diff --git a/.autoskillit/.gitignore b/.autoskillit/.gitignore index 3081171d70..3b77c8496f 100644 --- a/.autoskillit/.gitignore +++ b/.autoskillit/.gitignore @@ -2,3 +2,5 @@ temp/ .secrets.yaml .onboarded sync_manifest.json +test-filter-manifest.yaml +validation-errors/ diff --git a/.autoskillit/config.yaml b/.autoskillit/config.yaml index ada27986d9..f5ea571a71 100644 --- a/.autoskillit/config.yaml +++ b/.autoskillit/config.yaml @@ -1,5 +1,7 @@ test_check: command: ["task", "test-all"] + filter_mode: "conservative" + base_ref: "develop" # timeout: 600 safety: @@ -21,7 +23,7 @@ safety: # skill_names: ["/autoskillit:implement-worktree", "/autoskillit:implement-worktree-no-merge"] branching: - default_base_branch: "integration" + default_base_branch: "develop" ci: workflow: "tests.yml" @@ -29,6 +31,20 @@ ci: github: default_repo: "TalonT-Org/AutoSkillit" -quota_guard: - enabled: false - short_window_threshold: 95.0 +features: + providers: true + +providers: + profiles: + minimax: + ANTHROPIC_BASE_URL: "https://api.minimax.io/anthropic" + ANTHROPIC_MODEL: "MiniMax-M2.7-highspeed" + ANTHROPIC_TIMEOUT: "3000000" + ENABLE_TOOL_SEARCH: "false" + CLAUDE_CODE_DISABLE_NONESSENTIAL_TRAFFIC: "1" + step_overrides: + retry_worktree: minimax + compose_pr: minimax + diagnose_ci: minimax + prepare_pr: minimax + diff --git a/.autoskillit/recipes/full-audit.yaml b/.autoskillit/recipes/full-audit.yaml deleted file mode 100644 index 1200d95c90..0000000000 --- a/.autoskillit/recipes/full-audit.yaml +++ /dev/null @@ -1,149 +0,0 @@ -name: full-audit -description: >- - Run audit-tests, audit-cohesion, and audit-arch in parallel, validate each - report individually, and create a GitHub issue from each validated report. -summary: "branch → audit×3 ∥ → validate×3 ∥ → issue×3 ∥ → done" - -ingredients: - workspace: - description: "Path to the git repository workspace" - required: true - branch: - description: "The branch to audit against" - required: true - -kitchen_rules: - - >- - Each audit chain (tests, cohesion, arch) is independent. A failure in one - chain MUST NOT block or affect the other two chains. Continue with - successful chains and note failures for escalation. - - >- - Within each wave, fire all run_skill calls in a single message to achieve - parallelism. Do not serialize audit/validate/issue calls. - - >- - Report paths are discovered from session output — look for file paths - in .autoskillit/temp/ directories. Pass discovered paths as positional - arguments to downstream skills. - -steps: - checkout: - tool: run_cmd - description: "Check out the target branch" - with: - cmd: "git checkout ${{ inputs.branch }}" - cwd: "${{ inputs.workspace }}" - step_name: "checkout" - note: > - Ensure the working directory is on the target branch before audits run. - If the checkout fails (e.g., dirty working tree, unknown branch), route - to escalate — audits cannot proceed on the wrong branch. - on_success: run_audits - on_failure: escalate_stop - retries: 0 - on_exhausted: escalate_stop - - run_audits: - tool: run_skill - description: "Run three audit skills in parallel" - with: - skill_command: "/autoskillit:audit-tests" - cwd: "${{ inputs.workspace }}" - step_name: "run_audits" - note: > - PARALLEL AUDIT WAVE — Fire three run_skill calls in a single message: - - 1. run_skill(skill_command="/autoskillit:audit-tests", cwd="${{ inputs.workspace }}", step_name="audit_tests") - 2. run_skill(skill_command="/autoskillit:audit-cohesion", cwd="${{ inputs.workspace }}", step_name="audit_cohesion") - 3. run_skill(skill_command="/autoskillit:audit-arch", cwd="${{ inputs.workspace }}", step_name="audit_arch") - - Each audit writes its report to a well-known temp directory: - - .autoskillit/temp/audit-tests/{name}_{timestamp}.md - - .autoskillit/temp/audit-cohesion/cohesion_audit_{timestamp}.md - - .autoskillit/temp/audit-arch/arch_audit_{timestamp}.md - - After all three complete, discover the most-recently-modified .md file in - each temp directory from the session output. Record which audits succeeded - and their report paths for the validation wave. - - If any audit fails, note the failure but continue with the other two. - Proceed to validate_audits even if only one or two audits succeeded. - Route to escalate_stop ONLY if all three audits fail. - on_success: validate_audits - on_failure: escalate_stop - retries: 1 - on_exhausted: escalate_stop - - validate_audits: - tool: run_skill - description: "Validate each audit report in parallel" - with: - skill_command: "/autoskillit:validate-audit" - cwd: "${{ inputs.workspace }}" - step_name: "validate_audits" - note: > - PARALLEL VALIDATION WAVE — For each audit that succeeded in the previous - wave, fire a run_skill call in a single message. Pass the audit report - path as a positional argument: - - 1. run_skill(skill_command="/autoskillit:validate-audit {tests_report_path}", cwd="${{ inputs.workspace }}", step_name="validate_tests") - 2. run_skill(skill_command="/autoskillit:validate-audit {cohesion_report_path}", cwd="${{ inputs.workspace }}", step_name="validate_cohesion") - 3. run_skill(skill_command="/autoskillit:validate-audit {arch_report_path}", cwd="${{ inputs.workspace }}", step_name="validate_arch") - - Skip validation for any audit that failed in wave 1. - Each validation writes to .autoskillit/temp/validate-audit/: - - validated_report_{source}_{timestamp}.md (first line: "validated: true") - - contested_findings_{source}_{timestamp}.md (only if contested > 0) - - After all complete, discover the validated_report_*.md files from session - output. Record paths for the issue creation wave. - - If any validation fails, note it but continue with the others. - Route to escalate_stop ONLY if all validations fail. - on_success: create_issues - on_failure: escalate_stop - retries: 1 - on_exhausted: escalate_stop - - create_issues: - tool: run_skill - description: "Create GitHub issues from validated reports" - with: - skill_command: "/autoskillit:prepare-issue" - cwd: "${{ inputs.workspace }}" - step_name: "create_issues" - note: > - PARALLEL ISSUE CREATION WAVE — For each validated audit report, fire a - run_skill call in a single message. Pass the validated report path as - the description argument: - - 1. run_skill(skill_command="/autoskillit:prepare-issue {validated_tests_path}", cwd="${{ inputs.workspace }}", step_name="issue_tests") - 2. run_skill(skill_command="/autoskillit:prepare-issue {validated_cohesion_path}", cwd="${{ inputs.workspace }}", step_name="issue_cohesion") - 3. run_skill(skill_command="/autoskillit:prepare-issue {validated_arch_path}", cwd="${{ inputs.workspace }}", step_name="issue_arch") - - Skip issue creation for any validation that failed in wave 2. - The prepare-issue skill auto-detects the "validated: true" sentinel in - the report's first line and uses the audit-report input path (derives - title from H1 heading, strips artifact paths from body). - - Collect the issue URLs from each successful result. - If any issue creation fails, note it but continue with the others. - capture_list: - issue_urls: "${{ result.issue_url }}" - on_success: done - on_failure: escalate_stop - retries: 1 - on_exhausted: escalate_stop - - done: - action: stop - message: >- - Full audit complete. GitHub issues created from validated audit reports. - Check context.issue_urls for the created issue links and - .autoskillit/temp/ for all audit artifacts. - - escalate_stop: - action: stop - message: >- - Full audit pipeline failed — one or more audit chains could not complete. - Check session logs and .autoskillit/temp/ directories for partial results. - Individual chain failures are noted in the session output above. diff --git a/.autoskillit/test-filter-manifest.yaml b/.autoskillit/test-filter-manifest.yaml new file mode 100644 index 0000000000..b1e2af3706 --- /dev/null +++ b/.autoskillit/test-filter-manifest.yaml @@ -0,0 +1,180 @@ +# Test Filter Manifest +# +# Maps non-Python file glob patterns (repo-relative) to test directories +# (relative to tests/). Used by _test_filter.py to select which tests +# to run when non-Python files change. +# +# Files already handled by Bucket A (pyproject.toml, uv.lock, conftest.py) +# are intentionally absent — Bucket A triggers a full run before the +# manifest is consulted. + +src/autoskillit/recipes/*.yaml: + - recipe/ + - contracts/ + +src/autoskillit/recipes/*.json: + - recipe/ + - contracts/ + +src/autoskillit/recipes/campaigns/*.yaml: + - recipe/ + - cli/ + +src/autoskillit/recipes/campaigns/*.json: + - recipe/ + - cli/ + +src/autoskillit/recipes/examples/*.yaml: + - recipe/ + +src/autoskillit/recipes/examples/*.json: + - recipe/ + +src/autoskillit/recipes/diagrams/*.md: + - recipe/ + +src/autoskillit/recipes/contracts/*.yaml: + - recipe/ + - contracts/ + +src/autoskillit/recipes/contracts/*.json: + - recipe/ + - contracts/ + +src/autoskillit/recipes/experiment-types/*.yaml: + - recipe/ + +src/autoskillit/recipes/experiment-types/*.json: + - recipe/ + +src/autoskillit/recipes/methodology-traditions/*.yaml: + - recipe/ + +src/autoskillit/recipes/methodology-traditions/*.json: + - recipe/ + +src/autoskillit/recipes/sub-recipes/*.yaml: + - recipe/ + +src/autoskillit/recipes/sub-recipes/*.json: + - recipe/ + +src/autoskillit/recipe/skill_contracts.yaml: + - contracts/ + - skills/ + - recipe/ + - execution/ + +src/autoskillit/recipe/block_budgets.yaml: + - recipe/ + +src/autoskillit/skills/*/SKILL.md: + - skills/ + - contracts/ + +src/autoskillit/skills_extended/*/SKILL.md: + - skills/ + - contracts/ + - recipe/ + - skills_extended/ + +src/autoskillit/config/defaults.yaml: + - config/ + - infra/ + - docs/ + - workspace/ + +src/autoskillit/hooks/registry.sha256: + - infra/ + +src/autoskillit/.claude-plugin/plugin.json: + - contracts/ + - server/ + - cli/ + - test_version.py + +src/autoskillit/migrations/*.yaml: + - migration/ + +.github/workflows/*.yml: + - infra/ + +CLAUDE.md: + - infra/ + - contracts/ + - docs/ + +Taskfile.yml: + - infra/ + +.gitattributes: + - infra/ + +.gitleaks.toml: + - infra/ + +docs/**/*.md: + - docs/ + +README.md: + - docs/ + +src/autoskillit/assets/**/*: + - assets/ + +tests/recipe/fixtures/*: + - recipe/ + +# Local .autoskillit/ config and state files +.autoskillit/test-source-map.json: + - infra/ + +.autoskillit/test-filter-manifest.yaml: + - infra/ + +.autoskillit/config.yaml: + - infra/ + +.autoskillit/recipes/*.yaml: + - recipe/ + +.autoskillit/recipes/contracts/*.yaml: + - recipe/ + - contracts/ + +.autoskillit/recipes/diagrams/*.md: + - recipe/ + +# External skills installed under .claude/skills/ +.claude/skills/*/SKILL.md: + - skills/ + - contracts/ + +# Root config / tooling files +.gitignore: + - infra/ + +.python-version: + - infra/ + +install.sh: + - infra/ + - cli/ + +scripts/verify-test-filter.sh: + - infra/ + +# Source non-Python files +src/autoskillit/.mcp.json: + - server/ + - cli/ + +scripts/recipe/*.sh: + - recipe/ + +src/autoskillit/core/__init__.pyi: + - core/ + - arch/ + - contracts/ + +tests/**/CLAUDE.md: [] diff --git a/.autoskillit/test-source-map.json b/.autoskillit/test-source-map.json new file mode 100644 index 0000000000..f313e9cfec --- /dev/null +++ b/.autoskillit/test-source-map.json @@ -0,0 +1,3865 @@ +{ + "src/autoskillit/_llm_triage.py": [ + "tests/server/test_mcp_overrides.py", + "tests/server/test_tools_kitchen_gate.py", + "tests/server/test_tools_kitchen_envelope.py", + "tests/server/test_tools_kitchen_visibility.py", + "tests/test_llm_triage.py" + ], + "src/autoskillit/_test_filter.py": [ + "tests/test_test_filter.py" + ], + "src/autoskillit/cli/_ansi.py": [ + "tests/cli/test_ansi.py", + "tests/cli/test_cli_prompts.py", + "tests/cli/test_cook.py", + "tests/cli/test_cook_env_scrub.py", + "tests/cli/test_cook_interactive.py", + "tests/cli/test_franchise_cli.py", + "tests/cli/test_init.py", + "tests/cli/test_input_tty_contracts.py", + "tests/cli/test_onboarding.py", + "tests/cli/test_reload_loop.py", + "tests/cli/test_terminal.py", + "tests/cli/test_update_checks.py", + "tests/cli/test_workspace.py", + "tests/contracts/test_claude_code_interface_contracts.py", + "tests/execution/test_commands.py" + ], + "src/autoskillit/cli/_cook.py": [ + "tests/cli/test_cook.py", + "tests/cli/test_cook_env_scrub.py", + "tests/cli/test_cook_interactive.py", + "tests/cli/test_input_tty_contracts.py", + "tests/cli/test_reload_loop.py", + "tests/cli/test_terminal.py", + "tests/contracts/test_claude_code_interface_contracts.py", + "tests/execution/test_commands.py" + ], + "src/autoskillit/cli/_doctor.py": [ + "tests/cli/test_doctor.py" + ], + "src/autoskillit/cli/_features.py": [ + "tests/cli/test_features_cli.py" + ], + "src/autoskillit/cli/_franchise.py": [ + "tests/cli/test_franchise_cli.py", + "tests/cli/test_reap.py", + "tests/cli/test_reload_loop.py", + "tests/cli/test_signal_guard.py", + "tests/franchise/test_franchise_e2e.py" + ], + "src/autoskillit/cli/_hooks.py": [ + "tests/cli/test_cli_hooks.py", + "tests/cli/test_doctor.py", + "tests/cli/test_init.py", + "tests/cli/test_install.py", + "tests/infra/test_hook_executability.py" + ], + "src/autoskillit/cli/_init_helpers.py": [ + "tests/cli/test_cli_hooks.py", + "tests/cli/test_cli_serve_logging.py", + "tests/cli/test_cook.py", + "tests/cli/test_cook_env_scrub.py", + "tests/cli/test_cook_interactive.py", + "tests/cli/test_doctor.py", + "tests/cli/test_franchise_cli.py", + "tests/cli/test_init.py", + "tests/cli/test_input_tty_contracts.py", + "tests/cli/test_install.py", + "tests/cli/test_onboarding.py", + "tests/cli/test_reload_loop.py", + "tests/cli/test_serve_sigterm.py", + "tests/cli/test_startup_budget.py", + "tests/cli/test_terminal.py", + "tests/cli/test_update_checks.py", + "tests/cli/test_workspace.py", + "tests/contracts/test_claude_code_interface_contracts.py", + "tests/execution/test_commands.py" + ], + "src/autoskillit/cli/_install_info.py": [ + "tests/cli/test_doctor.py", + "tests/cli/test_install_info.py", + "tests/cli/test_update_checks.py", + "tests/cli/test_update_command.py" + ], + "src/autoskillit/cli/_installed_plugins.py": [ + "tests/cli/test_cli_hooks.py", + "tests/cli/test_doctor.py", + "tests/cli/test_install.py", + "tests/cli/test_installed_plugins_file.py" + ], + "src/autoskillit/cli/_marketplace.py": [ + "tests/cli/test_cli_hooks.py", + "tests/cli/test_install.py", + "tests/contracts/test_version_consistency.py" + ], + "src/autoskillit/cli/_onboarding.py": [ + "tests/cli/test_cook_env_scrub.py", + "tests/cli/test_cook_interactive.py", + "tests/cli/test_onboarding.py", + "tests/contracts/test_claude_code_interface_contracts.py", + "tests/execution/test_commands.py" + ], + "src/autoskillit/cli/_prompts.py": [ + "tests/cli/test_cli_prompts.py", + "tests/cli/test_cook.py", + "tests/cli/test_franchise_cli.py", + "tests/cli/test_l3_orchestrator_prompt.py", + "tests/cli/test_orchestrator_prompt_contract.py", + "tests/contracts/test_exogenous_string_coupling.py", + "tests/contracts/test_instruction_surface.py" + ], + "src/autoskillit/cli/_reload.py": [ + "tests/cli/test_cook.py", + "tests/cli/test_cook_env_scrub.py", + "tests/cli/test_cook_ide_isolation.py", + "tests/cli/test_cook_interactive.py", + "tests/cli/test_franchise_cli.py", + "tests/cli/test_reload_loop.py", + "tests/cli/test_session_launch.py", + "tests/contracts/test_claude_code_interface_contracts.py", + "tests/execution/test_commands.py" + ], + "src/autoskillit/cli/_session_launch.py": [ + "tests/cli/test_cook.py", + "tests/cli/test_cook_env_scrub.py", + "tests/cli/test_cook_ide_isolation.py", + "tests/cli/test_franchise_cli.py", + "tests/cli/test_reload_loop.py", + "tests/cli/test_session_launch.py", + "tests/cli/test_terminal.py" + ], + "src/autoskillit/cli/_terminal.py": [ + "tests/cli/test_cook.py", + "tests/cli/test_cook_env_scrub.py", + "tests/cli/test_cook_ide_isolation.py", + "tests/cli/test_cook_interactive.py", + "tests/cli/test_franchise_cli.py", + "tests/cli/test_session_launch.py", + "tests/cli/test_terminal.py", + "tests/contracts/test_claude_code_interface_contracts.py", + "tests/execution/test_commands.py" + ], + "src/autoskillit/cli/_terminal_table.py": [ + "tests/core/test_core_terminal_table.py" + ], + "src/autoskillit/cli/_timed_input.py": [ + "tests/cli/test_cook.py", + "tests/cli/test_cook_env_scrub.py", + "tests/cli/test_cook_interactive.py", + "tests/cli/test_init.py", + "tests/cli/test_input_tty_contracts.py", + "tests/cli/test_onboarding.py", + "tests/cli/test_reload_loop.py", + "tests/cli/test_terminal.py", + "tests/cli/test_update_checks.py", + "tests/cli/test_workspace.py", + "tests/contracts/test_claude_code_interface_contracts.py", + "tests/execution/test_commands.py" + ], + "src/autoskillit/cli/_update.py": [ + "tests/cli/test_update_command.py" + ], + "src/autoskillit/cli/_update_checks.py": [ + "tests/cli/test_doctor.py", + "tests/cli/test_init.py", + "tests/cli/test_update_checks.py", + "tests/cli/test_update_command.py" + ], + "src/autoskillit/cli/_workspace.py": [ + "tests/cli/test_cook.py", + "tests/cli/test_input_tty_contracts.py", + "tests/cli/test_workspace.py" + ], + "src/autoskillit/cli/app.py": [ + "tests/cli/test_cli_serve_logging.py", + "tests/cli/test_cook.py", + "tests/cli/test_cook_interactive.py", + "tests/cli/test_doctor.py", + "tests/cli/test_init.py", + "tests/cli/test_install.py", + "tests/cli/test_serve_sigterm.py", + "tests/cli/test_startup_budget.py" + ], + "src/autoskillit/config/ingredient_defaults.py": [ + "tests/cli/test_cli_prompts.py", + "tests/cli/test_cook.py", + "tests/config/test_helpers.py", + "tests/server/test_mcp_overrides.py", + "tests/server/test_tools_kitchen_gate.py", + "tests/server/test_tools_kitchen_envelope.py", + "tests/server/test_tools_kitchen_visibility.py", + "tests/server/test_tools_load_recipe.py" + ], + "src/autoskillit/config/settings.py": [ + "tests/arch/test_feature_registry.py", + "tests/cli/test_cli_prompts.py", + "tests/cli/test_cli_serve_logging.py", + "tests/cli/test_cook.py", + "tests/cli/test_cook_env_scrub.py", + "tests/cli/test_cook_interactive.py", + "tests/cli/test_doctor.py", + "tests/cli/test_franchise_cli.py", + "tests/cli/test_init.py", + "tests/cli/test_input_tty_contracts.py", + "tests/cli/test_install.py", + "tests/cli/test_reload_loop.py", + "tests/cli/test_startup_budget.py", + "tests/cli/test_terminal.py", + "tests/cli/test_workspace.py", + "tests/config/test_config.py", + "tests/config/test_defaults.py", + "tests/config/test_helpers.py", + "tests/config/test_settings_allowed_labels.py", + "tests/config/test_settings_staged_label.py", + "tests/config/test_workspace_temp_dir_config.py", + "tests/contracts/test_claude_code_interface_contracts.py", + "tests/contracts/test_hook_bridge_coverage.py", + "tests/contracts/test_package_gateways.py", + "tests/contracts/test_target_skill_invocability.py", + "tests/execution/test_commands.py", + "tests/execution/test_headless.py", + "tests/execution/test_headless_add_dirs.py", + "tests/execution/test_headless_debug_logging.py", + "tests/execution/test_headless_env_injection.py", + "tests/execution/test_headless_env_scrub.py", + "tests/execution/test_idle_output_env.py", + "tests/execution/test_linux_tracing.py", + "tests/execution/test_output_format_contract.py", + "tests/execution/test_process_run.py", + "tests/execution/test_quota.py", + "tests/execution/test_quota_http.py", + "tests/execution/test_session_log_integration.py", + "tests/execution/test_testing.py", + "tests/franchise/test_franchise_e2e.py", + "tests/pipeline/test_context.py", + "tests/server/test_factory.py", + "tests/server/test_factory_recording.py", + "tests/server/test_git.py", + "tests/server/test_mcp_overrides.py", + "tests/server/test_perform_merge_editable_guard.py", + "tests/server/test_quota_refresh_loop.py", + "tests/server/test_server_init_gate.py", + "tests/server/test_server_init_session_visibility.py", + "tests/server/test_server_tool_registration.py", + "tests/server/test_tools_execution_command.py", + "tests/server/test_tools_execution_input_gates.py", + "tests/server/test_tools_execution_results.py", + "tests/server/test_tools_execution_routing.py", + "tests/server/test_tools_execution_response.py", + "tests/server/test_tools_github.py", + "tests/server/test_tools_integrations.py", + "tests/server/test_tools_integrations_release.py", + "tests/server/test_tools_issue_lifecycle.py", + "tests/server/test_tools_kitchen_gate.py", + "tests/server/test_tools_kitchen_envelope.py", + "tests/server/test_tools_kitchen_visibility.py", + "tests/server/test_tools_label_validation.py", + "tests/server/test_tools_load_recipe.py", + "tests/server/test_tools_run_skill_retry.py", + "tests/server/test_tools_session_diagnostics.py", + "tests/server/test_tools_status_kitchen.py", + "tests/server/test_tools_status_summaries.py", + "tests/server/test_tools_status_quota_and_db.py", + "tests/server/test_tools_workspace.py", + "tests/test_conftest.py", + "tests/workspace/test_project_local_overrides.py", + "tests/workspace/test_session_skills.py", + "tests/workspace/test_skills.py" + ], + "src/autoskillit/core/_claude_env.py": [ + "tests/cli/test_cook.py", + "tests/cli/test_cook_env_scrub.py", + "tests/cli/test_cook_ide_isolation.py", + "tests/cli/test_cook_interactive.py", + "tests/cli/test_franchise_cli.py", + "tests/cli/test_reload_loop.py", + "tests/cli/test_session_launch.py", + "tests/cli/test_terminal.py", + "tests/contracts/test_claude_code_interface_contracts.py", + "tests/core/test_claude_env.py", + "tests/execution/test_commands.py", + "tests/execution/test_flag_contracts.py", + "tests/execution/test_headless.py", + "tests/execution/test_headless_add_dirs.py", + "tests/execution/test_headless_env_injection.py", + "tests/execution/test_headless_env_scrub.py", + "tests/execution/test_idle_output_env.py", + "tests/execution/test_output_format_contract.py", + "tests/execution/test_recording.py", + "tests/franchise/test_franchise_e2e.py", + "tests/server/test_factory_recording.py", + "tests/server/test_server_tool_registration.py", + "tests/server/test_tools_execution_command.py", + "tests/server/test_tools_execution_input_gates.py", + "tests/server/test_tools_execution_results.py", + "tests/server/test_tools_execution_routing.py", + "tests/server/test_tools_execution_response.py", + "tests/server/test_tools_run_skill_retry.py", + "tests/server/test_tools_status_kitchen.py", + "tests/server/test_tools_status_summaries.py", + "tests/server/test_tools_status_quota_and_db.py", + "tests/test_llm_triage.py" + ], + "src/autoskillit/core/_linux_proc.py": [ + "tests/execution/test_linux_tracing.py", + "tests/execution/test_linux_tracing_pty_integration.py", + "tests/execution/test_on_spawn_timing.py", + "tests/execution/test_process_run.py", + "tests/execution/test_session_log.py", + "tests/execution/test_session_log_integration.py", + "tests/execution/test_trace_target_resolver.py", + "tests/franchise/test_franchise_e2e.py", + "tests/server/test_headless_session.py", + "tests/server/test_lifespan.py", + "tests/server/test_server_init_gate.py", + "tests/server/test_server_init_session_visibility.py", + "tests/server/test_server_tool_registration.py", + "tests/server/test_tools_dispatch.py", + "tests/server/test_tools_recipe.py" + ], + "src/autoskillit/core/_plugin_cache.py": [ + "tests/arch/test_feature_markers.py", + "tests/cli/test_cli_hooks.py", + "tests/cli/test_install.py", + "tests/cli/test_plugin_cache.py", + "tests/cli/test_update_checks.py", + "tests/cli/test_update_command.py", + "tests/contracts/test_instruction_surface.py", + "tests/franchise/test_pack_enforcement.py", + "tests/server/test_headless_session.py", + "tests/server/test_lifespan.py", + "tests/server/test_server_init_gate.py", + "tests/server/test_server_init_session_visibility.py", + "tests/server/test_server_tool_registration.py", + "tests/server/test_tool_annotation_completeness.py", + "tests/server/test_tools_kitchen_gate.py", + "tests/server/test_tools_kitchen_envelope.py", + "tests/server/test_tools_kitchen_visibility.py", + "tests/server/test_tools_recipe.py" + ], + "src/autoskillit/core/_plugin_ids.py": [ + "tests/cli/test_cook.py", + "tests/cli/test_franchise_cli.py", + "tests/cli/test_mcp_names.py", + "tests/contracts/test_package_gateways.py", + "tests/server/test_factory.py" + ], + "src/autoskillit/core/_terminal_table.py": [ + "tests/cli/test_cook.py", + "tests/cli/test_features_cli.py", + "tests/cli/test_franchise_cli.py", + "tests/contracts/test_package_gateways.py", + "tests/core/test_core_terminal_table.py", + "tests/pipeline/test_telemetry_formatter.py", + "tests/recipe/test_api.py", + "tests/recipe/test_hidden_ingredients.py", + "tests/recipe/test_implementation_sprint_mode.py", + "tests/recipe/test_sub_recipe_loading.py", + "tests/recipe/test_sub_recipe_validation.py", + "tests/server/test_service_wrappers.py", + "tests/server/test_tools_kitchen_gate.py", + "tests/server/test_tools_kitchen_envelope.py", + "tests/server/test_tools_kitchen_visibility.py", + "tests/server/test_tools_load_recipe.py" + ], + "src/autoskillit/core/_type_enums.py": [ + "tests/arch/test_feature_markers.py", + "tests/core/test_session_type.py", + "tests/execution/test_clone_guard.py", + "tests/execution/test_headless.py", + "tests/execution/test_headless_add_dirs.py", + "tests/execution/test_headless_debug_logging.py", + "tests/execution/test_headless_env_injection.py", + "tests/execution/test_headless_env_scrub.py", + "tests/execution/test_idle_output_env.py", + "tests/execution/test_normalize_subtype.py", + "tests/execution/test_output_format_contract.py", + "tests/execution/test_process_channel_b.py", + "tests/execution/test_process_pty.py", + "tests/execution/test_session.py", + "tests/execution/test_session_adjudication.py", + "tests/execution/test_session_debug_logging.py", + "tests/execution/test_zero_write_detection.py", + "tests/franchise/test_franchise_e2e.py", + "tests/franchise/test_pack_enforcement.py", + "tests/infra/test_pretty_output.py", + "tests/server/test_factory_recording.py", + "tests/server/test_helpers_tier_guards.py", + "tests/server/test_server_init_gate.py", + "tests/server/test_server_init_session_visibility.py", + "tests/server/test_server_tool_registration.py", + "tests/server/test_server_version_telemetry.py", + "tests/server/test_tools_execution_command.py", + "tests/server/test_tools_execution_input_gates.py", + "tests/server/test_tools_execution_results.py", + "tests/server/test_tools_execution_routing.py", + "tests/server/test_tools_execution_response.py", + "tests/server/test_tools_kitchen_gate.py", + "tests/server/test_tools_kitchen_envelope.py", + "tests/server/test_tools_kitchen_visibility.py", + "tests/server/test_tools_run_cmd_unit.py", + "tests/server/test_tools_run_python.py", + "tests/server/test_tools_run_skill_retry.py", + "tests/server/test_tools_status_kitchen.py", + "tests/server/test_tools_status_summaries.py", + "tests/server/test_tools_status_quota_and_db.py", + "tests/test_llm_triage.py" + ], + "src/autoskillit/core/_type_helpers.py": [ + "tests/contracts/test_protocol_satisfaction.py", + "tests/contracts/test_target_skill_invocability.py", + "tests/core/test_skill_command_parsing.py", + "tests/execution/test_clone_guard.py", + "tests/execution/test_headless.py", + "tests/execution/test_headless_add_dirs.py", + "tests/execution/test_headless_debug_logging.py", + "tests/execution/test_headless_env_injection.py", + "tests/execution/test_headless_env_scrub.py", + "tests/execution/test_idle_output_env.py", + "tests/execution/test_output_format_contract.py", + "tests/execution/test_process_channel_b.py", + "tests/execution/test_process_pty.py", + "tests/execution/test_zero_write_detection.py", + "tests/franchise/test_dispatch_failure_semantics.py", + "tests/franchise/test_error_envelope.py", + "tests/franchise/test_franchise_e2e.py", + "tests/infra/test_pretty_output.py", + "tests/server/test_factory_recording.py", + "tests/server/test_git.py", + "tests/server/test_run_skill_add_dirs.py", + "tests/server/test_server_init_gate.py", + "tests/server/test_server_init_session_visibility.py", + "tests/server/test_server_tool_registration.py", + "tests/server/test_tools_dispatch.py", + "tests/server/test_tools_execution_command.py", + "tests/server/test_tools_execution_input_gates.py", + "tests/server/test_tools_execution_results.py", + "tests/server/test_tools_execution_routing.py", + "tests/server/test_tools_execution_response.py", + "tests/server/test_tools_git.py", + "tests/server/test_tools_run_cmd.py", + "tests/server/test_tools_run_cmd_unit.py", + "tests/server/test_tools_run_skill_retry.py", + "tests/server/test_tools_status_kitchen.py", + "tests/server/test_tools_status_summaries.py", + "tests/server/test_tools_status_quota_and_db.py", + "tests/server/test_tools_workspace.py" + ], + "src/autoskillit/core/_type_results.py": [ + "tests/cli/test_workspace.py", + "tests/contracts/test_claude_code_interface_contracts.py", + "tests/contracts/test_l1_packages.py", + "tests/core/test_add_dir_validation.py", + "tests/core/test_types.py", + "tests/execution/test_headless.py", + "tests/execution/test_headless_add_dirs.py", + "tests/execution/test_headless_env_injection.py", + "tests/execution/test_headless_env_scrub.py", + "tests/execution/test_idle_output_env.py", + "tests/execution/test_session.py", + "tests/execution/test_zero_write_detection.py", + "tests/infra/test_pretty_output.py", + "tests/pipeline/test_audit.py", + "tests/server/test_factory_recording.py", + "tests/server/test_git.py", + "tests/server/test_perform_merge_editable_guard.py", + "tests/server/test_run_skill_add_dirs.py", + "tests/server/test_server_tool_registration.py", + "tests/server/test_tools_execution_command.py", + "tests/server/test_tools_execution_input_gates.py", + "tests/server/test_tools_execution_results.py", + "tests/server/test_tools_execution_routing.py", + "tests/server/test_tools_execution_response.py", + "tests/server/test_tools_git.py", + "tests/server/test_tools_run_skill_retry.py", + "tests/server/test_tools_status_kitchen.py", + "tests/server/test_tools_status_summaries.py", + "tests/server/test_tools_status_quota_and_db.py", + "tests/server/test_tools_workspace.py", + "tests/workspace/test_cleanup.py", + "tests/workspace/test_project_local_overrides.py", + "tests/workspace/test_session_skills.py" + ], + "src/autoskillit/core/_type_resume.py": [ + "tests/cli/test_cook.py", + "tests/cli/test_cook_env_scrub.py", + "tests/cli/test_cook_interactive.py", + "tests/cli/test_reload_loop.py", + "tests/cli/test_terminal.py", + "tests/contracts/test_claude_code_interface_contracts.py", + "tests/execution/test_commands.py" + ], + "src/autoskillit/core/_version_snapshot.py": [ + "tests/core/test_version_snapshot.py", + "tests/execution/test_headless.py", + "tests/server/test_factory_recording.py", + "tests/server/test_tools_run_skill_retry.py" + ], + "src/autoskillit/core/branch_guard.py": [ + "tests/core/test_branch_guard.py", + "tests/server/test_git.py", + "tests/server/test_perform_merge_editable_guard.py", + "tests/server/test_server_tool_registration.py", + "tests/server/test_tools_git.py", + "tests/workspace/test_clone.py" + ], + "src/autoskillit/core/claude_conventions.py": [ + "tests/core/test_add_dir_validation.py", + "tests/server/test_run_skill_add_dirs.py", + "tests/server/test_server_tool_registration.py", + "tests/server/test_tools_execution_command.py", + "tests/server/test_tools_execution_input_gates.py", + "tests/server/test_tools_execution_results.py", + "tests/server/test_tools_execution_routing.py", + "tests/server/test_tools_execution_response.py", + "tests/server/test_tools_run_skill_retry.py", + "tests/server/test_tools_status_kitchen.py", + "tests/server/test_tools_status_summaries.py", + "tests/server/test_tools_status_quota_and_db.py" + ], + "src/autoskillit/core/feature_flags.py": [ + "tests/arch/test_feature_registry.py", + "tests/cli/test_cli_prompts.py", + "tests/cli/test_cli_serve_logging.py", + "tests/cli/test_cook.py", + "tests/cli/test_cook_env_scrub.py", + "tests/cli/test_cook_interactive.py", + "tests/cli/test_doctor.py", + "tests/cli/test_features_cli.py", + "tests/cli/test_franchise_cli.py", + "tests/cli/test_init.py", + "tests/cli/test_input_tty_contracts.py", + "tests/cli/test_install.py", + "tests/cli/test_reload_loop.py", + "tests/cli/test_startup_budget.py", + "tests/cli/test_terminal.py", + "tests/cli/test_workspace.py", + "tests/config/test_config.py", + "tests/config/test_helpers.py", + "tests/config/test_settings_allowed_labels.py", + "tests/config/test_settings_staged_label.py", + "tests/config/test_workspace_temp_dir_config.py", + "tests/contracts/test_claude_code_interface_contracts.py", + "tests/contracts/test_instruction_surface.py", + "tests/contracts/test_target_skill_invocability.py", + "tests/core/test_type_constants.py", + "tests/execution/test_commands.py", + "tests/execution/test_testing.py", + "tests/server/test_mcp_overrides.py", + "tests/server/test_run_skill_add_dirs.py", + "tests/server/test_server_init_gate.py", + "tests/server/test_server_init_session_visibility.py", + "tests/server/test_server_tool_registration.py", + "tests/server/test_tools_execution_command.py", + "tests/server/test_tools_execution_input_gates.py", + "tests/server/test_tools_execution_results.py", + "tests/server/test_tools_execution_routing.py", + "tests/server/test_tools_execution_response.py", + "tests/server/test_tools_kitchen_gate.py", + "tests/server/test_tools_kitchen_envelope.py", + "tests/server/test_tools_kitchen_visibility.py", + "tests/server/test_tools_load_recipe.py", + "tests/server/test_tools_run_skill_retry.py", + "tests/server/test_tools_status_kitchen.py", + "tests/server/test_tools_status_summaries.py", + "tests/server/test_tools_status_quota_and_db.py", + "tests/test_conftest.py", + "tests/workspace/test_project_local_overrides.py", + "tests/workspace/test_session_skills.py", + "tests/workspace/test_skill_content_substitution.py", + "tests/workspace/test_skills.py" + ], + "src/autoskillit/core/github_url.py": [ + "tests/core/test_github_url.py", + "tests/execution/test_ci.py", + "tests/execution/test_ci_params.py", + "tests/execution/test_github.py", + "tests/execution/test_remote_resolver.py", + "tests/server/test_helpers.py", + "tests/server/test_tools_ci.py", + "tests/server/test_tools_integrations.py", + "tests/server/test_tools_integrations_release.py", + "tests/server/test_tools_issue_lifecycle.py", + "tests/server/test_tools_label_validation.py", + "tests/workspace/test_clone_ci_contract.py" + ], + "src/autoskillit/core/io.py": [ + "tests/arch/test_feature_markers.py", + "tests/cli/test_cli_hooks.py", + "tests/cli/test_cli_prompts.py", + "tests/cli/test_cli_serve_logging.py", + "tests/cli/test_cook.py", + "tests/cli/test_cook_env_scrub.py", + "tests/cli/test_cook_interactive.py", + "tests/cli/test_doctor.py", + "tests/cli/test_franchise_cli.py", + "tests/cli/test_init.py", + "tests/cli/test_input_tty_contracts.py", + "tests/cli/test_install.py", + "tests/cli/test_installed_plugins_file.py", + "tests/cli/test_onboarding.py", + "tests/cli/test_plugin_cache.py", + "tests/cli/test_reap.py", + "tests/cli/test_reload_loop.py", + "tests/cli/test_signal_guard.py", + "tests/cli/test_startup_budget.py", + "tests/cli/test_terminal.py", + "tests/cli/test_update_checks.py", + "tests/cli/test_update_command.py", + "tests/cli/test_workspace.py", + "tests/config/test_config.py", + "tests/config/test_helpers.py", + "tests/config/test_settings_allowed_labels.py", + "tests/config/test_settings_staged_label.py", + "tests/config/test_workspace_temp_dir_config.py", + "tests/contracts/test_activate_deps_completeness.py", + "tests/contracts/test_claude_code_interface_contracts.py", + "tests/contracts/test_instruction_surface.py", + "tests/contracts/test_make_campaign_skill_contracts.py", + "tests/contracts/test_package_gateways.py", + "tests/contracts/test_skill_contracts.py", + "tests/contracts/test_target_skill_invocability.py", + "tests/contracts/test_version_consistency.py", + "tests/core/test_ensure_project_temp_with_config.py", + "tests/core/test_io.py", + "tests/core/test_resolve_temp_dir.py", + "tests/execution/test_commands.py", + "tests/execution/test_headless.py", + "tests/execution/test_headless_add_dirs.py", + "tests/execution/test_headless_env_injection.py", + "tests/execution/test_headless_env_scrub.py", + "tests/execution/test_idle_output_env.py", + "tests/execution/test_linux_tracing_pty_integration.py", + "tests/execution/test_quota.py", + "tests/execution/test_quota_http.py", + "tests/execution/test_recording.py", + "tests/execution/test_session_log.py", + "tests/execution/test_session_log_integration.py", + "tests/execution/test_testing.py", + "tests/franchise/test_dispatch_failure_semantics.py", + "tests/franchise/test_dispatch_lifespan.py", + "tests/franchise/test_franchise_e2e.py", + "tests/franchise/test_pack_enforcement.py", + "tests/franchise/test_pack_enforcement_e2e.py", + "tests/franchise/test_state.py", + "tests/franchise/test_state_schema.py", + "tests/infra/test_filter_activation.py", + "tests/infra/test_hook_executability.py", + "tests/infra/test_hook_settings.py", + "tests/infra/test_pretty_output_integration.py", + "tests/infra/test_quota_check.py", + "tests/migration/test_api.py", + "tests/migration/test_engine.py", + "tests/migration/test_loader.py", + "tests/migration/test_store.py", + "tests/planner/test_compiler.py", + "tests/planner/test_manifests.py", + "tests/planner/test_validation.py", + "tests/recipe/test_anti_pattern_guards.py", + "tests/recipe/test_api.py", + "tests/recipe/test_bundled_recipe_hidden_policy.py", + "tests/recipe/test_bundled_recipes.py", + "tests/recipe/test_bundled_recipes_no_inversions.py", + "tests/recipe/test_campaign_loader.py", + "tests/recipe/test_check_repo_merge_state_routing.py", + "tests/recipe/test_contracts.py", + "tests/recipe/test_contracts_block_fingerprint.py", + "tests/recipe/test_diagrams.py", + "tests/recipe/test_experiment_type_registry.py", + "tests/recipe/test_full_audit_recipe.py", + "tests/recipe/test_identity.py", + "tests/recipe/test_implementation_sprint_mode.py", + "tests/recipe/test_io.py", + "tests/recipe/test_issue_url_pipeline.py", + "tests/recipe/test_loader.py", + "tests/recipe/test_make_campaign_output_schema.py", + "tests/recipe/test_merge_prs_queue.py", + "tests/recipe/test_recipe_temp_substitution.py", + "tests/recipe/test_remediation_sprint_mode.py", + "tests/recipe/test_research_output_mode.py", + "tests/recipe/test_resolve_ci_routing_invariant.py", + "tests/recipe/test_review_loop_routing_integration.py", + "tests/recipe/test_rules_blocks.py", + "tests/recipe/test_rules_bypass.py", + "tests/recipe/test_rules_campaign.py", + "tests/recipe/test_rules_ci.py", + "tests/recipe/test_rules_clone.py", + "tests/recipe/test_rules_cmd.py", + "tests/recipe/test_rules_conditional_push.py", + "tests/recipe/test_rules_contracts.py", + "tests/recipe/test_rules_dataflow.py", + "tests/recipe/test_rules_features.py", + "tests/recipe/test_rules_graph.py", + "tests/recipe/test_rules_inputs.py", + "tests/recipe/test_rules_isolation.py", + "tests/recipe/test_rules_merge.py", + "tests/recipe/test_rules_packs.py", + "tests/recipe/test_rules_pipeline_internal.py", + "tests/recipe/test_rules_project_local_override.py", + "tests/recipe/test_rules_reachability.py", + "tests/recipe/test_rules_recipe.py", + "tests/recipe/test_rules_skill_content.py", + "tests/recipe/test_rules_skills.py", + "tests/recipe/test_rules_structure.py", + "tests/recipe/test_rules_subset_disabled.py", + "tests/recipe/test_rules_tools.py", + "tests/recipe/test_rules_verdict.py", + "tests/recipe/test_rules_worktree.py", + "tests/recipe/test_schema.py", + "tests/recipe/test_sprint_sub_recipe.py", + "tests/recipe/test_staleness_cache.py", + "tests/recipe/test_sub_recipe_loading.py", + "tests/recipe/test_sub_recipe_validation.py", + "tests/recipe/test_validator.py", + "tests/server/test_factory.py", + "tests/server/test_factory_recording.py", + "tests/server/test_git.py", + "tests/server/test_headless_session.py", + "tests/server/test_lifespan.py", + "tests/server/test_mcp_overrides.py", + "tests/server/test_perform_merge_editable_guard.py", + "tests/server/test_reload_session.py", + "tests/server/test_run_skill_add_dirs.py", + "tests/server/test_server_init_gate.py", + "tests/server/test_server_init_session_visibility.py", + "tests/server/test_server_tool_registration.py", + "tests/server/test_server_version_telemetry.py", + "tests/server/test_service_wrappers.py", + "tests/server/test_smoke_pipeline.py", + "tests/server/test_tool_annotation_completeness.py", + "tests/server/test_tools_clone.py", + "tests/server/test_tools_dispatch.py", + "tests/server/test_tools_execution_command.py", + "tests/server/test_tools_execution_input_gates.py", + "tests/server/test_tools_execution_results.py", + "tests/server/test_tools_execution_routing.py", + "tests/server/test_tools_execution_response.py", + "tests/server/test_tools_git.py", + "tests/server/test_tools_github.py", + "tests/server/test_tools_integrations.py", + "tests/server/test_tools_kitchen_gate.py", + "tests/server/test_tools_kitchen_envelope.py", + "tests/server/test_tools_kitchen_visibility.py", + "tests/server/test_tools_list_recipes.py", + "tests/server/test_tools_load_recipe.py", + "tests/server/test_tools_pr_ops.py", + "tests/server/test_tools_recipe.py", + "tests/server/test_tools_run_skill_retry.py", + "tests/server/test_tools_session_diagnostics.py", + "tests/server/test_tools_status_kitchen.py", + "tests/server/test_tools_status_summaries.py", + "tests/server/test_tools_status_quota_and_db.py", + "tests/server/test_tools_status_mcp_response.py", + "tests/skills/test_phase2_skills.py", + "tests/skills/test_review_design_guards.py", + "tests/skills/test_skill_compliance.py", + "tests/skills/test_skill_output_compliance.py", + "tests/skills/test_skill_preambles.py", + "tests/skills/test_troubleshoot_experiment_contracts.py", + "tests/skills/test_validate_audit_contracts.py", + "tests/test_conftest.py", + "tests/test_test_filter.py", + "tests/workspace/test_clone_registry.py", + "tests/workspace/test_project_local_overrides.py", + "tests/workspace/test_session_skills.py", + "tests/workspace/test_skill_content_substitution.py", + "tests/workspace/test_skills.py", + "tests/workspace/test_worktree.py" + ], + "src/autoskillit/core/kitchen_state.py": [ + "tests/arch/test_feature_markers.py", + "tests/cli/test_franchise_cli.py", + "tests/core/test_kitchen_state.py", + "tests/execution/test_recording_sigterm.py", + "tests/franchise/test_pack_enforcement.py", + "tests/server/test_headless_session.py", + "tests/server/test_lifespan.py", + "tests/server/test_reload_session.py", + "tests/server/test_server_init_gate.py", + "tests/server/test_server_init_session_visibility.py", + "tests/server/test_server_tool_registration.py", + "tests/server/test_tool_annotation_completeness.py", + "tests/server/test_tools_recipe.py" + ], + "src/autoskillit/core/logging.py": [ + "tests/cli/test_cli_serve_logging.py", + "tests/cli/test_cook_env_scrub.py", + "tests/cli/test_cook_interactive.py", + "tests/cli/test_init.py", + "tests/cli/test_reload_loop.py", + "tests/cli/test_serve_sigterm.py", + "tests/cli/test_startup_budget.py", + "tests/cli/test_terminal.py", + "tests/contracts/test_claude_code_interface_contracts.py", + "tests/core/test_logging.py", + "tests/execution/test_commands.py", + "tests/server/test_git.py", + "tests/server/test_mcp_overrides.py", + "tests/server/test_server_version_telemetry.py", + "tests/server/test_tools_kitchen_gate.py", + "tests/server/test_tools_kitchen_envelope.py", + "tests/server/test_tools_kitchen_visibility.py" + ], + "src/autoskillit/core/paths.py": [ + "tests/arch/test_feature_markers.py", + "tests/arch/test_feature_registry.py", + "tests/arch/test_protocol_names.py", + "tests/cli/test_cli_hooks.py", + "tests/cli/test_cli_prompts.py", + "tests/cli/test_cli_serve_logging.py", + "tests/cli/test_cook.py", + "tests/cli/test_cook_env_scrub.py", + "tests/cli/test_cook_ide_isolation.py", + "tests/cli/test_cook_interactive.py", + "tests/cli/test_doctor.py", + "tests/cli/test_food_truck_prompt.py", + "tests/cli/test_franchise_cli.py", + "tests/cli/test_init.py", + "tests/cli/test_input_tty_contracts.py", + "tests/cli/test_install.py", + "tests/cli/test_l3_orchestrator_prompt.py", + "tests/cli/test_orchestrator_prompt_contract.py", + "tests/cli/test_reload_loop.py", + "tests/cli/test_serve_sigterm.py", + "tests/cli/test_session_launch.py", + "tests/cli/test_signal_guard.py", + "tests/cli/test_startup_budget.py", + "tests/cli/test_terminal.py", + "tests/cli/test_workspace.py", + "tests/config/test_config.py", + "tests/config/test_helpers.py", + "tests/config/test_settings_allowed_labels.py", + "tests/config/test_settings_staged_label.py", + "tests/config/test_workspace_temp_dir_config.py", + "tests/contracts/test_activate_deps_completeness.py", + "tests/contracts/test_claude_code_interface_contracts.py", + "tests/contracts/test_exogenous_string_coupling.py", + "tests/contracts/test_github_ops.py", + "tests/contracts/test_instruction_surface.py", + "tests/contracts/test_issue_content_fidelity.py", + "tests/contracts/test_make_campaign_skill_contracts.py", + "tests/contracts/test_package_gateways.py", + "tests/contracts/test_process_issues_contracts.py", + "tests/contracts/test_protocol_definitions.py", + "tests/contracts/test_scope_contracts.py", + "tests/contracts/test_skill_contracts.py", + "tests/contracts/test_skill_yaml_validation.py", + "tests/contracts/test_target_skill_invocability.py", + "tests/contracts/test_version_consistency.py", + "tests/core/test_add_dir_validation.py", + "tests/core/test_io.py", + "tests/core/test_paths.py", + "tests/core/test_types_structure.py", + "tests/execution/test_commands.py", + "tests/execution/test_headless.py", + "tests/execution/test_headless_add_dirs.py", + "tests/execution/test_headless_env_injection.py", + "tests/execution/test_headless_env_scrub.py", + "tests/execution/test_idle_output_env.py", + "tests/execution/test_linux_tracing_pty_integration.py", + "tests/execution/test_session_log.py", + "tests/execution/test_session_log_integration.py", + "tests/execution/test_testing.py", + "tests/franchise/test_dispatch_failure_semantics.py", + "tests/franchise/test_dispatch_lifespan.py", + "tests/franchise/test_franchise_e2e.py", + "tests/franchise/test_pack_enforcement.py", + "tests/franchise/test_pack_enforcement_e2e.py", + "tests/infra/test_generated_file_write_guard.py", + "tests/infra/test_hook_executability.py", + "tests/infra/test_hook_settings.py", + "tests/infra/test_mcp_health_advisor.py", + "tests/infra/test_open_kitchen_guard.py", + "tests/infra/test_pretty_output.py", + "tests/infra/test_pretty_output_integration.py", + "tests/infra/test_quota_check.py", + "tests/infra/test_token_summary_appender.py", + "tests/migration/test_engine.py", + "tests/pipeline/test_gate.py", + "tests/recipe/test_anti_pattern_guards.py", + "tests/recipe/test_api.py", + "tests/recipe/test_bundled_recipe_hidden_policy.py", + "tests/recipe/test_bundled_recipes.py", + "tests/recipe/test_bundled_recipes_no_inversions.py", + "tests/recipe/test_campaign_loader.py", + "tests/recipe/test_check_repo_merge_state_routing.py", + "tests/recipe/test_contracts.py", + "tests/recipe/test_contracts_block_fingerprint.py", + "tests/recipe/test_full_audit_recipe.py", + "tests/recipe/test_identity.py", + "tests/recipe/test_implementation_sprint_mode.py", + "tests/recipe/test_io.py", + "tests/recipe/test_issue_url_pipeline.py", + "tests/recipe/test_loader.py", + "tests/recipe/test_make_campaign_output_schema.py", + "tests/recipe/test_merge_prs_queue.py", + "tests/recipe/test_plan_visualization_step.py", + "tests/recipe/test_remediation_sprint_mode.py", + "tests/recipe/test_repository.py", + "tests/recipe/test_research_output_mode.py", + "tests/recipe/test_review_loop_routing_integration.py", + "tests/recipe/test_rules_blocks.py", + "tests/recipe/test_rules_bypass.py", + "tests/recipe/test_rules_campaign.py", + "tests/recipe/test_rules_ci.py", + "tests/recipe/test_rules_clone.py", + "tests/recipe/test_rules_cmd.py", + "tests/recipe/test_rules_conditional_push.py", + "tests/recipe/test_rules_contracts.py", + "tests/recipe/test_rules_dataflow.py", + "tests/recipe/test_rules_features.py", + "tests/recipe/test_rules_graph.py", + "tests/recipe/test_rules_inputs.py", + "tests/recipe/test_rules_isolation.py", + "tests/recipe/test_rules_merge.py", + "tests/recipe/test_rules_packs.py", + "tests/recipe/test_rules_pipeline_internal.py", + "tests/recipe/test_rules_project_local_override.py", + "tests/recipe/test_rules_reachability.py", + "tests/recipe/test_rules_recipe.py", + "tests/recipe/test_rules_skill_content.py", + "tests/recipe/test_rules_skills.py", + "tests/recipe/test_rules_structure.py", + "tests/recipe/test_rules_subset_disabled.py", + "tests/recipe/test_rules_tools.py", + "tests/recipe/test_rules_verdict.py", + "tests/recipe/test_rules_worktree.py", + "tests/recipe/test_skill_emit_consistency.py", + "tests/recipe/test_sprint_sub_recipe.py", + "tests/recipe/test_sub_recipe_loading.py", + "tests/recipe/test_sub_recipe_validation.py", + "tests/recipe/test_validator.py", + "tests/server/test_factory.py", + "tests/server/test_factory_recording.py", + "tests/server/test_headless_session.py", + "tests/server/test_lifespan.py", + "tests/server/test_mcp_overrides.py", + "tests/server/test_run_skill_add_dirs.py", + "tests/server/test_server_init_gate.py", + "tests/server/test_server_init_session_visibility.py", + "tests/server/test_server_tool_registration.py", + "tests/server/test_server_version_telemetry.py", + "tests/server/test_service_wrappers.py", + "tests/server/test_smoke_pipeline.py", + "tests/server/test_tool_annotation_completeness.py", + "tests/server/test_tools_dispatch.py", + "tests/server/test_tools_execution_command.py", + "tests/server/test_tools_execution_input_gates.py", + "tests/server/test_tools_execution_results.py", + "tests/server/test_tools_execution_routing.py", + "tests/server/test_tools_execution_response.py", + "tests/server/test_tools_kitchen_gate.py", + "tests/server/test_tools_kitchen_envelope.py", + "tests/server/test_tools_kitchen_visibility.py", + "tests/server/test_tools_list_recipes.py", + "tests/server/test_tools_load_recipe.py", + "tests/server/test_tools_recipe.py", + "tests/server/test_tools_run_skill_retry.py", + "tests/server/test_tools_status_kitchen.py", + "tests/server/test_tools_status_summaries.py", + "tests/server/test_tools_status_quota_and_db.py", + "tests/skills/test_conflict_resolution_guards.py", + "tests/skills/test_phase2_skills.py", + "tests/skills/test_skill_compliance.py", + "tests/skills/test_skill_output_compliance.py", + "tests/skills/test_skill_preambles.py", + "tests/skills/test_troubleshoot_experiment_contracts.py", + "tests/skills/test_validate_audit_contracts.py", + "tests/test_conftest.py", + "tests/workspace/test_project_local_overrides.py", + "tests/workspace/test_session_skills.py", + "tests/workspace/test_skill_content_substitution.py", + "tests/workspace/test_skills.py" + ], + "src/autoskillit/core/readiness.py": [ + "tests/arch/test_feature_markers.py", + "tests/execution/test_recording_sigterm.py", + "tests/franchise/test_pack_enforcement.py", + "tests/server/test_headless_session.py", + "tests/server/test_lifespan.py", + "tests/server/test_server_init_gate.py", + "tests/server/test_server_init_session_visibility.py", + "tests/server/test_server_tool_registration.py", + "tests/server/test_tool_annotation_completeness.py", + "tests/server/test_tools_recipe.py" + ], + "src/autoskillit/execution/_headless_scan.py": [ + "tests/execution/test_headless.py", + "tests/execution/test_headless_add_dirs.py", + "tests/execution/test_headless_env_injection.py", + "tests/execution/test_headless_env_scrub.py", + "tests/execution/test_idle_output_env.py", + "tests/franchise/test_franchise_e2e.py", + "tests/server/test_factory_recording.py", + "tests/server/test_server_tool_registration.py", + "tests/server/test_tools_execution_command.py", + "tests/server/test_tools_execution_input_gates.py", + "tests/server/test_tools_execution_results.py", + "tests/server/test_tools_execution_routing.py", + "tests/server/test_tools_execution_response.py", + "tests/server/test_tools_run_skill_retry.py", + "tests/server/test_tools_status_kitchen.py", + "tests/server/test_tools_status_summaries.py", + "tests/server/test_tools_status_quota_and_db.py" + ], + "src/autoskillit/execution/_process_io.py": [ + "tests/execution/test_linux_tracing_pty_integration.py", + "tests/execution/test_on_spawn_timing.py", + "tests/execution/test_process_channel_b.py", + "tests/execution/test_process_debug_logging.py", + "tests/execution/test_process_kill.py", + "tests/execution/test_process_monitor.py", + "tests/execution/test_process_pty.py", + "tests/execution/test_process_run.py", + "tests/execution/test_session_log_integration.py" + ], + "src/autoskillit/execution/_process_jsonl.py": [ + "tests/execution/test_headless.py", + "tests/execution/test_linux_tracing_pty_integration.py", + "tests/execution/test_output_format_contract.py", + "tests/execution/test_process_channel_b.py", + "tests/execution/test_process_debug_logging.py", + "tests/execution/test_process_jsonl.py", + "tests/execution/test_process_kill.py", + "tests/execution/test_process_monitor.py", + "tests/execution/test_process_pty.py", + "tests/execution/test_process_run.py", + "tests/execution/test_session.py", + "tests/execution/test_session_log_integration.py" + ], + "src/autoskillit/execution/_process_kill.py": [ + "tests/execution/test_process_channel_b.py", + "tests/execution/test_process_debug_logging.py", + "tests/execution/test_process_kill.py", + "tests/execution/test_process_monitor.py", + "tests/execution/test_process_pty.py", + "tests/execution/test_process_run.py", + "tests/franchise/test_franchise_e2e.py" + ], + "src/autoskillit/execution/_process_monitor.py": [ + "tests/execution/test_linux_tracing_pty_integration.py", + "tests/execution/test_on_spawn_timing.py", + "tests/execution/test_process_channel_b.py", + "tests/execution/test_process_debug_logging.py", + "tests/execution/test_process_kill.py", + "tests/execution/test_process_monitor.py", + "tests/execution/test_process_pty.py", + "tests/execution/test_process_run.py", + "tests/execution/test_session_log_integration.py" + ], + "src/autoskillit/execution/_process_pty.py": [ + "tests/execution/test_linux_tracing_pty_integration.py", + "tests/execution/test_process_pty.py", + "tests/execution/test_session_log_integration.py" + ], + "src/autoskillit/execution/_process_race.py": [ + "tests/execution/test_linux_tracing_pty_integration.py", + "tests/execution/test_on_spawn_timing.py", + "tests/execution/test_process_channel_b.py", + "tests/execution/test_process_debug_logging.py", + "tests/execution/test_process_idle_watchdog.py", + "tests/execution/test_process_kill.py", + "tests/execution/test_process_monitor.py", + "tests/execution/test_process_pty.py", + "tests/execution/test_process_race.py", + "tests/execution/test_process_run.py", + "tests/execution/test_session_log_integration.py" + ], + "src/autoskillit/execution/anomaly_detection.py": [ + "tests/execution/test_anomaly_detection.py", + "tests/execution/test_headless.py", + "tests/execution/test_linux_tracing_pty_integration.py", + "tests/execution/test_session_log.py", + "tests/execution/test_session_log_integration.py", + "tests/franchise/test_franchise_e2e.py", + "tests/server/test_server_version_telemetry.py", + "tests/server/test_tools_execution_command.py", + "tests/server/test_tools_execution_input_gates.py", + "tests/server/test_tools_execution_results.py", + "tests/server/test_tools_execution_routing.py" + ], + "src/autoskillit/execution/ci.py": [ + "tests/cli/test_cli_serve_logging.py", + "tests/cli/test_init.py", + "tests/cli/test_startup_budget.py", + "tests/contracts/test_package_gateways.py", + "tests/execution/test_ci.py", + "tests/execution/test_ci_params.py", + "tests/server/test_factory.py", + "tests/server/test_factory_recording.py" + ], + "src/autoskillit/execution/clone_guard.py": [ + "tests/execution/test_clone_guard.py", + "tests/execution/test_headless.py", + "tests/execution/test_headless_add_dirs.py", + "tests/execution/test_headless_env_injection.py", + "tests/execution/test_headless_env_scrub.py", + "tests/execution/test_idle_output_env.py", + "tests/server/test_factory_recording.py", + "tests/server/test_server_tool_registration.py", + "tests/server/test_tools_execution_command.py", + "tests/server/test_tools_execution_input_gates.py", + "tests/server/test_tools_execution_results.py", + "tests/server/test_tools_execution_routing.py", + "tests/server/test_tools_execution_response.py", + "tests/server/test_tools_run_skill_retry.py", + "tests/server/test_tools_status_kitchen.py", + "tests/server/test_tools_status_summaries.py", + "tests/server/test_tools_status_quota_and_db.py" + ], + "src/autoskillit/execution/commands.py": [ + "tests/cli/test_cook.py", + "tests/cli/test_cook_env_scrub.py", + "tests/cli/test_cook_ide_isolation.py", + "tests/cli/test_cook_interactive.py", + "tests/cli/test_franchise_cli.py", + "tests/cli/test_reload_loop.py", + "tests/cli/test_session_launch.py", + "tests/cli/test_terminal.py", + "tests/contracts/test_claude_code_interface_contracts.py", + "tests/execution/test_commands.py", + "tests/execution/test_flag_contracts.py", + "tests/execution/test_headless.py", + "tests/execution/test_headless_add_dirs.py", + "tests/execution/test_headless_env_injection.py", + "tests/execution/test_headless_env_scrub.py", + "tests/execution/test_idle_output_env.py", + "tests/execution/test_output_format_contract.py", + "tests/execution/test_recording.py", + "tests/franchise/test_franchise_e2e.py", + "tests/server/test_factory_recording.py", + "tests/server/test_server_tool_registration.py", + "tests/server/test_tools_execution_command.py", + "tests/server/test_tools_execution_input_gates.py", + "tests/server/test_tools_execution_results.py", + "tests/server/test_tools_execution_routing.py", + "tests/server/test_tools_execution_response.py", + "tests/server/test_tools_run_skill_retry.py", + "tests/server/test_tools_status_kitchen.py", + "tests/server/test_tools_status_summaries.py", + "tests/server/test_tools_status_quota_and_db.py" + ], + "src/autoskillit/execution/db.py": [ + "tests/execution/test_db.py", + "tests/server/test_tools_status_kitchen.py", + "tests/server/test_tools_status_summaries.py", + "tests/server/test_tools_status_quota_and_db.py" + ], + "src/autoskillit/execution/diff_annotator.py": [ + "tests/execution/test_diff_annotator.py" + ], + "src/autoskillit/execution/github.py": [ + "tests/cli/test_cli_serve_logging.py", + "tests/cli/test_init.py", + "tests/cli/test_startup_budget.py", + "tests/contracts/test_package_gateways.py", + "tests/execution/test_check_repo_merge_state.py", + "tests/execution/test_ci.py", + "tests/execution/test_ci_params.py", + "tests/execution/test_github.py", + "tests/execution/test_github_headers.py", + "tests/execution/test_merge_queue.py", + "tests/infra/test_pretty_output_integration.py", + "tests/server/test_factory.py", + "tests/server/test_factory_recording.py", + "tests/server/test_tools_status_kitchen.py", + "tests/server/test_tools_status_summaries.py", + "tests/server/test_tools_status_quota_and_db.py" + ], + "src/autoskillit/execution/headless.py": [ + "tests/cli/test_cli_serve_logging.py", + "tests/cli/test_init.py", + "tests/cli/test_startup_budget.py", + "tests/contracts/test_package_gateways.py", + "tests/contracts/test_protocol_satisfaction.py", + "tests/execution/test_clone_guard.py", + "tests/execution/test_headless.py", + "tests/execution/test_headless_add_dirs.py", + "tests/execution/test_headless_debug_logging.py", + "tests/execution/test_headless_env_injection.py", + "tests/execution/test_headless_env_scrub.py", + "tests/execution/test_idle_output_env.py", + "tests/execution/test_output_format_contract.py", + "tests/execution/test_process_channel_b.py", + "tests/execution/test_process_pty.py", + "tests/execution/test_recording.py", + "tests/execution/test_zero_write_detection.py", + "tests/franchise/test_franchise_e2e.py", + "tests/infra/test_pretty_output.py", + "tests/server/test_factory.py", + "tests/server/test_factory_recording.py", + "tests/server/test_server_tool_registration.py", + "tests/server/test_tools_execution_command.py", + "tests/server/test_tools_execution_input_gates.py", + "tests/server/test_tools_execution_results.py", + "tests/server/test_tools_execution_routing.py", + "tests/server/test_tools_execution_response.py", + "tests/server/test_tools_load_recipe.py", + "tests/server/test_tools_run_skill_retry.py", + "tests/server/test_tools_status_kitchen.py", + "tests/server/test_tools_status_summaries.py", + "tests/server/test_tools_status_quota_and_db.py" + ], + "src/autoskillit/execution/linux_tracing.py": [ + "tests/execution/test_linux_tracing.py", + "tests/execution/test_linux_tracing_pty_integration.py", + "tests/execution/test_on_spawn_timing.py", + "tests/execution/test_process_channel_b.py", + "tests/execution/test_process_debug_logging.py", + "tests/execution/test_process_kill.py", + "tests/execution/test_process_pty.py", + "tests/execution/test_process_race.py", + "tests/execution/test_process_run.py", + "tests/execution/test_session_log.py", + "tests/execution/test_session_log_integration.py", + "tests/execution/test_trace_target_resolver.py" + ], + "src/autoskillit/execution/merge_queue.py": [ + "tests/cli/test_cli_serve_logging.py", + "tests/cli/test_init.py", + "tests/cli/test_startup_budget.py", + "tests/contracts/test_package_gateways.py", + "tests/execution/test_check_repo_merge_state.py", + "tests/execution/test_github_headers.py", + "tests/execution/test_merge_queue.py", + "tests/server/test_factory.py", + "tests/server/test_factory_recording.py" + ], + "src/autoskillit/execution/pr_analysis.py": [ + "tests/execution/test_pr_analysis.py" + ], + "src/autoskillit/execution/process.py": [ + "tests/execution/test_linux_tracing_pty_integration.py", + "tests/execution/test_on_spawn_timing.py", + "tests/execution/test_process_channel_b.py", + "tests/execution/test_process_debug_logging.py", + "tests/execution/test_process_kill.py", + "tests/execution/test_process_monitor.py", + "tests/execution/test_process_pty.py", + "tests/execution/test_process_race.py", + "tests/execution/test_process_run.py", + "tests/execution/test_session_log_integration.py", + "tests/execution/test_termination_action.py", + "tests/execution/test_termination_executor.py" + ], + "src/autoskillit/execution/quota.py": [ + "tests/execution/test_quota.py", + "tests/execution/test_quota_http.py", + "tests/server/test_run_skill_add_dirs.py", + "tests/server/test_server_tool_registration.py", + "tests/server/test_tools_execution_command.py", + "tests/server/test_tools_execution_input_gates.py", + "tests/server/test_tools_execution_results.py", + "tests/server/test_tools_execution_routing.py", + "tests/server/test_tools_execution_response.py", + "tests/server/test_tools_run_skill_retry.py", + "tests/server/test_tools_status_kitchen.py", + "tests/server/test_tools_status_summaries.py", + "tests/server/test_tools_status_quota_and_db.py" + ], + "src/autoskillit/execution/recording.py": [ + "tests/execution/test_recording.py", + "tests/server/test_factory_recording.py", + "tests/server/test_state.py", + "tests/server/test_tools_run_cmd_unit.py" + ], + "src/autoskillit/execution/remote_resolver.py": [ + "tests/execution/test_ci.py", + "tests/execution/test_ci_params.py", + "tests/execution/test_remote_resolver.py", + "tests/server/test_helpers.py", + "tests/server/test_server_tool_registration.py", + "tests/server/test_tools_ci.py", + "tests/server/test_tools_git.py", + "tests/workspace/test_clone_ci_contract.py" + ], + "src/autoskillit/execution/session.py": [ + "tests/execution/test_clone_guard.py", + "tests/execution/test_headless.py", + "tests/execution/test_headless_add_dirs.py", + "tests/execution/test_headless_debug_logging.py", + "tests/execution/test_headless_env_injection.py", + "tests/execution/test_headless_env_scrub.py", + "tests/execution/test_idle_output_env.py", + "tests/execution/test_normalize_subtype.py", + "tests/execution/test_output_format_contract.py", + "tests/execution/test_process_channel_b.py", + "tests/execution/test_process_pty.py", + "tests/execution/test_session.py", + "tests/execution/test_session_adjudication.py", + "tests/execution/test_session_debug_logging.py", + "tests/execution/test_zero_write_detection.py", + "tests/franchise/test_franchise_e2e.py", + "tests/infra/test_pretty_output.py", + "tests/server/test_factory_recording.py", + "tests/server/test_server_tool_registration.py", + "tests/server/test_tools_execution_command.py", + "tests/server/test_tools_execution_input_gates.py", + "tests/server/test_tools_execution_results.py", + "tests/server/test_tools_execution_routing.py", + "tests/server/test_tools_execution_response.py", + "tests/server/test_tools_run_skill_retry.py", + "tests/server/test_tools_status_kitchen.py", + "tests/server/test_tools_status_summaries.py", + "tests/server/test_tools_status_quota_and_db.py", + "tests/test_llm_triage.py" + ], + "src/autoskillit/execution/session_log.py": [ + "tests/cli/test_franchise_cli.py", + "tests/execution/test_headless.py", + "tests/execution/test_headless_add_dirs.py", + "tests/execution/test_headless_env_injection.py", + "tests/execution/test_headless_env_scrub.py", + "tests/execution/test_idle_output_env.py", + "tests/execution/test_linux_tracing_pty_integration.py", + "tests/execution/test_session_log.py", + "tests/execution/test_session_log_integration.py", + "tests/franchise/test_franchise_e2e.py", + "tests/infra/test_quota_check.py", + "tests/server/test_factory_recording.py", + "tests/server/test_headless_session.py", + "tests/server/test_lifespan.py", + "tests/server/test_server_init_gate.py", + "tests/server/test_server_init_session_visibility.py", + "tests/server/test_server_tool_registration.py", + "tests/server/test_server_version_telemetry.py", + "tests/server/test_state.py", + "tests/server/test_tools_execution_command.py", + "tests/server/test_tools_execution_input_gates.py", + "tests/server/test_tools_execution_results.py", + "tests/server/test_tools_execution_routing.py", + "tests/server/test_tools_execution_response.py", + "tests/server/test_tools_github.py", + "tests/server/test_tools_kitchen_gate.py", + "tests/server/test_tools_kitchen_envelope.py", + "tests/server/test_tools_kitchen_visibility.py", + "tests/server/test_tools_recipe.py", + "tests/server/test_tools_run_skill_retry.py", + "tests/server/test_tools_session_diagnostics.py", + "tests/server/test_tools_status_kitchen.py", + "tests/server/test_tools_status_summaries.py", + "tests/server/test_tools_status_quota_and_db.py", + "tests/server/test_tools_status_mcp_response.py" + ], + "src/autoskillit/execution/testing.py": [ + "tests/cli/test_cli_serve_logging.py", + "tests/cli/test_init.py", + "tests/cli/test_startup_budget.py", + "tests/contracts/test_package_gateways.py", + "tests/contracts/test_protocol_satisfaction.py", + "tests/execution/test_testing.py", + "tests/server/test_factory.py", + "tests/server/test_factory_recording.py", + "tests/server/test_server_tool_registration.py", + "tests/server/test_tools_git.py", + "tests/server/test_tools_workspace.py" + ], + "src/autoskillit/franchise/_api.py": [ + "tests/franchise/test_dispatch_failure_semantics.py", + "tests/franchise/test_dispatch_lifespan.py", + "tests/franchise/test_franchise_e2e.py", + "tests/server/test_tools_dispatch.py" + ], + "src/autoskillit/franchise/_prompts.py": [ + "tests/cli/test_food_truck_prompt.py" + ], + "src/autoskillit/franchise/result_parser.py": [ + "tests/franchise/test_franchise_e2e.py", + "tests/franchise/test_result_parser.py", + "tests/server/test_tools_dispatch.py" + ], + "src/autoskillit/franchise/state.py": [ + "tests/cli/test_franchise_cli.py", + "tests/cli/test_reap.py", + "tests/cli/test_signal_guard.py", + "tests/franchise/test_dispatch_failure_semantics.py", + "tests/franchise/test_dispatch_lifespan.py", + "tests/franchise/test_franchise_e2e.py", + "tests/franchise/test_state.py", + "tests/franchise/test_state_schema.py", + "tests/server/test_tools_dispatch.py" + ], + "src/autoskillit/franchise/summary.py": [ + "tests/franchise/test_summary.py" + ], + "src/autoskillit/hook_registry.py": [ + "tests/arch/test_feature_markers.py", + "tests/cli/test_cli_hooks.py", + "tests/cli/test_doctor.py", + "tests/cli/test_init.py", + "tests/cli/test_install.py", + "tests/contracts/test_instruction_surface.py", + "tests/franchise/test_pack_enforcement.py", + "tests/infra/test_hook_executability.py", + "tests/infra/test_hook_registration_coverage.py", + "tests/infra/test_hook_registry.py", + "tests/infra/test_session_scope_enforcement.py", + "tests/server/test_headless_session.py", + "tests/server/test_lifespan.py", + "tests/server/test_mcp_overrides.py", + "tests/server/test_server_init_gate.py", + "tests/server/test_server_init_session_visibility.py", + "tests/server/test_server_tool_registration.py", + "tests/server/test_tool_annotation_completeness.py", + "tests/server/test_tools_kitchen_gate.py", + "tests/server/test_tools_kitchen_envelope.py", + "tests/server/test_tools_kitchen_visibility.py", + "tests/server/test_tools_recipe.py" + ], + "src/autoskillit/hooks/_fmt_execution.py": [ + "tests/infra/test_pretty_output.py" + ], + "src/autoskillit/hooks/_fmt_primitives.py": [ + "tests/infra/test_pretty_output.py", + "tests/infra/test_pretty_output_integration.py" + ], + "src/autoskillit/hooks/_fmt_recipe.py": [ + "tests/infra/test_pretty_output.py", + "tests/infra/test_pretty_output_integration.py" + ], + "src/autoskillit/hooks/_fmt_status.py": [ + "tests/hooks/test_fmt_status.py", + "tests/infra/test_pretty_output.py", + "tests/infra/test_pretty_output_integration.py" + ], + "src/autoskillit/hooks/_hook_settings.py": [ + "tests/execution/test_quota.py", + "tests/infra/test_hook_settings.py", + "tests/infra/test_quota_check.py", + "tests/infra/test_quota_post_check.py" + ], + "src/autoskillit/hooks/franchise_dispatch_guard.py": [ + "tests/infra/test_franchise_dispatch_guard.py" + ], + "src/autoskillit/hooks/grep_pattern_lint_guard.py": [ + "tests/infra/test_grep_pattern_lint_guard.py" + ], + "src/autoskillit/hooks/leaf_orchestration_guard.py": [ + "tests/infra/test_leaf_orchestration_guard.py" + ], + "src/autoskillit/hooks/open_kitchen_guard.py": [ + "tests/infra/test_open_kitchen_guard.py" + ], + "src/autoskillit/hooks/pr_create_guard.py": [ + "tests/infra/test_pr_create_guard.py" + ], + "src/autoskillit/hooks/pretty_output_hook.py": [ + "tests/infra/test_pretty_output.py", + "tests/infra/test_pretty_output_integration.py" + ], + "src/autoskillit/hooks/quota_guard.py": [ + "tests/execution/test_quota.py", + "tests/infra/test_quota_check.py" + ], + "src/autoskillit/hooks/quota_post_hook.py": [ + "tests/infra/test_quota_post_check.py" + ], + "src/autoskillit/hooks/remove_clone_guard.py": [ + "tests/infra/test_remove_clone_guard.py" + ], + "src/autoskillit/hooks/skill_cmd_guard.py": [ + "tests/contracts/test_instruction_surface.py", + "tests/infra/test_skill_cmd_check.py" + ], + "src/autoskillit/hooks/skill_command_guard.py": [ + "tests/infra/test_skill_command_guard.py" + ], + "src/autoskillit/hooks/token_summary_hook.py": [ + "tests/infra/test_token_summary_appender.py" + ], + "src/autoskillit/hooks/unsafe_install_guard.py": [ + "tests/infra/test_unsafe_install_guard.py" + ], + "src/autoskillit/migration/_api.py": [ + "tests/contracts/test_package_gateways.py", + "tests/migration/test_api.py" + ], + "src/autoskillit/migration/engine.py": [ + "tests/cli/test_cli_serve_logging.py", + "tests/cli/test_init.py", + "tests/cli/test_startup_budget.py", + "tests/contracts/test_package_gateways.py", + "tests/contracts/test_protocol_satisfaction.py", + "tests/migration/test_engine.py", + "tests/server/test_factory.py", + "tests/server/test_factory_recording.py", + "tests/server/test_service_wrappers.py", + "tests/server/test_tools_load_recipe.py" + ], + "src/autoskillit/migration/loader.py": [ + "tests/cli/test_install.py", + "tests/contracts/test_package_gateways.py", + "tests/migration/test_loader.py", + "tests/server/test_service_wrappers.py", + "tests/server/test_tools_load_recipe.py" + ], + "src/autoskillit/migration/store.py": [ + "tests/cli/test_doctor.py", + "tests/migration/test_store.py", + "tests/server/test_service_wrappers.py", + "tests/server/test_tools_load_recipe.py" + ], + "src/autoskillit/pipeline/audit.py": [ + "tests/cli/test_cli_serve_logging.py", + "tests/cli/test_init.py", + "tests/cli/test_startup_budget.py", + "tests/contracts/test_package_gateways.py", + "tests/contracts/test_protocol_satisfaction.py", + "tests/execution/test_clone_guard.py", + "tests/execution/test_headless.py", + "tests/execution/test_headless_add_dirs.py", + "tests/execution/test_headless_env_injection.py", + "tests/execution/test_headless_env_scrub.py", + "tests/execution/test_idle_output_env.py", + "tests/franchise/test_franchise_e2e.py", + "tests/pipeline/test_audit.py", + "tests/pipeline/test_context.py", + "tests/pipeline/test_timings.py", + "tests/pipeline/test_tokens.py", + "tests/server/test_factory.py", + "tests/server/test_factory_recording.py", + "tests/server/test_headless_session.py", + "tests/server/test_run_skill_add_dirs.py", + "tests/server/test_server_init_gate.py", + "tests/server/test_server_init_session_visibility.py", + "tests/server/test_server_tool_registration.py", + "tests/server/test_state.py", + "tests/server/test_tools_execution_command.py", + "tests/server/test_tools_execution_input_gates.py", + "tests/server/test_tools_execution_results.py", + "tests/server/test_tools_execution_routing.py", + "tests/server/test_tools_execution_response.py", + "tests/server/test_tools_recipe.py", + "tests/server/test_tools_run_skill_retry.py", + "tests/server/test_tools_status_kitchen.py", + "tests/server/test_tools_status_summaries.py", + "tests/server/test_tools_status_quota_and_db.py", + "tests/test_conftest.py" + ], + "src/autoskillit/pipeline/background.py": [ + "tests/arch/test_feature_markers.py", + "tests/cli/test_cli_serve_logging.py", + "tests/cli/test_init.py", + "tests/cli/test_startup_budget.py", + "tests/contracts/test_instruction_surface.py", + "tests/contracts/test_package_gateways.py", + "tests/franchise/test_dispatch_failure_semantics.py", + "tests/franchise/test_dispatch_lifespan.py", + "tests/franchise/test_franchise_e2e.py", + "tests/franchise/test_pack_enforcement.py", + "tests/pipeline/test_background_supervisor.py", + "tests/pipeline/test_context.py", + "tests/server/test_factory.py", + "tests/server/test_factory_recording.py", + "tests/server/test_headless_session.py", + "tests/server/test_lifespan.py", + "tests/server/test_run_skill_add_dirs.py", + "tests/server/test_server_init_gate.py", + "tests/server/test_server_init_session_visibility.py", + "tests/server/test_server_tool_registration.py", + "tests/server/test_tool_annotation_completeness.py", + "tests/server/test_tools_dispatch.py", + "tests/server/test_tools_execution_command.py", + "tests/server/test_tools_execution_input_gates.py", + "tests/server/test_tools_execution_results.py", + "tests/server/test_tools_execution_routing.py", + "tests/server/test_tools_execution_response.py", + "tests/server/test_tools_github.py", + "tests/server/test_tools_kitchen_gate.py", + "tests/server/test_tools_kitchen_envelope.py", + "tests/server/test_tools_kitchen_visibility.py", + "tests/server/test_tools_recipe.py", + "tests/server/test_tools_run_skill_retry.py", + "tests/server/test_tools_session_diagnostics.py", + "tests/server/test_tools_status_kitchen.py", + "tests/server/test_tools_status_summaries.py", + "tests/server/test_tools_status_quota_and_db.py" + ], + "src/autoskillit/pipeline/context.py": [ + "tests/cli/test_cli_serve_logging.py", + "tests/cli/test_init.py", + "tests/cli/test_startup_budget.py", + "tests/contracts/test_package_gateways.py", + "tests/pipeline/test_context.py", + "tests/server/test_factory.py", + "tests/server/test_factory_recording.py", + "tests/server/test_tools_ci.py" + ], + "src/autoskillit/pipeline/gate.py": [ + "tests/contracts/test_instruction_surface.py", + "tests/contracts/test_protocol_satisfaction.py", + "tests/pipeline/test_gate.py", + "tests/server/test_helpers_gate.py", + "tests/server/test_helpers_tier_guards.py", + "tests/server/test_server_init_gate.py", + "tests/server/test_server_init_session_visibility.py", + "tests/server/test_server_tool_registration.py", + "tests/server/test_set_commit_status.py", + "tests/server/test_tools_ci.py", + "tests/server/test_tools_clone.py", + "tests/server/test_tools_dispatch.py", + "tests/server/test_tools_execution_command.py", + "tests/server/test_tools_execution_input_gates.py", + "tests/server/test_tools_execution_results.py", + "tests/server/test_tools_execution_routing.py", + "tests/server/test_tools_git.py", + "tests/server/test_tools_github.py", + "tests/server/test_tools_integrations.py", + "tests/server/test_tools_issue_lifecycle.py", + "tests/server/test_tools_kitchen_gate.py", + "tests/server/test_tools_kitchen_envelope.py", + "tests/server/test_tools_kitchen_visibility.py", + "tests/server/test_tools_list_recipes.py", + "tests/server/test_tools_load_recipe.py", + "tests/server/test_tools_pr_ops.py", + "tests/server/test_tools_run_cmd_unit.py", + "tests/server/test_tools_run_python.py", + "tests/server/test_tools_status_kitchen.py", + "tests/server/test_tools_status_summaries.py", + "tests/server/test_tools_status_quota_and_db.py" + ], + "src/autoskillit/pipeline/mcp_response.py": [ + "tests/cli/test_cli_serve_logging.py", + "tests/cli/test_init.py", + "tests/cli/test_startup_budget.py", + "tests/contracts/test_instruction_surface.py", + "tests/contracts/test_package_gateways.py", + "tests/infra/test_pretty_output_integration.py", + "tests/pipeline/test_context.py", + "tests/pipeline/test_mcp_response.py", + "tests/server/test_factory.py", + "tests/server/test_factory_recording.py", + "tests/server/test_mcp_overrides.py", + "tests/server/test_run_skill_add_dirs.py", + "tests/server/test_server_init_gate.py", + "tests/server/test_server_init_session_visibility.py", + "tests/server/test_server_tool_registration.py", + "tests/server/test_set_commit_status.py", + "tests/server/test_smoke_pipeline.py", + "tests/server/test_tool_exception_boundary.py", + "tests/server/test_tools_ci.py", + "tests/server/test_tools_ci_enqueue.py", + "tests/server/test_tools_clone.py", + "tests/server/test_tools_dispatch.py", + "tests/server/test_tools_execution_command.py", + "tests/server/test_tools_execution_input_gates.py", + "tests/server/test_tools_execution_results.py", + "tests/server/test_tools_execution_routing.py", + "tests/server/test_tools_execution_response.py", + "tests/server/test_tools_git.py", + "tests/server/test_tools_github.py", + "tests/server/test_tools_integrations.py", + "tests/server/test_tools_integrations_release.py", + "tests/server/test_tools_issue_lifecycle.py", + "tests/server/test_tools_kitchen_gate.py", + "tests/server/test_tools_kitchen_envelope.py", + "tests/server/test_tools_kitchen_visibility.py", + "tests/server/test_tools_label_validation.py", + "tests/server/test_tools_list_recipes.py", + "tests/server/test_tools_load_recipe.py", + "tests/server/test_tools_pr_ops.py", + "tests/server/test_tools_recipe.py", + "tests/server/test_tools_run_cmd.py", + "tests/server/test_tools_run_cmd_unit.py", + "tests/server/test_tools_run_python.py", + "tests/server/test_tools_run_skill_retry.py", + "tests/server/test_tools_session_diagnostics.py", + "tests/server/test_tools_status_kitchen.py", + "tests/server/test_tools_status_summaries.py", + "tests/server/test_tools_status_quota_and_db.py", + "tests/server/test_tools_status_mcp_response.py", + "tests/server/test_tools_workspace.py", + "tests/server/test_track_response_size.py" + ], + "src/autoskillit/pipeline/pr_gates.py": [ + "tests/pipeline/test_pr_gates.py" + ], + "src/autoskillit/pipeline/telemetry_fmt.py": [ + "tests/infra/test_pretty_output.py", + "tests/infra/test_pretty_output_integration.py", + "tests/pipeline/test_telemetry_formatter.py", + "tests/server/test_tools_status_kitchen.py", + "tests/server/test_tools_status_summaries.py", + "tests/server/test_tools_status_quota_and_db.py", + "tests/server/test_tools_status_mcp_response.py" + ], + "src/autoskillit/pipeline/timings.py": [ + "tests/cli/test_cli_serve_logging.py", + "tests/cli/test_init.py", + "tests/cli/test_startup_budget.py", + "tests/contracts/test_package_gateways.py", + "tests/contracts/test_protocol_satisfaction.py", + "tests/pipeline/test_context.py", + "tests/pipeline/test_timings.py", + "tests/server/test_factory.py", + "tests/server/test_factory_recording.py", + "tests/server/test_state.py", + "tests/server/test_tools_ci.py", + "tests/server/test_tools_clone.py", + "tests/server/test_tools_execution_command.py", + "tests/server/test_tools_execution_input_gates.py", + "tests/server/test_tools_execution_results.py", + "tests/server/test_tools_execution_routing.py", + "tests/server/test_tools_git.py", + "tests/server/test_tools_run_cmd_unit.py", + "tests/server/test_tools_status_kitchen.py", + "tests/server/test_tools_status_summaries.py", + "tests/server/test_tools_status_quota_and_db.py", + "tests/server/test_tools_status_mcp_response.py", + "tests/server/test_tools_workspace.py" + ], + "src/autoskillit/pipeline/tokens.py": [ + "tests/cli/test_cli_serve_logging.py", + "tests/cli/test_franchise_cli.py", + "tests/cli/test_init.py", + "tests/cli/test_startup_budget.py", + "tests/contracts/test_package_gateways.py", + "tests/contracts/test_protocol_satisfaction.py", + "tests/execution/test_headless.py", + "tests/pipeline/test_context.py", + "tests/pipeline/test_timings.py", + "tests/pipeline/test_tokens.py", + "tests/server/test_factory.py", + "tests/server/test_factory_recording.py", + "tests/server/test_server_version_telemetry.py", + "tests/server/test_state.py", + "tests/server/test_tools_ci.py", + "tests/server/test_tools_clone.py", + "tests/server/test_tools_execution_command.py", + "tests/server/test_tools_execution_input_gates.py", + "tests/server/test_tools_execution_results.py", + "tests/server/test_tools_execution_routing.py", + "tests/server/test_tools_git.py", + "tests/server/test_tools_run_cmd_unit.py", + "tests/server/test_tools_status_kitchen.py", + "tests/server/test_tools_status_summaries.py", + "tests/server/test_tools_status_quota_and_db.py", + "tests/server/test_tools_status_mcp_response.py", + "tests/server/test_tools_workspace.py", + "tests/test_conftest.py" + ], + "src/autoskillit/planner/compiler.py": [ + "tests/planner/test_compiler.py" + ], + "src/autoskillit/planner/manifests.py": [ + "tests/planner/test_manifests.py" + ], + "src/autoskillit/planner/validation.py": [ + "tests/planner/test_compiler.py", + "tests/planner/test_validation.py" + ], + "src/autoskillit/recipe/_analysis.py": [ + "tests/cli/test_cli_prompts.py", + "tests/cli/test_cook.py", + "tests/contracts/test_instruction_surface.py", + "tests/contracts/test_package_gateways.py", + "tests/contracts/test_target_skill_invocability.py", + "tests/franchise/test_pack_enforcement.py", + "tests/franchise/test_pack_enforcement_e2e.py", + "tests/migration/test_engine.py", + "tests/recipe/test_anti_pattern_guards.py", + "tests/recipe/test_api.py", + "tests/recipe/test_bundled_recipe_hidden_policy.py", + "tests/recipe/test_bundled_recipes.py", + "tests/recipe/test_bundled_recipes_no_inversions.py", + "tests/recipe/test_campaign_loader.py", + "tests/recipe/test_check_repo_merge_state_routing.py", + "tests/recipe/test_contracts_block_fingerprint.py", + "tests/recipe/test_diagrams.py", + "tests/recipe/test_full_audit_recipe.py", + "tests/recipe/test_identity.py", + "tests/recipe/test_implementation_sprint_mode.py", + "tests/recipe/test_io.py", + "tests/recipe/test_issue_url_pipeline.py", + "tests/recipe/test_make_campaign_output_schema.py", + "tests/recipe/test_merge_prs_queue.py", + "tests/recipe/test_recipe_temp_substitution.py", + "tests/recipe/test_remediation_sprint_mode.py", + "tests/recipe/test_research_output_mode.py", + "tests/recipe/test_resolve_ci_routing_invariant.py", + "tests/recipe/test_review_loop_routing_integration.py", + "tests/recipe/test_rules_blocks.py", + "tests/recipe/test_rules_bypass.py", + "tests/recipe/test_rules_campaign.py", + "tests/recipe/test_rules_ci.py", + "tests/recipe/test_rules_clone.py", + "tests/recipe/test_rules_cmd.py", + "tests/recipe/test_rules_conditional_push.py", + "tests/recipe/test_rules_contracts.py", + "tests/recipe/test_rules_dataflow.py", + "tests/recipe/test_rules_features.py", + "tests/recipe/test_rules_graph.py", + "tests/recipe/test_rules_inputs.py", + "tests/recipe/test_rules_isolation.py", + "tests/recipe/test_rules_merge.py", + "tests/recipe/test_rules_packs.py", + "tests/recipe/test_rules_pipeline_internal.py", + "tests/recipe/test_rules_project_local_override.py", + "tests/recipe/test_rules_reachability.py", + "tests/recipe/test_rules_recipe.py", + "tests/recipe/test_rules_skill_content.py", + "tests/recipe/test_rules_skills.py", + "tests/recipe/test_rules_structure.py", + "tests/recipe/test_rules_subset_disabled.py", + "tests/recipe/test_rules_tools.py", + "tests/recipe/test_rules_verdict.py", + "tests/recipe/test_rules_worktree.py", + "tests/recipe/test_schema.py", + "tests/recipe/test_sprint_sub_recipe.py", + "tests/recipe/test_sub_recipe_loading.py", + "tests/recipe/test_sub_recipe_validation.py", + "tests/recipe/test_validator.py", + "tests/server/test_server_tool_registration.py", + "tests/server/test_service_wrappers.py", + "tests/server/test_smoke_pipeline.py", + "tests/server/test_tools_kitchen_gate.py", + "tests/server/test_tools_kitchen_envelope.py", + "tests/server/test_tools_kitchen_visibility.py", + "tests/server/test_tools_load_recipe.py", + "tests/server/test_tools_recipe.py" + ], + "src/autoskillit/recipe/_api.py": [ + "tests/cli/test_cli_prompts.py", + "tests/cli/test_cook.py", + "tests/contracts/test_package_gateways.py", + "tests/infra/test_pretty_output_integration.py", + "tests/recipe/test__api.py", + "tests/recipe/test_api.py", + "tests/recipe/test_hidden_ingredients.py", + "tests/recipe/test_implementation_sprint_mode.py", + "tests/recipe/test_issue_url_pipeline.py", + "tests/recipe/test_merge_sub_recipe_hidden.py", + "tests/recipe/test_remediation_sprint_mode.py", + "tests/recipe/test_sub_recipe_loading.py", + "tests/recipe/test_sub_recipe_validation.py", + "tests/server/test_mcp_overrides.py", + "tests/server/test_service_wrappers.py", + "tests/server/test_smoke_pipeline.py", + "tests/server/test_tools_kitchen_gate.py", + "tests/server/test_tools_kitchen_envelope.py", + "tests/server/test_tools_kitchen_visibility.py", + "tests/server/test_tools_list_recipes.py", + "tests/server/test_tools_load_recipe.py", + "tests/server/test_tools_recipe.py" + ], + "src/autoskillit/recipe/_skill_placeholder_parser.py": [ + "tests/contracts/test_package_gateways.py", + "tests/recipe/test_api.py", + "tests/recipe/test_bundled_recipe_hidden_policy.py", + "tests/recipe/test_bundled_recipes.py", + "tests/recipe/test_bundled_recipes_no_inversions.py", + "tests/recipe/test_full_audit_recipe.py", + "tests/recipe/test_implementation_sprint_mode.py", + "tests/recipe/test_issue_url_pipeline.py", + "tests/recipe/test_merge_prs_queue.py", + "tests/recipe/test_remediation_sprint_mode.py", + "tests/recipe/test_research_output_mode.py", + "tests/recipe/test_rules_bypass.py", + "tests/recipe/test_rules_ci.py", + "tests/recipe/test_rules_clone.py", + "tests/recipe/test_rules_conditional_push.py", + "tests/recipe/test_rules_contracts.py", + "tests/recipe/test_rules_dataflow.py", + "tests/recipe/test_rules_features.py", + "tests/recipe/test_rules_graph.py", + "tests/recipe/test_rules_inputs.py", + "tests/recipe/test_rules_isolation.py", + "tests/recipe/test_rules_merge.py", + "tests/recipe/test_rules_project_local_override.py", + "tests/recipe/test_rules_reachability.py", + "tests/recipe/test_rules_recipe.py", + "tests/recipe/test_rules_skill_content.py", + "tests/recipe/test_rules_skills.py", + "tests/recipe/test_rules_structure.py", + "tests/recipe/test_rules_subset_disabled.py", + "tests/recipe/test_rules_tools.py", + "tests/recipe/test_rules_verdict.py", + "tests/recipe/test_rules_worktree.py", + "tests/recipe/test_sprint_sub_recipe.py", + "tests/server/test_service_wrappers.py", + "tests/server/test_smoke_pipeline.py", + "tests/server/test_tools_kitchen_gate.py", + "tests/server/test_tools_kitchen_envelope.py", + "tests/server/test_tools_kitchen_visibility.py", + "tests/server/test_tools_load_recipe.py", + "tests/server/test_tools_recipe.py", + "tests/skills/test_make_campaign_compliance.py", + "tests/skills/test_skill_placeholder_contracts.py", + "tests/skills/test_skill_tool_syntax_contracts.py" + ], + "src/autoskillit/recipe/contracts.py": [ + "tests/cli/test_cook.py", + "tests/contracts/test_instruction_surface.py", + "tests/contracts/test_package_gateways.py", + "tests/contracts/test_review_pr_diff_annotation.py", + "tests/migration/test_engine.py", + "tests/recipe/test_api.py", + "tests/recipe/test_bundled_recipe_hidden_policy.py", + "tests/recipe/test_bundled_recipes.py", + "tests/recipe/test_bundled_recipes_no_inversions.py", + "tests/recipe/test_contracts.py", + "tests/recipe/test_contracts_block_fingerprint.py", + "tests/recipe/test_full_audit_recipe.py", + "tests/recipe/test_identity.py", + "tests/recipe/test_implementation_sprint_mode.py", + "tests/recipe/test_issue_url_pipeline.py", + "tests/recipe/test_merge_prs_queue.py", + "tests/recipe/test_remediation_sprint_mode.py", + "tests/recipe/test_research_output_mode.py", + "tests/recipe/test_rules_bypass.py", + "tests/recipe/test_rules_ci.py", + "tests/recipe/test_rules_clone.py", + "tests/recipe/test_rules_cmd.py", + "tests/recipe/test_rules_conditional_push.py", + "tests/recipe/test_rules_contracts.py", + "tests/recipe/test_rules_dataflow.py", + "tests/recipe/test_rules_features.py", + "tests/recipe/test_rules_graph.py", + "tests/recipe/test_rules_inputs.py", + "tests/recipe/test_rules_isolation.py", + "tests/recipe/test_rules_merge.py", + "tests/recipe/test_rules_project_local_override.py", + "tests/recipe/test_rules_reachability.py", + "tests/recipe/test_rules_recipe.py", + "tests/recipe/test_rules_skill_content.py", + "tests/recipe/test_rules_skills.py", + "tests/recipe/test_rules_structure.py", + "tests/recipe/test_rules_subset_disabled.py", + "tests/recipe/test_rules_tools.py", + "tests/recipe/test_rules_verdict.py", + "tests/recipe/test_rules_worktree.py", + "tests/recipe/test_sprint_sub_recipe.py", + "tests/recipe/test_staleness_cache.py", + "tests/recipe/test_sub_recipe_validation.py", + "tests/server/test_factory.py", + "tests/server/test_run_skill_add_dirs.py", + "tests/server/test_server_tool_registration.py", + "tests/server/test_service_wrappers.py", + "tests/server/test_smoke_pipeline.py", + "tests/server/test_tools_execution_command.py", + "tests/server/test_tools_execution_input_gates.py", + "tests/server/test_tools_execution_results.py", + "tests/server/test_tools_execution_routing.py", + "tests/server/test_tools_execution_response.py", + "tests/server/test_tools_github.py", + "tests/server/test_tools_integrations.py", + "tests/server/test_tools_issue_lifecycle.py", + "tests/server/test_tools_kitchen_gate.py", + "tests/server/test_tools_kitchen_envelope.py", + "tests/server/test_tools_kitchen_visibility.py", + "tests/server/test_tools_label_validation.py", + "tests/server/test_tools_load_recipe.py", + "tests/server/test_tools_recipe.py", + "tests/server/test_tools_run_skill_retry.py", + "tests/server/test_tools_session_diagnostics.py", + "tests/server/test_tools_status_kitchen.py", + "tests/server/test_tools_status_summaries.py", + "tests/server/test_tools_status_quota_and_db.py" + ], + "src/autoskillit/recipe/diagrams.py": [ + "tests/cli/test_cli_prompts.py", + "tests/cli/test_cook.py", + "tests/contracts/test_package_gateways.py", + "tests/migration/test_engine.py", + "tests/recipe/test_api.py", + "tests/recipe/test_diagrams.py", + "tests/recipe/test_sub_recipe_validation.py", + "tests/server/test_service_wrappers.py", + "tests/server/test_tools_kitchen_gate.py", + "tests/server/test_tools_kitchen_envelope.py", + "tests/server/test_tools_kitchen_visibility.py", + "tests/server/test_tools_load_recipe.py" + ], + "src/autoskillit/recipe/experiment_type_registry.py": [ + "tests/recipe/test_experiment_type_registry.py", + "tests/skills/test_review_design_guards.py" + ], + "src/autoskillit/recipe/identity.py": [ + "tests/cli/test_cook.py", + "tests/contracts/test_package_gateways.py", + "tests/recipe/test_api.py", + "tests/recipe/test_identity.py", + "tests/recipe/test_sub_recipe_validation.py", + "tests/server/test_service_wrappers.py", + "tests/server/test_tools_kitchen_gate.py", + "tests/server/test_tools_kitchen_envelope.py", + "tests/server/test_tools_kitchen_visibility.py", + "tests/server/test_tools_load_recipe.py" + ], + "src/autoskillit/recipe/io.py": [ + "tests/cli/test_cli_prompts.py", + "tests/cli/test_cook.py", + "tests/cli/test_doctor.py", + "tests/cli/test_franchise_cli.py", + "tests/cli/test_input_tty_contracts.py", + "tests/cli/test_install.py", + "tests/contracts/test_instruction_surface.py", + "tests/contracts/test_package_gateways.py", + "tests/contracts/test_skill_yaml_validation.py", + "tests/contracts/test_target_skill_invocability.py", + "tests/franchise/test_pack_enforcement.py", + "tests/franchise/test_pack_enforcement_e2e.py", + "tests/infra/test_pretty_output_integration.py", + "tests/migration/test_engine.py", + "tests/recipe/test_anti_pattern_guards.py", + "tests/recipe/test_api.py", + "tests/recipe/test_bundled_recipe_hidden_policy.py", + "tests/recipe/test_bundled_recipes.py", + "tests/recipe/test_bundled_recipes_no_inversions.py", + "tests/recipe/test_campaign_loader.py", + "tests/recipe/test_check_repo_merge_state_routing.py", + "tests/recipe/test_contracts.py", + "tests/recipe/test_contracts_block_fingerprint.py", + "tests/recipe/test_diagrams.py", + "tests/recipe/test_full_audit_recipe.py", + "tests/recipe/test_hidden_ingredients.py", + "tests/recipe/test_identity.py", + "tests/recipe/test_implementation_pr_decomposition.py", + "tests/recipe/test_implementation_sprint_mode.py", + "tests/recipe/test_io.py", + "tests/recipe/test_issue_url_pipeline.py", + "tests/recipe/test_loader.py", + "tests/recipe/test_make_campaign_output_schema.py", + "tests/recipe/test_merge_prs.py", + "tests/recipe/test_merge_prs_queue.py", + "tests/recipe/test_recipe_temp_substitution.py", + "tests/recipe/test_remediation_pr_decomposition.py", + "tests/recipe/test_remediation_recipe.py", + "tests/recipe/test_remediation_sprint_mode.py", + "tests/recipe/test_repository.py", + "tests/recipe/test_research_output_mode.py", + "tests/recipe/test_research_recipe_diag.py", + "tests/recipe/test_research_stage_data_step.py", + "tests/recipe/test_resolve_ci_routing_invariant.py", + "tests/recipe/test_review_loop_routing_integration.py", + "tests/recipe/test_rules_blocks.py", + "tests/recipe/test_rules_bypass.py", + "tests/recipe/test_rules_campaign.py", + "tests/recipe/test_rules_ci.py", + "tests/recipe/test_rules_clone.py", + "tests/recipe/test_rules_cmd.py", + "tests/recipe/test_rules_conditional_push.py", + "tests/recipe/test_rules_contracts.py", + "tests/recipe/test_rules_dataflow.py", + "tests/recipe/test_rules_features.py", + "tests/recipe/test_rules_graph.py", + "tests/recipe/test_rules_inputs.py", + "tests/recipe/test_rules_isolation.py", + "tests/recipe/test_rules_merge.py", + "tests/recipe/test_rules_packs.py", + "tests/recipe/test_rules_pipeline_internal.py", + "tests/recipe/test_rules_project_local_override.py", + "tests/recipe/test_rules_reachability.py", + "tests/recipe/test_rules_recipe.py", + "tests/recipe/test_rules_skill_content.py", + "tests/recipe/test_rules_skills.py", + "tests/recipe/test_rules_structure.py", + "tests/recipe/test_rules_subset_disabled.py", + "tests/recipe/test_rules_tools.py", + "tests/recipe/test_rules_verdict.py", + "tests/recipe/test_rules_worktree.py", + "tests/recipe/test_schema.py", + "tests/recipe/test_sprint_sub_recipe.py", + "tests/recipe/test_sub_recipe_loading.py", + "tests/recipe/test_sub_recipe_schema.py", + "tests/recipe/test_sub_recipe_validation.py", + "tests/recipe/test_validator.py", + "tests/server/test_server_tool_registration.py", + "tests/server/test_service_wrappers.py", + "tests/server/test_smoke_pipeline.py", + "tests/server/test_tools_kitchen_gate.py", + "tests/server/test_tools_kitchen_envelope.py", + "tests/server/test_tools_kitchen_visibility.py", + "tests/server/test_tools_list_recipes.py", + "tests/server/test_tools_load_recipe.py", + "tests/server/test_tools_recipe.py" + ], + "src/autoskillit/recipe/loader.py": [ + "tests/migration/test_engine.py", + "tests/recipe/test_loader.py", + "tests/server/test_service_wrappers.py", + "tests/server/test_tools_load_recipe.py" + ], + "src/autoskillit/recipe/registry.py": [ + "tests/cli/test_cook.py", + "tests/contracts/test_package_gateways.py", + "tests/recipe/test_api.py", + "tests/recipe/test_bundled_recipe_hidden_policy.py", + "tests/recipe/test_bundled_recipes.py", + "tests/recipe/test_bundled_recipes_no_inversions.py", + "tests/recipe/test_full_audit_recipe.py", + "tests/recipe/test_implementation_sprint_mode.py", + "tests/recipe/test_issue_url_pipeline.py", + "tests/recipe/test_make_campaign_output_schema.py", + "tests/recipe/test_merge_prs_queue.py", + "tests/recipe/test_remediation_sprint_mode.py", + "tests/recipe/test_research_output_mode.py", + "tests/recipe/test_rules_blocks.py", + "tests/recipe/test_rules_bypass.py", + "tests/recipe/test_rules_campaign.py", + "tests/recipe/test_rules_ci.py", + "tests/recipe/test_rules_clone.py", + "tests/recipe/test_rules_cmd.py", + "tests/recipe/test_rules_conditional_push.py", + "tests/recipe/test_rules_contracts.py", + "tests/recipe/test_rules_dataflow.py", + "tests/recipe/test_rules_features.py", + "tests/recipe/test_rules_graph.py", + "tests/recipe/test_rules_inputs.py", + "tests/recipe/test_rules_isolation.py", + "tests/recipe/test_rules_merge.py", + "tests/recipe/test_rules_packs.py", + "tests/recipe/test_rules_pipeline_internal.py", + "tests/recipe/test_rules_project_local_override.py", + "tests/recipe/test_rules_reachability.py", + "tests/recipe/test_rules_recipe.py", + "tests/recipe/test_rules_skill_content.py", + "tests/recipe/test_rules_skills.py", + "tests/recipe/test_rules_structure.py", + "tests/recipe/test_rules_subset_disabled.py", + "tests/recipe/test_rules_tools.py", + "tests/recipe/test_rules_verdict.py", + "tests/recipe/test_rules_worktree.py", + "tests/recipe/test_sprint_sub_recipe.py", + "tests/recipe/test_sub_recipe_validation.py", + "tests/recipe/test_validator.py", + "tests/server/test_service_wrappers.py", + "tests/server/test_smoke_pipeline.py", + "tests/server/test_tools_kitchen_gate.py", + "tests/server/test_tools_kitchen_envelope.py", + "tests/server/test_tools_kitchen_visibility.py", + "tests/server/test_tools_load_recipe.py", + "tests/server/test_tools_recipe.py" + ], + "src/autoskillit/recipe/repository.py": [ + "tests/cli/test_cli_serve_logging.py", + "tests/cli/test_init.py", + "tests/cli/test_startup_budget.py", + "tests/contracts/test_package_gateways.py", + "tests/contracts/test_protocol_satisfaction.py", + "tests/infra/test_pretty_output_integration.py", + "tests/recipe/test_api.py", + "tests/recipe/test_repository.py", + "tests/server/test_factory.py", + "tests/server/test_factory_recording.py", + "tests/server/test_mcp_overrides.py", + "tests/server/test_service_wrappers.py", + "tests/server/test_smoke_pipeline.py", + "tests/server/test_tools_kitchen_gate.py", + "tests/server/test_tools_kitchen_envelope.py", + "tests/server/test_tools_kitchen_visibility.py", + "tests/server/test_tools_list_recipes.py", + "tests/server/test_tools_load_recipe.py", + "tests/server/test_tools_recipe.py" + ], + "src/autoskillit/recipe/rules_blocks.py": [ + "tests/contracts/test_package_gateways.py", + "tests/recipe/test_anti_pattern_guards.py", + "tests/recipe/test_bundled_recipe_hidden_policy.py", + "tests/recipe/test_bundled_recipes.py", + "tests/recipe/test_bundled_recipes_no_inversions.py", + "tests/recipe/test_implementation_sprint_mode.py", + "tests/recipe/test_issue_url_pipeline.py", + "tests/recipe/test_merge_prs_queue.py", + "tests/recipe/test_remediation_sprint_mode.py", + "tests/recipe/test_rules_blocks.py", + "tests/recipe/test_rules_ci.py", + "tests/recipe/test_rules_clone.py", + "tests/recipe/test_rules_contracts.py", + "tests/recipe/test_rules_dataflow.py", + "tests/recipe/test_rules_isolation.py", + "tests/recipe/test_rules_merge.py", + "tests/recipe/test_rules_reachability.py", + "tests/recipe/test_rules_recipe.py", + "tests/recipe/test_rules_skill_content.py", + "tests/recipe/test_rules_skills.py", + "tests/recipe/test_rules_structure.py", + "tests/recipe/test_rules_worktree.py", + "tests/server/test_service_wrappers.py", + "tests/server/test_tools_load_recipe.py" + ], + "src/autoskillit/recipe/rules_bypass.py": [ + "tests/cli/test_cook.py", + "tests/contracts/test_package_gateways.py", + "tests/recipe/test_api.py", + "tests/recipe/test_bundled_recipe_hidden_policy.py", + "tests/recipe/test_bundled_recipes.py", + "tests/recipe/test_bundled_recipes_no_inversions.py", + "tests/recipe/test_full_audit_recipe.py", + "tests/recipe/test_implementation_sprint_mode.py", + "tests/recipe/test_issue_url_pipeline.py", + "tests/recipe/test_make_campaign_output_schema.py", + "tests/recipe/test_merge_prs_queue.py", + "tests/recipe/test_remediation_sprint_mode.py", + "tests/recipe/test_research_output_mode.py", + "tests/recipe/test_rules_blocks.py", + "tests/recipe/test_rules_bypass.py", + "tests/recipe/test_rules_campaign.py", + "tests/recipe/test_rules_ci.py", + "tests/recipe/test_rules_clone.py", + "tests/recipe/test_rules_cmd.py", + "tests/recipe/test_rules_conditional_push.py", + "tests/recipe/test_rules_contracts.py", + "tests/recipe/test_rules_dataflow.py", + "tests/recipe/test_rules_features.py", + "tests/recipe/test_rules_graph.py", + "tests/recipe/test_rules_inputs.py", + "tests/recipe/test_rules_isolation.py", + "tests/recipe/test_rules_merge.py", + "tests/recipe/test_rules_packs.py", + "tests/recipe/test_rules_pipeline_internal.py", + "tests/recipe/test_rules_project_local_override.py", + "tests/recipe/test_rules_reachability.py", + "tests/recipe/test_rules_recipe.py", + "tests/recipe/test_rules_skill_content.py", + "tests/recipe/test_rules_skills.py", + "tests/recipe/test_rules_structure.py", + "tests/recipe/test_rules_subset_disabled.py", + "tests/recipe/test_rules_tools.py", + "tests/recipe/test_rules_verdict.py", + "tests/recipe/test_rules_worktree.py", + "tests/recipe/test_sprint_sub_recipe.py", + "tests/recipe/test_sub_recipe_validation.py", + "tests/recipe/test_validator.py", + "tests/server/test_service_wrappers.py", + "tests/server/test_smoke_pipeline.py", + "tests/server/test_tools_kitchen_gate.py", + "tests/server/test_tools_kitchen_envelope.py", + "tests/server/test_tools_kitchen_visibility.py", + "tests/server/test_tools_load_recipe.py", + "tests/server/test_tools_recipe.py" + ], + "src/autoskillit/recipe/rules_campaign.py": [ + "tests/cli/test_cook.py", + "tests/contracts/test_package_gateways.py", + "tests/recipe/test_api.py", + "tests/recipe/test_bundled_recipe_hidden_policy.py", + "tests/recipe/test_bundled_recipes.py", + "tests/recipe/test_bundled_recipes_no_inversions.py", + "tests/recipe/test_full_audit_recipe.py", + "tests/recipe/test_implementation_sprint_mode.py", + "tests/recipe/test_issue_url_pipeline.py", + "tests/recipe/test_make_campaign_output_schema.py", + "tests/recipe/test_merge_prs_queue.py", + "tests/recipe/test_remediation_sprint_mode.py", + "tests/recipe/test_research_output_mode.py", + "tests/recipe/test_rules_blocks.py", + "tests/recipe/test_rules_bypass.py", + "tests/recipe/test_rules_campaign.py", + "tests/recipe/test_rules_ci.py", + "tests/recipe/test_rules_clone.py", + "tests/recipe/test_rules_cmd.py", + "tests/recipe/test_rules_conditional_push.py", + "tests/recipe/test_rules_contracts.py", + "tests/recipe/test_rules_dataflow.py", + "tests/recipe/test_rules_features.py", + "tests/recipe/test_rules_graph.py", + "tests/recipe/test_rules_inputs.py", + "tests/recipe/test_rules_isolation.py", + "tests/recipe/test_rules_merge.py", + "tests/recipe/test_rules_packs.py", + "tests/recipe/test_rules_pipeline_internal.py", + "tests/recipe/test_rules_project_local_override.py", + "tests/recipe/test_rules_reachability.py", + "tests/recipe/test_rules_recipe.py", + "tests/recipe/test_rules_skill_content.py", + "tests/recipe/test_rules_skills.py", + "tests/recipe/test_rules_structure.py", + "tests/recipe/test_rules_subset_disabled.py", + "tests/recipe/test_rules_tools.py", + "tests/recipe/test_rules_verdict.py", + "tests/recipe/test_rules_worktree.py", + "tests/recipe/test_sprint_sub_recipe.py", + "tests/recipe/test_sub_recipe_validation.py", + "tests/recipe/test_validator.py", + "tests/server/test_service_wrappers.py", + "tests/server/test_smoke_pipeline.py", + "tests/server/test_tools_kitchen_gate.py", + "tests/server/test_tools_kitchen_envelope.py", + "tests/server/test_tools_kitchen_visibility.py", + "tests/server/test_tools_load_recipe.py", + "tests/server/test_tools_recipe.py" + ], + "src/autoskillit/recipe/rules_ci.py": [ + "tests/cli/test_cook.py", + "tests/contracts/test_package_gateways.py", + "tests/recipe/test_api.py", + "tests/recipe/test_bundled_recipe_hidden_policy.py", + "tests/recipe/test_bundled_recipes.py", + "tests/recipe/test_bundled_recipes_no_inversions.py", + "tests/recipe/test_full_audit_recipe.py", + "tests/recipe/test_implementation_sprint_mode.py", + "tests/recipe/test_issue_url_pipeline.py", + "tests/recipe/test_make_campaign_output_schema.py", + "tests/recipe/test_merge_prs_queue.py", + "tests/recipe/test_remediation_sprint_mode.py", + "tests/recipe/test_research_output_mode.py", + "tests/recipe/test_rules_blocks.py", + "tests/recipe/test_rules_bypass.py", + "tests/recipe/test_rules_campaign.py", + "tests/recipe/test_rules_ci.py", + "tests/recipe/test_rules_clone.py", + "tests/recipe/test_rules_cmd.py", + "tests/recipe/test_rules_conditional_push.py", + "tests/recipe/test_rules_contracts.py", + "tests/recipe/test_rules_dataflow.py", + "tests/recipe/test_rules_features.py", + "tests/recipe/test_rules_graph.py", + "tests/recipe/test_rules_inputs.py", + "tests/recipe/test_rules_isolation.py", + "tests/recipe/test_rules_merge.py", + "tests/recipe/test_rules_packs.py", + "tests/recipe/test_rules_pipeline_internal.py", + "tests/recipe/test_rules_project_local_override.py", + "tests/recipe/test_rules_reachability.py", + "tests/recipe/test_rules_recipe.py", + "tests/recipe/test_rules_skill_content.py", + "tests/recipe/test_rules_skills.py", + "tests/recipe/test_rules_structure.py", + "tests/recipe/test_rules_subset_disabled.py", + "tests/recipe/test_rules_tools.py", + "tests/recipe/test_rules_verdict.py", + "tests/recipe/test_rules_worktree.py", + "tests/recipe/test_sprint_sub_recipe.py", + "tests/recipe/test_sub_recipe_validation.py", + "tests/recipe/test_validator.py", + "tests/server/test_service_wrappers.py", + "tests/server/test_smoke_pipeline.py", + "tests/server/test_tools_kitchen_gate.py", + "tests/server/test_tools_kitchen_envelope.py", + "tests/server/test_tools_kitchen_visibility.py", + "tests/server/test_tools_load_recipe.py", + "tests/server/test_tools_recipe.py" + ], + "src/autoskillit/recipe/rules_clone.py": [ + "tests/cli/test_cook.py", + "tests/contracts/test_package_gateways.py", + "tests/recipe/test_api.py", + "tests/recipe/test_bundled_recipe_hidden_policy.py", + "tests/recipe/test_bundled_recipes.py", + "tests/recipe/test_bundled_recipes_no_inversions.py", + "tests/recipe/test_full_audit_recipe.py", + "tests/recipe/test_implementation_sprint_mode.py", + "tests/recipe/test_issue_url_pipeline.py", + "tests/recipe/test_make_campaign_output_schema.py", + "tests/recipe/test_merge_prs_queue.py", + "tests/recipe/test_remediation_sprint_mode.py", + "tests/recipe/test_research_output_mode.py", + "tests/recipe/test_rules_blocks.py", + "tests/recipe/test_rules_bypass.py", + "tests/recipe/test_rules_campaign.py", + "tests/recipe/test_rules_ci.py", + "tests/recipe/test_rules_clone.py", + "tests/recipe/test_rules_cmd.py", + "tests/recipe/test_rules_conditional_push.py", + "tests/recipe/test_rules_contracts.py", + "tests/recipe/test_rules_dataflow.py", + "tests/recipe/test_rules_features.py", + "tests/recipe/test_rules_graph.py", + "tests/recipe/test_rules_inputs.py", + "tests/recipe/test_rules_isolation.py", + "tests/recipe/test_rules_merge.py", + "tests/recipe/test_rules_packs.py", + "tests/recipe/test_rules_pipeline_internal.py", + "tests/recipe/test_rules_project_local_override.py", + "tests/recipe/test_rules_reachability.py", + "tests/recipe/test_rules_recipe.py", + "tests/recipe/test_rules_skill_content.py", + "tests/recipe/test_rules_skills.py", + "tests/recipe/test_rules_structure.py", + "tests/recipe/test_rules_subset_disabled.py", + "tests/recipe/test_rules_tools.py", + "tests/recipe/test_rules_verdict.py", + "tests/recipe/test_rules_worktree.py", + "tests/recipe/test_sprint_sub_recipe.py", + "tests/recipe/test_sub_recipe_validation.py", + "tests/recipe/test_validator.py", + "tests/server/test_service_wrappers.py", + "tests/server/test_smoke_pipeline.py", + "tests/server/test_tools_kitchen_gate.py", + "tests/server/test_tools_kitchen_envelope.py", + "tests/server/test_tools_kitchen_visibility.py", + "tests/server/test_tools_load_recipe.py", + "tests/server/test_tools_recipe.py" + ], + "src/autoskillit/recipe/rules_cmd.py": [ + "tests/cli/test_cook.py", + "tests/contracts/test_package_gateways.py", + "tests/recipe/test_api.py", + "tests/recipe/test_bundled_recipe_hidden_policy.py", + "tests/recipe/test_bundled_recipes.py", + "tests/recipe/test_bundled_recipes_no_inversions.py", + "tests/recipe/test_full_audit_recipe.py", + "tests/recipe/test_implementation_sprint_mode.py", + "tests/recipe/test_issue_url_pipeline.py", + "tests/recipe/test_make_campaign_output_schema.py", + "tests/recipe/test_merge_prs_queue.py", + "tests/recipe/test_remediation_sprint_mode.py", + "tests/recipe/test_research_output_mode.py", + "tests/recipe/test_rules_blocks.py", + "tests/recipe/test_rules_bypass.py", + "tests/recipe/test_rules_campaign.py", + "tests/recipe/test_rules_ci.py", + "tests/recipe/test_rules_clone.py", + "tests/recipe/test_rules_cmd.py", + "tests/recipe/test_rules_conditional_push.py", + "tests/recipe/test_rules_contracts.py", + "tests/recipe/test_rules_dataflow.py", + "tests/recipe/test_rules_features.py", + "tests/recipe/test_rules_graph.py", + "tests/recipe/test_rules_inputs.py", + "tests/recipe/test_rules_isolation.py", + "tests/recipe/test_rules_merge.py", + "tests/recipe/test_rules_packs.py", + "tests/recipe/test_rules_pipeline_internal.py", + "tests/recipe/test_rules_project_local_override.py", + "tests/recipe/test_rules_reachability.py", + "tests/recipe/test_rules_recipe.py", + "tests/recipe/test_rules_skill_content.py", + "tests/recipe/test_rules_skills.py", + "tests/recipe/test_rules_structure.py", + "tests/recipe/test_rules_subset_disabled.py", + "tests/recipe/test_rules_tools.py", + "tests/recipe/test_rules_verdict.py", + "tests/recipe/test_rules_worktree.py", + "tests/recipe/test_sprint_sub_recipe.py", + "tests/recipe/test_sub_recipe_validation.py", + "tests/recipe/test_validator.py", + "tests/server/test_service_wrappers.py", + "tests/server/test_smoke_pipeline.py", + "tests/server/test_tools_kitchen_gate.py", + "tests/server/test_tools_kitchen_envelope.py", + "tests/server/test_tools_kitchen_visibility.py", + "tests/server/test_tools_load_recipe.py", + "tests/server/test_tools_recipe.py" + ], + "src/autoskillit/recipe/rules_contracts.py": [ + "tests/cli/test_cook.py", + "tests/contracts/test_package_gateways.py", + "tests/recipe/test_api.py", + "tests/recipe/test_bundled_recipe_hidden_policy.py", + "tests/recipe/test_bundled_recipes.py", + "tests/recipe/test_bundled_recipes_no_inversions.py", + "tests/recipe/test_full_audit_recipe.py", + "tests/recipe/test_implementation_sprint_mode.py", + "tests/recipe/test_issue_url_pipeline.py", + "tests/recipe/test_make_campaign_output_schema.py", + "tests/recipe/test_merge_prs_queue.py", + "tests/recipe/test_remediation_sprint_mode.py", + "tests/recipe/test_research_output_mode.py", + "tests/recipe/test_rules_blocks.py", + "tests/recipe/test_rules_bypass.py", + "tests/recipe/test_rules_campaign.py", + "tests/recipe/test_rules_ci.py", + "tests/recipe/test_rules_clone.py", + "tests/recipe/test_rules_cmd.py", + "tests/recipe/test_rules_conditional_push.py", + "tests/recipe/test_rules_contracts.py", + "tests/recipe/test_rules_dataflow.py", + "tests/recipe/test_rules_features.py", + "tests/recipe/test_rules_graph.py", + "tests/recipe/test_rules_inputs.py", + "tests/recipe/test_rules_isolation.py", + "tests/recipe/test_rules_merge.py", + "tests/recipe/test_rules_packs.py", + "tests/recipe/test_rules_pipeline_internal.py", + "tests/recipe/test_rules_project_local_override.py", + "tests/recipe/test_rules_reachability.py", + "tests/recipe/test_rules_recipe.py", + "tests/recipe/test_rules_skill_content.py", + "tests/recipe/test_rules_skills.py", + "tests/recipe/test_rules_structure.py", + "tests/recipe/test_rules_subset_disabled.py", + "tests/recipe/test_rules_tools.py", + "tests/recipe/test_rules_verdict.py", + "tests/recipe/test_rules_worktree.py", + "tests/recipe/test_sprint_sub_recipe.py", + "tests/recipe/test_sub_recipe_validation.py", + "tests/recipe/test_validator.py", + "tests/server/test_service_wrappers.py", + "tests/server/test_smoke_pipeline.py", + "tests/server/test_tools_kitchen_gate.py", + "tests/server/test_tools_kitchen_envelope.py", + "tests/server/test_tools_kitchen_visibility.py", + "tests/server/test_tools_load_recipe.py", + "tests/server/test_tools_recipe.py" + ], + "src/autoskillit/recipe/rules_dataflow.py": [ + "tests/cli/test_cook.py", + "tests/contracts/test_package_gateways.py", + "tests/recipe/test_api.py", + "tests/recipe/test_bundled_recipe_hidden_policy.py", + "tests/recipe/test_bundled_recipes.py", + "tests/recipe/test_bundled_recipes_no_inversions.py", + "tests/recipe/test_full_audit_recipe.py", + "tests/recipe/test_implementation_sprint_mode.py", + "tests/recipe/test_issue_url_pipeline.py", + "tests/recipe/test_make_campaign_output_schema.py", + "tests/recipe/test_merge_prs_queue.py", + "tests/recipe/test_remediation_sprint_mode.py", + "tests/recipe/test_research_output_mode.py", + "tests/recipe/test_rules_blocks.py", + "tests/recipe/test_rules_bypass.py", + "tests/recipe/test_rules_campaign.py", + "tests/recipe/test_rules_ci.py", + "tests/recipe/test_rules_clone.py", + "tests/recipe/test_rules_cmd.py", + "tests/recipe/test_rules_conditional_push.py", + "tests/recipe/test_rules_contracts.py", + "tests/recipe/test_rules_dataflow.py", + "tests/recipe/test_rules_features.py", + "tests/recipe/test_rules_graph.py", + "tests/recipe/test_rules_inputs.py", + "tests/recipe/test_rules_isolation.py", + "tests/recipe/test_rules_merge.py", + "tests/recipe/test_rules_packs.py", + "tests/recipe/test_rules_pipeline_internal.py", + "tests/recipe/test_rules_project_local_override.py", + "tests/recipe/test_rules_reachability.py", + "tests/recipe/test_rules_recipe.py", + "tests/recipe/test_rules_skill_content.py", + "tests/recipe/test_rules_skills.py", + "tests/recipe/test_rules_structure.py", + "tests/recipe/test_rules_subset_disabled.py", + "tests/recipe/test_rules_tools.py", + "tests/recipe/test_rules_verdict.py", + "tests/recipe/test_rules_worktree.py", + "tests/recipe/test_sprint_sub_recipe.py", + "tests/recipe/test_sub_recipe_validation.py", + "tests/recipe/test_validator.py", + "tests/server/test_service_wrappers.py", + "tests/server/test_smoke_pipeline.py", + "tests/server/test_tools_kitchen_gate.py", + "tests/server/test_tools_kitchen_envelope.py", + "tests/server/test_tools_kitchen_visibility.py", + "tests/server/test_tools_load_recipe.py", + "tests/server/test_tools_recipe.py" + ], + "src/autoskillit/recipe/rules_features.py": [ + "tests/cli/test_cook.py", + "tests/contracts/test_package_gateways.py", + "tests/recipe/test_api.py", + "tests/recipe/test_bundled_recipe_hidden_policy.py", + "tests/recipe/test_bundled_recipes.py", + "tests/recipe/test_bundled_recipes_no_inversions.py", + "tests/recipe/test_full_audit_recipe.py", + "tests/recipe/test_implementation_sprint_mode.py", + "tests/recipe/test_issue_url_pipeline.py", + "tests/recipe/test_make_campaign_output_schema.py", + "tests/recipe/test_merge_prs_queue.py", + "tests/recipe/test_remediation_sprint_mode.py", + "tests/recipe/test_research_output_mode.py", + "tests/recipe/test_rules_blocks.py", + "tests/recipe/test_rules_bypass.py", + "tests/recipe/test_rules_campaign.py", + "tests/recipe/test_rules_ci.py", + "tests/recipe/test_rules_clone.py", + "tests/recipe/test_rules_cmd.py", + "tests/recipe/test_rules_conditional_push.py", + "tests/recipe/test_rules_contracts.py", + "tests/recipe/test_rules_dataflow.py", + "tests/recipe/test_rules_features.py", + "tests/recipe/test_rules_graph.py", + "tests/recipe/test_rules_inputs.py", + "tests/recipe/test_rules_isolation.py", + "tests/recipe/test_rules_merge.py", + "tests/recipe/test_rules_packs.py", + "tests/recipe/test_rules_pipeline_internal.py", + "tests/recipe/test_rules_project_local_override.py", + "tests/recipe/test_rules_reachability.py", + "tests/recipe/test_rules_recipe.py", + "tests/recipe/test_rules_skill_content.py", + "tests/recipe/test_rules_skills.py", + "tests/recipe/test_rules_structure.py", + "tests/recipe/test_rules_subset_disabled.py", + "tests/recipe/test_rules_tools.py", + "tests/recipe/test_rules_verdict.py", + "tests/recipe/test_rules_worktree.py", + "tests/recipe/test_sprint_sub_recipe.py", + "tests/recipe/test_sub_recipe_validation.py", + "tests/recipe/test_validator.py", + "tests/server/test_service_wrappers.py", + "tests/server/test_smoke_pipeline.py", + "tests/server/test_tools_kitchen_gate.py", + "tests/server/test_tools_kitchen_envelope.py", + "tests/server/test_tools_kitchen_visibility.py", + "tests/server/test_tools_load_recipe.py", + "tests/server/test_tools_recipe.py" + ], + "src/autoskillit/recipe/rules_fixing.py": [ + "tests/cli/test_cook.py", + "tests/contracts/test_package_gateways.py", + "tests/recipe/test_api.py", + "tests/recipe/test_bundled_recipe_hidden_policy.py", + "tests/recipe/test_bundled_recipes.py", + "tests/recipe/test_bundled_recipes_no_inversions.py", + "tests/recipe/test_full_audit_recipe.py", + "tests/recipe/test_implementation_sprint_mode.py", + "tests/recipe/test_issue_url_pipeline.py", + "tests/recipe/test_make_campaign_output_schema.py", + "tests/recipe/test_merge_prs_queue.py", + "tests/recipe/test_remediation_sprint_mode.py", + "tests/recipe/test_research_output_mode.py", + "tests/recipe/test_rules_blocks.py", + "tests/recipe/test_rules_bypass.py", + "tests/recipe/test_rules_campaign.py", + "tests/recipe/test_rules_ci.py", + "tests/recipe/test_rules_clone.py", + "tests/recipe/test_rules_cmd.py", + "tests/recipe/test_rules_conditional_push.py", + "tests/recipe/test_rules_contracts.py", + "tests/recipe/test_rules_dataflow.py", + "tests/recipe/test_rules_features.py", + "tests/recipe/test_rules_graph.py", + "tests/recipe/test_rules_inputs.py", + "tests/recipe/test_rules_isolation.py", + "tests/recipe/test_rules_merge.py", + "tests/recipe/test_rules_packs.py", + "tests/recipe/test_rules_pipeline_internal.py", + "tests/recipe/test_rules_project_local_override.py", + "tests/recipe/test_rules_reachability.py", + "tests/recipe/test_rules_recipe.py", + "tests/recipe/test_rules_skill_content.py", + "tests/recipe/test_rules_skills.py", + "tests/recipe/test_rules_structure.py", + "tests/recipe/test_rules_subset_disabled.py", + "tests/recipe/test_rules_tools.py", + "tests/recipe/test_rules_verdict.py", + "tests/recipe/test_rules_worktree.py", + "tests/recipe/test_sprint_sub_recipe.py", + "tests/recipe/test_sub_recipe_validation.py", + "tests/recipe/test_validator.py", + "tests/server/test_service_wrappers.py", + "tests/server/test_smoke_pipeline.py", + "tests/server/test_tools_kitchen_gate.py", + "tests/server/test_tools_kitchen_envelope.py", + "tests/server/test_tools_kitchen_visibility.py", + "tests/server/test_tools_load_recipe.py", + "tests/server/test_tools_recipe.py" + ], + "src/autoskillit/recipe/rules_graph.py": [ + "tests/cli/test_cook.py", + "tests/contracts/test_package_gateways.py", + "tests/recipe/test_api.py", + "tests/recipe/test_bundled_recipe_hidden_policy.py", + "tests/recipe/test_bundled_recipes.py", + "tests/recipe/test_bundled_recipes_no_inversions.py", + "tests/recipe/test_full_audit_recipe.py", + "tests/recipe/test_implementation_sprint_mode.py", + "tests/recipe/test_issue_url_pipeline.py", + "tests/recipe/test_make_campaign_output_schema.py", + "tests/recipe/test_merge_prs_queue.py", + "tests/recipe/test_remediation_sprint_mode.py", + "tests/recipe/test_research_output_mode.py", + "tests/recipe/test_rules_blocks.py", + "tests/recipe/test_rules_bypass.py", + "tests/recipe/test_rules_campaign.py", + "tests/recipe/test_rules_ci.py", + "tests/recipe/test_rules_clone.py", + "tests/recipe/test_rules_cmd.py", + "tests/recipe/test_rules_conditional_push.py", + "tests/recipe/test_rules_contracts.py", + "tests/recipe/test_rules_dataflow.py", + "tests/recipe/test_rules_features.py", + "tests/recipe/test_rules_graph.py", + "tests/recipe/test_rules_inputs.py", + "tests/recipe/test_rules_isolation.py", + "tests/recipe/test_rules_merge.py", + "tests/recipe/test_rules_packs.py", + "tests/recipe/test_rules_pipeline_internal.py", + "tests/recipe/test_rules_project_local_override.py", + "tests/recipe/test_rules_reachability.py", + "tests/recipe/test_rules_recipe.py", + "tests/recipe/test_rules_skill_content.py", + "tests/recipe/test_rules_skills.py", + "tests/recipe/test_rules_structure.py", + "tests/recipe/test_rules_subset_disabled.py", + "tests/recipe/test_rules_tools.py", + "tests/recipe/test_rules_verdict.py", + "tests/recipe/test_rules_worktree.py", + "tests/recipe/test_sprint_sub_recipe.py", + "tests/recipe/test_sub_recipe_validation.py", + "tests/recipe/test_validator.py", + "tests/server/test_service_wrappers.py", + "tests/server/test_smoke_pipeline.py", + "tests/server/test_tools_kitchen_gate.py", + "tests/server/test_tools_kitchen_envelope.py", + "tests/server/test_tools_kitchen_visibility.py", + "tests/server/test_tools_load_recipe.py", + "tests/server/test_tools_recipe.py" + ], + "src/autoskillit/recipe/rules_inputs.py": [ + "tests/cli/test_cook.py", + "tests/contracts/test_package_gateways.py", + "tests/recipe/test_api.py", + "tests/recipe/test_bundled_recipe_hidden_policy.py", + "tests/recipe/test_bundled_recipes.py", + "tests/recipe/test_bundled_recipes_no_inversions.py", + "tests/recipe/test_full_audit_recipe.py", + "tests/recipe/test_implementation_sprint_mode.py", + "tests/recipe/test_issue_url_pipeline.py", + "tests/recipe/test_make_campaign_output_schema.py", + "tests/recipe/test_merge_prs_queue.py", + "tests/recipe/test_remediation_sprint_mode.py", + "tests/recipe/test_research_output_mode.py", + "tests/recipe/test_rules_blocks.py", + "tests/recipe/test_rules_bypass.py", + "tests/recipe/test_rules_campaign.py", + "tests/recipe/test_rules_ci.py", + "tests/recipe/test_rules_clone.py", + "tests/recipe/test_rules_cmd.py", + "tests/recipe/test_rules_conditional_push.py", + "tests/recipe/test_rules_contracts.py", + "tests/recipe/test_rules_dataflow.py", + "tests/recipe/test_rules_features.py", + "tests/recipe/test_rules_graph.py", + "tests/recipe/test_rules_inputs.py", + "tests/recipe/test_rules_isolation.py", + "tests/recipe/test_rules_merge.py", + "tests/recipe/test_rules_packs.py", + "tests/recipe/test_rules_pipeline_internal.py", + "tests/recipe/test_rules_project_local_override.py", + "tests/recipe/test_rules_reachability.py", + "tests/recipe/test_rules_recipe.py", + "tests/recipe/test_rules_skill_content.py", + "tests/recipe/test_rules_skills.py", + "tests/recipe/test_rules_structure.py", + "tests/recipe/test_rules_subset_disabled.py", + "tests/recipe/test_rules_tools.py", + "tests/recipe/test_rules_verdict.py", + "tests/recipe/test_rules_worktree.py", + "tests/recipe/test_sprint_sub_recipe.py", + "tests/recipe/test_sub_recipe_validation.py", + "tests/recipe/test_validator.py", + "tests/server/test_service_wrappers.py", + "tests/server/test_smoke_pipeline.py", + "tests/server/test_tools_kitchen_gate.py", + "tests/server/test_tools_kitchen_envelope.py", + "tests/server/test_tools_kitchen_visibility.py", + "tests/server/test_tools_load_recipe.py", + "tests/server/test_tools_recipe.py" + ], + "src/autoskillit/recipe/rules_isolation.py": [ + "tests/cli/test_cook.py", + "tests/contracts/test_package_gateways.py", + "tests/recipe/test_api.py", + "tests/recipe/test_bundled_recipe_hidden_policy.py", + "tests/recipe/test_bundled_recipes.py", + "tests/recipe/test_bundled_recipes_no_inversions.py", + "tests/recipe/test_full_audit_recipe.py", + "tests/recipe/test_implementation_sprint_mode.py", + "tests/recipe/test_issue_url_pipeline.py", + "tests/recipe/test_make_campaign_output_schema.py", + "tests/recipe/test_merge_prs_queue.py", + "tests/recipe/test_remediation_sprint_mode.py", + "tests/recipe/test_research_output_mode.py", + "tests/recipe/test_rules_blocks.py", + "tests/recipe/test_rules_bypass.py", + "tests/recipe/test_rules_campaign.py", + "tests/recipe/test_rules_ci.py", + "tests/recipe/test_rules_clone.py", + "tests/recipe/test_rules_cmd.py", + "tests/recipe/test_rules_conditional_push.py", + "tests/recipe/test_rules_contracts.py", + "tests/recipe/test_rules_dataflow.py", + "tests/recipe/test_rules_features.py", + "tests/recipe/test_rules_graph.py", + "tests/recipe/test_rules_inputs.py", + "tests/recipe/test_rules_isolation.py", + "tests/recipe/test_rules_merge.py", + "tests/recipe/test_rules_packs.py", + "tests/recipe/test_rules_pipeline_internal.py", + "tests/recipe/test_rules_project_local_override.py", + "tests/recipe/test_rules_reachability.py", + "tests/recipe/test_rules_recipe.py", + "tests/recipe/test_rules_skill_content.py", + "tests/recipe/test_rules_skills.py", + "tests/recipe/test_rules_structure.py", + "tests/recipe/test_rules_subset_disabled.py", + "tests/recipe/test_rules_tools.py", + "tests/recipe/test_rules_verdict.py", + "tests/recipe/test_rules_worktree.py", + "tests/recipe/test_sprint_sub_recipe.py", + "tests/recipe/test_sub_recipe_validation.py", + "tests/recipe/test_validator.py", + "tests/server/test_service_wrappers.py", + "tests/server/test_smoke_pipeline.py", + "tests/server/test_tools_kitchen_gate.py", + "tests/server/test_tools_kitchen_envelope.py", + "tests/server/test_tools_kitchen_visibility.py", + "tests/server/test_tools_load_recipe.py", + "tests/server/test_tools_recipe.py" + ], + "src/autoskillit/recipe/rules_merge.py": [ + "tests/cli/test_cook.py", + "tests/contracts/test_package_gateways.py", + "tests/recipe/test_api.py", + "tests/recipe/test_bundled_recipe_hidden_policy.py", + "tests/recipe/test_bundled_recipes.py", + "tests/recipe/test_bundled_recipes_no_inversions.py", + "tests/recipe/test_full_audit_recipe.py", + "tests/recipe/test_implementation_sprint_mode.py", + "tests/recipe/test_issue_url_pipeline.py", + "tests/recipe/test_make_campaign_output_schema.py", + "tests/recipe/test_merge_prs_queue.py", + "tests/recipe/test_remediation_sprint_mode.py", + "tests/recipe/test_research_output_mode.py", + "tests/recipe/test_rules_blocks.py", + "tests/recipe/test_rules_bypass.py", + "tests/recipe/test_rules_campaign.py", + "tests/recipe/test_rules_ci.py", + "tests/recipe/test_rules_clone.py", + "tests/recipe/test_rules_cmd.py", + "tests/recipe/test_rules_conditional_push.py", + "tests/recipe/test_rules_contracts.py", + "tests/recipe/test_rules_dataflow.py", + "tests/recipe/test_rules_features.py", + "tests/recipe/test_rules_graph.py", + "tests/recipe/test_rules_inputs.py", + "tests/recipe/test_rules_isolation.py", + "tests/recipe/test_rules_merge.py", + "tests/recipe/test_rules_packs.py", + "tests/recipe/test_rules_pipeline_internal.py", + "tests/recipe/test_rules_project_local_override.py", + "tests/recipe/test_rules_reachability.py", + "tests/recipe/test_rules_recipe.py", + "tests/recipe/test_rules_skill_content.py", + "tests/recipe/test_rules_skills.py", + "tests/recipe/test_rules_structure.py", + "tests/recipe/test_rules_subset_disabled.py", + "tests/recipe/test_rules_tools.py", + "tests/recipe/test_rules_verdict.py", + "tests/recipe/test_rules_worktree.py", + "tests/recipe/test_sprint_sub_recipe.py", + "tests/recipe/test_sub_recipe_validation.py", + "tests/recipe/test_validator.py", + "tests/server/test_service_wrappers.py", + "tests/server/test_smoke_pipeline.py", + "tests/server/test_tools_kitchen_gate.py", + "tests/server/test_tools_kitchen_envelope.py", + "tests/server/test_tools_kitchen_visibility.py", + "tests/server/test_tools_load_recipe.py", + "tests/server/test_tools_recipe.py" + ], + "src/autoskillit/recipe/rules_packs.py": [ + "tests/cli/test_cook.py", + "tests/contracts/test_package_gateways.py", + "tests/recipe/test_api.py", + "tests/recipe/test_bundled_recipe_hidden_policy.py", + "tests/recipe/test_bundled_recipes.py", + "tests/recipe/test_bundled_recipes_no_inversions.py", + "tests/recipe/test_full_audit_recipe.py", + "tests/recipe/test_implementation_sprint_mode.py", + "tests/recipe/test_issue_url_pipeline.py", + "tests/recipe/test_make_campaign_output_schema.py", + "tests/recipe/test_merge_prs_queue.py", + "tests/recipe/test_remediation_sprint_mode.py", + "tests/recipe/test_research_output_mode.py", + "tests/recipe/test_rules_blocks.py", + "tests/recipe/test_rules_bypass.py", + "tests/recipe/test_rules_campaign.py", + "tests/recipe/test_rules_ci.py", + "tests/recipe/test_rules_clone.py", + "tests/recipe/test_rules_cmd.py", + "tests/recipe/test_rules_conditional_push.py", + "tests/recipe/test_rules_contracts.py", + "tests/recipe/test_rules_dataflow.py", + "tests/recipe/test_rules_features.py", + "tests/recipe/test_rules_graph.py", + "tests/recipe/test_rules_inputs.py", + "tests/recipe/test_rules_isolation.py", + "tests/recipe/test_rules_merge.py", + "tests/recipe/test_rules_packs.py", + "tests/recipe/test_rules_pipeline_internal.py", + "tests/recipe/test_rules_project_local_override.py", + "tests/recipe/test_rules_reachability.py", + "tests/recipe/test_rules_recipe.py", + "tests/recipe/test_rules_skill_content.py", + "tests/recipe/test_rules_skills.py", + "tests/recipe/test_rules_structure.py", + "tests/recipe/test_rules_subset_disabled.py", + "tests/recipe/test_rules_tools.py", + "tests/recipe/test_rules_verdict.py", + "tests/recipe/test_rules_worktree.py", + "tests/recipe/test_sprint_sub_recipe.py", + "tests/recipe/test_sub_recipe_validation.py", + "tests/recipe/test_validator.py", + "tests/server/test_service_wrappers.py", + "tests/server/test_smoke_pipeline.py", + "tests/server/test_tools_kitchen_gate.py", + "tests/server/test_tools_kitchen_envelope.py", + "tests/server/test_tools_kitchen_visibility.py", + "tests/server/test_tools_load_recipe.py", + "tests/server/test_tools_recipe.py" + ], + "src/autoskillit/recipe/rules_reachability.py": [ + "tests/cli/test_cook.py", + "tests/contracts/test_package_gateways.py", + "tests/recipe/test_api.py", + "tests/recipe/test_bundled_recipe_hidden_policy.py", + "tests/recipe/test_bundled_recipes.py", + "tests/recipe/test_bundled_recipes_no_inversions.py", + "tests/recipe/test_full_audit_recipe.py", + "tests/recipe/test_implementation_sprint_mode.py", + "tests/recipe/test_issue_url_pipeline.py", + "tests/recipe/test_make_campaign_output_schema.py", + "tests/recipe/test_merge_prs_queue.py", + "tests/recipe/test_remediation_sprint_mode.py", + "tests/recipe/test_research_output_mode.py", + "tests/recipe/test_rules_blocks.py", + "tests/recipe/test_rules_bypass.py", + "tests/recipe/test_rules_campaign.py", + "tests/recipe/test_rules_ci.py", + "tests/recipe/test_rules_clone.py", + "tests/recipe/test_rules_cmd.py", + "tests/recipe/test_rules_conditional_push.py", + "tests/recipe/test_rules_contracts.py", + "tests/recipe/test_rules_dataflow.py", + "tests/recipe/test_rules_features.py", + "tests/recipe/test_rules_graph.py", + "tests/recipe/test_rules_inputs.py", + "tests/recipe/test_rules_isolation.py", + "tests/recipe/test_rules_merge.py", + "tests/recipe/test_rules_packs.py", + "tests/recipe/test_rules_pipeline_internal.py", + "tests/recipe/test_rules_project_local_override.py", + "tests/recipe/test_rules_reachability.py", + "tests/recipe/test_rules_recipe.py", + "tests/recipe/test_rules_skill_content.py", + "tests/recipe/test_rules_skills.py", + "tests/recipe/test_rules_structure.py", + "tests/recipe/test_rules_subset_disabled.py", + "tests/recipe/test_rules_tools.py", + "tests/recipe/test_rules_verdict.py", + "tests/recipe/test_rules_worktree.py", + "tests/recipe/test_sprint_sub_recipe.py", + "tests/recipe/test_sub_recipe_validation.py", + "tests/recipe/test_validator.py", + "tests/server/test_service_wrappers.py", + "tests/server/test_smoke_pipeline.py", + "tests/server/test_tools_kitchen_gate.py", + "tests/server/test_tools_kitchen_envelope.py", + "tests/server/test_tools_kitchen_visibility.py", + "tests/server/test_tools_load_recipe.py", + "tests/server/test_tools_recipe.py" + ], + "src/autoskillit/recipe/rules_recipe.py": [ + "tests/cli/test_cook.py", + "tests/contracts/test_package_gateways.py", + "tests/recipe/test_api.py", + "tests/recipe/test_bundled_recipe_hidden_policy.py", + "tests/recipe/test_bundled_recipes.py", + "tests/recipe/test_bundled_recipes_no_inversions.py", + "tests/recipe/test_full_audit_recipe.py", + "tests/recipe/test_implementation_sprint_mode.py", + "tests/recipe/test_issue_url_pipeline.py", + "tests/recipe/test_make_campaign_output_schema.py", + "tests/recipe/test_merge_prs_queue.py", + "tests/recipe/test_remediation_sprint_mode.py", + "tests/recipe/test_research_output_mode.py", + "tests/recipe/test_rules_blocks.py", + "tests/recipe/test_rules_bypass.py", + "tests/recipe/test_rules_campaign.py", + "tests/recipe/test_rules_ci.py", + "tests/recipe/test_rules_clone.py", + "tests/recipe/test_rules_cmd.py", + "tests/recipe/test_rules_conditional_push.py", + "tests/recipe/test_rules_contracts.py", + "tests/recipe/test_rules_dataflow.py", + "tests/recipe/test_rules_features.py", + "tests/recipe/test_rules_graph.py", + "tests/recipe/test_rules_inputs.py", + "tests/recipe/test_rules_isolation.py", + "tests/recipe/test_rules_merge.py", + "tests/recipe/test_rules_packs.py", + "tests/recipe/test_rules_pipeline_internal.py", + "tests/recipe/test_rules_project_local_override.py", + "tests/recipe/test_rules_reachability.py", + "tests/recipe/test_rules_recipe.py", + "tests/recipe/test_rules_skill_content.py", + "tests/recipe/test_rules_skills.py", + "tests/recipe/test_rules_structure.py", + "tests/recipe/test_rules_subset_disabled.py", + "tests/recipe/test_rules_tools.py", + "tests/recipe/test_rules_verdict.py", + "tests/recipe/test_rules_worktree.py", + "tests/recipe/test_sprint_sub_recipe.py", + "tests/recipe/test_sub_recipe_validation.py", + "tests/recipe/test_validator.py", + "tests/server/test_service_wrappers.py", + "tests/server/test_smoke_pipeline.py", + "tests/server/test_tools_kitchen_gate.py", + "tests/server/test_tools_kitchen_envelope.py", + "tests/server/test_tools_kitchen_visibility.py", + "tests/server/test_tools_load_recipe.py", + "tests/server/test_tools_recipe.py" + ], + "src/autoskillit/recipe/rules_skill_content.py": [ + "tests/cli/test_cook.py", + "tests/contracts/test_package_gateways.py", + "tests/recipe/test_api.py", + "tests/recipe/test_bundled_recipe_hidden_policy.py", + "tests/recipe/test_bundled_recipes.py", + "tests/recipe/test_bundled_recipes_no_inversions.py", + "tests/recipe/test_full_audit_recipe.py", + "tests/recipe/test_implementation_sprint_mode.py", + "tests/recipe/test_issue_url_pipeline.py", + "tests/recipe/test_make_campaign_output_schema.py", + "tests/recipe/test_merge_prs_queue.py", + "tests/recipe/test_remediation_sprint_mode.py", + "tests/recipe/test_research_output_mode.py", + "tests/recipe/test_rules_blocks.py", + "tests/recipe/test_rules_bypass.py", + "tests/recipe/test_rules_campaign.py", + "tests/recipe/test_rules_ci.py", + "tests/recipe/test_rules_clone.py", + "tests/recipe/test_rules_cmd.py", + "tests/recipe/test_rules_conditional_push.py", + "tests/recipe/test_rules_contracts.py", + "tests/recipe/test_rules_dataflow.py", + "tests/recipe/test_rules_features.py", + "tests/recipe/test_rules_graph.py", + "tests/recipe/test_rules_inputs.py", + "tests/recipe/test_rules_isolation.py", + "tests/recipe/test_rules_merge.py", + "tests/recipe/test_rules_packs.py", + "tests/recipe/test_rules_pipeline_internal.py", + "tests/recipe/test_rules_project_local_override.py", + "tests/recipe/test_rules_reachability.py", + "tests/recipe/test_rules_recipe.py", + "tests/recipe/test_rules_skill_content.py", + "tests/recipe/test_rules_skills.py", + "tests/recipe/test_rules_structure.py", + "tests/recipe/test_rules_subset_disabled.py", + "tests/recipe/test_rules_tools.py", + "tests/recipe/test_rules_verdict.py", + "tests/recipe/test_rules_worktree.py", + "tests/recipe/test_sprint_sub_recipe.py", + "tests/recipe/test_sub_recipe_validation.py", + "tests/recipe/test_validator.py", + "tests/server/test_service_wrappers.py", + "tests/server/test_smoke_pipeline.py", + "tests/server/test_tools_kitchen_gate.py", + "tests/server/test_tools_kitchen_envelope.py", + "tests/server/test_tools_kitchen_visibility.py", + "tests/server/test_tools_load_recipe.py", + "tests/server/test_tools_recipe.py" + ], + "src/autoskillit/recipe/rules_skills.py": [ + "tests/cli/test_cook.py", + "tests/contracts/test_package_gateways.py", + "tests/recipe/test_api.py", + "tests/recipe/test_bundled_recipe_hidden_policy.py", + "tests/recipe/test_bundled_recipes.py", + "tests/recipe/test_bundled_recipes_no_inversions.py", + "tests/recipe/test_full_audit_recipe.py", + "tests/recipe/test_implementation_sprint_mode.py", + "tests/recipe/test_issue_url_pipeline.py", + "tests/recipe/test_make_campaign_output_schema.py", + "tests/recipe/test_merge_prs_queue.py", + "tests/recipe/test_remediation_sprint_mode.py", + "tests/recipe/test_research_output_mode.py", + "tests/recipe/test_rules_blocks.py", + "tests/recipe/test_rules_bypass.py", + "tests/recipe/test_rules_campaign.py", + "tests/recipe/test_rules_ci.py", + "tests/recipe/test_rules_clone.py", + "tests/recipe/test_rules_cmd.py", + "tests/recipe/test_rules_conditional_push.py", + "tests/recipe/test_rules_contracts.py", + "tests/recipe/test_rules_dataflow.py", + "tests/recipe/test_rules_features.py", + "tests/recipe/test_rules_graph.py", + "tests/recipe/test_rules_inputs.py", + "tests/recipe/test_rules_isolation.py", + "tests/recipe/test_rules_merge.py", + "tests/recipe/test_rules_packs.py", + "tests/recipe/test_rules_pipeline_internal.py", + "tests/recipe/test_rules_project_local_override.py", + "tests/recipe/test_rules_reachability.py", + "tests/recipe/test_rules_recipe.py", + "tests/recipe/test_rules_skill_content.py", + "tests/recipe/test_rules_skills.py", + "tests/recipe/test_rules_structure.py", + "tests/recipe/test_rules_subset_disabled.py", + "tests/recipe/test_rules_tools.py", + "tests/recipe/test_rules_verdict.py", + "tests/recipe/test_rules_worktree.py", + "tests/recipe/test_sprint_sub_recipe.py", + "tests/recipe/test_sub_recipe_validation.py", + "tests/recipe/test_validator.py", + "tests/server/test_service_wrappers.py", + "tests/server/test_smoke_pipeline.py", + "tests/server/test_tools_kitchen_gate.py", + "tests/server/test_tools_kitchen_envelope.py", + "tests/server/test_tools_kitchen_visibility.py", + "tests/server/test_tools_load_recipe.py", + "tests/server/test_tools_recipe.py" + ], + "src/autoskillit/recipe/rules_tools.py": [ + "tests/cli/test_cook.py", + "tests/contracts/test_package_gateways.py", + "tests/recipe/test_api.py", + "tests/recipe/test_bundled_recipe_hidden_policy.py", + "tests/recipe/test_bundled_recipes.py", + "tests/recipe/test_bundled_recipes_no_inversions.py", + "tests/recipe/test_full_audit_recipe.py", + "tests/recipe/test_implementation_sprint_mode.py", + "tests/recipe/test_issue_url_pipeline.py", + "tests/recipe/test_make_campaign_output_schema.py", + "tests/recipe/test_merge_prs_queue.py", + "tests/recipe/test_remediation_sprint_mode.py", + "tests/recipe/test_research_output_mode.py", + "tests/recipe/test_rules_blocks.py", + "tests/recipe/test_rules_bypass.py", + "tests/recipe/test_rules_campaign.py", + "tests/recipe/test_rules_ci.py", + "tests/recipe/test_rules_clone.py", + "tests/recipe/test_rules_cmd.py", + "tests/recipe/test_rules_conditional_push.py", + "tests/recipe/test_rules_contracts.py", + "tests/recipe/test_rules_dataflow.py", + "tests/recipe/test_rules_features.py", + "tests/recipe/test_rules_graph.py", + "tests/recipe/test_rules_inputs.py", + "tests/recipe/test_rules_isolation.py", + "tests/recipe/test_rules_merge.py", + "tests/recipe/test_rules_packs.py", + "tests/recipe/test_rules_pipeline_internal.py", + "tests/recipe/test_rules_project_local_override.py", + "tests/recipe/test_rules_reachability.py", + "tests/recipe/test_rules_recipe.py", + "tests/recipe/test_rules_skill_content.py", + "tests/recipe/test_rules_skills.py", + "tests/recipe/test_rules_structure.py", + "tests/recipe/test_rules_subset_disabled.py", + "tests/recipe/test_rules_tools.py", + "tests/recipe/test_rules_verdict.py", + "tests/recipe/test_rules_worktree.py", + "tests/recipe/test_sprint_sub_recipe.py", + "tests/recipe/test_sub_recipe_validation.py", + "tests/recipe/test_validator.py", + "tests/server/test_service_wrappers.py", + "tests/server/test_smoke_pipeline.py", + "tests/server/test_tools_kitchen_gate.py", + "tests/server/test_tools_kitchen_envelope.py", + "tests/server/test_tools_kitchen_visibility.py", + "tests/server/test_tools_load_recipe.py", + "tests/server/test_tools_recipe.py" + ], + "src/autoskillit/recipe/rules_verdict.py": [ + "tests/cli/test_cook.py", + "tests/contracts/test_package_gateways.py", + "tests/recipe/test_api.py", + "tests/recipe/test_bundled_recipe_hidden_policy.py", + "tests/recipe/test_bundled_recipes.py", + "tests/recipe/test_bundled_recipes_no_inversions.py", + "tests/recipe/test_full_audit_recipe.py", + "tests/recipe/test_implementation_sprint_mode.py", + "tests/recipe/test_issue_url_pipeline.py", + "tests/recipe/test_make_campaign_output_schema.py", + "tests/recipe/test_merge_prs_queue.py", + "tests/recipe/test_remediation_sprint_mode.py", + "tests/recipe/test_research_output_mode.py", + "tests/recipe/test_resolve_ci_routing_invariant.py", + "tests/recipe/test_rules_blocks.py", + "tests/recipe/test_rules_bypass.py", + "tests/recipe/test_rules_campaign.py", + "tests/recipe/test_rules_ci.py", + "tests/recipe/test_rules_clone.py", + "tests/recipe/test_rules_cmd.py", + "tests/recipe/test_rules_conditional_push.py", + "tests/recipe/test_rules_contracts.py", + "tests/recipe/test_rules_dataflow.py", + "tests/recipe/test_rules_features.py", + "tests/recipe/test_rules_graph.py", + "tests/recipe/test_rules_inputs.py", + "tests/recipe/test_rules_isolation.py", + "tests/recipe/test_rules_merge.py", + "tests/recipe/test_rules_packs.py", + "tests/recipe/test_rules_pipeline_internal.py", + "tests/recipe/test_rules_project_local_override.py", + "tests/recipe/test_rules_reachability.py", + "tests/recipe/test_rules_recipe.py", + "tests/recipe/test_rules_skill_content.py", + "tests/recipe/test_rules_skills.py", + "tests/recipe/test_rules_structure.py", + "tests/recipe/test_rules_subset_disabled.py", + "tests/recipe/test_rules_tools.py", + "tests/recipe/test_rules_verdict.py", + "tests/recipe/test_rules_worktree.py", + "tests/recipe/test_sprint_sub_recipe.py", + "tests/recipe/test_sub_recipe_validation.py", + "tests/recipe/test_validator.py", + "tests/server/test_service_wrappers.py", + "tests/server/test_smoke_pipeline.py", + "tests/server/test_tools_kitchen_gate.py", + "tests/server/test_tools_kitchen_envelope.py", + "tests/server/test_tools_kitchen_visibility.py", + "tests/server/test_tools_load_recipe.py", + "tests/server/test_tools_recipe.py" + ], + "src/autoskillit/recipe/rules_worktree.py": [ + "tests/cli/test_cook.py", + "tests/contracts/test_package_gateways.py", + "tests/recipe/test_api.py", + "tests/recipe/test_bundled_recipe_hidden_policy.py", + "tests/recipe/test_bundled_recipes.py", + "tests/recipe/test_bundled_recipes_no_inversions.py", + "tests/recipe/test_full_audit_recipe.py", + "tests/recipe/test_implementation_sprint_mode.py", + "tests/recipe/test_issue_url_pipeline.py", + "tests/recipe/test_make_campaign_output_schema.py", + "tests/recipe/test_merge_prs_queue.py", + "tests/recipe/test_remediation_sprint_mode.py", + "tests/recipe/test_research_output_mode.py", + "tests/recipe/test_rules_blocks.py", + "tests/recipe/test_rules_bypass.py", + "tests/recipe/test_rules_campaign.py", + "tests/recipe/test_rules_ci.py", + "tests/recipe/test_rules_clone.py", + "tests/recipe/test_rules_cmd.py", + "tests/recipe/test_rules_conditional_push.py", + "tests/recipe/test_rules_contracts.py", + "tests/recipe/test_rules_dataflow.py", + "tests/recipe/test_rules_features.py", + "tests/recipe/test_rules_graph.py", + "tests/recipe/test_rules_inputs.py", + "tests/recipe/test_rules_isolation.py", + "tests/recipe/test_rules_merge.py", + "tests/recipe/test_rules_packs.py", + "tests/recipe/test_rules_pipeline_internal.py", + "tests/recipe/test_rules_project_local_override.py", + "tests/recipe/test_rules_reachability.py", + "tests/recipe/test_rules_recipe.py", + "tests/recipe/test_rules_skill_content.py", + "tests/recipe/test_rules_skills.py", + "tests/recipe/test_rules_structure.py", + "tests/recipe/test_rules_subset_disabled.py", + "tests/recipe/test_rules_tools.py", + "tests/recipe/test_rules_verdict.py", + "tests/recipe/test_rules_worktree.py", + "tests/recipe/test_sprint_sub_recipe.py", + "tests/recipe/test_sub_recipe_validation.py", + "tests/recipe/test_validator.py", + "tests/server/test_service_wrappers.py", + "tests/server/test_smoke_pipeline.py", + "tests/server/test_tools_kitchen_gate.py", + "tests/server/test_tools_kitchen_envelope.py", + "tests/server/test_tools_kitchen_visibility.py", + "tests/server/test_tools_load_recipe.py", + "tests/server/test_tools_recipe.py" + ], + "src/autoskillit/recipe/schema.py": [ + "tests/cli/test_cli_prompts.py", + "tests/cli/test_cook.py", + "tests/cli/test_doctor.py", + "tests/cli/test_franchise_cli.py", + "tests/cli/test_input_tty_contracts.py", + "tests/cli/test_install.py", + "tests/cli/test_l3_orchestrator_prompt.py", + "tests/contracts/test_instruction_surface.py", + "tests/contracts/test_package_gateways.py", + "tests/contracts/test_skill_yaml_validation.py", + "tests/contracts/test_target_skill_invocability.py", + "tests/franchise/test_dispatch_failure_semantics.py", + "tests/franchise/test_dispatch_lifespan.py", + "tests/franchise/test_franchise_e2e.py", + "tests/franchise/test_pack_enforcement.py", + "tests/franchise/test_pack_enforcement_e2e.py", + "tests/infra/test_pretty_output_integration.py", + "tests/migration/test_engine.py", + "tests/recipe/test__api.py", + "tests/recipe/test_anti_pattern_guards.py", + "tests/recipe/test_api.py", + "tests/recipe/test_bundled_recipe_hidden_policy.py", + "tests/recipe/test_bundled_recipes.py", + "tests/recipe/test_bundled_recipes_no_inversions.py", + "tests/recipe/test_campaign_loader.py", + "tests/recipe/test_check_repo_merge_state_routing.py", + "tests/recipe/test_contracts.py", + "tests/recipe/test_contracts_block_fingerprint.py", + "tests/recipe/test_diagrams.py", + "tests/recipe/test_full_audit_recipe.py", + "tests/recipe/test_hidden_ingredients.py", + "tests/recipe/test_identity.py", + "tests/recipe/test_implementation_sprint_mode.py", + "tests/recipe/test_io.py", + "tests/recipe/test_issue_url_pipeline.py", + "tests/recipe/test_make_campaign_output_schema.py", + "tests/recipe/test_merge_prs_queue.py", + "tests/recipe/test_merge_sub_recipe_hidden.py", + "tests/recipe/test_recipe_temp_substitution.py", + "tests/recipe/test_remediation_sprint_mode.py", + "tests/recipe/test_repository.py", + "tests/recipe/test_research_output_mode.py", + "tests/recipe/test_resolve_ci_routing_invariant.py", + "tests/recipe/test_review_loop_routing_integration.py", + "tests/recipe/test_rules_blocks.py", + "tests/recipe/test_rules_bypass.py", + "tests/recipe/test_rules_campaign.py", + "tests/recipe/test_rules_ci.py", + "tests/recipe/test_rules_clone.py", + "tests/recipe/test_rules_cmd.py", + "tests/recipe/test_rules_conditional_push.py", + "tests/recipe/test_rules_contracts.py", + "tests/recipe/test_rules_dataflow.py", + "tests/recipe/test_rules_features.py", + "tests/recipe/test_rules_graph.py", + "tests/recipe/test_rules_inputs.py", + "tests/recipe/test_rules_isolation.py", + "tests/recipe/test_rules_merge.py", + "tests/recipe/test_rules_packs.py", + "tests/recipe/test_rules_pipeline_internal.py", + "tests/recipe/test_rules_project_local_override.py", + "tests/recipe/test_rules_reachability.py", + "tests/recipe/test_rules_recipe.py", + "tests/recipe/test_rules_skill_content.py", + "tests/recipe/test_rules_skills.py", + "tests/recipe/test_rules_structure.py", + "tests/recipe/test_rules_subset_disabled.py", + "tests/recipe/test_rules_tools.py", + "tests/recipe/test_rules_verdict.py", + "tests/recipe/test_rules_worktree.py", + "tests/recipe/test_schema.py", + "tests/recipe/test_sub_recipe_loading.py", + "tests/recipe/test_sub_recipe_schema.py", + "tests/recipe/test_sub_recipe_validation.py", + "tests/recipe/test_validator.py", + "tests/server/test_mcp_overrides.py", + "tests/server/test_server_tool_registration.py", + "tests/server/test_service_wrappers.py", + "tests/server/test_smoke_pipeline.py", + "tests/server/test_tools_dispatch.py", + "tests/server/test_tools_kitchen_gate.py", + "tests/server/test_tools_kitchen_envelope.py", + "tests/server/test_tools_kitchen_visibility.py", + "tests/server/test_tools_list_recipes.py", + "tests/server/test_tools_load_recipe.py", + "tests/server/test_tools_recipe.py" + ], + "src/autoskillit/recipe/staleness_cache.py": [ + "tests/cli/test_cli_prompts.py", + "tests/cli/test_cook.py", + "tests/cli/test_doctor.py", + "tests/cli/test_franchise_cli.py", + "tests/cli/test_input_tty_contracts.py", + "tests/cli/test_install.py", + "tests/contracts/test_instruction_surface.py", + "tests/contracts/test_package_gateways.py", + "tests/contracts/test_target_skill_invocability.py", + "tests/franchise/test_pack_enforcement.py", + "tests/franchise/test_pack_enforcement_e2e.py", + "tests/infra/test_pretty_output_integration.py", + "tests/migration/test_engine.py", + "tests/recipe/test_anti_pattern_guards.py", + "tests/recipe/test_api.py", + "tests/recipe/test_bundled_recipe_hidden_policy.py", + "tests/recipe/test_bundled_recipes.py", + "tests/recipe/test_bundled_recipes_no_inversions.py", + "tests/recipe/test_campaign_loader.py", + "tests/recipe/test_check_repo_merge_state_routing.py", + "tests/recipe/test_contracts.py", + "tests/recipe/test_contracts_block_fingerprint.py", + "tests/recipe/test_diagrams.py", + "tests/recipe/test_full_audit_recipe.py", + "tests/recipe/test_identity.py", + "tests/recipe/test_implementation_sprint_mode.py", + "tests/recipe/test_io.py", + "tests/recipe/test_issue_url_pipeline.py", + "tests/recipe/test_make_campaign_output_schema.py", + "tests/recipe/test_merge_prs_queue.py", + "tests/recipe/test_recipe_temp_substitution.py", + "tests/recipe/test_remediation_sprint_mode.py", + "tests/recipe/test_resolve_ci_routing_invariant.py", + "tests/recipe/test_review_loop_routing_integration.py", + "tests/recipe/test_rules_blocks.py", + "tests/recipe/test_rules_campaign.py", + "tests/recipe/test_rules_ci.py", + "tests/recipe/test_rules_clone.py", + "tests/recipe/test_rules_contracts.py", + "tests/recipe/test_rules_dataflow.py", + "tests/recipe/test_rules_isolation.py", + "tests/recipe/test_rules_merge.py", + "tests/recipe/test_rules_reachability.py", + "tests/recipe/test_rules_recipe.py", + "tests/recipe/test_rules_skill_content.py", + "tests/recipe/test_rules_skills.py", + "tests/recipe/test_rules_structure.py", + "tests/recipe/test_rules_verdict.py", + "tests/recipe/test_rules_worktree.py", + "tests/recipe/test_schema.py", + "tests/recipe/test_staleness_cache.py", + "tests/recipe/test_sub_recipe_loading.py", + "tests/recipe/test_sub_recipe_validation.py", + "tests/recipe/test_validator.py", + "tests/server/test_server_tool_registration.py", + "tests/server/test_service_wrappers.py", + "tests/server/test_smoke_pipeline.py", + "tests/server/test_tools_kitchen_gate.py", + "tests/server/test_tools_kitchen_envelope.py", + "tests/server/test_tools_kitchen_visibility.py", + "tests/server/test_tools_list_recipes.py", + "tests/server/test_tools_load_recipe.py" + ], + "src/autoskillit/recipe/validator.py": [ + "tests/cli/test_cook.py", + "tests/contracts/test_package_gateways.py", + "tests/contracts/test_skill_yaml_validation.py", + "tests/migration/test_engine.py", + "tests/recipe/test_api.py", + "tests/recipe/test_bundled_recipes.py", + "tests/recipe/test_campaign_loader.py", + "tests/recipe/test_full_audit_recipe.py", + "tests/recipe/test_implementation_pr_decomposition.py", + "tests/recipe/test_implementation_sprint_mode.py", + "tests/recipe/test_issue_url_pipeline.py", + "tests/recipe/test_merge_prs_queue.py", + "tests/recipe/test_remediation_pr_decomposition.py", + "tests/recipe/test_remediation_recipe.py", + "tests/recipe/test_remediation_sprint_mode.py", + "tests/recipe/test_research_recipe_diag.py", + "tests/recipe/test_research_stage_data_step.py", + "tests/recipe/test_rules_structure.py", + "tests/recipe/test_sprint_sub_recipe.py", + "tests/recipe/test_sub_recipe_schema.py", + "tests/recipe/test_sub_recipe_validation.py", + "tests/recipe/test_validator.py", + "tests/server/test_service_wrappers.py", + "tests/server/test_smoke_pipeline.py", + "tests/server/test_tools_kitchen_gate.py", + "tests/server/test_tools_kitchen_envelope.py", + "tests/server/test_tools_kitchen_visibility.py", + "tests/server/test_tools_load_recipe.py", + "tests/server/test_tools_recipe.py" + ], + "src/autoskillit/server/__init__.py": [ + "tests/server/test_server_version_telemetry.py" + ], + "src/autoskillit/server/_editable_guard.py": [ + "tests/server/test_editable_guard.py", + "tests/server/test_git.py", + "tests/server/test_server_tool_registration.py", + "tests/server/test_tools_git.py" + ], + "src/autoskillit/server/_factory.py": [ + "tests/cli/test_cli_serve_logging.py", + "tests/cli/test_init.py", + "tests/cli/test_startup_budget.py", + "tests/contracts/test_package_gateways.py", + "tests/infra/test_pretty_output_integration.py", + "tests/server/test_factory.py", + "tests/server/test_factory_recording.py", + "tests/server/test_run_skill_add_dirs.py", + "tests/server/test_server_tool_registration.py", + "tests/server/test_tools_ci.py", + "tests/server/test_tools_execution_command.py", + "tests/server/test_tools_execution_input_gates.py", + "tests/server/test_tools_execution_results.py", + "tests/server/test_tools_execution_routing.py", + "tests/server/test_tools_execution_response.py", + "tests/server/test_tools_github.py", + "tests/server/test_tools_integrations.py", + "tests/server/test_tools_issue_lifecycle.py", + "tests/server/test_tools_label_validation.py", + "tests/server/test_tools_run_skill_retry.py", + "tests/server/test_tools_session_diagnostics.py", + "tests/server/test_tools_status_kitchen.py", + "tests/server/test_tools_status_summaries.py", + "tests/server/test_tools_status_quota_and_db.py" + ], + "src/autoskillit/server/_lifespan.py": [ + "tests/arch/test_feature_markers.py", + "tests/franchise/test_pack_enforcement.py", + "tests/server/test_headless_session.py", + "tests/server/test_lifespan.py", + "tests/server/test_server_init_gate.py", + "tests/server/test_server_init_session_visibility.py", + "tests/server/test_server_tool_registration.py", + "tests/server/test_tool_annotation_completeness.py", + "tests/server/test_tools_recipe.py" + ], + "src/autoskillit/server/_session_type.py": [ + "tests/arch/test_feature_markers.py", + "tests/franchise/test_pack_enforcement.py", + "tests/server/test_server_init_gate.py", + "tests/server/test_server_init_session_visibility.py", + "tests/server/test_server_version_telemetry.py" + ], + "src/autoskillit/server/_state.py": [ + "tests/arch/test_feature_markers.py", + "tests/cli/test_cli_serve_logging.py", + "tests/cli/test_init.py", + "tests/cli/test_startup_budget.py", + "tests/contracts/test_instruction_surface.py", + "tests/franchise/test_pack_enforcement.py", + "tests/infra/test_pretty_output_integration.py", + "tests/server/test_headless_session.py", + "tests/server/test_helpers_gate.py", + "tests/server/test_lifespan.py", + "tests/server/test_mcp_overrides.py", + "tests/server/test_run_skill_add_dirs.py", + "tests/server/test_server_init_gate.py", + "tests/server/test_server_init_session_visibility.py", + "tests/server/test_server_tool_registration.py", + "tests/server/test_server_version_telemetry.py", + "tests/server/test_set_commit_status.py", + "tests/server/test_smoke_pipeline.py", + "tests/server/test_state.py", + "tests/server/test_tool_annotation_completeness.py", + "tests/server/test_tools_ci.py", + "tests/server/test_tools_ci_enqueue.py", + "tests/server/test_tools_clone.py", + "tests/server/test_tools_dispatch.py", + "tests/server/test_tools_execution_command.py", + "tests/server/test_tools_execution_input_gates.py", + "tests/server/test_tools_execution_results.py", + "tests/server/test_tools_execution_routing.py", + "tests/server/test_tools_execution_response.py", + "tests/server/test_tools_git.py", + "tests/server/test_tools_github.py", + "tests/server/test_tools_integrations.py", + "tests/server/test_tools_integrations_release.py", + "tests/server/test_tools_issue_lifecycle.py", + "tests/server/test_tools_kitchen_gate.py", + "tests/server/test_tools_kitchen_envelope.py", + "tests/server/test_tools_kitchen_visibility.py", + "tests/server/test_tools_label_validation.py", + "tests/server/test_tools_list_recipes.py", + "tests/server/test_tools_load_recipe.py", + "tests/server/test_tools_pr_ops.py", + "tests/server/test_tools_recipe.py", + "tests/server/test_tools_run_cmd.py", + "tests/server/test_tools_run_cmd_unit.py", + "tests/server/test_tools_run_python.py", + "tests/server/test_tools_run_skill_retry.py", + "tests/server/test_tools_session_diagnostics.py", + "tests/server/test_tools_status_kitchen.py", + "tests/server/test_tools_status_summaries.py", + "tests/server/test_tools_status_quota_and_db.py", + "tests/server/test_tools_status_mcp_response.py", + "tests/server/test_tools_workspace.py" + ], + "src/autoskillit/server/_wire_compat.py": [ + "tests/arch/test_feature_markers.py", + "tests/franchise/test_pack_enforcement.py", + "tests/server/test_headless_session.py", + "tests/server/test_server_init_gate.py", + "tests/server/test_server_init_session_visibility.py", + "tests/server/test_server_tool_registration.py", + "tests/server/test_tool_annotation_completeness.py", + "tests/server/test_tools_recipe.py", + "tests/server/test_tools_status_kitchen.py", + "tests/server/test_tools_status_summaries.py", + "tests/server/test_tools_status_quota_and_db.py" + ], + "src/autoskillit/server/git.py": [ + "tests/server/test_git.py", + "tests/server/test_perform_merge_editable_guard.py", + "tests/server/test_server_tool_registration.py", + "tests/server/test_tools_git.py" + ], + "src/autoskillit/server/helpers.py": [ + "tests/arch/test_layer_enforcement.py", + "tests/contracts/test_instruction_surface.py", + "tests/infra/test_pretty_output_integration.py", + "tests/server/test_helpers.py", + "tests/server/test_helpers_gate.py", + "tests/server/test_helpers_tier_guards.py", + "tests/server/test_mcp_overrides.py", + "tests/server/test_quota_refresh_loop.py", + "tests/server/test_run_skill_add_dirs.py", + "tests/server/test_server_init_gate.py", + "tests/server/test_server_init_session_visibility.py", + "tests/server/test_server_tool_registration.py", + "tests/server/test_set_commit_status.py", + "tests/server/test_smoke_pipeline.py", + "tests/server/test_tool_exception_boundary.py", + "tests/server/test_tools_ci.py", + "tests/server/test_tools_ci_enqueue.py", + "tests/server/test_tools_clone.py", + "tests/server/test_tools_dispatch.py", + "tests/server/test_tools_execution_command.py", + "tests/server/test_tools_execution_input_gates.py", + "tests/server/test_tools_execution_results.py", + "tests/server/test_tools_execution_routing.py", + "tests/server/test_tools_execution_response.py", + "tests/server/test_tools_git.py", + "tests/server/test_tools_github.py", + "tests/server/test_tools_integrations.py", + "tests/server/test_tools_integrations_release.py", + "tests/server/test_tools_issue_lifecycle.py", + "tests/server/test_tools_kitchen_gate.py", + "tests/server/test_tools_kitchen_envelope.py", + "tests/server/test_tools_kitchen_visibility.py", + "tests/server/test_tools_label_validation.py", + "tests/server/test_tools_list_recipes.py", + "tests/server/test_tools_load_recipe.py", + "tests/server/test_tools_pr_ops.py", + "tests/server/test_tools_recipe.py", + "tests/server/test_tools_run_cmd.py", + "tests/server/test_tools_run_cmd_unit.py", + "tests/server/test_tools_run_python.py", + "tests/server/test_tools_run_skill_retry.py", + "tests/server/test_tools_session_diagnostics.py", + "tests/server/test_tools_status_kitchen.py", + "tests/server/test_tools_status_summaries.py", + "tests/server/test_tools_status_quota_and_db.py", + "tests/server/test_tools_status_mcp_response.py", + "tests/server/test_tools_workspace.py", + "tests/server/test_track_response_size.py" + ], + "src/autoskillit/server/tools_ci.py": [ + "tests/server/test_set_commit_status.py", + "tests/server/test_tools_ci.py", + "tests/server/test_tools_ci_enqueue.py" + ], + "src/autoskillit/server/tools_clone.py": [ + "tests/server/test_tools_clone.py" + ], + "src/autoskillit/server/tools_execution.py": [ + "tests/server/test_run_skill_add_dirs.py", + "tests/server/test_server_init_gate.py", + "tests/server/test_server_init_session_visibility.py", + "tests/server/test_server_tool_registration.py", + "tests/server/test_tools_dispatch.py", + "tests/server/test_tools_execution_command.py", + "tests/server/test_tools_execution_input_gates.py", + "tests/server/test_tools_execution_results.py", + "tests/server/test_tools_execution_routing.py", + "tests/server/test_tools_execution_response.py", + "tests/server/test_tools_run_cmd.py", + "tests/server/test_tools_run_cmd_unit.py", + "tests/server/test_tools_run_python.py", + "tests/server/test_tools_run_skill_retry.py", + "tests/server/test_tools_status_kitchen.py", + "tests/server/test_tools_status_summaries.py", + "tests/server/test_tools_status_quota_and_db.py" + ], + "src/autoskillit/server/tools_git.py": [ + "tests/server/test_server_tool_registration.py", + "tests/server/test_tools_git.py" + ], + "src/autoskillit/server/tools_github.py": [ + "tests/server/test_tools_github.py", + "tests/server/test_tools_session_diagnostics.py" + ], + "src/autoskillit/server/tools_issue_lifecycle.py": [ + "tests/server/test_tools_github.py", + "tests/server/test_tools_integrations.py", + "tests/server/test_tools_integrations_release.py", + "tests/server/test_tools_issue_lifecycle.py", + "tests/server/test_tools_label_validation.py" + ], + "src/autoskillit/server/tools_kitchen.py": [ + "tests/contracts/test_hook_bridge_coverage.py", + "tests/contracts/test_instruction_surface.py", + "tests/server/test_mcp_overrides.py", + "tests/server/test_reload_session.py", + "tests/server/test_server_init_gate.py", + "tests/server/test_server_init_session_visibility.py", + "tests/server/test_tools_execution_command.py", + "tests/server/test_tools_execution_input_gates.py", + "tests/server/test_tools_execution_results.py", + "tests/server/test_tools_execution_routing.py", + "tests/server/test_tools_kitchen_gate.py", + "tests/server/test_tools_kitchen_envelope.py", + "tests/server/test_tools_kitchen_visibility.py" + ], + "src/autoskillit/server/tools_pr_ops.py": [ + "tests/server/test_tools_integrations.py", + "tests/server/test_tools_pr_ops.py" + ], + "src/autoskillit/server/tools_recipe.py": [ + "tests/infra/test_pretty_output_integration.py", + "tests/server/test_mcp_overrides.py", + "tests/server/test_smoke_pipeline.py", + "tests/server/test_tools_list_recipes.py", + "tests/server/test_tools_load_recipe.py", + "tests/server/test_tools_recipe.py" + ], + "src/autoskillit/server/tools_status.py": [ + "tests/infra/test_pretty_output_integration.py", + "tests/server/test_tools_status_kitchen.py", + "tests/server/test_tools_status_summaries.py", + "tests/server/test_tools_status_quota_and_db.py", + "tests/server/test_tools_status_mcp_response.py" + ], + "src/autoskillit/server/tools_workspace.py": [ + "tests/server/test_server_tool_registration.py", + "tests/server/test_tools_workspace.py" + ], + "src/autoskillit/smoke_utils.py": [ + "tests/recipe/test_review_loop_routing_integration.py", + "tests/test_smoke_utils.py" + ], + "src/autoskillit/version.py": [ + "tests/cli/test_doctor.py", + "tests/contracts/test_version_consistency.py", + "tests/infra/test_pretty_output_integration.py", + "tests/server/test_server_version_telemetry.py", + "tests/server/test_tools_status_kitchen.py", + "tests/server/test_tools_status_summaries.py", + "tests/server/test_tools_status_quota_and_db.py", + "tests/test_version.py" + ], + "src/autoskillit/workspace/cleanup.py": [ + "tests/server/test_server_tool_registration.py", + "tests/server/test_tools_workspace.py", + "tests/workspace/test_cleanup.py" + ], + "src/autoskillit/workspace/clone.py": [ + "tests/workspace/test_clone.py", + "tests/workspace/test_clone_ci_contract.py" + ], + "src/autoskillit/workspace/clone_registry.py": [ + "tests/server/test_tools_clone.py", + "tests/workspace/test_clone_registry.py" + ], + "src/autoskillit/workspace/session_skills.py": [ + "tests/cli/test_cli_serve_logging.py", + "tests/cli/test_cook_env_scrub.py", + "tests/cli/test_cook_interactive.py", + "tests/cli/test_franchise_cli.py", + "tests/cli/test_init.py", + "tests/cli/test_reload_loop.py", + "tests/cli/test_startup_budget.py", + "tests/cli/test_terminal.py", + "tests/contracts/test_activate_deps_completeness.py", + "tests/contracts/test_claude_code_interface_contracts.py", + "tests/contracts/test_package_gateways.py", + "tests/contracts/test_target_skill_invocability.py", + "tests/execution/test_commands.py", + "tests/franchise/test_dispatch_failure_semantics.py", + "tests/franchise/test_franchise_e2e.py", + "tests/server/test_factory.py", + "tests/server/test_factory_recording.py", + "tests/server/test_headless_session.py", + "tests/server/test_run_skill_add_dirs.py", + "tests/server/test_server_init_gate.py", + "tests/server/test_server_init_session_visibility.py", + "tests/server/test_server_tool_registration.py", + "tests/server/test_tools_dispatch.py", + "tests/server/test_tools_execution_command.py", + "tests/server/test_tools_execution_input_gates.py", + "tests/server/test_tools_execution_results.py", + "tests/server/test_tools_execution_routing.py", + "tests/server/test_tools_execution_response.py", + "tests/server/test_tools_recipe.py", + "tests/server/test_tools_run_skill_retry.py", + "tests/server/test_tools_status_kitchen.py", + "tests/server/test_tools_status_summaries.py", + "tests/server/test_tools_status_quota_and_db.py", + "tests/workspace/test_project_local_overrides.py", + "tests/workspace/test_session_skills.py", + "tests/workspace/test_skill_content_substitution.py", + "tests/workspace/test_skills.py" + ], + "src/autoskillit/workspace/skills.py": [ + "tests/arch/test_protocol_names.py", + "tests/cli/test_cli_serve_logging.py", + "tests/cli/test_cook.py", + "tests/cli/test_cook_env_scrub.py", + "tests/cli/test_cook_interactive.py", + "tests/cli/test_franchise_cli.py", + "tests/cli/test_init.py", + "tests/cli/test_onboarding.py", + "tests/cli/test_reload_loop.py", + "tests/cli/test_startup_budget.py", + "tests/cli/test_terminal.py", + "tests/contracts/test_activate_deps_completeness.py", + "tests/contracts/test_claude_code_interface_contracts.py", + "tests/contracts/test_github_ops.py", + "tests/contracts/test_issue_content_fidelity.py", + "tests/contracts/test_make_campaign_skill_contracts.py", + "tests/contracts/test_package_gateways.py", + "tests/contracts/test_process_issues_contracts.py", + "tests/contracts/test_protocol_definitions.py", + "tests/contracts/test_skill_contracts.py", + "tests/contracts/test_skill_yaml_validation.py", + "tests/contracts/test_target_skill_invocability.py", + "tests/execution/test_commands.py", + "tests/migration/test_engine.py", + "tests/recipe/test_api.py", + "tests/recipe/test_bundled_recipe_hidden_policy.py", + "tests/recipe/test_bundled_recipes.py", + "tests/recipe/test_bundled_recipes_no_inversions.py", + "tests/recipe/test_contracts.py", + "tests/recipe/test_full_audit_recipe.py", + "tests/recipe/test_implementation_sprint_mode.py", + "tests/recipe/test_issue_url_pipeline.py", + "tests/recipe/test_make_campaign_output_schema.py", + "tests/recipe/test_merge_prs_queue.py", + "tests/recipe/test_remediation_sprint_mode.py", + "tests/recipe/test_research_output_mode.py", + "tests/recipe/test_rules_blocks.py", + "tests/recipe/test_rules_bypass.py", + "tests/recipe/test_rules_campaign.py", + "tests/recipe/test_rules_ci.py", + "tests/recipe/test_rules_clone.py", + "tests/recipe/test_rules_cmd.py", + "tests/recipe/test_rules_conditional_push.py", + "tests/recipe/test_rules_contracts.py", + "tests/recipe/test_rules_dataflow.py", + "tests/recipe/test_rules_features.py", + "tests/recipe/test_rules_graph.py", + "tests/recipe/test_rules_inputs.py", + "tests/recipe/test_rules_isolation.py", + "tests/recipe/test_rules_merge.py", + "tests/recipe/test_rules_packs.py", + "tests/recipe/test_rules_pipeline_internal.py", + "tests/recipe/test_rules_project_local_override.py", + "tests/recipe/test_rules_reachability.py", + "tests/recipe/test_rules_recipe.py", + "tests/recipe/test_rules_skill_content.py", + "tests/recipe/test_rules_skills.py", + "tests/recipe/test_rules_structure.py", + "tests/recipe/test_rules_subset_disabled.py", + "tests/recipe/test_rules_tools.py", + "tests/recipe/test_rules_verdict.py", + "tests/recipe/test_rules_worktree.py", + "tests/recipe/test_sprint_sub_recipe.py", + "tests/recipe/test_sub_recipe_validation.py", + "tests/recipe/test_validator.py", + "tests/server/test_factory.py", + "tests/server/test_factory_recording.py", + "tests/server/test_run_skill_add_dirs.py", + "tests/server/test_server_tool_registration.py", + "tests/server/test_service_wrappers.py", + "tests/server/test_smoke_pipeline.py", + "tests/server/test_tools_execution_command.py", + "tests/server/test_tools_execution_input_gates.py", + "tests/server/test_tools_execution_results.py", + "tests/server/test_tools_execution_routing.py", + "tests/server/test_tools_execution_response.py", + "tests/server/test_tools_kitchen_gate.py", + "tests/server/test_tools_kitchen_envelope.py", + "tests/server/test_tools_kitchen_visibility.py", + "tests/server/test_tools_load_recipe.py", + "tests/server/test_tools_recipe.py", + "tests/server/test_tools_run_skill_retry.py", + "tests/server/test_tools_status_kitchen.py", + "tests/server/test_tools_status_summaries.py", + "tests/server/test_tools_status_quota_and_db.py", + "tests/skills/test_phase2_skills.py", + "tests/skills/test_skill_compliance.py", + "tests/skills/test_skill_output_compliance.py", + "tests/skills/test_skill_preambles.py", + "tests/skills/test_troubleshoot_experiment_contracts.py", + "tests/skills/test_validate_audit_contracts.py", + "tests/workspace/test_project_local_overrides.py", + "tests/workspace/test_session_skills.py", + "tests/workspace/test_skill_content_substitution.py", + "tests/workspace/test_skills.py" + ], + "src/autoskillit/workspace/worktree.py": [ + "tests/cli/test_workspace.py", + "tests/server/test_git.py", + "tests/server/test_perform_merge_editable_guard.py", + "tests/server/test_server_tool_registration.py", + "tests/server/test_tools_git.py", + "tests/workspace/test_worktree.py" + ] +} \ No newline at end of file diff --git a/.claude/skills/audit-tests/SKILL.md b/.claude/skills/audit-tests/SKILL.md index 0cf960c0d8..f2f642a528 100644 --- a/.claude/skills/audit-tests/SKILL.md +++ b/.claude/skills/audit-tests/SKILL.md @@ -1,7 +1,7 @@ --- name: audit-tests categories: [audit] -description: Audit the test suite for useless tests, consolidation opportunities, over-mocking, weak assertions, placement/organization issues, xdist safety violations, and other test quality issues. Use when user says "audit tests", "audit test suite", "review tests", or "test quality check". Generates an improvement plan in temp/ with explanations for each proposed change. +description: Audit the test suite for useless tests, consolidation opportunities, over-mocking, weak assertions, placement/organization issues, xdist safety violations, test path filter integrity, and other test quality issues. Use when user says "audit tests", "audit test suite", "review tests", or "test quality check". Generates an improvement plan in temp/ with explanations for each proposed change. hooks: PreToolUse: - matcher: "*" @@ -23,21 +23,21 @@ Audit the test suite to identify useless tests, consolidation opportunities, qua ## Intended Test Directory Structure -The test suite mirrors the source sub-package hierarchy rather than using a type-based unit/integration split — the project's L0–L3 layer boundaries already encode isolation levels. Each sub-package gets its own test directory. Cross-cutting concerns go in dedicated directories: +The test suite mirrors the source sub-package hierarchy rather than using a type-based unit/integration split — the project's IL-0–IL-3 layer boundaries already encode isolation levels. Each sub-package gets its own test directory. Cross-cutting concerns go in dedicated directories: ``` tests/ ├── conftest.py # Shared: MockSubprocessRunner, tool_ctx fixture (wired via make_context) ├── CLAUDE.md # xdist safety guidelines -├── core/ # L0: tests for autoskillit/core/ (io, logging, paths, types, protocols) -├── config/ # L1: tests for autoskillit/config/ -├── pipeline/ # L1: tests for autoskillit/pipeline/ (audit, gate, context, tokens) -├── execution/ # L1: tests for autoskillit/execution/ (db, github, headless, process, quota, session, testing) -├── workspace/ # L1: tests for autoskillit/workspace/ (cleanup, clone, skills) -├── recipe/ # L2: tests for autoskillit/recipe/ (io, loader, schema, validator, rules, smoke) -├── migration/ # L2: tests for autoskillit/migration/ (engine, loader, store) -├── server/ # L3: tests for autoskillit/server/ (_factory, git, helpers, tools_*) -├── cli/ # L3: tests for autoskillit/cli/ +├── core/ # IL-0: tests for autoskillit/core/ (io, logging, paths, types, protocols) +├── config/ # IL-1: tests for autoskillit/config/ +├── pipeline/ # IL-1: tests for autoskillit/pipeline/ (audit, gate, context, tokens) +├── execution/ # IL-1: tests for autoskillit/execution/ (db, github, headless, process, quota, session, testing) +├── workspace/ # IL-1: tests for autoskillit/workspace/ (cleanup, clone, skills) +├── recipe/ # IL-2: tests for autoskillit/recipe/ (io, loader, schema, validator, rules, smoke) +├── migration/ # IL-2: tests for autoskillit/migration/ (engine, loader, store) +├── server/ # IL-3: tests for autoskillit/server/ (_factory, git, helpers, tools_*) +├── cli/ # IL-3: tests for autoskillit/cli/ ├── arch/ # Cross-cutting: AST architecture enforcement, import contracts, layer rules ├── contracts/ # Cross-cutting: instruction surface, MCP/API surface, version consistency └── infra/ # Cross-cutting: CI/dev config checks (.pre-commit, .gitleaks, .github/) @@ -139,6 +139,9 @@ Tests that no longer align with the current codebase. - Tests that are always skipped or conditionally disabled - Tests whose setup creates state the production code no longer uses - `CLAUDE.md` test file list that diverges from the actual files on disk — flag if the documented inventory is incomplete or contains stale entries +- `LAYER_CASCADE_CONSERVATIVE` or `LAYER_CASCADE_AGGRESSIVE` keys in `tests/_test_filter.py` that don't match the current set of subpackages under `src/autoskillit/` +- `.autoskillit/test-filter-manifest.yaml` patterns that match zero tracked files (orphaned entries) +- `.autoskillit/test-source-map.json` not regenerated within the quarterly schedule ### Category 7: Fixture Issues (LOW) @@ -159,16 +162,18 @@ Tests placed in the wrong directory relative to the intended sub-package hierarc **What to look for:** - Any test file living at the `tests/` root other than `conftest.py` and `CLAUDE.md` — all test files should be inside sub-directories -- Tests for L0 (`core/`) modules not in `tests/core/` -- Tests for L1 modules (`config`, `pipeline`, `execution`, `workspace`) not in their corresponding `tests//` directory -- Tests for L2 modules (`recipe`, `migration`) not in their corresponding `tests//` directory -- Tests for L3 modules (`server`, `cli`) not in their corresponding `tests//` directory +- Tests for IL-0 (`core/`) modules not in `tests/core/` +- Tests for IL-1 modules (`config`, `pipeline`, `execution`, `workspace`) not in their corresponding `tests//` directory +- Tests for IL-2 modules (`recipe`, `migration`) not in their corresponding `tests//` directory +- Tests for IL-3 modules (`server`, `cli`) not in their corresponding `tests//` directory - AST-based architecture enforcement tests (import contracts, layer rules, structural checks) outside `tests/arch/` - Instruction surface / MCP contract / SKILL.md contract tests outside `tests/contracts/` - CI/dev infrastructure tests checking `.pre-commit-config.yaml`, `.gitleaks.toml`, or `.github/workflows/` files outside `tests/infra/` - Test file names carrying redundant sub-package prefixes (e.g., `test_recipe_io.py` inside `tests/recipe/` should be `test_io.py`) - Test file names that don't reflect the source module: `test_db_tools.py` → `test_db.py`, `test_test_runner.py` → `test_testing.py`, `test_yaml_extended.py` → `test_io.py`, `test_process_lifecycle.py` → `test_process.py`, `test_session_result.py` → `test_session.py`, `test_headless_runner.py` → `test_headless.py`, `test_skill_resolver.py` → `test_skills.py`, `test_git_operations.py` → `test_git.py` - Test files that cover modules from two different layers without a clear rationale (candidates for splitting) +- Test files that import production code from a source layer outside their directory's cascade entry — even if correctly placed by sub-package, the import creates an invisible dependency the filter cannot track (cross-layer import, see C11) +- Test files at the `tests/` root that are part of the filter infrastructure (`test_test_filter.py`, `test_test_filter_plugin.py`, `test_test_filter_step7.py`) — these are correctly placed and should NOT be flagged for relocation ### Category 9: Oversized Files (MEDIUM) @@ -176,7 +181,7 @@ Test files or supporting files exceeding 750 lines. Flag files approaching the t ### Category 10: xdist Safety Violations (HIGH) -Tests that are unsafe for parallel execution under pytest-xdist (`-n 4 --dist worksteal`). This project runs tests in parallel by default with explicit safety rules defined in `tests/CLAUDE.md`. +Tests that are unsafe for parallel execution under pytest-xdist (`-n 4 --dist load`). This project runs tests in parallel by default with explicit safety rules defined in `tests/CLAUDE.md`. **What to look for:** - Tests that create temporary files or directories at fixed paths instead of using `tmp_path` (pytest provides unique per-test paths; on Linux these should land under `/dev/shm/pytest-tmp` per `tests/CLAUDE.md` guidance) @@ -187,20 +192,54 @@ Tests that are unsafe for parallel execution under pytest-xdist (`-n 4 --dist wo - Tests that write to or read from paths shared with other tests (e.g., a hardcoded `temp/` subpath without a unique suffix) - Tests that rely on execution order (pass in isolation, fail when another test runs first) — detect by checking for setup that assumes prior state +### Category 11: Test Path Filter Integrity (HIGH) + +Tests and configuration that maintain the path-based test filter's correctness. The filter system (`tests/_test_filter.py`) provides a 5-minute local feedback loop by selecting only affected test directories. False negatives here mean broken code passes CI; false positives mean slow developer feedback. + +**Cascade alignment:** +- Test files that import production code from a source layer NOT in their directory's cascade entry in `LAYER_CASCADE_CONSERVATIVE` — these tests are silently skipped when the imported module changes (false negative). Check by walking `from autoskillit.` imports in each test file and verifying the test's directory appears in `LAYER_CASCADE_CONSERVATIVE[]` +- New source subpackages under `src/autoskillit/` not present as keys in both `LAYER_CASCADE_CONSERVATIVE` and `LAYER_CASCADE_AGGRESSIVE` +- Cross-layer test imports invisible to the filter's AST walker (e.g., dynamic imports via `importlib.import_module`) + +**Manifest coverage:** +- Non-Python tracked files without a matching entry in `.autoskillit/test-filter-manifest.yaml` — causes unnecessary full runs when only that file changed +- Manifest entries pointing to nonexistent test directories +- Manifest patterns that match zero tracked files (orphaned entries) + +**Size marker correctness:** +- Always-run directories (`arch/`, `contracts/`) that have zero size markers — these are fully deselected by aggressive mode's size filter, nullifying the always-run safety net +- `small`-marked tests that spawn subprocesses or perform real filesystem I/O (should be `medium`) +- `medium`-marked tests that access the network (should be `large`) +- `_SIZE_DIRS` in `conftest.py` diverging from `SIZE_DIRECTORIES` in `tests/arch/test_size_markers.py` + +**Bucket A discipline:** +- Files in `BUCKET_A_PATTERNS` that could be narrowed to specific test directories via the manifest instead of triggering a full run +- High-churn files in Bucket A that frequently negate filter performance gains + +**Test file naming for oracle traceability:** +- Source modules under `src/autoskillit/` with zero corresponding `test_.py` files — the coverage oracle (`test-source-map.json`) and aggressive mode step 7 file-level filtering rely on naming correspondence +- Cross-reference C8's naming rules — the filter-specific concern is that misnamed test files block file-level filtering in aggressive mode step 7 + +**Filter infrastructure staleness:** +- `LAYER_CASCADE_CONSERVATIVE` or `LAYER_CASCADE_AGGRESSIVE` keys in `tests/_test_filter.py` that don't match the current set of subpackages under `src/autoskillit/` +- Hardcoded count assertions in filter tests that have drifted from reality +- `test-source-map.json` (coverage oracle) not regenerated within the quarterly schedule +- `ALWAYS_RUN_CONSERVATIVE` or `ALWAYS_RUN_AGGRESSIVE` sets containing directories that no longer exist under `tests/` + --- ## Audit Workflow ### Step 1: Launch Parallel Subagents -Spawn 6 domain-based subagents. Each covers all issue categories (C1–C10) within its area. Group by source domain, not by issue category. Each subagent must read both test files AND the corresponding production code before making judgements. +Spawn 6 domain-based subagents. Each covers all issue categories (C1–C11) within its area. Group by source domain, not by issue category. Each subagent must read both test files AND the corresponding production code before making judgements. -- **Group 1 — Core + Config (L0 + L1):** Tests for `core/` and `config/` sub-packages. Also check `conftest.py` for fixture quality. -- **Group 2 — Pipeline + Workspace (L1):** Tests for `pipeline/` and `workspace/` sub-packages. -- **Group 3 — Execution (L1):** Tests for `execution/` sub-package. -- **Group 4 — Recipe + Migration (L2):** Tests for `recipe/` and `migration/` sub-packages. -- **Group 5 — Server + CLI (L3):** Tests for `server/` and `cli/` sub-packages. -- **Group 6 — Cross-cutting:** Architecture enforcement tests, instruction surface/contract tests, CI/dev infrastructure tests. Also audit `tests/CLAUDE.md` for accuracy against the actual test files on disk. +- **Group 1 — Core + Config (IL-0 + IL-1):** Tests for `core/` and `config/` sub-packages. Also check `conftest.py` for fixture quality. +- **Group 2 — Pipeline + Workspace (IL-1):** Tests for `pipeline/` and `workspace/` sub-packages. +- **Group 3 — Execution (IL-1):** Tests for `execution/` sub-package. +- **Group 4 — Recipe + Migration (IL-2):** Tests for `recipe/` and `migration/` sub-packages. +- **Group 5 — Server + CLI (IL-3):** Tests for `server/` and `cli/` sub-packages. +- **Group 6 — Cross-cutting:** Architecture enforcement tests, instruction surface/contract tests, CI/dev infrastructure tests. Also audit `tests/CLAUDE.md` for accuracy against the actual test files on disk. Additionally, perform filter integrity checks: verify filter cascade maps (`LAYER_CASCADE_CONSERVATIVE`, `LAYER_CASCADE_AGGRESSIVE`) against actual source subpackages under `src/autoskillit/`; check manifest completeness against `git ls-files`; verify size marker rollup coverage against `_SIZE_DIRS` in `conftest.py`; check Bucket A minimality (files that could use manifest instead); verify always-run directories (`ALWAYS_RUN_CONSERVATIVE`, `ALWAYS_RUN_AGGRESSIVE`) have appropriate size markers or are exempted from size filtering. For each finding, note the file, line range, issue category, and a brief explanation of why it's a problem and what should change. @@ -249,3 +288,5 @@ Do NOT flag: - Tests that are intentionally minimal as smoke tests - Shared fixtures in centralized test configuration - Architecture enforcement tests (`test_architecture.py`, `test_import_paths.py`) for having "weak assertions" or "no meaningful coverage" — AST rule enforcement tests validate source code structure, not runtime behavior, and are correctly evaluated against whether the structural rule they enforce is correct and complete, not by normal assertion-strength standards +- Test files at the `tests/` root that are part of the filter infrastructure (`test_test_filter.py`, `test_test_filter_plugin.py`, `test_test_filter_step7.py`, `_test_filter.py`) — these test root-level infrastructure and are correctly placed at the root +- Hardcoded count assertions in filter/manifest tests (e.g., `>= 22 patterns`) — these are intentional drift detectors, not weak assertions diff --git a/.claude/skills/check-bearing/SKILL.md b/.claude/skills/check-bearing/SKILL.md index 7ccfa947c8..947b42e6ee 100644 --- a/.claude/skills/check-bearing/SKILL.md +++ b/.claude/skills/check-bearing/SKILL.md @@ -40,7 +40,7 @@ hooks: ### Target Resolution **Branch name:** -- Detect base branch via `git merge-base` against `main` and `integration` +- Detect base branch via `git merge-base` against `main` and `develop` - Use the closer base (fewer commits ahead) - Diff: `git diff {merge_base}...{branch}` diff --git a/.claude/skills/promote-to-main/SKILL.md b/.claude/skills/promote-to-main/SKILL.md index 09109af767..fab2c8e459 100644 --- a/.claude/skills/promote-to-main/SKILL.md +++ b/.claude/skills/promote-to-main/SKILL.md @@ -18,17 +18,17 @@ and creates a comprehensive promotion PR. ## Arguments ``` -/autoskillit:promote-to-main [integration_branch] [base_branch] [--dry-run] +/autoskillit:promote-to-main [batch_branch] [base_branch] [--dry-run] ``` -- `integration_branch` (optional) — source branch to promote. Defaults to `integration`. +- `batch_branch` (optional) — source branch to promote. Defaults to `integration`. - `base_branch` (optional) — target branch. Defaults to `main`. - `--dry-run` — generate the full PR body without creating a PR. ## When to Use -- When the integration branch is stable and ready to be promoted to main -- After feature PRs, bug fixes, and cleanup work have been collected and tested on integration +- When the develop branch is stable and ready to be promoted to main +- After feature PRs, bug fixes, and cleanup work have been collected and tested on develop - This is the final gate before changes land on main ## Critical Constraints @@ -49,20 +49,7 @@ and creates a comprehensive promotion PR. - Output `verdict = ` as a structured token - Carry forward ALL `Closes #N`, `Fixes #N`, and `Resolves #N` references from merged PR bodies - Use `gh pr create --body-file` (never inline body via `--body`) -- Grant every subagent explicit permission to spawn their own sub-subagents -- `report_path` must be an absolute path (prepend CWD) - -## Subagent Autonomy Grant - -Every subagent spawned by this skill receives this standing instruction: - -> You may spawn additional subagents (Task tool, model: sonnet) at your discretion -> to parallelize your research, fill gaps you discover during analysis, or decompose -> large tasks into focused sub-investigations. You do not need permission — use your -> judgment about when deeper exploration would improve the quality of your findings. - -Include this grant verbatim in every Task tool prompt throughout this skill. - +- `pr_body_path` must be an absolute path (prepend CWD) ## Workflow ### Phase 0: Setup @@ -70,13 +57,13 @@ Include this grant verbatim in every Task tool prompt throughout this skill. #### Step 0.1: Parse Arguments Parse optional positional arguments and flags: -- `integration_branch` — default `"integration"` if absent or empty +- `batch_branch` — default `"develop"` if absent or empty - `base_branch` — default `"main"` if absent or empty - `dry_run` — `true` if `--dry-run` present in ARGUMENTS Validate that both branches exist locally: ```bash -git rev-parse --verify {integration_branch} 2>/dev/null +git rev-parse --verify {batch_branch} 2>/dev/null git rev-parse --verify {base_branch} 2>/dev/null ``` If either fails, try fetching: @@ -88,14 +75,14 @@ If still missing, print error to stderr and exit 1. #### Step 0.2: Compute Divergence Point ```bash -git merge-base {base_branch} {integration_branch} +git merge-base {base_branch} {batch_branch} ``` Store as `merge_base_sha`. Get commit count and timestamp: ```bash -git rev-list --count {merge_base_sha}..{integration_branch} +git rev-list --count {merge_base_sha}..{batch_branch} git show -s --format=%cI {merge_base_sha} ``` Store as `commit_count` and `merge_base_date`. @@ -142,17 +129,14 @@ EOF Spawn three parallel Task subagents (model: sonnet) to validate promotion readiness. All three must pass before analysis proceeds. If any fails, report the failure clearly and exit 1. Do NOT create a PR when pre-flight fails. - -**Include the Subagent Autonomy Grant in each prompt.** - #### Subagent 1A: CI and Branch Status Check: 1. CI is green on the integration branch — run `gh pr checks` for any open PR from - integration, or `gh run list --branch {integration_branch} --workflow tests.yml --limit 1 --json conclusion` -2. The integration branch is not behind base — run `git rev-list --count {integration_branch}..{base_branch}` + integration, or `gh run list --branch {batch_branch} --workflow tests.yml --limit 1 --json conclusion` +2. The integration branch is not behind base — run `git rev-list --count {batch_branch}..{base_branch}` to check if base has commits not in integration (if > 0, warn that a rebase may be needed) -3. No open PRs targeting integration with failing CI — `gh pr list --base {integration_branch} --state open --json number,title,statusCheckRollup` +3. No open PRs targeting integration with failing CI — `gh pr list --base {batch_branch} --state open --json number,title,statusCheckRollup` Return JSON: ```json @@ -188,7 +172,7 @@ Return JSON: Check: 1. No open PRs targeting integration with `CHANGES_REQUESTED` reviews — - `gh pr list --base {integration_branch} --state open --json number,title,reviews` + `gh pr list --base {batch_branch} --state open --json number,title,reviews` 2. No `in-progress` labeled issues that might indicate incomplete work — `gh issue list --label in-progress --state open --json number,title` @@ -207,14 +191,13 @@ treat as a warning (non-blocking) and note it in the report. ### Phase 2: Change Inventory (parallel subagents) -Spawn four parallel Task subagents (model: sonnet). **Include the Subagent Autonomy Grant -in each prompt.** +Spawn four parallel Task subagents (model: sonnet). #### Subagent 2A: Commit Categorization Receive the output of: ```bash -git log {merge_base_sha}..{integration_branch} --format="%H %s" +git log {merge_base_sha}..{batch_branch} --format="%H %s" ``` Categorize each commit into exactly one category based on its subject line: @@ -245,13 +228,13 @@ Return JSON: Run: ```bash -gh pr list --base {integration_branch} --state merged --limit 200 --json number,title,author,mergedAt,body,headRefName,additions,deletions,labels,url +gh pr list --base {batch_branch} --state merged --limit 200 --json number,title,author,mergedAt,body,headRefName,additions,deletions,labels,url ``` Filter to PRs merged after `merge_base_date`. If empty, fall back to commit-subject discovery: ```bash -git log {merge_base_sha}..{integration_branch} --oneline --grep="(#" --format="%s" +git log {merge_base_sha}..{batch_branch} --oneline --grep="(#" --format="%s" ``` For each PR, extract `Closes|Fixes|Resolves #N` references (case-insensitive). @@ -277,12 +260,12 @@ Return JSON: Run: ```bash -git diff --name-only {base_branch}..{integration_branch} -git diff --diff-filter=A --name-only {base_branch}..{integration_branch} -git diff --diff-filter=M --name-only {base_branch}..{integration_branch} -git diff --diff-filter=D --name-only {base_branch}..{integration_branch} -git diff --diff-filter=R --name-only {base_branch}..{integration_branch} -git diff --stat {base_branch}..{integration_branch} | tail -1 +git diff --name-only {base_branch}..{batch_branch} +git diff --diff-filter=A --name-only {base_branch}..{batch_branch} +git diff --diff-filter=M --name-only {base_branch}..{batch_branch} +git diff --diff-filter=D --name-only {base_branch}..{batch_branch} +git diff --diff-filter=R --name-only {base_branch}..{batch_branch} +git diff --stat {base_branch}..{batch_branch} | tail -1 ``` Return JSON: @@ -400,9 +383,6 @@ new/modified nodes, add to `validated_diagrams`. Otherwise discard. Spawn one Task subagent (model: sonnet) with results from Phase 2 ONLY (no domain analysis or quality assessment data). - -**Include the Subagent Autonomy Grant.** - The subagent receives: - Commit categorization from Subagent 2A - PR and issue data from Subagent 2B @@ -424,7 +404,7 @@ Write to `.autoskillit/temp/promote-to-main/pr_body_{YYYY-MM-DD_HHMMSS}.md` (relative to the current working directory). Sections in order: -1. `## Promotion: {integration_branch} to {base_branch}` — executive summary + stats (`diff_stat_summary`, `commit_count`, PR count) +1. `## Promotion: {batch_branch} to {base_branch}` — executive summary + stats (`diff_stat_summary`, `commit_count`, PR count) 2. `### Highlights` — from synthesis 3. `## Release Notes` — from synthesis 4. `## Merged PRs` — table (PR, Title, Author, Labels) from Subagent 2B @@ -446,12 +426,12 @@ gh auth status 2>/dev/null If exit code non-zero: output `pr_url = ` and exit successfully. Construct PR title using actual branch names: -`Promote {integration_branch} to {base_branch} ({len(prs)} PRs, {len(linked_issue_numbers)} issues, {category_summary})` +`Promote {batch_branch} to {base_branch} ({len(prs)} PRs, {len(linked_issue_numbers)} issues, {category_summary})` ```bash gh pr create \ --base {base_branch} \ - --head {integration_branch} \ + --head {batch_branch} \ --title "{pr_title}" \ --body-file .autoskillit/temp/promote-to-main/pr_body_{timestamp}.md ``` @@ -468,7 +448,7 @@ gh pr edit {pr_url} --add-label "promotion" 2>/dev/null || true Always emit these structured output tokens as the final lines: ``` -report_path = {absolute path to .autoskillit/temp/promote-to-main/pr_body_{timestamp}.md} +pr_body_path = {absolute path to .autoskillit/temp/promote-to-main/pr_body_{timestamp}.md} pr_url = {pr_url, empty if dry-run or gh unavailable} verdict = {created|dry_run|preflight_failed} category_summary = {e.g., "14 fixes, 13 features, 3 infra"} diff --git a/.claude/skills/review-promotion/SKILL.md b/.claude/skills/review-promotion/SKILL.md index 5c59e6f3e7..cd4a31624f 100644 --- a/.claude/skills/review-promotion/SKILL.md +++ b/.claude/skills/review-promotion/SKILL.md @@ -18,10 +18,10 @@ posts the review report as a PR comment. ## Arguments ``` -/autoskillit:review-promotion [integration_branch] [base_branch] [--post-to-pr] +/autoskillit:review-promotion [batch_branch] [base_branch] [--post-to-pr] ``` -- `integration_branch` (optional) — source branch to analyze. Defaults to `integration`. +- `batch_branch` (optional) — source branch to analyze. Defaults to `develop`. - `base_branch` (optional) — target branch. Defaults to `main`. - `--post-to-pr` — if present, post the review report as a comment on the open promotion PR. @@ -42,20 +42,6 @@ posts the review report as a PR comment. **ALWAYS:** - Output `report_path = ` as a structured token (absolute path, prepend CWD) - Output `verdict = ` as a structured token -- Include Subagent Autonomy Grant in every Task tool prompt -- Grant every subagent explicit permission to spawn their own sub-subagents - -## Subagent Autonomy Grant - -Every subagent spawned by this skill receives this standing instruction: - -> You may spawn additional subagents (Task tool, model: sonnet) at your discretion -> to parallelize your research, fill gaps you discover during analysis, or decompose -> large tasks into focused sub-investigations. You do not need permission — use your -> judgment about when deeper exploration would improve the quality of your findings. - -Include this grant verbatim in every Task tool prompt throughout this skill. - ## Workflow ### Phase 0: Setup @@ -63,17 +49,17 @@ Include this grant verbatim in every Task tool prompt throughout this skill. #### Step 0.1: Parse Arguments Parse optional positional arguments and flags: -- `integration_branch` — default `"integration"` if absent or empty +- `batch_branch` — default `"develop"` if absent or empty - `base_branch` — default `"main"` if absent or empty - `post_to_pr` — `true` if `--post-to-pr` present in ARGUMENTS #### Step 0.2: Compute Divergence Point ```bash -git merge-base {base_branch} {integration_branch} -git diff --name-only {base_branch}..{integration_branch} -git diff --name-only --diff-filter=A {base_branch}..{integration_branch} -git diff --name-only --diff-filter=M {base_branch}..{integration_branch} +git merge-base {base_branch} {batch_branch} +git diff --name-only {base_branch}..{batch_branch} +git diff --name-only --diff-filter=A {base_branch}..{batch_branch} +git diff --name-only --diff-filter=M {base_branch}..{batch_branch} ``` Store as `merge_base_sha`, `changed_files`, `new_files` (added files), and @@ -83,14 +69,14 @@ If the `git merge-base` or `git diff` command exits non-zero (e.g., unknown bran emit a clear error and exit 1: ``` -Error: could not compute divergence point between '{base_branch}' and '{integration_branch}'. +Error: could not compute divergence point between '{base_branch}' and '{batch_branch}'. Check that both branches exist locally or are fetchable. ``` If `changed_files` is empty after a successful `git diff`, emit: ``` -Error: no changed files found between '{base_branch}' and '{integration_branch}'. +Error: no changed files found between '{base_branch}' and '{batch_branch}'. Verify the branches are not identical and that the correct branch names were supplied. ``` @@ -99,7 +85,7 @@ Then exit 1. #### Step 0.3: Find PR (only if --post-to-pr) ```bash -gh pr list --base {base_branch} --head {integration_branch} --state open --json number,url --limit 1 +gh pr list --base {base_branch} --head {batch_branch} --state open --json number,url --limit 1 ``` Store `pr_number` and `pr_url`. If none found, warn and continue — the review report will @@ -126,7 +112,7 @@ Store as `domain_partitions`. Skip if `changed_files` is empty. For each domain `D` in `domain_partitions` with a non-empty file list, run in parallel: ```bash -git diff {base_branch}..{integration_branch} -- {space-separated files in domain D} +git diff {base_branch}..{batch_branch} -- {space-separated files in domain D} ``` Truncate diffs exceeding 12,000 characters. Drop domains with empty diffs. @@ -136,7 +122,7 @@ Truncate diffs exceeding 12,000 characters. Drop domains with empty diffs. For each domain in `domain_diffs`, run in parallel: ```bash -git log {base_branch}..{integration_branch} --oneline -- {space-separated files in domain D} +git log {base_branch}..{batch_branch} --oneline -- {space-separated files in domain D} ``` #### Step 1.4: Identify PRs per Domain @@ -152,7 +138,7 @@ Store as `domain_pr_numbers`. #### Step 1.5: Parallel Domain Analysis Subagents For each domain `D` in `domain_diffs`, spawn a Task subagent (model: sonnet) in a -single parallel message. **Include the Subagent Autonomy Grant in each prompt.** +single parallel message. Each subagent receives: - Domain name and file list @@ -179,7 +165,6 @@ Each subagent returns ONLY a JSON object: #### Step 1.6: Cross-Domain Dependency Analysis Spawn one Task subagent (model: sonnet) with ALL domain summaries from Step 1.5. -**Include the Subagent Autonomy Grant.** Analyze cross-domain dependencies: - Do recipe schema changes require corresponding server tool updates? @@ -198,8 +183,7 @@ Return JSON: ### Phase 2: Quality Assessment (parallel subagents) -Spawn three parallel Task subagents (model: sonnet). **Include the Subagent Autonomy -Grant in each prompt.** +Spawn three parallel Task subagents (model: sonnet). #### Subagent 2A: Test Coverage Delta @@ -282,7 +266,6 @@ Return JSON: ### Phase 3: Review Summary Synthesis Spawn one Task subagent (model: sonnet) with ALL results from Phases 1–2. -**Include the Subagent Autonomy Grant.** The subagent synthesizes a reviewer-focused verdict based on: - Domain risk scores from Phase 1 @@ -319,7 +302,7 @@ Write to `.autoskillit/temp/review-promotion/review_report_{YYYY-MM-DD_HHMMSS}.m (relative to the current working directory): ```markdown -# Promotion Review: {integration_branch} → {base_branch} +# Promotion Review: {batch_branch} → {base_branch} ## Verdict: {verdict} diff --git a/.claude/skills/validate-audit/SKILL.md b/.claude/skills/validate-audit/SKILL.md index 10f1c6b35a..c97e663c7b 100644 --- a/.claude/skills/validate-audit/SKILL.md +++ b/.claude/skills/validate-audit/SKILL.md @@ -40,10 +40,11 @@ signal downstream processing. **NEVER:** - Modify any source code files -- Create files outside `{{AUTOSKILLIT_TEMP}}/validate-audit/` +- Create files outside the per-run output directory (`{{AUTOSKILLIT_TEMP}}/validate-audit-{YYYY-MM-DD_HHMMSS}/`) - Issue subagent Task calls sequentially — ALL must be in a single parallel message - Write output files before synthesizing ALL subagent results - Subagents must NOT create their own files — they return findings in response text only +- Do NOT include VALID BUT EXCEPTION WARRANTED findings in the validated report body — they belong in the validation summary only **ALWAYS:** - Use `model: "sonnet"` when spawning all subagents via the Task tool @@ -195,10 +196,11 @@ After all agents return: ### Step 5 — Generate Output Files -Ensure `{{AUTOSKILLIT_TEMP}}/validate-audit/` exists (`mkdir -p`). +Generate a run timestamp: `{YYYY-MM-DD_HHMMSS}` (current UTC). Create the per-run directory: +`mkdir -p {{AUTOSKILLIT_TEMP}}/validate-audit-{YYYY-MM-DD_HHMMSS}/` **File 1 — Validated report** -Path: `{{AUTOSKILLIT_TEMP}}/validate-audit/validated_report_{source}_{YYYY-MM-DD_HHMMSS}.md` +Path: `{{AUTOSKILLIT_TEMP}}/validate-audit-{YYYY-MM-DD_HHMMSS}/validated_report_{source}.md` Structure: @@ -222,19 +224,17 @@ validated: true ## Validated Findings -{Each VALID finding: original text, verdict badge, severity adjustment if any.} - -## Findings with Exceptions - -{Each VALID BUT EXCEPTION WARRANTED finding: original text + exception note.} +{Each **VALID** finding only — do NOT include VALID BUT EXCEPTION WARRANTED findings here. +Exception-warranted findings go exclusively in the validation summary file. +Format: original finding text, VALID verdict badge, severity adjustment note if applicable.} --- -*{N_contested} finding(s) contested and excluded — see contested_findings_{source}_{ts}.md* +*{N_contested} finding(s) contested and excluded — see contested_findings_{source}.md* ``` **File 2 — Contested findings** (write only when `N_contested > 0`) -Path: `{{AUTOSKILLIT_TEMP}}/validate-audit/contested_findings_{source}_{YYYY-MM-DD_HHMMSS}.md` +Path: `{{AUTOSKILLIT_TEMP}}/validate-audit-{YYYY-MM-DD_HHMMSS}/contested_findings_{source}.md` Structure: @@ -251,47 +251,232 @@ Structure: **Historical context:** {from history agent, if relevant; else omit} ``` -### Step 6 — Interactive vs Headless Approval +### Step 5b — Write Validation Summary + +Write the full audit trail to a separate file. This file is NOT part of the issue body — +it is posted as a comment after issue creation. + +Path: `{{AUTOSKILLIT_TEMP}}/validate-audit-{YYYY-MM-DD_HHMMSS}/validation_summary_{source}.md` + +Structure: + +```markdown +# Validation Summary — {source} ({YYYY-MM-DD}) + +**Original report:** {audit_report_path} +**Total findings:** {total} | **Valid:** {N_valid} | **Exception warranted:** {N_exception} | **Contested:** {N_contested} + +--- + +## Per-Finding Verdicts + +| Finding ID | Verdict | Severity (adj.) | Reasoning summary | +|------------|---------|-----------------|-------------------| +| ... | ... | ... | ... | + +--- + +## Exception-Warranted Findings + +{For each VALID BUT EXCEPTION WARRANTED finding:} + +### [{ID}] {short description} + +**Original severity:** {severity} +**Exception note:** {constraint that warrants exception} +**Code evidence:** {file:line + what code shows} +**Historical context:** {from history agent, if relevant; else omit} + +--- + +## Contested Findings (Removed) + +{For each CONTESTED finding: full text, contest rationale, code evidence.} + +--- + +## Severity Adjustments + +{For each finding where severity was adjusted: original → adjusted, rationale.} +``` + +### Step 6 — Parallel Post-Validation (SINGLE MESSAGE, READ-ONLY) + +After both the validated report and validation summary are written, launch **two read-only +subagents in a single message**. Neither subagent may use Write, Edit, or any file-creation +tool — they return findings as response text only. + +**Subagent A — Cross-Validator** + +Receives paths to three files: +1. Original audit report (`{audit_report_path}`) +2. Validated report (`validated_report_{source}.md`) +3. Validation summary (`validation_summary_{source}.md`) + +Instructions: +> You are cross-validating three audit artifacts for consistency. Read all three files. +> Check: +> 1. **No accidental deletions** — every finding in the validated report traces to a finding in the original +> 2. **No accidental survivors** — every CONTESTED finding in the summary is absent from the validated report +> 3. **No exception-warranted leakage** — no VALID BUT EXCEPTION WARRANTED finding appears in the validated report's `## Validated Findings` section +> 4. **Structural integrity** — valid markdown, Summary Table counts match actual finding count, finding IDs sequential, no orphaned references +> 5. **Count reconciliation** — N_valid + N_exception + N_contested equals original total; consistent between summary and validated report +> Return a structured discrepancy report. If no issues found, return "CROSS-VALIDATION PASSED". +> Do NOT create any files. Return structured text only. + +Output format: +``` +## Cross-Validation Report + +Status: PASSED | DISCREPANCIES FOUND + +### Discrepancy [{N}]: {type} +- **Finding ID**: {id} +- **Issue**: {what is wrong} +- **Expected**: {what should be there} +- **Actual**: {what is there} +``` + +**Subagent B — Ticket Grouper** + +Receives the validated report path. + +Instructions: +> You are analyzing validated audit findings to propose ticket groupings. Read the validated report. +> For each finding, assess scope: lines of code affected, complexity, criticality, file overlap. +> Grouping rules: +> - **Standalone ticket**: finding is large in scope (many files/lines), complex refactor, or touches a critical path +> - **Grouped ticket**: finding is small, low-risk, non-conflicting. Group same-category small findings together. +> - **Conflict awareness**: findings touching the same file(s) must be in the same ticket or explicitly sequenced +> - No rigid severity-to-grouping rule: a HIGH can be grouped if small; a LOW can be standalone if complex +> +> Return a grouping manifest listing each proposed ticket with: +> - Ticket title (descriptive, scoped) +> - Finding IDs included (e.g., P1-F09, P1-F11, P3-F18) +> - Rationale for grouping or standalone +> - Estimated scope: small / medium / large +> - File overlap notes (which findings touch the same files) +> Do NOT create any files. Return structured text only. + +Output format: +``` +## Grouping Manifest + +### Ticket Group 1: {title} +- **Finding IDs**: {id1}, {id2}, ... +- **Rationale**: {why grouped or standalone} +- **Scope**: small | medium | large +- **File overlap**: {files touched by multiple findings in this group, or "none"} + +### Ticket Group 2: {title} +... +``` + +### Step 7 — Apply Cross-Validation Corrections + +After both parallel subagents return: + +**From Cross-Validator:** +- If status is `CROSS-VALIDATION PASSED`: proceed directly to Step 8. +- If discrepancies found: for each discrepancy, re-read the relevant section of the validated + report and validation summary, write the corrected content to a `.tmp` file first, then + atomically move it over the original (to prevent partial-write corruption), and note the + correction applied. Limit to at most 3 correction passes; after 3 passes, record any + remaining discrepancies and continue to Step 8. +- Corrections are writes to existing output files only — no new findings are introduced. + +**From Ticket Grouper:** +- Record the grouping manifest (it will be written to disk in Step 8). +- If the grouper returned fewer than 1 group: treat the entire validated report as a single ticket. + +### Step 8 — Split Validated Report by Grouping Manifest + +Before writing any ticket body files, verify `$AUTOSKILLIT_TEMP` is non-empty +(`test -n "${AUTOSKILLIT_TEMP}"`); abort with an error message if unset to prevent +path collapse to filesystem root. + +For each ticket group in the grouping manifest: + +1. Extract the subset of findings assigned to this group from the validated report. +2. Build a per-ticket body file with: + - The `validated: true` sentinel on line 1 + - An H1 heading: `# {ticket title}` (from grouping manifest) + - A subset Summary Table (only the rows for included finding IDs) + - Only the `## Validated Findings` sub-sections for included finding IDs + - A footer: `*Part of validated {source} audit — see full report for remaining tickets.*` +3. Write to: `{{AUTOSKILLIT_TEMP}}/validate-audit-{YYYY-MM-DD_HHMMSS}/ticket_body_{source}_{N}.md` + where `{N}` is 1-indexed from the grouping manifest. + +Also write the grouping manifest itself to: +`{{AUTOSKILLIT_TEMP}}/validate-audit-{YYYY-MM-DD_HHMMSS}/grouping_manifest_{source}.md` + +The grouping manifest file is the structured text returned by the ticket grouper subagent, +prefixed with: +```markdown +# Ticket Grouping Manifest — {source} ({YYYY-MM-DD}) + +**Validated report:** {validated_report_path} +**Total groups:** {N} + +--- +``` + +### Step 9 — Interactive vs Headless Approval Detect headless mode: run `echo "${AUTOSKILLIT_HEADLESS:-0}"` via Bash. Output `1` means headless. -**Headless mode:** Write both output files immediately without prompting. Print to terminal: +**Headless mode:** Write all output files immediately without prompting. Print to terminal: ``` [validate-audit] Done. Valid: {N_valid} | Exceptions: {N_exception} | Contested: {N_contested} - Report: {{AUTOSKILLIT_TEMP}}/validate-audit/validated_report_{source}_{ts}.md - Contested: {{AUTOSKILLIT_TEMP}}/validate-audit/contested_findings_{source}_{ts}.md + Summary: {validation_summary_path} + Manifest: {grouping_manifest_path} + Tickets: {ticket_body_1_path} + {ticket_body_2_path} (one line per ticket group) + Contested: {contested_findings_path} (omit if N_contested == 0) + Report: {validated_report_path} +validated_report_path = {validated_report_path} +verdict = validated ``` -(Omit the "Contested:" line if `N_contested == 0`.) - **Interactive mode:** Display the validation status table (verdict counts), then ask: > Write validated report and contested findings files? [Y/n] -On Y or empty input, write both files. After writing, if `N_contested > 0`, offer: +On Y or empty input, write all files. After writing, offer: -> Run `/autoskillit:prepare-issue` for contested findings? [y/N] +> Run `/autoskillit:prepare-issue` for each ticket group? [Y/n] -If the user confirms, pass the contested findings file path to `prepare-issue`. +On Y, call `prepare-issue` for each ticket body file (in parallel). After issue creation, +append the validation summary to each created issue body using `gh issue edit --body-file`: +fetch the current issue body, verify the fetched body is non-empty (abort the append for +that issue if empty to avoid overwriting with summary-only content), append a horizontal +rule and the validation summary content, write the combined text to a temp file, then run +`gh issue edit {issue_number} --body-file` with that temp file. Do NOT use `gh issue comment`. --- ## Output Location +All output files are written under `{{AUTOSKILLIT_TEMP}}/validate-audit-{YYYY-MM-DD_HHMMSS}/`: + ``` -{{AUTOSKILLIT_TEMP}}/validate-audit/ -├── validated_report_{source}_{YYYY-MM-DD_HHMMSS}.md (always written) -└── contested_findings_{source}_{YYYY-MM-DD_HHMMSS}.md (when N_contested > 0) +{{AUTOSKILLIT_TEMP}}/validate-audit-{YYYY-MM-DD_HHMMSS}/ +├── validated_report_{source}.md (always written; VALID findings only) +├── contested_findings_{source}.md (when N_contested > 0) +├── validation_summary_{source}.md (always written; audit trail) +├── grouping_manifest_{source}.md (always written; ticket grouping) +└── ticket_body_{source}_{N}.md (one per ticket group, N ≥ 1) ``` -`{source}` is `arch`, `tests`, or `cohesion` based on the input report. +`{source}` is `arch`, `tests`, `cohesion`, or `feature_gates` based on the input report. ## Related Skills - `/autoskillit:audit-arch` — produces reports this skill validates - `/autoskillit:audit-tests` — produces reports this skill validates - `/autoskillit:audit-cohesion` — produces reports this skill validates +- `/autoskillit:audit-feature-gates` — produces reports this skill validates - `/autoskillit:prepare-issue` — offered interactively for contested findings diff --git a/.github/CLAUDE.md b/.github/CLAUDE.md new file mode 100644 index 0000000000..d7031832f2 --- /dev/null +++ b/.github/CLAUDE.md @@ -0,0 +1,48 @@ +# CI Workflow Constraints + +Rules for maintaining `.github/workflows/` files. Enforced for both human and AI contributors. + +## setup-uv parameter names + +The correct input parameter for pinning the uv version is `version`, not `uv-version`. + +```yaml +# CORRECT +- uses: astral-sh/setup-uv@v7 + with: + version: "0.9.21" + +# WRONG — silently ignored, installs latest uv instead of pinned version +- uses: astral-sh/setup-uv@v7 + with: + uv-version: "0.9.21" +``` + +`uv-version` is an *output* of the action (the resolved installed version), not an input. + +## Caching discipline + +Jobs that do not run `uv sync` MUST disable setup-uv caching to avoid poisoning the +wheel cache consumed by downstream jobs: + +```yaml +- uses: astral-sh/setup-uv@v7 + with: + version: "0.9.21" + enable-cache: false # required for jobs that skip uv sync +``` + +Jobs that run `uv sync --locked` may omit `enable-cache` (defaults to `auto`, enabled on +GitHub-hosted runners). + +## Rust toolchain scope + +`dtolnay/rust-toolchain` is only needed in jobs that compile native Python extensions +(i.e., jobs that run `uv sync`). Do not add it to preflight, lint, or other utility jobs +that only run `uv lock --check` or similar commands. + +## Lockfile verification + +The `preflight` job exists to validate the lockfile early and cheaply. It must NOT install +the full dependency tree — only `uv lock --check`. Adding `uv sync` to preflight defeats +the purpose of the job separation. \ No newline at end of file diff --git a/.github/workflows/patch-bump-integration.yml b/.github/workflows/patch-bump-develop.yml similarity index 79% rename from .github/workflows/patch-bump-integration.yml rename to .github/workflows/patch-bump-develop.yml index a4dc0a6171..7bbe647e9f 100644 --- a/.github/workflows/patch-bump-integration.yml +++ b/.github/workflows/patch-bump-develop.yml @@ -1,12 +1,12 @@ -name: Auto Patch Version Bump on Integration PR +name: Auto Patch Version Bump on Develop PR on: pull_request: types: [closed] - branches: [integration] + branches: [develop] concurrency: - group: patch-bump-integration + group: patch-bump-develop cancel-in-progress: false jobs: @@ -19,14 +19,14 @@ jobs: steps: - uses: actions/checkout@v4 with: - ref: integration + ref: develop fetch-depth: 0 token: ${{ secrets.GH_PAT }} - name: Install uv uses: astral-sh/setup-uv@v7 with: - uv-version: "0.9.21" + version: "0.9.21" - name: Install Rust toolchain uses: dtolnay/rust-toolchain@stable @@ -66,17 +66,8 @@ jobs: OLD_VERSION: ${{ steps.version.outputs.old }} NEW_VERSION: ${{ steps.version.outputs.new }} - - name: Update plugin.json - run: | - python3 - <<'EOF' - import json, pathlib, os - p = pathlib.Path("src/autoskillit/.claude-plugin/plugin.json") - data = json.loads(p.read_text()) - data["version"] = os.environ["NEW_VERSION"] - p.write_text(json.dumps(data, indent=2) + "\n") - EOF - env: - NEW_VERSION: ${{ steps.version.outputs.new }} + - name: Sync version artifacts + run: python3 scripts/sync_versions.py - name: Configure git auth for private deps run: git config --global url."https://${{ secrets.GH_USER }}:${{ secrets.GH_PAT }}@github.com/TalonT-Org/".insteadOf "https://github.com/TalonT-Org/" @@ -84,7 +75,7 @@ jobs: - name: Regenerate uv.lock run: uv lock - - name: Commit and push version bump to integration + - name: Commit and push version bump to develop run: | set -euo pipefail git config user.name "github-actions[bot]" @@ -94,5 +85,5 @@ jobs: echo "Nothing to commit" else git commit -m "chore: bump version to ${{ steps.version.outputs.new }}" - git push origin integration + git push origin develop fi diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 216317d450..3835a95e7a 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -21,7 +21,7 @@ jobs: - name: Install uv uses: astral-sh/setup-uv@v7 with: - uv-version: "0.9.21" + version: "0.9.21" - name: Compute new minor version id: version @@ -50,17 +50,8 @@ jobs: OLD_VERSION: ${{ steps.version.outputs.old }} NEW_VERSION: ${{ steps.version.outputs.new }} - - name: Update plugin.json - run: | - python3 - <<'EOF' - import json, pathlib, os - p = pathlib.Path("src/autoskillit/.claude-plugin/plugin.json") - data = json.loads(p.read_text()) - data["version"] = os.environ["NEW_VERSION"] - p.write_text(json.dumps(data, indent=2) + "\n") - EOF - env: - NEW_VERSION: ${{ steps.version.outputs.new }} + - name: Sync plugin.json and recipe versions + run: python3 scripts/sync_versions.py - name: Regenerate uv.lock run: uv lock diff --git a/.github/workflows/test-filter-audit.yml b/.github/workflows/test-filter-audit.yml new file mode 100644 index 0000000000..6a8148dd38 --- /dev/null +++ b/.github/workflows/test-filter-audit.yml @@ -0,0 +1,56 @@ +name: Test Filter Audit + +on: + schedule: + - cron: "17 3 * * 1" # Weekly Monday 3:17am UTC (offset avoids peak contention) + push: + branches: [main] + paths: + - "tests/**" + - "tests/conftest.py" + - "tests/_test_filter.py" + - "src/autoskillit/_test_filter.py" + - ".autoskillit/test-filter-manifest.yaml" + - "Taskfile.yml" + workflow_dispatch: + +permissions: + contents: read + +jobs: + audit: + name: Shadow-mode filter verification + runs-on: ubuntu-latest + timeout-minutes: 15 + env: + TMPDIR: /dev/shm/pytest-audit + + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 10 # Need history for HEAD~5 + + - name: Install uv + uses: astral-sh/setup-uv@v7 + with: + uv-version: "0.9.21" + + - name: Install go-task + uses: arduino/setup-task@v2 + with: + version: "3.43.3" + repo-token: ${{ secrets.GITHUB_TOKEN }} + + - name: Configure git auth for private deps + run: git config --global url."https://${{ secrets.GH_USER }}:${{ secrets.GH_PAT }}@github.com/TalonT-Org/".insteadOf "https://github.com/TalonT-Org/" + + - name: Install Rust toolchain + uses: dtolnay/rust-toolchain@stable + + - name: Set up environment + run: | + mkdir -p "$TMPDIR" + task install-worktree + + - name: Run test filter audit + run: scripts/verify-test-filter.sh HEAD~5 diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index c561c4188d..d18cbf6521 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -4,7 +4,7 @@ on: push: branches: [main, stable] pull_request: - branches: [main, integration, stable] + branches: [main, develop, stable] merge_group: permissions: @@ -32,10 +32,8 @@ jobs: - name: Install uv uses: astral-sh/setup-uv@v7 with: - uv-version: "0.9.21" - - - name: Install Rust toolchain - uses: dtolnay/rust-toolchain@stable + version: "0.9.21" + enable-cache: false - name: Verify lockfile is up to date run: uv lock --check @@ -56,11 +54,13 @@ jobs: steps: - uses: actions/checkout@v4 + with: + fetch-depth: 0 - name: Install uv uses: astral-sh/setup-uv@v7 with: - uv-version: "0.9.21" + version: "0.9.21" - name: Install go-task uses: arduino/setup-task@v2 @@ -88,4 +88,19 @@ jobs: run: uv run autoskillit recipes render - name: Run tests - run: task test-all + env: + AUTOSKILLIT_FILTER_STATS_FILE: ${{ github.workspace }}/.autoskillit/temp/filter-stats.json + run: | + mkdir -p .autoskillit/temp + if [[ "${{ github.event_name }}" == "pull_request" && "${{ github.base_ref }}" == "develop" ]]; then + export AUTOSKILLIT_TEST_FILTER=conservative + fi + task test-all + + - name: Upload filter stats + if: always() + uses: actions/upload-artifact@v4 + with: + name: filter-stats-${{ matrix.os }} + path: .autoskillit/temp/filter-stats.json + if-no-files-found: ignore diff --git a/.github/workflows/version-bump.yml b/.github/workflows/version-bump.yml index 9ef0c58b8b..c2866ae592 100644 --- a/.github/workflows/version-bump.yml +++ b/.github/workflows/version-bump.yml @@ -11,7 +11,7 @@ jobs: contents: write if: > github.event.pull_request.merged == true && - github.event.pull_request.head.ref == 'integration' + github.event.pull_request.head.ref == 'develop' runs-on: ubuntu-latest steps: @@ -24,7 +24,7 @@ jobs: - name: Install uv uses: astral-sh/setup-uv@v7 with: - uv-version: "0.9.21" + version: "0.9.21" - name: Install Rust toolchain uses: dtolnay/rust-toolchain@stable @@ -59,17 +59,8 @@ jobs: OLD_VERSION: ${{ steps.version.outputs.old }} NEW_VERSION: ${{ steps.version.outputs.new_main }} - - name: Update plugin.json (main) - run: | - python3 - <<'EOF' - import json, pathlib, os - p = pathlib.Path("src/autoskillit/.claude-plugin/plugin.json") - data = json.loads(p.read_text()) - data["version"] = os.environ["NEW_VERSION"] - p.write_text(json.dumps(data, indent=2) + "\n") - EOF - env: - NEW_VERSION: ${{ steps.version.outputs.new_main }} + - name: Sync version artifacts (main) + run: python3 scripts/sync_versions.py - name: Configure git auth for private deps run: git config --global url."https://${{ secrets.GH_USER }}:${{ secrets.GH_PAT }}@github.com/TalonT-Org/".insteadOf "https://github.com/TalonT-Org/" @@ -90,13 +81,13 @@ jobs: git push origin main fi - - name: Checkout integration branch + - name: Checkout develop branch run: | set -euo pipefail - git fetch origin integration - git checkout -B integration origin/integration + git fetch origin develop + git checkout -B develop origin/develop - - name: Read integration current version + - name: Read develop current version id: int_version run: | set -euo pipefail @@ -107,7 +98,7 @@ jobs: [ -n "$CURRENT" ] || { echo "ERROR: int_version: CURRENT is empty — checkout may have failed or pyproject.toml is absent" >&2; exit 1; } echo "old=$CURRENT" >> "$GITHUB_OUTPUT" - - name: Update pyproject.toml (integration) + - name: Update pyproject.toml (develop) run: | python3 - <<'EOF' import pathlib, os, sys @@ -117,7 +108,7 @@ jobs: old_parts = tuple(int(x) for x in old_str.split(".")) new_parts = tuple(int(x) for x in new_str.split(".")) if new_parts <= old_parts: - print(f"ERROR: new_int {new_str!r} would downgrade integration from {old_str!r}", file=sys.stderr) + print(f"ERROR: new_int {new_str!r} would downgrade develop from {old_str!r}", file=sys.stderr) sys.exit(1) old_ver = f'version = "{old_str}"' new_ver = f'version = "{new_str}"' @@ -132,28 +123,19 @@ jobs: OLD_VERSION: ${{ steps.int_version.outputs.old }} NEW_VERSION: ${{ steps.version.outputs.new_int }} - - name: Update plugin.json (integration) - run: | - python3 - <<'EOF' - import json, pathlib, os - p = pathlib.Path("src/autoskillit/.claude-plugin/plugin.json") - data = json.loads(p.read_text()) - data["version"] = os.environ["NEW_VERSION"] - p.write_text(json.dumps(data, indent=2) + "\n") - EOF - env: - NEW_VERSION: ${{ steps.version.outputs.new_int }} + - name: Sync version artifacts (develop) + run: python3 scripts/sync_versions.py - - name: Regenerate uv.lock (integration) + - name: Regenerate uv.lock (develop) run: uv lock - - name: Commit and push integration version bump + - name: Commit and push develop version bump run: | set -euo pipefail git add pyproject.toml src/autoskillit/.claude-plugin/plugin.json uv.lock if git diff --cached --quiet; then echo "Nothing to commit" else - git commit -m "chore: bump integration version to ${{ steps.version.outputs.new_int }}" - git push origin integration + git commit -m "chore: bump develop version to ${{ steps.version.outputs.new_int }}" + git push origin develop fi diff --git a/.gitignore b/.gitignore index cd41f4ac77..5438fa4ffb 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,6 @@ __pycache__/ *.egg-info/ +src/autoskillit/franchise/ dist/ build/ .venv/ @@ -8,6 +9,9 @@ build/ temp/ .mypy_cache/ .pytest_cache/ +.coverage +.testmondata +.testmondata-journal *.code-workspace @@ -22,5 +26,6 @@ src/autoskillit/hooks/hooks.json # Runtime sync state — generated on each install, not source .autoskillit/sync_manifest.json +.autoskillit/validation-errors/ .secrets.yaml .autoskillit/.secrets.yaml \ No newline at end of file diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index b9599a388f..e452e0da4f 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -15,7 +15,7 @@ repos: - id: mypy name: mypy - entry: mypy src/ --ignore-missing-imports + entry: mypy src/ tests/_test_filter.py --ignore-missing-imports language: system pass_filenames: false @@ -39,6 +39,27 @@ repos: files: ^(docs/|README\.md|src/autoskillit/skills/|src/autoskillit/recipes/|src/autoskillit/core/_type_constants\.py) pass_filenames: false + - id: check-version-consistency + name: Check version consistency across artifacts + language: system + entry: python scripts/sync_versions.py --check + files: '(pyproject\.toml|plugin\.json|recipes/.*\.yaml)$' + pass_filenames: false + + - id: check-tool-annotations + name: Check MCP tool readOnlyHint annotations + language: system + entry: python scripts/check_tool_annotations.py + files: ^src/autoskillit/server/tools_ + pass_filenames: false + + - id: check-sub-claude-md + name: Check sub-CLAUDE.md file table completeness + language: system + entry: python scripts/check_sub_claude_md.py + files: ^(tests/|src/autoskillit/).*(\.py|CLAUDE\.md)$ + pass_filenames: false + - repo: https://github.com/gitleaks/gitleaks rev: v8.30.0 hooks: diff --git a/CLAUDE.md b/CLAUDE.md index 4f2413ee91..a1de805a56 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -28,9 +28,18 @@ A Claude Code plugin that orchestrates automated skill-driven workflows using he * **No Backward Compatibility Hacks**: No comments about dead code. Remove dead code entirely. * **Avoid Redundancy**: Do not duplicate logic or utilities. * **Use Current Package Versions**: Web search for current stable versions when adding dependencies. - * **Version Bumps**: When bumping the package version, update `pyproject.toml` and run `task sync-plugin-version && uv lock`; then search tests for hardcoded version strings (e.g. `AUTOSKILLIT_INSTALLED_VERSION` monkeypatches) and update them. + * **Version Bumps**: When bumping the package version, update `pyproject.toml` and run `task sync-versions && uv lock`; then search tests for hardcoded version strings (e.g. `AUTOSKILLIT_INSTALLED_VERSION` monkeypatches) and update them. * **Run pre-commit before committing**: Always run `pre-commit run --all-files` before committing. Do not skip this step even when code appears clean — hooks auto-fix formatting and abort the commit, requiring re-stage and retry. * **Hook Renames**: Renaming a hook script under `src/autoskillit/hooks/` must update `HOOK_REGISTRY` in `hook_registry.py` AND add the old basename to `RETIRED_SCRIPT_BASENAMES` in the same commit. `test_no_retired_name_has_a_live_file` will fail otherwise. + * **Skill Renames**: Renaming a skill under `src/autoskillit/skills_extended/` (or `src/autoskillit/skills/`) must update the skill's `SKILL.md` `name:` field AND add the old directory name to `RETIRED_SKILL_NAMES` in `src/autoskillit/core/types/_type_constants.py` in the SAME commit. `test_no_retired_skill_name_has_a_live_directory` will fail otherwise. + * **Grep tool uses ripgrep (ERE) syntax**: Use `|` for OR-alternation in Grep tool `pattern` + arguments. `\|` is Bash grep BRE syntax — ripgrep treats it as a literal backslash-pipe + and returns 0 results. Example: `Grep(pattern="foo|bar")` not `Grep(pattern="foo\|bar")`. + In production, `grep_pattern_lint_guard.py` (hooks/guards/) blocks `\|` calls with a correction hint. The silent zero-results failure described above only occurs in hook-inactive contexts (e.g., raw CLI usage without the plugin installed). + * **Worktree Init Prohibition**: Never run `autoskillit init` from within a git worktree. `sync_hooks_to_settings()` will raise `RuntimeError` if `pkg_root()` resolves to a worktree. Use `task install-worktree` for worktree setup — it does NOT call `init`. + * **Naming convention — `*Def` vs `*Spec` suffixes**: + - `*Def` — static definition of a registered entity (e.g., `HookDef`, `PackDef`, `FeatureDef`, `RuleDef`). Typically a `NamedTuple` or `@dataclass(frozen=True)`, used as elements in a registry or lookup table. Typically lives in `core/`; stdlib-only types importable from hook scripts may live at the package root (e.g., `HookDef` in `hook_registry.py`). + - `*Spec` — behavioral specification or validation rule (e.g., `ExperimentTypeSpec`, `WriteBehaviorSpec`). Typically a `@dataclass` or `TypedDict` configuring a pipeline or validation stage. Typically lives in `recipe/` or domain layers; `*Spec` types used by IL-0 core protocols live in `core/` (e.g., `WriteBehaviorSpec` in `core/types/_type_results.py`). ### **3.2. File System** @@ -38,19 +47,58 @@ A Claude Code plugin that orchestrates automated skill-driven workflows using he * **Do Not Add Root Files**: Never create new root files unless explicitly required. * **Never commit unless told to do so** -### **3.3. Code Index MCP Usage** - - * **Initialize before use**: Always call `set_project_path` with the project root as the first action in any session that will use code-index tools. Without this call, all code-index tools fail with "Project path not set" and cascade-cancel sibling parallel calls. - * **Index is locked to the main project root**: The `code-index` MCP server is indexed against the source repo and must never be redirected to a worktree or branch. Its value is for exploration before code changes — at that point any worktree is identical to main, so the index is accurate regardless of where you are working. - * **Prefer code-index tools over native search** when exploring the codebase: `find_files`, `search_code_advanced`, `get_file_summary`, `get_symbol_body` (includes `called_by` call graph). - * **Do not rely on code-index for code added or modified during a branch** — use Read/Grep directly for that. - * **Fall back to native Grep/Glob** for multiline patterns or paths outside the project root. - -### **3.4. CLAUDE.md Modifications** +### **3.3. CLAUDE.md Modifications** * **Correcting existing content is permitted**: If you discover that CLAUDE.md contains inaccurate information (wrong file paths, stale names, incorrect tool attributions), you may correct it without being asked. * **Adding new content requires explicit instruction**: Never add new sections, bullet points, entries, or any new information to CLAUDE.md unless the user has explicitly asked you to update or extend it. Corrections to existing facts ≠ permission to expand scope. +### **3.4. GitHub API Call Discipline** + + * **Batch inline review comments** via `POST /pulls/{N}/reviews` with `comments[]` array — never post comments individually unless the batch call fails. + * **Batch GraphQL mutations** via aliases (N mutations in 1 request = 5 pts total, not N × 5 pts). Use for thread resolution, bulk PR queries, and any operation touching multiple entities. + * **Delay 1s between POST/PATCH/PUT/DELETE calls** — add `sleep 1` (in shell) or `await asyncio.sleep(1)` (in Python) between consecutive mutating GitHub API calls. + * **Pre-fetch entity lists** upfront in a single call; pass results via manifest files or variables rather than querying per-entity. + * **Use `--json` field selection** to request only needed fields from `gh` CLI commands. + * **Prefer GraphQL** for multi-entity reads — alias queries cost 1 point regardless of entity count. + * **Never check response body for `comments` array length** after `POST /pulls/{N}/reviews` — GitHub does not echo back the comments array; HTTP 200 is the success signal. + +### **3.5. GitHub Issue Body is the Source of Truth** + + * **Never use `gh issue comment`** to communicate issue status, triage feedback, tracking + info, or occurrence data. Comments are not read by downstream consumers and fragment the + record. + * **All issue content updates must use `gh issue edit --body-file`**: fetch the current + body, append the new section, write to `${{AUTOSKILLIT_TEMP}}`, then run + `gh issue edit {number} --body-file "$FILE"`. + * The `update_issue_body()` method on `GitHubFetcher` is the Python API equivalent. + +### **3.6. Pyright LSP Usage** + +The `LSP` tool provides type-aware code intelligence via Pyright. Use it for precise +navigation instead of grep when tracing symbols through imports, re-exports, or protocols. + +**Available operations** (all take `filePath`, `line`, `character` — 1-based): + +| Operation | Use case | +|-----------|----------| +| `goToDefinition` | Jump to where a symbol is defined (follows imports/re-exports) | +| `findReferences` | Find all usages of a symbol across the codebase | +| `documentSymbol` | List all classes, functions, and variables in a file | +| `goToImplementation` | Find concrete implementations of a Protocol or ABC | +| `prepareCallHierarchy` | Get the call hierarchy item at a position | +| `incomingCalls` | Find all callers of a function/method | +| `outgoingCalls` | Find all functions/methods called by a function | + +**When to use LSP over grep:** +- Tracing a symbol through re-exports (e.g., `core/__init__.py` -> actual definition) +- Finding all implementations of a Protocol +- Mapping call hierarchies (who calls X, what does X call) +- Understanding a file's structure before editing + +**When grep is still better:** +- Searching for string literals, comments, or non-symbol patterns +- Searching across non-Python files (YAML, JSON, markdown) + ## **4. Testing Guidelines** The project uses pytest with pytest-asyncio. Tests run in parallel via pytest-xdist (`-n 4`). All tests must be safe for parallel execution. @@ -60,10 +108,11 @@ The project uses pytest with pytest-asyncio. Tests run in parallel via pytest-xd * **Add tests for new features** * **Follow existing test patterns** in `tests/` — avoid test code redundancy * **Worktree setup**: Use `task install-worktree` in worktrees. Never hardcode `uv venv`/`pip install` in skills or plans. + * **Filtered tests**: `task test-filtered` runs path-filtered tests (defaults `AUTOSKILLIT_TEST_FILTER=conservative`). Set `AUTOSKILLIT_TEST_BASE_REF` to control the diff base. See `tests/CLAUDE.md` for filter modes and algorithm details. ## **5. Pre-commit Hooks** -Install hooks after cloning: `pre-commit install`. Run manually with `pre-commit run --all-files`. +Run manually with `pre-commit run --all-files`. Configured hooks: ruff format (auto-fix), ruff check (auto-fix), mypy type checking, uv lock check, gitleaks secret scanning. @@ -83,211 +132,38 @@ generic_automation_mcp/ └── pyproject.toml ``` -`src/autoskillit/`: +`src/autoskillit/` packages — each has its own CLAUDE.md with file-level detail (except `recipes/`, `skills/`, and `skills_extended/` — CLAUDE.md files for these are pending): + +| Package | IL | Purpose | +|---|---|---| +| `./` | — | Package root: `__init__`, `__main__`, `hook_registry`, `version`, `_test_filter`, `_llm_triage`, `smoke_utils` | +| `core/` | IL-0 | Foundation — types/, runtime/, paths, IO, feature flags (zero autoskillit imports) | +| `config/` | IL-1 | `AutomationConfig` + Dynaconf loader + 24 leaf dataclasses | +| `pipeline/` | IL-1 | Pipeline state — `ToolContext` DI, gate, audit log, telemetry | +| `execution/` | IL-1 | Headless sessions (headless/, process/, merge_queue/, session/), CI/GitHub | +| `workspace/` | IL-1 | Clone management, worktrees, skill resolution | +| `planner/` | IL-1 | Progressive resolution planner — phases, assignments, WPs, validation | +| `recipe/` | IL-2 | Recipe schema, validation, semantic rules/ | +| `migration/` | IL-2 | Versioned migration engine + failure store | +| `fleet/` | IL-2 | Campaign dispatch, semaphore, sidecar, liveness, state persistence | +| `server/` | IL-3 | FastMCP server — tools/, kitchen gating, session-type dispatch | +| `cli/` | IL-3 | CLI — doctor/, update/, fleet/ subcommands, ui/, session/ management | +| `hooks/` | — | Claude Code hook scripts — guards/, formatters/ | +| `recipes/` | — | Bundled recipe YAML + contracts, diagrams, sub-recipes | +| `skills/` | — | Tier 1 skills: open-kitchen, close-kitchen, sous-chef | +| `skills_extended/` | — | Tier 2 (interactive) + Tier 3 (pipeline) skills, incl. arch-lens-* (13), exp-lens-* (18), vis-lens-* (12) | + +**Session diagnostics logs** live at `~/.local/share/autoskillit/logs/` (Linux) or `~/Library/Application Support/autoskillit/logs/` (macOS). Override with `linux_tracing.log_dir`. Session directories are named by Claude Code session UUID when available (parsed from stdout, or discovered from JSONL filename via Channel B (the JSONL stream written by the Claude Code subprocess)). Fallback: `no_session_{timestamp}`. Query the index: `jq 'select(.success == false)' ~/.local/share/autoskillit/logs/sessions.jsonl`. -``` -├── __init__.py -├── __main__.py -├── _llm_triage.py # Contract staleness triage (Haiku subprocess) -├── smoke_utils.py # Callables for smoke-test pipeline run_python steps -├── hook_registry.py # HookDef, HOOK_REGISTRY, generate_hooks_json -├── version.py # Version health utilities (L0) -├── .claude-plugin/ # plugin.json -├── .mcp.json -│ -├── core/ # L0 foundation (zero autoskillit imports) -│ ├── __init__.py # Re-exports public surface -│ ├── io.py # atomic_write, ensure_project_temp, YAML helpers -│ ├── logging.py -│ ├── paths.py # pkg_root(), is_git_worktree() -│ ├── types.py # Re-export hub for _type_*.py -│ ├── _type_enums.py # StrEnums -│ ├── _type_subprocess.py -│ ├── _type_constants.py # GATED_TOOLS, FREE_RANGE_TOOLS, SKILL_TOOLS, etc. -│ ├── _type_results.py # LoadResult, SkillResult, FailureRecord, CleanupResult, etc. -│ ├── _type_protocols.py # Protocols: GatePolicy, HeadlessExecutor, CIWatcher, etc. -│ ├── _type_helpers.py -│ ├── _claude_env.py # IDE-scrubbing canonical env builder for claude subprocesses -│ ├── _terminal_table.py # L0 color-agnostic terminal table primitive -│ ├── branch_guard.py -│ ├── claude_conventions.py # Skill discovery directory layout constants -│ ├── github_url.py # parse_github_repo -│ ├── kitchen_state.py # Kitchen-open session marker (stdlib-only; readable from hooks) -│ └── readiness.py # Filesystem readiness sentinel primitives for MCP server startup (L0) -│ -├── config/ # L1 -│ ├── __init__.py -│ ├── defaults.yaml -│ ├── ingredient_defaults.py -│ └── settings.py # Dataclass config + dynaconf layered resolution -│ -├── pipeline/ # L1 pipeline state -│ ├── __init__.py -│ ├── audit.py # FailureRecord, DefaultAuditLog -│ ├── background.py # DefaultBackgroundSupervisor -│ ├── context.py # ToolContext DI container -│ ├── gate.py # DefaultGateState, gate_error_result -│ ├── mcp_response.py # Per-tool response size tracking -│ ├── telemetry_fmt.py # Canonical token/timing display -│ ├── timings.py -│ ├── tokens.py -│ └── pr_gates.py # is_ci_passing, is_review_passing, partition_prs -│ -├── execution/ # L1 -│ ├── __init__.py -│ ├── commands.py # Claude{Interactive,Headless}Cmd builders -│ ├── db.py # Read-only SQLite with defence-in-depth -│ ├── diff_annotator.py # Diff annotation + findings filter for review-pr -│ ├── headless.py # Headless Claude session orchestration -│ ├── _headless_scan.py # Write-path JSONL scanning (extracted from headless.py) -│ ├── linux_tracing.py # /proc + psutil process tracing (Linux) -│ ├── anomaly_detection.py # Post-hoc anomaly detection over snapshots -│ ├── session_log.py # XDG-aware session diagnostics log writer -│ ├── recording.py # Record/replay subprocess runners via api-simulator -│ ├── process.py # Facade re-exporting from _process_*.py -│ ├── _process_io.py -│ ├── _process_jsonl.py -│ ├── _process_kill.py -│ ├── _process_monitor.py -│ ├── _process_pty.py -│ ├── _process_race.py # RaceAccumulator, resolve_termination -│ ├── quota.py # QuotaStatus, cache, check_and_sleep_if_needed -│ ├── ci.py # GitHub Actions CI watcher (httpx, never raises) -│ ├── merge_queue.py # GitHub merge queue watcher -│ ├── github.py # GitHub issue fetcher -│ ├── session.py # ClaudeSessionResult, extract_token_usage -│ ├── remote_resolver.py # upstream > origin, clone-aware -│ ├── testing.py # Pytest output parsing + pass/fail adjudication -│ └── pr_analysis.py # extract_linked_issues, DOMAIN_PATHS, partition_files_by_domain -│ -├── workspace/ # L1 -│ ├── __init__.py -│ ├── cleanup.py # CleanupResult, preserve list -│ ├── clone.py # Clone-based run isolation -│ ├── session_skills.py # Per-session ephemeral skill dirs; subset filtering -│ ├── clone_registry.py # Shared file-based coordination for deferred cleanup -│ ├── skills.py # SkillResolver — bundled skill listing -│ └── worktree.py -│ -├── recipe/ # L2 -│ ├── __init__.py -│ ├── contracts.py # Contract card generation + staleness triage -│ ├── io.py # load_recipe, list_recipes, iter_steps_with_context -│ ├── loader.py # Path-based recipe metadata utilities -│ ├── _api.py # Orchestration API -│ ├── diagrams.py # Flow diagram generation + staleness detection -│ ├── experiment_type_registry.py # ExperimentTypeSpec, load_all_experiment_types -│ ├── registry.py # RuleFinding, RuleSpec, semantic_rule decorator -│ ├── repository.py -│ ├── _analysis.py # Step graph building + dataflow analysis -│ ├── rules_blocks.py # Block-level budget rules (block-run-cmd-budget, etc.) -│ ├── rules_bypass.py -│ ├── rules_ci.py -│ ├── rules_clone.py -│ ├── rules_cmd.py -│ ├── rules_contracts.py -│ ├── rules_dataflow.py -│ ├── rules_fixing.py -│ ├── rules_graph.py -│ ├── rules_inputs.py -│ ├── rules_isolation.py -│ ├── rules_merge.py -│ ├── rules_packs.py -│ ├── rules_reachability.py # Symbolic reachability rules (capture-inversion-detection, event-scope-requires-upstream-capture) -│ ├── rules_recipe.py -│ ├── rules_skill_content.py -│ ├── rules_skills.py -│ ├── rules_tools.py -│ ├── rules_verdict.py -│ ├── rules_worktree.py # Semantic validation rule modules -│ ├── _skill_placeholder_parser.py -│ ├── schema.py # Recipe, RecipeStep, DataFlowWarning -│ ├── staleness_cache.py -│ └── validator.py # validate_recipe, analyze_dataflow -│ -├── migration/ # L2 -│ ├── __init__.py -│ ├── engine.py # MigrationEngine, adapter ABC hierarchy -│ ├── _api.py # check_and_migrate -│ ├── loader.py # Migration note discovery + version chaining -│ └── store.py # FailureStore (JSON, atomic writes) -│ -├── server/ # L3 FastMCP server -│ ├── __init__.py # FastMCP app, kitchen gating, headless tool reveal -│ ├── git.py # Merge workflow for merge_worktree -│ ├── _editable_guard.py # Pre-deletion editable install guard (stdlib-only) -│ ├── _lifespan.py # FastMCP lifespan: recorder teardown on shutdown -│ ├── _wire_compat.py # Claude Code wire-format sanitization middleware -│ ├── helpers.py -│ ├── tools_kitchen.py # open_kitchen, close_kitchen + recipe:// resource -│ ├── tools_ci.py # CI/merge-queue tool handlers -│ ├── tools_clone.py -│ ├── tools_execution.py # run_cmd, run_python, run_skill -│ ├── tools_git.py # merge_worktree, classify_fix, etc. -│ ├── tools_recipe.py -│ ├── tools_status.py # kitchen_status, reports, summaries, quota events, read_db -│ ├── tools_github.py # fetch_github_issue, get_issue_title, report_bug -│ ├── tools_issue_lifecycle.py # prepare/enrich/claim/release issue -│ ├── tools_pr_ops.py # get_pr_reviews, bulk_close_issues -│ ├── tools_workspace.py # test_check, reset_test_dir, reset_workspace -│ ├── _factory.py # Composition Root: make_context() -│ └── _state.py # Lazy init, plugin dir resolution -│ -├── cli/ # L3 CLI -│ ├── __init__.py -│ ├── _ansi.py # supports_color, NO_COLOR/TERM=dumb -│ ├── _terminal.py # terminal_guard() TTY restore -│ ├── _terminal_table.py # Re-export shim from core/_terminal_table -│ ├── _cook.py # cook: ephemeral skill session launcher -│ ├── _doctor.py # 16 project setup checks -│ ├── _hooks.py # PreToolUse hook registration helpers -│ ├── _init_helpers.py -│ ├── _installed_plugins.py # InstalledPluginsFile — canonical accessor for installed_plugins.json -│ ├── _install_info.py # InstallInfo, InstallType, detect_install(), comparison_branch(), dismissal_window(), upgrade_command() -│ ├── _marketplace.py # Plugin install/upgrade -│ ├── _mcp_names.py # MCP prefix detection -│ ├── _onboarding.py # First-run detection + guided menu -│ ├── _prompts.py # Orchestrator prompt builder -│ ├── _timed_input.py # timed_prompt() and status_line() CLI primitives -│ ├── _update.py # run_update_command(): first-class upgrade path for `autoskillit update` -│ ├── _update_checks.py # Unified startup update check: version/hook/source-drift signals, branch-aware dismissal -│ ├── _serve_guard.py # Async signal-guarded MCP server bootstrap (extracted from app.py) -│ ├── _workspace.py # Workspace clean helpers -│ └── app.py # CLI entry: serve, init, config, skills, recipes, doctor, update, etc. -│ -├── hooks/ # Claude Code PreToolUse/PostToolUse/SessionStart scripts -│ ├── __init__.py -│ ├── hooks.json # Plugin hook registration -│ ├── branch_protection_guard.py -│ ├── _hook_settings.py # Shared stdlib-only settings resolver for quota guard hooks -│ ├── quota_guard.py # Blocks run_skill when threshold exceeded -│ ├── quota_post_hook.py # Appends quota warning to run_skill output -│ ├── remove_clone_guard.py -│ ├── skill_cmd_guard.py -│ ├── skill_command_guard.py -│ ├── ask_user_question_guard.py # Blocks AskUserQuestion if kitchen is not open -│ ├── open_kitchen_guard.py -│ ├── unsafe_install_guard.py -│ ├── generated_file_write_guard.py -│ ├── headless_orchestration_guard.py -│ ├── pretty_output_hook.py # Dispatch entrypoint for MCP JSON → Markdown-KV reformatter -│ ├── _fmt_primitives.py # Payload dataclasses, token formatter, pipeline-mode + short-name -│ ├── _fmt_execution.py # run_skill, run_cmd, test_check, merge_worktree formatters -│ ├── _fmt_status.py # token/timing summary, kitchen_status, clone_repo formatters -│ ├── _fmt_recipe.py # load_recipe, open_kitchen, list_recipes formatters -│ ├── token_summary_hook.py # Appends Token Usage Summary to PR body -│ └── session_start_hook.py # Injects open-kitchen reminder on resume -│ -├── migrations/ # Versioned migration YAML notes -├── recipes/ # Bundled recipe YAML + contracts/, diagrams/, sub-recipes/ -├── skills/ # Tier 1: open-kitchen, close-kitchen, sous-chef -└── skills_extended/ # Tier 2 (interactive) + Tier 3 (pipeline/automation) skills - # incl. arch-lens-* (13) and exp-lens-* (18) diagram families -``` +**CRITICAL**: When using subagents, invoke with `CLAUDE_CODE_EXIT_AFTER_STOP_DELAY=120000` to ensure subagents exit when finished. -**Session diagnostics logs** live at `~/.local/share/autoskillit/logs/` (Linux) or `~/Library/Application Support/autoskillit/logs/` (macOS). Override with `linux_tracing.log_dir`. Session directories are named by Claude Code session UUID when available (parsed from stdout, or discovered from JSONL filename via Channel B). Fallback: `no_session_{timestamp}`. Query the index: `jq 'select(.success == false)' ~/.local/share/autoskillit/logs/sessions.jsonl`. +**Import layer vs. orchestration level — disambiguation table:** -**CRITICAL**: When using subagents, invoke with `CLAUDE_CODE_EXIT_AFTER_STOP_DELAY=120000` to ensure subagents exit when finished. +| Notation | System | Example | Meaning | +|----------|--------|---------|---------| +| IL-N (single digit) | Import layer level | IL-0, IL-2 | Module's position in the import DAG | +| IL-NNN (three-digit) | Import-linter contract ID | IL-001, IL-009 | Specific pyproject.toml contract | +| L-N | Orchestration level | L0, L3 | Runtime session spawning tier (see `docs/orchestration-levels.md`) | ## 7. Session Diagnostics diff --git a/README.md b/README.md index 384fa9cb7f..5856c6be90 100644 --- a/README.md +++ b/README.md @@ -4,8 +4,8 @@ AutoSkillit is a Claude Code plugin that runs YAML recipes through a two-tier orchestrator. Bundled recipes turn GitHub issues into merged PRs by chaining -plan, dry-walkthrough, worktree, test, and PR-review skills against 42 MCP -tools and 109 bundled skills. +plan, dry-walkthrough, worktree, test, and PR-review skills against 48 MCP +tools and 133 bundled skills. https://github.com/user-attachments/assets/bcd910c8-7269-46d6-a496-53b2cb24d212 @@ -33,8 +33,8 @@ autoskillit order implementation ## What it does Each bundled recipe is a sequenced graph of skill invocations. The orchestrator -holds a kitchen of 40 kitchen-tagged MCP tools plus 2 free range tools (`open_kitchen`, -`close_kitchen`), launches headless Claude sessions for the heavy work, and +holds a kitchen of 44 kitchen-tagged MCP tools plus 4 free range tools (`open_kitchen`, +`close_kitchen`, `disable_quota_guard`, `reload_session`), launches headless Claude sessions for the heavy work, and routes verdicts through retry, merge, and review gates. The 5 bundled recipes are `implementation`, `implementation-groups`, `merge-prs`, `remediation`, and `research`. diff --git a/Taskfile.yml b/Taskfile.yml index b499f3c39d..5050003138 100644 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -38,7 +38,7 @@ tasks: set -o pipefail set +e - $PYTEST_CMD tests/ -q --tb=short -n 4 -m "not smoke" --disable-warnings -o "addopts=" --basetemp={{.PYTEST_TMPDIR}} -o "cache_dir={{.PYTEST_CACHEDIR}}" 2>&1 | tee "$TEST_OUTPUT" + $PYTEST_CMD tests/ -q --tb=short -n 4 -m "not smoke and not canary" --disable-warnings -o "addopts=" --basetemp={{.PYTEST_TMPDIR}} -o "cache_dir={{.PYTEST_CACHEDIR}}" 2>&1 | tee "$TEST_OUTPUT" PYTEST_EXIT=$? set +o pipefail set -e @@ -67,7 +67,7 @@ tasks: set -o pipefail set +e - $PYTEST_CMD tests/ -q --tb=short -n 4 -m "not smoke" --disable-warnings -o "addopts=" --basetemp={{.PYTEST_TMPDIR}} -o "cache_dir={{.PYTEST_CACHEDIR}}" 2>&1 | tee "$TEST_OUTPUT" + $PYTEST_CMD tests/ -q --tb=short -n 4 -m "not smoke and not canary" --disable-warnings -o "addopts=" --basetemp={{.PYTEST_TMPDIR}} -o "cache_dir={{.PYTEST_CACHEDIR}}" 2>&1 | tee "$TEST_OUTPUT" PYTEST_EXIT=$? set +o pipefail set -e @@ -86,6 +86,60 @@ tasks: exit 0 fi + test-filtered: + desc: Run path-filtered tests (delegates to test-check with filter env passthrough) + cmds: + - | + export AUTOSKILLIT_TEST_FILTER="${AUTOSKILLIT_TEST_FILTER:-conservative}" + task test-check + + coverage-audit: + desc: "Run pytest with coverage collection and compare against AST-derived function map" + deps: [install-worktree, _tmpdir-setup] + env: + PYTHONDONTWRITEBYTECODE: "1" + OMP_NUM_THREADS: "1" + TMPDIR: "{{.PYTEST_TMPDIR}}" + cmds: + - | + mkdir -p temp + + if [[ -d ".venv" ]]; then + PYTEST_CMD=".venv/bin/python -m pytest" + PYTHON=".venv/bin/python" + else + PYTEST_CMD="pytest" + PYTHON="python" + fi + + set +e + $PYTEST_CMD tests/ -q --tb=short \ + -n 4 \ + -m "not smoke and not canary" \ + --disable-warnings \ + -o "addopts=" \ + --basetemp="{{.PYTEST_TMPDIR}}" \ + -o "cache_dir={{.PYTEST_CACHEDIR}}" \ + --cov=src/autoskillit \ + --cov-context=test \ + --cov-branch \ + --cov-report=term-missing:skip-covered \ + --cov-report= + set -e + + echo "" + echo "--- Coverage AST Comparison ---" + $PYTHON scripts/compare-coverage-ast.py \ + --coverage-db .coverage \ + --output "temp/coverage-audit-$(date +%Y%m%d-%H%M%S).json" + + echo "" + echo "--- Building test-source map ---" + $PYTHON scripts/compare-coverage-ast.py \ + --mode build-test-source-map \ + --coverage-db .coverage \ + --output ".autoskillit/test-source-map.json" + test-smoke: desc: Run smoke tests (requires ANTHROPIC_API_KEY) deps: [_tmpdir-setup] @@ -109,7 +163,7 @@ tasks: set -o pipefail set +e - $PYTEST_CMD tests/recipe/test_smoke_pipeline.py -m smoke -v --tb=short -o "addopts=" --basetemp={{.PYTEST_TMPDIR}} -o "cache_dir={{.PYTEST_CACHEDIR}}" 2>&1 | tee "$TEST_OUTPUT" + $PYTEST_CMD tests/server/test_smoke_pipeline.py -m smoke -v --tb=short -o "addopts=" --basetemp={{.PYTEST_TMPDIR}} -o "cache_dir={{.PYTEST_CACHEDIR}}" 2>&1 | tee "$TEST_OUTPUT" PYTEST_EXIT=$? set +o pipefail set -e @@ -118,6 +172,41 @@ tasks: echo "Smoke test output saved to: $TEST_OUTPUT" exit $PYTEST_EXIT + test-smoke-research: + desc: Run research recipe smoke tests (requires ANTHROPIC_API_KEY) + deps: [_tmpdir-setup] + env: + PYTHONDONTWRITEBYTECODE: "1" + OMP_NUM_THREADS: "1" + RESEARCH_SMOKE_TEST: "1" + TMPDIR: "{{.PYTEST_TMPDIR}}" + preconditions: + - sh: test -n "$ANTHROPIC_API_KEY" + msg: "ANTHROPIC_API_KEY must be set" + cmds: + - | + mkdir -p temp + TEST_OUTPUT="temp/smoke-research-$(date +%Y-%m-%d_%H%M%S).txt" + echo "Running research smoke tests" + echo "Output: $TEST_OUTPUT" + + if [[ -d ".venv" ]]; then + PYTEST_CMD=".venv/bin/python -m pytest" + else + PYTEST_CMD="pytest" + fi + + set -o pipefail + set +e + $PYTEST_CMD tests/server/test_research_smoke_pipeline.py -m smoke -v --tb=short -o "addopts=" --basetemp={{.PYTEST_TMPDIR}} -o "cache_dir={{.PYTEST_CACHEDIR}}" 2>&1 | tee "$TEST_OUTPUT" + PYTEST_EXIT=$? + set +o pipefail + set -e + + echo "" + echo "Research smoke test output saved to: $TEST_OUTPUT" + exit $PYTEST_EXIT + test-smoke-record: desc: Record smoke test scenario by running autoskillit order with recording enabled env: @@ -137,18 +226,24 @@ tasks: cmds: - python scripts/check_doc_counts.py - sync-plugin-version: - desc: Sync plugin.json version from pyproject.toml + sync-versions: + desc: Sync all version-bearing files from pyproject.toml cmds: - - python scripts/sync_plugin_json.py + - python scripts/sync_versions.py sync-hooks-hash: desc: "Regenerate src/autoskillit/hooks/registry.sha256 from the live HOOK_REGISTRY" cmds: - uv run python -c "from autoskillit.hook_registry import HOOK_REGISTRY_HASH, HOOKS_DIR; (HOOKS_DIR / 'registry.sha256').write_text(HOOK_REGISTRY_HASH + '\n')" + compile-recipes: + desc: Pre-compile bundled recipe YAML files to JSON + cmds: + - uv run python scripts/compile_recipes.py + install-worktree: desc: Set up development environment (isolated venv in worktrees, direct install in main) + deps: [compile-recipes] status: - test -x .venv/bin/python - uv sync --check --all-extras 2>/dev/null @@ -167,12 +262,12 @@ tasks: uv venv --clear .venv uv pip install -e '.[dev]' --python .venv/bin/python fi - echo "Done: $(python --version 2>/dev/null || .venv/bin/python --version)" + echo "Done: $(.venv/bin/python --version)" install-dev: - desc: Install dev version of autoskillit from the integration branch + desc: Install dev version of autoskillit from the develop branch cmds: - - uv tool install --force "autoskillit[dev] @ git+https://github.com/TalonT-Org/AutoSkillit.git@integration" + - uv tool install --force "autoskillit[dev] @ git+https://github.com/TalonT-Org/AutoSkillit.git@develop" - "$(uv tool dir)/autoskillit/bin/autoskillit install" vendor-mermaid: @@ -187,3 +282,107 @@ tasks: -o src/autoskillit/assets/mermaid/mermaid.min.js echo "${VERSION}" > src/autoskillit/assets/mermaid/VERSION echo "Vendored mermaid ${VERSION} ($(wc -c < src/autoskillit/assets/mermaid/mermaid.min.js) bytes)" + + testmon-build: + desc: Build .testmondata DB by running full suite with testmon (serial, no xdist) + deps: [install-worktree, _tmpdir-setup] + env: + PYTHONDONTWRITEBYTECODE: "1" + OMP_NUM_THREADS: "1" + TMPDIR: "{{.PYTEST_TMPDIR}}" + AUTOSKILLIT_TEST_FILTER: "none" + cmds: + - | + mkdir -p temp + echo "Building .testmondata (serial run — this will take a while)" + if [[ -d ".venv" ]]; then + PYTEST_CMD=".venv/bin/python -m pytest" + else + PYTEST_CMD="pytest" + fi + + set -o pipefail + set +e + $PYTEST_CMD tests/ -q --tb=short --testmon \ + -m "not smoke and not canary" \ + --disable-warnings -o "addopts=" \ + --basetemp={{.PYTEST_TMPDIR}} \ + -o "cache_dir={{.PYTEST_CACHEDIR}}" \ + 2>&1 | tee "temp/testmon-build-$(date +%Y-%m-%d_%H%M%S).txt" + PYTEST_EXIT=$? + set +o pipefail + set -e + + if [ -f ".testmondata" ]; then + SIZE=$(du -h .testmondata | cut -f1) + echo "" + echo ".testmondata built successfully: $SIZE" + else + echo "" + echo "WARNING: .testmondata was not created" + fi + exit $PYTEST_EXIT + + testmon-run: + desc: Run tests incrementally with testmon selection (requires prior testmon-build) + deps: [install-worktree, _tmpdir-setup] + env: + PYTHONDONTWRITEBYTECODE: "1" + OMP_NUM_THREADS: "1" + TMPDIR: "{{.PYTEST_TMPDIR}}" + AUTOSKILLIT_TEST_FILTER: "none" + cmds: + - | + mkdir -p temp + if [ ! -f ".testmondata" ]; then + echo "ERROR: .testmondata not found. Run 'task testmon-build' first." + exit 1 + fi + + if [[ -d ".venv" ]]; then + PYTEST_CMD=".venv/bin/python -m pytest" + else + PYTEST_CMD="pytest" + fi + + XDIST_FLAGS="" + if [ "${TESTMON_XDIST:-0}" = "1" ]; then + XDIST_FLAGS="-n 4" + echo "Running testmon with xdist (-n 4)" + else + echo "Running testmon serial (set TESTMON_XDIST=1 for parallel)" + fi + + set -o pipefail + set +e + $PYTEST_CMD tests/ -q --tb=short --testmon $XDIST_FLAGS \ + -m "not smoke and not canary" \ + --disable-warnings -o "addopts=" \ + --basetemp={{.PYTEST_TMPDIR}} \ + -o "cache_dir={{.PYTEST_CACHEDIR}}" \ + 2>&1 | tee "temp/testmon-run-$(date +%Y-%m-%d_%H%M%S).txt" + PYTEST_EXIT=$? + set +o pipefail + set -e + + exit $PYTEST_EXIT + + testmon-benchmark: + desc: Compare testmon selection vs cascade selection for changed files + deps: [install-worktree] + cmds: + - | + if [ ! -f ".testmondata" ]; then + echo "ERROR: .testmondata not found. Run 'task testmon-build' first." + exit 1 + fi + + if [[ -d ".venv" ]]; then + PYTHON=".venv/bin/python" + else + PYTHON="python" + fi + + $PYTHON scripts/benchmark-testmon.py \ + --mode="${TESTMON_BENCH_MODE:-compare}" \ + ${TESTMON_CHANGED_FILES:+--changed-files="$TESTMON_CHANGED_FILES"} diff --git a/docs/README.md b/docs/README.md index 842b9ad257..3e605efdc5 100644 --- a/docs/README.md +++ b/docs/README.md @@ -1,8 +1,8 @@ # AutoSkillit documentation AutoSkillit is a Claude Code plugin that runs YAML recipes through a -two-tier orchestrator. The bundled recipes implement issue → plan → worktree -→ tests → PR → merge pipelines using 44 MCP tools and 109 bundled skills. +multi-level orchestrator. The bundled recipes implement issue → plan → worktree +→ tests → PR → merge pipelines using 52 MCP tools and 133 bundled skills. ## Start here @@ -16,6 +16,8 @@ two-tier orchestrator. The bundled recipes implement issue → plan → worktree - [update-checks.md](update-checks.md) — update checks, dismissal windows, `autoskillit update` - [faq.md](faq.md) — common questions - [glossary.md](glossary.md) — canonical terms +- [orchestration-levels.md](orchestration-levels.md) — L0–L3 orchestration hierarchy +- [version-pipeline.md](version-pipeline.md) — CI versioning pipeline, sync_versions.py, workflow reference ## Topic-based subdirectories @@ -23,6 +25,11 @@ two-tier orchestrator. The bundled recipes implement issue → plan → worktree - [skills/](skills/README.md) — Tier model, catalog, subsets, overrides - [execution/](execution/README.md) — architecture, tool access, orchestration - [safety/](safety/README.md) — hooks, workspace isolation -- [operations/](operations/README.md) — sprint guide, observability +- [operations/](operations/README.md) — observability - [developer/](developer/README.md) — contributing, diagnostics, end-turn hazards - [examples/](examples/README.md) — end-to-end pipeline runs +- [decisions/](decisions/README.md) — architecture decision records +- [design/](design/README.md) — design specifications for planned features and skills +- [research/experiment-type-rationale.md](research/experiment-type-rationale.md) — dimension-weight rationale for experiment types +- [research/silent-type-convention.md](research/silent-type-convention.md) — silent type detection and advisory convention +- [research/audit-trail-format.md](research/audit-trail-format.md) — audit/ artifact structure and lifecycle diff --git a/docs/cli.md b/docs/cli.md index ee30505148..8ee9e6215c 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -25,7 +25,7 @@ Register AutoSkillit as a Claude Code plugin. 3. Installs the plugin 4. Syncs hooks to `settings.json` -Run after every upgrade. +Syncs hooks and plugin cache. Called automatically by `autoskillit update`. --- @@ -82,14 +82,15 @@ Run health checks on your setup. **Flags:** - `--output-json` — Output results as JSON -Runs 14 checks enumerated by `run_doctor` in `cli/_doctor.py` (12 numbered -plus the lettered sub-checks `4b` *Config secrets placement* and `7b` *Hook -registry drift*). The checks cover stale MCP servers, plugin registration, -PATH, project config, secrets placement, version consistency, hook health, -hook registration, hook registry drift, recipe version health, gitignore -completeness, secret-scanning hook, editable install source, and stale entry -points. See [installation.md](installation.md#post-install-verification) for -the full table. +Runs 28 checks (up to 33 with fleet enabled) enumerated by `run_doctor` in +`cli/_doctor.py` — 23 numbered plus lettered sub-checks `2b`, `2c`, `2d`, +`4b`, and `7b`. The checks cover stale MCP servers, plugin registration, PATH, +project config, secrets placement, version consistency, hook health, hook +registration, hook registry drift, recipe version health, gitignore +completeness, secret-scanning hook, editable install source, stale entry +points, source drift, quota cache schema, process state, install classification, +update dismissal state, ambient env leaks, and feature gate consistency. See +[installation.md](installation.md#post-install-verification) for the full table. --- diff --git a/docs/configuration.md b/docs/configuration.md index 7d5ca6ef5c..4e2405fc79 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -140,13 +140,13 @@ Default: `command: null` (disabled), `preserve_dirs: []`. quota_guard: enabled: true short_window_threshold: 85.0 # block at this % for short windows (e.g. five_hour) - long_window_threshold: 98.0 # block at this % for long windows (weekly, sonnet, opus) + long_window_threshold: 95.0 # block at this % for long windows (weekly, sonnet, opus) long_window_patterns: # substrings (case-insensitive) that classify a - weekly # window name as long-window - sonnet - opus buffer_seconds: 60 # extra buffer after quota reset before resuming - cache_max_age: 60 # seconds before a live quota fetch is triggered + cache_max_age: 300 # seconds before a live quota fetch is triggered ``` Check current quota: `autoskillit quota-status`. @@ -173,11 +173,11 @@ branching: default_base_branch: main # default base branch for recipes ``` -Default: `main`. Override if your project uses a different integration branch (e.g. `integration` or `develop`): +Default: `main`. Override if your project uses a different integration branch (e.g. `develop`): ```yaml branching: - default_base_branch: integration + default_base_branch: develop ``` ## Session Diagnostics (Linux) @@ -271,7 +271,7 @@ read_db: ```yaml report_bug: - output_dir: null # null = {cwd}/.autoskillit/temp/bug-reports/ + report_dir: null # null = {cwd}/.autoskillit/temp/bug-reports/ timeout: 600 github_filing: true github_labels: ["autoreported", "bug"] @@ -369,7 +369,7 @@ with GitHub's merge queue feature. For best results with automation use cases: ### `min_entries_to_merge_wait_minutes` = 0 GitHub branch rulesets expose a `min_entries_to_merge_wait_minutes` setting that adds -latency before a queued PR is eligible to merge. For the `integration` branch (or any +latency before a queued PR is eligible to merge. For the `develop` branch (or any branch where AutoSkillit manages the PR queue), set this to `0`. **Why:** AutoSkillit enters PRs one at a time or in small batches. A non-zero wait @@ -377,5 +377,37 @@ multiplier adds unnecessary latency per PR. Setting it to `0` lets PRs merge as as their CI passes. **Location:** GitHub → Repository Settings → Branches → Branch protection rules → -select the integration ruleset → Merge queue → "Minimum entries to merge — wait X minutes". +select the develop ruleset → Merge queue → "Minimum entries to merge — wait X minutes". Set to `0`. + +## Feature Flags + +Features are gated by lifecycle state. The `features:` config section and +`AUTOSKILLIT_FEATURES__*` env vars let you override individual features. + +### Lifecycle States + +| Lifecycle | Behavior | Can Override? | +|-----------|----------|---------------| +| STABLE | On everywhere | Yes — opt out via `features: {name: false}` | +| EXPERIMENTAL | On when `experimental_enabled: true` (default on develop; off on main) | Yes — per-feature entry overrides blanket | +| DEPRECATED | Follows `default_enabled`; may be removed without warning | Yes | +| DISABLED | Always off; cannot be enabled | No — config entry setting `true` is rejected | + +### Blanket Toggle + +`experimental_enabled: true` in `defaults.yaml` means all `EXPERIMENTAL` features are +active on develop, worktrees, and feature branches without any per-feature config. + +Main and stable branches commit `features: {experimental_enabled: false}` to opt out. + +### Per-Feature Override + +Any config layer can override an individual feature regardless of the blanket toggle: + +```yaml +features: + planner: false # disable planner even on develop +``` + +Env var takes highest priority: `AUTOSKILLIT_FEATURES__PLANNER=false`. diff --git a/docs/decisions/0001-prohibit-background-subagent-execution.md b/docs/decisions/0001-prohibit-background-subagent-execution.md new file mode 100644 index 0000000000..0d27c75a56 --- /dev/null +++ b/docs/decisions/0001-prohibit-background-subagent-execution.md @@ -0,0 +1,86 @@ +# ADR-0001: Prohibit Background Subagent Execution in All Skills + +**Status:** Accepted +**Date:** 2026-05-01 +**Issue:** [#1582](https://github.com/TalonT-Org/AutoSkillit/issues/1582) + +## Context + +Skills that spawn subagents via the Agent/Task tool have no universal rule governing +whether those subagents run in the foreground (blocking) or background (non-blocking). +When left to LLM discretion, background execution is occasionally chosen and produces +silent failures: + +- **Lost results**: Background agents complete after the parent has moved on; findings + are never incorporated into skill output. +- **Race conditions**: The parent acts on incomplete information while background agents + are still gathering evidence. +- **Unobservable failures**: A failed background agent goes unnoticed — the parent is + not blocked waiting for it. +- **Synthesis responsibility violation**: Synthesis requires all results to be available + simultaneously. Background execution makes this impossible. + +## Decision + +**All skills that instruct sessions to spawn subagents MUST include the explicit rule in +their NEVER block:** + +> Run subagents in the background (`run_in_background: true` is prohibited) + +This applies to all SKILL.md files in `skills/` and `skills_extended/` that contain +Agent/Task tool spawning instructions. + +This rule does NOT prohibit parallel execution. Multiple foreground subagents launched +in a single message execute concurrently — the parent blocks until all complete, then +synthesizes their combined results. + +## Rationale + +The foreground-parallel pattern (multiple Agent calls in one message, no +`run_in_background`) provides all the throughput benefits of parallelism while +guaranteeing the parent has complete information before synthesis. There is no valid +use case for background subagents in skill execution — if results are not needed, the +subagent should not be spawned at all. + +## Enforcement + +The compliance test `test_no_background_subagent_in_spawning_skills` in +`tests/skills/test_skill_compliance.py` automatically detects skills with spawn +indicators and asserts the prohibition string is present. This test runs as part of +`task test-all` and `task test-check`. + +## Scope + +Applies to 77 SKILL.md files. Listed by group: + +**Core spawning skills (23):** implement-worktree, implement-worktree-no-merge, +investigate, make-plan, make-groups, validate-audit, scope, rectify, process-issues, +dry-walkthrough, generate-report, implement-experiment, plan-experiment, +plan-visualization, run-experiment, review-approach, review-design, setup-project, +triage-issues, audit-claims, review-pr, review-research-pr, retry-worktree + +**Planner L1+L0 skills (8):** planner-analyze, planner-extract-domain, +planner-elaborate-wps, planner-elaborate-assignments, planner-refine-phases, +planner-refine-assignments, planner-refine-wps, planner-elaborate-phase + +**Resolve skills (3):** resolve-review, resolve-claims-review, resolve-research-review + +**Audit/design skills (7):** audit-arch, audit-cohesion, audit-defense-standards, +design-guards, audit-tests, audit-bugs, stage-data + +**Arch-lens skills (13):** arch-lens-state-lifecycle, arch-lens-c4-container, +arch-lens-concurrency, arch-lens-data-lineage, arch-lens-deployment, +arch-lens-development, arch-lens-error-resilience, arch-lens-module-dependency, +arch-lens-operational, arch-lens-process-flow, arch-lens-repository-access, +arch-lens-scenarios, arch-lens-security + +**Exp-lens skills (18):** exp-lens-fair-comparison, exp-lens-unit-interference, +exp-lens-causal-assumptions, exp-lens-benchmark-representativeness, +exp-lens-comparator-construction, exp-lens-error-budget, exp-lens-estimand-clarity, +exp-lens-exploratory-confirmatory, exp-lens-governance-risk, exp-lens-iterative-learning, +exp-lens-measurement-validity, exp-lens-pipeline-integrity, +exp-lens-randomization-blocking, exp-lens-reproducibility-artifacts, +exp-lens-sensitivity-robustness, exp-lens-severity-testing, exp-lens-validity-threats, +exp-lens-variance-stability + +**Other (5):** analyze-prs, build-execution-map, verify-diag, audit-impl, elaborate-phase diff --git a/docs/decisions/0002-ban-inline-shell-scripts-from-cmd.md b/docs/decisions/0002-ban-inline-shell-scripts-from-cmd.md new file mode 100644 index 0000000000..d9a8fa2023 --- /dev/null +++ b/docs/decisions/0002-ban-inline-shell-scripts-from-cmd.md @@ -0,0 +1,44 @@ +# ADR-0002: Ban Inline Shell Scripts from Recipe cmd Fields + +**Status:** Accepted +**Date:** 2026-05-01 +**Issue:** [#1584](https://github.com/TalonT-Org/AutoSkillit/issues/1584) + +## Context + +- Recipe YAML files are declarative workflow graphs defining *what* happens and in *what order*. +- PR #1583 introduced a 12-line bash script with variable assignments, arithmetic, file I/O, and control flow directly in a `cmd:` field. +- A systematic scan reveals 61 `run_cmd` steps across 6 recipe files containing inline shell scripts. +- Inline scripts make recipes unreadable, prevent reuse, defeat syntax highlighting, and make it trivially easy for automated PRs to introduce more. + +## Decision + +> **Code belongs in code files. Recipes are purely declarative.** + +- **Python logic** → `.py` file, invoked via `run_python` callable. +- **Bash logic** → `.sh` file in `scripts/recipe/`, invoked via `run_cmd cmd: bash scripts/recipe/foo.sh`. +- **Recipe YAML** → declarative only. Names the thing to call, passes parameters. Zero logic. + +A recipe step's `cmd:` field must never contain shell control flow (`if/then/else/fi`, `for/do/done`, `while/do/done`, `case/esac`), variable assignments beyond simple parameter passing, loops, or embedded Python (`python3 -c`). + +## Rationale + +- Recipes should be auditable at a glance — scanning step names and their targets tells the story. +- Shell scripts in `.sh` files get syntax highlighting, linting (shellcheck), and version control diffs. +- Python callables get type checking, testing, and IDE support. +- Deduplication becomes trivial — shared logic lives in one callable/script, referenced by name. +- The lint rule prevents regression automatically. + +## Enforcement + +- **Lint rule:** `inline-script-in-cmd` in `src/autoskillit/recipe/rules_inline_script.py` +- **Lint rule:** `inline-python-in-cmd` in the same module (catches `python3 -c`) +- **Validation:** `validate_recipe` → `run_semantic_rules` fires on every recipe load +- **Existing violations:** Grandfathered via `_INLINE_SCRIPT_ALLOWLIST` frozen set until externalized +- **Test command:** `task test-all` (includes recipe validation) + +## Scope + +- All recipe YAML files under `src/autoskillit/recipes/` +- All sub-recipe YAML files under `src/autoskillit/recipes/sub-recipes/` +- The `cmd` field of any `run_cmd` step in any recipe diff --git a/docs/decisions/0003-skill-args.md b/docs/decisions/0003-skill-args.md new file mode 100644 index 0000000000..a33c20a50c --- /dev/null +++ b/docs/decisions/0003-skill-args.md @@ -0,0 +1,37 @@ +# ADR-0003: Pass Skill Inputs as Arguments, Not Environment Variables + +**Status:** Accepted +**Date:** 2026-05-02 +**Issue:** [#1625](https://github.com/TalonT-Org/AutoSkillit/issues/1625) + +## Context + +- Recipe steps can declare an `env:` key inside `with:` which is stored in `with_args["env"]` by the YAML parser. +- The `run_skill` MCP tool has no `env` parameter and `build_leaf_headless_cmd` inherits only `os.environ`. +- Environment variables declared in recipe `env:` blocks are never propagated to headless subprocesses. +- The planner recipe's `inputs.task` and `inputs.task_file` were delivered via `env:` to `planner-generate-phases` and `planner-extract-domain`, meaning those skills never received their task input. + +## Decision + +> **All skill inputs go through positional arguments in `skill_command`. Environment variables MUST NOT be used to deliver skill inputs.** + +A semantic rule (`env-key-in-with-args`) rejects any step that contains an `env:` key in `with_args` at recipe validation time. + +## Rationale + +Positional arguments in `skill_command` flow through the entire pipeline: + +1. Recipe YAML (`skill_command: "/autoskillit:skill-name $1 $2 $3"`) +2. Orchestrator LLM (reads and forwards the command string) +3. `run_skill` MCP call (passes `skill_command` as-is) +4. `build_leaf_headless_cmd` (embeds in `--skill-command`) +5. Headless session (receives `$1`, `$2`, `$3` as positional args) +6. `SKILL.md` (documents and reads `$N` references) + +Environment variables have no delivery path from recipe YAML to the headless subprocess. The `env:` key is silently stored and ignored, creating a failure mode that is invisible at recipe load time. + +## Consequences + +- Existing `env:` blocks in recipe steps must be migrated to positional args in `skill_command`. +- SKILL.md files must document `$N` positional args instead of `ENV_VAR` references. +- The semantic rule catches future regressions at validation time. diff --git a/docs/decisions/README.md b/docs/decisions/README.md new file mode 100644 index 0000000000..e3727494c4 --- /dev/null +++ b/docs/decisions/README.md @@ -0,0 +1,5 @@ +# Architecture Decision Records + +- [0001-prohibit-background-subagent-execution.md](0001-prohibit-background-subagent-execution.md) — prohibit `run_in_background: true` in all skills +- [0002-ban-inline-shell-scripts-from-cmd.md](0002-ban-inline-shell-scripts-from-cmd.md) — Prohibit inline shell scripts in recipe cmd fields; require externalization to .sh files or run_python callables +- [0003-skill-args.md](0003-skill-args.md) — Pass skill inputs as positional arguments, not environment variables diff --git a/docs/design/README.md b/docs/design/README.md new file mode 100644 index 0000000000..39ce1b9ffa --- /dev/null +++ b/docs/design/README.md @@ -0,0 +1,7 @@ +# Design Specifications + +Design specifications for planned features and skills. + +| Document | Description | +|---|---| +| [env-setup-design.md](env-setup-design.md) | Design spec for the dedicated `setup-environment` skill — Docker vs micromamba-host decision tree, structured output tokens, and recipe integration | diff --git a/docs/design/env-setup-design.md b/docs/design/env-setup-design.md new file mode 100644 index 0000000000..12ac78ba71 --- /dev/null +++ b/docs/design/env-setup-design.md @@ -0,0 +1,350 @@ +# Design Spec: Dedicated Environment-Setup Skill + +**Status:** Draft +**Date:** 2026-05-05 +**Issue:** [#838](https://github.com/TalonT-Org/AutoSkillit/issues/838) +**Blocks:** [#839](https://github.com/TalonT-Org/AutoSkillit/issues/839) (skill implementation) + +## Overview + +This document specifies the `setup-environment` skill — a discrete recipe step inserted between `stage_data` and `decompose_phases` in `research.yaml`. The skill decouples environment decisions (Docker vs host-micromamba vs none) from `implement-experiment`'s inline Docker build (Step 3c) and makes the environment-mode decision a testable, pre-flight gate with structured output tokens consumed by downstream steps. + +## Skill Identity + +- **Skill name:** `setup-environment` +- **Location:** `src/autoskillit/skills_extended/setup-environment/SKILL.md` +- **Recipe step name:** `setup_environment` +- **Invocation:** `/autoskillit:setup-environment ` + +### Arguments + +| Position | Name | Type | Required | Description | +|---|---|---|---|---| +| 1 | `worktree_path` | `directory_path` | Yes | Absolute path to the experiment worktree | +| 2 | `experiment_plan` | `file_path` | Yes | Absolute path to the experiment plan YAML | + +## Structured Output Tokens + +The skill emits exactly four tokens before `%%ORDER_UP%%`: + +``` +env_mode = {none|docker|micromamba-host|unavailable} +env_report = {absolute_path_to_environment_setup_report.md} +verdict = {PASS|WARN|FAIL} +%%ORDER_UP%% +``` + +| Token | Description | Consumed by | +|---|---|---| +| `env_mode` | Resolved environment execution mode | `implement_phase`, `run_experiment`, `generate_report` via `${{ context.env_mode }}` | +| `env_report` | Absolute path to detailed report file | `troubleshoot-experiment` for post-failure diagnostics | +| `verdict` | Routing signal | `on_result` in the recipe step | + +The `env_report` file is always written (`write_behavior: always`), providing a diagnostic trail for `troubleshoot-experiment` when later steps fail. + +## Environment-Mode Verdict Enum + +Four values extending the issue's three-value suggestion to handle the `environment.type = standard` case from `plan-experiment`: + +| `env_mode` value | Meaning | `verdict` | Downstream behavior | +|---|---|---|---| +| `none` | Experiment does not need a custom environment (`environment.type = standard` in plan) | `PASS` | `implement_phase` skips Docker/env setup entirely; `run_experiment` runs scripts directly on host | +| `docker` | Docker runtime available, image built successfully | `PASS` | `implement_phase` skips Step 3 (env already built); `run_experiment` uses `docker run`; `generate-report` uses existing image for visualizations | +| `micromamba-host` | Docker unavailable but host micromamba environment created | `WARN` | `implement_phase` skips Step 3; `run_experiment` uses `micromamba run -n {env}`; `generate-report` runs visualization scripts in micromamba env instead of Docker | +| `unavailable` | Custom environment required but neither Docker nor host micromamba viable | `FAIL` | Recipe routes to `escalate_stop`; experiment is blocked | + +The `WARN` verdict for `micromamba-host` reflects degraded isolation (no container boundary) while still being a viable execution path. Downstream steps treat `WARN` the same as `PASS` for routing purposes but log the degradation. + +## Process Flow: Environment-Setup Skill Decision Logic + +```mermaid +%%{init: {'flowchart': {'nodeSpacing': 50, 'rankSpacing': 60, 'curve': 'basis'}}}%% +flowchart TB + classDef cli fill:#1a237e,stroke:#7986cb,stroke-width:2px,color:#fff; + classDef stateNode fill:#004d40,stroke:#4db6ac,stroke-width:2px,color:#fff; + classDef handler fill:#e65100,stroke:#ffb74d,stroke-width:2px,color:#fff; + classDef newComponent fill:#2e7d32,stroke:#81c784,stroke-width:2px,color:#fff; + classDef terminal fill:#1a237e,stroke:#7986cb,stroke-width:2px,color:#fff; + + START([START
━━━━━━━━━━
setup_environment step invoked]) + + subgraph InputParsing ["Input Parsing"] + direction TB + ReadPlan["Read Experiment Plan
━━━━━━━━━━
Parse YAML frontmatter
environment.type + spec_path"] + CheckType{"environment.type?
━━━━━━━━━━
standard vs custom"} + end + + subgraph DockerPath ["Docker Assessment"] + direction TB + CheckDocker{"Docker Runtime
Available?
━━━━━━━━━━
docker info --format ok"} + WriteDockerfile["Write Dockerfile
━━━━━━━━━━
From Dockerfile.template
+ environment.yml"] + BuildImage["docker build
━━━━━━━━━━
--build-arg MAMBA_ENV=slug
-t research-slug"] + CheckBuild{"Build
Succeeded?"} + end + + subgraph FallbackPath ["Host Micromamba Fallback"] + direction TB + AssessViability{"Viability
Assessment
━━━━━━━━━━
Parse environment.yml
deps classification"} + CreateHostEnv["micromamba create
━━━━━━━━━━
-f environment.yml
-n experiment-slug"] + CheckInstall{"Install
Succeeded?"} + end + + subgraph Verdicts ["Verdict Emission"] + direction TB + EmitNone["env_mode = none
verdict = PASS
━━━━━━━━━━
No custom env needed"] + EmitDocker["env_mode = docker
verdict = PASS
━━━━━━━━━━
Image built OK"] + EmitHost["env_mode = micromamba-host
verdict = WARN
━━━━━━━━━━
Degraded isolation"] + EmitFail["env_mode = unavailable
verdict = FAIL
━━━━━━━━━━
Cannot establish env"] + end + + COMPLETE([COMPLETE
━━━━━━━━━━
%%ORDER_UP%%]) + + START --> ReadPlan + ReadPlan --> CheckType + CheckType -->|"standard"| EmitNone + CheckType -->|"custom"| CheckDocker + CheckDocker -->|"yes"| WriteDockerfile + CheckDocker -->|"no"| AssessViability + WriteDockerfile --> BuildImage + BuildImage --> CheckBuild + CheckBuild -->|"success"| EmitDocker + CheckBuild -->|"failure"| AssessViability + AssessViability -->|"viable:
pure conda-forge Python"| CreateHostEnv + AssessViability -->|"not viable:
system pkgs / CUDA / Bioconductor"| EmitFail + CreateHostEnv --> CheckInstall + CheckInstall -->|"success"| EmitHost + CheckInstall -->|"failure"| EmitFail + + EmitNone --> COMPLETE + EmitDocker --> COMPLETE + EmitHost --> COMPLETE + EmitFail --> COMPLETE + + class START,COMPLETE terminal; + class ReadPlan,WriteDockerfile,BuildImage,CreateHostEnv handler; + class CheckType,CheckDocker,CheckBuild,AssessViability,CheckInstall stateNode; + class EmitNone,EmitDocker,EmitHost,EmitFail newComponent; +``` + +## Per-Task Fallback Viability Rules + +When Docker is unavailable (or Docker build fails), the skill decides whether `micromamba-host` is a viable fallback by parsing the experiment's `environment.yml` and the Dockerfile template's `apt-get install` layer. + +### Viable (micromamba-host OK) + +- All dependencies are pure conda-forge Python packages (e.g., `numpy`, `pandas`, `scipy`, `scikit-learn`, `pytest`) +- No system packages beyond what the host already provides +- No compiled C/Fortran extensions that require specific system libraries not available via conda-forge + +### Not Viable (must use Docker or fail) + +- `environment.yml` references non-conda-forge channels that require system-level setup +- The Dockerfile template's `apt-get install` layer includes packages not present on the host AND those packages are required at runtime (not just build-time) +- Dependencies include CUDA libraries (`cudatoolkit`, `cuda-*`, `nvidia-*`) +- Dependencies include Bioconductor R packages that require system-level R installation +- Dependencies include packages with known native compilation requirements that conda-forge cannot satisfy on the host architecture + +### Assessment Algorithm + +1. Parse `environment.yml` — extract `channels` list and `dependencies` list +2. Check channels: if only `conda-forge` (and optionally `defaults`), mark channels as viable +3. Classify each dependency: + - Look up in a hardcoded allowlist of known pure-Python conda-forge packages → `viable` + - Check for CUDA/GPU markers (`cuda`, `nvidia`, `cudnn`, `nccl`) → `not_viable` + - Check for Bioconductor markers (`bioconductor-*`, `r-*` with system deps) → `not_viable` + - Unknown packages → `viable` (optimistic default; failure caught at install time) +4. Parse the Dockerfile template's `apt-get install` line — extract system packages +5. For each system package, check if it exists on the host via `dpkg -l` or `which` +6. If any required system package is missing AND is not installable via conda-forge → `not_viable` +7. Final verdict: all checks pass → `viable`; any `not_viable` → `not_viable` + +### Viability Examples + +| Scenario | environment.yml | Viability | Reason | +|---|---|---|---| +| Statistical analysis | `numpy`, `scipy`, `pandas`, `matplotlib` | Viable | Pure conda-forge Python packages | +| GPU ML training | `pytorch`, `cudatoolkit=11.8` | Not viable | CUDA system library required | +| Bioinformatics | `bioconductor-deseq2`, `r-base` | Not viable | System R + Bioconductor ecosystem | +| Web scraping | `requests`, `beautifulsoup4`, `selenium` | Viable | Pure Python; selenium webdriver is a user concern | +| Image processing | `opencv`, `pillow`, `scikit-image` | Viable | conda-forge provides pre-built binaries | + +## Downstream Consumption + +The `env_mode` value is captured into recipe context and passed to downstream steps via `${{ context.env_mode }}`. + +### `implement_phase` (via `implement-experiment` SKILL.md, modified in WI 3.2) + +``` +If context.env_mode == "none": + Skip Step 3 entirely (no Dockerfile, no Taskfile, no docker build) + Write scripts that run directly on host Python + +If context.env_mode == "docker": + Skip Step 3a–3c (Dockerfile already written, image already built by setup-environment) + Step 3b: Write Taskfile.yml referencing existing image "research-{slug}" + Verify image exists: docker image inspect "research-{slug}" + +If context.env_mode == "micromamba-host": + Skip Step 3a–3c entirely + Write Taskfile.yml with micromamba-based tasks: + run-experiment: micromamba run -n {env} python {script} + test: micromamba run -n {env} pytest {test_dir} + Verify env exists: micromamba env list | grep {env} + +If context.env_mode == "unavailable": + This case never reaches implement_phase (recipe routes to escalate_stop) +``` + +### `run_experiment` (via `run-experiment` SKILL.md) + +`run-experiment` is already environment-agnostic — it reads the plan and executes what the plan describes. The Taskfile.yml written by `implement_phase` encapsulates the execution mode. No changes are needed to `run-experiment` itself. + +### `generate_report` (via `generate-report` SKILL.md, modified in WI 3.2) + +Step 2.5 currently does `docker run` for visualizations. It must branch on `env_mode`: + +``` +If context.env_mode == "docker": + (current behavior) docker run --rm -v ... "research-{slug}" bash -c "..." + +If context.env_mode == "micromamba-host": + micromamba run -n {env} python /workspace/scripts/fig{N}_{slug}.py + +If context.env_mode == "none": + python scripts/fig{N}_{slug}.py (host Python, no container) +``` + +## Canonical Dockerfile Formula Reference + +The canonical Dockerfile template lives at: + +``` +src/autoskillit/assets/research/Dockerfile.template +``` + +### Version Pin + +- **Base image:** `mambaorg/micromamba:1.0-bullseye-slim` +- **System packages:** `procps git curl build-essential` +- **Taskfile installer:** `https://taskfile.dev/install.sh` +- **Shell hook mechanism:** `BASH_ENV=/etc/container.bashrc` + `micromamba shell hook --shell=bash` + +### Docker Build Context Staging Requirement + +The template uses `COPY ${MAMBA_ENV}.yaml /opt/research/env/`. The build arg `MAMBA_ENV` is the experiment slug, not the literal string `environment.yml`. The setup-environment skill must stage the build context before running `docker build`: + +1. Create a temp build directory under `.autoskillit/temp/` +2. Copy `Dockerfile.template` into the build dir as `Dockerfile` +3. Copy the experiment's `environment.yml` into the build dir as `{slug}.yaml` +4. Run: `docker build --build-arg MAMBA_ENV={slug} -t research-{slug} {build_dir}` + +This staging requirement exists so that WI 3.2 implementers do not place `environment.yml` directly in the build context (which would be silently ignored by the `COPY ${MAMBA_ENV}.yaml` instruction). + +### Single Source of Truth + +The `Dockerfile.template` asset file is the single source of truth for the Docker build formula. Any future helper-agents mirror must be kept in sync via a contract test. + +**Contract test for WI 3.2:** `test_dockerfile_template_base_image_pinned` — asserts `mambaorg/micromamba:1.0-bullseye-slim` appears in the template. (Already partially covered by existing `test_dockerfile_template_uses_micromamba_base` in `tests/contracts/test_generate_report_contracts.py`.) + +## Recipe Integration + +New step `setup_environment` inserted between `stage_data` and `decompose_phases`. The `stage_data` step's PASS/WARN routes change from `decompose_phases` to `setup_environment`. + +### Recipe Flow Diagram + +```mermaid +%%{init: {'flowchart': {'nodeSpacing': 40, 'rankSpacing': 50, 'curve': 'basis'}}}%% +flowchart TB + classDef stateNode fill:#004d40,stroke:#4db6ac,stroke-width:2px,color:#fff; + classDef handler fill:#e65100,stroke:#ffb74d,stroke-width:2px,color:#fff; + classDef newComponent fill:#2e7d32,stroke:#81c784,stroke-width:2px,color:#fff; + classDef detector fill:#b71c1c,stroke:#ef5350,stroke-width:2px,color:#fff; + + CW["create_worktree
━━━━━━━━━━
captures: worktree_path,
research_dir"] + SD["stage_data
━━━━━━━━━━
captures: verdict,
resource_report"] + SDRoute{"stage_data
verdict?"} + SE["setup_environment
━━━━━━━━━━
captures: env_mode,
env_report, verdict"] + SERoute{"setup_environment
verdict?"} + DP["decompose_phases
━━━━━━━━━━
captures: group_files"] + PP["plan_phase
━━━━━━━━━━
captures: phase_plan_path"] + IP["implement_phase
━━━━━━━━━━
reads context.env_mode
skips Step 3 if env ready"] + ESC["escalate_stop
━━━━━━━━━━
action: stop"] + + CW --> SD + SD --> SDRoute + SDRoute -->|"PASS / WARN"| SE + SDRoute -->|"FAIL"| ESC + SE --> SERoute + SERoute -->|"PASS / WARN"| DP + SERoute -->|"FAIL"| ESC + DP --> PP + PP --> IP + + class CW,SD,DP,PP handler; + class SE newComponent; + class IP stateNode; + class SDRoute,SERoute stateNode; + class ESC detector; +``` + +### Proposed Recipe YAML (for WI 3.2 implementation) + +```yaml +setup_environment: + tool: run_skill + model: "" + with: + skill_command: "/autoskillit:setup-environment ${{ context.worktree_path }} ${{ context.experiment_plan }}" + cwd: "${{ context.worktree_path }}" + step_name: setup_environment + capture: + env_mode: "${{ result.env_mode }}" + env_report: "${{ result.env_report }}" + on_result: + field: verdict + routes: + PASS: decompose_phases + WARN: decompose_phases + FAIL: escalate_stop + on_failure: escalate_stop + stale_threshold: 600 + retries: 1 + on_exhausted: escalate_stop +``` + +### Changes to Existing Steps (for WI 3.2 implementation) + +1. `stage_data.on_result.routes.PASS` and `WARN`: change target from `decompose_phases` to `setup_environment` +2. `implement_phase.with.skill_command`: append `${{ context.env_mode }}` so the skill knows what environment mode to use +3. `generate_report` step: append `${{ context.env_mode }}` to its `skill_command` (or add `optional_context_refs: [env_mode]`) + +## Skill Contract + +Entry for `src/autoskillit/recipe/skill_contracts.yaml` (added in WI 3.2): + +```yaml +setup-environment: + inputs: + - name: worktree_path + type: directory_path + required: true + - name: experiment_plan + type: file_path + required: true + outputs: + - name: env_mode + type: string + - name: env_report + type: file_path + - name: verdict + type: string + expected_output_patterns: + - "env_mode\\s*=\\s*(none|docker|micromamba-host|unavailable)" + - "verdict\\s*=\\s*(PASS|WARN|FAIL)" + - "env_report\\s*=\\s*/.+" + pattern_examples: + - "env_mode = docker\nenv_report = /tmp/env_setup_report.md\nverdict = PASS\n%%ORDER_UP%%" + - "env_mode = micromamba-host\nenv_report = /tmp/env_setup_report.md\nverdict = WARN\n%%ORDER_UP%%" + - "env_mode = unavailable\nenv_report = /tmp/env_setup_report.md\nverdict = FAIL\n%%ORDER_UP%%" + write_behavior: always +``` diff --git a/docs/developer/end-turn-hazards.md b/docs/developer/end-turn-hazards.md index 1acde7b99c..d34198bfc9 100644 --- a/docs/developer/end-turn-hazards.md +++ b/docs/developer/end-turn-hazards.md @@ -190,7 +190,7 @@ two detectors on every SKILL.md in the project: | `_check_loop_boundary()` | "For each" loops with tool invocations but no anti-prose guard | These run as part of `test_no_text_then_tool_in_any_step`, a parametrized -test that scans all 109 bundled skills. Any new skill with an unguarded +test that scans all 133 bundled skills. Any new skill with an unguarded loop fails CI automatically. ## If This Gets Fixed Upstream diff --git a/docs/execution/README.md b/docs/execution/README.md index 8ffe23a6d3..4256f3a811 100644 --- a/docs/execution/README.md +++ b/docs/execution/README.md @@ -5,4 +5,5 @@ merge gates. - [architecture.md](architecture.md) — orchestrator, kitchen, sous-chef - [tool-access.md](tool-access.md) — kitchen vs free range, the 42 MCP tools -- [orchestration.md](orchestration.md) — tier model, retry reasons, merge decision tree +- [orchestration.md](orchestration.md) — orchestration levels, retry reasons, merge decision tree +- [orchestration-levels.md](../orchestration-levels.md) — L0–L3 level definitions, mapping table diff --git a/docs/execution/architecture.md b/docs/execution/architecture.md index 4eed2963a2..a9494327d5 100644 --- a/docs/execution/architecture.md +++ b/docs/execution/architecture.md @@ -4,7 +4,7 @@ How AutoSkillit runs a recipe end to end: orchestrator, kitchen gating, clone an ## Overview -AutoSkillit is a Claude Code plugin that orchestrates automated workflows using headless sessions. It provides 43 MCP tools and 109 bundled skills, organized into a gated visibility system. +AutoSkillit is a Claude Code plugin that orchestrates automated workflows using headless sessions. It provides 52 MCP tools and 133 bundled skills, organized into a gated visibility system. ## Core Concepts @@ -21,14 +21,14 @@ When you run `autoskillit order`, Claude Code acts as a pipeline orchestrator. I AutoSkillit uses a three-tier tool visibility model: -- **Free-range (2 tools)**: Always visible — `open_kitchen`, `close_kitchen` +- **Free-range (4 tools)**: Always visible — `open_kitchen`, `close_kitchen`, `disable_quota_guard`, `reload_session` - **Headless tools (1 tool)**: Revealed in headless sessions via `mcp.enable({'headless'})` — `test_check` -- **Kitchen-tagged tools (41 tools total)**: Gated behind `open_kitchen` — `run_skill`, +- **Kitchen-tagged tools (37 tools total)**: Gated behind `open_kitchen` — `run_skill`, `run_cmd`, `run_python`, `merge_worktree`, `clone_repo`, `push_to_remote`, and 31 more. One kitchen tool (`test_check`) also carries the `headless` tag and is additionally pre-enabled in headless sessions. -When you call `open_kitchen` (automatically done by `order`), all 41 kitchen-tagged tools become +When you call `open_kitchen` (automatically done by `order`), all 37 kitchen-tagged tools become available for that session. This keeps normal Claude Code sessions clean — no pipeline tools cluttering the tool list. @@ -59,8 +59,8 @@ Within the clone, implementation happens in git worktrees: AutoSkillit supports four session modes with different tool and skill visibility: - **`$ claude` (plugin, no kitchen)**: Regular Claude Code session with the AutoSkillit plugin - loaded. Sees 2 Free Range MCP tools (`open_kitchen`, `close_kitchen`) and Tier 1 skills only - (`open-kitchen`, `close-kitchen`). After calling `/open-kitchen`, all 41 kitchen-tagged MCP + loaded. Sees 4 Free Range MCP tools (`open_kitchen`, `close_kitchen`, `disable_quota_guard`, `reload_session`) and Tier 1 skills only + (`open-kitchen`, `close-kitchen`). After calling `/open-kitchen`, all 37 kitchen-tagged MCP tools become available. - **`$ autoskillit cook`**: Interactive development session. Sees all three skill tiers @@ -68,10 +68,10 @@ AutoSkillit supports four session modes with different tool and skill visibility `$ claude`); `/open-kitchen` reveals kitchen tools. - **`$ autoskillit order`**: Pipeline orchestrator session. Kitchen is pre-opened at startup — - all 44 MCP tools are available immediately. All skill tiers are accessible. The orchestrator + all 52 MCP tools are available immediately. All skill tiers are accessible. The orchestrator delegates work through `run_skill` (headless sessions) and `run_cmd` (shell commands). -- **`run_skill` (headless)**: Worker sessions launched by the orchestrator. Sees 2 Free Range +- **`run_skill` (headless)**: Worker sessions launched by the orchestrator. Sees 4 Free Range tools + `test_check` (headless-tagged). Cannot call `run_skill`, `run_cmd`, or `run_python` — enforced by hooks and code guards. Has access to all native Claude Code tools (Read, Write, Bash, etc.) and all skill tiers via `--add-dir skills_extended/`. diff --git a/docs/execution/orchestration.md b/docs/execution/orchestration.md index 83d230049f..2a876a7f61 100644 --- a/docs/execution/orchestration.md +++ b/docs/execution/orchestration.md @@ -1,24 +1,28 @@ # Orchestration -How AutoSkillit routes work between the Tier 1 orchestrator and the Tier 2/3 +> **See also:** [`docs/orchestration-levels.md`](../orchestration-levels.md) for +> the formal definition of the L0–L3 orchestration hierarchy. + +How AutoSkillit routes work between the **L2** orchestrator and the **L1** worker sessions, what the orchestrator does on every retry verdict, and how the merge pipeline decides whether a worktree is ready to land. -## Two-tier model +## Multi-level orchestration model -AutoSkillit splits agent execution into two layers: +AutoSkillit defines four orchestration levels (L0–L3). The recipe execution +pipeline primarily connects the L2 orchestrator and L1 workers: -- **Tier 1 — orchestrator.** A Claude Code session running the +- **L2 — orchestrator.** A Claude Code session running the `autoskillit order` CLI command, with the kitchen pre-opened. Sees all 42 MCP tools, spawns headless workers, and routes verdicts. Never reads or writes code itself. -- **Tier 2/3 — worker.** A headless Claude session launched by `run_skill`. +- **L1 — worker.** A headless Claude session launched by `run_skill`. Sees the 2 free range tools plus `test_check` (the only `headless`-tagged tool). Cannot call `run_skill`, `run_cmd`, or `run_python`. The boundary is enforced three ways: FastMCP visibility, the -`headless_orchestration_guard.py` PreToolUse hook, and the -`_require_not_headless()` runtime guard inside `tools_execution.py`. All +`skill_orchestration_guard.py` PreToolUse hook, and the +`_require_orchestrator_or_higher()` runtime guard inside `tools_execution.py`. All three must independently agree before any orchestration tool can fire. ## Recipe as a program @@ -43,10 +47,10 @@ tool result. The routing rules per tool: - `clone_repo` / `remove_clone` — read `clone_id` and stash for later cleanup via `register_clone_status` and `batch_cleanup_clones`. -## The 11 `retry_reason` values +## The 13 `retry_reason` values `RetryReason` is a `StrEnum` in `src/autoskillit/core/_type_enums.py` with -11 distinct values. Each value triggers a different recovery route: +13 distinct values. Each value triggers a different recovery route: | Value | When the orchestrator sets it | Recovery | |-------|-------------------------------|----------| @@ -56,11 +60,13 @@ tool result. The routing rules per tool: | `budget_exhausted` | Token-budget cap reached for the step | Re-plan or escalate; do not auto-retry | | `early_stop` | Worker emitted a structured `early_stop` token | Skip remaining sub-steps and route to a fallback | | `zero_writes` | Worker exited cleanly but produced no file writes | Re-spawn once, then escalate | -| `empty_output` | Natural exit with rc=0 and no stdout, no partial progress | Treat as a transient failure; one retry then escalate | +| `empty_output` | Natural exit with rc=0, no stdout, and no write evidence | Treat as a transient failure; one retry then escalate | +| `completed_no_flush` | Natural exit with rc=0 and no stdout, but write evidence confirms work was performed | Route to on_context_limit (same as drain_race/resume); partial progress exists on disk | | `drain_race` | Channel-confirmed completion but stdout was not flushed before kill | Replay the captured channel record; do not re-spawn | | `path_contamination` | Worker wrote outside its CWD boundary | Hard-fail; do not retry — this is an isolation breach | | `contract_recovery` | Marker present and write evidence on disk, but the structured contract token was missing | Treat as success and synthesise the contract from disk | | `clone_contamination` | Worker mutated the source clone instead of the worktree | Hard-fail; abort the entire `order` | +| `thinking_stall` | Thinking-only final turn: model consumed tokens (thinking blocks present) but produced no text or tool output | If lifespan_started, route to on_context_limit; else route to on_failure | `recipe/rules_isolation.py` enforces matching `clone_contamination` and `path_contamination` defenses at recipe-validation time. diff --git a/docs/execution/tool-access.md b/docs/execution/tool-access.md index fb6c36f1d1..d13b4dba27 100644 --- a/docs/execution/tool-access.md +++ b/docs/execution/tool-access.md @@ -1,14 +1,15 @@ # MCP Tool Access Control -AutoSkillit provides 44 MCP tools organized into three access levels that control which +AutoSkillit provides 52 MCP tools organized into three access levels that control which session types can see each tool. ## Three Access Levels ``` ┌─────────────────────────────────────────────────────────┐ -│ FREE RANGE (3 tools, always visible) │ -│ open_kitchen, close_kitchen, disable_quota_guard │ +│ FREE RANGE (4 tools, always visible) │ +│ open_kitchen, close_kitchen, disable_quota_guard, │ +│ reload_session │ │ Always visible — no gating, no headless restriction │ ├─────────────────────────────────────────────────────────┤ │ HEADLESS-TAGGED (1 tool) │ @@ -41,7 +42,7 @@ remain hidden even after `open_kitchen`. | Tag | Meaning | |-----|---------| | `autoskillit` | Identifies the tool as belonging to AutoSkillit. Present on every tool. | -| `kitchen` | Tool is hidden at startup via `mcp.disable(tags={'kitchen'})`. 38 tools carry this tag. | +| `kitchen` | Tool is hidden at startup via `mcp.disable(tags={'kitchen'})`. 37 tools carry this tag. | | `headless` | Tool is revealed in headless sessions via `mcp.enable(tags={'headless'})`. Additive — also carries `kitchen`. | | `github` | Functional category: GitHub-interacting tools. Can be disabled as a subset. | | `ci` | Functional category: CI/merge-queue polling tools. Can be disabled as a subset. | @@ -54,7 +55,7 @@ Server startup sequence: ``` 1. mcp.disable(tags={"kitchen"}) - → hides 38 kitchen-tagged tools (including the 1 headless-tagged tool) + → hides 37 kitchen-tagged tools (including the 1 headless-tagged tool) 2. mcp.disable(tags={subset}) for each entry in config.subsets.disabled → e.g. hides all github-tagged tools if "github" is disabled @@ -64,7 +65,7 @@ Server startup sequence: → reveals test_check only (the sole headless-tagged tool) 4. When open_kitchen is called: - ctx.enable_components(tags={"kitchen"}) → reveals all 41 kitchen tools + ctx.enable_components(tags={"kitchen"}) → reveals kitchen-tagged tools (not fleet) ctx.disable_components(tags={subset}) → re-hides each disabled subset (session-level enable overwrites server-level disable, so re-disabling is required) ``` @@ -76,8 +77,8 @@ Three independent layers prevent headless sessions from calling orchestration to | Layer | Mechanism | What It Blocks | |-------|-----------|----------------| | 1. FastMCP | Kitchen tools remain hidden (`mcp.enable(headless)` does not reveal kitchen-only tools) | `run_skill`, `run_cmd`, `run_python`, `merge_worktree`, and all other kitchen-only tools | -| 2. Hook | `headless_orchestration_guard.py` PreToolUse hook | `run_skill`, `run_cmd`, `run_python` | -| 3. Code | `_require_not_headless()` guard in `tools_execution.py` | `run_skill`, `run_cmd`, `run_python` | +| 2. Hook | `skill_orchestration_guard.py` PreToolUse hook | `run_skill`, `run_cmd`, `run_python` | +| 3. Code | `_require_orchestrator_or_higher()` guard in `tools_execution.py` | `run_skill`, `run_cmd`, `run_python` | All three layers must independently agree before any orchestration tool can execute. A bypassed hook is caught by the code guard; a bypassed code guard is caught by the @@ -85,10 +86,10 @@ missing kitchen visibility. ## Complete MCP Tool Access Control Map -All 44 tools with their access level, tags, source file, and functional category. +All 52 tools with their access level, tags, source file, and functional category. **Tag abbreviations**: AS = `autoskillit`, K = `kitchen`, HL = `headless`, -GH = `github`, CI = `ci`, CL = `clone`, TL = `telemetry` +GH = `github`, CI = `ci`, CL = `clone`, TL = `telemetry`, FL = `fleet` --- @@ -99,6 +100,7 @@ GH = `github`, CI = `ci`, CL = `clone`, TL = `telemetry` | `open_kitchen` | AS | `server/tools_kitchen.py` | | `close_kitchen` | AS | `server/tools_kitchen.py` | | `disable_quota_guard` | AS | `server/tools_kitchen.py` | +| `reload_session` | AS | `server/tools_kitchen.py` | --- @@ -127,6 +129,7 @@ GH = `github`, CI = `ci`, CL = `clone`, TL = `telemetry` | `merge_worktree` | AS, K | `server/tools_git.py` | | `classify_fix` | AS, K | `server/tools_git.py` | | `create_unique_branch` | AS, K, GH | `server/tools_git.py` | +| `create_and_publish_branch` | AS, K, GH | `server/tools_git.py` | | `check_pr_mergeable` | AS, K, GH | `server/tools_git.py` | | `reset_test_dir` | AS, K | `server/tools_workspace.py` | | `reset_workspace` | AS, K | `server/tools_workspace.py` | @@ -141,7 +144,8 @@ GH = `github`, CI = `ci`, CL = `clone`, TL = `telemetry` | `remove_clone` | AS, K, CL | `server/tools_clone.py` | | `push_to_remote` | AS, K, GH | `server/tools_clone.py` | | `register_clone_status` | AS, K, CL | `server/tools_clone.py` | -| `batch_cleanup_clones` | AS, K, CL | `server/tools_clone.py` | +| `batch_cleanup_clones` | AS, K, CL, FL | `server/tools_clone.py` | +| `bootstrap_clone` | AS, K, CL | `server/tools_clone.py` | --- @@ -154,6 +158,7 @@ GH = `github`, CI = `ci`, CL = `clone`, TL = `telemetry` | `wait_for_merge_queue` | AS, K, CI | `server/tools_ci.py` | | `check_repo_merge_state` | AS, K, CI | `server/tools_ci.py` | | `toggle_auto_merge` | AS, K, CI | `server/tools_ci.py` | +| `enqueue_pr` | AS, K, CI | `server/tools_ci.py` | | `set_commit_status` | AS, K, GH | `server/tools_ci.py` | --- @@ -169,6 +174,7 @@ GH = `github`, CI = `ci`, CL = `clone`, TL = `telemetry` | `enrich_issues` | AS, K, GH | `server/tools_issue_lifecycle.py` | | `claim_issue` | AS, K, GH | `server/tools_issue_lifecycle.py` | | `release_issue` | AS, K, GH | `server/tools_issue_lifecycle.py` | +| `claim_and_resolve_issue` | AS, K, GH | `server/tools_issue_composite.py` | | `get_pr_reviews` | AS, K, GH | `server/tools_pr_ops.py` | | `bulk_close_issues` | AS, K, GH | `server/tools_pr_ops.py` | @@ -185,6 +191,7 @@ GH = `github`, CI = `ci`, CL = `clone`, TL = `telemetry` | `get_quota_events` | AS, K, TL | `server/tools_status.py` | | `write_telemetry_files` | AS, K, TL | `server/tools_status.py` | | `read_db` | AS, K | `server/tools_status.py` | +| `analyze_tool_sequences` | AS, K, TL | `server/tools_status.py` | --- @@ -199,7 +206,16 @@ GH = `github`, CI = `ci`, CL = `clone`, TL = `telemetry` --- -**Total: 43 tools** — 2 Free Range + 41 Kitchen-tagged (of which 1, `test_check`, additionally carries the `headless` tag and is revealed inside headless sessions) +### KITCHEN — Fleet + +| Tool | Tags | Source File | +|------|------|-------------| +| `dispatch_food_truck` | AS, K, KC, fleet | `server/tools_execution.py` | +| `record_gate_dispatch` | AS, K, KC, fleet | `server/tools_execution.py` | + +--- + +**Total: 52 tools** — 4 Free Range + 11 Fleet + 37 Kitchen-tagged (of which 1, `test_check`, additionally carries the `headless` tag and is revealed inside headless sessions) For subset configuration that can hide functional-category tools, see [Subset Categories](../skills/subsets.md). diff --git a/docs/faq.md b/docs/faq.md index ec4483d32d..47044b15b6 100644 --- a/docs/faq.md +++ b/docs/faq.md @@ -9,14 +9,16 @@ dry-walkthrough, worktree, test, and PR-review skills. See ### How many MCP tools does it expose? -42. Two are free range (`open_kitchen`, `close_kitchen`) and 40 are +52. Fifteen are free-range: four always-visible (`open_kitchen`, `close_kitchen`, +`disable_quota_guard`, `reload_session`) plus eleven fleet tools revealed only in +fleet sessions via the `fleet`/`fleet-dispatch` tags. The remaining 37 are kitchen-tagged (gated behind `open_kitchen`). One kitchen tool, `test_check`, -also carries the `headless` tag and is revealed inside headless sessions. +also carries the `headless` tag and is revealed only inside headless sessions. See [execution/tool-access.md](execution/tool-access.md). ### How many bundled skills are there? -95: 3 in `src/autoskillit/skills/` (Tier 1) and 92 in +125: 3 in `src/autoskillit/skills/` (Tier 1) and 122 in `src/autoskillit/skills_extended/` (Tier 2 and 3). See [skills/catalog.md](skills/catalog.md). @@ -27,13 +29,12 @@ and `research`. See [recipes/overview.md](recipes/overview.md). ### What does the doctor command actually check? -14 things: 12 numbered checks plus the lettered sub-checks `4b` (config -secrets placement) and `7b` (hook registry drift). The full table lives in +28+ things: base checks span 28 numbered slots (with sub-checks 2b/2c/2d, 4b, 7b) plus up to 5 additional fleet-specific checks. The full table lives in [installation.md](installation.md#post-install-verification). ### Why are some MCP tools hidden by default? -To keep normal Claude Code sessions clean. The 40 kitchen-tagged tools only +To keep normal Claude Code sessions clean. The 37 kitchen-tagged tools only appear after the orchestrator calls `open_kitchen`. See [execution/tool-access.md](execution/tool-access.md). @@ -68,10 +69,9 @@ tree. See [safety/workspace.md](safety/workspace.md). ### How do I tune the API quota guard? Set `quota_guard.short_window_threshold` (default 85.0) for short windows -(e.g. `five_hour`), `quota_guard.long_window_threshold` (default 98.0) for +(e.g. `five_hour`), `quota_guard.long_window_threshold` (default 95.0) for long windows (weekly, sonnet, opus), and `quota_guard.buffer_seconds` -(default 60) in `.autoskillit/config.yaml`. See -[operations/sprint-guide.md](operations/sprint-guide.md). +(default 60) in `.autoskillit/config.yaml`. ### Where do session diagnostics go? @@ -100,5 +100,4 @@ contract card. See [recipes/authoring.md](recipes/authoring.md). Open an issue in the GitHub repository. AutoSkillit also has a built-in `report_bug` MCP tool that the `pipeline-summary` skill calls automatically when an overnight pipeline surfaces a bug. The tool deduplicates against -existing open issues by fingerprint. See -[operations/sprint-guide.md](operations/sprint-guide.md). +existing open issues by fingerprint. diff --git a/docs/glossary.md b/docs/glossary.md index b403758196..e6ddf6bdaf 100644 --- a/docs/glossary.md +++ b/docs/glossary.md @@ -13,8 +13,11 @@ Common mistake: `Arch Lens` or `arch lens`. ### bundled recipes -The 5 YAML recipes shipped under `src/autoskillit/recipes/`: `implementation`, -`implementation-groups`, `merge-prs`, `remediation`, and `research`. +The 14 YAML recipes shipped under `src/autoskillit/recipes/`: `bem-wrapper`, +`full-audit`, `implement-findings`, `implementation`, `implementation-groups`, +`merge-prs`, `planner`, `promote-to-main-wrapper`, `remediation`, +`research`, `research-archive`, `research-design`, `research-implement`, +and `research-review`. Distinguished from project-local recipes that live under `.autoskillit/recipes/`. Common mistake: "the built-in recipes". @@ -54,27 +57,67 @@ A composition pattern (see `recipe/registry.py`) where one recipe references sub-recipes via `requires_packs` instead of inlining their steps. Distinct from sub-recipes themselves. +### food truck + +A headless L2 session dispatched by an L3 fleet orchestrator. Self-contained: +clones the target repository, runs a full recipe, and returns results via the +campaign sidecar. Named by analogy — the fleet (L3) dispatches food trucks (L2) +to remote jobs. + ### free range tools -The 2 MCP tools that are always visible regardless of kitchen state: -`open_kitchen` and `close_kitchen`. Tagged only with `autoskillit`, never with -`kitchen`. Always two words, no hyphen. Common mistake: `free-range tools`. +The 4 MCP tools that are always visible regardless of kitchen state: +`open_kitchen`, `close_kitchen`, `disable_quota_guard`, and `reload_session`. Tagged +only with `autoskillit`, never with `kitchen`. Always two words, no hyphen. Common +mistake: `free-range tools`. + +### Ghost Kitchen + +The remote compute adapter system for dispatching food trucks to cloud +infrastructure. An L3 fleet sends work to a Ghost Kitchen when the L2 workers +need to run outside the local machine. ### kitchen -The collection of 40 kitchen-tagged MCP tools that the orchestrator must +The collection of 37 kitchen-tagged MCP tools that the orchestrator must explicitly reveal via `open_kitchen` before they can be called. Hidden at server startup via `mcp.disable(tags={'kitchen'})`. ### kitchen tools -Synonym for the 40 kitchen-tagged MCP tools. Two words, no hyphen. +Synonym for the 37 kitchen-tagged MCP tools. Two words, no hyphen. ### kitchen_id The unique identifier assigned to a kitchen instance (typically a session UUID). Recorded in pipeline telemetry. +### L0 + +Leaf subagent. Terminal node in the orchestration hierarchy. Cannot launch +sub-agents or headless sessions. Cannot call `run_skill`, `run_cmd`, or +`run_python`. Spawned by an L1 via Claude Code's Agent/Task tool. See +`docs/orchestration-levels.md`. + +### L1 + +A Claude Code session (interactive or headless) that can launch L0 subagents. +When headless, spawned by an L2 via `run_skill`. Headless L1 workers cannot +call `run_skill`, `run_cmd`, or `run_python`. Interactive variant: +`autoskillit cook`. See `docs/orchestration-levels.md`. + +### L2 + +Orchestrator of sessions. Launches L1 headless sessions via `run_skill`. +Interactive: `autoskillit order`. Headless: food truck (dispatched by L3). +See `docs/orchestration-levels.md`. + +### L3 + +Fleet dispatcher. Launches L2 food trucks. Interactive: `autoskillit fleet`. +No headless variant (no L4 exists to dispatch it). +See `docs/orchestration-levels.md`. + ### order A single execution of a recipe via `autoskillit order `. The @@ -87,7 +130,7 @@ and used to correlate logs. ### orchestrator -The Tier 1 Claude Code session that runs `autoskillit order`. It reads the +The L2 Claude Code session that runs `autoskillit order`. It reads the recipe, calls MCP tools, and spawns headless worker sessions via `run_skill`. Never reads or writes code itself. @@ -123,20 +166,32 @@ The Haiku-assisted contract-card freshness check implemented in ### Tier 1 -The free range tier — 3 skills under `src/autoskillit/skills/` +The free range **skill access tier** — 3 skills under `src/autoskillit/skills/` (`open-kitchen`, `close-kitchen`, `sous-chef`). Capitalised, space between `Tier` and the digit. Common mistake: `tier-1`, `tier 1` (lowercase). +> Not to be confused with orchestration levels (L0–L3). Skill access tiers +> control which skills are visible in which session modes. See +> `docs/orchestration-levels.md`. + ### Tier 2 Cook + headless skills under `src/autoskillit/skills_extended/`. Same capitalisation rules as Tier 1. +> Not to be confused with orchestration levels (L0–L3). Skill access tiers +> control which skills are visible in which session modes. See +> `docs/orchestration-levels.md`. + ### Tier 3 Pipeline / automation skills under `src/autoskillit/skills_extended/`. Same capitalisation rules as Tier 1. +> Not to be confused with orchestration levels (L0–L3). Skill access tiers +> control which skills are visible in which session modes. See +> `docs/orchestration-levels.md`. + ### wavefront scheduling The orchestration pattern implemented in `recipes/implementation-groups.yaml` @@ -144,8 +199,8 @@ that runs many sibling implementations in parallel waves rather than serially. ### worker -A headless Claude session spawned by the orchestrator via `run_skill`. Tier 2 -boundary — workers cannot call `run_skill`, `run_cmd`, or `run_python`. +A headless Claude session spawned by the orchestrator via `run_skill`. L1 +boundary — headless workers cannot call `run_skill`, `run_cmd`, or `run_python`. ### worktree diff --git a/docs/installation.md b/docs/installation.md index 6dce2343c6..1135d2f0c1 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -67,17 +67,20 @@ projects need. autoskillit doctor -Doctor runs 17 checks (15 numbered + 2 lettered sub-checks `4b` and `7b`), -enumerated by `run_doctor` in `src/autoskillit/cli/_doctor.py`: +Doctor runs 28 checks (23 numbered + 5 lettered sub-checks: `2b`, `2c`, `2d`, `4b`, `7b`); up to 33 with the fleet feature enabled. +Enumerated by `run_doctor` in `src/autoskillit/cli/_doctor.py`: | # | Check | What it verifies | |---|-------|------------------| | 1 | Stale MCP servers | No dead binaries or nonexistent paths in `~/.claude.json` | | 2 | MCP server registered | AutoSkillit MCP server is registered (direct entry or via plugin) | +| 2b | Dual MCP registration | No duplicate direct + marketplace registration | +| 2c | Plugin cache exists | `~/.claude/plugins/cache/autoskillit-local/` directory exists | +| 2d | installed_plugins.json | AutoSkillit entry present in installed_plugins.json | | 3 | `autoskillit` on PATH | The CLI command is reachable | | 4 | Config exists | `.autoskillit/config.yaml` is present | | 4b | Config secrets placement | Secrets live in `.autoskillit/.secrets.yaml`, never in `config.yaml` | -| 5 | Version consistency | Installed package version matches `plugin.json` | +| 5 | Version consistency | Cached plugin.json version matches installed package version | | 6 | Hook executability | Deployed hook scripts exist and are executable for every event type | | 7 | Hook registration | Hooks are registered in `settings.json` | | 7b | Hook registry drift | Structural diff against `generate_hooks_json()` from `hook_registry.py` | @@ -91,6 +94,13 @@ enumerated by `run_doctor` in `src/autoskillit/cli/_doctor.py`: | 15 | Claude process state | Reports D-state and CPU breakdown of running `claude` processes via `ps` | | 16 | Install classification | `direct_url.json` classifies the install type and requested revision | | 17 | Update dismissal state | Active update-prompt dismissal window and conditions, if any | +| 18 | Ambient SESSION_TYPE=leaf | No stray `SESSION_TYPE=leaf` env var in interactive shell | +| 19 | Ambient SESSION_TYPE=orchestrator | No stray `SESSION_TYPE=orchestrator` env var | +| 20 | Ambient SESSION_TYPE=fleet | No stray `SESSION_TYPE=fleet` env var | +| 21 | Ambient CAMPAIGN_ID | No stray `CAMPAIGN_ID` env var in interactive shell | +| 22 | Feature dependency consistency | Enabled features satisfy their declared dependencies | +| 23 | Feature registry import consistency | All feature gate modules import without errors | +| 24–28 | Fleet infrastructure | Sous-chef skill, dispatch guard, stale state, onboarding, clone collisions (fleet feature only) | See **[Hooks](safety/hooks.md)** for what each PreToolUse / PostToolUse / SessionStart hook actually enforces. @@ -141,7 +151,13 @@ If missing, run `autoskillit init` in your project directory. ### Upgrading +The recommended upgrade path: + + autoskillit update + +This fetches the latest version and runs `autoskillit install` automatically. + +For manual upgrades (fallback): + uv tool install --force "git+https://github.com/TalonT-Org/AutoSkillit.git@stable" autoskillit install - -Always run `autoskillit install` after upgrading to sync the plugin cache and hooks. diff --git a/docs/operations/README.md b/docs/operations/README.md index af57bf5b58..141a7a5837 100644 --- a/docs/operations/README.md +++ b/docs/operations/README.md @@ -2,5 +2,4 @@ Day-to-day running, observability, and recovery. -- [sprint-guide.md](sprint-guide.md) — overnight pipelines, quota tuning, scheduling - [observability.md](observability.md) — telemetry accumulators, session logs, anomaly detection diff --git a/docs/operations/observability.md b/docs/operations/observability.md index 82667972ff..4d23efcf64 100644 --- a/docs/operations/observability.md +++ b/docs/operations/observability.md @@ -21,10 +21,16 @@ and are written out at the end via `write_telemetry_files`. ## TelemetryFormatter `pipeline/telemetry_fmt.py:TelemetryFormatter` is the single source of truth -for the human-readable token and timing tables. Both -`get_token_summary` and the `token_summary_appender.py` PostToolUse hook -delegate to it so the format never drifts between the CLI and the PR body -appender. +for the human-readable token and timing tables. The MCP tool +`get_token_summary` delegates to it directly. The `token_summary_hook.py` +PostToolUse hook maintains stdlib-only parallel implementations of +`_format_efficiency_table` and `_format_table` (cannot import from +`autoskillit.*` — enforced by `tests/arch/test_ast_rules.py`). Output +equivalence between the canonical formatter and the hook is enforced by +`test_efficiency_table_equivalence` and `test_token_table_equivalence` in +`tests/infra/test_token_summary_core.py`. The canonical formatter derives +markdown headers from `_EFFICIENCY_COLUMNS` / `_TOKEN_COLUMNS` via +label-mapping dicts rather than hardcoding header strings. ## Mid-run accessors @@ -69,9 +75,9 @@ Per-session layout: ``` sessions/ / - proc.jsonl # ProcSnapshot stream + proc_trace.jsonl # ProcSnapshot stream anomalies.jsonl # detected anomalies - stdout.log # captured headless stdout + raw_stdout.jsonl # captured headless stdout sessions.jsonl # one summary line per session ``` @@ -92,8 +98,8 @@ jq 'select(.anomaly_count > 0)' ~/.local/share/autoskillit/logs/sessions.jsonl ## 500-directory retention `execution/session_log.py` keeps the most recent 500 session directories and -prunes older ones at every new session start. The `sessions.jsonl` index is -append-only and is not pruned. +prunes older ones at every new session start. `sessions.jsonl` is also rewritten +on each prune to remove entries for deleted session directories. ## Recording and replay diff --git a/docs/operations/sprint-guide.md b/docs/operations/sprint-guide.md deleted file mode 100644 index 1390d8c857..0000000000 --- a/docs/operations/sprint-guide.md +++ /dev/null @@ -1,140 +0,0 @@ -# Sprint guide - -How to drive a multi-issue overnight pipeline with AutoSkillit, tune the -quota guard so it neither blocks nor over-runs, and recover from common -stall modes. - -## Four-skill chain - -The sprint workflow chains four skills end to end: - -1. **`triage-issues`** — reads open GitHub issues, classifies them by - complexity and domain, and writes a triage report. -2. **`sprint-planner`** — consumes the triage report, groups issues into - sprints by domain coupling, and emits a sprint plan. -3. **`process-issues`** — runs the `implementation-groups` recipe over each - sprint, in wavefront-scheduled parallel batches. -4. **`pipeline-summary`** — reads the audit log, the token log, and the - timing log; opens a GitHub issue and PR summarising what landed and what - bugs surfaced during the run. - -Each skill consumes the previous skill's structured output via the -orchestrator's capture-and-route mechanism (see -[../execution/orchestration.md](../execution/orchestration.md)). - -## `triage-issues` - -Skill: `/autoskillit:triage-issues`. Reads open issues via `gh issue list`, -groups by label and file path, and writes -`.autoskillit/temp/triage/triage_.md`. The report classifies every -issue as `simple`, `needs_check`, or `needs_plan`. - -## `sprint-planner` - -Skill: `/autoskillit:sprint-planner`. Consumes the triage report and groups -issues into independent batches by file overlap. The output is a sprint plan -with explicit "wave 1", "wave 2", … markers that map onto the wavefront -scheduling rule in `recipes/implementation-groups.yaml`. - -## `process-issues` - -Skill: `/autoskillit:process-issues`. Drives `autoskillit order -implementation-groups` against the sprint plan. Each wave runs in parallel -under `pipeline/background.DefaultBackgroundSupervisor`. - -## `pipeline-summary` - -Skill: `/autoskillit:pipeline-summary`. Reads the accumulated audit log, -token log, timing log, and quota events, then opens a GitHub issue listing -every PR that landed, every test that flaked, every quota event, and every -bug surfaced during the run. - -## Quota tuning + buffer - -The quota guard blocks new headless sessions when the binding rate-limit -window crosses its own per-window threshold. Short windows (e.g. `five_hour`) -use `quota_guard.short_window_threshold` (default 85.0%); long windows -matched by `quota_guard.long_window_patterns` (default `weekly`, `sonnet`, -`opus`) use `quota_guard.long_window_threshold` (default 98.0%) — short-window -exhaustion means imminent throttling, while 15% headroom on a multi-day -weekly window is comfortable. - -```yaml -quota_guard: - enabled: true - short_window_threshold: 85.0 # block at 85% for short windows - long_window_threshold: 98.0 # block at 98% for long windows (weekly, sonnet, opus) - long_window_patterns: - - weekly - - sonnet - - opus - buffer_seconds: 60 # extra delay above the strict reset time - cache_path: "~/.claude/autoskillit_quota_cache.json" -``` - -The `buffer_seconds` value is the slack the guard adds to the cache hit -when computing how long to sleep, so multiple parallel pipelines do not -hammer the API at the exact reset second. - -## Interpreting `get_quota_events` - -`get_quota_events` returns the list of throttle events the quota guard has -fired during the current orchestrator session. Each event has a timestamp, -the observed utilization percentage, and the sleep duration the orchestrator -was instructed to wait. Use the event stream to confirm that overnight -pipelines are pacing correctly. - -## Stale session detection - -If a worker emits no output for `run_skill.stale_threshold` seconds (default -1200, 20 minutes), the orchestrator records a `stale` `retry_reason` and -re-spawns from scratch. Stale events are visible in the audit log under the -step that produced them. - -## Overnight scheduling pattern - -The recommended pattern for an overnight run on a 50-issue sprint: - -| Phase | Time | Activity | -|-------|------|----------| -| 19:00 | 5 min | Run `triage-issues` and review the report | -| 19:10 | 5 min | Run `sprint-planner` and adjust sprint groupings | -| 19:15 | 8 hours | Launch `process-issues` and walk away | -| 03:30 | 5 min | Read `pipeline-summary` output the next morning | - -## Per-pipeline env-var overrides - -Individual pipeline runs can override config values via `AUTOSKILLIT_*` -environment variables (see `cli/_prompts.py` for the full list). The most -common override is the API key per project: - -```bash -AUTOSKILLIT_GITHUB__TOKEN=ghp_xxx autoskillit order implementation -``` - -## CI integration with `autoskillit migrate --check` - -Add `autoskillit migrate --check` to the project's CI matrix to fail the -build if any project-local recipes are stale relative to the running -package. Pair with the gitleaks pre-commit hook to enforce the secrets-in- -secrets-file invariant on every commit. - -## `report_bug` auto-dedup - -When `pipeline-summary` discovers a bug fingerprint that already has an -open GitHub issue, `report_bug` deduplicates by appending a comment to the -existing issue rather than opening a new one. Dedup is keyed by the bug -fingerprint emitted by the bug-investigation skill. - -## Quota-guard multi-window selection - -`execution/quota.py` reads up to three rolling windows from the API quota -endpoint (1-hour, 5-hour, daily). The guard selects the window with the -highest current utilization and blocks against that one. This avoids the -common pitfall of being well under the daily budget but pegged on the -hourly rate. - -## See also - -- [../examples/research-pipeline.md](../examples/research-pipeline.md) — - end-to-end example with merged research and archive PRs. diff --git a/docs/orchestration-levels.md b/docs/orchestration-levels.md new file mode 100644 index 0000000000..a522a4a3f4 --- /dev/null +++ b/docs/orchestration-levels.md @@ -0,0 +1,127 @@ +# Orchestration Levels + +AutoSkillit uses four orchestration levels (L0–L3) that describe *who can spawn +whom* at runtime. These are entirely separate from the import-layer IL-N labels +used in module docstrings and `pyproject.toml` import-linter contracts. + +## Level Definitions + +### L0 — Leaf Subagent + +A terminal node (actual leaf) in the execution graph. L0 agents are always +headless, spawned by an L1 session via Claude Code's Agent or Task tool. They +cannot launch sub-agents or headless sessions of their own. + +Key properties: + +- Always headless (never interactive) +- Spawned via the Agent/Task tool, not `run_skill` +- Cannot call `run_skill`, `run_cmd`, or `run_python` +- Session type: n/a (Claude Agent, not a full session) + +### L1 — Session + +A Claude Code session (interactive or headless) that can spawn L0 leaf +subagents. When running headless, an L1 is a `run_skill` worker dispatched by +an L2 orchestrator. + +Key properties: + +- Interactive variant: `autoskillit cook` +- Headless variant: `run_skill` worker +- SessionType: `SKILL` (headless variant); interactive cook carries CLI label `"cook"` (not a SessionType enum member) +- Can spawn L0 subagents via Agent/Task tool +- Headless variant cannot call `run_skill` *in headless mode* (enforced by + `skill_orchestration_guard.py` and `skill_cmd_guard.py`); interactive L1 sessions + bypass all tier guards via the ``AUTOSKILLIT_HEADLESS`` short-circuit + +``` +L1 (interactive cook) +└── L0 subagent (Agent/Task tool) + └── [terminal — spawns nothing] + +L1 (headless run_skill worker) +└── L0 subagent (Agent/Task tool) + └── [terminal — spawns nothing] +``` + +### L2 — Orchestrator + +Orchestrates L1 headless sessions by dispatching them via `run_skill`. The L2 +reads the recipe, calls MCP tools, and routes verdicts. It never reads or writes +code itself. + +Key properties: + +- Interactive variant: `autoskillit order` (CLI label `"order"`; headless equivalent carries SessionType `ORCHESTRATOR`) +- Headless variant: food truck (dispatched by L3, SessionType `ORCHESTRATOR`) +- Spawns L1 workers via `run_skill` +- Has full kitchen access (37 kitchen-tagged MCP tools) + +``` +L2 (interactive order) +└── L1 worker (run_skill) + └── L0 subagent (Agent/Task tool) + +L2 (headless food truck, dispatched by L3) +└── L1 worker (run_skill) + └── L0 subagent (Agent/Task tool) +``` + +### L3 — Fleet Dispatcher + +Manages a fleet of L2 food trucks, dispatching them to process batches of +issues or repositories. There is no L4, so L3 has no headless variant. + +Key properties: + +- Interactive only: `autoskillit fleet` (SessionType `FLEET`) +- No headless variant (nothing above L3 to dispatch it) +- Dispatches L2 food trucks via `run_skill` +- Manages campaign state via the sidecar JSONL file + +``` +L3 (interactive fleet) +└── L2 food truck (run_skill → headless L2) + └── L1 worker (run_skill) + └── L0 subagent (Agent/Task tool) +``` + +## Mapping Table + +| Orchestration Level | SessionType enum (headless) | CLI label (interactive) | CLI command | Headless variant | +|---|---|---|---|---| +| L0 (leaf) | n/a — Claude Agent | n/a | n/a | Always headless | +| L1 (session) | `SKILL` | `"cook"` | `autoskillit cook` | `run_skill` worker | +| L2 (orchestrator) | `ORCHESTRATOR` | `"order"` | `autoskillit order` | Food truck | +| L3 (fleet) | `FLEET` | `"fleet"` | `autoskillit fleet` | None — no L4 exists | + +> **Note:** `SessionType` enum values (`SKILL`, `ORCHESTRATOR`, `FLEET`) apply to headless +> sessions only and are read from the `AUTOSKILLIT_SESSION_TYPE` environment variable. +> Interactive sessions carry CLI display labels (`"cook"`, `"order"`, `"fleet"`) that are +> not `SessionType` enum members and bypass tier enforcement via the +> `AUTOSKILLIT_HEADLESS` short-circuit in tier guards. + +## Key Rules + +- **Headless L1 workers cannot call `run_skill`.** The boundary is enforced three ways: + FastMCP visibility tags, the `skill_orchestration_guard.py` PreToolUse hook, + and the `_require_orchestrator_or_higher()` runtime guard in + `tools_execution.py`. All three must independently agree. +- **L0 agents cannot launch anything.** They are terminal nodes — they cannot + call `run_skill`, cannot invoke the Agent tool to spawn sub-agents, and + cannot open sub-sessions. (L0 agents are themselves spawned via Agent/Task + by an L1 — the constraint is on outbound calls only.) +- **L3 has no headless variant.** There is no L4 to dispatch an L3. Fleet + always runs interactively. +- **Spawning is strictly downward.** An L2 dispatches L1, an L1 spawns L0. + No level can spawn a peer or a higher level. +- **food trucks are L2, not L1.** A food truck is a headless L2 session + dispatched by an L3 fleet. It retains full orchestrator capabilities + (it can call `run_skill` to spawn L1 workers). + +## Disambiguation + +> Module docstrings and import-linter comments use IL-N (IL-0 through IL-3) for +> the import dependency hierarchy — these are NOT orchestration levels. See the +> import-linter contracts IL-001 through IL-009 in `pyproject.toml`. diff --git a/docs/recipes/authoring.md b/docs/recipes/authoring.md index 7c281dc56c..0011f9cd04 100644 --- a/docs/recipes/authoring.md +++ b/docs/recipes/authoring.md @@ -6,8 +6,8 @@ contract / migration / staleness pipeline keeps recipes from rotting. ## Family vs bundled A **bundled recipe** lives under `src/autoskillit/recipes/*.yaml` and ships -with the plugin (5 today: `implementation`, `implementation-groups`, -`merge-prs`, `remediation`, `research`). A **project-local recipe** lives +with the plugin (6 today: `implementation`, `implementation-groups`, +`merge-prs`, `planner`, `remediation`, `research`). A **project-local recipe** lives under `.autoskillit/recipes/*.yaml` in a downstream project and overrides the bundled name. @@ -29,8 +29,8 @@ top-level keys (defined in `recipe/schema.py:Recipe`): | `requires_packs` | list[str] | Sub-recipes or named packs to import | | `steps` | list[RecipeStep] | The sequenced step graph | -`RecipeStep` adds: `id`, `tool`, `params`, `capture`, `verdict_routes`, -`skip_when_false`, `retry`. +`RecipeStep` adds: `name`, `tool`, `with_args`, `capture`, `on_result`, +`skip_when_false`, `retries`. ## Authoring flow @@ -44,32 +44,39 @@ top-level keys (defined in `recipe/schema.py:Recipe`): 5. Add the recipe to your project's `.autoskillit/recipes/` and run a small `order` against a sandbox issue. -## The 17 semantic rule families +## The 24 semantic rule families `validate_recipe` runs every rule registered with `recipe/registry.py:semantic_rule`. -The rule families live in `src/autoskillit/recipe/rules_*.py` (17 files): +The rule families live in `src/autoskillit/recipe/rules_*.py` (24 files): | File | What it catches | |------|-----------------| +| `rules_actions.py` | Action-type semantic rules: `stop-step-has-no-routing`, `recipe-has-terminal-step`, `route-step-requires-on-result` | +| `rules_blocks.py` | Block-level budget rules: per-block `run_cmd` and `run_skill` call-count budgets | | `rules_bypass.py` | `skip_when_false` routes that have no fallthrough — a step that gets bypassed must also have a downstream consumer that handles the bypass | +| `rules_campaign.py` | Campaign recipe structural rules (step count, required ingredient presence for campaign mode) | | `rules_ci.py` | CI polling steps written as inline shell commands instead of `wait_for_ci` | | `rules_clone.py` | Clone/push workflow integrity — every `clone_repo` must be paired with a `register_clone_status` and a `remove_clone` route | | `rules_cmd.py` | `run_cmd` echo-capture alignment and `find` rediscovery anti-patterns | | `rules_contracts.py` | Skills referenced in steps that are missing required output patterns in their contract card | | `rules_dataflow.py` | Capture/output dataflow — a captured value must be consumed by a downstream step or removed | +| `rules_features.py` | Feature-flag gating rules: steps that reference features not declared in `requires_packs` | +| `rules_fixing.py` | Fixing/Rectify workflow rules: rectify loops missing a terminal success route | | `rules_graph.py` | Step graph reachability and cycle detection | | `rules_inputs.py` | Ingredient/version validation — required ingredients are present, types are correct | | `rules_isolation.py` | Workspace isolation breaches: `source-isolation-violation` and `git-mutation-on-source` | | `rules_merge.py` | `merge_worktree` routing completeness — every `MergeFailedStep` value must have a route | | `rules_packs.py` | Unknown pack names referenced in `requires_packs` | +| `rules_reachability.py` | Symbolic reachability: `capture-inversion-detection`, `event-scope-requires-upstream-capture` | | `rules_recipe.py` | Unknown sub-recipe references inside a `requires_packs` entry | | `rules_skill_content.py` | SKILL.md bash-block placeholder validation (`undefined-bash-placeholder`) | +| `rules_temp_path.py` | Temp-path hygiene: steps that write to paths outside `.autoskillit/temp/` | | `rules_skills.py` | `skill_command` resolvability against the bundled skill registry (`unknown-skill-command`) | | `rules_tools.py` | MCP tool name and parameter validity (`unknown-tool`, `dead-with-param`) | | `rules_verdict.py` | Skill verdict routing completeness — every emitted verdict value must be routed | | `rules_worktree.py` | Worktree retry lifecycle — every step that creates a worktree must have a downstream merge or cleanup | -The 17-family count is enforced by `tests/docs/test_doc_counts.py`. +The 24-family count is enforced by `tests/docs/test_doc_counts.py`. ## Contract cards diff --git a/docs/recipes/composition.md b/docs/recipes/composition.md index 5837541dba..a021ce63f0 100644 --- a/docs/recipes/composition.md +++ b/docs/recipes/composition.md @@ -13,13 +13,13 @@ In a recipe YAML, add a step with `sub_recipe:` and `gate:` fields: ```yaml steps: - sprint_entry: - sub_recipe: sprint-prefix # name of sub-recipe (no extension) - gate: sprint_mode # ingredient controlling activation - on_success: done # where to route after sub-recipe completes + prefix_entry: + sub_recipe: my-prefix # name of sub-recipe (no extension) + gate: enable_prefix # ingredient controlling activation + on_success: done # where to route after sub-recipe completes on_failure: escalate_stop note: > - When gate=true, sprint-prefix steps are merged here. + When gate=true, my-prefix steps are merged here. When gate=false (default), this step is dropped at load time. ``` @@ -27,8 +27,8 @@ The referenced ingredient must be declared as hidden so it is not shown to agent ```yaml ingredients: - sprint_mode: - description: Enable sprint sub-recipe prefix + enable_prefix: + description: Enable the prefix sub-recipe default: "false" hidden: true ``` @@ -43,18 +43,11 @@ When the gate ingredient is `"true"`, the sub-recipe's steps are **merged inline the attachment point, with step names prefixed to avoid collisions. The sub-recipe's kitchen rules are merged with the parent's kitchen rules. -## Activating Sprint Mode - -Pass `overrides` when loading the recipe via the `open_kitchen` MCP tool: +Pass `overrides` when loading the recipe via the `open_kitchen` or `load_recipe` MCP +tool to activate a gated sub-recipe: ``` -open_kitchen(name="implementation", overrides={"sprint_mode": "true"}) -``` - -Or via the `load_recipe` MCP tool: - -``` -load_recipe(name="implementation", overrides={"sprint_mode": "true"}) +open_kitchen(name="my-recipe", overrides={"enable_prefix": "true"}) ``` ## Sub-Recipe File Location @@ -73,27 +66,3 @@ Hidden ingredients (`hidden: true`) are suppressed from the ingredients table sh to agents at recipe load time. Agents will not see them unless the ingredient is explicitly passed via `overrides`. This keeps the default runtime experience clean while allowing advanced activation patterns. - -## Example: Sprint Mode in the Implementation Recipe - -The `implementation` recipe includes a `sprint_entry` step gated on `sprint_mode`: - -- **Default** (`sprint_mode="false"`): `sprint_entry` is dropped; the recipe starts - at `clone` and runs the standard single-issue pipeline. -- **Sprint mode** (`sprint_mode="true"`): `sprint-prefix` steps are merged as a prefix, - running the full sprint workflow (triage → sprint planning → user confirmation → - per-issue dispatch → sprint report). - -The same pattern applies to the `remediation` recipe. - -### How Sprint Mode Works - -Sprint mode transforms a single-issue recipe into a batch processor: - -1. **Triage** — `triage-issues` analyzes open GitHub issues and produces a sequenced plan -2. **Sprint Planning** — `sprint-planner` selects a conflict-free subset of issues for the sprint -3. **User Confirmation** — the orchestrator presents the sprint plan and waits for approval -4. **Dispatch** — for each issue in the sprint, the standard pipeline runs (plan → implement → test → merge → PR) -5. **Sprint Report** — a summary issue is created linking all PRs from the sprint - -This reuses the existing recipe steps — the sub-recipe only adds the triage/planning prefix and the per-issue dispatch loop. diff --git a/docs/recipes/overview.md b/docs/recipes/overview.md index 5c9d6072fd..dec32cbd04 100644 --- a/docs/recipes/overview.md +++ b/docs/recipes/overview.md @@ -4,7 +4,7 @@ Recipes are YAML pipeline definitions that automate multi-step workflows. Each r ## Bundled Recipes -AutoSkillit ships with 5 recipes: +AutoSkillit ships with 6 bundled recipes: ### implementation @@ -78,6 +78,21 @@ Consolidate multiple open PRs into a single integration branch and PR. | `base_branch` | *(auto-detect)* | Branch that all PRs target | | `audit` | `true` | Run quality audit on conflict resolutions | +### planner + +Progressive decomposition pipeline. Analyzes a codebase, generates phases, elaborates assignments and work packages in iterative loops, reconciles dependencies, validates the dependency graph, and compiles a manifest ready for issue creation. + +**Flow:** init → analyze → [extract_domain?] → phases-loop → assignments-loop → wps-loop → reconcile → validate/refine → compile + +**When to use:** When you need to decompose a large task into a structured set of GitHub issues with a validated dependency graph. Produces per-WP issue markdown files and a plan manifest. + +**Key ingredients:** + +| Ingredient | Default | Description | +|-----------|---------|-------------| +| `task` | *(required)* | High-level task or feature description to decompose into a plan | +| `source_dir` | *(required)* | Absolute path to the repository root to analyze | + ### research Two-phase technical research recipe. Phase 1 scopes a research question and opens an experiment design issue for human review. Phase 2 implements the experiment, runs it, writes a report to `research/`, and opens a PR. @@ -132,7 +147,7 @@ steps: message: "Failed." ``` -See **[Sub-Recipe Composition](composition.md)** for advanced patterns like sprint mode. +See **[Sub-Recipe Composition](composition.md)** for advanced patterns like gated sub-recipe prefixes. ## Recipe Management diff --git a/docs/research/audit-trail-format.md b/docs/research/audit-trail-format.md new file mode 100644 index 0000000000..cf46b952b9 --- /dev/null +++ b/docs/research/audit-trail-format.md @@ -0,0 +1,78 @@ +# Audit Trail Format + +Documents the structure of `research/{slug}/audit/` artifacts produced by the +research recipe pipeline. + +## Directory Structure + +``` +research/{YYYY-MM-DD}-{slug}/ +├── audit/ +│ ├── design-review-dashboard.md ← copied from AUTOSKILLIT_TEMP/review-design/ +│ └── visualization-plan-trace.md ← copied from AUTOSKILLIT_TEMP/plan-visualization/ +├── report.md ← references audit/ via YAML frontmatter +├── scripts/ +│ └── ... +└── data/ + └── ... +``` + +## Artifact Descriptions + +### `audit/design-review-dashboard.md` + +**Source:** `{{AUTOSKILLIT_TEMP}}/review-design/evaluation_dashboard_{slug}_{timestamp}.md` +**Copied by:** `scripts/recipe/create_worktree.sh` +**When:** During the `create_worktree` recipe step + +Contains: +- Verdict banner (GO / REVISE / STOP) +- Classification summary (experiment type, methodology tradition) +- Dimension rationale (per-dimension weight table) +- Dimension scorecard (dimension → weight → findings → severity) +- Adversarial (red-team) findings +- "Cannot Assess" entries +- Mechanizable check log +- Machine-readable YAML summary block +- Silent-type advisories (appended when applicable — see `silent-type-convention.md`) + +### `audit/visualization-plan-trace.md` + +**Source:** `{{AUTOSKILLIT_TEMP}}/plan-visualization/visualization-plan-trace.md` +**Copied by:** `scripts/recipe/create_worktree.sh` +**When:** During the `create_worktree` recipe step + +Contains: +- `primary_tradition` — the methodology tradition selected by Tier-C routing +- `applied_union_rules` — which union rules were applied (from disambiguation) +- `precedence_trace` — the precedence resolution chain +- `tier_c_lens` — which vis-lens was dispatched (e.g., `vis-lens-methodology-norms`) +- `disambiguation_rule_applied` — which disambiguation rule fired, or null +- Silent-type advisories (appended when applicable — see `silent-type-convention.md`) + +## Report Frontmatter Reference + +`report.md` references audit artifacts via YAML frontmatter: + +```yaml +--- +experiment_type: causal_inference +methodology_traditions: + - controlled_intervention +disambiguation_rule_applied: null +tier_c_lens: vis-lens-methodology-norms +design_review_verdict: GO +classification_timestamp: "2026-04-13T15:32:00Z" +audit_trail_path: + design_review: research/{slug}/audit/design-review-dashboard.md + visualization_trace: research/{slug}/audit/visualization-plan-trace.md +--- +``` + +## Lifecycle + +1. **Ephemeral:** Artifacts are written to `{{AUTOSKILLIT_TEMP}}/` during recipe execution +2. **Persisted:** `create_worktree.sh` copies them to `research/{slug}/audit/` +3. **Committed:** The `git add research/ && git commit` at the end of `create_worktree.sh` commits them +4. **Referenced:** `generate-report` writes relative paths into the report frontmatter +5. **Archived:** When the research bundle is archived (PR or local export), audit artifacts travel with it diff --git a/docs/research/experiment-type-rationale.md b/docs/research/experiment-type-rationale.md new file mode 100644 index 0000000000..8ee775cbee --- /dev/null +++ b/docs/research/experiment-type-rationale.md @@ -0,0 +1,400 @@ +# Experiment-Type Dimension-Weight Rationale + +This document records the authoritative rationale for every non-default dimension weight +assigned to the seven new experiment types introduced in this project: +`evidence_synthesis`, `observational_correlational`, `instrument_validation`, +`simulation_modeling`, `factorial_design`, `single_subject`, and +`qualitative_interpretive`. + +**Scope.** Only non-default weights are documented here. The dimension +`resource_proportionality` defaults to `L` for every type and is not discussed +further. Weights of `M` (medium) are the implicit baseline for most dimensions; any +departure to `H` (high), `L` (low), or `S` (silent) is explained below. + +**Citation removal note.** An earlier draft of this material contained synthetic +deep-research citation markers (left-double-angle-bracket + line-reference fragments). +All such markers have been removed. Every verifiable claim now cites a source from the +Reference List at the end of this document; claims that cannot be traced to a specific +published passage use appropriately hedged language ("may require", "is often +recommended", "typically involves"). + +--- + +## 1. `evidence_synthesis` + +Evidence synthesis — systematic reviews and meta-analyses — aggregates effect estimates +from a heterogeneous body of primary studies. The design does not generate new +observations directly; it reanalyses existing data. This derivation character shapes +nearly every dimension weight. + +### `variance_protocol: H` + +Heterogeneity quantification is a first-order concern in any meta-analysis. +Borenstein, Hedges, Higgins, and Rothstein [B1] devote substantial treatment to the +choice between fixed-effect and random-effects models, showing that the between-study +variance component (tau-squared) must be estimated and reported regardless of which +model is chosen. The *Cochrane Handbook* [C1] §9.5 similarly requires heterogeneity +statistics (I-squared, Q-statistic) in every forest plot. Because the analyst has no +direct control over primary-study design, all variance-reduction levers — sampling +decisions, blocking, stratification — must be handled at the protocol level through +inclusion/exclusion criteria and subgroup definitions. This makes the variance-protocol +dimension substantially more demanding than in primary studies where randomisation +handles most confounding directly. + +### `statistical_corrections: H` + +Systematic reviews commonly test multiple subgroup hypotheses, moderators, and +sensitivity analyses in a single synthesis. The *Cochrane Handbook* [C1] §16.7.2 notes +that multiplicity across subgroup comparisons inflates the Type I error rate and +recommends pre-registering a small number of a-priori subgroups and treating post-hoc +analyses as exploratory. Borenstein et al. [B1] further discuss publication-bias +corrections (trim-and-fill, Egger's test, PET-PEESE) that constitute an additional +layer of statistical adjustment. The combination of heterogeneity modelling, subgroup +testing, and publication-bias assessment makes the statistical-corrections burden +substantially higher than in a single, pre-registered primary study. + +### `ecological_validity: L` + +A meta-analysis does not generate primary data; its ecological fidelity is derived +entirely from the studies it aggregates. If the included studies themselves were +conducted in naturalistic settings the synthesis may inherit high ecological validity, +but the synthesis design has no direct mechanism to ensure this. Rating ecological +validity as a weight-worthy concern at the synthesis level is generally not meaningful +— reviewers can note effect-size heterogeneity across applied versus laboratory studies, +but they cannot redesign the primary studies. An `L` weight reflects this structural +limitation. + +### `causal_structure: M` + +Random-effects meta-analysis can support causal inference when the included studies are +randomised controlled trials, but the inference remains contingent on the quality of +randomisation in primary studies [B1], [C1]. When the synthesis includes both +experimental and observational studies, causal claims require careful qualification. +The weight is retained at `M` (medium) rather than elevated to `H` because the causal +machinery belongs to the primary studies, not the synthesis design itself, and because +many syntheses are explicitly descriptive rather than causal. + +--- + +## 2. `observational_correlational` + +Observational studies — cross-sectional surveys, cohort studies, case-control designs +— observe naturally occurring variation without experimental manipulation. Their +distinctive challenge is separating association from causation in the absence of +randomisation. + +### `causal_structure: L` + +Without randomisation or an equivalent quasi-experimental device, observed associations +cannot be given unambiguous causal interpretations. Rosenbaum [R1] provides a +systematic treatment of sensitivity analysis — the tools used to ask how strong an +unmeasured confounder would have to be to explain away a detected association — and +concludes that such analyses are necessary precisely because the observational setting +cannot rule out confounding by design. Shadish, Cook, and Campbell [S1] Chapter 1 +similarly distinguish between descriptive, predictive, and causal purposes, noting that +observational designs are strongest for the first two. A low weight on causal structure +reflects not that causality is irrelevant but that the design affords limited leverage +over it. + +### `variance_protocol: H` + +In the absence of randomisation, confounder selection and adjustment strategy become +the primary design decisions. Rosenbaum [R1] Chapter 3 argues that the choice of +matched or stratified design — which covariates to include, which balance metric to +optimise — is the observational analogue of randomisation and demands equally careful +prespecification. Failing to document the variance protocol makes replication and +sensitivity analysis impossible. The high weight reflects that observational analysts +effectively substitute design-time variance decisions for the automated balance that +randomisation provides. + +### `ecological_validity: H` + +Observational studies are conducted in real-world settings, which is precisely their +comparative advantage over laboratory experiments. Shadish et al. [S1] §3.4 discuss +external validity at length, noting that observational cohort studies and surveys +typically recruit from broader and more representative populations than convenience +samples used in laboratory research. When the research question concerns naturally +occurring behaviour, ecological validity is a deliberate strength of the design, not an +incidental feature, warranting a high weight. + +### `measurement_alignment: M` + +When constructs are measured in naturalistic settings, instrument performance may differ +from the psychometric conditions under which the instrument was developed. Kline [K1] +Chapter 2 discusses reliability generalisation — the observation that a coefficient +alpha obtained in a standardisation sample need not transfer to a new population or +context. Measurement alignment remains a medium priority rather than high because the +observational analyst typically cannot redesign the instrument mid-study; the concern is +real but bounded in scope compared to an instrument-validation study where measurement +is the primary object. + +--- + +## 3. `instrument_validation` + +Instrument-validation studies exist to characterise the psychometric properties of a +measure: reliability, factorial structure, convergent and discriminant validity, and +population-specific functioning (differential item functioning, DIF). + +### `measurement_alignment: H` + +Measurement is the design's primary object. Kline [K1] provides comprehensive treatment +of reliability coefficients (Cronbach's alpha, test-retest, parallel forms), exploratory +and confirmatory factor analysis, and the multi-trait multi-method matrix as a framework +for construct validity. Every design decision in an instrument-validation study — sample +size, calibration versus holdout split, anchor item selection — is motivated by the goal +of characterising measurement properties. No other dimension approaches this centrality, +which justifies the high weight. + +### `causal_structure: L` + +Instrument validation is fundamentally correlational in nature. The goal is to show +that items hang together (internal consistency), that the scale correlates with related +constructs (convergent validity), and that it does not correlate with unrelated +constructs (discriminant validity). None of these goals require or support causal +claims. Attributing a causal direction to an item-factor relationship would be a +category error in standard psychometric theory [K1] Chapter 5. + +### `ecological_validity: M` + +A validation study must demonstrate that the instrument functions adequately in the +population for which it is intended. Kline [K1] Chapter 11 discusses DIF analysis and +population portability: a scale validated on a university convenience sample may behave +differently in clinical or community samples. Ecological validity is therefore relevant +at a medium level — the analyst must show generalisability beyond the calibration +sample — but it is subordinate to measurement alignment. + +### `statistical_corrections: H` + +Instrument validation typically involves iterative model fitting: initial exploratory +factor analysis guides item selection, which informs a confirmatory factor analysis on +a separate sample. Kline [K1] §8.6 warns explicitly about capitalising on chance when +item selection is data-driven, recommending split-sample or cross-validation strategies. +DIF analysis across subgroups adds further multiplicity. The statistical corrections +burden is high because the same dataset is often interrogated for reliability, +factorial validity, and convergent validity simultaneously. + +--- + +## 4. `simulation_modeling` + +Simulation studies — discrete-event, Monte Carlo, agent-based — generate synthetic data +from a formal model of a system. They are used to study systems that are too costly, +too dangerous, or too slow to observe directly. + +### `variance_protocol: H` + +Simulation output is stochastic, and the analyst controls the variance entirely through +experimental design choices: number of replications, common random number streams, +antithetic variates, and control variates. Law and Kelton [L1] Chapter 11 provide +detailed guidance on replication-and-deletion warm-up analysis and on variance-reduction +techniques. Without a careful variance protocol, confidence intervals on simulation +output metrics may be unreliable regardless of how faithful the model is to the real +system. The high weight reflects that variance management is an intrinsic part of the +simulation design, not an afterthought. + +### `ecological_validity: L` + +A simulation model is an approximation. Law and Kelton [L1] §5.4 discuss model +validation — the process of determining whether a model is an accurate enough +representation for its intended purpose — and note that no model is valid in an absolute +sense. The degree to which simulation results generalise to the real system depends on +how thoroughly the model has been validated and on the assumptions embedded in its +structure. Because ecological validity is structurally bounded by the approximation +inherent to any model, it warrants a low weight relative to internal (within-model) +validity. + +### `causal_structure: M` + +Within a well-specified simulation model, causal statements are valid by construction: +the analyst controls all mechanisms and can apply counterfactual interventions cleanly. +However, the real-world causal interpretation of those results is conditional on model +validity [L1]. Because causal claims are unambiguous within the model but uncertain in +their external application, the weight is medium — significant but not the primary +design challenge. + +### `statistical_corrections: M` + +When a simulation study compares alternative system configurations or parameter +settings, multiple comparisons arise in the same way as in any designed experiment. Law +and Kelton [L1] §12.3 discuss procedures for comparing means across alternatives, +including Bonferroni corrections and ranking-and-selection methods. The medium weight +reflects that multiplicity is a real concern in comparative simulation studies but is +often managed by the structured experimental design (e.g., paired replications via +common random numbers) rather than post-hoc corrections alone. + +--- + +## 5. `factorial_design` + +Factorial experiments manipulate two or more factors simultaneously, allowing estimation +of both main effects and interaction effects. They are the primary vehicle for testing +hypotheses about how factors combine. + +### `causal_structure: H` + +Full factorial designs with randomisation provide the strongest causal infrastructure +of any standard experimental design. Montgomery [M1] Chapter 5 explains that the +factorial structure allows unconfounded estimation of every main effect and interaction +simultaneously, and that randomisation to factor-level combinations justifies causal +interpretation of all effects. The high weight reflects that causal structure is not +merely an aspiration but a built-in property of the design — the reason factorial +experiments are preferred over one-factor-at-a-time investigations. + +### `statistical_corrections: H` + +A full two-level factorial design with k factors yields 2^k − 1 effects (k main +effects plus all interactions). Montgomery [M1] §6.4 discusses the multiplicity +problem in detail: with many contrasts, the familywise error rate climbs rapidly. Common +approaches include Lenth's method for unreplicated factorials, Bonferroni correction +for planned comparisons, and graphical half-normal plots to distinguish signal from +noise. The high weight reflects the systematic multiplicity inherent in factorial +designs rather than researcher degrees of freedom. + +### `variance_protocol: M` + +Blocking and split-plot structures are common in factorial designs when complete +randomisation is impractical. Montgomery [M1] Chapter 7 covers randomised complete +block designs and their factorial extensions, showing that the choice of blocking factor +and the associated error term affect the power of tests for each effect. The medium +weight reflects that variance protocol decisions are important but are resolved at the +design stage rather than constituting an ongoing analytical challenge. + +### `ecological_validity: L` + +Factorial designs are optimised for internal validity: the careful crossing of factor +levels and randomisation of units to cells gives precise causal estimates within the +experiment. This precision often comes at the cost of external validity — factor levels +may be set at values chosen for orthogonality rather than for representativeness of +real-world conditions. The low weight reflects the design's deliberate prioritisation +of internal over external validity, not a deficiency. + +--- + +## 6. `single_subject` + +Single-subject research designs — also called single-case experimental designs — +examine behaviour change within one or a small number of individuals using repeated +measurement and phase alternation (e.g., ABAB reversal, multiple baseline). + +### `causal_structure: M` + +Single-subject designs can support causal inference through intra-subject replication: +demonstrating that behaviour changes when and only when the intervention is introduced +[Ka1]. Kazdin [Ka1] Chapter 4 explains that the logic of replication within a single +participant — across phases, across baselines, across participants in a multiple-baseline +design — provides the same inferential warrant that between-group replication provides +in group experiments. However, the causal inference is limited to the individual(s) +studied and may not generalise, which prevents elevation to a high weight. + +### `ecological_validity: H` + +Single-subject research emerged from applied behaviour analysis, where interventions are +almost always implemented in naturalistic settings: schools, clinics, homes, workplaces. +Kazdin [Ka1] Chapter 2 emphasises ecological fidelity as a design value: the +intervention must work in the environment where behaviour naturally occurs, not in a +specially constructed laboratory environment. This applied orientation makes ecological +validity a primary design concern warranting a high weight. + +### `variance_protocol: L` + +The inferential logic of single-subject designs rests on visual analysis of graphed +data — the stability of baseline phases and the magnitude and immediacy of change upon +intervention — rather than on probabilistic sampling theory [Ka1] Chapter 8. Effect +size indices for single-case designs (PND, IRD, Tau-U) exist but have not supplanted +visual analysis as the primary judgement criterion. Because there is no probability +sample from a population to manage, the variance-protocol machinery of randomised group +designs (power analysis, allocation ratios, stratification) is largely inapplicable, +justifying a low weight. + +--- + +## 7. `qualitative_interpretive` + +Qualitative and interpretive research traditions — phenomenology, grounded theory, +ethnography, case study — seek to understand meaning, process, and context through +non-numerical data (interviews, observations, documents). + +### `ecological_validity: H` + +The central commitment of naturalistic inquiry is to study phenomena in their natural +settings rather than in controlled conditions. Lincoln and Guba [LG1] Part II articulate +"prolonged engagement" and "persistent observation" in the field as core credibility +strategies precisely because ecological authenticity is the design's primary source of +validity. An interview conducted in a participant's workplace or home, or an ethnographic +observation of naturally occurring interaction, maximises ecological validity by +definition. The high weight reflects this foundational commitment. + +### `causal_structure: L` + +Lincoln and Guba [LG1] Chapter 1 explicitly reject linear causality as a goal of +naturalistic inquiry, arguing instead for mutual, simultaneous shaping — the idea that +causes and effects are so intertwined in human social systems that isolating a +unidirectional causal chain is neither feasible nor theoretically appropriate. This is +not a weakness but a deliberate epistemological stance: the goal is to understand how +things are related and what they mean, not to establish that X produces Y. The low +weight follows directly from this stance. + +### `statistical_corrections: L` + +Null hypothesis significance testing does not apply in qualitative research. There are +no sampling distributions, no alpha levels, and no multiple-comparison problems in the +classical statistical sense. Credibility and trustworthiness are established through +other means — member checking, triangulation, negative case analysis, audit trails +[LG1] Chapter 11 — none of which involve statistical correction. A low weight reflects +the structural inapplicability of this dimension, not a quality deficit. + +### `measurement_alignment: M` + +Qualitative research employs a trustworthiness framework in place of the +reliability/validity framework used in quantitative research [LG1]. Transferability +(the qualitative analogue of external validity) requires thick description so that +readers can judge applicability to their own contexts. Dependability (the analogue of +reliability) is established through an inquiry audit. These mechanisms partially address +the concerns captured by measurement alignment, making this dimension relevant at a +medium level — present but not the primary analytical challenge. + +### `variance_protocol: L` + +Qualitative research uses purposive or theoretical sampling rather than probability +sampling. The goal is to select cases that are information-rich for the phenomenon under +study [LG1] Chapter 9, not to obtain a representative sample from a defined population. +Because there is no sampling distribution to manage, the variance-protocol machinery +(sample-size formulae, power analysis, stratification) is structurally inapplicable. +The low weight reflects this difference in sampling logic. + +--- + +## Reference List + +[B1] Borenstein, M., Hedges, L. V., Higgins, J. P. T., & Rothstein, H. R. +*Introduction to Meta-Analysis*. Wiley, 2009. ISBN 978-0-470-05724-7. + +[C1] Higgins, J. P. T., Thomas, J., Chandler, J., Cumpston, M., Li, T., Page, M. J., +& Welch, V. A. (Eds.). *Cochrane Handbook for Systematic Reviews of Interventions* +(2nd ed.). Wiley, 2019. ISBN 978-1-119-53660-4. + +[K1] Kline, P. *Handbook of Psychological Testing* (2nd ed.). Routledge, 1999. +ISBN 978-0-415-21158-1. + +[Ka1] Kazdin, A. E. *Single-Case Research Designs: Methods for Clinical and Applied +Settings* (3rd ed.). Oxford University Press, 2020. ISBN 978-0-190-07997-0. + +[L1] Law, A. M., & Kelton, W. D. *Simulation Modeling and Analysis* (3rd ed.). +McGraw-Hill, 2000. ISBN 978-0-070-59266-7. + +[LG1] Lincoln, Y. S., & Guba, E. G. *Naturalistic Inquiry*. Sage Publications, 1985. +ISBN 978-0-803-92431-4. + +[M1] Montgomery, D. C. *Design and Analysis of Experiments* (9th ed.). Wiley, 2017. +ISBN 978-1-119-58906-8. + +[R1] Rosenbaum, P. R. *Design of Observational Studies*. Springer, 2010. +ISBN 978-1-441-91212-1. + +[R2] Rosenbaum, P. R. *Observational Studies* (2nd ed.). Springer, 2002. +ISBN 978-0-387-98967-9. + +[S1] Shadish, W. R., Cook, T. D., & Campbell, D. T. *Experimental and +Quasi-Experimental Designs for Generalized Causal Inference*. Houghton Mifflin, 2002. +ISBN 978-0-395-61556-0. diff --git a/docs/research/silent-type-convention.md b/docs/research/silent-type-convention.md new file mode 100644 index 0000000000..c0e2fccd58 --- /dev/null +++ b/docs/research/silent-type-convention.md @@ -0,0 +1,69 @@ +# Silent-Type Convention + +Shared convention for detecting and handling "silent" experiment types and methodology +traditions — those that lack strong dimensional signals or mandatory figure requirements. + +Consumed by Work Items 2.3 (#835) and 4.7 (#846). + +## Detection Criteria + +### Experiment Types + +An experiment type is "silent" when **≥6 of 8 `dimension_weights`** are equal to `S` +(suppressed). Silent experiment types produce valid classifications but lack the +dimensional signal strength to drive opinionated design-review scoring. + +### Methodology Traditions + +A methodology tradition is "silent" when its `mandatory_figures` list is **empty**. +Silent traditions produce valid Tier-C routing results but do not constrain the +visualization plan to specific figure types. + +## Canonical Advisory Schema + +Both experiment-type and methodology-tradition subsystems emit advisories using +this schema when a silent type/tradition is detected: + +```yaml +verdict: GO +advisory_context: + subject_kind: experiment_type | methodology_tradition + subject_name: + reasoning: + reference_framework: + strongly_expected_figures: [, ...] # optional; only populated by vis-lens side +requires_decision: false +``` + +### Field Definitions + +| Field | Type | Description | +|-------|------|-------------| +| `verdict` | `GO` | Always GO — silent types are valid, not errors | +| `subject_kind` | enum | `experiment_type` or `methodology_tradition` | +| `subject_name` | string | Snake-case name of the detected silent type/tradition | +| `reasoning` | string | Human-readable explanation of why this is silent | +| `reference_framework` | string | Which registry/framework detected the silent signal | +| `strongly_expected_figures` | list | Optional; populated only by vis-lens side for traditions | +| `requires_decision` | `false` | Informational only — no human decision needed | + +## Write Targets + +| Detection Source | Advisory Written To | +|-----------------|---------------------| +| Experiment-type subsystem | `research/{slug}/audit/design-review-dashboard.md` | +| Methodology-tradition subsystem | `research/{slug}/audit/visualization-plan-trace.md` | + +Advisories are **appended** to the corresponding audit-dashboard file. They are NOT +written into the report frontmatter — the frontmatter records classification decisions +only, not advisory metadata. + +## Integration Test Expectations + +A qualitative-interpretive fixture plan should produce matching advisories from both +subsystems: +- The experiment-type side detects silent dimensions and writes an advisory to + `design-review-dashboard.md` +- The methodology-tradition side detects empty `mandatory_figures` and writes an + advisory to `visualization-plan-trace.md` +- Both advisories have `verdict: GO` and `requires_decision: false` diff --git a/docs/safety/hooks.md b/docs/safety/hooks.md index 45576e0305..c7ee84d73c 100644 --- a/docs/safety/hooks.md +++ b/docs/safety/hooks.md @@ -1,27 +1,27 @@ # Hooks -AutoSkillit registers 13 Claude Code hook scripts: 9 PreToolUse, 3 PostToolUse, +AutoSkillit registers 21 Claude Code hook scripts: 16 PreToolUse, 4 PostToolUse, and 1 SessionStart. Every script is stdlib-only Python so it can run before the project virtualenv is on the path. Scripts live in `src/autoskillit/hooks/` and are bound to event types in `src/autoskillit/hook_registry.py` via the `HOOK_REGISTRY` list of `HookDef` entries; `generate_hooks_json()` then materializes the canonical `hooks.json` that Claude Code reads. -## PreToolUse hooks (9) +## PreToolUse hooks (16) ### `branch_protection_guard.py` **Guarded tools:** `merge_worktree`, `push_to_remote` Denies merges and pushes targeting branches in `safety.protected_branches` -(`main`, `integration`, `stable` by default). Pure-function check via +(`main`, `develop`, `stable` by default). Pure-function check via `core/branch_guard.is_protected_branch`. -### `quota_check.py` +### `quota_guard.py` **Guarded tool:** `run_skill` Blocks launching new headless sessions when the cached binding window marks `should_block=True`. The threshold is per-window: short windows (e.g. `five_hour`) use `quota_guard.short_window_threshold` (default 85.0%); long windows matched by `quota_guard.long_window_patterns` (default `weekly`, -`sonnet`, `opus`) use `quota_guard.long_window_threshold` (default 98.0%). +`sonnet`, `opus`) use `quota_guard.long_window_threshold` (default 95.0%). Reports the exact sleep duration the orchestrator must wait. ### `skill_command_guard.py` @@ -30,7 +30,7 @@ Blocks `run_skill` calls where `skill_command` does not start with `/`. Catches the case where the orchestrator passed prose instead of a slash command. -### `skill_cmd_check.py` +### `skill_cmd_guard.py` **Guarded tool:** `run_skill` Validates that path-argument skills (`implement-worktree-no-merge`, `resolve-failures`, etc.) receive the file path as the first token rather @@ -46,48 +46,133 @@ unintended deletion of clones that may still have unpushed work. Blocks `open_kitchen` from running inside a headless session. Only human operators may open the kitchen. -### `headless_orchestration_guard.py` +### `ask_user_question_guard.py` +**Guarded tool:** `AskUserQuestion` +Blocks `AskUserQuestion` in headless sessions unless a fresh kitchen-open +marker exists (TTL: 24 hours). Prevents leaf workers from attempting +interactive user prompts that can never be answered. Fails open on parse +errors or missing session ID. Session scope: headless only. + +### `skill_orchestration_guard.py` **Guarded tools:** `run_skill`, `run_cmd`, `run_python` -Blocks orchestration tools from headless sessions. Enforces the two-tier -invariant: only the Tier 1 orchestrator may spawn headless sessions. +Blocks orchestration tools from skill-tier sessions. Enforces the tier +invariant: orchestrator and fleet sessions may call orchestration tools; +skill workers use native Claude Code tools only. ### `unsafe_install_guard.py` **Guarded tool:** `run_cmd` Denies `run_cmd` calls that perform editable installs without `--python .venv`. Prevents pollution of the global Python environment. +### `pr_create_guard.py` +**Guarded tool:** `run_cmd` +Blocks `gh pr create` called via `run_cmd` while the kitchen is open. Uses +`shlex.split` tokenisation to avoid false positives from quoted shell +arguments (e.g. `echo 'do not gh pr create'` does not match). Directs the +caller to use the `prepare_pr → compose_pr` pipeline instead. + ### `generated_file_write_guard.py` **Guarded tools:** `Write`, `Edit` Denies writes to generated files (`hooks.json`, `settings.json`). The hooks file must be regenerated through `generate_hooks_json()`, never edited by hand. -## PostToolUse hooks (3) - -### `pretty_output.py` +### `recipe_write_advisor.py` +**Matched tools:** `Write`, `Edit` +Non-blocking advisory: suggests `/autoskillit:write-recipe` or +`/autoskillit:make-campaign` when writing recipe YAML files in +`.autoskillit/recipes/` or `src/autoskillit/recipes/`. Silently skips +headless sessions to avoid noise in automated runs. Never blocks tool +execution. Session scope: interactive only. + +### `grep_pattern_lint_guard.py` +**Guarded tool:** `Grep` +Denies `Grep` calls that contain `\|` (POSIX BRE alternation) in the +pattern. The Grep tool wraps ripgrep, which uses ERE/PCRE syntax where bare +`|` is alternation; `\|` matches a literal backslash-pipe, causing silent +zero-result failures. Returns the corrected ERE pattern (replacing `\|` with +`|`) in the deny message. + +### `mcp_health_advisor.py` +**Matched tools:** `Bash`, `Write`, `Edit`, `Read`, `Glob`, `Grep` +Detects MCP server disconnection by reading `active_kitchens.json` and checking +PID liveness. Injects informational message suggesting `/MCP` reconnection when +all registered server PIDs for the project are dead. Never blocks tool execution. +Interactive sessions only. + +### `fleet_dispatch_guard.py` +**Guarded tool:** `dispatch_food_truck` +Blocks `dispatch_food_truck` from headless callers. Prevents recursive +L3→L3 fleet session creation where a headless session launches another fleet +of headless sessions. Fails open on malformed input. Session scope: headless +calls are denied; interactive callers pass through. + +### `review_loop_gate.py` +**Guarded tools:** `wait_for_ci`, `enqueue_pr` +Blocks these tools when `review_gate_state.json` has `gate == "LOOP_REQUIRED"` +and `check_review_loop` has not yet been called. Enforces the review-loop +invariant: after a `changes_requested` verdict the orchestrator must call +`run_python` with `callable='autoskillit.smoke_utils.check_review_loop'` +before proceeding to CI/merge steps. + +## PostToolUse hooks (4) + +### `pretty_output_hook.py` **Guarded tools:** all AutoSkillit MCP tools Reformats raw JSON responses into Markdown key-value pairs for readable display and reduced token usage. -### `token_summary_appender.py` +### `token_summary_hook.py` **Guarded tool:** `run_skill` After `run_skill` returns a GitHub PR URL, appends a `## Token Usage Summary` table to the PR body so reviewers can see per-step token cost. -### `quota_post_check.py` +### `quota_post_hook.py` **Guarded tool:** `run_skill` After `run_skill` returns, appends a quota warning to the tool output when the cached binding window marks `should_block=True` (per-window threshold — -see `quota_check.py` above). Gives the orchestrator a chance to back off +see `quota_guard.py` above). Gives the orchestrator a chance to back off voluntarily before the PreToolUse hook starts denying. +### `review_gate_post_hook.py` +**Guarded tools:** `run_skill`, `run_python` +Writes, updates, or clears `review_gate_state.json` in response to gate +sentinel tags in `run_skill` output: `%%REVIEW_GATE::LOOP_REQUIRED%%` sets +the gate and records the PR number; `%%REVIEW_GATE::CLEAR%%` removes the +state file. When `run_python` calls `check_review_loop`, marks +`check_review_loop_called: True` in the state so `review_loop_gate.py` will +unblock `wait_for_ci`/`enqueue_pr`. + ## SessionStart hook (1) -### `session_start_reminder.py` +### `session_start_hook.py` Injects a reminder to call `/autoskillit:open-kitchen` when resuming a prior session (transcript_path size > 0). Without this, resumed orchestrator sessions silently lose access to the kitchen tools. +## Fail Modes + +All guard scripts fail-**open** for malformed or unparseable input: a JSON decode +failure produces exit 0 (approve). This prevents a broken hook from blocking the +entire tool chain. + +Three guards additionally fail-**closed** for valid input with unrecognized values, +as a defense-in-depth measure against privilege escalation: + +| Guard | Fail-closed condition | Rationale | +|-------|----------------------|-----------| +| `skill_command_guard.py` | Unexpected runtime error (not JSON parse) | Unknown failure mode = deny rather than risk executing an unvalidated command | +| `open_kitchen_guard.py` | Unrecognized `AUTOSKILLIT_SESSION_TYPE` | Unknown session type should not gain kitchen access | +| `skill_orchestration_guard.py` | Unrecognized `AUTOSKILLIT_SESSION_TYPE` | Unknown session type should not call orchestration tools (`run_skill`, `run_cmd`, `run_python`) | + +**Design principle:** Garbage-in (malformed hook input) = fail-open. Unknown-tier +(valid input, unrecognized value) = fail-closed. + +All remaining guards (`fleet_dispatch_guard.py`, `quota_guard.py`, +`mcp_health_advisor.py`, `branch_protection_guard.py`, etc.) fail-open in every +failure scenario — malformed input, unrecognized session types, runtime errors, +and missing data. + ## Drift detection `cli/_doctor.py:_check_hook_registry_drift` calls `generate_hooks_json()` and @@ -110,7 +195,7 @@ the global Python. ```yaml # .autoskillit/config.yaml safety: - protected_branches: ["main", "integration", "stable"] + protected_branches: ["main", "develop", "stable"] require_dry_walkthrough: true test_gate_on_merge: true reset_guard_marker: ".autoskillit-workspace" @@ -118,7 +203,7 @@ safety: quota_guard: enabled: true short_window_threshold: 85.0 - long_window_threshold: 98.0 + long_window_threshold: 95.0 long_window_patterns: ["weekly", "sonnet", "opus"] buffer_seconds: 60 ``` diff --git a/docs/skills/catalog.md b/docs/skills/catalog.md index 6cd7240c8f..5f7f93bc6b 100644 --- a/docs/skills/catalog.md +++ b/docs/skills/catalog.md @@ -1,7 +1,7 @@ # Skill catalog -The complete list of bundled skills (109 total: 3 in `src/autoskillit/skills/`, -106 in `src/autoskillit/skills_extended/`). Filesystem walk this directory if +The complete list of bundled skills (133 total: 3 in `src/autoskillit/skills/`, +130 in `src/autoskillit/skills_extended/`). Filesystem walk this directory if you need an exhaustive listing; this catalog groups by purpose. ## Tier 1 — free range (3) @@ -19,31 +19,38 @@ Located under `src/autoskillit/skills_extended/`. Grouped by purpose: ### Plan and implementation `investigate`, `make-plan`, `dry-walkthrough`, `review-approach`, `implement-worktree`, `rectify`, `make-groups`, `mermaid`, `make-arch-diag`, -`make-experiment-diag` +`make-experiment-diag`, `build-execution-map`, `plan-visualization` ### Audit suite `audit-arch`, `audit-cohesion`, `audit-tests`, `audit-defense-standards`, -`audit-bugs`, `audit-friction`, `validate-audit` +`audit-bugs`, `audit-friction`, `validate-audit`, `validate-test-audit`, `validate-review-decisions`, +`audit-claims`, `audit-docs`, `audit-feature-gates`, `audit-review-decisions` ### Requirements and planning `make-req`, `elaborate-phase`, `write-recipe`, `migrate-recipes`, -`setup-project`, `sprint-planner`, `design-guards`, `triage-issues`, +`setup-project`, `design-guards`, `triage-issues`, `collapse-issues`, `issue-splitter`, `enrich-issues`, `prepare-issue`, -`process-issues` +`process-issues`, `make-campaign` ### Experiment family `scope`, `plan-experiment`, `implement-experiment`, `run-experiment`, `generate-report`, `troubleshoot-experiment` +### Research and review +`review-design`, `stage-data`, `setup-environment`, `bundle-local-report`, `reload-session` + ## Tier 3 — pipeline / automation Also under `src/autoskillit/skills_extended/`. Used by recipes for unattended runs: -`open-pr`, `open-integration-pr`, `merge-pr`, `analyze-prs`, `review-pr`, +`prepare-pr`, `compose-pr`, `open-integration-pr`, `merge-pr`, `analyze-prs`, `review-pr`, `resolve-review`, `implement-worktree-no-merge`, `resolve-failures`, `retry-worktree`, `resolve-merge-conflicts`, `audit-impl`, `smoke-task`, -`report-bug`, `pipeline-summary`, `diagnose-ci`, `verify-diag` +`report-bug`, `pipeline-summary`, `diagnose-ci`, `verify-diag`, +`resolve-claims-review`, `resolve-design-review`, `resolve-research-review`, +`compose-research-pr`, `prepare-research-pr`, `review-research-pr`, +`promote-to-main` ## arch-lens family (13) @@ -102,7 +109,7 @@ figure set: | 1 | `vis-lens-always-on` | Composite | Is everything correct by default? | P0 | | 2 | `vis-lens-antipattern` | Diagnostic | What visualization antipatterns are present? | P0 | | 3 | `vis-lens-chart-select` | Typological | What chart type fits this data shape? | P0 | -| 4 | `vis-lens-domain-norms` | Normative | Does the figure follow domain conventions? | P0 | +| 4 | `vis-lens-methodology-norms` | Normative | Does the figure follow methodology-tradition conventions? | P0 | | 5 | `vis-lens-uncertainty` | Probabilistic | Is uncertainty properly communicated? | P0 | | 6 | `vis-lens-color-access` | Chromatic | Is the color encoding accessible and perceptually uniform? | P1 | | 7 | `vis-lens-figure-table` | Decisional | Should this result be a figure or a table? | P1 | @@ -112,6 +119,17 @@ figure set: | 11 | `vis-lens-story-arc` | Narrative | Do the figures tell a coherent story across the report? | P2 | | 12 | `vis-lens-reproducibility` | Replicative | Can the figures be reproduced from the data and code? | P2 | +## Planner family (14) + +14 progressive-decomposition sub-skills under `skills_extended/planner-*/`. Invoked +internally by the `planner` recipe to break a roadmap into GitHub milestones and issues: + +`planner-analyze`, `planner-assess-review-approach`, `planner-consolidate-wps`, +`planner-elaborate-assignments`, `planner-elaborate-phase`, +`planner-elaborate-wps`, `planner-extract-domain`, `planner-generate-phases`, +`planner-reconcile-deps`, `planner-refine`, `planner-refine-assignments`, +`planner-refine-phases`, `planner-refine-wps`, `planner-validate-task-alignment` + ## Rectify doctrine Several Tier 2 skills (`rectify`, `audit-bugs`, `design-guards`, @@ -121,8 +139,8 @@ symptom, and the audit suite is updated so the same class of bug cannot recur. Commit messages prefix with `Rectify:` for traceability; the count of `Rectify:` commits is reported in `docs/developer/contributing.md`. -## Total: 109 +## Total: 132 -3 (Tier 1) + 106 (`skills_extended/`) = 109 bundled skills. The total is +3 (Tier 1) + 129 (`skills_extended/`) = 132 bundled skills. The total is verified by `tests/docs/test_doc_counts.py` against a filesystem walk so any addition or removal is caught immediately. diff --git a/docs/skills/subsets.md b/docs/skills/subsets.md index 1bacc4a613..1746da85ad 100644 --- a/docs/skills/subsets.md +++ b/docs/skills/subsets.md @@ -20,6 +20,10 @@ manually reclassifying each of its tools and skills one by one. A project withou | `telemetry` | `get_token_summary`, `get_timing_summary`, `write_telemetry_files`, `get_quota_events` | — | | `arch-lens` | — | All 13 `arch-lens-*` skills, `make-arch-diag`, `verify-diag` | | `audit` | — | `audit-arch`, `audit-cohesion`, `audit-tests`, `audit-defense-standards`, `audit-bugs`, `audit-friction`, `audit-impl` | +| `exp-lens` | — | All 18 `exp-lens-*` skills | +| `kitchen-core` | `run_cmd`, `run_python`, `run_skill`, `test_check`, `reset_test_dir`, `reset_workspace`, `classify_fix`, `list_recipes`, `load_recipe`, `validate_recipe`, `migrate_recipe`, `kitchen_status`, `read_db`, `get_pipeline_report`, `dispatch_food_truck`, `merge_worktree`, `get_token_summary`, `get_timing_summary`, `write_telemetry_files`, `get_quota_events`, `analyze_tool_sequences` | — | +| `research` | — | `compose-research-pr`, `prepare-research-pr`, `review-research-pr`, `resolve-research-review`, `resolve-claims-review`, `resolve-design-review` | +| `vis-lens` | — | All 12 `vis-lens-*` skills, `plan-visualization` | ## Disabling a Subset diff --git a/docs/skills/visibility.md b/docs/skills/visibility.md index 85bc9aed83..0176f34097 100644 --- a/docs/skills/visibility.md +++ b/docs/skills/visibility.md @@ -2,7 +2,7 @@ ## Overview -AutoSkillit's 109 bundled skills are organized into three tiers that control when and where +AutoSkillit's 133 bundled skills are organized into three tiers that control when and where they appear as slash commands. The tier system is orthogonal to subset categories — you can disable a subset across all tiers simultaneously, or reclassify individual skills between tiers. See [Subset Categories](subsets.md) for subset configuration. @@ -22,14 +22,21 @@ tiers. See [Subset Categories](subsets.md) for subset configuration. ### Tier 2 — Cook (Interactive Skills) - **Location**: `src/autoskillit/skills_extended/` (NOT plugin-scanned) -- **Default members** (41 total): +- **Default members** (102 total): `investigate`, `make-plan`, `implement-worktree`, `rectify`, `dry-walkthrough`, `make-groups`, `review-approach`, `mermaid`, `make-arch-diag`, - all 13 `arch-lens-*` skills, `audit-arch`, `audit-cohesion`, `audit-tests`, - `audit-defense-standards`, `audit-bugs`, `audit-friction`, `make-req`, - `elaborate-phase`, `write-recipe`, `migrate-recipes`, `setup-project`, - `sprint-planner`, `design-guards`, `triage-issues`, `collapse-issues`, - `issue-splitter`, `enrich-issues`, `prepare-issue`, `process-issues` + `make-experiment-diag`, `plan-visualization`, + all 13 `arch-lens-*` skills, all 18 `exp-lens-*` skills, all 12 `vis-lens-*` skills, + all 14 `planner-*` skills, + `audit-arch`, `audit-cohesion`, `audit-tests`, + `audit-defense-standards`, `audit-bugs`, `audit-friction`, `validate-audit`, + `audit-docs`, `audit-feature-gates`, `audit-review-decisions`, + `make-req`, `elaborate-phase`, `write-recipe`, `migrate-recipes`, `setup-project`, + `design-guards`, `triage-issues`, `collapse-issues`, + `issue-splitter`, `enrich-issues`, `prepare-issue`, `process-issues`, `make-campaign`, + `scope`, `plan-experiment`, `implement-experiment`, `run-experiment`, + `generate-report`, `validate-test-audit`, `validate-review-decisions`, + `stage-data`, `setup-environment`, `bundle-local-report`, `reload-session` - **Visible in**: cook and headless sessions - **Mechanism**: copied to an ephemeral session directory (cook) or exposed via `--add-dir` (headless sessions launched by `run_skill`) @@ -37,11 +44,15 @@ tiers. See [Subset Categories](subsets.md) for subset configuration. ### Tier 3 — Pipeline-Only (Automation Skills) - **Location**: `src/autoskillit/skills_extended/` (same directory as Tier 2) -- **Default members** (16 total): - `open-pr`, `open-integration-pr`, `merge-pr`, `analyze-prs`, +- **Default members** (28 total): + `prepare-pr`, `compose-pr`, `open-integration-pr`, `merge-pr`, `analyze-prs`, `review-pr`, `resolve-review`, `implement-worktree-no-merge`, `resolve-failures`, `retry-worktree`, `resolve-merge-conflicts`, `audit-impl`, `smoke-task`, - `report-bug`, `pipeline-summary`, `diagnose-ci`, `verify-diag` + `report-bug`, `pipeline-summary`, `diagnose-ci`, `verify-diag`, + `compose-research-pr`, `prepare-research-pr`, `resolve-claims-review`, + `resolve-design-review`, `resolve-research-review`, `review-research-pr`, + `audit-claims`, `build-execution-map`, `promote-to-main`, + `review-design`, `troubleshoot-experiment` - **Visible in**: cook and headless sessions - **Distinction from Tier 2**: semantic only — both tiers live in `skills_extended/` and are available in the same session modes. The tier distinction lets users reclassify @@ -84,7 +95,7 @@ and `/autoskillit:close-kitchen`. Skills in `skills_extended/` are never seen. Order is similar to cook: AutoSkillit launches Claude Code with access to all tiers. The key difference is the orchestrator (`sous-chef` skill) is injected and the kitchen -is pre-opened so all 44 MCP tools are available from the start. +is pre-opened so all 52 MCP tools are available from the start. ### Headless session (launched by `run_skill`) diff --git a/docs/update-checks.md b/docs/update-checks.md index e444791563..52f3ace57c 100644 --- a/docs/update-checks.md +++ b/docs/update-checks.md @@ -20,7 +20,7 @@ Dismissal windows vary by install type to balance convenience and safety: | Install | Window | |---------|--------| | stable / main / release-tag | 7 days | -| integration / local-editable | 12 hours | +| develop / local-editable | 12 hours | The window is determined at check time from the current `direct_url.json` — not from what was stored when you dismissed. @@ -45,8 +45,9 @@ For unknown install types (e.g. installed from PyPI without a VCS reference), ## Escape hatches -Set either env var to silence all update checks for a single invocation: +Set any of these env vars to silence all update checks for a single invocation: + AUTOSKILLIT_SKIP_UPDATE_CHECK=1 autoskillit AUTOSKILLIT_SKIP_STALE_CHECK=1 autoskillit AUTOSKILLIT_SKIP_SOURCE_DRIFT_CHECK=1 autoskillit diff --git a/docs/version-pipeline.md b/docs/version-pipeline.md new file mode 100644 index 0000000000..f077a53a9b --- /dev/null +++ b/docs/version-pipeline.md @@ -0,0 +1,51 @@ +# Versioning Pipeline + +## Canonical Source + +`pyproject.toml` `[project].version` is the single source of truth for the package version. + +## Version Propagation + +`scripts/sync_versions.py` reads the canonical version from `pyproject.toml` via `tomllib`, then +atomically updates `src/autoskillit/.claude-plugin/plugin.json` `"version"` field. + +Pre-commit enforcement: `sync_versions.py --check` (exits 1 if any artifact is out of sync, without modifying files). + +## CI Workflows + +### `patch-bump-develop.yml` + +- **Trigger:** any PR merged into `develop` +- **Action:** `MAJOR.MINOR.(PATCH+1)` on `develop` +- **Calls `sync_versions.py`:** yes +- **Staged files:** `pyproject.toml`, `src/autoskillit/.claude-plugin/plugin.json`, `uv.lock` +- **Concurrency:** serialized (`cancel-in-progress: false`) + +### `version-bump.yml` + +- **Trigger:** PR merged into `main` from `develop` (promotion) +- **Action:** + - `main` → `MAJOR.(MINOR+1).0` + - `develop` → `MAJOR.(MINOR+1).1` (forward-bumped to stay ahead of main) +- **Calls `sync_versions.py`:** yes (on both `main` and `develop` in sequence) +- **Guard:** rejects if `new_develop <= old_develop` (downgrade protection) + +### `release.yml` + +- **Trigger:** PR merged into `stable` +- **Action:** `MAJOR.(MINOR+1).0` on `stable`, annotated git tag `vX.Y.Z`, GitHub Release +- **Calls `sync_versions.py`:** yes — `plugin.json` is synced +- **Staged files:** `pyproject.toml`, `src/autoskillit/.claude-plugin/plugin.json`, `uv.lock` + +## Runtime Health + +`version.py:version_info(plugin_dir)` — LRU-cached, reads: + +- `importlib.metadata.version("autoskillit")` → `package_version` +- `plugin.json` → `plugin_json_version` + +Returns `match` (`package_version == plugin_json_version`). + +## Invariant for PRs + +Never include manual version bump commits in PRs. CI handles all bumps automatically on merge. Including a manual bump causes a conflict with the CI-committed bump and must be reverted. diff --git a/pyproject.toml b/pyproject.toml index 1bd841eeae..baa7c1961f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "hatchling.build" [project] name = "autoskillit" -version = "0.9.0" +version = "0.9.569" description = "MCP server for orchestrating automated skill-driven workflows with Claude Code" readme = "README.md" license = {text = "MIT"} @@ -27,16 +27,21 @@ dependencies = [ "anyio>=4.6.0", "dynaconf>=3.2.13,<4.0", "fastmcp>=3.2.0,<4.0", - "httpx>=0.28,<0.29", + "httpx>=0.28,<1.0", "igraph>=1.0,<2.0", + "lazy-loader>=0.4", "markdown-it-py>=3.0", "packaging>=23.0", + "pathspec>=1.0,<2.0", + "orjson>=3.11", "psutil>=7.2.0", "pygments>=2.20.0", "pyjwt>=2.12.0", "pyyaml>=6.0", "cyclopts>=4.0", + "regex>=2026.1", "structlog>=25.0,<26.0", + "uvloop>=0.21.0; sys_platform != 'win32'", ] [project.optional-dependencies] @@ -45,11 +50,13 @@ dev = [ "pytest-asyncio>=1.0.0", # asyncio_default_fixture_loop_scope requires >=1.0.0 "pytest-httpx>=0.36", "pytest-xdist>=3.5.0", + "pytest-cov>=7.0", "pytest-timeout>=2.3", "ruff>=0.15.0", "import-linter>=2.1", "packaging>=25.0", - "api-simulator", + "pytest-testmon>=2.2.0", + "api-simulator; sys_platform == 'linux'", ] [project.urls] @@ -90,8 +97,18 @@ timeout_method = "signal" markers = [ "smoke: end-to-end smoke tests requiring Claude API access (deselect with: -m 'not smoke')", "integration: integration tests that spawn real subprocesses (deselect with: -m 'not integration')", + "canary: development-time filter verification tests (deselect with: -m 'not canary')", + "layer(name): source-layer marker for test file classification (core, config, pipeline, execution, workspace, recipe, migration, fleet, server, cli)", + "small: pure unit test — no persistent I/O, no network, no subprocess (RAM-backed tmpfs via tmp_path IS allowed)", + "medium: filesystem and subprocess allowed — no network, no external services", + "large: full integration — everything allowed (default for unannotated tests)", + "feature(name): marks tests requiring a specific feature gate to be enabled", ] +[tool.coverage.run] +parallel = true +source = ["src/autoskillit"] + [tool.ruff] target-version = "py311" line-length = 99 @@ -101,6 +118,15 @@ select = ["E", "F", "I", "UP", "TID251"] [tool.ruff.lint.per-file-ignores] "tests/execution/test_session_classification_e2e.py" = ["E402"] +"tests/execution/test_process_channel_b.py" = ["E501"] +"tests/test_test_filter_cascade.py" = ["E501"] +"tests/execution/test_quota_io.py" = ["E501"] +"tests/execution/test_merge_queue_ejection.py" = ["E501"] +"tests/execution/test_quota_binding.py" = ["E501"] +"tests/arch/test_layer_enforcement.py" = ["E501"] +"tests/execution/test_quota_sleep.py" = ["E501"] +"tests/execution/test_merge_queue_classifier.py" = ["E501"] +"tests/skills/test_vis_lens_methodology_norms.py" = ["E501"] [tool.ruff.lint.flake8-tidy-imports.banned-api] "logging.getLogger" = {msg = "Use get_logger() from autoskillit._logging instead."} @@ -113,11 +139,15 @@ root_packages = ["autoskillit"] # inside TYPE_CHECKING blocks to avoid circular imports at runtime. exclude_type_checking_imports = true +# Contract IDs below use IL-NNN (three-digit serial numbers, e.g. IL-001). +# These are NOT import layer levels (IL-0 through IL-3). +# IL-002 = config's contract; IL-2 = the recipe/migration/fleet import layer. + # IL-001 | Lens: module-dependency | DS-001 -# Rationale: L0 (core) is the foundation layer; it must have zero upward imports to +# Rationale: IL-0 (core) is the foundation layer; it must have zero upward imports to # remain independently testable and importable outside the autoskillit package. [[tool.importlinter.contracts]] -name = "L0 core has no autoskillit upward imports" +name = "IL-0 core has no autoskillit upward imports" type = "forbidden" source_modules = ["autoskillit.core"] forbidden_modules = [ @@ -132,11 +162,11 @@ forbidden_modules = [ ] # IL-002 | Lens: module-dependency | DS-001 -# Rationale: L1 config resolves settings from layered sources; allowing it to import +# Rationale: IL-1 config resolves settings from layered sources; allowing it to import # pipeline/execution/recipe would create hidden initialization-time coupling that # breaks independent layered testing. [[tool.importlinter.contracts]] -name = "L1 config imports only L0" +name = "IL-1 config imports only IL-0" type = "forbidden" source_modules = ["autoskillit.config"] forbidden_modules = [ @@ -150,18 +180,18 @@ forbidden_modules = [ ] # IL-003 | Lens: module-dependency | DS-001 -# Rationale: pipeline/ holds cross-session gate and audit state; importing from L2 -# (recipe, migration) would create initialization-order coupling and prevent L1 -# packages from being tested in isolation without a full L2 import graph. +# Rationale: pipeline/ holds cross-session gate and audit state; importing from IL-2 +# (recipe, migration) would create initialization-order coupling and prevent IL-1 +# packages from being tested in isolation without a full IL-2 import graph. [[tool.importlinter.contracts]] -name = "L1 pipeline does not import L2 or L3" +name = "IL-1 pipeline does not import IL-2 or IL-3" type = "forbidden" source_modules = ["autoskillit.pipeline"] # EXCEPTION: autoskillit.config is intentionally omitted from the # forbidden_modules list. pipeline.context.ToolContext owns # AutomationConfig as the DI wiring point — see the docstrings on # pipeline/context.py and pipeline/__init__.py. The coupling is safe -# because config/ depends only on L0 (enforced by IL-002), so no +# because config/ depends only on IL-0 (enforced by IL-002), so no # cycle is possible. This exception is asserted by # tests/arch/test_import_linter_contracts.py # ::test_il003_pipeline_config_exception_documented. @@ -176,10 +206,10 @@ forbidden_modules = [ # IL-004 | Lens: module-dependency | DS-001 # Rationale: execution/ manages subprocess lifecycle; it must not import config, -# pipeline, workspace, recipe, migration, server, or cli to remain a leaf L1 service +# pipeline, workspace, recipe, migration, server, or cli to remain a leaf IL-1 service # that can be tested without spinning up the full autoskillit stack. [[tool.importlinter.contracts]] -name = "L1 execution imports only L0" +name = "IL-1 execution imports only IL-0" type = "forbidden" source_modules = ["autoskillit.execution"] forbidden_modules = [ @@ -197,7 +227,7 @@ forbidden_modules = [ # or pipeline would introduce a circular risk since config and pipeline both may # depend on workspace for path resolution. [[tool.importlinter.contracts]] -name = "L1 workspace imports only L0" +name = "IL-1 workspace imports only IL-0" type = "forbidden" source_modules = ["autoskillit.workspace"] forbidden_modules = [ @@ -211,11 +241,11 @@ forbidden_modules = [ ] # IL-006 | Lens: module-dependency | DS-001 -# Rationale: recipe/ is an L2 service that orchestrates validation over workspace +# Rationale: recipe/ is an IL-2 service that orchestrates validation over workspace # artifacts; importing config, pipeline, execution, migration, server, or cli # would conflate recipe semantics with runtime session management. [[tool.importlinter.contracts]] -name = "L2 recipe imports only L0 and workspace" +name = "IL-2 recipe imports only IL-0 and workspace" type = "forbidden" source_modules = ["autoskillit.recipe"] forbidden_modules = [ @@ -231,7 +261,7 @@ forbidden_modules = [ # Rationale: migration/ must not import config or pipeline to stay independently # runnable as a schema-upgrade step without a live pipeline context. [[tool.importlinter.contracts]] -name = "L2 migration does not import L3 or config" +name = "IL-2 migration does not import IL-3 or config" type = "forbidden" source_modules = ["autoskillit.migration"] forbidden_modules = [ @@ -241,5 +271,48 @@ forbidden_modules = [ "autoskillit.cli", ] +# IL-008 | Lens: module-dependency | DS-001 +# Rationale: The four IL-1 peers (config, pipeline, execution, workspace) form an +# independent service tier. Lateral imports between them create hidden coupling +# that prevents each package from being tested and deployed in isolation. The +# independence contract makes this boundary machine-verifiable and surfaces any +# future regression immediately via lint-imports. +[[tool.importlinter.contracts]] +name = "IL-1 siblings are independent of each other" +type = "independence" +modules = [ + "autoskillit.config", + "autoskillit.pipeline", + "autoskillit.execution", + "autoskillit.workspace", +] +# EXCEPTION: autoskillit.pipeline.context may import autoskillit.config. +# pipeline.context.ToolContext owns AutomationConfig as the DI wiring point. +# config/ depends only on IL-0 (IL-002), so no cycle is introduced. +# This exception mirrors the one already documented in IL-003. +ignore_imports = [ + "autoskillit.pipeline.context -> autoskillit.config", +] + +# IL-009 | Lens: module-dependency | DS-001 +# Rationale: fleet is IL-2 — imports IL-0 (core) only; workspace allowed for path utilities +[[tool.importlinter.contracts]] +name = "IL-2 fleet does not import IL-3 or most IL-1 peers" +type = "forbidden" +source_modules = ["autoskillit.fleet"] +forbidden_modules = [ + # EXCEPTION: autoskillit.workspace is intentionally absent from this list. + # fleet may import workspace solely for path-lookup utilities (e.g., worktree paths). + # This is a deliberate, narrow allowance — do not expand workspace use in fleet + # beyond path utilities without updating this contract and the rationale comment above. + "autoskillit.config", + "autoskillit.pipeline", + "autoskillit.execution", + "autoskillit.migration", + "autoskillit.recipe", + "autoskillit.server", + "autoskillit.cli", +] + [tool.uv.sources] api-simulator = { git = "https://github.com/TalonT-Org/api-simulator", rev = "8f711f958d4d93a8e53a3c3b51279fb420870f3a", subdirectory = "crates/api-simulator-py" } diff --git a/scripts/benchmark-testmon.py b/scripts/benchmark-testmon.py new file mode 100755 index 0000000000..ec74b1dbea --- /dev/null +++ b/scripts/benchmark-testmon.py @@ -0,0 +1,246 @@ +#!/usr/bin/env python3 +"""Compare pytest-testmon selection against cascade filter selection. + +Modes: + --mode=compare Set comparison of testmon vs cascade selection (default) + --mode=overhead Measure collection-time overhead with/without testmon + --mode=db-stats Report .testmondata DB statistics + +Exit codes: + 0 Always (evaluation tool, not a gate) +""" + +from __future__ import annotations + +import argparse +import importlib.util +import json +import sqlite3 +import subprocess +import sys +import time +from pathlib import Path +from typing import Any + +PROJECT_ROOT = Path(__file__).resolve().parent.parent + + +def compute_selection_metrics( + testmon_selected: set[str], + cascade_selected: set[str], +) -> dict[str, Any]: + overlap = testmon_selected & cascade_selected + union = testmon_selected | cascade_selected + return { + "testmon_only": testmon_selected - cascade_selected, + "cascade_only": cascade_selected - testmon_selected, + "overlap": overlap, + "testmon_count": len(testmon_selected), + "cascade_count": len(cascade_selected), + "overlap_count": len(overlap), + "jaccard_similarity": len(overlap) / len(union) if union else 1.0, + } + + +def _pytest_cmd() -> list[str]: + venv_python = PROJECT_ROOT / ".venv" / "bin" / "python" + if venv_python.exists(): + return [str(venv_python), "-m", "pytest"] + return ["pytest"] + + +def _collect_testmon_selection() -> set[str]: + cmd = [*_pytest_cmd(), "--collect-only", "-q", "--testmon", "-o", "addopts="] + result = subprocess.run(cmd, capture_output=True, text=True, cwd=str(PROJECT_ROOT)) + selected: set[str] = set() + for line in result.stdout.splitlines(): + line = line.strip() + if line and "::" in line and not line.startswith(("=", "-", "no tests")): + test_file = line.split("::")[0] + selected.add(test_file) + return selected + + +def _load_test_filter_module() -> Any: + test_filter_path = PROJECT_ROOT / "tests" / "_test_filter.py" + spec = importlib.util.spec_from_file_location("_test_filter", test_filter_path) + assert spec is not None and spec.loader is not None + mod = importlib.util.module_from_spec(spec) + spec.loader.exec_module(mod) + return mod + + +def _collect_cascade_selection(changed_files: list[str]) -> set[str]: + mod = _load_test_filter_module() + manifest = mod.load_manifest(PROJECT_ROOT / ".autoskillit" / "test-filter-manifest.yaml") + scope = mod.build_test_scope( + changed=[Path(f) for f in changed_files], + repo_root=PROJECT_ROOT, + manifest=manifest, + mode=mod.FilterMode.CONSERVATIVE, + ) + if scope is None: + tests_dir = PROJECT_ROOT / "tests" + return {str(p.relative_to(tests_dir)) for p in tests_dir.rglob("test_*.py")} + return {str(p.relative_to(PROJECT_ROOT / "tests")) for p in scope} + + +def _run_compare(changed_files: list[str]) -> int: + if not changed_files: + print("ERROR: --changed-files required for compare mode") + return 0 + + print(f"Changed files: {changed_files}") + print() + + testmon_sel = _collect_testmon_selection() + cascade_sel = _collect_cascade_selection(changed_files) + + metrics = compute_selection_metrics(testmon_sel, cascade_sel) + + print("=== Selection Comparison ===") + print(f"Testmon selected: {metrics['testmon_count']} test files") + print(f"Cascade selected: {metrics['cascade_count']} test files") + print(f"Overlap: {metrics['overlap_count']} test files") + print(f"Jaccard similarity: {metrics['jaccard_similarity']:.3f}") + print() + + if metrics["testmon_only"]: + print("Testmon-only (cascade misses):") + for f in sorted(metrics["testmon_only"]): + print(f" {f}") + print() + + if metrics["cascade_only"]: + print("Cascade-only (testmon skips):") + for f in sorted(metrics["cascade_only"]): + print(f" {f}") + print() + + report = { + "mode": "compare", + "changed_files": changed_files, + "testmon_count": metrics["testmon_count"], + "cascade_count": metrics["cascade_count"], + "overlap_count": metrics["overlap_count"], + "jaccard_similarity": metrics["jaccard_similarity"], + "testmon_only": sorted(metrics["testmon_only"]), + "cascade_only": sorted(metrics["cascade_only"]), + "overlap": sorted(metrics["overlap"]), + } + _write_report(report) + return 0 + + +def _run_overhead() -> int: + base_cmd = [*_pytest_cmd(), "tests/", "-q", "--co", "-o", "addopts="] + testmon_cmd = [*base_cmd, "--testmon"] + + print("Measuring collection overhead...") + print() + + t0 = time.monotonic() + subprocess.run(base_cmd, capture_output=True, cwd=str(PROJECT_ROOT)) + base_time = time.monotonic() - t0 + + t0 = time.monotonic() + subprocess.run(testmon_cmd, capture_output=True, cwd=str(PROJECT_ROOT)) + testmon_time = time.monotonic() - t0 + + overhead_pct = ((testmon_time - base_time) / base_time * 100) if base_time > 0 else 0 + + print(f"Base collection: {base_time:.2f}s") + print(f"Testmon collection: {testmon_time:.2f}s") + print(f"Overhead: {overhead_pct:+.1f}%") + + report = { + "mode": "overhead", + "base_time_s": round(base_time, 3), + "testmon_time_s": round(testmon_time, 3), + "overhead_pct": round(overhead_pct, 1), + } + _write_report(report) + return 0 + + +def _run_db_stats() -> int: + db_path = PROJECT_ROOT / ".testmondata" + if not db_path.exists(): + print("ERROR: .testmondata not found. Run 'task testmon-build' first.") + return 0 + + size_bytes = db_path.stat().st_size + size_mb = size_bytes / (1024 * 1024) + + conn = sqlite3.connect(str(db_path)) + try: + cursor = conn.cursor() + cursor.execute("SELECT COUNT(*) FROM node") + node_count = cursor.fetchone()[0] + + cursor.execute("SELECT COUNT(DISTINCT file_name) FROM node_file") + source_file_count = cursor.fetchone()[0] + + cursor.execute( + "SELECT AVG(dep_count) FROM " + "(SELECT node_id, COUNT(*) as dep_count FROM node_file GROUP BY node_id)" + ) + avg_deps = cursor.fetchone()[0] or 0 + finally: + conn.close() + + print("=== .testmondata Statistics ===") + print(f"DB size: {size_mb:.1f} MB ({size_bytes:,} bytes)") + print(f"Test nodes: {node_count:,}") + print(f"Source files: {source_file_count:,}") + print(f"Avg deps per test: {avg_deps:.1f}") + + report = { + "mode": "db-stats", + "db_size_bytes": size_bytes, + "db_size_mb": round(size_mb, 1), + "test_node_count": node_count, + "source_file_count": source_file_count, + "avg_deps_per_test": round(avg_deps, 1), + } + _write_report(report) + return 0 + + +def _write_report(data: dict[str, Any]) -> None: + report_dir = PROJECT_ROOT / ".autoskillit" / "temp" + report_dir.mkdir(parents=True, exist_ok=True) + timestamp = time.strftime("%Y%m%d-%H%M%S") + report_path = report_dir / f"testmon-eval-{timestamp}.json" + report_path.write_text(json.dumps(data, indent=2, default=str) + "\n") + print(f"\nReport written: {report_path}") + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--mode", + choices=["compare", "overhead", "db-stats"], + default="compare", + ) + parser.add_argument( + "--changed-files", + help="Comma-separated list of changed file paths", + ) + args = parser.parse_args() + + changed_files: list[str] = [] + if args.changed_files: + changed_files = [f.strip() for f in args.changed_files.split(",") if f.strip()] + + if args.mode == "compare": + return _run_compare(changed_files) + elif args.mode == "overhead": + return _run_overhead() + elif args.mode == "db-stats": + return _run_db_stats() + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/check_doc_counts.py b/scripts/check_doc_counts.py index f3a1a15ef8..1d38a6d94c 100755 --- a/scripts/check_doc_counts.py +++ b/scripts/check_doc_counts.py @@ -18,25 +18,17 @@ SKILLS_DIR = PROJECT_ROOT / "src" / "autoskillit" / "skills" SKILLS_EXTENDED_DIR = PROJECT_ROOT / "src" / "autoskillit" / "skills_extended" RECIPES_DIR = PROJECT_ROOT / "src" / "autoskillit" / "recipes" -TYPES_FILE = PROJECT_ROOT / "src" / "autoskillit" / "core" / "_type_constants.py" +TYPES_FILE = PROJECT_ROOT / "src" / "autoskillit" / "core" / "types" / "_type_constants.py" def count_skills() -> int: - """Count public skills in skills/ and skills_extended/. - - Only counts skills whose SKILL.md begins with YAML frontmatter (``---``), - matching the behaviour of DefaultSkillResolver().list_all() which excludes - internal bootstrap documents such as sous-chef. - """ + """Count all bundled skill directories in skills/ and skills_extended/.""" count = 0 for skills_dir in (SKILLS_DIR, SKILLS_EXTENDED_DIR): if skills_dir.is_dir(): for d in skills_dir.iterdir(): - skill_md = d / "SKILL.md" - if d.is_dir() and skill_md.exists(): - text = skill_md.read_text(encoding="utf-8") - if text.startswith("---\n"): - count += 1 + if d.is_dir(): + count += 1 return count diff --git a/scripts/check_sub_claude_md.py b/scripts/check_sub_claude_md.py new file mode 100644 index 0000000000..5630300cfe --- /dev/null +++ b/scripts/check_sub_claude_md.py @@ -0,0 +1,110 @@ +#!/usr/bin/env python3 +"""Validate sub-CLAUDE.md file tables cover all .py files in their directories. + +Pre-commit hook (validate-only). Exits 1 with structured messages when a .py +file exists in a directory whose CLAUDE.md does not mention it. +""" + +from __future__ import annotations + +import sys +from pathlib import Path + +PROJECT_ROOT = Path(__file__).resolve().parent.parent +SRC_ROOT = PROJECT_ROOT / "src" / "autoskillit" +TESTS_ROOT = PROJECT_ROOT / "tests" + +SRC_EXPECTED = [ + "core/types/CLAUDE.md", + "core/runtime/CLAUDE.md", + "execution/headless/CLAUDE.md", + "execution/process/CLAUDE.md", + "execution/session/CLAUDE.md", + "execution/merge_queue/CLAUDE.md", + "recipe/rules/CLAUDE.md", + "server/tools/CLAUDE.md", + "cli/doctor/CLAUDE.md", + "cli/fleet/CLAUDE.md", + "cli/session/CLAUDE.md", + "cli/ui/CLAUDE.md", + "cli/update/CLAUDE.md", + "hooks/guards/CLAUDE.md", + "hooks/formatters/CLAUDE.md", + "CLAUDE.md", + "core/CLAUDE.md", + "config/CLAUDE.md", + "pipeline/CLAUDE.md", + "execution/CLAUDE.md", + "workspace/CLAUDE.md", + "planner/CLAUDE.md", + "recipe/CLAUDE.md", + "migration/CLAUDE.md", + "fleet/CLAUDE.md", + "cli/CLAUDE.md", + "hooks/CLAUDE.md", +] + +TESTS_EXPECTED = [ + "arch/CLAUDE.md", + "assets/CLAUDE.md", + "cli/CLAUDE.md", + "config/CLAUDE.md", + "contracts/CLAUDE.md", + "core/CLAUDE.md", + "docs/CLAUDE.md", + "execution/CLAUDE.md", + "fleet/CLAUDE.md", + "hooks/CLAUDE.md", + "infra/CLAUDE.md", + "migration/CLAUDE.md", + "pipeline/CLAUDE.md", + "planner/CLAUDE.md", + "recipe/CLAUDE.md", + "server/CLAUDE.md", + "skills/CLAUDE.md", + "skills_extended/CLAUDE.md", + "workspace/CLAUDE.md", +] + + +def check_coverage(root: Path, expected: list[str]) -> list[str]: + """Check that each CLAUDE.md in expected mentions all .py files in its directory. + + Returns a list of failure messages (empty if all coverage is complete). + """ + failures: list[str] = [] + for rel_path in expected: + claude_md = root / rel_path + if not claude_md.exists(): + failures.append(f"{rel_path}: CLAUDE.md not found") + continue + content = claude_md.read_text(encoding="utf-8") + directory = claude_md.parent + for py_file in directory.glob("*.py"): + if py_file.name == "__init__.py": + if "`__init__.py`" not in content: + failures.append(f"{rel_path}: missing `__init__.py` in file table") + else: + if f"`{py_file.name}`" not in content: + failures.append(f"{rel_path}: missing {py_file.name}") + return failures + + +def main() -> int: + src_failures = check_coverage(SRC_ROOT, SRC_EXPECTED) + tests_failures = check_coverage(TESTS_ROOT, TESTS_EXPECTED) + all_failures = src_failures + tests_failures + if all_failures: + print("sub-CLAUDE.md file table gaps found:\n") + for f in all_failures: + print(f" {f}") + print(f"\nTotal: {len(all_failures)} gap(s)") + print("\nTo fix: add the missing file(s) to the CLAUDE.md file table in the") + print("directory where the .py file(s) were added.") + return 1 + print("All sub-CLAUDE.md file tables are complete.") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/check_tool_annotations.py b/scripts/check_tool_annotations.py new file mode 100644 index 0000000000..4c0f56350c --- /dev/null +++ b/scripts/check_tool_annotations.py @@ -0,0 +1,79 @@ +#!/usr/bin/env python3 +"""Verify all MCP tool decorators use readOnlyHint: True. + +AST-scans src/autoskillit/server/tools_*.py for @mcp.tool() decorators +and rejects any that: (1) lack the annotations= keyword, (2) have an +annotations dict missing readOnlyHint, or (3) set readOnlyHint to a non-True value. + +Exit 0 if all annotations are correct. Exit 1 with details on violations. +""" + +import ast +import sys +from pathlib import Path + +SERVER_DIR = Path(__file__).resolve().parent.parent / "src" / "autoskillit" / "server" + + +def check() -> list[str]: + violations = [] + for path in sorted(SERVER_DIR.glob("tools_*.py")): + tree = ast.parse(path.read_text(encoding="utf-8"), filename=str(path)) + for node in ast.walk(tree): + if not isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)): + continue + for dec in node.decorator_list: + if not ( + isinstance(dec, ast.Call) + and isinstance(dec.func, ast.Attribute) + and dec.func.attr == "tool" + and isinstance(dec.func.value, ast.Name) + and dec.func.value.id == "mcp" + ): + continue + ann_kw = None + for kw in dec.keywords: + if kw.arg == "annotations" and isinstance(kw.value, ast.Dict): + ann_kw = kw + break + if ann_kw is None: + violations.append( + f"{path.name}:{dec.lineno}: {node.name} " + f"missing annotations= keyword (must have readOnlyHint=True)" + ) + continue + key_names = [k.value for k in ann_kw.value.keys if isinstance(k, ast.Constant)] + if "readOnlyHint" not in key_names: + violations.append( + f"{path.name}:{dec.lineno}: {node.name} " + f"annotations dict missing readOnlyHint key (must be True)" + ) + continue + for key, val in zip(ann_kw.value.keys, ann_kw.value.values): + if ( + isinstance(key, ast.Constant) + and key.value == "readOnlyHint" + and isinstance(val, ast.Constant) + and val.value is not True + ): + violations.append( + f"{path.name}:{dec.lineno}: {node.name} " + f"has readOnlyHint={val.value!r} (must be True)" + ) + return violations + + +def main() -> int: + violations = check() + if violations: + print("readOnlyHint violations found:\n") + for v in violations: + print(f" {v}") + print("\nAll tools must have readOnlyHint=True. See server/CLAUDE.md for rationale.") + return 1 + print("All tool annotations correct: readOnlyHint=True.") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/compare-coverage-ast.py b/scripts/compare-coverage-ast.py new file mode 100755 index 0000000000..64d8510e3d --- /dev/null +++ b/scripts/compare-coverage-ast.py @@ -0,0 +1,391 @@ +#!/usr/bin/env python3 +"""Compare AST-derived function map against pytest-cov coverage database. + +Walks all Python files under a source root, extracts every function/method +definition via AST, then queries the .coverage SQLite database (via the +coverage.py public API) to determine which functions have test coverage. + +Inputs: + --coverage-db Path to .coverage file (default: .coverage in project root) + --src-root Source directory to scan (default: src/autoskillit) + --output Optional JSON report output path + +Exit codes: + 0 Always (audit tool, not a gate) +""" + +from __future__ import annotations + +import argparse +import ast +import json +import sys +from dataclasses import asdict, dataclass, field +from pathlib import Path + +PROJECT_ROOT = Path(__file__).resolve().parent.parent + + +@dataclass +class FuncInfo: + name: str + qualname: str + filepath: str + lineno: int + end_lineno: int + is_async: bool + + +@dataclass +class FuncCoverage: + func: FuncInfo + total_lines: int + covered_lines: int + status: str # "covered", "partial", "uncovered" + + +@dataclass +class Report: + total: int = 0 + covered: int = 0 + partial: int = 0 + uncovered: int = 0 + details: dict[str, list[FuncCoverage]] = field(default_factory=dict) + + +class _FuncVisitor(ast.NodeVisitor): + """AST visitor that tracks class/function nesting to build qualified names.""" + + def __init__(self, filepath: str) -> None: + self._filepath = filepath + self._name_stack: list[str] = [] + self.functions: list[FuncInfo] = [] + + def _visit_func(self, node: ast.FunctionDef | ast.AsyncFunctionDef) -> None: + qualname = ".".join([*self._name_stack, node.name]) + start = node.decorator_list[0].lineno if node.decorator_list else node.lineno + end = node.end_lineno or node.lineno + self.functions.append( + FuncInfo( + name=node.name, + qualname=qualname, + filepath=self._filepath, + lineno=start, + end_lineno=end, + is_async=isinstance(node, ast.AsyncFunctionDef), + ) + ) + self._name_stack.append(node.name) + try: + self.generic_visit(node) + finally: + self._name_stack.pop() + + def visit_FunctionDef(self, node: ast.FunctionDef) -> None: + self._visit_func(node) + + def visit_AsyncFunctionDef(self, node: ast.AsyncFunctionDef) -> None: + self._visit_func(node) + + def visit_ClassDef(self, node: ast.ClassDef) -> None: + self._name_stack.append(node.name) + try: + self.generic_visit(node) + finally: + self._name_stack.pop() + + +def extract_functions(source_path: Path, filepath: str = "") -> list[FuncInfo]: + """Parse a Python file and return FuncInfo for every function/method. + + Args: + source_path: Absolute path to the Python file on disk. + filepath: Value to store in FuncInfo.filepath (e.g. a relative path). + Defaults to str(source_path) when empty. + """ + try: + tree = ast.parse(source_path.read_text(encoding="utf-8"), filename=str(source_path)) + except SyntaxError: + print(f"WARNING: Skipping {source_path} (SyntaxError)", file=sys.stderr) + return [] + visitor = _FuncVisitor(filepath or str(source_path)) + visitor.visit(tree) + return visitor.functions + + +def build_ast_map(src_root: Path) -> dict[str, list[FuncInfo]]: + """Walk source root and build {relative_path: [FuncInfo]} map.""" + result: dict[str, list[FuncInfo]] = {} + for py_file in sorted(src_root.rglob("*.py")): + rel = str(py_file.relative_to(PROJECT_ROOT)) + funcs = extract_functions(py_file, filepath=rel) + if funcs: + result[rel] = funcs + return result + + +def query_coverage_db( + db_path: Path, +) -> tuple[dict[str, set[int]], dict[str, set[int]]]: + """Query .coverage database and return (covered_map, executable_map). + + covered_map: {relative_path: {covered_line_numbers}} + executable_map: {relative_path: {all_executable_line_numbers}} + + Uses coverage.Coverage.analysis2() to obtain the full set of executable + lines (statements) per file, not just the executed subset. + """ + import coverage + + try: + cov = coverage.Coverage(data_file=str(db_path)) + cov.load() + except Exception as exc: + print( + f"ERROR: Failed to read coverage database {db_path}: {exc}", + file=sys.stderr, + ) + return {}, {} + covered_result: dict[str, set[int]] = {} + executable_result: dict[str, set[int]] = {} + for measured_file in cov.get_data().measured_files(): + try: + rel = str(Path(measured_file).relative_to(PROJECT_ROOT)) + except ValueError: + continue + try: + analysis = cov.analysis2(measured_file) + except Exception: + continue + statements = set(analysis[1]) + missing = set(analysis[3]) + covered = statements - missing + if covered: + covered_result[rel] = covered + if statements: + executable_result[rel] = statements + return covered_result, executable_result + + +def query_contexts_map(db_path: Path) -> dict[str, set[str]]: + """Query .coverage DB and return {source_file: {test_file_paths}}. + + Uses CoverageData.contexts_by_lineno() to build the inversion. + Only includes source files under src/ and test contexts from tests/. + Context names from --cov-context=test are test node IDs like + 'tests/recipe/test_rules_dataflow.py::TestClass::test_method|run'. + Only |run phase contexts are included to exclude fixture-inflation from + |setup and |teardown phases. + """ + import coverage + + try: + data = coverage.CoverageData(basename=str(db_path)) + data.read() + except (OSError, coverage.CoverageException) as exc: + print(f"ERROR: Failed to read coverage database {db_path}: {exc}", file=sys.stderr) + return {} + + result: dict[str, set[str]] = {} + for measured_file in data.measured_files(): + try: + rel = str(Path(measured_file).relative_to(PROJECT_ROOT)) + except ValueError: + continue + if not rel.startswith("src/"): + continue + contexts_by_line = data.contexts_by_lineno(measured_file) + test_files: set[str] = set() + for contexts in contexts_by_line.values(): + for ctx in contexts: + if "::" in ctx and ctx.endswith("|run"): + test_file = ctx.split("::")[0] + if test_file.startswith("tests/") and test_file.endswith(".py"): + test_files.add(test_file) + if test_files: + result[rel] = test_files + return result + + +def build_test_source_map( + db_path: Path, + output_path: Path, +) -> bool: + """Build and write {source_file: [test_files]} map from coverage DB. + + Args: + db_path: Path to .coverage SQLite database. + output_path: Path where test-source-map.json will be written. + + Returns: + True on success, False when the coverage DB is not found. + """ + if not db_path.exists(): + print(f"WARNING: Coverage database not found: {db_path}", file=sys.stderr) + print("Run 'task coverage-audit' first to generate coverage data.", file=sys.stderr) + return False + + mapping = query_contexts_map(db_path) + # Convert sets to sorted lists for stable, human-readable JSON + serializable = {src: sorted(tests) for src, tests in sorted(mapping.items())} + output_path.parent.mkdir(parents=True, exist_ok=True) + output_path.write_text(json.dumps(serializable, indent=2) + "\n", encoding="utf-8") + print(f"Test-source map written to: {output_path} ({len(serializable)} source files)") + return True + + +def compare( + ast_map: dict[str, list[FuncInfo]], + coverage_map: dict[str, set[int]], + executable_map: dict[str, set[int]] | None = None, +) -> Report: + """Compare AST function ranges against coverage data.""" + report = Report() + for filepath, funcs in sorted(ast_map.items()): + covered_lines = coverage_map.get(filepath, set()) + executable_lines = executable_map.get(filepath, set()) if executable_map else set() + file_details: list[FuncCoverage] = [] + for func in funcs: + func_lines = set(range(func.lineno, func.end_lineno + 1)) + # Use executable lines within the function range when available, + # otherwise fall back to the full AST line range. + if executable_lines: + func_executable = func_lines & executable_lines + else: + func_executable = func_lines + intersection = func_executable & covered_lines + total = len(func_executable) if func_executable else len(func_lines) + covered_count = len(intersection) + + if covered_count == 0: + status = "uncovered" + report.uncovered += 1 + elif covered_count >= total: + status = "covered" + report.covered += 1 + else: + status = "partial" + report.partial += 1 + + report.total += 1 + file_details.append( + FuncCoverage( + func=func, + total_lines=total, + covered_lines=covered_count, + status=status, + ) + ) + if file_details: + report.details[filepath] = file_details + return report + + +def _print_report(report: Report) -> None: + """Print human-readable summary to stdout.""" + print() + print("Coverage Audit Report") + print("=====================") + if report.total == 0: + print("No functions found.") + return + + pct_c = round(report.covered * 100 / report.total) + pct_p = round(report.partial * 100 / report.total) + pct_u = round(report.uncovered * 100 / report.total) + print(f"Total functions: {report.total}") + print(f"Fully covered: {report.covered} ({pct_c}%)") + print(f"Partially covered: {report.partial} ({pct_p}%)") + print(f"Uncovered: {report.uncovered} ({pct_u}%)") + + uncovered_entries = { + fp: [fc for fc in fcs if fc.status == "uncovered"] for fp, fcs in report.details.items() + } + uncovered_entries = {fp: fcs for fp, fcs in uncovered_entries.items() if fcs} + + if uncovered_entries: + print() + print("BLIND SPOTS (uncovered functions):") + print("\u2500" * 34) + for filepath, fcs in sorted(uncovered_entries.items()): + print(f" {filepath}:") + for fc in fcs: + print(f" L{fc.func.lineno}-L{fc.func.end_lineno} {fc.func.qualname}") + + +def _write_json(report: Report, output_path: Path) -> None: + """Write structured JSON report.""" + data = { + "total": report.total, + "covered": report.covered, + "partial": report.partial, + "uncovered": report.uncovered, + "details": {fp: [asdict(fc) for fc in fcs] for fp, fcs in report.details.items()}, + } + output_path.parent.mkdir(parents=True, exist_ok=True) + output_path.write_text(json.dumps(data, indent=2) + "\n", encoding="utf-8") + print(f"\nJSON report written to: {output_path}") + + +def main() -> int: + parser = argparse.ArgumentParser( + description="Compare AST function map against coverage database" + ) + parser.add_argument( + "--mode", + choices=["audit", "build-test-source-map"], + default="audit", + help="Operation mode (default: audit)", + ) + parser.add_argument( + "--coverage-db", + type=Path, + default=PROJECT_ROOT / ".coverage", + help="Path to .coverage database (default: .coverage in project root)", + ) + parser.add_argument( + "--src-root", + type=Path, + default=PROJECT_ROOT / "src" / "autoskillit", + help="Source directory to scan (default: src/autoskillit)", + ) + parser.add_argument( + "--output", + type=Path, + default=None, + help="Path for JSON report output (audit mode) or map output (build-test-source-map mode)", + ) + args = parser.parse_args() + + if args.mode == "build-test-source-map": + output_path = args.output or (PROJECT_ROOT / ".autoskillit" / "test-source-map.json") + ok = build_test_source_map(args.coverage_db, output_path) + return 0 if ok else 1 + + if not args.src_root.is_dir(): + print(f"ERROR: Source root not found: {args.src_root}", file=sys.stderr) + return 0 + + ast_map = build_ast_map(args.src_root) + if not args.coverage_db.exists(): + print(f"WARNING: Coverage database not found: {args.coverage_db}", file=sys.stderr) + print("Run pytest with --cov first to generate coverage data.", file=sys.stderr) + total_funcs = sum(len(v) for v in ast_map.values()) + print( + f"\nNo coverage data available ({total_funcs} functions discovered, " + "coverage status unknown)." + ) + return 0 + + coverage_map, executable_map = query_coverage_db(args.coverage_db) + report = compare(ast_map, coverage_map, executable_map) + _print_report(report) + + if args.output: + _write_json(report, args.output) + + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/compile_recipes.py b/scripts/compile_recipes.py new file mode 100644 index 0000000000..2dc3764a6a --- /dev/null +++ b/scripts/compile_recipes.py @@ -0,0 +1,57 @@ +"""Pre-compile bundled recipe YAML files to JSON for faster runtime loading.""" + +from __future__ import annotations + +import json +import sys +from pathlib import Path + +import yaml + +try: + from yaml import CSafeLoader as Loader +except ImportError: + Loader = yaml.SafeLoader # type: ignore[misc,assignment] + +RECIPES_DIR = Path(__file__).resolve().parent.parent / "src" / "autoskillit" / "recipes" + + +class CompileError(Exception): + """Raised when a single recipe file fails to compile.""" + + +def _compile_one(yaml_path: Path) -> None: + try: + data = yaml.load(yaml_path.read_bytes(), Loader=Loader) + except yaml.YAMLError as exc: + raise CompileError(f"ERROR: YAML parse failed in {yaml_path}: {exc}") from exc + if not isinstance(data, dict): + raise CompileError( + f"ERROR: {yaml_path} top-level value is {type(data).__name__}, expected mapping" + ) + json_path = yaml_path.with_suffix(".json") + json_path.write_text( + json.dumps(data, indent=2, ensure_ascii=False) + "\n", + encoding="utf-8", + ) + + +def main() -> int: + if not RECIPES_DIR.is_dir(): + print(f"ERROR: recipes dir not found: {RECIPES_DIR}", file=sys.stderr) + return 1 + count = 0 + errors = 0 + for yaml_path in sorted(RECIPES_DIR.rglob("*.yaml")): + try: + _compile_one(yaml_path) + count += 1 + except CompileError as exc: + print(exc, file=sys.stderr) + errors += 1 + print(f"Compiled {count} YAML files to JSON" + (f" ({errors} errors)" if errors else "")) + return 1 if errors else 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/recipe/advance_queue_pr.sh b/scripts/recipe/advance_queue_pr.sh new file mode 100755 index 0000000000..18fc768c28 --- /dev/null +++ b/scripts/recipe/advance_queue_pr.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Args: $1=current_pr_number $2=pr_order_file +CURRENT="$1" +PR_ORDER_FILE="$2" + +NEXT=$(jq -r --arg cur "$CURRENT" ' + (to_entries | map(select(.value.number == ($cur | tonumber))) | .[0].key) as $idx | + if ($idx + 1) < length then .[$idx + 1].number | tostring + else "done" + end +' "$PR_ORDER_FILE") + +echo "$NEXT" diff --git a/scripts/recipe/create_artifact_branch.sh b/scripts/recipe/create_artifact_branch.sh new file mode 100755 index 0000000000..58de153913 --- /dev/null +++ b/scripts/recipe/create_artifact_branch.sh @@ -0,0 +1,30 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Args: $1=worktree_path $2=experiment_branch $3=base_branch $4=research_dir +WORKTREE_PATH="$1" +EXPERIMENT_BRANCH="$2" +BASE_BRANCH="$3" +RESEARCH_DIR="$4" + +SLUG=$(printf '%s' "${EXPERIMENT_BRANCH}" | sed 's/^research-//') +ARTIFACT_BRANCH="research-artifacts-${SLUG}" + +ARTIFACT_DIR="$(mktemp -d)" +trap 'git -C "${WORKTREE_PATH}" worktree remove "${ARTIFACT_DIR}" --force 2>/dev/null; rm -rf "${ARTIFACT_DIR}"' EXIT + +git -C "${WORKTREE_PATH}" fetch origin "${BASE_BRANCH}" +git -C "${WORKTREE_PATH}" branch -D "${ARTIFACT_BRANCH}" 2>/dev/null || true +git -C "${WORKTREE_PATH}" worktree add "${ARTIFACT_DIR}" -b "${ARTIFACT_BRANCH}" "origin/${BASE_BRANCH}" + +RESEARCH_SUBDIR="research/$(basename "${RESEARCH_DIR}")" +git -C "${ARTIFACT_DIR}" checkout "${EXPERIMENT_BRANCH}" -- "${RESEARCH_SUBDIR}" +cd "${ARTIFACT_DIR}" +git add research/ +git commit -m "Add research artifacts: ${SLUG}" +git push -u origin "${ARTIFACT_BRANCH}" +cd - > /dev/null + +git -C "${WORKTREE_PATH}" worktree remove "${ARTIFACT_DIR}" --force + +echo "artifact_branch=${ARTIFACT_BRANCH}" diff --git a/scripts/recipe/create_worktree.sh b/scripts/recipe/create_worktree.sh new file mode 100755 index 0000000000..fd0ae23714 --- /dev/null +++ b/scripts/recipe/create_worktree.sh @@ -0,0 +1,78 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Args: $1=source_dir $2=task $3=experiment_plan $4=scope_report $5=evaluation_dashboard +# $6=visualization_plan_path $7=report_plan_path $8=autoskillit_temp_name +# $9=visualization_plan_trace_path +SOURCE_DIR="$1" +TASK="$2" +EXPERIMENT_PLAN="$3" +SCOPE_REPORT="${4:-}" +EVAL_DASHBOARD="${5:-}" +VISUALIZATION_PLAN="${6:-}" +REPORT_PLAN="${7:-}" +TEMP_NAME="${8:-.autoskillit/temp}" +VIS_TRACE_PATH="${9:-}" + +if [ ! -d "$SOURCE_DIR/.git" ]; then + git init -q "$SOURCE_DIR" + git -C "$SOURCE_DIR" commit --allow-empty -m "autoskillit: init for research recipe" -q +fi + +BRANCH="research-$(date +%Y%m%d-%H%M%S)" +WORKTREE_PATH="../worktrees/${BRANCH}" +git -C "$SOURCE_DIR" worktree add -b "${BRANCH}" "${WORKTREE_PATH}" +RESOLVED="$(cd "${SOURCE_DIR}/${WORKTREE_PATH}" && pwd)" + +case "${RESOLVED}" in /*) ;; *) echo "error: resolved worktree path is not absolute: ${RESOLVED}" >&2; exit 1;; esac + +mkdir -p "${SOURCE_DIR}/${TEMP_NAME}/worktrees/${BRANCH}" +git -C "$SOURCE_DIR" rev-parse --abbrev-ref HEAD > "${SOURCE_DIR}/${TEMP_NAME}/worktrees/${BRANCH}/base-branch" + +mkdir -p "${RESOLVED}/${TEMP_NAME}" +cp "${EXPERIMENT_PLAN}" "${RESOLVED}/${TEMP_NAME}/experiment-plan.md" + +SLUG=$(printf '%s' "${TASK}" | tr '[:upper:]' '[:lower:]' | tr -cs 'a-z0-9' '-' | sed 's/--*/-/g;s/^-//;s/-$//' | cut -c1-30) +RESEARCH_DIR="${RESOLVED}/research/$(date +%Y-%m-%d)-${SLUG:-experiment}" +mkdir -p "${RESEARCH_DIR}/artifacts" +cp "${EXPERIMENT_PLAN}" "${RESEARCH_DIR}/experiment-plan.md" + +if [ -n "${SCOPE_REPORT}" ] && [ -f "${SCOPE_REPORT}" ]; then + cp "${SCOPE_REPORT}" "${RESEARCH_DIR}/artifacts/scope-report.md" +fi +if [ -n "${EVAL_DASHBOARD}" ] && [ -f "${EVAL_DASHBOARD}" ]; then + cp "${EVAL_DASHBOARD}" "${RESEARCH_DIR}/artifacts/design-evaluation.md" +fi +if [ -n "${VISUALIZATION_PLAN}" ] && [ -f "${VISUALIZATION_PLAN}" ]; then + cp "${VISUALIZATION_PLAN}" "${RESEARCH_DIR}/visualization-plan.md" +fi +if [ -n "${REPORT_PLAN}" ] && [ -f "${REPORT_PLAN}" ]; then + cp "${REPORT_PLAN}" "${RESEARCH_DIR}/report-plan.md" +fi + +SRC_TEMP="${SOURCE_DIR}/${TEMP_NAME}" +mkdir -p "${RESEARCH_DIR}/artifacts/review-cycles" +mkdir -p "${RESEARCH_DIR}/artifacts/plan-versions" +for f in "${SRC_TEMP}"/review-design/evaluation_dashboard_*.md; do [ -f "$f" ] && cp "$f" "${RESEARCH_DIR}/artifacts/review-cycles/"; done +for f in "${SRC_TEMP}"/review-design/revision_guidance_*.md; do [ -f "$f" ] && cp "$f" "${RESEARCH_DIR}/artifacts/review-cycles/"; done +for f in "${SRC_TEMP}"/plan-experiment/experiment_plan_*.md; do [ -f "$f" ] && cp "$f" "${RESEARCH_DIR}/artifacts/plan-versions/"; done +for f in "${SRC_TEMP}"/resolve-design-review/*.md; do [ -f "$f" ] && cp "$f" "${RESEARCH_DIR}/artifacts/review-cycles/"; done + +# --- Audit trail artifacts --- +AUDIT_DIR="${RESEARCH_DIR}/audit" +mkdir -p "${AUDIT_DIR}" + +if [ -n "${EVAL_DASHBOARD}" ] && [ -f "${EVAL_DASHBOARD}" ]; then + cp "${EVAL_DASHBOARD}" "${AUDIT_DIR}/design-review-dashboard.md" +fi + +if [ -n "${VIS_TRACE_PATH}" ] && [ -f "${VIS_TRACE_PATH}" ]; then + cp "${VIS_TRACE_PATH}" "${AUDIT_DIR}/visualization-plan-trace.md" +elif [ -f "${SRC_TEMP}/plan-visualization/visualization-plan-trace.md" ]; then + cp "${SRC_TEMP}/plan-visualization/visualization-plan-trace.md" "${AUDIT_DIR}/visualization-plan-trace.md" +fi + +cd "${RESOLVED}" && git add research/ && git commit -m "Add experiment plan and scope to research/" + +echo "research_dir=${RESEARCH_DIR}" +echo "worktree_path=${RESOLVED}" diff --git a/scripts/recipe/finalize_bundle.sh b/scripts/recipe/finalize_bundle.sh new file mode 100755 index 0000000000..db5e59337f --- /dev/null +++ b/scripts/recipe/finalize_bundle.sh @@ -0,0 +1,71 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Args: $1=output_mode $2=research_dir $3=worktree_path +OUTPUT_MODE="$1" +RESEARCH_DIR="$2" +WORKTREE_PATH="$3" + +if [ -z "${RESEARCH_DIR}" ] || [ ! -d "${RESEARCH_DIR}" ]; then + echo "finalize_bundle: no research directory found — artifact commit skipped" + exit 0 +fi + +if [ "${OUTPUT_MODE}" != "local" ]; then + if [ -f "${RESEARCH_DIR}/report.md" ]; then + cp "${RESEARCH_DIR}/report.md" "${RESEARCH_DIR}/README.md" + rm "${RESEARCH_DIR}/report.md" + fi +fi + +REPORT_FILE=$( [ "${OUTPUT_MODE}" = "local" ] && echo "report.md" || echo "README.md" ) + +if [ "${OUTPUT_MODE}" = "local" ]; then + EXCLUDE_PATTERN='^(report\.html|mermaid\.min\.js|images|report\.md|artifacts\.tar\.gz)$' +else + EXCLUDE_PATTERN='^(README\.md|artifacts\.tar\.gz)$' +fi + +mapfile -t TAR_ITEMS < <(ls -1 "${RESEARCH_DIR}" | grep -vE "${EXCLUDE_PATTERN}") + +if [ ${#TAR_ITEMS[@]} -gt 0 ]; then + tar czf "${RESEARCH_DIR}/artifacts.tar.gz" -C "${RESEARCH_DIR}" "${TAR_ITEMS[@]}" + for item in "${TAR_ITEMS[@]}"; do rm -rf "${RESEARCH_DIR}/${item}"; done +fi + +if [ -f "${RESEARCH_DIR}/artifacts.tar.gz" ] && [ -f "${RESEARCH_DIR}/${REPORT_FILE}" ]; then + MANIFEST=$(tar tzf "${RESEARCH_DIR}/artifacts.tar.gz" | sort) + { + echo "" + echo "## Archive Manifest" + echo "" + echo "Contents of \`artifacts.tar.gz\`:" + echo "" + echo '```' + echo "${MANIFEST}" + echo '```' + } >> "${RESEARCH_DIR}/${REPORT_FILE}" +fi + +[ -f "${RESEARCH_DIR}/artifacts.tar.gz" ] || \ + { echo "ERROR: artifacts.tar.gz not created in ${RESEARCH_DIR}"; exit 1; } + +if [ "${OUTPUT_MODE}" = "local" ]; then + LEFTOVER_PATTERN='^(report\.md|report\.html|mermaid\.min\.js|images|artifacts\.tar\.gz)$' +else + LEFTOVER_PATTERN='^(README\.md|artifacts\.tar\.gz)$' +fi + +LEFTOVERS=$(ls -1 "${RESEARCH_DIR}" | grep -vE "${LEFTOVER_PATTERN}" || true) +[ -z "${LEFTOVERS}" ] || \ + { echo "ERROR: uncompressed entries still present in ${RESEARCH_DIR}: ${LEFTOVERS}"; exit 1; } + +[ -f "${RESEARCH_DIR}/${REPORT_FILE}" ] || \ + { echo "ERROR: ${REPORT_FILE} not found in ${RESEARCH_DIR}"; exit 1; } + +if [ "${OUTPUT_MODE}" != "local" ]; then + cd "${WORKTREE_PATH}" && git add "${RESEARCH_DIR}" && { git diff --cached --quiet || git commit -m "Add compressed research artifacts"; } +fi + +echo "report_path=${RESEARCH_DIR}/${REPORT_FILE}" +echo "report_path_after_finalize=${RESEARCH_DIR}/${REPORT_FILE}" diff --git a/scripts/recipe/open_artifact_pr.sh b/scripts/recipe/open_artifact_pr.sh new file mode 100755 index 0000000000..a1470112c7 --- /dev/null +++ b/scripts/recipe/open_artifact_pr.sh @@ -0,0 +1,29 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Args: $1=worktree_path $2=experiment_branch $3=artifact_branch $4=pr_url $5=base_branch $6=research_dir +WORKTREE_PATH="$1" +EXPERIMENT_BRANCH="$2" +ARTIFACT_BRANCH="$3" +ORIG_PR="$4" +BASE_BRANCH="$5" +RESEARCH_DIR="$6" + +RESEARCH_SUBDIR="research/$(basename "${RESEARCH_DIR}")" +ARTIFACT_DIR="${WORKTREE_PATH}" + +if [ -f "${ARTIFACT_DIR}/${RESEARCH_SUBDIR}/artifacts.tar.gz" ]; then + STRUCT_DETAIL=$'- `artifacts.tar.gz` \xe2\x80\x94 compressed artifact tree\n\nExtract:\n```bash\ntar xzf artifacts.tar.gz\n```' +else + STRUCT_DETAIL='- `artifacts/` — uncompressed artifact tree' +fi + +PR_BODY=$(printf '## Research Artifacts\n\nFrom experiment branch `%s`.\n\nOriginal experiment PR: %s\n\n### Structure\n\n- **README.md** — Research conclusions (browsable on GitHub)\n%s\n\nContains only files under `research/` — zero production code changes.' "${EXPERIMENT_BRANCH}" "${ORIG_PR}" "${STRUCT_DETAIL}") + +PR_URL=$(gh pr create \ + --head "${ARTIFACT_BRANCH}" \ + --base "${BASE_BRANCH}" \ + --title "Research artifacts: ${EXPERIMENT_BRANCH}" \ + --body "${PR_BODY}") + +echo "artifact_pr_url=${PR_URL}" diff --git a/scripts/recipe/push_branch.sh b/scripts/recipe/push_branch.sh new file mode 100755 index 0000000000..0a1f895e92 --- /dev/null +++ b/scripts/recipe/push_branch.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Args: $1=output_mode $2=worktree_path +OUTPUT_MODE="$1" +WORKTREE_PATH="$2" + +if [ "${OUTPUT_MODE}" = "local" ]; then + echo "push_branch: skipped in local mode" + exit 0 +fi + +cd "${WORKTREE_PATH}" && git push -u origin HEAD diff --git a/scripts/recipe/stage_bundle.sh b/scripts/recipe/stage_bundle.sh new file mode 100755 index 0000000000..ad182cd988 --- /dev/null +++ b/scripts/recipe/stage_bundle.sh @@ -0,0 +1,25 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Args: $1=research_dir $2=worktree_path $3=autoskillit_temp_name +RESEARCH_DIR="$1" +WORKTREE_PATH="$2" +TEMP_NAME="${3:-.autoskillit/temp}" + +if [ -z "${RESEARCH_DIR}" ] || [ ! -d "${RESEARCH_DIR}" ]; then + echo "stage_bundle: no research directory found — skipped" + exit 0 +fi + +WT_TEMP="${WORKTREE_PATH}/${TEMP_NAME}" +mkdir -p "${RESEARCH_DIR}/artifacts/phase-groups" +mkdir -p "${RESEARCH_DIR}/artifacts/phase-plans" +mkdir -p "${RESEARCH_DIR}/artifacts/images" +mkdir -p "${RESEARCH_DIR}/artifacts/scripts" + +for f in "${WT_TEMP}"/make-groups/*.md; do [ -f "$f" ] && cp "$f" "${RESEARCH_DIR}/artifacts/phase-groups/"; done +for f in "${WT_TEMP}"/make-groups/*.yaml; do [ -f "$f" ] && cp "$f" "${RESEARCH_DIR}/artifacts/phase-groups/"; done +for f in "${WT_TEMP}"/make-plan/*.md; do [ -f "$f" ] && cp "$f" "${RESEARCH_DIR}/artifacts/phase-plans/"; done +for f in "${WT_TEMP}"/exp-lens-*/*.md; do [ -f "$f" ] && cp "$f" "${RESEARCH_DIR}/artifacts/images/"; done + +echo "stage_bundle=done" diff --git a/scripts/recipe/tag_experiment_branch.sh b/scripts/recipe/tag_experiment_branch.sh new file mode 100755 index 0000000000..abf794876f --- /dev/null +++ b/scripts/recipe/tag_experiment_branch.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Args: $1=worktree_path $2=experiment_branch $3=artifact_pr_url +WORKTREE_PATH="$1" +EXPERIMENT_BRANCH="$2" +ARTIFACT_PR_URL="$3" + +TAG="archive/research/${EXPERIMENT_BRANCH}" + +git -C "${WORKTREE_PATH}" tag -a "${TAG}" "${EXPERIMENT_BRANCH}" \ + -m "Research experiment: ${EXPERIMENT_BRANCH}. Report merged via artifact PR ${ARTIFACT_PR_URL}. Variant code preserved here for reference." +git -C "${WORKTREE_PATH}" push origin "${TAG}" + +echo "archive_tag=${TAG}" diff --git a/scripts/sync_plugin_json.py b/scripts/sync_plugin_json.py deleted file mode 100644 index ce555919c2..0000000000 --- a/scripts/sync_plugin_json.py +++ /dev/null @@ -1,43 +0,0 @@ -#!/usr/bin/env python3 -"""Sync plugin.json version from pyproject.toml. - -Reads the version from pyproject.toml and writes it into -src/autoskillit/.claude-plugin/plugin.json atomically. - -Exit 0 if the file was updated (or already in sync). Exit 1 on error. -""" - -from __future__ import annotations - -import json -import sys -import tomllib -from pathlib import Path - -PROJECT_ROOT = Path(__file__).resolve().parent.parent -PYPROJECT = PROJECT_ROOT / "pyproject.toml" -PLUGIN_JSON = PROJECT_ROOT / "src" / "autoskillit" / ".claude-plugin" / "plugin.json" - - -def main() -> int: - version = tomllib.loads(PYPROJECT.read_text(encoding="utf-8"))["project"]["version"] - data = json.loads(PLUGIN_JSON.read_text(encoding="utf-8")) - if data.get("version") == version: - print(f"plugin.json already at version {version}") - return 0 - data["version"] = version - content = json.dumps(data, indent=2) + "\n" - tmp = PLUGIN_JSON.parent / (PLUGIN_JSON.name + ".tmp") - try: - tmp.write_text(content, encoding="utf-8") - tmp.replace(PLUGIN_JSON) - except Exception as e: - tmp.unlink(missing_ok=True) - print(f"Error: {e}", file=sys.stderr) - return 1 - print(f"plugin.json updated to version {version}") - return 0 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/scripts/sync_versions.py b/scripts/sync_versions.py new file mode 100644 index 0000000000..2166313590 --- /dev/null +++ b/scripts/sync_versions.py @@ -0,0 +1,68 @@ +#!/usr/bin/env python3 +"""Sync all version-bearing artifacts from pyproject.toml. + +Discovers plugin.json and updates it to match the canonical version +in pyproject.toml. + +Exit 0 if plugin.json is in sync (or was updated). +Exit 1 on error or (in --check mode) if plugin.json is out of sync. +""" + +from __future__ import annotations + +import json +import sys +import tomllib +from pathlib import Path + +PROJECT_ROOT = Path(__file__).resolve().parent.parent +PYPROJECT = PROJECT_ROOT / "pyproject.toml" +PLUGIN_JSON = PROJECT_ROOT / "src" / "autoskillit" / ".claude-plugin" / "plugin.json" + + +def _sync_plugin_json(version: str, *, check: bool) -> tuple[bool, bool]: + """Sync plugin.json. Returns (was_stale, updated_ok).""" + data = json.loads(PLUGIN_JSON.read_text(encoding="utf-8")) + if data.get("version") == version: + return False, True + if check: + return True, True + data["version"] = version + content = json.dumps(data, indent=2) + "\n" + tmp = PLUGIN_JSON.parent / (PLUGIN_JSON.name + ".tmp") + try: + tmp.write_text(content, encoding="utf-8") + tmp.replace(PLUGIN_JSON) + except Exception as e: + tmp.unlink(missing_ok=True) + print(f"Error updating plugin.json: {e}", file=sys.stderr) + return True, False + return True, True + + +def main() -> int: + check = "--check" in sys.argv + + version = tomllib.loads(PYPROJECT.read_text(encoding="utf-8"))["project"]["version"] + + was_stale, ok = _sync_plugin_json(version, check=check) + if not ok: + print("Error updating plugin.json", file=sys.stderr) + return 1 + + if check: + if was_stale: + print(f"plugin.json is out of sync (expected {version})") + return 1 + print(f"plugin.json in sync at {version}") + return 0 + + if was_stale: + print(f"Updated plugin.json to {version}") + else: + print(f"plugin.json already at {version}") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/verify-test-filter.sh b/scripts/verify-test-filter.sh new file mode 100755 index 0000000000..b21e8c84ab --- /dev/null +++ b/scripts/verify-test-filter.sh @@ -0,0 +1,122 @@ +#!/usr/bin/env bash +# Anti-drift verification for test path filtering. +# +# Shadow-mode comparison: collects tests with and without the filter, +# identifies missed tests, runs them to detect false negatives, and +# reports reduction percentage. +# +# Usage: scripts/verify-test-filter.sh +# base_ref: git ref for AUTOSKILLIT_TEST_BASE_REF (e.g. HEAD~5) +# +# NOTE: This script uses .venv/bin/python -m pytest directly (not task +# test-check) because it needs raw --collect-only output without the +# Taskfile harness. This is an intentional exception to the "no bare +# pytest" convention. + +set -euo pipefail + +# Deterministic byte-order for sort and comm — prevents locale-dependent +# collation surprises with parametrized test IDs (brackets, hyphens, etc.) +export LC_ALL=C + +BASE_REF="${1:?Usage: $0 }" + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +PROJECT_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" +cd "$PROJECT_ROOT" + +PYTEST=".venv/bin/python -m pytest" +if [[ ! -x ".venv/bin/python" ]]; then + echo "ERROR: .venv/bin/python not found. Run 'task install-worktree' first." >&2 + exit 1 +fi + +WORK_DIR="$(mktemp -d "${TMPDIR:-/tmp}/test-filter-audit.XXXXXX")" +trap 'rm -rf "$WORK_DIR"' EXIT + +FULL_FILE="$WORK_DIR/full_selected.txt" +FILTER_FILE="$WORK_DIR/filter_selected.txt" +MISSED_FILE="$WORK_DIR/missed_tests.txt" + +echo "=== Test Filter Anti-Drift Verification ===" +echo "Base ref: $BASE_REF" +echo "" + +# --- Step 1: Full collection (no filter) --- +echo "--- Step 1: Collecting all tests (no filter) ---" +$PYTEST tests/ --collect-only -q --no-header --disable-warnings -o "addopts=" -m "not smoke and not canary" 2>/dev/null \ + | grep '::' \ + | sort > "$FULL_FILE" + +TOTAL=$(wc -l < "$FULL_FILE") +echo "Total tests collected: $TOTAL" + +if [[ "$TOTAL" -eq 0 ]]; then + echo "ERROR: No tests collected in full run. Something is wrong." >&2 + exit 1 +fi + +# --- Step 2: Filtered collection (conservative) --- +echo "" +echo "--- Step 2: Collecting tests with conservative filter (base_ref=$BASE_REF) ---" +AUTOSKILLIT_TEST_BASE_REF="$BASE_REF" \ +AUTOSKILLIT_TEST_FILTER=conservative \ +$PYTEST tests/ --collect-only -q --no-header --disable-warnings -o "addopts=" -m "not smoke and not canary" 2>/dev/null \ + | grep '::' \ + | sort > "$FILTER_FILE" + +FILTERED=$(wc -l < "$FILTER_FILE") +echo "Filtered tests collected: $FILTERED" + +# --- Step 3: Find missed tests --- +echo "" +echo "--- Step 3: Computing shadow diff ---" +comm -23 "$FULL_FILE" "$FILTER_FILE" > "$MISSED_FILE" + +MISSED=$(wc -l < "$MISSED_FILE") +echo "Missed tests (in full but not in filtered): $MISSED" + +# --- Step 4: Report reduction --- +echo "" +echo "--- Report ---" +if [[ "$TOTAL" -gt 0 ]]; then + REDUCTION=$(awk "BEGIN {printf \"%.1f\", ($TOTAL - $FILTERED) * 100 / $TOTAL}") + echo "Reduction: ${REDUCTION}% ($TOTAL → $FILTERED)" +else + REDUCTION="0.0" + echo "Reduction: 0.0% (no tests collected)" +fi + +echo "Total tests: $TOTAL" +echo "Filtered tests: $FILTERED" +echo "Missed tests: $MISSED" +echo "Reduction: ${REDUCTION}%" + +# --- Step 5: Run missed tests if any --- +if [[ "$MISSED" -gt 0 ]]; then + echo "" + echo "--- Step 5: Running $MISSED missed tests to detect false negatives ---" + + set +e + $PYTEST $(cat "$MISSED_FILE" | tr '\n' ' ') \ + --tb=short -q --no-header -o "addopts=" \ + -m "not smoke and not canary" 2>&1 + MISSED_EXIT=$? + set -e + + if [[ "$MISSED_EXIT" -ne 0 ]]; then + echo "" + echo "FALSE NEGATIVE DETECTED: $MISSED missed tests and some FAILED." + echo "The test filter is dropping tests that would have caught regressions." + exit 1 + else + echo "" + echo "Missed tests all passed — no false negatives detected." + echo "These tests were excluded by the filter but would not have caught regressions." + exit 0 + fi +else + echo "" + echo "No missed tests — filter is not dropping any tests for this diff range." + exit 0 +fi diff --git a/src/autoskillit/.claude-plugin/plugin.json b/src/autoskillit/.claude-plugin/plugin.json index 03df03157d..8e8ce395a8 100644 --- a/src/autoskillit/.claude-plugin/plugin.json +++ b/src/autoskillit/.claude-plugin/plugin.json @@ -1,5 +1,5 @@ { "name": "autoskillit", - "version": "0.9.0", + "version": "0.9.569", "description": "Orchestrated skill-driven workflows using Claude Code headless sessions" } diff --git a/src/autoskillit/CLAUDE.md b/src/autoskillit/CLAUDE.md new file mode 100644 index 0000000000..108a72dc26 --- /dev/null +++ b/src/autoskillit/CLAUDE.md @@ -0,0 +1,21 @@ +# autoskillit/ + +Package root — entry points, hook registry, and cross-cutting utilities. + +## Files + +| File | Purpose | +|------|---------| +| `__init__.py` | Package exports | +| `__main__.py` | `python -m autoskillit` entry point | +| `_llm_triage.py` | Contract staleness triage (Haiku subprocess) | +| `smoke_utils.py` | Callables for smoke-test pipeline `run_python` steps | +| `hook_registry.py` | `HookDef`, `HOOK_REGISTRY`, `generate_hooks_json` | +| `_test_filter.py` | Test filter manifest: glob-to-test-directory mapping | +| `version.py` | Version health utilities (IL-0) | + +## Architecture Notes + +`hook_registry.py` is stdlib-only (safe for hook subprocesses). `_test_filter.py` drives +`task test-filtered` — it maps changed-file globs to test directory subsets. `_llm_triage.py` +and `smoke_utils.py` are callable by headless recipe steps via `run_python`. diff --git a/src/autoskillit/_llm_triage.py b/src/autoskillit/_llm_triage.py index 8da813550b..215fe68849 100644 --- a/src/autoskillit/_llm_triage.py +++ b/src/autoskillit/_llm_triage.py @@ -129,7 +129,7 @@ async def _triage_batch( if result.termination == TerminationReason.TIMED_OUT: raise TimeoutError("triage_staleness batch timed out") session = parse_session_result(result.stdout) - if session.is_error or not session.result: + if session.is_error or session.needs_retry or not session.result: logger.warning( "triage_staleness batch parse failed; treating all as meaningful", batch=[i.skill for i in triageable], diff --git a/src/autoskillit/_test_filter.py b/src/autoskillit/_test_filter.py new file mode 100644 index 0000000000..6e17fe7082 --- /dev/null +++ b/src/autoskillit/_test_filter.py @@ -0,0 +1,74 @@ +"""Test filter manifest: glob-to-test-directory mapping for non-Python files. + +Loads ``.autoskillit/test-filter-manifest.yaml`` and resolves changed non-Python +file paths to the minimal set of test directories that must run. +""" + +from __future__ import annotations + +from pathlib import Path + +import pathspec + +from autoskillit.core import load_yaml + + +def load_manifest(path: Path) -> dict[str, list[str]]: + """Load the test filter manifest YAML. + + Args: + path: Absolute path to the manifest YAML file. + + Returns: + Dict mapping glob pattern strings to lists of test directory strings. + + Raises: + FileNotFoundError: If the manifest file does not exist. + """ + data = load_yaml(path) + if not isinstance(data, dict): + msg = f"Manifest must be a YAML mapping, got {type(data).__name__}" + raise TypeError(msg) + return data + + +def apply_manifest( + changed_files: list[str], + manifest: dict[str, list[str]], +) -> set[str] | None: + """Resolve changed non-Python files to test directories via the manifest. + + For each manifest entry, compiles the glob pattern using pathspec's + gitwildmatch and checks whether any changed file matches. Collects the + union of all matched test directories. + + Args: + changed_files: Repo-relative paths of changed non-Python files. + manifest: Output of ``load_manifest()``. + + Returns: + A set of test directory strings (relative to ``tests/``) if at least + one file matched a manifest pattern. ``None`` if any changed file + matched no pattern (fail-open: caller should run the full suite). + """ + if not changed_files: + return set() + + compiled = {pat: pathspec.PathSpec.from_lines("gitwildmatch", [pat]) for pat in manifest} + + matched_dirs: set[str] = set() + unmatched: list[str] = [] + + for file_path in changed_files: + file_matched = False + for pattern, spec in compiled.items(): + if spec.match_file(file_path): + matched_dirs.update(manifest[pattern]) + file_matched = True + if not file_matched: + unmatched.append(file_path) + + if unmatched: + return None + + return matched_dirs diff --git a/src/autoskillit/cli/CLAUDE.md b/src/autoskillit/cli/CLAUDE.md new file mode 100644 index 0000000000..485428cb2b --- /dev/null +++ b/src/autoskillit/cli/CLAUDE.md @@ -0,0 +1,36 @@ +# cli/ + +IL-3 CLI layer — entry points for all user-facing commands. +Sub-packages: doctor/ (see doctor/CLAUDE.md), fleet/ (see fleet/CLAUDE.md), +session/ (see session/CLAUDE.md), ui/ (see ui/CLAUDE.md), update/ (see update/CLAUDE.md). + +## Files + +| File | Purpose | +|------|---------| +| `__init__.py` | Package marker | +| `app.py` | CLI entry: `serve`, `init`, `config`, `skills`, `recipes`, `doctor`, `update`, etc. | +| `_restart.py` | `perform_restart()` → `NoReturn`: sets `SKIP_UPDATE_CHECK`, calls `os.execv` | +| `_hooks.py` | `PreToolUse` hook registration helpers | +| `_init_helpers.py` | `autoskillit init` implementation helpers | +| `_installed_plugins.py` | `InstalledPluginsFile` — canonical accessor for `installed_plugins.json` | +| `_install_info.py` | `InstallInfo`, `InstallType`, `detect_install()`, `comparison_branch()`, `dismissal_window()`, `upgrade_command()` | +| `_marketplace.py` | Plugin install/upgrade | +| `_mcp_names.py` | MCP prefix detection | +| `_onboarding.py` | First-run detection + guided menu | +| `_prompts.py` | Prompt builder re-export hub + shared helpers (`_MCP_RETRY_INSTRUCTION`, `_read_full_sous_chef`) | +| `_prompts_campaign.py` | L3 campaign dispatcher prompt builder | +| `_prompts_orchestrator.py` | L1/L2 cook session prompt builder + ingredients table + greetings | +| `_prompts_kitchen.py` | Open-kitchen and fleet-dispatch prompt builders | +| `_preview.py` | Shared pre-launch preview: flow diagram + ingredient table display | +| `_serve_guard.py` | Async signal-guarded MCP server bootstrap (extracted from `app.py`) | +| `_features.py` | `features` subcommand group: list/status commands for feature gate inspection | +| `_workspace.py` | Workspace clean helpers | +| `_sessions.py` | `sessions analyze` CLI subcommand for cross-session DFG visualization | +| `_validate.py` | `validate registries` subcommand for validating user-override registry files | + +## Architecture Notes + +`app.py` is the Click application root; all sub-packages register their subcommand groups +against the root Click group. `_serve_guard.py` was extracted from `app.py` to isolate +the asyncio/signal machinery for testability. diff --git a/src/autoskillit/cli/__init__.py b/src/autoskillit/cli/__init__.py index a11a10228e..8ed3f0acda 100644 --- a/src/autoskillit/cli/__init__.py +++ b/src/autoskillit/cli/__init__.py @@ -9,24 +9,21 @@ import subprocess # noqa: F401 — tests patch autoskillit.cli.subprocess.run from pathlib import Path # noqa: F401 — tests patch autoskillit.cli.Path.home -from autoskillit.cli._cook import cook -from autoskillit.cli._doctor import DoctorResult from autoskillit.cli._hooks import _claude_settings_path from autoskillit.cli._init_helpers import _prompt_recipe_choice +from autoskillit.cli._mcp_names import detect_autoskillit_mcp_prefix from autoskillit.cli._prompts import ( - _OPEN_KITCHEN_CHOICE, _build_open_kitchen_prompt, _build_orchestrator_prompt, - _resolve_recipe_input, ) +from autoskillit.cli._validate import validate_app from autoskillit.cli.app import ( _generate_config_yaml, - _is_plugin_installed, _prompt_test_command, app, config_app, config_show, - doctor, + fleet_app, init, install, main, @@ -46,11 +43,17 @@ workspace_clean, workspace_init, ) +from autoskillit.cli.app import ( + doctor as _doctor_cmd, +) +from autoskillit.cli.doctor import DoctorResult +from autoskillit.cli.fleet import fleet_campaign, fleet_dispatch, fleet_list, fleet_status +from autoskillit.cli.session._cook import cook from autoskillit.hook_registry import HookDriftResult +doctor_cmd = _doctor_cmd + __all__ = [ - "_OPEN_KITCHEN_CHOICE", - "_is_plugin_installed", "_build_open_kitchen_prompt", "_build_orchestrator_prompt", "DoctorResult", @@ -59,12 +62,17 @@ "_generate_config_yaml", "_prompt_recipe_choice", "_prompt_test_command", - "_resolve_recipe_input", "app", "config_app", "config_show", "cook", - "doctor", + "detect_autoskillit_mcp_prefix", + "doctor_cmd", + "fleet_app", + "fleet_campaign", + "fleet_dispatch", + "fleet_list", + "fleet_status", "init", "install", "main", @@ -83,4 +91,5 @@ "workspace_app", "workspace_clean", "workspace_init", + "validate_app", ] diff --git a/src/autoskillit/cli/_cook.py b/src/autoskillit/cli/_cook.py deleted file mode 100644 index 2484b0df4d..0000000000 --- a/src/autoskillit/cli/_cook.py +++ /dev/null @@ -1,175 +0,0 @@ -"""cook command: interactive skill session launcher.""" - -from __future__ import annotations - -import shutil -import subprocess -import uuid -from collections.abc import Mapping -from pathlib import Path - -from autoskillit.cli._terminal import terminal_guard - -_DISPLAY_CATEGORIES: tuple[tuple[str, tuple[str, ...]], ...] = ( - ("Execution", ("run_cmd", "run_python", "run_skill")), - ("Testing & Workspace", ("test_check", "reset_test_dir", "classify_fix", "reset_workspace")), - ( - "Git Operations", - ("merge_worktree", "create_unique_branch", "check_pr_mergeable", "set_commit_status"), - ), - ("Recipes", ("migrate_recipe", "list_recipes", "load_recipe", "validate_recipe")), - ( - "Clone & Remote", - ( - "clone_repo", - "remove_clone", - "push_to_remote", - "register_clone_status", - "batch_cleanup_clones", - ), - ), - ( - "GitHub", - ( - "fetch_github_issue", - "get_issue_title", - "report_bug", - "prepare_issue", - "enrich_issues", - "claim_issue", - "release_issue", - "get_pr_reviews", - "bulk_close_issues", - ), - ), - ( - "CI & Automation", - ( - "wait_for_ci", - "wait_for_merge_queue", - "check_repo_merge_state", - "toggle_auto_merge", - "get_ci_status", - ), - ), - ( - "Telemetry & Diagnostics", - ( - "read_db", - "write_telemetry_files", - "kitchen_status", - "get_pipeline_report", - "get_token_summary", - "get_timing_summary", - "get_quota_events", - ), - ), - ("Kitchen", ("open_kitchen", "close_kitchen", "disable_quota_guard")), -) - - -def _run_cook_session( - *, - cmd: list[str], - env: Mapping[str, str], - _first_run: bool, - initial_prompt: str | None, - project_dir: Path, -) -> None: - """Run the cook subprocess and gate mark_onboarded on success.""" - with terminal_guard(): - result = subprocess.run(cmd, env=env) - if result.returncode == 0: - if _first_run and initial_prompt is not None: - from autoskillit.cli._onboarding import mark_onboarded - - mark_onboarded(project_dir) - else: - raise SystemExit(result.returncode) - - -def cook(*, resume: bool = False, session_id: str | None = None) -> None: - """Launch Claude with all bundled AutoSkillit skills as slash commands.""" - from autoskillit.workspace import ( - DefaultSessionSkillManager, - SkillsDirectoryProvider, - resolve_ephemeral_root, - ) - - if not shutil.which("claude"): - print("'claude' not found on PATH. Install Claude Code to use cook.") - raise SystemExit(1) - - from autoskillit import __version__ - from autoskillit.cli._ansi import supports_color - - color = supports_color() - _B = "\x1b[1m" if color else "" - _C = "\x1b[96m" if color else "" - _D = "\x1b[2m" if color else "" - _G = "\x1b[32m" if color else "" - _Y = "\x1b[33m" if color else "" - _R = "\x1b[0m" if color else "" - - print(f"{_B}{_C}AUTOSKILLIT {__version__}{_R} {_D}Kitchen open. All tools active.{_R}") - skip = {"Telemetry & Diagnostics", "Kitchen"} - for name, tools in _DISPLAY_CATEGORIES: - if name in skip: - continue - tool_list = f"{_D}, {_R}".join(f"{_G}{t}{_R}" for t in tools) - print(f" {_Y}{name:>20}{_R} {tool_list}") - print() - - from autoskillit.cli._ansi import permissions_warning - from autoskillit.cli._timed_input import timed_prompt - - print(permissions_warning()) - confirm = timed_prompt( - "\nLaunch session? [Enter/n]", default="", timeout=120, label="autoskillit cook" - ) - if confirm.lower() in ("n", "no"): - return - - from autoskillit.cli._init_helpers import _is_plugin_installed - from autoskillit.cli._onboarding import is_first_run, run_onboarding_menu - from autoskillit.config import load_config - from autoskillit.core import configure_logging, find_latest_session_id, pkg_root - from autoskillit.execution import build_interactive_cmd - - configure_logging() - - resume_session_id: str | None = None - if resume: - resume_session_id = session_id or find_latest_session_id() - if resume_session_id is None: - print("No previous session found. Starting a fresh session.") - - project_dir = Path.cwd() - initial_prompt: str | None = None - _first_run = is_first_run(project_dir) - if _first_run: - initial_prompt = run_onboarding_menu(project_dir, color=color) - - session_id_local = uuid.uuid4().hex[:16] - ephemeral_root = resolve_ephemeral_root() - session_mgr = DefaultSessionSkillManager(SkillsDirectoryProvider(), ephemeral_root) - session_mgr.cleanup_stale() - config = load_config() - skills_dir = session_mgr.init_session( - session_id_local, cook_session=True, config=config, project_dir=project_dir - ) - - plugin_dir = None if _is_plugin_installed() else pkg_root() - spec = build_interactive_cmd( - plugin_dir=plugin_dir, - add_dirs=[skills_dir], - initial_prompt=initial_prompt, - resume_session_id=resume_session_id, - ) - _run_cook_session( - cmd=spec.cmd, - env=spec.env, - _first_run=_first_run, - initial_prompt=initial_prompt, - project_dir=project_dir, - ) diff --git a/src/autoskillit/cli/_doctor.py b/src/autoskillit/cli/_doctor.py deleted file mode 100644 index 969f636e82..0000000000 --- a/src/autoskillit/cli/_doctor.py +++ /dev/null @@ -1,880 +0,0 @@ -"""Doctor command implementation — project setup checks.""" - -from __future__ import annotations - -import json -import shutil -import subprocess -import urllib.parse -from dataclasses import dataclass -from pathlib import Path - -from autoskillit.cli._hooks import _claude_settings_path, _load_settings_data -from autoskillit.cli._init_helpers import ( - _KNOWN_SCANNERS, - _check_dual_mcp_files, - _detect_secret_scanner, -) -from autoskillit.core import Severity, get_logger -from autoskillit.execution import QUOTA_CACHE_SCHEMA_VERSION -from autoskillit.hook_registry import ( - _count_hook_registry_drift, - canonical_script_basenames, - find_broken_hook_scripts, -) - -_log = get_logger(__name__) - - -@dataclass -class DoctorResult: - """Outcome of a single doctor check.""" - - severity: Severity - check: str - message: str - - -def _check_mcp_server_registered(claude_json_path: Path | None = None) -> DoctorResult: - """Check that autoskillit MCP server is registered (via mcpServers or plugin).""" - if claude_json_path is None: - claude_json_path = Path.home() / ".claude.json" - - # Check 1: direct mcpServers entry (legacy / init-based registration) - if claude_json_path.exists(): - try: - data = json.loads(claude_json_path.read_text()) - if "autoskillit" in data.get("mcpServers", {}): - return DoctorResult( - severity=Severity.OK, - check="mcp_server_registered", - message="autoskillit registered in mcpServers", - ) - except OSError as exc: - return DoctorResult( - severity=Severity.ERROR, - check="mcp_server_registered", - message=f"~/.claude.json could not be read: {exc}", - ) - except json.JSONDecodeError as exc: - return DoctorResult( - severity=Severity.ERROR, - check="mcp_server_registered", - message=f"~/.claude.json is not valid JSON: {exc}", - ) - - # Check 2: plugin-based registration (install-based) - _plugin_check_detail = "" - try: - result = subprocess.run( - ["claude", "plugin", "list"], - capture_output=True, - text=True, - timeout=10, - ) - if result.returncode == 0 and "autoskillit" in result.stdout: - return DoctorResult( - severity=Severity.OK, - check="mcp_server_registered", - message="autoskillit registered as Claude plugin", - ) - except (FileNotFoundError, subprocess.TimeoutExpired, OSError) as exc: - _plugin_check_detail = f" (claude plugin list unavailable: {type(exc).__name__})" - else: - _plugin_check_detail = "" - - return DoctorResult( - severity=Severity.WARNING, - check="mcp_server_registered", - message=( - "autoskillit not registered. Run 'autoskillit install' to install as a plugin, " - "or 'autoskillit init' to register in mcpServers." + _plugin_check_detail - ), - ) - - -def _check_dual_mcp_registration( - claude_json_path: Path | None = None, - plugins_json_path: Path | None = None, -) -> DoctorResult: - """Check that autoskillit is not registered both as a direct entry and as a marketplace plugin. - - Returns WARNING if both registrations are simultaneously present (split-brain condition). - Fail-open: unreadable files → cannot confirm dual registration, return OK. - """ - _claude_json = claude_json_path or (Path.home() / ".claude.json") - _plugins_json = plugins_json_path or ( - Path.home() / ".claude" / "plugins" / "installed_plugins.json" - ) - if _check_dual_mcp_files(_claude_json, _plugins_json): - return DoctorResult( - severity=Severity.WARNING, - check="dual_mcp_registration", - message=( - "autoskillit is registered both as a direct mcpServers entry " - "(~/.claude.json) and as a marketplace plugin. This spawns two " - "independent server processes per session with split gate state. " - "Run `autoskillit install` to remove the stale direct entry." - ), - ) - return DoctorResult( - severity=Severity.OK, - check="dual_mcp_registration", - message="", - ) - - -def _check_hook_registration(settings_path: Path) -> DoctorResult: - data = _load_settings_data(settings_path) - registered = " ".join( - hook.get("command", "") - for event_entries in data.get("hooks", {}).values() - if isinstance(event_entries, list) - for entry in event_entries - for hook in entry.get("hooks", []) - ) - missing = [script for script in canonical_script_basenames() if script not in registered] - if missing: - return DoctorResult( - severity=Severity.WARNING, - check="hook_registration", - message=f"Missing hooks: {', '.join(missing)}. Run 'autoskillit init'.", - ) - return DoctorResult( - severity=Severity.OK, - check="hook_registration", - message="All HOOK_REGISTRY scripts present in settings.json.", - ) - - -def _check_hook_registry_drift( - settings_path: Path, scope_label: str | None = None -) -> DoctorResult: - """Compare generate_hooks_json() with what is deployed in settings.json.""" - result = _count_hook_registry_drift(settings_path) - if result.orphaned > 0: - ghost_scripts = sorted(result.orphaned_cmds) - msg = ( - f"Orphaned hook entries detected: {', '.join(ghost_scripts)}. " - f"These scripts are missing from HOOK_REGISTRY but present in " - f"settings.json — every matching tool call will be denied with ENOENT. " - f"Run 'autoskillit install' to regenerate hooks." - ) - if scope_label: - msg = f"[{scope_label}] {msg}" - return DoctorResult(Severity.ERROR, "hook_registry_drift", msg) - if result.missing > 0: - msg = ( - f"Hook registry has changed since last install. " - f"Run 'autoskillit install' to deploy {result.missing} new/changed hook(s)." - ) - if scope_label: - msg = f"[{scope_label}] {msg}" - return DoctorResult( - severity=Severity.WARNING, - check="hook_registry_drift", - message=msg, - ) - msg = "Deployed hooks match HOOK_REGISTRY." - if scope_label: - msg = f"[{scope_label}] {msg}" - return DoctorResult( - severity=Severity.OK, - check="hook_registry_drift", - message=msg, - ) - - -def _check_hook_health(settings_path: Path) -> DoctorResult: - """Verify all deployed hook scripts exist on disk for all event types.""" - broken_hooks = find_broken_hook_scripts(settings_path) - if broken_hooks: - return DoctorResult( - severity=Severity.ERROR, - check="hook_health", - message=f"Hook scripts not found: {', '.join(broken_hooks)}", - ) - return DoctorResult(Severity.OK, "hook_health", "All hook scripts accessible") - - -def _check_gitignore_completeness(project_dir: Path) -> DoctorResult: - """Check that every file in .autoskillit/ is gitignored or in the committed allowlist.""" - from autoskillit.core import _AUTOSKILLIT_GITIGNORE_ENTRIES, _COMMITTED_BY_DESIGN - - autoskillit_dir = project_dir / ".autoskillit" - gitignore_path = autoskillit_dir / ".gitignore" - if not autoskillit_dir.is_dir(): - return DoctorResult(Severity.OK, "gitignore_completeness", "No .autoskillit/ directory.") - if not gitignore_path.exists(): - return DoctorResult( - Severity.WARNING, - "gitignore_completeness", - ".autoskillit/.gitignore missing. Run 'autoskillit init'.", - ) - gitignore_content = gitignore_path.read_text(encoding="utf-8") - uncovered: list[str] = [] - for item in sorted(autoskillit_dir.iterdir()): - if item.name == ".gitignore": - continue - if item.name in _COMMITTED_BY_DESIGN: - continue - check_name = item.name + "/" if item.is_dir() else item.name - if check_name not in gitignore_content: - uncovered.append(item.name) - # Also check that every entry in the canonical list is present - for entry in _AUTOSKILLIT_GITIGNORE_ENTRIES: - if entry not in gitignore_content: - entry_name = entry.rstrip("/") - if entry_name not in uncovered: - uncovered.append(entry_name) - if uncovered: - return DoctorResult( - Severity.WARNING, - "gitignore_completeness", - f"Files in .autoskillit/ not covered by .gitignore: {', '.join(uncovered)}. " - "Add to _AUTOSKILLIT_GITIGNORE_ENTRIES or _COMMITTED_BY_DESIGN.", - ) - return DoctorResult(Severity.OK, "gitignore_completeness", "All .autoskillit/ files covered.") - - -def _check_secret_scanning_hook(project_dir: Path) -> DoctorResult: - """Check that .pre-commit-config.yaml includes a known secret scanning hook.""" - if _detect_secret_scanner(project_dir): - return DoctorResult( - Severity.OK, - "secret_scanning_hook", - "Secret scanning hook detected in .pre-commit-config.yaml.", - ) - pre_commit_path = project_dir / ".pre-commit-config.yaml" - if not pre_commit_path.exists(): - msg = ( - "No .pre-commit-config.yaml found. AutoSkillit commits code automatically — " - "add a secret scanner (gitleaks, detect-secrets, trufflehog, or git-secrets) " - "to prevent credential leaks." - ) - else: - scanners = ", ".join(sorted(_KNOWN_SCANNERS)) - msg = ( - f".pre-commit-config.yaml exists but contains no known secret scanner " - f"({scanners}). Add one to prevent credential leaks." - ) - return DoctorResult(Severity.ERROR, "secret_scanning_hook", msg) - - -def _check_editable_install_source_exists() -> DoctorResult: - """Detect editable autoskillit installs whose source directory no longer exists.""" - import importlib.metadata as meta - - check_name = "editable_install_source_exists" - try: - dist = meta.Distribution.from_name("autoskillit") - except meta.PackageNotFoundError: - return DoctorResult(Severity.OK, check_name, "autoskillit not installed in this env") - - direct_url_text = dist.read_text("direct_url.json") - if not direct_url_text: - return DoctorResult(Severity.OK, check_name, "Not an editable install") - - try: - direct_url = json.loads(direct_url_text) - except json.JSONDecodeError: - return DoctorResult(Severity.OK, check_name, "direct_url.json unreadable — skipped") - - is_editable = ( - direct_url.get("dir_info", {}).get("editable") is True - or direct_url.get("editable") is True - ) - if not is_editable: - return DoctorResult(Severity.OK, check_name, "Not an editable install") - - url = direct_url.get("url", "") - src_path = urllib.parse.urlparse(url).path if url.startswith("file://") else "" - if not src_path or Path(src_path).exists(): - return DoctorResult(Severity.OK, check_name, "Editable install source directory exists") - - return DoctorResult( - Severity.ERROR, - check_name, - f"autoskillit is installed from a deleted directory: {src_path}. " - f"Fix: uv tool install --force autoskillit && autoskillit install", - ) - - -def _check_stale_entry_points() -> DoctorResult: - """Detect autoskillit binaries on PATH outside ~/.local/bin (stale/poisoned installs).""" - check_name = "stale_entry_points" - primary = shutil.which("autoskillit") - if not primary: - return DoctorResult(Severity.OK, check_name, "autoskillit not found on PATH") - - try: - result = subprocess.run( - ["which", "-a", "autoskillit"], - capture_output=True, - text=True, - timeout=5, - ) - all_paths = [p.strip() for p in result.stdout.splitlines() if p.strip()] - except (subprocess.TimeoutExpired, FileNotFoundError, OSError): - all_paths = [primary] - - expected_prefix = Path.home() / ".local" - stale = [p for p in all_paths if not Path(p).is_relative_to(expected_prefix)] - if not stale: - return DoctorResult(Severity.OK, check_name, "No stale autoskillit entry points found") - - stale_list = ", ".join(stale) - return DoctorResult( - Severity.WARNING, - check_name, - f"Found autoskillit entry point(s) outside ~/.local/bin: {stale_list}. " - f"These may be stale editable installs. " - f"Fix: uv tool install --force autoskillit && autoskillit install", - ) - - -def _check_config_layers_for_secrets( - project_dir: Path | None = None, -) -> DoctorResult: - """Check all config.yaml layers for _SECRETS_ONLY_KEYS violations. - - Scans the user-level and project-level config.yaml files for any keys - that belong only in .secrets.yaml. Reports ERROR with exact fix guidance. - """ - from autoskillit.config import ConfigSchemaError, validate_layer_keys - from autoskillit.core import YAMLError, load_yaml - - root = project_dir or Path.cwd() - config_paths = [ - Path.home() / ".autoskillit" / "config.yaml", - root / ".autoskillit" / "config.yaml", - ] - for config_path in config_paths: - if not config_path.is_file(): - continue - try: - data = load_yaml(config_path) or {} - except YAMLError as exc: - return DoctorResult( - severity=Severity.WARNING, - check="config_secrets_placement", - message=f"Could not parse {str(config_path)!r} as YAML: {exc}", - ) - if not isinstance(data, dict): - continue - try: - validate_layer_keys(data, config_path, is_secrets_layer=False) - except ConfigSchemaError as exc: - return DoctorResult( - severity=Severity.ERROR, - check="config_secrets_placement", - message=str(exc), - ) - return DoctorResult( - severity=Severity.OK, - check="config_secrets_placement", - message="No secrets found in config.yaml layers", - ) - - -def _check_source_version_drift(home: Path | None = None) -> DoctorResult: - """Network source-drift check. - - Compares the installed commit SHA against the current HEAD of the branch - the binary was installed from. Uses a network request to get the latest - SHA (with disk-cache TTL fallback). - """ - check_name = "source_version_drift" - _home = home or Path.home() - - try: - from autoskillit.cli._install_info import InstallType, detect_install - from autoskillit.cli._update_checks import resolve_reference_sha - - info = detect_install() - - if info.install_type == InstallType.LOCAL_EDITABLE: - return DoctorResult( - Severity.OK, check_name, "Local editable install — drift check not applicable" - ) - - if info.install_type in (InstallType.UNKNOWN, InstallType.LOCAL_PATH): - return DoctorResult( - Severity.OK, - check_name, - "Not a source-tracked install — drift check not applicable", - ) - - # GIT_VCS: resolve SHA via network (with disk-cache fallback) - ref_sha = resolve_reference_sha(info, _home, network=True) - - if ref_sha is None: - return DoctorResult( - Severity.OK, - check_name, - "Source drift reference SHA unavailable — check network connectivity", - ) - - if info.commit_id == ref_sha: - return DoctorResult(Severity.OK, check_name, "No source drift detected") - - installed_short = (info.commit_id or "unknown")[:8] - ref_short = ref_sha[:8] - return DoctorResult( - Severity.WARNING, - check_name, - f"Source drift: installed={installed_short}, reference={ref_short}. " - f"Run the appropriate install command to update.", - ) - - except Exception: - _log.debug("Source drift check failed", exc_info=True) - return DoctorResult( - Severity.OK, check_name, "Source drift check skipped (unexpected error)" - ) - - -def _check_install_classification() -> DoctorResult: - """Classify the current autoskillit install type via direct_url.json.""" - check_name = "install_classification" - try: - from autoskillit.cli._install_info import InstallType, detect_install - - info = detect_install() - if info.install_type == InstallType.UNKNOWN: - return DoctorResult( - Severity.WARNING, - check_name, - "install type could not be detected from direct_url.json", - ) - commit_short = (info.commit_id or "")[:8] - return DoctorResult( - Severity.OK, - check_name, - f"install_type={info.install_type}, " - f"requested_revision={info.requested_revision}, " - f"commit_id={commit_short}", - ) - except Exception: - _log.debug("Install classification check failed", exc_info=True) - return DoctorResult( - Severity.OK, check_name, "Install classification check skipped (unexpected error)" - ) - - -def _check_update_dismissal_state(home: Path | None = None) -> DoctorResult: - """Report the current update-prompt dismissal state.""" - check_name = "update_dismissal_state" - _home = home or Path.home() - try: - from autoskillit.cli._install_info import detect_install, dismissal_window - from autoskillit.cli._update_checks import _read_dismiss_state - - state = _read_dismiss_state(_home) - entry = state.get("update_prompt") - if not isinstance(entry, dict) or "dismissed_at" not in entry: - return DoctorResult(Severity.OK, check_name, "No active dismissal") - - from datetime import datetime - - info = detect_install() - window = dismissal_window(info) - dismissed_at = datetime.fromisoformat(str(entry["dismissed_at"])) - expiry = (dismissed_at + window).strftime("%Y-%m-%d") - conditions = entry.get("conditions", []) - return DoctorResult( - Severity.OK, - check_name, - f"update_prompt dismissed until {expiry}; conditions={conditions}", - ) - except Exception: - _log.debug("Update dismissal state check failed", exc_info=True) - return DoctorResult( - Severity.OK, check_name, "Update dismissal state check skipped (unexpected error)" - ) - - -def _check_quota_cache_schema(cache_path: Path | None = None) -> DoctorResult: - """Check the quota cache file for schema version drift.""" - check_name = "quota_cache_schema" - path = cache_path or (Path.home() / ".claude" / "autoskillit_quota_cache.json") - if not path.exists(): - return DoctorResult(Severity.OK, check_name, "No quota cache present.") - try: - raw = json.loads(path.read_text()) - except Exception as exc: - _log.warning("quota_cache_parse_error", path=str(path), exc_info=True) - return DoctorResult( - Severity.WARNING, - check_name, - f"Quota cache at {path} could not be parsed: {type(exc).__name__}.", - ) - observed = raw.get("schema_version") if isinstance(raw, dict) else None - if observed == QUOTA_CACHE_SCHEMA_VERSION: - return DoctorResult( - Severity.OK, - check_name, - f"Quota cache schema v{QUOTA_CACHE_SCHEMA_VERSION} at {path}.", - ) - return DoctorResult( - Severity.WARNING, - check_name, - f"Quota cache schema drift at {path}: observed={observed!r}, " - f"expected={QUOTA_CACHE_SCHEMA_VERSION}.", - ) - - -def _check_claude_process_state_breakdown() -> DoctorResult: - """Check current D-state and CPU usage of claude processes via ps.""" - check_name = "claude_process_state" - try: - result = subprocess.run( - ["ps", "-axo", "pid,state,pcpu,comm"], - capture_output=True, - text=True, - timeout=5, - ) - except (FileNotFoundError, subprocess.TimeoutExpired, OSError) as exc: - return DoctorResult( - Severity.OK, - check_name, - f"ps unavailable ({type(exc).__name__}); skipping claude process check", - ) - - if result.returncode != 0: - return DoctorResult( - Severity.OK, - check_name, - f"ps exited {result.returncode}; skipping claude process check", - ) - - claude_rows: list[tuple[int, str, float]] = [] - for line in result.stdout.splitlines()[1:]: - parts = line.split(maxsplit=3) - if len(parts) < 4: - continue - comm = parts[3] - if comm != "claude": - continue - try: - claude_rows.append((int(parts[0]), parts[1], float(parts[2]))) - except ValueError: - continue - - if not claude_rows: - return DoctorResult(Severity.OK, check_name, "No claude processes running") - - breakdown: dict[str, int] = {} - for _, state, _ in claude_rows: - breakdown[state] = breakdown.get(state, 0) + 1 - - summary = ", ".join(f"{s}={c}" for s, c in sorted(breakdown.items())) - - d_rows = [f"pid={pid} pcpu={pcpu}" for pid, state, pcpu in claude_rows if state == "D"] - if d_rows: - return DoctorResult( - Severity.WARNING, - check_name, - f"claude processes in D state: {', '.join(d_rows)} (breakdown: {summary})", - ) - - return DoctorResult( - Severity.OK, - check_name, - f"claude process state breakdown: {summary}", - ) - - -def _check_plugin_cache_exists( - cache_dir: Path | None = None, -) -> DoctorResult: - """Check that the plugin cache directory exists.""" - from autoskillit.cli._install_info import InstallType, detect_install - - info = detect_install() - if info.install_type == InstallType.LOCAL_EDITABLE: - return DoctorResult( - Severity.OK, - "plugin_cache_exists", - "Plugin cache check skipped (editable dev install)", - ) - - _cache_dir = cache_dir or ( - Path.home() / ".claude" / "plugins" / "cache" / "autoskillit-local" / "autoskillit" - ) - if _cache_dir.is_dir(): - return DoctorResult( - Severity.OK, - "plugin_cache_exists", - "Plugin cache directory exists", - ) - return DoctorResult( - Severity.WARNING, - "plugin_cache_exists", - f"Plugin cache directory missing: {_cache_dir}. Run `autoskillit install` to recreate it.", - ) - - -def _check_installed_plugins_entry( - plugins_json_path: Path | None = None, -) -> DoctorResult: - """Check that installed_plugins.json contains the autoskillit entry.""" - from autoskillit.cli._installed_plugins import InstalledPluginsFile - - store = InstalledPluginsFile(plugins_json_path) - if not store.path.exists(): - return DoctorResult( - Severity.WARNING, - "installed_plugins_entry", - "installed_plugins.json not found. Run `autoskillit install`.", - ) - if store.contains("autoskillit@autoskillit-local"): - return DoctorResult( - Severity.OK, - "installed_plugins_entry", - "autoskillit entry present in installed_plugins.json", - ) - return DoctorResult( - Severity.WARNING, - "installed_plugins_entry", - "autoskillit entry missing from installed_plugins.json. Run `autoskillit install` to fix.", - ) - - -def run_doctor(*, output_json: bool = False) -> None: - """Check project setup for common issues.""" - results: list[DoctorResult] = [] - - # Check 1: Stale MCP servers — dead binaries or nonexistent paths - stale_servers: list[str] = [] - _stale_parse_error = False - claude_json = Path.home() / ".claude.json" - if claude_json.is_file(): - try: - data = json.loads(claude_json.read_text()) - except (json.JSONDecodeError, OSError) as exc: - results.append( - DoctorResult( - Severity.ERROR, - "stale_mcp_servers", - f"~/.claude.json could not be parsed: {exc}", - ) - ) - _stale_parse_error = True - data = {} - servers = data.get("mcpServers", {}) - for name, entry in servers.items(): - cmd = entry.get("command", "") - if not cmd: - continue - cmd_path = Path(cmd) - if cmd_path.is_absolute() and not cmd_path.exists(): - stale_servers.append( - f"MCP server '{name}' has dead command path: {cmd}. " - f"Remove with: claude mcp remove --scope user {name}" - ) - elif not cmd_path.is_absolute() and shutil.which(cmd) is None: - stale_servers.append( - f"MCP server '{name}' command not found: {cmd}. " - f"Remove with: claude mcp remove --scope user {name}" - ) - if not _stale_parse_error: - if stale_servers: - for msg in stale_servers: - results.append(DoctorResult(Severity.ERROR, "stale_mcp_servers", msg)) - else: - results.append( - DoctorResult(Severity.OK, "stale_mcp_servers", "No stale MCP servers detected") - ) - - # Check 2: MCP server registered in ~/.claude.json or via plugin - results.append(_check_mcp_server_registered(claude_json_path=Path.home() / ".claude.json")) - - # Check 2b: Dual MCP registration — direct entry and marketplace plugin both present - results.append(_check_dual_mcp_registration()) - - # Check 2c: Plugin cache directory exists - results.append(_check_plugin_cache_exists()) - - # Check 2d: installed_plugins.json has autoskillit entry - results.append(_check_installed_plugins_entry()) - - # Check 3: autoskillit command on PATH - if shutil.which("autoskillit") is None: - results.append( - DoctorResult( - Severity.WARNING, - "autoskillit_on_path", - "'autoskillit' command not found on PATH.", - ) - ) - else: - results.append( - DoctorResult(Severity.OK, "autoskillit_on_path", "autoskillit command found on PATH") - ) - - # Check 4: Config exists - if not (Path.cwd() / ".autoskillit" / "config.yaml").is_file(): - results.append( - DoctorResult( - Severity.WARNING, - "project_config", - "No project config found. Run: autoskillit init", - ) - ) - else: - results.append(DoctorResult(Severity.OK, "project_config", "Project config exists")) - - # Check 4b: Config secrets placement - results.append(_check_config_layers_for_secrets()) - - # Check 5: Version consistency — package version must match plugin.json - import importlib.metadata - import importlib.resources as _ir - - pkg_version = importlib.metadata.version("autoskillit") - plugin_json_path = Path(str(_ir.files("autoskillit"))) / ".claude-plugin" / "plugin.json" - try: - plugin_version = json.loads(plugin_json_path.read_text()).get("version") - except (json.JSONDecodeError, OSError): - plugin_version = None - if plugin_version == pkg_version: - results.append( - DoctorResult( - Severity.OK, - "version_consistency", - f"Version {pkg_version} installed", - ) - ) - else: - results.append( - DoctorResult( - Severity.WARNING, - "version_consistency", - f"Package version {pkg_version} does not match " - f"plugin.json {plugin_version!r}. Reinstall autoskillit to fix.", - ) - ) - - # Check 6: Hook executability — validates deployed scripts for all event types - results.append(_check_hook_health(_claude_settings_path("user"))) - - # Check 7: Hook registration in settings.json - results.append(_check_hook_registration(_claude_settings_path("user"))) - - # Check 7b: Hook registry drift (multi-scope) - from autoskillit.hook_registry import iter_all_scope_paths - - for scope_label, settings_path in iter_all_scope_paths(Path.cwd()): - results.append(_check_hook_registry_drift(settings_path, scope_label=scope_label)) - - # Check 8: Script version health - from autoskillit import __version__ - from autoskillit.core import RecipeSource - from autoskillit.migration import FailureStore, default_store_path - from autoskillit.recipe import list_recipes as _list_all_recipes - - _all_result = _list_all_recipes(Path.cwd()) - scripts_result_items = [r for r in _all_result.items if r.source == RecipeSource.PROJECT] - if not scripts_result_items: - results.append( - DoctorResult( - Severity.OK, - "script_version_health", - "No pipeline scripts found", - ) - ) - else: - from packaging.version import Version - - failure_store = FailureStore(default_store_path(Path.cwd())) - known_failures = failure_store.load() - - failed_migrations: list[str] = [] - outdated: list[str] = [] - for script in scripts_result_items: - if script.name in known_failures: - f = known_failures[script.name] - failed_migrations.append( - f"{script.name} (failed after {f.retries_attempted} retries)" - ) - elif script.version is None or Version(script.version) < Version(__version__): - outdated.append(script.name) - - if failed_migrations: - results.append( - DoctorResult( - Severity.ERROR, - "script_version_health", - "Migration failed — manual intervention required: " - + ", ".join(failed_migrations), - ) - ) - elif outdated: - results.append( - DoctorResult( - Severity.WARNING, - "script_version_health", - "Outdated recipes: " - + ", ".join(outdated) - + ". Will be auto-migrated on next load.", - ) - ) - else: - results.append( - DoctorResult( - Severity.OK, - "script_version_health", - "All recipes up to date", - ) - ) - - # Check 9: gitignore completeness - results.append(_check_gitignore_completeness(Path.cwd())) - - # Check 10: Secret scanning hook - results.append(_check_secret_scanning_hook(Path.cwd())) - - # Check 11: Editable install source directory still exists - results.append(_check_editable_install_source_exists()) - - # Check 12: No stale autoskillit entry points outside ~/.local/bin - results.append(_check_stale_entry_points()) - - # Check 13: Source version drift (network, with disk-cache TTL fallback) - results.append(_check_source_version_drift()) - - # Check 14: Quota cache schema version - results.append(_check_quota_cache_schema()) - - # Check 15: claude process state breakdown - results.append(_check_claude_process_state_breakdown()) - - # Check 16: Install classification from direct_url.json - results.append(_check_install_classification()) - - # Check 17: Update-prompt dismissal state - results.append(_check_update_dismissal_state()) - - # Output - if output_json: - print( - json.dumps( - { - "results": [ - {"severity": r.severity, "check": r.check, "message": r.message} - for r in results - ] - }, - indent=2, - ) - ) - else: - has_problems = any(r.severity != Severity.OK for r in results) - if has_problems: - for r in results: - if r.severity != Severity.OK: - print(f"{r.severity.upper()}: {r.message}") - else: - for r in results: - print(f"{r.severity}: {r.message}") diff --git a/src/autoskillit/cli/_features.py b/src/autoskillit/cli/_features.py new file mode 100644 index 0000000000..587b66857c --- /dev/null +++ b/src/autoskillit/cli/_features.py @@ -0,0 +1,112 @@ +"""Features subapp: list and inspect feature gate state.""" + +from __future__ import annotations + +import sys +from pathlib import Path + +from cyclopts import App + +features_app = App(name="features", help="Feature gate inspection.") + + +@features_app.command(name="list") +def features_list() -> None: + """List all registered features with their effective state.""" + from autoskillit.config import load_config + from autoskillit.core import ( + FEATURE_REGISTRY, + FeatureLifecycle, + TerminalColumn, + _render_terminal_table, + is_feature_enabled, + ) + + cfg = load_config(Path.cwd()) + + columns = [ + TerminalColumn("FEATURE", 20, "<"), + TerminalColumn("TIER", 5, ">"), + TerminalColumn("LIFECYCLE", 14, "<"), + TerminalColumn("DEFAULT", 9, "<"), + TerminalColumn("EFFECTIVE", 18, "<"), + TerminalColumn("SOURCE", 14, "<"), + ] + + rows = [] + for name, defn in sorted(FEATURE_REGISTRY.items()): + effective = is_feature_enabled( + name, cfg.features, experimental_enabled=cfg.experimental_enabled + ) + if ( + defn.lifecycle == FeatureLifecycle.EXPERIMENTAL + and effective + and name not in cfg.features + ): + effective_str = "on (experimental)" + source = "experimental" + elif name in cfg.features and not effective: + effective_str = "off (override)" + source = "config" + elif name in cfg.features: + effective_str = str(effective).lower() + source = "config" + else: + effective_str = str(effective).lower() + source = "default" + rows.append( + ( + name, + str(defn.tier), + str(defn.lifecycle), + str(defn.default_enabled).lower(), + effective_str, + source, + ) + ) + + print(_render_terminal_table(columns, rows)) + + +@features_app.command(name="status") +def features_status(name: str) -> None: + """Show detailed state for a single feature.""" + from autoskillit.config import load_config + from autoskillit.core import FEATURE_REGISTRY, FeatureLifecycle, is_feature_enabled + + if name not in FEATURE_REGISTRY: + known = sorted(FEATURE_REGISTRY.keys()) + print( + f"Unknown feature: {name!r}\nKnown features: {', '.join(known)}", + file=sys.stderr, + ) + raise SystemExit(1) + + defn = FEATURE_REGISTRY[name] + cfg = load_config(Path.cwd()) + effective = is_feature_enabled( + name, cfg.features, experimental_enabled=cfg.experimental_enabled + ) + + if defn.lifecycle == FeatureLifecycle.EXPERIMENTAL and effective and name not in cfg.features: + source = "on by experimental_enabled blanket" + elif name in cfg.features: + source = "overridden by config" + else: + source = "default" + enabled_str = f"{'true' if effective else 'false'} ({source})" + + tool_tags = ", ".join(sorted(defn.tool_tags)) if defn.tool_tags else "(none)" + skill_cats = ", ".join(sorted(defn.skill_categories)) if defn.skill_categories else "(none)" + depends = ", ".join(sorted(defn.depends_on)) if defn.depends_on else "(none)" + + print(f"Feature: {name}") + print(f" Lifecycle: {defn.lifecycle}") + print(f" Tier: {defn.tier}") + print(f" Enabled: {enabled_str}") + print(f" Package: {defn.import_package or '(none)'}") + print(f" Tool tags: {tool_tags}") + print(f" Skill cats: {skill_cats}") + print(f" Since: {f'v{defn.since_version}' if defn.since_version else '(none)'}") + print(f" Depends on: {depends}") + print(f" Sunset date: {defn.sunset_date or '(none)'}") diff --git a/src/autoskillit/cli/_hooks.py b/src/autoskillit/cli/_hooks.py index d8ec18992e..5f0e3db549 100644 --- a/src/autoskillit/cli/_hooks.py +++ b/src/autoskillit/cli/_hooks.py @@ -5,8 +5,9 @@ import json from pathlib import Path -from autoskillit.core import atomic_write, pkg_root +from autoskillit.core import atomic_write, is_git_worktree, pkg_root from autoskillit.hook_registry import ( + _build_hook_command, _build_hook_entry, _claude_settings_path, # noqa: F401 — re-exported; cli/__init__ + _stale_check + _init_helpers import from here _load_settings_data, @@ -56,23 +57,32 @@ def sync_hooks_to_settings(settings_path: Path) -> None: """ from autoskillit.hook_registry import HOOK_REGISTRY_HASH - hooks_dir = pkg_root() / "hooks" + root = pkg_root() + if is_git_worktree(root): + raise RuntimeError( + f"Refusing to sync hooks: pkg_root() resolves to a git linked worktree " + f"({root}). Hook paths written from a transient worktree would become " + f"dangling after worktree deletion. Use 'task install-worktree' instead " + f"of 'autoskillit init' when working in a worktree." + ) + hooks_dir = root / "hooks" data = _load_settings_data(settings_path) + # Consolidate HookDef entries sharing the same (event_type, matcher) into a + # single settings.json entry so Claude Code sees no duplicate matchers. + groups: dict[tuple[str, str], dict] = {} for hook_def in HOOK_REGISTRY: - event_list: list[dict] = data.setdefault("hooks", {}).setdefault(hook_def.event_type, []) + key = (hook_def.event_type, hook_def.matcher) hooks_list = [ - { - "type": "command", - "command": f"python3 {hooks_dir / script}", - **( - {"timeout": hook_def.timeout_seconds} - if hook_def.timeout_seconds is not None - else {} - ), - } + _build_hook_command(hooks_dir, script, hook_def.timeout_seconds) for script in hook_def.scripts ] - event_list.append(_build_hook_entry(hook_def, hooks_list)) + if key not in groups: + groups[key] = _build_hook_entry(hook_def, hooks_list) + else: + groups[key]["hooks"].extend(hooks_list) + for (event_type, _), entry in groups.items(): + event_list: list[dict] = data.setdefault("hooks", {}).setdefault(event_type, []) + event_list.append(entry) data["_autoskillit_registry_hash"] = HOOK_REGISTRY_HASH _write_settings_data(settings_path, data) diff --git a/src/autoskillit/cli/_init_helpers.py b/src/autoskillit/cli/_init_helpers.py index 7763df0500..6245667ad8 100644 --- a/src/autoskillit/cli/_init_helpers.py +++ b/src/autoskillit/cli/_init_helpers.py @@ -11,7 +11,6 @@ from autoskillit.config import write_config_layer from autoskillit.core import YAMLError, atomic_write, dump_yaml_str, get_logger, load_yaml -from autoskillit.recipe import list_recipes logger = get_logger(__name__) @@ -23,7 +22,7 @@ class _ScanResult(NamedTuple): def _colors() -> tuple[str, str, str, str, str, str]: """Return (Bold, Cyan, Dim, Green, Yellow, Reset) respecting NO_COLOR.""" - from autoskillit.cli._ansi import supports_color + from autoskillit.cli.ui._ansi import supports_color c = supports_color() return ( @@ -75,7 +74,8 @@ def _require_interactive_stdin(command_name: str) -> None: def _prompt_recipe_choice() -> str: - from autoskillit.cli._timed_input import timed_prompt + from autoskillit.cli.ui._timed_input import timed_prompt + from autoskillit.recipe import list_recipes available = list_recipes(Path.cwd()).items if not available: @@ -88,7 +88,7 @@ def _prompt_recipe_choice() -> str: def _prompt_test_command() -> list[str]: - from autoskillit.cli._timed_input import timed_prompt + from autoskillit.cli.ui._timed_input import timed_prompt default = "task test-all" answer = timed_prompt( @@ -122,7 +122,7 @@ def _prompt_github_repo() -> str | None: _B, _C, _D, _G, _Y, _R = _colors() detected = _detect_github_repo() - from autoskillit.cli._timed_input import timed_prompt + from autoskillit.cli.ui._timed_input import timed_prompt if detected: print(f"\n {_Y}GitHub repo{_R} {_G}{detected}{_R} {_D}(detected from git remote){_R}") @@ -264,7 +264,7 @@ def _check_secret_scanning(project_dir: Path) -> _ScanResult: " Recommended: add gitleaks to .pre-commit-config.yaml\n" " before proceeding.\n" ) - from autoskillit.cli._timed_input import timed_prompt + from autoskillit.cli.ui._timed_input import timed_prompt print(" To bypass, type exactly:\n") print(f" {_D}{_SECRET_SCAN_BYPASS_PHRASE}{_R}\n") @@ -320,7 +320,7 @@ def _generate_config_yaml(test_command: list[str]) -> str: # # implement_gate: # marker: "Dry-walkthrough verified = TRUE" -# skill_names: ["/autoskillit:implement-worktree", "/autoskillit:implement-worktree-no-merge"] +# skill_names: ["/implement-worktree", "/implement-worktree-no-merge"] # # run_skill: # timeout: 7200 @@ -439,6 +439,15 @@ def _register_all(scope: str, project_dir: Path) -> None: "`cd` to a different directory first." ) + if _core_paths.is_git_worktree(_core_paths.pkg_root()): + from autoskillit.cli.app import CliError + + raise CliError( + "Refusing to run `autoskillit init` from a git linked worktree — " + "hook paths would be written from a transient pkg_root(). " + "Use `task install-worktree` instead." + ) + # Cross-scope sweep: evict stale hooks from all scopes before writing canonical entries. sweep_all_scopes_for_orphans(project_dir) diff --git a/src/autoskillit/cli/_install_info.py b/src/autoskillit/cli/_install_info.py index 52027ec8be..5ab0b57d09 100644 --- a/src/autoskillit/cli/_install_info.py +++ b/src/autoskillit/cli/_install_info.py @@ -7,18 +7,18 @@ from __future__ import annotations -import json -import re from dataclasses import dataclass from datetime import timedelta from enum import StrEnum from pathlib import Path -from autoskillit.core import get_logger +from autoskillit.core import _is_stable_track, get_logger, parse_direct_url logger = get_logger(__name__) -_INSTALL_FROM_INTEGRATION = "git+https://github.com/TalonT-Org/AutoSkillit.git@integration" +_INSTALL_FROM_DEVELOP = "git+https://github.com/TalonT-Org/AutoSkillit.git@develop" +_STABLE_DISMISS_WINDOW = timedelta(days=7) +_DEV_DISMISS_WINDOW = timedelta(hours=12) class InstallType(StrEnum): @@ -28,7 +28,13 @@ class InstallType(StrEnum): UNKNOWN = "unknown" -@dataclass(frozen=True) +class InstallTrack(StrEnum): + STABLE = "stable" + DEV = "dev" + LOCAL = "local" + + +@dataclass(frozen=True, slots=True) class InstallInfo: install_type: InstallType commit_id: str | None @@ -45,28 +51,17 @@ def detect_install() -> InstallInfo: """ _unknown = InstallInfo(InstallType.UNKNOWN, None, None, None, None) try: - import importlib.metadata - - dist = importlib.metadata.Distribution.from_name("autoskillit") - raw = dist.read_text("direct_url.json") - if not raw: - return _unknown - - data = json.loads(raw) - - vcs_info = data.get("vcs_info", {}) - if isinstance(vcs_info, dict) and vcs_info.get("vcs") == "git": + info = parse_direct_url() + url = info["url"] or "" + if info["install_type"] == "git-vcs": return InstallInfo( install_type=InstallType.GIT_VCS, - commit_id=vcs_info.get("commit_id") or None, - requested_revision=vcs_info.get("requested_revision") or None, - url=data.get("url") or None, + commit_id=info["commit_id"], + requested_revision=info["requested_revision"], + url=url or None, editable_source=None, ) - - dir_info = data.get("dir_info", {}) - url = data.get("url", "") - if isinstance(dir_info, dict) and dir_info.get("editable") is True: + if info["install_type"] == "local-editable": if isinstance(url, str) and url.startswith("file://"): src_path = url[len("file://") :] return InstallInfo( @@ -76,40 +71,41 @@ def detect_install() -> InstallInfo: url=url, editable_source=Path(src_path), ) - - if isinstance(url, str) and url.startswith("file://"): + if info["install_type"] == "local-path": return InstallInfo( install_type=InstallType.LOCAL_PATH, commit_id=None, requested_revision=None, - url=url, + url=url or None, editable_source=None, ) - return _unknown - except Exception: logger.debug("install classification failed", exc_info=True) return _unknown -def _is_release_tag(rev: str) -> bool: - """Return True if ``rev`` looks like a version tag (e.g. 'v0.7.75', '0.7.75').""" - return bool(re.fullmatch(r"v?\d+(\.\d+)*", rev)) +def classify_track(info: InstallInfo) -> InstallTrack: + if info.install_type in (InstallType.LOCAL_EDITABLE, InstallType.LOCAL_PATH): + return InstallTrack.LOCAL + rev = info.requested_revision or "" + if _is_stable_track(rev): + return InstallTrack.STABLE + return InstallTrack.DEV def comparison_branch(info: InstallInfo) -> str | None: """Return the GitHub branch/tag to compare for update availability. - - ``stable`` / ``main`` / release-tag / ``UNKNOWN`` → ``"releases/latest"`` - - ``integration`` → ``"integration"`` + - stable / main / release-tag / UNKNOWN → ``"releases/latest"`` + - any other GIT_VCS revision (dev-track) → ``"develop"`` - ``LOCAL_EDITABLE`` / ``LOCAL_PATH`` → ``None`` (not applicable) """ - if info.install_type in (InstallType.LOCAL_EDITABLE, InstallType.LOCAL_PATH): + track = classify_track(info) + if track == InstallTrack.LOCAL: return None - rev = info.requested_revision or "" - if rev == "integration": - return "integration" + if track == InstallTrack.DEV: + return "develop" return "releases/latest" @@ -118,28 +114,29 @@ def dismissal_window(info: InstallInfo) -> timedelta: Branch-aware windows: - - ``stable`` / ``main`` / release-tag / ``UNKNOWN`` → ``timedelta(days=7)`` - - ``integration`` / ``LOCAL_EDITABLE`` → ``timedelta(hours=12)`` + - stable / main / release-tag / UNKNOWN → ``timedelta(days=7)`` + - dev-track / LOCAL → ``timedelta(hours=12)`` """ - rev = info.requested_revision or "" - if rev == "integration" or info.install_type == InstallType.LOCAL_EDITABLE: - return timedelta(hours=12) - return timedelta(days=7) + track = classify_track(info) + # LOCAL_EDITABLE is reachable only via AUTOSKILLIT_FORCE_UPDATE_CHECK; not dead code. + if track in (InstallTrack.DEV, InstallTrack.LOCAL): + return _DEV_DISMISS_WINDOW + return _STABLE_DISMISS_WINDOW def upgrade_command(info: InstallInfo) -> list[str] | None: """Return the subprocess command to upgrade autoskillit for this install. - - ``stable`` / ``main`` / release-tag → ``["uv", "tool", "upgrade", "autoskillit"]`` - - ``integration`` → ``["uv", "tool", "install", "--force", ]`` + - stable / main / release-tag → ``["uv", "tool", "upgrade", "autoskillit"]`` + - dev-track → ``["uv", "tool", "install", "--force", ]`` - ``LOCAL_EDITABLE`` → ``["uv", "pip", "install", "-e", str(info.editable_source)]`` - ``UNKNOWN`` / ``LOCAL_PATH`` → ``None`` """ - if info.install_type == InstallType.GIT_VCS: - rev = info.requested_revision or "" - if rev == "integration": - return ["uv", "tool", "install", "--force", _INSTALL_FROM_INTEGRATION] - return ["uv", "tool", "upgrade", "autoskillit"] if info.install_type == InstallType.LOCAL_EDITABLE and info.editable_source is not None: return ["uv", "pip", "install", "-e", str(info.editable_source)] - return None + if info.install_type != InstallType.GIT_VCS: + return None + track = classify_track(info) + if track == InstallTrack.DEV: + return ["uv", "tool", "install", "--force", _INSTALL_FROM_DEVELOP] + return ["uv", "tool", "upgrade", "autoskillit"] diff --git a/src/autoskillit/cli/_installed_plugins.py b/src/autoskillit/cli/_installed_plugins.py index 205e18740d..2467be7c19 100644 --- a/src/autoskillit/cli/_installed_plugins.py +++ b/src/autoskillit/cli/_installed_plugins.py @@ -13,7 +13,9 @@ from pathlib import Path from typing import Any -from autoskillit.core import atomic_write +from autoskillit.core import atomic_write, get_logger + +logger = get_logger(__name__) def _default_path() -> Path: @@ -38,7 +40,11 @@ def _read(self) -> dict[str, Any]: return {} try: return json.loads(self._path.read_text()) - except (json.JSONDecodeError, OSError): + except json.JSONDecodeError as exc: + logger.warning("installed_plugins.json is corrupt (%s): %s", self._path, exc) + return {} + except OSError as exc: + logger.warning("Could not read installed_plugins.json (%s): %s", self._path, exc) return {} def get_plugins(self) -> dict[str, Any]: diff --git a/src/autoskillit/cli/_marketplace.py b/src/autoskillit/cli/_marketplace.py index 2886f1a576..8ad55e2bd6 100644 --- a/src/autoskillit/cli/_marketplace.py +++ b/src/autoskillit/cli/_marketplace.py @@ -4,25 +4,40 @@ import json import os -import re import shutil import subprocess import sys from pathlib import Path +import regex as re + import autoskillit.cli._hooks as _hooks_mod from autoskillit.cli._hooks import ( sweep_all_scopes_for_orphans, sync_hooks_to_settings, ) from autoskillit.cli._init_helpers import _user_claude_json_path, evict_direct_mcp_entry -from autoskillit.core import atomic_write, is_git_worktree, pkg_root +from autoskillit.core import ( + DIRECT_INSTALL_CACHE_SUBDIR, + atomic_write, + get_logger, + is_git_worktree, + pkg_root, +) from autoskillit.hooks import generate_hooks_json +logger = get_logger(__name__) + _VALID_SCOPES = {"user", "project", "local"} _MARKETPLACE_NAME = "autoskillit-local" +def _plugin_cache_dir() -> Path: + return ( + Path.home() / ".claude" / "plugins" / "cache" / DIRECT_INSTALL_CACHE_SUBDIR / "autoskillit" + ) + + def _clear_plugin_cache() -> None: """Remove the cached plugin snapshot and installed_plugins.json entry. @@ -32,16 +47,16 @@ def _clear_plugin_cache() -> None: the cache beforehand ensures a single ``autoskillit install`` is always sufficient. """ - cache_dir = Path.home() / ".claude" / "plugins" / "cache" / _MARKETPLACE_NAME / "autoskillit" - if cache_dir.is_dir(): - shutil.rmtree(cache_dir) + _d = _plugin_cache_dir() + if _d.is_dir(): + from autoskillit import __version__ as _new_version + from autoskillit.core import _retire_old_versions - from autoskillit.cli._installed_plugins import InstalledPluginsFile + _retire_old_versions(_d, _new_version) + else: + from autoskillit.core import sweep_retiring_cache - try: - InstalledPluginsFile().remove(f"autoskillit@{_MARKETPLACE_NAME}") - except OSError: - pass # non-fatal — install will proceed regardless + sweep_retiring_cache() def _ensure_marketplace() -> Path: @@ -154,38 +169,53 @@ def install(*, scope: str = "user") -> bool: sys.exit(1) _ensure_workspace_ready() - _clear_plugin_cache() - - # Regenerate hooks.json from the canonical registry with absolute paths - hooks_json_path = pkg_root() / "hooks" / "hooks.json" - atomic_write(hooks_json_path, json.dumps(generate_hooks_json(), indent=2) + "\n") - - # Register the marketplace (idempotent) - result = subprocess.run( - ["claude", "plugin", "marketplace", "add", str(marketplace_dir)], - capture_output=True, - text=True, - stdin=subprocess.DEVNULL, - timeout=30, - ) - if result.returncode != 0: - print(f"Failed to register marketplace: {result.stderr.strip()}") - sys.exit(1) - print("Marketplace registered.") - - # Install the plugin - result = subprocess.run( - ["claude", "plugin", "install", plugin_ref, "--scope", scope], - capture_output=True, - text=True, - stdin=subprocess.DEVNULL, - timeout=30, - ) - if result.returncode != 0: - print(f"Failed to install plugin: {result.stderr.strip()}") - sys.exit(1) + + _cache = _plugin_cache_dir() + + from autoskillit.core import _InstallLock + + with _InstallLock(): + _clear_plugin_cache() + + # Regenerate hooks.json from the canonical registry with absolute paths + hooks_json_path = pkg_root() / "hooks" / "hooks.json" + atomic_write(hooks_json_path, json.dumps(generate_hooks_json(), indent=2) + "\n") + + # Register the marketplace (idempotent) + result = subprocess.run( + ["claude", "plugin", "marketplace", "add", str(marketplace_dir)], + capture_output=True, + text=True, + stdin=subprocess.DEVNULL, + timeout=30, + ) + if result.returncode != 0: + print(f"Failed to register marketplace: {result.stderr.strip()}") + sys.exit(1) + print("Marketplace registered.") + + # Install the plugin + result = subprocess.run( + ["claude", "plugin", "install", plugin_ref, "--scope", scope], + capture_output=True, + text=True, + stdin=subprocess.DEVNULL, + timeout=30, + ) + if result.returncode != 0: + print(f"Failed to install plugin: {result.stderr.strip()}") + sys.exit(1) print(f"Plugin installed: {plugin_ref} (scope: {scope})") + from autoskillit.hook_registry import validate_plugin_cache_hooks + + post_install_broken = validate_plugin_cache_hooks(cache_dir=_cache) + if post_install_broken: + logger.warning( + "broken_hook_paths_after_install", + count=len(post_install_broken), + paths=list(post_install_broken), + ) if evict_direct_mcp_entry(_user_claude_json_path()): print("Removed stale direct MCP entry from ~/.claude.json") # Cross-scope sweep: evict orphaned autoskillit hooks from ALL scopes before @@ -193,6 +223,9 @@ def install(*, scope: str = "user") -> bool: sweep_all_scopes_for_orphans(Path.cwd()) settings_path = _hooks_mod._claude_settings_path(scope) sync_hooks_to_settings(settings_path) + from autoskillit.cli.update._update_checks import invalidate_fetch_cache + + invalidate_fetch_cache(Path.home()) return True diff --git a/src/autoskillit/cli/_mcp_names.py b/src/autoskillit/cli/_mcp_names.py index e8fa6342e8..a8c7989111 100644 --- a/src/autoskillit/cli/_mcp_names.py +++ b/src/autoskillit/cli/_mcp_names.py @@ -1,46 +1,23 @@ -"""MCP tool name prefix detection for CLI prompt builders. +"""MCP tool name prefix detection and resolution for the CLI. -Detects whether autoskillit is marketplace-installed or running under -direct --plugin-dir only, and derives the correct fully-qualified MCP -tool name prefix. Detection is pure Python I/O — no LLM, no subprocess, -no network calls. +Re-exports core detection primitives and provides resolve_tool_name for +constructing fully-qualified tool names within CLI prompt builders. """ from __future__ import annotations -import json -from pathlib import Path - -# The key written to installed_plugins.json by `autoskillit install` -_AUTOSKILLIT_PLUGIN_KEY = "autoskillit@autoskillit-local" - -# Single source of truth for both known prefix forms -DIRECT_PREFIX = "mcp__autoskillit__" -MARKETPLACE_PREFIX = "mcp__plugin_autoskillit_autoskillit__" - - -def _installed_plugins_path() -> Path: - """Return the path to Claude Code's installed plugins registry.""" - return Path.home() / ".claude" / "plugins" / "installed_plugins.json" - - -def detect_autoskillit_mcp_prefix() -> str: - """Return the MCP prefix that autoskillit tools will use in a spawned session. - - Reads ~/.claude/plugins/installed_plugins.json to determine whether - autoskillit is marketplace-installed. When it is, the marketplace - prefix takes precedence even when --plugin-dir is also passed. - - Falls back to DIRECT_PREFIX if the file is absent, unreadable, or - does not contain the autoskillit key. - """ - try: - data = json.loads(_installed_plugins_path().read_text()) - if _AUTOSKILLIT_PLUGIN_KEY in data.get("plugins", {}): - return MARKETPLACE_PREFIX - except (OSError, json.JSONDecodeError, AttributeError, TypeError): - pass - return DIRECT_PREFIX +from autoskillit.core import ( + DIRECT_PREFIX, + MARKETPLACE_PREFIX, + detect_autoskillit_mcp_prefix, +) + +__all__ = [ + "DIRECT_PREFIX", + "MARKETPLACE_PREFIX", + "detect_autoskillit_mcp_prefix", + "resolve_tool_name", +] def resolve_tool_name(short_name: str, prefix: str | None = None) -> str: diff --git a/src/autoskillit/cli/_onboarding.py b/src/autoskillit/cli/_onboarding.py index 73ebe8badd..4e9264bad8 100644 --- a/src/autoskillit/cli/_onboarding.py +++ b/src/autoskillit/cli/_onboarding.py @@ -2,43 +2,26 @@ from __future__ import annotations -import json -import subprocess as _sp -from concurrent.futures import ThreadPoolExecutor -from dataclasses import dataclass, field from pathlib import Path -from typing import Any -from autoskillit.cli._init_helpers import _KNOWN_SCANNERS from autoskillit.core import get_logger -_log = get_logger(__name__) - - -@dataclass -class OnboardingIntel: - scanner_found: str | None = None - build_tools: list[str] = field(default_factory=list) - github_issues: list[str] = field(default_factory=list) +logger = get_logger(__name__) def is_first_run(project_dir: Path) -> bool: """Return True when the project is initialized but has never been onboarded.""" - # 1. autoskillit init must have run if not (project_dir / ".autoskillit" / "config.yaml").exists(): return False - # 2. onboarding already completed if (project_dir / ".autoskillit" / ".onboarded").exists(): return False - # 3. project already has local recipes (beyond the fresh state) recipes_dir = project_dir / ".autoskillit" / "recipes" if recipes_dir.exists(): try: if any(recipes_dir.iterdir()): return False except OSError: - _log.debug("Could not list recipes dir %s", recipes_dir, exc_info=True) - # 4. project already has skill overrides (already customized) + logger.debug("Could not list recipes dir %s", recipes_dir, exc_info=True) from autoskillit.workspace import detect_project_local_overrides if detect_project_local_overrides(project_dir): @@ -55,94 +38,9 @@ def mark_onboarded(project_dir: Path) -> None: atomic_write(marker, "") -_KNOWN_BUILD_FILES: dict[str, str] = { - "Taskfile.yml": "Taskfile", - "Taskfile.yaml": "Taskfile", - "Makefile": "Makefile", - "package.json": "npm/yarn", - "pyproject.toml": "uv/pip", -} - - -def _detect_scanner(project_dir: Path) -> str | None: - """Return first known scanner hook-id found in .pre-commit-config.yaml, else None.""" - pre_commit = project_dir / ".pre-commit-config.yaml" - try: - content = pre_commit.read_text(encoding="utf-8") - except (OSError, UnicodeDecodeError): - return None - for scanner in _KNOWN_SCANNERS: - if scanner in content: - return scanner - return None - - -def _detect_build_tools(project_dir: Path) -> list[str]: - """Return names of detected build tools present in project_dir.""" - found: list[str] = [] - seen: set[str] = set() - for filename, label in _KNOWN_BUILD_FILES.items(): - if (project_dir / filename).exists() and label not in seen: - found.append(label) - seen.add(label) - return found - - -def _fetch_good_first_issues(project_dir: Path) -> list[str]: # noqa: ARG001 - """Return up to 3 'good first issue' titles from the GitHub remote. - - Returns [] on any failure (gh not installed, not authenticated, no remote). - """ - try: - result = _sp.run( - [ - "gh", - "issue", - "list", - "--label", - "good first issue", - "--limit", - "3", - "--json", - "number,title", - ], - capture_output=True, - text=True, - timeout=8, - ) - if result.returncode != 0: - return [] - items: Any = json.loads(result.stdout or "[]") - return [f"#{i['number']}: {i['title']}" for i in items if "number" in i and "title" in i] - except Exception: - return [] - - -def gather_intel(project_dir: Path) -> OnboardingIntel: - """Concurrently gather project intelligence (scanner, build tools, issues).""" - with ThreadPoolExecutor(max_workers=3) as ex: - sf = ex.submit(_detect_scanner, project_dir) - bf = ex.submit(_detect_build_tools, project_dir) - gf = ex.submit(_fetch_good_first_issues, project_dir) - try: - scanner = sf.result() - except Exception: - scanner = None - try: - tools = bf.result() - except Exception: - tools = [] - try: - issues = gf.result() - except Exception: - issues = [] - return OnboardingIntel(scanner_found=scanner, build_tools=tools, github_issues=issues) - - def run_onboarding_menu(project_dir: Path, *, color: bool = True) -> str | None: """Display the guided onboarding menu. - Starts background intel gathering before showing the menu. Returns an initial_prompt string for A/B/C/D paths (mark_onboarded called by the caller's finally block), or None for E/decline (mark_onboarded called internally). @@ -152,7 +50,7 @@ def run_onboarding_menu(project_dir: Path, *, color: bool = True) -> str | None: _Y = "\x1b[33m" if color else "" _R = "\x1b[0m" if color else "" - from autoskillit.cli._timed_input import timed_prompt + from autoskillit.cli.ui._timed_input import timed_prompt print(f"\n{_B}It looks like this is your first time using AutoSkillit in this project.{_R}") ans = timed_prompt( @@ -162,10 +60,6 @@ def run_onboarding_menu(project_dir: Path, *, color: bool = True) -> str | None: mark_onboarded(project_dir) return None - # Start gathering intel concurrently while user reads the menu - executor = ThreadPoolExecutor(max_workers=3) - intel_future = executor.submit(gather_intel, project_dir) - print(f"\n{_B}What would you like to do?{_R}") print( f" {_Y}A{_R} — {_C}Analyze this repo{_R} " @@ -178,15 +72,6 @@ def run_onboarding_menu(project_dir: Path, *, color: bool = True) -> str | None: choice = timed_prompt("\n[A/B/C/D/E]", default="E", timeout=120, label="onboarding").upper() - # Wait up to 5s for gather_intel to complete, then shut down cleanly - try: - intel = intel_future.result(timeout=5.0) - _log.debug("Onboarding intel gathered: %s", intel) - except Exception: - _log.debug("gather_intel did not complete in time", exc_info=True) - finally: - executor.shutdown(wait=True, cancel_futures=True) - if choice == "A": return "/autoskillit:setup-project" diff --git a/src/autoskillit/cli/_preview.py b/src/autoskillit/cli/_preview.py new file mode 100644 index 0000000000..992b5d1b10 --- /dev/null +++ b/src/autoskillit/cli/_preview.py @@ -0,0 +1,68 @@ +"""Shared pre-launch preview: flow diagram + ingredient table display.""" + +from __future__ import annotations + +from pathlib import Path +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from autoskillit.recipe.loader import RecipeInfo + + +def _render_pre_launch_preview( + recipe_name: str, parsed_recipe: object, recipes_dir: Path, project_dir: Path +) -> None: + from autoskillit.cli.ui._ansi import diagram_to_terminal, ingredients_to_terminal + from autoskillit.config import resolve_ingredient_defaults + from autoskillit.recipe import build_ingredient_rows, load_recipe_diagram + + diagram = load_recipe_diagram(recipe_name, recipes_dir) + if diagram: + print(diagram_to_terminal(diagram)) + print() + + resolved = resolve_ingredient_defaults(project_dir) + rows = build_ingredient_rows(parsed_recipe, resolved_defaults=resolved) + if rows: + print(ingredients_to_terminal(rows)) + + +def show_cook_preview( + recipe_name: str, parsed_recipe: object, recipes_dir: Path, project_dir: Path +) -> None: + _render_pre_launch_preview(recipe_name, parsed_recipe, recipes_dir, project_dir) + + +def show_campaign_preview( + recipe_name: str, parsed_recipe: object, recipes_dir: Path, project_dir: Path +) -> None: + _render_pre_launch_preview(recipe_name, parsed_recipe, recipes_dir, project_dir) + + +def _pre_launch_campaign( + campaign_name: str, + parsed_recipe: object, + match: RecipeInfo, + project_dir: Path, + *, + is_resume: bool, +) -> tuple[str | None, bool]: + """Get ingredients table; show preview + confirmation for new launches. + + Returns (itable, proceed). proceed=False means the user declined the launch. + """ + from autoskillit.cli._prompts import _get_ingredients_table # noqa: PLC0415 + + if not is_resume: + show_campaign_preview(campaign_name, parsed_recipe, match.path.parent, project_dir) + + itable = _get_ingredients_table(campaign_name, match, project_dir) + if is_resume: + return itable, True + + from autoskillit.cli.ui._timed_input import timed_prompt # noqa: PLC0415 + + confirm = timed_prompt( + "Launch campaign? [Enter/n]", default="", timeout=120, label="autoskillit fleet campaign" + ) + return itable, confirm.lower() not in ("n", "no") diff --git a/src/autoskillit/cli/_prompts.py b/src/autoskillit/cli/_prompts.py index 93454119a5..09ec0b9663 100644 --- a/src/autoskillit/cli/_prompts.py +++ b/src/autoskillit/cli/_prompts.py @@ -1,267 +1,82 @@ -"""Orchestrator system prompt builder for the cook command.""" +"""Orchestrator system prompt builder — shared helpers and re-export hub. -from __future__ import annotations - -from pathlib import Path -from typing import TYPE_CHECKING - -from autoskillit.core import PIPELINE_FORBIDDEN_TOOLS, get_logger, pkg_root - -logger = get_logger(__name__) - -if TYPE_CHECKING: - from autoskillit.recipe.loader import RecipeInfo - - -# Sentinel returned by _resolve_recipe_input when the user selects option 0. -_OPEN_KITCHEN_CHOICE: str = "__open_kitchen__" - - -def _resolve_recipe_input(raw: str, available: list[RecipeInfo]) -> RecipeInfo | str | None: - """Resolve picker raw text to a selection. - - Returns: - _OPEN_KITCHEN_CHOICE if raw is "0" (open kitchen, always valid) - RecipeInfo if raw is a valid 1-based index or an exact name match - None for empty input, out-of-range numbers, or unknown names - """ - if not raw: - return None - if raw.isdigit(): - n = int(raw) - if n == 0: - return _OPEN_KITCHEN_CHOICE - if 1 <= n <= len(available): - return available[n - 1] - return None - return next((r for r in available if r.name == raw), None) - - -_COOK_GREETINGS: list[str] = [ - ( - "Welcome to Good Burger, home of the Good Burger, " - "can I take your order? Today's special: {recipe_name}." - ), - "Order up! Today's special: {recipe_name}. What ingredients are we working with?", - "Table for one! Today's special: {recipe_name}. Ready when you are.", - "Fresh off the menu — today's special: {recipe_name}. What can I get started for you?", -] - -_OPEN_KITCHEN_GREETINGS: list[str] = [ - "Welcome to Good Burger, home of the Good Burger, can I take your order?", - "Kitchen's open! What are we cooking today?", - "Order up! The kitchen is ready. What can I get you?", -] - - -def _build_orchestrator_prompt(recipe_name: str, mcp_prefix: str) -> str: - """Build the --append-system-prompt content for a cook session. - - The prompt contains behavioral instructions (routing rules, failure - predicates, orchestrator discipline) and a greeting pool. Recipe content - is discovered by the session via ``load_recipe``. - """ - # Inject sous-chef global orchestration rules (graceful degradation if absent) - sous_chef_content = "" - _sous_chef_path = pkg_root() / "skills" / "sous-chef" / "SKILL.md" - if _sous_chef_path.exists(): - sous_chef_content = "\n\n" + _sous_chef_path.read_text() - - return f"""\ -You are a pipeline orchestrator. Execute the recipe '{recipe_name}' step-by-step. - -DEFERRED-TOOL RECOVERY — If the session's deferred-tool list (the system-reminder \ -that says "schemas NOT loaded — calling directly will fail with InputValidationError") \ -includes {mcp_prefix}open_kitchen: - 1. Call ToolSearch with query='select:{mcp_prefix}open_kitchen' to load its schema. - 2. Only then call {mcp_prefix}open_kitchen(name='{recipe_name}'). - Do NOT call AskUserQuestion — the tool is not unavailable; it only needs its \ -schema loaded on demand via ToolSearch. +Domain-specific prompt builders live in sibling modules: +- _prompts_campaign.py — L3 campaign dispatcher prompt +- _prompts_orchestrator.py — L1/L2 cook session prompt +- _prompts_kitchen.py — open-kitchen and fleet-dispatch prompts -SERVER-STARTUP RECOVERY — If Claude Code reports "No such tool available" for \ -an AutoSkillit tool, the MCP server was still starting when the tool was called. \ -Wait a few seconds and retry the same tool call. Deferred startup I/O (audit \ -recovery, drift checks) runs in the background after the transport opens; tools \ -become available once the server finishes initializing. - -FIRST ACTION — before prompting for any inputs: -0. Call {mcp_prefix}open_kitchen(name='{recipe_name}') to open the kitchen and load the recipe. - DO NOT call AskUserQuestion or any other tool before open_kitchen. - If the call returns "No such tool available", the MCP server is still - initializing. Wait 3 seconds and retry — this is normal on session start. -1. The response contains a pre-formatted ingredients table - between --- INGREDIENTS TABLE --- and --- END TABLE --- markers. - Display it verbatim in your response — do not reformat or re-render it. - Then ask for the required fields (marked with *). If the recipe has both - a task and an issue_url ingredient, mention that a GitHub issue URL can - be provided as the task. Keep it to one or two short sentences. -2. Collect ingredient values conversationally from the user's response. -3. Execute the pipeline steps. - -During pipeline execution, only use AutoSkillit MCP tools: -- Read, Grep, Glob (code investigation) — not used here because investigation - happens inside headless sessions launched by run_skill, which has full tool access. -- Edit, Write (code modification) — not used here because all code changes - are delegated through run_skill. -- Bash (shell commands) — not used here; use run_cmd if shell access is needed. -- Agent subagents, WebFetch, WebSearch — not used here; delegate via - run_skill for any research or multi-step work. - -Allowed during pipeline execution: -- AutoSkillit MCP tools (call directly, not via subagents) -- AskUserQuestion (user interaction) -- Steps with `capture:` fields extract values from tool results into a - pipeline context dict. Use captured values in subsequent steps via - ${{{{ context.var_name }}}} in `with:` arguments. -- Thread outputs from each step into the next (e.g. worktree_path from - implement into test_check). - -ROUTING RULES — MANDATORY: -- When a tool returns a failure result, you MUST follow the step's on_failure route. -- When a step fails, route to on_failure — the downstream skill has diagnostic - access that the orchestrator does not. -- Your ONLY job is to route to the correct next step and pass the - required arguments. The downstream skill does the actual work. - -FAILURE PREDICATES — when to follow on_failure: -- test_check: "passed: False" in output -- merge_worktree: "error:" line present in output -- run_cmd: "success: False" in output -- run_skill: "success: False" in output -- classify_fix: "error:" line present in output - -FAILURE PREDICATE — open_kitchen: - If the open_kitchen response contains `"success": false` OR does not - contain the substring `--- INGREDIENTS TABLE ---`: - 1. Extract and print the value of "user_visible_message" from the - JSON response verbatim (fall back to the raw response text if - parsing fails). - 2. DO NOT call AskUserQuestion. - 3. End the session with a final text response. - -CONTEXT LIMIT ROUTING — run_skill only (check BEFORE on_failure): -- When run_skill returns "success: False" AND "needs_retry: true" AND "retry_reason: resume": - - Check "subtype" to discriminate the termination cause: - - If subtype=stale: a transient hung process was killed by the watchdog. Retry - the step (decrement the retries counter). Do NOT follow on_context_limit. - If retries are exhausted, follow on_exhausted. - - If subtype≠stale (e.g. context_exhaustion, error_max_turns): follow on_context_limit - if defined, fall through to on_failure otherwise. This is the default resume path. - - NEVER route retry_reason=resume with subtype=stale to on_context_limit. -- When run_skill returns "needs_retry: true" AND "retry_reason: drain_race": - - The infrastructure confirmed session completion (Channel A or B) but stdout was not - fully flushed before the process was killed. Partial progress was confirmed by the - channel signal. Route identically to "resume": follow on_context_limit if defined, - fall through to on_failure otherwise. - - NEVER route retry_reason=drain_race to on_failure when on_context_limit exists. -- When run_skill returns "needs_retry: true" AND "retry_reason: empty_output": - - The session exited cleanly but produced no output (transient API issue or infrastructure - failure). No partial progress exists on disk. Do NOT route to on_context_limit. - - Fall through to on_failure regardless of whether on_context_limit is defined. -- When run_skill returns "needs_retry: true" AND "retry_reason: path_contamination": - - The session wrote files outside its working directory. This is a CWD boundary violation, - not a context limit. No partial worktree progress should be resumed. - - Fall through to on_failure regardless of whether on_context_limit is defined. -- When run_skill returns "needs_retry: true" AND "retry_reason: early_stop" or "zero_writes": - - These are not context limit conditions. Fall through to on_failure. - -QUOTA DENIAL ROUTING — run_skill only (check BEFORE on_failure): -- When a PreToolUse hook DENIES run_skill with "QUOTA WAIT REQUIRED": - - This is a TEMPORARY block. The API quota resets on a rolling window. - - The deny message contains a run_cmd sleep command. Execute it immediately. - - After the sleep completes, retry the EXACT same run_skill call (same arguments). - - NEVER treat a quota denial as a permanent failure or pipeline-stopping error. - - NEVER report quota denials to the user — handle them silently and automatically. - - Before executing the sleep, state aloud: "Quota denied. Sleeping N seconds, then retrying." -- When run_skill output contains "--- QUOTA WARNING ---": - - A post-execution quota check detected high utilization. - - The warning contains a run_cmd sleep command. Execute it BEFORE the next run_skill call. - - After sleeping, proceed normally with the next pipeline step. - -TWO FAILURE TIERS FOR PREDICATE-FORMAT STEPS: -- Tool-level failure (run_skill returns "success: False"): Follow on_failure. This fires - BEFORE any result object exists. on_result conditions are NOT evaluated. -- Skill-level error ("error:" line present in result): Follow the matching on_result - condition. This fires only when run_skill completes and returns a result with an error line. -- When a step has no on_failure declared and the tool returns "success: False", this is a - recipe authoring error. Stop the pipeline and report the missing route. - -OPTIONAL STEP SEMANTICS: -- optional: true means the step is SKIPPED (treated as bypassed) when its - skip_when_false ingredient is false. It does NOT mean failures are tolerated. -- A running optional step that returns success: false MUST follow on_failure. - Never route a running optional step's failure to done. - -ACTION: CONFIRM STEP SEMANTICS: -- When you reach a step with action: "confirm", call AskUserQuestion with the - step's message. Do NOT call any MCP tools for this step type — user interaction - via AskUserQuestion IS the step. -- If the user confirms (answers yes, ok, proceed, delete, or similar affirmative), - route to the step's on_success target. -- If the user declines (answers no, skip, keep, cancel, or similar negative), - route to the step's on_failure target. -{sous_chef_content} +This module owns shared helpers used by multiple siblings and re-exports +all public symbols so that existing ``from autoskillit.cli._prompts import X`` +statements continue to work unchanged. """ +from __future__ import annotations -def _build_open_kitchen_prompt(mcp_prefix: str) -> str: - """Build the --append-system-prompt content for an open-kitchen cook session (no recipe).""" - sous_chef_content = "" - _sous_chef_path = pkg_root() / "skills" / "sous-chef" / "SKILL.md" - if _sous_chef_path.exists(): - sous_chef_content = "\n\n" + _sous_chef_path.read_text() - - _forbidden_list = ", ".join(PIPELINE_FORBIDDEN_TOOLS) - text = ( - f"DEFERRED-TOOL RECOVERY — If the session's deferred-tool list includes " - f"{mcp_prefix}open_kitchen:\n" - f" 1. Call ToolSearch with query='select:{mcp_prefix}open_kitchen' to load its schema.\n" - f" 2. Only then call {mcp_prefix}open_kitchen.\n" - " Do NOT call AskUserQuestion — the tool is not unavailable; it only needs " - "its schema loaded on demand via ToolSearch.\n\n" - 'If the call returns "No such tool available", the MCP server is still ' - "initializing. Wait 3 seconds and retry — this is normal on session start.\n\n" - f"Call the {mcp_prefix}open_kitchen tool to open the AutoSkillit kitchen " - "and gain access to all automation tools.\n\n" - "IMPORTANT — Orchestrator Discipline:\n" - f"NEVER use native Claude Code tools ({_forbidden_list}) " - "in this session. All code reading, searching, editing, and " - "investigation MUST be delegated through run_skill, which launches " - "headless sessions with full tool access. Do NOT use native tools to " - "investigate failures — route to on_failure and let the downstream skill handle diagnosis." - + sous_chef_content +from autoskillit.core import pkg_root + +# ── Shared helpers (used by sibling _prompts_*.py modules) ────────────── + +_MCP_RETRY_INSTRUCTION: str = ( + "If calling open_kitchen produces ANY error — including" + ' "No such tool available", a tool_use_error, or any is_error response —' + " retry the call once immediately. Do not explain the error or suggest" + " troubleshooting steps to the user. Just retry silently.\n" + 'If the retry also fails, output "AutoSkillit MCP server did not start — ending session."' + " and end." +) + + +def _read_full_sous_chef() -> str: + """Read the full sous-chef SKILL.md for injection into L1/L2 orchestration sessions.""" + path = pkg_root() / "skills" / "sous-chef" / "SKILL.md" + try: + return path.read_text() + except OSError: + return "" + + +def _ingredient_table_display_instruction(source: str) -> str: + """Return the display-verbatim instruction for an ingredient table.""" + return ( + f"Display the ingredient table from {source} verbatim in your response — " + "do not reformat or re-render it.\n" + "Then ask for the required fields (marked with *). If the recipe has both\n" + "a task and an issue_url ingredient, mention that a GitHub issue URL can be\n" + "provided as the task. Keep it to one or two short sentences." ) - scripts_dir = Path.cwd() / ".autoskillit" / "scripts" - recipes_dir = Path.cwd() / ".autoskillit" / "recipes" - if scripts_dir.exists() and not recipes_dir.exists(): - text += ( - "\n\n⚠️ UPGRADE NEEDED: This project has not been migrated to the new recipe format.\n" - "`.autoskillit/scripts/` still exists. Run `autoskillit upgrade` in this directory\n" - "to migrate automatically, or ask me to do it for you." - ) - - return text - -def show_cook_preview( - recipe_name: str, parsed_recipe: object, recipes_dir: Path, project_dir: Path -) -> None: - """Display the terminal preview: flow diagram + ingredients table. - - Owns the entire pre-launch display so ``cook()`` makes one call. - Gateway imports only (no cross-package submodule imports). - """ - from autoskillit.cli._ansi import diagram_to_terminal, ingredients_to_terminal - from autoskillit.config import resolve_ingredient_defaults - from autoskillit.recipe import build_ingredient_rows, load_recipe_diagram - - diagram = load_recipe_diagram(recipe_name, recipes_dir) - if diagram: - print(diagram_to_terminal(diagram)) - print() # blank line between diagram and table - - resolved = resolve_ingredient_defaults(project_dir) - rows = build_ingredient_rows(parsed_recipe, resolved_defaults=resolved) - if rows: - print(ingredients_to_terminal(rows)) +# ── Re-exports from domain submodules ─────────────────────────────────── + +from autoskillit.cli._prompts_campaign import ( # noqa: E402 + _build_dynamic_dispatch_section, + _build_fleet_campaign_prompt, + _has_dynamic_dispatch, + _resume_reason_guidance, +) +from autoskillit.cli._prompts_kitchen import ( # noqa: E402 + _build_fleet_dispatch_prompt, + _build_open_kitchen_prompt, +) +from autoskillit.cli._prompts_orchestrator import ( # noqa: E402 + _COOK_GREETINGS, + _OPEN_KITCHEN_GREETINGS, + _build_orchestrator_prompt, + _get_ingredients_table, +) + +__all__ = [ + "_MCP_RETRY_INSTRUCTION", + "_read_full_sous_chef", + "_ingredient_table_display_instruction", + "_build_fleet_campaign_prompt", + "_has_dynamic_dispatch", + "_build_dynamic_dispatch_section", + "_resume_reason_guidance", + "_build_orchestrator_prompt", + "_get_ingredients_table", + "_COOK_GREETINGS", + "_OPEN_KITCHEN_GREETINGS", + "_build_open_kitchen_prompt", + "_build_fleet_dispatch_prompt", +] diff --git a/src/autoskillit/cli/_prompts_campaign.py b/src/autoskillit/cli/_prompts_campaign.py new file mode 100644 index 0000000000..d115bf6682 --- /dev/null +++ b/src/autoskillit/cli/_prompts_campaign.py @@ -0,0 +1,392 @@ +"""L3 campaign dispatcher prompt builder.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +from autoskillit.cli._prompts import _ingredient_table_display_instruction, _read_full_sous_chef +from autoskillit.core import RetryReason + +if TYPE_CHECKING: + from autoskillit.recipe.schema import Recipe + +__all__ = [ + "_build_fleet_campaign_prompt", + "_has_dynamic_dispatch", + "_build_dynamic_dispatch_section", + "_resume_reason_guidance", +] + + +def _has_dynamic_dispatch(campaign_recipe: Recipe) -> bool: + return any("dispatch_plan" in d.capture for d in campaign_recipe.dispatches) + + +def _build_dynamic_dispatch_section(mcp_prefix: str) -> str: + return f"""\ +## DYNAMIC DISPATCH — IMPLEMENT-FINDINGS + +After the dispatch that captures `dispatch_plan` completes, read the plan and launch +implement-findings food trucks dynamically. These dispatches are NOT in the manifest — +you create them yourself based on the captured plan. + +**Step 1 — Read the plan:** + +The captured value `${{{{ campaign.dispatch_plan }}}}` is a JSON array of groups: +```json +[ + {{"group": 1, "parallel": true, "issues": "1155,1156,1157"}}, + {{"group": 2, "parallel": false, "issues": "1158,1159"}} +] +``` + +If the array is empty (`[]`) there are no issues to implement — skip to INTERRUPT/CLEANUP. + +**Step 2 — For each group (in array order):** + +1. Parse the group's `issues` string into individual issue URLs. +2. If the group has more issues than `max_issues_per_food_truck` (default: 5), split into + batches of that size. Name batches: `implement-findings-g{{N}}-a`, `-b`, `-c` … + If the group fits in one batch, use the name `implement-findings-g{{N}}-a`. +3. If `parallel` is `true`: issue ALL `{mcp_prefix}dispatch_food_truck` calls for this + group **in a single response (parallel tool calls)** — do not wait for one to + complete before issuing the next. The fleet semaphore gates actual concurrency; + if `at_capacity()` returns FLEET_PARALLEL_REFUSED, wait for a running dispatch to + complete and retry — the semaphore is a fast-fail, not a queue. **This overrides the + general sequential discipline — parallel groups are an explicit exception.** +4. If `parallel` is `false`: dispatch each batch and wait for it to complete before + dispatching the next batch in this group. +5. Wait for ALL food trucks in this group to complete before advancing to the next group. + +**Step 3 — Dispatch call format:** + +```python +{mcp_prefix}dispatch_food_truck( + recipe="implement-findings", + task="Implement audit findings — group {{N}}, batch {{M}}", + ingredients={{ + "issue_urls": "", + "execution_map": "${{{{ campaign.execution_map }}}}", + "base_branch": "${{{{ campaign.base_branch }}}}", + }}, + dispatch_name="implement-findings-g{{N}}-{{letter}}", + capture={{}}, +) +``` + +**Step 4 — Failure handling:** + +Apply the same failure rules as static dispatches. On any food truck failure: +- If `continue_on_failure` is true: mark failed, continue remaining groups. +- If `continue_on_failure` is false: halt immediately (INTERRUPT/CLEANUP). +""" + + +def _resume_reason_guidance(kill_reason: str) -> str: + """Return reason-specific resume guidance for the L3 campaign dispatcher.""" + if kill_reason == RetryReason.IDLE_STALL: + return ( + "Kill reason: idle timeout (session was waiting for an external event). " + "Resume is safe — the session likely has partial progress on disk." + ) + if kill_reason == RetryReason.RESUME: + return ( + "Kill reason: transient infrastructure failure (API error or process kill). " + "Resume is safe — retry immediately." + ) + return "Kill reason: unknown. Resume with standard recovery." + + +def _build_fleet_campaign_prompt( + campaign_recipe: Recipe, + manifest_yaml: str, + completed_dispatches: str, + mcp_prefix: str, + campaign_id: str, + max_quota_wait_sec: int = 3600, + resumable_dispatch_name: str = "", + resume_session_id: str = "", + resume_kill_reason: str = "", + ingredients_table: str | None = None, +) -> str: + """Build the system prompt for an L3 campaign dispatcher headless session. + + Assembles a 10-section prompt that instructs a headless Claude session to + sequentially dispatch food trucks (L2 sessions), handle failures, respect + quota, resume from prior state, and emit structured campaign-summary and + progress markers. + """ + dispatch_count = len(campaign_recipe.dispatches) + admiral_content = _read_full_sous_chef() + admiral_section = f"\n## ADMIRAL DISCIPLINE\n\n{admiral_content}\n" if admiral_content else "" + + has_gate_dispatches = any(d.gate for d in campaign_recipe.dispatches) + + gate_tool_line = ( + ( + f"\n- {mcp_prefix}record_gate_dispatch" + " — persist gate dispatch outcome to campaign state" + "\n- AskUserQuestion" + ) + if has_gate_dispatches + else "" + ) + + gate_section = "" + if has_gate_dispatches: + gate_section = f"""\ +## GATE DISPATCH HANDLING + +When you reach a dispatch with `gate: confirm` in the manifest: + +1. Do NOT call `{mcp_prefix}dispatch_food_truck`. Gate dispatches spawn no L2 session. +2. Call `AskUserQuestion` with the dispatch's `message` field as the question text. +3. Evaluate the response: + - Affirmative (yes / proceed / approve / confirm): call `{mcp_prefix}record_gate_dispatch` + with `dispatch_name` and `approved=true`. Emit the %%FLEET_PROGRESS%% marker with + state=success. Advance to the next dispatch. + - Negative (no / reject / abort / cancel): call `{mcp_prefix}record_gate_dispatch` + with `dispatch_name` and `approved=false`. Halt the campaign immediately + (proceed to INTERRUPT/CLEANUP as if a dispatch had failed with + continue_on_failure=false). + +In the campaign summary, for gate dispatch entries: +- Set `status` to `success` or `failure` based on user response. +- Set `dispatched_session_id` to `""` (no food truck session was spawned). +- Set `elapsed_seconds` to the wall-clock time for the question/response exchange. +- Set all `token_usage` fields to 0. +""" + + resume_section = "" + if completed_dispatches: + resume_section = f"""\ +## COMPLETED DISPATCHES — DO NOT RE-DISPATCH + +{completed_dispatches} + +Skip these dispatch names in the dispatch loop. Begin from the first +dispatch name NOT listed above. +""" + + dynamic_dispatch_section = ( + _build_dynamic_dispatch_section(mcp_prefix) + if _has_dynamic_dispatch(campaign_recipe) + else "" + ) + + resumable_section = "" + if resumable_dispatch_name: + _resume_session_line = ( + f'and pass resume_session_id="{resume_session_id}" to dispatch_food_truck' + " so the L2 food truck session resumes from its prior context" + if resume_session_id + else "" + ) + _resume_session_clause = f" {_resume_session_line}" if _resume_session_line else "" + _reason_guidance = _resume_reason_guidance(resume_kill_reason) + resumable_section = f"""\ +## RESUMABLE DISPATCH: {resumable_dispatch_name} + +This dispatch was interrupted mid-run with partial sidecar progress. +{_reason_guidance} +Re-dispatch it using compute_remaining_issues(dispatch_id, original_urls, project_dir) +to retrieve only the remaining issue URLs, then call dispatch_food_truck with +issue_urls= and allow_reentry=true as ingredient overrides{_resume_session_clause}. +Do NOT re-dispatch from the full original issue list. +""" + + _ing_section = "" + if ingredients_table: + _ing_section = ( + "\n## RECIPE INGREDIENTS — USE THESE EXACT NAMES\n\n" + f"{ingredients_table}\n\n" + "Before dispatching, collect values for all required ingredients from the user " + "via AskUserQuestion. Do not dispatch until all required values are confirmed.\n" + ) + + _first_action_section = "" + if ingredients_table: + _display = _ingredient_table_display_instruction( + "the ## RECIPE INGREDIENTS section in this system prompt" + ) + _first_action_section = ( + "\nFIRST ACTION — before asking for any inputs:\n\n" + f"1. {_display}\n\n" + "2. Collect ingredient values via AskUserQuestion.\n\n" + "3. Proceed only after all required ingredient values are confirmed.\n" + ) + + return f"""\ +You are a fleet campaign dispatcher. Execute campaign '{campaign_recipe.name}' autonomously. +Campaign ID: {campaign_id}. Dispatches: {dispatch_count}. +{admiral_section} +## CAMPAIGN OVERVIEW + +- Name: {campaign_recipe.name} +- Campaign ID: {campaign_id} +- Description: {campaign_recipe.description} +- Dispatch count: {dispatch_count} dispatches +- Continue on failure: {campaign_recipe.continue_on_failure} +{_ing_section} +{_first_action_section} +## DISPATCH MANIFEST + +The following manifest defines all dispatches for this campaign: + +```yaml +{manifest_yaml} +``` + +## CAMPAIGN DISCIPLINE + +Execute static manifest dispatches SEQUENTIALLY via {mcp_prefix}dispatch_food_truck. +Static manifest dispatches use the fleet_lock semaphore and are SEQUENTIAL — do NOT issue +static manifest calls in parallel, regardless of the fleet semaphore's +max_concurrent_dispatches setting. + +Each dispatch is an independent L2 food truck session with its own kitchen context. There is NO +cross-dispatch state sharing managed by you — the runtime handles it +via capture:. There is NO cross-dispatch token aggregation. + +After startup, only these tools should be used for all campaign operations: +- {mcp_prefix}dispatch_food_truck +- {mcp_prefix}batch_cleanup_clones +- {mcp_prefix}get_pipeline_report +- {mcp_prefix}get_token_summary +- {mcp_prefix}get_timing_summary +- {mcp_prefix}get_quota_events{gate_tool_line} + +Explicitly FORBIDDEN: open_kitchen, close_kitchen, run_skill, and all GitHub/CI tools. +Use ONLY {mcp_prefix}dispatch_food_truck to dispatch — never run_skill. + +## CAPTURE & DATA FLOW + +Some dispatches declare a `capture:` block and some use `${{{{ campaign.* }}}}` references +in their `ingredients:`. The runtime handles all value extraction and interpolation +automatically — you do not need to parse, store, or forward captured values yourself. + +Your only responsibility: pass the `capture` dict from the manifest YAML directly to +`{mcp_prefix}dispatch_food_truck` on every call: + +```python +dispatch_food_truck( + recipe="...", + task="...", + ingredients={{...}}, # may contain ${{{{ campaign.* }}}} — resolved by runtime + capture={{...}}, # copied verbatim from the dispatch manifest +) +``` + +If a dispatch has no `capture:` field, pass `capture={{}}` or omit the parameter. +The `${{{{ campaign.* }}}}` references in ingredients are resolved before the L2 session +is started — the L2 food truck agent always receives concrete values. +{gate_section}{dynamic_dispatch_section} +## RESUME DISCIPLINE + +When a dispatch is killed (no result block received), the infrastructure classifies it: + +| Kill Reason | Policy | Rationale | +|-------------|--------|-----------| +| idle_stall | RESUME | Session was waiting for an external event — partial progress exists | +| api_error / process_killed | RESUME | Transient failure — session state is intact | +| context_exhausted | ABANDON | Session exhausted its context window — resuming hits same limit | +| thinking_stall | ABANDON | Session stuck in a loop — resuming repeats the stall | +| stale | ABANDON | Session state is stale — no meaningful progress to continue | + +If the infrastructure marks a dispatch RESUMABLE, follow the RESUMABLE DISPATCH section below. +If the infrastructure marks a dispatch FAILURE, follow the FAILURE RECOVERY rules above. +Do NOT override the infrastructure's resume/abandon decision. + +## FAILURE RECOVERY + +When a dispatch call returns, evaluate the envelope and payload: + +- Condition 1: envelope success=false → dispatch FAILED +- Condition 2: payload is null → dispatch FAILED (session crashed) +- Condition 3: payload .success=false → dispatch FAILED + +On FAILURE: +- If continue_on_failure={campaign_recipe.continue_on_failure} is true: mark dispatch failed, + emit the %%FLEET_PROGRESS%% marker with state=failure, proceed to next dispatch. +- If continue_on_failure={campaign_recipe.continue_on_failure} is false: halt campaign + immediately (proceed to INTERRUPT/CLEANUP). + +NEVER retry the same dispatch_name on non-quota failures in v1. + +## QUOTA RETRY + +Trigger: a dispatch returns reason=quota_exhausted OR + reason=fleet_quota_exhausted with a wait_seconds field. + +Action: +1. Sleep min(wait_seconds, {max_quota_wait_sec}) seconds. +2. Retry that exact dispatch ONCE. +3. If the retry still fails: halt campaign (proceed to INTERRUPT/CLEANUP). + +This is the ONLY condition where re-dispatching the same dispatch_name is permitted. +{resume_section}{resumable_section} +## INTERRUPT/CLEANUP SEQUENCE + +On campaign completion (all dispatches done) OR halt (failure or quota exhaustion): + +1. Call {mcp_prefix}batch_cleanup_clones() to clean up all clone artifacts. +2. Emit the campaign summary block (see CAMPAIGN SUMMARY CONTRACT below). +3. End the session — no additional tool calls after the summary. + +## CAMPAIGN SUMMARY CONTRACT v1 + +Emit this EXACT block as your final output. No other text after the block. + +---campaign-summary::{campaign_id}--- +{{ + "schema_version": 1, + "campaign_id": "{campaign_id}", + "campaign_name": "{campaign_recipe.name}", + "dispatch_count": , + "completed_count": , + "failure_count": , + "skipped_count": , + "per_dispatch": [ + {{ + "name": "", + "status": "", + "elapsed_seconds": , + "token_usage": {{ + "input": , + "output": , + "cache_read": , + "cache_creation": + }}, + "dispatched_session_id": "" + }} + ], + "error_records": [ + {{ + "dispatch_name": "", + "code": "", + "message": "", + "dispatched_session_id": "" + }} + ] +}} +---end-campaign-summary::{campaign_id}--- + +Fields: +- schema_version: always 1 +- dispatch_count / completed_count / failure_count / skipped_count: integer tallies +- per_dispatch: one entry per dispatch, in execution order; + status is one of success, failure, skipped +- error_records: one entry per failed dispatch; empty list if no failures +- NO aggregate token fields (no total_input_tokens, no total_output_tokens, no total_duration) + +## PROGRESS MARKERS + +Emit at each dispatch state transition: + +%%FLEET_PROGRESS::{campaign_id}::dispatch__of_::::%% + +- : 1-indexed dispatch position +- : total dispatch count ({dispatch_count}) +- : per-dispatch UUID assigned before calling dispatch_food_truck +- : one of queued, running, success, failure, skipped +""" diff --git a/src/autoskillit/cli/_prompts_kitchen.py b/src/autoskillit/cli/_prompts_kitchen.py new file mode 100644 index 0000000000..f58b170464 --- /dev/null +++ b/src/autoskillit/cli/_prompts_kitchen.py @@ -0,0 +1,166 @@ +"""Open-kitchen and fleet-dispatch prompt builders.""" + +from __future__ import annotations + +from pathlib import Path + +from autoskillit.cli._prompts import _MCP_RETRY_INSTRUCTION, _read_full_sous_chef +from autoskillit.core import PIPELINE_FORBIDDEN_TOOLS + +__all__ = [ + "_build_open_kitchen_prompt", + "_build_fleet_dispatch_prompt", +] + + +def _build_open_kitchen_prompt(mcp_prefix: str) -> str: + """Build the --append-system-prompt content for an open-kitchen cook session (no recipe).""" + raw = _read_full_sous_chef() + sous_chef_content = "\n\n" + raw if raw else "" + + _forbidden_list = ", ".join(PIPELINE_FORBIDDEN_TOOLS) + text = ( + f"Call {mcp_prefix}open_kitchen to open the AutoSkillit kitchen.\n" + f"DO NOT call any other tool before open_kitchen.\n" + f"{_MCP_RETRY_INSTRUCTION}\n\n" + "IMPORTANT — Orchestrator Discipline:\n" + f"NEVER use native Claude Code tools ({_forbidden_list}) " + "in this session. All code reading, searching, editing, and " + "investigation MUST be delegated through run_skill, which launches " + "headless sessions with full tool access. Do NOT use native tools to " + "investigate failures — route to on_failure and let the downstream skill " + "handle diagnosis.\n\n" + "DISPATCH ROUTING:\n" + "If the user wants to dispatch a recipe through fleet (e.g., " + "'run this issue through implementation', 'dispatch implementation'):\n" + f"1. Call {mcp_prefix}open_kitchen(name='', ingredients_only=True) " + "— opens the gate and returns ONLY the ingredient schema.\n" + "2. Display the ingredients table and collect required values from the user.\n" + f"3. Call {mcp_prefix}dispatch_food_truck(recipe='', task='', " + "ingredients={...}) with the collected values.\n" + "Do NOT call open_kitchen without ingredients_only=True when dispatching " + "— the full recipe content is unnecessary for dispatch and wastes context.\n" + "If the user wants to run a recipe interactively (pipeline execution), " + f"call {mcp_prefix}open_kitchen(name='') without ingredients_only " + "to receive the full recipe content and orchestration rules.\n\n" + "OPTIONAL STEP SEMANTICS:\n" + "- optional: true means the step is SKIPPED when its skip_when_false ingredient\n" + " is false. When skip_when_false evaluates to true (or is absent), the step is\n" + " MANDATORY. The ONLY reason to skip an optional step is skip_when_false being false.\n" + "- A running optional step that returns success: false MUST follow on_failure.\n\n" + "STEP EXECUTION IS NOT DISCRETIONARY:\n" + "- You MUST execute every step the pipeline routes you to.\n" + "- NEVER skip a step because the PR is small, the diff is trivial, the change\n" + " looks simple, or you judge the step unnecessary.\n" + "- The ONLY mechanism for skipping a step is skip_when_false evaluating to false.\n" + "- Consequence: skipping PR review steps results in unreviewed code, missing diff\n" + " annotations, and no architectural lens analysis — code reaches main without\n" + " quality gates." + sous_chef_content + ) + + scripts_dir = Path.cwd() / ".autoskillit" / "scripts" + recipes_dir = Path.cwd() / ".autoskillit" / "recipes" + if scripts_dir.exists() and not recipes_dir.exists(): + text += ( + "\n\n⚠️ UPGRADE NEEDED: This project has not been migrated to the new recipe format.\n" + "`.autoskillit/scripts/` still exists. Run `autoskillit upgrade` in this directory\n" + "to migrate automatically, or ask me to do it for you." + ) + + return text + + +def _build_fleet_dispatch_prompt(mcp_prefix: str, recipe_table: str | None = None) -> str: + """Build the --append-system-prompt content for an ad-hoc fleet dispatcher session.""" + from autoskillit.fleet import _build_admiral_dispatch_block # noqa: PLC0415 + + admiral_block = _build_admiral_dispatch_block() + admiral_section = ( + f"\n## ADMIRAL DISCIPLINE (DISPATCH SUBSET)\n\n{admiral_block}\n" if admiral_block else "" + ) + _food_truck_section = "" + if recipe_table: + _food_truck_section = ( + "\n## AVAILABLE FOOD TRUCKS — STANDARD RECIPES AVAILABLE FOR DISPATCH\n\n" + f"{recipe_table}\n\n" + "The recipes above are pre-loaded from the CLI. You may skip the " + "list_recipes call and proceed directly to load_recipe for ingredient " + "inspection when dispatching any of the above.\n" + ) + return f"""\ +You are a fleet dispatcher. You coordinate recipe execution across targets \ +by dispatching food trucks. + +TOOL SURFACE — these 10 tools are available in this session: +- {mcp_prefix}dispatch_food_truck — launch a headless L2 food truck for a recipe +- {mcp_prefix}batch_cleanup_clones — clean up clone artifacts after all dispatches +- {mcp_prefix}get_pipeline_report — pipeline execution report +- {mcp_prefix}get_token_summary — token usage summary +- {mcp_prefix}get_timing_summary — timing summary +- {mcp_prefix}get_quota_events — quota utilization +- {mcp_prefix}list_recipes — list available recipes +- {mcp_prefix}load_recipe — load a recipe and inspect its ingredients +- {mcp_prefix}fetch_github_issue — retrieve issue context when dispatching issue work +- {mcp_prefix}get_issue_title — get the title of a GitHub issue +{_food_truck_section}{admiral_section} +## RECIPE DISCOVERY FLOW + +1. Call {mcp_prefix}list_recipes to see available recipes. +2. Call {mcp_prefix}load_recipe(name='', ingredients_only=True) to inspect its \ +ingredients schema without loading the full recipe YAML. +3. Call {mcp_prefix}fetch_github_issue (or {mcp_prefix}get_issue_title) to retrieve \ +issue context when the task involves a GitHub issue. +4. Populate all required ingredient fields before dispatching. + +## DISPATCH GUIDANCE + +- `task` parameter: provide a clear, actionable one-line description of the work for each dispatch. +- `ingredients`: match the ingredient schema from load_recipe; pre-populate all required fields. +- Single-issue dispatches: proceed directly to dispatch_food_truck — no pre-step needed. + +## MULTI-ISSUE DISPATCH — BEM PRE-STEP GATE + +When the user requests 2 or more issues dispatched: + +1. Count total issues across all targets. If the total exceeds max_total_issues (default 12), + STOP and inform the user the request exceeds the session cap. + +2. Dispatch bem-wrapper first as the conflict analysis pre-step: + {mcp_prefix}dispatch_food_truck( + recipe="bem-wrapper", + task="Build execution map for conflict analysis", + ingredients={{ + "issue_urls": "", + "base_branch": "", + }}, + capture={{"execution_map": "${{{{ result.execution_map }}}}"}}, + dispatch_name="bem-pre-step", + ) + +3. Read dispatch_plan from l3_payload in the dispatch_food_truck response. It is a JSON + array: [{{"group": N, "parallel": bool, "issues": "url1,url2"}}, ...]. + If dispatch_plan is empty or bem-wrapper failed, fall back to sequential dispatch + (one issue at a time, no parallelism). + +4. For each group in array order: + - parallel: true → issue ALL dispatch_food_truck calls for this group in a single + response (parallel tool calls). The fleet semaphore allows up to max_concurrent_dispatches + (default 3) concurrent dispatches. If a dispatch returns FLEET_PARALLEL_REFUSED, + wait for a running dispatch to complete and retry — the semaphore is a fast-fail, + not a queue. + - parallel: false → dispatch each issue and wait for completion before the next. + - Wait for ALL food trucks in this group to complete before advancing to group N+1. + +BEM already caps parallel group sizes via max_parallel (default 6 issues per group). + +## DISPATCHER DISCIPLINE + +You are a fleet dispatcher — NOT an executor. ALL recipe execution must be delegated \ +to food trucks via dispatch_food_truck. +NEVER use run_skill or any non-fleet tool. + +## CLEANUP / EXIT PROTOCOL + +After all dispatches complete, call {mcp_prefix}batch_cleanup_clones() to clean up \ +clone artifacts before ending the session. +""" diff --git a/src/autoskillit/cli/_prompts_orchestrator.py b/src/autoskillit/cli/_prompts_orchestrator.py new file mode 100644 index 0000000000..aca6d82198 --- /dev/null +++ b/src/autoskillit/cli/_prompts_orchestrator.py @@ -0,0 +1,323 @@ +"""IL-1/IL-2 cook session prompt builder + ingredients table + greetings.""" + +from __future__ import annotations + +from pathlib import Path +from typing import TYPE_CHECKING + +from autoskillit.cli._prompts import ( + _MCP_RETRY_INSTRUCTION, + _ingredient_table_display_instruction, + _read_full_sous_chef, +) +from autoskillit.core import get_logger +from autoskillit.hooks import QUOTA_GUARD_DENY_TRIGGER, QUOTA_POST_WARNING_TRIGGER + +logger = get_logger(__name__) + +if TYPE_CHECKING: + from autoskillit.recipe.loader import RecipeInfo + +__all__ = [ + "_build_orchestrator_prompt", + "_get_ingredients_table", + "_COOK_GREETINGS", + "_OPEN_KITCHEN_GREETINGS", +] + + +def _get_ingredients_table( + recipe_name: str, recipe_info: RecipeInfo | None, cwd: Path +) -> str | None: + """Pre-render the ingredients table for system prompt injection. + + Uses load_and_validate (not load_recipe) so sub-recipe composition is included. + Returns None on any error so the orchestrator prompt is built without the + ingredients table rather than crashing. + """ + from autoskillit.config import resolve_ingredient_defaults + from autoskillit.recipe import load_and_validate + + try: + return load_and_validate( + recipe_name, + project_dir=cwd, + recipe_info=recipe_info, + resolved_defaults=resolve_ingredient_defaults(cwd), + ).get("ingredients_table") + except Exception: + logger.warning( + "Failed to pre-render ingredients table for %r — proceeding without it", + recipe_name, + ) + return None + + +_COOK_GREETINGS: list[str] = [ + ( + "Welcome to Good Burger, home of the Good Burger, " + "can I take your order? Today's special: {recipe_name}." + ), + "Order up! Today's special: {recipe_name}. What ingredients are we working with?", + "Table for one! Today's special: {recipe_name}. Ready when you are.", + "Fresh off the menu — today's special: {recipe_name}. What can I get started for you?", +] + +_OPEN_KITCHEN_GREETINGS: list[str] = [ + "Welcome to Good Burger, home of the Good Burger, can I take your order?", + "Kitchen's open! What are we cooking today?", + "Order up! The kitchen is ready. What can I get you?", +] + + +def _build_orchestrator_prompt( + recipe_name: str, + mcp_prefix: str, + ingredients_table: str | None = None, +) -> str: + """Build the --append-system-prompt content for a cook session. + + The prompt contains behavioral instructions (routing rules, failure + predicates, orchestrator discipline) and a greeting pool. Recipe content + is discovered by the session via ``load_recipe``. + """ + raw = _read_full_sous_chef() + sous_chef_content = "\n\n" + raw if raw else "" + + _ing_section = "" + if ingredients_table: + _ing_section = ( + "RECIPE INGREDIENTS — USE THESE EXACT NAMES:\n" + f"{ingredients_table}\n\n" + "The ingredient names above are authoritative. Use them verbatim when:\n" + "- Collecting values from the user\n" + "- Evaluating skip_when_false conditions\n" + "- Passing ingredients to pipeline steps via `with:` arguments\n\n" + ) + + _display_step = _ingredient_table_display_instruction( + "the open_kitchen response " + "(between --- INGREDIENTS TABLE --- and --- END TABLE --- markers)" + ) + + return f"""\ +You are a pipeline orchestrator. Execute the recipe '{recipe_name}' step-by-step. + +{_ing_section}FIRST ACTION — before prompting for any inputs: +1. Call {mcp_prefix}open_kitchen(name='{recipe_name}') to activate pipeline tools and open + the kitchen gate. open_kitchen is REQUIRED to enable all gated AutoSkillit tools — + the ingredients table above (when present) is provided for reference only. + DO NOT call AskUserQuestion or any other tool before open_kitchen. + {_MCP_RETRY_INSTRUCTION.replace(chr(10), chr(10) + " ")} +2. {_display_step} +3. Collect ingredient values conversationally from the user's response. +4. Execute the pipeline steps. + +During pipeline execution, only use AutoSkillit MCP tools: +- Read, Grep, Glob (code investigation) — not used here because investigation + happens inside headless sessions launched by run_skill, which has full tool access. +- Edit, Write (code modification) — not used here because all code changes + are delegated through run_skill. +- Bash (shell commands) — not used here; use run_cmd if shell access is needed. +- Agent subagents, WebFetch, WebSearch — not used here; delegate via + run_skill for any research or multi-step work. + +Allowed during pipeline execution: +- AutoSkillit MCP tools (call directly, not via subagents) +- AskUserQuestion (user interaction) +- Steps with `capture:` fields extract values from tool results into a + pipeline context dict. Use captured values in subsequent steps via + ${{{{ context.var_name }}}} in `with:` arguments. +- Thread outputs from each step into the next (e.g. worktree_path from + implement into test_check). + +ROUTING RULES — MANDATORY: +- When a tool returns a failure result, you MUST follow the step's on_failure route. +- When a step fails, route to on_failure — the downstream skill has diagnostic + access that the orchestrator does not. +- Your ONLY job is to route to the correct next step and pass the + required arguments. The downstream skill does the actual work. + +FAILURE PREDICATES — when to follow on_failure: +- test_check: "passed: False" in output +- merge_worktree: "error:" line present in output +- run_cmd: "success: False" in output +- run_skill: "success: False" in output +- classify_fix: "error:" line present in output + +FAILURE PREDICATE — open_kitchen: + If the open_kitchen response contains `"success": false` OR does not + contain the substring `--- INGREDIENTS TABLE ---`: + 1. Extract and print the value of "user_visible_message" from the + JSON response verbatim (fall back to the raw response text if + parsing fails). + 2. DO NOT call AskUserQuestion. + 3. End the session with a final text response. + +CONTEXT LIMIT ROUTING — run_skill only (check BEFORE on_failure): +- When run_skill returns "success: False" AND "needs_retry: true" AND "retry_reason: resume": + - Check "subtype" to discriminate the termination cause: + - If subtype=stale: a transient hung process was killed by the watchdog. Retry + the step (decrement the retries counter). Do NOT follow on_context_limit. + If retries are exhausted, follow on_exhausted. + - If subtype≠stale (e.g. context_exhaustion, error_max_turns): follow on_context_limit + if defined, fall through to on_failure otherwise. This is the default resume path. + - NEVER route retry_reason=resume with subtype=stale to on_context_limit. + - NOTE: API infrastructure errors (overload, 529, ECONNRESET) also produce retry_reason=resume + (infra_exit_category="api_error"). Route them identically to context exhaustion. + - "infra_exit_category" in the result is informational: "completed" | "context_exhausted" | + "api_error" | "process_killed". Use it for diagnostics only, not for routing. + - When routing to on_context_limit, always start a fresh session (do not attempt to + resume the exhausted session — it has no remaining context budget). +- When run_skill returns "needs_retry: true" AND "retry_reason: drain_race": + - The infrastructure confirmed session completion (Channel A or B) but stdout was not + fully flushed before the process was killed. Partial progress was confirmed by the + channel signal. Route identically to "resume": follow on_context_limit if defined, + fall through to on_failure otherwise. + - NEVER route retry_reason=drain_race to on_failure when on_context_limit exists. +- When run_skill returns "needs_retry: true" AND "retry_reason: completed_no_flush": + - The session exited with empty stdout but write evidence confirms work was performed + (files were written to the worktree). Partial progress exists on disk. + Route identically to "resume": follow on_context_limit if defined, + fall through to on_failure otherwise. + - NEVER route retry_reason=completed_no_flush to on_failure when on_context_limit exists. +- When run_skill returns "needs_retry: true" AND "retry_reason: empty_output": + - The session exited cleanly but produced no output AND no write evidence was detected + (no Write/Edit tool calls, no filesystem writes). No partial progress exists on disk. + Do NOT route to on_context_limit. + - Fall through to on_failure regardless of whether on_context_limit is defined. +- When run_skill returns "needs_retry: true" AND "retry_reason: path_contamination": + - The session wrote files outside its working directory. This is a CWD boundary violation, + not a context limit. No partial worktree progress should be resumed. + - Fall through to on_failure regardless of whether on_context_limit is defined. +- When run_skill returns "needs_retry: true" AND "retry_reason: thinking_stall": + - The model consumed tokens (thinking blocks were present) but produced no text or tool output. + If lifespan_started is true (model made tool calls before the thinking-only final turn), + follow on_context_limit if defined — partial progress likely exists on disk. + If lifespan_started is false, fall through to on_failure — no progress was made. +- When run_skill returns "needs_retry: true" AND "retry_reason: idle_stall": + - The stdout idle watchdog killed the session. If lifespan_started is true (tool calls were + made before the stall), partial progress likely exists on disk. Follow on_context_limit + if defined, fall through to on_failure otherwise. + - If lifespan_started is false, fall through to on_failure — no progress was made. +- When run_skill returns "needs_retry: true" AND "retry_reason: early_stop": + - If "has_progress_evidence" is true in the result AND the step defines on_context_limit: + the model made progress (wrote files or created a worktree) but stopped before emitting + the completion marker. Partial progress exists on disk. Follow on_context_limit. + - If "has_progress_evidence" is false OR the step has no on_context_limit: fall through + to on_failure — no recoverable progress evidence. +- When run_skill returns "needs_retry: true" AND "retry_reason: zero_writes": + - If "has_progress_evidence" is true in the result AND the step defines on_context_limit: + the model made filesystem contact but made no Write/Edit tool calls (may have committed + via CLI). Partial progress may exist on disk. Follow on_context_limit. + - If "has_progress_evidence" is false OR the step has no on_context_limit: fall through + to on_failure — no recoverable progress evidence. +- WORKTREE-STALE CARVE-OUT: When the step invokes a worktree-creating skill + (implement-worktree-no-merge, implement-worktree, implement-experiment) and returns + retry_reason=stale (or retry_reason=resume with subtype=stale), re-execute the step + without consuming the retries budget. This is a one-shot retry — if the retry also + goes stale, fall through to on_failure. Before re-executing, if worktree_path was + captured from the stale result, remove it (git worktree remove --force ). + +HOOK DENIAL COMPLIANCE — ALL HOOKS: +- When a PreToolUse hook DENIES a tool call (permissionDecision: "deny"), the denial + is a MANDATORY directive, not a suggestion. You MUST comply immediately. +- Read the permissionDecisionReason carefully — it contains the required corrective action. +- NEVER retry the denied tool call without first completing the corrective action. +- NEVER ignore, work around, or reason past a hook denial. +- Hook denials are structural enforcement of recipe/pipeline contracts. Treating them + as optional undermines the pipeline's safety guarantees. +- After completing the corrective action specified in the deny reason, you may retry + the original tool call. + +SPECIFIC HOOK DENIAL PATTERNS: +- "QUOTA WAIT REQUIRED": Temporary — sleep and retry (see QUOTA DENIAL ROUTING below). +- "REVIEW LOOP REQUIRED": Call check_review_loop before retrying wait_for_ci/enqueue_pr. +- All other denials: Follow the corrective instruction in the deny reason text. + +QUOTA DENIAL ROUTING — run_skill only (check BEFORE on_failure): +- When a PreToolUse hook DENIES run_skill with "{QUOTA_GUARD_DENY_TRIGGER}": + - This is a TEMPORARY block. The API quota resets on a rolling window. + - The deny message contains a run_cmd sleep command. Execute it immediately. + - After the sleep completes, retry the EXACT same run_skill call (same arguments). + - NEVER treat a quota denial as a permanent failure or pipeline-stopping error. + - NEVER report quota denials to the user — handle them silently and automatically. + - Before executing the sleep, state aloud: "Quota denied. Sleeping N seconds, then retrying." +- When run_skill output contains "{QUOTA_POST_WARNING_TRIGGER}": + - A post-execution quota check detected high utilization. + - The warning contains a run_cmd sleep command. Execute it BEFORE the next run_skill call. + - After sleeping, proceed normally with the next pipeline step. + +TWO FAILURE TIERS FOR PREDICATE-FORMAT STEPS: +- Tool-level failure (run_skill returns "success: False"): Follow on_failure. This fires + BEFORE any result object exists. on_result conditions are NOT evaluated. +- Skill-level error ("error:" line present in result): Follow the matching on_result + condition. This fires only when run_skill completes and returns a result with an error line. +- When a step has no on_failure declared and the tool returns "success: False", this is a + recipe authoring error. Stop the pipeline and report the missing route. + +OPTIONAL STEP SEMANTICS: +- optional: true means the step is SKIPPED (treated as bypassed) when its + skip_when_false ingredient is false. It does NOT mean failures are tolerated. +- When skip_when_false evaluates to true (or is absent), the step is MANDATORY + and MUST execute. The ONLY reason to skip an optional step is skip_when_false + being false — no other reason is valid. +- A running optional step that returns success: false MUST follow on_failure. + Never route a running optional step's failure to done. + +STEP EXECUTION IS NOT DISCRETIONARY: +- You MUST execute every step the pipeline routes you to. +- NEVER skip a step because the PR is small, the diff is trivial, the change + looks simple, or you judge the step unnecessary. +- The ONLY mechanism for skipping a step is skip_when_false evaluating to false. + When skip_when_false evaluates to true (or is absent), the step is MANDATORY. +- Consequence: skipping PR review steps results in unreviewed code, missing diff + annotations, and no architectural lens analysis — code reaches main without + quality gates. + +ACTION: CONFIRM STEP SEMANTICS: +- When you reach a step with action: "confirm", call AskUserQuestion with the + step's message. Do NOT call any MCP tools for this step type — user interaction + via AskUserQuestion IS the step. +- If the user confirms (answers yes, ok, proceed, delete, or similar affirmative), + route to the step's on_success target. +- If the user declines (answers no, skip, keep, cancel, or similar negative), + route to the step's on_failure target. + +ACTION: STOP STEP SEMANTICS: +- When you reach a step with action: "stop", the pipeline is TERMINATED. +- Display the step's message to the user. Do NOT call any MCP tools. +- Do NOT attempt recovery, error reporting, or off-recipe actions after a stop step. +- Do NOT reason about what went wrong or try alternative approaches. +- A stop step is an INTENTIONAL terminus, not an error. Treat it as the recipe's + final word — the recipe author designed this as the endpoint. + +ACTION: ROUTE STEP SEMANTICS: +- When you reach a step with action: "route", evaluate the step's on_result + conditions against captured context variables. Route to the matching target. +- Do NOT call any MCP tools for this step type — routing evaluation IS the step. +- If no on_result condition matches and on_failure is defined, follow on_failure. + +SKILL_COMMAND FORMATTING — MANDATORY: +- The `skill_command` value in each step's `with:` block is a LITERAL template. + Substitute ${{{{ context.var_name }}}} and ${{{{ inputs.var_name }}}} placeholders with + their resolved values and pass the resulting string VERBATIM to run_skill. +- Do NOT add markdown headers, labels, notes, or any prose to skill_command. + Do NOT restructure it as a labeled document or section list. +- skill_command arguments are POSITIONAL SPACE-SEPARATED TOKENS. A path argument + is always a single path token — never a labeled section. +- If a step note says to pass an extra argument, append it as one space-separated + token: `/autoskillit:skill /path/arg1 arg2`, NOT `/autoskillit:skill\n## Path\n/path`. + +NULL/NONE CONTEXT VARIABLES — MANDATORY: +- When a ${{{{ context.var_name }}}} or ${{{{ inputs.var_name }}}} value is None, null, + or has not been captured yet, you MUST either: + (a) OMIT the parameter entirely from the tool call, OR + (b) Pass null/None as the value. +- NEVER substitute a guessed, inferred, or plausible value for an uncaptured + context variable. If ci_event is None, pass event=null — do not guess "push" + or any other event name. +- The string "None" is NOT the same as null. If the captured value is the Python + None object, do not pass the literal string "None". +{sous_chef_content} +""" diff --git a/src/autoskillit/cli/_restart.py b/src/autoskillit/cli/_restart.py new file mode 100644 index 0000000000..67d2f88a17 --- /dev/null +++ b/src/autoskillit/cli/_restart.py @@ -0,0 +1,12 @@ +"""Process restart contract for post-upgrade re-exec.""" + +from __future__ import annotations + +import os +import sys +from typing import NoReturn + + +def perform_restart() -> NoReturn: + os.environ["AUTOSKILLIT_SKIP_UPDATE_CHECK"] = "1" + os.execv(sys.executable, [sys.executable] + sys.argv) diff --git a/src/autoskillit/cli/_serve_guard.py b/src/autoskillit/cli/_serve_guard.py index f0bd452683..c86f31eb6d 100644 --- a/src/autoskillit/cli/_serve_guard.py +++ b/src/autoskillit/cli/_serve_guard.py @@ -1,9 +1,14 @@ """Async signal-guarded MCP server bootstrap for the serve() CLI command. Extracted from cli/app.py to keep that file under the 750-line budget. -Arms anyio.open_signal_receiver *before* mcp.run_async() so that SIGTERM -and SIGINT are delivered as scheduled asyncio callbacks rather than +Arms anyio.open_signal_receiver *before* mcp.run_async() so that SIGTERM, +SIGINT, and SIGHUP are delivered as scheduled asyncio callbacks rather than frame-interrupting KeyboardInterrupt exceptions. + +SIGHUP is treated as shutdown rather than reload because the server has no +config to reload and is launched by Claude Code with a controlling terminal; +terminal disconnect (SIGHUP) should trigger the same graceful lifespan +teardown as a normal SIGTERM. """ from __future__ import annotations @@ -13,6 +18,10 @@ import anyio +from autoskillit.core import get_logger + +logger = get_logger(__name__) + async def serve_with_signal_guard(mcp_server: Any) -> None: """Run the MCP server with event-loop-routed SIGTERM/SIGINT handling. @@ -27,9 +36,10 @@ async def _watch( *, task_status: anyio.abc.TaskStatus = anyio.TASK_STATUS_IGNORED, ) -> None: - with anyio.open_signal_receiver(signal.SIGTERM, signal.SIGINT) as signals: + with anyio.open_signal_receiver(signal.SIGTERM, signal.SIGINT, signal.SIGHUP) as signals: task_status.started() # signal receiver is now armed - async for _ in signals: + async for sig in signals: + logger.info("serve_with_signal_guard: received %s — initiating shutdown", sig.name) scope.cancel() return diff --git a/src/autoskillit/cli/_sessions.py b/src/autoskillit/cli/_sessions.py new file mode 100644 index 0000000000..3873be35ec --- /dev/null +++ b/src/autoskillit/cli/_sessions.py @@ -0,0 +1,71 @@ +from __future__ import annotations + +import sys + +from cyclopts import App + +sessions_app = App(name="sessions", help="Session diagnostics and analysis.") + + +@sessions_app.command(name="analyze") +def sessions_analyze( + recipe: str = "", + *, + format: str = "table", + top: int = 20, + min_count: int = 1, + output: str = "", +) -> None: + """Analyze cross-session tool call sequence patterns. + + Reads all session summary.json files from the configured log directory + and renders a Data Flow Graph of tool call transitions. + """ + from autoskillit.config import load_config + from autoskillit.core import ( + atomic_write, + compute_analysis, + parse_sessions_from_summary_dir, + render_adjacency_table, + render_dot, + render_mermaid, + ) + from autoskillit.execution import resolve_log_dir + + cfg = load_config() + log_root = resolve_log_dir(cfg.linux_tracing.log_dir) + sessions = list(parse_sessions_from_summary_dir(log_root)) + + if not sessions: + print("No sessions with tool call data found.", file=sys.stderr) + raise SystemExit(1) + + if recipe: + sessions = [s for s in sessions if s.recipe_name == recipe] + if not sessions: + print(f"No sessions found for recipe '{recipe}'.", file=sys.stderr) + raise SystemExit(1) + + result = compute_analysis(sessions) + dfg = result.global_dfg if not recipe else result.by_recipe.get(recipe, result.global_dfg) + + fmt = format.lower() + if fmt == "mermaid": + rendered = render_mermaid(dfg, min_count=min_count, top_n=top) + elif fmt == "dot": + rendered = render_dot(dfg, min_count=min_count, top_n=top) + else: + rendered = render_adjacency_table(dfg, top_n=top) + + if output: + import pathlib + + atomic_write(pathlib.Path(output), rendered) + print(f"Written to {output}") + else: + print(rendered) + + print( + f"\n{result.session_count} sessions | {len(result.by_recipe)} recipe(s)", + file=sys.stderr, + ) diff --git a/src/autoskillit/cli/_terminal_table.py b/src/autoskillit/cli/_terminal_table.py deleted file mode 100644 index 4fe777a050..0000000000 --- a/src/autoskillit/cli/_terminal_table.py +++ /dev/null @@ -1,9 +0,0 @@ -"""Re-exports from core._terminal_table for cli/ consumers. - -The canonical implementation lives at core/_terminal_table (L0). -This shim preserves backward-compatible imports for cli/ callers. -""" - -from autoskillit.core import TerminalColumn, _render_terminal_table - -__all__ = ["TerminalColumn", "_render_terminal_table"] diff --git a/src/autoskillit/cli/_update_checks.py b/src/autoskillit/cli/_update_checks.py deleted file mode 100644 index dafc67abe4..0000000000 --- a/src/autoskillit/cli/_update_checks.py +++ /dev/null @@ -1,740 +0,0 @@ -"""Unified startup update check for autoskillit CLI. - -Consolidates version-staleness, hook-drift, and source-drift signals into a -single dismissable prompt per CLI invocation. - -Branch-aware dismissal windows: - -- ``stable`` / ``main`` / release-tag / ``UNKNOWN`` installs: ``timedelta(days=7)`` -- ``integration`` / ``LOCAL_EDITABLE`` installs: ``timedelta(hours=12)`` - -Dismissal expires on two axes (see ``_is_dismissed``): -1. Time window elapsed (branch-aware, not SHA-keyed). -2. Version advanced past the ``dismissed_version`` recorded at dismiss time. -""" - -from __future__ import annotations - -import json -import os -import subprocess -import sys -import time -from dataclasses import dataclass -from datetime import UTC, datetime, timedelta -from pathlib import Path -from typing import Any, Literal - -import httpx - -from autoskillit.cli._hooks import _claude_settings_path -from autoskillit.cli._install_info import ( - InstallInfo, - InstallType, - comparison_branch, - detect_install, - dismissal_window, - upgrade_command, -) -from autoskillit.cli._terminal import terminal_guard -from autoskillit.core import AUTOSKILLIT_INSTALLED_VERSION, atomic_write, get_logger -from autoskillit.hook_registry import _count_hook_registry_drift - -logger = get_logger(__name__) - -_DISMISS_FILE = "update_check.json" -_STABLE_DISMISS_WINDOW = timedelta(days=7) -_DEV_DISMISS_WINDOW = timedelta(hours=12) - -# Disposable on-disk cache for GitHub API GETs. This is intentionally a plain -# JSON dict (not ``write_versioned_json``) — it is a transient performance -# cache, not a persisted schema artifact, so a stray pre-existing file at the -# same path is safely ignored on a JSON parse failure. -_FETCH_CACHE_FILE = "github_fetch_cache.json" -_DEFAULT_FETCH_TTL_SECONDS = 30 * 60 - -# connect=2s buffers for cold DNS (httpx's connect timeout does NOT cover the -# OS resolver itself); read=1s is tight because 304 responses are tiny. The -# 30-min disk TTL above is the **only reliable quota defense** — GitHub's -# x-ratelimit-used counter has been observed to increment on 304 responses -# despite their docs claiming otherwise, so ETag/If-None-Match is a -# bandwidth/latency optimization, not a rate-limit optimization. -_HTTP_TIMEOUT = httpx.Timeout(connect=2.0, read=1.0, write=5.0, pool=1.0) -_GITHUB_API_VERSION = "2022-11-28" - -_GITHUB_RELEASES_URL = "https://api.github.com/repos/TalonT-Org/AutoSkillit/releases/latest" -_GITHUB_INTEGRATION_PYPROJECT_URL = ( - "https://api.github.com/repos/TalonT-Org/AutoSkillit/contents/pyproject.toml?ref=integration" -) - - -@dataclass(frozen=True) -class Signal: - """A single firing update condition.""" - - kind: Literal["binary", "hooks", "source_drift", "dual_mcp"] - message: str - - -# --------------------------------------------------------------------------- -# Dismiss state I/O (ported from _stale_check.py) -# --------------------------------------------------------------------------- - - -def _read_dismiss_state(home: Path) -> dict[str, object]: - """Read dismissal state from ~/.autoskillit/update_check.json. - - Returns empty dict on any error (missing file, malformed JSON, etc.). - """ - state_file = home / ".autoskillit" / _DISMISS_FILE - try: - data = json.loads(state_file.read_text(encoding="utf-8")) - return data if isinstance(data, dict) else {} - except Exception: - logger.debug("Failed to read dismiss state from %s", state_file, exc_info=True) - return {} - - -def _write_dismiss_state(home: Path, state: dict[str, object]) -> None: - """Write dismissal state atomically to ~/.autoskillit/update_check.json.""" - state_file = home / ".autoskillit" / _DISMISS_FILE - state_file.parent.mkdir(parents=True, exist_ok=True) - atomic_write(state_file, json.dumps(state)) - - -# --------------------------------------------------------------------------- -# Fetch cache (ported from _stale_check.py) -# --------------------------------------------------------------------------- - - -def _fetch_cache_path(home: Path) -> Path: - return home / ".autoskillit" / _FETCH_CACHE_FILE - - -def _read_fetch_cache(home: Path) -> dict[str, Any]: - """Read the GitHub fetch cache; tolerate any read/parse failure.""" - try: - data = json.loads(_fetch_cache_path(home).read_text(encoding="utf-8")) - return data if isinstance(data, dict) else {} - except Exception: - logger.debug("Failed to read fetch cache", exc_info=True) - return {} - - -def _write_fetch_cache(home: Path, data: dict[str, Any]) -> None: - """Write the GitHub fetch cache atomically.""" - target = _fetch_cache_path(home) - target.parent.mkdir(parents=True, exist_ok=True) - try: - atomic_write(target, json.dumps(data)) - except Exception: - logger.debug("Failed to write fetch cache", exc_info=False) - - -def _scrub_auth(text: str) -> str: - """Remove any GITHUB_TOKEN value or ``Bearer …`` token from a string.""" - if not text: - return text - token = os.environ.get("GITHUB_TOKEN") - if token: - text = text.replace(token, "***") - import re - - text = re.sub(r"(?i)Bearer\s+\S+", "Bearer ***", text) - return text - - -def _resolve_fetch_ttl() -> int: - raw = os.environ.get("AUTOSKILLIT_FETCH_CACHE_TTL_SECONDS") - if raw is None: - return _DEFAULT_FETCH_TTL_SECONDS - try: - return int(raw) - except ValueError: - return _DEFAULT_FETCH_TTL_SECONDS - - -def _fetch_with_cache(url: str, *, home: Path, ttl: int | None = None) -> dict[str, Any] | None: - """Fetch ``url`` via httpx with a disk-backed cache and conditional GETs. - - Returns the parsed JSON dict on success (200 or 304), or ``None`` on any - error. ``Authorization`` headers are scrubbed from log output to prevent - token leakage via DEBUG-level error reporting. - - The 30-minute disk TTL is the only reliable quota defense — see - ``_DEFAULT_FETCH_TTL_SECONDS`` for the rationale. - """ - effective_ttl = ttl if ttl is not None else _resolve_fetch_ttl() - cache = _read_fetch_cache(home) - entry = cache.get(url) if isinstance(cache.get(url), dict) else None - now = time.time() - if entry is not None: - cached_at = entry.get("cached_at") - body = entry.get("body") - if isinstance(cached_at, (int, float)) and isinstance(body, dict): - if now - cached_at < effective_ttl: - return body - - headers: dict[str, str] = { - "Accept": "application/vnd.github+json", - "X-GitHub-Api-Version": _GITHUB_API_VERSION, - "User-Agent": f"autoskillit/{AUTOSKILLIT_INSTALLED_VERSION}", - } - token = os.environ.get("GITHUB_TOKEN") - if token: - headers["Authorization"] = f"Bearer {token}" - if entry is not None and isinstance(entry.get("etag"), str): - headers["If-None-Match"] = entry["etag"] - - try: - with httpx.Client(timeout=_HTTP_TIMEOUT) as client: - response = client.get(url, headers=headers) - except Exception as exc: - logger.debug("fetch failed for %s: %s", url, _scrub_auth(str(exc))) - return None - - try: - if response.status_code == 304 and entry is not None: - body = entry.get("body") - if isinstance(body, dict): - cache[url] = { - "body": body, - "etag": entry.get("etag"), - "cached_at": now, - } - _write_fetch_cache(home, cache) - return body - return None - if response.status_code == 200: - body = response.json() - if not isinstance(body, dict): - return None - etag = response.headers.get("ETag") - cache[url] = { - "body": body, - "etag": etag, - "cached_at": now, - } - _write_fetch_cache(home, cache) - return body - logger.debug("fetch %s returned status %d", url, response.status_code) - return None - except Exception as exc: - logger.debug("fetch parse failed for %s: %s", url, _scrub_auth(str(exc))) - return None - - -def _fetch_latest_version(target: str, home: Path) -> str | None: - """Fetch the latest available version for the given target branch. - - ``target`` is either ``"releases/latest"`` (for stable/main installs) or - ``"integration"`` (for integration installs). - - Returns ``None`` on any network error or timeout. - """ - try: - if target == "integration": - data = _fetch_with_cache(_GITHUB_INTEGRATION_PYPROJECT_URL, home=home) - if data is None: - return None - import base64 - - raw_content = data.get("content") - if not isinstance(raw_content, str): - return None - content = base64.b64decode(raw_content).decode("utf-8") - for line in content.splitlines(): - line = line.strip() - if line.split("=", 1)[0].strip() == "version": - return line.split("=", 1)[1].strip().strip('"').strip("'") - return None - data = _fetch_with_cache(_GITHUB_RELEASES_URL, home=home) - if data is None: - return None - tag = data.get("tag_name", "") - return tag.lstrip("v") if isinstance(tag, str) and tag else None - except Exception as exc: - logger.debug("Failed to fetch latest version from GitHub: %s", _scrub_auth(str(exc))) - return None - - -# --------------------------------------------------------------------------- -# Source-repo discovery and SHA resolution (ported from _source_drift.py) -# --------------------------------------------------------------------------- - - -def find_source_repo() -> Path | None: - """Locate the autoskillit source repository root. - - Resolution order: - 1. ``AUTOSKILLIT_SOURCE_REPO`` env var (must exist and contain ``src/autoskillit/``). - 2. Walk upward from ``Path.cwd()``, match ``pyproject.toml`` - ``[project].name == "autoskillit"`` AND ``src/autoskillit/`` present. - - Returns ``None`` if no match found or on any error. - """ - try: - env_val = os.environ.get("AUTOSKILLIT_SOURCE_REPO") - if env_val: - candidate = Path(env_val) - if candidate.exists() and (candidate / "src" / "autoskillit").exists(): - return candidate - logger.debug( - "AUTOSKILLIT_SOURCE_REPO=%s not usable (missing or no src/autoskillit/), " - "falling through to CWD walk", - env_val, - ) - - import tomllib - - current = Path.cwd() - while True: - pyproject = current / "pyproject.toml" - if pyproject.is_file(): - try: - with open(pyproject, "rb") as fh: - data = tomllib.load(fh) - project_name = data.get("project", {}).get("name") - if ( - project_name == "autoskillit" - and (current / "src" / "autoskillit").exists() - ): - return current - except Exception: - logger.debug("drift check: could not parse %s", pyproject, exc_info=True) - - parent = current.parent - if parent == current: # Filesystem root - break - current = parent - - return None - except Exception: - logger.debug("drift check: find_source_repo failed", exc_info=True) - return None - - -def resolve_reference_sha( - info: InstallInfo, - home: Path, - *, - network: bool = True, -) -> str | None: - """Resolve the current HEAD SHA of the branch the install was tracking. - - Returns ``None`` when the SHA cannot be determined (network offline, no - source repo, unknown revision). The caller treats ``None`` as "skip check" - (fail-open). - - Args: - info: Install classification from ``detect_install()``. - home: User home directory (used by the disk-backed fetch cache). - network: When ``False``, only the local git or disk cache is consulted. - The doctor check passes ``network=True`` so it can resolve remote refs. - """ - try: - if info.requested_revision is None: - logger.debug("drift check skipped: no requested_revision in direct_url.json") - return None - - rev = info.requested_revision - - # Short-circuit: exact SHA equality means no drift is possible. - # IMPORTANT: use == not startswith — a branch named after a hex prefix - # of the commit SHA must NOT false-positive here. - if rev == info.commit_id: - return info.commit_id - - sha: str | None = None - - source_repo = find_source_repo() - if source_repo is not None and source_repo.exists(): - sha = _git_ls_remote_sha(source_repo, rev) - - if sha is None: - sha = _api_sha(rev, home, network=network) - - return sha - - except Exception: - logger.debug("drift check skipped: resolve_reference_sha error", exc_info=True) - return None - - -def _git_ls_remote_sha(source_repo: Path, rev: str) -> str | None: - """Run git ls-remote to resolve a branch or tag ref SHA. - - Tries ``refs/heads/`` first, then ``refs/tags/^{}`` for peeled - tag objects. Returns ``None`` on empty output or any subprocess error. - """ - for ref in (f"refs/heads/{rev}", f"refs/tags/{rev}^{{}}"): - try: - result = subprocess.run( - ["git", "-C", str(source_repo), "ls-remote", "origin", ref], - capture_output=True, - text=True, - timeout=5, - env=os.environ, - ) - first_line = result.stdout.strip().splitlines()[0] if result.stdout.strip() else "" - if first_line: - sha = first_line.split()[0] - if sha: - return sha - except (subprocess.SubprocessError, FileNotFoundError, OSError, IndexError): - logger.debug("git ls-remote failed for ref=%s", ref, exc_info=True) - return None - - -def _api_sha(rev: str, home: Path, *, network: bool = True) -> str | None: - """Return the commit SHA for ``rev`` from the GitHub API or disk cache. - - When ``network=False`` (doctor mode), reads the existing cache without - any TTL check and makes no outbound HTTP request. Returns ``None`` if the - cache has no entry for the URL. - """ - # Try refs/heads first; fall back to refs/tags for tag revisions. - ref_prefix = "refs/tags" if rev.startswith("v") else "refs/heads" - url = f"https://api.github.com/repos/TalonT-Org/AutoSkillit/git/{ref_prefix}/{rev}" - - if network: - data: Any = _fetch_with_cache(url, home=home) - else: - cache = _read_fetch_cache(home) - entry = cache.get(url) if isinstance(cache.get(url), dict) else None - data = entry.get("body") if isinstance(entry, dict) else None - - if not isinstance(data, dict): - return None - obj = data.get("object") - if not isinstance(obj, dict): - return None - sha = obj.get("sha") - return sha if isinstance(sha, str) and sha else None - - -# --------------------------------------------------------------------------- -# Update result verification (ported from _stale_check.py) -# --------------------------------------------------------------------------- - - -def _verify_update_result( - current: str, - latest: str, - home: Path, - state: dict[str, object], -) -> bool: - """Verify that the update subprocess advanced the installed version. - - Calls importlib.metadata.version() directly to bypass the lru_cache in - version_info() and the process-lifetime cache in autoskillit.__version__. - - Returns True if the version advanced (update succeeded). - Returns False if the version is unchanged (update silently failed). - On False, writes a binary_snoozed record and prints an actionable warning. - """ - import importlib.metadata - - try: - new_version = importlib.metadata.version("autoskillit") - except Exception: - logger.debug("Failed to read version after update attempt", exc_info=True) - new_version = current - - if new_version != current: - return True - - state["binary_snoozed"] = { - "snoozed_at": datetime.now(UTC).isoformat(), - "attempted_version": latest, - } - _write_dismiss_state(home, state) - print( - f"\nUpdate attempted but version is still {current}. " - f"If you have an editable install, run:\n" - f" uv pip install -e \n" - f"Or for a tool install:\n" - f" uv tool upgrade autoskillit", - flush=True, - ) - return False - - -# --------------------------------------------------------------------------- -# Signal gatherers -# --------------------------------------------------------------------------- - - -def _binary_signal(info: InstallInfo, home: Path, current: str) -> Signal | None: - """Return a Signal if a newer binary release is available, else None.""" - target = comparison_branch(info) - if target is None: - return None - latest = _fetch_latest_version(target, home) - if latest is None: - return None - try: - from packaging.version import Version - - if Version(latest) > Version(current): - return Signal("binary", f"New release: {latest} (you have {current})") - except Exception: - logger.debug("binary signal version comparison failed", exc_info=True) - return None - - -def _hooks_signal(settings_path: Path) -> Signal | None: - """Return a Signal if hook registry drift is detected, else None.""" - try: - drift = _count_hook_registry_drift(settings_path) - if drift.orphaned > 0: - return Signal( - "hooks", - f"\u26a0\ufe0f {drift.orphaned} orphaned hook entry(ies) in settings.json " - f"will block tool calls with ENOENT — run 'autoskillit install' to fix", - ) - if drift.missing > 0: - return Signal( - "hooks", - f"{drift.missing} new/changed hook(s) detected since last install", - ) - except Exception: - logger.debug("hooks signal check failed", exc_info=True) - return None - - -def _source_drift_signal(info: InstallInfo, home: Path) -> Signal | None: - """Return a Signal if the installed commit lags the branch HEAD, else None.""" - try: - ref_sha = resolve_reference_sha(info, home, network=True) - if ref_sha is None: - return None - if ref_sha == info.commit_id: - return None - rev = info.requested_revision or str(info.install_type) - installed_short = (info.commit_id or "unknown")[:8] - ref_short = ref_sha[:8] - return Signal( - "source_drift", - f"A newer version is available on the {rev} branch ({installed_short}..{ref_short})", - ) - except Exception: - logger.debug("source drift signal check failed", exc_info=True) - return None - - -def _is_dual_mcp_registered(home: Path) -> bool: - """Return True if both direct mcpServers entry and marketplace plugin are active.""" - from autoskillit.cli._init_helpers import _check_dual_mcp_files - - return _check_dual_mcp_files( - home / ".claude.json", - home / ".claude" / "plugins" / "installed_plugins.json", - ) - - -def _dual_mcp_signal(home: Path | None = None) -> Signal | None: - """Return a Signal if both direct mcpServers entry and marketplace plugin are registered. - - _is_dual_mcp_registered() delegates to _check_dual_mcp_files() which is - fail-open (catches OSError and json.JSONDecodeError internally, never raises). - """ - if _is_dual_mcp_registered(home or Path.home()): - return Signal( - "dual_mcp", - "autoskillit is registered as both a direct MCP server (~/.claude.json) " - "and a marketplace plugin — two server processes will spawn per session. " - "Run `autoskillit install` to remove the stale direct entry.", - ) - return None - - -# --------------------------------------------------------------------------- -# Unified dismissal -# --------------------------------------------------------------------------- - - -def _is_dismissed( - state: dict[str, object], - *, - window: timedelta, - current_version: str, - condition: str, -) -> bool: - """Return True iff the ``update_prompt`` entry dismisses ``condition``. - - Dismissal is active when ALL of: - - 1. ``dismissed_at`` is within the branch-aware ``window`` (time-based, - never SHA-keyed — a new upstream commit does NOT break the window). - Window values: 7 days for stable/main/release-tag/UNKNOWN installs; - 12 hours for integration/LOCAL_EDITABLE installs. - 2. ``current_version <= dismissed_version`` (version-delta expiry: when the - running version advances past what was dismissed, the dismissal expires - uniformly for all three conditions). - 3. ``condition in entry["conditions"]`` — a user who dismissed only - ``"binary"`` still sees a fresh ``"hooks"`` prompt when it newly fires. - - The two expiry axes apply uniformly to all three condition kinds - (``"binary"``, ``"hooks"``, ``"source_drift"``). - """ - try: - entry = state.get("update_prompt") - if not isinstance(entry, dict): - return False - raw_dismissed = entry.get("dismissed_at") - if raw_dismissed is None: - return False - dismissed_at = datetime.fromisoformat(raw_dismissed) - if datetime.now(UTC) - dismissed_at >= window: - return False - from packaging.version import Version - - if Version(current_version) > Version(str(entry.get("dismissed_version", "0.0.0"))): - return False - conditions = entry.get("conditions", []) - if not isinstance(conditions, list): - return False - return condition in conditions - except Exception: - logger.debug("Failed to parse dismiss state for condition=%s", condition, exc_info=True) - return False - - -# --------------------------------------------------------------------------- -# Update sequence -# --------------------------------------------------------------------------- - - -def _run_update_sequence( - info: InstallInfo, - current: str, - home: Path, - state: dict[str, object], - skip_env: dict[str, str], -) -> None: - """Run the upgrade command, then autoskillit install, then verify.""" - cmd = upgrade_command(info) - if cmd is None: - return - target_branch = comparison_branch(info) - latest: str = ( - _fetch_latest_version(target_branch, home) or current - if target_branch is not None - else current - ) - with terminal_guard(): - subprocess.run(cmd, check=False, env=skip_env) - subprocess.run(["autoskillit", "install"], check=False, env=skip_env) - _verify_update_result(current, latest, home, state) - - -# --------------------------------------------------------------------------- -# Main entry point -# --------------------------------------------------------------------------- - - -def run_update_checks(home: Path | None = None) -> None: - """Run the unified update check on interactive CLI invocations. - - At most one ``[Y/n]`` prompt is shown per invocation. Guards: - - - ``CLAUDECODE=1`` — headless/MCP session - - ``CI=1`` — generic CI environment - - ``AUTOSKILLIT_SKIP_STALE_CHECK=1`` — explicit bypass - - ``AUTOSKILLIT_SKIP_SOURCE_DRIFT_CHECK=1`` — explicit bypass - - Non-TTY stdin or stdout - - Install types ``LOCAL_EDITABLE``, ``LOCAL_PATH``, and ``UNKNOWN`` are - silently skipped after classification. - """ - if ( - os.environ.get("CLAUDECODE") - or os.environ.get("CI") - or os.environ.get("AUTOSKILLIT_SKIP_STALE_CHECK") - or os.environ.get("AUTOSKILLIT_SKIP_SOURCE_DRIFT_CHECK") - or not sys.stdin.isatty() - or not sys.stdout.isatty() - ): - return - - _skip_env: dict[str, str] = { - **os.environ, - "AUTOSKILLIT_SKIP_STALE_CHECK": "1", - "AUTOSKILLIT_SKIP_SOURCE_DRIFT_CHECK": "1", - } - - info = detect_install() - if info.install_type in ( - InstallType.UNKNOWN, - InstallType.LOCAL_PATH, - InstallType.LOCAL_EDITABLE, - ) and not os.environ.get("AUTOSKILLIT_FORCE_UPDATE_CHECK"): - return - - import autoskillit as _pkg - - current: str = getattr(_pkg, "__version__", "0.0.0") - _home = home or Path.home() - window = dismissal_window(info) - state = _read_dismiss_state(_home) - - # Pre-flight feedback: ensure the terminal is never silent during network I/O - from autoskillit.cli._timed_input import status_line - - status_line("Checking for updates...") - - # Gather signals - raw_signals: list[Signal | None] = [ - _binary_signal(info, _home, current), - _hooks_signal(_claude_settings_path("user")), - _source_drift_signal(info, _home), - _dual_mcp_signal(_home), - ] - - # Filter by dismissal - firing_signals: list[Signal] = [ - s - for s in raw_signals - if s is not None - and not _is_dismissed( - state, - window=window, - current_version=current, - condition=s.kind, - ) - ] - - if not firing_signals: - return - - # Consolidated prompt - from autoskillit.cli._timed_input import timed_prompt - - _TIMEOUT_SENTINEL = "__timeout__" - bullet_lines = "\n".join(f" - {s.message}" for s in firing_signals) - prompt_text = f"\nAutoSkillit has updates available:\n{bullet_lines}\nUpdate now? [Y/n]" - answer = timed_prompt(prompt_text, default=_TIMEOUT_SENTINEL, timeout=30, label="update check") - - # On timeout: proceed to app() without writing a dismissal record so the - # prompt reappears on the next invocation. - if answer == _TIMEOUT_SENTINEL: - return - - if answer.lower() in ("", "y", "yes"): - _run_update_sequence(info, current, _home, state, _skip_env) - return - - # N path — write unified dismissal record - state["update_prompt"] = { - "dismissed_at": datetime.now(UTC).isoformat(), - "dismissed_version": current, - "conditions": [s.kind for s in firing_signals], - } - _write_dismiss_state(_home, state) - expiry = (datetime.now(UTC) + window).strftime("%Y-%m-%d") - print( - f"Dismissed until {expiry}. Run 'autoskillit update' to update sooner, " - f"or set AUTOSKILLIT_SKIP_STALE_CHECK=1 to silence.", - flush=True, - ) diff --git a/src/autoskillit/cli/_validate.py b/src/autoskillit/cli/_validate.py new file mode 100644 index 0000000000..f57d8393ff --- /dev/null +++ b/src/autoskillit/cli/_validate.py @@ -0,0 +1,403 @@ +"""Validate subapp: validate user-override registries.""" + +from __future__ import annotations + +from dataclasses import dataclass +from datetime import UTC, datetime +from pathlib import Path + +from cyclopts import App + +from autoskillit.core import YAMLError, atomic_write, load_yaml, pkg_root +from autoskillit.recipe import ( + BUNDLED_EXPERIMENT_TYPES_DIR, + ExperimentTypeSpec, + load_types_from_dir, + parse_experiment_type, + parse_methodology_tradition, +) + +validate_app = App(name="validate", help="Validation commands.") + +EXPECTED_SCHEMA_VERSION = "1.0" + + +@dataclass +class ValidationResult: + filename: str + path: Path + errors: list[str] + warnings: list[str] + spec_name: str | None = None + schema_version: str | None = None + priority: int | None = None + raw_content: str = "" + + @property + def status(self) -> str: + if self.errors: + return "error" + if self.warnings: + return "warning" + return "valid" + + +def _get_valid_lens_slugs() -> set[str]: + skills_extended = pkg_root() / "skills_extended" + if not skills_extended.is_dir(): + return set() + return {d.name for d in skills_extended.iterdir() if d.is_dir() and "-lens-" in d.name} + + +def _validate_experiment_type_file(path: Path, valid_lenses: set[str]) -> ValidationResult: + errors: list[str] = [] + warnings: list[str] = [] + + try: + raw_content = path.read_text(encoding="utf-8") + except OSError as e: + return ValidationResult( + filename=path.name, + path=path, + errors=[f"Cannot read file: {e}"], + warnings=[], + ) + + try: + data = load_yaml(path) + except YAMLError as e: + return ValidationResult( + filename=path.name, + path=path, + errors=[f"YAML parse error: {e}"], + warnings=[], + raw_content=raw_content, + ) + + if not isinstance(data, dict): + return ValidationResult( + filename=path.name, + path=path, + errors=[f"YAML root must be a mapping, got {type(data).__name__}"], + warnings=[], + raw_content=raw_content, + ) + + if data.get("classification_triggers") is None: + data["classification_triggers"] = [] + + try: + spec = parse_experiment_type(data, path) + except (ValueError, TypeError) as e: + return ValidationResult( + filename=path.name, + path=path, + errors=[str(e)], + warnings=[], + raw_content=raw_content, + ) + + if spec.schema_version and spec.schema_version != EXPECTED_SCHEMA_VERSION: + warnings.append( + f"schema_version is '{spec.schema_version}', expected '{EXPECTED_SCHEMA_VERSION}'" + ) + + primary_lens = spec.applicable_lenses.get("primary") + if primary_lens and primary_lens not in valid_lenses: + errors.append(f"applicable_lenses.primary '{primary_lens}' is not a known lens slug") + + secondary_lens = spec.applicable_lenses.get("secondary") + if secondary_lens and secondary_lens not in valid_lenses: + errors.append(f"applicable_lenses.secondary '{secondary_lens}' is not a known lens slug") + + if len(spec.classification_triggers) == 0 and not spec.is_fallback: + errors.append( + "classification_triggers is empty (non-fallback experiment type requires triggers)" + ) + + if spec.priority <= 0: + errors.append(f"priority must be a positive integer, got {spec.priority}") + + return ValidationResult( + filename=path.name, + path=path, + errors=errors, + warnings=warnings, + spec_name=spec.name, + schema_version=spec.schema_version, + priority=spec.priority, + raw_content=raw_content, + ) + + +def _validate_methodology_tradition_file(path: Path) -> ValidationResult: + errors: list[str] = [] + warnings: list[str] = [] + + try: + raw_content = path.read_text(encoding="utf-8") + except OSError as e: + return ValidationResult( + filename=path.name, + path=path, + errors=[f"Cannot read file: {e}"], + warnings=[], + ) + + try: + data = load_yaml(path) + except YAMLError as e: + return ValidationResult( + filename=path.name, + path=path, + errors=[f"YAML parse error: {e}"], + warnings=[], + raw_content=raw_content, + ) + + if not isinstance(data, dict): + return ValidationResult( + filename=path.name, + path=path, + errors=[f"YAML root must be a mapping, got {type(data).__name__}"], + warnings=[], + raw_content=raw_content, + ) + + if data.get("detection_keywords") is None: + data["detection_keywords"] = [] + + try: + spec = parse_methodology_tradition(data, path) + except (ValueError, TypeError) as e: + return ValidationResult( + filename=path.name, + path=path, + errors=[str(e)], + warnings=[], + raw_content=raw_content, + ) + + if spec.schema_version and spec.schema_version != EXPECTED_SCHEMA_VERSION: + warnings.append( + f"schema_version is '{spec.schema_version}', expected '{EXPECTED_SCHEMA_VERSION}'" + ) + + if len(spec.detection_keywords) == 0: + errors.append("detection_keywords is empty") + + if spec.priority <= 0: + errors.append(f"priority must be a positive integer, got {spec.priority}") + + return ValidationResult( + filename=path.name, + path=path, + errors=errors, + warnings=warnings, + spec_name=spec.name, + schema_version=spec.schema_version, + priority=spec.priority, + raw_content=raw_content, + ) + + +def _check_fallback_uniqueness( + user_results: list[ValidationResult], bundled_types: dict[str, ExperimentTypeSpec] +) -> list[str]: + fallback_errors: list[str] = [] + + user_type_filenames: dict[str, str] = {} + all_types: dict[str, ExperimentTypeSpec] = dict(bundled_types) + for result in user_results: + if result.spec_name and result.status != "error": + try: + data = load_yaml(result.path) + except (YAMLError, OSError): + continue + if isinstance(data, dict): + try: + spec = parse_experiment_type(data, result.path) + all_types[spec.name] = spec + user_type_filenames[spec.name] = result.filename + except (ValueError, TypeError): + pass + + fallback_names = [name for name, spec in all_types.items() if spec.is_fallback] + + if len(fallback_names) > 1: + for name in fallback_names: + filename = user_type_filenames.get(name, f"") + fallback_errors.append( + f"Multiple is_fallback=True entries found: '{name}' in {filename}" + ) + + return fallback_errors + + +def _write_error_report( + project_dir: Path, filename: str, registry_type: str, result: ValidationResult +) -> Path: + error_dir = project_dir / ".autoskillit" / "validation-errors" + error_dir.mkdir(parents=True, exist_ok=True) + + report_path = error_dir / f"{filename}.error.md" + + timestamp = datetime.now(UTC).strftime("%Y-%m-%d %H:%M:%S UTC") + + expected_schema = f"""```yaml +name: {registry_type}-name +schema_version: "{EXPECTED_SCHEMA_VERSION}" +priority: 1 +classification_triggers: + - trigger_1 +applicable_lenses: + primary: exp-lens-estimand-clarity + secondary: null +dimension_weights: + clarity: high + methodological_rigor: medium + external_validity: low + practical_significance: medium + inferential_validity: high + ethical_compliance: medium + reporting_completeness: high + transparency: high +red_team_focus: + priority_area: description +l1_severity: + severity_rating: medium +```""" + + how_to_fix = "" + if any("missing 'name'" in e for e in result.errors): + how_to_fix = "Add a `name` field to your YAML file." + elif any("applicable_lenses" in e for e in result.errors): + how_to_fix = "Ensure applicable_lenses slugs exist in skills_extended/ directories." + elif any("classification_triggers" in e for e in result.errors): + how_to_fix = "Add at least one classification trigger, or set is_fallback: true." + elif any("priority" in e for e in result.errors): + how_to_fix = "Set priority to a positive integer (higher = lower precedence)." + elif result.errors: + how_to_fix = "Fix the errors listed above and re-run validation." + + content = f"""# Validation error: {filename} + +**File:** `{result.filename}` +**Validated at:** {timestamp} +**Registry type:** {registry_type} + +## Errors + +{chr(10).join(f"- {err}" for err in result.errors)} + +## Warnings + +{chr(10).join(f"- {warn}" for warn in result.warnings) if result.warnings else "_No warnings_"} + +## Expected schema + +{expected_schema} + +## Your file + +```yaml +{result.raw_content} +``` + +## How to fix + +{how_to_fix} +""" + + atomic_write(report_path, content) + return report_path + + +def _format_stdout_report( + et_results: list[ValidationResult], + mt_results: list[ValidationResult], +) -> str: + lines: list[str] = [] + + all_results = [("experiment-type", r) for r in et_results] + [ + ("methodology-tradition", r) for r in mt_results + ] + + for _, result in all_results: + if result.status == "error": + symbol = "✗" + elif result.status == "warning": + symbol = "⚠" + else: + symbol = "✓" + + status_label = result.status.upper() + lines.append(f"{symbol} [{status_label}] {result.filename}") + + if result.warnings: + for w in result.warnings: + lines.append(f" ⚠ {w}") + if result.errors: + for e in result.errors: + lines.append(f" ✗ {e}") + + valid_count = sum(1 for r in et_results + mt_results if r.status == "valid") + warn_count = sum(1 for r in et_results + mt_results if r.status == "warning") + error_count = sum(1 for r in et_results + mt_results if r.status == "error") + + lines.append("") + lines.append(f"Summary: {valid_count} valid | {warn_count} warning | {error_count} error") + + return "\n".join(lines) + + +@validate_app.command(name="registries") +def validate_registries() -> None: + """Validate user-override registries in .autoskillit/.""" + project_dir = Path.cwd() + + et_dir = project_dir / ".autoskillit" / "experiment-types" + mt_dir = project_dir / ".autoskillit" / "methodology-traditions" + + if not et_dir.exists() and not mt_dir.exists(): + print( + "No user registry directories found" + " (.autoskillit/experiment-types/ or .autoskillit/methodology-traditions/)." + ) + return + + valid_lenses = _get_valid_lens_slugs() + + et_results: list[ValidationResult] = [] + if et_dir.exists(): + for path in sorted(et_dir.glob("*.yaml")): + result = _validate_experiment_type_file(path, valid_lenses) + et_results.append(result) + + mt_results: list[ValidationResult] = [] + if mt_dir.exists(): + for path in sorted(mt_dir.glob("*.yaml")): + result = _validate_methodology_tradition_file(path) + mt_results.append(result) + + bundled_types = load_types_from_dir(BUNDLED_EXPERIMENT_TYPES_DIR) + fallback_errors = _check_fallback_uniqueness(et_results, bundled_types) + + if fallback_errors: + for result in et_results: + if result.spec_name: + for error in fallback_errors: + if f"'{result.spec_name}'" in error: + result.errors.append(error) + + print(_format_stdout_report(et_results, mt_results)) + + for result in et_results: + if result.errors: + _write_error_report(project_dir, result.filename, "experiment-type", result) + for result in mt_results: + if result.errors: + _write_error_report(project_dir, result.filename, "methodology-tradition", result) + + if any(r.status == "error" for r in et_results + mt_results): + raise SystemExit(1) diff --git a/src/autoskillit/cli/_workspace.py b/src/autoskillit/cli/_workspace.py index 119a3a4950..df2e618f80 100644 --- a/src/autoskillit/cli/_workspace.py +++ b/src/autoskillit/cli/_workspace.py @@ -74,7 +74,7 @@ async def run_workspace_clean( print() if not force: - from autoskillit.cli._timed_input import timed_prompt + from autoskillit.cli.ui._timed_input import timed_prompt suffix = "ies" if len(stale) != 1 else "y" answer = timed_prompt( @@ -152,7 +152,7 @@ def _safe_mtime(p: Path) -> float | None: print() if not force: - from autoskillit.cli._timed_input import timed_prompt + from autoskillit.cli.ui._timed_input import timed_prompt suffix = "ies" if len(stale_wts) != 1 else "y" answer = timed_prompt( diff --git a/src/autoskillit/cli/app.py b/src/autoskillit/cli/app.py index 81c60a3ee0..8078285e2c 100644 --- a/src/autoskillit/cli/app.py +++ b/src/autoskillit/cli/app.py @@ -5,36 +5,33 @@ import dataclasses import json import os -import random -import shutil -import subprocess import sys from datetime import UTC from pathlib import Path -from typing import TYPE_CHECKING, Annotated +from typing import Annotated import anyio - -from autoskillit.cli._serve_guard import serve_with_signal_guard - -if TYPE_CHECKING: - from autoskillit.recipe import Recipe, RecipeInfo - from cyclopts import App, Parameter -from autoskillit.cli._cook import cook as cook_interactive +from autoskillit.cli._features import features_app from autoskillit.cli._init_helpers import ( _MARKER_CONTENT, _check_secret_scanning, _generate_config_yaml, - _is_plugin_installed, _log_secret_scan_bypass, _prompt_test_command, _register_all, ) -from autoskillit.cli._terminal import terminal_guard -from autoskillit.core import ClaudeFlags, RecipeSource, atomic_write, pkg_root -from autoskillit.execution import build_interactive_cmd +from autoskillit.cli._serve_guard import serve_with_signal_guard +from autoskillit.cli._sessions import sessions_app +from autoskillit.cli._validate import validate_app +from autoskillit.cli.fleet import fleet_app +from autoskillit.cli.session._cook import cook as cook_interactive +from autoskillit.cli.session._order import _recipes_dir_for, order +from autoskillit.core import ( + RecipeSource, + atomic_write, +) app = App( name="autoskillit", @@ -50,6 +47,11 @@ app.command(skills_app) app.command(recipes_app) app.command(workspace_app) +app.command(fleet_app) +app.command(features_app) +app.command(sessions_app) +app.command(validate_app) +app.command(order) class CliError(Exception): @@ -107,7 +109,7 @@ def serve(*, verbose: Annotated[bool, Parameter(name=["--verbose", "-v"])] = Fal get_logger(__name__).info( "serve_startup", config_path=resolved_path, - test_check_command=cfg.test_check.command, + test_check_command=cfg.test_check.commands or cfg.test_check.command, ) # Inject config-derived protected branches so hook scripts read consistent values. @@ -119,12 +121,12 @@ def serve(*, verbose: Annotated[bool, Parameter(name=["--verbose", "-v"])] = Fal ",".join(cfg.safety.protected_branches), ) - plugin_dir: str | None = None if _is_plugin_installed() else str(pkg_root()) - ctx = make_context(cfg, plugin_dir=plugin_dir) + ctx = make_context(cfg) _initialize(ctx) try: - anyio.run(serve_with_signal_guard, mcp) + backend_options = {"use_uvloop": sys.platform != "win32"} + anyio.run(serve_with_signal_guard, mcp, backend="asyncio", backend_options=backend_options) except KeyboardInterrupt: pass # Ctrl+C before anyio loop starts — rare during heavy import phase @@ -210,7 +212,7 @@ def upgrade() -> None: @app.command def update() -> None: """Upgrade autoskillit to the latest version on your install's branch.""" - from autoskillit.cli._update import run_update_command + from autoskillit.cli.update._update import run_update_command run_update_command() @@ -218,7 +220,7 @@ def update() -> None: @app.command def doctor(*, output_json: bool = False): """Check project setup for common issues.""" - from autoskillit.cli._doctor import run_doctor + from autoskillit.cli.doctor import run_doctor run_doctor(output_json=output_json) @@ -233,7 +235,6 @@ def migrate(*, check: bool = False): Exit with code 1 if any recipes need migration (useful for CI). """ from autoskillit import __version__ - from autoskillit.core import RecipeSource from autoskillit.migration import applicable_migrations from autoskillit.recipe import list_recipes as _list_all_recipes @@ -391,19 +392,9 @@ def workspace_clean( @recipes_app.command(name="list") def recipes_list(): """List available recipes with sources.""" - from autoskillit.recipe import list_recipes - - recipes = list_recipes(Path.cwd()).items - if not recipes: - print("No recipes found.") - return + from autoskillit.cli.session._cook import _print_recipes_list - name_w = max(len(r.name) for r in recipes) - src_w = max(len(r.source) for r in recipes) - print(f"{'NAME':<{name_w}} {'SOURCE':<{src_w}} DESCRIPTION") - print(f"{'-' * name_w} {'-' * src_w} {'-' * 11}") - for r in recipes: - print(f"{r.name:<{name_w}} {r.source:<{src_w}} {r.description}") + _print_recipes_list() @recipes_app.command(name="show") @@ -418,12 +409,6 @@ def recipes_show(name: str): print(match.path.read_text()) -def _recipes_dir_for(info: RecipeInfo) -> Path: - if getattr(info, "source", None) == RecipeSource.BUILTIN: - return pkg_root() / "recipes" - return Path.cwd() / ".autoskillit" / "recipes" - - @recipes_app.command(name="render") def recipes_render(name: str | None = None) -> None: """Show pre-rendered diagram. Diagrams are generated by /render-recipe.""" @@ -441,297 +426,17 @@ def recipes_render(name: str | None = None) -> None: print(diagram if diagram else f"No diagram. Run /render-recipe {name}") -def _launch_cook_session( - system_prompt: str, - *, - initial_message: str | None = None, - extra_env: dict[str, str] | None = None, - resume_session_id: str | None = None, -) -> None: - """Launch an interactive Claude Code cook session with the given system prompt.""" - if shutil.which("claude") is None: - print("ERROR: 'claude' not found. Install: https://docs.anthropic.com/en/docs/claude-code") - sys.exit(1) - spec = build_interactive_cmd( - initial_prompt=initial_message, - resume_session_id=resume_session_id, - env_extras=extra_env, - ) - plugin_flags = [] if _is_plugin_installed() else [ClaudeFlags.PLUGIN_DIR, str(pkg_root())] - cmd = [ - *spec.cmd, - *plugin_flags, - ClaudeFlags.TOOLS, - "AskUserQuestion", - ClaudeFlags.APPEND_SYSTEM_PROMPT, - system_prompt, - ] - with terminal_guard(): - result = subprocess.run(cmd, env=spec.env) - if result.returncode != 0: - sys.exit(result.returncode) - - -def _get_subsets_needed(recipe: Recipe, disabled_subsets: frozenset[str]) -> frozenset[str]: - """Return the subset names from disabled_subsets that are actually referenced in recipe.""" - import re - - from autoskillit.recipe import make_validation_context, run_semantic_rules - - ctx = make_validation_context(recipe, disabled_subsets=disabled_subsets) - findings = run_semantic_rules(ctx) - needed: set[str] = set() - for f in findings: - if f.rule not in ("subset-disabled-skill", "subset-disabled-tool"): - continue - m = re.search(r"disabled subset '([^']+)'", f.message) - if m: - needed.add(m.group(1)) - return frozenset(needed) - - -def _get_packs_needed(recipe: Recipe, default_disabled_packs: frozenset[str]) -> frozenset[str]: - """Return pack names from default_disabled_packs that are required by recipe.""" - requires = frozenset(getattr(recipe, "requires_packs", [])) - return requires & default_disabled_packs - - -def _enable_packs_permanently(project_dir: Path, packs: frozenset[str]) -> None: - """Add specified packs to packs.enabled in .autoskillit/config.yaml.""" - from autoskillit.core import YAMLError, atomic_write, dump_yaml_str, load_yaml - - config_path = project_dir / ".autoskillit" / "config.yaml" - try: - data: dict = (load_yaml(config_path) or {}) if config_path.exists() else {} - except YAMLError: - data = {} - packs_section = data.setdefault("packs", {}) - current_enabled: list[str] = packs_section.get("enabled", []) - new_enabled = sorted(set(current_enabled) | packs) - packs_section["enabled"] = new_enabled - config_path.parent.mkdir(parents=True, exist_ok=True) - atomic_write(config_path, dump_yaml_str(data, default_flow_style=False, allow_unicode=True)) - print(f"Updated {config_path}: added {sorted(packs)} to packs.enabled") - - -def _enable_subsets_permanently(project_dir: Path, subsets: frozenset[str]) -> None: - """Remove specified subsets from subsets.disabled in .autoskillit/config.yaml.""" - from autoskillit.core import YAMLError, atomic_write, dump_yaml_str, load_yaml - - config_path = project_dir / ".autoskillit" / "config.yaml" - try: - data: dict = (load_yaml(config_path) or {}) if config_path.exists() else {} - except YAMLError: - data = {} - subsets_section = data.setdefault("subsets", {}) - current_disabled: list[str] = subsets_section.get("disabled", []) - subsets_section["disabled"] = [s for s in current_disabled if s not in subsets] - config_path.parent.mkdir(parents=True, exist_ok=True) - atomic_write(config_path, dump_yaml_str(data, default_flow_style=False, allow_unicode=True)) - print(f"Updated {config_path}: removed {sorted(subsets)} from subsets.disabled") - - @app.command(name="cook", alias="c") -def _cook_cmd(session_id: str | None = None, *, resume: bool = False) -> None: +def _cook_cmd( + session_id: str | None = None, *, resume: bool = False, profile: str | None = None +) -> None: """Launch an interactive Claude session with all skills and kitchen tools. - Use --resume to restore a previous session. Optionally provide a session ID - directly after --resume to target a specific session; omit for automatic - discovery of the most recent session. - """ - cook_interactive(resume=resume or (session_id is not None), session_id=session_id) - - -@app.command -def order(recipe: str | None = None, session_id: str | None = None, *, resume: bool = False): - """Launch an interactive Claude Code session to execute a recipe. - - Starts Claude Code with hard tool restrictions: only AskUserQuestion - (built-in) and AutoSkillit MCP tools are available. The session - discovers recipe content by calling load_recipe as its first action. - - Parameters - ---------- - recipe - Name of the recipe (from .autoskillit/recipes/). Prompts if omitted. - session_id - Explicit session ID to resume. Provide after the recipe name. - Implies --resume when non-None. - resume - When True, attempt to restore a previous session. + Use --resume to restore a previous session. Pass a session ID to target a + specific one, or omit for Claude Code's interactive picker. """ - from autoskillit.cli._mcp_names import detect_autoskillit_mcp_prefix - from autoskillit.cli._prompts import _build_orchestrator_prompt - from autoskillit.recipe import ( - find_recipe_by_name, - list_recipes, - load_recipe, - validate_recipe, - ) - - if os.environ.get("CLAUDECODE"): - print("ERROR: 'order' cannot run inside a Claude Code session.") - print("Run this command in a regular terminal.") - sys.exit(1) - - # Resolve resume session ID — must come before the 'if recipe is None:' block - from autoskillit.core import find_latest_session_id - - resume_session_id: str | None = None - if resume or session_id: - resume_session_id = session_id or find_latest_session_id() - if resume_session_id is None: - print("No previous session found. Starting a fresh session.") - - mcp_prefix = detect_autoskillit_mcp_prefix() - - from autoskillit.cli._timed_input import timed_prompt - - if recipe is None: - from autoskillit.cli._prompts import ( - _OPEN_KITCHEN_CHOICE, - _build_open_kitchen_prompt, - _resolve_recipe_input, - ) - - available = list_recipes(Path.cwd()).items - if not available: - print("No recipes found. Run 'autoskillit recipes list' to check.") - sys.exit(1) - print("Available recipes:") - print(" 0. Open kitchen (no recipe)") - for i, r in enumerate(available, 1): - print(f" {i}. {r.name}") - raw = timed_prompt( - f"Select recipe [0-{len(available)}]:", - default="", - timeout=120, - label="autoskillit order", - ) - resolved = _resolve_recipe_input(raw, available) - if resolved is _OPEN_KITCHEN_CHOICE: - from autoskillit.cli._prompts import _OPEN_KITCHEN_GREETINGS - - greeting = random.choice(_OPEN_KITCHEN_GREETINGS) - _launch_cook_session( - _build_open_kitchen_prompt(mcp_prefix=mcp_prefix), - initial_message=greeting, - resume_session_id=resume_session_id, - ) - return - elif resolved is None: - print(f"Invalid selection: '{raw}'") - sys.exit(1) - else: - if isinstance(resolved, str): - raise TypeError(f"Expected RecipeInfo, got str: {resolved!r}") - recipe = resolved.name - - from autoskillit.core import YAMLError - - _match = find_recipe_by_name(recipe, Path.cwd()) - if _match is None: - available = list_recipes(Path.cwd()).items - print(f"Recipe not found: '{recipe}'") - if available: - print("Available recipes:") - for r in available: - print(f" - {r.name}") - else: - print("No recipes found") - sys.exit(1) - # Validate recipe before launching session - try: - parsed = load_recipe(_match.path) - except YAMLError as exc: - print(f"Recipe YAML parse error: {exc}") - sys.exit(1) - except ValueError as exc: - print(f"Recipe structure error: {exc}") - sys.exit(1) - - errors = validate_recipe(parsed) - if errors: - print(f"Recipe '{recipe}' failed validation:") - for err in errors: - print(f" - {err}") - sys.exit(1) - - # Subset-disabled gate (REQ-VAL-004) - from autoskillit.config import load_config as _load_config - - _cfg = _load_config(Path.cwd()) - _disabled = frozenset(_cfg.subsets.disabled) - _extra_env: dict[str, str] = {} - - if _disabled: - _needed = _get_subsets_needed(parsed, _disabled) - if _needed: - subset_list = ", ".join(sorted(_needed)) - print(f"\nThis recipe requires subset(s): {subset_list}") - print(" 1. Enable temporarily (for this run only)") - print(" 2. Enable permanently (update .autoskillit/config.yaml)") - print(" 3. Cancel") - _choice = timed_prompt( - "Choose [1/2/3]:", default="3", timeout=120, label="autoskillit order" - ) - if _choice == "1": - _extra_env["AUTOSKILLIT_SUBSETS__DISABLED"] = "@json []" - elif _choice == "2": - _enable_subsets_permanently(Path.cwd(), _needed) - else: - return - - # Pack gate — check default-disabled packs (REQ-PACK-010) - from autoskillit.core import PACK_REGISTRY as _PACK_REGISTRY - - _default_disabled = frozenset( - tag for tag, pack_def in _PACK_REGISTRY.items() if not pack_def.default_enabled - ) - _pack_enabled = frozenset(_cfg.packs.enabled) - _default_disabled_packs = _default_disabled - _pack_enabled - - if _default_disabled_packs: - _packs_needed = _get_packs_needed(parsed, _default_disabled_packs) - if _packs_needed: - pack_list = ", ".join(sorted(_packs_needed)) - print(f"\nThis recipe requires pack(s): {pack_list}") - print(" 1. Enable temporarily (for this run only)") - print(" 2. Enable permanently (update .autoskillit/config.yaml)") - print(" 3. Cancel") - _pack_choice = timed_prompt( - "Choose [1/2/3]:", default="3", timeout=120, label="autoskillit order" - ) - if _pack_choice == "1": - import json as _json - - _extra_env["AUTOSKILLIT_PACKS__ENABLED"] = "@json " + _json.dumps( - sorted(_packs_needed) - ) - elif _pack_choice == "2": - _enable_packs_permanently(Path.cwd(), _packs_needed) - else: - return - - from autoskillit.cli._prompts import _COOK_GREETINGS, show_cook_preview - - show_cook_preview(recipe, parsed, _recipes_dir_for(_match), Path.cwd()) - - from autoskillit.cli._ansi import permissions_warning - - print(permissions_warning()) - confirm = timed_prompt( - "Launch session? [Enter/n]", default="", timeout=120, label="autoskillit order" - ) - if confirm.lower() in ("n", "no"): - return - - greeting = random.choice(_COOK_GREETINGS).format(recipe_name=recipe) - _launch_cook_session( - _build_orchestrator_prompt(recipe, mcp_prefix=mcp_prefix), - initial_message=greeting, - extra_env=_extra_env if _extra_env else None, - resume_session_id=resume_session_id, + cook_interactive( + resume=resume or (session_id is not None), session_id=session_id, profile=profile ) @@ -743,7 +448,7 @@ def main() -> None: evict_direct_mcp_entry(_user_claude_json_path()) - from autoskillit.cli._update_checks import run_update_checks + from autoskillit.cli.update._update_checks import run_update_checks run_update_checks() app() diff --git a/src/autoskillit/cli/doctor/CLAUDE.md b/src/autoskillit/cli/doctor/CLAUDE.md new file mode 100644 index 0000000000..ba44da0efd --- /dev/null +++ b/src/autoskillit/cli/doctor/CLAUDE.md @@ -0,0 +1,22 @@ +# doctor/ + +Diagnostic health checks for the autoskillit installation (28 checks). + +## Files + +| File | Purpose | +|------|---------| +| `__init__.py` | Orchestration facade: imports all check functions and assembles `run_doctor()` | +| `_doctor_types.py` | `DoctorResult` dataclass and `_NON_PROBLEM` severity set | +| `_doctor_config.py` | Project config, gitignore, and secret scanning checks | +| `_doctor_env.py` | Ambient env var leak detection (`SESSION_TYPE`, `CAMPAIGN_ID`) | +| `_doctor_features.py` | Feature flag dependency consistency and registry import checks | +| `_doctor_fleet.py` | Fleet infrastructure, campaign state, and sous-chef checks | +| `_doctor_hooks.py` | Hook registration, executability, and registry drift checks | +| `_doctor_install.py` | Install path, entry points, version drift, update dismissal checks | +| `_doctor_mcp.py` | MCP server registration, dual registration, plugin cache checks | +| `_doctor_runtime.py` | Quota cache schema version and claude process state checks | + +## Architecture Notes + +Hub-and-spoke: `__init__.py` is the single orchestration point. Each `_doctor_*` module is an independent check group returning `list[DoctorResult]`. Fleet checks are conditionally run only when the fleet feature is enabled. diff --git a/src/autoskillit/cli/doctor/__init__.py b/src/autoskillit/cli/doctor/__init__.py new file mode 100644 index 0000000000..c8c8df3f23 --- /dev/null +++ b/src/autoskillit/cli/doctor/__init__.py @@ -0,0 +1,201 @@ +"""Doctor command implementation — project setup checks.""" + +from __future__ import annotations + +import json +from pathlib import Path + +from autoskillit.cli._hooks import _claude_settings_path +from autoskillit.config import load_config +from autoskillit.core import Severity, get_logger, is_feature_enabled + +from ._doctor_config import ( + _check_config_layers_for_secrets, + _check_gitignore_completeness, + _check_project_config, + _check_secret_scanning_hook, +) +from ._doctor_env import ( + _check_ambient_campaign_id, + _check_ambient_session_type_fleet, + _check_ambient_session_type_orchestrator, + _check_ambient_session_type_skill, +) +from ._doctor_features import ( + _check_feature_dependencies, + _check_feature_registry_consistency, +) +from ._doctor_fleet import ( + _check_campaign_manifest_clone_dests, + _check_campaign_onboarding_hint, + _check_fleet_dispatch_guard_registered, + _check_script_version_health, + _check_sous_chef_bundled, + _check_stale_fleet_state, +) +from ._doctor_hooks import ( + _check_hook_health_all_scopes, + _check_hook_registration, + _check_hook_registry_drift_all_scopes, +) +from ._doctor_install import ( + _check_autoskillit_on_path, + _check_editable_install_source_exists, + _check_install_classification, + _check_source_version_drift, + _check_stale_entry_points, + _check_update_dismissal_state, +) +from ._doctor_mcp import ( + _check_cache_version_mismatch, + _check_dual_mcp_registration, + _check_installed_plugins_entry, + _check_mcp_server_registered, + _check_plugin_cache_exists, + _check_plugin_cache_integrity, + _check_stale_mcp_servers, +) +from ._doctor_runtime import ( + _check_claude_process_state_breakdown, + _check_quota_cache_schema, +) +from ._doctor_types import _NON_PROBLEM, DoctorResult + +logger = get_logger(__name__) + +__all__ = ["DoctorResult", "Severity", "run_doctor"] + + +def run_doctor(*, output_json: bool = False) -> None: + """Check project setup for common issues.""" + cfg = load_config(Path.cwd()) + results: list[DoctorResult] = [] + + # Check 1: Stale MCP servers — dead binaries or nonexistent paths + results.extend(_check_stale_mcp_servers(Path.home() / ".claude.json")) + + # Check 2: MCP server registered in ~/.claude.json or via plugin + results.append(_check_mcp_server_registered(claude_json_path=Path.home() / ".claude.json")) + + # Check 2b: Dual MCP registration — direct entry and marketplace plugin both present + results.append(_check_dual_mcp_registration()) + + # Check 2c: Plugin cache directory exists + results.append(_check_plugin_cache_exists()) + + # Check 2d: installed_plugins.json has autoskillit entry + results.append(_check_installed_plugins_entry()) + + # Check 2e: Plugin cache hooks.json paths resolve to real files + results.append(_check_plugin_cache_integrity()) + + # Check 3: autoskillit command on PATH + results.append(_check_autoskillit_on_path()) + + # Check 4: Config exists + results.append(_check_project_config()) + + # Check 4b: Config secrets placement + results.append(_check_config_layers_for_secrets()) + + # Check 5: Version consistency — cached plugin.json must match installed package + results.append(_check_cache_version_mismatch()) + + # Check 6: Hook executability — validates deployed scripts for all event types (all scopes) + results.extend(_check_hook_health_all_scopes(Path.cwd())) + + # Check 7: Hook registration in settings.json + results.append(_check_hook_registration(_claude_settings_path("user"))) + + # Check 7b: Hook registry drift (multi-scope) + results.extend(_check_hook_registry_drift_all_scopes(Path.cwd())) + + # Check 8: Script version health + results.append(_check_script_version_health()) + + # Check 9: gitignore completeness + results.append(_check_gitignore_completeness(Path.cwd())) + + # Check 10: Secret scanning hook + results.append(_check_secret_scanning_hook(Path.cwd())) + + # Check 11: Editable install source directory still exists + results.append(_check_editable_install_source_exists()) + + # Check 12: No stale autoskillit entry points outside ~/.local/bin + results.append(_check_stale_entry_points()) + + # Check 13: Source version drift (network, with disk-cache TTL fallback) + results.append(_check_source_version_drift()) + + # Check 14: Quota cache schema version + results.append(_check_quota_cache_schema()) + + # Check 15: claude process state breakdown + results.append(_check_claude_process_state_breakdown()) + + # Check 16: Install classification from direct_url.json + results.append(_check_install_classification()) + + # Check 17: Update-prompt dismissal state + results.append(_check_update_dismissal_state()) + + # -- Fleet doctor checks (ambient env + infrastructure health) -- + + # Check 18: Ambient SESSION_TYPE=skill leak detection + results.append(_check_ambient_session_type_skill()) + + # Check 19: Ambient SESSION_TYPE=orchestrator leak detection + results.append(_check_ambient_session_type_orchestrator()) + + # Check 20: Ambient SESSION_TYPE=fleet leak detection + results.append(_check_ambient_session_type_fleet()) + + # Check 21: Ambient CAMPAIGN_ID leak detection + results.append(_check_ambient_campaign_id()) + + # Check 22: Feature dependency consistency + results.append(_check_feature_dependencies(cfg.features)) + + # Check 23: Feature registry import consistency + results.append(_check_feature_registry_consistency()) + + # Checks 24–28: Fleet infrastructure — only when fleet feature is enabled + if is_feature_enabled("fleet", cfg.features, experimental_enabled=cfg.experimental_enabled): + # Check 24: Sous-chef skill directory exists + results.append(_check_sous_chef_bundled()) + + # Check 25: Fleet dispatch guard registered + results.append(_check_fleet_dispatch_guard_registered()) + + # Check 26: Stale fleet state (running > 7 days) + results.append(_check_stale_fleet_state()) + + # Check 27: Campaign onboarding hint + results.append(_check_campaign_onboarding_hint()) + + # Check 28: Campaign manifest clone destination collisions + results.append(_check_campaign_manifest_clone_dests()) + + # Output + if output_json: + print( + json.dumps( + { + "results": [ + {"severity": r.severity, "check": r.check, "message": r.message} + for r in results + ] + }, + indent=2, + ) + ) + else: + has_problems = any(r.severity not in _NON_PROBLEM for r in results) + if has_problems: + for r in results: + if r.severity not in _NON_PROBLEM: + print(f"{r.severity.upper()}: {r.message}") + else: + for r in results: + print(f"{r.severity}: {r.message}") diff --git a/src/autoskillit/cli/doctor/_doctor_config.py b/src/autoskillit/cli/doctor/_doctor_config.py new file mode 100644 index 0000000000..e5df2e2eb3 --- /dev/null +++ b/src/autoskillit/cli/doctor/_doctor_config.py @@ -0,0 +1,130 @@ +"""Config, gitignore, and secret scanning doctor checks.""" + +from __future__ import annotations + +from pathlib import Path + +from autoskillit.core import Severity, get_logger + +from ._doctor_types import DoctorResult + +logger = get_logger(__name__) + + +def _check_project_config(project_dir: Path | None = None) -> DoctorResult: + """Check that .autoskillit/config.yaml exists.""" + root = project_dir or Path.cwd() + if not (root / ".autoskillit" / "config.yaml").is_file(): + return DoctorResult( + Severity.WARNING, + "project_config", + "No project config found. Run: autoskillit init", + ) + return DoctorResult(Severity.OK, "project_config", "Project config exists") + + +def _check_config_layers_for_secrets(project_dir: Path | None = None) -> DoctorResult: + """Check all config.yaml layers for _SECRETS_ONLY_KEYS violations. + + Scans the user-level and project-level config.yaml files for any keys + that belong only in .secrets.yaml. Reports ERROR with exact fix guidance. + """ + from autoskillit.config import ConfigSchemaError, validate_layer_keys + from autoskillit.core import YAMLError, load_yaml + + root = project_dir or Path.cwd() + config_paths = [ + Path.home() / ".autoskillit" / "config.yaml", + root / ".autoskillit" / "config.yaml", + ] + for config_path in config_paths: + if not config_path.is_file(): + continue + try: + data = load_yaml(config_path) or {} + except YAMLError as exc: + return DoctorResult( + severity=Severity.WARNING, + check="config_secrets_placement", + message=f"Could not parse {str(config_path)!r} as YAML: {exc}", + ) + if not isinstance(data, dict): + continue + try: + validate_layer_keys(data, config_path, is_secrets_layer=False) + except ConfigSchemaError as exc: + return DoctorResult( + severity=Severity.ERROR, + check="config_secrets_placement", + message=str(exc), + ) + return DoctorResult( + severity=Severity.OK, + check="config_secrets_placement", + message="No secrets found in config.yaml layers", + ) + + +def _check_gitignore_completeness(project_dir: Path) -> DoctorResult: + """Check that every file in .autoskillit/ is gitignored or in the committed allowlist.""" + from autoskillit.core import _AUTOSKILLIT_GITIGNORE_ENTRIES, _COMMITTED_BY_DESIGN + + autoskillit_dir = project_dir / ".autoskillit" + gitignore_path = autoskillit_dir / ".gitignore" + if not autoskillit_dir.is_dir(): + return DoctorResult(Severity.OK, "gitignore_completeness", "No .autoskillit/ directory.") + if not gitignore_path.exists(): + return DoctorResult( + Severity.WARNING, + "gitignore_completeness", + ".autoskillit/.gitignore missing. Run 'autoskillit init'.", + ) + gitignore_content = gitignore_path.read_text(encoding="utf-8") + uncovered: list[str] = [] + for item in sorted(autoskillit_dir.iterdir()): + if item.name == ".gitignore": + continue + if item.name in _COMMITTED_BY_DESIGN: + continue + check_name = item.name + "/" if item.is_dir() else item.name + if check_name not in gitignore_content: + uncovered.append(item.name) + for entry in _AUTOSKILLIT_GITIGNORE_ENTRIES: + if entry not in gitignore_content: + entry_name = entry.rstrip("/") + if entry_name not in uncovered: + uncovered.append(entry_name) + if uncovered: + return DoctorResult( + Severity.WARNING, + "gitignore_completeness", + f"Files in .autoskillit/ not covered by .gitignore: {', '.join(uncovered)}. " + "Add to _AUTOSKILLIT_GITIGNORE_ENTRIES or _COMMITTED_BY_DESIGN.", + ) + return DoctorResult(Severity.OK, "gitignore_completeness", "All .autoskillit/ files covered.") + + +def _check_secret_scanning_hook(project_dir: Path) -> DoctorResult: + """Check that .pre-commit-config.yaml includes a known secret scanning hook.""" + from autoskillit.cli._init_helpers import _KNOWN_SCANNERS, _detect_secret_scanner + + if _detect_secret_scanner(project_dir): + return DoctorResult( + Severity.OK, + "secret_scanning_hook", + "Secret scanning hook detected in .pre-commit-config.yaml.", + ) + pre_commit_path = project_dir / ".pre-commit-config.yaml" + if not pre_commit_path.exists(): + msg = ( + "No .pre-commit-config.yaml found. AutoSkillit commits code automatically — " + "add a secret scanner (gitleaks, detect-secrets, trufflehog, or git-secrets) " + "to prevent credential leaks." + ) + else: + scanners = ", ".join(sorted(_KNOWN_SCANNERS)) + msg = ( + f".pre-commit-config.yaml exists but contains no known secret scanner " + f"({scanners}). Add one to prevent credential leaks." + ) + return DoctorResult(Severity.ERROR, "secret_scanning_hook", msg) diff --git a/src/autoskillit/cli/doctor/_doctor_env.py b/src/autoskillit/cli/doctor/_doctor_env.py new file mode 100644 index 0000000000..23aef8c14e --- /dev/null +++ b/src/autoskillit/cli/doctor/_doctor_env.py @@ -0,0 +1,94 @@ +"""Ambient environment variable leak doctor checks.""" + +from __future__ import annotations + +import os +import warnings + +from autoskillit.core import SESSION_TYPE_ENV_VAR, Severity, get_logger + +from ._doctor_types import DoctorResult + +logger = get_logger(__name__) + + +def _check_ambient_session_type_skill() -> DoctorResult: + """Detect ambient SESSION_TYPE=skill — common env leakage from fleet subprocesses.""" + raw = os.environ.get(SESSION_TYPE_ENV_VAR, "") + raw_lower = raw.lower() + if raw_lower == "leaf": + warnings.warn( + "AUTOSKILLIT_SESSION_TYPE='leaf' is deprecated. Use 'skill' instead.", + DeprecationWarning, + stacklevel=2, + ) + return DoctorResult( + Severity.WARNING, + "ambient_session_type_skill", + "Ambient SESSION_TYPE=leaf detected (deprecated). " + "Update to SESSION_TYPE=skill. Fleet sessions should set " + "SESSION_TYPE=skill only in launched subprocesses.", + ) + if raw_lower == "skill": + return DoctorResult( + Severity.WARNING, + "ambient_session_type_skill", + "Ambient SESSION_TYPE=skill detected. " + "Did you intend to set SESSION_TYPE=skill? Fleet sessions should set " + "SESSION_TYPE=skill only in launched subprocesses.", + ) + return DoctorResult( + Severity.OK, + "ambient_session_type_skill", + f"SESSION_TYPE={raw!r} (not skill)", + ) + + +def _check_ambient_session_type_orchestrator() -> DoctorResult: + """Detect ambient SESSION_TYPE=orchestrator outside a launched session.""" + raw = os.environ.get(SESSION_TYPE_ENV_VAR, "") + if raw.lower() == "orchestrator": + return DoctorResult( + Severity.WARNING, + "ambient_session_type_orchestrator", + "Ambient SESSION_TYPE=orchestrator outside of a launched session " + "— should only be set by autoskillit CLIs.", + ) + return DoctorResult( + Severity.OK, + "ambient_session_type_orchestrator", + "No ambient orchestrator session type", + ) + + +def _check_ambient_session_type_fleet() -> DoctorResult: + """Detect ambient SESSION_TYPE=fleet outside a fleet CLI session.""" + raw = os.environ.get(SESSION_TYPE_ENV_VAR, "") + if raw.lower() == "fleet": + return DoctorResult( + Severity.WARNING, + "ambient_session_type_fleet", + "Ambient SESSION_TYPE=fleet outside of a fleet CLI session " + "— highest-privilege env, suspicious.", + ) + return DoctorResult( + Severity.OK, + "ambient_session_type_fleet", + "No ambient fleet session type", + ) + + +def _check_ambient_campaign_id() -> DoctorResult: + """Detect ambient CAMPAIGN_ID — should only be set by dispatch_food_truck.""" + campaign_id = os.environ.get("AUTOSKILLIT_CAMPAIGN_ID", "") + if campaign_id: + return DoctorResult( + Severity.WARNING, + "ambient_campaign_id", + f"Ambient CAMPAIGN_ID={campaign_id} — should only be set by dispatch_food_truck.", + ) + return DoctorResult( + Severity.OK, + "ambient_campaign_id", + "No ambient CAMPAIGN_ID", + ) diff --git a/src/autoskillit/cli/doctor/_doctor_features.py b/src/autoskillit/cli/doctor/_doctor_features.py new file mode 100644 index 0000000000..72cd91c17f --- /dev/null +++ b/src/autoskillit/cli/doctor/_doctor_features.py @@ -0,0 +1,65 @@ +"""Feature flag dependency and registry consistency doctor checks.""" + +from __future__ import annotations + +import autoskillit.core as _core +from autoskillit.core import Severity, get_logger + +from ._doctor_types import DoctorResult + +logger = get_logger(__name__) + + +def _check_feature_dependencies(features: dict[str, bool]) -> DoctorResult: + """Verify all enabled features have their dependencies satisfied.""" + violations: list[str] = [] + for name, defn in _core.FEATURE_REGISTRY.items(): + if not features.get(name, defn.default_enabled) or not defn.depends_on: + continue + for dep in defn.depends_on: + dep_defn = _core.FEATURE_REGISTRY.get(dep) + dep_default = dep_defn.default_enabled if dep_defn is not None else False + if not features.get(dep, dep_default): + violations.append( + f"Feature '{name}' requires '{dep}' but '{dep}' is disabled. " + f"Enable '{dep}' in features config first." + ) + if violations: + return DoctorResult( + severity=Severity.ERROR, + check="feature_dependencies", + message="; ".join(violations), + ) + return DoctorResult( + severity=Severity.OK, + check="feature_dependencies", + message="All feature dependencies satisfied", + ) + + +def _check_feature_registry_consistency() -> DoctorResult: + """Verify FEATURE_REGISTRY import_package entries resolve to importable modules.""" + import importlib + + failures: list[str] = [] + for name, defn in _core.FEATURE_REGISTRY.items(): + if not defn.import_package: + continue + try: + importlib.import_module(defn.import_package) + except ImportError as exc: + failures.append( + f"Feature '{name}' import_package {defn.import_package!r} " + f"cannot be imported: {exc}" + ) + if failures: + return DoctorResult( + severity=Severity.ERROR, + check="feature_registry_consistency", + message="; ".join(failures), + ) + return DoctorResult( + severity=Severity.OK, + check="feature_registry_consistency", + message="All FEATURE_REGISTRY import packages resolve", + ) diff --git a/src/autoskillit/cli/doctor/_doctor_fleet.py b/src/autoskillit/cli/doctor/_doctor_fleet.py new file mode 100644 index 0000000000..e3de0e73ce --- /dev/null +++ b/src/autoskillit/cli/doctor/_doctor_fleet.py @@ -0,0 +1,215 @@ +"""Fleet infrastructure and campaign state doctor checks.""" + +from __future__ import annotations + +import json +from pathlib import Path + +from autoskillit.core import Severity, get_logger, pkg_root +from autoskillit.hook_registry import canonical_script_basenames + +from ._doctor_types import DoctorResult + +logger = get_logger(__name__) + +_STALE_THRESHOLD_DAYS = 7 + + +def _check_script_version_health(project_dir: Path | None = None) -> DoctorResult: + """Check recipe migration status for all project pipeline scripts.""" + from autoskillit import __version__ + from autoskillit.core import RecipeSource + from autoskillit.migration import FailureStore, default_store_path + from autoskillit.recipe import list_recipes as _list_all_recipes + + root = project_dir or Path.cwd() + _all_result = _list_all_recipes(root) + scripts_result_items = [r for r in _all_result.items if r.source == RecipeSource.PROJECT] + if not scripts_result_items: + return DoctorResult(Severity.OK, "script_version_health", "No pipeline scripts found") + + from packaging.version import Version + + failure_store = FailureStore(default_store_path(root)) + known_failures = failure_store.load() + + failed_migrations: list[str] = [] + outdated: list[str] = [] + for script in scripts_result_items: + if script.name in known_failures: + f = known_failures[script.name] + failed_migrations.append(f"{script.name} (failed after {f.retries_attempted} retries)") + elif script.version is None or Version(script.version) < Version(__version__): + outdated.append(script.name) + + if failed_migrations: + return DoctorResult( + Severity.ERROR, + "script_version_health", + "Migration failed — manual intervention required: " + ", ".join(failed_migrations), + ) + if outdated: + return DoctorResult( + Severity.WARNING, + "script_version_health", + "Outdated recipes: " + ", ".join(outdated) + ". Will be auto-migrated on next load.", + ) + return DoctorResult(Severity.OK, "script_version_health", "All recipes up to date") + + +def _check_sous_chef_bundled() -> DoctorResult: + """Check that the sous-chef skill directory exists.""" + sous_chef_dir = pkg_root() / "skills" / "sous-chef" + if sous_chef_dir.is_dir(): + return DoctorResult( + Severity.OK, + "sous_chef_bundled", + "Sous-chef skill directory exists", + ) + return DoctorResult( + Severity.ERROR, + "sous_chef_bundled", + f"Sous-chef skill not found at {sous_chef_dir}/. Fatal prerequisite.", + ) + + +def _check_fleet_dispatch_guard_registered() -> DoctorResult: + """Check that fleet dispatch guard is registered in HOOK_REGISTRY.""" + from autoskillit.hook_registry import HOOKS_DIR + + check_name = "fleet_dispatch_guard_registered" + if "guards/fleet_dispatch_guard.py" not in canonical_script_basenames(): + return DoctorResult( + Severity.ERROR, + check_name, + "Fleet dispatch guard not registered in hooks.json. " + "Run: autoskillit config sync-hooks", + ) + if not (HOOKS_DIR / "guards" / "fleet_dispatch_guard.py").is_file(): + return DoctorResult( + Severity.ERROR, + check_name, + "Fleet dispatch guard registered but script file missing on disk. " + "Run: autoskillit install", + ) + return DoctorResult( + Severity.OK, + check_name, + "Fleet dispatch guard registered and accessible", + ) + + +def _check_stale_fleet_state(project_dir: Path | None = None) -> DoctorResult: + """Check for stale campaign state files with running dispatches > 7 days old.""" + import time + + root = project_dir or Path.cwd() + fleet_dir = root / ".autoskillit" / "temp" / "fleet" + check_name = "stale_fleet_state" + if not fleet_dir.is_dir(): + return DoctorResult(Severity.OK, check_name, "No fleet state directory") + threshold = time.time() - (_STALE_THRESHOLD_DAYS * 86400) + stale_paths: list[str] = [] + for campaign_dir in fleet_dir.iterdir(): + if not campaign_dir.is_dir(): + continue + state_file = campaign_dir / "state.json" + if not state_file.is_file(): + continue + if state_file.stat().st_mtime > threshold: + continue + try: + data = json.loads(state_file.read_text(encoding="utf-8")) + dispatches = data.get("dispatches", []) + has_running = any(d.get("status") == "running" for d in dispatches) + if has_running: + stale_paths.append(str(state_file)) + except (json.JSONDecodeError, OSError, KeyError): + continue + if stale_paths: + paths_str = ", ".join(stale_paths) + return DoctorResult( + Severity.WARNING, + check_name, + f"Stale campaign state > {_STALE_THRESHOLD_DAYS} days: {paths_str}. " + f"Run: autoskillit fleet status --reap", + ) + return DoctorResult(Severity.OK, check_name, "No stale fleet state files") + + +def _check_campaign_onboarding_hint(project_dir: Path | None = None) -> DoctorResult: + """Hint when no campaign recipes exist yet.""" + root = project_dir or Path.cwd() + campaigns_dir = root / ".autoskillit" / "recipes" / "campaigns" + check_name = "campaign_onboarding_hint" + if not campaigns_dir.is_dir(): + return DoctorResult( + Severity.INFO, + check_name, + "No campaign recipes found. Get started: /autoskillit:make-campaign ", + ) + yaml_files = [ + f for f in campaigns_dir.iterdir() if f.suffix in (".yaml", ".yml") and f.is_file() + ] + if not yaml_files: + return DoctorResult( + Severity.INFO, + check_name, + "No campaign recipes found. Get started: /autoskillit:make-campaign ", + ) + return DoctorResult( + Severity.OK, + check_name, + f"{len(yaml_files)} campaign recipe(s) found", + ) + + +def _check_campaign_manifest_clone_dests(project_dir: Path | None = None) -> DoctorResult: + """Check that dispatches within campaign recipes use unique clone destinations.""" + from autoskillit.core import YAMLError, load_yaml + + root = project_dir or Path.cwd() + campaigns_dir = root / ".autoskillit" / "recipes" / "campaigns" + check_name = "campaign_manifest_clone_dests" + if not campaigns_dir.is_dir(): + return DoctorResult(Severity.OK, check_name, "No campaigns directory") + yaml_files = [ + f for f in campaigns_dir.iterdir() if f.suffix in (".yaml", ".yml") and f.is_file() + ] + if not yaml_files: + return DoctorResult(Severity.OK, check_name, "No campaign recipes to check") + seen_paths: dict[str, list[str]] = {} + for yaml_file in yaml_files: + try: + data = load_yaml(yaml_file) + except (YAMLError, OSError): + continue + if not isinstance(data, dict): + continue + dispatches = data.get("dispatches", []) + if not isinstance(dispatches, list): + continue + recipe_name = data.get("name", yaml_file.stem) + for dispatch in dispatches: + if not isinstance(dispatch, dict): + continue + ingredients = dispatch.get("ingredients", {}) + if not isinstance(ingredients, dict): + continue + clone_path = ingredients.get("clone_path", "") + if clone_path: + key = str(clone_path) + label = f"{recipe_name}:{dispatch.get('name', '?')}" + seen_paths.setdefault(key, []).append(label) + duplicates = {path: users for path, users in seen_paths.items() if len(users) > 1} + if duplicates: + dup_details = "; ".join( + f"{path} (used by {', '.join(users)})" for path, users in duplicates.items() + ) + return DoctorResult( + Severity.WARNING, + check_name, + f"Dispatches share a literal clone destination: {dup_details}. " + f"Use unique clone paths per dispatch.", + ) + return DoctorResult(Severity.OK, check_name, "All dispatch clone destinations unique") diff --git a/src/autoskillit/cli/doctor/_doctor_hooks.py b/src/autoskillit/cli/doctor/_doctor_hooks.py new file mode 100644 index 0000000000..0cf9c7ec4a --- /dev/null +++ b/src/autoskillit/cli/doctor/_doctor_hooks.py @@ -0,0 +1,129 @@ +"""Hook registration and health doctor checks.""" + +from __future__ import annotations + +from pathlib import Path + +from autoskillit.core import Severity, get_logger +from autoskillit.hook_registry import ( + _count_hook_registry_drift, + canonical_script_basenames, + find_broken_hook_scripts, +) + +from ._doctor_types import DoctorResult + +logger = get_logger(__name__) + + +def _check_hook_registration(settings_path: Path) -> DoctorResult: + from autoskillit.cli._hooks import _load_settings_data + + data = _load_settings_data(settings_path) + commands = [ + hook.get("command", "") + for event_entries in data.get("hooks", {}).values() + if isinstance(event_entries, list) + for entry in event_entries + for hook in entry.get("hooks", []) + ] + registered_logical = {cmd.split()[-1] for cmd in commands if "_dispatch.py" in cmd} + missing = [ + script + for script in canonical_script_basenames() + if not any(script in cmd for cmd in commands) + and script.removesuffix(".py") not in registered_logical + ] + if missing: + return DoctorResult( + severity=Severity.WARNING, + check="hook_registration", + message=f"Missing hooks: {', '.join(missing)}. Run 'autoskillit init'.", + ) + return DoctorResult( + severity=Severity.OK, + check="hook_registration", + message="All HOOK_REGISTRY scripts present in settings.json.", + ) + + +def _check_hook_registry_drift( + settings_path: Path, scope_label: str | None = None +) -> DoctorResult: + """Compare generate_hooks_json() with what is deployed in settings.json.""" + result = _count_hook_registry_drift(settings_path) + if result.orphaned > 0: + ghost_scripts = sorted(result.orphaned_cmds) + msg = ( + f"Orphaned hook entries detected: {', '.join(ghost_scripts)}. " + f"These scripts are missing from HOOK_REGISTRY but present in " + f"settings.json — every matching tool call will be denied with ENOENT. " + f"Run 'autoskillit install' to regenerate hooks." + ) + if scope_label: + msg = f"[{scope_label}] {msg}" + return DoctorResult(Severity.ERROR, "hook_registry_drift", msg) + if result.missing > 0: + msg = ( + f"Hook registry has changed since last install. " + f"Run 'autoskillit install' to deploy {result.missing} new/changed hook(s)." + ) + if scope_label: + msg = f"[{scope_label}] {msg}" + return DoctorResult( + severity=Severity.WARNING, + check="hook_registry_drift", + message=msg, + ) + msg = "Deployed hooks match HOOK_REGISTRY." + if scope_label: + msg = f"[{scope_label}] {msg}" + return DoctorResult( + severity=Severity.OK, + check="hook_registry_drift", + message=msg, + ) + + +def _check_hook_health(settings_path: Path) -> DoctorResult: + """Verify all deployed hook scripts in a single settings file exist on disk.""" + broken = find_broken_hook_scripts(settings_path) + if broken: + return DoctorResult( + severity=Severity.ERROR, + check="hook_health", + message=f"Hook scripts not found: {', '.join(broken)}", + ) + return DoctorResult(Severity.OK, "hook_health", "All hook scripts accessible") + + +def _check_hook_registry_drift_all_scopes(project_root: Path | None = None) -> list[DoctorResult]: + """Check hook registry drift across ALL scopes.""" + from autoskillit.hook_registry import iter_all_scope_paths + + results: list[DoctorResult] = [] + for scope_label, settings_path in iter_all_scope_paths(project_root): + results.append(_check_hook_registry_drift(settings_path, scope_label=scope_label)) + return results + + +def _check_hook_health_all_scopes(project_root: Path | None = None) -> list[DoctorResult]: + """Verify all deployed hook scripts exist on disk across ALL scopes.""" + from autoskillit.hook_registry import iter_all_scope_paths + + results: list[DoctorResult] = [] + for scope_label, settings_path in iter_all_scope_paths(project_root): + broken = find_broken_hook_scripts(settings_path) + if broken: + results.append( + DoctorResult( + severity=Severity.ERROR, + check="hook_health", + message=f"[{scope_label}] Hook scripts not found: {', '.join(broken)}", + ) + ) + if not results: + results.append( + DoctorResult(Severity.OK, "hook_health", "All hook scripts accessible (all scopes)") + ) + return results diff --git a/src/autoskillit/cli/doctor/_doctor_install.py b/src/autoskillit/cli/doctor/_doctor_install.py new file mode 100644 index 0000000000..c10f666741 --- /dev/null +++ b/src/autoskillit/cli/doctor/_doctor_install.py @@ -0,0 +1,226 @@ +"""Installation, entry points, and version drift doctor checks.""" + +from __future__ import annotations + +import json +import shutil +import subprocess +import urllib.parse +from pathlib import Path + +from autoskillit.core import Severity, get_logger + +from ._doctor_types import DoctorResult + +logger = get_logger(__name__) + + +def _format_upgrade_cmd(info: object) -> str: + from autoskillit.cli._install_info import upgrade_command + + cmd = upgrade_command(info) # type: ignore[arg-type] + return " ".join(cmd) if cmd else "autoskillit update" + + +def _check_autoskillit_on_path() -> DoctorResult: + """Check that the autoskillit command is available on PATH.""" + if shutil.which("autoskillit") is None: + return DoctorResult( + Severity.WARNING, + "autoskillit_on_path", + "'autoskillit' command not found on PATH.", + ) + return DoctorResult(Severity.OK, "autoskillit_on_path", "autoskillit command found on PATH") + + +def _check_editable_install_source_exists() -> DoctorResult: + """Detect editable autoskillit installs whose source directory no longer exists.""" + import importlib.metadata as meta + + check_name = "editable_install_source_exists" + try: + dist = meta.Distribution.from_name("autoskillit") + except meta.PackageNotFoundError: + return DoctorResult(Severity.OK, check_name, "autoskillit not installed in this env") + + direct_url_text = dist.read_text("direct_url.json") + if not direct_url_text: + return DoctorResult(Severity.OK, check_name, "Not an editable install") + + try: + direct_url = json.loads(direct_url_text) + except json.JSONDecodeError: + return DoctorResult(Severity.OK, check_name, "direct_url.json unreadable — skipped") + + is_editable = ( + direct_url.get("dir_info", {}).get("editable") is True + or direct_url.get("editable") is True + ) + if not is_editable: + return DoctorResult(Severity.OK, check_name, "Not an editable install") + + url = direct_url.get("url", "") + src_path = urllib.parse.urlparse(url).path if url.startswith("file://") else "" + if not src_path or Path(src_path).exists(): + return DoctorResult(Severity.OK, check_name, "Editable install source directory exists") + + return DoctorResult( + Severity.ERROR, + check_name, + f"autoskillit is installed from a deleted directory: {src_path}. " + f"Fix: restore the editable source directory or re-run installation.", + ) + + +def _check_stale_entry_points() -> DoctorResult: + """Detect autoskillit binaries on PATH outside ~/.local/bin (stale/poisoned installs).""" + check_name = "stale_entry_points" + primary = shutil.which("autoskillit") + if not primary: + return DoctorResult(Severity.OK, check_name, "autoskillit not found on PATH") + + try: + result = subprocess.run( + ["which", "-a", "autoskillit"], + capture_output=True, + text=True, + timeout=5, + ) + all_paths = [p.strip() for p in result.stdout.splitlines() if p.strip()] + except (subprocess.TimeoutExpired, FileNotFoundError, OSError): + all_paths = [primary] + + expected_prefix = Path.home() / ".local" + stale = [p for p in all_paths if not Path(p).is_relative_to(expected_prefix)] + if not stale: + return DoctorResult(Severity.OK, check_name, "No stale autoskillit entry points found") + + stale_list = ", ".join(stale) + from autoskillit.cli._install_info import detect_install + + _info = detect_install() + _cmd_str = _format_upgrade_cmd(_info) + return DoctorResult( + Severity.WARNING, + check_name, + f"Found autoskillit entry point(s) outside ~/.local/bin: {stale_list}. " + f"These may be stale editable installs. " + f"Fix: {_cmd_str} && autoskillit install", + ) + + +def _check_source_version_drift(home: Path | None = None) -> DoctorResult: + """Network source-drift check. + + Compares the installed commit SHA against the current HEAD of the branch + the binary was installed from. Uses a network request to get the latest + SHA (with disk-cache TTL fallback). + """ + check_name = "source_version_drift" + _home = home or Path.home() + + try: + from autoskillit.cli._install_info import InstallType, detect_install + from autoskillit.cli.update._update_checks import resolve_reference_sha + + info = detect_install() + + if info.install_type == InstallType.LOCAL_EDITABLE: + return DoctorResult( + Severity.OK, check_name, "Local editable install — drift check not applicable" + ) + + if info.install_type in (InstallType.UNKNOWN, InstallType.LOCAL_PATH): + return DoctorResult( + Severity.OK, + check_name, + "Not a source-tracked install — drift check not applicable", + ) + + ref_sha = resolve_reference_sha(info, _home, network=True) + + if ref_sha is None: + return DoctorResult( + Severity.OK, + check_name, + "Source drift reference SHA unavailable — check network connectivity", + ) + + if info.commit_id == ref_sha: + return DoctorResult(Severity.OK, check_name, "No source drift detected") + + installed_short = (info.commit_id or "unknown")[:8] + ref_short = ref_sha[:8] + _cmd_str = _format_upgrade_cmd(info) + return DoctorResult( + Severity.WARNING, + check_name, + f"Source drift: installed={installed_short}, reference={ref_short}. " + f"Run: {_cmd_str} && autoskillit install", + ) + + except Exception: + logger.debug("Source drift check failed", exc_info=True) + return DoctorResult( + Severity.OK, check_name, "Source drift check skipped (unexpected error)" + ) + + +def _check_install_classification() -> DoctorResult: + """Classify the current autoskillit install type via direct_url.json.""" + check_name = "install_classification" + try: + from autoskillit.cli._install_info import InstallType, detect_install + + info = detect_install() + if info.install_type == InstallType.UNKNOWN: + return DoctorResult( + Severity.WARNING, + check_name, + "install type could not be detected from direct_url.json", + ) + commit_short = (info.commit_id or "")[:8] + return DoctorResult( + Severity.OK, + check_name, + f"install_type={info.install_type}, " + f"requested_revision={info.requested_revision}, " + f"commit_id={commit_short}", + ) + except Exception: + logger.debug("Install classification check failed", exc_info=True) + return DoctorResult( + Severity.OK, check_name, "Install classification check skipped (unexpected error)" + ) + + +def _check_update_dismissal_state(home: Path | None = None) -> DoctorResult: + """Report the current update-prompt dismissal state.""" + check_name = "update_dismissal_state" + _home = home or Path.home() + try: + from autoskillit.cli._install_info import detect_install, dismissal_window + from autoskillit.cli.update._update_checks import _read_dismiss_state + + state = _read_dismiss_state(_home) + entry = state.get("update_prompt") + if not isinstance(entry, dict) or "dismissed_at" not in entry: + return DoctorResult(Severity.OK, check_name, "No active dismissal") + + from datetime import datetime + + info = detect_install() + window = dismissal_window(info) + dismissed_at = datetime.fromisoformat(str(entry["dismissed_at"])) + expiry = (dismissed_at + window).strftime("%Y-%m-%d") + conditions = entry.get("conditions", []) + return DoctorResult( + Severity.OK, + check_name, + f"update_prompt dismissed until {expiry}; conditions={conditions}", + ) + except Exception: + logger.debug("Update dismissal state check failed", exc_info=True) + return DoctorResult( + Severity.OK, check_name, "Update dismissal state check skipped (unexpected error)" + ) diff --git a/src/autoskillit/cli/doctor/_doctor_mcp.py b/src/autoskillit/cli/doctor/_doctor_mcp.py new file mode 100644 index 0000000000..51e0b75b1d --- /dev/null +++ b/src/autoskillit/cli/doctor/_doctor_mcp.py @@ -0,0 +1,259 @@ +"""MCP server registration and plugin cache doctor checks.""" + +from __future__ import annotations + +import json +import shutil +import subprocess +from pathlib import Path + +from autoskillit.core import DIRECT_INSTALL_CACHE_SUBDIR, Severity, build_claude_env, get_logger + +from ._doctor_types import DoctorResult + +logger = get_logger(__name__) + + +def _check_stale_mcp_servers(claude_json_path: Path | None = None) -> list[DoctorResult]: + """Check ~/.claude.json for stale autoskillit* MCP server entries with dead paths.""" + _path = claude_json_path or (Path.home() / ".claude.json") + if not _path.is_file(): + return [DoctorResult(Severity.OK, "stale_mcp_servers", "No stale MCP servers detected")] + + try: + data = json.loads(_path.read_text()) + except (json.JSONDecodeError, OSError) as exc: + return [ + DoctorResult( + Severity.ERROR, + "stale_mcp_servers", + f"~/.claude.json could not be parsed: {exc}", + ) + ] + + servers = data.get("mcpServers", {}) + stale_msgs: list[str] = [] + for name, entry in servers.items(): + cmd = entry.get("command", "") + if not cmd: + continue + cmd_path = Path(cmd) + if cmd_path.is_absolute() and not cmd_path.exists(): + stale_msgs.append( + f"MCP server '{name}' has dead command path: {cmd}. " + f"Remove with: claude mcp remove --scope user {name}" + ) + elif not cmd_path.is_absolute() and shutil.which(cmd) is None: + stale_msgs.append( + f"MCP server '{name}' command not found: {cmd}. " + f"Remove with: claude mcp remove --scope user {name}" + ) + + if stale_msgs: + return [DoctorResult(Severity.ERROR, "stale_mcp_servers", msg) for msg in stale_msgs] + return [DoctorResult(Severity.OK, "stale_mcp_servers", "No stale MCP servers detected")] + + +def _check_mcp_server_registered(claude_json_path: Path | None = None) -> DoctorResult: + """Check that autoskillit MCP server is registered (via mcpServers or plugin).""" + if claude_json_path is None: + claude_json_path = Path.home() / ".claude.json" + + if claude_json_path.exists(): + try: + data = json.loads(claude_json_path.read_text()) + if "autoskillit" in data.get("mcpServers", {}): + return DoctorResult( + severity=Severity.OK, + check="mcp_server_registered", + message="autoskillit registered in mcpServers", + ) + except OSError as exc: + return DoctorResult( + severity=Severity.ERROR, + check="mcp_server_registered", + message=f"~/.claude.json could not be read: {exc}", + ) + except json.JSONDecodeError as exc: + return DoctorResult( + severity=Severity.ERROR, + check="mcp_server_registered", + message=f"~/.claude.json is not valid JSON: {exc}", + ) + + _plugin_check_detail = "" + try: + result = subprocess.run( + ["claude", "plugin", "list"], + capture_output=True, + text=True, + timeout=10, + env=build_claude_env(), + ) + if result.returncode == 0 and "autoskillit" in result.stdout: + return DoctorResult( + severity=Severity.OK, + check="mcp_server_registered", + message="autoskillit registered as Claude plugin", + ) + except (FileNotFoundError, subprocess.TimeoutExpired, OSError) as exc: + _plugin_check_detail = f" (claude plugin list unavailable: {type(exc).__name__})" + else: + _plugin_check_detail = "" + + return DoctorResult( + severity=Severity.WARNING, + check="mcp_server_registered", + message=( + "autoskillit not registered. Run 'autoskillit install' to install as a plugin, " + "or 'autoskillit init' to register in mcpServers." + _plugin_check_detail + ), + ) + + +def _check_dual_mcp_registration( + claude_json_path: Path | None = None, + plugins_json_path: Path | None = None, +) -> DoctorResult: + """Check that autoskillit is not registered both as a direct entry and as a marketplace plugin. + + Returns WARNING if both registrations are simultaneously present (split-brain condition). + Fail-open: unreadable files → cannot confirm dual registration, return OK. + """ + from autoskillit.cli._init_helpers import _check_dual_mcp_files + + _claude_json = claude_json_path or (Path.home() / ".claude.json") + _plugins_json = plugins_json_path or ( + Path.home() / ".claude" / "plugins" / "installed_plugins.json" + ) + if _check_dual_mcp_files(_claude_json, _plugins_json): + return DoctorResult( + severity=Severity.WARNING, + check="dual_mcp_registration", + message=( + "autoskillit is registered both as a direct mcpServers entry " + "(~/.claude.json) and as a marketplace plugin. This spawns two " + "independent server processes per session with split gate state. " + "Run `autoskillit install` to remove the stale direct entry." + ), + ) + return DoctorResult( + severity=Severity.OK, + check="dual_mcp_registration", + message="", + ) + + +def _check_plugin_cache_exists(cache_dir: Path | None = None) -> DoctorResult: + """Check that the plugin cache directory exists.""" + from autoskillit.cli._install_info import InstallType, detect_install + + info = detect_install() + if info.install_type == InstallType.LOCAL_EDITABLE: + return DoctorResult( + Severity.OK, + "plugin_cache_exists", + "Plugin cache check skipped (editable dev install)", + ) + + _cache_dir = cache_dir or ( + Path.home() / ".claude" / "plugins" / "cache" / DIRECT_INSTALL_CACHE_SUBDIR / "autoskillit" + ) + if _cache_dir.is_dir(): + return DoctorResult( + Severity.OK, + "plugin_cache_exists", + "Plugin cache directory exists", + ) + return DoctorResult( + Severity.WARNING, + "plugin_cache_exists", + f"Plugin cache directory missing: {_cache_dir}. Run `autoskillit install` to recreate it.", + ) + + +def _check_installed_plugins_entry(plugins_json_path: Path | None = None) -> DoctorResult: + """Check that installed_plugins.json contains the autoskillit entry.""" + from autoskillit.cli._installed_plugins import InstalledPluginsFile + + store = InstalledPluginsFile(plugins_json_path) + if not store.path.exists(): + return DoctorResult( + Severity.WARNING, + "installed_plugins_entry", + "installed_plugins.json not found. Run `autoskillit install`.", + ) + if store.contains("autoskillit@autoskillit-local"): + return DoctorResult( + Severity.OK, + "installed_plugins_entry", + "autoskillit entry present in installed_plugins.json", + ) + return DoctorResult( + Severity.WARNING, + "installed_plugins_entry", + "autoskillit entry missing from installed_plugins.json. Run `autoskillit install` to fix.", + ) + + +def _check_plugin_cache_integrity(cache_dir: Path | None = None) -> DoctorResult: + """Validate that plugin cache hooks.json paths resolve to real files.""" + from autoskillit.hook_registry import validate_plugin_cache_hooks + + broken = validate_plugin_cache_hooks(cache_dir=cache_dir) + if broken: + broken_str = ", ".join(broken) + return DoctorResult( + Severity.ERROR, + "plugin_cache_integrity", + f"Plugin cache hooks.json has {len(broken)} broken path(s): {broken_str}. " + f"Run `autoskillit install` to rebuild the cache.", + ) + return DoctorResult( + Severity.OK, + "plugin_cache_integrity", + "Plugin cache hook paths are valid", + ) + + +def _check_cache_version_mismatch(cache_dir: Path | None = None) -> DoctorResult: + """Check plugin cache version. ERROR if kitchen is open and versions mismatch.""" + from autoskillit.core import any_kitchen_open + from autoskillit.version import version_info + + _cache_plugin_dir = cache_dir or ( + Path.home() / ".claude" / "plugins" / "cache" / DIRECT_INSTALL_CACHE_SUBDIR / "autoskillit" + ) + try: + vi = version_info(plugin_dir=str(_cache_plugin_dir)) + except Exception as exc: + logger.warning("version_info_failed", plugin_dir=str(_cache_plugin_dir), exc_info=True) + return DoctorResult( + Severity.ERROR, + "version_consistency", + f"Could not read plugin cache version info: {exc}. " + "Run `autoskillit install` to rebuild.", + ) + if vi["match"]: + return DoctorResult( + Severity.OK, + "version_consistency", + f"Version {vi['package_version']} — plugin cache is current", + ) + mismatch_msg = ( + f"Plugin cache version {vi['plugin_json_version']!r} does not match " + f"installed package {vi['package_version']!r}. " + f"Run 'autoskillit install' to sync." + ) + if any_kitchen_open(project_path=str(Path.cwd())): + return DoctorResult( + Severity.ERROR, + "version_consistency", + mismatch_msg + " Kitchen is open — tool calls may fail with ENOENT until kitchens are " + "closed and `autoskillit install` is re-run.", + ) + return DoctorResult( + Severity.WARNING, + "version_consistency", + mismatch_msg, + ) diff --git a/src/autoskillit/cli/doctor/_doctor_runtime.py b/src/autoskillit/cli/doctor/_doctor_runtime.py new file mode 100644 index 0000000000..015e13474f --- /dev/null +++ b/src/autoskillit/cli/doctor/_doctor_runtime.py @@ -0,0 +1,105 @@ +"""Quota cache schema and claude process state doctor checks.""" + +from __future__ import annotations + +import json +import subprocess +from pathlib import Path + +from autoskillit.core import Severity, get_logger +from autoskillit.execution import QUOTA_CACHE_SCHEMA_VERSION + +from ._doctor_types import DoctorResult + +logger = get_logger(__name__) + + +def _check_quota_cache_schema(cache_path: Path | None = None) -> DoctorResult: + """Check the quota cache file for schema version drift.""" + check_name = "quota_cache_schema" + path = cache_path or (Path.home() / ".claude" / "autoskillit_quota_cache.json") + if not path.exists(): + return DoctorResult(Severity.OK, check_name, "No quota cache present.") + try: + raw = json.loads(path.read_text()) + except Exception as exc: + logger.warning("quota_cache_parse_error", path=str(path), exc_info=True) + return DoctorResult( + Severity.WARNING, + check_name, + f"Quota cache at {path} could not be parsed: {type(exc).__name__}.", + ) + observed = raw.get("schema_version") if isinstance(raw, dict) else None + if observed == QUOTA_CACHE_SCHEMA_VERSION: + return DoctorResult( + Severity.OK, + check_name, + f"Quota cache schema v{QUOTA_CACHE_SCHEMA_VERSION} at {path}.", + ) + return DoctorResult( + Severity.WARNING, + check_name, + f"Quota cache schema drift at {path}: observed={observed!r}, " + f"expected={QUOTA_CACHE_SCHEMA_VERSION}.", + ) + + +def _check_claude_process_state_breakdown() -> DoctorResult: + """Check current D-state and CPU usage of claude processes via ps.""" + check_name = "claude_process_state" + try: + result = subprocess.run( + ["ps", "-axo", "pid,state,pcpu,comm"], + capture_output=True, + text=True, + timeout=5, + ) + except (FileNotFoundError, subprocess.TimeoutExpired, OSError) as exc: + return DoctorResult( + Severity.OK, + check_name, + f"ps unavailable ({type(exc).__name__}); skipping claude process check", + ) + + if result.returncode != 0: + return DoctorResult( + Severity.OK, + check_name, + f"ps exited {result.returncode}; skipping claude process check", + ) + + claude_rows: list[tuple[int, str, float]] = [] + for line in result.stdout.splitlines()[1:]: + parts = line.split(maxsplit=3) + if len(parts) < 4: + continue + comm = parts[3] + if comm != "claude": + continue + try: + claude_rows.append((int(parts[0]), parts[1], float(parts[2]))) + except ValueError: + continue + + if not claude_rows: + return DoctorResult(Severity.OK, check_name, "No claude processes running") + + breakdown: dict[str, int] = {} + for _, state, _ in claude_rows: + breakdown[state] = breakdown.get(state, 0) + 1 + + summary = ", ".join(f"{s}={c}" for s, c in sorted(breakdown.items())) + + d_rows = [f"pid={pid} pcpu={pcpu}" for pid, state, pcpu in claude_rows if state == "D"] + if d_rows: + return DoctorResult( + Severity.WARNING, + check_name, + f"claude processes in D state: {', '.join(d_rows)} (breakdown: {summary})", + ) + + return DoctorResult( + Severity.OK, + check_name, + f"claude process state breakdown: {summary}", + ) diff --git a/src/autoskillit/cli/doctor/_doctor_types.py b/src/autoskillit/cli/doctor/_doctor_types.py new file mode 100644 index 0000000000..5d9bf171b0 --- /dev/null +++ b/src/autoskillit/cli/doctor/_doctor_types.py @@ -0,0 +1,19 @@ +"""Shared DoctorResult type — imported by all _doctor_* sub-modules.""" + +from __future__ import annotations + +from dataclasses import dataclass + +from autoskillit.core import Severity + + +@dataclass +class DoctorResult: + """Outcome of a single doctor check.""" + + severity: Severity + check: str + message: str + + +_NON_PROBLEM: frozenset[Severity] = frozenset({Severity.OK, Severity.INFO}) diff --git a/src/autoskillit/cli/fleet/CLAUDE.md b/src/autoskillit/cli/fleet/CLAUDE.md new file mode 100644 index 0000000000..29f4a4d38f --- /dev/null +++ b/src/autoskillit/cli/fleet/CLAUDE.md @@ -0,0 +1,13 @@ +# fleet/ + +Fleet campaign CLI subcommands for multi-issue dispatch orchestration. + +## Files + +| File | Purpose | +|------|---------| +| `__init__.py` | Main module: `fleet_app` Cyclopts sub-app with `campaign`, `dispatch`, `list`, `status` commands | +| `_fleet_display.py` | Status display: `_render_status_display()`, `_watch_loop()`, `_STATUS_COLUMNS`, `render_fleet_error()` | +| `_fleet_lifecycle.py` | Signal guard, stale dispatch reaping, `_pick_resume_campaign()` | +| `_fleet_preview.py` | Pre-launch dispatch preview: `_build_dispatch_recipe_table()`, `_print_dispatch_preview()` | +| `_fleet_session.py` | `_launch_fleet_session()` — builds Claude interactive session for fleet campaigns | diff --git a/src/autoskillit/cli/fleet/__init__.py b/src/autoskillit/cli/fleet/__init__.py new file mode 100644 index 0000000000..b719a02948 --- /dev/null +++ b/src/autoskillit/cli/fleet/__init__.py @@ -0,0 +1,400 @@ +"""Fleet CLI sub-app: campaign management commands.""" + +from __future__ import annotations + +import json +import os +import random +import shutil +import sys +from datetime import UTC, datetime +from pathlib import Path +from typing import TYPE_CHECKING, Annotated +from uuid import uuid4 + +from cyclopts import App, Parameter + +from autoskillit.cli.fleet._fleet_display import ( + _aggregate_totals, + _compute_exit_code, + _cross_check_tokens, + _render_status_display, + _watch_loop, + render_fleet_error, +) +from autoskillit.cli.fleet._fleet_lifecycle import ( + _pick_resume_campaign, + _reap_stale_dispatches, +) +from autoskillit.cli.fleet._fleet_preview import ( + _FLEET_CAMPAIGN_GREETINGS, + _FLEET_DISPATCH_GREETINGS, + _print_dispatch_preview, +) +from autoskillit.cli.fleet._fleet_session import _launch_fleet_session +from autoskillit.core import TerminalColumn, get_logger, is_feature_enabled + +logger = get_logger(__name__) + + +def _require_fleet(cfg: AutomationConfig) -> None: + """Exit with clear message if fleet feature is not enabled.""" + if not is_feature_enabled( + "fleet", cfg.features, experimental_enabled=cfg.experimental_enabled + ): + print( + "The 'fleet' feature is not enabled.\n" + "Enable it with: features.experimental_enabled: true in your config\n" + "Or set: AUTOSKILLIT_FEATURES__FLEET=true", + file=sys.stderr, + ) + raise SystemExit(1) + + +if TYPE_CHECKING: + from autoskillit.config import AutomationConfig + from autoskillit.fleet import ResumeDecision + + +fleet_app = App(name="fleet", help="Campaign fleet management.") +__all__ = ["fleet_app", "render_fleet_error"] + + +def _remove_clone_fn(path: str, _flag: str) -> dict[str, str]: + """Remove a clone directory for batch_delete.""" + try: + shutil.rmtree(path) + return {"removed": "true"} + except Exception as exc: + logger.warning("Failed to remove clone %s: %s", path, exc, exc_info=True) + return {"removed": "false", "reason": str(exc)} + + +@fleet_app.command(name="dispatch") +def fleet_dispatch() -> None: + """Launch a bare fleet dispatcher session (free-flow mode).""" + if os.environ.get("CLAUDECODE"): + print("ERROR: 'fleet dispatch' cannot run inside a Claude Code session.") + print("Run this command in a regular terminal.") + sys.exit(1) + if os.environ.get("AUTOSKILLIT_SESSION_TYPE") in ("skill", "leaf"): + print("ERROR: 'fleet dispatch' cannot run inside a skill or leaf (deprecated) session.") + sys.exit(1) + + from autoskillit.config import load_config + + cfg = load_config(Path.cwd()) + _require_fleet(cfg) + + recipe_table = _print_dispatch_preview() + + from autoskillit.cli.ui._timed_input import timed_prompt + + confirm = timed_prompt( + "\nLaunch session? [Enter/n]", default="", timeout=120, label="autoskillit fleet dispatch" + ) + if confirm.lower() in ("n", "no"): + return + + greeting = random.choice(_FLEET_DISPATCH_GREETINGS).format(recipe_table=recipe_table) + _launch_fleet_session( + None, + None, + None, + None, + fleet_mode="dispatch", + initial_message=greeting, + recipe_table=recipe_table, + ) + + +@fleet_app.command(name="campaign") +def fleet_campaign( + campaign_name: str | None = None, + *, + resume_campaign: Annotated[str | None, Parameter(name=["--resume"])] = None, +) -> None: + """Launch an interactive Claude Code session to execute a named campaign.""" + if os.environ.get("CLAUDECODE"): + print("ERROR: 'fleet campaign' cannot run inside a Claude Code session.") + print("Run this command in a regular terminal.") + sys.exit(1) + if os.environ.get("AUTOSKILLIT_SESSION_TYPE") in ("skill", "leaf"): + print("ERROR: 'fleet campaign' cannot run inside a skill or leaf (deprecated) session.") + sys.exit(1) + + from autoskillit.config import load_config + + cfg = load_config(Path.cwd()) + _require_fleet(cfg) + from autoskillit.cli.ui._menu import run_selection_menu + + if campaign_name is None and resume_campaign is None: + from autoskillit.recipe import list_campaign_recipes + + result = list_campaign_recipes(Path.cwd()) + if not result.items: + print("No campaigns found. Place campaign recipes in .autoskillit/recipes/campaigns/") + sys.exit(1) + + selected = run_selection_menu( + result.items, + header="Available campaigns:", + display_fn=lambda r: f"{r.name} {r.description[:60]}" if r.description else r.name, + name_key=lambda r: r.name, + timeout=120, + label="autoskillit fleet campaign", + ) + if selected is None or isinstance(selected, str): + print("No campaign selected.") + sys.exit(1) + campaign_name = selected.name + + elif campaign_name is None and resume_campaign is not None: + campaign_name, resume_campaign = _pick_resume_campaign(Path.cwd()) + + if campaign_name is None: + raise RuntimeError("campaign_name must be set before launching fleet session") + + from autoskillit.core import YAMLError + from autoskillit.fleet import ( + FLEET_HALTED_SENTINEL, + DispatchRecord, + resume_campaign_from_state, + write_initial_state, + ) + from autoskillit.recipe import find_campaign_by_name, load_recipe, validate_recipe_structure + + match = find_campaign_by_name(campaign_name, Path.cwd()) + if match is None: + print(f"Campaign not found: '{campaign_name}'") + sys.exit(1) + + try: + parsed = load_recipe(match.path) + except YAMLError as exc: + print(f"Campaign YAML parse error: {exc}") + sys.exit(1) + except ValueError as exc: + print(f"Campaign structure error: {exc}") + sys.exit(1) + + errors = validate_recipe_structure(parsed) + if errors: + print(f"Campaign '{campaign_name}' failed validation:") + for err in errors: + print(f" - {err}") + sys.exit(1) + + resume_metadata: ResumeDecision | None = None + campaign_id: str + state_path: Path + fleet_dir = Path.cwd() / ".autoskillit" / "temp" / "fleet" + + if resume_campaign is not None: + campaign_id = resume_campaign + state_path = fleet_dir / campaign_id / "state.json" + resume_metadata = resume_campaign_from_state( + state_path, parsed.continue_on_failure, reset_on_retry=True + ) + if resume_metadata is None: + print(f"ERROR: Campaign state not found or corrupted for '{campaign_id}'") + sys.exit(1) + if resume_metadata.completed_dispatches_block == FLEET_HALTED_SENTINEL: + print( + f"ERROR: Campaign '{campaign_id}' halted on dispatch failure. " + "Set 'continue_on_failure: true' in the campaign recipe to resume past failures.", + file=sys.stderr, + ) + sys.exit(1) + else: + campaign_id = uuid4().hex[:16] + state_dir = fleet_dir / campaign_id + state_dir.mkdir(parents=True, exist_ok=True) + state_path = state_dir / "state.json" + dispatches = [DispatchRecord(name=d.name) for d in parsed.dispatches] + write_initial_state(state_path, campaign_id, campaign_name, str(match.path), dispatches) + + from autoskillit.cli._preview import _pre_launch_campaign # noqa: PLC0415 + + _itable, proceed = _pre_launch_campaign( + campaign_name, parsed, match, Path.cwd(), is_resume=resume_campaign is not None + ) + if not proceed: + return + + greeting = random.choice(_FLEET_CAMPAIGN_GREETINGS).format(campaign_name=campaign_name) + _launch_fleet_session( + parsed, + campaign_id, + state_path, + resume_metadata, + fleet_mode="campaign", + ingredients_table=_itable, + initial_message=greeting, + ) + + +@fleet_app.command(name="list") +def fleet_list() -> None: + """List available campaign recipes.""" + from autoskillit.config import load_config + + cfg = load_config(Path.cwd()) + _require_fleet(cfg) + + from autoskillit.core import TerminalColumn, _render_terminal_table + from autoskillit.recipe import list_campaign_recipes + + result = list_campaign_recipes(Path.cwd()) + if not result.items: + print("No campaigns found.") + return + + columns = [ + TerminalColumn("NAME", 30, "<"), + TerminalColumn("SOURCE", 10, "<"), + TerminalColumn("DIR", 40, "<"), + ] + rows = [(r.name, r.source.value, str(r.path.parent.name)) for r in result.items] + print(_render_terminal_table(columns, rows)) + + +@fleet_app.command(name="status") +def fleet_status( + campaign_id: str | None = None, + *, + cleanup: bool = False, + reap: bool = False, + dry_run: bool = False, + watch: bool = False, + json_output: Annotated[bool, Parameter(name=["--json"])] = False, +) -> None: + """Show fleet campaign status.""" + from autoskillit.config import load_config + + cfg = load_config(Path.cwd()) + _require_fleet(cfg) + from autoskillit.fleet import read_state # noqa: PLC0415 + + fleet_dir = Path.cwd() / ".autoskillit" / "temp" / "fleet" + + if campaign_id is not None: + state_path = fleet_dir / campaign_id / "state.json" + state = read_state(state_path) + if state is None: + print(f"ERROR: Campaign '{campaign_id}' not found or state corrupted.") + sys.exit(3) + + if json_output: + totals = _aggregate_totals(state) + data = { + "campaign_id": state.campaign_id, + "campaign_name": state.campaign_name, + "started_at": state.started_at, + "dispatches": [d.to_dict() for d in state.dispatches], + "totals": totals, + } + print(json.dumps(data)) + _cross_check_tokens(state, totals) + sys.exit(_compute_exit_code(state)) + + if watch and cleanup: + print("ERROR: --watch and --cleanup are mutually exclusive.") + sys.exit(3) + + if watch: + sys.exit(_watch_loop(state_path)) + + _render_status_display(state) + + if cleanup: + from autoskillit.core import sweep_stale_markers + from autoskillit.workspace import ( + DefaultSessionSkillManager, + SkillsDirectoryProvider, + batch_delete, + resolve_ephemeral_root, + ) + + batch_delete("", _remove_clone_fn, owner=campaign_id) + try: + skill_mgr = DefaultSessionSkillManager( + provider=SkillsDirectoryProvider(), + ephemeral_root=resolve_ephemeral_root(), + ) + for d in state.dispatches: + if d.dispatched_session_id: + skill_mgr.cleanup_session(d.dispatched_session_id) + skill_mgr.cleanup_session(campaign_id) + except Exception: + logger.warning( + "Session skill cleanup failed for campaign %s", campaign_id, exc_info=True + ) + sweep_stale_markers() + kitchen_state_dir = ( + Path.cwd() / ".autoskillit" / "temp" / "kitchen_state" / campaign_id + ) + if kitchen_state_dir.is_dir(): + shutil.rmtree(kitchen_state_dir, ignore_errors=True) + print(f"Cleanup complete for campaign '{campaign_id}'.") + + if reap or dry_run: + _reap_stale_dispatches(state_path, dry_run=dry_run) + + totals = _aggregate_totals(state) + _cross_check_tokens(state, totals) + sys.exit(_compute_exit_code(state)) + + else: + from autoskillit.core import _render_terminal_table + + if not fleet_dir.exists(): + print("No campaigns found.") + return + + subdirs = [d for d in fleet_dir.iterdir() if d.is_dir()] + if not subdirs: + print("No campaigns found.") + return + + if json_output: + summaries = [] + for subdir in sorted(subdirs): + s = read_state(subdir / "state.json") + if s is None: + continue + status_counts: dict[str, int] = {} + for d in s.dispatches: + status_counts[d.status] = status_counts.get(d.status, 0) + 1 + summaries.append( + { + "campaign_id": s.campaign_id, + "campaign_name": s.campaign_name, + "started_at": s.started_at, + "dispatch_count": len(s.dispatches), + "status_counts": status_counts, + } + ) + print(json.dumps(summaries)) + return + + columns = [ + TerminalColumn("CAMPAIGN_NAME", 30, "<"), + TerminalColumn("ID", 18, "<"), + TerminalColumn("DISPATCHES", 10, "<"), + TerminalColumn("STARTED", 24, "<"), + ] + rows_list = [] + for subdir in sorted(subdirs): + s = read_state(subdir / "state.json") + if s is None: + continue + started = datetime.fromtimestamp(s.started_at, tz=UTC).strftime("%Y-%m-%d %H:%M UTC") + rows_list.append((s.campaign_name, s.campaign_id, str(len(s.dispatches)), started)) + + if not rows_list: + print("No campaigns found.") + return + + print(_render_terminal_table(columns, rows_list)) diff --git a/src/autoskillit/cli/fleet/_fleet_display.py b/src/autoskillit/cli/fleet/_fleet_display.py new file mode 100644 index 0000000000..d0c6ca39a7 --- /dev/null +++ b/src/autoskillit/cli/fleet/_fleet_display.py @@ -0,0 +1,292 @@ +"""Fleet status display helpers extracted from _fleet.py.""" + +from __future__ import annotations + +import json +import sys +import time +from datetime import UTC, datetime +from pathlib import Path +from typing import TYPE_CHECKING + +from autoskillit.core import TerminalColumn, get_logger + +logger = get_logger(__name__) + +if TYPE_CHECKING: + from autoskillit.fleet import CampaignState, DispatchRecord + +_STATUS_COLUMNS = ( + TerminalColumn("NAME", 30, "<"), + TerminalColumn("STATUS", 12, "<"), + TerminalColumn("ELAPSED", 10, ">"), + TerminalColumn("INPUT", 10, ">"), + TerminalColumn("OUTPUT", 10, ">"), + TerminalColumn("CACHE_RD", 10, ">"), + TerminalColumn("CACHE_WR", 10, ">"), + TerminalColumn("SESSION_LOG", None, "<"), +) + + +def _compute_exit_code(state: CampaignState) -> int: + """Compute CLI exit code from dispatch statuses. + + 0 = all success/skipped, 1 = any failure, 2 = any in-progress. + """ + from autoskillit.fleet import DispatchStatus # noqa: PLC0415 + + _failure = frozenset( + { + DispatchStatus.FAILURE, + DispatchStatus.INTERRUPTED, + DispatchStatus.REFUSED, + DispatchStatus.RELEASED, + } + ) + _in_progress = frozenset( + {DispatchStatus.RUNNING, DispatchStatus.PENDING, DispatchStatus.RESUMABLE} + ) + has_failure = any(d.status in _failure for d in state.dispatches) + has_in_progress = any(d.status in _in_progress for d in state.dispatches) + if has_failure: + return 1 + if has_in_progress: + return 2 + return 0 + + +def _fmt_elapsed(dispatch: DispatchRecord) -> str: + """Format dispatch elapsed time as human-readable string.""" + from autoskillit.fleet import DispatchStatus # noqa: PLC0415 + + if dispatch.started_at <= 0: + return "-" + if dispatch.status == DispatchStatus.RUNNING: + seconds = time.time() - dispatch.started_at + elif dispatch.ended_at > 0: + seconds = dispatch.ended_at - dispatch.started_at + else: + return "-" + seconds = max(0, seconds) + if seconds < 60: + return f"{seconds:.0f}s" + minutes = int(seconds // 60) + secs = int(seconds % 60) + if minutes < 60: + return f"{minutes}m {secs}s" + hours = minutes // 60 + mins = minutes % 60 + return f"{hours}h {mins}m" + + +def _humanize(n: int | float | None) -> str: + """Humanize token counts: 1234 -> '1.2k', 1234567 -> '1.2M'.""" + if n is None or n == 0: + return "0" + if not isinstance(n, (int, float)): + return "0" + if n >= 1_000_000: + return f"{n / 1_000_000:.1f}M" + if n >= 1_000: + return f"{n / 1_000:.1f}k" + return str(int(n)) + + +def _aggregate_totals(state: CampaignState) -> dict[str, int]: + """Sum token_usage across all dispatches.""" + totals: dict[str, int] = { + "input_tokens": 0, + "output_tokens": 0, + "cache_read": 0, + "cache_creation": 0, + } + for d in state.dispatches: + tu = d.token_usage + totals["input_tokens"] += tu.get("input_tokens", 0) + totals["output_tokens"] += tu.get("output_tokens", 0) + totals["cache_read"] += tu.get("cache_read_input_tokens", 0) + totals["cache_creation"] += tu.get("cache_creation_input_tokens", 0) + return totals + + +def _build_status_rows(state: CampaignState) -> list[tuple[str, ...]]: + """Build table rows from campaign state dispatches, including separator and TOTAL rows.""" + rows: list[tuple[str, ...]] = [] + for d in state.dispatches: + tu = d.token_usage + rows.append( + ( + d.name, + str(d.status), + _fmt_elapsed(d), + _humanize(tu.get("input_tokens", 0)), + _humanize(tu.get("output_tokens", 0)), + _humanize(tu.get("cache_read_input_tokens", 0)), + _humanize(tu.get("cache_creation_input_tokens", 0)), + d.dispatched_session_log_dir or "-", + ) + ) + totals = _aggregate_totals(state) + rows.append(("─" * 6, "", "", "", "", "", "", "")) + rows.append( + ( + "TOTAL", + "", + "", + _humanize(totals["input_tokens"]), + _humanize(totals["output_tokens"]), + _humanize(totals["cache_read"]), + _humanize(totals["cache_creation"]), + "", + ) + ) + return rows + + +def _load_log_totals(state: CampaignState) -> dict[str, int] | None: + """Load token totals from sessions.jsonl for the campaign. Returns None if unavailable.""" + from autoskillit.execution import resolve_log_dir + from autoskillit.pipeline import DefaultTokenLog + + log_root = resolve_log_dir("") + sessions_index = log_root / "sessions.jsonl" + if not sessions_index.exists(): + return None + + token_log = DefaultTokenLog() + loaded = token_log.load_from_log_dir(log_root, campaign_id_filter=state.campaign_id) + if loaded == 0: + return None + + return token_log.compute_total() + + +def _cross_check_tokens(state: CampaignState, state_totals: dict[str, int]) -> None: + """Warn on >5% token divergence between state.json and sessions.jsonl.""" + log_totals = _load_log_totals(state) + if log_totals is None: + return + + for label, state_key, log_key in [ + ("input_tokens", "input_tokens", "input_tokens"), + ("output_tokens", "output_tokens", "output_tokens"), + ("cache_read", "cache_read", "cache_read_input_tokens"), + ("cache_creation", "cache_creation", "cache_creation_input_tokens"), + ]: + sv = state_totals.get(state_key, 0) + lv = log_totals.get(log_key, 0) + if sv > 0 and abs(sv - lv) / sv > 0.05: + sys.stderr.write( + f"WARNING: {label} diverge {abs(sv - lv) / sv:.1%} (>5%)" + f" (state={sv}, sessionlog={lv}); state.json wins\n" + ) + + +def _render_status_display(state: CampaignState) -> int: + """Print campaign header and 8-column dispatch table to stdout. + + Returns the number of lines printed (for cursor-based screen refresh). + """ + from autoskillit.cli.ui._ansi import _render_terminal_table + + started = datetime.fromtimestamp(state.started_at, tz=UTC).strftime("%Y-%m-%d %H:%M:%S UTC") + header = f"Campaign: {state.campaign_name} ID: {state.campaign_id} Started: {started}" + print(header) + + rows = _build_status_rows(state) + table_str = _render_terminal_table(_STATUS_COLUMNS, rows) + print(table_str) + return 1 + table_str.rstrip("\n").count("\n") + 1 + + +def _watch_loop(state_path: Path) -> int: + """1 Hz polling loop for fleet status. Returns exit code.""" + import select + import termios + import tty + + from autoskillit.fleet import DispatchStatus, read_state # noqa: PLC0415 + + _in_progress = frozenset({DispatchStatus.RUNNING, DispatchStatus.PENDING}) + + state = read_state(state_path) + if state is None: + sys.stderr.write("ERROR: state file disappeared or corrupted\n") + return 3 + + # If campaign already terminal, render once and exit without needing a TTY + if all(d.status not in _in_progress for d in state.dispatches): + _render_status_display(state) + print("\nAll dispatches complete.") + return _compute_exit_code(state) + + if not sys.stdin.isatty() or not sys.stdout.isatty(): + sys.stderr.write( + "ERROR: --watch requires an interactive terminal" + " (both stdin and stdout must be TTYs).\n" + ) + return 1 + + fd = sys.stdin.fileno() + old_settings = termios.tcgetattr(fd) + prev_lines = 0 + try: + tty.setcbreak(fd) + while True: + state = read_state(state_path) + if state is None: + sys.stdout.flush() + sys.stderr.write("ERROR: state file disappeared or corrupted\n") + return 3 + + if prev_lines > 0: + sys.stdout.write(f"\033[{prev_lines}A") + for _ in range(prev_lines): + sys.stdout.write("\033[2K\033[1B") + sys.stdout.write(f"\033[{prev_lines}A") + sys.stdout.flush() + + prev_lines = _render_status_display(state) + + if all(d.status not in _in_progress for d in state.dispatches): + print("\nAll dispatches complete.") + return _compute_exit_code(state) + + rlist, _, _ = select.select([sys.stdin], [], [], 1.0) + if rlist: + ch = sys.stdin.read(1) + if ch.lower() == "q": + return _compute_exit_code(state) + except KeyboardInterrupt: + state = read_state(state_path) + return _compute_exit_code(state) if state else 3 + except Exception as exc: + logger.error("unexpected error in --watch loop: %s", exc, exc_info=True) + sys.stderr.write(f"ERROR: unexpected error in --watch loop: {exc}\n") + return 3 + finally: + try: + termios.tcsetattr(fd, termios.TCSADRAIN, old_settings) + except (termios.error, OSError): + pass + + +def render_fleet_error(envelope_json: str) -> int: + """Render a fleet error envelope to stderr. + + Returns exit code: 3 for fleet envelope errors, 0 for non-error envelopes. + """ + + try: + data = json.loads(envelope_json) + except (json.JSONDecodeError, TypeError): + return 0 + if data.get("success") is not False: + return 0 + msg = data.get("user_visible_message") or "unknown error" + code = data.get("error", "") + sys.stderr.write(f"fleet error [{code}]: {msg}\n") + details = data.get("details") + if details: + sys.stderr.write(f" details: {json.dumps(details)}\n") + return 3 diff --git a/src/autoskillit/cli/fleet/_fleet_lifecycle.py b/src/autoskillit/cli/fleet/_fleet_lifecycle.py new file mode 100644 index 0000000000..b753e25349 --- /dev/null +++ b/src/autoskillit/cli/fleet/_fleet_lifecycle.py @@ -0,0 +1,304 @@ +"""Fleet signal guard and stale-dispatch reaping extracted from _fleet.py.""" + +from __future__ import annotations + +import fcntl +import signal +import sys +from collections.abc import AsyncIterator +from contextlib import asynccontextmanager +from pathlib import Path +from typing import TYPE_CHECKING + +import anyio +import anyio.abc +import psutil + +from autoskillit.core import get_logger + +logger = get_logger(__name__) + +if TYPE_CHECKING: + from autoskillit.fleet import DispatchRecord, DispatchStatus + + +def _transition_dead_dispatch( + state_path: Path, record: DispatchRecord, reason: str +) -> DispatchStatus | None: + """Transition a confirmed-dead RUNNING dispatch to RESUMABLE or INTERRUPTED. + + Delegates sidecar decision logic to the fleet layer. Returns the new status, + or None if both write attempts failed. Never raises. + """ + from autoskillit.fleet import crash_recover_dispatch # noqa: PLC0415 + + return crash_recover_dispatch(state_path, record, reason=reason) + + +@asynccontextmanager +async def _fleet_signal_guard( + state_path: Path, + campaign_id: str, + *, + campaign_name: str | None = None, + cleanup_on_interrupt: bool = False, +) -> AsyncIterator[None]: + """Async context manager that installs SIGINT/SIGTERM handlers. + + On signal receipt: + - Cancels the enclosing task group scope first. + - Reads state.json and marks RUNNING dispatches as INTERRUPTED. + - Verifies PID identity via starttime_ticks before killing. + - Optionally runs workspace cleanup. + - Logs a resume hint. + """ + + async def _watch( + scope: anyio.CancelScope, + *, + task_status: anyio.abc.TaskStatus = anyio.TASK_STATUS_IGNORED, + ) -> None: + with anyio.open_signal_receiver(signal.SIGTERM, signal.SIGINT, signal.SIGHUP) as signals: + task_status.started() + async for sig in signals: + signame = sig.name + + # Cancel the enclosing scope FIRST to unwind any in-flight dispatch + # coroutine before the cleanup writes state (prevents ordering races). + scope.cancel() + + # Shield the cleanup from the now-cancelled scope so that async + # operations (kill, state write) are not interrupted. + with anyio.CancelScope(shield=True): + from autoskillit.execution import async_kill_process_tree, read_starttime_ticks + from autoskillit.fleet import DispatchStatus, read_state # noqa: PLC0415 + + state = read_state(state_path) + if state is not None: + for dispatch in state.dispatches: + if dispatch.status != DispatchStatus.RUNNING: + continue + if dispatch.dispatched_pid == 0: + _transition_dead_dispatch( + state_path, dispatch, reason=f"signal_{signame}" + ) + continue + + # Verify PID identity before killing + current_ticks = read_starttime_ticks(dispatch.dispatched_pid) + if current_ticks is not None: + if ( + dispatch.dispatched_starttime_ticks > 0 + and current_ticks == dispatch.dispatched_starttime_ticks + ): + try: + await async_kill_process_tree( + dispatch.dispatched_pid, timeout=5.0 + ) + except Exception: + logger.warning( + "signal_guard: kill_process_tree failed", + exc_info=True, + ) + else: + logger.warning( + "signal_guard: PID %d recycled (ticks mismatch)", + dispatch.dispatched_pid, + ) + else: + # Non-Linux fallback: psutil.pid_exists without identity check + if psutil.pid_exists(dispatch.dispatched_pid): + try: + await async_kill_process_tree( + dispatch.dispatched_pid, timeout=5.0 + ) + except Exception: + logger.warning( + "signal_guard: kill_process_tree failed (non-linux)", + exc_info=True, + ) + + _transition_dead_dispatch( + state_path, dispatch, reason=f"signal_{signame}" + ) + + if cleanup_on_interrupt: + try: + from autoskillit.core import ensure_project_temp + from autoskillit.workspace import DefaultWorkspaceManager + + workspace_dir = ensure_project_temp(Path.cwd()) + mgr = DefaultWorkspaceManager() + mgr.delete_contents(workspace_dir) + except Exception: + logger.warning("signal_guard: workspace cleanup failed", exc_info=True) + + if campaign_name is not None: + resume_cmd = ( + f"autoskillit fleet campaign {campaign_name} --resume {campaign_id}" + ) + else: + resume_cmd = f"autoskillit fleet campaign --resume {campaign_id}" + sys.stderr.write(f"Campaign {campaign_id} interrupted. Resume: {resume_cmd}\n") + return + + async with anyio.create_task_group() as tg: + await tg.start(_watch, tg.cancel_scope) + try: + yield + finally: + tg.cancel_scope.cancel() + + +def _reap_stale_dispatches(state_path: Path, *, dry_run: bool = False) -> None: + """Reap stale RUNNING dispatches with PID-recycling-safe identity checks. + + Uses fcntl.flock() to protect against concurrent reap invocations. + For each RUNNING dispatch: + - Boot-ID mismatch → reaped_pid_recycled (no kill) + - Process dead → reaped_dead_pid + - Process alive + ticks match → kill + reaped_orphan + - Process alive + ticks mismatch → reaped_pid_recycled (no kill) + """ + from autoskillit.execution import kill_process_tree, read_boot_id, read_starttime_ticks + from autoskillit.fleet import DispatchStatus, read_state # noqa: PLC0415 + + current_boot_id = read_boot_id() + + if not state_path.exists(): + logger.info("reap: state file not found, nothing to reap: %s", state_path) + return + + with open(state_path, "r+") as _lock_fh: + fcntl.flock(_lock_fh, fcntl.LOCK_EX) + try: + state = read_state(state_path) + if state is None: + logger.warning("reap: cannot read state file: %s", state_path) + return + + running = [d for d in state.dispatches if d.status == DispatchStatus.RUNNING] + if not running: + logger.info("reap: no running dispatches in campaign %s", state.campaign_id) + return + + logger.info( + "reap: scanning %d dispatches in campaign %s", len(running), state.campaign_id + ) + + for dispatch in running: + name = dispatch.name + pid = dispatch.dispatched_pid + + if pid == 0: + if dry_run: + logger.info("reap: [WOULD MARK] %s pid=0 (no PID recorded)", name) + else: + new_status = _transition_dead_dispatch( + state_path, dispatch, reason="reaped_dead_pid" + ) + if new_status is not None: + logger.info("reap: [MARKED] %s (no PID recorded)", name) + else: + logger.warning("reap: [MARK_FAILED] %s (no PID recorded)", name) + continue + + # Boot ID check: if machine rebooted, all PIDs are recycled + if ( + dispatch.dispatched_boot_id != "" + and current_boot_id is not None + and dispatch.dispatched_boot_id != current_boot_id + ): + if dry_run: + logger.info( + "reap: [WOULD MARK] %s pid=%d (rebooted, pid_recycled)", name, pid + ) + else: + _transition_dead_dispatch( + state_path, dispatch, reason="reaped_pid_recycled" + ) + logger.info( + "reap: [MARKED] %s pid=%d (rebooted, pid_recycled)", + name, + pid, + ) + continue + + if not psutil.pid_exists(pid): + if dry_run: + logger.info("reap: [WOULD MARK] %s pid=%d (process dead)", name, pid) + else: + _transition_dead_dispatch(state_path, dispatch, reason="reaped_dead_pid") + logger.info("reap: [MARKED] %s pid=%d (process dead)", name, pid) + continue + + # Process is alive — check identity + current_ticks = read_starttime_ticks(pid) + if ( + current_ticks is not None + and current_ticks == dispatch.dispatched_starttime_ticks + ): + if dry_run: + logger.info( + "reap: [WOULD KILL] %s pid=%d (orphan, identity match)", name, pid + ) + else: + try: + kill_process_tree(pid) + except Exception: + logger.warning( + "reap: kill_process_tree failed for pid=%d", pid, exc_info=True + ) + _transition_dead_dispatch(state_path, dispatch, reason="reaped_orphan") + logger.info("reap: [KILLED] %s pid=%d (orphan reaped)", name, pid) + else: + if dry_run: + logger.info( + "reap: [WOULD MARK] %s pid=%d (PID recycled, no kill)", name, pid + ) + else: + _transition_dead_dispatch( + state_path, dispatch, reason="reaped_pid_recycled" + ) + logger.info( + "reap: [MARKED] %s pid=%d (PID recycled, no kill)", + name, + pid, + ) + finally: + fcntl.flock(_lock_fh, fcntl.LOCK_UN) + + +def _pick_resume_campaign(project_dir: Path) -> tuple[str, str]: + """Interactively pick a resumable campaign. Returns (campaign_name, campaign_id) or exits.""" + from autoskillit.cli.ui._menu import run_selection_menu # noqa: PLC0415 + from autoskillit.fleet import TERMINAL_DISPATCH_STATUSES, read_state # noqa: PLC0415 + + fleet_dir = project_dir / ".autoskillit" / "temp" / "fleet" + active = [] + if fleet_dir.is_dir(): + for subdir in sorted(fleet_dir.iterdir()): + if not subdir.is_dir(): + continue + state = read_state(subdir / "state.json") + if state is None: + continue + if any(d.status not in TERMINAL_DISPATCH_STATUSES for d in state.dispatches): + active.append(state) + + if not active: + print("No active campaigns to resume.") + sys.exit(1) + + selected = run_selection_menu( + active, + header="Active campaigns (resumable):", + display_fn=lambda s: f"{s.campaign_name} [{(s.campaign_id or '')[:8]}…]", + name_key=lambda s: s.campaign_name, + timeout=120, + label="autoskillit fleet campaign --resume", + ) + if selected is None or isinstance(selected, str): + print("No campaign selected.") + sys.exit(1) + return selected.campaign_name, selected.campaign_id # type: ignore[union-attr] diff --git a/src/autoskillit/cli/fleet/_fleet_preview.py b/src/autoskillit/cli/fleet/_fleet_preview.py new file mode 100644 index 0000000000..21c2f356ba --- /dev/null +++ b/src/autoskillit/cli/fleet/_fleet_preview.py @@ -0,0 +1,90 @@ +"""Pre-launch dispatch preview: recipe roster + tool surface display.""" + +from __future__ import annotations + +from pathlib import Path + +_DISPATCH_TOOL_CATEGORIES: list[tuple[str, tuple[str, ...]]] = [ + ("Dispatch", ("dispatch_food_truck",)), + ("Cleanup", ("batch_cleanup_clones",)), + ( + "Telemetry", + ("get_pipeline_report", "get_token_summary", "get_timing_summary", "get_quota_events"), + ), + ("Recipes", ("list_recipes", "load_recipe")), + ("GitHub", ("fetch_github_issue", "get_issue_title")), +] + +_FLEET_DISPATCH_GREETINGS: list[str] = [ + ( + "Fleet dispatcher online. Your available food trucks:\n\n" + "{recipe_table}\n\n" + "Ready for dispatch orders." + ), + ( + "Welcome to fleet dispatch — ad-hoc food truck coordination.\n\n" + "Available food trucks:\n\n" + "{recipe_table}\n\n" + "What targets are we dispatching to?" + ), + ( + "Dispatcher standing by. Food truck roster:\n\n" + "{recipe_table}\n\n" + "Issue your dispatch orders when ready." + ), +] + +_FLEET_CAMPAIGN_GREETINGS: list[str] = [ + "Campaign ready: {campaign_name}. Let me display the ingredients and get started.", + "Launching campaign: {campaign_name}. Reviewing ingredients now.", + "Campaign {campaign_name} loaded. Checking ingredients before dispatch.", +] + + +def _print_dispatch_preview() -> str: + """Print the pre-launch summary for fleet dispatch (mirrors cook's pre-launch display). + + Returns the recipe table string (name + description) for greeting injection. + """ + from autoskillit.cli.ui._ansi import permissions_warning, supports_color + from autoskillit.recipe import RecipeKind, list_recipes + + color = supports_color() + _B = "\x1b[1m" if color else "" + _C = "\x1b[96m" if color else "" + _D = "\x1b[2m" if color else "" + _G = "\x1b[32m" if color else "" + _Y = "\x1b[33m" if color else "" + _R = "\x1b[0m" if color else "" + + from autoskillit import __version__ + + print( + f"{_B}{_C}AUTOSKILLIT {__version__}{_R}" + f" {_D}Fleet dispatcher. Ad-hoc food truck coordination.{_R}" + ) + + recipes = list_recipes(Path.cwd(), exclude_kinds=frozenset({RecipeKind.CAMPAIGN})).items + if recipes: + name_w = max(len(r.name or "") for r in recipes) + src_w = max(len(r.source or "") for r in recipes) + print(f"\n{_B}Available food trucks:{_R}") + print(f" {'NAME':<{name_w}} {'SOURCE':<{src_w}} DESCRIPTION") + print(f" {'-' * name_w} {'-' * src_w} {'-' * 11}") + for r in recipes: + name = r.name or "" + src = r.source or "" + print(f" {_G}{name:<{name_w}}{_R} {_D}{src:<{src_w}}{_R} {r.description}") + greeting_table = "\n".join(f"{(r.name or ''):<{name_w}} {r.description}" for r in recipes) + else: + print(f"\n{_D}No recipes found.{_R}") + greeting_table = "(no recipes found)" + + print() + for name, tools in _DISPATCH_TOOL_CATEGORIES: + tool_list = f"{_D}, {_R}".join(f"{_G}{t}{_R}" for t in tools) + print(f" {_Y}{name:>20}{_R} {tool_list}") + print() + + print(permissions_warning()) + return greeting_table diff --git a/src/autoskillit/cli/fleet/_fleet_session.py b/src/autoskillit/cli/fleet/_fleet_session.py new file mode 100644 index 0000000000..c2d923a617 --- /dev/null +++ b/src/autoskillit/cli/fleet/_fleet_session.py @@ -0,0 +1,197 @@ +"""Fleet session launch helper extracted from _fleet.py.""" + +from __future__ import annotations + +import dataclasses +from pathlib import Path +from typing import TYPE_CHECKING, Literal + +from autoskillit.core import NamedResume, NoResume, dump_yaml_str, get_logger + +logger = get_logger(__name__) + +if TYPE_CHECKING: + from autoskillit.core import ResumeSpec + from autoskillit.fleet import ResumeDecision + from autoskillit.recipe.schema import Recipe + +_MAX_RELOADS = 10 +_MAX_INFRA_RESUMES = 3 + + +def _check_reload_guard(reload_id: str, seen_reload_ids: set[str]) -> None: + """Enforce max-reload cap and duplicate-ID detection, then register the ID.""" + if len(seen_reload_ids) >= _MAX_RELOADS: + raise SystemExit(f"Too many reloads ({_MAX_RELOADS} max). Check for infinite loop.") + if reload_id in seen_reload_ids: + raise SystemExit(f"Repeated reload_id {reload_id!r} — aborting.") + seen_reload_ids.add(reload_id) + + +def _launch_fleet_session( + campaign_recipe: Recipe | None, + campaign_id: str | None, + state_path: Path | None, + resume_metadata: ResumeDecision | None, + *, + fleet_mode: Literal["dispatch", "campaign"], + ingredients_table: str | None = None, + initial_message: str | None = None, + recipe_table: str | None = None, +) -> None: + """Build the L3 orchestrator prompt and launch an interactive fleet session.""" + from autoskillit.cli import detect_autoskillit_mcp_prefix # noqa: PLC0415 + from autoskillit.cli.session._session_launch import ( + _run_interactive_session, + ) + + mcp_prefix = detect_autoskillit_mcp_prefix() + + project_dir = Path.cwd() + + if campaign_recipe is None: + # Ad-hoc mode: no campaign, no state, bare kitchen open + from autoskillit.cli._prompts import _build_fleet_dispatch_prompt + + prompt = _build_fleet_dispatch_prompt(mcp_prefix, recipe_table=recipe_table) + extra_env: dict[str, str] = { + "AUTOSKILLIT_SESSION_TYPE": "fleet", + "AUTOSKILLIT_FLEET_MODE": fleet_mode, + "AUTOSKILLIT_HEADLESS": "0", + } + seen_reload_ids: set[str] = set() + current_resume_spec: ResumeSpec = NoResume() + current_initial_message = initial_message + infra_resume_count = 0 + while True: + session_signal = _run_interactive_session( + prompt, + initial_message=current_initial_message, + extra_env=extra_env, + resume_spec=current_resume_spec, + project_dir=project_dir, + ) + if session_signal is None: + break + if not isinstance(session_signal, str): + infra_resume_count += 1 + if infra_resume_count >= _MAX_INFRA_RESUMES: + raise SystemExit( + f"Too many infrastructure resumes ({_MAX_INFRA_RESUMES} max). " + f"Last exit: {session_signal.category}" + ) + current_resume_spec = NamedResume(session_id=session_signal.session_id) + continue + _check_reload_guard(session_signal, seen_reload_ids) + current_resume_spec = NamedResume(session_id=session_signal) + current_initial_message = None # greeting only on first launch + else: + # Campaign-driven mode: full orchestrator prompt with manifest and state + if campaign_id is None: + raise ValueError("campaign_id must not be None in campaign-driven mode") + if state_path is None: + raise ValueError("state_path must not be None in campaign-driven mode") + from autoskillit.cli._prompts import _build_fleet_campaign_prompt + from autoskillit.fleet import FLEET_HALTED_SENTINEL, resume_campaign_from_state + + manifest_yaml = dump_yaml_str( + [dataclasses.asdict(d) for d in campaign_recipe.dispatches], + default_flow_style=False, + allow_unicode=True, + ) + completed_dispatches = ( + resume_metadata.completed_dispatches_block if resume_metadata is not None else "" + ) + resumable_dispatch_name = ( + resume_metadata.next_dispatch_name + if resume_metadata is not None and resume_metadata.is_resumable + else "" + ) + resume_session_id = ( + resume_metadata.dispatched_session_id + if resume_metadata is not None and resume_metadata.is_resumable + else "" + ) + resume_kill_reason = ( + resume_metadata.kill_reason + if resume_metadata is not None and resume_metadata.is_resumable + else "" + ) + prompt = _build_fleet_campaign_prompt( + campaign_recipe, + manifest_yaml, + completed_dispatches, + mcp_prefix, + campaign_id, + resumable_dispatch_name=resumable_dispatch_name, + resume_session_id=resume_session_id, + resume_kill_reason=resume_kill_reason, + ingredients_table=ingredients_table, + ) + extra_env = { + "AUTOSKILLIT_SESSION_TYPE": "fleet", + "AUTOSKILLIT_FLEET_MODE": fleet_mode, + "AUTOSKILLIT_CAMPAIGN_ID": campaign_id, + "AUTOSKILLIT_CAMPAIGN_STATE_PATH": str(state_path), + "AUTOSKILLIT_CONTINUE_ON_FAILURE": str(campaign_recipe.continue_on_failure).lower(), + "AUTOSKILLIT_HEADLESS": "0", + } + + seen_reload_ids = set[str]() + current_resume_spec = NoResume() + infra_resume_count = 0 + current_initial_message = initial_message + + while True: + session_signal = _run_interactive_session( + prompt, + initial_message=current_initial_message, + extra_env=extra_env, + resume_spec=current_resume_spec, + project_dir=project_dir, + ) + if session_signal is None: + break + if not isinstance(session_signal, str): + infra_resume_count += 1 + if infra_resume_count >= _MAX_INFRA_RESUMES: + raise SystemExit( + f"Too many infrastructure resumes ({_MAX_INFRA_RESUMES} max). " + f"Last exit: {session_signal.category}" + ) + current_resume_spec = NamedResume(session_id=session_signal.session_id) + else: + _check_reload_guard(session_signal, seen_reload_ids) + current_resume_spec = NamedResume(session_id=session_signal) + + current_initial_message = None + + fresh_metadata = resume_campaign_from_state( + state_path, campaign_recipe.continue_on_failure + ) + if fresh_metadata is None: + logger.error("Campaign state corrupted during resume — exiting") + break + if fresh_metadata.completed_dispatches_block == FLEET_HALTED_SENTINEL: + logger.info("Campaign halted on failure during resume — exiting") + break + + completed_dispatches = fresh_metadata.completed_dispatches_block + resumable_dispatch_name = ( + fresh_metadata.next_dispatch_name if fresh_metadata.is_resumable else "" + ) + resume_session_id = ( + fresh_metadata.dispatched_session_id if fresh_metadata.is_resumable else "" + ) + resume_kill_reason = fresh_metadata.kill_reason if fresh_metadata.is_resumable else "" + prompt = _build_fleet_campaign_prompt( + campaign_recipe, + manifest_yaml, + completed_dispatches, + mcp_prefix, + campaign_id, + resumable_dispatch_name=resumable_dispatch_name, + resume_session_id=resume_session_id, + resume_kill_reason=resume_kill_reason, + ingredients_table=ingredients_table, + ) diff --git a/src/autoskillit/cli/session/CLAUDE.md b/src/autoskillit/cli/session/CLAUDE.md new file mode 100644 index 0000000000..ccab57f832 --- /dev/null +++ b/src/autoskillit/cli/session/CLAUDE.md @@ -0,0 +1,15 @@ +# session/ + +Interactive session management — cook (ephemeral) and order (orchestrator) entry points. + +## Files + +| File | Purpose | +|------|---------| +| `__init__.py` | Re-exports `cook` and `order` commands | +| `_constants.py` | CLI session label constants (`SESSION_TYPE_COOK`, `SESSION_TYPE_ORDER`) — not `SessionType` enum members | +| `_cook.py` | `cook` command: ephemeral skill session launcher | +| `_order.py` | `order` command: orchestrator prompt builder with recipe selection | +| `_reload.py` | `consume_reload_sentinel()` — detects reload sentinel written by MCP reload tool | +| `_session_launch.py` | Shared prelude: `_launch_cook_session()`, `_run_interactive_session()` | +| `_session_picker.py` | Scoped resume picker: filters session history by greeting prefix | diff --git a/src/autoskillit/cli/session/__init__.py b/src/autoskillit/cli/session/__init__.py new file mode 100644 index 0000000000..07574fc571 --- /dev/null +++ b/src/autoskillit/cli/session/__init__.py @@ -0,0 +1,4 @@ +"""CLI session management — cook, order, resume, reload.""" + +from autoskillit.cli.session._cook import cook # noqa: F401 +from autoskillit.cli.session._order import order # noqa: F401 diff --git a/src/autoskillit/cli/session/_constants.py b/src/autoskillit/cli/session/_constants.py new file mode 100644 index 0000000000..d4eeb531f7 --- /dev/null +++ b/src/autoskillit/cli/session/_constants.py @@ -0,0 +1,13 @@ +"""CLI session labels — display/routing labels for the session picker. + +These are NOT ``SessionType`` enum members. They are not recognized by +``session_type()``, guards, or ``_session_type.py`` visibility logic. +Do not use in ``AUTOSKILLIT_SESSION_TYPE`` for headless sessions. +""" + +from __future__ import annotations + +from typing import Final + +SESSION_TYPE_COOK: Final = "cook" +SESSION_TYPE_ORDER: Final = "order" diff --git a/src/autoskillit/cli/session/_cook.py b/src/autoskillit/cli/session/_cook.py new file mode 100644 index 0000000000..1fbf6cb239 --- /dev/null +++ b/src/autoskillit/cli/session/_cook.py @@ -0,0 +1,237 @@ +"""cook command: interactive skill session launcher.""" + +from __future__ import annotations + +import shutil +import subprocess +import sys +import uuid +from collections.abc import Mapping +from pathlib import Path + +from autoskillit.cli.ui._terminal import terminal_guard +from autoskillit.core import is_feature_enabled + + +def _print_recipes_list() -> None: + """Print available recipes grouped by category to stdout.""" + from autoskillit.recipe import GROUP_LABELS, group_rank, list_recipes + + recipes = list_recipes(Path.cwd()).items + if not recipes: + print("No recipes found.") + return + + name_w = max(len(r.name) for r in recipes) + src_w = max(len(r.source) for r in recipes) + current_rank = -1 + for r in recipes: + rank = group_rank(r) + if rank != current_rank: + current_rank = rank + print(f"\n{GROUP_LABELS.get(rank, str(rank))}") + print(f"{'NAME':<{name_w}} {'SOURCE':<{src_w}} DESCRIPTION") + print(f"{'-' * name_w} {'-' * src_w} {'-' * 11}") + print(f"{r.name:<{name_w}} {r.source:<{src_w}} {r.description}") + + +def _run_cook_session( + *, + cmd: list[str], + env: Mapping[str, str], + _first_run: bool, + initial_prompt: str | None, + project_dir: Path, +) -> str | None: + """Run the cook subprocess; return session_id if a reload sentinel was written.""" + from autoskillit.cli.session._reload import consume_reload_sentinel + + with terminal_guard(): + result = subprocess.run(cmd, env=env) + reload_session_id = consume_reload_sentinel(project_dir) + if reload_session_id is not None: + return reload_session_id + if result.returncode == 0: + if _first_run and initial_prompt is not None: + from autoskillit.cli._onboarding import mark_onboarded + + mark_onboarded(project_dir) + else: + raise SystemExit(result.returncode) + return None + + +def cook( + *, resume: bool = False, session_id: str | None = None, profile: str | None = None +) -> None: + """Launch Claude with all bundled AutoSkillit skills as slash commands.""" + from autoskillit.workspace import ( + DefaultSessionSkillManager, + SkillsDirectoryProvider, + resolve_ephemeral_root, + ) + + if not shutil.which("claude"): + print("'claude' not found on PATH. Install Claude Code to use cook.") + raise SystemExit(1) + + from autoskillit import __version__ + from autoskillit.cli.ui._ansi import supports_color + + color = supports_color() + _B = "\x1b[1m" if color else "" + _C = "\x1b[96m" if color else "" + _D = "\x1b[2m" if color else "" + _G = "\x1b[32m" if color else "" + _Y = "\x1b[33m" if color else "" + _R = "\x1b[0m" if color else "" + + from autoskillit.config import iter_display_categories, load_config # noqa: PLC0415 + + config = load_config() + + if profile is not None: + if not is_feature_enabled( + "providers", config.features, experimental_enabled=config.experimental_enabled + ): + print( + "Error: --profile requires the 'providers' feature to be enabled.\n" + "Enable it in .autoskillit/config.yaml:\n" + " features:\n" + " providers: true", + file=sys.stderr, + ) + raise SystemExit(1) + if profile not in config.providers.profiles: + known = ", ".join(sorted(config.providers.profiles)) or "(none defined)" + print( + f"Error: Unknown provider profile {profile!r}. Known profiles: {known}\n" + "Define profiles in .autoskillit/config.yaml under providers.profiles.", + file=sys.stderr, + ) + raise SystemExit(1) + + print(f"{_B}{_C}AUTOSKILLIT {__version__}{_R} {_D}Kitchen open. All tools active.{_R}") + skip = {"Telemetry & Diagnostics", "Kitchen"} + for name, tools in iter_display_categories( + config.features, experimental_enabled=config.experimental_enabled + ): + if name in skip: + continue + tool_list = f"{_D}, {_R}".join(f"{_G}{t}{_R}" for t in tools) + print(f" {_Y}{name:>20}{_R} {tool_list}") + print() + + from autoskillit.cli.ui._ansi import permissions_warning + from autoskillit.cli.ui._timed_input import timed_prompt + + print(permissions_warning()) + confirm = timed_prompt( + "\nLaunch session? [Enter/n]", default="", timeout=120, label="autoskillit cook" + ) + if confirm.lower() in ("n", "no"): + return + + from autoskillit.cli._installed_plugins import InstalledPluginsFile + from autoskillit.cli._onboarding import is_first_run, run_onboarding_menu + from autoskillit.cli.session._constants import SESSION_TYPE_COOK + from autoskillit.core import ( + _AUTOSKILLIT_PLUGIN_KEY, + LAUNCH_ID_ENV_VAR, + SESSION_TYPE_ENV_VAR, + BareResume, + DirectInstall, + MarketplaceInstall, + NamedResume, + NoResume, + _get_autoskillit_install_path, + configure_logging, + get_logger, + pkg_root, + resume_spec_from_cli, + write_registry_entry, + ) + + logger = get_logger(__name__) + from autoskillit.execution import build_interactive_cmd + + configure_logging() + + resume_spec = resume_spec_from_cli(resume=resume, session_id=session_id) + + project_dir = Path.cwd() + initial_prompt: str | None = None + _first_run = is_first_run(project_dir) + if _first_run: + initial_prompt = run_onboarding_menu(project_dir, color=color) + + session_id_local = uuid.uuid4().hex[:16] + write_registry_entry(project_dir, session_id_local, SESSION_TYPE_COOK, None) + ephemeral_root = resolve_ephemeral_root() + session_mgr = DefaultSessionSkillManager(SkillsDirectoryProvider(), ephemeral_root) + session_mgr.cleanup_stale() + skills_dir = session_mgr.init_session( + session_id_local, cook_session=True, config=config, project_dir=project_dir + ) + + plugin_source: MarketplaceInstall | DirectInstall + if InstalledPluginsFile().contains(_AUTOSKILLIT_PLUGIN_KEY): + try: + plugin_source = MarketplaceInstall(cache_path=_get_autoskillit_install_path()) + except (KeyError, ValueError) as exc: + logger.warning( + "marketplace install path unavailable (%s) — falling back to direct install", + exc, + ) + plugin_source = DirectInstall(plugin_dir=pkg_root()) + else: + plugin_source = DirectInstall(plugin_dir=pkg_root()) + + if isinstance(resume_spec, BareResume): + from autoskillit.cli.session._session_picker import pick_session + + selected_id = pick_session(SESSION_TYPE_COOK, project_dir) + if selected_id is not None: + resume_spec = NamedResume(session_id=selected_id) + else: + resume_spec = NoResume() + + _cook_env_extras: dict[str, str] = { + SESSION_TYPE_ENV_VAR: SESSION_TYPE_COOK, + LAUNCH_ID_ENV_VAR: session_id_local, + } + if profile is not None: + _cook_env_extras["AUTOSKILLIT_PROVIDER_PROFILE"] = profile + _cook_env_extras.update(config.providers.profiles[profile]) + + current_resume_spec = resume_spec + _current_first_run = _first_run + _current_initial_prompt = initial_prompt + + _max_reloads = 10 + seen_reload_ids: set[str] = set() + while True: + spec = build_interactive_cmd( + plugin_source=plugin_source, + add_dirs=[skills_dir], + initial_prompt=_current_initial_prompt, + resume_spec=current_resume_spec, + env_extras=_cook_env_extras, + ) + reload_session_id = _run_cook_session( + cmd=spec.cmd, + env=spec.env, + _first_run=_current_first_run, + initial_prompt=_current_initial_prompt, + project_dir=project_dir, + ) + if reload_session_id is None: + break + if len(seen_reload_ids) >= _max_reloads: + raise SystemExit(f"Too many reloads ({_max_reloads} max). Check for infinite loop.") + if reload_session_id in seen_reload_ids: + raise SystemExit(f"Repeated reload_id {reload_session_id!r} — aborting.") + seen_reload_ids.add(reload_session_id) + current_resume_spec = NamedResume(session_id=reload_session_id) + _current_first_run = False + _current_initial_prompt = None diff --git a/src/autoskillit/cli/session/_order.py b/src/autoskillit/cli/session/_order.py new file mode 100644 index 0000000000..9904a94261 --- /dev/null +++ b/src/autoskillit/cli/session/_order.py @@ -0,0 +1,304 @@ +"""Order command and helpers extracted from app.py.""" + +from __future__ import annotations + +import os +import random +import sys +from pathlib import Path +from typing import TYPE_CHECKING + +import regex as re + +from autoskillit.cli._mcp_names import detect_autoskillit_mcp_prefix +from autoskillit.cli._prompts import _build_orchestrator_prompt, _get_ingredients_table +from autoskillit.cli.session._session_launch import _launch_cook_session, _write_order_entry +from autoskillit.core import ( + RecipeSource, + atomic_write, + pkg_root, + resume_spec_from_cli, +) + +if TYPE_CHECKING: + from autoskillit.recipe import Recipe, RecipeInfo + +_UUID_RE = re.compile(r"^[0-9a-f]{8}(-[0-9a-f]{4}){3}-[0-9a-f]{12}$", re.IGNORECASE) + + +def _recipes_dir_for(info: RecipeInfo) -> Path: + if getattr(info, "source", None) == RecipeSource.BUILTIN: + return pkg_root() / "recipes" + return Path.cwd() / ".autoskillit" / "recipes" + + +def _get_subsets_needed(recipe: Recipe, disabled_subsets: frozenset[str]) -> frozenset[str]: + """Return the subset names from disabled_subsets that are actually referenced in recipe.""" + from autoskillit.recipe import make_validation_context, run_semantic_rules + + ctx = make_validation_context(recipe, disabled_subsets=disabled_subsets) + findings = run_semantic_rules(ctx) + needed: set[str] = set() + for f in findings: + if f.rule not in ("subset-disabled-skill", "subset-disabled-tool"): + continue + m = re.search(r"disabled subset '([^']+)'", f.message) + if m: + needed.add(m.group(1)) + return frozenset(needed) + + +def _get_packs_needed(recipe: Recipe, default_disabled_packs: frozenset[str]) -> frozenset[str]: + """Return pack names from default_disabled_packs that are required by recipe.""" + requires = frozenset(getattr(recipe, "requires_packs", [])) + return requires & default_disabled_packs + + +def _enable_packs_permanently(project_dir: Path, packs: frozenset[str]) -> None: + """Add specified packs to packs.enabled in .autoskillit/config.yaml.""" + from autoskillit.core import YAMLError, dump_yaml_str, load_yaml + + config_path = project_dir / ".autoskillit" / "config.yaml" + try: + data: dict = (load_yaml(config_path) or {}) if config_path.exists() else {} + except YAMLError: + data = {} + packs_section = data.setdefault("packs", {}) + current_enabled: list[str] = packs_section.get("enabled", []) + new_enabled = sorted(set(current_enabled) | packs) + packs_section["enabled"] = new_enabled + config_path.parent.mkdir(parents=True, exist_ok=True) + atomic_write(config_path, dump_yaml_str(data, default_flow_style=False, allow_unicode=True)) + print(f"Updated {config_path}: added {sorted(packs)} to packs.enabled") + + +def _enable_subsets_permanently(project_dir: Path, subsets: frozenset[str]) -> None: + """Remove specified subsets from subsets.disabled in .autoskillit/config.yaml.""" + from autoskillit.core import YAMLError, dump_yaml_str, load_yaml + + config_path = project_dir / ".autoskillit" / "config.yaml" + try: + data: dict = (load_yaml(config_path) or {}) if config_path.exists() else {} + except YAMLError: + data = {} + subsets_section = data.setdefault("subsets", {}) + current_disabled: list[str] = subsets_section.get("disabled", []) + subsets_section["disabled"] = [s for s in current_disabled if s not in subsets] + config_path.parent.mkdir(parents=True, exist_ok=True) + atomic_write(config_path, dump_yaml_str(data, default_flow_style=False, allow_unicode=True)) + print(f"Updated {config_path}: removed {sorted(subsets)} from subsets.disabled") + + +def order( + recipe: str | None = None, session_id: str | None = None, *, resume: bool = False +) -> None: + """Launch an interactive Claude Code session to execute a recipe. + + Starts Claude Code with hard tool restrictions: only AskUserQuestion + (built-in) and AutoSkillit MCP tools are available. The session + discovers recipe content by calling load_recipe as its first action. + + Parameters + ---------- + recipe + Name of the recipe (from .autoskillit/recipes/). Prompts if omitted. + session_id + Explicit session ID to resume. Provide after the recipe name. + Implies --resume when non-None. + resume + When True, attempt to restore a previous session. + """ + from autoskillit.recipe import ( + NON_INTERACTIVE_KINDS, + find_recipe_by_name, + list_recipes, + load_recipe, + validate_recipe_structure, + ) + + if os.environ.get("CLAUDECODE"): + print("ERROR: 'order' cannot run inside a Claude Code session.") + print("Run this command in a regular terminal.") + sys.exit(1) + _resume = resume or (session_id is not None) + resume_spec = resume_spec_from_cli(resume=_resume, session_id=session_id) + + if _resume and recipe is not None and session_id is None and _UUID_RE.match(recipe): + session_id = recipe + recipe = None + resume_spec = resume_spec_from_cli(resume=True, session_id=session_id) + + if _resume and recipe is None: + from autoskillit.cli._prompts import _OPEN_KITCHEN_GREETINGS + from autoskillit.cli.session._session_picker import pick_session as _pick_session + from autoskillit.core import BareResume, NamedResume, NoResume + + if isinstance(resume_spec, BareResume): + _sel = _pick_session("order", Path.cwd()) + resume_spec = NamedResume(session_id=_sel) if _sel else NoResume() + _launch_cook_session( + "", + initial_message=random.choice(_OPEN_KITCHEN_GREETINGS), + resume_spec=resume_spec, + extra_env=_write_order_entry(Path.cwd(), None), + ) + return + + mcp_prefix = detect_autoskillit_mcp_prefix() + + from autoskillit.cli.ui._timed_input import timed_prompt + + if recipe is None: + from autoskillit.cli._prompts import _build_open_kitchen_prompt + from autoskillit.cli.ui._menu import SLOT_ZERO_SELECTED, run_selection_menu + from autoskillit.recipe import GROUP_LABELS, group_rank + + available = list_recipes( + Path.cwd(), + exclude_kinds=NON_INTERACTIVE_KINDS, + ).items + if not available: + print("No recipes found. Run 'autoskillit recipes list' to check.") + sys.exit(1) + + resolved = run_selection_menu( + available, + header="Available recipes:", + slot_zero_label="Open kitchen (no recipe)", + group_classifier=group_rank, + group_labels=GROUP_LABELS, + name_key=lambda r: r.name, + timeout=120, + label="autoskillit order", + ) + if resolved is SLOT_ZERO_SELECTED: + from autoskillit.cli._prompts import _OPEN_KITCHEN_GREETINGS + + _launch_cook_session( + _build_open_kitchen_prompt(mcp_prefix=mcp_prefix), + initial_message=random.choice(_OPEN_KITCHEN_GREETINGS), + resume_spec=resume_spec, + project_dir=Path.cwd(), + extra_env=_write_order_entry(Path.cwd(), None), + ) + return + elif resolved is None: + print("Invalid selection.") + sys.exit(1) + else: + if isinstance(resolved, str): + raise TypeError(f"Expected RecipeInfo, got str: {resolved!r}") + recipe = resolved.name + + from autoskillit.core import YAMLError + + _match = find_recipe_by_name(recipe, Path.cwd()) + if _match is None: + available = list_recipes( + Path.cwd(), + exclude_kinds=NON_INTERACTIVE_KINDS, + ).items + print(f"Recipe not found: '{recipe}'") + if available: + print("Available recipes:") + for r in available: + print(f" - {r.name}") + else: + print("No recipes found") + sys.exit(1) + # Validate recipe before launching session + try: + parsed = load_recipe(_match.path) + except YAMLError as exc: + print(f"Recipe YAML parse error: {exc}") + sys.exit(1) + except ValueError as exc: + print(f"Recipe structure error: {exc}") + sys.exit(1) + + errors = validate_recipe_structure(parsed) + if errors: + print(f"Recipe '{recipe}' failed validation:") + for err in errors: + print(f" - {err}") + sys.exit(1) + + # Subset-disabled gate (REQ-VAL-004) + from autoskillit.config import load_config as _load_config + + _cfg = _load_config(Path.cwd()) + _disabled = frozenset(_cfg.subsets.disabled) + _extra_env: dict[str, str] = {} + + if _disabled: + _needed = _get_subsets_needed(parsed, _disabled) + if _needed: + subset_list = ", ".join(sorted(_needed)) + print(f"\nThis recipe requires subset(s): {subset_list}") + print(" 1. Enable temporarily (for this run only)") + print(" 2. Enable permanently (update .autoskillit/config.yaml)") + print(" 3. Cancel") + _choice = timed_prompt( + "Choose [1/2/3]:", default="3", timeout=120, label="autoskillit order" + ) + if _choice == "1": + _extra_env["AUTOSKILLIT_SUBSETS__DISABLED"] = "@json []" + elif _choice == "2": + _enable_subsets_permanently(Path.cwd(), _needed) + else: + return + + # Pack gate — check default-disabled packs (REQ-PACK-010) + from autoskillit.core import PACK_REGISTRY as _PACK_REGISTRY + + _default_disabled = frozenset( + tag for tag, pack_def in _PACK_REGISTRY.items() if not pack_def.default_enabled + ) + _pack_enabled = frozenset(_cfg.packs.enabled) + _default_disabled_packs = _default_disabled - _pack_enabled + + if _default_disabled_packs: + _packs_needed = _get_packs_needed(parsed, _default_disabled_packs) + if _packs_needed: + pack_list = ", ".join(sorted(_packs_needed)) + print(f"\nThis recipe requires pack(s): {pack_list}") + print(" 1. Enable temporarily (for this run only)") + print(" 2. Enable permanently (update .autoskillit/config.yaml)") + print(" 3. Cancel") + _pack_choice = timed_prompt( + "Choose [1/2/3]:", default="3", timeout=120, label="autoskillit order" + ) + if _pack_choice == "1": + import json as _json + + _extra_env["AUTOSKILLIT_PACKS__ENABLED"] = "@json " + _json.dumps( + sorted(_packs_needed) + ) + elif _pack_choice == "2": + _enable_packs_permanently(Path.cwd(), _packs_needed) + else: + return + + from autoskillit.cli._preview import show_cook_preview + from autoskillit.cli._prompts import _COOK_GREETINGS + + _itable = _get_ingredients_table(recipe, _match, Path.cwd()) + show_cook_preview(recipe, parsed, _recipes_dir_for(_match), Path.cwd()) + + from autoskillit.cli.ui._ansi import permissions_warning + + print(permissions_warning()) + confirm = timed_prompt( + "Launch session? [Enter/n]", default="", timeout=120, label="autoskillit order" + ) + if confirm.lower() in ("n", "no"): + return + greeting = random.choice(_COOK_GREETINGS).format(recipe_name=recipe) + _extra_env |= _write_order_entry(Path.cwd(), recipe) + _launch_cook_session( + _build_orchestrator_prompt(recipe, mcp_prefix=mcp_prefix, ingredients_table=_itable), + initial_message=greeting, + extra_env=_extra_env, + resume_spec=resume_spec, + project_dir=Path.cwd(), + ) diff --git a/src/autoskillit/cli/session/_reload.py b/src/autoskillit/cli/session/_reload.py new file mode 100644 index 0000000000..563e992162 --- /dev/null +++ b/src/autoskillit/cli/session/_reload.py @@ -0,0 +1,36 @@ +"""Reload sentinel detection for interactive session re-launch loops.""" + +from __future__ import annotations + +import json +from pathlib import Path + + +def _reload_sentinel_dir(project_dir: Path) -> Path: + return project_dir / ".autoskillit" / "temp" / "reload_sentinel" + + +def consume_reload_sentinel(project_dir: Path) -> str | None: + """Scan for a reload sentinel file; if found, consume and return session_id.""" + sentinel_dir = _reload_sentinel_dir(project_dir) + if not sentinel_dir.is_dir(): + return None + candidates = sorted(sentinel_dir.glob("*.json"), key=lambda p: p.stat().st_mtime, reverse=True) + if not candidates: + return None + for stale in candidates[1:]: + try: + stale.unlink(missing_ok=True) + except OSError: + pass + sentinel = candidates[0] + try: + data = json.loads(sentinel.read_text(encoding="utf-8")) + session_id = data.get("session_id", "") + except (OSError, json.JSONDecodeError, ValueError): + return None + try: + sentinel.unlink(missing_ok=True) + except OSError: + pass + return session_id or None diff --git a/src/autoskillit/cli/session/_session_launch.py b/src/autoskillit/cli/session/_session_launch.py new file mode 100644 index 0000000000..2694682db3 --- /dev/null +++ b/src/autoskillit/cli/session/_session_launch.py @@ -0,0 +1,154 @@ +"""Shared interactive session launch prelude for CLI commands.""" + +from __future__ import annotations + +import shutil +import subprocess +import sys +from dataclasses import dataclass +from pathlib import Path +from typing import TYPE_CHECKING + +from autoskillit.core import NamedResume, NoResume + +if TYPE_CHECKING: + from autoskillit.core import ResumeSpec + + +@dataclass(frozen=True, slots=True) +class _InfraExitSignal: + session_id: str + category: str + + +def _run_interactive_session( + system_prompt: str, + *, + initial_message: str | None = None, + extra_env: dict[str, str] | None = None, + resume_spec: ResumeSpec | None = None, + project_dir: Path | None = None, +) -> str | _InfraExitSignal | None: + """Launch an interactive Claude Code session. + + Returns: + str — session_id when a reload sentinel is found + _InfraExitSignal — when an infrastructure exit is detected + None — clean exit + """ + if shutil.which("claude") is None: + print("ERROR: 'claude' not found. Install: https://docs.anthropic.com/en/docs/claude-code") + sys.exit(1) + from autoskillit.cli.session._reload import consume_reload_sentinel + from autoskillit.cli.ui._terminal import terminal_guard + from autoskillit.core import ( + MARKETPLACE_PREFIX, + BareResume, + ClaudeFlags, + InfraExitCategory, + NamedResume, + NoResume, + detect_autoskillit_mcp_prefix, + pkg_root, + ) + from autoskillit.execution import build_interactive_cmd, read_session_state + + _project_dir = project_dir if project_dir is not None else Path.cwd() + spec = build_interactive_cmd( + initial_prompt=initial_message, + resume_spec=resume_spec if resume_spec is not None else NoResume(), + env_extras=extra_env, + ) + plugin_flags = ( + [] + if detect_autoskillit_mcp_prefix() == MARKETPLACE_PREFIX + else [ClaudeFlags.PLUGIN_DIR, str(pkg_root())] + ) + _is_resume = isinstance(resume_spec, (BareResume, NamedResume)) + cmd = [ + *spec.cmd, + *plugin_flags, + ClaudeFlags.TOOLS, + "AskUserQuestion", + *([] if _is_resume else [ClaudeFlags.APPEND_SYSTEM_PROMPT, system_prompt]), + ] + with terminal_guard(): + result = subprocess.run(cmd, env=spec.env) + reload_session_id = consume_reload_sentinel(_project_dir) + if reload_session_id is not None: + return reload_session_id + if result.returncode != 0: + from autoskillit.core import ensure_project_temp + + state_dir = ensure_project_temp(_project_dir) / "session_state" + state = read_session_state(state_dir) + if ( + state is not None + and state.infra_exit_category + and state.infra_exit_category != InfraExitCategory.COMPLETED + and state.session_id + ): + return _InfraExitSignal( + session_id=state.session_id, category=state.infra_exit_category + ) + sys.exit(result.returncode) + return None + + +def _write_order_entry(project_dir: Path, recipe_name: str | None) -> dict[str, str]: + import uuid + + from autoskillit.cli.session._constants import SESSION_TYPE_ORDER + from autoskillit.core import ( + LAUNCH_ID_ENV_VAR, + SESSION_TYPE_ENV_VAR, + write_registry_entry, + ) + + lid = uuid.uuid4().hex[:16] + write_registry_entry(project_dir, lid, SESSION_TYPE_ORDER, recipe_name) + return {SESSION_TYPE_ENV_VAR: SESSION_TYPE_ORDER, LAUNCH_ID_ENV_VAR: lid} + + +def _launch_cook_session( + system_prompt: str, + *, + initial_message: str | None = None, + extra_env: dict[str, str] | None = None, + resume_spec: ResumeSpec = NoResume(), + project_dir: Path | None = None, +) -> None: + """Launch an interactive Claude Code cook session with reload and infra-resume support.""" + _max_reloads = 10 + _max_infra_resumes = 3 + current_resume_spec: ResumeSpec = resume_spec + _current_initial_message = initial_message + seen_reload_ids: set[str] = set() + infra_resume_count = 0 + while True: + session_signal = _run_interactive_session( + system_prompt, + initial_message=_current_initial_message, + extra_env=extra_env, + resume_spec=current_resume_spec, + project_dir=project_dir, + ) + if session_signal is None: + break + if isinstance(session_signal, _InfraExitSignal): + infra_resume_count += 1 + if infra_resume_count >= _max_infra_resumes: + raise SystemExit( + f"Too many infrastructure resumes ({_max_infra_resumes} max). " + f"Last exit: {session_signal.category}" + ) + current_resume_spec = NamedResume(session_id=session_signal.session_id) + _current_initial_message = None + continue + if len(seen_reload_ids) >= _max_reloads: + raise SystemExit(f"Too many reloads ({_max_reloads} max). Check for infinite loop.") + if session_signal in seen_reload_ids: + raise SystemExit(f"Repeated reload_id {session_signal!r} — aborting.") + seen_reload_ids.add(session_signal) + current_resume_spec = NamedResume(session_id=session_signal) + _current_initial_message = None diff --git a/src/autoskillit/cli/session/_session_picker.py b/src/autoskillit/cli/session/_session_picker.py new file mode 100644 index 0000000000..ce6279b110 --- /dev/null +++ b/src/autoskillit/cli/session/_session_picker.py @@ -0,0 +1,129 @@ +"""Scoped resume picker: shows a filtered list of cook or order sessions.""" + +from __future__ import annotations + +import json +from pathlib import Path + +from autoskillit.core import claude_code_project_dir + +_ORDER_GREETING_PREFIXES = ( + "Today's special:", + "Order up! Today's special:", + "Order up! The kitchen", + "Kitchen's open!", + "Table for one!", + "Fresh off the menu", + "Welcome to Good Burger, home of the Good Burger, can I take your order?", +) + + +def pick_session(session_type: str, project_dir: Path) -> str | None: + """Show filtered picker. Returns selected Claude session UUID or None (fresh start).""" + from autoskillit.core import read_registry + + registry = read_registry(project_dir) + sessions = _load_sessions_index(project_dir) + filtered = [s for s in sessions if _classify_session(s, registry) == session_type] + + if not filtered: + print(f"No {session_type} sessions found. Starting fresh.") + return None + + return _run_picker(filtered, session_type, registry) + + +def _load_sessions_index(project_dir: Path) -> list[dict]: + """Load sessions-index.json, filtering out sidechain entries.""" + index_path = claude_code_project_dir(str(project_dir)) / "sessions-index.json" + try: + entries: list[dict] = json.loads(index_path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError): + return [] + + return [e for e in entries if not e.get("isSidechain")] + + +def _classify_session(entry: dict, registry: dict[str, dict]) -> str: + """Classify session as 'cook' or 'order'. + + Uses registry lookup first, then greeting heuristic fallback. + """ + session_id = entry.get("sessionId", "") + for reg_entry in registry.values(): + if reg_entry.get("claude_session_id") == session_id: + return str(reg_entry.get("session_type", "cook")) + + first_prompt = entry.get("firstPrompt", "") + for prefix in _ORDER_GREETING_PREFIXES: + if first_prompt.startswith(prefix): + return "order" + return "cook" + + +def _format_session_row(entry: dict, session_type: str, registry: dict[str, dict]) -> str: + """Format a session entry as a display row.""" + recipe_name: str | None = None + session_id = entry.get("sessionId", "") + for reg_entry in registry.values(): + if reg_entry.get("claude_session_id") == session_id: + recipe_name = reg_entry.get("recipe_name") + break + + if session_type == "order" and recipe_name: + badge = f"[order: {recipe_name}]" + elif session_type == "order": + badge = "[order]" + else: + badge = "[cook]" + + summary = (entry.get("summary") or entry.get("firstPrompt", ""))[:60] + branch = entry.get("gitBranch", "") + modified = entry.get("modified", "") + + parts = [badge, summary] + if branch: + parts.append(branch) + if modified: + parts.append(modified) + return " ".join(p for p in parts if p) + + +def _run_picker(sessions: list[dict], session_type: str, registry: dict[str, dict]) -> str | None: + """Print numbered list and prompt user for selection. + + Returns sessions[n-1]["sessionId"] on valid selection, None on 0. + Re-prompts on invalid input (max 3 retries, then returns None). + """ + print(f"\nResume a {session_type} session:") + print(" 0. Start fresh session") + for i, entry in enumerate(sessions, 1): + row = _format_session_row(entry, session_type, registry) + print(f" {i}. {row}") + + from autoskillit.cli.ui._timed_input import timed_prompt + + max_retries = 3 + for _ in range(max_retries): + try: + raw = timed_prompt( + f"\nSelect [0-{len(sessions)}]: ", timeout=0, label="session picker" + ) + except KeyboardInterrupt: + return None + + if not raw: + continue + try: + choice = int(raw) + except ValueError: + print(f"Invalid input '{raw}'. Enter a number between 0 and {len(sessions)}.") + continue + + if choice == 0: + return None + if 1 <= choice <= len(sessions): + return str(sessions[choice - 1]["sessionId"]) + print(f"Out of range. Enter a number between 0 and {len(sessions)}.") + + return None diff --git a/src/autoskillit/cli/ui/CLAUDE.md b/src/autoskillit/cli/ui/CLAUDE.md new file mode 100644 index 0000000000..2c40ae58ae --- /dev/null +++ b/src/autoskillit/cli/ui/CLAUDE.md @@ -0,0 +1,17 @@ +# ui/ + +Terminal UI primitives for the CLI layer. + +## Files + +| File | Purpose | +|------|---------| +| `__init__.py` | Package marker (no imports) | +| `_ansi.py` | `supports_color()` — respects `NO_COLOR` and `TERM=dumb` | +| `_menu.py` | `run_selection_menu()`, `render_numbered_menu()`, `SLOT_ZERO_SELECTED` sentinel | +| `_terminal.py` | `terminal_guard()` — saves/restores TTY attributes around interactive subprocess launch | +| `_timed_input.py` | `timed_prompt()` and `status_line()` — every CLI `input()` must go through `timed_prompt()` | + +## Architecture Notes + +`_timed_input.py` is the lowest-level primitive; `_menu.py` depends on it. `_terminal.py` is independent. The `timed_prompt()` contract is enforced by `test_input_tty_contracts.py`. diff --git a/src/autoskillit/cli/ui/__init__.py b/src/autoskillit/cli/ui/__init__.py new file mode 100644 index 0000000000..d0091b0781 --- /dev/null +++ b/src/autoskillit/cli/ui/__init__.py @@ -0,0 +1 @@ +"""CLI UI primitives — color, terminal, prompts, menus.""" diff --git a/src/autoskillit/cli/_ansi.py b/src/autoskillit/cli/ui/_ansi.py similarity index 90% rename from src/autoskillit/cli/_ansi.py rename to src/autoskillit/cli/ui/_ansi.py index 6f1448691b..5edd61a9bf 100644 --- a/src/autoskillit/cli/_ansi.py +++ b/src/autoskillit/cli/ui/_ansi.py @@ -1,4 +1,4 @@ -"""Terminal color utilities (L3 — CLI layer only).""" +"""Terminal color utilities (IL-3 — CLI layer only).""" from __future__ import annotations @@ -121,11 +121,23 @@ def diagram_to_terminal(md: str) -> str: _C = "\x1b[96m" if color else "" _R = "\x1b[0m" if color else "" - _skip = {"|"{count}"| {bid}["{b}"]') + return "\n".join(lines) + + +def render_adjacency_table(dfg: DFG, *, top_n: int = 20) -> str: + """Render top-N bigrams as a plain-text ASCII table.""" + top = list(islice(dfg.bigrams.most_common(top_n), top_n)) + col_a = max((len(a) for (a, _), _ in top), default=6) + col_b = max((len(b) for (_, b), _ in top), default=6) + col_a = max(col_a, len("tool_a")) + col_b = max(col_b, len("tool_b")) + header = f"{'tool_a':<{col_a}} {'tool_b':<{col_b}} count" + sep = "-" * len(header) + rows = [header, sep] + for (a, b), count in top: + rows.append(f"{a:<{col_a}} {b:<{col_b}} {count}") + return "\n".join(rows) + + +def render_dot(dfg: DFG, *, min_count: int = 1, top_n: int = 30) -> str: + """Render bigram DFG as Graphviz DOT digraph.""" + lines = ["digraph tool_sequences {", ' rankdir="LR";'] + for (a, b), count in dfg.bigrams.most_common(top_n): + if count < min_count: + continue + lines.append(f' "{a}" -> "{b}" [label="{count}"];') + lines.append("}") + return "\n".join(lines) + + +def filter_sessions_by_recipe(sessions: Sequence[TurnSequence], recipe: str) -> list[TurnSequence]: + """Return sessions matching recipe_name == recipe.""" + result = [] + for s in sessions: + if s.recipe_name == recipe: + result.append(s) + return result + + +def format_top_bigrams(dfg: DFG, top_n: int, min_count: int) -> list[dict[str, object]]: + """Return top-N bigrams as a list of dicts for JSON serialization.""" + result: list[dict[str, object]] = [] + for (a, b), c in dfg.bigrams.most_common(top_n): + if c >= min_count: + result.append({"from": a, "to": b, "count": c}) + return result diff --git a/src/autoskillit/core/types.py b/src/autoskillit/core/types.py deleted file mode 100644 index 7d72cd57b0..0000000000 --- a/src/autoskillit/core/types.py +++ /dev/null @@ -1,24 +0,0 @@ -"""Core type contracts: re-export hub. - -All symbols are defined in the _type_*.py sub-modules. Import from -autoskillit.core (the package gateway) in production code — not from here. -""" - -from __future__ import annotations - -from ._type_constants import * # noqa: F401, F403 -from ._type_constants import __all__ as _constants_all -from ._type_enums import * # noqa: F401, F403 -from ._type_enums import __all__ as _enums_all -from ._type_helpers import * # noqa: F401, F403 -from ._type_helpers import __all__ as _helpers_all -from ._type_protocols import * # noqa: F401, F403 -from ._type_protocols import __all__ as _protocols_all -from ._type_results import * # noqa: F401, F403 -from ._type_results import __all__ as _results_all -from ._type_subprocess import * # noqa: F401, F403 -from ._type_subprocess import __all__ as _subprocess_all - -__all__ = ( - _constants_all + _enums_all + _helpers_all + _protocols_all + _results_all + _subprocess_all -) diff --git a/src/autoskillit/core/types/CLAUDE.md b/src/autoskillit/core/types/CLAUDE.md new file mode 100644 index 0000000000..a5648f1229 --- /dev/null +++ b/src/autoskillit/core/types/CLAUDE.md @@ -0,0 +1,32 @@ +# types/ + +Type re-export hub and all typed building blocks for the autoskillit package (IL-0). + +## Files + +| File | Purpose | +|------|---------| +| `__init__.py` | Re-export hub — aggregates `__all__` from all `_type_*.py` modules | +| `_type_enums.py` | All `StrEnum` discriminators (`RetryReason`, `KillReason`, `Severity`, etc.) | +| `_type_constants.py` | Shared constants: tool lists, env var names, version string | +| `_type_subprocess.py` | `SubprocessResult` dataclass and `SubprocessRunner` protocol | +| `_type_results_execution.py` | Execution-scoped result dataclasses: `SessionTelemetry`, `RecipeIdentity`, `CIRunScope` | +| `_type_results.py` | Core result dataclasses: `SkillResult`, `ProviderOutcome`, `LoadResult`, `FailureRecord`, `WriteBehaviorSpec` | +| `_type_protocols_logging.py` | Protocols: `AuditLog`, `TokenLog`, `TimingLog`, `McpResponseLog`, `GitHubApiLog`, `SupportsDebug`, `SupportsLogger` | +| `_type_protocols_execution.py` | Protocols: `TestRunner`, `HeadlessExecutor`, `OutputPatternResolver`, `WriteExpectedResolver` | +| `_type_protocols_github.py` | Protocols: `GitHubFetcher`, `CIWatcher`, `MergeQueueWatcher` | +| `_type_protocols_workspace.py` | Protocols: `WorkspaceManager`, `CloneManager`, `SessionSkillManager`, `SkillLister`, `SkillResolver` | +| `_type_protocols_recipe.py` | Protocols: `RecipeRepository`, `MigrationService`, `DatabaseReader`, `ReadOnlyResolver` | +| `_type_protocols_infra.py` | Protocols: `GateState`, `BackgroundSupervisor`, `FleetLock`, `QuotaRefreshTask`, `TokenFactory`, `CampaignProtector` | +| `_type_checkpoint.py` | `SessionCheckpoint` frozen dataclass and `compute_remaining()` helper for session resume | +| `_type_helpers.py` | Text processing and skill-name extraction utilities | +| `_type_resume.py` | `ResumeSpec` discriminated union: `NoResume | BareResume | NamedResume` | +| `_type_plugin_source.py` | `PluginSource` discriminated union: `DirectInstall | MarketplaceInstall` | + +## Architecture Notes + +Internal dependency DAG: enums -> constants -> subprocess -> results_execution; enums -> results -> protocols -> helpers. All modules have zero `autoskillit` imports outside this sub-package (IL-0 hard constraint). Production code imports from `autoskillit.core`, not from this package directly. + +## Extension Bundle Pattern + +New feature fields go on frozen dataclass bundles (`InfraOutcome`, `ProviderOutcome`), not flat on `SkillResult`. Bundles are embedded as `field(default_factory=...)` on `SkillResult`. The `to_json()` method flattens bundle fields to top-level JSON keys for backward compatibility. diff --git a/src/autoskillit/core/types/__init__.py b/src/autoskillit/core/types/__init__.py new file mode 100644 index 0000000000..4de1adb51a --- /dev/null +++ b/src/autoskillit/core/types/__init__.py @@ -0,0 +1,56 @@ +"""Core type contracts: re-export hub. + +All symbols are defined in the _type_*.py sub-modules. Import from +autoskillit.core (the package gateway) in production code — not from here. +""" + +from __future__ import annotations + +from ._type_checkpoint import * # noqa: F401, F403 +from ._type_checkpoint import __all__ as _checkpoint_all +from ._type_constants import * # noqa: F401, F403 +from ._type_constants import __all__ as _constants_all +from ._type_enums import * # noqa: F401, F403 +from ._type_enums import __all__ as _enums_all +from ._type_helpers import * # noqa: F401, F403 +from ._type_helpers import __all__ as _helpers_all +from ._type_plugin_source import * # noqa: F401, F403 +from ._type_plugin_source import __all__ as _plugin_source_all +from ._type_protocols_execution import * # noqa: F401, F403 +from ._type_protocols_execution import __all__ as _protocols_execution_all +from ._type_protocols_github import * # noqa: F401, F403 +from ._type_protocols_github import __all__ as _protocols_github_all +from ._type_protocols_infra import * # noqa: F401, F403 +from ._type_protocols_infra import __all__ as _protocols_infra_all +from ._type_protocols_logging import * # noqa: F401, F403 +from ._type_protocols_logging import __all__ as _protocols_logging_all +from ._type_protocols_recipe import * # noqa: F401, F403 +from ._type_protocols_recipe import __all__ as _protocols_recipe_all +from ._type_protocols_workspace import * # noqa: F401, F403 +from ._type_protocols_workspace import __all__ as _protocols_workspace_all +from ._type_results import * # noqa: F401, F403 +from ._type_results import __all__ as _results_all +from ._type_results_execution import * # noqa: F401, F403 +from ._type_results_execution import __all__ as _results_execution_all +from ._type_resume import * # noqa: F401, F403 +from ._type_resume import __all__ as _resume_all +from ._type_subprocess import * # noqa: F401, F403 +from ._type_subprocess import __all__ as _subprocess_all + +__all__ = ( + _checkpoint_all + + _constants_all + + _enums_all + + _helpers_all + + _plugin_source_all + + _protocols_logging_all + + _protocols_execution_all + + _protocols_github_all + + _protocols_workspace_all + + _protocols_recipe_all + + _protocols_infra_all + + _results_all + + _results_execution_all + + _resume_all + + _subprocess_all +) diff --git a/src/autoskillit/core/types/_type_checkpoint.py b/src/autoskillit/core/types/_type_checkpoint.py new file mode 100644 index 0000000000..c85def411e --- /dev/null +++ b/src/autoskillit/core/types/_type_checkpoint.py @@ -0,0 +1,58 @@ +"""Session checkpoint for resume progress tracking. + +Zero autoskillit imports outside this sub-package. IL-0 type contract. +""" + +from __future__ import annotations + +from collections.abc import Sequence +from dataclasses import dataclass, field +from datetime import UTC, datetime +from typing import Any + + +@dataclass(frozen=True, slots=True) +class SessionCheckpoint: + completed_items: list[str] = field(default_factory=list) + step_name: str = "" + progress_pct: float = 0.0 + ts: str = "" + + def to_dict(self) -> dict[str, Any]: + return { + "completed_items": list(self.completed_items), + "step_name": self.step_name, + "progress_pct": self.progress_pct, + "ts": self.ts, + } + + @classmethod + def from_dict(cls, data: dict[str, Any]) -> SessionCheckpoint: + return cls( + completed_items=list(data.get("completed_items", [])), + step_name=str(data.get("step_name", "")), + progress_pct=float(data.get("progress_pct", 0.0)), + ts=str(data.get("ts", "")), + ) + + @classmethod + def now( + cls, completed_items: list[str], step_name: str = "", progress_pct: float = 0.0 + ) -> SessionCheckpoint: + return cls( + completed_items=completed_items, + step_name=step_name, + progress_pct=progress_pct, + ts=datetime.now(tz=UTC).isoformat(), + ) + + +def compute_remaining(original_items: Sequence[str], checkpoint: SessionCheckpoint) -> list[str]: + completed = set(checkpoint.completed_items) + return [item for item in original_items if item not in completed] + + +__all__ = [ + "SessionCheckpoint", + "compute_remaining", +] diff --git a/src/autoskillit/core/types/_type_constants.py b/src/autoskillit/core/types/_type_constants.py new file mode 100644 index 0000000000..022ac39586 --- /dev/null +++ b/src/autoskillit/core/types/_type_constants.py @@ -0,0 +1,641 @@ +"""Core constants for autoskillit. + +Zero autoskillit imports. Provides the shared constant vocabulary for all higher layers. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from datetime import date +from importlib.metadata import version +from typing import NamedTuple + +from ._type_enums import FeatureLifecycle, FleetErrorCode, IssueLabelState + +__all__ = [ + "AUTOSKILLIT_INSTALLED_VERSION", + "AUTOSKILLIT_PRIVATE_ENV_VARS", + "CONTEXT_EXHAUSTION_MARKER", + "RESERVED_LOG_RECORD_KEYS", + "PIPELINE_FORBIDDEN_TOOLS", + "SKILL_TOOLS", + "GATED_TOOLS", + "HEADLESS_TOOLS", + "FLEET_TOOLS", + "FLEET_MENU_TOOLS", + "FREE_RANGE_TOOLS", + "UNGATED_TOOLS", + "PackDef", + "PACK_REGISTRY", + "RecipePackDef", + "RECIPE_PACK_REGISTRY", + "RECIPE_PACK_TAGS", + "CORE_PACKS", + "CATEGORY_TAGS", + "TOOL_SUBSET_TAGS", + "ALL_VISIBILITY_TAGS", + "SKILL_COMMAND_PREFIX", + "AUTOSKILLIT_SKILL_PREFIX", + "RETIRED_READINESS_TOKENS", + "SESSION_TYPE_ENV_VAR", + "SESSION_TYPE_FLEET", + "SESSION_TYPE_ORCHESTRATOR", + "SESSION_TYPE_SKILL", + "HEADLESS_ENV_VAR", + "FLEET_MODE_ENV_VAR", + "FLEET_DISPATCH_MODE", + "CAMPAIGN_ID_ENV_VAR", + "DISPATCH_ID_ENV_VAR", + "KITCHEN_SESSION_ID_ENV_VAR", + "LAUNCH_ID_ENV_VAR", + "FOOD_TRUCK_TOOL_TAGS_ENV_VAR", + "FLEET_DISPATCH_TOOLS", + "FLEET_ERROR_CODES", + "FeatureDef", + "FEATURE_REGISTRY", + "RETIRED_FEATURES", + "LabelDef", + "LABEL_LIFECYCLE_REGISTRY", + "LABEL_TRANSITIONS", + "validate_label_transition", + "RETIRED_SKILL_NAMES", + "SKILL_FILE_ADVISORY_MAP", + "SKILL_ACTIVATE_DEPS_REQUIRED", + "SOUS_CHEF_MANDATORY_SECTIONS", + "ADMIRAL_DISPATCH_SECTIONS", + "KNOWN_CI_EVENTS", +] + +AUTOSKILLIT_INSTALLED_VERSION: str = version("autoskillit") + +# Session type environment variable and valid values. +# String aliases for consumers that cannot import SessionType StrEnum +# (hook scripts, shell wrappers, env builders). +SESSION_TYPE_ENV_VAR: str = "AUTOSKILLIT_SESSION_TYPE" +SESSION_TYPE_ORCHESTRATOR: str = "orchestrator" +SESSION_TYPE_FLEET: str = "fleet" +SESSION_TYPE_SKILL: str = "skill" +HEADLESS_ENV_VAR: str = "AUTOSKILLIT_HEADLESS" +CAMPAIGN_ID_ENV_VAR: str = "AUTOSKILLIT_CAMPAIGN_ID" +FLEET_MODE_ENV_VAR: str = "AUTOSKILLIT_FLEET_MODE" +FLEET_DISPATCH_MODE: str = "dispatch" +DISPATCH_ID_ENV_VAR: str = "AUTOSKILLIT_DISPATCH_ID" +KITCHEN_SESSION_ID_ENV_VAR: str = "AUTOSKILLIT_KITCHEN_SESSION_ID" +LAUNCH_ID_ENV_VAR: str = "AUTOSKILLIT_LAUNCH_ID" +FOOD_TRUCK_TOOL_TAGS_ENV_VAR: str = "AUTOSKILLIT_FOOD_TRUCK_TOOL_TAGS" + +# Env vars that control MCP server-level behavior and must not leak into +# user-code subprocesses (pytest runs, shell commands, etc.). +# Add new internal vars here as they are introduced. +AUTOSKILLIT_PRIVATE_ENV_VARS: frozenset[str] = frozenset( + { + "AUTOSKILLIT_HEADLESS", + "AUTOSKILLIT_SKIP_STALE_CHECK", + "AUTOSKILLIT_SKIP_UPDATE_CHECK", + "AUTOSKILLIT_SKIP_SOURCE_DRIFT_CHECK", + "AUTOSKILLIT_FORCE_UPDATE_CHECK", + # Fleet tier vars — must not leak into user-code subprocesses + "AUTOSKILLIT_SESSION_TYPE", + "AUTOSKILLIT_FLEET_MODE", + "AUTOSKILLIT_CAMPAIGN_ID", + "AUTOSKILLIT_DISPATCH_ID", + "AUTOSKILLIT_KITCHEN_SESSION_ID", + "AUTOSKILLIT_CAMPAIGN_STATE_PATH", + "AUTOSKILLIT_PROJECT_DIR", + FOOD_TRUCK_TOOL_TAGS_ENV_VAR, + "AUTOSKILLIT_LAUNCH_ID", + "AUTOSKILLIT_SKILL_NAME", + # Provider-routing vars — must not leak into user-code subprocesses + "AUTOSKILLIT_PROVIDER_PROFILE", + } +) + +# The substring Claude CLI emits when the context window is full. +# Used by ClaudeSessionResult._is_context_exhausted() for detection. +# Centralized here so tests can reference the canonical value. +CONTEXT_EXHAUSTION_MARKER = "prompt is too long" + +# Attribute names set unconditionally by logging.LogRecord.__init__ and makeRecord(). +# Passing any of these as keys in the extra={} dict to ctx.info/ctx.error causes +# FastMCP's stdlib logging bridge to raise KeyError at runtime. +# Used by server/_notify._notify() for pre-dispatch validation. +RESERVED_LOG_RECORD_KEYS: frozenset[str] = frozenset( + { + # Set unconditionally in LogRecord.__init__ + "name", + "msg", + "args", + "levelname", + "levelno", + "pathname", + "filename", + "module", + "exc_info", + "exc_text", + "stack_info", + "lineno", + "funcName", + "created", + "msecs", + "relativeCreated", + "thread", + "threadName", + "processName", + "process", + # Python 3.12+ addition + "taskName", + # Additional keys checked explicitly in makeRecord (not in __init__) + "message", + "asctime", + } +) + +# Native Claude Code tools that pipeline orchestrators must NEVER use directly. +# Canonical source of truth — imported by server.py and tests. +PIPELINE_FORBIDDEN_TOOLS: tuple[str, ...] = ( + "Read", + "Grep", + "Glob", + "Edit", + "Write", + "Bash", + "Agent", # actual tool name; "Explore" is a subagent_type parameter, blocked via Agent + "WebFetch", + "WebSearch", + "NotebookEdit", +) + +# Skill tools that route headless Claude sessions — canonical constant used by +# recipe_validator.py. +SKILL_TOOLS: frozenset[str] = frozenset({"run_skill"}) + +# Authoritative MCP tool registries. Defined here (IL-0) so both pipeline/ (IL-1) +# and recipe/ (IL-2) can reference them without cross-layer import violations. +GATED_TOOLS: frozenset[str] = frozenset( + { + "run_cmd", + "run_python", + "read_db", + "run_skill", + "merge_worktree", + "reset_test_dir", + "classify_fix", + "reset_workspace", + "migrate_recipe", + "clone_repo", + "remove_clone", + "push_to_remote", + "report_bug", + "prepare_issue", + "enrich_issues", + "claim_issue", + "release_issue", + "wait_for_ci", + "wait_for_merge_queue", + "check_repo_merge_state", + "toggle_auto_merge", + "enqueue_pr", + "create_unique_branch", + "write_telemetry_files", + "get_pr_reviews", + "bulk_close_issues", + "check_pr_mergeable", + "set_commit_status", + "analyze_tool_sequences", + # Formerly ungated — now kitchen-gated: + "fetch_github_issue", + "get_issue_title", + "get_ci_status", + "get_pipeline_report", + "get_quota_events", + "get_timing_summary", + "get_token_summary", + "kitchen_status", + "list_recipes", + "load_recipe", + "validate_recipe", + "register_clone_status", + "batch_cleanup_clones", + "dispatch_food_truck", + "record_gate_dispatch", + "bootstrap_clone", + "claim_and_resolve_issue", + "create_and_publish_branch", + } +) + +HEADLESS_TOOLS: frozenset[str] = frozenset({"test_check"}) + +FLEET_TOOLS: frozenset[str] = frozenset( + { + "batch_cleanup_clones", + "get_pipeline_report", + "get_token_summary", + "get_timing_summary", + "get_quota_events", + "dispatch_food_truck", + "record_gate_dispatch", + } +) + +FLEET_DISPATCH_TOOLS: frozenset[str] = frozenset( + { + "list_recipes", + "load_recipe", + "fetch_github_issue", + "get_issue_title", + } +) + +# Tools that appear in the Fleet group in the cook menu and open_kitchen response. +# Defined here (IL-0) so menu modules can import the constant without loading the fleet package. +FLEET_MENU_TOOLS: tuple[str, ...] = ("dispatch_food_truck", "record_gate_dispatch") + +FLEET_ERROR_CODES: frozenset[str] = frozenset(FleetErrorCode) + +FREE_RANGE_TOOLS: frozenset[str] = frozenset( + {"open_kitchen", "close_kitchen", "disable_quota_guard", "reload_session"} +) + +UNGATED_TOOLS: frozenset[str] = FREE_RANGE_TOOLS + + +class PackDef(NamedTuple): + """Definition of a named skill pack with default visibility state.""" + + default_enabled: bool + description: str + + +class RecipePackDef(NamedTuple): + """Definition of a named recipe pack with default visibility state.""" + + default_enabled: bool + description: str + + +PACK_REGISTRY: dict[str, PackDef] = { + "kitchen-core": PackDef(True, "Core kitchen orchestration tools"), + "github": PackDef(True, "GitHub issue and PR tools"), + "ci": PackDef(True, "CI polling and merge queue tools"), + "clone": PackDef(True, "Clone-based run isolation tools"), + "telemetry": PackDef(True, "Token, timing, and quota reporting"), + "arch-lens": PackDef(True, "Architecture diagram lenses"), + "audit": PackDef(True, "Codebase audit skills"), + "research": PackDef(False, "Research recipe and experiment skills"), + "exp-lens": PackDef(False, "Experimental design audit lenses"), + "vis-lens": PackDef(False, "Visualization planning lenses"), +} + +CATEGORY_TAGS: frozenset[str] = frozenset(PACK_REGISTRY.keys()) + +RECIPE_PACK_REGISTRY: dict[str, RecipePackDef] = { + "implementation-family": RecipePackDef(True, "Implementation and refactoring recipes"), + "research-family": RecipePackDef(False, "Research and exploration recipes"), + "orchestration-family": RecipePackDef(True, "Campaign orchestration and automation"), +} + +RECIPE_PACK_TAGS: frozenset[str] = frozenset(RECIPE_PACK_REGISTRY.keys()) + +CORE_PACKS: frozenset[str] = frozenset({"github", "ci", "clone", "telemetry"}) + +if any(k != k.lower() for k in PACK_REGISTRY): + raise AssertionError( + "PACK_REGISTRY keys must be lowercase. " + f"Offending: {sorted(k for k in PACK_REGISTRY if k != k.lower())}" + ) +if any(k != k.lower() for k in RECIPE_PACK_REGISTRY): + raise AssertionError( + "RECIPE_PACK_REGISTRY keys must be lowercase. " + f"Offending: {sorted(k for k in RECIPE_PACK_REGISTRY if k != k.lower())}" + ) + +# Maps each MCP tool name to its functional category subset tags. +# Mirrors the FastMCP @mcp.tool(tags=...) category assignments in the server layer. +# Tools with no functional category are absent from this map (empty intersection = no finding). +TOOL_SUBSET_TAGS: dict[str, frozenset[str]] = { + # github + "fetch_github_issue": frozenset({"github", "fleet-dispatch"}), + "get_issue_title": frozenset({"github", "fleet-dispatch"}), + "report_bug": frozenset({"github"}), + "prepare_issue": frozenset({"github"}), + "enrich_issues": frozenset({"github"}), + "claim_issue": frozenset({"github"}), + "release_issue": frozenset({"github"}), + "get_pr_reviews": frozenset({"github"}), + "bulk_close_issues": frozenset({"github"}), + "check_pr_mergeable": frozenset({"github"}), + "push_to_remote": frozenset({"github"}), + "create_unique_branch": frozenset({"github"}), + "set_commit_status": frozenset({"github"}), + "claim_and_resolve_issue": frozenset({"github"}), + "create_and_publish_branch": frozenset({"github"}), + # ci + "wait_for_ci": frozenset({"ci"}), + "wait_for_merge_queue": frozenset({"ci"}), + "check_repo_merge_state": frozenset({"ci"}), + "toggle_auto_merge": frozenset({"ci"}), + "enqueue_pr": frozenset({"ci"}), + "get_ci_status": frozenset({"ci"}), + # clone + "clone_repo": frozenset({"clone"}), + "remove_clone": frozenset({"clone"}), + "register_clone_status": frozenset({"clone"}), + "batch_cleanup_clones": frozenset({"clone", "fleet"}), + "bootstrap_clone": frozenset({"clone"}), + # kitchen-core — telemetry + "get_token_summary": frozenset({"kitchen-core", "telemetry", "fleet"}), + "get_timing_summary": frozenset({"kitchen-core", "telemetry", "fleet"}), + "write_telemetry_files": frozenset({"kitchen-core", "telemetry"}), + "get_quota_events": frozenset({"kitchen-core", "telemetry", "fleet"}), + "analyze_tool_sequences": frozenset({"kitchen-core", "telemetry"}), + # kitchen-core — execution + "run_cmd": frozenset({"kitchen-core"}), + "run_python": frozenset({"kitchen-core"}), + "run_skill": frozenset({"kitchen-core"}), + # kitchen-core — workspace + "test_check": frozenset({"kitchen-core"}), + "reset_test_dir": frozenset({"kitchen-core"}), + "reset_workspace": frozenset({"kitchen-core"}), + "classify_fix": frozenset({"kitchen-core"}), + # kitchen-core — recipe + "list_recipes": frozenset({"kitchen-core", "fleet-dispatch"}), + "load_recipe": frozenset({"kitchen-core", "fleet-dispatch"}), + "validate_recipe": frozenset({"kitchen-core"}), + "migrate_recipe": frozenset({"kitchen-core"}), + # kitchen-core — status + "kitchen_status": frozenset({"kitchen-core"}), + "read_db": frozenset({"kitchen-core"}), + "get_pipeline_report": frozenset({"kitchen-core", "fleet"}), + "dispatch_food_truck": frozenset({"kitchen-core", "fleet"}), + "record_gate_dispatch": frozenset({"kitchen-core", "fleet"}), + # kitchen-core — git + "merge_worktree": frozenset({"kitchen-core"}), +} + +ALL_VISIBILITY_TAGS: frozenset[str] = frozenset( + {"kitchen", "headless", "fleet", "fleet-dispatch", "kitchen-core"} +) + +if not TOOL_SUBSET_TAGS: + raise RuntimeError("TOOL_SUBSET_TAGS is empty — cannot validate ALL_VISIBILITY_TAGS coverage") +_all_tool_tags = {tag for tags in TOOL_SUBSET_TAGS.values() for tag in tags} +_non_category_tool_tags = _all_tool_tags - CATEGORY_TAGS +if not _non_category_tool_tags <= ALL_VISIBILITY_TAGS: + _missing = _non_category_tool_tags - ALL_VISIBILITY_TAGS + raise RuntimeError( + f"ALL_VISIBILITY_TAGS is missing non-category tags found in TOOL_SUBSET_TAGS: " + f"{sorted(_missing)}. Add the missing tags to ALL_VISIBILITY_TAGS." + ) + + +@dataclass(frozen=True, slots=True) +class FeatureDef: + """Definition of a named feature gate.""" + + lifecycle: FeatureLifecycle + description: str + tool_tags: frozenset[str] + skill_categories: frozenset[str] + import_package: str | None + tier: int = 1 + default_enabled: bool = False + depends_on: frozenset[str] = field(default_factory=frozenset) + since_version: str | None = None + sunset_date: date | None = None + + +FEATURE_REGISTRY: dict[str, FeatureDef] = { + "fleet": FeatureDef( + lifecycle=FeatureLifecycle.EXPERIMENTAL, + description="L3 Fleet Orchestrator — multi-session campaign dispatch", + tool_tags=frozenset({"fleet"}), + skill_categories=frozenset({"fleet"}), + import_package="autoskillit.fleet", + tier=1, + default_enabled=False, + since_version="0.9.119", + ), + "planner": FeatureDef( + lifecycle=FeatureLifecycle.EXPERIMENTAL, + description=( + "Progressive resolution planner — 3-pass sequential decomposition" + " into GitHub-issue-ready work packages" + ), + tool_tags=frozenset(), + skill_categories=frozenset({"planner"}), + import_package="autoskillit.planner", + tier=1, + default_enabled=False, + since_version="0.9.119", + ), + "providers": FeatureDef( + lifecycle=FeatureLifecycle.EXPERIMENTAL, + description=( + "Provider routing — route recipe steps to a non-Anthropic LLM" + " provider (e.g. MiniMax M2.7-highspeed)" + ), + tool_tags=frozenset(), + skill_categories=frozenset(), + import_package=None, + tier=1, + default_enabled=False, + since_version="0.9.351", + ), +} + + +@dataclass(frozen=True, slots=True) +class LabelDef: + """Metadata for a lifecycle label managed by the issue state machine.""" + + state: IssueLabelState + color: str + description: str + removes_on_entry: frozenset[IssueLabelState] + + +LABEL_LIFECYCLE_REGISTRY: dict[IssueLabelState, LabelDef] = { + IssueLabelState.QUEUED: LabelDef( + state=IssueLabelState.QUEUED, + color="c2e0c6", + description="Issue claimed by orchestrator, waiting for recipe pickup", + removes_on_entry=frozenset({IssueLabelState.FAIL}), + ), + IssueLabelState.IN_PROGRESS: LabelDef( + state=IssueLabelState.IN_PROGRESS, + color="fbca04", + description="Issue is actively being processed by a pipeline session", + removes_on_entry=frozenset({IssueLabelState.QUEUED, IssueLabelState.FAIL}), + ), + IssueLabelState.STAGED: LabelDef( + state=IssueLabelState.STAGED, + color="0075ca", + description="Issue resolved, PR staged for promotion", + removes_on_entry=frozenset( + {IssueLabelState.IN_PROGRESS, IssueLabelState.FAIL, IssueLabelState.QUEUED} + ), + ), + IssueLabelState.FAIL: LabelDef( + state=IssueLabelState.FAIL, + color="d73a4a", + description="Recipe execution failed", + removes_on_entry=frozenset({IssueLabelState.IN_PROGRESS, IssueLabelState.QUEUED}), + ), +} + +LABEL_TRANSITIONS: dict[IssueLabelState | None, frozenset[IssueLabelState | None]] = { + None: frozenset({IssueLabelState.QUEUED, IssueLabelState.IN_PROGRESS}), + IssueLabelState.QUEUED: frozenset({IssueLabelState.IN_PROGRESS, None}), + IssueLabelState.IN_PROGRESS: frozenset( + { + IssueLabelState.STAGED, + IssueLabelState.FAIL, + None, + } + ), + IssueLabelState.STAGED: frozenset(), + IssueLabelState.FAIL: frozenset({IssueLabelState.QUEUED, IssueLabelState.IN_PROGRESS}), +} + + +def validate_label_transition( + current: IssueLabelState | None, + target: IssueLabelState | None, +) -> None: + """Raise ValueError if the label state transition is not allowed.""" + allowed = LABEL_TRANSITIONS.get(current) + if allowed is not None and target not in allowed: + msg = f"Invalid label transition: {current!r} -> {target!r}" + raise ValueError(msg) + + +for _ls in IssueLabelState: + if _ls not in LABEL_LIFECYCLE_REGISTRY: + raise AssertionError(f"IssueLabelState.{_ls.name} missing from LABEL_LIFECYCLE_REGISTRY") + if _ls not in LABEL_TRANSITIONS: + raise AssertionError(f"IssueLabelState.{_ls.name} missing from LABEL_TRANSITIONS") +if None not in LABEL_TRANSITIONS: + raise AssertionError("LABEL_TRANSITIONS must contain a None (unlabeled) entry") +del _ls + + +RETIRED_FEATURES: frozenset[str] = frozenset() + +if any(k != k.lower() for k in FEATURE_REGISTRY): + raise AssertionError( + "FEATURE_REGISTRY keys must be lowercase. " + f"Offending: {sorted(k for k in FEATURE_REGISTRY if k != k.lower())}" + ) +if any(k != k.lower() for k in RETIRED_FEATURES): + raise AssertionError( + "RETIRED_FEATURES entries must be lowercase. " + f"Offending: {sorted(k for k in RETIRED_FEATURES if k != k.lower())}" + ) + +RETIRED_SKILL_NAMES: frozenset[str] = frozenset( + { + # Skill directory names that have been renamed or removed. + # Append retired names here atomically with the rename/deletion commit. + # DO NOT REMOVE entries — this registry is append-only. + "open-research-pr", # Retired; replaced by decomposed skills + "sprint-planner", # Retired; no replacement + "vis-lens-domain-norms", # Retired; renamed to vis-lens-methodology-norms + } +) + +if any(n != n.lower() for n in RETIRED_SKILL_NAMES): + raise AssertionError( + "RETIRED_SKILL_NAMES entries must be lowercase. " + f"Offending: {sorted(n for n in RETIRED_SKILL_NAMES if n != n.lower())}" + ) + +# Guard: FeatureDef.tool_tags must be in TOOL_SUBSET_TAGS — checked at import time. +_ALL_REGISTERED_TOOL_TAGS: frozenset[str] = frozenset( + tag for tags in TOOL_SUBSET_TAGS.values() for tag in tags +) +if not all( + tag in _ALL_REGISTERED_TOOL_TAGS + for defn in FEATURE_REGISTRY.values() + for tag in defn.tool_tags +): + raise AssertionError( + "FeatureDef.tool_tags contains a tag not present in TOOL_SUBSET_TAGS values. " + "Add the tag to the appropriate tool entry in TOOL_SUBSET_TAGS first." + ) + + +# Canonical prefix required for all skill_command values passed to run_skill. +# Enforced at the Claude Code hook boundary by skill_command_guard.py. +SKILL_COMMAND_PREFIX: str = "/" + +# Canonical prefix for bundled autoskillit slash commands. +AUTOSKILLIT_SKILL_PREFIX: str = "/autoskillit:" + +# Log message tokens that were once used as subprocess readiness sync primitives +# and have since been retired. Any logger call using these tokens as its first +# positional argument is a structural anti-pattern — the lifespan's try: block +# and the anyio signal receiver replaced them with a filesystem sentinel. +# Consumed by test_lifespan_readiness_structural.py (AST Assertion C). +RETIRED_READINESS_TOKENS: frozenset[str] = frozenset( + { + "lifespan_started", + "sigterm_handler_ready", + } +) + +# Maps file-path regex patterns to the advisory skill name to suggest when that +# path is written or edited. Patterns are tried in order; first match wins. +# Campaign paths must appear before the general recipe pattern. +# Stdlib-only hooks inline a copy of the recipe-related subset; the contract +# test test_hook_patterns_match_type_constants asserts they stay in sync. +SKILL_FILE_ADVISORY_MAP: dict[str, str] = { + r"(?:\.autoskillit|src/autoskillit)/recipes/campaigns/.*\.ya?ml$": "make-campaign", + r"(?:\.autoskillit|src/autoskillit)/recipes/.*\.ya?ml$": "write-recipe", +} + +# Pipeline skills that must declare specific activate_deps. Contract test +# test_required_activate_deps_present enforces this invariant at CI time. +SKILL_ACTIVATE_DEPS_REQUIRED: dict[str, frozenset[str]] = { + "make-plan": frozenset({"arch-lens", "write-recipe"}), + "implement-worktree": frozenset({"write-recipe"}), + "implement-worktree-no-merge": frozenset({"write-recipe"}), +} + +# Single registration point: adding a section here surfaces any path that fails to deliver it. +SOUS_CHEF_MANDATORY_SECTIONS: tuple[str, ...] = ( + "MULTI-PART PLAN SEQUENCING", + "SKILL_COMMAND FORMATTING", + "CONTEXT LIMIT ROUTING", + "AUDIT-IMPL ACROSS MULTI-GROUP PIPELINES", + "READING AND ACTING ON `plan_parts=` OUTPUT", + "MULTIPLE ISSUES", + "PARALLEL STEP SCHEDULING", + "EXECUTION MAP — GROUP DISPATCH", + "STEP NAME IMMUTABILITY", + "MERGE PHASE", + "QUOTA WAIT PROTOCOL", + "STEP EXECUTION IS NOT DISCRETIONARY", + "NARRATION SUPPRESSION", +) + +# Strict subset of SOUS_CHEF_MANDATORY_SECTIONS delivered to L3 dispatch sessions. +ADMIRAL_DISPATCH_SECTIONS: tuple[str, ...] = ( + "CONTEXT LIMIT ROUTING", + "STEP NAME IMMUTABILITY", + "MERGE PHASE", + "QUOTA WAIT PROTOCOL", +) +assert set(ADMIRAL_DISPATCH_SECTIONS).issubset(set(SOUS_CHEF_MANDATORY_SECTIONS)) + +KNOWN_CI_EVENTS: frozenset[str] = frozenset( + { + "push", + "pull_request", + "pull_request_target", + "merge_group", + "workflow_dispatch", + "schedule", + "workflow_call", + } +) diff --git a/src/autoskillit/core/_type_enums.py b/src/autoskillit/core/types/_type_enums.py similarity index 65% rename from src/autoskillit/core/_type_enums.py rename to src/autoskillit/core/types/_type_enums.py index c761b3359d..370e73c212 100644 --- a/src/autoskillit/core/_type_enums.py +++ b/src/autoskillit/core/types/_type_enums.py @@ -5,7 +5,7 @@ from __future__ import annotations -from enum import StrEnum +from enum import StrEnum, unique __all__ = [ "RetryReason", @@ -25,6 +25,13 @@ "CliSubtype", "ChannelBStatus", "PRState", + "SessionType", + "FleetErrorCode", + "FeatureLifecycle", + "IssueLabelState", + "DispatchGateType", + "ClaudeContentBlockType", + "InfraExitCategory", ] @@ -35,13 +42,32 @@ class RetryReason(StrEnum): BUDGET_EXHAUSTED = "budget_exhausted" EARLY_STOP = "early_stop" ZERO_WRITES = "zero_writes" - EMPTY_OUTPUT = "empty_output" # NATURAL_EXIT + rc=0 + no output, no partial progress + EMPTY_OUTPUT = "empty_output" # NATURAL_EXIT + rc=0 + no output; no write evidence at exit + COMPLETED_NO_FLUSH = ( + "completed_no_flush" # EMPTY_OUTPUT + write evidence; stdout absent (not merely unflushed) + ) DRAIN_RACE = "drain_race" # channel-confirmed completion, stdout not fully flushed before kill PATH_CONTAMINATION = "path_contamination" # CWD boundary violation, not a context limit CONTRACT_RECOVERY = ( "contract_recovery" # marker present + write evidence — omission not structural ) CLONE_CONTAMINATION = "clone_contamination" + THINKING_STALL = "thinking_stall" # final turn: thinking blocks only, no text or tool output + IDLE_STALL = "idle_stall" # stdout idle watchdog kill — session may have partial progress + + +class InfraExitCategory(StrEnum): + """Infrastructure-level exit classification for headless sessions. + + Distinguishes infrastructure failures (context exhaustion, API errors, + process kills) from logical agent failures (completed with error). + Used for telemetry and resume routing — not a retry discriminant itself. + """ + + COMPLETED = "completed" + CONTEXT_EXHAUSTED = "context_exhausted" + API_ERROR = "api_error" + PROCESS_KILLED = "process_killed" class MergeFailedStep(StrEnum): @@ -49,6 +75,7 @@ class MergeFailedStep(StrEnum): PROTECTED_BRANCH = "protected_branch" BRANCH_DETECTION = "branch_detection" DIRTY_TREE = "dirty_tree" + DIRTY_MAIN_REPO = "dirty_main_repo" TEST_GATE = "test_gate" FETCH = "fetch" PRE_REBASE_CHECK = "pre_rebase_check" @@ -170,6 +197,7 @@ class Severity(StrEnum): OK = "ok" ERROR = "error" WARNING = "warning" + INFO = "info" class TerminationReason(StrEnum): @@ -323,5 +351,126 @@ class PRState(StrEnum): EJECTED_CI_FAILURE = "ejected_ci_failure" STALLED = "stalled" DROPPED_HEALTHY = "dropped_healthy" + DROPPED_MERGE_GROUP_CI = "dropped_merge_group_ci" + NOT_ENROLLED = "not_enrolled" TIMEOUT = "timeout" ERROR = "error" + + +class SessionType(StrEnum): + """Orchestration level discriminator for the session hierarchy. + + Each level can only call the level directly below it: + + L3 (FLEET) -> L2 (ORCHESTRATOR) -> L1 (headless worker) -> L0 (subagent) + + FLEET -- L3: top-level campaign dispatcher. + Launches L2 food trucks via dispatch_food_truck. + ORCHESTRATOR -- L2: recipe runner (interactive via order, or headless food truck). + Launches L1 headless workers via run_skill. + SKILL -- L1 skill session (headless worker launched by an orchestrator via + run_skill). L0 subagents are the actual leaf nodes — they never + set AUTOSKILLIT_SESSION_TYPE and cannot call run_skill. + + Note: interactive L1 sessions (autoskillit cook, bare Claude Code) have no + SessionType value -- they bypass tier checks because AUTOSKILLIT_HEADLESS is unset. + """ + + FLEET = "fleet" + ORCHESTRATOR = "orchestrator" + SKILL = "skill" + + +@unique +class FleetErrorCode(StrEnum): + """Registered error codes for fleet dispatch failures. + + Every fleet error envelope must use one of these codes. + Unregistered codes are rejected by fleet_error() at runtime. + """ + + FLEET_PARALLEL_REFUSED = "fleet_parallel_refused" + FLEET_UNKNOWN_INGREDIENT = "fleet_unknown_ingredient" + FLEET_MISSING_INGREDIENT = "fleet_missing_ingredient" + FLEET_CAMPAIGN_HALTED = "fleet_campaign_halted" + FLEET_RECIPE_NOT_FOUND = "fleet_recipe_not_found" + FLEET_INVALID_RECIPE_KIND = "fleet_invalid_recipe_kind" + FLEET_HARD_REFUSAL_HEADLESS = "fleet_hard_refusal_headless" + FLEET_FEATURE_DISABLED = "fleet_feature_disabled" + FLEET_MANIFEST_MISSING = "fleet_manifest_missing" + FLEET_MANIFEST_CORRUPTED = "fleet_manifest_corrupted" + FLEET_LOCK_NOT_INITIALIZED = "fleet_lock_not_initialized" + FLEET_L3_TIMEOUT = "fleet_l3_timeout" + FLEET_L3_NO_RESULT_BLOCK = "fleet_l3_no_result_block" + FLEET_L3_PARSE_FAILED = "fleet_l3_parse_failed" + FLEET_L3_STARTUP_OR_CRASH = "fleet_l3_startup_or_crash" + FLEET_BUDGET_EXCEEDED = "fleet_budget_exceeded" + FLEET_QUOTA_EXHAUSTED = "fleet_quota_exhausted" + FLEET_CLEANUP_FAILED = "fleet_cleanup_failed" + FLEET_GATE_UNKNOWN_DISPATCH = "fleet_gate_unknown_dispatch" + FLEET_GATE_ALREADY_RECORDED = "fleet_gate_already_recorded" + FLEET_GATE_NO_CAMPAIGN = "fleet_gate_no_campaign" + + +class FeatureLifecycle(StrEnum): + """Lifecycle stage of a registered feature gate. + + EXPERIMENTAL — On by default when experimental_enabled=True; disabled on main/stable. + STABLE — On by default everywhere; opt-out via config. + DEPRECATED — Scheduled for removal; follows default_enabled. + DISABLED — Never enabled; config cannot override. For in-progress/unsafe features. + """ + + EXPERIMENTAL = "experimental" + STABLE = "stable" + DEPRECATED = "deprecated" + DISABLED = "disabled" + + +class IssueLabelState(StrEnum): + """Lifecycle state of a GitHub issue managed by the label state machine.""" + + QUEUED = "queued" + IN_PROGRESS = "in-progress" + STAGED = "staged" + FAIL = "fail" + + +@unique +class DispatchGateType(StrEnum): + """Valid gate types for campaign dispatch entries.""" + + CONFIRM = "confirm" + + +class ClaudeContentBlockType(StrEnum): + """Sealed enum for Claude API content block types. + + Every block type that can appear in an assistant message content array + MUST be a member of this enum. The from_api() constructor maps unknown + block types to UNKNOWN instead of raising ValueError, because the Claude + API may introduce new block types in future versions. + + Exhaustive match dispatch over this enum (with assert_never on the + fallthrough arm) provides compile-time enforcement that all block types + are handled — adding a new member forces the developer to handle it. + """ + + TEXT = "text" + TOOL_USE = "tool_use" + TOOL_RESULT = "tool_result" + THINKING = "thinking" + REDACTED_THINKING = "redacted_thinking" + IMAGE = "image" + UNKNOWN = "unknown" + + @classmethod + def from_api(cls, raw: str) -> ClaudeContentBlockType: + """Convert a raw API block type string to a ClaudeContentBlockType member. + + Unknown strings map to UNKNOWN instead of raising ValueError. + """ + try: + return cls(raw) + except ValueError: + return cls.UNKNOWN diff --git a/src/autoskillit/core/types/_type_helpers.py b/src/autoskillit/core/types/_type_helpers.py new file mode 100644 index 0000000000..b1e5c73a40 --- /dev/null +++ b/src/autoskillit/core/types/_type_helpers.py @@ -0,0 +1,198 @@ +"""Core skill name resolution and text-processing helpers. + +Zero autoskillit imports outside this sub-package. Provides extract_skill_name, +extract_path_arg, resolve_target_skill, truncate_text, fleet_error, and session_type. +""" + +from __future__ import annotations + +import json +import os +import re +import warnings +from typing import Any + +from ._type_constants import ( + AUTOSKILLIT_SKILL_PREFIX, + FLEET_ERROR_CODES, + HEADLESS_ENV_VAR, + SESSION_TYPE_ENV_VAR, + SKILL_COMMAND_PREFIX, +) +from ._type_enums import SessionType, SkillSource +from ._type_protocols_workspace import SkillResolver + +__all__ = [ + "extract_path_arg", + "extract_skill_name", + "fleet_error", + "resolve_skill_name", + "resolve_target_skill", + "session_type", + "truncate_text", +] + +_SKILL_CMD_RE = re.compile( + r"^/(?:autoskillit:)?([\w-]+)" +) # anchored: strict leading-slash for extraction +_SKILL_RESOLVE_RE = re.compile( + r"/(?:autoskillit:)?([\w-]+)" +) # unanchored: supports "Use /..." prefix forms + +_PATH_PREFIXES: tuple[str, ...] = ("/", "./", ".autoskillit/") + + +def _looks_like_path(token: str) -> bool: + return any(token.startswith(p) for p in _PATH_PREFIXES) + + +def extract_path_arg(skill_command: str) -> str | None: + """Extract the first path-like positional argument from a skill_command string. + + Tolerates trailing text (markdown headers, extra tokens, embedded newlines) + after the path. Returns None if no path-like token is found. + Strips enclosing quotes from the returned path token. + """ + stripped = skill_command.strip() + m = _SKILL_CMD_RE.match(stripped) + if m is None: + return None + tokens = stripped[m.end() :].split() + for token in tokens: + cleaned = token.strip('"').strip("'") + if _looks_like_path(cleaned): + return cleaned + return None + + +def extract_skill_name(skill_command: str) -> str | None: + """Extract the bare skill name from a skill_command string. + + Handles both ``/autoskillit:make-plan ...`` and ``/make-plan ...`` forms. + Returns None if the command is not a slash-command. + """ + m = _SKILL_CMD_RE.match(skill_command.strip()) + return m.group(1) if m else None + + +def resolve_skill_name(skill_command: str) -> str | None: + """Extract and validate skill name from command string. + + Handles both ``/name`` and ``/autoskillit:name`` forms. Returns None if + no match, name contains template expressions, or is followed by a + bash-style ``{placeholder}`` token. + """ + stripped = skill_command.strip() + match = _SKILL_RESOLVE_RE.search(stripped) + if not match: + return None + name = match.group(1) + if "${{" in name: + return None + remainder = stripped[match.end() :] + if remainder.startswith("{") or remainder.startswith("${{"): + return None + return name + + +def resolve_target_skill( + skill_command: str, + resolver: SkillResolver, +) -> tuple[str, str | None]: + """Resolve a skill_command to the correct invocation namespace. + + Returns (resolved_command, skill_name). + skill_name is None if skill_command is not a slash command. + + - Skills in ``skills/`` (BUNDLED) → ``/autoskillit:name`` namespace + - Skills in ``skills_extended/`` (BUNDLED_EXTENDED) → ``/name`` namespace + """ + name = extract_skill_name(skill_command) + if name is None: + return skill_command, None + + info = resolver.resolve(name) + if info is None: + return skill_command, name + + # Determine correct prefix based on physical location + if info.source == SkillSource.BUNDLED: + correct_prefix = AUTOSKILLIT_SKILL_PREFIX + name + else: + correct_prefix = SKILL_COMMAND_PREFIX + name + + # Reconstruct: replace the skill reference, preserve trailing arguments + stripped = skill_command.strip() + m = _SKILL_CMD_RE.match(stripped) + if m is None: + raise RuntimeError(f"regex failed after extract_skill_name succeeded: {stripped!r}") + remainder = stripped[m.end() :] + return correct_prefix + remainder, name + + +def truncate_text(text: str, max_len: int = 5000) -> str: + """Truncate text to max_len, appending a count of truncated chars.""" + if len(text) <= max_len: + return text + return f"...[truncated {len(text) - max_len} chars]...\n" + text[-max_len:] + + +def fleet_error( + code: str, + message: str, + *, + details: dict[str, Any] | None = None, +) -> str: + """Return canonical JSON error envelope for fleet dispatch failures. + + Validates that code is a registered FleetErrorCode. Raises ValueError + for unregistered codes. The details dict must be JSON-serializable. + """ + if code not in FLEET_ERROR_CODES: + msg = f"Unregistered fleet error code: {code!r}" + raise ValueError(msg) + return json.dumps( + { + "success": False, + "error": str(code), + "user_visible_message": message, + "details": details, + } + ) + + +def session_type() -> SessionType: + """Resolve current session type from AUTOSKILLIT_SESSION_TYPE env var. + + Fail-closed: returns SKILL on unset or invalid values. + Transitional bridge: HEADLESS=1 without SESSION_TYPE emits DeprecationWarning. + """ + raw = os.environ.get(SESSION_TYPE_ENV_VAR, "") + if raw: + raw_lower = raw.lower() + if raw_lower == "leaf": + warnings.warn( + "AUTOSKILLIT_SESSION_TYPE='leaf' is deprecated. " + "Use 'skill' instead. Mapped to SessionType.SKILL.", + DeprecationWarning, + stacklevel=2, + ) + return SessionType.SKILL + try: + return SessionType(raw_lower) + except ValueError: + warnings.warn( + f"Invalid {SESSION_TYPE_ENV_VAR}={raw!r}, defaulting to SKILL. " + f"Valid values: {', '.join(m.value for m in SessionType)}", + DeprecationWarning, + stacklevel=2, + ) + return SessionType.SKILL + if os.environ.get(HEADLESS_ENV_VAR) == "1": + warnings.warn( + f"{HEADLESS_ENV_VAR}=1 without {SESSION_TYPE_ENV_VAR} set. " + "Defaulting to SKILL. Set AUTOSKILLIT_SESSION_TYPE explicitly.", + DeprecationWarning, + stacklevel=2, + ) + return SessionType.SKILL diff --git a/src/autoskillit/core/types/_type_plugin_source.py b/src/autoskillit/core/types/_type_plugin_source.py new file mode 100644 index 0000000000..fa44b2fe4d --- /dev/null +++ b/src/autoskillit/core/types/_type_plugin_source.py @@ -0,0 +1,31 @@ +"""PluginSource discriminated union — how autoskillit is loaded into Claude Code. + +Replaces the `plugin_dir: str | None` nullable sentinel in ToolContext. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from pathlib import Path + +__all__ = ["DirectInstall", "MarketplaceInstall", "PluginSource"] + + +@dataclass(frozen=True, slots=True) +class DirectInstall: + """Plugin loaded via --plugin-dir. plugin_dir is the package root (pkg_root()).""" + + plugin_dir: Path + + +@dataclass(frozen=True, slots=True) +class MarketplaceInstall: + """Plugin loaded via Claude marketplace. + + cache_path is the installPath field from installed_plugins.json. + """ + + cache_path: Path + + +PluginSource = DirectInstall | MarketplaceInstall diff --git a/src/autoskillit/core/types/_type_protocols_execution.py b/src/autoskillit/core/types/_type_protocols_execution.py new file mode 100644 index 0000000000..29907bf4df --- /dev/null +++ b/src/autoskillit/core/types/_type_protocols_execution.py @@ -0,0 +1,106 @@ +"""Execution-layer protocol definitions.""" + +from __future__ import annotations + +from collections.abc import Callable, Mapping, Sequence +from pathlib import Path +from typing import Protocol, runtime_checkable + +from ._type_checkpoint import SessionCheckpoint # noqa: F401, TC001 +from ._type_results import SkillResult, TestResult, ValidatedAddDir, WriteBehaviorSpec + +__all__ = [ + "TestRunner", + "HeadlessExecutor", + "OutputPatternResolver", + "WriteExpectedResolver", +] + + +@runtime_checkable +class TestRunner(Protocol): + """Protocol for running a test suite and reporting pass/fail. + + Returns a TestResult with passed, stdout, and stderr from the test run. + """ + + async def run(self, cwd: Path) -> TestResult: ... + + +@runtime_checkable +class HeadlessExecutor(Protocol): + """Protocol for running headless Claude Code sessions.""" + + async def run( + self, + skill_command: str, + cwd: str, + *, + model: str = "", + step_name: str = "", + kitchen_id: str = "", + order_id: str = "", + add_dirs: Sequence[ValidatedAddDir] = (), + timeout: float | None = None, + stale_threshold: float | None = None, + idle_output_timeout: float | None = None, + expected_output_patterns: Sequence[str] = (), + write_behavior: WriteBehaviorSpec | None = None, + completion_marker: str = "", + recipe_name: str = "", + recipe_content_hash: str = "", + recipe_composite_hash: str = "", + recipe_version: str = "", + allowed_write_prefix: str = "", + readonly_skill: bool = False, + write_watch_dirs: Sequence[Path] = (), + provider_extras: Mapping[str, str] | None = None, + profile_name: str = "", + provider_name: str = "", + provider_fallback_env: dict[str, str] | None = None, + provider_fallback_name: str = "", + resume_session_id: str = "", + resume_checkpoint: SessionCheckpoint | None = None, + ) -> SkillResult: ... + + async def dispatch_food_truck( + self, + orchestrator_prompt: str, + cwd: str, + *, + completion_marker: str, + resume_session_id: str | None = None, + resume_checkpoint: SessionCheckpoint | None = None, + model: str = "", + step_name: str = "", + kitchen_id: str = "", + order_id: str = "", + campaign_id: str = "", + dispatch_id: str = "", + caller_session_id: str = "", + project_dir: str = "", + timeout: float | None = None, + stale_threshold: float | None = None, + idle_output_timeout: float | None = None, + env_extras: Mapping[str, str] | None = None, + requires_packs: Sequence[str] = (), + on_spawn: Callable[[int, int], None] | None = None, + allowed_write_prefix: str = "", + provider_name: str = "", + provider_fallback_env: dict[str, str] | None = None, + provider_fallback_name: str = "", + ) -> SkillResult: ... + + +@runtime_checkable +class OutputPatternResolver(Protocol): + """Protocol for resolving expected output patterns from a skill command.""" + + def __call__(self, skill_command: str) -> Sequence[str]: ... + + +@runtime_checkable +class WriteExpectedResolver(Protocol): + """Protocol for resolving write-expectation metadata from skill contracts.""" + + def __call__(self, skill_command: str) -> WriteBehaviorSpec: ... diff --git a/src/autoskillit/core/types/_type_protocols_github.py b/src/autoskillit/core/types/_type_protocols_github.py new file mode 100644 index 0000000000..2ec3aab410 --- /dev/null +++ b/src/autoskillit/core/types/_type_protocols_github.py @@ -0,0 +1,166 @@ +"""GitHub integration protocol definitions.""" + +from __future__ import annotations + +from typing import Any, Protocol, runtime_checkable + +from ._type_results_execution import CIRunScope + +__all__ = ["GitHubFetcher", "CIWatcher", "MergeQueueWatcher"] + + +@runtime_checkable +class GitHubFetcher(Protocol): + """Protocol for fetching and filing GitHub issue content. + + Implementations must never raise — all errors must be captured and + returned in the result dict with success=False. + """ + + @property + def has_token(self) -> bool: + """True if this fetcher was constructed with an authentication token.""" + ... + + async def fetch_issue( + self, + issue_ref_or_owner: str, + repo: str | None = None, + number: int | None = None, + *, + include_comments: bool = True, + ) -> dict[str, Any]: ... + + async def search_issues( + self, + query: str, + owner: str, + repo: str, + *, + state: str = "open", + ) -> dict[str, Any]: ... + + async def create_issue( + self, + owner: str, + repo: str, + title: str, + body: str, + *, + labels: list[str] | None = None, + ) -> dict[str, Any]: ... + + async def update_issue_body( + self, + owner: str, + repo: str, + issue_number: int, + new_body: str, + ) -> dict[str, Any]: ... + + async def fetch_title(self, issue_url: str) -> dict[str, object]: ... + + async def add_labels( + self, + owner: str, + repo: str, + issue_number: int, + labels: list[str], + ) -> dict[str, Any]: ... + + async def remove_label( + self, + owner: str, + repo: str, + issue_number: int, + label: str, + ) -> dict[str, Any]: ... + + async def swap_labels( + self, + owner: str, + repo: str, + issue_number: int, + remove_labels: list[str], + add_labels: list[str], + ) -> dict[str, Any]: ... + + async def ensure_label( + self, + owner: str, + repo: str, + label: str, + color: str = "ededed", + description: str = "", + ) -> dict[str, Any]: ... + + +@runtime_checkable +class CIWatcher(Protocol): + """Protocol for watching GitHub Actions CI runs. + + Implementations must never raise — all errors must be captured and + returned in the result dict with appropriate conclusion values. + """ + + async def wait( + self, + branch: str, + *, + repo: str | None = None, + scope: CIRunScope = CIRunScope(), + timeout_seconds: int = 300, + lookback_seconds: int = 120, + cwd: str = "", + ) -> dict[str, Any]: ... + + async def status( + self, + branch: str, + *, + repo: str | None = None, + run_id: int | None = None, + scope: CIRunScope = CIRunScope(), + cwd: str = "", + ) -> dict[str, Any]: ... + + +@runtime_checkable +class MergeQueueWatcher(Protocol): + """Protocol for watching a PR's progress through GitHub's merge queue. + + Implementations must never raise — all errors must be captured and + returned in the result dict with appropriate pr_state values. + """ + + async def wait( + self, + pr_number: int, + target_branch: str, + repo: str | None = None, + cwd: str = ".", + timeout_seconds: int = 600, + poll_interval: int = 15, + stall_grace_period: int = 60, + max_stall_retries: int = 3, + not_in_queue_confirmation_cycles: int = 2, + max_inconclusive_retries: int = 5, + auto_merge_available: bool = True, + ) -> dict[str, Any]: ... + + async def toggle( + self, + pr_number: int, + target_branch: str, + repo: str | None = None, + cwd: str = ".", + ) -> dict[str, Any]: ... + + async def enqueue( + self, + pr_number: int, + target_branch: str, + repo: str | None = None, + cwd: str = ".", + auto_merge_available: bool = True, + ) -> dict[str, Any]: ... diff --git a/src/autoskillit/core/types/_type_protocols_infra.py b/src/autoskillit/core/types/_type_protocols_infra.py new file mode 100644 index 0000000000..e896fa70e0 --- /dev/null +++ b/src/autoskillit/core/types/_type_protocols_infra.py @@ -0,0 +1,103 @@ +"""Infrastructure and pipeline-control protocol definitions.""" + +from __future__ import annotations + +from pathlib import Path +from typing import Any, Protocol, runtime_checkable + +__all__ = [ + "GateState", + "BackgroundSupervisor", + "FleetLock", + "QuotaRefreshTask", + "TokenFactory", + "CampaignProtector", +] + + +@runtime_checkable +class GateState(Protocol): + """Protocol for gate enable/disable state.""" + + @property + def enabled(self) -> bool: ... + + def enable(self) -> None: ... + + def disable(self) -> None: ... + + +@runtime_checkable +class BackgroundSupervisor(Protocol): + """Protocol for supervised background task execution.""" + + @property + def pending_count(self) -> int: ... + + def submit( + self, + coro: Any, + *, + on_exception: Any | None = None, + status_path: Any | None = None, + label: str = "", + ) -> Any: ... + + async def drain(self) -> None: ... + + +@runtime_checkable +class FleetLock(Protocol): + """Protocol for a semaphore-style fleet dispatch guard. + + Default implementation is FleetSemaphore in server/_factory.py. + """ + + def at_capacity(self) -> bool: ... + + async def acquire(self) -> None: ... + + def release(self) -> None: ... + + @property + def active_count(self) -> int: ... + + @property + def max_concurrent(self) -> int: ... + + +@runtime_checkable +class QuotaRefreshTask(Protocol): + """Protocol for a cancellable background task handle. + + Satisfied by asyncio.Task — used to type the kitchen-scoped quota + refresh task stored in ToolContext without leaking asyncio.Task into the + core layer. + """ + + def cancel(self, msg: Any = None) -> bool: ... + + +@runtime_checkable +class TokenFactory(Protocol): + """Protocol for resolving a GitHub token via the config → env → CLI fallback chain. + + Satisfied by any zero-argument callable that returns a token string or None. + Set by make_context() on ToolContext; None in test ToolContext instances unless + explicitly provided. + """ + + def __call__(self) -> str | None: ... + + +@runtime_checkable +class CampaignProtector(Protocol): + """Protocol for resolving the set of protected campaign IDs for session retention. + + Satisfied by any callable that accepts a project root Path and returns a frozenset + of campaign ID strings that should not be purged during log retention. + Set by make_context() on ToolContext; None in test ToolContext instances unless + explicitly provided. + """ + + def __call__(self, project_dir: Path) -> frozenset[str]: ... diff --git a/src/autoskillit/core/types/_type_protocols_logging.py b/src/autoskillit/core/types/_type_protocols_logging.py new file mode 100644 index 0000000000..459da0e194 --- /dev/null +++ b/src/autoskillit/core/types/_type_protocols_logging.py @@ -0,0 +1,176 @@ +"""Logging and observer protocol definitions.""" + +from __future__ import annotations + +from pathlib import Path +from typing import Any, Protocol, runtime_checkable + +from ._type_results import FailureRecord, ModelTotalEntry + +__all__ = [ + "AuditLog", + "TokenLog", + "TimingLog", + "McpResponseLog", + "GitHubApiLog", + "SupportsDebug", + "SupportsLogger", +] + + +@runtime_checkable +class AuditLog(Protocol): + """Protocol for pipeline failure accumulation.""" + + def record_failure(self, record: FailureRecord) -> None: ... + + def get_report(self) -> list[FailureRecord]: ... + + def get_report_as_dicts(self) -> list[dict[str, Any]]: ... + + def clear(self) -> None: ... + + def consecutive_failures(self, skill_command: str) -> int: ... + + def record_success(self, skill_command: str) -> None: ... + + def load_from_log_dir( + self, + log_root: Path, + *, + since: str = "", + cwd_filter: str = "", + kitchen_id_filter: str = "", + campaign_id_filter: str = "", + order_id_filter: str = "", + dispatch_id_filter: str = "", + ) -> int: ... + + +@runtime_checkable +class TokenLog(Protocol): + """Protocol for per-step token usage accumulation.""" + + def record( + self, + step_name: str, + token_usage: dict[str, Any] | None, + *, + start_ts: str = "", + end_ts: str = "", + elapsed_seconds: float | None = None, + order_id: str = "", + loc_insertions: int = 0, + loc_deletions: int = 0, + ) -> None: ... + + def get_report(self, *, order_id: str = "") -> list[dict[str, Any]]: ... + + def compute_total(self, *, order_id: str = "") -> dict[str, Any]: ... + + def compute_model_totals(self, *, order_id: str = "") -> list[ModelTotalEntry]: ... + + def clear(self) -> None: ... + + def load_from_log_dir( + self, + log_root: Path, + *, + since: str = "", + cwd_filter: str = "", + kitchen_id_filter: str = "", + campaign_id_filter: str = "", + order_id_filter: str = "", + dispatch_id_filter: str = "", + ) -> int: ... + + +@runtime_checkable +class TimingLog(Protocol): + """Protocol for per-step wall-clock timing accumulation.""" + + def record(self, step_name: str, duration_seconds: float, *, order_id: str = "") -> None: ... + + def get_report(self, *, order_id: str = "") -> list[dict[str, Any]]: ... + + def compute_total(self, *, order_id: str = "") -> dict[str, Any]: ... + + def clear(self) -> None: ... + + def load_from_log_dir( + self, + log_root: Path, + *, + since: str = "", + cwd_filter: str = "", + kitchen_id_filter: str = "", + campaign_id_filter: str = "", + order_id_filter: str = "", + dispatch_id_filter: str = "", + ) -> int: ... + + +@runtime_checkable +class McpResponseLog(Protocol): + """Protocol for per-tool MCP response size accumulation.""" + + def record( + self, + tool_name: str, + response: str, + *, + alert_threshold_tokens: int = 0, + ) -> bool: ... + + def get_report(self) -> list[dict[str, Any]]: ... + + def compute_total(self) -> dict[str, Any]: ... + + def clear(self) -> None: ... + + +@runtime_checkable +class GitHubApiLog(Protocol): + """Protocol for session-scoped GitHub API request accumulation.""" + + async def record_httpx( + self, + *, + method: str, + path: str, + status_code: int, + latency_ms: float, + rate_limit_remaining: int, + rate_limit_used: int, + rate_limit_reset: int, + timestamp: str, + ) -> None: ... + + async def record_gh_cli( + self, + *, + subcommand: str, + exit_code: int, + latency_ms: float, + timestamp: str, + ) -> None: ... + + def to_usage(self, session_id: str) -> dict[str, Any] | None: ... + + def drain(self, session_id: str) -> dict[str, Any] | None: ... + + def clear(self) -> None: ... + + +# Not @runtime_checkable: structural (duck-typing) protocol; isinstance() checks not needed. +class SupportsDebug(Protocol): + """Structural logger protocol — only the debug() method is required.""" + + def debug(self, event: str, **kwargs: Any) -> None: ... + + +# Not @runtime_checkable: structural (duck-typing) protocol; isinstance() checks not needed. +class SupportsLogger(SupportsDebug, Protocol): + """Structural logger protocol — debug() and error() methods required.""" + + def error(self, event: str, **kwargs: Any) -> None: ... diff --git a/src/autoskillit/core/types/_type_protocols_recipe.py b/src/autoskillit/core/types/_type_protocols_recipe.py new file mode 100644 index 0000000000..aeea1560d2 --- /dev/null +++ b/src/autoskillit/core/types/_type_protocols_recipe.py @@ -0,0 +1,88 @@ +"""Recipe and data access protocol definitions.""" + +from __future__ import annotations + +from collections.abc import Awaitable, Callable, Sequence +from pathlib import Path +from typing import TYPE_CHECKING, Any, Protocol, runtime_checkable + +if TYPE_CHECKING: + from autoskillit.recipe.schema import Recipe, RecipeInfo + + from ._type_protocols_logging import SupportsDebug + +from ._type_results import LoadResult + +__all__ = ["RecipeRepository", "MigrationService", "DatabaseReader", "ReadOnlyResolver"] + + +@runtime_checkable +class RecipeRepository(Protocol): + """Protocol for recipe discovery and loading.""" + + def find(self, name: str, project_dir: Path) -> RecipeInfo | None: ... + + def load(self, path: Path) -> Recipe: ... + + def list(self, project_dir: Path) -> LoadResult[RecipeInfo]: ... + + def load_and_validate( + self, + name: str, + project_dir: Path | str, + *, + suppressed: Sequence[str] | None = None, + resolved_defaults: dict[str, str] | None = None, + ingredient_overrides: dict[str, str] | None = None, + temp_dir: Path | None = None, + temp_dir_relpath: str | None = None, + ) -> dict[str, Any]: ... + + def validate_from_path( + self, script_path: Any, temp_dir_relpath: str = ".autoskillit/temp" + ) -> dict[str, Any]: ... + + def list_all( + self, + project_dir: Any | None = None, + *, + features: dict[str, bool] | None = None, + ) -> dict[str, Any]: ... + + async def apply_triage_gate( + self, + result: dict[str, Any], + recipe_name: str, + recipe_info: Any, + temp_dir: Path, + logger: SupportsDebug, + triage_fn: Callable[..., Awaitable[Sequence[dict[str, Any]]]] | None = None, + ) -> dict[str, Any]: ... + + +@runtime_checkable +class MigrationService(Protocol): + """Protocol for applying migration notes to a recipe file.""" + + async def migrate(self, recipe_path: Path) -> dict[str, Any]: ... + + +@runtime_checkable +class DatabaseReader(Protocol): + """Protocol for read-only SQLite query execution.""" + + def query( + self, + db_path: str, + sql: str, + params: list | dict, # type: ignore[type-arg] # sqlite3 cursor.execute accepts any sequence or mapping + timeout_sec: int, + max_rows: int, + ) -> dict[str, Any]: ... + + +@runtime_checkable +class ReadOnlyResolver(Protocol): + """Protocol for resolving whether a skill is read-only from skill contracts.""" + + def __call__(self, skill_command: str) -> bool: ... diff --git a/src/autoskillit/core/types/_type_protocols_workspace.py b/src/autoskillit/core/types/_type_protocols_workspace.py new file mode 100644 index 0000000000..3824e30cbc --- /dev/null +++ b/src/autoskillit/core/types/_type_protocols_workspace.py @@ -0,0 +1,100 @@ +"""Workspace and skill management protocol definitions.""" + +from __future__ import annotations + +from pathlib import Path +from typing import Any, Protocol, runtime_checkable + +from ._type_results import CleanupResult, CloneResult, ValidatedAddDir + +__all__ = [ + "WorkspaceManager", + "CloneManager", + "SessionSkillManager", + "SkillLister", + "SkillResolver", +] + + +@runtime_checkable +class WorkspaceManager(Protocol): + """Protocol for directory teardown operations.""" + + def delete_contents( + self, + directory: Path, + preserve: set[str] | None = None, + ) -> CleanupResult: ... + + +@runtime_checkable +class CloneManager(Protocol): + """Protocol for clone-based pipeline run isolation.""" + + def clone_repo( + self, + source_dir: str, + run_name: str, + branch: str = "", + strategy: str = "", + remote_url: str = "", + ) -> CloneResult: ... + + def remove_clone(self, clone_path: str, keep: str = "false") -> dict[str, str]: ... + + def push_to_remote( + self, + clone_path: str, + source_dir: str = "", + branch: str = "", + *, + remote_url: str = "", + protected_branches: list[str] | None = None, + force: bool = False, + ) -> dict[str, str | bool]: ... + + +@runtime_checkable +class SessionSkillManager(Protocol): + """Protocol for managing per-session ephemeral skill directories.""" + + def init_session( + self, + session_id: str, + *, + cook_session: bool = False, + config: Any | None = None, + project_dir: Path | None = None, + recipe_packs: frozenset[str] | None = None, + recipe_features: frozenset[str] | None = None, + allow_only: frozenset[str] | None = None, + ) -> ValidatedAddDir: ... + + def compute_skill_closure(self, skill_name: str) -> frozenset[str]: ... + + def activate_skill_deps(self, session_id: str, skill_name: str) -> bool: ... + + def cleanup_session(self, session_id: str) -> bool: ... + + def cleanup_stale(self, max_age_seconds: int = 86400) -> int: ... + + +@runtime_checkable +class SkillResolver(Protocol): + """Protocol for resolving skill names to their source tier.""" + + def resolve(self, name: str) -> Any: ... + + +@runtime_checkable +class SkillLister(Protocol): + """IL-0 contract for listing all available skills. + + Allows IL-2 recipe rules to type their skill-listing dependency + against an IL-0 protocol instead of binding to the IL-1 workspace + concrete class. The default implementation lives at + autoskillit.workspace.skills.DefaultSkillResolver and satisfies this + protocol structurally. + """ + + def list_all(self) -> list[Any]: ... diff --git a/src/autoskillit/core/_type_results.py b/src/autoskillit/core/types/_type_results.py similarity index 61% rename from src/autoskillit/core/_type_results.py rename to src/autoskillit/core/types/_type_results.py index 465cad19d1..66831dcc8f 100644 --- a/src/autoskillit/core/_type_results.py +++ b/src/autoskillit/core/types/_type_results.py @@ -1,13 +1,15 @@ -"""Core result dataclasses. +"""Core result dataclasses — universal types. -Zero autoskillit imports outside this sub-package. Provides LoadResult, SkillResult, -CleanupResult, and related dataclasses. +Execution-scoped types (SessionTelemetry, RecipeIdentity, CIRunScope) live in +_type_results_execution.py for narrower test cascade. ProviderOutcome stays here +because SkillResult.provider references it, and SkillResult is consumed by 13+ +directories — a cross-import would undermine the cascade narrowing. """ from __future__ import annotations import json -from dataclasses import asdict, dataclass, field +from dataclasses import dataclass, field from enum import Enum from pathlib import Path from typing import Any, Generic, Literal, TypedDict, TypeVar @@ -23,13 +25,17 @@ "ValidatedAddDir", "WriteBehaviorSpec", "FailureRecord", + "ProviderOutcome", + "InfraOutcome", "SkillResult", "CleanupResult", - "CIRunScope", "CloneSuccessResult", "CloneGateUncommitted", "CloneGateUnpublished", "CloneResult", + "ModelTotalEntry", + "TokenUsageFileEntry", + "SessionIndexEntry", ] @@ -40,6 +46,11 @@ class TestResult: passed: bool stdout: str stderr: str + duration_seconds: float | None = None + tests_selected: int | None = None + tests_deselected: int | None = None + filter_mode: str | None = None + full_run_reason: str | None = None @dataclass @@ -58,7 +69,7 @@ class LoadResult(Generic[T]): errors: list[LoadReport] = field(default_factory=list) -@dataclass(frozen=True) +@dataclass(frozen=True, slots=True) class ValidatedAddDir: """An --add-dir path validated for Claude Code convention compliance. @@ -92,7 +103,7 @@ def glob(self, pattern: str) -> list[Path]: return list(Path(self.path).glob(pattern)) -@dataclass(frozen=True) +@dataclass(frozen=True, slots=True) class WriteBehaviorSpec: """Write-expectation metadata resolved from skill contracts. @@ -126,8 +137,40 @@ class FailureRecord: retry_reason: str # RetryReason.value string stderr: str # truncated to STDERR_MAX_LEN - def to_dict(self) -> dict: # type: ignore[type-arg] - return asdict(self) + def to_dict(self) -> dict[str, Any]: + return { + "timestamp": self.timestamp, + "skill_command": self.skill_command, + "exit_code": self.exit_code, + "subtype": self.subtype, + "needs_retry": self.needs_retry, + "retry_reason": self.retry_reason, + "stderr": self.stderr, + } + + +@dataclass(frozen=True, slots=True) +class ProviderOutcome: + """Typed bundle of provider execution outcome fields. + + All fields are required — constructing without any field is a TypeError, + making omissions visible at construction time rather than silently defaulting. + """ + + provider_used: str + fallback_activated: bool + + @classmethod + def none_used(cls) -> ProviderOutcome: + """Sentinel for paths where no provider selection occurred.""" + return cls(provider_used="", fallback_activated=False) + + +@dataclass(frozen=True, slots=True) +class InfraOutcome: + """Infrastructure exit classification bundle.""" + + exit_category: str = "" @dataclass @@ -148,6 +191,7 @@ class SkillResult: cli_subtype: str = field(default="") write_path_warnings: list[str] = field(default_factory=list) write_call_count: int = 0 + fs_writes_detected: bool = False order_id: str = "" kill_reason: KillReason = KillReason.NATURAL_EXIT """Why the subprocess was (or was not) killed after the race loop. @@ -155,6 +199,13 @@ class SkillResult: Surfaces from SubprocessResult so the formatter can annotate exit_code with the kill cause, resolving the "success=True + exit_code=-9" contradiction. """ + last_stop_reason: str = "" + lifespan_started: bool = False + """True when the headless session called an MCP tool (heuristic for server lifespan).""" + provider: ProviderOutcome = field(default_factory=ProviderOutcome.none_used) + """Provider execution outcome bundle.""" + infra: InfraOutcome = field(default_factory=InfraOutcome) + """Infrastructure exit classification bundle.""" def to_json(self) -> str: data: dict[str, Any] = { @@ -172,11 +223,17 @@ def to_json(self) -> str: "token_usage": self.token_usage, "write_path_warnings": self.write_path_warnings, "write_call_count": self.write_call_count, + "fs_writes_detected": self.fs_writes_detected, + "has_progress_evidence": self.has_progress_evidence, + "last_stop_reason": self.last_stop_reason, + "lifespan_started": self.lifespan_started, + "provider_fallback": self.provider.fallback_activated, + "provider_used": self.provider.provider_used, + "infra_exit_category": self.infra.exit_category, } if self.worktree_path is not None: data["worktree_path"] = self.worktree_path - if self.order_id: - data["order_id"] = self.order_id + data["order_id"] = self.order_id return json.dumps(data, default=lambda o: o.value if isinstance(o, Enum) else str(o)) @classmethod @@ -222,6 +279,18 @@ def outcome(self) -> SessionOutcome: return SessionOutcome.RETRIABLE return SessionOutcome.FAILED + @property + def has_progress_evidence(self) -> bool: + """Whether any evidence of meaningful progress exists. + + Synthesizes worktree_path, fs_writes_detected, and write_call_count + into a single routing-ready signal. Used by routing rules instead of + checking individual artifact fields. + """ + return ( + self.worktree_path is not None or self.fs_writes_detected or self.write_call_count > 0 + ) + @dataclass class CleanupResult: @@ -242,20 +311,6 @@ def to_dict(self) -> dict: # type: ignore[type-arg] } -@dataclass(frozen=True) -class CIRunScope: - """Immutable scope parameters that uniquely identify which CI workflow runs are relevant. - - Passed as a single argument through the CIWatcher protocol so that adding a new - scope axis requires changing only this dataclass and the API params builder — - not every method signature in the call chain. - """ - - workflow: str | None = None # workflow filename, e.g. "tests.yml" - head_sha: str | None = None # commit SHA to pin results to - event: str | None = None # trigger event, e.g. "push", "pull_request" - - class CloneSuccessResult(TypedDict): """Typed return contract for a successful clone_repo invocation. @@ -289,3 +344,76 @@ class CloneGateUnpublished(TypedDict): CloneResult = CloneSuccessResult | CloneGateUncommitted | CloneGateUnpublished + + +class ModelTotalEntry(TypedDict): + """Per-model aggregate token counts produced by compute_model_totals().""" + + model: str + step_count: int + input_tokens: int + output_tokens: int + cache_creation_input_tokens: int + cache_read_input_tokens: int + elapsed_seconds: float + + +class TokenUsageFileEntry(TypedDict): + """Schema contract for token_usage.json written by flush_session_log.""" + + session_label: str + input_tokens: int + output_tokens: int + cache_creation_input_tokens: int + cache_read_input_tokens: int + peak_context: int + turn_count: int + timing_seconds: float + order_id: str + loc_insertions: int + loc_deletions: int + provider_used: str + model_identifier: str + dispatch_id: str + campaign_id: str + + +class SessionIndexEntry(TypedDict): + """Schema contract for sessions.jsonl entries written by flush_session_log.""" + + session_id: str + dir_name: str + timestamp: str + cwd: str + kitchen_id: str + order_id: str + campaign_id: str + dispatch_id: str + claude_code_log: str + skill_command: str + success: bool + subtype: str + cli_subtype: str + exit_code: int + snapshot_count: int + anomaly_count: int + peak_rss_kb: int + peak_oom_score: int + step_name: str + input_tokens: int + output_tokens: int + cache_creation_input_tokens: int + cache_read_input_tokens: int + write_call_count: int + tracked_comm: str | None + tracked_comm_drift: bool + autoskillit_version: str + claude_code_version: str + recipe_name: str + recipe_content_hash: str + recipe_composite_hash: str + recipe_version: str + duration_seconds: float + github_api_requests: int + provider_used: str + provider_fallback: bool diff --git a/src/autoskillit/core/types/_type_results_execution.py b/src/autoskillit/core/types/_type_results_execution.py new file mode 100644 index 0000000000..28ac6cade0 --- /dev/null +++ b/src/autoskillit/core/types/_type_results_execution.py @@ -0,0 +1,93 @@ +"""Execution-scoped result dataclasses. + +Narrow-cascade peer of _type_results.py. These types are consumed primarily by +execution/, server/, and pipeline/ — not by workspace/, recipe/, migration/, or +the root-level utility modules. Splitting them here means changes cascade to +4 test directories instead of 13. ProviderOutcome lives in _type_results.py +because SkillResult.provider references it (universal consumer surface). + +Zero autoskillit imports outside this sub-package (IL-0). +""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import Any + +from ._type_constants import KNOWN_CI_EVENTS + +__all__ = [ + "SessionTelemetry", + "RecipeIdentity", + "CIRunScope", +] + + +@dataclass(frozen=True, slots=True) +class SessionTelemetry: + """Typed bundle of all per-session telemetry fields passed to flush_session_log. + + All fields are required — constructing without any field is a TypeError, + making omissions visible at construction time rather than silently defaulting + to None and skipping the corresponding write gate inside flush_session_log. + """ + + token_usage: dict[str, Any] | None + timing_seconds: float | None + audit_record: dict[str, Any] | None + github_api_usage: dict[str, Any] | None + github_api_requests: int + loc_insertions: int + loc_deletions: int + + @classmethod + def empty(cls) -> SessionTelemetry: + """Zero-value sentinel for error paths where no telemetry is available.""" + return cls( + token_usage=None, + timing_seconds=None, + audit_record=None, + github_api_usage=None, + github_api_requests=0, + loc_insertions=0, + loc_deletions=0, + ) + + +@dataclass(frozen=True, slots=True) +class RecipeIdentity: + """Typed bundle of recipe identification fields for session logging. + + All fields required — prevents silent empty-string drift when new recipe + fields are added to flush_session_log but not wired from callers. + """ + + name: str + content_hash: str + composite_hash: str + version: str + + @classmethod + def empty(cls) -> RecipeIdentity: + """Sentinel for sessions not driven by a recipe.""" + return cls(name="", content_hash="", composite_hash="", version="") + + +@dataclass(frozen=True, slots=True) +class CIRunScope: + """Immutable scope parameters that uniquely identify which CI workflow runs are relevant. + + Passed as a single argument through the CIWatcher protocol so that adding a new + scope axis requires changing only this dataclass and the API params builder — + not every method signature in the call chain. + """ + + workflow: str | None = None + head_sha: str | None = None + event: str | None = None + + def __post_init__(self) -> None: + if self.event is not None and self.event not in KNOWN_CI_EVENTS: + raise ValueError( + f"Invalid CI event {self.event!r}. Valid events: {sorted(KNOWN_CI_EVENTS)}" + ) diff --git a/src/autoskillit/core/types/_type_resume.py b/src/autoskillit/core/types/_type_resume.py new file mode 100644 index 0000000000..d0ee7ec968 --- /dev/null +++ b/src/autoskillit/core/types/_type_resume.py @@ -0,0 +1,48 @@ +"""Resume intent discriminated union for interactive session launch. + +Three sealed variants make the three user intents structurally distinct: +- NoResume: start a fresh session (no --resume flag) +- BareResume: pass --resume without an ID (delegate to Claude Code's picker) +- NamedResume: pass --resume (resume a specific session) +""" + +from __future__ import annotations + +from dataclasses import dataclass + +__all__ = [ + "NoResume", + "BareResume", + "NamedResume", + "ResumeSpec", + "resume_spec_from_cli", +] + + +@dataclass(frozen=True, slots=True) +class NoResume: + """No resume: start a fresh session.""" + + +@dataclass(frozen=True, slots=True) +class BareResume: + """Bare --resume: delegate session selection to Claude Code's picker.""" + + +@dataclass(frozen=True, slots=True) +class NamedResume: + """--resume : resume a specific named session.""" + + session_id: str + + +ResumeSpec = NoResume | BareResume | NamedResume + + +def resume_spec_from_cli(*, resume: bool, session_id: str | None) -> ResumeSpec: + """Construct a ResumeSpec from CLI input without any I/O.""" + if session_id: + return NamedResume(session_id=session_id) + if resume: + return BareResume() + return NoResume() diff --git a/src/autoskillit/core/_type_subprocess.py b/src/autoskillit/core/types/_type_subprocess.py similarity index 93% rename from src/autoskillit/core/_type_subprocess.py rename to src/autoskillit/core/types/_type_subprocess.py index f59d083a2c..5122e60fc4 100644 --- a/src/autoskillit/core/_type_subprocess.py +++ b/src/autoskillit/core/types/_type_subprocess.py @@ -6,7 +6,7 @@ from __future__ import annotations -from collections.abc import Awaitable, Mapping +from collections.abc import Awaitable, Callable, Mapping from dataclasses import dataclass from pathlib import Path from typing import Any, Protocol, runtime_checkable @@ -16,7 +16,6 @@ __all__ = [ "SubprocessResult", "SubprocessRunner", - "_TERMINATION_CONTRACT", ] #: Semantic contract for SubprocessResult fields per TerminationReason. @@ -117,6 +116,11 @@ class SubprocessResult: that downstream telemetry (sessions.jsonl, summary.json, GitHub issue bodies) can self-identify which process was actually observed. (Issue #806) """ + orphaned_tool_result: bool = False + """True when Channel B fired STALE and the last JSONL record was type=user. + Diagnostic only — does not affect termination behavior. + Always False for non-STALE outcomes. + """ @runtime_checkable @@ -139,4 +143,7 @@ def __call__( linux_tracing_config: Any | None = None, idle_output_timeout: float | None = None, max_suppression_seconds: float | None = None, + on_pid_resolved: Callable[[int, int], None] | None = None, + enable_deadline_extension: bool = False, + max_extension_seconds: float = 7200, ) -> Awaitable[SubprocessResult]: ... diff --git a/src/autoskillit/execution/CLAUDE.md b/src/autoskillit/execution/CLAUDE.md new file mode 100644 index 0000000000..384daaac59 --- /dev/null +++ b/src/autoskillit/execution/CLAUDE.md @@ -0,0 +1,32 @@ +# execution/ + +IL-1 execution layer — headless Claude sessions, process lifecycle, CI/GitHub integration. +Sub-packages: headless/ (see headless/CLAUDE.md), process/ (see process/CLAUDE.md), +merge_queue/ (see merge_queue/CLAUDE.md), session/ (see session/CLAUDE.md). + +## Files + +| File | Purpose | +|------|---------| +| `__init__.py` | Re-exports `DefaultHeadlessExecutor`, `run_headless_core` | +| `commands.py` | `ClaudeInteractiveCmd`, `ClaudeHeadlessCmd` builders | +| `db.py` | Read-only SQLite with defence-in-depth | +| `diff_annotator.py` | Diff annotation + findings filter for review-pr | +| `linux_tracing.py` | `/proc` + psutil process tracing (Linux) | +| `anomaly_detection.py` | Post-hoc anomaly detection over snapshots | +| `session_log.py` | XDG-aware session diagnostics log writer | +| `recording.py` | Record/replay subprocess runners via api-simulator | +| `_recording_skills.py` | Skill dir snapshot/restore for record/replay sessions | +| `quota.py` | `QuotaStatus`, cache, `check_and_sleep_if_needed` | +| `ci.py` | GitHub Actions CI watcher (httpx, never raises) | +| `github.py` | GitHub issue fetcher | +| `remote_resolver.py` | Upstream > origin, clone-aware remote resolution | +| `testing.py` | Pytest output parsing + pass/fail adjudication | +| `clone_guard.py` | Clone contamination guard — detect and revert direct changes to clone CWD | +| `pr_analysis.py` | `extract_linked_issues`, `DOMAIN_PATHS`, `partition_files_by_domain` | + +## Architecture Notes + +`session_log.py` uses XDG base dir spec; log directory names use hyphens (never +underscores). `recording.py` and `_recording_skills.py` only activate when +`AUTOSKILLIT_RECORD_SESSION` is set; production paths never touch them. diff --git a/src/autoskillit/execution/__init__.py b/src/autoskillit/execution/__init__.py index 86a60a4014..29d1ce5596 100644 --- a/src/autoskillit/execution/__init__.py +++ b/src/autoskillit/execution/__init__.py @@ -1,4 +1,4 @@ -"""execution/ L1 package: subprocess lifecycle, session parsing, headless runner, testing, DB. +"""execution/ IL-1 package: subprocess lifecycle, session parsing, headless runner, testing, DB. Re-exports the full public surface of the six execution sub-modules. All sub-modules depend only on autoskillit.core.* at runtime; @@ -6,6 +6,11 @@ """ from autoskillit.core import SkillResult +from autoskillit.execution._recording_skills import ( + restore_skill_snapshot, + scan_skill_snapshots, + snapshot_skill_dir, +) from autoskillit.execution.anomaly_detection import ( AnomalyKind, AnomalySeverity, @@ -27,10 +32,14 @@ _execute_readonly_query as execute_readonly_query, ) from autoskillit.execution.diff_annotator import ( + DiffMetrics, FilterResult, annotate_diff, + compute_diff_metrics, + extract_code_region, filter_findings, parse_hunk_ranges, + select_review_agents, ) from autoskillit.execution.github import ( DefaultGitHubFetcher, @@ -58,14 +67,17 @@ ) from autoskillit.execution.process import ( DefaultSubprocessRunner, + async_kill_process_tree, + kill_process_tree, run_managed_async, run_managed_sync, ) from autoskillit.execution.quota import ( QUOTA_CACHE_SCHEMA_VERSION, QuotaStatus, - _refresh_quota_cache, # noqa: F401 — imported for re-export via server.helpers; not in __all__ + _refresh_quota_cache, # noqa: F401 — re-exported for server consumers; not in __all__ check_and_sleep_if_needed, + invalidate_cache, ) from autoskillit.execution.recording import ( RECORD_SCENARIO_DIR_ENV, @@ -79,12 +91,21 @@ ScenarioReplayError, build_replay_runner, ) -from autoskillit.execution.remote_resolver import REMOTE_PRECEDENCE, resolve_remote_repo +from autoskillit.execution.remote_resolver import ( + REMOTE_PRECEDENCE, + resolve_remote_name, + resolve_remote_repo, +) from autoskillit.execution.session import ( ClaudeSessionResult, ContentState, + SessionState, + classify_infra_exit, + clear_session_state, extract_token_usage, parse_session_result, + persist_session_state, + read_session_state, ) from autoskillit.execution.session_log import ( flush_session_log, @@ -100,6 +121,9 @@ ) __all__ = [ + # _process_kill + "kill_process_tree", + "async_kill_process_tree", # commands "ClaudeInteractiveCmd", "ClaudeHeadlessCmd", @@ -121,16 +145,25 @@ "REPLAY_SCENARIO_ENV", "REPLAY_SCENARIO_DIR_ENV", "SCENARIO_STEP_NAME_ENV", + "restore_skill_snapshot", + "scan_skill_snapshots", + "snapshot_skill_dir", # quota "QUOTA_CACHE_SCHEMA_VERSION", "QuotaStatus", "check_and_sleep_if_needed", + "invalidate_cache", # session "ClaudeSessionResult", "ContentState", + "SessionState", "SkillResult", + "classify_infra_exit", + "clear_session_state", "extract_token_usage", "parse_session_result", + "persist_session_state", + "read_session_state", # headless "run_headless_core", "DefaultHeadlessExecutor", @@ -145,12 +178,17 @@ "fetch_repo_merge_state", # remote_resolver "REMOTE_PRECEDENCE", + "resolve_remote_name", "resolve_remote_repo", # diff_annotator + "DiffMetrics", "FilterResult", "annotate_diff", + "compute_diff_metrics", + "extract_code_region", "filter_findings", "parse_hunk_ranges", + "select_review_agents", # db "execute_readonly_query", "DefaultDatabaseReader", diff --git a/src/autoskillit/execution/_recording_skills.py b/src/autoskillit/execution/_recording_skills.py new file mode 100644 index 0000000000..483e92bd2f --- /dev/null +++ b/src/autoskillit/execution/_recording_skills.py @@ -0,0 +1,136 @@ +"""Snapshot and restore ephemeral skill directories for record/replay sessions.""" + +from __future__ import annotations + +import hashlib +import shutil +from datetime import UTC, datetime +from pathlib import Path +from typing import Any + +import regex as re + +from autoskillit.core import ValidatedAddDir, write_versioned_json + +SKILLS_SNAPSHOT_DIR = "skill-snapshots" +_EPHEMERAL_SESSION_PATTERN = "autoskillit-sessions" +_GATED_PATTERN = re.compile(r"disable-model-invocation\s*:\s*true", re.IGNORECASE) + + +def _extract_ephemeral_add_dir(cmd: list[str]) -> Path | None: + """Extract the ephemeral skill dir path from --add-dir CLI args. + + Identifies ephemeral dirs by the 'autoskillit-sessions' path component. + Returns the first matching --add-dir path, or None. + """ + for i, token in enumerate(cmd): + if token == "--add-dir" and i + 1 < len(cmd): + candidate = cmd[i + 1] + if _EPHEMERAL_SESSION_PATTERN in candidate: + return Path(candidate) + return None + + +def build_skills_manifest(skills_dir: Path) -> dict[str, Any]: + """Build a manifest dict from a .claude/skills/ directory.""" + if not skills_dir.is_dir(): + return { + "schema_version": 1, + "captured_at": datetime.now(tz=UTC).isoformat(), + "skill_count": 0, + "skills": {}, + } + skills: dict[str, Any] = {} + for skill_dir in sorted(skills_dir.iterdir()): + if not skill_dir.is_dir(): + continue + skill_md = skill_dir / "SKILL.md" + if not skill_md.exists(): + continue + content = skill_md.read_bytes() + sha256 = hashlib.sha256(content).hexdigest() + gated = bool(_GATED_PATTERN.search(content.decode("utf-8", errors="replace"))) + skills[skill_dir.name] = { + "content_sha256": sha256, + "size_bytes": len(content), + "gated": gated, + } + return { + "schema_version": 1, + "captured_at": datetime.now(tz=UTC).isoformat(), + "skill_count": len(skills), + "skills": skills, + } + + +def snapshot_skill_dir(scenario_dir: Path, step_name: str, add_dir_path: Path) -> Path | None: + """Copy the ephemeral skill dir tree into the scenario dir. + + Copies {add_dir_path}/.claude/skills/ → + {scenario_dir}/skill-snapshots/{step_name}/.claude/skills/ + Writes manifest.json alongside the .claude/ dir. + Returns the snapshot dir path, or None if no skills to snapshot. + """ + if not step_name or "/" in step_name or ".." in step_name: + raise ValueError(f"Invalid step_name for path construction: {step_name!r}") + skills_src = add_dir_path / ".claude" / "skills" + if not skills_src.exists() or not skills_src.is_dir(): + return None + + skill_subdirs = [d for d in skills_src.iterdir() if d.is_dir()] + if not skill_subdirs: + return None + + snapshot_dir = scenario_dir / SKILLS_SNAPSHOT_DIR / step_name + snapshot_dir.mkdir(parents=True, exist_ok=True) + + dest_skills = snapshot_dir / ".claude" / "skills" + if dest_skills.exists(): + shutil.rmtree(dest_skills) + try: + shutil.copytree(skills_src, dest_skills) + except Exception: + shutil.rmtree(snapshot_dir, ignore_errors=True) + raise + + manifest = build_skills_manifest(skills_src) + write_versioned_json(snapshot_dir / "manifest.json", manifest, 1) + + return snapshot_dir + + +def restore_skill_snapshot( + snapshot_path: Path, ephemeral_root: Path, session_id: str +) -> ValidatedAddDir | None: + """Restore skill dir from a snapshot into a new ephemeral session dir. + + Copies {snapshot_path}/.claude/skills/ → + {ephemeral_root}/{session_id}/.claude/skills/ + Returns ValidatedAddDir pointing to {ephemeral_root}/{session_id}. + """ + skills_src = snapshot_path / ".claude" / "skills" + if not skills_src.exists(): + return None + + session_dir = ephemeral_root / session_id + dest_skills = session_dir / ".claude" / "skills" + shutil.rmtree(dest_skills, ignore_errors=True) + dest_skills.parent.mkdir(parents=True, exist_ok=True) + + try: + shutil.copytree(skills_src, dest_skills) + except Exception: + shutil.rmtree(dest_skills, ignore_errors=True) + raise + return ValidatedAddDir(path=str(session_dir)) + + +def scan_skill_snapshots(scenario_dir: Path) -> dict[str, Path]: + """Scan {scenario_dir}/skill-snapshots/ for per-step snapshot dirs. + + Returns {step_name: snapshot_path} for each subdirectory. + """ + snapshots_root = scenario_dir / SKILLS_SNAPSHOT_DIR + if not snapshots_root.exists() or not snapshots_root.is_dir(): + return {} + return {entry.name: entry for entry in sorted(snapshots_root.iterdir()) if entry.is_dir()} diff --git a/src/autoskillit/execution/anomaly_detection.py b/src/autoskillit/execution/anomaly_detection.py index f8cd957e63..b07a4d2eb6 100644 --- a/src/autoskillit/execution/anomaly_detection.py +++ b/src/autoskillit/execution/anomaly_detection.py @@ -21,6 +21,8 @@ class AnomalyKind(StrEnum): D_STATE_SUSTAINED = "d_state_sustained" HIGH_CPU_SUSTAINED = "high_cpu_sustained" IDENTITY_DRIFT = "identity_drift" + EMPTY_RESULT_WITH_TOKENS = "empty_result_with_tokens" + THINKING_ONLY_FINAL_TURN = "thinking_only_final_turn" class AnomalySeverity(StrEnum): @@ -29,6 +31,13 @@ class AnomalySeverity(StrEnum): CRITICAL = "critical" +# Sentinel values for outcome anomalies (e.g. EMPTY_RESULT_WITH_TOKENS) that are not +# derived from a live ProcSnapshot. Downstream consumers must treat these as "no +# snapshot" indicators rather than real pid/seq values. +OUTCOME_ANOMALY_PID_SENTINEL: int = 0 +OUTCOME_ANOMALY_SEQ_SENTINEL: int = -1 + + # Kernel wait-channel values that are normal for a healthy process in # uninterruptible sleep. D-state snapshots whose wchan matches any of these # are NOT counted toward the D_STATE_SUSTAINED threshold. @@ -305,3 +314,48 @@ def detect_identity_drift( break return anomalies + + +def detect_outcome_anomalies( + token_usage: dict[str, object], + subtype: str, + has_thinking_only_turn: bool = False, +) -> list[dict[str, object]]: + """Detect outcome-level anomalies that require correlating session result with token usage. + + Detects: + - THINKING_ONLY_FINAL_TURN: final turn contained only thinking blocks (no text/tool output) + - EMPTY_RESULT_WITH_TOKENS: session produced output_tokens > 0 but subtype is 'empty_result' + """ + anomalies: list[dict[str, object]] = [] + output_tokens = token_usage.get("output_tokens", 0) + if has_thinking_only_turn and subtype == "empty_result": + anomalies.append( + { + "ts": datetime.now(UTC).isoformat(), + "seq": OUTCOME_ANOMALY_SEQ_SENTINEL, + "event": "anomaly", + "kind": str(AnomalyKind.THINKING_ONLY_FINAL_TURN), + "severity": str(AnomalySeverity.WARNING), + "pid": OUTCOME_ANOMALY_PID_SENTINEL, + "detail": { + "output_tokens": output_tokens if isinstance(output_tokens, int) else 0, + "subtype": subtype, + }, + "snapshot": {}, + } + ) + elif isinstance(output_tokens, int) and output_tokens > 0 and subtype == "empty_result": + anomalies.append( + { + "ts": datetime.now(UTC).isoformat(), + "seq": OUTCOME_ANOMALY_SEQ_SENTINEL, + "event": "anomaly", + "kind": str(AnomalyKind.EMPTY_RESULT_WITH_TOKENS), + "severity": str(AnomalySeverity.WARNING), + "pid": OUTCOME_ANOMALY_PID_SENTINEL, + "detail": {"output_tokens": output_tokens, "subtype": subtype}, + "snapshot": {}, + } + ) + return anomalies diff --git a/src/autoskillit/execution/ci.py b/src/autoskillit/execution/ci.py index eb82edef63..9749315709 100644 --- a/src/autoskillit/execution/ci.py +++ b/src/autoskillit/execution/ci.py @@ -1,6 +1,6 @@ """GitHub Actions CI watcher service. -L1 module: depends only on stdlib, httpx, and core/logging. +IL-1 module: depends only on stdlib, httpx, and core/logging. Never raises — all errors are captured and returned as structured dicts. Three-phase algorithm eliminates the race condition where CI completes @@ -15,20 +15,39 @@ import asyncio import random import time +import urllib.parse from collections.abc import Callable from datetime import UTC, datetime, timedelta -from typing import Any +from typing import TYPE_CHECKING, Any import httpx from autoskillit.core import CIRunScope, get_logger -from autoskillit.execution.github import github_headers +from autoskillit.execution.github import github_headers, make_tracked_httpx_client -_log = get_logger(__name__) +if TYPE_CHECKING: + from autoskillit.core import GitHubApiLog -# Backoff schedule constants -_BACKOFF_BASE = 5 # seconds -_BACKOFF_CAP = 30 # seconds +logger = get_logger(__name__) + +# Backoff schedule: (floor, ceiling) per attempt band. +_BACKOFF_BANDS: tuple[tuple[int, int], ...] = ((5, 10), (10, 20), (15, 30)) + +# All GitHub Actions check run conclusion values known to be returned by the REST API. +# https://docs.github.com/en/rest/checks/runs +KNOWN_CI_CONCLUSIONS: frozenset[str] = frozenset( + { + "success", + "failure", + "neutral", + "cancelled", + "skipped", + "timed_out", + "action_required", + "startup_failure", + "stale", + } +) # GitHub run-level conclusions that indicate a job-level failure worth inspecting. # "action_required" is intentionally excluded — it signals a billing/permissions @@ -44,9 +63,9 @@ def _jittered_sleep(attempt: int) -> float: - """Compute full-jitter exponential backoff: random(0, min(cap, base * 2^attempt)).""" - ceiling = min(_BACKOFF_CAP, _BACKOFF_BASE * (2**attempt)) - return random.uniform(0, ceiling) + """Full-jitter exponential backoff with per-attempt floor bands.""" + floor, ceiling = _BACKOFF_BANDS[min(attempt, len(_BACKOFF_BANDS) - 1)] + return random.uniform(floor, ceiling) def _validate_run_matches_scope(run: dict[str, Any], scope: CIRunScope) -> bool: @@ -72,7 +91,12 @@ class DefaultCIWatcher: _UNRESOLVED = object() - def __init__(self, *, token: str | None | Callable[[], str | None] = None) -> None: + def __init__( + self, + *, + token: str | None | Callable[[], str | None] = None, + tracker: GitHubApiLog | None = None, + ) -> None: self._token_factory: Callable[[], str | None] | None if callable(token): self._token_factory = token @@ -80,6 +104,8 @@ def __init__(self, *, token: str | None | Callable[[], str | None] = None) -> No else: self._token_factory = None self._token = token + self._tracker = tracker + self._etag_cache: dict[str, tuple[str, Any]] = {} # url -> (etag, cached_json) def _resolve_token(self) -> str | None: if self._token is self._UNRESOLVED: @@ -89,6 +115,45 @@ def _resolve_token(self) -> str | None: def _headers(self) -> dict[str, str]: return github_headers(self._resolve_token()) + async def _get_with_etag( + self, + client: httpx.AsyncClient, + headers: dict[str, str], + url: str, + params: dict[str, str | int] | None = None, + ) -> Any: + """GET with ETag conditional request support. + + Returns parsed JSON. On 304 (Not Modified), returns cached body. + On 200, stores ETag + body for future conditional requests. + """ + cache_key = url + ( + "?" + urllib.parse.urlencode(sorted((str(k), str(v)) for k, v in params.items())) + if params + else "" + ) + req_headers = dict(headers) + cached = self._etag_cache.get(cache_key) + if cached: + req_headers["If-None-Match"] = cached[0] + + resp = await client.get(url, headers=req_headers, params=params) + if resp.status_code == 304: + if cached: + return cached[1] + raise RuntimeError( + f"Server returned 304 Not Modified for {url!r} " + "but no cached response exists for this URL. " + "This indicates a server-side inconsistency." + ) + resp.raise_for_status() + data = resp.json() + + etag = resp.headers.get("ETag") + if etag: + self._etag_cache[cache_key] = (etag, data) + return data + async def _resolve_repo(self, repo: str | None, cwd: str) -> str | None: """Resolve owner/repo from argument or git remote.""" if not cwd and not repo: @@ -104,7 +169,7 @@ async def _fetch_completed_runs( owner_repo: str, branch: str, scope: CIRunScope, - lookback_seconds: int, + cutoff_dt: datetime | None, ) -> list[dict[str, Any]]: """Phase 1: Look-back — fetch recently completed runs for the branch.""" url = f"https://api.github.com/repos/{owner_repo}/actions/runs" @@ -115,23 +180,18 @@ async def _fetch_completed_runs( } if scope.workflow: params["workflow_id"] = scope.workflow - if scope.head_sha: - params["head_sha"] = scope.head_sha if scope.event: params["event"] = scope.event - resp = await client.get(url, headers=headers, params=params) - resp.raise_for_status() - data = resp.json() + data = await self._get_with_etag(client, headers, url, params) - cutoff = datetime.now(UTC) - timedelta(seconds=lookback_seconds) runs = [] for run in data.get("workflow_runs", []): updated = run.get("updated_at", "") if updated: try: run_time = datetime.fromisoformat(updated.replace("Z", "+00:00")) - if run_time >= cutoff: + if cutoff_dt is None or run_time >= cutoff_dt: runs.append(run) except ValueError: continue @@ -153,14 +213,10 @@ async def _fetch_active_runs( } if scope.workflow: params["workflow_id"] = scope.workflow - if scope.head_sha: - params["head_sha"] = scope.head_sha if scope.event: params["event"] = scope.event - resp = await client.get(url, headers=headers, params=params) - resp.raise_for_status() - data = resp.json() + data = await self._get_with_etag(client, headers, url, params) return [r for r in data.get("workflow_runs", []) if r.get("status") != "completed"] @@ -230,12 +286,18 @@ async def wait( } headers = self._headers() - deadline = time.monotonic() + timeout_seconds + _start_time = time.monotonic() + deadline = _start_time + timeout_seconds + cutoff_dt = datetime.now(UTC) - timedelta(seconds=lookback_seconds) try: - async with httpx.AsyncClient(timeout=httpx.Timeout(15.0, connect=5.0)) as client: + async with make_tracked_httpx_client( + self._tracker, + timeout=httpx.Timeout(15.0, connect=5.0), + headers=github_headers(self._resolve_token()), + ) as client: # Phase 1: Look-back — check for recently-completed runs - _log.info( + logger.info( "ci_watcher_lookback", branch=branch, repo=owner_repo, @@ -248,17 +310,22 @@ async def wait( owner_repo, branch, scope, - lookback_seconds, + cutoff_dt, ) if completed: valid_completed = [ r for r in completed if _validate_run_matches_scope(r, scope) ] if not valid_completed: - _log.warning( + found_shas = list( + {r.get("head_sha") for r in completed if r.get("head_sha")} + ) + logger.warning( "ci_watcher_scope_mismatch", count=len(completed), scope=str(scope), + expected_sha=scope.head_sha, + found_shas=found_shas, ) else: run = valid_completed[0] @@ -274,7 +341,9 @@ async def wait( if conclusion in FAILED_CONCLUSIONS else [] ) - _log.info("ci_watcher_lookback_hit", run_id=run_id, conclusion=conclusion) + logger.info( + "ci_watcher_lookback_hit", run_id=run_id, conclusion=conclusion + ) return { "run_id": run_id, "conclusion": conclusion, @@ -282,7 +351,7 @@ async def wait( } # Phase 2: Poll for active runs - _log.info("ci_watcher_polling", branch=branch, repo=owner_repo) + logger.info("ci_watcher_polling", branch=branch, repo=owner_repo) attempt = 0 found_run: dict[str, Any] | None = None while time.monotonic() < deadline: @@ -305,7 +374,7 @@ async def wait( owner_repo, branch, scope, - lookback_seconds, + cutoff_dt, ) if completed: valid_completed = [ @@ -339,12 +408,32 @@ async def wait( attempt += 1 if found_run is None: - _log.warning("ci_watcher_no_runs", branch=branch, repo=owner_repo) - return {"run_id": None, "conclusion": "no_runs", "failed_jobs": []} + poll_duration = time.monotonic() - _start_time + logger.warning( + "ci_watcher_no_runs", + branch=branch, + repo=owner_repo, + scope_event=scope.event, + scope_workflow=scope.workflow, + scope_head_sha=scope.head_sha, + poll_duration_s=round(poll_duration, 1), + ) + return { + "run_id": None, + "conclusion": "no_runs", + "failed_jobs": [], + "branch": branch, + "poll_duration_s": round(poll_duration, 1), + "scope_used": { + "event": scope.event, + "workflow": scope.workflow, + "head_sha": scope.head_sha, + }, + } # Phase 3: Wait for the found run to complete run_id = found_run["id"] - _log.info("ci_watcher_waiting", run_id=run_id) + logger.info("ci_watcher_waiting", run_id=run_id) attempt = 0 while time.monotonic() < deadline: run_data = await self._poll_run_status( @@ -365,7 +454,7 @@ async def wait( if conclusion in FAILED_CONCLUSIONS else [] ) - _log.info("ci_watcher_completed", run_id=run_id, conclusion=conclusion) + logger.info("ci_watcher_completed", run_id=run_id, conclusion=conclusion) return { "run_id": run_id, "conclusion": conclusion, @@ -379,11 +468,20 @@ async def wait( await asyncio.sleep(min(sleep_duration, remaining)) attempt += 1 - _log.warning("ci_watcher_timeout", run_id=run_id, timeout=timeout_seconds) - return {"run_id": run_id, "conclusion": "timed_out", "failed_jobs": []} + logger.warning("ci_watcher_timeout", run_id=run_id, timeout=timeout_seconds) + return { + "run_id": run_id, + "conclusion": "timed_out", + "failed_jobs": [], + "run_status": run_data.get("status", "in_progress"), + "hint": ( + f"CI run {run_id} is still in progress (not failed). " + "Call wait_for_ci again to continue watching." + ), + } except httpx.HTTPStatusError as exc: - _log.warning("ci_watcher_http_error", status=exc.response.status_code, branch=branch) + logger.warning("ci_watcher_http_error", status=exc.response.status_code, branch=branch) return { "run_id": None, "conclusion": "error", @@ -391,7 +489,7 @@ async def wait( "error": f"HTTP {exc.response.status_code}: {exc.response.text[:200]}", } except httpx.RequestError as exc: - _log.warning("ci_watcher_request_error", branch=branch, error=str(exc)) + logger.warning("ci_watcher_request_error", branch=branch, error=str(exc)) return { "run_id": None, "conclusion": "error", @@ -420,7 +518,11 @@ async def status( headers = self._headers() try: - async with httpx.AsyncClient(timeout=httpx.Timeout(15.0, connect=5.0)) as client: + async with make_tracked_httpx_client( + self._tracker, + timeout=httpx.Timeout(15.0, connect=5.0), + headers=github_headers(self._resolve_token()), + ) as client: if run_id is not None: run_data = await self._poll_run_status( client, @@ -482,11 +584,11 @@ async def status( return {"runs": runs} except httpx.HTTPStatusError as exc: - _log.warning("ci_status_http_error", status=exc.response.status_code) + logger.warning("ci_status_http_error", status=exc.response.status_code) return { "runs": [], "error": f"HTTP {exc.response.status_code}: {exc.response.text[:200]}", } except httpx.RequestError as exc: - _log.warning("ci_status_request_error", error=str(exc)) + logger.warning("ci_status_request_error", error=str(exc)) return {"runs": [], "error": f"Request error: {exc}"} diff --git a/src/autoskillit/execution/clone_guard.py b/src/autoskillit/execution/clone_guard.py index 9f8340a4c3..69210fd892 100644 --- a/src/autoskillit/execution/clone_guard.py +++ b/src/autoskillit/execution/clone_guard.py @@ -1,8 +1,8 @@ """Clone contamination guard — detect and revert direct changes to clone CWD. -L1 module (execution/). Detects when a worktree-based skill session modified -the clone directory directly (without creating a worktree) and reverts those -changes to prevent contamination from propagating to retry sessions. +IL-1 module (execution/). Detects when a worktree-based or read-only skill +session modified the clone directory directly and reverts those changes to +prevent contamination from propagating to retry sessions. Public API: is_worktree_skill(skill_command) -> bool @@ -42,14 +42,14 @@ _GIT_TIMEOUT: float = 10.0 -@dataclass(frozen=True) +@dataclass(frozen=True, slots=True) class CloneSnapshot: """Pre-session state of the clone directory.""" head_sha: str -@dataclass(frozen=True) +@dataclass(frozen=True, slots=True) class ContaminationReport: """Details of detected clone contamination.""" @@ -141,14 +141,45 @@ async def revert_contamination( report: ContaminationReport, cwd: str, runner: SubprocessRunner, + *, + selective: bool = False, ) -> ContaminationReport: - """Revert the clone to its pre-session state.""" + """Revert the clone to its pre-session state. + + When *selective* is True (read-only skills), uses ``git checkout -- .`` + and ``git clean -fd --exclude=.autoskillit/`` to preserve temp output. + Falls back to ``git reset --hard`` only when direct commits are present. + """ logger.info( "reverting_clone_contamination", pre_sha=snapshot.head_sha, direct_commits=report.direct_commits, uncommitted_file_count=len(report.uncommitted_files), + selective=selective, ) + if selective: + if report.direct_commits: + reset_result = await runner( + ["git", "reset", "--hard", snapshot.head_sha], + cwd=Path(cwd), + timeout=_GIT_TIMEOUT, + ) + if reset_result.returncode != 0: + return dataclasses.replace(report, reverted=False) + checkout_result = await runner( + ["git", "checkout", "--", "."], + cwd=Path(cwd), + timeout=_GIT_TIMEOUT, + ) + if checkout_result.returncode != 0: + return dataclasses.replace(report, reverted=False) + clean_result = await runner( + ["git", "clean", "-fd", "--exclude=.autoskillit/"], + cwd=Path(cwd), + timeout=_GIT_TIMEOUT, + ) + return dataclasses.replace(report, reverted=clean_result.returncode == 0) + reset_result = await runner( ["git", "reset", "--hard", snapshot.head_sha], cwd=Path(cwd), @@ -182,15 +213,21 @@ async def check_and_revert_clone_contamination( runner: SubprocessRunner, audit: AuditLog | None, skill_command: str = "", + *, + readonly_skill: bool = False, ) -> tuple[SkillResult, bool]: - """Top-level guard: detect and revert clone contamination after a failed session. + """Top-level guard: detect and revert clone contamination. + + For worktree skills, fires only on failure. For read-only skills + (``readonly_skill=True``), fires on both success and failure to catch + any stray writes that bypassed import layer 1 (IL-1). Returns (skill_result, reverted) where reverted is True if contamination was found and cleaned up. """ if snapshot is None: return skill_result, False - if skill_result.success: + if skill_result.success and not readonly_skill: return skill_result, False if skill_result.worktree_path is not None: return skill_result, False @@ -199,7 +236,7 @@ async def check_and_revert_clone_contamination( if report is None: return skill_result, False - report = await revert_contamination(snapshot, report, cwd, runner) + report = await revert_contamination(snapshot, report, cwd, runner, selective=readonly_skill) if audit is not None: audit.record_failure( diff --git a/src/autoskillit/execution/commands.py b/src/autoskillit/execution/commands.py index b5df08e549..c701216451 100644 --- a/src/autoskillit/execution/commands.py +++ b/src/autoskillit/execution/commands.py @@ -9,14 +9,28 @@ from types import MappingProxyType from autoskillit.core import ( + CAMPAIGN_ID_ENV_VAR, + KITCHEN_SESSION_ID_ENV_VAR, + SESSION_TYPE_ORCHESTRATOR, + SESSION_TYPE_SKILL, + BareResume, ClaudeFlags, + DirectInstall, + MarketplaceInstall, + NamedResume, + NoResume, + OutputFormat, + PluginSource, + ResumeSpec, + SessionCheckpoint, # noqa: F401, TC001 ValidatedAddDir, build_claude_env, + extract_skill_name, temp_dir_display_str, ) -@dataclass(frozen=True) +@dataclass(frozen=True, slots=True) class ClaudeInteractiveCmd: """Resolved argv + env for a claude interactive subprocess. @@ -30,7 +44,7 @@ class ClaudeInteractiveCmd: env: Mapping[str, str] = field(default_factory=dict) -@dataclass(frozen=True) +@dataclass(frozen=True, slots=True) class ClaudeHeadlessCmd: """Resolved argv + env for a claude headless subprocess. @@ -47,9 +61,9 @@ def build_interactive_cmd( *, initial_prompt: str | None = None, model: str | None = None, - plugin_dir: Path | None = None, + plugin_source: PluginSource | None = None, add_dirs: Sequence[Path | str | ValidatedAddDir] = (), - resume_session_id: str | None = None, + resume_spec: ResumeSpec = NoResume(), env_extras: Mapping[str, str] | None = None, ) -> ClaudeInteractiveCmd: """Build a Claude interactive session command. @@ -62,22 +76,45 @@ def build_interactive_cmd( session start. model Optional model override. - plugin_dir - When provided, appended as ``--plugin-dir ``. + plugin_source + When provided, determines the ``--plugin-dir`` flag. DirectInstall uses + the plugin_dir path; MarketplaceInstall omits the flag (parent session + already has it loaded). add_dirs Each entry is appended as ``--add-dir ``. - resume_session_id - When provided, appended as ``--resume `` before any positional prompt. + resume_spec + Resume intent discriminated union. ``NoResume`` (default) starts a fresh + session. ``BareResume`` passes ``--resume`` without an ID (Claude Code's + interactive picker). ``NamedResume`` passes ``--resume ``. env_extras Optional caller overrides merged into the resolved env after IDE scrubbing. + + Orchestration level + ------------------- + An interactive session operates at L1 (cook, ``autoskillit cook``) by default. + It becomes an L2 orchestrator (order, ``autoskillit order``) when the user + calls ``open_kitchen``, granting full kitchen access and the ability to dispatch + L1 ``run_skill`` workers. Unlike headless sessions, the orchestration level of + an interactive session is determined at runtime by kitchen state, not by a + ``SESSION_TYPE`` env variable. """ cmd = ["claude", ClaudeFlags.DANGEROUSLY_SKIP_PERMISSIONS] - if resume_session_id is not None: - cmd += [ClaudeFlags.RESUME, resume_session_id] + match resume_spec: + case NamedResume(session_id=sid): + cmd += [ClaudeFlags.RESUME, sid] + case BareResume(): + cmd.append(ClaudeFlags.RESUME) + case NoResume(): + pass if model: cmd += [ClaudeFlags.MODEL, model] - if plugin_dir is not None: - cmd += [ClaudeFlags.PLUGIN_DIR, str(plugin_dir)] + match plugin_source: + case DirectInstall(plugin_dir=p): + cmd += [ClaudeFlags.PLUGIN_DIR, str(p)] + case MarketplaceInstall(): + pass # parent session already has the marketplace plugin loaded + case None: + pass for d in add_dirs: cmd += [ClaudeFlags.ADD_DIR, str(d)] if initial_prompt is not None: @@ -115,30 +152,52 @@ def build_headless_cmd( _SESSION_BASELINE_ENV: Mapping[str, str] = MappingProxyType( { "MAX_MCP_OUTPUT_TOKENS": _MAX_MCP_OUTPUT_TOKENS_VALUE, + "MCP_CONNECTION_NONBLOCKING": "0", } ) -# Variables that build_full_headless_cmd controls exclusively. They must not +# Variables that build_skill_session_cmd controls exclusively. They must not # leak from the host process environment — the caller opts in via explicit -# parameters (exit_after_stop_delay_ms, scenario_step_name). -# Note: these overlap with IDE_ENV_DENYLIST in core/_claude_env.py, which -# strips the same keys as part of the broader IDE env scrubbing layer. -# Both lists must be kept in sync when adding new exclusive variables. +# parameters (exit_after_stop_delay_ms, scenario_step_name, allowed_write_prefix, etc.). +# Note: CLAUDE_CODE_EXIT_AFTER_STOP_DELAY, SCENARIO_STEP_NAME, and +# MAX_MCP_OUTPUT_TOKENS also overlap with IDE_ENV_DENYLIST in +# core/_claude_env.py. AUTOSKILLIT_SESSION_TYPE, AUTOSKILLIT_CAMPAIGN_ID, and +# AUTOSKILLIT_PROVIDER_PROFILE overlap with AUTOSKILLIT_PRIVATE_ENV_VARS +# (scrubbed by build_claude_env). +# All lists must be kept in sync when adding new exclusive variables. _HEADLESS_EXCLUSIVE_VARS: frozenset[str] = frozenset( { + "ANTHROPIC_API_KEY", + "ANTHROPIC_AUTH_TOKEN", + "ANTHROPIC_BASE_URL", + "AUTOSKILLIT_ALLOWED_WRITE_PREFIX", + "AUTOSKILLIT_CAMPAIGN_ID", + "AUTOSKILLIT_KITCHEN_SESSION_ID", + "AUTOSKILLIT_LAUNCH_ID", + "AUTOSKILLIT_PROVIDER_PROFILE", + "AUTOSKILLIT_SESSION_TYPE", + "AUTOSKILLIT_SKILL_NAME", "CLAUDE_CODE_EXIT_AFTER_STOP_DELAY", - "SCENARIO_STEP_NAME", "MAX_MCP_OUTPUT_TOKENS", + "SCENARIO_STEP_NAME", } ) +def _apply_output_format(cmd: list[str], output_format: OutputFormat) -> None: + """Append --output-format and all required CLI flags, deduplicating.""" + cmd += [ClaudeFlags.OUTPUT_FORMAT, output_format.value] + for flag in output_format.required_cli_flags: + if flag not in cmd: + cmd.append(flag) + + def build_headless_resume_cmd( *, resume_session_id: str, prompt: str, - output_format: str = "json", - plugin_dir: Path | None = None, + output_format: OutputFormat = OutputFormat.JSON, + plugin_source: PluginSource | None = None, env_extras: Mapping[str, str] | None = None, ) -> ClaudeHeadlessCmd: """Build a headless resume command for contract recovery nudge. @@ -154,26 +213,48 @@ def build_headless_resume_cmd( ClaudeFlags.RESUME, resume_session_id, ClaudeFlags.DANGEROUSLY_SKIP_PERMISSIONS, - ClaudeFlags.OUTPUT_FORMAT, - output_format, ] - if plugin_dir is not None: - cmd += [ClaudeFlags.PLUGIN_DIR, str(plugin_dir)] + _apply_output_format(cmd, output_format) + match plugin_source: + case DirectInstall(plugin_dir=p): + cmd += [ClaudeFlags.PLUGIN_DIR, str(p)] + case MarketplaceInstall(): + pass # parent session already has the marketplace plugin loaded + case None: + pass merged: dict[str, str] = dict(_SESSION_BASELINE_ENV) if env_extras: merged.update(env_extras) - return ClaudeHeadlessCmd(cmd=cmd, env=build_claude_env(extras=merged)) + return ClaudeHeadlessCmd(cmd=cmd, env=build_claude_env(base={}, extras=merged)) -def _ensure_skill_prefix(skill_command: str) -> str: - """Prompt-formatting helper: prepend 'Use ' to slash-commands for headless session loading. +def _ensure_skill_prefix(skill_command: str, *, provider_profile: str = "") -> str: + """Prompt-formatting helper: wrap slash-commands for headless session loading. + + Transforms `/foo args` into `Use the /foo skill args` so non-Claude models + recognize the slash command as a Skill tool invocation rather than a task description. This is NOT a validator. Non-slash input passes through unchanged by design — runtime validation is enforced by the skill_command_guard PreToolUse hook. """ stripped = skill_command.strip() if stripped.startswith("/"): - return f"Use {stripped}" + parts = stripped.split(None, 1) + slash_cmd = parts[0] + rest = parts[1] if len(parts) > 1 else "" + formatted = f"Use the {slash_cmd} skill" + if rest: + formatted += f" {rest}" + if provider_profile: + skill_name = extract_skill_name(stripped) or slash_cmd.lstrip("/") + formatted = ( + f"FIRST ACTION: Your first action should be to load the skill instructions " + f'by calling the Skill tool with skill="{skill_name}". ' + f"If the Skill tool is unavailable or returns an error, read the skill " + f"instructions from the skill's SKILL.md file instead.\n\n" + f"{formatted}" + ) + return formatted return skill_command @@ -197,14 +278,12 @@ def _inject_cwd_anchor(skill_command: str, cwd: str, temp_dir_relpath: str | Non f"\n\nWORKING DIRECTORY ANCHOR: Your working directory is {cwd}. " f"All relative paths ({relpath}/, .autoskillit/, etc.) " f"MUST resolve against {cwd}. " - f"Do NOT use any other directory as a base for relative paths, regardless of " - f"what paths appear in code-index tool responses or set_project_path results. " - f"The code-index project path is for READ-ONLY exploration only." + f"Do NOT use any other directory as a base for relative paths." ) return skill_command + directive -def _inject_narration_suppression(skill_command: str) -> str: +def _inject_narration_suppression(skill_command: str, *, has_skill_prefix: bool = False) -> str: """Append an efficiency directive to suppress inter-tool narration. Targets prose status text and phase announcements emitted between tool @@ -212,8 +291,9 @@ def _inject_narration_suppression(skill_command: str) -> str: long-running sessions. Does NOT suppress the final response, which is where structured output tokens (worktree_path, plan_path, etc.) live. """ + opener = "After loading the skill instructions, d" if has_skill_prefix else "D" directive = ( - "\n\nEFFICIENCY DIRECTIVE: Do NOT output prose status text, phase " + f"\n\nEFFICIENCY DIRECTIVE: {opener}o NOT output prose status text, phase " "announcements, or progress summaries between tool calls. Every " "non-final assistant turn MUST invoke at least one tool. The only " "permitted text-only turn is the final response required by the " @@ -222,21 +302,54 @@ def _inject_narration_suppression(skill_command: str) -> str: return skill_command + directive -def build_full_headless_cmd( +def _inject_completion_reminder(prompt: str, marker: str) -> str: + """Append a short completion marker reminder as the final prompt line. + + Provides recency-priority reinforcement for models that attend more strongly + to end-of-prompt instructions. + """ + if not marker: + return prompt + return f"{prompt}\nRemember: end your final response with {marker}" + + +def _build_resume_context(checkpoint: SessionCheckpoint) -> str: + lines = [ + "RESUME CONTEXT: The following items were completed in the previous session " + "and MUST be skipped. Do NOT redo any of them — continue from where the " + "previous session left off.", + "", + ] + for item in checkpoint.completed_items: + lines.append(f" - COMPLETED: {item}") + if checkpoint.step_name: + lines.append(f"\nLast active step: {checkpoint.step_name}") + return "\n".join(lines) + + +def build_skill_session_cmd( skill_command: str, *, cwd: str, completion_marker: str, model: str | None, - plugin_dir: str | Path | None, - output_format_value: str, - output_format_required_flags: Sequence[str] = (), + plugin_source: PluginSource | None, + output_format: OutputFormat, add_dirs: Sequence[ValidatedAddDir] = (), exit_after_stop_delay_ms: int = 0, scenario_step_name: str = "", temp_dir_relpath: str | None = None, + allowed_write_prefix: str = "", + provider_extras: Mapping[str, str] | None = None, + profile_name: str = "", + resume_session_id: str = "", + resume_checkpoint: SessionCheckpoint | None = None, ) -> ClaudeHeadlessCmd: - """Build the complete headless command spec ready for subprocess invocation. + """Build the complete headless command spec for a skill session. + + A skill session is a direct child of an orchestrator: it runs a skill, + always carries ``AUTOSKILLIT_SESSION_TYPE=skill``, and forwards + ``AUTOSKILLIT_CAMPAIGN_ID`` from the parent env when present. Applies prompt transformations (skill prefix, completion directive, cwd anchor, narration suppression), then constructs the full CLI command including plugin-dir, @@ -254,45 +367,239 @@ def build_full_headless_cmd( Marker string appended to the prompt as a completion directive. model Optional model override; passed through to ``build_headless_cmd``. - plugin_dir - Path passed as ``--plugin-dir`` flag. - output_format_value - String value passed as ``--output-format`` flag. - output_format_required_flags - Additional CLI flags required by the output format; deduplicated. + plugin_source + PluginSource determining the ``--plugin-dir`` flag. DirectInstall uses the + path; MarketplaceInstall omits the flag. + output_format + OutputFormat enum; ``--output-format`` and any required flags are self-applied. add_dirs Each entry is appended as ``--add-dir ``. exit_after_stop_delay_ms When > 0, carried as ``CLAUDE_CODE_EXIT_AFTER_STOP_DELAY=`` in ``.env``. scenario_step_name When non-empty, carried as ``SCENARIO_STEP_NAME=`` in ``.env`` for recording. + temp_dir_relpath + Relative path to the temp directory injected into the CWD anchor directive. + Falls back to the canonical default when None. + allowed_write_prefix + When non-empty, carried as ``AUTOSKILLIT_ALLOWED_WRITE_PREFIX`` in ``.env``. + provider_extras + Provider-profile env vars merged into the session environment after + session-identity keys are set. ``AUTOSKILLIT_SESSION_TYPE`` and + ``AUTOSKILLIT_HEADLESS`` keys are silently ignored. + profile_name + When non-empty, carried as ``AUTOSKILLIT_PROVIDER_PROFILE`` in ``.env``. """ - prompt = _inject_narration_suppression( - _inject_cwd_anchor( - _inject_completion_directive(_ensure_skill_prefix(skill_command), completion_marker), - cwd, - temp_dir_relpath=temp_dir_relpath, + if resume_session_id: + _resume_instruction = ( + "Your previous session was interrupted before completion. " + "Continue your work from where you left off. " + "Do NOT restart from scratch — pick up exactly where you stopped." + ) + if resume_checkpoint and resume_checkpoint.completed_items: + _resume_instruction += "\n\n" + _build_resume_context(resume_checkpoint) + prompt = _inject_completion_reminder( + _inject_narration_suppression( + _inject_cwd_anchor( + _inject_completion_directive(_resume_instruction, completion_marker), + cwd, + temp_dir_relpath=temp_dir_relpath, + ) + ), + completion_marker, + ) + else: + _has_prefix = bool(profile_name) and skill_command.strip().startswith("/") + prompt = _inject_completion_reminder( + _inject_narration_suppression( + _inject_cwd_anchor( + _inject_completion_directive( + _ensure_skill_prefix(skill_command, provider_profile=profile_name or ""), + completion_marker, + ), + cwd, + temp_dir_relpath=temp_dir_relpath, + ), + has_skill_prefix=_has_prefix, + ), + completion_marker, ) - ) extras: dict[str, str] = { "AUTOSKILLIT_HEADLESS": "1", + "AUTOSKILLIT_SESSION_TYPE": SESSION_TYPE_SKILL, "MAX_MCP_OUTPUT_TOKENS": _MAX_MCP_OUTPUT_TOKENS_VALUE, + "MCP_CONNECTION_NONBLOCKING": "0", } if exit_after_stop_delay_ms > 0: extras["CLAUDE_CODE_EXIT_AFTER_STOP_DELAY"] = str(exit_after_stop_delay_ms) if scenario_step_name: extras["SCENARIO_STEP_NAME"] = scenario_step_name + campaign_id = os.environ.get(CAMPAIGN_ID_ENV_VAR) + if campaign_id: + extras[CAMPAIGN_ID_ENV_VAR] = campaign_id + kitchen_session_id = os.environ.get(KITCHEN_SESSION_ID_ENV_VAR) + if kitchen_session_id: + extras[KITCHEN_SESSION_ID_ENV_VAR] = kitchen_session_id + if allowed_write_prefix: + extras["AUTOSKILLIT_ALLOWED_WRITE_PREFIX"] = allowed_write_prefix + extras["AUTOSKILLIT_SKILL_NAME"] = extract_skill_name(skill_command) or "" + if provider_extras: + for k, v in provider_extras.items(): + if k not in ("AUTOSKILLIT_SESSION_TYPE", "AUTOSKILLIT_HEADLESS"): + extras[k] = v + if profile_name: + extras["AUTOSKILLIT_PROVIDER_PROFILE"] = profile_name + extras["AUTOSKILLIT_COMPLETION_MARKER"] = completion_marker filtered_base = {k: v for k, v in os.environ.items() if k not in _HEADLESS_EXCLUSIVE_VARS} spec = build_headless_cmd(prompt, model=model, env_extras=extras, base=filtered_base) cmd: list[str] = [*spec.cmd] - if plugin_dir is not None: - cmd += [ClaudeFlags.PLUGIN_DIR, str(plugin_dir)] - cmd += [ClaudeFlags.OUTPUT_FORMAT, output_format_value] - for flag in output_format_required_flags: - if flag not in cmd: - cmd.append(flag) + match plugin_source: + case DirectInstall(plugin_dir=p): + cmd += [ClaudeFlags.PLUGIN_DIR, str(p)] + case MarketplaceInstall(): + pass # parent session already has the marketplace plugin loaded + case None: + pass + _apply_output_format(cmd, output_format) for validated_dir in add_dirs: cmd.extend([ClaudeFlags.ADD_DIR, validated_dir.path]) + if resume_session_id: + cmd += [ClaudeFlags.RESUME, resume_session_id] + + return ClaudeHeadlessCmd(cmd=cmd, env=spec.env) + + +def build_food_truck_cmd( + *, + orchestrator_prompt: str, + plugin_source: PluginSource, + cwd: str, + completion_marker: str, + resume_session_id: str | None = None, + resume_checkpoint: SessionCheckpoint | None = None, + model: str | None = None, + env_extras: Mapping[str, str] | None = None, + output_format: OutputFormat = OutputFormat.STREAM_JSON, + exit_after_stop_delay_ms: int = 0, + scenario_step_name: str = "", + temp_dir_relpath: str | None = None, + allowed_write_prefix: str = "", +) -> ClaudeHeadlessCmd: + """Build the complete headless command spec for an L2 food truck session. + + A food truck session is an L2 orchestrator: it runs a full recipe + autonomously, always carries ``AUTOSKILLIT_SESSION_TYPE=orchestrator``, + and restricts Claude native tools to ``--tools AskUserQuestion``. + + Unlike ``build_skill_session_cmd``, this builder: + - Does NOT call ``_ensure_skill_prefix`` (prompt is a complete orchestrator prompt) + - Sets ``SESSION_TYPE=orchestrator`` (not ``skill``) + - Accepts caller-provided ``env_extras`` for campaign-specific variables + (CAMPAIGN_ID, CAMPAIGN_STATE_PATH, PROJECT_DIR, FOOD_TRUCK_TOOL_TAGS, etc.) + - Always emits ``--plugin-dir``: DirectInstall uses plugin_dir, MarketplaceInstall + uses cache_path (food truck sessions are fresh subprocesses that need explicit + plugin loading, unlike skill sessions where the parent already has it). + + Parameters + ---------- + orchestrator_prompt + Complete system prompt built by the fleet caller. + plugin_source + PluginSource determining the ``--plugin-dir`` path. Both DirectInstall and + MarketplaceInstall produce a ``--plugin-dir`` flag (paths differ). + cwd + Absolute path to the working directory. Injected as anchor directive. + completion_marker + Marker string appended to the prompt as a completion directive. + model + Optional model override. + env_extras + Caller-provided env variables layered on top of the baseline. + Used for CAMPAIGN_ID, CAMPAIGN_STATE_PATH, PROJECT_DIR, FOOD_TRUCK_TOOL_TAGS, + IDLE_OUTPUT_TIMEOUT. These override baseline but cannot override + SESSION_TYPE or HEADLESS (applied last). + output_format + OutputFormat enum; ``--output-format`` and any required flags are self-applied. + Defaults to ``STREAM_JSON``. + exit_after_stop_delay_ms + If positive, sets ``CLAUDE_CODE_EXIT_AFTER_STOP_DELAY`` in the subprocess env. + Zero (default) means the variable is omitted entirely. + scenario_step_name + If non-empty, sets ``SCENARIO_STEP_NAME`` in the subprocess env. + temp_dir_relpath + Relative path to the temp directory injected into the CWD anchor directive. + Falls back to the canonical default when None. + allowed_write_prefix + If non-empty, sets ``AUTOSKILLIT_ALLOWED_WRITE_PREFIX`` in the subprocess env. + """ + if resume_session_id: + _resume_instruction = ( + "Your previous session was interrupted before completion. " + "Continue your work from where you left off. " + "Do NOT restart from scratch — pick up exactly where you stopped." + ) + if resume_checkpoint and resume_checkpoint.completed_items: + _resume_instruction += "\n\n" + _build_resume_context(resume_checkpoint) + prompt = _inject_completion_reminder( + _inject_narration_suppression( + _inject_cwd_anchor( + _inject_completion_directive(_resume_instruction, completion_marker), + cwd, + temp_dir_relpath=temp_dir_relpath, + ) + ), + completion_marker, + ) + else: + # No _ensure_skill_prefix — orchestrator_prompt is a complete system prompt. + prompt = _inject_completion_reminder( + _inject_narration_suppression( + _inject_cwd_anchor( + _inject_completion_directive(orchestrator_prompt, completion_marker), + cwd, + temp_dir_relpath=temp_dir_relpath, + ) + ), + completion_marker, + ) + + # Baseline env: headless + orchestrator session type + MCP settings. + extras: dict[str, str] = { + "AUTOSKILLIT_HEADLESS": "1", + "AUTOSKILLIT_SESSION_TYPE": SESSION_TYPE_ORCHESTRATOR, + "MAX_MCP_OUTPUT_TOKENS": _MAX_MCP_OUTPUT_TOKENS_VALUE, + "MCP_CONNECTION_NONBLOCKING": "0", + } + if exit_after_stop_delay_ms > 0: + extras["CLAUDE_CODE_EXIT_AFTER_STOP_DELAY"] = str(exit_after_stop_delay_ms) + if scenario_step_name: + extras["SCENARIO_STEP_NAME"] = scenario_step_name + kitchen_session_id = os.environ.get(KITCHEN_SESSION_ID_ENV_VAR) + if kitchen_session_id: + extras[KITCHEN_SESSION_ID_ENV_VAR] = kitchen_session_id + if allowed_write_prefix: + extras["AUTOSKILLIT_ALLOWED_WRITE_PREFIX"] = allowed_write_prefix + # Layer caller env_extras (campaign vars) UNDER the mandatory keys. + # This ensures SESSION_TYPE and HEADLESS cannot be accidentally overridden. + if env_extras: + for k, v in env_extras.items(): + if k not in ("AUTOSKILLIT_SESSION_TYPE", "AUTOSKILLIT_HEADLESS"): + extras[k] = v + + filtered_base = {k: v for k, v in os.environ.items() if k not in _HEADLESS_EXCLUSIVE_VARS} + spec = build_headless_cmd(prompt, model=model, env_extras=extras, base=filtered_base) + + cmd: list[str] = [*spec.cmd] + # Both install modes require --plugin-dir for food truck sessions (fresh subprocess). + match plugin_source: + case DirectInstall(plugin_dir=p): + cmd += [ClaudeFlags.PLUGIN_DIR, str(p)] + case MarketplaceInstall(cache_path=cp): + cmd += [ClaudeFlags.PLUGIN_DIR, str(cp)] + _apply_output_format(cmd, output_format) + cmd += [ClaudeFlags.TOOLS, "AskUserQuestion"] + if resume_session_id: + cmd += [ClaudeFlags.RESUME, resume_session_id] return ClaudeHeadlessCmd(cmd=cmd, env=spec.env) diff --git a/src/autoskillit/execution/db.py b/src/autoskillit/execution/db.py index 5749c10322..2896590215 100644 --- a/src/autoskillit/execution/db.py +++ b/src/autoskillit/execution/db.py @@ -6,10 +6,11 @@ from __future__ import annotations import base64 -import re import sqlite3 import threading +import regex as re + from autoskillit.core import get_logger logger = get_logger(__name__) diff --git a/src/autoskillit/execution/diff_annotator.py b/src/autoskillit/execution/diff_annotator.py index aaf897678b..cc44f228d2 100644 --- a/src/autoskillit/execution/diff_annotator.py +++ b/src/autoskillit/execution/diff_annotator.py @@ -7,9 +7,10 @@ from __future__ import annotations -import re from dataclasses import dataclass, field +import regex as re + _HUNK_HEADER = re.compile(r"^@@ -\d+(?:,\d+)? \+(\d+)(?:,(\d+))? @@") _FILE_HEADER = re.compile(r"^\+\+\+ b/(.+)$") @@ -23,6 +24,72 @@ class FilterResult: all_unpostable: bool = False +@dataclass +class DiffMetrics: + """Size metrics computed from a unified diff.""" + + added_lines: int + removed_lines: int + changed_files: int + file_paths: list[str] = field(default_factory=list) + + +_STRUCTURAL_SUFFIXES = ( + "/__init__.py", + "/setup.py", + "/setup.cfg", + "/pyproject.toml", + "/Makefile", + "/Taskfile.yml", + "/Dockerfile", + "/docker-compose.yml", + "/docker-compose.yaml", +) + +_ALL_STANDARD_AGENTS = ("arch", "tests", "defense", "bugs", "cohesion", "slop") +_SMALL_DIFF_CORE_AGENTS = ("tests", "cohesion") + + +def compute_diff_metrics(diff_text: str) -> DiffMetrics: + added = 0 + removed = 0 + file_paths: list[str] = [] + + for line in diff_text.splitlines(): + if line.startswith("+++ b/"): + file_paths.append(line[6:]) + continue + if line.startswith("+++"): + continue + if line.startswith("---"): + continue + if line.startswith("+"): + added += 1 + elif line.startswith("-"): + removed += 1 + + return DiffMetrics( + added_lines=added, + removed_lines=removed, + changed_files=len(file_paths), + file_paths=file_paths, + ) + + +def select_review_agents( + metrics: DiffMetrics, + *, + loc_threshold: int = 200, + file_threshold: int = 5, +) -> list[str]: + if metrics.added_lines < loc_threshold and metrics.changed_files < file_threshold: + agents: list[str] = list(_SMALL_DIFF_CORE_AGENTS) + if any(("/" + fp).endswith(s) for fp in metrics.file_paths for s in _STRUCTURAL_SUFFIXES): + agents.insert(0, "arch") + return agents + return list(_ALL_STANDARD_AGENTS) + + def parse_hunk_ranges(diff_text: str) -> dict[str, list[tuple[int, int]]]: """Extract per-file valid line ranges from unified diff @@ headers. @@ -122,3 +189,42 @@ def filter_findings( unpostable=unpostable, all_unpostable=len(findings) > 0 and len(filtered) == 0, ) + + +_LINE_MARKER = re.compile(r"^\[L(\d+)\]") + + +def extract_code_region( + annotated_diff: str, + file_path: str, + line: int, + context_lines: int = 50, +) -> str: + if not annotated_diff: + return "" + + lines = annotated_diff.splitlines() + in_file = False + collected: list[str] = [] + lo = line - context_lines + hi = line + context_lines + + for raw_line in lines: + header_match = _FILE_HEADER.match(raw_line) + if header_match: + if in_file: + break + if header_match.group(1) == file_path: + in_file = True + continue + + if not in_file: + continue + + marker_match = _LINE_MARKER.match(raw_line) + if marker_match: + marker_line = int(marker_match.group(1)) + if lo <= marker_line <= hi: + collected.append(raw_line) + + return "\n".join(collected) diff --git a/src/autoskillit/execution/github.py b/src/autoskillit/execution/github.py index 27a0b12ffe..3fb3c72934 100644 --- a/src/autoskillit/execution/github.py +++ b/src/autoskillit/execution/github.py @@ -1,20 +1,26 @@ """GitHub issue fetcher. -L1 module: depends only on stdlib, httpx, and core/logging. +IL-1 module: depends only on stdlib, httpx, and core/logging. Never raises — all errors are captured and returned as {"success": False, "error": "..."}. """ from __future__ import annotations -import re +import asyncio +import time from collections.abc import Callable -from typing import Any +from datetime import UTC, datetime +from typing import TYPE_CHECKING, Any import httpx +import regex as re from autoskillit.core import _parse_issue_ref, get_logger -_log = get_logger(__name__) +if TYPE_CHECKING: + from autoskillit.core import GitHubApiLog + +logger = get_logger(__name__) def _slugify(title: str) -> str: @@ -78,6 +84,66 @@ def github_headers(token: str | None) -> dict[str, str]: return h +class _TrackingTransport(httpx.AsyncBaseTransport): + """httpx transport wrapper that records each request into a GitHubApiLog.""" + + def __init__( + self, + wrapped: httpx.AsyncBaseTransport, + tracker: GitHubApiLog, + ) -> None: + self._wrapped = wrapped + self._tracker = tracker + + async def handle_async_request(self, request: httpx.Request) -> httpx.Response: + start = time.monotonic() + response = await self._wrapped.handle_async_request(request) + latency_ms = (time.monotonic() - start) * 1000.0 + headers = response.headers + + def _int(key: str) -> int: + try: + return int(headers.get(key, "-1")) + except ValueError: + return -1 + + await self._tracker.record_httpx( + method=request.method, + path=str(request.url.path), + status_code=response.status_code, + latency_ms=latency_ms, + rate_limit_remaining=_int("x-ratelimit-remaining"), + rate_limit_used=_int("x-ratelimit-used"), + rate_limit_reset=_int("x-ratelimit-reset"), + timestamp=datetime.now(UTC).isoformat(), + ) + return response + + async def aclose(self) -> None: + await self._wrapped.aclose() + + +def make_tracked_httpx_client( + tracker: GitHubApiLog | None, + *, + timeout: httpx.Timeout, + headers: dict[str, str] | None = None, + limits: httpx.Limits | None = None, + base_url: str | None = None, +) -> httpx.AsyncClient: + """Build an httpx.AsyncClient, optionally wrapping transport with tracking.""" + kwargs: dict[str, Any] = {"timeout": timeout} + if headers is not None: + kwargs["headers"] = headers + if limits is not None: + kwargs["limits"] = limits + if base_url is not None: + kwargs["base_url"] = base_url + if tracker is not None: + kwargs["transport"] = _TrackingTransport(httpx.AsyncHTTPTransport(), tracker) + return httpx.AsyncClient(**kwargs) + + def parse_merge_queue_response(data: dict[str, Any]) -> list[dict[str, Any]]: """Parse a GitHub GraphQL merge queue response into a sorted entry list. @@ -92,7 +158,7 @@ def parse_merge_queue_response(data: dict[str, Any]) -> list[dict[str, Any]]: rather than skipped, so callers can inspect and handle partial entries. """ if "errors" in data: - _log.warning("parse_merge_queue_response graphql errors", errors=data["errors"]) + logger.warning("parse_merge_queue_response graphql errors", errors=data["errors"]) return [] try: queue = data.get("data", {}).get("repository", {}).get("mergeQueue") or {} @@ -125,7 +191,13 @@ class DefaultGitHubFetcher: _UNRESOLVED = object() - def __init__(self, *, token: str | None | Callable[[], str | None] = None) -> None: + def __init__( + self, + *, + token: str | None | Callable[[], str | None] = None, + tracker: GitHubApiLog | None = None, + base_url: str = "https://api.github.com", + ) -> None: self._token_factory: Callable[[], str | None] | None if callable(token): self._token_factory = token @@ -133,6 +205,11 @@ def __init__(self, *, token: str | None | Callable[[], str | None] = None) -> No else: self._token_factory = None self._token = token + self._tracker = tracker + self._base_url = base_url.rstrip("/") + self._last_mutating_ts: float = 0.0 + self._mutating_lock: asyncio.Lock = asyncio.Lock() + self._label_cache: set[tuple[str, str, str]] = set() def _resolve_token(self) -> str | None: if self._token is self._UNRESOLVED: @@ -147,27 +224,55 @@ def has_token(self) -> bool: def _headers(self) -> dict[str, str]: return github_headers(self._resolve_token()) + async def _throttle_mutating(self) -> None: + """Enforce >= 1s gap between mutating GitHub API calls. + + Acquires an async lock to serialize concurrent mutating calls, + then sleeps for the remaining time if < 1s has elapsed since + the last mutating call. + """ + async with self._mutating_lock: + now = time.monotonic() + elapsed = now - self._last_mutating_ts + if elapsed < 1.0: + await asyncio.sleep(1.0 - elapsed) + self._last_mutating_ts = time.monotonic() + async def fetch_issue( self, - issue_ref: str, + issue_ref_or_owner: str, + repo: str | None = None, + number: int | None = None, *, include_comments: bool = True, ) -> dict[str, Any]: """Fetch a GitHub issue. Returns structured data + Markdown content. - Never raises. + Accepts either a single issue_ref string ("owner/repo#N") or separate + (owner, repo, number) positional arguments. Never raises. """ - try: - owner, repo, number = _parse_issue_ref(issue_ref) - except ValueError as exc: - return {"success": False, "error": str(exc)} + if (repo is None) != (number is None): + return {"success": False, "error": "repo and number must both be provided together"} + if repo is not None and number is not None: + owner = issue_ref_or_owner + issue_ref = f"{owner}/{repo}#{number}" + else: + try: + owner, repo, number = _parse_issue_ref(issue_ref_or_owner) + except ValueError as exc: + return {"success": False, "error": str(exc)} + issue_ref = issue_ref_or_owner - headers = self._headers() - base = f"https://api.github.com/repos/{owner}/{repo}/issues/{number}" + path = f"/repos/{owner}/{repo}/issues/{number}" try: - async with httpx.AsyncClient(timeout=httpx.Timeout(15.0, connect=5.0)) as client: - resp = await client.get(base, headers=headers) + async with make_tracked_httpx_client( + self._tracker, + timeout=httpx.Timeout(15.0, connect=5.0), + headers=github_headers(self._resolve_token()), + base_url=self._base_url, + ) as client: + resp = await client.get(path) if resp.status_code == 404: if not self.has_token: return { @@ -187,9 +292,7 @@ async def fetch_issue( comments: list[dict[str, str]] = [] if include_comments and issue_data.get("comments", 0) > 0: - c_resp = await client.get( - f"{base}/comments", headers=headers, params={"per_page": 100} - ) + c_resp = await client.get(f"{path}/comments", params={"per_page": 100}) c_resp.raise_for_status() comments = [ {"author": c["user"]["login"], "body": c["body"] or ""} @@ -197,7 +300,7 @@ async def fetch_issue( ] except httpx.HTTPStatusError as exc: - _log.warning( + logger.warning( "github fetch http error", status=exc.response.status_code, ref=issue_ref, @@ -207,7 +310,7 @@ async def fetch_issue( "error": f"HTTP {exc.response.status_code}: {exc.response.text[:200]}", } except httpx.RequestError as exc: - _log.warning("github fetch request error", ref=issue_ref, error=str(exc)) + logger.warning("github fetch request error", ref=issue_ref, error=str(exc)) return {"success": False, "error": f"Request error: {exc}"} labels = [lbl["name"] for lbl in issue_data.get("labels", [])] @@ -254,21 +357,26 @@ async def search_issues( Never raises. """ search_query = f'repo:{owner}/{repo} is:issue state:{state} "{query}"' - headers = self._headers() try: - async with httpx.AsyncClient(timeout=httpx.Timeout(15.0, connect=5.0)) as client: + async with make_tracked_httpx_client( + self._tracker, + timeout=httpx.Timeout(15.0, connect=5.0), + headers=github_headers(self._resolve_token()), + base_url=self._base_url, + ) as client: resp = await client.get( - "https://api.github.com/search/issues", - headers=headers, + "/search/issues", params={"q": search_query, "per_page": 5}, ) resp.raise_for_status() data = resp.json() except httpx.HTTPStatusError as exc: - _log.warning("github search http error", status=exc.response.status_code, query=query) + logger.warning( + "github search http error", status=exc.response.status_code, query=query + ) return {"success": False, "error": f"HTTP {exc.response.status_code}"} except httpx.RequestError as exc: - _log.warning("github search request error", query=query, error=str(exc)) + logger.warning("github search request error", query=query, error=str(exc)) return {"success": False, "error": f"Request error: {exc}"} items = [ @@ -296,27 +404,31 @@ async def create_issue( Returns {success, issue_number, url}. Never raises. """ - headers = self._headers() + await self._throttle_mutating() payload: dict[str, Any] = {"title": title, "body": body} if labels: payload["labels"] = labels try: - async with httpx.AsyncClient(timeout=httpx.Timeout(15.0, connect=5.0)) as client: + async with make_tracked_httpx_client( + self._tracker, + timeout=httpx.Timeout(15.0, connect=5.0), + headers=github_headers(self._resolve_token()), + base_url=self._base_url, + ) as client: resp = await client.post( - f"https://api.github.com/repos/{owner}/{repo}/issues", - headers=headers, + f"/repos/{owner}/{repo}/issues", json=payload, ) resp.raise_for_status() data = resp.json() except httpx.HTTPStatusError as exc: - _log.warning("github create_issue http error", status=exc.response.status_code) + logger.warning("github create_issue http error", status=exc.response.status_code) return { "success": False, "error": f"HTTP {exc.response.status_code}: {exc.response.text[:200]}", } except httpx.RequestError as exc: - _log.warning("github create_issue request error", error=str(exc)) + logger.warning("github create_issue request error", error=str(exc)) return {"success": False, "error": f"Request error: {exc}"} return { @@ -325,46 +437,49 @@ async def create_issue( "url": data.get("html_url", ""), } - async def add_comment( + async def update_issue_body( self, owner: str, repo: str, issue_number: int, - body: str, + new_body: str, ) -> dict[str, Any]: - """Post a comment on an existing issue. + """PATCH an existing issue's body field. - Returns {success, comment_id, url}. Never raises. + Returns {success, issue_url}. Never raises. """ - headers = self._headers() + await self._throttle_mutating() try: - async with httpx.AsyncClient(timeout=httpx.Timeout(15.0, connect=5.0)) as client: - resp = await client.post( - f"https://api.github.com/repos/{owner}/{repo}/issues/{issue_number}/comments", - headers=headers, - json={"body": body}, + async with make_tracked_httpx_client( + self._tracker, + timeout=httpx.Timeout(15.0, connect=5.0), + headers=github_headers(self._resolve_token()), + base_url=self._base_url, + ) as client: + resp = await client.patch( + f"/repos/{owner}/{repo}/issues/{issue_number}", + json={"body": new_body}, ) resp.raise_for_status() data = resp.json() except httpx.HTTPStatusError as exc: - _log.warning( - "github add_comment http error", + logger.warning( + "github update_issue_body http error", status=exc.response.status_code, issue=issue_number, + exc_info=True, ) return { "success": False, "error": f"HTTP {exc.response.status_code}: {exc.response.text[:200]}", } except httpx.RequestError as exc: - _log.warning("github add_comment request error", issue=issue_number, error=str(exc)) + logger.warning( + "github update_issue_body request error", issue=issue_number, error=str(exc) + ) return {"success": False, "error": f"Request error: {exc}"} - return { - "success": True, - "comment_id": data.get("id"), - "url": data.get("html_url", ""), - } + return {"success": True, "issue_url": data.get("html_url", "")} async def fetch_title(self, issue_url: str) -> dict[str, object]: """Fetch only the title and slug for a GitHub issue — no body, no comments. @@ -376,10 +491,15 @@ async def fetch_title(self, issue_url: str) -> dict[str, object]: owner, repo, number = _parse_issue_ref(issue_url) except ValueError as exc: return {"success": False, "error": str(exc)} - url = f"https://api.github.com/repos/{owner}/{repo}/issues/{number}" + path = f"/repos/{owner}/{repo}/issues/{number}" try: - async with httpx.AsyncClient(timeout=httpx.Timeout(15.0, connect=5.0)) as client: - resp = await client.get(url, headers=self._headers()) + async with make_tracked_httpx_client( + self._tracker, + timeout=httpx.Timeout(15.0, connect=5.0), + headers=github_headers(self._resolve_token()), + base_url=self._base_url, + ) as client: + resp = await client.get(path) if resp.status_code == 404: hint = ( " Set github.token in .autoskillit/.secrets.yaml," @@ -414,24 +534,28 @@ async def add_labels( labels: list[str], ) -> dict[str, Any]: """Add labels to an issue. Returns {success, labels}. Never raises.""" - headers = self._headers() + await self._throttle_mutating() try: - async with httpx.AsyncClient(timeout=httpx.Timeout(15.0, connect=5.0)) as client: + async with make_tracked_httpx_client( + self._tracker, + timeout=httpx.Timeout(15.0, connect=5.0), + headers=github_headers(self._resolve_token()), + base_url=self._base_url, + ) as client: resp = await client.post( - f"https://api.github.com/repos/{owner}/{repo}/issues/{issue_number}/labels", - headers=headers, + f"/repos/{owner}/{repo}/issues/{issue_number}/labels", json={"labels": labels}, ) resp.raise_for_status() data = resp.json() except httpx.HTTPStatusError as exc: - _log.warning("github add_labels http error", status=exc.response.status_code) + logger.warning("github add_labels http error", status=exc.response.status_code) return { "success": False, "error": f"HTTP {exc.response.status_code}: {exc.response.text[:200]}", } except httpx.RequestError as exc: - _log.warning("github add_labels request error", error=str(exc)) + logger.warning("github add_labels request error", error=str(exc)) return {"success": False, "error": f"Request error: {exc}"} applied = [lbl["name"] for lbl in data] return {"success": True, "labels": applied} @@ -445,28 +569,88 @@ async def remove_label( ) -> dict[str, Any]: """Remove a label from an issue. 404 is treated as success (idempotent). Returns {success}. Never raises.""" - headers = self._headers() + await self._throttle_mutating() encoded = label.replace(" ", "%20") try: - async with httpx.AsyncClient(timeout=httpx.Timeout(15.0, connect=5.0)) as client: + async with make_tracked_httpx_client( + self._tracker, + timeout=httpx.Timeout(15.0, connect=5.0), + headers=github_headers(self._resolve_token()), + base_url=self._base_url, + ) as client: resp = await client.delete( - f"https://api.github.com/repos/{owner}/{repo}/issues/{issue_number}/labels/{encoded}", - headers=headers, + f"/repos/{owner}/{repo}/issues/{issue_number}/labels/{encoded}", ) if resp.status_code == 404: return {"success": True} # already removed — idempotent resp.raise_for_status() except httpx.HTTPStatusError as exc: - _log.warning("github remove_label http error", status=exc.response.status_code) + logger.warning("github remove_label http error", status=exc.response.status_code) return { "success": False, "error": f"HTTP {exc.response.status_code}: {exc.response.text[:200]}", } except httpx.RequestError as exc: - _log.warning("github remove_label request error", error=str(exc)) + logger.warning("github remove_label request error", error=str(exc)) return {"success": False, "error": f"Request error: {exc}"} return {"success": True} + async def swap_labels( + self, + owner: str, + repo: str, + issue_number: int, + remove_labels: list[str], + add_labels: list[str], + ) -> dict[str, Any]: + """Swap labels on an issue via GET-then-PUT (reduced-race replacement). + + Fetches the current label set, computes (current - remove_labels) | add_labels, + then PUTs the result. The PUT endpoint replaces all labels in one HTTP call, + reducing the race window compared to DELETE→POST→POST, but is not truly atomic: + a concurrent writer between the GET (fetch) and PUT (replace) can silently + overwrite interleaved changes. + + Returns {success, labels}. Never raises. + """ + try: + async with make_tracked_httpx_client( + self._tracker, + timeout=httpx.Timeout(15.0, connect=5.0), + headers=github_headers(self._resolve_token()), + base_url=self._base_url, + ) as client: + # Step 1: GET current labels (read-only, no throttle needed) + get_resp = await client.get( + f"/repos/{owner}/{repo}/issues/{issue_number}/labels", + ) + get_resp.raise_for_status() + current = {lbl["name"] for lbl in get_resp.json()} + + # Step 2: Compute target set + remove_set = set(remove_labels) + add_set = set(add_labels) + target = sorted((current - remove_set) | add_set) + + # Step 3: PUT atomically (throttle the single mutating call) + await self._throttle_mutating() + put_resp = await client.put( + f"/repos/{owner}/{repo}/issues/{issue_number}/labels", + json={"labels": target}, + ) + put_resp.raise_for_status() + applied = [lbl["name"] for lbl in put_resp.json()] + except httpx.HTTPStatusError as exc: + logger.warning("github swap_labels http error", status=exc.response.status_code) + return { + "success": False, + "error": f"HTTP {exc.response.status_code}: {exc.response.text[:200]}", + } + except httpx.RequestError as exc: + logger.warning("github swap_labels request error", error=str(exc)) + return {"success": False, "error": f"Request error: {exc}"} + return {"success": True, "labels": applied} + async def ensure_label( self, owner: str, @@ -476,25 +660,38 @@ async def ensure_label( description: str = "", ) -> dict[str, Any]: """Create a label if it doesn't exist. 422 (already exists) is success. - Returns {success, created}. Never raises.""" - headers = self._headers() + Returns {success, created}. Never raises. + Caches successful results per (owner, repo, label) triple to skip + redundant API calls within a session. + """ + cache_key = (owner, repo, label) + if cache_key in self._label_cache: + return {"success": True, "created": False} + + await self._throttle_mutating() try: - async with httpx.AsyncClient(timeout=httpx.Timeout(15.0, connect=5.0)) as client: + async with make_tracked_httpx_client( + self._tracker, + timeout=httpx.Timeout(15.0, connect=5.0), + headers=github_headers(self._resolve_token()), + base_url=self._base_url, + ) as client: resp = await client.post( - f"https://api.github.com/repos/{owner}/{repo}/labels", - headers=headers, + f"/repos/{owner}/{repo}/labels", json={"name": label, "color": color, "description": description}, ) if resp.status_code == 422: - return {"success": True, "created": False} # already exists + self._label_cache.add(cache_key) + return {"success": True, "created": False} resp.raise_for_status() except httpx.HTTPStatusError as exc: - _log.warning("github ensure_label http error", status=exc.response.status_code) + logger.warning("github ensure_label http error", status=exc.response.status_code) return { "success": False, "error": f"HTTP {exc.response.status_code}: {exc.response.text[:200]}", } except httpx.RequestError as exc: - _log.warning("github ensure_label request error", error=str(exc)) + logger.warning("github ensure_label request error", error=str(exc)) return {"success": False, "error": f"Request error: {exc}"} + self._label_cache.add(cache_key) return {"success": True, "created": True} diff --git a/src/autoskillit/execution/headless.py b/src/autoskillit/execution/headless.py deleted file mode 100644 index f312c1034a..0000000000 --- a/src/autoskillit/execution/headless.py +++ /dev/null @@ -1,1249 +0,0 @@ -"""Headless Claude Code session orchestration. - -L1 module (execution/). Owns the full lifecycle of a headless claude CLI session: -command preparation, subprocess invocation via the injected runner, and -SkillResult construction. - -Public API: - run_headless_core(skill_command, cwd, ctx, *, ...) -> SkillResult -""" - -from __future__ import annotations - -import dataclasses -import os -import re -import time -import traceback -from collections.abc import Sequence -from datetime import UTC, datetime, timedelta -from pathlib import Path -from typing import TYPE_CHECKING, assert_never - -import structlog - -from autoskillit.core import ( - ChannelConfirmation, - CliSubtype, - FailureRecord, - RetryReason, - SessionOutcome, - SkillResult, - TerminationReason, - ValidatedAddDir, - WriteBehaviorSpec, - claude_code_project_dir, - get_logger, - is_git_worktree, - load_yaml, - pkg_root, - temp_dir_display_str, -) -from autoskillit.execution._headless_scan import _scan_jsonl_write_paths -from autoskillit.execution.clone_guard import ( - check_and_revert_clone_contamination, - is_worktree_skill, - snapshot_clone_state, -) -from autoskillit.execution.commands import build_full_headless_cmd, build_headless_resume_cmd -from autoskillit.execution.process import _marker_is_standalone -from autoskillit.execution.recording import RecordingSubprocessRunner -from autoskillit.execution.session import ( - ClaudeSessionResult, - _check_expected_patterns, - _compute_outcome, - _compute_success, - _normalize_subtype, - _truncate, - parse_session_result, -) - -if TYPE_CHECKING: - from autoskillit.config import ( - AutomationConfig, - ) - from autoskillit.core import AuditLog, SubprocessResult, SubprocessRunner - from autoskillit.pipeline.context import ( - ToolContext, - ) - -logger = get_logger(__name__) - -# Subtypes eligible for Channel B drain-race recovery. -# -# These are the failure subtypes that can arise when Claude Code defers ``type=result`` -# until all background agents finish. The deferred record is never flushed if the -# process tree is killed after Channel B fires on the session JSONL marker. -# -# TIMEOUT is excluded — it indicates a genuine time limit breach, not a drain-race. -# UNKNOWN is excluded — it indicates unrecognised CLI behaviour, not a missing record. -_CHANNEL_B_RECOVERABLE_SUBTYPES: frozenset[CliSubtype] = frozenset( - {CliSubtype.UNPARSEABLE, CliSubtype.EMPTY_OUTPUT} -) - -_PATH_CAPTURE: re.Pattern[str] = re.compile(r"^(\w+)\\s\*=\\s\*/.+") - - -def _session_log_dir(cwd: str) -> Path: - """Derive Claude Code session log directory from project cwd. - - Pre-creates the directory if absent so Channel B always has a directory - to poll. Without this, a fresh clone path whose encoded project dir - doesn't exist yet causes ``_session_log_monitor`` to burn its entire - phase-1 timeout absorbing ``OSError``, ultimately producing a false - ``EMPTY_OUTPUT`` retry. - """ - log_dir = claude_code_project_dir(cwd) - logger.info("session_log_dir_computed", path=str(log_dir), cwd=cwd) - if not log_dir.exists(): - logger.info("session_log_dir_precreating", path=str(log_dir), cwd=cwd) - try: - log_dir.mkdir(parents=True, exist_ok=True) - except OSError: - logger.warning("session_log_dir_mkdir_failed", path=str(log_dir), cwd=cwd) - raise - return log_dir - - -def _capture_failure( - skill_command: str, - exit_code: int, - subtype: str, - needs_retry: bool, - retry_reason: str, - stderr: str, - audit: AuditLog | None, -) -> None: - """Record a failure in the audit log. No-op if skill_command is empty or audit is None.""" - if not skill_command or audit is None: - return - audit.record_failure( - FailureRecord( - timestamp=datetime.now(UTC).isoformat(), - skill_command=skill_command, - exit_code=exit_code, - subtype=subtype, - needs_retry=needs_retry, - retry_reason=retry_reason, - stderr=stderr, - ) - ) - - -def _recover_from_separate_marker( - session: ClaudeSessionResult, - completion_marker: str, -) -> ClaudeSessionResult | None: - """Attempt recovery when the model emitted the completion marker as a standalone - final message rather than inline with its substantive output. - - Returns a reconstructed ClaudeSessionResult whose result field contains the - combined assistant message content (including the marker), or None if recovery - is not possible (no assistant content, or no substantive content beyond the marker). - """ - if not session.assistant_messages: - return None - if not any( - _marker_is_standalone(msg, completion_marker) for msg in session.assistant_messages - ): - return None - combined = "\n\n".join(session.assistant_messages) - stripped = combined.replace(completion_marker, "").strip() - if not stripped: - return None # only the marker exists — genuine failure, do not recover - logger.warning( - "completion_marker_in_separate_message", - recovery="rebuilding result from assistant_messages", - ) - return dataclasses.replace(session, result=combined) - - -def _recover_block_from_assistant_messages( - session: ClaudeSessionResult, - expected_output_patterns: Sequence[str], -) -> ClaudeSessionResult | None: - """When session.result lacks expected_output_patterns (drain-race condition - on either channel), attempt to find the patterns in session.assistant_messages. - If found, return a new ClaudeSessionResult with result reconstructed from - assistant_messages. Return None if patterns cannot be found there either. - """ - if not session.assistant_messages or not expected_output_patterns: - return None - combined = "\n\n".join(session.assistant_messages) - if not _check_expected_patterns(combined, expected_output_patterns): - return None - logger.warning( - "pattern_recovered_from_assistant_messages", - patterns=list(expected_output_patterns), - ) - # Preserve any content already drained into session.result before appending - # the recovered assistant_messages block. - recovered = (session.result + "\n\n" + combined) if session.result else combined - return dataclasses.replace(session, result=recovered) - - -def _synthesize_from_write_artifacts( - session: ClaudeSessionResult, - expected_output_patterns: list[str], - write_call_count: int, -) -> ClaudeSessionResult | None: - """Synthesize missing structured output tokens from Write/Edit tool_use file_path data. - - When the session has write evidence (write_call_count >= 1) and expected_output_patterns - contain path-capture patterns (e.g., ``plan_path\\s*=\\s*/.+``), scan tool_uses for - Write/Edit entries with absolute file_path values. For each pattern whose token name can - be extracted, inject ``{token_name} = {file_path}`` into session.result so that - _compute_outcome sees the token as if the model had emitted it. - - Returns a new ClaudeSessionResult with the injected line prepended to result, or None if - synthesis is not possible (no matching file_path, no path-capture patterns, or pattern - already satisfied). - """ - if write_call_count == 0: - return None - - # Only synthesize for path-capture patterns (token_name\s*=\s*/.+). - # Non-path patterns (verdict=, merged=) must remain text-compliance-only. - - synthesized_lines: list[str] = [] - for pattern in expected_output_patterns: - m = _PATH_CAPTURE.match(pattern) - if not m: - continue - token_name = m.group(1) - # Skip if the pattern is already satisfied in the current result. - if re.search(pattern, session.result): - continue - # Collect ALL absolute Write/Edit paths; use the LAST one (final deliverable). - # Multi-artifact skills write intermediate files first, final deliverable last. - candidate_paths = [ - t.get("file_path", "") - for t in session.tool_uses - if t.get("name") in {"Write", "Edit"} and t.get("file_path", "").startswith("/") - ] - if candidate_paths: - synthesized_lines.append(f"{token_name} = {candidate_paths[-1]}") - - if not synthesized_lines: - return None - - injected = "\n".join(synthesized_lines) + "\n" + session.result - return dataclasses.replace(session, result=injected) - - -def _extract_missing_token_hints( - stdout: str, - expected_output_patterns: Sequence[str], -) -> list[tuple[str, str]]: - """Extract (token_name, write_path) pairs for patterns missing from the result. - - Parses raw NDJSON stdout to find Write/Edit tool_use file_path entries, - then matches them against path-capture patterns that are NOT satisfied in - the result text. Returns the hints needed to build the nudge prompt. - """ - session = parse_session_result(stdout) - hints: list[tuple[str, str]] = [] - - for pattern in expected_output_patterns: - m = _PATH_CAPTURE.match(pattern) - if not m: - continue - token_name = m.group(1) - # Skip if already satisfied - if re.search(pattern, session.result): - continue - # Collect absolute Write/Edit paths; use the LAST one (final deliverable) - candidate_paths = [ - t.get("file_path", "") - for t in session.tool_uses - if t.get("name") in {"Write", "Edit"} and t.get("file_path", "").startswith("/") - ] - if candidate_paths: - hints.append((token_name, candidate_paths[-1])) - - return hints - - -_NUDGE_TIMEOUT: float = 60.0 - - -async def _attempt_contract_nudge( - skill_result: SkillResult, - subprocess_result: SubprocessResult, - expected_output_patterns: Sequence[str], - completion_marker: str, - cwd: str, - runner: SubprocessRunner, -) -> SkillResult | None: - """Attempt a lightweight resume nudge to recover missing structured output tokens. - - When ``_build_skill_result`` returns CONTRACT_RECOVERY, the model wrote the artifact - but omitted the structured output token. Instead of a full retry, resume the same - session with a short feedback prompt asking the model to emit the missing token. - - Returns a patched SkillResult(success=True) on success, or None to indicate the - nudge failed and the caller should fall through to the original CONTRACT_RECOVERY path. - """ - hints = _extract_missing_token_hints(subprocess_result.stdout, expected_output_patterns) - if not hints: - logger.debug("nudge_skip_no_hints") - return None - - # Build the feedback prompt - token_lines = "\n".join(f"{name} = {path}" for name, path in hints) - prompt = ( - "You completed your task and wrote the output file, but you omitted the " - "required structured output token in your final text response.\n\n" - f"Please emit ONLY the following (no other text):\n" - f"{token_lines}\n" - f"{completion_marker}" - ) - - spec = build_headless_resume_cmd( - resume_session_id=skill_result.session_id, - prompt=prompt, - output_format="json", - ) - - try: - nudge_result = await runner( - spec.cmd, - cwd=Path(cwd), - timeout=_NUDGE_TIMEOUT, - env=spec.env, - ) - except OSError: - logger.debug("nudge_runner_failed", exc_info=True) - return None - except Exception: - logger.warning("nudge_runner_failed_unexpected", exc_info=True) - return None - - # Parse the nudge response and check for the missing patterns - nudge_session = parse_session_result(nudge_result.stdout) - combined_result = skill_result.result + "\n" + nudge_session.result - - if not _check_expected_patterns(combined_result, list(expected_output_patterns)): - logger.debug( - "nudge_patterns_not_found", - nudge_result_len=len(nudge_session.result), - ) - return None - - logger.info( - "nudge_recovery_success", - session_id=skill_result.session_id, - nudge_output_count=nudge_session.token_usage.get("output_tokens", 0) - if nudge_session.token_usage - else 0, - ) - return dataclasses.replace( - skill_result, - success=True, - result=combined_result, - subtype="success", - needs_retry=False, - retry_reason=RetryReason.NONE, - token_usage=_merge_token_usage(skill_result.token_usage, nudge_session.token_usage), - ) - - -def _merge_token_usage( - base: dict[str, object] | None, - nudge: dict[str, object] | None, -) -> dict[str, object] | None: - """Additively merge token usage dicts from main session and nudge.""" - if base is None: - return nudge - if nudge is None: - return base - merged = dict(base) - for key in ( - "input_tokens", - "output_tokens", - "cache_creation_input_tokens", - "cache_read_input_tokens", - ): - base_val = base.get(key, 0) - nudge_val = nudge.get(key, 0) - if isinstance(base_val, (int, float)) and isinstance(nudge_val, (int, float)): - merged[key] = base_val + nudge_val - return merged - - -def _resolve_model(step_model: str, config: AutomationConfig) -> str | None: - """Resolve model selection: config override > step > config default.""" - if config.model.override: - logger.debug("model_resolved", tier="override", model=config.model.override) - return config.model.override - if step_model: - logger.debug("model_resolved", tier="step", model=step_model) - return step_model - if config.model.default: - logger.debug("model_resolved", tier="default", model=config.model.default) - return config.model.default - logger.debug("model_resolved", tier="none", model=None) - return None - - -_WORKTREE_PATH_PATTERN: re.Pattern[str] = re.compile(r"^worktree_path\s*=\s*(.+)$", re.MULTILINE) - - -def _extract_worktree_path(assistant_messages: list[str]) -> str | None: - """Return the last absolute path emitted as worktree_path=.""" - last: str | None = None - for msg in assistant_messages: - m = _WORKTREE_PATH_PATTERN.search(msg) - if m: - candidate = m.group(1).strip() - if os.path.isabs(candidate): - last = candidate - return last - - -# Intentionally excluded: these tokens are handled by dedicated extractors -# (_WORKTREE_PATH_PATTERN for worktree_path; branch_name is used as a string, -# not for path-contamination checks). -_INTENTIONALLY_EXCLUDED_PATH_TOKENS: frozenset[str] = frozenset( - { - "worktree_path", - "branch_name", - } -) - - -def _build_path_token_set() -> frozenset[str]: - """Derive the set of file-path output token names from skill_contracts.yaml. - - This replaces the manually-maintained frozenset and ensures new skills added - to the contracts file are automatically included in path-contamination checks. - Falls back to an empty frozenset if the manifest is unavailable (e.g., in - test environments where the package is not installed). - - Filters outputs where type starts with "file_path" (covers both "file_path" - and "file_path_list"). The outputs section in skill_contracts.yaml is a list - of dicts with "name" and "type" keys — not a mapping. - - Loads the YAML directly via L0 core utilities to avoid an upward L1→L2 import. - """ - try: - manifest_path = pkg_root() / "recipe" / "skill_contracts.yaml" - manifest = load_yaml(manifest_path) - if not isinstance(manifest, dict): - logger.debug( - "skill_contracts.yaml is empty or non-dict; _OUTPUT_PATH_TOKENS will be empty" - ) - return frozenset() - result = ( - frozenset( - out["name"] - for skill_data in manifest.get("skills", {}).values() - for out in skill_data.get("outputs", []) - if isinstance(out, dict) and out.get("type", "").startswith("file_path") - ) - - _INTENTIONALLY_EXCLUDED_PATH_TOKENS - ) - logger.debug("_OUTPUT_PATH_TOKENS derived from contracts", count=len(result)) - return result - except FileNotFoundError: - logger.debug("skill_contracts.yaml not found; _OUTPUT_PATH_TOKENS will be empty") - return frozenset() - except Exception: - logger.warning("Failed to derive _OUTPUT_PATH_TOKENS from contracts YAML", exc_info=True) - return frozenset() - - -_OUTPUT_PATH_TOKENS: frozenset[str] = _build_path_token_set() - -_OUTPUT_PATH_PATTERN: re.Pattern[str] = ( - re.compile( - r"^(" + "|".join(re.escape(t) for t in sorted(_OUTPUT_PATH_TOKENS)) + r")\s*=\s*(.+)$", - re.MULTILINE, - ) - if _OUTPUT_PATH_TOKENS - else re.compile(r"(?!)") # never-matches sentinel when token set is empty -) - - -def _extract_output_paths(assistant_messages: list[str]) -> dict[str, str]: - """Extract structured output path tokens from session output.""" - paths: dict[str, str] = {} - for msg in assistant_messages: - for m in _OUTPUT_PATH_PATTERN.finditer(msg): - token, value = m.group(1), m.group(2).strip() - if os.path.isabs(value): - paths[token] = value - return paths - - -def _validate_output_paths( - extracted_paths: dict[str, str], - cwd: str, -) -> str | None: - """Return a diagnostic string if any path is outside cwd, else None.""" - if not os.path.isabs(cwd) or cwd == "/": - return None - cwd_prefix = cwd.rstrip("/") + "/" - violations = [] - for token, path in extracted_paths.items(): - if not path.startswith(cwd_prefix) and path != cwd.rstrip("/"): - violations.append(f"{token} '{path}' is outside session cwd '{cwd}'") - return "; ".join(violations) if violations else None - - -def _apply_budget_guard( - sr: SkillResult, - skill_command: str, - audit: AuditLog | None, - max_consecutive_retries: int, -) -> SkillResult: - """Override needs_retry to False when the consecutive-failure budget is exhausted. - - The audit log records the raw failure (needs_retry=True) before this guard - runs; the guard is a post-processing filter on the returned SkillResult only. - """ - if not sr.needs_retry or audit is None or not skill_command: - return sr - consecutive = audit.consecutive_failures(skill_command) - # current failure already recorded; consecutive count includes this attempt - if consecutive > max_consecutive_retries: - logger.warning( - "retry_budget_exhausted", - skill_command=skill_command, - consecutive_failures=consecutive, - max_consecutive_retries=max_consecutive_retries, - ) - return dataclasses.replace( - sr, - needs_retry=False, - retry_reason=RetryReason.BUDGET_EXHAUSTED, - ) - return sr - - -def _resolve_skill_session_id( - session: ClaudeSessionResult | None, - result: SubprocessResult, -) -> str: - """Return the best-available Claude session UUID. - - Precedence: stdout-parsed session_id (Channel A) > transport-resolved - session_id (process.py) > Channel B JSONL filename stem. - Returns "" only when all sources are empty. - """ - if session is not None and session.session_id: - return session.session_id - return result.session_id or result.channel_b_session_id - - -def _build_skill_result( - result: SubprocessResult, - completion_marker: str = "", - skill_command: str = "", - audit: AuditLog | None = None, - max_consecutive_retries: int = 3, - expected_output_patterns: Sequence[str] = (), - cwd: str = "", - write_behavior: WriteBehaviorSpec | None = None, -) -> SkillResult: - """Route SubprocessResult fields into the standard run_skill response.""" - branch = ( - "idle_stall" - if result.termination == TerminationReason.IDLE_STALL - else "stale" - if result.termination == TerminationReason.STALE - else "timed_out" - if result.termination == TerminationReason.TIMED_OUT - else "normal" - ) - logger.debug( - "build_skill_result_entry", - termination=str(result.termination), - returncode=result.returncode, - channel=str(result.channel_confirmation), - pid=result.pid, - stdout_len=len(result.stdout), - stderr_len=len(result.stderr), - branch=branch, - ) - if result.termination == TerminationReason.STALE: - # Attempt to recover from stdout before declaring stale failure. - stale_session = parse_session_result(result.stdout) - stale_returncode = result.returncode if result.returncode is not None else -1 - can_attempt_stale_recovery = ( - stale_session.subtype == CliSubtype.SUCCESS - and stale_session.result.strip() - and not stale_session.is_error - ) - if can_attempt_stale_recovery: - success = _compute_success( - stale_session, - stale_returncode, - TerminationReason.COMPLETED, - completion_marker=completion_marker, - channel_confirmation=result.channel_confirmation, - ) - if success: - logger.warning( - "Session went stale but stdout contained a valid result; recovering" - ) - return SkillResult( - success=True, - result=_truncate(stale_session.agent_result), - session_id=stale_session.session_id or _resolve_skill_session_id(None, result), - subtype="recovered_from_stale", - is_error=False, - exit_code=stale_returncode, - needs_retry=False, - retry_reason=RetryReason.NONE, - stderr=result.stderr if result.stderr else "", - token_usage=stale_session.token_usage, - ) - # No valid result in stdout — fall through to original stale response - _capture_failure( - skill_command, - exit_code=result.returncode if result.returncode is not None else -1, - subtype="stale", - needs_retry=True, - retry_reason=RetryReason.STALE, - stderr=result.stderr if result.stderr else "", - audit=audit, - ) - stale_sr = SkillResult( - success=False, - result=( - "Session went stale (no activity for configured threshold). " - "Partial progress may have been made. Retry to continue." - ), - session_id=_resolve_skill_session_id(None, result), - subtype="stale", - is_error=False, - exit_code=-1, - needs_retry=True, - retry_reason=RetryReason.STALE, - stderr="", - token_usage=None, - ) - return _apply_budget_guard(stale_sr, skill_command, audit, max_consecutive_retries) - - if result.termination == TerminationReason.IDLE_STALL: - _capture_failure( - skill_command, - exit_code=result.returncode if result.returncode is not None else -1, - subtype="idle_stall", - needs_retry=True, - retry_reason=RetryReason.STALE, - stderr=result.stderr if result.stderr else "", - audit=audit, - ) - logger.warning( - "Headless session killed: stdout idle for configured threshold (IDLE_STALL)" - ) - idle_sr = SkillResult( - success=False, - result=( - "Session killed: stdout idle for configured threshold (no output growth). " - "Partial progress may have been made. Retry to continue." - ), - session_id=_resolve_skill_session_id(None, result), - subtype="idle_stall", - is_error=True, - exit_code=-1, - needs_retry=True, - retry_reason=RetryReason.STALE, - stderr="", - token_usage=None, - ) - return _apply_budget_guard(idle_sr, skill_command, audit, max_consecutive_retries) - - if result.termination == TerminationReason.TIMED_OUT: - returncode = -1 - if result.stdout.strip(): - session = parse_session_result(result.stdout) - if session.subtype == CliSubtype.SUCCESS: - session = dataclasses.replace(session, subtype=CliSubtype.TIMEOUT, is_error=True) - else: - session = ClaudeSessionResult( - subtype=CliSubtype.TIMEOUT, - is_error=True, - result="", - session_id=_resolve_skill_session_id(None, result), - errors=[], - ) - else: - returncode = result.returncode if result.returncode is not None else -1 - session = parse_session_result(result.stdout) - - # Moved earlier: needed by synthesis recovery step before _compute_outcome. - write_call_count = sum(1 for t in session.tool_uses if t.get("name") in {"Write", "Edit"}) - - # ── Channel B drain-race recovery ────────────────────────────────────── - # When Channel B confirmed completion but stdout never received the - # type=result record (UNPARSEABLE / EMPTY_OUTPUT), the session completed - # but Claude Code deferred type=result until all background agents finished. - # If we killed the process tree after Channel B fired, the deferred record - # was never flushed to stdout. - # - # assistant_messages are accumulated from stdout NDJSON records of type - # "assistant" — these are written BEFORE the deferred type=result. If the - # completion marker is standalone in assistant_messages with substantive - # content, reconstruct the result and promote the session so downstream - # recovery paths and the Channel B bypass in _compute_success operate on - # valid state. - match result.channel_confirmation: - case ChannelConfirmation.CHANNEL_B if ( - session.subtype in _CHANNEL_B_RECOVERABLE_SUBTYPES and completion_marker - ): - cb_recovered = _recover_from_separate_marker(session, completion_marker) - if cb_recovered is not None: - original_subtype = session.subtype - session = dataclasses.replace( - cb_recovered, - subtype=CliSubtype.SUCCESS, - is_error=False, - ) - logger.warning( - "channel_b_drain_race_recovery", - original_subtype=str(original_subtype), - assistant_message_count=len(session.assistant_messages), - ) - case ChannelConfirmation.DIR_MISSING if ( - session.subtype in _CHANNEL_B_RECOVERABLE_SUBTYPES and completion_marker - ): - # Late-bind recovery: the directory may have been created by - # Claude Code during the run even though it was absent at - # monitor start. Attempt the same marker-based recovery as - # the CHANNEL_B arm. - cb_recovered = _recover_from_separate_marker(session, completion_marker) - if cb_recovered is not None: - original_subtype = session.subtype - session = dataclasses.replace( - cb_recovered, - subtype=CliSubtype.SUCCESS, - is_error=False, - ) - logger.warning( - "dir_missing_late_bind_recovery", - original_subtype=str(original_subtype), - assistant_message_count=len(session.assistant_messages), - ) - else: - logger.warning( - "dir_missing_late_bind_recovery_failed", - subtype=str(session.subtype), - assistant_message_count=len(session.assistant_messages), - ) - case ( - ChannelConfirmation.CHANNEL_B - | ChannelConfirmation.CHANNEL_A - | ChannelConfirmation.UNMONITORED - | ChannelConfirmation.DIR_MISSING - ): - pass # no drain-race recovery applicable - case _ as _unreachable_cc: - assert_never(_unreachable_cc) - - # Recovery is only valid for sessions that completed normally. - # For incomplete sessions (UNPARSEABLE, TIMEOUT, etc.), any Write calls were - # intermediate artifacts, not final deliverables. Recovery or synthesis on these - # sessions would fabricate success evidence for a session that never finished. - if session.session_complete: - # Recovery check: attempt before _compute_outcome so the recovered session - # is the input for outcome computation rather than the original. - if completion_marker: - recovered = _recover_from_separate_marker(session, completion_marker) - if recovered is not None: - session = recovered - - # Pattern recovery: when a drain-race occurs on either channel, expected_output_patterns - # content may only exist in assistant_messages. Attempt recovery so that _compute_success - # sees the block in session.result. - if ( - result.channel_confirmation != ChannelConfirmation.UNMONITORED - and expected_output_patterns - and not _check_expected_patterns(session.result.strip(), expected_output_patterns) - ): - pattern_recovered = _recover_block_from_assistant_messages( - session, expected_output_patterns - ) - if pattern_recovered is not None: - session = pattern_recovered - - # Artifact-aware synthesis: only for UNMONITORED sessions where - # _recover_block_from_assistant_messages is unavailable. For CHANNEL_A/B - # sessions, if the pattern was absent from assistant_messages the agent never - # emitted it — synthesis would fabricate a token the agent did not produce. - if ( - expected_output_patterns - and write_call_count >= 1 - and result.channel_confirmation == ChannelConfirmation.UNMONITORED - and not _check_expected_patterns(session.result.strip(), expected_output_patterns) - ): - artifact_recovered = _synthesize_from_write_artifacts( - session, list(expected_output_patterns), write_call_count - ) - if artifact_recovered is not None: - session = artifact_recovered - - outcome, retry_reason = _compute_outcome( - session, - returncode, - result.termination, - completion_marker, - channel_confirmation=result.channel_confirmation, - expected_output_patterns=expected_output_patterns, - ) - success = outcome == SessionOutcome.SUCCEEDED - needs_retry = outcome == SessionOutcome.RETRIABLE - - normalized_subtype = _normalize_subtype(session.subtype, outcome, session, completion_marker) - - # For adjudicated_failure with write evidence, record as retriable in the audit so - # the consecutive chain is intact for the budget guard inside the CONTRACT_RECOVERY gate. - # CONTRACT_RECOVERY failures are genuinely retriable (the gate promotes them), so - # recording needs_retry=True is architecturally correct. - _audit_needs_retry = needs_retry - _audit_retry_reason = retry_reason - if ( - not success - and not needs_retry - and normalized_subtype == "adjudicated_failure" - and write_call_count >= 1 - ): - _audit_needs_retry = True - _audit_retry_reason = RetryReason.CONTRACT_RECOVERY - - if not success or needs_retry: - _capture_failure( - skill_command, - exit_code=returncode, - subtype=normalized_subtype, - needs_retry=_audit_needs_retry, - retry_reason=_audit_retry_reason.value, - stderr=result.stderr if result.stderr else "", - audit=audit, - ) - - result_text = _truncate(session.agent_result) - if completion_marker: - result_text = result_text.replace(completion_marker, "").strip() - - extracted_worktree_path = _extract_worktree_path(session.assistant_messages) - - # Path contamination detection - path_contamination: str | None = None - if not cwd: - logger.debug("path_contamination_check_skipped", reason="cwd not provided") - else: - extracted_paths = _extract_output_paths(session.assistant_messages) - path_contamination = _validate_output_paths(extracted_paths, cwd) - if path_contamination: - logger.warning("path_contamination_detected", detail=path_contamination, cwd=cwd) - - write_path_warnings: list[str] = [] - if cwd: - write_path_warnings = _scan_jsonl_write_paths(result.stdout, cwd) - if write_path_warnings: - logger.warning( - "write_path_warnings_detected", - count=len(write_path_warnings), - cwd=cwd, - warnings=write_path_warnings[:5], - ) - - if path_contamination: - sr = SkillResult( - success=False, - result=result_text, - session_id=session.session_id or result.session_id, - subtype="path_contamination", - is_error=session.is_error, - exit_code=returncode, - needs_retry=True, - retry_reason=RetryReason.PATH_CONTAMINATION, - stderr=_truncate(result.stderr), - token_usage=session.token_usage, - worktree_path=extracted_worktree_path, - cli_subtype=session.subtype, - write_path_warnings=write_path_warnings, - write_call_count=write_call_count, - ) - else: - sr = SkillResult( - success=success, - result=result_text, - session_id=session.session_id or result.session_id, - subtype=normalized_subtype, - is_error=session.is_error, - exit_code=returncode, - needs_retry=needs_retry, - retry_reason=retry_reason, - stderr=_truncate(result.stderr), - token_usage=session.token_usage, - worktree_path=extracted_worktree_path, - cli_subtype=session.subtype, - write_path_warnings=write_path_warnings, - write_call_count=write_call_count, - kill_reason=result.kill_reason, - ) - sr = _apply_budget_guard(sr, skill_command, audit, max_consecutive_retries) - - # CONTRACT_RECOVERY gate: when the session was classified as adjudicated_failure but - # write evidence exists (write_call_count >= 1), the model wrote the artifact but - # omitted the structured output token — an emission omission, not a structural contract - # failure. Promote to RETRIABLE(CONTRACT_RECOVERY) so the pipeline can recover. - # The first _apply_budget_guard call skips CONTRACT_VIOLATION cases because - # needs_retry is False at that point. Re-apply budget_guard after promoting so that - # budget exhaustion can still cap CONTRACT_RECOVERY retries (diagram: CRG → BG). - if ( - not sr.success - and not sr.needs_retry - and sr.subtype == "adjudicated_failure" - and write_call_count >= 1 - ): - sr = dataclasses.replace( - sr, - needs_retry=True, - retry_reason=RetryReason.CONTRACT_RECOVERY, - ) - sr = _apply_budget_guard(sr, skill_command, audit, max_consecutive_retries) - - # Zero-write gate: demote success to retriable failure when a write-expected - # skill produced zero Edit/Write calls (silent degradation detection). - # Write expectation is resolved from skill_contracts.yaml via WriteBehaviorSpec. - if sr.success and sr.write_call_count == 0 and write_behavior is not None: - write_expected = False - if write_behavior.mode == "always": - write_expected = True - elif write_behavior.mode == "conditional" and write_behavior.expected_when: - write_expected = _check_expected_patterns( - sr.result, - write_behavior.expected_when, - ) - if write_expected: - sr = dataclasses.replace( - sr, - success=False, - subtype="zero_writes", - needs_retry=True, - retry_reason=RetryReason.ZERO_WRITES, - ) - - logger.debug( - "build_skill_result_exit", - success=sr.success, - subtype=sr.subtype, - needs_retry=sr.needs_retry, - retry_reason=str(sr.retry_reason), - is_error=sr.is_error, - result_len=len(sr.result), - write_call_count=sr.write_call_count, - ) - return sr - - -def _derive_step_name_from_skill_command(skill_command: str) -> str: - """Extract a recording step name from a skill command string. - - Examples: - "/autoskillit:smoke-task arg1" -> "smoke-task" - "/investigate foo" -> "investigate" - "/autoskillit:make-plan" -> "make-plan" - "" -> "" - """ - stripped = skill_command.strip() - if not stripped: - return "" - token = stripped.split()[0].lstrip("/") - if ":" in token: - token = token.rsplit(":", 1)[-1] - return token - - -async def run_headless_core( - skill_command: str, - cwd: str, - ctx: ToolContext, - *, - model: str = "", - step_name: str = "", - kitchen_id: str = "", - order_id: str = "", - add_dirs: Sequence[ValidatedAddDir] = (), - timeout: float | None = None, - stale_threshold: float | None = None, - idle_output_timeout: float | None = None, - expected_output_patterns: Sequence[str] = (), - write_behavior: WriteBehaviorSpec | None = None, - completion_marker: str = "", -) -> SkillResult: - """Shared headless runner used by run_skill. - - Does NOT check open_kitchen gate — callers in server.py are responsible. - Accepts explicit ToolContext so this module has no server.py dependency. - """ - cfg = ctx.config.run_skill - effective_marker = completion_marker or cfg.completion_marker - original_skill_command = skill_command - - if not step_name and isinstance(ctx.runner, RecordingSubprocessRunner): - step_name = _derive_step_name_from_skill_command(skill_command) - - with structlog.contextvars.bound_contextvars( - skill_command=original_skill_command[:100], - step_name=step_name or None, - ): - effective_plugin_dir = ctx.plugin_dir - resolved_model = _resolve_model(model, ctx.config) - spec = build_full_headless_cmd( - skill_command, - cwd=cwd, - completion_marker=effective_marker, - model=resolved_model, - plugin_dir=effective_plugin_dir, - output_format_value=cfg.output_format.value, - output_format_required_flags=cfg.output_format.required_cli_flags, - add_dirs=add_dirs, - exit_after_stop_delay_ms=cfg.exit_after_stop_delay_ms, - scenario_step_name=step_name, - temp_dir_relpath=temp_dir_display_str(ctx.config.workspace.temp_dir), - ) - - effective_timeout = timeout if timeout is not None else cfg.timeout - effective_stale = stale_threshold if stale_threshold is not None else cfg.stale_threshold - _raw_idle = ( - idle_output_timeout - if idle_output_timeout is not None - else float(cfg.idle_output_timeout) - ) - effective_idle: float | None = _raw_idle if _raw_idle > 0.0 else None - - logger.debug( - "run_headless_core_entry", - cwd=cwd, - resolved_model=resolved_model, - timeout=effective_timeout, - stale_threshold=effective_stale, - plugin_dir=str(effective_plugin_dir), - add_dirs=list(add_dirs) if add_dirs else None, - ) - - runner = ctx.runner - assert runner is not None, "No subprocess runner configured" - - linux_tracing_cfg = ctx.config.linux_tracing - _start_ts = datetime.now(UTC).isoformat() - _start_mono = time.monotonic() - - _clone_snapshot = None - if is_worktree_skill(original_skill_command) and not is_git_worktree(Path(cwd)): - _clone_snapshot = await snapshot_clone_state(cwd, runner) - - _result: SubprocessResult | None = None - try: - _result = await runner( - spec.cmd, - cwd=Path(cwd), - timeout=effective_timeout, - env=spec.env, - pty_mode=True, - session_log_dir=_session_log_dir(cwd), - completion_marker=effective_marker, - stale_threshold=effective_stale, - completion_drain_timeout=cfg.completion_drain_timeout, - linux_tracing_config=linux_tracing_cfg, - idle_output_timeout=effective_idle, - max_suppression_seconds=cfg.max_suppression_seconds, - ) - except Exception as exc: - logger.error("run_headless_core runner crashed", exc_info=True) - _exc_text = traceback.format_exc() - _log_dir = ctx.config.linux_tracing.log_dir - try: - # Deferred: autoskillit.execution.__init__ imports headless.py (L39-42); - # a top-level import of autoskillit.execution would be circular. - from autoskillit.execution import flush_session_log - - flush_session_log( - log_dir=_log_dir, - cwd=str(cwd), - kitchen_id=kitchen_id, - order_id=order_id, - session_id="", - pid=0, - skill_command=original_skill_command, - success=False, - subtype="crashed", - exit_code=-1, - start_ts=_start_ts, - proc_snapshots=None, - termination_reason="CRASHED", - exception_text=_exc_text, - ) - except Exception: - logger.debug("flush_session_log during crash failed", exc_info=True) - return SkillResult.crashed( - exception=exc, - skill_command=original_skill_command, - order_id=order_id, - ) - if _result is None: - return SkillResult.crashed( - exception=RuntimeError("runner() did not return a result"), - order_id=order_id, - ) - _elapsed = time.monotonic() - _start_mono - _end_ts = (datetime.fromisoformat(_start_ts) + timedelta(seconds=_elapsed)).isoformat() - result = dataclasses.replace( # type: ignore[arg-type] - _result, start_ts=_start_ts, end_ts=_end_ts, elapsed_seconds=_elapsed - ) - - audit_count_before = len(ctx.audit.get_report()) - skill_result = _build_skill_result( - result, - completion_marker=effective_marker, - skill_command=original_skill_command, - audit=ctx.audit, - expected_output_patterns=expected_output_patterns, - cwd=cwd, - write_behavior=write_behavior, - ) - - # CONTRACT NUDGE: lightweight resume recovery before full retry. - # Fires only when _build_skill_result returns CONTRACT_RECOVERY with a - # valid session_id (budget-exhausted cases have retry_reason=BUDGET_EXHAUSTED). - if ( - skill_result.retry_reason == RetryReason.CONTRACT_RECOVERY - and skill_result.needs_retry - and skill_result.session_id - ): - nudge_success = await _attempt_contract_nudge( - skill_result, - result, - expected_output_patterns, - effective_marker, - cwd, - runner, - ) - if nudge_success is not None: - skill_result = nudge_success - - _clone_reverted = False - if _clone_snapshot is not None: - skill_result, _clone_reverted = await check_and_revert_clone_contamination( - _clone_snapshot, - skill_result, - cwd, - runner, - ctx.audit, - skill_command=original_skill_command, - ) - - # Use monotonic elapsed_seconds — authoritative wall-clock timing set by time.monotonic() - # brackets in run_managed_async. Never re-derive from ISO strings (backward-clock risk). - timing_seconds: float = result.elapsed_seconds - - # Extract the audit record (if any) added by this session - new_audit_records = ctx.audit.get_report_as_dicts()[audit_count_before:] - audit_record = new_audit_records[0] if new_audit_records else None - - if result.proc_snapshots is not None or not skill_result.success or bool(step_name): - from autoskillit.execution.session_log import flush_session_log - - try: - flush_session_log( - log_dir=ctx.config.linux_tracing.log_dir, - cwd=cwd, - kitchen_id=kitchen_id, - order_id=order_id, - session_id=skill_result.session_id, - pid=result.pid, - skill_command=original_skill_command, - success=skill_result.success, - subtype=skill_result.subtype, - cli_subtype=skill_result.cli_subtype, - exit_code=skill_result.exit_code, - start_ts=result.start_ts, - end_ts=result.end_ts, - elapsed_seconds=result.elapsed_seconds, - termination_reason=result.termination.value, - kill_reason=skill_result.kill_reason.value, - snapshot_interval_seconds=ctx.config.linux_tracing.proc_interval, - proc_snapshots=result.proc_snapshots, - step_name=step_name, - token_usage=skill_result.token_usage, - timing_seconds=timing_seconds, - audit_record=audit_record, - write_path_warnings=skill_result.write_path_warnings, - write_call_count=skill_result.write_call_count, - clone_contamination_reverted=_clone_reverted, - tracked_comm=result.tracked_comm, - ) - except Exception: - logger.debug("session_log_flush_failed", exc_info=True) - - logger.debug( - "run_headless_core_exit", - success=skill_result.success, - needs_retry=skill_result.needs_retry, - subtype=skill_result.subtype, - session_id=skill_result.session_id, - ) - - if step_name: - try: - ctx.token_log.record( - step_name, - skill_result.token_usage, - start_ts=result.start_ts, - end_ts=result.end_ts, - elapsed_seconds=result.elapsed_seconds, - order_id=order_id, - ) - except Exception: - logger.debug("token_log_record_failed", exc_info=True) - return skill_result - - -class DefaultHeadlessExecutor: - """Concrete HeadlessExecutor backed by run_headless_core.""" - - def __init__(self, ctx: ToolContext) -> None: - self._ctx = ctx - - async def run( - self, - skill_command: str, - cwd: str, - *, - model: str = "", - step_name: str = "", - kitchen_id: str = "", - order_id: str = "", - add_dirs: Sequence[ValidatedAddDir] = (), - timeout: float | None = None, - stale_threshold: float | None = None, - idle_output_timeout: float | None = None, - expected_output_patterns: Sequence[str] = (), - write_behavior: WriteBehaviorSpec | None = None, - completion_marker: str = "", - ) -> SkillResult: - cfg = self._ctx.config.run_skill - effective_timeout = timeout if timeout is not None else cfg.timeout - effective_stale = stale_threshold if stale_threshold is not None else cfg.stale_threshold - return await run_headless_core( - skill_command, - cwd, - self._ctx, - model=model, - step_name=step_name, - kitchen_id=kitchen_id, - order_id=order_id, - add_dirs=add_dirs, - timeout=effective_timeout, - stale_threshold=effective_stale, - idle_output_timeout=idle_output_timeout, - expected_output_patterns=expected_output_patterns, - write_behavior=write_behavior, - completion_marker=completion_marker, - ) diff --git a/src/autoskillit/execution/headless/CLAUDE.md b/src/autoskillit/execution/headless/CLAUDE.md new file mode 100644 index 0000000000..661dde76f1 --- /dev/null +++ b/src/autoskillit/execution/headless/CLAUDE.md @@ -0,0 +1,18 @@ +# headless/ + +Headless Claude session orchestration — command prep, subprocess invocation, result construction. + +## Files + +| File | Purpose | +|------|---------| +| `__init__.py` | Main module: `run_headless_core()`, `DefaultHeadlessExecutor`, `_execute_claude_headless()` | +| `_headless_git.py` | Git LOC tracking: `_capture_git_head_sha()`, `_compute_loc_changed()` | +| `_headless_path_tokens.py` | Path-token extraction and output-path validation from assistant messages | +| `_headless_recovery.py` | Session recovery: `_recover_from_separate_marker`, `_synthesize_from_write_artifacts` | +| `_headless_result.py` | `SkillResult` construction: `_build_skill_result`, `_build_session_telemetry`, `_apply_budget_guard` | +| `_headless_scan.py` | `_scan_jsonl_write_paths()` — scans stdout JSONL for Write/Edit/Bash tool calls | + +## Architecture Notes + +The `__init__.py` IS the main module body (not a thin facade). It uses a deferred import for `flush_session_log` to avoid circular imports. `_execute_claude_headless` is the shared path for both `run_skill` (skill session) and `dispatch_food_truck` (fleet) flows. diff --git a/src/autoskillit/execution/headless/__init__.py b/src/autoskillit/execution/headless/__init__.py new file mode 100644 index 0000000000..e70c12225b --- /dev/null +++ b/src/autoskillit/execution/headless/__init__.py @@ -0,0 +1,898 @@ +"""Headless Claude Code session orchestration. + +IL-1 module (execution/). Owns the full lifecycle of a headless claude CLI session: +command preparation, subprocess invocation via the injected runner, and +SkillResult construction. + +Public API: + run_headless_core(skill_command, cwd, ctx, *, ...) -> SkillResult +""" + +from __future__ import annotations + +import dataclasses +import os +import time +import traceback +from collections.abc import Callable, Mapping, Sequence +from datetime import UTC, datetime, timedelta +from pathlib import Path +from typing import TYPE_CHECKING + +import anyio +import structlog + +from autoskillit.core import ( + CAMPAIGN_ID_ENV_VAR, + DISPATCH_ID_ENV_VAR, + FOOD_TRUCK_TOOL_TAGS_ENV_VAR, + KillReason, + ProviderOutcome, + RecipeIdentity, + RetryReason, + SessionCheckpoint, # noqa: F401, TC001 + SkillResult, + ValidatedAddDir, + WriteBehaviorSpec, + claude_code_project_dir, + collect_version_snapshot, + extract_skill_name, + get_logger, + is_feature_enabled, + is_git_worktree, + temp_dir_display_str, +) +from autoskillit.execution.clone_guard import ( + check_and_revert_clone_contamination, + is_worktree_skill, + snapshot_clone_state, +) +from autoskillit.execution.commands import ( + build_food_truck_cmd, + build_skill_session_cmd, +) +from autoskillit.execution.headless._headless_git import ( + _capture_git_head_sha, + _compute_loc_changed, +) +from autoskillit.execution.headless._headless_path_tokens import ( # noqa: F401 + _INTENTIONALLY_EXCLUDED_PATH_TOKENS, + _OUTPUT_PATH_PATTERN, + _OUTPUT_PATH_TOKENS, + _RECOVERABLE_PATH_TOKENS, + _WORKTREE_PATH_PATTERN, + _build_path_token_set, + _extract_output_paths, + _extract_worktree_path, + _validate_output_paths, +) +from autoskillit.execution.headless._headless_recovery import ( + _CHANNEL_B_RECOVERABLE_SUBTYPES, # noqa: F401 + _NUDGE_TIMEOUT, # noqa: F401 + _TOKEN_NAME_RE, # noqa: F401 + _attempt_contract_nudge, + _extract_missing_token_hints, # noqa: F401 + _is_path_capture_pattern, # noqa: F401 + _merge_token_usage, # noqa: F401 + _recover_block_from_assistant_messages, # noqa: F401 + _recover_from_separate_marker, # noqa: F401 + _synthesize_from_write_artifacts, # noqa: F401 +) +from autoskillit.execution.headless._headless_result import ( + _apply_budget_guard, # noqa: F401 + _build_error_path_telemetry, + _build_session_telemetry, + _build_skill_result, + _capture_failure, # noqa: F401 + _resolve_skill_session_id, # noqa: F401 +) +from autoskillit.execution.headless._headless_scan import _scan_jsonl_write_paths # noqa: F401 +from autoskillit.execution.recording import RecordingSubprocessRunner + +if TYPE_CHECKING: + from autoskillit.config import ( + AutomationConfig, + ) + from autoskillit.core import SubprocessResult + from autoskillit.execution.commands import ClaudeHeadlessCmd + from autoskillit.pipeline.context import ( + ToolContext, + ) + +__all__ = [ + "DefaultHeadlessExecutor", + "PostSessionMetrics", + "run_headless_core", +] + +logger = get_logger(__name__) + + +def _session_log_dir(cwd: str) -> Path: + """Derive Claude Code session log directory from project cwd. + + Pre-creates the directory if absent so Channel B always has a directory + to poll. Without this, a fresh clone path whose encoded project dir + doesn't exist yet causes ``_session_log_monitor`` to burn its entire + phase-1 timeout absorbing ``OSError``, ultimately producing a false + ``EMPTY_OUTPUT`` retry. + """ + log_dir = claude_code_project_dir(cwd) + logger.info("session_log_dir_computed", path=str(log_dir), cwd=cwd) + if not log_dir.exists(): + logger.info("session_log_dir_precreating", path=str(log_dir), cwd=cwd) + try: + log_dir.mkdir(parents=True, exist_ok=True) + except OSError: + logger.warning("session_log_dir_mkdir_failed", path=str(log_dir), cwd=cwd) + raise + return log_dir + + +def _resolve_model(step_model: str, config: AutomationConfig) -> str | None: + """Resolve model selection: config override > step > config default.""" + if config.model.override: + logger.debug("model_resolved", tier="override", model=config.model.override) + return config.model.override + if step_model: + logger.debug("model_resolved", tier="step", model=step_model) + return step_model + if config.model.default: + logger.debug("model_resolved", tier="default", model=config.model.default) + return config.model.default + logger.debug("model_resolved", tier="none", model=None) + return None + + +def _derive_step_name_from_skill_command(skill_command: str) -> str: + """Extract a recording step name from a skill command string. + + Examples: + "/autoskillit:smoke-task arg1" -> "smoke-task" + "/investigate foo" -> "investigate" + "/autoskillit:make-plan" -> "make-plan" + "" -> "" + """ + stripped = skill_command.strip() + if not stripped: + return "" + token = stripped.split()[0].lstrip("/") + if ":" in token: + token = token.rsplit(":", 1)[-1] + return token + + +def _resolve_skill_temp_dir(cwd: str, skill_command: str) -> Path | None: + name = extract_skill_name(skill_command) + if not name: + return None + return Path(cwd) / ".autoskillit" / "temp" / name + + +def _recursive_snapshot(directory: Path) -> set[str]: + """Recursively enumerate all files under directory as relative paths.""" + return { + str(Path(dp).relative_to(directory) / f) for dp, _, fns in os.walk(directory) for f in fns + } + + +@dataclasses.dataclass(frozen=True, slots=True) +class PostSessionMetrics: + loc_insertions: int + loc_deletions: int + effective_cwd: str + + +def _compute_post_session_metrics( + cwd: str, + pre_session_sha: str, + skill_result: SkillResult, +) -> PostSessionMetrics: + effective_cwd = skill_result.worktree_path or cwd + loc_ins, loc_del = _compute_loc_changed(effective_cwd, pre_session_sha) + return PostSessionMetrics( + loc_insertions=loc_ins, + loc_deletions=loc_del, + effective_cwd=effective_cwd, + ) + + +async def _execute_claude_headless( + spec: ClaudeHeadlessCmd, + cwd: str, + ctx: ToolContext, + *, + skill_command: str = "", + step_name: str = "", + kitchen_id: str = "", + caller_session_id: str = "", + order_id: str = "", + campaign_id: str = "", + dispatch_id: str = "", + project_dir: str = "", + timeout: float, + stale_threshold: float, + idle_output_timeout: float | None = None, + expected_output_patterns: Sequence[str] = (), + write_behavior: WriteBehaviorSpec | None = None, + completion_marker: str = "", + recipe_name: str = "", + recipe_content_hash: str = "", + recipe_composite_hash: str = "", + recipe_version: str = "", + on_spawn: Callable[[int, int], None] | None = None, + skip_clone_guard: bool = False, + readonly_skill: bool = False, + write_watch_dirs: Sequence[Path] = (), + provider_name: str = "", + provider_fallback_env: dict[str, str] | None = None, + provider_fallback_name: str = "", + provider_extras: Mapping[str, str] | None = None, + enable_deadline_extension: bool = False, + max_extension_seconds: float = 7200, +) -> SkillResult: + """Shared subprocess execution for headless Claude sessions. + + Accepts an already-built ClaudeHeadlessCmd and handles runner invocation, + exception handling, _build_skill_result, and session log flushing. + Used by both run_headless_core (leaf path) and + DefaultHeadlessExecutor.dispatch_food_truck (food truck path). + """ + campaign_id = campaign_id or os.environ.get(CAMPAIGN_ID_ENV_VAR, "") + dispatch_id = dispatch_id or os.environ.get(DISPATCH_ID_ENV_VAR, "") + + cfg = ctx.config.run_skill + if idle_output_timeout is not None: + _raw_idle = idle_output_timeout + else: + env_idle = os.environ.get("AUTOSKILLIT_IDLE_OUTPUT_TIMEOUT") + if env_idle is not None: + try: + _raw_idle = float(env_idle) + except ValueError: + logger.warning( + "AUTOSKILLIT_IDLE_OUTPUT_TIMEOUT: invalid float — falling back to config", + env_value=env_idle, + fallback=cfg.idle_output_timeout, + ) + _raw_idle = float(cfg.idle_output_timeout) + else: + _raw_idle = float(cfg.idle_output_timeout) + effective_idle: float | None = _raw_idle if _raw_idle > 0.0 else None + + current_provider_name: str = provider_name + fallback_activated: bool = False + remaining_attempts = ctx.config.providers.provider_retry_limit if provider_fallback_env else 0 + + runner = ctx.runner + if runner is None: + raise RuntimeError("No subprocess runner configured") + + linux_tracing_cfg = ctx.config.linux_tracing + _start_ts = datetime.now(UTC).isoformat() + _start_mono = time.monotonic() + _versions = collect_version_snapshot() + + _readonly_skill = readonly_skill + _clone_snapshot = None + if ( + not skip_clone_guard + and not is_git_worktree(Path(cwd)) + and (is_worktree_skill(skill_command) or _readonly_skill) + ): + _clone_snapshot = await snapshot_clone_state(cwd, runner) + + _watch_dirs: list[Path] = list(write_watch_dirs) if write_watch_dirs else [] + if not _watch_dirs: + _default = _resolve_skill_temp_dir(cwd, skill_command) + if _default: + _watch_dirs.append(_default) + + _temp_snapshots_pre: dict[Path, set[str]] = {} + for _wd in _watch_dirs: + if _wd.is_dir(): + try: + _temp_snapshots_pre[_wd] = _recursive_snapshot(_wd) + except OSError: + logger.warning("watch_dir_pre_scan_failed", watch_dir=str(_wd), exc_info=True) + _temp_snapshots_pre[_wd] = set() + + _pre_session_sha = _capture_git_head_sha(cwd) + _result: SubprocessResult | None = None + result: SubprocessResult + skill_result: SkillResult + while True: + try: + _result = await runner( + spec.cmd, + cwd=Path(cwd), + timeout=timeout, + env=spec.env, + pty_mode=True, + session_log_dir=_session_log_dir(cwd), + completion_marker=completion_marker, + stale_threshold=stale_threshold, + completion_drain_timeout=cfg.completion_drain_timeout, + linux_tracing_config=linux_tracing_cfg, + idle_output_timeout=effective_idle, + max_suppression_seconds=cfg.max_suppression_seconds, + on_pid_resolved=on_spawn, + enable_deadline_extension=enable_deadline_extension, + max_extension_seconds=max_extension_seconds, + ) + except Exception as exc: + logger.error("headless_runner_crashed", exc_info=True) + _exc_text = traceback.format_exc() + _log_dir = ctx.config.linux_tracing.log_dir + try: + from autoskillit.execution import flush_session_log + + flush_session_log( + log_dir=_log_dir, + cwd=str(cwd), + kitchen_id=kitchen_id, + caller_session_id=caller_session_id, + order_id=order_id, + campaign_id=campaign_id, + dispatch_id=dispatch_id, + project_dir=project_dir, + build_protected_campaign_ids=ctx.build_protected_campaign_ids, + session_id="", + pid=0, + skill_command=skill_command, + success=False, + subtype="crashed", + exit_code=-1, + start_ts=_start_ts, + proc_snapshots=None, + termination_reason="CRASHED", + exception_text=_exc_text, + versions=_versions, + provider_outcome=ProviderOutcome( + provider_used=current_provider_name, + fallback_activated=fallback_activated, + ), + recipe_identity=RecipeIdentity( + name=recipe_name, + content_hash=recipe_content_hash, + composite_hash=recipe_composite_hash, + version=recipe_version, + ), + max_sessions=ctx.config.linux_tracing.max_sessions, + telemetry=_build_error_path_telemetry(ctx.github_api_log), + ) + except Exception: + logger.debug("flush_session_log during crash failed", exc_info=True) + _crashed = SkillResult.crashed( + exception=exc, + skill_command=skill_command, + order_id=order_id, + ) + return dataclasses.replace( + _crashed, + provider=ProviderOutcome( + provider_used=current_provider_name, fallback_activated=fallback_activated + ), + ) + except BaseException: + logger.warning("headless_runner_cancelled", exc_info=True) + _exc_text = traceback.format_exc() + _log_dir = ctx.config.linux_tracing.log_dir + try: + from autoskillit.execution import flush_session_log + + with anyio.CancelScope(shield=True): + flush_session_log( + log_dir=_log_dir, + cwd=str(cwd), + kitchen_id=kitchen_id, + caller_session_id=caller_session_id, + order_id=order_id, + campaign_id=campaign_id, + dispatch_id=dispatch_id, + project_dir=project_dir, + build_protected_campaign_ids=ctx.build_protected_campaign_ids, + session_id="", + pid=0, + skill_command=skill_command, + success=False, + subtype="cancelled", + exit_code=-1, + start_ts=_start_ts, + proc_snapshots=None, + termination_reason="CANCELLED", + exception_text=_exc_text, + versions=_versions, + provider_outcome=ProviderOutcome( + provider_used=current_provider_name, + fallback_activated=fallback_activated, + ), + recipe_identity=RecipeIdentity( + name=recipe_name, + content_hash=recipe_content_hash, + composite_hash=recipe_composite_hash, + version=recipe_version, + ), + max_sessions=ctx.config.linux_tracing.max_sessions, + telemetry=_build_error_path_telemetry(ctx.github_api_log), + ) + except Exception: + logger.debug("flush_session_log during cancel failed", exc_info=True) + raise + _elapsed = time.monotonic() - _start_mono + _end_ts = (datetime.fromisoformat(_start_ts) + timedelta(seconds=_elapsed)).isoformat() + result = dataclasses.replace( # type: ignore[arg-type] + _result, start_ts=_start_ts, end_ts=_end_ts, elapsed_seconds=_elapsed + ) + + _fs_writes_detected = False + for _wd in _watch_dirs: + if _wd.is_dir(): + try: + _post = _recursive_snapshot(_wd) + except OSError: + logger.warning("watch_dir_post_scan_failed", watch_dir=str(_wd), exc_info=True) + _post = set() + _pre = _temp_snapshots_pre.get(_wd, set()) + if _post - _pre: + _fs_writes_detected = True + break + + audit_count_before = len(ctx.audit.get_report()) + skill_result = _build_skill_result( + result, + completion_marker=completion_marker, + skill_command=skill_command, + audit=ctx.audit, + expected_output_patterns=expected_output_patterns, + cwd=cwd, + write_behavior=write_behavior, + fs_writes_detected=_fs_writes_detected, + provider_used=current_provider_name, + ) + + if ( + skill_result.needs_retry + and skill_result.session_id + and skill_result.retry_reason + in (RetryReason.CONTRACT_RECOVERY, RetryReason.EARLY_STOP) + ): + nudge_success = await _attempt_contract_nudge( + skill_result, + result, + expected_output_patterns, + completion_marker, + cwd, + runner, + provider_extras=provider_extras, + retry_reason=skill_result.retry_reason, + ) + if nudge_success is not None: + skill_result = nudge_success + + _clone_reverted = False + if _clone_snapshot is not None: + skill_result, _clone_reverted = await check_and_revert_clone_contamination( + _clone_snapshot, + skill_result, + cwd, + runner, + ctx.audit, + skill_command=skill_command, + readonly_skill=_readonly_skill, + ) + + if ( + skill_result.retry_reason in {RetryReason.STALE, RetryReason.BUDGET_EXHAUSTED} + and provider_fallback_env is not None + and remaining_attempts > 0 + and provider_name + and is_feature_enabled("providers", ctx.config.features) + ): + if not fallback_activated: + spec = dataclasses.replace(spec, env={**spec.env, **provider_fallback_env}) + if provider_fallback_name: + current_provider_name = provider_fallback_name + fallback_activated = True + remaining_attempts -= 1 + continue + break + + _metrics = _compute_post_session_metrics(cwd, _pre_session_sha, skill_result) + + timing_seconds: float = result.elapsed_seconds + + # Extract the audit record (if any) added by this session + new_audit_records = ctx.audit.get_report_as_dicts()[audit_count_before:] + audit_record = new_audit_records[0] if new_audit_records else None + + if ( + result.proc_snapshots is not None + or not skill_result.success + or bool(step_name) + or skill_result.token_usage is not None + ): + from autoskillit.execution.session_log import flush_session_log + + try: + flush_session_log( + log_dir=ctx.config.linux_tracing.log_dir, + cwd=cwd, + kitchen_id=kitchen_id, + caller_session_id=caller_session_id, + order_id=order_id, + campaign_id=campaign_id, + dispatch_id=dispatch_id, + project_dir=project_dir, + build_protected_campaign_ids=ctx.build_protected_campaign_ids, + session_id=skill_result.session_id, + pid=result.pid, + skill_command=skill_command, + success=skill_result.success, + subtype=skill_result.subtype, + cli_subtype=skill_result.cli_subtype, + exit_code=skill_result.exit_code, + start_ts=result.start_ts, + end_ts=result.end_ts, + elapsed_seconds=result.elapsed_seconds, + termination_reason=result.termination.value, + kill_reason=skill_result.kill_reason.value, + snapshot_interval_seconds=ctx.config.linux_tracing.proc_interval, + proc_snapshots=result.proc_snapshots, + step_name=step_name, + telemetry=_build_session_telemetry( + skill_result=skill_result, + timing_seconds=timing_seconds, + audit_record=audit_record, + github_api_log=ctx.github_api_log, + loc_insertions=_metrics.loc_insertions, + loc_deletions=_metrics.loc_deletions, + ), + write_path_warnings=skill_result.write_path_warnings, + write_call_count=skill_result.write_call_count, + clone_contamination_reverted=_clone_reverted, + tracked_comm=result.tracked_comm, + orphaned_tool_result=result.orphaned_tool_result, + raw_stdout=result.stdout + if ( + not skill_result.success or skill_result.kill_reason != KillReason.NATURAL_EXIT + ) + else "", + last_stop_reason=skill_result.last_stop_reason, + versions=_versions, + provider_outcome=ProviderOutcome( + provider_used=current_provider_name, + fallback_activated=fallback_activated, + ), + recipe_identity=RecipeIdentity( + name=recipe_name, + content_hash=recipe_content_hash, + composite_hash=recipe_composite_hash, + version=recipe_version, + ), + max_sessions=ctx.config.linux_tracing.max_sessions, + ) + except Exception: + logger.debug("session_log_flush_failed", exc_info=True) + + logger.debug( + "headless_session_exit", + success=skill_result.success, + needs_retry=skill_result.needs_retry, + subtype=skill_result.subtype, + session_id=skill_result.session_id, + ) + + from autoskillit.execution.session_log import _resolve_session_label + + _token_label = _resolve_session_label(step_name, dispatch_id) + try: + ctx.token_log.record( + _token_label, + skill_result.token_usage, + start_ts=result.start_ts, + end_ts=result.end_ts, + elapsed_seconds=result.elapsed_seconds, + order_id=order_id, + loc_insertions=_metrics.loc_insertions, + loc_deletions=_metrics.loc_deletions, + ) + except Exception: + logger.debug("token_log_record_failed", exc_info=True) + skill_result = dataclasses.replace( + skill_result, + provider=ProviderOutcome( + provider_used=current_provider_name, fallback_activated=fallback_activated + ), + ) + return skill_result + + +async def run_headless_core( + skill_command: str, + cwd: str, + ctx: ToolContext, + *, + model: str = "", + step_name: str = "", + kitchen_id: str = "", + order_id: str = "", + campaign_id: str = "", + dispatch_id: str = "", + project_dir: str = "", + add_dirs: Sequence[ValidatedAddDir] = (), + timeout: float | None = None, + stale_threshold: float | None = None, + idle_output_timeout: float | None = None, + expected_output_patterns: Sequence[str] = (), + write_behavior: WriteBehaviorSpec | None = None, + completion_marker: str = "", + recipe_name: str = "", + recipe_content_hash: str = "", + recipe_composite_hash: str = "", + recipe_version: str = "", + allowed_write_prefix: str = "", + readonly_skill: bool = False, + write_watch_dirs: Sequence[Path] = (), + provider_extras: Mapping[str, str] | None = None, + profile_name: str = "", + provider_name: str = "", + provider_fallback_env: dict[str, str] | None = None, + provider_fallback_name: str = "", + resume_session_id: str = "", + resume_checkpoint: SessionCheckpoint | None = None, +) -> SkillResult: + """Shared headless runner used by run_skill. + + Does NOT check open_kitchen gate — callers in server.py are responsible. + Accepts explicit ToolContext so this module has no server.py dependency. + """ + cfg = ctx.config.run_skill + effective_marker = completion_marker or cfg.completion_marker + original_skill_command = skill_command + + if not step_name and isinstance(ctx.runner, RecordingSubprocessRunner): + step_name = _derive_step_name_from_skill_command(skill_command) + + with structlog.contextvars.bound_contextvars( + skill_command=original_skill_command[:100], + step_name=step_name or None, + ): + resolved_model = _resolve_model(model, ctx.config) + spec = build_skill_session_cmd( + skill_command, + cwd=cwd, + completion_marker=effective_marker, + model=resolved_model, + plugin_source=ctx.plugin_source, + output_format=cfg.output_format, + add_dirs=add_dirs, + exit_after_stop_delay_ms=cfg.exit_after_stop_delay_ms, + scenario_step_name=step_name, + temp_dir_relpath=temp_dir_display_str(ctx.config.workspace.temp_dir), + allowed_write_prefix=allowed_write_prefix, + provider_extras=provider_extras, + profile_name=profile_name, + resume_session_id=resume_session_id, + resume_checkpoint=resume_checkpoint, + ) + + effective_timeout = timeout if timeout is not None else cfg.timeout + effective_stale = stale_threshold if stale_threshold is not None else cfg.stale_threshold + + logger.debug( + "run_headless_core_entry", + cwd=cwd, + resolved_model=resolved_model, + timeout=effective_timeout, + stale_threshold=effective_stale, + plugin_source=repr(ctx.plugin_source), + add_dirs=list(add_dirs) if add_dirs else None, + ) + + return await _execute_claude_headless( + spec, + cwd, + ctx, + skill_command=original_skill_command, + step_name=step_name, + kitchen_id=kitchen_id, + order_id=order_id, + campaign_id=campaign_id, + dispatch_id=dispatch_id, + project_dir=project_dir, + timeout=float(effective_timeout), + stale_threshold=float(effective_stale), + idle_output_timeout=idle_output_timeout, + expected_output_patterns=expected_output_patterns, + write_behavior=write_behavior, + completion_marker=effective_marker, + recipe_name=recipe_name, + recipe_content_hash=recipe_content_hash, + recipe_composite_hash=recipe_composite_hash, + recipe_version=recipe_version, + readonly_skill=readonly_skill, + write_watch_dirs=write_watch_dirs, + provider_name=provider_name, + provider_fallback_env=provider_fallback_env, + provider_fallback_name=provider_fallback_name, + provider_extras=provider_extras, + ) + + +class DefaultHeadlessExecutor: + """Concrete HeadlessExecutor backed by run_headless_core.""" + + def __init__(self, ctx: ToolContext) -> None: + self._ctx = ctx + + async def run( + self, + skill_command: str, + cwd: str, + *, + model: str = "", + step_name: str = "", + kitchen_id: str = "", + order_id: str = "", + add_dirs: Sequence[ValidatedAddDir] = (), + timeout: float | None = None, + stale_threshold: float | None = None, + idle_output_timeout: float | None = None, + expected_output_patterns: Sequence[str] = (), + write_behavior: WriteBehaviorSpec | None = None, + completion_marker: str = "", + recipe_name: str = "", + recipe_content_hash: str = "", + recipe_composite_hash: str = "", + recipe_version: str = "", + allowed_write_prefix: str = "", + readonly_skill: bool = False, + write_watch_dirs: Sequence[Path] = (), + provider_extras: Mapping[str, str] | None = None, + profile_name: str = "", + provider_name: str = "", + provider_fallback_env: dict[str, str] | None = None, + provider_fallback_name: str = "", + resume_session_id: str = "", + resume_checkpoint: SessionCheckpoint | None = None, + ) -> SkillResult: + cfg = self._ctx.config.run_skill + effective_timeout = timeout if timeout is not None else cfg.timeout + effective_stale = stale_threshold if stale_threshold is not None else cfg.stale_threshold + return await run_headless_core( + skill_command, + cwd, + self._ctx, + model=model, + step_name=step_name, + kitchen_id=kitchen_id, + order_id=order_id, + add_dirs=add_dirs, + timeout=effective_timeout, + stale_threshold=effective_stale, + idle_output_timeout=idle_output_timeout, + expected_output_patterns=expected_output_patterns, + write_behavior=write_behavior, + completion_marker=completion_marker, + recipe_name=recipe_name, + recipe_content_hash=recipe_content_hash, + recipe_composite_hash=recipe_composite_hash, + recipe_version=recipe_version, + allowed_write_prefix=allowed_write_prefix, + readonly_skill=readonly_skill, + write_watch_dirs=write_watch_dirs, + provider_extras=provider_extras, + profile_name=profile_name, + provider_name=provider_name, + provider_fallback_env=provider_fallback_env, + provider_fallback_name=provider_fallback_name, + resume_session_id=resume_session_id, + resume_checkpoint=resume_checkpoint, + ) + + async def dispatch_food_truck( + self, + orchestrator_prompt: str, + cwd: str, + *, + completion_marker: str, + resume_session_id: str | None = None, + resume_checkpoint: SessionCheckpoint | None = None, + model: str = "", + step_name: str = "", + kitchen_id: str = "", + order_id: str = "", + campaign_id: str = "", + dispatch_id: str = "", + caller_session_id: str = "", + project_dir: str = "", + timeout: float | None = None, + stale_threshold: float | None = None, + idle_output_timeout: float | None = None, + env_extras: Mapping[str, str] | None = None, + requires_packs: Sequence[str] = (), + on_spawn: Callable[[int, int], None] | None = None, + allowed_write_prefix: str = "", + provider_name: str = "", + provider_fallback_env: dict[str, str] | None = None, + provider_fallback_name: str = "", + ) -> SkillResult: + cfg = self._ctx.config + resolved_model = _resolve_model(model, cfg) + fleet_cfg = cfg.fleet + + merged_extras: dict[str, str] = dict(env_extras) if env_extras else {} + if requires_packs: + if FOOD_TRUCK_TOOL_TAGS_ENV_VAR in merged_extras: + raise ValueError( + f"dispatch_food_truck: requires_packs and env_extras both specify " + f"{FOOD_TRUCK_TOOL_TAGS_ENV_VAR} — use requires_packs exclusively" + ) + merged_extras[FOOD_TRUCK_TOOL_TAGS_ENV_VAR] = ",".join(sorted(requires_packs)) + + fleet_idle = fleet_cfg.idle_output_timeout + if idle_output_timeout is not None: + merged_extras["AUTOSKILLIT_IDLE_OUTPUT_TIMEOUT"] = str(idle_output_timeout) + elif fleet_idle > 0: + merged_extras.setdefault("AUTOSKILLIT_IDLE_OUTPUT_TIMEOUT", str(fleet_idle)) + else: + idle_cfg_val = cfg.run_skill.idle_output_timeout + if idle_cfg_val > 0: + merged_extras.setdefault("AUTOSKILLIT_IDLE_OUTPUT_TIMEOUT", str(idle_cfg_val)) + + spec = build_food_truck_cmd( + orchestrator_prompt=orchestrator_prompt, + plugin_source=self._ctx.plugin_source, + cwd=cwd, + completion_marker=completion_marker, + resume_session_id=resume_session_id, + resume_checkpoint=resume_checkpoint, + model=resolved_model, + env_extras=merged_extras or None, + output_format=cfg.run_skill.output_format, + exit_after_stop_delay_ms=cfg.run_skill.exit_after_stop_delay_ms, + scenario_step_name=step_name, + temp_dir_relpath=temp_dir_display_str(cfg.workspace.temp_dir), + allowed_write_prefix=allowed_write_prefix, + ) + + effective_timeout = timeout if timeout is not None else fleet_cfg.default_timeout_sec + effective_stale = ( + stale_threshold if stale_threshold is not None else cfg.run_skill.stale_threshold + ) + effective_deadline_ext = fleet_cfg.enable_deadline_extension + effective_max_ext = float(fleet_cfg.max_extension_seconds) + + effective_idle_out: float | None = ( + idle_output_timeout + if idle_output_timeout is not None + else float(fleet_idle) + if fleet_idle > 0 + else None + ) + + return await _execute_claude_headless( + spec, + cwd, + self._ctx, + skill_command="", + step_name=step_name, + kitchen_id=kitchen_id, + caller_session_id=caller_session_id, + order_id=order_id, + campaign_id=campaign_id, + dispatch_id=dispatch_id, + project_dir=project_dir, + timeout=float(effective_timeout), + stale_threshold=float(effective_stale), + idle_output_timeout=effective_idle_out, + completion_marker=completion_marker, + on_spawn=on_spawn, + skip_clone_guard=True, + provider_name=provider_name, + provider_fallback_env=provider_fallback_env, + provider_fallback_name=provider_fallback_name, + enable_deadline_extension=effective_deadline_ext, + max_extension_seconds=effective_max_ext, + ) diff --git a/src/autoskillit/execution/headless/_headless_git.py b/src/autoskillit/execution/headless/_headless_git.py new file mode 100644 index 0000000000..2668fd7298 --- /dev/null +++ b/src/autoskillit/execution/headless/_headless_git.py @@ -0,0 +1,67 @@ +"""Git helpers for headless session LOC tracking. + +Extracted from headless.py to keep that module below the architectural line budget. +IL-1 module (execution/). +""" + +from __future__ import annotations + +import subprocess + +from autoskillit.core import get_logger + +logger = get_logger(__name__) + + +def _capture_git_head_sha(cwd: str) -> str: + """Return current HEAD SHA in cwd. Returns '' on any error (non-git dirs).""" + try: + result = subprocess.run( + ["git", "rev-parse", "HEAD"], + cwd=cwd, + capture_output=True, + text=True, + timeout=10, + ) + return result.stdout.strip() if result.returncode == 0 else "" + except Exception: + logger.debug("capture_git_head_sha_failed", cwd=cwd, exc_info=True) + return "" + + +def _parse_numstat(numstat_output: str) -> tuple[int, int]: + """Parse `git diff --numstat` output into (insertions, deletions). + + Binary file lines (-\\t-\\tfilename) are skipped. + """ + insertions = deletions = 0 + for line in numstat_output.splitlines(): + parts = line.split("\t", 2) + if len(parts) < 2: + continue + try: + insertions += int(parts[0]) + deletions += int(parts[1]) + except ValueError: + continue # binary file row: "-\t-\tfilename" + return insertions, deletions + + +def _compute_loc_changed(cwd: str, pre_sha: str) -> tuple[int, int]: + """Run git diff --numstat in cwd. Returns (0, 0) on any error.""" + if not pre_sha: + return 0, 0 + try: + result = subprocess.run( + ["git", "diff", "--numstat", pre_sha], + cwd=cwd, + capture_output=True, + text=True, + timeout=15, + ) + if result.returncode != 0: + return 0, 0 + return _parse_numstat(result.stdout) + except Exception: + logger.debug("compute_loc_changed_failed", cwd=cwd, pre_sha=pre_sha, exc_info=True) + return 0, 0 diff --git a/src/autoskillit/execution/headless/_headless_path_tokens.py b/src/autoskillit/execution/headless/_headless_path_tokens.py new file mode 100644 index 0000000000..c8e85df5cd --- /dev/null +++ b/src/autoskillit/execution/headless/_headless_path_tokens.py @@ -0,0 +1,160 @@ +"""Path-token extraction and validation for headless Claude session output.""" + +from __future__ import annotations + +import os + +import regex as re + +from autoskillit.core import get_logger, load_yaml, pkg_root + +logger = get_logger(__name__) + +_WORKTREE_PATH_PATTERN: re.Pattern[str] = re.compile(r"^worktree_path\s*=\s*(.+)$", re.MULTILINE) + + +def _extract_worktree_path(assistant_messages: list[str]) -> str | None: + """Return the last absolute path emitted as worktree_path=.""" + last: str | None = None + for msg in assistant_messages: + m = _WORKTREE_PATH_PATTERN.search(msg) + if m: + candidate = m.group(1).strip() + if os.path.isabs(candidate): + last = candidate + return last + + +# Intentionally excluded: these tokens are handled by dedicated extractors +# (_WORKTREE_PATH_PATTERN for worktree_path; branch_name is used as a string, +# not for path-contamination checks). +_INTENTIONALLY_EXCLUDED_PATH_TOKENS: frozenset[str] = frozenset( + { + "worktree_path", + "branch_name", + } +) + + +def _build_path_token_set() -> frozenset[str]: + """Derive the set of file-path output token names from skill_contracts.yaml. + + This replaces the manually-maintained frozenset and ensures new skills added + to the contracts file are automatically included in path-contamination checks. + Falls back to an empty frozenset if the manifest is unavailable (e.g., in + test environments where the package is not installed). + + Filters outputs where type starts with "file_path" (covers both "file_path" + and "file_path_list"). The outputs section in skill_contracts.yaml is a list + of dicts with "name" and "type" keys — not a mapping. + + Loads the YAML directly via IL-0 core utilities to avoid an upward IL-1→IL-2 import. + """ + try: + manifest_path = pkg_root() / "recipe" / "skill_contracts.yaml" + manifest = load_yaml(manifest_path) + if not isinstance(manifest, dict): + logger.debug( + "skill_contracts.yaml is empty or non-dict; _OUTPUT_PATH_TOKENS will be empty" + ) + return frozenset() + result = ( + frozenset( + out.get("name", "") + for skill_data in manifest.get("skills", {}).values() + for out in skill_data.get("outputs", []) + if isinstance(out, dict) + and out.get("name", "") + and out.get("type", "").startswith("file_path") + ) + - _INTENTIONALLY_EXCLUDED_PATH_TOKENS + ) + logger.debug("_OUTPUT_PATH_TOKENS derived from contracts", count=len(result)) + return result + except FileNotFoundError: + logger.debug("skill_contracts.yaml not found; _OUTPUT_PATH_TOKENS will be empty") + return frozenset() + except Exception: + logger.warning("Failed to derive _OUTPUT_PATH_TOKENS from contracts YAML", exc_info=True) + return frozenset() + + +_OUTPUT_PATH_TOKENS: frozenset[str] = _build_path_token_set() + + +def _build_recoverable_path_tokens() -> frozenset[str]: + """Derive token names eligible for write-artifact recovery from skill_contracts.yaml. + + Includes outputs where type starts with "file_path" OR equals "directory_path". + Broader than _OUTPUT_PATH_TOKENS (which excludes directory_path). Used by + _headless_recovery.py for synthesis and nudge classification. + + Loads the YAML directly via IL-0 core utilities to avoid an upward IL-1→IL-2 import. + """ + try: + manifest_path = pkg_root() / "recipe" / "skill_contracts.yaml" + manifest = load_yaml(manifest_path) + if not isinstance(manifest, dict): + return frozenset() + result = ( + frozenset( + out.get("name", "") + for skill_data in manifest.get("skills", {}).values() + for out in skill_data.get("outputs", []) + if isinstance(out, dict) + and out.get("name", "") + and ( + out.get("type", "").startswith("file_path") + or out.get("type") == "directory_path" + ) + ) + - _INTENTIONALLY_EXCLUDED_PATH_TOKENS + ) + logger.debug("_RECOVERABLE_PATH_TOKENS derived from contracts", count=len(result)) + return result + except FileNotFoundError: + logger.debug("skill_contracts.yaml not found; _RECOVERABLE_PATH_TOKENS will be empty") + return frozenset() + except Exception: + logger.warning( + "Failed to derive _RECOVERABLE_PATH_TOKENS from contracts YAML", exc_info=True + ) + return frozenset() + + +_RECOVERABLE_PATH_TOKENS: frozenset[str] = _build_recoverable_path_tokens() + +_OUTPUT_PATH_PATTERN: re.Pattern[str] = ( + re.compile( + r"^(" + "|".join(re.escape(t) for t in sorted(_OUTPUT_PATH_TOKENS)) + r")\s*=\s*(.+)$", + re.MULTILINE, + ) + if _OUTPUT_PATH_TOKENS + else re.compile(r"(?!)") # never-matches sentinel when token set is empty +) + + +def _extract_output_paths(assistant_messages: list[str]) -> dict[str, str]: + """Extract structured output path tokens from session output.""" + paths: dict[str, str] = {} + for msg in assistant_messages: + for m in _OUTPUT_PATH_PATTERN.finditer(msg): + token, value = m.group(1), m.group(2).strip() + if os.path.isabs(value): + paths[token] = value + return paths + + +def _validate_output_paths( + extracted_paths: dict[str, str], + cwd: str, +) -> str | None: + """Return a diagnostic string if any path is outside cwd, else None.""" + if not os.path.isabs(cwd) or cwd == "/": + return None + cwd_prefix = cwd.rstrip("/") + "/" + violations = [] + for token, path in extracted_paths.items(): + if not path.startswith(cwd_prefix) and path != cwd.rstrip("/"): + violations.append(f"{token} '{path}' is outside session cwd '{cwd}'") + return "; ".join(violations) if violations else None diff --git a/src/autoskillit/execution/headless/_headless_recovery.py b/src/autoskillit/execution/headless/_headless_recovery.py new file mode 100644 index 0000000000..b21d2f563c --- /dev/null +++ b/src/autoskillit/execution/headless/_headless_recovery.py @@ -0,0 +1,336 @@ +"""Recovery helpers for headless Claude session result reconstruction.""" + +from __future__ import annotations + +import dataclasses +from collections.abc import Mapping, Sequence +from pathlib import Path +from typing import TYPE_CHECKING + +import regex as re + +from autoskillit.core import CliSubtype, OutputFormat, RetryReason, SkillResult, get_logger +from autoskillit.execution.commands import build_headless_resume_cmd +from autoskillit.execution.headless._headless_path_tokens import _RECOVERABLE_PATH_TOKENS +from autoskillit.execution.process import _marker_is_standalone +from autoskillit.execution.session import ( + ClaudeSessionResult, + _check_expected_patterns, + parse_session_result, +) + +if TYPE_CHECKING: + from autoskillit.core import SubprocessResult, SubprocessRunner + +logger = get_logger(__name__) + +# Subtypes eligible for Channel B drain-race recovery. +# +# These are the failure subtypes that can arise when Claude Code defers ``type=result`` +# until all background agents finish. The deferred record is never flushed if the +# process tree is killed after Channel B fires on the session JSONL marker. +# +# TIMEOUT is excluded — it indicates a genuine time limit breach, not a drain-race. +# UNKNOWN is excluded — it indicates unrecognised CLI behaviour, not a missing record. +_CHANNEL_B_RECOVERABLE_SUBTYPES: frozenset[CliSubtype] = frozenset( + {CliSubtype.UNPARSEABLE, CliSubtype.EMPTY_OUTPUT} +) + +_TOKEN_NAME_RE: re.Pattern[str] = re.compile(r"^(\w+)") + +_NUDGE_TIMEOUT: float = 60.0 + + +def _is_path_capture_pattern(pattern: str) -> str | None: + """Return the token name if pattern is a path-capture pattern, else None. + + Classification uses outputs[].type metadata from skill_contracts.yaml rather than + the pattern string suffix format, so all path-capture patterns are covered regardless + of whether they end in /.+, \\S+, .+, or any other suffix. + """ + m = _TOKEN_NAME_RE.match(pattern) + if not m: + return None + token_name = m.group(1) + if token_name not in _RECOVERABLE_PATH_TOKENS: + return None + if "=" not in pattern[m.end() :]: + return None + return token_name + + +def _recover_from_separate_marker( + session: ClaudeSessionResult, + completion_marker: str, +) -> ClaudeSessionResult | None: + """Attempt recovery when the model emitted the completion marker as a standalone + final message rather than inline with its substantive output. + + Returns a reconstructed ClaudeSessionResult whose result field contains the + combined assistant message content (including the marker), or None if recovery + is not possible (no assistant content, or no substantive content beyond the marker). + """ + if not session.assistant_messages: + return None + if not any( + _marker_is_standalone(msg, completion_marker) for msg in session.assistant_messages + ): + return None + combined = "\n\n".join(session.assistant_messages) + stripped = combined.replace(completion_marker, "").strip() + if not stripped: + return None # only the marker exists — genuine failure, do not recover + logger.warning( + "completion_marker_in_separate_message", + recovery="rebuilding result from assistant_messages", + ) + return dataclasses.replace(session, result=combined) + + +def _recover_block_from_assistant_messages( + session: ClaudeSessionResult, + expected_output_patterns: Sequence[str], +) -> ClaudeSessionResult | None: + """When session.result lacks expected_output_patterns (drain-race condition + on either channel), attempt to find the patterns in session.assistant_messages. + If found, return a new ClaudeSessionResult with result reconstructed from + assistant_messages. Return None if patterns cannot be found there either. + """ + if not session.assistant_messages or not expected_output_patterns: + return None + combined = "\n\n".join(session.assistant_messages) + if not _check_expected_patterns(combined, expected_output_patterns): + return None + logger.warning( + "pattern_recovered_from_assistant_messages", + patterns=list(expected_output_patterns), + ) + # Preserve any content already drained into session.result before appending + # the recovered assistant_messages block. + recovered = (session.result + "\n\n" + combined) if session.result else combined + return dataclasses.replace(session, result=recovered) + + +def _synthesize_from_write_artifacts( + session: ClaudeSessionResult, + expected_output_patterns: list[str], + write_call_count: int, + fs_writes_detected: bool = False, +) -> ClaudeSessionResult | None: + """Synthesize missing structured output tokens from Write/Edit tool_use file_path data. + + When the session has write evidence (write_call_count >= 1) and expected_output_patterns + contain path-capture patterns (e.g., ``plan_path\\s*=\\s*/.+``), scan tool_uses for + Write/Edit entries with absolute file_path values. For each pattern whose token name can + be extracted, inject ``{token_name} = {file_path}`` into session.result so that + _compute_outcome sees the token as if the model had emitted it. + + Returns a new ClaudeSessionResult with the injected line prepended to result, or None if + synthesis is not possible (no matching file_path, no path-capture patterns, or pattern + already satisfied). + """ + if write_call_count == 0: + return None + + # Only synthesize for path-capture patterns (token_name\s*=\s*/.+). + # Non-path patterns (verdict=, merged=) must remain text-compliance-only. + + synthesized_lines: list[str] = [] + for pattern in expected_output_patterns: + token_name = _is_path_capture_pattern(pattern) + if not token_name: + continue + # Skip if the pattern is already satisfied in the current result. + if re.search(pattern, session.result): + continue + # Collect ALL absolute Write/Edit paths; use the LAST one (final deliverable). + # Multi-artifact skills write intermediate files first, final deliverable last. + candidate_paths = [ + t.get("file_path", "") + for t in session.tool_uses + if t.get("name") in {"Write", "Edit"} and t.get("file_path", "").startswith("/") + ] + if candidate_paths: + synthesized_lines.append(f"{token_name} = {candidate_paths[-1]}") + + if not synthesized_lines: + return None + + injected = "\n".join(synthesized_lines) + "\n" + session.result + return dataclasses.replace(session, result=injected) + + +def _extract_missing_token_hints( + stdout: str, + expected_output_patterns: Sequence[str], +) -> list[tuple[str, str]]: + """Extract (token_name, write_path) pairs for patterns missing from the result. + + Parses raw NDJSON stdout to find Write/Edit tool_use file_path entries, + then matches them against path-capture patterns that are NOT satisfied in + the result text. Returns the hints needed to build the nudge prompt. + """ + session = parse_session_result(stdout) + hints: list[tuple[str, str]] = [] + + for pattern in expected_output_patterns: + token_name = _is_path_capture_pattern(pattern) + if not token_name: + continue + # Skip if already satisfied + if re.search(pattern, session.result): + continue + # Collect absolute Write/Edit paths; use the LAST one (final deliverable) + candidate_paths = [ + t.get("file_path", "") + for t in session.tool_uses + if t.get("name") in {"Write", "Edit"} and t.get("file_path", "").startswith("/") + ] + if candidate_paths: + hints.append((token_name, candidate_paths[-1])) + + return hints + + +async def _attempt_contract_nudge( + skill_result: SkillResult, + subprocess_result: SubprocessResult, + expected_output_patterns: Sequence[str], + completion_marker: str, + cwd: str, + runner: SubprocessRunner, + *, + provider_extras: Mapping[str, str] | None = None, + retry_reason: RetryReason = RetryReason.CONTRACT_RECOVERY, +) -> SkillResult | None: + """Attempt a lightweight resume nudge to recover missing structured output tokens. + + When ``_build_skill_result`` returns CONTRACT_RECOVERY, the model wrote the artifact + but omitted the structured output token. Instead of a full retry, resume the same + session with a short feedback prompt asking the model to emit the missing token. + + When retry_reason is EARLY_STOP, the model produced substantive output but omitted + the completion marker. A simpler nudge prompt is used that bypasses the hints guard. + + Returns a patched SkillResult(success=True) on success, or None to indicate the + nudge failed and the caller should fall through to the original path. + """ + if retry_reason == RetryReason.EARLY_STOP: + prompt = ( + "Your response was complete but you omitted the required completion marker. " + f"Please emit ONLY the following text (nothing else):\n" + f"{completion_marker}" + ) + patterns_to_check: Sequence[str] = [] + else: + hints = _extract_missing_token_hints(subprocess_result.stdout, expected_output_patterns) + if not hints: + logger.debug("nudge_skip_no_hints") + return None + token_lines = "\n".join(f"{name} = {path}" for name, path in hints) + prompt = ( + "You completed your task and wrote the output file, but you omitted the " + "required structured output token in your final text response.\n\n" + f"Please emit ONLY the following (no other text):\n" + f"{token_lines}\n" + f"{completion_marker}" + ) + patterns_to_check = list(expected_output_patterns) + + spec = build_headless_resume_cmd( + resume_session_id=skill_result.session_id, + prompt=prompt, + output_format=OutputFormat.JSON, + env_extras=dict(provider_extras) if provider_extras else None, + ) + + try: + nudge_result = await runner( + spec.cmd, + cwd=Path(cwd), + timeout=_NUDGE_TIMEOUT, + env=spec.env, + ) + except OSError: + logger.debug("nudge_runner_failed", exc_info=True) + return None + except Exception: + logger.warning("nudge_runner_failed_unexpected", exc_info=True) + return None + + # Parse the nudge response and check for the missing patterns + nudge_session = parse_session_result(nudge_result.stdout) + combined_result = skill_result.result + "\n" + nudge_session.result + + if retry_reason == RetryReason.EARLY_STOP: + # For EARLY_STOP, success is determined by the marker being present + if completion_marker in nudge_session.result: + logger.info( + "nudge_recovery_success", + session_id=skill_result.session_id, + nudge_output_count=nudge_session.token_usage.get("output_tokens", 0) + if nudge_session.token_usage + else 0, + ) + return dataclasses.replace( + skill_result, + success=True, + result=combined_result, + subtype="success", + needs_retry=False, + retry_reason=RetryReason.NONE, + token_usage=_merge_token_usage( + skill_result.token_usage, nudge_session.token_usage + ), + ) + logger.debug( + "nudge_early_stop_marker_not_found", nudge_result_len=len(nudge_session.result) + ) + return None + + if not _check_expected_patterns(combined_result, patterns_to_check): + logger.debug( + "nudge_patterns_not_found", + nudge_result_len=len(nudge_session.result), + ) + return None + + logger.info( + "nudge_recovery_success", + session_id=skill_result.session_id, + nudge_output_count=nudge_session.token_usage.get("output_tokens", 0) + if nudge_session.token_usage + else 0, + ) + return dataclasses.replace( + skill_result, + success=True, + result=combined_result, + subtype="success", + needs_retry=False, + retry_reason=RetryReason.NONE, + token_usage=_merge_token_usage(skill_result.token_usage, nudge_session.token_usage), + ) + + +def _merge_token_usage( + base: dict[str, object] | None, + nudge: dict[str, object] | None, +) -> dict[str, object] | None: + """Additively merge token usage dicts from main session and nudge.""" + if base is None: + return nudge + if nudge is None: + return base + merged = dict(base) + for key in ( + "input_tokens", + "output_tokens", + "cache_creation_input_tokens", + "cache_read_input_tokens", + ): + base_val = base.get(key, 0) + nudge_val = nudge.get(key, 0) + if isinstance(base_val, (int, float)) and isinstance(nudge_val, (int, float)): + merged[key] = base_val + nudge_val + return merged diff --git a/src/autoskillit/execution/headless/_headless_result.py b/src/autoskillit/execution/headless/_headless_result.py new file mode 100644 index 0000000000..63291786b3 --- /dev/null +++ b/src/autoskillit/execution/headless/_headless_result.py @@ -0,0 +1,657 @@ +"""SkillResult construction and adjudication for headless Claude sessions.""" + +from __future__ import annotations + +import dataclasses +from collections.abc import Sequence +from datetime import UTC, datetime +from typing import TYPE_CHECKING, assert_never + +from autoskillit.core import ( + ChannelConfirmation, + CliSubtype, + FailureRecord, + InfraExitCategory, + InfraOutcome, + KillReason, + ProviderOutcome, + RetryReason, + SessionOutcome, + SessionTelemetry, + SkillResult, + TerminationReason, + WriteBehaviorSpec, + get_logger, + truncate_text, +) +from autoskillit.execution.headless._headless_path_tokens import ( + _extract_output_paths, + _extract_worktree_path, + _validate_output_paths, +) +from autoskillit.execution.headless._headless_recovery import ( + _CHANNEL_B_RECOVERABLE_SUBTYPES, + _recover_block_from_assistant_messages, + _recover_from_separate_marker, + _synthesize_from_write_artifacts, +) +from autoskillit.execution.headless._headless_scan import _scan_jsonl_write_paths +from autoskillit.execution.session._exit_classification import classify_infra_exit +from autoskillit.execution.session._session_content import _check_expected_patterns +from autoskillit.execution.session._session_model import ( + ClaudeSessionResult, + parse_session_result, +) +from autoskillit.execution.session._session_outcome import ( + _compute_outcome, + _compute_success, +) + +if TYPE_CHECKING: + from autoskillit.core import AuditLog, GitHubApiLog, SubprocessResult + +logger = get_logger(__name__) +_truncate = truncate_text + +__all__ = [ + "_build_error_path_telemetry", + "_build_session_telemetry", + "_capture_failure", + "_apply_budget_guard", + "_resolve_skill_session_id", + "_build_skill_result", +] + + +def _capture_failure( + skill_command: str, + exit_code: int, + subtype: str, + needs_retry: bool, + retry_reason: str, + stderr: str, + audit: AuditLog | None, +) -> None: + """Record a failure in the audit log. No-op if skill_command is empty or audit is None.""" + if not skill_command or audit is None: + return + audit.record_failure( + FailureRecord( + timestamp=datetime.now(UTC).isoformat(), + skill_command=skill_command, + exit_code=exit_code, + subtype=subtype, + needs_retry=needs_retry, + retry_reason=retry_reason, + stderr=stderr, + ) + ) + + +def _apply_budget_guard( + sr: SkillResult, + skill_command: str, + audit: AuditLog | None, + max_consecutive_retries: int, +) -> SkillResult: + """Override needs_retry to False when the consecutive-failure budget is exhausted. + + The audit log records the raw failure (needs_retry=True) before this guard + runs; the guard is a post-processing filter on the returned SkillResult only. + """ + if not sr.needs_retry or audit is None or not skill_command: + return sr + consecutive = audit.consecutive_failures(skill_command) + # current failure already recorded; consecutive count includes this attempt + if consecutive > max_consecutive_retries: + logger.warning( + "retry_budget_exhausted", + skill_command=skill_command, + consecutive_failures=consecutive, + max_consecutive_retries=max_consecutive_retries, + ) + return dataclasses.replace( + sr, + needs_retry=False, + retry_reason=RetryReason.BUDGET_EXHAUSTED, + ) + return sr + + +def _resolve_skill_session_id( + session: ClaudeSessionResult | None, + result: SubprocessResult, +) -> str: + """Return the best-available Claude session UUID. + + Precedence: stdout-parsed session_id (Channel A) > transport-resolved + session_id (process.py) > Channel B JSONL filename stem. + Returns "" only when all sources are empty. + """ + if session is not None and session.session_id: + return session.session_id + return result.session_id or result.channel_b_session_id + + +def _build_skill_result( + result: SubprocessResult, + completion_marker: str = "", + skill_command: str = "", + audit: AuditLog | None = None, + max_consecutive_retries: int = 3, + expected_output_patterns: Sequence[str] = (), + cwd: str = "", + write_behavior: WriteBehaviorSpec | None = None, + fs_writes_detected: bool = False, + *, + provider_used: str = "", +) -> SkillResult: + """Route SubprocessResult fields into the standard run_skill response.""" + branch = ( + "idle_stall" + if result.termination == TerminationReason.IDLE_STALL + else "stale" + if result.termination == TerminationReason.STALE + else "timed_out" + if result.termination == TerminationReason.TIMED_OUT + else "normal" + ) + logger.debug( + "build_skill_result_entry", + termination=str(result.termination), + returncode=result.returncode, + channel=str(result.channel_confirmation), + pid=result.pid, + stdout_len=len(result.stdout), + stderr_len=len(result.stderr), + branch=branch, + ) + if result.termination == TerminationReason.STALE: + # Attempt to recover from stdout before declaring stale failure. + stale_session = parse_session_result(result.stdout) + stale_returncode = result.returncode if result.returncode is not None else -1 + can_attempt_stale_recovery = ( + stale_session.subtype == CliSubtype.SUCCESS + and stale_session.result.strip() + and not stale_session.is_error + ) + if can_attempt_stale_recovery: + success = _compute_success( + stale_session, + stale_returncode, + TerminationReason.COMPLETED, + completion_marker=completion_marker, + channel_confirmation=result.channel_confirmation, + ) + if success: + logger.warning( + "Session went stale but stdout contained a valid result; recovering" + ) + return SkillResult( + success=True, + result=_truncate(stale_session.agent_result), + session_id=stale_session.session_id or _resolve_skill_session_id(None, result), + subtype="recovered_from_stale", + is_error=False, + exit_code=stale_returncode, + needs_retry=False, + retry_reason=RetryReason.NONE, + stderr=result.stderr if result.stderr else "", + token_usage=stale_session.token_usage, + last_stop_reason=stale_session.last_stop_reason, + provider=ProviderOutcome( + provider_used=provider_used, fallback_activated=False + ), + ) + # No valid result in stdout — fall through to original stale response + _capture_failure( + skill_command, + exit_code=result.returncode if result.returncode is not None else -1, + subtype="stale", + needs_retry=True, + retry_reason=RetryReason.STALE, + stderr=result.stderr if result.stderr else "", + audit=audit, + ) + stale_sr = SkillResult( + success=False, + result=( + "Session went stale (no activity for configured threshold). " + "Partial progress may have been made. Retry to continue." + ), + session_id=_resolve_skill_session_id(None, result), + subtype="stale", + is_error=False, + exit_code=-1, + needs_retry=True, + retry_reason=RetryReason.STALE, + stderr="", + token_usage=None, + provider=ProviderOutcome(provider_used=provider_used, fallback_activated=False), + ) + return _apply_budget_guard(stale_sr, skill_command, audit, max_consecutive_retries) + + if result.termination == TerminationReason.IDLE_STALL: + idle_session = parse_session_result(result.stdout) + idle_returncode = result.returncode if result.returncode is not None else -1 + can_attempt_idle_stall_recovery = ( + idle_session.subtype == CliSubtype.SUCCESS + and idle_session.result.strip() + and not idle_session.is_error + ) + if can_attempt_idle_stall_recovery: + success = _compute_success( + idle_session, + idle_returncode, + TerminationReason.COMPLETED, + completion_marker=completion_marker, + channel_confirmation=result.channel_confirmation, + ) + if success: + logger.warning( + "Session idle-stalled but stdout contained a valid result; recovering" + ) + return SkillResult( + success=True, + result=_truncate(idle_session.agent_result), + session_id=idle_session.session_id or _resolve_skill_session_id(None, result), + subtype="recovered_from_idle_stall", + is_error=False, + exit_code=idle_returncode, + needs_retry=False, + retry_reason=RetryReason.NONE, + stderr=result.stderr if result.stderr else "", + token_usage=idle_session.token_usage, + last_stop_reason=idle_session.last_stop_reason, + lifespan_started=idle_session.lifespan_started, + provider=ProviderOutcome( + provider_used=provider_used, fallback_activated=False + ), + ) + _capture_failure( + skill_command, + exit_code=result.returncode if result.returncode is not None else -1, + subtype="idle_stall", + needs_retry=True, + retry_reason=RetryReason.IDLE_STALL, + stderr=result.stderr if result.stderr else "", + audit=audit, + ) + logger.warning( + "Headless session killed: stdout idle for configured threshold (IDLE_STALL)" + ) + idle_sr = SkillResult( + success=False, + result=( + "Session killed: stdout idle for configured threshold (no output growth). " + "Partial progress may have been made. Retry to continue." + ), + session_id=idle_session.session_id or _resolve_skill_session_id(None, result), + subtype="idle_stall", + is_error=False, + exit_code=-1, + needs_retry=True, + retry_reason=RetryReason.IDLE_STALL, + stderr="", + token_usage=idle_session.token_usage, + lifespan_started=idle_session.lifespan_started, + provider=ProviderOutcome(provider_used=provider_used, fallback_activated=False), + ) + return _apply_budget_guard(idle_sr, skill_command, audit, max_consecutive_retries) + + if result.termination == TerminationReason.TIMED_OUT: + returncode = -1 + if result.stdout.strip(): + session = parse_session_result(result.stdout) + if session.subtype == CliSubtype.SUCCESS: + session = dataclasses.replace(session, subtype=CliSubtype.TIMEOUT, is_error=True) + else: + session = ClaudeSessionResult( + subtype=CliSubtype.TIMEOUT, + is_error=True, + result="", + session_id=_resolve_skill_session_id(None, result), + errors=[], + ) + else: + returncode = result.returncode if result.returncode is not None else -1 + session = parse_session_result(result.stdout) + + write_call_count = sum(1 for t in session.tool_uses if t.get("name") in {"Write", "Edit"}) + _has_write_evidence = write_call_count >= 1 or fs_writes_detected + + # Channel B drain-race: recover from assistant_messages if type=result was not flushed. + match result.channel_confirmation: + case ChannelConfirmation.CHANNEL_B if ( + session.subtype in _CHANNEL_B_RECOVERABLE_SUBTYPES and completion_marker + ): + cb_recovered = _recover_from_separate_marker(session, completion_marker) + if cb_recovered is not None: + original_subtype = session.subtype + session = dataclasses.replace( + cb_recovered, + subtype=CliSubtype.SUCCESS, + is_error=False, + ) + logger.warning( + "channel_b_drain_race_recovery", + original_subtype=str(original_subtype), + assistant_message_count=len(session.assistant_messages), + ) + case ChannelConfirmation.DIR_MISSING if ( + session.subtype in _CHANNEL_B_RECOVERABLE_SUBTYPES and completion_marker + ): + # Late-bind recovery: the directory may have been created by + # Claude Code during the run even though it was absent at + # monitor start. Attempt the same marker-based recovery as + # the CHANNEL_B arm. + cb_recovered = _recover_from_separate_marker(session, completion_marker) + if cb_recovered is not None: + original_subtype = session.subtype + session = dataclasses.replace( + cb_recovered, + subtype=CliSubtype.SUCCESS, + is_error=False, + ) + logger.warning( + "dir_missing_late_bind_recovery", + original_subtype=str(original_subtype), + assistant_message_count=len(session.assistant_messages), + ) + else: + logger.warning( + "dir_missing_late_bind_recovery_failed", + subtype=str(session.subtype), + assistant_message_count=len(session.assistant_messages), + ) + case ( + ChannelConfirmation.CHANNEL_B + | ChannelConfirmation.CHANNEL_A + | ChannelConfirmation.UNMONITORED + | ChannelConfirmation.DIR_MISSING + ): + pass # no drain-race recovery applicable + case _ as _unreachable_cc: + assert_never(_unreachable_cc) + + # Recovery is only valid for sessions that completed normally. + # For incomplete sessions (UNPARSEABLE, TIMEOUT, etc.), any Write calls were + # intermediate artifacts, not final deliverables. Recovery or synthesis on these + # sessions would fabricate success evidence for a session that never finished. + if session.session_complete: + # Recovery check: attempt before _compute_outcome so the recovered session + # is the input for outcome computation rather than the original. + if completion_marker: + recovered = _recover_from_separate_marker(session, completion_marker) + if recovered is not None: + session = recovered + + # Pattern recovery: when a drain-race occurs on either channel, expected_output_patterns + # content may only exist in assistant_messages. Attempt recovery so that _compute_success + # sees the block in session.result. + if ( + result.channel_confirmation != ChannelConfirmation.UNMONITORED + and expected_output_patterns + and not _check_expected_patterns(session.result.strip(), expected_output_patterns) + ): + pattern_recovered = _recover_block_from_assistant_messages( + session, expected_output_patterns + ) + if pattern_recovered is not None: + session = pattern_recovered + + # Artifact-aware synthesis: only for UNMONITORED sessions where + # _recover_block_from_assistant_messages is unavailable. For CHANNEL_A/B + # sessions, if the pattern was absent from assistant_messages the agent never + # emitted it — synthesis would fabricate a token the agent did not produce. + if ( + expected_output_patterns + and _has_write_evidence + and result.channel_confirmation == ChannelConfirmation.UNMONITORED + and not _check_expected_patterns(session.result.strip(), expected_output_patterns) + ): + artifact_recovered = _synthesize_from_write_artifacts( + session, list(expected_output_patterns), write_call_count, fs_writes_detected + ) + if artifact_recovered is not None: + session = artifact_recovered + + outcome, retry_reason = _compute_outcome( + session, + returncode, + result.termination, + completion_marker, + channel_confirmation=result.channel_confirmation, + expected_output_patterns=expected_output_patterns, + ) + success = outcome == SessionOutcome.SUCCEEDED + needs_retry = outcome == SessionOutcome.RETRIABLE + + infra_category = classify_infra_exit(session, result) + + # API error override: when the session failed due to an API infrastructure error + # (overload, 529, ECONNRESET), promote to RESUME so the orchestrator routes to + # on_context_limit instead of on_failure (partial progress may exist). + if not success and infra_category == InfraExitCategory.API_ERROR: + logger.info( + "api_error_override", + original_retry_reason=retry_reason.value, + promoted_to="resume", + ) + retry_reason = RetryReason.RESUME + needs_retry = True + + # Process kill override: external kills (SIGKILL/OOM, not autoskillit-initiated) + # route to RESUME so the orchestrator can attempt recovery. + # TIMED_OUT uses a synthetic returncode=-1 but is a wall-clock timeout (non-recoverable). + if ( + not success + and not needs_retry + and infra_category == InfraExitCategory.PROCESS_KILLED + and result.kill_reason == KillReason.NATURAL_EXIT + and result.termination != TerminationReason.TIMED_OUT + ): + retry_reason = RetryReason.RESUME + needs_retry = True + outcome = SessionOutcome.RETRIABLE + + normalized_subtype = session.normalize_subtype(outcome, completion_marker) + + # For adjudicated_failure + write evidence: record as retriable so the consecutive + # chain is intact for the CONTRACT_RECOVERY budget guard (genuinely retriable). + _audit_needs_retry = needs_retry + _audit_retry_reason = retry_reason + if ( + not success + and not needs_retry + and normalized_subtype == "adjudicated_failure" + and _has_write_evidence + ): + _audit_needs_retry = True + _audit_retry_reason = RetryReason.CONTRACT_RECOVERY + if retry_reason == RetryReason.EMPTY_OUTPUT and _has_write_evidence: + _audit_retry_reason = RetryReason.COMPLETED_NO_FLUSH + + if not success or needs_retry: + _capture_failure( + skill_command, + exit_code=returncode, + subtype=normalized_subtype, + needs_retry=_audit_needs_retry, + retry_reason=_audit_retry_reason.value, + stderr=result.stderr if result.stderr else "", + audit=audit, + ) + + result_text = _truncate(session.agent_result) + if completion_marker: + result_text = result_text.replace(completion_marker, "").strip() + + extracted_worktree_path = _extract_worktree_path(session.assistant_messages) + + # Path contamination detection + path_contamination: str | None = None + if not cwd: + logger.debug("path_contamination_check_skipped", reason="cwd not provided") + else: + extracted_paths = _extract_output_paths(session.assistant_messages) + path_contamination = _validate_output_paths(extracted_paths, cwd) + if path_contamination: + logger.warning("path_contamination_detected", detail=path_contamination, cwd=cwd) + + write_path_warnings: list[str] = [] + if cwd: + write_path_warnings = _scan_jsonl_write_paths(result.stdout, cwd) + if write_path_warnings: + logger.warning( + "write_path_warnings_detected", + count=len(write_path_warnings), + cwd=cwd, + warnings=write_path_warnings[:5], + ) + + if path_contamination: + sr = SkillResult( + success=False, + result=result_text, + session_id=session.session_id or result.session_id, + subtype="path_contamination", + is_error=session.is_error, + exit_code=returncode, + needs_retry=True, + retry_reason=RetryReason.PATH_CONTAMINATION, + stderr=_truncate(result.stderr), + token_usage=session.token_usage, + worktree_path=extracted_worktree_path, + cli_subtype=session.subtype, + write_path_warnings=write_path_warnings, + write_call_count=write_call_count, + fs_writes_detected=fs_writes_detected, + last_stop_reason=session.last_stop_reason, + lifespan_started=session.lifespan_started, + provider=ProviderOutcome(provider_used=provider_used, fallback_activated=False), + infra=InfraOutcome(exit_category=infra_category.value), + ) + else: + sr = SkillResult( + success=success, + result=result_text, + session_id=session.session_id or result.session_id, + subtype=normalized_subtype, + is_error=session.is_error, + exit_code=returncode, + needs_retry=needs_retry, + retry_reason=retry_reason, + stderr=_truncate(result.stderr), + token_usage=session.token_usage, + worktree_path=extracted_worktree_path, + cli_subtype=session.subtype, + write_path_warnings=write_path_warnings, + write_call_count=write_call_count, + fs_writes_detected=fs_writes_detected, + kill_reason=result.kill_reason, + last_stop_reason=session.last_stop_reason, + lifespan_started=session.lifespan_started, + provider=ProviderOutcome(provider_used=provider_used, fallback_activated=False), + infra=InfraOutcome(exit_category=infra_category.value), + ) + sr = _apply_budget_guard(sr, skill_command, audit, max_consecutive_retries) + + # CONTRACT_RECOVERY gate: when the session was classified as adjudicated_failure but + # write evidence exists, the model wrote the artifact but omitted the structured output + # token — promote to RETRIABLE(CONTRACT_RECOVERY). Re-apply budget_guard after + # promoting so budget exhaustion can still cap CONTRACT_RECOVERY retries. + # The first _apply_budget_guard skips this case because needs_retry is False then. + if ( + not sr.success + and not sr.needs_retry + and sr.subtype == "adjudicated_failure" + and _has_write_evidence + ): + sr = dataclasses.replace( + sr, + needs_retry=True, + retry_reason=RetryReason.CONTRACT_RECOVERY, + ) + sr = _apply_budget_guard(sr, skill_command, audit, max_consecutive_retries) + + # Zero-write gate: demote success to retriable failure when a write-expected + # skill produced zero Edit/Write calls (silent degradation detection). + # Write expectation is resolved from skill_contracts.yaml via WriteBehaviorSpec. + if sr.success and not _has_write_evidence and write_behavior is not None: + write_expected = False + if write_behavior.mode == "always": + write_expected = True + elif write_behavior.mode == "conditional" and write_behavior.expected_when: + write_expected = _check_expected_patterns( + sr.result, + write_behavior.expected_when, + ) + if write_expected: + sr = dataclasses.replace( + sr, + success=False, + subtype="zero_writes", + needs_retry=True, + retry_reason=RetryReason.ZERO_WRITES, + ) + + if sr.needs_retry and sr.retry_reason == RetryReason.EMPTY_OUTPUT and _has_write_evidence: + sr = dataclasses.replace( + sr, + subtype="completed_no_flush", + retry_reason=RetryReason.COMPLETED_NO_FLUSH, + ) + sr = _apply_budget_guard(sr, skill_command, audit, max_consecutive_retries) + + logger.debug( + "build_skill_result_exit", + success=sr.success, + subtype=sr.subtype, + needs_retry=sr.needs_retry, + retry_reason=str(sr.retry_reason), + is_error=sr.is_error, + result_len=len(sr.result), + write_call_count=sr.write_call_count, + ) + return sr + + +def _build_session_telemetry( + *, + skill_result: SkillResult, + timing_seconds: float | None, + audit_record: dict | None, + github_api_log: GitHubApiLog | None, + loc_insertions: int, + loc_deletions: int, +) -> SessionTelemetry: + _api_usage = ( + github_api_log.drain(skill_result.session_id) if github_api_log is not None else None + ) + return SessionTelemetry( + token_usage=skill_result.token_usage, + timing_seconds=timing_seconds, + audit_record=audit_record, + github_api_usage=_api_usage, + github_api_requests=_api_usage.get("total_requests", 0) if _api_usage else 0, + loc_insertions=loc_insertions, + loc_deletions=loc_deletions, + ) + + +def _build_error_path_telemetry( + github_api_log: GitHubApiLog | None, + session_id: str = "", +) -> SessionTelemetry: + """Build SessionTelemetry for crash/cancel paths where no SkillResult exists.""" + _api_usage = github_api_log.drain(session_id) if github_api_log is not None else None + return SessionTelemetry( + token_usage=None, + timing_seconds=None, + audit_record=None, + github_api_usage=_api_usage, + github_api_requests=_api_usage.get("total_requests", 0) if _api_usage else 0, + loc_insertions=0, + loc_deletions=0, + ) diff --git a/src/autoskillit/execution/_headless_scan.py b/src/autoskillit/execution/headless/_headless_scan.py similarity index 97% rename from src/autoskillit/execution/_headless_scan.py rename to src/autoskillit/execution/headless/_headless_scan.py index d960674686..0edf2ccdc4 100644 --- a/src/autoskillit/execution/_headless_scan.py +++ b/src/autoskillit/execution/headless/_headless_scan.py @@ -3,14 +3,15 @@ Extracted from headless.py to keep that module below the 1100-line architectural limit (REQ-CNST-010-E2). -L1 module (execution/). No autoskillit imports — stdlib only. +IL-1 module (execution/). No autoskillit imports — stdlib only. """ from __future__ import annotations import json import os -import re + +import regex as re _WRITE_TOOL_NAMES: frozenset[str] = frozenset({"Write", "Edit"}) _BASH_TOOL_NAME: str = "Bash" diff --git a/src/autoskillit/execution/linux_tracing.py b/src/autoskillit/execution/linux_tracing.py index 6788c1149c..f68b39199c 100644 --- a/src/autoskillit/execution/linux_tracing.py +++ b/src/autoskillit/execution/linux_tracing.py @@ -23,7 +23,7 @@ import tempfile import time from collections.abc import AsyncIterator -from dataclasses import dataclass, field +from dataclasses import asdict, dataclass, field from datetime import UTC, datetime, timedelta from pathlib import Path from typing import IO, TYPE_CHECKING @@ -32,7 +32,10 @@ import anyio.abc import psutil +from autoskillit.core import fast_dumps as _fast_dumps from autoskillit.core import get_logger +from autoskillit.core import read_boot_id as read_boot_id +from autoskillit.core import read_starttime_ticks as read_starttime_ticks if TYPE_CHECKING: from autoskillit.config import LinuxTracingConfig @@ -42,35 +45,6 @@ LINUX_TRACING_AVAILABLE = sys.platform == "linux" -def read_boot_id() -> str | None: - """Read the system boot ID from /proc/sys/kernel/random/boot_id.""" - try: - return Path("/proc/sys/kernel/random/boot_id").read_text().strip() - except OSError: - return None - - -def read_starttime_ticks(pid: int) -> int | None: - """Read process starttime ticks from /proc/pid/stat. - - Uses rfind(")") to correctly locate the field boundary even when the - process comm contains a ")" character. Matches psutil's own _parse_stat_file() - which uses rfind(b")") for the same reason. - """ - try: - stat = Path(f"/proc/{pid}/stat").read_text() - # comm may contain ")" — use rfind to find the *last* ")" as the boundary - rpar = stat.rfind(")") - if rpar == -1: - return None - fields = stat[rpar + 2 :].split() - # starttime is field 22 (1-indexed per man page), offset 19 from the field after ")" - return int(fields[19]) - except (OSError, ValueError, IndexError): - pass - return None - - # --------------------------------------------------------------------------- # TraceTarget: workload process identity with provenance # --------------------------------------------------------------------------- @@ -95,7 +69,7 @@ def __init__(self, root_pid: int, expected_basename: str) -> None: ) -@dataclass(frozen=True) +@dataclass(frozen=True, slots=True) class TraceTarget: """Workload process identity with provenance — the correct target for the tracer. @@ -117,7 +91,7 @@ class TraceTarget: resolved_at: datetime -def resolve_trace_target( +async def resolve_trace_target( root_pid: int, expected_basename: str, timeout: float = 2.0, @@ -153,8 +127,10 @@ def resolve_trace_target( try: name = child.name() cmdline = child.cmdline() + if not cmdline: + continue basename_matches = name == expected_basename or ( - cmdline and Path(cmdline[0]).name == expected_basename + Path(cmdline[0]).name == expected_basename ) if not basename_matches: continue @@ -174,7 +150,7 @@ def resolve_trace_target( except (psutil.NoSuchProcess, psutil.AccessDenied, psutil.ZombieProcess, OSError): continue - time.sleep(0.05) + await anyio.sleep(0.05) raise TraceTargetResolutionError(root_pid=root_pid, expected_basename=expected_basename) @@ -213,7 +189,7 @@ def trace_target_from_pid(pid: int) -> TraceTarget: # --------------------------------------------------------------------------- -@dataclass(frozen=True) +@dataclass(frozen=True, slots=True) class TraceEnrollmentRecord: """Identity triple written atomically at trace-open time. @@ -238,7 +214,7 @@ class TraceEnrollmentRecord: def _write_enrollment_atomic(path: Path, record: TraceEnrollmentRecord) -> None: """Write enrollment sidecar atomically using tempfile + os.replace.""" - content = json.dumps(dataclasses.asdict(record)) + content = _fast_dumps(dataclasses.asdict(record)) fd, tmp = tempfile.mkstemp(dir=path.parent, suffix=".tmp") try: with os.fdopen(fd, "w", encoding="utf-8") as f: @@ -276,7 +252,7 @@ def read_enrollment(path: Path) -> TraceEnrollmentRecord | None: # --------------------------------------------------------------------------- -@dataclass(frozen=True) +@dataclass(frozen=True, slots=True) class ProcSnapshot: """Point-in-time snapshot of process state. @@ -305,6 +281,12 @@ class ProcSnapshot: wchan: str # CPU utilisation (0.0 when process arg is not supplied to read_proc_snapshot) cpu_percent: float + # Best-effort /proc/{pid}/net/tcp fields (Linux only; None when unavailable) + api_connection_established: int | None = None + api_connection_states: dict[str, int] | None = None + # Best-effort /proc/{pid}/io fields (Linux only; None when unavailable) + io_read_bytes: int | None = None + io_write_bytes: int | None = None def _parse_proc_status(content: str) -> dict[str, str]: @@ -328,6 +310,68 @@ def _parse_proc_status(content: str) -> dict[str, str]: return fields +_TCP_STATE_NAMES: dict[str, str] = { + "01": "ESTABLISHED", + "02": "SYN_SENT", + "03": "SYN_RECV", + "04": "FIN_WAIT1", + "05": "FIN_WAIT2", + "06": "TIME_WAIT", + "07": "CLOSE", + "08": "CLOSE_WAIT", + "09": "LAST_ACK", + "0A": "LISTEN", + "0B": "CLOSING", +} +_API_PORT_HEX: str = "01BB" # port 443 in big-endian hex (Linux /proc/net/tcp format) + + +def _parse_net_tcp(content: str) -> dict[str, int]: + """Parse /proc/{pid}/net/tcp content for connections to port 443. + + Returns a dict mapping TCP state name to connection count. + Returns {} on empty or header-only content. + Lines with fewer than 4 fields are skipped silently. + """ + counts: dict[str, int] = {} + for line in content.splitlines()[1:]: # skip header + parts = line.split() + if len(parts) < 4: + continue + rem_addr = parts[2] + state_hex = parts[3].upper() + if ":" not in rem_addr: + continue + rem_port = rem_addr.split(":")[1].upper() + if rem_port != _API_PORT_HEX: + continue + state_name = _TCP_STATE_NAMES.get(state_hex, state_hex) + counts[state_name] = counts.get(state_name, 0) + 1 + return counts + + +def _parse_proc_io(content: str) -> tuple[int | None, int | None]: + """Parse /proc/{pid}/io content for read_bytes and write_bytes. + + Returns (read_bytes, write_bytes). Either may be None if the field + is absent or unparseable. + """ + read_b: int | None = None + write_b: int | None = None + for line in content.splitlines(): + if line.startswith("read_bytes:"): + try: + read_b = int(line.split(":", 1)[1].strip()) + except (ValueError, IndexError): + pass + elif line.startswith("write_bytes:"): + try: + write_b = int(line.split(":", 1)[1].strip()) + except (ValueError, IndexError): + pass + return read_b, write_b + + def read_proc_snapshot(pid: int, *, process: psutil.Process | None = None) -> ProcSnapshot | None: """Read a complete snapshot for pid. Returns None if process gone. @@ -355,26 +399,54 @@ def read_proc_snapshot(pid: int, *, process: psutil.Process | None = None) -> Pr # Read process name from /proc/{pid}/comm (max 15 chars, kernel-truncated) try: comm = Path(f"/proc/{pid}/comm").read_text().strip() - except (FileNotFoundError, PermissionError): + except (FileNotFoundError, PermissionError, ProcessLookupError): comm = "" # Hand-rolled /proc reads for fields psutil doesn't expose try: status_content = Path(f"/proc/{pid}/status").read_text() - except (FileNotFoundError, PermissionError): + except (FileNotFoundError, PermissionError, ProcessLookupError): return None sig_fields = _parse_proc_status(status_content) try: oom = int(Path(f"/proc/{pid}/oom_score").read_text().strip()) - except (FileNotFoundError, PermissionError, ValueError): + except (FileNotFoundError, PermissionError, ProcessLookupError, ValueError): oom = -1 try: wchan = Path(f"/proc/{pid}/wchan").read_text().strip() - except (FileNotFoundError, PermissionError): + except (FileNotFoundError, PermissionError, ProcessLookupError): wchan = "" + # Best-effort: /proc/{pid}/net/tcp and tcp6 (Linux network namespace for this PID) + _api_conn_states: dict[str, int] | None = None + try: + _tcp_content = Path(f"/proc/{pid}/net/tcp").read_text() + _states = _parse_net_tcp(_tcp_content) + try: + _tcp6_content = Path(f"/proc/{pid}/net/tcp6").read_text() + for k, v in _parse_net_tcp(_tcp6_content).items(): + _states[k] = _states.get(k, 0) + v + except (FileNotFoundError, PermissionError, OSError): + pass + _api_conn_states = _states if _states else {} + except (FileNotFoundError, PermissionError, OSError, ValueError): + pass + + _api_conns_established: int | None = None + if _api_conn_states is not None: + _api_conns_established = _api_conn_states.get("ESTABLISHED", 0) + + # Best-effort: /proc/{pid}/io + _io_read: int | None = None + _io_write: int | None = None + try: + _io_content = Path(f"/proc/{pid}/io").read_text() + _io_read, _io_write = _parse_proc_io(_io_content) + except (FileNotFoundError, PermissionError, OSError): + pass + return ProcSnapshot( captured_at=captured_at, comm=comm, @@ -391,6 +463,10 @@ def read_proc_snapshot(pid: int, *, process: psutil.Process | None = None) -> Pr oom_score=oom, wchan=wchan, cpu_percent=cpu_pct, + api_connection_established=_api_conns_established, + api_connection_states=_api_conn_states, + io_read_bytes=_io_read, + io_write_bytes=_io_write, ) @@ -526,7 +602,7 @@ async def _run_monitor() -> None: handle._snapshots.append(snap) if handle._trace_file is not None: try: - handle._trace_file.write(json.dumps(snap.__dict__) + "\n") + handle._trace_file.write(_fast_dumps(asdict(snap)) + "\n") except OSError: # Close broken file; degrade to in-memory only try: diff --git a/src/autoskillit/execution/merge_queue.py b/src/autoskillit/execution/merge_queue.py deleted file mode 100644 index e308f934ae..0000000000 --- a/src/autoskillit/execution/merge_queue.py +++ /dev/null @@ -1,750 +0,0 @@ -"""Merge queue polling service (L1) — monitors GitHub merge queue for a PR. - -Never raises. All errors are returned as structured results. -""" - -from __future__ import annotations - -import asyncio -import random -import re -import time -from collections.abc import Callable -from dataclasses import dataclass -from datetime import UTC, datetime -from typing import Any, TypedDict, assert_never - -import httpx - -from autoskillit.core import PRState, get_logger -from autoskillit.execution.github import github_headers - -_log = get_logger(__name__) - - -class PRFetchState(TypedDict): - """Typed contract for _fetch_pr_and_queue_state return value.""" - - merged: bool - state: str - mergeable: str # "MERGEABLE" | "CONFLICTING" | "UNKNOWN" - merge_state_status: str - auto_merge_present: bool # True when autoMergeRequest is not null - auto_merge_enabled_at: datetime | None - pr_node_id: str - in_queue: bool - queue_state: str | None - checks_state: str | None # statusCheckRollup.state; None = no checks configured - - -_GRAPHQL_ENDPOINT = "https://api.github.com/graphql" - -_QUERY = """ -query GetPRAndQueueState($owner: String!, $repo: String!, $prNumber: Int!, $branch: String!) { - repository(owner: $owner, name: $repo) { - pullRequest(number: $prNumber) { - id - merged - state - mergeable - mergeStateStatus - autoMergeRequest { - enabledAt - } - statusCheckRollup { - state - } - } - mergeQueue(branch: $branch) { - entries(first: 100) { - nodes { - pullRequest { number } - state - } - } - } - } -} -""" - -# Maps every PRFetchState key to its GraphQL source path. -# "" means the field is derived from query results, not directly selected. -# This constant is validated at import time against PRFetchState.__required_keys__ -# (mirroring the pattern in recipe/io.py:126-161). -_QUERY_FIELD_MAP: dict[str, str] = { - "merged": "merged", - "state": "state", - "mergeable": "mergeable", - "merge_state_status": "mergeStateStatus", - "auto_merge_present": "autoMergeRequest", - "auto_merge_enabled_at": "autoMergeRequest.enabledAt", - "pr_node_id": "id", - "in_queue": "", - "queue_state": "", - "checks_state": "statusCheckRollup.state", -} - -_ALL_FETCH_STATE_KEYS = PRFetchState.__required_keys__ | PRFetchState.__optional_keys__ -if set(_QUERY_FIELD_MAP) != _ALL_FETCH_STATE_KEYS: - raise RuntimeError( - "_QUERY_FIELD_MAP is out of sync with PRFetchState keys.\n" - f"Missing from map: {_ALL_FETCH_STATE_KEYS - set(_QUERY_FIELD_MAP)}\n" - f"Missing from state: {set(_QUERY_FIELD_MAP) - _ALL_FETCH_STATE_KEYS}" - ) -for _key, _path in _QUERY_FIELD_MAP.items(): - if _path.startswith("<"): - continue - _head = _path.split(".", 1)[0] - # Word-boundary search prevents "state" from matching inside "mergeStateStatus". - if not re.search(r"\b" + re.escape(_head) + r"\b", _QUERY): - raise RuntimeError( - f"_QUERY is missing GraphQL field {_head!r} required by PRFetchState[{_key!r}]" - ) - -_MUTATION_DISABLE_AUTO_MERGE = """ -mutation DisableAutoMerge($prId: ID!) { - disablePullRequestAutoMerge(input: {pullRequestId: $prId}) { - pullRequest { number } - } -} -""" - -_MUTATION_ENABLE_AUTO_MERGE = """ -mutation EnableAutoMerge($prId: ID!, $mergeMethod: PullRequestMergeMethod!) { - enablePullRequestAutoMerge(input: {pullRequestId: $prId, mergeMethod: $mergeMethod}) { - pullRequest { number } - } -} -""" - -# Repo-level state query: consolidates three former run_cmd steps into one HTTP round-trip. -# Distinct from _QUERY (PR-level + queue-entries); do not merge these constants. -# Returns: mergeQueue presence, autoMergeAllowed flag, and workflow file texts for -# merge_group trigger detection — all in a single GraphQL call. -_REPO_STATE_QUERY = """ -query GetRepoMergeState($owner: String!, $repo: String!, $branch: String!) { - repository(owner: $owner, name: $repo) { - mergeQueue(branch: $branch) { - id - } - autoMergeAllowed - object(expression: "HEAD:.github/workflows") { - ... on Tree { - entries { - name - object { - ... on Blob { - text - } - } - } - } - } - } -} -""" - - -# --------------------------------------------------------------------------- -# Classifier primitives (pure functions — no I/O, no async) -# --------------------------------------------------------------------------- - - -@dataclass(frozen=True) -class ClassificationResult: - """Positive-signal classification outcome from _classify_pr_state.""" - - terminal: PRState - reason: str - - -class ClassifierInconclusive(Exception): - """Raised when no positive gate matched — caller must continue polling. - - The .state attribute exposes the full PRFetchState that was inspected, - enabling callers to log or surface the ambiguous fields. - """ - - def __init__(self, state: PRFetchState, reason: str) -> None: - super().__init__(reason) - self.state = state - self.reason = reason - - -class CIStillRunning(ClassifierInconclusive): - """CI checks are legitimately in-progress (PENDING or EXPECTED). - - Expected transient state — outer timeout_seconds bounds the wait. - Must NOT consume the inconclusive budget. - """ - - -class NoPositiveSignal(ClassifierInconclusive): - """No positive classifier gate matched — state is genuinely ambiguous. - - Bounded by max_inconclusive_retries. Counts against the inconclusive budget. - """ - - -def _is_positive_stall(state: PRFetchState) -> bool: - """True when auto-merge is enabled and merge_state_status indicates the PR is - stuck in a state where it should be in queue but is not.""" - return state["auto_merge_enabled_at"] is not None and state["merge_state_status"] in { - "CLEAN", - "HAS_HOOKS", - } - - -def _is_positive_dropped_healthy(state: PRFetchState) -> bool: - """True when the PR is fully healthy but auto_merge was cleared externally.""" - return ( - state["state"] == "OPEN" - and state["mergeable"] == "MERGEABLE" - and state["merge_state_status"] == "CLEAN" - and state["checks_state"] in (None, "SUCCESS") - and state["auto_merge_present"] is False - and state["in_queue"] is False - ) - - -def _classify_pr_state(state: PRFetchState) -> ClassificationResult: - """Classify PR state using positive signals only — no fall-through to EJECTED. - - Every return originates from a direct positive signal. When no positive gate - matches, raises ClassifierInconclusive so the caller can continue polling - within a bounded retry budget rather than silently misclassifying. - - Args: - state: Current PR fetch state snapshot. - - Returns: - ClassificationResult with the matched terminal PRState. - - Raises: - ClassifierInconclusive: When no positive signal matched. .state exposes - the full snapshot for logging/surfacing. - """ - if state["merged"]: - return ClassificationResult(PRState.MERGED, "PR merged") - - if state["state"] == "CLOSED": - if state["checks_state"] in {"FAILURE", "ERROR"}: - return ClassificationResult(PRState.EJECTED_CI_FAILURE, "PR closed after CI failure") - return ClassificationResult(PRState.EJECTED, "PR closed while not merged") - - if state["checks_state"] in {"FAILURE", "ERROR"}: - return ClassificationResult(PRState.EJECTED_CI_FAILURE, "checks terminal failure") - - if _is_positive_stall(state): - return ClassificationResult(PRState.STALLED, "stall signals present") - - if state["mergeable"] == "CONFLICTING": - return ClassificationResult(PRState.EJECTED, "conflicting changes prevent merge") - - if _is_positive_dropped_healthy(state): - return ClassificationResult(PRState.DROPPED_HEALTHY, "PR healthy but auto_merge cleared") - - if state["checks_state"] in {"PENDING", "EXPECTED"}: - raise CIStillRunning(state, "checks still running") - - raise NoPositiveSignal(state, "no positive signal matched") - - -class DefaultMergeQueueWatcher: - """Polls GitHub merge queue state until merged, ejected, stalled, or timed out. - - Uses a single consolidated GraphQL query per poll cycle to atomically capture - both PR state and queue entries, eliminating the race condition where two - separate API calls straddle a merge event. - - Never raises; all errors are returned as structured dicts. - """ - - def __init__( - self, - token: str | None | Callable[[], str | None], - ) -> None: - self._token_factory: Callable[[], str | None] | None - if callable(token): - self._token_factory = token - self._client: httpx.AsyncClient | None = None - else: - self._token_factory = None - self._client = httpx.AsyncClient( - headers=github_headers(token), - limits=httpx.Limits(keepalive_expiry=60), - timeout=30.0, - ) - - def _ensure_client(self) -> httpx.AsyncClient: - if self._client is None: - resolved = self._token_factory() if self._token_factory is not None else None - self._client = httpx.AsyncClient( - headers=github_headers(resolved), - limits=httpx.Limits(keepalive_expiry=60), - timeout=30.0, - ) - return self._client - - async def aclose(self) -> None: - if self._client is not None: - await self._client.aclose() - - async def wait( - self, - pr_number: int, - target_branch: str, - repo: str | None = None, - cwd: str = ".", - timeout_seconds: int = 600, - poll_interval: int = 15, - stall_grace_period: int = 60, - max_stall_retries: int = 3, - not_in_queue_confirmation_cycles: int = 2, - max_inconclusive_retries: int = 5, - ) -> dict[str, Any]: - """Poll until PR is merged, ejected, stalled, dropped, or timeout expires. - - Args: - pr_number: PR number to monitor. - target_branch: Branch the merge queue targets. - repo: "owner/name" format. Required. - cwd: Working directory. Present to satisfy the MergeQueueWatcher protocol. - timeout_seconds: Maximum polling duration (default 600s). - poll_interval: Seconds between poll cycles (default 15s). - stall_grace_period: Seconds after autoMergeRequest.enabledAt before stall - recovery may trigger (default 60s). Prevents intervention during normal - queue processing. - max_stall_retries: Maximum disable/re-enable toggle attempts before - returning pr_state="stalled" (default 3). - not_in_queue_confirmation_cycles: Consecutive "not in queue" cycles required - before acting on absence. Guards against race between queue exit and - merged=true propagation (default 2). - max_inconclusive_retries: Maximum NoPositiveSignal cycles (beyond the - confirmation window) before returning pr_state="timeout" (default 5). - - Returns: - { - "success": bool, - "pr_state": PRState value string, - "reason": str, - "stall_retries_attempted": int, - } - """ - if not repo or "/" not in repo: - return { - "success": False, - "pr_state": PRState.ERROR.value, - "reason": f"Invalid repo format: {repo!r}. Expected 'owner/name'.", - "stall_retries_attempted": 0, - } - owner, repo_name = repo.split("/", 1) - - deadline = time.monotonic() + timeout_seconds - stall_retries_attempted: int = 0 - not_in_queue_cycles: int = 0 - inconclusive_count: int = 0 - - def _make_result( - success: bool, pr_state: PRState, reason: str, ejection_cause: str = "" - ) -> dict[str, Any]: - result: dict[str, Any] = { - "success": success, - "pr_state": pr_state.value, - "reason": reason, - "stall_retries_attempted": stall_retries_attempted, - } - if ejection_cause: - result["ejection_cause"] = ejection_cause - return result - - while time.monotonic() < deadline: - try: - state = await self._fetch_pr_and_queue_state( - pr_number, owner, repo_name, target_branch - ) - except Exception: - _log.warning("fetch_pr_and_queue_state failed, retrying", exc_info=True) - await asyncio.sleep(poll_interval) - continue - - # In queue: reset window and continue - if state["in_queue"]: - not_in_queue_cycles = 0 - inconclusive_count = 0 - if state["queue_state"] == "UNMERGEABLE": - return _make_result(False, PRState.EJECTED, "PR is UNMERGEABLE in merge queue") - await asyncio.sleep(poll_interval) - continue - - not_in_queue_cycles += 1 - - # Classify state using positive-signal gates - try: - classification = _classify_pr_state(state) - except CIStillRunning: - await asyncio.sleep(poll_interval) - continue - except NoPositiveSignal as exc: - if not_in_queue_cycles < not_in_queue_confirmation_cycles: - await asyncio.sleep(poll_interval) - continue - inconclusive_count += 1 - if inconclusive_count >= max_inconclusive_retries: - return _make_result( - False, - PRState.TIMEOUT, - f"Inconclusive after {max_inconclusive_retries} retries: {exc.reason}", - ) - await asyncio.sleep(poll_interval) - continue - - # MERGED: definitive — bypasses confirmation window - if classification.terminal == PRState.MERGED: - return _make_result(True, PRState.MERGED, classification.reason) - - # CLOSED: definitive — bypasses confirmation window - if state["state"] == "CLOSED": - ejection_cause = ( - "ci_failure" if classification.terminal == PRState.EJECTED_CI_FAILURE else "" - ) - return _make_result( - False, classification.terminal, classification.reason, ejection_cause - ) - - # All other positive terminals require the confirmation window - if not_in_queue_cycles < not_in_queue_confirmation_cycles: - await asyncio.sleep(poll_interval) - continue - - # Post-confirmation terminal dispatch - if classification.terminal == PRState.STALLED: - enabled_at = state["auto_merge_enabled_at"] - assert enabled_at is not None # guaranteed by _is_positive_stall - now = datetime.now(UTC) - stall_duration = max(0.0, (now - enabled_at).total_seconds()) - if stall_duration < stall_grace_period: - await asyncio.sleep(poll_interval) - continue - if stall_retries_attempted < max_stall_retries: - backoff = min(30 * (2**stall_retries_attempted), 120) - _log.info( - "merge_queue_stall_detected", - stall_duration=stall_duration, - attempt=stall_retries_attempted + 1, - backoff=backoff, - ) - try: - await self._toggle_auto_merge(state["pr_node_id"]) - except Exception: - _log.warning("toggle_auto_merge failed", exc_info=True) - stall_retries_attempted += 1 - not_in_queue_cycles = 0 - await asyncio.sleep(backoff) - continue - return _make_result( - False, - PRState.STALLED, - f"PR #{pr_number} stall unresolved after {max_stall_retries} toggle attempts", - ) - - elif classification.terminal == PRState.EJECTED_CI_FAILURE: - return _make_result( - False, - PRState.EJECTED_CI_FAILURE, - classification.reason, - ejection_cause="ci_failure", - ) - - elif classification.terminal == PRState.EJECTED: - return _make_result(False, PRState.EJECTED, classification.reason) - - elif classification.terminal == PRState.DROPPED_HEALTHY: - return _make_result(False, PRState.DROPPED_HEALTHY, classification.reason) - - else: - # Unreachable: _classify_pr_state never returns TIMEOUT or ERROR. - # The assert_never call provides static exhaustiveness for future - # additions to PRState (pyright/mypy will flag any new unhandled member). - assert_never(classification.terminal) # type: ignore[arg-type] - - # Deadline exceeded - return _make_result( - False, - PRState.TIMEOUT, - f"Timed out after {timeout_seconds}s waiting for PR #{pr_number}", - ) - - async def _fetch_pr_and_queue_state( - self, pr_number: int, owner: str, repo: str, target_branch: str - ) -> PRFetchState: - """Single GraphQL round-trip returning PR state and merge queue entries.""" - variables = { - "owner": owner, - "repo": repo, - "prNumber": pr_number, - "branch": target_branch, - } - resp = await self._ensure_client().post( - _GRAPHQL_ENDPOINT, json={"query": _QUERY, "variables": variables} - ) - resp.raise_for_status() - data = resp.json() - if "errors" in data: - raise RuntimeError(f"GraphQL error: {data['errors']}") - - if not data.get("data") or data["data"] is None or "repository" not in data["data"]: - raise RuntimeError(f"GraphQL response missing 'repository' key: {str(data)[:200]}") - repo_data = data["data"]["repository"] - pr = repo_data["pullRequest"] - if pr is None: - raise RuntimeError(f"GraphQL returned null pullRequest for PR #{pr_number}") - entries_raw = repo_data.get("mergeQueue") or {} - entries = entries_raw if isinstance(entries_raw, dict) else {} - nodes_raw = (entries.get("entries") or {}).get("nodes") - nodes = nodes_raw if isinstance(nodes_raw, list) else [] - - auto_merge_raw = pr.get("autoMergeRequest") - auto_merge_present: bool = auto_merge_raw is not None - auto_merge = auto_merge_raw or {} - enabled_at_raw = auto_merge.get("enabledAt") - enabled_at: datetime | None = None - if enabled_at_raw: - try: - enabled_at = datetime.fromisoformat(enabled_at_raw.replace("Z", "+00:00")) - except ValueError as e: - raise RuntimeError( - f"Unexpected autoMergeRequest.enabledAt format: {enabled_at_raw!r}" - ) from e - - checks_rollup = pr.get("statusCheckRollup") or {} - checks_state: str | None = checks_rollup.get("state") - - queue_entry = next((n for n in nodes if n["pullRequest"]["number"] == pr_number), None) - return PRFetchState( - merged=pr["merged"], - state=pr["state"], - mergeable=pr.get("mergeable") or "UNKNOWN", - merge_state_status=pr.get("mergeStateStatus", "UNKNOWN"), - auto_merge_present=auto_merge_present, - auto_merge_enabled_at=enabled_at, - pr_node_id=pr["id"], - in_queue=queue_entry is not None, - queue_state=queue_entry["state"] if queue_entry else None, - checks_state=checks_state, - ) - - async def toggle( - self, - pr_number: int, - target_branch: str, - repo: str | None = None, - cwd: str = ".", - ) -> dict[str, Any]: - """Disable then re-enable auto-merge for a PR to re-enroll it in the merge queue. - - Fetches the PR node ID via a single GraphQL query, then applies the - disable/re-enable toggle. Never raises; returns a structured result dict. - """ - if not repo or "/" not in repo: - return { - "success": False, - "error": f"Invalid repo format: {repo!r}. Expected 'owner/name'.", - } - owner, repo_name = repo.split("/", 1) - try: - state = await self._fetch_pr_and_queue_state( - pr_number, owner, repo_name, target_branch - ) - await self._toggle_auto_merge(state["pr_node_id"]) - return {"success": True, "pr_number": pr_number} - except Exception as exc: - _log.warning("toggle_auto_merge failed", exc_info=True) - return {"success": False, "error": f"toggle failed: {exc}"} - - async def _toggle_auto_merge(self, pr_node_id: str) -> None: - """Disable then re-enable auto-merge via GraphQL mutations.""" - if not pr_node_id: - raise ValueError("pr_node_id must be a non-empty string") - mutations = [ - (_MUTATION_DISABLE_AUTO_MERGE, {"prId": pr_node_id}), - (_MUTATION_ENABLE_AUTO_MERGE, {"prId": pr_node_id, "mergeMethod": "SQUASH"}), - ] - for i, (mutation, variables) in enumerate(mutations): - resp = await self._ensure_client().post( - _GRAPHQL_ENDPOINT, json={"query": mutation, "variables": variables} - ) - resp.raise_for_status() - body = resp.json() - if "errors" in body: - raise RuntimeError(f"GraphQL mutation error: {body['errors']}") - if i < len(mutations) - 1: - await asyncio.sleep(2) - - -# --------------------------------------------------------------------------- -# Repo-level state helper (used by check_repo_merge_state MCP tool) -# --------------------------------------------------------------------------- - - -def _text_has_push_trigger(text: str) -> bool: - """Return True if a workflow file text declares a push trigger. - - Checks for the three YAML forms GitHub supports for a push trigger: - - ``on: [push, ...]`` (list form) - - ``on: push`` (scalar form) - - ``push:`` (mapping form under ``on:``) - - Note: the first two patterns (``on: [push``, ``on: push``) are reliably - scoped to YAML trigger syntax. The third (``push:``) may match inside - comments or run-step values — acceptable for the positive-signal-only - classification heuristic but not precise. - """ - return any(pat in text for pat in ("on: [push", "on: push", "push:")) - - -_RATE_LIMIT_MAX_ATTEMPTS = 3 -_RATE_LIMIT_SECONDARY_MARKER = "secondary rate limit" - - -def _is_secondary_rate_limit(resp: httpx.Response) -> bool: - """Return True when a 403 response is a GitHub secondary rate limit. - - GitHub returns HTTP 403 (not 429) for secondary rate limits. - The response body contains the phrase "secondary rate limit". - Primary rate limits use HTTP 429 or include x-ratelimit-remaining: 0. - """ - if resp.status_code != 403: - return False - try: - text = resp.text.lower() - except Exception: - _log.warning("Failed to read response body for rate-limit check", exc_info=True) - return False - return _RATE_LIMIT_SECONDARY_MARKER in text - - -def _retry_after_seconds(attempt: int, resp: httpx.Response) -> float: - """Return seconds to sleep before the next retry attempt. - - Prefers the Retry-After header (integer seconds) when present and valid. - Falls back to full-jitter exponential backoff: random(0, min(60, 1 * 2^attempt)). - """ - try: - header_val = resp.headers.get("Retry-After", "") - if header_val: - return float(header_val) - except (ValueError, AttributeError): - pass - return random.uniform(0, min(60.0, 1.0 * (2**attempt))) - - -async def fetch_repo_merge_state( - owner: str, - repo: str, - branch: str, - token: str | None, -) -> dict[str, bool | str | None]: - """Fetch repository merge-state in a single GraphQL round-trip. - - Returns a dict with four keys: - - ``queue_available``: the branch has an active GitHub merge queue (bool). - - ``merge_group_trigger``: at least one CI workflow declares the - ``merge_group`` event trigger (bool). - - ``auto_merge_available``: the repository has auto-merge enabled (bool). - - ``ci_event``: the recommended CI event to poll — ``"push"`` when any - workflow declares a push trigger, ``"merge_group"`` when only merge_group - triggers are found, or ``None`` when no push/merge_group triggers exist - (ci.py scope.event=None matches any trigger). - - Null-handling: - - ``mergeQueue is null`` → ``queue_available: False`` (no queue) - - ``object is null`` → ``merge_group_trigger: False``, ``ci_event: None`` (no workflows dir) - - ``entry.object.text is null`` → skip entry (binary/large file) - - GraphQL ``autoMergeAllowed`` field error (GHES 3.0.x) → ``auto_merge_available: False`` - - Only transport-level failures (network timeout, non-200 HTTP status) are - allowed to propagate; callers are expected to handle them. - - Historical note: Issue #498 ("Merge queue detection should validate workflow has - merge_group trigger") established the merge_group_trigger field. The ci_event - field is a closely related extension — verify that the push-trigger scan does - not regress the merge_group-only detection that #498 established. - """ - resp: httpx.Response | None = None - for attempt in range(_RATE_LIMIT_MAX_ATTEMPTS): - async with httpx.AsyncClient( - headers=github_headers(token), - timeout=30.0, - ) as client: - resp = await client.post( - _GRAPHQL_ENDPOINT, - json={ - "query": _REPO_STATE_QUERY, - "variables": {"owner": owner, "repo": repo, "branch": branch}, - }, - ) - if resp.status_code == 429 or _is_secondary_rate_limit(resp): - sleep_secs = _retry_after_seconds(attempt, resp) - _log.warning( - "fetch_repo_merge_state rate limited", - status=resp.status_code, - attempt=attempt, - sleep_secs=sleep_secs, - ) - await asyncio.sleep(sleep_secs) - continue - resp.raise_for_status() - break - else: - assert resp is not None, "_RATE_LIMIT_MAX_ATTEMPTS must be >= 1" - resp.raise_for_status() - - assert resp is not None - body = resp.json() - - # GitHub GraphQL always returns a JSON object; guard against unexpected shapes. - if not isinstance(body, dict): - body = {} - - # Gracefully handle GHES 3.0.x where autoMergeAllowed doesn't exist. - auto_merge_field_missing = any( - "autoMergeAllowed" in str(e.get("message", "")) for e in body.get("errors", []) - ) - - repo_data: dict[str, Any] = (body.get("data") or {}).get("repository") or {} - queue_available = repo_data.get("mergeQueue") is not None - auto_merge_available = ( - False if auto_merge_field_missing else bool(repo_data.get("autoMergeAllowed", False)) - ) - - # Scan workflow files for push and merge_group trigger declarations. - # Both flags are derived from the same Blob.text scan — no extra round-trips. - merge_group_trigger = False - has_push_trigger = False - workflows_tree = repo_data.get("object") - if workflows_tree is not None: - for entry in workflows_tree.get("entries", []): - blob = entry.get("object") or {} - text = blob.get("text") - if text is None: - continue # binary or oversized blob — skip - if "merge_group" in text: - merge_group_trigger = True - if _text_has_push_trigger(text): - has_push_trigger = True - - # Derive ci_event: prefer push for historical compatibility when both are present. - if has_push_trigger: - ci_event: str | None = "push" - elif merge_group_trigger: - ci_event = "merge_group" - else: - ci_event = None # schedule-only or workflow_dispatch-only: match any - - return { - "queue_available": queue_available, - "merge_group_trigger": merge_group_trigger, - "auto_merge_available": auto_merge_available, - "ci_event": ci_event, - } diff --git a/src/autoskillit/execution/merge_queue/CLAUDE.md b/src/autoskillit/execution/merge_queue/CLAUDE.md new file mode 100644 index 0000000000..5b3109f48f --- /dev/null +++ b/src/autoskillit/execution/merge_queue/CLAUDE.md @@ -0,0 +1,16 @@ +# merge_queue/ + +GitHub merge queue watcher — polls PR state until merged, ejected, or timed out. + +## Files + +| File | Purpose | +|------|---------| +| `__init__.py` | Main module: `DefaultMergeQueueWatcher` with single-GraphQL-round-trip polling | +| `_merge_queue_classifier.py` | `_classify_pr_state()` pure function: MERGED/EJECTED/STALLED/DROPPED classification | +| `_merge_queue_group_ci.py` | `_query_merge_group_ci()` and GraphQL mutation strings for auto-merge/enqueue | +| `_merge_queue_repo_state.py` | `fetch_repo_merge_state()`, push/merge-group trigger detection, rate-limit retry | + +## Architecture Notes + +The `random` module is explicitly re-exported from `__init__.py` to enable test monkeypatching of `merge_queue.random.uniform` for jitter control. diff --git a/src/autoskillit/execution/merge_queue/__init__.py b/src/autoskillit/execution/merge_queue/__init__.py new file mode 100644 index 0000000000..3b4c39139f --- /dev/null +++ b/src/autoskillit/execution/merge_queue/__init__.py @@ -0,0 +1,499 @@ +"""Merge queue polling service (IL-1) — monitors GitHub merge queue for a PR. + +Never raises. All errors are returned as structured results. + +Facade: re-exports from _merge_queue_classifier and _merge_queue_repo_state. +""" + +from __future__ import annotations + +import asyncio +import random # noqa: F401 — re-exported for test monkeypatching (_mq.random.uniform) +import time +from collections.abc import Callable +from datetime import UTC, datetime +from typing import TYPE_CHECKING, Any, assert_never, cast + +import httpx + +from autoskillit.core import PRState, get_logger +from autoskillit.execution.github import github_headers, make_tracked_httpx_client +from autoskillit.execution.merge_queue._merge_queue_classifier import ( + _QUERY_FIELD_MAP, # noqa: F401 — re-export: tests access merge_queue._QUERY_FIELD_MAP + KNOWN_MQ_MERGE_STATE_STATUSES, # noqa: F401 — re-export for callers + CIStillRunning, + ClassificationResult, # noqa: F401 — re-export for callers + ClassifierInconclusive, # noqa: F401 — re-export for callers + NoPositiveSignal, + PRFetchState, + _classify_pr_state, + _is_not_enrolled, # noqa: F401 — re-export for callers + _is_positive_dropped_healthy, # noqa: F401 — re-export for callers + _is_positive_dropped_merge_group_ci, # noqa: F401 — re-export for callers + _is_positive_stall, # noqa: F401 — re-export for callers +) +from autoskillit.execution.merge_queue._merge_queue_group_ci import ( + _MUTATION_DISABLE_AUTO_MERGE, + _MUTATION_ENABLE_AUTO_MERGE, + _MUTATION_ENQUEUE_PR, + _QUERY, # noqa: F401 — re-export: tests assert merge_queue._QUERY exists + _QUERY_AUTO_MERGE_STATUS, + _query_merge_group_ci, # noqa: F401 — re-export: tests patch merge_queue._query_merge_group_ci +) +from autoskillit.execution.merge_queue._merge_queue_repo_state import ( + _GRAPHQL_ENDPOINT, + _RATE_LIMIT_MAX_ATTEMPTS, # noqa: F401 — re-export for callers + _RATE_LIMIT_SECONDARY_MARKER, # noqa: F401 — re-export for callers + _REPO_STATE_QUERY, # noqa: F401 — re-export for callers + _has_merge_group_trigger, # noqa: F401 — re-export for callers + _is_secondary_rate_limit, # noqa: F401 — re-export for callers + _push_trigger_applies_to_branch, # noqa: F401 — re-export for callers + _retry_after_seconds, # noqa: F401 — re-export for callers + _text_has_push_trigger, # noqa: F401 — re-export for callers + fetch_repo_merge_state, # noqa: F401 — re-export for callers +) + +if TYPE_CHECKING: + from autoskillit.core import GitHubApiLog + +logger = get_logger(__name__) + + +class DefaultMergeQueueWatcher: + """Polls GitHub merge queue state until merged, ejected, stalled, or timed out. + + Never raises; all errors are returned as structured dicts. + """ + + def __init__( + self, + token: str | None | Callable[[], str | None], + *, + tracker: GitHubApiLog | None = None, + ) -> None: + self._token_factory: Callable[[], str | None] | None + self._tracker = tracker + if callable(token): + self._token_factory = token + self._client: httpx.AsyncClient | None = None + else: + self._token_factory = None + self._client = make_tracked_httpx_client( + self._tracker, + timeout=httpx.Timeout(30.0), + headers=github_headers(token), + limits=httpx.Limits(keepalive_expiry=60), + ) + + def _ensure_client(self) -> httpx.AsyncClient: + if self._client is None: + resolved = self._token_factory() if self._token_factory is not None else None + self._client = make_tracked_httpx_client( + self._tracker, + timeout=httpx.Timeout(30.0), + headers=github_headers(resolved), + limits=httpx.Limits(keepalive_expiry=60), + ) + return self._client + + async def aclose(self) -> None: + if self._client is not None: + await self._client.aclose() + + async def wait( + self, + pr_number: int, + target_branch: str, + repo: str | None = None, + cwd: str = ".", + timeout_seconds: int = 600, + poll_interval: int = 15, + stall_grace_period: int = 60, + max_stall_retries: int = 3, + not_in_queue_confirmation_cycles: int = 2, + max_inconclusive_retries: int = 5, + auto_merge_available: bool = True, + ) -> dict[str, Any]: + """Poll until PR is merged, ejected, stalled, dropped, or timeout expires. + + Returns {"success": bool, "pr_state": str, "reason": str, "stall_retries_attempted": int}. + """ + if not repo or "/" not in repo: + return { + "success": False, + "pr_state": PRState.ERROR.value, + "reason": f"Invalid repo format: {repo!r}. Expected 'owner/name'.", + "stall_retries_attempted": 0, + } + owner, repo_name = repo.split("/", 1) + + deadline = time.monotonic() + timeout_seconds + stall_retries_attempted: int = 0 + not_in_queue_cycles: int = 0 + inconclusive_count: int = 0 + ever_enrolled: bool = False + was_in_queue: bool = False + merge_group_ci_cache: str | None = None + pending_ejection: PRState | None = None + pending_reason: str = "" + + def _make_result( + success: bool, pr_state: PRState, reason: str, ejection_cause: str = "" + ) -> dict[str, Any]: + result: dict[str, Any] = { + "success": success, + "pr_state": pr_state.value, + "reason": reason, + "stall_retries_attempted": stall_retries_attempted, + } + if ejection_cause: + result["ejection_cause"] = ejection_cause + return result + + while time.monotonic() < deadline: + try: + state = await self._fetch_pr_and_queue_state( + pr_number, owner, repo_name, target_branch + ) + except Exception: + logger.warning("fetch_pr_and_queue_state failed, retrying", exc_info=True) + await asyncio.sleep(poll_interval) + continue + + if state["in_queue"] or state["auto_merge_present"]: + ever_enrolled = True + + if state["in_queue"]: + was_in_queue = True + merge_group_ci_cache = None + not_in_queue_cycles = 0 + inconclusive_count = 0 + if state["queue_state"] == "UNMERGEABLE": + pending_ejection = PRState.EJECTED + pending_reason = "PR is UNMERGEABLE in merge queue" + else: + pending_ejection = None + pending_reason = "" + await asyncio.sleep(poll_interval) + continue + + not_in_queue_cycles += 1 + + if was_in_queue and state["checks_state"] not in ("FAILURE", "ERROR"): + merge_group_ci_cache = await _query_merge_group_ci( + repo=repo, + pr_number=pr_number, + base_branch=target_branch, + github_token=None, + ) + # Only stop retrying once we have a result or the confirmation window expires. + # If _query_merge_group_ci returns None (run not yet visible), we keep + # was_in_queue=True to retry on the next poll cycle. + if ( + merge_group_ci_cache is not None + or not_in_queue_cycles >= not_in_queue_confirmation_cycles + ): + was_in_queue = False + else: + was_in_queue = False + if merge_group_ci_cache is not None: + state = cast( + PRFetchState, {**state, "merge_group_checks_state": merge_group_ci_cache} + ) + + try: + classification = _classify_pr_state(state, ever_enrolled=ever_enrolled) + except CIStillRunning: + await asyncio.sleep(poll_interval) + continue + except NoPositiveSignal as exc: + if not_in_queue_cycles < not_in_queue_confirmation_cycles: + await asyncio.sleep(poll_interval) + continue + inconclusive_count += 1 + if inconclusive_count >= max_inconclusive_retries: + return _make_result( + False, + PRState.TIMEOUT, + f"Inconclusive after {max_inconclusive_retries} retries: {exc.reason}", + ) + await asyncio.sleep(poll_interval) + continue + + if classification.terminal == PRState.MERGED: + return _make_result(True, PRState.MERGED, classification.reason) + + if not_in_queue_cycles < not_in_queue_confirmation_cycles: + await asyncio.sleep(poll_interval) + continue + + if pending_ejection is not None: + return _make_result(False, pending_ejection, pending_reason) + + if state["state"] == "CLOSED": + ejection_cause = ( + "ci_failure" if classification.terminal == PRState.EJECTED_CI_FAILURE else "" + ) + return _make_result( + False, classification.terminal, classification.reason, ejection_cause + ) + + if classification.terminal == PRState.STALLED: + enabled_at = state["auto_merge_enabled_at"] + assert enabled_at is not None # guaranteed by _is_positive_stall + now = datetime.now(UTC) + stall_duration = max(0.0, (now - enabled_at).total_seconds()) + if stall_duration < stall_grace_period: + await asyncio.sleep(poll_interval) + continue + if stall_retries_attempted < max_stall_retries: + backoff = min(30 * (2**stall_retries_attempted), 120) + logger.info( + "merge_queue_stall_detected", + stall_duration=stall_duration, + attempt=stall_retries_attempted + 1, + backoff=backoff, + ) + try: + await self._toggle_auto_merge( + state["pr_node_id"], + auto_merge_available=auto_merge_available, + ) + except Exception: + logger.warning("toggle_auto_merge failed", exc_info=True) + stall_retries_attempted += 1 + not_in_queue_cycles = 0 + await asyncio.sleep(backoff) + continue + return _make_result( + False, + PRState.STALLED, + f"PR #{pr_number} stall unresolved after {max_stall_retries} toggle attempts", + ) + + elif classification.terminal == PRState.EJECTED_CI_FAILURE: + return _make_result( + False, + PRState.EJECTED_CI_FAILURE, + classification.reason, + ejection_cause="ci_failure", + ) + + elif classification.terminal == PRState.EJECTED: + return _make_result(False, PRState.EJECTED, classification.reason) + + elif classification.terminal == PRState.DROPPED_HEALTHY: + return _make_result(False, PRState.DROPPED_HEALTHY, classification.reason) + + elif classification.terminal == PRState.DROPPED_MERGE_GROUP_CI: + return _make_result(False, PRState.DROPPED_MERGE_GROUP_CI, classification.reason) + + elif classification.terminal == PRState.NOT_ENROLLED: + return _make_result(False, PRState.NOT_ENROLLED, classification.reason) + + else: + # Unreachable: _classify_pr_state never returns TIMEOUT or ERROR. + # The assert_never call provides static exhaustiveness for future + # additions to PRState (pyright/mypy will flag any new unhandled member). + assert_never(classification.terminal) # type: ignore[arg-type] + + return _make_result( + False, + PRState.TIMEOUT, + f"Timed out after {timeout_seconds}s waiting for PR #{pr_number}", + ) + + async def _fetch_pr_and_queue_state( + self, pr_number: int, owner: str, repo: str, target_branch: str + ) -> PRFetchState: + """Single GraphQL round-trip returning PR state and merge queue entries.""" + variables = { + "owner": owner, + "repo": repo, + "prNumber": pr_number, + "branch": target_branch, + } + resp = await self._ensure_client().post( + _GRAPHQL_ENDPOINT, json={"query": _QUERY, "variables": variables} + ) + resp.raise_for_status() + data = resp.json() + if "errors" in data: + raise RuntimeError(f"GraphQL error: {data['errors']}") + + if not data.get("data") or data["data"] is None or "repository" not in data["data"]: + raise RuntimeError(f"GraphQL response missing 'repository' key: {str(data)[:200]}") + repo_data = data["data"]["repository"] + pr = repo_data["pullRequest"] + if pr is None: + raise RuntimeError(f"GraphQL returned null pullRequest for PR #{pr_number}") + entries_raw = repo_data.get("mergeQueue") or {} + entries = entries_raw if isinstance(entries_raw, dict) else {} + nodes_raw = (entries.get("entries") or {}).get("nodes") + nodes = nodes_raw if isinstance(nodes_raw, list) else [] + + auto_merge_raw = pr.get("autoMergeRequest") + auto_merge_present: bool = auto_merge_raw is not None + auto_merge = auto_merge_raw or {} + enabled_at_raw = auto_merge.get("enabledAt") + enabled_at: datetime | None = None + if enabled_at_raw: + try: + enabled_at = datetime.fromisoformat(enabled_at_raw.replace("Z", "+00:00")) + except ValueError as e: + raise RuntimeError( + f"Unexpected autoMergeRequest.enabledAt format: {enabled_at_raw!r}" + ) from e + + checks_rollup = pr.get("statusCheckRollup") or {} + checks_state: str | None = checks_rollup.get("state") + + queue_entry = next((n for n in nodes if n["pullRequest"]["number"] == pr_number), None) + return PRFetchState( + merged=pr["merged"], + state=pr["state"], + mergeable=pr.get("mergeable") or "UNKNOWN", + merge_state_status=pr.get("mergeStateStatus", "UNKNOWN"), + auto_merge_present=auto_merge_present, + auto_merge_enabled_at=enabled_at, + pr_node_id=pr["id"], + in_queue=queue_entry is not None, + queue_state=queue_entry["state"] if queue_entry else None, + checks_state=checks_state, + merge_group_checks_state=None, + ) + + async def toggle( + self, + pr_number: int, + target_branch: str, + repo: str | None = None, + cwd: str = ".", + ) -> dict[str, Any]: + """Toggle auto-merge off/on to re-enroll the PR. Never raises.""" + if not repo or "/" not in repo: + return { + "success": False, + "error": f"Invalid repo format: {repo!r}. Expected 'owner/name'.", + } + owner, repo_name = repo.split("/", 1) + try: + state = await self._fetch_pr_and_queue_state( + pr_number, owner, repo_name, target_branch + ) + await self._toggle_auto_merge(state["pr_node_id"]) + return {"success": True, "pr_number": pr_number} + except Exception as exc: + logger.warning("toggle_auto_merge failed", exc_info=True) + return {"success": False, "error": f"toggle failed: {exc}"} + + async def _enqueue_direct(self, pr_node_id: str) -> None: + """Enqueue a PR directly via the enqueuePullRequest GraphQL mutation.""" + if not pr_node_id: + raise ValueError("pr_node_id must be a non-empty string") + resp = await self._ensure_client().post( + _GRAPHQL_ENDPOINT, + json={"query": _MUTATION_ENQUEUE_PR, "variables": {"prId": pr_node_id}}, + ) + resp.raise_for_status() + body = resp.json() + if "errors" in body: + raise RuntimeError(f"GraphQL mutation error: {body['errors']}") + + async def _enable_auto_merge_direct(self, pr_node_id: str) -> None: + """Enable auto-merge for a PR via the enablePullRequestAutoMerge mutation.""" + if not pr_node_id: + raise ValueError("pr_node_id must be a non-empty string") + resp = await self._ensure_client().post( + _GRAPHQL_ENDPOINT, + json={ + "query": _MUTATION_ENABLE_AUTO_MERGE, + "variables": {"prId": pr_node_id, "mergeMethod": "SQUASH"}, + }, + ) + resp.raise_for_status() + body = resp.json() + if "errors" in body: + raise RuntimeError(f"GraphQL mutation error: {body['errors']}") + + async def enqueue( + self, + pr_number: int, + target_branch: str, + repo: str | None = None, + cwd: str = ".", + auto_merge_available: bool = True, + ) -> dict[str, Any]: + """Enqueue a PR (via auto-merge or direct enqueue). Never raises.""" + if not repo or "/" not in repo: + return { + "success": False, + "error": f"Invalid repo format: {repo!r}. Expected 'owner/name'.", + } + owner, repo_name = repo.split("/", 1) + if not owner or not repo_name: + return { + "success": False, + "error": f"Invalid repo format: {repo!r}. Expected 'owner/name'.", + } + try: + state = await self._fetch_pr_and_queue_state( + pr_number, owner, repo_name, target_branch + ) + pr_node_id = state["pr_node_id"] + if auto_merge_available: + await self._enable_auto_merge_direct(pr_node_id) + enrollment_method = "auto_merge" + else: + await self._enqueue_direct(pr_node_id) + enrollment_method = "direct_enqueue" + return { + "success": True, + "pr_number": pr_number, + "enrollment_method": enrollment_method, + } + except Exception as exc: + logger.warning("enqueue failed", exc_info=True) + return {"success": False, "error": f"enqueue failed: {exc}"} + + async def _toggle_auto_merge( + self, pr_node_id: str, *, auto_merge_available: bool = True + ) -> None: + """Re-enroll a PR: toggle disable/re-enable or enqueuePullRequest directly.""" + if not pr_node_id: + raise ValueError("pr_node_id must be a non-empty string") + if not auto_merge_available: + await self._enqueue_direct(pr_node_id) + return + disable_payload = { + "query": _MUTATION_DISABLE_AUTO_MERGE, + "variables": {"prId": pr_node_id}, + } + resp = await self._ensure_client().post(_GRAPHQL_ENDPOINT, json=disable_payload) + resp.raise_for_status() + body = resp.json() + if "errors" in body: + raise RuntimeError(f"GraphQL mutation error: {body['errors']}") + await self._confirm_disable(pr_node_id) + await self._enable_auto_merge_direct(pr_node_id) + + async def _confirm_disable(self, pr_node_id: str) -> None: + """Poll until auto-merge disable is confirmed or timeout (best-effort).""" + payload = {"query": _QUERY_AUTO_MERGE_STATUS, "variables": {"nodeId": pr_node_id}} + for _ in range(6): + await asyncio.sleep(5) + resp = await self._ensure_client().post(_GRAPHQL_ENDPOINT, json=payload) + if resp.status_code != 200: + logger.warning( + "confirm_disable_poll_error", + pr_node_id=pr_node_id, + status_code=resp.status_code, + ) + continue + try: + data = resp.json().get("data", {}) + except (ValueError, httpx.DecodingError): + continue + if not (data.get("node") or {}).get("autoMergeRequest"): + return + logger.warning("confirm_disable_timeout", pr_node_id=pr_node_id) diff --git a/src/autoskillit/execution/merge_queue/_merge_queue_classifier.py b/src/autoskillit/execution/merge_queue/_merge_queue_classifier.py new file mode 100644 index 0000000000..5a59184a55 --- /dev/null +++ b/src/autoskillit/execution/merge_queue/_merge_queue_classifier.py @@ -0,0 +1,175 @@ +"""PR state classifier primitives — private sub-module of execution/merge_queue.py.""" + +from __future__ import annotations + +from dataclasses import dataclass +from datetime import datetime +from typing import TypedDict + +from autoskillit.core import PRState, get_logger + +logger = get_logger(__name__) + +# https://docs.github.com/en/graphql/reference/enums#mergestatestatus +KNOWN_MQ_MERGE_STATE_STATUSES: frozenset[str] = frozenset( + { + "BEHIND", + "BLOCKED", + "CLEAN", + "DIRTY", + "HAS_HOOKS", + "UNKNOWN", + "UNSTABLE", + } +) +assert "CLEAN" in KNOWN_MQ_MERGE_STATE_STATUSES # Import-time drift guard + + +class PRFetchState(TypedDict): + """Typed contract for _fetch_pr_and_queue_state return value.""" + + merged: bool + state: str + mergeable: str # "MERGEABLE" | "CONFLICTING" | "UNKNOWN" + merge_state_status: str + auto_merge_present: bool # True when autoMergeRequest is not null + auto_merge_enabled_at: datetime | None + pr_node_id: str + in_queue: bool + queue_state: str | None + checks_state: str | None # statusCheckRollup.state; None = no checks configured + merge_group_checks_state: str | None # None = not queried; populated on in_queue True→False + + +# Maps PRFetchState keys to GraphQL source paths. Validated at import time. +_QUERY_FIELD_MAP: dict[str, str] = { + "merged": "merged", + "state": "state", + "mergeable": "mergeable", + "merge_state_status": "mergeStateStatus", + "auto_merge_present": "autoMergeRequest", + "auto_merge_enabled_at": "autoMergeRequest.enabledAt", + "pr_node_id": "id", + "in_queue": "", + "queue_state": "", + "checks_state": "statusCheckRollup.state", + "merge_group_checks_state": "", +} + +_ALL_FETCH_STATE_KEYS = PRFetchState.__required_keys__ | PRFetchState.__optional_keys__ +if set(_QUERY_FIELD_MAP) != _ALL_FETCH_STATE_KEYS: + raise RuntimeError( + "_QUERY_FIELD_MAP is out of sync with PRFetchState keys.\n" + f"Missing from map: {_ALL_FETCH_STATE_KEYS - set(_QUERY_FIELD_MAP)}\n" + f"Missing from state: {set(_QUERY_FIELD_MAP) - _ALL_FETCH_STATE_KEYS}" + ) + + +@dataclass(frozen=True, slots=True) +class ClassificationResult: + """Positive-signal classification outcome from _classify_pr_state.""" + + terminal: PRState + reason: str + + +class ClassifierInconclusive(Exception): + """Raised when no positive gate matched — caller must continue polling.""" + + def __init__(self, state: PRFetchState, reason: str) -> None: + super().__init__(reason) + self.state = state + self.reason = reason + + +class CIStillRunning(ClassifierInconclusive): + """CI checks legitimately in-progress — must NOT consume the inconclusive budget.""" + + +class NoPositiveSignal(ClassifierInconclusive): + """No positive gate matched — counts against the inconclusive budget.""" + + +def _is_positive_stall(state: PRFetchState) -> bool: + """True when auto-merge is enabled and merge_state_status is CLEAN or HAS_HOOKS.""" + return state["auto_merge_enabled_at"] is not None and state["merge_state_status"] in { + "CLEAN", + "HAS_HOOKS", + } + + +def _is_positive_dropped_healthy(state: PRFetchState, *, ever_enrolled: bool) -> bool: + """True when the PR is fully healthy but auto_merge was cleared externally.""" + if not ever_enrolled: + return False + return ( + state["state"] == "OPEN" + and state["mergeable"] == "MERGEABLE" + and state["merge_state_status"] == "CLEAN" + and state["checks_state"] in (None, "SUCCESS") + and state["auto_merge_present"] is False + and state["in_queue"] is False + ) + + +def _is_positive_dropped_merge_group_ci(state: PRFetchState, *, ever_enrolled: bool) -> bool: + """True when PR dropped from queue with CI failure; extends _is_positive_dropped_healthy.""" + return _is_positive_dropped_healthy(state, ever_enrolled=ever_enrolled) and state[ + "merge_group_checks_state" + ] in ("FAILURE", "ERROR") + + +def _is_not_enrolled(state: PRFetchState, *, ever_enrolled: bool) -> bool: + """True when the PR is healthy but was never enrolled in the merge queue.""" + if ever_enrolled: + return False + return ( + state["state"] == "OPEN" + and state["mergeable"] == "MERGEABLE" + and state["merge_state_status"] == "CLEAN" + and state["checks_state"] in (None, "SUCCESS") + and state["auto_merge_present"] is False + and state["in_queue"] is False + ) + + +def _classify_pr_state(state: PRFetchState, *, ever_enrolled: bool) -> ClassificationResult: + """Classify PR state using positive signals only. + + Raises ClassifierInconclusive when no positive gate matches. + """ + if state["merged"]: + return ClassificationResult(PRState.MERGED, "PR merged") + + if state["state"] == "CLOSED": + if state["checks_state"] in {"FAILURE", "ERROR"}: + return ClassificationResult(PRState.EJECTED_CI_FAILURE, "PR closed after CI failure") + return ClassificationResult(PRState.EJECTED, "PR closed while not merged") + + if state["checks_state"] in {"FAILURE", "ERROR"}: + return ClassificationResult(PRState.EJECTED_CI_FAILURE, "checks terminal failure") + + if _is_positive_stall(state): + return ClassificationResult(PRState.STALLED, "stall signals present") + + if state["mergeable"] == "CONFLICTING": + return ClassificationResult(PRState.EJECTED, "conflicting changes prevent merge") + + if _is_positive_dropped_merge_group_ci(state, ever_enrolled=ever_enrolled): + return ClassificationResult( + PRState.DROPPED_MERGE_GROUP_CI, + "merge-group CI failed; PR ejected from queue (merge-group check run FAILURE)", + ) + + if _is_positive_dropped_healthy(state, ever_enrolled=ever_enrolled): + return ClassificationResult(PRState.DROPPED_HEALTHY, "PR healthy but auto_merge cleared") + + if _is_not_enrolled(state, ever_enrolled=ever_enrolled): + return ClassificationResult( + PRState.NOT_ENROLLED, "PR was never enrolled in the merge queue" + ) + + if state["checks_state"] in {"PENDING", "EXPECTED"}: + raise CIStillRunning(state, "checks still running") + + raise NoPositiveSignal(state, "no positive signal matched") diff --git a/src/autoskillit/execution/merge_queue/_merge_queue_group_ci.py b/src/autoskillit/execution/merge_queue/_merge_queue_group_ci.py new file mode 100644 index 0000000000..4b90a96cff --- /dev/null +++ b/src/autoskillit/execution/merge_queue/_merge_queue_group_ci.py @@ -0,0 +1,139 @@ +"""Merge-group CI helpers and GraphQL mutation/query strings — private sub-module. + +Extracted from merge_queue.py to satisfy the 500-line size budget. +Import public symbols from autoskillit.execution.merge_queue, not here. +""" + +from __future__ import annotations + +import asyncio +import json +import os + +import regex as re + +from autoskillit.core import get_logger +from autoskillit.execution.merge_queue._merge_queue_classifier import _QUERY_FIELD_MAP + +logger = get_logger(__name__) + +_QUERY = """ +query GetPRAndQueueState($owner: String!, $repo: String!, $prNumber: Int!, $branch: String!) { + repository(owner: $owner, name: $repo) { + pullRequest(number: $prNumber) { + id + merged + state + mergeable + mergeStateStatus + autoMergeRequest { + enabledAt + } + statusCheckRollup { + state + } + } + mergeQueue(branch: $branch) { + entries(first: 100) { + nodes { + pullRequest { number } + state + } + } + } + } +} +""" + +# Part 2 of the _QUERY_FIELD_MAP validation: checks that every non-computed field +# path head appears as a GraphQL field name in _QUERY. +# Part 1 (keys match PRFetchState) lives in _merge_queue_classifier.py. +for _key, _path in _QUERY_FIELD_MAP.items(): + if _path.startswith("<"): + continue + _head = _path.split(".", 1)[0] + # Word-boundary search prevents "state" from matching inside "mergeStateStatus". + if not re.search(r"\b" + re.escape(_head) + r"\b", _QUERY): + raise RuntimeError( + f"_QUERY is missing GraphQL field {_head!r} required by PRFetchState[{_key!r}]" + ) + +_MUTATION_DISABLE_AUTO_MERGE = """ +mutation DisableAutoMerge($prId: ID!) { + disablePullRequestAutoMerge(input: {pullRequestId: $prId}) { + pullRequest { number } + } +} +""" + +_MUTATION_ENABLE_AUTO_MERGE = """ +mutation EnableAutoMerge($prId: ID!, $mergeMethod: PullRequestMergeMethod!) { + enablePullRequestAutoMerge(input: {pullRequestId: $prId, mergeMethod: $mergeMethod}) { + pullRequest { number } + } +} +""" + +_MUTATION_ENQUEUE_PR = """ +mutation EnqueuePullRequest($prId: ID!) { + enqueuePullRequest(input: {pullRequestId: $prId}) { + mergeQueueEntry { id } + } +} +""" + +_QUERY_AUTO_MERGE_STATUS = """ +query AutoMergeStatus($nodeId: ID!) { + node(id: $nodeId) { + ... on PullRequest { + autoMergeRequest { enabledAt } + } + } +} +""" + + +async def _query_merge_group_ci( + repo: str, + pr_number: int, + base_branch: str, + github_token: str | None, +) -> str | None: + """Query the most recent workflow run on the merge-group ref for this PR. + + Uses `gh run list` with branch prefix matching for the gh-readonly-queue ref. + Returns 'SUCCESS', 'FAILURE', or None (not found / still running / query failed). + Never raises. + """ + branch_prefix = f"gh-readonly-queue/{base_branch}/pr-{pr_number}-" + env = {**os.environ} + if github_token: + env["GH_TOKEN"] = github_token + try: + proc = await asyncio.create_subprocess_exec( + "gh", + "run", + "list", + "--repo", + repo, + "--json", + "conclusion,headBranch", + "--limit", + "10", + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + env=env, + ) + stdout, _ = await asyncio.wait_for(proc.communicate(), timeout=15) + runs: list[dict[str, str]] = json.loads(stdout.decode()) + for run in runs: + if run.get("headBranch", "").startswith(branch_prefix): + conclusion = run.get("conclusion", "") + if conclusion in ("failure", "cancelled", "timed_out", "action_required"): + return "FAILURE" + if conclusion == "success": + return "SUCCESS" + return None + except Exception as e: + logger.warning("_query_merge_group_ci failed: %s", type(e).__name__, exc_info=True) + return None diff --git a/src/autoskillit/execution/merge_queue/_merge_queue_repo_state.py b/src/autoskillit/execution/merge_queue/_merge_queue_repo_state.py new file mode 100644 index 0000000000..609dccee9b --- /dev/null +++ b/src/autoskillit/execution/merge_queue/_merge_queue_repo_state.py @@ -0,0 +1,270 @@ +"""Repository-level merge state helpers extracted from merge_queue.py. + +Contains the repo-state GraphQL query, push-trigger detection, and rate-limit +retry logic. Private sub-module — import from autoskillit.execution.merge_queue +for the public API. +""" + +from __future__ import annotations + +import asyncio +import fnmatch +import random +from typing import Any, Literal + +import httpx + +from autoskillit.core import YAMLError, get_logger, load_yaml +from autoskillit.execution.github import github_headers + +logger = get_logger(__name__) + +_GRAPHQL_ENDPOINT = "https://api.github.com/graphql" + +# Repo-level state query: consolidates three former run_cmd steps into one HTTP round-trip. +# Distinct from _QUERY (PR-level + queue-entries); do not merge these constants. +# Returns: mergeQueue presence, autoMergeAllowed flag, and workflow file texts for +# merge_group trigger detection — all in a single GraphQL call. +_REPO_STATE_QUERY = """ +query GetRepoMergeState($owner: String!, $repo: String!, $branch: String!) { + repository(owner: $owner, name: $repo) { + mergeQueue(branch: $branch) { + id + } + autoMergeAllowed + object(expression: "HEAD:.github/workflows") { + ... on Tree { + entries { + name + object { + ... on Blob { + text + } + } + } + } + } + } +} +""" + +_RATE_LIMIT_MAX_ATTEMPTS = 3 +_RATE_LIMIT_SECONDARY_MARKER = "secondary rate limit" + + +def _text_has_push_trigger(text: str) -> bool: + """Return True if a workflow file text declares a push trigger. + + Checks for the three YAML forms GitHub supports for a push trigger: + - ``on: [push, ...]`` (list form) + - ``on: push`` (scalar form) + - ``push:`` (mapping form under ``on:``) + + Note: the first two patterns (``on: [push``, ``on: push``) are reliably + scoped to YAML trigger syntax. The third (``push:``) may match inside + comments or run-step values — acceptable for the positive-signal-only + classification heuristic but not precise. + """ + return any(pat in text for pat in ("on: [push", "on: push", "push:")) + + +def _push_trigger_applies_to_branch(text: str, branch: str) -> bool: + """Return True if the workflow push trigger fires for the given branch. + + Parses the YAML to inspect push.branches / push.branches-ignore filters. + Falls back to presence-only heuristic on YAML parse failure (safe for + ambiguous/binary blobs). Supports GitHub's fnmatch-compatible glob patterns + (e.g. 'feature/**', 'release-*'). + """ + try: + parsed = load_yaml(text) + except YAMLError: + return _text_has_push_trigger(text) + + if not isinstance(parsed, dict): + return False + + # PyYAML (YAML 1.1) parses the bare key `on` as boolean True. + # Accept both to be safe. + on_value = parsed.get(True, parsed.get("on")) + if on_value == "push": + return True + if isinstance(on_value, list): + return "push" in on_value + if not isinstance(on_value, dict) or "push" not in on_value: + return False + + push_cfg = on_value["push"] + if not isinstance(push_cfg, dict) or not push_cfg: + # push: null or push: {} — no branch filter, fires for all branches + return True + + branches = push_cfg.get("branches") + branches_ignore = push_cfg.get("branches-ignore") + + if branches is not None: + return any(fnmatch.fnmatch(branch, pat) for pat in branches) + if branches_ignore is not None: + return not any(fnmatch.fnmatch(branch, pat) for pat in branches_ignore) + return True + + +def _has_merge_group_trigger(text: str) -> bool: + """Return True if the workflow declares a merge_group trigger. + + Parses YAML to inspect the on: key rather than relying on substring + matching, which can false-positive on comments or shell strings. + Falls back to substring heuristic on YAML parse failure. + """ + try: + parsed = load_yaml(text) + except YAMLError: + return "merge_group" in text + if not isinstance(parsed, dict): + return False + # PyYAML (YAML 1.1) parses the bare key `on` as boolean True. + # Accept both to be safe. + on_value = parsed.get(True, parsed.get("on")) + if on_value == "merge_group": + return True + if isinstance(on_value, list): + return "merge_group" in on_value + if isinstance(on_value, dict): + return "merge_group" in on_value + return False + + +def _is_secondary_rate_limit(resp: httpx.Response) -> bool: + """Return True when a 403 response is a GitHub secondary rate limit. + + GitHub returns HTTP 403 (not 429) for secondary rate limits. + The response body contains the phrase "secondary rate limit". + Primary rate limits use HTTP 429 or include x-ratelimit-remaining: 0. + """ + if resp.status_code != 403: + return False + try: + text = resp.text.lower() + except Exception: + logger.warning("Failed to read response body for rate-limit check", exc_info=True) + return False + return _RATE_LIMIT_SECONDARY_MARKER in text + + +def _retry_after_seconds(attempt: int, resp: httpx.Response) -> float: + """Return seconds to sleep before the next retry attempt. + + Prefers the Retry-After header (integer seconds) when present and valid. + Falls back to full-jitter exponential backoff: random(0, min(60, 1 * 2^attempt)). + """ + try: + header_val = resp.headers.get("Retry-After", "") + if header_val: + return float(header_val) + except (ValueError, AttributeError): + pass + return random.uniform(0, min(60.0, 1.0 * (2**attempt))) + + +async def fetch_repo_merge_state( + owner: str, + repo: str, + branch: str, + token: str | None, +) -> dict[str, bool | str | None]: + """Fetch repository merge-state in a single GraphQL round-trip. + + Returns a dict with four keys: + - ``queue_available``: the branch has an active GitHub merge queue (bool). + - ``merge_group_trigger``: at least one CI workflow declares the + ``merge_group`` event trigger (bool). + - ``auto_merge_available``: the repository has auto-merge enabled (bool). + - ``ci_event``: ``"push"`` when any workflow declares a push trigger + that fires for the given branch, or ``None`` otherwise (match-any — + ci.py scope.event=None lets head_sha provide correctness). + + Null-handling: + - ``mergeQueue is null`` → ``queue_available: False`` (no queue) + - ``object is null`` → ``merge_group_trigger: False``, ``ci_event: None`` (no workflows dir) + - ``entry.object.text is null`` → skip entry (binary/large file) + - GraphQL ``autoMergeAllowed`` field error (GHES 3.0.x) → ``auto_merge_available: False`` + + Only transport-level failures (network timeout, non-200 HTTP status) are + allowed to propagate; callers are expected to handle them. + + Historical note: Issue #498 ("Merge queue detection should validate workflow has + merge_group trigger") established the merge_group_trigger field. The ci_event + field is a closely related extension — verify that the push-trigger scan does + not regress the merge_group-only detection that #498 established. + """ + resp: httpx.Response | None = None + for attempt in range(_RATE_LIMIT_MAX_ATTEMPTS): + async with httpx.AsyncClient( + headers=github_headers(token), + timeout=30.0, + ) as client: + resp = await client.post( + _GRAPHQL_ENDPOINT, + json={ + "query": _REPO_STATE_QUERY, + "variables": {"owner": owner, "repo": repo, "branch": branch}, + }, + ) + if resp.status_code == 429 or _is_secondary_rate_limit(resp): + sleep_secs = _retry_after_seconds(attempt, resp) + logger.warning( + "fetch_repo_merge_state rate limited", + status=resp.status_code, + attempt=attempt, + sleep_secs=sleep_secs, + ) + await asyncio.sleep(sleep_secs) + continue + resp.raise_for_status() + break + else: + assert resp is not None, "_RATE_LIMIT_MAX_ATTEMPTS must be >= 1" + resp.raise_for_status() + + assert resp is not None + body = resp.json() + + # GitHub GraphQL always returns a JSON object; guard against unexpected shapes. + if not isinstance(body, dict): + body = {} + + # Gracefully handle GHES 3.0.x where autoMergeAllowed doesn't exist. + auto_merge_field_missing = any( + "autoMergeAllowed" in str(e.get("message", "")) for e in body.get("errors", []) + ) + + repo_data: dict[str, Any] = (body.get("data") or {}).get("repository") or {} + queue_available = repo_data.get("mergeQueue") is not None + auto_merge_available = ( + False if auto_merge_field_missing else bool(repo_data.get("autoMergeAllowed", False)) + ) + + # Scan workflow files for push and merge_group trigger declarations. + # Both flags are derived from the same Blob.text scan — no extra round-trips. + merge_group_trigger = False + has_push_trigger = False + workflows_tree = repo_data.get("object") + if workflows_tree is not None: + for entry in workflows_tree.get("entries", []): + blob = entry.get("object") or {} + text = blob.get("text") + if text is None: + continue # binary or oversized blob — skip + if _has_merge_group_trigger(text): + merge_group_trigger = True + if _push_trigger_applies_to_branch(text, branch): + has_push_trigger = True + + ci_event: Literal["push"] | None = "push" if has_push_trigger else None + + return { + "queue_available": queue_available, + "merge_group_trigger": merge_group_trigger, + "auto_merge_available": auto_merge_available, + "ci_event": ci_event, + } diff --git a/src/autoskillit/execution/pr_analysis.py b/src/autoskillit/execution/pr_analysis.py index 8e71d689d5..386376ea66 100644 --- a/src/autoskillit/execution/pr_analysis.py +++ b/src/autoskillit/execution/pr_analysis.py @@ -8,9 +8,10 @@ from __future__ import annotations -import re from typing import Any +import regex as re + # --------------------------------------------------------------------------- # Linked issue extraction (used by review-pr fidelity subagent) # --------------------------------------------------------------------------- diff --git a/src/autoskillit/execution/process/CLAUDE.md b/src/autoskillit/execution/process/CLAUDE.md new file mode 100644 index 0000000000..ebc0faf7ec --- /dev/null +++ b/src/autoskillit/execution/process/CLAUDE.md @@ -0,0 +1,26 @@ +# process/ + +Subprocess lifecycle management — spawn, monitor, race, kill. + +## Files + +| File | Purpose | +|------|---------| +| `__init__.py` | Main module: `run_managed_async()`, `run_managed_sync()`, `DefaultSubprocessRunner` | +| `_process_io.py` | `create_temp_io()` context manager for temp file stdin/stdout/stderr | +| `_process_jsonl.py` | JSONL parsing: `_jsonl_contains_marker`, `_jsonl_has_record_type` | +| `_process_kill.py` | `kill_process_tree()` (sync) and `async_kill_process_tree()` (async): SIGTERM -> wait -> SIGKILL | +| `_process_monitor.py` | Async monitor coroutines: `_heartbeat()` (Channel A), `_session_log_monitor()` (Channel B) | +| `_process_pty.py` | `pty_wrap_command()` — wraps command with `script(1)` for PTY allocation | +| `_process_race.py` | `RaceAccumulator`, `RaceSignals`, watcher coroutines, `resolve_termination()` | + +## Architecture Notes + +**Two-channel completion detection:** + +- **Channel A** (stdout heartbeat): polls the subprocess stdout temp file for a `type=result` NDJSON record containing the completion marker. Guarantees stdout data is available. +- **Channel B** (session JSONL monitor): watches the Claude Code session JSONL log file (written by the Claude Code subprocess to its session log directory) for the completion marker in an `assistant`-type record. Provides an orthogonal confirmation signal and session ID discovery via the JSONL filename stem. + +Both channels race concurrently in an `anyio` task group. `resolve_termination()` reads the frozen `RaceSignals` and returns `(TerminationReason, ChannelConfirmation)`. Channel A takes precedence if both fire in the same tick. + +`execute_termination_action()` is the sole authorized caller of `async_kill_process_tree` (enforced by test). diff --git a/src/autoskillit/execution/process.py b/src/autoskillit/execution/process/__init__.py similarity index 84% rename from src/autoskillit/execution/process.py rename to src/autoskillit/execution/process/__init__.py index 2b0d2ae5f7..69bcd4feaf 100644 --- a/src/autoskillit/execution/process.py +++ b/src/autoskillit/execution/process/__init__.py @@ -13,7 +13,8 @@ import subprocess import time -from collections.abc import Mapping +from collections.abc import Callable, Mapping +from dataclasses import asdict from pathlib import Path from typing import TYPE_CHECKING, assert_never @@ -27,29 +28,32 @@ TerminationAction, TerminationReason, get_logger, + read_starttime_ticks, ) -from autoskillit.execution._process_io import create_temp_io, read_temp_output -from autoskillit.execution._process_jsonl import ( +from autoskillit.execution.process._process_io import create_temp_io, read_temp_output +from autoskillit.execution.process._process_jsonl import ( _jsonl_contains_marker, _jsonl_has_record_type, + _jsonl_last_record_type, _marker_is_standalone, ) -from autoskillit.execution._process_kill import ( +from autoskillit.execution.process._process_kill import ( _wait_process_dead, async_kill_process_tree, kill_process_tree, ) -from autoskillit.execution._process_monitor import ( +from autoskillit.execution.process._process_monitor import ( _has_active_api_connection, _has_active_child_processes, _heartbeat, _session_log_monitor, ) -from autoskillit.execution._process_pty import pty_wrap_command -from autoskillit.execution._process_race import ( +from autoskillit.execution.process._process_pty import pty_wrap_command +from autoskillit.execution.process._process_race import ( RaceAccumulator, RaceSignals, _extract_stdout_session_id, + _watch_child_activity, _watch_heartbeat, _watch_process, _watch_session_log, @@ -80,6 +84,7 @@ "_heartbeat", "_jsonl_contains_marker", "_jsonl_has_record_type", + "_jsonl_last_record_type", "_marker_is_standalone", "_session_log_monitor", "_wait_process_dead", @@ -193,10 +198,14 @@ async def run_managed_async( linux_tracing_config: LinuxTracingConfig | None = None, idle_output_timeout: float | None = None, max_suppression_seconds: float | None = None, + on_pid_resolved: Callable[[int, int], None] | None = None, + enable_deadline_extension: bool = False, + max_extension_seconds: float = 7200, _phase1_poll: float = 1.0, _phase2_poll: float = 2.0, _heartbeat_poll: float = 0.5, _phase1_timeout: float = 30.0, + _session_id_timeout: float = 1.0, ) -> SubprocessResult: """Async subprocess execution with temp file I/O and process tree cleanup. @@ -226,13 +235,16 @@ async def run_managed_async( stdin_handle = open(stdin_path) # noqa: SIM115 try: + # uvloop's Cython layer requires type(env) is dict — coerce at + # the external API boundary to preserve MappingProxyType internally. + _env: dict[str, str] | None = dict(env) if env is not None else None proc = await anyio.open_process( cmd, stdout=stdout_file, stderr=stderr_file, stdin=stdin_handle if stdin_handle is not None else subprocess.DEVNULL, cwd=cwd, - env=env, + env=_env, start_new_session=True, ) @@ -253,7 +265,7 @@ async def run_managed_async( if pty_mode and LINUX_TRACING_AVAILABLE: # PTY mode: proc.pid is the script(1) wrapper — resolve to workload - _target = resolve_trace_target( + _target = await resolve_trace_target( root_pid=proc.pid, expected_basename=_workload_basename, timeout=2.0, @@ -265,6 +277,14 @@ async def run_managed_async( _observed_pid = _target.pid _tracked_comm = _target.comm + if on_pid_resolved is not None and _observed_pid > 0: + _ticks: int = 0 + if _target is not None and hasattr(_target, "starttime_ticks"): + _ticks = _target.starttime_ticks or 0 + if _ticks == 0: + _ticks = read_starttime_ticks(_observed_pid) or 0 + on_pid_resolved(_observed_pid, _ticks) + termination = TerminationReason.NATURAL_EXIT _channel_confirmation = ChannelConfirmation.UNMONITORED @@ -283,7 +303,7 @@ async def run_managed_async( trigger = anyio.Event() channel_b_ready = anyio.Event() stdout_session_id_ready = anyio.Event() - timeout_scope = None # bound inside task group body; initialized for safety + timeout_scope_ref: list[anyio.CancelScope | None] = [None] async with anyio.create_task_group() as tg: tg.start_soon(_watch_process, proc, acc, trigger) @@ -318,6 +338,7 @@ async def run_managed_async( _phase1_poll, _phase2_poll, _phase1_timeout, + _session_id_timeout, stdout_session_id_ready, max_suppression_seconds, ) @@ -338,8 +359,19 @@ async def run_managed_async( config=linux_tracing_config, tg=tg, ) - with anyio.move_on_after(timeout) as timeout_scope: + if enable_deadline_extension and _observed_pid is not None: + tg.start_soon( + _watch_child_activity, + _observed_pid, + timeout_scope_ref, + max_extension_seconds, + trigger, + ) + timeout_scope: anyio.CancelScope | None + with anyio.move_on_after(timeout) as _ts: + timeout_scope_ref[0] = _ts await trigger.wait() + timeout_scope = timeout_scope_ref[0] # Symmetric drain: if the process exited before Channel B deposited, # give the session monitor a bounded window to complete its current # poll cycle and deposit its signal. @@ -367,13 +399,12 @@ async def run_managed_async( snapshots_data: list[dict[str, object]] | None = None if tracing_handle is not None: - from autoskillit.execution.linux_tracing import read_proc_snapshot - accumulated = tracing_handle.stop() - final_snap = read_proc_snapshot(_observed_pid) - if final_snap: - accumulated.append(final_snap) - snapshots_data = [s.__dict__ for s in accumulated] + snapshots_data = [asdict(s) for s in accumulated] + if signals.exit_snapshot is not None: + snapshots_data.append(signals.exit_snapshot) + elif signals.exit_snapshot is not None: + snapshots_data = [signals.exit_snapshot] if timeout_scope is not None and timeout_scope.cancelled_caught: termination = TerminationReason.TIMED_OUT @@ -419,6 +450,7 @@ async def run_managed_async( ), kill_reason=kill_reason, tracked_comm=_tracked_comm, + orphaned_tool_result=signals.channel_b_orphaned_tool_result, ) proc_log.debug( "run_managed_async_result", @@ -430,11 +462,16 @@ async def run_managed_async( ) return sub_result except BaseException: - # Ensure cleanup on unexpected errors (including CancelledError) - if "tracing_handle" in locals() and tracing_handle is not None: - tracing_handle.stop() # idempotent: flushes and closes trace file - if "proc" in locals() and proc.returncode is None: - kill_process_tree(proc.pid) + # Shielded cleanup: when a task is cancelled, the BaseException handler + # runs with cancellation active. Without shielding, the await in + # async_kill_process_tree would be immediately cancelled, leaking the + # subprocess tree. CancelScope(shield=True) suspends the outer cancel + # so cleanup completes before re-raising. + with anyio.CancelScope(shield=True): + if "tracing_handle" in locals() and tracing_handle is not None: + tracing_handle.stop() + if "proc" in locals() and proc.returncode is None: + await async_kill_process_tree(proc.pid) raise finally: if stdin_handle is not None: @@ -469,13 +506,14 @@ def run_managed_sync( process = None try: + _env: dict[str, str] | None = dict(env) if env is not None else None process = subprocess.Popen( cmd, stdout=stdout_file, stderr=stderr_file, stdin=stdin_handle if stdin_handle is not None else subprocess.DEVNULL, cwd=cwd, - env=env, + env=_env, start_new_session=True, ) @@ -536,6 +574,9 @@ async def __call__( linux_tracing_config: LinuxTracingConfig | None = None, idle_output_timeout: float | None = None, max_suppression_seconds: float | None = None, + on_pid_resolved: Callable[[int, int], None] | None = None, + enable_deadline_extension: bool = False, + max_extension_seconds: float = 7200, ) -> SubprocessResult: return await run_managed_async( cmd, @@ -551,4 +592,7 @@ async def __call__( linux_tracing_config=linux_tracing_config, idle_output_timeout=idle_output_timeout, max_suppression_seconds=max_suppression_seconds, + on_pid_resolved=on_pid_resolved, + enable_deadline_extension=enable_deadline_extension, + max_extension_seconds=max_extension_seconds, ) diff --git a/src/autoskillit/execution/_process_io.py b/src/autoskillit/execution/process/_process_io.py similarity index 100% rename from src/autoskillit/execution/_process_io.py rename to src/autoskillit/execution/process/_process_io.py diff --git a/src/autoskillit/execution/_process_jsonl.py b/src/autoskillit/execution/process/_process_jsonl.py similarity index 74% rename from src/autoskillit/execution/_process_jsonl.py rename to src/autoskillit/execution/process/_process_jsonl.py index 26c9abf1c7..17e57c126b 100644 --- a/src/autoskillit/execution/_process_jsonl.py +++ b/src/autoskillit/execution/process/_process_jsonl.py @@ -2,7 +2,8 @@ from __future__ import annotations -from autoskillit.core import get_logger +from autoskillit.core import ClaudeContentBlockType, get_logger +from autoskillit.core import fast_loads as _fast_loads logger = get_logger(__name__) @@ -29,16 +30,16 @@ def _jsonl_contains_marker( This prevents false-fires when the model quotes the marker directive in discussion (e.g. ``"I will emit %%AUTOSKILLIT_COMPLETE%% when done"``). + Thinking blocks are excluded — a marker inside a thinking block is internal + reasoning, not a structural completion signal. """ - import json as _json - for line in content.splitlines(): line = line.strip() if not line: continue try: - obj = _json.loads(line) - except (ValueError, _json.JSONDecodeError): + obj = _fast_loads(line) + except ValueError: continue if not isinstance(obj, dict): continue @@ -54,7 +55,13 @@ def _jsonl_contains_marker( raw = " ".join(v for v in obj.values() if isinstance(v, str)) if isinstance(raw, list): - text = "\n".join(b.get("text", "") if isinstance(b, dict) else str(b) for b in raw) + text = "\n".join( + b.get("text", "") + for b in raw + if isinstance(b, dict) + and ClaudeContentBlockType.from_api(b.get("type", "")) + == ClaudeContentBlockType.TEXT + ) elif not isinstance(raw, str): text = "" if raw is None else str(raw) else: @@ -81,15 +88,13 @@ def _jsonl_has_record_type( This prevents Channel A from confirming on premature exits where the model produced output but did not complete its task. """ - import json as _json - for line in content.splitlines(): line = line.strip() if not line: continue try: - obj = _json.loads(line) - except (ValueError, _json.JSONDecodeError): + obj = _fast_loads(line) + except ValueError: continue if not isinstance(obj, dict): continue @@ -104,3 +109,21 @@ def _jsonl_has_record_type( continue # marker configured but absent — do not confirm return True return False + + +def _jsonl_last_record_type(content: str) -> str | None: + """Return the type field of the last parseable JSONL record in content, or None.""" + last_type: str | None = None + for line in content.splitlines(): + line = line.strip() + if not line: + continue + try: + obj = _fast_loads(line) + except ValueError: + continue + if isinstance(obj, dict): + t = obj.get("type") + if isinstance(t, str): + last_type = t + return last_type diff --git a/src/autoskillit/execution/_process_kill.py b/src/autoskillit/execution/process/_process_kill.py similarity index 100% rename from src/autoskillit/execution/_process_kill.py rename to src/autoskillit/execution/process/_process_kill.py diff --git a/src/autoskillit/execution/_process_monitor.py b/src/autoskillit/execution/process/_process_monitor.py similarity index 95% rename from src/autoskillit/execution/_process_monitor.py rename to src/autoskillit/execution/process/_process_monitor.py index ae443902a9..3503664a97 100644 --- a/src/autoskillit/execution/_process_monitor.py +++ b/src/autoskillit/execution/process/_process_monitor.py @@ -10,7 +10,11 @@ import psutil from autoskillit.core import ChannelBStatus, get_logger -from autoskillit.execution._process_jsonl import _jsonl_contains_marker, _jsonl_has_record_type +from autoskillit.execution.process._process_jsonl import ( + _jsonl_contains_marker, + _jsonl_has_record_type, + _jsonl_last_record_type, +) logger = get_logger(__name__) @@ -20,6 +24,7 @@ class SessionMonitorResult(NamedTuple): status: ChannelBStatus session_id: str # Claude Code session ID from JSONL filename stem, or "" + orphaned_tool_result: bool = False async def _heartbeat( @@ -245,6 +250,7 @@ async def _session_log_monitor( os_error_count = 0 suppression_start_api: float | None = None suppression_start_child: float | None = None + _last_record_type: str | None = None while True: await anyio.sleep(_phase2_poll) @@ -278,6 +284,9 @@ async def _session_log_monitor( scan_pos=scan_pos, ) return SessionMonitorResult(ChannelBStatus.COMPLETION, _session_id) + last_type_in_chunk = _jsonl_last_record_type(new_content) + if last_type_in_chunk is not None: + _last_record_type = last_type_in_chunk except OSError: pass else: @@ -325,4 +334,8 @@ async def _session_log_monitor( pid, ) else: - return SessionMonitorResult(ChannelBStatus.STALE, _session_id) + return SessionMonitorResult( + ChannelBStatus.STALE, + _session_id, + orphaned_tool_result=(_last_record_type == "user"), + ) diff --git a/src/autoskillit/execution/_process_pty.py b/src/autoskillit/execution/process/_process_pty.py similarity index 100% rename from src/autoskillit/execution/_process_pty.py rename to src/autoskillit/execution/process/_process_pty.py diff --git a/src/autoskillit/execution/_process_race.py b/src/autoskillit/execution/process/_process_race.py similarity index 78% rename from src/autoskillit/execution/_process_race.py rename to src/autoskillit/execution/process/_process_race.py index 2fdd9bd20d..16613c47d6 100644 --- a/src/autoskillit/execution/_process_race.py +++ b/src/autoskillit/execution/process/_process_race.py @@ -2,8 +2,7 @@ from __future__ import annotations -import json -from dataclasses import dataclass, field +from dataclasses import asdict, dataclass, field from pathlib import Path from typing import assert_never @@ -11,12 +10,18 @@ import anyio.abc from autoskillit.core import ChannelBStatus, ChannelConfirmation, TerminationReason, get_logger -from autoskillit.execution._process_monitor import _heartbeat, _session_log_monitor +from autoskillit.core import fast_loads as _fast_loads +from autoskillit.execution.process._process_monitor import ( + _has_active_api_connection, + _has_active_child_processes, + _heartbeat, + _session_log_monitor, +) logger = get_logger(__name__) -@dataclass(frozen=True) +@dataclass(frozen=True, slots=True) class RaceSignals: """Accumulated signals produced by the anyio task group race in run_managed_async. @@ -36,7 +41,9 @@ class RaceSignals: channel_b_session_id: str = "" # Claude Code session ID from JSONL filename stem, or "" stdout_session_id: str | None = None # Session ID extracted from stdout type=system record idle_stall: bool = False + channel_b_orphaned_tool_result: bool = False process_exited_event: anyio.Event = field(default_factory=anyio.Event) + exit_snapshot: dict[str, object] | None = None @dataclass @@ -60,7 +67,9 @@ class RaceAccumulator: channel_b_session_id: str = "" stdout_session_id: str | None = None idle_stall: bool = False + channel_b_orphaned_tool_result: bool = False process_exited_event: anyio.Event = field(default_factory=anyio.Event) + exit_snapshot: dict[str, object] | None = None def to_race_signals(self) -> RaceSignals: return RaceSignals( @@ -71,7 +80,9 @@ def to_race_signals(self) -> RaceSignals: channel_b_session_id=self.channel_b_session_id, stdout_session_id=self.stdout_session_id, idle_stall=self.idle_stall, + channel_b_orphaned_tool_result=self.channel_b_orphaned_tool_result, process_exited_event=self.process_exited_event, + exit_snapshot=self.exit_snapshot, ) @@ -88,6 +99,20 @@ async def _watch_process( """ await proc.wait() logger.debug("process_exited", pid=proc.pid, returncode=proc.returncode) + # Exit snapshot: best-effort capture at exact exit time. + # waitpid() reaps the process atomically, so /proc/[pid] is already gone for + # normally-exiting processes — acc.exit_snapshot will be None in most cases. + try: + # Deferred import: linux_tracing depends on psutil and reads /proc, which is + # Linux-only. Importing at module level would fail on non-Linux platforms where + # LINUX_TRACING_AVAILABLE is False. The bare except below degrades gracefully. + from autoskillit.execution.linux_tracing import read_proc_snapshot + + snap = read_proc_snapshot(proc.pid) + if snap is not None: + acc.exit_snapshot = {**asdict(snap), "event": "exit_snapshot"} + except Exception: + logger.debug("exit_snapshot_failed", pid=proc.pid, exc_info=True) acc.process_exited = True acc.process_returncode = proc.returncode acc.process_exited_event.set() @@ -153,6 +178,57 @@ async def _watch_stdout_idle( return +async def _watch_child_activity( + pid: int, + timeout_scope_ref: list[anyio.CancelScope | None], + max_extension_seconds: float, + trigger: anyio.Event, + _poll_interval: float = 30.0, +) -> None: + """Extend the wall-clock CancelScope.deadline when child processes are active. + + Polls _has_active_child_processes and _has_active_api_connection every + _poll_interval seconds. When either returns True, pushes + timeout_scope.deadline forward (up to max_extension_seconds beyond the + original deadline). + + Terminates when trigger fires (session completed normally). Crash is + fail-closed — anyio propagates exceptions in the task group, cancelling + siblings. + """ + _first_observed_deadline: float | None = None + + while not trigger.is_set(): + await anyio.sleep(_poll_interval) + if trigger.is_set(): + return + + scope = timeout_scope_ref[0] + if scope is None: + continue + + if _first_observed_deadline is None: + _first_observed_deadline = scope.deadline + + active = _has_active_child_processes(pid) or _has_active_api_connection(pid) + if not active: + continue + + cap = _first_observed_deadline + max_extension_seconds + desired = anyio.current_time() + _poll_interval * 2 + new_deadline = min(desired, cap) + if new_deadline > scope.deadline: + logger.debug( + "deadline_extended", + extension=new_deadline - scope.deadline, + new_deadline=new_deadline, + cap=cap, + ) + scope.deadline = new_deadline + if trigger.is_set(): + return + + async def _extract_stdout_session_id( stdout_path: Path, acc: RaceAccumulator, @@ -186,8 +262,8 @@ async def _extract_stdout_session_id( if not line: continue try: - obj = json.loads(line) - except (json.JSONDecodeError, ValueError): + obj = _fast_loads(line) + except ValueError: continue if isinstance(obj, dict) and obj.get("type") == "system": sid = obj.get("session_id") @@ -214,6 +290,7 @@ async def _watch_session_log( _phase1_poll: float, _phase2_poll: float, _phase1_timeout: float, + _session_id_timeout: float = 1.0, stdout_session_id_ready: anyio.Event | None = None, max_suppression_seconds: float | None = None, ) -> None: @@ -227,7 +304,7 @@ async def _watch_session_log( extraction before starting Phase 1 to enable identity-based JSONL selection. """ if stdout_session_id_ready is not None: - with anyio.move_on_after(1.0): + with anyio.move_on_after(_session_id_timeout): await stdout_session_id_ready.wait() _monitor_kwargs: dict[str, object] = { @@ -263,6 +340,7 @@ async def _watch_session_log( # there is no await between them and the function return. acc.channel_b_status = monitor_result.status acc.channel_b_session_id = monitor_result.session_id + acc.channel_b_orphaned_tool_result = monitor_result.orphaned_tool_result channel_b_ready.set() trigger.set() diff --git a/src/autoskillit/execution/quota.py b/src/autoskillit/execution/quota.py index cd1a76ad4d..bfde92bd17 100644 --- a/src/autoskillit/execution/quota.py +++ b/src/autoskillit/execution/quota.py @@ -1,6 +1,6 @@ """Quota-aware check for long-running pipeline recipes. -L1 module: depends only on stdlib, httpx (FastMCP transitive dep), and core/logging. +IL-1 module: depends only on stdlib, httpx (FastMCP transitive dep), and core/logging. Does NOT sleep. Returns metadata; the orchestrator sleeps via run_cmd. """ @@ -17,13 +17,34 @@ from autoskillit.core import get_logger, write_versioned_json -_log = get_logger(__name__) +logger = get_logger(__name__) _DEFAULT_BASE_URL: str = "https://api.anthropic.com" QUOTA_CACHE_SCHEMA_VERSION: int = 3 _SCHEMA_DRIFT_LOGGED: set[str] = set() +# Canonical Anthropic quota API window names, as returned by GET /api/oauth/usage. +# Update these when Anthropic adds or renames windows — the contract tests will +# catch any mismatch between these constants and the configured long_window_patterns. +KNOWN_QUOTA_WINDOW_NAMES: frozenset[str] = frozenset( + { + "five_hour", + "one_hour", + "one_day", + "seven_day", + } +) + +# Subset of KNOWN_QUOTA_WINDOW_NAMES that represent long (multi-day) rate-limit windows. +# These names must be matched by at least one pattern in long_window_patterns. +# When Anthropic adds a new long window, add it here and update long_window_patterns defaults. +LONG_WINDOW_NAMES: frozenset[str] = frozenset( + { + "seven_day", + } +) + def _reset_schema_drift_logged_for_tests() -> None: """Test-only helper: clear the once-per-process drift-log set.""" @@ -174,7 +195,7 @@ def _read_cache(cache_path: str, max_age: int) -> QuotaStatus | None: cache_key = str(Path(cache_path).expanduser()) if cache_key not in _SCHEMA_DRIFT_LOGGED: _SCHEMA_DRIFT_LOGGED.add(cache_key) - _log.warning( + logger.warning( "quota_cache_schema_drift", cache_path=cache_key, observed=raw.get("schema_version"), @@ -224,7 +245,20 @@ def _write_cache(cache_path: str, result: QuotaFetchResult) -> None: path.parent.mkdir(parents=True, exist_ok=True) write_versioned_json(path, payload, schema_version=QUOTA_CACHE_SCHEMA_VERSION) except OSError as exc: - _log.warning("quota cache write failed", path=cache_path, error=str(exc)) + logger.warning("quota cache write failed", path=cache_path, error=str(exc)) + + +def invalidate_cache(cache_path: str) -> None: + """Remove the quota cache file so the next read triggers a live fetch. + + Tolerates missing files and logs a warning on permission errors. + """ + try: + Path(cache_path).expanduser().unlink() + except FileNotFoundError: + pass + except OSError as exc: + logger.warning("quota cache invalidation failed", path=cache_path, error=str(exc)) async def _fetch_quota( @@ -260,6 +294,14 @@ async def _fetch_quota( utilization=float(raw_util), resets_at=_parse_resets_at(w.get("resets_at")), ) + novel = {name for name in windows if name not in KNOWN_QUOTA_WINDOW_NAMES} + if novel: + logger.warning( + "Anthropic quota API returned unknown quota window names. " + "If this is a new rate-limit window, add it to KNOWN_QUOTA_WINDOW_NAMES in quota.py " + "and update LONG_WINDOW_NAMES and long_window_patterns if it is a long window.", + novel_windows=sorted(novel), + ) if not windows: return QuotaFetchResult( windows={}, binding=QuotaStatus(0.0, None, effective_threshold=100.0) @@ -365,7 +407,7 @@ async def check_and_sleep_if_needed( if status.resets_at is None: fallback_seconds = max(config.buffer_seconds, 60) - _log.warning( + logger.warning( "quota above threshold but resets_at is None — blocking with fallback", utilization=status.utilization, fallback_sleep_seconds=fallback_seconds, @@ -387,7 +429,7 @@ async def check_and_sleep_if_needed( if status.resets_at is None: fallback_seconds = max(config.buffer_seconds, 60) - _log.warning( + logger.warning( "quota above threshold but resets_at is None after re-fetch" " — blocking with fallback", utilization=status.utilization, @@ -406,7 +448,7 @@ async def check_and_sleep_if_needed( now = datetime.now(UTC) wake_at = status.resets_at + timedelta(seconds=config.buffer_seconds) sleep_secs = max(0, int((wake_at - now).total_seconds())) - _log.info( + logger.info( "quota threshold exceeded — caller should sleep", utilization=status.utilization, effective_threshold=status.effective_threshold, @@ -438,14 +480,14 @@ async def check_and_sleep_if_needed( httpx.HTTPError, ) if isinstance(exc, _operational_types): - _log.warning( + logger.warning( "quota check failed — continuing without sleep", error=str(exc), error_type=type(exc).__name__, exc_info=True, ) else: - _log.error( + logger.error( "quota check failed (unexpected error) — continuing without sleep", error=str(exc), error_type=type(exc).__name__, diff --git a/src/autoskillit/execution/recording.py b/src/autoskillit/execution/recording.py index e2a59922c0..0f87d6de93 100644 --- a/src/autoskillit/execution/recording.py +++ b/src/autoskillit/execution/recording.py @@ -5,11 +5,25 @@ import asyncio import tempfile from collections import deque -from collections.abc import Mapping +from collections.abc import Callable, Mapping from pathlib import Path from typing import TYPE_CHECKING, Any -from autoskillit.core import SubprocessResult, SubprocessRunner, TerminationReason, get_logger +from autoskillit.core import ( + SubprocessResult, + SubprocessRunner, + TerminationReason, + ValidatedAddDir, + get_logger, +) +from autoskillit.execution._recording_skills import ( + _extract_ephemeral_add_dir, + scan_skill_snapshots, + snapshot_skill_dir, +) +from autoskillit.execution._recording_skills import ( + restore_skill_snapshot as _restore_skill_snapshot, +) if TYPE_CHECKING: from api_simulator.claude import ScenarioPlayer, ScenarioRecorder @@ -60,8 +74,11 @@ def __init__( self, recorder: ScenarioRecorder, inner: SubprocessRunner | None = None, + *, + scenario_dir: Path | None = None, ) -> None: self.recorder = recorder + self._scenario_dir = scenario_dir if inner is None: from autoskillit.execution.process import DefaultSubprocessRunner @@ -84,6 +101,9 @@ async def __call__( linux_tracing_config: Any | None = None, idle_output_timeout: float | None = None, max_suppression_seconds: float | None = None, + on_pid_resolved: Callable[[int, int], None] | None = None, + enable_deadline_extension: bool = False, + max_extension_seconds: float = 7200, ) -> SubprocessResult: step_name = (env or {}).get(SCENARIO_STEP_NAME_ENV, "") @@ -109,6 +129,9 @@ async def __call__( linux_tracing_config=linux_tracing_config, idle_output_timeout=idle_output_timeout, max_suppression_seconds=max_suppression_seconds, + on_pid_resolved=on_pid_resolved, + enable_deadline_extension=enable_deadline_extension, + max_extension_seconds=max_extension_seconds, ) if step_name: @@ -151,6 +174,17 @@ async def _record_session( if cassette_stdout.exists(): stdout = cassette_stdout.read_text(encoding="utf-8") + ephemeral_dir = _extract_ephemeral_add_dir(cmd) + if ephemeral_dir is not None and step_result.cassette_path: + _scenario_dir = ( + self._scenario_dir + if self._scenario_dir is not None + else Path(step_result.cassette_path).parent.parent + ) + _snap = snapshot_skill_dir(_scenario_dir, step_name, ephemeral_dir) + if _snap: + logger.debug("skill_dir_snapshot_written", step=step_name, path=str(_snap)) + return SubprocessResult( returncode=step_result.cassette_exit_code, stdout=stdout, @@ -178,13 +212,24 @@ def __init__( non_session_results: dict[str, dict[str, Any]], *, player: ScenarioPlayer | None = None, + skill_snapshots: dict[str, Path] | None = None, ) -> None: self._sessions = session_map self._non_session = non_session_results self.player: ScenarioPlayer | None = player + self.skill_snapshots: dict[str, Path] = skill_snapshots or {} self.call_log: list[tuple[str, list[str]]] = [] self._tmp_replay_dir = None + def restore_skill_snapshot( + self, step_name: str, ephemeral_root: Path, session_id: str + ) -> ValidatedAddDir | None: + """Restore a skill snapshot for step_name into a fresh ephemeral session dir.""" + snap_path = self.skill_snapshots.get(step_name) + if snap_path is None: + return None + return _restore_skill_snapshot(snap_path, ephemeral_root, session_id) + async def __call__( self, cmd: list[str], @@ -201,6 +246,9 @@ async def __call__( linux_tracing_config: Any | None = None, idle_output_timeout: float | None = None, max_suppression_seconds: float | None = None, + on_pid_resolved: Callable[[int, int], None] | None = None, + enable_deadline_extension: bool = False, + max_extension_seconds: float = 7200, ) -> SubprocessResult: step_name = (env or {}).get(SCENARIO_STEP_NAME_ENV, "") @@ -245,7 +293,7 @@ def build_replay_runner(replay_dir: str) -> ReplayingSubprocessRunner: Creates a temporary output directory for the player, then parses the scenario manifest and constructs the deque-based session map. All domain logic for replay setup lives here - (L1) rather than in the L3 composition root. + (IL-1) rather than in the IL-3 composition root. Args: replay_dir: Path to a scenario directory produced by a recording run. @@ -295,6 +343,9 @@ def build_replay_runner(replay_dir: str) -> ReplayingSubprocessRunner: raise session_map = {k: deque(v) for k, v in raw_map.items()} - runner = ReplayingSubprocessRunner(session_map, non_session, player=player) + skill_snapshots = scan_skill_snapshots(Path(replay_dir)) + runner = ReplayingSubprocessRunner( + session_map, non_session, player=player, skill_snapshots=skill_snapshots + ) runner._tmp_replay_dir = _tmp_replay_dir return runner diff --git a/src/autoskillit/execution/remote_resolver.py b/src/autoskillit/execution/remote_resolver.py index dcc64e46ef..dc573cce77 100644 --- a/src/autoskillit/execution/remote_resolver.py +++ b/src/autoskillit/execution/remote_resolver.py @@ -1,6 +1,6 @@ """Canonical GitHub remote repository resolver. -L1 module: resolves 'owner/repo' from a git working directory, +IL-1 module: resolves 'owner/repo' from a git working directory, encoding the clone isolation contract (upstream > origin priority). """ @@ -10,7 +10,7 @@ from autoskillit.core import get_logger, normalize_owner_repo, parse_github_repo -_log = get_logger(__name__) +logger = get_logger(__name__) # Canonical remote precedence order: upstream before origin encodes the clone # isolation contract (upstream = real GitHub URL; origin = file:// local path). @@ -59,7 +59,7 @@ async def resolve_remote_repo( proc.kill() await proc.wait() await asyncio.gather(io_task, return_exceptions=True) - _log.warning("Timed out getting URL for remote %r in %r", remote, cwd) + logger.warning("Timed out getting URL for remote %r in %r", remote, cwd) continue stdout, _ = await io_task if proc.returncode == 0: @@ -67,6 +67,47 @@ async def resolve_remote_repo( if parsed: return parsed except OSError: - _log.warning("Failed to get URL for remote %r in %r", remote, cwd, exc_info=True) + logger.warning("Failed to get URL for remote %r in %r", remote, cwd, exc_info=True) return None + + +async def resolve_remote_name( + cwd: str, + remotes: tuple[str, ...] = REMOTE_PRECEDENCE, +) -> str: + """Return the git remote name to use for fetch/rebase operations. + + Tries remotes in precedence order (upstream before origin). + Rejects file:// URLs — those indicate a clone-isolation origin + that should not be used for real git operations. + Falls back to "origin" if no remote qualifies. + """ + for name in remotes: + proc = None + try: + proc = await asyncio.create_subprocess_exec( + "git", + "remote", + "get-url", + name, + cwd=cwd, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.DEVNULL, + ) + stdout, _ = await asyncio.wait_for(proc.communicate(), timeout=15.0) + if proc.returncode != 0: + continue + url = stdout.decode().strip() + if url.startswith("file://"): + continue + return name + except (TimeoutError, OSError): + if proc is not None and proc.returncode is None: + try: + proc.kill() + await proc.wait() + except OSError: + pass + continue + return "origin" diff --git a/src/autoskillit/execution/session.py b/src/autoskillit/execution/session.py deleted file mode 100644 index fe030a2fe0..0000000000 --- a/src/autoskillit/execution/session.py +++ /dev/null @@ -1,1035 +0,0 @@ -"""Domain model for Claude Code headless session results. - -L2 module: imports only from L0 (types, _logging). No server side-effects. -Centralizes all session-parsing concerns so callers can work with typed -objects instead of raw JSON strings. -""" - -from __future__ import annotations - -import json -import re -from collections.abc import Sequence -from dataclasses import dataclass, field -from enum import StrEnum -from typing import Any, assert_never - -from autoskillit.core import ( - CONTEXT_EXHAUSTION_MARKER, - ChannelConfirmation, - CliSubtype, - RetryReason, - SessionOutcome, - SkillResult, - TerminationReason, - get_logger, - truncate_text, -) - -logger = get_logger(__name__) - -_truncate = truncate_text - -# Re-export SkillResult so existing callers (tests, migration_engine) can still -# import from this module. Canonical definition lives in core/types.py. -__all__ = ["CliSubtype", "SkillResult"] - - -_TOKEN_FIELDS = ( - "input_tokens", - "output_tokens", - "cache_creation_input_tokens", - "cache_read_input_tokens", -) - -FAILURE_SUBTYPES: frozenset[CliSubtype] = frozenset( - { - CliSubtype.UNKNOWN, - CliSubtype.EMPTY_OUTPUT, - CliSubtype.UNPARSEABLE, - CliSubtype.TIMEOUT, - CliSubtype.IDLE_STALL, - } -) - - -class ContentState(StrEnum): - """Describes why a session's content evaluation succeeded or failed. - - Separates three distinct failure categories so that the dead-end guard - can distinguish drain-race artifacts (transient, retriable) from pattern- - contract violations and session errors (terminal, not retriable). - """ - - COMPLETE = "complete" - ABSENT = "absent" # Empty result or missing marker — drain-race candidate - CONTRACT_VIOLATION = "contract_violation" # Result + marker present, but patterns fail - SESSION_ERROR = "session_error" # is_error=True or process-level failure - - -@dataclass -class ClaudeSessionResult: - """Parsed result from a Claude Code headless session.""" - - subtype: CliSubtype - is_error: bool - result: str - session_id: str - errors: list[str] = field(default_factory=list) - token_usage: dict[str, Any] | None = None - assistant_messages: list[str] = field(default_factory=list) - tool_uses: list[dict[str, Any]] = field(default_factory=list) - jsonl_context_exhausted: bool = False - - def __post_init__(self) -> None: - if not isinstance(self.result, str): - if isinstance(self.result, list): - self.result = "\n".join( - b.get("text", "") if isinstance(b, dict) else str(b) for b in self.result - ) - elif not isinstance(self.result, str): - self.result = "" if self.result is None else str(self.result) - if not isinstance(self.errors, list): - self.errors = [] if self.errors is None else [str(self.errors)] - if not isinstance(self.subtype, CliSubtype): - self.subtype = ( - CliSubtype.UNKNOWN - if self.subtype is None - else CliSubtype.from_cli(str(self.subtype)) - ) - if not isinstance(self.session_id, str): - self.session_id = "" if self.session_id is None else str(self.session_id) - if not isinstance(self.jsonl_context_exhausted, bool): - self.jsonl_context_exhausted = bool(self.jsonl_context_exhausted) - - def _is_context_exhausted(self) -> bool: - """Detect context window exhaustion from Claude's error output. - - Three detection paths (evaluated in priority order): - - 1. JSONL flat-record fallback (Channel B): when parse_session_result detected - a flat assistant record with zero output tokens and the marker text. This - path fires regardless of is_error — it is race-resilient because it reads - the raw NDJSON stream rather than the drained result field. - - 2. Primary: is_error=True AND marker in the structured errors list. - - 3. Fallback: is_error=True AND marker in session.result, restricted to known - error subtypes to avoid false positives from model prose. - """ - # Path 1: JSONL flat-record detection — bypasses is_error guard - if self.jsonl_context_exhausted: - return True - if not self.is_error: - return False - # Path 2: structured errors list from Claude CLI - marker = CONTEXT_EXHAUSTION_MARKER - if any(marker in e.lower() for e in self.errors): - return True - # Path 3: result text — restricted to error subtypes - if ( - self.subtype in (CliSubtype.SUCCESS, CliSubtype.ERROR_MAX_TURNS) - and marker in self.result.lower() - ): - return True - return False - - @property - def agent_result(self) -> str: - """Result text rewritten for LLM agent consumption. - - When the session ended due to a retriable condition (context exhaustion, - max turns), the raw result text from Claude CLI can be misleading to - LLM callers. This property returns semantically correct, actionable text. - The raw result is preserved in self.result for debugging. - """ - if self._is_context_exhausted(): - return ( - "Context limit reached during session execution. " - "The session made partial progress. " - "Use needs_retry and retry_reason to continue from where it left off." - ) - if self.subtype == CliSubtype.ERROR_MAX_TURNS: - return ( - "Turn limit reached during session execution. " - "The session made partial progress. " - "Use needs_retry and retry_reason to continue from where it left off." - ) - return self.result - - @property - def needs_retry(self) -> bool: - """Whether the session didn't finish and should be retried.""" - if self.subtype == CliSubtype.ERROR_MAX_TURNS: - return True - if self._is_context_exhausted(): - return True - return False - - @property - def retry_reason(self) -> RetryReason: - """Why retry is needed. NONE if needs_retry is False.""" - if self.needs_retry: - return RetryReason.RESUME - return RetryReason.NONE - - @property - def session_complete(self) -> bool: - """True when this session reached a normal completion state. - - A session is complete when it is not in an error state AND its subtype - is not in the failure set (UNKNOWN, EMPTY_OUTPUT, UNPARSEABLE, TIMEOUT). - Recovery operations and CHANNEL_B bypass may only run on complete sessions. - """ - return not self.is_error and self.subtype not in FAILURE_SUBTYPES - - -def extract_token_usage(stdout: str) -> dict[str, Any] | None: - """Extract token usage from Claude CLI NDJSON output. - - Takes raw NDJSON *stdout* (not a parsed ClaudeSessionResult) because this - function is called inside parse_session_result() *during construction* of - ClaudeSessionResult — before that object exists. A (result: ClaudeSessionResult) - parameter would create a circular bootstrapping dependency and is therefore - architecturally incorrect for this call site. - - Scans assistant records for per-model usage and the result record for - authoritative aggregated totals. Returns None if no usage data is found. - """ - if not stdout.strip(): - return None - - model_buckets: dict[str, dict[str, int]] = {} - result_usage: dict[str, int] | None = None - - for line in stdout.strip().splitlines(): - line = line.strip() - if not line: - continue - try: - obj = json.loads(line) - except json.JSONDecodeError: - continue - if not isinstance(obj, dict): - continue - - record_type = obj.get("type") - - if record_type == "assistant": - msg = obj.get("message") - if not isinstance(msg, dict): - continue - usage = msg.get("usage") - if not isinstance(usage, dict): - continue - model = msg.get("model", "unknown") - bucket = model_buckets.setdefault(model, {f: 0 for f in _TOKEN_FIELDS}) - for f in _TOKEN_FIELDS: - bucket[f] += usage.get(f, 0) - - elif record_type == "result": - usage = obj.get("usage") - if isinstance(usage, dict): - result_usage = {f: usage.get(f, 0) for f in _TOKEN_FIELDS} - - if not model_buckets and result_usage is None: - return None - - # Aggregated totals: prefer result record, fall back to assistant sum - if result_usage is not None: - totals = dict(result_usage) - else: - totals = {f: 0 for f in _TOKEN_FIELDS} - for bucket in model_buckets.values(): - for f in _TOKEN_FIELDS: - totals[f] += bucket[f] - - return { - **totals, - "model_breakdown": dict(model_buckets) if model_buckets else {}, - } - - -_KNOWN_RESULT_KEYS: frozenset[str] = frozenset( - {"type", "subtype", "is_error", "result", "session_id", "errors", "usage"} -) - - -def _detect_flat_context_exhaustion(stdout: str) -> bool: - """Detect context exhaustion from a flat assistant JSONL record. - - Claude Code CLI emits a flat (no 'message' wrapper) assistant record when - the context window is exhausted with is_error=False: - - {"type": "assistant", - "content": [{"type": "text", "text": "Prompt is too long"}], - "output_tokens": 0, "input_tokens": 0, ...} - - This record does NOT appear in session.result (stdout drain race) and is - NOT captured by the existing is_error-gated detection paths. Scanning the - raw NDJSON stream for this pattern makes detection race-resilient. - - Returns True only when: - - record type is "assistant" - - record has no "message" key (flat structure) - - "output_tokens" at top level is 0 - - at least one content text block contains CONTEXT_EXHAUSTION_MARKER - """ - marker = CONTEXT_EXHAUSTION_MARKER - for line in stdout.strip().splitlines(): - line = line.strip() - if not line: - continue - try: - obj = json.loads(line) - except json.JSONDecodeError: - continue - if not isinstance(obj, dict): - continue - if obj.get("type") != "assistant": - continue - if "message" in obj: - continue # standard wrapped record — not the flat context-exhaustion format - if obj.get("output_tokens", -1) != 0: - continue - content = obj.get("content", []) - if not isinstance(content, list): - continue - if any( - isinstance(block, dict) - and block.get("type") == "text" - and marker in block.get("text", "").lower() - for block in content - ): - return True - return False - - -@dataclass -class _ParseAccumulator: - """Mutable accumulator for parse_session_result's NDJSON scan. - - Collects tool_uses, assistant_messages, and context exhaustion signals - across all NDJSON records. A single ClaudeSessionResult is constructed - from the accumulated state at the end of parsing — no early returns. - """ - - result_obj: dict[str, Any] | None = None - tool_uses: list[dict[str, Any]] = field(default_factory=list) - assistant_messages: list[str] = field(default_factory=list) - jsonl_context_exhausted: bool = False - - -def parse_session_result(stdout: str) -> ClaudeSessionResult: - """Parse Claude Code's --output-format json stdout into a typed result. - - Handles multi-line NDJSON (Claude Code may emit multiple JSON objects; - the last 'result' type object is authoritative). - Falls back gracefully for non-JSON or missing fields. - - Uses an accumulator pattern: all NDJSON records are scanned unconditionally - into a _ParseAccumulator, then a single ClaudeSessionResult is constructed - from the accumulated state. This ensures tool_uses, assistant_messages, and - token_usage are preserved on all paths — including fallback/unparseable. - """ - if not stdout.strip(): - return ClaudeSessionResult( - subtype=CliSubtype.EMPTY_OUTPUT, - is_error=True, - result="", - session_id="", - errors=[], - ) - - acc = _ParseAccumulator() - marker = CONTEXT_EXHAUSTION_MARKER - - for line in stdout.strip().splitlines(): - line = line.strip() - if not line: - continue - try: - obj = json.loads(line) - if not isinstance(obj, dict): - continue - record_type = obj.get("type") - if record_type == "result": - acc.result_obj = obj - elif record_type == "assistant": - msg = obj.get("message") - if isinstance(msg, dict): - content = msg.get("content", "") - if isinstance(content, list): - for block in content: - if isinstance(block, dict) and block.get("type") == "tool_use": - name = block.get("name", "") - entry: dict[str, str] = {"name": name, "id": block.get("id", "")} - if name in {"Write", "Edit"} and isinstance( - block.get("input"), dict - ): - fp = block["input"].get("file_path", "") - if fp: - entry["file_path"] = fp - acc.tool_uses.append(entry) - text = "\n".join( - block.get("text", "") for block in content if isinstance(block, dict) - ).strip() - else: - text = str(content).strip() - if text: - acc.assistant_messages.append(text) - elif "message" not in obj: - # Flat assistant record (no "message" wrapper) — detect - # context exhaustion inline instead of a separate scan pass. - if obj.get("output_tokens", -1) == 0: - flat_content = obj.get("content", []) - if isinstance(flat_content, list) and any( - isinstance(block, dict) - and block.get("type") == "text" - and marker in block.get("text", "").lower() - for block in flat_content - ): - acc.jsonl_context_exhausted = True - except json.JSONDecodeError: - continue - - # Fallback: try to parse stdout as a single JSON object - if acc.result_obj is None: - try: - fallback = json.loads(stdout) - if isinstance(fallback, dict) and fallback.get("type") == "result": - acc.result_obj = fallback - except json.JSONDecodeError: - pass - - # Token usage extraction runs on ALL paths (not just when result_obj exists) - token_usage = extract_token_usage(stdout) - - # Single construction point: build ClaudeSessionResult from accumulated state - if acc.result_obj is not None: - extra_keys = frozenset(acc.result_obj.keys()) - _KNOWN_RESULT_KEYS - if extra_keys: - logger.debug("unknown_result_keys", unknown_fields=sorted(extra_keys)) - - subtype = CliSubtype.from_cli(acc.result_obj.get("subtype") or "unknown") - is_error: bool = acc.result_obj.get("is_error", False) - result_text: str = acc.result_obj.get("result") or "" - session_id: str = acc.result_obj.get("session_id") or "" - errors: list[str] = acc.result_obj.get("errors") or [] - else: - # No result record found: classify based on accumulated signals - if acc.jsonl_context_exhausted: - subtype = CliSubtype.CONTEXT_EXHAUSTION - else: - subtype = CliSubtype.UNPARSEABLE - is_error = True - result_text = stdout - session_id = "" - errors = [] - - return ClaudeSessionResult( - subtype=subtype, - is_error=is_error, - result=result_text, - session_id=session_id, - errors=errors, - token_usage=token_usage, - assistant_messages=acc.assistant_messages, - tool_uses=acc.tool_uses, - jsonl_context_exhausted=acc.jsonl_context_exhausted, - ) - - -# Compiled once at module level — no per-call overhead -_MARKDOWN_TOKEN_RE: re.Pattern[str] = re.compile(r"\*{1,2}(\w[\w_-]*)\*{1,2}(\s*=)", re.MULTILINE) - - -def _strip_markdown_from_tokens(text: str) -> str: - """Remove bold/italic markdown decorators from structured output token names. - - Transforms model output like: - **plan_path** = /abs/path/plan.md - into the canonical form: - plan_path = /abs/path/plan.md - - Applied before regex pattern matching to make adjudication tolerant of the - model's choice to visually style its output summary. Only `*word*` and - `**word**` patterns adjacent to `=` are normalized — decorators elsewhere - in the text are left unchanged. - """ - return _MARKDOWN_TOKEN_RE.sub(r"\1\2", text) - - -def _check_expected_patterns(result: str, patterns: Sequence[str]) -> bool: - """Return True if ALL expected_output_patterns are found in result, or if - no patterns are configured. This check MUST run on all session outcome paths, - including the Channel B bypass path. - - AND semantics are intentional: patterns represent content contracts (e.g., - block start/end delimiters) that must all be present simultaneously. - - Normalizes bold/italic markdown decorators on token names before matching, - so ``**plan_path** = /path`` is treated identically to ``plan_path = /path``. - - If any pattern is an invalid regex, returns False rather than raising. - """ - if not patterns: - return True - normalized = _strip_markdown_from_tokens(result) - for p in patterns: - try: - if not re.search(p, normalized): - return False - except re.error: - logger.warning("invalid_expected_output_pattern", pattern=p) - return False - return True - - -def _check_session_content( - session: ClaudeSessionResult, - completion_marker: str, - expected_output_patterns: Sequence[str] = (), -) -> bool: - """Validate session content fields after termination-specific gates pass.""" - if session.is_error: - logger.debug("content_check_failed", reason="is_error", is_error=True) - return False - if not session.result.strip(): - logger.debug("content_check_failed", reason="empty_result") - return False - if session.subtype in FAILURE_SUBTYPES: - logger.debug("content_check_failed", reason="failure_subtype", subtype=session.subtype) - return False - if completion_marker: - result_text = session.result.strip() - marker_stripped = result_text.replace(completion_marker, "").strip() - if not marker_stripped: - logger.debug("content_check_failed", reason="result_is_only_marker") - return False - if completion_marker not in result_text: - logger.debug( - "content_check_failed", - reason="completion_marker_absent", - result_tail=result_text[-200:] if len(result_text) > 200 else result_text, - ) - return False - if not _check_expected_patterns(session.result.strip(), expected_output_patterns): - logger.warning( - "content_check_failed", - reason="expected_artifact_absent", - patterns=list(expected_output_patterns), - ) - return False - logger.debug("content_check_passed") - return True - - -def _evaluate_content_state( - session: ClaudeSessionResult, - completion_marker: str, - expected_output_patterns: Sequence[str], -) -> ContentState: - """Classify the content completeness and contract compliance of a session result. - - Returns: - ContentState.COMPLETE: Result is non-empty, marker present (if configured), - and all expected_output_patterns match. Session is fully successful. - ContentState.ABSENT: Result is empty OR completion marker is absent from a - non-empty result. Indicates a drain-race artifact — the session may have - completed but stdout was not fully flushed. Retriable. - ContentState.CONTRACT_VIOLATION: Result is non-empty and contains the marker, - but one or more expected_output_patterns are absent. The session ran to - completion but the model did not produce the required output tokens. - Terminal — retrying will not produce different output. - ContentState.SESSION_ERROR: The CLI session itself reported an error - (is_error=True) or produced a failure subtype. Terminal. - """ - # Process-level / CLI-level failure — terminal regardless of content - if session.is_error: - return ContentState.SESSION_ERROR - - result = session.result.strip() - - # Empty result — drain-race candidate regardless of content requirements. - # This must come before the "no requirements" shortcut so that CHANNEL_A - # dead-end guard can detect drain-race artifacts even when no marker or - # patterns are configured. - if not result: - return ContentState.ABSENT - - # No content requirements configured and result is non-empty: CHANNEL_B - # confirmation alone is sufficient. Returning COMPLETE here preserves the - # existing behaviour for skills that produce non-empty output without a - # marker (e.g. fire-and-forget commands with plain text output). - if not completion_marker and not expected_output_patterns: - return ContentState.COMPLETE - - # Marker absent — partial drain candidate - if completion_marker and completion_marker not in result: - return ContentState.ABSENT - - # Result non-empty, marker present (or not configured) — check pattern contract - if expected_output_patterns and not _check_expected_patterns(result, expected_output_patterns): - return ContentState.CONTRACT_VIOLATION - - return ContentState.COMPLETE - - -def _compute_success( - session: ClaudeSessionResult, - returncode: int, - termination: TerminationReason, - completion_marker: str = "", - channel_confirmation: ChannelConfirmation = ChannelConfirmation.UNMONITORED, - expected_output_patterns: Sequence[str] = (), -) -> bool: - """Cross-validate all signals to determine unambiguous success/failure. - - Exhaustive match dispatch over TerminationReason ensures mypy flags any - unhandled value when the enum is extended (ARCH-007). - - Gate 0.5 (provenance bypass): when ``channel_confirmation=CHANNEL_B``, - Channel B's session-JSONL marker is the authoritative signal that the - session completed successfully. Stdout content is not required. - """ - # Gate 0.5: Channel B provenance bypass — session JSONL is authoritative. - # When expected_output_patterns is non-empty, the bypass requires session_complete - # to guard against synthesis-injected false positives: a failure-subtype session - # (UNPARSEABLE, TIMEOUT) whose result was mutated by synthesis must not be accepted - # as successful via CHANNEL_B. When no patterns are configured, the bypass fires - # unconditionally — the JSONL marker is sufficient evidence of completion (drain race). - match channel_confirmation: - case ChannelConfirmation.CHANNEL_B: - if expected_output_patterns and not session.session_complete: - # Patterns exist but session is a failure subtype — fall through to - # termination dispatch so standard content validation rejects it. - logger.debug( - "channel_b_bypass_skipped_incomplete_session", - subtype=str(session.subtype), - is_error=session.is_error, - pattern_count=len(expected_output_patterns), - ) - else: - # PRECONDITION: _recover_block_from_assistant_messages MUST be called - # (and its recovered session substituted) before this function when - # channel_confirmation=CHANNEL_B and expected_output_patterns is set. - # Callers that bypass _build_skill_result must honour this contract. - if not _check_expected_patterns(session.result.strip(), expected_output_patterns): - logger.debug( - "channel_b_content_check_failed", - result_len=len(session.result), - pattern_count=len(expected_output_patterns), - ) - return False - logger.debug("compute_success_bypass", channel="CHANNEL_B", result=True) - return True - case ( - ChannelConfirmation.CHANNEL_A - | ChannelConfirmation.UNMONITORED - | ChannelConfirmation.DIR_MISSING - ): - pass # fall through to termination dispatch - case _ as _unreachable_cc: - assert_never(_unreachable_cc) - - match termination: - case TerminationReason.TIMED_OUT: - return False - - case TerminationReason.STALE: - return False - - case TerminationReason.IDLE_STALL: - return False - - case TerminationReason.COMPLETED: - # The process was killed by our own async_kill_process_tree - # (signal -15 or -9), so a non-zero returncode is expected and - # trustworthy when the session envelope says "success". - if returncode != 0 and not ( - session.subtype == CliSubtype.SUCCESS and session.result.strip() - ): - return False - content_ok = _check_session_content( - session, completion_marker, expected_output_patterns - ) - logger.debug( - "compute_success_termination", - termination="COMPLETED", - returncode=returncode, - content_check=content_ok, - ) - return content_ok - - case TerminationReason.NATURAL_EXIT: - # The process exited on its own. A non-zero returncode is normally - # authoritative evidence of failure — no asymmetric bypass. - # - # Post-completion kill bypass: when an external watchdog kills the - # process AFTER it finished its work (e.g. trailing async task cleanup), - # the signal is a teardown artifact. The completion marker in the result - # provides strong evidence of completion — trust it over the returncode. - if returncode != 0: - if ( - session.subtype == CliSubtype.SUCCESS - and session.result.strip() - and completion_marker - and completion_marker in session.result - ): - content_ok = _check_session_content( - session, completion_marker, expected_output_patterns - ) - logger.debug( - "compute_success_natural_exit_post_completion_kill", - returncode=returncode, - content_check=content_ok, - ) - return content_ok - return False - content_ok = _check_session_content( - session, completion_marker, expected_output_patterns - ) - logger.debug( - "compute_success_termination", - termination="NATURAL_EXIT", - returncode=returncode, - content_check=content_ok, - ) - return content_ok - - case _ as unreachable: - assert_never(unreachable) - - -_KILL_ANOMALY_SUBTYPES: frozenset[CliSubtype] = frozenset( - { - CliSubtype.UNPARSEABLE, # killed mid-write → partial NDJSON - CliSubtype.EMPTY_OUTPUT, # killed before any stdout was written - CliSubtype.INTERRUPTED, # killed mid-generation → real Claude CLI subtype - } -) - - -def _is_kill_anomaly(session: ClaudeSessionResult) -> bool: - """True if the session result looks like a kill-induced incomplete flush. - - Covers anomalies from both infrastructure kills (COMPLETED) and voluntary - self-exits (NATURAL_EXIT with returncode==0). Callers must apply their own - termination-specific discriminators before calling this function. - - Covers: - - unparseable: process killed mid-write, stdout is partial NDJSON - - empty_output: process killed before any stdout was written - - interrupted: process killed mid-generation (real Claude CLI subtype) - - success with empty result: kill occurred after result record was written - but with empty content (Channel B / Channel A drain-race, or - CLAUDE_CODE_EXIT_AFTER_STOP_DELAY timer-based self-exit) - """ - if session.subtype in _KILL_ANOMALY_SUBTYPES: - return True - if session.subtype == CliSubtype.SUCCESS and not session.result.strip(): - return True - return False - - -def _compute_retry( - session: ClaudeSessionResult, - returncode: int, - termination: TerminationReason, - channel_confirmation: ChannelConfirmation = ChannelConfirmation.UNMONITORED, - completion_marker: str = "", -) -> tuple[bool, RetryReason]: - """Compute whether the session result warrants a retry. - - Phase 1: API-level signals are termination-agnostic. - Phase 2: Exhaustive match dispatch over TerminationReason ensures mypy - flags any unhandled value when the enum is extended. - - When ``channel_confirmation=CHANNEL_B`` and ``termination=COMPLETED``, - the provenance bypass applies: Channel B's session-JSONL signal is - authoritative, so kill-anomaly appearance is a drain-race artifact. - No retry is needed. - """ - # Phase 1: API-level retry signals (context exhaustion, max_turns) - if session.needs_retry: - logger.debug("compute_retry_api_signal", needs_retry=True, reason="resume") - return True, RetryReason.RESUME - - # Phase 2: Exhaustive termination dispatch - match termination: - case TerminationReason.NATURAL_EXIT: - # NATURAL_EXIT covers both deliberate CLI exits AND - # CLAUDE_CODE_EXIT_AFTER_STOP_DELAY timer-based self-exits. - # returncode==0 discriminates: clean exit (possible kill-race artifact) - # vs genuine crash (nonzero returncode — not a timing artifact). - # Channel confirmation means the session completed — any kill-anomaly - # appearance is a drain artifact, not a real incomplete flush. - if channel_confirmation in ( - ChannelConfirmation.CHANNEL_A, - ChannelConfirmation.CHANNEL_B, - ): - logger.debug( - "compute_retry_result", - termination="NATURAL_EXIT", - channel=str(channel_confirmation), - needs_retry=False, - ) - return False, RetryReason.NONE - if returncode == 0 and _is_kill_anomaly(session): - reason = ( - RetryReason.RESUME - if session._is_context_exhausted() - else RetryReason.EMPTY_OUTPUT - ) - logger.debug( - "compute_retry_result", - termination="NATURAL_EXIT", - channel=str(channel_confirmation), - needs_retry=True, - kill_anomaly=True, - context_exhausted=reason == RetryReason.RESUME, - ) - return True, reason - # EARLY_STOP: model produced substantive output but stopped before - # emitting the completion marker (text-then-tool boundary). - if ( - returncode == 0 - and session.subtype == CliSubtype.SUCCESS - and session.result.strip() - and completion_marker - and completion_marker not in session.result - ): - skill_tool_calls = [t for t in session.tool_uses if t.get("name") == "Skill"] - logger.debug( - "compute_retry_early_stop", - termination="NATURAL_EXIT", - has_skill_calls=bool(skill_tool_calls), - skill_call_count=len(skill_tool_calls), - ) - return True, RetryReason.EARLY_STOP - logger.debug( - "compute_retry_result", - termination="NATURAL_EXIT", - channel=str(channel_confirmation), - needs_retry=False, - ) - return False, RetryReason.NONE - - case TerminationReason.COMPLETED: - # Infrastructure killed the process. SIGTERM/SIGKILL produce nonzero - # returncode by design — do not gate on returncode here. - # Exhaustive ChannelConfirmation dispatch (ARCH-007 extension): - match channel_confirmation: - case ChannelConfirmation.CHANNEL_B: - # Channel B is authoritative — kill-anomaly appearance is - # a drain-race artifact, not a real incomplete flush. - logger.debug( - "compute_retry_result", - termination="COMPLETED", - channel="CHANNEL_B", - needs_retry=False, - ) - return False, RetryReason.NONE - case ( - ChannelConfirmation.CHANNEL_A - | ChannelConfirmation.UNMONITORED - | ChannelConfirmation.DIR_MISSING - ): - is_anomaly = _is_kill_anomaly(session) - logger.debug( - "compute_retry_result", - termination="COMPLETED", - channel=str(channel_confirmation), - needs_retry=is_anomaly, - kill_anomaly=is_anomaly, - ) - if is_anomaly: - return True, RetryReason.RESUME - return False, RetryReason.NONE - case _ as _unreachable_cc: - assert_never(_unreachable_cc) - - case TerminationReason.STALE: - # _build_skill_result intercepts STALE before calling _compute_retry. - # Explicit arm exists for exhaustiveness; unreachable in production. - logger.debug("compute_retry_result", termination="STALE", needs_retry=False) - return False, RetryReason.NONE - - case TerminationReason.IDLE_STALL: - # _build_skill_result intercepts IDLE_STALL before calling _compute_retry. - # Explicit arm exists for exhaustiveness; unreachable in production. - logger.debug("compute_retry_result", termination="IDLE_STALL", needs_retry=False) - return False, RetryReason.NONE - - case TerminationReason.TIMED_OUT: - # Wall-clock timeout: non-retriable (permanent infrastructure limit). - logger.debug("compute_retry_result", termination="TIMED_OUT", needs_retry=False) - return False, RetryReason.NONE - - case _ as unreachable: - assert_never(unreachable) - - -def _normalize_subtype( - raw_subtype: CliSubtype, - outcome: SessionOutcome, - session: ClaudeSessionResult, - completion_marker: str, -) -> str: - """Normalize raw_subtype against the adjudicated outcome to eliminate contradictions. - - Maps ``(raw_subtype, outcome, session, completion_marker) → adjudicated subtype``. - Ensures ``subtype == "success"`` iff ``outcome == SUCCEEDED``. - - Exhaustive match over CliSubtype ensures mypy flags any unhandled member - when the enum is extended. - - Class 2 fix (upward normalization): - SUCCEEDED + error/diagnostic subtype → "success" - - Class 1 fix (downward normalization): - non-SUCCEEDED + "success" → synthesized failure label based on why. - - Return type is str (not CliSubtype) because downward normalization synthesizes - labels that are SkillResult-level subtypes, not CliSubtype members. - """ - match raw_subtype: - case CliSubtype.SUCCESS: - if outcome == SessionOutcome.SUCCEEDED: - return raw_subtype - # Downward normalization: success subtype but adjudicated failure/retriable - if not session.result.strip(): - return "empty_result" - if session.jsonl_context_exhausted: - return "context_exhausted" - if completion_marker and completion_marker not in session.result: - return "missing_completion_marker" - return "adjudicated_failure" - - case ( - CliSubtype.UNKNOWN - | CliSubtype.EMPTY_OUTPUT - | CliSubtype.UNPARSEABLE - | CliSubtype.TIMEOUT - | CliSubtype.IDLE_STALL - ): - # Failure subtypes: upward normalize to "success" when adjudicated SUCCEEDED - if outcome == SessionOutcome.SUCCEEDED: - return "success" - return raw_subtype - - case ( - CliSubtype.ERROR_MAX_TURNS - | CliSubtype.ERROR_DURING_EXECUTION - | CliSubtype.CONTEXT_EXHAUSTION - | CliSubtype.INTERRUPTED - ): - return raw_subtype - - case _ as unreachable: - assert_never(unreachable) - - -def _compute_outcome( - session: ClaudeSessionResult, - returncode: int, - termination: TerminationReason, - completion_marker: str = "", - channel_confirmation: ChannelConfirmation = ChannelConfirmation.UNMONITORED, - expected_output_patterns: Sequence[str] = (), -) -> tuple[SessionOutcome, RetryReason]: - """Compose _compute_success and _compute_retry into a single SessionOutcome. - - Applies both composition guards (contradiction and dead-end) before mapping - the resulting (success, needs_retry) pair to the bijective SessionOutcome enum. - - Returns (SessionOutcome, RetryReason). The outcome is never the impossible - (success=True, needs_retry=True) state — the contradiction guard structurally - prevents it from reaching the mapping step. - - Does NOT handle _recover_from_separate_marker recovery; that recovery path - lives in _build_skill_result (headless.py), which calls _compute_success on - the recovered session then delegates to _compute_outcome. - """ - success = _compute_success( - session, - returncode, - termination, - completion_marker, - channel_confirmation, - expected_output_patterns, - ) - needs_retry, retry_reason = _compute_retry( - session, returncode, termination, channel_confirmation, completion_marker - ) - - logger.debug( - "compute_outcome_inputs", - success=success, - needs_retry=needs_retry, - retry_reason=str(retry_reason), - returncode=returncode, - termination=str(termination), - channel=str(channel_confirmation), - subtype=session.subtype, - is_error=session.is_error, - result_empty=not session.result.strip(), - ) - - # Contradiction guard: retry signal is authoritative over the channel bypass. - if success and needs_retry: - success = False - logger.debug( - "contradiction_guard", - action="demoted_success", - reason="retry_signal_authoritative", - ) - - # Dead-end guard: channel confirmation means the session signalled completion, but - # content checks failed and no retry was scheduled by _compute_retry. - # Only promote to RETRIABLE when the failure is a drain-race artifact (ABSENT state): - # - empty result → stdout not fully flushed - # - marker missing → partial flush - # Do NOT promote CONTRACT_VIOLATION or SESSION_ERROR — these are terminal failures - # that retrying will never resolve. - if not success and not needs_retry: - match channel_confirmation: - case ChannelConfirmation.CHANNEL_A | ChannelConfirmation.CHANNEL_B: - content_state = _evaluate_content_state( - session, completion_marker, expected_output_patterns - ) - if content_state == ContentState.ABSENT: - needs_retry = True - retry_reason = RetryReason.DRAIN_RACE - logger.debug( - "dead_end_guard", - action="promoted_to_retriable", - content_state=content_state.value, - channel=channel_confirmation.value, - ) - else: - logger.debug( - "dead_end_guard", - action="terminal_failure_not_promoted", - content_state=content_state.value, - channel=channel_confirmation.value, - ) - case ChannelConfirmation.UNMONITORED | ChannelConfirmation.DIR_MISSING: - pass # legitimate terminal failure — no channel confirmed completion - case _ as unreachable_cc: - assert_never(unreachable_cc) - - if success: - outcome = SessionOutcome.SUCCEEDED - elif needs_retry: - outcome = SessionOutcome.RETRIABLE - else: - outcome = SessionOutcome.FAILED - - logger.debug( - "compute_outcome_result", - outcome=str(outcome), - retry_reason=str(retry_reason), - ) - return outcome, retry_reason diff --git a/src/autoskillit/execution/session/CLAUDE.md b/src/autoskillit/execution/session/CLAUDE.md new file mode 100644 index 0000000000..60b8257c7f --- /dev/null +++ b/src/autoskillit/execution/session/CLAUDE.md @@ -0,0 +1,19 @@ +# session/ + +Session result processing — parse, validate content, compute retry, adjudicate outcome. + +## Files + +| File | Purpose | +|------|---------| +| `__init__.py` | Thin facade re-exporting all sub-module symbols | +| `_exit_classification.py` | Infrastructure exit classification: `classify_infra_exit()`, `InfraExitCategory` detection from session + stderr | +| `_session_state.py` | Session state persistence for resume: `SessionState`, `persist_session_state()`, `read_session_state()`, `clear_session_state()` | +| `_session_model.py` | `ClaudeSessionResult`, `ContentState`, `parse_session_result()`, `extract_token_usage()` | +| `_session_content.py` | Content validation: `_check_session_content()`, `_check_expected_patterns()` | +| `_retry_fsm.py` | Retry FSM: `_compute_retry()`, maps `(TerminationReason, CliSubtype)` to `RetryReason` | +| `_session_outcome.py` | High-level adjudication: `_compute_success()`, `_compute_outcome()` | + +## Architecture Notes + +The sub-modules form a pipeline: parse -> check content -> compute retry -> compute outcome. `_exit_classification.py` provides a parallel infrastructure classification (context exhaustion, API error, process kill) used by `_headless_result.py` for resume routing. When Channel B is the sole confirmation source, `_compute_success` applies a provenance bypass — the session JSONL marker is treated as authoritative proof of success without requiring stdout content. diff --git a/src/autoskillit/execution/session/__init__.py b/src/autoskillit/execution/session/__init__.py new file mode 100644 index 0000000000..99d3ae480c --- /dev/null +++ b/src/autoskillit/execution/session/__init__.py @@ -0,0 +1,63 @@ +"""Domain model for Claude Code headless session results. + +IL-1 module: imports only from IL-0 (types, _logging). No server side-effects. +Centralizes all session-parsing concerns so callers can work with typed +objects instead of raw JSON strings. + +Facade: re-exports from _session_model, _session_content, _retry_fsm, and +_session_outcome sub-modules. +""" + +from __future__ import annotations + +from autoskillit.core import ( + CliSubtype, + SkillResult, + get_logger, + truncate_text, +) +from autoskillit.execution.session._exit_classification import ( + classify_infra_exit, # noqa: F401 — re-export for callers +) +from autoskillit.execution.session._retry_fsm import ( + _KILL_ANOMALY_SUBTYPES, # noqa: F401 — re-export for callers + _compute_retry, # noqa: F401 — re-export for callers + _is_kill_anomaly, # noqa: F401 — re-export for callers +) +from autoskillit.execution.session._session_content import ( + _check_expected_patterns, # noqa: F401 — re-export for callers + _check_session_content, # noqa: F401 — re-export for callers + _evaluate_content_state, # noqa: F401 — re-export for callers +) +from autoskillit.execution.session._session_model import ( + FAILURE_SUBTYPES, # noqa: F401 — re-export for callers + ClaudeSessionResult, # noqa: F401 — re-export for callers + ContentState, # noqa: F401 — re-export for callers + extract_token_usage, # noqa: F401 — re-export for callers + parse_session_result, # noqa: F401 — re-export for callers +) +from autoskillit.execution.session._session_outcome import ( + _compute_outcome, # noqa: F401 — re-export for callers + _compute_success, # noqa: F401 — re-export for callers +) +from autoskillit.execution.session._session_state import ( + SessionState, # noqa: F401 — re-export for callers + SessionStateLock, # noqa: F401 — re-export for callers + clear_session_state, # noqa: F401 — re-export for callers + persist_session_state, # noqa: F401 — re-export for callers + read_session_state, # noqa: F401 — re-export for callers +) + +logger = get_logger(__name__) +_truncate = truncate_text +# Re-export SkillResult so existing callers can import from this module. +__all__ = [ + "CliSubtype", + "SessionState", + "SessionStateLock", + "SkillResult", + "classify_infra_exit", + "clear_session_state", + "persist_session_state", + "read_session_state", +] diff --git a/src/autoskillit/execution/session/_exit_classification.py b/src/autoskillit/execution/session/_exit_classification.py new file mode 100644 index 0000000000..ea235c7e3e --- /dev/null +++ b/src/autoskillit/execution/session/_exit_classification.py @@ -0,0 +1,49 @@ +"""Infrastructure exit classification for headless sessions.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +import regex as re + +from autoskillit.core import InfraExitCategory, get_logger + +if TYPE_CHECKING: + from autoskillit.core import SubprocessResult + from autoskillit.execution.session._session_model import ClaudeSessionResult + +logger = get_logger(__name__) + +_API_ERROR_PATTERNS: tuple[re.Pattern[str], ...] = ( + re.compile(r"overloaded", re.IGNORECASE), + re.compile(r"\b529\b"), + re.compile(r"\b503\b"), + re.compile(r"ECONNRESET", re.IGNORECASE), + re.compile(r"ECONNREFUSED", re.IGNORECASE), + re.compile(r"socket hang up", re.IGNORECASE), + re.compile(r"network error", re.IGNORECASE), + re.compile(r"connection reset", re.IGNORECASE), +) + + +def _detect_api_error(stderr: str) -> bool: + return any(p.search(stderr) for p in _API_ERROR_PATTERNS) + + +def classify_infra_exit( + session: ClaudeSessionResult, + result: SubprocessResult, +) -> InfraExitCategory: + """Classify why a headless session exited at the infrastructure level. + + Priority order: context exhaustion > API error > process kill > completed. + Context exhaustion takes precedence because it is more specific — an + overloaded API response may co-occur with the final truncation error. + """ + if session._is_context_exhausted(): + return InfraExitCategory.CONTEXT_EXHAUSTED + if _detect_api_error(result.stderr): + return InfraExitCategory.API_ERROR + if result.returncode is not None and result.returncode < 0: + return InfraExitCategory.PROCESS_KILLED + return InfraExitCategory.COMPLETED diff --git a/src/autoskillit/execution/session/_retry_fsm.py b/src/autoskillit/execution/session/_retry_fsm.py new file mode 100644 index 0000000000..bc1bdaafbd --- /dev/null +++ b/src/autoskillit/execution/session/_retry_fsm.py @@ -0,0 +1,182 @@ +"""Retry FSM sub-module for headless session adjudication.""" + +from __future__ import annotations + +from typing import assert_never + +from autoskillit.core import ( + ChannelConfirmation, + CliSubtype, + RetryReason, + TerminationReason, + get_logger, +) +from autoskillit.execution.session._session_model import ClaudeSessionResult + +logger = get_logger(__name__) + +__all__ = ["_KILL_ANOMALY_SUBTYPES", "_is_kill_anomaly", "_compute_retry"] + + +_KILL_ANOMALY_SUBTYPES: frozenset[CliSubtype] = frozenset( + { + CliSubtype.UNPARSEABLE, # killed mid-write → partial NDJSON + CliSubtype.EMPTY_OUTPUT, # killed before any stdout was written + CliSubtype.INTERRUPTED, # killed mid-generation → real Claude CLI subtype + } +) + + +def _is_kill_anomaly(session: ClaudeSessionResult) -> bool: + """True if the session result looks like a kill-induced incomplete flush. + + Covers anomalies from both infrastructure kills (COMPLETED) and voluntary + self-exits (NATURAL_EXIT with returncode==0). Callers must apply their own + termination-specific discriminators before calling this function. + + Covers: + - unparseable: process killed mid-write, stdout is partial NDJSON + - empty_output: process killed before any stdout was written + - interrupted: process killed mid-generation (real Claude CLI subtype) + - success with empty result: kill occurred after result record was written + but with empty content (Channel B / Channel A drain-race, or + CLAUDE_CODE_EXIT_AFTER_STOP_DELAY timer-based self-exit) + """ + if session.subtype in _KILL_ANOMALY_SUBTYPES: + return True + if session.subtype == CliSubtype.SUCCESS and not session.result.strip(): + return True + return False + + +def _compute_retry( + session: ClaudeSessionResult, + returncode: int, + termination: TerminationReason, + channel_confirmation: ChannelConfirmation = ChannelConfirmation.UNMONITORED, + completion_marker: str = "", +) -> tuple[bool, RetryReason]: + """Compute whether the session result warrants a retry. + + Phase 1: API-level signals are termination-agnostic. + Phase 2: Exhaustive match dispatch over TerminationReason ensures mypy + flags any unhandled value when the enum is extended. + + When ``channel_confirmation=CHANNEL_B`` and ``termination=COMPLETED``, + the provenance bypass applies: Channel B's session-JSONL signal is + authoritative, so kill-anomaly appearance is a drain-race artifact. + No retry is needed. + """ + # Phase 1: API-level retry signals (context exhaustion, max_turns) + if session.needs_retry: + logger.debug("compute_retry_api_signal", needs_retry=True, reason="resume") + return True, RetryReason.RESUME + + # Phase 2: Exhaustive termination dispatch + match termination: + case TerminationReason.NATURAL_EXIT: + if channel_confirmation in ( + ChannelConfirmation.CHANNEL_A, + ChannelConfirmation.CHANNEL_B, + ): + logger.debug( + "compute_retry_result", + termination="NATURAL_EXIT", + channel=str(channel_confirmation), + needs_retry=False, + ) + return False, RetryReason.NONE + if returncode == 0 and _is_kill_anomaly(session): + if session.has_thinking_only_turn: + reason = RetryReason.THINKING_STALL + elif session._is_context_exhausted(): + reason = RetryReason.RESUME + else: + reason = RetryReason.EMPTY_OUTPUT + logger.debug( + "compute_retry_result", + termination="NATURAL_EXIT", + channel=str(channel_confirmation), + needs_retry=True, + kill_anomaly=True, + context_exhausted=reason == RetryReason.RESUME, + ) + return True, reason + # EARLY_STOP: model produced substantive output but stopped before + # emitting the completion marker (text-then-tool boundary). + if ( + returncode == 0 + and session.subtype == CliSubtype.SUCCESS + and session.result.strip() + and completion_marker + and completion_marker not in session.result + ): + skill_tool_calls = [t for t in session.tool_uses if t.get("name") == "Skill"] + logger.debug( + "compute_retry_early_stop", + termination="NATURAL_EXIT", + has_skill_calls=bool(skill_tool_calls), + skill_call_count=len(skill_tool_calls), + ) + return True, RetryReason.EARLY_STOP + logger.debug( + "compute_retry_result", + termination="NATURAL_EXIT", + channel=str(channel_confirmation), + needs_retry=False, + ) + return False, RetryReason.NONE + + case TerminationReason.COMPLETED: + # Infrastructure killed the process. SIGTERM/SIGKILL produce nonzero + # returncode by design — do not gate on returncode here. + # Exhaustive ChannelConfirmation dispatch (ARCH-007 extension): + match channel_confirmation: + case ChannelConfirmation.CHANNEL_B: + # Channel B is authoritative — kill-anomaly appearance is + # a drain-race artifact, not a real incomplete flush. + logger.debug( + "compute_retry_result", + termination="COMPLETED", + channel="CHANNEL_B", + needs_retry=False, + ) + return False, RetryReason.NONE + case ( + ChannelConfirmation.CHANNEL_A + | ChannelConfirmation.UNMONITORED + | ChannelConfirmation.DIR_MISSING + ): + is_anomaly = _is_kill_anomaly(session) + logger.debug( + "compute_retry_result", + termination="COMPLETED", + channel=str(channel_confirmation), + needs_retry=is_anomaly, + kill_anomaly=is_anomaly, + ) + if is_anomaly: + return True, RetryReason.RESUME + return False, RetryReason.NONE + case _ as _unreachable_cc: + assert_never(_unreachable_cc) + + case TerminationReason.STALE: + # _build_skill_result intercepts STALE before calling _compute_retry. + # Explicit arm exists for exhaustiveness; unreachable in production. + logger.debug("compute_retry_result", termination="STALE", needs_retry=False) + return False, RetryReason.NONE + + case TerminationReason.IDLE_STALL: + # _build_skill_result intercepts IDLE_STALL before calling _compute_retry. + # Explicit arm exists for exhaustiveness; unreachable in production. + logger.debug("compute_retry_result", termination="IDLE_STALL", needs_retry=False) + return False, RetryReason.NONE + + case TerminationReason.TIMED_OUT: + # Wall-clock timeout: non-retriable (permanent infrastructure limit). + logger.debug("compute_retry_result", termination="TIMED_OUT", needs_retry=False) + return False, RetryReason.NONE + + case _ as unreachable: + assert_never(unreachable) diff --git a/src/autoskillit/execution/session/_session_content.py b/src/autoskillit/execution/session/_session_content.py new file mode 100644 index 0000000000..185f1a79cb --- /dev/null +++ b/src/autoskillit/execution/session/_session_content.py @@ -0,0 +1,178 @@ +"""Session content validation functions extracted from session.py. + +Private sub-module — import from autoskillit.execution.session for public API. +""" + +from __future__ import annotations + +from collections.abc import Sequence + +import regex as re + +from autoskillit.core import get_logger +from autoskillit.execution.session._session_model import ( + FAILURE_SUBTYPES, + ClaudeSessionResult, + ContentState, +) + +logger = get_logger(__name__) + +# Bold + equals: **key** = value → key = value +_MARKDOWN_BOLD_EQUALS_RE: re.Pattern[str] = re.compile( + r"\*\*{1,2}(\w[\w_-]*)\*{1,2}(\s*=)", re.MULTILINE +) +# Bold + colon: **key:** value OR **key**: value OR **key** : value → key = value +# Handles colon inside bold (**key:**), immediately after (**key**:), or spaced (**key** :). +_MARKDOWN_BOLD_COLON_RE: re.Pattern[str] = re.compile( + r"\*\*(\w[\w_-]*)(?:\*{0,2}:\*{0,2}|\*{1,2}\s*:)\s+", re.MULTILINE +) +# Italic + equals: *key* = value → key = value +_MARKDOWN_ITALIC_EQUALS_RE: re.Pattern[str] = re.compile( + r"(? str: + """Remove bold/italic markdown decorators from structured output token names. + + Transforms model output like: + **plan_path** = /abs/path/plan.md + **Plan_Path:** /abs/path/plan.md + `plan_path` = /abs/path/plan.md + into the canonical form: + plan_path = /abs/path/plan.md + + Applied before regex pattern matching to make adjudication tolerant of the + model's choice to visually style its output summary. Three decorator variants + are handled: bold/italic with equals, bold/italic with colon, and backtick + wrapping. Decorators elsewhere in the text are left unchanged. + """ + text = _MARKDOWN_BOLD_EQUALS_RE.sub(lambda m: m.group(1).lower() + m.group(2), text) + text = _MARKDOWN_BOLD_COLON_RE.sub(lambda m: m.group(1).lower() + " = ", text) + text = _MARKDOWN_ITALIC_EQUALS_RE.sub(lambda m: m.group(1).lower() + m.group(2), text) + text = _MARKDOWN_ITALIC_COLON_RE.sub(lambda m: m.group(1).lower() + " = ", text) + text = _MARKDOWN_BACKTICK_RE.sub(lambda m: m.group(1).lower() + m.group(2), text) + return text + + +def _check_expected_patterns(result: str, patterns: Sequence[str]) -> bool: + """Return True if ALL expected_output_patterns are found in result, or if + no patterns are configured. This check MUST run on all session outcome paths, + including the Channel B bypass path. + + AND semantics are intentional: patterns represent content contracts (e.g., + block start/end delimiters) that must all be present simultaneously. + + Normalizes bold/italic markdown decorators on token names before matching, + so ``**plan_path** = /path`` is treated identically to ``plan_path = /path``. + + If any pattern is an invalid regex, returns False rather than raising. + """ + if not patterns: + return True + normalized = _strip_markdown_from_tokens(result) + for p in patterns: + try: + if not re.search(p, normalized): + return False + except re.error: + logger.warning("invalid_expected_output_pattern", pattern=p) + return False + return True + + +def _check_session_content( + session: ClaudeSessionResult, + completion_marker: str, + expected_output_patterns: Sequence[str] = (), +) -> bool: + """Validate session content fields after termination-specific gates pass.""" + if session.is_error: + logger.debug("content_check_failed", reason="is_error", is_error=True) + return False + if not session.result.strip(): + logger.debug("content_check_failed", reason="empty_result") + return False + if session.subtype in FAILURE_SUBTYPES: + logger.debug("content_check_failed", reason="failure_subtype", subtype=session.subtype) + return False + if completion_marker: + result_text = session.result.strip() + marker_stripped = result_text.replace(completion_marker, "").strip() + if not marker_stripped: + logger.debug("content_check_failed", reason="result_is_only_marker") + return False + if completion_marker not in result_text: + logger.debug( + "content_check_failed", + reason="completion_marker_absent", + result_tail=result_text[-200:] if len(result_text) > 200 else result_text, + ) + return False + if not _check_expected_patterns(session.result.strip(), expected_output_patterns): + logger.warning( + "content_check_failed", + reason="expected_artifact_absent", + patterns=list(expected_output_patterns), + ) + return False + logger.debug("content_check_passed") + return True + + +def _evaluate_content_state( + session: ClaudeSessionResult, + completion_marker: str, + expected_output_patterns: Sequence[str], +) -> ContentState: + """Classify the content completeness and contract compliance of a session result. + + Returns: + ContentState.COMPLETE: Result is non-empty, marker present (if configured), + and all expected_output_patterns match. Session is fully successful. + ContentState.ABSENT: Result is empty OR completion marker is absent from a + non-empty result. Indicates a drain-race artifact — the session may have + completed but stdout was not fully flushed. Retriable. + ContentState.CONTRACT_VIOLATION: Result is non-empty and contains the marker, + but one or more expected_output_patterns are absent. The session ran to + completion but the model did not produce the required output tokens. + Terminal — retrying will not produce different output. + ContentState.SESSION_ERROR: The CLI session itself reported an error + (is_error=True) or produced a failure subtype. Terminal. + """ + # Process-level / CLI-level failure — terminal regardless of content + if session.is_error: + return ContentState.SESSION_ERROR + + result = session.result.strip() + + # Empty result — drain-race candidate regardless of content requirements. + # This must come before the "no requirements" shortcut so that CHANNEL_A + # dead-end guard can detect drain-race artifacts even when no marker or + # patterns are configured. + if not result: + return ContentState.ABSENT + + # No content requirements configured and result is non-empty: CHANNEL_B + # confirmation alone is sufficient. Returning COMPLETE here preserves the + # existing behaviour for skills that produce non-empty output without a + # marker (e.g. fire-and-forget commands with plain text output). + if not completion_marker and not expected_output_patterns: + return ContentState.COMPLETE + + # Marker absent — partial drain candidate + if completion_marker and completion_marker not in result: + return ContentState.ABSENT + + # Result non-empty, marker present (or not configured) — check pattern contract + if expected_output_patterns and not _check_expected_patterns(result, expected_output_patterns): + return ContentState.CONTRACT_VIOLATION + + return ContentState.COMPLETE diff --git a/src/autoskillit/execution/session/_session_model.py b/src/autoskillit/execution/session/_session_model.py new file mode 100644 index 0000000000..e5096123b6 --- /dev/null +++ b/src/autoskillit/execution/session/_session_model.py @@ -0,0 +1,436 @@ +"""Session result model and parser — private sub-module of execution/session.py.""" + +from __future__ import annotations + +import json +from dataclasses import dataclass, field +from enum import StrEnum +from typing import Any, assert_never + +import regex as re + +from autoskillit.core import ( + CONTEXT_EXHAUSTION_MARKER, + ClaudeContentBlockType, + CliSubtype, + RetryReason, + SessionOutcome, + get_logger, +) + +logger = get_logger(__name__) + +_ABS_PATH_RE: re.Pattern[str] = re.compile(r'(?:^|[\s="\'])(/(?:[a-zA-Z0-9._/~@+-]+))') + +_TOKEN_FIELDS = ( + "input_tokens", + "output_tokens", + "cache_creation_input_tokens", + "cache_read_input_tokens", +) + +FAILURE_SUBTYPES: frozenset[CliSubtype] = frozenset( + { + CliSubtype.UNKNOWN, + CliSubtype.EMPTY_OUTPUT, + CliSubtype.UNPARSEABLE, + CliSubtype.TIMEOUT, + CliSubtype.IDLE_STALL, + } +) + + +class ContentState(StrEnum): + """Content evaluation state for dead-end / drain-race guard dispatch.""" + + COMPLETE = "complete" + ABSENT = "absent" # Empty result or missing marker — drain-race candidate + CONTRACT_VIOLATION = "contract_violation" # Result + marker present, but patterns fail + SESSION_ERROR = "session_error" # is_error=True or process-level failure + + +@dataclass +class ClaudeSessionResult: + """Parsed result from a Claude Code headless session.""" + + subtype: CliSubtype + is_error: bool + result: str + session_id: str + errors: list[str] = field(default_factory=list) + token_usage: dict[str, Any] | None = None + assistant_messages: list[str] = field(default_factory=list) + tool_uses: list[dict[str, Any]] = field(default_factory=list) + jsonl_context_exhausted: bool = False + stop_reasons: list[str] = field(default_factory=list) + has_thinking_only_turn: bool = False + seen_block_types: frozenset[str] = field(default_factory=frozenset) + + def __post_init__(self) -> None: + if not isinstance(self.result, str): + if isinstance(self.result, list): + parts: list[str] = [] + for b in self.result: + if not isinstance(b, dict): + parts.append(str(b)) + continue + block_type = ClaudeContentBlockType.from_api(b.get("type", "")) + if block_type == ClaudeContentBlockType.TEXT: + parts.append(b.get("text", "")) + # Non-text blocks (thinking, tool_use, etc.) contribute no text + self.result = "\n".join(parts) + elif not isinstance(self.result, str): + self.result = "" if self.result is None else str(self.result) + if not isinstance(self.errors, list): + self.errors = [] if self.errors is None else [str(self.errors)] + if not isinstance(self.subtype, CliSubtype): + self.subtype = ( + CliSubtype.UNKNOWN + if self.subtype is None + else CliSubtype.from_cli(str(self.subtype)) + ) + if not isinstance(self.session_id, str): + self.session_id = "" if self.session_id is None else str(self.session_id) + if not isinstance(self.jsonl_context_exhausted, bool): + self.jsonl_context_exhausted = bool(self.jsonl_context_exhausted) + + def _is_context_exhausted(self) -> bool: + """True when the session hit Claude's context window limit.""" + if self.jsonl_context_exhausted: + return True + if not self.is_error: + return False + marker = CONTEXT_EXHAUSTION_MARKER + if any(marker in e.lower() for e in self.errors): + return True + if ( + self.subtype in (CliSubtype.SUCCESS, CliSubtype.ERROR_MAX_TURNS) + and marker in self.result.lower() + ): + return True + return False + + @property + def agent_result(self) -> str: + """Result text rewritten for LLM agent consumption.""" + if self._is_context_exhausted(): + return ( + "Context limit reached during session execution. " + "The session made partial progress. " + "Use needs_retry and retry_reason to continue from where it left off." + ) + if self.subtype == CliSubtype.ERROR_MAX_TURNS: + return ( + "Turn limit reached during session execution. " + "The session made partial progress. " + "Use needs_retry and retry_reason to continue from where it left off." + ) + return self.result + + @property + def needs_retry(self) -> bool: + """Whether the session didn't finish and should be retried.""" + if self.subtype == CliSubtype.ERROR_MAX_TURNS: + return True + if self._is_context_exhausted(): + return True + return False + + @property + def retry_reason(self) -> RetryReason: + """Why retry is needed. NONE if needs_retry is False.""" + if self.needs_retry: + return RetryReason.RESUME + return RetryReason.NONE + + def normalize_subtype(self, outcome: SessionOutcome, completion_marker: str) -> str: + """Map (outcome, completion_marker) → adjudicated subtype string. + + Class 2 (upward): SUCCEEDED + failure subtype → "success". + Class 1 (downward): non-SUCCEEDED + success subtype → synthesized label. + Return type is str because downward normalization synthesizes labels not in CliSubtype. + """ + match self.subtype: + case CliSubtype.SUCCESS: + if outcome == SessionOutcome.SUCCEEDED: + return self.subtype + if not self.result.strip(): + return "empty_result" + if self._is_context_exhausted(): + return "context_exhausted" + if completion_marker and completion_marker not in self.result: + return "missing_completion_marker" + return "adjudicated_failure" + case ( + CliSubtype.UNKNOWN + | CliSubtype.EMPTY_OUTPUT + | CliSubtype.UNPARSEABLE + | CliSubtype.TIMEOUT + | CliSubtype.IDLE_STALL + ): + if outcome == SessionOutcome.SUCCEEDED: + return "success" + return self.subtype + case ( + CliSubtype.ERROR_MAX_TURNS + | CliSubtype.ERROR_DURING_EXECUTION + | CliSubtype.CONTEXT_EXHAUSTION + | CliSubtype.INTERRUPTED + ): + return self.subtype + case _ as unreachable: + assert_never(unreachable) + + @property + def session_complete(self) -> bool: + """True when not in error state and subtype is not in the failure set.""" + return not self.is_error and self.subtype not in FAILURE_SUBTYPES + + @property + def last_stop_reason(self) -> str: + """The stop_reason from the final assistant turn, or empty string.""" + return self.stop_reasons[-1] if self.stop_reasons else "" + + @property + def lifespan_started(self) -> bool: + """Heuristic: True when at least one MCP tool call was observed.""" + return bool(self.tool_uses) + + +def extract_token_usage(stdout: str) -> dict[str, Any] | None: + """Extract token usage from Claude CLI NDJSON output. + + Takes raw stdout (not ClaudeSessionResult) — called during parse_session_result + construction before the object exists. Returns None if no usage data found. + """ + if not stdout.strip(): + return None + + model_buckets: dict[str, dict[str, int]] = {} + result_usage: dict[str, int] | None = None + peak_context = 0 + turn_count = 0 + + for line in stdout.strip().splitlines(): + line = line.strip() + if not line: + continue + try: + obj = json.loads(line) + except json.JSONDecodeError: + continue + if not isinstance(obj, dict): + continue + + record_type = obj.get("type") + if record_type == "assistant": + msg = obj.get("message") + if not isinstance(msg, dict): + continue + usage = msg.get("usage") + if not isinstance(usage, dict): + continue + model = msg.get("model", "unknown") + bucket = model_buckets.setdefault(model, {f: 0 for f in _TOKEN_FIELDS}) + for f in _TOKEN_FIELDS: + bucket[f] += usage.get(f, 0) + cr = usage.get("cache_read_input_tokens", 0) + if cr > peak_context: + peak_context = cr + turn_count += 1 + elif record_type == "result": + usage = obj.get("usage") + if isinstance(usage, dict): + result_usage = {f: usage.get(f, 0) for f in _TOKEN_FIELDS} + + if not model_buckets and result_usage is None: + return None + + if result_usage is not None: + totals = dict(result_usage) + else: + totals = {f: 0 for f in _TOKEN_FIELDS} + for bucket in model_buckets.values(): + for f in _TOKEN_FIELDS: + totals[f] += bucket[f] + + return { + **totals, + "model_breakdown": dict(model_buckets) if model_buckets else {}, + "peak_context": peak_context, + "turn_count": turn_count, + } + + +_KNOWN_RESULT_KEYS: frozenset[str] = frozenset( + {"type", "subtype", "is_error", "result", "session_id", "errors", "usage"} +) + + +@dataclass +class _ParseAccumulator: + """Mutable accumulator for parse_session_result's NDJSON scan.""" + + result_obj: dict[str, Any] | None = None + tool_uses: list[dict[str, Any]] = field(default_factory=list) + assistant_messages: list[str] = field(default_factory=list) + jsonl_context_exhausted: bool = False + stop_reasons: list[str] = field(default_factory=list) + seen_block_types: set[str] = field(default_factory=set) + has_thinking_only_turn: bool = False + + +def parse_session_result(stdout: str) -> ClaudeSessionResult: + """Parse Claude Code NDJSON stdout into a typed result. + + Scans all NDJSON records into a _ParseAccumulator then constructs a single + ClaudeSessionResult — no early returns, ensuring tool_uses and token_usage + are preserved on all paths including fallback/unparseable. + """ + if not stdout.strip(): + return ClaudeSessionResult( + subtype=CliSubtype.EMPTY_OUTPUT, + is_error=True, + result="", + session_id="", + errors=[], + ) + + acc = _ParseAccumulator() + marker = CONTEXT_EXHAUSTION_MARKER + + for line in stdout.strip().splitlines(): + line = line.strip() + if not line: + continue + try: + obj = json.loads(line) + if not isinstance(obj, dict): + continue + record_type = obj.get("type") + if record_type == "result": + acc.result_obj = obj + elif record_type == "assistant": + msg = obj.get("message") + if isinstance(msg, dict): + content = msg.get("content", "") + if isinstance(content, list): + text_parts: list[str] = [] + turn_has_thinking = False + turn_has_tool_use = False + for block in content: + if not isinstance(block, dict): + continue + block_type = ClaudeContentBlockType.from_api(block.get("type", "")) + match block_type: + case ClaudeContentBlockType.TEXT: + text_parts.append(block.get("text", "")) + case ClaudeContentBlockType.TOOL_USE: + turn_has_tool_use = True + name = block.get("name", "") + entry: dict[str, str | list[str]] = { + "name": name, + "id": block.get("id", ""), + } + if name in {"Write", "Edit"} and isinstance( + block.get("input"), dict + ): + fp = block["input"].get("file_path", "") + if fp: + entry["file_path"] = fp + elif name == "Bash" and isinstance(block.get("input"), dict): + command = block["input"].get("command", "") + if isinstance(command, str): + paths = [ + m.group(1) + for m in _ABS_PATH_RE.finditer(command) + if len(m.group(1)) >= 5 + ] + if paths: + entry["bash_paths"] = paths + acc.tool_uses.append(entry) + case ( + ClaudeContentBlockType.THINKING + | ClaudeContentBlockType.REDACTED_THINKING + ): + turn_has_thinking = True + case ( + ClaudeContentBlockType.TOOL_RESULT + | ClaudeContentBlockType.IMAGE + ): + pass # Not expected in assistant content; skip gracefully + case ClaudeContentBlockType.UNKNOWN: + raw_type = block.get("type", "") + logger.debug("unknown_content_block_type", block_type=raw_type) + acc.seen_block_types.add(raw_type) + case _ as unreachable: + assert_never(unreachable) + text = "\n".join(text_parts).strip() + if turn_has_thinking and not text_parts and not turn_has_tool_use: + acc.has_thinking_only_turn = True + else: + text = str(content).strip() + if text: + acc.assistant_messages.append(text) + _stop = msg.get("stop_reason", "") + if _stop: + acc.stop_reasons.append(str(_stop)) + elif "message" not in obj: + # Flat assistant record — detect context exhaustion inline. + if obj.get("output_tokens", -1) == 0: + flat_content = obj.get("content", []) + if isinstance(flat_content, list) and any( + isinstance(block, dict) + and block.get("type") == "text" + and marker in block.get("text", "").lower() + for block in flat_content + ): + acc.jsonl_context_exhausted = True + except json.JSONDecodeError: + continue + + if acc.result_obj is None: + try: + fallback = json.loads(stdout) + if isinstance(fallback, dict) and fallback.get("type") == "result": + acc.result_obj = fallback + except json.JSONDecodeError: + pass + + token_usage = extract_token_usage(stdout) + + if acc.result_obj is not None: + extra_keys = frozenset(acc.result_obj.keys()) - _KNOWN_RESULT_KEYS + if extra_keys: + logger.debug("unknown_result_keys", unknown_fields=sorted(extra_keys)) + subtype = CliSubtype.from_cli(acc.result_obj.get("subtype") or "unknown") + is_error: bool = acc.result_obj.get("is_error", False) + result_text: str = acc.result_obj.get("result") or "" + session_id: str = acc.result_obj.get("session_id") or "" + errors: list[str] = acc.result_obj.get("errors") or [] + else: + if acc.jsonl_context_exhausted: + subtype = CliSubtype.CONTEXT_EXHAUSTION + else: + subtype = CliSubtype.UNPARSEABLE + is_error = True + result_text = stdout + session_id = "" + errors = [] + + return ClaudeSessionResult( + subtype=subtype, + is_error=is_error, + result=result_text, + session_id=session_id, + errors=errors, + token_usage=token_usage, + assistant_messages=acc.assistant_messages, + tool_uses=acc.tool_uses, + jsonl_context_exhausted=acc.jsonl_context_exhausted, + stop_reasons=acc.stop_reasons, + has_thinking_only_turn=acc.has_thinking_only_turn, + seen_block_types=frozenset( + acc.seen_block_types + ), # accumulator uses set; result is immutable + ) diff --git a/src/autoskillit/execution/session/_session_outcome.py b/src/autoskillit/execution/session/_session_outcome.py new file mode 100644 index 0000000000..ade9758555 --- /dev/null +++ b/src/autoskillit/execution/session/_session_outcome.py @@ -0,0 +1,237 @@ +"""Session outcome adjudication sub-module.""" + +from __future__ import annotations + +from collections.abc import Sequence +from typing import assert_never + +from autoskillit.core import ( + ChannelConfirmation, + CliSubtype, + RetryReason, + SessionOutcome, + TerminationReason, + get_logger, +) +from autoskillit.execution.session._retry_fsm import _compute_retry +from autoskillit.execution.session._session_content import ( + _check_expected_patterns, + _check_session_content, + _evaluate_content_state, +) +from autoskillit.execution.session._session_model import ClaudeSessionResult, ContentState + +logger = get_logger(__name__) + +__all__ = ["_compute_success", "_compute_outcome"] + + +def _compute_success( + session: ClaudeSessionResult, + returncode: int, + termination: TerminationReason, + completion_marker: str = "", + channel_confirmation: ChannelConfirmation = ChannelConfirmation.UNMONITORED, + expected_output_patterns: Sequence[str] = (), +) -> bool: + """Cross-validate all signals to determine unambiguous success/failure. + + Exhaustive match dispatch over TerminationReason ensures mypy flags any + unhandled value when the enum is extended (ARCH-007). + + Gate 0.5 (provenance bypass): when ``channel_confirmation=CHANNEL_B``, + Channel B's session-JSONL marker is the authoritative signal that the + session completed successfully. Stdout content is not required. + """ + # Gate 0.5: Channel B provenance bypass — session JSONL is authoritative. + match channel_confirmation: + case ChannelConfirmation.CHANNEL_B: + if expected_output_patterns and not session.session_complete: + logger.debug( + "channel_b_bypass_skipped_incomplete_session", + subtype=str(session.subtype), + is_error=session.is_error, + pattern_count=len(expected_output_patterns), + ) + else: + if not _check_expected_patterns(session.result.strip(), expected_output_patterns): + logger.debug( + "channel_b_content_check_failed", + result_len=len(session.result), + pattern_count=len(expected_output_patterns), + ) + return False + logger.debug("compute_success_bypass", channel="CHANNEL_B", result=True) + return True + case ( + ChannelConfirmation.CHANNEL_A + | ChannelConfirmation.UNMONITORED + | ChannelConfirmation.DIR_MISSING + ): + pass # fall through to termination dispatch + case _ as _unreachable_cc: + assert_never(_unreachable_cc) + + match termination: + case TerminationReason.TIMED_OUT: + return False + + case TerminationReason.STALE: + return False + + case TerminationReason.IDLE_STALL: + return False + + case TerminationReason.COMPLETED: + # The process was killed by our own async_kill_process_tree + # (signal -15 or -9), so a non-zero returncode is expected and + # trustworthy when the session envelope says "success". + if returncode != 0 and not ( + session.subtype == CliSubtype.SUCCESS and session.result.strip() + ): + return False + content_ok = _check_session_content( + session, completion_marker, expected_output_patterns + ) + logger.debug( + "compute_success_termination", + termination="COMPLETED", + returncode=returncode, + content_check=content_ok, + ) + return content_ok + + case TerminationReason.NATURAL_EXIT: + # The process exited on its own. A non-zero returncode is normally + # authoritative evidence of failure — no asymmetric bypass. + # + # Post-completion kill bypass: when an external watchdog kills the + # process AFTER it finished its work (e.g. trailing async task cleanup), + # the signal is a teardown artifact. The completion marker in the result + # provides strong evidence of completion — trust it over the returncode. + if returncode != 0: + if ( + session.subtype == CliSubtype.SUCCESS + and session.result.strip() + and completion_marker + and completion_marker in session.result + ): + content_ok = _check_session_content( + session, completion_marker, expected_output_patterns + ) + logger.debug( + "compute_success_natural_exit_post_completion_kill", + returncode=returncode, + content_check=content_ok, + ) + return content_ok + return False + content_ok = _check_session_content( + session, completion_marker, expected_output_patterns + ) + logger.debug( + "compute_success_termination", + termination="NATURAL_EXIT", + returncode=returncode, + content_check=content_ok, + ) + return content_ok + + case _ as unreachable: + assert_never(unreachable) + + +def _compute_outcome( + session: ClaudeSessionResult, + returncode: int, + termination: TerminationReason, + completion_marker: str = "", + channel_confirmation: ChannelConfirmation = ChannelConfirmation.UNMONITORED, + expected_output_patterns: Sequence[str] = (), +) -> tuple[SessionOutcome, RetryReason]: + """Compose _compute_success and _compute_retry into a (SessionOutcome, RetryReason) pair. + + Applies contradiction guard (retry demotes success) and dead-end guard (ABSENT + drain-race promotes to RETRIABLE) before mapping to the bijective SessionOutcome enum. + """ + success = _compute_success( + session, + returncode, + termination, + completion_marker, + channel_confirmation, + expected_output_patterns, + ) + needs_retry, retry_reason = _compute_retry( + session, returncode, termination, channel_confirmation, completion_marker + ) + + logger.debug( + "compute_outcome_inputs", + success=success, + needs_retry=needs_retry, + retry_reason=str(retry_reason), + returncode=returncode, + termination=str(termination), + channel=str(channel_confirmation), + subtype=session.subtype, + is_error=session.is_error, + result_empty=not session.result.strip(), + ) + + # Contradiction guard: retry signal is authoritative over the channel bypass. + if success and needs_retry: + success = False + logger.debug( + "contradiction_guard", + action="demoted_success", + reason="retry_signal_authoritative", + ) + + # Dead-end guard: channel confirmation means the session signalled completion, but + # content checks failed and no retry was scheduled by _compute_retry. + # Only promote to RETRIABLE when the failure is a drain-race artifact (ABSENT state): + # - empty result → stdout not fully flushed + # - marker missing → partial flush + # Do NOT promote CONTRACT_VIOLATION or SESSION_ERROR — these are terminal failures + # that retrying will never resolve. + if not success and not needs_retry: + match channel_confirmation: + case ChannelConfirmation.CHANNEL_A | ChannelConfirmation.CHANNEL_B: + content_state = _evaluate_content_state( + session, completion_marker, expected_output_patterns + ) + if content_state == ContentState.ABSENT: + needs_retry = True + retry_reason = RetryReason.DRAIN_RACE + logger.debug( + "dead_end_guard", + action="promoted_to_retriable", + content_state=content_state.value, + channel=channel_confirmation.value, + ) + else: + logger.debug( + "dead_end_guard", + action="terminal_failure_not_promoted", + content_state=content_state.value, + channel=channel_confirmation.value, + ) + case ChannelConfirmation.UNMONITORED | ChannelConfirmation.DIR_MISSING: + pass # legitimate terminal failure — no channel confirmed completion + case _ as unreachable_cc: + assert_never(unreachable_cc) + + if success: + outcome = SessionOutcome.SUCCEEDED + elif needs_retry: + outcome = SessionOutcome.RETRIABLE + else: + outcome = SessionOutcome.FAILED + + logger.debug( + "compute_outcome_result", + outcome=str(outcome), + retry_reason=str(retry_reason), + ) + return outcome, retry_reason diff --git a/src/autoskillit/execution/session/_session_state.py b/src/autoskillit/execution/session/_session_state.py new file mode 100644 index 0000000000..1ff39c0b64 --- /dev/null +++ b/src/autoskillit/execution/session/_session_state.py @@ -0,0 +1,122 @@ +"""Session state persistence for resume support. + +IL-1 module: imports from core (IL-0) only. Provides atomic read/write of +dispatch session state files so a dead session can be identified and resumed. +""" + +from __future__ import annotations + +import fcntl +import json +import os +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +from autoskillit.core import atomic_write, get_logger +from autoskillit.core import fast_dumps as _fast_dumps + +logger = get_logger(__name__) + +_STATE_FILENAME = "dispatch_session_state.json" +_LOCK_FILENAME = "dispatch_session_state.lock" +_FD_CLOSED = -1 + + +@dataclass(frozen=True, slots=True) +class SessionState: + session_id: str + pid: int + boot_id: str + starttime_ticks: int + checkpoint_path: str | None = None + infra_exit_category: str | None = None + + def to_dict(self) -> dict[str, object]: + d: dict[str, object] = { + "session_id": self.session_id, + "pid": self.pid, + "boot_id": self.boot_id, + "starttime_ticks": self.starttime_ticks, + } + if self.checkpoint_path is not None: + d["checkpoint_path"] = self.checkpoint_path + if self.infra_exit_category is not None: + d["infra_exit_category"] = self.infra_exit_category + return d + + @classmethod + def from_dict(cls, data: dict[str, Any]) -> SessionState | None: + try: + return cls( + session_id=str(data["session_id"]), + pid=int(data["pid"]), + boot_id=str(data["boot_id"]), + starttime_ticks=int(data["starttime_ticks"]), + checkpoint_path=data.get("checkpoint_path"), + infra_exit_category=data.get("infra_exit_category"), + ) + except (KeyError, TypeError, ValueError): + return None + + +def persist_session_state(state: SessionState, state_dir: Path) -> Path: + state_dir.mkdir(parents=True, exist_ok=True) + path = state_dir / _STATE_FILENAME + atomic_write(path, _fast_dumps(state.to_dict(), indent=True)) + return path + + +def read_session_state(state_dir: Path) -> SessionState | None: + path = state_dir / _STATE_FILENAME + try: + data = json.loads(path.read_text()) + except (OSError, json.JSONDecodeError): + return None + return SessionState.from_dict(data) + + +def clear_session_state(state_dir: Path) -> None: + path = state_dir / _STATE_FILENAME + try: + path.unlink(missing_ok=True) + except OSError: + logger.debug("session_state: could not remove state file", path=str(path)) + + +class SessionStateLock: + """File-based concurrency guard for session resume operations.""" + + def __init__(self, state_dir: Path) -> None: + self._lock_path = state_dir / _LOCK_FILENAME + self._fd: int = _FD_CLOSED + + def acquire(self) -> bool: + self._lock_path.parent.mkdir(parents=True, exist_ok=True) + try: + self._fd = os.open(str(self._lock_path), os.O_WRONLY | os.O_CREAT, 0o644) + fcntl.flock(self._fd, fcntl.LOCK_EX | fcntl.LOCK_NB) + return True + except (OSError, BlockingIOError): + if self._fd != _FD_CLOSED: + try: + os.close(self._fd) + except OSError: + pass + self._fd = _FD_CLOSED + return False + + def release(self) -> None: + if self._fd != _FD_CLOSED: + try: + fcntl.flock(self._fd, fcntl.LOCK_UN) + os.close(self._fd) + except OSError: + pass + self._fd = _FD_CLOSED + + def __enter__(self) -> bool: + return self.acquire() + + def __exit__(self, *_: object) -> None: + self.release() diff --git a/src/autoskillit/execution/session_log.py b/src/autoskillit/execution/session_log.py index 25785afd47..a6d78d30ce 100644 --- a/src/autoskillit/execution/session_log.py +++ b/src/autoskillit/execution/session_log.py @@ -13,14 +13,28 @@ import shutil import sys import time +from collections.abc import Callable from datetime import UTC, datetime from pathlib import Path -from typing import Any +from typing import TYPE_CHECKING, Any + +if TYPE_CHECKING: + from autoskillit.core import ProviderOutcome, RecipeIdentity, SessionTelemetry import psutil -from autoskillit.core import atomic_write, claude_code_log_path, get_logger -from autoskillit.execution.anomaly_detection import detect_anomalies, detect_identity_drift +from autoskillit.core import ( + atomic_write, + claude_code_log_path, + get_logger, + iter_merged_assistant_turns, +) +from autoskillit.core import fast_dumps as _fast_dumps +from autoskillit.execution.anomaly_detection import ( + detect_anomalies, + detect_identity_drift, + detect_outcome_anomalies, +) from autoskillit.execution.linux_tracing import ( read_boot_id, read_enrollment, @@ -29,7 +43,22 @@ logger = get_logger(__name__) -_MAX_SESSIONS = 500 +_MAX_SESSIONS = 2000 + + +def _primary_model_identifier(token_usage: dict[str, Any] | None) -> str: + """Return the model name with the most total tokens from model_breakdown. + + Returns "" when token_usage is absent or model_breakdown is empty. + """ + if not token_usage: + return "" + mb = token_usage.get("model_breakdown", {}) + if not isinstance(mb, dict) or not mb: + return "" + return max(mb, key=lambda m: sum(mb[m].values()) if isinstance(mb[m], dict) else 0) + + _CLEAR_MARKER_FILENAME = ".telemetry_cleared_at" @@ -70,12 +99,30 @@ def read_telemetry_clear_marker(log_root: Path) -> datetime | None: return None +def _resolve_session_label(step_name: str, dispatch_id: str) -> str: + """Derive a non-empty session label for telemetry file identification. + + Recipe steps use step_name. Fleet dispatches use dispatch_id. + Ad-hoc sessions get a fallback label. + """ + if step_name: + return step_name + if dispatch_id: + return f"dispatch:{dispatch_id}" + return "(ad-hoc)" + + def flush_session_log( *, log_dir: str, cwd: str, kitchen_id: str = "", + caller_session_id: str = "", order_id: str = "", + campaign_id: str = "", + dispatch_id: str = "", + project_dir: str = "", + build_protected_campaign_ids: Callable[[Path], frozenset[str]] | None = None, session_id: str, pid: int, skill_command: str, @@ -88,28 +135,41 @@ def flush_session_log( elapsed_seconds: float | None = None, termination_reason: str = "", kill_reason: str = "", + provider_outcome: ProviderOutcome, + recipe_identity: RecipeIdentity, snapshot_interval_seconds: float = 0.0, step_name: str = "", - token_usage: dict[str, Any] | None = None, - timing_seconds: float | None = None, - audit_record: dict[str, Any] | None = None, cli_subtype: str = "", write_path_warnings: list[str] | None = None, write_call_count: int = 0, clone_contamination_reverted: bool = False, tracked_comm: str | None = None, exception_text: str = "", + orphaned_tool_result: bool = False, + raw_stdout: str = "", + last_stop_reason: str = "", + has_thinking_only_turn: bool = False, + versions: dict[str, Any] | None = None, + model_identifier: str = "", + max_sessions: int | None = None, + telemetry: SessionTelemetry, ) -> None: """Flush session diagnostics to disk. Writes proc_trace.jsonl, summary.json, anomalies.jsonl (if any), and appends to the global sessions.jsonl index. Applies retention - to keep at most 500 session directories. + to keep at most ``_MAX_SESSIONS`` session directories (default 2000, + configurable via ``linux_tracing.max_sessions``). - When step_name is provided along with telemetry kwargs, also writes - token_usage.json, step_timing.json, and (if audit_record) audit_log.json - to the session directory for recovery at next server startup. + When step_name is provided, also writes token_usage.json, step_timing.json, + and (if telemetry.audit_record is set) audit_log.json to the session directory + for recovery at next server startup. """ + token_usage = telemetry.token_usage + timing_seconds = telemetry.timing_seconds + audit_record = telemetry.audit_record + loc_insertions = telemetry.loc_insertions + loc_deletions = telemetry.loc_deletions effective_write_path_warnings: list[str] = ( write_path_warnings if write_path_warnings is not None else [] ) @@ -122,6 +182,30 @@ def flush_session_log( if cc_log and not cc_log.exists(): logger.warning("claude_code_log_not_found", path=cc_log_str, session_id=session_id) + silent_gap_seconds: float | None = None + if cc_log and cc_log.exists() and end_ts: + try: + cc_log_mtime = cc_log.stat().st_mtime + end_dt = datetime.fromisoformat(end_ts) + silent_gap_seconds = max(0.0, end_dt.timestamp() - cc_log_mtime) + except (OSError, ValueError): + pass + + _cb_request_ids: list[str] = [] + _cb_turn_timestamps: list[str] = [] + _cb_turn_tool_calls: list[tuple[str, ...]] = [] + if cc_log and cc_log.exists(): + try: + _text = cc_log.read_text(encoding="utf-8", errors="replace") + for _turn in iter_merged_assistant_turns(_text): + if not _turn.request_id: + continue + _cb_request_ids.append(_turn.request_id) + _cb_turn_timestamps.append(_turn.timestamp) + _cb_turn_tool_calls.append(_turn.tool_names) + except OSError: + logger.debug("channel_b_log_read_error", path=cc_log_str, exc_info=True) + session_dir = log_root / "sessions" / dir_name session_dir.mkdir(parents=True, exist_ok=True) @@ -142,11 +226,11 @@ def flush_session_log( record = { "ts": snap.get("captured_at") or start_ts, "seq": seq, - "event": "snapshot", "pid": pid, **snap, + "event": snap.get("event", "snapshot"), } - f.write(json.dumps(record, sort_keys=True) + "\n") + f.write(_fast_dumps(record, sort_keys=True) + "\n") # Track peaks rss = snap.get("vm_rss_kb", 0) @@ -186,12 +270,19 @@ def flush_session_log( drift_anomalies = detect_identity_drift(proc_snapshots, _effective_tracked_comm) anomalies.extend(drift_anomalies) + # Outcome anomaly detection (correlates session result with token usage) + if token_usage: + outcome_anomalies = detect_outcome_anomalies( + token_usage, subtype, has_thinking_only_turn=has_thinking_only_turn + ) + anomalies.extend(outcome_anomalies) + # Write anomalies.jsonl (only if anomalies exist) if anomalies: anomalies_path = session_dir / "anomalies.jsonl" with anomalies_path.open("w") as f: for a in anomalies: - f.write(json.dumps(a, sort_keys=True) + "\n") + f.write(_fast_dumps(a, sort_keys=True) + "\n") anomaly_count = len(anomalies) @@ -209,6 +300,14 @@ def flush_session_log( except ValueError: pass + # Write github_api_usage.json from pre-computed telemetry bundle + github_api_requests = telemetry.github_api_requests + if telemetry.github_api_usage is not None: + atomic_write( + session_dir / "github_api_usage.json", + _fast_dumps(telemetry.github_api_usage, sort_keys=True, indent=True) + "\n", + ) + # Write summary.json summary = { "session_id": session_id, @@ -224,6 +323,7 @@ def flush_session_log( "start_ts": start_ts, "end_ts": end_ts, "duration_seconds": duration_seconds, + "silent_gap_seconds": silent_gap_seconds, "snapshot_interval_seconds": snapshot_interval_seconds, "snapshot_count": snapshot_count, "anomaly_count": anomaly_count, @@ -232,6 +332,8 @@ def flush_session_log( "peak_fd_ratio": round(peak_fd_ratio, 3), "termination_reason": termination_reason, "kill_reason": kill_reason, + "provider_used": provider_outcome.provider_used, + "provider_fallback": provider_outcome.fallback_activated, "write_path_warnings": effective_write_path_warnings, "write_call_count": write_call_count, "clone_contamination_reverted": clone_contamination_reverted, @@ -239,32 +341,74 @@ def flush_session_log( "tracked_comm": _effective_tracked_comm, "tracked_comm_drift": _tracked_comm_drift, "tracer_target_resolution_version": 2, + "orphaned_tool_result": orphaned_tool_result, + "last_stop_reason": last_stop_reason, + "request_ids": _cb_request_ids, + "turn_timestamps": _cb_turn_timestamps, + "turn_tool_calls": _cb_turn_tool_calls, + "campaign_id": campaign_id, + "dispatch_id": dispatch_id, + "github_api_requests": github_api_requests, + "caller_session_id": caller_session_id, } + if versions is not None: + effective_model_id = model_identifier or _primary_model_identifier(token_usage) + summary["versions"] = { + **versions, + "model_identifier": effective_model_id, + } + if recipe_identity.name or recipe_identity.content_hash: + summary["recipe_provenance"] = { + "schema_version": 1, + "name": recipe_identity.name, + "version": recipe_identity.version, + "content_hash": recipe_identity.content_hash, + "composite_hash": recipe_identity.composite_hash, + } summary_path = session_dir / "summary.json" - atomic_write(summary_path, json.dumps(summary, sort_keys=True, indent=2) + "\n") + atomic_write(summary_path, _fast_dumps(summary, sort_keys=True, indent=True) + "\n") + + if campaign_id: + meta_path = session_dir / "meta.json" + atomic_write( + meta_path, + _fast_dumps({"campaign_id": campaign_id, "dispatch_id": dispatch_id}, sort_keys=True), + ) + + if not success and raw_stdout: + atomic_write(session_dir / "raw_stdout.jsonl", raw_stdout) if exception_text: atomic_write(session_dir / "crash_exception.txt", exception_text) - # Write per-session telemetry files when step_name is provided - if step_name and token_usage is not None: + # Write per-session telemetry files; gate on data presence, not session identity + label = _resolve_session_label(step_name, dispatch_id) + if token_usage is not None: tu_data = { - "step_name": step_name, + "session_label": label, "input_tokens": token_usage.get("input_tokens", 0), "output_tokens": token_usage.get("output_tokens", 0), "cache_creation_input_tokens": token_usage.get("cache_creation_input_tokens", 0), "cache_read_input_tokens": token_usage.get("cache_read_input_tokens", 0), "timing_seconds": timing_seconds if timing_seconds is not None else 0.0, "order_id": order_id, + "loc_insertions": loc_insertions, + "loc_deletions": loc_deletions, + "peak_context": token_usage.get("peak_context", 0), + "turn_count": token_usage.get("turn_count", 0), + "provider_used": provider_outcome.provider_used, + "model_identifier": model_identifier or _primary_model_identifier(token_usage), + "dispatch_id": dispatch_id, + "campaign_id": campaign_id, } - atomic_write(session_dir / "token_usage.json", json.dumps(tu_data)) + atomic_write(session_dir / "token_usage.json", _fast_dumps(tu_data)) - if step_name and timing_seconds is not None: + if timing_seconds is not None: atomic_write( session_dir / "step_timing.json", - json.dumps( + _fast_dumps( { - "step_name": step_name, + "step_name": label, "total_seconds": max(0.0, timing_seconds), "order_id": order_id, } @@ -272,7 +416,7 @@ def flush_session_log( ) if step_name and audit_record is not None: - atomic_write(session_dir / "audit_log.json", json.dumps([audit_record])) + atomic_write(session_dir / "audit_log.json", _fast_dumps([audit_record])) # Append to sessions.jsonl index index_entry = { @@ -282,6 +426,8 @@ def flush_session_log( "cwd": cwd, "kitchen_id": kitchen_id, "order_id": order_id, + "campaign_id": campaign_id, + "dispatch_id": dispatch_id, "claude_code_log": cc_log_str, "skill_command": skill_command[:100], "success": success, @@ -295,32 +441,97 @@ def flush_session_log( "step_name": step_name, "input_tokens": token_usage.get("input_tokens", 0) if token_usage else 0, "output_tokens": token_usage.get("output_tokens", 0) if token_usage else 0, + "cache_creation_input_tokens": token_usage.get("cache_creation_input_tokens", 0) + if token_usage + else 0, + "cache_read_input_tokens": token_usage.get("cache_read_input_tokens", 0) + if token_usage + else 0, "write_call_count": write_call_count, "tracked_comm": _effective_tracked_comm, "tracked_comm_drift": _tracked_comm_drift, + "autoskillit_version": versions.get("autoskillit_version", "") if versions else "", + "claude_code_version": versions.get("claude_code_version", "") if versions else "", + "recipe_name": recipe_identity.name, + "recipe_content_hash": recipe_identity.content_hash, + "recipe_composite_hash": recipe_identity.composite_hash, + "recipe_version": recipe_identity.version, + "duration_seconds": duration_seconds, + "github_api_requests": github_api_requests, + "provider_used": provider_outcome.provider_used, + "provider_fallback": provider_outcome.fallback_activated, + "caller_session_id": caller_session_id, } index_path = log_root / "sessions.jsonl" with index_path.open("a") as f: - f.write(json.dumps(index_entry, sort_keys=True) + "\n") + f.write(_fast_dumps(index_entry, sort_keys=True) + "\n") + + # Co-write session provenance record + if cwd: + from autoskillit.core import ProvenanceRecord, write_provenance_record + + write_provenance_record( + ProvenanceRecord( + session_id=session_id, + caller_session_id=caller_session_id, + kitchen_id=kitchen_id, + dispatch_id=dispatch_id, + recipe_name=recipe_identity.name, + step_name=step_name, + timestamp=start_ts or end_ts or "", + ), + project_dir=Path(cwd), + ) # Retention: keep at most _MAX_SESSIONS session directories - _enforce_retention(log_root) + _enforce_retention( + log_root, + project_dir=project_dir, + build_protected_campaign_ids=build_protected_campaign_ids, + max_sessions=max_sessions if max_sessions is not None else _MAX_SESSIONS, + ) + +def _enforce_retention( + log_root: Path, + project_dir: str | None = None, + build_protected_campaign_ids: Callable[[Path], frozenset[str]] | None = None, + *, + max_sessions: int = _MAX_SESSIONS, +) -> None: + """Delete oldest session directories if count exceeds *max_sessions*. -def _enforce_retention(log_root: Path) -> None: - """Delete oldest session directories if count exceeds _MAX_SESSIONS.""" + When ``project_dir`` is provided, reads fleet state files and ``meta.json`` + sidecars to skip deletion of sessions belonging to active campaigns. + """ sessions_dir = log_root / "sessions" if not sessions_dir.is_dir(): return dirs = sorted(sessions_dir.iterdir(), key=lambda p: p.stat().st_mtime) - if len(dirs) <= _MAX_SESSIONS: + if len(dirs) <= max_sessions: return - expired = dirs[: len(dirs) - _MAX_SESSIONS] - surviving_names = {d.name for d in dirs[len(dirs) - _MAX_SESSIONS :]} + expired = dirs[: len(dirs) - max_sessions] + surviving_names = {d.name for d in dirs[len(dirs) - max_sessions :]} + + protected_ids = ( + build_protected_campaign_ids(Path(project_dir)) + if project_dir and build_protected_campaign_ids is not None + else frozenset() + ) for d in expired: + if protected_ids: + meta_path = d / "meta.json" + if meta_path.is_file(): + try: + meta = json.loads(meta_path.read_text(encoding="utf-8")) + if meta.get("campaign_id") in protected_ids: + surviving_names.add(d.name) + continue + except (json.JSONDecodeError, OSError): + pass shutil.rmtree(d, ignore_errors=True) # Rewrite sessions.jsonl to remove expired entries @@ -436,6 +647,8 @@ def recover_crashed_sessions(tmpfs_path: str = "/dev/shm", log_dir: str = "") -> continue try: + from autoskillit.core import ProviderOutcome, RecipeIdentity, SessionTelemetry + flush_session_log( log_dir=log_dir, cwd="", @@ -448,9 +661,14 @@ def recover_crashed_sessions(tmpfs_path: str = "/dev/shm", log_dir: str = "") -> start_ts=mtime_ts, proc_snapshots=snapshots if snapshots else None, termination_reason="CRASHED", + provider_outcome=ProviderOutcome.none_used(), + recipe_identity=RecipeIdentity.empty(), + telemetry=SessionTelemetry.empty(), ) except Exception: - logger.debug("recover_crashed_sessions: failed to finalize %s", trace_file) + logger.debug( + "recover_crashed_sessions: failed to finalize %s", trace_file, exc_info=True + ) continue trace_file.unlink(missing_ok=True) diff --git a/src/autoskillit/execution/testing.py b/src/autoskillit/execution/testing.py index 61c319d07e..d62e614ed5 100644 --- a/src/autoskillit/execution/testing.py +++ b/src/autoskillit/execution/testing.py @@ -1,17 +1,22 @@ """Pytest output parsing and test pass/fail adjudication. -L3 service module. Used by server.py test_check tool and git_operations.py +IL-1 module. Used by server.py (IL-3) test_check tool and git_operations.py merge test gate. No autoskillit server imports — depends only on stdlib and -_logging. +IL-0 _logging. """ from __future__ import annotations +import asyncio +import json import os -import re +import tempfile +import time from pathlib import Path from typing import TYPE_CHECKING +import regex as re + from autoskillit.core import AUTOSKILLIT_PRIVATE_ENV_VARS, TestResult, get_logger if TYPE_CHECKING: @@ -32,6 +37,90 @@ def build_sanitized_env() -> dict[str, str]: return {k: v for k, v in os.environ.items() if k not in AUTOSKILLIT_PRIVATE_ENV_VARS} +def _read_sidecar_base_branch(cwd: Path) -> str | None: + """Read base-branch from worktree sidecar if cwd is a linked worktree. + + Worktree sidecars are written by implement-worktree skills at + ``/.autoskillit/temp/worktrees//base-branch``. + Returns None if cwd is not a worktree or no sidecar exists. + """ + git_path = cwd / ".git" + if not git_path.is_file(): + return None + try: + content = git_path.read_text().strip() + if not content.startswith("gitdir:"): + return None + gitdir = Path(content.split(":", 1)[1].strip()) + if not gitdir.is_absolute(): + gitdir = (cwd / gitdir).resolve() + main_git = gitdir.parent.parent + project_root = main_git.parent + wt_name = cwd.name + sidecar = project_root / ".autoskillit" / "temp" / "worktrees" / wt_name / "base-branch" + if sidecar.is_file(): + return sidecar.read_text().strip() or None + except (OSError, ValueError): + pass + return None + + +async def _resolve_base_ref( + config_base_ref: str | None, + cwd: Path, + *, + default_base_branch: str | None = None, +) -> str | None: + """Resolve the base ref for test filtering. + + Resolution chain (first non-None wins): + 1. Config override (explicit base_ref in TestCheckConfig) + 2. Worktree sidecar (base-branch file written by implement-worktree skills) + 3. Git upstream tracking ref (``@{upstream}`` of current branch) + 4. default_base_branch (from BranchingConfig) + 5. None (no base ref available) + """ + if config_base_ref is not None: + return config_base_ref + + sidecar_ref = _read_sidecar_base_branch(cwd) + if sidecar_ref: + return sidecar_ref + + try: + proc = await asyncio.create_subprocess_exec( + "git", + "rev-parse", + "--abbrev-ref", + "@{upstream}", + cwd=cwd, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.DEVNULL, + ) + try: + await asyncio.wait_for(proc.wait(), timeout=15.0) + except TimeoutError: + try: + proc.kill() + await proc.wait() + except OSError: + pass + return None + if proc.returncode == 0: + assert proc.stdout is not None + raw = await proc.stdout.read() + ref = raw.decode().strip() + if ref: + return ref + except OSError: + pass + + if default_base_branch is not None: + return default_base_branch + + return None + + _OUTCOME_PATTERN = re.compile( r"(\d+)\s+(passed|failed|errors?|xfailed|xpassed|skipped|warnings?|deselected)" ) @@ -113,9 +202,90 @@ def __init__(self, config: AutomationConfig, runner: SubprocessRunner) -> None: self._runner = runner async def run(self, cwd: Path) -> TestResult: - command = self._config.test_check.command + effective_commands = self._config.test_check.effective_commands timeout = float(self._config.test_check.timeout) env = build_sanitized_env() - result = await self._runner(command, cwd=cwd, timeout=timeout, env=env) - passed = check_test_passed(result.returncode, result.stdout, result.stderr) - return TestResult(passed=passed, stdout=result.stdout, stderr=result.stderr) + + filter_mode = self._config.test_check.filter_mode + if filter_mode: + env["AUTOSKILLIT_TEST_FILTER"] = filter_mode + + base_ref = await _resolve_base_ref( + self._config.test_check.base_ref, + cwd, + default_base_branch=self._config.branching.default_base_branch, + ) + if base_ref: + env["AUTOSKILLIT_TEST_BASE_REF"] = base_ref + + fd, sidecar_path = tempfile.mkstemp(suffix=".json", prefix="filter-stats-") + os.close(fd) + env["AUTOSKILLIT_FILTER_STATS_FILE"] = sidecar_path + + total = len(effective_commands) + stdout_parts: list[str] = [] + last_result = None + elapsed: float = 0.0 + stat_filter_mode: str | None = None + stat_tests_selected: int | None = None + stat_tests_deselected: int | None = None + stat_full_run_reason: str | None = None + start = time.monotonic() + deadline = start + timeout + + try: + for idx, command in enumerate(effective_commands, 1): + remaining = deadline - time.monotonic() + if remaining < 0.01: + break + result = await self._runner(command, cwd=cwd, timeout=remaining, env=env) + last_result = result + if total > 1: + stdout_parts.append( + f"=== [{idx}/{total}] {' '.join(command)} ===\n{result.stdout}" + ) + else: + stdout_parts.append(result.stdout) + if result.returncode != 0: + break + + elapsed = time.monotonic() - start + + sidecar = Path(sidecar_path) + if sidecar.is_file() and sidecar.stat().st_size > 0: + try: + raw = json.loads(sidecar.read_text()) + if isinstance(raw, dict): + fm = raw.get("filter_mode") + ts = raw.get("tests_selected") + td = raw.get("tests_deselected") + fr = raw.get("full_run_reason") + stat_filter_mode = fm if isinstance(fm, str) else None + stat_tests_selected = ts if isinstance(ts, int) else None + stat_tests_deselected = td if isinstance(td, int) else None + stat_full_run_reason = fr if isinstance(fr, str) else None + except (json.JSONDecodeError, OSError) as exc: + logger.debug("filter stats sidecar read error: %s", exc) + finally: + Path(sidecar_path).unlink(missing_ok=True) + + combined_stdout = "\n".join(stdout_parts) + if last_result is not None: + final_returncode = last_result.returncode + final_stderr = last_result.stderr + else: + final_returncode = 1 + final_stderr = "timeout exhausted before first command could run" + logger.warning("test_runner_timeout_before_first_command", timeout=timeout) + + passed = check_test_passed(final_returncode, combined_stdout, final_stderr) + return TestResult( + passed=passed, + stdout=combined_stdout, + stderr=final_stderr, + duration_seconds=elapsed, + filter_mode=stat_filter_mode, + tests_selected=stat_tests_selected, + tests_deselected=stat_tests_deselected, + full_run_reason=stat_full_run_reason, + ) diff --git a/src/autoskillit/fleet/CLAUDE.md b/src/autoskillit/fleet/CLAUDE.md new file mode 100644 index 0000000000..812e567b80 --- /dev/null +++ b/src/autoskillit/fleet/CLAUDE.md @@ -0,0 +1,31 @@ +# fleet/ + +IL-2 fleet campaign layer — parallel issue dispatch, semaphore, sidecar, liveness, state. + +## Files + +| File | Purpose | +|------|---------| +| `__init__.py` | Re-exports `CampaignSummary`, `parse_campaign_summary`, and dispatch callables | +| `_api.py` | Fleet campaign execution engine — dispatches L2 sessions, resolves campaign/result variable references | +| `_prompts.py` | Prompt builder for L2 fleet dispatch sessions — assembles admiral dispatch instruction block from SKILL.md sections | +| `result_parser.py` | L2 result block parser with Channel B JSONL fallback | +| `sidecar.py` | Per-issue JSONL sidecar — `IssueSidecarEntry`, append/read/`compute_remaining` helpers | +| `_liveness.py` | `is_dispatch_session_alive()` — boot_id + starttime_ticks liveness gate | +| `_semaphore.py` | `FleetSemaphore` — configurable `asyncio.BoundedSemaphore` implementing `FleetLock` | +| `_sidecar_rpc.py` | `run_python`-callable entry points: `write_sidecar_entry`, `get_remaining_issues` | +| `_findings_rpc.py` | `run_python`-callable entry points: `parse_and_resume`, `load_execution_map` | +| `_checkpoint_bridge.py` | `checkpoint_from_sidecar` — converts `IssueSidecarEntry` list to `SessionCheckpoint` | +| `state.py` | Campaign state I/O and mutations — `read_state`, `_write_state`, `mark_dispatch_*`, re-exports from `state_types`, `state_gates`, `state_recovery` | +| `state_types.py` | Campaign state types — `DispatchStatus`, `DispatchRecord`, `CampaignState`, `ResumeDecision`, `GateRecordResult`, constants | +| `state_gates.py` | Gate dispatch recording — `record_gate_outcome` | +| `state_recovery.py` | Crash recovery + resume — `has_failed_dispatch`, `crash_recover_dispatch`, `resume_campaign_from_state` | +| `summary.py` | Campaign summary schema v1: frozen dataclasses, sentinel parser, validator | + +## Architecture Notes + +`_api.py` is the primary entry point called by `server/tools/tools_execution.py:dispatch_food_truck`. +Sidecars are per-issue JSONL files appended atomically; `_sidecar_rpc.py` and +`_findings_rpc.py` expose sidecar operations to in-recipe `run_python` steps without +requiring a full server import. `_liveness.py` gates dispatch to prevent zombie sessions +from blocking campaign progress. diff --git a/src/autoskillit/fleet/__init__.py b/src/autoskillit/fleet/__init__.py new file mode 100644 index 0000000000..3ed3bd0f88 --- /dev/null +++ b/src/autoskillit/fleet/__init__.py @@ -0,0 +1,111 @@ +"""Fleet sub-package: campaign dispatch orchestration. + +Gateway exports per REQ-IMP-001 — consumers import from +``autoskillit.fleet``, not from sub-modules. +""" + +from ._api import _write_pid as _write_pid +from ._api import classify_dispatch_outcome, execute_dispatch +from ._checkpoint_bridge import checkpoint_from_sidecar +from ._liveness import is_dispatch_session_alive +from ._prompts import _build_admiral_dispatch_block as _build_admiral_dispatch_block +from ._prompts import _build_food_truck_prompt as _build_food_truck_prompt +from ._semaphore import FleetSemaphore +from .result_parser import L3ParseResult, parse_l3_result_block +from .sidecar import ( + IssueSidecarEntry, + append_sidecar_entry, + compute_remaining_issues, + read_sidecar, + read_sidecar_from_path, + sidecar_path, +) +from .state import ( + FLEET_HALTED_SENTINEL, + TERMINAL_DISPATCH_STATUSES, + CampaignState, + DispatchRecord, + DispatchStatus, + GateRecordResult, + ResumeDecision, + append_dispatch_record, + build_protected_campaign_ids, + crash_recover_dispatch, + has_failed_dispatch, + mark_dispatch_interrupted, + mark_dispatch_resumable, + mark_dispatch_running, + normalize_dispatch_token_usage, + read_all_campaign_captures, + read_state, + record_gate_outcome, + reset_failed_dispatch, + resume_campaign_from_state, + write_captured_values, + write_initial_state, +) +from .summary import ( + CampaignParseResult, + CampaignSummary, + CampaignSummaryStatus, + DispatchTokenUsage, + ParseFailure, + ParseFailureKind, + PerDispatchEntry, + SummaryErrorRecord, + parse_campaign_summary, + serialize_campaign_summary, + validate_campaign_summary, +) + +__all__ = [ + "_write_pid", + "classify_dispatch_outcome", + "execute_dispatch", + "_build_food_truck_prompt", + "_build_admiral_dispatch_block", + "L3ParseResult", + "parse_l3_result_block", + "CampaignParseResult", + "CampaignSummary", + "CampaignSummaryStatus", + "DispatchTokenUsage", + "ParseFailure", + "ParseFailureKind", + "PerDispatchEntry", + "SummaryErrorRecord", + "parse_campaign_summary", + "serialize_campaign_summary", + "validate_campaign_summary", + "TERMINAL_DISPATCH_STATUSES", + "FLEET_HALTED_SENTINEL", + "CampaignState", + "DispatchRecord", + "DispatchStatus", + "ResumeDecision", + "GateRecordResult", + "append_dispatch_record", + "crash_recover_dispatch", + "build_protected_campaign_ids", + "has_failed_dispatch", + "record_gate_outcome", + "mark_dispatch_interrupted", + "mark_dispatch_resumable", + "mark_dispatch_running", + "read_all_campaign_captures", + "read_state", + "reset_failed_dispatch", + "resume_campaign_from_state", + "write_captured_values", + "write_initial_state", + "normalize_dispatch_token_usage", + "checkpoint_from_sidecar", + "is_dispatch_session_alive", + "FleetSemaphore", + "IssueSidecarEntry", + "append_sidecar_entry", + "compute_remaining_issues", + "read_sidecar", + "read_sidecar_from_path", + "sidecar_path", +] diff --git a/src/autoskillit/fleet/_api.py b/src/autoskillit/fleet/_api.py new file mode 100644 index 0000000000..ac0973e097 --- /dev/null +++ b/src/autoskillit/fleet/_api.py @@ -0,0 +1,574 @@ +"""Fleet dispatch orchestration API.""" + +from __future__ import annotations + +import asyncio +import json +import time +from collections.abc import Callable +from pathlib import Path +from typing import TYPE_CHECKING, Any +from uuid import uuid4 + +import regex as re + +from autoskillit.core import ( + FleetErrorCode, + InfraExitCategory, + RetryReason, + SessionCheckpoint, # noqa: F401, TC001 + SkillResult, + claude_code_log_path, + fleet_error, + get_logger, +) +from autoskillit.fleet.result_parser import L3ParseResult, parse_l3_result_block +from autoskillit.fleet.state import DispatchStatus + +if TYPE_CHECKING: + from autoskillit.pipeline.context import ToolContext + +logger = get_logger(__name__) + +_CAMPAIGN_REF_RE = re.compile(r"\$\{\{\s*campaign\.(\w+)\s*\}\}") +_RESULT_REF_RE = re.compile(r"^\$\{\{\s*result\.([\w-]+)\s*\}\}$") + + +def _extract_captures( + capture_spec: dict[str, str], + payload: dict[str, object], +) -> dict[str, str]: + """Extract captured values from an L3 result payload. + + For each entry in `capture_spec` whose value matches ``${{ result.field }}``, + reads `payload[field]` and converts it to str. Missing payload keys are skipped. + """ + result: dict[str, str] = {} + for key, template in capture_spec.items(): + m = _RESULT_REF_RE.match(template.strip()) + if m is None: + continue + field_name = m.group(1) + if field_name in payload: + value = payload[field_name] + result[key] = value if isinstance(value, str) else json.dumps(value, default=str) + return result + + +def _interpolate_campaign_refs( + ingredients: dict[str, str], + captured: dict[str, str], +) -> dict[str, str]: + """Resolve ``${{ campaign.key }}`` references in ingredient values. + + Raises ValueError if a campaign reference cannot be resolved. + Non-campaign values are returned unchanged. + """ + out: dict[str, str] = {} + for k, v in ingredients.items(): + + def _replace(m: re.Match, _k: str = k) -> str: + ref = m.group(1) + if ref not in captured: + raise ValueError( + f"Ingredient '{_k}' references ${{{{ campaign.{ref} }}}} " + f"but '{ref}' has not been captured by any prior dispatch. " + f"Available: {sorted(captured)}" + ) + return captured[ref] + + out[k] = _CAMPAIGN_REF_RE.sub(_replace, v) + return out + + +def _write_pid( + state_path: Path, + dispatch_name: str, + dispatch_id: str, + pid: int, + starttime_ticks: int, + sidecar_path: str | None = None, +) -> None: + """on_spawn callback: atomically mark dispatch as running with dispatched_pid.""" + from autoskillit.core import read_boot_id + from autoskillit.fleet import mark_dispatch_running + + try: + mark_dispatch_running( + state_path, + dispatch_name, + dispatch_id=dispatch_id, + dispatched_pid=pid, + starttime_ticks=starttime_ticks, + boot_id=read_boot_id() or "", + sidecar_path=sidecar_path, + ) + except Exception: + logger.warning("_write_pid: failed to mark dispatch running", exc_info=True) + + +def _post_dispatch_cleanup( + tool_ctx: ToolContext, + skill_result: SkillResult, + cache_invalidator: Callable[[str], None] | None, + quota_refresher: Callable[..., Any], +) -> None: + """Run quota cache invalidation, background quota refresh, and session skill cleanup.""" + if cache_invalidator is not None: + cache_invalidator(tool_ctx.config.quota_guard.cache_path) + + if tool_ctx.background is not None: + tool_ctx.background.submit( + quota_refresher(tool_ctx.config.quota_guard), + label="quota_post_dispatch_refresh", + ) + + if tool_ctx.session_skill_manager is not None and skill_result.session_id: + try: + tool_ctx.session_skill_manager.cleanup_session(skill_result.session_id) + except Exception as exc: + logger.warning( + "session skills cleanup failed — dispatch not affected", + session_id=skill_result.session_id, + exc_class=type(exc).__name__, + exc_info=True, + ) + + +async def execute_dispatch( + tool_ctx: ToolContext, + recipe: str, + task: str, + ingredients: dict[str, str] | None, + dispatch_name: str | None, + timeout_sec: int | None, + prompt_builder: Callable[..., str], + quota_checker: Callable[..., Any], + quota_refresher: Callable[..., Any], + cache_invalidator: Callable[[str], None] | None = None, + capture: dict[str, str] | None = None, + resume_session_id: str | None = None, + resume_checkpoint: SessionCheckpoint | None = None, + idle_output_timeout: int | None = None, + caller_session_id: str = "", +) -> str: + """Execute a single food truck dispatch. + + Orchestrates: lock → validate → quota → prompt → dispatch → parse → state → cleanup. + Returns JSON envelope string. + """ + if ingredients is not None: + bad_vals = [k for k, v in ingredients.items() if not isinstance(v, str)] + if bad_vals: + return fleet_error( + FleetErrorCode.FLEET_UNKNOWN_INGREDIENT, + f"Ingredient values must be strings. Non-string keys: {bad_vals}", + ) + + lock = tool_ctx.fleet_lock + if lock is None: + return fleet_error( + FleetErrorCode.FLEET_LOCK_NOT_INITIALIZED, + "Fleet lock not initialized — open_kitchen with fleet mode.", + ) + if lock.at_capacity(): + return fleet_error( + FleetErrorCode.FLEET_PARALLEL_REFUSED, + f"Fleet at capacity ({lock.active_count}/{lock.max_concurrent} dispatches running).", + ) + + await lock.acquire() + try: + return await _run_dispatch( + tool_ctx=tool_ctx, + recipe=recipe, + task=task, + ingredients=ingredients, + dispatch_name=dispatch_name, + timeout_sec=timeout_sec, + prompt_builder=prompt_builder, + quota_checker=quota_checker, + quota_refresher=quota_refresher, + cache_invalidator=cache_invalidator, + capture=capture, + resume_session_id=resume_session_id, + resume_checkpoint=resume_checkpoint, + idle_output_timeout=idle_output_timeout, + caller_session_id=caller_session_id, + ) + except asyncio.CancelledError: + raise + except Exception as exc: + logger.error("execute_dispatch failed", exc_info=True) + return fleet_error( + FleetErrorCode.FLEET_L3_STARTUP_OR_CRASH, + f"{type(exc).__name__}: {exc}", + ) + finally: + lock.release() + + +_ABANDON_REASONS: frozenset[str] = frozenset( + { + RetryReason.STALE, + RetryReason.THINKING_STALL, + RetryReason.PATH_CONTAMINATION, + RetryReason.CLONE_CONTAMINATION, + } +) + + +def _is_abandon_reason(skill_result: SkillResult) -> bool: + """Return True when the kill reason indicates resume would be futile.""" + if skill_result.retry_reason in _ABANDON_REASONS: + return True + if ( + skill_result.retry_reason == RetryReason.RESUME + and skill_result.infra.exit_category == InfraExitCategory.CONTEXT_EXHAUSTED + ): + return True + return False + + +def classify_dispatch_outcome( + parsed: L3ParseResult, + skill_result: SkillResult, + *, + sidecar_exists: bool = False, + checkpoint: SessionCheckpoint | None = None, +) -> tuple[DispatchStatus, str]: + """Map L2 food truck subprocess signals to a (DispatchStatus, reason) pair. + + Pure function — no filesystem access, no side effects. + Rules applied in order: + 1. completed_clean + success flag → SUCCESS + 2. completed_clean + no success → FAILURE + 3. completed_dirty → FAILURE (fleet_l3_parse_failed) + 4. no_sentinel + session_id + lifespan_started + (checkpoint or sidecar) → RESUMABLE + 5. no_sentinel (any other case) → FAILURE (fleet_l3_no_result_block) + """ + if parsed.outcome == "completed_clean" and parsed.payload and parsed.payload.get("success"): + return DispatchStatus.SUCCESS, "" + if parsed.outcome == "completed_clean": + reason = parsed.payload.get("reason", "") if parsed.payload else "" + return DispatchStatus.FAILURE, reason + if parsed.outcome == "completed_dirty": + return DispatchStatus.FAILURE, FleetErrorCode.FLEET_L3_PARSE_FAILED + has_progress = checkpoint is not None or sidecar_exists + if skill_result.session_id and skill_result.lifespan_started and has_progress: + if _is_abandon_reason(skill_result): + return DispatchStatus.FAILURE, FleetErrorCode.FLEET_L3_NO_RESULT_BLOCK + return DispatchStatus.RESUMABLE, FleetErrorCode.FLEET_L3_NO_RESULT_BLOCK + return DispatchStatus.FAILURE, FleetErrorCode.FLEET_L3_NO_RESULT_BLOCK + + +async def _run_dispatch( + tool_ctx: ToolContext, + recipe: str, + task: str, + ingredients: dict[str, str] | None, + dispatch_name: str | None, + timeout_sec: int | None, + prompt_builder: Callable[..., str], + quota_checker: Callable[..., Any], + quota_refresher: Callable[..., Any], + cache_invalidator: Callable[[str], None] | None = None, + capture: dict[str, str] | None = None, + resume_session_id: str | None = None, + resume_checkpoint: SessionCheckpoint | None = None, + idle_output_timeout: int | None = None, + caller_session_id: str = "", +) -> str: + """Inner dispatch body — called after lock acquisition.""" + from autoskillit.fleet.state import ( + DispatchRecord, + DispatchStatus, + append_dispatch_record, + normalize_dispatch_token_usage, + write_initial_state, + ) + + if tool_ctx.recipes is None: + return fleet_error( + FleetErrorCode.FLEET_MANIFEST_MISSING, + "Recipe repository not configured.", + ) + + recipe_obj = tool_ctx.recipes.find(recipe, tool_ctx.project_dir) + if recipe_obj is None: + return fleet_error( + FleetErrorCode.FLEET_RECIPE_NOT_FOUND, + f"Recipe '{recipe}' not found.", + ) + + try: + full_recipe = tool_ctx.recipes.load(recipe_obj.path) + except Exception as exc: + logger.warning("load_recipe failed for '%s'", recipe, exc_info=True) + return fleet_error( + FleetErrorCode.FLEET_RECIPE_NOT_FOUND, + f"Recipe '{recipe}' could not be loaded: {exc}", + ) + + _DISPATCHABLE_KINDS = frozenset({"standard", "food-truck"}) + + if full_recipe.kind not in _DISPATCHABLE_KINDS: + return fleet_error( + FleetErrorCode.FLEET_INVALID_RECIPE_KIND, + f"Recipe '{recipe}' has kind '{full_recipe.kind}'. " + "Only standard and food-truck recipes can be dispatched.", + ) + + effective_ingredients = ingredients or {} + if "task" in full_recipe.ingredients and "task" not in effective_ingredients: + effective_ingredients = {"task": task, **effective_ingredients} + if effective_ingredients: + unknown = set(effective_ingredients.keys()) - set(full_recipe.ingredients.keys()) + if unknown: + return fleet_error( + FleetErrorCode.FLEET_UNKNOWN_INGREDIENT, + f"Unknown ingredient keys: {sorted(unknown)}. " + f"Valid keys: {sorted(full_recipe.ingredients.keys())}", + ) + + missing_required = [ + key + for key, ing in full_recipe.ingredients.items() + if getattr(ing, "required", False) + and getattr(ing, "default", None) is None + and key not in effective_ingredients + ] + if missing_required: + return fleet_error( + FleetErrorCode.FLEET_MISSING_INGREDIENT, + f"Missing required ingredients: {sorted(missing_required)}. " + f"These have no default and must be supplied.", + ) + + from autoskillit.fleet.state import read_all_campaign_captures # noqa: PLC0415 + + dispatches_dir = tool_ctx.temp_dir / "dispatches" + accumulated_captures = read_all_campaign_captures(dispatches_dir, tool_ctx.kitchen_id) + + _has_campaign_refs = any(_CAMPAIGN_REF_RE.search(v) for v in effective_ingredients.values()) + if _has_campaign_refs: + try: + effective_ingredients = _interpolate_campaign_refs( + effective_ingredients, accumulated_captures + ) + except ValueError as exc: + return fleet_error( + FleetErrorCode.FLEET_UNKNOWN_INGREDIENT, + str(exc), + ) + + quota_result = await quota_checker(tool_ctx.config.quota_guard) + if quota_result.get("should_sleep"): + await asyncio.sleep(quota_result.get("sleep_seconds", 0)) + + dispatch_id = str(uuid4()) + completion_marker = f"%%L3_DONE::{dispatch_id[:8]}%%" + from autoskillit.fleet.sidecar import sidecar_path as compute_sidecar_path # noqa: PLC0415 + + dispatch_sidecar_path = str(compute_sidecar_path(dispatch_id, tool_ctx.project_dir)) + effective_name = dispatch_name or recipe + + campaign_id = tool_ctx.kitchen_id + prompt = prompt_builder( + recipe=recipe, + task=task, + ingredients=effective_ingredients, + dispatch_id=dispatch_id, + campaign_id=campaign_id, + l3_timeout_sec=timeout_sec or 1800, + ) + + state_path = tool_ctx.temp_dir / "dispatches" / f"{dispatch_id}.json" + state_path.parent.mkdir(parents=True, exist_ok=True) + write_initial_state( + state_path, + campaign_id=campaign_id, + campaign_name=effective_name, + manifest_path="", + dispatches=[DispatchRecord(name=effective_name, caller_session_id=caller_session_id)], + ) + + if tool_ctx.executor is None: + return fleet_error( + FleetErrorCode.FLEET_MANIFEST_MISSING, + "Executor not configured.", + ) + + started_at = time.time() + _dispatched_pid: list[int] = [] + + def _on_spawn(pid: int, ticks: int) -> None: + _dispatched_pid.append(pid) + _write_pid(state_path, effective_name, dispatch_id, pid, ticks, dispatch_sidecar_path) + + skill_result = await tool_ctx.executor.dispatch_food_truck( + orchestrator_prompt=prompt, + cwd=str(tool_ctx.project_dir), + completion_marker=completion_marker, + resume_session_id=resume_session_id, + resume_checkpoint=resume_checkpoint, + kitchen_id=tool_ctx.kitchen_id, + order_id=dispatch_id, + campaign_id=campaign_id, + dispatch_id=dispatch_id, + caller_session_id=caller_session_id, + project_dir=str(tool_ctx.project_dir), + timeout=float(timeout_sec) if timeout_sec else None, + idle_output_timeout=float(idle_output_timeout) + if idle_output_timeout is not None + else None, + env_extras={ + "AUTOSKILLIT_PROJECT_DIR": str(tool_ctx.project_dir), + "AUTOSKILLIT_CAMPAIGN_ID": campaign_id, + "AUTOSKILLIT_DISPATCH_ID": dispatch_id, + "AUTOSKILLIT_SESSION_DEADLINE": str( + started_at + + ( + float(timeout_sec) + if timeout_sec is not None + else float(tool_ctx.config.fleet.default_timeout_sec) + ) + ), + }, + requires_packs=list(full_recipe.requires_packs) or ["kitchen-core"], + on_spawn=_on_spawn, + ) + ended_at = time.time() + + # --- Timeout pre-check: short-circuit before result-block parsing --- + if skill_result.subtype == "timeout": + append_dispatch_record( + state_path, + DispatchRecord( + name=effective_name, + status=DispatchStatus.FAILURE, + dispatch_id=dispatch_id, + campaign_id=campaign_id, + caller_session_id=caller_session_id, + dispatched_session_id=skill_result.session_id, + dispatched_pid=_dispatched_pid[0] if _dispatched_pid else 0, + reason=FleetErrorCode.FLEET_L3_TIMEOUT, + kill_reason=skill_result.retry_reason or "", + infra_exit_category=skill_result.infra.exit_category or "", + token_usage=normalize_dispatch_token_usage(skill_result.token_usage or {}), + started_at=started_at, + ended_at=ended_at, + ), + ) + _post_dispatch_cleanup(tool_ctx, skill_result, cache_invalidator, quota_refresher) + return fleet_error( + FleetErrorCode.FLEET_L3_TIMEOUT, + f"L3 dispatch '{effective_name}' timed out", + details={ + "dispatch_id": dispatch_id, + "dispatched_session_id": skill_result.session_id, + "lifespan_started": skill_result.lifespan_started, + "token_usage": skill_result.token_usage, + }, + ) + + jsonl_path = claude_code_log_path(str(tool_ctx.project_dir), skill_result.session_id or "") + parsed = parse_l3_result_block( + stdout=skill_result.result or "", + expected_dispatch_id=dispatch_id, + assistant_messages_path=jsonl_path, + ) + + sidecar_file = Path(dispatch_sidecar_path) + dispatch_checkpoint: SessionCheckpoint | None = None + if sidecar_file.exists(): + from autoskillit.fleet._checkpoint_bridge import checkpoint_from_sidecar # noqa: PLC0415 + from autoskillit.fleet.sidecar import read_sidecar_from_path # noqa: PLC0415 + + sidecar_entries = read_sidecar_from_path(sidecar_file) + if sidecar_entries: + dispatch_checkpoint = checkpoint_from_sidecar(sidecar_entries) + + final_status, reason = classify_dispatch_outcome( + parsed, + skill_result, + sidecar_exists=sidecar_file.exists(), + checkpoint=dispatch_checkpoint, + ) + + append_dispatch_record( + state_path, + DispatchRecord( + name=effective_name, + status=final_status, + dispatch_id=dispatch_id, + campaign_id=campaign_id, + caller_session_id=caller_session_id, + dispatched_session_id=skill_result.session_id, + dispatched_pid=_dispatched_pid[0] if _dispatched_pid else 0, + reason=reason, + kill_reason=skill_result.retry_reason or "", + infra_exit_category=skill_result.infra.exit_category or "", + token_usage=normalize_dispatch_token_usage(skill_result.token_usage or {}), + started_at=started_at, + ended_at=ended_at, + ), + ) + + if final_status == DispatchStatus.SUCCESS and capture and parsed.payload: + from autoskillit.fleet.state import write_captured_values # noqa: PLC0415 + + extracted = _extract_captures(capture, parsed.payload) + if extracted: + write_captured_values(state_path, extracted) + + _post_dispatch_cleanup(tool_ctx, skill_result, cache_invalidator, quota_refresher) + + if parsed.outcome == "completed_clean": + envelope_success = bool(parsed.payload and parsed.payload.get("success", False)) + return json.dumps( + { + "success": envelope_success, + "dispatch_status": final_status.value, + "dispatch_id": dispatch_id, + "dispatched_session_id": skill_result.session_id, + "l3_payload": parsed.payload, + "reason": reason, + "token_usage": skill_result.token_usage, + "l3_parse_source": parsed.source, + "lifespan_started": skill_result.lifespan_started, + } + ) + elif parsed.outcome == "completed_dirty": + return json.dumps( + { + "success": False, + "dispatch_status": final_status.value, + "dispatch_id": dispatch_id, + "dispatched_session_id": skill_result.session_id, + "l3_payload": None, + "reason": FleetErrorCode.FLEET_L3_PARSE_FAILED, + "l3_raw_body": parsed.raw_body, + "l3_parse_error": parsed.parse_error, + "token_usage": skill_result.token_usage, + "l3_parse_source": parsed.source, + "lifespan_started": skill_result.lifespan_started, + } + ) + else: + return json.dumps( + { + "success": False, + "dispatch_status": final_status.value, + "dispatch_id": dispatch_id, + "dispatched_session_id": skill_result.session_id, + "l3_payload": None, + "reason": FleetErrorCode.FLEET_L3_NO_RESULT_BLOCK, + "l3_parse_source": parsed.source, + "token_usage": skill_result.token_usage, + "lifespan_started": skill_result.lifespan_started, + "resume_checkpoint": dispatch_checkpoint.to_dict() + if dispatch_checkpoint + else None, + } + ) diff --git a/src/autoskillit/fleet/_checkpoint_bridge.py b/src/autoskillit/fleet/_checkpoint_bridge.py new file mode 100644 index 0000000000..c861e8fd18 --- /dev/null +++ b/src/autoskillit/fleet/_checkpoint_bridge.py @@ -0,0 +1,17 @@ +"""Bridge from fleet IssueSidecarEntry to core SessionCheckpoint.""" + +from __future__ import annotations + +from autoskillit.core import SessionCheckpoint +from autoskillit.fleet.sidecar import IssueSidecarEntry + + +def checkpoint_from_sidecar(entries: list[IssueSidecarEntry]) -> SessionCheckpoint: + completed = [e.issue_url for e in entries if e.status == "completed"] + ts = entries[-1].ts if entries else "" + return SessionCheckpoint( + completed_items=completed, + step_name="fleet_dispatch", + progress_pct=0.0, + ts=ts, + ) diff --git a/src/autoskillit/fleet/_findings_rpc.py b/src/autoskillit/fleet/_findings_rpc.py new file mode 100644 index 0000000000..f2f6180305 --- /dev/null +++ b/src/autoskillit/fleet/_findings_rpc.py @@ -0,0 +1,65 @@ +from __future__ import annotations + +import json +import os +from pathlib import Path + +from autoskillit.core import get_logger +from autoskillit.fleet.sidecar import compute_remaining_issues + +logger = get_logger(__name__) + + +def parse_and_resume( + issue_urls_csv: str, + project_dir: str = "", + dispatch_id: str = "", +) -> dict[str, str]: + """Parse CSV issue URLs and filter out already-completed issues via sidecar. + + Callable via run_python. Falls back to AUTOSKILLIT_DISPATCH_ID from env if + dispatch_id is not provided. + Returns {"remaining_urls_json": "[...]", "completed_count": "N"} on success + or {"ok": "false", "error": "..."} on failure. + """ + if not dispatch_id: + dispatch_id = os.environ.get("AUTOSKILLIT_DISPATCH_ID", "") + if not dispatch_id: + return {"ok": "false", "error": "AUTOSKILLIT_DISPATCH_ID env var not set"} + urls = [u.strip() for u in issue_urls_csv.split(",") if u.strip()] + root = Path(project_dir) if project_dir else Path.cwd() + try: + remaining = compute_remaining_issues(dispatch_id, urls, root) + except Exception as exc: + logger.warning("parse_and_resume: compute_remaining_issues failed", exc_info=True) + return {"ok": "false", "error": str(exc)} + completed = len(urls) - len(remaining) + return { + "remaining_urls_json": json.dumps(remaining), + "completed_count": str(completed), + } + + +def load_execution_map(map_path: str) -> dict[str, str]: + """Read a BEM JSON file and extract the groups array. + + Callable via run_python. + Returns {"groups_json": "[{...}]", "total_groups": "N"} on success + or {"error": "..."} on failure. + """ + path = Path(map_path) + if not path.exists(): + return {"error": f"BEM file not found: {map_path}"} + try: + data = json.loads(path.read_text()) + except (json.JSONDecodeError, OSError) as exc: + return {"error": f"failed to read BEM file: {exc}"} + if not isinstance(data, dict) or "groups" not in data: + return {"error": "BEM file missing top-level 'groups' key"} + groups = data["groups"] + if not isinstance(groups, list): + return {"error": f"BEM 'groups' must be a list, got {type(groups).__name__}"} + return { + "groups_json": json.dumps(groups), + "total_groups": str(len(groups)), + } diff --git a/src/autoskillit/fleet/_liveness.py b/src/autoskillit/fleet/_liveness.py new file mode 100644 index 0000000000..279527c553 --- /dev/null +++ b/src/autoskillit/fleet/_liveness.py @@ -0,0 +1,11 @@ +from __future__ import annotations + +from autoskillit.core import is_session_alive +from autoskillit.fleet.state import DispatchRecord + + +def is_dispatch_session_alive(record: DispatchRecord) -> bool: + """True only when boot_id, PID, and starttime_ticks all match — False on non-Linux.""" + return is_session_alive( + record.dispatched_pid, record.dispatched_boot_id, record.dispatched_starttime_ticks + ) diff --git a/src/autoskillit/fleet/_prompts.py b/src/autoskillit/fleet/_prompts.py new file mode 100644 index 0000000000..6a3e495408 --- /dev/null +++ b/src/autoskillit/fleet/_prompts.py @@ -0,0 +1,279 @@ +"""Food truck prompt builder for L2 food truck sessions. + +Moved from autoskillit.cli._prompts — this module +depends only on autoskillit.core and stdlib, making it importable from both +the server and CLI layers without introducing cross-L3 coupling. +""" + +from __future__ import annotations + +import json + +import regex as re + +from autoskillit.core import ADMIRAL_DISPATCH_SECTIONS, get_logger, pkg_root +from autoskillit.hooks import ( + QUOTA_BUDGET_EXCEEDED_TRIGGER, + QUOTA_GUARD_DENY_TRIGGER, + QUOTA_POST_WARNING_TRIGGER, +) + +logger = get_logger(__name__) + + +def _build_admiral_dispatch_block() -> str: + """Extract the dispatch-relevant subset of sous-chef SKILL.md. + + Uses regex to split on ``## `` section headers and retains only sections + whose title starts with one of the ADMIRAL_DISPATCH_SECTIONS prefixes. + Returns empty string if SKILL.md is absent (graceful degradation). + """ + path = pkg_root() / "skills" / "sous-chef" / "SKILL.md" + if not path.exists(): + return "" + try: + content = path.read_text() + except OSError: + return "" + + # Split into sections on ## boundaries (keeping the delimiter via lookahead) + sections = re.split(r"(?=^## )", content, flags=re.MULTILINE) + + retained: list[str] = [] + for section in sections: + for title in ADMIRAL_DISPATCH_SECTIONS: + if section.startswith(f"## {title}"): + retained.append(section.rstrip()) + break + + return "\n\n".join(retained) + + +def _build_food_truck_prompt( + recipe: str, + task: str, + ingredients: dict[str, str], + mcp_prefix: str, + dispatch_id: str, + campaign_id: str, + l3_timeout_sec: int, +) -> str: + """Build the system prompt for an L2 food truck headless session. + + The prompt is self-contained — the L2 food truck session needs no runtime reference + material beyond what is embedded here. It assembles 8 sections: + filtered sous-chef discipline, headless directives, routing/predicates, + budget guidance, quota awareness, campaign task, ingredient values, + and a sentinel-anchored result contract. + """ + dispatch_id_short = dispatch_id[:8] + ingredients_json = json.dumps(ingredients) + ingredients_pretty_json = json.dumps(ingredients, indent=2) + + admiral_block = _build_admiral_dispatch_block() + + return f"""\ +You are an L2 food truck orchestrator. Execute the recipe '{recipe}' autonomously. +Timeout: {l3_timeout_sec}s. Campaign: {campaign_id}. Dispatch: {dispatch_id}. + +--- SECTION 1: ADMIRAL DISCIPLINE (L2/FOOD-TRUCK SUBSET) --- + +{admiral_block} + +--- SECTION 2: HEADLESS OPERATING MODE --- + +H1 — FIRST ACTION (open_kitchen with overrides): + Call {mcp_prefix}open_kitchen(name='{recipe}', overrides={ingredients_json}) + to activate pipeline tools. Ingredient values are pre-applied via overrides. + DO NOT call Bash, ToolSearch, or any other tool before open_kitchen. + +H2 — NO INGREDIENT PROMPTING: + All ingredient values have been applied via open_kitchen overrides. + DO NOT prompt for ingredient values. DO NOT call AskUserQuestion to collect inputs. + Proceed directly to pipeline execution after open_kitchen succeeds. + +H3 — AUTO-ACCEPT CONFIRM STEPS: + When you reach a step with action: "confirm", treat it as automatically confirmed. + Route to the step's on_success target without calling AskUserQuestion. + Headless sessions have no human operator to prompt. + +H3b — STOP STEP SEMANTICS: + When you reach a step with action: "stop", the pipeline is TERMINATED. + Display the step's message in the sentinel block. Do NOT call any MCP tools. + Do NOT attempt recovery, error reporting, or off-recipe actions after a stop step. + A stop step is an INTENTIONAL terminus — the recipe author designed this as the endpoint. + +H3c — ROUTE STEP SEMANTICS: + When you reach a step with action: "route", evaluate the step's on_result + conditions against captured context variables. Route to the matching target. + Do NOT call any MCP tools for this step type — routing evaluation IS the step. + If no on_result condition matches and on_failure is defined, follow on_failure. + +H4 — AskUserQuestion PERMITTED (AUTO-ACCEPTS): + AskUserQuestion calls are permitted in this session. In the headless environment, + they auto-accept with the first option. Use them only when a recipe step requires + explicit user interaction routing — the auto-accept behavior handles it. + +H5 — SHELL ACCESS VIA run_cmd ONLY: + All shell commands must go through run_cmd. Do NOT use Bash directly. + run_cmd provides audit logging, timeout enforcement, and quota integration. + +H6 — SENTINEL-ONLY OUTPUT: + Do NOT produce conversational prose, status summaries, or progress narration. + Your ONLY output is the final sentinel block (see SECTION 8). + Every non-final turn MUST invoke at least one tool. + +--- SECTION 3: CORE ROUTING RULES + FAILURE PREDICATES --- + +ROUTING RULES — MANDATORY: +- When a tool returns a failure result, you MUST follow the step's on_failure route. +- When a step fails, route to on_failure — the downstream skill has diagnostic + access that the orchestrator does not. +- Your ONLY job is to route to the correct next step and pass the + required arguments. The downstream skill does the actual work. + +FAILURE PREDICATES — when to follow on_failure: +- test_check: "passed: False" in output +- merge_worktree: "error:" line present in output +- run_cmd: "success: False" in output +- run_skill: "success: False" in output +- classify_fix: "error:" line present in output + +FAILURE PREDICATE — open_kitchen: + If the open_kitchen response contains `"success": false` OR does not + contain the substring `--- INGREDIENTS TABLE ---`: + 1. Emit the sentinel block with success=false and reason="open_kitchen_failed". + 2. End the session. + +TWO FAILURE TIERS FOR PREDICATE-FORMAT STEPS: +- Tool-level failure (run_skill returns "success: False"): Follow on_failure. This fires + BEFORE any result object exists. on_result conditions are NOT evaluated. +- Skill-level error ("error:" line present in result): Follow the matching on_result + condition. This fires only when run_skill completes and returns a result with an error line. +- When a step has no on_failure declared and the tool returns "success: False", this is a + recipe authoring error. Emit the sentinel block with success=false + and reason="missing_on_failure". + +OPTIONAL STEP SEMANTICS: +- optional: true means the step is SKIPPED (treated as bypassed) when its + skip_when_false ingredient is false. It does NOT mean failures are tolerated. +- When skip_when_false evaluates to true (or is absent), the step is MANDATORY + and MUST execute. The ONLY reason to skip an optional step is skip_when_false + being false — no other reason is valid. +- A running optional step that returns success: false MUST follow on_failure. + Never route a running optional step's failure to done. + +STEP EXECUTION IS NOT DISCRETIONARY: +- You MUST execute every step the pipeline routes you to. +- NEVER skip a step because the PR is small, the diff is trivial, the change + looks simple, or you judge the step unnecessary. +- The ONLY mechanism for skipping a step is skip_when_false evaluating to false. + When skip_when_false evaluates to true (or is absent), the step is MANDATORY. + +HOOK DENIAL COMPLIANCE — ALL HOOKS: +- When a PreToolUse hook DENIES a tool call (permissionDecision: "deny"), the denial + is a MANDATORY directive, not a suggestion. You MUST comply immediately. +- Read the permissionDecisionReason carefully — it contains the required corrective action. +- NEVER retry the denied tool call without first completing the corrective action. +- NEVER ignore, work around, or reason past a hook denial. +- Hook denials are structural enforcement of recipe/pipeline contracts. +- After completing the corrective action specified in the deny reason, you may retry + the original tool call. + +SPECIFIC HOOK DENIAL PATTERNS: +- "QUOTA WAIT REQUIRED": Temporary — sleep and retry (see QUOTA DENIAL ROUTING below). +- "REVIEW LOOP REQUIRED": Call check_review_loop before retrying wait_for_ci/enqueue_pr. +- All other denials: Follow the corrective instruction in the deny reason text. + +QUOTA DENIAL ROUTING — run_skill only (check BEFORE on_failure): +- When a PreToolUse hook DENIES run_skill with "{QUOTA_GUARD_DENY_TRIGGER}": + - This is a TEMPORARY block. The API quota resets on a rolling window. + - The deny message contains a run_cmd sleep command. Execute it immediately. + - After the sleep completes, retry the EXACT same run_skill call (same arguments). + - NEVER treat a quota denial as a permanent failure or pipeline-stopping error. +- When a PreToolUse hook DENIES run_skill with "{QUOTA_BUDGET_EXCEEDED_TRIGGER}": + - The required quota sleep EXCEEDS the session's remaining wall-clock budget. + - Do NOT sleep. Instead, emit the sentinel block immediately with the fields + specified in the deny message (success=false, reason=fleet_quota_exhausted, + wait_seconds, summary). + - Then STOP — do not call any more tools after emitting the sentinel. +- When run_skill output contains "{QUOTA_POST_WARNING_TRIGGER}": + - A post-execution quota check detected high utilization. + - The warning contains a run_cmd sleep command. Execute it BEFORE the next run_skill call. + - After sleeping, proceed normally with the next pipeline step. + +SKILL_COMMAND FORMATTING — MANDATORY: +- The `skill_command` value in each step's `with:` block is a LITERAL template. + Substitute ${{{{ context.var_name }}}} and ${{{{ inputs.var_name }}}} placeholders with + their resolved values and pass the resulting string VERBATIM to run_skill. +- Do NOT add markdown headers, labels, notes, or any prose to skill_command. +- skill_command arguments are POSITIONAL SPACE-SEPARATED TOKENS. + +--- SECTION 4: PER-SKILL BUDGET GUIDANCE --- + +Skill invocations have time budgets based on complexity: +- Simple skills (single tool, no subagent): 120 seconds +- Single-subagent skills (one headless session): 900 seconds +- Multi-subagent skills (parallel or sequential sessions): 1800 seconds + +When a skill approaches its budget, prefer completing partial work and +emitting a result over running to timeout. Budget overruns degrade +campaign throughput. + +--- SECTION 5: QUOTA AWARENESS --- + +When you detect that the API quota is exhausted (via quota guard denial +or quota post-warning), and you cannot make further progress: + +Emit the sentinel block with: + "success": false, + "reason": "quota_exhausted", + "wait_seconds": + +The fleet dispatcher will schedule a retry after the wait period. +Do NOT loop indefinitely on quota denials — if 3 consecutive quota +denials occur with no successful run_skill between them, emit the +quota_exhausted sentinel and exit. + +--- SECTION 6: CAMPAIGN TASK --- + +Recipe: {recipe} +Task: {task} +Campaign ID: {campaign_id} +Dispatch ID: {dispatch_id} +Timeout: {l3_timeout_sec} seconds + +Execute the recipe pipeline for the task above. Follow all routing +rules and failure predicates. Emit the sentinel block upon completion. + +--- SECTION 7: INGREDIENT VALUES --- + +The following ingredient values have been applied via open_kitchen overrides. +They are provided here for reference only — do NOT re-apply or re-prompt. + +```json +{ingredients_pretty_json} +``` + +--- SECTION 8: FINAL OUTPUT CONTRACT --- + +When the pipeline completes (success or failure), emit this EXACT sentinel block +as your final output. No other text after the sentinel. + +``` +---l3-result::{dispatch_id}--- +{{"success": , "reason": "", "summary": ""}} +---end-l3-result::{dispatch_id}--- +%%L3_DONE::{dispatch_id_short}%% +``` + +Fields: +- success: true if all mandatory steps completed without unresolved failures +- reason: "completed", "failed", "quota_exhausted", "timeout", + "open_kitchen_failed", "missing_on_failure" +- summary: One-line description of what happened + +The sentinel markers ---l3-result::{dispatch_id}--- and ---end-l3-result::{dispatch_id}--- +are parsed by the fleet dispatcher. The %%L3_DONE::{dispatch_id_short}%% marker +signals session completion to the process monitor. +""" diff --git a/src/autoskillit/fleet/_semaphore.py b/src/autoskillit/fleet/_semaphore.py new file mode 100644 index 0000000000..3bf5d39e7a --- /dev/null +++ b/src/autoskillit/fleet/_semaphore.py @@ -0,0 +1,35 @@ +"""FleetSemaphore: configurable semaphore implementing the FleetLock protocol.""" + +from __future__ import annotations + +import asyncio + + +class FleetSemaphore: + """Configurable semaphore implementing FleetLock for fleet dispatch concurrency.""" + + def __init__(self, max_concurrent: int = 1) -> None: + if max_concurrent <= 0: + raise ValueError(f"max_concurrent must be > 0, got {max_concurrent}") + self._semaphore = asyncio.BoundedSemaphore(max_concurrent) + self._active = 0 + self._max = max_concurrent + + def at_capacity(self) -> bool: + return self._active >= self._max + + async def acquire(self) -> None: + await self._semaphore.acquire() + self._active += 1 + + def release(self) -> None: + self._semaphore.release() + self._active -= 1 + + @property + def active_count(self) -> int: + return self._active + + @property + def max_concurrent(self) -> int: + return self._max diff --git a/src/autoskillit/fleet/_sidecar_rpc.py b/src/autoskillit/fleet/_sidecar_rpc.py new file mode 100644 index 0000000000..914144da65 --- /dev/null +++ b/src/autoskillit/fleet/_sidecar_rpc.py @@ -0,0 +1,56 @@ +from __future__ import annotations + +import json +from datetime import UTC, datetime +from pathlib import Path +from typing import Literal, cast + +from autoskillit.fleet.sidecar import ( + IssueSidecarEntry, + append_sidecar_entry, + compute_remaining_issues, +) + + +def write_sidecar_entry( + dispatch_id: str, + issue_url: str, + status: str, + pr_url: str = "", + reason: str = "", + project_dir: str = "", +) -> dict[str, str]: + """Append one completion entry; callable via run_python. Returns {"ok": "true"} on success.""" + if status not in ("completed", "failed"): + return {"ok": "false", "error": f"invalid status: {status!r}"} + entry = IssueSidecarEntry( + issue_url=issue_url, + status=cast(Literal["completed", "failed"], status), + ts=datetime.now(tz=UTC).isoformat(), + pr_url=pr_url or None, + reason=reason or None, + ) + root = Path(project_dir) if project_dir else Path.cwd() + try: + append_sidecar_entry(dispatch_id, entry, root) + except OSError as exc: + return {"ok": "false", "error": str(exc)} + return {"ok": "true"} + + +def get_remaining_issues( + dispatch_id: str, + original_urls_json: str, + project_dir: str = "", +) -> dict[str, str]: + """Return remaining URLs as remaining_urls_json + remaining_count dict.""" + try: + original_urls: list[str] = json.loads(original_urls_json) + except (json.JSONDecodeError, TypeError) as exc: + return {"ok": "false", "error": f"invalid original_urls_json: {exc}"} + root = Path(project_dir) if project_dir else Path.cwd() + remaining = compute_remaining_issues(dispatch_id, original_urls, root) + return { + "remaining_urls_json": json.dumps(remaining), + "remaining_count": str(len(remaining)), + } diff --git a/src/autoskillit/fleet/result_parser.py b/src/autoskillit/fleet/result_parser.py new file mode 100644 index 0000000000..cf917b570b --- /dev/null +++ b/src/autoskillit/fleet/result_parser.py @@ -0,0 +1,186 @@ +"""L3 result block parser with Channel B JSONL fallback.""" + +from __future__ import annotations + +import json +from dataclasses import dataclass +from pathlib import Path +from typing import Literal + +import regex as re + +from autoskillit.core import ClaudeContentBlockType, get_logger + +logger = get_logger(__name__) + +_ANSI_RE = re.compile(r"\x1b\[[0-9;]*m") + + +@dataclass(frozen=True, slots=True) +class L3ParseResult: + """Tri-state outcome of parsing an L3 result block.""" + + outcome: Literal["completed_clean", "completed_dirty", "no_sentinel"] + payload: dict | None + raw_body: str | None + parse_error: str | None + source: Literal["stdout", "assistant_messages_jsonl"] + + +def _extract_text_from_jsonl(path: Path) -> str: + """Read a Claude Code session JSONL and extract assistant text blocks. + + Reads all lines, filters for type=="assistant" records, extracts text + from message.content blocks. Returns concatenated text (oldest-first). + """ + try: + raw = path.read_text(encoding="utf-8") + except (FileNotFoundError, OSError): + return "" + + texts: list[str] = [] + for line in raw.splitlines(): + line = line.strip() + if not line: + continue + try: + obj = json.loads(line) + except json.JSONDecodeError: + continue + if not isinstance(obj, dict): + continue + if obj.get("type") != "assistant": + continue + msg = obj.get("message") + if not isinstance(msg, dict): + continue + content = msg.get("content", "") + if isinstance(content, list): + text = "\n".join( + block.get("text", "") + for block in content + if isinstance(block, dict) + and ClaudeContentBlockType.from_api(block.get("type", "")) + == ClaudeContentBlockType.TEXT + ).strip() + else: + text = str(content).strip() + if text: + texts.append(text) + + return "\n\n".join(texts) + + +def _scan_for_sentinel( + text: str, + open_sentinel: str, + close_sentinel: str, +) -> tuple[int, int] | None: + """Find the last valid (open, close) sentinel pair in text. + + Returns (open_pos, close_pos) indices or None if not found / out-of-order. + """ + open_pos = text.rfind(open_sentinel) + if open_pos == -1: + return None + search_from = open_pos + len(open_sentinel) + close_pos = text.find(close_sentinel, search_from) + if close_pos == -1: + return None + return (open_pos, close_pos) + + +def _parse_body( + text: str, + open_pos: int, + close_pos: int, + open_sentinel: str, + source: Literal["stdout", "assistant_messages_jsonl"], +) -> L3ParseResult: + """Extract body between sentinels and attempt JSON decode.""" + after_open = open_pos + len(open_sentinel) + body = text[after_open:close_pos].strip() + + if not body: + return L3ParseResult( + outcome="completed_dirty", + payload=None, + raw_body="", + parse_error="empty body between sentinels", + source=source, + ) + + try: + parsed = json.loads(body) + except json.JSONDecodeError as exc: + return L3ParseResult( + outcome="completed_dirty", + payload=None, + raw_body=body, + parse_error=str(exc), + source=source, + ) + + if not isinstance(parsed, dict): + return L3ParseResult( + outcome="completed_dirty", + payload=None, + raw_body=body, + parse_error=f"expected JSON object, got {type(parsed).__name__}", + source=source, + ) + + return L3ParseResult( + outcome="completed_clean", + payload=parsed, + raw_body=None, + parse_error=None, + source=source, + ) + + +def parse_l3_result_block( + stdout: str, + expected_dispatch_id: str, + assistant_messages_path: Path | None = None, +) -> L3ParseResult: + """Parse an L3 result block from food truck dispatch output. + + Strips ANSI codes, scans stdout for the last occurrence of the sentinel + block keyed to expected_dispatch_id, and returns a tri-state outcome. + Falls back to reading the Channel B JSONL file when stdout is truncated. + """ + open_sentinel = f"---l3-result::{expected_dispatch_id}---" + close_sentinel = f"---end-l3-result::{expected_dispatch_id}---" + + cleaned = _ANSI_RE.sub("", stdout) + + positions = _scan_for_sentinel(cleaned, open_sentinel, close_sentinel) + if positions is not None: + open_pos, close_pos = positions + return _parse_body(cleaned, open_pos, close_pos, open_sentinel, "stdout") + + if assistant_messages_path is None: + return L3ParseResult( + outcome="no_sentinel", + payload=None, + raw_body=None, + parse_error=None, + source="stdout", + ) + + jsonl_text = _extract_text_from_jsonl(assistant_messages_path) + positions = _scan_for_sentinel(jsonl_text, open_sentinel, close_sentinel) + if positions is not None: + open_pos, close_pos = positions + return _parse_body( + jsonl_text, open_pos, close_pos, open_sentinel, "assistant_messages_jsonl" + ) + + return L3ParseResult( + outcome="no_sentinel", + payload=None, + raw_body=None, + parse_error=None, + source="stdout", + ) diff --git a/src/autoskillit/fleet/sidecar.py b/src/autoskillit/fleet/sidecar.py new file mode 100644 index 0000000000..9e3cb29321 --- /dev/null +++ b/src/autoskillit/fleet/sidecar.py @@ -0,0 +1,92 @@ +from __future__ import annotations + +import json +from dataclasses import asdict, dataclass +from pathlib import Path +from typing import Literal + +from autoskillit.core import ensure_project_temp, get_logger + +logger = get_logger() + + +@dataclass(frozen=True, slots=True) +class IssueSidecarEntry: + issue_url: str + status: Literal["completed", "failed"] + ts: str + pr_url: str | None = None + reason: str | None = None + + +def sidecar_path(dispatch_id: str, project_dir: Path) -> Path: + return ensure_project_temp(project_dir) / "dispatches" / f"{dispatch_id}_issues.jsonl" + + +def append_sidecar_entry(dispatch_id: str, entry: IssueSidecarEntry, project_dir: Path) -> None: + path = sidecar_path(dispatch_id, project_dir) + path.parent.mkdir(parents=True, exist_ok=True) + payload = {k: v for k, v in asdict(entry).items() if v is not None} + with path.open("a") as fh: + fh.write(json.dumps(payload) + "\n") + + +def read_sidecar(dispatch_id: str, project_dir: Path) -> list[IssueSidecarEntry]: + path = sidecar_path(dispatch_id, project_dir) + if not path.exists(): + return [] + entries: list[IssueSidecarEntry] = [] + for line in path.read_text().splitlines(): + line = line.strip() + if not line: + continue + try: + data = json.loads(line) + entries.append( + IssueSidecarEntry( + issue_url=data["issue_url"], + status=data["status"], + ts=data.get("ts", ""), + pr_url=data.get("pr_url"), + reason=data.get("reason"), + ) + ) + except (json.JSONDecodeError, KeyError) as exc: + logger.debug("sidecar: skipping corrupt JSONL line", path=str(path), error=str(exc)) + continue + return entries + + +def read_sidecar_from_path(path: Path) -> list[IssueSidecarEntry]: + """Read and parse a sidecar JSONL at path; skips corrupt lines; returns [] on OSError.""" + entries: list[IssueSidecarEntry] = [] + try: + lines = path.read_text().splitlines() + except OSError: + return [] + for line in lines: + line = line.strip() + if not line: + continue + try: + data = json.loads(line) + entries.append( + IssueSidecarEntry( + issue_url=data["issue_url"], + status=data["status"], + ts=data.get("ts", ""), + pr_url=data.get("pr_url"), + reason=data.get("reason"), + ) + ) + except (json.JSONDecodeError, KeyError) as exc: + logger.debug("sidecar: skipping corrupt JSONL line", path=str(path), error=str(exc)) + continue + return entries + + +def compute_remaining_issues( + dispatch_id: str, original_urls: list[str], project_dir: Path +) -> list[str]: + seen = {e.issue_url for e in read_sidecar(dispatch_id, project_dir)} + return [url for url in original_urls if url not in seen] diff --git a/src/autoskillit/fleet/state.py b/src/autoskillit/fleet/state.py new file mode 100644 index 0000000000..4c3fe79f43 --- /dev/null +++ b/src/autoskillit/fleet/state.py @@ -0,0 +1,363 @@ +"""Campaign state file management — DispatchRecord, atomic writes, resume algorithm. + +Provides the single-file state format for fleet campaign execution. +All writes use core.io.atomic_write for crash-safety (tmp + os.replace). +""" + +from __future__ import annotations + +import fcntl +import json +import time +from pathlib import Path +from typing import Any + +from autoskillit.core import get_logger, write_versioned_json +from autoskillit.fleet.state_gates import record_gate_outcome +from autoskillit.fleet.state_recovery import ( + crash_recover_dispatch, + has_failed_dispatch, + resume_campaign_from_state, +) +from autoskillit.fleet.state_types import ( + _ABANDON_KILL_REASONS, # noqa: F401 + _ALLOWED_TRANSITIONS, # noqa: F401 + _COMPLETED_STATUSES, # noqa: F401 + _INFRASTRUCTURE_FAILURE_REASONS, # noqa: F401 + _SCHEMA_VERSION, # noqa: F401 + _VISIBLE_IN_BLOCK_STATUSES, # noqa: F401 + FLEET_HALTED_SENTINEL, + TERMINAL_DISPATCH_STATUSES, + CampaignState, + DispatchRecord, + DispatchStatus, + GateRecordResult, + ResumeDecision, + _resume_lock, + _validate_transition, +) + +__all__ = [ + # re-exported from state_gates + "record_gate_outcome", + # re-exported from state_recovery + "crash_recover_dispatch", + "has_failed_dispatch", + "resume_campaign_from_state", + # re-exported from state_types + "FLEET_HALTED_SENTINEL", + "TERMINAL_DISPATCH_STATUSES", + "CampaignState", + "DispatchRecord", + "DispatchStatus", + "GateRecordResult", + "ResumeDecision", + # local + "write_initial_state", + "read_state", + "mark_dispatch_running", + "mark_dispatch_interrupted", + "mark_dispatch_resumable", + "reset_failed_dispatch", + "append_dispatch_record", + "build_protected_campaign_ids", + "write_captured_values", + "read_all_campaign_captures", +] + +logger = get_logger(__name__) + + +def write_initial_state( + state_path: Path, + campaign_id: str, + campaign_name: str, + manifest_path: str, + dispatches: list[DispatchRecord], +) -> None: + """Create the campaign state file with all dispatches in pending status. + + Uses write_versioned_json for schema_version convention compliance. + """ + payload = { + "campaign_id": campaign_id, + "campaign_name": campaign_name, + "manifest_path": manifest_path, + "started_at": time.time(), + "dispatches": [d.to_dict() for d in dispatches], + } + write_versioned_json(state_path, payload, schema_version=_SCHEMA_VERSION) + + +def _clear_dispatch_for_retry(d: DispatchRecord) -> None: + """Clear a dispatch record for retry.""" + _validate_transition(d.status, DispatchStatus.PENDING, d.name) + d.status = DispatchStatus.PENDING + d.reason = "" + d.dispatch_id = "" + d.dispatched_session_id = "" + d.dispatched_session_log_dir = "" + d.dispatched_pid = 0 + d.dispatched_starttime_ticks = 0 + d.dispatched_boot_id = "" + d.token_usage = {} + d.started_at = 0.0 + d.ended_at = 0.0 + d.sidecar_path = None + + +def reset_failed_dispatch(state_path: Path, dispatch_name: str) -> bool: + """Reset a FAILURE dispatch to PENDING, clearing all execution metadata. + + Returns True if the dispatch was found in FAILURE state and reset, + False if the dispatch was not found, not in FAILURE, or the state file + is missing/corrupted. OSError raised by _write_state propagates to + the caller — write failures are not silently converted to False. + + Thread-safe: uses _resume_lock + fcntl.LOCK_EX. + """ + with _resume_lock: + if not state_path.exists(): + return False + lock_path = state_path.with_suffix(".lock") + with open(lock_path, "wb") as _flock_handle: + fcntl.flock(_flock_handle, fcntl.LOCK_EX) + + state = read_state(state_path) + if state is None: + return False + + for d in state.dispatches: + if d.name == dispatch_name and d.status == DispatchStatus.FAILURE: + _clear_dispatch_for_retry(d) + _write_state(state_path, state) + return True + + return False + + +def read_state(state_path: Path) -> CampaignState | None: + """Load campaign state from disk. + + Returns None on missing file, malformed JSON, or schema mismatch. + Never raises. + """ + try: + data = json.loads(state_path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError): + return None + try: + dispatches = [DispatchRecord.from_dict(d) for d in data["dispatches"]] + return CampaignState( + schema_version=data["schema_version"], + campaign_id=data["campaign_id"], + campaign_name=data["campaign_name"], + manifest_path=data["manifest_path"], + started_at=data["started_at"], + dispatches=dispatches, + captured_values=data.get("captured_values", {}), + ) + except (KeyError, ValueError, TypeError) as exc: + logger.warning("read_state: schema mismatch or corrupt payload in %s: %s", state_path, exc) + return None + + +def _write_state(state_path: Path, state: CampaignState) -> None: + """Internal: atomic write of full state to disk.""" + payload = { + "campaign_id": state.campaign_id, + "campaign_name": state.campaign_name, + "manifest_path": state.manifest_path, + "started_at": state.started_at, + "dispatches": [d.to_dict() for d in state.dispatches], + "captured_values": state.captured_values, + } + write_versioned_json(state_path, payload, schema_version=state.schema_version) + + +def mark_dispatch_running( + state_path: Path, + dispatch_name: str, + *, + dispatch_id: str, + dispatched_pid: int, + starttime_ticks: int = 0, + boot_id: str = "", + sidecar_path: str | None = None, +) -> None: + """Atomically mark a dispatch as running with its dispatch_id and dispatched_pid.""" + state = read_state(state_path) + if state is None: + raise FileNotFoundError(f"State file not found or corrupted: {state_path}") + for d in state.dispatches: + if d.name == dispatch_name: + _validate_transition(d.status, DispatchStatus.RUNNING, d.name) + d.status = DispatchStatus.RUNNING + d.dispatch_id = dispatch_id + d.dispatched_pid = dispatched_pid + d.dispatched_starttime_ticks = starttime_ticks + d.dispatched_boot_id = boot_id + d.started_at = time.time() + d.sidecar_path = sidecar_path + break + else: + raise ValueError(f"Dispatch '{dispatch_name}' not found in state") + _write_state(state_path, state) + + +def mark_dispatch_interrupted( + state_path: Path, + dispatch_name: str, + *, + reason: str, +) -> None: + """Atomically mark a dispatch as interrupted with a reason.""" + state = read_state(state_path) + if state is None: + raise FileNotFoundError(f"State file not found or corrupted: {state_path}") + for d in state.dispatches: + if d.name == dispatch_name: + _validate_transition(d.status, DispatchStatus.INTERRUPTED, d.name) + d.status = DispatchStatus.INTERRUPTED + d.reason = reason + d.ended_at = time.time() + break + else: + raise ValueError(f"Dispatch '{dispatch_name}' not found in state") + _write_state(state_path, state) + + +def mark_dispatch_resumable( + state_path: Path, + dispatch_name: str, + *, + sidecar_path: str, +) -> None: + """Atomically transition a RUNNING dispatch to RESUMABLE, preserving the sidecar path.""" + state = read_state(state_path) + if state is None: + raise FileNotFoundError(f"State file not found or corrupted: {state_path}") + for d in state.dispatches: + if d.name == dispatch_name: + _validate_transition(d.status, DispatchStatus.RESUMABLE, d.name) + d.status = DispatchStatus.RESUMABLE + d.sidecar_path = sidecar_path + d.ended_at = time.time() + break + else: + raise ValueError(f"Dispatch '{dispatch_name}' not found in state") + _write_state(state_path, state) + + +def append_dispatch_record( + state_path: Path, + record: DispatchRecord, +) -> None: + """Atomically append or replace a dispatch record by name. + + If a dispatch with the same name exists, it is replaced in-place. + Otherwise the record is appended to the end. + """ + state = read_state(state_path) + if state is None: + raise FileNotFoundError(f"State file not found or corrupted: {state_path}") + for i, d in enumerate(state.dispatches): + if d.name == record.name: + _validate_transition(d.status, record.status, d.name) + state.dispatches[i] = record + _write_state(state_path, state) + return + state.dispatches.append(record) + _write_state(state_path, state) + + +def build_protected_campaign_ids(project_dir: Path) -> frozenset[str]: + """Return campaign IDs with at least one non-terminal dispatch. + + Reads fleet state files from ``{project_dir}/.autoskillit/temp/dispatches/``. + A campaign is protected if any of its dispatch records has a status that is NOT + in the terminal set {success, failure, skipped, released}. + Returns partially-accumulated results on unexpected errors rather than empty + frozenset, so active campaigns processed before a failure are still protected. + """ + protected: set[str] = set() + try: + dispatches_dir = project_dir / ".autoskillit" / "temp" / "dispatches" + if not dispatches_dir.is_dir(): + return frozenset() + for state_file in dispatches_dir.glob("*.json"): + try: + data = json.loads(state_file.read_text(encoding="utf-8")) + cid = data.get("campaign_id", "") + if not cid: + continue + dispatches = data.get("dispatches", []) + if not dispatches: + protected.add(cid) + continue + for record in dispatches: + status = record.get("status", "") + if status not in TERMINAL_DISPATCH_STATUSES: + protected.add(cid) + break + except (json.JSONDecodeError, OSError): + continue + return frozenset(protected) + except Exception: + logger.warning("campaign_ids_protection_error", exc_info=True) + return frozenset(protected) + + +def write_captured_values(state_path: Path, captures: dict[str, str]) -> None: + """Atomically merge new captures into an existing state file. + + Merges `captures` into the existing `captured_values` dict (new keys win). + No-op if state file is missing or corrupted (logs a warning). + """ + state = read_state(state_path) + if state is None: + logger.warning("write_captured_values: state not found at %s", state_path) + return + state.captured_values = {**state.captured_values, **captures} + _write_state(state_path, state) + + +def read_all_campaign_captures( + dispatches_dir: Path, + campaign_id: str, +) -> dict[str, str]: + """Accumulate captured_values from all SUCCESS dispatches for a campaign. + + Scans all *.json files in `dispatches_dir`. For each file matching + `campaign_id` where every dispatch record has status SUCCESS, merges + its `captured_values` into the result. Later files win on key collision. + """ + result: dict[str, str] = {} + if not dispatches_dir.is_dir(): + return result + for path in dispatches_dir.glob("*.json"): + try: + data = json.loads(path.read_text(encoding="utf-8")) + if data.get("campaign_id") != campaign_id: + continue + caps = data.get("captured_values", {}) + if not caps: + continue + dispatches = data.get("dispatches", []) + all_success = all(d.get("status") == DispatchStatus.SUCCESS for d in dispatches) + if all_success and dispatches: + result.update(caps) + except (json.JSONDecodeError, OSError) as exc: + logger.warning("read_all_campaign_captures: skipping %s: %s", path, exc) + continue + return result + + +def normalize_dispatch_token_usage(raw: dict[str, Any]) -> dict[str, int]: + """Map raw Claude session token keys to canonical DispatchTokenUsage key set.""" + return { + "input": int(raw.get("input_tokens", 0)), + "output": int(raw.get("output_tokens", 0)), + "cache_creation": int(raw.get("cache_creation_input_tokens", 0)), + "cache_read": int(raw.get("cache_read_input_tokens", 0)), + } diff --git a/src/autoskillit/fleet/state_gates.py b/src/autoskillit/fleet/state_gates.py new file mode 100644 index 0000000000..8c2a7298a1 --- /dev/null +++ b/src/autoskillit/fleet/state_gates.py @@ -0,0 +1,85 @@ +"""Gate dispatch recording.""" + +from __future__ import annotations + +import fcntl +import time +from pathlib import Path + +from autoskillit.fleet.state_types import ( + DispatchRecord, + DispatchStatus, + GateRecordResult, + _resume_lock, + _validate_transition, +) + + +def record_gate_outcome( + state_path: Path, + dispatch_name: str, + approved: bool, +) -> GateRecordResult: + """Record the outcome of a gate dispatch to the campaign state file. + + Returns a GateRecordResult with success/failure and error details. + + Thread-safe: _resume_lock (intra-process) + fcntl.flock(LOCK_EX) + (cross-process) prevent concurrent callers from corrupting state. + """ + from autoskillit.fleet.state import _write_state, read_state # noqa: PLC0415 + + with _resume_lock: + lock_path = state_path.with_suffix(".lock") + with open(lock_path, "wb") as _flock_handle: + fcntl.flock(_flock_handle, fcntl.LOCK_EX) + + state = read_state(state_path) + if state is None: + return GateRecordResult( + success=False, + dispatch_name=dispatch_name, + error_code="fleet_gate_no_campaign", + error_message=f"Campaign state file missing or corrupted: {state_path}", + ) + + match = next((d for d in state.dispatches if d.name == dispatch_name), None) + if match is None: + return GateRecordResult( + success=False, + dispatch_name=dispatch_name, + error_code="fleet_gate_unknown_dispatch", + error_message=f"Dispatch '{dispatch_name}' not found in campaign state.", + ) + + if match.status != DispatchStatus.PENDING: + return GateRecordResult( + success=False, + dispatch_name=dispatch_name, + error_code="fleet_gate_already_recorded", + error_message=( + f"Dispatch '{dispatch_name}' is already {match.status.value}, not PENDING." + ), + ) + + status = DispatchStatus.SUCCESS if approved else DispatchStatus.FAILURE + now = time.time() + new_record = DispatchRecord( + name=dispatch_name, + status=status, + reason="gate_approved" if approved else "gate_rejected", + started_at=now, + ended_at=now, + ) + for i, d in enumerate(state.dispatches): + if d.name == new_record.name: + _validate_transition(d.status, new_record.status, d.name) + state.dispatches[i] = new_record + _write_state(state_path, state) + break + + return GateRecordResult( + success=True, + dispatch_name=dispatch_name, + status=status.value, + ) diff --git a/src/autoskillit/fleet/state_recovery.py b/src/autoskillit/fleet/state_recovery.py new file mode 100644 index 0000000000..b5d0c62879 --- /dev/null +++ b/src/autoskillit/fleet/state_recovery.py @@ -0,0 +1,207 @@ +"""Crash recovery and campaign resume logic.""" + +from __future__ import annotations + +import fcntl +from pathlib import Path + +from autoskillit.core import InfraExitCategory, RetryReason, get_logger +from autoskillit.fleet.state_types import ( + _ABANDON_KILL_REASONS, + _INFRASTRUCTURE_FAILURE_REASONS, + _VISIBLE_IN_BLOCK_STATUSES, + FLEET_HALTED_SENTINEL, + DispatchRecord, + DispatchStatus, + ResumeDecision, + _resume_lock, +) + +logger = get_logger(__name__) + + +def has_failed_dispatch(state_path: Path) -> bool: + """Check whether any dispatch has a FAILURE status attributable to logic (not infrastructure). + + Infrastructure failures (e.g. fleet_l3_no_result_block) represent transient L3 + disconnections and do not halt the campaign. Logic failures (e.g. completed_clean + with success=false) represent genuine task failures and do halt the campaign. + + Returns False when the file is missing or corrupted (fail-open). + """ + from autoskillit.fleet.state import read_state # noqa: PLC0415 + + if not state_path.exists(): + return False + state = read_state(state_path) + if state is None: + return False + return any( + d.status == DispatchStatus.FAILURE and d.reason not in _INFRASTRUCTURE_FAILURE_REASONS + for d in state.dispatches + ) + + +def _is_abandon_kill_reason(kill_reason: str, infra_exit_category: str) -> bool: + """Check if stored kill metadata indicates resume would be futile.""" + if kill_reason in _ABANDON_KILL_REASONS: + return True + if ( + kill_reason == RetryReason.RESUME + and infra_exit_category == InfraExitCategory.CONTEXT_EXHAUSTED + ): + return True + return False + + +def crash_recover_dispatch( + state_path: Path, + record: DispatchRecord, + reason: str = "stale_running_on_resume", +) -> DispatchStatus | None: + """Recover a stale RUNNING dispatch to RESUMABLE or INTERRUPTED; None if both writes fail.""" + from autoskillit.fleet.sidecar import read_sidecar_from_path # noqa: PLC0415 + from autoskillit.fleet.state import ( # noqa: PLC0415 + mark_dispatch_interrupted, + mark_dispatch_resumable, + ) + + sidecar = Path(record.sidecar_path) if record.sidecar_path else None + if sidecar is not None and sidecar.exists(): + try: + raw_lines = [ln.strip() for ln in sidecar.read_text().splitlines() if ln.strip()] + except OSError: + logger.warning("crash_recover_dispatch: sidecar vanished during read", exc_info=True) + else: + if not raw_lines or read_sidecar_from_path(sidecar): + if _is_abandon_kill_reason(record.kill_reason, record.infra_exit_category): + try: + mark_dispatch_interrupted(state_path, record.name, reason=reason) + return DispatchStatus.INTERRUPTED + except Exception: + logger.warning( + "crash_recover_dispatch: failed to mark dispatch interrupted", + exc_info=True, + ) + return None + try: + mark_dispatch_resumable(state_path, record.name, sidecar_path=str(sidecar)) + return DispatchStatus.RESUMABLE + except Exception: + logger.warning( + "crash_recover_dispatch: failed to mark dispatch resumable", + exc_info=True, + ) + logger.debug( + "crash_recover_dispatch: no sidecar for %s — falling back to interrupted", + record.name, + ) + try: + mark_dispatch_interrupted(state_path, record.name, reason=reason) + return DispatchStatus.INTERRUPTED + except Exception: + logger.warning( + "crash_recover_dispatch: failed to mark dispatch interrupted", exc_info=True + ) + return None + + +def resume_campaign_from_state( + state_path: Path, + continue_on_failure: bool, + *, + reset_on_retry: bool = False, +) -> ResumeDecision | None: + """Determine the next dispatch for a resumed campaign. + + Algorithm: + 1. Read state.json; return None if absent or corrupted. + 2. Find first dispatch not in {success, skipped}. + 3. If running exists and stale, mark it interrupted; skip alive ones. + 4. If failure exists and continue_on_failure=False, return None + with reason fleet_halted_on_failure (encoded via a sentinel). + When reset_on_retry=True, reset all FAILURE dispatches to PENDING instead. + 5. Return ResumeDecision with next_dispatch_name and completed block. + + Returns None if the state file is missing/corrupted. Returns a + ResumeDecision with next_dispatch_name="" if all dispatches are + complete or the campaign is halted. + + Thread-safe: _resume_lock (intra-process) + fcntl.flock(LOCK_EX) + (cross-process) prevent concurrent callers from corrupting state. + """ + from autoskillit.fleet import is_dispatch_session_alive # noqa: PLC0415 + from autoskillit.fleet.state import ( # noqa: PLC0415 + _clear_dispatch_for_retry, + _write_state, + read_state, + ) + + with _resume_lock: + lock_path = state_path.with_suffix(".lock") + with open(lock_path, "wb") as _flock_handle: + fcntl.flock(_flock_handle, fcntl.LOCK_EX) + + state = read_state(state_path) + if state is None: + return None + + for d in state.dispatches: + if d.status == DispatchStatus.RUNNING: + if is_dispatch_session_alive(d): + continue + new_status = crash_recover_dispatch(state_path, d) + if new_status is not None: + d.status = new_status + d.reason = "stale_running_on_resume" + + did_reset = False + for d in state.dispatches: + if d.status == DispatchStatus.FAILURE and not continue_on_failure: + if reset_on_retry: + _clear_dispatch_for_retry(d) + did_reset = True + else: + return ResumeDecision( + next_dispatch_name="", + completed_dispatches_block=FLEET_HALTED_SENTINEL, + ) + if did_reset: + _write_state(state_path, state) + + completed_lines: list[str] = [] + next_name = "" + is_resumable = False + resumable_dispatched_session_id = "" + resumable_kill_reason = "" + for d in state.dispatches: + if d.status in _VISIBLE_IN_BLOCK_STATUSES: + completed_lines.append(f"- {d.name}: {d.status}") + elif d.status == DispatchStatus.RESUMABLE and not next_name: + next_name = d.name + is_resumable = True + resumable_dispatched_session_id = d.dispatched_session_id + resumable_kill_reason = d.kill_reason + elif ( + d.status + not in { + DispatchStatus.INTERRUPTED, + DispatchStatus.RUNNING, + DispatchStatus.REFUSED, + DispatchStatus.RELEASED, + DispatchStatus.FAILURE, + DispatchStatus.RESUMABLE, + } + and not next_name + ): + next_name = d.name + + completed_block = "\n".join(completed_lines) if completed_lines else "" + + return ResumeDecision( + next_dispatch_name=next_name, + completed_dispatches_block=completed_block, + is_resumable=is_resumable, + dispatched_session_id=resumable_dispatched_session_id, + kill_reason=resumable_kill_reason, + ) diff --git a/src/autoskillit/fleet/state_types.py b/src/autoskillit/fleet/state_types.py new file mode 100644 index 0000000000..2e17abd2c7 --- /dev/null +++ b/src/autoskillit/fleet/state_types.py @@ -0,0 +1,243 @@ +"""Campaign state types and constants.""" + +from __future__ import annotations + +import threading +from dataclasses import dataclass, field +from enum import StrEnum +from typing import Any + +from autoskillit.core import FleetErrorCode, RetryReason + +_resume_lock = threading.Lock() + +_SCHEMA_VERSION = 4 + +FLEET_HALTED_SENTINEL = "fleet_halted_on_failure" + + +class DispatchStatus(StrEnum): + """Status of a single dispatch within a campaign.""" + + PENDING = "pending" + RUNNING = "running" + SUCCESS = "success" + FAILURE = "failure" + INTERRUPTED = "interrupted" + RESUMABLE = "resumable" + SKIPPED = "skipped" + REFUSED = "refused" + RELEASED = "released" + + +@dataclass +class DispatchRecord: + """Runtime state of a single dispatch within a campaign. + + Mutable: status and metadata fields are updated as the dispatch progresses. + """ + + name: str + status: DispatchStatus = DispatchStatus.PENDING + dispatch_id: str = "" + campaign_id: str = "" + caller_session_id: str = "" + dispatched_session_id: str = "" + dispatched_session_log_dir: str = "" + dispatched_pid: int = 0 + dispatched_starttime_ticks: int = 0 + dispatched_boot_id: str = "" + reason: str = "" + kill_reason: str = "" + infra_exit_category: str = "" + token_usage: dict[str, Any] = field(default_factory=dict) + started_at: float = 0.0 + ended_at: float = 0.0 + sidecar_path: str | None = None + + def to_dict(self) -> dict[str, Any]: + return { + "name": self.name, + "status": self.status, + "dispatch_id": self.dispatch_id, + "campaign_id": self.campaign_id, + "caller_session_id": self.caller_session_id, + "dispatched_session_id": self.dispatched_session_id, + "dispatched_session_log_dir": self.dispatched_session_log_dir, + "dispatched_pid": self.dispatched_pid, + "dispatched_starttime_ticks": self.dispatched_starttime_ticks, + "dispatched_boot_id": self.dispatched_boot_id, + "reason": self.reason, + "kill_reason": self.kill_reason, + "infra_exit_category": self.infra_exit_category, + "token_usage": dict(self.token_usage), + "started_at": self.started_at, + "ended_at": self.ended_at, + "sidecar_path": self.sidecar_path, + } + + @classmethod + def from_dict(cls, d: dict[str, Any]) -> DispatchRecord: + pid_raw = d.get("dispatched_pid") + ticks_raw = d.get("dispatched_starttime_ticks") + return cls( + name=d["name"], + status=DispatchStatus(d.get("status", DispatchStatus.PENDING)), + dispatch_id=d.get("dispatch_id", ""), + campaign_id=d.get("campaign_id", ""), + caller_session_id=d.get("caller_session_id", ""), + dispatched_session_id=( + d.get("dispatched_session_id") + or d.get("l3_session_id") + or d.get("l2_session_id", "") + ), + dispatched_session_log_dir=( + d.get("dispatched_session_log_dir") + or d.get("l3_session_log_dir") + or d.get("l2_session_log_dir", "") + ), + dispatched_pid=( + pid_raw + if pid_raw is not None + else (l3_pid if (l3_pid := d.get("l3_pid")) is not None else d.get("l2_pid", 0)) + ), + dispatched_starttime_ticks=( + ticks_raw + if ticks_raw is not None + else ( + l3_ticks + if (l3_ticks := d.get("l3_starttime_ticks")) is not None + else d.get("l2_starttime_ticks", 0) + ) + ), + dispatched_boot_id=( + d.get("dispatched_boot_id") or d.get("l3_boot_id") or d.get("l2_boot_id", "") + ), + reason=d.get("reason", ""), + kill_reason=d.get("kill_reason", ""), + infra_exit_category=d.get("infra_exit_category", ""), + token_usage=d.get("token_usage", {}), + started_at=d.get("started_at", 0.0), + ended_at=d.get("ended_at", 0.0), + sidecar_path=d.get("sidecar_path"), + ) + + +@dataclass +class CampaignState: + """Top-level campaign state file content.""" + + schema_version: int + campaign_id: str + campaign_name: str + manifest_path: str + started_at: float + dispatches: list[DispatchRecord] = field(default_factory=list) + captured_values: dict[str, str] = field(default_factory=dict) + + +@dataclass(frozen=True, slots=True) +class ResumeDecision: + """Result of the resume algorithm.""" + + next_dispatch_name: str + completed_dispatches_block: str + is_resumable: bool = False + dispatched_session_id: str = "" + kill_reason: str = "" + + +_ALLOWED_TRANSITIONS: dict[str, frozenset[str]] = { + DispatchStatus.PENDING: frozenset( + { + DispatchStatus.RUNNING, + DispatchStatus.SUCCESS, + DispatchStatus.FAILURE, + DispatchStatus.SKIPPED, + DispatchStatus.REFUSED, + DispatchStatus.RELEASED, + } + ), + DispatchStatus.RUNNING: frozenset( + { + DispatchStatus.SUCCESS, + DispatchStatus.FAILURE, + DispatchStatus.INTERRUPTED, + DispatchStatus.RESUMABLE, + } + ), + DispatchStatus.RESUMABLE: frozenset( + { + DispatchStatus.RUNNING, + DispatchStatus.SUCCESS, + DispatchStatus.FAILURE, + DispatchStatus.INTERRUPTED, + } + ), + DispatchStatus.FAILURE: frozenset({DispatchStatus.PENDING}), + DispatchStatus.SUCCESS: frozenset(), + DispatchStatus.INTERRUPTED: frozenset(), + DispatchStatus.SKIPPED: frozenset(), + DispatchStatus.REFUSED: frozenset(), + DispatchStatus.RELEASED: frozenset(), +} + + +def _validate_transition(current: str, new: str, dispatch_name: str) -> None: + """Raise ValueError if the status transition is not allowed.""" + allowed = _ALLOWED_TRANSITIONS.get(current) + if allowed is not None and new not in allowed: + msg = f"Invalid transition for dispatch '{dispatch_name}': {current!r} -> {new!r}" + raise ValueError(msg) + + +_INFRASTRUCTURE_FAILURE_REASONS: frozenset[str] = frozenset( + { + FleetErrorCode.FLEET_L3_NO_RESULT_BLOCK, + FleetErrorCode.FLEET_QUOTA_EXHAUSTED, + } +) + + +@dataclass(frozen=True, slots=True) +class GateRecordResult: + """Result of a gate dispatch recording attempt.""" + + success: bool + dispatch_name: str + status: str = "" + error_code: str = "" + error_message: str = "" + + +_COMPLETED_STATUSES = frozenset( + {DispatchStatus.SUCCESS, DispatchStatus.SKIPPED, DispatchStatus.FAILURE} +) + +_VISIBLE_IN_BLOCK_STATUSES = _COMPLETED_STATUSES | frozenset( + { + DispatchStatus.INTERRUPTED, + DispatchStatus.REFUSED, + DispatchStatus.RELEASED, + DispatchStatus.RUNNING, + } +) + +TERMINAL_DISPATCH_STATUSES: frozenset[str] = frozenset( + { + DispatchStatus.SUCCESS, + DispatchStatus.FAILURE, + DispatchStatus.SKIPPED, + DispatchStatus.RELEASED, + } +) + + +_ABANDON_KILL_REASONS: frozenset[str] = frozenset( + { + RetryReason.STALE, + RetryReason.THINKING_STALL, + RetryReason.PATH_CONTAMINATION, + RetryReason.CLONE_CONTAMINATION, + } +) diff --git a/src/autoskillit/fleet/summary.py b/src/autoskillit/fleet/summary.py new file mode 100644 index 0000000000..8f35fce0d2 --- /dev/null +++ b/src/autoskillit/fleet/summary.py @@ -0,0 +1,271 @@ +"""Campaign summary schema v1 — dataclasses, parser, and validator. + +Provides the structured contract for the campaign summary sentinel block +emitted by L3 fleet sessions before exit. +""" + +from __future__ import annotations + +import json +from dataclasses import dataclass +from enum import StrEnum +from typing import Any + +import regex as re + +from autoskillit.core import FleetErrorCode + + +class CampaignSummaryStatus(StrEnum): + """Strict 3-value status enum for per-dispatch summary entries.""" + + SUCCESS = "success" + FAILURE = "failure" + SKIPPED = "skipped" + + +class ParseFailureKind(StrEnum): + SENTINEL_MISSING = "sentinel_missing" + CAMPAIGN_ID_MISMATCH = "campaign_id_mismatch" + JSON_DECODE_ERROR = "json_decode_error" + SCHEMA_VALIDATION_ERROR = "schema_validation_error" + FIELD_ERROR = "field_error" + + +@dataclass(frozen=True, slots=True) +class ParseFailure: + kind: ParseFailureKind + message: str + + def __post_init__(self) -> None: + if not self.message: + raise ValueError("ParseFailure.message must not be empty") + + +@dataclass(frozen=True, slots=True) +class DispatchTokenUsage: + """Per-dispatch token usage — exactly 4 fields, no extras.""" + + input: int + output: int + cache_read: int + cache_creation: int + + +@dataclass(frozen=True, slots=True) +class PerDispatchEntry: + """One entry per dispatch in execution order.""" + + name: str + status: CampaignSummaryStatus + elapsed_seconds: float + token_usage: DispatchTokenUsage + dispatched_session_id: str + dispatch_id: str + + +@dataclass(frozen=True, slots=True) +class SummaryErrorRecord: + """One entry per failed dispatch.""" + + dispatch_name: str + code: FleetErrorCode + message: str + dispatched_session_id: str + + +@dataclass(frozen=True, slots=True) +class CampaignSummary: + """Campaign summary schema v1 — top-level contract.""" + + schema_version: int + campaign_id: str + campaign_name: str + dispatch_count: int + completed_count: int + failure_count: int + skipped_count: int + per_dispatch: list[PerDispatchEntry] + error_records: list[SummaryErrorRecord] + + +CampaignParseResult = CampaignSummary | ParseFailure + +_SUMMARY_PATTERN = re.compile( + r"---campaign-summary::(?P.+?)---[^\n]*\n" + r"(?P.*?)\n" + r"---end-campaign-summary::(?P.+?)---", + re.DOTALL, +) + +_FORBIDDEN_AGGREGATE_KEYS = frozenset( + { + "total_input_tokens", + "total_output_tokens", + "total_cache_read_tokens", + "total_cache_creation_tokens", + "total_duration", + "total_elapsed_seconds", + } +) + +_REQUIRED_TOP_KEYS = frozenset( + { + "schema_version", + "campaign_id", + "campaign_name", + "dispatch_count", + "completed_count", + "failure_count", + "skipped_count", + "per_dispatch", + "error_records", + } +) + + +def validate_campaign_summary(data: dict[str, Any]) -> list[str]: + """Validate raw JSON dict against campaign summary schema v1. + + Returns list of error strings. Empty list = valid. + """ + errors: list[str] = [] + for key in _FORBIDDEN_AGGREGATE_KEYS: + if key in data: + errors.append(f"Forbidden aggregate field present: {key}") + for key in data: + if key.startswith("total_") and key not in _REQUIRED_TOP_KEYS: + errors.append(f"Forbidden aggregate field present: {key}") + missing = _REQUIRED_TOP_KEYS - set(data.keys()) + if missing: + errors.append(f"Missing required fields: {sorted(missing)}") + if data.get("schema_version") != 1: + errors.append(f"schema_version must be 1, got {data.get('schema_version')}") + for i, entry in enumerate(data.get("per_dispatch", [])): + status = entry.get("status") + try: + CampaignSummaryStatus(status) + except (ValueError, KeyError): + errors.append(f"per_dispatch[{i}].status invalid: {status!r}") + tu = entry.get("token_usage") + if isinstance(tu, dict): + tu_keys = set(tu.keys()) + expected = {"input", "output", "cache_read", "cache_creation"} + if tu_keys != expected: + errors.append( + f"per_dispatch[{i}].token_usage must have exactly " + f"{sorted(expected)}, got {sorted(tu_keys)}" + ) + return errors + + +def parse_campaign_summary(text: str, campaign_id: str) -> CampaignParseResult: + """Parse campaign summary from sentinel-wrapped text. + + Returns CampaignSummary on success, ParseFailure with a specific kind on any failure. + """ + match = _SUMMARY_PATTERN.search(text) + if match is None: + return ParseFailure( + ParseFailureKind.SENTINEL_MISSING, "No campaign summary sentinel found" + ) + if match.group("cid") != campaign_id or match.group("cid_end") != campaign_id: + cid, cid_end = match.group("cid"), match.group("cid_end") + return ParseFailure( + ParseFailureKind.CAMPAIGN_ID_MISMATCH, + f"Sentinel ids {cid!r}/{cid_end!r} do not match expected {campaign_id!r}", + ) + try: + data = json.loads(match.group("body")) + except json.JSONDecodeError as exc: + return ParseFailure( + ParseFailureKind.JSON_DECODE_ERROR, + f"{exc.msg} (line {exc.lineno} col {exc.colno})", + ) + errors = validate_campaign_summary(data) + if errors: + return ParseFailure(ParseFailureKind.SCHEMA_VALIDATION_ERROR, "; ".join(errors)) + try: + per_dispatch = [ + PerDispatchEntry( + name=e["name"], + status=CampaignSummaryStatus(e["status"]), + elapsed_seconds=float(e["elapsed_seconds"]), + token_usage=DispatchTokenUsage( + input=e["token_usage"]["input"], + output=e["token_usage"]["output"], + cache_read=e["token_usage"]["cache_read"], + cache_creation=e["token_usage"]["cache_creation"], + ), + dispatched_session_id=e.get("dispatched_session_id") + or e.get("l3_session_id") + or e.get("l2_session_id", ""), + dispatch_id=e.get("dispatch_id", ""), + ) + for e in data["per_dispatch"] + ] + error_records = [ + SummaryErrorRecord( + dispatch_name=r["dispatch_name"], + code=FleetErrorCode(r["code"]), + message=r["message"], + dispatched_session_id=r.get("dispatched_session_id") + or r.get("l3_session_id") + or r.get("l2_session_id", ""), + ) + for r in data["error_records"] + ] + return CampaignSummary( + schema_version=data["schema_version"], + campaign_id=data["campaign_id"], + campaign_name=data["campaign_name"], + dispatch_count=data["dispatch_count"], + completed_count=data["completed_count"], + failure_count=data["failure_count"], + skipped_count=data["skipped_count"], + per_dispatch=per_dispatch, + error_records=error_records, + ) + except (KeyError, TypeError, ValueError) as exc: + return ParseFailure(ParseFailureKind.FIELD_ERROR, f"Field extraction failed: {exc}") + + +def serialize_campaign_summary(summary: CampaignSummary) -> str: + """Serialize a CampaignSummary to sentinel-wrapped JSON text.""" + data = { + "schema_version": summary.schema_version, + "campaign_id": summary.campaign_id, + "campaign_name": summary.campaign_name, + "dispatch_count": summary.dispatch_count, + "completed_count": summary.completed_count, + "failure_count": summary.failure_count, + "skipped_count": summary.skipped_count, + "per_dispatch": [ + { + "name": e.name, + "status": e.status.value, + "elapsed_seconds": e.elapsed_seconds, + "token_usage": { + "input": e.token_usage.input, + "output": e.token_usage.output, + "cache_read": e.token_usage.cache_read, + "cache_creation": e.token_usage.cache_creation, + }, + "dispatched_session_id": e.dispatched_session_id, + "dispatch_id": e.dispatch_id, + } + for e in summary.per_dispatch + ], + "error_records": [ + { + "dispatch_name": r.dispatch_name, + "code": r.code, + "message": r.message, + "dispatched_session_id": r.dispatched_session_id, + } + for r in summary.error_records + ], + } + body = json.dumps(data, indent=2) + cid = summary.campaign_id + return f"---campaign-summary::{cid}---\n{body}\n---end-campaign-summary::{cid}---" diff --git a/src/autoskillit/hook_registry.py b/src/autoskillit/hook_registry.py index 7cdc3c7028..34392280c4 100644 --- a/src/autoskillit/hook_registry.py +++ b/src/autoskillit/hook_registry.py @@ -13,10 +13,10 @@ from pathlib import Path from typing import Literal, NamedTuple -from autoskillit.core import pkg_root +from autoskillit.core import DIRECT_INSTALL_CACHE_SUBDIR, pkg_root -@dataclass(frozen=True) +@dataclass(frozen=True, slots=True) class HookDef: """A single hook group: event type, matcher pattern, and ordered script list.""" @@ -36,54 +36,112 @@ def __post_init__(self) -> None: HOOK_REGISTRY: list[HookDef] = [ HookDef( matcher="mcp__.*autoskillit.*__run_skill.*", - scripts=["skill_cmd_guard.py", "quota_guard.py", "skill_command_guard.py"], + scripts=[ + "guards/skill_cmd_guard.py", + "guards/quota_guard.py", + "guards/skill_command_guard.py", + ], ), HookDef( matcher="mcp__.*autoskillit.*__remove_clone", - scripts=["remove_clone_guard.py"], + scripts=["guards/remove_clone_guard.py"], ), HookDef( matcher=r"mcp__.*autoskillit.*__open_kitchen.*", - scripts=["open_kitchen_guard.py"], + scripts=["guards/open_kitchen_guard.py"], timeout_seconds=5, ), HookDef( matcher="AskUserQuestion", - scripts=["ask_user_question_guard.py"], + scripts=["guards/ask_user_question_guard.py"], timeout_seconds=5, session_scope="headless_only", ), HookDef( matcher=r"mcp__.*autoskillit.*__merge_worktree", - scripts=["branch_protection_guard.py"], + scripts=["guards/branch_protection_guard.py"], ), HookDef( matcher=r"mcp__.*autoskillit.*__push_to_remote", - scripts=["branch_protection_guard.py"], + scripts=["guards/branch_protection_guard.py"], + ), + HookDef( + matcher=r"Bash|mcp__.*autoskillit.*__run_cmd", + scripts=["guards/unsafe_install_guard.py", "guards/pr_create_guard.py"], + ), + HookDef( + matcher=r"Bash|mcp__.*autoskillit.*__run_cmd", + scripts=["guards/planner_gh_discovery_guard.py"], + session_scope="headless_only", + ), + HookDef( + matcher=r"Write|Edit", + scripts=["guards/generated_file_write_guard.py"], ), HookDef( - matcher=r"mcp__.*autoskillit.*__run_cmd", - scripts=["unsafe_install_guard.py"], + matcher=r"Write|Edit", + scripts=["guards/write_guard.py"], + session_scope="headless_only", ), HookDef( matcher=r"Write|Edit", - scripts=["generated_file_write_guard.py"], + scripts=["guards/recipe_write_advisor.py"], + session_scope="interactive_only", + ), + HookDef( + matcher=r"Grep", + scripts=["guards/grep_pattern_lint_guard.py"], + ), + HookDef( + matcher=r"Bash|Write|Edit|Read|Glob|Grep", + scripts=["guards/mcp_health_advisor.py"], + timeout_seconds=5, + session_scope="interactive_only", ), HookDef( matcher=r"mcp__.*autoskillit.*__(run_skill|run_cmd|run_python).*", - scripts=["headless_orchestration_guard.py"], + scripts=["guards/skill_orchestration_guard.py"], session_scope="headless_only", ), + HookDef( + matcher=r"(mcp__.*autoskillit.*__)?dispatch_food_truck", + scripts=["guards/fleet_dispatch_guard.py", "guards/resume_ownership_guard.py"], + ), HookDef( event_type="PostToolUse", matcher="mcp__.*autoskillit.*", - scripts=["pretty_output_hook.py"], + scripts=["formatters/pretty_output_hook.py"], ), HookDef( event_type="PostToolUse", matcher=r"mcp__.*autoskillit.*__run_skill.*", scripts=["token_summary_hook.py", "quota_post_hook.py"], ), + HookDef( + event_type="PostToolUse", + matcher=r"mcp__.*autoskillit.*__(run_skill|run_python).*", + scripts=["review_gate_post_hook.py"], + ), + HookDef( + event_type="PostToolUse", + matcher=r"Write|Edit", + scripts=["lint_after_edit_hook.py"], + session_scope="headless_only", + ), + HookDef( + event_type="PostToolUse", + matcher="Skill", + scripts=["skill_load_post_hook.py"], + ), + HookDef( + matcher=r"Read|Write|Edit|Bash|Grep|Glob", + scripts=["guards/skill_load_guard.py"], + session_scope="headless_only", + ), + HookDef( + matcher=r"mcp__.*autoskillit.*__(wait_for_ci|enqueue_pr)", + scripts=["guards/review_loop_gate.py"], + ), HookDef( event_type="SessionStart", scripts=["session_start_hook.py"], @@ -100,10 +158,43 @@ def __post_init__(self) -> None: "pretty_output.py", "token_summary_appender.py", "session_start_reminder.py", + "headless_orchestration_guard.py", + "franchise_dispatch_guard.py", + "ask_user_question_guard.py", + "branch_protection_guard.py", + "fleet_dispatch_guard.py", + "generated_file_write_guard.py", + "grep_pattern_lint_guard.py", + "leaf_orchestration_guard.py", + "mcp_health_guard.py", + "open_kitchen_guard.py", + "planner_gh_discovery_guard.py", + "pr_create_guard.py", + "quota_guard.py", + "recipe_write_advisor.py", + "remove_clone_guard.py", + "review_loop_gate.py", + "skill_cmd_guard.py", + "skill_command_guard.py", + "unsafe_install_guard.py", + "write_guard.py", + "pretty_output_hook.py", # Append any future retired basenames here, atomically with the rename commit. } ) +# Basenames of scripts added directly to a subdirectory without ever having a flat path. +# Add the basename here when introducing a new subdir script that was never previously flat, +# so test_moved_scripts_must_be_in_retired does not false-positive on it. +NEW_SUBDIR_BASENAMES: frozenset[str] = frozenset( + { + "mcp_health_advisor.py", + "skill_orchestration_guard.py", + "skill_load_guard.py", + "resume_ownership_guard.py", + } +) + def _canonical_registry_payload( registry: list[HookDef], @@ -123,7 +214,7 @@ def _canonical_registry_payload( key=lambda row: (row["event_type"], row["matcher"], tuple(row["scripts"])), # type: ignore[arg-type] ) return json.dumps( - {"registry": registry_rows, "retired": sorted(retired)}, + {"format_version": 2, "registry": registry_rows, "retired": sorted(retired)}, sort_keys=True, separators=(",", ":"), ) @@ -160,31 +251,46 @@ def _build_hook_entry(hook_def: HookDef, hook_commands: list[dict]) -> dict: return {"matcher": hook_def.matcher, "hooks": hook_commands} +def _build_hook_command(hooks_dir: Path, script: str, timeout_seconds: int | None) -> dict: + """Build a single hook command dict using the stable dispatcher format.""" + logical_name = script.removesuffix(".py") + cmd: dict = { + "type": "command", + "command": f"python3 {hooks_dir / '_dispatch.py'} {logical_name}", + } + if timeout_seconds is not None: + cmd["timeout"] = timeout_seconds + return cmd + + def generate_hooks_json() -> dict: - """Generate the hooks.json structure from HOOK_REGISTRY using absolute paths.""" - by_event: dict[str, list] = {} + """Generate the hooks.json structure from HOOK_REGISTRY using the stable dispatcher. + + Multiple HookDef entries with the same (event_type, matcher) are consolidated + into a single settings.json entry so Claude Code sees no duplicate matchers. + """ + # Preserve insertion order; merge scripts from same (event_type, matcher) key. + groups: dict[tuple[str, str], dict] = {} for hook_def in HOOK_REGISTRY: + key = (hook_def.event_type, hook_def.matcher) hook_commands = [ - { - "type": "command", - "command": f"python3 {HOOKS_DIR / script}", - **( - {"timeout": hook_def.timeout_seconds} - if hook_def.timeout_seconds is not None - else {} - ), - } + _build_hook_command(HOOKS_DIR, script, hook_def.timeout_seconds) for script in hook_def.scripts ] - by_event.setdefault(hook_def.event_type, []).append( - _build_hook_entry(hook_def, hook_commands) - ) + if key not in groups: + groups[key] = _build_hook_entry(hook_def, hook_commands) + else: + groups[key]["hooks"].extend(hook_commands) + + by_event: dict[str, list] = {} + for (event_type, _), entry in groups.items(): + by_event.setdefault(event_type, []).append(entry) return {"hooks": by_event, "_autoskillit_registry_hash": HOOK_REGISTRY_HASH} # --------------------------------------------------------------------------- -# Hook diagnostic utilities — shared between cli/ and server/ (both L3). -# Placed here (package root, L0-accessible) to avoid L3-to-L3 peer imports. +# Hook diagnostic utilities — shared between cli/ and server/ (both IL-3). +# Placed here (package root, IL-0-accessible) to avoid IL-3-to-IL-3 peer imports. # --------------------------------------------------------------------------- @@ -232,24 +338,39 @@ def _is_own_hook(command: str) -> bool: """Check if a hook command belongs to autoskillit (any format).""" if "autoskillit" in command: return True + if "_dispatch.py" in command: + return True known = canonical_script_basenames() | RETIRED_SCRIPT_BASENAMES - return any(command.endswith(script) or f"/{script}" in command for script in known) + bare = {Path(s).name for s in known} + return any(command.endswith(script) or f"/{script}" in command for script in known | bare) def _extract_script_basenames(hooks_dict: dict) -> set[str]: - """Extract autoskillit hook script basenames from a hooks dict. + """Extract autoskillit hook script relative paths from a hooks dict. Filters to autoskillit-owned commands only, then normalizes - to bare script filenames for installation-path-agnostic comparison. + to hooks-dir-relative paths for installation-path-agnostic comparison. """ - return { - Path(cmd.split()[-1]).name - for event_entries in hooks_dict.values() - if isinstance(event_entries, list) - for entry in event_entries - for hook in entry.get("hooks", []) - if (cmd := hook.get("command", "")) and _is_own_hook(cmd) - } + result: set[str] = set() + for event_entries in hooks_dict.values(): + if not isinstance(event_entries, list): + continue + for entry in event_entries: + for hook in entry.get("hooks", []): + cmd = hook.get("command", "") + if not cmd or not _is_own_hook(cmd): + continue + parts = cmd.split() + if "_dispatch.py" in cmd and len(parts) >= 3: + logical_name = parts[-1] + result.add(logical_name + ".py") + else: + script_path = Path(parts[-1]) + bare = script_path.name + canonical = canonical_script_basenames() + matched = next((c for c in canonical if Path(c).name == bare), bare) + result.add(matched) + return result class HookDriftResult(NamedTuple): @@ -281,10 +402,30 @@ def find_broken_hook_scripts(settings_path: Path) -> list[str]: for entry in data.get("hooks", {}).get(event_type, []): for hook in entry.get("hooks", []): cmd = hook.get("command", "") - if not _is_own_hook(cmd): # skip non-autoskillit hooks + if not _is_own_hook(cmd): continue parts = cmd.split() - if len(parts) >= 2: + if len(parts) >= 3 and parts[-2].endswith("_dispatch.py"): + if not Path(parts[-2]).is_file(): + broken.append(cmd) + elif len(parts) >= 2: if not Path(parts[-1]).is_file(): broken.append(cmd) return broken + + +def validate_plugin_cache_hooks(cache_dir: Path | None = None) -> list[str]: + """Return list of broken hook commands from the plugin cache hooks.json. + + Reads each hooks.json found under cache_dir/*/hooks.json and checks that + every autoskillit hook script path exists on disk. + """ + _cache = cache_dir or ( + Path.home() / ".claude" / "plugins" / "cache" / DIRECT_INSTALL_CACHE_SUBDIR / "autoskillit" + ) + broken: list[str] = [] + if not _cache.is_dir(): + return broken + for hooks_json_path in _cache.glob("*/hooks.json"): + broken.extend(find_broken_hook_scripts(hooks_json_path)) + return broken diff --git a/src/autoskillit/hooks/CLAUDE.md b/src/autoskillit/hooks/CLAUDE.md new file mode 100644 index 0000000000..bf75dc451c --- /dev/null +++ b/src/autoskillit/hooks/CLAUDE.md @@ -0,0 +1,27 @@ +# hooks/ + +Claude Code `PreToolUse`/`PostToolUse`/`SessionStart` scripts. +Sub-packages: guards/ (see guards/CLAUDE.md), formatters/ (see formatters/CLAUDE.md). + +## Files + +| File | Purpose | +|------|---------| +| `__init__.py` | Package marker (no imports) | +| `_dispatch.py` | Stable hook dispatcher — resolves logical hook names to scripts (stdlib-only, NEVER RENAME) | +| `_hook_settings.py` | Shared stdlib-only settings resolver for quota guard hooks | +| `lint_after_edit_hook.py` | `PostToolUse`: runs ruff format+check on `.py` files after Edit/Write | +| `quota_post_hook.py` | Appends quota warning to `run_skill` output | +| `review_gate_post_hook.py` | `PostToolUse`: writes/clears `review_gate_state.json` | +| `token_summary_hook.py` | Appends Token Usage Summary to PR body | +| `session_start_hook.py` | Injects open-kitchen reminder on resume | +| `skill_load_post_hook.py` | `PostToolUse`: writes skill-loaded flag for non-Anthropic provider guard | +| `_hook_utils.py` | Shared stdlib-only utilities for hook scripts (e.g., `find_project_root`) | + +## Architecture Notes + +`_dispatch.py` must never be renamed — it is referenced by absolute path in `hooks.json` +and the `HOOK_REGISTRY`. All hook scripts are stdlib-only standalone executables; they do +not import from `autoskillit.*` except via `_dispatch.py`'s path-resolution logic. +Renaming any hook script requires updating `HOOK_REGISTRY` in `hook_registry.py` AND +adding the old basename to `RETIRED_SCRIPT_BASENAMES` in the same commit. diff --git a/src/autoskillit/hooks/__init__.py b/src/autoskillit/hooks/__init__.py index 15a36cbdc8..ba51cc3dcd 100644 --- a/src/autoskillit/hooks/__init__.py +++ b/src/autoskillit/hooks/__init__.py @@ -5,6 +5,25 @@ HookDef, generate_hooks_json, ) -from autoskillit.hooks._fmt_primitives import _HOOK_CONFIG_PATH_COMPONENTS +from autoskillit.hooks.formatters._fmt_primitives import _HOOK_CONFIG_PATH_COMPONENTS +from autoskillit.hooks.guards.branch_protection_guard import BRANCH_PROTECTION_DENY_TRIGGER +from autoskillit.hooks.guards.quota_guard import ( + QUOTA_BUDGET_EXCEEDED_TRIGGER, + QUOTA_GUARD_DENY_TRIGGER, +) +from autoskillit.hooks.guards.review_loop_gate import REVIEW_LOOP_DENY_TRIGGER +from autoskillit.hooks.guards.skill_orchestration_guard import SKILL_ORCHESTRATION_DENY_TRIGGER +from autoskillit.hooks.quota_post_hook import QUOTA_POST_WARNING_TRIGGER -__all__ = ["HOOK_REGISTRY", "HookDef", "_HOOK_CONFIG_PATH_COMPONENTS", "generate_hooks_json"] +__all__ = [ + "HOOK_REGISTRY", + "HookDef", + "BRANCH_PROTECTION_DENY_TRIGGER", + "SKILL_ORCHESTRATION_DENY_TRIGGER", + "QUOTA_GUARD_DENY_TRIGGER", + "QUOTA_BUDGET_EXCEEDED_TRIGGER", + "QUOTA_POST_WARNING_TRIGGER", + "REVIEW_LOOP_DENY_TRIGGER", + "_HOOK_CONFIG_PATH_COMPONENTS", + "generate_hooks_json", +] diff --git a/src/autoskillit/hooks/_dispatch.py b/src/autoskillit/hooks/_dispatch.py new file mode 100644 index 0000000000..7000219e3f --- /dev/null +++ b/src/autoskillit/hooks/_dispatch.py @@ -0,0 +1,68 @@ +"""Stable hook dispatcher — NEVER RENAME THIS FILE. + +This file is the sole hook command target for all Claude Code hook entries. +Its path stability is a contract: renaming it would break in-flight sessions. +""" + +from __future__ import annotations + +import subprocess +import sys +from pathlib import Path + +# Stdin is one-shot from Claude Code — must buffer before subprocess. +_HOOKS_DIR = Path(__file__).parent + +_RETIRED_MAPPING: dict[str, str] = { + "guards/leaf_orchestration_guard": "guards/skill_orchestration_guard", + "guards/franchise_dispatch_guard": "guards/fleet_dispatch_guard", + "guards/headless_orchestration_guard": "guards/skill_orchestration_guard", + "guards/mcp_health_guard": "guards/mcp_health_advisor", +} + + +def main() -> None: + if len(sys.argv) < 2: + print("usage: _dispatch.py ", file=sys.stderr) + sys.exit(1) + + logical_name = sys.argv[1] + target = _HOOKS_DIR / (logical_name + ".py") + + if not target.is_file(): + resolved = _RETIRED_MAPPING.get(logical_name) + if resolved: + target = _HOOKS_DIR / (resolved + ".py") + else: + print( + f"[autoskillit dispatch] unknown hook: {logical_name} — degrading gracefully", + file=sys.stderr, + ) + sys.exit(0) + + if not target.is_file(): + print( + f"[autoskillit dispatch] retired target missing: {target} — degrading gracefully", + file=sys.stderr, + ) + sys.exit(0) + + stdin_data = sys.stdin.buffer.read() + + try: + result = subprocess.run( + [sys.executable, str(target)], + input=stdin_data, + capture_output=False, + ) + except OSError as exc: + print( + f"[autoskillit dispatch] exec failed for {target}: {exc} — degrading gracefully", + file=sys.stderr, + ) + sys.exit(0) + sys.exit(result.returncode) + + +if __name__ == "__main__": + main() diff --git a/src/autoskillit/hooks/_hook_settings.py b/src/autoskillit/hooks/_hook_settings.py index 4aa3ba28ac..99b5394077 100644 --- a/src/autoskillit/hooks/_hook_settings.py +++ b/src/autoskillit/hooks/_hook_settings.py @@ -33,6 +33,36 @@ ENV_BUFFER_SECONDS = "AUTOSKILLIT_QUOTA_GUARD__BUFFER_SECONDS" ENV_DISABLED = "AUTOSKILLIT_QUOTA_GUARD__DISABLED" +# The exact keys this module reads from hook_config["quota_guard"]. +# The bridge contract test asserts equality between this set and the +# serializer's payload keys — update both together. +QUOTA_GUARD_HOOK_PAYLOAD_KEYS: frozenset[str] = frozenset( + {"cache_path", "cache_max_age", "buffer_seconds", "disabled"} +) + +# The exact keys that appear in token_usage.json files written by flush_session_log. +# The bridge contract test asserts equality between this set and TokenUsageFileEntry +# annotations — update both together. +TOKEN_USAGE_FILE_KEYS: frozenset[str] = frozenset( + { + "session_label", + "input_tokens", + "output_tokens", + "cache_creation_input_tokens", + "cache_read_input_tokens", + "peak_context", + "turn_count", + "timing_seconds", + "order_id", + "loc_insertions", + "loc_deletions", + "provider_used", + "model_identifier", + "dispatch_id", + "campaign_id", + } +) + @dataclass(frozen=True, slots=True) class QuotaHookSettings: diff --git a/src/autoskillit/hooks/_hook_utils.py b/src/autoskillit/hooks/_hook_utils.py new file mode 100644 index 0000000000..a35eca4251 --- /dev/null +++ b/src/autoskillit/hooks/_hook_utils.py @@ -0,0 +1,18 @@ +"""Shared stdlib-only utilities for hook scripts. + +Imported by both PreToolUse guards (in guards/) and PostToolUse hooks +via sys.path bootstrap. Stdlib-only — no autoskillit imports. +""" + +from __future__ import annotations + +from pathlib import Path + + +def find_project_root() -> Path: + """Walk up from CWD to find nearest ancestor containing .autoskillit/.""" + cwd = Path.cwd() + for ancestor in [cwd, *cwd.parents]: + if (ancestor / ".autoskillit").is_dir(): + return ancestor + return cwd diff --git a/src/autoskillit/hooks/formatters/CLAUDE.md b/src/autoskillit/hooks/formatters/CLAUDE.md new file mode 100644 index 0000000000..945c0a758b --- /dev/null +++ b/src/autoskillit/hooks/formatters/CLAUDE.md @@ -0,0 +1,18 @@ +# formatters/ + +PostToolUse output formatters — MCP JSON to Markdown-KV reformatter (30-77% token reduction). + +## Files + +| File | Purpose | +|------|---------| +| `__init__.py` | Package marker (no imports) | +| `pretty_output_hook.py` | Dispatch entrypoint: intercepts MCP tool responses, routes to per-tool formatters | +| `_fmt_primitives.py` | Shared primitives: `_CHECK_MARK`, `_CROSS_MARK`, payload dataclasses, token formatter | +| `_fmt_execution.py` | Formatters for `run_skill`, `run_cmd`, `test_check`, `merge_worktree` | +| `_fmt_recipe.py` | Formatters for `load_recipe`, `open_kitchen`, `list_recipes` | +| `_fmt_status.py` | Formatters for `get_token_summary`, `get_timing_summary`, `kitchen_status` | + +## Architecture Notes + +All `_fmt_*` modules use bare relative imports (`from _fmt_primitives import ...`) because hook scripts run as standalone executables with this directory as CWD — not via the Python package system. `pretty_output_hook.py` is the only entry point. diff --git a/src/autoskillit/hooks/formatters/__init__.py b/src/autoskillit/hooks/formatters/__init__.py new file mode 100644 index 0000000000..a633a33e16 --- /dev/null +++ b/src/autoskillit/hooks/formatters/__init__.py @@ -0,0 +1 @@ +"""PostToolUse formatter scripts — MCP JSON to Markdown-KV reformatter.""" diff --git a/src/autoskillit/hooks/_fmt_execution.py b/src/autoskillit/hooks/formatters/_fmt_execution.py similarity index 72% rename from src/autoskillit/hooks/_fmt_execution.py rename to src/autoskillit/hooks/formatters/_fmt_execution.py index bd29e7ae79..9a0b384705 100644 --- a/src/autoskillit/hooks/_fmt_execution.py +++ b/src/autoskillit/hooks/formatters/_fmt_execution.py @@ -15,7 +15,6 @@ def _format_exit_code_line(data: dict) -> str: """Return the formatted exit_code line, annotated with kill_reason when present. - Legacy payloads without kill_reason render as bare exit_code values so that JSON-line replay of old session logs still parses correctly. """ @@ -35,6 +34,13 @@ def _format_exit_code_line(data: dict) -> str: return f"exit_code: {exit_code}" +def _maybe_provider_line(data: dict, lines: list[str]) -> None: + provider_used = data.get("provider_used", "") + if provider_used and provider_used != "anthropic": + suffix = " [FALLBACK]" if data.get("provider_fallback", False) else "" + lines.append(f"provider: {provider_used}{suffix}") + + def _fmt_run_skill(data: dict, pipeline: bool) -> str: """Format run_skill result as Markdown-KV.""" success = data.get("success", False) @@ -56,6 +62,7 @@ def _fmt_run_skill(data: dict, pipeline: bool) -> str: worktree = data.get("worktree_path", "") if worktree: lines.append(f"worktree_path: {worktree}") + _maybe_provider_line(data, lines) result = data.get("result", "") if result: lines.append(f"\nresult:\n{result}") @@ -79,6 +86,7 @@ def _fmt_run_skill(data: dict, pipeline: bool) -> str: worktree = data.get("worktree_path", "") if worktree: lines.append(f"worktree_path: {worktree}") + _maybe_provider_line(data, lines) token_usage = data.get("token_usage") if isinstance(token_usage, dict): lines.append("") @@ -164,6 +172,24 @@ def _fmt_test_check(data: dict, _pipeline: bool) -> str: mark = _CHECK_MARK if passed else _CROSS_MARK status = "PASS" if passed else "FAIL" lines = [f"## test_check {mark} {status}", "", f"passed: {passed}"] + + duration = data.get("duration_seconds") + if duration is not None: + lines.append(f"duration: {duration:.1f}s") + + filter_mode = data.get("filter_mode") + if filter_mode: + full_run_reason = data.get("full_run_reason") + if full_run_reason: + reason_display = full_run_reason.replace("_", " ") + lines.append(f"filter: {filter_mode} (full run — {reason_display})") + else: + selected = data.get("tests_selected", "?") + deselected = data.get("tests_deselected", "?") + lines.append(f"filter: {filter_mode} ({selected} selected, {deselected} deselected)") + else: + lines.append("filter: off") + stdout = data.get("stdout", "") if stdout: filtered = _filter_pytest_output(stdout) @@ -177,6 +203,54 @@ def _fmt_test_check(data: dict, _pipeline: bool) -> str: return "\n".join(lines) +_FMT_RUN_SKILL_RENDERED: frozenset[str] = frozenset( + { + "success", + "subtype", + "session_id", + "exit_code", + "kill_reason", + "needs_retry", + "retry_reason", + "worktree_path", + "result", + "stderr", + "token_usage", + } +) +_FMT_RUN_SKILL_SUPPRESSED: frozenset[str] = frozenset( + { + "cli_subtype", + "is_error", + "write_path_warnings", + "write_call_count", + "fs_writes_detected", + "last_stop_reason", + "lifespan_started", + "order_id", + "infra_exit_category", + } +) + +_FMT_RUN_CMD_RENDERED: frozenset[str] = frozenset({"success", "exit_code", "stdout", "stderr"}) +_FMT_RUN_CMD_SUPPRESSED: frozenset[str] = frozenset({"error"}) + +_FMT_TEST_CHECK_RENDERED: frozenset[str] = frozenset( + { + "passed", + "duration_seconds", + "filter_mode", + "full_run_reason", + "tests_selected", + "tests_deselected", + "stdout", + "stderr", + "error", + } +) +_FMT_TEST_CHECK_SUPPRESSED: frozenset[str] = frozenset() + + def _fmt_merge_worktree(data: dict, _pipeline: bool) -> str: """Format merge_worktree result as Markdown-KV.""" succeeded = data.get("merge_succeeded") @@ -208,3 +282,29 @@ def _fmt_merge_worktree(data: dict, _pipeline: bool) -> str: if stderr: lines.extend(["", "### stderr", stderr]) return "\n".join(lines) + + +_FMT_MERGE_WORKTREE_RENDERED: frozenset[str] = frozenset( + { + "merge_succeeded", + "merged_branch", + "into_branch", + "worktree_removed", + "branch_deleted", + "cleanup_succeeded", + "error", + "failed_step", + "state", + "worktree_path", + "stderr", + "base_branch", + "dirty_files", + "merge_commits", + "test_stdout", + "test_stderr", + "abort_failed", + "abort_stderr", + "poisoned_installs", + } +) +_FMT_MERGE_WORKTREE_SUPPRESSED: frozenset[str] = frozenset() diff --git a/src/autoskillit/hooks/_fmt_primitives.py b/src/autoskillit/hooks/formatters/_fmt_primitives.py similarity index 91% rename from src/autoskillit/hooks/_fmt_primitives.py rename to src/autoskillit/hooks/formatters/_fmt_primitives.py index 3b46309911..ef0867dcd9 100644 --- a/src/autoskillit/hooks/_fmt_primitives.py +++ b/src/autoskillit/hooks/formatters/_fmt_primitives.py @@ -2,7 +2,7 @@ Stdlib-only at runtime — runs under any Python interpreter without the autoskillit package, so the four ``_fmt_*`` modules and ``pretty_output_hook.py`` -all import directly from this module without going through any L1+ layer. +all import directly from this module without going through any IL-1+ layer. """ from __future__ import annotations @@ -16,12 +16,12 @@ _HOOK_CONFIG_PATH_COMPONENTS = (".autoskillit", "temp", ".hook_config.json") -@dataclass(frozen=True) +@dataclass(frozen=True, slots=True) class _DictPayload: data: dict[str, Any] -@dataclass(frozen=True) +@dataclass(frozen=True, slots=True) class _PlainTextPayload: text: str diff --git a/src/autoskillit/hooks/_fmt_recipe.py b/src/autoskillit/hooks/formatters/_fmt_recipe.py similarity index 80% rename from src/autoskillit/hooks/_fmt_recipe.py rename to src/autoskillit/hooks/formatters/_fmt_recipe.py index 27abd796b9..6d8117057b 100644 --- a/src/autoskillit/hooks/_fmt_recipe.py +++ b/src/autoskillit/hooks/formatters/_fmt_recipe.py @@ -17,7 +17,7 @@ ) if TYPE_CHECKING: - from autoskillit.recipe import ListRecipesResult, LoadRecipeResult + from autoskillit.recipe import ListRecipesResult, LoadRecipeResult, OpenKitchenResult # Field coverage contract for _fmt_load_recipe ↔ LoadRecipeResult @@ -28,6 +28,7 @@ "error", "content", "ingredients_table", + "orchestration_rules", } ) _FMT_LOAD_RECIPE_SUPPRESSED: frozenset[str] = frozenset( @@ -36,6 +37,11 @@ "diagram", # user sees it in terminal preview; agent doesn't need it "kitchen_rules", # already in the YAML content "requires_packs", # internal field; used for skill gating, not display + "requires_features", # internal feature gate enablement field + "content_hash", # internal identity metadata + "composite_hash", # internal identity metadata + "recipe_version", # internal identity metadata + "stop_step_semantics", # delivered via open_kitchen response Channel B; not redisplayed } ) @@ -106,6 +112,9 @@ def _fmt_recipe_body(data: Mapping[str, Any]) -> list[str]: ] if errors: lines.append(f"\n{len(errors)} finding(s)") + orch_rules = data.get("orchestration_rules") + if orch_rules: + lines.append(f"\n{orch_rules}") return lines @@ -146,8 +155,37 @@ def _fmt_load_recipe(data: LoadRecipeResult, pipeline: bool) -> str: ) _FMT_RECIPE_LIST_ITEM_SUPPRESSED: frozenset[str] = frozenset() +# Field coverage contract for _fmt_open_kitchen ↔ OpenKitchenResult +_FMT_OPEN_KITCHEN_RENDERED: frozenset[str] = frozenset( + { + "valid", + "suggestions", + "error", + "content", + "ingredients_table", + "orchestration_rules", + "version", + } +) +_FMT_OPEN_KITCHEN_SUPPRESSED: frozenset[str] = frozenset( + { + "success", # metadata — model infers success from formatted output + "kitchen", # metadata — model knows kitchen state from context + "greeting", # delivered via CLI preview, not MCP response + "diagram", # rendered in terminal preview, not needed by agent + "kitchen_rules", # already embedded in YAML content + "requires_packs", # internal gating field + "requires_features", # internal feature gate enablement field + "content_hash", # internal identity metadata + "composite_hash", # internal identity metadata + "recipe_version", # internal identity metadata + "stop_step_semantics", # delivered via open_kitchen Channel B + "hook_warning", # edge-case diagnostic; not rendered in standard path + } +) + -def _fmt_open_kitchen(data: dict[str, Any], pipeline: bool) -> str: +def _fmt_open_kitchen(data: OpenKitchenResult, pipeline: bool) -> str: """Format open_kitchen combined kitchen+recipe result.""" version = data.get("version", "") diff --git a/src/autoskillit/hooks/_fmt_status.py b/src/autoskillit/hooks/formatters/_fmt_status.py similarity index 63% rename from src/autoskillit/hooks/_fmt_status.py rename to src/autoskillit/hooks/formatters/_fmt_status.py index c3d5006676..605e52d490 100644 --- a/src/autoskillit/hooks/_fmt_status.py +++ b/src/autoskillit/hooks/formatters/_fmt_status.py @@ -18,16 +18,26 @@ def _fmt_get_token_summary(data: dict, _pipeline: bool) -> str: """Format get_token_summary compact Markdown-KV output (JSON payload only).""" lines = ["## token_summary", ""] steps = data.get("steps", []) + has_non_anthropic = False for step in steps: name = step.get("step_name", "?") + model = step.get("model", "") + if model and not model.startswith("claude-"): + name = f"{name}*" + has_non_anthropic = True count = step.get("invocation_count", 1) inp = _fmt_tokens(step.get("input_tokens", 0)) out = _fmt_tokens(step.get("output_tokens", 0)) cache_rd = _fmt_tokens(step.get("cache_read_input_tokens", 0)) + peak_ctx = _fmt_tokens(step.get("peak_context", 0)) cache_wr = _fmt_tokens(step.get("cache_creation_input_tokens", 0)) + turns = step.get("turn_count", 0) wc = step.get("wall_clock_seconds", step.get("elapsed_seconds", 0.0)) + model_tag = f" model:{model}" if model else "" lines.append( - f"{name} x{count} [uc:{inp} out:{out} cr:{cache_rd} cw:{cache_wr} t:{wc:.1f}s]" + f"{name} x{count}" + f" [uc:{inp} out:{out} cr:{cache_rd} pk:{peak_ctx} cw:{cache_wr}" + f" turns:{turns} t:{wc:.1f}s{model_tag}]" ) total = data.get("total", {}) if total: @@ -35,6 +45,7 @@ def _fmt_get_token_summary(data: dict, _pipeline: bool) -> str: lines.append(f"total_uncached: {_fmt_tokens(total.get('input_tokens', 0))}") lines.append(f"total_out: {_fmt_tokens(total.get('output_tokens', 0))}") lines.append(f"total_cache_read: {_fmt_tokens(total.get('cache_read_input_tokens', 0))}") + lines.append(f"total_peak_context: {_fmt_tokens(total.get('peak_context', 0))}") lines.append( f"total_cache_write: {_fmt_tokens(total.get('cache_creation_input_tokens', 0))}" ) @@ -45,6 +56,18 @@ def _fmt_get_token_summary(data: dict, _pipeline: bool) -> str: lines.append(f"mcp_invocations: {mcp_total.get('total_invocations', 0)}") est_tokens = mcp_total.get("total_estimated_response_tokens", 0) lines.append(f"mcp_response_tokens: ~{_fmt_tokens(est_tokens)}") + if has_non_anthropic: + lines.append("") + lines.append("* Step used a non-Anthropic provider; caching behavior may differ.") + model_totals = data.get("model_totals", []) + if model_totals: + lines.append("") + for m in model_totals: + model_name = m.get("model", "unknown") + steps = m.get("step_count", 0) + inp = _fmt_tokens(m.get("input_tokens", 0)) + out = _fmt_tokens(m.get("output_tokens", 0)) + lines.append(f"model:{model_name} steps:{steps} [uncached:{inp} out:{out}]") return "\n".join(lines) @@ -132,3 +155,53 @@ def _fmt_clone_repo(data: dict, _pipeline: bool) -> str: continue lines.append(f"{key}: {val}") return "\n".join(lines) + + +_FMT_TOKEN_SUMMARY_RENDERED: frozenset[str] = frozenset( + {"steps", "total", "mcp_responses", "model_totals"} +) +_FMT_TOKEN_SUMMARY_SUPPRESSED: frozenset[str] = frozenset( + { + "success", # only emitted from exception guard paths, not accessed by the formatter + "error", # only emitted from exception guard paths, not accessed by the formatter + } +) + +_FMT_TIMING_SUMMARY_RENDERED: frozenset[str] = frozenset({"steps", "total"}) +_FMT_TIMING_SUMMARY_SUPPRESSED: frozenset[str] = frozenset( + { + "success", # only emitted from exception guard paths, not accessed by the formatter + "error", # only emitted from exception guard paths, not accessed by the formatter + } +) + +_FMT_KITCHEN_STATUS_RENDERED: frozenset[str] = frozenset( + { + "error", + "tools_enabled", + "package_version", + "plugin_json_version", + "versions_match", + "token_usage_verbosity", + "quota_guard_enabled", + "github_token_configured", + "github_default_repo", + "warning", + } +) +_FMT_KITCHEN_STATUS_SUPPRESSED: frozenset[str] = frozenset( + { + "success", # only emitted from exception guard paths, not accessed by the formatter + } +) + +_FMT_CLONE_REPO_RENDERED: frozenset[str] = frozenset( + { + "clone_path", + "source_dir", + "remote_url", + "clone_source_type", + "clone_source_reason", + } +) +_FMT_CLONE_REPO_SUPPRESSED: frozenset[str] = frozenset() diff --git a/src/autoskillit/hooks/pretty_output_hook.py b/src/autoskillit/hooks/formatters/pretty_output_hook.py similarity index 91% rename from src/autoskillit/hooks/pretty_output_hook.py rename to src/autoskillit/hooks/formatters/pretty_output_hook.py index 6a88b2675c..9b7b83131a 100644 --- a/src/autoskillit/hooks/pretty_output_hook.py +++ b/src/autoskillit/hooks/formatters/pretty_output_hook.py @@ -39,7 +39,15 @@ if _HOOKS_DIR not in sys.path: sys.path.insert(0, _HOOKS_DIR) -from _fmt_execution import ( # type: ignore[import-not-found] # noqa: E402 +from _fmt_execution import ( # type: ignore[import-not-found] # noqa: E402, F401 + _FMT_MERGE_WORKTREE_RENDERED, + _FMT_MERGE_WORKTREE_SUPPRESSED, + _FMT_RUN_CMD_RENDERED, + _FMT_RUN_CMD_SUPPRESSED, + _FMT_RUN_SKILL_RENDERED, + _FMT_RUN_SKILL_SUPPRESSED, + _FMT_TEST_CHECK_RENDERED, + _FMT_TEST_CHECK_SUPPRESSED, _fmt_merge_worktree, _fmt_run_cmd, _fmt_run_skill, @@ -60,6 +68,8 @@ _FMT_LIST_RECIPES_SUPPRESSED, _FMT_LOAD_RECIPE_RENDERED, _FMT_LOAD_RECIPE_SUPPRESSED, + _FMT_OPEN_KITCHEN_RENDERED, + _FMT_OPEN_KITCHEN_SUPPRESSED, _FMT_RECIPE_LIST_ITEM_RENDERED, _FMT_RECIPE_LIST_ITEM_SUPPRESSED, _LOAD_RECIPE_CONTENT_DERIVED_FROM, @@ -70,7 +80,15 @@ _fmt_recipe_body, _strip_yaml_ingredients_block, ) -from _fmt_status import ( # type: ignore[import-not-found] # noqa: E402 +from _fmt_status import ( # type: ignore[import-not-found] # noqa: E402, F401 + _FMT_CLONE_REPO_RENDERED, + _FMT_CLONE_REPO_SUPPRESSED, + _FMT_KITCHEN_STATUS_RENDERED, + _FMT_KITCHEN_STATUS_SUPPRESSED, + _FMT_TIMING_SUMMARY_RENDERED, + _FMT_TIMING_SUMMARY_SUPPRESSED, + _FMT_TOKEN_SUMMARY_RENDERED, + _FMT_TOKEN_SUMMARY_SUPPRESSED, _fmt_clone_repo, _fmt_get_timing_summary, _fmt_get_token_summary, @@ -176,6 +194,7 @@ def _fmt_generic(short_name: str, data: dict, _pipeline: bool) -> str: "wait_for_merge_queue", # merge queue result dict, generic renders correctly "check_repo_merge_state", # simple JSON boolean bundle, generic renders correctly "toggle_auto_merge", # simple success/error result dict, generic renders correctly + "enqueue_pr", # simple success/error result dict, generic renders correctly "create_unique_branch", # simple result "write_telemetry_files", # simple path results "get_pr_reviews", # list of reviews @@ -192,6 +211,13 @@ def _fmt_generic(short_name: str, data: dict, _pipeline: bool) -> str: "disable_quota_guard", # simple success/error result "register_clone_status", # simple registered/error result "batch_cleanup_clones", # bulk delete summary dict + "dispatch_food_truck", # JSON dispatch envelope, generic renders correctly + "reload_session", # simple status dict, generic renders correctly + "analyze_tool_sequences", # DFG analysis result, generic renders correctly + "record_gate_dispatch", # simple success/error result + "bootstrap_clone", # composite clone result dict + "claim_and_resolve_issue", # composite claim result dict + "create_and_publish_branch", # composite branch result dict } ) diff --git a/src/autoskillit/hooks/guards/CLAUDE.md b/src/autoskillit/hooks/guards/CLAUDE.md new file mode 100644 index 0000000000..b56b57484c --- /dev/null +++ b/src/autoskillit/hooks/guards/CLAUDE.md @@ -0,0 +1,64 @@ +# guards/ + +PreToolUse guard scripts — standalone Python processes enforcing tool-call policies. + +## Files + +| File | Purpose | +|------|---------| +| `__init__.py` | Package marker (no imports) | +| `ask_user_question_guard.py` | Blocks `AskUserQuestion` before kitchen is open | +| `branch_protection_guard.py` | Blocks merge/push targeting protected branches | +| `fleet_dispatch_guard.py` | Blocks `dispatch_food_truck` from headless sessions (prevents L3->L3 recursion) | +| `generated_file_write_guard.py` | Blocks Write/Edit to machine-generated files (`hooks.json`, `settings.json`) | +| `grep_pattern_lint_guard.py` | Blocks Grep with BRE `\|` syntax; surfaces corrected ERE pattern | +| `skill_orchestration_guard.py` | Blocks `run_skill`/`run_cmd`/`run_python` from L1 skill sessions | +| `mcp_health_advisor.py` | Detects MCP server disconnection (dead PID); non-blocking advisory | +| `open_kitchen_guard.py` | Blocks `open_kitchen` from headless sessions; writes kitchen marker | +| `planner_gh_discovery_guard.py` | Blocks GitHub issue/PR listing in planner sessions | +| `pr_create_guard.py` | Blocks `gh pr create` via `run_cmd` when kitchen is open | +| `quota_guard.py` | Blocks `run_skill` when quota threshold exceeded; fails open on missing cache | +| `recipe_write_advisor.py` | Non-blocking advisory for recipe YAML writes | +| `remove_clone_guard.py` | Blocks `remove_clone` if branch has unpushed commits | +| `review_loop_gate.py` | Blocks `wait_for_ci`/`enqueue_pr` until `check_review_loop` is called | +| `resume_ownership_guard.py` | Validates `resume_session_id` ownership at resume time; blocks unowned or L3 session resume | +| `skill_cmd_guard.py` | Validates `skill_command` path argument format | +| `skill_command_guard.py` | Blocks `run_skill` with non-slash `skill_command` | +| `unsafe_install_guard.py` | Blocks `pip install -e` targeting system Python | +| `skill_load_guard.py` | Denies native tools until Skill tool is called in non-Anthropic headless skill sessions; exempts subagents via `agent_id` | +| `write_guard.py` | Blocks Write/Edit outside allowed prefix in read-only sessions | + +## Architecture Notes + +Each guard is a standalone Python script executed as a subprocess (not imported as a module). Protocol: read PreToolUse JSON from stdin, write decision JSON to stdout, exit 0. Most are stdlib-only for fast startup. + +### Fail-Mode Contract + +All guards fail-**open** for malformed/unparseable input (JSON decode failure = exit 0 = approve). +This prevents a broken hook from blocking the entire tool chain. + +Three guards additionally fail-**closed** for valid input with unrecognized values, as a +defense-in-depth measure against privilege escalation: + +| Guard | Fail-closed condition | Rationale | +|-------|----------------------|-----------| +| `skill_command_guard.py` | Unexpected runtime error (not JSON parse) | Unknown failure mode = deny rather than risk executing an unvalidated command | +| `open_kitchen_guard.py` | Unrecognized `AUTOSKILLIT_SESSION_TYPE` | Unknown session type should not gain kitchen access | +| `skill_orchestration_guard.py` | Unrecognized `AUTOSKILLIT_SESSION_TYPE` | Unknown session type should not call orchestration tools (`run_skill`, `run_cmd`, `run_python`) | + +**Design principle:** Garbage-in (malformed hook input) = fail-open. Unknown-tier (valid input, unrecognized value) = fail-closed. + +## Hook Payload Fields for Guard Development + +### `agent_id` — Subagent Detection + +Claude Code includes `agent_id` in the hook JSON payload (stdin) when the hook fires +inside a subagent (L0). The field is absent in top-level sessions. This is the standard +mechanism for detecting subagent context in hook scripts. + +- `agent_id` — present ONLY in subagent context; absent at top-level +- `agent_type` — also present in subagent context (e.g., `"Explore"`, `"general-purpose"`) +- Subagents cannot spawn nested subagents, so the check is binary: top-level vs. one-level-deep +- Check: `if data.get("agent_id"): sys.exit(0)` for unconditional subagent exemption + +Used by: `skill_load_guard.py`, `skill_load_post_hook.py` diff --git a/src/autoskillit/hooks/guards/__init__.py b/src/autoskillit/hooks/guards/__init__.py new file mode 100644 index 0000000000..6d6c0a8824 --- /dev/null +++ b/src/autoskillit/hooks/guards/__init__.py @@ -0,0 +1 @@ +"""PreToolUse guard scripts — invoked as standalone python3 commands.""" diff --git a/src/autoskillit/hooks/ask_user_question_guard.py b/src/autoskillit/hooks/guards/ask_user_question_guard.py similarity index 90% rename from src/autoskillit/hooks/ask_user_question_guard.py rename to src/autoskillit/hooks/guards/ask_user_question_guard.py index fc1b52ce87..577c6be8dc 100644 --- a/src/autoskillit/hooks/ask_user_question_guard.py +++ b/src/autoskillit/hooks/guards/ask_user_question_guard.py @@ -16,12 +16,17 @@ from datetime import UTC from pathlib import Path +ASK_USER_QUESTION_DENY_TRIGGER: str = "AskUserQuestion is not available in headless sessions" + def _get_marker_path(session_id: str) -> Path: override = os.environ.get("AUTOSKILLIT_STATE_DIR") if override: return Path(override) / "kitchen_state" / f"{session_id}.json" - return Path.cwd() / ".autoskillit" / "temp" / "kitchen_state" / f"{session_id}.json" + campaign_id = os.environ.get("AUTOSKILLIT_CAMPAIGN_ID", "") + base = Path.cwd() / ".autoskillit" / "temp" / "kitchen_state" + state_dir = base / campaign_id if campaign_id else base + return state_dir / f"{session_id}.json" def _read_marker(session_id: str) -> dict | None: diff --git a/src/autoskillit/hooks/branch_protection_guard.py b/src/autoskillit/hooks/guards/branch_protection_guard.py similarity index 91% rename from src/autoskillit/hooks/branch_protection_guard.py rename to src/autoskillit/hooks/guards/branch_protection_guard.py index 70e9501018..8aeb318bd8 100644 --- a/src/autoskillit/hooks/branch_protection_guard.py +++ b/src/autoskillit/hooks/guards/branch_protection_guard.py @@ -2,7 +2,7 @@ """PreToolUse hook: branch protection guard for merge_worktree and push_to_remote. Reads AUTOSKILLIT_PROTECTED_BRANCHES env var (comma-separated, default: -main,integration,stable). Denies tool calls that target a protected branch. +main,develop,stable). Denies tool calls that target a protected branch. Matched tools: - merge_worktree: checks base_branch parameter @@ -13,6 +13,8 @@ import os import sys +BRANCH_PROTECTION_DENY_TRIGGER: str = "Branch '" + def main() -> None: raw = sys.stdin.read().strip() @@ -39,7 +41,7 @@ def main() -> None: return # Read protected branches from env (comma-separated) - env_val = os.environ.get("AUTOSKILLIT_PROTECTED_BRANCHES", "main,integration,stable") + env_val = os.environ.get("AUTOSKILLIT_PROTECTED_BRANCHES", "main,develop,stable") protected = [b.strip() for b in env_val.split(",") if b.strip()] if branch in protected: diff --git a/src/autoskillit/hooks/guards/fleet_dispatch_guard.py b/src/autoskillit/hooks/guards/fleet_dispatch_guard.py new file mode 100755 index 0000000000..73a0527a29 --- /dev/null +++ b/src/autoskillit/hooks/guards/fleet_dispatch_guard.py @@ -0,0 +1,76 @@ +#!/usr/bin/env python3 +"""PreToolUse hook — blocks dispatch_food_truck from headless callers. + +Defense-in-depth: dispatch_food_truck must never be called from a headless +session regardless of SESSION_TYPE. This closes L3→L3 recursion where a +fleet session spawns another fleet session via dispatch. + +Interactive callers (cook with kitchen open) are always permitted. +""" + +import json +import os +import sys + +FLEET_DISPATCH_DENY_TRIGGER: str = "dispatch_food_truck cannot be called from headless sessions" + + +def main() -> None: + try: + data = json.loads(sys.stdin.read()) + except (json.JSONDecodeError, ValueError, OSError): + sys.stderr.write("fleet_dispatch_guard: malformed stdin — failing open\n") + sys.exit(0) # fail-open on malformed input + + if not isinstance(data, dict): + sys.stderr.write("fleet_dispatch_guard: unexpected JSON root type — failing open\n") + sys.exit(0) + + # AUTOSKILLIT_HEADLESS is the sole discriminator. Hook payload cross-check + # (session_type field) would add defence-in-depth against local env + # manipulation, but the attack surface is narrowly local and the env var + # is set by our own launcher — not user-supplied input. + if os.environ.get("AUTOSKILLIT_HEADLESS") != "1": + sys.exit(0) + + session_type = os.environ.get("AUTOSKILLIT_SESSION_TYPE", "") + if session_type and session_type != "fleet": + payload = json.dumps( + { + "hookSpecificOutput": { + "hookEventName": "PreToolUse", + "permissionDecision": "deny", + "permissionDecisionReason": ( + f"dispatch_food_truck requires fleet session (current: {session_type})" + ), + } + } + ) + sys.stdout.write(payload + "\n") + sys.exit(0) + + # Headless: check if this is dispatch_food_truck + tool_name: str = data.get("tool_name", "") + tool = tool_name.split("__")[-1] + if tool == "dispatch_food_truck": + payload = json.dumps( + { + "hookSpecificOutput": { + "hookEventName": "PreToolUse", + "permissionDecision": "deny", + "permissionDecisionReason": ( + "dispatch_food_truck cannot be called from headless sessions. " + "This tool is only available to interactive callers (cook). " + "Headless dispatch would create recursive L2 (food truck) sessions." + ), + } + } + ) + sys.stdout.write(payload + "\n") + sys.exit(0) + + sys.exit(0) + + +if __name__ == "__main__": + main() diff --git a/src/autoskillit/hooks/generated_file_write_guard.py b/src/autoskillit/hooks/guards/generated_file_write_guard.py similarity index 96% rename from src/autoskillit/hooks/generated_file_write_guard.py rename to src/autoskillit/hooks/guards/generated_file_write_guard.py index a38d508d1a..ad08b02ed1 100644 --- a/src/autoskillit/hooks/generated_file_write_guard.py +++ b/src/autoskillit/hooks/guards/generated_file_write_guard.py @@ -10,6 +10,8 @@ import os import sys +GENERATED_FILE_DENY_TRIGGER: str = "is a generated file" + _GENERATED_FILE_SUFFIXES = ("/hooks/hooks.json", ".claude/settings.json") diff --git a/src/autoskillit/hooks/guards/grep_pattern_lint_guard.py b/src/autoskillit/hooks/guards/grep_pattern_lint_guard.py new file mode 100755 index 0000000000..0b8dc04c47 --- /dev/null +++ b/src/autoskillit/hooks/guards/grep_pattern_lint_guard.py @@ -0,0 +1,58 @@ +#!/usr/bin/env python3 +"""PreToolUse hook — denies Grep calls with BRE \\| alternation syntax. + +The Grep tool wraps ripgrep which uses ERE/PCRE syntax where `|` is +alternation. Models trained on Bash data emit `\\|` (POSIX BRE), which +ripgrep treats as a literal backslash-pipe, silently returning 0 results. + +This guard intercepts the pattern before ripgrep sees it, denies the call, +and surfaces the corrected pattern so the model can retry immediately. + +Stdlib-only — runs under any Python interpreter without the autoskillit package. +""" + +from __future__ import annotations + +import json +import sys + +# Sentinel: backslash followed by pipe — BRE alternation, invalid in ripgrep ERE +_BRE_ALTERNATION = "\\|" + + +def main() -> None: + try: + data = json.loads(sys.stdin.read()) + except (json.JSONDecodeError, ValueError, OSError): + sys.exit(0) # fail-open on malformed input + + tool_name = data.get("tool_name", "") + if tool_name != "Grep": + sys.exit(0) # guard is Grep-specific + + pattern = (data.get("tool_input") or {}).get("pattern", None) + if not isinstance(pattern, str) or _BRE_ALTERNATION not in pattern: + sys.exit(0) # no BRE alternation present — allow + + corrected = pattern.replace(_BRE_ALTERNATION, "|") + payload = json.dumps( + { + "hookSpecificOutput": { + "hookEventName": "PreToolUse", + "permissionDecision": "deny", + "permissionDecisionReason": ( + "The Grep tool uses ripgrep (ERE/PCRE) syntax where `|` is alternation. " + "`\\|` is not alternation in ripgrep — it matches a literal backslash-pipe " + "character, which is extremely rare in source code. " + f"Corrected pattern: `{corrected}` " + "(If you genuinely need to match a literal backslash-pipe, use `\\\\|`.)" + ), + } + } + ) + sys.stdout.write(payload + "\n") + sys.exit(0) + + +if __name__ == "__main__": + main() diff --git a/src/autoskillit/hooks/guards/mcp_health_advisor.py b/src/autoskillit/hooks/guards/mcp_health_advisor.py new file mode 100644 index 0000000000..143c42c9ee --- /dev/null +++ b/src/autoskillit/hooks/guards/mcp_health_advisor.py @@ -0,0 +1,97 @@ +#!/usr/bin/env python3 +"""PreToolUse hook — detects MCP server disconnection and suggests /MCP reconnect. + +Reads ~/.autoskillit/active_kitchens.json to find kitchen entries for the current +project. If entries exist but all PIDs are dead, injects an informational message +via hookSpecificOutput.message. Never blocks tool execution. + +SIGHUP treated as shutdown: when a terminal disconnect kills the MCP server, +active_kitchens.json still holds the dead PID. This hook surfaces that state to +the user on the next native tool call, prompting reconnection before the session +tries to use kitchen tools that no longer exist. + +Stdlib-only — runs under any Python interpreter without the autoskillit package. +""" + +from __future__ import annotations + +import json +import os +import sys +from pathlib import Path + + +def _active_kitchens_path() -> Path: + return Path.home() / ".autoskillit" / "active_kitchens.json" + + +def _read_kitchens() -> list[dict]: + """Read and return kitchen entries from active_kitchens.json.""" + try: + data = json.loads(_active_kitchens_path().read_text(encoding="utf-8")) + entries = data.get("kitchens", []) + return entries if isinstance(entries, list) else [] + except (OSError, json.JSONDecodeError, AttributeError): + return [] + + +def _pid_alive(pid: int) -> bool: + """Check if a PID is still running (stdlib-only, no create_time validation).""" + try: + os.kill(pid, 0) + return True + except ProcessLookupError: + return False + except (PermissionError, OSError): + # PermissionError means the process exists but we lack permission to signal it. + return True + + +def main() -> None: + # Validate stdin is well-formed JSON; result intentionally discarded — this + # hook fires on every tool regardless of tool_name, so the payload content + # is not used. Fail-open: malformed stdin exits cleanly rather than raising. + try: + _ = json.loads(sys.stdin.read()) + except (json.JSONDecodeError, ValueError, OSError): + sys.exit(0) + + # Never fire in headless sessions — the orchestrator handles reconnection itself. + if os.environ.get("AUTOSKILLIT_HEADLESS") == "1": + sys.exit(0) + + cwd = os.getcwd() + entries = _read_kitchens() + + # Filter to entries matching the current project + matching = [e for e in entries if isinstance(e, dict) and e.get("project_path") == cwd] + + if not matching: + sys.exit(0) # No kitchen registered for this project + + # Check if ANY matching PID is still alive + for entry in matching: + pid = entry.get("pid") + if type(pid) is int and _pid_alive(pid): + sys.exit(0) # Server is alive — no disconnect + + # All matching PIDs are dead — server disconnected + payload = json.dumps( + { + "hookSpecificOutput": { + "hookEventName": "PreToolUse", + "message": ( + "AutoSkillit MCP server appears disconnected — all registered " + "server PIDs for this project are dead. Kitchen state has been " + "lost. Ask the user to run /MCP to reconnect, then re-open " + "the kitchen with open_kitchen." + ), + } + } + ) + sys.stdout.write(payload + "\n") + sys.exit(0) + + +if __name__ == "__main__": + main() diff --git a/src/autoskillit/hooks/open_kitchen_guard.py b/src/autoskillit/hooks/guards/open_kitchen_guard.py similarity index 51% rename from src/autoskillit/hooks/open_kitchen_guard.py rename to src/autoskillit/hooks/guards/open_kitchen_guard.py index 1021693f25..e133739ac0 100644 --- a/src/autoskillit/hooks/open_kitchen_guard.py +++ b/src/autoskillit/hooks/guards/open_kitchen_guard.py @@ -14,6 +14,8 @@ import sys from datetime import UTC +OPEN_KITCHEN_DENY_TRIGGER: str = "open_kitchen cannot be called" + def _write_kitchen_marker(session_id: str, recipe_name: str | None) -> None: """Write the kitchen-open session marker (stdlib-only, inline implementation).""" @@ -25,7 +27,9 @@ def _write_kitchen_marker(session_id: str, recipe_name: str | None) -> None: if state_override: state_dir = _Path(state_override) / "kitchen_state" else: - state_dir = _Path.cwd() / ".autoskillit" / "temp" / "kitchen_state" + campaign_id = os.environ.get("AUTOSKILLIT_CAMPAIGN_ID", "") + base = _Path.cwd() / ".autoskillit" / "temp" / "kitchen_state" + state_dir = base / campaign_id if campaign_id else base state_dir.mkdir(parents=True, exist_ok=True) marker_path = state_dir / f"{session_id}.json" payload = json.dumps( @@ -34,6 +38,8 @@ def _write_kitchen_marker(session_id: str, recipe_name: str | None) -> None: "opened_at": datetime.now(UTC).isoformat(), "recipe_name": recipe_name, "marker_version": 1, + "content_hash": "", + "composite_hash": "", } ) fd, tmp = tempfile.mkstemp(dir=state_dir, suffix=".tmp") @@ -49,6 +55,41 @@ def _write_kitchen_marker(session_id: str, recipe_name: str | None) -> None: raise +def _bridge_session_registry(session_id: str) -> None: + """Bridge AUTOSKILLIT_LAUNCH_ID to claude_session_id in the session registry.""" + import tempfile + from pathlib import Path as _Path + + launch_id = os.environ.get("AUTOSKILLIT_LAUNCH_ID", "") + if not launch_id or not session_id: + return + + registry_file = _Path.cwd() / ".autoskillit" / "temp" / "session_registry.json" + try: + registry: dict = json.loads(registry_file.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError): + return + + if launch_id not in registry: + return + + registry[launch_id]["claude_session_id"] = session_id + + registry_file.parent.mkdir(parents=True, exist_ok=True) + content = json.dumps(registry) + fd, tmp = tempfile.mkstemp(dir=registry_file.parent, suffix=".tmp") + try: + with os.fdopen(fd, "w", encoding="utf-8") as f: + f.write(content) + os.replace(tmp, registry_file) + except Exception: + try: + os.unlink(tmp) + except OSError: + pass + raise + + def main() -> None: try: data = json.loads(sys.stdin.read()) @@ -56,20 +97,42 @@ def main() -> None: sys.exit(0) # fail-open on malformed input or broken pipe if os.environ.get("AUTOSKILLIT_HEADLESS") == "1": - payload = json.dumps( - { - "hookSpecificOutput": { - "hookEventName": "PreToolUse", - "permissionDecision": "deny", - "permissionDecisionReason": ( - "open_kitchen cannot be called from headless sessions. " - "Open the kitchen in your human session using /autoskillit:open-kitchen." - ), + session_type = os.environ.get("AUTOSKILLIT_SESSION_TYPE", "").lower() + + if session_type in ("fleet",): + payload = json.dumps( + { + "hookSpecificOutput": { + "hookEventName": "PreToolUse", + "permissionDecision": "deny", + "permissionDecisionReason": ( + f"open_kitchen cannot be called from {session_type!r} sessions." + ), + } } - } - ) - sys.stdout.write(payload + "\n") - sys.exit(0) + ) + sys.stdout.write(payload + "\n") + sys.exit(0) + + if session_type not in ("orchestrator",): + # skill session, unset, or invalid — deny (fail-closed) + payload = json.dumps( + { + "hookSpecificOutput": { + "hookEventName": "PreToolUse", + "permissionDecision": "deny", + "permissionDecisionReason": ( + "open_kitchen cannot be called from skill sessions. " + "Open the kitchen in your orchestrator session using " + "/autoskillit:open-kitchen." + ), + } + } + ) + sys.stdout.write(payload + "\n") + sys.exit(0) + + # HEADLESS + orchestrator — fall through to permit path # Permit path: write a kitchen-open session marker so ask_user_question_guard # can verify the kitchen is open before allowing AskUserQuestion. @@ -84,6 +147,13 @@ def main() -> None: recipe_name = tool_input.get("name") or None if session_id: _write_kitchen_marker(session_id, recipe_name) + try: + _bridge_session_registry(session_id) + except Exception as _bridge_err: + print( + f"[open_kitchen_guard] registry bridge failed: {_bridge_err}", + file=sys.stderr, + ) except Exception as e: print(f"[open_kitchen_guard] marker write failed: {e}", file=sys.stderr) # Surface the failure so the user knows AskUserQuestion will be blocked diff --git a/src/autoskillit/hooks/guards/planner_gh_discovery_guard.py b/src/autoskillit/hooks/guards/planner_gh_discovery_guard.py new file mode 100644 index 0000000000..16f8b00aee --- /dev/null +++ b/src/autoskillit/hooks/guards/planner_gh_discovery_guard.py @@ -0,0 +1,114 @@ +#!/usr/bin/env python3 +"""PreToolUse hook: block GitHub discovery commands in planner skill sessions. + +Planner sessions must not discover GitHub issues/PRs via listing or search +commands. If a planner skill needs a specific issue, it must be provided as +skill input. + +stdlib-only; no autoskillit imports. +""" + +from __future__ import annotations + +import json +import os +import re +import shlex +import sys + +DISCOVERY_DENY_TRIGGER: str = "Planner skills cannot discover GitHub issues" + +_SHELL_OPS = frozenset({"&&", "||", ";", "!", "|", "("}) + +_DENY_REASON = ( + "Planner skills cannot discover GitHub issues or PRs. " + "If you need to read a specific issue, it must be provided as a skill input." +) + +_DISCOVERY_SUBCOMMANDS: frozenset[tuple[str, str]] = frozenset( + { + ("issue", "list"), + ("pr", "list"), + ("search", "issues"), + ("search", "prs"), + } +) + +_TARGETED_SUBCOMMANDS: frozenset[tuple[str, str]] = frozenset( + { + ("issue", "view"), + ("pr", "view"), + } +) + +_API_LISTING_RE = re.compile(r"/repos/[^/]+/[^/]+/(issues|pulls)(?:\?.*)?$") + +_API_SPECIFIC_RE = re.compile(r"/repos/[^/]+/[^/]+/(issues|pulls)/\d+") + + +def _is_gh_discovery(cmd: str) -> bool: + """Return True when *cmd* contains a GitHub discovery subcommand. + + Targeted reads (``gh issue view ``, ``gh api .../issues/``) are + explicitly allowed. Tokenises with shlex to avoid false positives from + quoted arguments. + """ + try: + tokens = shlex.split(cmd) + except ValueError: + return False + for i, token in enumerate(tokens): + if token != "gh" or i + 2 >= len(tokens): + continue + if i != 0 and tokens[i - 1] not in _SHELL_OPS: + continue + pair = (tokens[i + 1], tokens[i + 2]) + if pair in _DISCOVERY_SUBCOMMANDS: + return True + if pair in _TARGETED_SUBCOMMANDS: + return False + if tokens[i + 1] == "api": + endpoint = tokens[i + 2] + if _API_SPECIFIC_RE.search(endpoint): + return False + if _API_LISTING_RE.search(endpoint): + return True + return False + + +def main() -> None: + try: + data = json.loads(sys.stdin.read()) + except (json.JSONDecodeError, AttributeError, OSError): + sys.exit(0) + + if os.environ.get("AUTOSKILLIT_HEADLESS") != "1": + sys.exit(0) + + skill_name = os.environ.get("AUTOSKILLIT_SKILL_NAME", "") + if not skill_name.startswith("planner-"): + sys.exit(0) + + tool_input = data.get("tool_input", {}) + if not isinstance(tool_input, dict): + sys.exit(0) + cmd = tool_input.get("command", "") or tool_input.get("cmd", "") + + if not _is_gh_discovery(cmd): + sys.exit(0) + + payload = json.dumps( + { + "hookSpecificOutput": { + "hookEventName": "PreToolUse", + "permissionDecision": "deny", + "permissionDecisionReason": _DENY_REASON, + } + } + ) + sys.stdout.write(payload + "\n") + sys.exit(0) + + +if __name__ == "__main__": + main() diff --git a/src/autoskillit/hooks/guards/pr_create_guard.py b/src/autoskillit/hooks/guards/pr_create_guard.py new file mode 100644 index 0000000000..c591f29dc9 --- /dev/null +++ b/src/autoskillit/hooks/guards/pr_create_guard.py @@ -0,0 +1,85 @@ +#!/usr/bin/env python3 +"""PreToolUse hook: block gh pr create via run_cmd or Bash tool when kitchen is open. + +Enforces the mandatory prepare_pr → compose_pr pipeline. Direct gh pr create +calls bypass mandatory arch-lens, diff annotation, and automated code review. + +stdlib-only; no autoskillit imports. +""" + +from __future__ import annotations + +import json +import shlex +import sys +from pathlib import Path + +PR_CREATE_DENY_TRIGGER: str = "PR creation via run_cmd is prohibited" + +# Shell-separator tokens that introduce a new subcommand. +_SHELL_OPS = frozenset({"&&", "||", ";", "!", "|", "("}) + +_DENY_REASON = ( + "PR creation via run_cmd is prohibited during recipe execution. " + "Use the prepare_pr → compose_pr pipeline instead. " + "Direct gh pr create bypasses mandatory arch-lens, annotation, and review steps." +) + + +def _is_gh_pr_create(cmd: str) -> bool: + """Return True only when `gh pr create` appears as an actual subcommand. + + Tokenises with shlex to avoid false positives from quoted arguments + (e.g. ``echo 'do not gh pr create'`` must not match). A `gh` token is + considered a subcommand start when it is at position 0 or immediately + follows a shell separator token (&&, ||, ;, |, !, (). + """ + try: + tokens = shlex.split(cmd) + except ValueError: + # Unclosed quotes — shlex cannot parse; fail-open (no block). + return False + for i, token in enumerate(tokens): + if token == "gh" and i + 2 < len(tokens): + if tokens[i + 1] == "pr" and tokens[i + 2] == "create": + if i == 0 or tokens[i - 1] in _SHELL_OPS: + return True + return False + + +def main() -> None: + try: + data = json.loads(sys.stdin.read()) + tool_input = data.get("tool_input", {}) + cmd = tool_input.get("command", "") or tool_input.get("cmd", "") + except (json.JSONDecodeError, AttributeError, OSError): + sys.exit(0) + + if not _is_gh_pr_create(cmd): + sys.exit(0) + # Hook config file is written by open_kitchen and removed by close_kitchen. + # Its presence reliably signals an open kitchen without needing session ID. + try: + # Keep in sync with HOOK_DIR_COMPONENTS/HOOK_CONFIG_FILENAME in _hook_settings.py + # (stdlib-only boundary prevents a shared import). + cfg_path = Path.cwd() / ".autoskillit" / "temp" / ".hook_config.json" + if not cfg_path.exists(): + sys.exit(0) # kitchen not open; fail-open + except OSError: + sys.exit(0) + # Kitchen is open and command matches: deny + payload = json.dumps( + { + "hookSpecificOutput": { + "hookEventName": "PreToolUse", + "permissionDecision": "deny", + "permissionDecisionReason": _DENY_REASON, + } + } + ) + sys.stdout.write(payload + "\n") + sys.exit(0) + + +if __name__ == "__main__": + main() diff --git a/src/autoskillit/hooks/quota_guard.py b/src/autoskillit/hooks/guards/quota_guard.py similarity index 62% rename from src/autoskillit/hooks/quota_guard.py rename to src/autoskillit/hooks/guards/quota_guard.py index cbe64349da..503bb38ea1 100644 --- a/src/autoskillit/hooks/quota_guard.py +++ b/src/autoskillit/hooks/guards/quota_guard.py @@ -15,6 +15,7 @@ import json import os import sys +import time from datetime import UTC, datetime from pathlib import Path @@ -23,7 +24,7 @@ # Placing the script's directory first on sys.path lets the bare-name import # below resolve to the shared stdlib-only settings module in both subprocess # and package-mode invocations. -_HOOKS_DIR = str(Path(__file__).resolve().parent) +_HOOKS_DIR = str(Path(__file__).resolve().parent.parent) if _HOOKS_DIR not in sys.path: sys.path.insert(0, _HOOKS_DIR) @@ -31,6 +32,14 @@ _AUTOSKILLIT_LOG_DIR_ENV = "AUTOSKILLIT_LOG_DIR" +# Emitted in deny messages; also referenced by orchestrator prompt QUOTA DENIAL ROUTING. +# Changing this value requires updating _prompts.py and sous-chef/SKILL.md in the same commit. +QUOTA_GUARD_DENY_TRIGGER: str = "QUOTA WAIT REQUIRED" + +# Emitted when required sleep exceeds remaining session wall-clock budget. +# Instructs the session to emit a clean sentinel and exit, rather than sleep-and-retry. +QUOTA_BUDGET_EXCEEDED_TRIGGER: str = "QUOTA BUDGET EXCEEDED" + def _read_quota_cache(cache_path_str: str, max_age: int) -> dict | None: """Read quota cache file. Returns parsed data or None if missing/stale/corrupt.""" @@ -107,6 +116,19 @@ def main(*, cache_path_override: str | None = None) -> None: log_dir = _resolve_quota_log_dir() ts = datetime.now(UTC).isoformat() + profile = os.environ.get("AUTOSKILLIT_PROVIDER_PROFILE", "").strip() + if profile and profile.casefold() != "anthropic": + _write_quota_log_event( + { + "ts": ts, + "event": "provider_bypass", + "profile": profile, + "cache_path": cache_path_str, + }, + log_dir, + ) + sys.exit(0) + cache = _read_quota_cache(cache_path_str, cache_max_age) if cache is None: _write_quota_log_event( @@ -153,39 +175,76 @@ def main(*, cache_path_override: str | None = None) -> None: else: n = settings.buffer_seconds + session_deadline_str = os.environ.get("AUTOSKILLIT_SESSION_DEADLINE") + budget_exceeded = False + if session_deadline_str: + try: + session_deadline = float(session_deadline_str) + remaining_budget = max(0, session_deadline - time.time()) + if n > remaining_budget: + budget_exceeded = True + except (ValueError, TypeError): + pass # malformed deadline — fall through to normal deny + _write_quota_log_event( { "ts": ts, - "event": "blocked", + "event": "blocked_budget_exceeded" if budget_exceeded else "blocked", "effective_threshold": effective_threshold, "window_name": window_name, "utilization": utilization, "sleep_seconds": n, "resets_at": resets_at_str, + "budget_exceeded": budget_exceeded, }, log_dir, ) - print( - json.dumps( - { - "hookSpecificOutput": { - "hookEventName": "PreToolUse", - "permissionDecision": "deny", - "permissionDecisionReason": ( - f"QUOTA WAIT REQUIRED (temporary — NOT a permanent error). " - f"Utilization: {utilization:.0f}% on window '{window_name}' " - f"(threshold: {effective_threshold:.0f}%). " - f"MANDATORY ACTION: Call run_cmd with: " - f'python3 -c "import time; time.sleep({n})" timeout={n + 30} — ' - f"then retry the SAME run_skill call with identical arguments. " - f"Before executing, state aloud: " - f"'Quota exceeded at {utilization:.0f}%. " - f"Sleeping {n}s, then retrying.'" - ), + + if budget_exceeded: + print( + json.dumps( + { + "hookSpecificOutput": { + "hookEventName": "PreToolUse", + "permissionDecision": "deny", + "permissionDecisionReason": ( + f"{QUOTA_BUDGET_EXCEEDED_TRIGGER} — " + f"Quota sleep ({n}s) exceeds session budget " + f"({remaining_budget:.0f}s remaining). " + f"MANDATORY ACTION: Emit your result block with " + f'"success": false, ' + f'"reason": "fleet_quota_exhausted", ' + f'"wait_seconds": {n}, ' + f'"summary": "Quota exceeded; session budget insufficient ' + f'for sleep. Resume after window resets." ' + f"Then STOP — do not call any more tools." + ), + } + } + ) + ) + else: + print( + json.dumps( + { + "hookSpecificOutput": { + "hookEventName": "PreToolUse", + "permissionDecision": "deny", + "permissionDecisionReason": ( + f"{QUOTA_GUARD_DENY_TRIGGER} (temporary — NOT a permanent error). " + f"Utilization: {utilization:.0f}% on window '{window_name}' " + f"(threshold: {effective_threshold:.0f}%). " + f"MANDATORY ACTION: Call run_cmd with: " + f'python3 -c "import time; time.sleep({n})" timeout={n + 30} — ' + f"then retry the SAME run_skill call with identical arguments. " + f"Before executing, state aloud: " + f"'Quota exceeded at {utilization:.0f}%. " + f"Sleeping {n}s, then retrying.'" + ), + } } - } + ) ) - ) else: _write_quota_log_event( { diff --git a/src/autoskillit/hooks/guards/recipe_write_advisor.py b/src/autoskillit/hooks/guards/recipe_write_advisor.py new file mode 100644 index 0000000000..ddabe5e230 --- /dev/null +++ b/src/autoskillit/hooks/guards/recipe_write_advisor.py @@ -0,0 +1,71 @@ +#!/usr/bin/env python3 +"""PreToolUse hook — suggests the appropriate skill when writing recipe YAML files. + +Non-blocking advisory: emits hookSpecificOutput.message, never permissionDecision. +Skips headless sessions (AUTOSKILLIT_HEADLESS=1) to avoid noise in automated runs. + +Stdlib-only — runs under any Python interpreter without the autoskillit package. +Patterns are inlined from SKILL_FILE_ADVISORY_MAP in core._type_constants; the +contract test test_hook_patterns_match_type_constants asserts they stay in sync. +""" + +from __future__ import annotations + +import json +import os +import re +import sys + +# Inlined subset of SKILL_FILE_ADVISORY_MAP (recipe-related entries only). +# Must stay in sync with core._type_constants.SKILL_FILE_ADVISORY_MAP. +# test_hook_patterns_match_type_constants enforces this. +_ADVISORY_PATTERNS: list[tuple[str, str]] = [ + (r"(?:\.autoskillit|src/autoskillit)/recipes/campaigns/.*\.ya?ml$", "make-campaign"), + (r"(?:\.autoskillit|src/autoskillit)/recipes/.*\.ya?ml$", "write-recipe"), +] + +_COMPILED: list[tuple[re.Pattern[str], str]] = [ + (re.compile(pat), skill) for pat, skill in _ADVISORY_PATTERNS +] + + +def main() -> None: + try: + data = json.loads(sys.stdin.read()) + except (json.JSONDecodeError, ValueError, OSError): + sys.exit(0) + + tool_name = data.get("tool_name", "") + if tool_name not in ("Write", "Edit"): + sys.exit(0) + + if os.environ.get("AUTOSKILLIT_HEADLESS") == "1": + sys.exit(0) + + file_path = data.get("tool_input", {}).get("file_path", "") + if not isinstance(file_path, str) or not file_path: + sys.exit(0) + + normalized = file_path.replace(os.sep, "/") + for pattern, skill_name in _COMPILED: + if pattern.search(normalized): + payload = json.dumps( + { + "hookSpecificOutput": { + "hookEventName": "PreToolUse", + "message": ( + f"Consider using /{skill_name} for this file. " + f"It provides schema validation, worked examples, and " + f"prevents common recipe errors." + ), + } + } + ) + sys.stdout.write(payload + "\n") + sys.exit(0) + + sys.exit(0) + + +if __name__ == "__main__": + main() diff --git a/src/autoskillit/hooks/remove_clone_guard.py b/src/autoskillit/hooks/guards/remove_clone_guard.py similarity index 99% rename from src/autoskillit/hooks/remove_clone_guard.py rename to src/autoskillit/hooks/guards/remove_clone_guard.py index 485cad14ce..03a2fcb362 100644 --- a/src/autoskillit/hooks/remove_clone_guard.py +++ b/src/autoskillit/hooks/guards/remove_clone_guard.py @@ -18,6 +18,8 @@ import subprocess import sys +REMOVE_CLONE_DENY_TRIGGER: str = "Clone at" + def _git(clone_path: str, *args: str, timeout: int = 10) -> tuple[int, str]: """Run a git command in clone_path. Returns (returncode, stdout.strip()).""" diff --git a/src/autoskillit/hooks/guards/resume_ownership_guard.py b/src/autoskillit/hooks/guards/resume_ownership_guard.py new file mode 100644 index 0000000000..7daa5d6367 --- /dev/null +++ b/src/autoskillit/hooks/guards/resume_ownership_guard.py @@ -0,0 +1,107 @@ +#!/usr/bin/env python3 +"""PreToolUse guard: validate resume_session_id ownership.""" + +from __future__ import annotations + +import json +import os +import sys +from pathlib import Path + +RESUME_OWNERSHIP_DENY_TRIGGER: str = "resume_session_id ownership validation failed" + + +def _deny(reason: str) -> None: + json.dump( + { + "hookSpecificOutput": { + "hookEventName": "PreToolUse", + "permissionDecision": "deny", + "permissionDecisionReason": reason, + } + }, + sys.stdout, + ) + + +def _resolve_provenance_path() -> Path: + override = os.environ.get("AUTOSKILLIT_STATE_DIR", "") + if override: + return Path(override) / "session_provenance.jsonl" + campaign = os.environ.get("AUTOSKILLIT_CAMPAIGN_ID", "") + base = Path.cwd() / ".autoskillit" / "temp" + if campaign: + base = base / campaign + return base / "session_provenance.jsonl" + + +def _find_provenance(session_id: str, prov_path: Path) -> dict | None: + if not prov_path.is_file(): + return None + try: + for line in prov_path.read_text().splitlines(): + line = line.strip() + if not line: + continue + try: + record = json.loads(line) + if record.get("session_id") == session_id: + return record + except json.JSONDecodeError: + continue + except OSError: + return None + return None + + +def main() -> None: + try: + data = json.loads(sys.stdin.read()) + except (json.JSONDecodeError, ValueError): + sys.exit(0) + + # Interactive sessions bypass ownership — the human user is the implicit owner. + if os.environ.get("AUTOSKILLIT_HEADLESS") != "1": + sys.exit(0) + + tool_input = data.get("tool_input", {}) + resume_session_id = tool_input.get("resume_session_id") + + if not resume_session_id: + sys.exit(0) + + prov_path = _resolve_provenance_path() + record = _find_provenance(resume_session_id, prov_path) + + if record is None: + sys.exit(0) + + if not record.get("recipe_name"): + _deny( + f"resume_session_id '{resume_session_id}' is not a food truck " + f"session (no recipe_name in provenance). Cannot resume an L3 " + f"orchestrator session." + ) + sys.exit(0) + + caller_session_id = data.get("session_id", "") + record_caller = record.get("caller_session_id", "") + record_kitchen = record.get("kitchen_id", "") + current_kitchen = os.environ.get("AUTOSKILLIT_CAMPAIGN_ID", "") + + caller_matches = record_caller and record_caller == caller_session_id + kitchen_matches = record_kitchen and record_kitchen == current_kitchen + + if not caller_matches and not kitchen_matches: + _deny( + f"resume_session_id '{resume_session_id}' is not owned by this " + f"session. Owner: caller={record_caller}, kitchen={record_kitchen}. " + f"Current: caller={caller_session_id}, kitchen={current_kitchen}." + ) + sys.exit(0) + + sys.exit(0) + + +if __name__ == "__main__": + main() diff --git a/src/autoskillit/hooks/guards/review_loop_gate.py b/src/autoskillit/hooks/guards/review_loop_gate.py new file mode 100644 index 0000000000..e8aa7a3787 --- /dev/null +++ b/src/autoskillit/hooks/guards/review_loop_gate.py @@ -0,0 +1,61 @@ +"""PreToolUse gate: block wait_for_ci/enqueue_pr when check_review_loop not yet called.""" + +from __future__ import annotations + +import json +import sys +from pathlib import Path + +REVIEW_LOOP_DENY_TRIGGER: str = "REVIEW LOOP REQUIRED" + +_DENY_REASON = ( + "REVIEW LOOP REQUIRED: review_pr returned changes_requested and " + "resolve_review applied fixes, but check_review_loop was not called. " + "You MUST call run_python with callable=" + "'autoskillit.smoke_utils.check_review_loop' with the correct " + "pr_number, cwd, current_iteration, max_iterations, and " + "previous_verdict parameters BEFORE proceeding to " + "wait_for_ci/enqueue_pr. " + "Recipe routing: re_push_review → on_success: check_review_loop." +) + +_STATE_FILE_RELPATH = (".autoskillit", "temp", "review_gate_state.json") + + +def main() -> None: + try: + json.loads(sys.stdin.read()) + except Exception: + sys.exit(0) + + state_file = Path.cwd().joinpath(*_STATE_FILE_RELPATH) + if not state_file.exists(): + sys.exit(0) + + try: + state = json.loads(state_file.read_text(encoding="utf-8")) + except Exception: + sys.exit(0) + + if state.get("gate") != "LOOP_REQUIRED": + sys.exit(0) + + if state.get("check_review_loop_called"): + sys.exit(0) + + print( + json.dumps( + { + "hookSpecificOutput": { + "hookEventName": "PreToolUse", + "permissionDecision": "deny", + "permissionDecisionReason": _DENY_REASON, + } + } + ) + ) + sys.exit(0) + + +if __name__ == "__main__": + main() diff --git a/src/autoskillit/hooks/skill_cmd_guard.py b/src/autoskillit/hooks/guards/skill_cmd_guard.py similarity index 94% rename from src/autoskillit/hooks/skill_cmd_guard.py rename to src/autoskillit/hooks/guards/skill_cmd_guard.py index bdeade69aa..b3b48d97ec 100644 --- a/src/autoskillit/hooks/skill_cmd_guard.py +++ b/src/autoskillit/hooks/guards/skill_cmd_guard.py @@ -4,7 +4,7 @@ Denies run_skill calls where a path-argument skill is invoked with extra descriptive text before the actual file path, e.g.: - /autoskillit:implement-worktree-no-merge the verified plan .autoskillit/temp/plan.md + /implement-worktree-no-merge the verified plan .autoskillit/temp/plan.md ^^^^^^^^^^^^^^^^^^^ extra words Detection logic: @@ -37,6 +37,8 @@ # headless session constructs an invalid path and fails with "not found". # This set must match exactly the skills whose SKILL.md files carry # path-detection instructions (verified by TestPathArgSkillsContract). +SKILL_CMD_DENY_TRIGGER: str = "skill_command format error for" + PATH_ARG_SKILLS: frozenset[str] = frozenset( { "implement-worktree-no-merge", @@ -50,7 +52,7 @@ _PATH_PREFIXES: tuple[str, ...] = ("/", "./", ".autoskillit/") # Captures the skill short-name from a skill_command string such as: -# /autoskillit:implement-worktree-no-merge ... +# /implement-worktree-no-merge ... # implement-worktree-no-merge ... _SKILL_RE = re.compile(r"^/?(?:autoskillit:)?(\S+)") @@ -121,10 +123,10 @@ def main() -> None: path_part = " ".join(path_tokens) if has_newlines and non_path_tokens: prose_block = " ".join(non_path_tokens) - correct_cmd = f"/autoskillit:{skill_name} {path_part}\n\n{prose_block}" + correct_cmd = f"/{skill_name} {path_part}\n\n{prose_block}" else: tail = (" " + " ".join(non_path_tokens)) if non_path_tokens else "" - correct_cmd = f"/autoskillit:{skill_name} {path_part}{tail}" + correct_cmd = f"/{skill_name} {path_part}{tail}" _deny( f"skill_command format error for '{skill_name}': " f"found extra descriptive text '{first}...' before the path argument " diff --git a/src/autoskillit/hooks/skill_command_guard.py b/src/autoskillit/hooks/guards/skill_command_guard.py similarity index 96% rename from src/autoskillit/hooks/skill_command_guard.py rename to src/autoskillit/hooks/guards/skill_command_guard.py index 11a80901b6..c773fbc66e 100644 --- a/src/autoskillit/hooks/skill_command_guard.py +++ b/src/autoskillit/hooks/guards/skill_command_guard.py @@ -9,6 +9,8 @@ import json import sys +SKILL_COMMAND_DENY_TRIGGER: str = "run_skill requires a slash-command prefix" + SKILL_COMMAND_PREFIX = "/" AUTOSKILLIT_SKILL_PREFIX = "/autoskillit:" diff --git a/src/autoskillit/hooks/guards/skill_load_guard.py b/src/autoskillit/hooks/guards/skill_load_guard.py new file mode 100644 index 0000000000..d1c2bd3177 --- /dev/null +++ b/src/autoskillit/hooks/guards/skill_load_guard.py @@ -0,0 +1,86 @@ +"""PreToolUse guard: deny native tools until Skill has been called. + +Gates on three conditions (ALL must be true): +- ``AUTOSKILLIT_PROVIDER_PROFILE`` is non-empty and not ``anthropic`` (case-insensitive) +- ``AUTOSKILLIT_HEADLESS == "1"`` +- ``AUTOSKILLIT_SESSION_TYPE == "skill"`` + +When gated, checks for ``.autoskillit/temp/skill_guard_{session_id}.flag``. +If absent, denies with a directive message instructing the model to call +the Skill tool first. + +Known limitation: when the model invokes Skill + native tools in a single +parallel message, the native tools fire PreToolUse before the Skill PostToolUse +writes the flag. This costs one wasted turn but self-resolves on the next turn. +This is inherent to the PreToolUse/PostToolUse timing model. + +Stdlib-only — no autoskillit imports. +""" + +from __future__ import annotations + +import json +import os +import sys +from pathlib import Path + +_HOOKS_DIR = str(Path(__file__).resolve().parent.parent) +if _HOOKS_DIR not in sys.path: + sys.path.insert(0, _HOOKS_DIR) + +from _hook_utils import find_project_root # type: ignore[import-not-found] # noqa: E402 + +SKILL_LOAD_DENY_TRIGGER: str = "SKILL LOADING REQUIRED" + +_DENY_MESSAGE: str = ( + "SKILL LOADING REQUIRED. You MUST call the Skill tool to load the skill " + "instructions before using any other tools. Call ToolSearch with query " + '"select:Skill" to load the Skill tool schema, then invoke Skill with the ' + "slash command name from your prompt. Do NOT use Read, Write, Edit, Bash, or " + "any other tool until the skill is loaded. This is a MANDATORY step — skipping " + "it will cause the session to fail." +) + + +def main() -> None: + try: + data = json.loads(sys.stdin.read()) + except Exception: + sys.exit(0) + + if data.get("agent_id"): + sys.exit(0) + + profile = os.environ.get("AUTOSKILLIT_PROVIDER_PROFILE", "").strip() + if not profile or profile.casefold() == "anthropic": + sys.exit(0) + + if os.environ.get("AUTOSKILLIT_HEADLESS") != "1": + sys.exit(0) + + if os.environ.get("AUTOSKILLIT_SESSION_TYPE") != "skill": + sys.exit(0) + + session_id: str = data.get("session_id", "") + if not session_id: + sys.exit(0) + + flag_path = find_project_root() / ".autoskillit" / "temp" / f"skill_guard_{session_id}.flag" + if flag_path.exists(): + sys.exit(0) + + payload = json.dumps( + { + "hookSpecificOutput": { + "hookEventName": "PreToolUse", + "permissionDecision": "deny", + "permissionDecisionReason": _DENY_MESSAGE, + } + } + ) + sys.stdout.write(payload + "\n") + sys.exit(0) + + +if __name__ == "__main__": + main() diff --git a/src/autoskillit/hooks/guards/skill_orchestration_guard.py b/src/autoskillit/hooks/guards/skill_orchestration_guard.py new file mode 100644 index 0000000000..319a1013fd --- /dev/null +++ b/src/autoskillit/hooks/guards/skill_orchestration_guard.py @@ -0,0 +1,71 @@ +#!/usr/bin/env python3 +"""PreToolUse hook — blocks orchestration tools from L1 skill sessions. + +Skill sessions (AUTOSKILLIT_SESSION_TYPE=skill or unset in headless mode) must +never call run_skill, run_cmd, or run_python. This is defense-in-depth over +the in-handler gate check in each tool. + +L2+ invariant: orchestrator (L2) and fleet (L3) sessions may call orchestration tools. +Skill sessions use native Claude Code tools only. +""" + +import json +import os +import sys + +SKILL_ORCHESTRATION_DENY_TRIGGER: str = "cannot be called from skill sessions" + +_ORCHESTRATION_TOOLS: frozenset[str] = frozenset({"run_skill", "run_cmd", "run_python"}) + + +def main() -> None: + try: + data = json.loads(sys.stdin.read()) + except (json.JSONDecodeError, ValueError, OSError): + sys.exit(0) # fail-open on malformed input + + # Interactive sessions always pass + if os.environ.get("AUTOSKILLIT_HEADLESS") != "1": + sys.exit(0) + + # Headless: resolve session type, fail-closed to skill session + raw_session_type = os.environ.get("AUTOSKILLIT_SESSION_TYPE", "") + session_type = raw_session_type.lower() + if session_type in ("orchestrator", "fleet"): + sys.exit(0) # permitted tiers — not a skill session + # skill, leaf (deprecated), unset → deny below; unrecognized non-empty values also denied + _unrecognized_tier = bool(session_type) and session_type not in ("skill", "leaf") + + tool_name: str = data.get("tool_name", "") + # MCP tool names are prefixed: mcp____ + # Check only the last __ segment — avoids false positives where a server + # name coincidentally contains an orchestration tool name. + tool = tool_name.split("__")[-1] + if tool in _ORCHESTRATION_TOOLS: + denial_reason = ( + f"{tool} cannot be called from skill sessions. " + "Only orchestrator or fleet sessions may call orchestration tools. " + "Skill sessions use native Claude Code tools only." + ) + if _unrecognized_tier: + denial_reason += ( + f" (AUTOSKILLIT_SESSION_TYPE={raw_session_type!r} is not a recognized tier;" + " expected: orchestrator, fleet, or skill)" + ) + payload = json.dumps( + { + "hookSpecificOutput": { + "hookEventName": "PreToolUse", + "permissionDecision": "deny", + "permissionDecisionReason": denial_reason, + } + } + ) + sys.stdout.write(payload + "\n") + sys.exit(0) + + sys.exit(0) + + +if __name__ == "__main__": + main() diff --git a/src/autoskillit/hooks/unsafe_install_guard.py b/src/autoskillit/hooks/guards/unsafe_install_guard.py similarity index 55% rename from src/autoskillit/hooks/unsafe_install_guard.py rename to src/autoskillit/hooks/guards/unsafe_install_guard.py index 320c0f70ed..66a3b75457 100644 --- a/src/autoskillit/hooks/unsafe_install_guard.py +++ b/src/autoskillit/hooks/guards/unsafe_install_guard.py @@ -1,21 +1,26 @@ """ -PreToolUse hook — blocks run_cmd calls that would install an editable package -into system Python without an explicit --python .venv target. +PreToolUse hook — blocks run_cmd and Bash tool calls that would install an editable +package into system Python without an explicit --python .venv target. -This guards the interactive orchestrator path (headless sessions cannot call -run_cmd at all — they are blocked by headless_orchestration_guard.py). +This guards the interactive orchestrator path (skill sessions cannot call +run_cmd at all — they are blocked by skill_orchestration_guard.py). """ import json import sys +UNSAFE_INSTALL_DENY_TRIGGER: str = "Blocked: editable install without --python .venv" + _UNSAFE_PATTERNS = ( "pip install -e", "pip install --editable", "uv pip install -e", "uv pip install --editable", + "maturin develop", ) +_SYSTEM_FLAG_PATTERN = "pip install" + def _is_unsafe_editable_install(cmd: str) -> bool: """Return True if cmd is an editable install not targeting a .venv Python.""" @@ -33,10 +38,20 @@ def _is_unsafe_editable_install(cmd: str) -> bool: return True +def _is_system_install(cmd: str) -> bool: + """Return True if cmd is a pip/uv install with --system flag.""" + cmd_lower = cmd.lower() + if _SYSTEM_FLAG_PATTERN not in cmd_lower: + return False + tokens = cmd_lower.split() + return "--system" in tokens + + def main() -> None: try: data = json.loads(sys.stdin.read()) - cmd = data.get("tool_input", {}).get("cmd", "") + tool_input = data.get("tool_input", {}) + cmd = tool_input.get("command", "") or tool_input.get("cmd", "") except (json.JSONDecodeError, AttributeError, OSError): sys.exit(0) @@ -59,6 +74,23 @@ def main() -> None: } ) sys.stdout.write(payload + "\n") + sys.exit(0) + + if _is_system_install(cmd): + payload = json.dumps( + { + "hookSpecificOutput": { + "hookEventName": "PreToolUse", + "permissionDecision": "deny", + "permissionDecisionReason": ( + "Blocked: --system install from worktree contaminates global environment. " + "Use `task install-worktree` or add `--python .venv/bin/python`." + ), + } + } + ) + sys.stdout.write(payload + "\n") + sys.exit(0) sys.exit(0) diff --git a/src/autoskillit/hooks/guards/write_guard.py b/src/autoskillit/hooks/guards/write_guard.py new file mode 100644 index 0000000000..c7c6744610 --- /dev/null +++ b/src/autoskillit/hooks/guards/write_guard.py @@ -0,0 +1,64 @@ +"""PreToolUse hook: blocks Write/Edit outside the allowed prefix in read-only sessions.""" + +from __future__ import annotations + +import json +import os +import sys + +WRITE_GUARD_DENY_TRIGGER = "read-only skill session" + + +def _deny(reason: str) -> None: + json.dump( + { + "hookSpecificOutput": { + "hookEventName": "PreToolUse", + "permissionDecision": "deny", + "permissionDecisionReason": reason, + } + }, + sys.stdout, + ) + sys.exit(0) + + +def main() -> None: + if not os.environ.get("AUTOSKILLIT_HEADLESS"): + sys.exit(0) + + allowed_prefix = os.environ.get("AUTOSKILLIT_ALLOWED_WRITE_PREFIX", "") + if not allowed_prefix: + sys.exit(0) + + try: + data = json.loads(sys.stdin.read()) + except (json.JSONDecodeError, ValueError, OSError): + _deny(f"Write/Edit blocked: {WRITE_GUARD_DENY_TRIGGER} (malformed hook input).") + return + + tool_name = data.get("tool_name", "") + if tool_name not in ("Write", "Edit"): + sys.exit(0) + + tool_input = data.get("tool_input", {}) + file_path = tool_input.get("file_path", "") + if not file_path: + _deny(f"Write/Edit blocked: {WRITE_GUARD_DENY_TRIGGER} (no file_path).") + return + + resolved = os.path.realpath(file_path) + real_prefix = os.path.realpath(allowed_prefix) + norm_prefix = real_prefix.rstrip("/") + "/" + + if resolved.startswith(norm_prefix) or resolved == norm_prefix.rstrip("/"): + sys.exit(0) + + _deny( + f"Write/Edit blocked: {WRITE_GUARD_DENY_TRIGGER}. " + f"Only writes to {allowed_prefix} are permitted." + ) + + +if __name__ == "__main__": + main() diff --git a/src/autoskillit/hooks/headless_orchestration_guard.py b/src/autoskillit/hooks/headless_orchestration_guard.py deleted file mode 100644 index ee6c3e931d..0000000000 --- a/src/autoskillit/hooks/headless_orchestration_guard.py +++ /dev/null @@ -1,54 +0,0 @@ -#!/usr/bin/env python3 -"""PreToolUse hook — blocks orchestration tools from headless sessions. - -Headless sessions (AUTOSKILLIT_HEADLESS=1) must never call run_skill, -run_cmd, or run_python. This is defense-in-depth over the in-handler -gate check in each tool. - -The two-tier invariant: Orchestrator (Tier 1) spawns workers via run_skill. -Workers (Tier 2) execute skills using native Claude Code tools. No nesting. -""" - -import json -import os -import sys - -_ORCHESTRATION_TOOLS: frozenset[str] = frozenset({"run_skill", "run_cmd", "run_python"}) - - -def main() -> None: - try: - data = json.loads(sys.stdin.read()) - except (json.JSONDecodeError, ValueError, OSError): - sys.exit(0) # fail-open on malformed input - - if os.environ.get("AUTOSKILLIT_HEADLESS") != "1": - sys.exit(0) # not headless — pass through - - tool_name: str = data.get("tool_name", "") - # MCP tool names are prefixed: mcp____ - # Check only the last __ segment — avoids false positives where a server - # name coincidentally contains an orchestration tool name. - tool = tool_name.split("__")[-1] - if tool in _ORCHESTRATION_TOOLS: - payload = json.dumps( - { - "hookSpecificOutput": { - "hookEventName": "PreToolUse", - "permissionDecision": "deny", - "permissionDecisionReason": ( - f"{tool} cannot be called from headless sessions. " - "Only the Tier 1 orchestrator may call orchestration tools. " - "Headless workers (Tier 2) use native Claude Code tools." - ), - } - } - ) - sys.stdout.write(payload + "\n") - sys.exit(0) - - sys.exit(0) - - -if __name__ == "__main__": - main() diff --git a/src/autoskillit/hooks/lint_after_edit_hook.py b/src/autoskillit/hooks/lint_after_edit_hook.py new file mode 100644 index 0000000000..8f4eda1587 --- /dev/null +++ b/src/autoskillit/hooks/lint_after_edit_hook.py @@ -0,0 +1,142 @@ +#!/usr/bin/env python3 +"""PostToolUse hook: runs ruff lint on Python files after Edit/Write in headless sessions.""" + +from __future__ import annotations + +import hashlib +import json +import os +import subprocess +import sys +from pathlib import Path + +_IMPLEMENT_PREFIXES = ("implement-", "resolve-") +LINT_AUTOFIX_TRIGGER = "--- RUFF AUTOFIX ---" +LINT_ERROR_TRIGGER = "--- RUFF LINT ---" + +_TIMEOUT_S = 15 + + +def _resolve_ruff() -> str: + """Resolve ruff binary co-located with the running Python interpreter.""" + candidate = Path(sys.executable).parent / "ruff" + if candidate.is_file(): + return str(candidate) + return "ruff" + + +def _file_sha256(path: str) -> str: + return hashlib.sha256(Path(path).read_bytes()).hexdigest() + + +def main() -> None: + if os.environ.get("AUTOSKILLIT_HEADLESS") != "1": + sys.exit(0) + + skill_name = os.environ.get("AUTOSKILLIT_SKILL_NAME", "") + if not any(skill_name.startswith(p) for p in _IMPLEMENT_PREFIXES): + sys.exit(0) + + try: + data = json.loads(sys.stdin.read()) + except (json.JSONDecodeError, ValueError, OSError): + sys.exit(0) + + tool_input = data.get("tool_input", {}) + if not isinstance(tool_input, dict): + sys.exit(0) + file_path = tool_input.get("file_path", "") + + if not file_path or not file_path.endswith(".py"): + sys.exit(0) + + if not Path(file_path).is_file(): + sys.exit(0) + + try: + hash_before = _file_sha256(file_path) + except OSError: + sys.exit(0) + + ruff_cmd = _resolve_ruff() + + try: + subprocess.run( + [ruff_cmd, "format", file_path], + capture_output=True, + text=True, + timeout=_TIMEOUT_S, + ) + except subprocess.TimeoutExpired: + pass + except (FileNotFoundError, OSError): + sys.exit(0) + + try: + subprocess.run( + [ruff_cmd, "check", "--fix", file_path], + capture_output=True, + text=True, + timeout=_TIMEOUT_S, + ) + except (FileNotFoundError, subprocess.TimeoutExpired, OSError): + sys.exit(0) + + remaining_errors = "" + try: + result = subprocess.run( + [ruff_cmd, "check", file_path], + capture_output=True, + text=True, + timeout=_TIMEOUT_S, + ) + if result.returncode != 0 and result.stdout.strip(): + remaining_errors = result.stdout.strip() + except (FileNotFoundError, subprocess.TimeoutExpired, OSError): + pass + + try: + hash_after = _file_sha256(file_path) + except OSError: + sys.exit(0) + + file_changed = hash_before != hash_after + + messages: list[str] = [] + + if file_changed: + messages.append( + f"{LINT_AUTOFIX_TRIGGER}\n" + "ruff auto-formatted this file. Re-read it before your next Edit " + "to avoid stale old_string mismatches." + ) + + if remaining_errors: + messages.append( + f"{LINT_ERROR_TRIGGER}\n" + "ruff found errors that --fix cannot resolve. " + f"Fix these before proceeding:\n{remaining_errors}" + ) + + if not messages: + sys.exit(0) + + tool_response = data.get("tool_response", "") + separator = "\n\n" if tool_response else "" + updated = f"{tool_response}{separator}" + "\n\n".join(messages) + + print( + json.dumps( + { + "hookSpecificOutput": { + "hookEventName": "PostToolUse", + "updatedToolResult": updated, + } + } + ) + ) + sys.exit(0) + + +if __name__ == "__main__": + main() diff --git a/src/autoskillit/hooks/quota_post_hook.py b/src/autoskillit/hooks/quota_post_hook.py index 90a1a40af5..5fb1dd26e2 100644 --- a/src/autoskillit/hooks/quota_post_hook.py +++ b/src/autoskillit/hooks/quota_post_hook.py @@ -28,6 +28,9 @@ _AUTOSKILLIT_LOG_DIR_ENV = "AUTOSKILLIT_LOG_DIR" +# Emitted in post-tool output; referenced by orchestrator prompt and sous-chef SKILL.md. +QUOTA_POST_WARNING_TRIGGER: str = "--- QUOTA WARNING ---" + def _read_quota_cache(cache_path_str: str, max_age: int) -> dict | None: """Read quota cache file. Returns parsed data or None if missing/stale/corrupt.""" @@ -57,7 +60,8 @@ def _resolve_quota_log_dir() -> Path | None: if xdg: return Path(xdg) / "autoskillit" / "logs" return Path.home() / ".local" / "share" / "autoskillit" / "logs" - except Exception: + except Exception as exc: + print(f"quota_post_hook: failed to resolve log directory: {exc}", file=sys.stderr) return None @@ -70,8 +74,8 @@ def _write_quota_log_event(event: dict, log_dir: Path | None) -> None: line = json.dumps(event) + "\n" with open(log_dir / "quota_events.jsonl", "a", encoding="utf-8") as f: f.write(line) - except Exception: - pass + except Exception as exc: + print(f"quota_post_hook: failed to write quota log event: {exc}", file=sys.stderr) def _extract_run_skill_result(tool_response: str | dict) -> str: @@ -118,6 +122,20 @@ def main(*, cache_path_override: str | None = None) -> None: log_dir = _resolve_quota_log_dir() ts = datetime.now(UTC).isoformat() + profile = os.environ.get("AUTOSKILLIT_PROVIDER_PROFILE", "").strip() + if profile and profile.casefold() != "anthropic": + _write_quota_log_event( + { + "ts": ts, + "event": "post_provider_bypass", + "profile": profile, + "tool_name": tool_name, + "cache_path": cache_path_str, + }, + log_dir, + ) + sys.exit(0) + cache = _read_quota_cache(cache_path_str, cache_max_age) if cache is None: sys.exit(0) @@ -165,7 +183,7 @@ def main(*, cache_path_override: str | None = None) -> None: warning_text = ( f"{result_summary}\n\n" - f"--- QUOTA WARNING ---\n" + f"{QUOTA_POST_WARNING_TRIGGER}\n" f"Post-execution utilization: {utilization:.0f}% on window '{window_name}' " f"(threshold: {effective_threshold:.0f}%)\n" f"MANDATORY ACTION before next run_skill: Call run_cmd with: " diff --git a/src/autoskillit/hooks/registry.sha256 b/src/autoskillit/hooks/registry.sha256 index 6bb952b83f..c0db202e92 100644 --- a/src/autoskillit/hooks/registry.sha256 +++ b/src/autoskillit/hooks/registry.sha256 @@ -1 +1 @@ -70827d7991056f6b4737f014b33fcdbb56681b2087dee455859d1e96fc5755a3 +543d874aa3773391eb21db21c034b7e315f8cd3b6c781a96dc14dc16267e93d6 diff --git a/src/autoskillit/hooks/review_gate_post_hook.py b/src/autoskillit/hooks/review_gate_post_hook.py new file mode 100644 index 0000000000..b70d9cf8b8 --- /dev/null +++ b/src/autoskillit/hooks/review_gate_post_hook.py @@ -0,0 +1,115 @@ +"""PostToolUse hook: capture review gate tags from run_skill and track check_review_loop calls.""" + +from __future__ import annotations + +import json +import os +import re +import sys +import tempfile +from datetime import UTC, datetime +from pathlib import Path + +_TAG_LOOP_REQUIRED = "%%REVIEW_GATE::LOOP_REQUIRED%%" +_TAG_CLEAR = "%%REVIEW_GATE::CLEAR%%" +_STATE_FILE_RELPATH = (".autoskillit", "temp", "review_gate_state.json") +_PR_NUMBER_RE = re.compile(r"\b(\d{1,6})\b") + + +def _atomic_write(path: Path, content: str) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + fd, tmp = tempfile.mkstemp(dir=path.parent, suffix=".tmp") + try: + with os.fdopen(fd, "w", encoding="utf-8") as f: + f.write(content) + f.flush() + os.fsync(f.fileno()) + os.replace(tmp, path) + except Exception: + try: + os.unlink(tmp) + except OSError: + pass + raise + + +def _extract_run_skill_result(tool_response: str | dict) -> str: # type: ignore[return] + """Unwrap double-wrapped run_skill JSON to get the inner result string.""" + try: + outer = json.loads(tool_response) if isinstance(tool_response, str) else tool_response + inner_str = outer["result"] + inner = json.loads(inner_str) if isinstance(inner_str, str) else inner_str + return inner.get("result", "") + except Exception: + return "" + + +def _extract_pr_number(skill_command: str) -> str: + """Best-effort PR number extraction from a skill_command string.""" + m = _PR_NUMBER_RE.search(skill_command) + return m.group(1) if m else "" + + +def main() -> None: + try: + data = json.loads(sys.stdin.read()) + except Exception: + sys.exit(0) + + tool_name: str = data.get("tool_name", "") + tool_input: dict = data.get("tool_input", {}) or {} + tool_response = data.get("tool_response", "") + + state_file = Path.cwd().joinpath(*_STATE_FILE_RELPATH) + + # Branch 1 & 2: run_skill output containing a gate tag + if "run_skill" in tool_name: + result_text = _extract_run_skill_result(tool_response) + if _TAG_LOOP_REQUIRED in result_text: + skill_command = tool_input.get("skill_command", "") or tool_input.get("cmd", "") + pr_number = _extract_pr_number(str(skill_command)) + state = { + "gate": "LOOP_REQUIRED", + "review_verdict": "changes_requested", + "check_review_loop_called": False, + "pr_number": pr_number, + "set_at": datetime.now(UTC).isoformat(), + } + try: + _atomic_write(state_file, json.dumps(state)) + except Exception as e: + print(f"review_gate_post_hook: failed to write gate state: {e}", file=sys.stderr) + sys.exit(0) + + if _TAG_CLEAR in result_text: + try: + state_file.unlink(missing_ok=True) + except OSError: + pass + sys.exit(0) + + # Branch 3: run_python with check_review_loop callable + if "run_python" in tool_name: + callable_name: str = tool_input.get("callable", "") or "" + if "check_review_loop" in callable_name: + if state_file.exists(): + try: + state = json.loads(state_file.read_text(encoding="utf-8")) + if state.get("gate") == "LOOP_REQUIRED": + state["check_review_loop_called"] = True + # Update pr_number from args if available + args = tool_input.get("args") or {} + if isinstance(args, dict) and args.get("pr_number"): + state["pr_number"] = str(args["pr_number"]) + _atomic_write(state_file, json.dumps(state)) + except Exception as e: + print( + f"review_gate_post_hook: failed to update gate state: {e}", file=sys.stderr + ) + sys.exit(0) + + sys.exit(0) + + +if __name__ == "__main__": + main() diff --git a/src/autoskillit/hooks/session_start_hook.py b/src/autoskillit/hooks/session_start_hook.py index 1651c1a2e9..cfbee8514d 100644 --- a/src/autoskillit/hooks/session_start_hook.py +++ b/src/autoskillit/hooks/session_start_hook.py @@ -24,12 +24,16 @@ def main() -> None: sys.exit(0) # fail-open on malformed input # Best-effort TTL sweep of stale kitchen markers. Fail-open — must not raise. + _best_recipe_name: str | None = None + _best_opened_at = None try: _state_override = os.environ.get("AUTOSKILLIT_STATE_DIR") if _state_override: _state_dir = Path(_state_override) / "kitchen_state" else: - _state_dir = Path.cwd() / ".autoskillit" / "temp" / "kitchen_state" + _campaign_id = os.environ.get("AUTOSKILLIT_CAMPAIGN_ID", "") + _base = Path.cwd() / ".autoskillit" / "temp" / "kitchen_state" + _state_dir = _base / _campaign_id if _campaign_id else _base if _state_dir.is_dir(): _ttl_hours = 24 for _p in _state_dir.glob("*.json"): @@ -39,6 +43,14 @@ def main() -> None: _age = datetime.now(UTC) - _opened_at if _age.total_seconds() >= _ttl_hours * 3600: _p.unlink() + else: + # Fresh — track most recent recipe name + _this_recipe = _d.get("recipe_name") + if _best_opened_at is None or _opened_at > _best_opened_at: + _best_opened_at = _opened_at + _best_recipe_name = ( + _this_recipe if isinstance(_this_recipe, str) else None + ) except Exception: try: _p.unlink() @@ -59,16 +71,22 @@ def main() -> None: if size == 0: sys.exit(0) # fresh session — no reminder needed - payload = json.dumps( - { - "additionalContext": ( - "RESUME REMINDER: You are resuming a previous AutoSkillit session. " - "MCP tool access (kitchen) is not automatically restored on resume. " - "Call /autoskillit:open-kitchen first to regain access to all " - "AutoSkillit MCP tools before continuing your work." - ) - } + _base_msg = ( + "RESUME REMINDER: You are resuming a previous AutoSkillit session. " + "MCP tool access (kitchen) is not automatically restored on resume. " ) + if _best_recipe_name: + _detail = ( + f"You were running recipe '{_best_recipe_name}' — " + f"call open_kitchen(name='{_best_recipe_name}') to regain access to all " + "AutoSkillit MCP tools before continuing your work." + ) + else: + _detail = ( + "Call /autoskillit:open-kitchen first to regain access to all " + "AutoSkillit MCP tools before continuing your work." + ) + payload = json.dumps({"additionalContext": _base_msg + _detail}) sys.stdout.write(payload + "\n") sys.exit(0) diff --git a/src/autoskillit/hooks/skill_load_post_hook.py b/src/autoskillit/hooks/skill_load_post_hook.py new file mode 100644 index 0000000000..8cb4a8f724 --- /dev/null +++ b/src/autoskillit/hooks/skill_load_post_hook.py @@ -0,0 +1,86 @@ +"""PostToolUse hook: write skill-loaded flag for non-Anthropic providers. + +Fires on ``Skill`` tool calls. When ``AUTOSKILLIT_PROVIDER_PROFILE`` is +non-empty, writes ``.autoskillit/temp/skill_guard_{session_id}.flag`` +containing the loaded skill name. The companion PreToolUse guard +(``guards/skill_load_guard.py``) checks this flag before allowing native +tool calls. + +Stdlib-only — no autoskillit imports. +""" + +from __future__ import annotations + +import json +import os +import sys +import tempfile +from pathlib import Path + +_HOOKS_DIR = str(Path(__file__).resolve().parent) +if _HOOKS_DIR not in sys.path: + sys.path.insert(0, _HOOKS_DIR) + +from _hook_utils import find_project_root # type: ignore[import-not-found] # noqa: E402 + + +def _atomic_write(path: Path, content: str) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + fd, tmp = tempfile.mkstemp(dir=path.parent, suffix=".tmp") + try: + with os.fdopen(fd, "w", encoding="utf-8") as f: + f.write(content) + f.flush() + os.fsync(f.fileno()) + os.replace(tmp, path) + except Exception: + try: + os.unlink(tmp) + except OSError: + pass + raise + + +def main() -> None: + try: + data = json.loads(sys.stdin.read()) + except Exception: + sys.exit(0) + + if data.get("agent_id"): + sys.exit(0) + + if not os.environ.get("AUTOSKILLIT_PROVIDER_PROFILE", "").strip(): + sys.exit(0) + + if data.get("tool_name") != "Skill": + sys.exit(0) + + tool_input: dict = data.get("tool_input", {}) or {} + skill_name: str = tool_input.get("skill", "") + session_id: str = data.get("session_id", "") + + if not session_id: + sys.exit(0) + + flag_path = find_project_root() / ".autoskillit" / "temp" / f"skill_guard_{session_id}.flag" + try: + _atomic_write(flag_path, skill_name) + except Exception as exc: + sys.stderr.write(f"skill_load_post_hook: failed to write flag {flag_path}: {exc}\n") + + marker = os.environ.get("AUTOSKILLIT_COMPLETION_MARKER", "").strip() + if marker: + reminder = ( + "COMPLETION REMINDER: After completing your task, your final text output " + f"MUST end with exactly: {marker}\n" + "This is mandatory regardless of what the skill's Output section specifies." + ) + payload = json.dumps({"additionalContext": reminder}) + sys.stdout.write(payload + "\n") + + sys.exit(0) + + +if __name__ == "__main__": + main() diff --git a/src/autoskillit/hooks/token_summary_hook.py b/src/autoskillit/hooks/token_summary_hook.py index e452cadecb..c225f27ba1 100644 --- a/src/autoskillit/hooks/token_summary_hook.py +++ b/src/autoskillit/hooks/token_summary_hook.py @@ -22,9 +22,9 @@ # stdlib-only subprocess hook: import _fmt_primitives by bare name via sys.path # (test_hooks_are_stdlib_only). Venv tests use the autoskillit.hooks package path. -_HOOKS_DIR = str(pathlib.Path(__file__).resolve().parent) -if _HOOKS_DIR not in sys.path: - sys.path.insert(0, _HOOKS_DIR) +_FORMATTERS_DIR = str(pathlib.Path(__file__).resolve().parent / "formatters") +if _FORMATTERS_DIR not in sys.path: + sys.path.insert(0, _FORMATTERS_DIR) from _fmt_primitives import ( # type: ignore[import-not-found] # noqa: E402 _HOOK_CONFIG_PATH_COMPONENTS, @@ -59,20 +59,25 @@ def _log_root() -> pathlib.Path: return base / "autoskillit/logs" -def _extract_pr_url(tool_name: str, tool_response_raw: str) -> str | None: - """Extract a GitHub PR URL from a PostToolUse tool_response string. +def _unwrap_mcp_response(tool_name: str, raw: str) -> dict | None: + """Parse and double-unwrap a PostToolUse tool_response string. - Replicates pretty_output._resolve_payload double-unwrap logic. - Returns the URL string or None if not found. + Returns the effective payload dict, or None if raw is not valid JSON or + not a dict. + + For MCP tools (tool_name starts with 'mcp__'), if the outer dict has + exactly one key 'result' whose value is a JSON string, attempts to parse + that string as a nested dict and returns it. Falls back to returning the + outer dict when inner parsing fails or yields a non-dict. Non-MCP tools + return the outer dict directly. """ try: - outer = json.loads(tool_response_raw) + outer = json.loads(raw) except (json.JSONDecodeError, ValueError): return None if not isinstance(outer, dict): return None - result_text: str | None = None if ( tool_name.startswith("mcp__") and list(outer.keys()) == ["result"] @@ -81,15 +86,24 @@ def _extract_pr_url(tool_name: str, tool_response_raw: str) -> str | None: try: inner = json.loads(outer["result"]) if isinstance(inner, dict): - result_text = inner.get("result", "") + return inner except (json.JSONDecodeError, ValueError): - result_text = outer["result"] - else: - result_text = outer.get("result", "") + pass + return outer + + +def _extract_pr_url(tool_name: str, tool_response_raw: str) -> str | None: + """Extract a GitHub PR URL from a PostToolUse tool_response string. + + Returns the URL string or None if not found. + """ + payload = _unwrap_mcp_response(tool_name, tool_response_raw) + if payload is None: + return None + result_text = payload.get("result", "") if not result_text or not isinstance(result_text, str): return None - m = re.search(r"https://github\.com/[^/\s]+/[^/\s]+/pull/\d+", result_text) return m.group() if m else None @@ -139,34 +153,12 @@ def _read_kitchen_id(base: pathlib.Path | None = None) -> str: def _extract_order_id(tool_name: str, tool_response_raw: str) -> str: """Extract order_id from a PostToolUse run_skill result JSON. - Replicates the same double-unwrap logic as _extract_pr_url. Returns '' if not found. """ - try: - outer = json.loads(tool_response_raw) - except (json.JSONDecodeError, ValueError): - return "" - if not isinstance(outer, dict): + payload = _unwrap_mcp_response(tool_name, tool_response_raw) + if payload is None: return "" - - inner_dict: dict | None = None - if ( - tool_name.startswith("mcp__") - and list(outer.keys()) == ["result"] - and isinstance(outer["result"], str) - ): - try: - parsed = json.loads(outer["result"]) - if isinstance(parsed, dict): - inner_dict = parsed - except (json.JSONDecodeError, ValueError): - pass - else: - inner_dict = outer - - if inner_dict is None: - return "" - return str(inner_dict.get("order_id", "")) + return str(payload.get("order_id", "")) def _load_sessions( @@ -221,7 +213,7 @@ def _load_sessions( except (json.JSONDecodeError, OSError): continue - raw_step = data.get("step_name", "") + raw_step = data.get("session_label", "") if not raw_step: continue @@ -229,14 +221,22 @@ def _load_sessions( if key not in aggregated: aggregated[key] = { "step_name": key, + "model": "", "input_tokens": 0, "output_tokens": 0, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 0, "elapsed_seconds": 0.0, "invocation_count": 0, + "loc_insertions": 0, + "loc_deletions": 0, + "peak_context": 0, + "turn_count": 0, } entry = aggregated[key] + _model = data.get("model_identifier", "") + if _model and not entry["model"]: + entry["model"] = _model entry["input_tokens"] += data.get("input_tokens", 0) entry["output_tokens"] += data.get("output_tokens", 0) entry["cache_creation_input_tokens"] += data.get("cache_creation_input_tokens", 0) @@ -244,6 +244,14 @@ def _load_sessions( _raw_timing = data.get("timing_seconds") entry["elapsed_seconds"] += float(_raw_timing) if _raw_timing is not None else 0.0 entry["invocation_count"] += 1 + entry["loc_insertions"] = entry.get("loc_insertions", 0) + data.get("loc_insertions", 0) + entry["loc_deletions"] = entry.get("loc_deletions", 0) + data.get("loc_deletions", 0) + _raw_peak = data.get("peak_context", 0) + if isinstance(_raw_peak, int) and _raw_peak > entry["peak_context"]: + entry["peak_context"] = _raw_peak + _raw_turns = data.get("turn_count", 0) + if isinstance(_raw_turns, int): + entry["turn_count"] = entry.get("turn_count", 0) + _raw_turns return aggregated @@ -253,42 +261,152 @@ def _format_table(aggregated: dict[str, dict[str, Any]]) -> str: lines = [ "## Token Usage Summary", "", - "| Step | uncached | output | cache_read | cache_write | count | time |", - "|------|----------|--------|------------|-------------|-------|------|", + "| Step | Model | count | uncached | output | cache_read | peak_ctx | turns | cache_write | time |", # noqa: E501 + "|------|-------|-------|----------|--------|------------|----------|-------|-------------|------|", ] total_input = 0 total_output = 0 total_cache_rd = 0 + total_peak = 0 total_cache_wr = 0 total_time = 0.0 + has_non_anthropic = False for entry in aggregated.values(): name = entry["step_name"] + model = entry.get("model", "") + if model and not model.startswith("claude-"): + name = f"{name}*" + has_non_anthropic = True + count = entry.get("invocation_count", 1) inp = entry["input_tokens"] out = entry["output_tokens"] cache_rd = entry["cache_read_input_tokens"] + peak_ctx = entry.get("peak_context", 0) + turns = entry.get("turn_count", 0) cache_wr = entry["cache_creation_input_tokens"] - count = entry["invocation_count"] elapsed = entry["elapsed_seconds"] lines.append( - f"| {name} | {_humanize(inp)} | {_humanize(out)} | {_humanize(cache_rd)}" - f" | {_humanize(cache_wr)} | {count} | {_fmt_duration(elapsed)} |" + f"| {name} | {model} | {count} | {_humanize(inp)} | {_humanize(out)}" + f" | {_humanize(cache_rd)} | {_humanize(peak_ctx)} | {turns} | {_humanize(cache_wr)}" + f" | {_fmt_duration(elapsed)} |" ) total_input += inp total_output += out total_cache_rd += cache_rd + if peak_ctx > total_peak: + total_peak = peak_ctx total_cache_wr += cache_wr total_time += elapsed lines.append( - f"| **Total** | {_humanize(total_input)} | {_humanize(total_output)}" - f" | {_humanize(total_cache_rd)} | {_humanize(total_cache_wr)}" - f" | | {_fmt_duration(total_time)} |" + f"| **Total** | | | {_humanize(total_input)} | {_humanize(total_output)}" + f" | {_humanize(total_cache_rd)} | {_humanize(total_peak)}" + f" | | {_humanize(total_cache_wr)} | {_fmt_duration(total_time)} |" + ) + if has_non_anthropic: + lines.append("") + lines.append(r"\* *Step used a non-Anthropic provider; caching behavior may differ.*") + + return "\n".join(lines) + + +def _format_efficiency_table(aggregated: dict[str, dict[str, Any]]) -> str: + """Format aggregated token data as a markdown ## Token Efficiency table. + + Returns '' when no session has LoC data (all zero). + """ + has_loc = any( + e.get("loc_insertions", 0) + e.get("loc_deletions", 0) > 0 for e in aggregated.values() + ) + if not has_loc: + return "" + + def _ratio(tokens: int, loc: int) -> str: + return f"{tokens / loc:.1f}" if loc > 0 else "—" + + lines = [ + "## Token Efficiency", + "", + "| Step | LoC Changed | cache_read/LoC | cache_write/LoC | output/LoC |", + "|------|-------------|----------------|-----------------|------------|", + ] + total_loc = total_cr = total_cw = total_out = 0 + for entry in aggregated.values(): + loc = entry.get("loc_insertions", 0) + entry.get("loc_deletions", 0) + cr = entry.get("cache_read_input_tokens", 0) + cw = entry.get("cache_creation_input_tokens", 0) + out = entry.get("output_tokens", 0) + lines.append( + f"| {entry['step_name']} | {loc}" + f" | {_ratio(cr, loc)}" + f" | {_ratio(cw, loc)} | {_ratio(out, loc)} |" + ) + total_loc += loc + total_cr += cr + total_cw += cw + total_out += out + + lines.append( + f"| **Total** | **{total_loc}**" + f" | {_ratio(total_cr, total_loc)}" + f" | {_ratio(total_cw, total_loc)} | {_ratio(total_out, total_loc)} |" ) + return "\n".join(lines) + + +def _format_model_table(aggregated: dict[str, dict[str, Any]]) -> str: + """Format per-model aggregate breakdown as ## Model Usage Breakdown table. + Returns '' when all entries have no model data (legacy sessions). + + Duplicates TelemetryFormatter.format_model_table by design: hook scripts are + stdlib-only and cannot import from autoskillit.*. + """ + model_data: dict[str, dict[str, Any]] = {} + for entry in aggregated.values(): + model = entry.get("model", "") + if not model or model == "unknown": + continue + if model not in model_data: + model_data[model] = { + "model": model, + "_steps": set(), + "input_tokens": 0, + "output_tokens": 0, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + "elapsed_seconds": 0.0, + } + md = model_data[model] + md["_steps"].add(entry.get("step_name", "")) + md["input_tokens"] += entry.get("input_tokens", 0) + md["output_tokens"] += entry.get("output_tokens", 0) + md["cache_creation_input_tokens"] += entry.get("cache_creation_input_tokens", 0) + md["cache_read_input_tokens"] += entry.get("cache_read_input_tokens", 0) + md["elapsed_seconds"] += entry.get("elapsed_seconds", 0.0) + + if not model_data: + return "" + + lines = [ + "## Model Usage Breakdown", + "", + "| Model | steps | uncached | output | cache_read | cache_write | time |", + "|-------|-------|----------|--------|------------|-------------|------|", + ] + for md in model_data.values(): + step_count = len(md.pop("_steps")) + lines.append( + f"| {md['model']} | {step_count}" + f" | {_humanize(md['input_tokens'])} | {_humanize(md['output_tokens'])}" + f" | {_humanize(md['cache_read_input_tokens'])}" + f" | {_humanize(md['cache_creation_input_tokens'])}" + f" | {_fmt_duration(md['elapsed_seconds'])} |" + ) return "\n".join(lines) @@ -338,7 +456,13 @@ def main() -> None: sys.exit(0) token_table = _format_table(aggregated) + efficiency_table = _format_efficiency_table(aggregated) + model_table = _format_model_table(aggregated) new_body = current_body + "\n\n" + token_table + if efficiency_table: + new_body += "\n\n" + efficiency_table + if model_table: + new_body += "\n\n" + model_table try: subprocess.run( diff --git a/src/autoskillit/migration/CLAUDE.md b/src/autoskillit/migration/CLAUDE.md new file mode 100644 index 0000000000..f0fc4b2063 --- /dev/null +++ b/src/autoskillit/migration/CLAUDE.md @@ -0,0 +1,20 @@ +# migration/ + +IL-2 migration engine — versioned config migration with adapter hierarchy and failure store. + +## Files + +| File | Purpose | +|------|---------| +| `__init__.py` | Re-exports `check_and_migrate`, `MigrationEngine` | +| `engine.py` | `MigrationEngine`, adapter ABC hierarchy | +| `_api.py` | `check_and_migrate` — top-level migration entry point | +| `loader.py` | Migration note discovery + version chaining | +| `store.py` | `FailureStore` (JSON, atomic writes) | + +## Architecture Notes + +Migration notes live in `src/autoskillit/migrations/` as YAML files discovered by +`loader.py` at startup. `store.py` persists failures to `.autoskillit/temp/` using atomic +writes. `engine.py` defines the adapter ABC; concrete adapters are registered per migration +note version. diff --git a/src/autoskillit/migration/__init__.py b/src/autoskillit/migration/__init__.py index 2fc98582dd..a129656680 100644 --- a/src/autoskillit/migration/__init__.py +++ b/src/autoskillit/migration/__init__.py @@ -1,4 +1,4 @@ -"""L2 migration domain — version graph, adapter dispatch, failure persistence.""" +"""IL-2 migration domain — version graph, adapter dispatch, failure persistence.""" from __future__ import annotations diff --git a/src/autoskillit/migration/_api.py b/src/autoskillit/migration/_api.py index 2842ab540d..dd9049b606 100644 --- a/src/autoskillit/migration/_api.py +++ b/src/autoskillit/migration/_api.py @@ -1,6 +1,6 @@ """Migration API: top-level check_and_migrate convenience function. -Cross-L2 coupling note: migration/ (L2) imports from recipe/ (L2). This is +Cross-IL-2 coupling note: migration/ (IL-2) imports from recipe/ (IL-2). This is architecturally correct — migration inherently needs to discover, parse, and validate recipes. The coupling is not extractable without inverting control in a way that would complicate callers. The # noqa: PLC0415 comments on the deferred diff --git a/src/autoskillit/migration/engine.py b/src/autoskillit/migration/engine.py index 40860643fe..bb37c69bb4 100644 --- a/src/autoskillit/migration/engine.py +++ b/src/autoskillit/migration/engine.py @@ -1,7 +1,6 @@ from __future__ import annotations import json -import re import shutil from abc import ABC, abstractmethod from collections.abc import Awaitable, Callable @@ -9,6 +8,8 @@ from pathlib import Path from typing import Any +import regex as re + from autoskillit import __version__ from autoskillit.core import ( RetryReason, @@ -42,6 +43,13 @@ class MigrationResult: migrated_content: str | None = None error: str | None = None retries_attempted: int = 0 + advisory: str | None = None + + +@dataclass +class AdvisoryResult: + name: str + suggestion: str class MigrationAdapter(ABC): @@ -89,7 +97,18 @@ async def migrate( """Apply migration deterministically; write-back handled by MigrationEngine.""" -_AnyAdapter = HeadlessMigrationAdapter | DeterministicMigrationAdapter +class AdvisoryMigrationAdapter(MigrationAdapter): + """Adapter for skill-crafted artifacts: detects staleness but never writes files. + + Returns advisory results (warnings/suggestions) that surface in migration + reports. File regeneration is deferred to the appropriate skill invocation. + """ + + @abstractmethod + def check_staleness(self, file: MigrationFile) -> AdvisoryResult: ... + + +_AnyAdapter = HeadlessMigrationAdapter | DeterministicMigrationAdapter | AdvisoryMigrationAdapter class MigrationEngine: @@ -116,7 +135,10 @@ async def migrate_file( if not adapter.needs_migration(file): return MigrationResult(success=True, name=file.name) - if isinstance(adapter, DeterministicMigrationAdapter): + if isinstance(adapter, AdvisoryMigrationAdapter): + advisory = adapter.check_staleness(file) + return MigrationResult(success=True, name=file.name, advisory=advisory.suggestion) + elif isinstance(adapter, DeterministicMigrationAdapter): result = await adapter.migrate(file, temp_dir=temp_dir) else: result = await adapter.migrate(file, run_headless=run_headless, temp_dir=temp_dir) @@ -137,7 +159,7 @@ class RecipeMigrationAdapter(HeadlessMigrationAdapter): file_type = "recipe" def discover(self, project_dir: Path) -> list[MigrationFile]: - from autoskillit.recipe import parse_recipe_metadata + from autoskillit.recipe import parse_recipe_metadata # noqa: PLC0415 recipes_dir = project_dir / ".autoskillit" / "recipes" if not recipes_dir.exists(): @@ -230,12 +252,12 @@ def get_temp_output_path(self, file: MigrationFile, temp_dir: Path) -> Path: return temp_dir / "migrations" / f"{file.path.stem}.yaml" def validate(self, path: Path) -> tuple[bool, str]: - from autoskillit.recipe import load_recipe as _parse_recipe - from autoskillit.recipe import validate_recipe + from autoskillit.recipe import load_recipe as _parse_recipe # noqa: PLC0415 + from autoskillit.recipe import validate_recipe_structure # noqa: PLC0415 try: recipe = _parse_recipe(path) - errors = validate_recipe(recipe) + errors = validate_recipe_structure(recipe) if errors: return False, "; ".join(str(e) for e in errors) return True, "" @@ -264,7 +286,7 @@ def discover(self, project_dir: Path) -> list[MigrationFile]: return files def needs_migration(self, file: MigrationFile) -> bool: - from autoskillit.recipe import check_contract_staleness, load_recipe_card + from autoskillit.recipe import check_contract_staleness, load_recipe_card # noqa: PLC0415 recipes_dir = file.path.parent.parent contract = load_recipe_card(file.name, recipes_dir) @@ -278,7 +300,7 @@ async def migrate( *, temp_dir: Path, ) -> MigrationResult: - from autoskillit.recipe import generate_recipe_card + from autoskillit.recipe import generate_recipe_card # noqa: PLC0415 recipes_dir = file.path.parent.parent recipe_path = recipes_dir / f"{file.name}.yaml" @@ -306,8 +328,12 @@ def validate(self, path: Path) -> tuple[bool, str]: return False, str(exc) -class DiagramMigrationAdapter(DeterministicMigrationAdapter): - """Adapter that regenerates stale recipe flow diagram Markdown files.""" +class DiagramMigrationAdapter(AdvisoryMigrationAdapter): + """Advisory adapter for skill-crafted recipe flow diagrams. + + Detects stale diagrams but never overwrites them — returns a suggestion + to run ``/render-recipe`` instead. + """ file_type = "diagram" @@ -321,21 +347,24 @@ def discover(self, project_dir: Path) -> list[MigrationFile]: ] def needs_migration(self, file: MigrationFile) -> bool: - from autoskillit.recipe import check_diagram_staleness + from autoskillit.recipe import check_diagram_staleness # noqa: PLC0415 - recipes_dir = file.path.parent.parent # diagrams/ -> recipes/ + if not file.path.exists(): + return False + recipes_dir = file.path.parent.parent recipe_path = recipes_dir / f"{file.name}.yaml" if not recipe_path.exists(): return False return check_diagram_staleness(file.name, recipes_dir, recipe_path) - async def migrate( - self, - file: MigrationFile, - *, - temp_dir: Path, - ) -> MigrationResult: - return MigrationResult(success=True, name=file.name) + def check_staleness(self, file: MigrationFile) -> AdvisoryResult: + from autoskillit.recipe import diagram_stale_to_suggestions # noqa: PLC0415 + + suggestions = diagram_stale_to_suggestions(file.name) + return AdvisoryResult( + name=file.name, + suggestion=suggestions[0]["message"] if suggestions else "", + ) def validate(self, path: Path) -> tuple[bool, str]: try: @@ -388,7 +417,7 @@ async def migrate(self, recipe_path: Path) -> dict[str, Any]: """ from autoskillit.migration.loader import applicable_migrations as _applicable from autoskillit.migration.store import FailureStore, default_store_path - from autoskillit.recipe import parse_recipe_metadata + from autoskillit.recipe import parse_recipe_metadata # noqa: PLC0415 meta = parse_recipe_metadata(recipe_path) name = meta.name @@ -451,6 +480,7 @@ async def run_headless(*args: Any, **kwargs: Any) -> SkillResult: # type: ignor ) return {"error": f"Migration failed: {migration_result.error}", "name": name} + advisories: list[str] = [] contracts_regenerated: list[str] = [] contract_adapter = self._engine.get_adapter("contract") if contract_adapter is not None: @@ -489,7 +519,9 @@ async def run_headless(*args: Any, **kwargs: Any) -> SkillResult: # type: ignor run_headless=run_headless, temp_dir=temp_dir, ) - if not diagram_result.success: + if diagram_result.advisory: + advisories.append(diagram_result.advisory) + elif not diagram_result.success: logger.warning( "diagram.migration_failed", name=name, @@ -497,9 +529,15 @@ async def run_headless(*args: Any, **kwargs: Any) -> SkillResult: # type: ignor ) if did_version_migrate or contracts_regenerated: - return { + result_dict: dict[str, object] = { "status": "migrated", "name": name, "contracts_regenerated": contracts_regenerated, } - return {"status": "up_to_date", "name": name} + if advisories: + result_dict["advisories"] = advisories + return result_dict + result_dict = {"status": "up_to_date", "name": name} + if advisories: + result_dict["advisories"] = advisories + return result_dict diff --git a/src/autoskillit/migration/loader.py b/src/autoskillit/migration/loader.py index 52fee74d4c..a1fb4fe75e 100644 --- a/src/autoskillit/migration/loader.py +++ b/src/autoskillit/migration/loader.py @@ -42,18 +42,29 @@ def _migrations_dir() -> Path: return pkg_root() / "migrations" +_migrations_cache: dict[tuple[str, float], list[MigrationNote]] = {} + + def list_migrations() -> list[MigrationNote]: """Discover and parse all migration YAML files, sorted by filename.""" mig_dir = _migrations_dir() if not mig_dir.is_dir(): return [] - + try: + mt = mig_dir.stat().st_mtime + except OSError: + mt = -1.0 + key = (str(mig_dir), mt) + cached = _migrations_cache.get(key) + if cached is not None: + return list(cached) notes: list[MigrationNote] = [] for f in sorted(mig_dir.iterdir()): if f.suffix in (".yaml", ".yml") and f.is_file(): note = _parse_migration(f) notes.append(note) - return notes + _migrations_cache[key] = notes + return list(notes) def applicable_migrations( @@ -62,15 +73,20 @@ def applicable_migrations( """Return migration notes applicable to a script at the given version. Chains migrations from *script_version* up to *installed_version*. + Returns empty when script_version is None (bundled recipes have no version + and are not subject to migration). Algorithm: - 1. current = Version(script_version) or Version("0.0.0") if None - 2. Sort migrations by to_version ascending - 3. For each migration: if from_version <= current < to_version, include it + 1. If script_version is None, return [] (not migratable) + 2. current = Version(script_version) + 3. Sort migrations by to_version ascending + 4. For each migration: if from_version <= current < to_version, include it and advance current = to_version - 4. Stop when current >= installed_version or no more migrations + 5. Stop when current >= installed_version or no more migrations """ - current = Version(script_version) if script_version else Version("0.0.0") + if script_version is None: + return [] + current = Version(script_version) target = Version(installed_version) if current >= target: diff --git a/src/autoskillit/migrations/0.9.134-to-0.9.135.yaml b/src/autoskillit/migrations/0.9.134-to-0.9.135.yaml new file mode 100644 index 0000000000..53124254cd --- /dev/null +++ b/src/autoskillit/migrations/0.9.134-to-0.9.135.yaml @@ -0,0 +1,25 @@ +from_version: "0.9.134" +to_version: "0.9.135" +description: > + Fleet restructure rename: features.franchise config key renamed to features.fleet. +changes: + - id: features-franchise-to-fleet + description: > + The features.franchise config key has been renamed to features.fleet. + Any project config using features.franchise: true must be updated. + instruction: | + In your project's .autoskillit/config.yaml, replace: + features: + franchise: true + with: + features: + fleet: true + The old 'franchise' key is no longer recognized and will have no effect. + detect: + key: features.franchise + example_before: | + features: + franchise: true + example_after: | + features: + fleet: true diff --git a/src/autoskillit/migrations/0.9.158-to-0.9.159.yaml b/src/autoskillit/migrations/0.9.158-to-0.9.159.yaml new file mode 100644 index 0000000000..9b1fbded08 --- /dev/null +++ b/src/autoskillit/migrations/0.9.158-to-0.9.159.yaml @@ -0,0 +1,43 @@ +from_version: "0.9.158" +to_version: "0.9.159" +description: > + FleetErrorCode prefix rationalization: all error codes now use the fleet_ prefix. + Persisted campaign summary JSON containing error_records with old code values must + be updated. +changes: + - id: fleet-error-code-prefix-normalization + description: > + FleetErrorCode enum members L2_*, DISPATCH_BUDGET_EXCEEDED, QUOTA_EXHAUSTED, and CLEANUP_FAILED + have been renamed to FLEET_L2_*, FLEET_BUDGET_EXCEEDED, FLEET_QUOTA_EXHAUSTED, and + FLEET_CLEANUP_FAILED respectively. Their string values changed from l2_*/dispatch_*/ + quota_exhausted/cleanup_failed to fleet_l2_*/fleet_budget_exceeded/ + fleet_quota_exhausted/fleet_cleanup_failed. + instruction: | + If you have persisted campaign summary JSON files (e.g. written by + serialize_campaign_summary), update any error_records[*].code values: + "l2_timeout" -> "fleet_l2_timeout" + "l2_no_result_block" -> "fleet_l2_no_result_block" + "l2_parse_failed" -> "fleet_l2_parse_failed" + "l2_startup_or_crash" -> "fleet_l2_startup_or_crash" + "dispatch_budget_exceeded" -> "fleet_budget_exceeded" + "quota_exhausted" -> "fleet_quota_exhausted" + "cleanup_failed" -> "fleet_cleanup_failed" + + Any code using FleetErrorCode enum members by name must update: + FleetErrorCode.L2_TIMEOUT -> FleetErrorCode.FLEET_L2_TIMEOUT + FleetErrorCode.L2_NO_RESULT_BLOCK -> FleetErrorCode.FLEET_L2_NO_RESULT_BLOCK + FleetErrorCode.L2_PARSE_FAILED -> FleetErrorCode.FLEET_L2_PARSE_FAILED + FleetErrorCode.L2_STARTUP_OR_CRASH -> FleetErrorCode.FLEET_L2_STARTUP_OR_CRASH + FleetErrorCode.DISPATCH_BUDGET_EXCEEDED -> FleetErrorCode.FLEET_BUDGET_EXCEEDED + FleetErrorCode.QUOTA_EXHAUSTED -> FleetErrorCode.FLEET_QUOTA_EXHAUSTED + FleetErrorCode.CLEANUP_FAILED -> FleetErrorCode.FLEET_CLEANUP_FAILED + example_before: | + error_records: + - dispatch_name: "my-dispatch" + code: "l2_timeout" + message: "Timed out after 300s" + example_after: | + error_records: + - dispatch_name: "my-dispatch" + code: "fleet_l2_timeout" + message: "Timed out after 300s" diff --git a/src/autoskillit/migrations/0.9.376-to-0.9.377.yaml b/src/autoskillit/migrations/0.9.376-to-0.9.377.yaml new file mode 100644 index 0000000000..d8ce428cd6 --- /dev/null +++ b/src/autoskillit/migrations/0.9.376-to-0.9.377.yaml @@ -0,0 +1,46 @@ +from_version: "0.9.376" +to_version: "0.9.377" +description: > + Rename L2 identifiers to L3 in fleet dispatch code. Campaign state + field names change; read_state accepts both old and new names. +changes: + - id: fleet-dispatch-record-field-rename + description: > + DispatchRecord fields renamed from l2_* to l3_*. + read_state falls back to old names for existing state files. + NOTE: l3_* fields were subsequently renamed to dispatched_* in 0.9.484 + (see 0.9.483-to-0.9.484.yaml). The fallback chain is now: + dispatched_* → l3_* → l2_*. + instruction: | + No manual action required — read_state auto-migrates on read. + After the first write-back, state files use new field names. + Old field names: l2_session_id, l2_session_log_dir, l2_pid, + l2_starttime_ticks, l2_boot_id. + New field names (0.9.484+): dispatched_session_id, dispatched_session_log_dir, + dispatched_pid, dispatched_starttime_ticks, dispatched_boot_id. + + - id: fleet-error-code-value-rename + description: > + FleetErrorCode string values renamed from fleet_l2_* to fleet_l3_*. + instruction: | + Old values: fleet_l2_timeout, fleet_l2_no_result_block, + fleet_l2_parse_failed, fleet_l2_startup_or_crash. + New values: fleet_l3_timeout, fleet_l3_no_result_block, + fleet_l3_parse_failed, fleet_l3_startup_or_crash. + Persisted error_records[*].code in campaign summaries will + retain old values until rewritten. + + - id: fleet-env-var-rename + description: > + Environment variable AUTOSKILLIT_L2_TOOL_TAGS renamed to + AUTOSKILLIT_L3_TOOL_TAGS. + instruction: | + Internal-only env var. No user action required. + + - id: fleet-sentinel-protocol-rename + description: > + Result block sentinel markers renamed from l2-result to l3-result. + Completion marker renamed from L2_DONE to L3_DONE. + instruction: | + In-flight campaigns using old sentinel format will not parse + correctly after upgrade. Complete all campaigns before upgrading. diff --git a/src/autoskillit/migrations/0.9.388-to-0.9.389.yaml b/src/autoskillit/migrations/0.9.388-to-0.9.389.yaml new file mode 100644 index 0000000000..7a67b8fc55 --- /dev/null +++ b/src/autoskillit/migrations/0.9.388-to-0.9.389.yaml @@ -0,0 +1,34 @@ +from_version: "0.9.388" +to_version: "0.9.389" +description: > + Rename L1 session terminology from "leaf" to "skill". SessionType.LEAF + is replaced by SessionType.SKILL; the wire value changes from "leaf" to + "skill". The old "leaf" value is recognized with a DeprecationWarning for + one release cycle. +changes: + - id: session-type-leaf-to-skill + description: > + SessionType.LEAF renamed to SessionType.SKILL; wire value "leaf" + deprecated in favor of "skill". + instruction: | + AUTOSKILLIT_SESSION_TYPE=leaf is still accepted but emits a + DeprecationWarning and maps to SessionType.SKILL. Update any + scripts or config that sets SESSION_TYPE=leaf to use SESSION_TYPE=skill. + The old value will be removed in a future release. + + - id: build-leaf-headless-cmd-renamed + description: > + build_leaf_headless_cmd() renamed to build_skill_session_cmd() in + autoskillit.execution.commands. + instruction: | + Update any direct callers of build_leaf_headless_cmd() to use + build_skill_session_cmd(). Function signature is unchanged. + + - id: leaf-orchestration-guard-renamed + description: > + hooks/guards/leaf_orchestration_guard.py retired; + hooks/guards/skill_orchestration_guard.py is the active guard. + instruction: | + No manual action required. hook_registry.py references the new path. + leaf_orchestration_guard.py is listed in RETIRED_SCRIPT_BASENAMES and + will be removed from existing installs automatically on next init. diff --git a/src/autoskillit/migrations/0.9.483-to-0.9.484.yaml b/src/autoskillit/migrations/0.9.483-to-0.9.484.yaml new file mode 100644 index 0000000000..6fc8b019b6 --- /dev/null +++ b/src/autoskillit/migrations/0.9.483-to-0.9.484.yaml @@ -0,0 +1,41 @@ +from_version: "0.9.483" +to_version: "0.9.484" +description: > + Rename L3 dispatch identifiers to dispatched_* in fleet state and summary. + The l3_* prefix reflected an internal chain perspective (L1→L2→L3) + that contradicts the user-facing L2/L3 terminology where L2=food-truck + and L3=orchestrator. Renaming clarifies ownership provenance. +changes: + - id: fleet-dispatch-record-field-rename + description: > + DispatchRecord fields renamed from l3_* to dispatched_*. + read_state falls back through three generations: dispatched_* → + l3_* → l2_* for existing state files. + instruction: | + No manual action required — read_state auto-migrates on read. + After the first write-back, state files use new field names. + Old field names: l3_session_id, l3_session_log_dir, l3_pid, + l3_starttime_ticks, l3_boot_id. + New field names: dispatched_session_id, dispatched_session_log_dir, + dispatched_pid, dispatched_starttime_ticks, dispatched_boot_id. + + - id: fleet-mark-dispatch-running-param-rename + description: > + mark_dispatch_running() parameter l3_pid renamed to dispatched_pid. + instruction: | + No manual action required — all call sites updated in same release. + + - id: fleet-resume-decision-field-rename + description: > + ResumeDecision.l3_session_id renamed to dispatched_session_id. + instruction: | + No manual action required — all call sites updated in same release. + + - id: fleet-summary-schema-rename + description: > + Campaign summary per_dispatch[*].l3_session_id and + error_records[*].l3_session_id renamed to dispatched_session_id. + parse_campaign_summary falls back through three generations. + instruction: | + No manual action required — parse_campaign_summary auto-migrates + on parse. Old summaries retain l3_session_id keys until rewritten. diff --git a/src/autoskillit/migrations/0.9.525-to-0.9.526.yaml b/src/autoskillit/migrations/0.9.525-to-0.9.526.yaml new file mode 100644 index 0000000000..b662046dc0 --- /dev/null +++ b/src/autoskillit/migrations/0.9.525-to-0.9.526.yaml @@ -0,0 +1,12 @@ +from_version: "0.9.525" +to_version: "0.9.526" +description: > + Rename L3 identifiers to food-truck in session/gating layer. +changes: + - id: food-truck-env-var-rename + description: > + Environment variable AUTOSKILLIT_L3_TOOL_TAGS renamed to + AUTOSKILLIT_FOOD_TRUCK_TOOL_TAGS. Internal-only env var passed + between dispatch_food_truck and child sessions. + instruction: | + Internal-only env var. No user action required. diff --git a/src/autoskillit/migrations/0.9.526-to-0.9.527.yaml b/src/autoskillit/migrations/0.9.526-to-0.9.527.yaml new file mode 100644 index 0000000000..b164b08fd9 --- /dev/null +++ b/src/autoskillit/migrations/0.9.526-to-0.9.527.yaml @@ -0,0 +1,30 @@ +from_version: "0.9.526" +to_version: "0.9.527" +description: > + Skill rename: vis-lens-domain-norms → vis-lens-methodology-norms. +changes: + - id: vis-lens-methodology-norms-rename + description: "Skill renamed: vis-lens-domain-norms → vis-lens-methodology-norms" + instruction: | + If your .autoskillit/recipes/*.yaml files or custom Taskfile invoke + vis-lens-domain-norms explicitly: + + 1. Replace '/autoskillit:vis-lens-domain-norms' with + '/autoskillit:vis-lens-methodology-norms' in every skill_command step. + 2. Update any downstream captures, routing references, or log greps that + hardcoded the old skill name. + + The skill's input/output contract is unchanged; only the slug is different. + Most users with only bundled-recipe workflows will see no disruption — the + rename is transparent inside the bundled research.yaml. + detect: + tool: run_skill + skill_pattern: "vis-lens-domain-norms" + example_before: | + - tool: run_skill + with: + skill_command: "/autoskillit:vis-lens-domain-norms ${{ context.experiment_plan }}" + example_after: | + - tool: run_skill + with: + skill_command: "/autoskillit:vis-lens-methodology-norms ${{ context.experiment_plan }}" diff --git a/src/autoskillit/pipeline/CLAUDE.md b/src/autoskillit/pipeline/CLAUDE.md new file mode 100644 index 0000000000..c82aaf18cc --- /dev/null +++ b/src/autoskillit/pipeline/CLAUDE.md @@ -0,0 +1,26 @@ +# pipeline/ + +IL-1 pipeline state — per-tool-call state containers, gate logic, audit log, telemetry. + +## Files + +| File | Purpose | +|------|---------| +| `__init__.py` | Re-exports public protocol implementations | +| `audit.py` | `FailureRecord`, `DefaultAuditLog` | +| `background.py` | `DefaultBackgroundSupervisor` | +| `context.py` | `ToolContext` DI container | +| `gate.py` | `DefaultGateState`, `gate_error_result` | +| `github_api_log.py` | `DefaultGitHubApiLog` — session-scoped GitHub API request accumulator | +| `mcp_response.py` | Per-tool response size tracking | +| `telemetry_fmt.py` | Canonical token/timing display | +| `timings.py` | `DefaultTimingLog` | +| `tokens.py` | `DefaultTokenLog` | +| `pr_gates.py` | `is_ci_passing`, `is_review_passing`, `partition_prs` | + +## Architecture Notes + +`ToolContext` is the composition root injected into every MCP tool handler via +`server/_factory.py:make_context()`. All implementations satisfy protocols defined in +`core/types/`. `gate.py` is the sole source of structured gate-error results; no tool +handler constructs gate errors directly. diff --git a/src/autoskillit/pipeline/__init__.py b/src/autoskillit/pipeline/__init__.py index fbc5e1e49e..f16f57d3b8 100644 --- a/src/autoskillit/pipeline/__init__.py +++ b/src/autoskillit/pipeline/__init__.py @@ -1,11 +1,17 @@ -"""pipeline/ L1 package: audit log, token tracking, gate policy, and ToolContext. +"""pipeline/ IL-1 package: audit log, token tracking, gate policy, and ToolContext. Re-exports the full public surface of the four pipeline sub-modules. Only pipeline/context.py imports from config/; the other three modules depend only on autoskillit.core.*. """ -from autoskillit.core import FailureRecord, is_protected_branch +from autoskillit.core import ( + GATED_TOOLS, + UNGATED_TOOLS, + FailureRecord, + fleet_error, + is_protected_branch, +) from autoskillit.pipeline.audit import ( COMMAND_MAX_LEN, STDERR_MAX_LEN, @@ -18,12 +24,11 @@ ) from autoskillit.pipeline.context import ToolContext from autoskillit.pipeline.gate import ( - GATED_TOOLS, - UNGATED_TOOLS, DefaultGateState, gate_error_result, headless_error_result, ) +from autoskillit.pipeline.github_api_log import DefaultGitHubApiLog from autoskillit.pipeline.mcp_response import DefaultMcpResponseLog, McpResponseEntry from autoskillit.pipeline.pr_gates import ( is_ci_passing, @@ -55,6 +60,7 @@ "DefaultGateState", "GATED_TOOLS", "UNGATED_TOOLS", + "fleet_error", "gate_error_result", "headless_error_result", # telemetry_fmt @@ -65,6 +71,8 @@ "write_status", # context "ToolContext", + # github_api_log + "DefaultGitHubApiLog", # pr_gates "is_ci_passing", "is_review_passing", diff --git a/src/autoskillit/pipeline/audit.py b/src/autoskillit/pipeline/audit.py index 81742bac9f..e148ceb663 100644 --- a/src/autoskillit/pipeline/audit.py +++ b/src/autoskillit/pipeline/audit.py @@ -69,6 +69,9 @@ def _iter_session_log_entries( filename: str, cwd_filter: str = "", kitchen_id_filter: str = "", + campaign_id_filter: str = "", + order_id_filter: str = "", + dispatch_id_filter: str = "", ) -> Iterator[Path]: """Yield per-session file paths from sessions.jsonl that pass the filters. @@ -91,6 +94,17 @@ def _iter_session_log_entries( sessions written before the rename. Empty string disables kitchen_id filtering. When both cwd_filter and kitchen_id_filter are provided, both must match (AND logic). + campaign_id_filter: If non-empty, only yield sessions whose ``campaign_id`` field + matches this string exactly. Uses ``.get()`` for backward + compatibility with pre-existing index entries lacking the field. + All active filters apply as AND logic. + order_id_filter: If non-empty, only yield sessions whose ``order_id`` field + matches this string exactly. Supersedes cwd_filter for fleet + sessions where a single order spans multiple clone directories. + All active filters apply as AND logic. + dispatch_id_filter: If non-empty, only yield sessions whose ``dispatch_id`` field + matches this string exactly. + All active filters apply as AND logic. Yields: Path to each matching per-session file that exists. @@ -135,6 +149,15 @@ def _iter_session_log_entries( if entry_kitchen_id != kitchen_id_filter: continue + if campaign_id_filter and idx.get("campaign_id") != campaign_id_filter: + continue + + if order_id_filter and idx.get("order_id") != order_id_filter: + continue + + if dispatch_id_filter and idx.get("dispatch_id") != dispatch_id_filter: + continue + dir_name = idx.get("dir_name", "") if not dir_name: continue @@ -234,6 +257,9 @@ def load_from_log_dir( since: str = "", cwd_filter: str = "", kitchen_id_filter: str = "", + campaign_id_filter: str = "", + order_id_filter: str = "", + dispatch_id_filter: str = "", ) -> int: """Reconstruct failure records from persisted session logs. @@ -244,12 +270,22 @@ def load_from_log_dir( cwd_filter: if non-empty, only sessions whose cwd matches are loaded. kitchen_id_filter: if non-empty, only sessions whose kitchen_id matches are loaded. Falls back to pipeline_id for sessions written before the rename. + campaign_id_filter: if non-empty, only sessions whose campaign_id matches are loaded. + order_id_filter: if non-empty, only sessions whose order_id matches are loaded. + dispatch_id_filter: if non-empty, only sessions whose dispatch_id matches are loaded. Returns the count of session directories successfully loaded. """ count = 0 for al_path in _iter_session_log_entries( - log_root, since, "audit_log.json", cwd_filter, kitchen_id_filter + log_root, + since, + "audit_log.json", + cwd_filter, + kitchen_id_filter, + campaign_id_filter, + order_id_filter, + dispatch_id_filter, ): try: data = json.loads(al_path.read_text(encoding="utf-8")) diff --git a/src/autoskillit/pipeline/background.py b/src/autoskillit/pipeline/background.py index 32c339462b..e83ff408da 100644 --- a/src/autoskillit/pipeline/background.py +++ b/src/autoskillit/pipeline/background.py @@ -3,13 +3,13 @@ from __future__ import annotations import asyncio -import json from collections.abc import Callable, Coroutine from datetime import UTC, datetime from pathlib import Path from typing import Any -from autoskillit.core import atomic_write, get_logger +from autoskillit.core import AuditLog, SupportsLogger, atomic_write, get_logger +from autoskillit.core import fast_dumps as _fast_dumps logger = get_logger(__name__) @@ -24,8 +24,8 @@ class DefaultBackgroundSupervisor: def __init__( self, - audit: Any | None = None, - log: Any | None = None, + audit: AuditLog | None = None, + log: SupportsLogger | None = None, ) -> None: self._tasks: set[asyncio.Task[Any]] = set() self._audit = audit @@ -129,6 +129,6 @@ def write_status(path: Path | None, status: str, *, error: str | None = None) -> } if error is not None: payload["error"] = error - atomic_write(path, json.dumps(payload, indent=2)) + atomic_write(path, _fast_dumps(payload, indent=True)) except Exception: logger.debug("write_status failed", path=str(path), exc_info=True) diff --git a/src/autoskillit/pipeline/context.py b/src/autoskillit/pipeline/context.py index e8526c732b..1340256260 100644 --- a/src/autoskillit/pipeline/context.py +++ b/src/autoskillit/pipeline/context.py @@ -9,23 +9,29 @@ from dataclasses import dataclass, field from pathlib import Path +from typing import Any from autoskillit.config import AutomationConfig from autoskillit.core import ( AuditLog, BackgroundSupervisor, + CampaignProtector, CIRunScope, CIWatcher, CloneManager, DatabaseReader, + FleetLock, GateState, + GitHubApiLog, GitHubFetcher, HeadlessExecutor, McpResponseLog, MergeQueueWatcher, MigrationService, OutputPatternResolver, + PluginSource, QuotaRefreshTask, + ReadOnlyResolver, RecipeRepository, SessionSkillManager, SkillResolver, @@ -40,6 +46,9 @@ from autoskillit.pipeline.background import DefaultBackgroundSupervisor from autoskillit.pipeline.mcp_response import DefaultMcpResponseLog +# Must-supply-or-raise: fields defaulting to _MISSING are required by __post_init__. +_MISSING: Any = object() + @dataclass class ToolContext: @@ -57,8 +66,8 @@ class ToolContext: timing_log: TimingLog — per-step wall-clock duration tracking response_log: McpResponseLog — per-tool MCP response size tracking gate: GateState — enables/disables gated tools - plugin_dir: Absolute path string to the autoskillit package directory, or None if - the marketplace plugin is installed and `--plugin-dir` should be omitted. + plugin_source: PluginSource — DirectInstall (dev/editable) or MarketplaceInstall. + Encodes how autoskillit is loaded into Claude Code sessions. runner: SubprocessRunner implementation (DefaultSubprocessRunner in production, MockSubprocessRunner in tests) executor: HeadlessExecutor — runs headless Claude Code sessions @@ -77,14 +86,18 @@ class ToolContext: to the current kitchen session lifetime. active_recipe_packs: frozenset[str] | None — pack names declared by the loaded recipe (frozenset() when kitchen open but no recipe loaded; None when closed) - temp_dir: Resolved temp directory for this project. MUST be supplied explicitly - by callers outside make_context(). The default_factory falls back to - Path.cwd() / ".autoskillit" / "temp", which is cwd-dependent and - ignores the configured workspace.temp_dir. + active_recipe_features: frozenset[str] | None — feature names declared by the loaded recipe + (frozenset() when kitchen open but no recipe loaded; None when closed) + temp_dir: Resolved temp directory for this project. Sentinel-guarded: raises + TypeError if not supplied explicitly. Use make_context() or pass + temp_dir=. token_factory: Optional callable that resolves a GitHub token via the config → GITHUB_TOKEN env → gh CLI fallback chain. Set by make_context(); None in test ToolContext instances unless explicitly provided. + project_dir: Resolved project root directory. Sentinel-guarded: raises TypeError + if not supplied explicitly. Use make_context() or pass + project_dir=. """ config: AutomationConfig @@ -92,15 +105,10 @@ class ToolContext: token_log: TokenLog timing_log: TimingLog gate: GateState - plugin_dir: str | None + plugin_source: PluginSource runner: SubprocessRunner | None - # Always supply temp_dir explicitly when constructing ToolContext directly. - # The default captures Path.cwd() at field-instantiation time, which is - # cwd-dependent and will differ from the configured workspace.temp_dir. - # Production callers must use make_context() (server/_factory.py), which - # resolves temp_dir from config via resolve_temp_dir(). Direct construction - # (e.g. in tests) must override this field before any file I/O that uses it. - temp_dir: Path = field(default_factory=lambda: Path.cwd() / ".autoskillit" / "temp") + temp_dir: Path = field(default=_MISSING) + project_dir: Path = field(default=_MISSING) response_log: McpResponseLog = field(default_factory=DefaultMcpResponseLog) executor: HeadlessExecutor | None = field(default=None) tester: TestRunner | None = field(default=None) @@ -112,17 +120,37 @@ class ToolContext: github_client: GitHubFetcher | None = field(default=None) ci_watcher: CIWatcher | None = field(default=None) merge_queue_watcher: MergeQueueWatcher | None = field(default=None) + github_api_log: GitHubApiLog | None = field(default=None) background: BackgroundSupervisor | None = field(default=None) output_pattern_resolver: OutputPatternResolver | None = field(default=None) write_expected_resolver: WriteExpectedResolver | None = field(default=None) + read_only_resolver: ReadOnlyResolver | None = field(default=None) session_skill_manager: SessionSkillManager | None = field(default=None) skill_resolver: SkillResolver | None = field(default=None) + recipe_name: str = field(default="") + recipe_content_hash: str = field(default="") + recipe_composite_hash: str = field(default="") + recipe_version: str = field(default="") kitchen_id: str = field(default="") active_recipe_packs: frozenset[str] | None = field(default_factory=lambda: None) + active_recipe_features: frozenset[str] | None = field(default_factory=lambda: None) quota_refresh_task: QuotaRefreshTask | None = field(default=None) token_factory: TokenFactory | None = field(default=None) + fleet_lock: FleetLock | None = field(default=None) + build_protected_campaign_ids: CampaignProtector | None = field(default=None) + ephemeral_root: Path | None = field(default_factory=lambda: None) def __post_init__(self) -> None: + if self.temp_dir is _MISSING: + raise TypeError( + "temp_dir must be supplied explicitly — do not rely on defaults. " + "Use make_context() or pass temp_dir= directly." + ) + if self.project_dir is _MISSING: + raise TypeError( + "project_dir must be supplied explicitly — do not rely on defaults. " + "Use make_context() or pass project_dir= directly." + ) if self.background is None: self.background = DefaultBackgroundSupervisor(audit=self.audit) diff --git a/src/autoskillit/pipeline/gate.py b/src/autoskillit/pipeline/gate.py index 61a7deb4ec..1289f29fdb 100644 --- a/src/autoskillit/pipeline/gate.py +++ b/src/autoskillit/pipeline/gate.py @@ -1,10 +1,10 @@ """Gate policy constants for AutoSkillit MCP tools. -L1 pipeline module. Declares which tools are gated vs. ungated and provides +IL-1 pipeline module. Declares which tools are gated vs. ungated and provides the canonical error response for a closed gate. -GATED_TOOLS and UNGATED_TOOLS are sourced from autoskillit.core.types (L0) -so that L2 modules (recipe, migration) can also reference the tool registry +GATED_TOOLS and UNGATED_TOOLS are sourced from autoskillit.core.types (IL-0) +so that IL-2 modules (recipe, migration) can also reference the tool registry without violating layer ordering. """ @@ -45,7 +45,7 @@ def gate_error_result(message: str | None = None) -> str: 'tools not enabled' message for gate-closed errors. Hardcodes retry_reason as "none" (the StrEnum value of RetryReason.NONE) - to preserve the L0 zero-internal-imports constraint. + to preserve the IL-0 zero-internal-imports constraint. """ return json.dumps( { @@ -69,8 +69,9 @@ def gate_error_result(message: str | None = None) -> str: _DEFAULT_HEADLESS_MESSAGE = ( "This tool cannot be called from headless sessions. " - "Headless workers (Tier 2) may only use native Claude Code tools and " - "HEADLESS_TOOLS. Orchestration tools are reserved for Tier 1 sessions." + "L1 headless workers may only use native Claude Code tools and " + "HEADLESS_TOOLS. Orchestration tools are reserved for L2+ sessions " + "(orchestrator or fleet)." ) diff --git a/src/autoskillit/pipeline/github_api_log.py b/src/autoskillit/pipeline/github_api_log.py new file mode 100644 index 0000000000..5d3c542a5a --- /dev/null +++ b/src/autoskillit/pipeline/github_api_log.py @@ -0,0 +1,153 @@ +"""Session-scoped GitHub API request accumulator. + +Mirrors the tokens.py / timings.py pattern: a lock-guarded list of entries +aggregated on demand by to_usage(). Flushed to github_api_usage.json at +session log-write time via flush_session_log(). +""" + +from __future__ import annotations + +import asyncio +from dataclasses import dataclass, field +from typing import Any + +import regex as re + + +@dataclass +class GitHubApiEntry: + method: str + path: str + status_code: int + latency_ms: float + rate_limit_remaining: int + rate_limit_used: int + rate_limit_reset: int + timestamp: str + source: str # "httpx" or "gh_cli" + subcommand: str = field(default="") + + +_CATEGORY_PATTERNS: list[tuple[re.Pattern[str], str]] = [ + (re.compile(r"^/repos/[^/]+/[^/]+/issues"), "issues"), + (re.compile(r"^/repos/[^/]+/[^/]+/pulls"), "pulls"), + (re.compile(r"^/repos/[^/]+/[^/]+/actions"), "actions"), + (re.compile(r"^/search/"), "search"), + (re.compile(r"^/graphql$"), "graphql"), +] + + +def _categorize(path: str) -> str: + for pattern, category in _CATEGORY_PATTERNS: + if pattern.match(path): + return category + return "other" + + +class DefaultGitHubApiLog: + """asyncio.Lock-guarded accumulator of GitHub API call entries.""" + + def __init__(self) -> None: + self._lock = asyncio.Lock() + self._entries: list[GitHubApiEntry] = [] + + async def record_httpx( + self, + *, + method: str, + path: str, + status_code: int, + latency_ms: float, + rate_limit_remaining: int, + rate_limit_used: int, + rate_limit_reset: int, + timestamp: str, + ) -> None: + async with self._lock: + self._entries.append( + GitHubApiEntry( + method=method, + path=path, + status_code=status_code, + latency_ms=latency_ms, + rate_limit_remaining=rate_limit_remaining, + rate_limit_used=rate_limit_used, + rate_limit_reset=rate_limit_reset, + timestamp=timestamp, + source="httpx", + ) + ) + + async def record_gh_cli( + self, + *, + subcommand: str, + exit_code: int, + latency_ms: float, + timestamp: str, + ) -> None: + async with self._lock: + self._entries.append( + GitHubApiEntry( + method="", + path="", + status_code=exit_code, + latency_ms=latency_ms, + rate_limit_remaining=-1, + rate_limit_used=0, + rate_limit_reset=0, + timestamp=timestamp, + source="gh_cli", + subcommand=subcommand, + ) + ) + + def to_usage(self, session_id: str) -> dict[str, Any] | None: + if not self._entries: + return None + by_category: dict[str, int] = {} + by_source: dict[str, int] = {} + total_latency = 0.0 + min_remaining: int | None = None + errors: dict[str, int] = {"4xx": 0, "5xx": 0} + timestamps = [e.timestamp for e in self._entries if e.timestamp] + for e in self._entries: + cat = _categorize(e.path) if e.source == "httpx" else "other" + by_category[cat] = by_category.get(cat, 0) + 1 + by_source[e.source] = by_source.get(e.source, 0) + 1 + total_latency += e.latency_ms + if e.source == "httpx" and e.rate_limit_remaining >= 0: + if min_remaining is None or e.rate_limit_remaining < min_remaining: + min_remaining = e.rate_limit_remaining + if e.source == "httpx": + if 400 <= e.status_code < 500: + errors["4xx"] += 1 + elif 500 <= e.status_code < 600: + errors["5xx"] += 1 + count = len(self._entries) + return { + "session_id": session_id, + "total_requests": count, + "by_category": by_category, + "by_source": by_source, + "total_latency_ms": round(total_latency, 2), + "avg_latency_ms": round(total_latency / count, 2) if count else 0.0, + "min_rate_limit_remaining": min_remaining, + "errors": errors, + "first_request_ts": min(timestamps) if timestamps else None, + "last_request_ts": max(timestamps) if timestamps else None, + } + + def drain(self, session_id: str) -> dict[str, Any] | None: + """Snapshot current entries as a usage dict and clear the accumulator. + + List replacement (not .clear()) is used so a concurrent record_* call that + appended to the old list after drain() snapshotted it will appear in the + next session's drain rather than being silently dropped. + """ + usage = self.to_usage(session_id) + self._entries = [] + return usage + + def clear(self) -> None: + self._entries = [] diff --git a/src/autoskillit/pipeline/pr_gates.py b/src/autoskillit/pipeline/pr_gates.py index 95850d745f..b8e59462eb 100644 --- a/src/autoskillit/pipeline/pr_gates.py +++ b/src/autoskillit/pipeline/pr_gates.py @@ -6,8 +6,27 @@ from __future__ import annotations +import regex as re + _CI_PASSING_CONCLUSIONS = frozenset({"success", "skipped", "neutral"}) +_PIPELINE_SIG_RE = re.compile(r"``. + PRs that pass CI and review gates but lack this signature are surfaced in + the ``provenance_blocked_prs`` bucket from ``partition_prs`` for operator + review — they are not automatically blocked from merge. + """ + return bool(_PIPELINE_SIG_RE.search(pr_body or "")) def partition_prs( @@ -44,18 +75,21 @@ def partition_prs( checks_by_number: dict[int, list[dict]], reviews_by_number: dict[int, list[dict]], ) -> dict[str, list[dict]]: - """Partition PRs into eligible, ci_blocked, and review_blocked lists. + """Partition PRs into eligible, ci_blocked, review_blocked, and provenance_blocked lists. Each PR in ``prs`` must have at minimum: ``number`` (int) and ``title`` (str). Returns a dict with keys: - - eligible_prs: list of PR dicts that passed both gates + - eligible_prs: list of PR dicts that passed both gates and carry the pipeline signature - ci_blocked_prs: list of {number, title, reason} for CI-failing PRs - review_blocked_prs: list of {number, title, reason} for review-blocked PRs + - provenance_blocked_prs: list of PR dicts that passed CI+review gates but lack the + autoskillit pipeline signature — surfaced for operator review, not auto-blocked """ eligible: list[dict] = [] ci_blocked: list[dict] = [] review_blocked: list[dict] = [] + provenance_blocked: list[dict] = [] for pr in prs: number = pr.get("number") @@ -78,15 +112,20 @@ def partition_prs( reviews = reviews_by_number.get(number, []) if not is_review_passing(reviews): - count = sum(1 for r in reviews if r.get("state") == "CHANGES_REQUESTED") - reason = f"{count} unresolved CHANGES_REQUESTED review(s)" + count = sum(1 for r in reviews if r.get("state") == _BLOCKING_REVIEW_STATE) + reason = f"{count} unresolved {_BLOCKING_REVIEW_STATE} review(s)" review_blocked.append({"number": number, "title": title, "reason": reason}) continue + if not is_pipeline_sourced(pr.get("body", "")): + provenance_blocked.append(pr) + continue + eligible.append(pr) return { "eligible_prs": eligible, "ci_blocked_prs": ci_blocked, "review_blocked_prs": review_blocked, + "provenance_blocked_prs": provenance_blocked, } diff --git a/src/autoskillit/pipeline/telemetry_fmt.py b/src/autoskillit/pipeline/telemetry_fmt.py index fc4392f671..6538f85462 100644 --- a/src/autoskillit/pipeline/telemetry_fmt.py +++ b/src/autoskillit/pipeline/telemetry_fmt.py @@ -8,15 +8,18 @@ from __future__ import annotations -from autoskillit.core import TerminalColumn, _render_terminal_table +from autoskillit.core import ModelTotalEntry, TerminalColumn, _render_terminal_table _TOKEN_COLUMNS = ( TerminalColumn("STEP", max_width=40, align="<"), + TerminalColumn("MODEL", max_width=30, align="<"), + TerminalColumn("COUNT", max_width=5, align=">"), TerminalColumn("UNCACHED", max_width=10, align=">"), TerminalColumn("OUTPUT", max_width=10, align=">"), TerminalColumn("CACHE_RD", max_width=10, align=">"), + TerminalColumn("PEAK_CTX", max_width=10, align=">"), + TerminalColumn("TURNS", max_width=7, align=">"), TerminalColumn("CACHE_WR", max_width=10, align=">"), - TerminalColumn("COUNT", max_width=7, align=">"), TerminalColumn("TIME", max_width=8, align=">"), ) @@ -26,6 +29,121 @@ TerminalColumn("INVOCATIONS", max_width=11, align=">"), ) +_EFFICIENCY_COLUMNS = ( + TerminalColumn("STEP", max_width=40, align="<"), + TerminalColumn("LOC_CHG", max_width=8, align=">"), + TerminalColumn("RD/LOC", max_width=8, align=">"), + TerminalColumn("WR/LOC", max_width=8, align=">"), + TerminalColumn("OUT/LOC", max_width=8, align=">"), +) + + +_EFFICIENCY_MD_LABELS: dict[str, str] = { + "STEP": "Step", + "LOC_CHG": "LoC Changed", + "RD/LOC": "cache_read/LoC", + "WR/LOC": "cache_write/LoC", + "OUT/LOC": "output/LoC", +} + +_eff_md_headers = [_EFFICIENCY_MD_LABELS[c.label] for c in _EFFICIENCY_COLUMNS] +_EFFICIENCY_MD_HEADER = "| " + " | ".join(_eff_md_headers) + " |" +_EFFICIENCY_MD_SEP = "|" + "|".join("-" * (len(h) + 2) for h in _eff_md_headers) + "|" + +_TOKEN_MD_LABELS: dict[str, str] = { + "STEP": "Step", + "MODEL": "Model", + "COUNT": "count", + "UNCACHED": "uncached", + "OUTPUT": "output", + "CACHE_RD": "cache_read", + "PEAK_CTX": "peak_ctx", + "TURNS": "turns", + "CACHE_WR": "cache_write", + "TIME": "time", +} + +_tok_md_headers = [_TOKEN_MD_LABELS[c.label] for c in _TOKEN_COLUMNS] +_TOKEN_MD_HEADER = "| " + " | ".join(_tok_md_headers) + " |" +_TOKEN_MD_SEP = "|" + "|".join("-" * (len(h) + 2) for h in _tok_md_headers) + "|" + +_TOKEN_DISPLAY_FIELDS: frozenset[str] = frozenset( + { + "step_name", + "model", + "input_tokens", + "output_tokens", + "cache_creation_input_tokens", + "cache_read_input_tokens", + "invocation_count", + "peak_context", + "turn_count", + } +) + +_TOKEN_EXCLUDED_FIELDS: frozenset[str] = frozenset( + { + "elapsed_seconds", + "loc_insertions", + "loc_deletions", + } +) + +_TOKEN_FIELD_TO_COLUMN: dict[str, str] = { + "step_name": "STEP", + "model": "MODEL", + "input_tokens": "UNCACHED", + "output_tokens": "OUTPUT", + "cache_read_input_tokens": "CACHE_RD", + "peak_context": "PEAK_CTX", + "turn_count": "TURNS", + "cache_creation_input_tokens": "CACHE_WR", + "invocation_count": "COUNT", +} + +_TIMING_MD_LABELS: dict[str, str] = { + "STEP": "Step", + "DURATION": "Duration", + "INVOCATIONS": "Invocations", +} + +_timing_md_headers = [_TIMING_MD_LABELS[c.label] for c in _TIMING_COLUMNS] +_TIMING_MD_HEADER = "| " + " | ".join(_timing_md_headers) + " |" +_TIMING_MD_SEP = "|" + "|".join("-" * (len(h) + 2) for h in _timing_md_headers) + "|" + + +_MODEL_COLUMNS = ( + TerminalColumn("MODEL", max_width=30, align="<"), + TerminalColumn("STEPS", max_width=7, align=">"), + TerminalColumn("INPUT", max_width=10, align=">"), + TerminalColumn("OUTPUT", max_width=10, align=">"), + TerminalColumn("CACHE_RD", max_width=10, align=">"), + TerminalColumn("CACHE_WR", max_width=10, align=">"), + TerminalColumn("TIME", max_width=8, align=">"), +) + +_MODEL_MD_LABELS: dict[str, str] = { + "MODEL": "Model", + "STEPS": "steps", + "INPUT": "uncached", + "OUTPUT": "output", + "CACHE_RD": "cache_read", + "CACHE_WR": "cache_write", + "TIME": "time", +} + +_model_md_headers = [_MODEL_MD_LABELS[c.label] for c in _MODEL_COLUMNS] +_MODEL_MD_HEADER = "| " + " | ".join(_model_md_headers) + " |" +_MODEL_MD_SEP = "|" + "|".join("-" * (len(h) + 2) for h in _model_md_headers) + "|" + + +def _is_non_anthropic(model: str) -> bool: + return bool(model) and not model.startswith("claude-") + + +def _ratio(tokens: int, loc: int) -> str: + return f"{tokens / loc:.1f}" if loc > 0 else "—" + class TelemetryFormatter: """Stateless formatter for token and timing telemetry data.""" @@ -65,30 +183,42 @@ def format_token_table(steps: list[dict], total: dict) -> str: lines = [ "## Token Usage Summary", "", - "| Step | uncached | output | cache_read | cache_write | count | time |", - "|------|----------|--------|------------|-------------|-------|------|", + _TOKEN_MD_HEADER, + _TOKEN_MD_SEP, ] + has_non_anthropic = False for step in steps: name = step.get("step_name", "?") + model = step.get("model", "") + if _is_non_anthropic(model): + name = f"{name}*" + has_non_anthropic = True + count = step.get("invocation_count", 1) inp = h(step.get("input_tokens", 0)) out = h(step.get("output_tokens", 0)) cache_rd = h(step.get("cache_read_input_tokens", 0)) + peak_ctx = h(step.get("peak_context", 0)) + turns = step.get("turn_count", 0) cache_wr = h(step.get("cache_creation_input_tokens", 0)) - count = step.get("invocation_count", 1) wc = step.get("wall_clock_seconds", step.get("elapsed_seconds", 0.0)) lines.append( - f"| {name} | {inp} | {out} | {cache_rd} | {cache_wr} | {count} | {fmt_dur(wc)} |" + f"| {name} | {model} | {count} | {inp} | {out} | {cache_rd} | {peak_ctx}" + f" | {turns} | {cache_wr} | {fmt_dur(wc)} |" ) total_in = h(total.get("input_tokens", 0)) total_out = h(total.get("output_tokens", 0)) total_cache_rd = h(total.get("cache_read_input_tokens", 0)) + total_peak = h(total.get("peak_context", 0)) total_cache_wr = h(total.get("cache_creation_input_tokens", 0)) total_time = total.get("total_elapsed_seconds", 0.0) lines.append( - f"| **Total** | {total_in} | {total_out} | {total_cache_rd}" - f" | {total_cache_wr} | | {fmt_dur(total_time)} |" + f"| **Total** | | | {total_in} | {total_out} | {total_cache_rd}" + f" | {total_peak} | | {total_cache_wr} | {fmt_dur(total_time)} |" ) + if has_non_anthropic: + lines.append("") + lines.append(r"\* *Step used a non-Anthropic provider; caching behavior may differ.*") return "\n".join(lines) @staticmethod @@ -99,8 +229,8 @@ def format_timing_table(steps: list[dict], total: dict) -> str: lines = [ "## Step Timing Summary", "", - "| Step | Duration | Invocations |", - "|------|----------|-------------|", + _TIMING_MD_HEADER, + _TIMING_MD_SEP, ] for step in steps: name = step.get("step_name", "?") @@ -118,31 +248,46 @@ def format_token_table_terminal(steps: list[dict], total: dict) -> str: h = TelemetryFormatter._humanize fmt_dur = TelemetryFormatter._fmt_duration - rows: list[tuple[str, str, str, str, str, str, str]] = [] + rows: list[tuple[str, str, str, str, str, str, str, str, str, str]] = [] + has_non_anthropic = False for step in steps: + step_name = step.get("step_name", "?") + model = step.get("model", "") + if _is_non_anthropic(model): + step_name = f"{step_name}*" + has_non_anthropic = True rows.append( ( - step.get("step_name", "?"), + step_name, + model, + str(step.get("invocation_count", 1)), h(step.get("input_tokens", 0)), h(step.get("output_tokens", 0)), h(step.get("cache_read_input_tokens", 0)), + h(step.get("peak_context", 0)), + str(step.get("turn_count", 0)), h(step.get("cache_creation_input_tokens", 0)), - str(step.get("invocation_count", 1)), fmt_dur(step.get("wall_clock_seconds", step.get("elapsed_seconds", 0.0))), ) ) total_row = ( "Total", + "", + "", h(total.get("input_tokens", 0)), h(total.get("output_tokens", 0)), h(total.get("cache_read_input_tokens", 0)), - h(total.get("cache_creation_input_tokens", 0)), + h(total.get("peak_context", 0)), "", + h(total.get("cache_creation_input_tokens", 0)), fmt_dur(total.get("total_elapsed_seconds", 0.0)), ) - return _render_terminal_table(_TOKEN_COLUMNS, rows + [total_row]) + result = _render_terminal_table(_TOKEN_COLUMNS, rows + [total_row]) + if has_non_anthropic: + result += "\n* Step used a non-Anthropic provider; caching behavior may differ." + return result @staticmethod def format_timing_table_terminal(steps: list[dict], total: dict) -> str: @@ -170,22 +315,33 @@ def format_compact_kv( h = TelemetryFormatter._humanize lines = ["## token_summary", ""] + has_non_anthropic = False for step in steps: name = step.get("step_name", "?") + model = step.get("model", "") + if _is_non_anthropic(model): + name = f"{name}*" + has_non_anthropic = True count = step.get("invocation_count", 1) inp = h(step.get("input_tokens", 0)) out = h(step.get("output_tokens", 0)) cache_rd = h(step.get("cache_read_input_tokens", 0)) + peak_ctx = h(step.get("peak_context", 0)) cache_wr = h(step.get("cache_creation_input_tokens", 0)) + turns = step.get("turn_count", 0) wc = step.get("wall_clock_seconds", step.get("elapsed_seconds", 0.0)) + model_tag = f" model:{model}" if model else "" lines.append( - f"{name} x{count} [uc:{inp} out:{out} cr:{cache_rd} cw:{cache_wr} t:{wc:.1f}s]" + f"{name} x{count}" + f" [uc:{inp} out:{out} cr:{cache_rd} pk:{peak_ctx} cw:{cache_wr}" + f" turns:{turns} t:{wc:.1f}s{model_tag}]" ) if total: lines.append("") lines.append(f"total_uncached: {h(total.get('input_tokens', 0))}") lines.append(f"total_out: {h(total.get('output_tokens', 0))}") lines.append(f"total_cache_read: {h(total.get('cache_read_input_tokens', 0))}") + lines.append(f"total_peak_context: {h(total.get('peak_context', 0))}") lines.append(f"total_cache_write: {h(total.get('cache_creation_input_tokens', 0))}") if mcp_responses: mcp_total = mcp_responses.get("total", {}) @@ -194,4 +350,118 @@ def format_compact_kv( lines.append(f"mcp_invocations: {mcp_total.get('total_invocations', 0)}") est_tokens = mcp_total.get("total_estimated_response_tokens", 0) lines.append(f"mcp_response_tokens: ~{h(est_tokens)}") + if has_non_anthropic: + lines.append("") + lines.append("* Step used a non-Anthropic provider; caching behavior may differ.") return "\n".join(lines) + + @staticmethod + def format_efficiency_table(steps: list[dict], total: dict) -> str: + """Produce a markdown Token Efficiency table. Returns '' when all LoC=0.""" + if not any(s.get("loc_insertions", 0) + s.get("loc_deletions", 0) > 0 for s in steps): + return "" + + lines = [ + "## Token Efficiency", + "", + _EFFICIENCY_MD_HEADER, + _EFFICIENCY_MD_SEP, + ] + for step in steps: + loc = step.get("loc_insertions", 0) + step.get("loc_deletions", 0) + cr = step.get("cache_read_input_tokens", 0) + cw = step.get("cache_creation_input_tokens", 0) + out = step.get("output_tokens", 0) + lines.append( + f"| {step.get('step_name', '?')} | {loc}" + f" | {_ratio(cr, loc)}" + f" | {_ratio(cw, loc)} | {_ratio(out, loc)} |" + ) + + total_loc = total.get("loc_insertions", 0) + total.get("loc_deletions", 0) + total_cr = total.get("cache_read_input_tokens", 0) + total_cw = total.get("cache_creation_input_tokens", 0) + total_out = total.get("output_tokens", 0) + lines.append( + f"| **Total** | **{total_loc}**" + f" | {_ratio(total_cr, total_loc)}" + f" | {_ratio(total_cw, total_loc)} | {_ratio(total_out, total_loc)} |" + ) + return "\n".join(lines) + + @staticmethod + def format_model_table(model_totals: list[ModelTotalEntry]) -> str: + """Produce markdown ## Model Usage Breakdown table. Returns '' when empty.""" + if not model_totals or all(m.get("model", "") == "unknown" for m in model_totals): + return "" + h = TelemetryFormatter._humanize + fmt_dur = TelemetryFormatter._fmt_duration + lines = [ + "## Model Usage Breakdown", + "", + _MODEL_MD_HEADER, + _MODEL_MD_SEP, + ] + for m in model_totals: + lines.append( + f"| {m.get('model', '')} | {m.get('step_count', 0)}" + f" | {h(m.get('input_tokens', 0))} | {h(m.get('output_tokens', 0))}" + f" | {h(m.get('cache_read_input_tokens', 0))}" + f" | {h(m.get('cache_creation_input_tokens', 0))}" + f" | {fmt_dur(m.get('elapsed_seconds', 0.0))} |" + ) + return "\n".join(lines) + + @staticmethod + def format_model_table_terminal(model_totals: list[ModelTotalEntry]) -> str: + """Produce padded-column terminal model breakdown table. Returns '' when empty.""" + if not model_totals or all(m.get("model", "") == "unknown" for m in model_totals): + return "" + h = TelemetryFormatter._humanize + fmt_dur = TelemetryFormatter._fmt_duration + rows: list[tuple[str, str, str, str, str, str, str]] = [] + for m in model_totals: + rows.append( + ( + m.get("model", ""), + str(m.get("step_count", 0)), + h(m.get("input_tokens", 0)), + h(m.get("output_tokens", 0)), + h(m.get("cache_read_input_tokens", 0)), + h(m.get("cache_creation_input_tokens", 0)), + fmt_dur(m.get("elapsed_seconds", 0.0)), + ) + ) + return _render_terminal_table(_MODEL_COLUMNS, rows) + + @staticmethod + def format_efficiency_table_terminal(steps: list[dict], total: dict) -> str: + """Produce a padded-column plain text efficiency table. Returns '' when all LoC=0.""" + if not any(s.get("loc_insertions", 0) + s.get("loc_deletions", 0) > 0 for s in steps): + return "" + + rows: list[tuple[str, str, str, str, str]] = [] + for step in steps: + loc = step.get("loc_insertions", 0) + step.get("loc_deletions", 0) + cr = step.get("cache_read_input_tokens", 0) + cw = step.get("cache_creation_input_tokens", 0) + out = step.get("output_tokens", 0) + rows.append( + ( + step.get("step_name", "?"), + str(loc), + _ratio(cr, loc), + _ratio(cw, loc), + _ratio(out, loc), + ) + ) + + total_loc = total.get("loc_insertions", 0) + total.get("loc_deletions", 0) + total_row = ( + "Total", + str(total_loc), + _ratio(total.get("cache_read_input_tokens", 0), total_loc), + _ratio(total.get("cache_creation_input_tokens", 0), total_loc), + _ratio(total.get("output_tokens", 0), total_loc), + ) + return _render_terminal_table(_EFFICIENCY_COLUMNS, rows + [total_row]) diff --git a/src/autoskillit/pipeline/timings.py b/src/autoskillit/pipeline/timings.py index fc95054a51..8e6d5202d2 100644 --- a/src/autoskillit/pipeline/timings.py +++ b/src/autoskillit/pipeline/timings.py @@ -7,7 +7,7 @@ from __future__ import annotations import json -from dataclasses import asdict, dataclass +from dataclasses import dataclass from pathlib import Path from typing import Any @@ -27,7 +27,11 @@ class TimingEntry: invocation_count: int = 0 def to_dict(self) -> dict[str, Any]: - return asdict(self) + return { + "step_name": self.step_name, + "total_seconds": self.total_seconds, + "invocation_count": self.invocation_count, + } __all__ = ["TimingEntry", "DefaultTimingLog"] @@ -108,6 +112,9 @@ def load_from_log_dir( since: str = "", cwd_filter: str = "", kitchen_id_filter: str = "", + campaign_id_filter: str = "", + order_id_filter: str = "", + dispatch_id_filter: str = "", ) -> int: """Reconstruct timing entries from persisted session logs. @@ -118,12 +125,22 @@ def load_from_log_dir( cwd_filter: if non-empty, only sessions whose cwd matches are loaded. kitchen_id_filter: if non-empty, only sessions whose kitchen_id matches are loaded. Falls back to pipeline_id for sessions written before the rename. + campaign_id_filter: if non-empty, only sessions whose campaign_id matches are loaded. + order_id_filter: if non-empty, only sessions whose order_id matches are loaded. + dispatch_id_filter: if non-empty, only sessions whose dispatch_id matches are loaded. Returns the count of session directories successfully loaded. """ count = 0 for st_path in _iter_session_log_entries( - log_root, since, "step_timing.json", cwd_filter, kitchen_id_filter + log_root, + since, + "step_timing.json", + cwd_filter, + kitchen_id_filter, + campaign_id_filter, + order_id_filter, + dispatch_id_filter, ): try: data = json.loads(st_path.read_text(encoding="utf-8")) diff --git a/src/autoskillit/pipeline/tokens.py b/src/autoskillit/pipeline/tokens.py index 1a19da9976..6d1f2d0d32 100644 --- a/src/autoskillit/pipeline/tokens.py +++ b/src/autoskillit/pipeline/tokens.py @@ -10,13 +10,14 @@ from __future__ import annotations import json -import re -from dataclasses import asdict, dataclass +from dataclasses import dataclass from datetime import datetime from pathlib import Path -from typing import Any +from typing import Any, cast -from autoskillit.core import get_logger +import regex as re + +from autoskillit.core import ModelTotalEntry, get_logger from autoskillit.pipeline.audit import _iter_session_log_entries logger = get_logger(__name__) @@ -39,7 +40,27 @@ def canonical_step_name(step_name: str) -> str: indistinguishable from an orchestrator-appended suffix and will be collapsed to the base name. """ - return re.sub(r"-\d+$", "", step_name) if step_name else step_name + if not step_name or ":" in step_name or step_name.startswith("("): + return step_name + return re.sub(r"-\d+$", "", step_name) + + +def _primary_model(token_usage: dict[str, Any]) -> str: + """Return the model name with the most total tokens from model_breakdown.""" + mb = token_usage.get("model_breakdown", {}) + if not isinstance(mb, dict) or not mb: + return "" + for m, v in mb.items(): + if not isinstance(v, dict): + logger.warning("Unexpected model_breakdown entry type for %r: %r", m, type(v).__name__) + return max( + mb, + key=lambda m: ( + sum(v for v in mb[m].values() if isinstance(v, (int, float))) + if isinstance(mb[m], dict) + else 0 + ), + ) @dataclass @@ -47,15 +68,33 @@ class TokenEntry: """Accumulated token usage for a single YAML step name.""" step_name: str + model: str = "" input_tokens: int = 0 output_tokens: int = 0 cache_creation_input_tokens: int = 0 cache_read_input_tokens: int = 0 invocation_count: int = 0 elapsed_seconds: float = 0.0 + loc_insertions: int = 0 + loc_deletions: int = 0 + peak_context: int = 0 + turn_count: int = 0 def to_dict(self) -> dict[str, Any]: - return asdict(self) + return { + "step_name": self.step_name, + "model": self.model, + "input_tokens": self.input_tokens, + "output_tokens": self.output_tokens, + "cache_creation_input_tokens": self.cache_creation_input_tokens, + "cache_read_input_tokens": self.cache_read_input_tokens, + "invocation_count": self.invocation_count, + "elapsed_seconds": self.elapsed_seconds, + "loc_insertions": self.loc_insertions, + "loc_deletions": self.loc_deletions, + "peak_context": self.peak_context, + "turn_count": self.turn_count, + } __all__ = ["TokenEntry", "DefaultTokenLog", "canonical_step_name"] @@ -85,6 +124,8 @@ def record( end_ts: str = "", elapsed_seconds: float | None = None, order_id: str = "", + loc_insertions: int = 0, + loc_deletions: int = 0, ) -> None: """Accumulate token usage for a step. @@ -98,11 +139,22 @@ def record( if key not in self._entries: self._entries[key] = TokenEntry(step_name=canonical) e = self._entries[key] + _model = _primary_model(token_usage) + if _model and not e.model: + e.model = _model e.input_tokens += token_usage.get("input_tokens", 0) e.output_tokens += token_usage.get("output_tokens", 0) e.cache_creation_input_tokens += token_usage.get("cache_creation_input_tokens", 0) e.cache_read_input_tokens += token_usage.get("cache_read_input_tokens", 0) e.invocation_count += 1 + e.loc_insertions += loc_insertions + e.loc_deletions += loc_deletions + _peak = token_usage.get("peak_context", 0) + if isinstance(_peak, int) and _peak > e.peak_context: + e.peak_context = _peak + _turns = token_usage.get("turn_count", 0) + if isinstance(_turns, int): + e.turn_count += _turns if elapsed_seconds is not None: e.elapsed_seconds += elapsed_seconds elif start_ts and end_ts: @@ -135,12 +187,19 @@ def get_report(self, *, order_id: str = "") -> list[dict[str, Any]]: if step not in aggregated: aggregated[step] = TokenEntry(step_name=step) agg = aggregated[step] + if e.model and not agg.model: + agg.model = e.model agg.input_tokens += e.input_tokens agg.output_tokens += e.output_tokens agg.cache_creation_input_tokens += e.cache_creation_input_tokens agg.cache_read_input_tokens += e.cache_read_input_tokens agg.elapsed_seconds += e.elapsed_seconds agg.invocation_count += e.invocation_count + agg.loc_insertions += e.loc_insertions + agg.loc_deletions += e.loc_deletions + if e.peak_context > agg.peak_context: + agg.peak_context = e.peak_context + agg.turn_count += e.turn_count return [e.to_dict() for e in aggregated.values()] def compute_total(self, *, order_id: str = "") -> dict[str, Any]: @@ -155,6 +214,10 @@ def compute_total(self, *, order_id: str = "") -> dict[str, Any]: "cache_creation_input_tokens": 0, "cache_read_input_tokens": 0, "total_elapsed_seconds": 0.0, + "loc_insertions": 0, + "loc_deletions": 0, + "peak_context": 0, + "turn_count": 0, } for (oid, _step), entry in self._entries.items(): if order_id and oid != order_id: @@ -164,8 +227,43 @@ def compute_total(self, *, order_id: str = "") -> dict[str, Any]: total["cache_creation_input_tokens"] += entry.cache_creation_input_tokens total["cache_read_input_tokens"] += entry.cache_read_input_tokens total["total_elapsed_seconds"] += entry.elapsed_seconds + total["loc_insertions"] += entry.loc_insertions + total["loc_deletions"] += entry.loc_deletions + if entry.peak_context > total["peak_context"]: + total["peak_context"] = entry.peak_context + total["turn_count"] += entry.turn_count return total + def compute_model_totals(self, *, order_id: str = "") -> list[ModelTotalEntry]: + """Compute per-model aggregate token counts across all steps.""" + model_data: dict[str, dict[str, Any]] = {} + for (oid, _step), entry in self._entries.items(): + if order_id and oid != order_id: + continue + model = entry.model or "unknown" + if model not in model_data: + model_data[model] = { + "model": model, + "_steps": set(), + "input_tokens": 0, + "output_tokens": 0, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + "elapsed_seconds": 0.0, + } + md = model_data[model] + md["_steps"].add(entry.step_name) + md["input_tokens"] += entry.input_tokens + md["output_tokens"] += entry.output_tokens + md["cache_creation_input_tokens"] += entry.cache_creation_input_tokens + md["cache_read_input_tokens"] += entry.cache_read_input_tokens + md["elapsed_seconds"] += entry.elapsed_seconds + result = [] + for md in model_data.values(): + md["step_count"] = len(md.pop("_steps")) + result.append(md) + return cast(list[ModelTotalEntry], result) + def clear(self) -> None: """Reset the store.""" self._entries = {} @@ -177,6 +275,9 @@ def load_from_log_dir( since: str = "", cwd_filter: str = "", kitchen_id_filter: str = "", + campaign_id_filter: str = "", + order_id_filter: str = "", + dispatch_id_filter: str = "", ) -> int: """Reconstruct token entries from persisted session logs. @@ -188,19 +289,29 @@ def load_from_log_dir( cwd_filter: if non-empty, only sessions whose cwd matches are loaded. kitchen_id_filter: if non-empty, only sessions whose kitchen_id matches are loaded. Falls back to pipeline_id for sessions written before the rename. + campaign_id_filter: if non-empty, only sessions whose campaign_id matches are loaded. + order_id_filter: if non-empty, only sessions whose order_id matches are loaded. + dispatch_id_filter: if non-empty, only sessions whose dispatch_id matches are loaded. Returns the count of session directories successfully loaded. """ count = 0 for tu_path in _iter_session_log_entries( - log_root, since, "token_usage.json", cwd_filter, kitchen_id_filter + log_root, + since, + "token_usage.json", + cwd_filter, + kitchen_id_filter, + campaign_id_filter, + order_id_filter, + dispatch_id_filter, ): try: data = json.loads(tu_path.read_text(encoding="utf-8")) except (json.JSONDecodeError, OSError): continue - raw_step = data.get("step_name", "") + raw_step = data.get("session_label") or data.get("step_name", "") if not raw_step: continue @@ -210,6 +321,9 @@ def load_from_log_dir( if key not in self._entries: self._entries[key] = TokenEntry(step_name=step_name) e = self._entries[key] + _model = data.get("model_identifier", "") + if _model and not e.model: + e.model = _model e.input_tokens += data.get("input_tokens", 0) e.output_tokens += data.get("output_tokens", 0) e.cache_creation_input_tokens += data.get("cache_creation_input_tokens", 0) @@ -218,6 +332,14 @@ def load_from_log_dir( # elapsed_seconds is the in-memory field name on TokenEntry. _raw_timing = data.get("timing_seconds") e.elapsed_seconds += float(_raw_timing) if _raw_timing is not None else 0.0 + e.loc_insertions += data.get("loc_insertions", 0) + e.loc_deletions += data.get("loc_deletions", 0) + _raw_peak = data.get("peak_context", 0) + if isinstance(_raw_peak, int) and _raw_peak > e.peak_context: + e.peak_context = _raw_peak + _raw_turns = data.get("turn_count", 0) + if isinstance(_raw_turns, int): + e.turn_count += _raw_turns # Each token_usage.json file represents a single run_skill invocation # (one file = one invocation). Incrementing here reconstructs the # invocation count that was accumulated live via record(). diff --git a/src/autoskillit/planner/CLAUDE.md b/src/autoskillit/planner/CLAUDE.md new file mode 100644 index 0000000000..0b7cb29189 --- /dev/null +++ b/src/autoskillit/planner/CLAUDE.md @@ -0,0 +1,21 @@ +# planner/ + +IL-1 progressive resolution planner — phases, work packages, manifest generation, DAG validation. + +## Files + +| File | Purpose | +|------|---------| +| `__init__.py` | Re-exports `expand_assignments`, `expand_wps`, `finalize_wp_manifest`, `validate_plan`, `compile_plan` | +| `_sort_utils.py` | Natural sort key utility: `_NATURAL_SORT_RE`, `_natural_sort_key()` | +| `manifests.py` | `expand_assignments`, `expand_wps`, `finalize_wp_manifest`, `build_phase_assignment_manifest`, `build_phase_wp_manifest` | +| `merge.py` | `merge_tier_dir`, `merge_files`, `build_plan_snapshot`, `extract_item`, `replace_item` — JSON interchange merge tooling | +| `validation.py` | `validate_plan` — DAG cycle check, structural completeness, sizing bounds, duplicate-deliverable detection | +| `schema.py` | Planner data contracts: `PhaseResult`, `AssignmentResult`, `WPResult`, `PlanDocument` TypedDicts | +| `compiler.py` | `compile_plan` — topological sort, issue body generation, milestone definitions, plan artifacts | +| `consolidation.py` | `consolidate_wps` — post-elaboration WP consolidation: reads manifests, merges trivial WPs, rewrites dep IDs | + +## Architecture Notes +import from `server/` or `recipe/`. `validation.py` performs a DAG cycle check before any +compilation proceeds. `consolidation.py` runs as a post-pass after all elaboration phases +complete. diff --git a/src/autoskillit/planner/__init__.py b/src/autoskillit/planner/__init__.py new file mode 100644 index 0000000000..6d3bda07c5 --- /dev/null +++ b/src/autoskillit/planner/__init__.py @@ -0,0 +1,87 @@ +"""planner/ IL-1 package: progressive resolution planner. + +Exports: manifests, merge, validation, compiler.""" + +from __future__ import annotations + +from autoskillit.planner.compiler import compile_plan +from autoskillit.planner.consolidation import consolidate_wps +from autoskillit.planner.manifests import ( + build_phase_assignment_manifest, + build_phase_wp_manifest, + create_run_dir, + expand_assignments, + expand_wps, + finalize_wp_manifest, + resolve_task_input, +) +from autoskillit.planner.merge import ( + build_plan_snapshot, + extract_item, + merge_files, + merge_refined_assignments, + merge_tier_results, + replace_item, +) +from autoskillit.planner.schema import ( + ASSIGN_RESULT_FILE_RE, + ASSIGNMENT_REQUIRED_KEYS, + DELIVERABLE_BOUNDS, + PHASE_REQUIRED_KEYS, + PHASE_RESULT_FILE_RE, + WP_REQUIRED_KEYS, + WP_RESULT_FILE_RE, + AssignmentElaborated, + AssignmentShort, + PhaseElaborated, + PhaseShort, + PlanDocument, + PlannerManifest, + PlannerManifestItem, + ValidationFinding, + WPElaborated, + WPShort, + resolve_wp_id, + validate_refined_assignments, + validate_refined_plan, +) +from autoskillit.planner.validation import validate_plan + +__all__ = [ + "ASSIGN_RESULT_FILE_RE", + "DELIVERABLE_BOUNDS", + "PHASE_RESULT_FILE_RE", + "WP_RESULT_FILE_RE", + "build_phase_assignment_manifest", + "consolidate_wps", + "build_phase_wp_manifest", + "compile_plan", + "create_run_dir", + "expand_assignments", + "expand_wps", + "finalize_wp_manifest", + "resolve_task_input", + "validate_plan", + "PlannerManifest", + "PlannerManifestItem", + "merge_files", + "merge_refined_assignments", + "merge_tier_results", + "extract_item", + "replace_item", + "build_plan_snapshot", + "PlanDocument", + "PhaseShort", + "PhaseElaborated", + "AssignmentShort", + "AssignmentElaborated", + "WPShort", + "WPElaborated", + "ASSIGNMENT_REQUIRED_KEYS", + "PHASE_REQUIRED_KEYS", + "WP_REQUIRED_KEYS", + "resolve_wp_id", + "validate_refined_assignments", + "validate_refined_plan", + "ValidationFinding", +] diff --git a/src/autoskillit/planner/_sort_utils.py b/src/autoskillit/planner/_sort_utils.py new file mode 100644 index 0000000000..148fc208c1 --- /dev/null +++ b/src/autoskillit/planner/_sort_utils.py @@ -0,0 +1,11 @@ +"""Natural sort key — shared within planner/.""" + +from __future__ import annotations + +import regex as re + +_NATURAL_SORT_RE = re.compile(r"(\d+)") + + +def _natural_sort_key(s: str) -> list[int | str]: + return [int(tok) if tok.isdigit() else tok for tok in _NATURAL_SORT_RE.split(s)] diff --git a/src/autoskillit/planner/compiler.py b/src/autoskillit/planner/compiler.py new file mode 100644 index 0000000000..f35f8a0576 --- /dev/null +++ b/src/autoskillit/planner/compiler.py @@ -0,0 +1,279 @@ +"""Planner compiler: topological sort, issue body generation, plan artifacts. + +Imports ``_load_*_results()`` from ``validation.py`` — see that module's +docstring for the combined-document exclusion rationale. +""" + +from __future__ import annotations + +import json +from collections import deque +from pathlib import Path +from typing import Any + +from autoskillit.core import atomic_write, get_logger, write_versioned_json +from autoskillit.planner.manifests import _derive_label +from autoskillit.planner.validation import ( + _load_assignment_results, + _load_phase_results, + _load_wp_results, +) + +logger = get_logger(__name__) + + +def _topological_sort(wp_results: dict[str, dict]) -> list[str]: + in_degree: dict[str, int] = {wp_id: 0 for wp_id in wp_results} + adjacency: dict[str, list[str]] = {wp_id: [] for wp_id in wp_results} + for wp_id, wp in wp_results.items(): + for dep in wp.get("depends_on", []): + if dep in wp_results: + adjacency[dep].append(wp_id) + in_degree[wp_id] += 1 + + queue: deque[str] = deque(sorted(k for k, v in in_degree.items() if v == 0)) + order: list[str] = [] + while queue: + node = queue.popleft() + order.append(node) + for neighbor in sorted(adjacency[node]): + in_degree[neighbor] -= 1 + if in_degree[neighbor] == 0: + queue.append(neighbor) + + if len(order) < len(wp_results): + cycle_nodes = [n for n in wp_results if n not in set(order)] + raise RuntimeError(f"Cycle detected among WPs: {', '.join(sorted(cycle_nodes))}") + return order + + +def _inject_forward_deps(wp_results: dict[str, dict], dep_graph: dict) -> None: + for wp_id, dependents in dep_graph.get("forward_deps", {}).items(): + if wp_id in wp_results: + wp_results[wp_id]["depended_on_by"] = list(dependents) + + +def _build_phase_lookup(phase_results: dict[str, dict]) -> dict[int, dict]: + return {v["phase_number"]: v for v in phase_results.values()} + + +def _build_assignment_lookup( + assignment_results: dict[str, dict], +) -> dict[tuple[int, int], dict]: + return {(v["phase_number"], v["assignment_number"]): v for v in assignment_results.values()} + + +def _parse_wp_id(wp_id: str) -> tuple[int, int, int]: + parts = wp_id.split("-") + if len(parts) != 3: + raise ValueError(f"Invalid WP id format (expected PX-AY-WPZ): {wp_id!r}") + try: + return int(parts[0][1:]), int(parts[1][1:]), int(parts[2][2:]) + except (IndexError, ValueError) as exc: + raise ValueError(f"Invalid WP id format (expected PX-AY-WPZ): {wp_id!r}") from exc + + +def _render_issue_body(wp: dict, phase: dict, assignment: dict) -> str: + phase_num = phase["phase_number"] + phase_name = phase["name"] + phase_slug = phase["name_slug"] + milestone = f"{phase_num}-{phase_slug}" + assign_id = f"P{phase['phase_number']}-A{assignment['assignment_number']}" + assign_name = assignment.get("name", "") + + depends_on = wp.get("depends_on", []) + depended_on_by = wp.get("depended_on_by", []) + + depends_str = ", ".join(depends_on) if depends_on else "None" + depended_str = ", ".join(depended_on_by) if depended_on_by else "None" + + deliverables_md = "\n".join(f"- `{d}`" for d in wp.get("deliverables", [])) + steps_md = "\n".join(f"{i + 1}. {s}" for i, s in enumerate(wp.get("technical_steps", []))) + criteria_md = "\n".join(f"- {c}" for c in wp.get("acceptance_criteria", [])) + + body = f"""## Goal + +{wp.get("goal", wp.get("summary", ""))} + +## Context + +- Phase: {phase_name} (Milestone: {milestone}) +- Assignment: {assign_id} ({assign_name}) +- Depends on: {depends_str} +- Depended on by: {depended_str} + +## Deliverables + +{deliverables_md} + +## Technical Steps + +{steps_md} + +## Acceptance Criteria + +{criteria_md} +""" + if wp.get("review_approach_recommended"): + reasoning = wp.get("review_approach_reasoning", "") + body += f"\n## Review Approach\n\n> **review-approach recommended**: {reasoning}\n" + return body + + +def compile_plan( + output_dir: str, task_file_path: str, source_dir: str, task_label: str = "", **kwargs: Any +) -> dict[str, str]: + task = Path(task_file_path).read_text(encoding="utf-8") if task_file_path else "" + task_label = task_label or _derive_label(task, "") + root = Path(output_dir) + + phase_results = _load_phase_results(root) + assignment_results = _load_assignment_results(root) + wp_results = _load_wp_results(root) + + validation_path = root / "validation.json" + if validation_path.exists(): + try: + validation = json.loads(validation_path.read_text()) + except json.JSONDecodeError as exc: + raise RuntimeError(f"Malformed validation file {validation_path}: {exc}") from exc + if validation.get("verdict") != "pass": + logger.warning( + "compile_plan called with non-passing validation", + verdict=validation.get("verdict"), + ) + + dep_graph: dict = {} + dep_graph_path = root / "dep_graph.json" + if dep_graph_path.exists(): + try: + dep_graph = json.loads(dep_graph_path.read_text()) + except json.JSONDecodeError as exc: + raise RuntimeError(f"Malformed dep graph file {dep_graph_path}: {exc}") from exc + + _inject_forward_deps(wp_results, dep_graph) + + assessment_path = root / "review_approach_assessment.json" + assessment_by_wp_id: dict[str, dict] = {} + if assessment_path.exists(): + try: + data = json.loads(assessment_path.read_text(encoding="utf-8")) + except (json.JSONDecodeError, OSError) as exc: + raise RuntimeError(f"Malformed assessment file {assessment_path}: {exc}") from exc + for entry in data.get("assessments", []): + wp_id = entry.get("wp_id") + if wp_id is None: + continue + assessment_by_wp_id[wp_id] = entry + + execution_order = _topological_sort(wp_results) + phase_lookup = _build_phase_lookup(phase_results) + assign_lookup = _build_assignment_lookup(assignment_results) + + issues_dir = root / "issues" + issues_dir.mkdir(exist_ok=True) + + issue_paths: dict[str, str] = {} + for wp_id in execution_order: + wp = wp_results[wp_id] + phase_num, assign_num, _ = _parse_wp_id(wp_id) + if phase_num not in phase_lookup: + raise RuntimeError( + f"WP {wp_id!r} references phase {phase_num} not in loaded phase results" + ) + phase = phase_lookup[phase_num] + if (phase_num, assign_num) not in assign_lookup: + raise RuntimeError( + f"WP {wp_id!r} references assignment P{phase_num}-A{assign_num}" + " not in loaded assignment results" + ) + assignment = assign_lookup[(phase_num, assign_num)] + if wp_id in assessment_by_wp_id: + wp = { + **wp, + **{ + k: assessment_by_wp_id[wp_id][k] + for k in ("review_approach_recommended", "review_approach_reasoning") + if k in assessment_by_wp_id[wp_id] + }, + } + body = _render_issue_body(wp, phase, assignment) + issue_path = issues_dir / f"{wp_id}_issue.md" + atomic_write(issue_path, body) + issue_paths[wp_id] = str(issue_path) + + milestones = [ + { + "phase_number": phase["phase_number"], + "name": phase["name"], + "name_slug": phase["name_slug"], + } + for phase in sorted(phase_results.values(), key=lambda p: p["phase_number"]) + ] + milestones_path = root / "milestones.json" + write_versioned_json(milestones_path, {"milestones": milestones}, schema_version=1) + + phases_nested = [] + for phase_id in sorted(phase_results, key=lambda k: phase_results[k]["phase_number"]): + phase = phase_results[phase_id] + phase_num = phase["phase_number"] + assignments_nested = [] + for (pn, an), assign in sorted(assign_lookup.items()): + if pn != phase_num: + continue + wps_in_assign = [ + wp_results[wid] for wid in execution_order if wid.startswith(f"P{pn}-A{an}-") + ] + assignments_nested.append({**assign, "work_packages": wps_in_assign}) + phases_nested.append({**phase, "assignments": assignments_nested}) + + plan_json_path = root / "plan.json" + write_versioned_json( + plan_json_path, + { + "task": task, + "source_dir": source_dir, + "phases": phases_nested, + "execution_order": execution_order, + }, + schema_version=1, + ) + + md_lines = [f"# Plan: {task_label}", ""] + for phase in sorted(phase_results.values(), key=lambda p: p["phase_number"]): + phase_num = phase["phase_number"] + md_lines.append(f"## Phase {phase_num}: {phase['name']}") + md_lines.append("") + for wp_id in execution_order: + pn, an, _ = _parse_wp_id(wp_id) + if pn != phase_num: + continue + wp = wp_results[wp_id] + md_lines.append(f"### {wp_id}: {wp.get('name', '')}") + md_lines.append("") + md_lines.append(wp.get("summary", wp.get("goal", ""))) + md_lines.append("") + + plan_md_path = root / "plan.md" + atomic_write(plan_md_path, "\n".join(md_lines)) + + manifest_path = root / "manifest.json" + write_versioned_json( + manifest_path, + { + "task": task, + "source_dir": source_dir, + "execution_order": execution_order, + "issues": issue_paths, + }, + schema_version=1, + ) + + plan_parts = "\n".join(issue_paths[wp_id] for wp_id in execution_order) + + logger.info("compile_plan", wp_count=len(execution_order)) + return { + "plan_path": str(plan_md_path), + "plan_json_path": str(plan_json_path), + "plan_parts": plan_parts, + } diff --git a/src/autoskillit/planner/consolidation.py b/src/autoskillit/planner/consolidation.py new file mode 100644 index 0000000000..72013c4c27 --- /dev/null +++ b/src/autoskillit/planner/consolidation.py @@ -0,0 +1,318 @@ +"""WP consolidation pass: merges work packages per manifests with file-sharing fallback.""" + +from __future__ import annotations + +import json +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +import regex as re + +from autoskillit.core import atomic_write, write_versioned_json +from autoskillit.planner._sort_utils import _natural_sort_key +from autoskillit.planner.schema import validate_wp_result + +_ASSIGNMENT_RE = re.compile(r"^(P\d+-A\d+)-") +_FALLBACK_MIN_WPS = 5 +_FALLBACK_MAX_GROUP_SIZE = 5 + + +@dataclass(frozen=True, slots=True) +class _ConsolidationGroup: + merged_id: str + source_wp_ids: list[str] + merge_order: list[str] + name: str | None + goal: str | None + + +def _load_manifests(consolidation_dir: Path) -> list[dict[str, Any]]: + manifests: list[dict[str, Any]] = [] + for path in sorted(consolidation_dir.glob("*_consolidation.json")): + try: + manifests.append(json.loads(path.read_text())) + except (json.JSONDecodeError, OSError) as exc: + raise ValueError(f"failed to load consolidation manifest {path}: {exc}") from exc + return manifests + + +def _build_group_maps( + manifests: list[dict[str, Any]], +) -> tuple[dict[str, str], dict[str, _ConsolidationGroup]]: + source_to_merged: dict[str, str] = {} + merged_groups: dict[str, _ConsolidationGroup] = {} + for manifest_idx, manifest in enumerate(manifests): + for group_idx, g in enumerate(manifest.get("groups", [])): + try: + merged_id = g["merged_id"] + source_wp_ids = g["source_wp_ids"] + except KeyError as exc: + raise ValueError( + f"malformed group at manifest[{manifest_idx}]" + f".groups[{group_idx}]: missing key {exc}" + ) from exc + if merged_id in merged_groups: + raise ValueError( + f"duplicate merged_id {merged_id!r} across consolidation manifests" + ) + group = _ConsolidationGroup( + merged_id=merged_id, + source_wp_ids=source_wp_ids, + merge_order=g.get("merge_order", source_wp_ids), + name=g.get("name"), + goal=g.get("goal"), + ) + merged_groups[group.merged_id] = group + for src_id in group.source_wp_ids: + source_to_merged[src_id] = group.merged_id + return source_to_merged, merged_groups + + +def _merge_group( + group: _ConsolidationGroup, + wp_by_id: dict[str, dict[str, Any]], +) -> dict[str, Any]: + primary = wp_by_id[group.merged_id] + sources_in_order = [wp_by_id[id_] for id_ in group.merge_order] + + merged = dict(primary) + merged["name"] = group.name or primary["name"] + merged["goal"] = group.goal or primary["goal"] + merged["technical_steps"] = [ + s for wp in sources_in_order for s in wp.get("technical_steps", []) + ] + + for field in ( + "deliverables", + "acceptance_criteria", + "files_touched", + "apis_defined", + "apis_consumed", + ): + seen: set[str] = set() + merged[field] = [ + x + for wp in sources_in_order + for x in wp.get(field, []) + if not (x in seen or seen.add(x)) # type: ignore[func-returns-value] + ] + + # Collect all external deps; intra-group removal done in rewrite pass + all_source_ids = set(group.source_wp_ids) + raw_deps: list[str] = [] + seen_deps: set[str] = set() + for wp in sources_in_order: + for dep in wp.get("depends_on", []): + if dep not in all_source_ids and dep not in seen_deps: + raw_deps.append(dep) + seen_deps.add(dep) + merged["depends_on"] = raw_deps + + return merged + + +def _rewrite_deps( + wp: dict[str, Any], + source_to_merged: dict[str, str], + own_group_sources: set[str], +) -> list[str]: + result: list[str] = [] + seen: set[str] = set() + for dep in wp.get("depends_on", []): + if dep in own_group_sources: + continue + canonical = source_to_merged.get(dep, dep) + if canonical not in seen: + result.append(canonical) + seen.add(canonical) + return result + + +def _cluster_by_shared_files(wps: list[dict[str, Any]]) -> list[list[dict[str, Any]]]: + """Cluster WPs that share at least one files_touched entry.""" + parent: dict[str, str] = {wp["id"]: wp["id"] for wp in wps} + + def find(x: str) -> str: + while parent[x] != x: + x = parent[x] + return x + + def union(a: str, b: str) -> None: + ra, rb = find(a), find(b) + if ra != rb: + parent[ra] = rb + + file_to_wps: dict[str, list[str]] = {} + for wp in wps: + for f in wp.get("files_touched", []): + file_to_wps.setdefault(f, []).append(wp["id"]) + + for wp_ids in file_to_wps.values(): + for i in range(1, len(wp_ids)): + union(wp_ids[0], wp_ids[i]) + + clusters: dict[str, list[dict[str, Any]]] = {} + for wp in wps: + root = find(wp["id"]) + clusters.setdefault(root, []).append(wp) + + return list(clusters.values()) + + +def _chunk_list(items: list[dict[str, Any]], size: int) -> list[list[dict[str, Any]]]: + return [items[i : i + size] for i in range(0, len(items), size)] + + +def _build_fallback_groups( + work_packages: list[dict[str, Any]], +) -> list[_ConsolidationGroup]: + """Heuristic fallback: merge same-assignment WPs sharing files.""" + if len(work_packages) < _FALLBACK_MIN_WPS: + return [] + + by_assignment: dict[str, list[dict[str, Any]]] = {} + for wp in work_packages: + m = _ASSIGNMENT_RE.match(wp["id"]) + if m: + by_assignment.setdefault(m.group(1), []).append(wp) + + groups: list[_ConsolidationGroup] = [] + for wps in by_assignment.values(): + if len(wps) < 2: + continue + clusters = _cluster_by_shared_files(wps) + for cluster in clusters: + if len(cluster) < 2: + continue + for chunk in _chunk_list(cluster, _FALLBACK_MAX_GROUP_SIZE): + if len(chunk) < 2: + continue + sorted_ids = sorted([wp["id"] for wp in chunk], key=_natural_sort_key) + groups.append( + _ConsolidationGroup( + merged_id=sorted_ids[0], + source_wp_ids=sorted_ids, + merge_order=sorted_ids, + name=None, + goal=None, + ) + ) + return groups + + +def consolidate_wps( + refined_wps_path: str, + planner_dir: str, + **kwargs: object, +) -> dict[str, str]: + """Merge trivial WPs according to consolidation manifests. + + Reads manifests from ``{planner_dir}/work_packages/consolidation/``. + Writes ``{planner_dir}/consolidated_wps.json`` and rebuilds + ``{planner_dir}/work_packages/wp_index.json``. Returns a result dict with string values. + """ + try: + wps_doc: dict[str, Any] = json.loads(Path(refined_wps_path).read_text()) + except (json.JSONDecodeError, OSError) as exc: + raise ValueError(f"failed to load refined WPs from {refined_wps_path}: {exc}") from exc + work_packages: list[dict[str, Any]] = wps_doc.get("work_packages", []) + task = wps_doc.get("task", "") + source_dir = wps_doc.get("source_dir", "") + + missing_id = [i for i, wp in enumerate(work_packages) if "id" not in wp] + if missing_id: + raise ValueError(f"work_packages entries at indices {missing_id} are missing 'id' field") + wp_by_id: dict[str, dict[str, Any]] = {wp["id"]: wp for wp in work_packages} + + consolidation_dir = Path(planner_dir) / "work_packages" / "consolidation" + manifests = _load_manifests(consolidation_dir) if consolidation_dir.exists() else [] + + source_to_merged, merged_groups = _build_group_maps(manifests) + + # Validate all IDs in each group exist in wp_by_id + for group in merged_groups.values(): + if group.merged_id not in wp_by_id: + raise ValueError(f"unknown merged_id: {group.merged_id}") + for src_id in group.source_wp_ids: + if src_id not in wp_by_id: + raise ValueError(f"unknown WP: {src_id}") + for ord_id in group.merge_order: + if ord_id not in wp_by_id: + raise ValueError(f"unknown merge_order element: {ord_id}") + + has_real_merges = any(len(g.source_wp_ids) > 1 for g in merged_groups.values()) + if not has_real_merges: + fallback_groups = _build_fallback_groups(work_packages) + for fg in fallback_groups: + merged_groups[fg.merged_id] = fg + for src_id in fg.source_wp_ids: + source_to_merged[src_id] = fg.merged_id + + # Build the output WP list + output_wps: list[dict[str, Any]] = [] + non_primary_sources: set[str] = set() + for group in merged_groups.values(): + for src_id in group.source_wp_ids: + if src_id != group.merged_id: + non_primary_sources.add(src_id) + + groups_applied = 0 + for wp in work_packages: + wp_id = wp["id"] + if wp_id in non_primary_sources: + # Absorbed into a merged WP + continue + if wp_id in merged_groups: + group = merged_groups[wp_id] + if len(group.source_wp_ids) > 1: + groups_applied += 1 + merged_wp = _merge_group(group, wp_by_id) + validate_wp_result(merged_wp) + output_wps.append(merged_wp) + else: + output_wps.append(dict(wp)) + + # Rewrite depends_on for every output WP + for wp in output_wps: + wp_id = wp["id"] + own_group_sources: set[str] = set() + if wp_id in merged_groups: + own_group_sources = set(merged_groups[wp_id].source_wp_ids) + wp["depends_on"] = _rewrite_deps(wp, source_to_merged, own_group_sources) + + planner_path = Path(planner_dir) + consolidated_path = planner_path / "consolidated_wps.json" + write_versioned_json( + consolidated_path, + {"task": task, "source_dir": source_dir, "work_packages": output_wps}, + 1, + ) + + # Rebuild wp_index.json as a sorted list (ListComp arg keeps AST scan from flagging as dict) + wp_index_path = planner_path / "work_packages" / "wp_index.json" + atomic_write( + wp_index_path, + json.dumps( + [ + {"id": wp["id"], "name": wp["name"], "summary": wp.get("summary", "")} + for wp in sorted(output_wps, key=lambda w: _natural_sort_key(w["id"])) + ], + indent=2, + ), + ) + + wp_dir = planner_path / "work_packages" + if wp_dir.exists(): + for wp in output_wps: + atomic_write(wp_dir / f"{wp['id']}_result.json", json.dumps(wp, indent=2)) + for absorbed_id in non_primary_sources: + result_file = wp_dir / f"{absorbed_id}_result.json" + if result_file.exists(): + result_file.unlink() + + return { + "consolidated_wps_path": str(consolidated_path), + "total_count": str(len(output_wps)), + "merged_count": str(groups_applied), + } diff --git a/src/autoskillit/planner/manifests.py b/src/autoskillit/planner/manifests.py new file mode 100644 index 0000000000..32172a0cad --- /dev/null +++ b/src/autoskillit/planner/manifests.py @@ -0,0 +1,455 @@ +from __future__ import annotations + +import json +import secrets +from datetime import UTC, datetime +from pathlib import Path +from typing import TypedDict + +from autoskillit.core import atomic_write, write_versioned_json +from autoskillit.planner._sort_utils import _natural_sort_key +from autoskillit.planner.schema import ( + ASSIGN_RESULT_FILE_RE, + PHASE_RESULT_FILE_RE, + WP_RESULT_FILE_RE, + RunDirResult, + TaskResolutionResult, + validate_assignment_result, + validate_phase_result, + validate_refined_assignments, + validate_refined_plan, + validate_wp_result, +) + + +class _PhaseBucket(TypedDict): + id: str + name: str + wp_ids: list[str] + wp_names: list[str] + wp_scopes: list[str] + wp_estimated_files: list[list[str]] + wp_count: int + + +def create_run_dir(temp_dir: str) -> RunDirResult: + if not temp_dir: + raise ValueError("temp_dir must be a non-empty path") + stamp = datetime.now(UTC).strftime("%Y%m%d-%H%M%S") + run_dir = Path(temp_dir) / "planner" / f"run-{stamp}-{secrets.token_hex(4)}" + for sub in ("phases", "assignments", "work_packages"): + (run_dir / sub).mkdir(parents=True, exist_ok=True) + return RunDirResult(planner_dir=str(run_dir)) + + +def _build_index_entry(result_data: dict[str, object]) -> dict[str, object]: + return { + "id": result_data.get("id", ""), + "name": result_data.get("name", ""), + "summary": result_data.get("summary", ""), + } + + +def build_phase_assignment_manifest(phases_dir: str, output_dir: str) -> dict[str, str]: + if not phases_dir or not output_dir: + raise ValueError("phases_dir and output_dir must not be empty") + + phases_path = Path(phases_dir) + out_dir = Path(output_dir) + assign_dir = out_dir.resolve() + + phase_files = [ + f for f in sorted(phases_path.glob("*_result.json")) if PHASE_RESULT_FILE_RE.match(f.name) + ] + parsed_phases = [] + for f in phase_files: + try: + raw = json.loads(f.read_text()) + except json.JSONDecodeError as exc: + raise json.JSONDecodeError( + f"Failed to parse {f}: {exc.msg}", exc.doc, exc.pos + ) from exc + try: + data = validate_phase_result(raw) + except (ValueError, KeyError) as exc: + raise ValueError(f"Invalid phase result in {f}: {exc}") from exc + parsed_phases.append(data) + parsed_phases.sort(key=lambda d: d["phase_number"]) + + items = [] + for phase_data in parsed_phases: + assignments = phase_data.get("assignments", []) + items.append( + { + "id": phase_data["id"], + "name": phase_data.get("name", ""), + "status": "pending", + "result_path": None, + "metadata": { + "assignment_count": len(assignments), + "assignment_ids": [ + a.get("metadata", {}).get("id", "") or a.get("id", "") for a in assignments + ], + "assignment_names": [a.get("name", "") for a in assignments], + }, + } + ) + + manifest = { + "pass_name": "phase_assignments", + "result_dir": str(assign_dir), + "created_at": datetime.now(tz=UTC).strftime("%Y-%m-%dT%H:%M:%SZ"), + "items": items, + } + manifest_path = assign_dir / "phase_assignment_manifest.json" + write_versioned_json(manifest_path, manifest, schema_version=1) + return {"manifest_path": str(manifest_path), "total_count": str(len(items))} + + +def build_phase_wp_manifest( + assignments_dir: str, output_dir: str, work_packages_dir: str = "" +) -> dict[str, str]: + if not assignments_dir or not output_dir: + raise ValueError("assignments_dir and output_dir must not be empty") + + assign_path = Path(assignments_dir) + if not assign_path.is_dir(): + raise FileNotFoundError(f"assignments_dir does not exist: {assign_path}") + out_dir = Path(output_dir) + wp_dir = ( + Path(work_packages_dir).resolve() + if work_packages_dir + else (out_dir / "work_packages").resolve() + ) + + assign_files = [ + f for f in assign_path.glob("*_result.json") if ASSIGN_RESULT_FILE_RE.match(f.name) + ] + parsed_assignments: list[dict] = [] + for f in assign_files: + try: + raw = json.loads(f.read_text()) + except json.JSONDecodeError as exc: + raise json.JSONDecodeError( + f"Failed to parse {f}: {exc.msg}", exc.doc, exc.pos + ) from exc + try: + data = validate_assignment_result(raw) + except (ValueError, KeyError) as exc: + raise ValueError(f"Invalid assignment result in {f}: {exc}") from exc + parsed_assignments.append(data) + parsed_assignments.sort(key=lambda d: (d["phase_number"], d["assignment_number"])) + + phase_buckets: dict[int, dict] = {} + for assign_data in parsed_assignments: + pn = assign_data["phase_number"] + an = assign_data["assignment_number"] + if pn not in phase_buckets: + phase_buckets[pn] = { + "phase_name": assign_data.get("phase_name", f"Phase {pn}"), + "phase_id": f"P{pn}", + "wp_ids": [], + "wp_names": [], + "wp_scopes": [], + "wp_estimated_files": [], + } + for wp_seq, wp in enumerate(assign_data.get("proposed_work_packages", []), start=1): + wp_id = f"P{pn}-A{an}-WP{wp_seq}" + phase_buckets[pn]["wp_ids"].append(wp_id) + phase_buckets[pn]["wp_names"].append(wp.get("name", "")) + phase_buckets[pn]["wp_scopes"].append(wp.get("scope", "")) + est_files = wp.get("estimated_files", []) + if not isinstance(est_files, list): + est_files = [] + phase_buckets[pn]["wp_estimated_files"].append(est_files) + + items = [] + for pn in sorted(phase_buckets): + bucket = phase_buckets[pn] + items.append( + { + "id": bucket["phase_id"], + "name": bucket["phase_name"], + "status": "pending", + "result_path": None, + "metadata": { + "wp_count": len(bucket["wp_ids"]), + "wp_ids": bucket["wp_ids"], + "wp_names": bucket["wp_names"], + "wp_scopes": bucket["wp_scopes"], + "wp_estimated_files": bucket["wp_estimated_files"], + }, + } + ) + + sentinel_dir = wp_dir / "wp_sentinels" + sentinel_dir.mkdir(parents=True, exist_ok=True) + + manifest = { + "pass_name": "phase_work_packages", + "result_dir": str(sentinel_dir), + "created_at": datetime.now(tz=UTC).strftime("%Y-%m-%dT%H:%M:%SZ"), + "items": items, + } + manifest_path = out_dir / "phase_wp_manifest.json" + write_versioned_json(manifest_path, manifest, schema_version=1) + atomic_write(wp_dir / "wp_index.json", "[]") + return {"manifest_path": str(manifest_path), "total_count": str(len(items))} + + +def finalize_wp_manifest(work_packages_dir: str, output_dir: str) -> dict[str, str]: + if not work_packages_dir or not output_dir: + raise ValueError("work_packages_dir and output_dir must not be empty") + + wp_dir = Path(work_packages_dir) + if not wp_dir.is_dir(): + raise FileNotFoundError(f"work_packages_dir does not exist: {wp_dir}") + + result_files = sorted( + (f for f in wp_dir.glob("*_result.json") if WP_RESULT_FILE_RE.match(f.name)), + key=lambda p: _natural_sort_key(p.name), + ) + items = [] + index_entries = [] + errors: list[str] = [] + for f in result_files: + try: + raw = json.loads(f.read_text()) + except json.JSONDecodeError as exc: + raise json.JSONDecodeError( + f"Failed to parse {f}: {exc.msg}", exc.doc, exc.pos + ) from exc + try: + data = validate_wp_result(raw, allow_stub=bool(raw.get("elaboration_failed"))) + except (ValueError, KeyError) as exc: + errors.append(f"{f.name}: {exc}") + continue + items.append( + { + "id": data["id"], + "name": data["name"], + "status": "done", + "result_path": str(f), + "metadata": {}, + } + ) + index_entries.append(_build_index_entry(data)) + + if errors: + detail = "; ".join(errors) + raise ValueError( + f"{len(errors)} WP validation error{'s' if len(errors) != 1 else ''}: {detail}" + ) + + items.sort(key=lambda i: _natural_sort_key(str(i["id"]))) + index_entries.sort(key=lambda e: _natural_sort_key(str(e["id"]))) + + manifest = { + "pass_name": "work_packages", + "result_dir": str(wp_dir.resolve()), + "created_at": datetime.now(tz=UTC).strftime("%Y-%m-%dT%H:%M:%SZ"), + "items": items, + } + manifest_path = wp_dir / "wp_manifest.json" + write_versioned_json(manifest_path, manifest, schema_version=1) + atomic_write(wp_dir / "wp_index.json", json.dumps(index_entries, indent=2)) + return {"manifest_path": str(manifest_path), "total_count": str(len(items))} + + +def expand_assignments( + refined_plan_path: str, output_dir: str, **kwargs: object +) -> dict[str, str]: + plan_file = Path(refined_plan_path) + try: + plan = json.loads(plan_file.read_text()) + except json.JSONDecodeError as exc: + raise json.JSONDecodeError( + f"Failed to parse {plan_file}: {exc.msg}", exc.doc, exc.pos + ) from exc + plan = validate_refined_plan(plan) + phases = plan.get("phases", []) + task_file_path: str = str(kwargs.get("task_file_path") or "") + assign_dir = Path(output_dir) / "assignments" + assign_dir.mkdir(parents=True, exist_ok=True) + + items: list[dict[str, object]] = [] + context_paths: list[str] = [] + item_ids: list[str] = [] + for phase in phases: + phase_id = phase["id"] + previews = phase.get("assignments_preview", []) + if not previews: + continue + assignment_ids: list[str] = [] + for idx, a in enumerate(previews, start=1): + if isinstance(a, dict): + aid = a.get("id", "") + if not aid: + aid = f"{phase_id}-A{idx}" + assignment_ids.append(aid) + else: + assignment_ids.append(str(a)) + assignment_names = [a.get("name", "") if isinstance(a, dict) else str(a) for a in previews] + metadata = { + "assignment_count": len(previews), + "assignment_ids": assignment_ids, + "assignment_names": assignment_names, + } + items.append( + { + "id": phase_id, + "name": phase.get("name", ""), + "status": "pending", + "result_path": None, + "metadata": metadata, + } + ) + context: dict[str, object] = { + "id": phase_id, + "name": phase.get("name", ""), + "task_file_path": task_file_path, + "metadata": metadata, + "prior_results": [], + } + ctx_path = assign_dir / f"context_{phase_id}.json" + write_versioned_json(ctx_path, context, schema_version=1) + context_paths.append(str(ctx_path)) + item_ids.append(phase_id) + + manifest = { + "pass_name": "phase_assignments", + "result_dir": str(assign_dir), + "created_at": datetime.now(tz=UTC).strftime("%Y-%m-%dT%H:%M:%SZ"), + "items": items, + } + manifest_path = assign_dir / "phase_assignment_manifest.json" + write_versioned_json(manifest_path, manifest, schema_version=1) + return { + "manifest_path": str(manifest_path), + "context_paths": ",".join(context_paths), + "item_ids": ",".join(item_ids), + } + + +def expand_wps(refined_assignments_path: str, output_dir: str, **kwargs: object) -> dict[str, str]: + assignments_file = Path(refined_assignments_path) + try: + data = json.loads(assignments_file.read_text()) + except json.JSONDecodeError as exc: + raise json.JSONDecodeError( + f"Failed to parse {assignments_file}: {exc.msg}", exc.doc, exc.pos + ) from exc + data = validate_refined_assignments(data) + assignments = data.get("assignments", []) + task_file_path: str = str(kwargs.get("task_file_path") or "") + out_dir = Path(output_dir) + wp_dir = out_dir / "work_packages" + wp_dir.mkdir(parents=True, exist_ok=True) + + phase_buckets: dict[str, _PhaseBucket] = {} + for assign in assignments: + phase_id = assign.get("phase_id", "") + if not phase_id: + pn = assign.get("phase_number", 0) + phase_id = f"P{pn}" + if phase_id not in phase_buckets: + phase_buckets[phase_id] = _PhaseBucket( + id=phase_id, + name=assign.get("phase_name", f"Phase {phase_id}"), + wp_ids=[], + wp_names=[], + wp_scopes=[], + wp_estimated_files=[], + wp_count=0, + ) + bucket = phase_buckets[phase_id] + wps = assign.get("proposed_work_packages", []) + for wp in wps: + wp_id = wp["id"] + bucket["wp_ids"].append(wp_id) + bucket["wp_names"].append(wp.get("name", "")) + bucket["wp_scopes"].append(wp.get("scope", "")) + est = wp.get("estimated_files", []) + if not isinstance(est, list): + est = [] + bucket["wp_estimated_files"].append(est) + bucket["wp_count"] += 1 + + items: list[dict[str, object]] = [] + context_paths: list[str] = [] + item_ids: list[str] = [] + for phase_id in sorted(phase_buckets): + bucket = phase_buckets[phase_id] + metadata = { + "wp_count": bucket["wp_count"], + "wp_ids": bucket["wp_ids"], + "wp_names": bucket["wp_names"], + "wp_scopes": bucket["wp_scopes"], + "wp_estimated_files": bucket["wp_estimated_files"], + } + items.append( + { + "id": phase_id, + "name": bucket["name"], + "status": "pending", + "result_path": None, + "metadata": metadata, + } + ) + context: dict[str, object] = { + "id": phase_id, + "name": bucket["name"], + "task_file_path": task_file_path, + "metadata": metadata, + "prior_results": [], + "wp_index_path": str(wp_dir / "wp_index.json"), + } + ctx_path = wp_dir / f"context_{phase_id}.json" + write_versioned_json(ctx_path, context, schema_version=1) + context_paths.append(str(ctx_path)) + item_ids.append(str(phase_id)) + + sentinel_dir = wp_dir / "wp_sentinels" + sentinel_dir.mkdir(parents=True, exist_ok=True) + manifest = { + "pass_name": "phase_work_packages", + "result_dir": str(sentinel_dir), + "created_at": datetime.now(tz=UTC).strftime("%Y-%m-%dT%H:%M:%SZ"), + "items": items, + } + manifest_path = out_dir / "phase_wp_manifest.json" + write_versioned_json(manifest_path, manifest, schema_version=1) + atomic_write(wp_dir / "wp_index.json", "[]") + return { + "manifest_path": str(manifest_path), + "context_paths": ",".join(context_paths), + "item_ids": ",".join(item_ids), + } + + +def _derive_label(content: str, filename_stem: str) -> str: + first_non_empty: str | None = None + for line in content.splitlines(): + stripped = line.strip() + if stripped.startswith("# "): + return stripped[2:].strip() + if first_non_empty is None and stripped: + first_non_empty = stripped[:80] + return first_non_empty or filename_stem or "Untitled" + + +def resolve_task_input(task: str, planner_dir: str) -> TaskResolutionResult: + if not task: + raise ValueError("task must be a non-empty string") + task_path = Path(task) + if task_path.is_file(): + try: + content = task_path.read_text(encoding="utf-8") + except OSError as exc: + raise OSError(f"Cannot read task file {task_path}: {exc}") from exc + label = _derive_label(content, task_path.stem) + return TaskResolutionResult(task_file_path=str(task_path), task_label=label) + out = Path(planner_dir) / "task_input.md" + atomic_write(out, task) + label = _derive_label(task, "") + return TaskResolutionResult(task_file_path=str(out), task_label=label) diff --git a/src/autoskillit/planner/merge.py b/src/autoskillit/planner/merge.py new file mode 100644 index 0000000000..8370b7bbd8 --- /dev/null +++ b/src/autoskillit/planner/merge.py @@ -0,0 +1,376 @@ +from __future__ import annotations + +import fcntl +import json +from pathlib import Path +from typing import Any + +import regex as re + +from autoskillit.core import get_logger, write_versioned_json +from autoskillit.planner.schema import ( + ASSIGN_RESULT_FILE_RE, + PHASE_RESULT_FILE_RE, + WP_RESULT_FILE_RE, + validate_phase_result, +) + +logger = get_logger(__name__) + +_TIER_KEYS = ("phases", "assignments", "work_packages") + +_TIER_FILE_RE: dict[str, re.Pattern[str]] = { + "phases": PHASE_RESULT_FILE_RE, + "assignments": ASSIGN_RESULT_FILE_RE, + "work_packages": WP_RESULT_FILE_RE, +} + + +def merge_files( + file_paths: list[str], + output_path: str, + key: str, + task_file_path: str = "", + source_dir: str = "", + strict: bool = True, + **kwargs: Any, +) -> dict[str, Any]: + task = Path(task_file_path).read_text(encoding="utf-8") if task_file_path else "" + if key not in _TIER_KEYS: + raise ValueError(f"Invalid key {key!r}; must be one of {_TIER_KEYS}") + + out = Path(output_path) + errors: list[str] = [] + skipped: int = 0 + existing_items: list[dict[str, Any]] = [] + + with open(out, "a+b") as fh: + fcntl.flock(fh, fcntl.LOCK_EX) + try: + fh.seek(0) + content = fh.read() + existing: dict[str, Any] = json.loads(content) if content else {} + + existing_task = existing.get("task", task) + existing_source_dir = existing.get("source_dir", source_dir) + + existing_items = existing.get(key, []) + existing_ids: set[str] = {item["id"] for item in existing_items if "id" in item} + + for fp in file_paths: + p = Path(fp) + if not p.exists(): + msg = f"File not found: {fp}" + if strict: + raise ValueError(msg) + errors.append(msg) + continue + try: + item = json.loads(p.read_text()) + except json.JSONDecodeError as exc: + msg = f"Invalid JSON in {fp}: {exc}" + if strict: + raise ValueError(msg) from exc + errors.append(msg) + continue + item_id = item.get("id") + if item_id is None: + logger.debug("Skipping item with no 'id' field from %s", fp) + skipped += 1 + continue + if item_id not in existing_ids: + existing_items.append(item) + existing_ids.add(item_id) + else: + logger.debug("Skipping duplicate id %r from %s", item_id, fp) + skipped += 1 + + document: dict[str, Any] = { + "task": existing_task, + "source_dir": existing_source_dir, + key: existing_items, + } + enriched = {**document, "schema_version": 1} + payload = json.dumps(enriched, indent=2).encode() + fh.seek(0) + fh.truncate() + fh.write(payload) + fh.flush() + finally: + fcntl.flock(fh, fcntl.LOCK_UN) + + result: dict[str, Any] = { + "merged_path": str(output_path), + "item_count": str(len(existing_items)), + } + if skipped: + result["skipped_count"] = str(skipped) + if errors: + result["errors"] = errors + return result + + +def _write_refine_contexts( + planner_dir: Path, + assignments: list[dict[str, Any]], + task_file_path: str, +) -> list[str]: + phase_groups: dict[str, list[dict[str, Any]]] = {} + for assignment in assignments: + phase_id = assignment.get("phase_id", "") + if phase_id: + phase_groups.setdefault(phase_id, []).append(assignment) + else: + logger.warning( + "Assignment %r has no phase_id — skipped from refine contexts", + assignment.get("id", ""), + ) + + contexts_dir = planner_dir / "refine_contexts" + contexts_dir.mkdir(parents=True, exist_ok=True) + + context_paths: list[str] = [] + for phase_id in sorted(phase_groups): + own = phase_groups[phase_id] + peer_summaries: list[dict[str, str]] = [ + {"id": a.get("id", ""), "name": a.get("name", ""), "goal": a.get("goal", "")} + for pid, peers in sorted(phase_groups.items()) + if pid != phase_id + for a in peers + ] + context: dict[str, Any] = { + "phase_id": phase_id, + "task_file_path": task_file_path, + "assignments": own, + "peer_summaries": peer_summaries, + } + if not re.fullmatch(r"[A-Za-z0-9_\-]+", phase_id): + raise ValueError( + f"phase_id {phase_id!r} contains disallowed characters — " + "only alphanumeric, underscore, and hyphen are permitted in context filenames" + ) + ctx_path = contexts_dir / f"context_{phase_id}.json" + write_versioned_json(ctx_path, context, schema_version=1) + context_paths.append(str(ctx_path)) + + return context_paths + + +def extract_item( + source_path: str, + item_id: str, + output_path: str, + **kwargs: Any, +) -> dict[str, Any]: + src = Path(source_path) + try: + data: dict[str, Any] = json.loads(src.read_text()) + except FileNotFoundError: + raise ValueError(f"Source file not found: {source_path}") from None + except json.JSONDecodeError as exc: + raise ValueError(f"Invalid JSON in {source_path}: {exc}") from exc + for tier_key in _TIER_KEYS: + for item in data.get(tier_key, []): + if item.get("id") == item_id: + write_versioned_json(Path(output_path), item, schema_version=1) + return {"extracted_path": str(output_path)} + raise ValueError(f"Item {item_id!r} not found in {source_path}") + + +def replace_item( + source_path: str, + item_id: str, + replacement_path: str, + **kwargs: Any, +) -> dict[str, Any]: + try: + replacement: dict[str, Any] = json.loads(Path(replacement_path).read_text()) + except FileNotFoundError: + raise ValueError(f"Replacement file not found: {replacement_path}") from None + except json.JSONDecodeError as exc: + raise ValueError(f"Invalid JSON in {replacement_path}: {exc}") from exc + src = Path(source_path) + + found = False + with open(src, "r+b") as fh: + try: + fcntl.flock(fh, fcntl.LOCK_EX) + data: dict[str, Any] = json.loads(fh.read()) + for tier_key in _TIER_KEYS: + tier: list[dict[str, Any]] = data.get(tier_key, []) + for idx, item in enumerate(tier): + if item.get("id") == item_id: + tier[idx] = replacement + enriched = {**data, "schema_version": 1} + payload = json.dumps(enriched, indent=2).encode() + fh.seek(0) + fh.truncate() + fh.write(payload) + fh.flush() + found = True + break + if found: + break + finally: + fcntl.flock(fh, fcntl.LOCK_UN) + + if not found: + raise ValueError(f"Item {item_id!r} not found in {source_path}") + return {"replaced_id": item_id, "updated_path": str(source_path)} + + +def merge_tier_results( + results_dir: str, + output_path: str, + key: str, + task_file_path: str = "", + source_dir: str = "", + **kwargs: Any, +) -> dict[str, Any]: + tier_re = _TIER_FILE_RE.get(key) + paths = sorted( + f + for f in Path(results_dir).glob("*_result.json") + if tier_re is None or tier_re.match(f.name) + ) + if not paths: + raise ValueError(f"No *_result.json files found in {results_dir}") + result = merge_files( + file_paths=[str(p) for p in paths], + output_path=output_path, + key=key, + task_file_path=task_file_path, + source_dir=source_dir, + ) + if key == "assignments": + merged_data = json.loads(Path(output_path).read_text(encoding="utf-8")) + assignments = merged_data.get("assignments", []) + if not assignments: + logger.warning( + "merge_tier_results: no assignments found in %s — refine contexts will be empty", + output_path, + ) + planner_dir = Path(output_path).parent + context_paths = _write_refine_contexts(planner_dir, assignments, task_file_path) + result["refine_context_paths"] = ",".join(context_paths) + return result + + +def merge_refined_assignments( + planner_dir: str, + **kwargs: Any, +) -> dict[str, Any]: + contexts_dir = Path(planner_dir) / "refine_contexts" + result_files = sorted(contexts_dir.glob("*_result.json")) + if not result_files: + raise ValueError( + f"No *_result.json files found in {contexts_dir}. " + "Run refine_assignments dispatch before calling this function." + ) + + all_assignments: list[dict[str, Any]] = [] + for path in result_files: + try: + data = json.loads(path.read_text(encoding="utf-8")) + except (json.JSONDecodeError, OSError) as exc: + logger.warning("Skipping malformed result file %s: %s", path, exc) + continue + all_assignments.extend(data.get("assignments", [])) + + valid_assignments: list[dict[str, Any]] = [] + for a in all_assignments: + if not a.get("id"): + logger.warning( + "merge_refined_assignments: skipping assignment with missing id: %r", + a.get("name", ""), + ) + else: + valid_assignments.append(a) + all_assignments = valid_assignments + + def _sort_key(assignment_id: str) -> tuple[int, ...]: + return tuple(int(n) for n in re.findall(r"\d+", assignment_id)) + + # Single-pass: for each (file, assignment_id) claim, keep the earliest assignment_id + file_owner: dict[str, str] = {} + for assignment in all_assignments: + aid = assignment.get("id", "") + for wp in assignment.get("proposed_work_packages", []): + for f in wp.get("files_touched", []): + if f not in file_owner or _sort_key(aid) < _sort_key(file_owner[f]): + file_owner[f] = aid + + # Count conflicts: files with more than one claimant + file_claimants: dict[str, set[str]] = {} + for assignment in all_assignments: + aid = assignment.get("id", "") + for wp in assignment.get("proposed_work_packages", []): + for f in wp.get("files_touched", []): + file_claimants.setdefault(f, set()).add(aid) + conflict_count = sum(1 for claimants in file_claimants.values() if len(claimants) > 1) + + # Strip files from losing assignments + for assignment in all_assignments: + aid = assignment.get("id", "") + for wp in assignment.get("proposed_work_packages", []): + wp["files_touched"] = [ + f for f in wp.get("files_touched", []) if file_owner.get(f) == aid + ] + if not wp["files_touched"]: + logger.warning( + "WP %s in assignment %s has empty files_touched after conflict resolution", + wp.get("id", wp.get("name", "")), + aid, + ) + + output_path = Path(planner_dir) / "refined_assignments.json" + write_versioned_json(output_path, {"assignments": all_assignments}, schema_version=1) + + return { + "refined_assignments_path": str(output_path), + "item_count": str(len(all_assignments)), + "conflict_count": str(conflict_count), + } + + +def build_plan_snapshot( + phases_dir: str, + output_path: str, + task_file_path: str = "", + source_dir: str = "", + **kwargs: Any, +) -> dict[str, Any]: + task = Path(task_file_path).read_text(encoding="utf-8") if task_file_path else "" + phase_pairs: list[tuple[int, dict[str, Any]]] = [] + for p in sorted( + f for f in Path(phases_dir).glob("*_result.json") if PHASE_RESULT_FILE_RE.match(f.name) + ): + try: + raw = json.loads(p.read_text()) + validated = validate_phase_result(raw) + except (ValueError, json.JSONDecodeError) as exc: + logger.warning("Skipping malformed phase file %s: %s", p, exc) + continue + short: dict[str, Any] = { + "id": validated["id"], + "name": validated["name"], + "goal": validated.get("goal", ""), + "scope": validated.get("scope", []), + "ordering": validated["ordering"], + } + phase_pairs.append((int(validated["ordering"]), short)) + + phase_pairs.sort(key=lambda x: x[0]) + phases = [pair[1] for pair in phase_pairs] + + document: dict[str, Any] = { + "task": task, + "source_dir": source_dir, + "phases": phases, + } + write_versioned_json(Path(output_path), document, schema_version=1) + + return { + "snapshot_path": str(output_path), + "phase_ids": ",".join(ph["id"] for ph in phases), + } diff --git a/src/autoskillit/planner/schema.py b/src/autoskillit/planner/schema.py new file mode 100644 index 0000000000..59d7c1f4d4 --- /dev/null +++ b/src/autoskillit/planner/schema.py @@ -0,0 +1,340 @@ +from __future__ import annotations + +import warnings +from typing import Any, Literal, TypedDict + +import regex as re + + +class PhaseResult(TypedDict): + id: str + phase_number: int + name: str + name_slug: str + goal: str + scope: list[str] + ordering: int + assignments_preview: list[str] + assignments: list[dict[str, Any]] + relationship_notes: str + + +class AssignmentResult(TypedDict): + id: str + phase_number: int + assignment_number: int + name: str + phase_id: str + goal: str + technical_approach: str + proposed_work_packages: list[dict[str, Any]] + + +class WPResult(TypedDict): + id: str + name: str + summary: str + goal: str + technical_steps: list[str] + files_touched: list[str] + apis_defined: list[str] + apis_consumed: list[str] + depends_on: list[str] + deliverables: list[str] + acceptance_criteria: list[str] + + +class PhaseShort(TypedDict): + id: str + name: str + goal: str + scope: list[str] + ordering: int + + +class PhaseElaborated(TypedDict): + id: str + name: str + goal: str + scope: list[str] + technical_approach: str + relationship_notes: str + assignments_preview: list[str] + ordering: int + + +class AssignmentShort(TypedDict): + id: str + phase_id: str + name: str + goal: str + + +class AssignmentElaborated(TypedDict): + id: str + phase_id: str + name: str + goal: str + technical_approach: str + proposed_work_packages: list[dict[str, Any]] + dependency_notes: str + overlap_notes: str + + +class WPShort(TypedDict): + id: str + assignment_id: str + phase_id: str + name: str + goal: str + scope: str + estimated_files: list[str] + + +class WPElaborated(TypedDict): + id: str + assignment_id: str + phase_id: str + name: str + scope: str + estimated_files: list[str] + goal: str + summary: str + technical_steps: list[str] + files_touched: list[str] + apis_defined: list[str] + apis_consumed: list[str] + depends_on: list[str] + deliverables: list[str] + acceptance_criteria: list[str] + + +class _PlanDocumentBase(TypedDict): + task: str + source_dir: str + + +class PlanDocument(_PlanDocumentBase, total=False): + phases: list[PhaseShort | PhaseElaborated] + assignments: list[AssignmentShort | AssignmentElaborated] + work_packages: list[WPShort | WPElaborated] + + +class PlannerManifestItem(TypedDict): + id: str + name: str + status: str + result_path: str | None + metadata: dict[str, Any] + + +class PlannerManifest(TypedDict): + pass_name: str + result_dir: str + created_at: str + items: list[PlannerManifestItem] + + +class RunDirResult(TypedDict): + planner_dir: str + + +class TaskResolutionResult(TypedDict): + task_file_path: str + task_label: str + + +class ValidationFinding(TypedDict): + message: str + severity: Literal["error", "warning"] + check: str + + +_PHASE_ID_RE = re.compile(r"^P\d+$") +_ASSIGN_ID_RE = re.compile(r"^P\d+-A\d+$") +_WP_ID_RE = re.compile(r"^P\d+-A\d+-WP\d+$") + +PHASE_RESULT_FILE_RE = re.compile(r"^P\d+_result\.json$") +ASSIGN_RESULT_FILE_RE = re.compile(r"^P\d+-A\d+_result\.json$") +WP_RESULT_FILE_RE = re.compile(r"^P\d+-A\d+-WP\d+_result\.json$") + +PHASE_REQUIRED_KEYS: frozenset[str] = frozenset({"id", "name", "ordering"}) +ASSIGNMENT_REQUIRED_KEYS: frozenset[str] = frozenset({"id", "name", "proposed_work_packages"}) +WP_REQUIRED_KEYS: frozenset[str] = frozenset({"id", "name", "deliverables"}) + +DELIVERABLE_BOUNDS: tuple[int, int] = (1, 5) + + +def _slugify(name: str) -> str: + slug = name.lower() + slug = re.sub(r"[^a-z0-9]+", "-", slug) + return slug.strip("-") + + +def _parse_phase_number(data: dict[str, Any]) -> int: + if "ordering" in data: + return int(data["ordering"]) + return int(str(data["id"])[1:]) + + +def _parse_assignment_numbers(data: dict[str, Any]) -> tuple[int, int]: + raw_id = str(data["id"]) + parts = raw_id.split("-") + if len(parts) < 2: + raise ValueError(f"Assignment id {raw_id!r} is missing '-' separator") + try: + return int(parts[0][1:]), int(parts[1][1:]) + except ValueError: + raise ValueError(f"Assignment id {raw_id!r} has malformed numeric segment") from None + + +def validate_phase_result(data: dict[str, Any]) -> dict[str, Any]: + missing = PHASE_REQUIRED_KEYS - data.keys() + if missing: + raise ValueError(f"Phase result missing required keys: {sorted(missing)}") + + result: dict[str, Any] = dict(data) + + if "phase_number" not in result: + result["phase_number"] = _parse_phase_number(data) + + if "name_slug" not in result: + result["name_slug"] = _slugify(data["name"]) + + if "assignments" not in result: + preview = result.get("assignments_preview", []) + result["assignments"] = [{"name": name, "metadata": {}} for name in preview] + + result.setdefault("assignments_preview", []) + result.setdefault("goal", "") + result.setdefault("scope", []) + result.setdefault("relationship_notes", "") + + return result + + +def validate_assignment_result(data: dict[str, Any]) -> dict[str, Any]: + missing = ASSIGNMENT_REQUIRED_KEYS - data.keys() + if missing: + raise ValueError(f"Assignment result missing required keys: {sorted(missing)}") + + result: dict[str, Any] = dict(data) + + if "phase_number" not in result or "assignment_number" not in result: + pn, an = _parse_assignment_numbers(data) + result.setdefault("phase_number", pn) + result.setdefault("assignment_number", an) + + result.setdefault("phase_id", f"P{result['phase_number']}") + result.setdefault("goal", "") + result.setdefault("technical_approach", "") + + return result + + +def validate_wp_result(data: dict[str, Any], *, allow_stub: bool = False) -> dict[str, Any]: + missing = WP_REQUIRED_KEYS - data.keys() + if missing: + raise ValueError(f"WP result missing required keys: {sorted(missing)}") + + _reject_empty_string_ids(data, "WP result") + + result: dict[str, Any] = dict(data) + + wp_id = result["id"] + if not _WP_ID_RE.match(wp_id): + warnings.warn( + f"WP id {wp_id!r} does not match expected PX-AY-WPZ format", + stacklevel=2, + ) + + if not allow_stub: + count = len(result.get("deliverables", [])) + lo, hi = DELIVERABLE_BOUNDS + if not (lo <= count <= hi): + raise ValueError(f"WP {wp_id} has {count} deliverables (must be {lo}–{hi})") + + result.setdefault("summary", "") + result.setdefault("goal", "") + result.setdefault("technical_steps", []) + result.setdefault("files_touched", []) + result.setdefault("apis_defined", []) + result.setdefault("apis_consumed", []) + result.setdefault("depends_on", []) + result.setdefault("acceptance_criteria", []) + + return result + + +def _reject_empty_string_ids(data: dict[str, Any], context: str) -> None: + if "id" in data and data["id"] == "": + raise ValueError(f"{context}: 'id' field is present but empty") + + +def resolve_wp_id(wp: dict[str, Any], assign_id: str) -> str: + wp_id = wp.get("id", "") + if wp_id: + return wp_id + id_suffix = wp.get("id_suffix", "") + if id_suffix: + return f"{assign_id}-{id_suffix}" + raise ValueError(f"Work package in assignment {assign_id!r} has neither 'id' nor 'id_suffix'") + + +def validate_refined_assignments(data: dict[str, Any]) -> dict[str, Any]: + """Validate and normalize refined_assignments.json at ingestion.""" + result: dict[str, Any] = dict(data) + assignments = result.get("assignments", []) + if not assignments: + raise ValueError("refined_assignments must contain non-empty 'assignments' list") + + result["assignments"] = [dict(a) for a in assignments] + + for assign in result["assignments"]: + assign_id = assign.get("id", "") + if not assign_id: + phase_id = assign.get("phase_id", "") + pn = assign.get("phase_number", 0) + an = assign.get("assignment_number", 0) + if not phase_id and not pn: + raise ValueError( + "assignment has no resolvable id: needs 'id', " + "or 'phase_id'+'assignment_number', " + "or 'phase_number'+'assignment_number'" + ) + if not phase_id: + phase_id = f"P{pn}" + assign_id = f"{phase_id}-A{an}" + assign["id"] = assign_id + + assign["proposed_work_packages"] = [ + dict(wp) for wp in assign.get("proposed_work_packages", []) + ] + for wp in assign["proposed_work_packages"]: + wp["id"] = resolve_wp_id(wp, assign_id) + + return result + + +def validate_refined_plan(data: dict[str, Any]) -> dict[str, Any]: + """Validate and normalize refined_plan.json at ingestion.""" + phases = data.get("phases", []) + if not phases: + raise ValueError("refined_plan must contain non-empty 'phases' list") + + for phase in phases: + phase_id = phase.get("id", "") + if not phase_id: + raise ValueError(f"Phase has empty 'id': {phase.get('name', '')}") + previews = phase.get("assignments_preview", []) + for i, preview in enumerate(previews): + if ( + isinstance(preview, dict) + and not preview.get("id", "") + and not preview.get("name", "") + ): + raise ValueError( + f"Phase {phase_id} assignments_preview[{i}] has neither 'id' nor 'name'" + ) + + return data diff --git a/src/autoskillit/planner/validation.py b/src/autoskillit/planner/validation.py new file mode 100644 index 0000000000..2b32df0d1b --- /dev/null +++ b/src/autoskillit/planner/validation.py @@ -0,0 +1,364 @@ +"""Planner validation: structural completeness, DAG acyclicity, sizing bounds. + +All loaders read from individual ``*_result.json`` files in the ``phases/``, +``assignments/``, and ``work_packages/`` subdirectories. Combined documents +(``combined_*.json``, ``refined_*.json``) are intermediate orchestration +artifacts produced by the merge/refine cycle — they are **not** authoritative +and are never consumed here. +""" + +from __future__ import annotations + +import json +from collections import deque +from collections.abc import Iterator +from pathlib import Path + +import regex as re + +from autoskillit.core import get_logger, write_versioned_json +from autoskillit.planner.schema import ( + ASSIGN_RESULT_FILE_RE, + DELIVERABLE_BOUNDS, + PHASE_RESULT_FILE_RE, + WP_RESULT_FILE_RE, + ValidationFinding, + validate_assignment_result, + validate_phase_result, + validate_wp_result, +) + +logger = get_logger(__name__) + +_VERSION_BUMP_PATTERNS: tuple[re.Pattern[str], ...] = tuple( + re.compile(p, re.IGNORECASE) + for p in [ + r"(?:bump|edit|set|update|modify|change)\s[\s\S]{0,40}pyproject\.toml", + r"pyproject\.toml[\s\S]{0,40}version\s*=", + r"sync[-_]versions", + r"task\s+sync-versions", + r"\bversion[\s_-]*bump\b", + ] +) + +_NEGATION_PREFIX_RE: re.Pattern[str] = re.compile( + r"^\s*(?:no|not|skip|avoid|omit|don'?t|do\s+not)\b", + re.IGNORECASE, +) + + +def _iter_tier_files(directory: Path, filename_re: re.Pattern[str]) -> Iterator[Path]: + """Yield *_result.json files whose names match the tier's naming convention.""" + for f in sorted(directory.glob("*_result.json")): + if filename_re.match(f.name): + yield f + + +def _load_phase_results(root: Path) -> dict[str, dict]: + results: dict[str, dict] = {} + for f in _iter_tier_files(root / "phases", PHASE_RESULT_FILE_RE): + try: + raw = json.loads(f.read_text()) + data = validate_phase_result(raw) + phase_id = f"P{data['phase_number']}" + except (json.JSONDecodeError, KeyError, ValueError) as exc: + raise RuntimeError(f"Malformed phase result file {f}: {exc}") from exc + results[phase_id] = data + return results + + +def _load_assignment_results(root: Path) -> dict[str, dict]: + results: dict[str, dict] = {} + assign_dir = root / "assignments" + if not assign_dir.exists(): + return results + for f in _iter_tier_files(assign_dir, ASSIGN_RESULT_FILE_RE): + try: + raw = json.loads(f.read_text()) + data = validate_assignment_result(raw) + assign_id = f"P{data['phase_number']}-A{data['assignment_number']}" + except (json.JSONDecodeError, KeyError, ValueError) as exc: + raise RuntimeError(f"Malformed assignment result file {f}: {exc}") from exc + results[assign_id] = data + return results + + +def _load_wp_results(root: Path) -> dict[str, dict]: + results: dict[str, dict] = {} + wp_dir = root / "work_packages" + if not wp_dir.exists(): + return results + for f in _iter_tier_files(wp_dir, WP_RESULT_FILE_RE): + try: + raw = json.loads(f.read_text()) + data = validate_wp_result(raw, allow_stub=True) + results[data["id"]] = data + except (json.JSONDecodeError, KeyError, ValueError) as exc: + raise RuntimeError(f"Malformed WP result file {f}: {exc}") from exc + return results + + +def _load_wp_manifest(root: Path) -> dict | None: + manifest_path = root / "work_packages" / "wp_manifest.json" + if not manifest_path.exists(): + return None + try: + return json.loads(manifest_path.read_text()) + except json.JSONDecodeError as exc: + raise RuntimeError(f"Malformed WP manifest file {manifest_path}: {exc}") from exc + + +def _inject_backward_deps(wp_results: dict[str, dict], dep_graph: dict) -> None: + for wp_id, extra_deps in dep_graph.get("added_backward_deps", {}).items(): + if wp_id not in wp_results: + continue + existing = wp_results[wp_id].setdefault("depends_on", []) + for dep in extra_deps: + if dep not in existing: + existing.append(dep) + + +def _check_phase_completeness( + phase_results: dict[str, dict], + assignment_results: dict[str, dict], +) -> list[ValidationFinding]: + findings: list[ValidationFinding] = [] + assigned_phase_nums = {v["phase_number"] for v in assignment_results.values()} + for phase_id, phase in phase_results.items(): + if phase["phase_number"] not in assigned_phase_nums: + findings.append( + { + "message": f"Phase {phase_id} has no assignments", + "severity": "error", + "check": "phase_completeness", + } + ) + return findings + + +def _check_assignment_completeness( + assignment_results: dict[str, dict], + wp_results: dict[str, dict], +) -> list[ValidationFinding]: + findings: list[ValidationFinding] = [] + wp_pairs: set[tuple[int, int]] = set() + for wp_id in wp_results: + parts = wp_id.split("-") + if len(parts) < 2: + findings.append( + { + "message": f"WP {wp_id!r} has malformed id (expected PX-AY-WPZ)", + "severity": "error", + "check": "assignment_completeness", + } + ) + continue + phase_num = int(parts[0][1:]) + assign_num = int(parts[1][1:]) + wp_pairs.add((phase_num, assign_num)) + for assign_id, assign in assignment_results.items(): + pair = (assign["phase_number"], assign["assignment_number"]) + if pair not in wp_pairs: + findings.append( + { + "message": f"Assignment {assign_id} has no work packages", + "severity": "error", + "check": "assignment_completeness", + } + ) + return findings + + +def _check_dep_references(wp_results: dict[str, dict]) -> list[ValidationFinding]: + findings: list[ValidationFinding] = [] + for wp_id, wp in wp_results.items(): + for dep in wp.get("depends_on", []): + if dep not in wp_results: + findings.append( + { + "message": f"WP {wp_id} depends on unknown WP {dep}", + "severity": "error", + "check": "dep_references", + } + ) + return findings + + +def _check_dag_acyclic(wp_results: dict[str, dict]) -> list[ValidationFinding]: + in_degree: dict[str, int] = {wp_id: 0 for wp_id in wp_results} + adjacency: dict[str, list[str]] = {wp_id: [] for wp_id in wp_results} + for wp_id, wp in wp_results.items(): + for dep in wp.get("depends_on", []): + if dep in wp_results: + adjacency[dep].append(wp_id) + in_degree[wp_id] += 1 + + queue: deque[str] = deque(k for k, v in in_degree.items() if v == 0) + sorted_nodes: list[str] = [] + while queue: + node = queue.popleft() + sorted_nodes.append(node) + for neighbor in adjacency[node]: + in_degree[neighbor] -= 1 + if in_degree[neighbor] == 0: + queue.append(neighbor) + + if len(sorted_nodes) < len(wp_results): + cycle_nodes = [n for n in wp_results if n not in set(sorted_nodes)] + return [ + { + "message": f"Cycle detected among WPs: {', '.join(sorted(cycle_nodes))}", + "severity": "error", + "check": "dag_acyclic", + } + ] + return [] + + +def _check_sizing_bounds(wp_results: dict[str, dict]) -> list[ValidationFinding]: + lo, hi = DELIVERABLE_BOUNDS + findings: list[ValidationFinding] = [] + for wp_id, wp in wp_results.items(): + count = len(wp.get("deliverables", [])) + if not (lo <= count <= hi): + findings.append( + { + "message": f"WP {wp_id} has {count} deliverables (must be {lo}–{hi})", + "severity": "error", + "check": "sizing_bounds", + } + ) + return findings + + +def _check_duplicate_deliverables(wp_results: dict[str, dict]) -> list[ValidationFinding]: + deliverable_map: dict[str, list[str]] = {} + for wp_id, wp in wp_results.items(): + for d in wp.get("deliverables", []): + deliverable_map.setdefault(d, []).append(wp_id) + findings: list[ValidationFinding] = [] + for path, owners in deliverable_map.items(): + if len(owners) > 1: + findings.append( + { + "message": ( + f"Deliverable '{path}' claimed by multiple WPs: " + f"{', '.join(sorted(owners))}" + ), + "severity": "error", + "check": "duplicate_deliverables", + } + ) + return findings + + +def _check_duplicate_files_touched(wp_results: dict[str, dict]) -> list[ValidationFinding]: + file_map: dict[str, list[str]] = {} + for wp_id, wp in wp_results.items(): + for path in wp.get("files_touched", []): + file_map.setdefault(path, []).append(wp_id) + findings: list[ValidationFinding] = [] + for path, owners in file_map.items(): + if len(owners) > 1: + findings.append( + { + "message": ( + f"File '{path}' touched by multiple WPs: {', '.join(sorted(owners))}" + ), + "severity": "warning", + "check": "duplicate_files_touched", + } + ) + return findings + + +def _check_version_bump_steps( + wp_results: dict[str, dict], +) -> list[ValidationFinding]: + findings: list[ValidationFinding] = [] + for wp_id, wp in wp_results.items(): + steps: list[str] = wp.get("technical_steps") or [] + fragments = [wp.get("name") or "", wp.get("summary") or ""] + steps + if any( + fragment + and not _NEGATION_PREFIX_RE.search(fragment) + and any(pattern.search(fragment) for pattern in _VERSION_BUMP_PATTERNS) + for fragment in fragments + ): + findings.append( + { + "message": ( + f"{wp_id}: contains manual version-bump steps; " + "CI handles version bumps automatically on merge — " + "remove these steps." + ), + "severity": "warning", + "check": "version_bump_step", + } + ) + return findings + + +def _check_failed_wps(wp_manifest: dict | None) -> list[ValidationFinding]: + if wp_manifest is None: + return [] + findings: list[ValidationFinding] = [] + for item in wp_manifest.get("items", []): + if item.get("status") == "failed": + findings.append( + { + "message": f"WP {item.get('id', '')} has status 'failed'", + "severity": "error", + "check": "failed_wps", + } + ) + return findings + + +def validate_plan(output_dir: str) -> dict[str, str]: + root = Path(output_dir) + phase_results = _load_phase_results(root) + assignment_results = _load_assignment_results(root) + wp_results = _load_wp_results(root) + wp_manifest = _load_wp_manifest(root) + + dep_graph_path = root / "dep_graph.json" + if dep_graph_path.exists(): + try: + dep_graph = json.loads(dep_graph_path.read_text()) + except json.JSONDecodeError as exc: + raise RuntimeError(f"Malformed dep graph file {dep_graph_path}: {exc}") from exc + _inject_backward_deps(wp_results, dep_graph) + + all_findings: list[ValidationFinding] = [] + all_findings.extend(_check_phase_completeness(phase_results, assignment_results)) + all_findings.extend(_check_assignment_completeness(assignment_results, wp_results)) + all_findings.extend(_check_dep_references(wp_results)) + all_findings.extend(_check_dag_acyclic(wp_results)) + all_findings.extend(_check_sizing_bounds(wp_results)) + all_findings.extend(_check_duplicate_deliverables(wp_results)) + all_findings.extend(_check_duplicate_files_touched(wp_results)) + all_findings.extend(_check_version_bump_steps(wp_results)) + all_findings.extend(_check_failed_wps(wp_manifest)) + + errors = [f for f in all_findings if f["severity"] == "error"] + warnings = [f for f in all_findings if f["severity"] == "warning"] + unrecognized = [f for f in all_findings if f["severity"] not in ("error", "warning")] + if unrecognized: + sev_vals = {f["severity"] for f in unrecognized} + raise ValueError(f"Unrecognized severity values in findings: {sev_vals}") + + verdict = "pass" if not errors else "fail" + validation_path = root / "validation.json" + write_versioned_json( + validation_path, + {"verdict": verdict, "findings": errors, "warnings": warnings}, + schema_version=2, + ) + logger.info("validate_plan", verdict=verdict, issue_count=len(errors)) + return { + "verdict": verdict, + "validation_path": str(validation_path), + "issue_count": str(len(errors)), + "warning_count": str(len(warnings)), + } diff --git a/src/autoskillit/recipe/CLAUDE.md b/src/autoskillit/recipe/CLAUDE.md new file mode 100644 index 0000000000..d1b51be9a5 --- /dev/null +++ b/src/autoskillit/recipe/CLAUDE.md @@ -0,0 +1,46 @@ +# recipe/ + +IL-2 recipe layer — YAML schema, validation, semantic rules, dataflow analysis. +Sub-package: rules/ (see rules/CLAUDE.md). + +## Files + +| File | Purpose | +|------|---------| +| `__init__.py` | Re-exports `load_recipe`, `validate_recipe_structure`, `analyze_dataflow` | +| `contracts.py` | Contract card generation + staleness triage | +| `io.py` | `load_recipe`, `list_recipes`, `iter_steps_with_context` | +| `order.py` | `BUNDLED_RECIPE_ORDER` — stable display order registry for Group 0 recipes | +| `loader.py` | Path-based recipe metadata utilities | +| `_api.py` | Orchestration API | +| `_cmd_rpc.py` | `run_python` callables for externalized recipe cmd scripts | +| `_recipe_ingredients.py` | `format_ingredients_table` + `LoadRecipeResult` TypedDicts | +| `_recipe_composition.py` | `_build_active_recipe` + sub-recipe merging | +| `_rule_helpers.py` | Shared helper utilities for recipe semantic rules | +| `diagrams.py` | Flow diagram generation + staleness detection | +| `_registry_utils.py` | `dir_mtime` — shared mtime helper for registry loaders | +| `experiment_type_registry.py` | `ExperimentTypeSpec`, `load_all_experiment_types`, `is_silent_type` | +| `methodology_tradition_registry.py` | `MethodologyTraditionSpec`, `VenueAppendixDef`, `load_all_methodology_traditions`, `get_methodology_tradition_by_name`, `is_out_of_scope_tradition` | +| `methodology_venue_appendix.py` | `AlternateParentDef`, `MLSubAreaFoldingDef`, `VenueAppendixMatch`, `load_ml_sub_area_folding`, `resolve_venue_appendices` — Stage B venue-appendix resolution | +| `methodology_tradition_router.py` | `TraditionRouterResult`, `UnionRuleDef`, `classify_methodology` — two-stage Tier-C router | +| `methodology_disambiguation.py` | `DisambiguationRuleDef`, `CrossTraditionOverlapDef`, `DisambiguationResult`, `disambiguate`, `load_disambiguation_rules` | +| `registry.py` | `RuleFinding`, `RuleDef`, `BlockRuleDef`, `semantic_rule` decorator | +| `repository.py` | `RecipeRepository` implementation | +| `_analysis.py` | `ValidationContext` + `make_validation_context` | +| `_analysis_graph.py` | `RouteEdge` + `build_recipe_graph` + step graph primitives | +| `_analysis_bfs.py` | `bfs_reachable` + symbolic BFS fact propagation | +| `_analysis_blocks.py` | `extract_blocks` — group steps by block annotation | +| `_analysis_detectors.py` | Dead outputs + ref invalidations + implicit handoffs | +| `_git_helpers.py` | Shared git-remote regex (`_GIT_REMOTE_COMMAND_RE`, `_LITERAL_ORIGIN_RE`) for lint rules | +| `_skill_helpers.py` | Shared helpers for skill-related semantic rules | +| `_skill_placeholder_parser.py` | Bash placeholder extraction from SKILL.md | +| `identity.py` | Recipe identity hashing — content and composite fingerprints | +| `schema.py` | `Recipe`, `RecipeStep`, `DataFlowWarning` | +| `staleness_cache.py` | Staleness cache for contract and diagram freshness checks | +| `validator.py` | `validate_recipe_structure`, `analyze_dataflow` | + +## Architecture Notes + +`registry.py` uses the `@semantic_rule` decorator pattern (same side-effect registration +as `rules/`). The `_analysis_*.py` modules form an internal BFS-based dataflow analysis +pipeline; callers use `make_validation_context` as the sole entry point. diff --git a/src/autoskillit/recipe/__init__.py b/src/autoskillit/recipe/__init__.py index 48dd98f856..abfe7789d2 100644 --- a/src/autoskillit/recipe/__init__.py +++ b/src/autoskillit/recipe/__init__.py @@ -1,42 +1,25 @@ -"""L2 recipe domain — schema, I/O, validation, and contract management.""" +"""IL-2 recipe domain — schema, I/O, validation, and contract management.""" from __future__ import annotations from autoskillit.core import get_logger -_logger = get_logger(__name__) +logger = get_logger(__name__) # Rule registration — import triggers @semantic_rule registration. -from autoskillit.recipe import rules_blocks as _rules_blocks # noqa: E402 F401 -from autoskillit.recipe import rules_bypass as _rules_bypass # noqa: E402 F401 -from autoskillit.recipe import rules_ci as _rules_ci # noqa: E402 F401 -from autoskillit.recipe import rules_clone as _rules_clone # noqa: E402 F401 -from autoskillit.recipe import rules_cmd as _rules_cmd # noqa: E402 F401 -from autoskillit.recipe import rules_contracts as _rules_contracts # noqa: E402 F401 -from autoskillit.recipe import rules_dataflow as _rules_dataflow # noqa: E402 F401 -from autoskillit.recipe import rules_fixing as _rules_fixing # noqa: E402 F401 -from autoskillit.recipe import rules_graph as _rules_graph # noqa: E402 F401 -from autoskillit.recipe import rules_inputs as _rules_inputs # noqa: E402 F401 -from autoskillit.recipe import rules_isolation as _rules_isolation # noqa: E402 F401 -from autoskillit.recipe import rules_merge as _rules_merge # noqa: E402 F401 -from autoskillit.recipe import rules_packs as _rules_packs # noqa: E402 F401 -from autoskillit.recipe import rules_reachability as _rules_reachability # noqa: E402 F401 -from autoskillit.recipe import rules_recipe as _rules_recipe # noqa: E402 F401 -from autoskillit.recipe import rules_skill_content as _rules_skill_content # noqa: E402 F401 -from autoskillit.recipe import rules_skills as _rules_skills # noqa: E402 F401 -from autoskillit.recipe import rules_tools as _rules_tools # noqa: E402 F401 -from autoskillit.recipe import rules_verdict as _rules_verdict # noqa: E402 F401 -from autoskillit.recipe import rules_worktree as _rules_worktree # noqa: E402 F401 from autoskillit.recipe._api import ( # noqa: E402 + format_recipe_list_response, + list_all, + load_and_validate, + validate_from_path, +) +from autoskillit.recipe._recipe_ingredients import ( # noqa: E402 ListRecipesResult, LoadRecipeResult, + OpenKitchenResult, RecipeListItem, build_ingredient_rows, format_ingredients_table, - format_recipe_list_response, - list_all, - load_and_validate, - validate_from_path, ) from autoskillit.recipe.contracts import ( # noqa: E402 StaleItem, @@ -54,26 +37,104 @@ load_recipe_diagram, ) from autoskillit.recipe.experiment_type_registry import ( # noqa: E402 + BUNDLED_EXPERIMENT_TYPES_DIR, ExperimentTypeSpec, + get_experiment_type_by_name, + is_silent_type, load_all_experiment_types, + load_types_from_dir, + parse_experiment_type, +) +from autoskillit.recipe.identity import ( # noqa: E402 + check_rerun_detection, + find_prior_runs, ) from autoskillit.recipe.io import ( # noqa: E402 + GROUP_LABELS, builtin_sub_recipes_dir, + find_campaign_by_name, find_recipe_by_name, find_sub_recipe_by_name, + group_rank, iter_steps_with_context, + list_campaign_recipes, list_recipes, + load_campaign_recipes_in_packs, load_recipe, ) from autoskillit.recipe.loader import parse_recipe_metadata # noqa: E402 +from autoskillit.recipe.methodology_disambiguation import ( # noqa: E402 + CrossTraditionOverlapDef, + DisambiguationExceptionDef, + DisambiguationResult, + DisambiguationRuleDef, + disambiguate, + load_disambiguation_rules, +) +from autoskillit.recipe.methodology_tradition_registry import ( # noqa: E402 + BUNDLED_METHODOLOGY_TRADITIONS_DIR, + MethodologyTraditionSpec, + VenueAppendixDef, + get_methodology_tradition_by_name, + is_out_of_scope_tradition, + load_all_methodology_traditions, + load_traditions_from_dir, + parse_methodology_tradition, +) +from autoskillit.recipe.methodology_tradition_router import ( # noqa: E402 + TraditionRouterResult, + UnionRuleDef, + classify_methodology, +) +from autoskillit.recipe.methodology_venue_appendix import ( # noqa: E402 + AlternateParentDef, + MLSubAreaFoldingDef, + VenueAppendixMatch, + load_ml_sub_area_folding, + resolve_venue_appendices, +) from autoskillit.recipe.repository import DefaultRecipeRepository # noqa: E402 +from autoskillit.recipe.rules import rules_actions as _rules_actions # noqa: E402 F401 +from autoskillit.recipe.rules import rules_blocks as _rules_blocks # noqa: E402 F401 +from autoskillit.recipe.rules import rules_bypass as _rules_bypass # noqa: E402 F401 +from autoskillit.recipe.rules import ( # noqa: E402 + rules_callable_scope as _rules_callable_scope, # noqa: F401 +) +from autoskillit.recipe.rules import rules_campaign as _rules_campaign # noqa: E402 F401 +from autoskillit.recipe.rules import rules_ci as _rules_ci # noqa: E402 F401 +from autoskillit.recipe.rules import rules_clone as _rules_clone # noqa: E402 F401 +from autoskillit.recipe.rules import rules_cmd as _rules_cmd # noqa: E402 F401 +from autoskillit.recipe.rules import rules_contracts as _rules_contracts # noqa: E402 F401 +from autoskillit.recipe.rules import rules_dataflow as _rules_dataflow # noqa: E402 F401 +from autoskillit.recipe.rules import rules_features as _rules_features # noqa: E402 F401 +from autoskillit.recipe.rules import rules_fixing as _rules_fixing # noqa: E402 F401 +from autoskillit.recipe.rules import rules_food_truck as _rules_food_truck # noqa: E402 F401 +from autoskillit.recipe.rules import rules_graph as _rules_graph # noqa: E402 F401 +from autoskillit.recipe.rules import rules_inline_script as _rules_inline_script # noqa: E402 F401 +from autoskillit.recipe.rules import rules_inputs as _rules_inputs # noqa: E402 F401 +from autoskillit.recipe.rules import rules_isolation as _rules_isolation # noqa: E402 F401 +from autoskillit.recipe.rules import rules_merge as _rules_merge # noqa: E402 F401 +from autoskillit.recipe.rules import rules_merge_queue as _rules_merge_queue # noqa: E402 F401 +from autoskillit.recipe.rules import rules_packs as _rules_packs # noqa: E402 F401 +from autoskillit.recipe.rules import rules_reachability as _rules_reachability # noqa: E402 F401 +from autoskillit.recipe.rules import rules_recipe as _rules_recipe # noqa: E402 F401 +from autoskillit.recipe.rules import rules_skill_content as _rules_skill_content # noqa: E402 F401 +from autoskillit.recipe.rules import rules_skills as _rules_skills # noqa: E402 F401 +from autoskillit.recipe.rules import rules_temp_path as _rules_temp_path # noqa: E402 F401 +from autoskillit.recipe.rules import rules_tools as _rules_tools # noqa: E402 F401 +from autoskillit.recipe.rules import rules_verdict as _rules_verdict # noqa: E402 F401 +from autoskillit.recipe.rules import rules_worktree as _rules_worktree # noqa: E402 F401 from autoskillit.recipe.schema import ( # noqa: E402 AUTOSKILLIT_VERSION_KEY, + CAMPAIGN_REF_RE, + NON_INTERACTIVE_KINDS, + CampaignDispatch, DataFlowReport, Recipe, RecipeBlock, RecipeInfo, RecipeIngredient, + RecipeKind, RecipeStep, StepResultCondition, StepResultRoute, @@ -89,12 +150,15 @@ analyze_dataflow, make_validation_context, run_semantic_rules, - validate_recipe, + validate_recipe_structure, ) __all__ = [ + "GROUP_LABELS", + "group_rank", "ListRecipesResult", "LoadRecipeResult", + "OpenKitchenResult", "RecipeListItem", "build_ingredient_rows", "Recipe", @@ -103,6 +167,7 @@ "RecipeIngredient", "RecipeStep", "AUTOSKILLIT_VERSION_KEY", + "CAMPAIGN_REF_RE", "StepResultCondition", "StepResultRoute", "DataFlowReport", @@ -116,7 +181,7 @@ "list_recipes", "find_recipe_by_name", "iter_steps_with_context", - "validate_recipe", + "validate_recipe_structure", "make_validation_context", "run_semantic_rules", "analyze_dataflow", @@ -139,6 +204,43 @@ "diagram_stale_to_suggestions", "builtin_sub_recipes_dir", "find_sub_recipe_by_name", + "BUNDLED_EXPERIMENT_TYPES_DIR", + "BUNDLED_METHODOLOGY_TRADITIONS_DIR", "ExperimentTypeSpec", + "load_types_from_dir", + "parse_experiment_type", + "get_experiment_type_by_name", + "is_silent_type", "load_all_experiment_types", + "MethodologyTraditionSpec", + "parse_methodology_tradition", + "load_traditions_from_dir", + "get_methodology_tradition_by_name", + "is_out_of_scope_tradition", + "load_all_methodology_traditions", + "VenueAppendixDef", + "AlternateParentDef", + "MLSubAreaFoldingDef", + "VenueAppendixMatch", + "load_ml_sub_area_folding", + "resolve_venue_appendices", + "CrossTraditionOverlapDef", + "DisambiguationExceptionDef", + "DisambiguationResult", + "DisambiguationRuleDef", + "disambiguate", + "load_disambiguation_rules", + # --- methodology tradition router --- + "TraditionRouterResult", + "UnionRuleDef", + "classify_methodology", + # --- rerun detection --- + "check_rerun_detection", + "find_prior_runs", + "CampaignDispatch", + "NON_INTERACTIVE_KINDS", + "RecipeKind", + "find_campaign_by_name", + "list_campaign_recipes", + "load_campaign_recipes_in_packs", ] diff --git a/src/autoskillit/recipe/_analysis.py b/src/autoskillit/recipe/_analysis.py index 4fc6f82e47..204c2d0856 100644 --- a/src/autoskillit/recipe/_analysis.py +++ b/src/autoskillit/recipe/_analysis.py @@ -10,197 +10,52 @@ from __future__ import annotations -import re -from collections import deque from dataclasses import dataclass, field from pathlib import Path +from typing import TYPE_CHECKING -import igraph +if TYPE_CHECKING: + from autoskillit.core import SkillResolver -from autoskillit.core import SKILL_TOOLS, get_logger -from autoskillit.recipe.contracts import _CONTEXT_REF_RE, RESULT_CAPTURE_RE +from autoskillit.recipe._analysis_bfs import _bfs_with_facts, bfs_reachable +from autoskillit.recipe._analysis_blocks import extract_blocks +from autoskillit.recipe._analysis_detectors import ( + _detect_dead_outputs, + _detect_implicit_handoffs, + _detect_ref_invalidations, +) +from autoskillit.recipe._analysis_graph import ( + RouteEdge, + _build_step_graph, + _extract_routing_edges, + _is_infrastructure_step, + build_recipe_graph, +) from autoskillit.recipe.io import iter_steps_with_context # noqa: F401 — re-exported for rules from autoskillit.recipe.schema import ( - _TERMINAL_TARGETS, DataFlowReport, DataFlowWarning, Recipe, RecipeBlock, - RecipeStep, ) -logger = get_logger(__name__) - - -# --------------------------------------------------------------------------- -# Infrastructure step classification (shared with diagrams rendering) -# --------------------------------------------------------------------------- - - -def _is_infrastructure_step(step: RecipeStep) -> bool: - """Return True if *step* is a plumbing step that should be hidden from diagrams. - - Infrastructure steps are ``run_cmd`` steps whose sole purpose is capturing - or setting a context value (git rev-parse, printf, echo one-liners). - They add no user-visible behaviour to the pipeline flow. - """ - if step.tool != "run_cmd": - return False - note_lower = (step.note or "").lower() - cmd = "" - if step.with_args and isinstance(step.with_args, dict): - cmd = step.with_args.get("cmd", "") or "" - return ( - "capture" in note_lower - or "set" in note_lower - or "printf" in cmd - or "git rev-parse" in cmd - or (cmd.strip().startswith("echo") and "\n" not in cmd) - ) - - -# --------------------------------------------------------------------------- -# igraph recipe graph builder -# --------------------------------------------------------------------------- - - -def build_recipe_graph(recipe: Recipe) -> igraph.Graph: - """Build a directed igraph.Graph from a Recipe dataclass. - - Nodes represent recipe steps. Each vertex carries attributes matching the - RecipeStep fields relevant to diagram rendering: - - ``name``: step name (str) - - ``tool``: tool identifier (str, empty string if None) - - ``action``: action identifier (str, empty string if None) - - ``note``: step note for semantic gate checks (str) - - ``retries``: retry count (int) - - ``skip_when_false``: optional condition (str, empty string if None) - - ``is_infra``: whether the step is a hidden infrastructure step (bool) - - ``is_terminal``: whether the step is a stop action (bool) - - ``is_confirm``: whether the step is a confirm action (bool) - - Edges represent routing connections. Each edge carries: - - ``edge_type``: one of ``"success"``, ``"failure"``, ``"context_limit"``, - ``"result_condition"``, ``"exhausted"`` - - ``condition``: for ``on_result`` edges, the ``when`` expression; otherwise ``""`` - - Args: - recipe: The loaded Recipe dataclass. - - Returns: - A directed ``igraph.Graph`` with vertex and edge attributes as described. - """ - step_names = list(recipe.steps.keys()) - name_to_id: dict[str, int] = {name: i for i, name in enumerate(step_names)} - - g = igraph.Graph(n=len(step_names), directed=True) - steps_list = list(recipe.steps.values()) - - g.vs["name"] = step_names - g.vs["tool"] = [s.tool or "" for s in steps_list] - g.vs["action"] = [s.action or "" for s in steps_list] - g.vs["note"] = [s.note or "" for s in steps_list] - g.vs["retries"] = [s.retries for s in steps_list] - g.vs["skip_when_false"] = [s.skip_when_false or "" for s in steps_list] - g.vs["is_infra"] = [_is_infrastructure_step(s) for s in steps_list] - g.vs["is_terminal"] = [s.action == "stop" for s in steps_list] - g.vs["is_confirm"] = [s.action == "confirm" for s in steps_list] - - edges: list[tuple[int, int]] = [] - edge_types: list[str] = [] - edge_conditions: list[str] = [] - - for name, step in recipe.steps.items(): - src = name_to_id[name] - for edge in _extract_routing_edges(step): - # Mirror _build_step_graph: skip on_exhausted edges for action steps - # (stop/confirm/route steps have no retry semantics). - if edge.edge_type == "exhausted" and step.action is not None: - continue - if edge.target in name_to_id: - edges.append((src, name_to_id[edge.target])) - edge_types.append(edge.edge_type) - edge_conditions.append(edge.condition or "") - elif edge.target in _TERMINAL_TARGETS: - # Known sentinel — valid target, no graph edge needed. - pass - else: - logger.warning( - "build_recipe_graph: step %r references unknown target %r — edge skipped", - name, - edge.target, - ) - - if edges: - g.add_edges(edges, attributes={"edge_type": edge_types, "condition": edge_conditions}) - - return g - - -# --------------------------------------------------------------------------- -# Routing edge extraction -# --------------------------------------------------------------------------- - - -@dataclass -class RouteEdge: - """A single routing edge from a recipe step to a target step. - - Attributes: - edge_type: One of ``"success"``, ``"failure"``, ``"context_limit"``, - ``"result_condition"``, ``"exhausted"``. - target: The target step name. - condition: Populated for ``on_result`` conditions — the ``when`` expression. - """ - - edge_type: str - target: str - condition: str | None = None - - -def _extract_routing_edges(step: RecipeStep) -> list[RouteEdge]: - """Return all routing edges declared on *step*. - - Covers every routing field on :class:`RecipeStep`: - ``on_success``, ``on_failure``, ``on_context_limit``, ``on_exhausted``, - ``on_result.conditions[].route``, and ``on_result.routes`` (dict form). - - None targets are skipped. The caller is responsible for filtering by - known step names if graph-membership checks are needed. - """ - edges: list[RouteEdge] = [] - - if step.on_success: - edges.append(RouteEdge(edge_type="success", target=step.on_success)) - if step.on_failure: - edges.append(RouteEdge(edge_type="failure", target=step.on_failure)) - if step.on_context_limit: - edges.append(RouteEdge(edge_type="context_limit", target=step.on_context_limit)) - if step.on_exhausted: - edges.append(RouteEdge(edge_type="exhausted", target=step.on_exhausted)) - - if step.on_result: - sr = step.on_result - if sr.conditions: - for cond in sr.conditions: - edges.append( - RouteEdge( - edge_type="result_condition", - target=cond.route, - condition=cond.when, - ) - ) - elif sr.routes: - for key, target in sr.routes.items(): - edges.append( - RouteEdge( - edge_type="result_condition", - target=target, - condition=key, - ) - ) - - return edges +__all__ = [ + "build_recipe_graph", + "RouteEdge", + "_extract_routing_edges", + "_build_step_graph", + "_is_infrastructure_step", + "bfs_reachable", + "_bfs_with_facts", + "extract_blocks", + "_detect_dead_outputs", + "_detect_ref_invalidations", + "_detect_implicit_handoffs", + "ValidationContext", + "analyze_dataflow", + "make_validation_context", + "iter_steps_with_context", +] @dataclass @@ -219,551 +74,15 @@ class ValidationContext: available_sub_recipes: frozenset[str] = field(default_factory=frozenset) project_dir: Path | None = None disabled_subsets: frozenset[str] = field(default_factory=frozenset) + disabled_features: frozenset[str] = field(default_factory=frozenset) + provider_profiles: frozenset[str] = field(default_factory=frozenset) skill_category_map: dict[str, frozenset[str]] | None = None overridden_skills: frozenset[str] | None = None + skill_resolver: SkillResolver | None = None blocks: tuple[RecipeBlock, ...] = field(default_factory=tuple) predecessors: dict[str, set[str]] = field(default_factory=dict) -# --------------------------------------------------------------------------- -# Step routing graph -# --------------------------------------------------------------------------- - - -def _build_step_graph(recipe: Recipe) -> dict[str, set[str]]: - """Build a routing adjacency list from all step routing fields. - - Each key is a step name, each value is the set of step names - reachable in one hop (successors). Terminal targets like "done" - are excluded since they are not real steps. - """ - step_names = set(recipe.steps.keys()) - graph: dict[str, set[str]] = {name: set() for name in step_names} - - for name, step in recipe.steps.items(): - for edge in _extract_routing_edges(step): - # exhausted only applies to non-terminal steps - if edge.edge_type == "exhausted" and step.action is not None: - continue - if edge.target in step_names: - graph[name].add(edge.target) - - # Build predecessor map for bypass edge injection below. - predecessors: dict[str, set[str]] = {name: set() for name in step_names} - for name, successors in graph.items(): - for s in successors: - predecessors[s].add(name) - - # For each step with skip_when_false, add bypass edges from all predecessors - # directly to the step's routing targets (the steps to route to when skipped). - # This makes optional-step bypass paths visible to graph-based rules. - for name, step in recipe.steps.items(): - if not step.skip_when_false: - continue - # on_success bypass: predecessor → step.on_success - if step.on_success and step.on_success in step_names: - for pred in predecessors[name]: - graph[pred].add(step.on_success) - # on_result bypass: predecessor → each on_result condition route target - if step.on_result: - if step.on_result.routes: - for target in step.on_result.routes.values(): - if target in step_names: - for pred in predecessors[name]: - graph[pred].add(target) - for condition in step.on_result.conditions: - if condition.route in step_names: - for pred in predecessors[name]: - graph[pred].add(condition.route) - - # For each sub_recipe placeholder step (gate-controlled), add a bypass edge - # to the next step in YAML order. When the gate is false the step is dropped - # at load time; without this edge the next step becomes unreachable in the - # raw recipe graph, breaking reachability-based semantic rules. - step_names_list = list(recipe.steps.keys()) - for i, (name, step) in enumerate(recipe.steps.items()): - if step.sub_recipe is None or i + 1 >= len(step_names_list): - continue - next_step = step_names_list[i + 1] - graph[name].add(next_step) - for pred in predecessors[name]: - graph[pred].add(next_step) - - return graph - - -# --------------------------------------------------------------------------- -# Block extraction helpers -# --------------------------------------------------------------------------- - - -def _count_by_tool(members: list[RecipeStep]) -> dict[str, int]: - """Count tool occurrences across a list of steps.""" - counts: dict[str, int] = {} - for step in members: - key = step.tool or "" - if key: - counts[key] = counts.get(key, 0) + 1 - return counts - - -def _count_gh_api(step: RecipeStep) -> int: - """Count occurrences of 'gh api' in a run_cmd step's cmd argument.""" - if step.tool != "run_cmd": - return 0 - cmd = (step.with_args or {}).get("cmd", "") or "" - return cmd.count("gh api") - - -def extract_blocks( - recipe: Recipe, - step_graph: dict[str, set[str]], - *, - predecessors: dict[str, set[str]] | None = None, -) -> tuple[RecipeBlock, ...]: - """Extract named block regions from a recipe's routing graph. - - Groups steps that share the same ``step.block`` value, then computes the - entry step (no in-block predecessor) and exit step (no in-block successor) - for each group. The step_graph is the forward adjacency dict produced by - ``_build_step_graph``; a local inverse is built here to find predecessors. - - Steps without a ``block`` annotation are ignored. Recipes with no block - annotations return an empty tuple. - """ - if predecessors is None: - # Build predecessor map by inverting the forward step_graph edges. - predecessors = {} - for src, successors in step_graph.items(): - for dst in successors: - predecessors.setdefault(dst, set()).add(src) - - by_name: dict[str, list[RecipeStep]] = {} - for step in recipe.steps.values(): - if step.block is not None: - by_name.setdefault(step.block, []).append(step) - - blocks: list[RecipeBlock] = [] - for name, members in by_name.items(): - member_names = {s.name for s in members} - # Entry: no in-block predecessor (predecessor set ∩ member_names is empty) - entry_candidates = [ - s for s in members if not (predecessors.get(s.name, set()) & member_names) - ] - # Exit: no in-block successor (successor set ∩ member_names is empty) - exit_candidates = [ - s for s in members if not (step_graph.get(s.name, set()) & member_names) - ] - entry_name: str - if entry_candidates: - entry_name = entry_candidates[0].name - else: - logger.warning( - "block %r has no graph-reachable entry step; falling back to first member", - name, - ) - entry_name = members[0].name - - exit_name: str - if exit_candidates: - exit_name = exit_candidates[0].name - else: - logger.warning( - "block %r has no graph-reachable exit step; falling back to last member", - name, - ) - exit_name = members[-1].name - - blocks.append( - RecipeBlock( - name=name, - entry=entry_name, - exit=exit_name, - members=tuple(members), - tool_counts=_count_by_tool(members), - gh_api_occurrences=sum(_count_gh_api(s) for s in members), - ) - ) - return tuple(blocks) - - -# --------------------------------------------------------------------------- -# BFS helpers -# --------------------------------------------------------------------------- - - -def _bfs_reachable(graph: dict[str, set[str]], start: str) -> set[str]: - """Return all step names reachable from ``start`` in the routing graph (excluding start).""" - visited: set[str] = set() - queue = list(graph.get(start, set())) - while queue: - node = queue.pop() - if node in visited: - continue - visited.add(node) - queue.extend(graph.get(node, set())) - return visited - - -def _build_capture_origin_map(recipe: Recipe) -> dict[str, str]: - """Return {context_var_name: result_key} for all capture declarations. - - Only captures with a single extractable result.KEY reference are included. - Used to determine provenance of context variables — specifically whether - a variable was sourced from an ephemeral resource key like worktree_path - or branch_name. - """ - origin: dict[str, str] = {} - for step in recipe.steps.values(): - for cap_var, cap_expr in (step.capture or {}).items(): - if isinstance(cap_expr, str): - keys = RESULT_CAPTURE_RE.findall(cap_expr) - if len(keys) == 1: - origin[cap_var] = keys[0] - return origin - - -# Keys that become invalid after the given tool runs -_INVALIDATING_TOOLS: dict[str, frozenset[str]] = { - "merge_worktree": frozenset({"worktree_path", "branch_name"}), - "remove_clone": frozenset({"clone_path"}), -} - - -def _bfs_capped( - graph: dict[str, set[str]], - start_nodes: set[str], - barrier_nodes: set[str], -) -> set[str]: - """BFS from start_nodes, visiting but not expanding barrier_nodes. - - Used to find steps reachable from a starting set where certain steps - (re-capture barriers) reset a variable's provenance — steps reachable - only through a barrier are excluded from the result's expansion. - - Returns all visited nodes (including barrier_nodes that were reached). - """ - visited: set[str] = set() - queue = list(start_nodes) - while queue: - node = queue.pop() - if node in visited: - continue - visited.add(node) - if node in barrier_nodes: - continue # Reached but do not expand — variable is refreshed here - queue.extend(graph.get(node, set())) - return visited - - -# --------------------------------------------------------------------------- -# Symbolic reachability: BFS with fact propagation -# --------------------------------------------------------------------------- - -# A FactSet is a frozen set of (variable, value) pairs established by -# conditional on_result.when edges. Each frozenset represents the facts -# that are known-to-be-true on one particular path through the routing graph. -_FactSet = frozenset[tuple[str, str]] - -# Matches simple equality conditions of the form: -# context.X == 'v' or ${{ context.X }} == "v" -# Capture groups: 1 = variable name, 2 = value. -# Non-equality expressions (inequalities, conjunctions) produce no match — -# conservative assumption (no fact is established). -_SIMPLE_WHEN_RE = re.compile( - r"(?:\$\{\{\s*)?context\.(\w+)(?:\s*\}\})?\s*==\s*" - r"(?:\"(\w+)\"|'(\w+)'|(\w+))" -) - - -def _parse_when_expr(expr: str) -> tuple[str, str] | None: - """Parse a simple equality when-expression into a (variable, value) fact. - - Returns ``None`` for conjunctions, inequalities, or non-context refs. - Conservative: only establishes facts for provably-simple equality conditions. - """ - m = _SIMPLE_WHEN_RE.fullmatch(expr.strip()) - if not m: - return None - # Groups 2/3/4 are mutually exclusive (double-quoted, single-quoted, unquoted) - value = m.group(2) or m.group(3) or m.group(4) - return (m.group(1), value) - - -def _edge_fact(recipe: Recipe, source: str, target: str) -> tuple[str, str] | None: - """Return the (variable, value) fact established by the edge source→target, or None. - - Only ``on_result.conditions`` edges with a parseable simple equality ``when`` - expression contribute a fact. ``on_success``/``on_failure`` edges contribute no fact. - """ - step = recipe.steps.get(source) - if step is None or step.on_result is None: - return None - for cond in step.on_result.conditions: - if cond.route == target and cond.when is not None: - return _parse_when_expr(cond.when) - return None - - -def _intersect_facts(fs: set[_FactSet]) -> _FactSet: - """Intersect all fact sets — only facts that hold on every incoming path survive.""" - if not fs: - return frozenset() - return frozenset.intersection(*fs) - - -def _bfs_with_facts( - graph: dict[str, set[str]], - recipe: Recipe, - start: str, -) -> dict[str, set[_FactSet]]: - """BFS from *start* propagating conditional edge facts. - - Each ``on_result`` edge whose ``when`` expression parses as - ``'context.X == "v"'`` extends the current fact set with ``(X, v)`` on - the target; other edges carry facts unchanged. At join points, the - returned fact set is the intersection of all incoming fact sets — a fact - is only "known" at a node if it holds on every path reaching that node. - - Returns ``{step_name: {intersected_fact_set}}``. Each value is a - single-element set containing one :class:`frozenset` of ``(var, val)`` - pairs. - """ - # facts maps step_name → set of fact-sets that have been discovered for it. - # visited tracks (node, fact_set) pairs to avoid reprocessing. - facts: dict[str, set[_FactSet]] = {start: {frozenset()}} - work: deque[str] = deque([start]) - visited: set[tuple[str, _FactSet]] = set() - - while work: - node = work.popleft() - for succ in graph.get(node, ()): - edge_fact = _edge_fact(recipe, node, succ) - for f in facts.get(node, {frozenset()}): - new_f: _FactSet = f | {edge_fact} if edge_fact else f - state = (succ, new_f) - if state in visited: - continue - visited.add(state) - facts.setdefault(succ, set()).add(new_f) - work.append(succ) - - return {n: {_intersect_facts(fs)} for n, fs in facts.items()} - - -# --------------------------------------------------------------------------- -# Detection helpers -# --------------------------------------------------------------------------- - - -def _detect_ref_invalidations(recipe: Recipe, graph: dict[str, set[str]]) -> list[DataFlowWarning]: - """Detect context variables consumed after the step that invalidated the - underlying resource. - - The resource lifecycle contract: - - ``merge_worktree`` on SUCCESS destroys the worktree directory and branch ref. - - ``remove_clone`` on SUCCESS destroys the clone directory. - - Only steps reachable via ``on_success`` from the invalidating step are checked. - Steps that re-capture the same variable are treated as barriers — they refresh - the variable to a new resource, so their successors are excluded from - the stale-ref check. - """ - origin = _build_capture_origin_map(recipe) - - # Map: result_key → set of context variable names sourced from that key - key_to_vars: dict[str, set[str]] = {} - for var, result_key in origin.items(): - key_to_vars.setdefault(result_key, set()).add(var) - - # Map: var_name → set of step names that re-capture (refresh) it - var_recapture_steps: dict[str, set[str]] = {} - for step_name, step in recipe.steps.items(): - for cap_var in step.capture or {}: - var_recapture_steps.setdefault(cap_var, set()).add(step_name) - - warnings: list[DataFlowWarning] = [] - - for step_name, step in recipe.steps.items(): - invalidated_result_keys = _INVALIDATING_TOOLS.get(step.tool or "", frozenset()) - if not invalidated_result_keys: - continue - - # Variables whose underlying resource is destroyed when this step SUCCEEDS - invalidated_vars: set[str] = set() - for result_key in invalidated_result_keys: - invalidated_vars.update(key_to_vars.get(result_key, set())) - - if not invalidated_vars: - continue - - # Only check steps reachable via on_success (failure path = resource not destroyed) - on_success_target = step.on_success - if not on_success_target or on_success_target not in recipe.steps: - continue - - for var in invalidated_vars: - # Steps that re-capture this var are barriers: they refresh the variable - # to a new resource, so their successors are NOT stale consumers. - barrier = var_recapture_steps.get(var, set()) - stale_reachable = _bfs_capped(graph, {on_success_target}, barrier) - # A loop may route the invalidating step back into the reachable set - stale_reachable.discard(step_name) - - for downstream_name in stale_reachable: - downstream = recipe.steps.get(downstream_name) - if downstream is None: - continue - - for arg_val in (downstream.with_args or {}).values(): - if not isinstance(arg_val, str): - continue - for ref_var in _CONTEXT_REF_RE.findall(arg_val): - if ref_var == var: - warnings.append( - DataFlowWarning( - code="REF_INVALIDATED", - step_name=downstream_name, - field=var, - message=( - f"Step '{downstream_name}' references " - f"context.{var} after step '{step_name}' " - f"({step.tool}) has invalidated the underlying " - f"resource. Replace with a stable alternative " - f"(e.g., a commit SHA captured before any merge " - f"begins)." - ), - ) - ) - - return warnings - - -# Observability captures: variables captured for human-readable logs, hook -# consumption, or note-driven orchestration rather than downstream recipe -# threading. Each entry is (cap_key, skill_command_fragment). A capture is -# exempt when cap_key matches AND skill_command_fragment appears in the -# step's skill_command (or tool/step_name for non-skill steps). -_OBSERVABILITY_CAPTURES: frozenset[tuple[str, str]] = frozenset( - { - ("diagnosis_path", "diagnose-ci"), - ("summary_path", "pipeline-summary"), - ("report_path", "generate-report"), - ("selected_lenses", "prepare-research-pr"), - ("selected_lenses", "prepare-pr"), - ("lens_context_paths", "prepare-research-pr"), - ("lens_context_paths", "prepare-pr"), - ("pr_url", "compose-pr"), - ("html_path", "bundle-local-report"), - ("resource_report", "stage-data"), - } -) - - -def _is_observability_capture(cap_key: str, step_name: str, step: RecipeStep) -> bool: - """Return True if *cap_key* is a known observability-only capture.""" - skill_cmd = step.with_args.get("skill_command", "") if step.with_args else "" - - # Skill-command-based exemptions (the common case). - for obs_key, fragment in _OBSERVABILITY_CAPTURES: - if cap_key == obs_key and fragment in skill_cmd: - return True - - # merge_worktree cleanup_succeeded: matched by tool name + capture value, - # not skill_command (merge_worktree is a direct tool, not a skill). - if step.tool == "merge_worktree" and "result.cleanup_succeeded" in str( - step.capture.get(cap_key, "") - ): - return True - - # export_local_bundle local_bundle_path: matched by step name (terminal - # step in the local-mode path that ends at a stop action). - if cap_key == "local_bundle_path" and step_name == "export_local_bundle": - return True - - return False - - -def _detect_dead_outputs(recipe: Recipe, graph: dict[str, set[str]]) -> list[DataFlowWarning]: - """Detect captured variables that are never consumed downstream.""" - warnings: list[DataFlowWarning] = [] - - for step_name, step in recipe.steps.items(): - if not step.capture: - continue - - # BFS: collect all steps reachable from this step - reachable = _bfs_reachable(graph, step_name) - - # Collect all context.X references in reachable steps' with_args and - # on_result condition when-expressions (route actions gate on context vars). - consumed: set[str] = set() - for reachable_name in reachable: - reachable_step = recipe.steps[reachable_name] - for arg_val in reachable_step.with_args.values(): - if not isinstance(arg_val, str): - continue - consumed.update(_CONTEXT_REF_RE.findall(arg_val)) - if reachable_step.on_result and reachable_step.on_result.conditions: - for cond in reachable_step.on_result.conditions: - if cond.when and isinstance(cond.when, str): - consumed.update(_CONTEXT_REF_RE.findall(cond.when)) - - # on_result routing — both legacy field and predicate conditions count - # as structural consumption of captured variables. - if step.on_result: - # Legacy field routing: field name matches a captured key - if step.on_result.field in step.capture: - consumed.add(step.on_result.field) - # Predicate condition routing — conditions gate on step result; - # treat all captured vars as structurally consumed. - if step.on_result.conditions: - consumed.update(step.capture.keys()) - - # Flag captured vars not consumed on any path - for cap_key in step.capture: - if cap_key not in consumed: - if _is_observability_capture(cap_key, step_name, step): - continue - warnings.append( - DataFlowWarning( - code="DEAD_OUTPUT", - step_name=step_name, - field=cap_key, - message=( - f"Step '{step_name}' captures '{cap_key}' but no " - f"reachable downstream step references " - f"${{{{ context.{cap_key} }}}}." - ), - ) - ) - - return warnings - - -def _detect_implicit_handoffs(recipe: Recipe) -> list[DataFlowWarning]: - """Detect skill-invoking steps with no capture block.""" - warnings: list[DataFlowWarning] = [] - - for step_name, step in recipe.steps.items(): - if step.tool in SKILL_TOOLS and not step.capture: - warnings.append( - DataFlowWarning( - code="IMPLICIT_HANDOFF", - step_name=step_name, - field=step.tool, - message=( - f"Step '{step_name}' calls '{step.tool}' but has no " - f"capture: block. Data flows to subsequent steps " - f"implicitly through agent context rather than " - f"explicit ${{{{ context.X }}}} wiring." - ), - ) - ) - - return warnings - - # --------------------------------------------------------------------------- # Public entry point # --------------------------------------------------------------------------- @@ -807,6 +126,9 @@ def make_validation_context( available_sub_recipes: frozenset[str] = frozenset(), project_dir: Path | None = None, disabled_subsets: frozenset[str] = frozenset(), + disabled_features: frozenset[str] = frozenset(), + provider_profiles: frozenset[str] = frozenset(), + skill_resolver: SkillResolver | None = None, ) -> ValidationContext: """Build a ``ValidationContext`` from a recipe. @@ -830,6 +152,9 @@ def make_validation_context( available_sub_recipes=available_sub_recipes, project_dir=project_dir, disabled_subsets=disabled_subsets, + disabled_features=disabled_features, + provider_profiles=provider_profiles, + skill_resolver=skill_resolver, blocks=extract_blocks(recipe, step_graph, predecessors=predecessors), predecessors=predecessors, ) diff --git a/src/autoskillit/recipe/_analysis_bfs.py b/src/autoskillit/recipe/_analysis_bfs.py new file mode 100644 index 0000000000..07490b4b1d --- /dev/null +++ b/src/autoskillit/recipe/_analysis_bfs.py @@ -0,0 +1,233 @@ +"""BFS reachability and symbolic fact-propagation over recipe step graphs.""" + +from __future__ import annotations + +from collections import deque + +import regex as re + +from autoskillit.recipe.contracts import RESULT_CAPTURE_RE +from autoskillit.recipe.schema import Recipe + +# --------------------------------------------------------------------------- +# BFS helpers +# --------------------------------------------------------------------------- + + +def bfs_reachable(graph: dict[str, set[str]], start: str) -> set[str]: + """Return all step names reachable from ``start`` in the routing graph (excluding start).""" + visited: set[str] = set() + queue = list(graph.get(start, set())) + while queue: + node = queue.pop() + if node in visited: + continue + visited.add(node) + queue.extend(graph.get(node, set())) + return visited + + +def _build_capture_origin_map(recipe: Recipe) -> dict[str, str]: + """Return {context_var_name: result_key} for all capture declarations. + + Only captures with a single extractable result.KEY reference are included. + Used to determine provenance of context variables — specifically whether + a variable was sourced from an ephemeral resource key like worktree_path + or branch_name. + """ + origin: dict[str, str] = {} + for step in recipe.steps.values(): + for cap_var, cap_expr in (step.capture or {}).items(): + if isinstance(cap_expr, str): + keys = RESULT_CAPTURE_RE.findall(cap_expr) + if len(keys) == 1: + origin[cap_var] = keys[0] + return origin + + +# Keys that become invalid after the given tool runs +_INVALIDATING_TOOLS: dict[str, frozenset[str]] = { + "merge_worktree": frozenset({"worktree_path", "branch_name"}), + "remove_clone": frozenset({"clone_path"}), +} + + +def _build_step_graph(recipe: Recipe) -> dict[str, set[str]]: + """Build an adjacency dict from recipe step routing edges. + + Builds the routing graph from on_result.condition route edges and + unconditional route edges (on_success, on_failure, on_context_limit). + """ + graph: dict[str, set[str]] = {name: set() for name in recipe.steps} + for step_name, step in recipe.steps.items(): + if step.on_result and step.on_result.conditions: + for cond in step.on_result.conditions: + if cond.route: + graph[step_name].add(cond.route) + if step.on_success: + graph[step_name].add(step.on_success) + if step.on_failure: + graph[step_name].add(step.on_failure) + if step.on_context_limit: + graph[step_name].add(step.on_context_limit) + return graph + + +def _build_success_step_graph(recipe: Recipe) -> dict[str, set[str]]: + """Build an adjacency dict from recipe step routing edges, success paths only. + + Includes on_result condition routes and on_success edges. + Excludes on_failure and on_context_limit — those are error-recovery paths, + not verdict-driven routing, and must not be checked by waypoint invariants. + """ + graph: dict[str, set[str]] = {name: set() for name in recipe.steps} + for step_name, step in recipe.steps.items(): + if step.on_result and step.on_result.conditions: + for cond in step.on_result.conditions: + if cond.route: + graph[step_name].add(cond.route) + if step.on_success: + graph[step_name].add(step.on_success) + return graph + + +def bfs_reachable_without_barrier( + recipe: Recipe, + start: str, + barrier: str, +) -> set[str]: + """BFS from ``start`` through success-path routing edges, stopping at ``barrier``. + + Returns all step names reachable from ``start`` without crossing ``barrier``. + The barrier step itself is not included in the returned set. + + Only follows on_result conditions and on_success edges — error paths + (on_failure, on_context_limit) are excluded because they are not + verdict-driven routing and must not be checked by waypoint invariants. + + This is the canonical implementation of the BFS-barrier pattern previously + duplicated inline in ``push-before-audit`` and ``merge-base-unpublished``. + """ + graph = _build_success_step_graph(recipe) + return _bfs_capped(graph, {start}, {barrier}) + + +def _bfs_capped( + graph: dict[str, set[str]], + start_nodes: set[str], + barrier_nodes: set[str], +) -> set[str]: + """BFS from start_nodes, visiting but not expanding barrier_nodes. + + Used to find steps reachable from a starting set where certain steps + (re-capture barriers) reset a variable's provenance — steps reachable + only through a barrier are excluded from the result's expansion. + + Returns all visited nodes (including barrier_nodes that were reached). + """ + visited: set[str] = set() + queue = list(start_nodes) + while queue: + node = queue.pop() + if node in visited: + continue + visited.add(node) + if node in barrier_nodes: + continue # Reached but do not expand — variable is refreshed here + queue.extend(graph.get(node, set())) + return visited + + +# --------------------------------------------------------------------------- +# Symbolic reachability: BFS with fact propagation +# --------------------------------------------------------------------------- + +# A FactSet is a frozen set of (variable, value) pairs established by +# conditional on_result.when edges. Each frozenset represents the facts +# that are known-to-be-true on one particular path through the routing graph. +_FactSet = frozenset[tuple[str, str]] + +# Matches simple equality conditions of the form: +# context.X == 'v' or ${{ context.X }} == "v" +# Capture groups: 1 = variable name, 2 = value. +# Non-equality expressions (inequalities, conjunctions) produce no match — +# conservative assumption (no fact is established). +_SIMPLE_WHEN_RE = re.compile( + r"(?:\$\{\{\s*)?context\.(\w+)(?:\s*\}\})?\s*==\s*" + r"(?:\"(\w+)\"|'(\w+)'|(\w+))" +) + + +def _parse_when_expr(expr: str) -> tuple[str, str] | None: + """Parse a simple equality when-expression into a (variable, value) fact. + + Returns ``None`` for conjunctions, inequalities, or non-context refs. + Conservative: only establishes facts for provably-simple equality conditions. + """ + m = _SIMPLE_WHEN_RE.fullmatch(expr.strip()) + if not m: + return None + # Groups 2/3/4 are mutually exclusive (double-quoted, single-quoted, unquoted) + value = m.group(2) or m.group(3) or m.group(4) + return (m.group(1), value) + + +def _edge_fact(recipe: Recipe, source: str, target: str) -> tuple[str, str] | None: + """Return the (variable, value) fact established by the edge source→target, or None. + + Only ``on_result.conditions`` edges with a parseable simple equality ``when`` + expression contribute a fact. ``on_success``/``on_failure`` edges contribute no fact. + """ + step = recipe.steps.get(source) + if step is None or step.on_result is None: + return None + for cond in step.on_result.conditions: + if cond.route == target and cond.when is not None: + return _parse_when_expr(cond.when) + return None + + +def _intersect_facts(fs: set[_FactSet]) -> _FactSet: + """Intersect all fact sets — only facts that hold on every incoming path survive.""" + if not fs: + return frozenset() + return frozenset.intersection(*fs) + + +def _bfs_with_facts( + graph: dict[str, set[str]], + recipe: Recipe, + start: str, +) -> dict[str, set[_FactSet]]: + """BFS from *start* propagating conditional edge facts. + + Each ``on_result`` edge whose ``when`` expression parses as + ``'context.X == "v"'`` extends the current fact set with ``(X, v)`` on + the target; other edges carry facts unchanged. At join points, the + returned fact set is the intersection of all incoming fact sets — a fact + is only "known" at a node if it holds on every path reaching that node. + + Returns ``{step_name: {intersected_fact_set}}``. Each value is a + single-element set containing one :class:`frozenset` of ``(var, val)`` + pairs. + """ + # facts maps step_name → set of fact-sets that have been discovered for it. + # visited tracks (node, fact_set) pairs to avoid reprocessing. + facts: dict[str, set[_FactSet]] = {start: {frozenset()}} + work: deque[str] = deque([start]) + visited: set[tuple[str, _FactSet]] = set() + + while work: + node = work.popleft() + for succ in graph.get(node, ()): + edge_fact = _edge_fact(recipe, node, succ) + for f in facts.get(node, {frozenset()}): + new_f: _FactSet = f | {edge_fact} if edge_fact else f + state = (succ, new_f) + if state in visited: + continue + visited.add(state) + facts.setdefault(succ, set()).add(new_f) + work.append(succ) + + return {n: {_intersect_facts(fs)} for n, fs in facts.items()} diff --git a/src/autoskillit/recipe/_analysis_blocks.py b/src/autoskillit/recipe/_analysis_blocks.py new file mode 100644 index 0000000000..156ae97703 --- /dev/null +++ b/src/autoskillit/recipe/_analysis_blocks.py @@ -0,0 +1,103 @@ +"""Block extraction: group steps by block annotation with entry/exit tracking.""" + +from __future__ import annotations + +from autoskillit.core import get_logger +from autoskillit.recipe.schema import Recipe, RecipeBlock, RecipeStep + +logger = get_logger(__name__) + + +# --------------------------------------------------------------------------- +# Block extraction helpers +# --------------------------------------------------------------------------- + + +def _count_by_tool(members: list[RecipeStep]) -> dict[str, int]: + """Count tool occurrences across a list of steps.""" + counts: dict[str, int] = {} + for step in members: + key = step.tool or "" + if key: + counts[key] = counts.get(key, 0) + 1 + return counts + + +def _count_gh_api(step: RecipeStep) -> int: + """Count occurrences of 'gh api' in a run_cmd step's cmd argument.""" + if step.tool != "run_cmd": + return 0 + cmd = (step.with_args or {}).get("cmd", "") or "" + return cmd.count("gh api") + + +def extract_blocks( + recipe: Recipe, + step_graph: dict[str, set[str]], + *, + predecessors: dict[str, set[str]] | None = None, +) -> tuple[RecipeBlock, ...]: + """Extract named block regions from a recipe's routing graph. + + Groups steps that share the same ``step.block`` value, then computes the + entry step (no in-block predecessor) and exit step (no in-block successor) + for each group. The step_graph is the forward adjacency dict produced by + ``_build_step_graph``; a local inverse is built here to find predecessors. + + Steps without a ``block`` annotation are ignored. Recipes with no block + annotations return an empty tuple. + """ + if predecessors is None: + # Build predecessor map by inverting the forward step_graph edges. + predecessors = {} + for src, successors in step_graph.items(): + for dst in successors: + predecessors.setdefault(dst, set()).add(src) + + by_name: dict[str, list[RecipeStep]] = {} + for step in recipe.steps.values(): + if step.block is not None: + by_name.setdefault(step.block, []).append(step) + + blocks: list[RecipeBlock] = [] + for name, members in by_name.items(): + member_names = {s.name for s in members} + # Entry: no in-block predecessor (predecessor set ∩ member_names is empty) + entry_candidates = [ + s for s in members if not (predecessors.get(s.name, set()) & member_names) + ] + # Exit: no in-block successor (successor set ∩ member_names is empty) + exit_candidates = [ + s for s in members if not (step_graph.get(s.name, set()) & member_names) + ] + entry_name: str + if entry_candidates: + entry_name = entry_candidates[0].name + else: + logger.warning( + "block %r has no graph-reachable entry step; falling back to first member", + name, + ) + entry_name = members[0].name + + exit_name: str + if exit_candidates: + exit_name = exit_candidates[0].name + else: + logger.warning( + "block %r has no graph-reachable exit step; falling back to last member", + name, + ) + exit_name = members[-1].name + + blocks.append( + RecipeBlock( + name=name, + entry=entry_name, + exit=exit_name, + members=tuple(members), + tool_counts=_count_by_tool(members), + gh_api_occurrences=sum(_count_gh_api(s) for s in members), + ) + ) + return tuple(blocks) diff --git a/src/autoskillit/recipe/_analysis_detectors.py b/src/autoskillit/recipe/_analysis_detectors.py new file mode 100644 index 0000000000..e592d924a4 --- /dev/null +++ b/src/autoskillit/recipe/_analysis_detectors.py @@ -0,0 +1,239 @@ +"""Dataflow warning detectors: dead outputs, ref invalidations, implicit handoffs.""" + +from __future__ import annotations + +from autoskillit.core import SKILL_TOOLS +from autoskillit.recipe._analysis_bfs import ( + _INVALIDATING_TOOLS, + _bfs_capped, + _build_capture_origin_map, + bfs_reachable, +) +from autoskillit.recipe.contracts import _CONTEXT_REF_RE +from autoskillit.recipe.schema import DataFlowWarning, Recipe, RecipeStep + +# --------------------------------------------------------------------------- +# Detection helpers +# --------------------------------------------------------------------------- + + +def _detect_ref_invalidations(recipe: Recipe, graph: dict[str, set[str]]) -> list[DataFlowWarning]: + """Detect context variables consumed after the step that invalidated the + underlying resource. + + The resource lifecycle contract: + - ``merge_worktree`` on SUCCESS destroys the worktree directory and branch ref. + - ``remove_clone`` on SUCCESS destroys the clone directory. + + Only steps reachable via ``on_success`` from the invalidating step are checked. + Steps that re-capture the same variable are treated as barriers — they refresh + the variable to a new resource, so their successors are excluded from + the stale-ref check. + """ + origin = _build_capture_origin_map(recipe) + + # Map: result_key → set of context variable names sourced from that key + key_to_vars: dict[str, set[str]] = {} + for var, result_key in origin.items(): + key_to_vars.setdefault(result_key, set()).add(var) + + # Map: var_name → set of step names that re-capture (refresh) it + var_recapture_steps: dict[str, set[str]] = {} + for step_name, step in recipe.steps.items(): + for cap_var in step.capture or {}: + var_recapture_steps.setdefault(cap_var, set()).add(step_name) + + warnings: list[DataFlowWarning] = [] + + for step_name, step in recipe.steps.items(): + invalidated_result_keys = _INVALIDATING_TOOLS.get(step.tool or "", frozenset()) + if not invalidated_result_keys: + continue + + # Variables whose underlying resource is destroyed when this step SUCCEEDS + invalidated_vars: set[str] = set() + for result_key in invalidated_result_keys: + invalidated_vars.update(key_to_vars.get(result_key, set())) + + if not invalidated_vars: + continue + + # Only check steps reachable via on_success (failure path = resource not destroyed) + on_success_target = step.on_success + if not on_success_target or on_success_target not in recipe.steps: + continue + + for var in invalidated_vars: + # Steps that re-capture this var are barriers: they refresh the variable + # to a new resource, so their successors are NOT stale consumers. + barrier = var_recapture_steps.get(var, set()) + stale_reachable = _bfs_capped(graph, {on_success_target}, barrier) + # A loop may route the invalidating step back into the reachable set + stale_reachable.discard(step_name) + + for downstream_name in stale_reachable: + downstream = recipe.steps.get(downstream_name) + if downstream is None: + continue + + for arg_val in (downstream.with_args or {}).values(): + if not isinstance(arg_val, str): + continue + for ref_var in _CONTEXT_REF_RE.findall(arg_val): + if ref_var == var: + warnings.append( + DataFlowWarning( + code="REF_INVALIDATED", + step_name=downstream_name, + field=var, + message=( + f"Step '{downstream_name}' references " + f"context.{var} after step '{step_name}' " + f"({step.tool}) has invalidated the underlying " + f"resource. Replace with a stable alternative " + f"(e.g., a commit SHA captured before any merge " + f"begins)." + ), + ) + ) + + return warnings + + +# Observability captures: variables captured for human-readable logs, hook +# consumption, or note-driven orchestration rather than downstream recipe +# threading. Each entry is (cap_key, skill_command_fragment). A capture is +# exempt when cap_key matches AND skill_command_fragment appears in the +# step's skill_command (or tool/step_name for non-skill steps). +_OBSERVABILITY_CAPTURES: frozenset[tuple[str, str]] = frozenset( + { + ("diagnosis_path", "diagnose-ci"), + ("summary_path", "pipeline-summary"), + ("report_path", "generate-report"), + ("selected_lenses", "prepare-research-pr"), + ("selected_lenses", "prepare-pr"), + ("lens_context_paths", "prepare-research-pr"), + ("lens_context_paths", "prepare-pr"), + ("pr_url", "compose-pr"), + ("html_path", "bundle-local-report"), + ("resource_report", "stage-data"), + ("alignment_findings_path", "planner-validate-task-alignment"), + ("review_approach_assessment_path", "planner-assess-review-approach"), + # plan-visualization terminal handoff captures: emitted in food-truck sentinel, + # not consumed by downstream recipe steps (route → stop action). + ("visualization_plan_path", "plan-visualization"), + ("report_plan_path", "plan-visualization"), + } +) + + +def _is_observability_capture(cap_key: str, step_name: str, step: RecipeStep) -> bool: + """Return True if *cap_key* is a known observability-only capture.""" + skill_cmd = step.with_args.get("skill_command", "") if step.with_args else "" + + # Skill-command-based exemptions (the common case). + for obs_key, fragment in _OBSERVABILITY_CAPTURES: + if cap_key == obs_key and fragment in skill_cmd: + return True + + # merge_worktree cleanup_succeeded: matched by tool name + capture value, + # not skill_command (merge_worktree is a direct tool, not a skill). + if step.tool == "merge_worktree" and "result.cleanup_succeeded" in str( + step.capture.get(cap_key, "") + ): + return True + + # export_local_bundle local_bundle_path: matched by step name (terminal + # step in the local-mode path that ends at a stop action). + if cap_key == "local_bundle_path" and step_name == "export_local_bundle": + return True + + # batch_create_issues issue_count: captured output used for telemetry/logging, + # not threaded to downstream recipe steps (the done message uses issue_urls only). + if cap_key == "issue_count" and step_name == "create_issues" and step.tool == "run_python": + return True + + return False + + +def _detect_dead_outputs(recipe: Recipe, graph: dict[str, set[str]]) -> list[DataFlowWarning]: + """Detect captured variables that are never consumed downstream.""" + warnings: list[DataFlowWarning] = [] + + for step_name, step in recipe.steps.items(): + if not step.capture: + continue + + # BFS: collect all steps reachable from this step + reachable = bfs_reachable(graph, step_name) + + # Collect all context.X references in reachable steps' with_args and + # on_result condition when-expressions (route actions gate on context vars). + consumed: set[str] = set() + for reachable_name in reachable: + reachable_step = recipe.steps[reachable_name] + for arg_val in reachable_step.with_args.values(): + if not isinstance(arg_val, str): + continue + consumed.update(_CONTEXT_REF_RE.findall(arg_val)) + # message fields are not recipe args; scanner must handle them separately. + if reachable_step.message and isinstance(reachable_step.message, str): + consumed.update(_CONTEXT_REF_RE.findall(reachable_step.message)) + if reachable_step.on_result and reachable_step.on_result.conditions: + for cond in reachable_step.on_result.conditions: + if cond.when and isinstance(cond.when, str): + consumed.update(_CONTEXT_REF_RE.findall(cond.when)) + + # on_result routing — both legacy field and predicate conditions count + # as structural consumption of captured variables. + if step.on_result: + # Legacy field routing: field name matches a captured key + if step.on_result.field in step.capture: + consumed.add(step.on_result.field) + # Predicate condition routing — conditions gate on step result; + # treat all captured vars as structurally consumed. + if step.on_result.conditions: + consumed.update(step.capture.keys()) + + # Flag captured vars not consumed on any path + for cap_key in step.capture: + if cap_key not in consumed: + if _is_observability_capture(cap_key, step_name, step): + continue + warnings.append( + DataFlowWarning( + code="DEAD_OUTPUT", + step_name=step_name, + field=cap_key, + message=( + f"Step '{step_name}' captures '{cap_key}' but no " + f"reachable downstream step references " + f"${{{{ context.{cap_key} }}}}." + ), + ) + ) + + return warnings + + +def _detect_implicit_handoffs(recipe: Recipe) -> list[DataFlowWarning]: + """Detect skill-invoking steps with no capture block.""" + warnings: list[DataFlowWarning] = [] + + for step_name, step in recipe.steps.items(): + if step.tool in SKILL_TOOLS and not step.capture: + warnings.append( + DataFlowWarning( + code="IMPLICIT_HANDOFF", + step_name=step_name, + field=step.tool, + message=( + f"Step '{step_name}' calls '{step.tool}' but has no " + f"capture: block. Data flows to subsequent steps " + f"implicitly through agent context rather than " + f"explicit ${{{{ context.X }}}} wiring." + ), + ) + ) + + return warnings diff --git a/src/autoskillit/recipe/_analysis_graph.py b/src/autoskillit/recipe/_analysis_graph.py new file mode 100644 index 0000000000..e924b0f8c0 --- /dev/null +++ b/src/autoskillit/recipe/_analysis_graph.py @@ -0,0 +1,254 @@ +"""Recipe graph builders: igraph visualization and step adjacency.""" + +from __future__ import annotations + +import dataclasses +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + import igraph + +from autoskillit.core import get_logger +from autoskillit.recipe.schema import _TERMINAL_TARGETS, Recipe, RecipeStep + +logger = get_logger(__name__) + + +# --------------------------------------------------------------------------- +# Infrastructure step classification (shared with diagrams rendering) +# --------------------------------------------------------------------------- + + +def _is_infrastructure_step(step: RecipeStep) -> bool: + """Return True if *step* is a plumbing step that should be hidden from diagrams. + + Infrastructure steps are ``run_cmd`` steps whose sole purpose is capturing + or setting a context value (git rev-parse, printf, echo one-liners). + They add no user-visible behaviour to the pipeline flow. + """ + if step.tool != "run_cmd": + return False + note_lower = (step.note or "").lower() + cmd = "" + if step.with_args and isinstance(step.with_args, dict): + cmd = step.with_args.get("cmd", "") or "" + return ( + "capture" in note_lower + or "set" in note_lower + or "printf" in cmd + or "git rev-parse" in cmd + or (cmd.strip().startswith("echo") and "\n" not in cmd) + ) + + +# --------------------------------------------------------------------------- +# igraph recipe graph builder +# --------------------------------------------------------------------------- + + +def build_recipe_graph(recipe: Recipe) -> igraph.Graph: + """Build a directed igraph.Graph from a Recipe dataclass. + + Nodes represent recipe steps. Each vertex carries attributes matching the + RecipeStep fields relevant to diagram rendering: + - ``name``: step name (str) + - ``tool``: tool identifier (str, empty string if None) + - ``action``: action identifier (str, empty string if None) + - ``note``: step note for semantic gate checks (str) + - ``retries``: retry count (int) + - ``skip_when_false``: optional condition (str, empty string if None) + - ``is_infra``: whether the step is a hidden infrastructure step (bool) + - ``is_terminal``: whether the step is a stop action (bool) + - ``is_confirm``: whether the step is a confirm action (bool) + + Edges represent routing connections. Each edge carries: + - ``edge_type``: one of ``"success"``, ``"failure"``, ``"context_limit"``, + ``"result_condition"``, ``"exhausted"`` + - ``condition``: for ``on_result`` edges, the ``when`` expression; otherwise ``""`` + + Args: + recipe: The loaded Recipe dataclass. + + Returns: + A directed ``igraph.Graph`` with vertex and edge attributes as described. + """ + import igraph # noqa: PLC0415 + + step_names = list(recipe.steps.keys()) + name_to_id: dict[str, int] = {name: i for i, name in enumerate(step_names)} + + g = igraph.Graph(n=len(step_names), directed=True) + steps_list = list(recipe.steps.values()) + + g.vs["name"] = step_names + g.vs["tool"] = [s.tool or "" for s in steps_list] + g.vs["action"] = [s.action or "" for s in steps_list] + g.vs["note"] = [s.note or "" for s in steps_list] + g.vs["retries"] = [s.retries for s in steps_list] + g.vs["skip_when_false"] = [s.skip_when_false or "" for s in steps_list] + g.vs["is_infra"] = [_is_infrastructure_step(s) for s in steps_list] + g.vs["is_terminal"] = [s.action == "stop" for s in steps_list] + g.vs["is_confirm"] = [s.action == "confirm" for s in steps_list] + + edges: list[tuple[int, int]] = [] + edge_types: list[str] = [] + edge_conditions: list[str] = [] + + for name, step in recipe.steps.items(): + src = name_to_id[name] + for edge in _extract_routing_edges(step): + # Mirror _build_step_graph: skip on_exhausted edges for action steps + # (stop/confirm/route steps have no retry semantics). + if edge.edge_type == "exhausted" and step.action is not None: + continue + if edge.target in name_to_id: + edges.append((src, name_to_id[edge.target])) + edge_types.append(edge.edge_type) + edge_conditions.append(edge.condition or "") + elif edge.target in _TERMINAL_TARGETS: + # Known sentinel — valid target, no graph edge needed. + pass + else: + logger.warning( + "build_recipe_graph: step %r references unknown target %r — edge skipped", + name, + edge.target, + ) + + if edges: + g.add_edges(edges, attributes={"edge_type": edge_types, "condition": edge_conditions}) + + return g + + +# --------------------------------------------------------------------------- +# Routing edge extraction +# --------------------------------------------------------------------------- + + +@dataclasses.dataclass +class RouteEdge: + """A single routing edge from a recipe step to a target step. + + Attributes: + edge_type: One of ``"success"``, ``"failure"``, ``"context_limit"``, + ``"result_condition"``, ``"exhausted"``. + target: The target step name. + condition: Populated for ``on_result`` conditions — the ``when`` expression. + """ + + edge_type: str + target: str + condition: str | None = None + + +def _extract_routing_edges(step: RecipeStep) -> list[RouteEdge]: + """Return all routing edges declared on *step*. + + Covers every routing field on :class:`RecipeStep`: + ``on_success``, ``on_failure``, ``on_context_limit``, ``on_exhausted``, + ``on_result.conditions[].route``, and ``on_result.routes`` (dict form). + + None targets are skipped. The caller is responsible for filtering by + known step names if graph-membership checks are needed. + """ + edges: list[RouteEdge] = [] + + if step.on_success: + edges.append(RouteEdge(edge_type="success", target=step.on_success)) + if step.on_failure: + edges.append(RouteEdge(edge_type="failure", target=step.on_failure)) + if step.on_context_limit: + edges.append(RouteEdge(edge_type="context_limit", target=step.on_context_limit)) + if step.on_exhausted: + edges.append(RouteEdge(edge_type="exhausted", target=step.on_exhausted)) + + if step.on_result: + sr = step.on_result + if sr.conditions: + for cond in sr.conditions: + edges.append( + RouteEdge( + edge_type="result_condition", + target=cond.route, + condition=cond.when, + ) + ) + elif sr.routes: + for key, target in sr.routes.items(): + edges.append( + RouteEdge( + edge_type="result_condition", + target=target, + condition=key, + ) + ) + + return edges + + +# --------------------------------------------------------------------------- +# Step routing graph +# --------------------------------------------------------------------------- + + +def _build_step_graph(recipe: Recipe) -> dict[str, set[str]]: + """Build a routing adjacency list from all step routing fields. + + Each key is a step name, each value is the set of step names + reachable in one hop (successors). Terminal targets like "done" + are excluded since they are not real steps. + """ + step_names = set(recipe.steps.keys()) + graph: dict[str, set[str]] = {name: set() for name in step_names} + + for name, step in recipe.steps.items(): + for edge in _extract_routing_edges(step): + # exhausted only applies to non-terminal steps + if edge.edge_type == "exhausted" and step.action is not None: + continue + if edge.target in step_names: + graph[name].add(edge.target) + + # Build predecessor map for bypass edge injection below. + predecessors: dict[str, set[str]] = {name: set() for name in step_names} + for name, successors in graph.items(): + for s in successors: + predecessors[s].add(name) + + # For each step with skip_when_false, add bypass edges from all predecessors + # directly to the step's routing targets (the steps to route to when skipped). + # This makes optional-step bypass paths visible to graph-based rules. + for name, step in recipe.steps.items(): + if not step.skip_when_false: + continue + # on_success bypass: predecessor → step.on_success + if step.on_success and step.on_success in step_names: + for pred in predecessors[name]: + graph[pred].add(step.on_success) + # on_result bypass: predecessor → each on_result condition route target + if step.on_result: + if step.on_result.routes: + for target in step.on_result.routes.values(): + if target in step_names: + for pred in predecessors[name]: + graph[pred].add(target) + for condition in step.on_result.conditions: + if condition.route in step_names: + for pred in predecessors[name]: + graph[pred].add(condition.route) + + # For each sub_recipe placeholder step (gate-controlled), add a bypass edge + # to the next step in YAML order. When the gate is false the step is dropped + # at load time; without this edge the next step becomes unreachable in the + # raw recipe graph, breaking reachability-based semantic rules. + step_names_list = list(recipe.steps.keys()) + for i, (name, step) in enumerate(recipe.steps.items()): + if step.sub_recipe is None or i + 1 >= len(step_names_list): + continue + next_step = step_names_list[i + 1] + graph[name].add(next_step) + for pred in predecessors[name]: + graph[pred].add(next_step) + + return graph diff --git a/src/autoskillit/recipe/_api.py b/src/autoskillit/recipe/_api.py index 82c5772032..153cb26bad 100644 --- a/src/autoskillit/recipe/_api.py +++ b/src/autoskillit/recipe/_api.py @@ -2,28 +2,36 @@ from __future__ import annotations -import dataclasses -import re +import hashlib import threading import time from collections.abc import Sequence from dataclasses import dataclass as _dc from pathlib import Path -from typing import Any, TypedDict +from typing import Any from autoskillit.core import ( LoadResult, RecipeSource, SkillLister, - TerminalColumn, YAMLError, - _render_gfm_table, get_logger, load_yaml, pkg_root, resolve_temp_dir, ) from autoskillit.recipe._analysis import make_validation_context +from autoskillit.recipe._recipe_composition import ( + _build_active_recipe, +) +from autoskillit.recipe._recipe_ingredients import ( + ListRecipesResult, # noqa: F401 + LoadRecipeResult, + OpenKitchenResult, # noqa: F401 + RecipeListItem, + build_ingredient_rows, # noqa: F401 + format_ingredients_table, +) from autoskillit.recipe.contracts import ( check_contract_staleness, load_recipe_card, @@ -37,151 +45,25 @@ ) from autoskillit.recipe.io import ( RecipeInfo, + _load_recipe_dict, _parse_recipe, builtin_recipes_dir, builtin_sub_recipes_dir, find_recipe_by_name, - find_sub_recipe_by_name, list_recipes, substitute_temp_placeholder, ) -from autoskillit.recipe.io import ( - load_recipe as _load_recipe_from_path, -) -from autoskillit.recipe.schema import StepResultCondition, StepResultRoute +from autoskillit.recipe.schema import Recipe from autoskillit.recipe.validator import ( build_quality_dict, compute_recipe_validity, filter_version_rule, findings_to_dicts, run_semantic_rules, - validate_recipe, -) - -_logger = get_logger(__name__) - -# --------------------------------------------------------------------------- -# GFM ingredient table column specs -# --------------------------------------------------------------------------- - -_GFM_DESC_MAX_WIDTH: int = 60 - -_GFM_INGREDIENT_COLUMNS: tuple[TerminalColumn, ...] = ( - TerminalColumn("Name", max_width=30, align=">"), - TerminalColumn("Description", max_width=_GFM_DESC_MAX_WIDTH, align="<"), - TerminalColumn("Default", max_width=20, align=">"), + validate_recipe_structure, ) - -# --------------------------------------------------------------------------- -# Schema contract: handler → formatter boundary -# --------------------------------------------------------------------------- - - -def _ingredient_sort_key(name: str, required: bool, default: object) -> tuple[int, str]: - """Sort ingredients: required > auto-detect > flags > optional > constants.""" - if required and default is None: - return (0, name) - if default == "": - return (1, name) - if default in ("true", "false"): - return (2, name) - if default is None: - return (3, name) - return (4, name) # has a non-empty default (constants, rarely changed) - - -def build_ingredient_rows( - recipe: Any, - resolved_defaults: dict[str, str] | None = None, -) -> list[tuple[str, str, str]]: - """Build (name, description, default) rows for a recipe's ingredients. - - This is the shared source of truth for ingredient row data, consumed by - both the GFM table formatter (LLM path) and the terminal renderer - (terminal path). Descriptions are full-length here — truncation is the - terminal renderer's responsibility. - - Returns rows sorted by ingredient priority (required first, then alphabetical). - """ - resolved = resolved_defaults or {} - raw: list[tuple[str, str, str, tuple[int, str]]] = [] - for name, ing in (getattr(recipe, "ingredients", None) or {}).items(): - if getattr(ing, "hidden", False): - continue - desc = getattr(ing, "description", "") - required = getattr(ing, "required", False) - default = getattr(ing, "default", None) - sort_key = _ingredient_sort_key(name, required, default) - if default is None and required: - default_str, name_str = "(required)", f"{name} *" - elif res := resolved.get(name): - default_str, name_str = res, name - elif default == "": - default_str, name_str = "auto-detect", name - elif default == "true": - default_str, name_str = "on", name - elif default == "false": - default_str, name_str = "off", name - elif default is None: - default_str, name_str = "--", name - else: - default_str, name_str = str(default), name - raw.append((name_str, desc, default_str, sort_key)) - raw.sort(key=lambda r: r[3]) - return [(r[0], r[1], r[2]) for r in raw] - - -def format_ingredients_table( - recipe: Any, resolved_defaults: dict[str, str] | None = None -) -> str | None: - """Build a pre-formatted ingredients table from a parsed Recipe. - - When ``resolved_defaults`` is provided, auto-detect ingredients (``default: ""``) - use the resolved value instead of showing "auto-detect". - """ - ingredients = getattr(recipe, "ingredients", None) - if not ingredients: - return None - - rows = build_ingredient_rows(recipe, resolved_defaults) - - if not rows: - return None - - return _render_gfm_table(_GFM_INGREDIENT_COLUMNS, rows) - - -class LoadRecipeResult(TypedDict, total=False): - """Typed schema for the load_recipe handler → formatter boundary.""" - - content: str - diagram: str | None - suggestions: list[dict[str, Any]] - valid: bool - kitchen_rules: list[str] - requires_packs: list[str] - error: str - greeting: str - ingredients_table: str - - -class RecipeListItem(TypedDict): - """Typed schema for a single recipe entry in the list_recipes response.""" - - name: str - description: str - summary: str - source: str - - -class ListRecipesResult(TypedDict, total=False): - """Typed schema for the list_recipes handler → formatter boundary.""" - - recipes: list[RecipeListItem] - count: int - errors: list[dict[str, str]] - +logger = get_logger(__name__) # --------------------------------------------------------------------------- # Stage timing helper @@ -195,7 +77,7 @@ def _t(label: str, t0: float, name: str) -> float: filtering without requiring an explicit isEnabledFor() guard. """ elapsed_ms = (time.perf_counter() - t0) * 1000 - _logger.debug("load_recipe_stage", recipe=name, stage=label, elapsed_ms=round(elapsed_ms, 1)) + logger.debug("load_recipe_stage", recipe=name, stage=label, elapsed_ms=round(elapsed_ms, 1)) return time.perf_counter() @@ -224,6 +106,19 @@ def _get_pkg_version() -> str: return __version__ +def _compute_registry_hash(experiment_types_dir: Path) -> str: + """Compute md5 hash of sorted (path, mtime_ns) pairs for experiment-type YAMLs.""" + if not experiment_types_dir.exists(): + return "" + entries: list[tuple[str, int]] = [] + for p in sorted(experiment_types_dir.glob("*.yaml")): + try: + entries.append((p.name, p.stat().st_mtime_ns)) + except OSError: + continue + return hashlib.md5(str(entries).encode(), usedforsecurity=False).hexdigest() + + @_dc class _LoadCacheEntry: recipe_path: Path @@ -259,193 +154,28 @@ def format_recipe_list_response(result: LoadResult[RecipeInfo]) -> dict[str, obj return response -def list_all(project_dir: Path | None = None) -> dict[str, Any]: +def list_all( + project_dir: Path | None = None, + *, + features: dict[str, bool] | None = None, +) -> dict[str, Any]: """List all recipes from project and built-in sources. Returns: {"recipes": list[{"name", "description", "summary"}]} Includes "errors" key when recipes fail to parse. """ + from autoskillit.core import is_feature_enabled # noqa: PLC0415 + from autoskillit.recipe.schema import NON_INTERACTIVE_KINDS # noqa: PLC0415 + _pdir = project_dir if project_dir is not None else Path.cwd() - result = list_recipes(_pdir) + _features = features or {} + fleet_enabled = is_feature_enabled("fleet", _features) + exclude_kinds = frozenset() if fleet_enabled else NON_INTERACTIVE_KINDS + result = list_recipes(_pdir, exclude_kinds=exclude_kinds) return format_recipe_list_response(result) -def _drop_sub_recipe_step(recipe: Any, step_name: str) -> Any: - """Return a new Recipe with the named sub_recipe placeholder step removed.""" - new_steps = {k: v for k, v in recipe.steps.items() if k != step_name} - return dataclasses.replace(recipe, steps=new_steps) - - -def _merge_sub_recipe(parent: Any, placeholder_name: str, sub: Any) -> Any: - """Replace the sub_recipe placeholder step with the sub-recipe's steps. - - Algorithm: - 1. Compute a safe name prefix from the sub-recipe name. - 2. For each step in sub, create a prefixed copy with routing fixed: - - Routes to "done" → parent placeholder's on_success - - Routes to "escalate" → parent placeholder's on_failure - - Routes to other sub-recipe step names → add prefix - 3. Insert sub-recipe steps in place of the placeholder. - 4. Merge ingredients: add sub-recipe's non-hidden ingredients into parent. - 5. Merge kitchen_rules: union (deduplicated), sub-recipe rules appended. - """ - if placeholder_name not in parent.steps: - raise KeyError( - f"_merge_sub_recipe: placeholder step '{placeholder_name}' not found in " - f"recipe '{parent.name}'. Available steps: {list(parent.steps.keys())}" - ) - placeholder = parent.steps[placeholder_name] - on_success = placeholder.on_success or "done" - on_failure = placeholder.on_failure or "escalate" - - # Build prefix: "sprint-prefix" → "sprint_prefix_", "my-sub" → "my_sub_" - raw_prefix = re.sub(r"[^a-z0-9]", "_", (sub.name or placeholder_name).lower()) - if not raw_prefix.endswith("_"): - raw_prefix += "_" - prefix = raw_prefix - - sub_step_names = set(sub.steps.keys()) - - def _fix_route(target: str | None) -> str | None: - if target is None: - return None - if target == "done": - return on_success - if target == "escalate": - return on_failure - if target in sub_step_names: - return prefix + target - return target - - def _fix_result_route(route: Any) -> Any: - if route is None: - return None - if route.conditions: - return StepResultRoute( - conditions=[ - StepResultCondition(when=c.when, route=_fix_route(c.route) or "") - for c in route.conditions - ] - ) - return StepResultRoute( - field=route.field, - routes={k: (_fix_route(v) or v) for k, v in route.routes.items()}, - ) - - prefixed_steps: dict[str, Any] = {} - for sub_step_name, sub_step in sub.steps.items(): - new_name = prefix + sub_step_name - new_step = dataclasses.replace( - sub_step, - on_success=_fix_route(sub_step.on_success), - on_failure=_fix_route(sub_step.on_failure), - on_context_limit=_fix_route(sub_step.on_context_limit), - on_exhausted=_fix_route(sub_step.on_exhausted), - on_result=_fix_result_route(sub_step.on_result), - ) - prefixed_steps[new_name] = new_step - - # Assemble new steps dict: sub-recipe steps injected in place of placeholder - new_steps: dict[str, Any] = {} - for step_name, step in parent.steps.items(): - if step_name == placeholder_name: - new_steps.update(prefixed_steps) - else: - new_steps[step_name] = step - - # Merge ingredients: sub-recipe non-hidden ingredients into parent - merged_ingredients = dict(parent.ingredients) - for ing_name, ing in sub.ingredients.items(): - if getattr(ing, "hidden", False): - continue # do not propagate hidden sub-recipe ingredients to parent - if ing_name not in merged_ingredients: - merged_ingredients[ing_name] = ing - - # Merge kitchen_rules: union (parent first, then sub-recipe additions) - seen_rules: set[str] = set(parent.kitchen_rules) - merged_rules = list(parent.kitchen_rules) - for rule in sub.kitchen_rules: - if rule not in seen_rules: - merged_rules.append(rule) - seen_rules.add(rule) - - # Merge requires_packs: union (parent first, then sub-recipe additions) - seen_packs: set[str] = set(parent.requires_packs) - merged_packs = list(parent.requires_packs) - for pack in sub.requires_packs: - if pack not in seen_packs: - merged_packs.append(pack) - seen_packs.add(pack) - - return dataclasses.replace( - parent, - steps=new_steps, - ingredients=merged_ingredients, - kitchen_rules=merged_rules, - requires_packs=merged_packs, - ) - - -def _build_active_recipe( - recipe: Any, - ingredient_overrides: dict[str, str] | None, - project_dir: Path, - temp_dir_relpath: str = ".autoskillit/temp", -) -> tuple[Any, Any | None]: - """Return (active_recipe, combined_recipe | None). - - active_recipe: the Recipe to serve to the agent. - - If no sub_recipe steps: returns recipe unchanged. - - If sub_recipe step with gate=false: returns recipe with sub_recipe step dropped. - - If sub_recipe step with gate=true: returns the merged (combined) recipe. - - combined_recipe: the merged Recipe if any gate was true, else None. - Used to run dual validation (REQ-VALID-004). - """ - overrides = ingredient_overrides or {} - sub_recipe_steps = [ - (name, step) for name, step in recipe.steps.items() if step.sub_recipe is not None - ] - if not sub_recipe_steps: - return recipe, None - - combined: Any | None = None - working = recipe - - # Re-read each step from working.steps to get the current state after prior - # merge/drop operations, rather than using the stale reference from recipe.steps. - for step_name, _orig_step in sub_recipe_steps: - current_step = working.steps.get(step_name) - if current_step is None or current_step.sub_recipe is None: - continue - gate_name = current_step.gate or "" - gate_ingredient = working.ingredients.get(gate_name) if gate_name else None - gate_default: str = (gate_ingredient.default or "false") if gate_ingredient else "false" - gate_value = overrides.get(gate_name, gate_default) - - if gate_value.lower() in ("true", "1", "yes"): - sr_path = find_sub_recipe_by_name(current_step.sub_recipe, project_dir) - if sr_path is None: - raise FileNotFoundError( - f"Sub-recipe '{current_step.sub_recipe}' not found. " - f"Expected in recipes/sub-recipes/{current_step.sub_recipe}.yaml" - ) - try: - sub_recipe = _load_recipe_from_path(sr_path, temp_dir_relpath) - except (YAMLError, ValueError, OSError) as exc: - raise ValueError( - f"Failed to load sub-recipe '{current_step.sub_recipe}' " - f"(gate: {gate_name}={gate_value}): {exc}" - ) from exc - working = _merge_sub_recipe(working, step_name, sub_recipe) - combined = working - else: - working = _drop_sub_recipe_step(working, step_name) - - return working, combined - - def validate_from_path( path: Path, temp_dir_relpath: str = ".autoskillit/temp", @@ -491,10 +221,16 @@ def validate_from_path( lister = DefaultSkillResolver() + from autoskillit.core import SkillResolver as _SkillResolver # noqa: PLC0415 + + _skill_resolver = lister if isinstance(lister, _SkillResolver) else None + recipe = _parse_recipe(data) - errors = validate_recipe(recipe) + errors = validate_recipe_structure(recipe) known_skills = frozenset(s.name for s in lister.list_all()) - ctx = make_validation_context(recipe, available_skills=known_skills) + ctx = make_validation_context( + recipe, available_skills=known_skills, skill_resolver=_skill_resolver + ) report = ctx.dataflow semantic_findings = run_semantic_rules(ctx) @@ -519,12 +255,60 @@ def validate_from_path( } +def _build_stop_step_semantics(recipe: Recipe) -> str: + stop_steps = {name: step for name, step in recipe.steps.items() if step.action == "stop"} + if not stop_steps: + return "" + names = ", ".join(f"'{n}'" for n in stop_steps) + lines = [ + "ACTION: STOP STEP SEMANTICS:", + f"- Steps {names} are terminal stop steps.", + "- When routed to a stop step, display its message and TERMINATE immediately.", + "- Do NOT call any MCP tools after a stop step.", + "- Do NOT attempt recovery, error reporting, or off-recipe actions.", + "- A stop step is an INTENTIONAL terminus — the recipe author designed this as" + " the endpoint.", + ] + for name, step in stop_steps.items(): + if step.message: + lines.append(f" Stop step '{name}' message: {step.message!r}") + return "\n".join(lines) + + +def _build_orchestration_rules( + recipe: Recipe | None = None, stop_semantics: str | None = None +) -> str: + parts = [ + "STEP EXECUTION IS NOT DISCRETIONARY:\n" + "You MUST execute every step the pipeline routes you to. " + "The ONLY mechanism for skipping a step is skip_when_false evaluating to false. " + "When skip_when_false evaluates to true (or is absent), the step is MANDATORY. " + "NEVER skip a step because the PR is small, the diff is trivial, or you judge " + "the step unnecessary. NEVER replace recipe steps with manual tool calls. " + "Consequence: skipping PR review steps results in unreviewed code, missing " + "diff annotations, and no architectural lens analysis." + ] + if recipe is not None: + sem = stop_semantics if stop_semantics is not None else _build_stop_step_semantics(recipe) + if sem: + parts.append(sem) + parts.append( + "ACTION: ROUTE STEP SEMANTICS:\n" + '- When you reach a step with action: "route", evaluate the step\'s on_result\n' + " conditions against captured context variables. Route to the matching target.\n" + "- Do NOT call any MCP tools for this step type — routing evaluation IS the step.\n" + "- If no on_result condition matches and on_failure is defined, follow on_failure." + ) + return "\n\n".join(parts) + + def load_and_validate( name: str, project_dir: Path | None = None, *, suppressed: Sequence[str] | None = None, recipe_info: RecipeInfo | None = None, + recipe_list: list[RecipeInfo] | None = None, resolved_defaults: dict[str, str] | None = None, ingredient_overrides: dict[str, str] | None = None, temp_dir: Path | None = None, @@ -550,11 +334,28 @@ def load_and_validate( pkg_version = _get_pkg_version() project_recipes_dir = _pdir / ".autoskillit" / "recipes" _builtin_dir = builtin_recipes_dir() + from autoskillit.recipe.experiment_type_registry import ( # noqa: PLC0415 + BUNDLED_EXPERIMENT_TYPES_DIR, + ) + from autoskillit.recipe.methodology_tradition_registry import ( # noqa: PLC0415 + BUNDLED_METHODOLOGY_TRADITIONS_DIR, + ) + + _exp_types_hash = _compute_registry_hash(BUNDLED_EXPERIMENT_TYPES_DIR) + _user_exp_types_dir = _pdir / ".autoskillit" / "experiment-types" + _user_exp_hash = _compute_registry_hash(_user_exp_types_dir) + _method_traditions_hash = _compute_registry_hash(BUNDLED_METHODOLOGY_TRADITIONS_DIR) + _user_method_traditions_dir = _pdir / ".autoskillit" / "methodology-traditions" + _user_method_traditions_hash = _compute_registry_hash(_user_method_traditions_dir) cache_key = ( name, str(_pdir), tuple(sorted(suppressed)) if suppressed else (), tuple(sorted(ingredient_overrides.items())) if ingredient_overrides else (), + _exp_types_hash, + _user_exp_hash, + _method_traditions_hash, + _user_method_traditions_hash, ) with _LOAD_CACHE_LOCK: @@ -571,7 +372,7 @@ def load_and_validate( and rm == cached.recipe_mtime and rs == cached.recipe_size ): - _logger.debug("load_recipe_cache_hit", recipe=name) + logger.debug("load_recipe_cache_hit", recipe=name) return cached.result t0 = time.perf_counter() @@ -579,8 +380,10 @@ def load_and_validate( # Stage: find recipe if recipe_info is not None: match: RecipeInfo | None = recipe_info + _recipe_list = recipe_list else: match = find_recipe_by_name(name, _pdir) + _recipe_list = None t0 = _t("find_recipe", t0, name) if match is None: @@ -588,7 +391,6 @@ def load_and_validate( raw = match.content if match.content is not None else match.path.read_text() _temp_relpath = temp_dir_relpath or ".autoskillit/temp" - raw = substitute_temp_placeholder(raw, _temp_relpath) suggestions: list[dict[str, Any]] = [] valid = True recipe = None @@ -602,24 +404,40 @@ def load_and_validate( try: # Stage: yaml parse - data = load_yaml(raw) + data = _load_recipe_dict(match.path, raw_text=raw, temp_dir_relpath=_temp_relpath) t0 = _t("yaml_parse", t0, name) if isinstance(data, dict) and "steps" in data: recipe = _parse_recipe(data) + from autoskillit.recipe.identity import compute_composite_hash # noqa: PLC0415 + + _recipe_bytes = match.path.read_bytes() + recipe.content_hash = ( + match.content_hash + if match.content_hash + else "sha256:" + hashlib.sha256(_recipe_bytes).hexdigest() + ) + recipe.composite_hash = compute_composite_hash( + match.path, + recipe, + skills_dir=pkg_root() / "skills", + project_dir=_pdir, + content_bytes=_recipe_bytes, + ) + # Stage: sub-recipe composition (lazy-loaded prefixes) active_recipe, combined_recipe = _build_active_recipe( recipe, ingredient_overrides, _pdir, _temp_relpath ) # Stage: structural validation on active recipe - errors = validate_recipe(active_recipe) + errors = validate_recipe_structure(active_recipe) if combined_recipe is not None: # Dual validation: also validate the combined (merged) graph - combined_errors = validate_recipe(combined_recipe) + combined_errors = validate_recipe_structure(combined_recipe) errors.extend(f"[combined] {e}" for e in combined_errors) - t0 = _t("validate_recipe", t0, name) + t0 = _t("validate_recipe_structure", t0, name) # Stage: semantic rules (builds ValidationContext once — shared computation) if lister is None: @@ -627,7 +445,14 @@ def load_and_validate( lister = DefaultSkillResolver() - known = frozenset(r.name for r in list_recipes(_pdir).items) + from autoskillit.core import SkillResolver as _SkillResolver # noqa: PLC0415 + + _skill_resolver = lister if isinstance(lister, _SkillResolver) else None + + known = frozenset( + r.name + for r in (_recipe_list if _recipe_list is not None else list_recipes(_pdir).items) + ) known_skills = frozenset(s.name for s in lister.list_all()) sub_recipes_dir = builtin_sub_recipes_dir() known_sub_recipes: frozenset[str] = ( @@ -644,6 +469,7 @@ def load_and_validate( available_skills=known_skills, available_sub_recipes=known_sub_recipes, project_dir=_pdir, + skill_resolver=_skill_resolver, ) semantic_findings = run_semantic_rules(val_ctx) semantic_suggestions = findings_to_dicts(semantic_findings) @@ -682,7 +508,7 @@ def load_and_validate( t0 = _t("yaml_parse", t0, name) except YAMLError as exc: - _logger.warning("Recipe YAML parse error", name=name, exc_info=True) + logger.warning("Recipe YAML parse error", name=name, exc_info=True) suggestions.append( { "rule": "validation-error", @@ -693,7 +519,7 @@ def load_and_validate( ) valid = False except ValueError as exc: - _logger.warning("Recipe structure invalid", name=name, exc_info=True) + logger.warning("Recipe structure invalid", name=name, exc_info=True) suggestions.append( { "rule": "validation-error", @@ -704,7 +530,7 @@ def load_and_validate( ) valid = False except (FileNotFoundError, OSError) as exc: - _logger.warning("Recipe file not found or unreadable", name=name, exc_info=True) + logger.warning("Recipe file not found or unreadable", name=name, exc_info=True) suggestions.append( { "rule": "validation-error", @@ -736,8 +562,22 @@ def load_and_validate( result["kitchen_rules"] = _serving_recipe.kitchen_rules if _serving_recipe is not None and _serving_recipe.requires_packs: result["requires_packs"] = _serving_recipe.requires_packs + if _serving_recipe is not None and _serving_recipe.requires_features: + result["requires_features"] = _serving_recipe.requires_features if ing_table: result["ingredients_table"] = ing_table + # Compute once; reused by both fields to avoid a second traversal of recipe.steps. + # Two delivery paths are intentional: orchestration_rules embeds the text for Channel A + # (open_kitchen response / system prompt); stop_step_semantics is a dedicated field for + # Channel B consumers (load_recipe docstring injection) that need the text in isolation. + _stop_semantics = _build_stop_step_semantics(recipe) if recipe else "" + result["orchestration_rules"] = _build_orchestration_rules( + recipe, stop_semantics=_stop_semantics + ) + result["stop_step_semantics"] = _stop_semantics + result["content_hash"] = recipe.content_hash if recipe else "" + result["composite_hash"] = recipe.composite_hash if recipe else "" + result["recipe_version"] = recipe.recipe_version if recipe else None # Write to cache (only when recipe was found and fully processed) if match is not None: diff --git a/src/autoskillit/recipe/_cmd_rpc.py b/src/autoskillit/recipe/_cmd_rpc.py new file mode 100644 index 0000000000..e79b43cbdc --- /dev/null +++ b/src/autoskillit/recipe/_cmd_rpc.py @@ -0,0 +1,725 @@ +"""Recipe cmd externalization callables — run_python entry points (IL-006).""" + +from __future__ import annotations + +import json +import secrets +import subprocess +import time +from datetime import UTC, date, datetime +from pathlib import Path + +import regex as re + +from autoskillit.core import atomic_write + + +def compute_branch( + issue_slug: str = "", + run_name: str = "", + issue_number: str = "", +) -> dict[str, str]: + """Compute branch name from slug + issue or date.""" + prefix = issue_slug or run_name + if issue_number: + return {"branch_name": f"{prefix}/{issue_number}"} + return {"branch_name": f"{prefix}/{date.today().strftime('%Y%m%d')}"} + + +def check_eject_limit( + counter_file: str, + max_ejects: str = "3", +) -> dict[str, str]: + """Increment counter file; return EJECT_OK or EJECT_LIMIT_EXCEEDED.""" + max_ejects = max_ejects or "3" + path = Path(counter_file) + path.parent.mkdir(parents=True, exist_ok=True) + try: + count = int(path.read_text().strip()) + except (FileNotFoundError, ValueError): + count = 0 + count += 1 + atomic_write(path, str(count)) + status = "EJECT_LIMIT_EXCEEDED" if count > int(max_ejects) else "EJECT_OK" + return {"status": status, "count": str(count)} + + +def check_dropped_healthy_loop( + counter_file: str, + max_drops: str = "2", +) -> dict[str, str]: + """Increment dropped-healthy counter; return DROPPED_OK or DROPPED_LIMIT_EXCEEDED.""" + max_drops = max_drops or "2" + path = Path(counter_file) + path.parent.mkdir(parents=True, exist_ok=True) + try: + count = int(path.read_text().strip()) + except (FileNotFoundError, ValueError): + count = 0 + count += 1 + atomic_write(path, str(count)) + status = "DROPPED_LIMIT_EXCEEDED" if count > int(max_drops) else "DROPPED_OK" + return {"status": status, "count": str(count)} + + +def commit_guard(worktree_path: str) -> dict[str, str]: + """Auto-commit pending changes if worktree is dirty.""" + result = subprocess.run( + ["git", "status", "--porcelain"], + cwd=worktree_path, + capture_output=True, + text=True, + ) + if result.stdout.strip(): + subprocess.run(["git", "add", "-A"], cwd=worktree_path, check=True) + subprocess.run( + ["git", "commit", "-m", "chore: commit pending session changes"], + cwd=worktree_path, + check=True, + ) + return {"committed": "true"} + return {"committed": "false"} + + +def _detect_remote(cwd: str) -> str: + """Detect preferred remote: upstream (non-file) or origin.""" + result = subprocess.run( + ["git", "remote", "get-url", "upstream"], + cwd=cwd, + capture_output=True, + text=True, + ) + if result.returncode == 0 and not result.stdout.strip().startswith("file://"): + return "upstream" + return "origin" + + +def queue_ejected_fix( + work_dir: str, + base_branch: str, +) -> dict[str, str]: + """Fetch and rebase onto base branch; return clean or conflicts.""" + remote = _detect_remote(work_dir) + fetch = subprocess.run( + ["git", "fetch", remote, base_branch], + cwd=work_dir, + capture_output=True, + text=True, + ) + if fetch.returncode != 0: + return {"status": "conflicts"} + rebase = subprocess.run( + ["git", "rebase", f"{remote}/{base_branch}"], + cwd=work_dir, + capture_output=True, + text=True, + ) + if rebase.returncode == 0: + return {"status": "clean"} + subprocess.run( + ["git", "rebase", "--abort"], + cwd=work_dir, + capture_output=True, + ) + return {"status": "conflicts"} + + +def direct_merge_conflict_fix( + work_dir: str, + base_branch: str, +) -> dict[str, str]: + """Attempt rebase for direct-merge path; return clean or conflicts.""" + return queue_ejected_fix(work_dir=work_dir, base_branch=base_branch) + + +def immediate_merge_conflict_fix( + work_dir: str, + base_branch: str, +) -> dict[str, str]: + """Attempt rebase for immediate-merge path; return clean or conflicts.""" + return queue_ejected_fix(work_dir=work_dir, base_branch=base_branch) + + +def wait_for_direct_merge( + pr_number: str, + max_polls: str = "90", + poll_interval: str = "10", +) -> dict[str, str]: + """Poll PR state until merged/closed/timeout.""" + + max_polls = max_polls or "90" + poll_interval = poll_interval or "10" + for _ in range(int(max_polls)): + result = subprocess.run( + ["gh", "pr", "view", str(pr_number), "--json", "state", "--jq", ".state"], + capture_output=True, + text=True, + ) + if result.returncode != 0: + time.sleep(int(poll_interval)) + continue + state = result.stdout.strip() + if state == "MERGED": + return {"state": "merged"} + if state == "CLOSED": + return {"state": "closed"} + time.sleep(int(poll_interval)) + return {"state": "timeout"} + + +def wait_for_immediate_merge( + pr_number: str, + max_polls: str = "30", + poll_interval: str = "10", +) -> dict[str, str]: + """Poll PR state until merged/closed/timeout (shorter).""" + return wait_for_direct_merge( + pr_number=pr_number, max_polls=max_polls, poll_interval=poll_interval + ) + + +def attempt_cheap_rebase( + work_dir: str, + ejected_pr_branch: str, + base_branch: str, +) -> dict[str, str]: + """Checkout ejected branch and attempt rebase.""" + remote = _detect_remote(work_dir) + subprocess.run( + ["git", "fetch", remote, ejected_pr_branch], + cwd=work_dir, + capture_output=True, + check=True, + ) + subprocess.run( + ["git", "checkout", ejected_pr_branch], + cwd=work_dir, + capture_output=True, + check=True, + ) + rebase = subprocess.run( + ["git", "rebase", f"{remote}/{base_branch}"], + cwd=work_dir, + capture_output=True, + text=True, + ) + if rebase.returncode == 0: + return {"status": "clean"} + subprocess.run( + ["git", "rebase", "--abort"], + cwd=work_dir, + capture_output=True, + ) + return {"status": "conflicts"} + + +def wait_for_review_pr_mergeability( + pr_url: str, + max_polls: str = "12", + poll_interval: str = "15", +) -> dict[str, str]: + """Extract PR number and poll until mergeability resolves.""" + + max_polls = max_polls or "12" + poll_interval = poll_interval or "15" + result = subprocess.run( + ["gh", "pr", "view", pr_url, "--json", "number", "-q", ".number"], + capture_output=True, + text=True, + ) + if result.returncode != 0: + msg = f"failed to resolve PR number: {result.stderr}" + raise RuntimeError(msg) + pr_number = result.stdout.strip() + for _ in range(int(max_polls)): + r = subprocess.run( + ["gh", "pr", "view", pr_number, "--json", "mergeable", "-q", ".mergeable"], + capture_output=True, + text=True, + ) + if r.returncode != 0: + time.sleep(int(poll_interval)) + continue + status = r.stdout.strip() + if status != "UNKNOWN": + return {"pr_number": pr_number} + time.sleep(int(poll_interval)) + msg = "Timed out waiting for mergeability" + raise RuntimeError(msg) + + +def create_persistent_integration( + work_dir: str, + base_branch: str, +) -> dict[str, str]: + """Create and push persistent integration branch from default branch.""" + remote = _detect_remote(work_dir) + result = subprocess.run( + ["git", "symbolic-ref", "refs/remotes/origin/HEAD"], + cwd=work_dir, + capture_output=True, + text=True, + ) + default_branch = "main" + if result.returncode == 0: + ref = result.stdout.strip() + default_branch = ref.replace("refs/remotes/origin/", "") + subprocess.run( + ["git", "checkout", default_branch], cwd=work_dir, check=True, capture_output=True + ) + subprocess.run(["git", "pull"], cwd=work_dir, check=True, capture_output=True) + subprocess.run( + ["git", "checkout", "-b", base_branch], cwd=work_dir, check=True, capture_output=True + ) + push = subprocess.run( + ["git", "push", remote, base_branch], + cwd=work_dir, + capture_output=True, + text=True, + ) + if push.returncode != 0: + msg = f"push failed: {push.stderr}" + raise RuntimeError(msg) + return {"ok": "true"} + + +def force_push_and_wait_mergeability( + work_dir: str, + batch_branch: str, + review_pr_number: str, + max_polls: str = "12", + poll_interval: str = "15", +) -> dict[str, str]: + """Force-push integration branch and wait for mergeability.""" + + max_polls = max_polls or "12" + poll_interval = poll_interval or "15" + remote = _detect_remote(work_dir) + push = subprocess.run( + ["git", "push", remote, batch_branch, "--force-with-lease"], + cwd=work_dir, + capture_output=True, + text=True, + ) + if push.returncode != 0: + msg = f"force-push failed: {push.stderr}" + raise RuntimeError(msg) + for _ in range(int(max_polls)): + r = subprocess.run( + ["gh", "pr", "view", str(review_pr_number), "--json", "mergeable", "-q", ".mergeable"], + capture_output=True, + text=True, + ) + if r.returncode != 0: + time.sleep(int(poll_interval)) + continue + status = r.stdout.strip() + if status != "UNKNOWN": + return {"ok": "true"} + time.sleep(int(poll_interval)) + msg = "Timed out waiting for post-rebase mergeability" + raise RuntimeError(msg) + + +def advance_queue_pr( + current_pr_number: str, + pr_order_file: str, +) -> dict[str, str]: + """Find next PR in queue order file.""" + if not current_pr_number: + return {"error": f"current_pr_number is required, got {current_pr_number!r}"} + try: + with open(pr_order_file) as f: + order = json.load(f) + except (OSError, json.JSONDecodeError) as exc: + raise RuntimeError(str(exc)) from exc + current = int(current_pr_number) + idx = None + for i, entry in enumerate(order): + if entry.get("number") == current: + idx = i + break + if idx is None: + return {"current_pr_number": "done"} + if (idx + 1) < len(order): + return {"current_pr_number": str(order[idx + 1]["number"])} + return {"current_pr_number": "done"} + + +def proactive_rebase_next_pr( + work_dir: str, + next_pr_branch: str, + base_branch: str, +) -> dict[str, str]: + """Fetch, checkout, and rebase next PR branch.""" + remote = _detect_remote(work_dir) + subprocess.run( + ["git", "fetch", remote, next_pr_branch], + cwd=work_dir, + capture_output=True, + check=True, + ) + subprocess.run( + ["git", "checkout", "-B", next_pr_branch, f"{remote}/{next_pr_branch}"], + cwd=work_dir, + capture_output=True, + check=True, + ) + rebase = subprocess.run( + ["git", "rebase", f"{remote}/{base_branch}"], + cwd=work_dir, + capture_output=True, + text=True, + ) + if rebase.returncode == 0: + return {"status": "clean"} + subprocess.run( + ["git", "rebase", "--abort"], + cwd=work_dir, + capture_output=True, + ) + return {"status": "conflicts"} + + +def refetch_issues(issue_urls: str) -> dict[str, str]: + """Build GraphQL query from issue URLs, fetch open issues.""" + urls = issue_urls.split(",") + parts = [] + for i, url in enumerate(urls): + url = url.strip() + if not url: + continue + m = re.match(r"https://github\.com/([^/]+)/([^/]+)/issues/(\d+)", url) + if m: + owner, repo, num = m.groups() + parts.append( + f'i{i}: repository(owner: "{owner}", name: "{repo}") ' + f"{{ issue(number: {num}) {{ number state }} }}" + ) + if not parts: + return {"issue_numbers": ""} + query = "{" + " ".join(parts) + "}" + result = subprocess.run( + [ + "gh", + "api", + "graphql", + "-f", + f"query={query}", + "--jq", + ( + '[.data[] | select(.issue != null and .issue.state == "OPEN")' + ' | .issue.number | tostring] | join(" ")' + ), + ], + capture_output=True, + text=True, + ) + if result.returncode != 0: + msg = f"gh graphql failed: {result.stderr}" + raise RuntimeError(msg) + return {"issue_numbers": result.stdout.strip()} + + +def emit_fallback_map( + issue_urls: str, + temp_dir: str, +) -> dict[str, str]: + """Build fallback execution map JSON from issue URLs.""" + nums: list[int] = [] + for url in issue_urls.split(","): + m = re.search(r"issues/(\d+)", url.strip()) + if m: + nums.append(int(m.group(1))) + if not nums: + msg = "no issue numbers extracted from issue URLs" + raise RuntimeError(msg) + issues = [{"number": n, "title": str(n)} for n in nums] + data = { + "groups": [{"group": 1, "parallel": False, "issues": issues}], + "merge_order": nums, + "pairwise_assessments": [], + } + map_file = Path(temp_dir) / "bem-fallback-map.json" + map_file.parent.mkdir(parents=True, exist_ok=True) + atomic_write(map_file, json.dumps(data)) + return {"execution_map": str(map_file)} + + +def ensure_results( + experiment_results: str, + worktree_path: str, + temp_subdir: str = ".autoskillit/temp", +) -> dict[str, str]: + """Ensure experiment_results file exists; create placeholder if empty.""" + if experiment_results: + return {"experiment_results": experiment_results} + results_path = Path(worktree_path) / temp_subdir / "run-experiment" / "results-inconclusive.md" + results_path.parent.mkdir(parents=True, exist_ok=True) + atomic_write( + results_path, + "# Experiment Results\n\n## Status\nINCONCLUSIVE\n\n" + "Experiment did not produce results — retries exhausted or adjustment failed.\n", + ) + return {"experiment_results": str(results_path)} + + +def export_local_bundle( + source_dir: str, + research_dir: str, +) -> dict[str, str]: + """Copy research dir to source_dir/research-bundles/{slug}/.""" + import shutil # noqa: PLC0415 + + local_root = Path(source_dir) / "research-bundles" + local_root.mkdir(parents=True, exist_ok=True) + slug = Path(research_dir).name + dest = local_root / slug + if dest.exists(): + shutil.rmtree(dest) + shutil.copytree(research_dir, dest) + return {"local_bundle_path": str(dest)} + + +# ─── batch_create_issues helpers ──────────────────────────────────────────── + + +def _extract_title(raw: str) -> str: + """Return the text following '# ' from the first H1 line, or a fallback.""" + m = re.search(r"^#\s+(.+)$", raw, re.MULTILINE) + return m.group(1).strip() if m else "Untitled audit finding" + + +def _strip_ticket_body(raw: str) -> str: + """Remove internal metadata and exception details from a ticket body.""" + lines = raw.splitlines() + result: list[str] = [] + skip_exceptions_section = False + for line in lines: + if line.strip().startswith("validated: true"): + continue + if ".autoskillit/" in line: + continue + if "contested_findings_" in line: + continue + if "| CONTESTED |" in line or "| VALID BUT EXCEPTION WARRANTED |" in line: + continue + if re.search(r"\*\*Contested:\*\*\s+\d+", line) or re.search( + r"\*\*Exception warranted:\*\*\s+\d+", line + ): + continue + if "**Exception note:**" in line: + continue + if re.match(r"## Findings with Exceptions\s*$", line): + skip_exceptions_section = True + continue + if skip_exceptions_section: + if line.strip().startswith("---"): + skip_exceptions_section = False + continue + result.append(line) + return "\n".join(result) + + +def _resolve_repo_identity(cwd: str) -> tuple[str, str, str]: + """Return (owner, repo_name, repo_node_id) for the given workspace.""" + result = subprocess.run( + ["gh", "repo", "view", "--json", "owner,name", "-q", '.owner.login + " " + .name'], + cwd=cwd, + capture_output=True, + text=True, + ) + if result.returncode != 0: + msg = f"gh repo view failed: {result.stderr}" + raise RuntimeError(msg) + parts = result.stdout.strip().split() + if len(parts) < 2: + msg = f"Unexpected gh repo view output: {result.stdout!r}" + raise RuntimeError(msg) + owner, repo_name = parts[0], parts[1] + safe_owner = json.dumps(owner)[1:-1] + safe_repo = json.dumps(repo_name)[1:-1] + query = f'{{ repository(owner: "{safe_owner}", name: "{safe_repo}") {{ id }} }}' + result = subprocess.run( + ["gh", "api", "graphql", "-f", f"query={query}"], + cwd=cwd, + capture_output=True, + text=True, + ) + if result.returncode != 0: + msg = f"gh graphql repo ID query failed: {result.stderr}" + raise RuntimeError(msg) + try: + data = json.loads(result.stdout) + except json.JSONDecodeError as exc: + msg = f"gh graphql repo ID: non-JSON output: {result.stdout!r}" + raise RuntimeError(msg) from exc + if "errors" in data: + raise RuntimeError(f"gh graphql repo ID errors: {data['errors']}") + node_id = data["data"]["repository"]["id"] + return owner, repo_name, node_id + + +def _ensure_and_resolve_labels(cwd: str, owner: str, repo_name: str) -> list[str]: + """Create labels if absent, resolve and return their node IDs.""" + label_defs = [ + ("recipe:implementation", "0E8A16"), + ("enhancement", "a2eeef"), + ] + for name, color in label_defs: + subprocess.run( + ["gh", "label", "create", name, "--force", "--color", color], + cwd=cwd, + capture_output=True, + ) + time.sleep(1) + safe_owner = json.dumps(owner)[1:-1] + safe_repo = json.dumps(repo_name)[1:-1] + query = ( + f'{{ repository(owner: "{safe_owner}", name: "{safe_repo}") {{' + f' impl: label(name: "recipe:implementation") {{ id }}' + f' enh: label(name: "enhancement") {{ id }} }} }}' + ) + result = subprocess.run( + ["gh", "api", "graphql", "-f", f"query={query}"], + cwd=cwd, + capture_output=True, + text=True, + ) + if result.returncode != 0: + msg = f"gh graphql label query failed: {result.stderr}" + raise RuntimeError(msg) + try: + data = json.loads(result.stdout) + except json.JSONDecodeError as exc: + msg = f"gh graphql label query: non-JSON output: {result.stdout!r}" + raise RuntimeError(msg) from exc + if "errors" in data: + raise RuntimeError(f"gh graphql label query errors: {data['errors']}") + repo = data["data"]["repository"] + if repo["impl"] is None or repo["enh"] is None: + raise RuntimeError( + f"Label resolution returned null: impl={repo['impl']!r} enh={repo['enh']!r}" + ) + return [repo["impl"]["id"], repo["enh"]["id"]] + + +def create_audit_run_dir(temp_dir: str) -> dict[str, str]: + """Create a unique per-run directory for validate-audit outputs. + + Follows the same pattern as ``create_run_dir`` in planner/manifests.py: + creates ``{temp_dir}/validate-audit/run-{stamp}-{hex}/`` so that each + pipeline invocation writes to an isolated subdirectory, preventing cross-run + file accumulation in the flat ``validate-audit/`` namespace. + """ + if not temp_dir: + raise ValueError("temp_dir must be a non-empty path") + stamp = datetime.now(UTC).strftime("%Y%m%d-%H%M%S") + run_dir = Path(temp_dir) / "validate-audit" / f"run-{stamp}-{secrets.token_hex(4)}" + try: + run_dir.mkdir(parents=True, exist_ok=True) + except OSError as exc: + raise ValueError(f"cannot create audit run directory {run_dir}: {exc}") from exc + return {"audit_run_dir": str(run_dir)} + + +def batch_create_issues( + workspace: str, + chunk_size: str = "20", + timeout: int = 120, + audit_run_dir: str = "", +) -> dict[str, str]: + """Batch-create GitHub issues from validated ticket body files via GraphQL. + + The ``audit_run_dir`` parameter scopes file discovery to a per-run directory + (created by ``create_audit_run_dir``). When provided, only ticket body files + within that directory are processed. When empty, falls back to the + workspace-derived path for direct CLI invocation compatibility. + """ + if not workspace or not Path(workspace).is_dir(): + raise ValueError(f"workspace must be an existing directory, got: {workspace!r}") + if audit_run_dir: + temp_dir = Path(audit_run_dir) + if not temp_dir.is_dir(): + raise ValueError( + f"audit_run_dir must be an existing directory, got: {audit_run_dir!r}" + ) + else: + temp_dir = Path(workspace) / ".autoskillit" / "temp" / "validate-audit" + ticket_bodies = sorted(temp_dir.glob("ticket_body_*.md")) + if not ticket_bodies: + return {"issue_urls": "", "issue_count": "0"} + + parsed: list[tuple[str, str, str]] = [] + for f in ticket_bodies: + raw = f.read_text() + m = re.match(r"ticket_body_\w+_\d+_(.+)\.md", f.name) + ts = m.group(1) if m else "" + title = _extract_title(raw) + body = _strip_ticket_body(raw) + parsed.append((title, body, ts)) + + owner, repo_name, repo_id = _resolve_repo_identity(workspace) + label_ids = _ensure_and_resolve_labels(workspace, owner, repo_name) + + all_urls: list[str] = [] + try: + chunk_sz = int(chunk_size) if chunk_size else 20 + except ValueError as exc: + raise ValueError(f"chunk_size must be a positive integer, got: {chunk_size!r}") from exc + if chunk_sz <= 0: + raise ValueError(f"chunk_size must be positive, got: {chunk_sz}") + for offset in range(0, len(parsed), chunk_sz): + chunk = parsed[offset : offset + chunk_sz] + mutation_parts = [] + variables: dict[str, object] = {} + for idx, (title, body, _) in enumerate(chunk): + alias = f"issue{idx}" + mutation_parts.append( + f"{alias}: createIssue(input: $i{idx}) {{ issue {{ number url }} }}" + ) + variables[f"i{idx}"] = { + "repositoryId": repo_id, + "title": title, + "body": body, + "labelIds": label_ids, + } + mutation = ( + "mutation(" + + ",".join(f"$i{k}: CreateIssueInput!" for k in range(len(chunk))) + + ") {" + + " ".join(mutation_parts) + + "}" + ) + payload = json.dumps({"query": mutation, "variables": variables}) + result = subprocess.run( + ["gh", "api", "graphql", "--input", "-"], + input=payload, + cwd=workspace, + capture_output=True, + text=True, + ) + if result.returncode != 0: + msg = f"gh graphql createIssue failed: {result.stderr}" + raise RuntimeError(msg) + try: + data = json.loads(result.stdout) + except json.JSONDecodeError as exc: + msg = f"gh graphql createIssue: non-JSON output: {result.stdout!r}" + raise RuntimeError(msg) from exc + if "errors" in data: + raise RuntimeError(f"gh graphql createIssue errors: {data['errors']}") + resp_data = data.get("data") or {} + for idx in range(len(chunk)): + alias = f"issue{idx}" + alias_result = resp_data.get(alias) + if alias_result is None: + raise RuntimeError(f"createIssue response missing alias {alias!r}: {data}") + issue_data = alias_result.get("issue") + if issue_data is None: + raise RuntimeError(f"createIssue alias {alias!r} returned null issue: {data}") + all_urls.append(issue_data["url"]) + if offset + chunk_sz < len(parsed): + time.sleep(1) + + return {"issue_urls": ",".join(all_urls), "issue_count": str(len(all_urls))} diff --git a/src/autoskillit/recipe/_git_helpers.py b/src/autoskillit/recipe/_git_helpers.py new file mode 100644 index 0000000000..ec0b4e3b5a --- /dev/null +++ b/src/autoskillit/recipe/_git_helpers.py @@ -0,0 +1,15 @@ +"""Shared regex helpers for git remote command lint rules.""" + +from __future__ import annotations + +import regex as re + +# Matches any line that contains a git command followed by a remote-aware verb. +# Expanded from rules_skill_content.py to include push, merge-base, diff, ls-remote. +_GIT_REMOTE_COMMAND_RE: re.Pattern[str] = re.compile( + r"\bgit\b.*?\b(?:fetch|rebase|log|show|rev-parse|push|merge-base|diff|ls-remote)\b" +) + +# Matches literal 'origin' not immediately preceded by $, {, or - (i.e., not a shell +# variable reference or shell default-value expression like ${REMOTE:-origin}). +_LITERAL_ORIGIN_RE: re.Pattern[str] = re.compile(r"(? Any: + """Return a new Recipe with the named sub_recipe placeholder step removed.""" + new_steps = {k: v for k, v in recipe.steps.items() if k != step_name} + return dataclasses.replace(recipe, steps=new_steps) + + +def _merge_sub_recipe(parent: Any, placeholder_name: str, sub: Any) -> Any: + """Replace the sub_recipe placeholder step with the sub-recipe's steps. + + Algorithm: + 1. Compute a safe name prefix from the sub-recipe name. + 2. For each step in sub, create a prefixed copy with routing fixed: + - Routes to "done" → parent placeholder's on_success + - Routes to "escalate" → parent placeholder's on_failure + - Routes to other sub-recipe step names → add prefix + 3. Insert sub-recipe steps in place of the placeholder. + 4. Merge ingredients: add sub-recipe's non-hidden ingredients into parent. + 5. Merge kitchen_rules: union (deduplicated), sub-recipe rules appended. + """ + if placeholder_name not in parent.steps: + raise KeyError( + f"_merge_sub_recipe: placeholder step '{placeholder_name}' not found in " + f"recipe '{parent.name}'. Available steps: {list(parent.steps.keys())}" + ) + placeholder = parent.steps[placeholder_name] + on_success = placeholder.on_success or "done" + on_failure = placeholder.on_failure or "escalate" + + # Build prefix: "sprint-prefix" → "sprint_prefix_", "my-sub" → "my_sub_" + raw_prefix = re.sub(r"[^a-z0-9]", "_", (sub.name or placeholder_name).lower()) + if not raw_prefix.endswith("_"): + raw_prefix += "_" + prefix = raw_prefix + + sub_step_names = set(sub.steps.keys()) + + def _fix_route(target: str | None) -> str | None: + if target is None: + return None + if target == "done": + return on_success + if target == "escalate": + return on_failure + if target in sub_step_names: + return prefix + target + return target + + def _fix_result_route(route: Any) -> Any: + if route is None: + return None + if route.conditions: + return StepResultRoute( + conditions=[ + StepResultCondition(when=c.when, route=_fix_route(c.route) or "") + for c in route.conditions + ] + ) + return StepResultRoute( + field=route.field, + routes={k: (_fix_route(v) or v) for k, v in route.routes.items()}, + ) + + prefixed_steps: dict[str, Any] = {} + for sub_step_name, sub_step in sub.steps.items(): + new_name = prefix + sub_step_name + new_step = dataclasses.replace( + sub_step, + on_success=_fix_route(sub_step.on_success), + on_failure=_fix_route(sub_step.on_failure), + on_context_limit=_fix_route(sub_step.on_context_limit), + on_exhausted=_fix_route(sub_step.on_exhausted), + on_result=_fix_result_route(sub_step.on_result), + ) + prefixed_steps[new_name] = new_step + + # Assemble new steps dict: sub-recipe steps injected in place of placeholder + new_steps: dict[str, Any] = {} + for step_name, step in parent.steps.items(): + if step_name == placeholder_name: + new_steps.update(prefixed_steps) + else: + new_steps[step_name] = step + + # Merge ingredients: sub-recipe non-hidden ingredients into parent + merged_ingredients = dict(parent.ingredients) + for ing_name, ing in sub.ingredients.items(): + if getattr(ing, "hidden", False): + continue # do not propagate hidden sub-recipe ingredients to parent + if ing_name not in merged_ingredients: + merged_ingredients[ing_name] = ing + + # Merge kitchen_rules: union (parent first, then sub-recipe additions) + seen_rules: set[str] = set(parent.kitchen_rules) + merged_rules = list(parent.kitchen_rules) + for rule in sub.kitchen_rules: + if rule not in seen_rules: + merged_rules.append(rule) + seen_rules.add(rule) + + # Merge requires_packs: union (parent first, then sub-recipe additions) + seen_packs: set[str] = set(parent.requires_packs) + merged_packs = list(parent.requires_packs) + for pack in sub.requires_packs: + if pack not in seen_packs: + merged_packs.append(pack) + seen_packs.add(pack) + + # Merge requires_features: union (parent first, then sub-recipe additions) + seen_features: set[str] = set(parent.requires_features) + merged_features = list(parent.requires_features) + for feat in sub.requires_features: + if feat not in seen_features: + merged_features.append(feat) + seen_features.add(feat) + + return dataclasses.replace( + parent, + steps=new_steps, + ingredients=merged_ingredients, + kitchen_rules=merged_rules, + requires_packs=merged_packs, + requires_features=merged_features, + ) + + +def _build_active_recipe( + recipe: Any, + ingredient_overrides: dict[str, str] | None, + project_dir: Path, + temp_dir_relpath: str = ".autoskillit/temp", +) -> tuple[Any, Any | None]: + """Return (active_recipe, combined_recipe | None). + + active_recipe: the Recipe to serve to the agent. + - If no sub_recipe steps: returns recipe unchanged. + - If sub_recipe step with gate=false: returns recipe with sub_recipe step dropped. + - If sub_recipe step with gate=true: returns the merged (combined) recipe. + + combined_recipe: the merged Recipe if any gate was true, else None. + Used to run dual validation (REQ-VALID-004). + """ + overrides = ingredient_overrides or {} + sub_recipe_steps = [ + (name, step) for name, step in recipe.steps.items() if step.sub_recipe is not None + ] + if not sub_recipe_steps: + return recipe, None + + combined: Any | None = None + working = recipe + + # Re-read each step from working.steps to get the current state after prior + # merge/drop operations, rather than using the stale reference from recipe.steps. + for step_name, _orig_step in sub_recipe_steps: + current_step = working.steps.get(step_name) + if current_step is None or current_step.sub_recipe is None: + continue + gate_name = current_step.gate or "" + gate_ingredient = working.ingredients.get(gate_name) if gate_name else None + gate_default: str = (gate_ingredient.default or "false") if gate_ingredient else "false" + gate_value = overrides.get(gate_name, gate_default) + + if gate_value.lower() in ("true", "1", "yes"): + sr_path = find_sub_recipe_by_name(current_step.sub_recipe, project_dir) + if sr_path is None: + raise FileNotFoundError( + f"Sub-recipe '{current_step.sub_recipe}' not found. " + f"Expected in recipes/sub-recipes/{current_step.sub_recipe}.yaml" + ) + try: + sub_recipe = _load_recipe_from_path(sr_path, temp_dir_relpath) + except (YAMLError, ValueError, OSError) as exc: + raise ValueError( + f"Failed to load sub-recipe '{current_step.sub_recipe}' " + f"(gate: {gate_name}={gate_value}): {exc}" + ) from exc + working = _merge_sub_recipe(working, step_name, sub_recipe) + combined = working + else: + working = _drop_sub_recipe_step(working, step_name) + + return working, combined diff --git a/src/autoskillit/recipe/_recipe_ingredients.py b/src/autoskillit/recipe/_recipe_ingredients.py new file mode 100644 index 0000000000..3a16f9d221 --- /dev/null +++ b/src/autoskillit/recipe/_recipe_ingredients.py @@ -0,0 +1,167 @@ +"""Ingredient formatting and response TypedDicts for recipe API.""" + +from __future__ import annotations + +from typing import Any, TypedDict + +from autoskillit.core import ( + TerminalColumn, + _render_gfm_table, +) + +# --------------------------------------------------------------------------- +# GFM ingredient table column specs +# --------------------------------------------------------------------------- + +_GFM_DESC_MAX_WIDTH: int = 60 + +_GFM_INGREDIENT_COLUMNS: tuple[TerminalColumn, ...] = ( + TerminalColumn("Name", max_width=30, align=">"), + TerminalColumn("Description", max_width=_GFM_DESC_MAX_WIDTH, align="<"), + TerminalColumn("Default", max_width=20, align=">"), +) + + +# --------------------------------------------------------------------------- +# Schema contract: handler → formatter boundary +# --------------------------------------------------------------------------- + + +def _ingredient_sort_key(name: str, required: bool, default: object) -> tuple[int, str]: + """Sort ingredients: required > auto-detect > flags > optional > constants.""" + if required and default is None: + return (0, name) + if default == "": + return (1, name) + if default in ("true", "false"): + return (2, name) + if default is None: + return (3, name) + return (4, name) # has a non-empty default (constants, rarely changed) + + +def build_ingredient_rows( + recipe: Any, + resolved_defaults: dict[str, str] | None = None, +) -> list[tuple[str, str, str]]: + """Build (name, description, default) rows for a recipe's ingredients. + + This is the shared source of truth for ingredient row data, consumed by + both the GFM table formatter (LLM path) and the terminal renderer + (terminal path). Descriptions are full-length here — truncation is the + terminal renderer's responsibility. + + Returns rows sorted by ingredient priority (required first, then alphabetical). + """ + resolved = resolved_defaults or {} + raw: list[tuple[str, str, str, tuple[int, str]]] = [] + for name, ing in (getattr(recipe, "ingredients", None) or {}).items(): + if getattr(ing, "hidden", False): + continue + desc = getattr(ing, "description", "") + required = getattr(ing, "required", False) + default = getattr(ing, "default", None) + sort_key = _ingredient_sort_key(name, required, default) + if default is None and required: + default_str, name_str = "(required)", f"{name} *" + elif res := resolved.get(name): + default_str, name_str = res, name + elif default == "": + default_str, name_str = "auto-detect", name + elif default == "true": + default_str, name_str = "on", name + elif default == "false": + default_str, name_str = "off", name + elif default is None: + default_str, name_str = "--", name + else: + default_str, name_str = str(default), name + raw.append((name_str, desc, default_str, sort_key)) + raw.sort(key=lambda r: r[3]) + return [(r[0], r[1], r[2]) for r in raw] + + +def format_ingredients_table( + recipe: Any, resolved_defaults: dict[str, str] | None = None +) -> str | None: + """Build a pre-formatted ingredients table from a parsed Recipe. + + When ``resolved_defaults`` is provided, auto-detect ingredients (``default: ""``) + use the resolved value instead of showing "auto-detect". + """ + ingredients = getattr(recipe, "ingredients", None) + if not ingredients: + return None + + rows = build_ingredient_rows(recipe, resolved_defaults) + + if not rows: + return None + + return _render_gfm_table(_GFM_INGREDIENT_COLUMNS, rows) + + +class LoadRecipeResult(TypedDict, total=False): + """Typed schema for the load_recipe handler → formatter boundary.""" + + content: str + diagram: str | None + suggestions: list[dict[str, Any]] + valid: bool + kitchen_rules: list[str] + requires_packs: list[str] + requires_features: list[str] + error: str + greeting: str + ingredients_table: str + orchestration_rules: str + stop_step_semantics: str + content_hash: str + composite_hash: str + recipe_version: str | None + + +class OpenKitchenResult(TypedDict, total=False): + """Typed schema for the open_kitchen named-recipe handler → formatter boundary. + + Extends LoadRecipeResult with three post-return keys injected by the handler. + """ + + # Inherited from LoadRecipeResult (15 keys) + content: str + diagram: str | None + suggestions: list[dict[str, Any]] + valid: bool + kitchen_rules: list[str] + requires_packs: list[str] + requires_features: list[str] + error: str + greeting: str + ingredients_table: str | None + orchestration_rules: str + stop_step_semantics: str + content_hash: str + composite_hash: str + recipe_version: str | None + # Post-return keys injected by open_kitchen handler (4 keys) + success: bool + kitchen: str + version: str + hook_warning: str + + +class RecipeListItem(TypedDict): + """Typed schema for a single recipe entry in the list_recipes response.""" + + name: str + description: str + summary: str + source: str + + +class ListRecipesResult(TypedDict, total=False): + """Typed schema for the list_recipes handler → formatter boundary.""" + + recipes: list[RecipeListItem] + count: int + errors: list[dict[str, str]] diff --git a/src/autoskillit/recipe/_registry_utils.py b/src/autoskillit/recipe/_registry_utils.py new file mode 100644 index 0000000000..c54b9f331d --- /dev/null +++ b/src/autoskillit/recipe/_registry_utils.py @@ -0,0 +1,15 @@ +"""Shared utilities for recipe registry loaders.""" + +from __future__ import annotations + +from pathlib import Path + +_MISSING_MTIME: float = -1.0 + + +def dir_mtime(path: Path) -> float: + """Return directory mtime, or ``_MISSING_MTIME`` if the path is inaccessible.""" + try: + return path.stat().st_mtime + except OSError: + return _MISSING_MTIME diff --git a/src/autoskillit/recipe/_rule_helpers.py b/src/autoskillit/recipe/_rule_helpers.py new file mode 100644 index 0000000000..a31dac09bf --- /dev/null +++ b/src/autoskillit/recipe/_rule_helpers.py @@ -0,0 +1,16 @@ +"""Shared helper utilities for recipe semantic rules.""" + +from __future__ import annotations + +from autoskillit.recipe._analysis import ValidationContext + + +def _is_loop_guard_step(step_name: str, ctx: ValidationContext) -> bool: + """Return True if step_name is a loop iteration guard via check_loop_iteration.""" + step = ctx.recipe.steps.get(step_name) + if step is None: + return False + if step.tool != "run_python": + return False + callable_str = step.with_args.get("callable", "") + return callable_str == "autoskillit.smoke_utils.check_loop_iteration" diff --git a/src/autoskillit/recipe/_skill_helpers.py b/src/autoskillit/recipe/_skill_helpers.py new file mode 100644 index 0000000000..6ec936f922 --- /dev/null +++ b/src/autoskillit/recipe/_skill_helpers.py @@ -0,0 +1,14 @@ +"""Shared helpers for skill-related semantic rules.""" + +from __future__ import annotations + +from autoskillit.core import SkillLister + + +def _get_skill_category_map(lister: SkillLister | None = None) -> dict[str, frozenset[str]]: + """Return {skill_name: categories} for all bundled skills.""" + if lister is None: + from autoskillit.workspace import DefaultSkillResolver # noqa: PLC0415 + + lister = DefaultSkillResolver() + return {s.name: s.categories for s in lister.list_all()} diff --git a/src/autoskillit/recipe/_skill_placeholder_parser.py b/src/autoskillit/recipe/_skill_placeholder_parser.py index 42e29dc2c1..bb8255b285 100644 --- a/src/autoskillit/recipe/_skill_placeholder_parser.py +++ b/src/autoskillit/recipe/_skill_placeholder_parser.py @@ -7,7 +7,7 @@ from __future__ import annotations -import re +import regex as re def extract_bash_blocks(content: str) -> list[str]: diff --git a/src/autoskillit/recipe/contracts.py b/src/autoskillit/recipe/contracts.py index f26c0042f0..d71c95c475 100644 --- a/src/autoskillit/recipe/contracts.py +++ b/src/autoskillit/recipe/contracts.py @@ -4,12 +4,13 @@ import dataclasses import hashlib -import re from datetime import UTC, datetime from functools import lru_cache from pathlib import Path from typing import TYPE_CHECKING, Any +import regex as re + if TYPE_CHECKING: from autoskillit.core import SkillResolver @@ -21,6 +22,7 @@ get_logger, load_yaml, pkg_root, + resolve_skill_name, ) from autoskillit.recipe.io import _parse_recipe from autoskillit.recipe.schema import Recipe, RecipeBlock @@ -45,6 +47,7 @@ class SkillInput: type: str required: bool recommended: bool = False + nullable: bool = True @dataclasses.dataclass @@ -53,6 +56,13 @@ class SkillOutput: type: str +@dataclasses.dataclass(frozen=True, slots=True) +class ResultFieldSpec: + name: str + type: str + required: bool = True + + @dataclasses.dataclass class SkillContract: inputs: list[SkillInput] @@ -61,6 +71,21 @@ class SkillContract: pattern_examples: list[str] = dataclasses.field(default_factory=list) write_behavior: str | None = None write_expected_when: list[str] = dataclasses.field(default_factory=list) + read_only: bool = False + result_fields: list[ResultFieldSpec] = dataclasses.field(default_factory=list) + + +@dataclasses.dataclass(frozen=True, slots=True) +class ToolOutputFieldSpec: + allowed_values: tuple[str, ...] + terminal_values: frozenset[str] + recoverable_values: frozenset[str] + + +@dataclasses.dataclass(frozen=True, slots=True) +class ToolOutputContractSpec: + result_field: str + fields: dict[str, ToolOutputFieldSpec] @dataclasses.dataclass @@ -79,7 +104,7 @@ class DataFlowEntry: produced: list[str] -@dataclasses.dataclass(frozen=True) +@dataclasses.dataclass(frozen=True, slots=True) class BlockFingerprint: """Structural fingerprint for a named recipe block. @@ -113,7 +138,6 @@ class RecipeCard: # Regex patterns # --------------------------------------------------------------------------- -_SKILL_NAME_RE = re.compile(r"/autoskillit:([\w-]+)") _CONTEXT_REF_RE = re.compile(r"\$\{\{\s*context\.(\w+)\s*\}\}") INPUT_REF_RE = re.compile(r"\$\{\{\s*inputs\.(\w+)\s*\}\}") _TEMPLATE_REF_RE = re.compile(r"\$\{\{[^}]+\}\}") @@ -132,27 +156,6 @@ def load_bundled_manifest() -> dict[str, Any]: return load_yaml(manifest_path) -def resolve_skill_name(skill_command: str) -> str | None: - """Extract the skill name from a command string. - - Returns the skill name (e.g. "retry-worktree") or None if the command - does not reference an autoskillit skill or contains dynamic expressions. - """ - match = _SKILL_NAME_RE.search(skill_command) - if not match: - return None - name = match.group(1) - # Reject dynamic names containing template expressions - if "${{" in name: - return None - # Reject names truncated by a bash-style {placeholder} token immediately - # following the match (e.g. "/autoskillit:exp-lens-{slug}" extracts - # "exp-lens-" but is dynamic — the true name is resolved at runtime). - if match.end() < len(skill_command) and skill_command[match.end()] == "{": - return None - return name - - def get_skill_contract(skill_name: str, manifest: dict[str, Any]) -> SkillContract | None: """Look up a skill in the manifest and return a SkillContract.""" skills = manifest.get("skills", {}) @@ -163,6 +166,7 @@ def get_skill_contract(skill_name: str, manifest: dict[str, Any]) -> SkillContra SkillInput( name=inp["name"], type=inp["type"], + # Skill inputs default to optional — skills are permissive by design. required=inp.get("required", False), recommended=inp.get("recommended", False), ) @@ -175,6 +179,20 @@ def get_skill_contract(skill_name: str, manifest: dict[str, Any]) -> SkillContra examples = skill_data.get("pattern_examples", []) write_behavior = skill_data.get("write_behavior") write_expected_when = skill_data.get("write_expected_when", []) + read_only = bool(skill_data.get("read_only", False)) + try: + result_fields = [ + ResultFieldSpec( + name=rf["name"], + type=rf["type"], + required=rf.get("required", True), + ) + for rf in skill_data.get("result_fields", []) + ] + except KeyError as exc: + raise KeyError( + f"Malformed result_fields entry for skill '{skill_name}': missing key {exc}" + ) from exc return SkillContract( inputs=inputs, outputs=outputs, @@ -182,9 +200,69 @@ def get_skill_contract(skill_name: str, manifest: dict[str, Any]) -> SkillContra pattern_examples=examples, write_behavior=write_behavior, write_expected_when=write_expected_when, + read_only=read_only, + result_fields=result_fields, ) +def get_callable_contract( + dotted_path: str, manifest: dict[str, Any] | None = None +) -> SkillContract | None: + """Look up a run_python callable in the manifest and return a SkillContract. + + Callable contracts live under the ``callable_contracts`` top-level key in + skill_contracts.yaml, keyed by the fully-qualified dotted Python path + (e.g. ``autoskillit.smoke_utils.check_review_loop``). + """ + if manifest is None: + manifest = load_bundled_manifest() + callables = manifest.get("callable_contracts", {}) + entry = callables.get(dotted_path) + if entry is None: + return None + inputs = [ + SkillInput( + name=inp["name"], + type=inp["type"], + # Callable inputs default to required — callables are strict by design. + required=inp.get("required", True), + nullable=inp.get("nullable", True), + ) + for inp in entry.get("inputs", []) + ] + outputs = [SkillOutput(name=out["name"], type=out["type"]) for out in entry.get("outputs", [])] + return SkillContract(inputs=inputs, outputs=outputs) + + +def get_tool_output_contract( + tool_name: str, manifest: dict[str, Any] | None = None +) -> ToolOutputContractSpec | None: + """Return the ToolOutputContractSpec for a named MCP tool, or None if not declared.""" + if manifest is None: + manifest = load_bundled_manifest() + entry = manifest.get("tool_output_contracts", {}).get(tool_name) + if entry is None: + return None + fields = {} + for field_name, field_data in entry.get("fields", {}).items(): + if not isinstance(field_data, dict): + raise ValueError( + f"tool_output_contracts entry for {tool_name!r}: " + f"field {field_name!r} must be a mapping, got {type(field_data).__name__!r}" + ) + fields[field_name] = ToolOutputFieldSpec( + allowed_values=tuple(field_data.get("allowed_values", [])), + terminal_values=frozenset(field_data.get("terminal_values", [])), + recoverable_values=frozenset(field_data.get("recoverable_values", [])), + ) + result_field = entry.get("result_field") + if not result_field: + raise ValueError( + f"tool_output_contracts entry for {tool_name!r} is missing required 'result_field'" + ) + return ToolOutputContractSpec(result_field=result_field, fields=fields) + + def compute_skill_hash(skill_name: str, *, skills_dir: Path) -> str: """Compute SHA256 hash of a skill's SKILL.md file.""" skill_md = skills_dir / skill_name / "SKILL.md" @@ -230,13 +308,15 @@ def count_positional_args(skill_command: str) -> int: Returns 0 if there are no extra tokens after the skill name. """ - match = _SKILL_NAME_RE.search(skill_command) - if not match: + name = resolve_skill_name(skill_command) + if not name: + return 0 + idx = skill_command.find(name) + if idx < 0: return 0 - after_skill = skill_command[match.end() :].strip() + after_skill = skill_command[idx + len(name) :].strip() if not after_skill: return 0 - # Remove template references before counting without_templates = _TEMPLATE_REF_RE.sub("", after_skill).strip() if not without_templates: return 0 @@ -378,10 +458,15 @@ def generate_recipe_card( skill_entry["write_behavior"] = contract.write_behavior if contract.write_expected_when: skill_entry["write_expected_when"] = contract.write_expected_when + if contract.read_only: + skill_entry["read_only"] = True skills[skill_name] = skill_entry - if count_positional_args(skill_cmd) > 0: - # Positional args used — can't verify named inputs by ref + pos_arg_count = count_positional_args(skill_cmd) + if pos_arg_count > 0: + # Positional args used — can't verify named inputs by ref. + # Record the count so downstream rules know args were supplied. entry["required"] = [] + entry["positional_args"] = pos_arg_count else: # Named template refs only — flag required inputs not referenced ctx_refs = extract_context_refs(step) diff --git a/src/autoskillit/recipe/diagrams.py b/src/autoskillit/recipe/diagrams.py index b40aa2b6f0..08f8fc2576 100644 --- a/src/autoskillit/recipe/diagrams.py +++ b/src/autoskillit/recipe/diagrams.py @@ -11,9 +11,10 @@ from __future__ import annotations -import re from pathlib import Path +import regex as re + from autoskillit.recipe.staleness_cache import compute_recipe_hash # Diagram format version — bump when the render-recipe skill spec changes diff --git a/src/autoskillit/recipe/experiment_type_registry.py b/src/autoskillit/recipe/experiment_type_registry.py index f94f9c884f..fc48c6a89e 100644 --- a/src/autoskillit/recipe/experiment_type_registry.py +++ b/src/autoskillit/recipe/experiment_type_registry.py @@ -2,12 +2,14 @@ from __future__ import annotations -from dataclasses import dataclass +from dataclasses import dataclass, field from pathlib import Path +from typing import Any from autoskillit.core import get_logger, load_yaml, pkg_root +from autoskillit.recipe._registry_utils import _MISSING_MTIME, dir_mtime -_log = get_logger(__name__) +logger = get_logger(__name__) BUNDLED_EXPERIMENT_TYPES_DIR: Path = pkg_root() / "recipes" / "experiment-types" @@ -22,16 +24,47 @@ class ExperimentTypeSpec: applicable_lenses: dict[str, str | None] red_team_focus: dict[str, str] l1_severity: dict[str, str] + schema_version: str = "" + priority: int = 999 + is_fallback: bool = False + dimension_weight_rationale: dict[str, str] = field(default_factory=dict) + + +def _parse_int_field(data: dict, field: str, default: int, source_path: Path) -> int: + val = data.get(field, default) + try: + return int(val) + except (ValueError, TypeError) as e: + name = data.get("name", "?") + raise TypeError( + f"Experiment type '{name}' field '{field}' must be an integer: {source_path}" + ) from e + + +def _parse_bool_field(data: dict, field: str, default: bool, source_path: Path) -> bool: + val = data.get(field, default) + if not isinstance(val, bool): + name = data.get("name", "?") + raise TypeError( + f"Experiment type '{name}' field '{field}' must be a boolean: {source_path}" + ) + return val def _parse_experiment_type(data: dict, source_path: Path) -> ExperimentTypeSpec: if "name" not in data: raise ValueError(f"Experiment type YAML missing 'name' field: {source_path}") - for field in ("dimension_weights", "applicable_lenses", "red_team_focus", "l1_severity"): - val = data.get(field) + for f in ( + "dimension_weights", + "applicable_lenses", + "red_team_focus", + "l1_severity", + "dimension_weight_rationale", + ): + val = data.get(f) if val is not None and not isinstance(val, dict): raise TypeError( - f"Experiment type '{data['name']}' field '{field}' must be a dict, " + f"Experiment type '{data['name']}' field '{f}' must be a dict, " f"got {type(val).__name__}: {source_path}" ) return ExperimentTypeSpec( @@ -41,6 +74,10 @@ def _parse_experiment_type(data: dict, source_path: Path) -> ExperimentTypeSpec: applicable_lenses=dict(data.get("applicable_lenses", {})), red_team_focus=dict(data.get("red_team_focus", {})), l1_severity=dict(data.get("l1_severity", {})), + schema_version=str(data.get("schema_version", "")), + priority=_parse_int_field(data, "priority", 999, source_path), + is_fallback=_parse_bool_field(data, "is_fallback", False, source_path), + dimension_weight_rationale=dict(data.get("dimension_weight_rationale", {})), ) @@ -55,32 +92,101 @@ def _load_types_from_dir(directory: Path) -> dict[str, ExperimentTypeSpec]: spec = _parse_experiment_type(data, path) result[spec.name] = spec except Exception: - _log.warning("Skipping malformed experiment type file: %s", path, exc_info=True) + logger.warning("Skipping malformed experiment type file: %s", path, exc_info=True) return result +def parse_experiment_type(data: dict[str, Any], source_path: Path) -> ExperimentTypeSpec: + return _parse_experiment_type(data, source_path) + + +def load_types_from_dir(directory: Path) -> dict[str, ExperimentTypeSpec]: + return _load_types_from_dir(directory) + + +_exp_types_cache: dict[tuple[str | None, float, float], list[ExperimentTypeSpec]] = {} + + def load_all_experiment_types( project_dir: Path | None = None, -) -> dict[str, ExperimentTypeSpec]: +) -> list[ExperimentTypeSpec]: """Load experiment types: bundled types merged with user-defined overrides. User-defined types with the same name as a bundled type replace the bundled type entirely — no field merging. User-defined types with a new name are added alongside bundled types. + The returned list is sorted by ``(priority, name)`` with ``is_fallback=True`` + entries always appended last. + Args: project_dir: Project root containing optional user-defined overrides at ``.autoskillit/experiment-types/``. When ``None``, only bundled types are returned. Returns: - Mapping of experiment type name to ``ExperimentTypeSpec``. + Sorted list of ``ExperimentTypeSpec``, fallback entries last. """ + bundled_mt = dir_mtime(BUNDLED_EXPERIMENT_TYPES_DIR) + if project_dir is not None: + user_dir = Path(project_dir) / ".autoskillit" / "experiment-types" + user_mt = dir_mtime(user_dir) + else: + user_mt = _MISSING_MTIME + key: tuple[str | None, float, float] = ( + str(project_dir) if project_dir is not None else None, + bundled_mt, + user_mt, + ) + cached = _exp_types_cache.get(key) + if cached is not None: + return list(cached) + types = _load_types_from_dir(BUNDLED_EXPERIMENT_TYPES_DIR) if project_dir is not None: user_dir = Path(project_dir) / ".autoskillit" / "experiment-types" user_types = _load_types_from_dir(user_dir) + for spec in user_types.values(): + if spec.schema_version and spec.schema_version != "1.0": + logger.warning( + "User experiment type has schema_version mismatch; loading continues", + type_name=spec.name, + schema_version=spec.schema_version, + expected_schema_version="1.0", + ) types.update(user_types) - return types + non_fallback = [s for s in types.values() if not s.is_fallback] + fallback = [s for s in types.values() if s.is_fallback] + non_fallback.sort(key=lambda s: (s.priority, s.name)) + fallback.sort(key=lambda s: (s.priority, s.name)) + result = non_fallback + fallback + _exp_types_cache[key] = result + return list(result) + + +def get_experiment_type_by_name( + name: str, + project_dir: Path | None = None, +) -> ExperimentTypeSpec | None: + """Look up a single experiment type by name. + + Returns the matching spec or None if not found. + """ + for spec in load_all_experiment_types(project_dir): + if spec.name == name: + return spec + return None + + +_SILENT_THRESHOLD = 6 + + +def is_silent_type(spec: ExperimentTypeSpec) -> bool: + """Return True when >=6 of 8 dimension_weights are 'S' (suppressed). + + Silent experiment types lack dimensional signal strength for opinionated + design-review scoring. See docs/research/silent-type-convention.md. + """ + return sum(1 for v in spec.dimension_weights.values() if v == "S") >= _SILENT_THRESHOLD diff --git a/src/autoskillit/recipe/identity.py b/src/autoskillit/recipe/identity.py new file mode 100644 index 0000000000..dfc85748af --- /dev/null +++ b/src/autoskillit/recipe/identity.py @@ -0,0 +1,153 @@ +"""Recipe identity hashing — content and composite fingerprints, query and re-run detection.""" + +from __future__ import annotations + +import hashlib +import json +from pathlib import Path +from typing import Any + +from autoskillit.core import get_logger +from autoskillit.recipe.contracts import compute_skill_hash, resolve_skill_name +from autoskillit.recipe.io import find_sub_recipe_by_name +from autoskillit.recipe.schema import Recipe +from autoskillit.recipe.staleness_cache import compute_recipe_hash + +logger = get_logger(__name__) + + +def compute_composite_hash( + recipe_path: Path, + recipe: Recipe, + *, + skills_dir: Path, + project_dir: Path, + content_bytes: bytes | None = None, + _seen: frozenset[Path] | None = None, +) -> str: + """Compute a composite hash covering the recipe, referenced skills, and sub-recipes. + + The hash is deterministic: skill and sub-recipe names are sorted before + hashing. Returns "sha256:<64-hex>". + """ + if _seen is None: + _seen = frozenset() + + hasher = hashlib.sha256() + + hasher.update(b"autoskillit-composite-v1\n") + + hasher.update(content_bytes if content_bytes is not None else recipe_path.read_bytes()) + + skill_names: set[str] = set() + for step in recipe.steps.values(): + if step.tool == "run_skill" and step.message: + name = resolve_skill_name(step.message) + if name: + skill_names.add(name) + + for name in sorted(skill_names): + h = compute_skill_hash(name, skills_dir=skills_dir) + hasher.update(f"skill:{name}:{h}\n".encode()) + + sub_names: set[str] = set() + for step in recipe.steps.values(): + if step.sub_recipe: + sub_names.add(step.sub_recipe) + + for name in sorted(sub_names): + sub_path = find_sub_recipe_by_name(name, project_dir) + if sub_path and sub_path.is_file(): + resolved = sub_path.resolve() + if resolved in _seen: + logger.debug("cycle_detected_in_sub_recipe", path=str(sub_path)) + continue + sub_recipe = _load_sub_recipe_for_hash(sub_path) + if sub_recipe is not None: + sub_hash = compute_composite_hash( + sub_path, + sub_recipe, + skills_dir=skills_dir, + project_dir=project_dir, + _seen=_seen | {resolved}, + ) + else: + sub_hash = compute_recipe_hash(sub_path) + hasher.update(f"sub:{name}:{sub_hash}\n".encode()) + + return "sha256:" + hasher.hexdigest() + + +def _load_sub_recipe_for_hash(path: Path) -> Recipe | None: + """Load a sub-recipe YAML and return a Recipe, or None on any parse failure.""" + from autoskillit.recipe.io import _parse_recipe # noqa: PLC0415 + + try: + from autoskillit.core import load_yaml # noqa: PLC0415 + + data = load_yaml(path.read_text(encoding="utf-8")) + if isinstance(data, dict) and "steps" in data: + return _parse_recipe(data) + except Exception: + logger.debug("sub_recipe_load_for_hash_failed", path=str(path), exc_info=True) + return None + + +def find_prior_runs( + sessions_jsonl: Path, + *, + composite_hash: str = "", + recipe_name: str = "", +) -> list[dict[str, Any]]: + """Return sessions.jsonl entries matching the given recipe identity. + + Filters by composite_hash if provided, then by recipe_name. + Returns matches sorted by timestamp descending. Skips malformed lines. + """ + if not sessions_jsonl.is_file(): + return [] + matches: list[dict[str, Any]] = [] + try: + lines = sessions_jsonl.read_text(encoding="utf-8").splitlines() + except (OSError, UnicodeDecodeError): + logger.debug("sessions_jsonl_read_failed", path=str(sessions_jsonl), exc_info=True) + return [] + for line in lines: + if not line.strip(): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + if composite_hash and entry.get("recipe_composite_hash") != composite_hash: + continue + if recipe_name and entry.get("recipe_name") != recipe_name: + continue + if not composite_hash and not recipe_name: + continue + matches.append(entry) + return sorted(matches, key=lambda e: e.get("timestamp", ""), reverse=True) + + +def check_rerun_detection( + sessions_jsonl: Path, + *, + composite_hash: str, +) -> dict[str, Any] | None: + """Check if this exact recipe composite hash has been run before. + + Returns a suggestion dict if a prior run is found, None otherwise. + """ + prior = find_prior_runs(sessions_jsonl, composite_hash=composite_hash) + if not prior: + return None + last = prior[0] + return { + "rule": "duplicate-run-detected", + "severity": "info", + "message": ( + f"This exact recipe version was last run on " + f"{last.get('timestamp', 'unknown')} " + f"(session {last.get('session_id', 'unknown')})." + ), + } diff --git a/src/autoskillit/recipe/io.py b/src/autoskillit/recipe/io.py index 0b3cc349d4..81d1e6d3a3 100644 --- a/src/autoskillit/recipe/io.py +++ b/src/autoskillit/recipe/io.py @@ -2,16 +2,31 @@ from __future__ import annotations +import json as _json from collections.abc import Iterator from pathlib import Path from typing import Any -from autoskillit.core import LoadReport, LoadResult, RecipeSource, get_logger, load_yaml, pkg_root +from autoskillit.core import ( + CORE_PACKS, + DispatchGateType, + LoadReport, + LoadResult, + RecipeSource, + get_logger, + load_yaml, + pkg_root, +) +from autoskillit.core import fast_loads as _fast_loads +from autoskillit.recipe.order import BUNDLED_RECIPE_ORDER from autoskillit.recipe.schema import ( AUTOSKILLIT_VERSION_KEY, + RECIPE_VERSION_KEY, + CampaignDispatch, Recipe, RecipeInfo, RecipeIngredient, + RecipeKind, RecipeStep, StepResultCondition, StepResultRoute, @@ -36,6 +51,46 @@ def substitute_temp_placeholder(text: str, temp_dir_relpath: str) -> str: return text.replace(_TEMP_PLACEHOLDER, temp_dir_relpath) +def _load_recipe_dict( + yaml_path: Path, + *, + raw_text: str | None = None, + temp_dir_relpath: str | None = None, +) -> dict[str, Any]: + """Load a recipe dict, preferring a pre-compiled JSON sibling when fresh. + + Args: + yaml_path: Path to the .yaml recipe file. + raw_text: Already-read YAML text (avoids redundant I/O on fallback). + temp_dir_relpath: When set, ``{{AUTOSKILLIT_TEMP}}`` is replaced in the + text before parsing (applies to both JSON and YAML paths). + """ + json_path = yaml_path.with_suffix(".json") + try: + if json_path.stat().st_mtime_ns > yaml_path.stat().st_mtime_ns: + text = json_path.read_text(encoding="utf-8") + if temp_dir_relpath is not None: + text = substitute_temp_placeholder(text, temp_dir_relpath) + data = _fast_loads(text) + if isinstance(data, dict): + return data + logger.warning( + "Pre-compiled JSON is not a mapping, falling back to YAML: %s", json_path + ) + except _json.JSONDecodeError: + logger.warning("Pre-compiled JSON is corrupt, falling back to YAML: %s", json_path) + except (FileNotFoundError, OSError): + pass + if raw_text is None: + raw_text = yaml_path.read_text(encoding="utf-8") + if temp_dir_relpath is not None: + raw_text = substitute_temp_placeholder(raw_text, temp_dir_relpath) + data = load_yaml(raw_text) + if not isinstance(data, dict): + raise ValueError(f"Recipe file must contain a YAML mapping: {yaml_path}") + return data + + def load_recipe(path: Path, temp_dir_relpath: str = ".autoskillit/temp") -> Recipe: """Parse a YAML recipe file into a Recipe dataclass. @@ -48,12 +103,11 @@ def load_recipe(path: Path, temp_dir_relpath: str = ".autoskillit/temp") -> Reci ``_analysis.py`` imports ``iter_steps_with_context`` from this module, so a top-level import here would create a cycle. """ - raw_text = path.read_text(encoding="utf-8") - substituted = substitute_temp_placeholder(raw_text, temp_dir_relpath) - data = load_yaml(substituted) - if not isinstance(data, dict): - raise ValueError(f"Recipe file must contain a YAML mapping: {path}") + data = _load_recipe_dict(path, temp_dir_relpath=temp_dir_relpath) recipe = _parse_recipe(data) + from autoskillit.recipe.staleness_cache import compute_recipe_hash # noqa: PLC0415 + + recipe.content_hash = compute_recipe_hash(path) # Deferred import breaks the circular dependency with _analysis.py. from autoskillit.recipe._analysis import _build_step_graph, extract_blocks # noqa: PLC0415 @@ -61,7 +115,38 @@ def load_recipe(path: Path, temp_dir_relpath: str = ".autoskillit/temp") -> Reci return recipe -def list_recipes(project_dir: Path) -> LoadResult[RecipeInfo]: +GROUP_LABELS: dict[int, str] = { + 0: "Bundled Recipes", + 1: "Bundled Add-ons", + 2: "Family Recipes", + 3: "Experimental", +} + + +def group_rank(r: RecipeInfo) -> int: + if r.experimental: + return 3 + if r.source == RecipeSource.PROJECT: + return 2 + if r.requires_packs and all(p in CORE_PACKS for p in r.requires_packs): + return 0 + return 1 + + +def _registry_position(r: RecipeInfo) -> int: + """Return sort position within Group 0; 0 (no-op) for all other groups.""" + if group_rank(r) == 0: + try: + return BUNDLED_RECIPE_ORDER.index(r.name) + except ValueError: + return len(BUNDLED_RECIPE_ORDER) + return 0 + + +def list_recipes( + project_dir: Path, + exclude_kinds: frozenset[RecipeKind] = frozenset(), +) -> LoadResult[RecipeInfo]: """Find available recipes from project and built-in sources.""" seen: set[str] = set() items: list[RecipeInfo] = [] @@ -73,8 +158,9 @@ def list_recipes(project_dir: Path) -> LoadResult[RecipeInfo]: builtin_dir = pkg_root() / "recipes" _collect_recipes(RecipeSource.BUILTIN, builtin_dir, seen, items, errors) + filtered = [r for r in items if r.kind not in exclude_kinds] if exclude_kinds else items return LoadResult( - items=sorted(items, key=lambda r: (r.source != RecipeSource.BUILTIN, r.name)), + items=sorted(filtered, key=lambda r: (group_rank(r), _registry_position(r), r.name)), errors=errors, ) @@ -128,6 +214,57 @@ def find_recipe_by_name(name: str, project_dir: Path) -> RecipeInfo | None: return next((r for r in result.items if r.name == name), None) +def list_campaign_recipes(project_dir: Path) -> LoadResult[RecipeInfo]: + """Find available campaign recipes from project and built-in sources.""" + seen: set[str] = set() + items: list[RecipeInfo] = [] + errors: list[LoadReport] = [] + + project_campaigns_dir = project_dir / ".autoskillit" / "recipes" / "campaigns" + _collect_recipes(RecipeSource.PROJECT, project_campaigns_dir, seen, items, errors) + + builtin_campaigns_dir = pkg_root() / "recipes" / "campaigns" + _collect_recipes(RecipeSource.BUILTIN, builtin_campaigns_dir, seen, items, errors) + + return LoadResult( + items=sorted(items, key=lambda r: (r.source != RecipeSource.BUILTIN, r.name)), + errors=errors, + ) + + +def find_campaign_by_name(name: str, project_dir: Path) -> RecipeInfo | None: + """Find a campaign recipe by name. + + Returns the first match (project takes precedence), or None if not found. + """ + result = list_campaign_recipes(project_dir) + return next((r for r in result.items if r.name == name), None) + + +def load_campaign_recipes_in_packs( + packs: frozenset[str], + project_dir: Path, + allowed_recipe_names: frozenset[str] = frozenset(), +) -> list[Recipe]: + """Return all campaign recipes whose categories overlap with the requested packs. + + Recipes explicitly named in ``allowed_recipe_names`` are also included regardless + of category membership, allowing callers to honour the requesting campaign's + ``allowed_recipes`` list. + """ + result = list_campaign_recipes(project_dir) + matching: list[Recipe] = [] + for info in result.items: + try: + recipe = load_recipe(info.path) + except Exception: + logger.warning("Failed to load campaign recipe", path=str(info.path)) + continue + if (set(recipe.categories) & packs) or (info.name in allowed_recipe_names): + matching.append(recipe) + return matching + + # --- internal helpers --- # Fields explicitly handled by _parse_step. Must match RecipeStep.__dataclass_fields__ @@ -154,6 +291,7 @@ def find_recipe_by_name(name: str, project_dir: Path) -> RecipeInfo | None: "optional", "skip_when_false", "model", + "provider", "description", "sub_recipe", "gate", @@ -172,6 +310,47 @@ def find_recipe_by_name(name: str, project_dir: Path) -> RecipeInfo | None: f"{_PARSE_STEP_HANDLED_FIELDS - frozenset(RecipeStep.__dataclass_fields__)}" ) +_PARSE_RECIPE_HANDLED_FIELDS: frozenset[str] = frozenset( + { + "name", + "description", + "summary", + "ingredients", + "steps", + "kitchen_rules", + "version", + "recipe_version", + "experimental", + "requires_packs", + "kind", + "dispatches", + "categories", + "requires_recipe_packs", + "allowed_recipes", + "continue_on_failure", + "requires_features", + } +) + +_RECIPE_COMPUTED_FIELDS: frozenset[str] = frozenset( + { + "content_hash", + "composite_hash", + "blocks", + } +) + +if _PARSE_RECIPE_HANDLED_FIELDS | _RECIPE_COMPUTED_FIELDS != frozenset( + Recipe.__dataclass_fields__ +): + _expected = frozenset(Recipe.__dataclass_fields__) + _combined = _PARSE_RECIPE_HANDLED_FIELDS | _RECIPE_COMPUTED_FIELDS + raise RuntimeError( + "_parse_recipe field list is out of sync with Recipe schema.\n" + f" Missing from handled+computed: {_expected - _combined}\n" + f" Extra in handled+computed: {_combined - _expected}" + ) + def _parse_recipe(data: dict[str, Any]) -> Recipe: name = data.get("name", "") @@ -185,6 +364,7 @@ def _parse_recipe(data: dict[str, Any]) -> Recipe: description=inp_data.get("description", ""), required=inp_data.get("required", False), default=inp_data.get("default"), + type=inp_data.get("type"), hidden=bool(inp_data.get("hidden", False)), ) @@ -205,6 +385,72 @@ def _parse_recipe(data: dict[str, Any]) -> Recipe: f"'requires_packs' must be a list, got {type(requires_packs_raw).__name__!r}" ) + _rv = data.get(RECIPE_VERSION_KEY) + if _rv is not None and not isinstance(_rv, str): + raise ValueError( + f"recipe_version must be a quoted string in YAML, got {type(_rv).__name__}: {_rv!r}. " + f"Use recipe_version: '{_rv}' (with quotes) in your recipe file." + ) + + kind_raw = data.get("kind", "standard") + try: + kind = RecipeKind(kind_raw) + except ValueError: + kind = RecipeKind.STANDARD + + dispatches_raw = data.get("dispatches") or [] + dispatches = [] + for d in dispatches_raw: + if isinstance(d, dict): + d_name = d.get("name", "") + _raw_gate = d.get("gate") or None + try: + d_gate: DispatchGateType | None = ( + DispatchGateType(_raw_gate) if _raw_gate else None + ) + except ValueError: + d_gate = _raw_gate # type: ignore[assignment] # Invalid; caught by validate_recipe_structure + d_recipe = d.get("recipe", "") + if not d_name: + raise ValueError(f"Campaign dispatch is missing required 'name' field: {d!r}") + if d_gate and d_recipe: + raise ValueError( + f"Campaign dispatch {d_name!r} has both 'gate' and 'recipe' set. " + "A dispatch must be either a gate dispatch (gate only) or a recipe " + "dispatch (recipe only), not both." + ) + if not d_gate and not d_recipe: + raise ValueError( + f"Campaign dispatch is missing required 'recipe' field " + f"(required when 'gate' is not set): {d!r}" + ) + dispatches.append( + CampaignDispatch( + name=d_name, + recipe=d_recipe, + task=d.get("task", ""), + ingredients=d.get("ingredients") or {}, + depends_on=d.get("depends_on") or [], + capture=d.get("capture") or {}, + gate=d_gate, + message=d.get("message") or None, + ) + ) + + categories = data.get("categories") or [] + requires_recipe_packs = data.get("requires_recipe_packs") or [] + allowed_recipes = data.get("allowed_recipes") or [] + continue_on_failure = bool(data.get("continue_on_failure", False)) + _rf_val = data.get("requires_features") + requires_features_raw = _rf_val if _rf_val is not None else [] + if not isinstance(requires_features_raw, list): + raise ValueError( + f"'requires_features' must be a list, got {type(requires_features_raw).__name__!r}" + ) + if not all(isinstance(f, str) for f in requires_features_raw): + bad = [f for f in requires_features_raw if not isinstance(f, str)] + raise ValueError(f"'requires_features' entries must be strings, got: {bad!r}") + return Recipe( name=name, description=description, @@ -213,8 +459,16 @@ def _parse_recipe(data: dict[str, Any]) -> Recipe: steps=steps, kitchen_rules=kitchen_rules, version=data.get(AUTOSKILLIT_VERSION_KEY), + recipe_version=_rv, experimental=bool(data.get("experimental", False)), requires_packs=requires_packs_raw, + kind=kind, + dispatches=dispatches, + categories=categories, + requires_recipe_packs=requires_recipe_packs, + allowed_recipes=allowed_recipes, + continue_on_failure=continue_on_failure, + requires_features=requires_features_raw, ) @@ -264,6 +518,7 @@ def _parse_step(data: dict[str, Any]) -> RecipeStep: optional=bool(data.get("optional", False)), skip_when_false=data.get("skip_when_false"), model=data.get("model"), + provider=data.get("provider"), description=data.get("description", ""), sub_recipe=data.get("sub_recipe"), gate=data.get("gate"), @@ -287,12 +542,14 @@ def _collect_recipes( if f.suffix in (".yaml", ".yml") and f.is_file(): try: raw = f.read_text(encoding="utf-8") - data = load_yaml(raw) - if not isinstance(data, dict): - raise ValueError("recipe must be a YAML mapping") + data = _load_recipe_dict(f, raw_text=raw) recipe = _parse_recipe(data) if recipe.name and recipe.name not in seen: seen.add(recipe.name) + from autoskillit.recipe.staleness_cache import ( # noqa: PLC0415 + compute_recipe_hash as _crh, + ) + result.append( RecipeInfo( name=recipe.name, @@ -301,7 +558,12 @@ def _collect_recipes( path=f, summary=recipe.summary, version=recipe.version, + recipe_version=recipe.recipe_version, + content_hash=_crh(f), content=raw, + kind=recipe.kind, + experimental=recipe.experimental, + requires_packs=recipe.requires_packs, ) ) except Exception as exc: diff --git a/src/autoskillit/recipe/loader.py b/src/autoskillit/recipe/loader.py index 4f4b353994..6811484ac3 100644 --- a/src/autoskillit/recipe/loader.py +++ b/src/autoskillit/recipe/loader.py @@ -5,7 +5,7 @@ from pathlib import Path from autoskillit.core import RecipeSource, load_yaml -from autoskillit.recipe.schema import AUTOSKILLIT_VERSION_KEY, RecipeInfo +from autoskillit.recipe.schema import AUTOSKILLIT_VERSION_KEY, RECIPE_VERSION_KEY, RecipeInfo def _extract_frontmatter(text: str) -> str: @@ -45,4 +45,5 @@ def parse_recipe_metadata(path: Path) -> RecipeInfo: path=path, source=RecipeSource.PROJECT, version=data.get(AUTOSKILLIT_VERSION_KEY), + recipe_version=data.get(RECIPE_VERSION_KEY), ) diff --git a/src/autoskillit/recipe/methodology_disambiguation.py b/src/autoskillit/recipe/methodology_disambiguation.py new file mode 100644 index 0000000000..501bbf5610 --- /dev/null +++ b/src/autoskillit/recipe/methodology_disambiguation.py @@ -0,0 +1,193 @@ +"""Methodology tradition disambiguation — resolves multi-tradition candidate sets.""" + +from __future__ import annotations + +from dataclasses import dataclass +from pathlib import Path +from typing import Literal + +from autoskillit.core import load_yaml, pkg_root + +BUNDLED_METHODOLOGY_TRADITIONS_DIR: Path = pkg_root() / "recipes" / "methodology-traditions" + + +@dataclass(frozen=True, slots=True) +class DisambiguationExceptionDef: + when_present: str + add_union_rules: tuple[str, ...] + + +@dataclass(frozen=True, slots=True) +class DisambiguationRuleDef: + name: str + order: int + description: str + trigger_traditions: frozenset[str] + trigger_mode: Literal["anchor_plus_any", "all_present"] + anchor: str | None + primary_tradition: str + applied_union_rules: tuple[str, ...] + exceptions: tuple[DisambiguationExceptionDef, ...] + + +@dataclass(frozen=True, slots=True) +class CrossTraditionOverlapDef: + name: str + order: int + description: str + trigger_traditions: frozenset[str] + primary_tradition: str + applied_union_rules: tuple[str, ...] + overrides_primary: bool = False + + +@dataclass(frozen=True, slots=True) +class DisambiguationResult: + primary_tradition: str | None + applied_union_rules: tuple[str, ...] + precedence_trace: str + candidate_set: tuple[str, ...] + + +def load_disambiguation_rules() -> tuple[ + list[DisambiguationRuleDef], list[CrossTraditionOverlapDef] +]: + yaml_path = BUNDLED_METHODOLOGY_TRADITIONS_DIR / "_disambiguation.yaml" + try: + data = load_yaml(yaml_path) + except Exception as exc: + raise RuntimeError(f"Failed to load disambiguation rules from {yaml_path}") from exc + + rules: list[DisambiguationRuleDef] = [] + for idx, r in enumerate(data.get("disambiguation_rules", [])): + try: + exceptions = tuple( + DisambiguationExceptionDef( + when_present=e["when_present"], + add_union_rules=tuple(e["add_union_rules"]), + ) + for e in r.get("exceptions", []) + ) + rules.append( + DisambiguationRuleDef( + name=r["name"], + order=r["order"], + description=r["description"], + trigger_traditions=frozenset(r["trigger_traditions"]), + trigger_mode=r["trigger_mode"], + anchor=r.get("anchor"), + primary_tradition=r["primary_tradition"], + applied_union_rules=tuple(r["applied_union_rules"]), + exceptions=exceptions, + ) + ) + except (KeyError, TypeError) as exc: + name = r.get("name", f"") if isinstance(r, dict) else f"" + raise RuntimeError(f"Invalid disambiguation_rules entry {name!r}: {exc}") from exc + + overlaps: list[CrossTraditionOverlapDef] = [] + for idx, o in enumerate(data.get("cross_tradition_overlaps", [])): + try: + overlaps.append( + CrossTraditionOverlapDef( + name=o["name"], + order=o["order"], + description=o["description"], + trigger_traditions=frozenset(o["trigger_traditions"]), + primary_tradition=o["primary_tradition"], + applied_union_rules=tuple(o["applied_union_rules"]), + overrides_primary=bool(o.get("overrides_primary", False)), + ) + ) + except (KeyError, TypeError) as exc: + name = o.get("name", f"") if isinstance(o, dict) else f"" + raise RuntimeError(f"Invalid cross_tradition_overlaps entry {name!r}: {exc}") from exc + + rules.sort(key=lambda r: r.order) + overlaps.sort(key=lambda o: o.order) + + return rules, overlaps + + +def _rule_matches(rule: DisambiguationRuleDef, candidates: set[str]) -> bool: + if rule.trigger_mode == "anchor_plus_any": + return rule.anchor is not None and rule.anchor in candidates and len(candidates) >= 2 + elif rule.trigger_mode == "all_present": + return rule.trigger_traditions.issubset(candidates) + return False + + +def disambiguate( + candidate_names: set[str], + *, + rules: list[DisambiguationRuleDef] | None = None, + overlaps: list[CrossTraditionOverlapDef] | None = None, + tradition_priority: dict[str, int] | None = None, +) -> DisambiguationResult: + if not candidate_names: + return DisambiguationResult( + primary_tradition=None, + applied_union_rules=(), + precedence_trace="no_candidates", + candidate_set=(), + ) + + if len(candidate_names) == 1: + name = next(iter(candidate_names)) + return DisambiguationResult( + primary_tradition=name, + applied_union_rules=(), + precedence_trace="single_candidate", + candidate_set=(name,), + ) + + if rules is None or overlaps is None: + loaded_rules, loaded_overlaps = load_disambiguation_rules() + if rules is None: + rules = loaded_rules + if overlaps is None: + overlaps = loaded_overlaps + + if tradition_priority is None: + from autoskillit.recipe.methodology_tradition_registry import ( + load_all_methodology_traditions, + ) + + tradition_priority = {s.name: s.priority for s in load_all_methodology_traditions()} + + primary: str | None = None + union_rules: list[str] = [] + trace_parts: list[str] = [] + sorted_candidates = tuple( + sorted(candidate_names, key=lambda n: (tradition_priority.get(n, 999), n)) + ) + + for rule in sorted(rules, key=lambda r: r.order): + if _rule_matches(rule, candidate_names): + primary = rule.primary_tradition + union_rules.extend(rule.applied_union_rules) + trace_parts.append(f"rule_{rule.name}") + for exc in rule.exceptions: + if exc.when_present in candidate_names: + union_rules.extend(exc.add_union_rules) + trace_parts.append(f"exception_{exc.when_present}") + break + + for overlap in sorted(overlaps, key=lambda o: o.order): + if overlap.trigger_traditions.issubset(candidate_names): + union_rules.extend(overlap.applied_union_rules) + trace_parts.append(f"overlap_{overlap.name}") + if primary is None or overlap.overrides_primary: + primary = overlap.primary_tradition + trace_parts.append(f"overlap_primary_{overlap.name}") + + if primary is None: + primary = sorted_candidates[0] + trace_parts.append("fallthrough_priority") + + return DisambiguationResult( + primary_tradition=primary, + applied_union_rules=tuple(union_rules), + precedence_trace="+".join(trace_parts) if trace_parts else "no_match", + candidate_set=sorted_candidates, + ) diff --git a/src/autoskillit/recipe/methodology_tradition_registry.py b/src/autoskillit/recipe/methodology_tradition_registry.py new file mode 100644 index 0000000000..3999d6bf4c --- /dev/null +++ b/src/autoskillit/recipe/methodology_tradition_registry.py @@ -0,0 +1,290 @@ +"""Methodology tradition registry — load bundled and user-defined tradition specs.""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any + +from autoskillit.core import get_logger, load_yaml, pkg_root +from autoskillit.recipe._registry_utils import _MISSING_MTIME, dir_mtime + +logger = get_logger(__name__) + +BUNDLED_METHODOLOGY_TRADITIONS_DIR: Path = pkg_root() / "recipes" / "methodology-traditions" + + +@dataclass(frozen=True, slots=True) +class VenueAppendixDef: + """A venue-specific appendix entry for an ML sub-area within a tradition.""" + + sub_area: str + trigger_keywords: tuple[str, ...] + expectations: tuple[dict[str, str], ...] + + +@dataclass(frozen=True, slots=True) +class MethodologyTraditionSpec: + """Specification for a single methodology tradition.""" + + name: str + display_name: str + canonical_guideline: dict[str, str | int | float | bool] + fields_spanned: list[str] + detection_keywords: list[str] + mandatory_figures: list[dict[str, str]] + strongly_expected_figures: list[dict[str, str]] + anti_patterns: list[dict[str, str]] + schema_version: str = "" + priority: int = 999 + venue_specific_appendices: tuple[VenueAppendixDef, ...] = field(default_factory=tuple) + + +def _parse_int_field(data: dict, field_name: str, default: int, source_path: Path) -> int: + val = data.get(field_name, default) + try: + return int(val) + except (ValueError, TypeError) as e: + name = data.get("name", "?") + raise TypeError( + f"Methodology tradition '{name}' field '{field_name}' must be an integer:" + f" {source_path}" + ) from e + + +def _parse_methodology_tradition(data: dict, source_path: Path) -> MethodologyTraditionSpec: + if "name" not in data: + raise ValueError(f"Methodology tradition YAML missing 'name' field: {source_path}") + + canonical_guideline = data.get("canonical_guideline") + if canonical_guideline is not None and not isinstance(canonical_guideline, dict): + raise TypeError( + f"Methodology tradition '{data['name']}' field 'canonical_guideline' must be a dict, " + f"got {type(canonical_guideline).__name__}: {source_path}" + ) + + for list_field in ( + "fields_spanned", + "detection_keywords", + "mandatory_figures", + "strongly_expected_figures", + "anti_patterns", + ): + val = data.get(list_field) + if val is not None and not isinstance(val, list): + raise TypeError( + f"Methodology tradition '{data['name']}' field '{list_field}' must be a list, " + f"got {type(val).__name__}: {source_path}" + ) + + def _coerce_dict_list(field_name: str, items: list) -> list[dict]: + for i, item in enumerate(items): + if not isinstance(item, dict): + raise TypeError( + f"Methodology tradition '{data['name']}' {field_name}[{i}] must be a" + f" dict, got {type(item).__name__}: {source_path}" + ) + return [dict(item) for item in items] + + return MethodologyTraditionSpec( + name=data["name"], + display_name=str(data.get("display_name", "")), + canonical_guideline=dict(canonical_guideline) if canonical_guideline else {}, + fields_spanned=list(data.get("fields_spanned", [])), + detection_keywords=list(data.get("detection_keywords", [])), + mandatory_figures=_coerce_dict_list( + "mandatory_figures", data.get("mandatory_figures", []) + ), + strongly_expected_figures=_coerce_dict_list( + "strongly_expected_figures", data.get("strongly_expected_figures", []) + ), + anti_patterns=_coerce_dict_list("anti_patterns", data.get("anti_patterns", [])), + schema_version=str(data.get("schema_version", "")), + priority=_parse_int_field(data, "priority", 999, source_path), + venue_specific_appendices=_parse_venue_appendices(data, source_path), + ) + + +def _parse_venue_appendices(data: dict, source_path: Path) -> tuple[VenueAppendixDef, ...]: + raw = data.get("venue_specific_appendices", []) + if not isinstance(raw, list): + raise TypeError( + f"Methodology tradition '{data['name']}' field 'venue_specific_appendices' " + f"must be a list, got {type(raw).__name__}: {source_path}" + ) + result: list[VenueAppendixDef] = [] + for i, item in enumerate(raw): + if not isinstance(item, dict): + raise TypeError( + f"Methodology tradition '{data['name']}' venue_specific_appendices[{i}] " + f"must be a dict, got {type(item).__name__}: {source_path}" + ) + sub_area = item.get("sub_area") + if not isinstance(sub_area, str) or not sub_area: + raise TypeError( + f"Methodology tradition '{data['name']}' venue_specific_appendices[{i}] " + f"'sub_area' must be a non-empty string: {source_path}" + ) + if "trigger_keywords" not in item: + raise TypeError( + f"Methodology tradition '{data['name']}' venue_specific_appendices[{i}] " + f"missing required key 'trigger_keywords': {source_path}" + ) + trigger_keywords = item["trigger_keywords"] + if not isinstance(trigger_keywords, list): + raise TypeError( + f"Methodology tradition '{data['name']}' venue_specific_appendices[{i}] " + f"'trigger_keywords' must be a list, " + f"got {type(trigger_keywords).__name__}: {source_path}" + ) + if "expectations" not in item: + raise TypeError( + f"Methodology tradition '{data['name']}' venue_specific_appendices[{i}] " + f"missing required key 'expectations': {source_path}" + ) + expectations = item["expectations"] + if not isinstance(expectations, list): + raise TypeError( + f"Methodology tradition '{data['name']}' venue_specific_appendices[{i}] " + f"'expectations' must be a list, got {type(expectations).__name__}: {source_path}" + ) + parsed_expectations: list[dict[str, str]] = [] + for j, exp in enumerate(expectations): + if not isinstance(exp, dict): + raise TypeError( + f"Methodology tradition '{data['name']}' venue_specific_appendices[{i}] " + f"expectations[{j}] must be a dict, got {type(exp).__name__}: {source_path}" + ) + fig = exp.get("figure") + src = exp.get("source") + if not isinstance(fig, str) or not fig: + raise TypeError( + f"Methodology tradition '{data['name']}' venue_specific_appendices[{i}] " + f"expectations[{j}] 'figure' must be a non-empty string: {source_path}" + ) + if not isinstance(src, str) or not src: + raise TypeError( + f"Methodology tradition '{data['name']}' venue_specific_appendices[{i}] " + f"expectations[{j}] 'source' must be a non-empty string: {source_path}" + ) + parsed_expectations.append({"figure": fig, "source": src}) + result.append( + VenueAppendixDef( + sub_area=sub_area, + trigger_keywords=tuple(str(kw) for kw in trigger_keywords), + expectations=tuple(parsed_expectations), + ) + ) + return tuple(result) + + +def _load_traditions_from_dir(directory: Path) -> dict[str, MethodologyTraditionSpec]: + if not directory.exists(): + return {} + result: dict[str, MethodologyTraditionSpec] = {} + for path in sorted(directory.glob("*.yaml")): + if path.name.startswith("_"): + continue + try: + data = load_yaml(path) + if isinstance(data, dict): + spec = _parse_methodology_tradition(data, path) + result[spec.name] = spec + else: + logger.warning( + "Skipping methodology tradition file with non-dict top-level structure: %s", + path, + ) + except (ValueError, TypeError, OSError): + logger.warning( + "Skipping malformed methodology tradition file: %s", path, exc_info=True + ) + return result + + +def parse_methodology_tradition( + data: dict[str, Any], source_path: Path +) -> MethodologyTraditionSpec: + return _parse_methodology_tradition(data, source_path) + + +def load_traditions_from_dir(directory: Path) -> dict[str, MethodologyTraditionSpec]: + return _load_traditions_from_dir(directory) + + +_traditions_cache: dict[tuple[str | None, float, float], list[MethodologyTraditionSpec]] = {} + + +def load_all_methodology_traditions( + project_dir: Path | None = None, +) -> list[MethodologyTraditionSpec]: + """Load methodology traditions: bundled traditions merged with user-defined overrides. + + User-defined traditions with the same name as a bundled tradition replace the bundled + tradition entirely — no field merging. User-defined traditions with a new name are added + alongside bundled traditions. + + The returned list is sorted by ``(priority, name)``. + + Args: + project_dir: Project root containing optional user-defined overrides at + ``.autoskillit/methodology-traditions/``. When ``None``, only bundled traditions + are returned. + + Returns: + Sorted list of ``MethodologyTraditionSpec``. + """ + bundled_mt = dir_mtime(BUNDLED_METHODOLOGY_TRADITIONS_DIR) + if project_dir is not None: + user_dir = Path(project_dir) / ".autoskillit" / "methodology-traditions" + user_mt = dir_mtime(user_dir) + else: + user_mt = _MISSING_MTIME + key: tuple[str | None, float, float] = ( + str(project_dir) if project_dir is not None else None, + bundled_mt, + user_mt, + ) + cached = _traditions_cache.get(key) + if cached is not None: + return list(cached) + + traditions = _load_traditions_from_dir(BUNDLED_METHODOLOGY_TRADITIONS_DIR) + + if project_dir is not None: + user_dir = Path(project_dir) / ".autoskillit" / "methodology-traditions" + user_traditions = _load_traditions_from_dir(user_dir) + for spec in user_traditions.values(): + if spec.schema_version and spec.schema_version != "1.0": + logger.warning( + "User methodology tradition has schema_version mismatch; loading continues", + tradition_name=spec.name, + schema_version=spec.schema_version, + expected_schema_version="1.0", + ) + traditions.update(user_traditions) + + sorted_traditions = sorted(traditions.values(), key=lambda s: (s.priority, s.name)) + _traditions_cache[key] = sorted_traditions + return list(sorted_traditions) + + +def get_methodology_tradition_by_name( + name: str, + project_dir: Path | None = None, +) -> MethodologyTraditionSpec | None: + """Look up a single methodology tradition by name. + + Returns the matching spec or None if not found. + """ + by_name = {s.name: s for s in load_all_methodology_traditions(project_dir)} + return by_name.get(name) + + +def is_out_of_scope_tradition(spec: MethodologyTraditionSpec) -> bool: + """Return True when the tradition has no mandatory figures. + + Qualitative traditions do not mandate specific figure types and are therefore + considered out of scope for automated figure requirement checking. + """ + return len(spec.mandatory_figures) == 0 diff --git a/src/autoskillit/recipe/methodology_tradition_router.py b/src/autoskillit/recipe/methodology_tradition_router.py new file mode 100644 index 0000000000..83e440cf00 --- /dev/null +++ b/src/autoskillit/recipe/methodology_tradition_router.py @@ -0,0 +1,135 @@ +"""Two-stage methodology tradition router for Tier-C lens selection.""" + +from __future__ import annotations + +from dataclasses import dataclass +from functools import cache +from pathlib import Path + +import regex as re + +from autoskillit.recipe.methodology_tradition_registry import ( + MethodologyTraditionSpec, + load_all_methodology_traditions, +) + + +@dataclass(frozen=True, slots=True) +class UnionRuleDef: + """Resolves multi-match ambiguity when candidate_set ⊆ member_traditions.""" + + name: str + member_traditions: frozenset[str] + resolved_tradition: str + + def __post_init__(self) -> None: + if self.resolved_tradition not in self.member_traditions: + raise ValueError( + f"UnionRuleDef '{self.name}': resolved_tradition " + f"'{self.resolved_tradition}' must be a member of " + f"member_traditions {self.member_traditions}" + ) + + +@dataclass(frozen=True, slots=True) +class TraditionRouterResult: + """Result from two-stage methodology classification.""" + + primary_tradition: str | None + applied_union_rules: tuple[str, ...] + precedence_trace: str + candidate_set: tuple[str, ...] + + +@cache +def _keyword_pattern(keyword: str) -> re.Pattern[str]: + escaped = re.escape(keyword) + return re.compile(r"(? int: + count = 0 + for kw in spec.detection_keywords: + if _keyword_pattern(kw.lower()).search(text): + count += 1 + return count + + +def _try_union_rules( + candidate_names: set[str], + union_rules: list[UnionRuleDef], +) -> tuple[str | None, tuple[str, ...], str]: + for rule in union_rules: + if candidate_names.issubset(rule.member_traditions): + return ( + rule.resolved_tradition, + (rule.name,), + f"stage2_tiebreak_by_rule_{rule.name}", + ) + return None, (), "" + + +def classify_methodology( + plan_text: str, + project_dir: Path | None = None, + *, + min_keyword_matches: int = 2, + union_rules: list[UnionRuleDef] | None = None, + resolve_by_priority: bool = False, +) -> TraditionRouterResult: + """Stage-1 deterministic keyword classification of plan methodology.""" + if min_keyword_matches < 1: + raise ValueError(f"min_keyword_matches must be >= 1, got {min_keyword_matches}") + traditions = load_all_methodology_traditions(project_dir) + + scored: list[tuple[MethodologyTraditionSpec, int]] = [] + for spec in traditions: + hits = _count_keyword_matches(plan_text, spec) + if hits >= min_keyword_matches: + scored.append((spec, hits)) + + # Lower priority number = higher precedence (priority=1 outranks priority=999) + scored.sort(key=lambda pair: (pair[0].priority, pair[0].name)) + candidate_set = tuple(spec.name for spec, _ in scored) + + if len(candidate_set) == 0: + return TraditionRouterResult( + primary_tradition=None, + applied_union_rules=(), + precedence_trace="stage1_no_match_fallback", + candidate_set=(), + ) + + if len(candidate_set) == 1: + return TraditionRouterResult( + primary_tradition=candidate_set[0], + applied_union_rules=(), + precedence_trace="stage1_single_match", + candidate_set=candidate_set, + ) + + effective_rules = union_rules or [] + candidate_names = set(candidate_set) + resolved, applied, trace = _try_union_rules(candidate_names, effective_rules) + if resolved is not None: + return TraditionRouterResult( + primary_tradition=resolved, + applied_union_rules=applied, + precedence_trace=trace, + candidate_set=candidate_set, + ) + + if resolve_by_priority: + return TraditionRouterResult( + primary_tradition=candidate_set[0], + applied_union_rules=(), + precedence_trace="stage1_multi_match_resolved_by_priority", + candidate_set=candidate_set, + ) + + return TraditionRouterResult( + primary_tradition=None, + applied_union_rules=(), + precedence_trace="stage1_multi_match", + candidate_set=candidate_set, + ) diff --git a/src/autoskillit/recipe/methodology_venue_appendix.py b/src/autoskillit/recipe/methodology_venue_appendix.py new file mode 100644 index 0000000000..b1179bc034 --- /dev/null +++ b/src/autoskillit/recipe/methodology_venue_appendix.py @@ -0,0 +1,193 @@ +"""Stage B venue-appendix resolution — conditional-branching ML sub-area matching.""" + +from __future__ import annotations + +import warnings +from collections.abc import Callable +from dataclasses import dataclass +from functools import cache, lru_cache +from pathlib import Path + +import regex as re + +from autoskillit.core import load_yaml +from autoskillit.recipe.methodology_tradition_registry import ( + BUNDLED_METHODOLOGY_TRADITIONS_DIR, + VenueAppendixDef, + load_all_methodology_traditions, +) + + +@dataclass(frozen=True, slots=True) +class AlternateParentDef: + parent: str + trigger_keywords: tuple[str, ...] + constraint: str | None = None + + +@dataclass(frozen=True, slots=True) +class MLSubAreaFoldingDef: + sub_area: str + display_name: str + primary_parent: str + alternate_parents: tuple[AlternateParentDef, ...] + + +@dataclass(frozen=True, slots=True) +class VenueAppendixMatch: + sub_area: str + resolved_parent: str + appendix: VenueAppendixDef + re_routed: bool + + +_CONSTRAINT_EVALUATORS: dict[str, Callable[[str], bool]] = { + "only_if_explicit_construct_measurement": lambda text: bool( + re.search( + r"(? re.Pattern[str]: + return _keyword_pattern_cached(keyword.lower()) + + +@cache +def _keyword_pattern_cached(keyword: str) -> re.Pattern[str]: + escaped = re.escape(keyword) + return re.compile(r"(? bool: + return any(_keyword_pattern(kw).search(text) for kw in keywords) + + +@lru_cache(maxsize=1) +def load_ml_sub_area_folding() -> tuple[MLSubAreaFoldingDef, ...]: + yaml_path = BUNDLED_METHODOLOGY_TRADITIONS_DIR / "_ml_sub_area_folding.yaml" + data = load_yaml(yaml_path) + if not isinstance(data, dict): + raise TypeError(f"Expected dict from {yaml_path}, got {type(data).__name__}") + entries: list[MLSubAreaFoldingDef] = [] + for i, item in enumerate(data.get("ml_sub_area_folding", [])): + if not isinstance(item, dict): + raise TypeError( + f"ml_sub_area_folding[{i}] must be a dict, got {type(item).__name__}: {yaml_path}" + ) + for key in ("sub_area", "display_name", "primary_parent"): + if not isinstance(item.get(key), str) or not item[key]: + raise TypeError( + f"ml_sub_area_folding[{i}] '{key}' must be a non-empty string: {yaml_path}" + ) + alternates_raw = item.get("alternate_parents", []) + if not isinstance(alternates_raw, list): + raise TypeError( + f"ml_sub_area_folding[{i}] 'alternate_parents' must be a list, " + f"got {type(alternates_raw).__name__}: {yaml_path}" + ) + alternates: list[AlternateParentDef] = [] + for j, a in enumerate(alternates_raw): + if not isinstance(a, dict): + raise TypeError( + f"ml_sub_area_folding[{i}] alternate_parents[{j}] must be a dict, " + f"got {type(a).__name__}: {yaml_path}" + ) + if not isinstance(a.get("parent"), str) or not a["parent"]: + raise TypeError( + f"ml_sub_area_folding[{i}] alternate_parents[{j}] 'parent' " + f"must be a non-empty string: {yaml_path}" + ) + if not isinstance(a.get("trigger_keywords"), list): + raise TypeError( + f"ml_sub_area_folding[{i}] alternate_parents[{j}] 'trigger_keywords' " + f"must be a list: {yaml_path}" + ) + constraint = a.get("constraint") + if constraint is not None and constraint not in _CONSTRAINT_EVALUATORS: + raise ValueError( + f"ml_sub_area_folding[{i}] alternate_parents[{j}] 'constraint' " + f"'{constraint}' is not a recognised evaluator key: {yaml_path}" + ) + alternates.append( + AlternateParentDef( + parent=a["parent"], + trigger_keywords=tuple(a["trigger_keywords"]), + constraint=constraint, + ) + ) + entries.append( + MLSubAreaFoldingDef( + sub_area=item["sub_area"], + display_name=item["display_name"], + primary_parent=item["primary_parent"], + alternate_parents=tuple(alternates), + ) + ) + return tuple(entries) + + +def _resolve_conditional_parent( + entry: MLSubAreaFoldingDef, + plan_text: str, +) -> tuple[str, bool]: + """Return (resolved_parent, re_routed).""" + for alt in entry.alternate_parents: + if _has_keyword_match(plan_text, alt.trigger_keywords): + if alt.constraint is not None: + evaluator = _CONSTRAINT_EVALUATORS.get(alt.constraint) + if evaluator is None: + warnings.warn( + f"_resolve_conditional_parent: unrecognised constraint " + f"'{alt.constraint}' on sub-area '{entry.sub_area}' — skipping alternate", + stacklevel=2, + ) + continue + if not evaluator(plan_text): + continue + return alt.parent, True + return entry.primary_parent, False + + +def resolve_venue_appendices( + plan_text: str, + project_dir: Path | None = None, +) -> list[VenueAppendixMatch]: + """Two-stage venue appendix resolution. + + 1. Detect ML sub-areas from plan text using folding map keywords + 2. For each detected sub-area, resolve conditional parent + 3. Collect venue appendices from resolved parent tradition + """ + + folding = load_ml_sub_area_folding() + traditions = {s.name: s for s in load_all_methodology_traditions(project_dir)} + + matches: list[VenueAppendixMatch] = [] + for entry in folding: + parent, re_routed = _resolve_conditional_parent(entry, plan_text) + spec = traditions.get(parent) + if spec is None: + warnings.warn( + f"resolve_venue_appendices: tradition '{parent}' referenced by sub-area " + f"'{entry.sub_area}' not found — skipping", + stacklevel=2, + ) + continue + for appendix in spec.venue_specific_appendices: + if appendix.sub_area == entry.sub_area: + if _has_keyword_match(plan_text, appendix.trigger_keywords): + matches.append( + VenueAppendixMatch( + sub_area=entry.sub_area, + resolved_parent=parent, + appendix=appendix, + re_routed=re_routed, + ) + ) + break + return matches diff --git a/src/autoskillit/recipe/order.py b/src/autoskillit/recipe/order.py new file mode 100644 index 0000000000..f8861a4995 --- /dev/null +++ b/src/autoskillit/recipe/order.py @@ -0,0 +1,18 @@ +"""Stable display order registry for bundled recipes. + +Controls the display position of bundled recipes within their group (Group 0 = Bundled +Recipes). Recipes listed here appear first in registry order; discovered recipes whose +names are absent sort alphabetically after the last registered entry. + +To add a new recipe at a specific position, insert its name here. To leave it at the +bottom, add the YAML without touching this file. +""" + +from __future__ import annotations + +BUNDLED_RECIPE_ORDER: list[str] = [ + "implementation", + "remediation", + "implementation-groups", + "merge-prs", +] diff --git a/src/autoskillit/recipe/registry.py b/src/autoskillit/recipe/registry.py index e1b273e62b..845c36ce73 100644 --- a/src/autoskillit/recipe/registry.py +++ b/src/autoskillit/recipe/registry.py @@ -37,9 +37,9 @@ def to_dict(self) -> dict[str, str]: } -@dataclass -class RuleSpec: - """Internal: metadata for one registered rule.""" +@dataclass(frozen=True, slots=True) +class RuleDef: + """Static definition of a registered semantic rule.""" name: str description: str @@ -47,10 +47,10 @@ class RuleSpec: check: Callable[[ValidationContext], list[RuleFinding]] -_RULE_REGISTRY: list[RuleSpec] = [] +_RULE_REGISTRY: list[RuleDef] = [] -@dataclass(frozen=True) +@dataclass(frozen=True, slots=True) class BlockContext: """Per-block dispatch context for block-level semantic rules. @@ -63,9 +63,9 @@ class BlockContext: parent: ValidationContext -@dataclass -class BlockRuleSpec: - """Internal: metadata for one registered block-level rule.""" +@dataclass(frozen=True, slots=True) +class BlockRuleDef: + """Static definition of a registered block-level semantic rule.""" name: str description: str @@ -73,7 +73,7 @@ class BlockRuleSpec: check: Callable[[BlockContext], list[RuleFinding]] -_BLOCK_RULE_REGISTRY: list[BlockRuleSpec] = [] +_BLOCK_RULE_REGISTRY: list[BlockRuleDef] = [] def semantic_rule( @@ -87,7 +87,7 @@ def decorator( fn: Callable[[ValidationContext], list[RuleFinding]], ) -> Callable[[ValidationContext], list[RuleFinding]]: _RULE_REGISTRY.append( - RuleSpec(name=name, description=description, severity=severity, check=fn) + RuleDef(name=name, description=description, severity=severity, check=fn) ) return fn @@ -110,7 +110,7 @@ def decorator( fn: Callable[[BlockContext], list[RuleFinding]], ) -> Callable[[BlockContext], list[RuleFinding]]: _BLOCK_RULE_REGISTRY.append( - BlockRuleSpec(name=name, description=description, severity=severity, check=fn) + BlockRuleDef(name=name, description=description, severity=severity, check=fn) ) return fn diff --git a/src/autoskillit/recipe/repository.py b/src/autoskillit/recipe/repository.py index a72fa3d9f0..2d55e32d1b 100644 --- a/src/autoskillit/recipe/repository.py +++ b/src/autoskillit/recipe/repository.py @@ -2,12 +2,25 @@ from __future__ import annotations +import time from collections.abc import Sequence +from datetime import UTC, datetime from pathlib import Path -from typing import Any, cast +from typing import TYPE_CHECKING, Any, cast + +if TYPE_CHECKING: + from autoskillit.core import LoadResult + from autoskillit.recipe.schema import Recipe, RecipeInfo import autoskillit.recipe._api as _api -from autoskillit.recipe.io import builtin_recipes_dir, list_recipes +from autoskillit.recipe.contracts import StaleItem, load_bundled_manifest +from autoskillit.recipe.io import builtin_recipes_dir, list_recipes, load_recipe +from autoskillit.recipe.staleness_cache import ( + StalenessEntry, + compute_recipe_hash, + read_staleness_cache, + write_staleness_cache, +) def _dir_mtime(path: Path) -> float: @@ -22,12 +35,12 @@ class DefaultRecipeRepository: """Concrete RecipeRepository backed by list_recipes with in-memory mtime cache.""" def __init__(self) -> None: - self._cached_list: Any | None = None + self._cached_list: LoadResult[RecipeInfo] | None = None self._cached_project_dir: Path | None = None self._cached_project_mtime: float = 0.0 self._cached_builtin_mtime: float = 0.0 - def _get_list(self, project_dir: Path) -> Any: + def _get_list(self, project_dir: Path) -> LoadResult[RecipeInfo]: pm = _dir_mtime(project_dir / ".autoskillit" / "recipes") bm = _dir_mtime(builtin_recipes_dir()) if ( @@ -44,17 +57,27 @@ def _get_list(self, project_dir: Path) -> Any: self._cached_builtin_mtime = bm return result - def find(self, name: str, project_dir: Path) -> Any: + def find(self, name: str, project_dir: Path) -> RecipeInfo | None: result = self._get_list(project_dir) return next((r for r in result.items if r.name == name), None) - def list(self, project_dir: Path) -> Any: + def load(self, path: Path) -> Recipe: + """Load and parse the recipe at *path*. + + Exceptions raised by :func:`load_recipe` (e.g. ``FileNotFoundError``, + ``yaml.YAMLError``, ``ValidationError``) are intentionally not wrapped — + thin delegation is the contract. Callers that need a uniform error type + should catch at the call site. + """ + return load_recipe(path) + + def list(self, project_dir: Path) -> LoadResult[RecipeInfo]: return self._get_list(project_dir) def load_and_validate( self, name: str, - project_dir: Any, + project_dir: Path | str, *, suppressed: Sequence[str] | None = None, resolved_defaults: dict[str, str] | None = None, @@ -62,7 +85,9 @@ def load_and_validate( temp_dir: Path | None = None, temp_dir_relpath: str | None = None, ) -> dict[str, Any]: - recipe_info = self.find(name, project_dir) + project_dir = Path(project_dir) + result = self._get_list(project_dir) + recipe_info = next((r for r in result.items if r.name == name), None) return cast( dict[str, Any], _api.load_and_validate( @@ -70,6 +95,7 @@ def load_and_validate( project_dir=project_dir, suppressed=suppressed, recipe_info=recipe_info, + recipe_list=result.items, resolved_defaults=resolved_defaults, ingredient_overrides=ingredient_overrides, temp_dir=temp_dir, @@ -82,5 +108,91 @@ def validate_from_path( ) -> dict[str, Any]: return _api.validate_from_path(script_path, temp_dir_relpath=temp_dir_relpath) - def list_all(self, project_dir: Any | None = None) -> dict[str, Any]: - return _api.list_all(project_dir=project_dir) + def list_all( + self, + project_dir: Any | None = None, + *, + features: dict[str, bool] | None = None, + ) -> dict[str, Any]: + return _api.list_all(project_dir=project_dir, features=features) + + async def apply_triage_gate( + self, + result: dict[str, Any], + recipe_name: str, + recipe_info: Any, + temp_dir: Path, + logger: Any, + triage_fn: Any = None, + ) -> dict[str, Any]: + """Apply LLM triage to stale-contract suggestions, suppressing cosmetic ones.""" + stale_suggs = [ + s for s in result.get("suggestions", []) if s.get("rule") == "stale-contract" + ] + if not stale_suggs: + return result + + if recipe_info is None: + recipe_info = self.find(recipe_name, Path.cwd()) + if recipe_info is None: + return result + + cache_path = temp_dir / "recipe_staleness_cache.json" + t0 = time.perf_counter() + cached = read_staleness_cache(cache_path, recipe_name) + logger.debug( + "triage_gate_cache_read", + recipe=recipe_name, + elapsed_ms=round((time.perf_counter() - t0) * 1000, 1), + ) + + if cached is not None and cached.triage_result == "cosmetic": + result["suggestions"] = [ + s for s in result["suggestions"] if s.get("rule") != "stale-contract" + ] + return result + + if cached is None or cached.triage_result is None: + hash_items = [ + StaleItem( + skill=s["skill"], + reason=s["reason"], + stored_value=s.get("stored_value", ""), + current_value=s.get("current_value", ""), + ) + for s in stale_suggs + if s.get("reason") == "hash_mismatch" + ] + if hash_items and triage_fn is not None: + t_llm = time.perf_counter() + triage = await triage_fn(hash_items) + logger.debug( + "triage_gate_llm_triage", + recipe=recipe_name, + elapsed_ms=round((time.perf_counter() - t_llm) * 1000, 1), + ) + if not triage: + return result + all_cosmetic = all(not r.get("meaningful", True) for r in triage) + triage_str = "cosmetic" if all_cosmetic else "meaningful" + current_hash = compute_recipe_hash(recipe_info.path) + current_ver = load_bundled_manifest().get("version", "") + write_staleness_cache( + cache_path, + recipe_name, + StalenessEntry( + recipe_hash=current_hash, + manifest_version=current_ver, + is_stale=True, + triage_result=triage_str, + checked_at=datetime.now(UTC).isoformat(), + ), + ) + if all_cosmetic and not any( + s.get("reason") == "version_mismatch" for s in stale_suggs + ): + result["suggestions"] = [ + s for s in result["suggestions"] if s.get("rule") != "stale-contract" + ] + + return result diff --git a/src/autoskillit/recipe/rules/CLAUDE.md b/src/autoskillit/recipe/rules/CLAUDE.md new file mode 100644 index 0000000000..e0dec10e70 --- /dev/null +++ b/src/autoskillit/recipe/rules/CLAUDE.md @@ -0,0 +1,41 @@ +# rules/ + +Semantic validation rule modules for recipe analysis (29 rule files). + +## Files + +| File | Purpose | +|------|---------| +| `__init__.py` | Docstring-only — rules register via `@semantic_rule` decorator on import | +| `rules_actions.py` | Semantic rules for `stop`/`route`/`confirm` action-type steps | +| `rules_blocks.py` | Block-level budget rules; loads `block_budgets.yaml` at import | +| `rules_bypass.py` | Rules for `skip_when_false` bypass routing contracts | +| `rules_callable_scope.py` | Enforces scoped directory args for file-discovering callables (e.g. `batch_create_issues` → `audit_run_dir`) | +| `rules_campaign.py` | Campaign recipe validation: dispatch names, ingredient refs | +| `rules_ci.py` | CI polling patterns: PR state handling, CI step ordering | +| `rules_clone.py` | Clone/push dataflow rules: missing remote URL, local-strategy capture | +| `rules_cmd.py` | `run_cmd` echo-capture alignment; git remote command detection | +| `rules_contracts.py` | Skill contract completeness rules | +| `rules_dataflow.py` | Dataflow analysis: pipeline-forbidden tool usage, output chaining | +| `rules_features.py` | Feature-gated tool/skill reference validation | +| `rules_food_truck.py` | Food-truck recipe validation: sentinel stop step requirement | +| `rules_fixing.py` | Conditional-write skill must gate on declared verdict output | +| `rules_graph.py` | Graph/routing analysis rules | +| `rules_inline_script.py` | Detects inline shell scripts in `run_cmd` cmd fields | +| `rules_inputs.py` | Input/ingredient validation; version compatibility checks | +| `rules_isolation.py` | Workspace isolation rules (prevents operating on source repo) | +| `rules_merge.py` | `merge_worktree` routing completeness | +| `rules_merge_queue.py` | Merge queue push routing: `queued_branch` error route enforcement | +| `rules_packs.py` | Pack validation (names must exist in `PACK_REGISTRY`) | +| `rules_reachability.py` | Symbolic BFS reachability; capture-inversion detection | +| `rules_recipe.py` | Sub-recipe reference validity and `with_args` hygiene | +| `rules_skill_content.py` | Undefined bash placeholder detection in SKILL.md | +| `rules_skills.py` | `skill_command` resolvability rules | +| `rules_temp_path.py` | Rejects bare `{{AUTOSKILLIT_TEMP}}/` without scope prefix | +| `rules_tools.py` | MCP tool name validity (must be in known tool sets) | +| `rules_verdict.py` | Skill verdict routing completeness and cross-step consistency | +| `rules_worktree.py` | Worktree and retry validation rules | + +## Architecture Notes + +Side-effect registration: callers import the package to trigger `@semantic_rule` decorator registration of all 29 rule modules. Each rule receives a `ValidationContext` argument. No cross-imports between rule modules. diff --git a/src/autoskillit/recipe/rules/__init__.py b/src/autoskillit/recipe/rules/__init__.py new file mode 100644 index 0000000000..f8eb6dd365 --- /dev/null +++ b/src/autoskillit/recipe/rules/__init__.py @@ -0,0 +1 @@ +"""Recipe semantic rule modules — imported for side-effect registration.""" diff --git a/src/autoskillit/recipe/rules/rules_actions.py b/src/autoskillit/recipe/rules/rules_actions.py new file mode 100644 index 0000000000..f1381fb6fe --- /dev/null +++ b/src/autoskillit/recipe/rules/rules_actions.py @@ -0,0 +1,127 @@ +"""Semantic validation rules for action-type steps (stop, route, confirm).""" + +from __future__ import annotations + +from autoskillit.core import Severity, get_logger +from autoskillit.recipe._analysis import ValidationContext +from autoskillit.recipe.registry import RuleFinding, semantic_rule +from autoskillit.recipe.schema import RecipeKind + +logger = get_logger(__name__) + +_PLACEHOLDER_MESSAGES: frozenset[str] = frozenset( + {"done", "ok", "complete", "finished", "end", "stop", "exit"} +) + + +@semantic_rule( + name="stop-step-has-no-routing", + description="Stop steps must not have outbound routing fields.", + severity=Severity.ERROR, +) +def _check_stop_step_has_no_routing(ctx: ValidationContext) -> list[RuleFinding]: + findings: list[RuleFinding] = [] + for step_name, step in ctx.recipe.steps.items(): + if step.action != "stop": + continue + has_routing = ( + step.on_success is not None + or step.on_failure is not None + or step.on_context_limit is not None + or step.on_result is not None + or step.on_exhausted + != "escalate" # "escalate" is RecipeStep's dataclass default (schema.py) + ) + if has_routing: + findings.append( + RuleFinding( + rule="stop-step-has-no-routing", + severity=Severity.ERROR, + step_name=step_name, + message=( + f"Stop step '{step_name}' has outbound routing fields. " + "Stop steps are terminal — routing fields are contradictory " + "and indicate author confusion." + ), + ) + ) + return findings + + +@semantic_rule( + name="stop-step-message-quality", + description="Stop step messages must be meaningful (≥10 characters and not a placeholder).", + severity=Severity.WARNING, +) +def _check_stop_step_message_quality(ctx: ValidationContext) -> list[RuleFinding]: + findings: list[RuleFinding] = [] + for step_name, step in ctx.recipe.steps.items(): + if step.action != "stop": + continue + msg = (step.message or "").strip() + # 10 chars: short enough to pass "Done." equivalents that name the outcome, + # long enough to reject single-word placeholders caught by _PLACEHOLDER_MESSAGES. + if len(msg) < 10 or msg.lower() in _PLACEHOLDER_MESSAGES: + findings.append( + RuleFinding( + rule="stop-step-message-quality", + severity=Severity.WARNING, + step_name=step_name, + message=( + f"Stop step '{step_name}' has a weak message: {msg!r}. " + "Use a meaningful description (≥10 characters) to give the " + "model a clear terminus signal." + ), + ) + ) + return findings + + +@semantic_rule( + name="recipe-has-terminal-step", + description="Every recipe must have at least one stop step.", + severity=Severity.ERROR, +) +def _check_recipe_has_terminal_step(ctx: ValidationContext) -> list[RuleFinding]: + if ctx.recipe.kind == RecipeKind.CAMPAIGN: + return [] + has_stop = any(step.action == "stop" for step in ctx.recipe.steps.values()) + if not has_stop: + return [ + RuleFinding( + rule="recipe-has-terminal-step", + severity=Severity.ERROR, + step_name="", + message=( + "Recipe has no stop step. Every recipe must have at least one " + "terminal node — without one, the pipeline has no defined end state." + ), + ) + ] + return [] + + +@semantic_rule( + name="route-step-requires-on-result", + description="Route steps must have an on_result block to serve their purpose.", + severity=Severity.WARNING, +) +def _check_route_step_requires_on_result(ctx: ValidationContext) -> list[RuleFinding]: + findings: list[RuleFinding] = [] + for step_name, step in ctx.recipe.steps.items(): + if step.action != "route": + continue + if step.on_result is None: + findings.append( + RuleFinding( + rule="route-step-requires-on-result", + severity=Severity.WARNING, + step_name=step_name, + message=( + f"Route step '{step_name}' has no on_result block. " + "Route steps exist to make conditional routing decisions; " + "without on_result, they serve no purpose." + ), + ) + ) + return findings diff --git a/src/autoskillit/recipe/rules_blocks.py b/src/autoskillit/recipe/rules/rules_blocks.py similarity index 100% rename from src/autoskillit/recipe/rules_blocks.py rename to src/autoskillit/recipe/rules/rules_blocks.py diff --git a/src/autoskillit/recipe/rules_bypass.py b/src/autoskillit/recipe/rules/rules_bypass.py similarity index 100% rename from src/autoskillit/recipe/rules_bypass.py rename to src/autoskillit/recipe/rules/rules_bypass.py diff --git a/src/autoskillit/recipe/rules/rules_callable_scope.py b/src/autoskillit/recipe/rules/rules_callable_scope.py new file mode 100644 index 0000000000..43fe5577b4 --- /dev/null +++ b/src/autoskillit/recipe/rules/rules_callable_scope.py @@ -0,0 +1,55 @@ +"""Semantic rules for run_python callable scoping requirements.""" + +from __future__ import annotations + +from autoskillit.core import Severity +from autoskillit.recipe._analysis import ValidationContext +from autoskillit.recipe.registry import RuleFinding, semantic_rule + +# Callables that discover files via glob and require a scoped directory argument. +SCOPED_CALLABLES: dict[str, str] = { + "batch_create_issues": "audit_run_dir", +} + + +@semantic_rule( + name="callable-requires-scoped-discovery", + description=( + "run_python steps calling file-discovering callables must pass " + "a scoped directory argument (audit_run_dir or similar) to prevent " + "cross-run file accumulation." + ), + severity=Severity.ERROR, +) +def _check_callable_scoped_discovery(ctx: ValidationContext) -> list[RuleFinding]: + """Error when a file-discovering callable is called without its required scoping parameter. + + batch_create_issues uses an unscoped glob on the flat validate-audit/ directory. + Without audit_run_dir, it cannot distinguish current-run files from prior-run files, + causing duplicate issue creation on re-runs. + + Other callables in SCOPED_CALLABLES follow the same pattern. + """ + findings: list[RuleFinding] = [] + for step_name, step in ctx.recipe.steps.items(): + if step.tool != "run_python": + continue + callable_val = str(step.with_args.get("callable", "")) + callable_leaf = callable_val.rsplit(".", 1)[-1] + for callable_name, required_key in SCOPED_CALLABLES.items(): + if callable_leaf == callable_name: + if required_key not in step.with_args: + findings.append( + RuleFinding( + rule="callable-requires-scoped-discovery", + severity=Severity.ERROR, + step_name=step_name, + message=( + f"step '{step_name}': {callable_name} call is " + f"missing '{required_key}' in with args. Without a " + f"scoped directory, the callable will glob all files " + f"including those from prior runs." + ), + ) + ) + return findings diff --git a/src/autoskillit/recipe/rules/rules_campaign.py b/src/autoskillit/recipe/rules/rules_campaign.py new file mode 100644 index 0000000000..356208b17f --- /dev/null +++ b/src/autoskillit/recipe/rules/rules_campaign.py @@ -0,0 +1,735 @@ +"""Semantic validation rules for campaign recipes.""" + +from __future__ import annotations + +from collections import Counter +from pathlib import Path +from typing import TYPE_CHECKING + +import regex as re + +from autoskillit.core import RECIPE_PACK_REGISTRY, DispatchGateType, Severity, get_logger +from autoskillit.recipe._analysis import ValidationContext +from autoskillit.recipe.registry import RuleFinding, semantic_rule +from autoskillit.recipe.schema import CAMPAIGN_REF_RE, CampaignDispatch, RecipeKind + +if TYPE_CHECKING: + from autoskillit.recipe.schema import Recipe + +logger = get_logger(__name__) + +_KEBAB_RE = re.compile(r"^[a-z0-9]+(-[a-z0-9]+)*$") + +# _run_dispatch() auto-injects these from dispatch-level fields (e.g. task: from the +# dispatch task: field), so campaigns that rely on this injection pattern should not +# be flagged for not explicitly forwarding them in the ingredients block. +_AUTO_INJECTED_CAMPAIGN_INGREDIENTS: frozenset[str] = frozenset({"task"}) + + +def _load_dispatch_target(dispatch: CampaignDispatch, project_dir: Path | None) -> Recipe | None: + """Load the target recipe for a dispatch. Returns None if not loadable.""" + if project_dir is None: + return None + try: + from autoskillit.recipe.io import find_recipe_by_name, load_recipe # noqa: PLC0415 + + info = find_recipe_by_name(dispatch.recipe, project_dir) + if info is None: + return None + return load_recipe(info.path) + except Exception: + logger.warning("dispatch_target_load_failed", recipe=dispatch.recipe, exc_info=True) + return None + + +@semantic_rule( + name="campaign-kind-is-campaign", + description="Recipe with dispatches must declare kind: campaign", + severity=Severity.ERROR, +) +def _check_campaign_kind_is_campaign(ctx: ValidationContext) -> list[RuleFinding]: + if not ctx.recipe.dispatches: + return [] + if ctx.recipe.kind == RecipeKind.CAMPAIGN: + return [] + return [ + RuleFinding( + rule="campaign-kind-is-campaign", + severity=Severity.ERROR, + step_name="(top-level)", + message=( + "Recipe has dispatches but kind is not 'campaign'. " + "Set 'kind: campaign' in the recipe header." + ), + ) + ] + + +@semantic_rule( + name="campaign-has-dispatches", + description="Campaign recipe must have at least one dispatch", + severity=Severity.ERROR, +) +def _check_campaign_has_dispatches(ctx: ValidationContext) -> list[RuleFinding]: + if ctx.recipe.kind != RecipeKind.CAMPAIGN: + return [] + if ctx.recipe.dispatches: + return [] + return [ + RuleFinding( + rule="campaign-has-dispatches", + severity=Severity.ERROR, + step_name="(top-level)", + message="Campaign recipe must have at least one dispatch in 'dispatches'.", + ) + ] + + +@semantic_rule( + name="dispatch-names-unique", + description="Dispatch names within a campaign must be unique", + severity=Severity.ERROR, +) +def _check_dispatch_names_unique(ctx: ValidationContext) -> list[RuleFinding]: + if ctx.recipe.kind != RecipeKind.CAMPAIGN: + return [] + counts = Counter(d.name for d in ctx.recipe.dispatches) + findings: list[RuleFinding] = [] + for name, count in counts.items(): + if count > 1: + findings.append( + RuleFinding( + rule="dispatch-names-unique", + severity=Severity.ERROR, + step_name="(top-level)", + message=f"Dispatch name {name!r} appears {count} times; names must be unique.", + ) + ) + return findings + + +@semantic_rule( + name="dispatch-names-kebab-case", + description="Dispatch names should use kebab-case", + severity=Severity.WARNING, +) +def _check_dispatch_names_kebab_case(ctx: ValidationContext) -> list[RuleFinding]: + if ctx.recipe.kind != RecipeKind.CAMPAIGN: + return [] + findings: list[RuleFinding] = [] + for d in ctx.recipe.dispatches: + if not _KEBAB_RE.match(d.name): + findings.append( + RuleFinding( + rule="dispatch-names-kebab-case", + severity=Severity.WARNING, + step_name="(top-level)", + message=( + f"Dispatch name {d.name!r} is not kebab-case. " + "Use lowercase letters, digits, and hyphens only." + ), + ) + ) + return findings + + +@semantic_rule( + name="dispatch-recipe-exists", + description="Each dispatch must reference a known recipe name", + severity=Severity.ERROR, +) +def _check_dispatch_recipe_exists(ctx: ValidationContext) -> list[RuleFinding]: + if ctx.recipe.kind != RecipeKind.CAMPAIGN: + return [] + if not ctx.available_recipes: + return [] + findings: list[RuleFinding] = [] + for d in ctx.recipe.dispatches: + if d.gate: + continue + if d.recipe not in ctx.available_recipes: + findings.append( + RuleFinding( + rule="dispatch-recipe-exists", + severity=Severity.ERROR, + step_name="(top-level)", + message=( + f"Dispatch {d.name!r} references recipe {d.recipe!r} " + "which is not in the known recipe set." + ), + ) + ) + return findings + + +@semantic_rule( + name="dispatch-recipe-is-standard", + description="Campaign dispatches must not target other campaign recipes", + severity=Severity.ERROR, +) +def _check_dispatch_recipe_is_standard(ctx: ValidationContext) -> list[RuleFinding]: + if ctx.recipe.kind != RecipeKind.CAMPAIGN: + return [] + findings: list[RuleFinding] = [] + for d in ctx.recipe.dispatches: + if d.gate: + continue + target = _load_dispatch_target(d, ctx.project_dir) + if target is None: + continue + if target.kind == RecipeKind.CAMPAIGN: + findings.append( + RuleFinding( + rule="dispatch-recipe-is-standard", + severity=Severity.ERROR, + step_name="(top-level)", + message=( + f"Dispatch {d.name!r} targets recipe {d.recipe!r} which is itself a " + "campaign recipe. Campaign nesting is not supported." + ), + ) + ) + return findings + + +@semantic_rule( + name="dispatch-recipe-in-declared-packs", + description="Dispatch target recipes should belong to the campaign's declared packs", + severity=Severity.WARNING, +) +def _check_dispatch_recipe_in_declared_packs(ctx: ValidationContext) -> list[RuleFinding]: + if ctx.recipe.kind != RecipeKind.CAMPAIGN: + return [] + if not ctx.recipe.requires_recipe_packs: + return [] + findings: list[RuleFinding] = [] + for d in ctx.recipe.dispatches: + if d.gate: + continue + if d.recipe in ctx.recipe.allowed_recipes: + continue + target = _load_dispatch_target(d, ctx.project_dir) + if target is None: + continue + if not (set(target.categories) & set(ctx.recipe.requires_recipe_packs)): + findings.append( + RuleFinding( + rule="dispatch-recipe-in-declared-packs", + severity=Severity.WARNING, + step_name="(top-level)", + message=( + f"Dispatch {d.name!r} targets recipe {d.recipe!r} whose categories " + f"{target.categories!r} do not overlap with the campaign's declared " + f"packs {ctx.recipe.requires_recipe_packs!r}." + ), + ) + ) + return findings + + +@semantic_rule( + name="campaign-requires-recipe-packs-exist", + description="Pack names in requires_recipe_packs must be in RECIPE_PACK_REGISTRY", + severity=Severity.WARNING, +) +def _check_campaign_requires_recipe_packs_exist(ctx: ValidationContext) -> list[RuleFinding]: + if ctx.recipe.kind != RecipeKind.CAMPAIGN: + return [] + findings: list[RuleFinding] = [] + seen: set[str] = set() + for pack_name in ctx.recipe.requires_recipe_packs: + if pack_name not in RECIPE_PACK_REGISTRY and pack_name not in seen: + seen.add(pack_name) + findings.append( + RuleFinding( + rule="campaign-requires-recipe-packs-exist", + severity=Severity.WARNING, + step_name="(top-level)", + message=( + f"Pack {pack_name!r} in requires_recipe_packs is not in " + f"RECIPE_PACK_REGISTRY. Known packs: {sorted(RECIPE_PACK_REGISTRY)}" + ), + ) + ) + return findings + + +@semantic_rule( + name="dispatch-ingredients-keys-in-target-schema", + description="Dispatch ingredient keys must exist in the target recipe's ingredients", + severity=Severity.ERROR, +) +def _check_dispatch_ingredients_keys_in_target_schema(ctx: ValidationContext) -> list[RuleFinding]: + if ctx.recipe.kind != RecipeKind.CAMPAIGN: + return [] + findings: list[RuleFinding] = [] + for d in ctx.recipe.dispatches: + if d.gate: + continue + if not d.ingredients: + continue + target = _load_dispatch_target(d, ctx.project_dir) + if target is None: + continue + for key in d.ingredients: + if key not in target.ingredients: + findings.append( + RuleFinding( + rule="dispatch-ingredients-keys-in-target-schema", + severity=Severity.ERROR, + step_name="(top-level)", + message=( + f"Dispatch {d.name!r} passes ingredient {key!r} to recipe " + f"{d.recipe!r}, but that recipe does not declare ingredient {key!r}. " + f"Known ingredients: {sorted(target.ingredients)}" + ), + ) + ) + return findings + + +@semantic_rule( + name="campaign-dangling-ingredient", + description=( + "Campaign ingredients should be forwarded to dispatches whose target recipe declares them" + ), + severity=Severity.WARNING, +) +def _check_campaign_dangling_ingredient(ctx: ValidationContext) -> list[RuleFinding]: + if ctx.recipe.kind != RecipeKind.CAMPAIGN: + return [] + campaign_ingredients = set(ctx.recipe.ingredients.keys()) + if not campaign_ingredients: + return [] + findings: list[RuleFinding] = [] + for d in ctx.recipe.dispatches: + # Gated dispatches are conditional — they may not run at all, so requiring + # them to forward every campaign ingredient would produce false positives. + if d.gate: + continue + target = _load_dispatch_target(d, ctx.project_dir) + if target is None: + continue + forwarded_keys = set(d.ingredients.keys()) + for ing_name in campaign_ingredients: + if ing_name in _AUTO_INJECTED_CAMPAIGN_INGREDIENTS: + continue + if ing_name in target.ingredients and ing_name not in forwarded_keys: + findings.append( + RuleFinding( + rule="campaign-dangling-ingredient", + severity=Severity.WARNING, + step_name="(top-level)", + message=( + f"Campaign ingredient {ing_name!r} is declared by target " + f"recipe {d.recipe!r} (dispatch {d.name!r}) but is not " + f"forwarded in the dispatch's ingredients block. The sub-recipe " + f"will use its own default instead of the campaign-level value." + ), + ) + ) + return findings + + +@semantic_rule( + name="dispatch-ingredient-values-are-strings", + description="All dispatch ingredient values must be strings", + severity=Severity.ERROR, +) +def _check_dispatch_ingredient_values_are_strings(ctx: ValidationContext) -> list[RuleFinding]: + if ctx.recipe.kind != RecipeKind.CAMPAIGN: + return [] + findings: list[RuleFinding] = [] + for d in ctx.recipe.dispatches: + for key, val in d.ingredients.items(): + if not isinstance(val, str): + findings.append( + RuleFinding( + rule="dispatch-ingredient-values-are-strings", + severity=Severity.ERROR, + step_name="(top-level)", + message=( + f"Dispatch {d.name!r} ingredient {key!r} has non-string value " + f"{val!r} ({type(val).__name__}). YAML auto-coercion detected — " + "quote the value in YAML." + ), + ) + ) + return findings + + +@semantic_rule( + name="depends-on-refers-to-valid-dispatches", + description="depends_on entries must reference known dispatch names", + severity=Severity.ERROR, +) +def _check_depends_on_refers_to_valid_dispatches(ctx: ValidationContext) -> list[RuleFinding]: + if ctx.recipe.kind != RecipeKind.CAMPAIGN: + return [] + all_names = {d.name for d in ctx.recipe.dispatches} + findings: list[RuleFinding] = [] + for d in ctx.recipe.dispatches: + for dep in d.depends_on: + if dep not in all_names: + findings.append( + RuleFinding( + rule="depends-on-refers-to-valid-dispatches", + severity=Severity.ERROR, + step_name="(top-level)", + message=( + f"Dispatch {d.name!r} depends_on {dep!r} which is not a known " + f"dispatch name. Known names: {sorted(all_names)}" + ), + ) + ) + return findings + + +@semantic_rule( + name="depends-on-acyclic", + description="Dispatch depends_on graph must be acyclic", + severity=Severity.ERROR, +) +def _check_depends_on_acyclic(ctx: ValidationContext) -> list[RuleFinding]: + if ctx.recipe.kind != RecipeKind.CAMPAIGN: + return [] + adjacency: dict[str, list[str]] = {d.name: list(d.depends_on) for d in ctx.recipe.dispatches} + visited: set[str] = set() + in_stack: set[str] = set() + findings: list[RuleFinding] = [] + + def dfs(node: str, path: list[str]) -> None: + visited.add(node) + in_stack.add(node) + for neighbor in adjacency.get(node, []): + if neighbor not in adjacency: + continue + if neighbor not in visited: + dfs(neighbor, path + [neighbor]) + elif neighbor in in_stack: + cycle_start = path.index(neighbor) if neighbor in path else 0 + cycle = path[cycle_start:] + [neighbor] + findings.append( + RuleFinding( + rule="depends-on-acyclic", + severity=Severity.ERROR, + step_name="(top-level)", + message=( + f"Circular dependency detected in dispatches: {' → '.join(cycle)}" + ), + ) + ) + in_stack.discard(node) + + for name in list(adjacency): + if name not in visited: + dfs(name, [name]) + + return findings + + +@semantic_rule( + name="campaign-dispatch-depends-on-is-sequential", + description="Each dispatch's depends_on must have at most one entry (linear chain only)", + severity=Severity.ERROR, +) +def _check_campaign_dispatch_depends_on_is_sequential(ctx: ValidationContext) -> list[RuleFinding]: + if ctx.recipe.kind != RecipeKind.CAMPAIGN: + return [] + findings: list[RuleFinding] = [] + for d in ctx.recipe.dispatches: + if len(d.depends_on) > 1: + findings.append( + RuleFinding( + rule="campaign-dispatch-depends-on-is-sequential", + severity=Severity.ERROR, + step_name="(top-level)", + message=( + f"Dispatch {d.name!r} has {len(d.depends_on)} entries in depends_on " + f"({d.depends_on!r}). Campaign dispatches must form a linear chain: " + "each dispatch may depend on at most one predecessor." + ), + ) + ) + return findings + + +@semantic_rule( + name="campaign-task-non-empty", + description="Each dispatch must have a non-empty task description", + severity=Severity.ERROR, +) +def _check_campaign_task_non_empty(ctx: ValidationContext) -> list[RuleFinding]: + if ctx.recipe.kind != RecipeKind.CAMPAIGN: + return [] + findings: list[RuleFinding] = [] + for d in ctx.recipe.dispatches: + if d.gate: + continue + if not d.task.strip(): + findings.append( + RuleFinding( + rule="campaign-task-non-empty", + severity=Severity.ERROR, + step_name="(top-level)", + message=f"Dispatch {d.name!r} has an empty 'task' field.", + ) + ) + return findings + + +_IDENT_RE = re.compile(r"^[a-zA-Z_][a-zA-Z0-9_]*$") +_RESULT_TEMPLATE_RE = re.compile(r"^\$\{\{\s*result\.[\w-]+\s*\}\}$") + + +@semantic_rule( + name="dispatch-capture-keys-are-identifiers", + description="Capture keys must be valid Python identifiers", + severity=Severity.ERROR, +) +def _check_dispatch_capture_keys_are_identifiers(ctx: ValidationContext) -> list[RuleFinding]: + if ctx.recipe.kind != RecipeKind.CAMPAIGN: + return [] + findings = [] + for d in ctx.recipe.dispatches: + for key in d.capture: + if not _IDENT_RE.match(key): + findings.append( + RuleFinding( + rule="dispatch-capture-keys-are-identifiers", + severity=Severity.ERROR, + step_name="(top-level)", + message=( + f"Dispatch {d.name!r} capture key {key!r} is not a valid" + " identifier. Use only letters, digits, and underscores" + " (must start with letter or _)." + ), + ) + ) + return findings + + +@semantic_rule( + name="dispatch-capture-value-references-result", + description="Capture values must use ${{ result.field }} syntax", + severity=Severity.ERROR, +) +def _check_dispatch_capture_value_references_result(ctx: ValidationContext) -> list[RuleFinding]: + if ctx.recipe.kind != RecipeKind.CAMPAIGN: + return [] + findings = [] + for d in ctx.recipe.dispatches: + for key, val in d.capture.items(): + if not _RESULT_TEMPLATE_RE.match(val.strip()): + findings.append( + RuleFinding( + rule="dispatch-capture-value-references-result", + severity=Severity.ERROR, + step_name="(top-level)", + message=( + f"Dispatch {d.name!r} capture[{key!r}] value {val!r} must use " + "${{ result. }} syntax." + ), + ) + ) + return findings + + +def _build_ancestors(name: str, adjacency: dict[str, list[str]]) -> set[str]: + """Transitive closure of depends_on for a given dispatch name.""" + ancestors: set[str] = set() + queue = list(adjacency.get(name, [])) + while queue: + dep = queue.pop() + if dep not in ancestors: + ancestors.add(dep) + queue.extend(adjacency.get(dep, [])) + return ancestors + + +@semantic_rule( + name="campaign-ingredient-refs-have-prior-capture", + description="${{ campaign.key }} in ingredients must be captured by an ancestor dispatch", + severity=Severity.ERROR, +) +def _check_campaign_ingredient_refs_have_prior_capture( + ctx: ValidationContext, +) -> list[RuleFinding]: + if ctx.recipe.kind != RecipeKind.CAMPAIGN: + return [] + adjacency = {d.name: list(d.depends_on) for d in ctx.recipe.dispatches} + dispatch_by_name = {d.name: d for d in ctx.recipe.dispatches} + findings = [] + for d in ctx.recipe.dispatches: + ancestors = _build_ancestors(d.name, adjacency) + available_captures: set[str] = set() + for ancestor_name in ancestors: + ancestor = dispatch_by_name.get(ancestor_name) + if ancestor: + available_captures.update(ancestor.capture.keys()) + for ing_key, ing_val in d.ingredients.items(): + if not isinstance(ing_val, str): + continue + for ref in CAMPAIGN_REF_RE.findall(ing_val): + if ref not in available_captures: + findings.append( + RuleFinding( + rule="campaign-ingredient-refs-have-prior-capture", + severity=Severity.ERROR, + step_name="(top-level)", + message=( + f"Dispatch {d.name!r} ingredient {ing_key!r} references " + f"${{{{ campaign.{ref} }}}} but no ancestor dispatch " + f"(via depends_on) captures {ref!r}. " + f"Available captured keys: {sorted(available_captures)}" + ), + ) + ) + return findings + + +@semantic_rule( + name="autoskillit-version-compatible", + description="Campaign recipe version requirement must be satisfied by installed version", + severity=Severity.WARNING, +) +def _check_autoskillit_version_compatible(ctx: ValidationContext) -> list[RuleFinding]: + if ctx.recipe.kind != RecipeKind.CAMPAIGN: + return [] + if not ctx.recipe.version: + return [] + try: + from importlib.metadata import version # noqa: PLC0415 + + from packaging.version import Version # noqa: PLC0415 + + installed = Version(version("autoskillit")) + required = Version(ctx.recipe.version) + if required > installed: + return [ + RuleFinding( + rule="autoskillit-version-compatible", + severity=Severity.WARNING, + step_name="(top-level)", + message=( + f"Campaign requires autoskillit>={ctx.recipe.version} " + f"but installed version is {installed}." + ), + ) + ] + except Exception: + logger.warning("autoskillit_version_check_failed", exc_info=True) + return [] + + +_VALID_GATE_TYPES: frozenset[DispatchGateType] = frozenset({DispatchGateType.CONFIRM}) + + +@semantic_rule( + name="gate-dispatch-valid-type", + description="gate value must be 'confirm'", + severity=Severity.ERROR, +) +def _check_gate_dispatch_valid_type(ctx: ValidationContext) -> list[RuleFinding]: + if ctx.recipe.kind != RecipeKind.CAMPAIGN: + return [] + findings: list[RuleFinding] = [] + for d in ctx.recipe.dispatches: + if d.gate is None: + continue + if d.gate not in _VALID_GATE_TYPES: + findings.append( + RuleFinding( + rule="gate-dispatch-valid-type", + severity=Severity.ERROR, + step_name="(top-level)", + message=( + f"Dispatch {d.name!r} has gate={d.gate!r} which is not a valid " + f"gate type. Supported types: {sorted(_VALID_GATE_TYPES)}" + ), + ) + ) + return findings + + +@semantic_rule( + name="gate-dispatch-has-message", + description="A gate dispatch must have a non-empty message", + severity=Severity.ERROR, +) +def _check_gate_dispatch_has_message(ctx: ValidationContext) -> list[RuleFinding]: + if ctx.recipe.kind != RecipeKind.CAMPAIGN: + return [] + findings: list[RuleFinding] = [] + for d in ctx.recipe.dispatches: + if d.gate is None: + continue + if not (d.message or "").strip(): + findings.append( + RuleFinding( + rule="gate-dispatch-has-message", + severity=Severity.ERROR, + step_name="(top-level)", + message=( + f"Dispatch {d.name!r} has gate={d.gate!r} but 'message' is empty. " + "A non-empty message is required for gate dispatches." + ), + ) + ) + return findings + + +@semantic_rule( + name="gate-dispatch-no-recipe", + description="A gate dispatch must not specify recipe or task", + severity=Severity.ERROR, +) +def _check_gate_dispatch_no_recipe(ctx: ValidationContext) -> list[RuleFinding]: + if ctx.recipe.kind != RecipeKind.CAMPAIGN: + return [] + findings: list[RuleFinding] = [] + for d in ctx.recipe.dispatches: + if d.gate is None: + continue + if d.recipe or d.task: + findings.append( + RuleFinding( + rule="gate-dispatch-no-recipe", + severity=Severity.ERROR, + step_name="(top-level)", + message=( + f"Dispatch {d.name!r} has gate={d.gate!r} but also specifies " + f"recipe={d.recipe!r} or task={d.task!r}. " + "Gate dispatches must not specify recipe or task." + ), + ) + ) + return findings + + +@semantic_rule( + name="gate-dispatch-no-capture", + description="A gate dispatch must not specify capture", + severity=Severity.ERROR, +) +def _check_gate_dispatch_no_capture(ctx: ValidationContext) -> list[RuleFinding]: + if ctx.recipe.kind != RecipeKind.CAMPAIGN: + return [] + findings: list[RuleFinding] = [] + for d in ctx.recipe.dispatches: + if d.gate is None: + continue + if d.capture: + findings.append( + RuleFinding( + rule="gate-dispatch-no-capture", + severity=Severity.ERROR, + step_name="(top-level)", + message=( + f"Dispatch {d.name!r} has gate={d.gate!r} but also specifies " + f"capture={d.capture!r}. Gate dispatches produce no L3 session " + "output and must not specify capture." + ), + ) + ) + return findings diff --git a/src/autoskillit/recipe/rules_ci.py b/src/autoskillit/recipe/rules/rules_ci.py similarity index 50% rename from src/autoskillit/recipe/rules_ci.py rename to src/autoskillit/recipe/rules/rules_ci.py index d5d12e82dd..b54a14d12a 100644 --- a/src/autoskillit/recipe/rules_ci.py +++ b/src/autoskillit/recipe/rules/rules_ci.py @@ -2,12 +2,15 @@ from __future__ import annotations -import re +import regex as re from autoskillit.core import PRState, Severity from autoskillit.recipe._analysis import ValidationContext from autoskillit.recipe.registry import RuleFinding, semantic_rule +_NO_RUNS_RE = re.compile(r"""==\s*['"]?no_runs['"]?""") +_TIMED_OUT_RE = re.compile(r"""==\s*['"]?timed_out['"]?""") + @semantic_rule( name="ci-polling-inline-shell", @@ -145,13 +148,96 @@ def _check_ci_failure_conflict_gate(ctx: ValidationContext) -> list[RuleFinding] return findings +@semantic_rule( + name="ci-no-runs-unguarded", + description=( + "Flags wait_for_ci steps that use bare on_success routing without " + "on_result conditions that intercept conclusion='no_runs'" + ), + severity=Severity.ERROR, +) +def _check_ci_no_runs_unguarded(ctx: ValidationContext) -> list[RuleFinding]: + findings: list[RuleFinding] = [] + for name, step in ctx.recipe.steps.items(): + if step.tool != "wait_for_ci": + continue + has_no_runs_guard = False + if step.on_result and step.on_result.conditions: + has_explicit_no_runs = any( + c.when and _NO_RUNS_RE.search(c.when) for c in step.on_result.conditions + ) + has_catch_all = any(not c.when for c in step.on_result.conditions) + has_no_runs_guard = has_explicit_no_runs or has_catch_all + if has_no_runs_guard: + continue + if step.on_success or step.on_result: + target = step.on_success or "on_result routing" + findings.append( + RuleFinding( + rule="ci-no-runs-unguarded", + severity=Severity.ERROR, + step_name=name, + message=( + f"Step '{name}' uses wait_for_ci without an on_result condition " + "that intercepts conclusion='no_runs'. wait_for_ci returns " + "conclusion='no_runs' on the success path — add on_result " + f"conditions to intercept no_runs before routing to '{target}'." + ), + ) + ) + return findings + + +@semantic_rule( + name="ci-timed-out-unguarded", + description=( + "Flags wait_for_ci steps whose on_result routing lacks an explicit " + "condition for conclusion='timed_out'. A catch-all arm is NOT " + "sufficient — timed_out means CI is still running, not failed." + ), + severity=Severity.ERROR, +) +def _check_ci_timed_out_unguarded(ctx: ValidationContext) -> list[RuleFinding]: + findings: list[RuleFinding] = [] + for name, step in ctx.recipe.steps.items(): + if step.tool != "wait_for_ci": + continue + if not (step.on_success or step.on_result): + continue + has_explicit_timed_out = False + if step.on_result and step.on_result.conditions: + has_explicit_timed_out = any( + c.when and _TIMED_OUT_RE.search(c.when) for c in step.on_result.conditions + ) + if has_explicit_timed_out: + continue + target = step.on_success or "on_result routing" + findings.append( + RuleFinding( + rule="ci-timed-out-unguarded", + severity=Severity.ERROR, + step_name=name, + message=( + f"Step '{name}' uses wait_for_ci without an on_result condition " + "that intercepts conclusion='timed_out'. timed_out means CI is " + "still in progress — routing it through a catch-all to the CI " + f"failure path is semantically wrong. Add an explicit timed_out " + f"arm before '{target}'." + ), + ) + ) + return findings + + _CI_EVENT_SCOPE_TOOLS = {"wait_for_ci", "get_ci_status"} @semantic_rule( name="ci-missing-event-scope", - description="CI tool step without event parameter risks cross-event confusion", - severity=Severity.WARNING, + description=( + "CI tool step without event parameter causes silent run exclusion on feature branches" + ), + severity=Severity.ERROR, ) def _check_ci_missing_event_scope(ctx: ValidationContext) -> list[RuleFinding]: findings: list[RuleFinding] = [] @@ -162,13 +248,15 @@ def _check_ci_missing_event_scope(ctx: ValidationContext) -> list[RuleFinding]: findings.append( RuleFinding( rule="ci-missing-event-scope", - severity=Severity.WARNING, + severity=Severity.ERROR, step_name=step_name, message=( f"Step '{step_name}' calls {step.tool} without an 'event' parameter. " - f"Without event filtering, a passing pull_request run can mask a " - f"failing push run. Add event: 'push' (or the appropriate trigger " - f"event) to the step's with_args, or set ci.event in project config." + f"On feature branches excluded from push triggers, the push-scoped " + f"filter returns no runs even when pull_request CI is active, causing " + f"no_runs timeout. Add event: '${{{{ context.ci_event }}}}' " + f"(requires check_repo_ci_event to run first) " + f"or set ci.event in project config." ), ) ) @@ -207,9 +295,11 @@ def _check_ci_hardcoded_workflow(ctx: ValidationContext) -> list[RuleFinding]: # wait_for_merge_queue routing rules (I7 + I8) # --------------------------------------------------------------------------- -_REQUIRED_MQ_PR_STATES: frozenset[str] = frozenset(s.value for s in PRState if s != PRState.ERROR) +_REQUIRED_MQ_PR_STATES: frozenset[str] = frozenset( + s.value for s in PRState if s not in {PRState.ERROR, PRState.NOT_ENROLLED} +) _PR_STATE_WHEN_RE = re.compile(r"\$\{\{\s*result\.pr_state\s*\}\}\s*==\s*(\w+)") -_MQ_EXPECTED_FALLBACK = "release_issue_timeout" +_MQ_EXPECTED_FALLBACK = "register_clone_unconfirmed" def _extract_mq_when_values(on_result: object) -> set[str]: @@ -224,12 +314,23 @@ def _extract_mq_when_values(on_result: object) -> set[str]: return values -def _recipe_uses_release_issue_timeout(ctx: ValidationContext) -> bool: - """Return True if this recipe family uses release_issue_timeout as its error escalation step. +def _recipe_has_mq_routing_step(ctx: ValidationContext) -> bool: + """Return True if any step uses wait_for_merge_queue with predicate on_result routing.""" + return any( + step.tool == "wait_for_merge_queue" + and step.on_result is not None + and getattr(step.on_result, "conditions", None) + for step in ctx.recipe.steps.values() + ) - The two new wait_for_merge_queue rules only apply to implementation/remediation-family - recipes that define release_issue_timeout. merge-prs.yaml and other recipe families - route queue timeouts/errors differently (e.g. register_clone_failure) and are exempt. + +def _recipe_uses_register_clone_unconfirmed(ctx: ValidationContext) -> bool: + """Return True if this recipe family uses register_clone_unconfirmed as timeout escalation. + + Used only by Rule I8 (conformance targets) — implementation/remediation-family recipes + that define register_clone_unconfirmed must route fallback and on_failure there. Other + recipe families (e.g. merge-prs.yaml) route queue timeouts/errors differently and are + exempt from target-specific conformance checks, but NOT from PRState completeness (I7). """ return _MQ_EXPECTED_FALLBACK in ctx.recipe.steps @@ -245,7 +346,7 @@ def _recipe_uses_release_issue_timeout(ctx: ValidationContext) -> bool: def _check_wait_for_merge_queue_routing_covers_all_pr_states( ctx: ValidationContext, ) -> list[RuleFinding]: - if not _recipe_uses_release_issue_timeout(ctx): + if not _recipe_has_mq_routing_step(ctx): return [] findings: list[RuleFinding] = [] for step_name, step in ctx.recipe.steps.items(): @@ -275,14 +376,14 @@ def _check_wait_for_merge_queue_routing_covers_all_pr_states( name="wait-for-merge-queue-routing-conforms-to-expected-targets", description=( "wait_for_merge_queue fallback and on_failure must both target " - "release_issue_timeout; prevents silent success routing on timeout/unknown states" + "register_clone_unconfirmed; prevents silent success routing on timeout/unknown states" ), severity=Severity.ERROR, ) def _check_wait_for_merge_queue_routing_conforms_to_expected_targets( ctx: ValidationContext, ) -> list[RuleFinding]: - if not _recipe_uses_release_issue_timeout(ctx): + if not _recipe_uses_register_clone_unconfirmed(ctx): return [] findings: list[RuleFinding] = [] for step_name, step in ctx.recipe.steps.items(): @@ -303,7 +404,7 @@ def _check_wait_for_merge_queue_routing_conforms_to_expected_targets( message=( f"Step {step_name!r} has fallback route {route!r} but " f"expected {_MQ_EXPECTED_FALLBACK!r}. The fallback must " - f"route to release_issue_timeout so unrecognised states " + f"route to register_clone_unconfirmed so unrecognised states " f"are escalated, not silently treated as success." ), ) @@ -318,8 +419,176 @@ def _check_wait_for_merge_queue_routing_conforms_to_expected_targets( message=( f"Step {step_name!r} has on_failure={step.on_failure!r} but " f"expected {_MQ_EXPECTED_FALLBACK!r}. Tool errors must route " - f"to release_issue_timeout, not a success-path step." + f"to register_clone_unconfirmed, not a success-path step." + ), + ) + ) + return findings + + +@semantic_rule( + name="ci-event-literal-merge-group", + description=( + "Flags wait_for_ci steps that hardcode event='merge_group' — use context.ci_event instead" + ), + severity=Severity.ERROR, +) +def _check_ci_event_literal_merge_group(ctx: ValidationContext) -> list[RuleFinding]: + """Flag wait_for_ci steps that hardcode event='merge_group'.""" + findings: list[RuleFinding] = [] + for name, step in ctx.recipe.steps.items(): + if step.tool != "wait_for_ci": + continue + event_value = (step.with_args or {}).get("event") + if event_value == "merge_group": + findings.append( + RuleFinding( + rule="ci-event-literal-merge-group", + severity=Severity.ERROR, + step_name=name, + message=( + "wait_for_ci must not hardcode event='merge_group'. " + "Use context.ci_event (which is 'push' or null) for pre-queue CI, " + "or add a lifecycle-aware condition for in-queue CI." + ), + ) + ) + return findings + + +_CI_TIMEOUT_MINIMUM = 600 + + +@semantic_rule( + name="ci-timeout-minimum", + description="Flags wait_for_ci steps with timeout_seconds below 600", + severity=Severity.WARNING, +) +def _check_ci_timeout_minimum(ctx: ValidationContext) -> list[RuleFinding]: + findings: list[RuleFinding] = [] + for name, step in ctx.recipe.steps.items(): + if step.tool != "wait_for_ci": + continue + raw = (step.with_args or {}).get("timeout_seconds") + if raw is None: + continue + try: + timeout = int(raw) + except (ValueError, TypeError): + continue + if timeout < _CI_TIMEOUT_MINIMUM: + findings.append( + RuleFinding( + rule="ci-timeout-minimum", + severity=Severity.WARNING, + step_name=name, + message=( + f"timeout_seconds={timeout} is below the {_CI_TIMEOUT_MINIMUM}s " + f"minimum — average CI duration is ~317s with peaks to 392s." + ), + ) + ) + return findings + + +@semantic_rule( + name="ci-timed-out-self-loop-unguarded", + description=( + "A wait_for_ci step whose on_result routes timed_out back to itself " + "must have a check_loop_iteration guard on that path to prevent unbounded looping. " + "timed_out means CI is still running — it should be polled with a bounded iteration " + "count, not routed to a bare self-loop with no cap." + ), + severity=Severity.ERROR, +) +def _check_ci_timed_out_self_loop_unguarded(ctx: ValidationContext) -> list[RuleFinding]: + findings: list[RuleFinding] = [] + for step_name, step in ctx.recipe.steps.items(): + if step.tool != "wait_for_ci": + continue + if not step.on_result or not step.on_result.conditions: + continue + conditions = step.on_result.conditions + timed_out_routes: list[str] = [] + for cond in conditions: + if cond.when is not None and "timed_out" in cond.when: + timed_out_routes.append(cond.route) + for route in timed_out_routes: + if route == step_name: + findings.append( + RuleFinding( + rule="ci-timed-out-self-loop-unguarded", + severity=Severity.ERROR, + step_name=step_name, + message=( + f"Step '{step_name}' has a timed_out self-loop with no " + f"check_loop_iteration guard on the path. " + f"Unbounded polling can result from this pattern." + ), + ) + ) + return findings + + +@semantic_rule( + name="ci-conflict-path-missing-auto-trigger", + description=( + "A wait_for_ci step on a conflict-resolution path must have auto_trigger: true. " + "After a force-push of a conflict fix, CI may not have been triggered yet — " + "auto_trigger ensures the empty-commit CI trigger fires on the next poll cycle." + ), + severity=Severity.ERROR, +) +def _check_ci_conflict_path_missing_auto_trigger(ctx: ValidationContext) -> list[RuleFinding]: + findings: list[RuleFinding] = [] + conflict_trigger_steps: set[str] = set() + for step_name, step in ctx.recipe.steps.items(): + if not step.on_result or not step.on_result.conditions: + continue + for cond in step.on_result.conditions: + if cond.when is not None and "CONFLICTING" in cond.when: + conflict_trigger_steps.add(step_name) + break + for step_name, step in ctx.recipe.steps.items(): + if step.tool != "wait_for_ci": + continue + is_on_conflict_path = ( + "conflict" in step_name.lower() + or step_name in conflict_trigger_steps + or _has_conflict_ancestor(step_name, conflict_trigger_steps, ctx) + ) + if not is_on_conflict_path: + continue + auto_trigger = (step.with_args or {}).get("auto_trigger", "") + if str(auto_trigger).lower() != "true": + findings.append( + RuleFinding( + rule="ci-conflict-path-missing-auto-trigger", + severity=Severity.ERROR, + step_name=step_name, + message=( + f"Step '{step_name}' is on a conflict-resolution path but lacks " + f"auto_trigger: 'true' in with_args. After a conflict fix push, " + f"CI may not have fired yet — auto_trigger is mandatory here." ), ) ) return findings + + +def _has_conflict_ancestor( + step_name: str, conflict_steps: set[str], ctx: ValidationContext +) -> bool: + from collections import deque + + visited: set[str] = set() + queue = deque(ctx.predecessors.get(step_name, set())) + while queue: + current = queue.popleft() + if current in visited: + continue + visited.add(current) + if current in conflict_steps: + return True + queue.extend(ctx.predecessors.get(current, set())) + return False diff --git a/src/autoskillit/recipe/rules_clone.py b/src/autoskillit/recipe/rules/rules_clone.py similarity index 99% rename from src/autoskillit/recipe/rules_clone.py rename to src/autoskillit/recipe/rules/rules_clone.py index 50e8228fc8..03e8beae30 100644 --- a/src/autoskillit/recipe/rules_clone.py +++ b/src/autoskillit/recipe/rules/rules_clone.py @@ -13,7 +13,7 @@ from __future__ import annotations -import re +import regex as re from autoskillit.core import ( SKILL_COMMAND_PREFIX, diff --git a/src/autoskillit/recipe/rules_cmd.py b/src/autoskillit/recipe/rules/rules_cmd.py similarity index 70% rename from src/autoskillit/recipe/rules_cmd.py rename to src/autoskillit/recipe/rules/rules_cmd.py index fd58645fba..a209f65fbd 100644 --- a/src/autoskillit/recipe/rules_cmd.py +++ b/src/autoskillit/recipe/rules/rules_cmd.py @@ -2,10 +2,11 @@ from __future__ import annotations -import re +import regex as re from autoskillit.core import Severity from autoskillit.recipe._analysis import ValidationContext +from autoskillit.recipe._git_helpers import _GIT_REMOTE_COMMAND_RE, _LITERAL_ORIGIN_RE from autoskillit.recipe.contracts import RESULT_CAPTURE_RE from autoskillit.recipe.registry import RuleFinding, semantic_rule @@ -34,6 +35,8 @@ def _check_run_cmd_emit_alignment(ctx: ValidationContext) -> list[RuleFinding]: cmd = (step.with_args or {}).get("cmd", "") if not isinstance(cmd, str): continue + if re.match(r"^\s*bash\s+scripts/", cmd): + continue for cap_key, cap_val in step.capture.items(): m = RESULT_CAPTURE_RE.search(cap_val) if m is None: @@ -92,3 +95,39 @@ def _check_run_cmd_find_rediscovery(ctx: ValidationContext) -> list[RuleFinding] ) ) return findings + + +@semantic_rule( + name="hardcoded-origin-in-run-cmd", + description="run_cmd step uses hardcoded 'origin' remote name", + severity=Severity.WARNING, +) +def _check_hardcoded_origin_in_run_cmd(ctx: ValidationContext) -> list[RuleFinding]: + findings: list[RuleFinding] = [] + for name, step in ctx.recipe.steps.items(): + if step.tool != "run_cmd": + continue + cmd = (step.with_args or {}).get("cmd", "") + if not isinstance(cmd, str): + continue + if "git remote set-url origin" in cmd: + continue + for line in cmd.splitlines(): + stripped = line.strip() + if not stripped or stripped.startswith("#"): + continue + if _GIT_REMOTE_COMMAND_RE.search(stripped) and _LITERAL_ORIGIN_RE.search(stripped): + findings.append( + RuleFinding( + rule="hardcoded-origin-in-run-cmd", + severity=Severity.WARNING, + step_name=name, + message=( + f"Step '{name}' uses hardcoded 'origin' in a git command. " + "In clone-isolated pipelines, origin is file://. " + "Use: REMOTE=$(git remote get-url upstream >/dev/null 2>&1 " + "&& echo upstream || echo origin)" + ), + ) + ) + return findings diff --git a/src/autoskillit/recipe/rules_contracts.py b/src/autoskillit/recipe/rules/rules_contracts.py similarity index 50% rename from src/autoskillit/recipe/rules_contracts.py rename to src/autoskillit/recipe/rules/rules_contracts.py index d1eb8c9aa8..b8b36c8554 100644 --- a/src/autoskillit/recipe/rules_contracts.py +++ b/src/autoskillit/recipe/rules/rules_contracts.py @@ -2,7 +2,7 @@ from __future__ import annotations -import re as _re +import regex as re from autoskillit.core import Severity, get_logger, pkg_root from autoskillit.recipe._analysis import ValidationContext @@ -13,6 +13,19 @@ ) from autoskillit.recipe.registry import RuleFinding, semantic_rule +# Must match _INTENTIONALLY_EXCLUDED_PATH_TOKENS in _headless_path_tokens.py. +# Cannot import directly: recipe (IL-2) cannot depend on execution (IL-1). +# test_rules_contracts.py::test_excluded_tokens_match_intentionally_excluded guards drift. +_PATH_RECOVERY_EXCLUDED_TOKENS: frozenset[str] = frozenset({"worktree_path", "branch_name"}) + +# Skill names covered by the result-field-drift rule. +_RESULT_FIELD_DRIFT_SKILLS = frozenset( + { + "planner-generate-phases", + "planner-elaborate-phase", + } +) + logger = get_logger(__name__) @@ -33,9 +46,6 @@ def _check_missing_output_patterns(ctx: ValidationContext) -> list[RuleFinding]: continue skill_cmd = step.with_args.get("skill_command", "") - if "${{" in skill_cmd: - continue - name = resolve_skill_name(skill_cmd) if not name: continue @@ -78,8 +88,6 @@ def _check_pattern_examples_match(ctx: ValidationContext) -> list[RuleFinding]: if step.tool != "run_skill": continue skill_cmd = step.with_args.get("skill_command", "") - if "${{" in skill_cmd: - continue name = resolve_skill_name(skill_cmd) if not name: continue @@ -88,8 +96,8 @@ def _check_pattern_examples_match(ctx: ValidationContext) -> list[RuleFinding]: continue for pattern in contract.expected_output_patterns: try: - matched = any(_re.search(pattern, ex) for ex in contract.pattern_examples) - except _re.error: + matched = any(re.search(pattern, ex) for ex in contract.pattern_examples) + except re.error: findings.append( RuleFinding( rule="pattern-examples-match", @@ -131,8 +139,6 @@ def _check_missing_pattern_examples(ctx: ValidationContext) -> list[RuleFinding] if step.tool != "run_skill": continue skill_cmd = step.with_args.get("skill_command", "") - if "${{" in skill_cmd: - continue name = resolve_skill_name(skill_cmd) if not name: continue @@ -219,8 +225,8 @@ def _check_write_behavior_consistency(ctx: ValidationContext) -> list[RuleFindin ) for pattern in wew: try: - _re.compile(pattern) - except _re.error as exc: + re.compile(pattern) + except re.error as exc: findings.append( RuleFinding( rule="write-behavior-consistency", @@ -248,7 +254,7 @@ def _check_write_behavior_consistency(ctx: ValidationContext) -> list[RuleFindin "already green", "graceful degradation", # resolve-review: "graceful degradation — do not fail" r"\bskip\b.{0,30}\bstep\b", # conditional path language - "exit 0", # graceful early exit + r"(?:and|,)\s*exit\s+0", # graceful early exit suffix (e.g. "and exit 0", ", exit 0") r"\bif no\b.{0,30}\bfound\b", r"if.*unavailable", "already complete", @@ -280,9 +286,6 @@ def _check_always_has_no_write_exit(ctx: ValidationContext) -> list[RuleFinding] continue skill_cmd = step.with_args.get("skill_command", "") - if "${{" in skill_cmd: - continue - name = resolve_skill_name(skill_cmd) if not name: continue @@ -302,9 +305,20 @@ def _check_always_has_no_write_exit(ctx: ValidationContext) -> list[RuleFinding] "could not read %s; skipping always-has-no-write-exit check", skill_md, ) + findings.append( + RuleFinding( + rule="always-has-no-write-exit", + severity=Severity.WARNING, + step_name=step_name, + message=( + f"Skill '{name}' SKILL.md at {skill_md} could not be read; " + f"always-has-no-write-exit check was skipped for this step." + ), + ) + ) break for phrase in _ALWAYS_WITH_NO_WRITE_EXIT_PHRASES: - if _re.search(phrase, content): + if re.search(phrase, content): findings.append( RuleFinding( rule="always-has-no-write-exit", @@ -324,3 +338,260 @@ def _check_always_has_no_write_exit(ctx: ValidationContext) -> list[RuleFinding] break return findings + + +@semantic_rule( + name="result-field-drift", + description=( + "result_fields declared in skill_contracts.yaml must match the TypedDict required keys " + "in planner/schema.py for planner skills that produce structured result files." + ), + severity=Severity.ERROR, +) +def _check_result_field_drift(ctx: ValidationContext) -> list[RuleFinding]: + """Error when a skill's declared result_fields diverge from the canonical TypedDict keys. + + Covers planner-generate-phases, elaborate-phase, elaborate-assignment, and elaborate-wp. + Comparison is one-directional: contract required fields vs TypedDict required keys. + """ + # Deferred import: recipe/ must not import planner/ at module level (REQ-COMP-009). + # planner/ is IL-1 and does not import recipe/, so no circular risk. + from autoskillit.planner import ( # noqa: PLC0415 + PHASE_REQUIRED_KEYS, + ) + + skill_schemas: dict[str, frozenset[str]] = { + "planner-generate-phases": PHASE_REQUIRED_KEYS, + "planner-elaborate-phase": PHASE_REQUIRED_KEYS, + } + + findings: list[RuleFinding] = [] + manifest = load_bundled_manifest() + + for step_name, step in ctx.recipe.steps.items(): + if step.tool != "run_skill": + continue + + skill_cmd = step.with_args.get("skill_command", "") + name = resolve_skill_name(skill_cmd) + if not name or name not in skill_schemas: + continue + + contract = get_skill_contract(name, manifest) + if contract is None or not contract.result_fields: + continue + + expected_keys = skill_schemas[name] + declared_required = {rf.name for rf in contract.result_fields if rf.required} + + added = declared_required - expected_keys + removed = expected_keys - declared_required + + if added or removed: + parts: list[str] = [] + if added: + parts.append(f"extra in contract: {sorted(added)}") + if removed: + parts.append(f"missing from contract: {sorted(removed)}") + findings.append( + RuleFinding( + rule="result-field-drift", + severity=Severity.ERROR, + step_name=step_name, + message=( + f"Skill '{name}' result_fields in skill_contracts.yaml diverge from " + f"the TypedDict required keys in planner/schema.py — {'; '.join(parts)}. " + f"Update result_fields to match the TypedDict definition." + ), + ) + ) + + return findings + + +@semantic_rule( + name="example-covers-all-allowed-values", + description=( + "Every allowed_value declared on a skill output must appear in at least " + "one pattern_examples entry, ensuring the example set covers all output paths" + ), + severity=Severity.ERROR, +) +def _check_example_covers_all_allowed_values(ctx: ValidationContext) -> list[RuleFinding]: + """Error when any allowed_value has no corresponding pattern_examples entry. + + Reads allowed_values from the raw manifest dict (not SkillContract, which does not + promote allowed_values into SkillOutput). An example 'covers' a value when the + example text contains a match for '{output_name}\\s*=\\s*{re.escape(value)}'. + """ + findings: list[RuleFinding] = [] + try: + manifest = load_bundled_manifest() + except Exception: + logger.warning( + "example-covers-all-allowed-values: failed to load bundled manifest; skipping", + exc_info=True, + ) + return findings + + for step_name, step in ctx.recipe.steps.items(): + if step.tool != "run_skill": + continue + skill_cmd = (step.with_args or {}).get("skill_command", "") + name = resolve_skill_name(skill_cmd) + if not name: + continue + contract = get_skill_contract(name, manifest) + if not contract or not contract.pattern_examples: + continue + + skill_dict = manifest.get("skills", {}).get(name, {}) + for output in skill_dict.get("outputs", []): + if "allowed_values" not in output: + continue + output_name: str = output["name"] + for value in output["allowed_values"]: + pattern = re.compile(re.escape(output_name) + r"\s*=\s*" + re.escape(value)) + if not any(pattern.search(ex) for ex in contract.pattern_examples): + findings.append( + RuleFinding( + rule="example-covers-all-allowed-values", + severity=Severity.ERROR, + step_name=step_name, + message=( + f"Skill '{name}': allowed_value '{value}' on output " + f"'{output_name}' has no pattern_examples entry. " + f"Add an example containing '{output_name} = {value}' " + f"to skill_contracts.yaml so all output paths are covered." + ), + ) + ) + + return findings + + +@semantic_rule( + name="all-examples-match-all-patterns", + description=( + "Every pattern_examples entry must satisfy ALL expected_output_patterns. " + "An example that fails a pattern indicates the pattern is conditional, " + "which will cause CONTRACT_VIOLATION at runtime due to AND semantics." + ), + severity=Severity.ERROR, +) +def _check_all_examples_match_all_patterns(ctx: ValidationContext) -> list[RuleFinding]: + """Error when any pattern_examples entry fails any expected_output_pattern. + + The existing pattern-examples-match rule checks ∀ pattern, ∃ example (pattern not dead). + This rule checks ∀ example, ∀ pattern (pattern not conditional). Together they form a + complete constraint: patterns are both necessary and sufficient for all output paths. + """ + findings: list[RuleFinding] = [] + try: + manifest = load_bundled_manifest() + except Exception: + logger.warning( + "all-examples-match-all-patterns: failed to load bundled manifest; skipping", + exc_info=True, + ) + return findings + + for step_name, step in ctx.recipe.steps.items(): + if step.tool != "run_skill": + continue + skill_cmd = (step.with_args or {}).get("skill_command", "") + name = resolve_skill_name(skill_cmd) + if not name: + continue + contract = get_skill_contract(name, manifest) + if not contract or not contract.pattern_examples or not contract.expected_output_patterns: + continue + + for example in contract.pattern_examples: + for pattern in contract.expected_output_patterns: + try: + matched = bool(re.search(pattern, example)) + except re.error: + continue # invalid regex — covered by pattern-examples-match + if not matched: + preview = repr(example[:60]) + findings.append( + RuleFinding( + rule="all-examples-match-all-patterns", + severity=Severity.ERROR, + step_name=step_name, + message=( + f"Skill '{name}': example {preview} does not match " + f"pattern {pattern!r}. This pattern is conditional — " + f"AND semantics in _check_expected_patterns will reject " + f"sessions producing this output." + ), + ) + ) + + return findings + + +@semantic_rule( + name="path-output-recovery-coverage", + description=( + "Every file_path/directory_path output must have a matching expected_output_pattern" + ), + severity=Severity.WARNING, +) +def _check_path_output_recovery_coverage(ctx: ValidationContext) -> list[RuleFinding]: + """Every file_path/directory_path output must have a matching expected_output_pattern. + + Without a matching pattern, the recovery system cannot classify the output as + path-capture and synthesis/nudge will be silently disabled. + """ + findings: list[RuleFinding] = [] + try: + manifest = load_bundled_manifest() + except Exception: + logger.warning( + "path-output-recovery-coverage: failed to load bundled manifest; skipping", + exc_info=True, + ) + return findings + + for step_name, step in ctx.recipe.steps.items(): + if step.tool != "run_skill": + continue + skill_cmd = (step.with_args or {}).get("skill_command", "") + name = resolve_skill_name(skill_cmd) + if not name: + continue + contract = get_skill_contract(name, manifest) + if not contract: + continue + + path_outputs = [ + o + for o in contract.outputs + if (o.type.startswith("file_path") or o.type == "directory_path") + and o.name not in _PATH_RECOVERY_EXCLUDED_TOKENS + ] + if not path_outputs: + continue + + for output in path_outputs: + matched = any( + re.match(rf"^{re.escape(output.name)}\s*=", p) + for p in contract.expected_output_patterns + ) + if not matched: + findings.append( + RuleFinding( + rule="path-output-recovery-coverage", + severity=Severity.WARNING, + step_name=step_name, + message=( + f"Skill '{name}': output '{output.name}' (type={output.type!r}) " + f"has no matching expected_output_pattern. The write-artifact " + f"recovery system will be silently disabled for this output." + ), + ) + ) + + return findings diff --git a/src/autoskillit/recipe/rules_dataflow.py b/src/autoskillit/recipe/rules/rules_dataflow.py similarity index 61% rename from src/autoskillit/recipe/rules_dataflow.py rename to src/autoskillit/recipe/rules/rules_dataflow.py index af7a6dcc1b..f60241c84c 100644 --- a/src/autoskillit/recipe/rules_dataflow.py +++ b/src/autoskillit/recipe/rules/rules_dataflow.py @@ -2,6 +2,9 @@ from __future__ import annotations +import importlib +import inspect + from autoskillit.core import ( PIPELINE_FORBIDDEN_TOOLS, SKILL_TOOLS, @@ -10,7 +13,9 @@ ) from autoskillit.recipe._analysis import ValidationContext from autoskillit.recipe.contracts import ( + _CONTEXT_REF_RE, RESULT_CAPTURE_RE, + get_callable_contract, get_skill_contract, load_bundled_manifest, resolve_skill_name, @@ -52,7 +57,7 @@ def _check_weak_constraint_text(ctx: ValidationContext) -> list[RuleFinding]: @semantic_rule( name="undeclared-capture-key", description="result.X captures must match skill output keys in skill_contracts.yaml", - severity=Severity.WARNING, + severity=Severity.ERROR, ) def _check_capture_output_coverage(ctx: ValidationContext) -> list[RuleFinding]: wf = ctx.recipe @@ -76,7 +81,7 @@ def _check_capture_output_coverage(ctx: ValidationContext) -> list[RuleFinding]: findings.append( RuleFinding( rule="undeclared-capture-key", - severity=Severity.WARNING, + severity=Severity.ERROR, step_name=step_name, message=( f"Step '{step_name}' captures from skill '{skill_name}' " @@ -95,7 +100,7 @@ def _check_capture_output_coverage(ctx: ValidationContext) -> list[RuleFinding]: findings.append( RuleFinding( rule="undeclared-capture-key", - severity=Severity.WARNING, + severity=Severity.ERROR, step_name=step_name, message=( f"Step '{step_name}' captures result.{ref_key} " @@ -111,7 +116,7 @@ def _check_capture_output_coverage(ctx: ValidationContext) -> list[RuleFinding]: findings.append( RuleFinding( rule="undeclared-capture-key", - severity=Severity.WARNING, + severity=Severity.ERROR, step_name=step_name, message=( f"Step '{step_name}' captures result.{ref_key} via capture_list " @@ -124,6 +129,80 @@ def _check_capture_output_coverage(ctx: ValidationContext) -> list[RuleFinding]: return findings +@semantic_rule( + name="undeclared-python-capture-key", + description="result.X references in run_python steps must match callable contract outputs", + severity=Severity.WARNING, +) +def _check_python_capture_output_coverage(ctx: ValidationContext) -> list[RuleFinding]: + """Validate that run_python steps only reference declared callable output fields. + + Checks both capture: mappings and on_result condition when: expressions + for result.* field references, and verifies each against the callable's + contract in callable_contracts section of skill_contracts.yaml. + """ + wf = ctx.recipe + findings: list[RuleFinding] = [] + manifest = load_bundled_manifest() + + for step_name, step in wf.steps.items(): + if step.tool != "run_python": + continue + + callable_path = step.with_args.get("callable", "") + if not callable_path: + continue + + # Collect all result.* references from capture, capture_list, and on_result + result_refs: list[str] = [] + for _capture_var, capture_expr in step.capture.items(): + result_refs.extend(RESULT_CAPTURE_RE.findall(capture_expr)) + for _capture_var, capture_expr in step.capture_list.items(): + result_refs.extend(RESULT_CAPTURE_RE.findall(capture_expr)) + if step.on_result is not None: + for cond in step.on_result.conditions: + if cond.when is not None: + result_refs.extend(RESULT_CAPTURE_RE.findall(cond.when)) + + if not result_refs: + continue + + contract = get_callable_contract(callable_path, manifest) + if contract is None: + findings.append( + RuleFinding( + rule="undeclared-python-capture-key", + severity=Severity.WARNING, + step_name=step_name, + message=( + f"Step '{step_name}' references result.* fields from callable " + f"'{callable_path}' which has no callable contract entry in " + f"skill_contracts.yaml. Add a callable_contracts section to " + f"verify capture correctness." + ), + ) + ) + continue + + declared_keys = {out.name for out in contract.outputs} + for ref_key in result_refs: + if ref_key not in declared_keys: + findings.append( + RuleFinding( + rule="undeclared-python-capture-key", + severity=Severity.WARNING, + step_name=step_name, + message=( + f"Step '{step_name}' references result.{ref_key} " + f"but callable '{callable_path}' does not declare " + f"'{ref_key}' in its outputs contract." + ), + ) + ) + + return findings + + @semantic_rule( name="dead-output", description="Captured variable never consumed downstream", @@ -172,7 +251,7 @@ def _check_implicit_handoff(ctx: ValidationContext) -> list[RuleFinding]: continue if not contract.get("outputs"): continue - if not step.capture: + if not step.capture and not step.capture_list: findings.append( RuleFinding( rule="implicit-handoff", @@ -181,7 +260,7 @@ def _check_implicit_handoff(ctx: ValidationContext) -> list[RuleFinding]: message=( f"Step '{step_name}' calls '/autoskillit:{skill_name}' " f"which declares {len(contract['outputs'])} output(s) " - f"but has no capture: block. Add a capture: block to " + f"but has no capture: or capture_list: block. Add a capture: block to " f"thread the skill's structured output into pipeline context." ), ) @@ -255,15 +334,16 @@ def _check_multipart_iteration_notes(ctx: ValidationContext) -> list[RuleFinding next_or_done = wf.steps.get("next_or_done") if next_or_done is not None and next_or_done.on_result is not None: - # Legacy format: field/routes dict with explicit "more_parts" → "verify" - has_more_parts_to_verify = next_or_done.on_result.routes.get("more_parts") == "verify" - # Predicate format: condition with "more_parts" in the when clause routing to "verify" - if not has_more_parts_to_verify: - has_more_parts_to_verify = any( - cond.route == "verify" and cond.when is not None and "more_parts" in cond.when + # Legacy format: field/routes dict with explicit "more_parts" → any step + more_parts_target = next_or_done.on_result.routes.get("more_parts") + has_more_parts_route = more_parts_target is not None and more_parts_target in wf.steps + # Predicate format: condition with "more_parts" in the when clause routing to any step + if not has_more_parts_route: + has_more_parts_route = any( + cond.route in wf.steps and cond.when is not None and "more_parts" in cond.when for cond in next_or_done.on_result.conditions ) - if not has_more_parts_to_verify: + if not has_more_parts_route: findings.append( RuleFinding( rule="multipart-route-back", @@ -271,8 +351,8 @@ def _check_multipart_iteration_notes(ctx: ValidationContext) -> list[RuleFinding step_name="next_or_done", message=( "Recipe uses make-plan or rectify but next_or_done does not route " - "'more_parts' back to 'verify'. Sequential part processing requires " - "more_parts → verify in the on_result routes." + "'more_parts' back to a recipe step. Sequential part processing requires " + "a more_parts → condition in the on_result routes." ), ) ) @@ -417,3 +497,145 @@ def _check_uncaptured_handoff_consumer(ctx: ValidationContext) -> list[RuleFindi ) return findings + + +def _get_provided_args(with_args: dict) -> set[str]: + _nested = with_args.get("args") + nested_args: set[str] = set(_nested.keys()) if isinstance(_nested, dict) else set() + top_level_args = set(with_args.keys()) - {"callable", "timeout", "args"} + return nested_args | top_level_args + + +def _get_args_values(with_args: dict) -> dict[str, object]: + result: dict[str, object] = {} + _nested = with_args.get("args") + if isinstance(_nested, dict): + result.update(_nested) + for key, val in with_args.items(): + if key not in {"callable", "timeout", "args"}: + result[key] = val + return result + + +@semantic_rule( + name="missing-callable-input", + description="run_python steps must pass all required inputs declared in callable contract", + severity=Severity.ERROR, +) +def _check_missing_callable_input(ctx: ValidationContext) -> list[RuleFinding]: + findings = [] + manifest = load_bundled_manifest() + for step_name, step in ctx.recipe.steps.items(): + if step.tool != "run_python": + continue + callable_path = step.with_args.get("callable", "") + if not callable_path: + continue + contract = get_callable_contract(callable_path, manifest) + if contract is None: + continue + required_inputs = {inp.name for inp in contract.inputs if inp.required} + provided_args = _get_provided_args(step.with_args) + missing = required_inputs - provided_args + for arg_name in sorted(missing): + findings.append( + RuleFinding( + rule="missing-callable-input", + severity=Severity.ERROR, + step_name=step_name, + message=( + f"Step '{step_name}' calls '{callable_path}' but does not pass " + f"required input '{arg_name}'. Add it to the step's args block." + ), + ) + ) + return findings + + +@semantic_rule( + name="callable-signature-mismatch", + description="run_python step args keys must match the callable's function signature", + severity=Severity.ERROR, +) +def _check_callable_signature_mismatch(ctx: ValidationContext) -> list[RuleFinding]: + findings = [] + for step_name, step in ctx.recipe.steps.items(): + if step.tool != "run_python": + continue + callable_path = step.with_args.get("callable", "") + if not callable_path: + continue + try: + module_path, attr_name = callable_path.rsplit(".", 1) + mod = importlib.import_module(module_path) + func = getattr(mod, attr_name) + sig = inspect.signature(func) + except (ImportError, AttributeError, ValueError): + continue + if any(p.kind == inspect.Parameter.VAR_KEYWORD for p in sig.parameters.values()): + continue + valid_params = set(sig.parameters.keys()) + provided_args = _get_provided_args(step.with_args) + invalid = provided_args - valid_params + for arg_name in sorted(invalid): + findings.append( + RuleFinding( + rule="callable-signature-mismatch", + severity=Severity.ERROR, + step_name=step_name, + message=( + f"Step '{step_name}' passes arg '{arg_name}' to '{callable_path}' " + f"but the callable does not accept that parameter." + ), + ) + ) + return findings + + +@semantic_rule( + name="nullable-optional-context-ref", + description=( + "run_python steps must not pass optional_context_refs values to non-nullable " + "callable inputs without null coercion in the callable" + ), + severity=Severity.ERROR, +) +def _check_nullable_optional_context_ref(ctx: ValidationContext) -> list[RuleFinding]: + findings = [] + manifest = load_bundled_manifest() + for step_name, step in ctx.recipe.steps.items(): + if step.tool != "run_python": + continue + callable_path = step.with_args.get("callable", "") + if not callable_path: + continue + contract = get_callable_contract(callable_path, manifest) + if contract is None: + continue + non_nullable = {inp.name for inp in contract.inputs if not inp.nullable} + if not non_nullable: + continue + optional_refs = set(step.optional_context_refs) + if not optional_refs: + continue + args_values = _get_args_values(step.with_args) + for inp_name in sorted(non_nullable): + arg_val = args_values.get(inp_name) + if not isinstance(arg_val, str): + continue + for ref in _CONTEXT_REF_RE.findall(arg_val): + if ref in optional_refs: + findings.append( + RuleFinding( + rule="nullable-optional-context-ref", + severity=Severity.ERROR, + step_name=step_name, + message=( + f"Step '{step_name}' passes optional context ref '{ref}' " + f"to non-nullable input '{inp_name}' of callable " + f"'{callable_path}'. Add null coercion in the callable " + f"or remove from optional_context_refs." + ), + ) + ) + return findings diff --git a/src/autoskillit/recipe/rules/rules_features.py b/src/autoskillit/recipe/rules/rules_features.py new file mode 100644 index 0000000000..4a8beb5684 --- /dev/null +++ b/src/autoskillit/recipe/rules/rules_features.py @@ -0,0 +1,185 @@ +"""Semantic rules for feature-gated tool and skill references. + +Validates that recipe steps do not reference tools or skills belonging to +features that are currently disabled in the project configuration. +""" + +from __future__ import annotations + +from autoskillit.core import ( + FEATURE_REGISTRY, + SKILL_TOOLS, + TOOL_SUBSET_TAGS, + FeatureDef, + Severity, +) +from autoskillit.recipe._analysis import ValidationContext +from autoskillit.recipe._skill_helpers import _get_skill_category_map +from autoskillit.recipe.contracts import resolve_skill_name +from autoskillit.recipe.registry import RuleFinding, semantic_rule + + +def _tools_for_feature(fdef: FeatureDef) -> frozenset[str]: + """Return all MCP tool names that carry at least one of this feature's tool_tags.""" + return frozenset(tool for tool, tags in TOOL_SUBSET_TAGS.items() if fdef.tool_tags & tags) + + +def _get_disabled_feature_defs(ctx: ValidationContext) -> dict[str, FeatureDef]: + """Return {feature_name: FeatureDef} for all features named in ctx.disabled_features.""" + return {name: fdef for name, fdef in FEATURE_REGISTRY.items() if name in ctx.disabled_features} + + +@semantic_rule( + name="feature-gate-tool-reference", + description="Steps must not reference tools or skills from disabled features", + severity=Severity.ERROR, +) +def check_feature_gated_tools(ctx: ValidationContext) -> list[RuleFinding]: + """Flag steps that reference tools or skills belonging to a disabled feature.""" + if not ctx.disabled_features: + return [] + + disabled_fdefs = _get_disabled_feature_defs(ctx) + if not disabled_fdefs: + return [] + + findings: list[RuleFinding] = [] + + # Hoist per-feature tool sets and category_map outside the step loop + feature_tools = {fdef: _tools_for_feature(fdef) for fdef in disabled_fdefs.values()} + category_map = ( + ctx.skill_category_map if ctx.skill_category_map is not None else _get_skill_category_map() + ) + + for step_name, step in ctx.recipe.steps.items(): + for fname, fdef in disabled_fdefs.items(): + # --- Tool check --- + if step.tool and step.tool in feature_tools[fdef]: + findings.append( + RuleFinding( + rule="feature-gate-tool-reference", + severity=Severity.ERROR, + step_name=step_name, + message=( + f"step '{step_name}': tool '{step.tool}' belongs to " + f"disabled feature '{fname}'. " + f"Enable '{fname}' in .autoskillit/config.yaml " + f"features to use this tool." + ), + ) + ) + + if ( + step.tool == "run_python" + and fdef.import_package is not None + and (step.with_args or {}).get("callable", "").startswith(fdef.import_package) + ): + findings.append( + RuleFinding( + rule="feature-gate-tool-reference", + severity=Severity.ERROR, + step_name=step_name, + message=( + f"step '{step_name}': run_python callable " + f"'{(step.with_args or {}).get('callable', '')}' " + f"belongs to disabled feature '{fname}'. " + f"Enable '{fname}' in .autoskillit/config.yaml " + f"features to use this callable." + ), + ) + ) + + # --- Skill check (only when the feature gates skill categories) --- + if step.tool not in SKILL_TOOLS or not fdef.skill_categories: + continue + skill_cmd = (step.with_args or {}).get("skill_command") or "" + skill_name = resolve_skill_name(skill_cmd) + if skill_name is None: + continue + categories = category_map.get(skill_name, frozenset()) + if categories & fdef.skill_categories: + findings.append( + RuleFinding( + rule="feature-gate-tool-reference", + severity=Severity.ERROR, + step_name=step_name, + message=( + f"step '{step_name}': skill_command '{skill_cmd}' references " + f"skill '{skill_name}' which belongs to disabled feature " + f"'{fname}'. " + f"Enable '{fname}' in .autoskillit/config.yaml " + f"features to use this skill." + ), + ) + ) + + return findings + + +@semantic_rule( + name="undeclared-feature-requirement", + description="Recipes dispatching feature-gated skills must declare requires_features", + severity=Severity.ERROR, +) +def check_requires_features_declared(ctx: ValidationContext) -> list[RuleFinding]: + """Flag recipes that dispatch skills in feature-gated categories without + declaring the corresponding requires_features entry.""" + category_map = ( + ctx.skill_category_map if ctx.skill_category_map is not None else _get_skill_category_map() + ) + declared_features: frozenset[str] = frozenset(ctx.recipe.requires_features) + + findings: list[RuleFinding] = [] + for step_name, step in ctx.recipe.steps.items(): + if step.tool not in SKILL_TOOLS: + continue + skill_cmd = (step.with_args or {}).get("skill_command") or "" + skill_name = resolve_skill_name(skill_cmd) + if skill_name is None: + continue + categories = category_map.get(skill_name, frozenset()) + for feat_name, fdef in FEATURE_REGISTRY.items(): + if not (categories & fdef.skill_categories): + continue + if feat_name not in declared_features: + findings.append( + RuleFinding( + rule="undeclared-feature-requirement", + severity=Severity.ERROR, + step_name=step_name, + message=( + f"step '{step_name}': skill_command '{skill_cmd}' references " + f"skill '{skill_name}' which belongs to feature '{feat_name}'. " + f"Add 'requires_features: [{feat_name}]' to this recipe so " + f"init_session can enable the '{feat_name}' feature gate." + ), + ) + ) + return findings + + +@semantic_rule( + name="provider-requires-profile", + description="Steps that reference a provider must name a configured provider profile", + severity=Severity.ERROR, +) +def check_provider_requires_profile(ctx: ValidationContext) -> list[RuleFinding]: + if not ctx.provider_profiles: + return [] + + findings: list[RuleFinding] = [] + for step_name, step in ctx.recipe.steps.items(): + if step.provider is not None and step.provider not in ctx.provider_profiles: + findings.append( + RuleFinding( + rule="provider-requires-profile", + severity=Severity.ERROR, + step_name=step_name, + message=( + f"step '{step_name}': provider '{step.provider}' is not a " + f"configured provider profile. Add it under " + f"providers.profiles in .autoskillit/config.yaml." + ), + ) + ) + return findings diff --git a/src/autoskillit/recipe/rules_fixing.py b/src/autoskillit/recipe/rules/rules_fixing.py similarity index 99% rename from src/autoskillit/recipe/rules_fixing.py rename to src/autoskillit/recipe/rules/rules_fixing.py index b6beb0cc21..d756e7e367 100644 --- a/src/autoskillit/recipe/rules_fixing.py +++ b/src/autoskillit/recipe/rules/rules_fixing.py @@ -13,9 +13,10 @@ from __future__ import annotations -import re from collections import deque +import regex as re + from autoskillit.core import SKILL_TOOLS, Severity, get_logger from autoskillit.recipe._analysis import ValidationContext from autoskillit.recipe.contracts import load_bundled_manifest, resolve_skill_name diff --git a/src/autoskillit/recipe/rules/rules_food_truck.py b/src/autoskillit/recipe/rules/rules_food_truck.py new file mode 100644 index 0000000000..4edaec3693 --- /dev/null +++ b/src/autoskillit/recipe/rules/rules_food_truck.py @@ -0,0 +1,32 @@ +"""Semantic validation rules for food-truck recipes.""" + +from __future__ import annotations + +from autoskillit.core import Severity +from autoskillit.recipe._analysis import ValidationContext +from autoskillit.recipe.registry import RuleFinding, semantic_rule +from autoskillit.recipe.schema import RecipeKind + + +@semantic_rule( + name="food-truck-has-sentinel-stop", + description="Food-truck recipes must have a stop step referencing L3 sentinel", + severity=Severity.WARNING, +) +def _check_food_truck_has_sentinel_stop(ctx: ValidationContext) -> list[RuleFinding]: + if ctx.recipe.kind != RecipeKind.FOOD_TRUCK: + return [] + for step_name, step in ctx.recipe.steps.items(): + if step.action == "stop" and step.message and "sentinel" in step.message.lower(): + return [] + return [ + RuleFinding( + rule="food-truck-has-sentinel-stop", + severity=Severity.WARNING, + step_name="(top-level)", + message=( + "Food-truck recipe has no stop step referencing L3 sentinel. " + "Food trucks must emit a sentinel JSON block on completion." + ), + ) + ] diff --git a/src/autoskillit/recipe/rules_graph.py b/src/autoskillit/recipe/rules/rules_graph.py similarity index 58% rename from src/autoskillit/recipe/rules_graph.py rename to src/autoskillit/recipe/rules/rules_graph.py index 482ac84254..b6dab12d42 100644 --- a/src/autoskillit/recipe/rules_graph.py +++ b/src/autoskillit/recipe/rules/rules_graph.py @@ -2,19 +2,27 @@ from __future__ import annotations -import re +import regex as re from autoskillit.core import ( SKILL_TOOLS, Severity, get_logger, + resolve_skill_name, ) from autoskillit.recipe._analysis import ValidationContext -from autoskillit.recipe.contracts import _CONTEXT_REF_RE +from autoskillit.recipe._analysis_bfs import bfs_reachable_without_barrier +from autoskillit.recipe.contracts import ( + _CONTEXT_REF_RE, + get_tool_output_contract, + load_bundled_manifest, +) from autoskillit.recipe.registry import RuleFinding, semantic_rule logger = get_logger(__name__) +_STRUCTURAL_ON_RESULT_TOOLS = {"run_python", "wait_for_ci"} + @semantic_rule( name="unbounded-cycle", @@ -33,9 +41,9 @@ def _check_unbounded_cycles(ctx: ValidationContext) -> list[RuleFinding]: def dfs(node: str, path: list[str]) -> None: visited.add(node) rec_stack.add(node) - for neighbor in graph.get(node, set()): + for neighbor in sorted(graph.get(node, set())): if neighbor not in recipe.steps: - continue # dead reference — caught by validate_recipe + continue # dead reference — caught by validate_recipe_structure if neighbor not in visited: dfs(neighbor, path + [neighbor]) elif neighbor in rec_stack: @@ -51,6 +59,27 @@ def dfs(node: str, path: list[str]) -> None: reported_cycles.add(cycle_key) cycle_set = set(cycle_steps) + has_on_result_exit = False + for s in cycle_steps: + if s not in recipe.steps: + continue + step = recipe.steps[s] + if step.tool not in _STRUCTURAL_ON_RESULT_TOOLS or step.on_result is None: + continue + targets: set[str] = set() + if step.on_result.conditions: + targets = {c.route for c in step.on_result.conditions} + elif step.on_result.routes: + targets = set(step.on_result.routes.values()) + routes_out = targets - cycle_set + if routes_out: + has_on_result_exit = True + break + + if has_on_result_exit: + rec_stack.discard(node) + return + has_retry_exit = any( recipe.steps[s].retries > 0 and recipe.steps[s].tool in SKILL_TOOLS @@ -70,20 +99,48 @@ def dfs(node: str, path: list[str]) -> None: and recipe.steps[s].tool in SKILL_TOOLS and recipe.steps[s].on_exhausted not in cycle_set ] - success_stays_in_cycle = any( - recipe.steps[s].on_success in cycle_set - for s in retrying_steps - if recipe.steps[s].on_success is not None - ) + # Check whether any non-failure successor of the retrying step + # stays within the cycle. Uses the step graph (which includes + # on_result routes) rather than on_success alone, so steps that + # route via on_result without an explicit on_success are handled. + success_stays_in_cycle = False + for _s in retrying_steps: + _step = recipe.steps[_s] + _fail_targets = { + t + for t in ( + _step.on_failure, + _step.on_exhausted, + _step.on_context_limit, + ) + if t + } + if any( + succ in cycle_set + for succ in graph.get(_s, set()) + if succ not in _fail_targets + ): + success_stays_in_cycle = True + break if not success_stays_in_cycle: # Success path exits the cycle — but does it loop back? # BFS from exit targets to check if they can reach any # cycle member through the step graph. exit_targets: set[str] = set() for _rs in retrying_steps: - _os = recipe.steps[_rs].on_success - if _os is not None and _os not in cycle_set: - exit_targets.add(_os) + _step_r = recipe.steps[_rs] + _fail_targets_r = { + t + for t in ( + _step_r.on_failure, + _step_r.on_exhausted, + _step_r.on_context_limit, + ) + if t + } + for succ in graph.get(_rs, set()): + if succ not in cycle_set and succ not in _fail_targets_r: + exit_targets.add(succ) loops_back = False visited_exit: set[str] = set() frontier = exit_targets @@ -102,19 +159,19 @@ def dfs(node: str, path: list[str]) -> None: return # Success path re-enters the cycle — retry exit only bounds - # individual step visits, not the outer loop. Emit WARNING. + # individual step visits, not the outer loop. Emit ERROR. findings.append( RuleFinding( rule="unbounded-cycle", - severity=Severity.WARNING, + severity=Severity.ERROR, step_name=node, message=( f"Routing cycle detected: {' → '.join(cycle_steps)} → {neighbor}. " f"Step(s) {', '.join(retrying_steps)} have retry exits, but their " f"success paths re-enter the cycle. The inner retry budget resets " f"on each loop iteration, so the outer loop is unbounded. " - "Add a global iteration counter or route success outside the " - "cycle." + "Add a check_loop_iteration guard step to enforce a hard " + "iteration cap, or route the success path outside the cycle." ), ) ) @@ -360,12 +417,18 @@ def _check_merge_base_unpublished(ctx: ValidationContext) -> list[RuleFinding]: if context_var is None: continue # literal branch name — always published, no check needed - # Collect push_to_remote steps that push this exact context variable. + # Collect steps that publish this exact context variable. + # push_to_remote steps are barriers when their branch arg is context.X. + # create_and_publish_branch steps are barriers when they capture X (they + # always push the created branch before capturing it as merge_target). push_steps = { name for name, s in recipe.steps.items() - if s.tool == "push_to_remote" - and _extract_context_var((s.with_args or {}).get("branch", "")) == context_var + if ( + s.tool == "push_to_remote" + and _extract_context_var((s.with_args or {}).get("branch", "")) == context_var + ) + or (s.tool == "create_and_publish_branch" and context_var in (s.capture or {})) } # BFS from entry treating push_steps as barriers. @@ -402,3 +465,187 @@ def _check_merge_base_unpublished(ctx: ValidationContext) -> list[RuleFinding]: ) return findings + + +@semantic_rule( + name="on-result-missing-tool-output-value", + description=( + "Recoverable tool output values falling through to a terminal catch-all " + "step. When a tool has declared recoverable_values in tool_output_contracts " + "and none of them are explicitly routed in on_result conditions, a catch-all " + "that routes to a terminal (action: stop) step silently drops those values." + ), + severity=Severity.WARNING, +) +def _check_on_result_missing_tool_output_value(ctx: ValidationContext) -> list[RuleFinding]: + recipe = ctx.recipe + findings: list[RuleFinding] = [] + for step_name, step in recipe.steps.items(): + if step.tool is None or step.on_result is None: + continue + contract = get_tool_output_contract(step.tool) + if contract is None: + continue + field_def = contract.fields.get(contract.result_field) + if field_def is None or not field_def.recoverable_values: + continue + conditions = step.on_result.conditions or [] + explicitly_routed: set[str] = set() + catchall_route: str | None = None + for cond in conditions: + if cond.when is None: + catchall_route = cond.route + else: + for val in field_def.allowed_values: + if val in cond.when: + explicitly_routed.add(val) + if catchall_route is None: + continue + catchall_step = recipe.steps.get(catchall_route) + if catchall_step is None or catchall_step.action != "stop": + continue + unrouted_recoverable = field_def.recoverable_values - explicitly_routed + if not unrouted_recoverable: + continue + findings.append( + RuleFinding( + rule="on-result-missing-tool-output-value", + severity=Severity.WARNING, + step_name=step_name, + message=( + f"Step '{step_name}' (tool: {step.tool}) has recoverable " + f"output values {sorted(unrouted_recoverable)} not explicitly " + f"routed in on_result. The catch-all routes to terminal step " + f"'{catchall_route}' (action: stop), silently terminating on " + f"these recoverable outcomes." + ), + ) + ) + return findings + + +@semantic_rule( + name="skill-result-routing-gap", + description=( + "A run_skill step that captures a skill output with declared allowed_values " + "must have an explicit on_result condition for every allowed value. " + "If the output is not captured at all and the catch-all routes to a " + "non-terminal step, unhandled values silently fall through to the success path." + ), + severity=Severity.ERROR, +) +def _check_skill_result_routing_gap(ctx: ValidationContext) -> list[RuleFinding]: + findings: list[RuleFinding] = [] + try: + manifest = load_bundled_manifest() + except (FileNotFoundError, OSError, ValueError): + logger.warning("failed to load bundled manifest", exc_info=True) + return findings + for step_name, step in ctx.recipe.steps.items(): + if step.tool != "run_skill": + continue + skill_command = (step.with_args or {}).get("skill_command", "") + skill_name = resolve_skill_name(skill_command) + if not skill_name: + continue + skill_contract = manifest.get("skills", {}).get(skill_name, {}) + outputs_with_allowed_values: dict[str, list[str]] = {} + for output in skill_contract.get("outputs", []): + if "allowed_values" in output: + outputs_with_allowed_values[output["name"]] = output["allowed_values"] + if not outputs_with_allowed_values: + continue + captured_outputs = set() + if step.capture: + for captured_var, capture_expr in step.capture.items(): + for output_name in outputs_with_allowed_values: + if f"result.{output_name}" in capture_expr: + captured_outputs.add(output_name) + if not step.on_result or not step.on_result.conditions: + if captured_outputs: + for output_name in captured_outputs: + findings.append( + RuleFinding( + rule="skill-result-routing-gap", + severity=Severity.ERROR, + step_name=step_name, + message=( + f"Step '{step_name}' captures '{output_name}' but has no " + f"on_result conditions to route its allowed values " + f"{outputs_with_allowed_values[output_name]}." + ), + ) + ) + continue + conditions = step.on_result.conditions or [] + for output_name, allowed_values in outputs_with_allowed_values.items(): + explicitly_routed: set[str] = set() + catchall_route: str | None = None + for cond in conditions: + if cond.when is None: + catchall_route = cond.route + else: + for val in allowed_values: + if val in cond.when: + explicitly_routed.add(val) + unrouted = [v for v in allowed_values if v not in explicitly_routed] + if not unrouted: + continue + if catchall_route is None: + continue + catchall_step = ctx.recipe.steps.get(catchall_route) + is_terminal = catchall_step is not None and catchall_step.action == "stop" + if is_terminal: + continue + findings.append( + RuleFinding( + rule="skill-result-routing-gap", + severity=Severity.ERROR, + step_name=step_name, + message=( + f"Step '{step_name}' has allowed value(s) {unrouted} of " + f"'{output_name}' not explicitly routed in on_result. " + f"Catch-all routes to non-terminal step '{catchall_route}'." + ), + ) + ) + return findings + + +@semantic_rule( + name="review-loop-waypoint-guard", + description=( + "check_repo_ci_event must not be reachable from review_pr without " + "traversing check_review_loop. Bypassing check_review_loop prevents " + "review_loop_count from incrementing, causing review_mode to remain " + "'local' permanently and no GitHub comments to be posted on clean PRs." + ), + severity=Severity.ERROR, +) +def _check_review_loop_waypoint(ctx: ValidationContext) -> list[RuleFinding]: + steps = ctx.recipe.steps + if not all(k in steps for k in ("review_pr", "check_review_loop", "check_repo_ci_event")): + return [] + + reachable = bfs_reachable_without_barrier( + recipe=ctx.recipe, + start="review_pr", + barrier="check_review_loop", + ) + + if "check_repo_ci_event" not in reachable: + return [] + + return [ + RuleFinding( + rule="review-loop-waypoint-guard", + severity=Severity.ERROR, + step_name="review_pr", + message=( + "check_repo_ci_event is reachable from review_pr without crossing " + "check_review_loop. All review_pr verdicts must route through " + "check_review_loop so review_loop_count is always incremented and " + "review_mode can graduate from 'local' to 'github'." + ), + ) + ] diff --git a/src/autoskillit/recipe/rules/rules_inline_script.py b/src/autoskillit/recipe/rules/rules_inline_script.py new file mode 100644 index 0000000000..198bd4c624 --- /dev/null +++ b/src/autoskillit/recipe/rules/rules_inline_script.py @@ -0,0 +1,169 @@ +"""Semantic rules: detect inline shell scripts in run_cmd cmd fields.""" + +from __future__ import annotations + +import regex as re + +from autoskillit.core import Severity +from autoskillit.recipe._analysis import ValidationContext +from autoskillit.recipe.registry import RuleFinding, semantic_rule + +# --------------------------------------------------------------------------- +# Detection patterns +# --------------------------------------------------------------------------- + +_CONTROL_FLOW_RE = re.compile( + r""" + (? str: + """Remove jq expression blocks to avoid false-positive control-flow matches.""" + return _JQ_BLOCK_RE.sub("", cmd) + + +def _count_logical_lines(cmd: str) -> int: + """Count non-blank, non-comment lines.""" + return sum(1 for line in cmd.splitlines() if line.strip() and not line.strip().startswith("#")) + + +@semantic_rule( + name="inline-script-in-cmd", + description=( + "Detects inline shell scripts in run_cmd cmd fields " + "(control flow, loops, excessive variable assignments)" + ), + severity=Severity.ERROR, +) +def _check_inline_script_in_cmd(ctx: ValidationContext) -> list[RuleFinding]: + findings: list[RuleFinding] = [] + for name, step in ctx.recipe.steps.items(): + if step.tool != "run_cmd": + continue + cmd = (step.with_args or {}).get("cmd", "") + if not isinstance(cmd, str): + continue + + stripped = _strip_jq_blocks(cmd) + + if _CONTROL_FLOW_RE.search(stripped): + findings.append( + RuleFinding( + rule="inline-script-in-cmd", + severity=Severity.ERROR, + step_name=name, + message=( + f"Step '{name}' contains shell control flow in cmd. " + "Extract to a .sh script or run_python callable." + ), + ) + ) + continue + + if _BASH_BUILTINS_RE.search(stripped): + findings.append( + RuleFinding( + rule="inline-script-in-cmd", + severity=Severity.ERROR, + step_name=name, + message=( + f"Step '{name}' contains bash builtins " + "(mapfile/declare/local/export/read -r) in cmd. " + "Extract to a .sh script." + ), + ) + ) + continue + + var_count = len(_VAR_ASSIGN_RE.findall(cmd)) + chain_count = len(_AND_CHAIN_RE.findall(cmd)) + if chain_count > 3 and var_count >= 1: + findings.append( + RuleFinding( + rule="inline-script-in-cmd", + severity=Severity.ERROR, + step_name=name, + message=( + f"Step '{name}' has {chain_count} &&-chains with " + f"{var_count} variable assignment(s). " + "Extract to a .sh script or run_python callable." + ), + ) + ) + continue + + logical_lines = _count_logical_lines(cmd) + if logical_lines > 3 and var_count > 2: + findings.append( + RuleFinding( + rule="inline-script-in-cmd", + severity=Severity.WARNING, + step_name=name, + message=( + f"Step '{name}' has {logical_lines} logical lines and " + f"{var_count} variable assignments. " + "Consider extracting to a script." + ), + ) + ) + + return findings + + +@semantic_rule( + name="inline-python-in-cmd", + description=( + "Detects python3 -c usage in run_cmd cmd fields (must use run_python callable instead)" + ), + severity=Severity.ERROR, +) +def _check_inline_python_in_cmd(ctx: ValidationContext) -> list[RuleFinding]: + findings: list[RuleFinding] = [] + for name, step in ctx.recipe.steps.items(): + if step.tool != "run_cmd": + continue + cmd = (step.with_args or {}).get("cmd", "") + if not isinstance(cmd, str): + continue + + if _PYTHON3_C_RE.search(cmd): + findings.append( + RuleFinding( + rule="inline-python-in-cmd", + severity=Severity.ERROR, + step_name=name, + message=( + f"Step '{name}' uses python3 -c in cmd. Convert to a run_python callable." + ), + ) + ) + + return findings diff --git a/src/autoskillit/recipe/rules_inputs.py b/src/autoskillit/recipe/rules/rules_inputs.py similarity index 83% rename from src/autoskillit/recipe/rules_inputs.py rename to src/autoskillit/recipe/rules/rules_inputs.py index 9cda08dd2a..109aecabb0 100644 --- a/src/autoskillit/recipe/rules_inputs.py +++ b/src/autoskillit/recipe/rules/rules_inputs.py @@ -2,8 +2,7 @@ from __future__ import annotations -import re - +import regex as re from packaging.version import Version from autoskillit.core import ( @@ -38,18 +37,7 @@ def _check_outdated_version(ctx: ValidationContext) -> list[RuleFinding]: wf = ctx.recipe script_ver = wf.version if script_ver is None: - return [ - RuleFinding( - rule="outdated-recipe-version", - severity=Severity.WARNING, - step_name="(top-level)", - message=( - f"Recipe has no autoskillit_version field. " - f"Current installed version is {AUTOSKILLIT_INSTALLED_VERSION}. " - f"Run 'autoskillit migrate' to update." - ), - ) - ] + return [] if Version(script_ver) < Version(AUTOSKILLIT_INSTALLED_VERSION): return [ @@ -392,3 +380,76 @@ def _check_research_output_mode_enum( ) ] return [] + + +@semantic_rule( + name="ingredient-type-default-invalid", + description=( + "Integer-typed ingredients must not use '' as default " + "(auto-detect sentinel is invalid for numerics)" + ), + severity=Severity.ERROR, +) +def _check_ingredient_type_default_invalid(ctx: ValidationContext) -> list[RuleFinding]: + """Reject integer-typed ingredients that use '' as default or have non-parseable defaults.""" + wf = ctx.recipe + findings: list[RuleFinding] = [] + + for ing_name, ing in (wf.ingredients or {}).items(): + if getattr(ing, "type", None) != "integer": + continue + + default = getattr(ing, "default", None) + required = getattr(ing, "required", False) + + # Empty string default is the auto-detect sentinel — invalid for integer types + if default == "": + findings.append( + RuleFinding( + rule="ingredient-type-default-invalid", + severity=Severity.ERROR, + step_name=ing_name, + message=( + f"Ingredient '{ing_name}' has type='integer' but default='' " + "(auto-detect sentinel). Integer-typed ingredients must use an " + "explicit numeric default. Suggested fix: default='3'." + ), + ) + ) + continue + + # Non-empty but non-parseable default + if default is not None and default != "": + try: + int(default) + continue + except ValueError as exc: + findings.append( + RuleFinding( + rule="ingredient-type-default-invalid", + severity=Severity.ERROR, + step_name=ing_name, + message=( + f"Ingredient '{ing_name}' has type='integer' but default={default!r} " + f"is not parseable as an integer ({exc})." + ), + ) + ) + continue + + # None default with required=False and no explicit value — silently absent numeric field + if default is None and not required: + findings.append( + RuleFinding( + rule="ingredient-type-default-invalid", + severity=Severity.ERROR, + step_name=ing_name, + message=( + f"Ingredient '{ing_name}' has type='integer', required=False, and " + f"default=None. Integer-typed non-required ingredients must declare " + f"an explicit default to avoid ambiguity." + ), + ) + ) + + return findings diff --git a/src/autoskillit/recipe/rules_isolation.py b/src/autoskillit/recipe/rules/rules_isolation.py similarity index 99% rename from src/autoskillit/recipe/rules_isolation.py rename to src/autoskillit/recipe/rules/rules_isolation.py index 2b1827ea97..5b306e7f29 100644 --- a/src/autoskillit/recipe/rules_isolation.py +++ b/src/autoskillit/recipe/rules/rules_isolation.py @@ -2,7 +2,7 @@ from __future__ import annotations -import re +import regex as re from autoskillit.core import Severity, get_logger from autoskillit.recipe._analysis import ValidationContext diff --git a/src/autoskillit/recipe/rules/rules_merge.py b/src/autoskillit/recipe/rules/rules_merge.py new file mode 100644 index 0000000000..6569228034 --- /dev/null +++ b/src/autoskillit/recipe/rules/rules_merge.py @@ -0,0 +1,449 @@ +"""Semantic rules for merge_worktree routing completeness.""" + +from __future__ import annotations + +import regex as re + +from autoskillit.core import MergeFailedStep, Severity, get_logger +from autoskillit.recipe._analysis import ValidationContext, bfs_reachable +from autoskillit.recipe._rule_helpers import _is_loop_guard_step +from autoskillit.recipe.registry import RuleFinding, semantic_rule + +logger = get_logger(__name__) + + +def _is_commit_guard(step_name: str, ctx: ValidationContext) -> bool: + """Return True if step_name is a commit_guard predecessor for merge_worktree. + + A commit_guard step is one whose name starts with 'commit_guard' OR whose + tool is 'run_cmd' and whose cmd contains 'git commit'. + """ + if step_name.startswith("commit_guard"): + return True + step = ctx.recipe.steps.get(step_name) + if step and step.tool == "run_cmd": + cmd = step.with_args.get("cmd", "") + if "git commit" in cmd: + return True + return False + + +_RECOVERABLE_FAILED_STEPS: frozenset[str] = frozenset( + { + MergeFailedStep.DIRTY_TREE, + MergeFailedStep.TEST_GATE, + MergeFailedStep.POST_REBASE_TEST_GATE, + MergeFailedStep.REBASE, + } +) + +_MERGE_FAILURE_DOMAINS: dict[str, str] = { + MergeFailedStep.DIRTY_TREE: "code", + MergeFailedStep.TEST_GATE: "code", + MergeFailedStep.POST_REBASE_TEST_GATE: "code", + MergeFailedStep.REBASE: "git_conflict", +} + +_REQUIRED_SKILL_BY_DOMAIN: dict[str, str] = { + "code": "resolve-failures", + "git_conflict": "resolve-merge-conflicts", +} + +_FAILED_STEP_PATTERN = re.compile(r"result\.failed_step\s*==\s*['\"](\w+)['\"]") + + +@semantic_rule( + name="merge-routing-incomplete", + description=( + "Every merge_worktree step with predicate on_result must explicitly route " + "all recoverable MergeFailedStep values to a recovery step. " + "Unhandled values fall through to the result.error catch-all, which typically " + "discards a recoverable worktree." + ), + severity=Severity.ERROR, +) +def _check_merge_routing_completeness(ctx: ValidationContext) -> list[RuleFinding]: + findings: list[RuleFinding] = [] + for step_name, step in ctx.recipe.steps.items(): + if step.tool != "merge_worktree": + continue + if not step.on_result or not step.on_result.conditions: + continue + + matched: set[str] = set() + for condition in step.on_result.conditions: + if condition.when is None: + continue + m = _FAILED_STEP_PATTERN.search(condition.when) + if m: + matched.add(m.group(1)) + + missing = _RECOVERABLE_FAILED_STEPS - matched + if missing: + findings.append( + RuleFinding( + rule="merge-routing-incomplete", + severity=Severity.ERROR, + step_name=step_name, + message=( + f"merge_worktree on_result is missing explicit routes for " + f"recoverable failures: {sorted(missing)}. " + f"These will fall through to the result.error catch-all, " + f"discarding a recoverable worktree." + ), + ) + ) + return findings + + +_SKILL_CMD_PATTERN = re.compile(r"/(?:autoskillit:)?([\w-]+)") + + +@semantic_rule( + name="merge-failure-skill-domain-mismatch", + description=( + "A merge_worktree on_result condition routes a recoverable failed_step to a " + "step whose skill does not match the failure domain. Git-conflict failures " + "(rebase) must route to resolve-merge-conflicts; code failures (dirty_tree, " + "test_gate, post_rebase_test_gate) must route to resolve-failures." + ), + severity=Severity.ERROR, +) +def _check_merge_failure_skill_domain_mismatch( + ctx: ValidationContext, +) -> list[RuleFinding]: + findings: list[RuleFinding] = [] + for step_name, step in ctx.recipe.steps.items(): + if step.tool != "merge_worktree": + continue + if not step.on_result or not step.on_result.conditions: + continue + + for condition in step.on_result.conditions: + if condition.when is None: + continue + m = _FAILED_STEP_PATTERN.search(condition.when) + if not m: + continue + failed_step_value = m.group(1) + domain = _MERGE_FAILURE_DOMAINS.get(failed_step_value) + if domain is None: + continue + + target_step = ctx.recipe.steps.get(condition.route) + if target_step is None or target_step.tool != "run_skill": + continue + + skill_cmd = target_step.with_args.get("skill_command", "") + skill_match = _SKILL_CMD_PATTERN.search(skill_cmd) + if not skill_match: + continue + actual_skill = skill_match.group(1) + + required_skill = _REQUIRED_SKILL_BY_DOMAIN[domain] + if actual_skill != required_skill: + findings.append( + RuleFinding( + rule="merge-failure-skill-domain-mismatch", + severity=Severity.ERROR, + step_name=step_name, + message=( + f"failed_step == '{failed_step_value}' (domain: {domain}) " + f"routes to step '{condition.route}' which invokes " + f"'{actual_skill}', but {domain} failures require " + f"'{required_skill}'." + ), + ) + ) + return findings + + +def _has_commit_guard_ancestor( + step_name: str, ctx: ValidationContext, *, max_depth: int = 5 +) -> bool: + """BFS over predecessors to find a commit_guard within *max_depth* hops.""" + visited: set[str] = set() + frontier = ctx.predecessors.get(step_name, set()) + for _ in range(max_depth): + if not frontier: + break + for p in frontier: + if _is_commit_guard(p, ctx): + return True + visited |= frontier + next_frontier: set[str] = set() + for p in frontier: + next_frontier |= ctx.predecessors.get(p, set()) - visited + frontier = next_frontier + return False + + +@semantic_rule( + name="merge-fix-cycle-without-iteration-guard", + description=( + "A merge_worktree step routes recoverable failures to a fix/assess step, " + "creating a merge→fix→test→merge cycle. Without a check_loop_iteration guard, " + "this cycle can loop unboundedly on structurally unresolvable conflicts." + ), + severity=Severity.ERROR, +) +def _check_merge_fix_cycle_without_guard(ctx: ValidationContext) -> list[RuleFinding]: + findings: list[RuleFinding] = [] + for step_name, step in ctx.recipe.steps.items(): + if step.tool != "merge_worktree": + continue + if not step.on_result or not step.on_result.conditions: + continue + + fix_routes: set[str] = set() + for cond in step.on_result.conditions: + if cond.when and re.search(r"\bfailed_step\b", cond.when): + fix_routes.add(cond.route) + + if not fix_routes: + continue + + has_guard = False + for fix_step_name in fix_routes: + reachable = bfs_reachable(ctx.step_graph, fix_step_name) | {fix_step_name} + for reached in reachable: + if _is_loop_guard_step(reached, ctx): + has_guard = True + break + if has_guard: + break + + if not has_guard: + findings.append( + RuleFinding( + rule="merge-fix-cycle-without-iteration-guard", + severity=Severity.ERROR, + step_name=step_name, + message=( + f"merge_worktree step '{step_name}' routes recoverable failures " + f"to {sorted(fix_routes)}, creating a merge→fix→test cycle " + f"with no check_loop_iteration guard. This can loop unboundedly " + f"on structurally unresolvable conflicts. Add a check_merge_fix_loop " + f"step (run_python calling check_loop_iteration) between test and " + f"commit_guard to cap the cycle at 3 iterations." + ), + ) + ) + return findings + + +@semantic_rule( + name="gh-pr-merge-silent-success-routing", + description=( + "A run_cmd step that executes 'gh pr merge' must not route its on_failure " + "to register_clone_success. A failed merge means the PR was NOT merged; routing " + "to the success terminal silently reports the PR as done when it is not. " + "Cleanup steps are exempt: steps with optional=True, or steps whose name starts " + "with 'release_issue_' (all release_issue_* steps are terminal cleanup steps by " + "convention — they never perform primary merge work)." + ), + severity=Severity.ERROR, +) +def _check_gh_pr_merge_silent_success_degradation(ctx: ValidationContext) -> list[RuleFinding]: + findings: list[RuleFinding] = [] + for step_name, step in ctx.recipe.steps.items(): + if step.tool != "run_cmd": + continue + cmd = step.with_args.get("cmd", "") + if not isinstance(cmd, str) or "gh pr merge" not in cmd: + continue + # Exempt cleanup steps: optional=True, or name starts with release_issue_ + # (release_issue_* steps are terminal cleanup steps by convention) + if step.optional or step_name.startswith("release_issue_"): + continue + if step.on_failure == "register_clone_success": + findings.append( + RuleFinding( + rule="gh-pr-merge-silent-success-routing", + severity=Severity.ERROR, + step_name=step_name, + message=( + f"Step '{step_name}' runs 'gh pr merge' but routes " + f"on_failure to 'register_clone_success' (a success terminal). " + f"A failed merge command means the PR was NOT merged. " + f"Route on_failure to an escalation target such as " + f"'release_issue_failure' or 'verify_queue_enrollment'." + ), + ) + ) + return findings + + +@semantic_rule( + name="merge-without-commit-guard", + description=( + "A merge_worktree step has no commit_guard predecessor. Any path reaching " + "merge with uncommitted changes will fail at the dirty-tree gate, burning " + "an expensive recovery cycle. Add a commit_guard run_cmd step before merge." + ), + severity=Severity.ERROR, +) +def _check_merge_without_commit_guard(ctx: ValidationContext) -> list[RuleFinding]: + findings: list[RuleFinding] = [] + for step_name, step in ctx.recipe.steps.items(): + if step.tool != "merge_worktree": + continue + if not _has_commit_guard_ancestor(step_name, ctx): + findings.append( + RuleFinding( + rule="merge-without-commit-guard", + severity=Severity.ERROR, + step_name=step_name, + message=( + f"merge_worktree step '{step_name}' has no commit_guard predecessor. " + f"Uncommitted changes from context-exhausted skills will trigger " + f"the dirty-tree gate, causing an expensive recovery cycle. " + f"Add a commit_guard run_cmd step immediately before this step." + ), + ) + ) + return findings + + +# --------------------------------------------------------------------------- +# release-issue-on-unconfirmed-merge rule +# --------------------------------------------------------------------------- + +_TIMEOUT_CONDITION_RE = re.compile(r"timeout", re.IGNORECASE) +_MERGE_WAIT_TOOLS = frozenset({"wait_for_merge_queue"}) +_DIRECT_WAIT_NAMES = re.compile(r"wait_for_(direct|immediate)_merge") +_REGISTER_CLONE_UNCONFIRMED = "register_clone_unconfirmed" + + +def _collect_timeout_exit_steps(ctx: ValidationContext) -> set[str]: + """Collect step names that are timeout exits from merge-wait steps. + + A timeout exit is any step name that appears as the route of an on_result + condition containing 'timeout' in its when-expression, where the source + step is a merge-wait step (wait_for_merge_queue, wait_for_direct_merge, + wait_for_immediate_merge run_cmd steps). The on_failure of any merge-wait + step is also treated as a timeout exit (tool error is also unconfirmed). + """ + exits: set[str] = set() + for step_name, step in ctx.recipe.steps.items(): + is_merge_wait = step.tool in _MERGE_WAIT_TOOLS or ( + step.tool == "run_cmd" and bool(_DIRECT_WAIT_NAMES.search(step_name)) + ) + if not is_merge_wait: + continue + if step.on_result and step.on_result.conditions: + for cond in step.on_result.conditions: + if cond.when and _TIMEOUT_CONDITION_RE.search(cond.when): + exits.add(cond.route) + if step.on_failure: + exits.add(step.on_failure) + return exits + + +@semantic_rule( + name="release-issue-on-unconfirmed-merge", + description=( + "A release_issue step must not be reachable from a merge-wait timeout exit. " + "When wait_for_merge_queue / wait_for_direct_merge / wait_for_immediate_merge " + "times out, the PR is still actively in the queue. Calling release_issue removes " + "the in-progress label, leaving the issue visually unclaimed while the merge is " + "still pending. Route timeout exits to register_clone_unconfirmed instead." + ), + severity=Severity.ERROR, +) +def _check_release_issue_on_unconfirmed_merge(ctx: ValidationContext) -> list[RuleFinding]: + timeout_exits = _collect_timeout_exit_steps(ctx) + if not timeout_exits: + return [] + + # BFS from all timeout exits using the forward step_graph + reachable: set[str] = set() + frontier = set(timeout_exits) & set(ctx.step_graph) + while frontier: + reachable |= frontier + next_frontier: set[str] = set() + for name in frontier: + next_frontier |= ctx.step_graph.get(name, set()) - reachable + frontier = next_frontier + + findings: list[RuleFinding] = [] + for step_name in reachable: + step = ctx.recipe.steps.get(step_name) + if step and step.tool == "release_issue": + findings.append( + RuleFinding( + rule="release-issue-on-unconfirmed-merge", + severity=Severity.ERROR, + step_name=step_name, + message=( + f"Step '{step_name}' calls release_issue but is reachable from a " + f"merge-wait timeout exit ({sorted(timeout_exits)}). " + f"Calling release_issue on a timeout path removes the in-progress label " + f"while the PR may still be queued. Replace with " + f"{_REGISTER_CLONE_UNCONFIRMED} (status: unconfirmed) so the label" + f" is kept." + ), + ) + ) + return findings + + +# --------------------------------------------------------------------------- +# merge-enrollment-auto-consistency rule +# --------------------------------------------------------------------------- + +_AUTO_MERGE_FALSE_PATTERN = re.compile( + r"auto_merge_available\s*==\s*['\"]?false['\"]?", re.IGNORECASE +) + + +def _is_auto_flagged_step(step_name: str, ctx: ValidationContext) -> bool: + """Return True if step uses --auto in a gh pr merge command or calls toggle_auto_merge.""" + step = ctx.recipe.steps.get(step_name) + if step is None: + return False + if step.tool == "run_cmd": + cmd = step.with_args.get("cmd", "") + if isinstance(cmd, str) and "gh pr merge" in cmd and "--auto" in cmd: + return True + if step.tool == "toggle_auto_merge": + return True + return False + + +@semantic_rule( + name="merge-enrollment-auto-consistency", + description=( + "gh pr merge steps with --auto must not be reachable from auto_merge_available=false " + "routing arms. When auto_merge_available is false, --auto and toggle_auto_merge will " + "fail because the repository does not support enablePullRequestAutoMerge." + ), + severity=Severity.ERROR, +) +def _check_merge_enrollment_auto_consistency(ctx: ValidationContext) -> list[RuleFinding]: + findings: list[RuleFinding] = [] + + no_auto_targets: set[str] = set() + for step_name, step in ctx.recipe.steps.items(): + if step.on_result and step.on_result.conditions: + for cond in step.on_result.conditions: + if cond.when and _AUTO_MERGE_FALSE_PATTERN.search(cond.when): + no_auto_targets.add(cond.route) + + for target in no_auto_targets: + reachable = bfs_reachable(ctx.step_graph, target) | {target} + for reached in reachable: + if _is_auto_flagged_step(reached, ctx): + findings.append( + RuleFinding( + rule="merge-enrollment-auto-consistency", + severity=Severity.ERROR, + step_name=reached, + message=( + f"Step '{reached}' uses --auto or toggle_auto_merge but is " + f"reachable from an auto_merge_available=false routing arm " + f"(via '{target}'). Use enqueue_pr instead." + ), + ) + ) + return findings diff --git a/src/autoskillit/recipe/rules/rules_merge_queue.py b/src/autoskillit/recipe/rules/rules_merge_queue.py new file mode 100644 index 0000000000..287ca4f89d --- /dev/null +++ b/src/autoskillit/recipe/rules/rules_merge_queue.py @@ -0,0 +1,137 @@ +"""Semantic rules for merge queue push routing completeness.""" + +from __future__ import annotations + +from autoskillit.core import Severity +from autoskillit.recipe._analysis import ValidationContext +from autoskillit.recipe._analysis_graph import _extract_routing_edges +from autoskillit.recipe.registry import RuleFinding, semantic_rule + +_MERGE_QUEUE_WAIT_TOOLS = frozenset({"wait_for_merge_queue"}) +_PUSH_TOOLS = frozenset({"push_to_remote"}) +_FLOW_BOUNDARY_TOOLS = frozenset( + { + "wait_for_merge_queue", + "wait_for_ci", + "wait_for_direct_merge", + "wait_for_immediate_merge", + } +) + + +def _build_raw_routing_graph(ctx: ValidationContext) -> dict[str, set[str]]: + """Build a routing adjacency list without skip_when_false bypass edges. + + ``ctx.step_graph`` includes bypass edges that jump past skippable steps, + creating paths around flow-boundary steps. For ejection-flow reachability + we need the direct routing graph only. + """ + step_names = set(ctx.recipe.steps.keys()) + graph: dict[str, set[str]] = {name: set() for name in step_names} + for name, step in ctx.recipe.steps.items(): + for edge in _extract_routing_edges(step): + if edge.edge_type == "exhausted" and step.action is not None: + continue + if edge.target in step_names: + graph[name].add(edge.target) + return graph + + +def _collect_ejection_exit_steps(ctx: ValidationContext) -> set[str]: + """Collect step names that are pure-ejection-route exits from merge-queue-wait steps. + + Only matches conditions for the ``ejected`` state, not ``ejected_ci_failure`` + or ``dropped_*`` variants. Excludes routes whose target is itself a + ``wait_for_merge_queue`` step (re-enrollment, not ejection recovery). + """ + exits: set[str] = set() + for _name, step in ctx.recipe.steps.items(): + if step.tool not in _MERGE_QUEUE_WAIT_TOOLS: + continue + if step.on_result and step.on_result.conditions: + for cond in step.on_result.conditions: + if not cond.when: + continue + when_lower = cond.when.lower() + if "ejected" in when_lower and "ejected_ci_failure" not in when_lower: + target = ctx.recipe.steps.get(cond.route) + if target and target.tool in _MERGE_QUEUE_WAIT_TOOLS: + continue + exits.add(cond.route) + return exits + + +def _bfs_forward(graph: dict[str, set[str]], starts: set[str], ctx: ValidationContext) -> set[str]: + """BFS from start nodes, stopping at flow-boundary step tools.""" + reachable: set[str] = set() + frontier = starts & set(graph) + while frontier: + reachable |= frontier + next_frontier: set[str] = set() + for name in frontier: + for succ in graph.get(name, set()) - reachable: + step = ctx.recipe.steps.get(succ) + if step and step.tool in _FLOW_BOUNDARY_TOOLS and succ not in starts: + continue + next_frontier.add(succ) + frontier = next_frontier + return reachable + + +def _step_has_queued_branch_route(step_name: str, ctx: ValidationContext) -> bool: + """Check if a step's failure target chain includes a queued_branch route.""" + step = ctx.recipe.steps.get(step_name) + if not step or not step.on_failure: + return False + failure_target = step.on_failure + target_step = ctx.recipe.steps.get(failure_target) + if not target_step: + return False + edges = _extract_routing_edges(target_step) + for edge in edges: + if edge.edge_type == "result_condition" and edge.condition: + if "queued_branch" in edge.condition: + return True + return False + + +@semantic_rule( + name="push-after-queue-requires-queued-branch-route", + description=( + "A push_to_remote step reachable from a merge-queue ejection route must have " + "its on_failure target include a queued_branch error route. Without this, a " + "GH006 rejection from pushing to a still-protected branch causes terminal " + "failure instead of recoverable retry." + ), + severity=Severity.ERROR, +) +def _check_push_after_queue_has_queued_branch_route( + ctx: ValidationContext, +) -> list[RuleFinding]: + ejection_exits = _collect_ejection_exit_steps(ctx) + if not ejection_exits: + return [] + + raw_graph = _build_raw_routing_graph(ctx) + reachable = _bfs_forward(raw_graph, ejection_exits, ctx) + + findings: list[RuleFinding] = [] + for step_name in reachable: + step = ctx.recipe.steps.get(step_name) + if not step or step.tool not in _PUSH_TOOLS: + continue + if not _step_has_queued_branch_route(step_name, ctx): + findings.append( + RuleFinding( + rule="push-after-queue-requires-queued-branch-route", + severity=Severity.ERROR, + step_name=step_name, + message=( + f"Step '{step_name}' uses push_to_remote and is reachable from a " + f"merge-queue ejection route, but its on_failure chain does not " + f"include a 'queued_branch' error route. Add a classify_push_failure " + f"step that routes queued_branch errors to a dequeue-retry path." + ), + ) + ) + return findings diff --git a/src/autoskillit/recipe/rules/rules_packs.py b/src/autoskillit/recipe/rules/rules_packs.py new file mode 100644 index 0000000000..ba784aa8f0 --- /dev/null +++ b/src/autoskillit/recipe/rules/rules_packs.py @@ -0,0 +1,86 @@ +"""Semantic rules for pack validation in recipe pipelines.""" + +from __future__ import annotations + +from autoskillit.core import PACK_REGISTRY, SKILL_TOOLS, Severity +from autoskillit.recipe._analysis import ValidationContext +from autoskillit.recipe._skill_helpers import _get_skill_category_map +from autoskillit.recipe.contracts import resolve_skill_name +from autoskillit.recipe.registry import RuleFinding, semantic_rule + + +@semantic_rule( + name="unknown-required-pack", + description="Pack name in requires_packs is not in PACK_REGISTRY", + severity=Severity.ERROR, +) +def _check_unknown_required_pack(ctx: ValidationContext) -> list[RuleFinding]: + findings = [] + seen_reported: set[str] = set() + for pack_name in ctx.recipe.requires_packs: + if pack_name not in PACK_REGISTRY and pack_name not in seen_reported: + seen_reported.add(pack_name) + findings.append( + RuleFinding( + rule="unknown-required-pack", + severity=Severity.ERROR, + step_name="(top-level)", + message=( + f"Pack {pack_name!r} in requires_packs is not in PACK_REGISTRY. " + f"Known packs: {sorted(PACK_REGISTRY)}" + ), + ) + ) + return findings + + +@semantic_rule( + name="undeclared-pack-requirement", + description=( + "Recipes dispatching skills in default-disabled pack categories " + "must declare those packs in requires_packs" + ), + severity=Severity.ERROR, +) +def _check_undeclared_pack_requirement(ctx: ValidationContext) -> list[RuleFinding]: + """Flag recipes that dispatch skills in default-disabled pack categories without + declaring the corresponding requires_packs entry. + + Mirrors the pattern of check_requires_features_declared in rules_features.py. + PACK_REGISTRY keys directly equal CATEGORY_TAGS, so no intermediate mapping + (like FeatureDef.skill_categories) is needed — a category that appears in + PACK_REGISTRY with default_enabled=False is exactly a pack that must be declared. + """ + category_map = ( + ctx.skill_category_map if ctx.skill_category_map is not None else _get_skill_category_map() + ) + declared_packs = frozenset(ctx.recipe.requires_packs) + default_disabled = frozenset( + name for name, pdef in PACK_REGISTRY.items() if not pdef.default_enabled + ) + + findings: list[RuleFinding] = [] + for step_name, step in ctx.recipe.steps.items(): + if step.tool not in SKILL_TOOLS: + continue + skill_cmd = (step.with_args or {}).get("skill_command") or "" + skill_name = resolve_skill_name(skill_cmd) + if skill_name is None: + continue + categories = category_map.get(skill_name, frozenset()) + for cat in sorted(categories & default_disabled): + if cat not in declared_packs: + findings.append( + RuleFinding( + rule="undeclared-pack-requirement", + severity=Severity.ERROR, + step_name=step_name, + message=( + f"step '{step_name}': skill_command '{skill_cmd}' references " + f"skill '{skill_name}' which belongs to default-disabled pack " + f"'{cat}'. Add '{cat}' to this recipe's requires_packs so " + f"init_session can enable the pack gate." + ), + ) + ) + return findings diff --git a/src/autoskillit/recipe/rules_reachability.py b/src/autoskillit/recipe/rules/rules_reachability.py similarity index 98% rename from src/autoskillit/recipe/rules_reachability.py rename to src/autoskillit/recipe/rules/rules_reachability.py index f2db3f82f8..75ea8b8fa3 100644 --- a/src/autoskillit/recipe/rules_reachability.py +++ b/src/autoskillit/recipe/rules/rules_reachability.py @@ -19,13 +19,13 @@ from __future__ import annotations -import re +import regex as re from autoskillit.core import Severity from autoskillit.recipe._analysis import ( ValidationContext, - _bfs_reachable, _bfs_with_facts, + bfs_reachable, ) from autoskillit.recipe.registry import RuleFinding, semantic_rule @@ -61,7 +61,7 @@ def _find_capture_producers(ctx: ValidationContext, var: str) -> list[str]: def _ancestors(ctx: ValidationContext, step_name: str) -> set[str]: """Return all steps reachable via backward BFS from step_name (i.e. ancestors).""" - return _bfs_reachable(ctx.predecessors, step_name) + return bfs_reachable(ctx.predecessors, step_name) @semantic_rule( diff --git a/src/autoskillit/recipe/rules_recipe.py b/src/autoskillit/recipe/rules/rules_recipe.py similarity index 79% rename from src/autoskillit/recipe/rules_recipe.py rename to src/autoskillit/recipe/rules/rules_recipe.py index 44e265577c..602943f34b 100644 --- a/src/autoskillit/recipe/rules_recipe.py +++ b/src/autoskillit/recipe/rules/rules_recipe.py @@ -1,4 +1,4 @@ -"""Semantic rules for sub-recipe reference validity.""" +"""Semantic rules for sub-recipe reference validity and with_args hygiene.""" from __future__ import annotations @@ -15,6 +15,31 @@ logger = get_logger(__name__) +@semantic_rule( + name="env-key-in-with-args", + description="step with_args must not contain an env: key (ADR-0003)", + severity=Severity.ERROR, +) +def _check_env_key_in_with_args(ctx: ValidationContext) -> list[RuleFinding]: + findings: list[RuleFinding] = [] + for step_name, step in ctx.recipe.steps.items(): + if "env" in step.with_args: + findings.append( + RuleFinding( + rule="env-key-in-with-args", + severity=Severity.ERROR, + step_name=step_name, + message=( + f"step '{step_name}' contains an env: key in with_args. " + f"Environment variables are not delivered to headless " + f"subprocesses — use positional arguments in skill_command " + f"instead. See ADR-0003." + ), + ) + ) + return findings + + @semantic_rule( name="unknown-sub-recipe", description="sub_recipe step must reference a known sub-recipe name", diff --git a/src/autoskillit/recipe/rules_skill_content.py b/src/autoskillit/recipe/rules/rules_skill_content.py similarity index 58% rename from src/autoskillit/recipe/rules_skill_content.py rename to src/autoskillit/recipe/rules/rules_skill_content.py index 6f9fc7ed05..0521f3d6ce 100644 --- a/src/autoskillit/recipe/rules_skill_content.py +++ b/src/autoskillit/recipe/rules/rules_skill_content.py @@ -7,15 +7,17 @@ from __future__ import annotations -import re from pathlib import Path from typing import TYPE_CHECKING +import regex as re + from autoskillit.core import Severity if TYPE_CHECKING: from autoskillit.core import SkillResolver from autoskillit.recipe._analysis import ValidationContext +from autoskillit.recipe._git_helpers import _GIT_REMOTE_COMMAND_RE, _LITERAL_ORIGIN_RE from autoskillit.recipe._skill_placeholder_parser import ( extract_bash_blocks, extract_bash_placeholders, @@ -40,6 +42,13 @@ # bash blocks that reference it. ("implement-experiment", "slug"), ("generate-report", "slug"), + ("setup-environment", "slug"), + ("promote-to-main", "branch"), + ("promote-to-main", "merge_base_sha"), + ("promote-to-main", "number"), + ("promote-to-main", "pr_title"), + ("promote-to-main", "pr_url"), + ("promote-to-main", "timestamp"), } ) @@ -87,7 +96,7 @@ def _check_undefined_bash_placeholder(ctx: ValidationContext) -> list[RuleFindin if skill_name is None: continue - skill_md = _resolve_skill_md(skill_name) + skill_md = _resolve_skill_md(skill_name, resolver=ctx.skill_resolver) if skill_md is None: continue # unknown-skill-command rule handles missing skills @@ -125,14 +134,6 @@ def _check_undefined_bash_placeholder(ctx: ValidationContext) -> list[RuleFindin return findings -_GIT_REMOTE_COMMAND_RE: re.Pattern[str] = re.compile( - r"\bgit\b.*?\b(?:fetch|rebase|log|show|rev-parse)\b" -) -# Matches literal 'origin' not immediately preceded by $, {, or - (i.e., not a shell -# variable reference or shell default-value expression like ${REMOTE:-origin}). -_LITERAL_ORIGIN_RE: re.Pattern[str] = re.compile(r"(? bool: """Return True if any non-comment bash line uses literal 'origin' in a git remote command.""" for block in bash_blocks: @@ -169,7 +170,7 @@ def _check_hardcoded_origin_remote(ctx: ValidationContext) -> list[RuleFinding]: skill_name = resolve_skill_name(skill_cmd) if skill_name is None: continue - skill_md = _resolve_skill_md(skill_name) + skill_md = _resolve_skill_md(skill_name, resolver=ctx.skill_resolver) if skill_md is None: continue # unknown-skill-command rule handles missing skills try: @@ -226,7 +227,7 @@ def _check_no_autoskillit_import(ctx: ValidationContext) -> list[RuleFinding]: skill_name = resolve_skill_name(skill_cmd) if skill_name is None: continue - skill_md = _resolve_skill_md(skill_name) + skill_md = _resolve_skill_md(skill_name, resolver=ctx.skill_resolver) if skill_md is None: continue try: @@ -254,6 +255,76 @@ def _check_no_autoskillit_import(ctx: ValidationContext) -> list[RuleFinding]: return findings +_GREP_BRE_ALTERNATION_RE: re.Pattern[str] = re.compile( + r""" + (? list[RuleFinding]: + findings: list[RuleFinding] = [] + for step_name, step in ctx.recipe.steps.items(): + if step.tool != "run_skill": + continue + skill_cmd = step.with_args.get("skill_command", "") + if not skill_cmd: + continue + skill_name = resolve_skill_name(skill_cmd) + if skill_name is None: + continue + skill_md = _resolve_skill_md(skill_name, resolver=ctx.skill_resolver) + if skill_md is None: + continue # unknown-skill-command rule handles missing skills + try: + content = skill_md.read_text(encoding="utf-8") + except OSError: + continue + bash_blocks = extract_bash_blocks(content) + if not bash_blocks: + continue + violations: list[str] = [] + for block in bash_blocks: + for line in block.splitlines(): + if _GIT_GREP_BRE_RE.search(line): + continue # git --grep= BRE context: allowed + if _GREP_BRE_ALTERNATION_RE.search(line): + violations.append(line.strip()) + if violations: + findings.append( + RuleFinding( + rule="grep-bre-alternation-in-skill", + severity=Severity.ERROR, + step_name=step_name, + message=( + f"Skill '{skill_name}' bash block uses grep BRE \\| alternation " + f"in {len(violations)} line(s). The Grep tool wraps ripgrep (ERE) " + f"where | (bare) is alternation — \\| silently returns 0 results. " + f"Fix: replace grep 'foo\\|bar' with rg 'foo|bar'. " + f"Violations: {violations!r}" + ), + ) + ) + return findings + + _NO_MARKDOWN_DIRECTIVE_PATTERN: re.Pattern[str] = re.compile( r"no\s+markdown\s+format|plain\s+text.*token|literal\s+plain\s+text", re.IGNORECASE, @@ -293,17 +364,17 @@ def _check_output_section_no_markdown_directive(ctx: ValidationContext) -> list[ if not skill_data or not skill_data.get("expected_output_patterns"): continue # Only check skills that have contracts with patterns - skill_md_path = _resolve_skill_md(skill_name) - if skill_md_path is None: + skill_md = _resolve_skill_md(skill_name, resolver=ctx.skill_resolver) + if skill_md is None: continue # unknown-skill-command rule handles missing skills try: - skill_md = skill_md_path.read_text(encoding="utf-8") + content = skill_md.read_text(encoding="utf-8") except OSError: continue # file deleted or unreadable between resolution and read output_section_match = re.search( - r"##\s+Output\b(.+?)(?:^##|\Z)", skill_md, re.DOTALL | re.MULTILINE + r"##\s+Output\b(.+?)(?:^##|\Z)", content, re.DOTALL | re.MULTILINE ) if not output_section_match: continue # No output section — other rules handle this @@ -325,3 +396,143 @@ def _check_output_section_no_markdown_directive(ctx: ValidationContext) -> list[ ) ) return findings + + +_GH_ISSUE_COMMENT_RE = re.compile(r"\bgh\s+issue\s+comment\b") + + +@semantic_rule( + name="skill-no-issue-comments", + description=( + "Skill content must not use 'gh issue comment'. " + "All issue updates belong in the body via 'gh issue edit --body-file'." + ), +) +def _check_no_gh_issue_comment(ctx: ValidationContext) -> list[RuleFinding]: + findings: list[RuleFinding] = [] + for step_name, step in ctx.recipe.steps.items(): + if step.tool != "run_skill": + continue + skill_cmd = step.with_args.get("skill_command", "") + skill_name = resolve_skill_name(skill_cmd) + if skill_name is None: + continue + skill_md = _resolve_skill_md(skill_name, resolver=ctx.skill_resolver) + if skill_md is None: + continue + try: + content = skill_md.read_text(encoding="utf-8") + except OSError: + continue + for block in extract_bash_blocks(content): + if _GH_ISSUE_COMMENT_RE.search(block): + findings.append( + RuleFinding( + rule="skill-no-issue-comments", + severity=Severity.ERROR, + step_name=step_name, + message=( + f"Skill '{skill_name}' contains 'gh issue comment'. " + "Use 'gh issue edit --body-file' instead." + ), + ) + ) + break + return findings + + +_TRANSITION_BOUNDARY_RES: list[tuple[str, re.Pattern[str]]] = [ + ( + "group-iteration", + re.compile( + r"group\s+(?:iteration|N\+1|dispatch|ordering)|between\s+groups" + r"|merge[- ]wait.*between|verify.*merged.*before.*(?:start|Group)", + re.IGNORECASE, + ), + ), + ( + "phase-loop", + re.compile( + r"(?:phase|step)\s+(?:loop|iteration|repeat)|between\s+phases" + r"|phase\s+by\s+phase|for\s+each\s+(?:[\w/]+\s+)?phase", + re.IGNORECASE, + ), + ), + ( + "batch-processing", + re.compile( + r"batch\s+(?:N|iteration|processing)|between\s+(?:batches|issues)", + re.IGNORECASE, + ), + ), +] + +_TRANSITION_ANTI_CONFIRM_RE: re.Pattern[str] = re.compile( + r"(?:never|do\s+not|must\s+not|prohibited)" + r"[^\n]{0,80}" + r"(?:ask|confirm|pause|AskUserQuestion" + r"|wait\s+for\s+[^\n]{0,30}(?:user|human|permission|approval|confirmation))" + r"|(?:proceed|dispatch|continue|start)" + r"[^\n]{0,60}" + r"(?:immediately|without[^\n]{0,30}(?:asking|confirming|pausing|waiting))", + re.IGNORECASE, +) + + +def _extract_sections(content: str) -> list[str]: + """Split SKILL.md content into heading-scoped sections at the ## level.""" + parts = re.split(r"(?m)^(?=##\s)", content) + return [p for p in parts if p.strip()] + + +@semantic_rule( + name="transition-boundary-anti-confirmation", + description=( + "A SKILL.md section describes a transition boundary (between groups, phases, or batches) " + "but does not contain an explicit anti-confirmation instruction. Without it, the model " + "may insert unsolicited confirmation prompts between iterations." + ), +) +def _check_transition_boundary_anti_confirmation(ctx: ValidationContext) -> list[RuleFinding]: + """Fire for any run_skill step whose SKILL.md has unprotected transition boundaries.""" + findings: list[RuleFinding] = [] + for step_name, step in ctx.recipe.steps.items(): + if step.tool != "run_skill": + continue + skill_cmd = step.with_args.get("skill_command", "") + if not skill_cmd: + continue + skill_name = resolve_skill_name(skill_cmd) + if skill_name is None: + continue + skill_md = _resolve_skill_md(skill_name, resolver=ctx.skill_resolver) + if skill_md is None: + continue + try: + content = skill_md.read_text(encoding="utf-8") + except OSError: + continue + unprotected: list[tuple[str, str]] = [] + for section in _extract_sections(content): + lines = section.splitlines() + heading = lines[0].strip() if lines else "" + for boundary_name, boundary_re in _TRANSITION_BOUNDARY_RES: + if boundary_re.search(section) and not _TRANSITION_ANTI_CONFIRM_RE.search(section): + unprotected.append((boundary_name, heading)) + break + if unprotected: + boundary_desc = "; ".join(f"'{b}' in section '{h}'" for b, h in unprotected) + findings.append( + RuleFinding( + rule="transition-boundary-anti-confirmation", + severity=Severity.WARNING, + step_name=step_name, + message=( + f"Skill '{skill_name}' has unprotected transition " + f"{'boundaries' if len(unprotected) > 1 else 'boundary'}: " + f"{boundary_desc}. Add explicit anti-confirmation instruction " + f"(e.g., 'NEVER use AskUserQuestion to confirm proceeding')." + ), + ) + ) + return findings diff --git a/src/autoskillit/recipe/rules_skills.py b/src/autoskillit/recipe/rules/rules_skills.py similarity index 94% rename from src/autoskillit/recipe/rules_skills.py rename to src/autoskillit/recipe/rules/rules_skills.py index 0be72cbbc9..e1f9548854 100644 --- a/src/autoskillit/recipe/rules_skills.py +++ b/src/autoskillit/recipe/rules/rules_skills.py @@ -2,10 +2,11 @@ from __future__ import annotations -import re +import regex as re from autoskillit.core import AUTOSKILLIT_SKILL_PREFIX, SKILL_TOOLS, Severity, SkillLister from autoskillit.recipe._analysis import ValidationContext +from autoskillit.recipe._skill_helpers import _get_skill_category_map from autoskillit.recipe.contracts import resolve_skill_name from autoskillit.recipe.registry import RuleFinding, semantic_rule @@ -18,15 +19,6 @@ def _get_bundled_skill_names(lister: SkillLister | None = None) -> frozenset[str return frozenset(s.name for s in lister.list_all()) -def _get_skill_category_map(lister: SkillLister | None = None) -> dict[str, frozenset[str]]: - """Return {skill_name: categories} for all bundled skills.""" - if lister is None: - from autoskillit.workspace import DefaultSkillResolver # noqa: PLC0415 - - lister = DefaultSkillResolver() - return {s.name: s.categories for s in lister.list_all()} - - _SKILL_TOKEN_RE = re.compile(r"/autoskillit:(\S+)") @@ -87,6 +79,12 @@ def _check_unknown_skill_command(ctx: ValidationContext) -> list[RuleFinding]: severity=Severity.WARNING, ) def _check_subset_disabled_skill(ctx: ValidationContext) -> list[RuleFinding]: + """Flag steps that reference skills disabled by the user's subsets.disabled config. + + Note: This rule checks USER CONFIG vs RECIPE compatibility. + For RECIPE DECLARATION completeness (requires_packs vs skill categories), + see 'undeclared-pack-requirement' in rules_packs.py. + """ if not ctx.disabled_subsets: return [] category_map = ( diff --git a/src/autoskillit/recipe/rules/rules_temp_path.py b/src/autoskillit/recipe/rules/rules_temp_path.py new file mode 100644 index 0000000000..e4e13431b3 --- /dev/null +++ b/src/autoskillit/recipe/rules/rules_temp_path.py @@ -0,0 +1,62 @@ +"""Lint rule: reject bare {{AUTOSKILLIT_TEMP}}/ paths without a context-variable scope prefix.""" + +from __future__ import annotations + +from collections.abc import Iterator + +import regex as re + +from autoskillit.core import Severity +from autoskillit.recipe._analysis import ValidationContext +from autoskillit.recipe.registry import RuleFinding, semantic_rule +from autoskillit.recipe.schema import RecipeStep + +_BARE_TEMP_RE = re.compile(r"\{\{AUTOSKILLIT_TEMP\}\}/") +_CONTEXT_SCOPED_RE = re.compile(r"\$\{\{\s*context\.\w+\s*\}\}") +_SKIP_KEYS = frozenset({"step_name", "callable", "pass_name"}) + + +def _iter_path_values(step: RecipeStep) -> Iterator[tuple[str, str]]: + for key, val in (step.with_args or {}).items(): + if key in _SKIP_KEYS: + continue + if key == "env" and isinstance(val, dict): + for env_key, env_val in val.items(): + if isinstance(env_val, str): + yield f"env.{env_key}", env_val + elif isinstance(val, str): + yield key, val + elif isinstance(val, list): + for i, item in enumerate(val): + if isinstance(item, str): + yield f"{key}[{i}]", item + + +@semantic_rule( + name="non-unique-output-path", + description=( + "Recipe steps must scope output paths through a per-run context variable. " + "Bare {{AUTOSKILLIT_TEMP}}/ paths are shared across runs and cause " + "stale-artifact failures." + ), + severity=Severity.ERROR, +) +def _check_non_unique_output_path(ctx: ValidationContext) -> list[RuleFinding]: + findings = [] + for step_name, step in ctx.recipe.steps.items(): + for key, val in _iter_path_values(step): + if _BARE_TEMP_RE.search(val) and not _CONTEXT_SCOPED_RE.search(val): + findings.append( + RuleFinding( + rule="non-unique-output-path", + severity=Severity.ERROR, + step_name=step_name, + message=( + f"Step '{step_name}' uses a bare '{{{{AUTOSKILLIT_TEMP}}}}/' " + f"path in '{key}' without a context-variable scope prefix. " + "Capture a unique per-run directory in the init step " + "and reference it via ${{{{ context.run_dir }}}} or similar." + ), + ) + ) + return findings diff --git a/src/autoskillit/recipe/rules_tools.py b/src/autoskillit/recipe/rules/rules_tools.py similarity index 73% rename from src/autoskillit/recipe/rules_tools.py rename to src/autoskillit/recipe/rules/rules_tools.py index a0eb096110..1f7a24565c 100644 --- a/src/autoskillit/recipe/rules_tools.py +++ b/src/autoskillit/recipe/rules/rules_tools.py @@ -21,6 +21,8 @@ "order_id", "stale_threshold", "idle_output_timeout", + "output_dir", + "resume_session_id", } ), "run_cmd": frozenset({"cmd", "cwd", "timeout", "step_name"}), @@ -34,7 +36,7 @@ # --- Recipe tools --- "validate_recipe": frozenset({"script_path"}), "migrate_recipe": frozenset({"name"}), - "load_recipe": frozenset({"name", "overrides"}), + "load_recipe": frozenset({"name", "overrides", "ingredients_only"}), "list_recipes": frozenset(), # --- Clone tools --- "clone_repo": frozenset( @@ -70,12 +72,14 @@ { "registry_path", "all_owners", + "owner_filter", "step_name", } ), # --- CI tools --- "wait_for_ci": frozenset( { + "auto_trigger", "branch", "repo", "remote_url", @@ -83,6 +87,7 @@ "workflow", "event", "timeout_seconds", + "lookback_seconds", "cwd", "step_name", } @@ -100,6 +105,18 @@ "max_stall_retries", "not_in_queue_confirmation_cycles", "max_inconclusive_retries", + "auto_merge_available", + "step_name", + } + ), + "enqueue_pr": frozenset( + { + "pr_number", + "target_branch", + "cwd", + "auto_merge_available", + "repo", + "remote_url", "step_name", } ), @@ -149,7 +166,9 @@ ), "enrich_issues": frozenset({"issue_number", "batch", "dry_run", "repo"}), "claim_issue": frozenset({"issue_url", "label", "allow_reentry"}), - "release_issue": frozenset({"issue_url", "label", "target_branch", "staged_label"}), + "release_issue": frozenset( + {"issue_url", "label", "target_branch", "staged_label", "fail_label"} + ), "fetch_github_issue": frozenset({"issue_url", "include_comments"}), "get_issue_title": frozenset({"issue_url"}), # --- Status tools --- @@ -320,3 +339,78 @@ def _check_rebase_then_push_requires_force(ctx: ValidationContext) -> list[RuleF ) break # one finding per push step is sufficient return findings + + +@semantic_rule( + name="release-issue-requires-disposition", + description="release_issue must have fail_label or target_branch to avoid bare removal", + severity=Severity.ERROR, +) +def _check_release_issue_requires_disposition(ctx: ValidationContext) -> list[RuleFinding]: + findings: list[RuleFinding] = [] + for step_name, step in ctx.recipe.steps.items(): + if step.tool is None or step.tool != "release_issue": + continue + has_fail_label = bool(step.with_args.get("fail_label")) + has_target_branch = bool(step.with_args.get("target_branch")) + if not has_fail_label and not has_target_branch: + findings.append( + RuleFinding( + rule="release-issue-requires-disposition", + severity=Severity.ERROR, + step_name=step_name, + message=( + f"Step '{step_name}' calls release_issue without fail_label or " + f"target_branch — this performs bare label removal with no " + f"replacement. Add fail_label for failure paths or target_branch " + f"for success/staging paths." + ), + ) + ) + return findings + + +@semantic_rule( + name="patch-token-summary-requires-order-id", + description=( + "run_python step calling patch_pr_token_summary should pass order_id " + "for correct multi-clone scoping" + ), + severity=Severity.WARNING, +) +def _check_patch_token_summary_order_id(ctx: ValidationContext) -> list[RuleFinding]: + """Warn when a patch_pr_token_summary step does not pass order_id. + + patch_pr_token_summary uses order_id as the canonical scoping key for fleet + sessions where a single order spans multiple clone directories. Without it, + the function falls back to cwd_filter which misses sessions from other clones. + + The env-based auto-propagation (AUTOSKILLIT_DISPATCH_ID) handles runtime + scoping for fleet sessions automatically; this rule serves as a documentation + guard alerting recipe authors to the scoping requirement when building + non-fleet callers or when explicit order_id is needed. + """ + findings: list[RuleFinding] = [] + for step_name, step in ctx.recipe.steps.items(): + if step.tool != "run_python": + continue + callable_val = step.with_args.get("callable", "") + if "patch_pr_token_summary" not in str(callable_val): + continue + if "order_id" not in step.with_args: + findings.append( + RuleFinding( + rule="patch-token-summary-requires-order-id", + severity=Severity.WARNING, + step_name=step_name, + message=( + f"step '{step_name}': patch_pr_token_summary call is missing " + f"'order_id' in with args. " + "Fleet sessions auto-propagate via AUTOSKILLIT_DISPATCH_ID, " + "but non-fleet callers need explicit order_id for cross-clone " + "scoping. Add 'order_id: \"${{{{ context.order_id }}}}\"' or " + "similar to suppress this warning." + ), + ) + ) + return findings diff --git a/src/autoskillit/recipe/rules/rules_verdict.py b/src/autoskillit/recipe/rules/rules_verdict.py new file mode 100644 index 0000000000..bc68988e27 --- /dev/null +++ b/src/autoskillit/recipe/rules/rules_verdict.py @@ -0,0 +1,295 @@ +"""Semantic rules for skill verdict routing completeness and consistency. + +Ensures that recipe steps capturing a skill verdict with declared allowed_values +have an explicit on_result condition for every allowed value. A catch-all +`when: true` route does not count as explicit handling. + +Also ensures that steps invoking the same skill route the same verdict value +to the same outcome category (continuation vs escalation). +""" + +from __future__ import annotations + +import regex as re + +from autoskillit.core import Severity, get_logger, resolve_skill_name +from autoskillit.recipe._analysis import ValidationContext +from autoskillit.recipe.contracts import load_bundled_manifest +from autoskillit.recipe.registry import RuleFinding, semantic_rule + +logger = get_logger(__name__) + + +def _get_allowed_values_for_skill(skill_name: str) -> dict[str, list[str]]: + """Return {output_name: [allowed_value, ...]} for a skill's outputs with allowed_values.""" + try: + manifest = load_bundled_manifest() + except Exception: + logger.warning( + "unrouted-verdict-value: failed to load bundled manifest; skipping", exc_info=True + ) + return {} + skill_contract = manifest.get("skills", {}).get(skill_name, {}) + result: dict[str, list[str]] = {} + for output in skill_contract.get("outputs", []): + if "allowed_values" in output: + result[output["name"]] = output["allowed_values"] + return result + + +def _is_explicit_condition(when: str | None, value: str) -> bool: + """A condition is explicit if it references the value and is not the catch-all.""" + if not when: + return False + if when.strip() == "true": + return False + return bool(re.search(r"\b" + re.escape(value) + r"\b", when)) + + +def _is_terminal_step(step: object) -> bool: + """Return True if the step has no outgoing routing edges (a true leaf step).""" + if step is None: + return False + return ( + getattr(step, "on_result", None) is None + and getattr(step, "on_success", None) is None + and getattr(step, "on_failure", None) is None + and getattr(step, "on_context_limit", None) is None + ) + + +@semantic_rule( + name="unrouted-verdict-value", + description=( + "Each allowed verdict value from a skill contract must have an explicit " + "on_result condition. A catch-all 'when: true' does not count as explicit handling." + ), + severity=Severity.ERROR, +) +def _check_unrouted_verdict_values(ctx: ValidationContext) -> list[RuleFinding]: + """Error when any on_result step lets an allowed verdict value fall to a catch-all. + + For each step that: + - uses tool: run_skill + - invokes a skill with declared allowed_values on a captured output + - has an on_result block + + Verify that every allowed value has an explicit on_result condition + (not just a when: "true" catch-all). + """ + findings: list[RuleFinding] = [] + + for step_name, step in ctx.recipe.steps.items(): + if step.tool != "run_skill": + continue + skill_command = (step.with_args or {}).get("skill_command", "") + skill_name = resolve_skill_name(skill_command) + if not skill_name: + continue + + allowed_by_output = _get_allowed_values_for_skill(skill_name) + if not allowed_by_output: + continue + + capture = step.capture or {} + for output_name, allowed_values in allowed_by_output.items(): + # Check that the output is captured + captured_key = None + for cap_key, cap_expr in capture.items(): + if not isinstance(cap_expr, str): + continue + if f"result.{output_name}" in cap_expr: + captured_key = cap_key + break + if not captured_key: + continue # Not captured — dataflow rules handle this separately + + if not step.on_result: + continue # No on_result — other rules handle missing routing + + conditions = step.on_result.conditions or [] + # Only check outputs that are actually used as routing discriminators. + # Context-forwarding captures (e.g. experiment_type) have allowed_values + # for schema validation, not for routing — skip them here. + if not any(c.when and f"result.{output_name}" in c.when for c in conditions): + continue + + unrouted = [ + value + for value in allowed_values + if not any(_is_explicit_condition(c.when, value) for c in conditions) + ] + # Allow one value to fall through the catch-all ONLY when the catch-all + # routes to a genuine terminal step (no on_result/on_success/on_failure). + # When the catch-all routes to a non-terminal step, the rule fires + # regardless of how many values are unrouted. + catch_all_arm = next( + (arm for arm in conditions if not arm.when or arm.when.strip() == "true"), + None, + ) + catch_all_is_terminal = catch_all_arm is not None and ( + catch_all_arm.route is None + or _is_terminal_step(ctx.recipe.steps.get(catch_all_arm.route)) + ) + if len(unrouted) <= 1 and catch_all_is_terminal: + continue + for value in unrouted: + findings.append( + RuleFinding( + rule="unrouted-verdict-value", + severity=Severity.ERROR, + step_name=step_name, + message=( + f"Step '{step_name}' captures '{output_name}' from " + f"'{skill_name}' but has no explicit on_result condition " + f"for allowed value '{value}'. Add a condition like: " + f'when: "${{{{ result.{output_name} }}}} == {value}". ' + f"A catch-all 'when: true' does not count." + ), + ) + ) + + return findings + + +def _classify_route_target(target: str) -> str: + """Classify a route target as 'escalation' or 'continuation'.""" + if "failure" in target or "escalat" in target or "stop" in target: + return "escalation" + return "continuation" + + +@semantic_rule( + name="verdict-routing-asymmetry", + description=( + "Steps invoking the same skill must route the same verdict value to the " + "same outcome category (continuation vs escalation). Asymmetric routing " + "causes flaky tests to abort in one path while retrying in another." + ), + severity=Severity.ERROR, +) +def _check_verdict_routing_asymmetry(ctx: ValidationContext) -> list[RuleFinding]: + """Error when the same verdict routes to different outcome categories across steps.""" + findings: list[RuleFinding] = [] + + # Build map: {(skill_name, verdict_value): [(step_name, classification), ...]} + routing_map: dict[tuple[str, str], list[tuple[str, str]]] = {} + + for step_name, step in ctx.recipe.steps.items(): + if step.tool != "run_skill": + continue + skill_command = (step.with_args or {}).get("skill_command", "") + skill_name = resolve_skill_name(skill_command) + if not skill_name: + continue + + allowed_by_output = _get_allowed_values_for_skill(skill_name) + if not allowed_by_output: + continue + + if not step.on_result: + continue + + conditions = step.on_result.conditions or [] + for _output_name, allowed_values in allowed_by_output.items(): + for value in allowed_values: + for cond in conditions: + if _is_explicit_condition(cond.when, value): + classification = _classify_route_target(cond.route) + key = (skill_name, value) + routing_map.setdefault(key, []).append((step_name, classification)) + break + + for (skill_name, value), entries in routing_map.items(): + classifications = {cls for _, cls in entries} + if len(classifications) <= 1: + continue + escalation_steps = [name for name, cls in entries if cls == "escalation"] + continuation_steps = [name for name, cls in entries if cls == "continuation"] + if escalation_steps and continuation_steps: + findings.append( + RuleFinding( + rule="verdict-routing-asymmetry", + severity=Severity.ERROR, + step_name=escalation_steps[0], + message=( + f"Verdict '{value}' from '{skill_name}' routes to escalation " + f"in step '{escalation_steps[0]}' but continuation in step " + f"'{continuation_steps[0]}'. All steps invoking the same skill " + f"should treat the same verdict consistently." + ), + ) + ) + + return findings + + +# Regex to extract a literal verdict value from a `when` expression. +# Handles both template form: ${{ result.verdict }} == value +# and bare dot-access form: result.verdict == 'value' +_VALUE_FROM_WHEN_RE = re.compile( + r"result\.(?P[\w]+)" # result. + r"\s*\}?\}?" # optional closing braces from template form + r"\s*==\s*" # equality operator + r"(?P'\w+'|\w+)" # balanced single-quoted or bare value +) + + +@semantic_rule( + name="on-result-values-in-allowed-values", + description=( + "Every literal verdict value referenced in a recipe on_result condition " + "must be present in the skill contract's allowed_values. Catches drift " + "between recipe routing and contract definitions." + ), + severity=Severity.ERROR, +) +def _check_on_result_values_in_allowed_values(ctx: ValidationContext) -> list[RuleFinding]: + """Error when a recipe routes a verdict value not declared in allowed_values. + + Completes the closed-loop constraint: recipe routes → allowed_values (this rule) + → examples (example-covers-all-allowed-values) → patterns (all-examples-match-all-patterns). + """ + findings: list[RuleFinding] = [] + + for step_name, step in ctx.recipe.steps.items(): + if step.tool != "run_skill": + continue + skill_command = (step.with_args or {}).get("skill_command", "") + skill_name = resolve_skill_name(skill_command) + if not skill_name: + continue + + allowed_by_output = _get_allowed_values_for_skill(skill_name) + if not allowed_by_output: + continue + + if not step.on_result: + continue + + conditions = step.on_result.conditions or [] + for condition in conditions: + when = condition.when + if not when or when.strip() == "true": + continue + for m in _VALUE_FROM_WHEN_RE.finditer(when): + output_name = m.group("output") + value = m.group("value").strip("'") + if output_name not in allowed_by_output: + continue + if value not in allowed_by_output[output_name]: + findings.append( + RuleFinding( + rule="on-result-values-in-allowed-values", + severity=Severity.ERROR, + step_name=step_name, + message=( + f"Step '{step_name}' routes {output_name} == '{value}' " + f"but skill '{skill_name}' contract allowed_values does " + f"not include '{value}'. Add '{value}' to allowed_values " + f"in skill_contracts.yaml." + ), + ) + ) + + return findings diff --git a/src/autoskillit/recipe/rules_worktree.py b/src/autoskillit/recipe/rules/rules_worktree.py similarity index 100% rename from src/autoskillit/recipe/rules_worktree.py rename to src/autoskillit/recipe/rules/rules_worktree.py diff --git a/src/autoskillit/recipe/rules_merge.py b/src/autoskillit/recipe/rules_merge.py deleted file mode 100644 index 062457f1d4..0000000000 --- a/src/autoskillit/recipe/rules_merge.py +++ /dev/null @@ -1,134 +0,0 @@ -"""Semantic rules for merge_worktree routing completeness.""" - -from __future__ import annotations - -import re - -from autoskillit.core import MergeFailedStep, Severity, get_logger -from autoskillit.recipe._analysis import ValidationContext -from autoskillit.recipe.registry import RuleFinding, semantic_rule - -logger = get_logger(__name__) - - -def _is_commit_guard(step_name: str, ctx: ValidationContext) -> bool: - """Return True if step_name is a commit_guard predecessor for merge_worktree. - - A commit_guard step is one whose name starts with 'commit_guard' OR whose - tool is 'run_cmd' and whose cmd contains 'git commit'. - """ - if step_name.startswith("commit_guard"): - return True - step = ctx.recipe.steps.get(step_name) - if step and step.tool == "run_cmd": - cmd = step.with_args.get("cmd", "") - if "git commit" in cmd: - return True - return False - - -_RECOVERABLE_FAILED_STEPS: frozenset[str] = frozenset( - { - MergeFailedStep.DIRTY_TREE, - MergeFailedStep.TEST_GATE, - MergeFailedStep.POST_REBASE_TEST_GATE, - MergeFailedStep.REBASE, - } -) - -_FAILED_STEP_PATTERN = re.compile(r"result\.failed_step\s*==\s*['\"](\w+)['\"]") - - -@semantic_rule( - name="merge-routing-incomplete", - description=( - "Every merge_worktree step with predicate on_result must explicitly route " - "all recoverable MergeFailedStep values to a recovery step. " - "Unhandled values fall through to the result.error catch-all, which typically " - "discards a recoverable worktree." - ), - severity=Severity.ERROR, -) -def _check_merge_routing_completeness(ctx: ValidationContext) -> list[RuleFinding]: - findings: list[RuleFinding] = [] - for step_name, step in ctx.recipe.steps.items(): - if step.tool != "merge_worktree": - continue - if not step.on_result or not step.on_result.conditions: - continue - - matched: set[str] = set() - for condition in step.on_result.conditions: - if condition.when is None: - continue - m = _FAILED_STEP_PATTERN.search(condition.when) - if m: - matched.add(m.group(1)) - - missing = _RECOVERABLE_FAILED_STEPS - matched - if missing: - findings.append( - RuleFinding( - rule="merge-routing-incomplete", - severity=Severity.ERROR, - step_name=step_name, - message=( - f"merge_worktree on_result is missing explicit routes for " - f"recoverable failures: {sorted(missing)}. " - f"These will fall through to the result.error catch-all, " - f"discarding a recoverable worktree." - ), - ) - ) - return findings - - -def _has_commit_guard_ancestor( - step_name: str, ctx: ValidationContext, *, max_depth: int = 5 -) -> bool: - """BFS over predecessors to find a commit_guard within *max_depth* hops.""" - visited: set[str] = set() - frontier = ctx.predecessors.get(step_name, set()) - for _ in range(max_depth): - if not frontier: - break - for p in frontier: - if _is_commit_guard(p, ctx): - return True - visited |= frontier - next_frontier: set[str] = set() - for p in frontier: - next_frontier |= ctx.predecessors.get(p, set()) - visited - frontier = next_frontier - return False - - -@semantic_rule( - name="merge-without-commit-guard", - description=( - "A merge_worktree step has no commit_guard predecessor. Any path reaching " - "merge with uncommitted changes will fail at the dirty-tree gate, burning " - "an expensive recovery cycle. Add a commit_guard run_cmd step before merge." - ), - severity=Severity.ERROR, -) -def _check_merge_without_commit_guard(ctx: ValidationContext) -> list[RuleFinding]: - findings: list[RuleFinding] = [] - for step_name, step in ctx.recipe.steps.items(): - if step.tool != "merge_worktree": - continue - if not _has_commit_guard_ancestor(step_name, ctx): - findings.append( - RuleFinding( - rule="merge-without-commit-guard", - severity=Severity.ERROR, - step_name=step_name, - message=( - f"merge_worktree step '{step_name}' has no commit_guard predecessor. " - f"Uncommitted changes from context-exhausted skills will trigger " - f"the dirty-tree gate, causing an expensive recovery cycle. " - f"Add a commit_guard run_cmd step immediately before this step." - ), - ) - ) - return findings diff --git a/src/autoskillit/recipe/rules_packs.py b/src/autoskillit/recipe/rules_packs.py deleted file mode 100644 index 3f151cd0e5..0000000000 --- a/src/autoskillit/recipe/rules_packs.py +++ /dev/null @@ -1,32 +0,0 @@ -"""Semantic rules for pack validation in recipe pipelines.""" - -from __future__ import annotations - -from autoskillit.core import PACK_REGISTRY, Severity -from autoskillit.recipe._analysis import ValidationContext -from autoskillit.recipe.registry import RuleFinding, semantic_rule - - -@semantic_rule( - name="unknown-required-pack", - description="Pack name in requires_packs is not in PACK_REGISTRY", - severity=Severity.WARNING, -) -def _check_unknown_required_pack(ctx: ValidationContext) -> list[RuleFinding]: - findings = [] - seen_reported: set[str] = set() - for pack_name in ctx.recipe.requires_packs: - if pack_name not in PACK_REGISTRY and pack_name not in seen_reported: - seen_reported.add(pack_name) - findings.append( - RuleFinding( - rule="unknown-required-pack", - severity=Severity.WARNING, - step_name="(top-level)", - message=( - f"Pack {pack_name!r} in requires_packs is not in PACK_REGISTRY. " - f"Known packs: {sorted(PACK_REGISTRY)}" - ), - ) - ) - return findings diff --git a/src/autoskillit/recipe/rules_verdict.py b/src/autoskillit/recipe/rules_verdict.py deleted file mode 100644 index 5c3941980a..0000000000 --- a/src/autoskillit/recipe/rules_verdict.py +++ /dev/null @@ -1,136 +0,0 @@ -"""Semantic rules for skill verdict routing completeness. - -Ensures that recipe steps capturing a skill verdict with declared allowed_values -have an explicit on_result condition for every allowed value. A catch-all -`when: true` route does not count as explicit handling. -""" - -from __future__ import annotations - -import re - -from autoskillit.core import Severity, get_logger -from autoskillit.recipe._analysis import ValidationContext -from autoskillit.recipe.contracts import load_bundled_manifest -from autoskillit.recipe.registry import RuleFinding, semantic_rule - -logger = get_logger(__name__) - -_SKILL_NAME_RE = re.compile(r"/autoskillit:([\w-]+)") - - -def _get_allowed_values_for_skill(skill_name: str) -> dict[str, list[str]]: - """Return {output_name: [allowed_value, ...]} for a skill's outputs with allowed_values.""" - try: - manifest = load_bundled_manifest() - except Exception: - logger.warning( - "unrouted-verdict-value: failed to load bundled manifest; skipping", exc_info=True - ) - return {} - skill_contract = manifest.get("skills", {}).get(skill_name, {}) - result: dict[str, list[str]] = {} - for output in skill_contract.get("outputs", []): - if "allowed_values" in output: - result[output["name"]] = output["allowed_values"] - return result - - -def _extract_skill_name(skill_command: str) -> str | None: - """Extract skill name from a skill_command like /autoskillit:review-pr ...""" - m = _SKILL_NAME_RE.match(skill_command.strip()) - return m.group(1) if m else None - - -def _is_explicit_condition(when: str | None, value: str) -> bool: - """A condition is explicit if it references the value and is not the catch-all.""" - if not when: - return False - if when.strip() == "true": - return False - return bool(re.search(r"\b" + re.escape(value) + r"\b", when)) - - -@semantic_rule( - name="unrouted-verdict-value", - description=( - "Each allowed verdict value from a skill contract must have an explicit " - "on_result condition. A catch-all 'when: true' does not count as explicit handling." - ), - severity=Severity.ERROR, -) -def _check_unrouted_verdict_values(ctx: ValidationContext) -> list[RuleFinding]: - """Error when any on_result step lets an allowed verdict value fall to a catch-all. - - For each step that: - - uses tool: run_skill - - invokes a skill with declared allowed_values on a captured output - - has an on_result block - - Verify that every allowed value has an explicit on_result condition - (not just a when: "true" catch-all). - """ - findings: list[RuleFinding] = [] - - for step_name, step in ctx.recipe.steps.items(): - if step.tool != "run_skill": - continue - skill_command = (step.with_args or {}).get("skill_command", "") - skill_name = _extract_skill_name(skill_command) - if not skill_name: - continue - - allowed_by_output = _get_allowed_values_for_skill(skill_name) - if not allowed_by_output: - continue - - capture = step.capture or {} - for output_name, allowed_values in allowed_by_output.items(): - # Check that the output is captured - captured_key = None - for cap_key, cap_expr in capture.items(): - if not isinstance(cap_expr, str): - continue - if f"result.{output_name}" in cap_expr: - captured_key = cap_key - break - if not captured_key: - continue # Not captured — dataflow rules handle this separately - - if not step.on_result: - continue # No on_result — other rules handle missing routing - - conditions = step.on_result.conditions or [] - # Only check outputs that are actually used as routing discriminators. - # Context-forwarding captures (e.g. experiment_type) have allowed_values - # for schema validation, not for routing — skip them here. - if not any(c.when and f"result.{output_name}" in c.when for c in conditions): - continue - - unrouted = [ - value - for value in allowed_values - if not any(_is_explicit_condition(c.when, value) for c in conditions) - ] - # Allow at most one value to fall through the catch-all (the intended default). - # Fire only when two or more values share the catch-all — that is the - # silent-fallthrough bug pattern (e.g. needs_human treated identically to approved). - if len(unrouted) <= 1: - continue - for value in unrouted: - findings.append( - RuleFinding( - rule="unrouted-verdict-value", - severity=Severity.ERROR, - step_name=step_name, - message=( - f"Step '{step_name}' captures '{output_name}' from " - f"'{skill_name}' but has no explicit on_result condition " - f"for allowed value '{value}'. Add a condition like: " - f'when: "${{{{ result.{output_name} }}}} == {value}". ' - f"A catch-all 'when: true' does not count." - ), - ) - ) - - return findings diff --git a/src/autoskillit/recipe/schema.py b/src/autoskillit/recipe/schema.py index 259b137a7b..3135aa5c9d 100644 --- a/src/autoskillit/recipe/schema.py +++ b/src/autoskillit/recipe/schema.py @@ -5,12 +5,28 @@ import dataclasses from collections.abc import Mapping from dataclasses import dataclass, field +from enum import StrEnum from pathlib import Path from typing import Final -from autoskillit.core import RecipeSource +import regex as re + +from autoskillit.core import FEATURE_REGISTRY, RECIPE_PACK_TAGS, DispatchGateType, RecipeSource AUTOSKILLIT_VERSION_KEY: Final = "autoskillit_version" +RECIPE_VERSION_KEY: Final = "recipe_version" +CAMPAIGN_REF_RE: Final = re.compile(r"\$\{\{\s*campaign\.(\w+)\s*\}\}") + + +class RecipeKind(StrEnum): + STANDARD = "standard" + CAMPAIGN = "campaign" + FOOD_TRUCK = "food-truck" + + +NON_INTERACTIVE_KINDS: Final[frozenset[RecipeKind]] = frozenset( + {RecipeKind.CAMPAIGN, RecipeKind.FOOD_TRUCK} +) @dataclass @@ -18,6 +34,7 @@ class RecipeIngredient: description: str required: bool = False default: str | None = None + type: str | None = None hidden: bool = False # When True, excluded from ingredients table shown to agent def __post_init__(self) -> None: @@ -84,6 +101,7 @@ class RecipeStep: optional: bool = False skip_when_false: str | None = None model: str | None = None + provider: str | None = None description: str = "" sub_recipe: str | None = None # Name of sub-recipe file (no extension) gate: str | None = None # Ingredient name whose value controls lazy loading @@ -95,7 +113,7 @@ class RecipeStep: block: str | None = None # Named block anchor this step belongs to (e.g. "pre_queue_gate") -@dataclass(frozen=True) +@dataclass(frozen=True, slots=True) class RecipeBlock: """A named contiguous region of the step routing graph with budget constraints. @@ -113,6 +131,22 @@ class RecipeBlock: gh_api_occurrences: int # total count of "gh api" substrings across all run_cmd cmds +@dataclass +class CampaignDispatch: + """A single dispatch entry in a campaign recipe.""" + + name: str + recipe: str = "" + task: str = "" + ingredients: dict[str, str] = field( + default_factory=dict + ) # string-only: YAML pass-through key-value pairs, not structured RecipeIngredient objects + depends_on: list[str] = field(default_factory=list) + capture: dict[str, str] = field(default_factory=dict) + gate: DispatchGateType | None = None + message: str | None = None + + @dataclass class Recipe: name: str @@ -122,16 +156,40 @@ class Recipe: steps: dict[str, RecipeStep] = field(default_factory=dict) kitchen_rules: list[str] = field(default_factory=list) version: str | None = None + recipe_version: str | None = None + content_hash: str = "" + composite_hash: str = "" experimental: bool = False requires_packs: list[str] = field(default_factory=list) + # Keys from PACK_REGISTRY (skill pack names, e.g. "fleet", "planner"). + # Named 'requires_packs' (not 'requires_skill_packs') for brevity. + kind: RecipeKind = RecipeKind.STANDARD + categories: list[str] = field(default_factory=list) + dispatches: list[CampaignDispatch] = field(default_factory=list) + requires_recipe_packs: list[str] = field(default_factory=list) + # Keys from RECIPE_PACK_REGISTRY. Named 'requires_recipe_packs' to + # distinguish from 'requires_packs'; the asymmetry is intentional. + allowed_recipes: list[str] = field(default_factory=list) + continue_on_failure: bool = False # Populated by extract_blocks() during load; empty tuple for recipes with no block: anchors. blocks: tuple[RecipeBlock, ...] = field(default_factory=tuple) + requires_features: list[str] = field(default_factory=list) + # Keys from FEATURE_REGISTRY. Recipes declare the features whose skill_categories + # they need so init_session can merge them into session_features at dispatch time. def __post_init__(self) -> None: self.name = self.name.strip() self.description = self.description.strip() self.summary = self.summary.strip() self.kitchen_rules = [rule.strip() for rule in self.kitchen_rules] + unknown_cats = set(self.categories) - RECIPE_PACK_TAGS + if unknown_cats: + raise ValueError( + f"Unknown categories {sorted(unknown_cats)!r}. Valid: {sorted(RECIPE_PACK_TAGS)}" + ) + unknown_features = set(self.requires_features) - set(FEATURE_REGISTRY) + if unknown_features: + raise ValueError(f"Unknown features in requires_features: {sorted(unknown_features)}") @dataclass @@ -142,7 +200,12 @@ class RecipeInfo: path: Path summary: str = "" version: str | None = None + recipe_version: str | None = None + content_hash: str = "" content: str | None = None # raw YAML text; None when set via parse_recipe_metadata + kind: RecipeKind = RecipeKind.STANDARD + experimental: bool = False + requires_packs: list[str] = field(default_factory=list) @dataclass diff --git a/src/autoskillit/recipe/skill_contracts.yaml b/src/autoskillit/recipe/skill_contracts.yaml index f530854a01..c9dd812e37 100644 --- a/src/autoskillit/recipe/skill_contracts.yaml +++ b/src/autoskillit/recipe/skill_contracts.yaml @@ -1,4 +1,8 @@ version: 0.1.0 +# Default semantics for 'required' when omitted from an input: +# skills section → required=false (skills are permissive) +# callable_contracts section → required=true (callables are strict) +# Best practice: always declare 'required' explicitly on every input. skills: troubleshoot-experiment: inputs: @@ -43,11 +47,20 @@ skills: - name: is_fixable type: string expected_output_patterns: - - "diagnosis_path\\s*=\\s*/.+" + - "diagnosis_path\\s*=\\s*(?:/.+)?" - "failure_subtype\\s*=\\s*(flaky|timing_race|deterministic|fixture|import|env|unknown)" pattern_examples: - "diagnosis_path = /tmp/diagnosis.md\nfailure_type = test\nfailure_subtype = deterministic\nis_fixable = true\n%%ORDER_UP%%" - "diagnosis_path = /tmp/diagnosis.md\nfailure_type = test\nfailure_subtype = flaky\nis_fixable = false\n%%ORDER_UP%%" + - "diagnosis_path = \nfailure_type = test\nfailure_subtype = unknown\nis_fixable = false\n%%ORDER_UP%%" + - "diagnosis_path = /tmp/diagnosis.md\nfailure_type = test\nfailure_subtype = timing_race\nis_fixable = true\n%%ORDER_UP%%" + - "diagnosis_path = /tmp/diagnosis.md\nfailure_type = test\nfailure_subtype = fixture\nis_fixable = true\n%%ORDER_UP%%" + - "diagnosis_path = /tmp/diagnosis.md\nfailure_type = test\nfailure_subtype = import\nis_fixable = true\n%%ORDER_UP%%" + - "diagnosis_path = /tmp/diagnosis.md\nfailure_type = lint\nfailure_subtype = unknown\nis_fixable = true\n%%ORDER_UP%%" + - "diagnosis_path = /tmp/diagnosis.md\nfailure_type = build\nfailure_subtype = unknown\nis_fixable = true\n%%ORDER_UP%%" + - "diagnosis_path = /tmp/diagnosis.md\nfailure_type = type_check\nfailure_subtype = unknown\nis_fixable = true\n%%ORDER_UP%%" + - "diagnosis_path = /tmp/diagnosis.md\nfailure_type = env\nfailure_subtype = env\nis_fixable = false\n%%ORDER_UP%%" + - "diagnosis_path = /tmp/diagnosis.md\nfailure_type = unknown\nfailure_subtype = unknown\nis_fixable = false\n%%ORDER_UP%%" write_behavior: always resolve-design-review: inputs: @@ -65,7 +78,7 @@ skills: type: file_path expected_output_patterns: - "resolution\\s*=\\s*(revised|failed)" - - "resolution\\s*=\\s*revised[\\s\\S]*?revision_guidance\\s*=\\s*/.+" + - "(?:revision_guidance\\s*=\\s*/.+|resolution\\s*=\\s*failed)" pattern_examples: - "resolution = revised\nrevision_guidance = /tmp/guidance.md\n%%ORDER_UP%%" - "resolution = failed\n%%ORDER_UP%%" @@ -134,6 +147,9 @@ skills: - name: base_branch type: string required: true + - name: mode + type: string + required: false outputs: - name: verdict type: string @@ -144,6 +160,14 @@ skills: - ci_only_failure - name: fixes_applied type: integer + - name: deferred_observations_path + type: string + description: "Written when mode=local (accumulated DISCUSS findings) or when mode=github (posted from prior local rounds)." + required: false + - name: reject_patterns_path + type: string + description: "Written in local mode (accumulating across rounds without timestamp) or github mode (timestamped, one-shot)." + required: false write_behavior: conditional write_expected_when: - "verdict\\s*=\\s*real_fix" @@ -196,6 +220,7 @@ skills: - "investigation_path\\s*=\\s*/.+" pattern_examples: - "investigation_path = /tmp/investigation.md\n%%ORDER_UP%%" + read_only: true make-groups: inputs: - name: source_doc @@ -212,6 +237,7 @@ skills: - "groups_path\\s*=\\s*/.+" pattern_examples: - "groups_path = /tmp/groups.json\n%%ORDER_UP%%" + write_behavior: always make-plan: inputs: - name: task @@ -237,6 +263,16 @@ skills: pattern_examples: - "recipe_path = /tmp/recipe.yaml\n%%ORDER_UP%%" write_behavior: always + make-campaign: + inputs: [] + outputs: + - name: campaign_path + type: file_path + expected_output_patterns: + - "campaign_path\\s*=\\s*/.+" + pattern_examples: + - "campaign_path = /home/user/project/.autoskillit/recipes/campaigns/my-campaign.yaml\n%%ORDER_UP%%" + write_behavior: always mermaid: inputs: [] outputs: [] @@ -346,7 +382,7 @@ skills: type: file_path - name: analysis_file type: file_path - - name: integration_branch + - name: batch_branch type: string - name: pr_count type: string @@ -371,6 +407,9 @@ skills: outputs: - name: merged type: string + allowed_values: + - "true" + - "false" - name: needs_plan type: string - name: escalation_required @@ -441,7 +480,9 @@ skills: - "lens_context_paths\\s*=\\s*/.+" pattern_examples: - "prep_path = /abs/path/pr_prep_2026-04-07_120000.md\nselected_lenses = module-dependency,process-flow\nlens_context_paths = /abs/ctx1.md,/abs/ctx2.md\n%%ORDER_UP%%" - write_behavior: always + write_behavior: conditional + write_expected_when: + - "prep_path\\s*=\\s*/.+" compose-pr: inputs: @@ -472,7 +513,7 @@ skills: open-integration-pr: inputs: - - name: integration_branch + - name: batch_branch type: string required: true - name: base_branch @@ -529,19 +570,31 @@ skills: type: string required: false recommended: true + - name: diff_metrics_path + type: string + required: false + - name: mode + type: string + required: false outputs: - name: verdict type: string allowed_values: - approved + - approved_with_comments - changes_requested - needs_human + - name: local_findings_path + type: string + description: "Written when mode=local. Path to local_findings_{pr_number}.json output." expected_output_patterns: - - "verdict\\s*=\\s*(approved|changes_requested|needs_human)" + - "verdict\\s*=\\s*(approved|approved_with_comments|changes_requested|needs_human)" + - "(?:%%REVIEW_GATE::(LOOP_REQUIRED|CLEAR)%%|verdict\\s*=\\s*approved_with_comments)" pattern_examples: - - "verdict = approved\n%%ORDER_UP%%" - - "verdict = changes_requested\n%%ORDER_UP%%" - - "verdict = needs_human\n%%ORDER_UP%%" + - "verdict = approved\n%%REVIEW_GATE::CLEAR%%\n%%ORDER_UP%%" + - "verdict = approved_with_comments\n%%ORDER_UP%%" + - "verdict = changes_requested\n%%REVIEW_GATE::LOOP_REQUIRED%%\n%%ORDER_UP%%" + - "verdict = needs_human\n%%REVIEW_GATE::CLEAR%%\n%%ORDER_UP%%" arch-lens-c4-container: inputs: @@ -797,7 +850,7 @@ skills: required: false - name: experiment_plan_path type: file_path - required: false + required: true - name: project_context type: string required: false @@ -816,7 +869,7 @@ skills: required: false - name: experiment_plan_path type: file_path - required: false + required: true - name: project_context type: string required: false @@ -835,7 +888,7 @@ skills: required: false - name: experiment_plan_path type: file_path - required: false + required: true - name: project_context type: string required: false @@ -854,7 +907,7 @@ skills: required: false - name: experiment_plan_path type: file_path - required: false + required: true - name: project_context type: string required: false @@ -873,7 +926,7 @@ skills: required: false - name: experiment_plan_path type: file_path - required: false + required: true - name: project_context type: string required: false @@ -892,7 +945,7 @@ skills: required: false - name: experiment_plan_path type: file_path - required: false + required: true - name: project_context type: string required: false @@ -911,7 +964,7 @@ skills: required: false - name: experiment_plan_path type: file_path - required: false + required: true - name: project_context type: string required: false @@ -930,7 +983,7 @@ skills: required: false - name: experiment_plan_path type: file_path - required: false + required: true - name: project_context type: string required: false @@ -949,7 +1002,7 @@ skills: required: false - name: experiment_plan_path type: file_path - required: false + required: true - name: project_context type: string required: false @@ -968,7 +1021,7 @@ skills: required: false - name: experiment_plan_path type: file_path - required: false + required: true - name: project_context type: string required: false @@ -987,7 +1040,7 @@ skills: required: false - name: experiment_plan_path type: file_path - required: false + required: true - name: project_context type: string required: false @@ -1006,7 +1059,7 @@ skills: required: false - name: experiment_plan_path type: file_path - required: false + required: true - name: project_context type: string required: false @@ -1025,7 +1078,7 @@ skills: required: false - name: experiment_plan_path type: file_path - required: false + required: true - name: project_context type: string required: false @@ -1044,7 +1097,7 @@ skills: required: false - name: experiment_plan_path type: file_path - required: false + required: true - name: project_context type: string required: false @@ -1063,7 +1116,7 @@ skills: required: false - name: experiment_plan_path type: file_path - required: false + required: true - name: project_context type: string required: false @@ -1082,7 +1135,7 @@ skills: required: false - name: experiment_plan_path type: file_path - required: false + required: true - name: project_context type: string required: false @@ -1101,7 +1154,7 @@ skills: required: false - name: experiment_plan_path type: file_path - required: false + required: true - name: project_context type: string required: false @@ -1120,7 +1173,7 @@ skills: required: false - name: experiment_plan_path type: file_path - required: false + required: true - name: project_context type: string required: false @@ -1134,7 +1187,9 @@ skills: prepare-issue: inputs: [] - outputs: [] + outputs: + - name: issue_url + type: str expected_output_patterns: - "---prepare-issue-result---" pattern_examples: @@ -1175,7 +1230,9 @@ skills: process-issues: inputs: [] - outputs: [] + outputs: + - name: dispatch_results + type: str expected_output_patterns: - "---process-issues-result---" pattern_examples: @@ -1236,6 +1293,14 @@ skills: type: file_path - name: report_plan_path type: file_path + - name: disambiguation_rule_applied + type: string + - name: tier_c_lens + type: string + - name: methodology_tradition + type: string + - name: visualization_plan_trace_path + type: file_path expected_output_patterns: - "visualization_plan_path\\s*=\\s*/.+" - "report_plan_path\\s*=\\s*/.+" @@ -1260,6 +1325,33 @@ skills: - "verdict = PASS\nresource_report = /tmp/stage-data/resource_feasibility_2026-04-13_120000.md\n%%ORDER_UP%%" write_behavior: always + setup-environment: + inputs: + - name: worktree_path + type: directory_path + required: true + - name: experiment_plan + type: file_path + required: true + outputs: + - name: env_mode + type: string + allowed_values: [none, docker, micromamba-host, unavailable] + - name: env_report + type: file_path + - name: verdict + type: string + expected_output_patterns: + - "env_mode\\s*=\\s*(none|docker|micromamba-host|unavailable)" + - "env_report\\s*=\\s*/.+" + - "verdict\\s*=\\s*(PASS|WARN|FAIL)" + pattern_examples: + - "env_mode = none\nenv_report = /tmp/setup-environment/env_setup_report.md\nverdict = PASS\n%%ORDER_UP%%" + - "env_mode = docker\nenv_report = /tmp/setup-environment/env_setup_report.md\nverdict = PASS\n%%ORDER_UP%%" + - "env_mode = micromamba-host\nenv_report = /tmp/setup-environment/env_setup_report.md\nverdict = WARN\n%%ORDER_UP%%" + - "env_mode = unavailable\nenv_report = /tmp/setup-environment/env_setup_report.md\nverdict = FAIL\n%%ORDER_UP%%" + write_behavior: always + implement-experiment: inputs: - name: plan_path @@ -1379,11 +1471,19 @@ skills: type: file_path - name: revision_guidance type: file_path + - name: classification_timestamp + type: string expected_output_patterns: - "verdict\\s*=\\s*(GO|REVISE|STOP)" - - "experiment_type\\s*=\\s*\\S+" + - "(?:experiment_type\\s*=\\s*\\S+|verdict\\s*=\\s*STOP)" pattern_examples: - "verdict = GO\nexperiment_type = benchmark\nevaluation_dashboard = /tmp/eval.md\nrevision_guidance = /tmp/guidance.md\n%%ORDER_UP%%" + - "verdict = REVISE\nexperiment_type = benchmark\nrevision_guidance = /tmp/guidance.md\n%%ORDER_UP%%" + - "verdict = STOP\n%%ORDER_UP%%" + - "verdict = GO\nexperiment_type = causal_inference\nevaluation_dashboard = /tmp/eval.md\n%%ORDER_UP%%" + - "verdict = GO\nexperiment_type = configuration_study\nevaluation_dashboard = /tmp/eval.md\n%%ORDER_UP%%" + - "verdict = GO\nexperiment_type = robustness_audit\nevaluation_dashboard = /tmp/eval.md\n%%ORDER_UP%%" + - "verdict = GO\nexperiment_type = exploratory\nevaluation_dashboard = /tmp/eval.md\n%%ORDER_UP%%" write_behavior: always review-research-pr: @@ -1455,6 +1555,8 @@ skills: pattern_examples: - "needs_rerun = true\nverdict = real_fix\nfixes_applied = 2\n%%ORDER_UP%%" - "needs_rerun = false\nverdict = already_green\nfixes_applied = 0\n%%ORDER_UP%%" + - "needs_rerun = true\nverdict = flake_suspected\nfixes_applied = 0\n%%ORDER_UP%%" + - "needs_rerun = false\nverdict = ci_only_failure\nfixes_applied = 0\n%%ORDER_UP%%" write_behavior: conditional write_expected_when: - "verdict\\s*=\\s*real_fix" @@ -1484,6 +1586,8 @@ skills: pattern_examples: - "needs_rerun = true\nverdict = real_fix\nfixes_applied = 2\n%%ORDER_UP%%" - "needs_rerun = false\nverdict = already_green\nfixes_applied = 0\n%%ORDER_UP%%" + - "needs_rerun = true\nverdict = flake_suspected\nfixes_applied = 0\n%%ORDER_UP%%" + - "needs_rerun = false\nverdict = ci_only_failure\nfixes_applied = 0\n%%ORDER_UP%%" write_behavior: conditional write_expected_when: - "verdict\\s*=\\s*real_fix" @@ -1545,7 +1649,7 @@ skills: pattern_examples: - "diagram_path = /tmp/diagram.md\n%%ORDER_UP%%" - vis-lens-domain-norms: + vis-lens-methodology-norms: inputs: - name: context_path type: file_path @@ -1712,6 +1816,30 @@ skills: - "diagram_path\\s*=\\s*/.+" pattern_examples: - "diagram_path = /tmp/diagram.md\n%%ORDER_UP%%" + build-execution-map: + inputs: [] + outputs: + - name: execution_map + type: file_path + - name: execution_map_report + type: file_path + - name: group_count + type: string + - name: total_issues + type: string + - name: dispatched_count + type: string + - name: deferred_count + type: string + - name: has_deferred + type: string + - name: review_approach_candidates + type: string + expected_output_patterns: + - "execution_map\\s*=\\s*/.+" + pattern_examples: + - "execution_map = /tmp/build-execution-map/execution_map_2026-04-22_190000.json\nexecution_map_report = /tmp/build-execution-map/execution_map_report_2026-04-22_190000.md\ngroup_count = 3\ntotal_issues = 5\ndispatched_count = 4\ndeferred_count = 1\nhas_deferred = true" + write_behavior: always bundle-local-report: inputs: - name: research_dir @@ -1733,4 +1861,838 @@ skills: - "html_path\\s*=\\s*\\S+" pattern_examples: - "html_path = /tmp/research/report.html\n%%ORDER_UP%%" + write_behavior: conditional + write_expected_when: + - "html_path\\s*=\\s*\\S+" + planner-analyze: + inputs: [] + outputs: [] + planner-extract-domain: + inputs: [] + outputs: [] + planner-generate-phases: + inputs: [] + outputs: + - name: phase_manifest_path + type: file_path + - name: phase_ids + type: str + result_fields: + - name: id + type: str + required: true + - name: name + type: str + required: true + - name: ordering + type: int + required: true + expected_output_patterns: + - "phase_manifest_path\\s*=\\s*\\S+" + pattern_examples: + - "phase_manifest_path = /tmp/autoskillit/planner/run-20260101-120000/phases/phase_manifest.json\n%%ORDER_UP%%" + write_behavior: always + planner-elaborate-phase: + inputs: + - name: plan_snapshot_path + type: file_path + required: true + - name: target_phase_id + type: str + required: true + - name: output_dir + type: directory_path + required: true + outputs: + - name: elab_result_path + type: file_path + expected_output_patterns: + - "elab_result_path\\s*=\\s*\\S+" + pattern_examples: + - "elab_result_path = /tmp/autoskillit/planner/run-20260101-120000/phases/P1_result.json\n%%ORDER_UP%%" + write_behavior: always + planner-elaborate-assignments: + inputs: + - name: context_file + type: file_path + required: true + - name: planner_dir + type: directory_path + required: true + outputs: + - name: phase_assignments_result_dir + type: directory_path + expected_output_patterns: + - "phase_assignments_result_dir\\s*=\\s*\\S+" + pattern_examples: + - "phase_assignments_result_dir = /tmp/autoskillit/planner/run-20260101-120000/assignments\n%%ORDER_UP%%" + write_behavior: always + planner-elaborate-wps: + inputs: + - name: context_file + type: file_path + required: true + - name: planner_dir + type: directory_path + required: true + outputs: + - name: phase_wps_result_dir + type: directory_path + expected_output_patterns: + - "phase_wps_result_dir\\s*=\\s*\\S+" + pattern_examples: + - "phase_wps_result_dir = /tmp/autoskillit/planner/run-20260101-120000/work_packages\n%%ORDER_UP%%" + write_behavior: always + planner-reconcile-deps: + inputs: [] + outputs: [] + planner-refine: + inputs: [] + outputs: [] + planner-refine-phases: + inputs: + - name: combined_plan_path + type: file_path + required: true + - name: output_dir + type: directory_path + required: true + outputs: + - name: refined_plan_path + type: file_path + expected_output_patterns: + - "refined_plan_path\\s*=\\s*\\S+" + pattern_examples: + - "refined_plan_path = /tmp/autoskillit/planner/run-20260101-120000/refined_plan.json\n%%ORDER_UP%%" + write_behavior: always + planner-refine-assignments: + inputs: + - name: context_file + type: file_path + required: true + - name: refined_plan_path + type: file_path + required: true + - name: output_dir + type: directory_path + required: true + outputs: + - name: phase_refined_path + type: file_path + expected_output_patterns: + - "phase_refined_path\\s*=\\s*\\S+" + pattern_examples: + - "phase_refined_path = /tmp/autoskillit/planner/run-20260101-120000/refine_contexts/P1_result.json\n%%ORDER_UP%%" + write_behavior: always + planner-refine-wps: + inputs: + - name: combined_wps_path + type: file_path + required: true + - name: refined_plan_path + type: file_path + required: true + - name: refined_assignments_path + type: file_path + required: true + - name: output_dir + type: directory_path + required: true + outputs: + - name: refined_wps_path + type: file_path + expected_output_patterns: + - "refined_wps_path\\s*=\\s*\\S+" + pattern_examples: + - "refined_wps_path = /tmp/autoskillit/planner/run-20260101-120000/refined_wps.json\n%%ORDER_UP%%" + write_behavior: always + planner-consolidate-wps: + inputs: + - name: refined_wps_path + type: file_path + required: true + - name: planner_dir + type: directory_path + required: true + outputs: [] + write_behavior: always + planner-validate-task-alignment: + inputs: + - name: refined_wps_path + type: file_path + required: true + - name: refined_plan_path + type: file_path + required: true + - name: output_dir + type: directory_path + required: true + outputs: + - name: alignment_findings_path + type: file_path + - name: alignment_finding_count + type: str + expected_output_patterns: + - "alignment_findings_path\\s*=\\s*\\S+" + - "alignment_finding_count\\s*=\\s*\\d+" + pattern_examples: + - "alignment_findings_path = /tmp/autoskillit/planner/run-20260101-120000/task_alignment.json\nalignment_finding_count = 0\n%%ORDER_UP%%" + write_behavior: always + planner-assess-review-approach: + inputs: + - name: refined_wps_path + type: file_path + required: true + - name: planner_dir + type: directory_path + required: true + outputs: + - name: review_approach_assessment_path + type: file_path + expected_output_patterns: + - "review_approach_assessment_path\\s*=\\s*\\S+" + pattern_examples: + - "review_approach_assessment_path = /tmp/autoskillit/planner/run-20260101-120000/review_approach_assessment.json\n%%ORDER_UP%%" + write_behavior: always + audit-tests: + inputs: [] + outputs: + - name: audit_report_path + type: file_path + expected_output_patterns: + - "audit_report_path\\s*=\\s*\\S+" + pattern_examples: + - "audit_report_path = /tmp/autoskillit/audit-tests/test_audit_2026-04-28_120000.md\n%%ORDER_UP%%" + write_behavior: always + validate-audit: + inputs: + - name: audit_report_path + type: file_path + required: false + outputs: + - name: validated_report_path + type: file_path + - name: verdict + type: str + expected_output_patterns: + - "validated_report_path\\s*=\\s*\\S+" + - "verdict\\s*=\\s*\\S+" + pattern_examples: + - "validated_report_path = /tmp/autoskillit/validate-audit-2026-04-28_120000/validated_report_tests.md\nverdict = validated" + write_behavior: conditional + write_expected_when: + - "verdict\\s*=\\s*validated" + validate-test-audit: + inputs: + - name: audit_report_path + type: file_path + required: false + outputs: + - name: validated_report_path + type: file_path + - name: verdict + type: str + expected_output_patterns: + - "validated_report_path\\s*=\\s*\\S+" + - "verdict\\s*=\\s*\\S+" + pattern_examples: + - "validated_report_path = /tmp/autoskillit/validate-audit-2026-05-05_140000/validated_report_tests.md\nverdict = validated" + write_behavior: conditional + write_expected_when: + - "verdict\\s*=\\s*validated" + promote-to-main: + inputs: + - name: batch_branch + type: string + required: false + - name: base_branch + type: string + required: false + outputs: + - name: pr_url + type: string + - name: verdict + type: string + allowed_values: [created, dry_run, preflight_failed] + - name: category_summary + type: string + - name: pr_body_path + type: file_path + expected_output_patterns: + - "verdict\\s*=\\s*(created|dry_run|preflight_failed)" + pattern_examples: + - "pr_url = https://github.com/org/repo/pull/42\nverdict = created\ncategory_summary = 14 fixes, 3 features\n%%ORDER_UP%%" + - "pr_url = \nverdict = dry_run\ncategory_summary = 14 fixes, 3 features\n%%ORDER_UP%%" + - "pr_url = \nverdict = preflight_failed\ncategory_summary = \n%%ORDER_UP%%" write_behavior: always +callable_contracts: + autoskillit.smoke_utils.check_loop_iteration: + inputs: + - name: current_iteration + type: str + required: false + - name: max_iterations + type: str + required: false + outputs: + - name: next_iteration + type: str + - name: max_exceeded + type: str + autoskillit.smoke_utils.check_review_loop: + inputs: + - name: pr_number + type: str + required: true + - name: current_iteration + type: str + required: false + - name: max_iterations + type: str + required: false + - name: previous_verdict + type: str + required: false + outputs: + - name: next_iteration + type: str + - name: max_exceeded + type: str + - name: had_blocking + type: str + autoskillit.smoke_utils.check_bug_report_non_empty: + inputs: + - name: workspace + type: directory_path + required: true + outputs: + - name: non_empty + type: str + autoskillit.smoke_utils.annotate_pr_diff: + inputs: + - name: pr_number + type: str + required: true + - name: cwd + type: directory_path + required: true + - name: output_dir + type: directory_path + required: true + - name: loc_threshold + type: str + required: false + - name: file_threshold + type: str + required: false + outputs: + - name: annotated_diff_path + type: file_path + - name: hunk_ranges_path + type: file_path + - name: diff_metrics_path + type: file_path + autoskillit.smoke_utils.compute_domain_partitions: + inputs: + - name: batch_branch + type: str + required: true + - name: base_branch + type: str + required: true + - name: cwd + type: directory_path + required: true + - name: output_dir + type: directory_path + required: true + outputs: + - name: domain_partitions_path + type: file_path + autoskillit.smoke_utils.fetch_merge_queue_data: + inputs: + - name: base_branch + type: str + required: true + - name: cwd + type: directory_path + required: true + - name: output_dir + type: directory_path + required: true + outputs: + - name: merge_queue_data_path + type: file_path + autoskillit.planner.validate_plan: + inputs: + - name: output_dir + type: directory_path + required: true + outputs: + - name: verdict + type: str + - name: validation_path + type: file_path + - name: issue_count + type: str + autoskillit.planner.compile_plan: + inputs: + - name: output_dir + type: directory_path + required: true + - name: task_file_path + type: file_path + required: true + - name: source_dir + type: directory_path + required: true + outputs: + - name: plan_path + type: file_path + - name: plan_json_path + type: file_path + - name: plan_parts + type: str + autoskillit.planner.merge.merge_files: + inputs: + - name: file_paths + type: str + required: true + - name: output_path + type: file_path + required: true + - name: key + type: str + required: true + outputs: + - name: merged_path + type: file_path + - name: item_count + type: str + autoskillit.planner.merge.extract_item: + inputs: + - name: source_path + type: file_path + required: true + - name: item_id + type: str + required: true + - name: output_path + type: file_path + required: true + outputs: + - name: extracted_path + type: file_path + autoskillit.planner.merge.replace_item: + inputs: + - name: source_path + type: file_path + required: true + - name: item_id + type: str + required: true + - name: replacement_path + type: file_path + required: true + outputs: + - name: updated_path + type: file_path + - name: replaced_id + type: str + autoskillit.planner.merge.build_plan_snapshot: + inputs: + - name: phases_dir + type: directory_path + required: true + - name: output_path + type: file_path + required: true + outputs: + - name: snapshot_path + type: file_path + - name: phase_ids + type: str + autoskillit.planner.merge.merge_tier_results: + inputs: + - name: results_dir + type: directory_path + required: true + - name: output_path + type: file_path + required: true + - name: key + type: str + required: true + outputs: + - name: merged_path + type: file_path + - name: item_count + type: str + - name: refine_context_paths + type: str + when: "key == 'assignments'" + autoskillit.planner.create_run_dir: + inputs: + - name: temp_dir + type: directory_path + required: true + outputs: + - name: planner_dir + type: directory_path + autoskillit.planner.finalize_wp_manifest: + inputs: + - name: work_packages_dir + type: directory_path + required: true + - name: output_dir + type: directory_path + required: true + outputs: + - name: manifest_path + type: file_path + - name: total_count + type: str + autoskillit.planner.expand_assignments: + inputs: + - name: refined_plan_path + type: file_path + required: true + - name: output_dir + type: directory_path + required: true + outputs: + - name: manifest_path + type: file_path + - name: context_paths + type: str + - name: item_ids + type: str + autoskillit.planner.expand_wps: + inputs: + - name: refined_assignments_path + type: file_path + required: true + - name: output_dir + type: directory_path + required: true + outputs: + - name: manifest_path + type: file_path + - name: context_paths + type: str + - name: item_ids + type: str + autoskillit.recipe._cmd_rpc.compute_branch: + inputs: + - name: issue_slug + type: str + required: false + - name: run_name + type: str + required: false + - name: issue_number + type: str + required: false + outputs: + - name: branch_name + type: str + autoskillit.recipe._cmd_rpc.check_eject_limit: + inputs: + - name: counter_file + type: str + required: true + - name: max_ejects + type: str + required: false + outputs: + - name: status + type: str + - name: count + type: str + autoskillit.recipe._cmd_rpc.check_dropped_healthy_loop: + inputs: + - name: counter_file + type: str + required: true + - name: max_drops + type: str + required: false + outputs: + - name: status + type: str + - name: count + type: str + autoskillit.recipe._cmd_rpc.commit_guard: + inputs: + - name: worktree_path + type: str + required: true + outputs: + - name: committed + type: str + autoskillit.recipe._cmd_rpc.queue_ejected_fix: + inputs: + - name: work_dir + type: str + required: true + - name: base_branch + type: str + required: true + outputs: + - name: status + type: str + autoskillit.recipe._cmd_rpc.direct_merge_conflict_fix: + inputs: + - name: work_dir + type: str + required: true + - name: base_branch + type: str + required: true + outputs: + - name: status + type: str + autoskillit.recipe._cmd_rpc.immediate_merge_conflict_fix: + inputs: + - name: work_dir + type: str + required: true + - name: base_branch + type: str + required: true + outputs: + - name: status + type: str + autoskillit.recipe._cmd_rpc.wait_for_direct_merge: + inputs: + - name: pr_number + type: str + required: true + - name: max_polls + type: str + required: false + - name: poll_interval + type: str + required: false + outputs: + - name: state + type: str + autoskillit.recipe._cmd_rpc.wait_for_immediate_merge: + inputs: + - name: pr_number + type: str + required: true + - name: max_polls + type: str + required: false + - name: poll_interval + type: str + required: false + outputs: + - name: state + type: str + autoskillit.recipe._cmd_rpc.attempt_cheap_rebase: + inputs: + - name: work_dir + type: str + required: true + - name: ejected_pr_branch + type: str + required: true + - name: base_branch + type: str + required: true + outputs: + - name: status + type: str + autoskillit.recipe._cmd_rpc.wait_for_review_pr_mergeability: + inputs: + - name: pr_url + type: str + required: true + - name: max_polls + type: str + required: false + - name: poll_interval + type: str + required: false + outputs: + - name: pr_number + type: str + autoskillit.recipe._cmd_rpc.create_persistent_integration: + inputs: + - name: work_dir + type: str + required: true + - name: base_branch + type: str + required: true + outputs: + - name: ok + type: str + autoskillit.recipe._cmd_rpc.force_push_and_wait_mergeability: + inputs: + - name: work_dir + type: str + required: true + - name: batch_branch + type: str + required: true + - name: review_pr_number + type: str + required: true + - name: max_polls + type: str + required: false + - name: poll_interval + type: str + required: false + outputs: + - name: ok + type: str + autoskillit.recipe._cmd_rpc.advance_queue_pr: + inputs: + - name: current_pr_number + type: str + required: true + nullable: false + - name: pr_order_file + type: str + required: true + outputs: + - name: current_pr_number + type: str + autoskillit.recipe._cmd_rpc.proactive_rebase_next_pr: + inputs: + - name: work_dir + type: str + required: true + - name: next_pr_branch + type: str + required: true + - name: base_branch + type: str + required: true + outputs: + - name: status + type: str + autoskillit.recipe._cmd_rpc.refetch_issues: + inputs: + - name: issue_urls + type: str + required: true + outputs: + - name: issue_numbers + type: str + autoskillit.recipe._cmd_rpc.emit_fallback_map: + inputs: + - name: issue_urls + type: str + required: true + - name: temp_dir + type: str + required: true + outputs: + - name: execution_map + type: str + autoskillit.recipe._cmd_rpc.ensure_results: + inputs: + - name: experiment_results + type: str + required: true + - name: worktree_path + type: str + required: true + - name: temp_subdir + type: str + required: false + outputs: + - name: experiment_results + type: str + autoskillit.recipe._cmd_rpc.export_local_bundle: + inputs: + - name: source_dir + type: str + required: true + - name: research_dir + type: str + required: true + outputs: + - name: local_bundle_path + type: str + autoskillit.recipe._cmd_rpc.batch_create_issues: + inputs: + - name: workspace + type: str + required: true + - name: chunk_size + type: str + required: false + - name: audit_run_dir + type: directory_path + required: false + outputs: + - name: issue_urls + type: str + - name: issue_count + type: str + autoskillit.recipe._cmd_rpc.create_audit_run_dir: + inputs: + - name: temp_dir + type: directory_path + required: true + outputs: + - name: audit_run_dir + type: directory_path + +tool_output_contracts: + wait_for_ci: + result_field: conclusion + fields: + conclusion: + type: str + allowed_values: + - success + - failure + - cancelled + - action_required + - timed_out + - startup_failure + - stale + - no_runs + - error + - unknown + terminal_values: + - error + - unknown + recoverable_values: + - failure + - cancelled + - timed_out + - no_runs + - action_required + get_ci_status: + result_field: conclusion + fields: + conclusion: + type: str + allowed_values: + - success + - failure + - cancelled + - action_required + - timed_out + - startup_failure + - stale + - no_runs + - error + - unknown + terminal_values: + - error + - unknown + recoverable_values: + - failure + - cancelled + - timed_out + - no_runs + - action_required diff --git a/src/autoskillit/recipe/validator.py b/src/autoskillit/recipe/validator.py index e84ecf19b4..0ce17691fe 100644 --- a/src/autoskillit/recipe/validator.py +++ b/src/autoskillit/recipe/validator.py @@ -23,8 +23,8 @@ from autoskillit.recipe.io import iter_steps_with_context from autoskillit.recipe.registry import ( _RULE_REGISTRY, + RuleDef, RuleFinding, - RuleSpec, build_quality_dict, compute_recipe_validity, filter_version_rule, @@ -32,7 +32,7 @@ run_semantic_rules, semantic_rule, ) -from autoskillit.recipe.schema import _TERMINAL_TARGETS, Recipe +from autoskillit.recipe.schema import _TERMINAL_TARGETS, Recipe, RecipeKind logger = get_logger(__name__) @@ -42,7 +42,7 @@ # registry). Callers import from validator.py as the single public entry point. __all__ = [ "RuleFinding", - "RuleSpec", + "RuleDef", "ValidationContext", "_RULE_REGISTRY", "_build_step_graph", @@ -60,15 +60,27 @@ # Structural validation # --------------------------------------------------------------------------- +_SKILL_HINT = " (Use /autoskillit:write-recipe for schema guidance)" -def validate_recipe(recipe: Recipe) -> list[str]: - """Return a list of validation errors (empty if valid).""" + +def validate_recipe_structure(recipe: Recipe) -> list[str]: + """Return structural validation errors (empty if valid). + + Does not run semantic rules or contract checks; use validate_from_path() + for complete validation. + """ errors: list[str] = [] if not recipe.name: - errors.append("Recipe must have a 'name'.") + errors.append("Recipe must have a 'name'." + _SKILL_HINT) + + if recipe.kind == RecipeKind.CAMPAIGN: + if not recipe.dispatches: + errors.append("Campaign recipe must have at least one dispatch.") + return errors + if not recipe.steps: - errors.append("Recipe must have at least one step.") + errors.append("Recipe must have at least one step." + _SKILL_HINT) step_names = set(recipe.steps.keys()) @@ -213,7 +225,7 @@ def validate_recipe(recipe: Recipe) -> list[str]: if not all_refs: errors.append( f"Step '{step_name}'.capture.{cap_key} must contain " - f"a ${{{{ result.* }}}} expression." + f"a ${{{{ result.* }}}} expression." + _SKILL_HINT ) for ref_match in all_refs: inner = ref_match[3:-2].strip() @@ -221,6 +233,7 @@ def validate_recipe(recipe: Recipe) -> list[str]: errors.append( f"Step '{step_name}'.capture.{cap_key} references " f"'{inner}'; capture values must use the 'result.' namespace." + + _SKILL_HINT ) # Validate input and context references in with_args using iter_steps_with_context @@ -235,6 +248,7 @@ def validate_recipe(recipe: Recipe) -> list[str]: if ref not in ingredient_names: errors.append( f"Step '{step_name}'.with.{arg_key} references undeclared input '{ref}'." + + _SKILL_HINT ) for ref in _CONTEXT_REF_RE.findall(arg_val): if ref not in available_context and ref not in step.optional_context_refs: @@ -247,12 +261,12 @@ def validate_recipe(recipe: Recipe) -> list[str]: if not recipe.kitchen_rules: errors.append( "Recipe has no 'kitchen_rules' field. Recipes should include " - "orchestrator discipline constraints." + "orchestrator discipline constraints." + _SKILL_HINT ) return errors # Re-export test-access symbols from their new locations. -from autoskillit.recipe.rules_inputs import _check_outdated_version # noqa: E402 F401 -from autoskillit.recipe.rules_worktree import _WORKTREE_MODIFYING_SKILLS # noqa: E402 F401 +from autoskillit.recipe.rules.rules_inputs import _check_outdated_version # noqa: E402 F401 +from autoskillit.recipe.rules.rules_worktree import _WORKTREE_MODIFYING_SKILLS # noqa: E402 F401 diff --git a/src/autoskillit/recipes/CLAUDE.md b/src/autoskillit/recipes/CLAUDE.md new file mode 100644 index 0000000000..191b384ba2 --- /dev/null +++ b/src/autoskillit/recipes/CLAUDE.md @@ -0,0 +1,3 @@ +# Recipes + +Recipe YAML files are read by an LLM orchestrator, not a code interpreter. Step-level fields like `model:`, `note:`, and `kitchen_rules:` are prompts — if a field is absent from a step, the orchestrator doesn't know to pass it. diff --git a/src/autoskillit/recipes/bem-wrapper.json b/src/autoskillit/recipes/bem-wrapper.json new file mode 100644 index 0000000000..6337e9f751 --- /dev/null +++ b/src/autoskillit/recipes/bem-wrapper.json @@ -0,0 +1,102 @@ +{ + "name": "bem-wrapper", + "description": "Re-fetch open audit issues, run build-execution-map, emit execution map path and compact dispatch plan. Used by the promote-to-main campaign to compute issue distribution before dispatching implementation L3s.", + "kind": "food-truck", + "recipe_version": "1.0.0", + "requires_packs": [ + "github" + ], + "kitchen_rules": [ + "NEVER use native Claude Code tools (Read, Grep, Glob, Edit, Write, Bash, Agent, WebFetch, WebSearch, NotebookEdit) from the orchestrator. All work is delegated through run_cmd and run_skill.", + "This recipe is a food truck dispatched by a campaign. Emit the L3 sentinel JSON block exactly as instructed in the done step message — no extra tool calls after reaching done." + ], + "ingredients": { + "issue_urls": { + "description": "Comma-separated GitHub issue URLs from full-audit", + "required": true + }, + "base_branch": { + "description": "Base branch for dependency analysis", + "default": "develop" + } + }, + "steps": { + "refetch_issues": { + "tool": "run_python", + "with": { + "callable": "autoskillit.recipe._cmd_rpc.refetch_issues", + "issue_urls": "${{ inputs.issue_urls }}" + }, + "capture": { + "issue_numbers": "${{ result.issue_numbers }}" + }, + "on_success": "check_issue_count", + "on_failure": "emit_fallback_map" + }, + "check_issue_count": { + "action": "route", + "on_result": [ + { + "when": "${{ context.issue_numbers }} == ''", + "route": "emit_empty_result" + }, + { + "route": "run_bem" + } + ] + }, + "run_bem": { + "tool": "run_skill", + "model": "", + "with": { + "skill_command": "/autoskillit:build-execution-map ${{ context.issue_numbers }} --base-ref ${{ inputs.base_branch }}", + "step_name": "run_bem" + }, + "retries": 1, + "capture": { + "execution_map": "${{ result.execution_map }}" + }, + "on_success": "emit_result", + "on_failure": "emit_fallback_map" + }, + "emit_fallback_map": { + "tool": "run_python", + "with": { + "callable": "autoskillit.recipe._cmd_rpc.emit_fallback_map", + "issue_urls": "${{ inputs.issue_urls }}", + "temp_dir": "{{AUTOSKILLIT_TEMP}}" + }, + "capture": { + "execution_map": "${{ result.execution_map }}" + }, + "on_success": "emit_result", + "on_failure": "escalate_stop" + }, + "emit_result": { + "tool": "run_cmd", + "with": { + "cmd": "echo 'execution_map=${{ context.execution_map }}'", + "step_name": "emit_result" + }, + "on_success": "done", + "on_failure": "done" + }, + "emit_empty_result": { + "tool": "run_cmd", + "with": { + "cmd": "echo 'execution_map=none'", + "step_name": "emit_empty_result" + }, + "on_success": "done", + "on_failure": "done" + }, + "done": { + "action": "stop", + "message": "Execution map computed. The context variable execution_map holds either a file path or \"none\" (if all issues were closed during review gate). Emit your L3 sentinel JSON block now. The sentinel must include: success=true, execution_map=, and dispatch_plan=. To build dispatch_plan: if execution_map is a file path, parse the JSON and for each group emit {\"group\": , \"parallel\": , \"issues\": \"\"}. If execution_map is \"none\", set dispatch_plan=[]. Example sentinel: {\"success\": true, \"execution_map\": \"/path/map.json\", \"dispatch_plan\": [{\"group\": 1, \"parallel\": true, \"issues\": \"1155,1156\"},\n {\"group\": 2, \"parallel\": false, \"issues\": \"1157\"}]}" + }, + "escalate_stop": { + "action": "stop", + "message": "BEM wrapper failed — fallback execution map could not be written to disk. Human intervention required. Check {{AUTOSKILLIT_TEMP}} permissions and issue URLs in the ingredients." + } + } +} diff --git a/src/autoskillit/recipes/bem-wrapper.yaml b/src/autoskillit/recipes/bem-wrapper.yaml new file mode 100644 index 0000000000..d4e626bea3 --- /dev/null +++ b/src/autoskillit/recipes/bem-wrapper.yaml @@ -0,0 +1,104 @@ +name: bem-wrapper +description: >- + Re-fetch open audit issues, run build-execution-map, emit execution map path + and compact dispatch plan. Used by the promote-to-main campaign to compute + issue distribution before dispatching implementation L3s. +kind: food-truck +recipe_version: "1.0.0" + +requires_packs: [github] + +kitchen_rules: + - "NEVER use native Claude Code tools (Read, Grep, Glob, Edit, Write, Bash, Agent, + WebFetch, WebSearch, NotebookEdit) from the orchestrator. All work is delegated + through run_cmd and run_skill." + - "This recipe is a food truck dispatched by a campaign. Emit the L3 sentinel JSON + block exactly as instructed in the done step message — no extra tool calls after + reaching done." + +ingredients: + issue_urls: + description: "Comma-separated GitHub issue URLs from full-audit" + required: true + base_branch: + description: "Base branch for dependency analysis" + default: "develop" + +steps: + refetch_issues: + tool: run_python + with: + callable: "autoskillit.recipe._cmd_rpc.refetch_issues" + issue_urls: "${{ inputs.issue_urls }}" + capture: + issue_numbers: "${{ result.issue_numbers }}" + on_success: check_issue_count + on_failure: emit_fallback_map + + check_issue_count: + action: route + on_result: + - when: "${{ context.issue_numbers }} == ''" + route: emit_empty_result + - route: run_bem + + run_bem: + tool: run_skill + model: "" + with: + skill_command: "/autoskillit:build-execution-map ${{ context.issue_numbers }} --base-ref ${{ inputs.base_branch }}" + step_name: run_bem + retries: 1 + capture: + execution_map: "${{ result.execution_map }}" + on_success: emit_result + on_failure: emit_fallback_map + + emit_fallback_map: + tool: run_python + with: + callable: "autoskillit.recipe._cmd_rpc.emit_fallback_map" + issue_urls: "${{ inputs.issue_urls }}" + temp_dir: "{{AUTOSKILLIT_TEMP}}" + capture: + execution_map: "${{ result.execution_map }}" + on_success: emit_result + on_failure: escalate_stop + + emit_result: + tool: run_cmd + with: + cmd: "echo 'execution_map=${{ context.execution_map }}'" + step_name: emit_result + on_success: done + on_failure: done + + emit_empty_result: + tool: run_cmd + with: + cmd: "echo 'execution_map=none'" + step_name: emit_empty_result + on_success: done + on_failure: done + + done: + action: stop + message: >- + Execution map computed. The context variable execution_map holds either a + file path or "none" (if all issues were closed during review gate). Emit + your L3 sentinel JSON block now. The sentinel must include: success=true, + execution_map=, and dispatch_plan=. + To build dispatch_plan: if execution_map is a file path, parse the JSON + and for each group emit {"group": , "parallel": , "issues": + ""}. If execution_map is "none", + set dispatch_plan=[]. Example sentinel: + {"success": true, "execution_map": "/path/map.json", "dispatch_plan": + [{"group": 1, "parallel": true, "issues": "1155,1156"}, + {"group": 2, "parallel": false, "issues": "1157"}]} + + escalate_stop: + action: stop + message: >- + BEM wrapper failed — fallback execution map could not be written to disk. + Human intervention required. Check {{AUTOSKILLIT_TEMP}} permissions and + issue URLs in the ingredients. diff --git a/src/autoskillit/recipes/campaigns/promote-to-main.json b/src/autoskillit/recipes/campaigns/promote-to-main.json new file mode 100644 index 0000000000..22651f4c0e --- /dev/null +++ b/src/autoskillit/recipes/campaigns/promote-to-main.json @@ -0,0 +1,116 @@ +{ + "name": "promote-to-main", + "description": "Full promotion lifecycle: audit integration branch, human review gate, implement audit findings with BEM-guided distribution, promote to main.", + "kind": "campaign", + "recipe_version": "1.0.0", + "categories": [ + "orchestration-family" + ], + "continue_on_failure": false, + "allowed_recipes": [ + "full-audit", + "bem-wrapper", + "implement-findings", + "promote-to-main-wrapper" + ], + "ingredients": { + "source_dir": { + "description": "Source repository path", + "required": true + }, + "batch_branch": { + "description": "Integration branch to promote", + "default": "develop" + }, + "base_branch": { + "description": "Target branch for promotion PR", + "default": "main" + }, + "max_issues_per_l2": { + "description": "Max issues per L2 food truck", + "default": "6", + "hidden": true + }, + "model_context_window": { + "description": "Model context window in tokens; set to 1000000 for Opus 1M", + "default": "200000", + "hidden": true + } + }, + "dispatches": [ + { + "name": "full-audit", + "recipe": "full-audit", + "task": "Run comprehensive audit on integration branch", + "ingredients": { + "workspace": "${{ inputs.source_dir }}", + "branch": "${{ inputs.batch_branch }}" + }, + "capture": { + "issue_urls": "${{ result.issue_urls }}" + }, + "depends_on": [] + }, + { + "name": "review-gate", + "gate": "confirm", + "message": "Full audit complete. Review the created GitHub issues. Close any false positives, then approve to proceed.", + "depends_on": [ + "full-audit" + ] + }, + { + "name": "build-map", + "recipe": "bem-wrapper", + "task": "Analyze issue dependencies and compute distribution plan", + "ingredients": { + "issue_urls": "${{ campaign.issue_urls }}", + "base_branch": "${{ inputs.batch_branch }}" + }, + "capture": { + "execution_map": "${{ result.execution_map }}" + }, + "depends_on": [ + "review-gate" + ] + }, + { + "name": "implement-findings", + "recipe": "implement-findings", + "task": "Implement audit findings guided by BEM execution map", + "ingredients": { + "issue_urls": "${{ campaign.issue_urls }}", + "execution_map": "${{ campaign.execution_map }}", + "base_branch": "${{ inputs.batch_branch }}", + "source_dir": "${{ inputs.source_dir }}", + "max_parallel": "${{ inputs.max_issues_per_l2 }}", + "model_context_window": "${{ inputs.model_context_window }}" + }, + "depends_on": [ + "build-map" + ] + }, + { + "name": "promote", + "recipe": "promote-to-main-wrapper", + "task": "Create promotion PR from integration branch to main", + "ingredients": { + "batch_branch": "${{ inputs.batch_branch }}", + "base_branch": "${{ inputs.base_branch }}", + "source_dir": "${{ inputs.source_dir }}" + }, + "capture": { + "pr_url": "${{ result.pr_url }}", + "verdict": "${{ result.verdict }}" + }, + "depends_on": [ + "implement-findings" + ] + } + ], + "kitchen_rules": [ + "The review-gate dispatch requires explicit human approval before proceeding.", + "Issues closed by the human during review are excluded by the BEM wrapper." + ], + "steps": {} +} diff --git a/src/autoskillit/recipes/campaigns/promote-to-main.yaml b/src/autoskillit/recipes/campaigns/promote-to-main.yaml new file mode 100644 index 0000000000..2f4bdbad04 --- /dev/null +++ b/src/autoskillit/recipes/campaigns/promote-to-main.yaml @@ -0,0 +1,97 @@ +name: promote-to-main +description: >- + Full promotion lifecycle: audit integration branch, human review gate, + implement audit findings with BEM-guided distribution, promote to main. +kind: campaign +recipe_version: "1.0.0" +categories: + - orchestration-family + +continue_on_failure: false + +allowed_recipes: + - full-audit + - bem-wrapper + - implement-findings + - promote-to-main-wrapper + +ingredients: + source_dir: + description: "Source repository path" + required: true + batch_branch: + description: "Integration branch to promote" + default: "develop" + base_branch: + description: "Target branch for promotion PR" + default: "main" + max_issues_per_l2: + description: "Max issues per L2 food truck" + default: "6" + hidden: true + model_context_window: + description: "Model context window in tokens; set to 1000000 for Opus 1M" + default: "200000" + hidden: true + +dispatches: + - name: full-audit + recipe: full-audit + task: "Run comprehensive audit on integration branch" + ingredients: + workspace: "${{ inputs.source_dir }}" + branch: "${{ inputs.batch_branch }}" + capture: + issue_urls: "${{ result.issue_urls }}" + depends_on: [] + + - name: review-gate + gate: confirm + message: >- + Full audit complete. Review the created GitHub issues. + Close any false positives, then approve to proceed. + depends_on: + - full-audit + + - name: build-map + recipe: bem-wrapper + task: "Analyze issue dependencies and compute distribution plan" + ingredients: + issue_urls: "${{ campaign.issue_urls }}" + base_branch: "${{ inputs.batch_branch }}" + capture: + execution_map: "${{ result.execution_map }}" + depends_on: + - review-gate + + - name: implement-findings + recipe: implement-findings + task: "Implement audit findings guided by BEM execution map" + ingredients: + issue_urls: "${{ campaign.issue_urls }}" + execution_map: "${{ campaign.execution_map }}" + base_branch: "${{ inputs.batch_branch }}" + source_dir: "${{ inputs.source_dir }}" + max_parallel: "${{ inputs.max_issues_per_l2 }}" + model_context_window: "${{ inputs.model_context_window }}" + depends_on: + - build-map + + - name: promote + recipe: promote-to-main-wrapper + task: "Create promotion PR from integration branch to main" + ingredients: + batch_branch: "${{ inputs.batch_branch }}" + base_branch: "${{ inputs.base_branch }}" + source_dir: "${{ inputs.source_dir }}" + capture: + pr_url: "${{ result.pr_url }}" + verdict: "${{ result.verdict }}" + depends_on: + - implement-findings + +kitchen_rules: + - "The review-gate dispatch requires explicit human approval before proceeding." + - "Issues closed by the human during review are excluded by the BEM wrapper." + +steps: {} diff --git a/src/autoskillit/recipes/campaigns/research-campaign.json b/src/autoskillit/recipes/campaigns/research-campaign.json new file mode 100644 index 0000000000..47fc4c3d3d --- /dev/null +++ b/src/autoskillit/recipes/campaigns/research-campaign.json @@ -0,0 +1,141 @@ +{ + "name": "research-campaign", + "description": "Four-phase research campaign lifecycle: design the experiment, implement and run it, review the results, then archive artifacts. Dispatches to research-design, research-implement, research-review, and research-archive sub-recipes in sequence.", + "kind": "campaign", + "recipe_version": "1.0.0", + "categories": [ + "research-family" + ], + "continue_on_failure": false, + "requires_recipe_packs": [ + "research-family" + ], + "allowed_recipes": [ + "research-design", + "research-implement", + "research-review", + "research-archive" + ], + "ingredients": { + "task": { + "description": "Research question or topic to investigate", + "required": true + }, + "source_dir": { + "description": "Path to the project root", + "required": true + }, + "issue_url": { + "description": "Optional URL to an existing GitHub issue describing the research goal. Provided as context to scope and plan_experiment.", + "required": false + }, + "base_branch": { + "description": "Branch to target for the research PR", + "default": "main" + }, + "review_design": { + "description": "When 'true' (the default), run the review-design skill after plan_experiment to validate the experiment plan before execution. When 'false', skip directly to create_worktree.", + "default": "true" + }, + "review_pr": { + "description": "When 'true', run the review-research-pr skill after the research PR is opened. Defaults to 'false' — the research PR is for human review.", + "default": "false" + }, + "audit_claims": { + "description": "When 'true', run audit-claims after review_research_pr to verify citation integrity and claim backing in the research report.", + "required": false, + "default": "false" + }, + "output_mode": { + "description": "Where research outputs land. 'local' (default) produces a self-contained report bundle. 'pr' opens a GitHub PR and runs the full review/archival flow.", + "required": false, + "default": "local" + } + }, + "dispatches": [ + { + "name": "run-design", + "recipe": "research-design", + "task": "Run the research design phase: scope the question, plan the experiment, and optionally review the design.", + "ingredients": { + "task": "${{ inputs.task }}", + "issue_url": "${{ inputs.issue_url }}", + "source_dir": "${{ inputs.source_dir }}", + "base_branch": "${{ inputs.base_branch }}", + "review_design": "${{ inputs.review_design }}" + }, + "capture": { + "worktree_path": "${{ result.worktree_path }}", + "research_dir": "${{ result.research_dir }}", + "experiment_plan": "${{ result.experiment_plan }}", + "visualization_plan_path": "${{ result.visualization_plan_path }}" + }, + "depends_on": [] + }, + { + "name": "run-implement", + "recipe": "research-implement", + "task": "Implement the research experiment plan.", + "ingredients": { + "worktree_path": "${{ campaign.worktree_path }}", + "research_dir": "${{ campaign.research_dir }}", + "experiment_plan": "${{ campaign.experiment_plan }}", + "visualization_plan_path": "${{ campaign.visualization_plan_path }}", + "source_dir": "${{ inputs.source_dir }}", + "base_branch": "${{ inputs.base_branch }}", + "issue_url": "${{ inputs.issue_url }}", + "output_mode": "${{ inputs.output_mode }}" + }, + "capture": { + "report_path": "${{ result.report_path }}" + }, + "depends_on": [ + "run-design" + ] + }, + { + "name": "run-review", + "recipe": "research-review", + "task": "Review the research experiment and open PR.", + "ingredients": { + "worktree_path": "${{ campaign.worktree_path }}", + "research_dir": "${{ campaign.research_dir }}", + "experiment_plan": "${{ campaign.experiment_plan }}", + "visualization_plan_path": "${{ campaign.visualization_plan_path }}", + "report_path": "${{ campaign.report_path }}", + "source_dir": "${{ inputs.source_dir }}", + "base_branch": "${{ inputs.base_branch }}", + "issue_url": "${{ inputs.issue_url }}", + "output_mode": "${{ inputs.output_mode }}", + "review_pr": "${{ inputs.review_pr }}", + "audit_claims": "${{ inputs.audit_claims }}" + }, + "capture": { + "pr_url": "${{ result.pr_url }}", + "all_diagram_paths": "${{ result.all_diagram_paths }}", + "report_path_after_finalize": "${{ result.report_path_after_finalize }}" + }, + "depends_on": [ + "run-implement" + ] + }, + { + "name": "run-archive", + "recipe": "research-archive", + "task": "Archive research artifacts and close experiment PR.", + "ingredients": { + "base_branch": "${{ inputs.base_branch }}", + "worktree_path": "${{ campaign.worktree_path }}", + "research_dir": "${{ campaign.research_dir }}", + "pr_url": "${{ campaign.pr_url }}", + "all_diagram_paths": "${{ campaign.all_diagram_paths }}", + "report_path_after_finalize": "${{ campaign.report_path_after_finalize }}" + }, + "capture": {}, + "depends_on": [ + "run-review" + ] + } + ], + "steps": {} +} diff --git a/src/autoskillit/recipes/campaigns/research-campaign.yaml b/src/autoskillit/recipes/campaigns/research-campaign.yaml new file mode 100644 index 0000000000..0014984261 --- /dev/null +++ b/src/autoskillit/recipes/campaigns/research-campaign.yaml @@ -0,0 +1,132 @@ +name: research-campaign +description: >- + Four-phase research campaign lifecycle: design the experiment, implement and + run it, review the results, then archive artifacts. Dispatches to + research-design, research-implement, research-review, and research-archive + sub-recipes in sequence. +kind: campaign +recipe_version: "1.0.0" +categories: + - research-family + +continue_on_failure: false + +requires_recipe_packs: + - research-family + +allowed_recipes: + - research-design + - research-implement + - research-review + - research-archive + +ingredients: + task: + description: Research question or topic to investigate + required: true + source_dir: + description: Path to the project root + required: true + issue_url: + description: >- + Optional URL to an existing GitHub issue describing the research goal. + Provided as context to scope and plan_experiment. + required: false + base_branch: + description: Branch to target for the research PR + default: "main" + review_design: + description: >- + When 'true' (the default), run the review-design skill after plan_experiment + to validate the experiment plan before execution. When 'false', skip directly + to create_worktree. + default: "true" + review_pr: + description: >- + When 'true', run the review-research-pr skill after the research PR is opened. + Defaults to 'false' — the research PR is for human review. + default: "false" + audit_claims: + description: >- + When 'true', run audit-claims after review_research_pr to verify citation + integrity and claim backing in the research report. + required: false + default: "false" + output_mode: + description: >- + Where research outputs land. 'local' (default) produces a self-contained + report bundle. 'pr' opens a GitHub PR and runs the full review/archival flow. + required: false + default: "local" + +dispatches: + - name: run-design + recipe: research-design + task: "Run the research design phase: scope the question, plan the experiment, and optionally review the design." + ingredients: + task: "${{ inputs.task }}" + issue_url: "${{ inputs.issue_url }}" + source_dir: "${{ inputs.source_dir }}" + base_branch: "${{ inputs.base_branch }}" + review_design: "${{ inputs.review_design }}" + capture: + worktree_path: "${{ result.worktree_path }}" + research_dir: "${{ result.research_dir }}" + experiment_plan: "${{ result.experiment_plan }}" + visualization_plan_path: "${{ result.visualization_plan_path }}" + depends_on: [] + + - name: run-implement + recipe: research-implement + task: "Implement the research experiment plan." + ingredients: + worktree_path: "${{ campaign.worktree_path }}" + research_dir: "${{ campaign.research_dir }}" + experiment_plan: "${{ campaign.experiment_plan }}" + visualization_plan_path: "${{ campaign.visualization_plan_path }}" + source_dir: "${{ inputs.source_dir }}" + base_branch: "${{ inputs.base_branch }}" + issue_url: "${{ inputs.issue_url }}" + output_mode: "${{ inputs.output_mode }}" + capture: + report_path: "${{ result.report_path }}" + depends_on: + - run-design + + - name: run-review + recipe: research-review + task: "Review the research experiment and open PR." + ingredients: + worktree_path: "${{ campaign.worktree_path }}" + research_dir: "${{ campaign.research_dir }}" + experiment_plan: "${{ campaign.experiment_plan }}" + visualization_plan_path: "${{ campaign.visualization_plan_path }}" + report_path: "${{ campaign.report_path }}" + source_dir: "${{ inputs.source_dir }}" + base_branch: "${{ inputs.base_branch }}" + issue_url: "${{ inputs.issue_url }}" + output_mode: "${{ inputs.output_mode }}" + review_pr: "${{ inputs.review_pr }}" + audit_claims: "${{ inputs.audit_claims }}" + capture: + pr_url: "${{ result.pr_url }}" + all_diagram_paths: "${{ result.all_diagram_paths }}" + report_path_after_finalize: "${{ result.report_path_after_finalize }}" + depends_on: + - run-implement + + - name: run-archive + recipe: research-archive + task: "Archive research artifacts and close experiment PR." + ingredients: + base_branch: "${{ inputs.base_branch }}" + worktree_path: "${{ campaign.worktree_path }}" + research_dir: "${{ campaign.research_dir }}" + pr_url: "${{ campaign.pr_url }}" + all_diagram_paths: "${{ campaign.all_diagram_paths }}" + report_path_after_finalize: "${{ campaign.report_path_after_finalize }}" + capture: {} + depends_on: + - run-review + +steps: {} diff --git a/src/autoskillit/recipes/contracts/bem-wrapper.json b/src/autoskillit/recipes/contracts/bem-wrapper.json new file mode 100644 index 0000000000..0b8a5861f9 --- /dev/null +++ b/src/autoskillit/recipes/contracts/bem-wrapper.json @@ -0,0 +1,132 @@ +{ + "generated_at": "2026-04-28T09:27:39.466273+00:00", + "bundled_manifest_version": "0.1.0", + "skill_hashes": {}, + "skills": { + "build-execution-map": { + "inputs": [], + "outputs": [ + { + "name": "execution_map", + "type": "file_path" + }, + { + "name": "execution_map_report", + "type": "file_path" + }, + { + "name": "group_count", + "type": "string" + }, + { + "name": "total_issues", + "type": "string" + }, + { + "name": "review_approach_candidates", + "type": "string" + } + ], + "expected_output_patterns": [ + "execution_map\\s*=\\s*/.+" + ], + "pattern_examples": [ + "execution_map = /tmp/build-execution-map/execution_map_2026-04-22_190000.json\nexecution_map_report = /tmp/build-execution-map/execution_map_report_2026-04-22_190000.md\ngroup_count = 3\ntotal_issues = 5" + ], + "write_behavior": "always" + } + }, + "dataflow": [ + { + "step": "refetch_issues", + "available": [ + "base_branch", + "issue_urls" + ], + "required": [], + "produced": [ + "issue_numbers" + ] + }, + { + "step": "check_issue_count", + "available": [ + "base_branch", + "issue_numbers", + "issue_urls" + ], + "required": [], + "produced": [] + }, + { + "step": "run_bem", + "available": [ + "base_branch", + "issue_numbers", + "issue_urls" + ], + "required": [], + "produced": [ + "execution_map" + ], + "positional_args": 1 + }, + { + "step": "emit_fallback_map", + "available": [ + "base_branch", + "execution_map", + "issue_numbers", + "issue_urls" + ], + "required": [], + "produced": [ + "execution_map" + ] + }, + { + "step": "emit_result", + "available": [ + "base_branch", + "execution_map", + "issue_numbers", + "issue_urls" + ], + "required": [], + "produced": [] + }, + { + "step": "emit_empty_result", + "available": [ + "base_branch", + "execution_map", + "issue_numbers", + "issue_urls" + ], + "required": [], + "produced": [] + }, + { + "step": "done", + "available": [ + "base_branch", + "execution_map", + "issue_numbers", + "issue_urls" + ], + "required": [], + "produced": [] + }, + { + "step": "escalate_stop", + "available": [ + "base_branch", + "execution_map", + "issue_numbers", + "issue_urls" + ], + "required": [], + "produced": [] + } + ] +} diff --git a/src/autoskillit/recipes/contracts/bem-wrapper.yaml b/src/autoskillit/recipes/contracts/bem-wrapper.yaml new file mode 100644 index 0000000000..5a177ce1e0 --- /dev/null +++ b/src/autoskillit/recipes/contracts/bem-wrapper.yaml @@ -0,0 +1,93 @@ +generated_at: '2026-04-28T09:27:39.466273+00:00' +bundled_manifest_version: 0.1.0 +skill_hashes: {} +skills: + build-execution-map: + inputs: [] + outputs: + - name: execution_map + type: file_path + - name: execution_map_report + type: file_path + - name: group_count + type: string + - name: total_issues + type: string + - name: review_approach_candidates + type: string + expected_output_patterns: + - execution_map\s*=\s*/.+ + pattern_examples: + - 'execution_map = /tmp/build-execution-map/execution_map_2026-04-22_190000.json + + execution_map_report = /tmp/build-execution-map/execution_map_report_2026-04-22_190000.md + + group_count = 3 + + total_issues = 5' + write_behavior: always +dataflow: +- step: refetch_issues + available: + - base_branch + - issue_urls + required: [] + produced: + - issue_numbers +- step: check_issue_count + available: + - base_branch + - issue_numbers + - issue_urls + required: [] + produced: [] +- step: run_bem + available: + - base_branch + - issue_numbers + - issue_urls + required: [] + produced: + - execution_map + positional_args: 1 +- step: emit_fallback_map + available: + - base_branch + - execution_map + - issue_numbers + - issue_urls + required: [] + produced: + - execution_map +- step: emit_result + available: + - base_branch + - execution_map + - issue_numbers + - issue_urls + required: [] + produced: [] +- step: emit_empty_result + available: + - base_branch + - execution_map + - issue_numbers + - issue_urls + required: [] + produced: [] +- step: done + available: + - base_branch + - execution_map + - issue_numbers + - issue_urls + required: [] + produced: [] +- step: escalate_stop + available: + - base_branch + - execution_map + - issue_numbers + - issue_urls + required: [] + produced: [] diff --git a/src/autoskillit/recipes/contracts/implementation-groups.json b/src/autoskillit/recipes/contracts/implementation-groups.json new file mode 100644 index 0000000000..34bace1da0 --- /dev/null +++ b/src/autoskillit/recipes/contracts/implementation-groups.json @@ -0,0 +1,2587 @@ +{ + "generated_at": "2026-04-05T02:19:56.037706+00:00", + "bundled_manifest_version": "0.1.0", + "skill_hashes": {}, + "skills": { + "make-groups": { + "inputs": [ + { + "name": "source_doc", + "type": "file_path", + "required": true + } + ], + "outputs": [ + { + "name": "groups_path", + "type": "file_path" + }, + { + "name": "manifest_path", + "type": "file_path" + }, + { + "name": "group_files", + "type": "file_path_list" + } + ], + "expected_output_patterns": [ + "groups_path\\s*=\\s*/.+" + ], + "pattern_examples": [ + "groups_path = /tmp/groups.json" + ] + }, + "make-plan": { + "inputs": [ + { + "name": "task", + "type": "string", + "required": true + } + ], + "outputs": [ + { + "name": "plan_path", + "type": "file_path" + }, + { + "name": "plan_parts", + "type": "file_path_list" + } + ], + "expected_output_patterns": [ + "plan_path\\s*=\\s*/.+" + ], + "pattern_examples": [ + "plan_path = /tmp/plan.md" + ], + "write_behavior": "always" + }, + "review-approach": { + "inputs": [ + { + "name": "plan_path", + "type": "file_path", + "required": false + } + ], + "outputs": [ + { + "name": "review_path", + "type": "file_path" + } + ], + "expected_output_patterns": [ + "review_path\\s*=\\s*/.+" + ], + "pattern_examples": [ + "review_path = /tmp/review.md" + ] + }, + "dry-walkthrough": { + "inputs": [ + { + "name": "plan_path", + "type": "file_path", + "required": true + } + ], + "outputs": [], + "expected_output_patterns": [], + "pattern_examples": [], + "write_behavior": "always" + }, + "implement-worktree-no-merge": { + "inputs": [ + { + "name": "plan_path", + "type": "file_path", + "required": true + } + ], + "outputs": [ + { + "name": "worktree_path", + "type": "directory_path" + }, + { + "name": "branch_name", + "type": "string" + } + ], + "expected_output_patterns": [ + "worktree_path\\s*=\\s*/.+" + ], + "pattern_examples": [ + "worktree_path = /tmp/worktrees/impl-foo-20260316" + ], + "write_behavior": "always" + }, + "retry-worktree": { + "inputs": [ + { + "name": "plan_path", + "type": "file_path", + "required": true + }, + { + "name": "worktree_path", + "type": "directory_path", + "required": true + } + ], + "outputs": [ + { + "name": "worktree_path", + "type": "directory_path" + }, + { + "name": "branch_name", + "type": "string" + } + ], + "expected_output_patterns": [ + "worktree_path\\s*=\\s*/.+" + ], + "pattern_examples": [ + "worktree_path = /tmp/worktrees/impl-foo-20260316" + ], + "write_behavior": "always" + }, + "resolve-failures": { + "inputs": [ + { + "name": "worktree_path", + "type": "directory_path", + "required": true + }, + { + "name": "plan_path", + "type": "file_path", + "required": true + }, + { + "name": "base_branch", + "type": "string", + "required": true + } + ], + "outputs": [], + "expected_output_patterns": [], + "pattern_examples": [], + "write_behavior": "always" + }, + "audit-impl": { + "inputs": [ + { + "name": "plan_path", + "type": "file_path", + "required": true + }, + { + "name": "branch_name", + "type": "string", + "required": false + }, + { + "name": "base_branch", + "type": "string", + "required": false + } + ], + "outputs": [ + { + "name": "verdict", + "type": "string" + }, + { + "name": "remediation_path", + "type": "file_path" + } + ], + "expected_output_patterns": [ + "verdict\\s*=\\s*(GO|NO GO)" + ], + "pattern_examples": [ + "verdict = GO", + "verdict = NO GO\nremediation_path = /tmp/remediation.md" + ] + }, + "open-pr": { + "inputs": [ + { + "name": "plan_paths", + "type": "string", + "required": true + }, + { + "name": "feature_branch", + "type": "string", + "required": true + }, + { + "name": "base_branch", + "type": "string", + "required": true + }, + { + "name": "closing_issue", + "type": "string", + "required": false + } + ], + "outputs": [], + "expected_output_patterns": [ + "https://github\\.com/.*/pull/\\d+" + ], + "pattern_examples": [ + "https://github.com/owner/repo/pull/42" + ] + }, + "review-pr": { + "inputs": [ + { + "name": "feature_branch", + "type": "string", + "required": true + }, + { + "name": "base_branch", + "type": "string", + "required": true + }, + { + "name": "annotated_diff_path", + "type": "string", + "required": false + }, + { + "name": "hunk_ranges_path", + "type": "string", + "required": false + } + ], + "outputs": [ + { + "name": "verdict", + "type": "string" + } + ], + "expected_output_patterns": [ + "verdict\\s*=\\s*(approved|approved_with_comments|changes_requested|needs_human)" + ], + "pattern_examples": [ + "verdict = approved", + "verdict = approved_with_comments", + "verdict = changes_requested", + "verdict = needs_human" + ] + }, + "resolve-review": { + "inputs": [ + { + "name": "feature_branch", + "type": "string", + "required": true + }, + { + "name": "base_branch", + "type": "string", + "required": true + } + ], + "outputs": [], + "expected_output_patterns": [], + "pattern_examples": [], + "write_behavior": "always" + }, + "diagnose-ci": { + "inputs": [ + { + "name": "branch", + "type": "string", + "required": true + }, + { + "name": "run_id", + "type": "string", + "required": false + }, + { + "name": "ci_failed_jobs", + "type": "string", + "required": false + } + ], + "outputs": [ + { + "name": "diagnosis_path", + "type": "file_path" + }, + { + "name": "failure_type", + "type": "string" + }, + { + "name": "is_fixable", + "type": "string" + } + ], + "expected_output_patterns": [ + "diagnosis_path\\s*=\\s*/.+" + ], + "pattern_examples": [ + "diagnosis_path = /tmp/diagnosis.md" + ], + "write_behavior": "always" + }, + "resolve-merge-conflicts": { + "inputs": [ + { + "name": "worktree_path", + "type": "directory_path", + "required": true + }, + { + "name": "plan_path", + "type": "file_path", + "required": true + }, + { + "name": "base_branch", + "type": "string", + "required": true + } + ], + "outputs": [ + { + "name": "worktree_path", + "type": "directory_path" + }, + { + "name": "branch_name", + "type": "string" + }, + { + "name": "conflict_report_path", + "type": "file_path" + }, + { + "name": "escalation_required", + "type": "string" + }, + { + "name": "escalation_reason", + "type": "string" + } + ], + "expected_output_patterns": [ + "worktree_path\\s*=\\s*/.+" + ], + "pattern_examples": [ + "worktree_path = /tmp/worktrees/impl-foo-20260316" + ], + "write_behavior": "conditional", + "write_expected_when": [ + "conflict_report_path\\s*=\\s*/.+" + ] + } + }, + "dataflow": [ + { + "step": "clone", + "available": [ + "audit", + "auto_merge", + "base_branch", + "issue_url", + "open_pr", + "review_approach", + "run_name", + "source_dir", + "source_doc", + "upfront_claimed" + ], + "required": [], + "produced": [ + "work_dir", + "remote_url" + ] + }, + { + "step": "capture_base_sha", + "available": [ + "audit", + "auto_merge", + "base_branch", + "issue_url", + "open_pr", + "remote_url", + "review_approach", + "run_name", + "source_dir", + "source_doc", + "upfront_claimed", + "work_dir" + ], + "required": [], + "produced": [ + "base_sha" + ] + }, + { + "step": "set_merge_target", + "available": [ + "audit", + "auto_merge", + "base_branch", + "base_sha", + "issue_url", + "open_pr", + "remote_url", + "review_approach", + "run_name", + "source_dir", + "source_doc", + "upfront_claimed", + "work_dir" + ], + "required": [], + "produced": [ + "merge_target" + ] + }, + { + "step": "get_issue_title", + "available": [ + "audit", + "auto_merge", + "base_branch", + "base_sha", + "issue_url", + "merge_target", + "open_pr", + "remote_url", + "review_approach", + "run_name", + "source_dir", + "source_doc", + "upfront_claimed", + "work_dir" + ], + "required": [], + "produced": [ + "issue_number", + "issue_title", + "issue_slug" + ] + }, + { + "step": "claim_issue", + "available": [ + "audit", + "auto_merge", + "base_branch", + "base_sha", + "issue_number", + "issue_slug", + "issue_title", + "issue_url", + "merge_target", + "open_pr", + "remote_url", + "review_approach", + "run_name", + "source_dir", + "source_doc", + "upfront_claimed", + "work_dir" + ], + "required": [], + "produced": [ + "claimed" + ] + }, + { + "step": "compute_branch", + "available": [ + "audit", + "auto_merge", + "base_branch", + "base_sha", + "claimed", + "issue_number", + "issue_slug", + "issue_title", + "issue_url", + "merge_target", + "open_pr", + "remote_url", + "review_approach", + "run_name", + "source_dir", + "source_doc", + "upfront_claimed", + "work_dir" + ], + "required": [], + "produced": [ + "proposed_branch" + ] + }, + { + "step": "create_branch", + "available": [ + "audit", + "auto_merge", + "base_branch", + "base_sha", + "claimed", + "issue_number", + "issue_slug", + "issue_title", + "issue_url", + "merge_target", + "open_pr", + "proposed_branch", + "remote_url", + "review_approach", + "run_name", + "source_dir", + "source_doc", + "upfront_claimed", + "work_dir" + ], + "required": [], + "produced": [ + "merge_target" + ] + }, + { + "step": "push_merge_target", + "available": [ + "audit", + "auto_merge", + "base_branch", + "base_sha", + "claimed", + "issue_number", + "issue_slug", + "issue_title", + "issue_url", + "merge_target", + "open_pr", + "proposed_branch", + "remote_url", + "review_approach", + "run_name", + "source_dir", + "source_doc", + "upfront_claimed", + "work_dir" + ], + "required": [], + "produced": [] + }, + { + "step": "group", + "available": [ + "audit", + "auto_merge", + "base_branch", + "base_sha", + "claimed", + "issue_number", + "issue_slug", + "issue_title", + "issue_url", + "merge_target", + "open_pr", + "proposed_branch", + "remote_url", + "review_approach", + "run_name", + "source_dir", + "source_doc", + "upfront_claimed", + "work_dir" + ], + "required": [], + "produced": [ + "group_files" + ] + }, + { + "step": "plan", + "available": [ + "audit", + "auto_merge", + "base_branch", + "base_sha", + "claimed", + "group_files", + "issue_number", + "issue_slug", + "issue_title", + "issue_url", + "merge_target", + "open_pr", + "proposed_branch", + "remote_url", + "review_approach", + "run_name", + "source_dir", + "source_doc", + "upfront_claimed", + "work_dir" + ], + "required": [], + "produced": [ + "plan_path", + "all_plan_paths" + ] + }, + { + "step": "review", + "available": [ + "all_plan_paths", + "audit", + "auto_merge", + "base_branch", + "base_sha", + "claimed", + "group_files", + "issue_number", + "issue_slug", + "issue_title", + "issue_url", + "merge_target", + "open_pr", + "plan_path", + "proposed_branch", + "remote_url", + "review_approach", + "run_name", + "source_dir", + "source_doc", + "upfront_claimed", + "work_dir" + ], + "required": [], + "produced": [ + "review_path" + ] + }, + { + "step": "verify", + "available": [ + "all_plan_paths", + "audit", + "auto_merge", + "base_branch", + "base_sha", + "claimed", + "group_files", + "issue_number", + "issue_slug", + "issue_title", + "issue_url", + "merge_target", + "open_pr", + "plan_path", + "proposed_branch", + "remote_url", + "review_approach", + "review_path", + "run_name", + "source_dir", + "source_doc", + "upfront_claimed", + "work_dir" + ], + "required": [], + "produced": [] + }, + { + "step": "implement", + "available": [ + "all_plan_paths", + "audit", + "auto_merge", + "base_branch", + "base_sha", + "claimed", + "group_files", + "issue_number", + "issue_slug", + "issue_title", + "issue_url", + "merge_target", + "open_pr", + "plan_path", + "proposed_branch", + "remote_url", + "review_approach", + "review_path", + "run_name", + "source_dir", + "source_doc", + "upfront_claimed", + "work_dir" + ], + "required": [], + "produced": [ + "worktree_path" + ] + }, + { + "step": "retry_worktree", + "available": [ + "all_plan_paths", + "audit", + "auto_merge", + "base_branch", + "base_sha", + "claimed", + "group_files", + "issue_number", + "issue_slug", + "issue_title", + "issue_url", + "merge_target", + "open_pr", + "plan_path", + "proposed_branch", + "remote_url", + "review_approach", + "review_path", + "run_name", + "source_dir", + "source_doc", + "upfront_claimed", + "work_dir", + "worktree_path" + ], + "required": [], + "produced": [ + "worktree_path" + ] + }, + { + "step": "test", + "available": [ + "all_plan_paths", + "audit", + "auto_merge", + "base_branch", + "base_sha", + "claimed", + "group_files", + "issue_number", + "issue_slug", + "issue_title", + "issue_url", + "merge_target", + "open_pr", + "plan_path", + "proposed_branch", + "remote_url", + "review_approach", + "review_path", + "run_name", + "source_dir", + "source_doc", + "upfront_claimed", + "work_dir", + "worktree_path" + ], + "required": [], + "produced": [] + }, + { + "step": "merge", + "available": [ + "all_plan_paths", + "audit", + "auto_merge", + "base_branch", + "base_sha", + "claimed", + "group_files", + "issue_number", + "issue_slug", + "issue_title", + "issue_url", + "merge_target", + "open_pr", + "plan_path", + "proposed_branch", + "remote_url", + "review_approach", + "review_path", + "run_name", + "source_dir", + "source_doc", + "upfront_claimed", + "work_dir", + "worktree_path" + ], + "required": [], + "produced": [ + "cleanup_succeeded", + "worktree_path" + ] + }, + { + "step": "push", + "available": [ + "all_plan_paths", + "audit", + "auto_merge", + "base_branch", + "base_sha", + "claimed", + "cleanup_succeeded", + "group_files", + "issue_number", + "issue_slug", + "issue_title", + "issue_url", + "merge_target", + "open_pr", + "plan_path", + "proposed_branch", + "remote_url", + "review_approach", + "review_path", + "run_name", + "source_dir", + "source_doc", + "upfront_claimed", + "work_dir", + "worktree_path" + ], + "required": [], + "produced": [] + }, + { + "step": "fix", + "available": [ + "all_plan_paths", + "audit", + "auto_merge", + "base_branch", + "base_sha", + "claimed", + "cleanup_succeeded", + "group_files", + "issue_number", + "issue_slug", + "issue_title", + "issue_url", + "merge_target", + "open_pr", + "plan_path", + "proposed_branch", + "remote_url", + "review_approach", + "review_path", + "run_name", + "source_dir", + "source_doc", + "upfront_claimed", + "work_dir", + "worktree_path" + ], + "required": [], + "produced": [] + }, + { + "step": "next_or_done", + "available": [ + "all_plan_paths", + "audit", + "auto_merge", + "base_branch", + "base_sha", + "claimed", + "cleanup_succeeded", + "group_files", + "issue_number", + "issue_slug", + "issue_title", + "issue_url", + "merge_target", + "open_pr", + "plan_path", + "proposed_branch", + "remote_url", + "review_approach", + "review_path", + "run_name", + "source_dir", + "source_doc", + "upfront_claimed", + "work_dir", + "worktree_path" + ], + "required": [], + "produced": [] + }, + { + "step": "audit_impl", + "available": [ + "all_plan_paths", + "audit", + "auto_merge", + "base_branch", + "base_sha", + "claimed", + "cleanup_succeeded", + "group_files", + "issue_number", + "issue_slug", + "issue_title", + "issue_url", + "merge_target", + "open_pr", + "plan_path", + "proposed_branch", + "remote_url", + "review_approach", + "review_path", + "run_name", + "source_dir", + "source_doc", + "upfront_claimed", + "work_dir", + "worktree_path" + ], + "required": [], + "produced": [ + "remediation_path" + ] + }, + { + "step": "remediate", + "available": [ + "all_plan_paths", + "audit", + "auto_merge", + "base_branch", + "base_sha", + "claimed", + "cleanup_succeeded", + "group_files", + "issue_number", + "issue_slug", + "issue_title", + "issue_url", + "merge_target", + "open_pr", + "plan_path", + "proposed_branch", + "remediation_path", + "remote_url", + "review_approach", + "review_path", + "run_name", + "source_dir", + "source_doc", + "upfront_claimed", + "work_dir", + "worktree_path" + ], + "required": [], + "produced": [] + }, + { + "step": "compose_pr", + "available": [ + "all_plan_paths", + "audit", + "auto_merge", + "base_branch", + "base_sha", + "claimed", + "cleanup_succeeded", + "group_files", + "issue_number", + "issue_slug", + "issue_title", + "issue_url", + "merge_target", + "open_pr", + "plan_path", + "proposed_branch", + "remediation_path", + "remote_url", + "review_approach", + "review_path", + "run_name", + "source_dir", + "source_doc", + "upfront_claimed", + "work_dir", + "worktree_path" + ], + "required": [ + "plan_paths", + "feature_branch" + ], + "produced": [] + }, + { + "step": "extract_pr_number", + "available": [ + "all_plan_paths", + "audit", + "auto_merge", + "base_branch", + "base_sha", + "claimed", + "cleanup_succeeded", + "group_files", + "issue_number", + "issue_slug", + "issue_title", + "issue_url", + "merge_target", + "open_pr", + "plan_path", + "proposed_branch", + "remediation_path", + "remote_url", + "review_approach", + "review_path", + "run_name", + "source_dir", + "source_doc", + "upfront_claimed", + "work_dir", + "worktree_path" + ], + "required": [], + "produced": [ + "pr_number" + ] + }, + { + "step": "review_pr", + "available": [ + "all_plan_paths", + "audit", + "auto_merge", + "base_branch", + "base_sha", + "claimed", + "cleanup_succeeded", + "group_files", + "issue_number", + "issue_slug", + "issue_title", + "issue_url", + "merge_target", + "open_pr", + "plan_path", + "pr_number", + "proposed_branch", + "remediation_path", + "remote_url", + "review_approach", + "review_path", + "run_name", + "source_dir", + "source_doc", + "upfront_claimed", + "work_dir", + "worktree_path" + ], + "required": [ + "feature_branch" + ], + "produced": [ + "verdict" + ] + }, + { + "step": "resolve_review", + "available": [ + "all_plan_paths", + "audit", + "auto_merge", + "base_branch", + "base_sha", + "claimed", + "cleanup_succeeded", + "group_files", + "issue_number", + "issue_slug", + "issue_title", + "issue_url", + "merge_target", + "open_pr", + "plan_path", + "pr_number", + "proposed_branch", + "remediation_path", + "remote_url", + "review_approach", + "review_path", + "run_name", + "source_dir", + "source_doc", + "upfront_claimed", + "verdict", + "work_dir", + "worktree_path" + ], + "required": [ + "feature_branch" + ], + "produced": [] + }, + { + "step": "re_push_review", + "available": [ + "all_plan_paths", + "audit", + "auto_merge", + "base_branch", + "base_sha", + "claimed", + "cleanup_succeeded", + "group_files", + "issue_number", + "issue_slug", + "issue_title", + "issue_url", + "merge_target", + "open_pr", + "plan_path", + "pr_number", + "proposed_branch", + "remediation_path", + "remote_url", + "review_approach", + "review_path", + "run_name", + "source_dir", + "source_doc", + "upfront_claimed", + "verdict", + "work_dir", + "worktree_path" + ], + "required": [], + "produced": [] + }, + { + "step": "ci_watch", + "available": [ + "all_plan_paths", + "audit", + "auto_merge", + "base_branch", + "base_sha", + "claimed", + "cleanup_succeeded", + "group_files", + "issue_number", + "issue_slug", + "issue_title", + "issue_url", + "merge_target", + "open_pr", + "plan_path", + "pr_number", + "proposed_branch", + "remediation_path", + "remote_url", + "review_approach", + "review_path", + "run_name", + "source_dir", + "source_doc", + "upfront_claimed", + "verdict", + "work_dir", + "worktree_path" + ], + "required": [], + "produced": [ + "ci_conclusion", + "ci_failed_jobs" + ] + }, + { + "step": "diagnose_ci", + "available": [ + "all_plan_paths", + "audit", + "auto_merge", + "base_branch", + "base_sha", + "ci_conclusion", + "ci_failed_jobs", + "claimed", + "cleanup_succeeded", + "group_files", + "issue_number", + "issue_slug", + "issue_title", + "issue_url", + "merge_target", + "open_pr", + "plan_path", + "pr_number", + "proposed_branch", + "remediation_path", + "remote_url", + "review_approach", + "review_path", + "run_name", + "source_dir", + "source_doc", + "upfront_claimed", + "verdict", + "work_dir", + "worktree_path" + ], + "required": [], + "produced": [ + "diagnosis_path" + ] + }, + { + "step": "resolve_ci", + "available": [ + "all_plan_paths", + "audit", + "auto_merge", + "base_branch", + "base_sha", + "ci_conclusion", + "ci_failed_jobs", + "claimed", + "cleanup_succeeded", + "diagnosis_path", + "group_files", + "issue_number", + "issue_slug", + "issue_title", + "issue_url", + "merge_target", + "open_pr", + "plan_path", + "pr_number", + "proposed_branch", + "remediation_path", + "remote_url", + "review_approach", + "review_path", + "run_name", + "source_dir", + "source_doc", + "upfront_claimed", + "verdict", + "work_dir", + "worktree_path" + ], + "required": [ + "worktree_path" + ], + "produced": [] + }, + { + "step": "re_push", + "available": [ + "all_plan_paths", + "audit", + "auto_merge", + "base_branch", + "base_sha", + "ci_conclusion", + "ci_failed_jobs", + "claimed", + "cleanup_succeeded", + "diagnosis_path", + "group_files", + "issue_number", + "issue_slug", + "issue_title", + "issue_url", + "merge_target", + "open_pr", + "plan_path", + "pr_number", + "proposed_branch", + "remediation_path", + "remote_url", + "review_approach", + "review_path", + "run_name", + "source_dir", + "source_doc", + "upfront_claimed", + "verdict", + "work_dir", + "worktree_path" + ], + "required": [], + "produced": [] + }, + { + "step": "check_merge_queue", + "available": [ + "all_plan_paths", + "audit", + "auto_merge", + "base_branch", + "base_sha", + "ci_conclusion", + "ci_failed_jobs", + "claimed", + "cleanup_succeeded", + "diagnosis_path", + "group_files", + "issue_number", + "issue_slug", + "issue_title", + "issue_url", + "merge_target", + "open_pr", + "plan_path", + "pr_number", + "proposed_branch", + "remediation_path", + "remote_url", + "review_approach", + "review_path", + "run_name", + "source_dir", + "source_doc", + "upfront_claimed", + "verdict", + "work_dir", + "worktree_path" + ], + "required": [], + "produced": [ + "queue_available" + ] + }, + { + "step": "check_merge_group_trigger", + "available": [ + "all_plan_paths", + "audit", + "auto_merge", + "base_branch", + "base_sha", + "ci_conclusion", + "ci_failed_jobs", + "claimed", + "cleanup_succeeded", + "diagnosis_path", + "group_files", + "issue_number", + "issue_slug", + "issue_title", + "issue_url", + "merge_target", + "open_pr", + "plan_path", + "pr_number", + "proposed_branch", + "queue_available", + "remediation_path", + "remote_url", + "review_approach", + "review_path", + "run_name", + "source_dir", + "source_doc", + "upfront_claimed", + "verdict", + "work_dir", + "worktree_path" + ], + "required": [], + "produced": [ + "merge_group_trigger" + ] + }, + { + "step": "check_auto_merge", + "available": [ + "all_plan_paths", + "audit", + "auto_merge", + "base_branch", + "base_sha", + "ci_conclusion", + "ci_failed_jobs", + "claimed", + "cleanup_succeeded", + "diagnosis_path", + "group_files", + "issue_number", + "issue_slug", + "issue_title", + "issue_url", + "merge_group_trigger", + "merge_target", + "open_pr", + "plan_path", + "pr_number", + "proposed_branch", + "queue_available", + "remediation_path", + "remote_url", + "review_approach", + "review_path", + "run_name", + "source_dir", + "source_doc", + "upfront_claimed", + "verdict", + "work_dir", + "worktree_path" + ], + "required": [], + "produced": [ + "auto_merge_available" + ] + }, + { + "step": "route_queue_mode", + "available": [ + "all_plan_paths", + "audit", + "auto_merge", + "auto_merge_available", + "base_branch", + "base_sha", + "ci_conclusion", + "ci_failed_jobs", + "claimed", + "cleanup_succeeded", + "diagnosis_path", + "group_files", + "issue_number", + "issue_slug", + "issue_title", + "issue_url", + "merge_group_trigger", + "merge_target", + "open_pr", + "plan_path", + "pr_number", + "proposed_branch", + "queue_available", + "remediation_path", + "remote_url", + "review_approach", + "review_path", + "run_name", + "source_dir", + "source_doc", + "upfront_claimed", + "verdict", + "work_dir", + "worktree_path" + ], + "required": [], + "produced": [] + }, + { + "step": "enable_auto_merge", + "available": [ + "all_plan_paths", + "audit", + "auto_merge", + "auto_merge_available", + "base_branch", + "base_sha", + "ci_conclusion", + "ci_failed_jobs", + "claimed", + "cleanup_succeeded", + "diagnosis_path", + "group_files", + "issue_number", + "issue_slug", + "issue_title", + "issue_url", + "merge_group_trigger", + "merge_target", + "open_pr", + "plan_path", + "pr_number", + "proposed_branch", + "queue_available", + "remediation_path", + "remote_url", + "review_approach", + "review_path", + "run_name", + "source_dir", + "source_doc", + "upfront_claimed", + "verdict", + "work_dir", + "worktree_path" + ], + "required": [], + "produced": [] + }, + { + "step": "wait_for_queue", + "available": [ + "all_plan_paths", + "audit", + "auto_merge", + "auto_merge_available", + "base_branch", + "base_sha", + "ci_conclusion", + "ci_failed_jobs", + "claimed", + "cleanup_succeeded", + "diagnosis_path", + "group_files", + "issue_number", + "issue_slug", + "issue_title", + "issue_url", + "merge_group_trigger", + "merge_target", + "open_pr", + "plan_path", + "pr_number", + "proposed_branch", + "queue_available", + "remediation_path", + "remote_url", + "review_approach", + "review_path", + "run_name", + "source_dir", + "source_doc", + "upfront_claimed", + "verdict", + "work_dir", + "worktree_path" + ], + "required": [], + "produced": [ + "queue_pr_state" + ] + }, + { + "step": "reenroll_stalled_pr", + "available": [ + "all_plan_paths", + "audit", + "auto_merge", + "auto_merge_available", + "base_branch", + "base_sha", + "ci_conclusion", + "ci_failed_jobs", + "claimed", + "cleanup_succeeded", + "diagnosis_path", + "group_files", + "issue_number", + "issue_slug", + "issue_title", + "issue_url", + "merge_group_trigger", + "merge_target", + "open_pr", + "plan_path", + "pr_number", + "proposed_branch", + "queue_available", + "queue_pr_state", + "remediation_path", + "remote_url", + "review_approach", + "review_path", + "run_name", + "source_dir", + "source_doc", + "upfront_claimed", + "verdict", + "work_dir", + "worktree_path" + ], + "required": [], + "produced": [] + }, + { + "step": "queue_ejected_fix", + "available": [ + "all_plan_paths", + "audit", + "auto_merge", + "auto_merge_available", + "base_branch", + "base_sha", + "ci_conclusion", + "ci_failed_jobs", + "claimed", + "cleanup_succeeded", + "diagnosis_path", + "group_files", + "issue_number", + "issue_slug", + "issue_title", + "issue_url", + "merge_group_trigger", + "merge_target", + "open_pr", + "plan_path", + "pr_number", + "proposed_branch", + "queue_available", + "queue_pr_state", + "remediation_path", + "remote_url", + "review_approach", + "review_path", + "run_name", + "source_dir", + "source_doc", + "upfront_claimed", + "verdict", + "work_dir", + "worktree_path" + ], + "required": [ + "worktree_path" + ], + "produced": [ + "conflict_escalation_required" + ] + }, + { + "step": "re_push_queue_fix", + "available": [ + "all_plan_paths", + "audit", + "auto_merge", + "auto_merge_available", + "base_branch", + "base_sha", + "ci_conclusion", + "ci_failed_jobs", + "claimed", + "cleanup_succeeded", + "conflict_escalation_required", + "diagnosis_path", + "group_files", + "issue_number", + "issue_slug", + "issue_title", + "issue_url", + "merge_group_trigger", + "merge_target", + "open_pr", + "plan_path", + "pr_number", + "proposed_branch", + "queue_available", + "queue_pr_state", + "remediation_path", + "remote_url", + "review_approach", + "review_path", + "run_name", + "source_dir", + "source_doc", + "upfront_claimed", + "verdict", + "work_dir", + "worktree_path" + ], + "required": [], + "produced": [] + }, + { + "step": "reenter_merge_queue", + "available": [ + "all_plan_paths", + "audit", + "auto_merge", + "auto_merge_available", + "base_branch", + "base_sha", + "ci_conclusion", + "ci_failed_jobs", + "claimed", + "cleanup_succeeded", + "conflict_escalation_required", + "diagnosis_path", + "group_files", + "issue_number", + "issue_slug", + "issue_title", + "issue_url", + "merge_group_trigger", + "merge_target", + "open_pr", + "plan_path", + "pr_number", + "proposed_branch", + "queue_available", + "queue_pr_state", + "remediation_path", + "remote_url", + "review_approach", + "review_path", + "run_name", + "source_dir", + "source_doc", + "upfront_claimed", + "verdict", + "work_dir", + "worktree_path" + ], + "required": [], + "produced": [] + }, + { + "step": "direct_merge", + "available": [ + "all_plan_paths", + "audit", + "auto_merge", + "auto_merge_available", + "base_branch", + "base_sha", + "ci_conclusion", + "ci_failed_jobs", + "claimed", + "cleanup_succeeded", + "conflict_escalation_required", + "diagnosis_path", + "group_files", + "issue_number", + "issue_slug", + "issue_title", + "issue_url", + "merge_group_trigger", + "merge_target", + "open_pr", + "plan_path", + "pr_number", + "proposed_branch", + "queue_available", + "queue_pr_state", + "remediation_path", + "remote_url", + "review_approach", + "review_path", + "run_name", + "source_dir", + "source_doc", + "upfront_claimed", + "verdict", + "work_dir", + "worktree_path" + ], + "required": [], + "produced": [] + }, + { + "step": "wait_for_direct_merge", + "available": [ + "all_plan_paths", + "audit", + "auto_merge", + "auto_merge_available", + "base_branch", + "base_sha", + "ci_conclusion", + "ci_failed_jobs", + "claimed", + "cleanup_succeeded", + "conflict_escalation_required", + "diagnosis_path", + "group_files", + "issue_number", + "issue_slug", + "issue_title", + "issue_url", + "merge_group_trigger", + "merge_target", + "open_pr", + "plan_path", + "pr_number", + "proposed_branch", + "queue_available", + "queue_pr_state", + "remediation_path", + "remote_url", + "review_approach", + "review_path", + "run_name", + "source_dir", + "source_doc", + "upfront_claimed", + "verdict", + "work_dir", + "worktree_path" + ], + "required": [], + "produced": [] + }, + { + "step": "direct_merge_conflict_fix", + "available": [ + "all_plan_paths", + "audit", + "auto_merge", + "auto_merge_available", + "base_branch", + "base_sha", + "ci_conclusion", + "ci_failed_jobs", + "claimed", + "cleanup_succeeded", + "conflict_escalation_required", + "diagnosis_path", + "group_files", + "issue_number", + "issue_slug", + "issue_title", + "issue_url", + "merge_group_trigger", + "merge_target", + "open_pr", + "plan_path", + "pr_number", + "proposed_branch", + "queue_available", + "queue_pr_state", + "remediation_path", + "remote_url", + "review_approach", + "review_path", + "run_name", + "source_dir", + "source_doc", + "upfront_claimed", + "verdict", + "work_dir", + "worktree_path" + ], + "required": [ + "worktree_path" + ], + "produced": [ + "conflict_escalation_required" + ] + }, + { + "step": "re_push_direct_fix", + "available": [ + "all_plan_paths", + "audit", + "auto_merge", + "auto_merge_available", + "base_branch", + "base_sha", + "ci_conclusion", + "ci_failed_jobs", + "claimed", + "cleanup_succeeded", + "conflict_escalation_required", + "diagnosis_path", + "group_files", + "issue_number", + "issue_slug", + "issue_title", + "issue_url", + "merge_group_trigger", + "merge_target", + "open_pr", + "plan_path", + "pr_number", + "proposed_branch", + "queue_available", + "queue_pr_state", + "remediation_path", + "remote_url", + "review_approach", + "review_path", + "run_name", + "source_dir", + "source_doc", + "upfront_claimed", + "verdict", + "work_dir", + "worktree_path" + ], + "required": [], + "produced": [] + }, + { + "step": "redirect_merge", + "available": [ + "all_plan_paths", + "audit", + "auto_merge", + "auto_merge_available", + "base_branch", + "base_sha", + "ci_conclusion", + "ci_failed_jobs", + "claimed", + "cleanup_succeeded", + "conflict_escalation_required", + "diagnosis_path", + "group_files", + "issue_number", + "issue_slug", + "issue_title", + "issue_url", + "merge_group_trigger", + "merge_target", + "open_pr", + "plan_path", + "pr_number", + "proposed_branch", + "queue_available", + "queue_pr_state", + "remediation_path", + "remote_url", + "review_approach", + "review_path", + "run_name", + "source_dir", + "source_doc", + "upfront_claimed", + "verdict", + "work_dir", + "worktree_path" + ], + "required": [], + "produced": [] + }, + { + "step": "immediate_merge", + "available": [ + "all_plan_paths", + "audit", + "auto_merge", + "auto_merge_available", + "base_branch", + "base_sha", + "ci_conclusion", + "ci_failed_jobs", + "claimed", + "cleanup_succeeded", + "conflict_escalation_required", + "diagnosis_path", + "group_files", + "issue_number", + "issue_slug", + "issue_title", + "issue_url", + "merge_group_trigger", + "merge_target", + "open_pr", + "plan_path", + "pr_number", + "proposed_branch", + "queue_available", + "queue_pr_state", + "remediation_path", + "remote_url", + "review_approach", + "review_path", + "run_name", + "source_dir", + "source_doc", + "upfront_claimed", + "verdict", + "work_dir", + "worktree_path" + ], + "required": [], + "produced": [] + }, + { + "step": "wait_for_immediate_merge", + "available": [ + "all_plan_paths", + "audit", + "auto_merge", + "auto_merge_available", + "base_branch", + "base_sha", + "ci_conclusion", + "ci_failed_jobs", + "claimed", + "cleanup_succeeded", + "conflict_escalation_required", + "diagnosis_path", + "group_files", + "issue_number", + "issue_slug", + "issue_title", + "issue_url", + "merge_group_trigger", + "merge_target", + "open_pr", + "plan_path", + "pr_number", + "proposed_branch", + "queue_available", + "queue_pr_state", + "remediation_path", + "remote_url", + "review_approach", + "review_path", + "run_name", + "source_dir", + "source_doc", + "upfront_claimed", + "verdict", + "work_dir", + "worktree_path" + ], + "required": [], + "produced": [] + }, + { + "step": "immediate_merge_conflict_fix", + "available": [ + "all_plan_paths", + "audit", + "auto_merge", + "auto_merge_available", + "base_branch", + "base_sha", + "ci_conclusion", + "ci_failed_jobs", + "claimed", + "cleanup_succeeded", + "conflict_escalation_required", + "diagnosis_path", + "group_files", + "issue_number", + "issue_slug", + "issue_title", + "issue_url", + "merge_group_trigger", + "merge_target", + "open_pr", + "plan_path", + "pr_number", + "proposed_branch", + "queue_available", + "queue_pr_state", + "remediation_path", + "remote_url", + "review_approach", + "review_path", + "run_name", + "source_dir", + "source_doc", + "upfront_claimed", + "verdict", + "work_dir", + "worktree_path" + ], + "required": [ + "worktree_path" + ], + "produced": [ + "conflict_escalation_required" + ] + }, + { + "step": "re_push_immediate_fix", + "available": [ + "all_plan_paths", + "audit", + "auto_merge", + "auto_merge_available", + "base_branch", + "base_sha", + "ci_conclusion", + "ci_failed_jobs", + "claimed", + "cleanup_succeeded", + "conflict_escalation_required", + "diagnosis_path", + "group_files", + "issue_number", + "issue_slug", + "issue_title", + "issue_url", + "merge_group_trigger", + "merge_target", + "open_pr", + "plan_path", + "pr_number", + "proposed_branch", + "queue_available", + "queue_pr_state", + "remediation_path", + "remote_url", + "review_approach", + "review_path", + "run_name", + "source_dir", + "source_doc", + "upfront_claimed", + "verdict", + "work_dir", + "worktree_path" + ], + "required": [], + "produced": [] + }, + { + "step": "remerge_immediate", + "available": [ + "all_plan_paths", + "audit", + "auto_merge", + "auto_merge_available", + "base_branch", + "base_sha", + "ci_conclusion", + "ci_failed_jobs", + "claimed", + "cleanup_succeeded", + "conflict_escalation_required", + "diagnosis_path", + "group_files", + "issue_number", + "issue_slug", + "issue_title", + "issue_url", + "merge_group_trigger", + "merge_target", + "open_pr", + "plan_path", + "pr_number", + "proposed_branch", + "queue_available", + "queue_pr_state", + "remediation_path", + "remote_url", + "review_approach", + "review_path", + "run_name", + "source_dir", + "source_doc", + "upfront_claimed", + "verdict", + "work_dir", + "worktree_path" + ], + "required": [], + "produced": [] + }, + { + "step": "register_clone_unconfirmed", + "available": [ + "all_plan_paths", + "audit", + "auto_merge", + "auto_merge_available", + "base_branch", + "base_sha", + "ci_conclusion", + "ci_failed_jobs", + "claimed", + "cleanup_succeeded", + "conflict_escalation_required", + "diagnosis_path", + "group_files", + "issue_number", + "issue_slug", + "issue_title", + "issue_url", + "merge_group_trigger", + "merge_target", + "open_pr", + "plan_path", + "pr_number", + "proposed_branch", + "queue_available", + "queue_pr_state", + "remediation_path", + "remote_url", + "review_approach", + "review_path", + "run_name", + "source_dir", + "source_doc", + "upfront_claimed", + "verdict", + "work_dir", + "worktree_path" + ], + "required": [], + "produced": [] + }, + { + "step": "detect_ci_conflict", + "available": [ + "all_plan_paths", + "audit", + "auto_merge", + "auto_merge_available", + "base_branch", + "base_sha", + "ci_conclusion", + "ci_failed_jobs", + "claimed", + "cleanup_succeeded", + "conflict_escalation_required", + "diagnosis_path", + "group_files", + "issue_number", + "issue_slug", + "issue_title", + "issue_url", + "merge_group_trigger", + "merge_target", + "open_pr", + "plan_path", + "pr_number", + "proposed_branch", + "queue_available", + "queue_pr_state", + "remediation_path", + "remote_url", + "review_approach", + "review_path", + "run_name", + "source_dir", + "source_doc", + "upfront_claimed", + "verdict", + "work_dir", + "worktree_path" + ], + "required": [], + "produced": [] + }, + { + "step": "ci_conflict_fix", + "available": [ + "all_plan_paths", + "audit", + "auto_merge", + "auto_merge_available", + "base_branch", + "base_sha", + "ci_conclusion", + "ci_failed_jobs", + "claimed", + "cleanup_succeeded", + "conflict_escalation_required", + "diagnosis_path", + "group_files", + "issue_number", + "issue_slug", + "issue_title", + "issue_url", + "merge_group_trigger", + "merge_target", + "open_pr", + "plan_path", + "pr_number", + "proposed_branch", + "queue_available", + "queue_pr_state", + "remediation_path", + "remote_url", + "review_approach", + "review_path", + "run_name", + "source_dir", + "source_doc", + "upfront_claimed", + "verdict", + "work_dir", + "worktree_path" + ], + "required": [ + "worktree_path" + ], + "produced": [ + "conflict_escalation_required" + ] + }, + { + "step": "release_issue_success", + "available": [ + "all_plan_paths", + "audit", + "auto_merge", + "auto_merge_available", + "base_branch", + "base_sha", + "ci_conclusion", + "ci_failed_jobs", + "claimed", + "cleanup_succeeded", + "conflict_escalation_required", + "diagnosis_path", + "group_files", + "issue_number", + "issue_slug", + "issue_title", + "issue_url", + "merge_group_trigger", + "merge_target", + "open_pr", + "plan_path", + "pr_number", + "proposed_branch", + "queue_available", + "queue_pr_state", + "remediation_path", + "remote_url", + "review_approach", + "review_path", + "run_name", + "source_dir", + "source_doc", + "upfront_claimed", + "verdict", + "work_dir", + "worktree_path" + ], + "required": [], + "produced": [] + }, + { + "step": "release_issue_failure", + "available": [ + "all_plan_paths", + "audit", + "auto_merge", + "auto_merge_available", + "base_branch", + "base_sha", + "ci_conclusion", + "ci_failed_jobs", + "claimed", + "cleanup_succeeded", + "conflict_escalation_required", + "diagnosis_path", + "group_files", + "issue_number", + "issue_slug", + "issue_title", + "issue_url", + "merge_group_trigger", + "merge_target", + "open_pr", + "plan_path", + "pr_number", + "proposed_branch", + "queue_available", + "queue_pr_state", + "remediation_path", + "remote_url", + "review_approach", + "review_path", + "run_name", + "source_dir", + "source_doc", + "upfront_claimed", + "verdict", + "work_dir", + "worktree_path" + ], + "required": [], + "produced": [] + }, + { + "step": "register_clone_success", + "available": [ + "all_plan_paths", + "audit", + "auto_merge", + "auto_merge_available", + "base_branch", + "base_sha", + "ci_conclusion", + "ci_failed_jobs", + "claimed", + "cleanup_succeeded", + "conflict_escalation_required", + "diagnosis_path", + "group_files", + "issue_number", + "issue_slug", + "issue_title", + "issue_url", + "merge_group_trigger", + "merge_target", + "open_pr", + "plan_path", + "pr_number", + "proposed_branch", + "queue_available", + "queue_pr_state", + "remediation_path", + "remote_url", + "review_approach", + "review_path", + "run_name", + "source_dir", + "source_doc", + "upfront_claimed", + "verdict", + "work_dir", + "worktree_path" + ], + "required": [], + "produced": [] + }, + { + "step": "register_clone_failure", + "available": [ + "all_plan_paths", + "audit", + "auto_merge", + "auto_merge_available", + "base_branch", + "base_sha", + "ci_conclusion", + "ci_failed_jobs", + "claimed", + "cleanup_succeeded", + "conflict_escalation_required", + "diagnosis_path", + "group_files", + "issue_number", + "issue_slug", + "issue_title", + "issue_url", + "merge_group_trigger", + "merge_target", + "open_pr", + "plan_path", + "pr_number", + "proposed_branch", + "queue_available", + "queue_pr_state", + "remediation_path", + "remote_url", + "review_approach", + "review_path", + "run_name", + "source_dir", + "source_doc", + "upfront_claimed", + "verdict", + "work_dir", + "worktree_path" + ], + "required": [], + "produced": [] + }, + { + "step": "done", + "available": [ + "all_plan_paths", + "audit", + "auto_merge", + "auto_merge_available", + "base_branch", + "base_sha", + "ci_conclusion", + "ci_failed_jobs", + "claimed", + "cleanup_succeeded", + "conflict_escalation_required", + "diagnosis_path", + "group_files", + "issue_number", + "issue_slug", + "issue_title", + "issue_url", + "merge_group_trigger", + "merge_target", + "open_pr", + "plan_path", + "pr_number", + "proposed_branch", + "queue_available", + "queue_pr_state", + "remediation_path", + "remote_url", + "review_approach", + "review_path", + "run_name", + "source_dir", + "source_doc", + "upfront_claimed", + "verdict", + "work_dir", + "worktree_path" + ], + "required": [], + "produced": [] + }, + { + "step": "escalate_stop", + "available": [ + "all_plan_paths", + "audit", + "auto_merge", + "auto_merge_available", + "base_branch", + "base_sha", + "ci_conclusion", + "ci_failed_jobs", + "claimed", + "cleanup_succeeded", + "conflict_escalation_required", + "diagnosis_path", + "group_files", + "issue_number", + "issue_slug", + "issue_title", + "issue_url", + "merge_group_trigger", + "merge_target", + "open_pr", + "plan_path", + "pr_number", + "proposed_branch", + "queue_available", + "queue_pr_state", + "remediation_path", + "remote_url", + "review_approach", + "review_path", + "run_name", + "source_dir", + "source_doc", + "upfront_claimed", + "verdict", + "work_dir", + "worktree_path" + ], + "required": [], + "produced": [] + } + ] +} diff --git a/src/autoskillit/recipes/contracts/implementation-groups.yaml b/src/autoskillit/recipes/contracts/implementation-groups.yaml index 591e8b55f1..ed73c64f57 100644 --- a/src/autoskillit/recipes/contracts/implementation-groups.yaml +++ b/src/autoskillit/recipes/contracts/implementation-groups.yaml @@ -162,9 +162,10 @@ skills: - name: verdict type: string expected_output_patterns: - - verdict\s*=\s*(approved|changes_requested|needs_human) + - verdict\s*=\s*(approved|approved_with_comments|changes_requested|needs_human) pattern_examples: - 'verdict = approved' + - 'verdict = approved_with_comments' - 'verdict = changes_requested' - 'verdict = needs_human' resolve-review: @@ -1853,7 +1854,7 @@ dataflow: - worktree_path required: [] produced: [] -- step: release_issue_timeout +- step: register_clone_unconfirmed available: - all_plan_paths - audit diff --git a/src/autoskillit/recipes/contracts/implementation.json b/src/autoskillit/recipes/contracts/implementation.json new file mode 100644 index 0000000000..85805a75b8 --- /dev/null +++ b/src/autoskillit/recipes/contracts/implementation.json @@ -0,0 +1,2591 @@ +{ + "generated_at": "2026-04-05T02:19:55.954339+00:00", + "bundled_manifest_version": "0.1.0", + "skill_hashes": {}, + "skills": { + "make-plan": { + "inputs": [ + { + "name": "task", + "type": "string", + "required": true + } + ], + "outputs": [ + { + "name": "plan_path", + "type": "file_path" + }, + { + "name": "plan_parts", + "type": "file_path_list" + } + ], + "expected_output_patterns": [ + "plan_path\\s*=\\s*/.+" + ], + "pattern_examples": [ + "plan_path = /tmp/plan.md" + ], + "write_behavior": "always" + }, + "review-approach": { + "inputs": [ + { + "name": "plan_path", + "type": "file_path", + "required": false + } + ], + "outputs": [ + { + "name": "review_path", + "type": "file_path" + } + ], + "expected_output_patterns": [ + "review_path\\s*=\\s*/.+" + ], + "pattern_examples": [ + "review_path = /tmp/review.md" + ] + }, + "dry-walkthrough": { + "inputs": [ + { + "name": "plan_path", + "type": "file_path", + "required": true + } + ], + "outputs": [], + "expected_output_patterns": [], + "pattern_examples": [], + "write_behavior": "always" + }, + "implement-worktree-no-merge": { + "inputs": [ + { + "name": "plan_path", + "type": "file_path", + "required": true + } + ], + "outputs": [ + { + "name": "worktree_path", + "type": "directory_path" + }, + { + "name": "branch_name", + "type": "string" + } + ], + "expected_output_patterns": [ + "worktree_path\\s*=\\s*/.+" + ], + "pattern_examples": [ + "worktree_path = /tmp/worktrees/impl-foo-20260316" + ], + "write_behavior": "always" + }, + "retry-worktree": { + "inputs": [ + { + "name": "plan_path", + "type": "file_path", + "required": true + }, + { + "name": "worktree_path", + "type": "directory_path", + "required": true + } + ], + "outputs": [ + { + "name": "worktree_path", + "type": "directory_path" + }, + { + "name": "branch_name", + "type": "string" + } + ], + "expected_output_patterns": [ + "worktree_path\\s*=\\s*/.+" + ], + "pattern_examples": [ + "worktree_path = /tmp/worktrees/impl-foo-20260316" + ], + "write_behavior": "always" + }, + "resolve-failures": { + "inputs": [ + { + "name": "worktree_path", + "type": "directory_path", + "required": true + }, + { + "name": "plan_path", + "type": "file_path", + "required": true + }, + { + "name": "base_branch", + "type": "string", + "required": true + } + ], + "outputs": [], + "expected_output_patterns": [], + "pattern_examples": [], + "write_behavior": "always" + }, + "audit-impl": { + "inputs": [ + { + "name": "plan_path", + "type": "file_path", + "required": true + }, + { + "name": "branch_name", + "type": "string", + "required": false + }, + { + "name": "base_branch", + "type": "string", + "required": false + } + ], + "outputs": [ + { + "name": "verdict", + "type": "string" + }, + { + "name": "remediation_path", + "type": "file_path" + } + ], + "expected_output_patterns": [ + "verdict\\s*=\\s*(GO|NO GO)" + ], + "pattern_examples": [ + "verdict = GO", + "verdict = NO GO\nremediation_path = /tmp/remediation.md" + ] + }, + "open-pr": { + "inputs": [ + { + "name": "plan_paths", + "type": "string", + "required": true + }, + { + "name": "feature_branch", + "type": "string", + "required": true + }, + { + "name": "base_branch", + "type": "string", + "required": true + }, + { + "name": "closing_issue", + "type": "string", + "required": false + } + ], + "outputs": [], + "expected_output_patterns": [ + "https://github\\.com/.*/pull/\\d+" + ], + "pattern_examples": [ + "https://github.com/owner/repo/pull/42" + ] + }, + "review-pr": { + "inputs": [ + { + "name": "feature_branch", + "type": "string", + "required": true + }, + { + "name": "base_branch", + "type": "string", + "required": true + }, + { + "name": "annotated_diff_path", + "type": "string", + "required": false + }, + { + "name": "hunk_ranges_path", + "type": "string", + "required": false + } + ], + "outputs": [ + { + "name": "verdict", + "type": "string" + } + ], + "expected_output_patterns": [ + "verdict\\s*=\\s*(approved|approved_with_comments|changes_requested|needs_human)" + ], + "pattern_examples": [ + "verdict = approved", + "verdict = approved_with_comments", + "verdict = changes_requested", + "verdict = needs_human" + ] + }, + "resolve-review": { + "inputs": [ + { + "name": "feature_branch", + "type": "string", + "required": true + }, + { + "name": "base_branch", + "type": "string", + "required": true + } + ], + "outputs": [], + "expected_output_patterns": [], + "pattern_examples": [], + "write_behavior": "always" + }, + "resolve-merge-conflicts": { + "inputs": [ + { + "name": "worktree_path", + "type": "directory_path", + "required": true + }, + { + "name": "plan_path", + "type": "file_path", + "required": true + }, + { + "name": "base_branch", + "type": "string", + "required": true + } + ], + "outputs": [ + { + "name": "worktree_path", + "type": "directory_path" + }, + { + "name": "branch_name", + "type": "string" + }, + { + "name": "conflict_report_path", + "type": "file_path" + }, + { + "name": "escalation_required", + "type": "string" + }, + { + "name": "escalation_reason", + "type": "string" + } + ], + "expected_output_patterns": [ + "worktree_path\\s*=\\s*/.+" + ], + "pattern_examples": [ + "worktree_path = /tmp/worktrees/impl-foo-20260316" + ], + "write_behavior": "conditional", + "write_expected_when": [ + "conflict_report_path\\s*=\\s*/.+" + ] + }, + "diagnose-ci": { + "inputs": [ + { + "name": "branch", + "type": "string", + "required": true + }, + { + "name": "run_id", + "type": "string", + "required": false + }, + { + "name": "ci_failed_jobs", + "type": "string", + "required": false + } + ], + "outputs": [ + { + "name": "diagnosis_path", + "type": "file_path" + }, + { + "name": "failure_type", + "type": "string" + }, + { + "name": "is_fixable", + "type": "string" + } + ], + "expected_output_patterns": [ + "diagnosis_path\\s*=\\s*/.+" + ], + "pattern_examples": [ + "diagnosis_path = /tmp/diagnosis.md" + ], + "write_behavior": "always" + } + }, + "dataflow": [ + { + "step": "clone", + "available": [ + "audit", + "auto_merge", + "base_branch", + "issue_url", + "max_parallel", + "open_pr", + "review_approach", + "run_mode", + "run_name", + "source_dir", + "task", + "upfront_claimed" + ], + "required": [], + "produced": [ + "work_dir", + "remote_url" + ] + }, + { + "step": "capture_base_sha", + "available": [ + "audit", + "auto_merge", + "base_branch", + "issue_url", + "max_parallel", + "open_pr", + "remote_url", + "review_approach", + "run_mode", + "run_name", + "source_dir", + "task", + "upfront_claimed", + "work_dir" + ], + "required": [], + "produced": [ + "base_sha" + ] + }, + { + "step": "set_merge_target", + "available": [ + "audit", + "auto_merge", + "base_branch", + "base_sha", + "issue_url", + "max_parallel", + "open_pr", + "remote_url", + "review_approach", + "run_mode", + "run_name", + "source_dir", + "task", + "upfront_claimed", + "work_dir" + ], + "required": [], + "produced": [ + "merge_target" + ] + }, + { + "step": "get_issue_title", + "available": [ + "audit", + "auto_merge", + "base_branch", + "base_sha", + "issue_url", + "max_parallel", + "merge_target", + "open_pr", + "remote_url", + "review_approach", + "run_mode", + "run_name", + "source_dir", + "task", + "upfront_claimed", + "work_dir" + ], + "required": [], + "produced": [ + "issue_number", + "issue_title", + "issue_slug" + ] + }, + { + "step": "claim_issue", + "available": [ + "audit", + "auto_merge", + "base_branch", + "base_sha", + "issue_number", + "issue_slug", + "issue_title", + "issue_url", + "max_parallel", + "merge_target", + "open_pr", + "remote_url", + "review_approach", + "run_mode", + "run_name", + "source_dir", + "task", + "upfront_claimed", + "work_dir" + ], + "required": [], + "produced": [ + "claimed" + ] + }, + { + "step": "compute_branch", + "available": [ + "audit", + "auto_merge", + "base_branch", + "base_sha", + "claimed", + "issue_number", + "issue_slug", + "issue_title", + "issue_url", + "max_parallel", + "merge_target", + "open_pr", + "remote_url", + "review_approach", + "run_mode", + "run_name", + "source_dir", + "task", + "upfront_claimed", + "work_dir" + ], + "required": [], + "produced": [ + "proposed_branch" + ] + }, + { + "step": "create_branch", + "available": [ + "audit", + "auto_merge", + "base_branch", + "base_sha", + "claimed", + "issue_number", + "issue_slug", + "issue_title", + "issue_url", + "max_parallel", + "merge_target", + "open_pr", + "proposed_branch", + "remote_url", + "review_approach", + "run_mode", + "run_name", + "source_dir", + "task", + "upfront_claimed", + "work_dir" + ], + "required": [], + "produced": [ + "merge_target" + ] + }, + { + "step": "push_merge_target", + "available": [ + "audit", + "auto_merge", + "base_branch", + "base_sha", + "claimed", + "issue_number", + "issue_slug", + "issue_title", + "issue_url", + "max_parallel", + "merge_target", + "open_pr", + "proposed_branch", + "remote_url", + "review_approach", + "run_mode", + "run_name", + "source_dir", + "task", + "upfront_claimed", + "work_dir" + ], + "required": [], + "produced": [] + }, + { + "step": "plan", + "available": [ + "audit", + "auto_merge", + "base_branch", + "base_sha", + "claimed", + "issue_number", + "issue_slug", + "issue_title", + "issue_url", + "max_parallel", + "merge_target", + "open_pr", + "proposed_branch", + "remote_url", + "review_approach", + "run_mode", + "run_name", + "source_dir", + "task", + "upfront_claimed", + "work_dir" + ], + "required": [], + "produced": [ + "plan_path", + "all_plan_paths" + ] + }, + { + "step": "review", + "available": [ + "all_plan_paths", + "audit", + "auto_merge", + "base_branch", + "base_sha", + "claimed", + "issue_number", + "issue_slug", + "issue_title", + "issue_url", + "max_parallel", + "merge_target", + "open_pr", + "plan_path", + "proposed_branch", + "remote_url", + "review_approach", + "run_mode", + "run_name", + "source_dir", + "task", + "upfront_claimed", + "work_dir" + ], + "required": [], + "produced": [ + "review_path" + ] + }, + { + "step": "verify", + "available": [ + "all_plan_paths", + "audit", + "auto_merge", + "base_branch", + "base_sha", + "claimed", + "issue_number", + "issue_slug", + "issue_title", + "issue_url", + "max_parallel", + "merge_target", + "open_pr", + "plan_path", + "proposed_branch", + "remote_url", + "review_approach", + "review_path", + "run_mode", + "run_name", + "source_dir", + "task", + "upfront_claimed", + "work_dir" + ], + "required": [], + "produced": [] + }, + { + "step": "implement", + "available": [ + "all_plan_paths", + "audit", + "auto_merge", + "base_branch", + "base_sha", + "claimed", + "issue_number", + "issue_slug", + "issue_title", + "issue_url", + "max_parallel", + "merge_target", + "open_pr", + "plan_path", + "proposed_branch", + "remote_url", + "review_approach", + "review_path", + "run_mode", + "run_name", + "source_dir", + "task", + "upfront_claimed", + "work_dir" + ], + "required": [], + "produced": [ + "worktree_path" + ] + }, + { + "step": "retry_worktree", + "available": [ + "all_plan_paths", + "audit", + "auto_merge", + "base_branch", + "base_sha", + "claimed", + "issue_number", + "issue_slug", + "issue_title", + "issue_url", + "max_parallel", + "merge_target", + "open_pr", + "plan_path", + "proposed_branch", + "remote_url", + "review_approach", + "review_path", + "run_mode", + "run_name", + "source_dir", + "task", + "upfront_claimed", + "work_dir", + "worktree_path" + ], + "required": [], + "produced": [ + "worktree_path" + ] + }, + { + "step": "test", + "available": [ + "all_plan_paths", + "audit", + "auto_merge", + "base_branch", + "base_sha", + "claimed", + "issue_number", + "issue_slug", + "issue_title", + "issue_url", + "max_parallel", + "merge_target", + "open_pr", + "plan_path", + "proposed_branch", + "remote_url", + "review_approach", + "review_path", + "run_mode", + "run_name", + "source_dir", + "task", + "upfront_claimed", + "work_dir", + "worktree_path" + ], + "required": [], + "produced": [] + }, + { + "step": "merge", + "available": [ + "all_plan_paths", + "audit", + "auto_merge", + "base_branch", + "base_sha", + "claimed", + "issue_number", + "issue_slug", + "issue_title", + "issue_url", + "max_parallel", + "merge_target", + "open_pr", + "plan_path", + "proposed_branch", + "remote_url", + "review_approach", + "review_path", + "run_mode", + "run_name", + "source_dir", + "task", + "upfront_claimed", + "work_dir", + "worktree_path" + ], + "required": [], + "produced": [ + "cleanup_succeeded", + "worktree_path" + ] + }, + { + "step": "push", + "available": [ + "all_plan_paths", + "audit", + "auto_merge", + "base_branch", + "base_sha", + "claimed", + "cleanup_succeeded", + "issue_number", + "issue_slug", + "issue_title", + "issue_url", + "max_parallel", + "merge_target", + "open_pr", + "plan_path", + "proposed_branch", + "remote_url", + "review_approach", + "review_path", + "run_mode", + "run_name", + "source_dir", + "task", + "upfront_claimed", + "work_dir", + "worktree_path" + ], + "required": [], + "produced": [] + }, + { + "step": "fix", + "available": [ + "all_plan_paths", + "audit", + "auto_merge", + "base_branch", + "base_sha", + "claimed", + "cleanup_succeeded", + "issue_number", + "issue_slug", + "issue_title", + "issue_url", + "max_parallel", + "merge_target", + "open_pr", + "plan_path", + "proposed_branch", + "remote_url", + "review_approach", + "review_path", + "run_mode", + "run_name", + "source_dir", + "task", + "upfront_claimed", + "work_dir", + "worktree_path" + ], + "required": [], + "produced": [] + }, + { + "step": "next_or_done", + "available": [ + "all_plan_paths", + "audit", + "auto_merge", + "base_branch", + "base_sha", + "claimed", + "cleanup_succeeded", + "issue_number", + "issue_slug", + "issue_title", + "issue_url", + "max_parallel", + "merge_target", + "open_pr", + "plan_path", + "proposed_branch", + "remote_url", + "review_approach", + "review_path", + "run_mode", + "run_name", + "source_dir", + "task", + "upfront_claimed", + "work_dir", + "worktree_path" + ], + "required": [], + "produced": [] + }, + { + "step": "audit_impl", + "available": [ + "all_plan_paths", + "audit", + "auto_merge", + "base_branch", + "base_sha", + "claimed", + "cleanup_succeeded", + "issue_number", + "issue_slug", + "issue_title", + "issue_url", + "max_parallel", + "merge_target", + "open_pr", + "plan_path", + "proposed_branch", + "remote_url", + "review_approach", + "review_path", + "run_mode", + "run_name", + "source_dir", + "task", + "upfront_claimed", + "work_dir", + "worktree_path" + ], + "required": [], + "produced": [ + "remediation_path" + ] + }, + { + "step": "remediate", + "available": [ + "all_plan_paths", + "audit", + "auto_merge", + "base_branch", + "base_sha", + "claimed", + "cleanup_succeeded", + "issue_number", + "issue_slug", + "issue_title", + "issue_url", + "max_parallel", + "merge_target", + "open_pr", + "plan_path", + "proposed_branch", + "remediation_path", + "remote_url", + "review_approach", + "review_path", + "run_mode", + "run_name", + "source_dir", + "task", + "upfront_claimed", + "work_dir", + "worktree_path" + ], + "required": [], + "produced": [] + }, + { + "step": "compose_pr", + "available": [ + "all_plan_paths", + "audit", + "auto_merge", + "base_branch", + "base_sha", + "claimed", + "cleanup_succeeded", + "issue_number", + "issue_slug", + "issue_title", + "issue_url", + "max_parallel", + "merge_target", + "open_pr", + "plan_path", + "proposed_branch", + "remediation_path", + "remote_url", + "review_approach", + "review_path", + "run_mode", + "run_name", + "source_dir", + "task", + "upfront_claimed", + "work_dir", + "worktree_path" + ], + "required": [ + "plan_paths", + "feature_branch" + ], + "produced": [] + }, + { + "step": "extract_pr_number", + "available": [ + "all_plan_paths", + "audit", + "auto_merge", + "base_branch", + "base_sha", + "claimed", + "cleanup_succeeded", + "issue_number", + "issue_slug", + "issue_title", + "issue_url", + "max_parallel", + "merge_target", + "open_pr", + "plan_path", + "proposed_branch", + "remediation_path", + "remote_url", + "review_approach", + "review_path", + "run_mode", + "run_name", + "source_dir", + "task", + "upfront_claimed", + "work_dir", + "worktree_path" + ], + "required": [], + "produced": [ + "pr_number" + ] + }, + { + "step": "review_pr", + "available": [ + "all_plan_paths", + "audit", + "auto_merge", + "base_branch", + "base_sha", + "claimed", + "cleanup_succeeded", + "issue_number", + "issue_slug", + "issue_title", + "issue_url", + "max_parallel", + "merge_target", + "open_pr", + "plan_path", + "pr_number", + "proposed_branch", + "remediation_path", + "remote_url", + "review_approach", + "review_path", + "run_mode", + "run_name", + "source_dir", + "task", + "upfront_claimed", + "work_dir", + "worktree_path" + ], + "required": [ + "feature_branch" + ], + "produced": [ + "verdict" + ] + }, + { + "step": "resolve_review", + "available": [ + "all_plan_paths", + "audit", + "auto_merge", + "base_branch", + "base_sha", + "claimed", + "cleanup_succeeded", + "issue_number", + "issue_slug", + "issue_title", + "issue_url", + "max_parallel", + "merge_target", + "open_pr", + "plan_path", + "pr_number", + "proposed_branch", + "remediation_path", + "remote_url", + "review_approach", + "review_path", + "run_mode", + "run_name", + "source_dir", + "task", + "upfront_claimed", + "verdict", + "work_dir", + "worktree_path" + ], + "required": [ + "feature_branch" + ], + "produced": [] + }, + { + "step": "re_push_review", + "available": [ + "all_plan_paths", + "audit", + "auto_merge", + "base_branch", + "base_sha", + "claimed", + "cleanup_succeeded", + "issue_number", + "issue_slug", + "issue_title", + "issue_url", + "max_parallel", + "merge_target", + "open_pr", + "plan_path", + "pr_number", + "proposed_branch", + "remediation_path", + "remote_url", + "review_approach", + "review_path", + "run_mode", + "run_name", + "source_dir", + "task", + "upfront_claimed", + "verdict", + "work_dir", + "worktree_path" + ], + "required": [], + "produced": [] + }, + { + "step": "ci_watch", + "available": [ + "all_plan_paths", + "audit", + "auto_merge", + "base_branch", + "base_sha", + "claimed", + "cleanup_succeeded", + "issue_number", + "issue_slug", + "issue_title", + "issue_url", + "max_parallel", + "merge_target", + "open_pr", + "plan_path", + "pr_number", + "proposed_branch", + "remediation_path", + "remote_url", + "review_approach", + "review_path", + "run_mode", + "run_name", + "source_dir", + "task", + "upfront_claimed", + "verdict", + "work_dir", + "worktree_path" + ], + "required": [], + "produced": [ + "ci_conclusion", + "ci_failed_jobs" + ] + }, + { + "step": "check_merge_queue", + "available": [ + "all_plan_paths", + "audit", + "auto_merge", + "base_branch", + "base_sha", + "ci_conclusion", + "ci_failed_jobs", + "claimed", + "cleanup_succeeded", + "issue_number", + "issue_slug", + "issue_title", + "issue_url", + "max_parallel", + "merge_target", + "open_pr", + "plan_path", + "pr_number", + "proposed_branch", + "remediation_path", + "remote_url", + "review_approach", + "review_path", + "run_mode", + "run_name", + "source_dir", + "task", + "upfront_claimed", + "verdict", + "work_dir", + "worktree_path" + ], + "required": [], + "produced": [ + "queue_available" + ] + }, + { + "step": "check_merge_group_trigger", + "available": [ + "all_plan_paths", + "audit", + "auto_merge", + "base_branch", + "base_sha", + "ci_conclusion", + "ci_failed_jobs", + "claimed", + "cleanup_succeeded", + "issue_number", + "issue_slug", + "issue_title", + "issue_url", + "max_parallel", + "merge_target", + "open_pr", + "plan_path", + "pr_number", + "proposed_branch", + "queue_available", + "remediation_path", + "remote_url", + "review_approach", + "review_path", + "run_mode", + "run_name", + "source_dir", + "task", + "upfront_claimed", + "verdict", + "work_dir", + "worktree_path" + ], + "required": [], + "produced": [ + "merge_group_trigger" + ] + }, + { + "step": "check_auto_merge", + "available": [ + "all_plan_paths", + "audit", + "auto_merge", + "base_branch", + "base_sha", + "ci_conclusion", + "ci_failed_jobs", + "claimed", + "cleanup_succeeded", + "issue_number", + "issue_slug", + "issue_title", + "issue_url", + "max_parallel", + "merge_group_trigger", + "merge_target", + "open_pr", + "plan_path", + "pr_number", + "proposed_branch", + "queue_available", + "remediation_path", + "remote_url", + "review_approach", + "review_path", + "run_mode", + "run_name", + "source_dir", + "task", + "upfront_claimed", + "verdict", + "work_dir", + "worktree_path" + ], + "required": [], + "produced": [ + "auto_merge_available" + ] + }, + { + "step": "route_queue_mode", + "available": [ + "all_plan_paths", + "audit", + "auto_merge", + "auto_merge_available", + "base_branch", + "base_sha", + "ci_conclusion", + "ci_failed_jobs", + "claimed", + "cleanup_succeeded", + "issue_number", + "issue_slug", + "issue_title", + "issue_url", + "max_parallel", + "merge_group_trigger", + "merge_target", + "open_pr", + "plan_path", + "pr_number", + "proposed_branch", + "queue_available", + "remediation_path", + "remote_url", + "review_approach", + "review_path", + "run_mode", + "run_name", + "source_dir", + "task", + "upfront_claimed", + "verdict", + "work_dir", + "worktree_path" + ], + "required": [], + "produced": [] + }, + { + "step": "enable_auto_merge", + "available": [ + "all_plan_paths", + "audit", + "auto_merge", + "auto_merge_available", + "base_branch", + "base_sha", + "ci_conclusion", + "ci_failed_jobs", + "claimed", + "cleanup_succeeded", + "issue_number", + "issue_slug", + "issue_title", + "issue_url", + "max_parallel", + "merge_group_trigger", + "merge_target", + "open_pr", + "plan_path", + "pr_number", + "proposed_branch", + "queue_available", + "remediation_path", + "remote_url", + "review_approach", + "review_path", + "run_mode", + "run_name", + "source_dir", + "task", + "upfront_claimed", + "verdict", + "work_dir", + "worktree_path" + ], + "required": [], + "produced": [] + }, + { + "step": "wait_for_queue", + "available": [ + "all_plan_paths", + "audit", + "auto_merge", + "auto_merge_available", + "base_branch", + "base_sha", + "ci_conclusion", + "ci_failed_jobs", + "claimed", + "cleanup_succeeded", + "issue_number", + "issue_slug", + "issue_title", + "issue_url", + "max_parallel", + "merge_group_trigger", + "merge_target", + "open_pr", + "plan_path", + "pr_number", + "proposed_branch", + "queue_available", + "remediation_path", + "remote_url", + "review_approach", + "review_path", + "run_mode", + "run_name", + "source_dir", + "task", + "upfront_claimed", + "verdict", + "work_dir", + "worktree_path" + ], + "required": [], + "produced": [ + "queue_pr_state" + ] + }, + { + "step": "reenroll_stalled_pr", + "available": [ + "all_plan_paths", + "audit", + "auto_merge", + "auto_merge_available", + "base_branch", + "base_sha", + "ci_conclusion", + "ci_failed_jobs", + "claimed", + "cleanup_succeeded", + "issue_number", + "issue_slug", + "issue_title", + "issue_url", + "max_parallel", + "merge_group_trigger", + "merge_target", + "open_pr", + "plan_path", + "pr_number", + "proposed_branch", + "queue_available", + "queue_pr_state", + "remediation_path", + "remote_url", + "review_approach", + "review_path", + "run_mode", + "run_name", + "source_dir", + "task", + "upfront_claimed", + "verdict", + "work_dir", + "worktree_path" + ], + "required": [], + "produced": [] + }, + { + "step": "queue_ejected_fix", + "available": [ + "all_plan_paths", + "audit", + "auto_merge", + "auto_merge_available", + "base_branch", + "base_sha", + "ci_conclusion", + "ci_failed_jobs", + "claimed", + "cleanup_succeeded", + "issue_number", + "issue_slug", + "issue_title", + "issue_url", + "max_parallel", + "merge_group_trigger", + "merge_target", + "open_pr", + "plan_path", + "pr_number", + "proposed_branch", + "queue_available", + "queue_pr_state", + "remediation_path", + "remote_url", + "review_approach", + "review_path", + "run_mode", + "run_name", + "source_dir", + "task", + "upfront_claimed", + "verdict", + "work_dir", + "worktree_path" + ], + "required": [ + "worktree_path" + ], + "produced": [ + "conflict_escalation_required" + ] + }, + { + "step": "re_push_queue_fix", + "available": [ + "all_plan_paths", + "audit", + "auto_merge", + "auto_merge_available", + "base_branch", + "base_sha", + "ci_conclusion", + "ci_failed_jobs", + "claimed", + "cleanup_succeeded", + "conflict_escalation_required", + "issue_number", + "issue_slug", + "issue_title", + "issue_url", + "max_parallel", + "merge_group_trigger", + "merge_target", + "open_pr", + "plan_path", + "pr_number", + "proposed_branch", + "queue_available", + "queue_pr_state", + "remediation_path", + "remote_url", + "review_approach", + "review_path", + "run_mode", + "run_name", + "source_dir", + "task", + "upfront_claimed", + "verdict", + "work_dir", + "worktree_path" + ], + "required": [], + "produced": [] + }, + { + "step": "reenter_merge_queue", + "available": [ + "all_plan_paths", + "audit", + "auto_merge", + "auto_merge_available", + "base_branch", + "base_sha", + "ci_conclusion", + "ci_failed_jobs", + "claimed", + "cleanup_succeeded", + "conflict_escalation_required", + "issue_number", + "issue_slug", + "issue_title", + "issue_url", + "max_parallel", + "merge_group_trigger", + "merge_target", + "open_pr", + "plan_path", + "pr_number", + "proposed_branch", + "queue_available", + "queue_pr_state", + "remediation_path", + "remote_url", + "review_approach", + "review_path", + "run_mode", + "run_name", + "source_dir", + "task", + "upfront_claimed", + "verdict", + "work_dir", + "worktree_path" + ], + "required": [], + "produced": [] + }, + { + "step": "direct_merge", + "available": [ + "all_plan_paths", + "audit", + "auto_merge", + "auto_merge_available", + "base_branch", + "base_sha", + "ci_conclusion", + "ci_failed_jobs", + "claimed", + "cleanup_succeeded", + "conflict_escalation_required", + "issue_number", + "issue_slug", + "issue_title", + "issue_url", + "max_parallel", + "merge_group_trigger", + "merge_target", + "open_pr", + "plan_path", + "pr_number", + "proposed_branch", + "queue_available", + "queue_pr_state", + "remediation_path", + "remote_url", + "review_approach", + "review_path", + "run_mode", + "run_name", + "source_dir", + "task", + "upfront_claimed", + "verdict", + "work_dir", + "worktree_path" + ], + "required": [], + "produced": [] + }, + { + "step": "wait_for_direct_merge", + "available": [ + "all_plan_paths", + "audit", + "auto_merge", + "auto_merge_available", + "base_branch", + "base_sha", + "ci_conclusion", + "ci_failed_jobs", + "claimed", + "cleanup_succeeded", + "conflict_escalation_required", + "issue_number", + "issue_slug", + "issue_title", + "issue_url", + "max_parallel", + "merge_group_trigger", + "merge_target", + "open_pr", + "plan_path", + "pr_number", + "proposed_branch", + "queue_available", + "queue_pr_state", + "remediation_path", + "remote_url", + "review_approach", + "review_path", + "run_mode", + "run_name", + "source_dir", + "task", + "upfront_claimed", + "verdict", + "work_dir", + "worktree_path" + ], + "required": [], + "produced": [] + }, + { + "step": "direct_merge_conflict_fix", + "available": [ + "all_plan_paths", + "audit", + "auto_merge", + "auto_merge_available", + "base_branch", + "base_sha", + "ci_conclusion", + "ci_failed_jobs", + "claimed", + "cleanup_succeeded", + "conflict_escalation_required", + "issue_number", + "issue_slug", + "issue_title", + "issue_url", + "max_parallel", + "merge_group_trigger", + "merge_target", + "open_pr", + "plan_path", + "pr_number", + "proposed_branch", + "queue_available", + "queue_pr_state", + "remediation_path", + "remote_url", + "review_approach", + "review_path", + "run_mode", + "run_name", + "source_dir", + "task", + "upfront_claimed", + "verdict", + "work_dir", + "worktree_path" + ], + "required": [ + "worktree_path" + ], + "produced": [ + "conflict_escalation_required" + ] + }, + { + "step": "re_push_direct_fix", + "available": [ + "all_plan_paths", + "audit", + "auto_merge", + "auto_merge_available", + "base_branch", + "base_sha", + "ci_conclusion", + "ci_failed_jobs", + "claimed", + "cleanup_succeeded", + "conflict_escalation_required", + "issue_number", + "issue_slug", + "issue_title", + "issue_url", + "max_parallel", + "merge_group_trigger", + "merge_target", + "open_pr", + "plan_path", + "pr_number", + "proposed_branch", + "queue_available", + "queue_pr_state", + "remediation_path", + "remote_url", + "review_approach", + "review_path", + "run_mode", + "run_name", + "source_dir", + "task", + "upfront_claimed", + "verdict", + "work_dir", + "worktree_path" + ], + "required": [], + "produced": [] + }, + { + "step": "redirect_merge", + "available": [ + "all_plan_paths", + "audit", + "auto_merge", + "auto_merge_available", + "base_branch", + "base_sha", + "ci_conclusion", + "ci_failed_jobs", + "claimed", + "cleanup_succeeded", + "conflict_escalation_required", + "issue_number", + "issue_slug", + "issue_title", + "issue_url", + "max_parallel", + "merge_group_trigger", + "merge_target", + "open_pr", + "plan_path", + "pr_number", + "proposed_branch", + "queue_available", + "queue_pr_state", + "remediation_path", + "remote_url", + "review_approach", + "review_path", + "run_mode", + "run_name", + "source_dir", + "task", + "upfront_claimed", + "verdict", + "work_dir", + "worktree_path" + ], + "required": [], + "produced": [] + }, + { + "step": "immediate_merge", + "available": [ + "all_plan_paths", + "audit", + "auto_merge", + "auto_merge_available", + "base_branch", + "base_sha", + "ci_conclusion", + "ci_failed_jobs", + "claimed", + "cleanup_succeeded", + "conflict_escalation_required", + "issue_number", + "issue_slug", + "issue_title", + "issue_url", + "max_parallel", + "merge_group_trigger", + "merge_target", + "open_pr", + "plan_path", + "pr_number", + "proposed_branch", + "queue_available", + "queue_pr_state", + "remediation_path", + "remote_url", + "review_approach", + "review_path", + "run_mode", + "run_name", + "source_dir", + "task", + "upfront_claimed", + "verdict", + "work_dir", + "worktree_path" + ], + "required": [], + "produced": [] + }, + { + "step": "wait_for_immediate_merge", + "available": [ + "all_plan_paths", + "audit", + "auto_merge", + "auto_merge_available", + "base_branch", + "base_sha", + "ci_conclusion", + "ci_failed_jobs", + "claimed", + "cleanup_succeeded", + "conflict_escalation_required", + "issue_number", + "issue_slug", + "issue_title", + "issue_url", + "max_parallel", + "merge_group_trigger", + "merge_target", + "open_pr", + "plan_path", + "pr_number", + "proposed_branch", + "queue_available", + "queue_pr_state", + "remediation_path", + "remote_url", + "review_approach", + "review_path", + "run_mode", + "run_name", + "source_dir", + "task", + "upfront_claimed", + "verdict", + "work_dir", + "worktree_path" + ], + "required": [], + "produced": [] + }, + { + "step": "immediate_merge_conflict_fix", + "available": [ + "all_plan_paths", + "audit", + "auto_merge", + "auto_merge_available", + "base_branch", + "base_sha", + "ci_conclusion", + "ci_failed_jobs", + "claimed", + "cleanup_succeeded", + "conflict_escalation_required", + "issue_number", + "issue_slug", + "issue_title", + "issue_url", + "max_parallel", + "merge_group_trigger", + "merge_target", + "open_pr", + "plan_path", + "pr_number", + "proposed_branch", + "queue_available", + "queue_pr_state", + "remediation_path", + "remote_url", + "review_approach", + "review_path", + "run_mode", + "run_name", + "source_dir", + "task", + "upfront_claimed", + "verdict", + "work_dir", + "worktree_path" + ], + "required": [ + "worktree_path" + ], + "produced": [ + "conflict_escalation_required" + ] + }, + { + "step": "re_push_immediate_fix", + "available": [ + "all_plan_paths", + "audit", + "auto_merge", + "auto_merge_available", + "base_branch", + "base_sha", + "ci_conclusion", + "ci_failed_jobs", + "claimed", + "cleanup_succeeded", + "conflict_escalation_required", + "issue_number", + "issue_slug", + "issue_title", + "issue_url", + "max_parallel", + "merge_group_trigger", + "merge_target", + "open_pr", + "plan_path", + "pr_number", + "proposed_branch", + "queue_available", + "queue_pr_state", + "remediation_path", + "remote_url", + "review_approach", + "review_path", + "run_mode", + "run_name", + "source_dir", + "task", + "upfront_claimed", + "verdict", + "work_dir", + "worktree_path" + ], + "required": [], + "produced": [] + }, + { + "step": "remerge_immediate", + "available": [ + "all_plan_paths", + "audit", + "auto_merge", + "auto_merge_available", + "base_branch", + "base_sha", + "ci_conclusion", + "ci_failed_jobs", + "claimed", + "cleanup_succeeded", + "conflict_escalation_required", + "issue_number", + "issue_slug", + "issue_title", + "issue_url", + "max_parallel", + "merge_group_trigger", + "merge_target", + "open_pr", + "plan_path", + "pr_number", + "proposed_branch", + "queue_available", + "queue_pr_state", + "remediation_path", + "remote_url", + "review_approach", + "review_path", + "run_mode", + "run_name", + "source_dir", + "task", + "upfront_claimed", + "verdict", + "work_dir", + "worktree_path" + ], + "required": [], + "produced": [] + }, + { + "step": "diagnose_ci", + "available": [ + "all_plan_paths", + "audit", + "auto_merge", + "auto_merge_available", + "base_branch", + "base_sha", + "ci_conclusion", + "ci_failed_jobs", + "claimed", + "cleanup_succeeded", + "conflict_escalation_required", + "issue_number", + "issue_slug", + "issue_title", + "issue_url", + "max_parallel", + "merge_group_trigger", + "merge_target", + "open_pr", + "plan_path", + "pr_number", + "proposed_branch", + "queue_available", + "queue_pr_state", + "remediation_path", + "remote_url", + "review_approach", + "review_path", + "run_mode", + "run_name", + "source_dir", + "task", + "upfront_claimed", + "verdict", + "work_dir", + "worktree_path" + ], + "required": [], + "produced": [ + "diagnosis_path" + ] + }, + { + "step": "resolve_ci", + "available": [ + "all_plan_paths", + "audit", + "auto_merge", + "auto_merge_available", + "base_branch", + "base_sha", + "ci_conclusion", + "ci_failed_jobs", + "claimed", + "cleanup_succeeded", + "conflict_escalation_required", + "diagnosis_path", + "issue_number", + "issue_slug", + "issue_title", + "issue_url", + "max_parallel", + "merge_group_trigger", + "merge_target", + "open_pr", + "plan_path", + "pr_number", + "proposed_branch", + "queue_available", + "queue_pr_state", + "remediation_path", + "remote_url", + "review_approach", + "review_path", + "run_mode", + "run_name", + "source_dir", + "task", + "upfront_claimed", + "verdict", + "work_dir", + "worktree_path" + ], + "required": [ + "worktree_path" + ], + "produced": [] + }, + { + "step": "re_push", + "available": [ + "all_plan_paths", + "audit", + "auto_merge", + "auto_merge_available", + "base_branch", + "base_sha", + "ci_conclusion", + "ci_failed_jobs", + "claimed", + "cleanup_succeeded", + "conflict_escalation_required", + "diagnosis_path", + "issue_number", + "issue_slug", + "issue_title", + "issue_url", + "max_parallel", + "merge_group_trigger", + "merge_target", + "open_pr", + "plan_path", + "pr_number", + "proposed_branch", + "queue_available", + "queue_pr_state", + "remediation_path", + "remote_url", + "review_approach", + "review_path", + "run_mode", + "run_name", + "source_dir", + "task", + "upfront_claimed", + "verdict", + "work_dir", + "worktree_path" + ], + "required": [], + "produced": [] + }, + { + "step": "detect_ci_conflict", + "available": [ + "all_plan_paths", + "audit", + "auto_merge", + "auto_merge_available", + "base_branch", + "base_sha", + "ci_conclusion", + "ci_failed_jobs", + "claimed", + "cleanup_succeeded", + "conflict_escalation_required", + "diagnosis_path", + "issue_number", + "issue_slug", + "issue_title", + "issue_url", + "max_parallel", + "merge_group_trigger", + "merge_target", + "open_pr", + "plan_path", + "pr_number", + "proposed_branch", + "queue_available", + "queue_pr_state", + "remediation_path", + "remote_url", + "review_approach", + "review_path", + "run_mode", + "run_name", + "source_dir", + "task", + "upfront_claimed", + "verdict", + "work_dir", + "worktree_path" + ], + "required": [], + "produced": [] + }, + { + "step": "ci_conflict_fix", + "available": [ + "all_plan_paths", + "audit", + "auto_merge", + "auto_merge_available", + "base_branch", + "base_sha", + "ci_conclusion", + "ci_failed_jobs", + "claimed", + "cleanup_succeeded", + "conflict_escalation_required", + "diagnosis_path", + "issue_number", + "issue_slug", + "issue_title", + "issue_url", + "max_parallel", + "merge_group_trigger", + "merge_target", + "open_pr", + "plan_path", + "pr_number", + "proposed_branch", + "queue_available", + "queue_pr_state", + "remediation_path", + "remote_url", + "review_approach", + "review_path", + "run_mode", + "run_name", + "source_dir", + "task", + "upfront_claimed", + "verdict", + "work_dir", + "worktree_path" + ], + "required": [ + "worktree_path" + ], + "produced": [ + "conflict_escalation_required" + ] + }, + { + "step": "release_issue_success", + "available": [ + "all_plan_paths", + "audit", + "auto_merge", + "auto_merge_available", + "base_branch", + "base_sha", + "ci_conclusion", + "ci_failed_jobs", + "claimed", + "cleanup_succeeded", + "conflict_escalation_required", + "diagnosis_path", + "issue_number", + "issue_slug", + "issue_title", + "issue_url", + "max_parallel", + "merge_group_trigger", + "merge_target", + "open_pr", + "plan_path", + "pr_number", + "proposed_branch", + "queue_available", + "queue_pr_state", + "remediation_path", + "remote_url", + "review_approach", + "review_path", + "run_mode", + "run_name", + "source_dir", + "task", + "upfront_claimed", + "verdict", + "work_dir", + "worktree_path" + ], + "required": [], + "produced": [] + }, + { + "step": "register_clone_unconfirmed", + "available": [ + "all_plan_paths", + "audit", + "auto_merge", + "auto_merge_available", + "base_branch", + "base_sha", + "ci_conclusion", + "ci_failed_jobs", + "claimed", + "cleanup_succeeded", + "conflict_escalation_required", + "diagnosis_path", + "issue_number", + "issue_slug", + "issue_title", + "issue_url", + "max_parallel", + "merge_group_trigger", + "merge_target", + "open_pr", + "plan_path", + "pr_number", + "proposed_branch", + "queue_available", + "queue_pr_state", + "remediation_path", + "remote_url", + "review_approach", + "review_path", + "run_mode", + "run_name", + "source_dir", + "task", + "upfront_claimed", + "verdict", + "work_dir", + "worktree_path" + ], + "required": [], + "produced": [] + }, + { + "step": "release_issue_failure", + "available": [ + "all_plan_paths", + "audit", + "auto_merge", + "auto_merge_available", + "base_branch", + "base_sha", + "ci_conclusion", + "ci_failed_jobs", + "claimed", + "cleanup_succeeded", + "conflict_escalation_required", + "diagnosis_path", + "issue_number", + "issue_slug", + "issue_title", + "issue_url", + "max_parallel", + "merge_group_trigger", + "merge_target", + "open_pr", + "plan_path", + "pr_number", + "proposed_branch", + "queue_available", + "queue_pr_state", + "remediation_path", + "remote_url", + "review_approach", + "review_path", + "run_mode", + "run_name", + "source_dir", + "task", + "upfront_claimed", + "verdict", + "work_dir", + "worktree_path" + ], + "required": [], + "produced": [] + }, + { + "step": "register_clone_success", + "available": [ + "all_plan_paths", + "audit", + "auto_merge", + "auto_merge_available", + "base_branch", + "base_sha", + "ci_conclusion", + "ci_failed_jobs", + "claimed", + "cleanup_succeeded", + "conflict_escalation_required", + "diagnosis_path", + "issue_number", + "issue_slug", + "issue_title", + "issue_url", + "max_parallel", + "merge_group_trigger", + "merge_target", + "open_pr", + "plan_path", + "pr_number", + "proposed_branch", + "queue_available", + "queue_pr_state", + "remediation_path", + "remote_url", + "review_approach", + "review_path", + "run_mode", + "run_name", + "source_dir", + "task", + "upfront_claimed", + "verdict", + "work_dir", + "worktree_path" + ], + "required": [], + "produced": [] + }, + { + "step": "register_clone_failure", + "available": [ + "all_plan_paths", + "audit", + "auto_merge", + "auto_merge_available", + "base_branch", + "base_sha", + "ci_conclusion", + "ci_failed_jobs", + "claimed", + "cleanup_succeeded", + "conflict_escalation_required", + "diagnosis_path", + "issue_number", + "issue_slug", + "issue_title", + "issue_url", + "max_parallel", + "merge_group_trigger", + "merge_target", + "open_pr", + "plan_path", + "pr_number", + "proposed_branch", + "queue_available", + "queue_pr_state", + "remediation_path", + "remote_url", + "review_approach", + "review_path", + "run_mode", + "run_name", + "source_dir", + "task", + "upfront_claimed", + "verdict", + "work_dir", + "worktree_path" + ], + "required": [], + "produced": [] + }, + { + "step": "done", + "available": [ + "all_plan_paths", + "audit", + "auto_merge", + "auto_merge_available", + "base_branch", + "base_sha", + "ci_conclusion", + "ci_failed_jobs", + "claimed", + "cleanup_succeeded", + "conflict_escalation_required", + "diagnosis_path", + "issue_number", + "issue_slug", + "issue_title", + "issue_url", + "max_parallel", + "merge_group_trigger", + "merge_target", + "open_pr", + "plan_path", + "pr_number", + "proposed_branch", + "queue_available", + "queue_pr_state", + "remediation_path", + "remote_url", + "review_approach", + "review_path", + "run_mode", + "run_name", + "source_dir", + "task", + "upfront_claimed", + "verdict", + "work_dir", + "worktree_path" + ], + "required": [], + "produced": [] + }, + { + "step": "escalate_stop", + "available": [ + "all_plan_paths", + "audit", + "auto_merge", + "auto_merge_available", + "base_branch", + "base_sha", + "ci_conclusion", + "ci_failed_jobs", + "claimed", + "cleanup_succeeded", + "conflict_escalation_required", + "diagnosis_path", + "issue_number", + "issue_slug", + "issue_title", + "issue_url", + "max_parallel", + "merge_group_trigger", + "merge_target", + "open_pr", + "plan_path", + "pr_number", + "proposed_branch", + "queue_available", + "queue_pr_state", + "remediation_path", + "remote_url", + "review_approach", + "review_path", + "run_mode", + "run_name", + "source_dir", + "task", + "upfront_claimed", + "verdict", + "work_dir", + "worktree_path" + ], + "required": [], + "produced": [] + } + ] +} diff --git a/src/autoskillit/recipes/contracts/implementation.yaml b/src/autoskillit/recipes/contracts/implementation.yaml index 02f4ac03ea..60c693c0ce 100644 --- a/src/autoskillit/recipes/contracts/implementation.yaml +++ b/src/autoskillit/recipes/contracts/implementation.yaml @@ -146,9 +146,10 @@ skills: - name: verdict type: string expected_output_patterns: - - verdict\s*=\s*(approved|changes_requested|needs_human) + - verdict\s*=\s*(approved|approved_with_comments|changes_requested|needs_human) pattern_examples: - 'verdict = approved' + - 'verdict = approved_with_comments' - 'verdict = changes_requested' - 'verdict = needs_human' resolve-review: @@ -216,34 +217,18 @@ skills: - 'diagnosis_path = /tmp/diagnosis.md' write_behavior: always dataflow: -- step: sprint_entry - available: - - audit - - auto_merge - - base_branch - - issue_url - - open_pr - - review_approach - - run_mode - - run_name - - source_dir - - sprint_mode - - task - - upfront_claimed - required: [] - produced: [] - step: clone available: - audit - auto_merge - base_branch - issue_url + - max_parallel - open_pr - review_approach - run_mode - run_name - source_dir - - sprint_mode - task - upfront_claimed required: [] @@ -256,13 +241,13 @@ dataflow: - auto_merge - base_branch - issue_url + - max_parallel - open_pr - remote_url - review_approach - run_mode - run_name - source_dir - - sprint_mode - task - upfront_claimed - work_dir @@ -276,13 +261,13 @@ dataflow: - base_branch - base_sha - issue_url + - max_parallel - open_pr - remote_url - review_approach - run_mode - run_name - source_dir - - sprint_mode - task - upfront_claimed - work_dir @@ -296,6 +281,7 @@ dataflow: - base_branch - base_sha - issue_url + - max_parallel - merge_target - open_pr - remote_url @@ -303,7 +289,6 @@ dataflow: - run_mode - run_name - source_dir - - sprint_mode - task - upfront_claimed - work_dir @@ -322,6 +307,7 @@ dataflow: - issue_slug - issue_title - issue_url + - max_parallel - merge_target - open_pr - remote_url @@ -329,7 +315,6 @@ dataflow: - run_mode - run_name - source_dir - - sprint_mode - task - upfront_claimed - work_dir @@ -347,6 +332,7 @@ dataflow: - issue_slug - issue_title - issue_url + - max_parallel - merge_target - open_pr - remote_url @@ -354,7 +340,6 @@ dataflow: - run_mode - run_name - source_dir - - sprint_mode - task - upfront_claimed - work_dir @@ -372,6 +357,7 @@ dataflow: - issue_slug - issue_title - issue_url + - max_parallel - merge_target - open_pr - proposed_branch @@ -380,7 +366,6 @@ dataflow: - run_mode - run_name - source_dir - - sprint_mode - task - upfront_claimed - work_dir @@ -398,6 +383,7 @@ dataflow: - issue_slug - issue_title - issue_url + - max_parallel - merge_target - open_pr - proposed_branch @@ -406,7 +392,6 @@ dataflow: - run_mode - run_name - source_dir - - sprint_mode - task - upfront_claimed - work_dir @@ -423,6 +408,7 @@ dataflow: - issue_slug - issue_title - issue_url + - max_parallel - merge_target - open_pr - proposed_branch @@ -431,7 +417,6 @@ dataflow: - run_mode - run_name - source_dir - - sprint_mode - task - upfront_claimed - work_dir @@ -451,6 +436,7 @@ dataflow: - issue_slug - issue_title - issue_url + - max_parallel - merge_target - open_pr - plan_path @@ -460,7 +446,6 @@ dataflow: - run_mode - run_name - source_dir - - sprint_mode - task - upfront_claimed - work_dir @@ -479,6 +464,7 @@ dataflow: - issue_slug - issue_title - issue_url + - max_parallel - merge_target - open_pr - plan_path @@ -489,7 +475,6 @@ dataflow: - run_mode - run_name - source_dir - - sprint_mode - task - upfront_claimed - work_dir @@ -507,6 +492,7 @@ dataflow: - issue_slug - issue_title - issue_url + - max_parallel - merge_target - open_pr - plan_path @@ -517,7 +503,6 @@ dataflow: - run_mode - run_name - source_dir - - sprint_mode - task - upfront_claimed - work_dir @@ -536,6 +521,7 @@ dataflow: - issue_slug - issue_title - issue_url + - max_parallel - merge_target - open_pr - plan_path @@ -546,7 +532,6 @@ dataflow: - run_mode - run_name - source_dir - - sprint_mode - task - upfront_claimed - work_dir @@ -566,6 +551,7 @@ dataflow: - issue_slug - issue_title - issue_url + - max_parallel - merge_target - open_pr - plan_path @@ -576,7 +562,6 @@ dataflow: - run_mode - run_name - source_dir - - sprint_mode - task - upfront_claimed - work_dir @@ -595,6 +580,7 @@ dataflow: - issue_slug - issue_title - issue_url + - max_parallel - merge_target - open_pr - plan_path @@ -605,7 +591,6 @@ dataflow: - run_mode - run_name - source_dir - - sprint_mode - task - upfront_claimed - work_dir @@ -627,6 +612,7 @@ dataflow: - issue_slug - issue_title - issue_url + - max_parallel - merge_target - open_pr - plan_path @@ -637,7 +623,6 @@ dataflow: - run_mode - run_name - source_dir - - sprint_mode - task - upfront_claimed - work_dir @@ -657,6 +642,7 @@ dataflow: - issue_slug - issue_title - issue_url + - max_parallel - merge_target - open_pr - plan_path @@ -667,7 +653,6 @@ dataflow: - run_mode - run_name - source_dir - - sprint_mode - task - upfront_claimed - work_dir @@ -687,6 +672,7 @@ dataflow: - issue_slug - issue_title - issue_url + - max_parallel - merge_target - open_pr - plan_path @@ -697,7 +683,6 @@ dataflow: - run_mode - run_name - source_dir - - sprint_mode - task - upfront_claimed - work_dir @@ -717,6 +702,7 @@ dataflow: - issue_slug - issue_title - issue_url + - max_parallel - merge_target - open_pr - plan_path @@ -727,7 +713,6 @@ dataflow: - run_mode - run_name - source_dir - - sprint_mode - task - upfront_claimed - work_dir @@ -748,6 +733,7 @@ dataflow: - issue_slug - issue_title - issue_url + - max_parallel - merge_target - open_pr - plan_path @@ -759,7 +745,6 @@ dataflow: - run_mode - run_name - source_dir - - sprint_mode - task - upfront_claimed - work_dir @@ -779,6 +764,7 @@ dataflow: - issue_slug - issue_title - issue_url + - max_parallel - merge_target - open_pr - plan_path @@ -790,7 +776,6 @@ dataflow: - run_mode - run_name - source_dir - - sprint_mode - task - upfront_claimed - work_dir @@ -812,6 +797,7 @@ dataflow: - issue_slug - issue_title - issue_url + - max_parallel - merge_target - open_pr - plan_path @@ -823,7 +809,6 @@ dataflow: - run_mode - run_name - source_dir - - sprint_mode - task - upfront_claimed - work_dir @@ -844,6 +829,7 @@ dataflow: - issue_slug - issue_title - issue_url + - max_parallel - merge_target - open_pr - plan_path @@ -856,7 +842,6 @@ dataflow: - run_mode - run_name - source_dir - - sprint_mode - task - upfront_claimed - work_dir @@ -878,6 +863,7 @@ dataflow: - issue_slug - issue_title - issue_url + - max_parallel - merge_target - open_pr - plan_path @@ -890,7 +876,6 @@ dataflow: - run_mode - run_name - source_dir - - sprint_mode - task - upfront_claimed - verdict @@ -912,6 +897,7 @@ dataflow: - issue_slug - issue_title - issue_url + - max_parallel - merge_target - open_pr - plan_path @@ -924,7 +910,6 @@ dataflow: - run_mode - run_name - source_dir - - sprint_mode - task - upfront_claimed - verdict @@ -945,6 +930,7 @@ dataflow: - issue_slug - issue_title - issue_url + - max_parallel - merge_target - open_pr - plan_path @@ -957,7 +943,6 @@ dataflow: - run_mode - run_name - source_dir - - sprint_mode - task - upfront_claimed - verdict @@ -982,6 +967,7 @@ dataflow: - issue_slug - issue_title - issue_url + - max_parallel - merge_target - open_pr - plan_path @@ -994,7 +980,6 @@ dataflow: - run_mode - run_name - source_dir - - sprint_mode - task - upfront_claimed - verdict @@ -1018,6 +1003,7 @@ dataflow: - issue_slug - issue_title - issue_url + - max_parallel - merge_target - open_pr - plan_path @@ -1031,7 +1017,6 @@ dataflow: - run_mode - run_name - source_dir - - sprint_mode - task - upfront_claimed - verdict @@ -1055,6 +1040,7 @@ dataflow: - issue_slug - issue_title - issue_url + - max_parallel - merge_group_trigger - merge_target - open_pr @@ -1069,7 +1055,6 @@ dataflow: - run_mode - run_name - source_dir - - sprint_mode - task - upfront_claimed - verdict @@ -1094,6 +1079,7 @@ dataflow: - issue_slug - issue_title - issue_url + - max_parallel - merge_group_trigger - merge_target - open_pr @@ -1108,7 +1094,6 @@ dataflow: - run_mode - run_name - source_dir - - sprint_mode - task - upfront_claimed - verdict @@ -1132,6 +1117,7 @@ dataflow: - issue_slug - issue_title - issue_url + - max_parallel - merge_group_trigger - merge_target - open_pr @@ -1146,7 +1132,6 @@ dataflow: - run_mode - run_name - source_dir - - sprint_mode - task - upfront_claimed - verdict @@ -1170,6 +1155,7 @@ dataflow: - issue_slug - issue_title - issue_url + - max_parallel - merge_group_trigger - merge_target - open_pr @@ -1184,7 +1170,6 @@ dataflow: - run_mode - run_name - source_dir - - sprint_mode - task - upfront_claimed - verdict @@ -1209,6 +1194,7 @@ dataflow: - issue_slug - issue_title - issue_url + - max_parallel - merge_group_trigger - merge_target - open_pr @@ -1224,7 +1210,6 @@ dataflow: - run_mode - run_name - source_dir - - sprint_mode - task - upfront_claimed - verdict @@ -1248,6 +1233,7 @@ dataflow: - issue_slug - issue_title - issue_url + - max_parallel - merge_group_trigger - merge_target - open_pr @@ -1263,7 +1249,6 @@ dataflow: - run_mode - run_name - source_dir - - sprint_mode - task - upfront_claimed - verdict @@ -1290,6 +1275,7 @@ dataflow: - issue_slug - issue_title - issue_url + - max_parallel - merge_group_trigger - merge_target - open_pr @@ -1305,7 +1291,6 @@ dataflow: - run_mode - run_name - source_dir - - sprint_mode - task - upfront_claimed - verdict @@ -1330,6 +1315,7 @@ dataflow: - issue_slug - issue_title - issue_url + - max_parallel - merge_group_trigger - merge_target - open_pr @@ -1345,7 +1331,6 @@ dataflow: - run_mode - run_name - source_dir - - sprint_mode - task - upfront_claimed - verdict @@ -1370,6 +1355,7 @@ dataflow: - issue_slug - issue_title - issue_url + - max_parallel - merge_group_trigger - merge_target - open_pr @@ -1385,7 +1371,6 @@ dataflow: - run_mode - run_name - source_dir - - sprint_mode - task - upfront_claimed - verdict @@ -1410,6 +1395,7 @@ dataflow: - issue_slug - issue_title - issue_url + - max_parallel - merge_group_trigger - merge_target - open_pr @@ -1425,7 +1411,6 @@ dataflow: - run_mode - run_name - source_dir - - sprint_mode - task - upfront_claimed - verdict @@ -1450,6 +1435,7 @@ dataflow: - issue_slug - issue_title - issue_url + - max_parallel - merge_group_trigger - merge_target - open_pr @@ -1465,7 +1451,6 @@ dataflow: - run_mode - run_name - source_dir - - sprint_mode - task - upfront_claimed - verdict @@ -1492,6 +1477,7 @@ dataflow: - issue_slug - issue_title - issue_url + - max_parallel - merge_group_trigger - merge_target - open_pr @@ -1507,7 +1493,6 @@ dataflow: - run_mode - run_name - source_dir - - sprint_mode - task - upfront_claimed - verdict @@ -1532,6 +1517,7 @@ dataflow: - issue_slug - issue_title - issue_url + - max_parallel - merge_group_trigger - merge_target - open_pr @@ -1547,7 +1533,6 @@ dataflow: - run_mode - run_name - source_dir - - sprint_mode - task - upfront_claimed - verdict @@ -1572,6 +1557,7 @@ dataflow: - issue_slug - issue_title - issue_url + - max_parallel - merge_group_trigger - merge_target - open_pr @@ -1587,7 +1573,6 @@ dataflow: - run_mode - run_name - source_dir - - sprint_mode - task - upfront_claimed - verdict @@ -1612,6 +1597,7 @@ dataflow: - issue_slug - issue_title - issue_url + - max_parallel - merge_group_trigger - merge_target - open_pr @@ -1627,7 +1613,6 @@ dataflow: - run_mode - run_name - source_dir - - sprint_mode - task - upfront_claimed - verdict @@ -1652,6 +1637,7 @@ dataflow: - issue_slug - issue_title - issue_url + - max_parallel - merge_group_trigger - merge_target - open_pr @@ -1667,7 +1653,6 @@ dataflow: - run_mode - run_name - source_dir - - sprint_mode - task - upfront_claimed - verdict @@ -1694,6 +1679,7 @@ dataflow: - issue_slug - issue_title - issue_url + - max_parallel - merge_group_trigger - merge_target - open_pr @@ -1709,7 +1695,6 @@ dataflow: - run_mode - run_name - source_dir - - sprint_mode - task - upfront_claimed - verdict @@ -1734,6 +1719,7 @@ dataflow: - issue_slug - issue_title - issue_url + - max_parallel - merge_group_trigger - merge_target - open_pr @@ -1749,7 +1735,6 @@ dataflow: - run_mode - run_name - source_dir - - sprint_mode - task - upfront_claimed - verdict @@ -1774,6 +1759,7 @@ dataflow: - issue_slug - issue_title - issue_url + - max_parallel - merge_group_trigger - merge_target - open_pr @@ -1789,7 +1775,6 @@ dataflow: - run_mode - run_name - source_dir - - sprint_mode - task - upfront_claimed - verdict @@ -1816,6 +1801,7 @@ dataflow: - issue_slug - issue_title - issue_url + - max_parallel - merge_group_trigger - merge_target - open_pr @@ -1831,7 +1817,6 @@ dataflow: - run_mode - run_name - source_dir - - sprint_mode - task - upfront_claimed - verdict @@ -1858,6 +1843,7 @@ dataflow: - issue_slug - issue_title - issue_url + - max_parallel - merge_group_trigger - merge_target - open_pr @@ -1873,7 +1859,6 @@ dataflow: - run_mode - run_name - source_dir - - sprint_mode - task - upfront_claimed - verdict @@ -1899,6 +1884,7 @@ dataflow: - issue_slug - issue_title - issue_url + - max_parallel - merge_group_trigger - merge_target - open_pr @@ -1914,7 +1900,6 @@ dataflow: - run_mode - run_name - source_dir - - sprint_mode - task - upfront_claimed - verdict @@ -1940,6 +1925,7 @@ dataflow: - issue_slug - issue_title - issue_url + - max_parallel - merge_group_trigger - merge_target - open_pr @@ -1955,7 +1941,6 @@ dataflow: - run_mode - run_name - source_dir - - sprint_mode - task - upfront_claimed - verdict @@ -1983,6 +1968,7 @@ dataflow: - issue_slug - issue_title - issue_url + - max_parallel - merge_group_trigger - merge_target - open_pr @@ -1998,7 +1984,6 @@ dataflow: - run_mode - run_name - source_dir - - sprint_mode - task - upfront_claimed - verdict @@ -2006,7 +1991,7 @@ dataflow: - worktree_path required: [] produced: [] -- step: release_issue_timeout +- step: register_clone_unconfirmed available: - all_plan_paths - audit @@ -2024,6 +2009,7 @@ dataflow: - issue_slug - issue_title - issue_url + - max_parallel - merge_group_trigger - merge_target - open_pr @@ -2039,7 +2025,6 @@ dataflow: - run_mode - run_name - source_dir - - sprint_mode - task - upfront_claimed - verdict @@ -2065,6 +2050,7 @@ dataflow: - issue_slug - issue_title - issue_url + - max_parallel - merge_group_trigger - merge_target - open_pr @@ -2080,7 +2066,6 @@ dataflow: - run_mode - run_name - source_dir - - sprint_mode - task - upfront_claimed - verdict @@ -2106,6 +2091,7 @@ dataflow: - issue_slug - issue_title - issue_url + - max_parallel - merge_group_trigger - merge_target - open_pr @@ -2121,7 +2107,6 @@ dataflow: - run_mode - run_name - source_dir - - sprint_mode - task - upfront_claimed - verdict @@ -2147,6 +2132,7 @@ dataflow: - issue_slug - issue_title - issue_url + - max_parallel - merge_group_trigger - merge_target - open_pr @@ -2162,7 +2148,6 @@ dataflow: - run_mode - run_name - source_dir - - sprint_mode - task - upfront_claimed - verdict @@ -2188,6 +2173,7 @@ dataflow: - issue_slug - issue_title - issue_url + - max_parallel - merge_group_trigger - merge_target - open_pr @@ -2203,7 +2189,6 @@ dataflow: - run_mode - run_name - source_dir - - sprint_mode - task - upfront_claimed - verdict @@ -2229,6 +2214,7 @@ dataflow: - issue_slug - issue_title - issue_url + - max_parallel - merge_group_trigger - merge_target - open_pr @@ -2244,7 +2230,6 @@ dataflow: - run_mode - run_name - source_dir - - sprint_mode - task - upfront_claimed - verdict diff --git a/src/autoskillit/recipes/contracts/merge-prs.json b/src/autoskillit/recipes/contracts/merge-prs.json new file mode 100644 index 0000000000..3c299ef0d4 --- /dev/null +++ b/src/autoskillit/recipes/contracts/merge-prs.json @@ -0,0 +1,2194 @@ +{ + "generated_at": "2026-04-25T21:57:00.000000+00:00", + "bundled_manifest_version": "0.1.0", + "skill_hashes": {}, + "skills": { + "analyze-prs": { + "inputs": [ + { + "name": "base_branch", + "type": "string", + "required": true + }, + { + "name": "merge_queue_data_path", + "type": "file_path", + "required": false + } + ], + "outputs": [ + { + "name": "pr_order_file", + "type": "file_path" + }, + { + "name": "analysis_file", + "type": "file_path" + }, + { + "name": "batch_branch", + "type": "string" + }, + { + "name": "pr_count", + "type": "string" + }, + { + "name": "simple_count", + "type": "string" + }, + { + "name": "needs_check_count", + "type": "string" + }, + { + "name": "queue_mode", + "type": "string" + } + ], + "expected_output_patterns": [ + "pr_order_file\\s*=\\s*.+" + ], + "pattern_examples": [ + "pr_order_file = /tmp/pr_order.json" + ] + }, + "resolve-merge-conflicts": { + "inputs": [ + { + "name": "worktree_path", + "type": "directory_path", + "required": true + }, + { + "name": "plan_path", + "type": "file_path", + "required": true + }, + { + "name": "base_branch", + "type": "string", + "required": true + } + ], + "outputs": [ + { + "name": "worktree_path", + "type": "directory_path" + }, + { + "name": "branch_name", + "type": "string" + }, + { + "name": "conflict_report_path", + "type": "file_path" + }, + { + "name": "escalation_required", + "type": "string" + }, + { + "name": "escalation_reason", + "type": "string" + } + ], + "expected_output_patterns": [ + "worktree_path\\s*=\\s*/.+" + ], + "pattern_examples": [ + "worktree_path = /tmp/worktrees/impl-foo-20260316" + ], + "write_behavior": "conditional", + "write_expected_when": [ + "conflict_report_path\\s*=\\s*/.+" + ] + }, + "merge-pr": { + "inputs": [ + { + "name": "pr_number", + "type": "string", + "required": true + }, + { + "name": "complexity", + "type": "string", + "required": true + } + ], + "outputs": [ + { + "name": "merged", + "type": "string", + "allowed_values": [ + "true", + "false" + ] + }, + { + "name": "needs_plan", + "type": "string" + }, + { + "name": "escalation_required", + "type": "string" + }, + { + "name": "escalation_reason", + "type": "string" + }, + { + "name": "pr_number", + "type": "string" + }, + { + "name": "pr_branch", + "type": "string" + }, + { + "name": "pr_title", + "type": "string" + }, + { + "name": "conflict_report_path", + "type": "file_path" + } + ], + "expected_output_patterns": [ + "merged\\s*=\\s*(true|false)" + ], + "pattern_examples": [ + "merged = true", + "merged = false" + ] + }, + "make-plan": { + "inputs": [ + { + "name": "task", + "type": "string", + "required": true + } + ], + "outputs": [ + { + "name": "plan_path", + "type": "file_path" + }, + { + "name": "plan_parts", + "type": "file_path_list" + } + ], + "expected_output_patterns": [ + "plan_path\\s*=\\s*/.+" + ], + "pattern_examples": [ + "plan_path = /tmp/plan.md" + ], + "write_behavior": "always" + }, + "dry-walkthrough": { + "inputs": [ + { + "name": "plan_path", + "type": "file_path", + "required": true + } + ], + "outputs": [], + "expected_output_patterns": [], + "pattern_examples": [], + "write_behavior": "always" + }, + "implement-worktree-no-merge": { + "inputs": [ + { + "name": "plan_path", + "type": "file_path", + "required": true + } + ], + "outputs": [ + { + "name": "worktree_path", + "type": "directory_path" + }, + { + "name": "branch_name", + "type": "string" + } + ], + "expected_output_patterns": [ + "worktree_path\\s*=\\s*/.+" + ], + "pattern_examples": [ + "worktree_path = /tmp/worktrees/impl-foo-20260316" + ], + "write_behavior": "always" + }, + "retry-worktree": { + "inputs": [ + { + "name": "plan_path", + "type": "file_path", + "required": true + }, + { + "name": "worktree_path", + "type": "directory_path", + "required": true + } + ], + "outputs": [ + { + "name": "worktree_path", + "type": "directory_path" + }, + { + "name": "branch_name", + "type": "string" + } + ], + "expected_output_patterns": [ + "worktree_path\\s*=\\s*/.+" + ], + "pattern_examples": [ + "worktree_path = /tmp/worktrees/impl-foo-20260316" + ], + "write_behavior": "always" + }, + "resolve-failures": { + "inputs": [ + { + "name": "worktree_path", + "type": "directory_path", + "required": true + }, + { + "name": "plan_path", + "type": "file_path", + "required": true + }, + { + "name": "base_branch", + "type": "string", + "required": true + } + ], + "outputs": [], + "expected_output_patterns": [], + "pattern_examples": [], + "write_behavior": "always" + }, + "audit-impl": { + "inputs": [ + { + "name": "plan_path", + "type": "file_path", + "required": true + }, + { + "name": "branch_name", + "type": "string", + "required": false + }, + { + "name": "base_branch", + "type": "string", + "required": false + } + ], + "outputs": [ + { + "name": "verdict", + "type": "string" + }, + { + "name": "remediation_path", + "type": "file_path" + } + ], + "expected_output_patterns": [ + "verdict\\s*=\\s*(GO|NO GO)" + ], + "pattern_examples": [ + "verdict = GO", + "verdict = NO GO\nremediation_path = /tmp/remediation.md" + ] + }, + "open-integration-pr": { + "inputs": [ + { + "name": "batch_branch", + "type": "string", + "required": true + }, + { + "name": "base_branch", + "type": "string", + "required": true + }, + { + "name": "pr_order_file", + "type": "file_path", + "required": true + }, + { + "name": "audit_verdict", + "type": "string", + "required": false + }, + { + "name": "domain_partitions_path", + "type": "file_path", + "required": false + } + ], + "outputs": [ + { + "name": "pr_url", + "type": "string" + } + ], + "expected_output_patterns": [ + "https://github\\.com/.*/pull/\\d+" + ], + "pattern_examples": [ + "https://github.com/owner/repo/pull/42" + ] + }, + "review-pr": { + "inputs": [ + { + "name": "feature_branch", + "type": "string", + "required": true + }, + { + "name": "base_branch", + "type": "string", + "required": true + }, + { + "name": "annotated_diff_path", + "type": "string", + "required": false + }, + { + "name": "hunk_ranges_path", + "type": "string", + "required": false + } + ], + "outputs": [ + { + "name": "verdict", + "type": "string" + } + ], + "expected_output_patterns": [ + "verdict\\s*=\\s*(approved|changes_requested|needs_human)" + ], + "pattern_examples": [ + "verdict = approved", + "verdict = changes_requested", + "verdict = needs_human" + ] + }, + "resolve-review": { + "inputs": [ + { + "name": "feature_branch", + "type": "string", + "required": true + }, + { + "name": "base_branch", + "type": "string", + "required": true + } + ], + "outputs": [], + "expected_output_patterns": [], + "pattern_examples": [], + "write_behavior": "always" + }, + "diagnose-ci": { + "inputs": [ + { + "name": "branch", + "type": "string", + "required": true + }, + { + "name": "run_id", + "type": "string", + "required": false + }, + { + "name": "ci_failed_jobs", + "type": "string", + "required": false + } + ], + "outputs": [ + { + "name": "diagnosis_path", + "type": "file_path" + }, + { + "name": "failure_type", + "type": "string" + }, + { + "name": "is_fixable", + "type": "string" + } + ], + "expected_output_patterns": [ + "diagnosis_path\\s*=\\s*/.+" + ], + "pattern_examples": [ + "diagnosis_path = /tmp/diagnosis.md" + ], + "write_behavior": "always" + } + }, + "dataflow": [ + { + "step": "clone", + "available": [ + "audit", + "base_branch", + "plans_dir", + "run_name", + "source_dir" + ], + "required": [], + "produced": [ + "work_dir", + "remote_url" + ] + }, + { + "step": "setup_remote", + "available": [ + "audit", + "base_branch", + "plans_dir", + "remote_url", + "run_name", + "source_dir", + "work_dir" + ], + "required": [], + "produced": [] + }, + { + "step": "check_integration_exists", + "available": [ + "audit", + "base_branch", + "plans_dir", + "remote_url", + "run_name", + "source_dir", + "work_dir" + ], + "required": [], + "produced": [] + }, + { + "step": "confirm_create_integration", + "available": [ + "audit", + "base_branch", + "plans_dir", + "remote_url", + "run_name", + "source_dir", + "work_dir" + ], + "required": [], + "produced": [] + }, + { + "step": "create_persistent_integration", + "available": [ + "audit", + "base_branch", + "plans_dir", + "remote_url", + "run_name", + "source_dir", + "work_dir" + ], + "required": [], + "produced": [] + }, + { + "step": "fetch_merge_queue_data", + "available": [ + "audit", + "base_branch", + "plans_dir", + "remote_url", + "run_name", + "source_dir", + "work_dir" + ], + "required": [], + "produced": [ + "merge_queue_data_path" + ] + }, + { + "step": "analyze_prs", + "available": [ + "audit", + "base_branch", + "merge_queue_data_path", + "plans_dir", + "remote_url", + "run_name", + "source_dir", + "work_dir" + ], + "required": [], + "produced": [ + "pr_order_file", + "batch_branch", + "queue_mode" + ] + }, + { + "step": "route_by_queue_mode", + "available": [ + "audit", + "base_branch", + "batch_branch", + "merge_queue_data_path", + "plans_dir", + "pr_order_file", + "queue_mode", + "remote_url", + "run_name", + "source_dir", + "work_dir" + ], + "required": [], + "produced": [] + }, + { + "step": "get_first_pr_number", + "available": [ + "audit", + "base_branch", + "batch_branch", + "merge_queue_data_path", + "plans_dir", + "pr_order_file", + "queue_mode", + "remote_url", + "run_name", + "source_dir", + "work_dir" + ], + "required": [], + "produced": [ + "current_pr_number" + ] + }, + { + "step": "enqueue_current_pr", + "available": [ + "audit", + "base_branch", + "current_pr_number", + "batch_branch", + "merge_queue_data_path", + "plans_dir", + "pr_order_file", + "queue_mode", + "remote_url", + "run_name", + "source_dir", + "work_dir" + ], + "required": [], + "produced": [] + }, + { + "step": "wait_queue_pr", + "available": [ + "audit", + "base_branch", + "current_pr_number", + "batch_branch", + "merge_queue_data_path", + "plans_dir", + "pr_order_file", + "queue_mode", + "remote_url", + "run_name", + "source_dir", + "work_dir" + ], + "required": [], + "produced": [ + "queue_pr_state" + ] + }, + { + "step": "get_ejected_pr_branch", + "available": [ + "audit", + "base_branch", + "current_pr_number", + "batch_branch", + "merge_queue_data_path", + "plans_dir", + "pr_order_file", + "queue_mode", + "queue_pr_state", + "remote_url", + "run_name", + "source_dir", + "work_dir" + ], + "required": [], + "produced": [ + "ejected_pr_branch" + ] + }, + { + "step": "attempt_cheap_rebase", + "available": [ + "audit", + "base_branch", + "current_pr_number", + "ejected_pr_branch", + "batch_branch", + "merge_queue_data_path", + "plans_dir", + "pr_order_file", + "queue_mode", + "queue_pr_state", + "remote_url", + "run_name", + "source_dir", + "work_dir" + ], + "required": [], + "produced": [] + }, + { + "step": "resolve_ejected_conflicts", + "available": [ + "audit", + "base_branch", + "current_pr_number", + "ejected_pr_branch", + "batch_branch", + "merge_queue_data_path", + "plans_dir", + "pr_order_file", + "queue_mode", + "queue_pr_state", + "remote_url", + "run_name", + "source_dir", + "work_dir" + ], + "required": [], + "produced": [ + "conflict_escalation_required" + ] + }, + { + "step": "push_ejected_fix", + "available": [ + "audit", + "base_branch", + "conflict_escalation_required", + "current_pr_number", + "ejected_pr_branch", + "batch_branch", + "merge_queue_data_path", + "plans_dir", + "pr_order_file", + "queue_mode", + "queue_pr_state", + "remote_url", + "run_name", + "source_dir", + "work_dir" + ], + "required": [], + "produced": [] + }, + { + "step": "ci_watch_post_queue_fix", + "available": [ + "audit", + "base_branch", + "conflict_escalation_required", + "current_pr_number", + "ejected_pr_branch", + "batch_branch", + "merge_queue_data_path", + "plans_dir", + "pr_order_file", + "queue_mode", + "queue_pr_state", + "remote_url", + "run_name", + "source_dir", + "work_dir" + ], + "required": [], + "produced": [] + }, + { + "step": "reenter_queue", + "available": [ + "audit", + "base_branch", + "conflict_escalation_required", + "current_pr_number", + "ejected_pr_branch", + "batch_branch", + "merge_queue_data_path", + "plans_dir", + "pr_order_file", + "queue_mode", + "queue_pr_state", + "remote_url", + "run_name", + "source_dir", + "work_dir" + ], + "required": [], + "produced": [] + }, + { + "step": "advance_queue_pr", + "available": [ + "audit", + "base_branch", + "conflict_escalation_required", + "current_pr_number", + "ejected_pr_branch", + "batch_branch", + "merge_queue_data_path", + "plans_dir", + "pr_order_file", + "queue_mode", + "queue_pr_state", + "remote_url", + "run_name", + "source_dir", + "work_dir" + ], + "required": [], + "produced": [ + "current_pr_number" + ] + }, + { + "step": "get_next_pr_branch", + "available": [ + "audit", + "base_branch", + "conflict_escalation_required", + "current_pr_number", + "ejected_pr_branch", + "batch_branch", + "merge_queue_data_path", + "plans_dir", + "pr_order_file", + "queue_mode", + "queue_pr_state", + "remote_url", + "run_name", + "source_dir", + "work_dir" + ], + "required": [], + "produced": [ + "next_pr_branch" + ] + }, + { + "step": "proactive_rebase_next_pr", + "available": [ + "audit", + "base_branch", + "conflict_escalation_required", + "current_pr_number", + "ejected_pr_branch", + "batch_branch", + "merge_queue_data_path", + "next_pr_branch", + "plans_dir", + "pr_order_file", + "queue_mode", + "queue_pr_state", + "remote_url", + "run_name", + "source_dir", + "work_dir" + ], + "required": [], + "produced": [] + }, + { + "step": "resolve_proactive_rebase_conflicts", + "available": [ + "audit", + "base_branch", + "conflict_escalation_required", + "current_pr_number", + "ejected_pr_branch", + "batch_branch", + "merge_queue_data_path", + "next_pr_branch", + "plans_dir", + "pr_order_file", + "queue_mode", + "queue_pr_state", + "remote_url", + "run_name", + "source_dir", + "work_dir" + ], + "required": [], + "produced": [ + "conflict_escalation_reason" + ] + }, + { + "step": "push_rebased_next_pr", + "available": [ + "audit", + "base_branch", + "conflict_escalation_reason", + "conflict_escalation_required", + "current_pr_number", + "ejected_pr_branch", + "batch_branch", + "merge_queue_data_path", + "next_pr_branch", + "plans_dir", + "pr_order_file", + "queue_mode", + "queue_pr_state", + "remote_url", + "run_name", + "source_dir", + "work_dir" + ], + "required": [], + "produced": [] + }, + { + "step": "diagnose_queue_ci", + "available": [ + "audit", + "base_branch", + "conflict_escalation_required", + "current_pr_number", + "ejected_pr_branch", + "batch_branch", + "merge_queue_data_path", + "plans_dir", + "pr_order_file", + "queue_mode", + "queue_pr_state", + "remote_url", + "run_name", + "source_dir", + "work_dir" + ], + "required": [], + "produced": [ + "diagnosis_path" + ] + }, + { + "step": "reenroll_stalled_queue_pr", + "available": [ + "audit", + "base_branch", + "conflict_escalation_required", + "current_pr_number", + "ejected_pr_branch", + "batch_branch", + "merge_queue_data_path", + "plans_dir", + "pr_order_file", + "queue_mode", + "queue_pr_state", + "remote_url", + "run_name", + "source_dir", + "work_dir" + ], + "required": [], + "produced": [] + }, + { + "step": "create_batch_branch", + "available": [ + "audit", + "base_branch", + "conflict_escalation_required", + "current_pr_number", + "ejected_pr_branch", + "batch_branch", + "merge_queue_data_path", + "plans_dir", + "pr_order_file", + "queue_mode", + "queue_pr_state", + "remote_url", + "run_name", + "source_dir", + "work_dir" + ], + "required": [], + "produced": [] + }, + { + "step": "publish_batch_branch", + "available": [ + "audit", + "base_branch", + "conflict_escalation_required", + "current_pr_number", + "ejected_pr_branch", + "batch_branch", + "merge_queue_data_path", + "plans_dir", + "pr_order_file", + "queue_mode", + "queue_pr_state", + "remote_url", + "run_name", + "source_dir", + "work_dir" + ], + "required": [], + "produced": [] + }, + { + "step": "merge_pr", + "available": [ + "audit", + "base_branch", + "conflict_escalation_required", + "current_pr_number", + "ejected_pr_branch", + "batch_branch", + "merge_queue_data_path", + "plans_dir", + "pr_order_file", + "queue_mode", + "queue_pr_state", + "remote_url", + "run_name", + "source_dir", + "work_dir" + ], + "required": [], + "produced": [ + "needs_plan", + "escalation_required", + "escalation_reason", + "task", + "current_pr_number" + ] + }, + { + "step": "plan", + "available": [ + "audit", + "base_branch", + "conflict_escalation_required", + "current_pr_number", + "ejected_pr_branch", + "escalation_reason", + "escalation_required", + "batch_branch", + "merge_queue_data_path", + "needs_plan", + "plans_dir", + "pr_order_file", + "queue_mode", + "queue_pr_state", + "remote_url", + "run_name", + "source_dir", + "task", + "work_dir" + ], + "required": [], + "produced": [ + "pr_plan_path" + ] + }, + { + "step": "verify", + "available": [ + "audit", + "base_branch", + "conflict_escalation_required", + "current_pr_number", + "ejected_pr_branch", + "escalation_reason", + "escalation_required", + "batch_branch", + "merge_queue_data_path", + "needs_plan", + "plans_dir", + "pr_order_file", + "pr_plan_path", + "queue_mode", + "queue_pr_state", + "remote_url", + "run_name", + "source_dir", + "task", + "work_dir" + ], + "required": [ + "plan_path" + ], + "produced": [] + }, + { + "step": "implement", + "available": [ + "audit", + "base_branch", + "conflict_escalation_required", + "current_pr_number", + "ejected_pr_branch", + "escalation_reason", + "escalation_required", + "batch_branch", + "merge_queue_data_path", + "needs_plan", + "plans_dir", + "pr_order_file", + "pr_plan_path", + "queue_mode", + "queue_pr_state", + "remote_url", + "run_name", + "source_dir", + "task", + "work_dir" + ], + "required": [ + "plan_path" + ], + "produced": [ + "worktree_path", + "worktree_branch_name" + ] + }, + { + "step": "retry_worktree", + "available": [ + "audit", + "base_branch", + "conflict_escalation_required", + "current_pr_number", + "ejected_pr_branch", + "escalation_reason", + "escalation_required", + "batch_branch", + "merge_queue_data_path", + "needs_plan", + "plans_dir", + "pr_order_file", + "pr_plan_path", + "queue_mode", + "queue_pr_state", + "remote_url", + "run_name", + "source_dir", + "task", + "work_dir", + "worktree_branch_name", + "worktree_path" + ], + "required": [ + "plan_path" + ], + "produced": [ + "worktree_path", + "worktree_branch_name" + ] + }, + { + "step": "test", + "available": [ + "audit", + "base_branch", + "conflict_escalation_required", + "current_pr_number", + "ejected_pr_branch", + "escalation_reason", + "escalation_required", + "batch_branch", + "merge_queue_data_path", + "needs_plan", + "plans_dir", + "pr_order_file", + "pr_plan_path", + "queue_mode", + "queue_pr_state", + "remote_url", + "run_name", + "source_dir", + "task", + "work_dir", + "worktree_branch_name", + "worktree_path" + ], + "required": [], + "produced": [] + }, + { + "step": "push_worktree_branch", + "available": [ + "audit", + "base_branch", + "conflict_escalation_required", + "current_pr_number", + "ejected_pr_branch", + "escalation_reason", + "escalation_required", + "batch_branch", + "merge_queue_data_path", + "needs_plan", + "plans_dir", + "pr_order_file", + "pr_plan_path", + "queue_mode", + "queue_pr_state", + "remote_url", + "run_name", + "source_dir", + "task", + "work_dir", + "worktree_branch_name", + "worktree_path" + ], + "required": [], + "produced": [] + }, + { + "step": "create_conflict_pr", + "available": [ + "audit", + "base_branch", + "conflict_escalation_required", + "current_pr_number", + "ejected_pr_branch", + "escalation_reason", + "escalation_required", + "batch_branch", + "merge_queue_data_path", + "needs_plan", + "plans_dir", + "pr_order_file", + "pr_plan_path", + "queue_mode", + "queue_pr_state", + "remote_url", + "run_name", + "source_dir", + "task", + "work_dir", + "worktree_branch_name", + "worktree_path" + ], + "required": [], + "produced": [ + "conflict_pr_number" + ] + }, + { + "step": "wait_for_conflict_ci", + "available": [ + "audit", + "base_branch", + "conflict_escalation_required", + "conflict_pr_number", + "current_pr_number", + "ejected_pr_branch", + "escalation_reason", + "escalation_required", + "batch_branch", + "merge_queue_data_path", + "needs_plan", + "plans_dir", + "pr_order_file", + "pr_plan_path", + "queue_mode", + "queue_pr_state", + "remote_url", + "run_name", + "source_dir", + "task", + "work_dir", + "worktree_branch_name", + "worktree_path" + ], + "required": [], + "produced": [] + }, + { + "step": "merge_conflict_pr", + "available": [ + "audit", + "base_branch", + "conflict_escalation_required", + "conflict_pr_number", + "current_pr_number", + "ejected_pr_branch", + "escalation_reason", + "escalation_required", + "batch_branch", + "merge_queue_data_path", + "needs_plan", + "plans_dir", + "pr_order_file", + "pr_plan_path", + "queue_mode", + "queue_pr_state", + "remote_url", + "run_name", + "source_dir", + "task", + "work_dir", + "worktree_branch_name", + "worktree_path" + ], + "required": [], + "produced": [] + }, + { + "step": "fix", + "available": [ + "audit", + "base_branch", + "conflict_escalation_required", + "conflict_pr_number", + "current_pr_number", + "ejected_pr_branch", + "escalation_reason", + "escalation_required", + "batch_branch", + "merge_queue_data_path", + "needs_plan", + "plans_dir", + "pr_order_file", + "pr_plan_path", + "queue_mode", + "queue_pr_state", + "remote_url", + "run_name", + "source_dir", + "task", + "work_dir", + "worktree_branch_name", + "worktree_path" + ], + "required": [ + "plan_path" + ], + "produced": [] + }, + { + "step": "next_part_or_next_pr", + "available": [ + "audit", + "base_branch", + "conflict_escalation_required", + "conflict_pr_number", + "current_pr_number", + "ejected_pr_branch", + "escalation_reason", + "escalation_required", + "batch_branch", + "merge_queue_data_path", + "needs_plan", + "plans_dir", + "pr_order_file", + "pr_plan_path", + "queue_mode", + "queue_pr_state", + "remote_url", + "run_name", + "source_dir", + "task", + "work_dir", + "worktree_branch_name", + "worktree_path" + ], + "required": [], + "produced": [] + }, + { + "step": "push_batch_branch", + "available": [ + "audit", + "base_branch", + "conflict_escalation_required", + "conflict_pr_number", + "current_pr_number", + "ejected_pr_branch", + "escalation_reason", + "escalation_required", + "batch_branch", + "merge_queue_data_path", + "needs_plan", + "plans_dir", + "pr_order_file", + "pr_plan_path", + "queue_mode", + "queue_pr_state", + "remote_url", + "run_name", + "source_dir", + "task", + "work_dir", + "worktree_branch_name", + "worktree_path" + ], + "required": [], + "produced": [] + }, + { + "step": "collect_artifacts", + "available": [ + "audit", + "base_branch", + "conflict_escalation_required", + "conflict_pr_number", + "current_pr_number", + "ejected_pr_branch", + "escalation_reason", + "escalation_required", + "batch_branch", + "merge_queue_data_path", + "needs_plan", + "plans_dir", + "pr_order_file", + "pr_plan_path", + "queue_mode", + "queue_pr_state", + "remote_url", + "run_name", + "source_dir", + "task", + "work_dir", + "worktree_branch_name", + "worktree_path" + ], + "required": [], + "produced": [] + }, + { + "step": "check_impl_plans", + "available": [ + "audit", + "base_branch", + "conflict_escalation_required", + "conflict_pr_number", + "current_pr_number", + "ejected_pr_branch", + "escalation_reason", + "escalation_required", + "batch_branch", + "merge_queue_data_path", + "needs_plan", + "plans_dir", + "pr_order_file", + "pr_plan_path", + "queue_mode", + "queue_pr_state", + "remote_url", + "run_name", + "source_dir", + "task", + "work_dir", + "worktree_branch_name", + "worktree_path" + ], + "required": [], + "produced": [] + }, + { + "step": "audit_impl", + "available": [ + "audit", + "base_branch", + "conflict_escalation_required", + "conflict_pr_number", + "current_pr_number", + "ejected_pr_branch", + "escalation_reason", + "escalation_required", + "batch_branch", + "merge_queue_data_path", + "needs_plan", + "plans_dir", + "pr_order_file", + "pr_plan_path", + "queue_mode", + "queue_pr_state", + "remote_url", + "run_name", + "source_dir", + "task", + "work_dir", + "worktree_branch_name", + "worktree_path" + ], + "required": [], + "produced": [ + "verdict", + "remediation_path" + ] + }, + { + "step": "remediate", + "available": [ + "audit", + "base_branch", + "conflict_escalation_required", + "conflict_pr_number", + "current_pr_number", + "ejected_pr_branch", + "escalation_reason", + "escalation_required", + "batch_branch", + "merge_queue_data_path", + "needs_plan", + "plans_dir", + "pr_order_file", + "pr_plan_path", + "queue_mode", + "queue_pr_state", + "remediation_path", + "remote_url", + "run_name", + "source_dir", + "task", + "verdict", + "work_dir", + "worktree_branch_name", + "worktree_path" + ], + "required": [], + "produced": [] + }, + { + "step": "compute_domain_partitions", + "available": [ + "audit", + "base_branch", + "conflict_escalation_required", + "conflict_pr_number", + "current_pr_number", + "ejected_pr_branch", + "escalation_reason", + "escalation_required", + "batch_branch", + "merge_queue_data_path", + "needs_plan", + "plans_dir", + "pr_order_file", + "pr_plan_path", + "queue_mode", + "queue_pr_state", + "remediation_path", + "remote_url", + "run_name", + "source_dir", + "task", + "verdict", + "work_dir", + "worktree_branch_name", + "worktree_path" + ], + "required": [], + "produced": [ + "domain_partitions_path" + ] + }, + { + "step": "open_integration_pr", + "available": [ + "audit", + "base_branch", + "conflict_escalation_required", + "conflict_pr_number", + "current_pr_number", + "domain_partitions_path", + "ejected_pr_branch", + "escalation_reason", + "escalation_required", + "batch_branch", + "merge_queue_data_path", + "needs_plan", + "plans_dir", + "pr_order_file", + "pr_plan_path", + "queue_mode", + "queue_pr_state", + "remediation_path", + "remote_url", + "run_name", + "source_dir", + "task", + "verdict", + "work_dir", + "worktree_branch_name", + "worktree_path" + ], + "required": [], + "produced": [ + "pr_url" + ] + }, + { + "step": "wait_for_review_pr_mergeability", + "available": [ + "audit", + "base_branch", + "conflict_escalation_required", + "conflict_pr_number", + "current_pr_number", + "domain_partitions_path", + "ejected_pr_branch", + "escalation_reason", + "escalation_required", + "batch_branch", + "merge_queue_data_path", + "needs_plan", + "plans_dir", + "pr_order_file", + "pr_plan_path", + "pr_url", + "queue_mode", + "queue_pr_state", + "remediation_path", + "remote_url", + "run_name", + "source_dir", + "task", + "verdict", + "work_dir", + "worktree_branch_name", + "worktree_path" + ], + "required": [], + "produced": [ + "review_pr_number" + ] + }, + { + "step": "check_mergeability", + "available": [ + "audit", + "base_branch", + "conflict_escalation_required", + "conflict_pr_number", + "current_pr_number", + "domain_partitions_path", + "ejected_pr_branch", + "escalation_reason", + "escalation_required", + "batch_branch", + "merge_queue_data_path", + "needs_plan", + "plans_dir", + "pr_order_file", + "pr_plan_path", + "pr_url", + "queue_mode", + "queue_pr_state", + "remediation_path", + "remote_url", + "review_pr_number", + "run_name", + "source_dir", + "task", + "verdict", + "work_dir", + "worktree_branch_name", + "worktree_path" + ], + "required": [], + "produced": [] + }, + { + "step": "resolve_integration_conflicts", + "available": [ + "audit", + "base_branch", + "conflict_escalation_required", + "conflict_pr_number", + "current_pr_number", + "domain_partitions_path", + "ejected_pr_branch", + "escalation_reason", + "escalation_required", + "batch_branch", + "merge_queue_data_path", + "needs_plan", + "plans_dir", + "pr_order_file", + "pr_plan_path", + "pr_url", + "queue_mode", + "queue_pr_state", + "remediation_path", + "remote_url", + "review_pr_number", + "run_name", + "source_dir", + "task", + "verdict", + "work_dir", + "worktree_branch_name", + "worktree_path" + ], + "required": [ + "worktree_path", + "plan_path" + ], + "produced": [ + "conflict_escalation_required", + "conflict_escalation_reason" + ] + }, + { + "step": "force_push_after_rebase", + "available": [ + "audit", + "base_branch", + "conflict_escalation_reason", + "conflict_escalation_required", + "conflict_pr_number", + "current_pr_number", + "domain_partitions_path", + "ejected_pr_branch", + "escalation_reason", + "escalation_required", + "batch_branch", + "merge_queue_data_path", + "needs_plan", + "plans_dir", + "pr_order_file", + "pr_plan_path", + "pr_url", + "queue_mode", + "queue_pr_state", + "remediation_path", + "remote_url", + "review_pr_number", + "run_name", + "source_dir", + "task", + "verdict", + "work_dir", + "worktree_branch_name", + "worktree_path" + ], + "required": [], + "produced": [] + }, + { + "step": "wait_for_post_rebase_mergeability", + "available": [ + "audit", + "base_branch", + "conflict_escalation_reason", + "conflict_escalation_required", + "conflict_pr_number", + "current_pr_number", + "domain_partitions_path", + "ejected_pr_branch", + "escalation_reason", + "escalation_required", + "batch_branch", + "merge_queue_data_path", + "needs_plan", + "plans_dir", + "pr_order_file", + "pr_plan_path", + "pr_url", + "queue_mode", + "queue_pr_state", + "remediation_path", + "remote_url", + "review_pr_number", + "run_name", + "source_dir", + "task", + "verdict", + "work_dir", + "worktree_branch_name", + "worktree_path" + ], + "required": [], + "produced": [] + }, + { + "step": "check_mergeability_post_rebase", + "available": [ + "audit", + "base_branch", + "conflict_escalation_reason", + "conflict_escalation_required", + "conflict_pr_number", + "current_pr_number", + "domain_partitions_path", + "ejected_pr_branch", + "escalation_reason", + "escalation_required", + "batch_branch", + "merge_queue_data_path", + "needs_plan", + "plans_dir", + "pr_order_file", + "pr_plan_path", + "pr_url", + "queue_mode", + "queue_pr_state", + "remediation_path", + "remote_url", + "review_pr_number", + "run_name", + "source_dir", + "task", + "verdict", + "work_dir", + "worktree_branch_name", + "worktree_path" + ], + "required": [], + "produced": [] + }, + { + "step": "annotate_pr_diff", + "available": [ + "audit", + "base_branch", + "conflict_escalation_reason", + "conflict_escalation_required", + "conflict_pr_number", + "current_pr_number", + "domain_partitions_path", + "ejected_pr_branch", + "escalation_reason", + "escalation_required", + "batch_branch", + "merge_queue_data_path", + "needs_plan", + "plans_dir", + "pr_order_file", + "pr_plan_path", + "pr_url", + "queue_mode", + "queue_pr_state", + "remediation_path", + "remote_url", + "review_pr_number", + "run_name", + "source_dir", + "task", + "verdict", + "work_dir", + "worktree_branch_name", + "worktree_path" + ], + "required": [], + "produced": [ + "annotated_diff_path", + "diff_metrics_path", + "hunk_ranges_path", + "review_mode" + ] + }, + { + "step": "review_pr_integration", + "available": [ + "annotated_diff_path", + "audit", + "base_branch", + "conflict_escalation_reason", + "conflict_escalation_required", + "conflict_pr_number", + "current_pr_number", + "domain_partitions_path", + "ejected_pr_branch", + "escalation_reason", + "escalation_required", + "hunk_ranges_path", + "batch_branch", + "merge_queue_data_path", + "needs_plan", + "plans_dir", + "pr_order_file", + "pr_plan_path", + "pr_url", + "queue_mode", + "queue_pr_state", + "remediation_path", + "remote_url", + "review_pr_number", + "run_name", + "source_dir", + "task", + "verdict", + "work_dir", + "worktree_branch_name", + "worktree_path" + ], + "required": [], + "produced": [ + "review_verdict" + ] + }, + { + "step": "resolve_review_integration", + "available": [ + "annotated_diff_path", + "audit", + "base_branch", + "conflict_escalation_reason", + "conflict_escalation_required", + "conflict_pr_number", + "current_pr_number", + "domain_partitions_path", + "ejected_pr_branch", + "escalation_reason", + "escalation_required", + "hunk_ranges_path", + "batch_branch", + "merge_queue_data_path", + "needs_plan", + "plans_dir", + "pr_order_file", + "pr_plan_path", + "pr_url", + "queue_mode", + "queue_pr_state", + "remediation_path", + "remote_url", + "review_pr_number", + "review_verdict", + "run_name", + "source_dir", + "task", + "verdict", + "work_dir", + "worktree_branch_name", + "worktree_path" + ], + "required": [ + "feature_branch" + ], + "produced": [] + }, + { + "step": "re_push_review_integration", + "available": [ + "annotated_diff_path", + "audit", + "base_branch", + "conflict_escalation_reason", + "conflict_escalation_required", + "conflict_pr_number", + "current_pr_number", + "domain_partitions_path", + "ejected_pr_branch", + "escalation_reason", + "escalation_required", + "hunk_ranges_path", + "batch_branch", + "merge_queue_data_path", + "needs_plan", + "plans_dir", + "pr_order_file", + "pr_plan_path", + "pr_url", + "queue_mode", + "queue_pr_state", + "remediation_path", + "remote_url", + "review_pr_number", + "review_verdict", + "run_name", + "source_dir", + "task", + "verdict", + "work_dir", + "worktree_branch_name", + "worktree_path" + ], + "required": [], + "produced": [] + }, + { + "step": "ci_watch_pr", + "available": [ + "annotated_diff_path", + "audit", + "base_branch", + "conflict_escalation_reason", + "conflict_escalation_required", + "conflict_pr_number", + "current_pr_number", + "domain_partitions_path", + "ejected_pr_branch", + "escalation_reason", + "escalation_required", + "hunk_ranges_path", + "batch_branch", + "merge_queue_data_path", + "needs_plan", + "plans_dir", + "pr_order_file", + "pr_plan_path", + "pr_url", + "queue_mode", + "queue_pr_state", + "remediation_path", + "remote_url", + "review_pr_number", + "review_verdict", + "run_name", + "source_dir", + "task", + "verdict", + "work_dir", + "worktree_branch_name", + "worktree_path" + ], + "required": [], + "produced": [] + }, + { + "step": "diagnose_ci", + "available": [ + "annotated_diff_path", + "audit", + "base_branch", + "conflict_escalation_reason", + "conflict_escalation_required", + "conflict_pr_number", + "current_pr_number", + "domain_partitions_path", + "ejected_pr_branch", + "escalation_reason", + "escalation_required", + "hunk_ranges_path", + "batch_branch", + "merge_queue_data_path", + "needs_plan", + "plans_dir", + "pr_order_file", + "pr_plan_path", + "pr_url", + "queue_mode", + "queue_pr_state", + "remediation_path", + "remote_url", + "review_pr_number", + "review_verdict", + "run_name", + "source_dir", + "task", + "verdict", + "work_dir", + "worktree_branch_name", + "worktree_path" + ], + "required": [], + "produced": [ + "diagnosis_path" + ] + }, + { + "step": "register_clone_success", + "available": [ + "annotated_diff_path", + "audit", + "base_branch", + "conflict_escalation_reason", + "conflict_escalation_required", + "conflict_pr_number", + "current_pr_number", + "diagnosis_path", + "domain_partitions_path", + "ejected_pr_branch", + "escalation_reason", + "escalation_required", + "hunk_ranges_path", + "batch_branch", + "merge_queue_data_path", + "needs_plan", + "plans_dir", + "pr_order_file", + "pr_plan_path", + "pr_url", + "queue_mode", + "queue_pr_state", + "remediation_path", + "remote_url", + "review_pr_number", + "review_verdict", + "run_name", + "source_dir", + "task", + "verdict", + "work_dir", + "worktree_branch_name", + "worktree_path" + ], + "required": [], + "produced": [] + }, + { + "step": "register_clone_failure", + "available": [ + "annotated_diff_path", + "audit", + "base_branch", + "conflict_escalation_reason", + "conflict_escalation_required", + "conflict_pr_number", + "current_pr_number", + "diagnosis_path", + "domain_partitions_path", + "ejected_pr_branch", + "escalation_reason", + "escalation_required", + "hunk_ranges_path", + "batch_branch", + "merge_queue_data_path", + "needs_plan", + "plans_dir", + "pr_order_file", + "pr_plan_path", + "pr_url", + "queue_mode", + "queue_pr_state", + "remediation_path", + "remote_url", + "review_pr_number", + "review_verdict", + "run_name", + "source_dir", + "task", + "verdict", + "work_dir", + "worktree_branch_name", + "worktree_path" + ], + "required": [], + "produced": [] + }, + { + "step": "done", + "available": [ + "annotated_diff_path", + "audit", + "base_branch", + "conflict_escalation_reason", + "conflict_escalation_required", + "conflict_pr_number", + "current_pr_number", + "diagnosis_path", + "domain_partitions_path", + "ejected_pr_branch", + "escalation_reason", + "escalation_required", + "hunk_ranges_path", + "batch_branch", + "merge_queue_data_path", + "needs_plan", + "plans_dir", + "pr_order_file", + "pr_plan_path", + "pr_url", + "queue_mode", + "queue_pr_state", + "remediation_path", + "remote_url", + "review_pr_number", + "review_verdict", + "run_name", + "source_dir", + "task", + "verdict", + "work_dir", + "worktree_branch_name", + "worktree_path" + ], + "required": [], + "produced": [] + }, + { + "step": "escalate_stop", + "available": [ + "annotated_diff_path", + "audit", + "base_branch", + "conflict_escalation_reason", + "conflict_escalation_required", + "conflict_pr_number", + "current_pr_number", + "diagnosis_path", + "domain_partitions_path", + "ejected_pr_branch", + "escalation_reason", + "escalation_required", + "hunk_ranges_path", + "batch_branch", + "merge_queue_data_path", + "needs_plan", + "plans_dir", + "pr_order_file", + "pr_plan_path", + "pr_url", + "queue_mode", + "queue_pr_state", + "remediation_path", + "remote_url", + "review_pr_number", + "review_verdict", + "run_name", + "source_dir", + "task", + "verdict", + "work_dir", + "worktree_branch_name", + "worktree_path" + ], + "required": [], + "produced": [] + } + ] +} diff --git a/src/autoskillit/recipes/contracts/merge-prs.yaml b/src/autoskillit/recipes/contracts/merge-prs.yaml index 479d954d9e..4073f5911f 100644 --- a/src/autoskillit/recipes/contracts/merge-prs.yaml +++ b/src/autoskillit/recipes/contracts/merge-prs.yaml @@ -1,4 +1,4 @@ -generated_at: '2026-04-05T02:19:56.074977+00:00' +generated_at: '2026-04-25T21:57:00.000000+00:00' bundled_manifest_version: 0.1.0 skill_hashes: {} skills: @@ -15,7 +15,7 @@ skills: type: file_path - name: analysis_file type: file_path - - name: integration_branch + - name: batch_branch type: string - name: pr_count type: string @@ -69,6 +69,9 @@ skills: outputs: - name: merged type: string + allowed_values: + - "true" + - "false" - name: needs_plan type: string - name: escalation_required @@ -185,7 +188,7 @@ skills: remediation_path = /tmp/remediation.md' open-integration-pr: inputs: - - name: integration_branch + - name: batch_branch type: string required: true - name: base_branch @@ -346,13 +349,13 @@ dataflow: required: [] produced: - pr_order_file - - integration_branch + - batch_branch - queue_mode - step: route_by_queue_mode available: - audit - base_branch - - integration_branch + - batch_branch - merge_queue_data_path - plans_dir - pr_order_file @@ -363,11 +366,11 @@ dataflow: - work_dir required: [] produced: [] -- step: enqueue_all_prs +- step: get_first_pr_number available: - audit - base_branch - - integration_branch + - batch_branch - merge_queue_data_path - plans_dir - pr_order_file @@ -377,12 +380,14 @@ dataflow: - source_dir - work_dir required: [] - produced: [] -- step: get_first_pr_number + produced: + - current_pr_number +- step: enqueue_current_pr available: - audit - base_branch - - integration_branch + - current_pr_number + - batch_branch - merge_queue_data_path - plans_dir - pr_order_file @@ -392,14 +397,13 @@ dataflow: - source_dir - work_dir required: [] - produced: - - current_pr_number + produced: [] - step: wait_queue_pr available: - audit - base_branch - current_pr_number - - integration_branch + - batch_branch - merge_queue_data_path - plans_dir - pr_order_file @@ -416,7 +420,7 @@ dataflow: - audit - base_branch - current_pr_number - - integration_branch + - batch_branch - merge_queue_data_path - plans_dir - pr_order_file @@ -429,13 +433,13 @@ dataflow: required: [] produced: - ejected_pr_branch -- step: checkout_ejected_pr +- step: attempt_cheap_rebase available: - audit - base_branch - current_pr_number - ejected_pr_branch - - integration_branch + - batch_branch - merge_queue_data_path - plans_dir - pr_order_file @@ -453,7 +457,7 @@ dataflow: - base_branch - current_pr_number - ejected_pr_branch - - integration_branch + - batch_branch - merge_queue_data_path - plans_dir - pr_order_file @@ -473,7 +477,26 @@ dataflow: - conflict_escalation_required - current_pr_number - ejected_pr_branch - - integration_branch + - batch_branch + - merge_queue_data_path + - plans_dir + - pr_order_file + - queue_mode + - queue_pr_state + - remote_url + - run_name + - source_dir + - work_dir + required: [] + produced: [] +- step: ci_watch_post_queue_fix + available: + - audit + - base_branch + - conflict_escalation_required + - current_pr_number + - ejected_pr_branch + - batch_branch - merge_queue_data_path - plans_dir - pr_order_file @@ -492,8 +515,110 @@ dataflow: - conflict_escalation_required - current_pr_number - ejected_pr_branch - - integration_branch + - batch_branch + - merge_queue_data_path + - plans_dir + - pr_order_file + - queue_mode + - queue_pr_state + - remote_url + - run_name + - source_dir + - work_dir + required: [] + produced: [] +- step: advance_queue_pr + available: + - audit + - base_branch + - conflict_escalation_required + - current_pr_number + - ejected_pr_branch + - batch_branch + - merge_queue_data_path + - plans_dir + - pr_order_file + - queue_mode + - queue_pr_state + - remote_url + - run_name + - source_dir + - work_dir + required: [] + produced: + - current_pr_number +- step: get_next_pr_branch + available: + - audit + - base_branch + - conflict_escalation_required + - current_pr_number + - ejected_pr_branch + - batch_branch + - merge_queue_data_path + - plans_dir + - pr_order_file + - queue_mode + - queue_pr_state + - remote_url + - run_name + - source_dir + - work_dir + required: [] + produced: + - next_pr_branch +- step: proactive_rebase_next_pr + available: + - audit + - base_branch + - conflict_escalation_required + - current_pr_number + - ejected_pr_branch + - batch_branch + - merge_queue_data_path + - next_pr_branch + - plans_dir + - pr_order_file + - queue_mode + - queue_pr_state + - remote_url + - run_name + - source_dir + - work_dir + required: [] + produced: [] +- step: resolve_proactive_rebase_conflicts + available: + - audit + - base_branch + - conflict_escalation_required + - current_pr_number + - ejected_pr_branch + - batch_branch + - merge_queue_data_path + - next_pr_branch + - plans_dir + - pr_order_file + - queue_mode + - queue_pr_state + - remote_url + - run_name + - source_dir + - work_dir + required: [] + produced: + - conflict_escalation_reason +- step: push_rebased_next_pr + available: + - audit + - base_branch + - conflict_escalation_reason + - conflict_escalation_required + - current_pr_number + - ejected_pr_branch + - batch_branch - merge_queue_data_path + - next_pr_branch - plans_dir - pr_order_file - queue_mode @@ -504,14 +629,34 @@ dataflow: - work_dir required: [] produced: [] -- step: next_queue_pr_or_done +- step: diagnose_queue_ci + available: + - audit + - base_branch + - conflict_escalation_required + - current_pr_number + - ejected_pr_branch + - batch_branch + - merge_queue_data_path + - plans_dir + - pr_order_file + - queue_mode + - queue_pr_state + - remote_url + - run_name + - source_dir + - work_dir + required: [] + produced: + - diagnosis_path +- step: reenroll_stalled_queue_pr available: - audit - base_branch - conflict_escalation_required - current_pr_number - ejected_pr_branch - - integration_branch + - batch_branch - merge_queue_data_path - plans_dir - pr_order_file @@ -523,14 +668,14 @@ dataflow: - work_dir required: [] produced: [] -- step: create_integration_branch +- step: create_batch_branch available: - audit - base_branch - conflict_escalation_required - current_pr_number - ejected_pr_branch - - integration_branch + - batch_branch - merge_queue_data_path - plans_dir - pr_order_file @@ -542,14 +687,14 @@ dataflow: - work_dir required: [] produced: [] -- step: publish_integration_branch +- step: publish_batch_branch available: - audit - base_branch - conflict_escalation_required - current_pr_number - ejected_pr_branch - - integration_branch + - batch_branch - merge_queue_data_path - plans_dir - pr_order_file @@ -568,7 +713,7 @@ dataflow: - conflict_escalation_required - current_pr_number - ejected_pr_branch - - integration_branch + - batch_branch - merge_queue_data_path - plans_dir - pr_order_file @@ -594,7 +739,7 @@ dataflow: - ejected_pr_branch - escalation_reason - escalation_required - - integration_branch + - batch_branch - merge_queue_data_path - needs_plan - plans_dir @@ -618,7 +763,7 @@ dataflow: - ejected_pr_branch - escalation_reason - escalation_required - - integration_branch + - batch_branch - merge_queue_data_path - needs_plan - plans_dir @@ -643,7 +788,7 @@ dataflow: - ejected_pr_branch - escalation_reason - escalation_required - - integration_branch + - batch_branch - merge_queue_data_path - needs_plan - plans_dir @@ -670,7 +815,7 @@ dataflow: - ejected_pr_branch - escalation_reason - escalation_required - - integration_branch + - batch_branch - merge_queue_data_path - needs_plan - plans_dir @@ -699,7 +844,7 @@ dataflow: - ejected_pr_branch - escalation_reason - escalation_required - - integration_branch + - batch_branch - merge_queue_data_path - needs_plan - plans_dir @@ -725,7 +870,7 @@ dataflow: - ejected_pr_branch - escalation_reason - escalation_required - - integration_branch + - batch_branch - merge_queue_data_path - needs_plan - plans_dir @@ -751,7 +896,7 @@ dataflow: - ejected_pr_branch - escalation_reason - escalation_required - - integration_branch + - batch_branch - merge_queue_data_path - needs_plan - plans_dir @@ -779,7 +924,7 @@ dataflow: - ejected_pr_branch - escalation_reason - escalation_required - - integration_branch + - batch_branch - merge_queue_data_path - needs_plan - plans_dir @@ -806,7 +951,7 @@ dataflow: - ejected_pr_branch - escalation_reason - escalation_required - - integration_branch + - batch_branch - merge_queue_data_path - needs_plan - plans_dir @@ -833,7 +978,7 @@ dataflow: - ejected_pr_branch - escalation_reason - escalation_required - - integration_branch + - batch_branch - merge_queue_data_path - needs_plan - plans_dir @@ -861,7 +1006,7 @@ dataflow: - ejected_pr_branch - escalation_reason - escalation_required - - integration_branch + - batch_branch - merge_queue_data_path - needs_plan - plans_dir @@ -878,7 +1023,7 @@ dataflow: - worktree_path required: [] produced: [] -- step: push_integration_branch +- step: push_batch_branch available: - audit - base_branch @@ -888,7 +1033,7 @@ dataflow: - ejected_pr_branch - escalation_reason - escalation_required - - integration_branch + - batch_branch - merge_queue_data_path - needs_plan - plans_dir @@ -915,7 +1060,7 @@ dataflow: - ejected_pr_branch - escalation_reason - escalation_required - - integration_branch + - batch_branch - merge_queue_data_path - needs_plan - plans_dir @@ -942,7 +1087,7 @@ dataflow: - ejected_pr_branch - escalation_reason - escalation_required - - integration_branch + - batch_branch - merge_queue_data_path - needs_plan - plans_dir @@ -969,7 +1114,7 @@ dataflow: - ejected_pr_branch - escalation_reason - escalation_required - - integration_branch + - batch_branch - merge_queue_data_path - needs_plan - plans_dir @@ -998,7 +1143,7 @@ dataflow: - ejected_pr_branch - escalation_reason - escalation_required - - integration_branch + - batch_branch - merge_queue_data_path - needs_plan - plans_dir @@ -1027,7 +1172,7 @@ dataflow: - ejected_pr_branch - escalation_reason - escalation_required - - integration_branch + - batch_branch - merge_queue_data_path - needs_plan - plans_dir @@ -1058,7 +1203,7 @@ dataflow: - ejected_pr_branch - escalation_reason - escalation_required - - integration_branch + - batch_branch - merge_queue_data_path - needs_plan - plans_dir @@ -1089,7 +1234,7 @@ dataflow: - ejected_pr_branch - escalation_reason - escalation_required - - integration_branch + - batch_branch - merge_queue_data_path - needs_plan - plans_dir @@ -1121,7 +1266,7 @@ dataflow: - ejected_pr_branch - escalation_reason - escalation_required - - integration_branch + - batch_branch - merge_queue_data_path - needs_plan - plans_dir @@ -1153,7 +1298,7 @@ dataflow: - ejected_pr_branch - escalation_reason - escalation_required - - integration_branch + - batch_branch - merge_queue_data_path - needs_plan - plans_dir @@ -1190,7 +1335,7 @@ dataflow: - ejected_pr_branch - escalation_reason - escalation_required - - integration_branch + - batch_branch - merge_queue_data_path - needs_plan - plans_dir @@ -1223,7 +1368,7 @@ dataflow: - ejected_pr_branch - escalation_reason - escalation_required - - integration_branch + - batch_branch - merge_queue_data_path - needs_plan - plans_dir @@ -1256,7 +1401,7 @@ dataflow: - ejected_pr_branch - escalation_reason - escalation_required - - integration_branch + - batch_branch - merge_queue_data_path - needs_plan - plans_dir @@ -1289,7 +1434,7 @@ dataflow: - ejected_pr_branch - escalation_reason - escalation_required - - integration_branch + - batch_branch - merge_queue_data_path - needs_plan - plans_dir @@ -1311,7 +1456,9 @@ dataflow: required: [] produced: - annotated_diff_path + - diff_metrics_path - hunk_ranges_path + - review_mode - step: review_pr_integration available: - annotated_diff_path @@ -1326,7 +1473,7 @@ dataflow: - escalation_reason - escalation_required - hunk_ranges_path - - integration_branch + - batch_branch - merge_queue_data_path - needs_plan - plans_dir @@ -1362,7 +1509,7 @@ dataflow: - escalation_reason - escalation_required - hunk_ranges_path - - integration_branch + - batch_branch - merge_queue_data_path - needs_plan - plans_dir @@ -1399,7 +1546,7 @@ dataflow: - escalation_reason - escalation_required - hunk_ranges_path - - integration_branch + - batch_branch - merge_queue_data_path - needs_plan - plans_dir @@ -1435,7 +1582,7 @@ dataflow: - escalation_reason - escalation_required - hunk_ranges_path - - integration_branch + - batch_branch - merge_queue_data_path - needs_plan - plans_dir @@ -1471,7 +1618,7 @@ dataflow: - escalation_reason - escalation_required - hunk_ranges_path - - integration_branch + - batch_branch - merge_queue_data_path - needs_plan - plans_dir @@ -1509,7 +1656,7 @@ dataflow: - escalation_reason - escalation_required - hunk_ranges_path - - integration_branch + - batch_branch - merge_queue_data_path - needs_plan - plans_dir @@ -1546,7 +1693,7 @@ dataflow: - escalation_reason - escalation_required - hunk_ranges_path - - integration_branch + - batch_branch - merge_queue_data_path - needs_plan - plans_dir @@ -1583,7 +1730,7 @@ dataflow: - escalation_reason - escalation_required - hunk_ranges_path - - integration_branch + - batch_branch - merge_queue_data_path - needs_plan - plans_dir @@ -1620,7 +1767,7 @@ dataflow: - escalation_reason - escalation_required - hunk_ranges_path - - integration_branch + - batch_branch - merge_queue_data_path - needs_plan - plans_dir diff --git a/src/autoskillit/recipes/contracts/planner.json b/src/autoskillit/recipes/contracts/planner.json new file mode 100644 index 0000000000..e05b5f0928 --- /dev/null +++ b/src/autoskillit/recipes/contracts/planner.json @@ -0,0 +1,1008 @@ +{ + "generated_at": "2026-05-03T01:40:00.180863+00:00", + "bundled_manifest_version": "0.1.0", + "skill_hashes": { + "planner-analyze": "sha256:bc64ba4984d3c1b32231c062e46550bc5e00d94689d5a8967b3f222abfe0e172", + "planner-extract-domain": "sha256:dcb246a33465b839a552719dff34684ea9421ad99e021232ae677448a69eff45", + "planner-generate-phases": "sha256:0904a31c377aee72e98e937915f8397e42b02dc7fab5572cb156e144bf613817", + "planner-elaborate-phase": "sha256:d0127bd85e030eecaabb727730ccb8d80fc05cb55227975320393ba17fcb2238", + "planner-refine-phases": "sha256:3a514d6a658cb3bcee2de18f0f3aea2f529c01a24268b7f6007ec391ba8fe91c", + "planner-elaborate-assignments": "sha256:1545a91c18d14d00031077833d3f37af1be7b0287e8239d469df3388496c649e", + "planner-refine-assignments": "sha256:584c1715729a91928994d4d18b91a8e48ca03ee94ce70dd1c613c7a5159e6318", + "planner-elaborate-wps": "sha256:301bebf89b29e658de254d5790d50878fde417a3359222c5b282065f13aa2b30", + "planner-refine-wps": "sha256:17d44c80abb78698c6173c74f6259df65ac09cd87063cfdf3e0e1de763043334", + "planner-consolidate-wps": "sha256:384ae024bfc9ef67fd25d4a929badcd5790577364a768d2f655b5172bc13a8e7", + "planner-validate-task-alignment": "sha256:f57d27bbf345f2da0c7fe4d8918f63e02b12882fd5fe1f7bb43e6c096f340b2b", + "planner-reconcile-deps": "sha256:4ea197d4d0c25af5b734f36c79ddb182bc140d98cdfe811f515ff182961ac9bb", + "planner-assess-review-approach": "sha256:4c3dfa00349ab2bec06c089d282f00d3e127e1c794b41daa6a71779612ab3558", + "planner-refine": "sha256:31e088ee392a4bee262818c5117eea98c57d910a6aec1363dc1a0c9e9f710e1d" + }, + "skills": { + "planner-analyze": { + "inputs": [], + "outputs": [], + "expected_output_patterns": [], + "pattern_examples": [] + }, + "planner-extract-domain": { + "inputs": [], + "outputs": [], + "expected_output_patterns": [], + "pattern_examples": [] + }, + "planner-generate-phases": { + "inputs": [], + "outputs": [ + { + "name": "phase_manifest_path", + "type": "file_path" + }, + { + "name": "phase_ids", + "type": "str" + } + ], + "expected_output_patterns": [ + "phase_manifest_path\\s*=\\s*\\S+" + ], + "pattern_examples": [ + "phase_manifest_path = /tmp/autoskillit/planner/run-20260101-120000/phases/phase_manifest.json\n%%ORDER_UP%%" + ], + "write_behavior": "always" + }, + "planner-elaborate-phase": { + "inputs": [ + { + "name": "plan_snapshot_path", + "type": "file_path", + "required": true, + "recommended": false + }, + { + "name": "target_phase_id", + "type": "str", + "required": true, + "recommended": false + }, + { + "name": "output_dir", + "type": "directory_path", + "required": true, + "recommended": false + } + ], + "outputs": [ + { + "name": "elab_result_path", + "type": "file_path" + } + ], + "expected_output_patterns": [ + "elab_result_path\\s*=\\s*\\S+" + ], + "pattern_examples": [ + "elab_result_path = /tmp/autoskillit/planner/run-20260101-120000/phases/P1_result.json\n%%ORDER_UP%%" + ], + "write_behavior": "always" + }, + "planner-refine-phases": { + "inputs": [ + { + "name": "combined_plan_path", + "type": "file_path", + "required": true, + "recommended": false + }, + { + "name": "output_dir", + "type": "directory_path", + "required": true, + "recommended": false + } + ], + "outputs": [ + { + "name": "refined_plan_path", + "type": "file_path" + } + ], + "expected_output_patterns": [ + "refined_plan_path\\s*=\\s*\\S+" + ], + "pattern_examples": [ + "refined_plan_path = /tmp/autoskillit/planner/run-20260101-120000/refined_plan.json\n%%ORDER_UP%%" + ], + "write_behavior": "always" + }, + "planner-elaborate-assignments": { + "inputs": [ + { + "name": "context_file", + "type": "file_path", + "required": true, + "recommended": false + }, + { + "name": "planner_dir", + "type": "directory_path", + "required": true, + "recommended": false + } + ], + "outputs": [ + { + "name": "phase_assignments_result_dir", + "type": "directory_path" + } + ], + "expected_output_patterns": [ + "phase_assignments_result_dir\\s*=\\s*\\S+" + ], + "pattern_examples": [ + "phase_assignments_result_dir = /tmp/autoskillit/planner/run-20260101-120000/assignments\n%%ORDER_UP%%" + ], + "write_behavior": "always" + }, + "planner-refine-assignments": { + "inputs": [ + { + "name": "context_file", + "type": "file_path", + "required": true, + "recommended": false + }, + { + "name": "refined_plan_path", + "type": "file_path", + "required": true, + "recommended": false + }, + { + "name": "output_dir", + "type": "directory_path", + "required": true, + "recommended": false + } + ], + "outputs": [ + { + "name": "phase_refined_path", + "type": "file_path" + } + ], + "expected_output_patterns": [ + "phase_refined_path\\s*=\\s*\\S+" + ], + "pattern_examples": [ + "phase_refined_path = /tmp/autoskillit/planner/run-20260101-120000/refine_contexts/P1_result.json\n%%ORDER_UP%%" + ], + "write_behavior": "always" + }, + "planner-elaborate-wps": { + "inputs": [ + { + "name": "context_file", + "type": "file_path", + "required": true, + "recommended": false + }, + { + "name": "planner_dir", + "type": "directory_path", + "required": true, + "recommended": false + } + ], + "outputs": [ + { + "name": "phase_wps_result_dir", + "type": "directory_path" + } + ], + "expected_output_patterns": [ + "phase_wps_result_dir\\s*=\\s*\\S+" + ], + "pattern_examples": [ + "phase_wps_result_dir = /tmp/autoskillit/planner/run-20260101-120000/work_packages\n%%ORDER_UP%%" + ], + "write_behavior": "always" + }, + "planner-refine-wps": { + "inputs": [ + { + "name": "combined_wps_path", + "type": "file_path", + "required": true, + "recommended": false + }, + { + "name": "refined_plan_path", + "type": "file_path", + "required": true, + "recommended": false + }, + { + "name": "refined_assignments_path", + "type": "file_path", + "required": true, + "recommended": false + }, + { + "name": "output_dir", + "type": "directory_path", + "required": true, + "recommended": false + } + ], + "outputs": [ + { + "name": "refined_wps_path", + "type": "file_path" + } + ], + "expected_output_patterns": [ + "refined_wps_path\\s*=\\s*\\S+" + ], + "pattern_examples": [ + "refined_wps_path = /tmp/autoskillit/planner/run-20260101-120000/refined_wps.json\n%%ORDER_UP%%" + ], + "write_behavior": "always" + }, + "planner-consolidate-wps": { + "inputs": [ + { + "name": "refined_wps_path", + "type": "file_path", + "required": true, + "recommended": false + }, + { + "name": "planner_dir", + "type": "directory_path", + "required": true, + "recommended": false + } + ], + "outputs": [], + "expected_output_patterns": [], + "pattern_examples": [], + "write_behavior": "always" + }, + "planner-validate-task-alignment": { + "inputs": [ + { + "name": "refined_wps_path", + "type": "file_path", + "required": true, + "recommended": false + }, + { + "name": "refined_plan_path", + "type": "file_path", + "required": true, + "recommended": false + }, + { + "name": "output_dir", + "type": "directory_path", + "required": true, + "recommended": false + } + ], + "outputs": [ + { + "name": "alignment_findings_path", + "type": "file_path" + }, + { + "name": "alignment_finding_count", + "type": "str" + } + ], + "expected_output_patterns": [ + "alignment_findings_path\\s*=\\s*\\S+", + "alignment_finding_count\\s*=\\s*\\d+" + ], + "pattern_examples": [ + "alignment_findings_path = /tmp/autoskillit/planner/run-20260101-120000/task_alignment.json\nalignment_finding_count = 0\n%%ORDER_UP%%" + ], + "write_behavior": "always" + }, + "planner-reconcile-deps": { + "inputs": [], + "outputs": [], + "expected_output_patterns": [], + "pattern_examples": [] + }, + "planner-assess-review-approach": { + "inputs": [ + { + "name": "refined_wps_path", + "type": "file_path", + "required": true, + "recommended": false + }, + { + "name": "planner_dir", + "type": "directory_path", + "required": true, + "recommended": false + } + ], + "outputs": [ + { + "name": "review_approach_assessment_path", + "type": "file_path" + } + ], + "expected_output_patterns": [ + "review_approach_assessment_path\\s*=\\s*\\S+" + ], + "pattern_examples": [ + "review_approach_assessment_path = /tmp/autoskillit/planner/run-20260101-120000/review_approach_assessment.json\n%%ORDER_UP%%" + ], + "write_behavior": "always" + }, + "planner-refine": { + "inputs": [], + "outputs": [], + "expected_output_patterns": [], + "pattern_examples": [] + } + }, + "dataflow": [ + { + "step": "init", + "available": [ + "planner_assess_review_approach", + "source_dir", + "task" + ], + "required": [], + "produced": [ + "planner_dir" + ] + }, + { + "step": "resolve_task", + "available": [ + "planner_assess_review_approach", + "planner_dir", + "source_dir", + "task" + ], + "required": [], + "produced": [ + "task_file_path", + "task_label" + ] + }, + { + "step": "analyze", + "available": [ + "planner_assess_review_approach", + "planner_dir", + "source_dir", + "task", + "task_file_path", + "task_label" + ], + "required": [], + "produced": [] + }, + { + "step": "extract_domain", + "available": [ + "planner_assess_review_approach", + "planner_dir", + "source_dir", + "task", + "task_file_path", + "task_label" + ], + "required": [], + "produced": [], + "positional_args": 2 + }, + { + "step": "generate_phases", + "available": [ + "planner_assess_review_approach", + "planner_dir", + "source_dir", + "task", + "task_file_path", + "task_label" + ], + "required": [], + "produced": [ + "phase_manifest_path" + ], + "positional_args": 3 + }, + { + "step": "build_plan_snapshot", + "available": [ + "phase_manifest_path", + "planner_assess_review_approach", + "planner_dir", + "source_dir", + "task", + "task_file_path", + "task_label" + ], + "required": [], + "produced": [ + "plan_snapshot_path", + "phase_ids" + ] + }, + { + "step": "elaborate_phases", + "available": [ + "phase_ids", + "phase_manifest_path", + "plan_snapshot_path", + "planner_assess_review_approach", + "planner_dir", + "source_dir", + "task", + "task_file_path", + "task_label" + ], + "required": [], + "produced": [], + "positional_args": 2 + }, + { + "step": "merge_phases", + "available": [ + "phase_ids", + "phase_manifest_path", + "plan_snapshot_path", + "planner_assess_review_approach", + "planner_dir", + "source_dir", + "task", + "task_file_path", + "task_label" + ], + "required": [], + "produced": [ + "combined_phases_path" + ] + }, + { + "step": "refine_phases", + "available": [ + "combined_phases_path", + "phase_ids", + "phase_manifest_path", + "plan_snapshot_path", + "planner_assess_review_approach", + "planner_dir", + "source_dir", + "task", + "task_file_path", + "task_label" + ], + "required": [ + "combined_plan_path", + "output_dir" + ], + "produced": [ + "refined_plan_path" + ] + }, + { + "step": "expand_assignments", + "available": [ + "combined_phases_path", + "phase_ids", + "phase_manifest_path", + "plan_snapshot_path", + "planner_assess_review_approach", + "planner_dir", + "refined_plan_path", + "source_dir", + "task", + "task_file_path", + "task_label" + ], + "required": [], + "produced": [ + "assignment_context_paths" + ] + }, + { + "step": "elaborate_assignments", + "available": [ + "assignment_context_paths", + "combined_phases_path", + "phase_ids", + "phase_manifest_path", + "plan_snapshot_path", + "planner_assess_review_approach", + "planner_dir", + "refined_plan_path", + "source_dir", + "task", + "task_file_path", + "task_label" + ], + "required": [], + "produced": [], + "positional_args": 1 + }, + { + "step": "merge_assignments", + "available": [ + "assignment_context_paths", + "combined_phases_path", + "phase_ids", + "phase_manifest_path", + "plan_snapshot_path", + "planner_assess_review_approach", + "planner_dir", + "refined_plan_path", + "source_dir", + "task", + "task_file_path", + "task_label" + ], + "required": [], + "produced": [ + "combined_assignments_path" + ] + }, + { + "step": "refine_assignments", + "available": [ + "assignment_context_paths", + "combined_assignments_path", + "combined_phases_path", + "phase_ids", + "phase_manifest_path", + "plan_snapshot_path", + "planner_assess_review_approach", + "planner_dir", + "refined_plan_path", + "source_dir", + "task", + "task_file_path", + "task_label" + ], + "required": [ + "output_dir" + ], + "produced": [ + "refined_assignments_path" + ] + }, + { + "step": "expand_wps", + "available": [ + "assignment_context_paths", + "combined_assignments_path", + "combined_phases_path", + "phase_ids", + "phase_manifest_path", + "plan_snapshot_path", + "planner_assess_review_approach", + "planner_dir", + "refined_assignments_path", + "refined_plan_path", + "source_dir", + "task", + "task_file_path", + "task_label" + ], + "required": [], + "produced": [ + "wp_context_paths" + ] + }, + { + "step": "elaborate_wps", + "available": [ + "assignment_context_paths", + "combined_assignments_path", + "combined_phases_path", + "phase_ids", + "phase_manifest_path", + "plan_snapshot_path", + "planner_assess_review_approach", + "planner_dir", + "refined_assignments_path", + "refined_plan_path", + "source_dir", + "task", + "task_file_path", + "task_label", + "wp_context_paths" + ], + "required": [], + "produced": [], + "positional_args": 1 + }, + { + "step": "finalize_wp_manifest", + "available": [ + "assignment_context_paths", + "combined_assignments_path", + "combined_phases_path", + "phase_ids", + "phase_manifest_path", + "plan_snapshot_path", + "planner_assess_review_approach", + "planner_dir", + "refined_assignments_path", + "refined_plan_path", + "source_dir", + "task", + "task_file_path", + "task_label", + "wp_context_paths" + ], + "required": [], + "produced": [] + }, + { + "step": "merge_wps", + "available": [ + "assignment_context_paths", + "combined_assignments_path", + "combined_phases_path", + "phase_ids", + "phase_manifest_path", + "plan_snapshot_path", + "planner_assess_review_approach", + "planner_dir", + "refined_assignments_path", + "refined_plan_path", + "source_dir", + "task", + "task_file_path", + "task_label", + "wp_context_paths" + ], + "required": [], + "produced": [ + "combined_wps_path" + ] + }, + { + "step": "refine_wps", + "available": [ + "assignment_context_paths", + "combined_assignments_path", + "combined_phases_path", + "combined_wps_path", + "phase_ids", + "phase_manifest_path", + "plan_snapshot_path", + "planner_assess_review_approach", + "planner_dir", + "refined_assignments_path", + "refined_plan_path", + "source_dir", + "task", + "task_file_path", + "task_label", + "wp_context_paths" + ], + "required": [ + "output_dir" + ], + "produced": [ + "refined_wps_path" + ] + }, + { + "step": "consolidate_wps_analyze", + "available": [ + "assignment_context_paths", + "combined_assignments_path", + "combined_phases_path", + "combined_wps_path", + "phase_ids", + "phase_manifest_path", + "plan_snapshot_path", + "planner_assess_review_approach", + "planner_dir", + "refined_assignments_path", + "refined_plan_path", + "refined_wps_path", + "source_dir", + "task", + "task_file_path", + "task_label", + "wp_context_paths" + ], + "required": [], + "produced": [] + }, + { + "step": "consolidate_wps_merge", + "available": [ + "assignment_context_paths", + "combined_assignments_path", + "combined_phases_path", + "combined_wps_path", + "phase_ids", + "phase_manifest_path", + "plan_snapshot_path", + "planner_assess_review_approach", + "planner_dir", + "refined_assignments_path", + "refined_plan_path", + "refined_wps_path", + "source_dir", + "task", + "task_file_path", + "task_label", + "wp_context_paths" + ], + "required": [], + "produced": [ + "consolidated_wps_path" + ] + }, + { + "step": "validate_task_alignment", + "available": [ + "assignment_context_paths", + "combined_assignments_path", + "combined_phases_path", + "combined_wps_path", + "consolidated_wps_path", + "phase_ids", + "phase_manifest_path", + "plan_snapshot_path", + "planner_assess_review_approach", + "planner_dir", + "refined_assignments_path", + "refined_plan_path", + "refined_wps_path", + "source_dir", + "task", + "task_file_path", + "task_label", + "wp_context_paths" + ], + "required": [ + "refined_wps_path", + "output_dir" + ], + "produced": [ + "alignment_findings_path" + ] + }, + { + "step": "reconcile_deps", + "available": [ + "alignment_findings_path", + "assignment_context_paths", + "combined_assignments_path", + "combined_phases_path", + "combined_wps_path", + "consolidated_wps_path", + "phase_ids", + "phase_manifest_path", + "plan_snapshot_path", + "planner_assess_review_approach", + "planner_dir", + "refined_assignments_path", + "refined_plan_path", + "refined_wps_path", + "source_dir", + "task", + "task_file_path", + "task_label", + "wp_context_paths" + ], + "required": [], + "produced": [] + }, + { + "step": "assess_review_approach_step", + "available": [ + "alignment_findings_path", + "assignment_context_paths", + "combined_assignments_path", + "combined_phases_path", + "combined_wps_path", + "consolidated_wps_path", + "phase_ids", + "phase_manifest_path", + "plan_snapshot_path", + "planner_assess_review_approach", + "planner_dir", + "refined_assignments_path", + "refined_plan_path", + "refined_wps_path", + "source_dir", + "task", + "task_file_path", + "task_label", + "wp_context_paths" + ], + "required": [ + "refined_wps_path" + ], + "produced": [ + "review_approach_assessment_path" + ] + }, + { + "step": "validate", + "available": [ + "alignment_findings_path", + "assignment_context_paths", + "combined_assignments_path", + "combined_phases_path", + "combined_wps_path", + "consolidated_wps_path", + "phase_ids", + "phase_manifest_path", + "plan_snapshot_path", + "planner_assess_review_approach", + "planner_dir", + "refined_assignments_path", + "refined_plan_path", + "refined_wps_path", + "review_approach_assessment_path", + "source_dir", + "task", + "task_file_path", + "task_label", + "wp_context_paths" + ], + "required": [], + "produced": [ + "verdict" + ] + }, + { + "step": "check_verdict", + "available": [ + "alignment_findings_path", + "assignment_context_paths", + "combined_assignments_path", + "combined_phases_path", + "combined_wps_path", + "consolidated_wps_path", + "phase_ids", + "phase_manifest_path", + "plan_snapshot_path", + "planner_assess_review_approach", + "planner_dir", + "refined_assignments_path", + "refined_plan_path", + "refined_wps_path", + "review_approach_assessment_path", + "source_dir", + "task", + "task_file_path", + "task_label", + "verdict", + "wp_context_paths" + ], + "required": [], + "produced": [] + }, + { + "step": "refine", + "available": [ + "alignment_findings_path", + "assignment_context_paths", + "combined_assignments_path", + "combined_phases_path", + "combined_wps_path", + "consolidated_wps_path", + "phase_ids", + "phase_manifest_path", + "plan_snapshot_path", + "planner_assess_review_approach", + "planner_dir", + "refined_assignments_path", + "refined_plan_path", + "refined_wps_path", + "review_approach_assessment_path", + "source_dir", + "task", + "task_file_path", + "task_label", + "verdict", + "wp_context_paths" + ], + "required": [], + "produced": [], + "positional_args": 1 + }, + { + "step": "compile", + "available": [ + "alignment_findings_path", + "assignment_context_paths", + "combined_assignments_path", + "combined_phases_path", + "combined_wps_path", + "consolidated_wps_path", + "phase_ids", + "phase_manifest_path", + "plan_snapshot_path", + "planner_assess_review_approach", + "planner_dir", + "refined_assignments_path", + "refined_plan_path", + "refined_wps_path", + "review_approach_assessment_path", + "source_dir", + "task", + "task_file_path", + "task_label", + "verdict", + "wp_context_paths" + ], + "required": [], + "produced": [] + }, + { + "step": "done", + "available": [ + "alignment_findings_path", + "assignment_context_paths", + "combined_assignments_path", + "combined_phases_path", + "combined_wps_path", + "consolidated_wps_path", + "phase_ids", + "phase_manifest_path", + "plan_snapshot_path", + "planner_assess_review_approach", + "planner_dir", + "refined_assignments_path", + "refined_plan_path", + "refined_wps_path", + "review_approach_assessment_path", + "source_dir", + "task", + "task_file_path", + "task_label", + "verdict", + "wp_context_paths" + ], + "required": [], + "produced": [] + }, + { + "step": "escalate_stop", + "available": [ + "alignment_findings_path", + "assignment_context_paths", + "combined_assignments_path", + "combined_phases_path", + "combined_wps_path", + "consolidated_wps_path", + "phase_ids", + "phase_manifest_path", + "plan_snapshot_path", + "planner_assess_review_approach", + "planner_dir", + "refined_assignments_path", + "refined_plan_path", + "refined_wps_path", + "review_approach_assessment_path", + "source_dir", + "task", + "task_file_path", + "task_label", + "verdict", + "wp_context_paths" + ], + "required": [], + "produced": [] + } + ] +} diff --git a/src/autoskillit/recipes/contracts/planner.yaml b/src/autoskillit/recipes/contracts/planner.yaml new file mode 100644 index 0000000000..fd0aceca9a --- /dev/null +++ b/src/autoskillit/recipes/contracts/planner.yaml @@ -0,0 +1,797 @@ +generated_at: '2026-05-03T01:40:00.180863+00:00' +bundled_manifest_version: 0.1.0 +skill_hashes: + planner-analyze: sha256:bc64ba4984d3c1b32231c062e46550bc5e00d94689d5a8967b3f222abfe0e172 + planner-extract-domain: sha256:dcb246a33465b839a552719dff34684ea9421ad99e021232ae677448a69eff45 + planner-generate-phases: sha256:0904a31c377aee72e98e937915f8397e42b02dc7fab5572cb156e144bf613817 + planner-elaborate-phase: sha256:d0127bd85e030eecaabb727730ccb8d80fc05cb55227975320393ba17fcb2238 + planner-refine-phases: sha256:3a514d6a658cb3bcee2de18f0f3aea2f529c01a24268b7f6007ec391ba8fe91c + planner-elaborate-assignments: sha256:1545a91c18d14d00031077833d3f37af1be7b0287e8239d469df3388496c649e + planner-refine-assignments: sha256:584c1715729a91928994d4d18b91a8e48ca03ee94ce70dd1c613c7a5159e6318 + planner-elaborate-wps: sha256:301bebf89b29e658de254d5790d50878fde417a3359222c5b282065f13aa2b30 + planner-refine-wps: sha256:17d44c80abb78698c6173c74f6259df65ac09cd87063cfdf3e0e1de763043334 + planner-consolidate-wps: sha256:384ae024bfc9ef67fd25d4a929badcd5790577364a768d2f655b5172bc13a8e7 + planner-validate-task-alignment: sha256:f57d27bbf345f2da0c7fe4d8918f63e02b12882fd5fe1f7bb43e6c096f340b2b + planner-reconcile-deps: sha256:4ea197d4d0c25af5b734f36c79ddb182bc140d98cdfe811f515ff182961ac9bb + planner-assess-review-approach: sha256:4c3dfa00349ab2bec06c089d282f00d3e127e1c794b41daa6a71779612ab3558 + planner-refine: sha256:31e088ee392a4bee262818c5117eea98c57d910a6aec1363dc1a0c9e9f710e1d +skills: + planner-analyze: + inputs: [] + outputs: [] + expected_output_patterns: [] + pattern_examples: [] + planner-extract-domain: + inputs: [] + outputs: [] + expected_output_patterns: [] + pattern_examples: [] + planner-generate-phases: + inputs: [] + outputs: + - name: phase_manifest_path + type: file_path + - name: phase_ids + type: str + expected_output_patterns: + - phase_manifest_path\s*=\s*\S+ + pattern_examples: + - 'phase_manifest_path = /tmp/autoskillit/planner/run-20260101-120000/phases/phase_manifest.json + + %%ORDER_UP%%' + write_behavior: always + planner-elaborate-phase: + inputs: + - name: plan_snapshot_path + type: file_path + required: true + recommended: false + - name: target_phase_id + type: str + required: true + recommended: false + - name: output_dir + type: directory_path + required: true + recommended: false + outputs: + - name: elab_result_path + type: file_path + expected_output_patterns: + - elab_result_path\s*=\s*\S+ + pattern_examples: + - 'elab_result_path = /tmp/autoskillit/planner/run-20260101-120000/phases/P1_result.json + + %%ORDER_UP%%' + write_behavior: always + planner-refine-phases: + inputs: + - name: combined_plan_path + type: file_path + required: true + recommended: false + - name: output_dir + type: directory_path + required: true + recommended: false + outputs: + - name: refined_plan_path + type: file_path + expected_output_patterns: + - refined_plan_path\s*=\s*\S+ + pattern_examples: + - 'refined_plan_path = /tmp/autoskillit/planner/run-20260101-120000/refined_plan.json + + %%ORDER_UP%%' + write_behavior: always + planner-elaborate-assignments: + inputs: + - name: context_file + type: file_path + required: true + recommended: false + - name: planner_dir + type: directory_path + required: true + recommended: false + outputs: + - name: phase_assignments_result_dir + type: directory_path + expected_output_patterns: + - phase_assignments_result_dir\s*=\s*\S+ + pattern_examples: + - 'phase_assignments_result_dir = /tmp/autoskillit/planner/run-20260101-120000/assignments + + %%ORDER_UP%%' + write_behavior: always + planner-refine-assignments: + inputs: + - name: context_file + type: file_path + required: true + recommended: false + - name: refined_plan_path + type: file_path + required: true + recommended: false + - name: output_dir + type: directory_path + required: true + recommended: false + outputs: + - name: phase_refined_path + type: file_path + expected_output_patterns: + - phase_refined_path\s*=\s*\S+ + pattern_examples: + - 'phase_refined_path = /tmp/autoskillit/planner/run-20260101-120000/refine_contexts/P1_result.json + + %%ORDER_UP%%' + write_behavior: always + planner-elaborate-wps: + inputs: + - name: context_file + type: file_path + required: true + recommended: false + - name: planner_dir + type: directory_path + required: true + recommended: false + outputs: + - name: phase_wps_result_dir + type: directory_path + expected_output_patterns: + - phase_wps_result_dir\s*=\s*\S+ + pattern_examples: + - 'phase_wps_result_dir = /tmp/autoskillit/planner/run-20260101-120000/work_packages + + %%ORDER_UP%%' + write_behavior: always + planner-refine-wps: + inputs: + - name: combined_wps_path + type: file_path + required: true + recommended: false + - name: refined_plan_path + type: file_path + required: true + recommended: false + - name: refined_assignments_path + type: file_path + required: true + recommended: false + - name: output_dir + type: directory_path + required: true + recommended: false + outputs: + - name: refined_wps_path + type: file_path + expected_output_patterns: + - refined_wps_path\s*=\s*\S+ + pattern_examples: + - 'refined_wps_path = /tmp/autoskillit/planner/run-20260101-120000/refined_wps.json + + %%ORDER_UP%%' + write_behavior: always + planner-consolidate-wps: + inputs: + - name: refined_wps_path + type: file_path + required: true + recommended: false + - name: planner_dir + type: directory_path + required: true + recommended: false + outputs: [] + expected_output_patterns: [] + pattern_examples: [] + write_behavior: always + planner-validate-task-alignment: + inputs: + - name: refined_wps_path + type: file_path + required: true + recommended: false + - name: refined_plan_path + type: file_path + required: true + recommended: false + - name: output_dir + type: directory_path + required: true + recommended: false + outputs: + - name: alignment_findings_path + type: file_path + - name: alignment_finding_count + type: str + expected_output_patterns: + - alignment_findings_path\s*=\s*\S+ + - alignment_finding_count\s*=\s*\d+ + pattern_examples: + - 'alignment_findings_path = /tmp/autoskillit/planner/run-20260101-120000/task_alignment.json + + alignment_finding_count = 0 + + %%ORDER_UP%%' + write_behavior: always + planner-reconcile-deps: + inputs: [] + outputs: [] + expected_output_patterns: [] + pattern_examples: [] + planner-assess-review-approach: + inputs: + - name: refined_wps_path + type: file_path + required: true + recommended: false + - name: planner_dir + type: directory_path + required: true + recommended: false + outputs: + - name: review_approach_assessment_path + type: file_path + expected_output_patterns: + - review_approach_assessment_path\s*=\s*\S+ + pattern_examples: + - 'review_approach_assessment_path = /tmp/autoskillit/planner/run-20260101-120000/review_approach_assessment.json + + %%ORDER_UP%%' + write_behavior: always + planner-refine: + inputs: [] + outputs: [] + expected_output_patterns: [] + pattern_examples: [] +dataflow: +- step: init + available: + - planner_assess_review_approach + - source_dir + - task + required: [] + produced: + - planner_dir +- step: resolve_task + available: + - planner_assess_review_approach + - planner_dir + - source_dir + - task + required: [] + produced: + - task_file_path + - task_label +- step: analyze + available: + - planner_assess_review_approach + - planner_dir + - source_dir + - task + - task_file_path + - task_label + required: [] + produced: [] +- step: extract_domain + available: + - planner_assess_review_approach + - planner_dir + - source_dir + - task + - task_file_path + - task_label + required: [] + produced: [] + positional_args: 2 +- step: generate_phases + available: + - planner_assess_review_approach + - planner_dir + - source_dir + - task + - task_file_path + - task_label + required: [] + produced: + - phase_manifest_path + positional_args: 3 +- step: build_plan_snapshot + available: + - phase_manifest_path + - planner_assess_review_approach + - planner_dir + - source_dir + - task + - task_file_path + - task_label + required: [] + produced: + - plan_snapshot_path + - phase_ids +- step: elaborate_phases + available: + - phase_ids + - phase_manifest_path + - plan_snapshot_path + - planner_assess_review_approach + - planner_dir + - source_dir + - task + - task_file_path + - task_label + required: [] + produced: [] + positional_args: 2 +- step: merge_phases + available: + - phase_ids + - phase_manifest_path + - plan_snapshot_path + - planner_assess_review_approach + - planner_dir + - source_dir + - task + - task_file_path + - task_label + required: [] + produced: + - combined_phases_path +- step: refine_phases + available: + - combined_phases_path + - phase_ids + - phase_manifest_path + - plan_snapshot_path + - planner_assess_review_approach + - planner_dir + - source_dir + - task + - task_file_path + - task_label + required: + - combined_plan_path + - output_dir + produced: + - refined_plan_path +- step: expand_assignments + available: + - combined_phases_path + - phase_ids + - phase_manifest_path + - plan_snapshot_path + - planner_assess_review_approach + - planner_dir + - refined_plan_path + - source_dir + - task + - task_file_path + - task_label + required: [] + produced: + - assignment_context_paths +- step: elaborate_assignments + available: + - assignment_context_paths + - combined_phases_path + - phase_ids + - phase_manifest_path + - plan_snapshot_path + - planner_assess_review_approach + - planner_dir + - refined_plan_path + - source_dir + - task + - task_file_path + - task_label + required: [] + produced: [] + positional_args: 1 +- step: merge_assignments + available: + - assignment_context_paths + - combined_phases_path + - phase_ids + - phase_manifest_path + - plan_snapshot_path + - planner_assess_review_approach + - planner_dir + - refined_plan_path + - source_dir + - task + - task_file_path + - task_label + required: [] + produced: + - combined_assignments_path +- step: refine_assignments + available: + - assignment_context_paths + - combined_assignments_path + - combined_phases_path + - phase_ids + - phase_manifest_path + - plan_snapshot_path + - planner_assess_review_approach + - planner_dir + - refined_plan_path + - source_dir + - task + - task_file_path + - task_label + required: + - output_dir + produced: + - refined_assignments_path +- step: expand_wps + available: + - assignment_context_paths + - combined_assignments_path + - combined_phases_path + - phase_ids + - phase_manifest_path + - plan_snapshot_path + - planner_assess_review_approach + - planner_dir + - refined_assignments_path + - refined_plan_path + - source_dir + - task + - task_file_path + - task_label + required: [] + produced: + - wp_context_paths +- step: elaborate_wps + available: + - assignment_context_paths + - combined_assignments_path + - combined_phases_path + - phase_ids + - phase_manifest_path + - plan_snapshot_path + - planner_assess_review_approach + - planner_dir + - refined_assignments_path + - refined_plan_path + - source_dir + - task + - task_file_path + - task_label + - wp_context_paths + required: [] + produced: [] + positional_args: 1 +- step: finalize_wp_manifest + available: + - assignment_context_paths + - combined_assignments_path + - combined_phases_path + - phase_ids + - phase_manifest_path + - plan_snapshot_path + - planner_assess_review_approach + - planner_dir + - refined_assignments_path + - refined_plan_path + - source_dir + - task + - task_file_path + - task_label + - wp_context_paths + required: [] + produced: [] +- step: merge_wps + available: + - assignment_context_paths + - combined_assignments_path + - combined_phases_path + - phase_ids + - phase_manifest_path + - plan_snapshot_path + - planner_assess_review_approach + - planner_dir + - refined_assignments_path + - refined_plan_path + - source_dir + - task + - task_file_path + - task_label + - wp_context_paths + required: [] + produced: + - combined_wps_path +- step: refine_wps + available: + - assignment_context_paths + - combined_assignments_path + - combined_phases_path + - combined_wps_path + - phase_ids + - phase_manifest_path + - plan_snapshot_path + - planner_assess_review_approach + - planner_dir + - refined_assignments_path + - refined_plan_path + - source_dir + - task + - task_file_path + - task_label + - wp_context_paths + required: + - output_dir + produced: + - refined_wps_path +- step: consolidate_wps_analyze + available: + - assignment_context_paths + - combined_assignments_path + - combined_phases_path + - combined_wps_path + - phase_ids + - phase_manifest_path + - plan_snapshot_path + - planner_assess_review_approach + - planner_dir + - refined_assignments_path + - refined_plan_path + - refined_wps_path + - source_dir + - task + - task_file_path + - task_label + - wp_context_paths + required: [] + produced: [] +- step: consolidate_wps_merge + available: + - assignment_context_paths + - combined_assignments_path + - combined_phases_path + - combined_wps_path + - phase_ids + - phase_manifest_path + - plan_snapshot_path + - planner_assess_review_approach + - planner_dir + - refined_assignments_path + - refined_plan_path + - refined_wps_path + - source_dir + - task + - task_file_path + - task_label + - wp_context_paths + required: [] + produced: + - consolidated_wps_path +- step: validate_task_alignment + available: + - assignment_context_paths + - combined_assignments_path + - combined_phases_path + - combined_wps_path + - consolidated_wps_path + - phase_ids + - phase_manifest_path + - plan_snapshot_path + - planner_assess_review_approach + - planner_dir + - refined_assignments_path + - refined_plan_path + - refined_wps_path + - source_dir + - task + - task_file_path + - task_label + - wp_context_paths + required: + - refined_wps_path + - output_dir + produced: + - alignment_findings_path +- step: reconcile_deps + available: + - alignment_findings_path + - assignment_context_paths + - combined_assignments_path + - combined_phases_path + - combined_wps_path + - consolidated_wps_path + - phase_ids + - phase_manifest_path + - plan_snapshot_path + - planner_assess_review_approach + - planner_dir + - refined_assignments_path + - refined_plan_path + - refined_wps_path + - source_dir + - task + - task_file_path + - task_label + - wp_context_paths + required: [] + produced: [] +- step: assess_review_approach_step + available: + - alignment_findings_path + - assignment_context_paths + - combined_assignments_path + - combined_phases_path + - combined_wps_path + - consolidated_wps_path + - phase_ids + - phase_manifest_path + - plan_snapshot_path + - planner_assess_review_approach + - planner_dir + - refined_assignments_path + - refined_plan_path + - refined_wps_path + - source_dir + - task + - task_file_path + - task_label + - wp_context_paths + required: + - refined_wps_path + produced: + - review_approach_assessment_path +- step: validate + available: + - alignment_findings_path + - assignment_context_paths + - combined_assignments_path + - combined_phases_path + - combined_wps_path + - consolidated_wps_path + - phase_ids + - phase_manifest_path + - plan_snapshot_path + - planner_assess_review_approach + - planner_dir + - refined_assignments_path + - refined_plan_path + - refined_wps_path + - review_approach_assessment_path + - source_dir + - task + - task_file_path + - task_label + - wp_context_paths + required: [] + produced: + - verdict +- step: check_verdict + available: + - alignment_findings_path + - assignment_context_paths + - combined_assignments_path + - combined_phases_path + - combined_wps_path + - consolidated_wps_path + - phase_ids + - phase_manifest_path + - plan_snapshot_path + - planner_assess_review_approach + - planner_dir + - refined_assignments_path + - refined_plan_path + - refined_wps_path + - review_approach_assessment_path + - source_dir + - task + - task_file_path + - task_label + - verdict + - wp_context_paths + required: [] + produced: [] +- step: refine + available: + - alignment_findings_path + - assignment_context_paths + - combined_assignments_path + - combined_phases_path + - combined_wps_path + - consolidated_wps_path + - phase_ids + - phase_manifest_path + - plan_snapshot_path + - planner_assess_review_approach + - planner_dir + - refined_assignments_path + - refined_plan_path + - refined_wps_path + - review_approach_assessment_path + - source_dir + - task + - task_file_path + - task_label + - verdict + - wp_context_paths + required: [] + produced: [] + positional_args: 1 +- step: compile + available: + - alignment_findings_path + - assignment_context_paths + - combined_assignments_path + - combined_phases_path + - combined_wps_path + - consolidated_wps_path + - phase_ids + - phase_manifest_path + - plan_snapshot_path + - planner_assess_review_approach + - planner_dir + - refined_assignments_path + - refined_plan_path + - refined_wps_path + - review_approach_assessment_path + - source_dir + - task + - task_file_path + - task_label + - verdict + - wp_context_paths + required: [] + produced: [] +- step: done + available: + - alignment_findings_path + - assignment_context_paths + - combined_assignments_path + - combined_phases_path + - combined_wps_path + - consolidated_wps_path + - phase_ids + - phase_manifest_path + - plan_snapshot_path + - planner_assess_review_approach + - planner_dir + - refined_assignments_path + - refined_plan_path + - refined_wps_path + - review_approach_assessment_path + - source_dir + - task + - task_file_path + - task_label + - verdict + - wp_context_paths + required: [] + produced: [] +- step: escalate_stop + available: + - alignment_findings_path + - assignment_context_paths + - combined_assignments_path + - combined_phases_path + - combined_wps_path + - consolidated_wps_path + - phase_ids + - phase_manifest_path + - plan_snapshot_path + - planner_assess_review_approach + - planner_dir + - refined_assignments_path + - refined_plan_path + - refined_wps_path + - review_approach_assessment_path + - source_dir + - task + - task_file_path + - task_label + - verdict + - wp_context_paths + required: [] + produced: [] diff --git a/src/autoskillit/recipes/contracts/remediation.json b/src/autoskillit/recipes/contracts/remediation.json new file mode 100644 index 0000000000..89b64bf6ad --- /dev/null +++ b/src/autoskillit/recipes/contracts/remediation.json @@ -0,0 +1,4991 @@ +{ + "generated_at": "2026-05-03T07:01:54.187185+00:00", + "bundled_manifest_version": "0.1.0", + "skill_hashes": {}, + "skills": { + "investigate": { + "inputs": [ + { + "name": "topic", + "type": "string", + "required": true, + "recommended": false + } + ], + "outputs": [ + { + "name": "investigation_path", + "type": "file_path" + } + ], + "expected_output_patterns": [ + "investigation_path\\s*=\\s*/.+" + ], + "pattern_examples": [ + "investigation_path = /tmp/investigation.md\n%%ORDER_UP%%" + ], + "read_only": true + }, + "rectify": { + "inputs": [ + { + "name": "investigation_path", + "type": "file_path", + "required": true, + "recommended": false + } + ], + "outputs": [ + { + "name": "plan_path", + "type": "file_path" + }, + { + "name": "plan_parts", + "type": "file_path_list" + } + ], + "expected_output_patterns": [ + "plan_path\\s*=\\s*/.+" + ], + "pattern_examples": [ + "plan_path = /tmp/plan.md\n%%ORDER_UP%%" + ], + "write_behavior": "always" + }, + "review-approach": { + "inputs": [ + { + "name": "plan_path", + "type": "file_path", + "required": false, + "recommended": false + } + ], + "outputs": [ + { + "name": "review_path", + "type": "file_path" + } + ], + "expected_output_patterns": [ + "review_path\\s*=\\s*/.+" + ], + "pattern_examples": [ + "review_path = /tmp/review.md\n%%ORDER_UP%%" + ] + }, + "dry-walkthrough": { + "inputs": [ + { + "name": "plan_path", + "type": "file_path", + "required": true, + "recommended": false + } + ], + "outputs": [], + "expected_output_patterns": [], + "pattern_examples": [], + "write_behavior": "always" + }, + "implement-worktree-no-merge": { + "inputs": [ + { + "name": "plan_path", + "type": "file_path", + "required": true, + "recommended": false + } + ], + "outputs": [ + { + "name": "worktree_path", + "type": "directory_path" + }, + { + "name": "branch_name", + "type": "string" + } + ], + "expected_output_patterns": [ + "worktree_path\\s*=\\s*/.+" + ], + "pattern_examples": [ + "worktree_path = /tmp/worktrees/impl-foo-20260316\n%%ORDER_UP%%" + ], + "write_behavior": "always" + }, + "retry-worktree": { + "inputs": [ + { + "name": "plan_path", + "type": "file_path", + "required": true, + "recommended": false + }, + { + "name": "worktree_path", + "type": "directory_path", + "required": true, + "recommended": false + } + ], + "outputs": [ + { + "name": "worktree_path", + "type": "directory_path" + }, + { + "name": "branch_name", + "type": "string" + }, + { + "name": "phases_implemented", + "type": "integer" + } + ], + "expected_output_patterns": [ + "worktree_path\\s*=\\s*/.+" + ], + "pattern_examples": [ + "worktree_path = /tmp/worktrees/impl-foo-20260316\nbranch_name = feature/123\nphases_implemented = 2\n%%ORDER_UP%%" + ], + "write_behavior": "conditional", + "write_expected_when": [ + "phases_implemented\\s*=\\s*[1-9][0-9]*" + ] + }, + "resolve-failures": { + "inputs": [ + { + "name": "worktree_path", + "type": "directory_path", + "required": true, + "recommended": false + }, + { + "name": "plan_path", + "type": "file_path", + "required": true, + "recommended": false + }, + { + "name": "base_branch", + "type": "string", + "required": true, + "recommended": false + }, + { + "name": "diagnosis_path", + "type": "file_path", + "required": false, + "recommended": false + } + ], + "outputs": [ + { + "name": "verdict", + "type": "string" + }, + { + "name": "fixes_applied", + "type": "integer" + } + ], + "expected_output_patterns": [], + "pattern_examples": [], + "write_behavior": "conditional", + "write_expected_when": [ + "verdict\\s*=\\s*real_fix" + ] + }, + "audit-impl": { + "inputs": [ + { + "name": "plan_path", + "type": "file_path", + "required": true, + "recommended": false + }, + { + "name": "branch_name", + "type": "string", + "required": false, + "recommended": false + }, + { + "name": "base_branch", + "type": "string", + "required": false, + "recommended": false + } + ], + "outputs": [ + { + "name": "verdict", + "type": "string" + }, + { + "name": "remediation_path", + "type": "file_path" + } + ], + "expected_output_patterns": [ + "verdict\\s*=\\s*(GO|NO GO)" + ], + "pattern_examples": [ + "verdict = GO\n%%ORDER_UP%%", + "verdict = NO GO\nremediation_path = /tmp/remediation.md\n%%ORDER_UP%%" + ] + }, + "make-plan": { + "inputs": [ + { + "name": "task", + "type": "string", + "required": true, + "recommended": false + } + ], + "outputs": [ + { + "name": "plan_path", + "type": "file_path" + }, + { + "name": "plan_parts", + "type": "file_path_list" + } + ], + "expected_output_patterns": [ + "plan_path\\s*=\\s*/.+" + ], + "pattern_examples": [ + "plan_path = /tmp/plan.md\n%%ORDER_UP%%" + ], + "write_behavior": "always" + }, + "prepare-pr": { + "inputs": [ + { + "name": "plan_paths", + "type": "string", + "required": true, + "recommended": false + }, + { + "name": "run_name", + "type": "string", + "required": true, + "recommended": false + }, + { + "name": "base_branch", + "type": "string", + "required": true, + "recommended": false + }, + { + "name": "closing_issue", + "type": "string", + "required": false, + "recommended": false + }, + { + "name": "conflict_report_path", + "type": "file_path", + "required": false, + "recommended": false + } + ], + "outputs": [ + { + "name": "prep_path", + "type": "file_path" + }, + { + "name": "selected_lenses", + "type": "string" + }, + { + "name": "lens_context_paths", + "type": "string" + } + ], + "expected_output_patterns": [ + "prep_path\\s*=\\s*/.+", + "selected_lenses\\s*=\\s*\\S+", + "lens_context_paths\\s*=\\s*/.+" + ], + "pattern_examples": [ + "prep_path = /abs/path/pr_prep_2026-04-07_120000.md\nselected_lenses = module-dependency,process-flow\nlens_context_paths = /abs/ctx1.md,/abs/ctx2.md\n%%ORDER_UP%%" + ], + "write_behavior": "conditional", + "write_expected_when": [ + "prep_path\\s*=\\s*/.+" + ] + }, + "compose-pr": { + "inputs": [ + { + "name": "prep_path", + "type": "file_path", + "required": true, + "recommended": false + }, + { + "name": "all_diagram_paths", + "type": "string", + "required": true, + "recommended": false + }, + { + "name": "work_dir", + "type": "directory_path", + "required": true, + "recommended": false + }, + { + "name": "base_branch", + "type": "string", + "required": true, + "recommended": false + }, + { + "name": "closing_issue", + "type": "string", + "required": false, + "recommended": false + } + ], + "outputs": [ + { + "name": "pr_url", + "type": "string" + } + ], + "expected_output_patterns": [ + "pr_url\\s*=\\s*(https://github\\.com/.*/pull/\\d+)?" + ], + "pattern_examples": [ + "pr_url = https://github.com/owner/repo/pull/42\n%%ORDER_UP%%", + "pr_url = \n%%ORDER_UP%%" + ], + "write_behavior": "always" + }, + "review-pr": { + "inputs": [ + { + "name": "feature_branch", + "type": "string", + "required": true, + "recommended": false + }, + { + "name": "base_branch", + "type": "string", + "required": true, + "recommended": false + }, + { + "name": "annotated_diff_path", + "type": "string", + "required": false, + "recommended": true + }, + { + "name": "hunk_ranges_path", + "type": "string", + "required": false, + "recommended": true + } + ], + "outputs": [ + { + "name": "verdict", + "type": "string" + } + ], + "expected_output_patterns": [ + "verdict\\s*=\\s*(approved|approved_with_comments|changes_requested|needs_human)", + "%%REVIEW_GATE::(LOOP_REQUIRED|CLEAR)%%" + ], + "pattern_examples": [ + "verdict = approved\n%%REVIEW_GATE::CLEAR%%\n%%ORDER_UP%%", + "verdict = approved_with_comments\n%%REVIEW_GATE::CLEAR%%\n%%ORDER_UP%%", + "verdict = changes_requested\n%%REVIEW_GATE::LOOP_REQUIRED%%\n%%ORDER_UP%%", + "verdict = needs_human\n%%REVIEW_GATE::CLEAR%%\n%%ORDER_UP%%" + ] + }, + "resolve-review": { + "inputs": [ + { + "name": "feature_branch", + "type": "string", + "required": true, + "recommended": false + }, + { + "name": "base_branch", + "type": "string", + "required": true, + "recommended": false + } + ], + "outputs": [ + { + "name": "verdict", + "type": "string" + }, + { + "name": "fixes_applied", + "type": "integer" + } + ], + "expected_output_patterns": [], + "pattern_examples": [], + "write_behavior": "conditional", + "write_expected_when": [ + "verdict\\s*=\\s*real_fix" + ] + }, + "resolve-merge-conflicts": { + "inputs": [ + { + "name": "worktree_path", + "type": "directory_path", + "required": true, + "recommended": false + }, + { + "name": "plan_path", + "type": "file_path", + "required": true, + "recommended": false + }, + { + "name": "base_branch", + "type": "string", + "required": true, + "recommended": false + } + ], + "outputs": [ + { + "name": "worktree_path", + "type": "directory_path" + }, + { + "name": "branch_name", + "type": "string" + }, + { + "name": "conflict_report_path", + "type": "file_path" + }, + { + "name": "escalation_required", + "type": "string" + }, + { + "name": "escalation_reason", + "type": "string" + } + ], + "expected_output_patterns": [ + "worktree_path\\s*=\\s*/.+" + ], + "pattern_examples": [ + "worktree_path = /tmp/worktrees/impl-foo-20260316\n%%ORDER_UP%%" + ], + "write_behavior": "conditional", + "write_expected_when": [ + "conflict_report_path\\s*=\\s*/.+" + ] + }, + "diagnose-ci": { + "inputs": [ + { + "name": "branch", + "type": "string", + "required": true, + "recommended": false + }, + { + "name": "run_id", + "type": "string", + "required": false, + "recommended": false + }, + { + "name": "ci_failed_jobs", + "type": "string", + "required": false, + "recommended": false + } + ], + "outputs": [ + { + "name": "diagnosis_path", + "type": "file_path" + }, + { + "name": "failure_type", + "type": "string" + }, + { + "name": "failure_subtype", + "type": "string" + }, + { + "name": "is_fixable", + "type": "string" + } + ], + "expected_output_patterns": [ + "diagnosis_path\\s*=\\s*/.+", + "failure_subtype\\s*=\\s*(flaky|timing_race|deterministic|fixture|import|env|unknown)" + ], + "pattern_examples": [ + "diagnosis_path = /tmp/diagnosis.md\nfailure_type = test\nfailure_subtype = deterministic\nis_fixable = true\n%%ORDER_UP%%", + "diagnosis_path = /tmp/diagnosis.md\nfailure_type = test\nfailure_subtype = flaky\nis_fixable = false\n%%ORDER_UP%%" + ], + "write_behavior": "always" + } + }, + "dataflow": [ + { + "step": "clone", + "available": [ + "arch_lenses", + "audit", + "auto_merge", + "base_branch", + "depth", + "issue_url", + "max_parallel", + "open_pr", + "review_approach", + "review_max_retries", + "run_mode", + "run_name", + "source_dir", + "task", + "upfront_claimed" + ], + "required": [], + "produced": [ + "work_dir", + "remote_url" + ] + }, + { + "step": "set_merge_target", + "available": [ + "arch_lenses", + "audit", + "auto_merge", + "base_branch", + "depth", + "issue_url", + "max_parallel", + "open_pr", + "remote_url", + "review_approach", + "review_max_retries", + "run_mode", + "run_name", + "source_dir", + "task", + "upfront_claimed", + "work_dir" + ], + "required": [], + "produced": [ + "merge_target" + ] + }, + { + "step": "get_issue_title", + "available": [ + "arch_lenses", + "audit", + "auto_merge", + "base_branch", + "depth", + "issue_url", + "max_parallel", + "merge_target", + "open_pr", + "remote_url", + "review_approach", + "review_max_retries", + "run_mode", + "run_name", + "source_dir", + "task", + "upfront_claimed", + "work_dir" + ], + "required": [], + "produced": [ + "issue_number", + "issue_title", + "issue_slug" + ] + }, + { + "step": "claim_issue", + "available": [ + "arch_lenses", + "audit", + "auto_merge", + "base_branch", + "depth", + "issue_number", + "issue_slug", + "issue_title", + "issue_url", + "max_parallel", + "merge_target", + "open_pr", + "remote_url", + "review_approach", + "review_max_retries", + "run_mode", + "run_name", + "source_dir", + "task", + "upfront_claimed", + "work_dir" + ], + "required": [], + "produced": [ + "claimed" + ] + }, + { + "step": "compute_branch", + "available": [ + "arch_lenses", + "audit", + "auto_merge", + "base_branch", + "claimed", + "depth", + "issue_number", + "issue_slug", + "issue_title", + "issue_url", + "max_parallel", + "merge_target", + "open_pr", + "remote_url", + "review_approach", + "review_max_retries", + "run_mode", + "run_name", + "source_dir", + "task", + "upfront_claimed", + "work_dir" + ], + "required": [], + "produced": [ + "proposed_branch" + ] + }, + { + "step": "create_branch", + "available": [ + "arch_lenses", + "audit", + "auto_merge", + "base_branch", + "claimed", + "depth", + "issue_number", + "issue_slug", + "issue_title", + "issue_url", + "max_parallel", + "merge_target", + "open_pr", + "proposed_branch", + "remote_url", + "review_approach", + "review_max_retries", + "run_mode", + "run_name", + "source_dir", + "task", + "upfront_claimed", + "work_dir" + ], + "required": [], + "produced": [ + "merge_target" + ] + }, + { + "step": "push_merge_target", + "available": [ + "arch_lenses", + "audit", + "auto_merge", + "base_branch", + "claimed", + "depth", + "issue_number", + "issue_slug", + "issue_title", + "issue_url", + "max_parallel", + "merge_target", + "open_pr", + "proposed_branch", + "remote_url", + "review_approach", + "review_max_retries", + "run_mode", + "run_name", + "source_dir", + "task", + "upfront_claimed", + "work_dir" + ], + "required": [], + "produced": [] + }, + { + "step": "investigate", + "available": [ + "arch_lenses", + "audit", + "auto_merge", + "base_branch", + "claimed", + "depth", + "issue_number", + "issue_slug", + "issue_title", + "issue_url", + "max_parallel", + "merge_target", + "open_pr", + "proposed_branch", + "remote_url", + "review_approach", + "review_max_retries", + "run_mode", + "run_name", + "source_dir", + "task", + "upfront_claimed", + "work_dir" + ], + "required": [ + "topic" + ], + "produced": [ + "investigation_path" + ] + }, + { + "step": "rectify", + "available": [ + "arch_lenses", + "audit", + "auto_merge", + "base_branch", + "claimed", + "depth", + "investigation_path", + "issue_number", + "issue_slug", + "issue_title", + "issue_url", + "max_parallel", + "merge_target", + "open_pr", + "proposed_branch", + "remote_url", + "review_approach", + "review_max_retries", + "run_mode", + "run_name", + "source_dir", + "task", + "upfront_claimed", + "work_dir" + ], + "required": [], + "produced": [ + "plan_path" + ] + }, + { + "step": "review", + "available": [ + "arch_lenses", + "audit", + "auto_merge", + "base_branch", + "claimed", + "depth", + "investigation_path", + "issue_number", + "issue_slug", + "issue_title", + "issue_url", + "max_parallel", + "merge_target", + "open_pr", + "plan_path", + "proposed_branch", + "remote_url", + "review_approach", + "review_max_retries", + "run_mode", + "run_name", + "source_dir", + "task", + "upfront_claimed", + "work_dir" + ], + "required": [], + "produced": [ + "review_path" + ] + }, + { + "step": "dry_walkthrough", + "available": [ + "arch_lenses", + "audit", + "auto_merge", + "base_branch", + "claimed", + "depth", + "investigation_path", + "issue_number", + "issue_slug", + "issue_title", + "issue_url", + "max_parallel", + "merge_target", + "open_pr", + "plan_path", + "proposed_branch", + "remote_url", + "review_approach", + "review_max_retries", + "review_path", + "run_mode", + "run_name", + "source_dir", + "task", + "upfront_claimed", + "work_dir" + ], + "required": [], + "produced": [] + }, + { + "step": "implement", + "available": [ + "arch_lenses", + "audit", + "auto_merge", + "base_branch", + "claimed", + "depth", + "investigation_path", + "issue_number", + "issue_slug", + "issue_title", + "issue_url", + "max_parallel", + "merge_target", + "open_pr", + "plan_path", + "proposed_branch", + "remote_url", + "review_approach", + "review_max_retries", + "review_path", + "run_mode", + "run_name", + "source_dir", + "task", + "upfront_claimed", + "work_dir" + ], + "required": [], + "produced": [ + "implementation_ref", + "worktree_path", + "branch_name" + ] + }, + { + "step": "retry_worktree", + "available": [ + "arch_lenses", + "audit", + "auto_merge", + "base_branch", + "branch_name", + "claimed", + "depth", + "implementation_ref", + "investigation_path", + "issue_number", + "issue_slug", + "issue_title", + "issue_url", + "max_parallel", + "merge_target", + "open_pr", + "plan_path", + "proposed_branch", + "remote_url", + "review_approach", + "review_max_retries", + "review_path", + "run_mode", + "run_name", + "source_dir", + "task", + "upfront_claimed", + "work_dir", + "worktree_path" + ], + "required": [], + "produced": [ + "implementation_ref", + "worktree_path", + "branch_name", + "phases_implemented" + ] + }, + { + "step": "test", + "available": [ + "arch_lenses", + "audit", + "auto_merge", + "base_branch", + "branch_name", + "claimed", + "depth", + "implementation_ref", + "investigation_path", + "issue_number", + "issue_slug", + "issue_title", + "issue_url", + "max_parallel", + "merge_target", + "open_pr", + "phases_implemented", + "plan_path", + "proposed_branch", + "remote_url", + "review_approach", + "review_max_retries", + "review_path", + "run_mode", + "run_name", + "source_dir", + "task", + "upfront_claimed", + "work_dir", + "worktree_path" + ], + "required": [], + "produced": [] + }, + { + "step": "assess", + "available": [ + "arch_lenses", + "audit", + "auto_merge", + "base_branch", + "branch_name", + "claimed", + "depth", + "implementation_ref", + "investigation_path", + "issue_number", + "issue_slug", + "issue_title", + "issue_url", + "max_parallel", + "merge_target", + "open_pr", + "phases_implemented", + "plan_path", + "proposed_branch", + "remote_url", + "review_approach", + "review_max_retries", + "review_path", + "run_mode", + "run_name", + "source_dir", + "task", + "upfront_claimed", + "work_dir", + "worktree_path" + ], + "required": [], + "produced": [ + "verdict", + "fixes_applied" + ] + }, + { + "step": "audit_impl", + "available": [ + "arch_lenses", + "audit", + "auto_merge", + "base_branch", + "branch_name", + "claimed", + "depth", + "fixes_applied", + "implementation_ref", + "investigation_path", + "issue_number", + "issue_slug", + "issue_title", + "issue_url", + "max_parallel", + "merge_target", + "open_pr", + "phases_implemented", + "plan_path", + "proposed_branch", + "remote_url", + "review_approach", + "review_max_retries", + "review_path", + "run_mode", + "run_name", + "source_dir", + "task", + "upfront_claimed", + "verdict", + "work_dir", + "worktree_path" + ], + "required": [], + "produced": [ + "remediation_path" + ] + }, + { + "step": "remediate", + "available": [ + "arch_lenses", + "audit", + "auto_merge", + "base_branch", + "branch_name", + "claimed", + "depth", + "fixes_applied", + "implementation_ref", + "investigation_path", + "issue_number", + "issue_slug", + "issue_title", + "issue_url", + "max_parallel", + "merge_target", + "open_pr", + "phases_implemented", + "plan_path", + "proposed_branch", + "remediation_path", + "remote_url", + "review_approach", + "review_max_retries", + "review_path", + "run_mode", + "run_name", + "source_dir", + "task", + "upfront_claimed", + "verdict", + "work_dir", + "worktree_path" + ], + "required": [], + "produced": [] + }, + { + "step": "make_plan", + "available": [ + "arch_lenses", + "audit", + "auto_merge", + "base_branch", + "branch_name", + "claimed", + "depth", + "fixes_applied", + "implementation_ref", + "investigation_path", + "issue_number", + "issue_slug", + "issue_title", + "issue_url", + "max_parallel", + "merge_target", + "open_pr", + "phases_implemented", + "plan_path", + "proposed_branch", + "remediation_path", + "remote_url", + "review_approach", + "review_max_retries", + "review_path", + "run_mode", + "run_name", + "source_dir", + "task", + "upfront_claimed", + "verdict", + "work_dir", + "worktree_path" + ], + "required": [], + "produced": [ + "plan_path" + ] + }, + { + "step": "commit_guard", + "available": [ + "arch_lenses", + "audit", + "auto_merge", + "base_branch", + "branch_name", + "claimed", + "depth", + "fixes_applied", + "implementation_ref", + "investigation_path", + "issue_number", + "issue_slug", + "issue_title", + "issue_url", + "max_parallel", + "merge_target", + "open_pr", + "phases_implemented", + "plan_path", + "proposed_branch", + "remediation_path", + "remote_url", + "review_approach", + "review_max_retries", + "review_path", + "run_mode", + "run_name", + "source_dir", + "task", + "upfront_claimed", + "verdict", + "work_dir", + "worktree_path" + ], + "required": [], + "produced": [] + }, + { + "step": "merge", + "available": [ + "arch_lenses", + "audit", + "auto_merge", + "base_branch", + "branch_name", + "claimed", + "depth", + "fixes_applied", + "implementation_ref", + "investigation_path", + "issue_number", + "issue_slug", + "issue_title", + "issue_url", + "max_parallel", + "merge_target", + "open_pr", + "phases_implemented", + "plan_path", + "proposed_branch", + "remediation_path", + "remote_url", + "review_approach", + "review_max_retries", + "review_path", + "run_mode", + "run_name", + "source_dir", + "task", + "upfront_claimed", + "verdict", + "work_dir", + "worktree_path" + ], + "required": [], + "produced": [ + "cleanup_succeeded", + "worktree_path" + ] + }, + { + "step": "next_or_done", + "available": [ + "arch_lenses", + "audit", + "auto_merge", + "base_branch", + "branch_name", + "claimed", + "cleanup_succeeded", + "depth", + "fixes_applied", + "implementation_ref", + "investigation_path", + "issue_number", + "issue_slug", + "issue_title", + "issue_url", + "max_parallel", + "merge_target", + "open_pr", + "phases_implemented", + "plan_path", + "proposed_branch", + "remediation_path", + "remote_url", + "review_approach", + "review_max_retries", + "review_path", + "run_mode", + "run_name", + "source_dir", + "task", + "upfront_claimed", + "verdict", + "work_dir", + "worktree_path" + ], + "required": [], + "produced": [] + }, + { + "step": "push", + "available": [ + "arch_lenses", + "audit", + "auto_merge", + "base_branch", + "branch_name", + "claimed", + "cleanup_succeeded", + "depth", + "fixes_applied", + "implementation_ref", + "investigation_path", + "issue_number", + "issue_slug", + "issue_title", + "issue_url", + "max_parallel", + "merge_target", + "open_pr", + "phases_implemented", + "plan_path", + "proposed_branch", + "remediation_path", + "remote_url", + "review_approach", + "review_max_retries", + "review_path", + "run_mode", + "run_name", + "source_dir", + "task", + "upfront_claimed", + "verdict", + "work_dir", + "worktree_path" + ], + "required": [], + "produced": [] + }, + { + "step": "prepare_pr", + "available": [ + "arch_lenses", + "audit", + "auto_merge", + "base_branch", + "branch_name", + "claimed", + "cleanup_succeeded", + "depth", + "fixes_applied", + "implementation_ref", + "investigation_path", + "issue_number", + "issue_slug", + "issue_title", + "issue_url", + "max_parallel", + "merge_target", + "open_pr", + "phases_implemented", + "plan_path", + "proposed_branch", + "remediation_path", + "remote_url", + "review_approach", + "review_max_retries", + "review_path", + "run_mode", + "run_name", + "source_dir", + "task", + "upfront_claimed", + "verdict", + "work_dir", + "worktree_path" + ], + "required": [ + "plan_paths" + ], + "produced": [ + "prep_path", + "selected_lenses", + "lens_context_paths" + ] + }, + { + "step": "run_arch_lenses", + "available": [ + "arch_lenses", + "audit", + "auto_merge", + "base_branch", + "branch_name", + "claimed", + "cleanup_succeeded", + "depth", + "fixes_applied", + "implementation_ref", + "investigation_path", + "issue_number", + "issue_slug", + "issue_title", + "issue_url", + "lens_context_paths", + "max_parallel", + "merge_target", + "open_pr", + "phases_implemented", + "plan_path", + "prep_path", + "proposed_branch", + "remediation_path", + "remote_url", + "review_approach", + "review_max_retries", + "review_path", + "run_mode", + "run_name", + "selected_lenses", + "source_dir", + "task", + "upfront_claimed", + "verdict", + "work_dir", + "worktree_path" + ], + "required": [], + "produced": [] + }, + { + "step": "compose_pr", + "available": [ + "arch_lenses", + "audit", + "auto_merge", + "base_branch", + "branch_name", + "claimed", + "cleanup_succeeded", + "depth", + "fixes_applied", + "implementation_ref", + "investigation_path", + "issue_number", + "issue_slug", + "issue_title", + "issue_url", + "lens_context_paths", + "max_parallel", + "merge_target", + "open_pr", + "phases_implemented", + "plan_path", + "prep_path", + "proposed_branch", + "remediation_path", + "remote_url", + "review_approach", + "review_max_retries", + "review_path", + "run_mode", + "run_name", + "selected_lenses", + "source_dir", + "task", + "upfront_claimed", + "verdict", + "work_dir", + "worktree_path" + ], + "required": [], + "produced": [ + "pr_url" + ], + "positional_args": 1 + }, + { + "step": "extract_pr_number", + "available": [ + "arch_lenses", + "audit", + "auto_merge", + "base_branch", + "branch_name", + "claimed", + "cleanup_succeeded", + "depth", + "fixes_applied", + "implementation_ref", + "investigation_path", + "issue_number", + "issue_slug", + "issue_title", + "issue_url", + "lens_context_paths", + "max_parallel", + "merge_target", + "open_pr", + "phases_implemented", + "plan_path", + "pr_url", + "prep_path", + "proposed_branch", + "remediation_path", + "remote_url", + "review_approach", + "review_max_retries", + "review_path", + "run_mode", + "run_name", + "selected_lenses", + "source_dir", + "task", + "upfront_claimed", + "verdict", + "work_dir", + "worktree_path" + ], + "required": [], + "produced": [ + "pr_number" + ] + }, + { + "step": "annotate_pr_diff", + "available": [ + "arch_lenses", + "audit", + "auto_merge", + "base_branch", + "branch_name", + "claimed", + "cleanup_succeeded", + "depth", + "fixes_applied", + "implementation_ref", + "investigation_path", + "issue_number", + "issue_slug", + "issue_title", + "issue_url", + "lens_context_paths", + "max_parallel", + "merge_target", + "open_pr", + "phases_implemented", + "plan_path", + "pr_number", + "pr_url", + "prep_path", + "proposed_branch", + "remediation_path", + "remote_url", + "review_approach", + "review_max_retries", + "review_path", + "run_mode", + "run_name", + "selected_lenses", + "source_dir", + "task", + "upfront_claimed", + "verdict", + "work_dir", + "worktree_path" + ], + "required": [], + "produced": [ + "annotated_diff_path", + "diff_metrics_path", + "hunk_ranges_path", + "review_mode" + ] + }, + { + "step": "review_pr", + "available": [ + "annotated_diff_path", + "arch_lenses", + "audit", + "auto_merge", + "base_branch", + "branch_name", + "claimed", + "cleanup_succeeded", + "depth", + "fixes_applied", + "hunk_ranges_path", + "implementation_ref", + "investigation_path", + "issue_number", + "issue_slug", + "issue_title", + "issue_url", + "lens_context_paths", + "max_parallel", + "merge_target", + "open_pr", + "phases_implemented", + "plan_path", + "pr_number", + "pr_url", + "prep_path", + "proposed_branch", + "remediation_path", + "remote_url", + "review_approach", + "review_max_retries", + "review_path", + "run_mode", + "run_name", + "selected_lenses", + "source_dir", + "task", + "upfront_claimed", + "verdict", + "work_dir", + "worktree_path" + ], + "required": [], + "produced": [ + "review_verdict" + ], + "positional_args": 2 + }, + { + "step": "resolve_review", + "available": [ + "annotated_diff_path", + "arch_lenses", + "audit", + "auto_merge", + "base_branch", + "branch_name", + "claimed", + "cleanup_succeeded", + "depth", + "fixes_applied", + "hunk_ranges_path", + "implementation_ref", + "investigation_path", + "issue_number", + "issue_slug", + "issue_title", + "issue_url", + "lens_context_paths", + "max_parallel", + "merge_target", + "open_pr", + "phases_implemented", + "plan_path", + "pr_number", + "pr_url", + "prep_path", + "proposed_branch", + "remediation_path", + "remote_url", + "review_approach", + "review_max_retries", + "review_path", + "review_verdict", + "run_mode", + "run_name", + "selected_lenses", + "source_dir", + "task", + "upfront_claimed", + "verdict", + "work_dir", + "worktree_path" + ], + "required": [ + "feature_branch" + ], + "produced": [ + "verdict" + ] + }, + { + "step": "re_push_review", + "available": [ + "annotated_diff_path", + "arch_lenses", + "audit", + "auto_merge", + "base_branch", + "branch_name", + "claimed", + "cleanup_succeeded", + "depth", + "fixes_applied", + "hunk_ranges_path", + "implementation_ref", + "investigation_path", + "issue_number", + "issue_slug", + "issue_title", + "issue_url", + "lens_context_paths", + "max_parallel", + "merge_target", + "open_pr", + "phases_implemented", + "plan_path", + "pr_number", + "pr_url", + "prep_path", + "proposed_branch", + "remediation_path", + "remote_url", + "review_approach", + "review_max_retries", + "review_path", + "review_verdict", + "run_mode", + "run_name", + "selected_lenses", + "source_dir", + "task", + "upfront_claimed", + "verdict", + "work_dir", + "worktree_path" + ], + "required": [], + "produced": [] + }, + { + "step": "check_review_loop", + "available": [ + "annotated_diff_path", + "arch_lenses", + "audit", + "auto_merge", + "base_branch", + "branch_name", + "claimed", + "cleanup_succeeded", + "depth", + "fixes_applied", + "hunk_ranges_path", + "implementation_ref", + "investigation_path", + "issue_number", + "issue_slug", + "issue_title", + "issue_url", + "lens_context_paths", + "max_parallel", + "merge_target", + "open_pr", + "phases_implemented", + "plan_path", + "pr_number", + "pr_url", + "prep_path", + "proposed_branch", + "remediation_path", + "remote_url", + "review_approach", + "review_max_retries", + "review_path", + "review_verdict", + "run_mode", + "run_name", + "selected_lenses", + "source_dir", + "task", + "upfront_claimed", + "verdict", + "work_dir", + "worktree_path" + ], + "required": [], + "produced": [ + "review_loop_count" + ] + }, + { + "step": "check_repo_ci_event", + "available": [ + "annotated_diff_path", + "arch_lenses", + "audit", + "auto_merge", + "base_branch", + "branch_name", + "claimed", + "cleanup_succeeded", + "depth", + "fixes_applied", + "hunk_ranges_path", + "implementation_ref", + "investigation_path", + "issue_number", + "issue_slug", + "issue_title", + "issue_url", + "lens_context_paths", + "max_parallel", + "merge_target", + "open_pr", + "phases_implemented", + "plan_path", + "pr_number", + "pr_url", + "prep_path", + "proposed_branch", + "remediation_path", + "remote_url", + "review_approach", + "review_loop_count", + "review_max_retries", + "review_path", + "review_verdict", + "run_mode", + "run_name", + "selected_lenses", + "source_dir", + "task", + "upfront_claimed", + "verdict", + "work_dir", + "worktree_path" + ], + "required": [], + "produced": [ + "ci_event" + ] + }, + { + "step": "check_pr_state", + "available": [ + "annotated_diff_path", + "arch_lenses", + "audit", + "auto_merge", + "base_branch", + "branch_name", + "ci_event", + "claimed", + "cleanup_succeeded", + "depth", + "fixes_applied", + "hunk_ranges_path", + "implementation_ref", + "investigation_path", + "issue_number", + "issue_slug", + "issue_title", + "issue_url", + "lens_context_paths", + "max_parallel", + "merge_target", + "open_pr", + "phases_implemented", + "plan_path", + "pr_number", + "pr_url", + "prep_path", + "proposed_branch", + "remediation_path", + "remote_url", + "review_approach", + "review_loop_count", + "review_max_retries", + "review_path", + "review_verdict", + "run_mode", + "run_name", + "selected_lenses", + "source_dir", + "task", + "upfront_claimed", + "verdict", + "work_dir", + "worktree_path" + ], + "required": [], + "produced": [] + }, + { + "step": "ci_watch", + "available": [ + "annotated_diff_path", + "arch_lenses", + "audit", + "auto_merge", + "base_branch", + "branch_name", + "ci_event", + "claimed", + "cleanup_succeeded", + "depth", + "fixes_applied", + "hunk_ranges_path", + "implementation_ref", + "investigation_path", + "issue_number", + "issue_slug", + "issue_title", + "issue_url", + "lens_context_paths", + "max_parallel", + "merge_target", + "open_pr", + "phases_implemented", + "plan_path", + "pr_number", + "pr_url", + "prep_path", + "proposed_branch", + "remediation_path", + "remote_url", + "review_approach", + "review_loop_count", + "review_max_retries", + "review_path", + "review_verdict", + "run_mode", + "run_name", + "selected_lenses", + "source_dir", + "task", + "upfront_claimed", + "verdict", + "work_dir", + "worktree_path" + ], + "required": [], + "produced": [ + "ci_conclusion", + "ci_failed_jobs" + ] + }, + { + "step": "handle_no_ci_runs", + "available": [ + "annotated_diff_path", + "arch_lenses", + "audit", + "auto_merge", + "base_branch", + "branch_name", + "ci_conclusion", + "ci_event", + "ci_failed_jobs", + "claimed", + "cleanup_succeeded", + "depth", + "fixes_applied", + "hunk_ranges_path", + "implementation_ref", + "investigation_path", + "issue_number", + "issue_slug", + "issue_title", + "issue_url", + "lens_context_paths", + "max_parallel", + "merge_target", + "open_pr", + "phases_implemented", + "plan_path", + "pr_number", + "pr_url", + "prep_path", + "proposed_branch", + "remediation_path", + "remote_url", + "review_approach", + "review_loop_count", + "review_max_retries", + "review_path", + "review_verdict", + "run_mode", + "run_name", + "selected_lenses", + "source_dir", + "task", + "upfront_claimed", + "verdict", + "work_dir", + "worktree_path" + ], + "required": [], + "produced": [ + "ci_event" + ] + }, + { + "step": "check_ci_loop", + "available": [ + "annotated_diff_path", + "arch_lenses", + "audit", + "auto_merge", + "base_branch", + "branch_name", + "ci_conclusion", + "ci_event", + "ci_failed_jobs", + "claimed", + "cleanup_succeeded", + "depth", + "fixes_applied", + "hunk_ranges_path", + "implementation_ref", + "investigation_path", + "issue_number", + "issue_slug", + "issue_title", + "issue_url", + "lens_context_paths", + "max_parallel", + "merge_target", + "open_pr", + "phases_implemented", + "plan_path", + "pr_number", + "pr_url", + "prep_path", + "proposed_branch", + "remediation_path", + "remote_url", + "review_approach", + "review_loop_count", + "review_max_retries", + "review_path", + "review_verdict", + "run_mode", + "run_name", + "selected_lenses", + "source_dir", + "task", + "upfront_claimed", + "verdict", + "work_dir", + "worktree_path" + ], + "required": [], + "produced": [ + "ci_loop_count" + ] + }, + { + "step": "check_active_trigger_loop", + "available": [ + "annotated_diff_path", + "arch_lenses", + "audit", + "auto_merge", + "base_branch", + "branch_name", + "ci_conclusion", + "ci_event", + "ci_failed_jobs", + "ci_loop_count", + "claimed", + "cleanup_succeeded", + "depth", + "fixes_applied", + "hunk_ranges_path", + "implementation_ref", + "investigation_path", + "issue_number", + "issue_slug", + "issue_title", + "issue_url", + "lens_context_paths", + "max_parallel", + "merge_target", + "open_pr", + "phases_implemented", + "plan_path", + "pr_number", + "pr_url", + "prep_path", + "proposed_branch", + "remediation_path", + "remote_url", + "review_approach", + "review_loop_count", + "review_max_retries", + "review_path", + "review_verdict", + "run_mode", + "run_name", + "selected_lenses", + "source_dir", + "task", + "upfront_claimed", + "verdict", + "work_dir", + "worktree_path" + ], + "required": [], + "produced": [ + "active_trigger_count" + ] + }, + { + "step": "trigger_ci_actively", + "available": [ + "active_trigger_count", + "annotated_diff_path", + "arch_lenses", + "audit", + "auto_merge", + "base_branch", + "branch_name", + "ci_conclusion", + "ci_event", + "ci_failed_jobs", + "ci_loop_count", + "claimed", + "cleanup_succeeded", + "depth", + "fixes_applied", + "hunk_ranges_path", + "implementation_ref", + "investigation_path", + "issue_number", + "issue_slug", + "issue_title", + "issue_url", + "lens_context_paths", + "max_parallel", + "merge_target", + "open_pr", + "phases_implemented", + "plan_path", + "pr_number", + "pr_url", + "prep_path", + "proposed_branch", + "remediation_path", + "remote_url", + "review_approach", + "review_loop_count", + "review_max_retries", + "review_path", + "review_verdict", + "run_mode", + "run_name", + "selected_lenses", + "source_dir", + "task", + "upfront_claimed", + "verdict", + "work_dir", + "worktree_path" + ], + "required": [], + "produced": [] + }, + { + "step": "check_ci_already_passed", + "available": [ + "active_trigger_count", + "annotated_diff_path", + "arch_lenses", + "audit", + "auto_merge", + "base_branch", + "branch_name", + "ci_conclusion", + "ci_event", + "ci_failed_jobs", + "ci_loop_count", + "claimed", + "cleanup_succeeded", + "depth", + "fixes_applied", + "hunk_ranges_path", + "implementation_ref", + "investigation_path", + "issue_number", + "issue_slug", + "issue_title", + "issue_url", + "lens_context_paths", + "max_parallel", + "merge_target", + "open_pr", + "phases_implemented", + "plan_path", + "pr_number", + "pr_url", + "prep_path", + "proposed_branch", + "remediation_path", + "remote_url", + "review_approach", + "review_loop_count", + "review_max_retries", + "review_path", + "review_verdict", + "run_mode", + "run_name", + "selected_lenses", + "source_dir", + "task", + "upfront_claimed", + "verdict", + "work_dir", + "worktree_path" + ], + "required": [], + "produced": [] + }, + { + "step": "mark_issue_failed_no_ci", + "available": [ + "active_trigger_count", + "annotated_diff_path", + "arch_lenses", + "audit", + "auto_merge", + "base_branch", + "branch_name", + "ci_conclusion", + "ci_event", + "ci_failed_jobs", + "ci_loop_count", + "claimed", + "cleanup_succeeded", + "depth", + "fixes_applied", + "hunk_ranges_path", + "implementation_ref", + "investigation_path", + "issue_number", + "issue_slug", + "issue_title", + "issue_url", + "lens_context_paths", + "max_parallel", + "merge_target", + "open_pr", + "phases_implemented", + "plan_path", + "pr_number", + "pr_url", + "prep_path", + "proposed_branch", + "remediation_path", + "remote_url", + "review_approach", + "review_loop_count", + "review_max_retries", + "review_path", + "review_verdict", + "run_mode", + "run_name", + "selected_lenses", + "source_dir", + "task", + "upfront_claimed", + "verdict", + "work_dir", + "worktree_path" + ], + "required": [], + "produced": [] + }, + { + "step": "register_clone_no_ci", + "available": [ + "active_trigger_count", + "annotated_diff_path", + "arch_lenses", + "audit", + "auto_merge", + "base_branch", + "branch_name", + "ci_conclusion", + "ci_event", + "ci_failed_jobs", + "ci_loop_count", + "claimed", + "cleanup_succeeded", + "depth", + "fixes_applied", + "hunk_ranges_path", + "implementation_ref", + "investigation_path", + "issue_number", + "issue_slug", + "issue_title", + "issue_url", + "lens_context_paths", + "max_parallel", + "merge_target", + "open_pr", + "phases_implemented", + "plan_path", + "pr_number", + "pr_url", + "prep_path", + "proposed_branch", + "remediation_path", + "remote_url", + "review_approach", + "review_loop_count", + "review_max_retries", + "review_path", + "review_verdict", + "run_mode", + "run_name", + "selected_lenses", + "source_dir", + "task", + "upfront_claimed", + "verdict", + "work_dir", + "worktree_path" + ], + "required": [], + "produced": [] + }, + { + "step": "escalate_stop_no_ci", + "available": [ + "active_trigger_count", + "annotated_diff_path", + "arch_lenses", + "audit", + "auto_merge", + "base_branch", + "branch_name", + "ci_conclusion", + "ci_event", + "ci_failed_jobs", + "ci_loop_count", + "claimed", + "cleanup_succeeded", + "depth", + "fixes_applied", + "hunk_ranges_path", + "implementation_ref", + "investigation_path", + "issue_number", + "issue_slug", + "issue_title", + "issue_url", + "lens_context_paths", + "max_parallel", + "merge_target", + "open_pr", + "phases_implemented", + "plan_path", + "pr_number", + "pr_url", + "prep_path", + "proposed_branch", + "remediation_path", + "remote_url", + "review_approach", + "review_loop_count", + "review_max_retries", + "review_path", + "review_verdict", + "run_mode", + "run_name", + "selected_lenses", + "source_dir", + "task", + "upfront_claimed", + "verdict", + "work_dir", + "worktree_path" + ], + "required": [], + "produced": [] + }, + { + "step": "check_repo_merge_state", + "available": [ + "active_trigger_count", + "annotated_diff_path", + "arch_lenses", + "audit", + "auto_merge", + "base_branch", + "branch_name", + "ci_conclusion", + "ci_event", + "ci_failed_jobs", + "ci_loop_count", + "claimed", + "cleanup_succeeded", + "depth", + "fixes_applied", + "hunk_ranges_path", + "implementation_ref", + "investigation_path", + "issue_number", + "issue_slug", + "issue_title", + "issue_url", + "lens_context_paths", + "max_parallel", + "merge_target", + "open_pr", + "phases_implemented", + "plan_path", + "pr_number", + "pr_url", + "prep_path", + "proposed_branch", + "remediation_path", + "remote_url", + "review_approach", + "review_loop_count", + "review_max_retries", + "review_path", + "review_verdict", + "run_mode", + "run_name", + "selected_lenses", + "source_dir", + "task", + "upfront_claimed", + "verdict", + "work_dir", + "worktree_path" + ], + "required": [], + "produced": [ + "queue_available", + "merge_group_trigger", + "auto_merge_available", + "ci_event" + ] + }, + { + "step": "route_queue_mode", + "available": [ + "active_trigger_count", + "annotated_diff_path", + "arch_lenses", + "audit", + "auto_merge", + "auto_merge_available", + "base_branch", + "branch_name", + "ci_conclusion", + "ci_event", + "ci_failed_jobs", + "ci_loop_count", + "claimed", + "cleanup_succeeded", + "depth", + "fixes_applied", + "hunk_ranges_path", + "implementation_ref", + "investigation_path", + "issue_number", + "issue_slug", + "issue_title", + "issue_url", + "lens_context_paths", + "max_parallel", + "merge_group_trigger", + "merge_target", + "open_pr", + "phases_implemented", + "plan_path", + "pr_number", + "pr_url", + "prep_path", + "proposed_branch", + "queue_available", + "remediation_path", + "remote_url", + "review_approach", + "review_loop_count", + "review_max_retries", + "review_path", + "review_verdict", + "run_mode", + "run_name", + "selected_lenses", + "source_dir", + "task", + "upfront_claimed", + "verdict", + "work_dir", + "worktree_path" + ], + "required": [], + "produced": [] + }, + { + "step": "enqueue_to_queue", + "available": [ + "active_trigger_count", + "annotated_diff_path", + "arch_lenses", + "audit", + "auto_merge", + "auto_merge_available", + "base_branch", + "branch_name", + "ci_conclusion", + "ci_event", + "ci_failed_jobs", + "ci_loop_count", + "claimed", + "cleanup_succeeded", + "depth", + "fixes_applied", + "hunk_ranges_path", + "implementation_ref", + "investigation_path", + "issue_number", + "issue_slug", + "issue_title", + "issue_url", + "lens_context_paths", + "max_parallel", + "merge_group_trigger", + "merge_target", + "open_pr", + "phases_implemented", + "plan_path", + "pr_number", + "pr_url", + "prep_path", + "proposed_branch", + "queue_available", + "remediation_path", + "remote_url", + "review_approach", + "review_loop_count", + "review_max_retries", + "review_path", + "review_verdict", + "run_mode", + "run_name", + "selected_lenses", + "source_dir", + "task", + "upfront_claimed", + "verdict", + "work_dir", + "worktree_path" + ], + "required": [], + "produced": [] + }, + { + "step": "verify_queue_enrollment", + "available": [ + "active_trigger_count", + "annotated_diff_path", + "arch_lenses", + "audit", + "auto_merge", + "auto_merge_available", + "base_branch", + "branch_name", + "ci_conclusion", + "ci_event", + "ci_failed_jobs", + "ci_loop_count", + "claimed", + "cleanup_succeeded", + "depth", + "fixes_applied", + "hunk_ranges_path", + "implementation_ref", + "investigation_path", + "issue_number", + "issue_slug", + "issue_title", + "issue_url", + "lens_context_paths", + "max_parallel", + "merge_group_trigger", + "merge_target", + "open_pr", + "phases_implemented", + "plan_path", + "pr_number", + "pr_url", + "prep_path", + "proposed_branch", + "queue_available", + "remediation_path", + "remote_url", + "review_approach", + "review_loop_count", + "review_max_retries", + "review_path", + "review_verdict", + "run_mode", + "run_name", + "selected_lenses", + "source_dir", + "task", + "upfront_claimed", + "verdict", + "work_dir", + "worktree_path" + ], + "required": [], + "produced": [] + }, + { + "step": "wait_for_queue", + "available": [ + "active_trigger_count", + "annotated_diff_path", + "arch_lenses", + "audit", + "auto_merge", + "auto_merge_available", + "base_branch", + "branch_name", + "ci_conclusion", + "ci_event", + "ci_failed_jobs", + "ci_loop_count", + "claimed", + "cleanup_succeeded", + "depth", + "fixes_applied", + "hunk_ranges_path", + "implementation_ref", + "investigation_path", + "issue_number", + "issue_slug", + "issue_title", + "issue_url", + "lens_context_paths", + "max_parallel", + "merge_group_trigger", + "merge_target", + "open_pr", + "phases_implemented", + "plan_path", + "pr_number", + "pr_url", + "prep_path", + "proposed_branch", + "queue_available", + "remediation_path", + "remote_url", + "review_approach", + "review_loop_count", + "review_max_retries", + "review_path", + "review_verdict", + "run_mode", + "run_name", + "selected_lenses", + "source_dir", + "task", + "upfront_claimed", + "verdict", + "work_dir", + "worktree_path" + ], + "required": [], + "produced": [ + "queue_pr_state" + ] + }, + { + "step": "reenroll_stalled_pr", + "available": [ + "active_trigger_count", + "annotated_diff_path", + "arch_lenses", + "audit", + "auto_merge", + "auto_merge_available", + "base_branch", + "branch_name", + "ci_conclusion", + "ci_event", + "ci_failed_jobs", + "ci_loop_count", + "claimed", + "cleanup_succeeded", + "depth", + "fixes_applied", + "hunk_ranges_path", + "implementation_ref", + "investigation_path", + "issue_number", + "issue_slug", + "issue_title", + "issue_url", + "lens_context_paths", + "max_parallel", + "merge_group_trigger", + "merge_target", + "open_pr", + "phases_implemented", + "plan_path", + "pr_number", + "pr_url", + "prep_path", + "proposed_branch", + "queue_available", + "queue_pr_state", + "remediation_path", + "remote_url", + "review_approach", + "review_loop_count", + "review_max_retries", + "review_path", + "review_verdict", + "run_mode", + "run_name", + "selected_lenses", + "source_dir", + "task", + "upfront_claimed", + "verdict", + "work_dir", + "worktree_path" + ], + "required": [], + "produced": [] + }, + { + "step": "check_stall_loop", + "available": [ + "active_trigger_count", + "annotated_diff_path", + "arch_lenses", + "audit", + "auto_merge", + "auto_merge_available", + "base_branch", + "branch_name", + "ci_conclusion", + "ci_event", + "ci_failed_jobs", + "ci_loop_count", + "claimed", + "cleanup_succeeded", + "depth", + "fixes_applied", + "hunk_ranges_path", + "implementation_ref", + "investigation_path", + "issue_number", + "issue_slug", + "issue_title", + "issue_url", + "lens_context_paths", + "max_parallel", + "merge_group_trigger", + "merge_target", + "open_pr", + "phases_implemented", + "plan_path", + "pr_number", + "pr_url", + "prep_path", + "proposed_branch", + "queue_available", + "queue_pr_state", + "remediation_path", + "remote_url", + "review_approach", + "review_loop_count", + "review_max_retries", + "review_path", + "review_verdict", + "run_mode", + "run_name", + "selected_lenses", + "source_dir", + "task", + "upfront_claimed", + "verdict", + "work_dir", + "worktree_path" + ], + "required": [], + "produced": [ + "stall_loop_count" + ] + }, + { + "step": "check_eject_limit", + "available": [ + "active_trigger_count", + "annotated_diff_path", + "arch_lenses", + "audit", + "auto_merge", + "auto_merge_available", + "base_branch", + "branch_name", + "ci_conclusion", + "ci_event", + "ci_failed_jobs", + "ci_loop_count", + "claimed", + "cleanup_succeeded", + "depth", + "fixes_applied", + "hunk_ranges_path", + "implementation_ref", + "investigation_path", + "issue_number", + "issue_slug", + "issue_title", + "issue_url", + "lens_context_paths", + "max_parallel", + "merge_group_trigger", + "merge_target", + "open_pr", + "phases_implemented", + "plan_path", + "pr_number", + "pr_url", + "prep_path", + "proposed_branch", + "queue_available", + "queue_pr_state", + "remediation_path", + "remote_url", + "review_approach", + "review_loop_count", + "review_max_retries", + "review_path", + "review_verdict", + "run_mode", + "run_name", + "selected_lenses", + "source_dir", + "stall_loop_count", + "task", + "upfront_claimed", + "verdict", + "work_dir", + "worktree_path" + ], + "required": [], + "produced": [] + }, + { + "step": "check_dropped_healthy_loop", + "available": [ + "active_trigger_count", + "annotated_diff_path", + "arch_lenses", + "audit", + "auto_merge", + "auto_merge_available", + "base_branch", + "branch_name", + "ci_conclusion", + "ci_event", + "ci_failed_jobs", + "ci_loop_count", + "claimed", + "cleanup_succeeded", + "depth", + "fixes_applied", + "hunk_ranges_path", + "implementation_ref", + "investigation_path", + "issue_number", + "issue_slug", + "issue_title", + "issue_url", + "lens_context_paths", + "max_parallel", + "merge_group_trigger", + "merge_target", + "open_pr", + "phases_implemented", + "plan_path", + "pr_number", + "pr_url", + "prep_path", + "proposed_branch", + "queue_available", + "queue_pr_state", + "remediation_path", + "remote_url", + "review_approach", + "review_loop_count", + "review_max_retries", + "review_path", + "review_verdict", + "run_mode", + "run_name", + "selected_lenses", + "source_dir", + "stall_loop_count", + "task", + "upfront_claimed", + "verdict", + "work_dir", + "worktree_path" + ], + "required": [], + "produced": [] + }, + { + "step": "queue_ejected_fix", + "available": [ + "active_trigger_count", + "annotated_diff_path", + "arch_lenses", + "audit", + "auto_merge", + "auto_merge_available", + "base_branch", + "branch_name", + "ci_conclusion", + "ci_event", + "ci_failed_jobs", + "ci_loop_count", + "claimed", + "cleanup_succeeded", + "depth", + "fixes_applied", + "hunk_ranges_path", + "implementation_ref", + "investigation_path", + "issue_number", + "issue_slug", + "issue_title", + "issue_url", + "lens_context_paths", + "max_parallel", + "merge_group_trigger", + "merge_target", + "open_pr", + "phases_implemented", + "plan_path", + "pr_number", + "pr_url", + "prep_path", + "proposed_branch", + "queue_available", + "queue_pr_state", + "remediation_path", + "remote_url", + "review_approach", + "review_loop_count", + "review_max_retries", + "review_path", + "review_verdict", + "run_mode", + "run_name", + "selected_lenses", + "source_dir", + "stall_loop_count", + "task", + "upfront_claimed", + "verdict", + "work_dir", + "worktree_path" + ], + "required": [], + "produced": [] + }, + { + "step": "resolve_queue_merge_conflicts", + "available": [ + "active_trigger_count", + "annotated_diff_path", + "arch_lenses", + "audit", + "auto_merge", + "auto_merge_available", + "base_branch", + "branch_name", + "ci_conclusion", + "ci_event", + "ci_failed_jobs", + "ci_loop_count", + "claimed", + "cleanup_succeeded", + "depth", + "fixes_applied", + "hunk_ranges_path", + "implementation_ref", + "investigation_path", + "issue_number", + "issue_slug", + "issue_title", + "issue_url", + "lens_context_paths", + "max_parallel", + "merge_group_trigger", + "merge_target", + "open_pr", + "phases_implemented", + "plan_path", + "pr_number", + "pr_url", + "prep_path", + "proposed_branch", + "queue_available", + "queue_pr_state", + "remediation_path", + "remote_url", + "review_approach", + "review_loop_count", + "review_max_retries", + "review_path", + "review_verdict", + "run_mode", + "run_name", + "selected_lenses", + "source_dir", + "stall_loop_count", + "task", + "upfront_claimed", + "verdict", + "work_dir", + "worktree_path" + ], + "required": [ + "worktree_path" + ], + "produced": [ + "conflict_escalation_required" + ] + }, + { + "step": "re_push_queue_fix", + "available": [ + "active_trigger_count", + "annotated_diff_path", + "arch_lenses", + "audit", + "auto_merge", + "auto_merge_available", + "base_branch", + "branch_name", + "ci_conclusion", + "ci_event", + "ci_failed_jobs", + "ci_loop_count", + "claimed", + "cleanup_succeeded", + "conflict_escalation_required", + "depth", + "fixes_applied", + "hunk_ranges_path", + "implementation_ref", + "investigation_path", + "issue_number", + "issue_slug", + "issue_title", + "issue_url", + "lens_context_paths", + "max_parallel", + "merge_group_trigger", + "merge_target", + "open_pr", + "phases_implemented", + "plan_path", + "pr_number", + "pr_url", + "prep_path", + "proposed_branch", + "queue_available", + "queue_pr_state", + "remediation_path", + "remote_url", + "review_approach", + "review_loop_count", + "review_max_retries", + "review_path", + "review_verdict", + "run_mode", + "run_name", + "selected_lenses", + "source_dir", + "stall_loop_count", + "task", + "upfront_claimed", + "verdict", + "work_dir", + "worktree_path" + ], + "required": [], + "produced": [] + }, + { + "step": "ci_watch_post_queue_fix", + "available": [ + "active_trigger_count", + "annotated_diff_path", + "arch_lenses", + "audit", + "auto_merge", + "auto_merge_available", + "base_branch", + "branch_name", + "ci_conclusion", + "ci_event", + "ci_failed_jobs", + "ci_loop_count", + "claimed", + "cleanup_succeeded", + "conflict_escalation_required", + "depth", + "fixes_applied", + "hunk_ranges_path", + "implementation_ref", + "investigation_path", + "issue_number", + "issue_slug", + "issue_title", + "issue_url", + "lens_context_paths", + "max_parallel", + "merge_group_trigger", + "merge_target", + "open_pr", + "phases_implemented", + "plan_path", + "pr_number", + "pr_url", + "prep_path", + "proposed_branch", + "queue_available", + "queue_pr_state", + "remediation_path", + "remote_url", + "review_approach", + "review_loop_count", + "review_max_retries", + "review_path", + "review_verdict", + "run_mode", + "run_name", + "selected_lenses", + "source_dir", + "stall_loop_count", + "task", + "upfront_claimed", + "verdict", + "work_dir", + "worktree_path" + ], + "required": [], + "produced": [ + "ci_conclusion", + "ci_failed_jobs" + ] + }, + { + "step": "check_ci_post_queue_loop", + "available": [ + "active_trigger_count", + "annotated_diff_path", + "arch_lenses", + "audit", + "auto_merge", + "auto_merge_available", + "base_branch", + "branch_name", + "ci_conclusion", + "ci_event", + "ci_failed_jobs", + "ci_loop_count", + "claimed", + "cleanup_succeeded", + "conflict_escalation_required", + "depth", + "fixes_applied", + "hunk_ranges_path", + "implementation_ref", + "investigation_path", + "issue_number", + "issue_slug", + "issue_title", + "issue_url", + "lens_context_paths", + "max_parallel", + "merge_group_trigger", + "merge_target", + "open_pr", + "phases_implemented", + "plan_path", + "pr_number", + "pr_url", + "prep_path", + "proposed_branch", + "queue_available", + "queue_pr_state", + "remediation_path", + "remote_url", + "review_approach", + "review_loop_count", + "review_max_retries", + "review_path", + "review_verdict", + "run_mode", + "run_name", + "selected_lenses", + "source_dir", + "stall_loop_count", + "task", + "upfront_claimed", + "verdict", + "work_dir", + "worktree_path" + ], + "required": [], + "produced": [ + "ci_post_queue_loop_count" + ] + }, + { + "step": "reenter_merge_queue", + "available": [ + "active_trigger_count", + "annotated_diff_path", + "arch_lenses", + "audit", + "auto_merge", + "auto_merge_available", + "base_branch", + "branch_name", + "ci_conclusion", + "ci_event", + "ci_failed_jobs", + "ci_loop_count", + "ci_post_queue_loop_count", + "claimed", + "cleanup_succeeded", + "conflict_escalation_required", + "depth", + "fixes_applied", + "hunk_ranges_path", + "implementation_ref", + "investigation_path", + "issue_number", + "issue_slug", + "issue_title", + "issue_url", + "lens_context_paths", + "max_parallel", + "merge_group_trigger", + "merge_target", + "open_pr", + "phases_implemented", + "plan_path", + "pr_number", + "pr_url", + "prep_path", + "proposed_branch", + "queue_available", + "queue_pr_state", + "remediation_path", + "remote_url", + "review_approach", + "review_loop_count", + "review_max_retries", + "review_path", + "review_verdict", + "run_mode", + "run_name", + "selected_lenses", + "source_dir", + "stall_loop_count", + "task", + "upfront_claimed", + "verdict", + "work_dir", + "worktree_path" + ], + "required": [], + "produced": [] + }, + { + "step": "reenter_merge_queue_cheap", + "available": [ + "active_trigger_count", + "annotated_diff_path", + "arch_lenses", + "audit", + "auto_merge", + "auto_merge_available", + "base_branch", + "branch_name", + "ci_conclusion", + "ci_event", + "ci_failed_jobs", + "ci_loop_count", + "ci_post_queue_loop_count", + "claimed", + "cleanup_succeeded", + "conflict_escalation_required", + "depth", + "fixes_applied", + "hunk_ranges_path", + "implementation_ref", + "investigation_path", + "issue_number", + "issue_slug", + "issue_title", + "issue_url", + "lens_context_paths", + "max_parallel", + "merge_group_trigger", + "merge_target", + "open_pr", + "phases_implemented", + "plan_path", + "pr_number", + "pr_url", + "prep_path", + "proposed_branch", + "queue_available", + "queue_pr_state", + "remediation_path", + "remote_url", + "review_approach", + "review_loop_count", + "review_max_retries", + "review_path", + "review_verdict", + "run_mode", + "run_name", + "selected_lenses", + "source_dir", + "stall_loop_count", + "task", + "upfront_claimed", + "verdict", + "work_dir", + "worktree_path" + ], + "required": [], + "produced": [] + }, + { + "step": "direct_merge", + "available": [ + "active_trigger_count", + "annotated_diff_path", + "arch_lenses", + "audit", + "auto_merge", + "auto_merge_available", + "base_branch", + "branch_name", + "ci_conclusion", + "ci_event", + "ci_failed_jobs", + "ci_loop_count", + "ci_post_queue_loop_count", + "claimed", + "cleanup_succeeded", + "conflict_escalation_required", + "depth", + "fixes_applied", + "hunk_ranges_path", + "implementation_ref", + "investigation_path", + "issue_number", + "issue_slug", + "issue_title", + "issue_url", + "lens_context_paths", + "max_parallel", + "merge_group_trigger", + "merge_target", + "open_pr", + "phases_implemented", + "plan_path", + "pr_number", + "pr_url", + "prep_path", + "proposed_branch", + "queue_available", + "queue_pr_state", + "remediation_path", + "remote_url", + "review_approach", + "review_loop_count", + "review_max_retries", + "review_path", + "review_verdict", + "run_mode", + "run_name", + "selected_lenses", + "source_dir", + "stall_loop_count", + "task", + "upfront_claimed", + "verdict", + "work_dir", + "worktree_path" + ], + "required": [], + "produced": [] + }, + { + "step": "wait_for_direct_merge", + "available": [ + "active_trigger_count", + "annotated_diff_path", + "arch_lenses", + "audit", + "auto_merge", + "auto_merge_available", + "base_branch", + "branch_name", + "ci_conclusion", + "ci_event", + "ci_failed_jobs", + "ci_loop_count", + "ci_post_queue_loop_count", + "claimed", + "cleanup_succeeded", + "conflict_escalation_required", + "depth", + "fixes_applied", + "hunk_ranges_path", + "implementation_ref", + "investigation_path", + "issue_number", + "issue_slug", + "issue_title", + "issue_url", + "lens_context_paths", + "max_parallel", + "merge_group_trigger", + "merge_target", + "open_pr", + "phases_implemented", + "plan_path", + "pr_number", + "pr_url", + "prep_path", + "proposed_branch", + "queue_available", + "queue_pr_state", + "remediation_path", + "remote_url", + "review_approach", + "review_loop_count", + "review_max_retries", + "review_path", + "review_verdict", + "run_mode", + "run_name", + "selected_lenses", + "source_dir", + "stall_loop_count", + "task", + "upfront_claimed", + "verdict", + "work_dir", + "worktree_path" + ], + "required": [], + "produced": [] + }, + { + "step": "direct_merge_conflict_fix", + "available": [ + "active_trigger_count", + "annotated_diff_path", + "arch_lenses", + "audit", + "auto_merge", + "auto_merge_available", + "base_branch", + "branch_name", + "ci_conclusion", + "ci_event", + "ci_failed_jobs", + "ci_loop_count", + "ci_post_queue_loop_count", + "claimed", + "cleanup_succeeded", + "conflict_escalation_required", + "depth", + "fixes_applied", + "hunk_ranges_path", + "implementation_ref", + "investigation_path", + "issue_number", + "issue_slug", + "issue_title", + "issue_url", + "lens_context_paths", + "max_parallel", + "merge_group_trigger", + "merge_target", + "open_pr", + "phases_implemented", + "plan_path", + "pr_number", + "pr_url", + "prep_path", + "proposed_branch", + "queue_available", + "queue_pr_state", + "remediation_path", + "remote_url", + "review_approach", + "review_loop_count", + "review_max_retries", + "review_path", + "review_verdict", + "run_mode", + "run_name", + "selected_lenses", + "source_dir", + "stall_loop_count", + "task", + "upfront_claimed", + "verdict", + "work_dir", + "worktree_path" + ], + "required": [], + "produced": [] + }, + { + "step": "resolve_direct_merge_conflicts", + "available": [ + "active_trigger_count", + "annotated_diff_path", + "arch_lenses", + "audit", + "auto_merge", + "auto_merge_available", + "base_branch", + "branch_name", + "ci_conclusion", + "ci_event", + "ci_failed_jobs", + "ci_loop_count", + "ci_post_queue_loop_count", + "claimed", + "cleanup_succeeded", + "conflict_escalation_required", + "depth", + "fixes_applied", + "hunk_ranges_path", + "implementation_ref", + "investigation_path", + "issue_number", + "issue_slug", + "issue_title", + "issue_url", + "lens_context_paths", + "max_parallel", + "merge_group_trigger", + "merge_target", + "open_pr", + "phases_implemented", + "plan_path", + "pr_number", + "pr_url", + "prep_path", + "proposed_branch", + "queue_available", + "queue_pr_state", + "remediation_path", + "remote_url", + "review_approach", + "review_loop_count", + "review_max_retries", + "review_path", + "review_verdict", + "run_mode", + "run_name", + "selected_lenses", + "source_dir", + "stall_loop_count", + "task", + "upfront_claimed", + "verdict", + "work_dir", + "worktree_path" + ], + "required": [ + "worktree_path" + ], + "produced": [ + "conflict_escalation_required" + ] + }, + { + "step": "re_push_direct_fix", + "available": [ + "active_trigger_count", + "annotated_diff_path", + "arch_lenses", + "audit", + "auto_merge", + "auto_merge_available", + "base_branch", + "branch_name", + "ci_conclusion", + "ci_event", + "ci_failed_jobs", + "ci_loop_count", + "ci_post_queue_loop_count", + "claimed", + "cleanup_succeeded", + "conflict_escalation_required", + "depth", + "fixes_applied", + "hunk_ranges_path", + "implementation_ref", + "investigation_path", + "issue_number", + "issue_slug", + "issue_title", + "issue_url", + "lens_context_paths", + "max_parallel", + "merge_group_trigger", + "merge_target", + "open_pr", + "phases_implemented", + "plan_path", + "pr_number", + "pr_url", + "prep_path", + "proposed_branch", + "queue_available", + "queue_pr_state", + "remediation_path", + "remote_url", + "review_approach", + "review_loop_count", + "review_max_retries", + "review_path", + "review_verdict", + "run_mode", + "run_name", + "selected_lenses", + "source_dir", + "stall_loop_count", + "task", + "upfront_claimed", + "verdict", + "work_dir", + "worktree_path" + ], + "required": [], + "produced": [] + }, + { + "step": "redirect_merge", + "available": [ + "active_trigger_count", + "annotated_diff_path", + "arch_lenses", + "audit", + "auto_merge", + "auto_merge_available", + "base_branch", + "branch_name", + "ci_conclusion", + "ci_event", + "ci_failed_jobs", + "ci_loop_count", + "ci_post_queue_loop_count", + "claimed", + "cleanup_succeeded", + "conflict_escalation_required", + "depth", + "fixes_applied", + "hunk_ranges_path", + "implementation_ref", + "investigation_path", + "issue_number", + "issue_slug", + "issue_title", + "issue_url", + "lens_context_paths", + "max_parallel", + "merge_group_trigger", + "merge_target", + "open_pr", + "phases_implemented", + "plan_path", + "pr_number", + "pr_url", + "prep_path", + "proposed_branch", + "queue_available", + "queue_pr_state", + "remediation_path", + "remote_url", + "review_approach", + "review_loop_count", + "review_max_retries", + "review_path", + "review_verdict", + "run_mode", + "run_name", + "selected_lenses", + "source_dir", + "stall_loop_count", + "task", + "upfront_claimed", + "verdict", + "work_dir", + "worktree_path" + ], + "required": [], + "produced": [] + }, + { + "step": "immediate_merge", + "available": [ + "active_trigger_count", + "annotated_diff_path", + "arch_lenses", + "audit", + "auto_merge", + "auto_merge_available", + "base_branch", + "branch_name", + "ci_conclusion", + "ci_event", + "ci_failed_jobs", + "ci_loop_count", + "ci_post_queue_loop_count", + "claimed", + "cleanup_succeeded", + "conflict_escalation_required", + "depth", + "fixes_applied", + "hunk_ranges_path", + "implementation_ref", + "investigation_path", + "issue_number", + "issue_slug", + "issue_title", + "issue_url", + "lens_context_paths", + "max_parallel", + "merge_group_trigger", + "merge_target", + "open_pr", + "phases_implemented", + "plan_path", + "pr_number", + "pr_url", + "prep_path", + "proposed_branch", + "queue_available", + "queue_pr_state", + "remediation_path", + "remote_url", + "review_approach", + "review_loop_count", + "review_max_retries", + "review_path", + "review_verdict", + "run_mode", + "run_name", + "selected_lenses", + "source_dir", + "stall_loop_count", + "task", + "upfront_claimed", + "verdict", + "work_dir", + "worktree_path" + ], + "required": [], + "produced": [] + }, + { + "step": "wait_for_immediate_merge", + "available": [ + "active_trigger_count", + "annotated_diff_path", + "arch_lenses", + "audit", + "auto_merge", + "auto_merge_available", + "base_branch", + "branch_name", + "ci_conclusion", + "ci_event", + "ci_failed_jobs", + "ci_loop_count", + "ci_post_queue_loop_count", + "claimed", + "cleanup_succeeded", + "conflict_escalation_required", + "depth", + "fixes_applied", + "hunk_ranges_path", + "implementation_ref", + "investigation_path", + "issue_number", + "issue_slug", + "issue_title", + "issue_url", + "lens_context_paths", + "max_parallel", + "merge_group_trigger", + "merge_target", + "open_pr", + "phases_implemented", + "plan_path", + "pr_number", + "pr_url", + "prep_path", + "proposed_branch", + "queue_available", + "queue_pr_state", + "remediation_path", + "remote_url", + "review_approach", + "review_loop_count", + "review_max_retries", + "review_path", + "review_verdict", + "run_mode", + "run_name", + "selected_lenses", + "source_dir", + "stall_loop_count", + "task", + "upfront_claimed", + "verdict", + "work_dir", + "worktree_path" + ], + "required": [], + "produced": [] + }, + { + "step": "immediate_merge_conflict_fix", + "available": [ + "active_trigger_count", + "annotated_diff_path", + "arch_lenses", + "audit", + "auto_merge", + "auto_merge_available", + "base_branch", + "branch_name", + "ci_conclusion", + "ci_event", + "ci_failed_jobs", + "ci_loop_count", + "ci_post_queue_loop_count", + "claimed", + "cleanup_succeeded", + "conflict_escalation_required", + "depth", + "fixes_applied", + "hunk_ranges_path", + "implementation_ref", + "investigation_path", + "issue_number", + "issue_slug", + "issue_title", + "issue_url", + "lens_context_paths", + "max_parallel", + "merge_group_trigger", + "merge_target", + "open_pr", + "phases_implemented", + "plan_path", + "pr_number", + "pr_url", + "prep_path", + "proposed_branch", + "queue_available", + "queue_pr_state", + "remediation_path", + "remote_url", + "review_approach", + "review_loop_count", + "review_max_retries", + "review_path", + "review_verdict", + "run_mode", + "run_name", + "selected_lenses", + "source_dir", + "stall_loop_count", + "task", + "upfront_claimed", + "verdict", + "work_dir", + "worktree_path" + ], + "required": [], + "produced": [] + }, + { + "step": "resolve_immediate_merge_conflicts", + "available": [ + "active_trigger_count", + "annotated_diff_path", + "arch_lenses", + "audit", + "auto_merge", + "auto_merge_available", + "base_branch", + "branch_name", + "ci_conclusion", + "ci_event", + "ci_failed_jobs", + "ci_loop_count", + "ci_post_queue_loop_count", + "claimed", + "cleanup_succeeded", + "conflict_escalation_required", + "depth", + "fixes_applied", + "hunk_ranges_path", + "implementation_ref", + "investigation_path", + "issue_number", + "issue_slug", + "issue_title", + "issue_url", + "lens_context_paths", + "max_parallel", + "merge_group_trigger", + "merge_target", + "open_pr", + "phases_implemented", + "plan_path", + "pr_number", + "pr_url", + "prep_path", + "proposed_branch", + "queue_available", + "queue_pr_state", + "remediation_path", + "remote_url", + "review_approach", + "review_loop_count", + "review_max_retries", + "review_path", + "review_verdict", + "run_mode", + "run_name", + "selected_lenses", + "source_dir", + "stall_loop_count", + "task", + "upfront_claimed", + "verdict", + "work_dir", + "worktree_path" + ], + "required": [ + "worktree_path" + ], + "produced": [ + "conflict_escalation_required" + ] + }, + { + "step": "re_push_immediate_fix", + "available": [ + "active_trigger_count", + "annotated_diff_path", + "arch_lenses", + "audit", + "auto_merge", + "auto_merge_available", + "base_branch", + "branch_name", + "ci_conclusion", + "ci_event", + "ci_failed_jobs", + "ci_loop_count", + "ci_post_queue_loop_count", + "claimed", + "cleanup_succeeded", + "conflict_escalation_required", + "depth", + "fixes_applied", + "hunk_ranges_path", + "implementation_ref", + "investigation_path", + "issue_number", + "issue_slug", + "issue_title", + "issue_url", + "lens_context_paths", + "max_parallel", + "merge_group_trigger", + "merge_target", + "open_pr", + "phases_implemented", + "plan_path", + "pr_number", + "pr_url", + "prep_path", + "proposed_branch", + "queue_available", + "queue_pr_state", + "remediation_path", + "remote_url", + "review_approach", + "review_loop_count", + "review_max_retries", + "review_path", + "review_verdict", + "run_mode", + "run_name", + "selected_lenses", + "source_dir", + "stall_loop_count", + "task", + "upfront_claimed", + "verdict", + "work_dir", + "worktree_path" + ], + "required": [], + "produced": [] + }, + { + "step": "remerge_immediate", + "available": [ + "active_trigger_count", + "annotated_diff_path", + "arch_lenses", + "audit", + "auto_merge", + "auto_merge_available", + "base_branch", + "branch_name", + "ci_conclusion", + "ci_event", + "ci_failed_jobs", + "ci_loop_count", + "ci_post_queue_loop_count", + "claimed", + "cleanup_succeeded", + "conflict_escalation_required", + "depth", + "fixes_applied", + "hunk_ranges_path", + "implementation_ref", + "investigation_path", + "issue_number", + "issue_slug", + "issue_title", + "issue_url", + "lens_context_paths", + "max_parallel", + "merge_group_trigger", + "merge_target", + "open_pr", + "phases_implemented", + "plan_path", + "pr_number", + "pr_url", + "prep_path", + "proposed_branch", + "queue_available", + "queue_pr_state", + "remediation_path", + "remote_url", + "review_approach", + "review_loop_count", + "review_max_retries", + "review_path", + "review_verdict", + "run_mode", + "run_name", + "selected_lenses", + "source_dir", + "stall_loop_count", + "task", + "upfront_claimed", + "verdict", + "work_dir", + "worktree_path" + ], + "required": [], + "produced": [] + }, + { + "step": "diagnose_ci", + "available": [ + "active_trigger_count", + "annotated_diff_path", + "arch_lenses", + "audit", + "auto_merge", + "auto_merge_available", + "base_branch", + "branch_name", + "ci_conclusion", + "ci_event", + "ci_failed_jobs", + "ci_loop_count", + "ci_post_queue_loop_count", + "claimed", + "cleanup_succeeded", + "conflict_escalation_required", + "depth", + "fixes_applied", + "hunk_ranges_path", + "implementation_ref", + "investigation_path", + "issue_number", + "issue_slug", + "issue_title", + "issue_url", + "lens_context_paths", + "max_parallel", + "merge_group_trigger", + "merge_target", + "open_pr", + "phases_implemented", + "plan_path", + "pr_number", + "pr_url", + "prep_path", + "proposed_branch", + "queue_available", + "queue_pr_state", + "remediation_path", + "remote_url", + "review_approach", + "review_loop_count", + "review_max_retries", + "review_path", + "review_verdict", + "run_mode", + "run_name", + "selected_lenses", + "source_dir", + "stall_loop_count", + "task", + "upfront_claimed", + "verdict", + "work_dir", + "worktree_path" + ], + "required": [], + "produced": [ + "diagnosis_path" + ], + "positional_args": 3 + }, + { + "step": "resolve_ci", + "available": [ + "active_trigger_count", + "annotated_diff_path", + "arch_lenses", + "audit", + "auto_merge", + "auto_merge_available", + "base_branch", + "branch_name", + "ci_conclusion", + "ci_event", + "ci_failed_jobs", + "ci_loop_count", + "ci_post_queue_loop_count", + "claimed", + "cleanup_succeeded", + "conflict_escalation_required", + "depth", + "diagnosis_path", + "fixes_applied", + "hunk_ranges_path", + "implementation_ref", + "investigation_path", + "issue_number", + "issue_slug", + "issue_title", + "issue_url", + "lens_context_paths", + "max_parallel", + "merge_group_trigger", + "merge_target", + "open_pr", + "phases_implemented", + "plan_path", + "pr_number", + "pr_url", + "prep_path", + "proposed_branch", + "queue_available", + "queue_pr_state", + "remediation_path", + "remote_url", + "review_approach", + "review_loop_count", + "review_max_retries", + "review_path", + "review_verdict", + "run_mode", + "run_name", + "selected_lenses", + "source_dir", + "stall_loop_count", + "task", + "upfront_claimed", + "verdict", + "work_dir", + "worktree_path" + ], + "required": [ + "worktree_path" + ], + "produced": [ + "verdict" + ] + }, + { + "step": "pre_resolve_rebase", + "available": [ + "active_trigger_count", + "annotated_diff_path", + "arch_lenses", + "audit", + "auto_merge", + "auto_merge_available", + "base_branch", + "branch_name", + "ci_conclusion", + "ci_event", + "ci_failed_jobs", + "ci_loop_count", + "ci_post_queue_loop_count", + "claimed", + "cleanup_succeeded", + "conflict_escalation_required", + "depth", + "diagnosis_path", + "fixes_applied", + "hunk_ranges_path", + "implementation_ref", + "investigation_path", + "issue_number", + "issue_slug", + "issue_title", + "issue_url", + "lens_context_paths", + "max_parallel", + "merge_group_trigger", + "merge_target", + "open_pr", + "phases_implemented", + "plan_path", + "pr_number", + "pr_url", + "prep_path", + "proposed_branch", + "queue_available", + "queue_pr_state", + "remediation_path", + "remote_url", + "review_approach", + "review_loop_count", + "review_max_retries", + "review_path", + "review_verdict", + "run_mode", + "run_name", + "selected_lenses", + "source_dir", + "stall_loop_count", + "task", + "upfront_claimed", + "verdict", + "work_dir", + "worktree_path" + ], + "required": [], + "produced": [] + }, + { + "step": "re_push", + "available": [ + "active_trigger_count", + "annotated_diff_path", + "arch_lenses", + "audit", + "auto_merge", + "auto_merge_available", + "base_branch", + "branch_name", + "ci_conclusion", + "ci_event", + "ci_failed_jobs", + "ci_loop_count", + "ci_post_queue_loop_count", + "claimed", + "cleanup_succeeded", + "conflict_escalation_required", + "depth", + "diagnosis_path", + "fixes_applied", + "hunk_ranges_path", + "implementation_ref", + "investigation_path", + "issue_number", + "issue_slug", + "issue_title", + "issue_url", + "lens_context_paths", + "max_parallel", + "merge_group_trigger", + "merge_target", + "open_pr", + "phases_implemented", + "plan_path", + "pr_number", + "pr_url", + "prep_path", + "proposed_branch", + "queue_available", + "queue_pr_state", + "remediation_path", + "remote_url", + "review_approach", + "review_loop_count", + "review_max_retries", + "review_path", + "review_verdict", + "run_mode", + "run_name", + "selected_lenses", + "source_dir", + "stall_loop_count", + "task", + "upfront_claimed", + "verdict", + "work_dir", + "worktree_path" + ], + "required": [], + "produced": [] + }, + { + "step": "detect_ci_conflict", + "available": [ + "active_trigger_count", + "annotated_diff_path", + "arch_lenses", + "audit", + "auto_merge", + "auto_merge_available", + "base_branch", + "branch_name", + "ci_conclusion", + "ci_event", + "ci_failed_jobs", + "ci_loop_count", + "ci_post_queue_loop_count", + "claimed", + "cleanup_succeeded", + "conflict_escalation_required", + "depth", + "diagnosis_path", + "fixes_applied", + "hunk_ranges_path", + "implementation_ref", + "investigation_path", + "issue_number", + "issue_slug", + "issue_title", + "issue_url", + "lens_context_paths", + "max_parallel", + "merge_group_trigger", + "merge_target", + "open_pr", + "phases_implemented", + "plan_path", + "pr_number", + "pr_url", + "prep_path", + "proposed_branch", + "queue_available", + "queue_pr_state", + "remediation_path", + "remote_url", + "review_approach", + "review_loop_count", + "review_max_retries", + "review_path", + "review_verdict", + "run_mode", + "run_name", + "selected_lenses", + "source_dir", + "stall_loop_count", + "task", + "upfront_claimed", + "verdict", + "work_dir", + "worktree_path" + ], + "required": [], + "produced": [] + }, + { + "step": "ci_conflict_fix", + "available": [ + "active_trigger_count", + "annotated_diff_path", + "arch_lenses", + "audit", + "auto_merge", + "auto_merge_available", + "base_branch", + "branch_name", + "ci_conclusion", + "ci_event", + "ci_failed_jobs", + "ci_loop_count", + "ci_post_queue_loop_count", + "claimed", + "cleanup_succeeded", + "conflict_escalation_required", + "depth", + "diagnosis_path", + "fixes_applied", + "hunk_ranges_path", + "implementation_ref", + "investigation_path", + "issue_number", + "issue_slug", + "issue_title", + "issue_url", + "lens_context_paths", + "max_parallel", + "merge_group_trigger", + "merge_target", + "open_pr", + "phases_implemented", + "plan_path", + "pr_number", + "pr_url", + "prep_path", + "proposed_branch", + "queue_available", + "queue_pr_state", + "remediation_path", + "remote_url", + "review_approach", + "review_loop_count", + "review_max_retries", + "review_path", + "review_verdict", + "run_mode", + "run_name", + "selected_lenses", + "source_dir", + "stall_loop_count", + "task", + "upfront_claimed", + "verdict", + "work_dir", + "worktree_path" + ], + "required": [ + "worktree_path" + ], + "produced": [ + "conflict_escalation_required" + ] + }, + { + "step": "register_clone_unconfirmed", + "available": [ + "active_trigger_count", + "annotated_diff_path", + "arch_lenses", + "audit", + "auto_merge", + "auto_merge_available", + "base_branch", + "branch_name", + "ci_conclusion", + "ci_event", + "ci_failed_jobs", + "ci_loop_count", + "ci_post_queue_loop_count", + "claimed", + "cleanup_succeeded", + "conflict_escalation_required", + "depth", + "diagnosis_path", + "fixes_applied", + "hunk_ranges_path", + "implementation_ref", + "investigation_path", + "issue_number", + "issue_slug", + "issue_title", + "issue_url", + "lens_context_paths", + "max_parallel", + "merge_group_trigger", + "merge_target", + "open_pr", + "phases_implemented", + "plan_path", + "pr_number", + "pr_url", + "prep_path", + "proposed_branch", + "queue_available", + "queue_pr_state", + "remediation_path", + "remote_url", + "review_approach", + "review_loop_count", + "review_max_retries", + "review_path", + "review_verdict", + "run_mode", + "run_name", + "selected_lenses", + "source_dir", + "stall_loop_count", + "task", + "upfront_claimed", + "verdict", + "work_dir", + "worktree_path" + ], + "required": [], + "produced": [] + }, + { + "step": "release_issue_success", + "available": [ + "active_trigger_count", + "annotated_diff_path", + "arch_lenses", + "audit", + "auto_merge", + "auto_merge_available", + "base_branch", + "branch_name", + "ci_conclusion", + "ci_event", + "ci_failed_jobs", + "ci_loop_count", + "ci_post_queue_loop_count", + "claimed", + "cleanup_succeeded", + "conflict_escalation_required", + "depth", + "diagnosis_path", + "fixes_applied", + "hunk_ranges_path", + "implementation_ref", + "investigation_path", + "issue_number", + "issue_slug", + "issue_title", + "issue_url", + "lens_context_paths", + "max_parallel", + "merge_group_trigger", + "merge_target", + "open_pr", + "phases_implemented", + "plan_path", + "pr_number", + "pr_url", + "prep_path", + "proposed_branch", + "queue_available", + "queue_pr_state", + "remediation_path", + "remote_url", + "review_approach", + "review_loop_count", + "review_max_retries", + "review_path", + "review_verdict", + "run_mode", + "run_name", + "selected_lenses", + "source_dir", + "stall_loop_count", + "task", + "upfront_claimed", + "verdict", + "work_dir", + "worktree_path" + ], + "required": [], + "produced": [] + }, + { + "step": "patch_token_summary", + "available": [ + "active_trigger_count", + "annotated_diff_path", + "arch_lenses", + "audit", + "auto_merge", + "auto_merge_available", + "base_branch", + "branch_name", + "ci_conclusion", + "ci_event", + "ci_failed_jobs", + "ci_loop_count", + "ci_post_queue_loop_count", + "claimed", + "cleanup_succeeded", + "conflict_escalation_required", + "depth", + "diagnosis_path", + "fixes_applied", + "hunk_ranges_path", + "implementation_ref", + "investigation_path", + "issue_number", + "issue_slug", + "issue_title", + "issue_url", + "lens_context_paths", + "max_parallel", + "merge_group_trigger", + "merge_target", + "open_pr", + "phases_implemented", + "plan_path", + "pr_number", + "pr_url", + "prep_path", + "proposed_branch", + "queue_available", + "queue_pr_state", + "remediation_path", + "remote_url", + "review_approach", + "review_loop_count", + "review_max_retries", + "review_path", + "review_verdict", + "run_mode", + "run_name", + "selected_lenses", + "source_dir", + "stall_loop_count", + "task", + "upfront_claimed", + "verdict", + "work_dir", + "worktree_path" + ], + "required": [], + "produced": [] + }, + { + "step": "release_issue_failure", + "available": [ + "active_trigger_count", + "annotated_diff_path", + "arch_lenses", + "audit", + "auto_merge", + "auto_merge_available", + "base_branch", + "branch_name", + "ci_conclusion", + "ci_event", + "ci_failed_jobs", + "ci_loop_count", + "ci_post_queue_loop_count", + "claimed", + "cleanup_succeeded", + "conflict_escalation_required", + "depth", + "diagnosis_path", + "fixes_applied", + "hunk_ranges_path", + "implementation_ref", + "investigation_path", + "issue_number", + "issue_slug", + "issue_title", + "issue_url", + "lens_context_paths", + "max_parallel", + "merge_group_trigger", + "merge_target", + "open_pr", + "phases_implemented", + "plan_path", + "pr_number", + "pr_url", + "prep_path", + "proposed_branch", + "queue_available", + "queue_pr_state", + "remediation_path", + "remote_url", + "review_approach", + "review_loop_count", + "review_max_retries", + "review_path", + "review_verdict", + "run_mode", + "run_name", + "selected_lenses", + "source_dir", + "stall_loop_count", + "task", + "upfront_claimed", + "verdict", + "work_dir", + "worktree_path" + ], + "required": [], + "produced": [] + }, + { + "step": "register_clone_success", + "available": [ + "active_trigger_count", + "annotated_diff_path", + "arch_lenses", + "audit", + "auto_merge", + "auto_merge_available", + "base_branch", + "branch_name", + "ci_conclusion", + "ci_event", + "ci_failed_jobs", + "ci_loop_count", + "ci_post_queue_loop_count", + "claimed", + "cleanup_succeeded", + "conflict_escalation_required", + "depth", + "diagnosis_path", + "fixes_applied", + "hunk_ranges_path", + "implementation_ref", + "investigation_path", + "issue_number", + "issue_slug", + "issue_title", + "issue_url", + "lens_context_paths", + "max_parallel", + "merge_group_trigger", + "merge_target", + "open_pr", + "phases_implemented", + "plan_path", + "pr_number", + "pr_url", + "prep_path", + "proposed_branch", + "queue_available", + "queue_pr_state", + "remediation_path", + "remote_url", + "review_approach", + "review_loop_count", + "review_max_retries", + "review_path", + "review_verdict", + "run_mode", + "run_name", + "selected_lenses", + "source_dir", + "stall_loop_count", + "task", + "upfront_claimed", + "verdict", + "work_dir", + "worktree_path" + ], + "required": [], + "produced": [] + }, + { + "step": "register_clone_failure", + "available": [ + "active_trigger_count", + "annotated_diff_path", + "arch_lenses", + "audit", + "auto_merge", + "auto_merge_available", + "base_branch", + "branch_name", + "ci_conclusion", + "ci_event", + "ci_failed_jobs", + "ci_loop_count", + "ci_post_queue_loop_count", + "claimed", + "cleanup_succeeded", + "conflict_escalation_required", + "depth", + "diagnosis_path", + "fixes_applied", + "hunk_ranges_path", + "implementation_ref", + "investigation_path", + "issue_number", + "issue_slug", + "issue_title", + "issue_url", + "lens_context_paths", + "max_parallel", + "merge_group_trigger", + "merge_target", + "open_pr", + "phases_implemented", + "plan_path", + "pr_number", + "pr_url", + "prep_path", + "proposed_branch", + "queue_available", + "queue_pr_state", + "remediation_path", + "remote_url", + "review_approach", + "review_loop_count", + "review_max_retries", + "review_path", + "review_verdict", + "run_mode", + "run_name", + "selected_lenses", + "source_dir", + "stall_loop_count", + "task", + "upfront_claimed", + "verdict", + "work_dir", + "worktree_path" + ], + "required": [], + "produced": [] + }, + { + "step": "done", + "available": [ + "active_trigger_count", + "annotated_diff_path", + "arch_lenses", + "audit", + "auto_merge", + "auto_merge_available", + "base_branch", + "branch_name", + "ci_conclusion", + "ci_event", + "ci_failed_jobs", + "ci_loop_count", + "ci_post_queue_loop_count", + "claimed", + "cleanup_succeeded", + "conflict_escalation_required", + "depth", + "diagnosis_path", + "fixes_applied", + "hunk_ranges_path", + "implementation_ref", + "investigation_path", + "issue_number", + "issue_slug", + "issue_title", + "issue_url", + "lens_context_paths", + "max_parallel", + "merge_group_trigger", + "merge_target", + "open_pr", + "phases_implemented", + "plan_path", + "pr_number", + "pr_url", + "prep_path", + "proposed_branch", + "queue_available", + "queue_pr_state", + "remediation_path", + "remote_url", + "review_approach", + "review_loop_count", + "review_max_retries", + "review_path", + "review_verdict", + "run_mode", + "run_name", + "selected_lenses", + "source_dir", + "stall_loop_count", + "task", + "upfront_claimed", + "verdict", + "work_dir", + "worktree_path" + ], + "required": [], + "produced": [] + }, + { + "step": "escalate_stop", + "available": [ + "active_trigger_count", + "annotated_diff_path", + "arch_lenses", + "audit", + "auto_merge", + "auto_merge_available", + "base_branch", + "branch_name", + "ci_conclusion", + "ci_event", + "ci_failed_jobs", + "ci_loop_count", + "ci_post_queue_loop_count", + "claimed", + "cleanup_succeeded", + "conflict_escalation_required", + "depth", + "diagnosis_path", + "fixes_applied", + "hunk_ranges_path", + "implementation_ref", + "investigation_path", + "issue_number", + "issue_slug", + "issue_title", + "issue_url", + "lens_context_paths", + "max_parallel", + "merge_group_trigger", + "merge_target", + "open_pr", + "phases_implemented", + "plan_path", + "pr_number", + "pr_url", + "prep_path", + "proposed_branch", + "queue_available", + "queue_pr_state", + "remediation_path", + "remote_url", + "review_approach", + "review_loop_count", + "review_max_retries", + "review_path", + "review_verdict", + "run_mode", + "run_name", + "selected_lenses", + "source_dir", + "stall_loop_count", + "task", + "upfront_claimed", + "verdict", + "work_dir", + "worktree_path" + ], + "required": [], + "produced": [] + } + ] +} diff --git a/src/autoskillit/recipes/contracts/remediation.yaml b/src/autoskillit/recipes/contracts/remediation.yaml index f49193786e..3be283e4ae 100644 --- a/src/autoskillit/recipes/contracts/remediation.yaml +++ b/src/autoskillit/recipes/contracts/remediation.yaml @@ -1,4 +1,4 @@ -generated_at: '2026-04-05T02:19:55.997695+00:00' +generated_at: '2026-05-03T07:01:54.187185+00:00' bundled_manifest_version: 0.1.0 skill_hashes: {} skills: @@ -7,18 +7,23 @@ skills: - name: topic type: string required: true + recommended: false outputs: - name: investigation_path type: file_path expected_output_patterns: - investigation_path\s*=\s*/.+ pattern_examples: - - 'investigation_path = /tmp/investigation.md' + - 'investigation_path = /tmp/investigation.md + + %%ORDER_UP%%' + read_only: true rectify: inputs: - name: investigation_path type: file_path required: true + recommended: false outputs: - name: plan_path type: file_path @@ -27,25 +32,31 @@ skills: expected_output_patterns: - plan_path\s*=\s*/.+ pattern_examples: - - 'plan_path = /tmp/plan.md' + - 'plan_path = /tmp/plan.md + + %%ORDER_UP%%' write_behavior: always review-approach: inputs: - name: plan_path type: file_path required: false + recommended: false outputs: - name: review_path type: file_path expected_output_patterns: - review_path\s*=\s*/.+ pattern_examples: - - 'review_path = /tmp/review.md' + - 'review_path = /tmp/review.md + + %%ORDER_UP%%' dry-walkthrough: inputs: - name: plan_path type: file_path required: true + recommended: false outputs: [] expected_output_patterns: [] pattern_examples: [] @@ -55,6 +66,7 @@ skills: - name: plan_path type: file_path required: true + recommended: false outputs: - name: worktree_path type: directory_path @@ -63,52 +75,82 @@ skills: expected_output_patterns: - worktree_path\s*=\s*/.+ pattern_examples: - - 'worktree_path = /tmp/worktrees/impl-foo-20260316' + - 'worktree_path = /tmp/worktrees/impl-foo-20260316 + + %%ORDER_UP%%' write_behavior: always retry-worktree: inputs: - name: plan_path type: file_path required: true + recommended: false - name: worktree_path type: directory_path required: true + recommended: false outputs: - name: worktree_path type: directory_path - name: branch_name type: string + - name: phases_implemented + type: integer expected_output_patterns: - worktree_path\s*=\s*/.+ pattern_examples: - - 'worktree_path = /tmp/worktrees/impl-foo-20260316' - write_behavior: always + - 'worktree_path = /tmp/worktrees/impl-foo-20260316 + + branch_name = feature/123 + + phases_implemented = 2 + + %%ORDER_UP%%' + write_behavior: conditional + write_expected_when: + - phases_implemented\s*=\s*[1-9][0-9]* resolve-failures: inputs: - name: worktree_path type: directory_path required: true + recommended: false - name: plan_path type: file_path required: true + recommended: false - name: base_branch type: string required: true - outputs: [] + recommended: false + - name: diagnosis_path + type: file_path + required: false + recommended: false + outputs: + - name: verdict + type: string + - name: fixes_applied + type: integer expected_output_patterns: [] pattern_examples: [] - write_behavior: always + write_behavior: conditional + write_expected_when: + - verdict\s*=\s*real_fix audit-impl: inputs: - name: plan_path type: file_path required: true + recommended: false - name: branch_name type: string required: false + recommended: false - name: base_branch type: string required: false + recommended: false outputs: - name: verdict type: string @@ -117,15 +159,20 @@ skills: expected_output_patterns: - verdict\s*=\s*(GO|NO GO) pattern_examples: - - 'verdict = GO' + - 'verdict = GO + + %%ORDER_UP%%' - 'verdict = NO GO - remediation_path = /tmp/remediation.md' + remediation_path = /tmp/remediation.md + + %%ORDER_UP%%' make-plan: inputs: - name: task type: string required: true + recommended: false outputs: - name: plan_path type: file_path @@ -134,73 +181,166 @@ skills: expected_output_patterns: - plan_path\s*=\s*/.+ pattern_examples: - - 'plan_path = /tmp/plan.md' + - 'plan_path = /tmp/plan.md + + %%ORDER_UP%%' write_behavior: always - open-pr: + prepare-pr: inputs: - name: plan_paths type: string required: true - - name: feature_branch + recommended: false + - name: run_name type: string required: true + recommended: false - name: base_branch type: string required: true + recommended: false - name: closing_issue type: string required: false - outputs: [] + recommended: false + - name: conflict_report_path + type: file_path + required: false + recommended: false + outputs: + - name: prep_path + type: file_path + - name: selected_lenses + type: string + - name: lens_context_paths + type: string expected_output_patterns: - - https://github\.com/.*/pull/\d+ + - prep_path\s*=\s*/.+ + - selected_lenses\s*=\s*\S+ + - lens_context_paths\s*=\s*/.+ pattern_examples: - - 'https://github.com/owner/repo/pull/42' + - 'prep_path = /abs/path/pr_prep_2026-04-07_120000.md + + selected_lenses = module-dependency,process-flow + + lens_context_paths = /abs/ctx1.md,/abs/ctx2.md + + %%ORDER_UP%%' + write_behavior: conditional + write_expected_when: + - prep_path\s*=\s*/.+ + compose-pr: + inputs: + - name: prep_path + type: file_path + required: true + recommended: false + - name: all_diagram_paths + type: string + required: true + recommended: false + - name: work_dir + type: directory_path + required: true + recommended: false + - name: base_branch + type: string + required: true + recommended: false + - name: closing_issue + type: string + required: false + recommended: false + outputs: + - name: pr_url + type: string + expected_output_patterns: + - pr_url\s*=\s*(https://github\.com/.*/pull/\d+)? + pattern_examples: + - 'pr_url = https://github.com/owner/repo/pull/42 + + %%ORDER_UP%%' + - "pr_url = \n%%ORDER_UP%%" + write_behavior: always review-pr: inputs: - name: feature_branch type: string required: true + recommended: false - name: base_branch type: string required: true + recommended: false - name: annotated_diff_path type: string required: false + recommended: true - name: hunk_ranges_path type: string required: false + recommended: true outputs: - name: verdict type: string expected_output_patterns: - - verdict\s*=\s*(approved|changes_requested|needs_human) + - verdict\s*=\s*(approved|approved_with_comments|changes_requested|needs_human) + - '%%REVIEW_GATE::(LOOP_REQUIRED|CLEAR)%%' pattern_examples: - - 'verdict = approved' - - 'verdict = changes_requested' - - 'verdict = needs_human' + - 'verdict = approved + + %%REVIEW_GATE::CLEAR%% + + %%ORDER_UP%%' + - 'verdict = approved_with_comments + + %%REVIEW_GATE::CLEAR%% + + %%ORDER_UP%%' + - 'verdict = changes_requested + + %%REVIEW_GATE::LOOP_REQUIRED%% + + %%ORDER_UP%%' + - 'verdict = needs_human + + %%REVIEW_GATE::CLEAR%% + + %%ORDER_UP%%' resolve-review: inputs: - name: feature_branch type: string required: true + recommended: false - name: base_branch type: string required: true - outputs: [] + recommended: false + outputs: + - name: verdict + type: string + - name: fixes_applied + type: integer expected_output_patterns: [] pattern_examples: [] - write_behavior: always + write_behavior: conditional + write_expected_when: + - verdict\s*=\s*real_fix resolve-merge-conflicts: inputs: - name: worktree_path type: directory_path required: true + recommended: false - name: plan_path type: file_path required: true + recommended: false - name: base_branch type: string required: true + recommended: false outputs: - name: worktree_path type: directory_path @@ -215,7 +355,9 @@ skills: expected_output_patterns: - worktree_path\s*=\s*/.+ pattern_examples: - - 'worktree_path = /tmp/worktrees/impl-foo-20260316' + - 'worktree_path = /tmp/worktrees/impl-foo-20260316 + + %%ORDER_UP%%' write_behavior: conditional write_expected_when: - conflict_report_path\s*=\s*/.+ @@ -224,53 +366,63 @@ skills: - name: branch type: string required: true + recommended: false - name: run_id type: string required: false + recommended: false - name: ci_failed_jobs type: string required: false + recommended: false outputs: - name: diagnosis_path type: file_path - name: failure_type type: string + - name: failure_subtype + type: string - name: is_fixable type: string expected_output_patterns: - diagnosis_path\s*=\s*/.+ + - failure_subtype\s*=\s*(flaky|timing_race|deterministic|fixture|import|env|unknown) pattern_examples: - - 'diagnosis_path = /tmp/diagnosis.md' + - 'diagnosis_path = /tmp/diagnosis.md + + failure_type = test + + failure_subtype = deterministic + + is_fixable = true + + %%ORDER_UP%%' + - 'diagnosis_path = /tmp/diagnosis.md + + failure_type = test + + failure_subtype = flaky + + is_fixable = false + + %%ORDER_UP%%' write_behavior: always dataflow: -- step: sprint_entry - available: - - audit - - auto_merge - - base_branch - - issue_url - - open_pr - - review_approach - - run_mode - - run_name - - source_dir - - sprint_mode - - task - - upfront_claimed - required: [] - produced: [] - step: clone available: + - arch_lenses - audit - auto_merge - base_branch + - depth - issue_url + - max_parallel - open_pr - review_approach + - review_max_retries - run_mode - run_name - source_dir - - sprint_mode - task - upfront_claimed required: [] @@ -279,17 +431,20 @@ dataflow: - remote_url - step: set_merge_target available: + - arch_lenses - audit - auto_merge - base_branch + - depth - issue_url + - max_parallel - open_pr - remote_url - review_approach + - review_max_retries - run_mode - run_name - source_dir - - sprint_mode - task - upfront_claimed - work_dir @@ -298,18 +453,21 @@ dataflow: - merge_target - step: get_issue_title available: + - arch_lenses - audit - auto_merge - base_branch + - depth - issue_url + - max_parallel - merge_target - open_pr - remote_url - review_approach + - review_max_retries - run_mode - run_name - source_dir - - sprint_mode - task - upfront_claimed - work_dir @@ -320,21 +478,24 @@ dataflow: - issue_slug - step: claim_issue available: + - arch_lenses - audit - auto_merge - base_branch + - depth - issue_number - issue_slug - issue_title - issue_url + - max_parallel - merge_target - open_pr - remote_url - review_approach + - review_max_retries - run_mode - run_name - source_dir - - sprint_mode - task - upfront_claimed - work_dir @@ -343,22 +504,25 @@ dataflow: - claimed - step: compute_branch available: + - arch_lenses - audit - auto_merge - base_branch - claimed + - depth - issue_number - issue_slug - issue_title - issue_url + - max_parallel - merge_target - open_pr - remote_url - review_approach + - review_max_retries - run_mode - run_name - source_dir - - sprint_mode - task - upfront_claimed - work_dir @@ -367,23 +531,26 @@ dataflow: - proposed_branch - step: create_branch available: + - arch_lenses - audit - auto_merge - base_branch - claimed + - depth - issue_number - issue_slug - issue_title - issue_url + - max_parallel - merge_target - open_pr - proposed_branch - remote_url - review_approach + - review_max_retries - run_mode - run_name - source_dir - - sprint_mode - task - upfront_claimed - work_dir @@ -392,23 +559,26 @@ dataflow: - merge_target - step: push_merge_target available: + - arch_lenses - audit - auto_merge - base_branch - claimed + - depth - issue_number - issue_slug - issue_title - issue_url + - max_parallel - merge_target - open_pr - proposed_branch - remote_url - review_approach + - review_max_retries - run_mode - run_name - source_dir - - sprint_mode - task - upfront_claimed - work_dir @@ -416,23 +586,26 @@ dataflow: produced: [] - step: investigate available: + - arch_lenses - audit - auto_merge - base_branch - claimed + - depth - issue_number - issue_slug - issue_title - issue_url + - max_parallel - merge_target - open_pr - proposed_branch - remote_url - review_approach + - review_max_retries - run_mode - run_name - source_dir - - sprint_mode - task - upfront_claimed - work_dir @@ -442,24 +615,27 @@ dataflow: - investigation_path - step: rectify available: + - arch_lenses - audit - auto_merge - base_branch - claimed + - depth - investigation_path - issue_number - issue_slug - issue_title - issue_url + - max_parallel - merge_target - open_pr - proposed_branch - remote_url - review_approach + - review_max_retries - run_mode - run_name - source_dir - - sprint_mode - task - upfront_claimed - work_dir @@ -468,25 +644,28 @@ dataflow: - plan_path - step: review available: + - arch_lenses - audit - auto_merge - base_branch - claimed + - depth - investigation_path - issue_number - issue_slug - issue_title - issue_url + - max_parallel - merge_target - open_pr - plan_path - proposed_branch - remote_url - review_approach + - review_max_retries - run_mode - run_name - source_dir - - sprint_mode - task - upfront_claimed - work_dir @@ -495,26 +674,29 @@ dataflow: - review_path - step: dry_walkthrough available: + - arch_lenses - audit - auto_merge - base_branch - claimed + - depth - investigation_path - issue_number - issue_slug - issue_title - issue_url + - max_parallel - merge_target - open_pr - plan_path - proposed_branch - remote_url - review_approach + - review_max_retries - review_path - run_mode - run_name - source_dir - - sprint_mode - task - upfront_claimed - work_dir @@ -522,26 +704,29 @@ dataflow: produced: [] - step: implement available: + - arch_lenses - audit - auto_merge - base_branch - claimed + - depth - investigation_path - issue_number - issue_slug - issue_title - issue_url + - max_parallel - merge_target - open_pr - plan_path - proposed_branch - remote_url - review_approach + - review_max_retries - review_path - run_mode - run_name - source_dir - - sprint_mode - task - upfront_claimed - work_dir @@ -552,28 +737,31 @@ dataflow: - branch_name - step: retry_worktree available: + - arch_lenses - audit - auto_merge - base_branch - branch_name - claimed + - depth - implementation_ref - investigation_path - issue_number - issue_slug - issue_title - issue_url + - max_parallel - merge_target - open_pr - plan_path - proposed_branch - remote_url - review_approach + - review_max_retries - review_path - run_mode - run_name - source_dir - - sprint_mode - task - upfront_claimed - work_dir @@ -583,30 +771,35 @@ dataflow: - implementation_ref - worktree_path - branch_name + - phases_implemented - step: test available: + - arch_lenses - audit - auto_merge - base_branch - branch_name - claimed + - depth - implementation_ref - investigation_path - issue_number - issue_slug - issue_title - issue_url + - max_parallel - merge_target - open_pr + - phases_implemented - plan_path - proposed_branch - remote_url - review_approach + - review_max_retries - review_path - run_mode - run_name - source_dir - - sprint_mode - task - upfront_claimed - work_dir @@ -615,60 +808,72 @@ dataflow: produced: [] - step: assess available: + - arch_lenses - audit - auto_merge - base_branch - branch_name - claimed + - depth - implementation_ref - investigation_path - issue_number - issue_slug - issue_title - issue_url + - max_parallel - merge_target - open_pr + - phases_implemented - plan_path - proposed_branch - remote_url - review_approach + - review_max_retries - review_path - run_mode - run_name - source_dir - - sprint_mode - task - upfront_claimed - work_dir - worktree_path required: [] - produced: [] + produced: + - verdict + - fixes_applied - step: audit_impl available: + - arch_lenses - audit - auto_merge - base_branch - branch_name - claimed + - depth + - fixes_applied - implementation_ref - investigation_path - issue_number - issue_slug - issue_title - issue_url + - max_parallel - merge_target - open_pr + - phases_implemented - plan_path - proposed_branch - remote_url - review_approach + - review_max_retries - review_path - run_mode - run_name - source_dir - - sprint_mode - task - upfront_claimed + - verdict - work_dir - worktree_path required: [] @@ -676,330 +881,392 @@ dataflow: - remediation_path - step: remediate available: + - arch_lenses - audit - auto_merge - base_branch - branch_name - claimed + - depth + - fixes_applied - implementation_ref - investigation_path - issue_number - issue_slug - issue_title - issue_url + - max_parallel - merge_target - open_pr + - phases_implemented - plan_path - proposed_branch - remediation_path - remote_url - review_approach + - review_max_retries - review_path - run_mode - run_name - source_dir - - sprint_mode - task - upfront_claimed + - verdict - work_dir - worktree_path required: [] produced: [] - step: make_plan available: + - arch_lenses - audit - auto_merge - base_branch - branch_name - claimed + - depth + - fixes_applied - implementation_ref - investigation_path - issue_number - issue_slug - issue_title - issue_url + - max_parallel - merge_target - open_pr + - phases_implemented - plan_path - proposed_branch - remediation_path - remote_url - review_approach + - review_max_retries - review_path - run_mode - run_name - source_dir - - sprint_mode - task - upfront_claimed + - verdict - work_dir - worktree_path required: [] produced: - plan_path -- step: merge +- step: commit_guard available: + - arch_lenses - audit - auto_merge - base_branch - branch_name - claimed + - depth + - fixes_applied - implementation_ref - investigation_path - issue_number - issue_slug - issue_title - issue_url + - max_parallel - merge_target - open_pr + - phases_implemented - plan_path - proposed_branch - remediation_path - remote_url - review_approach + - review_max_retries - review_path - run_mode - run_name - source_dir - - sprint_mode - task - upfront_claimed + - verdict - work_dir - worktree_path required: [] - produced: - - cleanup_succeeded - - worktree_path -- step: push + produced: [] +- step: merge available: + - arch_lenses - audit - auto_merge - base_branch - branch_name - claimed - - cleanup_succeeded + - depth + - fixes_applied - implementation_ref - investigation_path - issue_number - issue_slug - issue_title - issue_url + - max_parallel - merge_target - open_pr + - phases_implemented - plan_path - proposed_branch - remediation_path - remote_url - review_approach + - review_max_retries - review_path - run_mode - run_name - source_dir - - sprint_mode - task - upfront_claimed + - verdict - work_dir - worktree_path required: [] - produced: [] -- step: compose_pr + produced: + - cleanup_succeeded + - worktree_path +- step: next_or_done available: + - arch_lenses - audit - auto_merge - base_branch - branch_name - claimed - cleanup_succeeded + - depth + - fixes_applied - implementation_ref - investigation_path - issue_number - issue_slug - issue_title - issue_url + - max_parallel - merge_target - open_pr + - phases_implemented - plan_path - proposed_branch - remediation_path - remote_url - review_approach + - review_max_retries - review_path - run_mode - run_name - source_dir - - sprint_mode - task - upfront_claimed + - verdict - work_dir - worktree_path - required: - - plan_paths - - feature_branch + required: [] produced: [] -- step: extract_pr_number +- step: push available: + - arch_lenses - audit - auto_merge - base_branch - branch_name - claimed - cleanup_succeeded + - depth + - fixes_applied - implementation_ref - investigation_path - issue_number - issue_slug - issue_title - issue_url + - max_parallel - merge_target - open_pr + - phases_implemented - plan_path - proposed_branch - remediation_path - remote_url - review_approach + - review_max_retries - review_path - run_mode - run_name - source_dir - - sprint_mode - task - upfront_claimed + - verdict - work_dir - worktree_path required: [] - produced: - - pr_number -- step: review_pr + produced: [] +- step: prepare_pr available: + - arch_lenses - audit - auto_merge - base_branch - branch_name - claimed - cleanup_succeeded + - depth + - fixes_applied - implementation_ref - investigation_path - issue_number - issue_slug - issue_title - issue_url + - max_parallel - merge_target - open_pr + - phases_implemented - plan_path - - pr_number - proposed_branch - remediation_path - remote_url - review_approach + - review_max_retries - review_path - run_mode - run_name - source_dir - - sprint_mode - task - upfront_claimed + - verdict - work_dir - worktree_path required: - - feature_branch + - plan_paths produced: - - verdict -- step: resolve_review + - prep_path + - selected_lenses + - lens_context_paths +- step: run_arch_lenses available: + - arch_lenses - audit - auto_merge - base_branch - branch_name - claimed - cleanup_succeeded + - depth + - fixes_applied - implementation_ref - investigation_path - issue_number - issue_slug - issue_title - issue_url + - lens_context_paths + - max_parallel - merge_target - open_pr + - phases_implemented - plan_path - - pr_number + - prep_path - proposed_branch - remediation_path - remote_url - review_approach + - review_max_retries - review_path - run_mode - run_name + - selected_lenses - source_dir - - sprint_mode - task - upfront_claimed - verdict - work_dir - worktree_path - required: - - feature_branch + required: [] produced: [] -- step: re_push_review +- step: compose_pr available: + - arch_lenses - audit - auto_merge - base_branch - branch_name - claimed - cleanup_succeeded + - depth + - fixes_applied - implementation_ref - investigation_path - issue_number - issue_slug - issue_title - issue_url + - lens_context_paths + - max_parallel - merge_target - open_pr + - phases_implemented - plan_path - - pr_number + - prep_path - proposed_branch - remediation_path - remote_url - review_approach + - review_max_retries - review_path - run_mode - run_name + - selected_lenses - source_dir - - sprint_mode - task - upfront_claimed - verdict - work_dir - worktree_path required: [] - produced: [] -- step: ci_watch + produced: + - pr_url + positional_args: 1 +- step: extract_pr_number available: + - arch_lenses - audit - auto_merge - base_branch - branch_name - claimed - cleanup_succeeded + - depth + - fixes_applied - implementation_ref - investigation_path - issue_number - issue_slug - issue_title - issue_url + - lens_context_paths + - max_parallel - merge_target - open_pr + - phases_implemented - plan_path - - pr_number + - pr_url + - prep_path - proposed_branch - remediation_path - remote_url - review_approach + - review_max_retries - review_path - run_mode - run_name + - selected_lenses - source_dir - - sprint_mode - task - upfront_claimed - verdict @@ -1007,37 +1274,43 @@ dataflow: - worktree_path required: [] produced: - - ci_conclusion - - ci_failed_jobs -- step: check_merge_queue + - pr_number +- step: annotate_pr_diff available: + - arch_lenses - audit - auto_merge - base_branch - branch_name - - ci_conclusion - - ci_failed_jobs - claimed - cleanup_succeeded + - depth + - fixes_applied - implementation_ref - investigation_path - issue_number - issue_slug - issue_title - issue_url + - lens_context_paths + - max_parallel - merge_target - open_pr + - phases_implemented - plan_path - pr_number + - pr_url + - prep_path - proposed_branch - remediation_path - remote_url - review_approach + - review_max_retries - review_path - run_mode - run_name + - selected_lenses - source_dir - - sprint_mode - task - upfront_claimed - verdict @@ -1045,37 +1318,48 @@ dataflow: - worktree_path required: [] produced: - - queue_available -- step: check_merge_group_trigger + - annotated_diff_path + - diff_metrics_path + - hunk_ranges_path + - review_mode +- step: review_pr available: + - annotated_diff_path + - arch_lenses - audit - auto_merge - base_branch - branch_name - - ci_conclusion - - ci_failed_jobs - claimed - cleanup_succeeded + - depth + - fixes_applied + - hunk_ranges_path - implementation_ref - investigation_path - issue_number - issue_slug - issue_title - issue_url + - lens_context_paths + - max_parallel - merge_target - open_pr + - phases_implemented - plan_path - pr_number + - pr_url + - prep_path - proposed_branch - - queue_available - remediation_path - remote_url - review_approach + - review_max_retries - review_path - run_mode - run_name + - selected_lenses - source_dir - - sprint_mode - task - upfront_claimed - verdict @@ -1083,78 +1367,95 @@ dataflow: - worktree_path required: [] produced: - - merge_group_trigger -- step: check_auto_merge + - review_verdict + positional_args: 2 +- step: resolve_review available: + - annotated_diff_path + - arch_lenses - audit - auto_merge - base_branch - branch_name - - ci_conclusion - - ci_failed_jobs - claimed - cleanup_succeeded + - depth + - fixes_applied + - hunk_ranges_path - implementation_ref - investigation_path - issue_number - issue_slug - issue_title - issue_url - - merge_group_trigger + - lens_context_paths + - max_parallel - merge_target - open_pr + - phases_implemented - plan_path - pr_number + - pr_url + - prep_path - proposed_branch - - queue_available - remediation_path - remote_url - review_approach + - review_max_retries - review_path + - review_verdict - run_mode - run_name + - selected_lenses - source_dir - - sprint_mode - task - upfront_claimed - verdict - work_dir - worktree_path - required: [] + required: + - feature_branch produced: - - auto_merge_available -- step: route_queue_mode + - verdict +- step: re_push_review available: + - annotated_diff_path + - arch_lenses - audit - auto_merge - - auto_merge_available - base_branch - branch_name - - ci_conclusion - - ci_failed_jobs - claimed - cleanup_succeeded + - depth + - fixes_applied + - hunk_ranges_path - implementation_ref - investigation_path - issue_number - issue_slug - issue_title - issue_url - - merge_group_trigger + - lens_context_paths + - max_parallel - merge_target - open_pr + - phases_implemented - plan_path - pr_number + - pr_url + - prep_path - proposed_branch - - queue_available - remediation_path - remote_url - review_approach + - review_max_retries - review_path + - review_verdict - run_mode - run_name + - selected_lenses - source_dir - - sprint_mode - task - upfront_claimed - verdict @@ -1162,77 +1463,93 @@ dataflow: - worktree_path required: [] produced: [] -- step: enable_auto_merge +- step: check_review_loop available: + - annotated_diff_path + - arch_lenses - audit - auto_merge - - auto_merge_available - base_branch - branch_name - - ci_conclusion - - ci_failed_jobs - claimed - cleanup_succeeded + - depth + - fixes_applied + - hunk_ranges_path - implementation_ref - investigation_path - issue_number - issue_slug - issue_title - issue_url - - merge_group_trigger + - lens_context_paths + - max_parallel - merge_target - open_pr + - phases_implemented - plan_path - pr_number + - pr_url + - prep_path - proposed_branch - - queue_available - remediation_path - remote_url - review_approach + - review_max_retries - review_path + - review_verdict - run_mode - run_name + - selected_lenses - source_dir - - sprint_mode - task - upfront_claimed - verdict - work_dir - worktree_path required: [] - produced: [] -- step: wait_for_queue + produced: + - review_loop_count +- step: check_repo_ci_event available: + - annotated_diff_path + - arch_lenses - audit - auto_merge - - auto_merge_available - base_branch - branch_name - - ci_conclusion - - ci_failed_jobs - claimed - cleanup_succeeded + - depth + - fixes_applied + - hunk_ranges_path - implementation_ref - investigation_path - issue_number - issue_slug - issue_title - issue_url - - merge_group_trigger + - lens_context_paths + - max_parallel - merge_target - open_pr + - phases_implemented - plan_path - pr_number + - pr_url + - prep_path - proposed_branch - - queue_available - remediation_path - remote_url - review_approach + - review_loop_count + - review_max_retries - review_path + - review_verdict - run_mode - run_name + - selected_lenses - source_dir - - sprint_mode - task - upfront_claimed - verdict @@ -1240,40 +1557,48 @@ dataflow: - worktree_path required: [] produced: - - queue_pr_state -- step: reenroll_stalled_pr + - ci_event +- step: check_pr_state available: + - annotated_diff_path + - arch_lenses - audit - auto_merge - - auto_merge_available - base_branch - branch_name - - ci_conclusion - - ci_failed_jobs + - ci_event - claimed - cleanup_succeeded + - depth + - fixes_applied + - hunk_ranges_path - implementation_ref - investigation_path - issue_number - issue_slug - issue_title - issue_url - - merge_group_trigger + - lens_context_paths + - max_parallel - merge_target - open_pr + - phases_implemented - plan_path - pr_number + - pr_url + - prep_path - proposed_branch - - queue_available - - queue_pr_state - remediation_path - remote_url - review_approach + - review_loop_count + - review_max_retries - review_path + - review_verdict - run_mode - run_name + - selected_lenses - source_dir - - sprint_mode - task - upfront_claimed - verdict @@ -1281,205 +1606,255 @@ dataflow: - worktree_path required: [] produced: [] -- step: queue_ejected_fix +- step: ci_watch available: + - annotated_diff_path + - arch_lenses - audit - auto_merge - - auto_merge_available - base_branch - branch_name - - ci_conclusion - - ci_failed_jobs + - ci_event - claimed - cleanup_succeeded + - depth + - fixes_applied + - hunk_ranges_path - implementation_ref - investigation_path - issue_number - issue_slug - issue_title - issue_url - - merge_group_trigger + - lens_context_paths + - max_parallel - merge_target - open_pr + - phases_implemented - plan_path - pr_number + - pr_url + - prep_path - proposed_branch - - queue_available - - queue_pr_state - remediation_path - remote_url - review_approach + - review_loop_count + - review_max_retries - review_path + - review_verdict - run_mode - run_name + - selected_lenses - source_dir - - sprint_mode - task - upfront_claimed - verdict - work_dir - worktree_path - required: - - worktree_path + required: [] produced: - - conflict_escalation_required -- step: re_push_queue_fix + - ci_conclusion + - ci_failed_jobs +- step: handle_no_ci_runs available: + - annotated_diff_path + - arch_lenses - audit - auto_merge - - auto_merge_available - base_branch - branch_name - ci_conclusion + - ci_event - ci_failed_jobs - claimed - cleanup_succeeded - - conflict_escalation_required + - depth + - fixes_applied + - hunk_ranges_path - implementation_ref - investigation_path - issue_number - issue_slug - issue_title - issue_url - - merge_group_trigger + - lens_context_paths + - max_parallel - merge_target - open_pr + - phases_implemented - plan_path - pr_number + - pr_url + - prep_path - proposed_branch - - queue_available - - queue_pr_state - remediation_path - remote_url - review_approach + - review_loop_count + - review_max_retries - review_path + - review_verdict - run_mode - run_name + - selected_lenses - source_dir - - sprint_mode - task - upfront_claimed - verdict - work_dir - worktree_path required: [] - produced: [] -- step: reenter_merge_queue + produced: + - ci_event +- step: check_ci_loop available: + - annotated_diff_path + - arch_lenses - audit - auto_merge - - auto_merge_available - base_branch - branch_name - ci_conclusion + - ci_event - ci_failed_jobs - claimed - cleanup_succeeded - - conflict_escalation_required + - depth + - fixes_applied + - hunk_ranges_path - implementation_ref - investigation_path - issue_number - issue_slug - issue_title - issue_url - - merge_group_trigger + - lens_context_paths + - max_parallel - merge_target - open_pr + - phases_implemented - plan_path - pr_number + - pr_url + - prep_path - proposed_branch - - queue_available - - queue_pr_state - remediation_path - remote_url - review_approach + - review_loop_count + - review_max_retries - review_path + - review_verdict - run_mode - run_name + - selected_lenses - source_dir - - sprint_mode - task - upfront_claimed - verdict - work_dir - worktree_path required: [] - produced: [] -- step: direct_merge + produced: + - ci_loop_count +- step: check_active_trigger_loop available: + - annotated_diff_path + - arch_lenses - audit - auto_merge - - auto_merge_available - base_branch - branch_name - ci_conclusion + - ci_event - ci_failed_jobs + - ci_loop_count - claimed - cleanup_succeeded - - conflict_escalation_required + - depth + - fixes_applied + - hunk_ranges_path - implementation_ref - investigation_path - issue_number - issue_slug - issue_title - issue_url - - merge_group_trigger + - lens_context_paths + - max_parallel - merge_target - open_pr + - phases_implemented - plan_path - pr_number + - pr_url + - prep_path - proposed_branch - - queue_available - - queue_pr_state - remediation_path - remote_url - review_approach + - review_loop_count + - review_max_retries - review_path + - review_verdict - run_mode - run_name + - selected_lenses - source_dir - - sprint_mode - task - upfront_claimed - verdict - work_dir - worktree_path required: [] - produced: [] -- step: wait_for_direct_merge + produced: + - active_trigger_count +- step: trigger_ci_actively available: + - active_trigger_count + - annotated_diff_path + - arch_lenses - audit - auto_merge - - auto_merge_available - base_branch - branch_name - ci_conclusion + - ci_event - ci_failed_jobs + - ci_loop_count - claimed - cleanup_succeeded - - conflict_escalation_required + - depth + - fixes_applied + - hunk_ranges_path - implementation_ref - investigation_path - issue_number - issue_slug - issue_title - issue_url - - merge_group_trigger + - lens_context_paths + - max_parallel - merge_target - open_pr + - phases_implemented - plan_path - pr_number + - pr_url + - prep_path - proposed_branch - - queue_available - - queue_pr_state - remediation_path - remote_url - review_approach + - review_loop_count + - review_max_retries - review_path + - review_verdict - run_mode - run_name + - selected_lenses - source_dir - - sprint_mode - task - upfront_claimed - verdict @@ -1487,83 +1862,1419 @@ dataflow: - worktree_path required: [] produced: [] -- step: direct_merge_conflict_fix +- step: check_ci_already_passed available: + - active_trigger_count + - annotated_diff_path + - arch_lenses - audit - auto_merge - - auto_merge_available - base_branch - branch_name - ci_conclusion + - ci_event - ci_failed_jobs + - ci_loop_count - claimed - cleanup_succeeded - - conflict_escalation_required + - depth + - fixes_applied + - hunk_ranges_path - implementation_ref - investigation_path - issue_number - issue_slug - issue_title - issue_url - - merge_group_trigger + - lens_context_paths + - max_parallel - merge_target - open_pr + - phases_implemented - plan_path - pr_number + - pr_url + - prep_path - proposed_branch - - queue_available - - queue_pr_state - remediation_path - remote_url - review_approach + - review_loop_count + - review_max_retries - review_path + - review_verdict - run_mode - run_name + - selected_lenses - source_dir - - sprint_mode - task - upfront_claimed - verdict - work_dir - worktree_path - required: - - worktree_path - produced: - - conflict_escalation_required -- step: re_push_direct_fix + required: [] + produced: [] +- step: mark_issue_failed_no_ci available: + - active_trigger_count + - annotated_diff_path + - arch_lenses - audit - auto_merge - - auto_merge_available - base_branch - branch_name - ci_conclusion + - ci_event - ci_failed_jobs + - ci_loop_count - claimed - cleanup_succeeded - - conflict_escalation_required + - depth + - fixes_applied + - hunk_ranges_path - implementation_ref - investigation_path - issue_number - issue_slug - issue_title - issue_url - - merge_group_trigger + - lens_context_paths + - max_parallel - merge_target - open_pr + - phases_implemented - plan_path - pr_number + - pr_url + - prep_path - proposed_branch - - queue_available + - remediation_path + - remote_url + - review_approach + - review_loop_count + - review_max_retries + - review_path + - review_verdict + - run_mode + - run_name + - selected_lenses + - source_dir + - task + - upfront_claimed + - verdict + - work_dir + - worktree_path + required: [] + produced: [] +- step: register_clone_no_ci + available: + - active_trigger_count + - annotated_diff_path + - arch_lenses + - audit + - auto_merge + - base_branch + - branch_name + - ci_conclusion + - ci_event + - ci_failed_jobs + - ci_loop_count + - claimed + - cleanup_succeeded + - depth + - fixes_applied + - hunk_ranges_path + - implementation_ref + - investigation_path + - issue_number + - issue_slug + - issue_title + - issue_url + - lens_context_paths + - max_parallel + - merge_target + - open_pr + - phases_implemented + - plan_path + - pr_number + - pr_url + - prep_path + - proposed_branch + - remediation_path + - remote_url + - review_approach + - review_loop_count + - review_max_retries + - review_path + - review_verdict + - run_mode + - run_name + - selected_lenses + - source_dir + - task + - upfront_claimed + - verdict + - work_dir + - worktree_path + required: [] + produced: [] +- step: escalate_stop_no_ci + available: + - active_trigger_count + - annotated_diff_path + - arch_lenses + - audit + - auto_merge + - base_branch + - branch_name + - ci_conclusion + - ci_event + - ci_failed_jobs + - ci_loop_count + - claimed + - cleanup_succeeded + - depth + - fixes_applied + - hunk_ranges_path + - implementation_ref + - investigation_path + - issue_number + - issue_slug + - issue_title + - issue_url + - lens_context_paths + - max_parallel + - merge_target + - open_pr + - phases_implemented + - plan_path + - pr_number + - pr_url + - prep_path + - proposed_branch + - remediation_path + - remote_url + - review_approach + - review_loop_count + - review_max_retries + - review_path + - review_verdict + - run_mode + - run_name + - selected_lenses + - source_dir + - task + - upfront_claimed + - verdict + - work_dir + - worktree_path + required: [] + produced: [] +- step: check_repo_merge_state + available: + - active_trigger_count + - annotated_diff_path + - arch_lenses + - audit + - auto_merge + - base_branch + - branch_name + - ci_conclusion + - ci_event + - ci_failed_jobs + - ci_loop_count + - claimed + - cleanup_succeeded + - depth + - fixes_applied + - hunk_ranges_path + - implementation_ref + - investigation_path + - issue_number + - issue_slug + - issue_title + - issue_url + - lens_context_paths + - max_parallel + - merge_target + - open_pr + - phases_implemented + - plan_path + - pr_number + - pr_url + - prep_path + - proposed_branch + - remediation_path + - remote_url + - review_approach + - review_loop_count + - review_max_retries + - review_path + - review_verdict + - run_mode + - run_name + - selected_lenses + - source_dir + - task + - upfront_claimed + - verdict + - work_dir + - worktree_path + required: [] + produced: + - queue_available + - merge_group_trigger + - auto_merge_available + - ci_event +- step: route_queue_mode + available: + - active_trigger_count + - annotated_diff_path + - arch_lenses + - audit + - auto_merge + - auto_merge_available + - base_branch + - branch_name + - ci_conclusion + - ci_event + - ci_failed_jobs + - ci_loop_count + - claimed + - cleanup_succeeded + - depth + - fixes_applied + - hunk_ranges_path + - implementation_ref + - investigation_path + - issue_number + - issue_slug + - issue_title + - issue_url + - lens_context_paths + - max_parallel + - merge_group_trigger + - merge_target + - open_pr + - phases_implemented + - plan_path + - pr_number + - pr_url + - prep_path + - proposed_branch + - queue_available + - remediation_path + - remote_url + - review_approach + - review_loop_count + - review_max_retries + - review_path + - review_verdict + - run_mode + - run_name + - selected_lenses + - source_dir + - task + - upfront_claimed + - verdict + - work_dir + - worktree_path + required: [] + produced: [] +- step: enqueue_to_queue + available: + - active_trigger_count + - annotated_diff_path + - arch_lenses + - audit + - auto_merge + - auto_merge_available + - base_branch + - branch_name + - ci_conclusion + - ci_event + - ci_failed_jobs + - ci_loop_count + - claimed + - cleanup_succeeded + - depth + - fixes_applied + - hunk_ranges_path + - implementation_ref + - investigation_path + - issue_number + - issue_slug + - issue_title + - issue_url + - lens_context_paths + - max_parallel + - merge_group_trigger + - merge_target + - open_pr + - phases_implemented + - plan_path + - pr_number + - pr_url + - prep_path + - proposed_branch + - queue_available + - remediation_path + - remote_url + - review_approach + - review_loop_count + - review_max_retries + - review_path + - review_verdict + - run_mode + - run_name + - selected_lenses + - source_dir + - task + - upfront_claimed + - verdict + - work_dir + - worktree_path + required: [] + produced: [] +- step: verify_queue_enrollment + available: + - active_trigger_count + - annotated_diff_path + - arch_lenses + - audit + - auto_merge + - auto_merge_available + - base_branch + - branch_name + - ci_conclusion + - ci_event + - ci_failed_jobs + - ci_loop_count + - claimed + - cleanup_succeeded + - depth + - fixes_applied + - hunk_ranges_path + - implementation_ref + - investigation_path + - issue_number + - issue_slug + - issue_title + - issue_url + - lens_context_paths + - max_parallel + - merge_group_trigger + - merge_target + - open_pr + - phases_implemented + - plan_path + - pr_number + - pr_url + - prep_path + - proposed_branch + - queue_available + - remediation_path + - remote_url + - review_approach + - review_loop_count + - review_max_retries + - review_path + - review_verdict + - run_mode + - run_name + - selected_lenses + - source_dir + - task + - upfront_claimed + - verdict + - work_dir + - worktree_path + required: [] + produced: [] +- step: wait_for_queue + available: + - active_trigger_count + - annotated_diff_path + - arch_lenses + - audit + - auto_merge + - auto_merge_available + - base_branch + - branch_name + - ci_conclusion + - ci_event + - ci_failed_jobs + - ci_loop_count + - claimed + - cleanup_succeeded + - depth + - fixes_applied + - hunk_ranges_path + - implementation_ref + - investigation_path + - issue_number + - issue_slug + - issue_title + - issue_url + - lens_context_paths + - max_parallel + - merge_group_trigger + - merge_target + - open_pr + - phases_implemented + - plan_path + - pr_number + - pr_url + - prep_path + - proposed_branch + - queue_available + - remediation_path + - remote_url + - review_approach + - review_loop_count + - review_max_retries + - review_path + - review_verdict + - run_mode + - run_name + - selected_lenses + - source_dir + - task + - upfront_claimed + - verdict + - work_dir + - worktree_path + required: [] + produced: + - queue_pr_state +- step: reenroll_stalled_pr + available: + - active_trigger_count + - annotated_diff_path + - arch_lenses + - audit + - auto_merge + - auto_merge_available + - base_branch + - branch_name + - ci_conclusion + - ci_event + - ci_failed_jobs + - ci_loop_count + - claimed + - cleanup_succeeded + - depth + - fixes_applied + - hunk_ranges_path + - implementation_ref + - investigation_path + - issue_number + - issue_slug + - issue_title + - issue_url + - lens_context_paths + - max_parallel + - merge_group_trigger + - merge_target + - open_pr + - phases_implemented + - plan_path + - pr_number + - pr_url + - prep_path + - proposed_branch + - queue_available + - queue_pr_state + - remediation_path + - remote_url + - review_approach + - review_loop_count + - review_max_retries + - review_path + - review_verdict + - run_mode + - run_name + - selected_lenses + - source_dir + - task + - upfront_claimed + - verdict + - work_dir + - worktree_path + required: [] + produced: [] +- step: check_stall_loop + available: + - active_trigger_count + - annotated_diff_path + - arch_lenses + - audit + - auto_merge + - auto_merge_available + - base_branch + - branch_name + - ci_conclusion + - ci_event + - ci_failed_jobs + - ci_loop_count + - claimed + - cleanup_succeeded + - depth + - fixes_applied + - hunk_ranges_path + - implementation_ref + - investigation_path + - issue_number + - issue_slug + - issue_title + - issue_url + - lens_context_paths + - max_parallel + - merge_group_trigger + - merge_target + - open_pr + - phases_implemented + - plan_path + - pr_number + - pr_url + - prep_path + - proposed_branch + - queue_available + - queue_pr_state + - remediation_path + - remote_url + - review_approach + - review_loop_count + - review_max_retries + - review_path + - review_verdict + - run_mode + - run_name + - selected_lenses + - source_dir + - task + - upfront_claimed + - verdict + - work_dir + - worktree_path + required: [] + produced: + - stall_loop_count +- step: check_eject_limit + available: + - active_trigger_count + - annotated_diff_path + - arch_lenses + - audit + - auto_merge + - auto_merge_available + - base_branch + - branch_name + - ci_conclusion + - ci_event + - ci_failed_jobs + - ci_loop_count + - claimed + - cleanup_succeeded + - depth + - fixes_applied + - hunk_ranges_path + - implementation_ref + - investigation_path + - issue_number + - issue_slug + - issue_title + - issue_url + - lens_context_paths + - max_parallel + - merge_group_trigger + - merge_target + - open_pr + - phases_implemented + - plan_path + - pr_number + - pr_url + - prep_path + - proposed_branch + - queue_available + - queue_pr_state + - remediation_path + - remote_url + - review_approach + - review_loop_count + - review_max_retries + - review_path + - review_verdict + - run_mode + - run_name + - selected_lenses + - source_dir + - stall_loop_count + - task + - upfront_claimed + - verdict + - work_dir + - worktree_path + required: [] + produced: [] +- step: check_dropped_healthy_loop + available: + - active_trigger_count + - annotated_diff_path + - arch_lenses + - audit + - auto_merge + - auto_merge_available + - base_branch + - branch_name + - ci_conclusion + - ci_event + - ci_failed_jobs + - ci_loop_count + - claimed + - cleanup_succeeded + - depth + - fixes_applied + - hunk_ranges_path + - implementation_ref + - investigation_path + - issue_number + - issue_slug + - issue_title + - issue_url + - lens_context_paths + - max_parallel + - merge_group_trigger + - merge_target + - open_pr + - phases_implemented + - plan_path + - pr_number + - pr_url + - prep_path + - proposed_branch + - queue_available + - queue_pr_state + - remediation_path + - remote_url + - review_approach + - review_loop_count + - review_max_retries + - review_path + - review_verdict + - run_mode + - run_name + - selected_lenses + - source_dir + - stall_loop_count + - task + - upfront_claimed + - verdict + - work_dir + - worktree_path + required: [] + produced: [] +- step: queue_ejected_fix + available: + - active_trigger_count + - annotated_diff_path + - arch_lenses + - audit + - auto_merge + - auto_merge_available + - base_branch + - branch_name + - ci_conclusion + - ci_event + - ci_failed_jobs + - ci_loop_count + - claimed + - cleanup_succeeded + - depth + - fixes_applied + - hunk_ranges_path + - implementation_ref + - investigation_path + - issue_number + - issue_slug + - issue_title + - issue_url + - lens_context_paths + - max_parallel + - merge_group_trigger + - merge_target + - open_pr + - phases_implemented + - plan_path + - pr_number + - pr_url + - prep_path + - proposed_branch + - queue_available + - queue_pr_state + - remediation_path + - remote_url + - review_approach + - review_loop_count + - review_max_retries + - review_path + - review_verdict + - run_mode + - run_name + - selected_lenses + - source_dir + - stall_loop_count + - task + - upfront_claimed + - verdict + - work_dir + - worktree_path + required: [] + produced: [] +- step: resolve_queue_merge_conflicts + available: + - active_trigger_count + - annotated_diff_path + - arch_lenses + - audit + - auto_merge + - auto_merge_available + - base_branch + - branch_name + - ci_conclusion + - ci_event + - ci_failed_jobs + - ci_loop_count + - claimed + - cleanup_succeeded + - depth + - fixes_applied + - hunk_ranges_path + - implementation_ref + - investigation_path + - issue_number + - issue_slug + - issue_title + - issue_url + - lens_context_paths + - max_parallel + - merge_group_trigger + - merge_target + - open_pr + - phases_implemented + - plan_path + - pr_number + - pr_url + - prep_path + - proposed_branch + - queue_available + - queue_pr_state + - remediation_path + - remote_url + - review_approach + - review_loop_count + - review_max_retries + - review_path + - review_verdict + - run_mode + - run_name + - selected_lenses + - source_dir + - stall_loop_count + - task + - upfront_claimed + - verdict + - work_dir + - worktree_path + required: + - worktree_path + produced: + - conflict_escalation_required +- step: re_push_queue_fix + available: + - active_trigger_count + - annotated_diff_path + - arch_lenses + - audit + - auto_merge + - auto_merge_available + - base_branch + - branch_name + - ci_conclusion + - ci_event + - ci_failed_jobs + - ci_loop_count + - claimed + - cleanup_succeeded + - conflict_escalation_required + - depth + - fixes_applied + - hunk_ranges_path + - implementation_ref + - investigation_path + - issue_number + - issue_slug + - issue_title + - issue_url + - lens_context_paths + - max_parallel + - merge_group_trigger + - merge_target + - open_pr + - phases_implemented + - plan_path + - pr_number + - pr_url + - prep_path + - proposed_branch + - queue_available + - queue_pr_state + - remediation_path + - remote_url + - review_approach + - review_loop_count + - review_max_retries + - review_path + - review_verdict + - run_mode + - run_name + - selected_lenses + - source_dir + - stall_loop_count + - task + - upfront_claimed + - verdict + - work_dir + - worktree_path + required: [] + produced: [] +- step: ci_watch_post_queue_fix + available: + - active_trigger_count + - annotated_diff_path + - arch_lenses + - audit + - auto_merge + - auto_merge_available + - base_branch + - branch_name + - ci_conclusion + - ci_event + - ci_failed_jobs + - ci_loop_count + - claimed + - cleanup_succeeded + - conflict_escalation_required + - depth + - fixes_applied + - hunk_ranges_path + - implementation_ref + - investigation_path + - issue_number + - issue_slug + - issue_title + - issue_url + - lens_context_paths + - max_parallel + - merge_group_trigger + - merge_target + - open_pr + - phases_implemented + - plan_path + - pr_number + - pr_url + - prep_path + - proposed_branch + - queue_available + - queue_pr_state + - remediation_path + - remote_url + - review_approach + - review_loop_count + - review_max_retries + - review_path + - review_verdict + - run_mode + - run_name + - selected_lenses + - source_dir + - stall_loop_count + - task + - upfront_claimed + - verdict + - work_dir + - worktree_path + required: [] + produced: + - ci_conclusion + - ci_failed_jobs +- step: check_ci_post_queue_loop + available: + - active_trigger_count + - annotated_diff_path + - arch_lenses + - audit + - auto_merge + - auto_merge_available + - base_branch + - branch_name + - ci_conclusion + - ci_event + - ci_failed_jobs + - ci_loop_count + - claimed + - cleanup_succeeded + - conflict_escalation_required + - depth + - fixes_applied + - hunk_ranges_path + - implementation_ref + - investigation_path + - issue_number + - issue_slug + - issue_title + - issue_url + - lens_context_paths + - max_parallel + - merge_group_trigger + - merge_target + - open_pr + - phases_implemented + - plan_path + - pr_number + - pr_url + - prep_path + - proposed_branch + - queue_available + - queue_pr_state + - remediation_path + - remote_url + - review_approach + - review_loop_count + - review_max_retries + - review_path + - review_verdict + - run_mode + - run_name + - selected_lenses + - source_dir + - stall_loop_count + - task + - upfront_claimed + - verdict + - work_dir + - worktree_path + required: [] + produced: + - ci_post_queue_loop_count +- step: reenter_merge_queue + available: + - active_trigger_count + - annotated_diff_path + - arch_lenses + - audit + - auto_merge + - auto_merge_available + - base_branch + - branch_name + - ci_conclusion + - ci_event + - ci_failed_jobs + - ci_loop_count + - ci_post_queue_loop_count + - claimed + - cleanup_succeeded + - conflict_escalation_required + - depth + - fixes_applied + - hunk_ranges_path + - implementation_ref + - investigation_path + - issue_number + - issue_slug + - issue_title + - issue_url + - lens_context_paths + - max_parallel + - merge_group_trigger + - merge_target + - open_pr + - phases_implemented + - plan_path + - pr_number + - pr_url + - prep_path + - proposed_branch + - queue_available + - queue_pr_state + - remediation_path + - remote_url + - review_approach + - review_loop_count + - review_max_retries + - review_path + - review_verdict + - run_mode + - run_name + - selected_lenses + - source_dir + - stall_loop_count + - task + - upfront_claimed + - verdict + - work_dir + - worktree_path + required: [] + produced: [] +- step: reenter_merge_queue_cheap + available: + - active_trigger_count + - annotated_diff_path + - arch_lenses + - audit + - auto_merge + - auto_merge_available + - base_branch + - branch_name + - ci_conclusion + - ci_event + - ci_failed_jobs + - ci_loop_count + - ci_post_queue_loop_count + - claimed + - cleanup_succeeded + - conflict_escalation_required + - depth + - fixes_applied + - hunk_ranges_path + - implementation_ref + - investigation_path + - issue_number + - issue_slug + - issue_title + - issue_url + - lens_context_paths + - max_parallel + - merge_group_trigger + - merge_target + - open_pr + - phases_implemented + - plan_path + - pr_number + - pr_url + - prep_path + - proposed_branch + - queue_available + - queue_pr_state + - remediation_path + - remote_url + - review_approach + - review_loop_count + - review_max_retries + - review_path + - review_verdict + - run_mode + - run_name + - selected_lenses + - source_dir + - stall_loop_count + - task + - upfront_claimed + - verdict + - work_dir + - worktree_path + required: [] + produced: [] +- step: direct_merge + available: + - active_trigger_count + - annotated_diff_path + - arch_lenses + - audit + - auto_merge + - auto_merge_available + - base_branch + - branch_name + - ci_conclusion + - ci_event + - ci_failed_jobs + - ci_loop_count + - ci_post_queue_loop_count + - claimed + - cleanup_succeeded + - conflict_escalation_required + - depth + - fixes_applied + - hunk_ranges_path + - implementation_ref + - investigation_path + - issue_number + - issue_slug + - issue_title + - issue_url + - lens_context_paths + - max_parallel + - merge_group_trigger + - merge_target + - open_pr + - phases_implemented + - plan_path + - pr_number + - pr_url + - prep_path + - proposed_branch + - queue_available + - queue_pr_state + - remediation_path + - remote_url + - review_approach + - review_loop_count + - review_max_retries + - review_path + - review_verdict + - run_mode + - run_name + - selected_lenses + - source_dir + - stall_loop_count + - task + - upfront_claimed + - verdict + - work_dir + - worktree_path + required: [] + produced: [] +- step: wait_for_direct_merge + available: + - active_trigger_count + - annotated_diff_path + - arch_lenses + - audit + - auto_merge + - auto_merge_available + - base_branch + - branch_name + - ci_conclusion + - ci_event + - ci_failed_jobs + - ci_loop_count + - ci_post_queue_loop_count + - claimed + - cleanup_succeeded + - conflict_escalation_required + - depth + - fixes_applied + - hunk_ranges_path + - implementation_ref + - investigation_path + - issue_number + - issue_slug + - issue_title + - issue_url + - lens_context_paths + - max_parallel + - merge_group_trigger + - merge_target + - open_pr + - phases_implemented + - plan_path + - pr_number + - pr_url + - prep_path + - proposed_branch + - queue_available + - queue_pr_state + - remediation_path + - remote_url + - review_approach + - review_loop_count + - review_max_retries + - review_path + - review_verdict + - run_mode + - run_name + - selected_lenses + - source_dir + - stall_loop_count + - task + - upfront_claimed + - verdict + - work_dir + - worktree_path + required: [] + produced: [] +- step: direct_merge_conflict_fix + available: + - active_trigger_count + - annotated_diff_path + - arch_lenses + - audit + - auto_merge + - auto_merge_available + - base_branch + - branch_name + - ci_conclusion + - ci_event + - ci_failed_jobs + - ci_loop_count + - ci_post_queue_loop_count + - claimed + - cleanup_succeeded + - conflict_escalation_required + - depth + - fixes_applied + - hunk_ranges_path + - implementation_ref + - investigation_path + - issue_number + - issue_slug + - issue_title + - issue_url + - lens_context_paths + - max_parallel + - merge_group_trigger + - merge_target + - open_pr + - phases_implemented + - plan_path + - pr_number + - pr_url + - prep_path + - proposed_branch + - queue_available + - queue_pr_state + - remediation_path + - remote_url + - review_approach + - review_loop_count + - review_max_retries + - review_path + - review_verdict + - run_mode + - run_name + - selected_lenses + - source_dir + - stall_loop_count + - task + - upfront_claimed + - verdict + - work_dir + - worktree_path + required: [] + produced: [] +- step: resolve_direct_merge_conflicts + available: + - active_trigger_count + - annotated_diff_path + - arch_lenses + - audit + - auto_merge + - auto_merge_available + - base_branch + - branch_name + - ci_conclusion + - ci_event + - ci_failed_jobs + - ci_loop_count + - ci_post_queue_loop_count + - claimed + - cleanup_succeeded + - conflict_escalation_required + - depth + - fixes_applied + - hunk_ranges_path + - implementation_ref + - investigation_path + - issue_number + - issue_slug + - issue_title + - issue_url + - lens_context_paths + - max_parallel + - merge_group_trigger + - merge_target + - open_pr + - phases_implemented + - plan_path + - pr_number + - pr_url + - prep_path + - proposed_branch + - queue_available + - queue_pr_state + - remediation_path + - remote_url + - review_approach + - review_loop_count + - review_max_retries + - review_path + - review_verdict + - run_mode + - run_name + - selected_lenses + - source_dir + - stall_loop_count + - task + - upfront_claimed + - verdict + - work_dir + - worktree_path + required: + - worktree_path + produced: + - conflict_escalation_required +- step: re_push_direct_fix + available: + - active_trigger_count + - annotated_diff_path + - arch_lenses + - audit + - auto_merge + - auto_merge_available + - base_branch + - branch_name + - ci_conclusion + - ci_event + - ci_failed_jobs + - ci_loop_count + - ci_post_queue_loop_count + - claimed + - cleanup_succeeded + - conflict_escalation_required + - depth + - fixes_applied + - hunk_ranges_path + - implementation_ref + - investigation_path + - issue_number + - issue_slug + - issue_title + - issue_url + - lens_context_paths + - max_parallel + - merge_group_trigger + - merge_target + - open_pr + - phases_implemented + - plan_path + - pr_number + - pr_url + - prep_path + - proposed_branch + - queue_available - queue_pr_state - remediation_path - remote_url - review_approach + - review_loop_count + - review_max_retries - review_path + - review_verdict - run_mode - run_name + - selected_lenses - source_dir - - sprint_mode + - stall_loop_count - task - upfront_claimed - verdict @@ -1573,38 +3284,56 @@ dataflow: produced: [] - step: redirect_merge available: + - active_trigger_count + - annotated_diff_path + - arch_lenses - audit - auto_merge - auto_merge_available - base_branch - branch_name - ci_conclusion + - ci_event - ci_failed_jobs + - ci_loop_count + - ci_post_queue_loop_count - claimed - cleanup_succeeded - conflict_escalation_required + - depth + - fixes_applied + - hunk_ranges_path - implementation_ref - investigation_path - issue_number - issue_slug - issue_title - issue_url + - lens_context_paths + - max_parallel - merge_group_trigger - merge_target - open_pr + - phases_implemented - plan_path - pr_number + - pr_url + - prep_path - proposed_branch - queue_available - queue_pr_state - remediation_path - remote_url - review_approach + - review_loop_count + - review_max_retries - review_path + - review_verdict - run_mode - run_name + - selected_lenses - source_dir - - sprint_mode + - stall_loop_count - task - upfront_claimed - verdict @@ -1614,38 +3343,56 @@ dataflow: produced: [] - step: immediate_merge available: + - active_trigger_count + - annotated_diff_path + - arch_lenses - audit - auto_merge - auto_merge_available - base_branch - branch_name - ci_conclusion + - ci_event - ci_failed_jobs + - ci_loop_count + - ci_post_queue_loop_count - claimed - cleanup_succeeded - conflict_escalation_required + - depth + - fixes_applied + - hunk_ranges_path - implementation_ref - investigation_path - issue_number - issue_slug - issue_title - issue_url + - lens_context_paths + - max_parallel - merge_group_trigger - merge_target - open_pr + - phases_implemented - plan_path - pr_number + - pr_url + - prep_path - proposed_branch - queue_available - queue_pr_state - remediation_path - remote_url - review_approach + - review_loop_count + - review_max_retries - review_path + - review_verdict - run_mode - run_name + - selected_lenses - source_dir - - sprint_mode + - stall_loop_count - task - upfront_claimed - verdict @@ -1655,38 +3402,56 @@ dataflow: produced: [] - step: wait_for_immediate_merge available: + - active_trigger_count + - annotated_diff_path + - arch_lenses - audit - auto_merge - auto_merge_available - base_branch - branch_name - ci_conclusion + - ci_event - ci_failed_jobs + - ci_loop_count + - ci_post_queue_loop_count - claimed - cleanup_succeeded - conflict_escalation_required + - depth + - fixes_applied + - hunk_ranges_path - implementation_ref - investigation_path - issue_number - issue_slug - issue_title - issue_url + - lens_context_paths + - max_parallel - merge_group_trigger - merge_target - open_pr + - phases_implemented - plan_path - pr_number + - pr_url + - prep_path - proposed_branch - queue_available - queue_pr_state - remediation_path - remote_url - review_approach + - review_loop_count + - review_max_retries - review_path + - review_verdict - run_mode - run_name + - selected_lenses - source_dir - - sprint_mode + - stall_loop_count - task - upfront_claimed - verdict @@ -1696,38 +3461,115 @@ dataflow: produced: [] - step: immediate_merge_conflict_fix available: + - active_trigger_count + - annotated_diff_path + - arch_lenses + - audit + - auto_merge + - auto_merge_available + - base_branch + - branch_name + - ci_conclusion + - ci_event + - ci_failed_jobs + - ci_loop_count + - ci_post_queue_loop_count + - claimed + - cleanup_succeeded + - conflict_escalation_required + - depth + - fixes_applied + - hunk_ranges_path + - implementation_ref + - investigation_path + - issue_number + - issue_slug + - issue_title + - issue_url + - lens_context_paths + - max_parallel + - merge_group_trigger + - merge_target + - open_pr + - phases_implemented + - plan_path + - pr_number + - pr_url + - prep_path + - proposed_branch + - queue_available + - queue_pr_state + - remediation_path + - remote_url + - review_approach + - review_loop_count + - review_max_retries + - review_path + - review_verdict + - run_mode + - run_name + - selected_lenses + - source_dir + - stall_loop_count + - task + - upfront_claimed + - verdict + - work_dir + - worktree_path + required: [] + produced: [] +- step: resolve_immediate_merge_conflicts + available: + - active_trigger_count + - annotated_diff_path + - arch_lenses - audit - auto_merge - auto_merge_available - base_branch - branch_name - ci_conclusion + - ci_event - ci_failed_jobs + - ci_loop_count + - ci_post_queue_loop_count - claimed - cleanup_succeeded - conflict_escalation_required + - depth + - fixes_applied + - hunk_ranges_path - implementation_ref - investigation_path - issue_number - issue_slug - issue_title - issue_url + - lens_context_paths + - max_parallel - merge_group_trigger - merge_target - open_pr + - phases_implemented - plan_path - pr_number + - pr_url + - prep_path - proposed_branch - queue_available - queue_pr_state - remediation_path - remote_url - review_approach + - review_loop_count + - review_max_retries - review_path + - review_verdict - run_mode - run_name + - selected_lenses - source_dir - - sprint_mode + - stall_loop_count - task - upfront_claimed - verdict @@ -1739,38 +3581,56 @@ dataflow: - conflict_escalation_required - step: re_push_immediate_fix available: + - active_trigger_count + - annotated_diff_path + - arch_lenses - audit - auto_merge - auto_merge_available - base_branch - branch_name - ci_conclusion + - ci_event - ci_failed_jobs + - ci_loop_count + - ci_post_queue_loop_count - claimed - cleanup_succeeded - conflict_escalation_required + - depth + - fixes_applied + - hunk_ranges_path - implementation_ref - investigation_path - issue_number - issue_slug - issue_title - issue_url + - lens_context_paths + - max_parallel - merge_group_trigger - merge_target - open_pr + - phases_implemented - plan_path - pr_number + - pr_url + - prep_path - proposed_branch - queue_available - queue_pr_state - remediation_path - remote_url - review_approach + - review_loop_count + - review_max_retries - review_path + - review_verdict - run_mode - run_name + - selected_lenses - source_dir - - sprint_mode + - stall_loop_count - task - upfront_claimed - verdict @@ -1780,38 +3640,56 @@ dataflow: produced: [] - step: remerge_immediate available: + - active_trigger_count + - annotated_diff_path + - arch_lenses - audit - auto_merge - auto_merge_available - base_branch - branch_name - ci_conclusion + - ci_event - ci_failed_jobs + - ci_loop_count + - ci_post_queue_loop_count - claimed - cleanup_succeeded - conflict_escalation_required + - depth + - fixes_applied + - hunk_ranges_path - implementation_ref - investigation_path - issue_number - issue_slug - issue_title - issue_url + - lens_context_paths + - max_parallel - merge_group_trigger - merge_target - open_pr + - phases_implemented - plan_path - pr_number + - pr_url + - prep_path - proposed_branch - queue_available - queue_pr_state - remediation_path - remote_url - review_approach + - review_loop_count + - review_max_retries - review_path + - review_verdict - run_mode - run_name + - selected_lenses - source_dir - - sprint_mode + - stall_loop_count - task - upfront_claimed - verdict @@ -1821,38 +3699,56 @@ dataflow: produced: [] - step: diagnose_ci available: + - active_trigger_count + - annotated_diff_path + - arch_lenses - audit - auto_merge - auto_merge_available - base_branch - branch_name - ci_conclusion + - ci_event - ci_failed_jobs + - ci_loop_count + - ci_post_queue_loop_count - claimed - cleanup_succeeded - conflict_escalation_required + - depth + - fixes_applied + - hunk_ranges_path - implementation_ref - investigation_path - issue_number - issue_slug - issue_title - issue_url + - lens_context_paths + - max_parallel - merge_group_trigger - merge_target - open_pr + - phases_implemented - plan_path - pr_number + - pr_url + - prep_path - proposed_branch - queue_available - queue_pr_state - remediation_path - remote_url - review_approach + - review_loop_count + - review_max_retries - review_path + - review_verdict - run_mode - run_name + - selected_lenses - source_dir - - sprint_mode + - stall_loop_count - task - upfront_claimed - verdict @@ -1861,41 +3757,60 @@ dataflow: required: [] produced: - diagnosis_path + positional_args: 3 - step: resolve_ci available: + - active_trigger_count + - annotated_diff_path + - arch_lenses - audit - auto_merge - auto_merge_available - base_branch - branch_name - ci_conclusion + - ci_event - ci_failed_jobs + - ci_loop_count + - ci_post_queue_loop_count - claimed - cleanup_succeeded - conflict_escalation_required + - depth - diagnosis_path + - fixes_applied + - hunk_ranges_path - implementation_ref - investigation_path - issue_number - issue_slug - issue_title - issue_url + - lens_context_paths + - max_parallel - merge_group_trigger - merge_target - open_pr + - phases_implemented - plan_path - pr_number + - pr_url + - prep_path - proposed_branch - queue_available - queue_pr_state - remediation_path - remote_url - review_approach + - review_loop_count + - review_max_retries - review_path + - review_verdict - run_mode - run_name + - selected_lenses - source_dir - - sprint_mode + - stall_loop_count - task - upfront_claimed - verdict @@ -1903,42 +3818,121 @@ dataflow: - worktree_path required: - worktree_path + produced: + - verdict +- step: pre_resolve_rebase + available: + - active_trigger_count + - annotated_diff_path + - arch_lenses + - audit + - auto_merge + - auto_merge_available + - base_branch + - branch_name + - ci_conclusion + - ci_event + - ci_failed_jobs + - ci_loop_count + - ci_post_queue_loop_count + - claimed + - cleanup_succeeded + - conflict_escalation_required + - depth + - diagnosis_path + - fixes_applied + - hunk_ranges_path + - implementation_ref + - investigation_path + - issue_number + - issue_slug + - issue_title + - issue_url + - lens_context_paths + - max_parallel + - merge_group_trigger + - merge_target + - open_pr + - phases_implemented + - plan_path + - pr_number + - pr_url + - prep_path + - proposed_branch + - queue_available + - queue_pr_state + - remediation_path + - remote_url + - review_approach + - review_loop_count + - review_max_retries + - review_path + - review_verdict + - run_mode + - run_name + - selected_lenses + - source_dir + - stall_loop_count + - task + - upfront_claimed + - verdict + - work_dir + - worktree_path + required: [] produced: [] - step: re_push available: + - active_trigger_count + - annotated_diff_path + - arch_lenses - audit - auto_merge - auto_merge_available - base_branch - branch_name - ci_conclusion + - ci_event - ci_failed_jobs + - ci_loop_count + - ci_post_queue_loop_count - claimed - cleanup_succeeded - conflict_escalation_required + - depth - diagnosis_path + - fixes_applied + - hunk_ranges_path - implementation_ref - investigation_path - issue_number - issue_slug - issue_title - issue_url + - lens_context_paths + - max_parallel - merge_group_trigger - merge_target - open_pr + - phases_implemented - plan_path - pr_number + - pr_url + - prep_path - proposed_branch - queue_available - queue_pr_state - remediation_path - remote_url - review_approach + - review_loop_count + - review_max_retries - review_path + - review_verdict - run_mode - run_name + - selected_lenses - source_dir - - sprint_mode + - stall_loop_count - task - upfront_claimed - verdict @@ -1948,39 +3942,57 @@ dataflow: produced: [] - step: detect_ci_conflict available: + - active_trigger_count + - annotated_diff_path + - arch_lenses - audit - auto_merge - auto_merge_available - base_branch - branch_name - ci_conclusion + - ci_event - ci_failed_jobs + - ci_loop_count + - ci_post_queue_loop_count - claimed - cleanup_succeeded - conflict_escalation_required + - depth - diagnosis_path + - fixes_applied + - hunk_ranges_path - implementation_ref - investigation_path - issue_number - issue_slug - issue_title - issue_url + - lens_context_paths + - max_parallel - merge_group_trigger - merge_target - open_pr + - phases_implemented - plan_path - pr_number + - pr_url + - prep_path - proposed_branch - queue_available - queue_pr_state - remediation_path - remote_url - review_approach + - review_loop_count + - review_max_retries - review_path + - review_verdict - run_mode - run_name + - selected_lenses - source_dir - - sprint_mode + - stall_loop_count - task - upfront_claimed - verdict @@ -1990,39 +4002,57 @@ dataflow: produced: [] - step: ci_conflict_fix available: + - active_trigger_count + - annotated_diff_path + - arch_lenses - audit - auto_merge - auto_merge_available - base_branch - branch_name - ci_conclusion + - ci_event - ci_failed_jobs + - ci_loop_count + - ci_post_queue_loop_count - claimed - cleanup_succeeded - conflict_escalation_required + - depth - diagnosis_path + - fixes_applied + - hunk_ranges_path - implementation_ref - investigation_path - issue_number - issue_slug - issue_title - issue_url + - lens_context_paths + - max_parallel - merge_group_trigger - merge_target - open_pr + - phases_implemented - plan_path - pr_number + - pr_url + - prep_path - proposed_branch - queue_available - queue_pr_state - remediation_path - remote_url - review_approach + - review_loop_count + - review_max_retries - review_path + - review_verdict - run_mode - run_name + - selected_lenses - source_dir - - sprint_mode + - stall_loop_count - task - upfront_claimed - verdict @@ -2032,41 +4062,59 @@ dataflow: - worktree_path produced: - conflict_escalation_required -- step: release_issue_timeout +- step: register_clone_unconfirmed available: + - active_trigger_count + - annotated_diff_path + - arch_lenses - audit - auto_merge - auto_merge_available - base_branch - branch_name - ci_conclusion + - ci_event - ci_failed_jobs + - ci_loop_count + - ci_post_queue_loop_count - claimed - cleanup_succeeded - conflict_escalation_required + - depth - diagnosis_path + - fixes_applied + - hunk_ranges_path - implementation_ref - investigation_path - issue_number - issue_slug - issue_title - issue_url + - lens_context_paths + - max_parallel - merge_group_trigger - merge_target - open_pr + - phases_implemented - plan_path - pr_number + - pr_url + - prep_path - proposed_branch - queue_available - queue_pr_state - remediation_path - remote_url - review_approach + - review_loop_count + - review_max_retries - review_path + - review_verdict - run_mode - run_name + - selected_lenses - source_dir - - sprint_mode + - stall_loop_count - task - upfront_claimed - verdict @@ -2076,39 +4124,117 @@ dataflow: produced: [] - step: release_issue_success available: + - active_trigger_count + - annotated_diff_path + - arch_lenses + - audit + - auto_merge + - auto_merge_available + - base_branch + - branch_name + - ci_conclusion + - ci_event + - ci_failed_jobs + - ci_loop_count + - ci_post_queue_loop_count + - claimed + - cleanup_succeeded + - conflict_escalation_required + - depth + - diagnosis_path + - fixes_applied + - hunk_ranges_path + - implementation_ref + - investigation_path + - issue_number + - issue_slug + - issue_title + - issue_url + - lens_context_paths + - max_parallel + - merge_group_trigger + - merge_target + - open_pr + - phases_implemented + - plan_path + - pr_number + - pr_url + - prep_path + - proposed_branch + - queue_available + - queue_pr_state + - remediation_path + - remote_url + - review_approach + - review_loop_count + - review_max_retries + - review_path + - review_verdict + - run_mode + - run_name + - selected_lenses + - source_dir + - stall_loop_count + - task + - upfront_claimed + - verdict + - work_dir + - worktree_path + required: [] + produced: [] +- step: patch_token_summary + available: + - active_trigger_count + - annotated_diff_path + - arch_lenses - audit - auto_merge - auto_merge_available - base_branch - branch_name - ci_conclusion + - ci_event - ci_failed_jobs + - ci_loop_count + - ci_post_queue_loop_count - claimed - cleanup_succeeded - conflict_escalation_required + - depth - diagnosis_path + - fixes_applied + - hunk_ranges_path - implementation_ref - investigation_path - issue_number - issue_slug - issue_title - issue_url + - lens_context_paths + - max_parallel - merge_group_trigger - merge_target - open_pr + - phases_implemented - plan_path - pr_number + - pr_url + - prep_path - proposed_branch - queue_available - queue_pr_state - remediation_path - remote_url - review_approach + - review_loop_count + - review_max_retries - review_path + - review_verdict - run_mode - run_name + - selected_lenses - source_dir - - sprint_mode + - stall_loop_count - task - upfront_claimed - verdict @@ -2118,39 +4244,57 @@ dataflow: produced: [] - step: release_issue_failure available: + - active_trigger_count + - annotated_diff_path + - arch_lenses - audit - auto_merge - auto_merge_available - base_branch - branch_name - ci_conclusion + - ci_event - ci_failed_jobs + - ci_loop_count + - ci_post_queue_loop_count - claimed - cleanup_succeeded - conflict_escalation_required + - depth - diagnosis_path + - fixes_applied + - hunk_ranges_path - implementation_ref - investigation_path - issue_number - issue_slug - issue_title - issue_url + - lens_context_paths + - max_parallel - merge_group_trigger - merge_target - open_pr + - phases_implemented - plan_path - pr_number + - pr_url + - prep_path - proposed_branch - queue_available - queue_pr_state - remediation_path - remote_url - review_approach + - review_loop_count + - review_max_retries - review_path + - review_verdict - run_mode - run_name + - selected_lenses - source_dir - - sprint_mode + - stall_loop_count - task - upfront_claimed - verdict @@ -2160,39 +4304,57 @@ dataflow: produced: [] - step: register_clone_success available: + - active_trigger_count + - annotated_diff_path + - arch_lenses - audit - auto_merge - auto_merge_available - base_branch - branch_name - ci_conclusion + - ci_event - ci_failed_jobs + - ci_loop_count + - ci_post_queue_loop_count - claimed - cleanup_succeeded - conflict_escalation_required + - depth - diagnosis_path + - fixes_applied + - hunk_ranges_path - implementation_ref - investigation_path - issue_number - issue_slug - issue_title - issue_url + - lens_context_paths + - max_parallel - merge_group_trigger - merge_target - open_pr + - phases_implemented - plan_path - pr_number + - pr_url + - prep_path - proposed_branch - queue_available - queue_pr_state - remediation_path - remote_url - review_approach + - review_loop_count + - review_max_retries - review_path + - review_verdict - run_mode - run_name + - selected_lenses - source_dir - - sprint_mode + - stall_loop_count - task - upfront_claimed - verdict @@ -2202,39 +4364,57 @@ dataflow: produced: [] - step: register_clone_failure available: + - active_trigger_count + - annotated_diff_path + - arch_lenses - audit - auto_merge - auto_merge_available - base_branch - branch_name - ci_conclusion + - ci_event - ci_failed_jobs + - ci_loop_count + - ci_post_queue_loop_count - claimed - cleanup_succeeded - conflict_escalation_required + - depth - diagnosis_path + - fixes_applied + - hunk_ranges_path - implementation_ref - investigation_path - issue_number - issue_slug - issue_title - issue_url + - lens_context_paths + - max_parallel - merge_group_trigger - merge_target - open_pr + - phases_implemented - plan_path - pr_number + - pr_url + - prep_path - proposed_branch - queue_available - queue_pr_state - remediation_path - remote_url - review_approach + - review_loop_count + - review_max_retries - review_path + - review_verdict - run_mode - run_name + - selected_lenses - source_dir - - sprint_mode + - stall_loop_count - task - upfront_claimed - verdict @@ -2244,39 +4424,57 @@ dataflow: produced: [] - step: done available: + - active_trigger_count + - annotated_diff_path + - arch_lenses - audit - auto_merge - auto_merge_available - base_branch - branch_name - ci_conclusion + - ci_event - ci_failed_jobs + - ci_loop_count + - ci_post_queue_loop_count - claimed - cleanup_succeeded - conflict_escalation_required + - depth - diagnosis_path + - fixes_applied + - hunk_ranges_path - implementation_ref - investigation_path - issue_number - issue_slug - issue_title - issue_url + - lens_context_paths + - max_parallel - merge_group_trigger - merge_target - open_pr + - phases_implemented - plan_path - pr_number + - pr_url + - prep_path - proposed_branch - queue_available - queue_pr_state - remediation_path - remote_url - review_approach + - review_loop_count + - review_max_retries - review_path + - review_verdict - run_mode - run_name + - selected_lenses - source_dir - - sprint_mode + - stall_loop_count - task - upfront_claimed - verdict @@ -2286,39 +4484,57 @@ dataflow: produced: [] - step: escalate_stop available: + - active_trigger_count + - annotated_diff_path + - arch_lenses - audit - auto_merge - auto_merge_available - base_branch - branch_name - ci_conclusion + - ci_event - ci_failed_jobs + - ci_loop_count + - ci_post_queue_loop_count - claimed - cleanup_succeeded - conflict_escalation_required + - depth - diagnosis_path + - fixes_applied + - hunk_ranges_path - implementation_ref - investigation_path - issue_number - issue_slug - issue_title - issue_url + - lens_context_paths + - max_parallel - merge_group_trigger - merge_target - open_pr + - phases_implemented - plan_path - pr_number + - pr_url + - prep_path - proposed_branch - queue_available - queue_pr_state - remediation_path - remote_url - review_approach + - review_loop_count + - review_max_retries - review_path + - review_verdict - run_mode - run_name + - selected_lenses - source_dir - - sprint_mode + - stall_loop_count - task - upfront_claimed - verdict diff --git a/src/autoskillit/recipes/contracts/research-campaign.json b/src/autoskillit/recipes/contracts/research-campaign.json new file mode 100644 index 0000000000..192fdc246c --- /dev/null +++ b/src/autoskillit/recipes/contracts/research-campaign.json @@ -0,0 +1,7 @@ +{ + "generated_at": "2026-05-06T14:51:35.117429+00:00", + "bundled_manifest_version": "0.1.0", + "skill_hashes": {}, + "skills": {}, + "dataflow": [] +} diff --git a/src/autoskillit/recipes/contracts/research-campaign.yaml b/src/autoskillit/recipes/contracts/research-campaign.yaml new file mode 100644 index 0000000000..85dcb59b69 --- /dev/null +++ b/src/autoskillit/recipes/contracts/research-campaign.yaml @@ -0,0 +1,5 @@ +generated_at: '2026-05-06T14:51:35.117429+00:00' +bundled_manifest_version: 0.1.0 +skill_hashes: {} +skills: {} +dataflow: [] diff --git a/src/autoskillit/recipes/contracts/research.json b/src/autoskillit/recipes/contracts/research.json new file mode 100644 index 0000000000..07bb0a3206 --- /dev/null +++ b/src/autoskillit/recipes/contracts/research.json @@ -0,0 +1,3190 @@ +{ + "generated_at": "2026-05-07T16:23:58.667262+00:00", + "bundled_manifest_version": "0.1.0", + "skill_hashes": {}, + "skills": { + "scope": { + "inputs": [ + { + "name": "research_question", + "type": "string", + "required": true, + "recommended": false + } + ], + "outputs": [ + { + "name": "scope_report", + "type": "file_path" + } + ], + "expected_output_patterns": [ + "scope_report\\s*=\\s*/.+" + ], + "pattern_examples": [ + "scope_report = /tmp/scope_report.md\n%%ORDER_UP%%" + ], + "write_behavior": "always" + }, + "plan-experiment": { + "inputs": [ + { + "name": "scope_report_path", + "type": "file_path", + "required": true, + "recommended": false + }, + { + "name": "revision_guidance", + "type": "file_path", + "required": false, + "recommended": false + } + ], + "outputs": [ + { + "name": "experiment_plan", + "type": "file_path" + } + ], + "expected_output_patterns": [ + "experiment_plan\\s*=\\s*/.+" + ], + "pattern_examples": [ + "experiment_plan = /tmp/experiment_plan.md\n%%ORDER_UP%%" + ], + "write_behavior": "always" + }, + "review-design": { + "inputs": [ + { + "name": "experiment_plan", + "type": "file_path", + "required": true, + "recommended": false + } + ], + "outputs": [ + { + "name": "verdict", + "type": "string" + }, + { + "name": "experiment_type", + "type": "string" + }, + { + "name": "evaluation_dashboard", + "type": "file_path" + }, + { + "name": "revision_guidance", + "type": "file_path" + }, + { + "name": "classification_timestamp", + "type": "string" + } + ], + "expected_output_patterns": [ + "verdict\\s*=\\s*(GO|REVISE|STOP)", + "(?:experiment_type\\s*=\\s*\\S+|verdict\\s*=\\s*STOP)" + ], + "pattern_examples": [ + "verdict = GO\nexperiment_type = benchmark\nevaluation_dashboard = /tmp/eval.md\nrevision_guidance = /tmp/guidance.md\n%%ORDER_UP%%", + "verdict = REVISE\nexperiment_type = benchmark\nrevision_guidance = /tmp/guidance.md\n%%ORDER_UP%%", + "verdict = STOP\n%%ORDER_UP%%", + "verdict = GO\nexperiment_type = causal_inference\nevaluation_dashboard = /tmp/eval.md\n%%ORDER_UP%%", + "verdict = GO\nexperiment_type = configuration_study\nevaluation_dashboard = /tmp/eval.md\n%%ORDER_UP%%", + "verdict = GO\nexperiment_type = robustness_audit\nevaluation_dashboard = /tmp/eval.md\n%%ORDER_UP%%", + "verdict = GO\nexperiment_type = exploratory\nevaluation_dashboard = /tmp/eval.md\n%%ORDER_UP%%" + ], + "write_behavior": "always" + }, + "plan-visualization": { + "inputs": [ + { + "name": "source_dir", + "type": "directory_path", + "required": true, + "recommended": false + }, + { + "name": "experiment_plan_path", + "type": "file_path", + "required": true, + "recommended": false + }, + { + "name": "scope_report_path", + "type": "string", + "required": false, + "recommended": false + }, + { + "name": "experiment_type", + "type": "string", + "required": false, + "recommended": false + } + ], + "outputs": [ + { + "name": "visualization_plan_path", + "type": "file_path" + }, + { + "name": "report_plan_path", + "type": "file_path" + }, + { + "name": "disambiguation_rule_applied", + "type": "string" + }, + { + "name": "tier_c_lens", + "type": "string" + }, + { + "name": "methodology_tradition", + "type": "string" + }, + { + "name": "visualization_plan_trace_path", + "type": "file_path" + } + ], + "expected_output_patterns": [ + "visualization_plan_path\\s*=\\s*/.+", + "report_plan_path\\s*=\\s*/.+", + "methodology_tradition\\s*=\\s*\\S+" + ], + "pattern_examples": [ + "visualization_plan_path = /tmp/plan-visualization/visualization-plan.md\nreport_plan_path = /tmp/plan-visualization/report-plan.md\nmethodology_tradition = controlled_intervention\n%%ORDER_UP%%", + "visualization_plan_path = /tmp/plan-visualization/visualization-plan.md\nreport_plan_path = /tmp/plan-visualization/report-plan.md\nmethodology_tradition = null\n%%ORDER_UP%%" + ], + "write_behavior": "always" + }, + "resolve-design-review": { + "inputs": [ + { + "name": "evaluation_dashboard", + "type": "file_path", + "required": true, + "recommended": false + }, + { + "name": "experiment_plan", + "type": "file_path", + "required": true, + "recommended": false + } + ], + "outputs": [ + { + "name": "resolution", + "type": "string" + }, + { + "name": "revision_guidance", + "type": "file_path" + } + ], + "expected_output_patterns": [ + "resolution\\s*=\\s*(revised|failed)", + "(?:revision_guidance\\s*=\\s*/.+|resolution\\s*=\\s*failed)" + ], + "pattern_examples": [ + "resolution = revised\nrevision_guidance = /tmp/guidance.md\n%%ORDER_UP%%", + "resolution = failed\n%%ORDER_UP%%" + ], + "write_behavior": "always" + }, + "stage-data": { + "inputs": [ + { + "name": "experiment_plan", + "type": "file_path", + "required": true, + "recommended": false + } + ], + "outputs": [ + { + "name": "verdict", + "type": "string" + }, + { + "name": "resource_report", + "type": "file_path" + } + ], + "expected_output_patterns": [ + "verdict\\s*=\\s*(PASS|WARN|FAIL)", + "resource_report\\s*=\\s*/.+" + ], + "pattern_examples": [ + "verdict = PASS\nresource_report = /tmp/stage-data/resource_feasibility_2026-04-13_120000.md\n%%ORDER_UP%%" + ], + "write_behavior": "always" + }, + "setup-environment": { + "inputs": [ + { + "name": "worktree_path", + "type": "directory_path", + "required": true, + "recommended": false + }, + { + "name": "experiment_plan", + "type": "file_path", + "required": true, + "recommended": false + } + ], + "outputs": [ + { + "name": "env_mode", + "type": "string" + }, + { + "name": "env_report", + "type": "file_path" + }, + { + "name": "verdict", + "type": "string" + } + ], + "expected_output_patterns": [ + "env_mode\\s*=\\s*(none|docker|micromamba-host|unavailable)", + "env_report\\s*=\\s*/.+", + "verdict\\s*=\\s*(PASS|WARN|FAIL)" + ], + "pattern_examples": [ + "env_mode = none\nenv_report = /tmp/setup-environment/env_setup_report.md\nverdict = PASS\n%%ORDER_UP%%", + "env_mode = docker\nenv_report = /tmp/setup-environment/env_setup_report.md\nverdict = PASS\n%%ORDER_UP%%", + "env_mode = micromamba-host\nenv_report = /tmp/setup-environment/env_setup_report.md\nverdict = WARN\n%%ORDER_UP%%", + "env_mode = unavailable\nenv_report = /tmp/setup-environment/env_setup_report.md\nverdict = FAIL\n%%ORDER_UP%%" + ], + "write_behavior": "always" + }, + "make-groups": { + "inputs": [ + { + "name": "source_doc", + "type": "file_path", + "required": true, + "recommended": false + } + ], + "outputs": [ + { + "name": "groups_path", + "type": "file_path" + }, + { + "name": "manifest_path", + "type": "file_path" + }, + { + "name": "group_files", + "type": "file_path_list" + } + ], + "expected_output_patterns": [ + "groups_path\\s*=\\s*/.+" + ], + "pattern_examples": [ + "groups_path = /tmp/groups.json\n%%ORDER_UP%%" + ], + "write_behavior": "always" + }, + "make-plan": { + "inputs": [ + { + "name": "task", + "type": "string", + "required": true, + "recommended": false + } + ], + "outputs": [ + { + "name": "plan_path", + "type": "file_path" + }, + { + "name": "plan_parts", + "type": "file_path_list" + } + ], + "expected_output_patterns": [ + "plan_path\\s*=\\s*/.+" + ], + "pattern_examples": [ + "plan_path = /tmp/plan.md\n%%ORDER_UP%%" + ], + "write_behavior": "always" + }, + "implement-experiment": { + "inputs": [ + { + "name": "plan_path", + "type": "file_path", + "required": true, + "recommended": false + } + ], + "outputs": [ + { + "name": "worktree_path", + "type": "directory_path" + }, + { + "name": "branch_name", + "type": "string" + } + ], + "expected_output_patterns": [ + "worktree_path\\s*=\\s*/.+" + ], + "pattern_examples": [ + "worktree_path = /tmp/worktrees/research-20260101-120000\n%%ORDER_UP%%" + ], + "write_behavior": "always" + }, + "troubleshoot-experiment": { + "inputs": [ + { + "name": "worktree_path", + "type": "string", + "required": true, + "recommended": false + }, + { + "name": "step_name", + "type": "string", + "required": true, + "recommended": false + } + ], + "outputs": [ + { + "name": "diagnosis_path", + "type": "file_path" + }, + { + "name": "failure_type", + "type": "string" + }, + { + "name": "is_fixable", + "type": "string" + } + ], + "expected_output_patterns": [ + "is_fixable\\s*=\\s*(true|false)" + ], + "pattern_examples": [ + "diagnosis_path = /tmp/diagnosis.md\nfailure_type = stale_timeout\nis_fixable = true\n%%ORDER_UP%%" + ], + "write_behavior": "always" + }, + "run-experiment": { + "inputs": [ + { + "name": "worktree_path", + "type": "directory_path", + "required": true, + "recommended": false + } + ], + "outputs": [ + { + "name": "results_path", + "type": "file_path" + } + ], + "expected_output_patterns": [ + "results_path\\s*=\\s*/.+" + ], + "pattern_examples": [ + "results_path = /tmp/run-experiment/results.md\n%%ORDER_UP%%" + ], + "write_behavior": "always" + }, + "generate-report": { + "inputs": [ + { + "name": "worktree_path", + "type": "directory_path", + "required": true, + "recommended": false + }, + { + "name": "results_path", + "type": "file_path", + "required": true, + "recommended": false + }, + { + "name": "experiment_type", + "type": "string", + "required": false, + "recommended": false + }, + { + "name": "methodology_tradition", + "type": "string", + "required": false, + "recommended": false + } + ], + "outputs": [ + { + "name": "report_path", + "type": "file_path" + } + ], + "expected_output_patterns": [ + "report_path\\s*=\\s*/.+" + ], + "pattern_examples": [ + "report_path = /tmp/research/report.md\n%%ORDER_UP%%" + ], + "write_behavior": "always" + }, + "resolve-failures": { + "inputs": [ + { + "name": "worktree_path", + "type": "directory_path", + "required": true, + "recommended": false + }, + { + "name": "plan_path", + "type": "file_path", + "required": true, + "recommended": false + }, + { + "name": "base_branch", + "type": "string", + "required": true, + "recommended": false + }, + { + "name": "diagnosis_path", + "type": "file_path", + "required": false, + "recommended": false + } + ], + "outputs": [ + { + "name": "verdict", + "type": "string" + }, + { + "name": "fixes_applied", + "type": "integer" + } + ], + "expected_output_patterns": [], + "pattern_examples": [], + "write_behavior": "conditional", + "write_expected_when": [ + "verdict\\s*=\\s*real_fix" + ] + }, + "prepare-research-pr": { + "inputs": [ + { + "name": "report_path", + "type": "file_path", + "required": true, + "recommended": false + }, + { + "name": "experiment_plan_path", + "type": "file_path", + "required": true, + "recommended": false + }, + { + "name": "worktree_path", + "type": "directory_path", + "required": true, + "recommended": false + }, + { + "name": "base_branch", + "type": "string", + "required": true, + "recommended": false + } + ], + "outputs": [ + { + "name": "prep_path", + "type": "file_path" + }, + { + "name": "selected_lenses", + "type": "string" + }, + { + "name": "lens_context_paths", + "type": "string" + } + ], + "expected_output_patterns": [ + "prep_path\\s*=\\s*/.+", + "selected_lenses\\s*=\\s*\\S+", + "lens_context_paths\\s*=\\s*/.+" + ], + "pattern_examples": [ + "prep_path = /abs/path/pr_prep_2026-04-07_101349.md\nselected_lenses = fair-comparison,estimand-clarity\nlens_context_paths = /abs/ctx1.md,/abs/ctx2.md\n%%ORDER_UP%%" + ], + "write_behavior": "always" + }, + "compose-research-pr": { + "inputs": [ + { + "name": "prep_path", + "type": "file_path", + "required": true, + "recommended": false + }, + { + "name": "all_diagram_paths", + "type": "string", + "required": true, + "recommended": false + }, + { + "name": "worktree_path", + "type": "directory_path", + "required": true, + "recommended": false + }, + { + "name": "base_branch", + "type": "string", + "required": true, + "recommended": false + }, + { + "name": "closing_issue", + "type": "string", + "required": false, + "recommended": false + } + ], + "outputs": [ + { + "name": "pr_url", + "type": "string" + } + ], + "expected_output_patterns": [ + "pr_url\\s*=\\s*(https://github\\.com/.*/pull/\\d+)?" + ], + "pattern_examples": [ + "pr_url = https://github.com/owner/repo/pull/42\n%%ORDER_UP%%", + "pr_url = \n%%ORDER_UP%%" + ], + "write_behavior": "always" + }, + "review-research-pr": { + "inputs": [ + { + "name": "worktree_path", + "type": "directory_path", + "required": true, + "recommended": false + }, + { + "name": "base_branch", + "type": "string", + "required": true, + "recommended": false + }, + { + "name": "pr_url", + "type": "string", + "required": true, + "recommended": false + } + ], + "outputs": [ + { + "name": "verdict", + "type": "string" + } + ], + "expected_output_patterns": [ + "verdict\\s*=\\s*(approved|changes_requested|needs_human)" + ], + "pattern_examples": [ + "verdict = approved\n%%ORDER_UP%%", + "verdict = changes_requested\n%%ORDER_UP%%", + "verdict = needs_human\n%%ORDER_UP%%" + ] + }, + "audit-claims": { + "inputs": [ + { + "name": "worktree_path", + "type": "directory_path", + "required": true, + "recommended": false + }, + { + "name": "base_branch", + "type": "string", + "required": true, + "recommended": false + }, + { + "name": "pr_url", + "type": "string", + "required": true, + "recommended": false + } + ], + "outputs": [ + { + "name": "verdict", + "type": "string" + } + ], + "expected_output_patterns": [ + "verdict\\s*=\\s*(approved|changes_requested|needs_human)" + ], + "pattern_examples": [ + "verdict = approved\n%%ORDER_UP%%", + "verdict = changes_requested\n%%ORDER_UP%%", + "verdict = needs_human\n%%ORDER_UP%%" + ] + }, + "resolve-research-review": { + "inputs": [ + { + "name": "worktree_path", + "type": "directory_path", + "required": true, + "recommended": false + }, + { + "name": "base_branch", + "type": "string", + "required": true, + "recommended": false + } + ], + "outputs": [ + { + "name": "needs_rerun", + "type": "string" + }, + { + "name": "verdict", + "type": "string" + }, + { + "name": "fixes_applied", + "type": "integer" + } + ], + "expected_output_patterns": [ + "needs_rerun\\s*=\\s*(true|false)" + ], + "pattern_examples": [ + "needs_rerun = true\nverdict = real_fix\nfixes_applied = 2\n%%ORDER_UP%%", + "needs_rerun = false\nverdict = already_green\nfixes_applied = 0\n%%ORDER_UP%%", + "needs_rerun = true\nverdict = flake_suspected\nfixes_applied = 0\n%%ORDER_UP%%", + "needs_rerun = false\nverdict = ci_only_failure\nfixes_applied = 0\n%%ORDER_UP%%" + ], + "write_behavior": "conditional", + "write_expected_when": [ + "verdict\\s*=\\s*real_fix" + ] + }, + "resolve-claims-review": { + "inputs": [ + { + "name": "worktree_path", + "type": "directory_path", + "required": true, + "recommended": false + }, + { + "name": "base_branch", + "type": "string", + "required": true, + "recommended": false + } + ], + "outputs": [ + { + "name": "needs_rerun", + "type": "string" + }, + { + "name": "verdict", + "type": "string" + }, + { + "name": "fixes_applied", + "type": "integer" + } + ], + "expected_output_patterns": [ + "needs_rerun\\s*=\\s*(true|false)" + ], + "pattern_examples": [ + "needs_rerun = true\nverdict = real_fix\nfixes_applied = 2\n%%ORDER_UP%%", + "needs_rerun = false\nverdict = already_green\nfixes_applied = 0\n%%ORDER_UP%%", + "needs_rerun = true\nverdict = flake_suspected\nfixes_applied = 0\n%%ORDER_UP%%", + "needs_rerun = false\nverdict = ci_only_failure\nfixes_applied = 0\n%%ORDER_UP%%" + ], + "write_behavior": "conditional", + "write_expected_when": [ + "verdict\\s*=\\s*real_fix" + ] + }, + "bundle-local-report": { + "inputs": [ + { + "name": "research_dir", + "type": "directory_path", + "required": true, + "recommended": false + }, + { + "name": "report_path", + "type": "file_path", + "required": true, + "recommended": false + }, + { + "name": "all_diagram_paths", + "type": "string", + "required": false, + "recommended": false + }, + { + "name": "visualization_plan_path", + "type": "file_path", + "required": false, + "recommended": false + } + ], + "outputs": [ + { + "name": "html_path", + "type": "file_path" + } + ], + "expected_output_patterns": [ + "html_path\\s*=\\s*\\S+" + ], + "pattern_examples": [ + "html_path = /tmp/research/report.html\n%%ORDER_UP%%" + ], + "write_behavior": "conditional", + "write_expected_when": [ + "html_path\\s*=\\s*\\S+" + ] + } + }, + "dataflow": [ + { + "step": "scope", + "available": [ + "audit_claims", + "base_branch", + "issue_url", + "output_mode", + "review_design", + "review_pr", + "source_dir", + "task" + ], + "required": [ + "research_question" + ], + "produced": [ + "scope_report" + ] + }, + { + "step": "plan_experiment", + "available": [ + "audit_claims", + "base_branch", + "issue_url", + "output_mode", + "review_design", + "review_pr", + "scope_report", + "source_dir", + "task" + ], + "required": [ + "scope_report_path" + ], + "produced": [ + "experiment_plan" + ] + }, + { + "step": "review_design", + "available": [ + "audit_claims", + "base_branch", + "experiment_plan", + "issue_url", + "output_mode", + "review_design", + "review_pr", + "scope_report", + "source_dir", + "task" + ], + "required": [], + "produced": [ + "verdict", + "experiment_type", + "evaluation_dashboard", + "revision_guidance", + "classification_timestamp" + ] + }, + { + "step": "plan_visualization", + "available": [ + "audit_claims", + "base_branch", + "classification_timestamp", + "evaluation_dashboard", + "experiment_plan", + "experiment_type", + "issue_url", + "output_mode", + "review_design", + "review_pr", + "revision_guidance", + "scope_report", + "source_dir", + "task", + "verdict" + ], + "required": [ + "experiment_plan_path" + ], + "produced": [ + "visualization_plan_path", + "report_plan_path", + "disambiguation_rule_applied", + "tier_c_lens", + "methodology_tradition", + "visualization_plan_trace_path" + ] + }, + { + "step": "revise_design", + "available": [ + "audit_claims", + "base_branch", + "classification_timestamp", + "disambiguation_rule_applied", + "evaluation_dashboard", + "experiment_plan", + "experiment_type", + "issue_url", + "methodology_tradition", + "output_mode", + "report_plan_path", + "review_design", + "review_pr", + "revision_guidance", + "scope_report", + "source_dir", + "task", + "tier_c_lens", + "verdict", + "visualization_plan_path", + "visualization_plan_trace_path" + ], + "required": [], + "produced": [] + }, + { + "step": "resolve_design_review", + "available": [ + "audit_claims", + "base_branch", + "classification_timestamp", + "disambiguation_rule_applied", + "evaluation_dashboard", + "experiment_plan", + "experiment_type", + "issue_url", + "methodology_tradition", + "output_mode", + "report_plan_path", + "review_design", + "review_pr", + "revision_guidance", + "scope_report", + "source_dir", + "task", + "tier_c_lens", + "verdict", + "visualization_plan_path", + "visualization_plan_trace_path" + ], + "required": [], + "produced": [ + "revision_guidance" + ] + }, + { + "step": "design_rejected", + "available": [ + "audit_claims", + "base_branch", + "classification_timestamp", + "disambiguation_rule_applied", + "evaluation_dashboard", + "experiment_plan", + "experiment_type", + "issue_url", + "methodology_tradition", + "output_mode", + "report_plan_path", + "review_design", + "review_pr", + "revision_guidance", + "scope_report", + "source_dir", + "task", + "tier_c_lens", + "verdict", + "visualization_plan_path", + "visualization_plan_trace_path" + ], + "required": [], + "produced": [] + }, + { + "step": "create_worktree", + "available": [ + "audit_claims", + "base_branch", + "classification_timestamp", + "disambiguation_rule_applied", + "evaluation_dashboard", + "experiment_plan", + "experiment_type", + "issue_url", + "methodology_tradition", + "output_mode", + "report_plan_path", + "review_design", + "review_pr", + "revision_guidance", + "scope_report", + "source_dir", + "task", + "tier_c_lens", + "verdict", + "visualization_plan_path", + "visualization_plan_trace_path" + ], + "required": [], + "produced": [ + "worktree_path", + "research_dir" + ] + }, + { + "step": "stage_data", + "available": [ + "audit_claims", + "base_branch", + "classification_timestamp", + "disambiguation_rule_applied", + "evaluation_dashboard", + "experiment_plan", + "experiment_type", + "issue_url", + "methodology_tradition", + "output_mode", + "report_plan_path", + "research_dir", + "review_design", + "review_pr", + "revision_guidance", + "scope_report", + "source_dir", + "task", + "tier_c_lens", + "verdict", + "visualization_plan_path", + "visualization_plan_trace_path", + "worktree_path" + ], + "required": [], + "produced": [ + "verdict", + "resource_report" + ] + }, + { + "step": "setup_environment", + "available": [ + "audit_claims", + "base_branch", + "classification_timestamp", + "disambiguation_rule_applied", + "evaluation_dashboard", + "experiment_plan", + "experiment_type", + "issue_url", + "methodology_tradition", + "output_mode", + "report_plan_path", + "research_dir", + "resource_report", + "review_design", + "review_pr", + "revision_guidance", + "scope_report", + "source_dir", + "task", + "tier_c_lens", + "verdict", + "visualization_plan_path", + "visualization_plan_trace_path", + "worktree_path" + ], + "required": [], + "produced": [ + "verdict" + ] + }, + { + "step": "decompose_phases", + "available": [ + "audit_claims", + "base_branch", + "classification_timestamp", + "disambiguation_rule_applied", + "evaluation_dashboard", + "experiment_plan", + "experiment_type", + "issue_url", + "methodology_tradition", + "output_mode", + "report_plan_path", + "research_dir", + "resource_report", + "review_design", + "review_pr", + "revision_guidance", + "scope_report", + "source_dir", + "task", + "tier_c_lens", + "verdict", + "visualization_plan_path", + "visualization_plan_trace_path", + "worktree_path" + ], + "required": [ + "source_doc" + ], + "produced": [ + "group_files" + ] + }, + { + "step": "plan_phase", + "available": [ + "audit_claims", + "base_branch", + "classification_timestamp", + "disambiguation_rule_applied", + "evaluation_dashboard", + "experiment_plan", + "experiment_type", + "group_files", + "issue_url", + "methodology_tradition", + "output_mode", + "report_plan_path", + "research_dir", + "resource_report", + "review_design", + "review_pr", + "revision_guidance", + "scope_report", + "source_dir", + "task", + "tier_c_lens", + "verdict", + "visualization_plan_path", + "visualization_plan_trace_path", + "worktree_path" + ], + "required": [ + "task" + ], + "produced": [ + "phase_plan_path" + ] + }, + { + "step": "implement_phase", + "available": [ + "audit_claims", + "base_branch", + "classification_timestamp", + "disambiguation_rule_applied", + "evaluation_dashboard", + "experiment_plan", + "experiment_type", + "group_files", + "issue_url", + "methodology_tradition", + "output_mode", + "phase_plan_path", + "report_plan_path", + "research_dir", + "resource_report", + "review_design", + "review_pr", + "revision_guidance", + "scope_report", + "source_dir", + "task", + "tier_c_lens", + "verdict", + "visualization_plan_path", + "visualization_plan_trace_path", + "worktree_path" + ], + "required": [ + "plan_path" + ], + "produced": [ + "worktree_path" + ] + }, + { + "step": "troubleshoot_implement_failure", + "available": [ + "audit_claims", + "base_branch", + "classification_timestamp", + "disambiguation_rule_applied", + "evaluation_dashboard", + "experiment_plan", + "experiment_type", + "group_files", + "issue_url", + "methodology_tradition", + "output_mode", + "phase_plan_path", + "report_plan_path", + "research_dir", + "resource_report", + "review_design", + "review_pr", + "revision_guidance", + "scope_report", + "source_dir", + "task", + "tier_c_lens", + "verdict", + "visualization_plan_path", + "visualization_plan_trace_path", + "worktree_path" + ], + "required": [], + "produced": [ + "is_fixable" + ], + "positional_args": 1 + }, + { + "step": "route_implement_failure", + "available": [ + "audit_claims", + "base_branch", + "classification_timestamp", + "disambiguation_rule_applied", + "evaluation_dashboard", + "experiment_plan", + "experiment_type", + "group_files", + "is_fixable", + "issue_url", + "methodology_tradition", + "output_mode", + "phase_plan_path", + "report_plan_path", + "research_dir", + "resource_report", + "review_design", + "review_pr", + "revision_guidance", + "scope_report", + "source_dir", + "task", + "tier_c_lens", + "verdict", + "visualization_plan_path", + "visualization_plan_trace_path", + "worktree_path" + ], + "required": [], + "produced": [] + }, + { + "step": "next_phase_or_experiment", + "available": [ + "audit_claims", + "base_branch", + "classification_timestamp", + "disambiguation_rule_applied", + "evaluation_dashboard", + "experiment_plan", + "experiment_type", + "group_files", + "is_fixable", + "issue_url", + "methodology_tradition", + "output_mode", + "phase_plan_path", + "report_plan_path", + "research_dir", + "resource_report", + "review_design", + "review_pr", + "revision_guidance", + "scope_report", + "source_dir", + "task", + "tier_c_lens", + "verdict", + "visualization_plan_path", + "visualization_plan_trace_path", + "worktree_path" + ], + "required": [], + "produced": [] + }, + { + "step": "run_experiment", + "available": [ + "audit_claims", + "base_branch", + "classification_timestamp", + "disambiguation_rule_applied", + "evaluation_dashboard", + "experiment_plan", + "experiment_type", + "group_files", + "is_fixable", + "issue_url", + "methodology_tradition", + "output_mode", + "phase_plan_path", + "report_plan_path", + "research_dir", + "resource_report", + "review_design", + "review_pr", + "revision_guidance", + "scope_report", + "source_dir", + "task", + "tier_c_lens", + "verdict", + "visualization_plan_path", + "visualization_plan_trace_path", + "worktree_path" + ], + "required": [], + "produced": [ + "experiment_results" + ] + }, + { + "step": "troubleshoot_run_failure", + "available": [ + "audit_claims", + "base_branch", + "classification_timestamp", + "disambiguation_rule_applied", + "evaluation_dashboard", + "experiment_plan", + "experiment_results", + "experiment_type", + "group_files", + "is_fixable", + "issue_url", + "methodology_tradition", + "output_mode", + "phase_plan_path", + "report_plan_path", + "research_dir", + "resource_report", + "review_design", + "review_pr", + "revision_guidance", + "scope_report", + "source_dir", + "task", + "tier_c_lens", + "verdict", + "visualization_plan_path", + "visualization_plan_trace_path", + "worktree_path" + ], + "required": [], + "produced": [ + "run_is_fixable" + ], + "positional_args": 1 + }, + { + "step": "route_run_failure", + "available": [ + "audit_claims", + "base_branch", + "classification_timestamp", + "disambiguation_rule_applied", + "evaluation_dashboard", + "experiment_plan", + "experiment_results", + "experiment_type", + "group_files", + "is_fixable", + "issue_url", + "methodology_tradition", + "output_mode", + "phase_plan_path", + "report_plan_path", + "research_dir", + "resource_report", + "review_design", + "review_pr", + "revision_guidance", + "run_is_fixable", + "scope_report", + "source_dir", + "task", + "tier_c_lens", + "verdict", + "visualization_plan_path", + "visualization_plan_trace_path", + "worktree_path" + ], + "required": [], + "produced": [] + }, + { + "step": "adjust_experiment", + "available": [ + "audit_claims", + "base_branch", + "classification_timestamp", + "disambiguation_rule_applied", + "evaluation_dashboard", + "experiment_plan", + "experiment_results", + "experiment_type", + "group_files", + "is_fixable", + "issue_url", + "methodology_tradition", + "output_mode", + "phase_plan_path", + "report_plan_path", + "research_dir", + "resource_report", + "review_design", + "review_pr", + "revision_guidance", + "run_is_fixable", + "scope_report", + "source_dir", + "task", + "tier_c_lens", + "verdict", + "visualization_plan_path", + "visualization_plan_trace_path", + "worktree_path" + ], + "required": [], + "produced": [ + "experiment_results" + ], + "positional_args": 1 + }, + { + "step": "ensure_results", + "available": [ + "audit_claims", + "base_branch", + "classification_timestamp", + "disambiguation_rule_applied", + "evaluation_dashboard", + "experiment_plan", + "experiment_results", + "experiment_type", + "group_files", + "is_fixable", + "issue_url", + "methodology_tradition", + "output_mode", + "phase_plan_path", + "report_plan_path", + "research_dir", + "resource_report", + "review_design", + "review_pr", + "revision_guidance", + "run_is_fixable", + "scope_report", + "source_dir", + "task", + "tier_c_lens", + "verdict", + "visualization_plan_path", + "visualization_plan_trace_path", + "worktree_path" + ], + "required": [], + "produced": [ + "experiment_results" + ] + }, + { + "step": "generate_report", + "available": [ + "audit_claims", + "base_branch", + "classification_timestamp", + "disambiguation_rule_applied", + "evaluation_dashboard", + "experiment_plan", + "experiment_results", + "experiment_type", + "group_files", + "is_fixable", + "issue_url", + "methodology_tradition", + "output_mode", + "phase_plan_path", + "report_plan_path", + "research_dir", + "resource_report", + "review_design", + "review_pr", + "revision_guidance", + "run_is_fixable", + "scope_report", + "source_dir", + "task", + "tier_c_lens", + "verdict", + "visualization_plan_path", + "visualization_plan_trace_path", + "worktree_path" + ], + "required": [], + "produced": [ + "report_path" + ], + "positional_args": 14 + }, + { + "step": "generate_report_inconclusive", + "available": [ + "audit_claims", + "base_branch", + "classification_timestamp", + "disambiguation_rule_applied", + "evaluation_dashboard", + "experiment_plan", + "experiment_results", + "experiment_type", + "group_files", + "is_fixable", + "issue_url", + "methodology_tradition", + "output_mode", + "phase_plan_path", + "report_path", + "report_plan_path", + "research_dir", + "resource_report", + "review_design", + "review_pr", + "revision_guidance", + "run_is_fixable", + "scope_report", + "source_dir", + "task", + "tier_c_lens", + "verdict", + "visualization_plan_path", + "visualization_plan_trace_path", + "worktree_path" + ], + "required": [], + "produced": [ + "report_path" + ], + "positional_args": 15 + }, + { + "step": "test", + "available": [ + "audit_claims", + "base_branch", + "classification_timestamp", + "disambiguation_rule_applied", + "evaluation_dashboard", + "experiment_plan", + "experiment_results", + "experiment_type", + "group_files", + "is_fixable", + "issue_url", + "methodology_tradition", + "output_mode", + "phase_plan_path", + "report_path", + "report_plan_path", + "research_dir", + "resource_report", + "review_design", + "review_pr", + "revision_guidance", + "run_is_fixable", + "scope_report", + "source_dir", + "task", + "tier_c_lens", + "verdict", + "visualization_plan_path", + "visualization_plan_trace_path", + "worktree_path" + ], + "required": [], + "produced": [] + }, + { + "step": "fix_tests", + "available": [ + "audit_claims", + "base_branch", + "classification_timestamp", + "disambiguation_rule_applied", + "evaluation_dashboard", + "experiment_plan", + "experiment_results", + "experiment_type", + "group_files", + "is_fixable", + "issue_url", + "methodology_tradition", + "output_mode", + "phase_plan_path", + "report_path", + "report_plan_path", + "research_dir", + "resource_report", + "review_design", + "review_pr", + "revision_guidance", + "run_is_fixable", + "scope_report", + "source_dir", + "task", + "tier_c_lens", + "verdict", + "visualization_plan_path", + "visualization_plan_trace_path", + "worktree_path" + ], + "required": [ + "plan_path" + ], + "produced": [ + "verdict", + "fixes_applied" + ] + }, + { + "step": "retest", + "available": [ + "audit_claims", + "base_branch", + "classification_timestamp", + "disambiguation_rule_applied", + "evaluation_dashboard", + "experiment_plan", + "experiment_results", + "experiment_type", + "fixes_applied", + "group_files", + "is_fixable", + "issue_url", + "methodology_tradition", + "output_mode", + "phase_plan_path", + "report_path", + "report_plan_path", + "research_dir", + "resource_report", + "review_design", + "review_pr", + "revision_guidance", + "run_is_fixable", + "scope_report", + "source_dir", + "task", + "tier_c_lens", + "verdict", + "visualization_plan_path", + "visualization_plan_trace_path", + "worktree_path" + ], + "required": [], + "produced": [] + }, + { + "step": "push_branch", + "available": [ + "audit_claims", + "base_branch", + "classification_timestamp", + "disambiguation_rule_applied", + "evaluation_dashboard", + "experiment_plan", + "experiment_results", + "experiment_type", + "fixes_applied", + "group_files", + "is_fixable", + "issue_url", + "methodology_tradition", + "output_mode", + "phase_plan_path", + "report_path", + "report_plan_path", + "research_dir", + "resource_report", + "review_design", + "review_pr", + "revision_guidance", + "run_is_fixable", + "scope_report", + "source_dir", + "task", + "tier_c_lens", + "verdict", + "visualization_plan_path", + "visualization_plan_trace_path", + "worktree_path" + ], + "required": [], + "produced": [] + }, + { + "step": "prepare_research_pr", + "available": [ + "audit_claims", + "base_branch", + "classification_timestamp", + "disambiguation_rule_applied", + "evaluation_dashboard", + "experiment_plan", + "experiment_results", + "experiment_type", + "fixes_applied", + "group_files", + "is_fixable", + "issue_url", + "methodology_tradition", + "output_mode", + "phase_plan_path", + "report_path", + "report_plan_path", + "research_dir", + "resource_report", + "review_design", + "review_pr", + "revision_guidance", + "run_is_fixable", + "scope_report", + "source_dir", + "task", + "tier_c_lens", + "verdict", + "visualization_plan_path", + "visualization_plan_trace_path", + "worktree_path" + ], + "required": [ + "experiment_plan_path" + ], + "produced": [ + "prep_path", + "selected_lenses", + "lens_context_paths" + ] + }, + { + "step": "run_experiment_lenses", + "available": [ + "audit_claims", + "base_branch", + "classification_timestamp", + "disambiguation_rule_applied", + "evaluation_dashboard", + "experiment_plan", + "experiment_results", + "experiment_type", + "fixes_applied", + "group_files", + "is_fixable", + "issue_url", + "lens_context_paths", + "methodology_tradition", + "output_mode", + "phase_plan_path", + "prep_path", + "report_path", + "report_plan_path", + "research_dir", + "resource_report", + "review_design", + "review_pr", + "revision_guidance", + "run_is_fixable", + "scope_report", + "selected_lenses", + "source_dir", + "task", + "tier_c_lens", + "verdict", + "visualization_plan_path", + "visualization_plan_trace_path", + "worktree_path" + ], + "required": [], + "produced": [] + }, + { + "step": "stage_bundle", + "available": [ + "audit_claims", + "base_branch", + "classification_timestamp", + "disambiguation_rule_applied", + "evaluation_dashboard", + "experiment_plan", + "experiment_results", + "experiment_type", + "fixes_applied", + "group_files", + "is_fixable", + "issue_url", + "lens_context_paths", + "methodology_tradition", + "output_mode", + "phase_plan_path", + "prep_path", + "report_path", + "report_plan_path", + "research_dir", + "resource_report", + "review_design", + "review_pr", + "revision_guidance", + "run_is_fixable", + "scope_report", + "selected_lenses", + "source_dir", + "task", + "tier_c_lens", + "verdict", + "visualization_plan_path", + "visualization_plan_trace_path", + "worktree_path" + ], + "required": [], + "produced": [] + }, + { + "step": "route_pr_or_local", + "available": [ + "audit_claims", + "base_branch", + "classification_timestamp", + "disambiguation_rule_applied", + "evaluation_dashboard", + "experiment_plan", + "experiment_results", + "experiment_type", + "fixes_applied", + "group_files", + "is_fixable", + "issue_url", + "lens_context_paths", + "methodology_tradition", + "output_mode", + "phase_plan_path", + "prep_path", + "report_path", + "report_plan_path", + "research_dir", + "resource_report", + "review_design", + "review_pr", + "revision_guidance", + "run_is_fixable", + "scope_report", + "selected_lenses", + "source_dir", + "task", + "tier_c_lens", + "verdict", + "visualization_plan_path", + "visualization_plan_trace_path", + "worktree_path" + ], + "required": [], + "produced": [] + }, + { + "step": "compose_research_pr", + "available": [ + "audit_claims", + "base_branch", + "classification_timestamp", + "disambiguation_rule_applied", + "evaluation_dashboard", + "experiment_plan", + "experiment_results", + "experiment_type", + "fixes_applied", + "group_files", + "is_fixable", + "issue_url", + "lens_context_paths", + "methodology_tradition", + "output_mode", + "phase_plan_path", + "prep_path", + "report_path", + "report_plan_path", + "research_dir", + "resource_report", + "review_design", + "review_pr", + "revision_guidance", + "run_is_fixable", + "scope_report", + "selected_lenses", + "source_dir", + "task", + "tier_c_lens", + "verdict", + "visualization_plan_path", + "visualization_plan_trace_path", + "worktree_path" + ], + "required": [], + "produced": [ + "pr_url" + ], + "positional_args": 1 + }, + { + "step": "guard_pr_url", + "available": [ + "audit_claims", + "base_branch", + "classification_timestamp", + "disambiguation_rule_applied", + "evaluation_dashboard", + "experiment_plan", + "experiment_results", + "experiment_type", + "fixes_applied", + "group_files", + "is_fixable", + "issue_url", + "lens_context_paths", + "methodology_tradition", + "output_mode", + "phase_plan_path", + "pr_url", + "prep_path", + "report_path", + "report_plan_path", + "research_dir", + "resource_report", + "review_design", + "review_pr", + "revision_guidance", + "run_is_fixable", + "scope_report", + "selected_lenses", + "source_dir", + "task", + "tier_c_lens", + "verdict", + "visualization_plan_path", + "visualization_plan_trace_path", + "worktree_path" + ], + "required": [], + "produced": [] + }, + { + "step": "review_research_pr", + "available": [ + "audit_claims", + "base_branch", + "classification_timestamp", + "disambiguation_rule_applied", + "evaluation_dashboard", + "experiment_plan", + "experiment_results", + "experiment_type", + "fixes_applied", + "group_files", + "is_fixable", + "issue_url", + "lens_context_paths", + "methodology_tradition", + "output_mode", + "phase_plan_path", + "pr_url", + "prep_path", + "report_path", + "report_plan_path", + "research_dir", + "resource_report", + "review_design", + "review_pr", + "revision_guidance", + "run_is_fixable", + "scope_report", + "selected_lenses", + "source_dir", + "task", + "tier_c_lens", + "verdict", + "visualization_plan_path", + "visualization_plan_trace_path", + "worktree_path" + ], + "required": [], + "produced": [ + "review_verdict" + ] + }, + { + "step": "audit_claims", + "available": [ + "audit_claims", + "base_branch", + "classification_timestamp", + "disambiguation_rule_applied", + "evaluation_dashboard", + "experiment_plan", + "experiment_results", + "experiment_type", + "fixes_applied", + "group_files", + "is_fixable", + "issue_url", + "lens_context_paths", + "methodology_tradition", + "output_mode", + "phase_plan_path", + "pr_url", + "prep_path", + "report_path", + "report_plan_path", + "research_dir", + "resource_report", + "review_design", + "review_pr", + "review_verdict", + "revision_guidance", + "run_is_fixable", + "scope_report", + "selected_lenses", + "source_dir", + "task", + "tier_c_lens", + "verdict", + "visualization_plan_path", + "visualization_plan_trace_path", + "worktree_path" + ], + "required": [], + "produced": [ + "audit_verdict" + ] + }, + { + "step": "route_review_resolve", + "available": [ + "audit_claims", + "audit_verdict", + "base_branch", + "classification_timestamp", + "disambiguation_rule_applied", + "evaluation_dashboard", + "experiment_plan", + "experiment_results", + "experiment_type", + "fixes_applied", + "group_files", + "is_fixable", + "issue_url", + "lens_context_paths", + "methodology_tradition", + "output_mode", + "phase_plan_path", + "pr_url", + "prep_path", + "report_path", + "report_plan_path", + "research_dir", + "resource_report", + "review_design", + "review_pr", + "review_verdict", + "revision_guidance", + "run_is_fixable", + "scope_report", + "selected_lenses", + "source_dir", + "task", + "tier_c_lens", + "verdict", + "visualization_plan_path", + "visualization_plan_trace_path", + "worktree_path" + ], + "required": [], + "produced": [] + }, + { + "step": "resolve_research_review", + "available": [ + "audit_claims", + "audit_verdict", + "base_branch", + "classification_timestamp", + "disambiguation_rule_applied", + "evaluation_dashboard", + "experiment_plan", + "experiment_results", + "experiment_type", + "fixes_applied", + "group_files", + "is_fixable", + "issue_url", + "lens_context_paths", + "methodology_tradition", + "output_mode", + "phase_plan_path", + "pr_url", + "prep_path", + "report_path", + "report_plan_path", + "research_dir", + "resource_report", + "review_design", + "review_pr", + "review_verdict", + "revision_guidance", + "run_is_fixable", + "scope_report", + "selected_lenses", + "source_dir", + "task", + "tier_c_lens", + "verdict", + "visualization_plan_path", + "visualization_plan_trace_path", + "worktree_path" + ], + "required": [], + "produced": [ + "review_needs_rerun" + ] + }, + { + "step": "route_claims_resolve", + "available": [ + "audit_claims", + "audit_verdict", + "base_branch", + "classification_timestamp", + "disambiguation_rule_applied", + "evaluation_dashboard", + "experiment_plan", + "experiment_results", + "experiment_type", + "fixes_applied", + "group_files", + "is_fixable", + "issue_url", + "lens_context_paths", + "methodology_tradition", + "output_mode", + "phase_plan_path", + "pr_url", + "prep_path", + "report_path", + "report_plan_path", + "research_dir", + "resource_report", + "review_design", + "review_needs_rerun", + "review_pr", + "review_verdict", + "revision_guidance", + "run_is_fixable", + "scope_report", + "selected_lenses", + "source_dir", + "task", + "tier_c_lens", + "verdict", + "visualization_plan_path", + "visualization_plan_trace_path", + "worktree_path" + ], + "required": [], + "produced": [] + }, + { + "step": "resolve_claims_review", + "available": [ + "audit_claims", + "audit_verdict", + "base_branch", + "classification_timestamp", + "disambiguation_rule_applied", + "evaluation_dashboard", + "experiment_plan", + "experiment_results", + "experiment_type", + "fixes_applied", + "group_files", + "is_fixable", + "issue_url", + "lens_context_paths", + "methodology_tradition", + "output_mode", + "phase_plan_path", + "pr_url", + "prep_path", + "report_path", + "report_plan_path", + "research_dir", + "resource_report", + "review_design", + "review_needs_rerun", + "review_pr", + "review_verdict", + "revision_guidance", + "run_is_fixable", + "scope_report", + "selected_lenses", + "source_dir", + "task", + "tier_c_lens", + "verdict", + "visualization_plan_path", + "visualization_plan_trace_path", + "worktree_path" + ], + "required": [], + "produced": [ + "claims_needs_rerun" + ] + }, + { + "step": "merge_escalations", + "available": [ + "audit_claims", + "audit_verdict", + "base_branch", + "claims_needs_rerun", + "classification_timestamp", + "disambiguation_rule_applied", + "evaluation_dashboard", + "experiment_plan", + "experiment_results", + "experiment_type", + "fixes_applied", + "group_files", + "is_fixable", + "issue_url", + "lens_context_paths", + "methodology_tradition", + "output_mode", + "phase_plan_path", + "pr_url", + "prep_path", + "report_path", + "report_plan_path", + "research_dir", + "resource_report", + "review_design", + "review_needs_rerun", + "review_pr", + "review_verdict", + "revision_guidance", + "run_is_fixable", + "scope_report", + "selected_lenses", + "source_dir", + "task", + "tier_c_lens", + "verdict", + "visualization_plan_path", + "visualization_plan_trace_path", + "worktree_path" + ], + "required": [], + "produced": [] + }, + { + "step": "re_run_experiment", + "available": [ + "audit_claims", + "audit_verdict", + "base_branch", + "claims_needs_rerun", + "classification_timestamp", + "disambiguation_rule_applied", + "evaluation_dashboard", + "experiment_plan", + "experiment_results", + "experiment_type", + "fixes_applied", + "group_files", + "is_fixable", + "issue_url", + "lens_context_paths", + "methodology_tradition", + "output_mode", + "phase_plan_path", + "pr_url", + "prep_path", + "report_path", + "report_plan_path", + "research_dir", + "resource_report", + "review_design", + "review_needs_rerun", + "review_pr", + "review_verdict", + "revision_guidance", + "run_is_fixable", + "scope_report", + "selected_lenses", + "source_dir", + "task", + "tier_c_lens", + "verdict", + "visualization_plan_path", + "visualization_plan_trace_path", + "worktree_path" + ], + "required": [], + "produced": [ + "experiment_results" + ], + "positional_args": 1 + }, + { + "step": "re_generate_report", + "available": [ + "audit_claims", + "audit_verdict", + "base_branch", + "claims_needs_rerun", + "classification_timestamp", + "disambiguation_rule_applied", + "evaluation_dashboard", + "experiment_plan", + "experiment_results", + "experiment_type", + "fixes_applied", + "group_files", + "is_fixable", + "issue_url", + "lens_context_paths", + "methodology_tradition", + "output_mode", + "phase_plan_path", + "pr_url", + "prep_path", + "report_path", + "report_plan_path", + "research_dir", + "resource_report", + "review_design", + "review_needs_rerun", + "review_pr", + "review_verdict", + "revision_guidance", + "run_is_fixable", + "scope_report", + "selected_lenses", + "source_dir", + "task", + "tier_c_lens", + "verdict", + "visualization_plan_path", + "visualization_plan_trace_path", + "worktree_path" + ], + "required": [], + "produced": [ + "report_path" + ], + "positional_args": 14 + }, + { + "step": "re_stage_bundle", + "available": [ + "audit_claims", + "audit_verdict", + "base_branch", + "claims_needs_rerun", + "classification_timestamp", + "disambiguation_rule_applied", + "evaluation_dashboard", + "experiment_plan", + "experiment_results", + "experiment_type", + "fixes_applied", + "group_files", + "is_fixable", + "issue_url", + "lens_context_paths", + "methodology_tradition", + "output_mode", + "phase_plan_path", + "pr_url", + "prep_path", + "report_path", + "report_plan_path", + "research_dir", + "resource_report", + "review_design", + "review_needs_rerun", + "review_pr", + "review_verdict", + "revision_guidance", + "run_is_fixable", + "scope_report", + "selected_lenses", + "source_dir", + "task", + "tier_c_lens", + "verdict", + "visualization_plan_path", + "visualization_plan_trace_path", + "worktree_path" + ], + "required": [], + "produced": [] + }, + { + "step": "re_test", + "available": [ + "audit_claims", + "audit_verdict", + "base_branch", + "claims_needs_rerun", + "classification_timestamp", + "disambiguation_rule_applied", + "evaluation_dashboard", + "experiment_plan", + "experiment_results", + "experiment_type", + "fixes_applied", + "group_files", + "is_fixable", + "issue_url", + "lens_context_paths", + "methodology_tradition", + "output_mode", + "phase_plan_path", + "pr_url", + "prep_path", + "report_path", + "report_plan_path", + "research_dir", + "resource_report", + "review_design", + "review_needs_rerun", + "review_pr", + "review_verdict", + "revision_guidance", + "run_is_fixable", + "scope_report", + "selected_lenses", + "source_dir", + "task", + "tier_c_lens", + "verdict", + "visualization_plan_path", + "visualization_plan_trace_path", + "worktree_path" + ], + "required": [], + "produced": [] + }, + { + "step": "re_push_research", + "available": [ + "audit_claims", + "audit_verdict", + "base_branch", + "claims_needs_rerun", + "classification_timestamp", + "disambiguation_rule_applied", + "evaluation_dashboard", + "experiment_plan", + "experiment_results", + "experiment_type", + "fixes_applied", + "group_files", + "is_fixable", + "issue_url", + "lens_context_paths", + "methodology_tradition", + "output_mode", + "phase_plan_path", + "pr_url", + "prep_path", + "report_path", + "report_plan_path", + "research_dir", + "resource_report", + "review_design", + "review_needs_rerun", + "review_pr", + "review_verdict", + "revision_guidance", + "run_is_fixable", + "scope_report", + "selected_lenses", + "source_dir", + "task", + "tier_c_lens", + "verdict", + "visualization_plan_path", + "visualization_plan_trace_path", + "worktree_path" + ], + "required": [], + "produced": [] + }, + { + "step": "finalize_bundle", + "available": [ + "audit_claims", + "audit_verdict", + "base_branch", + "claims_needs_rerun", + "classification_timestamp", + "disambiguation_rule_applied", + "evaluation_dashboard", + "experiment_plan", + "experiment_results", + "experiment_type", + "fixes_applied", + "group_files", + "is_fixable", + "issue_url", + "lens_context_paths", + "methodology_tradition", + "output_mode", + "phase_plan_path", + "pr_url", + "prep_path", + "report_path", + "report_plan_path", + "research_dir", + "resource_report", + "review_design", + "review_needs_rerun", + "review_pr", + "review_verdict", + "revision_guidance", + "run_is_fixable", + "scope_report", + "selected_lenses", + "source_dir", + "task", + "tier_c_lens", + "verdict", + "visualization_plan_path", + "visualization_plan_trace_path", + "worktree_path" + ], + "required": [], + "produced": [ + "report_path_after_finalize" + ] + }, + { + "step": "finalize_bundle_render", + "available": [ + "audit_claims", + "audit_verdict", + "base_branch", + "claims_needs_rerun", + "classification_timestamp", + "disambiguation_rule_applied", + "evaluation_dashboard", + "experiment_plan", + "experiment_results", + "experiment_type", + "fixes_applied", + "group_files", + "is_fixable", + "issue_url", + "lens_context_paths", + "methodology_tradition", + "output_mode", + "phase_plan_path", + "pr_url", + "prep_path", + "report_path", + "report_path_after_finalize", + "report_plan_path", + "research_dir", + "resource_report", + "review_design", + "review_needs_rerun", + "review_pr", + "review_verdict", + "revision_guidance", + "run_is_fixable", + "scope_report", + "selected_lenses", + "source_dir", + "task", + "tier_c_lens", + "verdict", + "visualization_plan_path", + "visualization_plan_trace_path", + "worktree_path" + ], + "required": [ + "report_path" + ], + "produced": [ + "html_path" + ] + }, + { + "step": "route_archive_or_export", + "available": [ + "audit_claims", + "audit_verdict", + "base_branch", + "claims_needs_rerun", + "classification_timestamp", + "disambiguation_rule_applied", + "evaluation_dashboard", + "experiment_plan", + "experiment_results", + "experiment_type", + "fixes_applied", + "group_files", + "html_path", + "is_fixable", + "issue_url", + "lens_context_paths", + "methodology_tradition", + "output_mode", + "phase_plan_path", + "pr_url", + "prep_path", + "report_path", + "report_path_after_finalize", + "report_plan_path", + "research_dir", + "resource_report", + "review_design", + "review_needs_rerun", + "review_pr", + "review_verdict", + "revision_guidance", + "run_is_fixable", + "scope_report", + "selected_lenses", + "source_dir", + "task", + "tier_c_lens", + "verdict", + "visualization_plan_path", + "visualization_plan_trace_path", + "worktree_path" + ], + "required": [], + "produced": [] + }, + { + "step": "export_local_bundle", + "available": [ + "audit_claims", + "audit_verdict", + "base_branch", + "claims_needs_rerun", + "classification_timestamp", + "disambiguation_rule_applied", + "evaluation_dashboard", + "experiment_plan", + "experiment_results", + "experiment_type", + "fixes_applied", + "group_files", + "html_path", + "is_fixable", + "issue_url", + "lens_context_paths", + "methodology_tradition", + "output_mode", + "phase_plan_path", + "pr_url", + "prep_path", + "report_path", + "report_path_after_finalize", + "report_plan_path", + "research_dir", + "resource_report", + "review_design", + "review_needs_rerun", + "review_pr", + "review_verdict", + "revision_guidance", + "run_is_fixable", + "scope_report", + "selected_lenses", + "source_dir", + "task", + "tier_c_lens", + "verdict", + "visualization_plan_path", + "visualization_plan_trace_path", + "worktree_path" + ], + "required": [], + "produced": [ + "local_bundle_path" + ] + }, + { + "step": "begin_archival", + "available": [ + "audit_claims", + "audit_verdict", + "base_branch", + "claims_needs_rerun", + "classification_timestamp", + "disambiguation_rule_applied", + "evaluation_dashboard", + "experiment_plan", + "experiment_results", + "experiment_type", + "fixes_applied", + "group_files", + "html_path", + "is_fixable", + "issue_url", + "lens_context_paths", + "local_bundle_path", + "methodology_tradition", + "output_mode", + "phase_plan_path", + "pr_url", + "prep_path", + "report_path", + "report_path_after_finalize", + "report_plan_path", + "research_dir", + "resource_report", + "review_design", + "review_needs_rerun", + "review_pr", + "review_verdict", + "revision_guidance", + "run_is_fixable", + "scope_report", + "selected_lenses", + "source_dir", + "task", + "tier_c_lens", + "verdict", + "visualization_plan_path", + "visualization_plan_trace_path", + "worktree_path" + ], + "required": [], + "produced": [] + }, + { + "step": "capture_experiment_branch", + "available": [ + "audit_claims", + "audit_verdict", + "base_branch", + "claims_needs_rerun", + "classification_timestamp", + "disambiguation_rule_applied", + "evaluation_dashboard", + "experiment_plan", + "experiment_results", + "experiment_type", + "fixes_applied", + "group_files", + "html_path", + "is_fixable", + "issue_url", + "lens_context_paths", + "local_bundle_path", + "methodology_tradition", + "output_mode", + "phase_plan_path", + "pr_url", + "prep_path", + "report_path", + "report_path_after_finalize", + "report_plan_path", + "research_dir", + "resource_report", + "review_design", + "review_needs_rerun", + "review_pr", + "review_verdict", + "revision_guidance", + "run_is_fixable", + "scope_report", + "selected_lenses", + "source_dir", + "task", + "tier_c_lens", + "verdict", + "visualization_plan_path", + "visualization_plan_trace_path", + "worktree_path" + ], + "required": [], + "produced": [ + "experiment_branch" + ] + }, + { + "step": "create_artifact_branch", + "available": [ + "audit_claims", + "audit_verdict", + "base_branch", + "claims_needs_rerun", + "classification_timestamp", + "disambiguation_rule_applied", + "evaluation_dashboard", + "experiment_branch", + "experiment_plan", + "experiment_results", + "experiment_type", + "fixes_applied", + "group_files", + "html_path", + "is_fixable", + "issue_url", + "lens_context_paths", + "local_bundle_path", + "methodology_tradition", + "output_mode", + "phase_plan_path", + "pr_url", + "prep_path", + "report_path", + "report_path_after_finalize", + "report_plan_path", + "research_dir", + "resource_report", + "review_design", + "review_needs_rerun", + "review_pr", + "review_verdict", + "revision_guidance", + "run_is_fixable", + "scope_report", + "selected_lenses", + "source_dir", + "task", + "tier_c_lens", + "verdict", + "visualization_plan_path", + "visualization_plan_trace_path", + "worktree_path" + ], + "required": [], + "produced": [ + "artifact_branch" + ] + }, + { + "step": "open_artifact_pr", + "available": [ + "artifact_branch", + "audit_claims", + "audit_verdict", + "base_branch", + "claims_needs_rerun", + "classification_timestamp", + "disambiguation_rule_applied", + "evaluation_dashboard", + "experiment_branch", + "experiment_plan", + "experiment_results", + "experiment_type", + "fixes_applied", + "group_files", + "html_path", + "is_fixable", + "issue_url", + "lens_context_paths", + "local_bundle_path", + "methodology_tradition", + "output_mode", + "phase_plan_path", + "pr_url", + "prep_path", + "report_path", + "report_path_after_finalize", + "report_plan_path", + "research_dir", + "resource_report", + "review_design", + "review_needs_rerun", + "review_pr", + "review_verdict", + "revision_guidance", + "run_is_fixable", + "scope_report", + "selected_lenses", + "source_dir", + "task", + "tier_c_lens", + "verdict", + "visualization_plan_path", + "visualization_plan_trace_path", + "worktree_path" + ], + "required": [], + "produced": [ + "artifact_pr_url" + ] + }, + { + "step": "tag_experiment_branch", + "available": [ + "artifact_branch", + "artifact_pr_url", + "audit_claims", + "audit_verdict", + "base_branch", + "claims_needs_rerun", + "classification_timestamp", + "disambiguation_rule_applied", + "evaluation_dashboard", + "experiment_branch", + "experiment_plan", + "experiment_results", + "experiment_type", + "fixes_applied", + "group_files", + "html_path", + "is_fixable", + "issue_url", + "lens_context_paths", + "local_bundle_path", + "methodology_tradition", + "output_mode", + "phase_plan_path", + "pr_url", + "prep_path", + "report_path", + "report_path_after_finalize", + "report_plan_path", + "research_dir", + "resource_report", + "review_design", + "review_needs_rerun", + "review_pr", + "review_verdict", + "revision_guidance", + "run_is_fixable", + "scope_report", + "selected_lenses", + "source_dir", + "task", + "tier_c_lens", + "verdict", + "visualization_plan_path", + "visualization_plan_trace_path", + "worktree_path" + ], + "required": [], + "produced": [ + "archive_tag" + ] + }, + { + "step": "close_experiment_pr", + "available": [ + "archive_tag", + "artifact_branch", + "artifact_pr_url", + "audit_claims", + "audit_verdict", + "base_branch", + "claims_needs_rerun", + "classification_timestamp", + "disambiguation_rule_applied", + "evaluation_dashboard", + "experiment_branch", + "experiment_plan", + "experiment_results", + "experiment_type", + "fixes_applied", + "group_files", + "html_path", + "is_fixable", + "issue_url", + "lens_context_paths", + "local_bundle_path", + "methodology_tradition", + "output_mode", + "phase_plan_path", + "pr_url", + "prep_path", + "report_path", + "report_path_after_finalize", + "report_plan_path", + "research_dir", + "resource_report", + "review_design", + "review_needs_rerun", + "review_pr", + "review_verdict", + "revision_guidance", + "run_is_fixable", + "scope_report", + "selected_lenses", + "source_dir", + "task", + "tier_c_lens", + "verdict", + "visualization_plan_path", + "visualization_plan_trace_path", + "worktree_path" + ], + "required": [], + "produced": [] + }, + { + "step": "patch_token_summary", + "available": [ + "archive_tag", + "artifact_branch", + "artifact_pr_url", + "audit_claims", + "audit_verdict", + "base_branch", + "claims_needs_rerun", + "classification_timestamp", + "disambiguation_rule_applied", + "evaluation_dashboard", + "experiment_branch", + "experiment_plan", + "experiment_results", + "experiment_type", + "fixes_applied", + "group_files", + "html_path", + "is_fixable", + "issue_url", + "lens_context_paths", + "local_bundle_path", + "methodology_tradition", + "output_mode", + "phase_plan_path", + "pr_url", + "prep_path", + "report_path", + "report_path_after_finalize", + "report_plan_path", + "research_dir", + "resource_report", + "review_design", + "review_needs_rerun", + "review_pr", + "review_verdict", + "revision_guidance", + "run_is_fixable", + "scope_report", + "selected_lenses", + "source_dir", + "task", + "tier_c_lens", + "verdict", + "visualization_plan_path", + "visualization_plan_trace_path", + "worktree_path" + ], + "required": [], + "produced": [] + }, + { + "step": "research_complete", + "available": [ + "archive_tag", + "artifact_branch", + "artifact_pr_url", + "audit_claims", + "audit_verdict", + "base_branch", + "claims_needs_rerun", + "classification_timestamp", + "disambiguation_rule_applied", + "evaluation_dashboard", + "experiment_branch", + "experiment_plan", + "experiment_results", + "experiment_type", + "fixes_applied", + "group_files", + "html_path", + "is_fixable", + "issue_url", + "lens_context_paths", + "local_bundle_path", + "methodology_tradition", + "output_mode", + "phase_plan_path", + "pr_url", + "prep_path", + "report_path", + "report_path_after_finalize", + "report_plan_path", + "research_dir", + "resource_report", + "review_design", + "review_needs_rerun", + "review_pr", + "review_verdict", + "revision_guidance", + "run_is_fixable", + "scope_report", + "selected_lenses", + "source_dir", + "task", + "tier_c_lens", + "verdict", + "visualization_plan_path", + "visualization_plan_trace_path", + "worktree_path" + ], + "required": [], + "produced": [] + }, + { + "step": "escalate_stop", + "available": [ + "archive_tag", + "artifact_branch", + "artifact_pr_url", + "audit_claims", + "audit_verdict", + "base_branch", + "claims_needs_rerun", + "classification_timestamp", + "disambiguation_rule_applied", + "evaluation_dashboard", + "experiment_branch", + "experiment_plan", + "experiment_results", + "experiment_type", + "fixes_applied", + "group_files", + "html_path", + "is_fixable", + "issue_url", + "lens_context_paths", + "local_bundle_path", + "methodology_tradition", + "output_mode", + "phase_plan_path", + "pr_url", + "prep_path", + "report_path", + "report_path_after_finalize", + "report_plan_path", + "research_dir", + "resource_report", + "review_design", + "review_needs_rerun", + "review_pr", + "review_verdict", + "revision_guidance", + "run_is_fixable", + "scope_report", + "selected_lenses", + "source_dir", + "task", + "tier_c_lens", + "verdict", + "visualization_plan_path", + "visualization_plan_trace_path", + "worktree_path" + ], + "required": [], + "produced": [] + } + ] +} diff --git a/src/autoskillit/recipes/contracts/research.yaml b/src/autoskillit/recipes/contracts/research.yaml new file mode 100644 index 0000000000..2cf5bf932b --- /dev/null +++ b/src/autoskillit/recipes/contracts/research.yaml @@ -0,0 +1,2844 @@ +generated_at: '2026-05-07T16:23:58.667262+00:00' +bundled_manifest_version: 0.1.0 +skill_hashes: {} +skills: + scope: + inputs: + - name: research_question + type: string + required: true + recommended: false + outputs: + - name: scope_report + type: file_path + expected_output_patterns: + - scope_report\s*=\s*/.+ + pattern_examples: + - 'scope_report = /tmp/scope_report.md + + %%ORDER_UP%%' + write_behavior: always + plan-experiment: + inputs: + - name: scope_report_path + type: file_path + required: true + recommended: false + - name: revision_guidance + type: file_path + required: false + recommended: false + outputs: + - name: experiment_plan + type: file_path + expected_output_patterns: + - experiment_plan\s*=\s*/.+ + pattern_examples: + - 'experiment_plan = /tmp/experiment_plan.md + + %%ORDER_UP%%' + write_behavior: always + review-design: + inputs: + - name: experiment_plan + type: file_path + required: true + recommended: false + outputs: + - name: verdict + type: string + - name: experiment_type + type: string + - name: evaluation_dashboard + type: file_path + - name: revision_guidance + type: file_path + - name: classification_timestamp + type: string + expected_output_patterns: + - verdict\s*=\s*(GO|REVISE|STOP) + - (?:experiment_type\s*=\s*\S+|verdict\s*=\s*STOP) + pattern_examples: + - 'verdict = GO + + experiment_type = benchmark + + evaluation_dashboard = /tmp/eval.md + + revision_guidance = /tmp/guidance.md + + %%ORDER_UP%%' + - 'verdict = REVISE + + experiment_type = benchmark + + revision_guidance = /tmp/guidance.md + + %%ORDER_UP%%' + - 'verdict = STOP + + %%ORDER_UP%%' + - 'verdict = GO + + experiment_type = causal_inference + + evaluation_dashboard = /tmp/eval.md + + %%ORDER_UP%%' + - 'verdict = GO + + experiment_type = configuration_study + + evaluation_dashboard = /tmp/eval.md + + %%ORDER_UP%%' + - 'verdict = GO + + experiment_type = robustness_audit + + evaluation_dashboard = /tmp/eval.md + + %%ORDER_UP%%' + - 'verdict = GO + + experiment_type = exploratory + + evaluation_dashboard = /tmp/eval.md + + %%ORDER_UP%%' + write_behavior: always + plan-visualization: + inputs: + - name: source_dir + type: directory_path + required: true + recommended: false + - name: experiment_plan_path + type: file_path + required: true + recommended: false + - name: scope_report_path + type: string + required: false + recommended: false + - name: experiment_type + type: string + required: false + recommended: false + outputs: + - name: visualization_plan_path + type: file_path + - name: report_plan_path + type: file_path + - name: disambiguation_rule_applied + type: string + - name: tier_c_lens + type: string + - name: methodology_tradition + type: string + - name: visualization_plan_trace_path + type: file_path + expected_output_patterns: + - visualization_plan_path\s*=\s*/.+ + - report_plan_path\s*=\s*/.+ + - methodology_tradition\s*=\s*\S+ + pattern_examples: + - 'visualization_plan_path = /tmp/plan-visualization/visualization-plan.md + + report_plan_path = /tmp/plan-visualization/report-plan.md + + methodology_tradition = controlled_intervention + + %%ORDER_UP%%' + - 'visualization_plan_path = /tmp/plan-visualization/visualization-plan.md + + report_plan_path = /tmp/plan-visualization/report-plan.md + + methodology_tradition = null + + %%ORDER_UP%%' + write_behavior: always + resolve-design-review: + inputs: + - name: evaluation_dashboard + type: file_path + required: true + recommended: false + - name: experiment_plan + type: file_path + required: true + recommended: false + outputs: + - name: resolution + type: string + - name: revision_guidance + type: file_path + expected_output_patterns: + - resolution\s*=\s*(revised|failed) + - (?:revision_guidance\s*=\s*/.+|resolution\s*=\s*failed) + pattern_examples: + - 'resolution = revised + + revision_guidance = /tmp/guidance.md + + %%ORDER_UP%%' + - 'resolution = failed + + %%ORDER_UP%%' + write_behavior: always + stage-data: + inputs: + - name: experiment_plan + type: file_path + required: true + recommended: false + outputs: + - name: verdict + type: string + - name: resource_report + type: file_path + expected_output_patterns: + - verdict\s*=\s*(PASS|WARN|FAIL) + - resource_report\s*=\s*/.+ + pattern_examples: + - 'verdict = PASS + + resource_report = /tmp/stage-data/resource_feasibility_2026-04-13_120000.md + + %%ORDER_UP%%' + write_behavior: always + setup-environment: + inputs: + - name: worktree_path + type: directory_path + required: true + recommended: false + - name: experiment_plan + type: file_path + required: true + recommended: false + outputs: + - name: env_mode + type: string + - name: env_report + type: file_path + - name: verdict + type: string + expected_output_patterns: + - env_mode\s*=\s*(none|docker|micromamba-host|unavailable) + - env_report\s*=\s*/.+ + - verdict\s*=\s*(PASS|WARN|FAIL) + pattern_examples: + - 'env_mode = none + + env_report = /tmp/setup-environment/env_setup_report.md + + verdict = PASS + + %%ORDER_UP%%' + - 'env_mode = docker + + env_report = /tmp/setup-environment/env_setup_report.md + + verdict = PASS + + %%ORDER_UP%%' + - 'env_mode = micromamba-host + + env_report = /tmp/setup-environment/env_setup_report.md + + verdict = WARN + + %%ORDER_UP%%' + - 'env_mode = unavailable + + env_report = /tmp/setup-environment/env_setup_report.md + + verdict = FAIL + + %%ORDER_UP%%' + write_behavior: always + make-groups: + inputs: + - name: source_doc + type: file_path + required: true + recommended: false + outputs: + - name: groups_path + type: file_path + - name: manifest_path + type: file_path + - name: group_files + type: file_path_list + expected_output_patterns: + - groups_path\s*=\s*/.+ + pattern_examples: + - 'groups_path = /tmp/groups.json + + %%ORDER_UP%%' + write_behavior: always + make-plan: + inputs: + - name: task + type: string + required: true + recommended: false + outputs: + - name: plan_path + type: file_path + - name: plan_parts + type: file_path_list + expected_output_patterns: + - plan_path\s*=\s*/.+ + pattern_examples: + - 'plan_path = /tmp/plan.md + + %%ORDER_UP%%' + write_behavior: always + implement-experiment: + inputs: + - name: plan_path + type: file_path + required: true + recommended: false + outputs: + - name: worktree_path + type: directory_path + - name: branch_name + type: string + expected_output_patterns: + - worktree_path\s*=\s*/.+ + pattern_examples: + - 'worktree_path = /tmp/worktrees/research-20260101-120000 + + %%ORDER_UP%%' + write_behavior: always + troubleshoot-experiment: + inputs: + - name: worktree_path + type: string + required: true + recommended: false + - name: step_name + type: string + required: true + recommended: false + outputs: + - name: diagnosis_path + type: file_path + - name: failure_type + type: string + - name: is_fixable + type: string + expected_output_patterns: + - is_fixable\s*=\s*(true|false) + pattern_examples: + - 'diagnosis_path = /tmp/diagnosis.md + + failure_type = stale_timeout + + is_fixable = true + + %%ORDER_UP%%' + write_behavior: always + run-experiment: + inputs: + - name: worktree_path + type: directory_path + required: true + recommended: false + outputs: + - name: results_path + type: file_path + expected_output_patterns: + - results_path\s*=\s*/.+ + pattern_examples: + - 'results_path = /tmp/run-experiment/results.md + + %%ORDER_UP%%' + write_behavior: always + generate-report: + inputs: + - name: worktree_path + type: directory_path + required: true + recommended: false + - name: results_path + type: file_path + required: true + recommended: false + - name: experiment_type + type: string + required: false + recommended: false + - name: methodology_tradition + type: string + required: false + recommended: false + outputs: + - name: report_path + type: file_path + expected_output_patterns: + - report_path\s*=\s*/.+ + pattern_examples: + - 'report_path = /tmp/research/report.md + + %%ORDER_UP%%' + write_behavior: always + resolve-failures: + inputs: + - name: worktree_path + type: directory_path + required: true + recommended: false + - name: plan_path + type: file_path + required: true + recommended: false + - name: base_branch + type: string + required: true + recommended: false + - name: diagnosis_path + type: file_path + required: false + recommended: false + outputs: + - name: verdict + type: string + - name: fixes_applied + type: integer + expected_output_patterns: [] + pattern_examples: [] + write_behavior: conditional + write_expected_when: + - verdict\s*=\s*real_fix + prepare-research-pr: + inputs: + - name: report_path + type: file_path + required: true + recommended: false + - name: experiment_plan_path + type: file_path + required: true + recommended: false + - name: worktree_path + type: directory_path + required: true + recommended: false + - name: base_branch + type: string + required: true + recommended: false + outputs: + - name: prep_path + type: file_path + - name: selected_lenses + type: string + - name: lens_context_paths + type: string + expected_output_patterns: + - prep_path\s*=\s*/.+ + - selected_lenses\s*=\s*\S+ + - lens_context_paths\s*=\s*/.+ + pattern_examples: + - 'prep_path = /abs/path/pr_prep_2026-04-07_101349.md + + selected_lenses = fair-comparison,estimand-clarity + + lens_context_paths = /abs/ctx1.md,/abs/ctx2.md + + %%ORDER_UP%%' + write_behavior: always + compose-research-pr: + inputs: + - name: prep_path + type: file_path + required: true + recommended: false + - name: all_diagram_paths + type: string + required: true + recommended: false + - name: worktree_path + type: directory_path + required: true + recommended: false + - name: base_branch + type: string + required: true + recommended: false + - name: closing_issue + type: string + required: false + recommended: false + outputs: + - name: pr_url + type: string + expected_output_patterns: + - pr_url\s*=\s*(https://github\.com/.*/pull/\d+)? + pattern_examples: + - 'pr_url = https://github.com/owner/repo/pull/42 + + %%ORDER_UP%%' + - "pr_url = \n%%ORDER_UP%%" + write_behavior: always + review-research-pr: + inputs: + - name: worktree_path + type: directory_path + required: true + recommended: false + - name: base_branch + type: string + required: true + recommended: false + - name: pr_url + type: string + required: true + recommended: false + outputs: + - name: verdict + type: string + expected_output_patterns: + - verdict\s*=\s*(approved|changes_requested|needs_human) + pattern_examples: + - 'verdict = approved + + %%ORDER_UP%%' + - 'verdict = changes_requested + + %%ORDER_UP%%' + - 'verdict = needs_human + + %%ORDER_UP%%' + audit-claims: + inputs: + - name: worktree_path + type: directory_path + required: true + recommended: false + - name: base_branch + type: string + required: true + recommended: false + - name: pr_url + type: string + required: true + recommended: false + outputs: + - name: verdict + type: string + expected_output_patterns: + - verdict\s*=\s*(approved|changes_requested|needs_human) + pattern_examples: + - 'verdict = approved + + %%ORDER_UP%%' + - 'verdict = changes_requested + + %%ORDER_UP%%' + - 'verdict = needs_human + + %%ORDER_UP%%' + resolve-research-review: + inputs: + - name: worktree_path + type: directory_path + required: true + recommended: false + - name: base_branch + type: string + required: true + recommended: false + outputs: + - name: needs_rerun + type: string + - name: verdict + type: string + - name: fixes_applied + type: integer + expected_output_patterns: + - needs_rerun\s*=\s*(true|false) + pattern_examples: + - 'needs_rerun = true + + verdict = real_fix + + fixes_applied = 2 + + %%ORDER_UP%%' + - 'needs_rerun = false + + verdict = already_green + + fixes_applied = 0 + + %%ORDER_UP%%' + - 'needs_rerun = true + + verdict = flake_suspected + + fixes_applied = 0 + + %%ORDER_UP%%' + - 'needs_rerun = false + + verdict = ci_only_failure + + fixes_applied = 0 + + %%ORDER_UP%%' + write_behavior: conditional + write_expected_when: + - verdict\s*=\s*real_fix + resolve-claims-review: + inputs: + - name: worktree_path + type: directory_path + required: true + recommended: false + - name: base_branch + type: string + required: true + recommended: false + outputs: + - name: needs_rerun + type: string + - name: verdict + type: string + - name: fixes_applied + type: integer + expected_output_patterns: + - needs_rerun\s*=\s*(true|false) + pattern_examples: + - 'needs_rerun = true + + verdict = real_fix + + fixes_applied = 2 + + %%ORDER_UP%%' + - 'needs_rerun = false + + verdict = already_green + + fixes_applied = 0 + + %%ORDER_UP%%' + - 'needs_rerun = true + + verdict = flake_suspected + + fixes_applied = 0 + + %%ORDER_UP%%' + - 'needs_rerun = false + + verdict = ci_only_failure + + fixes_applied = 0 + + %%ORDER_UP%%' + write_behavior: conditional + write_expected_when: + - verdict\s*=\s*real_fix + bundle-local-report: + inputs: + - name: research_dir + type: directory_path + required: true + recommended: false + - name: report_path + type: file_path + required: true + recommended: false + - name: all_diagram_paths + type: string + required: false + recommended: false + - name: visualization_plan_path + type: file_path + required: false + recommended: false + outputs: + - name: html_path + type: file_path + expected_output_patterns: + - html_path\s*=\s*\S+ + pattern_examples: + - 'html_path = /tmp/research/report.html + + %%ORDER_UP%%' + write_behavior: conditional + write_expected_when: + - html_path\s*=\s*\S+ +dataflow: +- step: scope + available: + - audit_claims + - base_branch + - issue_url + - output_mode + - review_design + - review_pr + - source_dir + - task + required: + - research_question + produced: + - scope_report +- step: plan_experiment + available: + - audit_claims + - base_branch + - issue_url + - output_mode + - review_design + - review_pr + - scope_report + - source_dir + - task + required: + - scope_report_path + produced: + - experiment_plan +- step: review_design + available: + - audit_claims + - base_branch + - experiment_plan + - issue_url + - output_mode + - review_design + - review_pr + - scope_report + - source_dir + - task + required: [] + produced: + - verdict + - experiment_type + - evaluation_dashboard + - revision_guidance + - classification_timestamp +- step: plan_visualization + available: + - audit_claims + - base_branch + - classification_timestamp + - evaluation_dashboard + - experiment_plan + - experiment_type + - issue_url + - output_mode + - review_design + - review_pr + - revision_guidance + - scope_report + - source_dir + - task + - verdict + required: + - experiment_plan_path + produced: + - visualization_plan_path + - report_plan_path + - disambiguation_rule_applied + - tier_c_lens + - methodology_tradition + - visualization_plan_trace_path +- step: revise_design + available: + - audit_claims + - base_branch + - classification_timestamp + - disambiguation_rule_applied + - evaluation_dashboard + - experiment_plan + - experiment_type + - issue_url + - methodology_tradition + - output_mode + - report_plan_path + - review_design + - review_pr + - revision_guidance + - scope_report + - source_dir + - task + - tier_c_lens + - verdict + - visualization_plan_path + - visualization_plan_trace_path + required: [] + produced: [] +- step: resolve_design_review + available: + - audit_claims + - base_branch + - classification_timestamp + - disambiguation_rule_applied + - evaluation_dashboard + - experiment_plan + - experiment_type + - issue_url + - methodology_tradition + - output_mode + - report_plan_path + - review_design + - review_pr + - revision_guidance + - scope_report + - source_dir + - task + - tier_c_lens + - verdict + - visualization_plan_path + - visualization_plan_trace_path + required: [] + produced: + - revision_guidance +- step: design_rejected + available: + - audit_claims + - base_branch + - classification_timestamp + - disambiguation_rule_applied + - evaluation_dashboard + - experiment_plan + - experiment_type + - issue_url + - methodology_tradition + - output_mode + - report_plan_path + - review_design + - review_pr + - revision_guidance + - scope_report + - source_dir + - task + - tier_c_lens + - verdict + - visualization_plan_path + - visualization_plan_trace_path + required: [] + produced: [] +- step: create_worktree + available: + - audit_claims + - base_branch + - classification_timestamp + - disambiguation_rule_applied + - evaluation_dashboard + - experiment_plan + - experiment_type + - issue_url + - methodology_tradition + - output_mode + - report_plan_path + - review_design + - review_pr + - revision_guidance + - scope_report + - source_dir + - task + - tier_c_lens + - verdict + - visualization_plan_path + - visualization_plan_trace_path + required: [] + produced: + - worktree_path + - research_dir +- step: stage_data + available: + - audit_claims + - base_branch + - classification_timestamp + - disambiguation_rule_applied + - evaluation_dashboard + - experiment_plan + - experiment_type + - issue_url + - methodology_tradition + - output_mode + - report_plan_path + - research_dir + - review_design + - review_pr + - revision_guidance + - scope_report + - source_dir + - task + - tier_c_lens + - verdict + - visualization_plan_path + - visualization_plan_trace_path + - worktree_path + required: [] + produced: + - verdict + - resource_report +- step: setup_environment + available: + - audit_claims + - base_branch + - classification_timestamp + - disambiguation_rule_applied + - evaluation_dashboard + - experiment_plan + - experiment_type + - issue_url + - methodology_tradition + - output_mode + - report_plan_path + - research_dir + - resource_report + - review_design + - review_pr + - revision_guidance + - scope_report + - source_dir + - task + - tier_c_lens + - verdict + - visualization_plan_path + - visualization_plan_trace_path + - worktree_path + required: [] + produced: + - verdict +- step: decompose_phases + available: + - audit_claims + - base_branch + - classification_timestamp + - disambiguation_rule_applied + - evaluation_dashboard + - experiment_plan + - experiment_type + - issue_url + - methodology_tradition + - output_mode + - report_plan_path + - research_dir + - resource_report + - review_design + - review_pr + - revision_guidance + - scope_report + - source_dir + - task + - tier_c_lens + - verdict + - visualization_plan_path + - visualization_plan_trace_path + - worktree_path + required: + - source_doc + produced: + - group_files +- step: plan_phase + available: + - audit_claims + - base_branch + - classification_timestamp + - disambiguation_rule_applied + - evaluation_dashboard + - experiment_plan + - experiment_type + - group_files + - issue_url + - methodology_tradition + - output_mode + - report_plan_path + - research_dir + - resource_report + - review_design + - review_pr + - revision_guidance + - scope_report + - source_dir + - task + - tier_c_lens + - verdict + - visualization_plan_path + - visualization_plan_trace_path + - worktree_path + required: + - task + produced: + - phase_plan_path +- step: implement_phase + available: + - audit_claims + - base_branch + - classification_timestamp + - disambiguation_rule_applied + - evaluation_dashboard + - experiment_plan + - experiment_type + - group_files + - issue_url + - methodology_tradition + - output_mode + - phase_plan_path + - report_plan_path + - research_dir + - resource_report + - review_design + - review_pr + - revision_guidance + - scope_report + - source_dir + - task + - tier_c_lens + - verdict + - visualization_plan_path + - visualization_plan_trace_path + - worktree_path + required: + - plan_path + produced: + - worktree_path +- step: troubleshoot_implement_failure + available: + - audit_claims + - base_branch + - classification_timestamp + - disambiguation_rule_applied + - evaluation_dashboard + - experiment_plan + - experiment_type + - group_files + - issue_url + - methodology_tradition + - output_mode + - phase_plan_path + - report_plan_path + - research_dir + - resource_report + - review_design + - review_pr + - revision_guidance + - scope_report + - source_dir + - task + - tier_c_lens + - verdict + - visualization_plan_path + - visualization_plan_trace_path + - worktree_path + required: [] + produced: + - is_fixable + positional_args: 1 +- step: route_implement_failure + available: + - audit_claims + - base_branch + - classification_timestamp + - disambiguation_rule_applied + - evaluation_dashboard + - experiment_plan + - experiment_type + - group_files + - is_fixable + - issue_url + - methodology_tradition + - output_mode + - phase_plan_path + - report_plan_path + - research_dir + - resource_report + - review_design + - review_pr + - revision_guidance + - scope_report + - source_dir + - task + - tier_c_lens + - verdict + - visualization_plan_path + - visualization_plan_trace_path + - worktree_path + required: [] + produced: [] +- step: next_phase_or_experiment + available: + - audit_claims + - base_branch + - classification_timestamp + - disambiguation_rule_applied + - evaluation_dashboard + - experiment_plan + - experiment_type + - group_files + - is_fixable + - issue_url + - methodology_tradition + - output_mode + - phase_plan_path + - report_plan_path + - research_dir + - resource_report + - review_design + - review_pr + - revision_guidance + - scope_report + - source_dir + - task + - tier_c_lens + - verdict + - visualization_plan_path + - visualization_plan_trace_path + - worktree_path + required: [] + produced: [] +- step: run_experiment + available: + - audit_claims + - base_branch + - classification_timestamp + - disambiguation_rule_applied + - evaluation_dashboard + - experiment_plan + - experiment_type + - group_files + - is_fixable + - issue_url + - methodology_tradition + - output_mode + - phase_plan_path + - report_plan_path + - research_dir + - resource_report + - review_design + - review_pr + - revision_guidance + - scope_report + - source_dir + - task + - tier_c_lens + - verdict + - visualization_plan_path + - visualization_plan_trace_path + - worktree_path + required: [] + produced: + - experiment_results +- step: troubleshoot_run_failure + available: + - audit_claims + - base_branch + - classification_timestamp + - disambiguation_rule_applied + - evaluation_dashboard + - experiment_plan + - experiment_results + - experiment_type + - group_files + - is_fixable + - issue_url + - methodology_tradition + - output_mode + - phase_plan_path + - report_plan_path + - research_dir + - resource_report + - review_design + - review_pr + - revision_guidance + - scope_report + - source_dir + - task + - tier_c_lens + - verdict + - visualization_plan_path + - visualization_plan_trace_path + - worktree_path + required: [] + produced: + - run_is_fixable + positional_args: 1 +- step: route_run_failure + available: + - audit_claims + - base_branch + - classification_timestamp + - disambiguation_rule_applied + - evaluation_dashboard + - experiment_plan + - experiment_results + - experiment_type + - group_files + - is_fixable + - issue_url + - methodology_tradition + - output_mode + - phase_plan_path + - report_plan_path + - research_dir + - resource_report + - review_design + - review_pr + - revision_guidance + - run_is_fixable + - scope_report + - source_dir + - task + - tier_c_lens + - verdict + - visualization_plan_path + - visualization_plan_trace_path + - worktree_path + required: [] + produced: [] +- step: adjust_experiment + available: + - audit_claims + - base_branch + - classification_timestamp + - disambiguation_rule_applied + - evaluation_dashboard + - experiment_plan + - experiment_results + - experiment_type + - group_files + - is_fixable + - issue_url + - methodology_tradition + - output_mode + - phase_plan_path + - report_plan_path + - research_dir + - resource_report + - review_design + - review_pr + - revision_guidance + - run_is_fixable + - scope_report + - source_dir + - task + - tier_c_lens + - verdict + - visualization_plan_path + - visualization_plan_trace_path + - worktree_path + required: [] + produced: + - experiment_results + positional_args: 1 +- step: ensure_results + available: + - audit_claims + - base_branch + - classification_timestamp + - disambiguation_rule_applied + - evaluation_dashboard + - experiment_plan + - experiment_results + - experiment_type + - group_files + - is_fixable + - issue_url + - methodology_tradition + - output_mode + - phase_plan_path + - report_plan_path + - research_dir + - resource_report + - review_design + - review_pr + - revision_guidance + - run_is_fixable + - scope_report + - source_dir + - task + - tier_c_lens + - verdict + - visualization_plan_path + - visualization_plan_trace_path + - worktree_path + required: [] + produced: + - experiment_results +- step: generate_report + available: + - audit_claims + - base_branch + - classification_timestamp + - disambiguation_rule_applied + - evaluation_dashboard + - experiment_plan + - experiment_results + - experiment_type + - group_files + - is_fixable + - issue_url + - methodology_tradition + - output_mode + - phase_plan_path + - report_plan_path + - research_dir + - resource_report + - review_design + - review_pr + - revision_guidance + - run_is_fixable + - scope_report + - source_dir + - task + - tier_c_lens + - verdict + - visualization_plan_path + - visualization_plan_trace_path + - worktree_path + required: [] + produced: + - report_path + positional_args: 14 +- step: generate_report_inconclusive + available: + - audit_claims + - base_branch + - classification_timestamp + - disambiguation_rule_applied + - evaluation_dashboard + - experiment_plan + - experiment_results + - experiment_type + - group_files + - is_fixable + - issue_url + - methodology_tradition + - output_mode + - phase_plan_path + - report_path + - report_plan_path + - research_dir + - resource_report + - review_design + - review_pr + - revision_guidance + - run_is_fixable + - scope_report + - source_dir + - task + - tier_c_lens + - verdict + - visualization_plan_path + - visualization_plan_trace_path + - worktree_path + required: [] + produced: + - report_path + positional_args: 15 +- step: test + available: + - audit_claims + - base_branch + - classification_timestamp + - disambiguation_rule_applied + - evaluation_dashboard + - experiment_plan + - experiment_results + - experiment_type + - group_files + - is_fixable + - issue_url + - methodology_tradition + - output_mode + - phase_plan_path + - report_path + - report_plan_path + - research_dir + - resource_report + - review_design + - review_pr + - revision_guidance + - run_is_fixable + - scope_report + - source_dir + - task + - tier_c_lens + - verdict + - visualization_plan_path + - visualization_plan_trace_path + - worktree_path + required: [] + produced: [] +- step: fix_tests + available: + - audit_claims + - base_branch + - classification_timestamp + - disambiguation_rule_applied + - evaluation_dashboard + - experiment_plan + - experiment_results + - experiment_type + - group_files + - is_fixable + - issue_url + - methodology_tradition + - output_mode + - phase_plan_path + - report_path + - report_plan_path + - research_dir + - resource_report + - review_design + - review_pr + - revision_guidance + - run_is_fixable + - scope_report + - source_dir + - task + - tier_c_lens + - verdict + - visualization_plan_path + - visualization_plan_trace_path + - worktree_path + required: + - plan_path + produced: + - verdict + - fixes_applied +- step: retest + available: + - audit_claims + - base_branch + - classification_timestamp + - disambiguation_rule_applied + - evaluation_dashboard + - experiment_plan + - experiment_results + - experiment_type + - fixes_applied + - group_files + - is_fixable + - issue_url + - methodology_tradition + - output_mode + - phase_plan_path + - report_path + - report_plan_path + - research_dir + - resource_report + - review_design + - review_pr + - revision_guidance + - run_is_fixable + - scope_report + - source_dir + - task + - tier_c_lens + - verdict + - visualization_plan_path + - visualization_plan_trace_path + - worktree_path + required: [] + produced: [] +- step: push_branch + available: + - audit_claims + - base_branch + - classification_timestamp + - disambiguation_rule_applied + - evaluation_dashboard + - experiment_plan + - experiment_results + - experiment_type + - fixes_applied + - group_files + - is_fixable + - issue_url + - methodology_tradition + - output_mode + - phase_plan_path + - report_path + - report_plan_path + - research_dir + - resource_report + - review_design + - review_pr + - revision_guidance + - run_is_fixable + - scope_report + - source_dir + - task + - tier_c_lens + - verdict + - visualization_plan_path + - visualization_plan_trace_path + - worktree_path + required: [] + produced: [] +- step: prepare_research_pr + available: + - audit_claims + - base_branch + - classification_timestamp + - disambiguation_rule_applied + - evaluation_dashboard + - experiment_plan + - experiment_results + - experiment_type + - fixes_applied + - group_files + - is_fixable + - issue_url + - methodology_tradition + - output_mode + - phase_plan_path + - report_path + - report_plan_path + - research_dir + - resource_report + - review_design + - review_pr + - revision_guidance + - run_is_fixable + - scope_report + - source_dir + - task + - tier_c_lens + - verdict + - visualization_plan_path + - visualization_plan_trace_path + - worktree_path + required: + - experiment_plan_path + produced: + - prep_path + - selected_lenses + - lens_context_paths +- step: run_experiment_lenses + available: + - audit_claims + - base_branch + - classification_timestamp + - disambiguation_rule_applied + - evaluation_dashboard + - experiment_plan + - experiment_results + - experiment_type + - fixes_applied + - group_files + - is_fixable + - issue_url + - lens_context_paths + - methodology_tradition + - output_mode + - phase_plan_path + - prep_path + - report_path + - report_plan_path + - research_dir + - resource_report + - review_design + - review_pr + - revision_guidance + - run_is_fixable + - scope_report + - selected_lenses + - source_dir + - task + - tier_c_lens + - verdict + - visualization_plan_path + - visualization_plan_trace_path + - worktree_path + required: [] + produced: [] +- step: stage_bundle + available: + - audit_claims + - base_branch + - classification_timestamp + - disambiguation_rule_applied + - evaluation_dashboard + - experiment_plan + - experiment_results + - experiment_type + - fixes_applied + - group_files + - is_fixable + - issue_url + - lens_context_paths + - methodology_tradition + - output_mode + - phase_plan_path + - prep_path + - report_path + - report_plan_path + - research_dir + - resource_report + - review_design + - review_pr + - revision_guidance + - run_is_fixable + - scope_report + - selected_lenses + - source_dir + - task + - tier_c_lens + - verdict + - visualization_plan_path + - visualization_plan_trace_path + - worktree_path + required: [] + produced: [] +- step: route_pr_or_local + available: + - audit_claims + - base_branch + - classification_timestamp + - disambiguation_rule_applied + - evaluation_dashboard + - experiment_plan + - experiment_results + - experiment_type + - fixes_applied + - group_files + - is_fixable + - issue_url + - lens_context_paths + - methodology_tradition + - output_mode + - phase_plan_path + - prep_path + - report_path + - report_plan_path + - research_dir + - resource_report + - review_design + - review_pr + - revision_guidance + - run_is_fixable + - scope_report + - selected_lenses + - source_dir + - task + - tier_c_lens + - verdict + - visualization_plan_path + - visualization_plan_trace_path + - worktree_path + required: [] + produced: [] +- step: compose_research_pr + available: + - audit_claims + - base_branch + - classification_timestamp + - disambiguation_rule_applied + - evaluation_dashboard + - experiment_plan + - experiment_results + - experiment_type + - fixes_applied + - group_files + - is_fixable + - issue_url + - lens_context_paths + - methodology_tradition + - output_mode + - phase_plan_path + - prep_path + - report_path + - report_plan_path + - research_dir + - resource_report + - review_design + - review_pr + - revision_guidance + - run_is_fixable + - scope_report + - selected_lenses + - source_dir + - task + - tier_c_lens + - verdict + - visualization_plan_path + - visualization_plan_trace_path + - worktree_path + required: [] + produced: + - pr_url + positional_args: 1 +- step: guard_pr_url + available: + - audit_claims + - base_branch + - classification_timestamp + - disambiguation_rule_applied + - evaluation_dashboard + - experiment_plan + - experiment_results + - experiment_type + - fixes_applied + - group_files + - is_fixable + - issue_url + - lens_context_paths + - methodology_tradition + - output_mode + - phase_plan_path + - pr_url + - prep_path + - report_path + - report_plan_path + - research_dir + - resource_report + - review_design + - review_pr + - revision_guidance + - run_is_fixable + - scope_report + - selected_lenses + - source_dir + - task + - tier_c_lens + - verdict + - visualization_plan_path + - visualization_plan_trace_path + - worktree_path + required: [] + produced: [] +- step: review_research_pr + available: + - audit_claims + - base_branch + - classification_timestamp + - disambiguation_rule_applied + - evaluation_dashboard + - experiment_plan + - experiment_results + - experiment_type + - fixes_applied + - group_files + - is_fixable + - issue_url + - lens_context_paths + - methodology_tradition + - output_mode + - phase_plan_path + - pr_url + - prep_path + - report_path + - report_plan_path + - research_dir + - resource_report + - review_design + - review_pr + - revision_guidance + - run_is_fixable + - scope_report + - selected_lenses + - source_dir + - task + - tier_c_lens + - verdict + - visualization_plan_path + - visualization_plan_trace_path + - worktree_path + required: [] + produced: + - review_verdict +- step: audit_claims + available: + - audit_claims + - base_branch + - classification_timestamp + - disambiguation_rule_applied + - evaluation_dashboard + - experiment_plan + - experiment_results + - experiment_type + - fixes_applied + - group_files + - is_fixable + - issue_url + - lens_context_paths + - methodology_tradition + - output_mode + - phase_plan_path + - pr_url + - prep_path + - report_path + - report_plan_path + - research_dir + - resource_report + - review_design + - review_pr + - review_verdict + - revision_guidance + - run_is_fixable + - scope_report + - selected_lenses + - source_dir + - task + - tier_c_lens + - verdict + - visualization_plan_path + - visualization_plan_trace_path + - worktree_path + required: [] + produced: + - audit_verdict +- step: route_review_resolve + available: + - audit_claims + - audit_verdict + - base_branch + - classification_timestamp + - disambiguation_rule_applied + - evaluation_dashboard + - experiment_plan + - experiment_results + - experiment_type + - fixes_applied + - group_files + - is_fixable + - issue_url + - lens_context_paths + - methodology_tradition + - output_mode + - phase_plan_path + - pr_url + - prep_path + - report_path + - report_plan_path + - research_dir + - resource_report + - review_design + - review_pr + - review_verdict + - revision_guidance + - run_is_fixable + - scope_report + - selected_lenses + - source_dir + - task + - tier_c_lens + - verdict + - visualization_plan_path + - visualization_plan_trace_path + - worktree_path + required: [] + produced: [] +- step: resolve_research_review + available: + - audit_claims + - audit_verdict + - base_branch + - classification_timestamp + - disambiguation_rule_applied + - evaluation_dashboard + - experiment_plan + - experiment_results + - experiment_type + - fixes_applied + - group_files + - is_fixable + - issue_url + - lens_context_paths + - methodology_tradition + - output_mode + - phase_plan_path + - pr_url + - prep_path + - report_path + - report_plan_path + - research_dir + - resource_report + - review_design + - review_pr + - review_verdict + - revision_guidance + - run_is_fixable + - scope_report + - selected_lenses + - source_dir + - task + - tier_c_lens + - verdict + - visualization_plan_path + - visualization_plan_trace_path + - worktree_path + required: [] + produced: + - review_needs_rerun +- step: route_claims_resolve + available: + - audit_claims + - audit_verdict + - base_branch + - classification_timestamp + - disambiguation_rule_applied + - evaluation_dashboard + - experiment_plan + - experiment_results + - experiment_type + - fixes_applied + - group_files + - is_fixable + - issue_url + - lens_context_paths + - methodology_tradition + - output_mode + - phase_plan_path + - pr_url + - prep_path + - report_path + - report_plan_path + - research_dir + - resource_report + - review_design + - review_needs_rerun + - review_pr + - review_verdict + - revision_guidance + - run_is_fixable + - scope_report + - selected_lenses + - source_dir + - task + - tier_c_lens + - verdict + - visualization_plan_path + - visualization_plan_trace_path + - worktree_path + required: [] + produced: [] +- step: resolve_claims_review + available: + - audit_claims + - audit_verdict + - base_branch + - classification_timestamp + - disambiguation_rule_applied + - evaluation_dashboard + - experiment_plan + - experiment_results + - experiment_type + - fixes_applied + - group_files + - is_fixable + - issue_url + - lens_context_paths + - methodology_tradition + - output_mode + - phase_plan_path + - pr_url + - prep_path + - report_path + - report_plan_path + - research_dir + - resource_report + - review_design + - review_needs_rerun + - review_pr + - review_verdict + - revision_guidance + - run_is_fixable + - scope_report + - selected_lenses + - source_dir + - task + - tier_c_lens + - verdict + - visualization_plan_path + - visualization_plan_trace_path + - worktree_path + required: [] + produced: + - claims_needs_rerun +- step: merge_escalations + available: + - audit_claims + - audit_verdict + - base_branch + - claims_needs_rerun + - classification_timestamp + - disambiguation_rule_applied + - evaluation_dashboard + - experiment_plan + - experiment_results + - experiment_type + - fixes_applied + - group_files + - is_fixable + - issue_url + - lens_context_paths + - methodology_tradition + - output_mode + - phase_plan_path + - pr_url + - prep_path + - report_path + - report_plan_path + - research_dir + - resource_report + - review_design + - review_needs_rerun + - review_pr + - review_verdict + - revision_guidance + - run_is_fixable + - scope_report + - selected_lenses + - source_dir + - task + - tier_c_lens + - verdict + - visualization_plan_path + - visualization_plan_trace_path + - worktree_path + required: [] + produced: [] +- step: re_run_experiment + available: + - audit_claims + - audit_verdict + - base_branch + - claims_needs_rerun + - classification_timestamp + - disambiguation_rule_applied + - evaluation_dashboard + - experiment_plan + - experiment_results + - experiment_type + - fixes_applied + - group_files + - is_fixable + - issue_url + - lens_context_paths + - methodology_tradition + - output_mode + - phase_plan_path + - pr_url + - prep_path + - report_path + - report_plan_path + - research_dir + - resource_report + - review_design + - review_needs_rerun + - review_pr + - review_verdict + - revision_guidance + - run_is_fixable + - scope_report + - selected_lenses + - source_dir + - task + - tier_c_lens + - verdict + - visualization_plan_path + - visualization_plan_trace_path + - worktree_path + required: [] + produced: + - experiment_results + positional_args: 1 +- step: re_generate_report + available: + - audit_claims + - audit_verdict + - base_branch + - claims_needs_rerun + - classification_timestamp + - disambiguation_rule_applied + - evaluation_dashboard + - experiment_plan + - experiment_results + - experiment_type + - fixes_applied + - group_files + - is_fixable + - issue_url + - lens_context_paths + - methodology_tradition + - output_mode + - phase_plan_path + - pr_url + - prep_path + - report_path + - report_plan_path + - research_dir + - resource_report + - review_design + - review_needs_rerun + - review_pr + - review_verdict + - revision_guidance + - run_is_fixable + - scope_report + - selected_lenses + - source_dir + - task + - tier_c_lens + - verdict + - visualization_plan_path + - visualization_plan_trace_path + - worktree_path + required: [] + produced: + - report_path + positional_args: 14 +- step: re_stage_bundle + available: + - audit_claims + - audit_verdict + - base_branch + - claims_needs_rerun + - classification_timestamp + - disambiguation_rule_applied + - evaluation_dashboard + - experiment_plan + - experiment_results + - experiment_type + - fixes_applied + - group_files + - is_fixable + - issue_url + - lens_context_paths + - methodology_tradition + - output_mode + - phase_plan_path + - pr_url + - prep_path + - report_path + - report_plan_path + - research_dir + - resource_report + - review_design + - review_needs_rerun + - review_pr + - review_verdict + - revision_guidance + - run_is_fixable + - scope_report + - selected_lenses + - source_dir + - task + - tier_c_lens + - verdict + - visualization_plan_path + - visualization_plan_trace_path + - worktree_path + required: [] + produced: [] +- step: re_test + available: + - audit_claims + - audit_verdict + - base_branch + - claims_needs_rerun + - classification_timestamp + - disambiguation_rule_applied + - evaluation_dashboard + - experiment_plan + - experiment_results + - experiment_type + - fixes_applied + - group_files + - is_fixable + - issue_url + - lens_context_paths + - methodology_tradition + - output_mode + - phase_plan_path + - pr_url + - prep_path + - report_path + - report_plan_path + - research_dir + - resource_report + - review_design + - review_needs_rerun + - review_pr + - review_verdict + - revision_guidance + - run_is_fixable + - scope_report + - selected_lenses + - source_dir + - task + - tier_c_lens + - verdict + - visualization_plan_path + - visualization_plan_trace_path + - worktree_path + required: [] + produced: [] +- step: re_push_research + available: + - audit_claims + - audit_verdict + - base_branch + - claims_needs_rerun + - classification_timestamp + - disambiguation_rule_applied + - evaluation_dashboard + - experiment_plan + - experiment_results + - experiment_type + - fixes_applied + - group_files + - is_fixable + - issue_url + - lens_context_paths + - methodology_tradition + - output_mode + - phase_plan_path + - pr_url + - prep_path + - report_path + - report_plan_path + - research_dir + - resource_report + - review_design + - review_needs_rerun + - review_pr + - review_verdict + - revision_guidance + - run_is_fixable + - scope_report + - selected_lenses + - source_dir + - task + - tier_c_lens + - verdict + - visualization_plan_path + - visualization_plan_trace_path + - worktree_path + required: [] + produced: [] +- step: finalize_bundle + available: + - audit_claims + - audit_verdict + - base_branch + - claims_needs_rerun + - classification_timestamp + - disambiguation_rule_applied + - evaluation_dashboard + - experiment_plan + - experiment_results + - experiment_type + - fixes_applied + - group_files + - is_fixable + - issue_url + - lens_context_paths + - methodology_tradition + - output_mode + - phase_plan_path + - pr_url + - prep_path + - report_path + - report_plan_path + - research_dir + - resource_report + - review_design + - review_needs_rerun + - review_pr + - review_verdict + - revision_guidance + - run_is_fixable + - scope_report + - selected_lenses + - source_dir + - task + - tier_c_lens + - verdict + - visualization_plan_path + - visualization_plan_trace_path + - worktree_path + required: [] + produced: + - report_path_after_finalize +- step: finalize_bundle_render + available: + - audit_claims + - audit_verdict + - base_branch + - claims_needs_rerun + - classification_timestamp + - disambiguation_rule_applied + - evaluation_dashboard + - experiment_plan + - experiment_results + - experiment_type + - fixes_applied + - group_files + - is_fixable + - issue_url + - lens_context_paths + - methodology_tradition + - output_mode + - phase_plan_path + - pr_url + - prep_path + - report_path + - report_path_after_finalize + - report_plan_path + - research_dir + - resource_report + - review_design + - review_needs_rerun + - review_pr + - review_verdict + - revision_guidance + - run_is_fixable + - scope_report + - selected_lenses + - source_dir + - task + - tier_c_lens + - verdict + - visualization_plan_path + - visualization_plan_trace_path + - worktree_path + required: + - report_path + produced: + - html_path +- step: route_archive_or_export + available: + - audit_claims + - audit_verdict + - base_branch + - claims_needs_rerun + - classification_timestamp + - disambiguation_rule_applied + - evaluation_dashboard + - experiment_plan + - experiment_results + - experiment_type + - fixes_applied + - group_files + - html_path + - is_fixable + - issue_url + - lens_context_paths + - methodology_tradition + - output_mode + - phase_plan_path + - pr_url + - prep_path + - report_path + - report_path_after_finalize + - report_plan_path + - research_dir + - resource_report + - review_design + - review_needs_rerun + - review_pr + - review_verdict + - revision_guidance + - run_is_fixable + - scope_report + - selected_lenses + - source_dir + - task + - tier_c_lens + - verdict + - visualization_plan_path + - visualization_plan_trace_path + - worktree_path + required: [] + produced: [] +- step: export_local_bundle + available: + - audit_claims + - audit_verdict + - base_branch + - claims_needs_rerun + - classification_timestamp + - disambiguation_rule_applied + - evaluation_dashboard + - experiment_plan + - experiment_results + - experiment_type + - fixes_applied + - group_files + - html_path + - is_fixable + - issue_url + - lens_context_paths + - methodology_tradition + - output_mode + - phase_plan_path + - pr_url + - prep_path + - report_path + - report_path_after_finalize + - report_plan_path + - research_dir + - resource_report + - review_design + - review_needs_rerun + - review_pr + - review_verdict + - revision_guidance + - run_is_fixable + - scope_report + - selected_lenses + - source_dir + - task + - tier_c_lens + - verdict + - visualization_plan_path + - visualization_plan_trace_path + - worktree_path + required: [] + produced: + - local_bundle_path +- step: begin_archival + available: + - audit_claims + - audit_verdict + - base_branch + - claims_needs_rerun + - classification_timestamp + - disambiguation_rule_applied + - evaluation_dashboard + - experiment_plan + - experiment_results + - experiment_type + - fixes_applied + - group_files + - html_path + - is_fixable + - issue_url + - lens_context_paths + - local_bundle_path + - methodology_tradition + - output_mode + - phase_plan_path + - pr_url + - prep_path + - report_path + - report_path_after_finalize + - report_plan_path + - research_dir + - resource_report + - review_design + - review_needs_rerun + - review_pr + - review_verdict + - revision_guidance + - run_is_fixable + - scope_report + - selected_lenses + - source_dir + - task + - tier_c_lens + - verdict + - visualization_plan_path + - visualization_plan_trace_path + - worktree_path + required: [] + produced: [] +- step: capture_experiment_branch + available: + - audit_claims + - audit_verdict + - base_branch + - claims_needs_rerun + - classification_timestamp + - disambiguation_rule_applied + - evaluation_dashboard + - experiment_plan + - experiment_results + - experiment_type + - fixes_applied + - group_files + - html_path + - is_fixable + - issue_url + - lens_context_paths + - local_bundle_path + - methodology_tradition + - output_mode + - phase_plan_path + - pr_url + - prep_path + - report_path + - report_path_after_finalize + - report_plan_path + - research_dir + - resource_report + - review_design + - review_needs_rerun + - review_pr + - review_verdict + - revision_guidance + - run_is_fixable + - scope_report + - selected_lenses + - source_dir + - task + - tier_c_lens + - verdict + - visualization_plan_path + - visualization_plan_trace_path + - worktree_path + required: [] + produced: + - experiment_branch +- step: create_artifact_branch + available: + - audit_claims + - audit_verdict + - base_branch + - claims_needs_rerun + - classification_timestamp + - disambiguation_rule_applied + - evaluation_dashboard + - experiment_branch + - experiment_plan + - experiment_results + - experiment_type + - fixes_applied + - group_files + - html_path + - is_fixable + - issue_url + - lens_context_paths + - local_bundle_path + - methodology_tradition + - output_mode + - phase_plan_path + - pr_url + - prep_path + - report_path + - report_path_after_finalize + - report_plan_path + - research_dir + - resource_report + - review_design + - review_needs_rerun + - review_pr + - review_verdict + - revision_guidance + - run_is_fixable + - scope_report + - selected_lenses + - source_dir + - task + - tier_c_lens + - verdict + - visualization_plan_path + - visualization_plan_trace_path + - worktree_path + required: [] + produced: + - artifact_branch +- step: open_artifact_pr + available: + - artifact_branch + - audit_claims + - audit_verdict + - base_branch + - claims_needs_rerun + - classification_timestamp + - disambiguation_rule_applied + - evaluation_dashboard + - experiment_branch + - experiment_plan + - experiment_results + - experiment_type + - fixes_applied + - group_files + - html_path + - is_fixable + - issue_url + - lens_context_paths + - local_bundle_path + - methodology_tradition + - output_mode + - phase_plan_path + - pr_url + - prep_path + - report_path + - report_path_after_finalize + - report_plan_path + - research_dir + - resource_report + - review_design + - review_needs_rerun + - review_pr + - review_verdict + - revision_guidance + - run_is_fixable + - scope_report + - selected_lenses + - source_dir + - task + - tier_c_lens + - verdict + - visualization_plan_path + - visualization_plan_trace_path + - worktree_path + required: [] + produced: + - artifact_pr_url +- step: tag_experiment_branch + available: + - artifact_branch + - artifact_pr_url + - audit_claims + - audit_verdict + - base_branch + - claims_needs_rerun + - classification_timestamp + - disambiguation_rule_applied + - evaluation_dashboard + - experiment_branch + - experiment_plan + - experiment_results + - experiment_type + - fixes_applied + - group_files + - html_path + - is_fixable + - issue_url + - lens_context_paths + - local_bundle_path + - methodology_tradition + - output_mode + - phase_plan_path + - pr_url + - prep_path + - report_path + - report_path_after_finalize + - report_plan_path + - research_dir + - resource_report + - review_design + - review_needs_rerun + - review_pr + - review_verdict + - revision_guidance + - run_is_fixable + - scope_report + - selected_lenses + - source_dir + - task + - tier_c_lens + - verdict + - visualization_plan_path + - visualization_plan_trace_path + - worktree_path + required: [] + produced: + - archive_tag +- step: close_experiment_pr + available: + - archive_tag + - artifact_branch + - artifact_pr_url + - audit_claims + - audit_verdict + - base_branch + - claims_needs_rerun + - classification_timestamp + - disambiguation_rule_applied + - evaluation_dashboard + - experiment_branch + - experiment_plan + - experiment_results + - experiment_type + - fixes_applied + - group_files + - html_path + - is_fixable + - issue_url + - lens_context_paths + - local_bundle_path + - methodology_tradition + - output_mode + - phase_plan_path + - pr_url + - prep_path + - report_path + - report_path_after_finalize + - report_plan_path + - research_dir + - resource_report + - review_design + - review_needs_rerun + - review_pr + - review_verdict + - revision_guidance + - run_is_fixable + - scope_report + - selected_lenses + - source_dir + - task + - tier_c_lens + - verdict + - visualization_plan_path + - visualization_plan_trace_path + - worktree_path + required: [] + produced: [] +- step: patch_token_summary + available: + - archive_tag + - artifact_branch + - artifact_pr_url + - audit_claims + - audit_verdict + - base_branch + - claims_needs_rerun + - classification_timestamp + - disambiguation_rule_applied + - evaluation_dashboard + - experiment_branch + - experiment_plan + - experiment_results + - experiment_type + - fixes_applied + - group_files + - html_path + - is_fixable + - issue_url + - lens_context_paths + - local_bundle_path + - methodology_tradition + - output_mode + - phase_plan_path + - pr_url + - prep_path + - report_path + - report_path_after_finalize + - report_plan_path + - research_dir + - resource_report + - review_design + - review_needs_rerun + - review_pr + - review_verdict + - revision_guidance + - run_is_fixable + - scope_report + - selected_lenses + - source_dir + - task + - tier_c_lens + - verdict + - visualization_plan_path + - visualization_plan_trace_path + - worktree_path + required: [] + produced: [] +- step: research_complete + available: + - archive_tag + - artifact_branch + - artifact_pr_url + - audit_claims + - audit_verdict + - base_branch + - claims_needs_rerun + - classification_timestamp + - disambiguation_rule_applied + - evaluation_dashboard + - experiment_branch + - experiment_plan + - experiment_results + - experiment_type + - fixes_applied + - group_files + - html_path + - is_fixable + - issue_url + - lens_context_paths + - local_bundle_path + - methodology_tradition + - output_mode + - phase_plan_path + - pr_url + - prep_path + - report_path + - report_path_after_finalize + - report_plan_path + - research_dir + - resource_report + - review_design + - review_needs_rerun + - review_pr + - review_verdict + - revision_guidance + - run_is_fixable + - scope_report + - selected_lenses + - source_dir + - task + - tier_c_lens + - verdict + - visualization_plan_path + - visualization_plan_trace_path + - worktree_path + required: [] + produced: [] +- step: escalate_stop + available: + - archive_tag + - artifact_branch + - artifact_pr_url + - audit_claims + - audit_verdict + - base_branch + - claims_needs_rerun + - classification_timestamp + - disambiguation_rule_applied + - evaluation_dashboard + - experiment_branch + - experiment_plan + - experiment_results + - experiment_type + - fixes_applied + - group_files + - html_path + - is_fixable + - issue_url + - lens_context_paths + - local_bundle_path + - methodology_tradition + - output_mode + - phase_plan_path + - pr_url + - prep_path + - report_path + - report_path_after_finalize + - report_plan_path + - research_dir + - resource_report + - review_design + - review_needs_rerun + - review_pr + - review_verdict + - revision_guidance + - run_is_fixable + - scope_report + - selected_lenses + - source_dir + - task + - tier_c_lens + - verdict + - visualization_plan_path + - visualization_plan_trace_path + - worktree_path + required: [] + produced: [] diff --git a/src/autoskillit/recipes/diagrams/implementation-groups.md b/src/autoskillit/recipes/diagrams/implementation-groups.md index 3540773fc8..7d8b093869 100644 --- a/src/autoskillit/recipes/diagrams/implementation-groups.md +++ b/src/autoskillit/recipes/diagrams/implementation-groups.md @@ -1,16 +1,36 @@ + + ## implementation-groups +Group-based implementation with per-group plan/implement/test cycles and PR gates. -``` - make-groups - | - +----+ FOR EACH GROUP: - | | - | make-plan --- [review-approach] (optional) --- dry-walkthrough --- implement --- test <-> [x fail -> fix] - | - +----+ - | - +-- [audit] (optional) - | x fail [-> make-plan] - | - +-- [open-pr] (optional) -``` +### Graph +clone → get_issue_title → claim_issue → compute_branch +| ++-- create_branch → push_merge_target +| +group → plan → review (optional) +| +┌────┤ FOR EACH PLAN PART: +│ verify → implement ↔ [retry_worktree on context limit] +│ | +│ test → commit_guard → merge → push +│ | +│ fix (on failure) → next_or_done +└────┘ +| ++-- audit_impl → remediate (optional) +| ++-- [open-pr] (optional): +| prepare_pr → compose_pr → review_pr → resolve_review +| ci_watch → check_repo_merge_state +| → [queue | direct | immediate] merge path +| → diagnose_ci → resolve_ci (on CI failure) +| +release_issue_success / release_issue_failure +| ++-- patch_token_summary (optional) +| +register_clone_success / register_clone_failure +───────────────────────────────────── +done "Complete." +escalate_stop "Failed." diff --git a/src/autoskillit/recipes/diagrams/implementation.md b/src/autoskillit/recipes/diagrams/implementation.md index 8dec130de0..798aabbdd6 100644 --- a/src/autoskillit/recipes/diagrams/implementation.md +++ b/src/autoskillit/recipes/diagrams/implementation.md @@ -1,18 +1,35 @@ -## implementation + + +# implementation ``` - make-plan + clone → get_issue_title → claim_issue → compute_branch | - +-- [review-approach] (optional) + +-- create_branch (optional) + | + plan + | + +-- review (optional) | +----+ FOR EACH PLAN PART: | | - | dry-walkthrough --- implement --- test <-> [x fail -> fix] + | verify → implement → test ↔ [fix on failure] + | | + | merge → push → next_or_done | +----+ | - +-- [audit] (optional) - | x fail [-> make-plan] + +-- audit_impl → remediate (optional) | - +-- [open-pr] (optional) -``` + +-- [open-pr] (optional): + | prepare_pr → compose_pr → review_pr → resolve_review + | ci_watch → check_repo_merge_state + | → [queue | direct | immediate] merge path + | → diagnose_ci → resolve_ci (on CI failure) + | + release_issue_success / release_issue_failure + | + +-- patch_token_summary (optional) + | + register_clone_success / register_clone_failure +``` \ No newline at end of file diff --git a/src/autoskillit/recipes/diagrams/merge-prs.md b/src/autoskillit/recipes/diagrams/merge-prs.md index 7bb93966e6..47949d2911 100644 --- a/src/autoskillit/recipes/diagrams/merge-prs.md +++ b/src/autoskillit/recipes/diagrams/merge-prs.md @@ -1,29 +1,34 @@ + + ## merge-prs +Merge multiple PRs into an integration branch with conflict resolution and CI gates. -``` - analyze-prs - | - +--- queue mode: - | - | +----+ FOR EACH PR: - | | | - | | x ejected -> resolve-merge-conflicts - | | - | +----+ - | - +--- classic mode: - | - | +----+ FOR EACH PR: - | | | - | | merge-pr - | | x needs_plan -> make-plan --- dry-walkthrough --- implement --- test <-> [x fail -> fix] - | | - | +----+ - | - +-- [audit] (optional) - | x fail [-> make-plan] - | - open-pr - | - +-- [resolve-merge-conflicts] (on conflict) -``` +### Graph +clone → setup_remote → check_repo_ci_event +| +check_integration_exists → confirm_create_integration (optional) +| +fetch_merge_queue_data → analyze_prs → route_by_queue_mode +| ++-- [queue mode]: +| enqueue → wait → advance → next PR +| → resolve ejected conflicts on failure +| ++-- [integration mode]: +| create_batch_branch → publish +| | +| ┌────┤ FOR EACH PR: +| │ merge_pr → plan → verify → implement → test +| │ → merge → push → next_or_done +| └────┘ +| | +| audit_impl → remediate (optional) +| | +| open_integration_pr → ci_watch → review +| ++-- diagnose_ci → resolve_ci (on CI failure) +| +patch_token_summary (optional) +───────────────────────────────────── +done "Complete." +escalate_stop "Failed." diff --git a/src/autoskillit/recipes/diagrams/remediation.md b/src/autoskillit/recipes/diagrams/remediation.md index 4442fdb355..ccf9222fb3 100644 --- a/src/autoskillit/recipes/diagrams/remediation.md +++ b/src/autoskillit/recipes/diagrams/remediation.md @@ -1,18 +1,34 @@ + + ## remediation +Investigate, rectify, implement, and merge a bug fix with CI and PR gates. -``` - investigate - | - rectify - | - +-- [review-approach] (optional) - | - dry-walkthrough - | - implement --- test <-> [x fail -> fix] - | - +-- [audit] (optional) - | x fail [-> make-plan] - | - +-- [open-pr] (optional) -``` +### Graph +clone → get_issue_title → claim_issue → compute_branch +| ++-- create_branch → push_merge_target +| +investigate → rectify → review (optional) +| +dry_walkthrough → implement ↔ [retry_worktree on context limit] +| +test → assess +| ++-- audit_impl → remediate (optional) +| +make_plan → commit_guard → merge → push +| ++-- [open-pr] (optional): +| prepare_pr → compose_pr → review_pr → resolve_review +| ci_watch → check_repo_merge_state +| → [queue | direct | immediate] merge path +| → diagnose_ci → resolve_ci (on CI failure) +| +release_issue_success / release_issue_failure +| ++-- patch_token_summary (optional) +| +register_clone_success / register_clone_failure +───────────────────────────────────── +done "Complete." +escalate_stop "Failed." diff --git a/src/autoskillit/recipes/diagrams/research.md b/src/autoskillit/recipes/diagrams/research.md new file mode 100644 index 0000000000..d4387e70ff --- /dev/null +++ b/src/autoskillit/recipes/diagrams/research.md @@ -0,0 +1,96 @@ + + + +## research + +scope +| +plan_experiment +| +review_design +| +-- [plan_visualization] (optional) +| +-- [resolve_design_review] (on STOP verdict) +| x fail [-> escalate_stop] +| +stage_data +| +setup_environment +| +decompose_phases + ++----+ FOR EACH PHASE: +| | +| plan_phase +| | +| implement_phase <-> [x fail -> troubleshoot_implement_failure] +| | x exhausted [-> run_experiment] +| ++----+ + +run_experiment <-> [adjust_experiment] (optional) +| x fail [-> troubleshoot_run_failure] +| x exhausted [-> ensure_results] +| +generate_report +| +test <-> [x fail -> fix_tests -> retest] +| +push_branch +| +prepare_research_pr +| +run_experiment_lenses +| +stage_bundle +| +route_pr_or_local + ++--+ pr mode: +| | +| compose_research_pr +| | +-- [review_research_pr] (optional) +| | +-- [audit_claims] (optional) +| | +-- [resolve_research_review] (on changes_requested) +| | +-- [resolve_claims_review] (on changes_requested) +| | | +| | merge_escalations +| | | +-- [re_run_experiment] (optional) +| | | | | +| | | re_generate_report +| | | | x fail [-> re_push_research] +| | | | +| | | re_test <-> [x fail -> re_push_research] +| | | | +| | re_push_research +| | | x fail [-> begin_archival] +| | | +| | finalize_bundle_render +| | | x fail [-> route_archive_or_export] +| | | +| | begin_archival +| | | capture_experiment_branch +| | | | x fail [-> patch_token_summary] +| | | | +| | | create_artifact_branch +| | | | x fail [-> patch_token_summary] +| | | | +| | | open_artifact_pr +| | | | x fail [-> patch_token_summary] +| | | | +| | | tag_experiment_branch +| | | | x fail [-> patch_token_summary] +| | | | +| | | close_experiment_pr +| | | | x fail [-> patch_token_summary] +| | | +| | patch_token_summary +| | +| +-- finalize_bundle (local mode) +| | | +| | finalize_bundle_render +| | | x fail [-> route_archive_or_export] +| | | +| | export_local_bundle +| | | x fail [-> patch_token_summary] +| | | +| | patch_token_summary diff --git a/src/autoskillit/recipes/examples/example-campaign.json b/src/autoskillit/recipes/examples/example-campaign.json new file mode 100644 index 0000000000..3b8b27df0d --- /dev/null +++ b/src/autoskillit/recipes/examples/example-campaign.json @@ -0,0 +1,49 @@ +{ + "name": "example-campaign", + "description": "Example campaign recipe demonstrating all campaign schema fields. Use as a reference when authoring new campaigns.", + "kind": "campaign", + "recipe_version": "1.0.0", + "categories": [ + "implementation-family" + ], + "requires_recipe_packs": [ + "implementation-family" + ], + "allowed_recipes": [], + "continue_on_failure": false, + "ingredients": { + "source_dir": { + "description": "Source repository path", + "required": false, + "default": "" + } + }, + "dispatches": [ + { + "name": "phase-1-implement", + "recipe": "implementation", + "task": "Implement the core feature", + "ingredients": { + "task": "Implement the core feature", + "source_dir": "${{ inputs.source_dir }}" + }, + "depends_on": [] + }, + { + "name": "phase-2-follow-up", + "recipe": "implementation", + "task": "Add tests and documentation for the core feature", + "ingredients": { + "task": "Add tests and documentation for the core feature", + "source_dir": "${{ inputs.source_dir }}" + }, + "depends_on": [ + "phase-1-implement" + ] + } + ], + "kitchen_rules": [ + "NEVER modify files outside the project directory" + ], + "steps": {} +} diff --git a/src/autoskillit/recipes/examples/example-campaign.yaml b/src/autoskillit/recipes/examples/example-campaign.yaml new file mode 100644 index 0000000000..ba6824dbb6 --- /dev/null +++ b/src/autoskillit/recipes/examples/example-campaign.yaml @@ -0,0 +1,45 @@ +name: example-campaign +description: >- + Example campaign recipe demonstrating all campaign schema fields. + Use as a reference when authoring new campaigns. +kind: campaign +recipe_version: "1.0.0" + +categories: + - implementation-family + +requires_recipe_packs: + - implementation-family + +allowed_recipes: [] + +continue_on_failure: false + +ingredients: + source_dir: + description: Source repository path + required: false + default: "" + +dispatches: + - name: phase-1-implement + recipe: implementation + task: "Implement the core feature" + ingredients: + task: "Implement the core feature" + source_dir: "${{ inputs.source_dir }}" + depends_on: [] + + - name: phase-2-follow-up + recipe: implementation + task: "Add tests and documentation for the core feature" + ingredients: + task: "Add tests and documentation for the core feature" + source_dir: "${{ inputs.source_dir }}" + depends_on: + - phase-1-implement + +kitchen_rules: + - "NEVER modify files outside the project directory" + +steps: {} diff --git a/src/autoskillit/recipes/experiment-types/benchmark.json b/src/autoskillit/recipes/experiment-types/benchmark.json new file mode 100644 index 0000000000..83ca3d4330 --- /dev/null +++ b/src/autoskillit/recipes/experiment-types/benchmark.json @@ -0,0 +1,31 @@ +{ + "name": "benchmark", + "schema_version": "1.0", + "priority": 3, + "is_fallback": false, + "classification_triggers": [ + "IVs are system/method names, DVs are performance metrics, multiple comparators" + ], + "dimension_weights": { + "causal_structure": "S", + "variance_protocol": "H", + "statistical_corrections": "M", + "ecological_validity": "M", + "measurement_alignment": "M", + "resource_proportionality": "L", + "data_acquisition": "M", + "agent_implementability": "H" + }, + "applicable_lenses": { + "primary": "exp-lens-estimand-clarity", + "secondary": null + }, + "red_team_focus": { + "specific": "asymmetric effort", + "severity_cap": "warning" + }, + "l1_severity": { + "estimand_clarity": "warning", + "hypothesis_falsifiability": "warning" + } +} diff --git a/src/autoskillit/recipes/experiment-types/benchmark.yaml b/src/autoskillit/recipes/experiment-types/benchmark.yaml index f82897ca46..31e331d31e 100644 --- a/src/autoskillit/recipes/experiment-types/benchmark.yaml +++ b/src/autoskillit/recipes/experiment-types/benchmark.yaml @@ -1,4 +1,7 @@ name: benchmark +schema_version: "1.0" +priority: 3 +is_fallback: false classification_triggers: - "IVs are system/method names, DVs are performance metrics, multiple comparators" dimension_weights: diff --git a/src/autoskillit/recipes/experiment-types/causal_inference.json b/src/autoskillit/recipes/experiment-types/causal_inference.json new file mode 100644 index 0000000000..642c2af7b1 --- /dev/null +++ b/src/autoskillit/recipes/experiment-types/causal_inference.json @@ -0,0 +1,32 @@ +{ + "name": "causal_inference", + "schema_version": "1.0", + "priority": 1, + "is_fallback": false, + "classification_triggers": [ + "Explicit randomization, manipulation, or intervention assignment: 'randomly assigned', 'intervention applied', 'treatment group', 'control group', 'manipulated X', 'assigned to condition'", + "Causal claim ('test whether X causes Y') AND explicit confounder adjustment or treatment assignment described" + ], + "dimension_weights": { + "causal_structure": "H", + "variance_protocol": "L", + "statistical_corrections": "H", + "ecological_validity": "L", + "measurement_alignment": "M", + "resource_proportionality": "L", + "data_acquisition": "M", + "agent_implementability": "M" + }, + "applicable_lenses": { + "primary": "exp-lens-estimand-clarity", + "secondary": null + }, + "red_team_focus": { + "specific": "unblocked backdoor path", + "severity_cap": "critical" + }, + "l1_severity": { + "estimand_clarity": "critical", + "hypothesis_falsifiability": "critical" + } +} diff --git a/src/autoskillit/recipes/experiment-types/causal_inference.yaml b/src/autoskillit/recipes/experiment-types/causal_inference.yaml index 0ff2da0e81..7afd704ab5 100644 --- a/src/autoskillit/recipes/experiment-types/causal_inference.yaml +++ b/src/autoskillit/recipes/experiment-types/causal_inference.yaml @@ -1,6 +1,10 @@ name: causal_inference +schema_version: "1.0" +priority: 1 +is_fallback: false classification_triggers: - - "Causal language (\"causes\", \"effect of\"), confounders in threats" + - "Explicit randomization, manipulation, or intervention assignment: 'randomly assigned', 'intervention applied', 'treatment group', 'control group', 'manipulated X', 'assigned to condition'" + - "Causal claim ('test whether X causes Y') AND explicit confounder adjustment or treatment assignment described" dimension_weights: causal_structure: H variance_protocol: L diff --git a/src/autoskillit/recipes/experiment-types/configuration_study.json b/src/autoskillit/recipes/experiment-types/configuration_study.json new file mode 100644 index 0000000000..2aff04929e --- /dev/null +++ b/src/autoskillit/recipes/experiment-types/configuration_study.json @@ -0,0 +1,31 @@ +{ + "name": "configuration_study", + "schema_version": "1.0", + "priority": 5, + "is_fallback": false, + "classification_triggers": [ + "IVs are numeric parameters of one system, grid/sweep structure" + ], + "dimension_weights": { + "causal_structure": "S", + "variance_protocol": "H", + "statistical_corrections": "H", + "ecological_validity": "L", + "measurement_alignment": "M", + "resource_proportionality": "L", + "data_acquisition": "M", + "agent_implementability": "H" + }, + "applicable_lenses": { + "primary": "exp-lens-estimand-clarity", + "secondary": null + }, + "red_team_focus": { + "specific": "overfitting to held-out set", + "severity_cap": "warning" + }, + "l1_severity": { + "estimand_clarity": "warning", + "hypothesis_falsifiability": "warning" + } +} diff --git a/src/autoskillit/recipes/experiment-types/configuration_study.yaml b/src/autoskillit/recipes/experiment-types/configuration_study.yaml index 901bddab8b..9fcf587703 100644 --- a/src/autoskillit/recipes/experiment-types/configuration_study.yaml +++ b/src/autoskillit/recipes/experiment-types/configuration_study.yaml @@ -1,4 +1,7 @@ name: configuration_study +schema_version: "1.0" +priority: 5 +is_fallback: false classification_triggers: - "IVs are numeric parameters of one system, grid/sweep structure" dimension_weights: diff --git a/src/autoskillit/recipes/experiment-types/evidence_synthesis.json b/src/autoskillit/recipes/experiment-types/evidence_synthesis.json new file mode 100644 index 0000000000..a0c5c8bbf7 --- /dev/null +++ b/src/autoskillit/recipes/experiment-types/evidence_synthesis.json @@ -0,0 +1,40 @@ +{ + "name": "evidence_synthesis", + "schema_version": "1.0", + "priority": 2, + "is_fallback": false, + "classification_triggers": [ + "Aggregates results from multiple existing studies: 'meta-analysis', 'systematic review', 'literature synthesis', 'pooled estimate', 'effect size aggregation', 'PRISMA'", + "No new data collection; studies are the unit of analysis: 'included studies', 'search strategy', 'inclusion criteria', 'forest plot'" + ], + "dimension_weights": { + "causal_structure": "L", + "variance_protocol": "M", + "statistical_corrections": "H", + "ecological_validity": "H", + "measurement_alignment": "H", + "resource_proportionality": "L", + "data_acquisition": "H", + "agent_implementability": "M" + }, + "applicable_lenses": { + "primary": "exp-lens-measurement-validity", + "secondary": "exp-lens-sensitivity-robustness" + }, + "red_team_focus": { + "specific": "publication bias and outcome reporting bias", + "severity_cap": "critical" + }, + "l1_severity": { + "estimand_clarity": "critical", + "hypothesis_falsifiability": "warning" + }, + "dimension_weight_rationale": { + "variance_protocol": "Heterogeneity across included studies must be characterized; random-effects vs fixed-effects model choice is load-bearing", + "statistical_corrections": "Multiple comparisons across subgroup and sensitivity analyses require explicit correction", + "ecological_validity": "Generalizability across study populations is a primary deliverable of systematic reviews", + "measurement_alignment": "Outcome harmonization across heterogeneous measurement instruments is the defining data challenge", + "data_acquisition": "Search strategy completeness and PRISMA compliance determine which evidence enters the pool", + "agent_implementability": "Protocol steps are well-defined but access to literature databases may require external tools" + } +} diff --git a/src/autoskillit/recipes/experiment-types/evidence_synthesis.yaml b/src/autoskillit/recipes/experiment-types/evidence_synthesis.yaml new file mode 100644 index 0000000000..9ff478af90 --- /dev/null +++ b/src/autoskillit/recipes/experiment-types/evidence_synthesis.yaml @@ -0,0 +1,32 @@ +name: evidence_synthesis +schema_version: "1.0" +priority: 2 +is_fallback: false +classification_triggers: + - "Aggregates results from multiple existing studies: 'meta-analysis', 'systematic review', 'literature synthesis', 'pooled estimate', 'effect size aggregation', 'PRISMA'" + - "No new data collection; studies are the unit of analysis: 'included studies', 'search strategy', 'inclusion criteria', 'forest plot'" +dimension_weights: + causal_structure: L + variance_protocol: M + statistical_corrections: H + ecological_validity: H + measurement_alignment: H + resource_proportionality: L + data_acquisition: H + agent_implementability: M +applicable_lenses: + primary: exp-lens-measurement-validity + secondary: exp-lens-sensitivity-robustness +red_team_focus: + specific: publication bias and outcome reporting bias + severity_cap: critical +l1_severity: + estimand_clarity: critical + hypothesis_falsifiability: warning +dimension_weight_rationale: + variance_protocol: "Heterogeneity across included studies must be characterized; random-effects vs fixed-effects model choice is load-bearing" + statistical_corrections: "Multiple comparisons across subgroup and sensitivity analyses require explicit correction" + ecological_validity: "Generalizability across study populations is a primary deliverable of systematic reviews" + measurement_alignment: "Outcome harmonization across heterogeneous measurement instruments is the defining data challenge" + data_acquisition: "Search strategy completeness and PRISMA compliance determine which evidence enters the pool" + agent_implementability: "Protocol steps are well-defined but access to literature databases may require external tools" diff --git a/src/autoskillit/recipes/experiment-types/exploratory.json b/src/autoskillit/recipes/experiment-types/exploratory.json new file mode 100644 index 0000000000..f3ce622193 --- /dev/null +++ b/src/autoskillit/recipes/experiment-types/exploratory.json @@ -0,0 +1,31 @@ +{ + "name": "exploratory", + "schema_version": "1.0", + "priority": 999, + "is_fallback": true, + "classification_triggers": [ + "Default — no prior rule fires, or hypothesis absent" + ], + "dimension_weights": { + "causal_structure": "L", + "variance_protocol": "L", + "statistical_corrections": "S", + "ecological_validity": "M", + "measurement_alignment": "M", + "resource_proportionality": "L", + "data_acquisition": "M", + "agent_implementability": "L" + }, + "applicable_lenses": { + "primary": "exp-lens-estimand-clarity", + "secondary": null + }, + "red_team_focus": { + "specific": "HARKing vulnerability", + "severity_cap": "info" + }, + "l1_severity": { + "estimand_clarity": "info", + "hypothesis_falsifiability": "info" + } +} diff --git a/src/autoskillit/recipes/experiment-types/exploratory.yaml b/src/autoskillit/recipes/experiment-types/exploratory.yaml index a7dae8b654..1206963e73 100644 --- a/src/autoskillit/recipes/experiment-types/exploratory.yaml +++ b/src/autoskillit/recipes/experiment-types/exploratory.yaml @@ -1,4 +1,7 @@ name: exploratory +schema_version: "1.0" +priority: 999 +is_fallback: true classification_triggers: - "Default — no prior rule fires, or hypothesis absent" dimension_weights: diff --git a/src/autoskillit/recipes/experiment-types/factorial_design.json b/src/autoskillit/recipes/experiment-types/factorial_design.json new file mode 100644 index 0000000000..6089faed78 --- /dev/null +++ b/src/autoskillit/recipes/experiment-types/factorial_design.json @@ -0,0 +1,39 @@ +{ + "name": "factorial_design", + "schema_version": "1.0", + "priority": 4, + "is_fallback": false, + "classification_triggers": [ + "Multiple independent variables crossed to study main effects and interactions: 'factorial', '2x2', '2x3', 'interaction effect', 'factorial ANOVA', 'multi-factor', 'crossed design', 'full factorial', 'fractional factorial'", + "Explicit interest in how one factor's effect depends on another: 'interaction between', 'moderation', 'effect of A depends on B', 'synergistic effect'" + ], + "dimension_weights": { + "causal_structure": "H", + "variance_protocol": "L", + "statistical_corrections": "H", + "ecological_validity": "L", + "measurement_alignment": "M", + "resource_proportionality": "L", + "data_acquisition": "M", + "agent_implementability": "H" + }, + "applicable_lenses": { + "primary": "exp-lens-estimand-clarity", + "secondary": "exp-lens-error-budget" + }, + "red_team_focus": { + "specific": "underpowered interaction tests treated as confirmatory", + "severity_cap": "warning" + }, + "l1_severity": { + "estimand_clarity": "warning", + "hypothesis_falsifiability": "warning" + }, + "dimension_weight_rationale": { + "causal_structure": "Factorial designs include explicit manipulation and random assignment; interaction interpretation requires the correct causal model for each factor", + "statistical_corrections": "All main effects and interaction terms inflate family-wise error rate; pre-specification of the primary interaction is required", + "measurement_alignment": "Outcome metric must detect interaction effects, which are typically smaller than main effects", + "data_acquisition": "Cell balance across factor combinations and counterbalancing affect interaction estimate validity", + "agent_implementability": "Factorial ANOVA and regression with interaction terms are highly standardized and automatable" + } +} diff --git a/src/autoskillit/recipes/experiment-types/factorial_design.yaml b/src/autoskillit/recipes/experiment-types/factorial_design.yaml new file mode 100644 index 0000000000..ad5bff5ed8 --- /dev/null +++ b/src/autoskillit/recipes/experiment-types/factorial_design.yaml @@ -0,0 +1,31 @@ +name: factorial_design +schema_version: "1.0" +priority: 4 +is_fallback: false +classification_triggers: + - "Multiple independent variables crossed to study main effects and interactions: 'factorial', '2x2', '2x3', 'interaction effect', 'factorial ANOVA', 'multi-factor', 'crossed design', 'full factorial', 'fractional factorial'" + - "Explicit interest in how one factor's effect depends on another: 'interaction between', 'moderation', 'effect of A depends on B', 'synergistic effect'" +dimension_weights: + causal_structure: H + variance_protocol: L + statistical_corrections: H + ecological_validity: L + measurement_alignment: M + resource_proportionality: L + data_acquisition: M + agent_implementability: H +applicable_lenses: + primary: exp-lens-estimand-clarity + secondary: exp-lens-error-budget +red_team_focus: + specific: underpowered interaction tests treated as confirmatory + severity_cap: warning +l1_severity: + estimand_clarity: warning + hypothesis_falsifiability: warning +dimension_weight_rationale: + causal_structure: "Factorial designs include explicit manipulation and random assignment; interaction interpretation requires the correct causal model for each factor" + statistical_corrections: "All main effects and interaction terms inflate family-wise error rate; pre-specification of the primary interaction is required" + measurement_alignment: "Outcome metric must detect interaction effects, which are typically smaller than main effects" + data_acquisition: "Cell balance across factor combinations and counterbalancing affect interaction estimate validity" + agent_implementability: "Factorial ANOVA and regression with interaction terms are highly standardized and automatable" diff --git a/src/autoskillit/recipes/experiment-types/instrument_validation.json b/src/autoskillit/recipes/experiment-types/instrument_validation.json new file mode 100644 index 0000000000..c95ae58f19 --- /dev/null +++ b/src/autoskillit/recipes/experiment-types/instrument_validation.json @@ -0,0 +1,40 @@ +{ + "name": "instrument_validation", + "schema_version": "1.0", + "priority": 7, + "is_fallback": false, + "classification_triggers": [ + "Tests reliability, validity, or measurement properties of a scale, instrument, or test: 'construct validity', 'internal consistency', 'Cronbach alpha', 'confirmatory factor analysis', 'CFA', 'IRT', 'item response theory', 'convergent validity', 'discriminant validity'", + "Scale development or validation study: 'measure development', 'instrument validation', 'psychometric properties', 'factor structure', 'test-retest reliability'" + ], + "dimension_weights": { + "causal_structure": "S", + "variance_protocol": "H", + "statistical_corrections": "M", + "ecological_validity": "M", + "measurement_alignment": "H", + "resource_proportionality": "L", + "data_acquisition": "M", + "agent_implementability": "M" + }, + "applicable_lenses": { + "primary": "exp-lens-measurement-validity", + "secondary": "exp-lens-estimand-clarity" + }, + "red_team_focus": { + "specific": "construct underrepresentation inflating reliability", + "severity_cap": "warning" + }, + "l1_severity": { + "estimand_clarity": "warning", + "hypothesis_falsifiability": "warning" + }, + "dimension_weight_rationale": { + "variance_protocol": "Reliability decomposition is the primary deliverable; variance components must be explicitly estimated", + "statistical_corrections": "Multiple fit index comparisons in CFA and competing factor models introduce multiplicity", + "ecological_validity": "Measurement context affects generalizability of psychometric properties; cross-sample invariance is required", + "measurement_alignment": "The entire study purpose is to establish that the instrument measures the intended construct", + "data_acquisition": "Sample size for factor analysis and participant characteristics determine reliability estimate stability", + "agent_implementability": "CFA and IRT pipelines are computationally defined but require psychometric expertise for interpretation" + } +} diff --git a/src/autoskillit/recipes/experiment-types/instrument_validation.yaml b/src/autoskillit/recipes/experiment-types/instrument_validation.yaml new file mode 100644 index 0000000000..6e109966f4 --- /dev/null +++ b/src/autoskillit/recipes/experiment-types/instrument_validation.yaml @@ -0,0 +1,32 @@ +name: instrument_validation +schema_version: "1.0" +priority: 7 +is_fallback: false +classification_triggers: + - "Tests reliability, validity, or measurement properties of a scale, instrument, or test: 'construct validity', 'internal consistency', 'Cronbach alpha', 'confirmatory factor analysis', 'CFA', 'IRT', 'item response theory', 'convergent validity', 'discriminant validity'" + - "Scale development or validation study: 'measure development', 'instrument validation', 'psychometric properties', 'factor structure', 'test-retest reliability'" +dimension_weights: + causal_structure: S + variance_protocol: H + statistical_corrections: M + ecological_validity: M + measurement_alignment: H + resource_proportionality: L + data_acquisition: M + agent_implementability: M +applicable_lenses: + primary: exp-lens-measurement-validity + secondary: exp-lens-estimand-clarity +red_team_focus: + specific: construct underrepresentation inflating reliability + severity_cap: warning +l1_severity: + estimand_clarity: warning + hypothesis_falsifiability: warning +dimension_weight_rationale: + variance_protocol: "Reliability decomposition is the primary deliverable; variance components must be explicitly estimated" + statistical_corrections: "Multiple fit index comparisons in CFA and competing factor models introduce multiplicity" + ecological_validity: "Measurement context affects generalizability of psychometric properties; cross-sample invariance is required" + measurement_alignment: "The entire study purpose is to establish that the instrument measures the intended construct" + data_acquisition: "Sample size for factor analysis and participant characteristics determine reliability estimate stability" + agent_implementability: "CFA and IRT pipelines are computationally defined but require psychometric expertise for interpretation" diff --git a/src/autoskillit/recipes/experiment-types/observational_correlational.json b/src/autoskillit/recipes/experiment-types/observational_correlational.json new file mode 100644 index 0000000000..e9b770f48b --- /dev/null +++ b/src/autoskillit/recipes/experiment-types/observational_correlational.json @@ -0,0 +1,39 @@ +{ + "name": "observational_correlational", + "schema_version": "1.0", + "priority": 10, + "is_fallback": false, + "classification_triggers": [ + "No manipulation or intervention; variables measured as they naturally occur: 'observational study', 'correlational', 'cross-sectional', 'longitudinal survey', 'archival data', 'naturalistic'", + "Association or prediction claim without causal assignment: 'associated with', 'predicts', 'correlated with', 'relationship between' without randomization or intervention described" + ], + "dimension_weights": { + "causal_structure": "M", + "variance_protocol": "L", + "statistical_corrections": "M", + "ecological_validity": "H", + "measurement_alignment": "H", + "resource_proportionality": "L", + "data_acquisition": "H", + "agent_implementability": "L" + }, + "applicable_lenses": { + "primary": "exp-lens-validity-threats", + "secondary": "exp-lens-measurement-validity" + }, + "red_team_focus": { + "specific": "unmeasured confounder invalidating association claim", + "severity_cap": "warning" + }, + "l1_severity": { + "estimand_clarity": "warning", + "hypothesis_falsifiability": "warning" + }, + "dimension_weight_rationale": { + "causal_structure": "Confounders must be enumerated in a DAG; covariate adjustment strategy determines whether associations are interpretable", + "statistical_corrections": "Multiple predictor tests and post-hoc subgroup splits require correction; p-hacking risk is elevated in large-N surveys", + "ecological_validity": "Naturalistic observation maximizes ecological validity; the key question is sample representativeness", + "measurement_alignment": "Self-report scales and archival proxies introduce construct validity threats requiring convergent evidence", + "data_acquisition": "Sampling frame, response rate, and missing-not-at-random mechanisms determine population validity" + } +} diff --git a/src/autoskillit/recipes/experiment-types/observational_correlational.yaml b/src/autoskillit/recipes/experiment-types/observational_correlational.yaml new file mode 100644 index 0000000000..9cfbb34808 --- /dev/null +++ b/src/autoskillit/recipes/experiment-types/observational_correlational.yaml @@ -0,0 +1,31 @@ +name: observational_correlational +schema_version: "1.0" +priority: 10 +is_fallback: false +classification_triggers: + - "No manipulation or intervention; variables measured as they naturally occur: 'observational study', 'correlational', 'cross-sectional', 'longitudinal survey', 'archival data', 'naturalistic'" + - "Association or prediction claim without causal assignment: 'associated with', 'predicts', 'correlated with', 'relationship between' without randomization or intervention described" +dimension_weights: + causal_structure: M + variance_protocol: L + statistical_corrections: M + ecological_validity: H + measurement_alignment: H + resource_proportionality: L + data_acquisition: H + agent_implementability: L +applicable_lenses: + primary: exp-lens-validity-threats + secondary: exp-lens-measurement-validity +red_team_focus: + specific: unmeasured confounder invalidating association claim + severity_cap: warning +l1_severity: + estimand_clarity: warning + hypothesis_falsifiability: warning +dimension_weight_rationale: + causal_structure: "Confounders must be enumerated in a DAG; covariate adjustment strategy determines whether associations are interpretable" + statistical_corrections: "Multiple predictor tests and post-hoc subgroup splits require correction; p-hacking risk is elevated in large-N surveys" + ecological_validity: "Naturalistic observation maximizes ecological validity; the key question is sample representativeness" + measurement_alignment: "Self-report scales and archival proxies introduce construct validity threats requiring convergent evidence" + data_acquisition: "Sampling frame, response rate, and missing-not-at-random mechanisms determine population validity" diff --git a/src/autoskillit/recipes/experiment-types/qualitative_interpretive.json b/src/autoskillit/recipes/experiment-types/qualitative_interpretive.json new file mode 100644 index 0000000000..e5d63c6d32 --- /dev/null +++ b/src/autoskillit/recipes/experiment-types/qualitative_interpretive.json @@ -0,0 +1,35 @@ +{ + "name": "qualitative_interpretive", + "schema_version": "1.0", + "priority": 11, + "is_fallback": false, + "classification_triggers": [ + "Non-numerical data or interpretive epistemology: 'qualitative', 'thematic analysis', 'grounded theory', 'phenomenology', 'discourse analysis', 'ethnography', 'interpretive', 'narrative inquiry'", + "Meaning, experience, or process as the research target rather than a quantitative estimand: 'lived experience', 'meaning-making', 'thick description', 'saturation', 'purposive sampling', 'inductive'" + ], + "dimension_weights": { + "causal_structure": "S", + "variance_protocol": "S", + "statistical_corrections": "S", + "ecological_validity": "S", + "measurement_alignment": "S", + "resource_proportionality": "S", + "data_acquisition": "L", + "agent_implementability": "L" + }, + "applicable_lenses": { + "primary": "exp-lens-estimand-clarity", + "secondary": "exp-lens-validity-threats" + }, + "red_team_focus": { + "specific": "researcher positionality and reflexivity undocumented", + "severity_cap": "warning" + }, + "l1_severity": { + "estimand_clarity": "warning", + "hypothesis_falsifiability": "info" + }, + "dimension_weight_rationale": { + "data_acquisition": "Purposive sampling strategy remains assessable at reduced rigor; saturation criteria are checked but qualitative validity is outside the quantitative audit framework" + } +} diff --git a/src/autoskillit/recipes/experiment-types/qualitative_interpretive.yaml b/src/autoskillit/recipes/experiment-types/qualitative_interpretive.yaml new file mode 100644 index 0000000000..a64c7974b0 --- /dev/null +++ b/src/autoskillit/recipes/experiment-types/qualitative_interpretive.yaml @@ -0,0 +1,27 @@ +name: qualitative_interpretive +schema_version: "1.0" +priority: 11 +is_fallback: false +classification_triggers: + - "Non-numerical data or interpretive epistemology: 'qualitative', 'thematic analysis', 'grounded theory', 'phenomenology', 'discourse analysis', 'ethnography', 'interpretive', 'narrative inquiry'" + - "Meaning, experience, or process as the research target rather than a quantitative estimand: 'lived experience', 'meaning-making', 'thick description', 'saturation', 'purposive sampling', 'inductive'" +dimension_weights: + causal_structure: S + variance_protocol: S + statistical_corrections: S + ecological_validity: S + measurement_alignment: S + resource_proportionality: S + data_acquisition: L + agent_implementability: L +applicable_lenses: + primary: exp-lens-estimand-clarity + secondary: exp-lens-validity-threats +red_team_focus: + specific: researcher positionality and reflexivity undocumented + severity_cap: warning +l1_severity: + estimand_clarity: warning + hypothesis_falsifiability: info +dimension_weight_rationale: + data_acquisition: "Purposive sampling strategy remains assessable at reduced rigor; saturation criteria are checked but qualitative validity is outside the quantitative audit framework" diff --git a/src/autoskillit/recipes/experiment-types/robustness_audit.json b/src/autoskillit/recipes/experiment-types/robustness_audit.json new file mode 100644 index 0000000000..62921a5f01 --- /dev/null +++ b/src/autoskillit/recipes/experiment-types/robustness_audit.json @@ -0,0 +1,31 @@ +{ + "name": "robustness_audit", + "schema_version": "1.0", + "priority": 8, + "is_fallback": false, + "classification_triggers": [ + "Tests generalization/stability, deliberately varied conditions" + ], + "dimension_weights": { + "causal_structure": "M", + "variance_protocol": "M", + "statistical_corrections": "S", + "ecological_validity": "H", + "measurement_alignment": "H", + "resource_proportionality": "L", + "data_acquisition": "H", + "agent_implementability": "M" + }, + "applicable_lenses": { + "primary": "exp-lens-estimand-clarity", + "secondary": null + }, + "red_team_focus": { + "specific": "unrealistic threat distribution", + "severity_cap": "warning" + }, + "l1_severity": { + "estimand_clarity": "warning", + "hypothesis_falsifiability": "warning" + } +} diff --git a/src/autoskillit/recipes/experiment-types/robustness_audit.yaml b/src/autoskillit/recipes/experiment-types/robustness_audit.yaml index ab4fe78f8e..16f8240149 100644 --- a/src/autoskillit/recipes/experiment-types/robustness_audit.yaml +++ b/src/autoskillit/recipes/experiment-types/robustness_audit.yaml @@ -1,4 +1,7 @@ name: robustness_audit +schema_version: "1.0" +priority: 8 +is_fallback: false classification_triggers: - "Tests generalization/stability, deliberately varied conditions" dimension_weights: diff --git a/src/autoskillit/recipes/experiment-types/simulation_modeling.json b/src/autoskillit/recipes/experiment-types/simulation_modeling.json new file mode 100644 index 0000000000..5fbf2b1ad0 --- /dev/null +++ b/src/autoskillit/recipes/experiment-types/simulation_modeling.json @@ -0,0 +1,40 @@ +{ + "name": "simulation_modeling", + "schema_version": "1.0", + "priority": 6, + "is_fallback": false, + "classification_triggers": [ + "Generates synthetic data from a computational or theoretical model: 'simulation', 'agent-based model', 'ABM', 'Monte Carlo', 'computational model', 'synthetic data', 'stochastic simulation'", + "System behavior studied by varying model parameters rather than real-world conditions: 'parameter sweep', 'sensitivity analysis on model parameters', 'simulated scenarios', 'model-generated outcomes'" + ], + "dimension_weights": { + "causal_structure": "M", + "variance_protocol": "H", + "statistical_corrections": "M", + "ecological_validity": "H", + "measurement_alignment": "M", + "resource_proportionality": "L", + "data_acquisition": "L", + "agent_implementability": "H" + }, + "applicable_lenses": { + "primary": "exp-lens-sensitivity-robustness", + "secondary": "exp-lens-validity-threats" + }, + "red_team_focus": { + "specific": "model misspecification masquerading as empirical finding", + "severity_cap": "warning" + }, + "l1_severity": { + "estimand_clarity": "warning", + "hypothesis_falsifiability": "warning" + }, + "dimension_weight_rationale": { + "causal_structure": "Simulation encodes a causal model explicitly; bridging assumptions to real-world causal claims must be stated", + "variance_protocol": "Stochastic simulations require adequate replication across random seeds with convergence diagnostics", + "statistical_corrections": "Parameter sweep designs generate many output metrics with high multiple-comparison pressure", + "ecological_validity": "Whether the simulation accurately represents the target real-world system is the central validity question", + "measurement_alignment": "Simulation output metrics must be operationally linked to real-world constructs of interest", + "agent_implementability": "Computational simulation pipelines are highly structured; parameter sweeps and seed management are automatable" + } +} diff --git a/src/autoskillit/recipes/experiment-types/simulation_modeling.yaml b/src/autoskillit/recipes/experiment-types/simulation_modeling.yaml new file mode 100644 index 0000000000..c22dfbcb60 --- /dev/null +++ b/src/autoskillit/recipes/experiment-types/simulation_modeling.yaml @@ -0,0 +1,32 @@ +name: simulation_modeling +schema_version: "1.0" +priority: 6 +is_fallback: false +classification_triggers: + - "Generates synthetic data from a computational or theoretical model: 'simulation', 'agent-based model', 'ABM', 'Monte Carlo', 'computational model', 'synthetic data', 'stochastic simulation'" + - "System behavior studied by varying model parameters rather than real-world conditions: 'parameter sweep', 'sensitivity analysis on model parameters', 'simulated scenarios', 'model-generated outcomes'" +dimension_weights: + causal_structure: M + variance_protocol: H + statistical_corrections: M + ecological_validity: H + measurement_alignment: M + resource_proportionality: L + data_acquisition: L + agent_implementability: H +applicable_lenses: + primary: exp-lens-sensitivity-robustness + secondary: exp-lens-validity-threats +red_team_focus: + specific: model misspecification masquerading as empirical finding + severity_cap: warning +l1_severity: + estimand_clarity: warning + hypothesis_falsifiability: warning +dimension_weight_rationale: + causal_structure: "Simulation encodes a causal model explicitly; bridging assumptions to real-world causal claims must be stated" + variance_protocol: "Stochastic simulations require adequate replication across random seeds with convergence diagnostics" + statistical_corrections: "Parameter sweep designs generate many output metrics with high multiple-comparison pressure" + ecological_validity: "Whether the simulation accurately represents the target real-world system is the central validity question" + measurement_alignment: "Simulation output metrics must be operationally linked to real-world constructs of interest" + agent_implementability: "Computational simulation pipelines are highly structured; parameter sweeps and seed management are automatable" diff --git a/src/autoskillit/recipes/experiment-types/single_subject.json b/src/autoskillit/recipes/experiment-types/single_subject.json new file mode 100644 index 0000000000..c54698a3d2 --- /dev/null +++ b/src/autoskillit/recipes/experiment-types/single_subject.json @@ -0,0 +1,39 @@ +{ + "name": "single_subject", + "schema_version": "1.0", + "priority": 9, + "is_fallback": false, + "classification_triggers": [ + "Study of one or very few individuals with intensive repeated measurement: 'N-of-1', 'single-subject', 'single case', 'case study', 'ABA design', 'ABAB', 'multiple baseline', 'idiographic'", + "Within-person replication logic rather than between-group inference: 'withdrawal design', 'reversal design', 'phase', 'baseline phase', 'treatment phase'" + ], + "dimension_weights": { + "causal_structure": "M", + "variance_protocol": "H", + "statistical_corrections": "S", + "ecological_validity": "H", + "measurement_alignment": "H", + "resource_proportionality": "L", + "data_acquisition": "H", + "agent_implementability": "L" + }, + "applicable_lenses": { + "primary": "exp-lens-variance-stability", + "secondary": "exp-lens-validity-threats" + }, + "red_team_focus": { + "specific": "baseline instability confounding phase-change interpretation", + "severity_cap": "warning" + }, + "l1_severity": { + "estimand_clarity": "warning", + "hypothesis_falsifiability": "warning" + }, + "dimension_weight_rationale": { + "causal_structure": "Within-person causal inference via phase designs is valid but external generalization requires theoretical argument", + "variance_protocol": "Autocorrelation, trend, and level in time-series must be modeled; baseline stability is a structural prerequisite", + "ecological_validity": "Single subjects are studied in naturalistic settings; ecological validity is a strength but context limits generalization", + "measurement_alignment": "Repeated measurement across many occasions requires demonstrated test-retest reliability and sensitivity to change", + "data_acquisition": "Continuous measurement schedule, missing observations, and phase-boundary decisions are structural design elements" + } +} diff --git a/src/autoskillit/recipes/experiment-types/single_subject.yaml b/src/autoskillit/recipes/experiment-types/single_subject.yaml new file mode 100644 index 0000000000..0f1f0aa088 --- /dev/null +++ b/src/autoskillit/recipes/experiment-types/single_subject.yaml @@ -0,0 +1,31 @@ +name: single_subject +schema_version: "1.0" +priority: 9 +is_fallback: false +classification_triggers: + - "Study of one or very few individuals with intensive repeated measurement: 'N-of-1', 'single-subject', 'single case', 'case study', 'ABA design', 'ABAB', 'multiple baseline', 'idiographic'" + - "Within-person replication logic rather than between-group inference: 'withdrawal design', 'reversal design', 'phase', 'baseline phase', 'treatment phase'" +dimension_weights: + causal_structure: M + variance_protocol: H + statistical_corrections: S + ecological_validity: H + measurement_alignment: H + resource_proportionality: L + data_acquisition: H + agent_implementability: L +applicable_lenses: + primary: exp-lens-variance-stability + secondary: exp-lens-validity-threats +red_team_focus: + specific: baseline instability confounding phase-change interpretation + severity_cap: warning +l1_severity: + estimand_clarity: warning + hypothesis_falsifiability: warning +dimension_weight_rationale: + causal_structure: "Within-person causal inference via phase designs is valid but external generalization requires theoretical argument" + variance_protocol: "Autocorrelation, trend, and level in time-series must be modeled; baseline stability is a structural prerequisite" + ecological_validity: "Single subjects are studied in naturalistic settings; ecological validity is a strength but context limits generalization" + measurement_alignment: "Repeated measurement across many occasions requires demonstrated test-retest reliability and sensitivity to change" + data_acquisition: "Continuous measurement schedule, missing observations, and phase-boundary decisions are structural design elements" diff --git a/src/autoskillit/recipes/full-audit.json b/src/autoskillit/recipes/full-audit.json new file mode 100644 index 0000000000..82a7d95bb9 --- /dev/null +++ b/src/autoskillit/recipes/full-audit.json @@ -0,0 +1,125 @@ +{ + "name": "full-audit", + "description": "Run audit-tests, audit-cohesion, audit-arch, audit-feature-gates, audit-docs, and audit-review-decisions in parallel, validate each report individually, and create a GitHub issue from each validated report.", + "summary": "branch → audit×6 ∥ → validate×6 ∥ → issue×6 ∥ → done", + "recipe_version": "1.2.0", + "requires_packs": [ + "audit", + "github" + ], + "ingredients": { + "workspace": { + "description": "Path to the git repository workspace", + "required": true + }, + "branch": { + "description": "The branch to audit against", + "required": true + }, + "max_parallel": { + "description": "Max audits to run in parallel (half the audit count by default)", + "default": "3", + "hidden": true + } + }, + "kitchen_rules": [ + "NEVER use native Claude Code tools (Read, Grep, Glob, Edit, Write, Bash, Agent, WebFetch, WebSearch, NotebookEdit) from the orchestrator. All work is delegated through MCP tools (run_skill, run_cmd, run_python).", + "Each audit chain (tests, cohesion, arch, feature-gates, docs, review-decisions) is independent. A failure in one chain MUST NOT block or affect the other five chains. Continue with successful chains and note failures for escalation.", + "PREFER-COMPLETION SCHEDULING: Maintain up to ${{ inputs.max_parallel }} concurrent run_skill calls. When any call completes, immediately dispatch the next item from the queue. Do not wait for all slots to finish before refilling — fill each freed slot individually.", + "Report paths are discovered from session output — look for file paths in {{AUTOSKILLIT_TEMP}}/ directories. Pass discovered paths as positional arguments to downstream skills." + ], + "steps": { + "checkout": { + "tool": "run_cmd", + "description": "Check out the target branch", + "with": { + "cmd": "git checkout ${{ inputs.branch }}", + "cwd": "${{ inputs.workspace }}", + "step_name": "checkout" + }, + "note": "Ensure the working directory is on the target branch before audits run. If the checkout fails (e.g., dirty working tree, unknown branch), route to escalate — audits cannot proceed on the wrong branch.\n", + "on_success": "run_audits", + "on_failure": "escalate_stop", + "retries": 0, + "on_exhausted": "escalate_stop" + }, + "init_audit_run": { + "tool": "run_python", + "description": "Create a unique per-run directory for this audit pipeline", + "with": { + "callable": "autoskillit.recipe._cmd_rpc.create_audit_run_dir", + "temp_dir": "{{AUTOSKILLIT_TEMP}}" + }, + "capture": { + "audit_run_dir": "${{ result.audit_run_dir }}" + }, + "on_success": "run_audits", + "on_failure": "escalate_stop", + "retries": 0, + "on_exhausted": "escalate_stop" + }, + "run_audits": { + "tool": "run_skill", + "model": "", + "description": "Run six audit skills in parallel", + "with": { + "skill_command": "/autoskillit:audit-tests", + "cwd": "${{ inputs.workspace }}", + "step_name": "run_audits" + }, + "note": "PREFER-COMPLETION AUDIT DISPATCH — Dispatch audits using prefer-completion scheduling capped at ${{ inputs.max_parallel }} concurrent slots:\n1. Queue all six audits:\n - run_skill(skill_command=\"/autoskillit:audit-tests\", cwd=\"${{ inputs.workspace }}\", step_name=\"audit_tests\")\n - run_skill(skill_command=\"/autoskillit:audit-cohesion\", cwd=\"${{ inputs.workspace }}\", step_name=\"audit_cohesion\")\n - run_skill(skill_command=\"/autoskillit:audit-arch\", cwd=\"${{ inputs.workspace }}\", step_name=\"audit_arch\")\n - run_skill(skill_command=\"/autoskillit:audit-feature-gates\", cwd=\"${{ inputs.workspace }}\", step_name=\"audit_feature_gates\")\n - run_skill(skill_command=\"/autoskillit:audit-docs\", cwd=\"${{ inputs.workspace }}\", step_name=\"audit_docs\")\n - run_skill(skill_command=\"/autoskillit:audit-review-decisions\", cwd=\"${{ inputs.workspace }}\", step_name=\"audit_review_decisions\")\n\n2. Launch the first ${{ inputs.max_parallel }} audits as run_skill calls\n in a single message.\n\n3. When any audit completes (success or failure), immediately launch\n the next queued audit in the same message as any other pending\n responses. Maintain ${{ inputs.max_parallel }} in-flight at all times\n until the queue is empty.\n\nEach audit writes its report to a well-known temp directory: - {{AUTOSKILLIT_TEMP}}/audit-tests/{name}_{timestamp}.md - {{AUTOSKILLIT_TEMP}}/audit-cohesion/cohesion_audit_{timestamp}.md - {{AUTOSKILLIT_TEMP}}/audit-arch/arch_audit_{timestamp}.md - {{AUTOSKILLIT_TEMP}}/audit-feature-gates/feature_gate_audit_{timestamp}.md - {{AUTOSKILLIT_TEMP}}/audit-docs/docs_audit_{timestamp}.md - {{AUTOSKILLIT_TEMP}}/audit-review-decisions/review_decisions_audit_{timestamp}.md\nAfter all six complete, discover the most-recently-modified .md file in each temp directory from the session output. Record which audits succeeded and their report paths for the validation wave.\nIf any audit fails, note the failure but continue with the other five. Proceed to validate_audits even if only one, two, three, four, or five audits succeeded. Route to escalate_stop ONLY if all six audits fail.\n", + "capture_list": { + "audit_report_paths": "${{ result.audit_report_path }}" + }, + "on_success": "validate_audits", + "on_failure": "escalate_stop", + "retries": 1, + "on_exhausted": "escalate_stop" + }, + "validate_audits": { + "tool": "run_skill", + "model": "", + "description": "Validate each audit report in parallel", + "with": { + "skill_command": "/autoskillit:validate-audit --audit-run-dir ${{ context.audit_run_dir }}", + "cwd": "${{ inputs.workspace }}", + "step_name": "validate_audits" + }, + "note": "PREFER-COMPLETION VALIDATION — Start validating as each audit finishes. Do not wait for all audits to complete before beginning validation.\nFor each succeeded audit, queue a validation:\n 1. run_skill(skill_command=\"/autoskillit:validate-test-audit {tests_report_path}\", cwd=\"${{ inputs.workspace }}\", step_name=\"validate_tests\")\n 2. run_skill(skill_command=\"/autoskillit:validate-audit {cohesion_report_path}\", cwd=\"${{ inputs.workspace }}\", step_name=\"validate_cohesion\")\n 3. run_skill(skill_command=\"/autoskillit:validate-audit {arch_report_path}\", cwd=\"${{ inputs.workspace }}\", step_name=\"validate_arch\")\n 4. run_skill(skill_command=\"/autoskillit:validate-audit {feature_gates_report_path}\", cwd=\"${{ inputs.workspace }}\", step_name=\"validate_feature_gates\")\n 5. run_skill(skill_command=\"/autoskillit:validate-audit {docs_report_path}\", cwd=\"${{ inputs.workspace }}\", step_name=\"validate_docs\")\n 6. run_skill(skill_command=\"/autoskillit:validate-review-decisions {review_decisions_report_path}\", cwd=\"${{ inputs.workspace }}\", step_name=\"validate_review_decisions\")\n\nApply the same prefer-completion scheduling as run_audits — maintain up to ${{ inputs.max_parallel }} concurrent slots across validations. As each slot frees, dispatch the next queued validation immediately.\nSkip validation for any audit that failed in the audit dispatch phase. Each validation writes to ${{ context.audit_run_dir }}/: - validated_report_{source}_{timestamp}.md (first line: \"validated: true\") - contested_findings_{source}_{timestamp}.md (only if contested > 0) - validation_summary_{source}_{timestamp}.md (always written) - ticket_body_{source}_{N}_{timestamp}.md (one per ticket group)\nAfter all complete, discover the validated_report_*.md files from session output. Record paths for the issue creation wave.\nIf any validation fails, note it but continue with the others. Route to escalate_stop ONLY if all six validations fail.\n", + "capture_list": { + "validated_report_paths": "${{ result.validated_report_path }}" + }, + "on_success": "create_issues", + "on_failure": "escalate_stop", + "retries": 1, + "on_exhausted": "escalate_stop" + }, + "create_issues": { + "tool": "run_python", + "description": "Batch-create GitHub issues via GraphQL from validated ticket body files", + "with": { + "callable": "autoskillit.recipe._cmd_rpc.batch_create_issues", + "workspace": "${{ inputs.workspace }}", + "audit_run_dir": "${{ context.audit_run_dir }}", + "timeout": 120 + }, + "note": "BATCHED ISSUE CREATION via GraphQL — The callable discovers all ticket_body_{source}_{N}_{ts}.md files in the validate-audit temp directory, parses titles and bodies, includes validation summaries, resolves repository and label IDs, and executes a single batched createIssue mutation with aliases. Chunks at 20 issues per request if needed.\nNo headless sessions are spawned. Rate-limit cost: 5 pts per chunk (vs 90 pts for 18 sequential REST calls).\n", + "capture": { + "issue_urls": "${{ result.issue_urls }}", + "issue_count": "${{ result.issue_count }}" + }, + "on_success": "done", + "on_failure": "escalate_stop", + "retries": 1, + "on_exhausted": "escalate_stop" + }, + "done": { + "action": "stop", + "message": "Full audit complete. GitHub issues created from validated audit reports. Emit the L3 result sentinel. context.issue_urls is a comma-separated string of the created issue URLs — include it directly in the sentinel JSON: {\"success\": true, \"reason\": \"completed\", \"summary\": \"Full audit complete. GitHub issues created.\", \"issue_urls\": \"${{ context.issue_urls }}\"} Artifacts are in {{AUTOSKILLIT_TEMP}}/." + }, + "escalate_stop": { + "action": "stop", + "message": "Full audit pipeline failed — one or more audit chains could not complete. Check session logs and {{AUTOSKILLIT_TEMP}}/ directories for partial results. Individual chain failures are noted in the session output above." + } + } +} diff --git a/src/autoskillit/recipes/full-audit.yaml b/src/autoskillit/recipes/full-audit.yaml new file mode 100644 index 0000000000..875a9a4df5 --- /dev/null +++ b/src/autoskillit/recipes/full-audit.yaml @@ -0,0 +1,204 @@ +name: full-audit +description: >- + Run audit-tests, audit-cohesion, audit-arch, audit-feature-gates, audit-docs, and + audit-review-decisions in parallel, validate each report individually, and create a + GitHub issue from each validated report. +summary: "branch → audit×6 ∥ → validate×6 ∥ → issue×6 ∥ → done" +recipe_version: "1.2.0" +requires_packs: [audit, github] + +ingredients: + workspace: + description: "Path to the git repository workspace" + required: true + branch: + description: "The branch to audit against" + required: true + max_parallel: + description: "Max audits to run in parallel (half the audit count by default)" + default: "3" + hidden: true + +kitchen_rules: + - >- + NEVER use native Claude Code tools (Read, Grep, Glob, Edit, Write, Bash, Agent, + WebFetch, WebSearch, NotebookEdit) from the orchestrator. All work is delegated + through MCP tools (run_skill, run_cmd, run_python). + - >- + Each audit chain (tests, cohesion, arch, feature-gates, docs, review-decisions) is + independent. A failure in one chain MUST NOT block or affect the other five chains. + Continue with successful chains and note failures for escalation. + - >- + PREFER-COMPLETION SCHEDULING: Maintain up to ${{ inputs.max_parallel }} + concurrent run_skill calls. When any call completes, immediately dispatch + the next item from the queue. Do not wait for all slots to finish before + refilling — fill each freed slot individually. + - >- + Report paths are discovered from session output — look for file paths + in {{AUTOSKILLIT_TEMP}}/ directories. Pass discovered paths as positional + arguments to downstream skills. + +steps: + checkout: + tool: run_cmd + description: "Check out the target branch" + with: + cmd: "git checkout ${{ inputs.branch }}" + cwd: "${{ inputs.workspace }}" + step_name: "checkout" + note: > + Ensure the working directory is on the target branch before audits run. + If the checkout fails (e.g., dirty working tree, unknown branch), route + to escalate — audits cannot proceed on the wrong branch. + on_success: run_audits + on_failure: escalate_stop + retries: 0 + on_exhausted: escalate_stop + + init_audit_run: + tool: run_python + description: "Create a unique per-run directory for this audit pipeline" + with: + callable: "autoskillit.recipe._cmd_rpc.create_audit_run_dir" + temp_dir: "{{AUTOSKILLIT_TEMP}}" + capture: + audit_run_dir: "${{ result.audit_run_dir }}" + on_success: run_audits + on_failure: escalate_stop + retries: 0 + on_exhausted: escalate_stop + + run_audits: + tool: run_skill + model: "" + description: "Run six audit skills in parallel" + with: + skill_command: "/autoskillit:audit-tests" + cwd: "${{ inputs.workspace }}" + step_name: "run_audits" + note: > + PREFER-COMPLETION AUDIT DISPATCH — Dispatch audits using prefer-completion + scheduling capped at ${{ inputs.max_parallel }} concurrent slots: + + 1. Queue all six audits: + - run_skill(skill_command="/autoskillit:audit-tests", cwd="${{ inputs.workspace }}", step_name="audit_tests") + - run_skill(skill_command="/autoskillit:audit-cohesion", cwd="${{ inputs.workspace }}", step_name="audit_cohesion") + - run_skill(skill_command="/autoskillit:audit-arch", cwd="${{ inputs.workspace }}", step_name="audit_arch") + - run_skill(skill_command="/autoskillit:audit-feature-gates", cwd="${{ inputs.workspace }}", step_name="audit_feature_gates") + - run_skill(skill_command="/autoskillit:audit-docs", cwd="${{ inputs.workspace }}", step_name="audit_docs") + - run_skill(skill_command="/autoskillit:audit-review-decisions", cwd="${{ inputs.workspace }}", step_name="audit_review_decisions") + + 2. Launch the first ${{ inputs.max_parallel }} audits as run_skill calls + in a single message. + + 3. When any audit completes (success or failure), immediately launch + the next queued audit in the same message as any other pending + responses. Maintain ${{ inputs.max_parallel }} in-flight at all times + until the queue is empty. + + Each audit writes its report to a well-known temp directory: + - {{AUTOSKILLIT_TEMP}}/audit-tests/{name}_{timestamp}.md + - {{AUTOSKILLIT_TEMP}}/audit-cohesion/cohesion_audit_{timestamp}.md + - {{AUTOSKILLIT_TEMP}}/audit-arch/arch_audit_{timestamp}.md + - {{AUTOSKILLIT_TEMP}}/audit-feature-gates/feature_gate_audit_{timestamp}.md + - {{AUTOSKILLIT_TEMP}}/audit-docs/docs_audit_{timestamp}.md + - {{AUTOSKILLIT_TEMP}}/audit-review-decisions/review_decisions_audit_{timestamp}.md + + After all six complete, discover the most-recently-modified .md file in + each temp directory from the session output. Record which audits succeeded + and their report paths for the validation wave. + + If any audit fails, note the failure but continue with the other five. + Proceed to validate_audits even if only one, two, three, four, or five audits succeeded. + Route to escalate_stop ONLY if all six audits fail. + capture_list: + audit_report_paths: "${{ result.audit_report_path }}" + on_success: validate_audits + on_failure: escalate_stop + retries: 1 + on_exhausted: escalate_stop + + validate_audits: + tool: run_skill + model: "" + description: "Validate each audit report in parallel" + with: + skill_command: "/autoskillit:validate-audit --audit-run-dir ${{ context.audit_run_dir }}" + cwd: "${{ inputs.workspace }}" + step_name: "validate_audits" + note: > + PREFER-COMPLETION VALIDATION — Start validating as each audit finishes. + Do not wait for all audits to complete before beginning validation. + + For each succeeded audit, queue a validation: + 1. run_skill(skill_command="/autoskillit:validate-test-audit {tests_report_path}", cwd="${{ inputs.workspace }}", step_name="validate_tests") + 2. run_skill(skill_command="/autoskillit:validate-audit {cohesion_report_path}", cwd="${{ inputs.workspace }}", step_name="validate_cohesion") + 3. run_skill(skill_command="/autoskillit:validate-audit {arch_report_path}", cwd="${{ inputs.workspace }}", step_name="validate_arch") + 4. run_skill(skill_command="/autoskillit:validate-audit {feature_gates_report_path}", cwd="${{ inputs.workspace }}", step_name="validate_feature_gates") + 5. run_skill(skill_command="/autoskillit:validate-audit {docs_report_path}", cwd="${{ inputs.workspace }}", step_name="validate_docs") + 6. run_skill(skill_command="/autoskillit:validate-review-decisions {review_decisions_report_path}", cwd="${{ inputs.workspace }}", step_name="validate_review_decisions") + + Apply the same prefer-completion scheduling as run_audits — maintain up + to ${{ inputs.max_parallel }} concurrent slots across validations. As each + slot frees, dispatch the next queued validation immediately. + + Skip validation for any audit that failed in the audit dispatch phase. + Each validation writes to ${{ context.audit_run_dir }}/: + - validated_report_{source}_{timestamp}.md (first line: "validated: true") + - contested_findings_{source}_{timestamp}.md (only if contested > 0) + - validation_summary_{source}_{timestamp}.md (always written) + - ticket_body_{source}_{N}_{timestamp}.md (one per ticket group) + + After all complete, discover the validated_report_*.md files from session + output. Record paths for the issue creation wave. + + If any validation fails, note it but continue with the others. + Route to escalate_stop ONLY if all six validations fail. + capture_list: + validated_report_paths: "${{ result.validated_report_path }}" + on_success: create_issues + on_failure: escalate_stop + retries: 1 + on_exhausted: escalate_stop + + create_issues: + tool: run_python + description: "Batch-create GitHub issues via GraphQL from validated ticket body files" + with: + callable: "autoskillit.recipe._cmd_rpc.batch_create_issues" + workspace: "${{ inputs.workspace }}" + audit_run_dir: "${{ context.audit_run_dir }}" + timeout: 120 + note: > + BATCHED ISSUE CREATION via GraphQL — The callable discovers all + ticket_body_{source}_{N}_{ts}.md files in the validate-audit temp directory, + parses titles and bodies, includes validation summaries, resolves repository + and label IDs, and executes a single batched createIssue mutation with aliases. + Chunks at 20 issues per request if needed. + + No headless sessions are spawned. Rate-limit cost: 5 pts per chunk + (vs 90 pts for 18 sequential REST calls). + capture: + issue_urls: "${{ result.issue_urls }}" + issue_count: "${{ result.issue_count }}" + on_success: done + on_failure: escalate_stop + retries: 1 + on_exhausted: escalate_stop + + done: + action: stop + message: >- + Full audit complete. GitHub issues created from validated audit reports. + Emit the L3 result sentinel. context.issue_urls is a comma-separated + string of the created issue URLs — include it directly in the sentinel JSON: + {"success": true, "reason": "completed", "summary": "Full audit complete. + GitHub issues created.", "issue_urls": "${{ context.issue_urls }}"} + Artifacts are in {{AUTOSKILLIT_TEMP}}/. + + escalate_stop: + action: stop + message: >- + Full audit pipeline failed — one or more audit chains could not complete. + Check session logs and {{AUTOSKILLIT_TEMP}}/ directories for partial results. + Individual chain failures are noted in the session output above. diff --git a/src/autoskillit/recipes/implement-findings.json b/src/autoskillit/recipes/implement-findings.json new file mode 100644 index 0000000000..e1e1652eee --- /dev/null +++ b/src/autoskillit/recipes/implement-findings.json @@ -0,0 +1,180 @@ +{ + "name": "implement-findings", + "description": "Multi-issue L2 orchestration recipe for promote-to-main campaign. Processes a batch of audit findings through the full implementation or remediation lifecycle. Issues within the same BEM group run in parallel via wavefront scheduling; groups run sequentially.", + "kind": "standard", + "recipe_version": "1.0.0", + "requires_packs": [ + "github", + "clone", + "telemetry" + ], + "ingredients": { + "issue_urls": { + "description": "Comma-separated GitHub issue URLs to process", + "required": true + }, + "base_branch": { + "description": "Target branch for implementation PRs", + "default": "develop" + }, + "source_dir": { + "description": "Source repository path", + "required": false, + "default": "" + }, + "execution_map": { + "description": "Path to pre-computed BEM execution map JSON. When provided (campaign mode), skips internal BEM computation. When empty (standalone mode), runs /autoskillit:build-execution-map internally.", + "required": false, + "default": "", + "hidden": true + }, + "max_parallel": { + "description": "Max issues to run in parallel within a group (default suits 200K-context models)", + "default": "6", + "hidden": true + }, + "model_context_window": { + "description": "Model context window in tokens for dynamic issue-cap calculation", + "default": "200000", + "hidden": true + } + }, + "kitchen_rules": [ + "NEVER use native Claude Code tools (Read, Grep, Glob, Edit, Write, Bash, Agent, WebFetch, WebSearch, NotebookEdit) from the orchestrator. All work is delegated through run_skill.", + "Route to on_failure — never investigate or fix directly from the orchestrator.", + "Issues within a parallel group MUST complete all fast MCP steps (clone_repo, create_branch) for ALL issues before launching any run_skill calls. Wait for all fast steps to finish, then launch run_skill calls for all issues simultaneously.", + "Per-issue failures are non-fatal. Write the sidecar entry with status:failed, call release_issue, and continue to the next issue without stopping the batch.", + "After all groups complete, call batch_cleanup_clones once with all clone registry paths from the batch.", + "SOURCE ISOLATION: After clone_repo returns, the source_dir is strictly off-limits. Never run any command in source_dir — no git checkout, git status, or any other git operation. All implementation work must happen inside the cloned directory returned by clone_repo." + ], + "steps": { + "check_resume": { + "tool": "run_python", + "with": { + "callable": "autoskillit.fleet._findings_rpc.parse_and_resume", + "issue_urls_csv": "${{ inputs.issue_urls }}", + "project_dir": "${{ inputs.source_dir }}" + }, + "capture": { + "remaining_urls_json": "${{ result.remaining_urls_json }}", + "completed_count": "${{ result.completed_count }}" + }, + "on_success": "route_bem_mode", + "on_failure": "escalate_stop", + "note": "Reads AUTOSKILLIT_DISPATCH_ID from env (set by execute_dispatch). Parses comma-separated issue_urls, checks the per-dispatch sidecar at {{AUTOSKILLIT_TEMP}}/dispatches/{dispatch_id}_issues.jsonl, and returns only the URLs not yet completed or failed. On resume, already-processed issues are excluded automatically.\n" + }, + "route_bem_mode": { + "action": "route", + "on_result": [ + { + "when": "${{ inputs.execution_map }} != ''", + "route": "set_bem_path_from_ingredient" + }, + { + "route": "run_bem_internally" + } + ], + "note": "Campaign mode: execution_map ingredient is pre-computed by the L3 fleet dispatcher — skip BEM computation, read the provided file path directly. Standalone mode: execution_map is empty — run build-execution-map skill to compute groups from scratch.\n" + }, + "set_bem_path_from_ingredient": { + "tool": "run_cmd", + "with": { + "cmd": "printf '%s' '${{ inputs.execution_map }}'", + "cwd": "${{ inputs.source_dir }}", + "step_name": "set_bem_path_from_ingredient" + }, + "capture": { + "execution_map_path": "${{ result.stdout | trim }}" + }, + "on_success": "load_bem_from_file", + "on_failure": "escalate_stop", + "note": "Campaign mode only. Captures execution_map ingredient into context variable execution_map_path so load_bem_from_file can use a single source for both modes.\n" + }, + "run_bem_internally": { + "tool": "run_skill", + "model": "", + "with": { + "skill_command": "/autoskillit:build-execution-map ${{ context.remaining_urls_json }} ${{ inputs.base_branch }}", + "cwd": "${{ inputs.source_dir }}", + "step_name": "run_bem_internally" + }, + "capture": { + "execution_map_path": "${{ result.execution_map }}" + }, + "on_success": "load_bem_from_file", + "on_failure": "escalate_stop", + "on_context_limit": "escalate_stop", + "note": "Standalone mode only. Runs build-execution-map skill to compute the BEM groups from the remaining issue URLs. Captures the output map file path as execution_map_path for load_bem_from_file.\n" + }, + "load_bem_from_file": { + "tool": "run_python", + "with": { + "callable": "autoskillit.fleet._findings_rpc.load_execution_map", + "map_path": "${{ context.execution_map_path }}" + }, + "capture": { + "groups_json": "${{ result.groups_json }}", + "total_groups": "${{ result.total_groups }}" + }, + "on_success": "claim_all_issues", + "on_failure": "escalate_stop", + "note": "Reads the BEM JSON file and extracts the groups array. Each group has: name, parallel (bool), issues (list of URLs). Captured as groups_json (JSON string) and total_groups (count string).\n" + }, + "claim_all_issues": { + "tool": "run_cmd", + "with": { + "cmd": "echo 'claiming_all=${{ context.remaining_urls_json }} completed_already=${{ context.completed_count }} total_groups=${{ context.total_groups }} groups=${{ context.groups_json }}'", + "cwd": "${{ inputs.source_dir }}", + "step_name": "claim_all_issues" + }, + "on_success": "process_groups", + "on_failure": "escalate_stop", + "note": "Upfront claim: call claim_issue for EACH URL in context.remaining_urls_json. Use allow_reentry: 'true' to handle the resume case where issues were already claimed in a prior run. Do this before beginning group processing — this is the process-issues pattern that prevents concurrent claim conflicts. The run_cmd here serves as an anchor step. Make all claim_issue calls in the same turn before routing to process_groups.\n" + }, + "process_groups": { + "tool": "run_cmd", + "with": { + "cmd": "printf 'groups_total=%s\n' '${{ context.total_groups }}'", + "cwd": "${{ inputs.source_dir }}", + "step_name": "process_groups" + }, + "on_success": "next_or_done", + "on_failure": "escalate_stop", + "note": "GROUP LOOP — the agent maintains a group queue from context.groups_json. For each group (process in order, one group at a time):\n1. Parse the group's issue URLs and parallel flag.\n2. For each issue in the group, fetch its GitHub labels via\n fetch_github_issue to determine recipe type:\n - Label \"recipe:implementation\" -> use implementation recipe steps\n - Label \"recipe:remediation\" -> use remediation recipe steps\n\n3. WAVEFRONT SCHEDULING (for parallel: true groups):\n - First turn: call clone_repo for ALL issues simultaneously.\n Wait for all clones to complete.\n - Second turn: call create_branch + push_merge_target for ALL\n issues simultaneously. Wait for all to complete.\n - Third turn: launch run_skill for ALL issues simultaneously\n (implementation or remediation skill session per issue).\n For sequential groups (parallel: false): process one issue at a time.\n\n4. After each issue completes (success or failure):\n - Success: call run_python with callable\n \"autoskillit.fleet._sidecar_rpc.write_sidecar_entry\",\n args: dispatch_id=$AUTOSKILLIT_DISPATCH_ID, issue_url=,\n status=\"completed\", pr_url=\n - Failure: call run_python with callable\n \"autoskillit.fleet._sidecar_rpc.write_sidecar_entry\",\n args: status=\"failed\", reason=.\n Then call release_issue for that URL to remove the in-progress label.\n Continue to next issue — do NOT stop the batch.\n\n5. After all issues in the current group complete, advance to the\n next group. When all groups are done, route to next_or_done\n with result.next = \"all_done\".\n" + }, + "next_or_done": { + "action": "route", + "on_result": [ + { + "when": "${{ result.next }} == more_groups", + "route": "process_groups" + }, + { + "when": "${{ result.next }} == all_done", + "route": "batch_cleanup" + }, + { + "route": "batch_cleanup" + } + ], + "note": "Routing step — no tool call. The agent evaluates whether more groups remain in the queue (result.next = more_groups) or all groups have been processed (route to batch_cleanup).\n" + }, + "batch_cleanup": { + "tool": "batch_cleanup_clones", + "with": { + "step_name": "batch_cleanup" + }, + "on_success": "done", + "on_failure": "done", + "note": "Deletes all success-status clones registered during this batch run. Preserves failure-status clones for post-mortem inspection. Always routes to done regardless of outcome — cleanup failures are non-fatal.\n" + }, + "done": { + "action": "stop", + "message": "implement-findings batch complete. All groups processed. See sidecar for per-issue outcomes." + }, + "escalate_stop": { + "action": "stop", + "message": "implement-findings batch failed at setup. Manual investigation required. Check the step that routed here for root cause." + } + } +} diff --git a/src/autoskillit/recipes/implement-findings.yaml b/src/autoskillit/recipes/implement-findings.yaml new file mode 100644 index 0000000000..2950c7354d --- /dev/null +++ b/src/autoskillit/recipes/implement-findings.yaml @@ -0,0 +1,234 @@ +name: implement-findings +description: >- + Multi-issue L2 orchestration recipe for promote-to-main campaign. Processes + a batch of audit findings through the full implementation or remediation + lifecycle. Issues within the same BEM group run in parallel via wavefront + scheduling; groups run sequentially. +kind: standard +recipe_version: "1.0.0" +requires_packs: + - github + - clone + - telemetry + +ingredients: + issue_urls: + description: "Comma-separated GitHub issue URLs to process" + required: true + base_branch: + description: "Target branch for implementation PRs" + default: "develop" + source_dir: + description: "Source repository path" + required: false + default: "" + execution_map: + description: >- + Path to pre-computed BEM execution map JSON. When provided (campaign + mode), skips internal BEM computation. When empty (standalone mode), + runs /autoskillit:build-execution-map internally. + required: false + default: "" + hidden: true + max_parallel: + description: "Max issues to run in parallel within a group (default suits 200K-context models)" + default: "6" + hidden: true + model_context_window: + description: "Model context window in tokens for dynamic issue-cap calculation" + default: "200000" + hidden: true + +kitchen_rules: + - "NEVER use native Claude Code tools (Read, Grep, Glob, Edit, Write, + Bash, Agent, WebFetch, WebSearch, NotebookEdit) from the + orchestrator. All work is delegated through run_skill." + - "Route to on_failure — never investigate or fix directly from the orchestrator." + - "Issues within a parallel group MUST complete all fast MCP steps + (clone_repo, create_branch) for ALL issues before launching any run_skill + calls. Wait for all fast steps to finish, then launch run_skill calls for + all issues simultaneously." + - "Per-issue failures are non-fatal. Write the sidecar entry with + status:failed, call release_issue, and continue to the next issue without + stopping the batch." + - "After all groups complete, call batch_cleanup_clones once with all clone + registry paths from the batch." + - "SOURCE ISOLATION: After clone_repo returns, the source_dir is strictly + off-limits. Never run any command in source_dir — no git checkout, git + status, or any other git operation. All implementation work must happen + inside the cloned directory returned by clone_repo." + +steps: + check_resume: + tool: run_python + with: + callable: "autoskillit.fleet._findings_rpc.parse_and_resume" + issue_urls_csv: "${{ inputs.issue_urls }}" + project_dir: "${{ inputs.source_dir }}" + capture: + remaining_urls_json: "${{ result.remaining_urls_json }}" + completed_count: "${{ result.completed_count }}" + on_success: route_bem_mode + on_failure: escalate_stop + note: > + Reads AUTOSKILLIT_DISPATCH_ID from env (set by execute_dispatch). + Parses comma-separated issue_urls, checks the per-dispatch sidecar at + {{AUTOSKILLIT_TEMP}}/dispatches/{dispatch_id}_issues.jsonl, and returns + only the URLs not yet completed or failed. On resume, already-processed + issues are excluded automatically. + + route_bem_mode: + action: route + on_result: + - when: "${{ inputs.execution_map }} != ''" + route: set_bem_path_from_ingredient + - route: run_bem_internally + note: > + Campaign mode: execution_map ingredient is pre-computed by the L3 fleet + dispatcher — skip BEM computation, read the provided file path directly. + Standalone mode: execution_map is empty — run build-execution-map skill + to compute groups from scratch. + + set_bem_path_from_ingredient: + tool: run_cmd + with: + cmd: "printf '%s' '${{ inputs.execution_map }}'" + cwd: "${{ inputs.source_dir }}" + step_name: "set_bem_path_from_ingredient" + capture: + execution_map_path: "${{ result.stdout | trim }}" + on_success: load_bem_from_file + on_failure: escalate_stop + note: > + Campaign mode only. Captures execution_map ingredient into context + variable execution_map_path so load_bem_from_file can use a single + source for both modes. + + run_bem_internally: + tool: run_skill + model: "" + with: + skill_command: "/autoskillit:build-execution-map ${{ context.remaining_urls_json }} ${{ inputs.base_branch }}" + cwd: "${{ inputs.source_dir }}" + step_name: "run_bem_internally" + capture: + execution_map_path: "${{ result.execution_map }}" + on_success: load_bem_from_file + on_failure: escalate_stop + on_context_limit: escalate_stop + note: > + Standalone mode only. Runs build-execution-map skill to compute the + BEM groups from the remaining issue URLs. Captures the output map + file path as execution_map_path for load_bem_from_file. + + load_bem_from_file: + tool: run_python + with: + callable: "autoskillit.fleet._findings_rpc.load_execution_map" + map_path: "${{ context.execution_map_path }}" + capture: + groups_json: "${{ result.groups_json }}" + total_groups: "${{ result.total_groups }}" + on_success: claim_all_issues + on_failure: escalate_stop + note: > + Reads the BEM JSON file and extracts the groups array. Each group + has: name, parallel (bool), issues (list of URLs). Captured as + groups_json (JSON string) and total_groups (count string). + + claim_all_issues: + tool: run_cmd + with: + cmd: "echo 'claiming_all=${{ context.remaining_urls_json }} completed_already=${{ context.completed_count }} total_groups=${{ context.total_groups }} groups=${{ context.groups_json }}'" + cwd: "${{ inputs.source_dir }}" + step_name: "claim_all_issues" + on_success: process_groups + on_failure: escalate_stop + note: > + Upfront claim: call claim_issue for EACH URL in + context.remaining_urls_json. Use allow_reentry: 'true' to handle + the resume case where issues were already claimed in a prior run. + Do this before beginning group processing — this is the process-issues + pattern that prevents concurrent claim conflicts. + The run_cmd here serves as an anchor step. Make all claim_issue calls + in the same turn before routing to process_groups. + + process_groups: + tool: run_cmd + with: + cmd: "printf 'groups_total=%s\n' '${{ context.total_groups }}'" + cwd: "${{ inputs.source_dir }}" + step_name: "process_groups" + on_success: next_or_done + on_failure: escalate_stop + note: > + GROUP LOOP — the agent maintains a group queue from context.groups_json. + For each group (process in order, one group at a time): + + 1. Parse the group's issue URLs and parallel flag. + + 2. For each issue in the group, fetch its GitHub labels via + fetch_github_issue to determine recipe type: + - Label "recipe:implementation" -> use implementation recipe steps + - Label "recipe:remediation" -> use remediation recipe steps + + 3. WAVEFRONT SCHEDULING (for parallel: true groups): + - First turn: call clone_repo for ALL issues simultaneously. + Wait for all clones to complete. + - Second turn: call create_branch + push_merge_target for ALL + issues simultaneously. Wait for all to complete. + - Third turn: launch run_skill for ALL issues simultaneously + (implementation or remediation skill session per issue). + For sequential groups (parallel: false): process one issue at a time. + + 4. After each issue completes (success or failure): + - Success: call run_python with callable + "autoskillit.fleet._sidecar_rpc.write_sidecar_entry", + args: dispatch_id=$AUTOSKILLIT_DISPATCH_ID, issue_url=, + status="completed", pr_url= + - Failure: call run_python with callable + "autoskillit.fleet._sidecar_rpc.write_sidecar_entry", + args: status="failed", reason=. + Then call release_issue for that URL to remove the in-progress label. + Continue to next issue — do NOT stop the batch. + + 5. After all issues in the current group complete, advance to the + next group. When all groups are done, route to next_or_done + with result.next = "all_done". + + next_or_done: + action: route + on_result: + - when: "${{ result.next }} == more_groups" + route: process_groups + - when: "${{ result.next }} == all_done" + route: batch_cleanup + - route: batch_cleanup + note: > + Routing step — no tool call. The agent evaluates whether more groups + remain in the queue (result.next = more_groups) or all groups have + been processed (route to batch_cleanup). + + batch_cleanup: + tool: batch_cleanup_clones + with: + step_name: "batch_cleanup" + on_success: done + on_failure: done + note: > + Deletes all success-status clones registered during this batch run. + Preserves failure-status clones for post-mortem inspection. + Always routes to done regardless of outcome — cleanup failures are + non-fatal. + + done: + action: stop + message: >- + implement-findings batch complete. All groups processed. + See sidecar for per-issue outcomes. + + escalate_stop: + action: stop + message: >- + implement-findings batch failed at setup. Manual investigation required. + Check the step that routed here for root cause. diff --git a/src/autoskillit/recipes/implementation-groups.json b/src/autoskillit/recipes/implementation-groups.json new file mode 100644 index 0000000000..50a2674e48 --- /dev/null +++ b/src/autoskillit/recipes/implementation-groups.json @@ -0,0 +1,1787 @@ +{ + "name": "implementation-groups", + "description": "Decompose a source document into sequenced implementation groups, then plan, verify, implement, test, and merge each group end-to-end. Use when you have a large document or roadmap to implement via make-groups.", + "summary": "bootstrap_clone > (claim_and_resolve?) > (create_and_publish?) > make-groups > make-plan > (review-approach?) > dry-walkthrough > implement > test > merge (per group, per plan part) > (audit-impl?) > (open_pr?) > push > (review_pr?) > (ci_watch?) > cleanup", + "recipe_version": "1.0.0", + "requires_packs": [ + "github", + "ci", + "clone", + "telemetry" + ], + "ingredients": { + "source_doc": { + "description": "Source document for group decomposition", + "required": true + }, + "source_dir": { + "description": "Source repository", + "required": false, + "default": "" + }, + "run_name": { + "description": "Run name prefix", + "default": "impl" + }, + "base_branch": { + "description": "Merge target", + "default": "" + }, + "review_approach": { + "description": "Research approaches first", + "default": "false" + }, + "audit": { + "description": "Post-merge quality gate", + "default": "false" + }, + "open_pr": { + "description": "PR instead of direct merge", + "default": "true" + }, + "arch_lenses": { + "description": "Generate architecture lens diagrams for the PR", + "default": "false" + }, + "auto_merge": { + "description": "Automatically merge the PR after required checks pass — via merge queue when available, direct merge otherwise", + "default": "true" + }, + "review_max_retries": { + "description": "Maximum number of review-resolve retry cycles before proceeding to CI", + "default": "3" + }, + "local_review_rounds": { + "description": "Number of review-resolve cycles to run locally before switching to GitHub-backed review. 0 disables local mode entirely.", + "type": "integer", + "default": "3" + }, + "issue_url": { + "description": "GitHub issue to close on merge", + "required": false + }, + "upfront_claimed": { + "description": "Set to 'true' when process-issues has already claimed this issue upfront. Allows claim_issue defense step to pass through without aborting.", + "default": "false", + "hidden": true + } + }, + "kitchen_rules": [ + "NEVER use native Claude Code tools (Read, Grep, Glob, Edit, Write, Bash, Agent, WebFetch, WebSearch, NotebookEdit) from the orchestrator. All work is delegated through run_skill.", + "Route to on_failure — never investigate or fix directly from the orchestrator.", + "source_doc is required — it provides the work via group decomposition.", + "Process plan parts and groups sequentially. Complete the full cycle (verify → implement → test → merge) for one part before starting the next. Do NOT run verify for all parts upfront.", + "By default (open_pr=true), a feature branch is created with a unique name derived from inputs.run_name and context.issue_number (e.g. impl/124) or a date suffix (e.g. impl/20260304) when no issue is available. All worktree merges target the feature branch (context.merge_target), not base_branch directly. The push step publishes the feature branch, then prepare_pr → run_arch_lenses → compose_pr opens a PR to base_branch. When open_pr=false, merges target base_branch directly and prepare_pr is skipped.", + "When arch_lenses=true (and open_pr=true), prepare_pr routes through run_arch_lenses before compose_pr, adding architecture lens diagrams to the PR. By default arch_lenses is off and prepare_pr routes directly to compose_pr.", + "SOURCE ISOLATION: After bootstrap_clone returns, the source_dir is strictly off-limits. Never run any command in source_dir — no git checkout, git fetch, git reset, git pull, run_cmd, run_skill, or any other operation. All work — skill invocations, git operations, file reads — happens exclusively in the clone (work_dir). source_dir is used ONLY to read the remote URL inside push_to_remote." + ], + "steps": { + "clone": { + "tool": "bootstrap_clone", + "with": { + "source_dir": "${{ inputs.source_dir }}", + "run_name": "${{ inputs.run_name }}", + "base_branch": "${{ inputs.base_branch }}", + "step_name": "bootstrap_clone" + }, + "capture": { + "work_dir": "${{ result.work_dir }}", + "remote_url": "${{ result.remote_url }}", + "base_sha": "${{ result.base_sha }}", + "merge_target": "${{ result.merge_target }}" + }, + "on_success": "claim_and_resolve", + "on_failure": "escalate_stop", + "note": "Clones source_dir and captures base_sha + merge_target in one turn. Returns work_dir, remote_url, base_sha (SHA of base_branch before any merge), and merge_target (set to base_branch as fallback; overridden by create_and_publish when open_pr=true). On failure, escalate_stop immediately (no clone to remove).\n" + }, + "claim_and_resolve": { + "tool": "claim_and_resolve_issue", + "optional": true, + "skip_when_false": "inputs.issue_url", + "with": { + "issue_url": "${{ inputs.issue_url }}", + "allow_reentry": "${{ inputs.upfront_claimed }}" + }, + "capture": { + "issue_number": "${{ result.issue_number }}", + "issue_title": "${{ result.issue_title }}", + "issue_slug": "${{ result.issue_slug }}", + "claimed": "${{ result.claimed }}" + }, + "on_result": [ + { + "when": "${{ result.claimed }} == true", + "route": "create_and_publish" + }, + { + "route": "escalate_stop" + } + ], + "on_failure": "escalate_stop", + "note": "Fetches issue title/slug and claims the issue in one turn. Returns issue_number, issue_title, issue_slug, and claimed boolean. When claimed=false (another session owns it), routes to escalate_stop. issue_slug and issue_number are passed to create_and_publish for branch naming.\n" + }, + "create_and_publish": { + "tool": "create_and_publish_branch", + "optional": true, + "skip_when_false": "inputs.open_pr", + "with": { + "issue_slug": "${{ context.issue_slug }}", + "run_name": "${{ inputs.run_name }}", + "issue_number": "${{ context.issue_number }}", + "work_dir": "${{ context.work_dir }}", + "remote_url": "${{ context.remote_url }}", + "step_name": "create_and_publish_branch" + }, + "capture": { + "merge_target": "${{ result.merge_target }}" + }, + "on_success": "group", + "on_failure": "release_issue_failure", + "note": "Computes branch name, creates it (with collision suffix if needed), and pushes to origin in one turn. Overrides merge_target with the feature branch name. merge_worktree requires refs/remotes/origin/{branch} to exist after fetch (step 5.5). When open_pr=false, this step is skipped and merge_target remains as base_branch.\n" + }, + "group": { + "tool": "run_skill", + "model": "", + "with": { + "skill_command": "/autoskillit:make-groups ${{ inputs.source_doc }}", + "cwd": "${{ context.work_dir }}", + "step_name": "group" + }, + "capture": { + "group_files": "${{ result.group_files }}" + }, + "on_success": "plan", + "on_failure": "release_issue_failure", + "note": "Always runs — groups are the primary workflow in this recipe. Produces group files in {{AUTOSKILLIT_TEMP}}/make-groups/ containing sequenced groups (groupA, groupB, ...). GROUPS MODE: Pass the file paths (group_files) directly to the plan step via context.group_files — do NOT read the files. The current group's file path becomes the task passed to the plan step. ITERATION: The agent maintains a group queue. For each group, run the full cycle (plan → review → verify → implement → test → merge) before advancing to the next group.\n" + }, + "plan": { + "tool": "run_skill", + "model": "", + "with": { + "skill_command": "/autoskillit:make-plan CURRENT_GROUP_FILE", + "cwd": "${{ context.work_dir }}", + "step_name": "plan", + "group_files": "${{ context.group_files }}", + "issue_url": "${{ inputs.issue_url }}", + "issue_title": "${{ context.issue_title }}" + }, + "capture": { + "plan_path": "${{ result.plan_path }}", + "all_plan_paths": "${{ result.plan_path }}" + }, + "capture_list": { + "plan_parts": "${{ result.plan_parts }}" + }, + "on_success": "review", + "on_failure": "release_issue_failure", + "note": "Produces plan in {{AUTOSKILLIT_TEMP}}/make-plan/. Glob plan_dir for *_part_*.md or single plan file. plan_parts context key contains the full ordered list of part files (sorted alphabetically). For a single-part plan, plan_parts has one entry equal to plan_path. GROUPS MODE: Replace the skill_command task with the current group's file path from context.group_files extracted from the group step's capture — do NOT read the file; pass the path directly to the skill. REMEDIATION MODE: context.remediation_path is available when the orchestrator re-enters plan from the remediate step; pass it inline in the skill_command rather than as a named with: key. ACCUMULATION: After each plan step execution in groups mode, the agent MUST append the captured plan_path to context.all_plan_paths using a comma separator instead of letting the capture block overwrite the accumulated value. For multi-group runs, the accumulated value looks like \"/path/groupA_plan.md,/path/groupB_plan.md\". ISSUE CONTEXT: If inputs.issue_url is a non-empty string, pass it to the skill so the make-plan skill can call fetch_github_issue internally to get full issue content: \"/autoskillit:make-plan {group_path}\\n\\nGitHub Issue: {inputs.issue_url}\"\n" + }, + "review": { + "tool": "run_skill", + "model": "", + "with": { + "skill_command": "/autoskillit:review-approach ${{ context.plan_path }}", + "cwd": "${{ context.work_dir }}", + "step_name": "review" + }, + "capture": { + "review_path": "${{ result.review_path }}" + }, + "retries": 1, + "on_success": "verify", + "on_failure": "release_issue_failure", + "on_context_limit": "verify", + "skip_when_false": "inputs.review_approach", + "note": "Only execute this step if review_approach input is 'true'. Otherwise skip directly to verify." + }, + "verify": { + "tool": "run_skill", + "model": "", + "with": { + "skill_command": "/autoskillit:dry-walkthrough ${{ context.plan_path }}", + "cwd": "${{ context.work_dir }}", + "step_name": "verify", + "review_path": "${{ context.review_path }}" + }, + "on_success": "implement", + "on_failure": "release_issue_failure", + "note": "SEQUENTIAL EXECUTION: For each plan_part, run the FULL cycle — verify → implement → test → merge — for that part before advancing to the next part. Never run verify for all parts first. If review produced a report, context.review_path is available — pass it as an additional arg to dry-walkthrough." + }, + "implement": { + "tool": "run_skill", + "model": "", + "stale_threshold": 2400, + "with": { + "skill_command": "/autoskillit:implement-worktree-no-merge ${{ context.plan_path }}", + "cwd": "${{ context.work_dir }}", + "step_name": "implement" + }, + "capture": { + "worktree_path": "${{ result.worktree_path }}" + }, + "retries": 0, + "on_context_limit": "retry_worktree", + "on_success": "test", + "on_failure": "release_issue_failure", + "note": "Extracts worktree_path from result for use in subsequent steps." + }, + "retry_worktree": { + "tool": "run_skill", + "model": "", + "stale_threshold": 2400, + "with": { + "skill_command": "/autoskillit:retry-worktree ${{ context.plan_path }} ${{ context.worktree_path }}", + "cwd": "${{ context.worktree_path }}", + "step_name": "retry_worktree" + }, + "capture": { + "worktree_path": "${{ result.worktree_path }}", + "phases_implemented": "${{ result.phases_implemented }}" + }, + "on_result": [ + { + "when": "${{ result.phases_implemented }} > 0", + "route": "test" + }, + { + "route": "test" + } + ], + "on_exhausted": "release_issue_failure", + "on_failure": "release_issue_failure" + }, + "test": { + "tool": "test_check", + "with": { + "worktree_path": "${{ context.worktree_path }}" + }, + "on_success": "check_merge_fix_loop", + "on_failure": "fix" + }, + "check_merge_fix_loop": { + "tool": "run_python", + "with": { + "callable": "autoskillit.smoke_utils.check_loop_iteration", + "current_iteration": "${{ context.merge_fix_count }}", + "max_iterations": "3" + }, + "capture": { + "merge_fix_count": "${{ result.next_iteration }}" + }, + "on_result": [ + { + "when": "${{ result.max_exceeded }} == true", + "route": "release_issue_failure" + }, + { + "route": "commit_guard" + } + ], + "on_failure": "release_issue_failure", + "optional_context_refs": [ + "merge_fix_count" + ], + "note": "Iteration guard for the merge→fix→test cycle. Caps the loop at 3 iterations (enough for legitimate flake recovery but not enough to burn 30+ minutes on structurally unresolvable conflicts).\n" + }, + "commit_guard": { + "tool": "run_python", + "with": { + "callable": "autoskillit.recipe._cmd_rpc.commit_guard", + "worktree_path": "${{ context.worktree_path }}" + }, + "on_success": "merge", + "on_failure": "merge", + "note": "Belt-and-suspenders guard: auto-commits any uncommitted changes before merge. Catches dirty state from context-exhausted skills, cascading pre-commit auto-fixes, or any other source. on_failure routes to merge (dirty-tree gate will catch real issues).\n" + }, + "merge": { + "tool": "merge_worktree", + "with": { + "worktree_path": "${{ context.worktree_path }}", + "base_branch": "${{ context.merge_target }}" + }, + "capture": { + "cleanup_succeeded": "${{ result.cleanup_succeeded }}", + "worktree_path": "${{ result.worktree_path }}" + }, + "on_result": [ + { + "when": "result.failed_step == 'dirty_tree'", + "route": "fix" + }, + { + "when": "result.failed_step == 'test_gate'", + "route": "fix" + }, + { + "when": "result.failed_step == 'post_rebase_test_gate'", + "route": "fix" + }, + { + "when": "result.failed_step == 'rebase'", + "route": "rebase_conflict_fix" + }, + { + "when": "result.error", + "route": "release_issue_failure" + }, + { + "route": "next_or_done" + } + ], + "on_failure": "release_issue_failure", + "note": "After merge, route to next_or_done to process remaining parts/groups before pushing. GROUPS MODE: When all plan_parts for the current group are merged, advance to the next group (back to plan step with the next group's description). When all groups are complete, go to audit_impl (via next_or_done). By default (open_pr=true), merge_target is the feature branch named from run_name. When open_pr=false, merge_target equals base_branch — merges go directly to base. test_gate failures route to fix (resolve-failures) for recovery; all other failures route to release_issue_failure.\n" + }, + "push": { + "tool": "push_to_remote", + "with": { + "clone_path": "${{ context.work_dir }}", + "remote_url": "${{ context.remote_url }}", + "branch": "${{ context.merge_target }}" + }, + "on_success": "prepare_pr", + "on_failure": "release_issue_failure", + "note": "Runs once at the end, after audit_impl passes. Pushes merge_target branch from the clone to the upstream remote. By default (open_pr=true), this pushes the feature branch; when open_pr=false, pushes base_branch. Uses the pre-resolved remote_url captured at clone time, preserving clone isolation. On success, routes to prepare_pr (which opens the PR by default; skipped when open_pr=false). On failure, release_issue_failure registers the clone and escalates.\n" + }, + "fix": { + "tool": "run_skill", + "model": "", + "with": { + "skill_command": "/autoskillit:resolve-failures ${{ context.worktree_path }} ${{ context.plan_path }} ${{ inputs.base_branch }}", + "cwd": "${{ context.worktree_path }}", + "step_name": "fix" + }, + "capture": { + "verdict": "${{ result.verdict }}", + "fixes_applied": "${{ result.fixes_applied }}" + }, + "on_result": [ + { + "when": "${{ result.verdict }} == 'real_fix'", + "route": "test" + }, + { + "when": "${{ result.verdict }} == 'already_green'", + "route": "test" + }, + { + "when": "${{ result.verdict }} == 'flake_suspected'", + "route": "test" + }, + { + "when": "${{ result.verdict }} == 'ci_only_failure'", + "route": "release_issue_failure" + }, + { + "route": "release_issue_failure" + } + ], + "on_failure": "release_issue_failure", + "on_context_limit": "test", + "note": "Fixes test failures without merging. Routes back to test for re-validation before merge_worktree. on_context_limit routes needs_retry=True to test (worktree may already be green) instead of re-running fix blindly." + }, + "rebase_conflict_fix": { + "tool": "run_skill", + "model": "", + "with": { + "skill_command": "/autoskillit:resolve-merge-conflicts ${{ context.worktree_path }} ${{ context.plan_path }} ${{ inputs.base_branch }}", + "cwd": "${{ context.worktree_path }}", + "step_name": "rebase_conflict_fix" + }, + "capture": { + "conflict_escalation_required": "${{ result.escalation_required }}" + }, + "on_result": [ + { + "when": "${{ result.escalation_required }} == true", + "route": "release_issue_failure" + }, + { + "route": "test" + } + ], + "on_failure": "release_issue_failure", + "on_context_limit": "release_issue_failure", + "retries": 1, + "on_exhausted": "release_issue_failure" + }, + "next_or_done": { + "action": "route", + "on_result": [ + { + "when": "${{ result.next }} == more_parts", + "route": "verify" + }, + { + "when": "${{ result.next }} == more_groups", + "route": "plan" + }, + { + "route": "audit_impl" + } + ], + "note": "Routing step — no tool call. The agent evaluates what comes next: 1. If more plan_parts remain for the current group → go to verify with next part. 2. If all plan_parts done but more groups remain → go to plan with next group's task. 3. If all groups complete → go to audit_impl.\n" + }, + "audit_impl": { + "tool": "run_skill", + "model": "", + "with": { + "skill_command": "/autoskillit:audit-impl ${{ context.plan_path }} ${{ context.base_sha }} ${{ inputs.base_branch }}", + "cwd": "${{ context.work_dir }}", + "step_name": "audit_impl" + }, + "capture": { + "remediation_path": "${{ result.remediation_path }}" + }, + "on_result": [ + { + "when": "${{ result.verdict }} == GO", + "route": "push" + }, + { + "when": "result.error", + "route": "escalate_stop" + }, + { + "route": "remediate" + } + ], + "on_failure": "escalate_stop", + "on_context_limit": "escalate_stop", + "optional": true, + "skip_when_false": "inputs.audit", + "note": "Only execute if inputs.audit is 'true'. If false, skip directly to push. Captures verdict and remediation_path. Routes GO → push, NO GO → remediate which re-enters the plan step with the remediation file. All groups and parts have been merged at this point — no worktree exists. Uses context.base_sha (a stable commit SHA captured before any merge began) as implementation_ref. A SHA names a git object and survives git branch -D unconditionally. The diff git diff base_sha..base_branch (two-dot, SHA on the left) covers the full implementation across all plan groups.\n" + }, + "remediate": { + "action": "route", + "with": { + "remediation_path": "${{ context.remediation_path }}" + }, + "on_success": "plan" + }, + "prepare_pr": { + "tool": "run_skill", + "model": "", + "with": { + "skill_command": "/autoskillit:prepare-pr ${{ context.all_plan_paths }} ${{ inputs.run_name }} ${{ inputs.base_branch }}", + "cwd": "${{ context.work_dir }}", + "step_name": "prepare_pr", + "issue_number": "${{ context.issue_number }}" + }, + "capture": { + "prep_path": "${{ result.prep_path }}", + "selected_lenses": "${{ result.selected_lenses }}", + "lens_context_paths": "${{ result.lens_context_paths }}" + }, + "on_result": [ + { + "when": "${{ result.prep_path }} and ${{ inputs.arch_lenses }} == 'true'", + "route": "run_arch_lenses" + }, + { + "when": "${{ result.prep_path }}", + "route": "compose_pr" + }, + { + "route": "release_issue_failure" + } + ], + "on_failure": "release_issue_failure", + "on_context_limit": "release_issue_failure", + "optional": true, + "skip_when_false": "inputs.open_pr", + "note": "Reads plan(s), runs git diff, classifies changed files, selects 1–3 arch-lens slugs, writes one PR context file per lens, and writes a PR prep file. Does NOT invoke arch-lens skills. If context.issue_number is a non-empty string, append it as the 4th positional argument so prepare-pr fetches requirements from the closing issue. MULTI-GROUP: context.all_plan_paths may contain comma-separated plan paths accumulated across group iterations.\n" + }, + "run_arch_lenses": { + "tool": "run_skill", + "model": "", + "with": { + "skill_command": "/autoskillit:arch-lens-{slug} {context_path}", + "cwd": "${{ context.work_dir }}", + "step_name": "run_arch_lenses" + }, + "capture_list": { + "all_diagram_paths": "${{ result.diagram_path }}" + }, + "on_success": "compose_pr", + "on_failure": "compose_pr", + "on_context_limit": "compose_pr", + "optional": true, + "skip_when_false": "inputs.open_pr", + "note": "LENS ITERATION: context.selected_lenses contains comma-separated arch-lens slugs (e.g. \"module-dependency,process-flow\"). context.lens_context_paths contains the corresponding comma-separated context file paths (same order as slugs). For each lens slug, run:\n /autoskillit:arch-lens-{slug} {matching_context_path}\nRun all selected lenses in parallel (all run_skill calls in a single message). Accumulate each diagram_path value into context.all_diagram_paths via capture_list. If a lens fails, continue with remaining lenses — compose_pr handles empty lists gracefully by omitting the Architecture Impact section.\n" + }, + "compose_pr": { + "tool": "run_skill", + "model": "", + "with": { + "skill_command": "/autoskillit:compose-pr ${{ context.prep_path }} \"${{ context.all_diagram_paths }}\" ${{ context.work_dir }} ${{ inputs.base_branch }}", + "cwd": "${{ context.work_dir }}", + "step_name": "compose_pr", + "issue_number": "${{ context.issue_number }}" + }, + "capture": { + "pr_url": "${{ result.pr_url }}" + }, + "on_success": "extract_pr_number", + "on_failure": "release_issue_failure", + "on_context_limit": "release_issue_failure", + "optional": true, + "skip_when_false": "inputs.open_pr", + "note": "Reads the PR prep file, validates arch-lens diagrams (must contain ★ or ●), composes the PR body, and runs gh pr create. Degrades gracefully if gh is unavailable (emits pr_url=). If context.issue_number is set, append it as the 5th positional argument for Closes #N injection in the PR body.\n" + }, + "extract_pr_number": { + "tool": "run_cmd", + "with": { + "cmd": "gh pr view '${{ context.merge_target }}' --json number -q .number", + "cwd": "${{ context.work_dir }}", + "step_name": "extract_pr_number" + }, + "capture": { + "pr_number": "${{ result.stdout | trim }}" + }, + "on_success": "annotate_pr_diff", + "on_failure": "release_issue_failure", + "optional": true, + "skip_when_false": "inputs.open_pr", + "note": "Extracts the integer PR number from the feature branch opened by compose_pr. pr_number is required by the queue finale steps (enqueue_to_queue, wait_for_queue).\n" + }, + "annotate_pr_diff": { + "tool": "run_python", + "with": { + "callable": "autoskillit.smoke_utils.annotate_pr_diff", + "pr_number": "${{ context.pr_number }}", + "cwd": "${{ context.work_dir }}", + "output_dir": "${{ context.work_dir }}/{{AUTOSKILLIT_TEMP}}/review-pr", + "base_branch": "${{ inputs.base_branch }}", + "local_review_rounds": "${{ inputs.local_review_rounds }}", + "current_iteration": "${{ context.review_loop_count }}" + }, + "capture": { + "annotated_diff_path": "${{ result.annotated_diff_path }}", + "hunk_ranges_path": "${{ result.hunk_ranges_path }}", + "diff_metrics_path": "${{ result.diff_metrics_path }}", + "review_mode": "${{ result.review_mode }}" + }, + "on_success": "review_pr", + "on_failure": "review_pr", + "optional": true, + "skip_when_false": "inputs.open_pr", + "optional_context_refs": [ + "review_loop_count" + ], + "note": "Pre-computes annotated diff with [LNNN] markers and hunk ranges JSON for review-pr. On failure, falls through to review_pr which uses empty fallback.\n" + }, + "review_pr": { + "tool": "run_skill", + "model": "", + "with": { + "skill_command": "/autoskillit:review-pr ${{ context.merge_target }} ${{ inputs.base_branch }} annotated_diff_path=${{ context.annotated_diff_path }} hunk_ranges_path=${{ context.hunk_ranges_path }} diff_metrics_path=${{ context.diff_metrics_path }} mode=${{ context.review_mode }}", + "cwd": "${{ context.work_dir }}", + "step_name": "review_pr" + }, + "capture": { + "review_verdict": "${{ result.verdict }}" + }, + "on_result": [ + { + "when": "${{ result.verdict }} == changes_requested", + "route": "resolve_review" + }, + { + "when": "${{ result.verdict }} == approved_with_comments", + "route": "resolve_review" + }, + { + "when": "${{ result.verdict }} == needs_human", + "route": "check_review_loop" + }, + { + "when": "${{ result.verdict }} == approved", + "route": "check_review_loop" + }, + { + "when": "true", + "route": "check_review_loop" + } + ], + "on_failure": "check_repo_ci_event", + "on_context_limit": "check_repo_ci_event", + "optional": true, + "skip_when_false": "inputs.open_pr", + "optional_context_refs": [ + "review_loop_count", + "review_mode" + ], + "note": "Runs after compose_pr creates the PR. Invokes the review-pr skill as a headless session to leave inline review comments on the PR. The skill finds the open PR by feature branch name (context.merge_target). Captures verdict as review_verdict and routes via on_result: changes_requested/approved_with_comments → resolve_review; all other verdicts (including approved, needs_human) → check_review_loop (mandatory waypoint to increment review_loop_count). On tool-level failure (MCP error, gh unavailable), routes to check_repo_ci_event (no review to resolve). Skipped when open_pr=false.\n" + }, + "resolve_review": { + "tool": "run_skill", + "model": "", + "with": { + "skill_command": "/autoskillit:resolve-review ${{ context.merge_target }} ${{ inputs.base_branch }} mode=${{ context.review_mode }}", + "cwd": "${{ context.work_dir }}", + "step_name": "resolve_review" + }, + "capture": { + "verdict": "${{ result.verdict }}" + }, + "retries": 2, + "on_exhausted": "release_issue_failure", + "on_result": [ + { + "when": "${{ result.verdict }} == 'real_fix'", + "route": "re_push_review" + }, + { + "when": "${{ result.verdict }} == 'already_green'", + "route": "re_push_review" + }, + { + "when": "${{ result.verdict }} == 'flake_suspected'", + "route": "re_push_review" + }, + { + "when": "${{ result.verdict }} == 'ci_only_failure'", + "route": "release_issue_failure" + } + ], + "on_failure": "release_issue_failure", + "optional_context_refs": [ + "review_mode" + ], + "note": "Runs when review_pr reports issues (verdict=changes_requested). The clone (context.work_dir) has the feature branch (context.merge_target) checked out. resolve-review fetches inline and summary review comments via the GitHub API, applies fixes for each actionable finding (critical + warning severity), and commits each fix. Routes via on_result: verdict dispatch — real_fix/already_green/ flake_suspected all route to re_push_review (retry bounded by retries: 2); ci_only_failure escalates to release_issue_failure.\n" + }, + "re_push_review": { + "tool": "push_to_remote", + "with": { + "clone_path": "${{ context.work_dir }}", + "remote_url": "${{ context.remote_url }}", + "branch": "${{ context.merge_target }}" + }, + "on_success": "check_review_loop", + "on_failure": "release_issue_failure", + "note": "Pushes the review-fixed feature branch back to the remote after resolve_review succeeds. Routes to check_review_loop to determine if another review-resolve cycle is needed (bounded by review_max_retries). On push failure, escalates via release_issue_failure.\n" + }, + "check_review_loop": { + "tool": "run_python", + "with": { + "callable": "autoskillit.smoke_utils.check_review_loop", + "pr_number": "${{ context.pr_number }}", + "current_iteration": "${{ context.review_loop_count }}", + "max_iterations": "${{ inputs.review_max_retries }}", + "previous_verdict": "${{ context.review_verdict }}", + "local_review_rounds": "${{ inputs.local_review_rounds }}" + }, + "capture": { + "review_loop_count": "${{ result.next_iteration }}" + }, + "on_result": [ + { + "when": "${{ result.had_blocking }} == true and ${{ result.max_exceeded }} == false", + "route": "review_pr" + }, + { + "route": "check_repo_ci_event" + } + ], + "on_failure": "check_repo_ci_event", + "skip_when_false": "inputs.open_pr", + "optional_context_refs": [ + "review_loop_count", + "review_verdict" + ], + "note": "Compound iteration + blocking guard. Routes back to review_pr only when had_blocking=true AND max_exceeded=false. had_blocking is true when previous_verdict=changes_requested OR (local_review_rounds > 0 AND iteration < local_review_rounds), ensuring local review rounds are fully exhausted before exiting to CI.\n" + }, + "check_repo_ci_event": { + "tool": "check_repo_merge_state", + "with": { + "branch": "${{ context.merge_target }}", + "cwd": "${{ context.work_dir }}", + "remote_url": "${{ context.remote_url }}", + "step_name": "check_repo_ci_event" + }, + "capture": { + "ci_event": "${{ result.ci_event }}" + }, + "on_success": "check_pr_state", + "on_failure": "ci_watch", + "skip_when_false": "inputs.open_pr", + "note": "Derives ci_event for the feature branch BEFORE ci_watch runs. Uses context.merge_target (feature branch) so push.branches filter is evaluated against the actual branch being pushed, not the base branch. Non-blocking: on_failure routes to ci_watch with ci_event=null (match-any).\n" + }, + "check_pr_state": { + "tool": "check_pr_mergeable", + "with": { + "pr_number": "${{ context.pr_number }}", + "cwd": "${{ context.work_dir }}" + }, + "on_result": [ + { + "when": "${{ result.mergeable_status }} == 'CONFLICTING'", + "route": "detect_ci_conflict" + }, + { + "route": "ci_watch" + } + ], + "on_failure": "ci_watch", + "skip_when_false": "inputs.open_pr", + "note": "Pre-CI-watch gate: checks PR mergeable state before entering the CI polling loop. CONFLICTING PRs cannot trigger pull_request workflows (GitHub cannot create refs/pull/N/merge), so routing to ci_watch would produce a guaranteed no_runs. Routes CONFLICTING to detect_ci_conflict (which handles conflict resolution) and MERGEABLE/UNKNOWN to ci_watch.\n" + }, + "ci_watch": { + "tool": "wait_for_ci", + "with": { + "branch": "${{ context.merge_target }}", + "event": "${{ context.ci_event }}", + "timeout_seconds": 600, + "cwd": "${{ context.work_dir }}", + "remote_url": "${{ context.remote_url }}", + "auto_trigger": "true" + }, + "capture": { + "ci_conclusion": "${{ result.conclusion }}", + "ci_failed_jobs": "${{ result.failed_jobs }}" + }, + "on_result": [ + { + "when": "${{ result.conclusion }} == 'success'", + "route": "check_repo_merge_state" + }, + { + "when": "${{ result.conclusion }} == 'no_runs'", + "route": "handle_no_ci_runs" + }, + { + "when": "${{ result.conclusion }} == 'timed_out'", + "route": "check_ci_timed_out_loop" + }, + { + "route": "detect_ci_conflict" + } + ], + "on_failure": "detect_ci_conflict", + "optional": true, + "skip_when_false": "inputs.open_pr", + "note": "Waits for the GitHub Actions CI run on context.merge_target to complete using the wait_for_ci MCP tool (timeout: 300 s). Captures structured output (conclusion + failed job names) for downstream resolve_ci. Skipped when open_pr=false — no remote feature branch is opened in that mode. Routes on result.conclusion: success → check_repo_merge_state, no_runs → handle_no_ci_runs (auto_trigger fires first: empty commit + force-push + re-poll; handle_no_ci_runs reached only when push fails), timed_out → check_ci_timed_out_loop (iteration guard), failure → detect_ci_conflict.\n" + }, + "check_ci_timed_out_loop": { + "tool": "run_python", + "with": { + "callable": "autoskillit.smoke_utils.check_loop_iteration", + "current_iteration": "${{ context.ci_timed_out_loop_count }}", + "max_iterations": "2" + }, + "capture": { + "ci_timed_out_loop_count": "${{ result.next_iteration }}" + }, + "on_result": [ + { + "when": "${{ result.max_exceeded }} == true", + "route": "detect_ci_conflict" + }, + { + "route": "ci_watch" + } + ], + "on_failure": "detect_ci_conflict", + "optional_context_refs": [ + "ci_timed_out_loop_count" + ], + "note": "Caps the ci_watch re-poll loop at 2 iterations. After 2 timed_out results (1200s total), routes to detect_ci_conflict instead of spinning indefinitely.\n" + }, + "handle_no_ci_runs": { + "tool": "check_repo_merge_state", + "with": { + "branch": "${{ context.merge_target }}", + "cwd": "${{ context.work_dir }}", + "remote_url": "${{ context.remote_url }}", + "step_name": "handle_no_ci_runs" + }, + "capture": { + "ci_event": "${{ result.ci_event }}" + }, + "on_success": "check_ci_loop", + "on_failure": "release_issue_failure", + "retries": 1, + "on_exhausted": "release_issue_failure", + "note": "Intercepts no_runs conclusion from wait_for_ci. Re-derives ci_event using check_repo_merge_state (the original event may have been wrong for this branch's workflow configuration) and routes to check_ci_loop for iteration guard before retrying ci_watch. retries: 1 bounds the tool-call retry; check_ci_loop bounds the outer loop at 2 iterations.\n" + }, + "check_ci_loop": { + "tool": "run_python", + "with": { + "callable": "autoskillit.smoke_utils.check_loop_iteration", + "current_iteration": "${{ context.ci_loop_count }}", + "max_iterations": "2" + }, + "capture": { + "ci_loop_count": "${{ result.next_iteration }}" + }, + "on_result": [ + { + "when": "${{ result.max_exceeded }} == true", + "route": "check_active_trigger_loop" + }, + { + "route": "ci_watch" + } + ], + "on_failure": "check_ci_already_passed", + "optional_context_refs": [ + "ci_loop_count" + ], + "note": "Iteration guard for the ci_watch / handle_no_ci_runs cycle. Caps the loop at 2 iterations (initial + 1 retry after re-deriving ci_event). On budget exhaustion, routes to check_active_trigger_loop which caps the active trigger to a single attempt before escalating.\n" + }, + "check_active_trigger_loop": { + "tool": "run_python", + "with": { + "callable": "autoskillit.smoke_utils.check_loop_iteration", + "current_iteration": "${{ context.active_trigger_count }}", + "max_iterations": "2" + }, + "capture": { + "active_trigger_count": "${{ result.next_iteration }}" + }, + "on_result": [ + { + "when": "${{ result.max_exceeded }} == true", + "route": "check_ci_already_passed" + }, + { + "route": "trigger_ci_actively" + } + ], + "on_failure": "check_ci_already_passed", + "optional_context_refs": [ + "active_trigger_count" + ], + "note": "Bounds the trigger_ci_actively remediation to a single attempt. If ci_watch still returns no_runs after the active trigger, escalates to escalate_stop_no_ci rather than re-triggering indefinitely.\n" + }, + "trigger_ci_actively": { + "tool": "run_cmd", + "with": { + "cmd": "gh pr close ${{ context.pr_number }} && sleep 2 && gh pr reopen ${{ context.pr_number }}", + "cwd": "${{ context.work_dir }}", + "step_name": "trigger_ci_actively" + }, + "on_success": "ci_watch", + "on_failure": "check_ci_already_passed", + "retries": 1, + "on_exhausted": "check_ci_already_passed", + "note": "Active CI trigger remediation: closes and reopens the PR to force GitHub webhook re-delivery. This re-creates the pull_request event and the refs/pull/N/merge ref (if the PR is mergeable). Routes back to ci_watch for one final attempt. If the close/reopen fails or the subsequent ci_watch still returns no_runs, escalates to escalate_stop_no_ci.\n" + }, + "check_ci_already_passed": { + "tool": "run_cmd", + "with": { + "cmd": "gh pr view '${{ context.pr_number }}' --json statusCheckRollup --jq '.statusCheckRollup | if length == 0 then \"no_checks\" elif all(.state == \"SUCCESS\" or .state == \"NEUTRAL\" or .state == \"SKIPPED\") then \"passed\" elif any(.state == \"FAILURE\" or .state == \"ERROR\") then \"failed\" else \"pending\" end'", + "cwd": "${{ context.work_dir }}", + "step_name": "check_ci_already_passed" + }, + "on_result": [ + { + "when": "${{ result.stdout | trim }} == passed", + "route": "check_repo_merge_state" + }, + { + "when": "${{ result.stdout | trim }} == failed", + "route": "detect_ci_conflict" + }, + { + "when": "${{ result.stdout | trim }} == pending", + "route": "ci_watch" + }, + { + "route": "mark_issue_failed_no_ci" + } + ], + "on_failure": "mark_issue_failed_no_ci", + "skip_when_false": "inputs.open_pr", + "note": "Safety net before terminal escalation: checks if CI already passed despite the watcher failing to observe it (e.g. post-force-push SHA staleness). Uses statusCheckRollup as the authoritative signal. Outputs passed/failed/no_checks/pending: passed goes to merge gate, failed routes to CI conflict remediation, pending re-enters the CI watcher, no_checks falls through to terminal stop.\n" + }, + "mark_issue_failed_no_ci": { + "description": "Swap in-progress → fail label before CI failure escalation", + "tool": "release_issue", + "optional": true, + "skip_when_false": "inputs.issue_url", + "with": { + "issue_url": "${{ inputs.issue_url }}", + "fail_label": "fail" + }, + "on_success": "register_clone_no_ci", + "on_failure": "register_clone_no_ci" + }, + "register_clone_no_ci": { + "description": "Register clone as preserved (no-CI error pipeline)", + "tool": "register_clone_status", + "with": { + "clone_path": "${{ context.work_dir }}", + "status": "error", + "step_name": "register_clone_no_ci" + }, + "on_success": "escalate_stop_no_ci", + "on_failure": "escalate_stop_no_ci" + }, + "escalate_stop_no_ci": { + "action": "stop", + "message": "CI safety net: statusCheckRollup returned no checks for PR ${{ context.pr_number }}. This typically means no workflow is configured to trigger on this branch pattern. Check .github/workflows/ trigger configuration. This is NOT a test failure — CI never started." + }, + "diagnose_ci": { + "tool": "run_skill", + "model": "", + "with": { + "skill_command": "/autoskillit:diagnose-ci ${{ context.merge_target }} - - - ${{ context.ci_event }}", + "cwd": "${{ context.work_dir }}", + "step_name": "diagnose_ci", + "ci_failed_jobs": "${{ context.ci_failed_jobs }}" + }, + "capture": { + "diagnosis_path": "${{ result.diagnosis_path }}" + }, + "on_success": "resolve_ci", + "on_failure": "resolve_ci", + "on_context_limit": "resolve_ci", + "optional": true, + "optional_context_refs": [ + "ci_event" + ], + "skip_when_false": "inputs.open_pr", + "note": "Runs when ci_watch reports a CI failure. Fetches CI logs via gh API and writes a structured diagnosis to {{AUTOSKILLIT_TEMP}}/diagnose-ci/. Routes to resolve_ci on success OR failure (best-effort: resolve-failures proceeds even without a diagnosis). diagnosis_path is passed to resolve_ci for targeted remediation context.\n" + }, + "resolve_ci": { + "tool": "run_skill", + "model": "", + "with": { + "skill_command": "/autoskillit:resolve-failures ${{ context.work_dir }} ${{ context.plan_path }} ${{ inputs.base_branch }} ${{ context.ci_conclusion }} ${{ context.ci_failed_jobs }} ${{ context.diagnosis_path }}", + "cwd": "${{ context.work_dir }}", + "step_name": "resolve_ci" + }, + "capture": { + "verdict": "${{ result.verdict }}" + }, + "retries": 2, + "on_exhausted": "release_issue_failure", + "on_context_limit": "re_push", + "on_result": [ + { + "when": "${{ result.verdict }} == 'real_fix'", + "route": "re_push" + }, + { + "when": "${{ result.verdict }} == 'already_green'", + "route": "pre_resolve_rebase" + }, + { + "when": "${{ result.verdict }} == 'flake_suspected'", + "route": "re_push" + }, + { + "when": "${{ result.verdict }} == 'ci_only_failure'", + "route": "release_issue_failure" + } + ], + "on_failure": "release_issue_failure", + "note": "Runs when ci_watch reports a CI failure. Receives ci_failed_jobs from wait_for_ci structured output to avoid redundant CI failure re-investigation. Routes via on_result: verdict dispatch — real_fix pushes, already_green rebases and re-diagnoses (sibling fix may have landed), flake_suspected retries via re_push (bounded by retries: 2 / on_exhausted: release_issue_failure), ci_only_failure escalates to human. Retries up to 2 times (3 total attempts) before exhausting to release_issue_failure.\n" + }, + "pre_resolve_rebase": { + "tool": "run_cmd", + "with": { + "cmd": "REMOTE=$(git -C ${{ context.work_dir }} remote get-url upstream 2>/dev/null | grep -qv \"^file://\" && echo upstream || echo origin) &&\ngit -C ${{ context.work_dir }} fetch \"$REMOTE\" &&\ngit -C ${{ context.work_dir }} rebase \"$REMOTE\"/${{ inputs.base_branch }}\n", + "step_name": "pre_resolve_rebase" + }, + "retries": 1, + "on_success": "diagnose_ci", + "on_failure": "release_issue_failure", + "note": "Rebases the feature branch against integration head when resolve_ci emits already_green — the worktree was already clean because a sibling pipeline's fix already landed on integration. Pulling it in before re-running diagnose_ci gives the pipeline a real chance at emitting real_fix and reaching re_push.\n" + }, + "re_push": { + "tool": "push_to_remote", + "with": { + "clone_path": "${{ context.work_dir }}", + "remote_url": "${{ context.remote_url }}", + "branch": "${{ context.merge_target }}", + "force": "true" + }, + "on_success": "check_repo_ci_event", + "on_failure": "release_issue_failure", + "note": "Pushes the CI-fixed feature branch back to the remote after resolve_ci succeeds. Routes to check_repo_ci_event to derive ci_event before ci_watch verifies the fix. On push failure, escalates via release_issue_failure.\n" + }, + "check_repo_merge_state": { + "tool": "check_repo_merge_state", + "block": "pre_queue_gate", + "with": { + "branch": "${{ inputs.base_branch }}", + "cwd": "${{ context.work_dir }}", + "remote_url": "${{ context.remote_url }}", + "step_name": "check_repo_merge_state" + }, + "capture": { + "queue_available": "${{ result.queue_available }}", + "merge_group_trigger": "${{ result.merge_group_trigger }}", + "auto_merge_available": "${{ result.auto_merge_available }}", + "ci_event": "${{ result.ci_event }}" + }, + "on_success": "route_queue_mode", + "on_failure": "immediate_merge", + "skip_when_false": "inputs.open_pr", + "note": "Single GraphQL round-trip that captures queue_available, merge_group_trigger, auto_merge_available, and ci_event for the target branch. Consolidates the former three sequential run_cmd steps (check_merge_queue, check_merge_group_trigger, check_auto_merge) into one MCP tool call. On failure (network error, auth failure), falls through to immediate_merge (non-queue direct merge path). Skipped when open_pr=false — no PR was opened.\n" + }, + "route_queue_mode": { + "block": "pre_queue_gate", + "action": "route", + "on_result": [ + { + "when": "${{ inputs.auto_merge }} != 'true'", + "route": "patch_token_summary" + }, + { + "when": "${{ context.queue_available }} == true and ${{ context.merge_group_trigger }} == true", + "route": "enqueue_to_queue" + }, + { + "when": "${{ context.auto_merge_available }} == true", + "route": "direct_merge" + }, + { + "route": "immediate_merge" + } + ], + "skip_when_false": "inputs.open_pr" + }, + "enqueue_to_queue": { + "tool": "enqueue_pr", + "with": { + "pr_number": "${{ context.pr_number }}", + "target_branch": "${{ inputs.base_branch }}", + "cwd": "${{ context.work_dir }}", + "auto_merge_available": "${{ context.auto_merge_available }}" + }, + "on_success": "wait_for_queue", + "on_failure": "verify_queue_enrollment", + "skip_when_false": "inputs.open_pr", + "note": "Submits the PR to the merge queue via the unified enqueue_pr tool. The tool resolves the enrollment strategy internally based on auto_merge_available. On failure, probes the queue via verify_queue_enrollment to distinguish a transient enqueue error from a PR that is already merged or still in queue.\n" + }, + "verify_queue_enrollment": { + "description": "Probe merge queue after enqueue command failure to determine actual state", + "tool": "wait_for_merge_queue", + "with": { + "pr_number": "${{ context.pr_number }}", + "target_branch": "${{ inputs.base_branch }}", + "cwd": "${{ context.work_dir }}", + "remote_url": "${{ context.remote_url }}", + "timeout_seconds": 60, + "poll_interval": 10 + }, + "on_result": [ + { + "when": "${{ result.pr_state }} == merged", + "route": "release_issue_success" + }, + { + "when": "${{ result.pr_state }} == ejected_ci_failure", + "route": "diagnose_ci" + }, + { + "when": "${{ result.pr_state }} == not_enrolled", + "route": "release_issue_failure" + }, + { + "when": "${{ result.pr_state }} == dropped_healthy", + "route": "wait_for_queue" + }, + { + "when": "${{ result.pr_state }} == dropped_merge_group_ci", + "route": "diagnose_ci" + }, + { + "when": "${{ result.pr_state }} == ejected", + "route": "wait_for_queue" + }, + { + "when": "${{ result.pr_state }} == stalled", + "route": "wait_for_queue" + }, + { + "when": "${{ result.pr_state }} == timeout", + "route": "register_clone_unconfirmed" + }, + { + "route": "register_clone_unconfirmed" + } + ], + "on_failure": "register_clone_unconfirmed", + "note": "Short probe (60s) after enqueue_to_queue failure. If the PR is already merged, route to release_issue_success. If the queue already confirms CI failure (ejected_ci_failure or dropped_merge_group_ci), route directly to diagnose_ci. If enrollment was attempted but the PR never appeared in the queue (not_enrolled), route to release_issue_failure — this is a repo configuration error. For other active queue states (dropped_healthy, ejected, stalled), route to wait_for_queue for full 900s authoritative watch. If the probe timed out or returned an unexpected state, escalate via register_clone_unconfirmed.\n" + }, + "wait_for_queue": { + "tool": "wait_for_merge_queue", + "with": { + "pr_number": "${{ context.pr_number }}", + "target_branch": "${{ inputs.base_branch }}", + "timeout_seconds": 900, + "cwd": "${{ context.work_dir }}", + "remote_url": "${{ context.remote_url }}" + }, + "capture": { + "queue_pr_state": "${{ result.pr_state }}" + }, + "on_result": [ + { + "when": "${{ result.pr_state }} == merged", + "route": "release_issue_success" + }, + { + "when": "${{ result.pr_state }} == ejected_ci_failure", + "route": "diagnose_ci" + }, + { + "when": "${{ result.pr_state }} == dropped_merge_group_ci", + "route": "diagnose_ci" + }, + { + "when": "${{ result.pr_state }} == not_enrolled", + "route": "release_issue_failure" + }, + { + "when": "${{ result.pr_state }} == dropped_healthy", + "route": "check_dropped_healthy_loop" + }, + { + "when": "${{ result.pr_state }} == ejected", + "route": "check_eject_limit" + }, + { + "when": "${{ result.pr_state }} == stalled", + "route": "reenroll_stalled_pr" + }, + { + "when": "${{ result.pr_state }} == timeout", + "route": "register_clone_unconfirmed" + }, + { + "route": "register_clone_unconfirmed" + } + ], + "on_failure": "register_clone_unconfirmed", + "skip_when_false": "inputs.open_pr", + "note": "Waits up to 15 minutes for the PR to merge through the queue. merged → release_issue_success to transition issue state then register_clone_success. ejected_ci_failure → diagnose_ci (CI failure; conflict resolution cannot fix this). not_enrolled → release_issue_failure (enrollment failed; repo configuration error). ejected → conflict fix cycle then re-enter. stalled → lightweight re-enrollment via enqueue_pr. timeout → register_clone_unconfirmed (release claim without staged label; merge unconfirmed).\n" + }, + "reenroll_stalled_pr": { + "tool": "enqueue_pr", + "with": { + "pr_number": "${{ context.pr_number }}", + "target_branch": "${{ inputs.base_branch }}", + "cwd": "${{ context.work_dir }}", + "auto_merge_available": "${{ context.auto_merge_available }}" + }, + "on_success": "check_stall_loop", + "on_failure": "release_issue_failure", + "skip_when_false": "inputs.open_pr" + }, + "check_stall_loop": { + "tool": "run_python", + "with": { + "callable": "autoskillit.smoke_utils.check_loop_iteration", + "current_iteration": "${{ context.stall_loop_count }}", + "max_iterations": "3" + }, + "capture": { + "stall_loop_count": "${{ result.next_iteration }}" + }, + "on_result": [ + { + "when": "${{ result.max_exceeded }} == true", + "route": "register_clone_unconfirmed" + }, + { + "route": "wait_for_queue" + } + ], + "on_failure": "register_clone_unconfirmed", + "optional_context_refs": [ + "stall_loop_count" + ], + "note": "Iteration guard for the wait_for_queue / reenroll_stalled_pr cycle. Caps stall re-enrollment at 3 attempts. On budget exhaustion, routes to register_clone_unconfirmed (merge outcome unconfirmed).\n" + }, + "check_eject_limit": { + "tool": "run_python", + "with": { + "callable": "autoskillit.recipe._cmd_rpc.check_eject_limit", + "counter_file": "${{ context.work_dir }}/{{AUTOSKILLIT_TEMP}}/eject_count", + "max_ejects": "3" + }, + "on_result": [ + { + "when": "${{ result.status }} == EJECT_LIMIT_EXCEEDED", + "route": "release_issue_failure" + }, + { + "route": "queue_ejected_fix" + } + ], + "on_failure": "release_issue_failure", + "note": "Circuit breaker for the queue ejection loop. Tracks ejection count in a file-based counter under {{AUTOSKILLIT_TEMP}}/. After 3 ejections, routes to release_issue_failure to prevent infinite re-enqueue cycles.\n" + }, + "check_dropped_healthy_loop": { + "tool": "run_python", + "with": { + "callable": "autoskillit.recipe._cmd_rpc.check_dropped_healthy_loop", + "counter_file": "${{ context.work_dir }}/{{AUTOSKILLIT_TEMP}}/dropped_healthy_count", + "max_drops": "2" + }, + "on_result": [ + { + "when": "${{ result.status }} == DROPPED_LIMIT_EXCEEDED", + "route": "release_issue_failure" + }, + { + "route": "reenter_merge_queue_cheap" + } + ], + "on_failure": "release_issue_failure", + "note": "Circuit breaker for the dropped_healthy re-enrollment loop. After 2 consecutive unexplained drops with healthy PR-branch CI, escalates to release_issue_failure instead of looping. Matches the check_eject_limit pattern used for the ejected path. DROPPED_MERGE_GROUP_CI (confirmed merge-group CI failure) is routed directly to diagnose_ci and never reaches this step.\n" + }, + "queue_ejected_fix": { + "tool": "run_python", + "with": { + "callable": "autoskillit.recipe._cmd_rpc.queue_ejected_fix", + "work_dir": "${{ context.work_dir }}", + "base_branch": "${{ inputs.base_branch }}" + }, + "on_result": [ + { + "when": "${{ result.status }} == clean", + "route": "re_push_queue_fix" + }, + { + "route": "resolve_queue_merge_conflicts" + } + ], + "on_failure": "resolve_queue_merge_conflicts", + "skip_when_false": "inputs.open_pr", + "note": "Attempts a cheap rebase before invoking the full conflict-resolution skill. On a clean rebase (stdout \"clean\"), skips the skill and proceeds directly to re_push_queue_fix. On conflict, aborts the rebase and routes to resolve_queue_merge_conflicts for goal-aware resolution.\n" + }, + "resolve_queue_merge_conflicts": { + "tool": "run_skill", + "model": "", + "with": { + "skill_command": "/autoskillit:resolve-merge-conflicts ${{ context.work_dir }} ${{ context.plan_path }} ${{ inputs.base_branch }}", + "cwd": "${{ context.work_dir }}", + "step_name": "resolve_queue_merge_conflicts" + }, + "capture": { + "conflict_escalation_required": "${{ result.escalation_required }}" + }, + "on_result": [ + { + "when": "${{ result.escalation_required }} == true", + "route": "release_issue_failure" + }, + { + "route": "re_push_queue_fix" + } + ], + "on_failure": "release_issue_failure", + "on_context_limit": "release_issue_failure", + "retries": 1, + "on_exhausted": "release_issue_failure", + "skip_when_false": "inputs.open_pr", + "note": "Resolves rebase conflicts on the feature branch after queue ejection. Only reached when the cheap rebase in queue_ejected_fix encountered conflicts. escalation_required=true → release_issue_failure for human review.\n" + }, + "re_push_queue_fix": { + "tool": "push_to_remote", + "with": { + "clone_path": "${{ context.work_dir }}", + "remote_url": "${{ context.remote_url }}", + "branch": "${{ context.merge_target }}", + "force": "true" + }, + "capture": { + "push_error_type": "${{ result.error_type }}" + }, + "retries": 2, + "on_exhausted": "register_clone_unconfirmed", + "on_success": "ci_watch_post_queue_fix", + "on_failure": "classify_push_failure", + "skip_when_false": "inputs.open_pr" + }, + "classify_push_failure": { + "action": "route", + "on_result": [ + { + "when": "${{ context.push_error_type }} == queued_branch", + "route": "wait_dequeue_retry" + }, + { + "route": "release_issue_failure" + } + ] + }, + "wait_dequeue_retry": { + "tool": "wait_for_merge_queue", + "with": { + "pr_number": "${{ context.pr_number }}", + "target_branch": "${{ inputs.base_branch }}", + "cwd": "${{ context.work_dir }}", + "timeout_seconds": "60" + }, + "on_result": [ + { + "when": "${{ result.pr_state }} == merged", + "route": "release_issue_success" + }, + { + "when": "${{ result.pr_state }} == ejected_ci_failure", + "route": "register_clone_unconfirmed" + }, + { + "when": "${{ result.pr_state }} == dropped_merge_group_ci", + "route": "register_clone_unconfirmed" + }, + { + "when": "${{ result.pr_state }} == dropped_healthy", + "route": "register_clone_unconfirmed" + }, + { + "when": "${{ result.pr_state }} == ejected", + "route": "re_push_queue_fix" + }, + { + "when": "${{ result.pr_state }} == stalled", + "route": "register_clone_unconfirmed" + }, + { + "when": "${{ result.pr_state }} == timeout", + "route": "register_clone_unconfirmed" + }, + { + "route": "register_clone_unconfirmed" + } + ], + "on_failure": "register_clone_unconfirmed", + "optional_context_refs": [ + "push_error_type" + ] + }, + "ci_watch_post_queue_fix": { + "tool": "wait_for_ci", + "with": { + "branch": "${{ context.merge_target }}", + "event": "${{ context.ci_event }}", + "timeout_seconds": 600, + "cwd": "${{ context.work_dir }}", + "remote_url": "${{ context.remote_url }}", + "auto_trigger": "true" + }, + "capture": { + "ci_conclusion": "${{ result.conclusion }}", + "ci_failed_jobs": "${{ result.failed_jobs }}" + }, + "on_result": [ + { + "when": "${{ result.conclusion }} == 'success'", + "route": "reenter_merge_queue" + }, + { + "when": "${{ result.conclusion }} == 'no_runs'", + "route": "release_issue_failure" + }, + { + "when": "${{ result.conclusion }} == 'timed_out'", + "route": "check_ci_post_queue_loop" + }, + { + "route": "detect_ci_conflict" + } + ], + "on_failure": "detect_ci_conflict", + "skip_when_false": "inputs.open_pr", + "note": "Validates CI on the rebased feature branch after queue ejection fix and re-push. Routes on result.conclusion: success → reenter_merge_queue, no_runs → release_issue_failure (auto_trigger fires first: empty commit + force-push + re-poll; no_runs reached only when trigger recovery fails), timed_out → check_ci_post_queue_loop, failure → detect_ci_conflict.\n" + }, + "check_ci_post_queue_loop": { + "tool": "run_python", + "with": { + "callable": "autoskillit.smoke_utils.check_loop_iteration", + "current_iteration": "${{ context.ci_post_queue_loop_count }}", + "max_iterations": "2" + }, + "capture": { + "ci_post_queue_loop_count": "${{ result.next_iteration }}" + }, + "on_result": [ + { + "when": "${{ result.max_exceeded }} == true", + "route": "release_issue_failure" + }, + { + "route": "ci_watch_post_queue_fix" + } + ], + "on_failure": "release_issue_failure", + "optional_context_refs": [ + "ci_post_queue_loop_count" + ], + "skip_when_false": "inputs.open_pr", + "note": "Iteration guard for the ci_watch_post_queue_fix timed_out self-loop. Caps at 2 iterations (initial + 1 retry). On budget exhaustion, routes to release_issue_failure — this is already a post-ejection retry path so further escalation is not warranted.\n" + }, + "reenter_merge_queue": { + "tool": "enqueue_pr", + "with": { + "pr_number": "${{ context.pr_number }}", + "target_branch": "${{ inputs.base_branch }}", + "cwd": "${{ context.work_dir }}", + "auto_merge_available": "${{ context.auto_merge_available }}" + }, + "on_success": "wait_for_queue", + "on_failure": "release_issue_failure", + "skip_when_false": "inputs.open_pr", + "note": "Re-submits the PR after conflict resolution. Routes back to wait_for_queue. If re-entry fails, release_issue_failure preserves the clone for debugging.\n" + }, + "reenter_merge_queue_cheap": { + "tool": "enqueue_pr", + "with": { + "pr_number": "${{ context.pr_number }}", + "target_branch": "${{ inputs.base_branch }}", + "cwd": "${{ context.work_dir }}", + "auto_merge_available": "${{ context.auto_merge_available }}" + }, + "on_success": "wait_for_queue", + "on_failure": "release_issue_failure", + "skip_when_false": "inputs.open_pr", + "note": "Re-submits a healthy PR into the merge queue after auto_merge was cleared externally (DROPPED_HEALTHY state). No conflict-resolution CI wait needed. Routes back to wait_for_queue on success.\n" + }, + "direct_merge": { + "tool": "run_cmd", + "with": { + "cmd": "gh pr merge '${{ context.pr_number }}' --squash --auto", + "cwd": "${{ context.work_dir }}", + "step_name": "direct_merge" + }, + "on_success": "wait_for_direct_merge", + "on_failure": "release_issue_failure", + "skip_when_false": "inputs.open_pr", + "note": "Enables GitHub native auto-merge for direct merge (no merge queue). GitHub merges automatically once all required checks pass. On failure, escalates via release_issue_failure so the issue claim is released and the pipeline reports an error rather than silently succeeding.\n" + }, + "wait_for_direct_merge": { + "tool": "run_python", + "with": { + "callable": "autoskillit.recipe._cmd_rpc.wait_for_direct_merge", + "pr_number": "${{ context.pr_number }}", + "max_polls": "90", + "poll_interval": "10" + }, + "on_result": [ + { + "when": "${{ result.state }} == merged", + "route": "release_issue_success" + }, + { + "when": "${{ result.state }} == closed", + "route": "direct_merge_conflict_fix" + }, + { + "route": "register_clone_unconfirmed" + } + ], + "on_failure": "register_clone_unconfirmed", + "skip_when_false": "inputs.open_pr", + "note": "Polls GitHub PR state every 10 s for up to 15 minutes (90 iterations). merged → release_issue_success (issue labelled, cleanup runs). closed → PR was closed (likely conflict) — route to conflict fix. timeout / unexpected → register_clone_unconfirmed.\n" + }, + "direct_merge_conflict_fix": { + "tool": "run_python", + "with": { + "callable": "autoskillit.recipe._cmd_rpc.direct_merge_conflict_fix", + "work_dir": "${{ context.work_dir }}", + "base_branch": "${{ inputs.base_branch }}" + }, + "on_result": [ + { + "when": "${{ result.status }} == clean", + "route": "re_push_direct_fix" + }, + { + "route": "resolve_direct_merge_conflicts" + } + ], + "on_failure": "resolve_direct_merge_conflicts", + "skip_when_false": "inputs.open_pr", + "note": "Attempts a cheap rebase before invoking the full conflict-resolution skill. On clean rebase, proceeds to re_push_direct_fix. On conflict, routes to resolve_direct_merge_conflicts for goal-aware resolution.\n" + }, + "resolve_direct_merge_conflicts": { + "tool": "run_skill", + "model": "", + "with": { + "skill_command": "/autoskillit:resolve-merge-conflicts ${{ context.work_dir }} ${{ context.plan_path }} ${{ inputs.base_branch }}", + "cwd": "${{ context.work_dir }}", + "step_name": "resolve_direct_merge_conflicts" + }, + "capture": { + "conflict_escalation_required": "${{ result.escalation_required }}" + }, + "on_result": [ + { + "when": "${{ result.escalation_required }} == true", + "route": "release_issue_failure" + }, + { + "route": "re_push_direct_fix" + } + ], + "on_failure": "release_issue_failure", + "on_context_limit": "release_issue_failure", + "retries": 1, + "on_exhausted": "release_issue_failure", + "skip_when_false": "inputs.open_pr", + "note": "Resolves rebase conflicts after direct-merge rejection. Only reached when the cheap rebase in direct_merge_conflict_fix encountered conflicts. escalation_required=true → release_issue_failure for human review.\n" + }, + "re_push_direct_fix": { + "tool": "push_to_remote", + "with": { + "clone_path": "${{ context.work_dir }}", + "remote_url": "${{ context.remote_url }}", + "branch": "${{ context.merge_target }}", + "force": "true" + }, + "on_success": "redirect_merge", + "on_failure": "release_issue_failure", + "skip_when_false": "inputs.open_pr" + }, + "redirect_merge": { + "tool": "run_cmd", + "with": { + "cmd": "gh pr merge '${{ context.pr_number }}' --squash --auto", + "cwd": "${{ context.work_dir }}", + "step_name": "redirect_merge" + }, + "on_success": "wait_for_direct_merge", + "on_failure": "release_issue_failure", + "skip_when_false": "inputs.open_pr", + "note": "Re-enables auto-merge after conflict resolution and re-push. Routes back to wait_for_direct_merge to poll for completion. On failure, release_issue_failure preserves the clone for debugging.\n" + }, + "immediate_merge": { + "tool": "run_cmd", + "with": { + "cmd": "gh pr merge '${{ context.pr_number }}' --squash", + "cwd": "${{ context.work_dir }}", + "step_name": "immediate_merge" + }, + "on_success": "wait_for_immediate_merge", + "on_failure": "release_issue_failure", + "skip_when_false": "inputs.open_pr", + "note": "Merges the PR immediately using plain --squash when autoMergeAllowed=false. Unlike direct_merge, no --auto flag is used — GitHub executes the merge synchronously rather than queueing it for when checks pass. On failure, escalates via release_issue_failure so the issue claim is released and the pipeline reports an error rather than silently succeeding.\n" + }, + "wait_for_immediate_merge": { + "tool": "run_python", + "with": { + "callable": "autoskillit.recipe._cmd_rpc.wait_for_immediate_merge", + "pr_number": "${{ context.pr_number }}", + "max_polls": "30", + "poll_interval": "10" + }, + "on_result": [ + { + "when": "${{ result.state }} == merged", + "route": "release_issue_success" + }, + { + "when": "${{ result.state }} == closed", + "route": "immediate_merge_conflict_fix" + }, + { + "route": "register_clone_unconfirmed" + } + ], + "on_failure": "register_clone_unconfirmed", + "skip_when_false": "inputs.open_pr", + "note": "Polls GitHub PR state every 10s for up to 5 minutes (30 iterations). Since immediate_merge uses --squash without --auto, the merge completes synchronously — this poll is a confirmation window. merged → release_issue_success. closed → PR was rejected (stale base or conflict) — route to conflict fix. timeout → register_clone_unconfirmed.\n" + }, + "immediate_merge_conflict_fix": { + "tool": "run_python", + "with": { + "callable": "autoskillit.recipe._cmd_rpc.immediate_merge_conflict_fix", + "work_dir": "${{ context.work_dir }}", + "base_branch": "${{ inputs.base_branch }}" + }, + "on_result": [ + { + "when": "${{ result.status }} == clean", + "route": "re_push_immediate_fix" + }, + { + "route": "resolve_immediate_merge_conflicts" + } + ], + "on_failure": "resolve_immediate_merge_conflicts", + "skip_when_false": "inputs.open_pr", + "note": "Attempts a cheap rebase before invoking the full conflict-resolution skill. On clean rebase, proceeds to re_push_immediate_fix. On conflict, routes to resolve_immediate_merge_conflicts for goal-aware resolution.\n" + }, + "resolve_immediate_merge_conflicts": { + "tool": "run_skill", + "model": "", + "with": { + "skill_command": "/autoskillit:resolve-merge-conflicts ${{ context.work_dir }} ${{ context.plan_path }} ${{ inputs.base_branch }}", + "cwd": "${{ context.work_dir }}", + "step_name": "resolve_immediate_merge_conflicts" + }, + "capture": { + "conflict_escalation_required": "${{ result.escalation_required }}" + }, + "on_result": [ + { + "when": "${{ result.escalation_required }} == true", + "route": "release_issue_failure" + }, + { + "route": "re_push_immediate_fix" + } + ], + "on_failure": "release_issue_failure", + "on_context_limit": "release_issue_failure", + "retries": 1, + "on_exhausted": "release_issue_failure", + "skip_when_false": "inputs.open_pr", + "note": "Resolves rebase conflicts after immediate-merge rejection. Only reached when the cheap rebase in immediate_merge_conflict_fix encountered conflicts. escalation_required=true → release_issue_failure for human review.\n" + }, + "re_push_immediate_fix": { + "tool": "push_to_remote", + "with": { + "clone_path": "${{ context.work_dir }}", + "remote_url": "${{ context.remote_url }}", + "branch": "${{ context.merge_target }}", + "force": "true" + }, + "on_success": "remerge_immediate", + "on_failure": "release_issue_failure", + "skip_when_false": "inputs.open_pr" + }, + "remerge_immediate": { + "tool": "run_cmd", + "with": { + "cmd": "gh pr merge '${{ context.pr_number }}' --squash", + "cwd": "${{ context.work_dir }}", + "step_name": "remerge_immediate" + }, + "on_success": "wait_for_immediate_merge", + "on_failure": "release_issue_failure", + "skip_when_false": "inputs.open_pr", + "note": "Re-attempts plain squash merge after conflict resolution and re-push. Routes back to wait_for_immediate_merge to confirm completion. On failure, release_issue_failure preserves the clone for debugging.\n" + }, + "register_clone_unconfirmed": { + "description": "Register clone as preserved — merge unconfirmed, in-progress label retained", + "tool": "register_clone_status", + "with": { + "clone_path": "${{ context.work_dir }}", + "status": "unconfirmed", + "step_name": "register_clone_unconfirmed" + }, + "on_success": "done", + "on_failure": "done", + "note": "Called when a merge-wait step times out before merge is confirmed. Does NOT call release_issue — the in-progress label stays on the issue so operators can see the PR is still actively queued. The clone is preserved (status: unconfirmed) for human inspection.\n" + }, + "detect_ci_conflict": { + "tool": "run_cmd", + "with": { + "cmd": "REMOTE=$(git remote get-url upstream 2>/dev/null | grep -qv \"^file://\" && echo upstream || echo origin) &&\ngit fetch \"$REMOTE\" ${{ inputs.base_branch }} &&\n! git merge-base --is-ancestor \"$REMOTE\"/${{ inputs.base_branch }} HEAD\n", + "cwd": "${{ context.work_dir }}", + "step_name": "detect_ci_conflict" + }, + "on_success": "ci_conflict_fix", + "on_failure": "diagnose_ci", + "optional": true, + "skip_when_false": "inputs.open_pr", + "note": "Detects whether the CI failure originates from a stale integration base rather than a code-level test failure. Exit 0 (fetch succeeded AND base is NOT an ancestor of HEAD) signals a stale base — routes to ci_conflict_fix. Exit 1 (fetch failed OR base is already an ancestor) signals a real code failure or an unresolvable state — falls through to diagnose_ci. This step insulates resolve_ci from being invoked for merge-conflict failures.\n" + }, + "ci_conflict_fix": { + "tool": "run_skill", + "model": "", + "with": { + "skill_command": "/autoskillit:resolve-merge-conflicts ${{ context.work_dir }} ${{ context.plan_path }} ${{ inputs.base_branch }}", + "cwd": "${{ context.work_dir }}", + "step_name": "ci_conflict_fix" + }, + "capture": { + "conflict_escalation_required": "${{ result.escalation_required }}" + }, + "on_result": [ + { + "when": "${{ result.escalation_required }} == true", + "route": "release_issue_failure" + }, + { + "route": "re_push" + } + ], + "on_failure": "release_issue_failure", + "on_context_limit": "release_issue_failure", + "retries": 1, + "on_exhausted": "release_issue_failure", + "skip_when_false": "inputs.open_pr", + "note": "Resolves a stale-base CI failure by rebasing the feature branch onto the latest integration HEAD. On clean rebase or goal-aware conflict resolution, routes to re_push to re-trigger CI on the rebased branch. On escalation (LOW confidence conflicts) or tool failure, routes to release_issue_failure. retries: 1 bounds the loop; after 2 total attempts, escalates.\n" + }, + "release_issue_success": { + "description": "Transition issue from in-progress to staged after successful pipeline completion", + "tool": "release_issue", + "optional": true, + "skip_when_false": "inputs.issue_url", + "with": { + "issue_url": "${{ inputs.issue_url }}", + "target_branch": "${{ inputs.base_branch }}" + }, + "on_success": "patch_token_summary", + "on_failure": "patch_token_summary", + "note": "Called on the success path after PR merge is confirmed. Passes target_branch so release_issue can determine whether to apply the staged label (non-default target) or just remove in-progress (default target). optional=true ensures pipeline still reaches register_clone_success if this step errors.\n" + }, + "patch_token_summary": { + "description": "Collect all pipeline token data and patch the PR body with complete summary", + "tool": "run_python", + "optional": true, + "skip_when_false": "inputs.open_pr", + "with": { + "callable": "autoskillit.smoke_utils.patch_pr_token_summary", + "timeout": 60, + "pr_url": "${{ context.pr_url }}", + "cwd": "${{ context.work_dir }}" + }, + "on_success": "register_clone_success", + "on_failure": "register_clone_success", + "note": "Patches PR body with complete token summary; failure never blocks pipeline completion.\n" + }, + "release_issue_failure": { + "description": "Release the in-progress claim after pipeline failure", + "tool": "release_issue", + "optional": true, + "skip_when_false": "inputs.issue_url", + "with": { + "issue_url": "${{ inputs.issue_url }}", + "fail_label": "fail" + }, + "on_success": "register_clone_failure", + "on_failure": "register_clone_failure" + }, + "register_clone_success": { + "description": "Register clone as cleanup candidate for batch deletion", + "tool": "register_clone_status", + "with": { + "clone_path": "${{ context.work_dir }}", + "status": "success", + "step_name": "register_clone_success" + }, + "on_success": "done", + "on_failure": "done" + }, + "register_clone_failure": { + "description": "Register clone as preserved (error pipeline)", + "tool": "register_clone_status", + "with": { + "clone_path": "${{ context.work_dir }}", + "status": "error", + "step_name": "register_clone_failure" + }, + "on_success": "escalate_stop", + "on_failure": "escalate_stop" + }, + "done": { + "action": "stop", + "message": "Implementation pipeline complete. All groups/tasks have been planned, implemented, tested, and merged." + }, + "escalate_stop": { + "action": "stop", + "message": "Pipeline failed — human intervention needed. Check the worktree and plan for details." + } + } +} diff --git a/src/autoskillit/recipes/implementation-groups.yaml b/src/autoskillit/recipes/implementation-groups.yaml index a3f38c16b0..878e351bc8 100644 --- a/src/autoskillit/recipes/implementation-groups.yaml +++ b/src/autoskillit/recipes/implementation-groups.yaml @@ -1,7 +1,8 @@ name: implementation-groups description: Decompose a source document into sequenced implementation groups, then plan, verify, implement, test, and merge each group end-to-end. Use when you have a large document or roadmap to implement via make-groups. -summary: clone > capture_base_sha > set_merge_target > (create_branch?) > make-groups > make-plan > (review-approach?) > dry-walkthrough > implement > test > merge (per group, per plan part) > (audit-impl?) > (open_pr?) > push > (review_pr?) > (ci_watch?) > cleanup -autoskillit_version: "0.3.0" +summary: "bootstrap_clone > (claim_and_resolve?) > (create_and_publish?) > make-groups > make-plan > (review-approach?) > dry-walkthrough > implement > test > merge (per group, per plan part) > (audit-impl?) > (open_pr?) > push > (review_pr?) > (ci_watch?) > cleanup" +recipe_version: "1.0.0" +requires_packs: [github, ci, clone, telemetry] ingredients: source_doc: @@ -26,11 +27,23 @@ ingredients: open_pr: description: PR instead of direct merge default: "true" + arch_lenses: + description: Generate architecture lens diagrams for the PR + default: "false" auto_merge: description: >- Automatically merge the PR after required checks pass — via merge queue when available, direct merge otherwise default: "true" + review_max_retries: + description: Maximum number of review-resolve retry cycles before proceeding to CI + default: "3" + local_review_rounds: + description: >- + Number of review-resolve cycles to run locally before switching to + GitHub-backed review. 0 disables local mode entirely. + type: integer + default: "3" issue_url: description: GitHub issue to close on merge required: false @@ -53,7 +66,11 @@ kitchen_rules: directly. The push step publishes the feature branch, then prepare_pr → run_arch_lenses → compose_pr opens a PR to base_branch. When open_pr=false, merges target base_branch directly and prepare_pr is skipped." - - "SOURCE ISOLATION: After clone_repo returns, the source_dir is strictly + - "When arch_lenses=true (and open_pr=true), prepare_pr routes through + run_arch_lenses before compose_pr, adding architecture lens diagrams + to the PR. By default arch_lenses is off and prepare_pr routes + directly to compose_pr." + - "SOURCE ISOLATION: After bootstrap_clone returns, the source_dir is strictly off-limits. Never run any command in source_dir — no git checkout, git fetch, git reset, git pull, run_cmd, run_skill, or any other operation. All work — skill invocations, git operations, file reads — happens @@ -62,150 +79,72 @@ kitchen_rules: steps: clone: - tool: clone_repo + tool: bootstrap_clone with: source_dir: "${{ inputs.source_dir }}" run_name: "${{ inputs.run_name }}" - branch: "${{ inputs.base_branch }}" + base_branch: "${{ inputs.base_branch }}" + step_name: "bootstrap_clone" capture: - work_dir: "${{ result.clone_path }}" + work_dir: "${{ result.work_dir }}" remote_url: "${{ result.remote_url }}" - on_success: capture_base_sha - on_failure: escalate_stop - note: > - Clones source_dir into ../autoskillit-runs/-/. - Captures work_dir context used by all subsequent steps as cwd. - On failure, escalate_stop is reached immediately (no clone to remove). - - capture_base_sha: - tool: run_cmd - with: - cmd: "git -C ${{ context.work_dir }} rev-parse ${{ inputs.base_branch }}" - cwd: "${{ context.work_dir }}" - step_name: "capture_base_sha" - capture: - base_sha: "${{ result.stdout | trim }}" - on_success: set_merge_target + base_sha: "${{ result.base_sha }}" + merge_target: "${{ result.merge_target }}" + on_success: claim_and_resolve on_failure: escalate_stop note: > - Capture the SHA of base_branch before any merge begins. A commit SHA names - a git object (not a ref), so it survives git branch -D unconditionally. This - SHA is passed to audit_impl as implementation_ref to diff the full - multi-group implementation against the pre-implementation state. + Clones source_dir and captures base_sha + merge_target in one turn. + Returns work_dir, remote_url, base_sha (SHA of base_branch before any merge), + and merge_target (set to base_branch as fallback; overridden by create_and_publish + when open_pr=true). On failure, escalate_stop immediately (no clone to remove). - set_merge_target: - tool: run_cmd - with: - cmd: "printf '%s' '${{ inputs.base_branch }}'" - cwd: "${{ context.work_dir }}" - step_name: "set_merge_target" - capture: - merge_target: "${{ result.stdout }}" - on_success: get_issue_title - on_failure: escalate_stop - note: > - Unconditionally sets merge_target to inputs.base_branch as a fallback. - In the standard flow (open_pr=true), the subsequent create_branch step - overrides merge_target with the feature branch name. When open_pr=false, - create_branch is skipped and this fallback stands. This eliminates the - undefined-context-variable gap that existed when create_branch was optional. - - get_issue_title: - tool: get_issue_title - with: - issue_url: "${{ inputs.issue_url }}" - capture: - issue_number: "${{ result.number }}" - issue_title: "${{ result.title }}" - issue_slug: "${{ result.slug }}" - on_success: claim_issue - on_failure: escalate_stop - optional: true - skip_when_false: "inputs.issue_url" - note: > - Runs when inputs.issue_url is non-empty. Skipped otherwise. - Fetches only the issue title and slug (no body, no comments). - issue_number is threaded to prepare_pr for "Closes #N" in the PR body. - issue_slug provides a descriptive branch prefix. - The make-plan skill running in a headless session can call fetch_github_issue - directly if it needs full issue content — pass inputs.issue_url through. - - claim_issue: - description: "Claim this issue with the in-progress label to prevent parallel work" - tool: claim_issue + claim_and_resolve: + tool: claim_and_resolve_issue optional: true skip_when_false: inputs.issue_url with: - issue_url: ${{ inputs.issue_url }} - allow_reentry: ${{ inputs.upfront_claimed }} + issue_url: "${{ inputs.issue_url }}" + allow_reentry: "${{ inputs.upfront_claimed }}" capture: + issue_number: "${{ result.issue_number }}" + issue_title: "${{ result.issue_title }}" + issue_slug: "${{ result.issue_slug }}" claimed: "${{ result.claimed }}" on_result: - when: "${{ result.claimed }} == true" - route: compute_branch + route: create_and_publish - route: escalate_stop on_failure: escalate_stop - - compute_branch: - tool: run_cmd - with: - cmd: >- - SLUG="${{ context.issue_slug }}" && - RUN="${{ inputs.run_name }}" && - PREFIX="${SLUG:-$RUN}" && - ISSUE="${{ context.issue_number }}" && - if [ -n "$ISSUE" ]; then - printf '%s' "${PREFIX}/${ISSUE}" - else - printf '%s' "${PREFIX}/$(date +%Y%m%d)" - fi - cwd: "${{ context.work_dir }}" - step_name: "compute_branch" - capture: - proposed_branch: "${{ result.stdout | trim }}" - on_success: create_branch - on_failure: release_issue_failure - optional: true - skip_when_false: "inputs.open_pr" note: > - Computes the proposed branch name from slug/issue or slug/date — no git operations. - Result is passed to create_branch for atomic collision check and checkout. + Fetches issue title/slug and claims the issue in one turn. + Returns issue_number, issue_title, issue_slug, and claimed boolean. + When claimed=false (another session owns it), routes to escalate_stop. + issue_slug and issue_number are passed to create_and_publish for branch naming. - create_branch: - tool: create_unique_branch - with: - base_branch_name: "${{ context.proposed_branch }}" - cwd: "${{ context.work_dir }}" - capture: - merge_target: "${{ result.branch_name }}" - on_success: push_merge_target - on_failure: release_issue_failure + create_and_publish: + tool: create_and_publish_branch optional: true - skip_when_false: "inputs.open_pr" - note: > - Atomic branch creation: ls-remote collision check → checkout -b. Uses the - create_unique_branch MCP tool so the orchestrator cannot insert extra git fetch - or checkout calls between the collision check and branch creation. - - push_merge_target: - tool: push_to_remote + skip_when_false: inputs.open_pr with: - clone_path: "${{ context.work_dir }}" + issue_slug: "${{ context.issue_slug }}" + run_name: "${{ inputs.run_name }}" + issue_number: "${{ context.issue_number }}" + work_dir: "${{ context.work_dir }}" remote_url: "${{ context.remote_url }}" - branch: "${{ context.merge_target }}" + step_name: "create_and_publish_branch" + capture: + merge_target: "${{ result.merge_target }}" on_success: group on_failure: release_issue_failure - optional: true - skip_when_false: "inputs.open_pr" note: > - Runs by default (open_pr=true). Skipped when open_pr=false. - Publishes the feature branch to origin before any merge_worktree call. + Computes branch name, creates it (with collision suffix if needed), and pushes + to origin in one turn. Overrides merge_target with the feature branch name. merge_worktree requires refs/remotes/origin/{branch} to exist after fetch (step 5.5). - Without this step, merge_worktree fails with WORKTREE_INTACT_BASE_NOT_PUBLISHED. - Uses the pre-resolved remote_url from the clone step to preserve clone isolation. + When open_pr=false, this step is skipped and merge_target remains as base_branch. group: tool: run_skill + model: "" with: skill_command: "/autoskillit:make-groups ${{ inputs.source_doc }}" cwd: "${{ context.work_dir }}" @@ -224,6 +163,7 @@ steps: plan: tool: run_skill + model: "" with: skill_command: "/autoskillit:make-plan CURRENT_GROUP_FILE" cwd: "${{ context.work_dir }}" @@ -258,6 +198,7 @@ steps: review: tool: run_skill + model: "" with: skill_command: "/autoskillit:review-approach ${{ context.plan_path }}" cwd: "${{ context.work_dir }}" @@ -273,6 +214,7 @@ steps: verify: tool: run_skill + model: "" with: skill_command: "/autoskillit:dry-walkthrough ${{ context.plan_path }}" cwd: "${{ context.work_dir }}" @@ -284,6 +226,7 @@ steps: implement: tool: run_skill + model: "" stale_threshold: 2400 with: skill_command: "/autoskillit:implement-worktree-no-merge ${{ context.plan_path }}" @@ -299,6 +242,7 @@ steps: retry_worktree: tool: run_skill + model: "" stale_threshold: 2400 with: skill_command: "/autoskillit:retry-worktree ${{ context.plan_path }} ${{ context.worktree_path }}" @@ -318,15 +262,34 @@ steps: tool: test_check with: worktree_path: "${{ context.worktree_path }}" - on_success: commit_guard + on_success: check_merge_fix_loop on_failure: fix + check_merge_fix_loop: + tool: run_python + with: + callable: "autoskillit.smoke_utils.check_loop_iteration" + current_iteration: "${{ context.merge_fix_count }}" + max_iterations: "3" + capture: + merge_fix_count: "${{ result.next_iteration }}" + on_result: + - when: "${{ result.max_exceeded }} == true" + route: release_issue_failure + - route: commit_guard + on_failure: release_issue_failure + optional_context_refs: + - merge_fix_count + note: > + Iteration guard for the merge→fix→test cycle. Caps the loop at 3 + iterations (enough for legitimate flake recovery but not enough to + burn 30+ minutes on structurally unresolvable conflicts). + commit_guard: - tool: run_cmd + tool: run_python with: - cmd: "cd ${{ context.worktree_path }} && if [ -n \"$(git status --porcelain)\" ]; then git add -A && git commit -m 'chore: commit pending session changes'; fi" - cwd: "${{ context.worktree_path }}" - step_name: commit_guard + callable: "autoskillit.recipe._cmd_rpc.commit_guard" + worktree_path: "${{ context.worktree_path }}" on_success: merge on_failure: merge note: > @@ -350,7 +313,7 @@ steps: - when: "result.failed_step == 'post_rebase_test_gate'" route: fix - when: "result.failed_step == 'rebase'" - route: fix + route: rebase_conflict_fix - when: "result.error" route: release_issue_failure - route: next_or_done @@ -383,6 +346,7 @@ steps: fix: tool: run_skill + model: "" with: skill_command: "/autoskillit:resolve-failures ${{ context.worktree_path }} ${{ context.plan_path }} ${{ inputs.base_branch }}" cwd: "${{ context.worktree_path }}" @@ -404,6 +368,24 @@ steps: on_context_limit: test note: "Fixes test failures without merging. Routes back to test for re-validation before merge_worktree. on_context_limit routes needs_retry=True to test (worktree may already be green) instead of re-running fix blindly." + rebase_conflict_fix: + tool: run_skill + model: "" + with: + skill_command: "/autoskillit:resolve-merge-conflicts ${{ context.worktree_path }} ${{ context.plan_path }} ${{ inputs.base_branch }}" + cwd: "${{ context.worktree_path }}" + step_name: "rebase_conflict_fix" + capture: + conflict_escalation_required: "${{ result.escalation_required }}" + on_result: + - when: "${{ result.escalation_required }} == true" + route: release_issue_failure + - route: test + on_failure: release_issue_failure + on_context_limit: release_issue_failure + retries: 1 + on_exhausted: release_issue_failure + next_or_done: action: route on_result: @@ -420,6 +402,7 @@ steps: audit_impl: tool: run_skill + model: "" with: skill_command: "/autoskillit:audit-impl ${{ context.plan_path }} ${{ context.base_sha }} ${{ inputs.base_branch }}" cwd: "${{ context.work_dir }}" @@ -454,6 +437,7 @@ steps: prepare_pr: tool: run_skill + model: "" with: skill_command: "/autoskillit:prepare-pr ${{ context.all_plan_paths }} ${{ inputs.run_name }} ${{ inputs.base_branch }}" cwd: "${{ context.work_dir }}" @@ -463,7 +447,12 @@ steps: prep_path: "${{ result.prep_path }}" selected_lenses: "${{ result.selected_lenses }}" lens_context_paths: "${{ result.lens_context_paths }}" - on_success: run_arch_lenses + on_result: + - when: "${{ result.prep_path }} and ${{ inputs.arch_lenses }} == 'true'" + route: run_arch_lenses + - when: "${{ result.prep_path }}" + route: compose_pr + - route: release_issue_failure on_failure: release_issue_failure on_context_limit: release_issue_failure optional: true @@ -478,6 +467,7 @@ steps: run_arch_lenses: tool: run_skill + model: "" with: skill_command: "/autoskillit:arch-lens-{slug} {context_path}" cwd: "${{ context.work_dir }}" @@ -502,6 +492,7 @@ steps: compose_pr: tool: run_skill + model: "" with: skill_command: "/autoskillit:compose-pr ${{ context.prep_path }} \"${{ context.all_diagram_paths }}\" ${{ context.work_dir }} ${{ inputs.base_branch }}" cwd: "${{ context.work_dir }}" @@ -534,7 +525,7 @@ steps: skip_when_false: "inputs.open_pr" note: > Extracts the integer PR number from the feature branch opened by compose_pr. - pr_number is required by the queue finale steps (enable_auto_merge, wait_for_queue). + pr_number is required by the queue finale steps (enqueue_to_queue, wait_for_queue). annotate_pr_diff: tool: run_python @@ -543,48 +534,66 @@ steps: pr_number: "${{ context.pr_number }}" cwd: "${{ context.work_dir }}" output_dir: "${{ context.work_dir }}/{{AUTOSKILLIT_TEMP}}/review-pr" + base_branch: "${{ inputs.base_branch }}" + local_review_rounds: "${{ inputs.local_review_rounds }}" + current_iteration: "${{ context.review_loop_count }}" capture: annotated_diff_path: "${{ result.annotated_diff_path }}" hunk_ranges_path: "${{ result.hunk_ranges_path }}" + diff_metrics_path: "${{ result.diff_metrics_path }}" + review_mode: "${{ result.review_mode }}" on_success: review_pr on_failure: review_pr optional: true skip_when_false: "inputs.open_pr" + optional_context_refs: + - review_loop_count note: > Pre-computes annotated diff with [LNNN] markers and hunk ranges JSON for review-pr. On failure, falls through to review_pr which uses empty fallback. review_pr: tool: run_skill + model: "" with: - skill_command: "/autoskillit:review-pr ${{ context.merge_target }} ${{ inputs.base_branch }} annotated_diff_path=${{ context.annotated_diff_path }} hunk_ranges_path=${{ context.hunk_ranges_path }}" + skill_command: "/autoskillit:review-pr ${{ context.merge_target }} ${{ inputs.base_branch }} annotated_diff_path=${{ context.annotated_diff_path }} hunk_ranges_path=${{ context.hunk_ranges_path }} diff_metrics_path=${{ context.diff_metrics_path }} mode=${{ context.review_mode }}" cwd: "${{ context.work_dir }}" step_name: "review_pr" capture: - verdict: "${{ result.verdict }}" + review_verdict: "${{ result.verdict }}" on_result: - when: "${{ result.verdict }} == changes_requested" route: resolve_review + - when: "${{ result.verdict }} == approved_with_comments" + route: resolve_review - when: "${{ result.verdict }} == needs_human" - route: ci_watch + route: check_review_loop + - when: "${{ result.verdict }} == approved" + route: check_review_loop - when: "true" - route: ci_watch - on_failure: resolve_review - on_context_limit: ci_watch + route: check_review_loop + on_failure: check_repo_ci_event + on_context_limit: check_repo_ci_event optional: true skip_when_false: "inputs.open_pr" + optional_context_refs: + - review_loop_count + - review_mode note: > Runs after compose_pr creates the PR. Invokes the review-pr skill as a headless session to leave inline review comments on the PR. The skill finds the open PR by feature branch name (context.merge_target). Captures verdict - and routes via on_result: changes_requested → resolve_review, - needs_human → ci_watch (explicit), approved → ci_watch. On tool-level failure (MCP error, gh - unavailable), routes to resolve_review. Skipped when open_pr=false. + as review_verdict and routes via on_result: changes_requested/approved_with_comments + → resolve_review; all other verdicts (including approved, needs_human) + → check_review_loop (mandatory waypoint to increment review_loop_count). + On tool-level failure (MCP error, gh unavailable), routes to check_repo_ci_event + (no review to resolve). Skipped when open_pr=false. resolve_review: tool: run_skill + model: "" with: - skill_command: "/autoskillit:resolve-review ${{ context.merge_target }} ${{ inputs.base_branch }}" + skill_command: "/autoskillit:resolve-review ${{ context.merge_target }} ${{ inputs.base_branch }} mode=${{ context.review_mode }}" cwd: "${{ context.work_dir }}" step_name: "resolve_review" capture: @@ -597,18 +606,20 @@ steps: - when: "${{ result.verdict }} == 'already_green'" route: re_push_review - when: "${{ result.verdict }} == 'flake_suspected'" - route: release_issue_failure + route: re_push_review - when: "${{ result.verdict }} == 'ci_only_failure'" route: release_issue_failure on_failure: release_issue_failure + optional_context_refs: + - review_mode note: > Runs when review_pr reports issues (verdict=changes_requested). The clone (context.work_dir) has the feature branch (context.merge_target) checked out. resolve-review fetches inline and summary review comments via the GitHub API, applies fixes for each actionable finding (critical + warning severity), and - commits each fix. Routes via on_result: verdict dispatch — real_fix/already_green - both route to re_push_review; flake_suspected/ci_only_failure escalate. - Retries up to 2 times (3 total attempts) before exhausting to release_issue_failure. + commits each fix. Routes via on_result: verdict dispatch — real_fix/already_green/ + flake_suspected all route to re_push_review (retry bounded by retries: 2); + ci_only_failure escalates to release_issue_failure. re_push_review: tool: push_to_remote @@ -616,24 +627,96 @@ steps: clone_path: "${{ context.work_dir }}" remote_url: "${{ context.remote_url }}" branch: "${{ context.merge_target }}" - on_success: ci_watch + on_success: check_review_loop on_failure: release_issue_failure note: > Pushes the review-fixed feature branch back to the remote after resolve_review - succeeds. Routes to ci_watch to validate via CI (one-shot review gate). - On push failure, escalates via release_issue_failure. + succeeds. Routes to check_review_loop to determine if another review-resolve + cycle is needed (bounded by review_max_retries). On push failure, escalates + via release_issue_failure. + + check_review_loop: + tool: run_python + with: + callable: "autoskillit.smoke_utils.check_review_loop" + pr_number: "${{ context.pr_number }}" + current_iteration: "${{ context.review_loop_count }}" + max_iterations: "${{ inputs.review_max_retries }}" + previous_verdict: "${{ context.review_verdict }}" + local_review_rounds: "${{ inputs.local_review_rounds }}" + capture: + review_loop_count: "${{ result.next_iteration }}" + on_result: + - when: "${{ result.had_blocking }} == true and ${{ result.max_exceeded }} == false" + route: review_pr + - route: check_repo_ci_event + on_failure: check_repo_ci_event + skip_when_false: "inputs.open_pr" + optional_context_refs: + - review_loop_count + - review_verdict + note: > + Compound iteration + blocking guard. Routes back to review_pr only when + had_blocking=true AND max_exceeded=false. had_blocking is true when + previous_verdict=changes_requested OR (local_review_rounds > 0 AND iteration < local_review_rounds), + ensuring local review rounds are fully exhausted before exiting to CI. + + check_repo_ci_event: + tool: check_repo_merge_state + with: + branch: "${{ context.merge_target }}" + cwd: "${{ context.work_dir }}" + remote_url: "${{ context.remote_url }}" + step_name: "check_repo_ci_event" + capture: + ci_event: "${{ result.ci_event }}" + on_success: check_pr_state + on_failure: ci_watch + skip_when_false: "inputs.open_pr" + note: > + Derives ci_event for the feature branch BEFORE ci_watch runs. + Uses context.merge_target (feature branch) so push.branches filter + is evaluated against the actual branch being pushed, not the base branch. + Non-blocking: on_failure routes to ci_watch with ci_event=null (match-any). + + check_pr_state: + tool: check_pr_mergeable + with: + pr_number: "${{ context.pr_number }}" + cwd: "${{ context.work_dir }}" + on_result: + - when: "${{ result.mergeable_status }} == 'CONFLICTING'" + route: detect_ci_conflict + - route: ci_watch + on_failure: ci_watch + skip_when_false: "inputs.open_pr" + note: > + Pre-CI-watch gate: checks PR mergeable state before entering the CI + polling loop. CONFLICTING PRs cannot trigger pull_request workflows + (GitHub cannot create refs/pull/N/merge), so routing to ci_watch would + produce a guaranteed no_runs. Routes CONFLICTING to detect_ci_conflict + (which handles conflict resolution) and MERGEABLE/UNKNOWN to ci_watch. ci_watch: tool: wait_for_ci with: branch: "${{ context.merge_target }}" - timeout_seconds: 300 + event: "${{ context.ci_event }}" + timeout_seconds: 600 cwd: "${{ context.work_dir }}" remote_url: "${{ context.remote_url }}" + auto_trigger: "true" capture: ci_conclusion: "${{ result.conclusion }}" ci_failed_jobs: "${{ result.failed_jobs }}" - on_success: check_repo_merge_state + on_result: + - when: "${{ result.conclusion }} == 'success'" + route: check_repo_merge_state + - when: "${{ result.conclusion }} == 'no_runs'" + route: handle_no_ci_runs + - when: "${{ result.conclusion }} == 'timed_out'" + route: check_ci_timed_out_loop + - route: detect_ci_conflict on_failure: detect_ci_conflict optional: true skip_when_false: "inputs.open_pr" @@ -642,12 +725,170 @@ steps: using the wait_for_ci MCP tool (timeout: 300 s). Captures structured output (conclusion + failed job names) for downstream resolve_ci. Skipped when open_pr=false — no remote feature branch is opened in that mode. - On success, routes to check_repo_merge_state to determine if the PR should enter - the merge queue. On failure, routes to detect_ci_conflict which checks whether - the failure is caused by a stale base before falling through to diagnose_ci. + Routes on result.conclusion: success → check_repo_merge_state, + no_runs → handle_no_ci_runs (auto_trigger fires first: empty commit + force-push + + re-poll; handle_no_ci_runs reached only when push fails), + timed_out → check_ci_timed_out_loop (iteration guard), failure → detect_ci_conflict. + + check_ci_timed_out_loop: + tool: run_python + with: + callable: "autoskillit.smoke_utils.check_loop_iteration" + current_iteration: "${{ context.ci_timed_out_loop_count }}" + max_iterations: "2" + capture: + ci_timed_out_loop_count: "${{ result.next_iteration }}" + on_result: + - when: "${{ result.max_exceeded }} == true" + route: detect_ci_conflict + - route: ci_watch + on_failure: detect_ci_conflict + optional_context_refs: + - ci_timed_out_loop_count + note: > + Caps the ci_watch re-poll loop at 2 iterations. + After 2 timed_out results (1200s total), routes to detect_ci_conflict + instead of spinning indefinitely. + + handle_no_ci_runs: + tool: check_repo_merge_state + with: + branch: "${{ context.merge_target }}" + cwd: "${{ context.work_dir }}" + remote_url: "${{ context.remote_url }}" + step_name: "handle_no_ci_runs" + capture: + ci_event: "${{ result.ci_event }}" + on_success: check_ci_loop + on_failure: release_issue_failure + retries: 1 + on_exhausted: release_issue_failure + note: > + Intercepts no_runs conclusion from wait_for_ci. Re-derives ci_event using + check_repo_merge_state (the original event may have been wrong for this + branch's workflow configuration) and routes to check_ci_loop for iteration + guard before retrying ci_watch. retries: 1 bounds the tool-call retry; + check_ci_loop bounds the outer loop at 2 iterations. + + check_ci_loop: + tool: run_python + with: + callable: "autoskillit.smoke_utils.check_loop_iteration" + current_iteration: "${{ context.ci_loop_count }}" + max_iterations: "2" + capture: + ci_loop_count: "${{ result.next_iteration }}" + on_result: + - when: "${{ result.max_exceeded }} == true" + route: check_active_trigger_loop + - route: ci_watch + on_failure: check_ci_already_passed + optional_context_refs: + - ci_loop_count + note: > + Iteration guard for the ci_watch / handle_no_ci_runs cycle. Caps the + loop at 2 iterations (initial + 1 retry after re-deriving ci_event). + On budget exhaustion, routes to check_active_trigger_loop which caps + the active trigger to a single attempt before escalating. + + check_active_trigger_loop: + tool: run_python + with: + callable: "autoskillit.smoke_utils.check_loop_iteration" + current_iteration: "${{ context.active_trigger_count }}" + max_iterations: "2" + capture: + active_trigger_count: "${{ result.next_iteration }}" + on_result: + - when: "${{ result.max_exceeded }} == true" + route: check_ci_already_passed + - route: trigger_ci_actively + on_failure: check_ci_already_passed + optional_context_refs: + - active_trigger_count + note: > + Bounds the trigger_ci_actively remediation to a single attempt. If + ci_watch still returns no_runs after the active trigger, escalates + to escalate_stop_no_ci rather than re-triggering indefinitely. + + trigger_ci_actively: + tool: run_cmd + with: + cmd: "gh pr close ${{ context.pr_number }} && sleep 2 && gh pr reopen ${{ context.pr_number }}" + cwd: "${{ context.work_dir }}" + step_name: "trigger_ci_actively" + on_success: ci_watch + on_failure: check_ci_already_passed + retries: 1 + on_exhausted: check_ci_already_passed + note: > + Active CI trigger remediation: closes and reopens the PR to force GitHub + webhook re-delivery. This re-creates the pull_request event and the + refs/pull/N/merge ref (if the PR is mergeable). Routes back to ci_watch + for one final attempt. If the close/reopen fails or the subsequent + ci_watch still returns no_runs, escalates to escalate_stop_no_ci. + + check_ci_already_passed: + tool: run_cmd + with: + cmd: >- + gh pr view '${{ context.pr_number }}' --json statusCheckRollup + --jq '.statusCheckRollup | if length == 0 then "no_checks" elif + all(.state == "SUCCESS" or .state == "NEUTRAL" or .state == "SKIPPED") + then "passed" elif any(.state == "FAILURE" or .state == "ERROR") + then "failed" else "pending" end' + cwd: "${{ context.work_dir }}" + step_name: "check_ci_already_passed" + on_result: + - when: "${{ result.stdout | trim }} == passed" + route: check_repo_merge_state + - when: "${{ result.stdout | trim }} == failed" + route: detect_ci_conflict + - when: "${{ result.stdout | trim }} == pending" + route: ci_watch + - route: mark_issue_failed_no_ci + on_failure: mark_issue_failed_no_ci + skip_when_false: "inputs.open_pr" + note: > + Safety net before terminal escalation: checks if CI already passed + despite the watcher failing to observe it (e.g. post-force-push SHA + staleness). Uses statusCheckRollup as the authoritative signal. + Outputs passed/failed/no_checks/pending: passed goes to merge gate, + failed routes to CI conflict remediation, pending re-enters the CI + watcher, no_checks falls through to terminal stop. + + mark_issue_failed_no_ci: + description: "Swap in-progress → fail label before CI failure escalation" + tool: release_issue + optional: true + skip_when_false: inputs.issue_url + with: + issue_url: ${{ inputs.issue_url }} + fail_label: "fail" + on_success: register_clone_no_ci + on_failure: register_clone_no_ci + + register_clone_no_ci: + description: "Register clone as preserved (no-CI error pipeline)" + tool: register_clone_status + with: + clone_path: "${{ context.work_dir }}" + status: "error" + step_name: register_clone_no_ci + on_success: escalate_stop_no_ci + on_failure: escalate_stop_no_ci + + escalate_stop_no_ci: + action: stop + message: >- + CI safety net: statusCheckRollup returned no checks for PR + ${{ context.pr_number }}. This typically means no workflow is + configured to trigger on this branch pattern. Check .github/workflows/ + trigger configuration. This is NOT a test failure — CI never started. diagnose_ci: tool: run_skill + model: "" with: skill_command: "/autoskillit:diagnose-ci ${{ context.merge_target }} - - - ${{ context.ci_event }}" cwd: "${{ context.work_dir }}" @@ -669,6 +910,7 @@ steps: resolve_ci: tool: run_skill + model: "" with: skill_command: "/autoskillit:resolve-failures ${{ context.work_dir }} ${{ context.plan_path }} ${{ inputs.base_branch }} ${{ context.ci_conclusion }} ${{ context.ci_failed_jobs }} ${{ context.diagnosis_path }}" cwd: "${{ context.work_dir }}" @@ -684,7 +926,7 @@ steps: - when: "${{ result.verdict }} == 'already_green'" route: pre_resolve_rebase - when: "${{ result.verdict }} == 'flake_suspected'" - route: release_issue_failure + route: re_push - when: "${{ result.verdict }} == 'ci_only_failure'" route: release_issue_failure on_failure: release_issue_failure @@ -692,13 +934,17 @@ steps: Runs when ci_watch reports a CI failure. Receives ci_failed_jobs from wait_for_ci structured output to avoid redundant CI failure re-investigation. Routes via on_result: verdict dispatch — real_fix pushes, already_green rebases and re-diagnoses - (sibling fix may have landed), flake_suspected/ci_only_failure escalate to human. + (sibling fix may have landed), flake_suspected retries via re_push (bounded by + retries: 2 / on_exhausted: release_issue_failure), ci_only_failure escalates to human. Retries up to 2 times (3 total attempts) before exhausting to release_issue_failure. pre_resolve_rebase: tool: run_cmd with: - cmd: "git -C ${{ context.work_dir }} fetch origin && git -C ${{ context.work_dir }} rebase origin/${{ inputs.base_branch }}" + cmd: | + REMOTE=$(git -C ${{ context.work_dir }} remote get-url upstream 2>/dev/null | grep -qv "^file://" && echo upstream || echo origin) && + git -C ${{ context.work_dir }} fetch "$REMOTE" && + git -C ${{ context.work_dir }} rebase "$REMOTE"/${{ inputs.base_branch }} step_name: "pre_resolve_rebase" retries: 1 on_success: diagnose_ci @@ -716,12 +962,12 @@ steps: remote_url: "${{ context.remote_url }}" branch: "${{ context.merge_target }}" force: "true" - on_success: ci_watch + on_success: check_repo_ci_event on_failure: release_issue_failure note: > Pushes the CI-fixed feature branch back to the remote after resolve_ci succeeds. - Routes back to ci_watch to verify the fix resolves CI. On push failure, escalates - via release_issue_failure. + Routes to check_repo_ci_event to derive ci_event before ci_watch verifies the fix. + On push failure, escalates via release_issue_failure. check_repo_merge_state: tool: check_repo_merge_state @@ -752,46 +998,70 @@ steps: action: route on_result: - when: "${{ inputs.auto_merge }} != 'true'" - route: register_clone_success - - when: "${{ context.queue_available }} == true and ${{ context.merge_group_trigger }} == true and ${{ context.auto_merge_available }} == true" - route: enable_auto_merge - - when: "${{ context.queue_available }} == true and ${{ context.merge_group_trigger }} == true and ${{ context.auto_merge_available }} == false" - route: queue_enqueue_no_auto + route: patch_token_summary + - when: "${{ context.queue_available }} == true and ${{ context.merge_group_trigger }} == true" + route: enqueue_to_queue - when: "${{ context.auto_merge_available }} == true" route: direct_merge - route: immediate_merge skip_when_false: "inputs.open_pr" - enable_auto_merge: - tool: run_cmd + enqueue_to_queue: + tool: enqueue_pr with: - cmd: "gh pr merge '${{ context.pr_number }}' --squash --auto" + pr_number: "${{ context.pr_number }}" + target_branch: "${{ inputs.base_branch }}" cwd: "${{ context.work_dir }}" - step_name: "enable_auto_merge" + auto_merge_available: "${{ context.auto_merge_available }}" on_success: wait_for_queue - on_failure: register_clone_success + on_failure: verify_queue_enrollment skip_when_false: "inputs.open_pr" note: > - Submits the PR to the merge queue. On failure (PR not mergeable, already merged), - degrades to register_clone_success. - - queue_enqueue_no_auto: - tool: run_cmd + Submits the PR to the merge queue via the unified enqueue_pr tool. + The tool resolves the enrollment strategy internally based on + auto_merge_available. On failure, probes the queue via + verify_queue_enrollment to distinguish a transient enqueue error from a + PR that is already merged or still in queue. + + verify_queue_enrollment: + description: "Probe merge queue after enqueue command failure to determine actual state" + tool: wait_for_merge_queue with: - cmd: "gh pr merge '${{ context.pr_number }}' --squash" + pr_number: "${{ context.pr_number }}" + target_branch: "${{ inputs.base_branch }}" cwd: "${{ context.work_dir }}" - step_name: "queue_enqueue_no_auto" - on_success: wait_for_queue - on_failure: register_clone_success - skip_when_false: "inputs.open_pr" + remote_url: "${{ context.remote_url }}" + timeout_seconds: 60 + poll_interval: 10 + on_result: + - when: "${{ result.pr_state }} == merged" + route: release_issue_success + - when: "${{ result.pr_state }} == ejected_ci_failure" + route: diagnose_ci + - when: "${{ result.pr_state }} == not_enrolled" + route: release_issue_failure + - when: "${{ result.pr_state }} == dropped_healthy" + route: wait_for_queue + - when: "${{ result.pr_state }} == dropped_merge_group_ci" + route: diagnose_ci + - when: "${{ result.pr_state }} == ejected" + route: wait_for_queue + - when: "${{ result.pr_state }} == stalled" + route: wait_for_queue + - when: "${{ result.pr_state }} == timeout" + route: register_clone_unconfirmed + - route: register_clone_unconfirmed + on_failure: register_clone_unconfirmed note: > - Submits the PR to the merge queue when the target branch has a configured - merge queue but the repository disallows auto-merge (autoMergeAllowed=false). - Uses plain `gh pr merge --squash` (no --auto, which would be rejected by - GitHub's API auto-merge gate). GitHub intercepts the call and enqueues - the PR because the branch has a queue. Routes to wait_for_queue (the - shared queue waiter) — never to wait_for_immediate_merge, whose 5-minute - poll is too short for queue serialization. + Short probe (60s) after enqueue_to_queue failure. If the PR is already + merged, route to release_issue_success. If the queue already confirms CI + failure (ejected_ci_failure or dropped_merge_group_ci), route directly to + diagnose_ci. If enrollment was attempted but the PR never appeared in the + queue (not_enrolled), route to release_issue_failure — this is a repo + configuration error. For other active queue states (dropped_healthy, ejected, + stalled), route to wait_for_queue for full 900s authoritative watch. If the + probe timed out or returned an unexpected state, escalate via + register_clone_unconfirmed. wait_for_queue: tool: wait_for_merge_queue @@ -808,47 +1078,103 @@ steps: route: release_issue_success - when: "${{ result.pr_state }} == ejected_ci_failure" route: diagnose_ci + - when: "${{ result.pr_state }} == dropped_merge_group_ci" + route: diagnose_ci + - when: "${{ result.pr_state }} == not_enrolled" + route: release_issue_failure - when: "${{ result.pr_state }} == dropped_healthy" - route: reenter_merge_queue_cheap + route: check_dropped_healthy_loop - when: "${{ result.pr_state }} == ejected" - route: queue_ejected_fix + route: check_eject_limit - when: "${{ result.pr_state }} == stalled" route: reenroll_stalled_pr - when: "${{ result.pr_state }} == timeout" - route: release_issue_timeout - - route: release_issue_timeout - on_failure: release_issue_timeout + route: register_clone_unconfirmed + - route: register_clone_unconfirmed + on_failure: register_clone_unconfirmed skip_when_false: "inputs.open_pr" note: > Waits up to 15 minutes for the PR to merge through the queue. merged → release_issue_success to transition issue state then register_clone_success. ejected_ci_failure → diagnose_ci (CI failure; conflict resolution cannot fix this). + not_enrolled → release_issue_failure (enrollment failed; repo configuration error). ejected → conflict fix cycle then re-enter. - stalled → lightweight re-enrollment (disable then re-enable auto-merge). - timeout → release_issue_timeout (release claim without staged label; merge unconfirmed). + stalled → lightweight re-enrollment via enqueue_pr. + timeout → register_clone_unconfirmed (release claim without staged label; merge unconfirmed). reenroll_stalled_pr: - tool: toggle_auto_merge + tool: enqueue_pr with: pr_number: "${{ context.pr_number }}" target_branch: "${{ inputs.base_branch }}" cwd: "${{ context.work_dir }}" - on_success: wait_for_queue + auto_merge_available: "${{ context.auto_merge_available }}" + on_success: check_stall_loop on_failure: release_issue_failure skip_when_false: "inputs.open_pr" + check_stall_loop: + tool: run_python + with: + callable: "autoskillit.smoke_utils.check_loop_iteration" + current_iteration: "${{ context.stall_loop_count }}" + max_iterations: "3" + capture: + stall_loop_count: "${{ result.next_iteration }}" + on_result: + - when: "${{ result.max_exceeded }} == true" + route: register_clone_unconfirmed + - route: wait_for_queue + on_failure: register_clone_unconfirmed + optional_context_refs: + - stall_loop_count + note: > + Iteration guard for the wait_for_queue / reenroll_stalled_pr cycle. + Caps stall re-enrollment at 3 attempts. On budget exhaustion, routes + to register_clone_unconfirmed (merge outcome unconfirmed). + + check_eject_limit: + tool: run_python + with: + callable: "autoskillit.recipe._cmd_rpc.check_eject_limit" + counter_file: "${{ context.work_dir }}/{{AUTOSKILLIT_TEMP}}/eject_count" + max_ejects: "3" + on_result: + - when: "${{ result.status }} == EJECT_LIMIT_EXCEEDED" + route: release_issue_failure + - route: queue_ejected_fix + on_failure: release_issue_failure + note: > + Circuit breaker for the queue ejection loop. Tracks ejection count in a + file-based counter under {{AUTOSKILLIT_TEMP}}/. After 3 ejections, routes + to release_issue_failure to prevent infinite re-enqueue cycles. + + check_dropped_healthy_loop: + tool: run_python + with: + callable: "autoskillit.recipe._cmd_rpc.check_dropped_healthy_loop" + counter_file: "${{ context.work_dir }}/{{AUTOSKILLIT_TEMP}}/dropped_healthy_count" + max_drops: "2" + on_result: + - when: "${{ result.status }} == DROPPED_LIMIT_EXCEEDED" + route: release_issue_failure + - route: reenter_merge_queue_cheap + on_failure: release_issue_failure + note: > + Circuit breaker for the dropped_healthy re-enrollment loop. After 2 consecutive + unexplained drops with healthy PR-branch CI, escalates to release_issue_failure + instead of looping. Matches the check_eject_limit pattern used for the ejected + path. DROPPED_MERGE_GROUP_CI (confirmed merge-group CI failure) is routed + directly to diagnose_ci and never reaches this step. + queue_ejected_fix: - tool: run_cmd + tool: run_python with: - cmd: > - if git fetch origin ${{ inputs.base_branch }} && git rebase origin/${{ inputs.base_branch }}; - then echo 'clean'; - else git rebase --abort 2>/dev/null || true; echo 'conflicts'; - fi - cwd: "${{ context.work_dir }}" - step_name: "queue_ejected_fix" + callable: "autoskillit.recipe._cmd_rpc.queue_ejected_fix" + work_dir: "${{ context.work_dir }}" + base_branch: "${{ inputs.base_branch }}" on_result: - - when: "${{ result.stdout | trim }} == clean" + - when: "${{ result.status }} == clean" route: re_push_queue_fix - route: resolve_queue_merge_conflicts on_failure: resolve_queue_merge_conflicts @@ -861,6 +1187,7 @@ steps: resolve_queue_merge_conflicts: tool: run_skill + model: "" with: skill_command: "/autoskillit:resolve-merge-conflicts ${{ context.work_dir }} ${{ context.plan_path }} ${{ inputs.base_branch }}" cwd: "${{ context.work_dir }}" @@ -888,35 +1215,106 @@ steps: remote_url: "${{ context.remote_url }}" branch: "${{ context.merge_target }}" force: "true" + capture: + push_error_type: "${{ result.error_type }}" + retries: 2 + on_exhausted: register_clone_unconfirmed on_success: ci_watch_post_queue_fix - on_failure: release_issue_failure + on_failure: classify_push_failure skip_when_false: "inputs.open_pr" + classify_push_failure: + action: route + on_result: + - when: "${{ context.push_error_type }} == queued_branch" + route: wait_dequeue_retry + - route: release_issue_failure + + wait_dequeue_retry: + tool: wait_for_merge_queue + with: + pr_number: "${{ context.pr_number }}" + target_branch: "${{ inputs.base_branch }}" + cwd: "${{ context.work_dir }}" + timeout_seconds: "60" + on_result: + - when: "${{ result.pr_state }} == merged" + route: release_issue_success + - when: "${{ result.pr_state }} == ejected_ci_failure" + route: register_clone_unconfirmed + - when: "${{ result.pr_state }} == dropped_merge_group_ci" + route: register_clone_unconfirmed + - when: "${{ result.pr_state }} == dropped_healthy" + route: register_clone_unconfirmed + - when: "${{ result.pr_state }} == ejected" + route: re_push_queue_fix + - when: "${{ result.pr_state }} == stalled" + route: register_clone_unconfirmed + - when: "${{ result.pr_state }} == timeout" + route: register_clone_unconfirmed + - route: register_clone_unconfirmed + on_failure: register_clone_unconfirmed + optional_context_refs: + - push_error_type + ci_watch_post_queue_fix: tool: wait_for_ci with: branch: "${{ context.merge_target }}" event: "${{ context.ci_event }}" - timeout_seconds: 300 + timeout_seconds: 600 cwd: "${{ context.work_dir }}" remote_url: "${{ context.remote_url }}" + auto_trigger: "true" capture: ci_conclusion: "${{ result.conclusion }}" ci_failed_jobs: "${{ result.failed_jobs }}" - on_success: reenter_merge_queue + on_result: + - when: "${{ result.conclusion }} == 'success'" + route: reenter_merge_queue + - when: "${{ result.conclusion }} == 'no_runs'" + route: release_issue_failure + - when: "${{ result.conclusion }} == 'timed_out'" + route: check_ci_post_queue_loop + - route: detect_ci_conflict on_failure: detect_ci_conflict skip_when_false: "inputs.open_pr" note: > Validates CI on the rebased feature branch after queue ejection fix and re-push. - Mirrors ci_watch: routes to reenter_merge_queue on CI pass, to detect_ci_conflict - on CI failure. Prevents re-entering the merge queue with a broken manifest or code. + Routes on result.conclusion: success → reenter_merge_queue, + no_runs → release_issue_failure (auto_trigger fires first: empty commit + + force-push + re-poll; no_runs reached only when trigger recovery fails), + timed_out → check_ci_post_queue_loop, failure → detect_ci_conflict. + + check_ci_post_queue_loop: + tool: run_python + with: + callable: "autoskillit.smoke_utils.check_loop_iteration" + current_iteration: "${{ context.ci_post_queue_loop_count }}" + max_iterations: "2" + capture: + ci_post_queue_loop_count: "${{ result.next_iteration }}" + on_result: + - when: "${{ result.max_exceeded }} == true" + route: release_issue_failure + - route: ci_watch_post_queue_fix + on_failure: release_issue_failure + optional_context_refs: + - ci_post_queue_loop_count + skip_when_false: "inputs.open_pr" + note: > + Iteration guard for the ci_watch_post_queue_fix timed_out self-loop. + Caps at 2 iterations (initial + 1 retry). On budget exhaustion, + routes to release_issue_failure — this is already a post-ejection + retry path so further escalation is not warranted. reenter_merge_queue: - tool: run_cmd + tool: enqueue_pr with: - cmd: "gh pr merge '${{ context.pr_number }}' --squash --auto" + pr_number: "${{ context.pr_number }}" + target_branch: "${{ inputs.base_branch }}" cwd: "${{ context.work_dir }}" - step_name: "reenter_merge_queue" + auto_merge_available: "${{ context.auto_merge_available }}" on_success: wait_for_queue on_failure: release_issue_failure skip_when_false: "inputs.open_pr" @@ -925,11 +1323,12 @@ steps: If re-entry fails, release_issue_failure preserves the clone for debugging. reenter_merge_queue_cheap: - tool: run_cmd + tool: enqueue_pr with: - cmd: "gh pr merge '${{ context.pr_number }}' --squash --auto" + pr_number: "${{ context.pr_number }}" + target_branch: "${{ inputs.base_branch }}" cwd: "${{ context.work_dir }}" - step_name: "reenter_merge_queue_cheap" + auto_merge_available: "${{ context.auto_merge_available }}" on_success: wait_for_queue on_failure: release_issue_failure skip_when_false: "inputs.open_pr" @@ -945,51 +1344,43 @@ steps: cwd: "${{ context.work_dir }}" step_name: "direct_merge" on_success: wait_for_direct_merge - on_failure: register_clone_success + on_failure: release_issue_failure skip_when_false: "inputs.open_pr" note: > Enables GitHub native auto-merge for direct merge (no merge queue). GitHub - merges automatically once all required checks pass. On failure (PR already - merged, not mergeable), degrades gracefully to register_clone_success. + merges automatically once all required checks pass. On failure, escalates + via release_issue_failure so the issue claim is released and the pipeline + reports an error rather than silently succeeding. wait_for_direct_merge: - tool: run_cmd + tool: run_python with: - cmd: > - for i in $(seq 1 90); do - STATE=$(gh pr view '${{ context.pr_number }}' --json state --jq '.state'); - if [ "$STATE" = "MERGED" ]; then echo "merged"; exit 0; fi; - if [ "$STATE" = "CLOSED" ]; then echo "closed"; exit 0; fi; - sleep 10; - done; echo "timeout" - cwd: "${{ context.work_dir }}" - step_name: "wait_for_direct_merge" + callable: "autoskillit.recipe._cmd_rpc.wait_for_direct_merge" + pr_number: "${{ context.pr_number }}" + max_polls: "90" + poll_interval: "10" on_result: - - when: "${{ result.stdout | trim }} == merged" + - when: "${{ result.state }} == merged" route: release_issue_success - - when: "${{ result.stdout | trim }} == closed" + - when: "${{ result.state }} == closed" route: direct_merge_conflict_fix - - route: release_issue_timeout - on_failure: release_issue_timeout + - route: register_clone_unconfirmed + on_failure: register_clone_unconfirmed skip_when_false: "inputs.open_pr" note: > Polls GitHub PR state every 10 s for up to 15 minutes (90 iterations). merged → release_issue_success (issue labelled, cleanup runs). closed → PR was closed (likely conflict) — route to conflict fix. - timeout / unexpected → release_issue_timeout. + timeout / unexpected → register_clone_unconfirmed. direct_merge_conflict_fix: - tool: run_cmd + tool: run_python with: - cmd: > - if git fetch origin ${{ inputs.base_branch }} && git rebase origin/${{ inputs.base_branch }}; - then echo 'clean'; - else git rebase --abort 2>/dev/null || true; echo 'conflicts'; - fi - cwd: "${{ context.work_dir }}" - step_name: "direct_merge_conflict_fix" + callable: "autoskillit.recipe._cmd_rpc.direct_merge_conflict_fix" + work_dir: "${{ context.work_dir }}" + base_branch: "${{ inputs.base_branch }}" on_result: - - when: "${{ result.stdout | trim }} == clean" + - when: "${{ result.status }} == clean" route: re_push_direct_fix - route: resolve_direct_merge_conflicts on_failure: resolve_direct_merge_conflicts @@ -1001,6 +1392,7 @@ steps: resolve_direct_merge_conflicts: tool: run_skill + model: "" with: skill_command: "/autoskillit:resolve-merge-conflicts ${{ context.work_dir }} ${{ context.plan_path }} ${{ inputs.base_branch }}" cwd: "${{ context.work_dir }}" @@ -1053,33 +1445,29 @@ steps: cwd: "${{ context.work_dir }}" step_name: "immediate_merge" on_success: wait_for_immediate_merge - on_failure: register_clone_success + on_failure: release_issue_failure skip_when_false: "inputs.open_pr" note: > Merges the PR immediately using plain --squash when autoMergeAllowed=false. Unlike direct_merge, no --auto flag is used — GitHub executes the merge - synchronously rather than queueing it for when checks pass. On failure - (conflict or not-mergeable error from GitHub), degrades to register_clone_success. + synchronously rather than queueing it for when checks pass. On failure, + escalates via release_issue_failure so the issue claim is released and the + pipeline reports an error rather than silently succeeding. wait_for_immediate_merge: - tool: run_cmd + tool: run_python with: - cmd: > - for i in $(seq 1 30); do - STATE=$(gh pr view '${{ context.pr_number }}' --json state --jq '.state'); - if [ "$STATE" = "MERGED" ]; then echo "merged"; exit 0; fi; - if [ "$STATE" = "CLOSED" ]; then echo "closed"; exit 0; fi; - sleep 10; - done; echo "timeout" - cwd: "${{ context.work_dir }}" - step_name: "wait_for_immediate_merge" + callable: "autoskillit.recipe._cmd_rpc.wait_for_immediate_merge" + pr_number: "${{ context.pr_number }}" + max_polls: "30" + poll_interval: "10" on_result: - - when: "${{ result.stdout | trim }} == merged" + - when: "${{ result.state }} == merged" route: release_issue_success - - when: "${{ result.stdout | trim }} == closed" + - when: "${{ result.state }} == closed" route: immediate_merge_conflict_fix - - route: release_issue_timeout - on_failure: release_issue_timeout + - route: register_clone_unconfirmed + on_failure: register_clone_unconfirmed skip_when_false: "inputs.open_pr" note: > Polls GitHub PR state every 10s for up to 5 minutes (30 iterations). @@ -1087,20 +1475,16 @@ steps: synchronously — this poll is a confirmation window. merged → release_issue_success. closed → PR was rejected (stale base or conflict) — route to conflict fix. - timeout → release_issue_timeout. + timeout → register_clone_unconfirmed. immediate_merge_conflict_fix: - tool: run_cmd + tool: run_python with: - cmd: > - if git fetch origin ${{ inputs.base_branch }} && git rebase origin/${{ inputs.base_branch }}; - then echo 'clean'; - else git rebase --abort 2>/dev/null || true; echo 'conflicts'; - fi - cwd: "${{ context.work_dir }}" - step_name: "immediate_merge_conflict_fix" + callable: "autoskillit.recipe._cmd_rpc.immediate_merge_conflict_fix" + work_dir: "${{ context.work_dir }}" + base_branch: "${{ inputs.base_branch }}" on_result: - - when: "${{ result.stdout | trim }} == clean" + - when: "${{ result.status }} == clean" route: re_push_immediate_fix - route: resolve_immediate_merge_conflicts on_failure: resolve_immediate_merge_conflicts @@ -1112,6 +1496,7 @@ steps: resolve_immediate_merge_conflicts: tool: run_skill + model: "" with: skill_command: "/autoskillit:resolve-merge-conflicts ${{ context.work_dir }} ${{ context.plan_path }} ${{ inputs.base_branch }}" cwd: "${{ context.work_dir }}" @@ -1157,25 +1542,28 @@ steps: Routes back to wait_for_immediate_merge to confirm completion. On failure, release_issue_failure preserves the clone for debugging. - release_issue_timeout: - description: "Release in-progress claim after queue timeout without applying staged label" - tool: release_issue - optional: true - skip_when_false: inputs.issue_url + register_clone_unconfirmed: + description: "Register clone as preserved — merge unconfirmed, in-progress label retained" + tool: register_clone_status with: - issue_url: ${{ inputs.issue_url }} - on_success: register_clone_success - on_failure: register_clone_success + clone_path: "${{ context.work_dir }}" + status: "unconfirmed" + step_name: register_clone_unconfirmed + on_success: done + on_failure: done note: > - Called when wait_for_merge_queue times out or errors. Releases the in-progress - claim without passing target_branch, so the staged label is not applied. The - merge is unconfirmed; a human must verify queue state independently. - Routes to register_clone_success so the pipeline completes cleanly. + Called when a merge-wait step times out before merge is confirmed. + Does NOT call release_issue — the in-progress label stays on the issue + so operators can see the PR is still actively queued. The clone is + preserved (status: unconfirmed) for human inspection. detect_ci_conflict: tool: run_cmd with: - cmd: "git fetch origin ${{ inputs.base_branch }} && ! git merge-base --is-ancestor origin/${{ inputs.base_branch }} HEAD" + cmd: | + REMOTE=$(git remote get-url upstream 2>/dev/null | grep -qv "^file://" && echo upstream || echo origin) && + git fetch "$REMOTE" ${{ inputs.base_branch }} && + ! git merge-base --is-ancestor "$REMOTE"/${{ inputs.base_branch }} HEAD cwd: "${{ context.work_dir }}" step_name: "detect_ci_conflict" on_success: ci_conflict_fix @@ -1191,6 +1579,7 @@ steps: ci_conflict_fix: tool: run_skill + model: "" with: skill_command: "/autoskillit:resolve-merge-conflicts ${{ context.work_dir }} ${{ context.plan_path }} ${{ inputs.base_branch }}" cwd: "${{ context.work_dir }}" @@ -1221,14 +1610,29 @@ steps: with: issue_url: ${{ inputs.issue_url }} target_branch: ${{ inputs.base_branch }} - on_success: register_clone_success - on_failure: register_clone_success + on_success: patch_token_summary + on_failure: patch_token_summary note: > Called on the success path after PR merge is confirmed. Passes target_branch so release_issue can determine whether to apply the staged label (non-default target) or just remove in-progress (default target). optional=true ensures pipeline still reaches register_clone_success if this step errors. + patch_token_summary: + description: "Collect all pipeline token data and patch the PR body with complete summary" + tool: run_python + optional: true + skip_when_false: "inputs.open_pr" + with: + callable: "autoskillit.smoke_utils.patch_pr_token_summary" + timeout: 60 + pr_url: "${{ context.pr_url }}" + cwd: "${{ context.work_dir }}" + on_success: register_clone_success + on_failure: register_clone_success + note: > + Patches PR body with complete token summary; failure never blocks pipeline completion. + release_issue_failure: description: "Release the in-progress claim after pipeline failure" tool: release_issue @@ -1236,6 +1640,7 @@ steps: skip_when_false: inputs.issue_url with: issue_url: ${{ inputs.issue_url }} + fail_label: "fail" on_success: register_clone_failure on_failure: register_clone_failure diff --git a/src/autoskillit/recipes/implementation.json b/src/autoskillit/recipes/implementation.json new file mode 100644 index 0000000000..a4be5d14b5 --- /dev/null +++ b/src/autoskillit/recipes/implementation.json @@ -0,0 +1,1790 @@ +{ + "name": "implementation", + "description": "Plan, verify, implement, test, and merge a task end-to-end. Use when user says \"run pipeline\", \"implement task\", or \"auto implement\".", + "summary": "bootstrap_clone > (claim_and_resolve?) > (create_and_publish?) > make-plan > (review-approach?) > dry-walkthrough > implement > test > merge (per plan part) > (audit-impl?) > (open_pr?) > push > cleanup", + "recipe_version": "1.0.0", + "requires_packs": [ + "github", + "ci", + "clone", + "telemetry" + ], + "ingredients": { + "task": { + "description": "What to implement", + "required": true + }, + "source_dir": { + "description": "Source repository", + "required": false, + "default": "" + }, + "run_name": { + "description": "Run name prefix", + "default": "impl" + }, + "base_branch": { + "description": "Merge target", + "default": "" + }, + "review_approach": { + "description": "Research approaches first", + "default": "false" + }, + "audit": { + "description": "Post-merge quality gate", + "default": "false" + }, + "open_pr": { + "description": "PR instead of direct merge", + "default": "true" + }, + "arch_lenses": { + "description": "Generate architecture lens diagrams for the PR", + "default": "false" + }, + "auto_merge": { + "description": "Automatically merge the PR after required checks pass — via merge queue when available, direct merge otherwise", + "default": "true" + }, + "review_max_retries": { + "description": "Maximum number of review-resolve retry cycles before proceeding to CI", + "default": "3" + }, + "local_review_rounds": { + "description": "Number of review-resolve cycles to run locally before switching to GitHub-backed review. 0 disables local mode entirely.", + "type": "integer", + "default": "3" + }, + "issue_url": { + "description": "GitHub issue to close on merge", + "required": false + }, + "upfront_claimed": { + "description": "Set to 'true' when process-issues has already claimed this issue upfront. Allows claim_issue defense step to pass through without aborting.", + "default": "false", + "hidden": true + }, + "run_mode": { + "description": "Execution mode when processing multiple issues. Options: sequential (default, one issue at a time) or parallel (all issues simultaneously with wavefront step scheduling — fast steps for all pipelines before any slow step).", + "default": "sequential", + "hidden": true + }, + "max_parallel": { + "description": "Maximum number of issues to run in parallel within a single execution-map group. Groups exceeding this cap are split into sequential sub-groups. Passed to build-execution-map. Must be a positive integer (expressed as a string).", + "default": "6", + "hidden": true + } + }, + "kitchen_rules": [ + "NEVER use native Claude Code tools (Read, Grep, Glob, Edit, Write, Bash, Agent, WebFetch, WebSearch, NotebookEdit) from the orchestrator. All work is delegated through run_skill.", + "Route to on_failure — never investigate or fix directly from the orchestrator.", + "Process plan parts sequentially. Complete the full cycle (verify → implement → test → merge) for one part before starting the next. Do NOT run verify for all parts upfront.", + "By default (open_pr=true), a feature branch is created with a unique name derived from inputs.run_name and context.issue_number (e.g. impl/124) or a date suffix (e.g. impl/20260304) when no issue is available. All worktree merges target the feature branch (context.merge_target), not base_branch directly. The push step publishes the feature branch, then prepare_pr → run_arch_lenses → compose_pr opens a PR to base_branch. When open_pr=false, merges target base_branch directly and prepare_pr is skipped.", + "When arch_lenses=true (and open_pr=true), prepare_pr routes through run_arch_lenses before compose_pr, adding architecture lens diagrams to the PR. By default arch_lenses is off and prepare_pr routes directly to compose_pr.", + "SOURCE ISOLATION: After bootstrap_clone returns, the source_dir is strictly off-limits. Never run any command in source_dir — no git checkout, git fetch, git reset, git pull, run_cmd, run_skill, or any other operation. All work — skill invocations, git operations, file reads — happens exclusively in the clone (work_dir). source_dir is used ONLY to read the remote URL inside push_to_remote.", + "MERGE ROUTING: check_repo_merge_state performs a single GraphQL round-trip to detect queue_available, merge_group_trigger, and auto_merge_available for the target branch. route_queue_mode then routes to: enable_auto_merge → wait_for_queue ONLY when queue_available=true AND merge_group_trigger=true; direct_merge → wait_for_direct_merge (no queue or trigger missing, autoMergeAllowed=true); or immediate_merge → wait_for_immediate_merge (no queue, autoMergeAllowed=false). Never call gh pr merge directly via run_cmd — all merges go through these named recipe steps so CI enforcement and conflict routing are preserved." + ], + "steps": { + "clone": { + "tool": "bootstrap_clone", + "with": { + "source_dir": "${{ inputs.source_dir }}", + "run_name": "${{ inputs.run_name }}", + "base_branch": "${{ inputs.base_branch }}", + "step_name": "bootstrap_clone" + }, + "capture": { + "work_dir": "${{ result.work_dir }}", + "remote_url": "${{ result.remote_url }}", + "base_sha": "${{ result.base_sha }}", + "merge_target": "${{ result.merge_target }}" + }, + "on_success": "claim_and_resolve", + "on_failure": "escalate_stop", + "note": "Clones source_dir and captures base_sha + merge_target in one turn. Returns work_dir, remote_url, base_sha (SHA of base_branch before any merge), and merge_target (set to base_branch as fallback; overridden by create_and_publish when open_pr=true). On failure, escalate_stop immediately (no clone to remove).\n" + }, + "claim_and_resolve": { + "tool": "claim_and_resolve_issue", + "optional": true, + "skip_when_false": "inputs.issue_url", + "with": { + "issue_url": "${{ inputs.issue_url }}", + "allow_reentry": "${{ inputs.upfront_claimed }}" + }, + "capture": { + "issue_number": "${{ result.issue_number }}", + "issue_title": "${{ result.issue_title }}", + "issue_slug": "${{ result.issue_slug }}", + "claimed": "${{ result.claimed }}" + }, + "on_result": [ + { + "when": "${{ result.claimed }} == true", + "route": "create_and_publish" + }, + { + "route": "escalate_stop" + } + ], + "on_failure": "escalate_stop", + "note": "Fetches issue title/slug and claims the issue in one turn. Returns issue_number, issue_title, issue_slug, and claimed boolean. When claimed=false (another session owns it), routes to escalate_stop. issue_slug and issue_number are passed to create_and_publish for branch naming.\n" + }, + "create_and_publish": { + "tool": "create_and_publish_branch", + "optional": true, + "skip_when_false": "inputs.open_pr", + "with": { + "issue_slug": "${{ context.issue_slug }}", + "run_name": "${{ inputs.run_name }}", + "issue_number": "${{ context.issue_number }}", + "work_dir": "${{ context.work_dir }}", + "remote_url": "${{ context.remote_url }}", + "step_name": "create_and_publish_branch" + }, + "capture": { + "merge_target": "${{ result.merge_target }}" + }, + "on_success": "plan", + "on_failure": "release_issue_failure", + "note": "Computes branch name, creates it (with collision suffix if needed), and pushes to origin in one turn. Overrides merge_target with the feature branch name. merge_worktree requires refs/remotes/origin/{branch} to exist after fetch (step 5.5). When open_pr=false, this step is skipped and merge_target remains as base_branch.\n" + }, + "plan": { + "tool": "run_skill", + "model": "", + "with": { + "skill_command": "/autoskillit:make-plan ${{ inputs.task }}", + "cwd": "${{ context.work_dir }}", + "step_name": "plan", + "issue_url": "${{ inputs.issue_url }}", + "issue_title": "${{ context.issue_title }}" + }, + "capture": { + "plan_path": "${{ result.plan_path }}", + "all_plan_paths": "${{ result.plan_path }}" + }, + "capture_list": { + "plan_parts": "${{ result.plan_parts }}" + }, + "on_success": "review", + "on_failure": "release_issue_failure", + "note": "Produces plan in {{AUTOSKILLIT_TEMP}}/make-plan/. Glob plan_dir for *_part_*.md or single plan file. plan_parts context key contains the full ordered list of part files (sorted alphabetically). For a single-part plan, plan_parts has one entry equal to plan_path. REMEDIATION MODE: context.remediation_path is available when the orchestrator re-enters plan from the remediate step; pass it inline in the skill_command rather than as a named with: key. ACCUMULATION: all_plan_paths accumulates across any re-entries to the plan step (e.g. remediation). For a single-pass run, all_plan_paths equals plan_path (the capture block handles this automatically). ISSUE CONTEXT: If inputs.issue_url is a non-empty string, append it to the skill_command so the make-plan skill can detect and fetch the issue itself: \"/autoskillit:make-plan {task}\\n\\nGitHub Issue: {inputs.issue_url}\" The skill detects the URL pattern and calls fetch_github_issue internally.\n" + }, + "review": { + "tool": "run_skill", + "model": "", + "with": { + "skill_command": "/autoskillit:review-approach ${{ context.plan_path }}", + "cwd": "${{ context.work_dir }}", + "step_name": "review" + }, + "capture": { + "review_path": "${{ result.review_path }}" + }, + "retries": 1, + "on_success": "verify", + "on_failure": "release_issue_failure", + "on_context_limit": "verify", + "skip_when_false": "inputs.review_approach", + "note": "Only execute this step if review_approach input is 'true'. Otherwise skip directly to verify." + }, + "verify": { + "tool": "run_skill", + "model": "", + "with": { + "skill_command": "/autoskillit:dry-walkthrough ${{ context.plan_path }}", + "cwd": "${{ context.work_dir }}", + "step_name": "verify", + "review_path": "${{ context.review_path }}" + }, + "on_success": "implement", + "on_failure": "release_issue_failure", + "on_context_limit": "implement", + "note": "SEQUENTIAL EXECUTION: For each plan_part, run the FULL cycle — verify → implement → test → merge — for that part before advancing to the next part. Never run verify for all parts first. If review produced a report, context.review_path is available — pass it as an additional arg to dry-walkthrough." + }, + "implement": { + "tool": "run_skill", + "model": "", + "stale_threshold": 2400, + "with": { + "skill_command": "/autoskillit:implement-worktree-no-merge ${{ context.plan_path }}", + "cwd": "${{ context.work_dir }}", + "step_name": "implement" + }, + "capture": { + "worktree_path": "${{ result.worktree_path }}" + }, + "retries": 0, + "on_context_limit": "retry_worktree", + "on_success": "test", + "on_failure": "release_issue_failure", + "note": "Extracts worktree_path from result for use in subsequent steps." + }, + "retry_worktree": { + "tool": "run_skill", + "model": "", + "stale_threshold": 2400, + "with": { + "skill_command": "/autoskillit:retry-worktree ${{ context.plan_path }} ${{ context.worktree_path }}", + "cwd": "${{ context.worktree_path }}", + "step_name": "retry_worktree" + }, + "capture": { + "worktree_path": "${{ result.worktree_path }}", + "phases_implemented": "${{ result.phases_implemented }}" + }, + "on_result": [ + { + "when": "${{ result.phases_implemented }} > 0", + "route": "test" + }, + { + "route": "test" + } + ], + "on_exhausted": "release_issue_failure", + "on_failure": "release_issue_failure" + }, + "test": { + "tool": "test_check", + "with": { + "worktree_path": "${{ context.worktree_path }}" + }, + "on_success": "check_merge_fix_loop", + "on_failure": "fix" + }, + "check_merge_fix_loop": { + "tool": "run_python", + "with": { + "callable": "autoskillit.smoke_utils.check_loop_iteration", + "current_iteration": "${{ context.merge_fix_count }}", + "max_iterations": "3" + }, + "capture": { + "merge_fix_count": "${{ result.next_iteration }}" + }, + "on_result": [ + { + "when": "${{ result.max_exceeded }} == true", + "route": "release_issue_failure" + }, + { + "route": "commit_guard" + } + ], + "on_failure": "release_issue_failure", + "optional_context_refs": [ + "merge_fix_count" + ], + "note": "Iteration guard for the merge→fix→test cycle. Caps the loop at 3 iterations (enough for legitimate flake recovery but not enough to burn 30+ minutes on structurally unresolvable conflicts).\n" + }, + "commit_guard": { + "tool": "run_python", + "with": { + "callable": "autoskillit.recipe._cmd_rpc.commit_guard", + "worktree_path": "${{ context.worktree_path }}" + }, + "on_success": "merge", + "on_failure": "merge", + "note": "Belt-and-suspenders guard: auto-commits any uncommitted changes before merge. Catches dirty state from context-exhausted skills, cascading pre-commit auto-fixes, or any other source. on_failure routes to merge (dirty-tree gate will catch real issues).\n" + }, + "merge": { + "tool": "merge_worktree", + "with": { + "worktree_path": "${{ context.worktree_path }}", + "base_branch": "${{ context.merge_target }}" + }, + "capture": { + "cleanup_succeeded": "${{ result.cleanup_succeeded }}", + "worktree_path": "${{ result.worktree_path }}" + }, + "on_result": [ + { + "when": "result.failed_step == 'dirty_tree'", + "route": "fix" + }, + { + "when": "result.failed_step == 'test_gate'", + "route": "fix" + }, + { + "when": "result.failed_step == 'post_rebase_test_gate'", + "route": "fix" + }, + { + "when": "result.failed_step == 'rebase'", + "route": "rebase_conflict_fix" + }, + { + "when": "result.error", + "route": "release_issue_failure" + }, + { + "route": "next_or_done" + } + ], + "on_failure": "release_issue_failure", + "note": "After merge, route to next_or_done to process remaining parts before pushing. By default (open_pr=true), merge_target is the feature branch named from run_name. When open_pr=false, merge_target equals base_branch — merges go directly to base. dirty_tree and test_gate failures route to fix (resolve-failures) for recovery; all other failures route to release_issue_failure.\n" + }, + "push": { + "tool": "push_to_remote", + "with": { + "clone_path": "${{ context.work_dir }}", + "remote_url": "${{ context.remote_url }}", + "branch": "${{ context.merge_target }}" + }, + "on_success": "prepare_pr", + "on_failure": "release_issue_failure", + "note": "Runs once at the end, after audit_impl passes. Pushes merge_target branch from the clone to the upstream remote. By default (open_pr=true), this pushes the feature branch; when open_pr=false, pushes base_branch. Uses the pre-resolved remote_url captured at clone time, preserving clone isolation. On success, routes to prepare_pr (which opens the PR by default; skipped when open_pr=false). On failure, release_issue_failure registers the clone and escalates.\n" + }, + "fix": { + "tool": "run_skill", + "model": "", + "with": { + "skill_command": "/autoskillit:resolve-failures ${{ context.worktree_path }} ${{ context.plan_path }} ${{ inputs.base_branch }}", + "cwd": "${{ context.worktree_path }}", + "step_name": "fix" + }, + "capture": { + "verdict": "${{ result.verdict }}", + "fixes_applied": "${{ result.fixes_applied }}" + }, + "on_result": [ + { + "when": "${{ result.verdict }} == 'real_fix'", + "route": "test" + }, + { + "when": "${{ result.verdict }} == 'already_green'", + "route": "test" + }, + { + "when": "${{ result.verdict }} == 'flake_suspected'", + "route": "test" + }, + { + "when": "${{ result.verdict }} == 'ci_only_failure'", + "route": "release_issue_failure" + } + ], + "on_failure": "release_issue_failure", + "on_context_limit": "test", + "note": "Fixes test failures without merging. Routes back to test for re-validation before merge_worktree. on_context_limit routes needs_retry=True to test (worktree may already be green) instead of re-running fix blindly." + }, + "rebase_conflict_fix": { + "tool": "run_skill", + "model": "", + "with": { + "skill_command": "/autoskillit:resolve-merge-conflicts ${{ context.worktree_path }} ${{ context.plan_path }} ${{ inputs.base_branch }}", + "cwd": "${{ context.worktree_path }}", + "step_name": "rebase_conflict_fix" + }, + "capture": { + "conflict_escalation_required": "${{ result.escalation_required }}" + }, + "on_result": [ + { + "when": "${{ result.escalation_required }} == true", + "route": "release_issue_failure" + }, + { + "route": "test" + } + ], + "on_failure": "release_issue_failure", + "on_context_limit": "release_issue_failure", + "retries": 1, + "on_exhausted": "release_issue_failure" + }, + "next_or_done": { + "action": "route", + "on_result": [ + { + "when": "${{ result.next }} == more_parts", + "route": "verify" + }, + { + "route": "audit_impl" + } + ], + "note": "Routing step — no tool call. The agent evaluates what comes next: 1. If more plan_parts remain for the current plan → go to verify with next part. 2. If all plan_parts are complete → go to audit_impl.\n" + }, + "audit_impl": { + "tool": "run_skill", + "model": "", + "with": { + "skill_command": "/autoskillit:audit-impl ${{ context.plan_path }} ${{ context.base_sha }} ${{ inputs.base_branch }}", + "cwd": "${{ context.work_dir }}", + "step_name": "audit_impl" + }, + "capture": { + "remediation_path": "${{ result.remediation_path }}" + }, + "on_result": [ + { + "when": "${{ result.verdict }} == GO", + "route": "push" + }, + { + "when": "result.error", + "route": "escalate_stop" + }, + { + "route": "remediate" + } + ], + "on_failure": "escalate_stop", + "on_context_limit": "escalate_stop", + "optional": true, + "skip_when_false": "inputs.audit", + "note": "Only execute if inputs.audit is 'true'. If false, skip directly to push. Captures verdict and remediation_path. Routes GO → push, NO GO → remediate which re-enters the plan step with the remediation file. All parts have been merged at this point — no worktree exists. Uses context.base_sha (a stable commit SHA captured before any merge began) as implementation_ref. A SHA names a git object and survives git branch -D unconditionally. The diff git diff base_sha..base_branch (two-dot, SHA on the left) covers the full implementation.\n" + }, + "remediate": { + "action": "route", + "with": { + "remediation_path": "${{ context.remediation_path }}" + }, + "on_success": "plan" + }, + "prepare_pr": { + "tool": "run_skill", + "model": "", + "retries": 1, + "with": { + "skill_command": "/autoskillit:prepare-pr ${{ context.all_plan_paths }} ${{ inputs.run_name }} ${{ inputs.base_branch }}", + "cwd": "${{ context.work_dir }}", + "step_name": "prepare_pr", + "issue_number": "${{ context.issue_number }}" + }, + "capture": { + "prep_path": "${{ result.prep_path }}", + "selected_lenses": "${{ result.selected_lenses }}", + "lens_context_paths": "${{ result.lens_context_paths }}" + }, + "on_result": [ + { + "when": "${{ result.prep_path }} and ${{ inputs.arch_lenses }} == 'true'", + "route": "run_arch_lenses" + }, + { + "when": "${{ result.prep_path }}", + "route": "compose_pr" + }, + { + "route": "release_issue_failure" + } + ], + "on_failure": "release_issue_failure", + "on_context_limit": "prepare_pr", + "on_exhausted": "release_issue_failure", + "optional": true, + "skip_when_false": "inputs.open_pr", + "note": "Reads plan(s), runs git diff, classifies changed files, selects 1–3 arch-lens slugs, writes one PR context file per lens, and writes a PR prep file. Does NOT invoke arch-lens skills. If context.issue_number is a non-empty string, append it as the 4th positional argument so prepare-pr fetches requirements from the closing issue.\n" + }, + "run_arch_lenses": { + "tool": "run_skill", + "model": "", + "with": { + "skill_command": "/autoskillit:arch-lens-{slug} {context_path}", + "cwd": "${{ context.work_dir }}", + "step_name": "run_arch_lenses" + }, + "capture_list": { + "all_diagram_paths": "${{ result.diagram_path }}" + }, + "on_success": "compose_pr", + "on_failure": "compose_pr", + "on_context_limit": "compose_pr", + "optional": true, + "skip_when_false": "inputs.open_pr", + "note": "LENS ITERATION: context.selected_lenses contains comma-separated arch-lens slugs (e.g. \"module-dependency,process-flow\"). context.lens_context_paths contains the corresponding comma-separated context file paths (same order as slugs). For each lens slug, run:\n /autoskillit:arch-lens-{slug} {matching_context_path}\nRun all selected lenses in parallel (all run_skill calls in a single message). Accumulate each diagram_path value into context.all_diagram_paths via capture_list. If a lens fails, continue with remaining lenses — compose_pr handles empty lists gracefully by omitting the Architecture Impact section.\n" + }, + "compose_pr": { + "tool": "run_skill", + "model": "", + "with": { + "skill_command": "/autoskillit:compose-pr ${{ context.prep_path }} \"${{ context.all_diagram_paths }}\" ${{ context.work_dir }} ${{ inputs.base_branch }}", + "cwd": "${{ context.work_dir }}", + "step_name": "compose_pr", + "issue_number": "${{ context.issue_number }}" + }, + "capture": { + "pr_url": "${{ result.pr_url }}" + }, + "on_success": "extract_pr_number", + "on_failure": "release_issue_failure", + "on_context_limit": "release_issue_failure", + "optional": true, + "skip_when_false": "inputs.open_pr", + "note": "Reads the PR prep file, validates arch-lens diagrams (must contain ★ or ●), composes the PR body, and runs gh pr create. Degrades gracefully if gh is unavailable (emits pr_url=). If context.issue_number is set, append it as the 5th positional argument for Closes #N injection in the PR body.\n" + }, + "extract_pr_number": { + "tool": "run_cmd", + "with": { + "cmd": "gh pr view '${{ context.merge_target }}' --json number -q .number", + "cwd": "${{ context.work_dir }}", + "step_name": "extract_pr_number" + }, + "capture": { + "pr_number": "${{ result.stdout | trim }}" + }, + "on_success": "annotate_pr_diff", + "on_failure": "release_issue_failure", + "optional": true, + "skip_when_false": "inputs.open_pr", + "note": "Extracts the PR number as a string for downstream steps. pr_number is required by the queue finale steps (enable_auto_merge, wait_for_queue).\n" + }, + "annotate_pr_diff": { + "tool": "run_python", + "with": { + "callable": "autoskillit.smoke_utils.annotate_pr_diff", + "pr_number": "${{ context.pr_number }}", + "cwd": "${{ context.work_dir }}", + "output_dir": "${{ context.work_dir }}/{{AUTOSKILLIT_TEMP}}/review-pr", + "base_branch": "${{ inputs.base_branch }}", + "local_review_rounds": "${{ inputs.local_review_rounds }}", + "current_iteration": "${{ context.review_loop_count }}" + }, + "capture": { + "annotated_diff_path": "${{ result.annotated_diff_path }}", + "hunk_ranges_path": "${{ result.hunk_ranges_path }}", + "diff_metrics_path": "${{ result.diff_metrics_path }}", + "review_mode": "${{ result.review_mode }}" + }, + "on_success": "review_pr", + "on_failure": "review_pr", + "optional": true, + "skip_when_false": "inputs.open_pr", + "optional_context_refs": [ + "review_loop_count" + ], + "note": "Pre-computes annotated diff with [LNNN] markers and hunk ranges JSON for review-pr. On failure, falls through to review_pr which uses empty fallback.\n" + }, + "review_pr": { + "tool": "run_skill", + "model": "", + "with": { + "skill_command": "/autoskillit:review-pr ${{ context.merge_target }} ${{ inputs.base_branch }} annotated_diff_path=${{ context.annotated_diff_path }} hunk_ranges_path=${{ context.hunk_ranges_path }} diff_metrics_path=${{ context.diff_metrics_path }} mode=${{ context.review_mode }}", + "cwd": "${{ context.work_dir }}", + "step_name": "review_pr" + }, + "capture": { + "review_verdict": "${{ result.verdict }}" + }, + "on_result": [ + { + "when": "${{ result.verdict }} == changes_requested", + "route": "enrich_diff_context" + }, + { + "when": "${{ result.verdict }} == approved_with_comments", + "route": "enrich_diff_context" + }, + { + "when": "${{ result.verdict }} == needs_human", + "route": "check_review_loop" + }, + { + "when": "${{ result.verdict }} == approved", + "route": "check_review_loop" + }, + { + "when": "true", + "route": "check_review_loop" + } + ], + "on_failure": "check_repo_ci_event", + "on_context_limit": "check_repo_ci_event", + "optional": true, + "skip_when_false": "inputs.open_pr", + "optional_context_refs": [ + "review_loop_count", + "review_mode" + ], + "note": "Runs after compose_pr creates the PR. Invokes the review-pr skill as a headless session to leave inline review comments on the PR. The skill finds the open PR by feature branch name (context.merge_target). Captures verdict as review_verdict and routes via on_result: changes_requested/approved_with_comments → enrich_diff_context → resolve_review; all other verdicts (including approved, needs_human) → check_review_loop (mandatory waypoint to increment review_loop_count). On tool-level failure (MCP error, gh unavailable), routes to check_repo_ci_event (no review to resolve). Skipped when open_pr=false.\n" + }, + "enrich_diff_context": { + "tool": "run_python", + "with": { + "callable": "autoskillit.smoke_utils.enrich_diff_context", + "pr_number": "${{ context.pr_number }}", + "work_dir": "${{ context.work_dir }}" + }, + "on_success": "resolve_review", + "on_failure": "resolve_review", + "optional": true, + "skip_when_false": "inputs.open_pr", + "note": "Enriches the review-pr diff_context handoff file with deterministic code_region extraction from the annotated diff. On failure, falls through to resolve_review which uses fallback file-read behavior.\n" + }, + "resolve_review": { + "tool": "run_skill", + "model": "", + "with": { + "skill_command": "/autoskillit:resolve-review ${{ context.merge_target }} ${{ inputs.base_branch }} mode=${{ context.review_mode }}", + "cwd": "${{ context.work_dir }}", + "step_name": "resolve_review" + }, + "capture": { + "verdict": "${{ result.verdict }}" + }, + "retries": 2, + "on_exhausted": "release_issue_failure", + "on_result": [ + { + "when": "${{ result.verdict }} == 'real_fix'", + "route": "re_push_review" + }, + { + "when": "${{ result.verdict }} == 'already_green'", + "route": "re_push_review" + }, + { + "when": "${{ result.verdict }} == 'flake_suspected'", + "route": "re_push_review" + }, + { + "when": "${{ result.verdict }} == 'ci_only_failure'", + "route": "release_issue_failure" + } + ], + "on_failure": "release_issue_failure", + "optional_context_refs": [ + "review_mode" + ], + "note": "Runs when review_pr reports issues (verdict=changes_requested). The clone (context.work_dir) has the feature branch (context.merge_target) checked out. resolve-review fetches inline and summary review comments via the GitHub API, applies fixes for each actionable finding (critical + warning severity), and commits each fix. Routes via on_result: verdict dispatch — real_fix/already_green/ flake_suspected all route to re_push_review (retry bounded by retries: 2); ci_only_failure escalates to release_issue_failure.\n" + }, + "re_push_review": { + "tool": "push_to_remote", + "with": { + "clone_path": "${{ context.work_dir }}", + "remote_url": "${{ context.remote_url }}", + "branch": "${{ context.merge_target }}" + }, + "on_success": "check_review_loop", + "on_failure": "release_issue_failure", + "note": "Pushes the review-fixed feature branch back to the remote after resolve_review succeeds. Routes to check_review_loop to determine if another review-resolve cycle is needed (bounded by review_max_retries). On push failure, escalates via release_issue_failure.\n" + }, + "check_review_loop": { + "tool": "run_python", + "with": { + "callable": "autoskillit.smoke_utils.check_review_loop", + "pr_number": "${{ context.pr_number }}", + "current_iteration": "${{ context.review_loop_count }}", + "max_iterations": "${{ inputs.review_max_retries }}", + "previous_verdict": "${{ context.review_verdict }}", + "local_review_rounds": "${{ inputs.local_review_rounds }}" + }, + "capture": { + "review_loop_count": "${{ result.next_iteration }}" + }, + "on_result": [ + { + "when": "${{ result.had_blocking }} == true and ${{ result.max_exceeded }} == false", + "route": "review_pr" + }, + { + "route": "check_repo_ci_event" + } + ], + "on_failure": "check_repo_ci_event", + "skip_when_false": "inputs.open_pr", + "optional_context_refs": [ + "review_loop_count", + "review_verdict" + ], + "note": "Compound iteration + blocking guard. Routes back to review_pr only when had_blocking=true AND max_exceeded=false. had_blocking is true when previous_verdict=changes_requested OR (local_review_rounds > 0 AND iteration < local_review_rounds), ensuring local review rounds are fully exhausted before exiting to CI.\n" + }, + "check_repo_ci_event": { + "tool": "check_repo_merge_state", + "with": { + "branch": "${{ context.merge_target }}", + "cwd": "${{ context.work_dir }}", + "remote_url": "${{ context.remote_url }}", + "step_name": "check_repo_ci_event" + }, + "capture": { + "ci_event": "${{ result.ci_event }}" + }, + "on_success": "check_pr_state", + "on_failure": "ci_watch", + "skip_when_false": "inputs.open_pr", + "note": "Derives ci_event for the feature branch BEFORE ci_watch runs. Uses context.merge_target (feature branch) so push.branches filter is evaluated against the actual branch being pushed, not the base branch. Non-blocking: on_failure routes to ci_watch with ci_event=null (match-any).\n" + }, + "check_pr_state": { + "tool": "check_pr_mergeable", + "with": { + "pr_number": "${{ context.pr_number }}", + "cwd": "${{ context.work_dir }}" + }, + "on_result": [ + { + "when": "${{ result.mergeable_status }} == 'CONFLICTING'", + "route": "detect_ci_conflict" + }, + { + "route": "ci_watch" + } + ], + "on_failure": "ci_watch", + "skip_when_false": "inputs.open_pr", + "note": "Pre-CI-watch gate: checks PR mergeable state before entering the CI polling loop. CONFLICTING PRs cannot trigger pull_request workflows (GitHub cannot create refs/pull/N/merge), so routing to ci_watch would produce a guaranteed no_runs. Routes CONFLICTING to detect_ci_conflict (which handles conflict resolution) and MERGEABLE/UNKNOWN to ci_watch.\n" + }, + "ci_watch": { + "tool": "wait_for_ci", + "with": { + "branch": "${{ context.merge_target }}", + "event": "${{ context.ci_event }}", + "timeout_seconds": 600, + "cwd": "${{ context.work_dir }}", + "remote_url": "${{ context.remote_url }}", + "auto_trigger": "true" + }, + "capture": { + "ci_conclusion": "${{ result.conclusion }}", + "ci_failed_jobs": "${{ result.failed_jobs }}" + }, + "on_result": [ + { + "when": "${{ result.conclusion }} == 'success'", + "route": "check_repo_merge_state" + }, + { + "when": "${{ result.conclusion }} == 'no_runs'", + "route": "handle_no_ci_runs" + }, + { + "when": "${{ result.conclusion }} == 'timed_out'", + "route": "check_ci_timed_out_loop" + }, + { + "route": "detect_ci_conflict" + } + ], + "on_failure": "detect_ci_conflict", + "optional": true, + "skip_when_false": "inputs.open_pr", + "note": "Waits for the GitHub Actions CI run on context.merge_target to complete using the wait_for_ci MCP tool (timeout: 300 s). Captures structured output (conclusion + failed job names) for downstream resolve_ci. Skipped when open_pr=false — no remote feature branch is opened in that mode. Routes on result.conclusion: success → check_repo_merge_state, no_runs → handle_no_ci_runs (auto_trigger fires first: empty commit + force-push + re-poll; handle_no_ci_runs reached only when push fails), timed_out → check_ci_timed_out_loop (iteration guard), failure → detect_ci_conflict.\n" + }, + "check_ci_timed_out_loop": { + "tool": "run_python", + "with": { + "callable": "autoskillit.smoke_utils.check_loop_iteration", + "current_iteration": "${{ context.ci_timed_out_loop_count }}", + "max_iterations": "2" + }, + "capture": { + "ci_timed_out_loop_count": "${{ result.next_iteration }}" + }, + "on_result": [ + { + "when": "${{ result.max_exceeded }} == true", + "route": "detect_ci_conflict" + }, + { + "route": "ci_watch" + } + ], + "on_failure": "detect_ci_conflict", + "optional_context_refs": [ + "ci_timed_out_loop_count" + ], + "note": "Caps the ci_watch re-poll loop at 2 iterations. After 2 timed_out results (1200s total), routes to detect_ci_conflict instead of spinning indefinitely.\n" + }, + "handle_no_ci_runs": { + "tool": "check_repo_merge_state", + "with": { + "branch": "${{ context.merge_target }}", + "cwd": "${{ context.work_dir }}", + "remote_url": "${{ context.remote_url }}", + "step_name": "handle_no_ci_runs" + }, + "capture": { + "ci_event": "${{ result.ci_event }}" + }, + "on_success": "check_ci_loop", + "on_failure": "release_issue_failure", + "retries": 1, + "on_exhausted": "release_issue_failure", + "note": "Intercepts no_runs conclusion from wait_for_ci. Re-derives ci_event using check_repo_merge_state (the original event may have been wrong for this branch's workflow configuration) and routes to check_ci_loop for iteration guard before retrying ci_watch. retries: 1 bounds the tool-call retry; check_ci_loop bounds the outer loop at 2 iterations.\n" + }, + "check_ci_loop": { + "tool": "run_python", + "with": { + "callable": "autoskillit.smoke_utils.check_loop_iteration", + "current_iteration": "${{ context.ci_loop_count }}", + "max_iterations": "2" + }, + "capture": { + "ci_loop_count": "${{ result.next_iteration }}" + }, + "on_result": [ + { + "when": "${{ result.max_exceeded }} == true", + "route": "check_active_trigger_loop" + }, + { + "route": "ci_watch" + } + ], + "on_failure": "check_ci_already_passed", + "optional_context_refs": [ + "ci_loop_count" + ], + "note": "Iteration guard for the ci_watch / handle_no_ci_runs cycle. Caps the loop at 2 iterations (initial + 1 retry after re-deriving ci_event). On budget exhaustion, routes to check_active_trigger_loop which caps the active trigger to a single attempt before escalating.\n" + }, + "check_active_trigger_loop": { + "tool": "run_python", + "with": { + "callable": "autoskillit.smoke_utils.check_loop_iteration", + "current_iteration": "${{ context.active_trigger_count }}", + "max_iterations": "2" + }, + "capture": { + "active_trigger_count": "${{ result.next_iteration }}" + }, + "on_result": [ + { + "when": "${{ result.max_exceeded }} == true", + "route": "check_ci_already_passed" + }, + { + "route": "trigger_ci_actively" + } + ], + "on_failure": "check_ci_already_passed", + "optional_context_refs": [ + "active_trigger_count" + ], + "note": "Bounds the trigger_ci_actively remediation to a single attempt. If ci_watch still returns no_runs after the active trigger, escalates to escalate_stop_no_ci rather than re-triggering indefinitely.\n" + }, + "trigger_ci_actively": { + "tool": "run_cmd", + "with": { + "cmd": "gh pr close ${{ context.pr_number }} && sleep 2 && gh pr reopen ${{ context.pr_number }}", + "cwd": "${{ context.work_dir }}", + "step_name": "trigger_ci_actively" + }, + "on_success": "ci_watch", + "on_failure": "check_ci_already_passed", + "retries": 1, + "on_exhausted": "check_ci_already_passed", + "note": "Active CI trigger remediation: closes and reopens the PR to force GitHub webhook re-delivery. This re-creates the pull_request event and the refs/pull/N/merge ref (if the PR is mergeable). Routes back to ci_watch for one final attempt. If the close/reopen fails or the subsequent ci_watch still returns no_runs, escalates to escalate_stop_no_ci.\n" + }, + "check_ci_already_passed": { + "tool": "run_cmd", + "with": { + "cmd": "gh pr view '${{ context.pr_number }}' --json statusCheckRollup --jq '.statusCheckRollup | if length == 0 then \"no_checks\" elif all(.state == \"SUCCESS\" or .state == \"NEUTRAL\" or .state == \"SKIPPED\") then \"passed\" elif any(.state == \"FAILURE\" or .state == \"ERROR\") then \"failed\" else \"pending\" end'", + "cwd": "${{ context.work_dir }}", + "step_name": "check_ci_already_passed" + }, + "on_result": [ + { + "when": "${{ result.stdout | trim }} == passed", + "route": "check_repo_merge_state" + }, + { + "when": "${{ result.stdout | trim }} == failed", + "route": "detect_ci_conflict" + }, + { + "when": "${{ result.stdout | trim }} == pending", + "route": "ci_watch" + }, + { + "route": "mark_issue_failed_no_ci" + } + ], + "on_failure": "mark_issue_failed_no_ci", + "skip_when_false": "inputs.open_pr", + "note": "Safety net before terminal escalation: checks if CI already passed despite the watcher failing to observe it (e.g. post-force-push SHA staleness). Uses statusCheckRollup as the authoritative signal. Outputs passed/failed/no_checks/pending: passed goes to merge gate, failed routes to CI conflict remediation, pending re-enters the CI watcher, no_checks falls through to terminal stop.\n" + }, + "mark_issue_failed_no_ci": { + "description": "Swap in-progress → fail label before CI failure escalation", + "tool": "release_issue", + "optional": true, + "skip_when_false": "inputs.issue_url", + "with": { + "issue_url": "${{ inputs.issue_url }}", + "fail_label": "fail" + }, + "on_success": "register_clone_no_ci", + "on_failure": "register_clone_no_ci" + }, + "register_clone_no_ci": { + "description": "Register clone as preserved (no-CI error pipeline)", + "tool": "register_clone_status", + "with": { + "clone_path": "${{ context.work_dir }}", + "status": "error", + "step_name": "register_clone_no_ci" + }, + "on_success": "escalate_stop_no_ci", + "on_failure": "escalate_stop_no_ci" + }, + "escalate_stop_no_ci": { + "action": "stop", + "message": "CI safety net: statusCheckRollup returned no checks for PR ${{ context.pr_number }}. This typically means no workflow is configured to trigger on this branch pattern. Check .github/workflows/ trigger configuration. This is NOT a test failure — CI never started." + }, + "check_repo_merge_state": { + "tool": "check_repo_merge_state", + "block": "pre_queue_gate", + "with": { + "branch": "${{ inputs.base_branch }}", + "cwd": "${{ context.work_dir }}", + "remote_url": "${{ context.remote_url }}", + "step_name": "check_repo_merge_state" + }, + "capture": { + "queue_available": "${{ result.queue_available }}", + "merge_group_trigger": "${{ result.merge_group_trigger }}", + "auto_merge_available": "${{ result.auto_merge_available }}", + "ci_event": "${{ result.ci_event }}" + }, + "on_success": "route_queue_mode", + "on_failure": "immediate_merge", + "skip_when_false": "inputs.open_pr", + "note": "Single GraphQL round-trip that captures queue_available, merge_group_trigger, auto_merge_available, and ci_event for the target branch. Consolidates the former three sequential run_cmd steps (check_merge_queue, check_merge_group_trigger, check_auto_merge) into one MCP tool call. On failure (network error, auth failure), falls through to immediate_merge (non-queue direct merge path). Skipped when open_pr=false — no PR was opened.\n" + }, + "route_queue_mode": { + "block": "pre_queue_gate", + "action": "route", + "on_result": [ + { + "when": "${{ inputs.auto_merge }} != 'true'", + "route": "patch_token_summary" + }, + { + "when": "${{ context.queue_available }} == true and ${{ context.merge_group_trigger }} == true", + "route": "enqueue_to_queue" + }, + { + "when": "${{ context.auto_merge_available }} == true", + "route": "direct_merge" + }, + { + "route": "immediate_merge" + } + ], + "skip_when_false": "inputs.open_pr" + }, + "enqueue_to_queue": { + "tool": "enqueue_pr", + "with": { + "pr_number": "${{ context.pr_number }}", + "target_branch": "${{ inputs.base_branch }}", + "cwd": "${{ context.work_dir }}", + "auto_merge_available": "${{ context.auto_merge_available }}" + }, + "on_success": "wait_for_queue", + "on_failure": "verify_queue_enrollment", + "skip_when_false": "inputs.open_pr", + "note": "Submits the PR to the merge queue via the unified enqueue_pr tool. The tool resolves the enrollment strategy internally based on auto_merge_available. On failure, probes the queue via verify_queue_enrollment to distinguish a transient enqueue error from a PR that is already merged or still in queue.\n" + }, + "verify_queue_enrollment": { + "description": "Probe merge queue after enqueue command failure to determine actual state", + "tool": "wait_for_merge_queue", + "with": { + "pr_number": "${{ context.pr_number }}", + "target_branch": "${{ inputs.base_branch }}", + "cwd": "${{ context.work_dir }}", + "remote_url": "${{ context.remote_url }}", + "timeout_seconds": 60, + "poll_interval": 10 + }, + "on_result": [ + { + "when": "${{ result.pr_state }} == merged", + "route": "release_issue_success" + }, + { + "when": "${{ result.pr_state }} == ejected_ci_failure", + "route": "diagnose_ci" + }, + { + "when": "${{ result.pr_state }} == not_enrolled", + "route": "release_issue_failure" + }, + { + "when": "${{ result.pr_state }} == dropped_healthy", + "route": "wait_for_queue" + }, + { + "when": "${{ result.pr_state }} == dropped_merge_group_ci", + "route": "diagnose_ci" + }, + { + "when": "${{ result.pr_state }} == ejected", + "route": "wait_for_queue" + }, + { + "when": "${{ result.pr_state }} == stalled", + "route": "wait_for_queue" + }, + { + "when": "${{ result.pr_state }} == timeout", + "route": "register_clone_unconfirmed" + }, + { + "route": "register_clone_unconfirmed" + } + ], + "on_failure": "register_clone_unconfirmed", + "note": "Short probe (60s) after enqueue_to_queue failure. If the PR is already merged, route to release_issue_success. If the queue already confirms CI failure (ejected_ci_failure or dropped_merge_group_ci), route directly to diagnose_ci. If enrollment was attempted but the PR never appeared in the queue (not_enrolled), route to release_issue_failure — this is a repo configuration error. For other active queue states (dropped_healthy, ejected, stalled), route to wait_for_queue for full 900s authoritative watch. If the probe timed out or returned an unexpected state, escalate via register_clone_unconfirmed.\n" + }, + "wait_for_queue": { + "tool": "wait_for_merge_queue", + "with": { + "pr_number": "${{ context.pr_number }}", + "target_branch": "${{ inputs.base_branch }}", + "timeout_seconds": 900, + "cwd": "${{ context.work_dir }}", + "remote_url": "${{ context.remote_url }}" + }, + "capture": { + "queue_pr_state": "${{ result.pr_state }}" + }, + "on_result": [ + { + "when": "${{ result.pr_state }} == merged", + "route": "release_issue_success" + }, + { + "when": "${{ result.pr_state }} == ejected_ci_failure", + "route": "diagnose_ci" + }, + { + "when": "${{ result.pr_state }} == dropped_merge_group_ci", + "route": "diagnose_ci" + }, + { + "when": "${{ result.pr_state }} == not_enrolled", + "route": "release_issue_failure" + }, + { + "when": "${{ result.pr_state }} == dropped_healthy", + "route": "check_dropped_healthy_loop" + }, + { + "when": "${{ result.pr_state }} == ejected", + "route": "check_eject_limit" + }, + { + "when": "${{ result.pr_state }} == stalled", + "route": "reenroll_stalled_pr" + }, + { + "when": "${{ result.pr_state }} == timeout", + "route": "register_clone_unconfirmed" + }, + { + "route": "register_clone_unconfirmed" + } + ], + "on_failure": "register_clone_unconfirmed", + "skip_when_false": "inputs.open_pr", + "note": "Waits up to 15 minutes for the PR to merge through the queue. merged → release_issue_success to transition issue state then register_clone_success. ejected_ci_failure → diagnose_ci (CI failure; conflict resolution cannot fix this). not_enrolled → release_issue_failure (enrollment failed; repo configuration error). ejected → conflict fix cycle then re-enter. stalled → lightweight re-enrollment via enqueue_pr. timeout → register_clone_unconfirmed (release claim without staged label; merge unconfirmed).\n" + }, + "reenroll_stalled_pr": { + "tool": "enqueue_pr", + "with": { + "pr_number": "${{ context.pr_number }}", + "target_branch": "${{ inputs.base_branch }}", + "cwd": "${{ context.work_dir }}", + "auto_merge_available": "${{ context.auto_merge_available }}" + }, + "on_success": "check_stall_loop", + "on_failure": "release_issue_failure", + "skip_when_false": "inputs.open_pr" + }, + "check_stall_loop": { + "tool": "run_python", + "with": { + "callable": "autoskillit.smoke_utils.check_loop_iteration", + "current_iteration": "${{ context.stall_loop_count }}", + "max_iterations": "3" + }, + "capture": { + "stall_loop_count": "${{ result.next_iteration }}" + }, + "on_result": [ + { + "when": "${{ result.max_exceeded }} == true", + "route": "register_clone_unconfirmed" + }, + { + "route": "wait_for_queue" + } + ], + "on_failure": "register_clone_unconfirmed", + "optional_context_refs": [ + "stall_loop_count" + ], + "note": "Iteration guard for the wait_for_queue / reenroll_stalled_pr cycle. Caps stall re-enrollment at 3 attempts. On budget exhaustion, routes to register_clone_unconfirmed (merge outcome unconfirmed).\n" + }, + "check_eject_limit": { + "tool": "run_python", + "with": { + "callable": "autoskillit.recipe._cmd_rpc.check_eject_limit", + "counter_file": "${{ context.work_dir }}/{{AUTOSKILLIT_TEMP}}/eject_count", + "max_ejects": "3" + }, + "on_result": [ + { + "when": "${{ result.status }} == EJECT_LIMIT_EXCEEDED", + "route": "release_issue_failure" + }, + { + "route": "queue_ejected_fix" + } + ], + "on_failure": "release_issue_failure", + "note": "Circuit breaker for the queue ejection loop. Tracks ejection count in a file-based counter under {{AUTOSKILLIT_TEMP}}/. After 3 ejections, routes to release_issue_failure to prevent infinite re-enqueue cycles.\n" + }, + "check_dropped_healthy_loop": { + "tool": "run_python", + "with": { + "callable": "autoskillit.recipe._cmd_rpc.check_dropped_healthy_loop", + "counter_file": "${{ context.work_dir }}/{{AUTOSKILLIT_TEMP}}/dropped_healthy_count", + "max_drops": "2" + }, + "on_result": [ + { + "when": "${{ result.status }} == DROPPED_LIMIT_EXCEEDED", + "route": "release_issue_failure" + }, + { + "route": "reenter_merge_queue_cheap" + } + ], + "on_failure": "release_issue_failure", + "note": "Circuit breaker for the dropped_healthy re-enrollment loop. After 2 consecutive unexplained drops with healthy PR-branch CI, escalates to release_issue_failure instead of looping. Matches the check_eject_limit pattern used for the ejected path. DROPPED_MERGE_GROUP_CI (confirmed merge-group CI failure) is routed directly to diagnose_ci and never reaches this step.\n" + }, + "queue_ejected_fix": { + "tool": "run_python", + "with": { + "callable": "autoskillit.recipe._cmd_rpc.queue_ejected_fix", + "work_dir": "${{ context.work_dir }}", + "base_branch": "${{ inputs.base_branch }}" + }, + "on_result": [ + { + "when": "${{ result.status }} == clean", + "route": "re_push_queue_fix" + }, + { + "route": "resolve_queue_merge_conflicts" + } + ], + "on_failure": "resolve_queue_merge_conflicts", + "skip_when_false": "inputs.open_pr", + "note": "Attempts a cheap rebase before invoking the full conflict-resolution skill. On a clean rebase (stdout \"clean\"), skips the skill and proceeds directly to re_push_queue_fix. On conflict, aborts the rebase and routes to resolve_queue_merge_conflicts for goal-aware resolution.\n" + }, + "resolve_queue_merge_conflicts": { + "tool": "run_skill", + "model": "", + "with": { + "skill_command": "/autoskillit:resolve-merge-conflicts ${{ context.work_dir }} ${{ context.plan_path }} ${{ inputs.base_branch }}", + "cwd": "${{ context.work_dir }}", + "step_name": "resolve_queue_merge_conflicts" + }, + "capture": { + "conflict_escalation_required": "${{ result.escalation_required }}" + }, + "on_result": [ + { + "when": "${{ result.escalation_required }} == true", + "route": "release_issue_failure" + }, + { + "route": "re_push_queue_fix" + } + ], + "on_failure": "release_issue_failure", + "on_context_limit": "release_issue_failure", + "retries": 1, + "on_exhausted": "release_issue_failure", + "skip_when_false": "inputs.open_pr", + "note": "Resolves rebase conflicts on the feature branch after queue ejection. Only reached when the cheap rebase in queue_ejected_fix encountered conflicts. escalation_required=true → release_issue_failure for human review.\n" + }, + "re_push_queue_fix": { + "tool": "push_to_remote", + "with": { + "clone_path": "${{ context.work_dir }}", + "remote_url": "${{ context.remote_url }}", + "branch": "${{ context.merge_target }}", + "force": "true" + }, + "capture": { + "push_error_type": "${{ result.error_type }}" + }, + "retries": 2, + "on_exhausted": "register_clone_unconfirmed", + "on_success": "ci_watch_post_queue_fix", + "on_failure": "classify_push_failure", + "skip_when_false": "inputs.open_pr" + }, + "classify_push_failure": { + "action": "route", + "on_result": [ + { + "when": "${{ context.push_error_type }} == queued_branch", + "route": "wait_dequeue_retry" + }, + { + "route": "release_issue_failure" + } + ] + }, + "wait_dequeue_retry": { + "tool": "wait_for_merge_queue", + "with": { + "pr_number": "${{ context.pr_number }}", + "target_branch": "${{ inputs.base_branch }}", + "cwd": "${{ context.work_dir }}", + "timeout_seconds": "60" + }, + "on_result": [ + { + "when": "${{ result.pr_state }} == merged", + "route": "release_issue_success" + }, + { + "when": "${{ result.pr_state }} == ejected_ci_failure", + "route": "register_clone_unconfirmed" + }, + { + "when": "${{ result.pr_state }} == dropped_merge_group_ci", + "route": "register_clone_unconfirmed" + }, + { + "when": "${{ result.pr_state }} == dropped_healthy", + "route": "register_clone_unconfirmed" + }, + { + "when": "${{ result.pr_state }} == ejected", + "route": "re_push_queue_fix" + }, + { + "when": "${{ result.pr_state }} == stalled", + "route": "register_clone_unconfirmed" + }, + { + "when": "${{ result.pr_state }} == timeout", + "route": "register_clone_unconfirmed" + }, + { + "route": "register_clone_unconfirmed" + } + ], + "on_failure": "register_clone_unconfirmed", + "optional_context_refs": [ + "push_error_type" + ] + }, + "ci_watch_post_queue_fix": { + "tool": "wait_for_ci", + "with": { + "branch": "${{ context.merge_target }}", + "event": "${{ context.ci_event }}", + "timeout_seconds": 600, + "cwd": "${{ context.work_dir }}", + "remote_url": "${{ context.remote_url }}", + "auto_trigger": "true" + }, + "capture": { + "ci_conclusion": "${{ result.conclusion }}", + "ci_failed_jobs": "${{ result.failed_jobs }}" + }, + "on_result": [ + { + "when": "${{ result.conclusion }} == 'success'", + "route": "reenter_merge_queue" + }, + { + "when": "${{ result.conclusion }} == 'no_runs'", + "route": "release_issue_failure" + }, + { + "when": "${{ result.conclusion }} == 'timed_out'", + "route": "check_ci_post_queue_loop" + }, + { + "route": "detect_ci_conflict" + } + ], + "on_failure": "detect_ci_conflict", + "skip_when_false": "inputs.open_pr", + "note": "Validates CI on the rebased feature branch after queue ejection fix and re-push. Routes on result.conclusion: success → reenter_merge_queue, no_runs → release_issue_failure (auto_trigger fires first: empty commit + force-push + re-poll; no_runs reached only when trigger recovery fails), timed_out → check_ci_post_queue_loop, failure → detect_ci_conflict.\n" + }, + "check_ci_post_queue_loop": { + "tool": "run_python", + "with": { + "callable": "autoskillit.smoke_utils.check_loop_iteration", + "current_iteration": "${{ context.ci_post_queue_loop_count }}", + "max_iterations": "2" + }, + "capture": { + "ci_post_queue_loop_count": "${{ result.next_iteration }}" + }, + "on_result": [ + { + "when": "${{ result.max_exceeded }} == true", + "route": "release_issue_failure" + }, + { + "route": "ci_watch_post_queue_fix" + } + ], + "on_failure": "release_issue_failure", + "optional_context_refs": [ + "ci_post_queue_loop_count" + ], + "skip_when_false": "inputs.open_pr", + "note": "Iteration guard for the ci_watch_post_queue_fix timed_out self-loop. Caps at 2 iterations (initial + 1 retry). On budget exhaustion, routes to release_issue_failure — this is already a post-ejection retry path so further escalation is not warranted.\n" + }, + "reenter_merge_queue": { + "tool": "enqueue_pr", + "with": { + "pr_number": "${{ context.pr_number }}", + "target_branch": "${{ inputs.base_branch }}", + "cwd": "${{ context.work_dir }}", + "auto_merge_available": "${{ context.auto_merge_available }}" + }, + "on_success": "wait_for_queue", + "on_failure": "release_issue_failure", + "skip_when_false": "inputs.open_pr", + "note": "Re-submits the PR after conflict resolution. Routes back to wait_for_queue. If re-entry fails, release_issue_failure preserves the clone for debugging.\n" + }, + "reenter_merge_queue_cheap": { + "tool": "enqueue_pr", + "with": { + "pr_number": "${{ context.pr_number }}", + "target_branch": "${{ inputs.base_branch }}", + "cwd": "${{ context.work_dir }}", + "auto_merge_available": "${{ context.auto_merge_available }}" + }, + "on_success": "wait_for_queue", + "on_failure": "release_issue_failure", + "skip_when_false": "inputs.open_pr", + "note": "Re-submits a healthy PR into the merge queue after auto_merge was cleared externally (DROPPED_HEALTHY state). No conflict-resolution CI wait needed. Routes back to wait_for_queue on success.\n" + }, + "direct_merge": { + "tool": "run_cmd", + "with": { + "cmd": "gh pr merge '${{ context.pr_number }}' --squash --auto", + "cwd": "${{ context.work_dir }}", + "step_name": "direct_merge" + }, + "on_success": "wait_for_direct_merge", + "on_failure": "release_issue_failure", + "skip_when_false": "inputs.open_pr", + "note": "Enables GitHub native auto-merge for direct merge (no merge queue). GitHub merges automatically once all required checks pass. On failure, escalates via release_issue_failure so the issue claim is released and the pipeline reports an error rather than silently succeeding.\n" + }, + "wait_for_direct_merge": { + "tool": "run_python", + "with": { + "callable": "autoskillit.recipe._cmd_rpc.wait_for_direct_merge", + "pr_number": "${{ context.pr_number }}", + "max_polls": "90", + "poll_interval": "10" + }, + "on_result": [ + { + "when": "${{ result.state }} == merged", + "route": "release_issue_success" + }, + { + "when": "${{ result.state }} == closed", + "route": "direct_merge_conflict_fix" + }, + { + "route": "register_clone_unconfirmed" + } + ], + "on_failure": "register_clone_unconfirmed", + "skip_when_false": "inputs.open_pr", + "note": "Polls GitHub PR state every 10 s for up to 15 minutes (90 iterations). merged → release_issue_success (issue labelled, cleanup runs). closed → PR was closed (likely conflict) — route to conflict fix. timeout / unexpected → register_clone_unconfirmed.\n" + }, + "direct_merge_conflict_fix": { + "tool": "run_python", + "with": { + "callable": "autoskillit.recipe._cmd_rpc.direct_merge_conflict_fix", + "work_dir": "${{ context.work_dir }}", + "base_branch": "${{ inputs.base_branch }}" + }, + "on_result": [ + { + "when": "${{ result.status }} == clean", + "route": "re_push_direct_fix" + }, + { + "route": "resolve_direct_merge_conflicts" + } + ], + "on_failure": "resolve_direct_merge_conflicts", + "skip_when_false": "inputs.open_pr", + "note": "Attempts a cheap rebase before invoking the full conflict-resolution skill. On clean rebase, proceeds to re_push_direct_fix. On conflict, routes to resolve_direct_merge_conflicts for goal-aware resolution.\n" + }, + "resolve_direct_merge_conflicts": { + "tool": "run_skill", + "model": "", + "with": { + "skill_command": "/autoskillit:resolve-merge-conflicts ${{ context.work_dir }} ${{ context.plan_path }} ${{ inputs.base_branch }}", + "cwd": "${{ context.work_dir }}", + "step_name": "resolve_direct_merge_conflicts" + }, + "capture": { + "conflict_escalation_required": "${{ result.escalation_required }}" + }, + "on_result": [ + { + "when": "${{ result.escalation_required }} == true", + "route": "release_issue_failure" + }, + { + "route": "re_push_direct_fix" + } + ], + "on_failure": "release_issue_failure", + "on_context_limit": "release_issue_failure", + "retries": 1, + "on_exhausted": "release_issue_failure", + "skip_when_false": "inputs.open_pr", + "note": "Resolves rebase conflicts after direct-merge rejection. Only reached when the cheap rebase in direct_merge_conflict_fix encountered conflicts. escalation_required=true → release_issue_failure for human review.\n" + }, + "re_push_direct_fix": { + "tool": "push_to_remote", + "with": { + "clone_path": "${{ context.work_dir }}", + "remote_url": "${{ context.remote_url }}", + "branch": "${{ context.merge_target }}", + "force": "true" + }, + "on_success": "redirect_merge", + "on_failure": "release_issue_failure", + "skip_when_false": "inputs.open_pr" + }, + "redirect_merge": { + "tool": "run_cmd", + "with": { + "cmd": "gh pr merge '${{ context.pr_number }}' --squash --auto", + "cwd": "${{ context.work_dir }}", + "step_name": "redirect_merge" + }, + "on_success": "wait_for_direct_merge", + "on_failure": "release_issue_failure", + "skip_when_false": "inputs.open_pr", + "note": "Re-enables auto-merge after conflict resolution and re-push. Routes back to wait_for_direct_merge to poll for completion. On failure, release_issue_failure preserves the clone for debugging.\n" + }, + "immediate_merge": { + "tool": "run_cmd", + "with": { + "cmd": "gh pr merge '${{ context.pr_number }}' --squash", + "cwd": "${{ context.work_dir }}", + "step_name": "immediate_merge" + }, + "on_success": "wait_for_immediate_merge", + "on_failure": "release_issue_failure", + "skip_when_false": "inputs.open_pr", + "note": "Merges the PR immediately using plain --squash when autoMergeAllowed=false. Unlike direct_merge, no --auto flag is used — GitHub executes the merge synchronously rather than queueing it for when checks pass. On failure, escalates via release_issue_failure so the issue claim is released and the pipeline reports an error rather than silently succeeding.\n" + }, + "wait_for_immediate_merge": { + "tool": "run_python", + "with": { + "callable": "autoskillit.recipe._cmd_rpc.wait_for_immediate_merge", + "pr_number": "${{ context.pr_number }}", + "max_polls": "30", + "poll_interval": "10" + }, + "on_result": [ + { + "when": "${{ result.state }} == merged", + "route": "release_issue_success" + }, + { + "when": "${{ result.state }} == closed", + "route": "immediate_merge_conflict_fix" + }, + { + "route": "register_clone_unconfirmed" + } + ], + "on_failure": "register_clone_unconfirmed", + "skip_when_false": "inputs.open_pr", + "note": "Polls GitHub PR state every 10s for up to 5 minutes (30 iterations). Since immediate_merge uses --squash without --auto, the merge completes synchronously — this poll is a confirmation window. merged → release_issue_success. closed → PR was rejected (stale base or conflict) — route to conflict fix. timeout → register_clone_unconfirmed.\n" + }, + "immediate_merge_conflict_fix": { + "tool": "run_python", + "with": { + "callable": "autoskillit.recipe._cmd_rpc.immediate_merge_conflict_fix", + "work_dir": "${{ context.work_dir }}", + "base_branch": "${{ inputs.base_branch }}" + }, + "on_result": [ + { + "when": "${{ result.status }} == clean", + "route": "re_push_immediate_fix" + }, + { + "route": "resolve_immediate_merge_conflicts" + } + ], + "on_failure": "resolve_immediate_merge_conflicts", + "skip_when_false": "inputs.open_pr", + "note": "Attempts a cheap rebase before invoking the full conflict-resolution skill. On clean rebase, proceeds to re_push_immediate_fix. On conflict, routes to resolve_immediate_merge_conflicts for goal-aware resolution.\n" + }, + "resolve_immediate_merge_conflicts": { + "tool": "run_skill", + "model": "", + "with": { + "skill_command": "/autoskillit:resolve-merge-conflicts ${{ context.work_dir }} ${{ context.plan_path }} ${{ inputs.base_branch }}", + "cwd": "${{ context.work_dir }}", + "step_name": "resolve_immediate_merge_conflicts" + }, + "capture": { + "conflict_escalation_required": "${{ result.escalation_required }}" + }, + "on_result": [ + { + "when": "${{ result.escalation_required }} == true", + "route": "release_issue_failure" + }, + { + "route": "re_push_immediate_fix" + } + ], + "on_failure": "release_issue_failure", + "on_context_limit": "release_issue_failure", + "retries": 1, + "on_exhausted": "release_issue_failure", + "skip_when_false": "inputs.open_pr", + "note": "Resolves rebase conflicts after immediate-merge rejection. Only reached when the cheap rebase in immediate_merge_conflict_fix encountered conflicts. escalation_required=true → release_issue_failure for human review.\n" + }, + "re_push_immediate_fix": { + "tool": "push_to_remote", + "with": { + "clone_path": "${{ context.work_dir }}", + "remote_url": "${{ context.remote_url }}", + "branch": "${{ context.merge_target }}", + "force": "true" + }, + "on_success": "remerge_immediate", + "on_failure": "release_issue_failure", + "skip_when_false": "inputs.open_pr" + }, + "remerge_immediate": { + "tool": "run_cmd", + "with": { + "cmd": "gh pr merge '${{ context.pr_number }}' --squash", + "cwd": "${{ context.work_dir }}", + "step_name": "remerge_immediate" + }, + "on_success": "wait_for_immediate_merge", + "on_failure": "release_issue_failure", + "skip_when_false": "inputs.open_pr", + "note": "Re-attempts plain squash merge after conflict resolution and re-push. Routes back to wait_for_immediate_merge to confirm completion. On failure, release_issue_failure preserves the clone for debugging.\n" + }, + "diagnose_ci": { + "tool": "run_skill", + "model": "", + "with": { + "skill_command": "/autoskillit:diagnose-ci ${{ context.merge_target }} - - - ${{ context.ci_event }}", + "cwd": "${{ context.work_dir }}", + "step_name": "diagnose_ci", + "ci_failed_jobs": "${{ context.ci_failed_jobs }}" + }, + "capture": { + "diagnosis_path": "${{ result.diagnosis_path }}" + }, + "on_success": "resolve_ci", + "on_failure": "resolve_ci", + "on_context_limit": "resolve_ci", + "optional": true, + "skip_when_false": "inputs.open_pr", + "note": "Runs when ci_watch reports a CI failure. Fetches CI logs via gh API and writes a structured diagnosis to {{AUTOSKILLIT_TEMP}}/diagnose-ci/. Routes to resolve_ci on success OR failure (best-effort: resolve-failures proceeds even without a diagnosis). diagnosis_path is passed to resolve_ci for targeted remediation context.\n" + }, + "resolve_ci": { + "tool": "run_skill", + "model": "", + "with": { + "skill_command": "/autoskillit:resolve-failures ${{ context.work_dir }} ${{ context.plan_path }} ${{ inputs.base_branch }} ${{ context.ci_conclusion }} ${{ context.ci_failed_jobs }} ${{ context.diagnosis_path }}", + "cwd": "${{ context.work_dir }}", + "step_name": "resolve_ci" + }, + "capture": { + "verdict": "${{ result.verdict }}" + }, + "retries": 2, + "on_exhausted": "release_issue_failure", + "on_context_limit": "re_push", + "on_result": [ + { + "when": "${{ result.verdict }} == 'real_fix'", + "route": "re_push" + }, + { + "when": "${{ result.verdict }} == 'already_green'", + "route": "pre_resolve_rebase" + }, + { + "when": "${{ result.verdict }} == 'flake_suspected'", + "route": "re_push" + }, + { + "when": "${{ result.verdict }} == 'ci_only_failure'", + "route": "release_issue_failure" + }, + { + "route": "release_issue_failure" + } + ], + "on_failure": "release_issue_failure", + "note": "Runs when ci_watch reports a CI failure. Receives ci_failed_jobs from wait_for_ci structured output to avoid redundant CI failure re-investigation. Routes via on_result: verdict dispatch — real_fix pushes, already_green rebases and re-diagnoses (sibling fix may have landed), flake_suspected retries via re_push (bounded by retries: 2 / on_exhausted: release_issue_failure), ci_only_failure escalates to human. Retries up to 2 times (3 total attempts) before exhausting to release_issue_failure.\n" + }, + "pre_resolve_rebase": { + "tool": "run_cmd", + "with": { + "cmd": "REMOTE=$(git -C ${{ context.work_dir }} remote get-url upstream 2>/dev/null | grep -qv \"^file://\" && echo upstream || echo origin) &&\ngit -C ${{ context.work_dir }} fetch \"$REMOTE\" &&\ngit -C ${{ context.work_dir }} rebase \"$REMOTE\"/${{ inputs.base_branch }}\n", + "step_name": "pre_resolve_rebase" + }, + "retries": 1, + "on_success": "diagnose_ci", + "on_failure": "release_issue_failure", + "note": "Rebases the feature branch against integration head when resolve_ci emits already_green — the worktree was already clean because a sibling pipeline's fix already landed on integration. Pulling it in before re-running diagnose_ci gives the pipeline a real chance at emitting real_fix and reaching re_push.\n" + }, + "re_push": { + "tool": "push_to_remote", + "with": { + "clone_path": "${{ context.work_dir }}", + "remote_url": "${{ context.remote_url }}", + "branch": "${{ context.merge_target }}", + "force": "true" + }, + "on_success": "check_repo_ci_event", + "on_failure": "release_issue_failure", + "note": "Pushes the CI-fixed feature branch back to the remote after resolve_ci succeeds. Routes to check_repo_ci_event to derive ci_event before ci_watch verifies the fix. On push failure, escalates via release_issue_failure.\n" + }, + "detect_ci_conflict": { + "tool": "run_cmd", + "with": { + "cmd": "REMOTE=$(git remote get-url upstream 2>/dev/null | grep -qv \"^file://\" && echo upstream || echo origin) &&\ngit fetch \"$REMOTE\" ${{ inputs.base_branch }} &&\n! git merge-base --is-ancestor \"$REMOTE\"/${{ inputs.base_branch }} HEAD\n", + "cwd": "${{ context.work_dir }}", + "step_name": "detect_ci_conflict" + }, + "on_success": "ci_conflict_fix", + "on_failure": "diagnose_ci", + "optional": true, + "skip_when_false": "inputs.open_pr", + "note": "Detects whether the CI failure originates from a stale integration base rather than a code-level test failure. Exit 0 (fetch succeeded AND base is NOT an ancestor of HEAD) signals a stale base — routes to ci_conflict_fix. Exit 1 (fetch failed OR base is already an ancestor) signals a real code failure or an unresolvable state — falls through to diagnose_ci. This step insulates resolve_ci from being invoked for merge-conflict failures.\n" + }, + "ci_conflict_fix": { + "tool": "run_skill", + "model": "", + "with": { + "skill_command": "/autoskillit:resolve-merge-conflicts ${{ context.work_dir }} ${{ context.plan_path }} ${{ inputs.base_branch }}", + "cwd": "${{ context.work_dir }}", + "step_name": "ci_conflict_fix" + }, + "capture": { + "conflict_escalation_required": "${{ result.escalation_required }}" + }, + "on_result": [ + { + "when": "${{ result.escalation_required }} == true", + "route": "release_issue_failure" + }, + { + "route": "re_push" + } + ], + "on_failure": "release_issue_failure", + "on_context_limit": "release_issue_failure", + "retries": 1, + "on_exhausted": "release_issue_failure", + "skip_when_false": "inputs.open_pr", + "note": "Resolves a stale-base CI failure by rebasing the feature branch onto the latest integration HEAD. Mirrors queue_ejected_fix: on clean rebase or goal-aware conflict resolution, routes to re_push to re-trigger CI on the rebased branch. On escalation (LOW confidence conflicts) or tool failure, routes to release_issue_failure. retries: 1 bounds the loop; after 2 total attempts, escalates.\n" + }, + "release_issue_success": { + "description": "Transition issue from in-progress to staged after successful pipeline completion", + "tool": "release_issue", + "optional": true, + "skip_when_false": "inputs.issue_url", + "with": { + "issue_url": "${{ inputs.issue_url }}", + "target_branch": "${{ inputs.base_branch }}" + }, + "on_success": "patch_token_summary", + "on_failure": "patch_token_summary", + "note": "Called on the success path after PR merge is confirmed. Passes target_branch so release_issue can determine whether to apply the staged label (non-default target) or just remove in-progress (default target). optional=true ensures pipeline still reaches register_clone_success if this step errors.\n" + }, + "patch_token_summary": { + "description": "Collect all pipeline token data and patch the PR body with complete summary", + "tool": "run_python", + "optional": true, + "skip_when_false": "inputs.open_pr", + "with": { + "callable": "autoskillit.smoke_utils.patch_pr_token_summary", + "timeout": 60, + "pr_url": "${{ context.pr_url }}", + "cwd": "${{ context.work_dir }}" + }, + "on_success": "register_clone_success", + "on_failure": "register_clone_success", + "note": "Patches PR body with complete token summary; failure never blocks pipeline completion.\n" + }, + "register_clone_unconfirmed": { + "description": "Register clone as preserved — merge unconfirmed, in-progress label retained", + "tool": "register_clone_status", + "with": { + "clone_path": "${{ context.work_dir }}", + "status": "unconfirmed", + "step_name": "register_clone_unconfirmed" + }, + "on_success": "done", + "on_failure": "done", + "note": "Called when a merge-wait step times out before merge is confirmed. Does NOT call release_issue — the in-progress label stays on the issue so operators can see the PR is still actively queued. The clone is preserved (status: unconfirmed) for human inspection.\n" + }, + "release_issue_failure": { + "description": "Release the in-progress claim after pipeline failure", + "tool": "release_issue", + "optional": true, + "skip_when_false": "inputs.issue_url", + "with": { + "issue_url": "${{ inputs.issue_url }}", + "fail_label": "fail" + }, + "on_success": "register_clone_failure", + "on_failure": "register_clone_failure" + }, + "register_clone_success": { + "description": "Register clone as cleanup candidate for batch deletion", + "tool": "register_clone_status", + "with": { + "clone_path": "${{ context.work_dir }}", + "status": "success", + "step_name": "register_clone_success" + }, + "on_success": "done", + "on_failure": "done" + }, + "register_clone_failure": { + "description": "Register clone as preserved (error pipeline)", + "tool": "register_clone_status", + "with": { + "clone_path": "${{ context.work_dir }}", + "status": "error", + "step_name": "register_clone_failure" + }, + "on_success": "escalate_stop", + "on_failure": "escalate_stop" + }, + "done": { + "action": "stop", + "message": "Implementation pipeline complete. All tasks have been planned, implemented, tested, and merged." + }, + "escalate_stop": { + "action": "stop", + "message": "Pipeline failed — human intervention needed. Check the worktree and plan for details." + } + } +} diff --git a/src/autoskillit/recipes/implementation.yaml b/src/autoskillit/recipes/implementation.yaml index 54fcc2df78..8a8ece9046 100644 --- a/src/autoskillit/recipes/implementation.yaml +++ b/src/autoskillit/recipes/implementation.yaml @@ -1,7 +1,8 @@ name: implementation description: Plan, verify, implement, test, and merge a task end-to-end. Use when user says "run pipeline", "implement task", or "auto implement". -summary: "[sprint: triage > plan > confirm > dispatch > report] | clone > capture_base_sha > set_merge_target > (create_branch?) > make-plan > (review-approach?) > dry-walkthrough > implement > test > merge (per plan part) > (audit-impl?) > (open_pr?) > push > cleanup" -autoskillit_version: "0.3.0" +summary: "bootstrap_clone > (claim_and_resolve?) > (create_and_publish?) > make-plan > (review-approach?) > dry-walkthrough > implement > test > merge (per plan part) > (audit-impl?) > (open_pr?) > push > cleanup" +recipe_version: "1.0.0" +requires_packs: [github, ci, clone, telemetry] ingredients: task: @@ -26,18 +27,26 @@ ingredients: open_pr: description: PR instead of direct merge default: "true" + arch_lenses: + description: Generate architecture lens diagrams for the PR + default: "false" auto_merge: description: >- Automatically merge the PR after required checks pass — via merge queue when available, direct merge otherwise default: "true" + review_max_retries: + description: Maximum number of review-resolve retry cycles before proceeding to CI + default: "3" + local_review_rounds: + description: >- + Number of review-resolve cycles to run locally before switching to + GitHub-backed review. 0 disables local mode entirely. + type: integer + default: "3" issue_url: description: GitHub issue to close on merge required: false - sprint_mode: - description: Enable sprint sub-recipe prefix (triage → sprint selection → per-issue dispatch) - default: "false" - hidden: true upfront_claimed: description: "Set to 'true' when process-issues has already claimed this issue upfront. Allows claim_issue defense step to pass through without aborting." default: "false" @@ -49,6 +58,14 @@ ingredients: step scheduling — fast steps for all pipelines before any slow step). default: "sequential" hidden: true + max_parallel: + description: >- + Maximum number of issues to run in parallel within a single execution-map + group. Groups exceeding this cap are split into sequential sub-groups. + Passed to build-execution-map. Must be a positive integer (expressed as a + string). + default: "6" + hidden: true kitchen_rules: - "NEVER use native Claude Code tools (Read, Grep, Glob, Edit, Write, @@ -63,7 +80,11 @@ kitchen_rules: directly. The push step publishes the feature branch, then prepare_pr → run_arch_lenses → compose_pr opens a PR to base_branch. When open_pr=false, merges target base_branch directly and prepare_pr is skipped." - - "SOURCE ISOLATION: After clone_repo returns, the source_dir is strictly + - "When arch_lenses=true (and open_pr=true), prepare_pr routes through + run_arch_lenses before compose_pr, adding architecture lens diagrams + to the PR. By default arch_lenses is off and prepare_pr routes + directly to compose_pr." + - "SOURCE ISOLATION: After bootstrap_clone returns, the source_dir is strictly off-limits. Never run any command in source_dir — no git checkout, git fetch, git reset, git pull, run_cmd, run_skill, or any other operation. All work — skill invocations, git operations, file reads — happens @@ -79,162 +100,73 @@ kitchen_rules: enforcement and conflict routing are preserved." steps: - sprint_entry: - sub_recipe: sprint-prefix - gate: sprint_mode - on_success: done - on_failure: escalate_stop - note: > - Sprint sub-recipe entry point. When sprint_mode=true, the sprint-prefix - sub-recipe steps are merged here and handle the full multi-issue workflow - (triage → planning → user approval → per-issue dispatch → report). - When sprint_mode=false (default), this step is dropped at load time and - the recipe starts at 'clone' as usual. - clone: - tool: clone_repo + tool: bootstrap_clone with: source_dir: "${{ inputs.source_dir }}" run_name: "${{ inputs.run_name }}" - branch: "${{ inputs.base_branch }}" + base_branch: "${{ inputs.base_branch }}" + step_name: "bootstrap_clone" capture: - work_dir: "${{ result.clone_path }}" + work_dir: "${{ result.work_dir }}" remote_url: "${{ result.remote_url }}" - on_success: capture_base_sha + base_sha: "${{ result.base_sha }}" + merge_target: "${{ result.merge_target }}" + on_success: claim_and_resolve on_failure: escalate_stop note: > - Clones source_dir into ../autoskillit-runs/-/. - Captures work_dir context used by all subsequent steps as cwd. - On failure, escalate_stop is reached immediately (no clone to remove). + Clones source_dir and captures base_sha + merge_target in one turn. + Returns work_dir, remote_url, base_sha (SHA of base_branch before any merge), + and merge_target (set to base_branch as fallback; overridden by create_and_publish + when open_pr=true). On failure, escalate_stop immediately (no clone to remove). - capture_base_sha: - tool: run_cmd - with: - cmd: "git -C ${{ context.work_dir }} rev-parse ${{ inputs.base_branch }}" - cwd: "${{ context.work_dir }}" - step_name: "capture_base_sha" - capture: - base_sha: "${{ result.stdout | trim }}" - on_success: set_merge_target - on_failure: escalate_stop - note: > - Capture the SHA of base_branch before any merge begins. A commit SHA names - a git object (not a ref), so it survives git branch -D unconditionally. This - SHA is passed to audit_impl as implementation_ref to diff the full - implementation against the pre-implementation state. - - set_merge_target: - tool: run_cmd - with: - cmd: "printf '%s' '${{ inputs.base_branch }}'" - cwd: "${{ context.work_dir }}" - step_name: "set_merge_target" - capture: - merge_target: "${{ result.stdout }}" - on_success: get_issue_title - on_failure: escalate_stop - note: > - Unconditionally sets merge_target to inputs.base_branch as a fallback. - In the standard flow (open_pr=true), the subsequent create_branch step - overrides merge_target with the feature branch name. When open_pr=false, - create_branch is skipped and this fallback stands. This eliminates the - undefined-context-variable gap that existed when create_branch was optional. - - get_issue_title: - tool: get_issue_title - with: - issue_url: "${{ inputs.issue_url }}" - capture: - issue_number: "${{ result.number }}" - issue_title: "${{ result.title }}" - issue_slug: "${{ result.slug }}" - on_success: claim_issue - on_failure: escalate_stop - optional: true - skip_when_false: "inputs.issue_url" - note: > - Runs when inputs.issue_url is non-empty. Skipped otherwise. - Fetches only the issue title and slug (no body, no comments). - issue_number is used for feature branch naming and threaded to - prepare_pr for "Closes #N" in the PR body. - issue_slug provides a descriptive branch prefix (e.g. fix-merge-conflict-triage). - - claim_issue: - description: "Claim this issue with the in-progress label to prevent parallel work" - tool: claim_issue + claim_and_resolve: + tool: claim_and_resolve_issue optional: true skip_when_false: inputs.issue_url with: - issue_url: ${{ inputs.issue_url }} - allow_reentry: ${{ inputs.upfront_claimed }} + issue_url: "${{ inputs.issue_url }}" + allow_reentry: "${{ inputs.upfront_claimed }}" capture: + issue_number: "${{ result.issue_number }}" + issue_title: "${{ result.issue_title }}" + issue_slug: "${{ result.issue_slug }}" claimed: "${{ result.claimed }}" on_result: - when: "${{ result.claimed }} == true" - route: compute_branch + route: create_and_publish - route: escalate_stop on_failure: escalate_stop - - compute_branch: - tool: run_cmd - with: - cmd: >- - SLUG="${{ context.issue_slug }}" && - RUN="${{ inputs.run_name }}" && - PREFIX="${SLUG:-$RUN}" && - ISSUE="${{ context.issue_number }}" && - if [ -n "$ISSUE" ]; then - printf '%s' "${PREFIX}/${ISSUE}" - else - printf '%s' "${PREFIX}/$(date +%Y%m%d)" - fi - cwd: "${{ context.work_dir }}" - step_name: "compute_branch" - capture: - proposed_branch: "${{ result.stdout | trim }}" - on_success: create_branch - on_failure: release_issue_failure - optional: true - skip_when_false: "inputs.open_pr" note: > - Computes the proposed branch name from slug/issue or slug/date — no git operations. - Result is passed to create_branch for atomic collision check and checkout. + Fetches issue title/slug and claims the issue in one turn. + Returns issue_number, issue_title, issue_slug, and claimed boolean. + When claimed=false (another session owns it), routes to escalate_stop. + issue_slug and issue_number are passed to create_and_publish for branch naming. - create_branch: - tool: create_unique_branch - with: - base_branch_name: "${{ context.proposed_branch }}" - cwd: "${{ context.work_dir }}" - capture: - merge_target: "${{ result.branch_name }}" - on_success: push_merge_target - on_failure: release_issue_failure + create_and_publish: + tool: create_and_publish_branch optional: true - skip_when_false: "inputs.open_pr" - note: > - Atomic branch creation: ls-remote collision check → checkout -b. Uses the - create_unique_branch MCP tool so the orchestrator cannot insert extra git fetch - or checkout calls between the collision check and branch creation. - - push_merge_target: - tool: push_to_remote + skip_when_false: inputs.open_pr with: - clone_path: "${{ context.work_dir }}" + issue_slug: "${{ context.issue_slug }}" + run_name: "${{ inputs.run_name }}" + issue_number: "${{ context.issue_number }}" + work_dir: "${{ context.work_dir }}" remote_url: "${{ context.remote_url }}" - branch: "${{ context.merge_target }}" + step_name: "create_and_publish_branch" + capture: + merge_target: "${{ result.merge_target }}" on_success: plan on_failure: release_issue_failure - optional: true - skip_when_false: "inputs.open_pr" note: > - Runs by default (open_pr=true). Skipped when open_pr=false. - Publishes the feature branch to origin before any merge_worktree call. + Computes branch name, creates it (with collision suffix if needed), and pushes + to origin in one turn. Overrides merge_target with the feature branch name. merge_worktree requires refs/remotes/origin/{branch} to exist after fetch (step 5.5). - Without this step, merge_worktree fails with WORKTREE_INTACT_BASE_NOT_PUBLISHED. - Uses the pre-resolved remote_url from the clone step to preserve clone isolation. + When open_pr=false, this step is skipped and merge_target remains as base_branch. plan: tool: run_skill + model: "" with: skill_command: "/autoskillit:make-plan ${{ inputs.task }}" cwd: "${{ context.work_dir }}" @@ -265,6 +197,7 @@ steps: review: tool: run_skill + model: "" with: skill_command: "/autoskillit:review-approach ${{ context.plan_path }}" cwd: "${{ context.work_dir }}" @@ -280,6 +213,7 @@ steps: verify: tool: run_skill + model: "" with: skill_command: "/autoskillit:dry-walkthrough ${{ context.plan_path }}" cwd: "${{ context.work_dir }}" @@ -287,10 +221,12 @@ steps: review_path: "${{ context.review_path }}" on_success: implement on_failure: release_issue_failure + on_context_limit: implement note: "SEQUENTIAL EXECUTION: For each plan_part, run the FULL cycle — verify → implement → test → merge — for that part before advancing to the next part. Never run verify for all parts first. If review produced a report, context.review_path is available — pass it as an additional arg to dry-walkthrough." implement: tool: run_skill + model: "" stale_threshold: 2400 with: skill_command: "/autoskillit:implement-worktree-no-merge ${{ context.plan_path }}" @@ -306,6 +242,7 @@ steps: retry_worktree: tool: run_skill + model: "" stale_threshold: 2400 with: skill_command: "/autoskillit:retry-worktree ${{ context.plan_path }} ${{ context.worktree_path }}" @@ -325,15 +262,34 @@ steps: tool: test_check with: worktree_path: "${{ context.worktree_path }}" - on_success: commit_guard + on_success: check_merge_fix_loop on_failure: fix + check_merge_fix_loop: + tool: run_python + with: + callable: "autoskillit.smoke_utils.check_loop_iteration" + current_iteration: "${{ context.merge_fix_count }}" + max_iterations: "3" + capture: + merge_fix_count: "${{ result.next_iteration }}" + on_result: + - when: "${{ result.max_exceeded }} == true" + route: release_issue_failure + - route: commit_guard + on_failure: release_issue_failure + optional_context_refs: + - merge_fix_count + note: > + Iteration guard for the merge→fix→test cycle. Caps the loop at 3 + iterations (enough for legitimate flake recovery but not enough to + burn 30+ minutes on structurally unresolvable conflicts). + commit_guard: - tool: run_cmd + tool: run_python with: - cmd: "cd ${{ context.worktree_path }} && if [ -n \"$(git status --porcelain)\" ]; then git add -A && git commit -m 'chore: commit pending session changes'; fi" - cwd: "${{ context.worktree_path }}" - step_name: commit_guard + callable: "autoskillit.recipe._cmd_rpc.commit_guard" + worktree_path: "${{ context.worktree_path }}" on_success: merge on_failure: merge note: > @@ -357,7 +313,7 @@ steps: - when: "result.failed_step == 'post_rebase_test_gate'" route: fix - when: "result.failed_step == 'rebase'" - route: fix + route: rebase_conflict_fix - when: "result.error" route: release_issue_failure - route: next_or_done @@ -387,6 +343,7 @@ steps: fix: tool: run_skill + model: "" with: skill_command: "/autoskillit:resolve-failures ${{ context.worktree_path }} ${{ context.plan_path }} ${{ inputs.base_branch }}" cwd: "${{ context.worktree_path }}" @@ -407,6 +364,24 @@ steps: on_context_limit: test note: "Fixes test failures without merging. Routes back to test for re-validation before merge_worktree. on_context_limit routes needs_retry=True to test (worktree may already be green) instead of re-running fix blindly." + rebase_conflict_fix: + tool: run_skill + model: "" + with: + skill_command: "/autoskillit:resolve-merge-conflicts ${{ context.worktree_path }} ${{ context.plan_path }} ${{ inputs.base_branch }}" + cwd: "${{ context.worktree_path }}" + step_name: "rebase_conflict_fix" + capture: + conflict_escalation_required: "${{ result.escalation_required }}" + on_result: + - when: "${{ result.escalation_required }} == true" + route: release_issue_failure + - route: test + on_failure: release_issue_failure + on_context_limit: release_issue_failure + retries: 1 + on_exhausted: release_issue_failure + next_or_done: action: route on_result: @@ -420,6 +395,7 @@ steps: audit_impl: tool: run_skill + model: "" with: skill_command: "/autoskillit:audit-impl ${{ context.plan_path }} ${{ context.base_sha }} ${{ inputs.base_branch }}" cwd: "${{ context.work_dir }}" @@ -454,6 +430,8 @@ steps: prepare_pr: tool: run_skill + model: "" + retries: 1 with: skill_command: "/autoskillit:prepare-pr ${{ context.all_plan_paths }} ${{ inputs.run_name }} ${{ inputs.base_branch }}" cwd: "${{ context.work_dir }}" @@ -463,9 +441,15 @@ steps: prep_path: "${{ result.prep_path }}" selected_lenses: "${{ result.selected_lenses }}" lens_context_paths: "${{ result.lens_context_paths }}" - on_success: run_arch_lenses + on_result: + - when: "${{ result.prep_path }} and ${{ inputs.arch_lenses }} == 'true'" + route: run_arch_lenses + - when: "${{ result.prep_path }}" + route: compose_pr + - route: release_issue_failure on_failure: release_issue_failure - on_context_limit: release_issue_failure + on_context_limit: prepare_pr + on_exhausted: release_issue_failure optional: true skip_when_false: "inputs.open_pr" note: > @@ -476,6 +460,7 @@ steps: run_arch_lenses: tool: run_skill + model: "" with: skill_command: "/autoskillit:arch-lens-{slug} {context_path}" cwd: "${{ context.work_dir }}" @@ -500,6 +485,7 @@ steps: compose_pr: tool: run_skill + model: "" with: skill_command: "/autoskillit:compose-pr ${{ context.prep_path }} \"${{ context.all_diagram_paths }}\" ${{ context.work_dir }} ${{ inputs.base_branch }}" cwd: "${{ context.work_dir }}" @@ -531,7 +517,7 @@ steps: optional: true skip_when_false: "inputs.open_pr" note: > - Extracts the integer PR number from the feature branch opened by compose_pr. + Extracts the PR number as a string for downstream steps. pr_number is required by the queue finale steps (enable_auto_merge, wait_for_queue). annotate_pr_diff: @@ -541,48 +527,81 @@ steps: pr_number: "${{ context.pr_number }}" cwd: "${{ context.work_dir }}" output_dir: "${{ context.work_dir }}/{{AUTOSKILLIT_TEMP}}/review-pr" + base_branch: "${{ inputs.base_branch }}" + local_review_rounds: "${{ inputs.local_review_rounds }}" + current_iteration: "${{ context.review_loop_count }}" capture: annotated_diff_path: "${{ result.annotated_diff_path }}" hunk_ranges_path: "${{ result.hunk_ranges_path }}" + diff_metrics_path: "${{ result.diff_metrics_path }}" + review_mode: "${{ result.review_mode }}" on_success: review_pr on_failure: review_pr optional: true skip_when_false: "inputs.open_pr" + optional_context_refs: + - review_loop_count note: > Pre-computes annotated diff with [LNNN] markers and hunk ranges JSON for review-pr. On failure, falls through to review_pr which uses empty fallback. review_pr: tool: run_skill + model: "" with: - skill_command: "/autoskillit:review-pr ${{ context.merge_target }} ${{ inputs.base_branch }} annotated_diff_path=${{ context.annotated_diff_path }} hunk_ranges_path=${{ context.hunk_ranges_path }}" + skill_command: "/autoskillit:review-pr ${{ context.merge_target }} ${{ inputs.base_branch }} annotated_diff_path=${{ context.annotated_diff_path }} hunk_ranges_path=${{ context.hunk_ranges_path }} diff_metrics_path=${{ context.diff_metrics_path }} mode=${{ context.review_mode }}" cwd: "${{ context.work_dir }}" step_name: "review_pr" capture: - verdict: "${{ result.verdict }}" + review_verdict: "${{ result.verdict }}" on_result: - when: "${{ result.verdict }} == changes_requested" - route: resolve_review + route: enrich_diff_context + - when: "${{ result.verdict }} == approved_with_comments" + route: enrich_diff_context - when: "${{ result.verdict }} == needs_human" - route: ci_watch + route: check_review_loop + - when: "${{ result.verdict }} == approved" + route: check_review_loop - when: "true" - route: ci_watch - on_failure: resolve_review - on_context_limit: ci_watch + route: check_review_loop + on_failure: check_repo_ci_event + on_context_limit: check_repo_ci_event optional: true skip_when_false: "inputs.open_pr" + optional_context_refs: + - review_loop_count + - review_mode note: > Runs after compose_pr creates the PR. Invokes the review-pr skill as a headless session to leave inline review comments on the PR. The skill finds the open PR by feature branch name (context.merge_target). Captures verdict - and routes via on_result: changes_requested → resolve_review, - needs_human → ci_watch (explicit), approved → ci_watch. On tool-level failure (MCP error, gh - unavailable), routes to resolve_review. Skipped when open_pr=false. + as review_verdict and routes via on_result: changes_requested/approved_with_comments + → enrich_diff_context → resolve_review; all other verdicts (including approved, needs_human) + → check_review_loop (mandatory waypoint to increment review_loop_count). + On tool-level failure (MCP error, gh unavailable), routes to check_repo_ci_event + (no review to resolve). Skipped when open_pr=false. + + enrich_diff_context: + tool: run_python + with: + callable: "autoskillit.smoke_utils.enrich_diff_context" + pr_number: "${{ context.pr_number }}" + work_dir: "${{ context.work_dir }}" + on_success: resolve_review + on_failure: resolve_review + optional: true + skip_when_false: "inputs.open_pr" + note: > + Enriches the review-pr diff_context handoff file with deterministic + code_region extraction from the annotated diff. On failure, falls through + to resolve_review which uses fallback file-read behavior. resolve_review: tool: run_skill + model: "" with: - skill_command: "/autoskillit:resolve-review ${{ context.merge_target }} ${{ inputs.base_branch }}" + skill_command: "/autoskillit:resolve-review ${{ context.merge_target }} ${{ inputs.base_branch }} mode=${{ context.review_mode }}" cwd: "${{ context.work_dir }}" step_name: "resolve_review" capture: @@ -595,18 +614,20 @@ steps: - when: "${{ result.verdict }} == 'already_green'" route: re_push_review - when: "${{ result.verdict }} == 'flake_suspected'" - route: release_issue_failure + route: re_push_review - when: "${{ result.verdict }} == 'ci_only_failure'" route: release_issue_failure on_failure: release_issue_failure + optional_context_refs: + - review_mode note: > Runs when review_pr reports issues (verdict=changes_requested). The clone (context.work_dir) has the feature branch (context.merge_target) checked out. resolve-review fetches inline and summary review comments via the GitHub API, applies fixes for each actionable finding (critical + warning severity), and - commits each fix. Routes via on_result: verdict dispatch — real_fix/already_green - both route to re_push_review; flake_suspected/ci_only_failure escalate. - Retries up to 2 times (3 total attempts) before exhausting to release_issue_failure. + commits each fix. Routes via on_result: verdict dispatch — real_fix/already_green/ + flake_suspected all route to re_push_review (retry bounded by retries: 2); + ci_only_failure escalates to release_issue_failure. re_push_review: tool: push_to_remote @@ -614,24 +635,96 @@ steps: clone_path: "${{ context.work_dir }}" remote_url: "${{ context.remote_url }}" branch: "${{ context.merge_target }}" - on_success: ci_watch + on_success: check_review_loop on_failure: release_issue_failure note: > Pushes the review-fixed feature branch back to the remote after resolve_review - succeeds. Routes to ci_watch to validate via CI (one-shot review gate). - On push failure, escalates via release_issue_failure. + succeeds. Routes to check_review_loop to determine if another review-resolve + cycle is needed (bounded by review_max_retries). On push failure, escalates + via release_issue_failure. + + check_review_loop: + tool: run_python + with: + callable: "autoskillit.smoke_utils.check_review_loop" + pr_number: "${{ context.pr_number }}" + current_iteration: "${{ context.review_loop_count }}" + max_iterations: "${{ inputs.review_max_retries }}" + previous_verdict: "${{ context.review_verdict }}" + local_review_rounds: "${{ inputs.local_review_rounds }}" + capture: + review_loop_count: "${{ result.next_iteration }}" + on_result: + - when: "${{ result.had_blocking }} == true and ${{ result.max_exceeded }} == false" + route: review_pr + - route: check_repo_ci_event + on_failure: check_repo_ci_event + skip_when_false: "inputs.open_pr" + optional_context_refs: + - review_loop_count + - review_verdict + note: > + Compound iteration + blocking guard. Routes back to review_pr only when + had_blocking=true AND max_exceeded=false. had_blocking is true when + previous_verdict=changes_requested OR (local_review_rounds > 0 AND iteration < local_review_rounds), + ensuring local review rounds are fully exhausted before exiting to CI. + + check_repo_ci_event: + tool: check_repo_merge_state + with: + branch: "${{ context.merge_target }}" + cwd: "${{ context.work_dir }}" + remote_url: "${{ context.remote_url }}" + step_name: "check_repo_ci_event" + capture: + ci_event: "${{ result.ci_event }}" + on_success: check_pr_state + on_failure: ci_watch + skip_when_false: "inputs.open_pr" + note: > + Derives ci_event for the feature branch BEFORE ci_watch runs. + Uses context.merge_target (feature branch) so push.branches filter + is evaluated against the actual branch being pushed, not the base branch. + Non-blocking: on_failure routes to ci_watch with ci_event=null (match-any). + + check_pr_state: + tool: check_pr_mergeable + with: + pr_number: "${{ context.pr_number }}" + cwd: "${{ context.work_dir }}" + on_result: + - when: "${{ result.mergeable_status }} == 'CONFLICTING'" + route: detect_ci_conflict + - route: ci_watch + on_failure: ci_watch + skip_when_false: "inputs.open_pr" + note: > + Pre-CI-watch gate: checks PR mergeable state before entering the CI + polling loop. CONFLICTING PRs cannot trigger pull_request workflows + (GitHub cannot create refs/pull/N/merge), so routing to ci_watch would + produce a guaranteed no_runs. Routes CONFLICTING to detect_ci_conflict + (which handles conflict resolution) and MERGEABLE/UNKNOWN to ci_watch. ci_watch: tool: wait_for_ci with: branch: "${{ context.merge_target }}" - timeout_seconds: 300 + event: "${{ context.ci_event }}" + timeout_seconds: 600 cwd: "${{ context.work_dir }}" remote_url: "${{ context.remote_url }}" + auto_trigger: "true" capture: ci_conclusion: "${{ result.conclusion }}" ci_failed_jobs: "${{ result.failed_jobs }}" - on_success: check_repo_merge_state + on_result: + - when: "${{ result.conclusion }} == 'success'" + route: check_repo_merge_state + - when: "${{ result.conclusion }} == 'no_runs'" + route: handle_no_ci_runs + - when: "${{ result.conclusion }} == 'timed_out'" + route: check_ci_timed_out_loop + - route: detect_ci_conflict on_failure: detect_ci_conflict optional: true skip_when_false: "inputs.open_pr" @@ -640,9 +733,166 @@ steps: using the wait_for_ci MCP tool (timeout: 300 s). Captures structured output (conclusion + failed job names) for downstream resolve_ci. Skipped when open_pr=false — no remote feature branch is opened in that mode. - On success, routes to check_repo_merge_state to determine if the PR should enter - the merge queue. On failure, routes to detect_ci_conflict which checks whether - the failure is caused by a stale base before falling through to diagnose_ci. + Routes on result.conclusion: success → check_repo_merge_state, + no_runs → handle_no_ci_runs (auto_trigger fires first: empty commit + force-push + + re-poll; handle_no_ci_runs reached only when push fails), + timed_out → check_ci_timed_out_loop (iteration guard), failure → detect_ci_conflict. + + check_ci_timed_out_loop: + tool: run_python + with: + callable: "autoskillit.smoke_utils.check_loop_iteration" + current_iteration: "${{ context.ci_timed_out_loop_count }}" + max_iterations: "2" + capture: + ci_timed_out_loop_count: "${{ result.next_iteration }}" + on_result: + - when: "${{ result.max_exceeded }} == true" + route: detect_ci_conflict + - route: ci_watch + on_failure: detect_ci_conflict + optional_context_refs: + - ci_timed_out_loop_count + note: > + Caps the ci_watch re-poll loop at 2 iterations. + After 2 timed_out results (1200s total), routes to detect_ci_conflict + instead of spinning indefinitely. + + handle_no_ci_runs: + tool: check_repo_merge_state + with: + branch: "${{ context.merge_target }}" + cwd: "${{ context.work_dir }}" + remote_url: "${{ context.remote_url }}" + step_name: "handle_no_ci_runs" + capture: + ci_event: "${{ result.ci_event }}" + on_success: check_ci_loop + on_failure: release_issue_failure + retries: 1 + on_exhausted: release_issue_failure + note: > + Intercepts no_runs conclusion from wait_for_ci. Re-derives ci_event using + check_repo_merge_state (the original event may have been wrong for this + branch's workflow configuration) and routes to check_ci_loop for iteration + guard before retrying ci_watch. retries: 1 bounds the tool-call retry; + check_ci_loop bounds the outer loop at 2 iterations. + + check_ci_loop: + tool: run_python + with: + callable: "autoskillit.smoke_utils.check_loop_iteration" + current_iteration: "${{ context.ci_loop_count }}" + max_iterations: "2" + capture: + ci_loop_count: "${{ result.next_iteration }}" + on_result: + - when: "${{ result.max_exceeded }} == true" + route: check_active_trigger_loop + - route: ci_watch + on_failure: check_ci_already_passed + optional_context_refs: + - ci_loop_count + note: > + Iteration guard for the ci_watch / handle_no_ci_runs cycle. Caps the + loop at 2 iterations (initial + 1 retry after re-deriving ci_event). + On budget exhaustion, routes to check_active_trigger_loop which caps + the active trigger to a single attempt before escalating. + + check_active_trigger_loop: + tool: run_python + with: + callable: "autoskillit.smoke_utils.check_loop_iteration" + current_iteration: "${{ context.active_trigger_count }}" + max_iterations: "2" + capture: + active_trigger_count: "${{ result.next_iteration }}" + on_result: + - when: "${{ result.max_exceeded }} == true" + route: check_ci_already_passed + - route: trigger_ci_actively + on_failure: check_ci_already_passed + optional_context_refs: + - active_trigger_count + note: > + Bounds the trigger_ci_actively remediation to a single attempt. If + ci_watch still returns no_runs after the active trigger, escalates + to escalate_stop_no_ci rather than re-triggering indefinitely. + + trigger_ci_actively: + tool: run_cmd + with: + cmd: "gh pr close ${{ context.pr_number }} && sleep 2 && gh pr reopen ${{ context.pr_number }}" + cwd: "${{ context.work_dir }}" + step_name: "trigger_ci_actively" + on_success: ci_watch + on_failure: check_ci_already_passed + retries: 1 + on_exhausted: check_ci_already_passed + note: > + Active CI trigger remediation: closes and reopens the PR to force GitHub + webhook re-delivery. This re-creates the pull_request event and the + refs/pull/N/merge ref (if the PR is mergeable). Routes back to ci_watch + for one final attempt. If the close/reopen fails or the subsequent + ci_watch still returns no_runs, escalates to escalate_stop_no_ci. + + check_ci_already_passed: + tool: run_cmd + with: + cmd: >- + gh pr view '${{ context.pr_number }}' --json statusCheckRollup + --jq '.statusCheckRollup | if length == 0 then "no_checks" elif + all(.state == "SUCCESS" or .state == "NEUTRAL" or .state == "SKIPPED") + then "passed" elif any(.state == "FAILURE" or .state == "ERROR") + then "failed" else "pending" end' + cwd: "${{ context.work_dir }}" + step_name: "check_ci_already_passed" + on_result: + - when: "${{ result.stdout | trim }} == passed" + route: check_repo_merge_state + - when: "${{ result.stdout | trim }} == failed" + route: detect_ci_conflict + - when: "${{ result.stdout | trim }} == pending" + route: ci_watch + - route: mark_issue_failed_no_ci + on_failure: mark_issue_failed_no_ci + skip_when_false: "inputs.open_pr" + note: > + Safety net before terminal escalation: checks if CI already passed + despite the watcher failing to observe it (e.g. post-force-push SHA + staleness). Uses statusCheckRollup as the authoritative signal. + Outputs passed/failed/no_checks/pending: passed goes to merge gate, + failed routes to CI conflict remediation, pending re-enters the CI + watcher, no_checks falls through to terminal stop. + + mark_issue_failed_no_ci: + description: "Swap in-progress → fail label before CI failure escalation" + tool: release_issue + optional: true + skip_when_false: inputs.issue_url + with: + issue_url: ${{ inputs.issue_url }} + fail_label: "fail" + on_success: register_clone_no_ci + on_failure: register_clone_no_ci + + register_clone_no_ci: + description: "Register clone as preserved (no-CI error pipeline)" + tool: register_clone_status + with: + clone_path: "${{ context.work_dir }}" + status: "error" + step_name: register_clone_no_ci + on_success: escalate_stop_no_ci + on_failure: escalate_stop_no_ci + + escalate_stop_no_ci: + action: stop + message: >- + CI safety net: statusCheckRollup returned no checks for PR + ${{ context.pr_number }}. This typically means no workflow is + configured to trigger on this branch pattern. Check .github/workflows/ + trigger configuration. This is NOT a test failure — CI never started. check_repo_merge_state: tool: check_repo_merge_state @@ -673,46 +923,70 @@ steps: action: route on_result: - when: "${{ inputs.auto_merge }} != 'true'" - route: register_clone_success - - when: "${{ context.queue_available }} == true and ${{ context.merge_group_trigger }} == true and ${{ context.auto_merge_available }} == true" - route: enable_auto_merge - - when: "${{ context.queue_available }} == true and ${{ context.merge_group_trigger }} == true and ${{ context.auto_merge_available }} == false" - route: queue_enqueue_no_auto + route: patch_token_summary + - when: "${{ context.queue_available }} == true and ${{ context.merge_group_trigger }} == true" + route: enqueue_to_queue - when: "${{ context.auto_merge_available }} == true" route: direct_merge - route: immediate_merge skip_when_false: "inputs.open_pr" - enable_auto_merge: - tool: run_cmd + enqueue_to_queue: + tool: enqueue_pr with: - cmd: "gh pr merge '${{ context.pr_number }}' --squash --auto" + pr_number: "${{ context.pr_number }}" + target_branch: "${{ inputs.base_branch }}" cwd: "${{ context.work_dir }}" - step_name: "enable_auto_merge" + auto_merge_available: "${{ context.auto_merge_available }}" on_success: wait_for_queue - on_failure: register_clone_success + on_failure: verify_queue_enrollment skip_when_false: "inputs.open_pr" note: > - Submits the PR to the merge queue. On failure (PR not mergeable, already merged), - degrades to register_clone_success. - - queue_enqueue_no_auto: - tool: run_cmd + Submits the PR to the merge queue via the unified enqueue_pr tool. + The tool resolves the enrollment strategy internally based on + auto_merge_available. On failure, probes the queue via + verify_queue_enrollment to distinguish a transient enqueue error from a + PR that is already merged or still in queue. + + verify_queue_enrollment: + description: "Probe merge queue after enqueue command failure to determine actual state" + tool: wait_for_merge_queue with: - cmd: "gh pr merge '${{ context.pr_number }}' --squash" + pr_number: "${{ context.pr_number }}" + target_branch: "${{ inputs.base_branch }}" cwd: "${{ context.work_dir }}" - step_name: "queue_enqueue_no_auto" - on_success: wait_for_queue - on_failure: register_clone_success - skip_when_false: "inputs.open_pr" + remote_url: "${{ context.remote_url }}" + timeout_seconds: 60 + poll_interval: 10 + on_result: + - when: "${{ result.pr_state }} == merged" + route: release_issue_success + - when: "${{ result.pr_state }} == ejected_ci_failure" + route: diagnose_ci + - when: "${{ result.pr_state }} == not_enrolled" + route: release_issue_failure + - when: "${{ result.pr_state }} == dropped_healthy" + route: wait_for_queue + - when: "${{ result.pr_state }} == dropped_merge_group_ci" + route: diagnose_ci + - when: "${{ result.pr_state }} == ejected" + route: wait_for_queue + - when: "${{ result.pr_state }} == stalled" + route: wait_for_queue + - when: "${{ result.pr_state }} == timeout" + route: register_clone_unconfirmed + - route: register_clone_unconfirmed + on_failure: register_clone_unconfirmed note: > - Submits the PR to the merge queue when the target branch has a configured - merge queue but the repository disallows auto-merge (autoMergeAllowed=false). - Uses plain `gh pr merge --squash` (no --auto, which would be rejected by - GitHub's API auto-merge gate). GitHub intercepts the call and enqueues - the PR because the branch has a queue. Routes to wait_for_queue (the - shared queue waiter) — never to wait_for_immediate_merge, whose 5-minute - poll is too short for queue serialization. + Short probe (60s) after enqueue_to_queue failure. If the PR is already + merged, route to release_issue_success. If the queue already confirms CI + failure (ejected_ci_failure or dropped_merge_group_ci), route directly to + diagnose_ci. If enrollment was attempted but the PR never appeared in the + queue (not_enrolled), route to release_issue_failure — this is a repo + configuration error. For other active queue states (dropped_healthy, ejected, + stalled), route to wait_for_queue for full 900s authoritative watch. If the + probe timed out or returned an unexpected state, escalate via + register_clone_unconfirmed. wait_for_queue: tool: wait_for_merge_queue @@ -729,47 +1003,103 @@ steps: route: release_issue_success - when: "${{ result.pr_state }} == ejected_ci_failure" route: diagnose_ci + - when: "${{ result.pr_state }} == dropped_merge_group_ci" + route: diagnose_ci + - when: "${{ result.pr_state }} == not_enrolled" + route: release_issue_failure - when: "${{ result.pr_state }} == dropped_healthy" - route: reenter_merge_queue_cheap + route: check_dropped_healthy_loop - when: "${{ result.pr_state }} == ejected" - route: queue_ejected_fix + route: check_eject_limit - when: "${{ result.pr_state }} == stalled" route: reenroll_stalled_pr - when: "${{ result.pr_state }} == timeout" - route: release_issue_timeout - - route: release_issue_timeout - on_failure: release_issue_timeout + route: register_clone_unconfirmed + - route: register_clone_unconfirmed + on_failure: register_clone_unconfirmed skip_when_false: "inputs.open_pr" note: > Waits up to 15 minutes for the PR to merge through the queue. merged → release_issue_success to transition issue state then register_clone_success. ejected_ci_failure → diagnose_ci (CI failure; conflict resolution cannot fix this). + not_enrolled → release_issue_failure (enrollment failed; repo configuration error). ejected → conflict fix cycle then re-enter. - stalled → lightweight re-enrollment (disable then re-enable auto-merge). - timeout → release_issue_timeout (release claim without staged label; merge unconfirmed). + stalled → lightweight re-enrollment via enqueue_pr. + timeout → register_clone_unconfirmed (release claim without staged label; merge unconfirmed). reenroll_stalled_pr: - tool: toggle_auto_merge + tool: enqueue_pr with: pr_number: "${{ context.pr_number }}" target_branch: "${{ inputs.base_branch }}" cwd: "${{ context.work_dir }}" - on_success: wait_for_queue + auto_merge_available: "${{ context.auto_merge_available }}" + on_success: check_stall_loop on_failure: release_issue_failure skip_when_false: "inputs.open_pr" + check_stall_loop: + tool: run_python + with: + callable: "autoskillit.smoke_utils.check_loop_iteration" + current_iteration: "${{ context.stall_loop_count }}" + max_iterations: "3" + capture: + stall_loop_count: "${{ result.next_iteration }}" + on_result: + - when: "${{ result.max_exceeded }} == true" + route: register_clone_unconfirmed + - route: wait_for_queue + on_failure: register_clone_unconfirmed + optional_context_refs: + - stall_loop_count + note: > + Iteration guard for the wait_for_queue / reenroll_stalled_pr cycle. + Caps stall re-enrollment at 3 attempts. On budget exhaustion, routes + to register_clone_unconfirmed (merge outcome unconfirmed). + + check_eject_limit: + tool: run_python + with: + callable: "autoskillit.recipe._cmd_rpc.check_eject_limit" + counter_file: "${{ context.work_dir }}/{{AUTOSKILLIT_TEMP}}/eject_count" + max_ejects: "3" + on_result: + - when: "${{ result.status }} == EJECT_LIMIT_EXCEEDED" + route: release_issue_failure + - route: queue_ejected_fix + on_failure: release_issue_failure + note: > + Circuit breaker for the queue ejection loop. Tracks ejection count in a + file-based counter under {{AUTOSKILLIT_TEMP}}/. After 3 ejections, routes + to release_issue_failure to prevent infinite re-enqueue cycles. + + check_dropped_healthy_loop: + tool: run_python + with: + callable: "autoskillit.recipe._cmd_rpc.check_dropped_healthy_loop" + counter_file: "${{ context.work_dir }}/{{AUTOSKILLIT_TEMP}}/dropped_healthy_count" + max_drops: "2" + on_result: + - when: "${{ result.status }} == DROPPED_LIMIT_EXCEEDED" + route: release_issue_failure + - route: reenter_merge_queue_cheap + on_failure: release_issue_failure + note: > + Circuit breaker for the dropped_healthy re-enrollment loop. After 2 consecutive + unexplained drops with healthy PR-branch CI, escalates to release_issue_failure + instead of looping. Matches the check_eject_limit pattern used for the ejected + path. DROPPED_MERGE_GROUP_CI (confirmed merge-group CI failure) is routed + directly to diagnose_ci and never reaches this step. + queue_ejected_fix: - tool: run_cmd + tool: run_python with: - cmd: > - if git fetch origin ${{ inputs.base_branch }} && git rebase origin/${{ inputs.base_branch }}; - then echo 'clean'; - else git rebase --abort 2>/dev/null || true; echo 'conflicts'; - fi - cwd: "${{ context.work_dir }}" - step_name: "queue_ejected_fix" + callable: "autoskillit.recipe._cmd_rpc.queue_ejected_fix" + work_dir: "${{ context.work_dir }}" + base_branch: "${{ inputs.base_branch }}" on_result: - - when: "${{ result.stdout | trim }} == clean" + - when: "${{ result.status }} == clean" route: re_push_queue_fix - route: resolve_queue_merge_conflicts on_failure: resolve_queue_merge_conflicts @@ -782,6 +1112,7 @@ steps: resolve_queue_merge_conflicts: tool: run_skill + model: "" with: skill_command: "/autoskillit:resolve-merge-conflicts ${{ context.work_dir }} ${{ context.plan_path }} ${{ inputs.base_branch }}" cwd: "${{ context.work_dir }}" @@ -809,35 +1140,106 @@ steps: remote_url: "${{ context.remote_url }}" branch: "${{ context.merge_target }}" force: "true" + capture: + push_error_type: "${{ result.error_type }}" + retries: 2 + on_exhausted: register_clone_unconfirmed on_success: ci_watch_post_queue_fix - on_failure: release_issue_failure + on_failure: classify_push_failure skip_when_false: "inputs.open_pr" + classify_push_failure: + action: route + on_result: + - when: "${{ context.push_error_type }} == queued_branch" + route: wait_dequeue_retry + - route: release_issue_failure + + wait_dequeue_retry: + tool: wait_for_merge_queue + with: + pr_number: "${{ context.pr_number }}" + target_branch: "${{ inputs.base_branch }}" + cwd: "${{ context.work_dir }}" + timeout_seconds: "60" + on_result: + - when: "${{ result.pr_state }} == merged" + route: release_issue_success + - when: "${{ result.pr_state }} == ejected_ci_failure" + route: register_clone_unconfirmed + - when: "${{ result.pr_state }} == dropped_merge_group_ci" + route: register_clone_unconfirmed + - when: "${{ result.pr_state }} == dropped_healthy" + route: register_clone_unconfirmed + - when: "${{ result.pr_state }} == ejected" + route: re_push_queue_fix + - when: "${{ result.pr_state }} == stalled" + route: register_clone_unconfirmed + - when: "${{ result.pr_state }} == timeout" + route: register_clone_unconfirmed + - route: register_clone_unconfirmed + on_failure: register_clone_unconfirmed + optional_context_refs: + - push_error_type + ci_watch_post_queue_fix: tool: wait_for_ci with: branch: "${{ context.merge_target }}" event: "${{ context.ci_event }}" - timeout_seconds: 300 + timeout_seconds: 600 cwd: "${{ context.work_dir }}" remote_url: "${{ context.remote_url }}" + auto_trigger: "true" capture: ci_conclusion: "${{ result.conclusion }}" ci_failed_jobs: "${{ result.failed_jobs }}" - on_success: reenter_merge_queue + on_result: + - when: "${{ result.conclusion }} == 'success'" + route: reenter_merge_queue + - when: "${{ result.conclusion }} == 'no_runs'" + route: release_issue_failure + - when: "${{ result.conclusion }} == 'timed_out'" + route: check_ci_post_queue_loop + - route: detect_ci_conflict on_failure: detect_ci_conflict skip_when_false: "inputs.open_pr" note: > Validates CI on the rebased feature branch after queue ejection fix and re-push. - Mirrors ci_watch: routes to reenter_merge_queue on CI pass, to detect_ci_conflict - on CI failure. Prevents re-entering the merge queue with a broken manifest or code. + Routes on result.conclusion: success → reenter_merge_queue, + no_runs → release_issue_failure (auto_trigger fires first: empty commit + + force-push + re-poll; no_runs reached only when trigger recovery fails), + timed_out → check_ci_post_queue_loop, failure → detect_ci_conflict. + + check_ci_post_queue_loop: + tool: run_python + with: + callable: "autoskillit.smoke_utils.check_loop_iteration" + current_iteration: "${{ context.ci_post_queue_loop_count }}" + max_iterations: "2" + capture: + ci_post_queue_loop_count: "${{ result.next_iteration }}" + on_result: + - when: "${{ result.max_exceeded }} == true" + route: release_issue_failure + - route: ci_watch_post_queue_fix + on_failure: release_issue_failure + optional_context_refs: + - ci_post_queue_loop_count + skip_when_false: "inputs.open_pr" + note: > + Iteration guard for the ci_watch_post_queue_fix timed_out self-loop. + Caps at 2 iterations (initial + 1 retry). On budget exhaustion, + routes to release_issue_failure — this is already a post-ejection + retry path so further escalation is not warranted. reenter_merge_queue: - tool: run_cmd + tool: enqueue_pr with: - cmd: "gh pr merge '${{ context.pr_number }}' --squash --auto" + pr_number: "${{ context.pr_number }}" + target_branch: "${{ inputs.base_branch }}" cwd: "${{ context.work_dir }}" - step_name: "reenter_merge_queue" + auto_merge_available: "${{ context.auto_merge_available }}" on_success: wait_for_queue on_failure: release_issue_failure skip_when_false: "inputs.open_pr" @@ -846,11 +1248,12 @@ steps: If re-entry fails, release_issue_failure preserves the clone for debugging. reenter_merge_queue_cheap: - tool: run_cmd + tool: enqueue_pr with: - cmd: "gh pr merge '${{ context.pr_number }}' --squash --auto" + pr_number: "${{ context.pr_number }}" + target_branch: "${{ inputs.base_branch }}" cwd: "${{ context.work_dir }}" - step_name: "reenter_merge_queue_cheap" + auto_merge_available: "${{ context.auto_merge_available }}" on_success: wait_for_queue on_failure: release_issue_failure skip_when_false: "inputs.open_pr" @@ -866,51 +1269,43 @@ steps: cwd: "${{ context.work_dir }}" step_name: "direct_merge" on_success: wait_for_direct_merge - on_failure: register_clone_success + on_failure: release_issue_failure skip_when_false: "inputs.open_pr" note: > Enables GitHub native auto-merge for direct merge (no merge queue). GitHub - merges automatically once all required checks pass. On failure (PR already - merged, not mergeable), degrades gracefully to register_clone_success. + merges automatically once all required checks pass. On failure, escalates + via release_issue_failure so the issue claim is released and the pipeline + reports an error rather than silently succeeding. wait_for_direct_merge: - tool: run_cmd + tool: run_python with: - cmd: > - for i in $(seq 1 90); do - STATE=$(gh pr view '${{ context.pr_number }}' --json state --jq '.state'); - if [ "$STATE" = "MERGED" ]; then echo "merged"; exit 0; fi; - if [ "$STATE" = "CLOSED" ]; then echo "closed"; exit 0; fi; - sleep 10; - done; echo "timeout" - cwd: "${{ context.work_dir }}" - step_name: "wait_for_direct_merge" + callable: "autoskillit.recipe._cmd_rpc.wait_for_direct_merge" + pr_number: "${{ context.pr_number }}" + max_polls: "90" + poll_interval: "10" on_result: - - when: "${{ result.stdout | trim }} == merged" + - when: "${{ result.state }} == merged" route: release_issue_success - - when: "${{ result.stdout | trim }} == closed" + - when: "${{ result.state }} == closed" route: direct_merge_conflict_fix - - route: release_issue_timeout - on_failure: release_issue_timeout + - route: register_clone_unconfirmed + on_failure: register_clone_unconfirmed skip_when_false: "inputs.open_pr" note: > Polls GitHub PR state every 10 s for up to 15 minutes (90 iterations). merged → release_issue_success (issue labelled, cleanup runs). closed → PR was closed (likely conflict) — route to conflict fix. - timeout / unexpected → release_issue_timeout. + timeout / unexpected → register_clone_unconfirmed. direct_merge_conflict_fix: - tool: run_cmd + tool: run_python with: - cmd: > - if git fetch origin ${{ inputs.base_branch }} && git rebase origin/${{ inputs.base_branch }}; - then echo 'clean'; - else git rebase --abort 2>/dev/null || true; echo 'conflicts'; - fi - cwd: "${{ context.work_dir }}" - step_name: "direct_merge_conflict_fix" + callable: "autoskillit.recipe._cmd_rpc.direct_merge_conflict_fix" + work_dir: "${{ context.work_dir }}" + base_branch: "${{ inputs.base_branch }}" on_result: - - when: "${{ result.stdout | trim }} == clean" + - when: "${{ result.status }} == clean" route: re_push_direct_fix - route: resolve_direct_merge_conflicts on_failure: resolve_direct_merge_conflicts @@ -922,6 +1317,7 @@ steps: resolve_direct_merge_conflicts: tool: run_skill + model: "" with: skill_command: "/autoskillit:resolve-merge-conflicts ${{ context.work_dir }} ${{ context.plan_path }} ${{ inputs.base_branch }}" cwd: "${{ context.work_dir }}" @@ -974,33 +1370,29 @@ steps: cwd: "${{ context.work_dir }}" step_name: "immediate_merge" on_success: wait_for_immediate_merge - on_failure: register_clone_success + on_failure: release_issue_failure skip_when_false: "inputs.open_pr" note: > Merges the PR immediately using plain --squash when autoMergeAllowed=false. Unlike direct_merge, no --auto flag is used — GitHub executes the merge - synchronously rather than queueing it for when checks pass. On failure - (conflict or not-mergeable error from GitHub), degrades to register_clone_success. + synchronously rather than queueing it for when checks pass. On failure, + escalates via release_issue_failure so the issue claim is released and the + pipeline reports an error rather than silently succeeding. wait_for_immediate_merge: - tool: run_cmd + tool: run_python with: - cmd: > - for i in $(seq 1 30); do - STATE=$(gh pr view '${{ context.pr_number }}' --json state --jq '.state'); - if [ "$STATE" = "MERGED" ]; then echo "merged"; exit 0; fi; - if [ "$STATE" = "CLOSED" ]; then echo "closed"; exit 0; fi; - sleep 10; - done; echo "timeout" - cwd: "${{ context.work_dir }}" - step_name: "wait_for_immediate_merge" + callable: "autoskillit.recipe._cmd_rpc.wait_for_immediate_merge" + pr_number: "${{ context.pr_number }}" + max_polls: "30" + poll_interval: "10" on_result: - - when: "${{ result.stdout | trim }} == merged" + - when: "${{ result.state }} == merged" route: release_issue_success - - when: "${{ result.stdout | trim }} == closed" + - when: "${{ result.state }} == closed" route: immediate_merge_conflict_fix - - route: release_issue_timeout - on_failure: release_issue_timeout + - route: register_clone_unconfirmed + on_failure: register_clone_unconfirmed skip_when_false: "inputs.open_pr" note: > Polls GitHub PR state every 10s for up to 5 minutes (30 iterations). @@ -1008,20 +1400,16 @@ steps: synchronously — this poll is a confirmation window. merged → release_issue_success. closed → PR was rejected (stale base or conflict) — route to conflict fix. - timeout → release_issue_timeout. + timeout → register_clone_unconfirmed. immediate_merge_conflict_fix: - tool: run_cmd + tool: run_python with: - cmd: > - if git fetch origin ${{ inputs.base_branch }} && git rebase origin/${{ inputs.base_branch }}; - then echo 'clean'; - else git rebase --abort 2>/dev/null || true; echo 'conflicts'; - fi - cwd: "${{ context.work_dir }}" - step_name: "immediate_merge_conflict_fix" + callable: "autoskillit.recipe._cmd_rpc.immediate_merge_conflict_fix" + work_dir: "${{ context.work_dir }}" + base_branch: "${{ inputs.base_branch }}" on_result: - - when: "${{ result.stdout | trim }} == clean" + - when: "${{ result.status }} == clean" route: re_push_immediate_fix - route: resolve_immediate_merge_conflicts on_failure: resolve_immediate_merge_conflicts @@ -1033,6 +1421,7 @@ steps: resolve_immediate_merge_conflicts: tool: run_skill + model: "" with: skill_command: "/autoskillit:resolve-merge-conflicts ${{ context.work_dir }} ${{ context.plan_path }} ${{ inputs.base_branch }}" cwd: "${{ context.work_dir }}" @@ -1080,6 +1469,7 @@ steps: diagnose_ci: tool: run_skill + model: "" with: skill_command: "/autoskillit:diagnose-ci ${{ context.merge_target }} - - - ${{ context.ci_event }}" cwd: "${{ context.work_dir }}" @@ -1100,6 +1490,7 @@ steps: resolve_ci: tool: run_skill + model: "" with: skill_command: "/autoskillit:resolve-failures ${{ context.work_dir }} ${{ context.plan_path }} ${{ inputs.base_branch }} ${{ context.ci_conclusion }} ${{ context.ci_failed_jobs }} ${{ context.diagnosis_path }}" cwd: "${{ context.work_dir }}" @@ -1115,7 +1506,7 @@ steps: - when: "${{ result.verdict }} == 'already_green'" route: pre_resolve_rebase - when: "${{ result.verdict }} == 'flake_suspected'" - route: release_issue_failure + route: re_push - when: "${{ result.verdict }} == 'ci_only_failure'" route: release_issue_failure - route: release_issue_failure @@ -1124,13 +1515,17 @@ steps: Runs when ci_watch reports a CI failure. Receives ci_failed_jobs from wait_for_ci structured output to avoid redundant CI failure re-investigation. Routes via on_result: verdict dispatch — real_fix pushes, already_green rebases and re-diagnoses - (sibling fix may have landed), flake_suspected/ci_only_failure escalate to human. + (sibling fix may have landed), flake_suspected retries via re_push (bounded by + retries: 2 / on_exhausted: release_issue_failure), ci_only_failure escalates to human. Retries up to 2 times (3 total attempts) before exhausting to release_issue_failure. pre_resolve_rebase: tool: run_cmd with: - cmd: "git -C ${{ context.work_dir }} fetch origin && git -C ${{ context.work_dir }} rebase origin/${{ inputs.base_branch }}" + cmd: | + REMOTE=$(git -C ${{ context.work_dir }} remote get-url upstream 2>/dev/null | grep -qv "^file://" && echo upstream || echo origin) && + git -C ${{ context.work_dir }} fetch "$REMOTE" && + git -C ${{ context.work_dir }} rebase "$REMOTE"/${{ inputs.base_branch }} step_name: "pre_resolve_rebase" retries: 1 on_success: diagnose_ci @@ -1148,17 +1543,20 @@ steps: remote_url: "${{ context.remote_url }}" branch: "${{ context.merge_target }}" force: "true" - on_success: ci_watch + on_success: check_repo_ci_event on_failure: release_issue_failure note: > Pushes the CI-fixed feature branch back to the remote after resolve_ci succeeds. - Routes back to ci_watch to verify the fix resolves CI. On push failure, escalates - via release_issue_failure. + Routes to check_repo_ci_event to derive ci_event before ci_watch verifies the fix. + On push failure, escalates via release_issue_failure. detect_ci_conflict: tool: run_cmd with: - cmd: "git fetch origin ${{ inputs.base_branch }} && ! git merge-base --is-ancestor origin/${{ inputs.base_branch }} HEAD" + cmd: | + REMOTE=$(git remote get-url upstream 2>/dev/null | grep -qv "^file://" && echo upstream || echo origin) && + git fetch "$REMOTE" ${{ inputs.base_branch }} && + ! git merge-base --is-ancestor "$REMOTE"/${{ inputs.base_branch }} HEAD cwd: "${{ context.work_dir }}" step_name: "detect_ci_conflict" on_success: ci_conflict_fix @@ -1174,6 +1572,7 @@ steps: ci_conflict_fix: tool: run_skill + model: "" with: skill_command: "/autoskillit:resolve-merge-conflicts ${{ context.work_dir }} ${{ context.plan_path }} ${{ inputs.base_branch }}" cwd: "${{ context.work_dir }}" @@ -1204,28 +1603,43 @@ steps: with: issue_url: ${{ inputs.issue_url }} target_branch: ${{ inputs.base_branch }} - on_success: register_clone_success - on_failure: register_clone_success + on_success: patch_token_summary + on_failure: patch_token_summary note: > Called on the success path after PR merge is confirmed. Passes target_branch so release_issue can determine whether to apply the staged label (non-default target) or just remove in-progress (default target). optional=true ensures pipeline still reaches register_clone_success if this step errors. - release_issue_timeout: - description: "Release in-progress claim after queue timeout without applying staged label" - tool: release_issue + patch_token_summary: + description: "Collect all pipeline token data and patch the PR body with complete summary" + tool: run_python optional: true - skip_when_false: inputs.issue_url + skip_when_false: "inputs.open_pr" with: - issue_url: ${{ inputs.issue_url }} + callable: "autoskillit.smoke_utils.patch_pr_token_summary" + timeout: 60 + pr_url: "${{ context.pr_url }}" + cwd: "${{ context.work_dir }}" on_success: register_clone_success on_failure: register_clone_success note: > - Called when wait_for_merge_queue times out or errors. Releases the in-progress - claim without passing target_branch, so the staged label is not applied. The - merge is unconfirmed; a human must verify queue state independently. - Routes to register_clone_success so the pipeline completes cleanly. + Patches PR body with complete token summary; failure never blocks pipeline completion. + + register_clone_unconfirmed: + description: "Register clone as preserved — merge unconfirmed, in-progress label retained" + tool: register_clone_status + with: + clone_path: "${{ context.work_dir }}" + status: "unconfirmed" + step_name: register_clone_unconfirmed + on_success: done + on_failure: done + note: > + Called when a merge-wait step times out before merge is confirmed. + Does NOT call release_issue — the in-progress label stays on the issue + so operators can see the PR is still actively queued. The clone is + preserved (status: unconfirmed) for human inspection. release_issue_failure: description: "Release the in-progress claim after pipeline failure" @@ -1234,6 +1648,7 @@ steps: skip_when_false: inputs.issue_url with: issue_url: ${{ inputs.issue_url }} + fail_label: "fail" on_success: register_clone_failure on_failure: register_clone_failure diff --git a/src/autoskillit/recipes/merge-prs.json b/src/autoskillit/recipes/merge-prs.json new file mode 100644 index 0000000000..500015478a --- /dev/null +++ b/src/autoskillit/recipes/merge-prs.json @@ -0,0 +1,1315 @@ +{ + "name": "merge-prs", + "description": "Analyze open PRs, determine merge order, collapse them sequentially into an integration branch, and open a single review PR for human approval. Handles conflict resolution via plan+implement for complex PRs.", + "summary": "clone > setup_remote > analyze_prs > create_batch_branch > [loop per PR: merge_pr or (plan > verify > implement > test > push_worktree_branch > create_conflict_pr > wait_for_conflict_ci > merge_conflict_pr)] > push_batch_branch > collect_artifacts > check_impl_plans > (audit_impl?) > open_integration_pr > wait_for_review_pr_mergeability > check_mergeability > (resolve_integration_conflicts > force_push_after_rebase)? > review_pr_integration > (resolve_review_integration > re_push_review_integration)? > ci_watch_pr > cleanup", + "recipe_version": "1.0.0", + "requires_packs": [ + "github", + "ci", + "clone", + "telemetry" + ], + "ingredients": { + "source_dir": { + "description": "Source repository", + "required": false, + "default": "" + }, + "run_name": { + "description": "Run name prefix", + "default": "pr-merge" + }, + "base_branch": { + "description": "Branch that PRs target", + "required": false, + "default": "" + }, + "audit": { + "description": "Post-merge quality gate", + "default": "false" + }, + "plans_dir": { + "description": "Plan files for audit", + "default": "{{AUTOSKILLIT_TEMP}}/merge-prs" + } + }, + "kitchen_rules": [ + "NEVER use native Claude Code tools (Read, Grep, Glob, Edit, Write, Bash, Agent, WebFetch, WebSearch, NotebookEdit) from the orchestrator. All work is delegated through run_skill and run_cmd.", + "Route to on_failure when a step fails — do not investigate or fix directly.", + "SEQUENTIAL LOOP: Process one PR at a time through the full merge cycle before advancing to the next PR. Never batch-assess all PRs before starting merges.", + "SEQUENTIAL EXECUTION: complete full cycle (verify → implement → test → push_worktree_branch → create_conflict_pr → wait_for_conflict_ci → merge_conflict_pr) per plan part before advancing to the next part or PR.", + "INTEGRATION BRANCH: Two distinct branches exist. inputs.base_branch is the target branch that PRs point to (e.g. main, develop). context.batch_branch is the per-run batch branch (e.g. pr-batch/pr-merge-{ts}) created from base_branch for this pipeline run. All PR merges and worktree merges target context.batch_branch. The final review PR opens from context.batch_branch into inputs.base_branch.", + "PR QUEUE: The agent reads context.pr_order_file once at analyze_prs and maintains an ordered queue. context.current_pr_number tracks the current PR. advance_queue_pr captures the next PR number at the recipe level after each successful merge.", + "COMPLEX PR PATH: When merge_pr returns needs_plan=true, context.task is set to the conflict_report_path. Pass context.task to make-plan. The plan+implement cycle creates a worktree from the integration branch; the worktree branch is then pushed to origin and merged into the integration batch branch via GitHub PR (push_worktree_branch → create_conflict_pr → wait_for_conflict_ci → merge_conflict_pr).", + "SOURCE ISOLATION: After clone_repo returns, the source_dir is strictly off-limits. Never run any command in source_dir — no git checkout, git fetch, git reset, git pull, run_cmd, run_skill, or any other operation. All work — skill invocations, git operations, file reads — happens exclusively in the clone (work_dir). source_dir is used ONLY to read the remote URL inside push_to_remote.", + "GITHUB API MERGE: All PR merges into the integration branch go through GitHub's merge interface (gh pr merge --squash). This enforces the branch ruleset required status checks. Never use merge_worktree or local git merge to merge into the integration batch branch.", + "PUSH DISCIPLINE: push_worktree_branch (inside the complex PR cycle), publish_batch_branch / push_batch_branch, force_push_and_wait_mergeability (authorized for conflict-resolution rebases), and push_rebased_next_pr (authorized for proactive pre-enqueue rebase) are the only authorized push steps. Never add ad-hoc run_cmd git push calls elsewhere in the loop.", + "QUEUE MODE: When context.queue_mode == true, PRs merge directly into base_branch via the merge queue. The batch branch (context.batch_branch) is the same as base_branch. Never invoke create_batch_branch, publish_batch_branch, merge_pr, or push_batch_branch in queue mode.", + "QUEUE LOOP: In queue mode, context.current_pr_number tracks the current PR being waited on. Initialized by get_first_pr_number from pr_order_file[0]. advance_queue_pr looks up the current number in pr_order_file and captures the next entry's number.", + "CONFLICT REPORT PATHS: context.all_conflict_report_paths accumulates conflict_report_path values from resolve_ejected_conflicts and resolve_integration_conflicts via capture_list. Pass it as the 5th argument to open_integration_pr and 4th argument to audit_impl. It is empty string when no conflict resolution occurred — downstream skills treat empty string as absent." + ], + "steps": { + "clone": { + "python": "autoskillit.workspace.clone.clone_repo", + "with": { + "source_dir": "${{ inputs.source_dir }}", + "run_name": "${{ inputs.run_name }}" + }, + "capture": { + "work_dir": "${{ result.clone_path }}", + "remote_url": "${{ result.remote_url }}" + }, + "on_success": "setup_remote", + "on_failure": "escalate_stop", + "note": "Clones source_dir into ../autoskillit-runs/-/. Captures work_dir used as cwd by all subsequent steps. On failure, escalate_stop is reached immediately — no clone to remove.\n" + }, + "setup_remote": { + "tool": "run_cmd", + "with": { + "cmd": "git remote set-url origin \"${{ context.remote_url }}\"", + "cwd": "${{ context.work_dir }}", + "step_name": "setup_remote" + }, + "on_success": "check_repo_ci_event", + "on_failure": "register_clone_failure", + "note": "The clone's origin initially points to a file:// URL (set by clone_repo for isolation). This step replaces it with the actual GitHub remote URL so that gh CLI commands (gh pr merge, gh pr view, etc.) can identify the correct GitHub repository. Skills that contact remotes (resolve-merge-conflicts, retry-worktree, implement-worktree) now use REMOTE=$(upstream || origin) internally and do not depend on this step.\n" + }, + "check_repo_ci_event": { + "tool": "check_repo_merge_state", + "block": "pre_merge_prs_ci_gate", + "with": { + "branch": "${{ inputs.base_branch }}", + "cwd": "${{ context.work_dir }}", + "remote_url": "${{ context.remote_url }}", + "step_name": "check_repo_ci_event" + }, + "capture": { + "ci_event": "${{ result.ci_event }}", + "auto_merge_available": "${{ result.auto_merge_available }}" + }, + "on_success": "check_integration_exists", + "on_failure": "check_integration_exists", + "note": "Detects whether the repository's CI triggers on push or merge_group events. Captures ci_event for use by wait_for_ci steps (wait_for_conflict_ci, ci_watch_pr) and diagnose-ci skill invocations downstream. Routes to check_integration_exists regardless of outcome — non-blocking gate: ci_event defaults to null (match-any) on failure, which matches the ci.py scope.event=None behavior.\n" + }, + "check_integration_exists": { + "tool": "run_cmd", + "with": { + "cmd": "REMOTE=$(git remote get-url upstream 2>/dev/null | grep -qv \"^file://\" && echo upstream || echo origin)\ngit ls-remote --exit-code --heads \"$REMOTE\" \"${{ inputs.base_branch }}\"\n", + "cwd": "${{ context.work_dir }}", + "step_name": "check_integration_exists" + }, + "on_success": "fetch_merge_queue_data", + "on_failure": "confirm_create_integration", + "note": "Checks whether base_branch exists on the remote. Exit code 0 means the branch exists — proceed to analyze_prs. Non-zero exit means the branch is absent — route to user confirmation before creating it. Idempotent: re-running after the branch is created skips through on_success.\n" + }, + "confirm_create_integration": { + "action": "confirm", + "message": "The branch '${{ inputs.base_branch }}' does not exist on the remote. Create it now from the repository's default branch? Type yes to create it or no to abort.", + "on_success": "create_persistent_integration", + "on_failure": "escalate_stop", + "note": "User approval gate. On yes: create_persistent_integration creates and pushes the branch. On no: escalate_stop — the pipeline cannot proceed without the target branch.\n" + }, + "create_persistent_integration": { + "tool": "run_python", + "with": { + "callable": "autoskillit.recipe._cmd_rpc.create_persistent_integration", + "work_dir": "${{ context.work_dir }}", + "base_branch": "${{ inputs.base_branch }}" + }, + "on_success": "fetch_merge_queue_data", + "on_failure": "register_clone_failure", + "note": "Creates base_branch from the repository's default branch (auto-detected via origin/HEAD, falls back to main). Only executes when the user confirmed branch creation in confirm_create_integration. Once created, subsequent pipeline runs skip directly from check_integration_exists to analyze_prs.\n" + }, + "fetch_merge_queue_data": { + "tool": "run_python", + "with": { + "callable": "autoskillit.smoke_utils.fetch_merge_queue_data", + "base_branch": "${{ inputs.base_branch }}", + "cwd": "${{ context.work_dir }}", + "output_dir": "${{ context.work_dir }}/{{AUTOSKILLIT_TEMP}}/merge-prs" + }, + "capture": { + "merge_queue_data_path": "${{ result.merge_queue_data_path }}" + }, + "on_success": "analyze_prs", + "on_failure": "analyze_prs", + "note": "Pre-fetches GitHub merge queue data server-side (where autoskillit is always installed). Writes parsed queue entries to disk as JSON. On failure, falls through to analyze_prs which uses the fallback empty QUEUE_ENTRIES path (QUEUE_MODE=false). Eliminates the autoskillit import in analyze-prs SKILL.md.\n" + }, + "analyze_prs": { + "tool": "run_skill", + "model": "", + "with": { + "skill_command": "/autoskillit:analyze-prs ${{ inputs.base_branch }} merge_queue_data_path=${{ context.merge_queue_data_path }}", + "cwd": "${{ context.work_dir }}", + "step_name": "analyze_prs" + }, + "capture": { + "pr_order_file": "${{ result.pr_order_file }}", + "batch_branch": "${{ result.batch_branch }}", + "queue_mode": "${{ result.queue_mode }}" + }, + "on_success": "route_by_queue_mode", + "on_failure": "register_clone_failure", + "note": "Captures pr_order_file (path to JSON with ordered PRs + complexity tags), batch_branch (e.g. integration/pr-merge-20250228-143052), and queue_mode (\"true\" when a GitHub merge queue with MERGEABLE entries is active on base_branch; \"false\" otherwise). If the skill reports pr_count=0 (no open PRs), skip directly to cleanup_success — no integration branch is needed. merge_queue_data_path is pre-computed by fetch_merge_queue_data step.\n" + }, + "route_by_queue_mode": { + "action": "route", + "on_result": [ + { + "when": "${{ context.queue_mode }} == true", + "route": "get_first_pr_number" + }, + { + "route": "create_batch_branch" + } + ], + "note": "Routes to queue mode (get_first_pr_number) when analyze-prs detected a merge queue on the target branch. Falls back to the classic batch-branch path otherwise. context.queue_mode is captured from analyze_prs result.\n" + }, + "get_first_pr_number": { + "tool": "run_cmd", + "with": { + "cmd": "jq -r '.[0].number' '${{ context.pr_order_file }}'", + "cwd": "${{ context.work_dir }}", + "step_name": "get_first_pr_number" + }, + "capture": { + "current_pr_number": "${{ result.stdout | trim }}" + }, + "on_success": "enqueue_current_pr", + "on_failure": "register_clone_failure", + "note": "Queue mode only. Extracts the first PR's number from pr_order_file to initialize context.current_pr_number before the sequential enqueue loop.\n" + }, + "enqueue_current_pr": { + "tool": "enqueue_pr", + "with": { + "pr_number": "${{ context.current_pr_number }}", + "target_branch": "${{ inputs.base_branch }}", + "cwd": "${{ context.work_dir }}", + "auto_merge_available": "${{ context.auto_merge_available }}" + }, + "on_success": "wait_queue_pr", + "on_failure": "register_clone_failure", + "note": "Queue mode only. Submits a single PR to the merge queue via the unified enqueue_pr tool. Called once per PR in the sequential enqueue loop: get_first_pr_number → enqueue_current_pr → wait_queue_pr → (merged) → advance_queue_pr → enqueue_current_pr → ...\n" + }, + "wait_queue_pr": { + "tool": "wait_for_merge_queue", + "with": { + "pr_number": "${{ context.current_pr_number }}", + "target_branch": "${{ inputs.base_branch }}", + "timeout_seconds": 900, + "cwd": "${{ context.work_dir }}", + "remote_url": "${{ context.remote_url }}" + }, + "capture": { + "queue_pr_state": "${{ result.pr_state }}" + }, + "on_result": [ + { + "when": "${{ result.pr_state }} == merged", + "route": "advance_queue_pr" + }, + { + "when": "${{ result.pr_state }} == ejected_ci_failure", + "route": "diagnose_queue_ci" + }, + { + "when": "${{ result.pr_state }} == dropped_merge_group_ci", + "route": "diagnose_queue_ci" + }, + { + "when": "${{ result.pr_state }} == not_enrolled", + "route": "register_clone_failure" + }, + { + "when": "${{ result.pr_state }} == ejected", + "route": "check_eject_limit" + }, + { + "when": "${{ result.pr_state }} == stalled", + "route": "reenroll_stalled_queue_pr" + }, + { + "when": "${{ result.pr_state }} == dropped_healthy", + "route": "check_dropped_healthy_loop" + }, + { + "when": "${{ result.pr_state }} == timeout", + "route": "register_clone_failure" + }, + { + "route": "register_clone_failure" + } + ], + "on_failure": "register_clone_failure", + "note": "Queue mode monitor. All non-error PRState values are explicitly routed. ejected_ci_failure and dropped_merge_group_ci both route to diagnose_queue_ci; ejected_ci_failure must precede ejected (both match CLOSED state).\n" + }, + "check_eject_limit": { + "tool": "run_python", + "with": { + "callable": "autoskillit.recipe._cmd_rpc.check_eject_limit", + "counter_file": "${{ context.work_dir }}/{{AUTOSKILLIT_TEMP}}/eject_count", + "max_ejects": "3" + }, + "on_result": [ + { + "when": "${{ result.status }} == EJECT_LIMIT_EXCEEDED", + "route": "register_clone_failure" + }, + { + "route": "get_ejected_pr_branch" + } + ], + "on_failure": "register_clone_failure", + "note": "Circuit breaker for the queue ejection loop. After 3 ejections, routes to register_clone_failure to prevent infinite re-enqueue cycles.\n" + }, + "check_dropped_healthy_loop": { + "tool": "run_python", + "with": { + "callable": "autoskillit.recipe._cmd_rpc.check_dropped_healthy_loop", + "counter_file": "${{ context.work_dir }}/{{AUTOSKILLIT_TEMP}}/dropped_healthy_count", + "max_drops": "2" + }, + "on_result": [ + { + "when": "${{ result.status }} == DROPPED_LIMIT_EXCEEDED", + "route": "register_clone_failure" + }, + { + "route": "reenter_queue" + } + ], + "on_failure": "register_clone_failure", + "note": "Circuit breaker for the dropped_healthy re-enrollment loop. After 2 consecutive unexplained drops with healthy PR-branch CI, escalates to register_clone_failure instead of looping. Matches the check_eject_limit pattern used for the ejected path. DROPPED_MERGE_GROUP_CI (confirmed merge-group CI failure) is routed directly to diagnose_queue_ci and never reaches this step.\n" + }, + "get_ejected_pr_branch": { + "tool": "run_cmd", + "with": { + "cmd": "gh pr view '${{ context.current_pr_number }}' --json headRefName -q .headRefName", + "cwd": "${{ context.work_dir }}", + "step_name": "get_ejected_pr_branch" + }, + "capture": { + "ejected_pr_branch": "${{ result.stdout | trim }}" + }, + "on_success": "attempt_cheap_rebase", + "on_failure": "register_clone_failure" + }, + "attempt_cheap_rebase": { + "tool": "run_python", + "with": { + "callable": "autoskillit.recipe._cmd_rpc.attempt_cheap_rebase", + "work_dir": "${{ context.work_dir }}", + "ejected_pr_branch": "${{ context.ejected_pr_branch }}", + "base_branch": "${{ inputs.base_branch }}" + }, + "on_result": [ + { + "when": "${{ result.status }} == clean", + "route": "push_ejected_fix" + }, + { + "route": "resolve_ejected_conflicts" + } + ], + "on_failure": "resolve_ejected_conflicts", + "note": "Attempts a trivial rebase before invoking the full resolve-merge-conflicts skill. Uses ejected_pr_branch captured by get_ejected_pr_branch. On clean rebase, skips the skill and proceeds directly to push_ejected_fix. On conflict, aborts the rebase and routes to resolve_ejected_conflicts for goal-aware resolution.\n" + }, + "resolve_ejected_conflicts": { + "tool": "run_skill", + "model": "", + "with": { + "skill_command": "/autoskillit:resolve-merge-conflicts '${{ context.ejected_pr_branch }}' '${{ inputs.base_branch }}'", + "cwd": "${{ context.work_dir }}", + "step_name": "resolve_ejected_conflicts" + }, + "capture": { + "conflict_escalation_required": "${{ result.escalation_required }}" + }, + "capture_list": { + "all_conflict_report_paths": "${{ result.conflict_report_path }}" + }, + "on_result": [ + { + "when": "${{ result.escalation_required }} == true", + "route": "register_clone_failure" + }, + { + "route": "push_ejected_fix" + } + ], + "on_failure": "register_clone_failure", + "retries": 1, + "on_exhausted": "register_clone_failure", + "note": "Resolves git-level rebase conflicts between the ejected PR's branch and the current base_branch HEAD. Uses the same resolve-merge-conflicts skill as the classic path's integration conflict resolution.\n" + }, + "push_ejected_fix": { + "tool": "push_to_remote", + "with": { + "clone_path": "${{ context.work_dir }}", + "remote_url": "${{ context.remote_url }}", + "branch": "${{ context.ejected_pr_branch }}", + "force": "true" + }, + "capture": { + "push_error_type": "${{ result.error_type }}" + }, + "retries": 2, + "on_exhausted": "register_clone_failure", + "on_success": "ci_watch_post_queue_fix", + "on_failure": "classify_push_failure" + }, + "classify_push_failure": { + "action": "route", + "on_result": [ + { + "when": "${{ context.push_error_type }} == queued_branch", + "route": "wait_dequeue_retry" + }, + { + "route": "register_clone_failure" + } + ] + }, + "wait_dequeue_retry": { + "tool": "wait_for_merge_queue", + "with": { + "pr_number": "${{ context.current_pr_number }}", + "target_branch": "${{ inputs.base_branch }}", + "cwd": "${{ context.work_dir }}", + "timeout_seconds": "60" + }, + "on_result": [ + { + "when": "${{ result.pr_state }} == merged", + "route": "advance_queue_pr" + }, + { + "when": "${{ result.pr_state }} == ejected_ci_failure", + "route": "register_clone_failure" + }, + { + "when": "${{ result.pr_state }} == dropped_merge_group_ci", + "route": "register_clone_failure" + }, + { + "when": "${{ result.pr_state }} == dropped_healthy", + "route": "register_clone_failure" + }, + { + "when": "${{ result.pr_state }} == ejected", + "route": "push_ejected_fix" + }, + { + "when": "${{ result.pr_state }} == stalled", + "route": "register_clone_failure" + }, + { + "when": "${{ result.pr_state }} == timeout", + "route": "register_clone_failure" + }, + { + "route": "register_clone_failure" + } + ], + "on_failure": "register_clone_failure", + "optional_context_refs": [ + "push_error_type" + ] + }, + "ci_watch_post_queue_fix": { + "tool": "wait_for_ci", + "with": { + "branch": "${{ context.ejected_pr_branch }}", + "event": "${{ context.ci_event }}", + "timeout_seconds": 600, + "cwd": "${{ context.work_dir }}", + "remote_url": "${{ context.remote_url }}", + "auto_trigger": "true" + }, + "capture": { + "ci_conclusion": "${{ result.conclusion }}", + "ci_failed_jobs": "${{ result.failed_jobs }}" + }, + "on_result": [ + { + "when": "${{ result.conclusion }} == 'success'", + "route": "reenter_queue" + }, + { + "when": "${{ result.conclusion }} == 'no_runs'", + "route": "register_clone_failure" + }, + { + "when": "${{ result.conclusion }} == 'timed_out'", + "route": "check_ci_post_queue_loop" + }, + { + "route": "register_clone_failure" + } + ], + "on_failure": "register_clone_failure", + "note": "Validates CI passes on the rebased branch before re-entering the merge queue. Prevents re-enqueue of a broken branch that would be ejected again. Routes on result.conclusion: success → reenter_queue, no_runs → register_clone_failure (auto_trigger fires first: empty commit + force-push + re-poll; no_runs reached only when trigger recovery fails), timed_out → check_ci_post_queue_loop, failure → register_clone_failure.\n" + }, + "check_ci_post_queue_loop": { + "tool": "run_python", + "with": { + "callable": "autoskillit.smoke_utils.check_loop_iteration", + "current_iteration": "${{ context.ci_post_queue_loop_count }}", + "max_iterations": "2" + }, + "capture": { + "ci_post_queue_loop_count": "${{ result.next_iteration }}" + }, + "on_result": [ + { + "when": "${{ result.max_exceeded }} == true", + "route": "register_clone_failure" + }, + { + "route": "ci_watch_post_queue_fix" + } + ], + "on_failure": "register_clone_failure", + "optional_context_refs": [ + "ci_post_queue_loop_count" + ], + "note": "Iteration guard for the ci_watch_post_queue_fix timed_out self-loop. Caps at 2 iterations. On budget exhaustion, routes to register_clone_failure.\n" + }, + "reenter_queue": { + "tool": "enqueue_pr", + "with": { + "pr_number": "${{ context.current_pr_number }}", + "target_branch": "${{ inputs.base_branch }}", + "cwd": "${{ context.work_dir }}", + "auto_merge_available": "${{ context.auto_merge_available }}" + }, + "on_success": "wait_queue_pr", + "on_failure": "register_clone_failure", + "note": "Re-submits the fixed PR to the merge queue after conflict resolution. Routes back to wait_queue_pr to continue monitoring.\n" + }, + "advance_queue_pr": { + "tool": "run_python", + "with": { + "callable": "autoskillit.recipe._cmd_rpc.advance_queue_pr", + "current_pr_number": "${{ context.current_pr_number }}", + "pr_order_file": "${{ context.pr_order_file }}" + }, + "capture": { + "current_pr_number": "${{ result.current_pr_number }}" + }, + "on_result": [ + { + "when": "${{ result.current_pr_number }} == done", + "route": "collect_and_check_impl_plans" + }, + { + "route": "get_next_pr_branch" + } + ], + "on_failure": "register_clone_failure", + "note": "Advances to the next PR by looking up the current PR number in pr_order_file and returning the next entry. Outputs the next PR number (or \"done\"). The capture block sets current_pr_number at the recipe level — no agent-managed variable updates. Routes to get_next_pr_branch for proactive rebase, or collect_and_check_impl_plans when done.\n" + }, + "get_next_pr_branch": { + "tool": "run_cmd", + "with": { + "cmd": "gh pr view '${{ context.current_pr_number }}' --json headRefName -q .headRefName\n", + "cwd": "${{ context.work_dir }}", + "step_name": "get_next_pr_branch" + }, + "capture": { + "next_pr_branch": "${{ result.stdout | trim }}" + }, + "on_success": "proactive_rebase_next_pr", + "on_failure": "enqueue_current_pr", + "note": "Fetches the headRefName for the next PR to be enqueued. Captured as next_pr_branch for use by proactive_rebase_next_pr. On failure (e.g. transient gh CLI error), falls through to enqueue_current_pr — the queue cycle proceeds normally without rebase.\n" + }, + "proactive_rebase_next_pr": { + "tool": "run_python", + "with": { + "callable": "autoskillit.recipe._cmd_rpc.proactive_rebase_next_pr", + "work_dir": "${{ context.work_dir }}", + "next_pr_branch": "${{ context.next_pr_branch }}", + "base_branch": "${{ inputs.base_branch }}" + }, + "on_result": [ + { + "when": "${{ result.status }} == clean", + "route": "push_rebased_next_pr" + }, + { + "route": "resolve_proactive_rebase_conflicts" + } + ], + "on_failure": "enqueue_current_pr", + "note": "Attempts a proactive rebase of the next PR branch onto the current base_branch HEAD before submitting to the merge queue. On clean rebase, skips conflict resolution entirely and proceeds to push. On conflict, routes to resolve_proactive_rebase_conflicts. On git command failure, falls through to enqueue_current_pr (safety net).\n" + }, + "resolve_proactive_rebase_conflicts": { + "tool": "run_skill", + "model": "", + "with": { + "skill_command": "/autoskillit:resolve-merge-conflicts '${{ context.next_pr_branch }}' '${{ inputs.base_branch }}'", + "cwd": "${{ context.work_dir }}", + "step_name": "resolve_proactive_rebase_conflicts" + }, + "capture": { + "conflict_escalation_required": "${{ result.escalation_required }}", + "conflict_escalation_reason": "${{ result.escalation_reason }}" + }, + "capture_list": { + "all_conflict_report_paths": "${{ result.conflict_report_path }}" + }, + "on_result": [ + { + "when": "${{ result.escalation_required }} == true", + "route": "register_clone_failure" + }, + { + "route": "push_rebased_next_pr" + } + ], + "on_failure": "register_clone_failure", + "retries": 1, + "on_exhausted": "enqueue_current_pr", + "note": "Resolves git-level rebase conflicts between the next PR's branch and the current base_branch HEAD. Similar to resolve_ejected_conflicts but operates proactively before queue submission. Unresolvable conflicts (escalation_required=true) hard-stop at register_clone_failure. Retry exhaustion falls through to enqueue_current_pr — speculative rebase failure must not abort the current PR's queue cycle.\n" + }, + "push_rebased_next_pr": { + "tool": "push_to_remote", + "with": { + "clone_path": "${{ context.work_dir }}", + "remote_url": "${{ context.remote_url }}", + "branch": "${{ context.next_pr_branch }}", + "force": "true" + }, + "on_success": "enqueue_current_pr", + "on_failure": "register_clone_failure", + "note": "Force-pushes the proactively rebased next PR branch. force=true is required because rebase rewrites SHAs (non-fast-forward). Routes to enqueue_current_pr on success.\n" + }, + "diagnose_queue_ci": { + "tool": "run_skill", + "model": "", + "with": { + "skill_command": "/autoskillit:diagnose-ci '${{ context.current_pr_number }}' - - - ${{ context.ci_event }}", + "cwd": "${{ context.work_dir }}", + "step_name": "diagnose_queue_ci" + }, + "capture": { + "diagnosis_path": "${{ result.diagnosis_path }}" + }, + "on_success": "register_clone_failure", + "on_failure": "register_clone_failure", + "note": "Best-effort CI diagnosis for ejected_ci_failure state. The PR was ejected due to CI failure in the merge queue, not a conflict. After diagnosis, routes to register_clone_failure — CI failures in queue mode are not auto-recoverable without human intervention.\n" + }, + "reenroll_stalled_queue_pr": { + "tool": "enqueue_pr", + "with": { + "pr_number": "${{ context.current_pr_number }}", + "target_branch": "${{ inputs.base_branch }}", + "cwd": "${{ context.work_dir }}", + "auto_merge_available": "${{ context.auto_merge_available }}" + }, + "on_success": "check_queue_stall_loop", + "on_failure": "register_clone_failure", + "note": "Uses the unified enqueue_pr tool to re-enroll a stalled PR in the merge queue. Routes back to check_queue_stall_loop (bounded iteration guard) instead of directly to wait_queue_pr.\n" + }, + "check_queue_stall_loop": { + "tool": "run_python", + "with": { + "callable": "autoskillit.smoke_utils.check_loop_iteration", + "current_iteration": "${{ context.queue_stall_loop_count }}", + "max_iterations": "3" + }, + "capture": { + "queue_stall_loop_count": "${{ result.next_iteration }}" + }, + "on_result": [ + { + "when": "${{ result.max_exceeded }} == true", + "route": "register_clone_failure" + }, + { + "route": "wait_queue_pr" + } + ], + "on_failure": "register_clone_failure", + "optional_context_refs": [ + "queue_stall_loop_count" + ], + "note": "Iteration guard for the wait_queue_pr / reenroll_stalled_queue_pr cycle. Caps stall re-enrollment at 3 attempts. On budget exhaustion, routes to register_clone_failure (merge outcome unconfirmed).\n" + }, + "create_batch_branch": { + "tool": "run_cmd", + "with": { + "cmd": "git checkout ${{ inputs.base_branch }} && git pull && git checkout -b ${{ context.batch_branch }}", + "cwd": "${{ context.work_dir }}", + "step_name": "create_batch_branch" + }, + "on_success": "publish_batch_branch", + "on_failure": "register_clone_failure", + "note": "Creates the integration branch from the latest base_branch HEAD. context.batch_branch was computed by analyze_prs.\n" + }, + "publish_batch_branch": { + "tool": "push_to_remote", + "with": { + "clone_path": "${{ context.work_dir }}", + "remote_url": "${{ context.remote_url }}", + "branch": "${{ context.batch_branch }}" + }, + "on_success": "merge_pr", + "on_failure": "register_clone_failure", + "note": "Publishes the integration branch to origin immediately after creation so that merge_worktree can verify refs/remotes/origin/batch_branch exists before rebasing. The branch is re-pushed at push_batch_branch after all merges complete to update the remote with accumulated merge commits.\n" + }, + "merge_pr": { + "tool": "run_skill", + "model": "", + "with": { + "skill_command": "/autoskillit:merge-pr {pr_number} {complexity}", + "cwd": "${{ context.work_dir }}", + "step_name": "merge_pr", + "pr_order_file": "${{ context.pr_order_file }}" + }, + "capture": { + "needs_plan": "${{ result.needs_plan }}", + "escalation_required": "${{ result.escalation_required }}", + "escalation_reason": "${{ result.escalation_reason }}", + "task": "${{ result.conflict_report_path }}", + "current_pr_number": "${{ result.pr_number }}", + "merged": "${{ result.merged }}" + }, + "on_result": [ + { + "when": "${{ result.escalation_required }} == true", + "route": "escalate_stop" + }, + { + "when": "${{ result.needs_plan }} == true", + "route": "plan" + }, + { + "when": "${{ result.merged }} == false", + "route": "register_clone_failure" + }, + { + "route": "next_part_or_next_pr" + } + ], + "on_failure": "register_clone_failure", + "retries": 5, + "on_exhausted": "register_clone_failure", + "on_context_limit": "register_clone_failure", + "note": "The agent reads context.pr_order_file at context.current_pr_number to get the current PR's complexity. Appends number and complexity to skill_command:\n /autoskillit:merge-pr {pr_number} {complexity}\nescalation_required=true → ambiguous conflict, human intervention needed → escalate_stop. needs_plan=true → conflict_report_path set as context.task → plan. merged=false (timeout/conflict) → register_clone_failure. merged=true + fallthrough → PR merged directly → next_part_or_next_pr.\n" + }, + "plan": { + "tool": "run_skill", + "model": "", + "with": { + "skill_command": "/autoskillit:make-plan ${{ context.task }}", + "cwd": "${{ context.work_dir }}", + "step_name": "plan" + }, + "capture": { + "pr_plan_path": "${{ result.plan_path }}" + }, + "capture_list": { + "plan_parts": "${{ result.plan_parts }}", + "all_plan_paths": "${{ result.plan_path }}" + }, + "on_success": "verify", + "on_failure": "register_clone_failure", + "note": "Glob plan_dir for *_part_*.md or single plan file. Sort alphabetically into plan_parts[]. For a single-part plan, plan_parts has one entry equal to plan_path. PRIMARY PATH: context.task = conflict_report_path from merge_pr. REMEDIATION MODE: context.remediation_path is set when re-entering from the remediate step; pass it inline in the skill_command instead of context.task.\n" + }, + "verify": { + "tool": "run_skill", + "model": "", + "with": { + "skill_command": "/autoskillit:dry-walkthrough ${{ context.pr_plan_path }}", + "cwd": "${{ context.work_dir }}", + "step_name": "verify" + }, + "on_success": "implement", + "on_failure": "register_clone_failure", + "retries": 5, + "on_exhausted": "register_clone_failure", + "on_context_limit": "register_clone_failure", + "note": "SEQUENTIAL: For each plan_part, run the full cycle (verify → implement → test → push_worktree_branch → create_conflict_pr → wait_for_conflict_ci → merge_conflict_pr) before advancing.\n" + }, + "implement": { + "tool": "run_skill", + "model": "", + "stale_threshold": 2400, + "with": { + "skill_command": "/autoskillit:implement-worktree-no-merge ${{ context.pr_plan_path }}", + "cwd": "${{ context.work_dir }}", + "step_name": "implement" + }, + "capture": { + "worktree_path": "${{ result.worktree_path }}", + "worktree_branch_name": "${{ result.branch_name }}" + }, + "on_success": "test", + "on_failure": "register_clone_failure", + "retries": 0, + "on_exhausted": "retry_worktree", + "on_context_limit": "retry_worktree" + }, + "retry_worktree": { + "tool": "run_skill", + "model": "", + "stale_threshold": 2400, + "with": { + "skill_command": "/autoskillit:retry-worktree ${{ context.pr_plan_path }} ${{ context.worktree_path }}", + "cwd": "${{ context.worktree_path }}", + "step_name": "retry_worktree" + }, + "capture": { + "worktree_path": "${{ result.worktree_path }}", + "worktree_branch_name": "${{ result.branch_name }}" + }, + "on_success": "test", + "on_failure": "register_clone_failure", + "retries": 3, + "on_exhausted": "register_clone_failure", + "on_context_limit": "register_clone_failure" + }, + "test": { + "tool": "test_check", + "with": { + "worktree_path": "${{ context.worktree_path }}" + }, + "on_success": "push_worktree_branch", + "on_failure": "fix" + }, + "push_worktree_branch": { + "tool": "run_cmd", + "with": { + "cmd": "REMOTE=$(git remote get-url upstream 2>/dev/null | grep -qv \"^file://\" && echo upstream || echo origin)\ngit push \"$REMOTE\" HEAD:${{ context.worktree_branch_name }} --force-with-lease\n", + "cwd": "${{ context.worktree_path }}", + "step_name": "push_worktree_branch" + }, + "on_success": "create_conflict_pr", + "on_failure": "register_clone_failure", + "note": "Pushes the conflict-resolution worktree branch to origin so GitHub can run CI. Uses --force-with-lease to prevent clobbering concurrent updates. context.worktree_branch_name is captured from implement step output (result.branch_name). Both a lease violation (concurrent push) and a network failure route to register_clone_failure — the recipe cannot distinguish between the two, so no safe retry is possible without re-evaluating the worktree state.\n" + }, + "create_conflict_pr": { + "tool": "run_cmd", + "with": { + "cmd": "gh pr create --base \"${{ context.batch_branch }}\" --head \"${{ context.worktree_branch_name }}\" --title \"chore: conflict resolution for PR ${{ context.current_pr_number }}\" --body \"Auto-generated conflict resolution branch. Closes after merge.\" --json number --jq '.number'\n", + "cwd": "${{ context.work_dir }}", + "step_name": "create_conflict_pr" + }, + "capture": { + "conflict_pr_number": "${{ result.stdout | trim }}" + }, + "on_success": "wait_for_conflict_ci", + "on_failure": "register_clone_failure", + "note": "Creates a GitHub PR from the worktree branch into the integration batch branch. Captures conflict_pr_number for the subsequent gh pr merge call. The --json number --jq '.number' flags constrain gh output to a bare integer, but if gh emits unexpected warnings to stdout the trimmed value may be malformed and propagate silently to merge_conflict_pr. Verify conflict_pr_number is a non-empty integer before use.\n" + }, + "wait_for_conflict_ci": { + "tool": "wait_for_ci", + "with": { + "branch": "${{ context.worktree_branch_name }}", + "event": "${{ context.ci_event }}", + "timeout_seconds": 600, + "cwd": "${{ context.work_dir }}", + "auto_trigger": "true" + }, + "on_result": [ + { + "when": "${{ result.conclusion }} == 'success'", + "route": "merge_conflict_pr" + }, + { + "when": "${{ result.conclusion }} == 'timed_out'", + "route": "check_conflict_ci_loop" + }, + { + "route": "register_clone_failure" + } + ], + "on_failure": "register_clone_failure", + "note": "Waits for CI to pass on the worktree branch before merging. 600 second timeout matches the merge-pr skill poll timeout. Routes on result.conclusion: success → merge_conflict_pr, timed_out → check_conflict_ci_loop (iteration guard), no_runs/failure → register_clone_failure.\n" + }, + "check_conflict_ci_loop": { + "tool": "run_python", + "with": { + "callable": "autoskillit.smoke_utils.check_loop_iteration", + "current_iteration": "${{ context.conflict_ci_loop_count }}", + "max_iterations": "2" + }, + "capture": { + "conflict_ci_loop_count": "${{ result.next_iteration }}" + }, + "on_result": [ + { + "when": "${{ result.max_exceeded }} == true", + "route": "register_clone_failure" + }, + { + "route": "wait_for_conflict_ci" + } + ], + "on_failure": "register_clone_failure", + "optional_context_refs": [ + "conflict_ci_loop_count" + ], + "note": "Caps the wait_for_conflict_ci re-poll loop at 2 iterations. After 2 timed_out results (1200s total), routes to register_clone_failure instead of spinning indefinitely.\n" + }, + "merge_conflict_pr": { + "tool": "run_cmd", + "with": { + "cmd": "gh pr merge \"${{ context.conflict_pr_number }}\" --squash --delete-branch", + "cwd": "${{ context.work_dir }}", + "step_name": "merge_conflict_pr" + }, + "on_success": "next_part_or_next_pr", + "on_failure": "register_clone_failure", + "note": "Merges the conflict-resolution PR into the integration batch branch via GitHub API. Uses --squash to keep integration history linear. Uses --delete-branch to clean up the temporary worktree branch on GitHub. Non-auto merge (explicit --squash without --auto) is correct here because wait_for_conflict_ci already confirmed CI has passed.\n" + }, + "fix": { + "tool": "run_skill", + "model": "", + "with": { + "skill_command": "/autoskillit:resolve-failures ${{ context.worktree_path }} ${{ context.pr_plan_path }} ${{ context.batch_branch }}", + "cwd": "${{ context.worktree_path }}", + "step_name": "fix", + "base_branch": "${{ inputs.base_branch }}" + }, + "capture": { + "verdict": "${{ result.verdict }}", + "fixes_applied": "${{ result.fixes_applied }}" + }, + "on_result": [ + { + "when": "${{ result.verdict }} == 'real_fix'", + "route": "test" + }, + { + "when": "${{ result.verdict }} == 'already_green'", + "route": "test" + }, + { + "when": "${{ result.verdict }} == 'flake_suspected'", + "route": "test" + }, + { + "when": "${{ result.verdict }} == 'ci_only_failure'", + "route": "register_clone_failure" + } + ], + "on_failure": "register_clone_failure", + "retries": 3, + "on_exhausted": "register_clone_failure", + "on_context_limit": "register_clone_failure", + "note": "Fixes test failures without merging. Routes back to test for re-validation before the GitHub-API merge sequence. The base_branch with: key is present for contract compatibility only; the merge target is always the integration branch.\n" + }, + "next_part_or_next_pr": { + "action": "route", + "on_result": { + "field": "next", + "routes": { + "more_parts": "verify", + "more_prs": "merge_pr", + "all_done": "push_batch_branch" + } + }, + "note": "Routing step — no tool call. The agent evaluates what comes next: 1. If more plan_parts remain for the current PR → verify (next part). 2. If all parts done, advance current_pr_number. If more PRs remain → merge_pr. 3. If all PRs processed → push_batch_branch.\n" + }, + "push_batch_branch": { + "tool": "push_to_remote", + "with": { + "clone_path": "${{ context.work_dir }}", + "remote_url": "${{ context.remote_url }}", + "branch": "${{ context.batch_branch }}" + }, + "on_success": "collect_and_check_impl_plans", + "on_failure": "register_clone_failure", + "note": "Single push after ALL PR merges complete. Uses push_to_remote MCP tool (consistent with all other push steps) rather than run_cmd git push.\n" + }, + "collect_and_check_impl_plans": { + "tool": "run_cmd", + "with": { + "cmd": "mkdir -p ${{ inputs.plans_dir }} && find {{AUTOSKILLIT_TEMP}}/make-plan/ -name '*_plan_*.md' -exec cp {} ${{ inputs.plans_dir }}/ \\; 2>/dev/null || true\nfind ${{ inputs.plans_dir }} -maxdepth 1 -name '*_plan_*.md' ! -name 'pr_analysis_plan_*.md' 2>/dev/null | wc -l | tr -d ' \\n'\n", + "cwd": "${{ context.work_dir }}", + "step_name": "collect_and_check_impl_plans" + }, + "on_result": [ + { + "when": "${{ result.stdout | trim }} == 0", + "route": "compute_domain_partitions" + }, + { + "route": "audit_impl" + } + ], + "on_failure": "audit_impl", + "note": "Copies plan files from {{AUTOSKILLIT_TEMP}}/make-plan/ into plans_dir, then counts implementation plan files (excluding pr_analysis_plan_*.md). The copy uses || true so the count always runs even if no source files exist. The final stdout line is the count, used for on_result routing: 0 → compute_domain_partitions (no plans generated, all PRs merged cleanly), non-zero → audit_impl (conflict reports or make-plan outputs are present). Replaces the former collect_artifacts + check_impl_plans two-step pair.\n" + }, + "audit_impl": { + "tool": "run_skill", + "model": "", + "with": { + "skill_command": "/autoskillit:audit-impl \"${{ context.all_plan_paths }}\" ${{ context.batch_branch }} ${{ inputs.base_branch }} \"${{ context.all_conflict_report_paths }}\"", + "cwd": "${{ context.work_dir }}", + "step_name": "audit_impl" + }, + "capture": { + "verdict": "${{ result.verdict }}", + "remediation_path": "${{ result.remediation_path }}" + }, + "on_result": { + "field": "verdict", + "routes": { + "GO": "compute_domain_partitions", + "NO GO": "remediate" + } + }, + "on_failure": "register_clone_failure", + "on_context_limit": "register_clone_failure", + "optional": true, + "skip_when_false": "inputs.audit", + "note": "DEPENDENCY: context.all_plan_paths is only populated if the plan step executed at least once. The check_impl_plans gate ensures this step is only reached when implementation plan files exist (i.e. make-plan was invoked), so all_plan_paths is guaranteed to be set when audit_impl runs. Gated by check_impl_plans — only reached when implementation plan files exist in plans_dir. Only execute if inputs.audit is 'true'; if false, skip directly to open_integration_pr. Passes context.all_plan_paths (comma-separated list of plan file paths accumulated across the PR loop) as plans_input. audit-impl splits on commas and audits each plan file. implementation_ref is context.batch_branch (no worktree remains). GO → open_integration_pr. NO GO → remediate (re-enters plan with remediation file).\n" + }, + "remediate": { + "action": "route", + "with": { + "remediation_path": "${{ context.remediation_path }}" + }, + "on_success": "plan" + }, + "compute_domain_partitions": { + "tool": "run_python", + "with": { + "callable": "autoskillit.smoke_utils.compute_domain_partitions", + "batch_branch": "${{ context.batch_branch }}", + "base_branch": "${{ inputs.base_branch }}", + "cwd": "${{ context.work_dir }}", + "output_dir": "${{ context.work_dir }}/{{AUTOSKILLIT_TEMP}}/open-integration-pr" + }, + "capture": { + "domain_partitions_path": "${{ result.domain_partitions_path }}" + }, + "on_success": "open_integration_pr", + "on_failure": "open_integration_pr", + "note": "Pre-computes domain partitions for open-integration-pr server-side (where autoskillit is always installed). Writes the JSON result to disk. On failure, falls through to open_integration_pr which uses an empty domain_partitions_path fallback. Eliminates the autoskillit import in open-integration-pr SKILL.md.\n" + }, + "open_integration_pr": { + "tool": "run_skill", + "model": "", + "with": { + "skill_command": "/autoskillit:open-integration-pr ${{ context.batch_branch }} ${{ inputs.base_branch }} ${{ context.pr_order_file }} ${{ context.verdict }} \"${{ context.all_conflict_report_paths }}\" domain_partitions_path=${{ context.domain_partitions_path }}", + "cwd": "${{ context.work_dir }}", + "step_name": "open_integration_pr" + }, + "capture": { + "pr_url": "${{ result.pr_url }}" + }, + "on_success": "wait_for_review_pr_mergeability", + "on_failure": "register_clone_failure" + }, + "wait_for_review_pr_mergeability": { + "tool": "run_python", + "with": { + "callable": "autoskillit.recipe._cmd_rpc.wait_for_review_pr_mergeability", + "pr_url": "${{ context.pr_url }}", + "max_polls": "12", + "poll_interval": "15" + }, + "capture": { + "review_pr_number": "${{ result.pr_number }}" + }, + "on_success": "check_mergeability", + "on_failure": "register_clone_failure", + "note": "Extracts the review PR number from context.pr_url and polls GitHub until the mergeable field is no longer UNKNOWN (GitHub computes this asynchronously after PR creation). Polls up to 12 times with 15-second intervals (3 minutes maximum). Captures review_pr_number for use by check_mergeability and check_mergeability_post_rebase.\n" + }, + "check_mergeability": { + "tool": "check_pr_mergeable", + "with": { + "pr_number": "${{ context.review_pr_number }}", + "cwd": "${{ context.work_dir }}" + }, + "on_result": [ + { + "when": "${{ result.mergeable_status }} == MERGEABLE", + "route": "annotate_pr_diff" + }, + { + "when": "${{ result.mergeable_status }} == CONFLICTING", + "route": "resolve_integration_conflicts" + }, + { + "route": "register_clone_failure" + } + ], + "on_failure": "register_clone_failure", + "note": "Authoritative mergeability check via check_pr_mergeable MCP tool (REQ-MERGE-001). Routes MERGEABLE directly to review cycle. Routes CONFLICTING to automated conflict resolution. Any other state (UNKNOWN after timeout, tool error) escalates to register_clone_failure (REQ-MERGE-005).\n" + }, + "resolve_integration_conflicts": { + "tool": "run_skill", + "model": "", + "with": { + "skill_command": "/autoskillit:resolve-merge-conflicts ${{ context.batch_branch }} ${{ inputs.base_branch }}", + "cwd": "${{ context.work_dir }}", + "step_name": "resolve_integration_conflicts" + }, + "capture": { + "conflict_escalation_required": "${{ result.escalation_required }}", + "conflict_escalation_reason": "${{ result.escalation_reason }}" + }, + "capture_list": { + "all_conflict_report_paths": "${{ result.conflict_report_path }}" + }, + "on_result": [ + { + "when": "${{ result.escalation_required }} == true", + "route": "register_clone_failure" + }, + { + "route": "force_push_and_wait_mergeability" + } + ], + "on_failure": "register_clone_failure", + "note": "Automated conflict resolution when the integration PR is CONFLICTING (REQ-MERGE-003). Rebases context.batch_branch onto inputs.base_branch and resolves merge conflicts. Single attempt only — no retries — to prevent looping on unresolvable conflicts. On failure, escalates to register_clone_failure with diagnostics (REQ-MERGE-004).\n" + }, + "force_push_and_wait_mergeability": { + "tool": "run_python", + "with": { + "callable": "autoskillit.recipe._cmd_rpc.force_push_and_wait_mergeability", + "work_dir": "${{ context.work_dir }}", + "batch_branch": "${{ context.batch_branch }}", + "review_pr_number": "${{ context.review_pr_number }}", + "max_polls": "12", + "poll_interval": "15" + }, + "on_success": "check_mergeability_post_rebase", + "on_failure": "register_clone_failure", + "note": "Force-pushes the rebased integration branch (--force-with-lease rejects if remote diverged), then polls GitHub until the PR's mergeable field is no longer UNKNOWN. Polls up to 12 times with 15-second intervals (3 minutes maximum). The && chaining preserves causal ordering: the poll loop only runs after a successful push. Any failure (push rejected or mergeability timeout) routes to register_clone_failure. Replaces the former force_push_after_rebase + wait_for_post_rebase_mergeability two-step pair. Satisfies REQ-MERGE-002.\n" + }, + "check_mergeability_post_rebase": { + "tool": "check_pr_mergeable", + "with": { + "pr_number": "${{ context.review_pr_number }}", + "cwd": "${{ context.work_dir }}" + }, + "on_result": [ + { + "when": "${{ result.mergeable_status }} == MERGEABLE", + "route": "annotate_pr_diff" + }, + { + "route": "register_clone_failure" + } + ], + "on_failure": "register_clone_failure", + "note": "Re-checks mergeability after the rebase and force-push. MERGEABLE proceeds to the review cycle. Any other state means automated resolution did not succeed — escalates to register_clone_failure for human intervention (REQ-MERGE-004). No second conflict resolution attempt — one cycle only.\n" + }, + "annotate_pr_diff": { + "tool": "run_python", + "with": { + "callable": "autoskillit.smoke_utils.annotate_pr_diff", + "pr_number": "${{ context.review_pr_number }}", + "cwd": "${{ context.work_dir }}", + "output_dir": "${{ context.work_dir }}/{{AUTOSKILLIT_TEMP}}/review-pr", + "local_review_rounds": "0" + }, + "capture": { + "annotated_diff_path": "${{ result.annotated_diff_path }}", + "hunk_ranges_path": "${{ result.hunk_ranges_path }}", + "diff_metrics_path": "${{ result.diff_metrics_path }}" + }, + "on_success": "review_pr_integration", + "on_failure": "review_pr_integration", + "note": "Pre-computes annotated diff and hunk ranges for review-pr server-side (where autoskillit is always installed). Writes both files to disk. On failure, falls through to review_pr_integration which uses empty VALID_LINE_RANGES fallback. Eliminates the autoskillit import in review-pr SKILL.md.\n" + }, + "review_pr_integration": { + "tool": "run_skill", + "model": "", + "with": { + "skill_command": "/autoskillit:review-pr ${{ context.batch_branch }} ${{ inputs.base_branch }} annotated_diff_path=${{ context.annotated_diff_path }} hunk_ranges_path=${{ context.hunk_ranges_path }} diff_metrics_path=${{ context.diff_metrics_path }}", + "cwd": "${{ context.work_dir }}", + "step_name": "review_pr_integration" + }, + "capture": { + "review_verdict": "${{ result.verdict }}" + }, + "on_result": [ + { + "when": "${{ result.verdict }} == changes_requested", + "route": "resolve_review_integration" + }, + { + "when": "${{ result.verdict }} == approved_with_comments", + "route": "resolve_review_integration" + }, + { + "when": "${{ result.verdict }} == needs_human", + "route": "ci_watch_pr" + }, + { + "when": "${{ result.verdict }} == approved", + "route": "ci_watch_pr" + }, + { + "when": "true", + "route": "ci_watch_pr" + } + ], + "on_failure": "resolve_review_integration", + "note": "Automated review of the integration PR (REQ-REVIEW-001). Invokes review-pr against context.batch_branch. All four verdicts have explicit routes: changes_requested → resolve_review_integration, approved_with_comments → resolve_review_integration, needs_human → ci_watch_pr (human reviews inline), approved → ci_watch_pr. On tool-level failure, routes to resolve_review_integration (matches implementation.yaml pattern). needs_human is explicitly routed to satisfy the unrouted-verdict-value semantic rule.\n" + }, + "resolve_review_integration": { + "tool": "run_skill", + "model": "", + "with": { + "skill_command": "/autoskillit:resolve-review ${{ context.batch_branch }} ${{ inputs.base_branch }}", + "cwd": "${{ context.work_dir }}", + "step_name": "resolve_review_integration" + }, + "capture": { + "verdict": "${{ result.verdict }}" + }, + "retries": 2, + "on_exhausted": "register_clone_failure", + "on_context_limit": "re_push_review_integration", + "on_result": [ + { + "when": "${{ result.verdict }} == 'real_fix'", + "route": "re_push_review_integration" + }, + { + "when": "${{ result.verdict }} == 'already_green'", + "route": "re_push_review_integration" + }, + { + "when": "${{ result.verdict }} == 'flake_suspected'", + "route": "register_clone_failure" + }, + { + "when": "${{ result.verdict }} == 'ci_only_failure'", + "route": "register_clone_failure" + } + ], + "on_failure": "register_clone_failure", + "note": "Resolves review findings on the integration PR (REQ-REVIEW-002). Fetches inline and summary review comments from GitHub and applies fixes. Retries up to 2 times (3 total attempts) before escalating to register_clone_failure (REQ-REVIEW-005). Routes via on_result: verdict dispatch — real_fix/already_green push fixes; flake_suspected/ ci_only_failure escalate to register_clone_failure.\n" + }, + "re_push_review_integration": { + "tool": "push_to_remote", + "with": { + "clone_path": "${{ context.work_dir }}", + "remote_url": "${{ context.remote_url }}", + "branch": "${{ context.batch_branch }}" + }, + "on_success": "ci_watch_pr", + "on_failure": "register_clone_failure", + "note": "Pushes the review-fixed integration branch to the remote (REQ-REVIEW-003). Routes to ci_watch_pr for final CI validation. On push failure, escalates to register_clone_failure.\n" + }, + "ci_watch_pr": { + "tool": "wait_for_ci", + "with": { + "branch": "${{ context.batch_branch }}", + "event": "${{ context.ci_event }}", + "timeout_seconds": 600, + "cwd": "${{ context.work_dir }}", + "pr_url": "${{ context.pr_url }}", + "auto_trigger": "true" + }, + "on_result": [ + { + "when": "${{ result.conclusion }} == 'success'", + "route": "patch_token_summary" + }, + { + "when": "${{ result.conclusion }} == 'timed_out'", + "route": "check_ci_watch_pr_loop" + }, + { + "route": "register_clone_failure" + } + ], + "on_failure": "diagnose_ci", + "note": "Waits for the GitHub Actions CI run on context.batch_branch to complete using the wait_for_ci MCP tool (timeout: 300 s). Routes on result.conclusion: success → patch_token_summary, timed_out → check_ci_watch_pr_loop (iteration guard), no_runs/failure → register_clone_failure. Tool-level failure routes to diagnose_ci for log capture.\n" + }, + "check_ci_watch_pr_loop": { + "tool": "run_python", + "with": { + "callable": "autoskillit.smoke_utils.check_loop_iteration", + "current_iteration": "${{ context.ci_watch_pr_loop_count }}", + "max_iterations": "2" + }, + "capture": { + "ci_watch_pr_loop_count": "${{ result.next_iteration }}" + }, + "on_result": [ + { + "when": "${{ result.max_exceeded }} == true", + "route": "register_clone_failure" + }, + { + "route": "ci_watch_pr" + } + ], + "on_failure": "register_clone_failure", + "optional_context_refs": [ + "ci_watch_pr_loop_count" + ], + "note": "Caps the ci_watch_pr re-poll loop at 2 iterations.\n" + }, + "diagnose_ci": { + "tool": "run_skill", + "model": "", + "with": { + "skill_command": "/autoskillit:diagnose-ci ${{ context.batch_branch }} - - - ${{ context.ci_event }}", + "cwd": "${{ context.work_dir }}", + "step_name": "diagnose_ci" + }, + "capture": { + "diagnosis_path": "${{ result.diagnosis_path }}" + }, + "on_success": "register_clone_failure", + "on_failure": "register_clone_failure", + "note": "Runs when ci_watch_pr reports CI failure on the integration branch. Fetches logs and writes a diagnosis to {{AUTOSKILLIT_TEMP}}/diagnose-ci/ for human inspection. diagnosis_path is captured for observability — no automated remediation for integration CI, so always routes to register_clone_failure.\n" + }, + "patch_token_summary": { + "description": "Collect all pipeline token data and patch the PR body with complete summary", + "tool": "run_python", + "with": { + "callable": "autoskillit.smoke_utils.patch_pr_token_summary", + "timeout": 60, + "pr_url": "${{ context.pr_url }}", + "cwd": "${{ context.work_dir }}" + }, + "on_success": "register_clone_success", + "on_failure": "register_clone_success", + "note": "Patches PR body with complete token summary; failure never blocks pipeline completion.\n" + }, + "register_clone_success": { + "description": "Register clone as cleanup candidate for batch deletion", + "tool": "register_clone_status", + "with": { + "clone_path": "${{ context.work_dir }}", + "status": "success", + "step_name": "register_clone_success" + }, + "on_success": "done", + "on_failure": "done" + }, + "register_clone_failure": { + "description": "Register clone as preserved (error pipeline)", + "tool": "register_clone_status", + "with": { + "clone_path": "${{ context.work_dir }}", + "status": "error", + "step_name": "register_clone_failure" + }, + "on_success": "escalate_stop", + "on_failure": "escalate_stop" + }, + "done": { + "action": "stop", + "message": "PR consolidation complete." + }, + "escalate_stop": { + "action": "stop", + "message": "Pipeline failed — human intervention needed. Check the integration branch and {{AUTOSKILLIT_TEMP}}/merge-prs/ for details.", + "note": "context.task may be null when reached via escalation_required=true from merge_pr (merge-pr does not set conflict_report_path on escalation paths). Do not rely on context.task being populated when handling this stop.\n" + } + } +} diff --git a/src/autoskillit/recipes/merge-prs.yaml b/src/autoskillit/recipes/merge-prs.yaml index df4a523943..4eb59c9dfc 100644 --- a/src/autoskillit/recipes/merge-prs.yaml +++ b/src/autoskillit/recipes/merge-prs.yaml @@ -1,7 +1,8 @@ name: merge-prs description: Analyze open PRs, determine merge order, collapse them sequentially into an integration branch, and open a single review PR for human approval. Handles conflict resolution via plan+implement for complex PRs. -summary: "clone > setup_remote > analyze_prs > create_integration_branch > [loop per PR: merge_pr or (plan > verify > implement > test > push_worktree_branch > create_conflict_pr > wait_for_conflict_ci > merge_conflict_pr)] > push_integration_branch > collect_artifacts > check_impl_plans > (audit_impl?) > open_integration_pr > wait_for_review_pr_mergeability > check_mergeability > (resolve_integration_conflicts > force_push_after_rebase)? > review_pr_integration > (resolve_review_integration > re_push_review_integration)? > ci_watch_pr > cleanup" -autoskillit_version: "0.3.0" +summary: "clone > setup_remote > analyze_prs > create_batch_branch > [loop per PR: merge_pr or (plan > verify > implement > test > push_worktree_branch > create_conflict_pr > wait_for_conflict_ci > merge_conflict_pr)] > push_batch_branch > collect_artifacts > check_impl_plans > (audit_impl?) > open_integration_pr > wait_for_review_pr_mergeability > check_mergeability > (resolve_integration_conflicts > force_push_after_rebase)? > review_pr_integration > (resolve_review_integration > re_push_review_integration)? > ci_watch_pr > cleanup" +recipe_version: "1.0.0" +requires_packs: [github, ci, clone, telemetry] ingredients: source_dir: @@ -34,15 +35,15 @@ kitchen_rules: push_worktree_branch → create_conflict_pr → wait_for_conflict_ci → merge_conflict_pr) per plan part before advancing to the next part or PR." - "INTEGRATION BRANCH: Two distinct branches exist. inputs.base_branch is the - target branch that PRs point to (e.g. main, integration). - context.integration_branch is the per-run batch branch + target branch that PRs point to (e.g. main, develop). + context.batch_branch is the per-run batch branch (e.g. pr-batch/pr-merge-{ts}) created from base_branch for this pipeline run. - All PR merges and worktree merges target context.integration_branch. The final - review PR opens from context.integration_branch into inputs.base_branch." + All PR merges and worktree merges target context.batch_branch. The final + review PR opens from context.batch_branch into inputs.base_branch." - "PR QUEUE: The agent reads context.pr_order_file once at analyze_prs and - maintains an ordered queue. context.current_pr_index (agent-maintained, starts - at 0) tracks progress. Advance the index after each successful merge or - complex-pr cycle completes." + maintains an ordered queue. context.current_pr_number tracks the current PR. + advance_queue_pr captures the next PR number at the recipe level after each + successful merge." - "COMPLEX PR PATH: When merge_pr returns needs_plan=true, context.task is set to the conflict_report_path. Pass context.task to make-plan. The plan+implement cycle creates a worktree from the integration branch; the worktree branch is @@ -58,16 +59,17 @@ kitchen_rules: interface (gh pr merge --squash). This enforces the branch ruleset required status checks. Never use merge_worktree or local git merge to merge into the integration batch branch." - "PUSH DISCIPLINE: push_worktree_branch (inside the complex PR cycle), - publish_integration_branch / push_integration_branch, and force_push_after_rebase - (authorized for conflict-resolution rebases) are the only authorized push steps. + publish_batch_branch / push_batch_branch, force_push_and_wait_mergeability + (authorized for conflict-resolution rebases), and push_rebased_next_pr + (authorized for proactive pre-enqueue rebase) are the only authorized push steps. Never add ad-hoc run_cmd git push calls elsewhere in the loop." - "QUEUE MODE: When context.queue_mode == true, PRs merge directly into base_branch via the - merge queue. The batch branch (context.integration_branch) is the same as base_branch. - Never invoke create_integration_branch, publish_integration_branch, merge_pr, or - push_integration_branch in queue mode." - - "QUEUE LOOP: In queue mode, context.current_pr_index tracks the current PR being waited on. - Initialize to 0 after enqueue_all_prs. Advance by 1 in next_queue_pr_or_done after each - successful merge. Read pr_number from pr_order_file[current_pr_index].number." + merge queue. The batch branch (context.batch_branch) is the same as base_branch. + Never invoke create_batch_branch, publish_batch_branch, merge_pr, or + push_batch_branch in queue mode." + - "QUEUE LOOP: In queue mode, context.current_pr_number tracks the current PR being waited on. + Initialized by get_first_pr_number from pr_order_file[0]. advance_queue_pr looks up + the current number in pr_order_file and captures the next entry's number." - "CONFLICT REPORT PATHS: context.all_conflict_report_paths accumulates conflict_report_path values from resolve_ejected_conflicts and resolve_integration_conflicts via capture_list. Pass it as the 5th argument to open_integration_pr and 4th argument to audit_impl. @@ -115,6 +117,7 @@ steps: step_name: "check_repo_ci_event" capture: ci_event: "${{ result.ci_event }}" + auto_merge_available: "${{ result.auto_merge_available }}" on_success: check_integration_exists on_failure: check_integration_exists note: > @@ -127,7 +130,9 @@ steps: check_integration_exists: tool: run_cmd with: - cmd: "git ls-remote --exit-code --heads origin \"${{ inputs.base_branch }}\"" + cmd: | + REMOTE=$(git remote get-url upstream 2>/dev/null | grep -qv "^file://" && echo upstream || echo origin) + git ls-remote --exit-code --heads "$REMOTE" "${{ inputs.base_branch }}" cwd: "${{ context.work_dir }}" step_name: "check_integration_exists" on_success: fetch_merge_queue_data @@ -152,11 +157,11 @@ steps: proceed without the target branch. create_persistent_integration: - tool: run_cmd + tool: run_python with: - cmd: "DEFAULT_BRANCH=$(git symbolic-ref refs/remotes/origin/HEAD 2>/dev/null | sed 's|refs/remotes/origin/||') && DEFAULT_BRANCH=${DEFAULT_BRANCH:-main} && git checkout \"$DEFAULT_BRANCH\" && git pull && git checkout -b '${{ inputs.base_branch }}' && git push origin '${{ inputs.base_branch }}'" - cwd: "${{ context.work_dir }}" - step_name: "create_persistent_integration" + callable: "autoskillit.recipe._cmd_rpc.create_persistent_integration" + work_dir: "${{ context.work_dir }}" + base_branch: "${{ inputs.base_branch }}" on_success: fetch_merge_queue_data on_failure: register_clone_failure note: > @@ -184,19 +189,20 @@ steps: analyze_prs: tool: run_skill + model: "" with: skill_command: "/autoskillit:analyze-prs ${{ inputs.base_branch }} merge_queue_data_path=${{ context.merge_queue_data_path }}" cwd: "${{ context.work_dir }}" step_name: "analyze_prs" capture: pr_order_file: "${{ result.pr_order_file }}" - integration_branch: "${{ result.integration_branch }}" + batch_branch: "${{ result.batch_branch }}" queue_mode: "${{ result.queue_mode }}" on_success: route_by_queue_mode on_failure: register_clone_failure note: > Captures pr_order_file (path to JSON with ordered PRs + complexity tags), - integration_branch (e.g. integration/pr-merge-20250228-143052), and + batch_branch (e.g. integration/pr-merge-20250228-143052), and queue_mode ("true" when a GitHub merge queue with MERGEABLE entries is active on base_branch; "false" otherwise). If the skill reports pr_count=0 (no open PRs), skip directly to cleanup_success — no integration branch is needed. @@ -206,33 +212,13 @@ steps: action: route on_result: - when: "${{ context.queue_mode }} == true" - route: enqueue_all_prs - - route: create_integration_branch + route: get_first_pr_number + - route: create_batch_branch note: > - Routes to queue mode (enqueue_all_prs) when analyze-prs detected a merge queue + Routes to queue mode (get_first_pr_number) when analyze-prs detected a merge queue on the target branch. Falls back to the classic batch-branch path otherwise. context.queue_mode is captured from analyze_prs result. - enqueue_all_prs: - tool: run_cmd - with: - cmd: > - cat '${{ context.pr_order_file }}' | - jq -r '.[].number' | - while read pr; do - echo "Enqueueing PR #$pr..." && - gh pr merge "$pr" --squash --auto || echo "WARNING: Failed to enqueue PR #$pr"; - done - cwd: "${{ context.work_dir }}" - step_name: "enqueue_all_prs" - on_success: get_first_pr_number - on_failure: register_clone_failure - note: > - Queue mode only. Submits all PRs in analyzed order to the merge queue via - gh pr merge --squash --auto. Non-fatal per-PR failures are warned but do not - abort — a PR may already be in queue or have auto-merge enabled. The orchestrator - initializes context.current_pr_index = 0 after this step. - get_first_pr_number: tool: run_cmd with: @@ -241,13 +227,26 @@ steps: step_name: "get_first_pr_number" capture: current_pr_number: "${{ result.stdout | trim }}" - on_success: wait_queue_pr + on_success: enqueue_current_pr on_failure: register_clone_failure note: > Queue mode only. Extracts the first PR's number from pr_order_file to initialize - context.current_pr_number before the queue wait loop. The agent advances - current_pr_index and updates current_pr_number in next_queue_pr_or_done - for subsequent iterations. + context.current_pr_number before the sequential enqueue loop. + + enqueue_current_pr: + tool: enqueue_pr + with: + pr_number: "${{ context.current_pr_number }}" + target_branch: "${{ inputs.base_branch }}" + cwd: "${{ context.work_dir }}" + auto_merge_available: "${{ context.auto_merge_available }}" + on_success: wait_queue_pr + on_failure: register_clone_failure + note: > + Queue mode only. Submits a single PR to the merge queue via the unified + enqueue_pr tool. Called once per PR in the sequential enqueue loop: + get_first_pr_number → enqueue_current_pr → wait_queue_pr → (merged) → + advance_queue_pr → enqueue_current_pr → ... wait_queue_pr: tool: wait_for_merge_queue @@ -261,17 +260,60 @@ steps: queue_pr_state: "${{ result.pr_state }}" on_result: - when: "${{ result.pr_state }} == merged" - route: next_queue_pr_or_done + route: advance_queue_pr + - when: "${{ result.pr_state }} == ejected_ci_failure" + route: diagnose_queue_ci + - when: "${{ result.pr_state }} == dropped_merge_group_ci" + route: diagnose_queue_ci + - when: "${{ result.pr_state }} == not_enrolled" + route: register_clone_failure - when: "${{ result.pr_state }} == ejected" - route: get_ejected_pr_branch + route: check_eject_limit + - when: "${{ result.pr_state }} == stalled" + route: reenroll_stalled_queue_pr + - when: "${{ result.pr_state }} == dropped_healthy" + route: check_dropped_healthy_loop + - when: "${{ result.pr_state }} == timeout" + route: register_clone_failure - route: register_clone_failure on_failure: register_clone_failure note: > - Queue mode only. The agent reads context.pr_order_file at context.current_pr_index - to get the current PR's number and appends it to the pr_number argument. - merged → advance to next PR or finish. - ejected → conflict resolution cycle (get_ejected_pr_branch → checkout → resolve → push → reenter). - timeout → register_clone_failure (900s budget = 15 min per PR). + Queue mode monitor. All non-error PRState values are explicitly routed. + ejected_ci_failure and dropped_merge_group_ci both route to diagnose_queue_ci; + ejected_ci_failure must precede ejected (both match CLOSED state). + + check_eject_limit: + tool: run_python + with: + callable: "autoskillit.recipe._cmd_rpc.check_eject_limit" + counter_file: "${{ context.work_dir }}/{{AUTOSKILLIT_TEMP}}/eject_count" + max_ejects: "3" + on_result: + - when: "${{ result.status }} == EJECT_LIMIT_EXCEEDED" + route: register_clone_failure + - route: get_ejected_pr_branch + on_failure: register_clone_failure + note: > + Circuit breaker for the queue ejection loop. After 3 ejections, routes + to register_clone_failure to prevent infinite re-enqueue cycles. + + check_dropped_healthy_loop: + tool: run_python + with: + callable: "autoskillit.recipe._cmd_rpc.check_dropped_healthy_loop" + counter_file: "${{ context.work_dir }}/{{AUTOSKILLIT_TEMP}}/dropped_healthy_count" + max_drops: "2" + on_result: + - when: "${{ result.status }} == DROPPED_LIMIT_EXCEEDED" + route: register_clone_failure + - route: reenter_queue + on_failure: register_clone_failure + note: > + Circuit breaker for the dropped_healthy re-enrollment loop. After 2 consecutive + unexplained drops with healthy PR-branch CI, escalates to register_clone_failure + instead of looping. Matches the check_eject_limit pattern used for the ejected + path. DROPPED_MERGE_GROUP_CI (confirmed merge-group CI failure) is routed + directly to diagnose_queue_ci and never reaches this step. get_ejected_pr_branch: tool: run_cmd @@ -281,23 +323,30 @@ steps: step_name: "get_ejected_pr_branch" capture: ejected_pr_branch: "${{ result.stdout | trim }}" - on_success: checkout_ejected_pr + on_success: attempt_cheap_rebase on_failure: register_clone_failure - checkout_ejected_pr: - tool: run_cmd + attempt_cheap_rebase: + tool: run_python with: - cmd: > - git fetch origin '${{ context.ejected_pr_branch }}' && - git checkout '${{ context.ejected_pr_branch }}' && - git pull origin '${{ context.ejected_pr_branch }}' - cwd: "${{ context.work_dir }}" - step_name: "checkout_ejected_pr" - on_success: resolve_ejected_conflicts - on_failure: register_clone_failure + callable: "autoskillit.recipe._cmd_rpc.attempt_cheap_rebase" + work_dir: "${{ context.work_dir }}" + ejected_pr_branch: "${{ context.ejected_pr_branch }}" + base_branch: "${{ inputs.base_branch }}" + on_result: + - when: "${{ result.status }} == clean" + route: push_ejected_fix + - route: resolve_ejected_conflicts + on_failure: resolve_ejected_conflicts + note: > + Attempts a trivial rebase before invoking the full resolve-merge-conflicts skill. + Uses ejected_pr_branch captured by get_ejected_pr_branch. On clean rebase, + skips the skill and proceeds directly to push_ejected_fix. On conflict, aborts + the rebase and routes to resolve_ejected_conflicts for goal-aware resolution. resolve_ejected_conflicts: tool: run_skill + model: "" with: skill_command: "/autoskillit:resolve-merge-conflicts '${{ context.ejected_pr_branch }}' '${{ inputs.base_branch }}'" cwd: "${{ context.work_dir }}" @@ -325,64 +374,281 @@ steps: remote_url: "${{ context.remote_url }}" branch: "${{ context.ejected_pr_branch }}" force: "true" - on_success: reenter_queue + capture: + push_error_type: "${{ result.error_type }}" + retries: 2 + on_exhausted: register_clone_failure + on_success: ci_watch_post_queue_fix + on_failure: classify_push_failure + + classify_push_failure: + action: route + on_result: + - when: "${{ context.push_error_type }} == queued_branch" + route: wait_dequeue_retry + - route: register_clone_failure + + wait_dequeue_retry: + tool: wait_for_merge_queue + with: + pr_number: "${{ context.current_pr_number }}" + target_branch: "${{ inputs.base_branch }}" + cwd: "${{ context.work_dir }}" + timeout_seconds: "60" + on_result: + - when: "${{ result.pr_state }} == merged" + route: advance_queue_pr + - when: "${{ result.pr_state }} == ejected_ci_failure" + route: register_clone_failure + - when: "${{ result.pr_state }} == dropped_merge_group_ci" + route: register_clone_failure + - when: "${{ result.pr_state }} == dropped_healthy" + route: register_clone_failure + - when: "${{ result.pr_state }} == ejected" + route: push_ejected_fix + - when: "${{ result.pr_state }} == stalled" + route: register_clone_failure + - when: "${{ result.pr_state }} == timeout" + route: register_clone_failure + - route: register_clone_failure + on_failure: register_clone_failure + optional_context_refs: + - push_error_type + + ci_watch_post_queue_fix: + tool: wait_for_ci + with: + branch: "${{ context.ejected_pr_branch }}" + event: "${{ context.ci_event }}" + timeout_seconds: 600 + cwd: "${{ context.work_dir }}" + remote_url: "${{ context.remote_url }}" + auto_trigger: "true" + capture: + ci_conclusion: "${{ result.conclusion }}" + ci_failed_jobs: "${{ result.failed_jobs }}" + on_result: + - when: "${{ result.conclusion }} == 'success'" + route: reenter_queue + - when: "${{ result.conclusion }} == 'no_runs'" + route: register_clone_failure + - when: "${{ result.conclusion }} == 'timed_out'" + route: check_ci_post_queue_loop + - route: register_clone_failure on_failure: register_clone_failure + note: > + Validates CI passes on the rebased branch before re-entering the merge queue. + Prevents re-enqueue of a broken branch that would be ejected again. + Routes on result.conclusion: success → reenter_queue, + no_runs → register_clone_failure (auto_trigger fires first: empty commit + + force-push + re-poll; no_runs reached only when trigger recovery fails), + timed_out → check_ci_post_queue_loop, failure → register_clone_failure. + + check_ci_post_queue_loop: + tool: run_python + with: + callable: "autoskillit.smoke_utils.check_loop_iteration" + current_iteration: "${{ context.ci_post_queue_loop_count }}" + max_iterations: "2" + capture: + ci_post_queue_loop_count: "${{ result.next_iteration }}" + on_result: + - when: "${{ result.max_exceeded }} == true" + route: register_clone_failure + - route: ci_watch_post_queue_fix + on_failure: register_clone_failure + optional_context_refs: + - ci_post_queue_loop_count + note: > + Iteration guard for the ci_watch_post_queue_fix timed_out self-loop. + Caps at 2 iterations. On budget exhaustion, routes to register_clone_failure. reenter_queue: - tool: run_cmd + tool: enqueue_pr with: - cmd: "gh pr merge '${{ context.current_pr_number }}' --squash --auto" + pr_number: "${{ context.current_pr_number }}" + target_branch: "${{ inputs.base_branch }}" cwd: "${{ context.work_dir }}" - step_name: "reenter_queue" + auto_merge_available: "${{ context.auto_merge_available }}" on_success: wait_queue_pr on_failure: register_clone_failure note: > Re-submits the fixed PR to the merge queue after conflict resolution. Routes back to wait_queue_pr to continue monitoring. - next_queue_pr_or_done: - action: route + advance_queue_pr: + tool: run_python + with: + callable: "autoskillit.recipe._cmd_rpc.advance_queue_pr" + current_pr_number: "${{ context.current_pr_number }}" + pr_order_file: "${{ context.pr_order_file }}" + capture: + current_pr_number: "${{ result.current_pr_number }}" on_result: - field: next - routes: - more_prs: wait_queue_pr - all_done: collect_artifacts + - when: "${{ result.current_pr_number }} == done" + route: collect_and_check_impl_plans + - route: get_next_pr_branch + on_failure: register_clone_failure note: > - Queue mode routing only. Advance context.current_pr_index. If more PRs remain, - fetch the next PR's number from pr_order_file and route to wait_queue_pr. - When all PRs are merged, route to collect_artifacts. - In queue mode, integration_branch == base_branch — collect_artifacts proceeds - directly without push_integration_branch (there is no batch branch to push). + Advances to the next PR by looking up the current PR number in pr_order_file + and returning the next entry. Outputs the next PR number (or "done"). + The capture block sets current_pr_number at the recipe level — no agent-managed + variable updates. Routes to get_next_pr_branch for proactive rebase, or + collect_and_check_impl_plans when done. - create_integration_branch: + get_next_pr_branch: tool: run_cmd with: - cmd: "git checkout ${{ inputs.base_branch }} && git pull && git checkout -b ${{ context.integration_branch }}" + cmd: | + gh pr view '${{ context.current_pr_number }}' --json headRefName -q .headRefName cwd: "${{ context.work_dir }}" - step_name: "create_integration_branch" - on_success: publish_integration_branch + step_name: "get_next_pr_branch" + capture: + next_pr_branch: "${{ result.stdout | trim }}" + on_success: proactive_rebase_next_pr + on_failure: enqueue_current_pr + note: > + Fetches the headRefName for the next PR to be enqueued. Captured as next_pr_branch + for use by proactive_rebase_next_pr. On failure (e.g. transient gh CLI error), + falls through to enqueue_current_pr — the queue cycle proceeds normally without rebase. + + proactive_rebase_next_pr: + tool: run_python + with: + callable: "autoskillit.recipe._cmd_rpc.proactive_rebase_next_pr" + work_dir: "${{ context.work_dir }}" + next_pr_branch: "${{ context.next_pr_branch }}" + base_branch: "${{ inputs.base_branch }}" + on_result: + - when: "${{ result.status }} == clean" + route: push_rebased_next_pr + - route: resolve_proactive_rebase_conflicts + on_failure: enqueue_current_pr + note: > + Attempts a proactive rebase of the next PR branch onto the current base_branch HEAD + before submitting to the merge queue. On clean rebase, skips conflict resolution + entirely and proceeds to push. On conflict, routes to resolve_proactive_rebase_conflicts. + On git command failure, falls through to enqueue_current_pr (safety net). + + resolve_proactive_rebase_conflicts: + tool: run_skill + model: "" + with: + skill_command: "/autoskillit:resolve-merge-conflicts '${{ context.next_pr_branch }}' '${{ inputs.base_branch }}'" + cwd: "${{ context.work_dir }}" + step_name: "resolve_proactive_rebase_conflicts" + capture: + conflict_escalation_required: "${{ result.escalation_required }}" + conflict_escalation_reason: "${{ result.escalation_reason }}" + capture_list: + all_conflict_report_paths: "${{ result.conflict_report_path }}" + on_result: + - when: "${{ result.escalation_required }} == true" + route: register_clone_failure + - route: push_rebased_next_pr + on_failure: register_clone_failure + retries: 1 + on_exhausted: enqueue_current_pr + note: > + Resolves git-level rebase conflicts between the next PR's branch and the current + base_branch HEAD. Similar to resolve_ejected_conflicts but operates proactively + before queue submission. Unresolvable conflicts (escalation_required=true) hard-stop + at register_clone_failure. Retry exhaustion falls through to enqueue_current_pr — + speculative rebase failure must not abort the current PR's queue cycle. + + push_rebased_next_pr: + tool: push_to_remote + with: + clone_path: "${{ context.work_dir }}" + remote_url: "${{ context.remote_url }}" + branch: "${{ context.next_pr_branch }}" + force: "true" + on_success: enqueue_current_pr + on_failure: register_clone_failure + note: > + Force-pushes the proactively rebased next PR branch. force=true is required because + rebase rewrites SHAs (non-fast-forward). Routes to enqueue_current_pr on success. + + diagnose_queue_ci: + tool: run_skill + model: "" + with: + skill_command: "/autoskillit:diagnose-ci '${{ context.current_pr_number }}' - - - ${{ context.ci_event }}" + cwd: "${{ context.work_dir }}" + step_name: "diagnose_queue_ci" + capture: + diagnosis_path: "${{ result.diagnosis_path }}" + on_success: register_clone_failure + on_failure: register_clone_failure + note: > + Best-effort CI diagnosis for ejected_ci_failure state. The PR was ejected + due to CI failure in the merge queue, not a conflict. After diagnosis, + routes to register_clone_failure — CI failures in queue mode are not + auto-recoverable without human intervention. + + reenroll_stalled_queue_pr: + tool: enqueue_pr + with: + pr_number: "${{ context.current_pr_number }}" + target_branch: "${{ inputs.base_branch }}" + cwd: "${{ context.work_dir }}" + auto_merge_available: "${{ context.auto_merge_available }}" + on_success: check_queue_stall_loop + on_failure: register_clone_failure + note: > + Uses the unified enqueue_pr tool to re-enroll a stalled PR in the + merge queue. Routes back to check_queue_stall_loop (bounded iteration guard) + instead of directly to wait_queue_pr. + + check_queue_stall_loop: + tool: run_python + with: + callable: "autoskillit.smoke_utils.check_loop_iteration" + current_iteration: "${{ context.queue_stall_loop_count }}" + max_iterations: "3" + capture: + queue_stall_loop_count: "${{ result.next_iteration }}" + on_result: + - when: "${{ result.max_exceeded }} == true" + route: register_clone_failure + - route: wait_queue_pr + on_failure: register_clone_failure + optional_context_refs: + - queue_stall_loop_count + note: > + Iteration guard for the wait_queue_pr / reenroll_stalled_queue_pr cycle. + Caps stall re-enrollment at 3 attempts. On budget exhaustion, routes + to register_clone_failure (merge outcome unconfirmed). + + create_batch_branch: + tool: run_cmd + with: + cmd: "git checkout ${{ inputs.base_branch }} && git pull && git checkout -b ${{ context.batch_branch }}" + cwd: "${{ context.work_dir }}" + step_name: "create_batch_branch" + on_success: publish_batch_branch on_failure: register_clone_failure note: > Creates the integration branch from the latest base_branch HEAD. - context.integration_branch was computed by analyze_prs. - The agent initializes context.current_pr_index = 0 here. + context.batch_branch was computed by analyze_prs. - publish_integration_branch: + publish_batch_branch: tool: push_to_remote with: clone_path: "${{ context.work_dir }}" remote_url: "${{ context.remote_url }}" - branch: "${{ context.integration_branch }}" + branch: "${{ context.batch_branch }}" on_success: merge_pr on_failure: register_clone_failure note: > Publishes the integration branch to origin immediately after creation so - that merge_worktree can verify refs/remotes/origin/integration_branch exists - before rebasing. The branch is re-pushed at push_integration_branch after all + that merge_worktree can verify refs/remotes/origin/batch_branch exists + before rebasing. The branch is re-pushed at push_batch_branch after all merges complete to update the remote with accumulated merge commits. merge_pr: tool: run_skill + model: "" with: skill_command: "/autoskillit:merge-pr {pr_number} {complexity}" cwd: "${{ context.work_dir }}" @@ -394,26 +660,31 @@ steps: escalation_reason: "${{ result.escalation_reason }}" task: "${{ result.conflict_report_path }}" current_pr_number: "${{ result.pr_number }}" + merged: "${{ result.merged }}" on_result: - when: "${{ result.escalation_required }} == true" route: escalate_stop - when: "${{ result.needs_plan }} == true" route: plan + - when: "${{ result.merged }} == false" + route: register_clone_failure - route: next_part_or_next_pr on_failure: register_clone_failure retries: 5 on_exhausted: register_clone_failure on_context_limit: register_clone_failure note: > - The agent reads context.pr_order_file at context.current_pr_index to get the - current PR's number and complexity. Appends them to skill_command: + The agent reads context.pr_order_file at context.current_pr_number to get the + current PR's complexity. Appends number and complexity to skill_command: /autoskillit:merge-pr {pr_number} {complexity} escalation_required=true → ambiguous conflict, human intervention needed → escalate_stop. needs_plan=true → conflict_report_path set as context.task → plan. - needs_plan=false (fallthrough) → PR merged directly → next_part_or_next_pr. + merged=false (timeout/conflict) → register_clone_failure. + merged=true + fallthrough → PR merged directly → next_part_or_next_pr. plan: tool: run_skill + model: "" with: skill_command: "/autoskillit:make-plan ${{ context.task }}" cwd: "${{ context.work_dir }}" @@ -436,6 +707,7 @@ steps: verify: tool: run_skill + model: "" with: skill_command: "/autoskillit:dry-walkthrough ${{ context.pr_plan_path }}" cwd: "${{ context.work_dir }}" @@ -452,6 +724,7 @@ steps: implement: tool: run_skill + model: "" stale_threshold: 2400 with: skill_command: "/autoskillit:implement-worktree-no-merge ${{ context.pr_plan_path }}" @@ -468,6 +741,7 @@ steps: retry_worktree: tool: run_skill + model: "" stale_threshold: 2400 with: skill_command: "/autoskillit:retry-worktree ${{ context.pr_plan_path }} ${{ context.worktree_path }}" @@ -492,7 +766,9 @@ steps: push_worktree_branch: tool: run_cmd with: - cmd: "git push origin HEAD:${{ context.worktree_branch_name }} --force-with-lease" + cmd: | + REMOTE=$(git remote get-url upstream 2>/dev/null | grep -qv "^file://" && echo upstream || echo origin) + git push "$REMOTE" HEAD:${{ context.worktree_branch_name }} --force-with-lease cwd: "${{ context.worktree_path }}" step_name: "push_worktree_branch" on_success: create_conflict_pr @@ -510,7 +786,7 @@ steps: with: cmd: > gh pr create - --base "${{ context.integration_branch }}" + --base "${{ context.batch_branch }}" --head "${{ context.worktree_branch_name }}" --title "chore: conflict resolution for PR ${{ context.current_pr_number }}" --body "Auto-generated conflict resolution branch. Closes after merge." @@ -536,13 +812,39 @@ steps: event: "${{ context.ci_event }}" timeout_seconds: 600 cwd: "${{ context.work_dir }}" - on_success: merge_conflict_pr + auto_trigger: "true" + on_result: + - when: "${{ result.conclusion }} == 'success'" + route: merge_conflict_pr + - when: "${{ result.conclusion }} == 'timed_out'" + route: check_conflict_ci_loop + - route: register_clone_failure on_failure: register_clone_failure note: > Waits for CI to pass on the worktree branch before merging. 600 second timeout matches the merge-pr skill poll timeout. - On CI failure, routes to register_clone_failure — no automated remediation - for conflict-resolution CI failures. + Routes on result.conclusion: success → merge_conflict_pr, + timed_out → check_conflict_ci_loop (iteration guard), no_runs/failure → register_clone_failure. + + check_conflict_ci_loop: + tool: run_python + with: + callable: "autoskillit.smoke_utils.check_loop_iteration" + current_iteration: "${{ context.conflict_ci_loop_count }}" + max_iterations: "2" + capture: + conflict_ci_loop_count: "${{ result.next_iteration }}" + on_result: + - when: "${{ result.max_exceeded }} == true" + route: register_clone_failure + - route: wait_for_conflict_ci + on_failure: register_clone_failure + optional_context_refs: + - conflict_ci_loop_count + note: > + Caps the wait_for_conflict_ci re-poll loop at 2 iterations. + After 2 timed_out results (1200s total), routes to register_clone_failure + instead of spinning indefinitely. merge_conflict_pr: tool: run_cmd @@ -561,8 +863,9 @@ steps: fix: tool: run_skill + model: "" with: - skill_command: "/autoskillit:resolve-failures ${{ context.worktree_path }} ${{ context.pr_plan_path }} ${{ context.integration_branch }}" + skill_command: "/autoskillit:resolve-failures ${{ context.worktree_path }} ${{ context.pr_plan_path }} ${{ context.batch_branch }}" cwd: "${{ context.worktree_path }}" step_name: "fix" base_branch: "${{ inputs.base_branch }}" @@ -595,63 +898,51 @@ steps: routes: more_parts: verify more_prs: merge_pr - all_done: push_integration_branch + all_done: push_batch_branch note: > Routing step — no tool call. The agent evaluates what comes next: 1. If more plan_parts remain for the current PR → verify (next part). - 2. If all parts done, advance current_pr_index. If more PRs remain → merge_pr. - 3. If all PRs processed → push_integration_branch. + 2. If all parts done, advance current_pr_number. If more PRs remain → merge_pr. + 3. If all PRs processed → push_batch_branch. - push_integration_branch: + push_batch_branch: tool: push_to_remote with: clone_path: "${{ context.work_dir }}" remote_url: "${{ context.remote_url }}" - branch: "${{ context.integration_branch }}" - on_success: collect_artifacts + branch: "${{ context.batch_branch }}" + on_success: collect_and_check_impl_plans on_failure: register_clone_failure note: > Single push after ALL PR merges complete. Uses push_to_remote MCP tool (consistent with all other push steps) rather than run_cmd git push. - collect_artifacts: - tool: run_cmd - with: - cmd: "mkdir -p ${{ inputs.plans_dir }} && find {{AUTOSKILLIT_TEMP}}/make-plan/ -name '*_plan_*.md' -exec cp {} ${{ inputs.plans_dir }}/ \\; 2>/dev/null || true" - cwd: "${{ context.work_dir }}" - step_name: "collect_artifacts" - on_success: check_impl_plans - on_failure: check_impl_plans - note: > - Copies plan files generated during the pipeline (from {{AUTOSKILLIT_TEMP}}/make-plan/) into - {{AUTOSKILLIT_TEMP}}/merge-prs/ alongside any conflict reports already there. - Always routes to check_impl_plans, which determines whether audit_impl - is meaningful. In queue mode (context.queue_mode == true), the integration_branch - is the base_branch itself — no batch branch push is needed. The step after - collect_artifacts proceeds identically in both modes. - - check_impl_plans: + collect_and_check_impl_plans: tool: run_cmd with: - cmd: "find ${{ inputs.plans_dir }} -maxdepth 1 -name '*_plan_*.md' ! -name 'pr_analysis_plan_*.md' 2>/dev/null | wc -l | tr -d ' \n'" + cmd: | + mkdir -p ${{ inputs.plans_dir }} && find {{AUTOSKILLIT_TEMP}}/make-plan/ -name '*_plan_*.md' -exec cp {} ${{ inputs.plans_dir }}/ \; 2>/dev/null || true + find ${{ inputs.plans_dir }} -maxdepth 1 -name '*_plan_*.md' ! -name 'pr_analysis_plan_*.md' 2>/dev/null | wc -l | tr -d ' \n' cwd: "${{ context.work_dir }}" - step_name: "check_impl_plans" + step_name: "collect_and_check_impl_plans" on_result: - when: "${{ result.stdout | trim }} == 0" route: compute_domain_partitions - route: audit_impl on_failure: audit_impl note: > - Counts implementation plan files in plans_dir — excludes pr_analysis_plan_*.md, - which is always written by analyze-prs but is not an implementation plan. - Routes to open_integration_pr when zero implementation plans exist (all PRs merged - cleanly, make-plan was never invoked — "No plans generated; all PRs merged cleanly"). - Routes to audit_impl when conflict reports or make-plan outputs are present. + Copies plan files from {{AUTOSKILLIT_TEMP}}/make-plan/ into plans_dir, then counts + implementation plan files (excluding pr_analysis_plan_*.md). The copy uses || true + so the count always runs even if no source files exist. The final stdout line is the + count, used for on_result routing: 0 → compute_domain_partitions (no plans generated, + all PRs merged cleanly), non-zero → audit_impl (conflict reports or make-plan outputs + are present). Replaces the former collect_artifacts + check_impl_plans two-step pair. audit_impl: tool: run_skill + model: "" with: - skill_command: "/autoskillit:audit-impl \"${{ context.all_plan_paths }}\" ${{ context.integration_branch }} ${{ inputs.base_branch }} \"${{ context.all_conflict_report_paths }}\"" + skill_command: "/autoskillit:audit-impl \"${{ context.all_plan_paths }}\" ${{ context.batch_branch }} ${{ inputs.base_branch }} \"${{ context.all_conflict_report_paths }}\"" cwd: "${{ context.work_dir }}" step_name: "audit_impl" capture: @@ -676,7 +967,7 @@ steps: open_integration_pr. Passes context.all_plan_paths (comma-separated list of plan file paths accumulated across the PR loop) as plans_input. audit-impl splits on commas and audits each plan file. - implementation_ref is context.integration_branch (no worktree remains). + implementation_ref is context.batch_branch (no worktree remains). GO → open_integration_pr. NO GO → remediate (re-enters plan with remediation file). remediate: @@ -689,7 +980,7 @@ steps: tool: run_python with: callable: "autoskillit.smoke_utils.compute_domain_partitions" - integration_branch: "${{ context.integration_branch }}" + batch_branch: "${{ context.batch_branch }}" base_branch: "${{ inputs.base_branch }}" cwd: "${{ context.work_dir }}" output_dir: "${{ context.work_dir }}/{{AUTOSKILLIT_TEMP}}/open-integration-pr" @@ -705,8 +996,9 @@ steps: open_integration_pr: tool: run_skill + model: "" with: - skill_command: "/autoskillit:open-integration-pr ${{ context.integration_branch }} ${{ inputs.base_branch }} ${{ context.pr_order_file }} ${{ context.verdict }} \"${{ context.all_conflict_report_paths }}\" domain_partitions_path=${{ context.domain_partitions_path }}" + skill_command: "/autoskillit:open-integration-pr ${{ context.batch_branch }} ${{ inputs.base_branch }} ${{ context.pr_order_file }} ${{ context.verdict }} \"${{ context.all_conflict_report_paths }}\" domain_partitions_path=${{ context.domain_partitions_path }}" cwd: "${{ context.work_dir }}" step_name: "open_integration_pr" capture: @@ -715,21 +1007,14 @@ steps: on_failure: register_clone_failure wait_for_review_pr_mergeability: - tool: run_cmd + tool: run_python with: - cmd: > - PR_NUM=$(gh pr view "${{ context.pr_url }}" --json number -q .number) && - for i in $(seq 1 12); do - STATUS=$(gh pr view "$PR_NUM" --json mergeable -q .mergeable); - [ "$STATUS" != "UNKNOWN" ] && break; - sleep 15; - done && - [ "$STATUS" != "UNKNOWN" ] || { echo "Timed out waiting for mergeability" >&2; exit 1; } && - echo "$PR_NUM" - cwd: "${{ context.work_dir }}" - step_name: "wait_for_review_pr_mergeability" - capture: - review_pr_number: "${{ result.stdout | trim }}" + callable: "autoskillit.recipe._cmd_rpc.wait_for_review_pr_mergeability" + pr_url: "${{ context.pr_url }}" + max_polls: "12" + poll_interval: "15" + capture: + review_pr_number: "${{ result.pr_number }}" on_success: check_mergeability on_failure: register_clone_failure note: > @@ -758,8 +1043,9 @@ steps: resolve_integration_conflicts: tool: run_skill + model: "" with: - skill_command: "/autoskillit:resolve-merge-conflicts ${{ context.integration_branch }} ${{ inputs.base_branch }}" + skill_command: "/autoskillit:resolve-merge-conflicts ${{ context.batch_branch }} ${{ inputs.base_branch }}" cwd: "${{ context.work_dir }}" step_name: "resolve_integration_conflicts" capture: @@ -770,50 +1056,33 @@ steps: on_result: - when: "${{ result.escalation_required }} == true" route: register_clone_failure - - route: force_push_after_rebase + - route: force_push_and_wait_mergeability on_failure: register_clone_failure note: > Automated conflict resolution when the integration PR is CONFLICTING (REQ-MERGE-003). - Rebases context.integration_branch onto inputs.base_branch and resolves merge conflicts. + Rebases context.batch_branch onto inputs.base_branch and resolves merge conflicts. Single attempt only — no retries — to prevent looping on unresolvable conflicts. On failure, escalates to register_clone_failure with diagnostics (REQ-MERGE-004). - force_push_after_rebase: - tool: run_cmd - with: - cmd: "git push origin ${{ context.integration_branch }} --force-with-lease" - cwd: "${{ context.work_dir }}" - step_name: "force_push_after_rebase" - on_success: wait_for_post_rebase_mergeability - on_failure: register_clone_failure - note: > - Force-pushes the rebased integration branch to update the review PR after conflict - resolution. Uses --force-with-lease to reject if the remote diverged during the rebase. - This is an authorized push step (conflict-resolution rebase rewrites history, requiring - force push — a regular push would be rejected by the remote). Routes to - wait_for_post_rebase_mergeability to allow GitHub to recompute mergeability after the push. - - wait_for_post_rebase_mergeability: - tool: run_cmd + force_push_and_wait_mergeability: + tool: run_python with: - cmd: > - PR_NUM="${{ context.review_pr_number }}" && - for i in $(seq 1 12); do - STATUS=$(gh pr view "$PR_NUM" --json mergeable -q .mergeable); - [ "$STATUS" != "UNKNOWN" ] && break; - sleep 15; - done && - [ "$STATUS" != "UNKNOWN" ] || { echo "Timed out waiting for post-rebase mergeability" >&2; exit 1; } - cwd: "${{ context.work_dir }}" - step_name: "wait_for_post_rebase_mergeability" + callable: "autoskillit.recipe._cmd_rpc.force_push_and_wait_mergeability" + work_dir: "${{ context.work_dir }}" + batch_branch: "${{ context.batch_branch }}" + review_pr_number: "${{ context.review_pr_number }}" + max_polls: "12" + poll_interval: "15" on_success: check_mergeability_post_rebase on_failure: register_clone_failure note: > - Polls GitHub until the integration PR's mergeable field is no longer UNKNOWN after the - force-push following conflict resolution. GitHub resets mergeable to UNKNOWN asynchronously - after a force-push, just as it does after PR creation. Polls up to 12 times with - 15-second intervals (3 minutes maximum). Fails explicitly if UNKNOWN persists after all - attempts. Satisfies REQ-MERGE-002. + Force-pushes the rebased integration branch (--force-with-lease rejects if remote + diverged), then polls GitHub until the PR's mergeable field is no longer UNKNOWN. + Polls up to 12 times with 15-second intervals (3 minutes maximum). The && chaining + preserves causal ordering: the poll loop only runs after a successful push. Any + failure (push rejected or mergeability timeout) routes to register_clone_failure. + Replaces the former force_push_after_rebase + wait_for_post_rebase_mergeability + two-step pair. Satisfies REQ-MERGE-002. check_mergeability_post_rebase: tool: check_pr_mergeable @@ -838,9 +1107,11 @@ steps: pr_number: "${{ context.review_pr_number }}" cwd: "${{ context.work_dir }}" output_dir: "${{ context.work_dir }}/{{AUTOSKILLIT_TEMP}}/review-pr" + local_review_rounds: "0" capture: annotated_diff_path: "${{ result.annotated_diff_path }}" hunk_ranges_path: "${{ result.hunk_ranges_path }}" + diff_metrics_path: "${{ result.diff_metrics_path }}" on_success: review_pr_integration on_failure: review_pr_integration note: > @@ -851,8 +1122,9 @@ steps: review_pr_integration: tool: run_skill + model: "" with: - skill_command: "/autoskillit:review-pr ${{ context.integration_branch }} ${{ inputs.base_branch }} annotated_diff_path=${{ context.annotated_diff_path }} hunk_ranges_path=${{ context.hunk_ranges_path }}" + skill_command: "/autoskillit:review-pr ${{ context.batch_branch }} ${{ inputs.base_branch }} annotated_diff_path=${{ context.annotated_diff_path }} hunk_ranges_path=${{ context.hunk_ranges_path }} diff_metrics_path=${{ context.diff_metrics_path }}" cwd: "${{ context.work_dir }}" step_name: "review_pr_integration" capture: @@ -860,23 +1132,29 @@ steps: on_result: - when: "${{ result.verdict }} == changes_requested" route: resolve_review_integration + - when: "${{ result.verdict }} == approved_with_comments" + route: resolve_review_integration - when: "${{ result.verdict }} == needs_human" route: ci_watch_pr + - when: "${{ result.verdict }} == approved" + route: ci_watch_pr - when: "true" route: ci_watch_pr on_failure: resolve_review_integration note: > Automated review of the integration PR (REQ-REVIEW-001). Invokes review-pr against - context.integration_branch. All three verdicts have explicit routes: changes_requested - → resolve_review_integration, needs_human → ci_watch_pr (human reviews inline), - approved → ci_watch_pr. On tool-level failure, routes to resolve_review_integration - (matches implementation.yaml pattern). needs_human is explicitly routed to satisfy - the unrouted-verdict-value semantic rule. + context.batch_branch. All four verdicts have explicit routes: changes_requested + → resolve_review_integration, approved_with_comments → resolve_review_integration, + needs_human → ci_watch_pr (human reviews inline), approved → ci_watch_pr. On + tool-level failure, routes to resolve_review_integration (matches implementation.yaml + pattern). needs_human is explicitly routed to satisfy the unrouted-verdict-value + semantic rule. resolve_review_integration: tool: run_skill + model: "" with: - skill_command: "/autoskillit:resolve-review ${{ context.integration_branch }} ${{ inputs.base_branch }}" + skill_command: "/autoskillit:resolve-review ${{ context.batch_branch }} ${{ inputs.base_branch }}" cwd: "${{ context.work_dir }}" step_name: "resolve_review_integration" capture: @@ -906,7 +1184,7 @@ steps: with: clone_path: "${{ context.work_dir }}" remote_url: "${{ context.remote_url }}" - branch: "${{ context.integration_branch }}" + branch: "${{ context.batch_branch }}" on_success: ci_watch_pr on_failure: register_clone_failure note: > @@ -916,24 +1194,49 @@ steps: ci_watch_pr: tool: wait_for_ci with: - branch: "${{ context.integration_branch }}" + branch: "${{ context.batch_branch }}" event: "${{ context.ci_event }}" - timeout_seconds: 300 + timeout_seconds: 600 cwd: "${{ context.work_dir }}" pr_url: "${{ context.pr_url }}" - on_success: register_clone_success + auto_trigger: "true" + on_result: + - when: "${{ result.conclusion }} == 'success'" + route: patch_token_summary + - when: "${{ result.conclusion }} == 'timed_out'" + route: check_ci_watch_pr_loop + - route: register_clone_failure on_failure: diagnose_ci note: > - Waits for the GitHub Actions CI run on context.integration_branch to complete - using the wait_for_ci MCP tool (timeout: 300 s). CI failures here indicate the - integration branch has problems that the human reviewer should see. No automated - remediation; on failure, routes to diagnose_ci to capture logs before - register_clone_failure preserves the clone for manual inspection. + Waits for the GitHub Actions CI run on context.batch_branch to complete + using the wait_for_ci MCP tool (timeout: 300 s). Routes on result.conclusion: + success → patch_token_summary, timed_out → check_ci_watch_pr_loop (iteration guard), + no_runs/failure → register_clone_failure. + Tool-level failure routes to diagnose_ci for log capture. + + check_ci_watch_pr_loop: + tool: run_python + with: + callable: "autoskillit.smoke_utils.check_loop_iteration" + current_iteration: "${{ context.ci_watch_pr_loop_count }}" + max_iterations: "2" + capture: + ci_watch_pr_loop_count: "${{ result.next_iteration }}" + on_result: + - when: "${{ result.max_exceeded }} == true" + route: register_clone_failure + - route: ci_watch_pr + on_failure: register_clone_failure + optional_context_refs: + - ci_watch_pr_loop_count + note: > + Caps the ci_watch_pr re-poll loop at 2 iterations. diagnose_ci: tool: run_skill + model: "" with: - skill_command: "/autoskillit:diagnose-ci ${{ context.integration_branch }} - - - ${{ context.ci_event }}" + skill_command: "/autoskillit:diagnose-ci ${{ context.batch_branch }} - - - ${{ context.ci_event }}" cwd: "${{ context.work_dir }}" step_name: "diagnose_ci" capture: @@ -946,6 +1249,19 @@ steps: diagnosis_path is captured for observability — no automated remediation for integration CI, so always routes to register_clone_failure. + patch_token_summary: + description: "Collect all pipeline token data and patch the PR body with complete summary" + tool: run_python + with: + callable: "autoskillit.smoke_utils.patch_pr_token_summary" + timeout: 60 + pr_url: "${{ context.pr_url }}" + cwd: "${{ context.work_dir }}" + on_success: register_clone_success + on_failure: register_clone_success + note: > + Patches PR body with complete token summary; failure never blocks pipeline completion. + register_clone_success: description: "Register clone as cleanup candidate for batch deletion" tool: register_clone_status diff --git a/src/autoskillit/recipes/methodology-traditions/_disambiguation.json b/src/autoskillit/recipes/methodology-traditions/_disambiguation.json new file mode 100644 index 0000000000..c73f45f2d6 --- /dev/null +++ b/src/autoskillit/recipes/methodology-traditions/_disambiguation.json @@ -0,0 +1,133 @@ +{ + "schema_version": "1.0", + "disambiguation_rules": [ + { + "name": "prisma_dominance", + "order": 1, + "description": "PRISMA dominates any primary design tradition", + "trigger_mode": "anchor_plus_any", + "anchor": "systematic_synthesis", + "trigger_traditions": [ + "systematic_synthesis" + ], + "primary_tradition": "systematic_synthesis", + "applied_union_rules": [], + "exceptions": [ + { + "when_present": "prediction_model_validation", + "add_union_rules": [ + "TRIPOD_SRMA" + ] + } + ] + }, + { + "name": "rct_economic_union", + "order": 2, + "description": "RCT + Economic evaluation invoke both rule sets", + "trigger_mode": "all_present", + "trigger_traditions": [ + "controlled_intervention", + "economic_evaluation" + ], + "primary_tradition": "controlled_intervention", + "applied_union_rules": [ + "CHEERS_union" + ] + }, + { + "name": "arrive_supersedes_consort", + "order": 3, + "description": "ARRIVE supersedes CONSORT for animal studies", + "trigger_mode": "all_present", + "trigger_traditions": [ + "animal_preclinical", + "controlled_intervention" + ], + "primary_tradition": "animal_preclinical", + "applied_union_rules": [] + }, + { + "name": "benchmarking_prediction_nested", + "order": 4, + "description": "Benchmarking with TRIPOD+AI calibration/discrimination nested", + "trigger_mode": "all_present", + "trigger_traditions": [ + "method_comparison_benchmarking", + "prediction_model_validation" + ], + "primary_tradition": "method_comparison_benchmarking", + "applied_union_rules": [ + "TRIPOD_nested" + ] + } + ], + "cross_tradition_overlaps": [ + { + "name": "tripod_consort_union", + "order": 5, + "description": "Model validated within an RCT", + "trigger_traditions": [ + "prediction_model_validation", + "controlled_intervention" + ], + "primary_tradition": "controlled_intervention", + "applied_union_rules": [ + "TRIPOD_union" + ] + }, + { + "name": "strobe_prisma_moose", + "order": 6, + "description": "Meta-analysis of observational studies uses MOOSE", + "trigger_traditions": [ + "observational_correlational", + "systematic_synthesis" + ], + "primary_tradition": "systematic_synthesis", + "applied_union_rules": [ + "MOOSE_override" + ] + }, + { + "name": "odd_controlled_nesting", + "order": 7, + "description": "Simulation calibrated against experimental data", + "trigger_traditions": [ + "simulation_modeling_tradition", + "controlled_intervention" + ], + "primary_tradition": "simulation_modeling_tradition", + "applied_union_rules": [ + "controlled_intervention_secondary" + ] + }, + { + "name": "benchmarking_prisma_separation", + "order": 8, + "description": "Dataset curation via PRISMA; algorithm evaluation via Weber", + "trigger_traditions": [ + "method_comparison_benchmarking", + "systematic_synthesis" + ], + "primary_tradition": "method_comparison_benchmarking", + "applied_union_rules": [ + "PRISMA_curation_phase" + ], + "overrides_primary": true + }, + { + "name": "srqr_consort_parallel", + "order": 9, + "description": "Mixed-methods RCT qualitative component", + "trigger_traditions": [ + "qualitative_interpretive_tradition", + "controlled_intervention" + ], + "primary_tradition": "controlled_intervention", + "applied_union_rules": [ + "SRQR_parallel" + ] + } + ] +} diff --git a/src/autoskillit/recipes/methodology-traditions/_disambiguation.yaml b/src/autoskillit/recipes/methodology-traditions/_disambiguation.yaml new file mode 100644 index 0000000000..15feba5e6a --- /dev/null +++ b/src/autoskillit/recipes/methodology-traditions/_disambiguation.yaml @@ -0,0 +1,75 @@ +schema_version: "1.0" + +disambiguation_rules: + - name: prisma_dominance + order: 1 + description: "PRISMA dominates any primary design tradition" + trigger_mode: anchor_plus_any + anchor: systematic_synthesis + trigger_traditions: [systematic_synthesis] + primary_tradition: systematic_synthesis + applied_union_rules: [] + exceptions: + - when_present: prediction_model_validation + add_union_rules: [TRIPOD_SRMA] + + - name: rct_economic_union + order: 2 + description: "RCT + Economic evaluation invoke both rule sets" + trigger_mode: all_present + trigger_traditions: [controlled_intervention, economic_evaluation] + primary_tradition: controlled_intervention + applied_union_rules: [CHEERS_union] + + - name: arrive_supersedes_consort + order: 3 + description: "ARRIVE supersedes CONSORT for animal studies" + trigger_mode: all_present + trigger_traditions: [animal_preclinical, controlled_intervention] + primary_tradition: animal_preclinical + applied_union_rules: [] + + - name: benchmarking_prediction_nested + order: 4 + description: "Benchmarking with TRIPOD+AI calibration/discrimination nested" + trigger_mode: all_present + trigger_traditions: [method_comparison_benchmarking, prediction_model_validation] + primary_tradition: method_comparison_benchmarking + applied_union_rules: [TRIPOD_nested] + +cross_tradition_overlaps: + - name: tripod_consort_union + order: 5 + description: "Model validated within an RCT" + trigger_traditions: [prediction_model_validation, controlled_intervention] + primary_tradition: controlled_intervention + applied_union_rules: [TRIPOD_union] + + - name: strobe_prisma_moose + order: 6 + description: "Meta-analysis of observational studies uses MOOSE" + trigger_traditions: [observational_correlational, systematic_synthesis] + primary_tradition: systematic_synthesis + applied_union_rules: [MOOSE_override] + + - name: odd_controlled_nesting + order: 7 + description: "Simulation calibrated against experimental data" + trigger_traditions: [simulation_modeling_tradition, controlled_intervention] + primary_tradition: simulation_modeling_tradition + applied_union_rules: [controlled_intervention_secondary] + + - name: benchmarking_prisma_separation + order: 8 + description: "Dataset curation via PRISMA; algorithm evaluation via Weber" + trigger_traditions: [method_comparison_benchmarking, systematic_synthesis] + primary_tradition: method_comparison_benchmarking + applied_union_rules: [PRISMA_curation_phase] + overrides_primary: true + + - name: srqr_consort_parallel + order: 9 + description: "Mixed-methods RCT qualitative component" + trigger_traditions: [qualitative_interpretive_tradition, controlled_intervention] + primary_tradition: controlled_intervention + applied_union_rules: [SRQR_parallel] diff --git a/src/autoskillit/recipes/methodology-traditions/_ml_sub_area_folding.json b/src/autoskillit/recipes/methodology-traditions/_ml_sub_area_folding.json new file mode 100644 index 0000000000..c91ecf6724 --- /dev/null +++ b/src/autoskillit/recipes/methodology-traditions/_ml_sub_area_folding.json @@ -0,0 +1,207 @@ +{ + "schema_version": "1.0", + "ml_sub_area_folding": [ + { + "sub_area": "foundation_models", + "display_name": "Foundation Models", + "primary_parent": "method_comparison_benchmarking", + "alternate_parents": [ + { + "parent": "prediction_model_validation", + "trigger_keywords": [ + "calibration", + "held-out", + "few-shot scaling", + "zero-shot validation" + ] + }, + { + "parent": "measurement_instrument_validation_tradition", + "trigger_keywords": [ + "psychometric", + "latent trait", + "item response theory", + "Cronbach" + ], + "constraint": "only_if_explicit_construct_measurement" + } + ] + }, + { + "sub_area": "reinforcement_learning", + "display_name": "Reinforcement Learning", + "primary_parent": "method_comparison_benchmarking", + "alternate_parents": [ + { + "parent": "simulation_modeling_tradition", + "trigger_keywords": [ + "multi-agent", + "emergent behavior", + "population dynamics", + "swarm" + ] + }, + { + "parent": "prediction_model_validation", + "trigger_keywords": [ + "policy evaluation", + "held-out task", + "sample efficiency" + ] + } + ] + }, + { + "sub_area": "supervised_classification", + "display_name": "Supervised Classification", + "primary_parent": "method_comparison_benchmarking", + "alternate_parents": [ + { + "parent": "diagnostic_accuracy", + "trigger_keywords": [ + "medical imaging", + "radiology", + "pathology", + "sensitivity", + "specificity" + ] + }, + { + "parent": "prediction_model_validation", + "trigger_keywords": [ + "patient outcome", + "prognosis", + "risk prediction", + "clinical decision" + ] + } + ] + }, + { + "sub_area": "nlp", + "display_name": "NLP", + "primary_parent": "method_comparison_benchmarking", + "alternate_parents": [ + { + "parent": "prediction_model_validation", + "trigger_keywords": [ + "clinical NLP", + "risk scoring", + "patient outcome", + "clinical text" + ] + }, + { + "parent": "qualitative_interpretive_tradition", + "trigger_keywords": [ + "discourse analysis", + "conversation analysis", + "narrative", + "thematic" + ] + } + ] + }, + { + "sub_area": "computer_vision", + "display_name": "Computer Vision", + "primary_parent": "method_comparison_benchmarking", + "alternate_parents": [ + { + "parent": "diagnostic_accuracy", + "trigger_keywords": [ + "medical imaging", + "radiology", + "pathology", + "sensitivity", + "specificity" + ] + }, + { + "parent": "prediction_model_validation", + "trigger_keywords": [ + "patient outcome", + "prognosis", + "clinical decision", + "imaging biomarker" + ] + } + ] + }, + { + "sub_area": "generative_models", + "display_name": "Generative Models", + "primary_parent": "method_comparison_benchmarking", + "alternate_parents": [ + { + "parent": "measurement_instrument_validation_tradition", + "trigger_keywords": [ + "human evaluation instrument", + "inter-rater", + "construct validity", + "scale development" + ] + }, + { + "parent": "qualitative_interpretive_tradition", + "trigger_keywords": [ + "human evaluation", + "qualitative assessment", + "expert rating", + "content analysis" + ] + } + ] + }, + { + "sub_area": "agentic_systems", + "display_name": "Agentic Systems", + "primary_parent": "method_comparison_benchmarking", + "alternate_parents": [ + { + "parent": "simulation_modeling_tradition", + "trigger_keywords": [ + "multi-agent", + "emergent behavior", + "agent-based model", + "swarm intelligence" + ] + }, + { + "parent": "quality_improvement", + "trigger_keywords": [ + "process improvement", + "workflow optimization", + "task efficiency", + "error reduction" + ] + } + ] + }, + { + "sub_area": "time_series", + "display_name": "Time-Series", + "primary_parent": "method_comparison_benchmarking", + "alternate_parents": [ + { + "parent": "prediction_model_validation", + "trigger_keywords": [ + "clinical forecasting", + "patient trajectory", + "risk prediction", + "mortality prediction" + ] + }, + { + "parent": "simulation_modeling_tradition", + "trigger_keywords": [ + "dynamical system", + "differential equation", + "compartmental model", + "system dynamics" + ] + } + ] + } + ] +} diff --git a/src/autoskillit/recipes/methodology-traditions/_ml_sub_area_folding.yaml b/src/autoskillit/recipes/methodology-traditions/_ml_sub_area_folding.yaml new file mode 100644 index 0000000000..8884bc44af --- /dev/null +++ b/src/autoskillit/recipes/methodology-traditions/_ml_sub_area_folding.yaml @@ -0,0 +1,140 @@ +schema_version: "1.0" + +ml_sub_area_folding: + - sub_area: foundation_models + display_name: "Foundation Models" + primary_parent: method_comparison_benchmarking + alternate_parents: + - parent: prediction_model_validation + trigger_keywords: + - calibration + - held-out + - few-shot scaling + - zero-shot validation + - parent: measurement_instrument_validation_tradition + trigger_keywords: + - psychometric + - latent trait + - item response theory + - Cronbach + constraint: only_if_explicit_construct_measurement + + - sub_area: reinforcement_learning + display_name: "Reinforcement Learning" + primary_parent: method_comparison_benchmarking + alternate_parents: + - parent: simulation_modeling_tradition + trigger_keywords: + - multi-agent + - emergent behavior + - population dynamics + - swarm + - parent: prediction_model_validation + trigger_keywords: + - policy evaluation + - held-out task + - sample efficiency + + - sub_area: supervised_classification + display_name: "Supervised Classification" + primary_parent: method_comparison_benchmarking + alternate_parents: + - parent: diagnostic_accuracy + trigger_keywords: + - medical imaging + - radiology + - pathology + - sensitivity + - specificity + - parent: prediction_model_validation + trigger_keywords: + - patient outcome + - prognosis + - risk prediction + - clinical decision + + - sub_area: nlp + display_name: "NLP" + primary_parent: method_comparison_benchmarking + alternate_parents: + - parent: prediction_model_validation + trigger_keywords: + - clinical NLP + - risk scoring + - patient outcome + - clinical text + - parent: qualitative_interpretive_tradition + trigger_keywords: + - discourse analysis + - conversation analysis + - narrative + - thematic + + - sub_area: computer_vision + display_name: "Computer Vision" + primary_parent: method_comparison_benchmarking + alternate_parents: + - parent: diagnostic_accuracy + trigger_keywords: + - medical imaging + - radiology + - pathology + - sensitivity + - specificity + - parent: prediction_model_validation + trigger_keywords: + - patient outcome + - prognosis + - clinical decision + - imaging biomarker + + - sub_area: generative_models + display_name: "Generative Models" + primary_parent: method_comparison_benchmarking + alternate_parents: + - parent: measurement_instrument_validation_tradition + trigger_keywords: + - human evaluation instrument + - inter-rater + - construct validity + - scale development + - parent: qualitative_interpretive_tradition + trigger_keywords: + - human evaluation + - qualitative assessment + - expert rating + - content analysis + + - sub_area: agentic_systems + display_name: "Agentic Systems" + primary_parent: method_comparison_benchmarking + alternate_parents: + - parent: simulation_modeling_tradition + trigger_keywords: + - multi-agent + - emergent behavior + - agent-based model + - swarm intelligence + - parent: quality_improvement + trigger_keywords: + - process improvement + - workflow optimization + - task efficiency + - error reduction + + - sub_area: time_series + display_name: "Time-Series" + primary_parent: method_comparison_benchmarking + alternate_parents: + - parent: prediction_model_validation + trigger_keywords: + - clinical forecasting + - patient trajectory + - risk prediction + - mortality prediction + - parent: simulation_modeling_tradition + trigger_keywords: + - dynamical system + - differential equation + - compartmental model + - system dynamics diff --git a/src/autoskillit/recipes/methodology-traditions/animal_preclinical.json b/src/autoskillit/recipes/methodology-traditions/animal_preclinical.json new file mode 100644 index 0000000000..790a9c8889 --- /dev/null +++ b/src/autoskillit/recipes/methodology-traditions/animal_preclinical.json @@ -0,0 +1,73 @@ +{ + "name": "animal_preclinical", + "schema_version": "1.0", + "priority": 10, + "display_name": "Animal Preclinical Study", + "canonical_guideline": { + "name": "ARRIVE 2.0", + "governing_body": "EQUATOR", + "stable_for_decade": true, + "canonical": true + }, + "fields_spanned": [ + "medicine", + "biology", + "pharmacology", + "neuroscience", + "toxicology" + ], + "detection_keywords": [ + "animal study", + "ARRIVE", + "preclinical", + "in vivo", + "mouse model", + "rat model", + "randomization of animals", + "blinding of assessors", + "sample size calculation" + ], + "mandatory_figures": [ + { + "figure": "Experimental Timeline Diagram", + "shows": "Study schedule with interventions, measurements, and timepoints", + "citation": "ARRIVE item 9" + }, + { + "figure": "Animal Flow Diagram", + "shows": "Animals allocated, housed, intervened, and analyzed at each stage", + "citation": "ARRIVE item 15" + } + ], + "strongly_expected_figures": [ + { + "figure": "Body Weight Curves", + "shows": "Growth curves by intervention group over study duration", + "source": "ARRIVE item 9" + }, + { + "figure": "Dose-Response Plot", + "shows": "Response variable vs dose with confidence intervals", + "source": "ARRIVE item 16" + } + ], + "anti_patterns": [ + { + "pattern": "No sample size justification", + "reason": "ARRIVE requires power analysis or sample size calculation to ensure adequate statistical power" + }, + { + "pattern": "Unreported animal exclusions", + "reason": "All exclusions must be reported with reasons; the flow diagram requires complete accounting" + }, + { + "pattern": "No blinding of outcome assessors", + "reason": "Blinding reduces bias in outcome assessment" + }, + { + "pattern": "Housing conditions unreported", + "reason": "ARRIVE requires reporting of housing and husbandry details" + } + ], + "venue_specific_appendices": [] +} diff --git a/src/autoskillit/recipes/methodology-traditions/animal_preclinical.yaml b/src/autoskillit/recipes/methodology-traditions/animal_preclinical.yaml new file mode 100644 index 0000000000..89ac44e13d --- /dev/null +++ b/src/autoskillit/recipes/methodology-traditions/animal_preclinical.yaml @@ -0,0 +1,49 @@ +name: animal_preclinical +schema_version: "1.0" +priority: 10 +display_name: "Animal Preclinical Study" +canonical_guideline: + name: "ARRIVE 2.0" + governing_body: "EQUATOR" + stable_for_decade: true + canonical: true +fields_spanned: + - medicine + - biology + - pharmacology + - neuroscience + - toxicology +detection_keywords: + - animal study + - ARRIVE + - preclinical + - in vivo + - mouse model + - rat model + - randomization of animals + - blinding of assessors + - sample size calculation +mandatory_figures: + - figure: "Experimental Timeline Diagram" + shows: "Study schedule with interventions, measurements, and timepoints" + citation: "ARRIVE item 9" + - figure: "Animal Flow Diagram" + shows: "Animals allocated, housed, intervened, and analyzed at each stage" + citation: "ARRIVE item 15" +strongly_expected_figures: + - figure: "Body Weight Curves" + shows: "Growth curves by intervention group over study duration" + source: "ARRIVE item 9" + - figure: "Dose-Response Plot" + shows: "Response variable vs dose with confidence intervals" + source: "ARRIVE item 16" +anti_patterns: + - pattern: "No sample size justification" + reason: "ARRIVE requires power analysis or sample size calculation to ensure adequate statistical power" + - pattern: "Unreported animal exclusions" + reason: "All exclusions must be reported with reasons; the flow diagram requires complete accounting" + - pattern: "No blinding of outcome assessors" + reason: "Blinding reduces bias in outcome assessment" + - pattern: "Housing conditions unreported" + reason: "ARRIVE requires reporting of housing and husbandry details" +venue_specific_appendices: [] diff --git a/src/autoskillit/recipes/methodology-traditions/controlled_intervention.json b/src/autoskillit/recipes/methodology-traditions/controlled_intervention.json new file mode 100644 index 0000000000..4d3bcc4e91 --- /dev/null +++ b/src/autoskillit/recipes/methodology-traditions/controlled_intervention.json @@ -0,0 +1,64 @@ +{ + "name": "controlled_intervention", + "schema_version": "1.0", + "priority": 1, + "display_name": "Controlled Intervention (Randomized Trial)", + "canonical_guideline": { + "name": "CONSORT 2010", + "governing_body": "EQUATOR", + "stable_for_decade": true, + "canonical": true + }, + "fields_spanned": [ + "medicine", + "psychology", + "software_engineering", + "education", + "public_health" + ], + "detection_keywords": [ + "randomized controlled trial", + "RCT", + "randomisation", + "random allocation", + "parallel group", + "clinical trial", + "CONSORT", + "intent-to-treat", + "ITT" + ], + "mandatory_figures": [ + { + "figure": "CONSORT Flow Diagram", + "shows": "Participant flow showing enrollment, allocation, follow-up, and analysis", + "citation": "CONSORT item 13a" + } + ], + "strongly_expected_figures": [ + { + "figure": "Kaplan-Meier Survival Curve", + "shows": "Time-to-event outcomes by arm", + "source": "Standard survival analysis" + }, + { + "figure": "Forest Plot of Subgroup Effects", + "shows": "Hazard ratios across pre-specified subgroups", + "source": "CONSORT item 19" + } + ], + "anti_patterns": [ + { + "pattern": "Per-protocol analysis without intent-to-treat", + "reason": "Intent-to-treat is the gold standard for RCTs; per-protocol introduces selection bias" + }, + { + "pattern": "Missing allocation concealment description", + "reason": "Allocation concealment is essential to prevent selection bias at enrollment" + }, + { + "pattern": "No flow diagram", + "reason": "CONSORT requires a flow diagram to transparently show attrition and protocol deviations" + } + ], + "venue_specific_appendices": [] +} diff --git a/src/autoskillit/recipes/methodology-traditions/controlled_intervention.yaml b/src/autoskillit/recipes/methodology-traditions/controlled_intervention.yaml new file mode 100644 index 0000000000..1e347b733b --- /dev/null +++ b/src/autoskillit/recipes/methodology-traditions/controlled_intervention.yaml @@ -0,0 +1,44 @@ +name: controlled_intervention +schema_version: "1.0" +priority: 1 +display_name: "Controlled Intervention (Randomized Trial)" +canonical_guideline: + name: "CONSORT 2010" + governing_body: "EQUATOR" + stable_for_decade: true + canonical: true +fields_spanned: + - medicine + - psychology + - software_engineering + - education + - public_health +detection_keywords: + - randomized controlled trial + - RCT + - randomisation + - random allocation + - parallel group + - clinical trial + - CONSORT + - intent-to-treat + - ITT +mandatory_figures: + - figure: "CONSORT Flow Diagram" + shows: "Participant flow showing enrollment, allocation, follow-up, and analysis" + citation: "CONSORT item 13a" +strongly_expected_figures: + - figure: "Kaplan-Meier Survival Curve" + shows: "Time-to-event outcomes by arm" + source: "Standard survival analysis" + - figure: "Forest Plot of Subgroup Effects" + shows: "Hazard ratios across pre-specified subgroups" + source: "CONSORT item 19" +anti_patterns: + - pattern: "Per-protocol analysis without intent-to-treat" + reason: "Intent-to-treat is the gold standard for RCTs; per-protocol introduces selection bias" + - pattern: "Missing allocation concealment description" + reason: "Allocation concealment is essential to prevent selection bias at enrollment" + - pattern: "No flow diagram" + reason: "CONSORT requires a flow diagram to transparently show attrition and protocol deviations" +venue_specific_appendices: [] diff --git a/src/autoskillit/recipes/methodology-traditions/diagnostic_accuracy.json b/src/autoskillit/recipes/methodology-traditions/diagnostic_accuracy.json new file mode 100644 index 0000000000..a5a9e45449 --- /dev/null +++ b/src/autoskillit/recipes/methodology-traditions/diagnostic_accuracy.json @@ -0,0 +1,110 @@ +{ + "name": "diagnostic_accuracy", + "schema_version": "1.0", + "priority": 4, + "display_name": "Diagnostic Accuracy Study", + "canonical_guideline": { + "name": "STARD 2015", + "governing_body": "EQUATOR", + "stable_for_decade": true, + "canonical": true + }, + "fields_spanned": [ + "medicine", + "radiology", + "pathology", + "laboratory_science", + "bioinformatics" + ], + "detection_keywords": [ + "diagnostic accuracy", + "sensitivity", + "specificity", + "STARD", + "ROC curve", + "AUC", + "positive predictive value", + "reference standard", + "index test" + ], + "mandatory_figures": [ + { + "figure": "STARD Flow Diagram", + "shows": "Participant flow through index test and reference standard", + "citation": "STARD item 13" + }, + { + "figure": "ROC Curve", + "shows": "Receiver operating characteristic with AUC", + "citation": "STARD item 24" + } + ], + "strongly_expected_figures": [ + { + "figure": "Cross-Tabulation (2x2 Table)", + "shows": "True positives, false positives, false negatives, true negatives", + "source": "STARD item 17" + }, + { + "figure": "Calibration Plot", + "shows": "Agreement between predicted and observed probabilities", + "source": "STARD item 24" + } + ], + "anti_patterns": [ + { + "pattern": "No reference standard defined", + "reason": "Diagnostic accuracy requires a defensible reference standard against which the index test is compared" + }, + { + "pattern": "Partial verification bias", + "reason": "All participants should receive both index test and reference standard" + }, + { + "pattern": "Reporting accuracy without confidence intervals", + "reason": "Point estimates without CIs provide no sense of precision or uncertainty" + } + ], + "venue_specific_appendices": [ + { + "sub_area": "supervised_classification", + "trigger_keywords": [ + "medical imaging", + "radiology", + "pathology", + "sensitivity", + "specificity" + ], + "expectations": [ + { + "figure": "STARD Flow Diagram", + "source": "STARD 2015" + }, + { + "figure": "Sensitivity/Specificity at Operating Points", + "source": "STARD 2015" + } + ] + }, + { + "sub_area": "computer_vision", + "trigger_keywords": [ + "medical imaging", + "radiology", + "pathology", + "sensitivity", + "specificity" + ], + "expectations": [ + { + "figure": "STARD Flow Diagram for Imaging Pipeline", + "source": "STARD 2015" + }, + { + "figure": "Cross-Tabulation for Diagnostic Imaging", + "source": "STARD 2015" + } + ] + } + ] +} diff --git a/src/autoskillit/recipes/methodology-traditions/diagnostic_accuracy.yaml b/src/autoskillit/recipes/methodology-traditions/diagnostic_accuracy.yaml new file mode 100644 index 0000000000..8688c153ca --- /dev/null +++ b/src/autoskillit/recipes/methodology-traditions/diagnostic_accuracy.yaml @@ -0,0 +1,71 @@ +name: diagnostic_accuracy +schema_version: "1.0" +priority: 4 +display_name: "Diagnostic Accuracy Study" +canonical_guideline: + name: "STARD 2015" + governing_body: "EQUATOR" + stable_for_decade: true + canonical: true +fields_spanned: + - medicine + - radiology + - pathology + - laboratory_science + - bioinformatics +detection_keywords: + - diagnostic accuracy + - sensitivity + - specificity + - STARD + - ROC curve + - AUC + - positive predictive value + - reference standard + - index test +mandatory_figures: + - figure: "STARD Flow Diagram" + shows: "Participant flow through index test and reference standard" + citation: "STARD item 13" + - figure: "ROC Curve" + shows: "Receiver operating characteristic with AUC" + citation: "STARD item 24" +strongly_expected_figures: + - figure: "Cross-Tabulation (2x2 Table)" + shows: "True positives, false positives, false negatives, true negatives" + source: "STARD item 17" + - figure: "Calibration Plot" + shows: "Agreement between predicted and observed probabilities" + source: "STARD item 24" +anti_patterns: + - pattern: "No reference standard defined" + reason: "Diagnostic accuracy requires a defensible reference standard against which the index test is compared" + - pattern: "Partial verification bias" + reason: "All participants should receive both index test and reference standard" + - pattern: "Reporting accuracy without confidence intervals" + reason: "Point estimates without CIs provide no sense of precision or uncertainty" +venue_specific_appendices: + - sub_area: supervised_classification + trigger_keywords: + - medical imaging + - radiology + - pathology + - sensitivity + - specificity + expectations: + - figure: "STARD Flow Diagram" + source: "STARD 2015" + - figure: "Sensitivity/Specificity at Operating Points" + source: "STARD 2015" + - sub_area: computer_vision + trigger_keywords: + - medical imaging + - radiology + - pathology + - sensitivity + - specificity + expectations: + - figure: "STARD Flow Diagram for Imaging Pipeline" + source: "STARD 2015" + - figure: "Cross-Tabulation for Diagnostic Imaging" + source: "STARD 2015" diff --git a/src/autoskillit/recipes/methodology-traditions/economic_evaluation.json b/src/autoskillit/recipes/methodology-traditions/economic_evaluation.json new file mode 100644 index 0000000000..8ea048c929 --- /dev/null +++ b/src/autoskillit/recipes/methodology-traditions/economic_evaluation.json @@ -0,0 +1,74 @@ +{ + "name": "economic_evaluation", + "schema_version": "1.0", + "priority": 9, + "display_name": "Economic Evaluation", + "canonical_guideline": { + "name": "CHEERS 2022", + "governing_body": "ISPOR", + "stable_for_decade": false, + "canonical": true + }, + "fields_spanned": [ + "medicine", + "health_economics", + "pharmacy", + "public_health", + "health_policy" + ], + "detection_keywords": [ + "cost-effectiveness", + "CHEERS", + "ICER", + "QALY", + "cost-utility", + "cost-benefit", + "economic evaluation", + "Markov model", + "willingness to pay", + "budget impact" + ], + "mandatory_figures": [ + { + "figure": "Cost-Effectiveness Plane", + "shows": "Incremental costs vs effects for competing strategies", + "citation": "CHEERS item 20" + }, + { + "figure": "Model Structure Diagram", + "shows": "Markov states, transitions, and cycle length", + "citation": "CHEERS item 14" + } + ], + "strongly_expected_figures": [ + { + "figure": "Tornado Diagram", + "shows": "One-way sensitivity analysis across parameters", + "source": "CHEERS item 21" + }, + { + "figure": "Cost-Effectiveness Acceptability Curve", + "shows": "Probability of cost-effectiveness at varying willingness-to-pay thresholds", + "source": "CHEERS item 20" + } + ], + "anti_patterns": [ + { + "pattern": "No discounting of future costs", + "reason": "Future costs and outcomes must be discounted to present value (typically 3-5% annually)" + }, + { + "pattern": "Undisclosed perspective", + "reason": "The perspective (healthcare system, societal) determines which costs are included" + }, + { + "pattern": "No sensitivity analysis", + "reason": "Economic models have inherent uncertainty; deterministic and probabilistic sensitivity analysis is required" + }, + { + "pattern": "Missing time horizon", + "reason": "The time horizon must be justified and long enough to capture all relevant costs and outcomes" + } + ], + "venue_specific_appendices": [] +} diff --git a/src/autoskillit/recipes/methodology-traditions/economic_evaluation.yaml b/src/autoskillit/recipes/methodology-traditions/economic_evaluation.yaml new file mode 100644 index 0000000000..5b68a3e309 --- /dev/null +++ b/src/autoskillit/recipes/methodology-traditions/economic_evaluation.yaml @@ -0,0 +1,50 @@ +name: economic_evaluation +schema_version: "1.0" +priority: 9 +display_name: "Economic Evaluation" +canonical_guideline: + name: "CHEERS 2022" + governing_body: "ISPOR" + stable_for_decade: false + canonical: true +fields_spanned: + - medicine + - health_economics + - pharmacy + - public_health + - health_policy +detection_keywords: + - cost-effectiveness + - CHEERS + - ICER + - QALY + - cost-utility + - cost-benefit + - economic evaluation + - Markov model + - willingness to pay + - budget impact +mandatory_figures: + - figure: "Cost-Effectiveness Plane" + shows: "Incremental costs vs effects for competing strategies" + citation: "CHEERS item 20" + - figure: "Model Structure Diagram" + shows: "Markov states, transitions, and cycle length" + citation: "CHEERS item 14" +strongly_expected_figures: + - figure: "Tornado Diagram" + shows: "One-way sensitivity analysis across parameters" + source: "CHEERS item 21" + - figure: "Cost-Effectiveness Acceptability Curve" + shows: "Probability of cost-effectiveness at varying willingness-to-pay thresholds" + source: "CHEERS item 20" +anti_patterns: + - pattern: "No discounting of future costs" + reason: "Future costs and outcomes must be discounted to present value (typically 3-5% annually)" + - pattern: "Undisclosed perspective" + reason: "The perspective (healthcare system, societal) determines which costs are included" + - pattern: "No sensitivity analysis" + reason: "Economic models have inherent uncertainty; deterministic and probabilistic sensitivity analysis is required" + - pattern: "Missing time horizon" + reason: "The time horizon must be justified and long enough to capture all relevant costs and outcomes" +venue_specific_appendices: [] diff --git a/src/autoskillit/recipes/methodology-traditions/measurement_instrument_validation_tradition.json b/src/autoskillit/recipes/methodology-traditions/measurement_instrument_validation_tradition.json new file mode 100644 index 0000000000..9b84f89ddc --- /dev/null +++ b/src/autoskillit/recipes/methodology-traditions/measurement_instrument_validation_tradition.json @@ -0,0 +1,108 @@ +{ + "name": "measurement_instrument_validation_tradition", + "schema_version": "1.0", + "priority": 7, + "display_name": "Measurement Instrument Validation", + "canonical_guideline": { + "name": "COSMIN 2.0", + "governing_body": "EQUATOR", + "stable_for_decade": true, + "canonical": true + }, + "fields_spanned": [ + "medicine", + "psychology", + "education", + "nursing", + "rehabilitation" + ], + "detection_keywords": [ + "measurement properties", + "COSMIN", + "reliability", + "validity", + "responsiveness", + "internal consistency", + "Cronbach alpha", + "test-retest", + "content validity" + ], + "mandatory_figures": [ + { + "figure": "Structural Validity Diagram", + "shows": "Factor structure or measurement model path diagram", + "citation": "COSMIN Box 4" + }, + { + "figure": "Measurement Model Path Diagram", + "shows": "Latent variables and observed indicators with loadings", + "citation": "COSMIN Box 4" + } + ], + "strongly_expected_figures": [ + { + "figure": "Bland-Altman Plot", + "shows": "Agreement between two measurement methods", + "source": "COSMIN reliability" + }, + { + "figure": "Item Characteristic Curves", + "shows": "Probability of response as function of latent trait", + "source": "COSMIN validity" + } + ], + "anti_patterns": [ + { + "pattern": "Reporting only Cronbach alpha without factor analysis", + "reason": "Cronbach alpha alone cannot establish structural validity; factor analysis is required" + }, + { + "pattern": "No content validity study", + "reason": "Content validity is the foundational measurement property and must be established before other properties" + }, + { + "pattern": "Conflating reliability with validity", + "reason": "Reliability (consistency) is distinct from validity (accuracy); high reliability does not imply validity" + } + ], + "venue_specific_appendices": [ + { + "sub_area": "foundation_models", + "trigger_keywords": [ + "psychometric", + "latent trait", + "item response theory", + "Cronbach" + ], + "expectations": [ + { + "figure": "Structural Validity for Evaluation Framework", + "source": "COSMIN 2.0" + }, + { + "figure": "Item Characteristic Curves", + "source": "COSMIN 2.0" + } + ] + }, + { + "sub_area": "generative_models", + "trigger_keywords": [ + "human evaluation instrument", + "inter-rater", + "construct validity", + "scale development" + ], + "expectations": [ + { + "figure": "Human Evaluation Instrument Validation", + "source": "COSMIN 2.0" + }, + { + "figure": "Inter-Rater Reliability Plots", + "source": "COSMIN 2.0" + } + ] + } + ] +} diff --git a/src/autoskillit/recipes/methodology-traditions/measurement_instrument_validation_tradition.yaml b/src/autoskillit/recipes/methodology-traditions/measurement_instrument_validation_tradition.yaml new file mode 100644 index 0000000000..9bfeeed261 --- /dev/null +++ b/src/autoskillit/recipes/methodology-traditions/measurement_instrument_validation_tradition.yaml @@ -0,0 +1,69 @@ +name: measurement_instrument_validation_tradition +schema_version: "1.0" +priority: 7 +display_name: "Measurement Instrument Validation" +canonical_guideline: + name: "COSMIN 2.0" + governing_body: "EQUATOR" + stable_for_decade: true + canonical: true +fields_spanned: + - medicine + - psychology + - education + - nursing + - rehabilitation +detection_keywords: + - measurement properties + - COSMIN + - reliability + - validity + - responsiveness + - internal consistency + - Cronbach alpha + - test-retest + - content validity +mandatory_figures: + - figure: "Structural Validity Diagram" + shows: "Factor structure or measurement model path diagram" + citation: "COSMIN Box 4" + - figure: "Measurement Model Path Diagram" + shows: "Latent variables and observed indicators with loadings" + citation: "COSMIN Box 4" +strongly_expected_figures: + - figure: "Bland-Altman Plot" + shows: "Agreement between two measurement methods" + source: "COSMIN reliability" + - figure: "Item Characteristic Curves" + shows: "Probability of response as function of latent trait" + source: "COSMIN validity" +anti_patterns: + - pattern: "Reporting only Cronbach alpha without factor analysis" + reason: "Cronbach alpha alone cannot establish structural validity; factor analysis is required" + - pattern: "No content validity study" + reason: "Content validity is the foundational measurement property and must be established before other properties" + - pattern: "Conflating reliability with validity" + reason: "Reliability (consistency) is distinct from validity (accuracy); high reliability does not imply validity" +venue_specific_appendices: + - sub_area: foundation_models + trigger_keywords: + - psychometric + - latent trait + - item response theory + - Cronbach + expectations: + - figure: "Structural Validity for Evaluation Framework" + source: "COSMIN 2.0" + - figure: "Item Characteristic Curves" + source: "COSMIN 2.0" + - sub_area: generative_models + trigger_keywords: + - human evaluation instrument + - inter-rater + - construct validity + - scale development + expectations: + - figure: "Human Evaluation Instrument Validation" + source: "COSMIN 2.0" + - figure: "Inter-Rater Reliability Plots" + source: "COSMIN 2.0" diff --git a/src/autoskillit/recipes/methodology-traditions/method_comparison_benchmarking.json b/src/autoskillit/recipes/methodology-traditions/method_comparison_benchmarking.json new file mode 100644 index 0000000000..4b9ffa64da --- /dev/null +++ b/src/autoskillit/recipes/methodology-traditions/method_comparison_benchmarking.json @@ -0,0 +1,268 @@ +{ + "name": "method_comparison_benchmarking", + "schema_version": "1.0", + "priority": 12, + "display_name": "Method Comparison and Benchmarking", + "canonical_guideline": { + "name": "Weber et al. 2019", + "governing_body": "community", + "stable_for_decade": true, + "canonical": false + }, + "fields_spanned": [ + "computer_science", + "software_engineering", + "machine_learning", + "bioinformatics", + "operations_research" + ], + "detection_keywords": [ + "benchmark", + "method comparison", + "baseline comparison", + "state-of-the-art", + "SOTA", + "leaderboard", + "ablation study", + "performance comparison", + "evaluation protocol" + ], + "mandatory_figures": [ + { + "figure": "Performance Comparison Table or Figure", + "shows": "Quantitative metrics comparing all methods side-by-side", + "citation": "Weber et al. 2019 Section 4" + }, + { + "figure": "Statistical Significance Test Results", + "shows": "Tests comparing method performance with p-values or confidence intervals", + "citation": "Weber et al. 2019 Section 4" + } + ], + "strongly_expected_figures": [ + { + "figure": "Critical Difference Diagram", + "shows": "Post-hoc pairwise comparisons with statistical significance", + "source": "Demšar 2006" + }, + { + "figure": "Runtime-Accuracy Pareto Front", + "shows": "Trade-off between computational cost and accuracy metrics", + "source": "Weber et al. 2019" + } + ], + "anti_patterns": [ + { + "pattern": "No statistical significance testing", + "reason": "Performance differences must be tested for statistical significance" + }, + { + "pattern": "Cherry-picked baselines", + "reason": "All relevant competing methods should be included; omission of strong baselines biases conclusions" + }, + { + "pattern": "Unreported hyperparameter tuning", + "reason": "Methods must be fairly tuned before comparison to ensure valid assessment" + }, + { + "pattern": "No ablation study", + "reason": "Ablation studies isolate the contribution of individual components" + } + ], + "venue_specific_appendices": [ + { + "sub_area": "supervised_classification", + "trigger_keywords": [ + "confusion matrix", + "precision", + "recall", + "ROC-AUC", + "PR curve", + "classification", + "learning curve" + ], + "expectations": [ + { + "figure": "Confusion Matrix", + "source": "NeurIPS/ICML community norms" + }, + { + "figure": "PR Curve", + "source": "NeurIPS/ICML community norms" + }, + { + "figure": "ROC-AUC Plot", + "source": "NeurIPS/ICML community norms" + }, + { + "figure": "Learning Curves", + "source": "NeurIPS/ICML community norms" + } + ] + }, + { + "sub_area": "nlp", + "trigger_keywords": [ + "accuracy", + "F1", + "GLUE", + "SuperGLUE", + "NLP", + "language model", + "token" + ], + "expectations": [ + { + "figure": "Per-Task Accuracy Table", + "source": "ACL norms" + }, + { + "figure": "Error Analysis Examples", + "source": "ACL norms" + } + ] + }, + { + "sub_area": "computer_vision", + "trigger_keywords": [ + "mAP", + "COCO", + "detection", + "segmentation", + "object detection", + "image classification" + ], + "expectations": [ + { + "figure": "Sample Predictions Grid", + "source": "CVPR/ECCV norms" + }, + { + "figure": "Failure Gallery", + "source": "CVPR/ECCV norms" + }, + { + "figure": "Per-Class mAP Table", + "source": "CVPR/ECCV norms" + } + ] + }, + { + "sub_area": "reinforcement_learning", + "trigger_keywords": [ + "episode reward", + "RL", + "Atari", + "agent", + "policy gradient" + ], + "expectations": [ + { + "figure": "Episode Reward Curve", + "source": "NeurIPS RL norms" + }, + { + "figure": "Sample Efficiency Curve", + "source": "NeurIPS RL norms" + } + ] + }, + { + "sub_area": "generative_models", + "trigger_keywords": [ + "GAN", + "diffusion", + "FID", + "IS", + "generative", + "sample quality" + ], + "expectations": [ + { + "figure": "Sample Grid", + "source": "NeurIPS/ICLR norms" + }, + { + "figure": "FID/IS Table", + "source": "NeurIPS/ICLR norms" + }, + { + "figure": "Failure Modes Gallery", + "source": "NeurIPS/ICLR norms" + } + ] + }, + { + "sub_area": "foundation_models", + "trigger_keywords": [ + "LLM", + "few-shot", + "scaling", + "GPT", + "foundation model", + "zero-shot" + ], + "expectations": [ + { + "figure": "Few-Shot Scaling Curve", + "source": "LLM evaluation norms" + }, + { + "figure": "Contamination Analysis", + "source": "LLM evaluation norms" + }, + { + "figure": "Ablation Table", + "source": "LLM evaluation norms" + } + ] + }, + { + "sub_area": "agentic_systems", + "trigger_keywords": [ + "agentic", + "tool use", + "multi-agent", + "task success", + "workflow" + ], + "expectations": [ + { + "figure": "Task Success Rate Bar Chart", + "source": "Emerging norms" + }, + { + "figure": "Step-Level Execution Traces", + "source": "Emerging norms" + }, + { + "figure": "Tool Use Breakdown", + "source": "Emerging norms" + } + ] + }, + { + "sub_area": "time_series", + "trigger_keywords": [ + "forecasting", + "time series", + "temporal", + "prediction interval" + ], + "expectations": [ + { + "figure": "Forecasting Horizon Curve", + "source": "ICLR/NeurIPS norms" + }, + { + "figure": "Decomposition Plot", + "source": "ICLR/NeurIPS norms" + }, + { + "figure": "Residual ACF Plot", + "source": "ICLR/NeurIPS norms" + } + ] + } + ] +} diff --git a/src/autoskillit/recipes/methodology-traditions/method_comparison_benchmarking.yaml b/src/autoskillit/recipes/methodology-traditions/method_comparison_benchmarking.yaml new file mode 100644 index 0000000000..cb2990ea5d --- /dev/null +++ b/src/autoskillit/recipes/methodology-traditions/method_comparison_benchmarking.yaml @@ -0,0 +1,165 @@ +name: method_comparison_benchmarking +schema_version: "1.0" +priority: 12 +display_name: "Method Comparison and Benchmarking" +canonical_guideline: + name: "Weber et al. 2019" + governing_body: "community" + stable_for_decade: true + canonical: false +fields_spanned: + - computer_science + - software_engineering + - machine_learning + - bioinformatics + - operations_research +detection_keywords: + - benchmark + - method comparison + - baseline comparison + - state-of-the-art + - SOTA + - leaderboard + - ablation study + - performance comparison + - evaluation protocol +mandatory_figures: + - figure: "Performance Comparison Table or Figure" + shows: "Quantitative metrics comparing all methods side-by-side" + citation: "Weber et al. 2019 Section 4" + - figure: "Statistical Significance Test Results" + shows: "Tests comparing method performance with p-values or confidence intervals" + citation: "Weber et al. 2019 Section 4" +strongly_expected_figures: + - figure: "Critical Difference Diagram" + shows: "Post-hoc pairwise comparisons with statistical significance" + source: "Demšar 2006" + - figure: "Runtime-Accuracy Pareto Front" + shows: "Trade-off between computational cost and accuracy metrics" + source: "Weber et al. 2019" +anti_patterns: + - pattern: "No statistical significance testing" + reason: "Performance differences must be tested for statistical significance" + - pattern: "Cherry-picked baselines" + reason: "All relevant competing methods should be included; omission of strong baselines biases conclusions" + - pattern: "Unreported hyperparameter tuning" + reason: "Methods must be fairly tuned before comparison to ensure valid assessment" + - pattern: "No ablation study" + reason: "Ablation studies isolate the contribution of individual components" +venue_specific_appendices: + - sub_area: supervised_classification + trigger_keywords: + - confusion matrix + - precision + - recall + - ROC-AUC + - PR curve + - classification + - learning curve + expectations: + - figure: "Confusion Matrix" + source: "NeurIPS/ICML community norms" + - figure: "PR Curve" + source: "NeurIPS/ICML community norms" + - figure: "ROC-AUC Plot" + source: "NeurIPS/ICML community norms" + - figure: "Learning Curves" + source: "NeurIPS/ICML community norms" + - sub_area: nlp + trigger_keywords: + - accuracy + - F1 + - GLUE + - SuperGLUE + - NLP + - language model + - token + expectations: + - figure: "Per-Task Accuracy Table" + source: "ACL norms" + - figure: "Error Analysis Examples" + source: "ACL norms" + - sub_area: computer_vision + trigger_keywords: + - mAP + - COCO + - detection + - segmentation + - object detection + - image classification + expectations: + - figure: "Sample Predictions Grid" + source: "CVPR/ECCV norms" + - figure: "Failure Gallery" + source: "CVPR/ECCV norms" + - figure: "Per-Class mAP Table" + source: "CVPR/ECCV norms" + - sub_area: reinforcement_learning + trigger_keywords: + - episode reward + - RL + - Atari + - agent + - policy gradient + expectations: + - figure: "Episode Reward Curve" + source: "NeurIPS RL norms" + - figure: "Sample Efficiency Curve" + source: "NeurIPS RL norms" + - sub_area: generative_models + trigger_keywords: + - GAN + - diffusion + - FID + - IS + - generative + - sample quality + expectations: + - figure: "Sample Grid" + source: "NeurIPS/ICLR norms" + - figure: "FID/IS Table" + source: "NeurIPS/ICLR norms" + - figure: "Failure Modes Gallery" + source: "NeurIPS/ICLR norms" + - sub_area: foundation_models + trigger_keywords: + - LLM + - few-shot + - scaling + - GPT + - foundation model + - zero-shot + expectations: + - figure: "Few-Shot Scaling Curve" + source: "LLM evaluation norms" + - figure: "Contamination Analysis" + source: "LLM evaluation norms" + - figure: "Ablation Table" + source: "LLM evaluation norms" + - sub_area: agentic_systems + trigger_keywords: + - agentic + - tool use + - multi-agent + - task success + - workflow + expectations: + - figure: "Task Success Rate Bar Chart" + source: "Emerging norms" + - figure: "Step-Level Execution Traces" + source: "Emerging norms" + - figure: "Tool Use Breakdown" + source: "Emerging norms" + - sub_area: time_series + trigger_keywords: + - forecasting + - time series + - temporal + - prediction interval + expectations: + - figure: "Forecasting Horizon Curve" + source: "ICLR/NeurIPS norms" + - figure: "Decomposition Plot" + source: "ICLR/NeurIPS norms" + - figure: "Residual ACF Plot" + source: "ICLR/NeurIPS norms" diff --git a/src/autoskillit/recipes/methodology-traditions/observational_correlational.json b/src/autoskillit/recipes/methodology-traditions/observational_correlational.json new file mode 100644 index 0000000000..b1a8b1e433 --- /dev/null +++ b/src/autoskillit/recipes/methodology-traditions/observational_correlational.json @@ -0,0 +1,64 @@ +{ + "name": "observational_correlational", + "schema_version": "1.0", + "priority": 3, + "display_name": "Observational Correlational Study", + "canonical_guideline": { + "name": "STROBE", + "governing_body": "EQUATOR", + "stable_for_decade": true, + "canonical": true + }, + "fields_spanned": [ + "medicine", + "epidemiology", + "psychology", + "sociology", + "public_health" + ], + "detection_keywords": [ + "observational study", + "cohort", + "case-control", + "cross-sectional", + "STROBE", + "odds ratio", + "hazard ratio", + "incidence", + "prevalence" + ], + "mandatory_figures": [ + { + "figure": "Participant Flow Diagram", + "shows": "Enrollment, exposure status, and outcome assessment at each stage", + "citation": "STROBE item 13" + } + ], + "strongly_expected_figures": [ + { + "figure": "Directed Acyclic Graph (DAG)", + "shows": "Causal structure and confounding variables", + "source": "STROBE item 16" + }, + { + "figure": "Adjusted Effect Estimates Table", + "shows": "Multivariable regression results with confounders", + "source": "STROBE item 16" + } + ], + "anti_patterns": [ + { + "pattern": "Causal language without randomization", + "reason": "Observational studies cannot establish causation; language must reflect association only" + }, + { + "pattern": "No confounder adjustment", + "reason": "Failure to adjust for measured confounders introduces omitted variable bias" + }, + { + "pattern": "No DAG", + "reason": "DAGs are essential for transparent confounding control in observational studies" + } + ], + "venue_specific_appendices": [] +} diff --git a/src/autoskillit/recipes/methodology-traditions/observational_correlational.yaml b/src/autoskillit/recipes/methodology-traditions/observational_correlational.yaml new file mode 100644 index 0000000000..554bf690e8 --- /dev/null +++ b/src/autoskillit/recipes/methodology-traditions/observational_correlational.yaml @@ -0,0 +1,44 @@ +name: observational_correlational +schema_version: "1.0" +priority: 3 +display_name: "Observational Correlational Study" +canonical_guideline: + name: "STROBE" + governing_body: "EQUATOR" + stable_for_decade: true + canonical: true +fields_spanned: + - medicine + - epidemiology + - psychology + - sociology + - public_health +detection_keywords: + - observational study + - cohort + - case-control + - cross-sectional + - STROBE + - odds ratio + - hazard ratio + - incidence + - prevalence +mandatory_figures: + - figure: "Participant Flow Diagram" + shows: "Enrollment, exposure status, and outcome assessment at each stage" + citation: "STROBE item 13" +strongly_expected_figures: + - figure: "Directed Acyclic Graph (DAG)" + shows: "Causal structure and confounding variables" + source: "STROBE item 16" + - figure: "Adjusted Effect Estimates Table" + shows: "Multivariable regression results with confounders" + source: "STROBE item 16" +anti_patterns: + - pattern: "Causal language without randomization" + reason: "Observational studies cannot establish causation; language must reflect association only" + - pattern: "No confounder adjustment" + reason: "Failure to adjust for measured confounders introduces omitted variable bias" + - pattern: "No DAG" + reason: "DAGs are essential for transparent confounding control in observational studies" +venue_specific_appendices: [] diff --git a/src/autoskillit/recipes/methodology-traditions/prediction_model_validation.json b/src/autoskillit/recipes/methodology-traditions/prediction_model_validation.json new file mode 100644 index 0000000000..3ace537216 --- /dev/null +++ b/src/autoskillit/recipes/methodology-traditions/prediction_model_validation.json @@ -0,0 +1,182 @@ +{ + "name": "prediction_model_validation", + "schema_version": "1.0", + "priority": 5, + "display_name": "Prediction Model Validation", + "canonical_guideline": { + "name": "TRIPOD+AI 2024", + "governing_body": "EQUATOR", + "stable_for_decade": false, + "canonical": true + }, + "fields_spanned": [ + "medicine", + "machine_learning", + "bioinformatics", + "clinical_informatics", + "epidemiology" + ], + "detection_keywords": [ + "prediction model", + "TRIPOD", + "calibration", + "discrimination", + "c-statistic", + "external validation", + "clinical prediction rule", + "risk score", + "prognostic model" + ], + "mandatory_figures": [ + { + "figure": "Calibration Plot", + "shows": "Observed vs predicted probabilities across risk strata", + "citation": "TRIPOD item 16" + }, + { + "figure": "Participant Flow Diagram", + "shows": "Enrollment, model development, and validation flow", + "citation": "TRIPOD item 13" + } + ], + "strongly_expected_figures": [ + { + "figure": "Decision Curve Analysis", + "shows": "Net benefit across threshold probabilities", + "source": "TRIPOD+AI item 16" + }, + { + "figure": "Net Benefit Plot", + "shows": "Clinical utility at varying decision thresholds", + "source": "Vickers et al. 2008" + } + ], + "anti_patterns": [ + { + "pattern": "No external validation", + "reason": "Internal validation alone is insufficient; external validation demonstrates transportability" + }, + { + "pattern": "Reporting discrimination without calibration", + "reason": "Discrimination (AUC/c-statistic) measures ranking ability; calibration measures absolute probability accuracy" + }, + { + "pattern": "No sample size justification", + "reason": "Prediction models require sufficient events per variable to avoid overfitting" + } + ], + "venue_specific_appendices": [ + { + "sub_area": "foundation_models", + "trigger_keywords": [ + "calibration", + "held-out", + "few-shot scaling", + "zero-shot validation" + ], + "expectations": [ + { + "figure": "Calibration Plot for Scaling Predictions", + "source": "TRIPOD+AI norms" + }, + { + "figure": "Held-Out Performance Curves", + "source": "TRIPOD+AI norms" + } + ] + }, + { + "sub_area": "reinforcement_learning", + "trigger_keywords": [ + "policy evaluation", + "held-out task", + "sample efficiency" + ], + "expectations": [ + { + "figure": "Policy Evaluation Calibration", + "source": "TRIPOD+AI norms" + }, + { + "figure": "Held-Out Task Reward Curves", + "source": "TRIPOD+AI norms" + } + ] + }, + { + "sub_area": "supervised_classification", + "trigger_keywords": [ + "patient outcome", + "prognosis", + "risk prediction", + "clinical decision" + ], + "expectations": [ + { + "figure": "Clinical Calibration Plot", + "source": "TRIPOD+AI norms" + }, + { + "figure": "Decision Curve Analysis", + "source": "TRIPOD+AI norms" + } + ] + }, + { + "sub_area": "nlp", + "trigger_keywords": [ + "clinical NLP", + "risk scoring", + "patient outcome", + "clinical text" + ], + "expectations": [ + { + "figure": "Clinical Text Prediction Calibration", + "source": "TRIPOD+AI norms" + }, + { + "figure": "Risk Score Validation", + "source": "TRIPOD+AI norms" + } + ] + }, + { + "sub_area": "computer_vision", + "trigger_keywords": [ + "patient outcome", + "prognosis", + "clinical decision", + "imaging biomarker" + ], + "expectations": [ + { + "figure": "Imaging Biomarker Calibration", + "source": "TRIPOD+AI norms" + }, + { + "figure": "Imaging Prediction Curves", + "source": "TRIPOD+AI norms" + } + ] + }, + { + "sub_area": "time_series", + "trigger_keywords": [ + "clinical forecasting", + "patient trajectory", + "mortality prediction" + ], + "expectations": [ + { + "figure": "Temporal Calibration Plot", + "source": "TRIPOD+AI norms" + }, + { + "figure": "Forecast Horizon Validation", + "source": "TRIPOD+AI norms" + } + ] + } + ] +} diff --git a/src/autoskillit/recipes/methodology-traditions/prediction_model_validation.yaml b/src/autoskillit/recipes/methodology-traditions/prediction_model_validation.yaml new file mode 100644 index 0000000000..73d894a758 --- /dev/null +++ b/src/autoskillit/recipes/methodology-traditions/prediction_model_validation.yaml @@ -0,0 +1,111 @@ +name: prediction_model_validation +schema_version: "1.0" +priority: 5 +display_name: "Prediction Model Validation" +canonical_guideline: + name: "TRIPOD+AI 2024" + governing_body: "EQUATOR" + stable_for_decade: false + canonical: true +fields_spanned: + - medicine + - machine_learning + - bioinformatics + - clinical_informatics + - epidemiology +detection_keywords: + - prediction model + - TRIPOD + - calibration + - discrimination + - c-statistic + - external validation + - clinical prediction rule + - risk score + - prognostic model +mandatory_figures: + - figure: "Calibration Plot" + shows: "Observed vs predicted probabilities across risk strata" + citation: "TRIPOD item 16" + - figure: "Participant Flow Diagram" + shows: "Enrollment, model development, and validation flow" + citation: "TRIPOD item 13" +strongly_expected_figures: + - figure: "Decision Curve Analysis" + shows: "Net benefit across threshold probabilities" + source: "TRIPOD+AI item 16" + - figure: "Net Benefit Plot" + shows: "Clinical utility at varying decision thresholds" + source: "Vickers et al. 2008" +anti_patterns: + - pattern: "No external validation" + reason: "Internal validation alone is insufficient; external validation demonstrates transportability" + - pattern: "Reporting discrimination without calibration" + reason: "Discrimination (AUC/c-statistic) measures ranking ability; calibration measures absolute probability accuracy" + - pattern: "No sample size justification" + reason: "Prediction models require sufficient events per variable to avoid overfitting" +venue_specific_appendices: + - sub_area: foundation_models + trigger_keywords: + - calibration + - held-out + - few-shot scaling + - zero-shot validation + expectations: + - figure: "Calibration Plot for Scaling Predictions" + source: "TRIPOD+AI norms" + - figure: "Held-Out Performance Curves" + source: "TRIPOD+AI norms" + - sub_area: reinforcement_learning + trigger_keywords: + - policy evaluation + - held-out task + - sample efficiency + expectations: + - figure: "Policy Evaluation Calibration" + source: "TRIPOD+AI norms" + - figure: "Held-Out Task Reward Curves" + source: "TRIPOD+AI norms" + - sub_area: supervised_classification + trigger_keywords: + - patient outcome + - prognosis + - risk prediction + - clinical decision + expectations: + - figure: "Clinical Calibration Plot" + source: "TRIPOD+AI norms" + - figure: "Decision Curve Analysis" + source: "TRIPOD+AI norms" + - sub_area: nlp + trigger_keywords: + - clinical NLP + - risk scoring + - patient outcome + - clinical text + expectations: + - figure: "Clinical Text Prediction Calibration" + source: "TRIPOD+AI norms" + - figure: "Risk Score Validation" + source: "TRIPOD+AI norms" + - sub_area: computer_vision + trigger_keywords: + - patient outcome + - prognosis + - clinical decision + - imaging biomarker + expectations: + - figure: "Imaging Biomarker Calibration" + source: "TRIPOD+AI norms" + - figure: "Imaging Prediction Curves" + source: "TRIPOD+AI norms" + - sub_area: time_series + trigger_keywords: + - clinical forecasting + - patient trajectory + - mortality prediction + expectations: + - figure: "Temporal Calibration Plot" + source: "TRIPOD+AI norms" + - figure: "Forecast Horizon Validation" + source: "TRIPOD+AI norms" diff --git a/src/autoskillit/recipes/methodology-traditions/qualitative_interpretive_tradition.json b/src/autoskillit/recipes/methodology-traditions/qualitative_interpretive_tradition.json new file mode 100644 index 0000000000..4974979dfc --- /dev/null +++ b/src/autoskillit/recipes/methodology-traditions/qualitative_interpretive_tradition.json @@ -0,0 +1,103 @@ +{ + "name": "qualitative_interpretive_tradition", + "schema_version": "1.0", + "priority": 11, + "display_name": "Qualitative Interpretive Research", + "canonical_guideline": { + "name": "COREQ/SRQR", + "governing_body": "EQUATOR", + "stable_for_decade": true, + "canonical": true + }, + "fields_spanned": [ + "medicine", + "nursing", + "sociology", + "education", + "psychology", + "public_health" + ], + "detection_keywords": [ + "qualitative research", + "COREQ", + "SRQR", + "thematic analysis", + "grounded theory", + "phenomenology", + "ethnography", + "semi-structured interview", + "focus group", + "coding framework" + ], + "mandatory_figures": [], + "strongly_expected_figures": [ + { + "figure": "Coding Tree or Thematic Map", + "shows": "Hierarchy of themes and subthemes with relationships", + "source": "COREQ item 28" + }, + { + "figure": "Participant Characteristics Table", + "shows": "Demographics and context of qualitative participants", + "source": "COREQ item 22" + } + ], + "anti_patterns": [ + { + "pattern": "No reflexivity statement", + "reason": "COREQ and SRQR require researchers to reflect on their role and influence on data collection and analysis" + }, + { + "pattern": "Quantifying qualitative data inappropriately", + "reason": "Qualitative data is not meant to be reduced to frequencies or percentages" + }, + { + "pattern": "No saturation discussion", + "reason": "Researchers must discuss whether data saturation was achieved" + }, + { + "pattern": "Member checking unreported", + "reason": "COREQ encourages reporting of participant validation of findings" + } + ], + "venue_specific_appendices": [ + { + "sub_area": "nlp", + "trigger_keywords": [ + "discourse analysis", + "conversation analysis", + "narrative", + "thematic" + ], + "expectations": [ + { + "figure": "Thematic Map for Discourse Analysis", + "source": "COREQ/SRQR norms" + }, + { + "figure": "Coding Framework Visualization", + "source": "COREQ/SRQR norms" + } + ] + }, + { + "sub_area": "generative_models", + "trigger_keywords": [ + "human evaluation", + "qualitative assessment", + "expert rating", + "content analysis" + ], + "expectations": [ + { + "figure": "Qualitative Evaluation Coding Tree", + "source": "COREQ/SRQR norms" + }, + { + "figure": "Expert Assessment Summary", + "source": "COREQ/SRQR norms" + } + ] + } + ] +} diff --git a/src/autoskillit/recipes/methodology-traditions/qualitative_interpretive_tradition.yaml b/src/autoskillit/recipes/methodology-traditions/qualitative_interpretive_tradition.yaml new file mode 100644 index 0000000000..581996d481 --- /dev/null +++ b/src/autoskillit/recipes/methodology-traditions/qualitative_interpretive_tradition.yaml @@ -0,0 +1,67 @@ +name: qualitative_interpretive_tradition +schema_version: "1.0" +priority: 11 +display_name: "Qualitative Interpretive Research" +canonical_guideline: + name: "COREQ/SRQR" + governing_body: "EQUATOR" + stable_for_decade: true + canonical: true +fields_spanned: + - medicine + - nursing + - sociology + - education + - psychology + - public_health +detection_keywords: + - qualitative research + - COREQ + - SRQR + - thematic analysis + - grounded theory + - phenomenology + - ethnography + - semi-structured interview + - focus group + - coding framework +mandatory_figures: [] +strongly_expected_figures: + - figure: "Coding Tree or Thematic Map" + shows: "Hierarchy of themes and subthemes with relationships" + source: "COREQ item 28" + - figure: "Participant Characteristics Table" + shows: "Demographics and context of qualitative participants" + source: "COREQ item 22" +anti_patterns: + - pattern: "No reflexivity statement" + reason: "COREQ and SRQR require researchers to reflect on their role and influence on data collection and analysis" + - pattern: "Quantifying qualitative data inappropriately" + reason: "Qualitative data is not meant to be reduced to frequencies or percentages" + - pattern: "No saturation discussion" + reason: "Researchers must discuss whether data saturation was achieved" + - pattern: "Member checking unreported" + reason: "COREQ encourages reporting of participant validation of findings" +venue_specific_appendices: + - sub_area: nlp + trigger_keywords: + - discourse analysis + - conversation analysis + - narrative + - thematic + expectations: + - figure: "Thematic Map for Discourse Analysis" + source: "COREQ/SRQR norms" + - figure: "Coding Framework Visualization" + source: "COREQ/SRQR norms" + - sub_area: generative_models + trigger_keywords: + - human evaluation + - qualitative assessment + - expert rating + - content analysis + expectations: + - figure: "Qualitative Evaluation Coding Tree" + source: "COREQ/SRQR norms" + - figure: "Expert Assessment Summary" + source: "COREQ/SRQR norms" diff --git a/src/autoskillit/recipes/methodology-traditions/quality_improvement.json b/src/autoskillit/recipes/methodology-traditions/quality_improvement.json new file mode 100644 index 0000000000..fbdc2d5101 --- /dev/null +++ b/src/autoskillit/recipes/methodology-traditions/quality_improvement.json @@ -0,0 +1,92 @@ +{ + "name": "quality_improvement", + "schema_version": "1.0", + "priority": 8, + "display_name": "Quality Improvement and Patient Safety", + "canonical_guideline": { + "name": "SQUIRE 2.0", + "governing_body": "EQUATOR", + "stable_for_decade": true, + "canonical": true + }, + "fields_spanned": [ + "medicine", + "nursing", + "healthcare_management", + "public_health" + ], + "detection_keywords": [ + "quality improvement", + "SQUIRE", + "PDSA cycle", + "Plan-Do-Study-Act", + "process improvement", + "clinical pathway", + "implementation science", + "lean", + "six sigma" + ], + "mandatory_figures": [ + { + "figure": "Run Chart or Control Chart", + "shows": "Process performance over time with centerline and control limits", + "citation": "SQUIRE item 15" + }, + { + "figure": "Intervention Logic Model", + "shows": "Theory of change linking intervention to outcomes", + "citation": "SQUIRE item 10" + } + ], + "strongly_expected_figures": [ + { + "figure": "PDSA Cycle Diagram", + "shows": "Iterative improvement cycles with Plan, Do, Study, Act phases", + "source": "SQUIRE item 8" + }, + { + "figure": "Process Map", + "shows": "Current or proposed workflow with improvement targets", + "source": "Lean methodology" + } + ], + "anti_patterns": [ + { + "pattern": "No baseline measurement", + "reason": "QI requires pre-intervention data to establish the problem and measure improvement" + }, + { + "pattern": "Single post-intervention data point", + "reason": "Multiple measurements over time are needed to demonstrate sustained improvement" + }, + { + "pattern": "No control chart rules applied", + "reason": "Control chart rules detect special cause variation; without them, signals cannot be distinguished from noise" + }, + { + "pattern": "Confusing QI with research", + "reason": "QI aims to improve local processes; research aims to generate generalizable knowledge" + } + ], + "venue_specific_appendices": [ + { + "sub_area": "agentic_systems", + "trigger_keywords": [ + "process improvement", + "workflow optimization", + "task efficiency", + "error reduction" + ], + "expectations": [ + { + "figure": "Run Chart for Agent Performance", + "source": "SQUIRE 2.0" + }, + { + "figure": "Process Map for Task Workflow", + "source": "Lean methodology" + } + ] + } + ] +} diff --git a/src/autoskillit/recipes/methodology-traditions/quality_improvement.yaml b/src/autoskillit/recipes/methodology-traditions/quality_improvement.yaml new file mode 100644 index 0000000000..79bfa14478 --- /dev/null +++ b/src/autoskillit/recipes/methodology-traditions/quality_improvement.yaml @@ -0,0 +1,59 @@ +name: quality_improvement +schema_version: "1.0" +priority: 8 +display_name: "Quality Improvement and Patient Safety" +canonical_guideline: + name: "SQUIRE 2.0" + governing_body: "EQUATOR" + stable_for_decade: true + canonical: true +fields_spanned: + - medicine + - nursing + - healthcare_management + - public_health +detection_keywords: + - quality improvement + - SQUIRE + - PDSA cycle + - Plan-Do-Study-Act + - process improvement + - clinical pathway + - implementation science + - lean + - six sigma +mandatory_figures: + - figure: "Run Chart or Control Chart" + shows: "Process performance over time with centerline and control limits" + citation: "SQUIRE item 15" + - figure: "Intervention Logic Model" + shows: "Theory of change linking intervention to outcomes" + citation: "SQUIRE item 10" +strongly_expected_figures: + - figure: "PDSA Cycle Diagram" + shows: "Iterative improvement cycles with Plan, Do, Study, Act phases" + source: "SQUIRE item 8" + - figure: "Process Map" + shows: "Current or proposed workflow with improvement targets" + source: "Lean methodology" +anti_patterns: + - pattern: "No baseline measurement" + reason: "QI requires pre-intervention data to establish the problem and measure improvement" + - pattern: "Single post-intervention data point" + reason: "Multiple measurements over time are needed to demonstrate sustained improvement" + - pattern: "No control chart rules applied" + reason: "Control chart rules detect special cause variation; without them, signals cannot be distinguished from noise" + - pattern: "Confusing QI with research" + reason: "QI aims to improve local processes; research aims to generate generalizable knowledge" +venue_specific_appendices: + - sub_area: agentic_systems + trigger_keywords: + - process improvement + - workflow optimization + - task efficiency + - error reduction + expectations: + - figure: "Run Chart for Agent Performance" + source: "SQUIRE 2.0" + - figure: "Process Map for Task Workflow" + source: "Lean methodology" diff --git a/src/autoskillit/recipes/methodology-traditions/simulation_modeling_tradition.json b/src/autoskillit/recipes/methodology-traditions/simulation_modeling_tradition.json new file mode 100644 index 0000000000..bcd5619a9d --- /dev/null +++ b/src/autoskillit/recipes/methodology-traditions/simulation_modeling_tradition.json @@ -0,0 +1,131 @@ +{ + "name": "simulation_modeling_tradition", + "schema_version": "1.0", + "priority": 6, + "display_name": "Simulation and Modeling (ABM/Computational)", + "canonical_guideline": { + "name": "ODD Protocol", + "governing_body": "community", + "stable_for_decade": true, + "canonical": false + }, + "fields_spanned": [ + "ecology", + "economics", + "social_science", + "computer_science", + "epidemiology" + ], + "detection_keywords": [ + "agent-based model", + "ABM", + "ODD protocol", + "simulation model", + "computational model", + "NetLogo", + "parameter sweep", + "emergent behavior", + "individual-based model" + ], + "mandatory_figures": [ + { + "figure": "Model Overview Diagram", + "shows": "Entities, state variables, processes, and scheduling", + "citation": "ODD Overview section" + }, + { + "figure": "Sensitivity Analysis Heatmap", + "shows": "Parameter sensitivity across output metrics", + "citation": "ODD Submodels section" + } + ], + "strongly_expected_figures": [ + { + "figure": "Time-Series Output", + "shows": "Model dynamics over simulated time", + "source": "ODD Output section" + }, + { + "figure": "Parameter Sensitivity Tornado Plot", + "shows": "Ranked influence of each parameter on outputs", + "source": "Saltelli et al. 2008" + } + ], + "anti_patterns": [ + { + "pattern": "No ODD protocol description", + "reason": "ODD provides the standardized structure for transparent ABM reporting" + }, + { + "pattern": "Unreported parameter ranges", + "reason": "Readers need to know what parameter values were explored" + }, + { + "pattern": "No sensitivity analysis", + "reason": "Without sensitivity analysis, readers cannot assess which parameters drive outputs" + }, + { + "pattern": "Single-run conclusions", + "reason": "ABMs require multiple runs to characterize stochastic variability and emergent patterns" + } + ], + "venue_specific_appendices": [ + { + "sub_area": "reinforcement_learning", + "trigger_keywords": [ + "multi-agent", + "emergent behavior", + "population dynamics", + "swarm" + ], + "expectations": [ + { + "figure": "ODD Protocol for RL Environment", + "source": "ODD Protocol" + }, + { + "figure": "Parameter Sweep Heatmaps", + "source": "Saltelli et al. 2008" + } + ] + }, + { + "sub_area": "agentic_systems", + "trigger_keywords": [ + "multi-agent", + "emergent behavior", + "agent-based model", + "swarm intelligence" + ], + "expectations": [ + { + "figure": "ODD Protocol for Multi-Agent System", + "source": "ODD Protocol" + }, + { + "figure": "Emergence Visualization", + "source": "ODD Protocol" + } + ] + }, + { + "sub_area": "time_series", + "trigger_keywords": [ + "dynamical system", + "differential equation", + "compartmental model", + "system dynamics" + ], + "expectations": [ + { + "figure": "Model Dynamics over Simulated Time", + "source": "ODD Output section" + }, + { + "figure": "Parameter Sensitivity Analysis", + "source": "Saltelli et al. 2008" + } + ] + } + ] +} diff --git a/src/autoskillit/recipes/methodology-traditions/simulation_modeling_tradition.yaml b/src/autoskillit/recipes/methodology-traditions/simulation_modeling_tradition.yaml new file mode 100644 index 0000000000..d00e0fdc45 --- /dev/null +++ b/src/autoskillit/recipes/methodology-traditions/simulation_modeling_tradition.yaml @@ -0,0 +1,82 @@ +name: simulation_modeling_tradition +schema_version: "1.0" +priority: 6 +display_name: "Simulation and Modeling (ABM/Computational)" +canonical_guideline: + name: "ODD Protocol" + governing_body: "community" + stable_for_decade: true + canonical: false +fields_spanned: + - ecology + - economics + - social_science + - computer_science + - epidemiology +detection_keywords: + - agent-based model + - ABM + - ODD protocol + - simulation model + - computational model + - NetLogo + - parameter sweep + - emergent behavior + - individual-based model +mandatory_figures: + - figure: "Model Overview Diagram" + shows: "Entities, state variables, processes, and scheduling" + citation: "ODD Overview section" + - figure: "Sensitivity Analysis Heatmap" + shows: "Parameter sensitivity across output metrics" + citation: "ODD Submodels section" +strongly_expected_figures: + - figure: "Time-Series Output" + shows: "Model dynamics over simulated time" + source: "ODD Output section" + - figure: "Parameter Sensitivity Tornado Plot" + shows: "Ranked influence of each parameter on outputs" + source: "Saltelli et al. 2008" +anti_patterns: + - pattern: "No ODD protocol description" + reason: "ODD provides the standardized structure for transparent ABM reporting" + - pattern: "Unreported parameter ranges" + reason: "Readers need to know what parameter values were explored" + - pattern: "No sensitivity analysis" + reason: "Without sensitivity analysis, readers cannot assess which parameters drive outputs" + - pattern: "Single-run conclusions" + reason: "ABMs require multiple runs to characterize stochastic variability and emergent patterns" +venue_specific_appendices: + - sub_area: reinforcement_learning + trigger_keywords: + - multi-agent + - emergent behavior + - population dynamics + - swarm + expectations: + - figure: "ODD Protocol for RL Environment" + source: "ODD Protocol" + - figure: "Parameter Sweep Heatmaps" + source: "Saltelli et al. 2008" + - sub_area: agentic_systems + trigger_keywords: + - multi-agent + - emergent behavior + - agent-based model + - swarm intelligence + expectations: + - figure: "ODD Protocol for Multi-Agent System" + source: "ODD Protocol" + - figure: "Emergence Visualization" + source: "ODD Protocol" + - sub_area: time_series + trigger_keywords: + - dynamical system + - differential equation + - compartmental model + - system dynamics + expectations: + - figure: "Model Dynamics over Simulated Time" + source: "ODD Output section" + - figure: "Parameter Sensitivity Analysis" + source: "Saltelli et al. 2008" diff --git a/src/autoskillit/recipes/methodology-traditions/systematic_synthesis.json b/src/autoskillit/recipes/methodology-traditions/systematic_synthesis.json new file mode 100644 index 0000000000..a26f111091 --- /dev/null +++ b/src/autoskillit/recipes/methodology-traditions/systematic_synthesis.json @@ -0,0 +1,69 @@ +{ + "name": "systematic_synthesis", + "schema_version": "1.0", + "priority": 2, + "display_name": "Systematic Synthesis (Meta-Analysis)", + "canonical_guideline": { + "name": "PRISMA 2020", + "governing_body": "Cochrane", + "stable_for_decade": true, + "canonical": true + }, + "fields_spanned": [ + "medicine", + "software_engineering", + "education", + "environmental_science", + "psychology" + ], + "detection_keywords": [ + "systematic review", + "meta-analysis", + "PRISMA", + "forest plot", + "pooled estimate", + "heterogeneity", + "I-squared", + "search strategy", + "inclusion criteria" + ], + "mandatory_figures": [ + { + "figure": "PRISMA Flow Diagram", + "shows": "Study identification, screening, eligibility, and inclusion at each stage", + "citation": "PRISMA Section 16" + }, + { + "figure": "Forest Plot", + "shows": "Pooled effect estimates with confidence intervals per study", + "citation": "PRISMA item 20d" + } + ], + "strongly_expected_figures": [ + { + "figure": "Funnel Plot", + "shows": "Publication bias assessment via asymmetry", + "source": "Cochrane Handbook" + }, + { + "figure": "Risk of Bias Summary", + "shows": "Quality assessment across included studies", + "source": "Cochrane RoB 2.0" + } + ], + "anti_patterns": [ + { + "pattern": "Narrative review disguised as systematic review", + "reason": "Systematic reviews require explicit search strategy, inclusion criteria, and quantitative synthesis" + }, + { + "pattern": "No PRISMA flow diagram", + "reason": "PRISMA requires transparent reporting of study selection process" + }, + { + "pattern": "Vote counting instead of quantitative synthesis", + "reason": "Vote counting (counting significant vs non-significant) ignores effect magnitudes and is methodologically invalid" + } + ], + "venue_specific_appendices": [] +} diff --git a/src/autoskillit/recipes/methodology-traditions/systematic_synthesis.yaml b/src/autoskillit/recipes/methodology-traditions/systematic_synthesis.yaml new file mode 100644 index 0000000000..a2342a6cf4 --- /dev/null +++ b/src/autoskillit/recipes/methodology-traditions/systematic_synthesis.yaml @@ -0,0 +1,47 @@ +name: systematic_synthesis +schema_version: "1.0" +priority: 2 +display_name: "Systematic Synthesis (Meta-Analysis)" +canonical_guideline: + name: "PRISMA 2020" + governing_body: "Cochrane" + stable_for_decade: true + canonical: true +fields_spanned: + - medicine + - software_engineering + - education + - environmental_science + - psychology +detection_keywords: + - systematic review + - meta-analysis + - PRISMA + - forest plot + - pooled estimate + - heterogeneity + - I-squared + - search strategy + - inclusion criteria +mandatory_figures: + - figure: "PRISMA Flow Diagram" + shows: "Study identification, screening, eligibility, and inclusion at each stage" + citation: "PRISMA Section 16" + - figure: "Forest Plot" + shows: "Pooled effect estimates with confidence intervals per study" + citation: "PRISMA item 20d" +strongly_expected_figures: + - figure: "Funnel Plot" + shows: "Publication bias assessment via asymmetry" + source: "Cochrane Handbook" + - figure: "Risk of Bias Summary" + shows: "Quality assessment across included studies" + source: "Cochrane RoB 2.0" +anti_patterns: + - pattern: "Narrative review disguised as systematic review" + reason: "Systematic reviews require explicit search strategy, inclusion criteria, and quantitative synthesis" + - pattern: "No PRISMA flow diagram" + reason: "PRISMA requires transparent reporting of study selection process" + - pattern: "Vote counting instead of quantitative synthesis" + reason: "Vote counting (counting significant vs non-significant) ignores effect magnitudes and is methodologically invalid" +venue_specific_appendices: [] diff --git a/src/autoskillit/recipes/planner.json b/src/autoskillit/recipes/planner.json new file mode 100644 index 0000000000..281b81cf45 --- /dev/null +++ b/src/autoskillit/recipes/planner.json @@ -0,0 +1,437 @@ +{ + "name": "planner", + "recipe_version": "2", + "description": "Progressive decomposition pipeline with mixed dispatch. Analyzes a codebase,\ngenerates phases, then elaborates each tier (phases, assignments, work packages).\nPhases and assignments elaborate in parallel; work packages elaborate sequentially\n(one phase at a time) to smooth token spend rate. Each tier follows the pattern:\nelaborate → merge → refine. Reconciles dependencies, validates the dependency\ngraph, and compiles a manifest ready for issue creation.\n", + "summary": "init > analyze > [domain?] > gen-phases > parallel-phases > parallel-assignments > sequential-wps > reconcile > validate/refine > compile", + "requires_packs": [ + "kitchen-core" + ], + "requires_features": [ + "planner" + ], + "categories": [ + "orchestration-family" + ], + "kitchen_rules": [ + "Never invoke native Claude Code tools (Read, Grep, Glob, Edit, Write, Bash, Agent, WebFetch, WebSearch, NotebookEdit) from the orchestrator. All codebase interaction runs inside headless skill sessions.", + "Route to on_failure or escalate_stop rather than attempting inline recovery.", + "The refine/validate cycle is bounded by retries. If refine is exhausted, halt.", + "PARALLEL DISPATCH: When a step note says 'PARALLEL DISPATCH', iterate the comma-separated list in the referenced context variable. For each item, substitute it into the skill_command template placeholder and issue one run_skill call. Issue ALL calls in a single assistant message so they execute concurrently. Accumulate each result via capture_list.", + "SEQUENTIAL DISPATCH: When a step note says 'SEQUENTIAL DISPATCH', iterate the comma-separated list in the referenced context variable. For each item, substitute it into the skill_command template placeholder and issue one run_skill call. Wait for the call to complete before issuing the next. Accumulate each result via capture_list." + ], + "ingredients": { + "task": { + "description": "Task description to decompose into a plan. Accepts either an absolute file path or inline text.", + "required": true + }, + "source_dir": { + "description": "Absolute path to the repository root to analyze.", + "required": true + }, + "planner_assess_review_approach": { + "description": "When true, run planner-assess-review-approach after reconcile_deps to annotate WPs that would benefit from a review-approach research pass before implementation.\n", + "required": false, + "default": "false", + "hidden": false + } + }, + "steps": { + "init": { + "tool": "run_python", + "with": { + "callable": "autoskillit.planner.create_run_dir", + "args": { + "temp_dir": "{{AUTOSKILLIT_TEMP}}" + } + }, + "capture": { + "planner_dir": "${{ result.planner_dir }}" + }, + "on_success": "resolve_task", + "on_failure": "escalate_stop" + }, + "resolve_task": { + "tool": "run_python", + "with": { + "callable": "autoskillit.planner.resolve_task_input", + "task": "${{ inputs.task }}", + "planner_dir": "${{ context.planner_dir }}" + }, + "capture": { + "task_file_path": "${{ result.task_file_path }}", + "task_label": "${{ result.task_label }}" + }, + "on_success": "analyze", + "on_failure": "escalate_stop" + }, + "analyze": { + "tool": "run_skill", + "model": "", + "with": { + "skill_command": "/autoskillit:planner-analyze ${{ context.planner_dir }}", + "cwd": "${{ inputs.source_dir }}", + "output_dir": "${{ context.planner_dir }}" + }, + "on_success": "extract_domain", + "on_failure": "escalate_stop" + }, + "extract_domain": { + "tool": "run_skill", + "model": "", + "with": { + "skill_command": "/autoskillit:planner-extract-domain ${{ context.planner_dir }}/analysis.json \"${{ context.task_file_path }}\"\n", + "cwd": "${{ inputs.source_dir }}", + "output_dir": "${{ context.planner_dir }}" + }, + "on_success": "generate_phases", + "on_failure": "generate_phases" + }, + "generate_phases": { + "tool": "run_skill", + "model": "", + "with": { + "skill_command": "/autoskillit:planner-generate-phases ${{ context.planner_dir }}/analysis.json ${{ context.planner_dir }}/domain_knowledge.md \"${{ context.task_file_path }}\"\n", + "cwd": "${{ inputs.source_dir }}", + "output_dir": "${{ context.planner_dir }}" + }, + "capture": { + "phase_manifest_path": "${{ result.phase_manifest_path }}" + }, + "on_success": "build_plan_snapshot", + "on_failure": "escalate_stop" + }, + "build_plan_snapshot": { + "tool": "run_python", + "with": { + "callable": "autoskillit.planner.merge.build_plan_snapshot", + "phases_dir": "${{ context.planner_dir }}/phases", + "phase_manifest_path": "${{ context.phase_manifest_path }}", + "output_path": "${{ context.planner_dir }}/plan_snapshot.json", + "task_file_path": "${{ context.task_file_path }}", + "source_dir": "${{ inputs.source_dir }}" + }, + "capture": { + "plan_snapshot_path": "${{ result.snapshot_path }}", + "phase_ids": "${{ result.phase_ids }}" + }, + "on_success": "elaborate_phases", + "on_failure": "escalate_stop" + }, + "elaborate_phases": { + "tool": "run_skill", + "model": "", + "with": { + "skill_command": "/autoskillit:planner-elaborate-phase ${{ context.plan_snapshot_path }} {phase_id} ${{ context.planner_dir }}/phases", + "cwd": "${{ inputs.source_dir }}", + "output_dir": "${{ context.planner_dir }}", + "dispatch_items": "${{ context.phase_ids }}" + }, + "capture_list": { + "elab_result_path": "${{ result.elab_result_path }}" + }, + "on_success": "merge_phases", + "on_failure": "escalate_stop", + "note": "PARALLEL DISPATCH: context.phase_ids contains comma-separated phase IDs (e.g. \"P1,P2,P3\"). For each phase_id, substitute it into the {phase_id} placeholder in skill_command and issue one run_skill call. Run ALL elaborations in parallel (all run_skill calls in a single message). Accumulate each elab_result_path value via capture_list.\n" + }, + "merge_phases": { + "tool": "run_python", + "with": { + "callable": "autoskillit.planner.merge.merge_tier_results", + "results_dir": "${{ context.planner_dir }}/phases", + "output_path": "${{ context.planner_dir }}/combined_plan.json", + "key": "phases", + "task_file_path": "${{ context.task_file_path }}", + "source_dir": "${{ inputs.source_dir }}" + }, + "capture": { + "combined_phases_path": "${{ result.merged_path }}" + }, + "on_success": "refine_phases", + "on_failure": "escalate_stop" + }, + "refine_phases": { + "tool": "run_skill", + "model": "", + "with": { + "skill_command": "/autoskillit:planner-refine-phases ${{ context.combined_phases_path }} ${{ context.planner_dir }}", + "cwd": "${{ inputs.source_dir }}", + "output_dir": "${{ context.planner_dir }}" + }, + "capture": { + "refined_plan_path": "${{ result.refined_plan_path }}" + }, + "on_success": "expand_assignments", + "on_failure": "escalate_stop" + }, + "expand_assignments": { + "tool": "run_python", + "with": { + "callable": "autoskillit.planner.expand_assignments", + "refined_plan_path": "${{ context.refined_plan_path }}", + "output_dir": "${{ context.planner_dir }}", + "task_file_path": "${{ context.task_file_path }}" + }, + "capture": { + "assignment_context_paths": "${{ result.context_paths }}" + }, + "on_success": "elaborate_assignments", + "on_failure": "escalate_stop" + }, + "elaborate_assignments": { + "tool": "run_skill", + "model": "", + "with": { + "skill_command": "/autoskillit:planner-elaborate-assignments {context_path} ${{ context.planner_dir }}", + "cwd": "${{ inputs.source_dir }}", + "output_dir": "${{ context.planner_dir }}/assignments", + "dispatch_items": "${{ context.assignment_context_paths }}" + }, + "capture_list": { + "phase_assignments_result_dir": "${{ result.phase_assignments_result_dir }}" + }, + "on_success": "merge_assignments", + "on_failure": "escalate_stop", + "note": "PARALLEL DISPATCH: context.assignment_context_paths contains comma-separated paths to per-phase context files. For each context_path, substitute it into the {context_path} placeholder in skill_command and issue one run_skill call. Run ALL elaborations in parallel. Accumulate results via capture_list.\n" + }, + "merge_assignments": { + "tool": "run_python", + "with": { + "callable": "autoskillit.planner.merge.merge_tier_results", + "results_dir": "${{ context.planner_dir }}/assignments", + "output_path": "${{ context.planner_dir }}/combined_assignments.json", + "key": "assignments", + "task_file_path": "${{ context.task_file_path }}", + "source_dir": "${{ inputs.source_dir }}" + }, + "capture": { + "refine_context_paths": "${{ result.refine_context_paths }}" + }, + "on_success": "refine_assignments", + "on_failure": "escalate_stop" + }, + "refine_assignments": { + "tool": "run_skill", + "model": "", + "with": { + "skill_command": "/autoskillit:planner-refine-assignments {context_path} ${{ context.refined_plan_path }} ${{ context.planner_dir }}", + "cwd": "${{ inputs.source_dir }}", + "output_dir": "${{ context.planner_dir }}/refine_contexts", + "dispatch_items": "${{ context.refine_context_paths }}" + }, + "capture_list": { + "phase_refined_path": "${{ result.phase_refined_path }}" + }, + "on_success": "merge_refined_assignments", + "on_failure": "escalate_stop", + "note": "PARALLEL DISPATCH: context.refine_context_paths contains comma-separated paths to per-phase context files produced by merge_tier_results. For each context_path, substitute it into the {context_path} placeholder and issue one run_skill call. Run ALL refinements in parallel. Accumulate results via capture_list.\n" + }, + "merge_refined_assignments": { + "tool": "run_python", + "with": { + "callable": "autoskillit.planner.merge.merge_refined_assignments", + "planner_dir": "${{ context.planner_dir }}" + }, + "capture": { + "refined_assignments_path": "${{ result.refined_assignments_path }}" + }, + "on_success": "expand_wps", + "on_failure": "escalate_stop" + }, + "expand_wps": { + "tool": "run_python", + "with": { + "callable": "autoskillit.planner.expand_wps", + "refined_assignments_path": "${{ context.refined_assignments_path }}", + "output_dir": "${{ context.planner_dir }}", + "task_file_path": "${{ context.task_file_path }}" + }, + "capture": { + "wp_context_paths": "${{ result.context_paths }}" + }, + "on_success": "elaborate_wps", + "on_failure": "escalate_stop" + }, + "elaborate_wps": { + "tool": "run_skill", + "model": "", + "with": { + "skill_command": "/autoskillit:planner-elaborate-wps {context_path} ${{ context.planner_dir }}", + "cwd": "${{ inputs.source_dir }}", + "output_dir": "${{ context.planner_dir }}/work_packages", + "dispatch_items": "${{ context.wp_context_paths }}" + }, + "capture_list": { + "phase_wps_result_dir": "${{ result.phase_wps_result_dir }}" + }, + "on_success": "finalize_wp_manifest", + "on_failure": "escalate_stop", + "note": "SEQUENTIAL DISPATCH: context.wp_context_paths contains comma-separated paths to per-phase WP context files. For each context_path, substitute it into the {context_path} placeholder in skill_command and issue one run_skill call. Wait for each elaboration to complete before starting the next. Accumulate results via capture_list.\n" + }, + "finalize_wp_manifest": { + "tool": "run_python", + "with": { + "callable": "autoskillit.planner.finalize_wp_manifest", + "work_packages_dir": "${{ context.planner_dir }}/work_packages", + "output_dir": "${{ context.planner_dir }}" + }, + "on_success": "merge_wps", + "on_failure": "escalate_stop" + }, + "merge_wps": { + "tool": "run_python", + "with": { + "callable": "autoskillit.planner.merge.merge_tier_results", + "results_dir": "${{ context.planner_dir }}/work_packages", + "output_path": "${{ context.planner_dir }}/combined_wps.json", + "key": "work_packages", + "task_file_path": "${{ context.task_file_path }}", + "source_dir": "${{ inputs.source_dir }}" + }, + "capture": { + "combined_wps_path": "${{ result.merged_path }}" + }, + "on_success": "refine_wps", + "on_failure": "escalate_stop" + }, + "refine_wps": { + "tool": "run_skill", + "model": "", + "with": { + "skill_command": "/autoskillit:planner-refine-wps ${{ context.combined_wps_path }} ${{ context.refined_plan_path }} ${{ context.refined_assignments_path }} ${{ context.planner_dir }}\n", + "cwd": "${{ inputs.source_dir }}", + "output_dir": "${{ context.planner_dir }}" + }, + "capture": { + "refined_wps_path": "${{ result.refined_wps_path }}" + }, + "on_success": "consolidate_wps_analyze", + "on_failure": "escalate_stop" + }, + "consolidate_wps_analyze": { + "tool": "run_skill", + "model": "", + "with": { + "skill_command": "/autoskillit:planner-consolidate-wps ${{ context.refined_wps_path }} ${{ context.planner_dir }}\n", + "cwd": "${{ inputs.source_dir }}", + "output_dir": "${{ context.planner_dir }}" + }, + "on_success": "consolidate_wps_merge", + "on_failure": "consolidate_wps_merge", + "note": "Analysis-only step: always routes to consolidate_wps_merge regardless of success or failure. If the skill fails, the merge callable falls back to passing all WPs through unchanged (no consolidation manifests = no-op merge).\n" + }, + "consolidate_wps_merge": { + "tool": "run_python", + "with": { + "callable": "autoskillit.planner.consolidate_wps", + "refined_wps_path": "${{ context.refined_wps_path }}", + "planner_dir": "${{ context.planner_dir }}" + }, + "capture": { + "consolidated_wps_path": "${{ result.consolidated_wps_path }}" + }, + "on_success": "validate_task_alignment", + "on_failure": "escalate_stop" + }, + "validate_task_alignment": { + "tool": "run_skill", + "model": "", + "with": { + "skill_command": "/autoskillit:planner-validate-task-alignment ${{ context.consolidated_wps_path }} ${{ context.refined_plan_path }} ${{ context.planner_dir }}\n", + "cwd": "${{ inputs.source_dir }}", + "output_dir": "${{ context.planner_dir }}" + }, + "capture": { + "alignment_findings_path": "${{ result.alignment_findings_path }}" + }, + "on_success": "reconcile_deps", + "on_failure": "reconcile_deps" + }, + "reconcile_deps": { + "tool": "run_skill", + "model": "", + "with": { + "skill_command": "/autoskillit:planner-reconcile-deps ${{ context.planner_dir }}", + "cwd": "${{ inputs.source_dir }}", + "output_dir": "${{ context.planner_dir }}", + "consolidated_wps_path": "${{ context.consolidated_wps_path }}" + }, + "on_success": "assess_review_approach_step", + "on_failure": "escalate_stop" + }, + "assess_review_approach_step": { + "tool": "run_skill", + "model": "", + "optional": true, + "skip_when_false": "inputs.planner_assess_review_approach", + "with": { + "skill_command": "/autoskillit:planner-assess-review-approach ${{ context.consolidated_wps_path }} ${{ context.planner_dir }}\n", + "cwd": "${{ inputs.source_dir }}", + "output_dir": "${{ context.planner_dir }}" + }, + "capture": { + "review_approach_assessment_path": "${{ result.review_approach_assessment_path }}" + }, + "on_success": "validate", + "on_failure": "validate", + "on_context_limit": "validate", + "note": "Optional review-approach benefit assessment pass. Non-fatal: routes to validate on both success and failure. Skipped when planner_assess_review_approach ingredient is false (default).\n" + }, + "validate": { + "tool": "run_python", + "with": { + "callable": "autoskillit.planner.validate_plan", + "output_dir": "${{ context.planner_dir }}" + }, + "capture": { + "verdict": "${{ result.verdict }}" + }, + "on_success": "check_verdict", + "on_failure": "escalate_stop" + }, + "check_verdict": { + "action": "route", + "on_result": [ + { + "when": "${{ context.verdict }} == fail", + "route": "refine" + }, + { + "route": "compile" + } + ] + }, + "refine": { + "tool": "run_skill", + "model": "", + "with": { + "skill_command": "/autoskillit:planner-refine ${{ context.planner_dir }}/validation.json ${{ context.planner_dir }}\n", + "cwd": "${{ inputs.source_dir }}", + "output_dir": "${{ context.planner_dir }}" + }, + "retries": 2, + "on_success": "validate", + "on_exhausted": "escalate_stop", + "on_failure": "escalate_stop" + }, + "compile": { + "tool": "run_python", + "with": { + "callable": "autoskillit.planner.compile_plan", + "output_dir": "${{ context.planner_dir }}", + "task_file_path": "${{ context.task_file_path }}", + "task_label": "${{ context.task_label }}", + "source_dir": "${{ inputs.source_dir }}" + }, + "on_success": "done", + "on_failure": "escalate_stop" + }, + "done": { + "action": "stop", + "message": "Planning complete. Plan artifacts written to ${{ context.planner_dir }}/. Use compile output to create GitHub issues and milestones.\n" + }, + "escalate_stop": { + "action": "stop", + "message": "Planner pipeline failed — human intervention needed. Review the latest step output for details." + } + } +} diff --git a/src/autoskillit/recipes/planner.yaml b/src/autoskillit/recipes/planner.yaml new file mode 100644 index 0000000000..be684e8d32 --- /dev/null +++ b/src/autoskillit/recipes/planner.yaml @@ -0,0 +1,451 @@ +name: planner +recipe_version: "2" +description: | + Progressive decomposition pipeline with mixed dispatch. Analyzes a codebase, + generates phases, then elaborates each tier (phases, assignments, work packages). + Phases and assignments elaborate in parallel; work packages elaborate sequentially + (one phase at a time) to smooth token spend rate. Each tier follows the pattern: + elaborate → merge → refine. Reconciles dependencies, validates the dependency + graph, and compiles a manifest ready for issue creation. +summary: "init > analyze > [domain?] > gen-phases > parallel-phases > parallel-assignments > sequential-wps > reconcile > validate/refine > compile" + +requires_packs: [kitchen-core] +requires_features: [planner] +categories: [orchestration-family] + +kitchen_rules: + - Never invoke native Claude Code tools (Read, Grep, Glob, Edit, Write, Bash, Agent, + WebFetch, WebSearch, NotebookEdit) from the orchestrator. All codebase interaction + runs inside headless skill sessions. + - Route to on_failure or escalate_stop rather than attempting inline recovery. + - The refine/validate cycle is bounded by retries. If refine is exhausted, halt. + - "PARALLEL DISPATCH: When a step note says 'PARALLEL DISPATCH', iterate the + comma-separated list in the referenced context variable. For each item, substitute + it into the skill_command template placeholder and issue one run_skill call. Issue + ALL calls in a single assistant message so they execute concurrently. Accumulate + each result via capture_list." + - "SEQUENTIAL DISPATCH: When a step note says 'SEQUENTIAL DISPATCH', iterate the + comma-separated list in the referenced context variable. For each item, substitute + it into the skill_command template placeholder and issue one run_skill call. Wait + for the call to complete before issuing the next. Accumulate each result via + capture_list." + +ingredients: + task: + description: "Task description to decompose into a plan. Accepts either an absolute file path or inline text." + required: true + source_dir: + description: "Absolute path to the repository root to analyze." + required: true + planner_assess_review_approach: + description: > + When true, run planner-assess-review-approach after reconcile_deps to annotate + WPs that would benefit from a review-approach research pass before implementation. + required: false + default: "false" + hidden: false + +steps: + + init: + tool: run_python + with: + callable: "autoskillit.planner.create_run_dir" + args: + temp_dir: "{{AUTOSKILLIT_TEMP}}" + capture: + planner_dir: "${{ result.planner_dir }}" + on_success: resolve_task + on_failure: escalate_stop + + resolve_task: + tool: run_python + with: + callable: "autoskillit.planner.resolve_task_input" + task: "${{ inputs.task }}" + planner_dir: "${{ context.planner_dir }}" + capture: + task_file_path: "${{ result.task_file_path }}" + task_label: "${{ result.task_label }}" + on_success: analyze + on_failure: escalate_stop + + analyze: + tool: run_skill + model: "" + with: + skill_command: "/autoskillit:planner-analyze ${{ context.planner_dir }}" + cwd: "${{ inputs.source_dir }}" + output_dir: "${{ context.planner_dir }}" + on_success: extract_domain + on_failure: escalate_stop + + extract_domain: + tool: run_skill + model: "" + with: + skill_command: > + /autoskillit:planner-extract-domain + ${{ context.planner_dir }}/analysis.json + "${{ context.task_file_path }}" + cwd: "${{ inputs.source_dir }}" + output_dir: "${{ context.planner_dir }}" + on_success: generate_phases + on_failure: generate_phases + + generate_phases: + tool: run_skill + model: "" + with: + skill_command: > + /autoskillit:planner-generate-phases + ${{ context.planner_dir }}/analysis.json + ${{ context.planner_dir }}/domain_knowledge.md + "${{ context.task_file_path }}" + cwd: "${{ inputs.source_dir }}" + output_dir: "${{ context.planner_dir }}" + capture: + phase_manifest_path: "${{ result.phase_manifest_path }}" + on_success: build_plan_snapshot + on_failure: escalate_stop + + # --- PHASE TIER --- + + build_plan_snapshot: + tool: run_python + with: + callable: "autoskillit.planner.merge.build_plan_snapshot" + phases_dir: "${{ context.planner_dir }}/phases" + phase_manifest_path: "${{ context.phase_manifest_path }}" + output_path: "${{ context.planner_dir }}/plan_snapshot.json" + task_file_path: "${{ context.task_file_path }}" + source_dir: "${{ inputs.source_dir }}" + capture: + plan_snapshot_path: "${{ result.snapshot_path }}" + phase_ids: "${{ result.phase_ids }}" + on_success: elaborate_phases + on_failure: escalate_stop + + elaborate_phases: + tool: run_skill + model: "" + with: + skill_command: "/autoskillit:planner-elaborate-phase ${{ context.plan_snapshot_path }} {phase_id} ${{ context.planner_dir }}/phases" + cwd: "${{ inputs.source_dir }}" + output_dir: "${{ context.planner_dir }}" + dispatch_items: "${{ context.phase_ids }}" + capture_list: + elab_result_path: "${{ result.elab_result_path }}" + on_success: merge_phases + on_failure: escalate_stop + note: > + PARALLEL DISPATCH: context.phase_ids contains comma-separated phase IDs + (e.g. "P1,P2,P3"). For each phase_id, substitute it into the {phase_id} + placeholder in skill_command and issue one run_skill call. Run ALL + elaborations in parallel (all run_skill calls in a single message). + Accumulate each elab_result_path value via capture_list. + + merge_phases: + tool: run_python + with: + callable: "autoskillit.planner.merge.merge_tier_results" + results_dir: "${{ context.planner_dir }}/phases" + output_path: "${{ context.planner_dir }}/combined_plan.json" + key: "phases" + task_file_path: "${{ context.task_file_path }}" + source_dir: "${{ inputs.source_dir }}" + capture: + combined_phases_path: "${{ result.merged_path }}" + on_success: refine_phases + on_failure: escalate_stop + + refine_phases: + tool: run_skill + model: "" + with: + skill_command: "/autoskillit:planner-refine-phases ${{ context.combined_phases_path }} ${{ context.planner_dir }}" + cwd: "${{ inputs.source_dir }}" + output_dir: "${{ context.planner_dir }}" + capture: + refined_plan_path: "${{ result.refined_plan_path }}" + on_success: expand_assignments + on_failure: escalate_stop + + # --- ASSIGNMENT TIER --- + + expand_assignments: + tool: run_python + with: + callable: "autoskillit.planner.expand_assignments" + refined_plan_path: "${{ context.refined_plan_path }}" + output_dir: "${{ context.planner_dir }}" + task_file_path: "${{ context.task_file_path }}" + capture: + assignment_context_paths: "${{ result.context_paths }}" + on_success: elaborate_assignments + on_failure: escalate_stop + + elaborate_assignments: + tool: run_skill + model: "" + with: + skill_command: "/autoskillit:planner-elaborate-assignments {context_path} ${{ context.planner_dir }}" + cwd: "${{ inputs.source_dir }}" + output_dir: "${{ context.planner_dir }}/assignments" + dispatch_items: "${{ context.assignment_context_paths }}" + capture_list: + phase_assignments_result_dir: "${{ result.phase_assignments_result_dir }}" + on_success: merge_assignments + on_failure: escalate_stop + note: > + PARALLEL DISPATCH: context.assignment_context_paths contains comma-separated + paths to per-phase context files. For each context_path, substitute it into + the {context_path} placeholder in skill_command and issue one run_skill call. + Run ALL elaborations in parallel. Accumulate results via capture_list. + + merge_assignments: + tool: run_python + with: + callable: "autoskillit.planner.merge.merge_tier_results" + results_dir: "${{ context.planner_dir }}/assignments" + output_path: "${{ context.planner_dir }}/combined_assignments.json" + key: "assignments" + task_file_path: "${{ context.task_file_path }}" + source_dir: "${{ inputs.source_dir }}" + capture: + refine_context_paths: "${{ result.refine_context_paths }}" + on_success: refine_assignments + on_failure: escalate_stop + + refine_assignments: + tool: run_skill + model: "" + with: + skill_command: "/autoskillit:planner-refine-assignments {context_path} ${{ context.refined_plan_path }} ${{ context.planner_dir }}" + cwd: "${{ inputs.source_dir }}" + output_dir: "${{ context.planner_dir }}/refine_contexts" + dispatch_items: "${{ context.refine_context_paths }}" + capture_list: + phase_refined_path: "${{ result.phase_refined_path }}" + on_success: merge_refined_assignments + on_failure: escalate_stop + note: > + PARALLEL DISPATCH: context.refine_context_paths contains comma-separated + paths to per-phase context files produced by merge_tier_results. + For each context_path, substitute it into the {context_path} placeholder + and issue one run_skill call. Run ALL refinements in parallel. + Accumulate results via capture_list. + + merge_refined_assignments: + tool: run_python + with: + callable: "autoskillit.planner.merge.merge_refined_assignments" + planner_dir: "${{ context.planner_dir }}" + capture: + refined_assignments_path: "${{ result.refined_assignments_path }}" + on_success: expand_wps + on_failure: escalate_stop + + # --- WORK PACKAGE TIER --- + + expand_wps: + tool: run_python + with: + callable: "autoskillit.planner.expand_wps" + refined_assignments_path: "${{ context.refined_assignments_path }}" + output_dir: "${{ context.planner_dir }}" + task_file_path: "${{ context.task_file_path }}" + capture: + wp_context_paths: "${{ result.context_paths }}" + on_success: elaborate_wps + on_failure: escalate_stop + + elaborate_wps: + tool: run_skill + model: "" + with: + skill_command: "/autoskillit:planner-elaborate-wps {context_path} ${{ context.planner_dir }}" + cwd: "${{ inputs.source_dir }}" + output_dir: "${{ context.planner_dir }}/work_packages" + dispatch_items: "${{ context.wp_context_paths }}" + capture_list: + phase_wps_result_dir: "${{ result.phase_wps_result_dir }}" + on_success: finalize_wp_manifest + on_failure: escalate_stop + note: > + SEQUENTIAL DISPATCH: context.wp_context_paths contains comma-separated paths + to per-phase WP context files. For each context_path, substitute it into + the {context_path} placeholder in skill_command and issue one run_skill call. + Wait for each elaboration to complete before starting the next. Accumulate + results via capture_list. + + finalize_wp_manifest: + tool: run_python + with: + callable: "autoskillit.planner.finalize_wp_manifest" + work_packages_dir: "${{ context.planner_dir }}/work_packages" + output_dir: "${{ context.planner_dir }}" + on_success: merge_wps + on_failure: escalate_stop + + merge_wps: + tool: run_python + with: + callable: "autoskillit.planner.merge.merge_tier_results" + results_dir: "${{ context.planner_dir }}/work_packages" + output_path: "${{ context.planner_dir }}/combined_wps.json" + key: "work_packages" + task_file_path: "${{ context.task_file_path }}" + source_dir: "${{ inputs.source_dir }}" + capture: + combined_wps_path: "${{ result.merged_path }}" + on_success: refine_wps + on_failure: escalate_stop + + refine_wps: + tool: run_skill + model: "" + with: + skill_command: > + /autoskillit:planner-refine-wps + ${{ context.combined_wps_path }} + ${{ context.refined_plan_path }} + ${{ context.refined_assignments_path }} + ${{ context.planner_dir }} + cwd: "${{ inputs.source_dir }}" + output_dir: "${{ context.planner_dir }}" + capture: + refined_wps_path: "${{ result.refined_wps_path }}" + on_success: consolidate_wps_analyze + on_failure: escalate_stop + + consolidate_wps_analyze: + tool: run_skill + model: "" + with: + skill_command: > + /autoskillit:planner-consolidate-wps + ${{ context.refined_wps_path }} + ${{ context.planner_dir }} + cwd: "${{ inputs.source_dir }}" + output_dir: "${{ context.planner_dir }}" + on_success: consolidate_wps_merge + on_failure: consolidate_wps_merge + note: > + Analysis-only step: always routes to consolidate_wps_merge regardless of success + or failure. If the skill fails, the merge callable falls back to passing all WPs + through unchanged (no consolidation manifests = no-op merge). + + consolidate_wps_merge: + tool: run_python + with: + callable: autoskillit.planner.consolidate_wps + refined_wps_path: "${{ context.refined_wps_path }}" + planner_dir: "${{ context.planner_dir }}" + capture: + consolidated_wps_path: "${{ result.consolidated_wps_path }}" + on_success: validate_task_alignment + on_failure: escalate_stop + + validate_task_alignment: + tool: run_skill + model: "" + with: + skill_command: > + /autoskillit:planner-validate-task-alignment + ${{ context.consolidated_wps_path }} + ${{ context.refined_plan_path }} + ${{ context.planner_dir }} + cwd: "${{ inputs.source_dir }}" + output_dir: "${{ context.planner_dir }}" + capture: + alignment_findings_path: "${{ result.alignment_findings_path }}" + on_success: reconcile_deps + on_failure: reconcile_deps + + # --- FINALIZATION --- + + reconcile_deps: + tool: run_skill + model: "" + with: + skill_command: "/autoskillit:planner-reconcile-deps ${{ context.planner_dir }}" + cwd: "${{ inputs.source_dir }}" + output_dir: "${{ context.planner_dir }}" + consolidated_wps_path: "${{ context.consolidated_wps_path }}" + on_success: assess_review_approach_step + on_failure: escalate_stop + + assess_review_approach_step: + tool: run_skill + model: "" + optional: true + skip_when_false: "inputs.planner_assess_review_approach" + with: + skill_command: > + /autoskillit:planner-assess-review-approach + ${{ context.consolidated_wps_path }} + ${{ context.planner_dir }} + cwd: "${{ inputs.source_dir }}" + output_dir: "${{ context.planner_dir }}" + capture: + review_approach_assessment_path: "${{ result.review_approach_assessment_path }}" + on_success: validate + on_failure: validate + on_context_limit: validate + note: > + Optional review-approach benefit assessment pass. Non-fatal: routes to validate + on both success and failure. Skipped when planner_assess_review_approach ingredient is false (default). + + validate: + tool: run_python + with: + callable: "autoskillit.planner.validate_plan" + output_dir: "${{ context.planner_dir }}" + capture: + verdict: "${{ result.verdict }}" + on_success: check_verdict + on_failure: escalate_stop + + check_verdict: + action: route + on_result: + - when: "${{ context.verdict }} == fail" + route: refine + - route: compile + + refine: + tool: run_skill + model: "" + with: + skill_command: > + /autoskillit:planner-refine + ${{ context.planner_dir }}/validation.json + ${{ context.planner_dir }} + cwd: "${{ inputs.source_dir }}" + output_dir: "${{ context.planner_dir }}" + retries: 2 + on_success: validate + on_exhausted: escalate_stop + on_failure: escalate_stop + + compile: + tool: run_python + with: + callable: "autoskillit.planner.compile_plan" + output_dir: "${{ context.planner_dir }}" + task_file_path: "${{ context.task_file_path }}" + task_label: "${{ context.task_label }}" + source_dir: "${{ inputs.source_dir }}" + on_success: done + on_failure: escalate_stop + + done: + action: stop + message: > + Planning complete. Plan artifacts written to ${{ context.planner_dir }}/. + Use compile output to create GitHub issues and milestones. + + escalate_stop: + action: stop + message: "Planner pipeline failed — human intervention needed. Review the latest step output for details." diff --git a/src/autoskillit/recipes/promote-to-main-wrapper.json b/src/autoskillit/recipes/promote-to-main-wrapper.json new file mode 100644 index 0000000000..eeb060d80e --- /dev/null +++ b/src/autoskillit/recipes/promote-to-main-wrapper.json @@ -0,0 +1,79 @@ +{ + "name": "promote-to-main-wrapper", + "description": "Thin wrapper that invokes the promote-to-main skill via run_skill, enabling campaign dispatch of integration branch promotions as L2 food trucks.", + "kind": "food-truck", + "recipe_version": "1.0.0", + "requires_packs": [ + "github" + ], + "kitchen_rules": [ + "NEVER use native Claude Code tools (Read, Grep, Glob, Edit, Write, Bash, Agent, WebFetch, WebSearch, NotebookEdit) from the orchestrator. All work is delegated through run_skill.", + "Route to on_failure — never investigate or fix directly from the orchestrator." + ], + "ingredients": { + "batch_branch": { + "description": "Branch to promote (must exist ahead of base_branch). Must be a valid git branch name with no spaces or shell metacharacters.", + "default": "develop" + }, + "base_branch": { + "description": "Target branch for the promotion PR. Must be a valid git branch name with no spaces or shell metacharacters.", + "default": "main" + }, + "source_dir": { + "description": "Absolute path to the source repository. Required — no default prevents silent cwd inheritance from the MCP server when an empty string is passed.", + "required": true + } + }, + "steps": { + "promote": { + "tool": "run_skill", + "model": "", + "with": { + "skill_command": "/autoskillit:promote-to-main ${{ inputs.batch_branch }} ${{ inputs.base_branch }}", + "cwd": "${{ inputs.source_dir }}", + "step_name": "promote" + }, + "capture": { + "pr_url": "${{ result.pr_url }}", + "verdict": "${{ result.verdict }}", + "category_summary": "${{ result.category_summary }}" + }, + "on_result": [ + { + "when": "${{ result.verdict }} == 'preflight_failed'", + "route": "escalate_stop" + }, + { + "when": "${{ result.verdict }} == 'created'", + "route": "emit_result" + }, + { + "when": "${{ result.verdict }} == 'dry_run'", + "route": "emit_result" + }, + { + "route": "emit_result" + } + ], + "on_context_limit": "escalate_stop", + "on_failure": "escalate_stop" + }, + "emit_result": { + "tool": "run_cmd", + "with": { + "cmd": "echo 'pr_url = ${{ context.pr_url }}' && echo 'verdict = ${{ context.verdict }}' && echo 'category_summary = ${{ context.category_summary }}'", + "step_name": "emit_result" + }, + "on_success": "done", + "on_failure": "done" + }, + "done": { + "action": "stop", + "message": "Promotion complete. Emit your L3 sentinel JSON block now with success=true, pr_url, verdict, and category_summary." + }, + "escalate_stop": { + "action": "stop", + "message": "Promotion failed — escalating for manual review." + } + } +} diff --git a/src/autoskillit/recipes/promote-to-main-wrapper.yaml b/src/autoskillit/recipes/promote-to-main-wrapper.yaml new file mode 100644 index 0000000000..6c93d0d83e --- /dev/null +++ b/src/autoskillit/recipes/promote-to-main-wrapper.yaml @@ -0,0 +1,69 @@ +name: promote-to-main-wrapper +description: >- + Thin wrapper that invokes the promote-to-main skill via run_skill, enabling + campaign dispatch of integration branch promotions as L2 food trucks. +kind: food-truck +recipe_version: "1.0.0" + +requires_packs: + - github + +kitchen_rules: + - "NEVER use native Claude Code tools (Read, Grep, Glob, Edit, Write, + Bash, Agent, WebFetch, WebSearch, NotebookEdit) from the + orchestrator. All work is delegated through run_skill." + - "Route to on_failure — never investigate or fix directly from the orchestrator." + +ingredients: + batch_branch: + description: "Branch to promote (must exist ahead of base_branch). Must be a valid git branch name with no spaces or shell metacharacters." + default: "develop" + base_branch: + description: "Target branch for the promotion PR. Must be a valid git branch name with no spaces or shell metacharacters." + default: "main" + source_dir: + description: >- + Absolute path to the source repository. Required — no default prevents + silent cwd inheritance from the MCP server when an empty string is passed. + required: true + +steps: + promote: + tool: run_skill + model: "" + with: + skill_command: "/autoskillit:promote-to-main ${{ inputs.batch_branch }} ${{ inputs.base_branch }}" + cwd: "${{ inputs.source_dir }}" + step_name: "promote" + capture: + pr_url: "${{ result.pr_url }}" + verdict: "${{ result.verdict }}" + category_summary: "${{ result.category_summary }}" + on_result: + - when: "${{ result.verdict }} == 'preflight_failed'" + route: escalate_stop + - when: "${{ result.verdict }} == 'created'" + route: emit_result + - when: "${{ result.verdict }} == 'dry_run'" + route: emit_result + - route: emit_result + on_context_limit: escalate_stop + on_failure: escalate_stop + + emit_result: + tool: run_cmd + with: + cmd: "echo 'pr_url = ${{ context.pr_url }}' && echo 'verdict = ${{ context.verdict }}' && echo 'category_summary = ${{ context.category_summary }}'" + step_name: "emit_result" + on_success: done + on_failure: done + + done: + action: stop + message: >- + Promotion complete. Emit your L3 sentinel JSON block now with + success=true, pr_url, verdict, and category_summary. + + escalate_stop: + action: stop + message: "Promotion failed — escalating for manual review." diff --git a/src/autoskillit/recipes/remediation.json b/src/autoskillit/recipes/remediation.json new file mode 100644 index 0000000000..c2574114ff --- /dev/null +++ b/src/autoskillit/recipes/remediation.json @@ -0,0 +1,1817 @@ +{ + "name": "remediation", + "description": "Investigate deeply, plan architectural fix, implement and open a PR.", + "summary": "bootstrap_clone > (claim_and_resolve?) > (create_and_publish?) > investigate > rectify > dry-walkthrough > implement > test > merge (per plan part) > (audit-impl?) > (open_pr?) > push > cleanup", + "recipe_version": "1.0.0", + "requires_packs": [ + "github", + "ci", + "clone", + "telemetry" + ], + "kitchen_rules": [ + "NEVER use native Claude Code tools (Read, Grep, Glob, Edit, Write, Bash, Agent, WebFetch, WebSearch, NotebookEdit) from the orchestrator. All code changes and investigation happen through headless sessions via run_skill.", + "Route to on_failure when a step fails — the downstream skill (e.g., resolve-failures) has diagnostic access that the orchestrator does not. Do not investigate or attempt to fix failures directly.", + "SEQUENTIAL EXECUTION: complete full cycle per part before advancing. For each plan_part, run the full cycle (dry_walkthrough → implement → test → merge) before starting the next part.", + "By default (open_pr=true), a feature branch is created with a unique name derived from inputs.run_name and context.issue_number (e.g. investigate/42) or a date suffix (e.g. investigate/20260304) when no issue is available. All worktree merges target the feature branch (context.merge_target), not base_branch directly. The push step publishes the feature branch, then prepare_pr → run_arch_lenses → compose_pr opens a PR to base_branch. When open_pr=false, merges target base_branch directly and prepare_pr is skipped.", + "When arch_lenses=true (and open_pr=true), prepare_pr routes through run_arch_lenses before compose_pr, adding architecture lens diagrams to the PR. By default arch_lenses is off and prepare_pr routes directly to compose_pr.", + "SOURCE ISOLATION: After bootstrap_clone returns, the source_dir is strictly off-limits. Never run any command in source_dir — no git checkout, git fetch, git reset, git pull, run_cmd, run_skill, or any other operation. All work — skill invocations, git operations, file reads — happens exclusively in the clone (work_dir). source_dir is used ONLY to read the remote URL inside push_to_remote." + ], + "ingredients": { + "task": { + "description": "Bug, error, or question to investigate", + "required": true + }, + "source_dir": { + "description": "Source repository", + "required": false, + "default": "" + }, + "run_name": { + "description": "Run name prefix", + "default": "remediation" + }, + "base_branch": { + "description": "Merge target", + "default": "" + }, + "audit": { + "description": "Post-merge quality gate", + "default": "false" + }, + "review_approach": { + "description": "Research approaches first", + "default": "false" + }, + "open_pr": { + "description": "PR instead of direct merge", + "default": "true" + }, + "arch_lenses": { + "description": "Generate architecture lens diagrams for the PR", + "default": "false" + }, + "auto_merge": { + "description": "Automatically merge the PR after required checks pass — via merge queue when available, direct merge otherwise", + "default": "true" + }, + "review_max_retries": { + "description": "Maximum number of review-resolve retry cycles before proceeding to CI", + "default": "3" + }, + "local_review_rounds": { + "description": "Number of review-resolve cycles to run locally before switching to GitHub-backed review. 0 disables local mode entirely.", + "type": "integer", + "default": "3" + }, + "issue_url": { + "description": "GitHub issue to close on merge", + "required": false + }, + "upfront_claimed": { + "description": "Set to 'true' when process-issues has already claimed this issue upfront. Allows claim_issue defense step to pass through without aborting.", + "default": "false", + "hidden": true + }, + "run_mode": { + "description": "Execution mode when processing multiple issues. Options: sequential (default, one issue at a time) or parallel (all issues simultaneously with wavefront step scheduling — fast steps for all pipelines before any slow step).", + "default": "sequential", + "hidden": true + }, + "max_parallel": { + "description": "Maximum number of issues to run in parallel within a single execution-map group. Groups exceeding this cap are split into sequential sub-groups. Passed to build-execution-map. Must be a positive integer (expressed as a string).", + "default": "6", + "hidden": true + }, + "depth": { + "description": "Investigation depth mode", + "default": "standard", + "hidden": true + } + }, + "steps": { + "clone": { + "tool": "bootstrap_clone", + "with": { + "source_dir": "${{ inputs.source_dir }}", + "run_name": "${{ inputs.run_name }}", + "base_branch": "${{ inputs.base_branch }}", + "step_name": "bootstrap_clone" + }, + "capture": { + "work_dir": "${{ result.work_dir }}", + "remote_url": "${{ result.remote_url }}", + "merge_target": "${{ result.merge_target }}" + }, + "on_success": "claim_and_resolve", + "on_failure": "escalate_stop", + "note": "Clones source_dir and captures merge_target in one turn. Returns work_dir, remote_url, and merge_target (set to base_branch as fallback; overridden by create_and_publish when open_pr=true). On failure, escalate_stop immediately (no clone to remove).\n" + }, + "claim_and_resolve": { + "tool": "claim_and_resolve_issue", + "optional": true, + "skip_when_false": "inputs.issue_url", + "with": { + "issue_url": "${{ inputs.issue_url }}", + "allow_reentry": "${{ inputs.upfront_claimed }}" + }, + "capture": { + "issue_number": "${{ result.issue_number }}", + "issue_title": "${{ result.issue_title }}", + "issue_slug": "${{ result.issue_slug }}", + "claimed": "${{ result.claimed }}" + }, + "on_result": [ + { + "when": "${{ result.claimed }} == true", + "route": "create_and_publish" + }, + { + "route": "escalate_stop" + } + ], + "on_failure": "escalate_stop", + "note": "Fetches issue title/slug and claims the issue in one turn. Returns issue_number, issue_title, issue_slug, and claimed boolean. When claimed=false (another session owns it), routes to escalate_stop. issue_slug and issue_number are passed to create_and_publish for branch naming.\n" + }, + "create_and_publish": { + "tool": "create_and_publish_branch", + "optional": true, + "skip_when_false": "inputs.open_pr", + "with": { + "issue_slug": "${{ context.issue_slug }}", + "run_name": "${{ inputs.run_name }}", + "issue_number": "${{ context.issue_number }}", + "work_dir": "${{ context.work_dir }}", + "remote_url": "${{ context.remote_url }}", + "step_name": "create_and_publish_branch" + }, + "capture": { + "merge_target": "${{ result.merge_target }}" + }, + "on_success": "investigate", + "on_failure": "release_issue_failure", + "note": "Computes branch name, creates it (with collision suffix if needed), and pushes to origin in one turn. Overrides merge_target with the feature branch name. merge_worktree requires refs/remotes/origin/{branch} to exist after fetch. When open_pr=false, this step is skipped and merge_target remains as base_branch.\n" + }, + "investigate": { + "tool": "run_skill", + "model": "${{ 'opus[1m]' if inputs.depth == 'deep' else 'sonnet' }}", + "with": { + "skill_command": "/autoskillit:investigate ${{ '--depth deep ' if inputs.depth == 'deep' else '' }}${{ inputs.task }}", + "cwd": "${{ context.work_dir }}", + "step_name": "investigate", + "issue_url": "${{ inputs.issue_url }}", + "issue_title": "${{ context.issue_title }}" + }, + "capture": { + "investigation_path": "${{ result.investigation_path }}" + }, + "on_success": "rectify", + "on_failure": "release_issue_failure", + "on_context_limit": "release_issue_failure", + "note": "ISSUE CONTEXT: If inputs.issue_url is a non-empty string, append it to the skill_command so the investigate skill can detect and fetch the issue itself: \"/autoskillit:investigate {task}\\n\\nGitHub Issue: {inputs.issue_url}\" The skill detects the URL pattern and calls fetch_github_issue internally. DEPTH: When inputs.depth == 'deep', passes --depth deep flag and sets model to opus[1m] for the main skill session. Subagents within the skill always use model: sonnet regardless of depth setting.\n" + }, + "rectify": { + "tool": "run_skill", + "model": "", + "with": { + "skill_command": "/autoskillit:rectify ${{ context.investigation_path }}", + "cwd": "${{ context.work_dir }}", + "step_name": "rectify" + }, + "capture": { + "plan_path": "${{ result.plan_path }}" + }, + "capture_list": { + "plan_parts": "${{ result.plan_parts }}" + }, + "on_success": "review", + "on_failure": "release_issue_failure", + "note": "Glob plan_dir for *_part_*.md or single plan file. Sort alphabetically into plan_parts[].\n" + }, + "review": { + "tool": "run_skill", + "model": "", + "with": { + "skill_command": "/autoskillit:review-approach ${{ context.plan_path }}", + "cwd": "${{ context.work_dir }}", + "step_name": "review" + }, + "capture": { + "review_path": "${{ result.review_path }}" + }, + "retries": 1, + "on_success": "dry_walkthrough", + "on_failure": "release_issue_failure", + "on_context_limit": "dry_walkthrough", + "skip_when_false": "inputs.review_approach", + "note": "Only execute this step if review_approach input is 'true'. Otherwise skip directly to dry_walkthrough." + }, + "dry_walkthrough": { + "tool": "run_skill", + "model": "", + "with": { + "skill_command": "/autoskillit:dry-walkthrough ${{ context.plan_path }}", + "cwd": "${{ context.work_dir }}", + "review_path": "${{ context.review_path }}", + "step_name": "dry_walkthrough" + }, + "on_success": "implement", + "on_failure": "release_issue_failure", + "retries": 3, + "on_exhausted": "release_issue_failure", + "note": "Dry walkthrough retries on failure. It never routes back to a previous step." + }, + "implement": { + "tool": "run_skill", + "model": "", + "stale_threshold": 2400, + "with": { + "skill_command": "/autoskillit:implement-worktree-no-merge ${{ context.plan_path }}", + "cwd": "${{ context.work_dir }}", + "step_name": "implement" + }, + "capture": { + "implementation_ref": "${{ result.worktree_path }}", + "worktree_path": "${{ result.worktree_path }}", + "branch_name": "${{ result.branch_name }}" + }, + "retries": 0, + "on_context_limit": "retry_worktree", + "on_success": "test", + "on_failure": "release_issue_failure" + }, + "retry_worktree": { + "tool": "run_skill", + "model": "", + "stale_threshold": 2400, + "with": { + "skill_command": "/autoskillit:retry-worktree ${{ context.plan_path }} ${{ context.worktree_path }}", + "cwd": "${{ context.worktree_path }}", + "step_name": "retry_worktree" + }, + "capture": { + "implementation_ref": "${{ result.worktree_path }}", + "worktree_path": "${{ result.worktree_path }}", + "branch_name": "${{ result.branch_name }}", + "phases_implemented": "${{ result.phases_implemented }}" + }, + "on_result": [ + { + "when": "${{ result.phases_implemented }} > 0", + "route": "test" + }, + { + "route": "test" + } + ], + "on_exhausted": "release_issue_failure", + "on_failure": "release_issue_failure" + }, + "test": { + "tool": "test_check", + "with": { + "worktree_path": "${{ context.implementation_ref }}" + }, + "on_success": "check_merge_fix_loop", + "on_failure": "assess" + }, + "check_merge_fix_loop": { + "tool": "run_python", + "with": { + "callable": "autoskillit.smoke_utils.check_loop_iteration", + "current_iteration": "${{ context.merge_fix_count }}", + "max_iterations": "3" + }, + "capture": { + "merge_fix_count": "${{ result.next_iteration }}" + }, + "on_result": [ + { + "when": "${{ result.max_exceeded }} == true", + "route": "release_issue_failure" + }, + { + "route": "audit_impl" + } + ], + "on_failure": "release_issue_failure", + "optional_context_refs": [ + "merge_fix_count" + ], + "note": "Iteration guard for the merge→assess→test cycle. Caps the loop at 3 iterations (enough for legitimate flake recovery but not enough to burn 30+ minutes on structurally unresolvable conflicts).\n" + }, + "assess": { + "tool": "run_skill", + "model": "", + "with": { + "skill_command": "/autoskillit:resolve-failures ${{ context.worktree_path }} ${{ context.plan_path }} ${{ inputs.base_branch }}", + "cwd": "${{ context.worktree_path }}", + "step_name": "assess" + }, + "capture": { + "verdict": "${{ result.verdict }}", + "fixes_applied": "${{ result.fixes_applied }}" + }, + "on_result": [ + { + "when": "${{ result.verdict }} == 'real_fix'", + "route": "test" + }, + { + "when": "${{ result.verdict }} == 'already_green'", + "route": "test" + }, + { + "when": "${{ result.verdict }} == 'flake_suspected'", + "route": "test" + }, + { + "when": "${{ result.verdict }} == 'ci_only_failure'", + "route": "release_issue_failure" + } + ], + "on_context_limit": "test", + "on_failure": "release_issue_failure", + "on_exhausted": "release_issue_failure" + }, + "rebase_conflict_fix": { + "tool": "run_skill", + "model": "", + "with": { + "skill_command": "/autoskillit:resolve-merge-conflicts ${{ context.worktree_path }} ${{ context.plan_path }} ${{ inputs.base_branch }}", + "cwd": "${{ context.worktree_path }}", + "step_name": "rebase_conflict_fix" + }, + "capture": { + "conflict_escalation_required": "${{ result.escalation_required }}" + }, + "on_result": [ + { + "when": "${{ result.escalation_required }} == true", + "route": "release_issue_failure" + }, + { + "route": "test" + } + ], + "on_failure": "release_issue_failure", + "on_context_limit": "release_issue_failure", + "retries": 1, + "on_exhausted": "release_issue_failure" + }, + "audit_impl": { + "tool": "run_skill", + "model": "", + "with": { + "skill_command": "/autoskillit:audit-impl ${{ context.plan_path }} ${{ context.branch_name }} ${{ inputs.base_branch }}", + "cwd": "${{ context.work_dir }}", + "step_name": "audit_impl" + }, + "capture": { + "remediation_path": "${{ result.remediation_path }}" + }, + "on_result": [ + { + "when": "${{ result.verdict }} == GO", + "route": "commit_guard" + }, + { + "when": "result.error", + "route": "escalate_stop" + }, + { + "route": "remediate" + } + ], + "on_failure": "escalate_stop", + "on_context_limit": "escalate_stop", + "optional": true, + "skip_when_false": "inputs.audit", + "note": "Only run if inputs.audit = true. If inputs.audit = false, skip directly to commit_guard." + }, + "remediate": { + "action": "route", + "with": { + "remediation_path": "${{ context.remediation_path }}" + }, + "on_success": "make_plan" + }, + "make_plan": { + "tool": "run_skill", + "model": "", + "with": { + "skill_command": "/autoskillit:make-plan ${{ context.remediation_path }}", + "task": "${{ inputs.task }}", + "cwd": "${{ context.work_dir }}", + "step_name": "make_plan" + }, + "capture": { + "plan_path": "${{ result.plan_path }}" + }, + "capture_list": { + "plan_parts": "${{ result.plan_parts }}" + }, + "on_success": "dry_walkthrough", + "on_failure": "release_issue_failure" + }, + "commit_guard": { + "tool": "run_python", + "with": { + "callable": "autoskillit.recipe._cmd_rpc.commit_guard", + "worktree_path": "${{ context.implementation_ref }}" + }, + "on_success": "merge", + "on_failure": "merge", + "note": "Belt-and-suspenders guard: auto-commits any uncommitted changes before merge. Catches dirty state from context-exhausted skills, cascading pre-commit auto-fixes, or any other source. on_failure routes to merge (dirty-tree gate will catch real issues).\n" + }, + "merge": { + "tool": "merge_worktree", + "with": { + "worktree_path": "${{ context.implementation_ref }}", + "base_branch": "${{ context.merge_target }}" + }, + "capture": { + "cleanup_succeeded": "${{ result.cleanup_succeeded }}", + "worktree_path": "${{ result.worktree_path }}" + }, + "on_result": [ + { + "when": "result.failed_step == 'dirty_tree'", + "route": "assess" + }, + { + "when": "result.failed_step == 'test_gate'", + "route": "assess" + }, + { + "when": "result.failed_step == 'post_rebase_test_gate'", + "route": "assess" + }, + { + "when": "result.failed_step == 'rebase'", + "route": "rebase_conflict_fix" + }, + { + "when": "result.error", + "route": "release_issue_failure" + }, + { + "route": "next_or_done" + } + ], + "on_failure": "release_issue_failure", + "note": "After merge, route to next_or_done to process remaining parts before pushing.\n" + }, + "next_or_done": { + "action": "route", + "on_result": [ + { + "when": "${{ result.next }} == more_parts", + "route": "dry_walkthrough" + }, + { + "route": "push" + } + ], + "note": "Routing step — no tool call. The agent evaluates what comes next: 1. If more plan_parts remain for the current plan → go to dry_walkthrough with next part. 2. If all plan_parts are complete → go to push. Unlike implementation.yaml (which routes done → audit_impl → push), remediation runs audit_impl pre-merge (GO → commit_guard), so the fallthrough here goes directly to push.\n" + }, + "push": { + "tool": "push_to_remote", + "with": { + "clone_path": "${{ context.work_dir }}", + "remote_url": "${{ context.remote_url }}", + "branch": "${{ context.merge_target }}" + }, + "on_success": "prepare_pr", + "on_failure": "release_issue_failure", + "note": "Runs once at the end. Pushes merge_target branch from the clone to the upstream remote. By default (open_pr=true), this pushes the feature branch; when open_pr=false, pushes base_branch. Uses the pre-resolved remote_url captured at clone time, preserving clone isolation. On success, routes to prepare_pr (which opens the PR by default; skipped when open_pr=false). On failure, release_issue_failure registers the clone and escalates.\n" + }, + "prepare_pr": { + "tool": "run_skill", + "model": "", + "retries": 1, + "with": { + "skill_command": "/autoskillit:prepare-pr ${{ context.plan_path }} ${{ inputs.run_name }} ${{ inputs.base_branch }}", + "cwd": "${{ context.work_dir }}", + "step_name": "prepare_pr", + "issue_number": "${{ context.issue_number }}" + }, + "capture": { + "prep_path": "${{ result.prep_path }}", + "selected_lenses": "${{ result.selected_lenses }}", + "lens_context_paths": "${{ result.lens_context_paths }}" + }, + "on_result": [ + { + "when": "${{ result.prep_path }} and ${{ inputs.arch_lenses }} == 'true'", + "route": "run_arch_lenses" + }, + { + "when": "${{ result.prep_path }}", + "route": "compose_pr" + }, + { + "route": "release_issue_failure" + } + ], + "on_failure": "release_issue_failure", + "on_context_limit": "prepare_pr", + "on_exhausted": "release_issue_failure", + "optional": true, + "skip_when_false": "inputs.open_pr", + "note": "Reads plan, runs git diff, classifies changed files, selects 1–3 arch-lens slugs, writes one PR context file per lens, and writes a PR prep file. Does NOT invoke arch-lens skills. If context.issue_number is a non-empty string, append it as the 4th positional argument so prepare-pr fetches requirements from the closing issue.\n" + }, + "run_arch_lenses": { + "tool": "run_skill", + "model": "", + "with": { + "skill_command": "/autoskillit:arch-lens-{slug} {context_path}", + "cwd": "${{ context.work_dir }}", + "step_name": "run_arch_lenses" + }, + "capture_list": { + "all_diagram_paths": "${{ result.diagram_path }}" + }, + "on_success": "compose_pr", + "on_failure": "compose_pr", + "on_context_limit": "compose_pr", + "optional": true, + "skip_when_false": "inputs.open_pr", + "note": "LENS ITERATION: context.selected_lenses contains comma-separated arch-lens slugs (e.g. \"module-dependency,process-flow\"). context.lens_context_paths contains the corresponding comma-separated context file paths (same order as slugs). For each lens slug, run:\n /autoskillit:arch-lens-{slug} {matching_context_path}\nRun all selected lenses in parallel (all run_skill calls in a single message). Accumulate each diagram_path value into context.all_diagram_paths via capture_list. If a lens fails, continue with remaining lenses — compose_pr handles empty lists gracefully by omitting the Architecture Impact section.\n" + }, + "compose_pr": { + "tool": "run_skill", + "model": "", + "with": { + "skill_command": "/autoskillit:compose-pr ${{ context.prep_path }} \"${{ context.all_diagram_paths }}\" ${{ context.work_dir }} ${{ inputs.base_branch }}", + "cwd": "${{ context.work_dir }}", + "step_name": "compose_pr", + "issue_number": "${{ context.issue_number }}" + }, + "capture": { + "pr_url": "${{ result.pr_url }}" + }, + "on_success": "extract_pr_number", + "on_failure": "release_issue_failure", + "on_context_limit": "release_issue_failure", + "optional": true, + "skip_when_false": "inputs.open_pr", + "note": "Reads the PR prep file, validates arch-lens diagrams (must contain ★ or ●), composes the PR body, and runs gh pr create. Degrades gracefully if gh is unavailable (emits pr_url=). If context.issue_number is set, append it as the 5th positional argument for Closes #N injection in the PR body.\n" + }, + "extract_pr_number": { + "tool": "run_cmd", + "with": { + "cmd": "gh pr view '${{ context.merge_target }}' --json number -q .number", + "cwd": "${{ context.work_dir }}", + "step_name": "extract_pr_number" + }, + "capture": { + "pr_number": "${{ result.stdout | trim }}" + }, + "on_success": "annotate_pr_diff", + "on_failure": "release_issue_failure", + "optional": true, + "skip_when_false": "inputs.open_pr", + "note": "Extracts the integer PR number from the feature branch opened by compose_pr. pr_number is required by the queue finale steps (enable_auto_merge, wait_for_queue).\n" + }, + "annotate_pr_diff": { + "tool": "run_python", + "with": { + "callable": "autoskillit.smoke_utils.annotate_pr_diff", + "pr_number": "${{ context.pr_number }}", + "cwd": "${{ context.work_dir }}", + "output_dir": "${{ context.work_dir }}/{{AUTOSKILLIT_TEMP}}/review-pr", + "base_branch": "${{ inputs.base_branch }}", + "local_review_rounds": "${{ inputs.local_review_rounds }}", + "current_iteration": "${{ context.review_loop_count }}" + }, + "capture": { + "annotated_diff_path": "${{ result.annotated_diff_path }}", + "hunk_ranges_path": "${{ result.hunk_ranges_path }}", + "diff_metrics_path": "${{ result.diff_metrics_path }}", + "review_mode": "${{ result.review_mode }}" + }, + "on_success": "review_pr", + "on_failure": "review_pr", + "optional": true, + "skip_when_false": "inputs.open_pr", + "optional_context_refs": [ + "review_loop_count" + ], + "note": "Pre-computes annotated diff with [LNNN] markers and hunk ranges JSON for review-pr. On failure, falls through to review_pr which uses empty fallback.\n" + }, + "review_pr": { + "tool": "run_skill", + "model": "", + "with": { + "skill_command": "/autoskillit:review-pr ${{ context.merge_target }} ${{ inputs.base_branch }} annotated_diff_path=${{ context.annotated_diff_path }} hunk_ranges_path=${{ context.hunk_ranges_path }} diff_metrics_path=${{ context.diff_metrics_path }} mode=${{ context.review_mode }}", + "cwd": "${{ context.work_dir }}", + "step_name": "review_pr" + }, + "capture": { + "review_verdict": "${{ result.verdict }}" + }, + "on_result": [ + { + "when": "${{ result.verdict }} == changes_requested", + "route": "resolve_review" + }, + { + "when": "${{ result.verdict }} == approved_with_comments", + "route": "resolve_review" + }, + { + "when": "${{ result.verdict }} == needs_human", + "route": "check_review_loop" + }, + { + "when": "${{ result.verdict }} == approved", + "route": "check_review_loop" + }, + { + "when": "true", + "route": "check_review_loop" + } + ], + "on_failure": "check_repo_ci_event", + "on_context_limit": "check_repo_ci_event", + "optional": true, + "skip_when_false": "inputs.open_pr", + "optional_context_refs": [ + "review_loop_count", + "review_mode" + ], + "note": "Runs after compose_pr creates the PR. Invokes the review-pr skill as a headless session to leave inline review comments on the PR. The skill finds the open PR by feature branch name (context.merge_target). Captures verdict as review_verdict and routes via on_result: changes_requested/approved_with_comments → resolve_review; all other verdicts (including approved, needs_human) → check_review_loop (mandatory waypoint to increment review_loop_count). On tool-level failure (MCP error, gh unavailable), routes to check_repo_ci_event (no review to resolve). Skipped when open_pr=false.\n" + }, + "resolve_review": { + "tool": "run_skill", + "model": "", + "with": { + "skill_command": "/autoskillit:resolve-review ${{ context.merge_target }} ${{ inputs.base_branch }} mode=${{ context.review_mode }}", + "cwd": "${{ context.work_dir }}", + "step_name": "resolve_review" + }, + "capture": { + "verdict": "${{ result.verdict }}" + }, + "retries": 2, + "on_exhausted": "release_issue_failure", + "on_result": [ + { + "when": "${{ result.verdict }} == 'real_fix'", + "route": "re_push_review" + }, + { + "when": "${{ result.verdict }} == 'already_green'", + "route": "re_push_review" + }, + { + "when": "${{ result.verdict }} == 'flake_suspected'", + "route": "re_push_review" + }, + { + "when": "${{ result.verdict }} == 'ci_only_failure'", + "route": "release_issue_failure" + } + ], + "on_failure": "release_issue_failure", + "note": "Runs when review_pr reports issues (verdict=changes_requested). The clone (context.work_dir) has the feature branch (context.merge_target) checked out. resolve-review fetches inline and summary review comments via the GitHub API, applies fixes for each actionable finding (critical + warning severity), and commits each fix. Routes via on_result: verdict dispatch — real_fix/already_green/ flake_suspected all route to re_push_review (retry bounded by retries: 2); ci_only_failure escalates to release_issue_failure.\n", + "optional_context_refs": [ + "review_mode" + ] + }, + "re_push_review": { + "tool": "push_to_remote", + "with": { + "clone_path": "${{ context.work_dir }}", + "remote_url": "${{ context.remote_url }}", + "branch": "${{ context.merge_target }}" + }, + "on_success": "check_review_loop", + "on_failure": "release_issue_failure", + "note": "Pushes the review-fixed feature branch back to the remote after resolve_review succeeds. Routes to check_review_loop to determine if another review-resolve cycle is needed (bounded by review_max_retries). On push failure, escalates via release_issue_failure.\n" + }, + "check_review_loop": { + "tool": "run_python", + "with": { + "callable": "autoskillit.smoke_utils.check_review_loop", + "pr_number": "${{ context.pr_number }}", + "current_iteration": "${{ context.review_loop_count }}", + "max_iterations": "${{ inputs.review_max_retries }}", + "previous_verdict": "${{ context.review_verdict }}", + "local_review_rounds": "${{ inputs.local_review_rounds }}" + }, + "capture": { + "review_loop_count": "${{ result.next_iteration }}" + }, + "on_result": [ + { + "when": "${{ result.had_blocking }} == true and ${{ result.max_exceeded }} == false", + "route": "review_pr" + }, + { + "route": "check_repo_ci_event" + } + ], + "on_failure": "check_repo_ci_event", + "skip_when_false": "inputs.open_pr", + "optional_context_refs": [ + "review_loop_count", + "review_verdict" + ], + "note": "Compound iteration + blocking guard. Routes back to review_pr only when had_blocking=true AND max_exceeded=false. had_blocking is true when previous_verdict=changes_requested OR (local_review_rounds > 0 AND iteration < local_review_rounds), ensuring local review rounds are fully exhausted before exiting to CI.\n" + }, + "check_repo_ci_event": { + "tool": "check_repo_merge_state", + "with": { + "branch": "${{ context.merge_target }}", + "cwd": "${{ context.work_dir }}", + "remote_url": "${{ context.remote_url }}", + "step_name": "check_repo_ci_event" + }, + "capture": { + "ci_event": "${{ result.ci_event }}" + }, + "on_success": "check_pr_state", + "on_failure": "ci_watch", + "skip_when_false": "inputs.open_pr", + "note": "Derives ci_event for the feature branch BEFORE ci_watch runs. Uses context.merge_target (feature branch) so push.branches filter is evaluated against the actual branch being pushed, not the base branch. Non-blocking: on_failure routes to ci_watch with ci_event=null (match-any).\n" + }, + "check_pr_state": { + "tool": "check_pr_mergeable", + "with": { + "pr_number": "${{ context.pr_number }}", + "cwd": "${{ context.work_dir }}" + }, + "on_result": [ + { + "when": "${{ result.mergeable_status }} == 'CONFLICTING'", + "route": "detect_ci_conflict" + }, + { + "route": "ci_watch" + } + ], + "on_failure": "ci_watch", + "skip_when_false": "inputs.open_pr", + "note": "Pre-CI-watch gate: checks PR mergeable state before entering the CI polling loop. CONFLICTING PRs cannot trigger pull_request workflows (GitHub cannot create refs/pull/N/merge), so routing to ci_watch would produce a guaranteed no_runs. Routes CONFLICTING to detect_ci_conflict (which handles conflict resolution) and MERGEABLE/UNKNOWN to ci_watch.\n" + }, + "ci_watch": { + "tool": "wait_for_ci", + "with": { + "branch": "${{ context.merge_target }}", + "event": "${{ context.ci_event }}", + "timeout_seconds": 600, + "cwd": "${{ context.work_dir }}", + "remote_url": "${{ context.remote_url }}", + "auto_trigger": "true" + }, + "capture": { + "ci_conclusion": "${{ result.conclusion }}", + "ci_failed_jobs": "${{ result.failed_jobs }}" + }, + "on_result": [ + { + "when": "${{ result.conclusion }} == 'success'", + "route": "check_repo_merge_state" + }, + { + "when": "${{ result.conclusion }} == 'no_runs'", + "route": "handle_no_ci_runs" + }, + { + "when": "${{ result.conclusion }} == 'timed_out'", + "route": "check_ci_timed_out_loop" + }, + { + "route": "detect_ci_conflict" + } + ], + "on_failure": "detect_ci_conflict", + "optional": true, + "skip_when_false": "inputs.open_pr", + "note": "Waits for the GitHub Actions CI run on context.merge_target to complete using the wait_for_ci MCP tool (timeout: 300 s). Captures structured output (conclusion + failed job names) for downstream resolve_ci. Skipped when open_pr=false — no remote feature branch is opened in that mode. Routes on result.conclusion: success → check_repo_merge_state, no_runs → handle_no_ci_runs (auto_trigger fires first: empty commit + force-push + re-poll; handle_no_ci_runs reached only when push fails), timed_out → check_ci_timed_out_loop (iteration guard), failure → detect_ci_conflict.\n" + }, + "check_ci_timed_out_loop": { + "tool": "run_python", + "with": { + "callable": "autoskillit.smoke_utils.check_loop_iteration", + "current_iteration": "${{ context.ci_timed_out_loop_count }}", + "max_iterations": "2" + }, + "capture": { + "ci_timed_out_loop_count": "${{ result.next_iteration }}" + }, + "on_result": [ + { + "when": "${{ result.max_exceeded }} == true", + "route": "detect_ci_conflict" + }, + { + "route": "ci_watch" + } + ], + "on_failure": "detect_ci_conflict", + "optional_context_refs": [ + "ci_timed_out_loop_count" + ], + "note": "Caps the ci_watch re-poll loop at 2 iterations. After 2 timed_out results (1200s total), routes to detect_ci_conflict instead of spinning indefinitely.\n" + }, + "handle_no_ci_runs": { + "tool": "check_repo_merge_state", + "with": { + "branch": "${{ context.merge_target }}", + "cwd": "${{ context.work_dir }}", + "remote_url": "${{ context.remote_url }}", + "step_name": "handle_no_ci_runs" + }, + "capture": { + "ci_event": "${{ result.ci_event }}" + }, + "on_success": "check_ci_loop", + "on_failure": "release_issue_failure", + "retries": 1, + "on_exhausted": "release_issue_failure", + "note": "Intercepts no_runs conclusion from wait_for_ci. Re-derives ci_event using check_repo_merge_state (the original event may have been wrong for this branch's workflow configuration) and routes to check_ci_loop for iteration guard before retrying ci_watch. retries: 1 bounds the tool-call retry; check_ci_loop bounds the outer loop at 2 iterations.\n" + }, + "check_ci_loop": { + "tool": "run_python", + "with": { + "callable": "autoskillit.smoke_utils.check_loop_iteration", + "current_iteration": "${{ context.ci_loop_count }}", + "max_iterations": "2" + }, + "capture": { + "ci_loop_count": "${{ result.next_iteration }}" + }, + "on_result": [ + { + "when": "${{ result.max_exceeded }} == true", + "route": "check_active_trigger_loop" + }, + { + "route": "ci_watch" + } + ], + "on_failure": "check_ci_already_passed", + "optional_context_refs": [ + "ci_loop_count" + ], + "note": "Iteration guard for the ci_watch / handle_no_ci_runs cycle. Caps the loop at 2 iterations (initial + 1 retry after re-deriving ci_event). On budget exhaustion, routes to check_active_trigger_loop which caps the active trigger to a single attempt before escalating.\n" + }, + "check_active_trigger_loop": { + "tool": "run_python", + "with": { + "callable": "autoskillit.smoke_utils.check_loop_iteration", + "current_iteration": "${{ context.active_trigger_count }}", + "max_iterations": "2" + }, + "capture": { + "active_trigger_count": "${{ result.next_iteration }}" + }, + "on_result": [ + { + "when": "${{ result.max_exceeded }} == true", + "route": "check_ci_already_passed" + }, + { + "route": "trigger_ci_actively" + } + ], + "on_failure": "check_ci_already_passed", + "optional_context_refs": [ + "active_trigger_count" + ], + "note": "Bounds the trigger_ci_actively remediation to a single attempt. If ci_watch still returns no_runs after the active trigger, escalates to escalate_stop_no_ci rather than re-triggering indefinitely.\n" + }, + "trigger_ci_actively": { + "tool": "run_cmd", + "with": { + "cmd": "gh pr close ${{ context.pr_number }} && sleep 2 && gh pr reopen ${{ context.pr_number }}", + "cwd": "${{ context.work_dir }}", + "step_name": "trigger_ci_actively" + }, + "on_success": "ci_watch", + "on_failure": "check_ci_already_passed", + "retries": 1, + "on_exhausted": "check_ci_already_passed", + "note": "Active CI trigger remediation: closes and reopens the PR to force GitHub webhook re-delivery. This re-creates the pull_request event and the refs/pull/N/merge ref (if the PR is mergeable). Routes back to ci_watch for one final attempt. If the close/reopen fails or the subsequent ci_watch still returns no_runs, escalates to escalate_stop_no_ci.\n" + }, + "check_ci_already_passed": { + "tool": "run_cmd", + "with": { + "cmd": "gh pr view '${{ context.pr_number }}' --json statusCheckRollup --jq '.statusCheckRollup | if length == 0 then \"no_checks\" elif all(.state == \"SUCCESS\" or .state == \"NEUTRAL\" or .state == \"SKIPPED\") then \"passed\" elif any(.state == \"FAILURE\" or .state == \"ERROR\") then \"failed\" else \"pending\" end'", + "cwd": "${{ context.work_dir }}", + "step_name": "check_ci_already_passed" + }, + "on_result": [ + { + "when": "${{ result.stdout | trim }} == passed", + "route": "check_repo_merge_state" + }, + { + "when": "${{ result.stdout | trim }} == failed", + "route": "detect_ci_conflict" + }, + { + "when": "${{ result.stdout | trim }} == pending", + "route": "ci_watch" + }, + { + "route": "mark_issue_failed_no_ci" + } + ], + "on_failure": "mark_issue_failed_no_ci", + "skip_when_false": "inputs.open_pr", + "note": "Safety net before terminal escalation: checks if CI already passed despite the watcher failing to observe it (e.g. post-force-push SHA staleness). Uses statusCheckRollup as the authoritative signal. Outputs passed/failed/no_checks/pending: passed goes to merge gate, failed routes to CI conflict remediation, pending re-enters the CI watcher, no_checks falls through to terminal stop.\n" + }, + "mark_issue_failed_no_ci": { + "description": "Swap in-progress → fail label before CI failure escalation", + "tool": "release_issue", + "optional": true, + "skip_when_false": "inputs.issue_url", + "with": { + "issue_url": "${{ inputs.issue_url }}", + "fail_label": "fail" + }, + "on_success": "register_clone_no_ci", + "on_failure": "register_clone_no_ci" + }, + "register_clone_no_ci": { + "description": "Register clone as preserved (no-CI error pipeline)", + "tool": "register_clone_status", + "with": { + "clone_path": "${{ context.work_dir }}", + "status": "error", + "step_name": "register_clone_no_ci" + }, + "on_success": "escalate_stop_no_ci", + "on_failure": "escalate_stop_no_ci" + }, + "escalate_stop_no_ci": { + "action": "stop", + "message": "CI safety net: statusCheckRollup returned no checks for PR ${{ context.pr_number }}. This typically means no workflow is configured to trigger on this branch pattern. Check .github/workflows/ trigger configuration. This is NOT a test failure — CI never started." + }, + "check_repo_merge_state": { + "tool": "check_repo_merge_state", + "block": "pre_queue_gate", + "with": { + "branch": "${{ inputs.base_branch }}", + "cwd": "${{ context.work_dir }}", + "remote_url": "${{ context.remote_url }}", + "step_name": "check_repo_merge_state" + }, + "capture": { + "queue_available": "${{ result.queue_available }}", + "merge_group_trigger": "${{ result.merge_group_trigger }}", + "auto_merge_available": "${{ result.auto_merge_available }}", + "ci_event": "${{ result.ci_event }}" + }, + "on_success": "route_queue_mode", + "on_failure": "immediate_merge", + "skip_when_false": "inputs.open_pr", + "note": "Single GraphQL round-trip that captures queue_available, merge_group_trigger, auto_merge_available, and ci_event for the target branch. Consolidates the former three sequential run_cmd steps (check_merge_queue, check_merge_group_trigger, check_auto_merge) into one MCP tool call. On failure (network error, auth failure), falls through to immediate_merge (non-queue direct merge path). Skipped when open_pr=false — no PR was opened.\n" + }, + "route_queue_mode": { + "block": "pre_queue_gate", + "action": "route", + "on_result": [ + { + "when": "${{ inputs.auto_merge }} != 'true'", + "route": "patch_token_summary" + }, + { + "when": "${{ context.queue_available }} == true and ${{ context.merge_group_trigger }} == true", + "route": "enqueue_to_queue" + }, + { + "when": "${{ context.auto_merge_available }} == true", + "route": "direct_merge" + }, + { + "route": "immediate_merge" + } + ], + "skip_when_false": "inputs.open_pr" + }, + "enqueue_to_queue": { + "tool": "enqueue_pr", + "with": { + "pr_number": "${{ context.pr_number }}", + "target_branch": "${{ inputs.base_branch }}", + "cwd": "${{ context.work_dir }}", + "auto_merge_available": "${{ context.auto_merge_available }}" + }, + "on_success": "wait_for_queue", + "on_failure": "verify_queue_enrollment", + "skip_when_false": "inputs.open_pr", + "note": "Submits the PR to the merge queue via the unified enqueue_pr tool. The tool resolves the enrollment strategy internally based on auto_merge_available. On failure, probes the queue via verify_queue_enrollment to distinguish a transient enqueue error from a PR that is already merged or still in queue.\n" + }, + "verify_queue_enrollment": { + "description": "Probe merge queue after enqueue command failure to determine actual state", + "tool": "wait_for_merge_queue", + "with": { + "pr_number": "${{ context.pr_number }}", + "target_branch": "${{ inputs.base_branch }}", + "cwd": "${{ context.work_dir }}", + "remote_url": "${{ context.remote_url }}", + "timeout_seconds": 60, + "poll_interval": 10 + }, + "on_result": [ + { + "when": "${{ result.pr_state }} == merged", + "route": "release_issue_success" + }, + { + "when": "${{ result.pr_state }} == ejected_ci_failure", + "route": "diagnose_ci" + }, + { + "when": "${{ result.pr_state }} == not_enrolled", + "route": "release_issue_failure" + }, + { + "when": "${{ result.pr_state }} == dropped_healthy", + "route": "wait_for_queue" + }, + { + "when": "${{ result.pr_state }} == dropped_merge_group_ci", + "route": "diagnose_ci" + }, + { + "when": "${{ result.pr_state }} == ejected", + "route": "wait_for_queue" + }, + { + "when": "${{ result.pr_state }} == stalled", + "route": "wait_for_queue" + }, + { + "when": "${{ result.pr_state }} == timeout", + "route": "register_clone_unconfirmed" + }, + { + "route": "register_clone_unconfirmed" + } + ], + "on_failure": "register_clone_unconfirmed", + "note": "Short probe (60s) after enqueue_to_queue failure. If the PR is already merged, route to release_issue_success. If the queue already confirms CI failure (ejected_ci_failure or dropped_merge_group_ci), route directly to diagnose_ci. If enrollment was attempted but the PR never appeared in the queue (not_enrolled), route to release_issue_failure — this is a repo configuration error. For other active queue states (dropped_healthy, ejected, stalled), route to wait_for_queue for full 900s authoritative watch. If the probe timed out or returned an unexpected state, escalate via register_clone_unconfirmed.\n" + }, + "wait_for_queue": { + "tool": "wait_for_merge_queue", + "with": { + "pr_number": "${{ context.pr_number }}", + "target_branch": "${{ inputs.base_branch }}", + "timeout_seconds": 900, + "cwd": "${{ context.work_dir }}", + "remote_url": "${{ context.remote_url }}" + }, + "capture": { + "queue_pr_state": "${{ result.pr_state }}" + }, + "on_result": [ + { + "when": "${{ result.pr_state }} == merged", + "route": "release_issue_success" + }, + { + "when": "${{ result.pr_state }} == ejected_ci_failure", + "route": "diagnose_ci" + }, + { + "when": "${{ result.pr_state }} == dropped_merge_group_ci", + "route": "diagnose_ci" + }, + { + "when": "${{ result.pr_state }} == not_enrolled", + "route": "release_issue_failure" + }, + { + "when": "${{ result.pr_state }} == dropped_healthy", + "route": "check_dropped_healthy_loop" + }, + { + "when": "${{ result.pr_state }} == ejected", + "route": "check_eject_limit" + }, + { + "when": "${{ result.pr_state }} == stalled", + "route": "reenroll_stalled_pr" + }, + { + "when": "${{ result.pr_state }} == timeout", + "route": "register_clone_unconfirmed" + }, + { + "route": "register_clone_unconfirmed" + } + ], + "on_failure": "register_clone_unconfirmed", + "skip_when_false": "inputs.open_pr", + "note": "Waits up to 15 minutes for the PR to merge through the queue. merged → release_issue_success to apply staged label, then register_clone_success. release_issue_success is optional with skip_when_false: inputs.issue_url — when issue_url is absent the step is silently skipped and pipeline continues to register_clone_success. ejected_ci_failure → diagnose_ci (CI failure; conflict resolution cannot fix this). not_enrolled → release_issue_failure (enrollment failed; repo configuration error). ejected → conflict fix cycle then re-enter. stalled → lightweight re-enrollment via enqueue_pr. timeout → register_clone_unconfirmed (release claim without staged label; merge unconfirmed).\n" + }, + "reenroll_stalled_pr": { + "tool": "enqueue_pr", + "with": { + "pr_number": "${{ context.pr_number }}", + "target_branch": "${{ inputs.base_branch }}", + "cwd": "${{ context.work_dir }}", + "auto_merge_available": "${{ context.auto_merge_available }}" + }, + "on_success": "check_stall_loop", + "on_failure": "release_issue_failure", + "skip_when_false": "inputs.open_pr" + }, + "check_stall_loop": { + "tool": "run_python", + "with": { + "callable": "autoskillit.smoke_utils.check_loop_iteration", + "current_iteration": "${{ context.stall_loop_count }}", + "max_iterations": "3" + }, + "capture": { + "stall_loop_count": "${{ result.next_iteration }}" + }, + "on_result": [ + { + "when": "${{ result.max_exceeded }} == true", + "route": "register_clone_unconfirmed" + }, + { + "route": "wait_for_queue" + } + ], + "on_failure": "register_clone_unconfirmed", + "optional_context_refs": [ + "stall_loop_count" + ], + "note": "Iteration guard for the wait_for_queue / reenroll_stalled_pr cycle. Caps stall re-enrollment at 3 attempts. On budget exhaustion, routes to register_clone_unconfirmed (merge outcome unconfirmed).\n" + }, + "check_eject_limit": { + "tool": "run_python", + "with": { + "callable": "autoskillit.recipe._cmd_rpc.check_eject_limit", + "counter_file": "${{ context.work_dir }}/{{AUTOSKILLIT_TEMP}}/eject_count", + "max_ejects": "3" + }, + "on_result": [ + { + "when": "${{ result.status }} == EJECT_LIMIT_EXCEEDED", + "route": "release_issue_failure" + }, + { + "route": "queue_ejected_fix" + } + ], + "on_failure": "release_issue_failure", + "note": "Circuit breaker for the queue ejection loop. Tracks ejection count in a file-based counter under {{AUTOSKILLIT_TEMP}}/. After 3 ejections, routes to release_issue_failure to prevent infinite re-enqueue cycles.\n" + }, + "check_dropped_healthy_loop": { + "tool": "run_python", + "with": { + "callable": "autoskillit.recipe._cmd_rpc.check_dropped_healthy_loop", + "counter_file": "${{ context.work_dir }}/{{AUTOSKILLIT_TEMP}}/dropped_healthy_count", + "max_drops": "2" + }, + "on_result": [ + { + "when": "${{ result.status }} == DROPPED_LIMIT_EXCEEDED", + "route": "release_issue_failure" + }, + { + "route": "reenter_merge_queue_cheap" + } + ], + "on_failure": "release_issue_failure", + "note": "Circuit breaker for the dropped_healthy re-enrollment loop. After 2 consecutive unexplained drops with healthy PR-branch CI, escalates to release_issue_failure instead of looping. Matches the check_eject_limit pattern used for the ejected path. DROPPED_MERGE_GROUP_CI (confirmed merge-group CI failure) is routed directly to diagnose_ci and never reaches this step.\n" + }, + "queue_ejected_fix": { + "tool": "run_python", + "with": { + "callable": "autoskillit.recipe._cmd_rpc.queue_ejected_fix", + "work_dir": "${{ context.work_dir }}", + "base_branch": "${{ inputs.base_branch }}" + }, + "on_result": [ + { + "when": "${{ result.status }} == clean", + "route": "re_push_queue_fix" + }, + { + "route": "resolve_queue_merge_conflicts" + } + ], + "on_failure": "resolve_queue_merge_conflicts", + "skip_when_false": "inputs.open_pr", + "note": "Attempts a cheap rebase before invoking the full conflict-resolution skill. On a clean rebase (stdout \"clean\"), skips the skill and proceeds directly to re_push_queue_fix. On conflict, aborts the rebase and routes to resolve_queue_merge_conflicts for goal-aware resolution.\n" + }, + "resolve_queue_merge_conflicts": { + "tool": "run_skill", + "model": "", + "with": { + "skill_command": "/autoskillit:resolve-merge-conflicts ${{ context.work_dir }} ${{ context.plan_path }} ${{ inputs.base_branch }}", + "cwd": "${{ context.work_dir }}", + "step_name": "resolve_queue_merge_conflicts" + }, + "capture": { + "conflict_escalation_required": "${{ result.escalation_required }}" + }, + "on_result": [ + { + "when": "${{ result.escalation_required }} == true", + "route": "release_issue_failure" + }, + { + "route": "re_push_queue_fix" + } + ], + "on_failure": "release_issue_failure", + "on_context_limit": "release_issue_failure", + "retries": 1, + "on_exhausted": "release_issue_failure", + "skip_when_false": "inputs.open_pr", + "note": "Resolves rebase conflicts on the feature branch after queue ejection. Only reached when the cheap rebase in queue_ejected_fix encountered conflicts. escalation_required=true → release_issue_failure for human review.\n" + }, + "re_push_queue_fix": { + "tool": "push_to_remote", + "with": { + "clone_path": "${{ context.work_dir }}", + "remote_url": "${{ context.remote_url }}", + "branch": "${{ context.merge_target }}", + "force": "true" + }, + "capture": { + "push_error_type": "${{ result.error_type }}" + }, + "retries": 2, + "on_exhausted": "register_clone_unconfirmed", + "on_success": "ci_watch_post_queue_fix", + "on_failure": "classify_push_failure", + "skip_when_false": "inputs.open_pr" + }, + "classify_push_failure": { + "action": "route", + "on_result": [ + { + "when": "${{ context.push_error_type }} == queued_branch", + "route": "wait_dequeue_retry" + }, + { + "route": "release_issue_failure" + } + ] + }, + "wait_dequeue_retry": { + "tool": "wait_for_merge_queue", + "with": { + "pr_number": "${{ context.pr_number }}", + "target_branch": "${{ inputs.base_branch }}", + "cwd": "${{ context.work_dir }}", + "timeout_seconds": "60" + }, + "on_result": [ + { + "when": "${{ result.pr_state }} == merged", + "route": "release_issue_success" + }, + { + "when": "${{ result.pr_state }} == ejected_ci_failure", + "route": "register_clone_unconfirmed" + }, + { + "when": "${{ result.pr_state }} == dropped_merge_group_ci", + "route": "register_clone_unconfirmed" + }, + { + "when": "${{ result.pr_state }} == dropped_healthy", + "route": "register_clone_unconfirmed" + }, + { + "when": "${{ result.pr_state }} == ejected", + "route": "re_push_queue_fix" + }, + { + "when": "${{ result.pr_state }} == stalled", + "route": "register_clone_unconfirmed" + }, + { + "when": "${{ result.pr_state }} == timeout", + "route": "register_clone_unconfirmed" + }, + { + "route": "register_clone_unconfirmed" + } + ], + "on_failure": "register_clone_unconfirmed", + "optional_context_refs": [ + "push_error_type" + ] + }, + "ci_watch_post_queue_fix": { + "tool": "wait_for_ci", + "with": { + "branch": "${{ context.merge_target }}", + "event": "${{ context.ci_event }}", + "timeout_seconds": 600, + "cwd": "${{ context.work_dir }}", + "remote_url": "${{ context.remote_url }}", + "auto_trigger": "true" + }, + "capture": { + "ci_conclusion": "${{ result.conclusion }}", + "ci_failed_jobs": "${{ result.failed_jobs }}" + }, + "on_result": [ + { + "when": "${{ result.conclusion }} == 'success'", + "route": "reenter_merge_queue" + }, + { + "when": "${{ result.conclusion }} == 'no_runs'", + "route": "release_issue_failure" + }, + { + "when": "${{ result.conclusion }} == 'timed_out'", + "route": "check_ci_post_queue_loop" + }, + { + "route": "detect_ci_conflict" + } + ], + "on_failure": "detect_ci_conflict", + "skip_when_false": "inputs.open_pr", + "note": "Validates CI on the rebased feature branch after queue ejection fix and re-push. Routes on result.conclusion: success → reenter_merge_queue, no_runs → release_issue_failure (auto_trigger fires first: empty commit + force-push + re-poll; no_runs reached only when trigger recovery fails), timed_out → check_ci_post_queue_loop, failure → detect_ci_conflict.\n" + }, + "check_ci_post_queue_loop": { + "tool": "run_python", + "with": { + "callable": "autoskillit.smoke_utils.check_loop_iteration", + "current_iteration": "${{ context.ci_post_queue_loop_count }}", + "max_iterations": "2" + }, + "capture": { + "ci_post_queue_loop_count": "${{ result.next_iteration }}" + }, + "on_result": [ + { + "when": "${{ result.max_exceeded }} == true", + "route": "release_issue_failure" + }, + { + "route": "ci_watch_post_queue_fix" + } + ], + "on_failure": "release_issue_failure", + "optional_context_refs": [ + "ci_post_queue_loop_count" + ], + "skip_when_false": "inputs.open_pr", + "note": "Iteration guard for the ci_watch_post_queue_fix timed_out self-loop. Caps at 2 iterations (initial + 1 retry). On budget exhaustion, routes to release_issue_failure — this is already a post-ejection retry path so further escalation is not warranted.\n" + }, + "reenter_merge_queue": { + "tool": "enqueue_pr", + "with": { + "pr_number": "${{ context.pr_number }}", + "target_branch": "${{ inputs.base_branch }}", + "cwd": "${{ context.work_dir }}", + "auto_merge_available": "${{ context.auto_merge_available }}" + }, + "on_success": "wait_for_queue", + "on_failure": "release_issue_failure", + "skip_when_false": "inputs.open_pr", + "note": "Re-submits the PR after conflict resolution. Routes back to wait_for_queue. If re-entry fails, release_issue_failure preserves the clone for debugging.\n" + }, + "reenter_merge_queue_cheap": { + "tool": "enqueue_pr", + "with": { + "pr_number": "${{ context.pr_number }}", + "target_branch": "${{ inputs.base_branch }}", + "cwd": "${{ context.work_dir }}", + "auto_merge_available": "${{ context.auto_merge_available }}" + }, + "on_success": "wait_for_queue", + "on_failure": "release_issue_failure", + "skip_when_false": "inputs.open_pr", + "note": "Re-submits a healthy PR into the merge queue after auto_merge was cleared externally (DROPPED_HEALTHY state). No conflict-resolution CI wait needed. Routes back to wait_for_queue on success.\n" + }, + "direct_merge": { + "tool": "run_cmd", + "with": { + "cmd": "gh pr merge '${{ context.pr_number }}' --squash --auto", + "cwd": "${{ context.work_dir }}", + "step_name": "direct_merge" + }, + "on_success": "wait_for_direct_merge", + "on_failure": "release_issue_failure", + "skip_when_false": "inputs.open_pr", + "note": "Enables GitHub native auto-merge for direct merge (no merge queue). GitHub merges automatically once all required checks pass. On failure, escalates via release_issue_failure so the issue claim is released and the pipeline reports an error rather than silently succeeding.\n" + }, + "wait_for_direct_merge": { + "tool": "run_python", + "with": { + "callable": "autoskillit.recipe._cmd_rpc.wait_for_direct_merge", + "pr_number": "${{ context.pr_number }}", + "max_polls": "90", + "poll_interval": "10" + }, + "on_result": [ + { + "when": "${{ result.state }} == merged", + "route": "release_issue_success" + }, + { + "when": "${{ result.state }} == closed", + "route": "direct_merge_conflict_fix" + }, + { + "route": "register_clone_unconfirmed" + } + ], + "on_failure": "register_clone_unconfirmed", + "skip_when_false": "inputs.open_pr", + "note": "Polls GitHub PR state every 10 s for up to 15 minutes. merged → release_issue_success. closed → conflict fix cycle. timeout → register_clone_unconfirmed (releases claim; merge unconfirmed).\n" + }, + "direct_merge_conflict_fix": { + "tool": "run_python", + "with": { + "callable": "autoskillit.recipe._cmd_rpc.direct_merge_conflict_fix", + "work_dir": "${{ context.work_dir }}", + "base_branch": "${{ inputs.base_branch }}" + }, + "on_result": [ + { + "when": "${{ result.status }} == clean", + "route": "re_push_direct_fix" + }, + { + "route": "resolve_direct_merge_conflicts" + } + ], + "on_failure": "resolve_direct_merge_conflicts", + "skip_when_false": "inputs.open_pr", + "note": "Attempts a cheap rebase before invoking the full conflict-resolution skill. On clean rebase, proceeds to re_push_direct_fix. On conflict, routes to resolve_direct_merge_conflicts for goal-aware resolution.\n" + }, + "resolve_direct_merge_conflicts": { + "tool": "run_skill", + "model": "", + "with": { + "skill_command": "/autoskillit:resolve-merge-conflicts ${{ context.work_dir }} ${{ context.plan_path }} ${{ inputs.base_branch }}", + "cwd": "${{ context.work_dir }}", + "step_name": "resolve_direct_merge_conflicts" + }, + "capture": { + "conflict_escalation_required": "${{ result.escalation_required }}" + }, + "on_result": [ + { + "when": "${{ result.escalation_required }} == true", + "route": "release_issue_failure" + }, + { + "route": "re_push_direct_fix" + } + ], + "on_failure": "release_issue_failure", + "on_context_limit": "release_issue_failure", + "retries": 1, + "on_exhausted": "release_issue_failure", + "skip_when_false": "inputs.open_pr", + "note": "Resolves rebase conflicts after direct-merge rejection. Only reached when the cheap rebase in direct_merge_conflict_fix encountered conflicts. escalation_required=true → release_issue_failure for human review.\n" + }, + "re_push_direct_fix": { + "tool": "push_to_remote", + "with": { + "clone_path": "${{ context.work_dir }}", + "remote_url": "${{ context.remote_url }}", + "branch": "${{ context.merge_target }}", + "force": "true" + }, + "on_success": "redirect_merge", + "on_failure": "release_issue_failure", + "skip_when_false": "inputs.open_pr" + }, + "redirect_merge": { + "tool": "run_cmd", + "with": { + "cmd": "gh pr merge '${{ context.pr_number }}' --squash --auto", + "cwd": "${{ context.work_dir }}", + "step_name": "redirect_merge" + }, + "on_success": "wait_for_direct_merge", + "on_failure": "release_issue_failure", + "skip_when_false": "inputs.open_pr", + "note": "Re-enables auto-merge after conflict resolution and re-push. Routes back to wait_for_direct_merge to poll for completion. On failure, release_issue_failure preserves the clone for debugging.\n" + }, + "immediate_merge": { + "tool": "run_cmd", + "with": { + "cmd": "gh pr merge '${{ context.pr_number }}' --squash", + "cwd": "${{ context.work_dir }}", + "step_name": "immediate_merge" + }, + "on_success": "wait_for_immediate_merge", + "on_failure": "release_issue_failure", + "skip_when_false": "inputs.open_pr", + "note": "Merges the PR immediately using plain --squash when autoMergeAllowed=false. Unlike direct_merge, no --auto flag is used — GitHub executes the merge synchronously rather than queueing it for when checks pass. On failure, escalates via release_issue_failure so the issue claim is released and the pipeline reports an error rather than silently succeeding.\n" + }, + "wait_for_immediate_merge": { + "tool": "run_python", + "with": { + "callable": "autoskillit.recipe._cmd_rpc.wait_for_immediate_merge", + "pr_number": "${{ context.pr_number }}", + "max_polls": "30", + "poll_interval": "10" + }, + "on_result": [ + { + "when": "${{ result.state }} == merged", + "route": "release_issue_success" + }, + { + "when": "${{ result.state }} == closed", + "route": "immediate_merge_conflict_fix" + }, + { + "route": "register_clone_unconfirmed" + } + ], + "on_failure": "register_clone_unconfirmed", + "skip_when_false": "inputs.open_pr", + "note": "Polls GitHub PR state every 10s for up to 5 minutes (30 iterations). Since immediate_merge uses --squash without --auto, the merge completes synchronously — this poll is a confirmation window. merged → release_issue_success. closed → PR was rejected (stale base or conflict) — route to conflict fix. timeout → register_clone_unconfirmed.\n" + }, + "immediate_merge_conflict_fix": { + "tool": "run_python", + "with": { + "callable": "autoskillit.recipe._cmd_rpc.immediate_merge_conflict_fix", + "work_dir": "${{ context.work_dir }}", + "base_branch": "${{ inputs.base_branch }}" + }, + "on_result": [ + { + "when": "${{ result.status }} == clean", + "route": "re_push_immediate_fix" + }, + { + "route": "resolve_immediate_merge_conflicts" + } + ], + "on_failure": "resolve_immediate_merge_conflicts", + "skip_when_false": "inputs.open_pr", + "note": "Attempts a cheap rebase before invoking the full conflict-resolution skill. On clean rebase, proceeds to re_push_immediate_fix. On conflict, routes to resolve_immediate_merge_conflicts for goal-aware resolution.\n" + }, + "resolve_immediate_merge_conflicts": { + "tool": "run_skill", + "model": "", + "with": { + "skill_command": "/autoskillit:resolve-merge-conflicts ${{ context.work_dir }} ${{ context.plan_path }} ${{ inputs.base_branch }}", + "cwd": "${{ context.work_dir }}", + "step_name": "resolve_immediate_merge_conflicts" + }, + "capture": { + "conflict_escalation_required": "${{ result.escalation_required }}" + }, + "on_result": [ + { + "when": "${{ result.escalation_required }} == true", + "route": "release_issue_failure" + }, + { + "route": "re_push_immediate_fix" + } + ], + "on_failure": "release_issue_failure", + "on_context_limit": "release_issue_failure", + "retries": 1, + "on_exhausted": "release_issue_failure", + "skip_when_false": "inputs.open_pr", + "note": "Resolves rebase conflicts after immediate-merge rejection. Only reached when the cheap rebase in immediate_merge_conflict_fix encountered conflicts. escalation_required=true → release_issue_failure for human review.\n" + }, + "re_push_immediate_fix": { + "tool": "push_to_remote", + "with": { + "clone_path": "${{ context.work_dir }}", + "remote_url": "${{ context.remote_url }}", + "branch": "${{ context.merge_target }}", + "force": "true" + }, + "on_success": "remerge_immediate", + "on_failure": "release_issue_failure", + "skip_when_false": "inputs.open_pr" + }, + "remerge_immediate": { + "tool": "run_cmd", + "with": { + "cmd": "gh pr merge '${{ context.pr_number }}' --squash", + "cwd": "${{ context.work_dir }}", + "step_name": "remerge_immediate" + }, + "on_success": "wait_for_immediate_merge", + "on_failure": "release_issue_failure", + "skip_when_false": "inputs.open_pr", + "note": "Re-attempts plain squash merge after conflict resolution and re-push. Routes back to wait_for_immediate_merge to confirm completion. On failure, release_issue_failure preserves the clone for debugging.\n" + }, + "diagnose_ci": { + "tool": "run_skill", + "model": "", + "with": { + "skill_command": "/autoskillit:diagnose-ci ${{ context.merge_target }} - - - ${{ context.ci_event }}", + "cwd": "${{ context.work_dir }}", + "step_name": "diagnose_ci", + "ci_failed_jobs": "${{ context.ci_failed_jobs }}" + }, + "capture": { + "diagnosis_path": "${{ result.diagnosis_path }}" + }, + "on_success": "resolve_ci", + "on_failure": "resolve_ci", + "on_context_limit": "resolve_ci", + "optional": true, + "skip_when_false": "inputs.open_pr", + "note": "Runs when ci_watch reports a CI failure. Fetches CI logs via gh API and writes a structured diagnosis to {{AUTOSKILLIT_TEMP}}/diagnose-ci/. Routes to resolve_ci on success OR failure (best-effort: resolve-failures proceeds even without a diagnosis). diagnosis_path is passed to resolve_ci for targeted remediation context.\n" + }, + "resolve_ci": { + "tool": "run_skill", + "model": "", + "with": { + "skill_command": "/autoskillit:resolve-failures ${{ context.work_dir }} ${{ context.plan_path }} ${{ inputs.base_branch }} ${{ context.ci_conclusion }} ${{ context.ci_failed_jobs }} ${{ context.diagnosis_path }}", + "cwd": "${{ context.work_dir }}", + "step_name": "resolve_ci" + }, + "capture": { + "verdict": "${{ result.verdict }}" + }, + "retries": 2, + "on_exhausted": "release_issue_failure", + "on_context_limit": "re_push", + "on_result": [ + { + "when": "${{ result.verdict }} == 'real_fix'", + "route": "re_push" + }, + { + "when": "${{ result.verdict }} == 'already_green'", + "route": "pre_resolve_rebase" + }, + { + "when": "${{ result.verdict }} == 'flake_suspected'", + "route": "re_push" + }, + { + "when": "${{ result.verdict }} == 'ci_only_failure'", + "route": "release_issue_failure" + }, + { + "route": "release_issue_failure" + } + ], + "on_failure": "release_issue_failure", + "note": "Runs when ci_watch reports a CI failure. Receives ci_failed_jobs from wait_for_ci structured output to avoid redundant CI failure re-investigation. Routes via on_result: verdict dispatch — real_fix pushes, already_green rebases and re-diagnoses (sibling fix may have landed), flake_suspected retries via re_push (bounded by retries: 2 / on_exhausted: release_issue_failure), ci_only_failure escalates to human. Retries up to 2 times (3 total attempts) before exhausting to release_issue_failure.\n" + }, + "pre_resolve_rebase": { + "tool": "run_cmd", + "with": { + "cmd": "REMOTE=$(git -C ${{ context.work_dir }} remote get-url upstream 2>/dev/null | grep -qv \"^file://\" && echo upstream || echo origin) &&\ngit -C ${{ context.work_dir }} fetch \"$REMOTE\" &&\ngit -C ${{ context.work_dir }} rebase \"$REMOTE\"/${{ inputs.base_branch }}\n", + "step_name": "pre_resolve_rebase" + }, + "retries": 1, + "on_success": "diagnose_ci", + "on_failure": "release_issue_failure", + "note": "Rebases the feature branch against integration head when resolve_ci emits already_green — the worktree was already clean because a sibling pipeline's fix already landed on integration. Pulling it in before re-running diagnose_ci gives the pipeline a real chance at emitting real_fix and reaching re_push.\n" + }, + "re_push": { + "tool": "push_to_remote", + "with": { + "clone_path": "${{ context.work_dir }}", + "remote_url": "${{ context.remote_url }}", + "branch": "${{ context.merge_target }}", + "force": "true" + }, + "on_success": "check_repo_ci_event", + "on_failure": "release_issue_failure", + "note": "Pushes the CI-fixed feature branch back to the remote after resolve_ci succeeds. Routes to check_repo_ci_event to derive ci_event before ci_watch verifies the fix. On push failure, escalates via release_issue_failure.\n" + }, + "detect_ci_conflict": { + "tool": "run_cmd", + "with": { + "cmd": "REMOTE=$(git remote get-url upstream 2>/dev/null | grep -qv \"^file://\" && echo upstream || echo origin) &&\ngit fetch \"$REMOTE\" ${{ inputs.base_branch }} &&\n! git merge-base --is-ancestor \"$REMOTE\"/${{ inputs.base_branch }} HEAD\n", + "cwd": "${{ context.work_dir }}", + "step_name": "detect_ci_conflict" + }, + "on_success": "ci_conflict_fix", + "on_failure": "diagnose_ci", + "optional": true, + "skip_when_false": "inputs.open_pr", + "note": "Detects whether the CI failure originates from a stale integration base rather than a code-level test failure. Exit 0 (fetch succeeded AND base is NOT an ancestor of HEAD) signals a stale base — routes to ci_conflict_fix. Exit 1 (fetch failed OR base is already an ancestor) signals a real code failure or an unresolvable state — falls through to diagnose_ci. This step insulates resolve_ci from being invoked for merge-conflict failures.\n" + }, + "ci_conflict_fix": { + "tool": "run_skill", + "model": "", + "with": { + "skill_command": "/autoskillit:resolve-merge-conflicts ${{ context.work_dir }} ${{ context.plan_path }} ${{ inputs.base_branch }}", + "cwd": "${{ context.work_dir }}", + "step_name": "ci_conflict_fix" + }, + "capture": { + "conflict_escalation_required": "${{ result.escalation_required }}" + }, + "on_result": [ + { + "when": "${{ result.escalation_required }} == true", + "route": "release_issue_failure" + }, + { + "route": "re_push" + } + ], + "on_failure": "release_issue_failure", + "on_context_limit": "release_issue_failure", + "retries": 1, + "on_exhausted": "release_issue_failure", + "skip_when_false": "inputs.open_pr", + "note": "Resolves a stale-base CI failure by rebasing the feature branch onto the latest integration HEAD. Mirrors queue_ejected_fix: on clean rebase or goal-aware conflict resolution, routes to re_push to re-trigger CI on the rebased branch. On escalation (LOW confidence conflicts) or tool failure, routes to release_issue_failure. retries: 1 bounds the loop; after 2 total attempts, escalates.\n" + }, + "register_clone_unconfirmed": { + "description": "Register clone as preserved — merge unconfirmed, in-progress label retained", + "tool": "register_clone_status", + "with": { + "clone_path": "${{ context.work_dir }}", + "status": "unconfirmed", + "step_name": "register_clone_unconfirmed" + }, + "on_success": "done", + "on_failure": "done", + "note": "Called when a merge-wait step times out before merge is confirmed. Does NOT call release_issue — the in-progress label stays on the issue so operators can see the PR is still actively queued. The clone is preserved (status: unconfirmed) for human inspection.\n" + }, + "release_issue_success": { + "description": "Transition issue from in-progress to staged after successful pipeline completion", + "tool": "release_issue", + "optional": true, + "skip_when_false": "inputs.issue_url", + "with": { + "issue_url": "${{ inputs.issue_url }}", + "target_branch": "${{ inputs.base_branch }}" + }, + "on_success": "patch_token_summary", + "on_failure": "patch_token_summary", + "note": "Applies staged label after successful PR merge; skipped when no issue_url. optional=true so registration proceeds even if this step errors.\n" + }, + "patch_token_summary": { + "description": "Collect all pipeline token data and patch the PR body with complete summary", + "tool": "run_python", + "optional": true, + "skip_when_false": "inputs.open_pr", + "with": { + "callable": "autoskillit.smoke_utils.patch_pr_token_summary", + "timeout": 60, + "pr_url": "${{ context.pr_url }}", + "cwd": "${{ context.work_dir }}" + }, + "on_success": "register_clone_success", + "on_failure": "register_clone_success", + "note": "Patches PR body with complete token summary; failure never blocks pipeline completion.\n" + }, + "release_issue_failure": { + "description": "Release the in-progress claim after pipeline failure", + "tool": "release_issue", + "optional": true, + "skip_when_false": "inputs.issue_url", + "with": { + "issue_url": "${{ inputs.issue_url }}", + "fail_label": "fail" + }, + "on_success": "register_clone_failure", + "on_failure": "register_clone_failure" + }, + "register_clone_success": { + "description": "Register clone as cleanup candidate for batch deletion", + "tool": "register_clone_status", + "with": { + "clone_path": "${{ context.work_dir }}", + "status": "success", + "step_name": "register_clone_success" + }, + "on_success": "done", + "on_failure": "done" + }, + "register_clone_failure": { + "description": "Register clone as preserved (error pipeline)", + "tool": "register_clone_status", + "with": { + "clone_path": "${{ context.work_dir }}", + "status": "error", + "step_name": "register_clone_failure" + }, + "on_success": "escalate_stop", + "on_failure": "escalate_stop" + }, + "done": { + "action": "stop", + "message": "Investigation complete. Fix implemented and PR opened." + }, + "escalate_stop": { + "action": "stop", + "message": "Human intervention needed. Review the latest output for details." + } + } +} diff --git a/src/autoskillit/recipes/remediation.yaml b/src/autoskillit/recipes/remediation.yaml index 3d27da2d22..5dbddf34db 100644 --- a/src/autoskillit/recipes/remediation.yaml +++ b/src/autoskillit/recipes/remediation.yaml @@ -1,7 +1,8 @@ name: remediation description: Investigate deeply, plan architectural fix, implement and open a PR. -summary: "[sprint: triage > plan > confirm > dispatch > report] | clone > investigate > rectify > dry-walkthrough > implement > test > merge (per plan part) > (audit-impl?) > (open_pr?) > push > cleanup" -autoskillit_version: "0.3.0" +summary: "bootstrap_clone > (claim_and_resolve?) > (create_and_publish?) > investigate > rectify > dry-walkthrough > implement > test > merge (per plan part) > (audit-impl?) > (open_pr?) > push > cleanup" +recipe_version: "1.0.0" +requires_packs: [github, ci, clone, telemetry] kitchen_rules: - "NEVER use native Claude Code tools (Read, Grep, Glob, Edit, Write, @@ -21,7 +22,11 @@ kitchen_rules: directly. The push step publishes the feature branch, then prepare_pr → run_arch_lenses → compose_pr opens a PR to base_branch. When open_pr=false, merges target base_branch directly and prepare_pr is skipped." - - "SOURCE ISOLATION: After clone_repo returns, the source_dir is strictly + - "When arch_lenses=true (and open_pr=true), prepare_pr routes through + run_arch_lenses before compose_pr, adding architecture lens diagrams + to the PR. By default arch_lenses is off and prepare_pr routes + directly to compose_pr." + - "SOURCE ISOLATION: After bootstrap_clone returns, the source_dir is strictly off-limits. Never run any command in source_dir — no git checkout, git fetch, git reset, git pull, run_cmd, run_skill, or any other operation. All work — skill invocations, git operations, file reads — happens @@ -51,18 +56,26 @@ ingredients: open_pr: description: PR instead of direct merge default: "true" + arch_lenses: + description: Generate architecture lens diagrams for the PR + default: "false" auto_merge: description: >- Automatically merge the PR after required checks pass — via merge queue when available, direct merge otherwise default: "true" + review_max_retries: + description: Maximum number of review-resolve retry cycles before proceeding to CI + default: "3" + local_review_rounds: + description: >- + Number of review-resolve cycles to run locally before switching to + GitHub-backed review. 0 disables local mode entirely. + type: integer + default: "3" issue_url: description: GitHub issue to close on merge required: false - sprint_mode: - description: Enable sprint sub-recipe prefix (triage → sprint selection → per-issue dispatch) - default: "false" - hidden: true upfront_claimed: description: "Set to 'true' when process-issues has already claimed this issue upfront. Allows claim_issue defense step to pass through without aborting." default: "false" @@ -74,150 +87,88 @@ ingredients: step scheduling — fast steps for all pipelines before any slow step). default: "sequential" hidden: true + max_parallel: + description: >- + Maximum number of issues to run in parallel within a single execution-map + group. Groups exceeding this cap are split into sequential sub-groups. + Passed to build-execution-map. Must be a positive integer (expressed as a + string). + default: "6" + hidden: true + depth: + description: "Investigation depth mode" + default: "standard" + hidden: true steps: - sprint_entry: - sub_recipe: sprint-prefix - gate: sprint_mode - on_success: done - on_failure: escalate_stop - note: > - Sprint sub-recipe entry point. When sprint_mode=true, the sprint-prefix - sub-recipe steps are merged here and handle the full multi-issue workflow - (triage → planning → user approval → per-issue dispatch → report). - When sprint_mode=false (default), this step is dropped at load time and - the recipe starts at 'clone' as usual. - clone: - tool: clone_repo + tool: bootstrap_clone with: source_dir: "${{ inputs.source_dir }}" run_name: "${{ inputs.run_name }}" - branch: "${{ inputs.base_branch }}" + base_branch: "${{ inputs.base_branch }}" + step_name: "bootstrap_clone" capture: - work_dir: "${{ result.clone_path }}" + work_dir: "${{ result.work_dir }}" remote_url: "${{ result.remote_url }}" - on_success: set_merge_target - on_failure: escalate_stop - note: > - Clones source_dir into ../autoskillit-runs/-/. - Captures work_dir context used by all subsequent steps as cwd. - On failure, escalate_stop is reached immediately (no clone to remove). - - set_merge_target: - tool: run_cmd - with: - cmd: "printf '%s' '${{ inputs.base_branch }}'" - cwd: "${{ context.work_dir }}" - step_name: "set_merge_target" - capture: - merge_target: "${{ result.stdout }}" - on_success: get_issue_title + merge_target: "${{ result.merge_target }}" + on_success: claim_and_resolve on_failure: escalate_stop note: > - Unconditionally sets merge_target to inputs.base_branch as a fallback. - In the standard flow (open_pr=true), the subsequent create_branch step - overrides merge_target with the feature branch name. When open_pr=false, - create_branch is skipped and this fallback stands. This eliminates the - undefined-context-variable gap that existed when create_branch was optional. + Clones source_dir and captures merge_target in one turn. + Returns work_dir, remote_url, and merge_target (set to base_branch as fallback; + overridden by create_and_publish when open_pr=true). On failure, escalate_stop + immediately (no clone to remove). - get_issue_title: - tool: get_issue_title - with: - issue_url: "${{ inputs.issue_url }}" - capture: - issue_number: "${{ result.number }}" - issue_title: "${{ result.title }}" - issue_slug: "${{ result.slug }}" - on_success: claim_issue - on_failure: escalate_stop - optional: true - skip_when_false: "inputs.issue_url" - note: > - Runs when inputs.issue_url is non-empty. Skipped otherwise. - Fetches only the issue title and slug (no body, no comments). - issue_number is used for feature branch naming and threaded to - prepare_pr for "Closes #N" in the PR body. - issue_slug provides a descriptive branch prefix (e.g. fix-merge-conflict-triage). - - claim_issue: - description: Claim the issue with in-progress label to block parallel work - tool: claim_issue + claim_and_resolve: + tool: claim_and_resolve_issue optional: true skip_when_false: inputs.issue_url with: - issue_url: ${{ inputs.issue_url }} - allow_reentry: ${{ inputs.upfront_claimed }} + issue_url: "${{ inputs.issue_url }}" + allow_reentry: "${{ inputs.upfront_claimed }}" capture: + issue_number: "${{ result.issue_number }}" + issue_title: "${{ result.issue_title }}" + issue_slug: "${{ result.issue_slug }}" claimed: "${{ result.claimed }}" on_result: - when: "${{ result.claimed }} == true" - route: compute_branch + route: create_and_publish - route: escalate_stop on_failure: escalate_stop - - compute_branch: - tool: run_cmd - with: - cmd: >- - SLUG="${{ context.issue_slug }}" && - RUN="${{ inputs.run_name }}" && - PREFIX="${SLUG:-$RUN}" && - ISSUE="${{ context.issue_number }}" && - if [ -n "$ISSUE" ]; then - printf '%s' "${PREFIX}/${ISSUE}" - else - printf '%s' "${PREFIX}/$(date +%Y%m%d)" - fi - cwd: "${{ context.work_dir }}" - step_name: "compute_branch" - capture: - proposed_branch: "${{ result.stdout | trim }}" - on_success: create_branch - on_failure: release_issue_failure - optional: true - skip_when_false: "inputs.open_pr" note: > - Computes the proposed branch name from slug/issue or slug/date — no git operations. - Result is passed to create_branch for atomic collision check and checkout. + Fetches issue title/slug and claims the issue in one turn. + Returns issue_number, issue_title, issue_slug, and claimed boolean. + When claimed=false (another session owns it), routes to escalate_stop. + issue_slug and issue_number are passed to create_and_publish for branch naming. - create_branch: - tool: create_unique_branch - with: - base_branch_name: "${{ context.proposed_branch }}" - cwd: "${{ context.work_dir }}" - capture: - merge_target: "${{ result.branch_name }}" - on_success: push_merge_target - on_failure: release_issue_failure + create_and_publish: + tool: create_and_publish_branch optional: true - skip_when_false: "inputs.open_pr" - note: > - Atomic branch creation: ls-remote collision check → checkout -b. Uses the - create_unique_branch MCP tool so the orchestrator cannot insert extra git fetch - or checkout calls between the collision check and branch creation. - - push_merge_target: - tool: push_to_remote + skip_when_false: inputs.open_pr with: - clone_path: "${{ context.work_dir }}" + issue_slug: "${{ context.issue_slug }}" + run_name: "${{ inputs.run_name }}" + issue_number: "${{ context.issue_number }}" + work_dir: "${{ context.work_dir }}" remote_url: "${{ context.remote_url }}" - branch: "${{ context.merge_target }}" + step_name: "create_and_publish_branch" + capture: + merge_target: "${{ result.merge_target }}" on_success: investigate on_failure: release_issue_failure - optional: true - skip_when_false: "inputs.open_pr" note: > - Runs by default (open_pr=true). Skipped when open_pr=false. - Publishes the feature branch to origin before any merge_worktree call. + Computes branch name, creates it (with collision suffix if needed), and pushes + to origin in one turn. Overrides merge_target with the feature branch name. merge_worktree requires refs/remotes/origin/{branch} to exist after fetch. - Without this step, merge_worktree fails with WORKTREE_INTACT_BASE_NOT_PUBLISHED. - Uses the pre-resolved remote_url from the clone step to preserve clone isolation. + When open_pr=false, this step is skipped and merge_target remains as base_branch. investigate: tool: run_skill + model: "${{ 'opus[1m]' if inputs.depth == 'deep' else 'sonnet' }}" with: - skill_command: "/autoskillit:investigate ${{ inputs.task }}" + skill_command: "/autoskillit:investigate ${{ '--depth deep ' if inputs.depth == 'deep' else '' }}${{ inputs.task }}" cwd: "${{ context.work_dir }}" step_name: "investigate" issue_url: "${{ inputs.issue_url }}" @@ -226,14 +177,19 @@ steps: investigation_path: "${{ result.investigation_path }}" on_success: rectify on_failure: release_issue_failure + on_context_limit: release_issue_failure note: > ISSUE CONTEXT: If inputs.issue_url is a non-empty string, append it to the skill_command so the investigate skill can detect and fetch the issue itself: "/autoskillit:investigate {task}\n\nGitHub Issue: {inputs.issue_url}" The skill detects the URL pattern and calls fetch_github_issue internally. + DEPTH: When inputs.depth == 'deep', passes --depth deep flag and sets model + to opus[1m] for the main skill session. Subagents within the skill always + use model: sonnet regardless of depth setting. rectify: tool: run_skill + model: "" with: skill_command: "/autoskillit:rectify ${{ context.investigation_path }}" cwd: "${{ context.work_dir }}" @@ -249,6 +205,7 @@ steps: review: tool: run_skill + model: "" with: skill_command: "/autoskillit:review-approach ${{ context.plan_path }}" cwd: "${{ context.work_dir }}" @@ -264,6 +221,7 @@ steps: dry_walkthrough: tool: run_skill + model: "" with: skill_command: "/autoskillit:dry-walkthrough ${{ context.plan_path }}" cwd: "${{ context.work_dir }}" @@ -277,6 +235,7 @@ steps: implement: tool: run_skill + model: "" stale_threshold: 2400 with: skill_command: "/autoskillit:implement-worktree-no-merge ${{ context.plan_path }}" @@ -293,6 +252,7 @@ steps: retry_worktree: tool: run_skill + model: "" stale_threshold: 2400 with: skill_command: "/autoskillit:retry-worktree ${{ context.plan_path }} ${{ context.worktree_path }}" @@ -314,11 +274,32 @@ steps: tool: test_check with: worktree_path: "${{ context.implementation_ref }}" - on_success: audit_impl + on_success: check_merge_fix_loop on_failure: assess + check_merge_fix_loop: + tool: run_python + with: + callable: "autoskillit.smoke_utils.check_loop_iteration" + current_iteration: "${{ context.merge_fix_count }}" + max_iterations: "3" + capture: + merge_fix_count: "${{ result.next_iteration }}" + on_result: + - when: "${{ result.max_exceeded }} == true" + route: release_issue_failure + - route: audit_impl + on_failure: release_issue_failure + optional_context_refs: + - merge_fix_count + note: > + Iteration guard for the merge→assess→test cycle. Caps the loop at 3 + iterations (enough for legitimate flake recovery but not enough to + burn 30+ minutes on structurally unresolvable conflicts). + assess: tool: run_skill + model: "" with: skill_command: "/autoskillit:resolve-failures ${{ context.worktree_path }} ${{ context.plan_path }} ${{ inputs.base_branch }}" cwd: "${{ context.worktree_path }}" @@ -339,8 +320,27 @@ steps: on_failure: release_issue_failure on_exhausted: release_issue_failure + rebase_conflict_fix: + tool: run_skill + model: "" + with: + skill_command: "/autoskillit:resolve-merge-conflicts ${{ context.worktree_path }} ${{ context.plan_path }} ${{ inputs.base_branch }}" + cwd: "${{ context.worktree_path }}" + step_name: "rebase_conflict_fix" + capture: + conflict_escalation_required: "${{ result.escalation_required }}" + on_result: + - when: "${{ result.escalation_required }} == true" + route: release_issue_failure + - route: test + on_failure: release_issue_failure + on_context_limit: release_issue_failure + retries: 1 + on_exhausted: release_issue_failure + audit_impl: tool: run_skill + model: "" with: skill_command: "/autoskillit:audit-impl ${{ context.plan_path }} ${{ context.branch_name }} ${{ inputs.base_branch }}" cwd: "${{ context.work_dir }}" @@ -367,6 +367,7 @@ steps: make_plan: tool: run_skill + model: "" with: skill_command: "/autoskillit:make-plan ${{ context.remediation_path }}" task: "${{ inputs.task }}" @@ -380,11 +381,10 @@ steps: on_failure: release_issue_failure commit_guard: - tool: run_cmd + tool: run_python with: - cmd: "cd ${{ context.implementation_ref }} && if [ -n \"$(git status --porcelain)\" ]; then git add -A && git commit -m 'chore: commit pending session changes'; fi" - cwd: "${{ context.implementation_ref }}" - step_name: commit_guard + callable: "autoskillit.recipe._cmd_rpc.commit_guard" + worktree_path: "${{ context.implementation_ref }}" on_success: merge on_failure: merge note: > @@ -408,11 +408,26 @@ steps: - when: "result.failed_step == 'post_rebase_test_gate'" route: assess - when: "result.failed_step == 'rebase'" - route: assess + route: rebase_conflict_fix - when: "result.error" route: release_issue_failure - - route: push + - route: next_or_done on_failure: release_issue_failure + note: > + After merge, route to next_or_done to process remaining parts before pushing. + + next_or_done: + action: route + on_result: + - when: "${{ result.next }} == more_parts" + route: dry_walkthrough + - route: push + note: > + Routing step — no tool call. The agent evaluates what comes next: + 1. If more plan_parts remain for the current plan → go to dry_walkthrough with next part. + 2. If all plan_parts are complete → go to push. + Unlike implementation.yaml (which routes done → audit_impl → push), remediation runs + audit_impl pre-merge (GO → commit_guard), so the fallthrough here goes directly to push. push: tool: push_to_remote @@ -432,6 +447,8 @@ steps: prepare_pr: tool: run_skill + model: "" + retries: 1 with: skill_command: "/autoskillit:prepare-pr ${{ context.plan_path }} ${{ inputs.run_name }} ${{ inputs.base_branch }}" cwd: "${{ context.work_dir }}" @@ -441,9 +458,15 @@ steps: prep_path: "${{ result.prep_path }}" selected_lenses: "${{ result.selected_lenses }}" lens_context_paths: "${{ result.lens_context_paths }}" - on_success: run_arch_lenses + on_result: + - when: "${{ result.prep_path }} and ${{ inputs.arch_lenses }} == 'true'" + route: run_arch_lenses + - when: "${{ result.prep_path }}" + route: compose_pr + - route: release_issue_failure on_failure: release_issue_failure - on_context_limit: release_issue_failure + on_context_limit: prepare_pr + on_exhausted: release_issue_failure optional: true skip_when_false: "inputs.open_pr" note: > @@ -454,6 +477,7 @@ steps: run_arch_lenses: tool: run_skill + model: "" with: skill_command: "/autoskillit:arch-lens-{slug} {context_path}" cwd: "${{ context.work_dir }}" @@ -478,6 +502,7 @@ steps: compose_pr: tool: run_skill + model: "" with: skill_command: "/autoskillit:compose-pr ${{ context.prep_path }} \"${{ context.all_diagram_paths }}\" ${{ context.work_dir }} ${{ inputs.base_branch }}" cwd: "${{ context.work_dir }}" @@ -519,48 +544,66 @@ steps: pr_number: "${{ context.pr_number }}" cwd: "${{ context.work_dir }}" output_dir: "${{ context.work_dir }}/{{AUTOSKILLIT_TEMP}}/review-pr" + base_branch: "${{ inputs.base_branch }}" + local_review_rounds: "${{ inputs.local_review_rounds }}" + current_iteration: "${{ context.review_loop_count }}" capture: annotated_diff_path: "${{ result.annotated_diff_path }}" hunk_ranges_path: "${{ result.hunk_ranges_path }}" + diff_metrics_path: "${{ result.diff_metrics_path }}" + review_mode: "${{ result.review_mode }}" on_success: review_pr on_failure: review_pr optional: true skip_when_false: "inputs.open_pr" + optional_context_refs: + - review_loop_count note: > Pre-computes annotated diff with [LNNN] markers and hunk ranges JSON for review-pr. On failure, falls through to review_pr which uses empty fallback. review_pr: tool: run_skill + model: "" with: - skill_command: "/autoskillit:review-pr ${{ context.merge_target }} ${{ inputs.base_branch }} annotated_diff_path=${{ context.annotated_diff_path }} hunk_ranges_path=${{ context.hunk_ranges_path }}" + skill_command: "/autoskillit:review-pr ${{ context.merge_target }} ${{ inputs.base_branch }} annotated_diff_path=${{ context.annotated_diff_path }} hunk_ranges_path=${{ context.hunk_ranges_path }} diff_metrics_path=${{ context.diff_metrics_path }} mode=${{ context.review_mode }}" cwd: "${{ context.work_dir }}" step_name: "review_pr" capture: - verdict: "${{ result.verdict }}" + review_verdict: "${{ result.verdict }}" on_result: - when: "${{ result.verdict }} == changes_requested" route: resolve_review + - when: "${{ result.verdict }} == approved_with_comments" + route: resolve_review - when: "${{ result.verdict }} == needs_human" - route: ci_watch + route: check_review_loop + - when: "${{ result.verdict }} == approved" + route: check_review_loop - when: "true" - route: ci_watch - on_failure: resolve_review - on_context_limit: ci_watch + route: check_review_loop + on_failure: check_repo_ci_event + on_context_limit: check_repo_ci_event optional: true skip_when_false: "inputs.open_pr" + optional_context_refs: + - review_loop_count + - review_mode note: > Runs after compose_pr creates the PR. Invokes the review-pr skill as a headless session to leave inline review comments on the PR. The skill finds the open PR by feature branch name (context.merge_target). Captures verdict - and routes via on_result: changes_requested → resolve_review, - needs_human → ci_watch (explicit), approved → ci_watch. On tool-level failure (MCP error, gh - unavailable), routes to resolve_review. Skipped when open_pr=false. + as review_verdict and routes via on_result: changes_requested/approved_with_comments + → resolve_review; all other verdicts (including approved, needs_human) + → check_review_loop (mandatory waypoint to increment review_loop_count). + On tool-level failure (MCP error, gh unavailable), routes to check_repo_ci_event + (no review to resolve). Skipped when open_pr=false. resolve_review: tool: run_skill + model: "" with: - skill_command: "/autoskillit:resolve-review ${{ context.merge_target }} ${{ inputs.base_branch }}" + skill_command: "/autoskillit:resolve-review ${{ context.merge_target }} ${{ inputs.base_branch }} mode=${{ context.review_mode }}" cwd: "${{ context.work_dir }}" step_name: "resolve_review" capture: @@ -573,7 +616,7 @@ steps: - when: "${{ result.verdict }} == 'already_green'" route: re_push_review - when: "${{ result.verdict }} == 'flake_suspected'" - route: release_issue_failure + route: re_push_review - when: "${{ result.verdict }} == 'ci_only_failure'" route: release_issue_failure on_failure: release_issue_failure @@ -582,9 +625,11 @@ steps: (context.work_dir) has the feature branch (context.merge_target) checked out. resolve-review fetches inline and summary review comments via the GitHub API, applies fixes for each actionable finding (critical + warning severity), and - commits each fix. Routes via on_result: verdict dispatch — real_fix/already_green - both route to re_push_review; flake_suspected/ci_only_failure escalate. - Retries up to 2 times (3 total attempts) before exhausting to release_issue_failure. + commits each fix. Routes via on_result: verdict dispatch — real_fix/already_green/ + flake_suspected all route to re_push_review (retry bounded by retries: 2); + ci_only_failure escalates to release_issue_failure. + optional_context_refs: + - review_mode re_push_review: tool: push_to_remote @@ -592,24 +637,96 @@ steps: clone_path: "${{ context.work_dir }}" remote_url: "${{ context.remote_url }}" branch: "${{ context.merge_target }}" - on_success: ci_watch + on_success: check_review_loop on_failure: release_issue_failure note: > Pushes the review-fixed feature branch back to the remote after resolve_review - succeeds. Routes to ci_watch to validate via CI (one-shot review gate). - On push failure, escalates via release_issue_failure. + succeeds. Routes to check_review_loop to determine if another review-resolve + cycle is needed (bounded by review_max_retries). On push failure, escalates + via release_issue_failure. + + check_review_loop: + tool: run_python + with: + callable: "autoskillit.smoke_utils.check_review_loop" + pr_number: "${{ context.pr_number }}" + current_iteration: "${{ context.review_loop_count }}" + max_iterations: "${{ inputs.review_max_retries }}" + previous_verdict: "${{ context.review_verdict }}" + local_review_rounds: "${{ inputs.local_review_rounds }}" + capture: + review_loop_count: "${{ result.next_iteration }}" + on_result: + - when: "${{ result.had_blocking }} == true and ${{ result.max_exceeded }} == false" + route: review_pr + - route: check_repo_ci_event + on_failure: check_repo_ci_event + skip_when_false: "inputs.open_pr" + optional_context_refs: + - review_loop_count + - review_verdict + note: > + Compound iteration + blocking guard. Routes back to review_pr only when + had_blocking=true AND max_exceeded=false. had_blocking is true when + previous_verdict=changes_requested OR (local_review_rounds > 0 AND iteration < local_review_rounds), + ensuring local review rounds are fully exhausted before exiting to CI. + + check_repo_ci_event: + tool: check_repo_merge_state + with: + branch: "${{ context.merge_target }}" + cwd: "${{ context.work_dir }}" + remote_url: "${{ context.remote_url }}" + step_name: "check_repo_ci_event" + capture: + ci_event: "${{ result.ci_event }}" + on_success: check_pr_state + on_failure: ci_watch + skip_when_false: "inputs.open_pr" + note: > + Derives ci_event for the feature branch BEFORE ci_watch runs. + Uses context.merge_target (feature branch) so push.branches filter + is evaluated against the actual branch being pushed, not the base branch. + Non-blocking: on_failure routes to ci_watch with ci_event=null (match-any). + + check_pr_state: + tool: check_pr_mergeable + with: + pr_number: "${{ context.pr_number }}" + cwd: "${{ context.work_dir }}" + on_result: + - when: "${{ result.mergeable_status }} == 'CONFLICTING'" + route: detect_ci_conflict + - route: ci_watch + on_failure: ci_watch + skip_when_false: "inputs.open_pr" + note: > + Pre-CI-watch gate: checks PR mergeable state before entering the CI + polling loop. CONFLICTING PRs cannot trigger pull_request workflows + (GitHub cannot create refs/pull/N/merge), so routing to ci_watch would + produce a guaranteed no_runs. Routes CONFLICTING to detect_ci_conflict + (which handles conflict resolution) and MERGEABLE/UNKNOWN to ci_watch. ci_watch: tool: wait_for_ci with: branch: "${{ context.merge_target }}" - timeout_seconds: 300 + event: "${{ context.ci_event }}" + timeout_seconds: 600 cwd: "${{ context.work_dir }}" remote_url: "${{ context.remote_url }}" + auto_trigger: "true" capture: ci_conclusion: "${{ result.conclusion }}" ci_failed_jobs: "${{ result.failed_jobs }}" - on_success: check_repo_merge_state + on_result: + - when: "${{ result.conclusion }} == 'success'" + route: check_repo_merge_state + - when: "${{ result.conclusion }} == 'no_runs'" + route: handle_no_ci_runs + - when: "${{ result.conclusion }} == 'timed_out'" + route: check_ci_timed_out_loop + - route: detect_ci_conflict on_failure: detect_ci_conflict optional: true skip_when_false: "inputs.open_pr" @@ -618,9 +735,166 @@ steps: using the wait_for_ci MCP tool (timeout: 300 s). Captures structured output (conclusion + failed job names) for downstream resolve_ci. Skipped when open_pr=false — no remote feature branch is opened in that mode. - On success, routes to check_repo_merge_state to determine if the PR should enter - the merge queue. On failure, routes to detect_ci_conflict which checks whether - the failure is caused by a stale base before falling through to diagnose_ci. + Routes on result.conclusion: success → check_repo_merge_state, + no_runs → handle_no_ci_runs (auto_trigger fires first: empty commit + force-push + + re-poll; handle_no_ci_runs reached only when push fails), + timed_out → check_ci_timed_out_loop (iteration guard), failure → detect_ci_conflict. + + check_ci_timed_out_loop: + tool: run_python + with: + callable: "autoskillit.smoke_utils.check_loop_iteration" + current_iteration: "${{ context.ci_timed_out_loop_count }}" + max_iterations: "2" + capture: + ci_timed_out_loop_count: "${{ result.next_iteration }}" + on_result: + - when: "${{ result.max_exceeded }} == true" + route: detect_ci_conflict + - route: ci_watch + on_failure: detect_ci_conflict + optional_context_refs: + - ci_timed_out_loop_count + note: > + Caps the ci_watch re-poll loop at 2 iterations. + After 2 timed_out results (1200s total), routes to detect_ci_conflict + instead of spinning indefinitely. + + handle_no_ci_runs: + tool: check_repo_merge_state + with: + branch: "${{ context.merge_target }}" + cwd: "${{ context.work_dir }}" + remote_url: "${{ context.remote_url }}" + step_name: "handle_no_ci_runs" + capture: + ci_event: "${{ result.ci_event }}" + on_success: check_ci_loop + on_failure: release_issue_failure + retries: 1 + on_exhausted: release_issue_failure + note: > + Intercepts no_runs conclusion from wait_for_ci. Re-derives ci_event using + check_repo_merge_state (the original event may have been wrong for this + branch's workflow configuration) and routes to check_ci_loop for iteration + guard before retrying ci_watch. retries: 1 bounds the tool-call retry; + check_ci_loop bounds the outer loop at 2 iterations. + + check_ci_loop: + tool: run_python + with: + callable: "autoskillit.smoke_utils.check_loop_iteration" + current_iteration: "${{ context.ci_loop_count }}" + max_iterations: "2" + capture: + ci_loop_count: "${{ result.next_iteration }}" + on_result: + - when: "${{ result.max_exceeded }} == true" + route: check_active_trigger_loop + - route: ci_watch + on_failure: check_ci_already_passed + optional_context_refs: + - ci_loop_count + note: > + Iteration guard for the ci_watch / handle_no_ci_runs cycle. Caps the + loop at 2 iterations (initial + 1 retry after re-deriving ci_event). + On budget exhaustion, routes to check_active_trigger_loop which caps + the active trigger to a single attempt before escalating. + + check_active_trigger_loop: + tool: run_python + with: + callable: "autoskillit.smoke_utils.check_loop_iteration" + current_iteration: "${{ context.active_trigger_count }}" + max_iterations: "2" + capture: + active_trigger_count: "${{ result.next_iteration }}" + on_result: + - when: "${{ result.max_exceeded }} == true" + route: check_ci_already_passed + - route: trigger_ci_actively + on_failure: check_ci_already_passed + optional_context_refs: + - active_trigger_count + note: > + Bounds the trigger_ci_actively remediation to a single attempt. If + ci_watch still returns no_runs after the active trigger, escalates + to escalate_stop_no_ci rather than re-triggering indefinitely. + + trigger_ci_actively: + tool: run_cmd + with: + cmd: "gh pr close ${{ context.pr_number }} && sleep 2 && gh pr reopen ${{ context.pr_number }}" + cwd: "${{ context.work_dir }}" + step_name: "trigger_ci_actively" + on_success: ci_watch + on_failure: check_ci_already_passed + retries: 1 + on_exhausted: check_ci_already_passed + note: > + Active CI trigger remediation: closes and reopens the PR to force GitHub + webhook re-delivery. This re-creates the pull_request event and the + refs/pull/N/merge ref (if the PR is mergeable). Routes back to ci_watch + for one final attempt. If the close/reopen fails or the subsequent + ci_watch still returns no_runs, escalates to escalate_stop_no_ci. + + check_ci_already_passed: + tool: run_cmd + with: + cmd: >- + gh pr view '${{ context.pr_number }}' --json statusCheckRollup + --jq '.statusCheckRollup | if length == 0 then "no_checks" elif + all(.state == "SUCCESS" or .state == "NEUTRAL" or .state == "SKIPPED") + then "passed" elif any(.state == "FAILURE" or .state == "ERROR") + then "failed" else "pending" end' + cwd: "${{ context.work_dir }}" + step_name: "check_ci_already_passed" + on_result: + - when: "${{ result.stdout | trim }} == passed" + route: check_repo_merge_state + - when: "${{ result.stdout | trim }} == failed" + route: detect_ci_conflict + - when: "${{ result.stdout | trim }} == pending" + route: ci_watch + - route: mark_issue_failed_no_ci + on_failure: mark_issue_failed_no_ci + skip_when_false: "inputs.open_pr" + note: > + Safety net before terminal escalation: checks if CI already passed + despite the watcher failing to observe it (e.g. post-force-push SHA + staleness). Uses statusCheckRollup as the authoritative signal. + Outputs passed/failed/no_checks/pending: passed goes to merge gate, + failed routes to CI conflict remediation, pending re-enters the CI + watcher, no_checks falls through to terminal stop. + + mark_issue_failed_no_ci: + description: "Swap in-progress → fail label before CI failure escalation" + tool: release_issue + optional: true + skip_when_false: inputs.issue_url + with: + issue_url: ${{ inputs.issue_url }} + fail_label: "fail" + on_success: register_clone_no_ci + on_failure: register_clone_no_ci + + register_clone_no_ci: + description: "Register clone as preserved (no-CI error pipeline)" + tool: register_clone_status + with: + clone_path: "${{ context.work_dir }}" + status: "error" + step_name: register_clone_no_ci + on_success: escalate_stop_no_ci + on_failure: escalate_stop_no_ci + + escalate_stop_no_ci: + action: stop + message: >- + CI safety net: statusCheckRollup returned no checks for PR + ${{ context.pr_number }}. This typically means no workflow is + configured to trigger on this branch pattern. Check .github/workflows/ + trigger configuration. This is NOT a test failure — CI never started. check_repo_merge_state: tool: check_repo_merge_state @@ -651,46 +925,70 @@ steps: action: route on_result: - when: "${{ inputs.auto_merge }} != 'true'" - route: register_clone_success - - when: "${{ context.queue_available }} == true and ${{ context.merge_group_trigger }} == true and ${{ context.auto_merge_available }} == true" - route: enable_auto_merge - - when: "${{ context.queue_available }} == true and ${{ context.merge_group_trigger }} == true and ${{ context.auto_merge_available }} == false" - route: queue_enqueue_no_auto + route: patch_token_summary + - when: "${{ context.queue_available }} == true and ${{ context.merge_group_trigger }} == true" + route: enqueue_to_queue - when: "${{ context.auto_merge_available }} == true" route: direct_merge - route: immediate_merge skip_when_false: "inputs.open_pr" - enable_auto_merge: - tool: run_cmd + enqueue_to_queue: + tool: enqueue_pr with: - cmd: "gh pr merge '${{ context.pr_number }}' --squash --auto" + pr_number: "${{ context.pr_number }}" + target_branch: "${{ inputs.base_branch }}" cwd: "${{ context.work_dir }}" - step_name: "enable_auto_merge" + auto_merge_available: "${{ context.auto_merge_available }}" on_success: wait_for_queue - on_failure: register_clone_success + on_failure: verify_queue_enrollment skip_when_false: "inputs.open_pr" note: > - Submits the PR to the merge queue. On failure (PR not mergeable, already merged), - degrades to register_clone_success. - - queue_enqueue_no_auto: - tool: run_cmd + Submits the PR to the merge queue via the unified enqueue_pr tool. + The tool resolves the enrollment strategy internally based on + auto_merge_available. On failure, probes the queue via + verify_queue_enrollment to distinguish a transient enqueue error from a + PR that is already merged or still in queue. + + verify_queue_enrollment: + description: "Probe merge queue after enqueue command failure to determine actual state" + tool: wait_for_merge_queue with: - cmd: "gh pr merge '${{ context.pr_number }}' --squash" + pr_number: "${{ context.pr_number }}" + target_branch: "${{ inputs.base_branch }}" cwd: "${{ context.work_dir }}" - step_name: "queue_enqueue_no_auto" - on_success: wait_for_queue - on_failure: register_clone_success - skip_when_false: "inputs.open_pr" + remote_url: "${{ context.remote_url }}" + timeout_seconds: 60 + poll_interval: 10 + on_result: + - when: "${{ result.pr_state }} == merged" + route: release_issue_success + - when: "${{ result.pr_state }} == ejected_ci_failure" + route: diagnose_ci + - when: "${{ result.pr_state }} == not_enrolled" + route: release_issue_failure + - when: "${{ result.pr_state }} == dropped_healthy" + route: wait_for_queue + - when: "${{ result.pr_state }} == dropped_merge_group_ci" + route: diagnose_ci + - when: "${{ result.pr_state }} == ejected" + route: wait_for_queue + - when: "${{ result.pr_state }} == stalled" + route: wait_for_queue + - when: "${{ result.pr_state }} == timeout" + route: register_clone_unconfirmed + - route: register_clone_unconfirmed + on_failure: register_clone_unconfirmed note: > - Submits the PR to the merge queue when the target branch has a configured - merge queue but the repository disallows auto-merge (autoMergeAllowed=false). - Uses plain `gh pr merge --squash` (no --auto, which would be rejected by - GitHub's API auto-merge gate). GitHub intercepts the call and enqueues - the PR because the branch has a queue. Routes to wait_for_queue (the - shared queue waiter) — never to wait_for_immediate_merge, whose 5-minute - poll is too short for queue serialization. + Short probe (60s) after enqueue_to_queue failure. If the PR is already + merged, route to release_issue_success. If the queue already confirms CI + failure (ejected_ci_failure or dropped_merge_group_ci), route directly to + diagnose_ci. If enrollment was attempted but the PR never appeared in the + queue (not_enrolled), route to release_issue_failure — this is a repo + configuration error. For other active queue states (dropped_healthy, ejected, + stalled), route to wait_for_queue for full 900s authoritative watch. If the + probe timed out or returned an unexpected state, escalate via + register_clone_unconfirmed. wait_for_queue: tool: wait_for_merge_queue @@ -707,16 +1005,20 @@ steps: route: release_issue_success - when: "${{ result.pr_state }} == ejected_ci_failure" route: diagnose_ci + - when: "${{ result.pr_state }} == dropped_merge_group_ci" + route: diagnose_ci + - when: "${{ result.pr_state }} == not_enrolled" + route: release_issue_failure - when: "${{ result.pr_state }} == dropped_healthy" - route: reenter_merge_queue_cheap + route: check_dropped_healthy_loop - when: "${{ result.pr_state }} == ejected" - route: queue_ejected_fix + route: check_eject_limit - when: "${{ result.pr_state }} == stalled" route: reenroll_stalled_pr - when: "${{ result.pr_state }} == timeout" - route: release_issue_timeout - - route: release_issue_timeout - on_failure: release_issue_timeout + route: register_clone_unconfirmed + - route: register_clone_unconfirmed + on_failure: register_clone_unconfirmed skip_when_false: "inputs.open_pr" note: > Waits up to 15 minutes for the PR to merge through the queue. @@ -724,32 +1026,84 @@ steps: release_issue_success is optional with skip_when_false: inputs.issue_url — when issue_url is absent the step is silently skipped and pipeline continues to register_clone_success. ejected_ci_failure → diagnose_ci (CI failure; conflict resolution cannot fix this). + not_enrolled → release_issue_failure (enrollment failed; repo configuration error). ejected → conflict fix cycle then re-enter. - stalled → lightweight re-enrollment (disable then re-enable auto-merge). - timeout → release_issue_timeout (release claim without staged label; merge unconfirmed). + stalled → lightweight re-enrollment via enqueue_pr. + timeout → register_clone_unconfirmed (release claim without staged label; merge unconfirmed). reenroll_stalled_pr: - tool: toggle_auto_merge + tool: enqueue_pr with: pr_number: "${{ context.pr_number }}" target_branch: "${{ inputs.base_branch }}" cwd: "${{ context.work_dir }}" - on_success: wait_for_queue + auto_merge_available: "${{ context.auto_merge_available }}" + on_success: check_stall_loop on_failure: release_issue_failure skip_when_false: "inputs.open_pr" + check_stall_loop: + tool: run_python + with: + callable: "autoskillit.smoke_utils.check_loop_iteration" + current_iteration: "${{ context.stall_loop_count }}" + max_iterations: "3" + capture: + stall_loop_count: "${{ result.next_iteration }}" + on_result: + - when: "${{ result.max_exceeded }} == true" + route: register_clone_unconfirmed + - route: wait_for_queue + on_failure: register_clone_unconfirmed + optional_context_refs: + - stall_loop_count + note: > + Iteration guard for the wait_for_queue / reenroll_stalled_pr cycle. + Caps stall re-enrollment at 3 attempts. On budget exhaustion, routes + to register_clone_unconfirmed (merge outcome unconfirmed). + + check_eject_limit: + tool: run_python + with: + callable: "autoskillit.recipe._cmd_rpc.check_eject_limit" + counter_file: "${{ context.work_dir }}/{{AUTOSKILLIT_TEMP}}/eject_count" + max_ejects: "3" + on_result: + - when: "${{ result.status }} == EJECT_LIMIT_EXCEEDED" + route: release_issue_failure + - route: queue_ejected_fix + on_failure: release_issue_failure + note: > + Circuit breaker for the queue ejection loop. Tracks ejection count in a + file-based counter under {{AUTOSKILLIT_TEMP}}/. After 3 ejections, routes + to release_issue_failure to prevent infinite re-enqueue cycles. + + check_dropped_healthy_loop: + tool: run_python + with: + callable: "autoskillit.recipe._cmd_rpc.check_dropped_healthy_loop" + counter_file: "${{ context.work_dir }}/{{AUTOSKILLIT_TEMP}}/dropped_healthy_count" + max_drops: "2" + on_result: + - when: "${{ result.status }} == DROPPED_LIMIT_EXCEEDED" + route: release_issue_failure + - route: reenter_merge_queue_cheap + on_failure: release_issue_failure + note: > + Circuit breaker for the dropped_healthy re-enrollment loop. After 2 consecutive + unexplained drops with healthy PR-branch CI, escalates to release_issue_failure + instead of looping. Matches the check_eject_limit pattern used for the ejected + path. DROPPED_MERGE_GROUP_CI (confirmed merge-group CI failure) is routed + directly to diagnose_ci and never reaches this step. + queue_ejected_fix: - tool: run_cmd + tool: run_python with: - cmd: > - if git fetch origin ${{ inputs.base_branch }} && git rebase origin/${{ inputs.base_branch }}; - then echo 'clean'; - else git rebase --abort 2>/dev/null || true; echo 'conflicts'; - fi - cwd: "${{ context.work_dir }}" - step_name: "queue_ejected_fix" + callable: "autoskillit.recipe._cmd_rpc.queue_ejected_fix" + work_dir: "${{ context.work_dir }}" + base_branch: "${{ inputs.base_branch }}" on_result: - - when: "${{ result.stdout | trim }} == clean" + - when: "${{ result.status }} == clean" route: re_push_queue_fix - route: resolve_queue_merge_conflicts on_failure: resolve_queue_merge_conflicts @@ -762,6 +1116,7 @@ steps: resolve_queue_merge_conflicts: tool: run_skill + model: "" with: skill_command: "/autoskillit:resolve-merge-conflicts ${{ context.work_dir }} ${{ context.plan_path }} ${{ inputs.base_branch }}" cwd: "${{ context.work_dir }}" @@ -789,35 +1144,106 @@ steps: remote_url: "${{ context.remote_url }}" branch: "${{ context.merge_target }}" force: "true" + capture: + push_error_type: "${{ result.error_type }}" + retries: 2 + on_exhausted: register_clone_unconfirmed on_success: ci_watch_post_queue_fix - on_failure: release_issue_failure + on_failure: classify_push_failure skip_when_false: "inputs.open_pr" + classify_push_failure: + action: route + on_result: + - when: "${{ context.push_error_type }} == queued_branch" + route: wait_dequeue_retry + - route: release_issue_failure + + wait_dequeue_retry: + tool: wait_for_merge_queue + with: + pr_number: "${{ context.pr_number }}" + target_branch: "${{ inputs.base_branch }}" + cwd: "${{ context.work_dir }}" + timeout_seconds: "60" + on_result: + - when: "${{ result.pr_state }} == merged" + route: release_issue_success + - when: "${{ result.pr_state }} == ejected_ci_failure" + route: register_clone_unconfirmed + - when: "${{ result.pr_state }} == dropped_merge_group_ci" + route: register_clone_unconfirmed + - when: "${{ result.pr_state }} == dropped_healthy" + route: register_clone_unconfirmed + - when: "${{ result.pr_state }} == ejected" + route: re_push_queue_fix + - when: "${{ result.pr_state }} == stalled" + route: register_clone_unconfirmed + - when: "${{ result.pr_state }} == timeout" + route: register_clone_unconfirmed + - route: register_clone_unconfirmed + on_failure: register_clone_unconfirmed + optional_context_refs: + - push_error_type + ci_watch_post_queue_fix: tool: wait_for_ci with: branch: "${{ context.merge_target }}" event: "${{ context.ci_event }}" - timeout_seconds: 300 + timeout_seconds: 600 cwd: "${{ context.work_dir }}" remote_url: "${{ context.remote_url }}" + auto_trigger: "true" capture: ci_conclusion: "${{ result.conclusion }}" ci_failed_jobs: "${{ result.failed_jobs }}" - on_success: reenter_merge_queue + on_result: + - when: "${{ result.conclusion }} == 'success'" + route: reenter_merge_queue + - when: "${{ result.conclusion }} == 'no_runs'" + route: release_issue_failure + - when: "${{ result.conclusion }} == 'timed_out'" + route: check_ci_post_queue_loop + - route: detect_ci_conflict on_failure: detect_ci_conflict skip_when_false: "inputs.open_pr" note: > Validates CI on the rebased feature branch after queue ejection fix and re-push. - Mirrors ci_watch: routes to reenter_merge_queue on CI pass, to detect_ci_conflict - on CI failure. Prevents re-entering the merge queue with a broken manifest or code. + Routes on result.conclusion: success → reenter_merge_queue, + no_runs → release_issue_failure (auto_trigger fires first: empty commit + + force-push + re-poll; no_runs reached only when trigger recovery fails), + timed_out → check_ci_post_queue_loop, failure → detect_ci_conflict. + + check_ci_post_queue_loop: + tool: run_python + with: + callable: "autoskillit.smoke_utils.check_loop_iteration" + current_iteration: "${{ context.ci_post_queue_loop_count }}" + max_iterations: "2" + capture: + ci_post_queue_loop_count: "${{ result.next_iteration }}" + on_result: + - when: "${{ result.max_exceeded }} == true" + route: release_issue_failure + - route: ci_watch_post_queue_fix + on_failure: release_issue_failure + optional_context_refs: + - ci_post_queue_loop_count + skip_when_false: "inputs.open_pr" + note: > + Iteration guard for the ci_watch_post_queue_fix timed_out self-loop. + Caps at 2 iterations (initial + 1 retry). On budget exhaustion, + routes to release_issue_failure — this is already a post-ejection + retry path so further escalation is not warranted. reenter_merge_queue: - tool: run_cmd + tool: enqueue_pr with: - cmd: "gh pr merge '${{ context.pr_number }}' --squash --auto" + pr_number: "${{ context.pr_number }}" + target_branch: "${{ inputs.base_branch }}" cwd: "${{ context.work_dir }}" - step_name: "reenter_merge_queue" + auto_merge_available: "${{ context.auto_merge_available }}" on_success: wait_for_queue on_failure: release_issue_failure skip_when_false: "inputs.open_pr" @@ -826,11 +1252,12 @@ steps: If re-entry fails, release_issue_failure preserves the clone for debugging. reenter_merge_queue_cheap: - tool: run_cmd + tool: enqueue_pr with: - cmd: "gh pr merge '${{ context.pr_number }}' --squash --auto" + pr_number: "${{ context.pr_number }}" + target_branch: "${{ inputs.base_branch }}" cwd: "${{ context.work_dir }}" - step_name: "reenter_merge_queue_cheap" + auto_merge_available: "${{ context.auto_merge_available }}" on_success: wait_for_queue on_failure: release_issue_failure skip_when_false: "inputs.open_pr" @@ -846,51 +1273,43 @@ steps: cwd: "${{ context.work_dir }}" step_name: "direct_merge" on_success: wait_for_direct_merge - on_failure: register_clone_success + on_failure: release_issue_failure skip_when_false: "inputs.open_pr" note: > Enables GitHub native auto-merge for direct merge (no merge queue). GitHub - merges automatically once all required checks pass. On failure (PR already - merged, not mergeable), degrades gracefully to register_clone_success. + merges automatically once all required checks pass. On failure, escalates + via release_issue_failure so the issue claim is released and the pipeline + reports an error rather than silently succeeding. wait_for_direct_merge: - tool: run_cmd + tool: run_python with: - cmd: > - for i in $(seq 1 90); do - STATE=$(gh pr view '${{ context.pr_number }}' --json state --jq '.state'); - if [ "$STATE" = "MERGED" ]; then echo "merged"; exit 0; fi; - if [ "$STATE" = "CLOSED" ]; then echo "closed"; exit 0; fi; - sleep 10; - done; echo "timeout" - cwd: "${{ context.work_dir }}" - step_name: "wait_for_direct_merge" + callable: "autoskillit.recipe._cmd_rpc.wait_for_direct_merge" + pr_number: "${{ context.pr_number }}" + max_polls: "90" + poll_interval: "10" on_result: - - when: "${{ result.stdout | trim }} == merged" + - when: "${{ result.state }} == merged" route: release_issue_success - - when: "${{ result.stdout | trim }} == closed" + - when: "${{ result.state }} == closed" route: direct_merge_conflict_fix - - route: release_issue_timeout - on_failure: register_clone_success + - route: register_clone_unconfirmed + on_failure: register_clone_unconfirmed skip_when_false: "inputs.open_pr" note: > Polls GitHub PR state every 10 s for up to 15 minutes. merged → release_issue_success. closed → conflict fix cycle. - timeout → release_issue_timeout (releases claim; merge unconfirmed). + timeout → register_clone_unconfirmed (releases claim; merge unconfirmed). direct_merge_conflict_fix: - tool: run_cmd + tool: run_python with: - cmd: > - if git fetch origin ${{ inputs.base_branch }} && git rebase origin/${{ inputs.base_branch }}; - then echo 'clean'; - else git rebase --abort 2>/dev/null || true; echo 'conflicts'; - fi - cwd: "${{ context.work_dir }}" - step_name: "direct_merge_conflict_fix" + callable: "autoskillit.recipe._cmd_rpc.direct_merge_conflict_fix" + work_dir: "${{ context.work_dir }}" + base_branch: "${{ inputs.base_branch }}" on_result: - - when: "${{ result.stdout | trim }} == clean" + - when: "${{ result.status }} == clean" route: re_push_direct_fix - route: resolve_direct_merge_conflicts on_failure: resolve_direct_merge_conflicts @@ -902,6 +1321,7 @@ steps: resolve_direct_merge_conflicts: tool: run_skill + model: "" with: skill_command: "/autoskillit:resolve-merge-conflicts ${{ context.work_dir }} ${{ context.plan_path }} ${{ inputs.base_branch }}" cwd: "${{ context.work_dir }}" @@ -954,33 +1374,29 @@ steps: cwd: "${{ context.work_dir }}" step_name: "immediate_merge" on_success: wait_for_immediate_merge - on_failure: register_clone_success + on_failure: release_issue_failure skip_when_false: "inputs.open_pr" note: > Merges the PR immediately using plain --squash when autoMergeAllowed=false. Unlike direct_merge, no --auto flag is used — GitHub executes the merge - synchronously rather than queueing it for when checks pass. On failure - (conflict or not-mergeable error from GitHub), degrades to register_clone_success. + synchronously rather than queueing it for when checks pass. On failure, + escalates via release_issue_failure so the issue claim is released and the + pipeline reports an error rather than silently succeeding. wait_for_immediate_merge: - tool: run_cmd + tool: run_python with: - cmd: > - for i in $(seq 1 30); do - STATE=$(gh pr view '${{ context.pr_number }}' --json state --jq '.state'); - if [ "$STATE" = "MERGED" ]; then echo "merged"; exit 0; fi; - if [ "$STATE" = "CLOSED" ]; then echo "closed"; exit 0; fi; - sleep 10; - done; echo "timeout" - cwd: "${{ context.work_dir }}" - step_name: "wait_for_immediate_merge" + callable: "autoskillit.recipe._cmd_rpc.wait_for_immediate_merge" + pr_number: "${{ context.pr_number }}" + max_polls: "30" + poll_interval: "10" on_result: - - when: "${{ result.stdout | trim }} == merged" + - when: "${{ result.state }} == merged" route: release_issue_success - - when: "${{ result.stdout | trim }} == closed" + - when: "${{ result.state }} == closed" route: immediate_merge_conflict_fix - - route: release_issue_timeout - on_failure: release_issue_timeout + - route: register_clone_unconfirmed + on_failure: register_clone_unconfirmed skip_when_false: "inputs.open_pr" note: > Polls GitHub PR state every 10s for up to 5 minutes (30 iterations). @@ -988,20 +1404,16 @@ steps: synchronously — this poll is a confirmation window. merged → release_issue_success. closed → PR was rejected (stale base or conflict) — route to conflict fix. - timeout → release_issue_timeout. + timeout → register_clone_unconfirmed. immediate_merge_conflict_fix: - tool: run_cmd + tool: run_python with: - cmd: > - if git fetch origin ${{ inputs.base_branch }} && git rebase origin/${{ inputs.base_branch }}; - then echo 'clean'; - else git rebase --abort 2>/dev/null || true; echo 'conflicts'; - fi - cwd: "${{ context.work_dir }}" - step_name: "immediate_merge_conflict_fix" + callable: "autoskillit.recipe._cmd_rpc.immediate_merge_conflict_fix" + work_dir: "${{ context.work_dir }}" + base_branch: "${{ inputs.base_branch }}" on_result: - - when: "${{ result.stdout | trim }} == clean" + - when: "${{ result.status }} == clean" route: re_push_immediate_fix - route: resolve_immediate_merge_conflicts on_failure: resolve_immediate_merge_conflicts @@ -1013,6 +1425,7 @@ steps: resolve_immediate_merge_conflicts: tool: run_skill + model: "" with: skill_command: "/autoskillit:resolve-merge-conflicts ${{ context.work_dir }} ${{ context.plan_path }} ${{ inputs.base_branch }}" cwd: "${{ context.work_dir }}" @@ -1060,6 +1473,7 @@ steps: diagnose_ci: tool: run_skill + model: "" with: skill_command: "/autoskillit:diagnose-ci ${{ context.merge_target }} - - - ${{ context.ci_event }}" cwd: "${{ context.work_dir }}" @@ -1080,6 +1494,7 @@ steps: resolve_ci: tool: run_skill + model: "" with: skill_command: "/autoskillit:resolve-failures ${{ context.work_dir }} ${{ context.plan_path }} ${{ inputs.base_branch }} ${{ context.ci_conclusion }} ${{ context.ci_failed_jobs }} ${{ context.diagnosis_path }}" cwd: "${{ context.work_dir }}" @@ -1095,7 +1510,7 @@ steps: - when: "${{ result.verdict }} == 'already_green'" route: pre_resolve_rebase - when: "${{ result.verdict }} == 'flake_suspected'" - route: release_issue_failure + route: re_push - when: "${{ result.verdict }} == 'ci_only_failure'" route: release_issue_failure - route: release_issue_failure @@ -1104,13 +1519,17 @@ steps: Runs when ci_watch reports a CI failure. Receives ci_failed_jobs from wait_for_ci structured output to avoid redundant CI failure re-investigation. Routes via on_result: verdict dispatch — real_fix pushes, already_green rebases and re-diagnoses - (sibling fix may have landed), flake_suspected/ci_only_failure escalate to human. + (sibling fix may have landed), flake_suspected retries via re_push (bounded by + retries: 2 / on_exhausted: release_issue_failure), ci_only_failure escalates to human. Retries up to 2 times (3 total attempts) before exhausting to release_issue_failure. pre_resolve_rebase: tool: run_cmd with: - cmd: "git -C ${{ context.work_dir }} fetch origin && git -C ${{ context.work_dir }} rebase origin/${{ inputs.base_branch }}" + cmd: | + REMOTE=$(git -C ${{ context.work_dir }} remote get-url upstream 2>/dev/null | grep -qv "^file://" && echo upstream || echo origin) && + git -C ${{ context.work_dir }} fetch "$REMOTE" && + git -C ${{ context.work_dir }} rebase "$REMOTE"/${{ inputs.base_branch }} step_name: "pre_resolve_rebase" retries: 1 on_success: diagnose_ci @@ -1128,17 +1547,20 @@ steps: remote_url: "${{ context.remote_url }}" branch: "${{ context.merge_target }}" force: "true" - on_success: ci_watch + on_success: check_repo_ci_event on_failure: release_issue_failure note: > Pushes the CI-fixed feature branch back to the remote after resolve_ci succeeds. - Routes back to ci_watch to verify the fix resolves CI. On push failure, escalates - via release_issue_failure. + Routes to check_repo_ci_event to derive ci_event before ci_watch verifies the fix. + On push failure, escalates via release_issue_failure. detect_ci_conflict: tool: run_cmd with: - cmd: "git fetch origin ${{ inputs.base_branch }} && ! git merge-base --is-ancestor origin/${{ inputs.base_branch }} HEAD" + cmd: | + REMOTE=$(git remote get-url upstream 2>/dev/null | grep -qv "^file://" && echo upstream || echo origin) && + git fetch "$REMOTE" ${{ inputs.base_branch }} && + ! git merge-base --is-ancestor "$REMOTE"/${{ inputs.base_branch }} HEAD cwd: "${{ context.work_dir }}" step_name: "detect_ci_conflict" on_success: ci_conflict_fix @@ -1154,6 +1576,7 @@ steps: ci_conflict_fix: tool: run_skill + model: "" with: skill_command: "/autoskillit:resolve-merge-conflicts ${{ context.work_dir }} ${{ context.plan_path }} ${{ inputs.base_branch }}" cwd: "${{ context.work_dir }}" @@ -1176,20 +1599,20 @@ steps: (LOW confidence conflicts) or tool failure, routes to release_issue_failure. retries: 1 bounds the loop; after 2 total attempts, escalates. - release_issue_timeout: - description: "Release in-progress claim after direct merge timeout without applying staged label" - tool: release_issue - optional: true - skip_when_false: inputs.issue_url + register_clone_unconfirmed: + description: "Register clone as preserved — merge unconfirmed, in-progress label retained" + tool: register_clone_status with: - issue_url: ${{ inputs.issue_url }} - on_success: register_clone_success - on_failure: register_clone_success + clone_path: "${{ context.work_dir }}" + status: "unconfirmed" + step_name: register_clone_unconfirmed + on_success: done + on_failure: done note: > - Called when wait_for_direct_merge times out. Releases the in-progress - claim without passing target_branch, so the staged label is not applied. The - merge is unconfirmed; a human must verify PR state independently. - Routes to register_clone_success so the pipeline completes cleanly. + Called when a merge-wait step times out before merge is confirmed. + Does NOT call release_issue — the in-progress label stays on the issue + so operators can see the PR is still actively queued. The clone is + preserved (status: unconfirmed) for human inspection. release_issue_success: description: "Transition issue from in-progress to staged after successful pipeline completion" @@ -1199,12 +1622,27 @@ steps: with: issue_url: ${{ inputs.issue_url }} target_branch: ${{ inputs.base_branch }} - on_success: register_clone_success - on_failure: register_clone_success + on_success: patch_token_summary + on_failure: patch_token_summary note: > Applies staged label after successful PR merge; skipped when no issue_url. optional=true so registration proceeds even if this step errors. + patch_token_summary: + description: "Collect all pipeline token data and patch the PR body with complete summary" + tool: run_python + optional: true + skip_when_false: "inputs.open_pr" + with: + callable: "autoskillit.smoke_utils.patch_pr_token_summary" + timeout: 60 + pr_url: "${{ context.pr_url }}" + cwd: "${{ context.work_dir }}" + on_success: register_clone_success + on_failure: register_clone_success + note: > + Patches PR body with complete token summary; failure never blocks pipeline completion. + release_issue_failure: description: "Release the in-progress claim after pipeline failure" tool: release_issue @@ -1212,6 +1650,7 @@ steps: skip_when_false: inputs.issue_url with: issue_url: ${{ inputs.issue_url }} + fail_label: "fail" on_success: register_clone_failure on_failure: register_clone_failure diff --git a/src/autoskillit/recipes/research-archive.json b/src/autoskillit/recipes/research-archive.json new file mode 100644 index 0000000000..aa45322092 --- /dev/null +++ b/src/autoskillit/recipes/research-archive.json @@ -0,0 +1,146 @@ +{ + "name": "research-archive", + "recipe_version": "1.0.0", + "categories": [ + "research-family" + ], + "requires_packs": [ + "github", + "ci" + ], + "description": "Archival-phase sub-recipe extracted from research.yaml. Separates research artifacts from experimental code, opens an artifact-only PR containing only the research/ directory, tags the experiment branch for preservation, closes the original PR with cross-references, and patches the token summary. All steps are best-effort — failures route to patch_token_summary for graceful degradation.\n", + "summary": "begin_archival > capture > branch > pr > tag > close > patch > complete | escalate", + "ingredients": { + "worktree_path": { + "description": "Path to the pre-created worktree (passed by campaign dispatch)", + "required": true, + "hidden": true + }, + "research_dir": { + "description": "Path to the research/ directory in the worktree", + "required": true, + "hidden": true + }, + "pr_url": { + "description": "URL of the original experiment PR (passed by campaign dispatch)", + "required": true, + "hidden": true + }, + "all_diagram_paths": { + "description": "Paths to all generated diagrams (passed by campaign dispatch)", + "required": true, + "hidden": true + }, + "report_path_after_finalize": { + "description": "Path to the finalized research report (passed by campaign dispatch)", + "required": true, + "hidden": true + }, + "base_branch": { + "description": "Branch to target for the artifact PR", + "default": "main" + } + }, + "kitchen_rules": [ + "NEVER use native Claude Code tools from the orchestrator: Read, Grep, Glob, Edit, Write, Bash, Agent, WebFetch, WebSearch, NotebookEdit are ALL forbidden. All work is delegated through run_cmd, run_python.\n", + "All archival steps are best-effort — route to patch_token_summary on failure.", + "Do NOT auto-merge the artifact PR — it is for human review only." + ], + "steps": { + "begin_archival": { + "action": "route", + "on_result": [ + { + "when": "${{ inputs.pr_url }}", + "route": "capture_experiment_branch" + }, + { + "route": "patch_token_summary" + } + ], + "note": "Gate step for the archival phase. Routes to artifact extraction when a PR URL is available, otherwise skips directly to patch_token_summary.\n" + }, + "capture_experiment_branch": { + "tool": "run_cmd", + "with": { + "cmd": "git -C '${{ inputs.worktree_path }}' rev-parse --abbrev-ref HEAD", + "cwd": "${{ inputs.worktree_path }}", + "step_name": "capture_experiment_branch" + }, + "capture": { + "experiment_branch": "${{ result.stdout | trim }}" + }, + "on_success": "create_artifact_branch", + "on_failure": "patch_token_summary" + }, + "create_artifact_branch": { + "tool": "run_cmd", + "with": { + "cmd": "bash scripts/recipe/create_artifact_branch.sh '${{ inputs.worktree_path }}' '${{ context.experiment_branch }}' '${{ inputs.base_branch }}' '${{ inputs.research_dir }}'", + "cwd": "${{ inputs.worktree_path }}", + "step_name": "create_artifact_branch" + }, + "capture": { + "artifact_branch": "${{ result.artifact_branch }}" + }, + "on_success": "open_artifact_pr", + "on_failure": "patch_token_summary" + }, + "open_artifact_pr": { + "tool": "run_cmd", + "with": { + "cmd": "bash scripts/recipe/open_artifact_pr.sh '${{ inputs.worktree_path }}' '${{ context.experiment_branch }}' '${{ context.artifact_branch }}' '${{ inputs.pr_url }}' '${{ inputs.base_branch }}' '${{ inputs.research_dir }}'", + "cwd": "${{ inputs.worktree_path }}", + "step_name": "open_artifact_pr" + }, + "capture": { + "artifact_pr_url": "${{ result.artifact_pr_url }}" + }, + "on_success": "tag_experiment_branch", + "on_failure": "patch_token_summary" + }, + "tag_experiment_branch": { + "tool": "run_cmd", + "with": { + "cmd": "bash scripts/recipe/tag_experiment_branch.sh '${{ inputs.worktree_path }}' '${{ context.experiment_branch }}' '${{ context.artifact_pr_url }}'", + "cwd": "${{ inputs.worktree_path }}", + "step_name": "tag_experiment_branch" + }, + "capture": { + "archive_tag": "${{ result.archive_tag }}" + }, + "on_success": "close_experiment_pr", + "on_failure": "patch_token_summary" + }, + "close_experiment_pr": { + "tool": "run_cmd", + "with": { + "cmd": "ARTIFACT_PR=\"${{ context.artifact_pr_url }}\" && ARCHIVE_TAG=\"${{ context.archive_tag }}\" && CLOSE_BODY=$(printf '## Archived\\n\\nThis PR contained both research artifacts and experimental code.\\n\\n**Research record merged via:** %s\\n**Full experiment branch preserved at:** `%s`\\n\\nThe artifact PR contains only files under `research/` (reports, scripts, results). The archive tag preserves the full experiment branch including variant code for reference.\\n\\nTo restore the full experiment: `git checkout %s`' \"${ARTIFACT_PR}\" \"${ARCHIVE_TAG}\" \"${ARCHIVE_TAG}\") && gh pr close '${{ inputs.pr_url }}' --comment \"${CLOSE_BODY}\"\n", + "cwd": "${{ inputs.worktree_path }}", + "step_name": "close_experiment_pr" + }, + "on_success": "patch_token_summary", + "on_failure": "patch_token_summary" + }, + "patch_token_summary": { + "description": "Collect all pipeline token data and patch the PR body with complete summary", + "tool": "run_python", + "with": { + "callable": "autoskillit.smoke_utils.patch_pr_token_summary", + "timeout": 60, + "pr_url": "${{ inputs.pr_url }}", + "cwd": "${{ inputs.worktree_path }}" + }, + "on_success": "research_complete", + "on_failure": "escalate_stop" + }, + "research_complete": { + "action": "stop", + "message": "Research archival complete. Artifacts separated into a dedicated PR, experiment branch tagged and preserved, original PR closed with cross-references to the artifact PR and archive tag.\n" + }, + "escalate_stop": { + "action": "stop", + "message": "Research archival pipeline failed — human intervention needed. Check the latest step output for details." + } + } +} diff --git a/src/autoskillit/recipes/research-archive.yaml b/src/autoskillit/recipes/research-archive.yaml new file mode 100644 index 0000000000..571b138cf3 --- /dev/null +++ b/src/autoskillit/recipes/research-archive.yaml @@ -0,0 +1,138 @@ +name: research-archive +recipe_version: "1.0.0" +categories: [research-family] +requires_packs: [github, ci] +description: > + Archival-phase sub-recipe extracted from research.yaml. Separates research + artifacts from experimental code, opens an artifact-only PR containing only + the research/ directory, tags the experiment branch for preservation, closes + the original PR with cross-references, and patches the token summary. All + steps are best-effort — failures route to patch_token_summary for graceful + degradation. +summary: begin_archival > capture > branch > pr > tag > close > patch > complete | escalate + +ingredients: + # --- campaign-sourced (hidden) --- + worktree_path: + description: Path to the pre-created worktree (passed by campaign dispatch) + required: true + hidden: true + research_dir: + description: Path to the research/ directory in the worktree + required: true + hidden: true + pr_url: + description: URL of the original experiment PR (passed by campaign dispatch) + required: true + hidden: true + all_diagram_paths: + description: Paths to all generated diagrams (passed by campaign dispatch) + required: true + hidden: true + report_path_after_finalize: + description: Path to the finalized research report (passed by campaign dispatch) + required: true + hidden: true + + # --- user-input --- + base_branch: + description: Branch to target for the artifact PR + default: "main" + +kitchen_rules: + - > + NEVER use native Claude Code tools from the orchestrator: Read, Grep, Glob, + Edit, Write, Bash, Agent, WebFetch, WebSearch, NotebookEdit are ALL forbidden. + All work is delegated through run_cmd, run_python. + - "All archival steps are best-effort — route to patch_token_summary on failure." + - "Do NOT auto-merge the artifact PR — it is for human review only." + +steps: + begin_archival: + action: route + on_result: + - when: "${{ inputs.pr_url }}" + route: capture_experiment_branch + - route: patch_token_summary + note: > + Gate step for the archival phase. Routes to artifact extraction when + a PR URL is available, otherwise skips directly to patch_token_summary. + + capture_experiment_branch: + tool: run_cmd + with: + cmd: "git -C '${{ inputs.worktree_path }}' rev-parse --abbrev-ref HEAD" + cwd: "${{ inputs.worktree_path }}" + step_name: capture_experiment_branch + capture: + experiment_branch: "${{ result.stdout | trim }}" + on_success: create_artifact_branch + on_failure: patch_token_summary + + create_artifact_branch: + tool: run_cmd + with: + cmd: "bash scripts/recipe/create_artifact_branch.sh '${{ inputs.worktree_path }}' '${{ context.experiment_branch }}' '${{ inputs.base_branch }}' '${{ inputs.research_dir }}'" + cwd: "${{ inputs.worktree_path }}" + step_name: create_artifact_branch + capture: + artifact_branch: "${{ result.artifact_branch }}" + on_success: open_artifact_pr + on_failure: patch_token_summary + + open_artifact_pr: + tool: run_cmd + with: + cmd: "bash scripts/recipe/open_artifact_pr.sh '${{ inputs.worktree_path }}' '${{ context.experiment_branch }}' '${{ context.artifact_branch }}' '${{ inputs.pr_url }}' '${{ inputs.base_branch }}' '${{ inputs.research_dir }}'" + cwd: "${{ inputs.worktree_path }}" + step_name: open_artifact_pr + capture: + artifact_pr_url: "${{ result.artifact_pr_url }}" + on_success: tag_experiment_branch + on_failure: patch_token_summary + + tag_experiment_branch: + tool: run_cmd + with: + cmd: "bash scripts/recipe/tag_experiment_branch.sh '${{ inputs.worktree_path }}' '${{ context.experiment_branch }}' '${{ context.artifact_pr_url }}'" + cwd: "${{ inputs.worktree_path }}" + step_name: tag_experiment_branch + capture: + archive_tag: "${{ result.archive_tag }}" + on_success: close_experiment_pr + on_failure: patch_token_summary + + close_experiment_pr: + tool: run_cmd + with: + cmd: > + ARTIFACT_PR="${{ context.artifact_pr_url }}" && + ARCHIVE_TAG="${{ context.archive_tag }}" && + CLOSE_BODY=$(printf '## Archived\n\nThis PR contained both research artifacts and experimental code.\n\n**Research record merged via:** %s\n**Full experiment branch preserved at:** `%s`\n\nThe artifact PR contains only files under `research/` (reports, scripts, results). The archive tag preserves the full experiment branch including variant code for reference.\n\nTo restore the full experiment: `git checkout %s`' "${ARTIFACT_PR}" "${ARCHIVE_TAG}" "${ARCHIVE_TAG}") && + gh pr close '${{ inputs.pr_url }}' --comment "${CLOSE_BODY}" + cwd: "${{ inputs.worktree_path }}" + step_name: close_experiment_pr + on_success: patch_token_summary + on_failure: patch_token_summary + + patch_token_summary: + description: "Collect all pipeline token data and patch the PR body with complete summary" + tool: run_python + with: + callable: "autoskillit.smoke_utils.patch_pr_token_summary" + timeout: 60 + pr_url: "${{ inputs.pr_url }}" + cwd: "${{ inputs.worktree_path }}" + on_success: research_complete + on_failure: escalate_stop + + research_complete: + action: stop + message: > + Research archival complete. Artifacts separated into a dedicated PR, + experiment branch tagged and preserved, original PR closed with + cross-references to the artifact PR and archive tag. + + escalate_stop: + action: stop + message: "Research archival pipeline failed — human intervention needed. Check the latest step output for details." \ No newline at end of file diff --git a/src/autoskillit/recipes/research-design.json b/src/autoskillit/recipes/research-design.json new file mode 100644 index 0000000000..6cd31185bf --- /dev/null +++ b/src/autoskillit/recipes/research-design.json @@ -0,0 +1,204 @@ +{ + "name": "research-design", + "recipe_version": "1.0.0", + "categories": [ + "research-family" + ], + "requires_packs": [ + "research", + "vis-lens" + ], + "description": "Design-phase sub-recipe extracted from research.yaml. Scopes a research question, plans an experiment, optionally reviews the design (bounded by check_design_review_loop, max 3 iterations), plans visualizations, then terminates with a cross-dispatch handoff sentinel containing the design artifacts. Dispatched as a food truck by campaign recipes that separate design from execution.\n", + "summary": "scope > plan > [review?] > visualize > design_complete", + "ingredients": { + "task": { + "description": "Research question or topic to investigate", + "required": true + }, + "source_dir": { + "description": "Path to the project root", + "required": true + }, + "base_branch": { + "description": "Branch to target for the research PR", + "default": "main" + }, + "review_design": { + "description": "When 'true' (the default), run the review-design skill after plan_experiment to validate the experiment plan before execution. Emits a verdict: GO proceeds, REVISE loops back to plan_experiment with revision guidance (bounded by check_design_review_loop at 3 iterations), STOP triggers resolve_design_review for triage — addressable findings loop back, structural findings halt the pipeline. When 'false', skip directly to plan_visualization.\n", + "default": "true" + }, + "issue_url": { + "description": "Optional URL to an existing GitHub issue describing the research goal. Provided as context to scope and plan_experiment.\n", + "required": false + } + }, + "kitchen_rules": [ + "NEVER use native Claude Code tools from the orchestrator: Read, Grep, Glob, Edit, Write, Bash, Agent, WebFetch, WebSearch, NotebookEdit are ALL forbidden. All work is delegated through run_skill, run_cmd, run_python.\n", + "This recipe is a food truck dispatched by a campaign. Emit the L3 sentinel JSON block exactly as instructed in the design_complete step message — no extra tool calls after reaching design_complete.\n" + ], + "steps": { + "scope": { + "tool": "run_skill", + "model": "", + "with": { + "skill_command": "/autoskillit:scope ${{ inputs.task }}", + "cwd": "${{ inputs.source_dir }}", + "step_name": "scope" + }, + "capture": { + "scope_report": "${{ result.scope_report }}" + }, + "on_success": "plan_experiment", + "on_failure": "escalate_stop" + }, + "plan_experiment": { + "tool": "run_skill", + "model": "", + "with": { + "skill_command": "/autoskillit:plan-experiment ${{ context.scope_report }} ${{ context.revision_guidance }}", + "cwd": "${{ inputs.source_dir }}", + "step_name": "plan_experiment" + }, + "optional_context_refs": [ + "revision_guidance" + ], + "capture": { + "experiment_plan": "${{ result.experiment_plan }}" + }, + "on_success": "review_design", + "on_failure": "escalate_stop" + }, + "review_design": { + "tool": "run_skill", + "model": "", + "with": { + "skill_command": "/autoskillit:review-design ${{ context.experiment_plan }}", + "cwd": "${{ inputs.source_dir }}", + "step_name": "review_design" + }, + "capture": { + "verdict": "${{ result.verdict }}", + "experiment_type": "${{ result.experiment_type }}", + "evaluation_dashboard": "${{ result.evaluation_dashboard }}", + "revision_guidance": "${{ result.revision_guidance }}" + }, + "skip_when_false": "inputs.review_design", + "retries": 2, + "on_exhausted": "plan_visualization", + "on_context_limit": "plan_visualization", + "on_failure": "plan_visualization", + "on_result": [ + { + "when": "${{ result.verdict }} == GO", + "route": "plan_visualization" + }, + { + "when": "${{ result.verdict }} == REVISE", + "route": "revise_design" + }, + { + "when": "${{ result.verdict }} == STOP", + "route": "resolve_design_review" + }, + { + "route": "plan_visualization" + } + ] + }, + "plan_visualization": { + "tool": "run_skill", + "model": "", + "with": { + "skill_command": "/autoskillit:plan-visualization ${{ inputs.source_dir }} ${{ context.experiment_plan }} ${{ context.scope_report }}", + "cwd": "${{ inputs.source_dir }}", + "step_name": "plan_visualization" + }, + "capture": { + "visualization_plan_path": "${{ result.visualization_plan_path }}", + "report_plan_path": "${{ result.report_plan_path }}" + }, + "on_success": "design_complete", + "on_failure": "escalate_stop", + "note": "Selects 2–4 vis-lens skills via three-tier logic, runs them in parallel, and synthesizes outputs into visualization-plan.md and report-plan.md. Runs before worktree creation so outputs can be committed into the research dir.\n" + }, + "revise_design": { + "action": "route", + "on_result": [ + { + "route": "check_design_review_loop" + } + ] + }, + "check_design_review_loop": { + "tool": "run_python", + "with": { + "callable": "autoskillit.smoke_utils.check_loop_iteration", + "current_iteration": "${{ context.design_review_count }}", + "max_iterations": "3" + }, + "capture": { + "design_review_count": "${{ result.next_iteration }}" + }, + "on_result": [ + { + "when": "${{ result.max_exceeded }} == true", + "route": "plan_visualization" + }, + { + "route": "plan_experiment" + } + ], + "on_failure": "plan_visualization", + "optional_context_refs": [ + "design_review_count" + ], + "note": "Iteration guard for the review_design → revise_design → plan_experiment cycle. Caps design revision loops at 3 iterations. Both REVISE and STOP→resolve→revised paths flow through revise_design and hit this guard. On exhaustion, proceeds to plan_visualization with the best available design.\n" + }, + "resolve_design_review": { + "tool": "run_skill", + "model": "", + "with": { + "skill_command": "/autoskillit:resolve-design-review ${{ context.evaluation_dashboard }} ${{ context.experiment_plan }} ${{ context.revision_guidance }}", + "cwd": "${{ inputs.source_dir }}", + "step_name": "resolve_design_review" + }, + "optional_context_refs": [ + "evaluation_dashboard", + "experiment_plan", + "revision_guidance" + ], + "capture": { + "revision_guidance": "${{ result.revision_guidance }}" + }, + "retries": 1, + "on_failure": "design_rejected", + "on_context_limit": "design_rejected", + "on_result": [ + { + "when": "${{ result.resolution }} == revised", + "route": "revise_design" + }, + { + "when": "${{ result.resolution }} == failed", + "route": "design_rejected" + }, + { + "route": "design_rejected" + } + ], + "note": "Triages STOP-verdict findings from review_design. ADDRESSABLE/DISCUSS findings generate revision_guidance and loop back. Only all-STRUCTURAL findings halt at design_rejected.\n" + }, + "design_rejected": { + "action": "stop", + "message": "Research pipeline halted: experiment design rejected (verdict=STOP, resolution=failed). All stop-trigger findings are structural — revision cannot address the identified flaws. Check {{AUTOSKILLIT_TEMP}}/resolve-design-review/ for the triage report and {{AUTOSKILLIT_TEMP}}/review-design/ for the evaluation dashboard.\n" + }, + "design_complete": { + "action": "stop", + "message": "Design phase complete. Emit the L3 result sentinel JSON block now. The sentinel must include the following fields from captured context: success=true, scope_report=, experiment_plan=, visualization_plan_path=, report_plan_path=, experiment_type=. Example sentinel: {\"success\": true, \"scope_report\": \"\", \"experiment_plan\": \"\", \"visualization_plan_path\": \"\", \"report_plan_path\": \"\", \"experiment_type\": \"benchmark\"}" + }, + "escalate_stop": { + "action": "stop", + "message": "Research pipeline failed — human intervention needed. Check the latest step output for details." + } + } +} diff --git a/src/autoskillit/recipes/research-design.yaml b/src/autoskillit/recipes/research-design.yaml new file mode 100644 index 0000000000..02f9d42c1c --- /dev/null +++ b/src/autoskillit/recipes/research-design.yaml @@ -0,0 +1,193 @@ +name: research-design +recipe_version: "1.0.0" +categories: [research-family] +requires_packs: [research, vis-lens] +description: > + Design-phase sub-recipe extracted from research.yaml. Scopes a research + question, plans an experiment, optionally reviews the design (bounded by + check_design_review_loop, max 3 iterations), plans visualizations, then + terminates with a cross-dispatch handoff sentinel containing the design + artifacts. Dispatched as a food truck by campaign recipes that separate + design from execution. +summary: scope > plan > [review?] > visualize > design_complete + +ingredients: + task: + description: Research question or topic to investigate + required: true + source_dir: + description: Path to the project root + required: true + base_branch: + description: Branch to target for the research PR + default: "main" + review_design: + description: > + When 'true' (the default), run the review-design skill after plan_experiment + to validate the experiment plan before execution. Emits a verdict: GO proceeds, + REVISE loops back to plan_experiment with revision guidance (bounded by + check_design_review_loop at 3 iterations), STOP triggers resolve_design_review + for triage — addressable findings loop back, structural findings halt the + pipeline. When 'false', skip directly to plan_visualization. + default: "true" + issue_url: + description: > + Optional URL to an existing GitHub issue describing the research goal. + Provided as context to scope and plan_experiment. + required: false + +kitchen_rules: + - > + NEVER use native Claude Code tools from the orchestrator: Read, Grep, Glob, + Edit, Write, Bash, Agent, WebFetch, WebSearch, NotebookEdit are ALL forbidden. + All work is delegated through run_skill, run_cmd, run_python. + - > + This recipe is a food truck dispatched by a campaign. Emit the L3 sentinel + JSON block exactly as instructed in the design_complete step message — no + extra tool calls after reaching design_complete. + +steps: + scope: + tool: run_skill + model: "" + with: + skill_command: "/autoskillit:scope ${{ inputs.task }}" + cwd: "${{ inputs.source_dir }}" + step_name: scope + capture: + scope_report: "${{ result.scope_report }}" + on_success: plan_experiment + on_failure: escalate_stop + + plan_experiment: + tool: run_skill + model: "" + with: + skill_command: "/autoskillit:plan-experiment ${{ context.scope_report }} ${{ context.revision_guidance }}" + cwd: "${{ inputs.source_dir }}" + step_name: plan_experiment + optional_context_refs: [revision_guidance] + capture: + experiment_plan: "${{ result.experiment_plan }}" + on_success: review_design + on_failure: escalate_stop + + review_design: + tool: run_skill + model: "" + with: + skill_command: "/autoskillit:review-design ${{ context.experiment_plan }}" + cwd: "${{ inputs.source_dir }}" + step_name: review_design + capture: + verdict: "${{ result.verdict }}" + experiment_type: "${{ result.experiment_type }}" + evaluation_dashboard: "${{ result.evaluation_dashboard }}" + revision_guidance: "${{ result.revision_guidance }}" + skip_when_false: inputs.review_design + retries: 2 + on_exhausted: plan_visualization + on_context_limit: plan_visualization + on_failure: plan_visualization + on_result: + - when: "${{ result.verdict }} == GO" + route: plan_visualization + - when: "${{ result.verdict }} == REVISE" + route: revise_design + - when: "${{ result.verdict }} == STOP" + route: resolve_design_review + - route: plan_visualization + + plan_visualization: + tool: run_skill + model: "" + with: + skill_command: "/autoskillit:plan-visualization ${{ inputs.source_dir }} ${{ context.experiment_plan }} ${{ context.scope_report }}" + cwd: "${{ inputs.source_dir }}" + step_name: plan_visualization + capture: + visualization_plan_path: "${{ result.visualization_plan_path }}" + report_plan_path: "${{ result.report_plan_path }}" + on_success: design_complete + on_failure: escalate_stop + note: > + Selects 2–4 vis-lens skills via three-tier logic, runs them in parallel, and + synthesizes outputs into visualization-plan.md and report-plan.md. + Runs before worktree creation so outputs can be committed into the research dir. + + revise_design: + action: route + on_result: + - route: check_design_review_loop + + check_design_review_loop: + tool: run_python + with: + callable: "autoskillit.smoke_utils.check_loop_iteration" + current_iteration: "${{ context.design_review_count }}" + max_iterations: "3" + capture: + design_review_count: "${{ result.next_iteration }}" + on_result: + - when: "${{ result.max_exceeded }} == true" + route: plan_visualization + - route: plan_experiment + on_failure: plan_visualization + optional_context_refs: + - design_review_count + note: > + Iteration guard for the review_design → revise_design → plan_experiment cycle. + Caps design revision loops at 3 iterations. Both REVISE and + STOP→resolve→revised paths flow through revise_design and hit this guard. + On exhaustion, proceeds to plan_visualization with the best available design. + + resolve_design_review: + tool: run_skill + model: "" + with: + skill_command: "/autoskillit:resolve-design-review ${{ context.evaluation_dashboard }} ${{ context.experiment_plan }} ${{ context.revision_guidance }}" + cwd: "${{ inputs.source_dir }}" + step_name: resolve_design_review + optional_context_refs: [evaluation_dashboard, experiment_plan, revision_guidance] + capture: + revision_guidance: "${{ result.revision_guidance }}" + retries: 1 + on_failure: design_rejected + on_context_limit: design_rejected + on_result: + - when: "${{ result.resolution }} == revised" + route: revise_design + - when: "${{ result.resolution }} == failed" + route: design_rejected + - route: design_rejected + note: > + Triages STOP-verdict findings from review_design. ADDRESSABLE/DISCUSS findings + generate revision_guidance and loop back. Only all-STRUCTURAL findings halt at + design_rejected. + + design_rejected: + action: stop + message: > + Research pipeline halted: experiment design rejected (verdict=STOP, resolution=failed). + All stop-trigger findings are structural — revision cannot address the identified flaws. + Check {{AUTOSKILLIT_TEMP}}/resolve-design-review/ for the triage report and + {{AUTOSKILLIT_TEMP}}/review-design/ for the evaluation dashboard. + + design_complete: + action: stop + message: >- + Design phase complete. Emit the L3 result sentinel JSON block now. + The sentinel must include the following fields from captured context: + success=true, scope_report=, + experiment_plan=, + visualization_plan_path=, + report_plan_path=, + experiment_type=. + Example sentinel: + {"success": true, "scope_report": "", "experiment_plan": "", + "visualization_plan_path": "", "report_plan_path": "", + "experiment_type": "benchmark"} + + escalate_stop: + action: stop + message: "Research pipeline failed — human intervention needed. Check the latest step output for details." diff --git a/src/autoskillit/recipes/research-implement.json b/src/autoskillit/recipes/research-implement.json new file mode 100644 index 0000000000..ef100a6719 --- /dev/null +++ b/src/autoskillit/recipes/research-implement.json @@ -0,0 +1,404 @@ +{ + "name": "research-implement", + "recipe_version": "1.0.0", + "categories": [ + "research-family" + ], + "requires_packs": [ + "research" + ], + "description": "Execution-phase sub-recipe extracted from research.yaml. Receives a pre-created worktree and experiment plan from a campaign dispatch, then runs the full 20-step implementation pipeline: stage data, decompose into phases, iteratively plan and implement each phase, troubleshoot failures, run the experiment, generate a report, test, and push. Terminates with sentinel emission of output paths.\n", + "summary": "stage > decompose > [plan > implement]* > experiment > report > test > push > complete", + "ingredients": { + "task": { + "description": "Research question or topic to investigate", + "required": true + }, + "source_dir": { + "description": "Path to the project root", + "required": true + }, + "base_branch": { + "description": "Branch to target for the research PR", + "default": "main" + }, + "output_mode": { + "description": "Where research outputs land. 'local' (default) produces a self-contained bundle. 'pr' opens a GitHub PR alongside the bundle.\n", + "default": "local" + }, + "issue_url": { + "description": "Optional URL to an existing GitHub issue describing the research goal.\n", + "required": false + }, + "worktree_path": { + "description": "Path to the pre-created worktree (passed by campaign dispatch)", + "required": true + }, + "research_dir": { + "description": "Path to the research/ directory in the worktree", + "required": true + }, + "experiment_plan": { + "description": "Path to the experiment plan file (passed by campaign dispatch)", + "required": true + }, + "visualization_plan_path": { + "description": "Path to the visualization plan (passed by campaign dispatch)", + "required": true, + "hidden": true + } + }, + "kitchen_rules": [ + "NEVER use native Claude Code tools from the orchestrator: Read, Grep, Glob, Edit, Write, Bash, Agent, WebFetch, WebSearch, NotebookEdit are ALL forbidden. All work is delegated through run_skill, run_cmd, run_python.\n", + "PHASE ITERATION: decompose_phases always runs. The agent maintains a phase queue from group_files. For each phase, run plan_phase then implement_phase. Advance to run_experiment when all phases complete.\n", + "SENTINEL EMISSION: When reaching implement_complete, emit the L3 result sentinel JSON block exactly as instructed in the step message — include worktree_path, research_dir, report_path, and experiment_results (when available).\n" + ], + "steps": { + "stage_data": { + "tool": "run_skill", + "model": "", + "with": { + "skill_command": "/autoskillit:stage-data ${{ inputs.experiment_plan }}", + "cwd": "${{ inputs.worktree_path }}", + "step_name": "stage_data" + }, + "capture": { + "verdict": "${{ result.verdict }}", + "resource_report": "${{ result.resource_report }}" + }, + "on_result": { + "field": "verdict", + "routes": { + "PASS": "decompose_phases", + "WARN": "decompose_phases", + "FAIL": "escalate_stop" + } + }, + "on_failure": "escalate_stop", + "note": "Pre-flight resource gate. Reads the experiment plan's data_manifest, checks available disk space and network connectivity for external/gitignored data entries, and creates the required directory structure in the worktree. PASS and WARN proceed to decompose_phases. FAIL escalates immediately with a detailed resource feasibility report rather than wasting compute on doomed downloads.\n" + }, + "decompose_phases": { + "tool": "run_skill", + "model": "", + "with": { + "skill_command": "/autoskillit:make-groups ${{ inputs.experiment_plan }}", + "cwd": "${{ inputs.worktree_path }}", + "step_name": "decompose" + }, + "capture": { + "group_files": "${{ result.group_files }}" + }, + "on_success": "plan_phase", + "on_failure": "escalate_stop", + "note": "Decomposes the experiment plan into sequenced phases. Each phase becomes an independently plannable implementation unit. Typical decomposition: infrastructure/logging -> data generators -> plotting/analysis scripts -> experiment implementation.\n" + }, + "plan_phase": { + "tool": "run_skill", + "model": "", + "with": { + "skill_command": "/autoskillit:make-plan ${{ context.group_files }}", + "cwd": "${{ inputs.worktree_path }}", + "step_name": "plan_phase", + "group_files": "${{ context.group_files }}" + }, + "capture": { + "phase_plan_path": "${{ result.plan_path }}" + }, + "capture_list": { + "plan_parts": "${{ result.plan_parts }}" + }, + "on_success": "implement_phase", + "on_failure": "escalate_stop", + "note": "Plans the current setup phase. Replace ${{ context.group_files }} in the skill_command with the FIRST REMAINING phase file path from the list — do NOT pass the entire list. Pass the path directly to the skill (do not read it). Glob plan_dir for *_part_*.md or single plan file. ITERATION: After each implement_phase cycle, advance to the next entry in context.group_files for the following plan_phase iteration.\n" + }, + "implement_phase": { + "tool": "run_skill", + "model": "", + "stale_threshold": 2400, + "idle_output_timeout": 0, + "with": { + "skill_command": "/autoskillit:implement-experiment ${{ context.phase_plan_path }}", + "cwd": "${{ inputs.worktree_path }}", + "step_name": "implement_phase" + }, + "capture": { + "worktree_path": "${{ result.worktree_path }}" + }, + "retries": 0, + "on_success": "next_phase_or_experiment", + "on_failure": "troubleshoot_implement_failure", + "on_exhausted": "run_experiment", + "on_context_limit": "run_experiment", + "note": "Implements the current phase plan in an isolated worktree via implement-experiment. Context exhaustion is a normal termination — the worktree is intact and run_experiment picks up what was committed. Failures route to troubleshoot for classification before re-planning or escalation.\n" + }, + "troubleshoot_implement_failure": { + "tool": "run_skill", + "model": "", + "stale_threshold": 2400, + "idle_output_timeout": 0, + "with": { + "skill_command": "/autoskillit:troubleshoot-experiment ${{ context.worktree_path }} implement_phase", + "cwd": "${{ inputs.source_dir }}", + "step_name": "troubleshoot_implement_failure" + }, + "capture": { + "is_fixable": "${{ result.is_fixable }}" + }, + "on_success": "route_implement_failure", + "on_failure": "escalate_stop" + }, + "route_implement_failure": { + "action": "route", + "on_result": [ + { + "when": "${{ context.is_fixable }} == true", + "route": "check_implement_fix_loop" + }, + { + "route": "escalate_stop" + } + ] + }, + "check_implement_fix_loop": { + "tool": "run_python", + "with": { + "callable": "autoskillit.smoke_utils.check_loop_iteration", + "current_iteration": "${{ context.implement_fix_count }}", + "max_iterations": "3" + }, + "capture": { + "implement_fix_count": "${{ result.next_iteration }}" + }, + "on_result": [ + { + "when": "${{ result.max_exceeded }} == true", + "route": "run_experiment" + }, + { + "route": "plan_phase" + } + ], + "on_failure": "run_experiment", + "optional_context_refs": [ + "implement_fix_count" + ], + "note": "Iteration guard for implement_phase → troubleshoot → plan_phase cycle. Caps fix-and-retry loops at 3 iterations. On exhaustion, skips to run_experiment with whatever implementation is committed.\n" + }, + "next_phase_or_experiment": { + "action": "route", + "on_result": [ + { + "when": "remaining phases exist in context.group_files", + "route": "plan_phase" + }, + { + "route": "run_experiment" + } + ], + "note": "Routing step. The agent evaluates remaining phases: 1. If more phases remain in group_files -> route to plan_phase. 2. If all phases complete -> route to run_experiment.\n" + }, + "run_experiment": { + "tool": "run_skill", + "model": "", + "stale_threshold": 2400, + "idle_output_timeout": 0, + "with": { + "skill_command": "/autoskillit:run-experiment ${{ context.worktree_path }}", + "cwd": "${{ context.worktree_path }}", + "step_name": "run_experiment" + }, + "capture": { + "experiment_results": "${{ result.results_path }}" + }, + "retries": 2, + "on_success": "generate_report", + "on_failure": "troubleshoot_run_failure", + "on_exhausted": "ensure_results", + "on_context_limit": "ensure_results" + }, + "troubleshoot_run_failure": { + "tool": "run_skill", + "model": "", + "stale_threshold": 2400, + "idle_output_timeout": 0, + "with": { + "skill_command": "/autoskillit:troubleshoot-experiment ${{ context.worktree_path }} run_experiment", + "cwd": "${{ inputs.source_dir }}", + "step_name": "troubleshoot_run_failure" + }, + "capture": { + "run_is_fixable": "${{ result.is_fixable }}" + }, + "on_success": "route_run_failure", + "on_failure": "escalate_stop" + }, + "route_run_failure": { + "action": "route", + "on_result": [ + { + "when": "${{ context.run_is_fixable }} == true", + "route": "adjust_experiment" + }, + { + "route": "escalate_stop" + } + ] + }, + "adjust_experiment": { + "tool": "run_skill", + "model": "", + "stale_threshold": 2400, + "idle_output_timeout": 0, + "with": { + "skill_command": "/autoskillit:run-experiment ${{ context.worktree_path }} --adjust", + "cwd": "${{ context.worktree_path }}", + "step_name": "adjust" + }, + "capture": { + "experiment_results": "${{ result.results_path }}" + }, + "on_success": "check_run_fix_loop", + "on_failure": "ensure_results", + "on_context_limit": "ensure_results" + }, + "check_run_fix_loop": { + "tool": "run_python", + "with": { + "callable": "autoskillit.smoke_utils.check_loop_iteration", + "current_iteration": "${{ context.run_fix_count }}", + "max_iterations": "3" + }, + "capture": { + "run_fix_count": "${{ result.next_iteration }}" + }, + "on_result": [ + { + "when": "${{ result.max_exceeded }} == true", + "route": "ensure_results" + }, + { + "route": "run_experiment" + } + ], + "on_failure": "ensure_results", + "optional_context_refs": [ + "run_fix_count" + ], + "note": "Iteration guard for the adjust_experiment → check_run_fix_loop → run_experiment sub-loop (entered via troubleshoot_run_failure → route_run_failure → adjust_experiment). Caps experiment fix loops at 3 iterations. On exhaustion, proceeds to ensure_results to salvage whatever data exists.\n" + }, + "ensure_results": { + "tool": "run_python", + "with": { + "callable": "autoskillit.recipe._cmd_rpc.ensure_results", + "experiment_results": "${{ context.experiment_results }}", + "worktree_path": "${{ context.worktree_path }}", + "temp_subdir": "{{AUTOSKILLIT_TEMP}}" + }, + "capture": { + "experiment_results": "${{ result.experiment_results }}" + }, + "on_success": "generate_report_inconclusive", + "on_failure": "escalate_stop", + "note": "Ensures context.experiment_results is a valid file path before calling generate_report_inconclusive. Creates a minimal placeholder results file when run-experiment never produced output (retries exhausted or adjust failed).\n" + }, + "generate_report": { + "tool": "run_skill", + "model": "", + "with": { + "skill_command": "/autoskillit:generate-report ${{ context.worktree_path }} ${{ context.experiment_results }} --output-mode ${{ inputs.output_mode }} --issue-url ${{ inputs.issue_url }}", + "cwd": "${{ context.worktree_path }}", + "step_name": "generate_report" + }, + "capture": { + "report_path": "${{ result.report_path }}" + }, + "on_success": "test", + "on_failure": "escalate_stop", + "on_context_limit": "test", + "note": "Writes a conclusive research report to research/ in the worktree." + }, + "generate_report_inconclusive": { + "tool": "run_skill", + "model": "", + "with": { + "skill_command": "/autoskillit:generate-report ${{ context.worktree_path }} ${{ context.experiment_results }} --inconclusive --output-mode ${{ inputs.output_mode }} --issue-url ${{ inputs.issue_url }}", + "cwd": "${{ context.worktree_path }}", + "step_name": "generate_report_inconclusive" + }, + "capture": { + "report_path": "${{ result.report_path }}" + }, + "on_success": "test", + "on_failure": "escalate_stop", + "on_context_limit": "test", + "note": "Writes an inconclusive research report. Experiments could not prove or disprove the hypothesis. This is a valid outcome.\n" + }, + "test": { + "tool": "test_check", + "with": { + "worktree_path": "${{ context.worktree_path }}" + }, + "on_success": "push_branch", + "on_failure": "fix_tests" + }, + "fix_tests": { + "tool": "run_skill", + "model": "", + "with": { + "skill_command": "/autoskillit:resolve-failures ${{ context.worktree_path }} ${{ inputs.experiment_plan }} ${{ inputs.base_branch }}", + "cwd": "${{ inputs.source_dir }}", + "step_name": "fix" + }, + "capture": { + "verdict": "${{ result.verdict }}", + "fixes_applied": "${{ result.fixes_applied }}" + }, + "on_result": [ + { + "when": "${{ context.verdict }} == 'real_fix'", + "route": "retest" + }, + { + "when": "${{ context.verdict }} == 'already_green'", + "route": "retest" + }, + { + "when": "${{ context.verdict }} == 'flake_suspected'", + "route": "retest" + }, + { + "when": "${{ context.verdict }} == 'ci_only_failure'", + "route": "escalate_stop" + } + ], + "on_failure": "escalate_stop", + "on_context_limit": "retest" + }, + "retest": { + "tool": "test_check", + "with": { + "worktree_path": "${{ context.worktree_path }}" + }, + "on_success": "push_branch", + "on_failure": "escalate_stop" + }, + "push_branch": { + "tool": "run_cmd", + "with": { + "cmd": "bash scripts/recipe/push_branch.sh '${{ inputs.output_mode }}' '${{ context.worktree_path }}'", + "cwd": "${{ context.worktree_path }}", + "step_name": "push_branch" + }, + "on_success": "implement_complete", + "on_failure": "escalate_stop" + }, + "escalate_stop": { + "action": "stop", + "message": "Implementation pipeline failed — human intervention needed. Check the latest step output for details." + }, + "implement_complete": { + "action": "stop", + "message": "Implementation phase complete. Emit the L3 result sentinel JSON block now. The sentinel must include the following fields: success=true, worktree_path=${{ context.worktree_path }}, research_dir=${{ inputs.research_dir }}, report_path=${{ context.report_path }}, experiment_results=${{ context.experiment_results }}. Example sentinel: {\"success\": true, \"worktree_path\": \"\", \"research_dir\": \"\", \"report_path\": \"\", \"experiment_results\": \"\"}" + } + } +} diff --git a/src/autoskillit/recipes/research-implement.yaml b/src/autoskillit/recipes/research-implement.yaml new file mode 100644 index 0000000000..3d797d3302 --- /dev/null +++ b/src/autoskillit/recipes/research-implement.yaml @@ -0,0 +1,386 @@ +name: research-implement +recipe_version: "1.0.0" +categories: [research-family] +requires_packs: [research] +description: > + Execution-phase sub-recipe extracted from research.yaml. Receives a pre-created + worktree and experiment plan from a campaign dispatch, then runs the full 20-step + implementation pipeline: stage data, decompose into phases, iteratively plan and + implement each phase, troubleshoot failures, run the experiment, generate a report, + test, and push. Terminates with sentinel emission of output paths. +summary: stage > decompose > [plan > implement]* > experiment > report > test > push > complete + +ingredients: + # --- user-input --- + task: + description: Research question or topic to investigate + required: true + source_dir: + description: Path to the project root + required: true + base_branch: + description: Branch to target for the research PR + default: "main" + output_mode: + description: > + Where research outputs land. 'local' (default) produces a self-contained + bundle. 'pr' opens a GitHub PR alongside the bundle. + default: "local" + issue_url: + description: > + Optional URL to an existing GitHub issue describing the research goal. + required: false + # --- campaign-passed --- + worktree_path: + description: Path to the pre-created worktree (passed by campaign dispatch) + required: true + research_dir: + description: Path to the research/ directory in the worktree + required: true + experiment_plan: + description: Path to the experiment plan file (passed by campaign dispatch) + required: true + visualization_plan_path: + description: Path to the visualization plan (passed by campaign dispatch) + required: true + hidden: true + +kitchen_rules: + - > + NEVER use native Claude Code tools from the orchestrator: Read, Grep, Glob, + Edit, Write, Bash, Agent, WebFetch, WebSearch, NotebookEdit are ALL forbidden. + All work is delegated through run_skill, run_cmd, run_python. + - > + PHASE ITERATION: decompose_phases always runs. The agent maintains + a phase queue from group_files. For each phase, run plan_phase then + implement_phase. Advance to run_experiment when all phases complete. + - > + SENTINEL EMISSION: When reaching implement_complete, emit the L3 result + sentinel JSON block exactly as instructed in the step message — include + worktree_path, research_dir, report_path, and experiment_results (when available). + +steps: + # --- Pre-Flight --- + stage_data: + tool: run_skill + model: "" + with: + skill_command: "/autoskillit:stage-data ${{ inputs.experiment_plan }}" + cwd: "${{ inputs.worktree_path }}" + step_name: stage_data + capture: + verdict: "${{ result.verdict }}" + resource_report: "${{ result.resource_report }}" + on_result: + field: verdict + routes: + PASS: decompose_phases + WARN: decompose_phases + FAIL: escalate_stop + on_failure: escalate_stop + note: > + Pre-flight resource gate. Reads the experiment plan's data_manifest, checks + available disk space and network connectivity for external/gitignored data + entries, and creates the required directory structure in the worktree. PASS + and WARN proceed to decompose_phases. FAIL escalates immediately with a + detailed resource feasibility report rather than wasting compute on doomed + downloads. + + # --- Phase Iteration --- + decompose_phases: + tool: run_skill + model: "" + with: + skill_command: "/autoskillit:make-groups ${{ inputs.experiment_plan }}" + cwd: "${{ inputs.worktree_path }}" + step_name: decompose + capture: + group_files: "${{ result.group_files }}" + on_success: plan_phase + on_failure: escalate_stop + note: > + Decomposes the experiment plan into sequenced phases. Each phase + becomes an independently plannable implementation unit. Typical + decomposition: infrastructure/logging -> data generators -> + plotting/analysis scripts -> experiment implementation. + + plan_phase: + tool: run_skill + model: "" + with: + skill_command: "/autoskillit:make-plan ${{ context.group_files }}" + cwd: "${{ inputs.worktree_path }}" + step_name: plan_phase + group_files: "${{ context.group_files }}" + capture: + phase_plan_path: "${{ result.plan_path }}" + capture_list: + plan_parts: "${{ result.plan_parts }}" + on_success: implement_phase + on_failure: escalate_stop + note: > + Plans the current setup phase. Replace ${{ context.group_files }} in the + skill_command with the FIRST REMAINING phase file path from the list — + do NOT pass the entire list. Pass the path directly to the skill (do not read it). + Glob plan_dir for *_part_*.md or single plan file. + ITERATION: After each implement_phase cycle, advance to the next entry in + context.group_files for the following plan_phase iteration. + + implement_phase: + tool: run_skill + model: "" + stale_threshold: 2400 + idle_output_timeout: 0 + with: + skill_command: "/autoskillit:implement-experiment ${{ context.phase_plan_path }}" + cwd: "${{ inputs.worktree_path }}" + step_name: implement_phase + capture: + worktree_path: "${{ result.worktree_path }}" + retries: 0 + on_success: next_phase_or_experiment + on_failure: troubleshoot_implement_failure + on_exhausted: run_experiment + on_context_limit: run_experiment + note: > + Implements the current phase plan in an isolated worktree via implement-experiment. + Context exhaustion is a normal termination — the worktree is intact and + run_experiment picks up what was committed. Failures route to troubleshoot + for classification before re-planning or escalation. + + troubleshoot_implement_failure: + tool: run_skill + model: "" + stale_threshold: 2400 + idle_output_timeout: 0 + with: + skill_command: "/autoskillit:troubleshoot-experiment ${{ context.worktree_path }} implement_phase" + cwd: "${{ inputs.source_dir }}" + step_name: troubleshoot_implement_failure + capture: + is_fixable: "${{ result.is_fixable }}" + on_success: route_implement_failure + on_failure: escalate_stop + + route_implement_failure: + action: route + on_result: + - when: "${{ context.is_fixable }} == true" + route: check_implement_fix_loop + - route: escalate_stop + + check_implement_fix_loop: + tool: run_python + with: + callable: "autoskillit.smoke_utils.check_loop_iteration" + current_iteration: "${{ context.implement_fix_count }}" + max_iterations: "3" + capture: + implement_fix_count: "${{ result.next_iteration }}" + on_result: + - when: "${{ result.max_exceeded }} == true" + route: run_experiment + - route: plan_phase + on_failure: run_experiment + optional_context_refs: + - implement_fix_count + note: > + Iteration guard for implement_phase → troubleshoot → plan_phase cycle. + Caps fix-and-retry loops at 3 iterations. On exhaustion, skips to + run_experiment with whatever implementation is committed. + + next_phase_or_experiment: + action: route + on_result: + - when: "remaining phases exist in context.group_files" + route: plan_phase + - route: run_experiment + note: > + Routing step. The agent evaluates remaining phases: + 1. If more phases remain in group_files -> route to plan_phase. + 2. If all phases complete -> route to run_experiment. + + # --- Experiment + Report --- + run_experiment: + tool: run_skill + model: "" + stale_threshold: 2400 + idle_output_timeout: 0 + with: + skill_command: "/autoskillit:run-experiment ${{ context.worktree_path }}" + cwd: "${{ context.worktree_path }}" + step_name: run_experiment + capture: + experiment_results: "${{ result.results_path }}" + retries: 2 + on_success: generate_report + on_failure: troubleshoot_run_failure + on_exhausted: ensure_results + on_context_limit: ensure_results + + troubleshoot_run_failure: + tool: run_skill + model: "" + stale_threshold: 2400 + idle_output_timeout: 0 + with: + skill_command: "/autoskillit:troubleshoot-experiment ${{ context.worktree_path }} run_experiment" + cwd: "${{ inputs.source_dir }}" + step_name: troubleshoot_run_failure + capture: + run_is_fixable: "${{ result.is_fixable }}" + on_success: route_run_failure + on_failure: escalate_stop + + route_run_failure: + action: route + on_result: + - when: "${{ context.run_is_fixable }} == true" + route: adjust_experiment + - route: escalate_stop + + adjust_experiment: + tool: run_skill + model: "" + stale_threshold: 2400 + idle_output_timeout: 0 + with: + skill_command: "/autoskillit:run-experiment ${{ context.worktree_path }} --adjust" + cwd: "${{ context.worktree_path }}" + step_name: adjust + capture: + experiment_results: "${{ result.results_path }}" + on_success: check_run_fix_loop + on_failure: ensure_results + on_context_limit: ensure_results + + check_run_fix_loop: + tool: run_python + with: + callable: "autoskillit.smoke_utils.check_loop_iteration" + current_iteration: "${{ context.run_fix_count }}" + max_iterations: "3" + capture: + run_fix_count: "${{ result.next_iteration }}" + on_result: + - when: "${{ result.max_exceeded }} == true" + route: ensure_results + - route: run_experiment + on_failure: ensure_results + optional_context_refs: + - run_fix_count + note: > + Iteration guard for the adjust_experiment → check_run_fix_loop → run_experiment + sub-loop (entered via troubleshoot_run_failure → route_run_failure → adjust_experiment). + Caps experiment fix loops at 3 iterations. On exhaustion, proceeds to + ensure_results to salvage whatever data exists. + + ensure_results: + tool: run_python + with: + callable: "autoskillit.recipe._cmd_rpc.ensure_results" + experiment_results: "${{ context.experiment_results }}" + worktree_path: "${{ context.worktree_path }}" + temp_subdir: "{{AUTOSKILLIT_TEMP}}" + capture: + experiment_results: "${{ result.experiment_results }}" + on_success: generate_report_inconclusive + on_failure: escalate_stop + note: > + Ensures context.experiment_results is a valid file path before calling + generate_report_inconclusive. Creates a minimal placeholder results file when + run-experiment never produced output (retries exhausted or adjust failed). + + generate_report: + tool: run_skill + model: "" + with: + skill_command: "/autoskillit:generate-report ${{ context.worktree_path }} ${{ context.experiment_results }} --output-mode ${{ inputs.output_mode }} --issue-url ${{ inputs.issue_url }}" + cwd: "${{ context.worktree_path }}" + step_name: generate_report + capture: + report_path: "${{ result.report_path }}" + on_success: test + on_failure: escalate_stop + on_context_limit: test + note: "Writes a conclusive research report to research/ in the worktree." + + generate_report_inconclusive: + tool: run_skill + model: "" + with: + skill_command: "/autoskillit:generate-report ${{ context.worktree_path }} ${{ context.experiment_results }} --inconclusive --output-mode ${{ inputs.output_mode }} --issue-url ${{ inputs.issue_url }}" + cwd: "${{ context.worktree_path }}" + step_name: generate_report_inconclusive + capture: + report_path: "${{ result.report_path }}" + on_success: test + on_failure: escalate_stop + on_context_limit: test + note: > + Writes an inconclusive research report. Experiments could not + prove or disprove the hypothesis. This is a valid outcome. + + # --- Test + Push --- + test: + tool: test_check + with: + worktree_path: "${{ context.worktree_path }}" + on_success: push_branch + on_failure: fix_tests + + fix_tests: + tool: run_skill + model: "" + with: + skill_command: "/autoskillit:resolve-failures ${{ context.worktree_path }} ${{ inputs.experiment_plan }} ${{ inputs.base_branch }}" + cwd: "${{ inputs.source_dir }}" + step_name: fix + capture: + verdict: "${{ result.verdict }}" + fixes_applied: "${{ result.fixes_applied }}" + on_result: + - when: "${{ context.verdict }} == 'real_fix'" + route: retest + - when: "${{ context.verdict }} == 'already_green'" + route: retest + - when: "${{ context.verdict }} == 'flake_suspected'" + route: retest + - when: "${{ context.verdict }} == 'ci_only_failure'" + route: escalate_stop + on_failure: escalate_stop + on_context_limit: retest + + retest: + tool: test_check + with: + worktree_path: "${{ context.worktree_path }}" + on_success: push_branch + on_failure: escalate_stop + + push_branch: + tool: run_cmd + with: + cmd: "bash scripts/recipe/push_branch.sh '${{ inputs.output_mode }}' '${{ context.worktree_path }}'" + cwd: "${{ context.worktree_path }}" + step_name: push_branch + on_success: implement_complete + on_failure: escalate_stop + + # --- Terminal Stops --- + escalate_stop: + action: stop + message: "Implementation pipeline failed — human intervention needed. Check the latest step output for details." + + implement_complete: + action: stop + message: >- + Implementation phase complete. Emit the L3 result sentinel JSON block now. + The sentinel must include the following fields: + success=true, worktree_path=${{ context.worktree_path }}, + research_dir=${{ inputs.research_dir }}, + report_path=${{ context.report_path }}, + experiment_results=${{ context.experiment_results }}. + Example sentinel: + {"success": true, "worktree_path": "", "research_dir": "", + "report_path": "", "experiment_results": ""} diff --git a/src/autoskillit/recipes/research-review.json b/src/autoskillit/recipes/research-review.json new file mode 100644 index 0000000000..09f88543c8 --- /dev/null +++ b/src/autoskillit/recipes/research-review.json @@ -0,0 +1,482 @@ +{ + "name": "research-review", + "recipe_version": "1.0.0", + "categories": [ + "research-family" + ], + "requires_packs": [ + "research", + "exp-lens", + "vis-lens" + ], + "description": "Review-phase sub-recipe extracted from research.yaml. Receives a pre-created worktree and completed experiment outputs from a campaign dispatch, then runs the 22-step PR/review pipeline: prepare PR, run experiment lenses, stage bundle, compose PR, review, audit claims, resolve feedback, re-run if needed, finalize, and export. Terminates with sentinel emission of output paths.\n", + "summary": "prepare > lenses > stage > [pr-compose > review? > audit? > resolve > re-run?] > finalize > export/complete", + "ingredients": { + "task": { + "description": "Research question or topic to investigate", + "required": true + }, + "source_dir": { + "description": "Path to the project root", + "required": true + }, + "base_branch": { + "description": "Branch to target for the research PR", + "default": "main" + }, + "output_mode": { + "description": "Where research outputs land. 'local' produces a self-contained bundle. 'pr' opens a GitHub PR alongside the bundle.\n", + "default": "local" + }, + "review_pr": { + "description": "When 'true', run the review-research-pr skill after the research PR is opened.\n", + "default": "false" + }, + "audit_claims": { + "description": "When 'true', run audit-claims after review_research_pr to verify citation integrity.\n", + "required": false, + "default": "false" + }, + "issue_url": { + "description": "Optional URL to an existing GitHub issue describing the research goal.\n", + "required": false + }, + "worktree_path": { + "description": "Path to the pre-created worktree (passed by campaign dispatch)", + "required": true + }, + "research_dir": { + "description": "Path to the research/ directory in the worktree", + "required": true + }, + "report_path": { + "description": "Path to the generated research report", + "required": true + }, + "experiment_plan": { + "description": "Path to the experiment plan file", + "required": true + }, + "experiment_results": { + "description": "Path to the experiment results file", + "required": true + }, + "experiment_type": { + "description": "Type of experiment (e.g. benchmark, comparison)", + "required": true + }, + "scope_report": { + "description": "Path to the scope report", + "required": true + }, + "visualization_plan_path": { + "description": "Path to the visualization plan", + "required": true + } + }, + "kitchen_rules": [ + "NEVER use native Claude Code tools from the orchestrator: Read, Grep, Glob, Edit, Write, Bash, Agent, WebFetch, WebSearch, NotebookEdit are ALL forbidden. All work is delegated through run_skill, run_cmd, run_python.\n", + "Do NOT auto-merge. The research PR is for human review only.", + "Route to on_failure when a step fails — do not investigate directly.", + "LENS ITERATION: run_experiment_lenses iterates over context.selected_lenses (comma-separated lens slugs, e.g. \"fair-comparison,estimand-clarity\") and context.lens_context_paths (comma-separated context file paths). For each lens slug, run /autoskillit:exp-lens-{slug} {matching_context_path} with the matching context file path. Run all selected lenses in parallel (multiple run_skill calls in a single assistant message). Accumulate each diagram_path value into context.all_diagram_paths using comma separation via capture_list. If a lens fails, continue with remaining lenses — partial diagrams are acceptable (compose_research_pr handles empty lists).\n" + ], + "steps": { + "prepare_research_pr": { + "tool": "run_skill", + "model": "", + "optional_context_refs": [ + "worktree_path", + "research_dir", + "report_path", + "experiment_plan", + "experiment_results", + "experiment_type", + "scope_report", + "visualization_plan_path" + ], + "with": { + "skill_command": "/autoskillit:prepare-research-pr ${{ context.report_path }} ${{ context.experiment_plan }} ${{ context.worktree_path }} ${{ inputs.base_branch }}", + "cwd": "${{ context.worktree_path }}", + "step_name": "prepare_research_pr" + }, + "capture": { + "prep_path": "${{ result.prep_path }}", + "selected_lenses": "${{ result.selected_lenses }}", + "lens_context_paths": "${{ result.lens_context_paths }}" + }, + "on_success": "run_experiment_lenses", + "on_failure": "escalate_stop" + }, + "run_experiment_lenses": { + "tool": "run_skill", + "model": "", + "optional_context_refs": [ + "worktree_path", + "experiment_plan" + ], + "with": { + "skill_command": "/autoskillit:exp-lens-{slug} {context_path} ${{ context.experiment_plan }}", + "cwd": "${{ context.worktree_path }}", + "step_name": "run_experiment_lenses" + }, + "capture_list": { + "all_diagram_paths": "${{ result.diagram_path }}" + }, + "on_success": "stage_bundle", + "on_failure": "stage_bundle" + }, + "stage_bundle": { + "tool": "run_cmd", + "optional_context_refs": [ + "research_dir", + "worktree_path" + ], + "with": { + "cmd": "bash scripts/recipe/stage_bundle.sh '${{ context.research_dir }}' '${{ context.worktree_path }}' '{{AUTOSKILLIT_TEMP}}'", + "cwd": "${{ context.worktree_path }}", + "step_name": "stage_bundle" + }, + "on_success": "route_pr_or_local", + "on_failure": "route_pr_or_local", + "note": "Idempotent file staging step. Copies phase-group, phase-plan, and exp-lens diagram artifacts from the worktree temp directory into research/{slug}/artifacts/. Creates images/ and scripts/ subdirectories. No compression, no rename, no commit. Safe to run multiple times — cp overwrites are idempotent. Runs after run_experiment_lenses to capture all exp-lens diagrams written to AUTOSKILLIT_TEMP/exp-lens-*/.\n" + }, + "route_pr_or_local": { + "action": "route", + "on_result": [ + { + "when": "${{ inputs.output_mode }} == local", + "route": "finalize_bundle" + }, + { + "route": "compose_research_pr" + } + ], + "note": "Branches on output_mode after bundle staging. local → bypass PR pipeline entirely, jump to finalize_bundle. pr (fall-through) → compose_research_pr to open PR and run the full review/archival flow. Uses predicate route because skip_when_false only accepts bare inputs. with no expression evaluation.\n" + }, + "compose_research_pr": { + "tool": "run_skill", + "model": "", + "optional_context_refs": [ + "worktree_path" + ], + "with": { + "skill_command": "/autoskillit:compose-research-pr ${{ context.prep_path }} \"${{ context.all_diagram_paths }}\" ${{ context.worktree_path }} ${{ inputs.base_branch }}", + "cwd": "${{ context.worktree_path }}", + "step_name": "compose_research_pr" + }, + "capture": { + "pr_url": "${{ result.pr_url }}" + }, + "on_success": "guard_pr_url", + "on_failure": "escalate_stop" + }, + "guard_pr_url": { + "action": "route", + "on_result": [ + { + "when": "${{ context.pr_url }}", + "route": "review_research_pr" + }, + { + "route": "review_pr_complete" + } + ], + "note": "Guard — skip review if pr_url is empty (graceful degradation from open_research_pr)." + }, + "review_research_pr": { + "tool": "run_skill", + "model": "", + "optional_context_refs": [ + "worktree_path" + ], + "with": { + "skill_command": "/autoskillit:review-research-pr ${{ context.worktree_path }} ${{ inputs.base_branch }} ${{ context.pr_url }}", + "cwd": "${{ context.worktree_path }}", + "step_name": "review_research_pr" + }, + "capture": { + "review_verdict": "${{ result.verdict }}" + }, + "skip_when_false": "inputs.review_pr", + "on_context_limit": "audit_claims", + "on_failure": "audit_claims", + "on_result": [ + { + "route": "audit_claims" + } + ], + "note": "Optional research PR review step, gated by the review_pr ingredient. Verdict captured as review_verdict for deferred routing. All paths (skip, success, failure, context-limit) route to audit_claims so both gates complete before any resolution step begins. When on_context_limit or on_failure fires, the capture block does not execute and context.review_verdict remains absent; the recipe engine treats absent context variables as empty string in comparisons, so downstream route_review_resolve safely falls through to route_claims_resolve.\n" + }, + "audit_claims": { + "tool": "run_skill", + "model": "", + "optional_context_refs": [ + "worktree_path" + ], + "with": { + "skill_command": "/autoskillit:audit-claims ${{ context.worktree_path }} ${{ inputs.base_branch }} ${{ context.pr_url }}", + "cwd": "${{ context.worktree_path }}", + "step_name": "audit_claims" + }, + "capture": { + "audit_verdict": "${{ result.verdict }}" + }, + "skip_when_false": "inputs.audit_claims", + "retries": 1, + "on_exhausted": "route_review_resolve", + "on_context_limit": "route_review_resolve", + "on_failure": "route_review_resolve", + "on_result": [ + { + "route": "route_review_resolve" + } + ], + "note": "Optional citation-integrity gate, gated by the audit_claims ingredient. Runs after review_research_pr regardless of that step's verdict. All paths (skip, success, failure) route to route_review_resolve so both gates complete before resolution.\n" + }, + "route_review_resolve": { + "action": "route", + "on_result": [ + { + "when": "${{ context.review_verdict }} == changes_requested", + "route": "resolve_research_review" + }, + { + "route": "route_claims_resolve" + } + ], + "note": "Routes to resolve_research_review only when the PR review found changes_requested. All other verdicts (approved, needs_human, empty/skipped) proceed directly to route_claims_resolve.\n" + }, + "resolve_research_review": { + "tool": "run_skill", + "model": "", + "optional_context_refs": [ + "worktree_path" + ], + "with": { + "skill_command": "/autoskillit:resolve-research-review ${{ context.worktree_path }} ${{ inputs.base_branch }}", + "cwd": "${{ context.worktree_path }}", + "step_name": "resolve_research_review" + }, + "capture": { + "review_needs_rerun": "${{ result.needs_rerun }}" + }, + "retries": 2, + "on_exhausted": "route_claims_resolve", + "on_context_limit": "route_claims_resolve", + "on_success": "route_claims_resolve", + "on_failure": "route_claims_resolve" + }, + "route_claims_resolve": { + "action": "route", + "on_result": [ + { + "when": "${{ context.audit_verdict }} == changes_requested", + "route": "resolve_claims_review" + }, + { + "route": "merge_escalations" + } + ], + "note": "Routes to resolve_claims_review only when the claims audit found changes_requested. All other verdicts (approved, needs_human, empty/skipped) proceed directly to merge_escalations. When audit_claims is skipped (inputs.audit_claims=false), context.audit_verdict is absent; the recipe engine treats absent variables as empty string, so the condition evaluates false and the fallback route to merge_escalations fires safely.\n" + }, + "resolve_claims_review": { + "tool": "run_skill", + "model": "", + "optional_context_refs": [ + "worktree_path" + ], + "with": { + "skill_command": "/autoskillit:resolve-claims-review ${{ context.worktree_path }} ${{ inputs.base_branch }}", + "cwd": "${{ context.worktree_path }}", + "step_name": "resolve_claims_review" + }, + "capture": { + "claims_needs_rerun": "${{ result.needs_rerun }}" + }, + "retries": 2, + "on_exhausted": "merge_escalations", + "on_context_limit": "merge_escalations", + "on_success": "merge_escalations", + "on_failure": "merge_escalations" + }, + "merge_escalations": { + "action": "route", + "on_result": [ + { + "when": "${{ context.review_needs_rerun }} == true", + "route": "re_run_experiment" + }, + { + "when": "${{ context.claims_needs_rerun }} == true", + "route": "re_run_experiment" + }, + { + "route": "finalize_bundle" + } + ], + "note": "Merged escalation consumer replacing check_escalations. Triggers a single re_run_experiment if either review resolution or claims resolution flagged a rerun. When only design_flaw escalations exist (or none), routes directly to finalize_bundle — design_flaw escalations are already posted as inline PR comments by the resolve skills. Both context.review_needs_rerun and context.claims_needs_rerun may be absent if their respective resolution steps were never reached (e.g., no changes_requested verdict on that gate); the recipe engine evaluates absent context variables as empty string, so absent == true is always false and routes safely fall through to finalize_bundle.\n" + }, + "re_run_experiment": { + "tool": "run_skill", + "model": "", + "stale_threshold": 2400, + "idle_output_timeout": 0, + "optional_context_refs": [ + "worktree_path" + ], + "with": { + "skill_command": "/autoskillit:run-experiment ${{ context.worktree_path }} --adjust", + "cwd": "${{ context.worktree_path }}", + "step_name": "re_run_experiment" + }, + "capture": { + "experiment_results": "${{ result.results_path }}" + }, + "on_success": "re_generate_report", + "on_failure": "re_push_research", + "on_context_limit": "re_push_research", + "note": "Targeted re-run of affected benchmarks after review resolution applied code fixes. Uses --adjust to read previous results and re-run only what changed. Intentionally shadows the experiment_results ingredient (L58): after this step, context.experiment_results holds the re-run results_path so re_generate_report uses fresh data. on_failure falls through to push as best-effort — the code fixes from resolve are already committed and should still be pushed.\n" + }, + "re_generate_report": { + "tool": "run_skill", + "model": "", + "optional_context_refs": [ + "worktree_path", + "experiment_results" + ], + "with": { + "skill_command": "/autoskillit:generate-report ${{ context.worktree_path }} ${{ context.experiment_results }} --output-mode ${{ inputs.output_mode }} --issue-url ${{ inputs.issue_url }}", + "cwd": "${{ context.worktree_path }}", + "step_name": "re_generate_report" + }, + "capture": { + "report_path": "${{ result.report_path }}" + }, + "on_success": "re_stage_bundle", + "on_failure": "re_push_research", + "on_context_limit": "re_push_research", + "note": "Re-generate the research report with updated benchmark results from the re-validation run. Passes context.experiment_results which was updated by re_run_experiment to point to the new results file. Intentionally shadows the report_path ingredient (L52): after this step, context.report_path holds the regenerated path so downstream steps receive the updated report. Falls through to push on failure.\n" + }, + "re_stage_bundle": { + "tool": "run_cmd", + "optional_context_refs": [ + "research_dir", + "worktree_path" + ], + "with": { + "cmd": "bash scripts/recipe/stage_bundle.sh '${{ context.research_dir }}' '${{ context.worktree_path }}' '{{AUTOSKILLIT_TEMP}}'", + "cwd": "${{ context.worktree_path }}", + "step_name": "re_stage_bundle" + }, + "on_success": "re_test", + "on_failure": "re_push_research", + "note": "Idempotent re-staging after report rewrite. Same body as stage_bundle — updates artifact directory with any newly written phase artifacts before re-testing. Falls through to re_push_research on failure (best-effort).\n" + }, + "re_test": { + "tool": "test_check", + "optional_context_refs": [ + "worktree_path" + ], + "with": { + "worktree_path": "${{ context.worktree_path }}" + }, + "on_success": "re_push_research", + "on_failure": "re_push_research", + "note": "Post-revalidation test gate. Both pass and fail route to push — the re-validation results should be pushed regardless so the PR reflects the current state of the code and measurements.\n" + }, + "re_push_research": { + "tool": "run_cmd", + "optional_context_refs": [ + "worktree_path" + ], + "with": { + "cmd": "git push", + "cwd": "${{ context.worktree_path }}", + "step_name": "re_push_research" + }, + "on_success": "finalize_bundle", + "on_failure": "escalate_stop" + }, + "finalize_bundle": { + "tool": "run_cmd", + "optional_context_refs": [ + "research_dir", + "worktree_path" + ], + "with": { + "cmd": "bash scripts/recipe/finalize_bundle.sh '${{ inputs.output_mode }}' '${{ context.research_dir }}' '${{ context.worktree_path }}'", + "cwd": "${{ context.worktree_path }}", + "step_name": "finalize_bundle" + }, + "capture": { + "report_path_after_finalize": "${{ result.report_path_after_finalize }}" + }, + "on_success": "finalize_bundle_render", + "on_failure": "escalate_stop", + "note": "Single compression point. Runs exactly once per recipe run. In pr mode: renames report.md → README.md, archives supporting artifacts, appends Archive Manifest, commits. In local mode: keeps report.md, excludes report.html/mermaid.min.js/images from the tar (they stay for local browsing), skips git commit. finalize_bundle_render always runs after this step regardless of mode.\n" + }, + "finalize_bundle_render": { + "tool": "run_skill", + "model": "", + "optional_context_refs": [ + "research_dir", + "worktree_path", + "all_diagram_paths", + "visualization_plan_path" + ], + "with": { + "skill_command": "/autoskillit:bundle-local-report ${{ context.research_dir }} ${{ context.report_path_after_finalize }} ${{ context.all_diagram_paths }} ${{ context.visualization_plan_path }}", + "cwd": "${{ context.worktree_path }}", + "step_name": "finalize_bundle_render" + }, + "capture": { + "html_path": "${{ result.html_path }}" + }, + "on_success": "route_archive_or_export", + "on_failure": "route_archive_or_export", + "note": "HTML report rendering. Always runs after finalize_bundle on_success. Non-fatal: routes to route_archive_or_export on both outcomes, which branches on output_mode to either export_local_bundle (local) or review_pr_complete (pr).\n" + }, + "route_archive_or_export": { + "action": "route", + "on_result": [ + { + "when": "${{ inputs.output_mode }} == local", + "route": "export_local_bundle" + }, + { + "route": "review_pr_complete" + } + ], + "note": "Branches after HTML bundle rendering. local → export_local_bundle (copy to {source_dir}/research-bundles/). pr (fall-through) → review_pr_complete to terminate the review pipeline.\n" + }, + "export_local_bundle": { + "tool": "run_python", + "optional_context_refs": [ + "research_dir" + ], + "with": { + "callable": "autoskillit.recipe._cmd_rpc.export_local_bundle", + "source_dir": "${{ inputs.source_dir }}", + "research_dir": "${{ context.research_dir }}" + }, + "capture": { + "local_bundle_path": "${{ result.local_bundle_path }}" + }, + "on_success": "review_local_complete", + "on_failure": "review_local_complete", + "note": "Local mode terminal export. Copies the finalized research_dir (containing report.html, mermaid.min.js, images/, report.md, artifacts.tar.gz) to {source_dir}/research-bundles/{slug}/. Non-fatal — routes to review_local_complete on both outcomes.\n" + }, + "review_pr_complete": { + "action": "stop", + "message": "Review phase complete (PR mode). Emit the L3 result sentinel JSON block now. The sentinel must include the following fields: success=true, pr_url=${{ context.pr_url }}, report_path_after_finalize=${{ context.report_path_after_finalize }}, worktree_path=${{ context.worktree_path }}, research_dir=${{ context.research_dir }}. Note: report_path_after_finalize is empty string when reached via the guard_pr_url fallback (pr_url absent) — finalize_bundle is bypassed in that path. Emit empty string for this field in that case. Example sentinel: {\"success\": true, \"pr_url\": \"\", \"report_path_after_finalize\": \"\", \"worktree_path\": \"\", \"research_dir\": \"\"}" + }, + "review_local_complete": { + "action": "stop", + "message": "Review phase complete (local mode). Local bundle exported successfully. Emit the L3 result sentinel JSON block now. The sentinel must include: success=true, local_bundle_path=${{ context.local_bundle_path }}. Example sentinel: {\"success\": true, \"local_bundle_path\": \"\"}" + }, + "escalate_stop": { + "action": "stop", + "message": "Review pipeline failed — human intervention needed. Check the latest step output for details." + } + } +} diff --git a/src/autoskillit/recipes/research-review.yaml b/src/autoskillit/recipes/research-review.yaml new file mode 100644 index 0000000000..19b736433f --- /dev/null +++ b/src/autoskillit/recipes/research-review.yaml @@ -0,0 +1,470 @@ +name: research-review +recipe_version: "1.0.0" +categories: [research-family] +requires_packs: [research, exp-lens, vis-lens] +description: > + Review-phase sub-recipe extracted from research.yaml. Receives a pre-created + worktree and completed experiment outputs from a campaign dispatch, then runs + the 22-step PR/review pipeline: prepare PR, run experiment lenses, stage bundle, + compose PR, review, audit claims, resolve feedback, re-run if needed, finalize, + and export. Terminates with sentinel emission of output paths. +summary: prepare > lenses > stage > [pr-compose > review? > audit? > resolve > re-run?] > finalize > export/complete + +ingredients: + # --- user-input --- + task: + description: Research question or topic to investigate + required: true + source_dir: + description: Path to the project root + required: true + base_branch: + description: Branch to target for the research PR + default: "main" + output_mode: + description: > + Where research outputs land. 'local' produces a self-contained bundle. + 'pr' opens a GitHub PR alongside the bundle. + default: "local" + review_pr: + description: > + When 'true', run the review-research-pr skill after the research PR + is opened. + default: "false" + audit_claims: + description: > + When 'true', run audit-claims after review_research_pr to verify citation + integrity. + required: false + default: "false" + issue_url: + description: > + Optional URL to an existing GitHub issue describing the research goal. + required: false + + # --- campaign-passed --- + worktree_path: + description: Path to the pre-created worktree (passed by campaign dispatch) + required: true + research_dir: + description: Path to the research/ directory in the worktree + required: true + report_path: + description: Path to the generated research report + required: true + experiment_plan: + description: Path to the experiment plan file + required: true + experiment_results: + description: Path to the experiment results file + required: true + experiment_type: + description: Type of experiment (e.g. benchmark, comparison) + required: true + scope_report: + description: Path to the scope report + required: true + visualization_plan_path: + description: Path to the visualization plan + required: true + +kitchen_rules: + - > + NEVER use native Claude Code tools from the orchestrator: Read, Grep, Glob, + Edit, Write, Bash, Agent, WebFetch, WebSearch, NotebookEdit are ALL forbidden. + All work is delegated through run_skill, run_cmd, run_python. + - "Do NOT auto-merge. The research PR is for human review only." + - "Route to on_failure when a step fails — do not investigate directly." + - > + LENS ITERATION: run_experiment_lenses iterates over context.selected_lenses + (comma-separated lens slugs, e.g. "fair-comparison,estimand-clarity") and + context.lens_context_paths (comma-separated context file paths). For each + lens slug, run /autoskillit:exp-lens-{slug} {matching_context_path} with the + matching context file path. Run all selected lenses in parallel (multiple + run_skill calls in a single assistant message). Accumulate each diagram_path + value into context.all_diagram_paths using comma separation via capture_list. + If a lens fails, continue with remaining lenses — partial diagrams are + acceptable (compose_research_pr handles empty lists). + +steps: + # --- LENS + STAGING PHASE --- + prepare_research_pr: + tool: run_skill + model: "" + optional_context_refs: [worktree_path, research_dir, report_path, experiment_plan, experiment_results, experiment_type, scope_report, visualization_plan_path] + with: + skill_command: "/autoskillit:prepare-research-pr ${{ context.report_path }} ${{ context.experiment_plan }} ${{ context.worktree_path }} ${{ inputs.base_branch }}" + cwd: "${{ context.worktree_path }}" + step_name: prepare_research_pr + capture: + prep_path: "${{ result.prep_path }}" + selected_lenses: "${{ result.selected_lenses }}" + lens_context_paths: "${{ result.lens_context_paths }}" + on_success: run_experiment_lenses + on_failure: escalate_stop + + run_experiment_lenses: + tool: run_skill + model: "" + optional_context_refs: [worktree_path, experiment_plan] + with: + skill_command: "/autoskillit:exp-lens-{slug} {context_path} ${{ context.experiment_plan }}" + cwd: "${{ context.worktree_path }}" + step_name: run_experiment_lenses + capture_list: + all_diagram_paths: "${{ result.diagram_path }}" + on_success: stage_bundle + on_failure: stage_bundle + + stage_bundle: + tool: run_cmd + optional_context_refs: [research_dir, worktree_path] + with: + cmd: "bash scripts/recipe/stage_bundle.sh '${{ context.research_dir }}' '${{ context.worktree_path }}' '{{AUTOSKILLIT_TEMP}}'" + cwd: "${{ context.worktree_path }}" + step_name: stage_bundle + on_success: route_pr_or_local + on_failure: route_pr_or_local + note: > + Idempotent file staging step. Copies phase-group, phase-plan, and exp-lens diagram + artifacts from the worktree temp directory into research/{slug}/artifacts/. Creates + images/ and scripts/ subdirectories. No compression, no rename, no commit. Safe to + run multiple times — cp overwrites are idempotent. Runs after run_experiment_lenses + to capture all exp-lens diagrams written to AUTOSKILLIT_TEMP/exp-lens-*/. + + route_pr_or_local: + action: route + on_result: + - when: "${{ inputs.output_mode }} == local" + route: finalize_bundle + - route: compose_research_pr + note: > + Branches on output_mode after bundle staging. local → bypass PR pipeline entirely, + jump to finalize_bundle. pr (fall-through) → compose_research_pr to open PR and run + the full review/archival flow. Uses predicate route because skip_when_false only + accepts bare inputs. with no expression evaluation. + + # --- PR COMPOSE + REVIEW PHASE --- + compose_research_pr: + tool: run_skill + model: "" + optional_context_refs: [worktree_path] + with: + skill_command: "/autoskillit:compose-research-pr ${{ context.prep_path }} \"${{ context.all_diagram_paths }}\" ${{ context.worktree_path }} ${{ inputs.base_branch }}" + cwd: "${{ context.worktree_path }}" + step_name: compose_research_pr + capture: + pr_url: "${{ result.pr_url }}" + on_success: guard_pr_url + on_failure: escalate_stop + + guard_pr_url: + action: route + on_result: + - when: "${{ context.pr_url }}" + route: review_research_pr + - route: review_pr_complete + note: Guard — skip review if pr_url is empty (graceful degradation from open_research_pr). + + review_research_pr: + tool: run_skill + model: "" + optional_context_refs: [worktree_path] + with: + skill_command: "/autoskillit:review-research-pr ${{ context.worktree_path }} ${{ inputs.base_branch }} ${{ context.pr_url }}" + cwd: "${{ context.worktree_path }}" + step_name: review_research_pr + capture: + review_verdict: "${{ result.verdict }}" + skip_when_false: inputs.review_pr + on_context_limit: audit_claims + on_failure: audit_claims + on_result: + - route: audit_claims + note: > + Optional research PR review step, gated by the review_pr ingredient. + Verdict captured as review_verdict for deferred routing. All paths + (skip, success, failure, context-limit) route to audit_claims so both + gates complete before any resolution step begins. When on_context_limit + or on_failure fires, the capture block does not execute and + context.review_verdict remains absent; the recipe engine treats absent + context variables as empty string in comparisons, so downstream + route_review_resolve safely falls through to route_claims_resolve. + + audit_claims: + tool: run_skill + model: "" + optional_context_refs: [worktree_path] + with: + skill_command: "/autoskillit:audit-claims ${{ context.worktree_path }} ${{ inputs.base_branch }} ${{ context.pr_url }}" + cwd: "${{ context.worktree_path }}" + step_name: audit_claims + capture: + audit_verdict: "${{ result.verdict }}" + skip_when_false: inputs.audit_claims + retries: 1 + on_exhausted: route_review_resolve + on_context_limit: route_review_resolve + on_failure: route_review_resolve + on_result: + - route: route_review_resolve + note: > + Optional citation-integrity gate, gated by the audit_claims ingredient. + Runs after review_research_pr regardless of that step's verdict. + All paths (skip, success, failure) route to route_review_resolve so both + gates complete before resolution. + + # --- RESOLUTION + ESCALATION PHASE --- + route_review_resolve: + action: route + on_result: + - when: "${{ context.review_verdict }} == changes_requested" + route: resolve_research_review + - route: route_claims_resolve + note: > + Routes to resolve_research_review only when the PR review found + changes_requested. All other verdicts (approved, needs_human, empty/skipped) + proceed directly to route_claims_resolve. + + resolve_research_review: + tool: run_skill + model: "" + optional_context_refs: [worktree_path] + with: + skill_command: "/autoskillit:resolve-research-review ${{ context.worktree_path }} ${{ inputs.base_branch }}" + cwd: "${{ context.worktree_path }}" + step_name: resolve_research_review + capture: + review_needs_rerun: "${{ result.needs_rerun }}" + retries: 2 + on_exhausted: route_claims_resolve + on_context_limit: route_claims_resolve + on_success: route_claims_resolve + on_failure: route_claims_resolve + + route_claims_resolve: + action: route + on_result: + - when: "${{ context.audit_verdict }} == changes_requested" + route: resolve_claims_review + - route: merge_escalations + note: > + Routes to resolve_claims_review only when the claims audit found + changes_requested. All other verdicts (approved, needs_human, empty/skipped) + proceed directly to merge_escalations. When audit_claims is skipped + (inputs.audit_claims=false), context.audit_verdict is absent; the recipe + engine treats absent variables as empty string, so the condition evaluates + false and the fallback route to merge_escalations fires safely. + + resolve_claims_review: + tool: run_skill + model: "" + optional_context_refs: [worktree_path] + with: + skill_command: "/autoskillit:resolve-claims-review ${{ context.worktree_path }} ${{ inputs.base_branch }}" + cwd: "${{ context.worktree_path }}" + step_name: resolve_claims_review + capture: + claims_needs_rerun: "${{ result.needs_rerun }}" + retries: 2 + on_exhausted: merge_escalations + on_context_limit: merge_escalations + on_success: merge_escalations + on_failure: merge_escalations + + merge_escalations: + action: route + on_result: + - when: "${{ context.review_needs_rerun }} == true" + route: re_run_experiment + - when: "${{ context.claims_needs_rerun }} == true" + route: re_run_experiment + - route: finalize_bundle + note: > + Merged escalation consumer replacing check_escalations. Triggers a single + re_run_experiment if either review resolution or claims resolution flagged + a rerun. When only design_flaw escalations exist (or none), routes directly + to finalize_bundle — design_flaw escalations are already posted as inline PR comments + by the resolve skills. Both context.review_needs_rerun and + context.claims_needs_rerun may be absent if their respective resolution + steps were never reached (e.g., no changes_requested verdict on that gate); + the recipe engine evaluates absent context variables as empty string, so + absent == true is always false and routes safely fall through to + finalize_bundle. + + # --- RE-VALIDATION LOOP --- + re_run_experiment: + tool: run_skill + model: "" + stale_threshold: 2400 + idle_output_timeout: 0 + optional_context_refs: [worktree_path] + with: + skill_command: "/autoskillit:run-experiment ${{ context.worktree_path }} --adjust" + cwd: "${{ context.worktree_path }}" + step_name: re_run_experiment + capture: + experiment_results: "${{ result.results_path }}" + on_success: re_generate_report + on_failure: re_push_research + on_context_limit: re_push_research + note: > + Targeted re-run of affected benchmarks after review resolution applied + code fixes. Uses --adjust to read previous results and re-run only what + changed. Intentionally shadows the experiment_results ingredient (L58): + after this step, context.experiment_results holds the re-run results_path + so re_generate_report uses fresh data. on_failure falls through to push + as best-effort — the code fixes from resolve are already committed and + should still be pushed. + + re_generate_report: + tool: run_skill + model: "" + optional_context_refs: [worktree_path, experiment_results] + with: + skill_command: "/autoskillit:generate-report ${{ context.worktree_path }} ${{ context.experiment_results }} --output-mode ${{ inputs.output_mode }} --issue-url ${{ inputs.issue_url }}" + cwd: "${{ context.worktree_path }}" + step_name: re_generate_report + capture: + report_path: "${{ result.report_path }}" + on_success: re_stage_bundle + on_failure: re_push_research + on_context_limit: re_push_research + note: > + Re-generate the research report with updated benchmark results from the + re-validation run. Passes context.experiment_results which was updated + by re_run_experiment to point to the new results file. Intentionally + shadows the report_path ingredient (L52): after this step, + context.report_path holds the regenerated path so downstream steps + receive the updated report. Falls through to push on failure. + + re_stage_bundle: + tool: run_cmd + optional_context_refs: [research_dir, worktree_path] + with: + cmd: "bash scripts/recipe/stage_bundle.sh '${{ context.research_dir }}' '${{ context.worktree_path }}' '{{AUTOSKILLIT_TEMP}}'" + cwd: "${{ context.worktree_path }}" + step_name: re_stage_bundle + on_success: re_test + on_failure: re_push_research + note: > + Idempotent re-staging after report rewrite. Same body as stage_bundle — updates + artifact directory with any newly written phase artifacts before re-testing. Falls + through to re_push_research on failure (best-effort). + + re_test: + tool: test_check + optional_context_refs: [worktree_path] + with: + worktree_path: "${{ context.worktree_path }}" + on_success: re_push_research + on_failure: re_push_research + note: > + Post-revalidation test gate. Both pass and fail route to push — + the re-validation results should be pushed regardless so the PR + reflects the current state of the code and measurements. + + re_push_research: + tool: run_cmd + optional_context_refs: [worktree_path] + with: + cmd: "git push" + cwd: "${{ context.worktree_path }}" + step_name: re_push_research + on_success: finalize_bundle + on_failure: escalate_stop + + # --- FINALIZE + EXPORT PHASE --- + finalize_bundle: + tool: run_cmd + optional_context_refs: [research_dir, worktree_path] + with: + cmd: "bash scripts/recipe/finalize_bundle.sh '${{ inputs.output_mode }}' '${{ context.research_dir }}' '${{ context.worktree_path }}'" + cwd: "${{ context.worktree_path }}" + step_name: finalize_bundle + capture: + report_path_after_finalize: "${{ result.report_path_after_finalize }}" + on_success: finalize_bundle_render + on_failure: escalate_stop + note: > + Single compression point. Runs exactly once per recipe run. In pr mode: renames + report.md → README.md, archives supporting artifacts, appends Archive Manifest, + commits. In local mode: keeps report.md, excludes report.html/mermaid.min.js/images + from the tar (they stay for local browsing), skips git commit. finalize_bundle_render + always runs after this step regardless of mode. + + finalize_bundle_render: + tool: run_skill + model: "" + optional_context_refs: [research_dir, worktree_path, all_diagram_paths, visualization_plan_path] + with: + skill_command: "/autoskillit:bundle-local-report ${{ context.research_dir }} ${{ context.report_path_after_finalize }} ${{ context.all_diagram_paths }} ${{ context.visualization_plan_path }}" + cwd: "${{ context.worktree_path }}" + step_name: finalize_bundle_render + capture: + html_path: "${{ result.html_path }}" + on_success: route_archive_or_export + on_failure: route_archive_or_export + note: > + HTML report rendering. Always runs after finalize_bundle on_success. Non-fatal: + routes to route_archive_or_export on both outcomes, which branches on output_mode + to either export_local_bundle (local) or review_pr_complete (pr). + + route_archive_or_export: + action: route + on_result: + - when: "${{ inputs.output_mode }} == local" + route: export_local_bundle + - route: review_pr_complete + note: > + Branches after HTML bundle rendering. local → export_local_bundle (copy to + {source_dir}/research-bundles/). pr (fall-through) → review_pr_complete + to terminate the review pipeline. + + export_local_bundle: + tool: run_python + optional_context_refs: [research_dir] + with: + callable: "autoskillit.recipe._cmd_rpc.export_local_bundle" + source_dir: "${{ inputs.source_dir }}" + research_dir: "${{ context.research_dir }}" + capture: + local_bundle_path: "${{ result.local_bundle_path }}" + on_success: review_local_complete + on_failure: review_local_complete + note: > + Local mode terminal export. Copies the finalized research_dir (containing + report.html, mermaid.min.js, images/, report.md, artifacts.tar.gz) to + {source_dir}/research-bundles/{slug}/. Non-fatal — routes to review_local_complete + on both outcomes. + + # --- TERMINAL STOPS --- + review_pr_complete: + action: stop + message: >- + Review phase complete (PR mode). Emit the L3 result sentinel JSON block now. + The sentinel must include the following fields: + success=true, pr_url=${{ context.pr_url }}, + report_path_after_finalize=${{ context.report_path_after_finalize }}, + worktree_path=${{ context.worktree_path }}, + research_dir=${{ context.research_dir }}. + Note: report_path_after_finalize is empty string when reached via the + guard_pr_url fallback (pr_url absent) — finalize_bundle is bypassed in + that path. Emit empty string for this field in that case. + Example sentinel: + {"success": true, "pr_url": "", "report_path_after_finalize": "", + "worktree_path": "", "research_dir": ""} + + review_local_complete: + action: stop + message: >- + Review phase complete (local mode). Local bundle exported successfully. + Emit the L3 result sentinel JSON block now. + The sentinel must include: success=true, + local_bundle_path=${{ context.local_bundle_path }}. + Example sentinel: + {"success": true, "local_bundle_path": ""} + + escalate_stop: + action: stop + message: "Review pipeline failed — human intervention needed. Check the latest step output for details." diff --git a/src/autoskillit/recipes/research.json b/src/autoskillit/recipes/research.json new file mode 100644 index 0000000000..e2e2a53fff --- /dev/null +++ b/src/autoskillit/recipes/research.json @@ -0,0 +1,1021 @@ +{ + "name": "research", + "recipe_version": "1.0.0", + "requires_packs": [ + "research", + "exp-lens", + "vis-lens" + ], + "description": "Single-phase technical research recipe. Scopes a research question, plans an experiment, optionally reviews the design, then always decomposes into phases, implements, runs the experiment, writes a report to research/, and opens a PR. Supports a skippable design review loop (bounded by check_design_review_loop at 3 iterations), review resolution routing after PR open, and a post-review re-validation loop that re-runs affected benchmarks when resolve-research-review produces rerun_required escalations. After the final push (or when review completes), an archival phase separates research artifacts from experimental code: opens a clean artifact-only PR, tags the experiment branch, and closes the original PR. No auto-merge.\n", + "summary": "scope > plan > [review?] > worktree > env-setup > decompose > phases > experiment > report > open-pr > [review? > resolve > [re-run?] > push] > archive", + "ingredients": { + "task": { + "description": "Research question or topic to investigate", + "required": true + }, + "issue_url": { + "description": "Optional URL to an existing GitHub issue describing the research goal. Provided as context to scope and plan_experiment.\n", + "required": false + }, + "source_dir": { + "description": "Path to the project root", + "required": true + }, + "base_branch": { + "description": "Branch to target for the research PR", + "default": "main" + }, + "review_design": { + "description": "When 'true' (the default), run the review-design skill after plan_experiment to validate the experiment plan before execution. Emits a verdict: GO proceeds, REVISE loops back to plan_experiment with revision guidance (bounded by check_design_review_loop at 3 iterations), STOP triggers resolve_design_review for triage — addressable findings loop back, structural findings halt the pipeline. When 'false', skip directly to create_worktree.\n", + "default": "true" + }, + "review_pr": { + "description": "When 'true', run the review-research-pr skill after the research PR is opened. Posts inline review comments on research quality dimensions (methodology, reproducibility, report-quality, etc.) and emits a verdict. Defaults to 'false' — the research PR is for human review.\n", + "default": "false" + }, + "audit_claims": { + "required": false, + "default": "false", + "description": "When 'true', run audit-claims after review_research_pr to verify citation integrity and claim backing in the research report. Emits a verdict that routes to resolve-claims-review when changes are needed.\n" + }, + "output_mode": { + "required": false, + "default": "local", + "description": "Where research outputs land. 'local' (default) produces a self-contained report.html bundle under {source_dir}/research-bundles/{date-slug}/ with no GitHub interaction. 'pr' opens a GitHub PR alongside the HTML bundle and runs the full review/archival flow.\n" + } + }, + "kitchen_rules": [ + "NEVER use native Claude Code tools from the orchestrator: Read, Grep, Glob, Edit, Write, Bash, Agent, WebFetch, WebSearch, NotebookEdit are ALL forbidden. All work is delegated through run_skill, run_cmd, run_python.\n", + "Do NOT auto-merge. The research PR is for human review only.", + "The research/ folder is the target for all permanent output.", + "Inconclusive results are valid findings, not failures.", + "Route to on_failure when a step fails — do not investigate directly.", + "DESIGN REVIEW LOOP: review_design validates the plan and may route back to plan_experiment with revision guidance. Bounded by check_design_review_loop (max 3 iterations). On exhaustion or failure, proceed to execution with the best available plan — do not stop. Exception: a STOP verdict triggers resolve_design_review, which triages each stop-trigger finding as ADDRESSABLE/STRUCTURAL/DISCUSS. If any are ADDRESSABLE or DISCUSS, revision guidance is generated and the plan loops back for revision (via revise_design). If all are STRUCTURAL, the pipeline halts at design_rejected (truly terminal — revision cannot address the flaws).\n", + "PHASE ITERATION: decompose_phases always runs. The agent maintains a phase queue from group_files. For each phase, run plan_phase then implement_phase. Advance to run_experiment when all phases complete.\n", + "PERMANENT ARTIFACTS: create_worktree commits the experiment plan to research/ and supporting artifacts (scope report, design evaluation) to research/artifacts/. These are permanent records in the PR.\n", + "ARCHIVAL PHASE: After all review and re-validation cycles complete, the pipeline extracts research/ into a clean artifact PR, tags the full experiment branch under archive/research/, and closes the original PR. Every archival step degrades gracefully — failures route to research_complete without blocking the pipeline.\n", + "LOCAL MODE OUTPUT: In local mode, the finalized bundle (report.html, report.md, artifacts.tar.gz) is exported to {source_dir}/research-bundles/{date-slug}/. This directory is created only when output_mode=local and is not committed to git.\n" + ], + "steps": { + "scope": { + "tool": "run_skill", + "model": "", + "with": { + "skill_command": "/autoskillit:scope ${{ inputs.task }}", + "cwd": "${{ inputs.source_dir }}", + "step_name": "scope" + }, + "capture": { + "scope_report": "${{ result.scope_report }}" + }, + "on_success": "plan_experiment", + "on_failure": "escalate_stop" + }, + "plan_experiment": { + "tool": "run_skill", + "model": "", + "with": { + "skill_command": "/autoskillit:plan-experiment ${{ context.scope_report }} ${{ context.revision_guidance }}", + "cwd": "${{ inputs.source_dir }}", + "step_name": "plan_experiment" + }, + "optional_context_refs": [ + "revision_guidance" + ], + "capture": { + "experiment_plan": "${{ result.experiment_plan }}" + }, + "on_success": "review_design", + "on_failure": "escalate_stop" + }, + "review_design": { + "tool": "run_skill", + "model": "", + "with": { + "skill_command": "/autoskillit:review-design ${{ context.experiment_plan }}", + "cwd": "${{ inputs.source_dir }}", + "step_name": "review_design" + }, + "capture": { + "verdict": "${{ result.verdict }}", + "experiment_type": "${{ result.experiment_type }}", + "evaluation_dashboard": "${{ result.evaluation_dashboard }}", + "revision_guidance": "${{ result.revision_guidance }}", + "classification_timestamp": "${{ result.classification_timestamp }}" + }, + "skip_when_false": "inputs.review_design", + "retries": 2, + "on_exhausted": "create_worktree", + "on_context_limit": "create_worktree", + "on_failure": "create_worktree", + "on_result": [ + { + "when": "${{ result.verdict }} == GO", + "route": "plan_visualization" + }, + { + "when": "${{ result.verdict }} == REVISE", + "route": "revise_design" + }, + { + "when": "${{ result.verdict }} == STOP", + "route": "resolve_design_review" + }, + { + "route": "create_worktree" + } + ] + }, + "plan_visualization": { + "tool": "run_skill", + "model": "", + "with": { + "skill_command": "/autoskillit:plan-visualization ${{ inputs.source_dir }} ${{ context.experiment_plan }} ${{ context.scope_report }}", + "cwd": "${{ inputs.source_dir }}", + "step_name": "plan_visualization" + }, + "capture": { + "visualization_plan_path": "${{ result.visualization_plan_path }}", + "report_plan_path": "${{ result.report_plan_path }}", + "disambiguation_rule_applied": "${{ result.disambiguation_rule_applied }}", + "tier_c_lens": "${{ result.tier_c_lens }}", + "methodology_tradition": "${{ result.methodology_tradition }}", + "visualization_plan_trace_path": "${{ result.visualization_plan_trace_path }}" + }, + "on_success": "create_worktree", + "on_failure": "escalate_stop", + "note": "Selects 2–4 vis-lens skills via three-tier logic, runs them in parallel, and synthesizes outputs into visualization-plan.md and report-plan.md. Runs before worktree creation so outputs can be committed into the research dir.\n" + }, + "revise_design": { + "action": "route", + "on_result": [ + { + "route": "check_design_review_loop" + } + ] + }, + "check_design_review_loop": { + "tool": "run_python", + "with": { + "callable": "autoskillit.smoke_utils.check_loop_iteration", + "current_iteration": "${{ context.design_review_count }}", + "max_iterations": "3" + }, + "capture": { + "design_review_count": "${{ result.next_iteration }}" + }, + "on_result": [ + { + "when": "${{ result.max_exceeded }} == true", + "route": "create_worktree" + }, + { + "route": "plan_experiment" + } + ], + "on_failure": "create_worktree", + "optional_context_refs": [ + "design_review_count" + ], + "note": "Iteration guard for the review_design → revise_design → plan_experiment cycle. Caps design revision loops at 3 iterations. On exhaustion, proceeds to create_worktree with the best available design.\n" + }, + "resolve_design_review": { + "tool": "run_skill", + "model": "", + "with": { + "skill_command": "/autoskillit:resolve-design-review ${{ context.evaluation_dashboard }} ${{ context.experiment_plan }} ${{ context.revision_guidance }}", + "cwd": "${{ inputs.source_dir }}", + "step_name": "resolve_design_review" + }, + "optional_context_refs": [ + "evaluation_dashboard", + "experiment_plan", + "revision_guidance" + ], + "capture": { + "revision_guidance": "${{ result.revision_guidance }}" + }, + "retries": 1, + "on_failure": "design_rejected", + "on_context_limit": "design_rejected", + "on_result": [ + { + "when": "${{ result.resolution }} == revised", + "route": "revise_design" + }, + { + "when": "${{ result.resolution }} == failed", + "route": "design_rejected" + }, + { + "route": "design_rejected" + } + ], + "note": "Triages STOP-verdict findings from review_design. ADDRESSABLE/DISCUSS findings generate revision_guidance and loop back. Only all-STRUCTURAL findings halt at design_rejected.\n" + }, + "design_rejected": { + "action": "stop", + "message": "Research pipeline halted: experiment design rejected (verdict=STOP, resolution=failed). All stop-trigger findings are structural — revision cannot address the identified flaws. Check {{AUTOSKILLIT_TEMP}}/resolve-design-review/ for the triage report and {{AUTOSKILLIT_TEMP}}/review-design/ for the evaluation dashboard.\n" + }, + "create_worktree": { + "tool": "run_cmd", + "with": { + "cmd": "bash scripts/recipe/create_worktree.sh '${{ inputs.source_dir }}' '${{ inputs.task }}' '${{ context.experiment_plan }}' '${{ context.scope_report }}' '${{ context.evaluation_dashboard }}' '${{ context.visualization_plan_path }}' '${{ context.report_plan_path }}' '{{AUTOSKILLIT_TEMP}}' '${{ context.visualization_plan_trace_path }}'", + "cwd": "${{ inputs.source_dir }}", + "step_name": "create_worktree" + }, + "capture": { + "worktree_path": "${{ result.worktree_path }}", + "research_dir": "${{ result.research_dir }}" + }, + "on_success": "stage_data", + "on_failure": "escalate_stop", + "note": "Creates the shared worktree and commits the experiment plan and scope report to research/ as permanent artifacts. All phases and experiment implementation share this worktree. Auto-initializes a git repo with an empty seed commit when the source directory has no .git directory.\n" + }, + "stage_data": { + "tool": "run_skill", + "model": "", + "with": { + "skill_command": "/autoskillit:stage-data ${{ context.experiment_plan }}", + "cwd": "${{ context.worktree_path }}", + "step_name": "stage_data" + }, + "capture": { + "verdict": "${{ result.verdict }}", + "resource_report": "${{ result.resource_report }}" + }, + "on_result": { + "field": "verdict", + "routes": { + "PASS": "setup_environment", + "WARN": "setup_environment", + "FAIL": "escalate_stop" + } + }, + "on_failure": "escalate_stop", + "note": "Pre-flight resource gate. Reads the experiment plan's data_manifest, checks available disk space and network connectivity for external/gitignored data entries, and creates the required directory structure in the worktree. PASS and WARN proceed to setup_environment. FAIL escalates immediately with a detailed resource feasibility report rather than wasting compute on doomed downloads.\n" + }, + "setup_environment": { + "tool": "run_skill", + "model": "", + "with": { + "skill_command": "/autoskillit:setup-environment ${{ context.worktree_path }} ${{ context.experiment_plan }}", + "cwd": "${{ context.worktree_path }}", + "step_name": "setup_environment" + }, + "capture": { + "verdict": "${{ result.verdict }}" + }, + "on_result": { + "field": "verdict", + "routes": { + "PASS": "decompose_phases", + "WARN": "decompose_phases", + "FAIL": "escalate_stop" + } + }, + "on_failure": "escalate_stop", + "stale_threshold": 600, + "retries": 1, + "on_exhausted": "escalate_stop", + "note": "Pre-flight environment gate. Reads the experiment plan to determine the required environment type (standard, Docker, or micromamba-host). Probes Docker availability, builds the image if possible, falls back to host micromamba if Docker is unavailable and deps are pure conda-forge Python. PASS (none or docker) and WARN (micromamba-host) proceed to decompose_phases. FAIL (unavailable) escalates immediately.\n" + }, + "decompose_phases": { + "tool": "run_skill", + "model": "", + "with": { + "skill_command": "/autoskillit:make-groups ${{ context.experiment_plan }}", + "cwd": "${{ context.worktree_path }}", + "step_name": "decompose" + }, + "capture": { + "group_files": "${{ result.group_files }}" + }, + "on_success": "plan_phase", + "on_failure": "escalate_stop", + "note": "Decomposes the experiment plan into sequenced phases. Each phase becomes an independently plannable implementation unit. Typical decomposition: infrastructure/logging -> data generators -> plotting/analysis scripts -> experiment implementation. ITERATION: The agent maintains a phase queue from group_files. For each phase, run plan_phase then implement_phase before advancing.\n" + }, + "plan_phase": { + "tool": "run_skill", + "model": "", + "with": { + "skill_command": "/autoskillit:make-plan ${{ context.group_files }}", + "cwd": "${{ context.worktree_path }}", + "step_name": "plan_phase", + "group_files": "${{ context.group_files }}" + }, + "capture": { + "phase_plan_path": "${{ result.plan_path }}" + }, + "capture_list": { + "plan_parts": "${{ result.plan_parts }}" + }, + "on_success": "implement_phase", + "on_failure": "escalate_stop", + "note": "Plans the current setup phase. Replace ${{ context.group_files }} in the skill_command with the FIRST REMAINING phase file path from the list — do NOT pass the entire list. Pass the path directly to the skill (do not read it). Glob plan_dir for *_part_*.md or single plan file. ITERATION: After each implement_phase cycle, advance to the next entry in context.group_files for the following plan_phase iteration.\n" + }, + "implement_phase": { + "tool": "run_skill", + "model": "", + "stale_threshold": 2400, + "idle_output_timeout": 0, + "with": { + "skill_command": "/autoskillit:implement-experiment ${{ context.phase_plan_path }}", + "cwd": "${{ context.worktree_path }}", + "step_name": "implement_phase" + }, + "capture": { + "worktree_path": "${{ result.worktree_path }}" + }, + "retries": 0, + "on_success": "next_phase_or_experiment", + "on_failure": "troubleshoot_implement_failure", + "on_exhausted": "run_experiment", + "on_context_limit": "run_experiment", + "note": "Implements the current phase plan in an isolated worktree via implement-experiment. Context exhaustion is a normal termination — the worktree is intact and run_experiment picks up what was committed. Failures route to troubleshoot for classification before re-planning or escalation.\n" + }, + "troubleshoot_implement_failure": { + "tool": "run_skill", + "model": "", + "stale_threshold": 2400, + "idle_output_timeout": 0, + "with": { + "skill_command": "/autoskillit:troubleshoot-experiment ${{ context.worktree_path }} implement_phase", + "cwd": "${{ inputs.source_dir }}", + "step_name": "troubleshoot_implement_failure" + }, + "capture": { + "is_fixable": "${{ result.is_fixable }}" + }, + "on_success": "route_implement_failure", + "on_failure": "escalate_stop" + }, + "route_implement_failure": { + "action": "route", + "on_result": [ + { + "when": "${{ context.is_fixable }} == true", + "route": "check_implement_fix_loop" + }, + { + "route": "escalate_stop" + } + ] + }, + "check_implement_fix_loop": { + "tool": "run_python", + "with": { + "callable": "autoskillit.smoke_utils.check_loop_iteration", + "current_iteration": "${{ context.implement_fix_count }}", + "max_iterations": "3" + }, + "capture": { + "implement_fix_count": "${{ result.next_iteration }}" + }, + "on_result": [ + { + "when": "${{ result.max_exceeded }} == true", + "route": "run_experiment" + }, + { + "route": "plan_phase" + } + ], + "on_failure": "run_experiment", + "optional_context_refs": [ + "implement_fix_count" + ], + "note": "Iteration guard for implement_phase → troubleshoot → plan_phase cycle. Caps fix-and-retry loops at 3 iterations. On exhaustion, skips to run_experiment with whatever implementation is committed.\n" + }, + "next_phase_or_experiment": { + "action": "route", + "on_result": [ + { + "when": "${{ result.next }} == more_phases", + "route": "plan_phase" + }, + { + "route": "run_experiment" + } + ], + "note": "Routing step. The agent evaluates remaining phases: 1. If more phases remain in group_files -> route to plan_phase. 2. If all phases complete -> route to run_experiment.\n" + }, + "run_experiment": { + "tool": "run_skill", + "model": "", + "stale_threshold": 2400, + "idle_output_timeout": 0, + "with": { + "skill_command": "/autoskillit:run-experiment ${{ context.worktree_path }}", + "cwd": "${{ context.worktree_path }}", + "step_name": "run_experiment" + }, + "capture": { + "experiment_results": "${{ result.results_path }}" + }, + "retries": 2, + "on_success": "generate_report", + "on_failure": "troubleshoot_run_failure", + "on_exhausted": "ensure_results" + }, + "troubleshoot_run_failure": { + "tool": "run_skill", + "model": "", + "stale_threshold": 2400, + "idle_output_timeout": 0, + "with": { + "skill_command": "/autoskillit:troubleshoot-experiment ${{ context.worktree_path }} run_experiment", + "cwd": "${{ inputs.source_dir }}", + "step_name": "troubleshoot_run_failure" + }, + "capture": { + "run_is_fixable": "${{ result.is_fixable }}" + }, + "on_success": "route_run_failure", + "on_failure": "escalate_stop" + }, + "route_run_failure": { + "action": "route", + "on_result": [ + { + "when": "${{ context.run_is_fixable }} == true", + "route": "adjust_experiment" + }, + { + "route": "escalate_stop" + } + ] + }, + "adjust_experiment": { + "tool": "run_skill", + "model": "", + "stale_threshold": 2400, + "idle_output_timeout": 0, + "with": { + "skill_command": "/autoskillit:run-experiment ${{ context.worktree_path }} --adjust", + "cwd": "${{ context.worktree_path }}", + "step_name": "adjust" + }, + "capture": { + "experiment_results": "${{ result.results_path }}" + }, + "on_success": "check_run_fix_loop", + "on_failure": "ensure_results", + "on_context_limit": "ensure_results" + }, + "check_run_fix_loop": { + "tool": "run_python", + "with": { + "callable": "autoskillit.smoke_utils.check_loop_iteration", + "current_iteration": "${{ context.run_fix_count }}", + "max_iterations": "3" + }, + "capture": { + "run_fix_count": "${{ result.next_iteration }}" + }, + "on_result": [ + { + "when": "${{ result.max_exceeded }} == true", + "route": "ensure_results" + }, + { + "route": "run_experiment" + } + ], + "on_failure": "ensure_results", + "optional_context_refs": [ + "run_fix_count" + ], + "note": "Iteration guard for run_experiment → troubleshoot → adjust_experiment cycle. Caps experiment fix loops at 3 iterations. On exhaustion, proceeds to ensure_results to salvage whatever data exists.\n" + }, + "ensure_results": { + "tool": "run_python", + "with": { + "callable": "autoskillit.recipe._cmd_rpc.ensure_results", + "experiment_results": "${{ context.experiment_results }}", + "worktree_path": "${{ context.worktree_path }}", + "temp_subdir": "{{AUTOSKILLIT_TEMP}}" + }, + "capture": { + "experiment_results": "${{ result.experiment_results }}" + }, + "on_success": "generate_report_inconclusive", + "on_failure": "escalate_stop", + "note": "Ensures context.experiment_results is a valid file path before calling generate_report_inconclusive. Creates a minimal placeholder results file when run-experiment never produced output (retries exhausted or adjust failed).\n" + }, + "generate_report": { + "tool": "run_skill", + "model": "", + "with": { + "skill_command": "/autoskillit:generate-report ${{ context.worktree_path }} ${{ context.experiment_results }} --output-mode ${{ inputs.output_mode }} --issue-url ${{ inputs.issue_url }} --experiment-type '${{ context.experiment_type }}' --methodology-traditions '${{ context.methodology_tradition }}' --design-review-verdict '${{ context.verdict }}' --disambiguation-rule-applied '${{ context.disambiguation_rule_applied }}' --tier-c-lens '${{ context.tier_c_lens }}' --classification-timestamp '${{ context.classification_timestamp }}'", + "cwd": "${{ context.worktree_path }}", + "step_name": "generate_report" + }, + "capture": { + "report_path": "${{ result.report_path }}" + }, + "on_success": "test", + "on_failure": "escalate_stop", + "on_context_limit": "test", + "note": "Writes a conclusive research report to research/ in the worktree." + }, + "generate_report_inconclusive": { + "tool": "run_skill", + "model": "", + "with": { + "skill_command": "/autoskillit:generate-report ${{ context.worktree_path }} ${{ context.experiment_results }} --inconclusive --output-mode ${{ inputs.output_mode }} --issue-url ${{ inputs.issue_url }} --experiment-type '${{ context.experiment_type }}' --methodology-traditions '${{ context.methodology_tradition }}' --design-review-verdict '${{ context.verdict }}' --disambiguation-rule-applied '${{ context.disambiguation_rule_applied }}' --tier-c-lens '${{ context.tier_c_lens }}' --classification-timestamp '${{ context.classification_timestamp }}'", + "cwd": "${{ context.worktree_path }}", + "step_name": "generate_report" + }, + "capture": { + "report_path": "${{ result.report_path }}" + }, + "on_success": "test", + "on_failure": "escalate_stop", + "on_context_limit": "test", + "note": "Writes an inconclusive research report. Experiments could not prove or disprove the hypothesis. This is a valid outcome.\n" + }, + "test": { + "tool": "test_check", + "with": { + "worktree_path": "${{ context.worktree_path }}" + }, + "on_success": "push_branch", + "on_failure": "fix_tests" + }, + "fix_tests": { + "tool": "run_skill", + "model": "", + "with": { + "skill_command": "/autoskillit:resolve-failures ${{ context.worktree_path }} ${{ context.experiment_plan }} ${{ inputs.base_branch }}", + "cwd": "${{ inputs.source_dir }}", + "step_name": "fix" + }, + "capture": { + "verdict": "${{ result.verdict }}", + "fixes_applied": "${{ result.fixes_applied }}" + }, + "on_result": [ + { + "when": "${{ result.verdict }} == 'real_fix'", + "route": "retest" + }, + { + "when": "${{ result.verdict }} == 'already_green'", + "route": "retest" + }, + { + "when": "${{ result.verdict }} == 'flake_suspected'", + "route": "retest" + }, + { + "when": "${{ result.verdict }} == 'ci_only_failure'", + "route": "escalate_stop" + } + ], + "on_failure": "escalate_stop", + "on_context_limit": "retest" + }, + "retest": { + "tool": "test_check", + "with": { + "worktree_path": "${{ context.worktree_path }}" + }, + "on_success": "push_branch", + "on_failure": "escalate_stop" + }, + "push_branch": { + "tool": "run_cmd", + "with": { + "cmd": "bash scripts/recipe/push_branch.sh '${{ inputs.output_mode }}' '${{ context.worktree_path }}'", + "cwd": "${{ context.worktree_path }}", + "step_name": "push_branch" + }, + "on_success": "prepare_research_pr", + "on_failure": "escalate_stop" + }, + "prepare_research_pr": { + "tool": "run_skill", + "model": "", + "with": { + "skill_command": "/autoskillit:prepare-research-pr ${{ context.report_path }} ${{ context.experiment_plan }} ${{ context.worktree_path }} ${{ inputs.base_branch }}", + "cwd": "${{ context.worktree_path }}", + "step_name": "prepare_research_pr" + }, + "capture": { + "prep_path": "${{ result.prep_path }}", + "selected_lenses": "${{ result.selected_lenses }}", + "lens_context_paths": "${{ result.lens_context_paths }}" + }, + "on_success": "run_experiment_lenses", + "on_failure": "escalate_stop" + }, + "run_experiment_lenses": { + "tool": "run_skill", + "model": "", + "with": { + "skill_command": "/autoskillit:exp-lens-{slug} {context_path} ${{ context.experiment_plan }}", + "cwd": "${{ context.worktree_path }}", + "step_name": "run_experiment_lenses" + }, + "capture_list": { + "all_diagram_paths": "${{ result.diagram_path }}" + }, + "on_success": "stage_bundle", + "on_failure": "stage_bundle", + "note": "LENS ITERATION: context.selected_lenses contains comma-separated lens slugs (e.g. \"fair-comparison,estimand-clarity\"). context.lens_context_paths contains the corresponding comma-separated context file paths. For each lens slug, run:\n /autoskillit:exp-lens-{slug} {matching_context_path} ${{ context.experiment_plan }}\nRun all selected lenses in parallel (multiple run_skill calls in a single assistant message). Accumulate each diagram_path value into context.all_diagram_paths using comma separation via capture_list. If a lens fails, continue with remaining lenses — partial diagrams are acceptable (compose_research_pr handles empty lists).\n" + }, + "stage_bundle": { + "tool": "run_cmd", + "with": { + "cmd": "bash scripts/recipe/stage_bundle.sh '${{ context.research_dir }}' '${{ context.worktree_path }}' '{{AUTOSKILLIT_TEMP}}'", + "cwd": "${{ context.worktree_path }}", + "step_name": "stage_bundle" + }, + "on_success": "route_pr_or_local", + "on_failure": "route_pr_or_local", + "note": "Idempotent file staging step. Copies phase-group, phase-plan, and exp-lens diagram artifacts from the worktree temp directory into research/{slug}/artifacts/. Creates images/ and scripts/ subdirectories. No compression, no rename, no commit. Safe to run multiple times — cp overwrites are idempotent. Runs after run_experiment_lenses to capture all exp-lens diagrams written to AUTOSKILLIT_TEMP/exp-lens-*/.\n" + }, + "route_pr_or_local": { + "action": "route", + "on_result": [ + { + "when": "${{ inputs.output_mode }} == local", + "route": "finalize_bundle" + }, + { + "route": "compose_research_pr" + } + ], + "note": "Branches on output_mode after bundle staging. local → bypass PR pipeline entirely, jump to finalize_bundle. pr (fall-through) → compose_research_pr to open PR and run the full review/archival flow. Uses predicate route because skip_when_false only accepts bare inputs. with no expression evaluation.\n" + }, + "compose_research_pr": { + "tool": "run_skill", + "model": "", + "with": { + "skill_command": "/autoskillit:compose-research-pr ${{ context.prep_path }} \"${{ context.all_diagram_paths }}\" ${{ context.worktree_path }} ${{ inputs.base_branch }}", + "cwd": "${{ context.worktree_path }}", + "step_name": "compose_research_pr" + }, + "capture": { + "pr_url": "${{ result.pr_url }}" + }, + "on_success": "guard_pr_url", + "on_failure": "escalate_stop" + }, + "guard_pr_url": { + "action": "route", + "on_result": [ + { + "when": "${{ context.pr_url }}", + "route": "review_research_pr" + }, + { + "route": "patch_token_summary" + } + ], + "note": "Guard — skip review if pr_url is empty (graceful degradation from open_research_pr)." + }, + "review_research_pr": { + "tool": "run_skill", + "model": "", + "with": { + "skill_command": "/autoskillit:review-research-pr ${{ context.worktree_path }} ${{ inputs.base_branch }} ${{ context.pr_url }}", + "cwd": "${{ context.worktree_path }}", + "step_name": "review_research_pr" + }, + "capture": { + "review_verdict": "${{ result.verdict }}" + }, + "skip_when_false": "inputs.review_pr", + "on_context_limit": "audit_claims", + "on_failure": "audit_claims", + "on_result": [ + { + "route": "audit_claims" + } + ], + "note": "Optional research PR review step, gated by the review_pr ingredient. Verdict captured as review_verdict for deferred routing. All paths (skip, success, failure, context-limit) route to audit_claims so both gates complete before any resolution step begins. When on_context_limit or on_failure fires, the capture block does not execute and context.review_verdict remains absent; the recipe engine treats absent context variables as empty string in comparisons, so downstream route_review_resolve safely falls through to route_claims_resolve.\n" + }, + "audit_claims": { + "tool": "run_skill", + "model": "", + "with": { + "skill_command": "/autoskillit:audit-claims ${{ context.worktree_path }} ${{ inputs.base_branch }} ${{ context.pr_url }}", + "cwd": "${{ context.worktree_path }}", + "step_name": "audit_claims" + }, + "capture": { + "audit_verdict": "${{ result.verdict }}" + }, + "skip_when_false": "inputs.audit_claims", + "retries": 1, + "on_exhausted": "route_review_resolve", + "on_context_limit": "route_review_resolve", + "on_failure": "route_review_resolve", + "on_result": [ + { + "route": "route_review_resolve" + } + ], + "note": "Optional citation-integrity gate, gated by the audit_claims ingredient. Runs after review_research_pr regardless of that step's verdict. All paths (skip, success, failure) route to route_review_resolve so both gates complete before resolution.\n" + }, + "route_review_resolve": { + "action": "route", + "on_result": [ + { + "when": "${{ context.review_verdict }} == changes_requested", + "route": "resolve_research_review" + }, + { + "route": "route_claims_resolve" + } + ], + "note": "Routes to resolve_research_review only when the PR review found changes_requested. All other verdicts (approved, needs_human, empty/skipped) proceed directly to route_claims_resolve.\n" + }, + "resolve_research_review": { + "tool": "run_skill", + "model": "", + "with": { + "skill_command": "/autoskillit:resolve-research-review ${{ context.worktree_path }} ${{ inputs.base_branch }}", + "cwd": "${{ context.worktree_path }}", + "step_name": "resolve_research_review" + }, + "capture": { + "review_needs_rerun": "${{ result.needs_rerun }}" + }, + "retries": 2, + "on_exhausted": "route_claims_resolve", + "on_context_limit": "route_claims_resolve", + "on_success": "route_claims_resolve", + "on_failure": "route_claims_resolve" + }, + "route_claims_resolve": { + "action": "route", + "on_result": [ + { + "when": "${{ context.audit_verdict }} == changes_requested", + "route": "resolve_claims_review" + }, + { + "route": "merge_escalations" + } + ], + "note": "Routes to resolve_claims_review only when the claims audit found changes_requested. All other verdicts (approved, needs_human, empty/skipped) proceed directly to merge_escalations. When audit_claims is skipped (inputs.audit_claims=false), context.audit_verdict is absent; the recipe engine treats absent variables as empty string, so the condition evaluates false and the fallback route to merge_escalations fires safely.\n" + }, + "resolve_claims_review": { + "tool": "run_skill", + "model": "", + "with": { + "skill_command": "/autoskillit:resolve-claims-review ${{ context.worktree_path }} ${{ inputs.base_branch }}", + "cwd": "${{ context.worktree_path }}", + "step_name": "resolve_claims_review" + }, + "capture": { + "claims_needs_rerun": "${{ result.needs_rerun }}" + }, + "retries": 2, + "on_exhausted": "merge_escalations", + "on_context_limit": "merge_escalations", + "on_success": "merge_escalations", + "on_failure": "merge_escalations" + }, + "merge_escalations": { + "action": "route", + "on_result": [ + { + "when": "${{ context.review_needs_rerun }} == true", + "route": "re_run_experiment" + }, + { + "when": "${{ context.claims_needs_rerun }} == true", + "route": "re_run_experiment" + }, + { + "route": "finalize_bundle" + } + ], + "note": "Merged escalation consumer replacing check_escalations. Triggers a single re_run_experiment if either review resolution or claims resolution flagged a rerun. When only design_flaw escalations exist (or none), routes directly to finalize_bundle — design_flaw escalations are already posted as inline PR comments by the resolve skills. Both context.review_needs_rerun and context.claims_needs_rerun may be absent if their respective resolution steps were never reached (e.g., no changes_requested verdict on that gate); the recipe engine evaluates absent context variables as empty string, so absent == true is always false and routes safely fall through to finalize_bundle.\n" + }, + "re_run_experiment": { + "tool": "run_skill", + "model": "", + "stale_threshold": 2400, + "idle_output_timeout": 0, + "with": { + "skill_command": "/autoskillit:run-experiment ${{ context.worktree_path }} --adjust", + "cwd": "${{ context.worktree_path }}", + "step_name": "re_run_experiment" + }, + "capture": { + "experiment_results": "${{ result.results_path }}" + }, + "on_success": "re_generate_report", + "on_failure": "re_push_research", + "on_context_limit": "re_push_research", + "note": "Targeted re-run of affected benchmarks after review resolution applied code fixes. Uses --adjust to read previous results and re-run only what changed. Captures the updated results_path so re_generate_report uses the new results. on_failure falls through to push as best-effort — the code fixes from resolve are already committed and should still be pushed.\n" + }, + "re_generate_report": { + "tool": "run_skill", + "model": "", + "with": { + "skill_command": "/autoskillit:generate-report ${{ context.worktree_path }} ${{ context.experiment_results }} --output-mode ${{ inputs.output_mode }} --issue-url ${{ inputs.issue_url }} --experiment-type '${{ context.experiment_type }}' --methodology-traditions '${{ context.methodology_tradition }}' --design-review-verdict '${{ context.verdict }}' --disambiguation-rule-applied '${{ context.disambiguation_rule_applied }}' --tier-c-lens '${{ context.tier_c_lens }}' --classification-timestamp '${{ context.classification_timestamp }}'", + "cwd": "${{ context.worktree_path }}", + "step_name": "re_generate_report" + }, + "capture": { + "report_path": "${{ result.report_path }}" + }, + "on_success": "re_stage_bundle", + "on_failure": "re_push_research", + "on_context_limit": "re_push_research", + "note": "Re-generate the research report with updated benchmark results from the re-validation run. Passes context.experiment_results which was updated by re_run_experiment to point to the new results file. Falls through to push on failure.\n" + }, + "re_stage_bundle": { + "tool": "run_cmd", + "with": { + "cmd": "bash scripts/recipe/stage_bundle.sh '${{ context.research_dir }}' '${{ context.worktree_path }}' '{{AUTOSKILLIT_TEMP}}'", + "cwd": "${{ context.worktree_path }}", + "step_name": "re_stage_bundle" + }, + "on_success": "re_test", + "on_failure": "re_push_research", + "note": "Idempotent re-staging after report rewrite. Same body as stage_bundle — updates artifact directory with any newly written phase artifacts before re-testing. Falls through to re_push_research on failure (best-effort).\n" + }, + "re_test": { + "tool": "test_check", + "with": { + "worktree_path": "${{ context.worktree_path }}" + }, + "on_success": "re_push_research", + "on_failure": "re_push_research", + "note": "Post-revalidation test gate. Both pass and fail route to push — the re-validation results should be pushed regardless so the PR reflects the current state of the code and measurements.\n" + }, + "re_push_research": { + "tool": "run_cmd", + "with": { + "cmd": "cd '${{ context.worktree_path }}' && git push", + "cwd": "${{ context.worktree_path }}", + "step_name": "re_push_research" + }, + "on_success": "finalize_bundle_render", + "on_failure": "begin_archival" + }, + "finalize_bundle": { + "tool": "run_cmd", + "with": { + "cmd": "bash scripts/recipe/finalize_bundle.sh '${{ inputs.output_mode }}' '${{ context.research_dir }}' '${{ context.worktree_path }}'", + "cwd": "${{ context.worktree_path }}", + "step_name": "finalize_bundle" + }, + "capture": { + "report_path_after_finalize": "${{ result.report_path_after_finalize }}" + }, + "on_success": "re_push_research", + "on_failure": "begin_archival", + "note": "Single compression point. Runs exactly once per recipe run. In pr mode: renames report.md → README.md, archives supporting artifacts, appends Archive Manifest, commits. In local mode: keeps report.md, excludes report.html/mermaid.min.js/images from the tar (they stay for local browsing), skips git commit. finalize_bundle_render always runs after this step regardless of mode.\n" + }, + "finalize_bundle_render": { + "tool": "run_skill", + "model": "", + "with": { + "skill_command": "/autoskillit:bundle-local-report ${{ context.research_dir }} ${{ context.report_path_after_finalize }} ${{ context.all_diagram_paths }} ${{ context.visualization_plan_path }}", + "cwd": "${{ context.worktree_path }}", + "step_name": "finalize_bundle_render" + }, + "capture": { + "html_path": "${{ result.html_path }}" + }, + "on_success": "route_archive_or_export", + "on_failure": "route_archive_or_export", + "note": "HTML report rendering. Always runs after finalize_bundle on_success. Non-fatal: routes to route_archive_or_export on both outcomes, which branches on output_mode to either export_local_bundle (local) or begin_archival (pr).\n" + }, + "route_archive_or_export": { + "action": "route", + "on_result": [ + { + "when": "${{ inputs.output_mode }} == local", + "route": "export_local_bundle" + }, + { + "route": "begin_archival" + } + ], + "note": "Branches after HTML bundle rendering. local → export_local_bundle (copy to {source_dir}/research-bundles/). pr (fall-through) → begin_archival to run the existing archival phase.\n" + }, + "export_local_bundle": { + "tool": "run_python", + "with": { + "callable": "autoskillit.recipe._cmd_rpc.export_local_bundle", + "source_dir": "${{ inputs.source_dir }}", + "research_dir": "${{ context.research_dir }}" + }, + "capture": { + "local_bundle_path": "${{ result.local_bundle_path }}" + }, + "on_success": "patch_token_summary", + "on_failure": "patch_token_summary", + "note": "Local mode terminal export. Copies the finalized research_dir (containing report.html, mermaid.min.js, images/, report.md, artifacts.tar.gz) to {source_dir}/research-bundles/{slug}/. Non-fatal — routes to patch_token_summary on both outcomes. The captured local_bundle_path is available to the orchestrator via research_complete.\n" + }, + "begin_archival": { + "action": "route", + "on_result": [ + { + "when": "${{ context.pr_url }}", + "route": "capture_experiment_branch" + }, + { + "route": "patch_token_summary" + } + ], + "note": "Gate step for the archival phase. Routes to artifact extraction when a research PR exists, otherwise skips directly to patch_token_summary. All paths from review/resolve/re-push converge here.\n" + }, + "capture_experiment_branch": { + "tool": "run_cmd", + "with": { + "cmd": "git -C '${{ context.worktree_path }}' rev-parse --abbrev-ref HEAD", + "cwd": "${{ context.worktree_path }}", + "step_name": "capture_experiment_branch" + }, + "capture": { + "experiment_branch": "${{ result.stdout | trim }}" + }, + "on_success": "create_artifact_branch", + "on_failure": "patch_token_summary" + }, + "create_artifact_branch": { + "tool": "run_cmd", + "with": { + "cmd": "bash scripts/recipe/create_artifact_branch.sh '${{ context.worktree_path }}' '${{ context.experiment_branch }}' '${{ inputs.base_branch }}' '${{ context.research_dir }}'", + "cwd": "${{ context.worktree_path }}", + "step_name": "create_artifact_branch" + }, + "capture": { + "artifact_branch": "${{ result.artifact_branch }}" + }, + "on_success": "open_artifact_pr", + "on_failure": "patch_token_summary", + "note": "Creates a temporary worktree from the base branch, checks out only the research/ directory from the experiment branch, commits and pushes, then removes the temporary worktree. Produces a single clean commit with zero production code changes.\n" + }, + "open_artifact_pr": { + "tool": "run_cmd", + "with": { + "cmd": "bash scripts/recipe/open_artifact_pr.sh '${{ context.worktree_path }}' '${{ context.experiment_branch }}' '${{ context.artifact_branch }}' '${{ context.pr_url }}' '${{ inputs.base_branch }}' '${{ context.research_dir }}'", + "cwd": "${{ context.worktree_path }}", + "step_name": "open_artifact_pr" + }, + "capture": { + "artifact_pr_url": "${{ result.artifact_pr_url }}" + }, + "on_success": "tag_experiment_branch", + "on_failure": "patch_token_summary", + "note": "Opens a PR containing only the research/ directory. The PR body documents the two-file structure (README.md + artifacts.tar.gz) and provides the extraction command. This PR is safe to merge — no production code.\n" + }, + "tag_experiment_branch": { + "tool": "run_cmd", + "with": { + "cmd": "bash scripts/recipe/tag_experiment_branch.sh '${{ context.worktree_path }}' '${{ context.experiment_branch }}' '${{ context.artifact_pr_url }}'", + "cwd": "${{ context.worktree_path }}", + "step_name": "tag_experiment_branch" + }, + "capture": { + "archive_tag": "${{ result.archive_tag }}" + }, + "on_success": "close_experiment_pr", + "on_failure": "patch_token_summary", + "note": "Creates an annotated tag preserving the full experiment branch (including all variant code, benchmark scaffolding, etc.) under archive/research/. Pushed to remote before any branch cleanup can occur.\n" + }, + "close_experiment_pr": { + "tool": "run_cmd", + "with": { + "cmd": "ARTIFACT_PR=\"${{ context.artifact_pr_url }}\" && ARCHIVE_TAG=\"${{ context.archive_tag }}\" && CLOSE_BODY=$(printf '## Archived\\n\\nThis PR contained both research artifacts and experimental code.\\n\\n**Research record merged via:** %s\\n**Full experiment branch preserved at:** `%s`\\n\\nThe artifact PR contains only files under `research/` (reports, scripts, results). The archive tag preserves the full experiment branch including variant code for reference.\\n\\nTo restore the full experiment: `git checkout %s`' \"${ARTIFACT_PR}\" \"${ARCHIVE_TAG}\" \"${ARCHIVE_TAG}\") && gh pr close '${{ context.pr_url }}' --comment \"${CLOSE_BODY}\"\n", + "cwd": "${{ context.worktree_path }}", + "step_name": "close_experiment_pr" + }, + "on_success": "patch_token_summary", + "on_failure": "patch_token_summary", + "note": "Closes the original experiment PR with a structured comment linking to the artifact PR (where research/ was merged) and the archive tag (where the full experiment branch is preserved). Both success and failure route to patch_token_summary — closing the PR is best-effort.\n" + }, + "patch_token_summary": { + "description": "Collect all pipeline token data and patch the PR body with complete summary", + "tool": "run_python", + "with": { + "callable": "autoskillit.smoke_utils.patch_pr_token_summary", + "timeout": 60, + "pr_url": "${{ context.pr_url }}", + "cwd": "${{ context.worktree_path }}" + }, + "on_success": "research_complete", + "on_failure": "research_complete", + "note": "Patches PR body with complete token summary; failure never blocks pipeline completion.\n" + }, + "research_complete": { + "action": "stop", + "message": "Research complete. pr mode: report added to research/ folder and PR opened for review. local mode: bundle exported to {source_dir}/research-bundles/{slug}/ (context.local_bundle_path).\n" + }, + "escalate_stop": { + "action": "stop", + "message": "Research pipeline failed — human intervention needed. Check the latest step output for details." + } + } +} diff --git a/src/autoskillit/recipes/research.yaml b/src/autoskillit/recipes/research.yaml index 6b8a7bef7c..ce693afb67 100644 --- a/src/autoskillit/recipes/research.yaml +++ b/src/autoskillit/recipes/research.yaml @@ -1,18 +1,18 @@ name: research -autoskillit_version: "0.8.0" +recipe_version: "1.0.0" requires_packs: [research, exp-lens, vis-lens] description: > Single-phase technical research recipe. Scopes a research question, plans an experiment, optionally reviews the design, then always decomposes into phases, implements, runs the experiment, writes a report to research/, and opens a PR. - Supports a skippable design review loop (bounded by retries: 2), review + Supports a skippable design review loop (bounded by check_design_review_loop at 3 iterations), review resolution routing after PR open, and a post-review re-validation loop that re-runs affected benchmarks when resolve-research-review produces rerun_required escalations. After the final push (or when review completes), an archival phase separates research artifacts from experimental code: opens a clean artifact-only PR, tags the experiment branch, and closes the original PR. No auto-merge. -summary: scope > plan > [review?] > worktree > decompose > phases > experiment > report > open-pr > [review? > resolve > [re-run?] > push] > archive +summary: scope > plan > [review?] > worktree > env-setup > decompose > phases > experiment > report > open-pr > [review? > resolve > [re-run?] > push] > archive ingredients: task: @@ -34,9 +34,9 @@ ingredients: When 'true' (the default), run the review-design skill after plan_experiment to validate the experiment plan before execution. Emits a verdict: GO proceeds, REVISE loops back to plan_experiment with revision guidance (bounded by - retries: 2), STOP triggers resolve_design_review for triage — addressable - findings loop back, structural findings halt the pipeline. When 'false', skip - directly to create_worktree. + check_design_review_loop at 3 iterations), STOP triggers resolve_design_review + for triage — addressable findings loop back, structural findings halt the + pipeline. When 'false', skip directly to create_worktree. default: "true" review_pr: description: > @@ -73,7 +73,7 @@ kitchen_rules: - > DESIGN REVIEW LOOP: review_design validates the plan and may route back to plan_experiment with revision guidance. Bounded by - retries: 2 (max 3 review cycles). On exhaustion or failure, + check_design_review_loop (max 3 iterations). On exhaustion or failure, proceed to execution with the best available plan — do not stop. Exception: a STOP verdict triggers resolve_design_review, which triages each stop-trigger finding as ADDRESSABLE/STRUCTURAL/DISCUSS. If any are @@ -103,6 +103,7 @@ steps: # --- DESIGN PHASE --- scope: tool: run_skill + model: "" with: skill_command: "/autoskillit:scope ${{ inputs.task }}" cwd: "${{ inputs.source_dir }}" @@ -114,6 +115,7 @@ steps: plan_experiment: tool: run_skill + model: "" with: skill_command: "/autoskillit:plan-experiment ${{ context.scope_report }} ${{ context.revision_guidance }}" cwd: "${{ inputs.source_dir }}" @@ -126,6 +128,7 @@ steps: review_design: tool: run_skill + model: "" with: skill_command: "/autoskillit:review-design ${{ context.experiment_plan }}" cwd: "${{ inputs.source_dir }}" @@ -135,6 +138,7 @@ steps: experiment_type: "${{ result.experiment_type }}" evaluation_dashboard: "${{ result.evaluation_dashboard }}" revision_guidance: "${{ result.revision_guidance }}" + classification_timestamp: "${{ result.classification_timestamp }}" skip_when_false: inputs.review_design retries: 2 on_exhausted: create_worktree @@ -151,6 +155,7 @@ steps: plan_visualization: tool: run_skill + model: "" with: skill_command: "/autoskillit:plan-visualization ${{ inputs.source_dir }} ${{ context.experiment_plan }} ${{ context.scope_report }}" cwd: "${{ inputs.source_dir }}" @@ -158,6 +163,10 @@ steps: capture: visualization_plan_path: "${{ result.visualization_plan_path }}" report_plan_path: "${{ result.report_plan_path }}" + disambiguation_rule_applied: "${{ result.disambiguation_rule_applied }}" + tier_c_lens: "${{ result.tier_c_lens }}" + methodology_tradition: "${{ result.methodology_tradition }}" + visualization_plan_trace_path: "${{ result.visualization_plan_trace_path }}" on_success: create_worktree on_failure: escalate_stop note: > @@ -168,10 +177,31 @@ steps: revise_design: action: route on_result: + - route: check_design_review_loop + + check_design_review_loop: + tool: run_python + with: + callable: "autoskillit.smoke_utils.check_loop_iteration" + current_iteration: "${{ context.design_review_count }}" + max_iterations: "3" + capture: + design_review_count: "${{ result.next_iteration }}" + on_result: + - when: "${{ result.max_exceeded }} == true" + route: create_worktree - route: plan_experiment + on_failure: create_worktree + optional_context_refs: + - design_review_count + note: > + Iteration guard for the review_design → revise_design → plan_experiment cycle. + Caps design revision loops at 3 iterations. On exhaustion, proceeds to + create_worktree with the best available design. resolve_design_review: tool: run_skill + model: "" with: skill_command: "/autoskillit:resolve-design-review ${{ context.evaluation_dashboard }} ${{ context.experiment_plan }} ${{ context.revision_guidance }}" cwd: "${{ inputs.source_dir }}" @@ -205,40 +235,7 @@ steps: create_worktree: tool: run_cmd with: - cmd: > - BRANCH="research-$(date +%Y%m%d-%H%M%S)" && - WORKTREE_PATH="../worktrees/${BRANCH}" && - git worktree add -b "${BRANCH}" "${WORKTREE_PATH}" && - RESOLVED="$(cd "${WORKTREE_PATH}" && pwd)" && - case "${RESOLVED}" in /*) ;; *) echo "error: resolved worktree path is not absolute: ${RESOLVED}" >&2; exit 1;; esac && - mkdir -p "{{AUTOSKILLIT_TEMP}}/worktrees/${BRANCH}" && - echo "$(git rev-parse --abbrev-ref HEAD)" > "{{AUTOSKILLIT_TEMP}}/worktrees/${BRANCH}/base-branch" && - mkdir -p "${RESOLVED}/{{AUTOSKILLIT_TEMP}}" && - EXPERIMENT_PLAN="${{ context.experiment_plan }}" && - cp "${EXPERIMENT_PLAN}" "${RESOLVED}/{{AUTOSKILLIT_TEMP}}/experiment-plan.md" && - TASK="${{ inputs.task }}" && - SLUG=$(printf '%s' "${TASK}" | tr '[:upper:]' '[:lower:]' | tr -cs 'a-z0-9' '-' | sed 's/--*/-/g;s/^-//;s/-$//' | cut -c1-30) && - RESEARCH_DIR="${RESOLVED}/research/$(date +%Y-%m-%d)-${SLUG:-experiment}" && - mkdir -p "${RESEARCH_DIR}/artifacts" && - cp "${EXPERIMENT_PLAN}" "${RESEARCH_DIR}/experiment-plan.md" && - SCOPE_REPORT="${{ context.scope_report }}" && - EVAL_DASHBOARD="${{ context.evaluation_dashboard }}" && - if [ -n "${SCOPE_REPORT}" ] && [ -f "${SCOPE_REPORT}" ]; then cp "${SCOPE_REPORT}" "${RESEARCH_DIR}/artifacts/scope-report.md"; fi && - if [ -n "${EVAL_DASHBOARD}" ] && [ -f "${EVAL_DASHBOARD}" ]; then cp "${EVAL_DASHBOARD}" "${RESEARCH_DIR}/artifacts/design-evaluation.md"; fi && - VISUALIZATION_PLAN="${{ context.visualization_plan_path }}" && - REPORT_PLAN="${{ context.report_plan_path }}" && - if [ -n "${VISUALIZATION_PLAN}" ] && [ -f "${VISUALIZATION_PLAN}" ]; then cp "${VISUALIZATION_PLAN}" "${RESEARCH_DIR}/visualization-plan.md"; fi && - if [ -n "${REPORT_PLAN}" ] && [ -f "${REPORT_PLAN}" ]; then cp "${REPORT_PLAN}" "${RESEARCH_DIR}/report-plan.md"; fi && - SRC_TEMP="${{ inputs.source_dir }}/{{AUTOSKILLIT_TEMP}}" && - mkdir -p "${RESEARCH_DIR}/artifacts/review-cycles" && - mkdir -p "${RESEARCH_DIR}/artifacts/plan-versions" && - for f in "${SRC_TEMP}"/review-design/evaluation_dashboard_*.md; do [ -f "$f" ] && cp "$f" "${RESEARCH_DIR}/artifacts/review-cycles/"; done; - for f in "${SRC_TEMP}"/review-design/revision_guidance_*.md; do [ -f "$f" ] && cp "$f" "${RESEARCH_DIR}/artifacts/review-cycles/"; done; - for f in "${SRC_TEMP}"/plan-experiment/experiment_plan_*.md; do [ -f "$f" ] && cp "$f" "${RESEARCH_DIR}/artifacts/plan-versions/"; done; - for f in "${SRC_TEMP}"/resolve-design-review/*.md; do [ -f "$f" ] && cp "$f" "${RESEARCH_DIR}/artifacts/review-cycles/"; done; - cd "${RESOLVED}" && git add research/ && git commit -m "Add experiment plan and scope to research/" && - echo "research_dir=${RESEARCH_DIR}" && - echo "worktree_path=${RESOLVED}" + cmd: "bash scripts/recipe/create_worktree.sh '${{ inputs.source_dir }}' '${{ inputs.task }}' '${{ context.experiment_plan }}' '${{ context.scope_report }}' '${{ context.evaluation_dashboard }}' '${{ context.visualization_plan_path }}' '${{ context.report_plan_path }}' '{{AUTOSKILLIT_TEMP}}' '${{ context.visualization_plan_trace_path }}'" cwd: "${{ inputs.source_dir }}" step_name: create_worktree capture: @@ -249,10 +246,12 @@ steps: note: > Creates the shared worktree and commits the experiment plan and scope report to research/ as permanent artifacts. All phases and experiment implementation - share this worktree. + share this worktree. Auto-initializes a git repo with an empty seed commit + when the source directory has no .git directory. stage_data: tool: run_skill + model: "" with: skill_command: "/autoskillit:stage-data ${{ context.experiment_plan }}" cwd: "${{ context.worktree_path }}" @@ -263,20 +262,48 @@ steps: on_result: field: verdict routes: - PASS: decompose_phases - WARN: decompose_phases + PASS: setup_environment + WARN: setup_environment FAIL: escalate_stop on_failure: escalate_stop note: > Pre-flight resource gate. Reads the experiment plan's data_manifest, checks available disk space and network connectivity for external/gitignored data entries, and creates the required directory structure in the worktree. PASS - and WARN proceed to decompose_phases. FAIL escalates immediately with a + and WARN proceed to setup_environment. FAIL escalates immediately with a detailed resource feasibility report rather than wasting compute on doomed downloads. + setup_environment: + tool: run_skill + model: "" + with: + skill_command: "/autoskillit:setup-environment ${{ context.worktree_path }} ${{ context.experiment_plan }}" + cwd: "${{ context.worktree_path }}" + step_name: setup_environment + capture: + verdict: "${{ result.verdict }}" + on_result: + field: verdict + routes: + PASS: decompose_phases + WARN: decompose_phases + FAIL: escalate_stop + on_failure: escalate_stop + stale_threshold: 600 + retries: 1 + on_exhausted: escalate_stop + note: > + Pre-flight environment gate. Reads the experiment plan to determine the + required environment type (standard, Docker, or micromamba-host). Probes + Docker availability, builds the image if possible, falls back to host + micromamba if Docker is unavailable and deps are pure conda-forge Python. + PASS (none or docker) and WARN (micromamba-host) proceed to decompose_phases. + FAIL (unavailable) escalates immediately. + decompose_phases: tool: run_skill + model: "" with: skill_command: "/autoskillit:make-groups ${{ context.experiment_plan }}" cwd: "${{ context.worktree_path }}" @@ -295,6 +322,7 @@ steps: plan_phase: tool: run_skill + model: "" with: skill_command: "/autoskillit:make-plan ${{ context.group_files }}" cwd: "${{ context.worktree_path }}" @@ -316,6 +344,7 @@ steps: implement_phase: tool: run_skill + model: "" stale_threshold: 2400 idle_output_timeout: 0 with: @@ -337,6 +366,7 @@ steps: troubleshoot_implement_failure: tool: run_skill + model: "" stale_threshold: 2400 idle_output_timeout: 0 with: @@ -352,9 +382,29 @@ steps: action: route on_result: - when: "${{ context.is_fixable }} == true" - route: plan_phase + route: check_implement_fix_loop - route: escalate_stop + check_implement_fix_loop: + tool: run_python + with: + callable: "autoskillit.smoke_utils.check_loop_iteration" + current_iteration: "${{ context.implement_fix_count }}" + max_iterations: "3" + capture: + implement_fix_count: "${{ result.next_iteration }}" + on_result: + - when: "${{ result.max_exceeded }} == true" + route: run_experiment + - route: plan_phase + on_failure: run_experiment + optional_context_refs: + - implement_fix_count + note: > + Iteration guard for implement_phase → troubleshoot → plan_phase cycle. + Caps fix-and-retry loops at 3 iterations. On exhaustion, skips to + run_experiment with whatever implementation is committed. + next_phase_or_experiment: action: route on_result: @@ -369,6 +419,7 @@ steps: # --- EXPERIMENT + REPORT PHASE --- run_experiment: tool: run_skill + model: "" stale_threshold: 2400 idle_output_timeout: 0 with: @@ -379,11 +430,33 @@ steps: experiment_results: "${{ result.results_path }}" retries: 2 on_success: generate_report - on_failure: adjust_experiment + on_failure: troubleshoot_run_failure on_exhausted: ensure_results + troubleshoot_run_failure: + tool: run_skill + model: "" + stale_threshold: 2400 + idle_output_timeout: 0 + with: + skill_command: "/autoskillit:troubleshoot-experiment ${{ context.worktree_path }} run_experiment" + cwd: "${{ inputs.source_dir }}" + step_name: troubleshoot_run_failure + capture: + run_is_fixable: "${{ result.is_fixable }}" + on_success: route_run_failure + on_failure: escalate_stop + + route_run_failure: + action: route + on_result: + - when: "${{ context.run_is_fixable }} == true" + route: adjust_experiment + - route: escalate_stop + adjust_experiment: tool: run_skill + model: "" stale_threshold: 2400 idle_output_timeout: 0 with: @@ -392,22 +465,37 @@ steps: step_name: adjust capture: experiment_results: "${{ result.results_path }}" - on_success: run_experiment + on_success: check_run_fix_loop + on_failure: ensure_results + on_context_limit: ensure_results + + check_run_fix_loop: + tool: run_python + with: + callable: "autoskillit.smoke_utils.check_loop_iteration" + current_iteration: "${{ context.run_fix_count }}" + max_iterations: "3" + capture: + run_fix_count: "${{ result.next_iteration }}" + on_result: + - when: "${{ result.max_exceeded }} == true" + route: ensure_results + - route: run_experiment on_failure: ensure_results + optional_context_refs: + - run_fix_count + note: > + Iteration guard for run_experiment → troubleshoot → adjust_experiment cycle. + Caps experiment fix loops at 3 iterations. On exhaustion, proceeds to + ensure_results to salvage whatever data exists. ensure_results: - tool: run_cmd + tool: run_python with: - cmd: > - RESULTS="${{ context.experiment_results }}" && - if [ -z "${RESULTS}" ]; then - RESULTS="${{ context.worktree_path }}/{{AUTOSKILLIT_TEMP}}/run-experiment/results-inconclusive.md" && - mkdir -p "$(dirname "${RESULTS}")" && - printf '# Experiment Results\n\n## Status\nINCONCLUSIVE\n\nExperiment did not produce results — retries exhausted or adjustment failed.\n' > "${RESULTS}" - fi && - echo "experiment_results=${RESULTS}" - cwd: "${{ context.worktree_path }}" - step_name: ensure_results + callable: "autoskillit.recipe._cmd_rpc.ensure_results" + experiment_results: "${{ context.experiment_results }}" + worktree_path: "${{ context.worktree_path }}" + temp_subdir: "{{AUTOSKILLIT_TEMP}}" capture: experiment_results: "${{ result.experiment_results }}" on_success: generate_report_inconclusive @@ -419,8 +507,9 @@ steps: generate_report: tool: run_skill + model: "" with: - skill_command: "/autoskillit:generate-report ${{ context.worktree_path }} ${{ context.experiment_results }} --output-mode ${{ inputs.output_mode }} --issue-url ${{ inputs.issue_url }}" + skill_command: "/autoskillit:generate-report ${{ context.worktree_path }} ${{ context.experiment_results }} --output-mode ${{ inputs.output_mode }} --issue-url ${{ inputs.issue_url }} --experiment-type '${{ context.experiment_type }}' --methodology-traditions '${{ context.methodology_tradition }}' --design-review-verdict '${{ context.verdict }}' --disambiguation-rule-applied '${{ context.disambiguation_rule_applied }}' --tier-c-lens '${{ context.tier_c_lens }}' --classification-timestamp '${{ context.classification_timestamp }}'" cwd: "${{ context.worktree_path }}" step_name: generate_report capture: @@ -432,8 +521,9 @@ steps: generate_report_inconclusive: tool: run_skill + model: "" with: - skill_command: "/autoskillit:generate-report ${{ context.worktree_path }} ${{ context.experiment_results }} --inconclusive --output-mode ${{ inputs.output_mode }} --issue-url ${{ inputs.issue_url }}" + skill_command: "/autoskillit:generate-report ${{ context.worktree_path }} ${{ context.experiment_results }} --inconclusive --output-mode ${{ inputs.output_mode }} --issue-url ${{ inputs.issue_url }} --experiment-type '${{ context.experiment_type }}' --methodology-traditions '${{ context.methodology_tradition }}' --design-review-verdict '${{ context.verdict }}' --disambiguation-rule-applied '${{ context.disambiguation_rule_applied }}' --tier-c-lens '${{ context.tier_c_lens }}' --classification-timestamp '${{ context.classification_timestamp }}'" cwd: "${{ context.worktree_path }}" step_name: generate_report capture: @@ -454,6 +544,7 @@ steps: fix_tests: tool: run_skill + model: "" with: skill_command: "/autoskillit:resolve-failures ${{ context.worktree_path }} ${{ context.experiment_plan }} ${{ inputs.base_branch }}" cwd: "${{ inputs.source_dir }}" @@ -483,7 +574,7 @@ steps: push_branch: tool: run_cmd with: - cmd: "cd '${{ context.worktree_path }}' && git push -u origin HEAD" + cmd: "bash scripts/recipe/push_branch.sh '${{ inputs.output_mode }}' '${{ context.worktree_path }}'" cwd: "${{ context.worktree_path }}" step_name: push_branch on_success: prepare_research_pr @@ -492,6 +583,7 @@ steps: # --- PR + REVIEW PHASE --- prepare_research_pr: tool: run_skill + model: "" with: skill_command: "/autoskillit:prepare-research-pr ${{ context.report_path }} ${{ context.experiment_plan }} ${{ context.worktree_path }} ${{ inputs.base_branch }}" cwd: "${{ context.worktree_path }}" @@ -505,6 +597,7 @@ steps: run_experiment_lenses: tool: run_skill + model: "" with: skill_command: "/autoskillit:exp-lens-{slug} {context_path} ${{ context.experiment_plan }}" cwd: "${{ context.worktree_path }}" @@ -526,22 +619,7 @@ steps: stage_bundle: tool: run_cmd with: - cmd: > - RESEARCH_DIR="${{ context.research_dir }}" && - if [ -n "${RESEARCH_DIR}" ] && [ -d "${RESEARCH_DIR}" ]; then - WT_TEMP="${{ context.worktree_path }}/{{AUTOSKILLIT_TEMP}}" && - mkdir -p "${RESEARCH_DIR}/artifacts/phase-groups" && - mkdir -p "${RESEARCH_DIR}/artifacts/phase-plans" && - mkdir -p "${RESEARCH_DIR}/artifacts/images" && - mkdir -p "${RESEARCH_DIR}/artifacts/scripts" && - for f in "${WT_TEMP}"/make-groups/*.md; do [ -f "$f" ] && cp "$f" "${RESEARCH_DIR}/artifacts/phase-groups/"; done && - for f in "${WT_TEMP}"/make-groups/*.yaml; do [ -f "$f" ] && cp "$f" "${RESEARCH_DIR}/artifacts/phase-groups/"; done && - for f in "${WT_TEMP}"/make-plan/*.md; do [ -f "$f" ] && cp "$f" "${RESEARCH_DIR}/artifacts/phase-plans/"; done && - for f in "${WT_TEMP}"/exp-lens-*/*.md; do [ -f "$f" ] && cp "$f" "${RESEARCH_DIR}/artifacts/images/"; done && - echo "stage_bundle=done"; - else - echo "stage_bundle: no research directory found — skipped"; - fi + cmd: "bash scripts/recipe/stage_bundle.sh '${{ context.research_dir }}' '${{ context.worktree_path }}' '{{AUTOSKILLIT_TEMP}}'" cwd: "${{ context.worktree_path }}" step_name: stage_bundle on_success: route_pr_or_local @@ -567,6 +645,7 @@ steps: compose_research_pr: tool: run_skill + model: "" with: skill_command: "/autoskillit:compose-research-pr ${{ context.prep_path }} \"${{ context.all_diagram_paths }}\" ${{ context.worktree_path }} ${{ inputs.base_branch }}" cwd: "${{ context.worktree_path }}" @@ -581,11 +660,12 @@ steps: on_result: - when: "${{ context.pr_url }}" route: review_research_pr - - route: research_complete + - route: patch_token_summary note: Guard — skip review if pr_url is empty (graceful degradation from open_research_pr). review_research_pr: tool: run_skill + model: "" with: skill_command: "/autoskillit:review-research-pr ${{ context.worktree_path }} ${{ inputs.base_branch }} ${{ context.pr_url }}" cwd: "${{ context.worktree_path }}" @@ -609,6 +689,7 @@ steps: audit_claims: tool: run_skill + model: "" with: skill_command: "/autoskillit:audit-claims ${{ context.worktree_path }} ${{ inputs.base_branch }} ${{ context.pr_url }}" cwd: "${{ context.worktree_path }}" @@ -641,6 +722,7 @@ steps: resolve_research_review: tool: run_skill + model: "" with: skill_command: "/autoskillit:resolve-research-review ${{ context.worktree_path }} ${{ inputs.base_branch }}" cwd: "${{ context.worktree_path }}" @@ -669,6 +751,7 @@ steps: resolve_claims_review: tool: run_skill + model: "" with: skill_command: "/autoskillit:resolve-claims-review ${{ context.worktree_path }} ${{ inputs.base_branch }}" cwd: "${{ context.worktree_path }}" @@ -688,21 +771,22 @@ steps: route: re_run_experiment - when: "${{ context.claims_needs_rerun }} == true" route: re_run_experiment - - route: re_push_research + - route: finalize_bundle note: > Merged escalation consumer replacing check_escalations. Triggers a single re_run_experiment if either review resolution or claims resolution flagged a rerun. When only design_flaw escalations exist (or none), routes directly - to push — design_flaw escalations are already posted as inline PR comments + to finalize_bundle — design_flaw escalations are already posted as inline PR comments by the resolve skills. Both context.review_needs_rerun and context.claims_needs_rerun may be absent if their respective resolution steps were never reached (e.g., no changes_requested verdict on that gate); the recipe engine evaluates absent context variables as empty string, so absent == true is always false and routes safely fall through to - re_push_research. + finalize_bundle. re_run_experiment: tool: run_skill + model: "" stale_threshold: 2400 idle_output_timeout: 0 with: @@ -723,8 +807,9 @@ steps: re_generate_report: tool: run_skill + model: "" with: - skill_command: "/autoskillit:generate-report ${{ context.worktree_path }} ${{ context.experiment_results }} --output-mode ${{ inputs.output_mode }} --issue-url ${{ inputs.issue_url }}" + skill_command: "/autoskillit:generate-report ${{ context.worktree_path }} ${{ context.experiment_results }} --output-mode ${{ inputs.output_mode }} --issue-url ${{ inputs.issue_url }} --experiment-type '${{ context.experiment_type }}' --methodology-traditions '${{ context.methodology_tradition }}' --design-review-verdict '${{ context.verdict }}' --disambiguation-rule-applied '${{ context.disambiguation_rule_applied }}' --tier-c-lens '${{ context.tier_c_lens }}' --classification-timestamp '${{ context.classification_timestamp }}'" cwd: "${{ context.worktree_path }}" step_name: re_generate_report capture: @@ -741,22 +826,7 @@ steps: re_stage_bundle: tool: run_cmd with: - cmd: > - RESEARCH_DIR="${{ context.research_dir }}" && - if [ -n "${RESEARCH_DIR}" ] && [ -d "${RESEARCH_DIR}" ]; then - WT_TEMP="${{ context.worktree_path }}/{{AUTOSKILLIT_TEMP}}" && - mkdir -p "${RESEARCH_DIR}/artifacts/phase-groups" && - mkdir -p "${RESEARCH_DIR}/artifacts/phase-plans" && - mkdir -p "${RESEARCH_DIR}/artifacts/images" && - mkdir -p "${RESEARCH_DIR}/artifacts/scripts" && - for f in "${WT_TEMP}"/make-groups/*.md; do [ -f "$f" ] && cp "$f" "${RESEARCH_DIR}/artifacts/phase-groups/"; done && - for f in "${WT_TEMP}"/make-groups/*.yaml; do [ -f "$f" ] && cp "$f" "${RESEARCH_DIR}/artifacts/phase-groups/"; done && - for f in "${WT_TEMP}"/make-plan/*.md; do [ -f "$f" ] && cp "$f" "${RESEARCH_DIR}/artifacts/phase-plans/"; done && - for f in "${WT_TEMP}"/exp-lens-*/*.md; do [ -f "$f" ] && cp "$f" "${RESEARCH_DIR}/artifacts/images/"; done && - echo "re_stage_bundle=done"; - else - echo "re_stage_bundle: no research directory found — skipped"; - fi + cmd: "bash scripts/recipe/stage_bundle.sh '${{ context.research_dir }}' '${{ context.worktree_path }}' '{{AUTOSKILLIT_TEMP}}'" cwd: "${{ context.worktree_path }}" step_name: re_stage_bundle on_success: re_test @@ -783,63 +853,19 @@ steps: cmd: "cd '${{ context.worktree_path }}' && git push" cwd: "${{ context.worktree_path }}" step_name: re_push_research - on_success: finalize_bundle + on_success: finalize_bundle_render on_failure: begin_archival # --- ARCHIVAL PHASE --- finalize_bundle: tool: run_cmd with: - cmd: > - OUTPUT_MODE="${{ inputs.output_mode }}" && - RESEARCH_DIR="${{ context.research_dir }}" && - if [ -n "${RESEARCH_DIR}" ] && [ -d "${RESEARCH_DIR}" ]; then - if [ "${OUTPUT_MODE}" != "local" ]; then - if [ -f "${RESEARCH_DIR}/report.md" ]; then - cp "${RESEARCH_DIR}/report.md" "${RESEARCH_DIR}/README.md" && - rm "${RESEARCH_DIR}/report.md"; - fi; - fi && - REPORT_FILE=$( [ "${OUTPUT_MODE}" = "local" ] && echo "report.md" || echo "README.md" ) && - if [ "${OUTPUT_MODE}" = "local" ]; then - EXCLUDE_PATTERN='^(report\.html|mermaid\.min\.js|images|report\.md|artifacts\.tar\.gz)$'; - else - EXCLUDE_PATTERN='^(README\.md|artifacts\.tar\.gz)$'; - fi && - mapfile -t TAR_ITEMS < <(ls -1 "${RESEARCH_DIR}" | grep -vE "${EXCLUDE_PATTERN}") && - if [ ${#TAR_ITEMS[@]} -gt 0 ]; then - tar czf "${RESEARCH_DIR}/artifacts.tar.gz" -C "${RESEARCH_DIR}" "${TAR_ITEMS[@]}" && - for item in "${TAR_ITEMS[@]}"; do rm -rf "${RESEARCH_DIR}/${item}"; done; - fi && - if [ -f "${RESEARCH_DIR}/artifacts.tar.gz" ] && [ -f "${RESEARCH_DIR}/${REPORT_FILE}" ]; then - MANIFEST=$(tar tzf "${RESEARCH_DIR}/artifacts.tar.gz" | sort) && - { echo ""; echo "## Archive Manifest"; echo ""; echo "Contents of \`artifacts.tar.gz\`:"; echo ""; echo '```'; echo "${MANIFEST}"; echo '```'; } >> "${RESEARCH_DIR}/${REPORT_FILE}"; - fi && - [ -f "${RESEARCH_DIR}/artifacts.tar.gz" ] || \ - { echo "ERROR: artifacts.tar.gz not created in ${RESEARCH_DIR}"; exit 1; } && - if [ "${OUTPUT_MODE}" = "local" ]; then - LEFTOVER_PATTERN='^(report\.md|report\.html|mermaid\.min\.js|images|artifacts\.tar\.gz)$'; - else - LEFTOVER_PATTERN='^(README\.md|artifacts\.tar\.gz)$'; - fi && - LEFTOVERS=$(ls -1 "${RESEARCH_DIR}" | grep -vE "${LEFTOVER_PATTERN}") && - [ -z "${LEFTOVERS}" ] || \ - { echo "ERROR: uncompressed entries still present in ${RESEARCH_DIR}: ${LEFTOVERS}"; exit 1; } && - [ -f "${RESEARCH_DIR}/${REPORT_FILE}" ] || \ - { echo "ERROR: ${REPORT_FILE} not found in ${RESEARCH_DIR}"; exit 1; } && - if [ "${OUTPUT_MODE}" != "local" ]; then - cd "${{ context.worktree_path }}" && git add "${RESEARCH_DIR}" && { git diff --cached --quiet || git commit -m "Add compressed research artifacts"; }; - fi; - echo "report_path=${RESEARCH_DIR}/${REPORT_FILE}"; - echo "report_path_after_finalize=${RESEARCH_DIR}/${REPORT_FILE}"; - else - echo "finalize_bundle: no research directory found — artifact commit skipped"; - fi + cmd: "bash scripts/recipe/finalize_bundle.sh '${{ inputs.output_mode }}' '${{ context.research_dir }}' '${{ context.worktree_path }}'" cwd: "${{ context.worktree_path }}" step_name: finalize_bundle capture: report_path_after_finalize: "${{ result.report_path_after_finalize }}" - on_success: finalize_bundle_render + on_success: re_push_research on_failure: begin_archival note: > Single compression point. Runs exactly once per recipe run. In pr mode: renames @@ -850,6 +876,7 @@ steps: finalize_bundle_render: tool: run_skill + model: "" with: skill_command: "/autoskillit:bundle-local-report ${{ context.research_dir }} ${{ context.report_path_after_finalize }} ${{ context.all_diagram_paths }} ${{ context.visualization_plan_path }}" cwd: "${{ context.worktree_path }}" @@ -875,26 +902,19 @@ steps: the existing archival phase. export_local_bundle: - tool: run_cmd + tool: run_python with: - cmd: > - LOCAL_ROOT="${{ inputs.source_dir }}/research-bundles" && - mkdir -p "${LOCAL_ROOT}" && - SLUG=$(basename "${{ context.research_dir }}") && - DEST="${LOCAL_ROOT}/${SLUG}" && - rm -rf "${DEST}" && - cp -a "${{ context.research_dir }}/." "${DEST}/" && - echo "local_bundle_path=${DEST}" - cwd: "${{ context.worktree_path }}" - step_name: export_local_bundle + callable: "autoskillit.recipe._cmd_rpc.export_local_bundle" + source_dir: "${{ inputs.source_dir }}" + research_dir: "${{ context.research_dir }}" capture: local_bundle_path: "${{ result.local_bundle_path }}" - on_success: research_complete - on_failure: research_complete + on_success: patch_token_summary + on_failure: patch_token_summary note: > Local mode terminal export. Copies the finalized research_dir (containing report.html, mermaid.min.js, images/, report.md, artifacts.tar.gz) to - {source_dir}/research-bundles/{slug}/. Non-fatal — routes to research_complete + {source_dir}/research-bundles/{slug}/. Non-fatal — routes to patch_token_summary on both outcomes. The captured local_bundle_path is available to the orchestrator via research_complete. @@ -903,11 +923,11 @@ steps: on_result: - when: "${{ context.pr_url }}" route: capture_experiment_branch - - route: research_complete + - route: patch_token_summary note: > Gate step for the archival phase. Routes to artifact extraction when - a research PR exists, otherwise skips directly to completion. All paths - from review/resolve/re-push converge here. + a research PR exists, otherwise skips directly to patch_token_summary. + All paths from review/resolve/re-push converge here. capture_experiment_branch: tool: run_cmd @@ -918,35 +938,18 @@ steps: capture: experiment_branch: "${{ result.stdout | trim }}" on_success: create_artifact_branch - on_failure: research_complete + on_failure: patch_token_summary create_artifact_branch: tool: run_cmd with: - cmd: > - EXPERIMENT_BRANCH="${{ context.experiment_branch }}" && - SLUG=$(printf '%s' "${EXPERIMENT_BRANCH}" | sed 's/^research-//') && - ARTIFACT_BRANCH="research-artifacts-${SLUG}" && - ARTIFACT_DIR="$(mktemp -d)" && - trap 'git -C "${{ context.worktree_path }}" worktree remove "${ARTIFACT_DIR}" --force 2>/dev/null; rm -rf "${ARTIFACT_DIR}"' EXIT && - git -C '${{ context.worktree_path }}' fetch origin '${{ inputs.base_branch }}' && - (git -C '${{ context.worktree_path }}' branch -D "${ARTIFACT_BRANCH}" 2>/dev/null || true) && - git -C '${{ context.worktree_path }}' worktree add "${ARTIFACT_DIR}" -b "${ARTIFACT_BRANCH}" "origin/${{ inputs.base_branch }}" && - RESEARCH_SUBDIR="research/$(basename "${{ context.research_dir }}")" && - git -C "${ARTIFACT_DIR}" checkout "${EXPERIMENT_BRANCH}" -- "${RESEARCH_SUBDIR}" && - cd "${ARTIFACT_DIR}" && - git add research/ && - git commit -m "Add research artifacts: ${SLUG}" && - git push -u origin "${ARTIFACT_BRANCH}" && - cd - > /dev/null && - git -C '${{ context.worktree_path }}' worktree remove "${ARTIFACT_DIR}" --force && - echo "artifact_branch=${ARTIFACT_BRANCH}" + cmd: "bash scripts/recipe/create_artifact_branch.sh '${{ context.worktree_path }}' '${{ context.experiment_branch }}' '${{ inputs.base_branch }}' '${{ context.research_dir }}'" cwd: "${{ context.worktree_path }}" step_name: create_artifact_branch capture: artifact_branch: "${{ result.artifact_branch }}" on_success: open_artifact_pr - on_failure: research_complete + on_failure: patch_token_summary note: > Creates a temporary worktree from the base branch, checks out only the research/ directory from the experiment branch, commits and pushes, then @@ -956,30 +959,13 @@ steps: open_artifact_pr: tool: run_cmd with: - cmd: > - EXPERIMENT="${{ context.experiment_branch }}" && - ARTIFACT="${{ context.artifact_branch }}" && - ORIG_PR="${{ context.pr_url }}" && - RESEARCH_SUBDIR="research/$(basename "${{ context.research_dir }}")" && - ARTIFACT_DIR="${{ context.worktree_path }}" && - if [ -f "${ARTIFACT_DIR}/${RESEARCH_SUBDIR}/artifacts.tar.gz" ]; then - STRUCT_DETAIL=$'- `artifacts.tar.gz` \xe2\x80\x94 compressed artifact tree\n\nExtract:\n```bash\ntar xzf artifacts.tar.gz\n```'; - else - STRUCT_DETAIL="- \`artifacts/\` — uncompressed artifact tree"; - fi && - PR_BODY=$(printf '## Research Artifacts\n\nFrom experiment branch `%s`.\n\nOriginal experiment PR: %s\n\n### Structure\n\n- **README.md** — Research conclusions (browsable on GitHub)\n%s\n\nContains only files under `research/` — zero production code changes.' "${EXPERIMENT}" "${ORIG_PR}" "${STRUCT_DETAIL}") && - PR_URL=$(gh pr create - --head "${ARTIFACT}" - --base '${{ inputs.base_branch }}' - --title "Research artifacts: ${EXPERIMENT}" - --body "${PR_BODY}") && - echo "artifact_pr_url=${PR_URL}" + cmd: "bash scripts/recipe/open_artifact_pr.sh '${{ context.worktree_path }}' '${{ context.experiment_branch }}' '${{ context.artifact_branch }}' '${{ context.pr_url }}' '${{ inputs.base_branch }}' '${{ context.research_dir }}'" cwd: "${{ context.worktree_path }}" step_name: open_artifact_pr capture: artifact_pr_url: "${{ result.artifact_pr_url }}" on_success: tag_experiment_branch - on_failure: research_complete + on_failure: patch_token_summary note: > Opens a PR containing only the research/ directory. The PR body documents the two-file structure (README.md + artifacts.tar.gz) and provides the @@ -988,19 +974,13 @@ steps: tag_experiment_branch: tool: run_cmd with: - cmd: > - TAG="archive/research/${{ context.experiment_branch }}" && - ARTIFACT_PR="${{ context.artifact_pr_url }}" && - git -C '${{ context.worktree_path }}' tag -a "${TAG}" '${{ context.experiment_branch }}' - -m "Research experiment: ${{ context.experiment_branch }}. Report merged via artifact PR ${ARTIFACT_PR}. Variant code preserved here for reference." && - git -C '${{ context.worktree_path }}' push origin "${TAG}" && - echo "archive_tag=${TAG}" + cmd: "bash scripts/recipe/tag_experiment_branch.sh '${{ context.worktree_path }}' '${{ context.experiment_branch }}' '${{ context.artifact_pr_url }}'" cwd: "${{ context.worktree_path }}" step_name: tag_experiment_branch capture: archive_tag: "${{ result.archive_tag }}" on_success: close_experiment_pr - on_failure: research_complete + on_failure: patch_token_summary note: > Creates an annotated tag preserving the full experiment branch (including all variant code, benchmark scaffolding, etc.) under archive/research/. @@ -1016,13 +996,26 @@ steps: gh pr close '${{ context.pr_url }}' --comment "${CLOSE_BODY}" cwd: "${{ context.worktree_path }}" step_name: close_experiment_pr - on_success: research_complete - on_failure: research_complete + on_success: patch_token_summary + on_failure: patch_token_summary note: > Closes the original experiment PR with a structured comment linking to the artifact PR (where research/ was merged) and the archive tag (where the full experiment branch is preserved). Both success and failure route - to research_complete — closing the PR is best-effort. + to patch_token_summary — closing the PR is best-effort. + + patch_token_summary: + description: "Collect all pipeline token data and patch the PR body with complete summary" + tool: run_python + with: + callable: "autoskillit.smoke_utils.patch_pr_token_summary" + timeout: 60 + pr_url: "${{ context.pr_url }}" + cwd: "${{ context.worktree_path }}" + on_success: research_complete + on_failure: research_complete + note: > + Patches PR body with complete token summary; failure never blocks pipeline completion. research_complete: action: stop diff --git a/src/autoskillit/recipes/sub-recipes/diagrams/sprint-prefix.md b/src/autoskillit/recipes/sub-recipes/diagrams/sprint-prefix.md deleted file mode 100644 index 392742873c..0000000000 --- a/src/autoskillit/recipes/sub-recipes/diagrams/sprint-prefix.md +++ /dev/null @@ -1,26 +0,0 @@ -## sprint-prefix - -``` - triage - | - plan (issue-splitter / collapse-issues / enrich-issues as needed) - | - confirm ←— user approval —→ [abort → done] - | - dispatch (process-issues: per-issue implementation / remediation) - | | - | [failure] - | | - report (sprint summary) -``` - -### Inputs - -| Ingredient | Default | Description | -|---|---|---| -| sprint_size | 4 | Max issues in sprint | -| enrich | on | Enrich issues before planning | -| dry_run | off | Triage/selection only, skip dispatch | -| source_dir | auto-detect | Source repository | -| base_branch | auto-detect | Merge target branch | -| run_name | sprint | Run name prefix | diff --git a/src/autoskillit/recipes/sub-recipes/research.json b/src/autoskillit/recipes/sub-recipes/research.json new file mode 100644 index 0000000000..330afcb41a --- /dev/null +++ b/src/autoskillit/recipes/sub-recipes/research.json @@ -0,0 +1,17 @@ +{ + "name": "research", + "description": "Research sub-recipe: gather context, analyze findings, and produce a structured research report for the parent recipe.\n", + "summary": "gather → analyze → report", + "ingredients": { + "source_dir": { + "description": "Source repository (inherited from parent recipe context)", + "required": false, + "default": "" + }, + "base_branch": { + "description": "Merge target branch (inherited from parent recipe context)", + "default": "" + } + }, + "steps": {} +} diff --git a/src/autoskillit/recipes/sub-recipes/research.yaml b/src/autoskillit/recipes/sub-recipes/research.yaml new file mode 100644 index 0000000000..f7fdf16038 --- /dev/null +++ b/src/autoskillit/recipes/sub-recipes/research.yaml @@ -0,0 +1,16 @@ +name: research +description: > + Research sub-recipe: gather context, analyze findings, and produce a + structured research report for the parent recipe. +summary: gather → analyze → report + +ingredients: + source_dir: + description: Source repository (inherited from parent recipe context) + required: false + default: "" + base_branch: + description: Merge target branch (inherited from parent recipe context) + default: "" + +steps: {} diff --git a/src/autoskillit/recipes/sub-recipes/sprint-prefix.yaml b/src/autoskillit/recipes/sub-recipes/sprint-prefix.yaml deleted file mode 100644 index d01a8c9a57..0000000000 --- a/src/autoskillit/recipes/sub-recipes/sprint-prefix.yaml +++ /dev/null @@ -1,140 +0,0 @@ -name: sprint-prefix -description: > - Sprint workflow prefix: triage open issues, plan the sprint for maximum - parallelism, get user approval, then dispatch each approved issue to the - parent recipe for implementation or remediation. -summary: triage → sprint-plan → confirm → dispatch → report -autoskillit_version: "0.3.0" - -ingredients: - source_dir: - description: Source repository (inherited from parent recipe context) - required: false - default: "" - base_branch: - description: Merge target branch (inherited from parent recipe context) - default: "" - run_name: - description: Run name prefix (inherited from parent recipe context) - default: "sprint" - sprint_size: - description: Maximum number of issues to include in the sprint - default: "4" - enrich: - description: Enrich issues with structured requirements before planning - default: "true" - dry_run: - description: Triage and selection only — skip dispatch and implementation - default: "false" - -kitchen_rules: - - "NEVER use native Claude Code tools (Read, Grep, Glob, Edit, Write, Bash, - Agent, WebFetch, WebSearch, NotebookEdit) from the orchestrator. All work - is delegated through run_skill or MCP tools." - - "Route to on_failure — never investigate or fix failures directly from the - orchestrator. The downstream skill handles diagnosis." - - "After the confirm step: if the user approves, route to dispatch. - If the user wants to adjust the sprint (add/remove issues), re-run - plan with the adjusted context before re-presenting confirm. - If the user aborts, route to done." - - "The dispatch step calls process-issues with the sprint_manifest. It handles - per-issue implementation/remediation orchestration internally." - - "After dispatch completes (success or failure), always route to report." - -steps: - triage: - tool: run_skill - with: - skill_command: /autoskillit:triage-issues - cwd: ${{ inputs.source_dir }} - step_name: triage - capture: - triage_manifest: "${{ result.triage_manifest }}" - on_success: plan - on_failure: escalate - note: > - Analyzes open GitHub issues, classifies them, and partitions them into - conflict-free parallel groups. The triage_manifest captures the output - for use in sprint planning. - - plan: - tool: run_skill - with: - skill_command: /autoskillit:sprint-planner - cwd: ${{ inputs.source_dir }} - step_name: plan - triage_manifest: ${{ context.triage_manifest }} - sprint_size: ${{ inputs.sprint_size }} - enrich: ${{ inputs.enrich }} - capture: - sprint_manifest: "${{ result.sprint_manifest }}" - on_success: confirm - on_failure: escalate - note: > - Sprint planner skill: reads triage_manifest, analyzes issue overlap, - optionally calls issue-splitter/collapse-issues/enrich-issues MCP tools, - and produces a sprint_manifest with the planned sprint composition. - - confirm: - action: confirm - message: > - Sprint Plan Ready. Review the sprint below and confirm to proceed, - or abort to cancel. - - Sprint manifest: ${{ context.sprint_manifest }} - - To proceed: approve this sprint. - To adjust: describe which issues to add, remove, or swap. - To abort: say "abort". - on_success: dispatch - on_failure: done - note: > - Presents the sprint to the user. on_success routes to dispatch when - the user approves. on_failure routes to done when the user aborts. - For sprint adjustments, the orchestrator should re-run the plan step - with updated context before returning to confirm. - - dispatch: - tool: run_skill - with: - skill_command: /autoskillit:process-issues - cwd: ${{ inputs.source_dir }} - step_name: dispatch - sprint_manifest: ${{ context.sprint_manifest }} - base_branch: ${{ inputs.base_branch }} - run_name: ${{ inputs.run_name }} - dry_run: ${{ inputs.dry_run }} - capture: - dispatch_results: "${{ result.dispatch_results }}" - on_success: report - on_failure: report - note: > - Dispatches each approved sprint issue to the parent recipe via process-issues. - Routes to report regardless of success/failure so the sprint summary is - always produced. - - report: - action: stop - message: > - Sprint complete. - - Results summary: ${{ context.dispatch_results }} - - Sprint manifest: ${{ context.sprint_manifest }} - - Review individual PRs before merging. Run the merge-prs pipeline when - ready to integrate. - with: - sprint_manifest: "${{ context.sprint_manifest }}" - dispatch_results: "${{ context.dispatch_results }}" - - done: - action: stop - message: > - Sprint aborted by user. No issues were dispatched. - - escalate: - action: stop - message: > - Sprint failed. A step returned failure before the sprint could complete. - Check the logs for the failing step and retry after resolving the issue. diff --git a/src/autoskillit/server/CLAUDE.md b/src/autoskillit/server/CLAUDE.md new file mode 100644 index 0000000000..33b017e14d --- /dev/null +++ b/src/autoskillit/server/CLAUDE.md @@ -0,0 +1,97 @@ +# server/ + +IL-3 FastMCP server — MCP tools, kitchen gating, session-type dispatch. +Sub-package: tools/ (see tools/CLAUDE.md). + +## Files + +| File | Purpose | +|------|---------| +| `__init__.py` | Re-exports `mcp`, `ToolContext`, `make_context`; applies `mcp.disable(tags={'kitchen'})` at import | +| `_editable_guard.py` | Pre-deletion editable install guard for `perform_merge()` — scans site-packages for PEP 610 direct_url.json links into the worktree | +| `_factory.py` | Composition root — `make_context()` is the sole legal instantiation point for all 22 service contracts | +| `_guards.py` | Orchestration-level gate functions for MCP tool access control | +| `_lifespan.py` | FastMCP lifespan context manager — deferred startup (recovery, audit loading, stale cleanup, drift check) | +| `_misc.py` | Quota, hook-config, triage, and miscellaneous server utilities; re-exports selected execution/workspace symbols for tools | +| `_notify.py` | MCP notification dispatch and response-size tracking | +| `_session_type.py` | Session-type tag visibility dispatcher — controls which tools are visible per session type | +| `_state.py` | Mutable singleton state and context accessor functions (`_ctx` sentinel, `get_ctx`, `set_ctx`) | +| `_subprocess.py` | Subprocess execution helpers for MCP tools | +| `_wire_compat.py` | Wire-format compatibility middleware — strips `outputSchema`/`title` fields to work around Claude Code bug #25081 | +| `git.py` | Git merge workflow for `merge_worktree` — path validation, branch detection, test gate, fetch, rebase, merge, cleanup | + +## readOnlyHint: All MCP tools MUST have `readOnlyHint: True` + +Every pipeline operates on independent branches and worktrees with zero cross-pipeline +interference. `readOnlyHint: False` serializes parallel tool calls and causes catastrophic +pipeline slowdowns (40+ minutes instead of 5 minutes for concurrent CI watches). + +This has regressed three times. Defense-in-depth: +- Pre-commit: `scripts/check_tool_annotations.py` (AST scan, blocks commit) +- Tests: `test_all_tools_have_readonly_hint_true` (universal assertion, no registry) +- Tests: `test_all_annotations_are_readonly_true` (AST-level, no server import) + +If you believe a tool genuinely needs `readOnlyHint: False`, you are wrong. All pipelines +use independent branches. There is no shared mutable state between concurrent tool calls. + +## Tool Gating Architecture + +Tools are controlled by two independent mechanisms. A tool may be affected by one, both, or neither. + +### Tag-Visibility (FastMCP layer) + +Controls whether the tool appears in `tools/list` (whether the agent can see it): + +- `mcp.disable(tags={"kitchen"})` at startup hides all `kitchen`-tagged tools +- `_apply_session_type_visibility()` selectively reveals tags per session type: + - **FLEET** sessions: `fleet`-tagged tools revealed; `fleet-dispatch` revealed only in dispatch mode + - **ORCHESTRATOR + HEADLESS**: `kitchen` (or `kitchen-core` + pack tags) revealed + - **SKILL + HEADLESS**: `headless`-tagged tools revealed (`test_check`) + - **Interactive** (no HEADLESS): nothing pre-revealed; `open_kitchen` reveals `kitchen` tag +- Tags not disabled at startup (`kitchen-core`, `fleet-dispatch`, `fleet`, `headless`) remain + visible unless a session-type or feature-gate transform explicitly disables them +- `ALL_VISIBILITY_TAGS` in `core/types/_type_constants.py` is the canonical set: + `{"kitchen", "headless", "fleet", "fleet-dispatch", "kitchen-core"}` + +### Application-Gate (Python layer) + +Controls whether the tool succeeds when called (independent of visibility): + +- Most kitchen tools call `_require_enabled()` as their first statement, which checks `ctx.gate.enabled` +- Returns a `gate_error` JSON envelope if the kitchen hasn't been opened +- `_require_enabled()` is defined in `server/_guards.py`; the error envelope is defined in `pipeline/gate.py` +- Enforcement is validated by `test_gated_tools_call_require_enabled_first` in `tests/arch/test_layer_enforcement.py` + +### The Anomalies + +1. **Fleet-dispatch tools are tag-visible but application-gated.** `fetch_github_issue`, `get_issue_title`, + `list_recipes`, and `load_recipe` carry the `fleet-dispatch` tag (not `kitchen`), so they are + NOT hidden by `mcp.disable(tags={"kitchen"})`. In interactive sessions they appear in `tools/list` + without `open_kitchen`. But they call `_require_enabled()` internally — an agent that sees them + and calls them gets an unexpected gate error. + +2. **`test_check` is tag-hidden but NOT application-gated.** It carries the `kitchen`, `kitchen-core`, + `headless`, and `autoskillit` tags (hidden at startup), but does NOT call `_require_enabled()`. Headless skill sessions need + `test_check` without opening the kitchen — the `headless` tag provides visibility in SKILL sessions, + and skipping `_require_enabled()` lets the call succeed without a gate open. + +### Tool Gating Matrix + +| Category | Tag(s) | Hidden at startup? | Application-gated? | Example tools | +|----------|--------|-------------------|--------------------|--------------| +| Standard kitchen | `kitchen` | Yes | Yes (`_require_enabled`) | `run_cmd`, `run_skill`, `report_bug` | +| Fleet tool | `fleet`, `kitchen-core` | No (no `kitchen` tag) | Yes (`_require_fleet` or `_require_enabled`) | `dispatch_food_truck`, `record_gate_dispatch` | +| Fleet-dispatch tool | `fleet-dispatch` (± `kitchen-core`) | No (no `kitchen` tag) | Yes (`_require_enabled`) | `fetch_github_issue`, `list_recipes` | +| Headless-exempt | `kitchen`, `headless` | Yes | No | `test_check` | +| Free-range | _(none of the above)_ | No | No | `open_kitchen`, `close_kitchen` | + +### Registry Constants + +The canonical tool sets are in `core/types/_type_constants.py`: + +- `GATED_TOOLS` — all tools that call `_require_enabled()` (validated by arch test) +- `UNGATED_TOOLS` = `FREE_RANGE_TOOLS` — tools with no gating at all +- `HEADLESS_TOOLS` — `{"test_check"}` — kitchen-tagged but not application-gated +- `FLEET_TOOLS` — fleet-session-only tools +- `FLEET_DISPATCH_TOOLS` — fleet-dispatch-mode tools (always tag-visible, application-gated) +- `ALL_VISIBILITY_TAGS` — `{"kitchen", "headless", "fleet", "fleet-dispatch", "kitchen-core"}` diff --git a/src/autoskillit/server/__init__.py b/src/autoskillit/server/__init__.py index e75955a9d6..86f9fdaa0f 100644 --- a/src/autoskillit/server/__init__.py +++ b/src/autoskillit/server/__init__.py @@ -1,13 +1,22 @@ """MCP server for orchestrating automated skill-driven workflows. -Kitchen tools (40 gated + 1 headless-tagged) are hidden at startup via FastMCP v3 -mcp.disable(tags={'kitchen'}) applied once after all tool modules are imported. -Each new session sees only the 3 free-range tools (open_kitchen, close_kitchen, -and disable_quota_guard). -Headless sessions (AUTOSKILLIT_HEADLESS=1) pre-reveal only headless-tagged tools -(test_check) via mcp.enable(tags={'headless'}) — not all kitchen tools. -Calling the open_kitchen tool reveals all 41 kitchen-tagged tools for that session -via ctx.enable_components(tags={'kitchen'}). +Tool gating uses two independent layers — see server/CLAUDE.md "Tool Gating +Architecture" section for the full matrix. + +Tag-Visibility (FastMCP): kitchen-tagged tools are hidden at startup via +mcp.disable(tags={'kitchen'}). Session-type dispatch (_apply_session_type_visibility) +selectively reveals tags per session type. open_kitchen reveals all kitchen-tagged +tools for interactive sessions. + +Application-Gate (Python): most tools call _require_enabled() internally, which +checks ctx.gate.enabled and returns a gate_error envelope if the kitchen is closed. +See GATED_TOOLS and UNGATED_TOOLS in core/types/_type_constants.py. + +Startup tag visibility is determined by AUTOSKILLIT_SESSION_TYPE (3-branch dispatch): + FLEET — fleet-tagged tools pre-revealed + ORCHESTRATOR + HEADLESS=1 — all kitchen-tagged tools pre-revealed + SKILL + HEADLESS=1 — headless-tagged tools (test_check) pre-revealed + ORCHESTRATOR/SKILL (interactive) — no pre-reveal; open_kitchen unlocks Transport: stdio (default for FastMCP). """ @@ -48,30 +57,66 @@ "make_context", # Wire-format compatibility middleware "ClaudeCodeCompatMiddleware", + # Session-type visibility dispatcher (callable by tests) + "_apply_session_type_visibility", ] # Import all tool sub-modules to trigger @mcp.tool() registration. # These imports must come AFTER mcp, _get_ctx, _get_config are defined # because tool modules import `mcp` from this package at import time. -import os # noqa: E402 - from autoskillit.core import PIPELINE_FORBIDDEN_TOOLS # noqa: E402, F401 from autoskillit.server import ( # noqa: E402, F401 - helpers, - tools_ci, - tools_clone, - tools_execution, - tools_git, - tools_github, - tools_issue_lifecycle, - tools_kitchen, - tools_pr_ops, - tools_recipe, - tools_status, - tools_workspace, + _misc, + _notify, ) from autoskillit.server._factory import make_context # noqa: E402, F401 -from autoskillit.server.tools_kitchen import _build_tool_category_listing # noqa: E402, F401 +from autoskillit.server._session_type import _apply_session_type_visibility # noqa: E402, F401 +from autoskillit.server.tools import ( # noqa: E402, F401 + tools_ci as _tools_ci, +) +from autoskillit.server.tools import ( # noqa: E402, F401 + tools_ci_merge_queue as _tools_ci_merge_queue, +) +from autoskillit.server.tools import ( # noqa: E402, F401 + tools_ci_watch as _tools_ci_watch, +) +from autoskillit.server.tools import ( # noqa: E402, F401 + tools_clone as _tools_clone, +) +from autoskillit.server.tools import ( # noqa: E402, F401 + tools_execution as _tools_execution, +) +from autoskillit.server.tools import ( # noqa: E402, F401 + tools_fleet_dispatch as _tools_fleet_dispatch, +) +from autoskillit.server.tools import ( # noqa: E402, F401 + tools_git as _tools_git, +) +from autoskillit.server.tools import ( # noqa: E402, F401 + tools_github as _tools_github, +) +from autoskillit.server.tools import ( # noqa: E402, F401 + tools_issue_composite as _tools_issue_composite, +) +from autoskillit.server.tools import ( # noqa: E402, F401 + tools_issue_lifecycle as _tools_issue_lifecycle, +) +from autoskillit.server.tools import ( # noqa: E402, F401 + tools_kitchen as _tools_kitchen, +) +from autoskillit.server.tools import ( # noqa: E402, F401 + tools_pr_ops as _tools_pr_ops, +) +from autoskillit.server.tools import ( # noqa: E402, F401 + tools_recipe as _tools_recipe, +) +from autoskillit.server.tools import ( # noqa: E402, F401 + tools_status as _tools_status, +) +from autoskillit.server.tools import ( # noqa: E402, F401 + tools_workspace as _tools_workspace, +) +from autoskillit.server.tools.tools_kitchen import _build_tool_category_listing # noqa: E402, F401 # Apply global visibility transform: all sessions start with kitchen tools hidden. # Must appear after all tool module imports so the registered tools are in place. @@ -83,7 +128,4 @@ mcp.add_middleware(ClaudeCodeCompatMiddleware()) -# Headless sessions (AUTOSKILLIT_HEADLESS=1) pre-reveal only headless-tagged tools -# (test_check) so the session starts with test_check visible without calling open_kitchen. -if os.environ.get("AUTOSKILLIT_HEADLESS") == "1": - mcp.enable(tags={"headless"}) +_apply_session_type_visibility() diff --git a/src/autoskillit/server/_factory.py b/src/autoskillit/server/_factory.py index e2399f6200..f0baa42315 100644 --- a/src/autoskillit/server/_factory.py +++ b/src/autoskillit/server/_factory.py @@ -1,14 +1,15 @@ """Composition Root: make_context() is the only location that legally instantiates all 22 service contracts simultaneously. -server/ is L3 — the only layer permitted to import from both L1 (pipeline/) -and L2 (recipe/, migration/) at the same time. This module is the canonical +server/ is IL-3 — the only layer permitted to import from both IL-1 (pipeline/) +and IL-2 (recipe/, migration/) at the same time. This module is the canonical factory for wiring a fully-populated ToolContext, replacing the ad-hoc construction scattered across callers. """ from __future__ import annotations +import dataclasses import os import subprocess from collections.abc import Callable @@ -17,8 +18,14 @@ from autoskillit.config import AutomationConfig from autoskillit.core import ( + MARKETPLACE_PREFIX, + DirectInstall, + FleetLock, + MarketplaceInstall, + PluginSource, SubprocessRunner, WriteBehaviorSpec, + detect_autoskillit_mcp_prefix, get_logger, pkg_root, resolve_temp_dir, @@ -43,6 +50,7 @@ DefaultAuditLog, DefaultBackgroundSupervisor, DefaultGateState, + DefaultGitHubApiLog, DefaultTimingLog, DefaultTokenLog, ToolContext, @@ -63,11 +71,11 @@ logger = get_logger(__name__) -# Sentinel: distinguish "caller passed runner=None explicitly" from "not provided" +# Optional-override: _UNSET means "not provided, use factory-computed default" (None is valid). _UNSET: Any = object() -class TokenFactory: +class _LazyTokenFactory: """Lazy-resolving, caching token factory. Wraps the config -> env -> gh CLI token resolution chain. Does NOT @@ -94,9 +102,16 @@ def is_resolved(self) -> bool: return self._resolved is not self._UNRESOLVED -def _default_plugin_dir() -> str: +def _default_plugin_dir() -> Path: """Resolve the autoskillit package root.""" - return str(pkg_root()) + return pkg_root() + + +def _resolve_marketplace_cache_path() -> Path: + """Read the installPath for autoskillit from installed_plugins.json.""" + from autoskillit.core import _get_autoskillit_install_path + + return _get_autoskillit_install_path() def _gh_cli_token() -> str | None: @@ -121,10 +136,27 @@ def _gh_cli_token() -> str | None: def _check_plugin_installed() -> bool: - """Deferred import to avoid server→cli module-level dependency.""" - from autoskillit.cli import _is_plugin_installed # noqa: PLC0415 + """Detect if autoskillit is marketplace-installed via installed_plugins.json.""" + return detect_autoskillit_mcp_prefix() == MARKETPLACE_PREFIX - return _is_plugin_installed() + +def _resolve_project_dir() -> Path: + """Resolve the project root: AUTOSKILLIT_PROJECT_DIR env -> git toplevel -> cwd.""" + env_project_dir = os.environ.get("AUTOSKILLIT_PROJECT_DIR", "") + if env_project_dir: + return Path(env_project_dir) + try: + result = subprocess.run( + ["git", "rev-parse", "--show-toplevel"], + capture_output=True, + text=True, + timeout=5, + ) + if result.returncode == 0 and result.stdout.strip(): + return Path(result.stdout.strip()) + except (OSError, subprocess.SubprocessError): + pass + return Path.cwd() def make_context( @@ -132,6 +164,8 @@ def make_context( *, runner: SubprocessRunner | None = _UNSET, plugin_dir: str | None = _UNSET, + plugin_source: PluginSource = _UNSET, + fleet_lock: FleetLock | None = None, ) -> ToolContext: """Create a fully-wired ToolContext with all 22 service fields populated. @@ -147,10 +181,16 @@ def make_context( runner: Subprocess runner implementation. Defaults to DefaultSubprocessRunner() for production use. Pass runner=None explicitly to disable the tester (useful in tests that don't need real subprocess execution). - plugin_dir: Absolute path to the autoskillit plugin directory. - Pass None explicitly to indicate the plugin is installed - (marketplace install; no --plugin-dir needed). When omitted - (sentinel), auto-detects via _check_plugin_installed(). + plugin_dir: Absolute path to the autoskillit plugin directory for a + direct install. When omitted (sentinel), auto-detects via + _check_plugin_installed(). When plugin_source is also provided, + plugin_source takes precedence. + plugin_source: PluginSource override. When supplied, used directly + without detection. For tests and CLI that construct + install mode explicitly. + fleet_lock: FleetLock implementation to inject. Defaults to + FleetSemaphore(max_concurrent_dispatches) when None. Pass a + custom implementation in tests to substitute without monkey-patching. Returns: ToolContext with gate starting closed (enabled=False) in all contexts. @@ -159,6 +199,16 @@ def make_context( All service fields are populated. When runner=None is passed explicitly, tester is left as None. """ + env_profile = os.environ.get("AUTOSKILLIT_PROVIDER_PROFILE") + if env_profile and env_profile in config.providers.profiles: + config = dataclasses.replace( + config, providers=dataclasses.replace(config.providers, default_provider=env_profile) + ) + elif env_profile: + logger.warning( + "AUTOSKILLIT_PROVIDER_PROFILE %r not in known profiles — ignored", env_profile + ) + if runner is _UNSET: from autoskillit.execution import DefaultSubprocessRunner @@ -203,32 +253,59 @@ def make_context( recorder = make_scenario_recorder( output_dir=scenario_dir, recipe_name=recipe_name ) - runner = RecordingSubprocessRunner(recorder=recorder, inner=runner) + runner = RecordingSubprocessRunner( + recorder=recorder, + inner=runner, + scenario_dir=Path(scenario_dir), + ) # Lazy token resolution: config → GITHUB_TOKEN env var → gh CLI → None. # The _gh_cli_token() subprocess (up to 5s) is deferred until the first # gated tool actually needs a GitHub token, keeping the MCP server startup # path free of subprocess calls (REQ-STARTUP-001). - token_factory = TokenFactory( + token_factory = _LazyTokenFactory( lambda: config.github.token or os.environ.get("GITHUB_TOKEN") or _gh_cli_token() ) - resolved_dir = ( - plugin_dir - if plugin_dir is not _UNSET - else (_default_plugin_dir() if not _check_plugin_installed() else None) - ) + resolved_plugin_source: PluginSource + if plugin_source is not _UNSET: + resolved_plugin_source = plugin_source # type: ignore[assignment] + elif plugin_dir is not _UNSET and isinstance(plugin_dir, (str, Path)): + resolved_plugin_source = DirectInstall(plugin_dir=Path(plugin_dir)) + elif _check_plugin_installed(): + try: + resolved_plugin_source = MarketplaceInstall( + cache_path=_resolve_marketplace_cache_path() + ) + except (KeyError, ValueError) as exc: + logger.warning( + "marketplace install path unavailable (%s) — falling back to direct install", + exc, + ) + resolved_plugin_source = DirectInstall(plugin_dir=_default_plugin_dir()) + else: + resolved_plugin_source = DirectInstall(plugin_dir=_default_plugin_dir()) + plugin_source = resolved_plugin_source gate = DefaultGateState(enabled=False) - project_dir = Path.cwd() + project_dir = _resolve_project_dir() temp_dir = resolve_temp_dir(project_dir, config.workspace.temp_dir) temp_dir_relpath = temp_dir_display_str(config.workspace.temp_dir) - provider = SkillsDirectoryProvider(temp_dir_relpath=temp_dir_relpath) + provider = SkillsDirectoryProvider( + temp_dir_relpath=temp_dir_relpath, + default_base_branch=config.branching.default_base_branch, + ) ephemeral_root = resolve_ephemeral_root() session_mgr = DefaultSessionSkillManager(provider, ephemeral_root) + from autoskillit.fleet import ( # lazy: avoids fleet init on server import + FleetSemaphore, + build_protected_campaign_ids, + ) + audit = DefaultAuditLog() + github_api_log = DefaultGitHubApiLog() ctx = ToolContext( config=config, audit=audit, @@ -236,20 +313,28 @@ def make_context( token_log=DefaultTokenLog(), timing_log=DefaultTimingLog(), gate=gate, - plugin_dir=resolved_dir, + plugin_source=plugin_source, runner=runner, temp_dir=temp_dir, + project_dir=project_dir, tester=DefaultTestRunner(config=config, runner=runner) if runner is not None else None, recipes=DefaultRecipeRepository(), db_reader=DefaultDatabaseReader(), workspace_mgr=DefaultWorkspaceManager(), clone_mgr=DefaultCloneManager(), - github_client=DefaultGitHubFetcher(token=token_factory), - ci_watcher=DefaultCIWatcher(token=token_factory), - merge_queue_watcher=DefaultMergeQueueWatcher(token=token_factory), + github_client=DefaultGitHubFetcher(token=token_factory, tracker=github_api_log), + ci_watcher=DefaultCIWatcher(token=token_factory, tracker=github_api_log), + merge_queue_watcher=DefaultMergeQueueWatcher(token=token_factory, tracker=github_api_log), + github_api_log=github_api_log, session_skill_manager=session_mgr, skill_resolver=provider.resolver, + ephemeral_root=ephemeral_root, quota_refresh_task=None, + fleet_lock=( + fleet_lock + if fleet_lock is not None + else FleetSemaphore(max_concurrent=config.fleet.max_concurrent_dispatches) + ), ) def _resolve_output_patterns(skill_command: str) -> list[str]: @@ -273,9 +358,18 @@ def _resolve_write_behavior(skill_command: str) -> WriteBehaviorSpec: expected_when=tuple(contract.write_expected_when), ) + def _resolve_read_only(skill_command: str) -> bool: + name = resolve_skill_name(skill_command) + if not name: + return False + contract = get_skill_contract(name, load_bundled_manifest()) + return contract.read_only if contract else False + ctx.output_pattern_resolver = _resolve_output_patterns ctx.write_expected_resolver = _resolve_write_behavior + ctx.read_only_resolver = _resolve_read_only ctx.token_factory = token_factory + ctx.build_protected_campaign_ids = build_protected_campaign_ids ctx.executor = DefaultHeadlessExecutor(ctx) ctx.migrations = DefaultMigrationService( default_migration_engine(), run_headless=ctx.executor.run diff --git a/src/autoskillit/server/_guards.py b/src/autoskillit/server/_guards.py new file mode 100644 index 0000000000..8dc8f96d39 --- /dev/null +++ b/src/autoskillit/server/_guards.py @@ -0,0 +1,246 @@ +"""Orchestration-level gate functions for MCP tool access control.""" + +from __future__ import annotations + +import os +from pathlib import Path +from typing import TYPE_CHECKING + +from autoskillit.core import SessionType, extract_path_arg, get_logger, session_type +from autoskillit.pipeline import gate_error_result, headless_error_result + +if TYPE_CHECKING: + from autoskillit.config._config_dataclasses import ProvidersConfig + +logger = get_logger(__name__) + + +def _get_ctx(): # type: ignore[return] + from autoskillit.server._state import _get_ctx as _ctx_fn + + return _ctx_fn() + + +def _get_config(): # type: ignore[return] + from autoskillit.server._state import _get_config as _cfg_fn + + return _cfg_fn() + + +def _require_orchestrator_or_higher(tool_name: str = "") -> str | None: + """Return headless_error JSON if session is L1 (skill session); None if permitted. + + Interactive sessions (HEADLESS not set) always pass. + Headless sessions must be L2 (orchestrator) or L3 (fleet). + Fail-closed: unset/invalid SESSION_TYPE → SKILL → deny. + """ + if os.environ.get("AUTOSKILLIT_HEADLESS") != "1": + return None + + st = session_type() + if st in (SessionType.ORCHESTRATOR, SessionType.FLEET): + return None + + msg = ( + f"{tool_name} cannot be called from skill sessions. " + "Only orchestrator or fleet sessions may call this tool." + if tool_name + else None + ) + return headless_error_result(msg) + + +def _require_orchestrator_exact(tool_name: str = "") -> str | None: + """Return headless_error JSON if session is not exactly L2; None if permitted. + + Interactive sessions (HEADLESS not set) always pass. + Headless sessions must be exactly L2 (orchestrator). + L1 (skill session) and L3 (fleet) are both denied. + """ + if os.environ.get("AUTOSKILLIT_HEADLESS") != "1": + return None + + st = session_type() + if st is SessionType.ORCHESTRATOR: + return None + + if st is SessionType.FLEET: + msg = ( + f"{tool_name} cannot be called from {st.value} sessions. " + f"{st.value.capitalize()} sessions have an auto-opened gate." + " open_kitchen is unnecessary." + if tool_name + else None + ) + else: + msg = ( + f"{tool_name} cannot be called from skill sessions. " + "Only the orchestrator may call this tool." + if tool_name + else None + ) + return headless_error_result(msg) + + +def _require_fleet(tool_name: str = "") -> str | None: + """Return headless_error JSON if session is not L3 (fleet); None if permitted. + + No interactive bypass — fleet is a specific orchestration level, not a headless guard. + L1 (skill session) and L2 (orchestrator) sessions are both denied. + """ + st = session_type() + if st is SessionType.FLEET: + return None + + msg = ( + f"{tool_name} requires a fleet session. Current session type is not fleet." + if tool_name + else None + ) + return headless_error_result(msg) + + +def _require_enabled() -> str | None: + """Return error JSON if tools are not enabled, None if OK. + + All tools are gated by default and can only be activated by the user + typing the open_kitchen prompt. The prompt name is prefixed by Claude + Code based on how the server was loaded (plugin vs --plugin-dir). + This survives --dangerously-skip-permissions because MCP prompts are + outside the permission system. + """ + if not _get_ctx().gate.enabled: + return gate_error_result() + return None + + +def _validate_skill_command(skill_command: str) -> str | None: + """Return error JSON if skill_command does not start with '/', None if OK.""" + if not skill_command.strip().startswith("/"): + return gate_error_result( + "run_skill requires a slash-command as skill_command.\n" + f"Got: {skill_command!r}\n" + "Expected: skill_command must start with '/' " + "(e.g. /autoskillit:open-kitchen, /make-plan, /audit-arch).\n" + "Prose task descriptions are not valid skill invocations." + ) + return None + + +def _check_dry_walkthrough(skill_command: str, cwd: str) -> str | None: + """If skill_command is an implement skill, verify the plan has been dry-walked. + + Returns an error JSON string if validation fails, None if OK. + """ + tokens = skill_command.strip().split() + if not tokens or tokens[0] not in _get_config().implement_gate.skill_names: + return None + skill_name = tokens[0] + plan_path_str = extract_path_arg(skill_command) + if plan_path_str is None: + return gate_error_result(f"Missing plan path argument for {skill_name}") + plan_path = Path(cwd) / plan_path_str + if not plan_path.is_file(): + return gate_error_result(f"Plan file not found: {plan_path}") + + first_line = plan_path.read_text().split("\n", 1)[0].strip() + if first_line != _get_config().implement_gate.marker: + return gate_error_result( + f"Plan has NOT been dry-walked. Run /dry-walkthrough on the plan first. " + f"Expected first line: {_get_config().implement_gate.marker!r}, " + f"actual: {first_line[:100]!r}" + ) + + return None + + +def _resolve_provider_profile( + step_name: str, + recipe_name: str, + config_providers: ProvidersConfig, +) -> tuple[str, dict[str, str]]: + + # Tier 1: per-step config override (requires recipe context) + if recipe_name and step_name: + step_override = config_providers.step_overrides.get(step_name) + if step_override: + logger.debug( + "provider_profile_resolved", + tier="step_override", + profile=step_override, + ) + if step_override == "anthropic": + return ("anthropic", {}) + return (step_override, config_providers.profiles.get(step_override, {})) + + # Tier 2: wildcard override (requires recipe context) + if recipe_name: + wildcard = config_providers.step_overrides.get("*") + if wildcard: + logger.debug( + "provider_profile_resolved", + tier="recipe_wildcard", + profile=wildcard, + ) + if wildcard == "anthropic": + return ("anthropic", {}) + return (wildcard, config_providers.profiles.get(wildcard, {})) + + # Tier 3: step YAML provider field + if step_name: + logger.debug("provider_profile_resolved", tier="step_provider_field", profile=step_name) + if step_name == "anthropic": + return ("anthropic", {}) + return (step_name, config_providers.profiles.get(step_name, {})) + + # Tier 4: default + name = config_providers.default_provider or "anthropic" + logger.debug("provider_profile_resolved", tier="default", profile=name) + if name == "anthropic": + return ("anthropic", {}) + return (name, config_providers.profiles.get(name, {})) + + +def _resolve_model_as_profile( + model_value: str, + config_providers: ProvidersConfig, +) -> tuple[str, str, dict[str, str] | None]: + """Check if a model value names a configured provider profile. + + When a model parameter matches a key in ``providers.profiles``, the profile's + ``ANTHROPIC_MODEL`` entry becomes the effective model for ``--model``, and the + remaining profile entries become environment variable overrides for the subprocess. + + Returns: + (effective_model, profile_name, provider_extras): + - Profile match with ANTHROPIC_MODEL: + (profile's model, profile_key, env_dict without ANTHROPIC_MODEL) + - Profile match without ANTHROPIC_MODEL: + ("", "", None) — caller should fall through to config default + - No match: + (model_value, "", None) — treat as literal model name + """ + if not model_value: + return (model_value, "", None) + + profile = config_providers.profiles.get(model_value) + if profile is None: + return (model_value, "", None) + + actual_model = profile.get("ANTHROPIC_MODEL") + if not actual_model: + logger.warning( + "provider_profile_no_model", + profile=model_value, + hint="Add ANTHROPIC_MODEL to the profile or use a literal model name", + ) + return ("", "", None) + + logger.debug( + "model_as_profile_resolved", + input_model=model_value, + resolved_model=actual_model, + profile=model_value, + ) + env_dict = {k: v for k, v in profile.items() if k != "ANTHROPIC_MODEL"} + return (actual_model, model_value, env_dict) diff --git a/src/autoskillit/server/_lifespan.py b/src/autoskillit/server/_lifespan.py index 4c76e249a4..bd320dc5be 100644 --- a/src/autoskillit/server/_lifespan.py +++ b/src/autoskillit/server/_lifespan.py @@ -17,10 +17,17 @@ from __future__ import annotations import asyncio as _asyncio +import os +from collections.abc import Awaitable, Callable from contextlib import asynccontextmanager from typing import Any -from autoskillit.core import cleanup_readiness_sentinel, get_logger, write_readiness_sentinel +from autoskillit.core import ( + SessionType, + cleanup_readiness_sentinel, + get_logger, + write_readiness_sentinel, +) from autoskillit.execution import RecordingSubprocessRunner from autoskillit.server._state import _get_ctx_or_none, deferred_initialize @@ -63,6 +70,43 @@ def run_startup_drift_check() -> None: logger.exception("startup_drift_check_failed") +def run_startup_hook_health_check() -> list[str]: + """Detect broken hook scripts across all settings scopes on MCP startup. + + Called as a background task alongside run_startup_drift_check(). + Returns list of broken hook commands. Any failure is logged and swallowed. + """ + try: + from autoskillit.hook_registry import ( + find_broken_hook_scripts, + iter_all_scope_paths, + validate_plugin_cache_hooks, + ) + + broken: list[str] = [] + for scope_label, settings_path in iter_all_scope_paths(None): + scope_broken = find_broken_hook_scripts(settings_path) + if scope_broken: + broken.extend(scope_broken) + logger.warning( + "stale_hook_paths_detected", + scope=scope_label, + broken=scope_broken, + ) + cache_broken = validate_plugin_cache_hooks() + if cache_broken: + broken.extend(cache_broken) + logger.warning( + "stale_plugin_cache_hooks_detected", + broken=cache_broken, + remediation="Run `autoskillit install` from an external terminal", + ) + return broken + except Exception: + logger.exception("startup_hook_health_check_failed") + return [] + + def _finalize_recorder() -> None: """Finalize the recording subprocess runner if one is active.""" ctx = _get_ctx_or_none() @@ -79,6 +123,20 @@ async def _run_drift_check_async() -> None: await loop.run_in_executor(None, run_startup_drift_check) +async def _run_retiring_sweep_async() -> None: + """Offload blocking retiring cache sweep to a thread.""" + from autoskillit.core import sweep_retiring_cache # noqa: PLC0415 + + loop = _asyncio.get_running_loop() + await loop.run_in_executor(None, sweep_retiring_cache) + + +async def _run_hook_health_check_async() -> None: + """Offload blocking hook health check to a thread.""" + loop = _asyncio.get_running_loop() + await loop.run_in_executor(None, run_startup_hook_health_check) + + async def _run_deferred_init(ready_event: _asyncio.Event) -> None: """Run deferred_initialize, signalling *ready_event* when done.""" ctx = _get_ctx_or_none() @@ -88,6 +146,158 @@ async def _run_deferred_init(ready_event: _asyncio.Event) -> None: ready_event.set() +async def _fleet_auto_gate_boot(ctx: Any) -> None: + """Auto-open the kitchen gate and prime quota/registry state for fleet sessions. + + Called synchronously in _autoskillit_lifespan before yield, ensuring gate + is open before any tool call arrives. Fails open: any step failure is + logged as a warning and does not abort gate activation. + """ + import os as _os + from pathlib import Path + from uuid import uuid4 + + from autoskillit.core import register_active_kitchen + from autoskillit.pipeline import create_background_task + from autoskillit.server._misc import ( + _prime_quota_cache, + _quota_refresh_loop, + ) + from autoskillit.server.tools.tools_kitchen import _write_hook_config + + ctx.kitchen_id = str(uuid4()) + ctx.active_recipe_packs = frozenset() + ctx.active_recipe_features = frozenset() + if ctx.gate is None: + logger.warning("fleet_auto_gate_boot_no_gate") + return + ctx.gate.enable() + logger.info("fleet_auto_gate_boot", gate_state="open", kitchen_id=ctx.kitchen_id) + + try: + from autoskillit.core import _collect_disabled_feature_tags + from autoskillit.server import mcp as _mcp + + _features = ctx.config.features if ctx.config is not None else {} + _exp_enabled = ctx.config.experimental_enabled if ctx.config is not None else False + for _tag in _collect_disabled_feature_tags(_features, experimental_enabled=_exp_enabled): + _mcp.disable(tags={_tag}) + except Exception: + logger.warning("fleet_auto_gate_boot_feature_suppression_failed", exc_info=True) + + try: + _write_hook_config() + except Exception: + logger.warning("fleet_auto_gate_boot_write_hook_config_failed", exc_info=True) + + try: + await _prime_quota_cache() + except Exception: + logger.warning("fleet_auto_gate_boot_prime_quota_cache_failed", exc_info=True) + + try: + ctx.quota_refresh_task = create_background_task( + _quota_refresh_loop(ctx.config.quota_guard), + label="quota_refresh_loop", + ) + except Exception: + logger.warning("fleet_auto_gate_boot_quota_refresh_failed", exc_info=True) + + try: + register_active_kitchen(ctx.kitchen_id, _os.getpid(), str(Path.cwd())) + except Exception: + logger.warning("fleet_auto_gate_boot_registry_failed", exc_info=True) + + +async def _food_truck_auto_gate_boot(ctx: Any) -> None: + """Auto-open gate for headless food truck (ORCHESTRATOR) sessions. + + Runs at lifespan startup when AUTOSKILLIT_HEADLESS=1 and + AUTOSKILLIT_FOOD_TRUCK_TOOL_TAGS is set. No-ops for interactive + ORCHESTRATOR sessions (open_kitchen handles the gate there). + """ + from pathlib import Path + from uuid import uuid4 + + from autoskillit.core import ( + CAMPAIGN_ID_ENV_VAR, + FOOD_TRUCK_TOOL_TAGS_ENV_VAR, + HEADLESS_ENV_VAR, + register_active_kitchen, + ) + from autoskillit.pipeline import create_background_task + from autoskillit.server._misc import ( + _prime_quota_cache, + _quota_refresh_loop, + ) + from autoskillit.server.tools.tools_kitchen import _write_hook_config + + if os.environ.get(HEADLESS_ENV_VAR) != "1": + return + _raw_tags = os.environ.get(FOOD_TRUCK_TOOL_TAGS_ENV_VAR, "") + if not _raw_tags: + return + + _packs = frozenset(p.strip() for p in _raw_tags.split(",") if p.strip()) + ctx.kitchen_id = os.environ.get(CAMPAIGN_ID_ENV_VAR) or str(uuid4()) + ctx.active_recipe_packs = _packs + ctx.active_recipe_features = frozenset() + + if ctx.gate is None: + logger.warning("food_truck_auto_gate_boot_no_gate") + return + + ctx.gate.enable() + logger.info( + "food_truck_auto_gate_boot", + gate_state="open", + kitchen_id=ctx.kitchen_id, + packs=sorted(_packs), + ) + + try: + from autoskillit.core import _collect_disabled_feature_tags + from autoskillit.server import mcp as _mcp + + _features = ctx.config.features if ctx.config is not None else {} + _exp_enabled = ctx.config.experimental_enabled if ctx.config is not None else False + for _tag in _collect_disabled_feature_tags(_features, experimental_enabled=_exp_enabled): + _mcp.disable(tags={_tag}) + except Exception: + logger.warning("food_truck_auto_gate_boot_feature_suppression_failed", exc_info=True) + + try: + _write_hook_config() + except Exception: + logger.warning("food_truck_auto_gate_boot_hook_config_failed", exc_info=True) + + try: + await _prime_quota_cache() + except Exception: + logger.warning("food_truck_auto_gate_boot_quota_cache_failed", exc_info=True) + + try: + if ctx.config is not None: + ctx.quota_refresh_task = create_background_task( + _quota_refresh_loop(ctx.config.quota_guard), + label="quota_refresh_loop", + ) + except Exception: + logger.warning("food_truck_auto_gate_boot_refresh_loop_failed", exc_info=True) + + try: + register_active_kitchen(ctx.kitchen_id, os.getpid(), str(Path.cwd())) + except Exception: + logger.warning("food_truck_auto_gate_boot_registry_failed", exc_info=True) + + +_LIFESPAN_BOOT_REGISTRY: dict[SessionType, Callable[[Any], Awaitable[None]] | None] = { + SessionType.FLEET: _fleet_auto_gate_boot, + SessionType.ORCHESTRATOR: _food_truck_auto_gate_boot, + SessionType.SKILL: None, +} + + @asynccontextmanager async def _autoskillit_lifespan(server: Any) -> Any: """Server lifecycle: write readiness sentinel, yield, then finalize recording. @@ -112,14 +322,25 @@ async def _autoskillit_lifespan(server: Any) -> Any: """ bg_tasks: list[_asyncio.Task[None]] = [] try: - from autoskillit.pipeline import create_background_task # noqa: PLC0415 - from autoskillit.server import _state # noqa: PLC0415 + from autoskillit.pipeline import create_background_task + from autoskillit.server import _state event = _asyncio.Event() _state._startup_ready = event write_readiness_sentinel() bg_tasks.append(create_background_task(_run_drift_check_async(), label="drift_check")) + bg_tasks.append(create_background_task(_run_retiring_sweep_async(), label="cache_sweep")) + bg_tasks.append( + create_background_task(_run_hook_health_check_async(), label="hook_health") + ) bg_tasks.append(create_background_task(_run_deferred_init(event), label="deferred_init")) + from autoskillit.core import session_type as _resolve_session_type + + _boot_fn = _LIFESPAN_BOOT_REGISTRY.get(_resolve_session_type()) + if _boot_fn is not None: + _boot_ctx = _get_ctx_or_none() + if _boot_ctx is not None: + await _boot_fn(_boot_ctx) yield finally: for task in bg_tasks: @@ -131,6 +352,12 @@ async def _autoskillit_lifespan(server: Any) -> Any: cleanup_readiness_sentinel() except Exception: logger.exception("lifespan sentinel cleanup error") + try: + from autoskillit.core import clear_kitchens_for_pid # noqa: PLC0415 + + clear_kitchens_for_pid(os.getpid()) + except Exception: + logger.exception("lifespan kitchen registry cleanup error") try: _finalize_recorder() except Exception: diff --git a/src/autoskillit/server/_misc.py b/src/autoskillit/server/_misc.py new file mode 100644 index 0000000000..ef514d5446 --- /dev/null +++ b/src/autoskillit/server/_misc.py @@ -0,0 +1,226 @@ +"""Quota, hook-config, triage, and miscellaneous server utilities. + +Also re-exports selected execution/workspace symbols so that tools_*.py files +(which are restricted to autoskillit.{core,pipeline,server,config,fleet} by +REQ-IMP-003 / REQ-ARCH-003) can access them without violating layer rules. +""" + +from __future__ import annotations + +import asyncio +from pathlib import Path +from typing import TYPE_CHECKING, Any + +from autoskillit.core import get_logger +from autoskillit.execution import ( + SCENARIO_STEP_NAME_ENV as SCENARIO_STEP_NAME_ENV, +) +from autoskillit.execution import ( + _refresh_quota_cache as _refresh_quota_cache, +) +from autoskillit.execution import ( + check_and_sleep_if_needed as check_and_sleep_if_needed, +) +from autoskillit.execution import ( + fetch_repo_merge_state as fetch_repo_merge_state, +) +from autoskillit.execution import ( + invalidate_cache as invalidate_cache, +) +from autoskillit.execution import ( + resolve_log_dir as resolve_log_dir, +) +from autoskillit.execution import ( + resolve_remote_name as resolve_remote_name, +) +from autoskillit.execution import ( + write_telemetry_clear_marker as write_telemetry_clear_marker, +) +from autoskillit.hooks import _HOOK_CONFIG_PATH_COMPONENTS +from autoskillit.workspace import clone_registry as clone_registry + +if TYPE_CHECKING: + from autoskillit.config import QuotaGuardConfig + from autoskillit.core import SkillResult + +logger = get_logger(__name__) + +_HOOK_CONFIG_FILENAME: str = _HOOK_CONFIG_PATH_COMPONENTS[-1] +_HOOK_DIR_COMPONENTS: tuple[str, ...] = _HOOK_CONFIG_PATH_COMPONENTS[:-1] + + +def _hook_config_path(project_root: Path) -> Path: + """Return the canonical path to the hook configuration JSON file.""" + return project_root.joinpath(*_HOOK_DIR_COMPONENTS, _HOOK_CONFIG_FILENAME) + + +def _extract_block(text: str, start_delim: str, end_delim: str) -> list[str]: + """Return all lines between start_delim and end_delim (exclusive). + + Returns an empty list if either delimiter is absent or the block is empty. + Lines are returned as-is (no stripping) to preserve JSON-parseable content. + """ + in_block = False + block_lines: list[str] = [] + for line in text.splitlines(): + if line.strip() == start_delim: + in_block = True + continue + if line.strip() == end_delim: + if not in_block: + return [] + return block_lines + if in_block: + block_lines.append(line) + return [] + + +def _build_hook_diagnostic_warning() -> str | None: + """Run hook health and drift checks. Return a warning string if issues are found.""" + from autoskillit.core import DIRECT_INSTALL_CACHE_SUBDIR + from autoskillit.hook_registry import ( + _claude_settings_path, + _count_hook_registry_drift, + find_broken_hook_scripts, + validate_plugin_cache_hooks, + ) + + issues: list[str] = [] + + settings_path = _claude_settings_path("user") + if settings_path.exists(): + broken = find_broken_hook_scripts(settings_path) + drift = _count_hook_registry_drift(settings_path) + if broken: + issues.append(f"Hook scripts not found: {', '.join(broken)}") + if drift.orphaned > 0: + issues.append( + f"{drift.orphaned} orphaned hook entry(ies) in settings.json are not in " + f"HOOK_REGISTRY — every matching tool call will be denied with ENOENT." + ) + if drift.missing > 0: + issues.append( + f"{drift.missing} hook(s) from HOOK_REGISTRY are not deployed in settings.json." + ) + + _cache_dir = ( + Path.home() / ".claude" / "plugins" / "cache" / DIRECT_INSTALL_CACHE_SUBDIR / "autoskillit" + ) + cache_broken = validate_plugin_cache_hooks(cache_dir=_cache_dir) + if cache_broken: + issues.append( + f"Plugin cache has {len(cache_broken)} stale hook path(s): {', '.join(cache_broken)}" + ) + + if not issues: + return None + + lines = ["\n⚠️ Hook configuration issues detected:"] + for issue in issues: + lines.append(f" • {issue}") + lines.append(" → Run 'autoskillit install' to regenerate hook configuration.\n") + return "\n".join(lines) + + +async def _apply_triage_gate( + result: dict[str, Any], name: str, recipe_info: Any = None +) -> dict[str, Any]: + """Apply LLM triage to stale-contract suggestions, suppressing cosmetic ones. + + Delegates to the RecipeRepository implementation via the Composition Root. + """ + from autoskillit.server._state import _ctx + + if _ctx is None or _ctx.recipes is None: + return result + + from autoskillit._llm_triage import triage_staleness + + return await _ctx.recipes.apply_triage_gate( + result, name, recipe_info, _ctx.temp_dir, logger, triage_fn=triage_staleness + ) + + +async def resolve_repo_from_remote(cwd: str, hint: str | None = None) -> str: + """Return 'owner/repo' from git remote URL, or '' on failure. + + hint: optional owner/repo string or full GitHub URL; parsed before + git remote inference. Passes through to resolve_remote_repo. + """ + from autoskillit.execution import resolve_remote_repo + + return await resolve_remote_repo(cwd, hint=hint) or "" + + +async def _prime_quota_cache() -> None: + """Fetch quota from the Anthropic API and write the local cache. + + Called at open_kitchen so the cache is primed before any run_skill hook fires. + Fails open: a quota fetch failure must not abort kitchen open. + """ + from autoskillit.server._state import _get_ctx as _ctx_fn + + try: + await check_and_sleep_if_needed(_ctx_fn().config.quota_guard) + except Exception: + logger.warning("quota_prime_failed", exc_info=True) + + +async def _quota_refresh_loop(config: QuotaGuardConfig) -> None: + """Long-running coroutine: refreshes the quota cache every cache_refresh_interval seconds. + + Designed to run as a background asyncio.Task for the duration of a kitchen session. + The loop sleeps first, then refreshes — ensuring _prime_quota_cache's initial write + is not immediately overwritten. CancelledError from asyncio.sleep propagates + uncaught, terminating the loop cleanly when the task is cancelled. + + Guarantee: with cache_refresh_interval < cache_max_age, the cache written by any + loop tick will still be fresh when the next tick fires. The hook never sees a stale + cache as long as this loop is running. + """ + while True: + await asyncio.sleep(config.cache_refresh_interval) + try: + await _refresh_quota_cache(config) + except Exception as exc: + logger.warning("quota_refresh_loop_error", exc_info=True, error=str(exc)) + + +def persist_run_skill_state(skill_result: SkillResult, project_dir: Path) -> None: + import os # noqa: PLC0415 + + from autoskillit.core import ensure_project_temp # noqa: PLC0415 + from autoskillit.execution import ( # noqa: PLC0415 + SessionState, + persist_session_state, + ) + + if not skill_result.session_id: + return + try: + state = SessionState( + session_id=skill_result.session_id, + pid=os.getpid(), + boot_id="", + starttime_ticks=0, + infra_exit_category=skill_result.infra.exit_category + if skill_result.infra.exit_category + else None, + ) + state_dir = ensure_project_temp(project_dir) / "session_state" + persist_session_state(state, state_dir) + except Exception: + logger.debug("run_skill: could not persist session state", exc_info=True) + + +def clear_run_skill_state(project_dir: Path) -> None: + from autoskillit.core import ensure_project_temp # noqa: PLC0415 + from autoskillit.execution import ( # noqa: PLC0415 + clear_session_state, + ) + + try: + state_dir = ensure_project_temp(project_dir) / "session_state" + clear_session_state(state_dir) + except Exception: + logger.debug("run_skill: could not clear session state", exc_info=True) diff --git a/src/autoskillit/server/_notify.py b/src/autoskillit/server/_notify.py new file mode 100644 index 0000000000..94798efa69 --- /dev/null +++ b/src/autoskillit/server/_notify.py @@ -0,0 +1,131 @@ +"""MCP notification dispatch and response-size tracking.""" + +from __future__ import annotations + +import functools +import json +from collections.abc import Awaitable, Callable +from typing import TYPE_CHECKING, Any, Literal + +from autoskillit.core import RESERVED_LOG_RECORD_KEYS, get_logger + +if TYPE_CHECKING: + from fastmcp import Context + +logger = get_logger(__name__) + + +def _get_ctx_or_none(): # type: ignore[return] + from autoskillit.server._state import _get_ctx_or_none as _ctx_none_fn + + return _ctx_none_fn() + + +async def _notify( + ctx: Context, + level: Literal["info", "error"], + message: str, + logger_name: str, + extra: dict[str, Any] | None = None, +) -> None: + """Send an MCP progress notification via FastMCP's Context. + + Validates extra dict keys against RESERVED_LOG_RECORD_KEYS before + dispatching. Raises ValueError if any reserved key is found — this + surfaces programming errors in tests rather than silently crashing + at runtime only when DEBUG logging is active. + + Catches (RuntimeError, AttributeError, KeyError) from FastMCP internals: + - RuntimeError: no active MCP session (Context.session raises) + - AttributeError: ctx is CurrentContext() sentinel during testing + - KeyError: makeRecord() collision (defense-in-depth; prevented by validation) + """ + if extra: + invalid = RESERVED_LOG_RECORD_KEYS & extra.keys() + if invalid: + raise ValueError( + f"extra dict contains reserved LogRecord keys: {sorted(invalid)!r}. " + "Rename these keys to avoid stdlib logging collisions." + ) + try: + if level == "info": + await ctx.info(message, logger_name=logger_name, extra=extra) + elif level == "error": + await ctx.error(message, logger_name=logger_name, extra=extra) + except (RuntimeError, AttributeError, KeyError): + pass + + +def track_response_size( + tool_name: str, +) -> Callable[[Callable[..., Awaitable[Any]]], Callable[..., Awaitable[Any]]]: + """Decorator: measure the JSON string size of a tool response and record to response_log. + + Last-resort safety net. Tool implementations SHOULD catch exceptions locally + and emit domain-specific envelopes with more helpful ``user_visible_message`` + values; this decorator only catches what slips through. + + Apply BELOW @mcp.tool() so the wrapped function is what FastMCP registers: + + @mcp.tool(tags={"automation"}) + @track_response_size("get_token_summary") + async def get_token_summary(...) -> str: + ... + """ + + def decorator(fn: Callable[..., Awaitable[Any]]) -> Callable[..., Awaitable[Any]]: + @functools.wraps(fn) + async def wrapper(*args: Any, **kwargs: Any) -> Any: + try: + result = await fn(*args, **kwargs) + except Exception as exc: + result = json.dumps( + { + "success": False, + "error": f"{type(exc).__name__}: {exc}", + "exit_code": -1, + "subtype": "tool_exception", + "user_visible_message": ( + f"An internal error occurred in {tool_name}: " + f"{type(exc).__name__}. Run 'autoskillit doctor' or reinstall." + ), + } + ) + logger.exception("Unhandled exception in tool %s", tool_name) + try: + ctx = _get_ctx_or_none() + if ctx is not None: + response_str = result if isinstance(result, str) else json.dumps(result) + threshold = ctx.config.mcp_response.alert_threshold_tokens + exceeded = ctx.response_log.record( + tool_name, response_str, alert_threshold_tokens=threshold + ) + if exceeded: + from fastmcp import Context as FmcpContext + + mcp_ctx = next( + (a for a in args if isinstance(a, FmcpContext)), + next( + (v for v in kwargs.values() if isinstance(v, FmcpContext)), + None, + ), + ) + if mcp_ctx is not None: + await _notify( + mcp_ctx, + "info", + f"MCP tool '{tool_name}' response exceeded " + f"{threshold} estimated token threshold", + logger_name="autoskillit.server.response_size", + ) + except Exception: + logger.warning( + "track_response_size_failed", + tool_name=tool_name, + exc_info=True, + ) + return result + + return wrapper + + return decorator diff --git a/src/autoskillit/server/_session_type.py b/src/autoskillit/server/_session_type.py new file mode 100644 index 0000000000..3bb397bd9f --- /dev/null +++ b/src/autoskillit/server/_session_type.py @@ -0,0 +1,71 @@ +"""Session-type tag visibility dispatcher for the FastMCP server. + +Separated from server/__init__.py to satisfy the pure-facade constraint on +sub-package __init__ files. +""" + +from __future__ import annotations + +import os + +from autoskillit.core import ( + CATEGORY_TAGS, + FEATURE_REGISTRY, + FLEET_DISPATCH_MODE, + FLEET_MODE_ENV_VAR, + FOOD_TRUCK_TOOL_TAGS_ENV_VAR, + HEADLESS_ENV_VAR, + SessionType, + get_logger, +) +from autoskillit.core import session_type as _resolve_session_type + +logger = get_logger(__name__) + + +def _collect_fleet_tool_tags() -> frozenset[str]: + """Return the union of all tool_tags across FEATURE_REGISTRY entries.""" + return frozenset().union(*(fdef.tool_tags for fdef in FEATURE_REGISTRY.values())) + + +def _apply_session_type_visibility() -> None: + """Apply FastMCP tag visibility based on session type + HEADLESS. + + Session-type dispatch only — feature gate suppression is handled at lifespan + time by _fleet_auto_gate_boot (fleet sessions) and _redisable_subsets + (open_kitchen sessions) where the full config pipeline is available. + """ + from autoskillit.server import mcp + + _session = _resolve_session_type() + _headless = os.environ.get(HEADLESS_ENV_VAR) == "1" + + if _session is SessionType.FLEET: + fleet_tags = _collect_fleet_tool_tags() + if fleet_tags: + mcp.enable(tags=set(fleet_tags)) + if os.environ.get(FLEET_MODE_ENV_VAR) == FLEET_DISPATCH_MODE: + mcp.enable(tags={"fleet-dispatch"}) + elif _session is SessionType.ORCHESTRATOR and _headless: + tool_tags = os.environ.get(FOOD_TRUCK_TOOL_TAGS_ENV_VAR, "") + if tool_tags: + mcp.enable(tags={"kitchen-core"}) + for pack in tool_tags.split(","): + pack = pack.strip() + if not pack: + continue + if pack not in CATEGORY_TAGS: + logger.warning( + "Unknown pack %r in AUTOSKILLIT_FOOD_TRUCK_TOOL_TAGS" + " — skipping mcp.enable(); valid packs: %s", + pack, + ", ".join(sorted(CATEGORY_TAGS)), + ) + continue + mcp.enable(tags={pack}) + else: + mcp.enable(tags={"kitchen"}) + elif _session is SessionType.SKILL and _headless: + mcp.enable(tags={"headless"}) + # ORCHESTRATOR+interactive and SKILL+interactive: no pre-reveal. + # Cook unlocks via open_kitchen (orchestrator) or stays minimal (skill session). diff --git a/src/autoskillit/server/_state.py b/src/autoskillit/server/_state.py index 7c7dbfd7d7..c1ba23779e 100644 --- a/src/autoskillit/server/_state.py +++ b/src/autoskillit/server/_state.py @@ -21,7 +21,7 @@ from datetime import UTC from autoskillit.config import AutomationConfig -from autoskillit.core import get_logger +from autoskillit.core import DirectInstall, MarketplaceInstall, get_logger from autoskillit.pipeline import ToolContext logger = get_logger(__name__) @@ -175,13 +175,30 @@ def _get_config() -> AutomationConfig: def _get_plugin_dir() -> str | None: - """Return plugin_dir from the current server context, or None if uninitialized.""" - return _ctx.plugin_dir if _ctx is not None else None + """Return the plugin path from the current server context, or None if uninitialized.""" + if _ctx is None: + return None + match _ctx.plugin_source: + case DirectInstall(plugin_dir=p): + return str(p) + case MarketplaceInstall(cache_path=cp): + return str(cp) + + +def _check_rerun(log_dir: str, composite_hash: str) -> dict[str, object] | None: + """Check sessions.jsonl for prior runs with the same composite hash.""" + if not composite_hash: + return None + from autoskillit.execution import resolve_log_dir # noqa: PLC0415 + from autoskillit.recipe import check_rerun_detection # noqa: PLC0415 + + sessions_path = resolve_log_dir(log_dir) / "sessions.jsonl" + return check_rerun_detection(sessions_path, composite_hash=composite_hash) def version_info() -> dict: """Return version health information for the running server.""" from autoskillit.version import version_info as _compute_version - plugin_dir = _ctx.plugin_dir if _ctx is not None else None + plugin_dir = _get_plugin_dir() return _compute_version(plugin_dir) diff --git a/src/autoskillit/server/_subprocess.py b/src/autoskillit/server/_subprocess.py new file mode 100644 index 0000000000..cced878e10 --- /dev/null +++ b/src/autoskillit/server/_subprocess.py @@ -0,0 +1,78 @@ +"""Subprocess execution helpers for MCP tools.""" + +from __future__ import annotations + +import time +from collections.abc import Mapping +from datetime import UTC, datetime +from pathlib import Path +from typing import TYPE_CHECKING + +from autoskillit.core import TerminationReason + +if TYPE_CHECKING: + from autoskillit.core import SubprocessResult + + +def _get_ctx(): # type: ignore[return] + """Deferred import of _get_ctx from _state to avoid circular imports.""" + from autoskillit.server._state import _get_ctx as _ctx_fn + + return _ctx_fn() + + +def _process_runner_result( + result: SubprocessResult, + timeout: float, +) -> tuple[int, str, str]: + """Convert a SubprocessResult to (returncode, stdout, stderr). + + Translates TIMED_OUT termination into (-1, stdout, "Process timed out after {timeout}s"). + """ + if result.termination == TerminationReason.TIMED_OUT: + return -1, result.stdout, f"Process timed out after {timeout}s" + return result.returncode, result.stdout, result.stderr + + +_GH_API_SUBCOMMANDS: frozenset[str] = frozenset( + {"api", "pr", "issue", "repo", "release", "run", "workflow", "search"} +) + + +def _is_github_cli_call(cmd: list[str]) -> bool: + return len(cmd) >= 2 and cmd[0] == "gh" and cmd[1] in _GH_API_SUBCOMMANDS + + +async def _run_subprocess( + cmd: list[str], + *, + cwd: str, + timeout: float, + env: Mapping[str, str] | None = None, +) -> tuple[int, str, str]: + """Run a subprocess asynchronously with timeout. Returns (returncode, stdout, stderr). + + Delegates to run_managed_async which uses temp file I/O (immune to + pipe-blocking from child FD inheritance) and psutil process tree cleanup. + """ + runner = _get_ctx().runner + assert runner is not None, "No subprocess runner configured" + + is_gh = _is_github_cli_call(cmd) + start = time.monotonic() if is_gh else 0.0 + + result = await runner(cmd, cwd=Path(cwd), timeout=timeout, env=env) + returncode, stdout, stderr = _process_runner_result(result, timeout) + + if is_gh: + latency_ms = (time.monotonic() - start) * 1000.0 + log = _get_ctx().github_api_log + if log is not None: + await log.record_gh_cli( + subcommand=" ".join(str(c) for c in cmd[:3]), + exit_code=returncode, + latency_ms=latency_ms, + timestamp=datetime.now(UTC).isoformat(), + ) + + return returncode, stdout, stderr diff --git a/src/autoskillit/server/_wire_compat.py b/src/autoskillit/server/_wire_compat.py index 54e323aff2..cc61dc2cd2 100644 --- a/src/autoskillit/server/_wire_compat.py +++ b/src/autoskillit/server/_wire_compat.py @@ -1,11 +1,13 @@ """Wire-format compatibility middleware for Claude Code MCP client. Claude Code bug #25081 silently drops ALL tools from servers whose -tools/list response includes ``outputSchema``, ``annotations``, or ``title`` -fields. FastMCP 3.2.3+ auto-generates these for typed return values and -annotated tools. This middleware strips them at the wire level so tool -registration code can use idiomatic FastMCP patterns without worrying -about client-side parser quirks. +tools/list response includes ``outputSchema`` or ``title`` fields. +FastMCP 3.2.3+ auto-generates these for typed return values and annotated +tools. This middleware strips only those fields at the wire level. + +``annotations`` is intentionally preserved: it carries ``readOnlyHint`` +which Claude Code uses to enable parallel tool execution (~7x pipeline +speedup). Stripping annotations forces all tools to run serially. Ref: https://github.com/anthropics/claude-code/issues/25081 """ @@ -22,6 +24,8 @@ from fastmcp.server.middleware import CallNext, MiddlewareContext from fastmcp.tools.tool import Tool +_STRIPPED_FIELDS: dict[str, None] = {"output_schema": None, "title": None} + class ClaudeCodeCompatMiddleware(Middleware): """Strip wire fields that trigger Claude Code #25081 tool-list rejection.""" @@ -34,8 +38,6 @@ async def on_list_tools( tools = await call_next(context) cleaned: list[Tool] = [] for tool in tools: - patched = tool.model_copy( - update={"output_schema": None, "annotations": None, "title": None} - ) + patched = tool.model_copy(update=_STRIPPED_FIELDS) cleaned.append(patched) return cleaned diff --git a/src/autoskillit/server/git.py b/src/autoskillit/server/git.py index 6991f5acbc..94c817b6e0 100644 --- a/src/autoskillit/server/git.py +++ b/src/autoskillit/server/git.py @@ -1,6 +1,6 @@ """Git merge workflow for the merge_worktree MCP tool. -L3 service module. Executes the full merge pipeline: +IL-3 service module. Executes the full merge pipeline: path validation → worktree verification → branch detection → test gate → fetch → rebase → main-repo merge → worktree cleanup. @@ -20,12 +20,12 @@ MergeFailedStep, MergeState, SubprocessRunner, - TerminationReason, get_logger, is_protected_branch, truncate_text, ) from autoskillit.server._editable_guard import scan_editable_installs_for_worktree +from autoskillit.server._subprocess import _process_runner_result from autoskillit.workspace import remove_git_worktree, remove_worktree_sidecar if TYPE_CHECKING: @@ -71,12 +71,10 @@ async def _run_git( Returns (returncode, stdout, stderr). Handles TIMED_OUT termination. """ result = await runner(cmd, cwd=Path(cwd), timeout=timeout) - if result.termination == TerminationReason.TIMED_OUT: - return -1, result.stdout, f"Process timed out after {timeout}s" - return result.returncode, result.stdout, result.stderr + return _process_runner_result(result, timeout) -@dataclasses.dataclass(frozen=True) +@dataclasses.dataclass(frozen=True, slots=True) class GitMergeTarget: """Verified merge target — proof that main_repo's branch was checked.""" @@ -127,6 +125,7 @@ async def perform_merge( worktree_path: str, base_branch: str, *, + remote: str = "origin", config: AutomationConfig, runner: SubprocessRunner, tester: TestRunner | None = None, @@ -291,12 +290,10 @@ async def perform_merge( } # 5. Fetch - fetch_rc, _, fetch_stderr = await _run_git( - ["git", "fetch", "origin"], worktree_path, 60, runner - ) + fetch_rc, _, fetch_stderr = await _run_git(["git", "fetch", remote], worktree_path, 60, runner) if fetch_rc != 0: return { - "error": "git fetch origin failed", + "error": f"git fetch {remote} failed", "failed_step": MergeFailedStep.FETCH, "state": MergeState.WORKTREE_INTACT, "stderr": truncate_text(fetch_stderr), @@ -305,7 +302,7 @@ async def perform_merge( # 5.5. Verify base branch remote tracking ref exists ref_rc, _, _ = await _run_git( - ["git", "rev-parse", "--verify", f"refs/remotes/origin/{base_branch}"], + ["git", "rev-parse", "--verify", f"refs/remotes/{remote}/{base_branch}"], worktree_path, 10, runner, @@ -314,8 +311,8 @@ async def perform_merge( return { "error": ( f"Base branch '{base_branch}' has no remote tracking ref — " - f"push it to origin before running this pipeline: " - f"git push -u origin {base_branch}" + f"push it to {remote} before running this pipeline: " + f"git push -u {remote} {base_branch}" ), "failed_step": MergeFailedStep.PRE_REBASE_CHECK, "state": MergeState.WORKTREE_INTACT_BASE_NOT_PUBLISHED, @@ -327,7 +324,7 @@ async def perform_merge( # Standard git rebase cannot replay merge commits and fails with a generic error. # Catch this early with a specific, actionable message. mc_rc, mc_out, _ = await _run_git( - ["git", "log", "--merges", "--oneline", f"origin/{base_branch}..HEAD"], + ["git", "log", "--merges", "--oneline", f"{remote}/{base_branch}..HEAD"], worktree_path, 15, runner, @@ -337,7 +334,7 @@ async def perform_merge( return { "error": ( f"Worktree branch contains {len(merge_list)} merge commit(s) not yet in " - f"origin/{base_branch}. Standard git rebase cannot replay merge commits. " + f"{remote}/{base_branch}. Standard git rebase cannot replay merge commits. " "The conflict-resolution plan must use 'git cherry-pick' or " "'git checkout -- ' to produce a linear commit history. " "Route this failure to cleanup_failure — do NOT use run_cmd to bypass." @@ -350,7 +347,7 @@ async def perform_merge( # 6. Rebase rc, _, rebase_stderr = await _run_git( - ["git", "rebase", "--autostash", f"origin/{base_branch}"], worktree_path, 120, runner + ["git", "rebase", "--autostash", f"{remote}/{base_branch}"], worktree_path, 120, runner ) if rc != 0: abort_rc, _, abort_stderr = await _run_git( @@ -415,6 +412,23 @@ async def perform_merge( return target_or_err target = target_or_err + # 7.6 Pre-merge dirty check — reject if main repo has uncommitted changes + rc, status_out, _ = await _run_git(["git", "status", "--porcelain"], target.path, 10, runner) + if rc == 0 and status_out.strip(): + dirty_files = [line for line in status_out.splitlines() if line.strip()] + return { + "error": ( + f"Main repository has {len(dirty_files)} dirty file(s): " + f"{', '.join(f.strip() for f in dirty_files[:10])}" + f"{' ...' if len(dirty_files) > 10 else ''}. " + f"Clean the working tree before merging." + ), + "failed_step": MergeFailedStep.DIRTY_MAIN_REPO, + "state": MergeState.WORKTREE_INTACT, + "worktree_path": worktree_path, + "dirty_files": dirty_files, + } + # 8. Merge rc, _, merge_stderr = await _run_git( ["git", "merge", "--no-edit", worktree_branch], target.path, 60, runner diff --git a/src/autoskillit/server/helpers.py b/src/autoskillit/server/helpers.py deleted file mode 100644 index 6583cefdc2..0000000000 --- a/src/autoskillit/server/helpers.py +++ /dev/null @@ -1,565 +0,0 @@ -"""Subprocess wrapper, dry-walkthrough check, and shared helpers for MCP tools.""" - -from __future__ import annotations - -import asyncio -import functools -import json -import os -import time -from collections.abc import Awaitable, Callable, Mapping -from pathlib import Path -from typing import TYPE_CHECKING, Any - -from autoskillit.core import RESERVED_LOG_RECORD_KEYS, TerminationReason, get_logger -from autoskillit.execution import ( - SCENARIO_STEP_NAME_ENV, # noqa: F401 — re-exported for tools_execution.py - _refresh_quota_cache, # noqa: F401 — re-exported for tools_execution.py; patched by tests - fetch_repo_merge_state, # noqa: F401 — re-exported for tools_ci.py - resolve_log_dir, # noqa: F401 — used by tools_github.py, tools_status.py - write_telemetry_clear_marker, # noqa: F401 — used by tools_status.py -) -from autoskillit.hooks import _HOOK_CONFIG_PATH_COMPONENTS -from autoskillit.pipeline import gate_error_result -from autoskillit.recipe import ( - StaleItem, - StalenessEntry, - compute_recipe_hash, - find_recipe_by_name, - load_bundled_manifest, - read_staleness_cache, - write_staleness_cache, -) -from autoskillit.workspace import clone_registry # noqa: F401 — re-exported for tools_clone.py - -if TYPE_CHECKING: - from fastmcp import Context - - from autoskillit.config import QuotaGuardConfig - from autoskillit.core import SubprocessResult - -logger = get_logger(__name__) - -_HOOK_CONFIG_FILENAME: str = _HOOK_CONFIG_PATH_COMPONENTS[-1] -_HOOK_DIR_COMPONENTS: tuple[str, ...] = _HOOK_CONFIG_PATH_COMPONENTS[:-1] - - -def _hook_config_path(project_root: Path) -> Path: - """Return the canonical path to the hook configuration JSON file.""" - return project_root.joinpath(*_HOOK_DIR_COMPONENTS, _HOOK_CONFIG_FILENAME) - - -async def _notify( - ctx: Context, - level: str, - message: str, - logger_name: str, - extra: dict[str, Any] | None = None, -) -> None: - """Send an MCP progress notification via FastMCP's Context. - - Validates extra dict keys against RESERVED_LOG_RECORD_KEYS before - dispatching. Raises ValueError if any reserved key is found — this - surfaces programming errors in tests rather than silently crashing - at runtime only when DEBUG logging is active. - - Catches (RuntimeError, AttributeError, KeyError) from FastMCP internals: - - RuntimeError: no active MCP session (Context.session raises) - - AttributeError: ctx is CurrentContext() sentinel during testing - - KeyError: makeRecord() collision (defense-in-depth; prevented by validation) - """ - if extra: - invalid = RESERVED_LOG_RECORD_KEYS & extra.keys() - if invalid: - raise ValueError( - f"extra dict contains reserved LogRecord keys: {sorted(invalid)!r}. " - "Rename these keys to avoid stdlib logging collisions." - ) - try: - if level == "info": - await ctx.info(message, logger_name=logger_name, extra=extra) - elif level == "error": - await ctx.error(message, logger_name=logger_name, extra=extra) - except (RuntimeError, AttributeError, KeyError): - pass - - -def _get_ctx(): # type: ignore[return] - """Deferred import of _get_ctx from _state to avoid circular imports.""" - from autoskillit.server._state import _get_ctx as _ctx_fn - - return _ctx_fn() - - -def _get_config(): # type: ignore[return] - """Deferred import of _get_config from _state to avoid circular imports.""" - from autoskillit.server._state import _get_config as _cfg_fn - - return _cfg_fn() - - -def _get_ctx_or_none(): # type: ignore[return] - """Deferred import of _get_ctx_or_none from _state to avoid circular imports.""" - from autoskillit.server._state import _get_ctx_or_none as _ctx_none_fn - - return _ctx_none_fn() - - -def track_response_size( - tool_name: str, -) -> Callable[[Callable[..., Awaitable[Any]]], Callable[..., Awaitable[Any]]]: - """Decorator: measure the JSON string size of a tool response and record to response_log. - - Last-resort safety net. Tool implementations SHOULD catch exceptions locally - and emit domain-specific envelopes with more helpful ``user_visible_message`` - values; this decorator only catches what slips through. - - Apply BELOW @mcp.tool() so the wrapped function is what FastMCP registers: - - @mcp.tool(tags={"automation"}) - @track_response_size("get_token_summary") - async def get_token_summary(...) -> str: - ... - """ - - def decorator(fn: Callable[..., Awaitable[Any]]) -> Callable[..., Awaitable[Any]]: - @functools.wraps(fn) - async def wrapper(*args: Any, **kwargs: Any) -> Any: - try: - result = await fn(*args, **kwargs) - except Exception as exc: - result = json.dumps( - { - "success": False, - "error": f"{type(exc).__name__}: {exc}", - "exit_code": -1, - "subtype": "tool_exception", - "user_visible_message": ( - f"An internal error occurred in {tool_name}: " - f"{type(exc).__name__}. Run 'autoskillit doctor' or reinstall." - ), - } - ) - logger.exception("Unhandled exception in tool %s", tool_name) - try: - ctx = _get_ctx_or_none() - if ctx is not None: - response_str = result if isinstance(result, str) else json.dumps(result) - threshold = ctx.config.mcp_response.alert_threshold_tokens - exceeded = ctx.response_log.record( - tool_name, response_str, alert_threshold_tokens=threshold - ) - if exceeded: - from fastmcp import Context as FmcpContext - - mcp_ctx = next( - (a for a in args if isinstance(a, FmcpContext)), - next( - (v for v in kwargs.values() if isinstance(v, FmcpContext)), - None, - ), - ) - if mcp_ctx is not None: - await _notify( - mcp_ctx, - "info", - f"MCP tool '{tool_name}' response exceeded " - f"{threshold} estimated token threshold", - logger_name="autoskillit.server.response_size", - ) - except Exception: - logger.warning( - "track_response_size_failed", - tool_name=tool_name, - exc_info=True, - ) - return result - - return wrapper - - return decorator - - -def _require_not_headless(tool_name: str = "") -> str | None: - """Return headless_error JSON if called from a headless session; None if safe.""" - if os.environ.get("AUTOSKILLIT_HEADLESS") == "1": - from autoskillit.pipeline import headless_error_result - - msg = ( - f"{tool_name} cannot be called from headless sessions. " - "Only the Tier 1 orchestrator may call this tool." - if tool_name - else None - ) - return headless_error_result(msg) - return None - - -def _require_enabled() -> str | None: - """Return error JSON if tools are not enabled, None if OK. - - All tools are gated by default and can only be activated by the user - typing the open_kitchen prompt. The prompt name is prefixed by Claude - Code based on how the server was loaded (plugin vs --plugin-dir). - This survives --dangerously-skip-permissions because MCP prompts are - outside the permission system. - """ - if not _get_ctx().gate.enabled: - return gate_error_result() - return None - - -def _validate_skill_command(skill_command: str) -> str | None: - """Return error JSON if skill_command does not start with '/', None if OK.""" - if not skill_command.strip().startswith("/"): - return gate_error_result( - "run_skill requires a slash-command as skill_command.\n" - f"Got: {skill_command!r}\n" - "Expected: skill_command must start with '/' " - "(e.g. /autoskillit:investigate, /autoskillit:make-plan, /audit-arch).\n" - "Prose task descriptions are not valid skill invocations." - ) - return None - - -def _extract_block(text: str, start_delim: str, end_delim: str) -> list[str]: - """Return all lines between start_delim and end_delim (exclusive). - - Returns an empty list if either delimiter is absent or the block is empty. - Lines are returned as-is (no stripping) to preserve JSON-parseable content. - """ - in_block = False - block_lines: list[str] = [] - for line in text.splitlines(): - if line.strip() == start_delim: - in_block = True - continue - if line.strip() == end_delim: - if not in_block: - return [] - return block_lines - if in_block: - block_lines.append(line) - return [] # end delimiter never found - - -async def _apply_triage_gate( - result: dict[str, Any], name: str, recipe_info: Any = None -) -> dict[str, Any]: - """Apply LLM triage to stale-contract suggestions, suppressing cosmetic ones. - - Checks the staleness cache for a cached triage result. If not cached, - runs triage_staleness() (a 30s Haiku call) for hash_mismatch items only. - version_mismatch items are always treated as meaningful and never suppressed. - - When ``recipe_info`` is provided by the caller, the internal find() call is - skipped, eliminating the duplicate YAML directory scan. - - Modifies ``result`` in-place and returns it. - """ - from autoskillit.server._state import _ctx - - if _ctx is None or _ctx.recipes is None: - return result - - stale_suggs = [s for s in result.get("suggestions", []) if s.get("rule") == "stale-contract"] - if not stale_suggs: - return result - - if recipe_info is None: - recipe_info = _ctx.recipes.find(name, Path.cwd()) - if recipe_info is None: - return result - - cache_path = _ctx.temp_dir / "recipe_staleness_cache.json" - t0 = time.perf_counter() - cached = read_staleness_cache(cache_path, name) - logger.debug( - "triage_gate_cache_read", - recipe=name, - elapsed_ms=round((time.perf_counter() - t0) * 1000, 1), - ) - - if cached is not None and cached.triage_result == "cosmetic": - result["suggestions"] = [ - s for s in result["suggestions"] if s.get("rule") != "stale-contract" - ] - return result - - if cached is None or cached.triage_result is None: - hash_items = [ - StaleItem( - skill=s["skill"], - reason=s["reason"], - stored_value=s.get("stored_value", ""), - current_value=s.get("current_value", ""), - ) - for s in stale_suggs - if s.get("reason") == "hash_mismatch" - ] - if hash_items: - from datetime import UTC, datetime - - from autoskillit._llm_triage import triage_staleness - - t_llm = time.perf_counter() - triage = await triage_staleness(hash_items) - logger.debug( - "triage_gate_llm_triage", - recipe=name, - elapsed_ms=round((time.perf_counter() - t_llm) * 1000, 1), - ) - all_cosmetic = all(not r.get("meaningful", True) for r in triage) - triage_str = "cosmetic" if all_cosmetic else "meaningful" - current_hash = compute_recipe_hash(recipe_info.path) - current_ver = load_bundled_manifest().get("version", "") - write_staleness_cache( - cache_path, - name, - StalenessEntry( - recipe_hash=current_hash, - manifest_version=current_ver, - is_stale=True, - triage_result=triage_str, - checked_at=datetime.now(UTC).isoformat(), - ), - ) - if all_cosmetic and not any( - s.get("reason") == "version_mismatch" for s in stale_suggs - ): - result["suggestions"] = [ - s for s in result["suggestions"] if s.get("rule") != "stale-contract" - ] - - return result - - -def _process_runner_result( - result: SubprocessResult, - timeout: float, -) -> tuple[int, str, str]: - """Convert a SubprocessResult to (returncode, stdout, stderr). - - Translates TIMED_OUT termination into (-1, stdout, "Process timed out after {timeout}s"). - Shared by _run_subprocess (helpers.py) and _run_git (git.py). - """ - if result.termination == TerminationReason.TIMED_OUT: - return -1, result.stdout, f"Process timed out after {timeout}s" - return result.returncode, result.stdout, result.stderr - - -async def _run_subprocess( - cmd: list[str], - *, - cwd: str, - timeout: float, - env: Mapping[str, str] | None = None, -) -> tuple[int, str, str]: - """Run a subprocess asynchronously with timeout. Returns (returncode, stdout, stderr). - - Delegates to run_managed_async which uses temp file I/O (immune to - pipe-blocking from child FD inheritance) and psutil process tree cleanup. - """ - runner = _get_ctx().runner - assert runner is not None, "No subprocess runner configured" - result = await runner(cmd, cwd=Path(cwd), timeout=timeout, env=env) - return _process_runner_result(result, timeout) - - -def _check_dry_walkthrough(skill_command: str, cwd: str) -> str | None: - """If skill_command is an implement skill, verify the plan has been dry-walked. - - Returns an error JSON string if validation fails, None if OK. - """ - parts = skill_command.strip().split(None, 1) - if not parts or parts[0] not in _get_config().implement_gate.skill_names: - return None - - skill_name = parts[0] - - if len(parts) < 2: - return gate_error_result(f"Missing plan path argument for {skill_name}") - - plan_path = Path(cwd) / parts[1].strip().strip('"').strip("'") - if not plan_path.is_file(): - return gate_error_result(f"Plan file not found: {plan_path}") - - # TOCTOU acceptance (option c, per P1-5): This function reads the plan file - # once here. If the file is modified or deleted between this gate check and - # the headless session acting on it, the gate condition will be stale. - # This is an accepted limitation: the plan file is user-controlled, and - # modification between check and execution is a user error. Options (a) and - # (b) — passing file content via stdin or re-verifying via content hash — - # involve significant scope and are deferred. - first_line = plan_path.read_text().split("\n", 1)[0].strip() - if first_line != _get_config().implement_gate.marker: - return gate_error_result( - f"Plan has NOT been dry-walked. Run /dry-walkthrough on the plan first. " - f"Expected first line: {_get_config().implement_gate.marker!r}, " - f"actual: {first_line[:100]!r}" - ) - - return None - - -async def _import_and_call( - dotted_path: str, - args: dict[str, object] | None = None, - timeout: float = 30, -) -> dict[str, object]: - """Import a Python callable by dotted path and invoke it. - - Returns dict with 'success', 'result' (or 'error'). - Handles sync and async callables, with timeout protection. - """ - import importlib - import inspect - - if args is None: - args = {} - - if "." not in dotted_path: - return {"success": False, "error": f"Invalid dotted path: {dotted_path!r}"} - - module_path, attr_name = dotted_path.rsplit(".", 1) - - try: - module = importlib.import_module(module_path) - except ImportError as exc: - return {"success": False, "error": f"Import failed for {module_path!r}: {exc}"} - - try: - func = getattr(module, attr_name) - except AttributeError: - return { - "success": False, - "error": f"Module {module_path!r} has no attribute {attr_name!r}", - } - - if not callable(func): - return {"success": False, "error": f"{dotted_path!r} is not callable"} - - try: - if inspect.iscoroutinefunction(func): - result = await asyncio.wait_for(func(**args), timeout=timeout) - else: - result = await asyncio.wait_for(asyncio.to_thread(func, **args), timeout=timeout) - except TimeoutError: - logger.warning( - "run_python timed out; sync thread may continue running", - dotted_path=dotted_path, - timeout=timeout, - ) - return {"success": False, "error": f"Timeout after {timeout}s calling {dotted_path}"} - except Exception as exc: - logger.warning( - "run_python execution failed", - dotted_path=dotted_path, - error=type(exc).__name__, - ) - return {"success": False, "error": f"{type(exc).__name__}: {exc}"} - - try: - import json as _json - - _json.dumps(result) - return {"success": True, "result": result} - except (TypeError, ValueError): - return {"success": True, "result": str(result)} - - -def _find_recipe(name: str, cwd: Path) -> Any: - """Look up a recipe by name. Delegates to recipe layer; exposed for tools_kitchen.py. - - tools_kitchen.py (a tools_*.py file) is restricted by REQ-IMP-003 to importing - only from autoskillit.core, autoskillit.pipeline, and autoskillit.server. - This function provides the architecture-compliant bridge to autoskillit.recipe. - """ - return find_recipe_by_name(name, cwd) - - -async def infer_repo_from_remote(cwd: str, hint: str | None = None) -> str: - """Return 'owner/repo' from git remote URL, or '' on failure. - - hint: optional owner/repo string or full GitHub URL; parsed before - git remote inference. Passes through to resolve_remote_repo. - """ - from autoskillit.execution import resolve_remote_repo - - return await resolve_remote_repo(cwd, hint=hint) or "" - - -async def _prime_quota_cache() -> None: - """Fetch quota from the Anthropic API and write the local cache. - - Called at open_kitchen so the cache is primed before any run_skill hook fires. - Fails open: a quota fetch failure must not abort kitchen open. - """ - from autoskillit.execution import check_and_sleep_if_needed - from autoskillit.server import _get_ctx - - try: - await check_and_sleep_if_needed(_get_ctx().config.quota_guard) - except Exception: - logger.warning("quota_prime_failed", exc_info=True) - - -async def _quota_refresh_loop(config: QuotaGuardConfig) -> None: - """Long-running coroutine: refreshes the quota cache every cache_refresh_interval seconds. - - Designed to run as a background asyncio.Task for the duration of a kitchen session. - The loop sleeps first, then refreshes — ensuring _prime_quota_cache's initial write - is not immediately overwritten. CancelledError from asyncio.sleep propagates - uncaught, terminating the loop cleanly when the task is cancelled. - - Guarantee: with cache_refresh_interval < cache_max_age, the cache written by any - loop tick will still be fresh when the next tick fires. The hook never sees a stale - cache as long as this loop is running. - """ - while True: - await asyncio.sleep(config.cache_refresh_interval) - try: - await _refresh_quota_cache(config) - except Exception as exc: - logger.warning("quota_refresh_loop_error", exc_info=True, error=str(exc)) - - -def _build_hook_diagnostic_warning() -> str | None: - """Run hook health and drift checks. Return a warning string if issues are found. - - Only reads; never writes or modifies state. Returns None when all hooks are healthy - or when settings.json does not yet exist (nothing to validate). - """ - from autoskillit.hook_registry import ( - _claude_settings_path, - _count_hook_registry_drift, - find_broken_hook_scripts, - ) - - settings_path = _claude_settings_path("user") - if not settings_path.exists(): - return None - - broken = find_broken_hook_scripts(settings_path) - drift = _count_hook_registry_drift(settings_path) - - issues: list[str] = [] - if broken: - issues.append(f"Hook scripts not found: {', '.join(broken)}") - if drift.orphaned > 0: - issues.append( - f"{drift.orphaned} orphaned hook entry(ies) in settings.json are not in " - f"HOOK_REGISTRY — every matching tool call will be denied with ENOENT." - ) - if drift.missing > 0: - issues.append( - f"{drift.missing} hook(s) from HOOK_REGISTRY are not deployed in settings.json." - ) - if not issues: - return None - - lines = ["\n⚠️ Hook configuration issues detected:"] - for issue in issues: - lines.append(f" • {issue}") - lines.append(" → Run 'autoskillit install' to regenerate hook configuration.\n") - return "\n".join(lines) diff --git a/src/autoskillit/server/tools/CLAUDE.md b/src/autoskillit/server/tools/CLAUDE.md new file mode 100644 index 0000000000..852f70e3ed --- /dev/null +++ b/src/autoskillit/server/tools/CLAUDE.md @@ -0,0 +1,30 @@ +# tools/ + +MCP `@mcp.tool()` handlers registered on import (15 tool modules). + +## Files + +| File | Purpose | +|------|---------| +| `__init__.py` | Docstring-only — tools register via `@mcp.tool()` on import | +| `_types.py` | TypedDict definitions for server tool JSON responses (RunSkillResult, RunCmdResult, etc.) | +| `tools_kitchen.py` | `open_kitchen`, `close_kitchen` (gate lifecycle), `recipe://` MCP resource | +| `tools_ci.py` | `set_commit_status`, `check_repo_merge_state` | +| `tools_ci_watch.py` | `wait_for_ci`, `get_ci_status`, `_auto_trigger_ci` | +| `tools_ci_merge_queue.py` | `toggle_auto_merge`, `enqueue_pr`, `wait_for_merge_queue` | +| `tools_clone.py` | `clone_repo`, `remove_clone`, `push_to_remote`, `register_clone_status`, `batch_cleanup_clones`, `bootstrap_clone` | +| `_execution_helpers.py` | `_import_and_call`, `_coerce_scalar` subprocess coercion helpers (no MCP tools) | +| `tools_execution.py` | `run_cmd`, `run_python`, `run_skill` | +| `tools_fleet_dispatch.py` | `dispatch_food_truck`, `record_gate_dispatch` | +| `tools_git.py` | `merge_worktree`, `classify_fix`, `create_unique_branch`, `create_and_publish_branch`, `check_pr_mergeable` | +| `tools_github.py` | `fetch_github_issue`, `get_issue_title`, `report_bug` | +| `tools_issue_lifecycle.py` | `prepare_issue`, `enrich_issues`, `claim_issue`, `release_issue` | +| `tools_issue_composite.py` | `claim_and_resolve_issue` | +| `tools_pr_ops.py` | `get_pr_reviews`, `bulk_close_issues` | +| `tools_recipe.py` | `load_recipe`, `list_recipes`, `validate_recipe`, `migrate_recipe` | +| `tools_status.py` | `kitchen_status`, `get_pipeline_report`, `get_token_summary`, `get_timing_summary`, `analyze_tool_sequences`, `get_quota_events`, `write_telemetry_files`, `read_db` | +| `tools_workspace.py` | `test_check`, `reset_test_dir`, `reset_workspace` | + +## Architecture Notes + +Side-effect registration (same pattern as `recipe/rules/`). The `server/__init__.py` owns the `mcp` app object; tool modules import it from the server layer. All tools require `readOnlyHint: True` (see `server/CLAUDE.md`). diff --git a/src/autoskillit/server/tools/__init__.py b/src/autoskillit/server/tools/__init__.py new file mode 100644 index 0000000000..29c1c293f4 --- /dev/null +++ b/src/autoskillit/server/tools/__init__.py @@ -0,0 +1 @@ +"""Server tool modules — @mcp.tool() handlers registered on import.""" diff --git a/src/autoskillit/server/tools/_execution_helpers.py b/src/autoskillit/server/tools/_execution_helpers.py new file mode 100644 index 0000000000..a3332af69f --- /dev/null +++ b/src/autoskillit/server/tools/_execution_helpers.py @@ -0,0 +1,192 @@ +"""Subprocess coercion helpers for run_python.""" + +from __future__ import annotations + +import asyncio +import json +import types +import typing + +from autoskillit.core import get_logger + +logger = get_logger(__name__) + + +_RUN_PYTHON_SENTINEL_KEYS: frozenset[str] = frozenset({"callable", "timeout"}) + + +def _coerce_scalar(val: object, annotation: object) -> object: + """Coerce val to match the annotated type. + + Handles str, int, float, and Optional[T] / T | None variants. + Skips containers, unions, bool, and unconvertible values. + """ + if isinstance(val, bool): + return val + + actual = annotation + + # Unwrap X | None (types.UnionType — bare union syntax, Python 3.10+) + if isinstance(annotation, types.UnionType): + non_none = [a for a in annotation.__args__ if a is not type(None)] + if len(non_none) == 1: + actual = non_none[0] + # Unwrap Optional[X] / Union[X, None] (typing.Union with __origin__) + elif hasattr(annotation, "__origin__") and hasattr(annotation, "__args__"): + ann: typing.Any = annotation # type: ignore[name-defined] + origin = ann.__origin__ + args = ann.__args__ + if origin is typing.Union: + non_none = [a for a in args if a is not type(None)] + if len(non_none) == 1: + actual = non_none[0] + + # str ← int/float + if actual is str and not isinstance(val, str): + if isinstance(val, (int, float)): + return str(val) + return val + # int ← str (try/except for unconvertible) + if actual is int and not isinstance(val, int): + if isinstance(val, str): + try: + return int(val) + except ValueError: + return val + return val + # float ← str/int (try/except for unconvertible) + if actual is float and not isinstance(val, float): + if isinstance(val, (str, int)): + try: + return float(val) + except ValueError: + return val + return val + return val + + +async def _import_and_call( + dotted_path: str, + args: dict[str, object] | None = None, + timeout: float = 30, +) -> dict[str, object]: + """Import a Python callable by dotted path and invoke it. + + Returns dict with 'success', 'result' (or 'error'). + Handles sync and async callables, with timeout protection. + """ + import importlib + import inspect + + if args is None: + args = {} + + if "." not in dotted_path: + return {"success": False, "error": f"Invalid dotted path: {dotted_path!r}"} + + module_path, attr_name = dotted_path.rsplit(".", 1) + + try: + module = importlib.import_module(module_path) + except ImportError as exc: + return {"success": False, "error": f"Import failed for {module_path!r}: {exc}"} + + try: + func = getattr(module, attr_name) + except AttributeError: + return { + "success": False, + "error": f"Module {module_path!r} has no attribute {attr_name!r}", + } + + if not callable(func): + return {"success": False, "error": f"{dotted_path!r} is not callable"} + + sig = inspect.signature(func) + + for key in list(args.keys()): + if key in _RUN_PYTHON_SENTINEL_KEYS: + logger.warning( + "run_python stripped sentinel key from args", + callable=dotted_path, + arg_name=key, + ) + del args[key] + + accepts_var_keyword = any( + p.kind == inspect.Parameter.VAR_KEYWORD for p in sig.parameters.values() + ) + if not accepts_var_keyword: + valid_keys = set(sig.parameters.keys()) + for key in list(args.keys()): + if key not in valid_keys: + logger.warning( + "run_python dropped unrecognized arg", + callable=dotted_path, + arg_name=key, + extra_args=[key], + ) + del args[key] + + try: + type_hints = typing.get_type_hints(func) + except (NameError, TypeError, AttributeError): + logger.warning( + "get_type_hints failed, skipping coercion", callable=dotted_path, exc_info=True + ) + type_hints = {} + coerced: dict[str, object] = {} + for key, val in args.items(): + if val is None and key in sig.parameters: + param = sig.parameters[key] + if param.default is not inspect.Parameter.empty and param.default is not None: + logger.warning( + "run_python null-arg coerced to default", + callable=dotted_path, + arg=key, + default=repr(param.default), + ) + coerced[key] = param.default + continue + if val is not None and key in type_hints: + annotation = type_hints[key] + coerced_val = _coerce_scalar(val, annotation) + if coerced_val is not val: + logger.warning( + "run_python type coerced", + callable=dotted_path, + arg=key, + from_type=type(val).__name__, + to_type=type(coerced_val).__name__, + ) + coerced[key] = coerced_val + continue + coerced[key] = val + args = coerced + + try: + if inspect.iscoroutinefunction(func): + result = await asyncio.wait_for(func(**args), timeout=timeout) + else: + result = await asyncio.wait_for(asyncio.to_thread(func, **args), timeout=timeout) + except TimeoutError: + logger.warning( + "run_python timed out; sync thread may continue running", + dotted_path=dotted_path, + timeout=timeout, + ) + return {"success": False, "error": f"Timeout after {timeout}s calling {dotted_path}"} + except Exception as exc: + logger.warning( + "run_python execution failed", + dotted_path=dotted_path, + error=type(exc).__name__, + exc_info=True, + ) + return {"success": False, "error": f"{type(exc).__name__}: {exc}"} + + try: + json.dumps(result) + return {"success": True, "result": result} + except (TypeError, ValueError): + return {"success": True, "result": str(result)} diff --git a/src/autoskillit/server/tools/_types.py b/src/autoskillit/server/tools/_types.py new file mode 100644 index 0000000000..23e60c9e2e --- /dev/null +++ b/src/autoskillit/server/tools/_types.py @@ -0,0 +1,146 @@ +"""Server tool response TypedDicts — typed contracts for MCP tool JSON responses. + +These types define the shape of JSON payloads returned by server tool handlers. +They live here (IL-3) because their only structured consumers are test infrastructure +and server internals — not cross-layer protocols. +""" + +from __future__ import annotations + +from typing import Any, TypedDict + +from autoskillit.core import ModelTotalEntry + +__all__ = [ + "RunSkillResult", + "RunCmdResult", + "TestCheckResult", + "MergeWorktreeResult", + "TokenSummaryResult", + "TimingSummaryResult", + "KitchenStatusResult", +] + + +class _RunSkillResultBase(TypedDict): + """Required fields always present in every run_skill response.""" + + success: bool + exit_code: int + + +class RunSkillResult(_RunSkillResultBase, total=False): + """Typed return contract for run_skill — mirrors SkillResult.to_json() output keys.""" + + result: str + session_id: str + subtype: str + cli_subtype: str + is_error: bool + kill_reason: str + needs_retry: bool + retry_reason: str + stderr: str + token_usage: dict[str, Any] | None + write_path_warnings: list[str] + write_call_count: int + fs_writes_detected: bool + last_stop_reason: str + lifespan_started: bool + worktree_path: str + order_id: str + infra_exit_category: str + + +class _RunCmdResultBase(TypedDict): + """Required fields always present in every run_cmd response.""" + + success: bool + exit_code: int + + +class RunCmdResult(_RunCmdResultBase, total=False): + """Typed return contract for run_cmd.""" + + stdout: str + stderr: str + error: str + + +class _TestCheckResultBase(TypedDict): + """Required field always present in every test_check response.""" + + passed: bool + + +class TestCheckResult(_TestCheckResultBase, total=False): + """Typed return contract for test_check.""" + + stdout: str + stderr: str + duration_seconds: float + filter_mode: str + tests_selected: int + tests_deselected: int + full_run_reason: str + error: str + + +class MergeWorktreeResult(TypedDict, total=False): + """Typed return contract for merge_worktree — union of all success and error path keys.""" + + merge_succeeded: bool + merged_branch: str + into_branch: str + worktree_removed: bool + branch_deleted: bool + cleanup_succeeded: bool + error: str + failed_step: str + state: str + worktree_path: str + stderr: str + base_branch: str + dirty_files: list[str] + merge_commits: list[str] + test_stdout: str + test_stderr: str + abort_failed: bool + abort_stderr: str + poisoned_installs: list[str] + + +class TokenSummaryResult(TypedDict, total=False): + """Typed return contract for get_token_summary (JSON payload path).""" + + steps: list[dict[str, Any]] + total: dict[str, Any] + mcp_responses: dict[str, Any] + model_totals: list[ModelTotalEntry] + success: bool + error: str + + +class TimingSummaryResult(TypedDict, total=False): + """Typed return contract for get_timing_summary (JSON payload path).""" + + steps: list[dict[str, Any]] + total: dict[str, Any] + success: bool + error: str + + +class KitchenStatusResult(TypedDict, total=False): + """Typed return contract for kitchen_status.""" + + package_version: str + plugin_json_version: str + versions_match: bool + tools_enabled: bool + token_usage_verbosity: str + quota_guard_enabled: bool + github_token_configured: bool + github_default_repo: str + warning: str + success: bool + error: str diff --git a/src/autoskillit/server/tools/tools_ci.py b/src/autoskillit/server/tools/tools_ci.py new file mode 100644 index 0000000000..d3714aa9a5 --- /dev/null +++ b/src/autoskillit/server/tools/tools_ci.py @@ -0,0 +1,203 @@ +"""MCP tool handlers: set_commit_status (gated), check_repo_merge_state (gated).""" + +from __future__ import annotations + +import json +import time +from typing import Literal + +import httpx +from fastmcp import Context +from fastmcp.dependencies import CurrentContext + +from autoskillit.core import DirectInstall, MarketplaceInstall, get_logger +from autoskillit.server import mcp +from autoskillit.server._guards import _require_enabled +from autoskillit.server._misc import fetch_repo_merge_state, resolve_repo_from_remote +from autoskillit.server._notify import track_response_size +from autoskillit.server._subprocess import _run_subprocess + +logger = get_logger(__name__) + + +@mcp.tool(tags={"autoskillit", "kitchen", "github"}, annotations={"readOnlyHint": True}) +@track_response_size("set_commit_status") +async def set_commit_status( + sha: str, + state: Literal["pending", "success", "failure", "error"], + context: str, + description: str = "", + target_url: str = "", + repo: str = "", + cwd: str = "", +) -> str: + """Post a GitHub Commit Status to a commit SHA. + + Use to implement review-first gating: post `pending` when review starts, + then `success` or `failure` when it completes. Combine with a required + status check in branch protection to block merge until the review resolves. + + Args: + sha: Full commit SHA to attach the status to. + state: One of: pending, success, failure, error. + context: Status context label (e.g. "autoskillit/ai-review"). + description: Short human-readable status description (max 140 chars). + target_url: Optional URL linking to review details. + repo: owner/repo format. Inferred from `cwd` git remote if absent. + cwd: Working directory for repo inference. Defaults to plugin_dir. + + Never raises. + """ + if (gate := _require_enabled()) is not None: + return gate + + if not sha: + return json.dumps({"success": False, "error": "sha must not be empty"}) + if not context: + return json.dumps({"success": False, "error": "context must not be empty"}) + if len(description) > 140: + return json.dumps( + { + "success": False, + "error": f"description exceeds 140 chars ({len(description)} chars)", + } + ) + + try: + from autoskillit.server import _get_ctx + + tool_ctx = _get_ctx() + if cwd: + effective_cwd = cwd + else: + match tool_ctx.plugin_source: + case DirectInstall(plugin_dir=p): + effective_cwd = str(p) + case MarketplaceInstall(cache_path=cp): + effective_cwd = str(cp) + + # Resolve owner/repo if not provided + owner_repo = repo + if not owner_repo: + owner_repo = await resolve_repo_from_remote(effective_cwd) + if not owner_repo: + return json.dumps( + {"success": False, "error": "Could not infer owner/repo from git remote"} + ) + + cmd = [ + "gh", + "api", + "--method", + "POST", + f"/repos/{owner_repo}/statuses/{sha}", + "-f", + f"state={state}", + "-f", + f"context={context}", + "-f", + f"description={description}", + ] + if target_url: + cmd += ["-f", f"target_url={target_url}"] + + rc, _stdout, stderr = await _run_subprocess(cmd, cwd=effective_cwd, timeout=30.0) + if rc != 0: + return json.dumps({"success": False, "error": stderr}) + + return json.dumps({"success": True, "sha": sha, "state": state, "context": context}) + except Exception as exc: + logger.error("set_commit_status unhandled exception", exc_info=True) + return json.dumps({"success": False, "error": f"{type(exc).__name__}: {exc}"}) + + +@mcp.tool(tags={"autoskillit", "kitchen", "ci"}, annotations={"readOnlyHint": True}) +@track_response_size("check_repo_merge_state") +async def check_repo_merge_state( + branch: str, + cwd: str = ".", + remote_url: str = "", + step_name: str = "", + ctx: Context = CurrentContext(), +) -> str: + """Single GraphQL round-trip returning queue_available, merge_group_trigger, + auto_merge_available, and ci_event for the given repository branch. + + Consolidates the three former run_cmd shell steps in the pre_queue_gate block + into a single MCP tool call, eliminating N+2 REST/GraphQL round-trips and + making the call budget auditable by block-level semantic rules. + + Args: + branch: Target branch name to check merge queue state for. + cwd: Working directory for git remote resolution. + remote_url: Full GitHub remote URL; parsed to owner/repo if provided. + step_name: Step name for timing telemetry. + + Returns a JSON object with keys: + - ``queue_available``: branch has an active GitHub merge queue. + - ``merge_group_trigger``: a CI workflow declares the merge_group event. + - ``auto_merge_available``: repository has auto-merge enabled. + - ``ci_event``: ``"push"`` | ``"merge_group"`` | ``null`` — recommended + event to use when polling CI via wait_for_ci. + On any error, returns an error field alongside the four boolean/null defaults. + + Never raises. + """ + if (gate := _require_enabled()) is not None: + return gate + try: + from autoskillit.server import _get_ctx + + tool_ctx = _get_ctx() + _start = time.monotonic() + try: + resolved_repo = await resolve_repo_from_remote(cwd, hint=remote_url or None) + if not resolved_repo or "/" not in resolved_repo: + return json.dumps( + { + "error": f"Could not resolve owner/repo from cwd={cwd!r}", + "queue_available": False, + "merge_group_trigger": False, + "auto_merge_available": False, + "ci_event": None, + } + ) + owner, repo = resolved_repo.split("/", 1) + resolved_token = ( + tool_ctx.token_factory() + if tool_ctx.token_factory is not None + else tool_ctx.config.github.token + ) + state = await fetch_repo_merge_state( + owner=owner, + repo=repo, + branch=branch, + token=resolved_token, + ) + return json.dumps(state) + except Exception as exc: + logger.error("autoskillit.check_repo_merge_state failed", exc_info=True) + envelope: dict[str, object] = { + "error": f"{type(exc).__name__}: {exc}", + "queue_available": False, + "merge_group_trigger": False, + "auto_merge_available": False, + "ci_event": None, + } + if isinstance(exc, httpx.HTTPStatusError): + envelope["http_status"] = exc.response.status_code + return json.dumps(envelope) + finally: + if step_name: + tool_ctx.timing_log.record(step_name, time.monotonic() - _start) + except Exception as exc: + logger.error("check_repo_merge_state unhandled exception", exc_info=True) + return json.dumps( + { + "error": f"{type(exc).__name__}: {exc}", + "queue_available": False, + "merge_group_trigger": False, + "auto_merge_available": False, + "ci_event": None, + } + ) diff --git a/src/autoskillit/server/tools/tools_ci_merge_queue.py b/src/autoskillit/server/tools/tools_ci_merge_queue.py new file mode 100644 index 0000000000..5c81a9efaa --- /dev/null +++ b/src/autoskillit/server/tools/tools_ci_merge_queue.py @@ -0,0 +1,296 @@ +"""Merge queue MCP tools: toggle_auto_merge, enqueue_pr, wait_for_merge_queue.""" + +from __future__ import annotations + +import json +import time + +import structlog +from fastmcp import Context +from fastmcp.dependencies import CurrentContext + +from autoskillit.core import get_logger +from autoskillit.server import mcp +from autoskillit.server._guards import _require_enabled +from autoskillit.server._misc import resolve_repo_from_remote +from autoskillit.server._notify import _notify, track_response_size + +logger = get_logger(__name__) + + +@mcp.tool(tags={"autoskillit", "kitchen", "ci"}, annotations={"readOnlyHint": True}) +@track_response_size("toggle_auto_merge") +async def toggle_auto_merge( + pr_number: int, + target_branch: str, + cwd: str, + repo: str = "", + remote_url: str = "", + ctx: Context = CurrentContext(), +) -> str: + """Disable then re-enable auto-merge for a PR to re-enroll it in the merge queue. + + Uses the same GraphQL mutation path as wait_for_merge_queue's stall recovery. + Call this when wait_for_merge_queue returns pr_state="stalled" and you want + to attempt one additional re-enrollment cycle. + + Args: + pr_number: PR number to re-enroll. + target_branch: Branch the merge queue targets (e.g. "develop"). + cwd: Working directory for git remote resolution when repo is not provided. + repo: Optional "owner/name" string. Inferred from git remote if empty. + remote_url: Full GitHub remote URL. Parsed to owner/repo before inference. + + Returns: + JSON: {"success": bool, "pr_number": int} on success, + {"success": false, "error": str} on failure. + + Never raises. + """ + if (gate := _require_enabled()) is not None: + return gate + try: + structlog.contextvars.clear_contextvars() + structlog.contextvars.bind_contextvars( + tool="toggle_auto_merge", pr_number=pr_number, target_branch=target_branch + ) + await _notify( + ctx, + "info", + f"Toggling auto-merge for PR #{pr_number} on {target_branch!r}", + "autoskillit.toggle_auto_merge", + extra={"pr_number": pr_number, "target_branch": target_branch}, + ) + + from autoskillit.server import _get_ctx + + tool_ctx = _get_ctx() + + if tool_ctx.merge_queue_watcher is None: + return json.dumps( + { + "success": False, + "error": "merge_queue_watcher not configured (missing GITHUB_TOKEN?)", + } + ) + + resolved_repo = await resolve_repo_from_remote(cwd, hint=remote_url or repo or None) + + result = await tool_ctx.merge_queue_watcher.toggle( + pr_number=pr_number, + target_branch=target_branch, + repo=resolved_repo or None, + cwd=cwd, + ) + return json.dumps(result) + except Exception as exc: + logger.error("toggle_auto_merge unhandled exception", exc_info=True) + return json.dumps({"success": False, "error": f"{type(exc).__name__}: {exc}"}) + + +@mcp.tool(tags={"autoskillit", "kitchen", "ci"}, annotations={"readOnlyHint": True}) +@track_response_size("enqueue_pr") +async def enqueue_pr( + pr_number: int, + target_branch: str, + cwd: str, + auto_merge_available: bool, + repo: str = "", + remote_url: str = "", + step_name: str = "", + ctx: Context = CurrentContext(), +) -> str: + """Enqueue a PR into the merge queue using the correct enrollment strategy. + + Uses enablePullRequestAutoMerge when auto_merge_available=true, + enqueuePullRequest GraphQL mutation when auto_merge_available=false. + + Args: + pr_number: PR number to enqueue. + target_branch: Branch the merge queue targets (e.g. "develop"). + cwd: Working directory for git remote resolution when repo is not provided. + auto_merge_available: Whether the repository allows auto-merge. + repo: Optional "owner/name" string. Inferred from git remote if empty. + remote_url: Full GitHub remote URL. Parsed to owner/repo before inference. + step_name: Optional YAML step key for wall-clock timing accumulation. + + Returns: + JSON: {"success": bool, "pr_number": int, "enrollment_method": str} on success, + {"success": false, "error": str} on failure. + + Never raises. + """ + if (gate := _require_enabled()) is not None: + return gate + try: + structlog.contextvars.clear_contextvars() + structlog.contextvars.bind_contextvars( + tool="enqueue_pr", pr_number=pr_number, target_branch=target_branch + ) + await _notify( + ctx, + "info", + f"Enrolling PR #{pr_number} in merge queue on {target_branch!r}", + "autoskillit.enqueue_pr", + extra={"pr_number": pr_number, "target_branch": target_branch}, + ) + + from autoskillit.server import _get_ctx + + tool_ctx = _get_ctx() + + if tool_ctx.merge_queue_watcher is None: + return json.dumps( + { + "success": False, + "error": "merge_queue_watcher not configured (missing GITHUB_TOKEN?)", + } + ) + + resolved_repo = await resolve_repo_from_remote(cwd, hint=remote_url or repo or None) + + _start = time.monotonic() + try: + result = await tool_ctx.merge_queue_watcher.enqueue( + pr_number=pr_number, + target_branch=target_branch, + repo=resolved_repo or None, + cwd=cwd, + auto_merge_available=auto_merge_available, + ) + return json.dumps(result) + except Exception as exc: + logger.error("enqueue_pr watcher error", exc_info=True) + return json.dumps({"success": False, "error": f"{type(exc).__name__}: {exc}"}) + finally: + if step_name: + tool_ctx.timing_log.record(step_name, time.monotonic() - _start) + except Exception as exc: + logger.error("enqueue_pr unhandled exception", exc_info=True) + return json.dumps({"success": False, "error": f"{type(exc).__name__}: {exc}"}) + + +@mcp.tool(tags={"autoskillit", "kitchen", "ci"}, annotations={"readOnlyHint": True}) +@track_response_size("wait_for_merge_queue") +async def wait_for_merge_queue( + pr_number: int, + target_branch: str, + cwd: str, + repo: str = "", + remote_url: str = "", + timeout_seconds: int = 600, + poll_interval: int = 15, + stall_grace_period: int = 60, + max_stall_retries: int = 3, + not_in_queue_confirmation_cycles: int = 2, + max_inconclusive_retries: int = 5, + auto_merge_available: bool = True, + step_name: str = "", + ctx: Context = CurrentContext(), +) -> str: + """Poll a PR's progress through GitHub's merge queue until merged, ejected, or timed out. + + Args: + pr_number: PR number to monitor. + target_branch: Branch the merge queue targets (e.g. "develop"). + cwd: Working directory for git remote resolution when repo is not provided. + repo: Optional "owner/name" string. Inferred from git remote if empty. + remote_url: Full GitHub remote URL (e.g. "https://github.com/owner/repo.git"). + Parsed to owner/repo before inference. Takes priority over repo + when both are provided. + timeout_seconds: Total polling budget (default 600s). + poll_interval: Seconds between polls (default 15s). + stall_grace_period: Seconds after auto-merge is enabled before stall recovery + may trigger. Prevents intervention during normal queue processing + (default 60s). + max_stall_retries: Maximum disable/re-enable toggle attempts before declaring + the PR stalled and returning pr_state="stalled" (default 3). + not_in_queue_confirmation_cycles: Consecutive "not in queue" cycles required + before treating absence as definitive. Guards against race between + queue exit and merged=true propagation (default 2). + max_inconclusive_retries: Maximum NoPositiveSignal cycles (beyond the + confirmation window) before returning pr_state="timeout" (default 5). + auto_merge_available: Whether the repository allows auto-merge. When False, + stall recovery uses enqueuePullRequest instead of toggle (default True). + step_name: Optional YAML step key for wall-clock timing accumulation. + + Returns: + JSON: { + "success": bool, + "pr_state": "merged"|"ejected"|"ejected_ci_failure"|"stalled"| + "dropped_healthy"|"dropped_merge_group_ci"| + "not_enrolled"|"timeout"|"error", + "reason": str, + "stall_retries_attempted": int, + } + + pr_state values: + merged — PR successfully merged through the queue. + ejected — PR removed from queue (conflict or other non-CI reason). + ejected_ci_failure — PR removed from queue because CI checks failed. + stalled — PR stuck in queue; max stall retries exhausted. + dropped_healthy — auto-merge disabled on a PR with no CI/conflict issues. + dropped_merge_group_ci — PR ejected due to merge-group CI failure (PR-branch CI clean). + not_enrolled — PR was never enrolled in the merge queue. + timeout — polling budget exhausted before a terminal state was reached. + error — watcher raised an unexpected exception. + + Never raises. + """ + if (gate := _require_enabled()) is not None: + return gate + try: + structlog.contextvars.clear_contextvars() + structlog.contextvars.bind_contextvars( + tool="wait_for_merge_queue", pr_number=pr_number, target_branch=target_branch + ) + await _notify( + ctx, + "info", + f"Waiting for PR #{pr_number} in merge queue on {target_branch!r}", + "autoskillit.wait_for_merge_queue", + extra={"pr_number": pr_number, "target_branch": target_branch}, + ) + + from autoskillit.server import _get_ctx + + tool_ctx = _get_ctx() + + if tool_ctx.merge_queue_watcher is None: + if step_name: + tool_ctx.timing_log.record(step_name, 0.0) + return json.dumps( + { + "success": False, + "pr_state": "error", + "reason": "merge_queue_watcher not configured (missing GITHUB_TOKEN?)", + } + ) + + resolved_repo = await resolve_repo_from_remote(cwd, hint=remote_url or repo or None) + + _start = time.monotonic() + try: + result = await tool_ctx.merge_queue_watcher.wait( + pr_number=pr_number, + target_branch=target_branch, + repo=resolved_repo or None, + cwd=cwd, + timeout_seconds=timeout_seconds, + poll_interval=poll_interval, + stall_grace_period=stall_grace_period, + max_stall_retries=max_stall_retries, + not_in_queue_confirmation_cycles=not_in_queue_confirmation_cycles, + max_inconclusive_retries=max_inconclusive_retries, + auto_merge_available=auto_merge_available, + ) + return json.dumps(result) + except Exception as exc: + logger.error("wait_for_merge_queue ci_watcher error", exc_info=True) + return json.dumps({"success": False, "error": f"{type(exc).__name__}: {exc}"}) + finally: + if step_name: + tool_ctx.timing_log.record(step_name, time.monotonic() - _start) + except Exception as exc: + logger.error("wait_for_merge_queue unhandled exception", exc_info=True) + return json.dumps({"success": False, "error": f"{type(exc).__name__}: {exc}"}) diff --git a/src/autoskillit/server/tools/tools_ci_watch.py b/src/autoskillit/server/tools/tools_ci_watch.py new file mode 100644 index 0000000000..4825b65e14 --- /dev/null +++ b/src/autoskillit/server/tools/tools_ci_watch.py @@ -0,0 +1,363 @@ +"""CI watching MCP tools: wait_for_ci and get_ci_status.""" + +from __future__ import annotations + +import asyncio +import json +import time +from typing import Any, Literal + +import structlog +from fastmcp import Context +from fastmcp.dependencies import CurrentContext + +from autoskillit.core import KNOWN_CI_EVENTS, CIRunScope, get_logger +from autoskillit.pipeline import ToolContext +from autoskillit.server import mcp +from autoskillit.server._guards import _require_enabled +from autoskillit.server._misc import resolve_repo_from_remote +from autoskillit.server._notify import _notify, track_response_size +from autoskillit.server._subprocess import _run_subprocess + +logger = get_logger(__name__) + + +def _coerce_none_string(value: str | None, *, tool: str) -> str | None: + """Coerce the string literal 'None' to Python None, logging a warning when coercion occurs.""" + if value == "None": + logger.warning("event coerced from string 'None' to null", tool=tool) + return None + return value + + +@mcp.tool(tags={"autoskillit", "kitchen", "ci"}, annotations={"readOnlyHint": True}) +@track_response_size("wait_for_ci") +async def wait_for_ci( + branch: str, + repo: str | None = None, + remote_url: str = "", + head_sha: str | None = None, + workflow: str | None = None, + event: str | None = None, + timeout_seconds: int = 300, + lookback_seconds: int = 3600, + cwd: str = ".", + step_name: str = "", + auto_trigger: bool = False, + ctx: Context = CurrentContext(), +) -> str: + """Wait for a GitHub Actions CI run to complete on the given branch. + + Uses a three-phase algorithm (look-back, poll, wait) that eliminates + the race condition where CI completes before polling starts. + + Args: + branch: Git branch name to watch CI for. + repo: GitHub owner/repo (e.g. "owner/repo"). If omitted, inferred + from git remote in cwd. + remote_url: Full GitHub remote URL (e.g. "https://github.com/owner/repo.git"). + Parsed to owner/repo before inference. Takes priority over repo + when both are provided. + head_sha: Specific commit SHA to match. If omitted, inferred from + HEAD in cwd. + workflow: Workflow filename to filter runs (e.g. "tests.yml"). If + omitted, falls back to the project-level ci.workflow config. + event: GitHub trigger event to filter runs (e.g. "push", "pull_request"). + If omitted, falls back to the project-level ci.event config. + timeout_seconds: Maximum time to wait (default 300s). + cwd: Working directory for git operations. + step_name: Optional YAML step key for wall-clock timing accumulation. + auto_trigger: When True and ci_watcher returns "no_runs", performs an active + self-healing sequence: checks PR mergeability, creates an empty + commit, and force-pushes the branch to re-trigger webhook delivery, + then re-polls CI with a fresh timeout. Result includes + "triggered": true when the sequence fires. Default False. + + Returns: + JSON with run_id, conclusion ("success", "failure", "cancelled", + "action_required", "timed_out", "no_runs", "error", "unknown"), + and failed_jobs list. Billing limit errors surface as + conclusion="action_required" with failed_jobs=[]. + + Never raises. + """ + if (gate := _require_enabled()) is not None: + return gate + try: + _start = time.monotonic() + _timing_ctx = None + event = _coerce_none_string(event, tool="wait_for_ci") + if event is not None and event not in KNOWN_CI_EVENTS: + return json.dumps( + { + "run_id": None, + "conclusion": "error", + "failed_jobs": [], + "error": f"Invalid event {event!r}. Valid events: {sorted(KNOWN_CI_EVENTS)}", + } + ) + structlog.contextvars.clear_contextvars() + structlog.contextvars.bind_contextvars(tool="wait_for_ci") + logger.info("wait_for_ci", branch=branch, repo=repo or "(infer)") + + from autoskillit.server import _get_ctx + + tool_ctx = _get_ctx() + _timing_ctx = tool_ctx + if tool_ctx.ci_watcher is None: + if step_name: + tool_ctx.timing_log.record(step_name, time.monotonic() - _start) + return json.dumps( + { + "run_id": None, + "conclusion": "error", + "failed_jobs": [], + "error": "CI watcher not configured", + } + ) + + # Infer head_sha from cwd if not provided + if head_sha is None and cwd: + try: + rc, stdout, _ = await _run_subprocess( + ["git", "rev-parse", "HEAD"], cwd=cwd, timeout=5.0 + ) + if rc == 0: + head_sha = stdout.strip() + except Exception: + logger.warning("git rev-parse HEAD failed", exc_info=True) + + scope = CIRunScope( + workflow=workflow or tool_ctx.default_ci_scope.workflow, + head_sha=head_sha, + event=event or tool_ctx.default_ci_scope.event, + ) + + resolved_repo = await resolve_repo_from_remote(cwd, hint=remote_url or repo or None) + + await _notify( + ctx, + "info", + f"Watching CI for branch {branch}", + "autoskillit.wait_for_ci", + extra={ + "repo": resolved_repo or "(infer)", + "head_sha": scope.head_sha or "(any)", + "workflow": scope.workflow or "(any)", + }, + ) + + try: + result = await tool_ctx.ci_watcher.wait( + branch, + repo=resolved_repo or None, + scope=scope, + timeout_seconds=timeout_seconds, + lookback_seconds=lookback_seconds, + cwd=cwd, + ) + + # Include head_sha used for this CI check so orchestrators can verify + # CI results correspond to the current HEAD after a force-push. + if scope.head_sha: + result = {**result, "head_sha": scope.head_sha} + + if auto_trigger and result.get("conclusion") == "no_runs": + result = await _auto_trigger_ci( + branch=branch, + cwd=cwd, + result=result, + scope=scope, + resolved_repo=resolved_repo, + tool_ctx=tool_ctx, + timeout_seconds=timeout_seconds, + lookback_seconds=lookback_seconds, + ) + + conclusion = result.get("conclusion", "unknown") + level: Literal["info", "error"] = "info" if conclusion == "success" else "error" + await _notify( + ctx, + level, + f"CI result: {conclusion}", + "autoskillit.wait_for_ci", + extra={"run_id": result.get("run_id")}, + ) + + return json.dumps(result) + except Exception as exc: + logger.error("wait_for_ci failed", exc_info=True) + return json.dumps( + { + "run_id": None, + "conclusion": "error", + "failed_jobs": [], + "error": f"{type(exc).__name__}: {exc}", + } + ) + finally: + if step_name: + tool_ctx.timing_log.record(step_name, time.monotonic() - _start) + except Exception as exc: + logger.error("wait_for_ci unhandled exception", exc_info=True) + if step_name and _timing_ctx is not None: + _timing_ctx.timing_log.record(step_name, time.monotonic() - _start) + return json.dumps( + { + "run_id": None, + "conclusion": "error", + "failed_jobs": [], + "error": f"{type(exc).__name__}: {exc}", + } + ) + + +@mcp.tool(tags={"autoskillit", "kitchen", "ci"}, annotations={"readOnlyHint": True}) +@track_response_size("get_ci_status") +async def get_ci_status( + branch: str | None = None, + run_id: int | None = None, + repo: str | None = None, + workflow: str | None = None, + event: str | None = None, + cwd: str = ".", +) -> str: + """Return current CI status for a branch or specific run without waiting. + + Args: + branch: Git branch name. Required if run_id is not provided. + run_id: Specific run ID to check. If provided, branch is ignored. + repo: GitHub owner/repo. If omitted, inferred from git remote in cwd. + workflow: Workflow filename to filter runs (e.g. "tests.yml"). If + omitted, falls back to the project-level ci.workflow config. + event: GitHub trigger event to filter runs (e.g. "push", "pull_request"). + If omitted, falls back to the project-level ci.event config. + cwd: Working directory for git operations. + + Returns: + JSON with runs list, each containing id, status, conclusion, failed_jobs. + + Never raises. + """ + if (gate := _require_enabled()) is not None: + return gate + try: + from autoskillit.server import _get_ctx + + tool_ctx = _get_ctx() + if tool_ctx.ci_watcher is None: + return json.dumps({"runs": [], "error": "CI watcher not configured"}) + + if branch is None and run_id is None: + return json.dumps({"runs": [], "error": "Provide branch or run_id"}) + + scope = CIRunScope( + workflow=workflow or tool_ctx.default_ci_scope.workflow, + event=event or tool_ctx.default_ci_scope.event, + ) + + result = await tool_ctx.ci_watcher.status( + branch or "", + repo=repo, + run_id=run_id, + scope=scope, + cwd=cwd, + ) + return json.dumps(result) + except Exception as exc: + logger.error("get_ci_status unhandled exception", exc_info=True) + return json.dumps({"runs": [], "error": f"{type(exc).__name__}: {exc}"}) + + +async def _auto_trigger_ci( + *, + branch: str, + cwd: str, + result: dict[str, Any], + scope: CIRunScope, + resolved_repo: str | None, + tool_ctx: ToolContext, + timeout_seconds: int, + lookback_seconds: int = 3600, +) -> dict[str, Any]: + """Active CI trigger recovery: empty commit + force-push + re-poll. + + Called when wait_for_ci returns no_runs and auto_trigger=True. + Returns result dict augmented with "triggered" key. + On any failure (merge conflict, push rejected, etc.) returns original + no_runs result so the recipe routes to handle_no_ci_runs as fallback. + """ + rc_m, out_m, _ = await _run_subprocess( + ["gh", "pr", "view", branch, "--json", "mergeable"], + cwd=cwd, + timeout=15.0, + ) + if rc_m != 0: + logger.warning("auto_trigger: gh pr view failed, cannot check mergeability", rc=rc_m) + return {**result, "conclusion": "gh_view_failed", "triggered": False} + try: + mergeable = json.loads(out_m).get("mergeable", "UNKNOWN") + except json.JSONDecodeError: + logger.warning("auto_trigger: failed to parse gh pr view JSON", exc_info=True) + mergeable = "UNKNOWN" + if mergeable == "CONFLICTING": + return {**result, "conclusion": "merge_conflict", "triggered": False} + + rc_c, _, err_c = await _run_subprocess( + ["git", "commit", "--allow-empty", "-m", "ci: trigger"], + cwd=cwd, + timeout=30.0, + ) + if rc_c != 0: + logger.warning("auto_trigger: empty commit failed", stderr=err_c) + return result + + rc_sha, sha_out, _ = await _run_subprocess(["git", "rev-parse", "HEAD"], cwd=cwd, timeout=5.0) + new_head_sha = (sha_out.strip() or None) if rc_sha == 0 else None + + remote_name = "origin" + for _candidate in ("upstream", "origin"): + rc_r, url_r, _ = await _run_subprocess( + ["git", "remote", "get-url", _candidate], cwd=cwd, timeout=5.0 + ) + if rc_r == 0 and not url_r.strip().startswith("file://"): + remote_name = _candidate + break + rc_p, _, err_p = await _run_subprocess( + ["git", "push", "--force-with-lease", remote_name, branch], + cwd=cwd, + timeout=60.0, + ) + if rc_p != 0: + logger.warning("auto_trigger: push failed", stderr=err_p) + rc_reset, _, _ = await _run_subprocess( + ["git", "reset", "--soft", "HEAD~1"], cwd=cwd, timeout=10.0 + ) + if rc_reset != 0: + logger.warning("auto_trigger: cleanup reset failed; branch may be diverged") + return result + + new_scope = CIRunScope( + workflow=scope.workflow, + head_sha=new_head_sha, + event=scope.event, + ) + if tool_ctx.ci_watcher is None: + raise RuntimeError("auto_trigger: ci_watcher not configured on tool_ctx") + try: + triggered_result = await tool_ctx.ci_watcher.wait( + branch, + repo=resolved_repo, + scope=new_scope, + timeout_seconds=timeout_seconds, + lookback_seconds=lookback_seconds, + cwd=cwd, + ) + if new_head_sha: + triggered_result = {**triggered_result, "head_sha": new_head_sha} + return {**triggered_result, "triggered": True} + except asyncio.CancelledError: + raise + except Exception: + logger.error("auto_trigger: second CI poll failed", exc_info=True) + return {**result, "conclusion": "auto_trigger_failed", "triggered": False} diff --git a/src/autoskillit/server/tools_clone.py b/src/autoskillit/server/tools/tools_clone.py similarity index 71% rename from src/autoskillit/server/tools_clone.py rename to src/autoskillit/server/tools/tools_clone.py index ae1701999e..f75756752c 100644 --- a/src/autoskillit/server/tools_clone.py +++ b/src/autoskillit/server/tools/tools_clone.py @@ -1,10 +1,11 @@ """MCP tool handlers: clone_repo, remove_clone, push_to_remote, -register_clone_status, batch_cleanup_clones.""" +register_clone_status, batch_cleanup_clones, bootstrap_clone.""" from __future__ import annotations import asyncio import json +import os import time from typing import Literal @@ -12,14 +13,12 @@ from fastmcp import Context from fastmcp.dependencies import CurrentContext -from autoskillit.core import get_logger +from autoskillit.core import CAMPAIGN_ID_ENV_VAR, get_logger from autoskillit.server import mcp -from autoskillit.server.helpers import ( - _notify, - _require_enabled, - clone_registry, - track_response_size, -) +from autoskillit.server._guards import _require_enabled +from autoskillit.server._misc import clone_registry +from autoskillit.server._notify import _notify, track_response_size +from autoskillit.server._subprocess import _run_subprocess logger = get_logger(__name__) @@ -167,7 +166,7 @@ async def remove_clone( if step_name: tool_ctx.timing_log.record(step_name, time.monotonic() - _start) except Exception as exc: - logger.error("remove_clone unhandled exception", exc_info=exc) + logger.error("remove_clone unhandled exception", exc_info=True) return json.dumps({"removed": "false", "reason": f"unexpected error: {exc}"}) @@ -198,7 +197,7 @@ async def push_to_remote( Args: clone_path: Absolute path to the clone directory to push from. - branch: Branch name to push (e.g. "integration"). + branch: Branch name to push (e.g. "develop"). source_dir: Source repo path (read-only URL lookup when remote_url is empty). remote_url: Pre-resolved upstream remote URL. When provided, source_dir is skipped. step_name: Optional YAML step key for wall-clock timing accumulation. @@ -281,7 +280,7 @@ async def push_to_remote( ) -@mcp.tool(tags={"autoskillit", "kitchen", "clone"}, annotations={"readOnlyHint": False}) +@mcp.tool(tags={"autoskillit", "kitchen", "clone"}, annotations={"readOnlyHint": True}) @track_response_size("register_clone_status") async def register_clone_status( clone_path: str, @@ -290,7 +289,7 @@ async def register_clone_status( step_name: str = "", ctx: Context = CurrentContext(), ) -> str: - """Register a clone path as 'success' or 'error' in the shared cleanup registry. + """Register a clone path in the shared cleanup registry. Called per-pipeline when defer_cleanup=true. Safe for parallel callers — clone_registry uses atomic writes. @@ -300,7 +299,9 @@ async def register_clone_status( Args: clone_path: Absolute path to the clone directory to register. - status: Completion status — "success" (safe to delete) or "error" (preserve). + status: Completion status — "success" (safe to delete), "error" (preserve for + investigation), or "unconfirmed" (preserve; merge was attempted but not + yet confirmed — in-progress label is retained on the issue). registry_path: Absolute path to the shared registry file. Defaults to .autoskillit/temp/clone-cleanup-registry.json in cwd. step_name: Optional YAML step key for wall-clock timing accumulation. @@ -314,18 +315,20 @@ async def register_clone_status( structlog.contextvars.bind_contextvars(tool="register_clone_status", clone_path=clone_path) logger.info("register_clone_status", clone_path=clone_path, status=status) - if status not in ("success", "error"): + if status not in ("success", "error", "unconfirmed"): return json.dumps( { "registered": "false", - "reason": f"Invalid status '{status}'. Must be 'success' or 'error'.", + "reason": ( + f"Invalid status '{status}'. Must be 'success', 'error', or 'unconfirmed'." + ), } ) from autoskillit.server import _get_ctx tool_ctx = _get_ctx() - owner = tool_ctx.kitchen_id + owner = os.environ.get(CAMPAIGN_ID_ENV_VAR, "") or tool_ctx.kitchen_id if owner == "": return json.dumps( { @@ -334,7 +337,13 @@ async def register_clone_status( } ) _start = time.monotonic() - typed_status: Literal["success", "error"] = "success" if status == "success" else "error" + typed_status: Literal["success", "error", "unconfirmed"] = ( + "success" + if status == "success" + else "unconfirmed" + if status == "unconfirmed" + else "error" + ) try: result = await asyncio.to_thread( clone_registry.register_clone, @@ -356,11 +365,12 @@ async def register_clone_status( return json.dumps({"registered": "false", "reason": f"{type(exc).__name__}: {exc}"}) -@mcp.tool(tags={"autoskillit", "kitchen", "clone"}, annotations={"readOnlyHint": False}) +@mcp.tool(tags={"autoskillit", "clone", "fleet"}, annotations={"readOnlyHint": True}) @track_response_size("batch_cleanup_clones") async def batch_cleanup_clones( registry_path: str = "", all_owners: str = "false", + owner_filter: str = "", step_name: str = "", ctx: Context = CurrentContext(), ) -> str: @@ -381,6 +391,8 @@ async def batch_cleanup_clones( all_owners: "false" (default) scopes deletion to the current kitchen's entries; "true" is the operator escape hatch that ignores owner and deletes all success-status entries including legacy orphans. + owner_filter: Explicit owner string for campaign-scoped cleanup. When non-empty, + overrides kitchen_id scoping. Ignored when all_owners="true". step_name: Optional YAML step key for wall-clock timing accumulation. """ try: @@ -404,19 +416,21 @@ async def batch_cleanup_clones( ) is_escape_hatch = all_owners.strip().lower() == "true" - owner_filter: str | None + owner_filter_resolved: str | None if is_escape_hatch: - owner_filter = None + owner_filter_resolved = None + elif owner_filter: + owner_filter_resolved = owner_filter else: - owner_filter = tool_ctx.kitchen_id - if owner_filter == "": + owner_filter_resolved = tool_ctx.kitchen_id + if owner_filter_resolved == "": return json.dumps( { "deleted": [], "delete_failures": [], "preserved": [], "error": "no active kitchen (kitchen_id empty) — " - "pass all_owners='true' to force-cleanup legacy entries", + "pass all_owners='true' or owner_filter to force-cleanup", } ) @@ -427,7 +441,7 @@ async def batch_cleanup_clones( registry_path, tool_ctx.clone_mgr.remove_clone, tool_ctx.temp_dir, - owner_filter, + owner_filter_resolved, ) return json.dumps(result) finally: @@ -436,3 +450,129 @@ async def batch_cleanup_clones( except Exception as exc: logger.warning("batch_cleanup_clones failed", exc_info=True) return json.dumps({"deleted": [], "preserved": [], "error": str(exc)}) + + +@mcp.tool(tags={"autoskillit", "kitchen", "clone"}, annotations={"readOnlyHint": True}) +@track_response_size("bootstrap_clone") +async def bootstrap_clone( + source_dir: str, + run_name: str, + base_branch: str, + branch: str = "", + strategy: str = "", + remote_url: str = "", + step_name: str = "", + ctx: Context = CurrentContext(), +) -> str: + """Clone a source repository and capture the base SHA in one orchestrator turn. + + Combines clone_repo + git rev-parse into a single call, reducing bootstrap + corridor turns from 2 to 1. + + Returns {"work_dir": str, "remote_url": str, "base_sha": str, + "merge_target": str, "timings": {"clone_ms": int, "rev_parse_ms": int}} + on success. Returns {"error": str} on clone failure or rev-parse failure. + + Args: + source_dir: Path to the source repository. Empty string auto-detects via git. + run_name: Name prefix for the clone directory (e.g. "impl", "audit-fix"). + base_branch: Branch to capture the SHA for (e.g. "main", "develop"). + branch: Branch to check out in the clone. Empty = auto-detect from HEAD. + strategy: On uncommitted changes: "" = return warning, "proceed" = clone + remote committed state, "clone_local" = copytree. + remote_url: Override remote URL for clone's origin. + step_name: Optional YAML step key for wall-clock timing accumulation. + + Never raises. + """ + if (gate := _require_enabled()) is not None: + return gate + try: + structlog.contextvars.clear_contextvars() + structlog.contextvars.bind_contextvars( + tool="bootstrap_clone", source_dir=source_dir, base_branch=base_branch + ) + logger.info( + "bootstrap_clone", source_dir=source_dir, run_name=run_name, base_branch=base_branch + ) + await _notify( + ctx, + "info", + f"bootstrap_clone: {source_dir!r} run_name={run_name!r} base_branch={base_branch!r}", + "autoskillit.bootstrap_clone", + extra={ + "source_dir": source_dir, + "run_name": run_name, + "base_branch": base_branch, + "branch": branch, + }, + ) + + from autoskillit.server import _get_ctx + + tool_ctx = _get_ctx() + if tool_ctx.clone_mgr is None: + return json.dumps({"error": "Clone manager not configured"}) + + _total_start = time.monotonic() + + _clone_start = time.monotonic() + try: + clone_result = await asyncio.to_thread( + tool_ctx.clone_mgr.clone_repo, source_dir, run_name, branch, strategy, remote_url + ) + except (ValueError, RuntimeError) as exc: + await _notify( + ctx, + "error", + "bootstrap_clone: clone failed", + "autoskillit.bootstrap_clone", + extra={"reason": str(exc)}, + ) + return json.dumps({"error": str(exc)}) + finally: + clone_ms = int((time.monotonic() - _clone_start) * 1000) + + clone_path: str = str(clone_result.get("clone_path", "")) + resolved_remote_url: str = str(clone_result.get("remote_url", remote_url)) + + _revparse_start = time.monotonic() + try: + rc, stdout, stderr = await _run_subprocess( + ["git", "rev-parse", base_branch], + cwd=clone_path, + timeout=30, + ) + finally: + rev_parse_ms = int((time.monotonic() - _revparse_start) * 1000) + + if rc != 0: + await _notify( + ctx, + "error", + "bootstrap_clone: rev-parse failed", + "autoskillit.bootstrap_clone", + extra={"stderr": stderr}, + ) + return json.dumps({"error": f"rev-parse failed: {stderr.strip()}"}) + + base_sha = stdout.strip() + + if step_name: + tool_ctx.timing_log.record(step_name, time.monotonic() - _total_start) + + return json.dumps( + { + "work_dir": clone_path, + "remote_url": resolved_remote_url, + "base_sha": base_sha, + "merge_target": base_branch, + "timings": { + "clone_ms": clone_ms, + "rev_parse_ms": rev_parse_ms, + }, + } + ) + except Exception as exc: + logger.error("bootstrap_clone unhandled exception", exc_info=True) + return json.dumps({"error": f"{type(exc).__name__}: {exc}"}) diff --git a/src/autoskillit/server/tools_execution.py b/src/autoskillit/server/tools/tools_execution.py similarity index 62% rename from src/autoskillit/server/tools_execution.py rename to src/autoskillit/server/tools/tools_execution.py index 879709aabc..705c75a694 100644 --- a/src/autoskillit/server/tools_execution.py +++ b/src/autoskillit/server/tools/tools_execution.py @@ -2,16 +2,19 @@ from __future__ import annotations +import asyncio import json import os import time from pathlib import Path +import regex as re import structlog from fastmcp import Context from fastmcp.dependencies import CurrentContext from autoskillit.core import ( + DISPATCH_ID_ENV_VAR, LayoutError, SkillResult, ValidatedAddDir, @@ -20,27 +23,30 @@ validate_add_dir, ) from autoskillit.server import mcp -from autoskillit.server.helpers import ( - SCENARIO_STEP_NAME_ENV, +from autoskillit.server._guards import ( _check_dry_walkthrough, - _import_and_call, - _notify, _require_enabled, - _require_not_headless, - _run_subprocess, + _require_orchestrator_or_higher, _validate_skill_command, - track_response_size, ) +from autoskillit.server._misc import SCENARIO_STEP_NAME_ENV +from autoskillit.server._notify import _notify, track_response_size +from autoskillit.server._subprocess import _run_subprocess logger = get_logger(__name__) +_PURE_SLEEP_RE = re.compile( + r'^(?:python3?\s+-c\s+["\']import time;\s*time\.sleep\((?P\d+(?:\.\d+)?)\)["\']' + r"|sleep\s+(?P\d+(?:\.\d+)?))$" +) + def _is_absolute_path(path: str) -> bool: """Return True if path is an absolute filesystem path.""" return Path(path).is_absolute() -@mcp.tool(tags={"autoskillit", "kitchen"}, annotations={"readOnlyHint": True}) +@mcp.tool(tags={"autoskillit", "kitchen", "kitchen-core"}, annotations={"readOnlyHint": True}) @track_response_size("run_cmd") async def run_cmd( cmd: str, @@ -59,8 +65,8 @@ async def run_cmd( Never raises. """ - if (headless := _require_not_headless("run_cmd")) is not None: - return headless + if (tier_gate := _require_orchestrator_or_higher("run_cmd")) is not None: + return tier_gate if (gate := _require_enabled()) is not None: return gate try: @@ -76,6 +82,11 @@ async def run_cmd( tool_ctx = _get_ctx() _start = time.monotonic() try: + m = _PURE_SLEEP_RE.match(cmd.strip()) + if m: + seconds = float(m.group("py_secs") or m.group("sh_secs")) + await asyncio.sleep(seconds) + return json.dumps({"success": True, "exit_code": 0, "stdout": "", "stderr": ""}) _env: dict[str, str] | None = ( {**os.environ, SCENARIO_STEP_NAME_ENV: step_name} if step_name else None ) @@ -115,7 +126,7 @@ async def run_cmd( ) -@mcp.tool(tags={"autoskillit", "kitchen"}, annotations={"readOnlyHint": True}) +@mcp.tool(tags={"autoskillit", "kitchen", "kitchen-core"}, annotations={"readOnlyHint": True}) @track_response_size("run_python") async def run_python( callable: str, @@ -140,8 +151,8 @@ async def run_python( Never raises. """ - if (headless := _require_not_headless("run_python")) is not None: - return headless + if (tier_gate := _require_orchestrator_or_higher("run_python")) is not None: + return tier_gate if (gate := _require_enabled()) is not None: return gate try: @@ -155,6 +166,8 @@ async def run_python( "autoskillit.run_python", extra={"callable": callable}, ) + from autoskillit.server.tools._execution_helpers import _import_and_call # noqa: PLC0415 + result = await _import_and_call(callable, args=args, timeout=float(timeout)) if not result.get("success"): await _notify( @@ -170,7 +183,19 @@ async def run_python( return json.dumps({"success": False, "error": f"{type(exc).__name__}: {exc}"}) -@mcp.tool(tags={"autoskillit", "kitchen"}, annotations={"readOnlyHint": True}) +def _persist_run_skill_state(skill_result: SkillResult, project_dir: Path) -> None: + from autoskillit.server._misc import persist_run_skill_state # noqa: PLC0415 + + persist_run_skill_state(skill_result, project_dir) + + +def _clear_run_skill_state(project_dir: Path) -> None: + from autoskillit.server._misc import clear_run_skill_state # noqa: PLC0415 + + clear_run_skill_state(project_dir) + + +@mcp.tool(tags={"autoskillit", "kitchen", "kitchen-core"}, annotations={"readOnlyHint": True}) @track_response_size("run_skill") async def run_skill( skill_command: str, @@ -180,6 +205,8 @@ async def run_skill( order_id: str = "", stale_threshold: int | None = None, idle_output_timeout: int | None = None, + output_dir: str = "", + resume_session_id: str = "", ctx: Context = CurrentContext(), ) -> str: """Run a Claude Code headless session with a skill command. @@ -189,12 +216,16 @@ async def run_skill( - "resume": context/turn limit hit — partial progress on disk, route to on_context_limit. - "drain_race": channel confirmed completion but stdout not fully flushed — route to on_context_limit (same as resume). - - "empty_output": session exited cleanly but produced no output — no partial progress, - route to on_failure. + - "completed_no_flush": session exited with empty stdout but write evidence confirms work was + performed — route to on_context_limit (same as drain_race/resume). + - "empty_output": session exited cleanly with no output AND no write evidence — no partial + progress, route to on_failure. - "path_contamination": session wrote files outside its working directory — route to on_failure. - "early_stop": model stopped before completion marker — route to on_failure. - "zero_writes": skill made no writes despite write expectation — route to on_failure. + - "thinking_stall": model produced thinking blocks only, no text/tool output — route to + on_context_limit if lifespan_started, else on_failure. This is the correct MCP tool to delegate work to a headless session during pipeline execution. NEVER use native tools (Read, Grep, Glob, Edit, Write, @@ -221,15 +252,19 @@ async def run_skill( config default (RunSkillConfig.stale_threshold, default 1200s). idle_output_timeout: Override the idle stdout kill threshold in seconds. 0 = disabled for this step. None = use global config - (RunSkillConfig.idle_output_timeout, default 600s). + (RunSkillConfig.idle_output_timeout, default 1000s). + resume_session_id: Session ID from a previous run_skill call that was interrupted. + When set, the session is resumed via --resume instead of starting fresh. + The skill_command becomes a continuation instruction (non-slash text is allowed). + Pass the session_id from the previous run_skill result JSON. Never raises. """ - if (headless := _require_not_headless("run_skill")) is not None: - return headless + if (tier_gate := _require_orchestrator_or_higher("run_skill")) is not None: + return tier_gate if (gate := _require_enabled()) is not None: return gate - if (cmd_error := _validate_skill_command(skill_command)) is not None: + if not resume_session_id and (cmd_error := _validate_skill_command(skill_command)) is not None: return cmd_error if cwd and not _is_absolute_path(cwd): return json.dumps( @@ -256,6 +291,13 @@ async def run_skill( from autoskillit.server import _get_config, _get_ctx + # Auto-enrich order_id from the fleet dispatcher's env variable when the + # caller did not pass an explicit value. AUTOSKILLIT_DISPATCH_ID is injected + # by fleet/_api.py into every L2 food truck session environment and inherited by all + # sub-sessions, ensuring token log entries carry the correct order_id without + # requiring recipe authors to thread it through every run_skill call. + effective_order_id = order_id or os.environ.get(DISPATCH_ID_ENV_VAR, "") + if _get_config().safety.require_dry_walkthrough: if (gate_error := _check_dry_walkthrough(skill_command, cwd)) is not None: return gate_error @@ -264,6 +306,35 @@ async def run_skill( if tool_ctx.executor is None: return json.dumps({"success": False, "error": "Executor not configured"}) + provider_extras: dict[str, str] | None = None + profile_name_out: str = "" + effective_model = model + + from autoskillit.core import is_feature_enabled + + _cfg = _get_config() + if is_feature_enabled( + "providers", _cfg.features, experimental_enabled=_cfg.experimental_enabled + ): + from autoskillit.server._guards import ( + _resolve_model_as_profile, + _resolve_provider_profile, + ) + + _profile, _env_dict = _resolve_provider_profile( + step_name or "", tool_ctx.recipe_name or "", _cfg.providers + ) + if _profile != "anthropic": + provider_extras = _env_dict + profile_name_out = _profile + else: + effective_model, prof_name, prof_extras = _resolve_model_as_profile( + model, _cfg.providers + ) + if prof_extras is not None: + provider_extras = prof_extras + profile_name_out = prof_name + # Look up artifact validation patterns from skill contract expected_output_patterns: list[str] = [] if tool_ctx.output_pattern_resolver: @@ -290,10 +361,59 @@ async def run_skill( skill_command, tool_ctx.skill_resolver ) + write_watch_dirs: list[Path] = [] + if output_dir: + resolved_dir = Path(output_dir) + if not resolved_dir.is_absolute(): + resolved_dir = Path(cwd) / output_dir + write_watch_dirs.append(resolved_dir) + + is_read_only = bool( + tool_ctx.read_only_resolver and tool_ctx.read_only_resolver(skill_command) + ) + allowed_write_prefix = "" + if is_read_only: + if write_watch_dirs: + allowed_write_prefix = str(write_watch_dirs[0]) + "/" + else: + _skill_temp_name = target_name or "" + if _skill_temp_name: + allowed_write_prefix = os.path.join( + cwd, ".autoskillit", "temp", _skill_temp_name, "" + ) + else: + logger.warning( + "read_only_skill_no_target_name", + skill_command=skill_command[:100], + ) + invocation_marker = f"%%ORDER_UP::{uuid4().hex[:8]}%%" skill_add_dirs: list[ValidatedAddDir] = [] - if tool_ctx.session_skill_manager is not None: + replay_snapshot_used = False + _runner = tool_ctx.runner + if ( + step_name + and _runner is not None + and getattr(_runner, "skill_snapshots", None) + and hasattr(_runner, "restore_skill_snapshot") + and tool_ctx.ephemeral_root is not None + ): + _ephemeral_root = tool_ctx.ephemeral_root + session_id = f"headless-{uuid4().hex[:12]}" + _restored = _runner.restore_skill_snapshot( # type: ignore[attr-defined] + step_name, _ephemeral_root, session_id + ) + if _restored is not None: + skill_add_dirs.append(_restored) + replay_snapshot_used = True + logger.debug( + "replay_skill_snapshot_restored", + step=step_name, + session_id=session_id, + ) + + if not replay_snapshot_used and tool_ctx.session_skill_manager is not None: allow_only: frozenset[str] | None = None if target_name: closure = tool_ctx.session_skill_manager.compute_skill_closure(target_name) @@ -306,11 +426,11 @@ async def run_skill( config=tool_ctx.config, project_dir=Path(cwd), recipe_packs=tool_ctx.active_recipe_packs, + recipe_features=tool_ctx.active_recipe_features, allow_only=allow_only, ) skill_add_dirs.append(session_root) - # Activate the target skill and its declared dependencies if target_name: tool_ctx.session_skill_manager.activate_skill_deps(session_id, target_name) try: @@ -323,11 +443,11 @@ async def run_skill( skill_result = await tool_ctx.executor.run( resolved_command, cwd, - model=model, + model=effective_model, add_dirs=skill_add_dirs, step_name=step_name, kitchen_id=tool_ctx.kitchen_id, - order_id=order_id, + order_id=effective_order_id, expected_output_patterns=expected_output_patterns, write_behavior=write_spec, stale_threshold=float(stale_threshold) if stale_threshold is not None else None, @@ -335,9 +455,20 @@ async def run_skill( if idle_output_timeout is not None else None, completion_marker=invocation_marker, + recipe_name=tool_ctx.recipe_name, + recipe_content_hash=tool_ctx.recipe_content_hash, + recipe_composite_hash=tool_ctx.recipe_composite_hash, + recipe_version=tool_ctx.recipe_version, + allowed_write_prefix=allowed_write_prefix, + readonly_skill=is_read_only, + write_watch_dirs=write_watch_dirs, + provider_extras=provider_extras, + profile_name=profile_name_out, + resume_session_id=resume_session_id, ) if skill_result.success: tool_ctx.audit.record_success(skill_command) + _clear_run_skill_state(tool_ctx.project_dir) else: await _notify( ctx, @@ -346,9 +477,12 @@ async def run_skill( "autoskillit.run_skill", extra={"exit_code": skill_result.exit_code, "subtype": skill_result.subtype}, ) - if order_id: - skill_result.order_id = order_id - from autoskillit.server.helpers import _refresh_quota_cache # noqa: PLC0415 + _persist_run_skill_state(skill_result, tool_ctx.project_dir) + if effective_order_id: + skill_result.order_id = effective_order_id + from autoskillit.server._misc import ( # noqa: PLC0415 + _refresh_quota_cache, + ) if tool_ctx.background is not None: tool_ctx.background.submit( @@ -361,11 +495,13 @@ async def run_skill( return SkillResult.crashed( exception=exc, skill_command=resolved_command, - order_id=order_id, + order_id=effective_order_id, ).to_json() finally: if step_name: - tool_ctx.timing_log.record(step_name, time.monotonic() - _start, order_id=order_id) + tool_ctx.timing_log.record( + step_name, time.monotonic() - _start, order_id=effective_order_id + ) except Exception as exc: logger.error("run_skill unhandled exception", exc_info=True) return SkillResult.crashed( @@ -373,3 +509,6 @@ async def run_skill( skill_command=skill_command, order_id=order_id, ).to_json() + except BaseException: + logger.warning("run_skill cancelled", exc_info=True) + raise diff --git a/src/autoskillit/server/tools/tools_fleet_dispatch.py b/src/autoskillit/server/tools/tools_fleet_dispatch.py new file mode 100644 index 0000000000..36a86c0438 --- /dev/null +++ b/src/autoskillit/server/tools/tools_fleet_dispatch.py @@ -0,0 +1,258 @@ +"""MCP tool handlers: dispatch_food_truck, record_gate_dispatch.""" + +from __future__ import annotations + +import functools +import json +import os +from collections.abc import Callable +from pathlib import Path + +from fastmcp import Context +from fastmcp.dependencies import CurrentContext + +from autoskillit.core import get_logger +from autoskillit.server import mcp +from autoskillit.server._guards import _require_enabled, _require_fleet +from autoskillit.server._notify import track_response_size + +logger = get_logger(__name__) + + +def _get_food_truck_prompt_builder() -> Callable[..., str]: + """Return the food truck prompt builder with mcp_prefix pre-bound.""" + from autoskillit.core import detect_autoskillit_mcp_prefix + from autoskillit.fleet import _build_food_truck_prompt + + mcp_prefix = detect_autoskillit_mcp_prefix() + return functools.partial(_build_food_truck_prompt, mcp_prefix=mcp_prefix) + + +@mcp.tool( + tags={"autoskillit", "kitchen-core", "fleet"}, + annotations={"readOnlyHint": True}, +) +@track_response_size("dispatch_food_truck") +async def dispatch_food_truck( + recipe: str, + task: str, + ingredients: dict[str, str] | None = None, + dispatch_name: str | None = None, + timeout_sec: int | None = None, + capture: dict[str, str] | None = None, + resume_session_id: str | None = None, + resume_checkpoint: dict[str, object] | None = None, + idle_output_timeout: int | None = None, + ctx: Context = CurrentContext(), +) -> str: + """Dispatch a single food truck L2 session for one recipe. + + Spawns a headless subprocess that executes the given recipe with the + provided task and ingredient overrides. Returns a JSON envelope with + dispatch_id, dispatched_session_id, l3_payload, and token_usage. + + Args: + recipe: Recipe name to dispatch (must be kind=standard). + task: Task description for the L2 food truck session. + ingredients: Optional ingredient overrides (all values must be strings). + dispatch_name: Optional display name for the dispatch record. + timeout_sec: Optional L2 session timeout override in seconds. + capture: Optional dict mapping capture keys to "${{ result.field }}" templates. + Extracted values are persisted in the campaign context for downstream + dispatches to reference via "${{ campaign.key }}" in their ingredients. + resume_checkpoint: Checkpoint dict from a prior RESUMABLE dispatch envelope. + Pass the "resume_checkpoint" field from the prior result to inject completed + items context into the resume prompt. + + Never raises. + """ + if (gate := _require_enabled()) is not None: + return gate + if (fleet_gate := _require_fleet("dispatch_food_truck")) is not None: + return fleet_gate + + try: + # Feature guard: config authority check independent of MCP visibility state. + # Fleet sessions open the gate unconditionally at boot; this catch-all ensures + # dispatch_food_truck never executes when features.fleet is disabled in config. + from autoskillit.core import FleetErrorCode, fleet_error, is_feature_enabled + from autoskillit.server import _get_ctx as _get_ctx_for_feature_check + + _feature_ctx = _get_ctx_for_feature_check() + if not is_feature_enabled( + "fleet", + _feature_ctx.config.features, + experimental_enabled=_feature_ctx.config.experimental_enabled, + ): + return fleet_error( + FleetErrorCode.FLEET_FEATURE_DISABLED, + "Fleet feature is disabled. Set features.experimental_enabled: true to enable.", + ) + + campaign_state_path_str = os.environ.get("AUTOSKILLIT_CAMPAIGN_STATE_PATH") + continue_on_failure_str = os.environ.get("AUTOSKILLIT_CONTINUE_ON_FAILURE", "false") + if campaign_state_path_str and continue_on_failure_str.lower() != "true": + from autoskillit.fleet import ( # noqa: PLC0415 + has_failed_dispatch, + reset_failed_dispatch, + ) + + campaign_sp = Path(campaign_state_path_str) + if dispatch_name: + reset_failed_dispatch(campaign_sp, dispatch_name) + if has_failed_dispatch(campaign_sp): + return fleet_error( + FleetErrorCode.FLEET_CAMPAIGN_HALTED, + "Campaign halted: a prior dispatch failed and " + "continue_on_failure is false. " + "No further dispatches permitted.", + ) + + from autoskillit.core import SessionCheckpoint # noqa: PLC0415 + from autoskillit.fleet import execute_dispatch + from autoskillit.server import _get_ctx + from autoskillit.server._misc import ( # noqa: PLC0415 + _refresh_quota_cache, + check_and_sleep_if_needed, + invalidate_cache, + ) + + parsed_checkpoint = ( + SessionCheckpoint.from_dict(resume_checkpoint) if resume_checkpoint else None + ) + tool_ctx = _get_ctx() + from autoskillit.core import find_caller_session_id + + caller_session_id = find_caller_session_id(project_dir=tool_ctx.project_dir) + result = await execute_dispatch( + tool_ctx=tool_ctx, + recipe=recipe, + task=task, + ingredients=ingredients, + dispatch_name=dispatch_name, + timeout_sec=timeout_sec, + prompt_builder=_get_food_truck_prompt_builder(), + quota_checker=check_and_sleep_if_needed, + quota_refresher=_refresh_quota_cache, + cache_invalidator=invalidate_cache, + capture=capture, + resume_session_id=resume_session_id, + resume_checkpoint=parsed_checkpoint, + idle_output_timeout=idle_output_timeout, + caller_session_id=caller_session_id, + ) + + campaign_state_path_str = os.environ.get("AUTOSKILLIT_CAMPAIGN_STATE_PATH") + if campaign_state_path_str and dispatch_name: + try: + if not isinstance(result, str): + raise TypeError( + f"execute_dispatch returned {type(result).__name__}, expected str" + ) + envelope = json.loads(result) + campaign_state_path = Path(campaign_state_path_str) + if campaign_state_path.exists(): + from autoskillit.fleet import ( + DispatchRecord, + DispatchStatus, + append_dispatch_record, + ) + + raw_status = envelope.get("dispatch_status") + if raw_status is None: + raise KeyError("dispatch_status missing from dispatch envelope") + status = DispatchStatus(raw_status) + append_dispatch_record( + campaign_state_path, + DispatchRecord( + name=dispatch_name, + status=status, + dispatch_id=envelope.get("dispatch_id", ""), + dispatched_session_id=envelope.get("dispatched_session_id", ""), + reason=envelope.get("reason", ""), + token_usage=envelope.get("token_usage") or {}, + ), + ) + except (json.JSONDecodeError, KeyError): + logger.warning("campaign state update failed", exc_info=True) + + return result + except Exception as exc: + logger.error("dispatch_food_truck unhandled exception", exc_info=True) + from autoskillit.core import FleetErrorCode, fleet_error + + return fleet_error( + FleetErrorCode.FLEET_L3_STARTUP_OR_CRASH, + f"{type(exc).__name__}: {exc}", + ) + + +@mcp.tool( + tags={"autoskillit", "kitchen-core", "fleet"}, + annotations={"readOnlyHint": True}, +) +@track_response_size("record_gate_dispatch") +async def record_gate_dispatch( + dispatch_name: str, + approved: bool, + ctx: Context = CurrentContext(), +) -> str: + """Record the outcome of a gate dispatch to the campaign state file. + + Gate dispatches are handled by AskUserQuestion (no L3 session). This tool + persists the user's approval/rejection so that campaign resume can skip + completed gates. + + Args: + dispatch_name: Name of the gate dispatch in the campaign manifest. + approved: True if the user approved the gate, False if rejected. + + Never raises. + """ + if (gate := _require_enabled()) is not None: + return gate + if (fleet_gate := _require_fleet("record_gate_dispatch")) is not None: + return fleet_gate + + try: + from autoskillit.core import FleetErrorCode, fleet_error, is_feature_enabled + from autoskillit.fleet import record_gate_outcome + from autoskillit.server import _get_ctx as _get_ctx_for_feature_check + + _feature_ctx = _get_ctx_for_feature_check() + if not is_feature_enabled( + "fleet", + _feature_ctx.config.features, + experimental_enabled=_feature_ctx.config.experimental_enabled, + ): + return fleet_error( + FleetErrorCode.FLEET_FEATURE_DISABLED, + "Fleet feature is disabled. Set features.experimental_enabled: true to enable.", + ) + + campaign_state_path_str = os.environ.get("AUTOSKILLIT_CAMPAIGN_STATE_PATH") + if not campaign_state_path_str: + return fleet_error( + FleetErrorCode.FLEET_GATE_NO_CAMPAIGN, + "No AUTOSKILLIT_CAMPAIGN_STATE_PATH set — not running in campaign mode.", + ) + + result = record_gate_outcome(Path(campaign_state_path_str), dispatch_name, approved) + if not result.success: + try: + error_code = FleetErrorCode(result.error_code) + except ValueError: + error_code = FleetErrorCode.FLEET_L3_STARTUP_OR_CRASH + return fleet_error(error_code, result.error_message) + + return json.dumps( + {"success": True, "dispatch_name": result.dispatch_name, "status": result.status} + ) + except Exception as exc: + logger.error("record_gate_dispatch unhandled exception", exc_info=True) + from autoskillit.core import FleetErrorCode, fleet_error + + return fleet_error( + FleetErrorCode.FLEET_L3_STARTUP_OR_CRASH, + f"{type(exc).__name__}: {exc}", + ) diff --git a/src/autoskillit/server/tools_git.py b/src/autoskillit/server/tools/tools_git.py similarity index 62% rename from src/autoskillit/server/tools_git.py rename to src/autoskillit/server/tools/tools_git.py index 840e36dff2..db35575fb5 100644 --- a/src/autoskillit/server/tools_git.py +++ b/src/autoskillit/server/tools/tools_git.py @@ -1,10 +1,12 @@ -"""MCP tool handlers: merge_worktree, classify_fix.""" +"""MCP tool handlers: merge_worktree, classify_fix, create_and_publish_branch.""" from __future__ import annotations +import asyncio import json import os import time +from typing import Any import structlog from fastmcp import Context @@ -12,17 +14,27 @@ from autoskillit.core import RestartScope, get_logger from autoskillit.server import mcp -from autoskillit.server.helpers import ( - _notify, - _require_enabled, - _run_subprocess, - track_response_size, -) +from autoskillit.server._guards import _require_enabled +from autoskillit.server._notify import _notify, track_response_size +from autoskillit.server._subprocess import _run_subprocess + +_BRANCH_DATE_FORMAT = "%Y%m%d" + + +def _compute_branch_name(issue_slug: str, run_name: str, issue_number: str) -> str: + """Compute branch name from slug + issue number or today's date.""" + from datetime import date + + prefix = issue_slug or run_name + if issue_number: + return f"{prefix}/{issue_number}" + return f"{prefix}/{date.today().strftime(_BRANCH_DATE_FORMAT)}" + logger = get_logger(__name__) -@mcp.tool(tags={"autoskillit", "kitchen"}, annotations={"readOnlyHint": True}) +@mcp.tool(tags={"autoskillit", "kitchen", "kitchen-core"}, annotations={"readOnlyHint": True}) @track_response_size("merge_worktree") async def merge_worktree( worktree_path: str, @@ -34,12 +46,12 @@ async def merge_worktree( Programmatic gate: runs the configured test command in the worktree before allowing merge. If tests fail, returns error without merging. - On failure, consider using /autoskillit:resolve-failures via run_skill + On failure, consider using /resolve-failures via run_skill for automated diagnosis and remediation. Args: worktree_path: Absolute path to the git worktree. - base_branch: Branch to merge into (e.g. "integration"). + base_branch: Branch to merge into (e.g. "develop"). step_name: Optional YAML step key for wall-clock timing accumulation. Never raises. @@ -59,16 +71,19 @@ async def merge_worktree( ) from autoskillit.server import _get_config, _get_ctx + from autoskillit.server._misc import resolve_remote_name from autoskillit.server.git import perform_merge tool_ctx = _get_ctx() runner = tool_ctx.runner assert runner is not None, "No subprocess runner configured" _start = time.monotonic() + remote = await resolve_remote_name(worktree_path) try: result = await perform_merge( worktree_path, base_branch, + remote=remote, config=_get_config(), runner=runner, tester=tool_ctx.tester, @@ -92,7 +107,7 @@ async def merge_worktree( return json.dumps({"success": False, "error": f"{type(exc).__name__}: {exc}"}) -@mcp.tool(tags={"autoskillit", "kitchen"}, annotations={"readOnlyHint": True}) +@mcp.tool(tags={"autoskillit", "kitchen", "kitchen-core"}, annotations={"readOnlyHint": True}) @track_response_size("classify_fix") async def classify_fix( worktree_path: str, @@ -109,9 +124,9 @@ async def classify_fix( Routing guidance: - full_restart: The fix touches critical paths. Re-run investigation and - plan creation (e.g. call /autoskillit:investigate via run_skill). + plan creation (e.g. call /investigate via run_skill). - partial_restart: The fix is localized. Re-run implementation only - (e.g. call /autoskillit:implement-worktree-no-merge via run_skill). + (e.g. call /implement-worktree-no-merge via run_skill). Args: worktree_path: Path to the git worktree with the implemented fix. @@ -147,13 +162,15 @@ async def classify_fix( ) from autoskillit.server import _get_config, _get_ctx + from autoskillit.server._misc import resolve_remote_name from autoskillit.server.git import _filter_changed_files tool_ctx = _get_ctx() _start = time.monotonic() + remote = await resolve_remote_name(worktree_path) try: fetch_rc, _, fetch_stderr = await _run_subprocess( - ["git", "fetch", "origin", base_branch], + ["git", "fetch", remote, base_branch], cwd=worktree_path, timeout=30, ) @@ -162,7 +179,7 @@ async def classify_fix( { "restart_scope": RestartScope.FULL_RESTART, "reason": ( - f"git fetch origin {base_branch} failed — " + f"git fetch {remote} {base_branch} failed — " "remote-tracking ref may be stale. " f"git error: {(fetch_stderr or '').strip()[:200]}" ), @@ -172,7 +189,7 @@ async def classify_fix( ) returncode, stdout, stderr = await _run_subprocess( - ["git", "diff", "--name-only", f"origin/{base_branch}...HEAD"], + ["git", "diff", "--name-only", f"{remote}/{base_branch}...HEAD"], cwd=worktree_path, timeout=30, ) @@ -193,7 +210,7 @@ async def classify_fix( { "restart_scope": RestartScope.FULL_RESTART, "reason": ( - f"Cannot diff against origin/{base_branch}" + f"Cannot diff against {remote}/{base_branch}" f" — ref may not exist locally. " f"git error: {stderr.strip()[:200]}" ), @@ -319,70 +336,10 @@ async def create_unique_branch( ) else: base_name = f"{slug}-{issue_number}" if issue_number is not None else slug - branch_name = base_name - was_unique = True try: - rc, stdout, _ = await _run_subprocess( - ["git", "ls-remote", remote, f"refs/heads/{branch_name}"], - cwd=cwd, - timeout=30, - ) - - if rc == 0 and stdout.strip(): - was_unique = False - suffix = 2 - _MAX_SUFFIX = 100 - while suffix <= _MAX_SUFFIX: - candidate = f"{base_name}-{suffix}" - rc2, stdout2, _ = await _run_subprocess( - ["git", "ls-remote", remote, f"refs/heads/{candidate}"], - cwd=cwd, - timeout=30, - ) - if rc2 != 0: - branch_name = candidate - break - if not stdout2.strip(): - branch_name = candidate - break - suffix += 1 - else: - return json.dumps( - { - "success": False, - "error": ( - f"All branch name candidates up to " - f"{base_name}-{_MAX_SUFFIX} are taken." - ), - } - ) - - rc_head, head_out, _ = await _run_subprocess( - ["git", "branch", "--show-current"], - cwd=cwd, - timeout=10, - ) - base_ref = head_out.strip() if rc_head == 0 and head_out.strip() else "DETACHED_HEAD" - - rc_checkout, _, _stderr_checkout = await _run_subprocess( - ["git", "checkout", "-b", branch_name], - cwd=cwd, - timeout=30, - ) - if rc_checkout != 0: - return json.dumps( - { - "success": False, - "error": ( - f"git checkout -b {branch_name!r} failed: {_stderr_checkout.strip()}" - ), - } - ) - - return json.dumps( - {"branch_name": branch_name, "was_unique": was_unique, "base_ref": base_ref} - ) + result = await _resolve_and_create_branch(base_name, remote, cwd) + return json.dumps(result) finally: if step_name: tool_ctx.timing_log.record(step_name, time.monotonic() - _start) @@ -454,3 +411,196 @@ async def check_pr_mergeable( except Exception as exc: logger.error("check_pr_mergeable unhandled exception", exc_info=True) return json.dumps({"success": False, "error": f"{type(exc).__name__}: {exc}"}) + + +async def _resolve_and_create_branch( + base_name: str, + remote: str, + cwd: str, +) -> dict[str, Any]: + """Core branch creation: ls-remote collision check + git checkout -b. + + Returns {"branch_name": str, "was_unique": bool, "base_ref": str} on success, + or {"success": False, "error": str} on failure. + """ + branch_name = base_name + was_unique = True + + rc, stdout, _ = await _run_subprocess( + ["git", "ls-remote", remote, f"refs/heads/{branch_name}"], + cwd=cwd, + timeout=30, + ) + + if rc == 0 and stdout.strip(): + was_unique = False + suffix = 2 + _MAX_SUFFIX = 100 + while suffix <= _MAX_SUFFIX: + candidate = f"{base_name}-{suffix}" + rc2, stdout2, _ = await _run_subprocess( + ["git", "ls-remote", remote, f"refs/heads/{candidate}"], + cwd=cwd, + timeout=30, + ) + if rc2 != 0: + branch_name = candidate + break + if not stdout2.strip(): + branch_name = candidate + break + suffix += 1 + else: + return { + "success": False, + "error": f"All branch name candidates up to {base_name}-{_MAX_SUFFIX} are taken.", + } + + rc_head, head_out, _ = await _run_subprocess( + ["git", "branch", "--show-current"], + cwd=cwd, + timeout=10, + ) + base_ref = head_out.strip() if rc_head == 0 and head_out.strip() else "DETACHED_HEAD" + + rc_checkout, _, _stderr_checkout = await _run_subprocess( + ["git", "checkout", "-b", branch_name], + cwd=cwd, + timeout=30, + ) + if rc_checkout != 0: + return { + "success": False, + "error": f"git checkout -b {branch_name!r} failed: {_stderr_checkout.strip()}", + } + + return {"branch_name": branch_name, "was_unique": was_unique, "base_ref": base_ref} + + +@mcp.tool(tags={"autoskillit", "kitchen", "github"}, annotations={"readOnlyHint": True}) +@track_response_size("create_and_publish_branch") +async def create_and_publish_branch( + issue_slug: str, + run_name: str, + issue_number: str, + work_dir: str, + remote_url: str, + step_name: str = "", + ctx: Context = CurrentContext(), +) -> str: + """Compute branch name, create it, and push to remote in one orchestrator turn. + + Combines compute_branch + create_unique_branch + push_to_remote into a single call. + + Returns {"merge_target": str, "was_unique": bool, "timings": {...}} on success. + Returns {"error": str, "error_type": str, "merge_target": str} on push failure — + merge_target is included so the orchestrator can reference it in release paths. + + Args: + issue_slug: Descriptive slug from the issue title (e.g. "fix-bug"). + run_name: Run name prefix (e.g. "impl"). Used as prefix when issue_slug is empty. + issue_number: GitHub issue number as a string. Empty string → date-based branch. + work_dir: Absolute path to the clone directory. + remote_url: Pre-resolved upstream remote URL for push isolation. + step_name: Optional YAML step key for wall-clock timing accumulation. + + Never raises. + """ + if (gate := _require_enabled()) is not None: + return gate + try: + structlog.contextvars.clear_contextvars() + structlog.contextvars.bind_contextvars( + tool="create_and_publish_branch", + work_dir=work_dir, + issue_number=issue_number, + ) + logger.info( + "create_and_publish_branch", + issue_slug=issue_slug, + run_name=run_name, + issue_number=issue_number, + ) + await _notify( + ctx, + "info", + f"create_and_publish_branch: {issue_slug or run_name}/{issue_number or 'date'}", + "autoskillit.create_and_publish_branch", + extra={"run_name": run_name, "issue_number": issue_number}, + ) + + from autoskillit.server import _get_ctx + + tool_ctx = _get_ctx() + clone_mgr = tool_ctx.clone_mgr + if clone_mgr is None: + return json.dumps({"error": "Clone manager not configured"}) + + _total_start = time.monotonic() + + _compute_start = time.monotonic() + base_name = _compute_branch_name(issue_slug, run_name, issue_number) + compute_ms = int((time.monotonic() - _compute_start) * 1000) + + _branch_start = time.monotonic() + branch_result = await _resolve_and_create_branch(base_name, "origin", work_dir) + branch_create_ms = int((time.monotonic() - _branch_start) * 1000) + + if "error" in branch_result: + return json.dumps( + { + "error": branch_result["error"], + "merge_target": base_name, + "timings": { + "compute_ms": compute_ms, + "branch_create_ms": branch_create_ms, + "push_ms": 0, + }, + } + ) + + branch_name: str = branch_result["branch_name"] + was_unique: bool = branch_result["was_unique"] + + _push_start = time.monotonic() + push_result = await asyncio.to_thread( + lambda: clone_mgr.push_to_remote( + work_dir, + "", + branch_name, + remote_url=remote_url, + protected_branches=tool_ctx.config.safety.protected_branches, + force=False, + ) + ) + push_ms = int((time.monotonic() - _push_start) * 1000) + + timings = { + "compute_ms": compute_ms, + "branch_create_ms": branch_create_ms, + "push_ms": push_ms, + } + + if step_name: + tool_ctx.timing_log.record(step_name, time.monotonic() - _total_start) + + if not push_result.get("success"): + return json.dumps( + { + "error": push_result.get("stderr", "push failed"), + "error_type": push_result.get("error_type", ""), + "merge_target": branch_name, + "timings": timings, + } + ) + + return json.dumps( + { + "merge_target": branch_name, + "was_unique": was_unique, + "timings": timings, + } + ) + except Exception as exc: + logger.error("create_and_publish_branch unhandled exception", exc_info=True) + return json.dumps({"error": f"{type(exc).__name__}: {exc}"}) diff --git a/src/autoskillit/server/tools_github.py b/src/autoskillit/server/tools/tools_github.py similarity index 81% rename from src/autoskillit/server/tools_github.py rename to src/autoskillit/server/tools/tools_github.py index e2b1bfa9af..e0dcd58659 100644 --- a/src/autoskillit/server/tools_github.py +++ b/src/autoskillit/server/tools/tools_github.py @@ -3,11 +3,11 @@ from __future__ import annotations import json -import re from datetime import UTC, datetime from pathlib import Path from typing import TYPE_CHECKING, Any +import regex as re import structlog from fastmcp import Context from fastmcp.dependencies import CurrentContext @@ -15,13 +15,9 @@ from autoskillit.core import atomic_write, get_logger from autoskillit.pipeline import write_status from autoskillit.server import mcp -from autoskillit.server.helpers import ( - _extract_block, - _notify, - _require_enabled, - resolve_log_dir, - track_response_size, -) +from autoskillit.server._guards import _require_enabled +from autoskillit.server._misc import _extract_block, resolve_log_dir +from autoskillit.server._notify import _notify, track_response_size if TYPE_CHECKING: from autoskillit.core import GitHubFetcher, HeadlessExecutor @@ -36,7 +32,7 @@ _FINGERPRINT_END = "---/bug-fingerprint---" -@mcp.tool(tags={"autoskillit", "kitchen", "github"}, annotations={"readOnlyHint": True}) +@mcp.tool(tags={"autoskillit", "github", "fleet-dispatch"}, annotations={"readOnlyHint": True}) @track_response_size("fetch_github_issue") async def fetch_github_issue( issue_url: str, @@ -44,15 +40,19 @@ async def fetch_github_issue( ) -> str: """Retrieve a GitHub issue as a formatted Markdown string. - Use this tool automatically whenever you encounter a GitHub issue URL, - shorthand reference (owner/repo#number), or bare issue number (when - default_repo is configured). Do not ask the user to paste the issue - content manually. + Call this tool when your session's role requires reading and acting on + the full issue content — for example, when writing an implementation + plan, conducting an investigation, or generating a scope report. + + Do NOT call this tool when your role is to route the issue URL downstream + as an ingredient (e.g. passing issue_url to dispatch_food_truck). The + downstream skill session will fetch the issue when it actually needs the + content; calling it here is wasteful and redundant. Returns JSON with: success, issue_number, title, url, state, labels, and content (Markdown). The content field is suitable for passing directly - as a prompt argument to skills like /autoskillit:make-plan, - /autoskillit:make-groups, or /autoskillit:investigate. + as a prompt argument to skills like /make-plan, + /make-groups, or /investigate. On failure: {"success": false, "error": "..."} — never raises. @@ -69,42 +69,49 @@ async def fetch_github_issue( try: from autoskillit.server import _get_config, _get_ctx - # Read-only query: structlog context binding is intentionally omitted. - logger.info("fetch_github_issue", issue_url=issue_url, include_comments=include_comments) + with structlog.contextvars.bound_contextvars( + tool="fetch_github_issue", issue_url=issue_url + ): + logger.info("fetch_github_issue", include_comments=include_comments) - tool_ctx = _get_ctx() - if tool_ctx.github_client is None: - return json.dumps({"success": False, "error": "GitHub client not configured"}) + tool_ctx = _get_ctx() + if tool_ctx.github_client is None: + return json.dumps({"success": False, "error": "GitHub client not configured"}) - config = _get_config() + config = _get_config() - # Resolve bare issue numbers using default_repo from config - resolved_ref = issue_url - if issue_url.strip().isdigit(): - if not config.github.default_repo: - return json.dumps( - { - "success": False, - "error": ( - f"Cannot resolve bare issue number {issue_url!r}: " - "github.default_repo is not set in .autoskillit/config.yaml" - ), - } - ) - resolved_ref = f"{config.github.default_repo}#{issue_url.strip()}" - logger.info("resolved bare number", resolved_ref=resolved_ref) + # Resolve bare issue numbers using default_repo from config + resolved_ref = issue_url + if issue_url.strip().isdigit(): + if not config.github.default_repo: + return json.dumps( + { + "success": False, + "error": ( + f"Cannot resolve bare issue number {issue_url!r}: " + "github.default_repo is not set in .autoskillit/config.yaml" + ), + } + ) + resolved_ref = f"{config.github.default_repo}#{issue_url.strip()}" + logger.info("resolved bare number", resolved_ref=resolved_ref) - result = await tool_ctx.github_client.fetch_issue( - resolved_ref, - include_comments=include_comments, - ) - return json.dumps(result) + result = await tool_ctx.github_client.fetch_issue( + resolved_ref, + include_comments=include_comments, + ) + return json.dumps(result) except Exception as exc: - logger.error("fetch_github_issue unhandled exception", exc_info=exc) + logger.error( + "fetch_github_issue unhandled exception", + exc_info=True, + tool="fetch_github_issue", + issue_url=issue_url, + ) return json.dumps({"success": False, "error": str(exc)}) -@mcp.tool(tags={"autoskillit", "kitchen", "github"}, annotations={"readOnlyHint": True}) +@mcp.tool(tags={"autoskillit", "github", "fleet-dispatch"}, annotations={"readOnlyHint": True}) @track_response_size("get_issue_title") async def get_issue_title(issue_url: str) -> str: """Fetch only the title and slug for a GitHub issue — no body, no comments. @@ -162,7 +169,7 @@ async def report_bug( ) -> str: """Run a headless investigation session for a bug and file or update a GitHub issue. - Launches /autoskillit:report-bug in a headless session to investigate the error, + Launches /report-bug in a headless session to investigate the error, produce a structured markdown report, and extract a deduplication fingerprint. The report is written to disk. If github.default_repo and a token are configured, the tool searches for an existing open issue matching the fingerprint: @@ -220,8 +227,35 @@ async def report_bug( report_path = report_dir / f"{timestamp}_report.md" effective_model = model or cfg.model or "" + report_provider_extras: dict[str, str] | None = None + report_profile_name: str = "" + + from autoskillit.core import is_feature_enabled + + if is_feature_enabled( + "providers", + config.features, + experimental_enabled=config.experimental_enabled, + ): + from autoskillit.server._guards import ( + _resolve_model_as_profile, + _resolve_provider_profile, + ) + + _rp_profile, _rp_env = _resolve_provider_profile("", "", config.providers) + if _rp_profile != "anthropic": + report_provider_extras = _rp_env + report_profile_name = _rp_profile + else: + effective_model, _prof_name, _prof_extras = _resolve_model_as_profile( + effective_model, config.providers + ) + if _prof_extras is not None: + report_provider_extras = _prof_extras + report_profile_name = _prof_name + skill_command = ( - f"/autoskillit:report-bug\n\n" + f"/report-bug\n\n" f"Error context:\n{error_context}\n\n" f"Report output path: {report_path}" ) @@ -252,6 +286,8 @@ async def report_bug( log_dir=log_dir, expected_output_patterns=expected_output_patterns, write_behavior=write_spec, + provider_extras=report_provider_extras, + profile_name=report_profile_name, ) if not result["success"]: await _notify( @@ -289,6 +325,8 @@ async def report_bug( expected_output_patterns=expected_output_patterns, write_behavior=write_spec, status_path=status_path, + provider_extras=report_provider_extras, + profile_name=report_profile_name, ), label=step_name or "report_bug", ) @@ -496,14 +534,14 @@ async def _file_or_update_github_issue( issue_url: str = existing["html_url"] existing_body: str = existing.get("body", "") or "" - # Skip comment if the exact error_context is already in the issue body. + # Skip update if the exact error_context is already in the issue body. if error_context.strip() in existing_body: logger.info( - "report_bug duplicate skipped comment", + "report_bug duplicate skipped update", issue=issue_number, reason="error_context already present", ) - return {"duplicate": True, "issue_url": issue_url, "comment_added": False} + return {"duplicate": True, "issue_url": issue_url, "body_updated": False} date_str = datetime.now(UTC).date().isoformat() diag_section = ( @@ -511,24 +549,26 @@ async def _file_or_update_github_issue( if diag is not None else "" ) - comment_body = ( - f"**New occurrence auto-reported on {date_str}**\n\n" + occurrence_section = ( + f"\n\n---\n\n" + f"## New Occurrence — {date_str}\n\n" f"**Error context:**\n```\n{error_context}\n```\n\n" f"**Local report:** `{report_path}`" f"{diag_section}" ) - comment_result = await github_client.add_comment( - owner, repo, issue_number, comment_body + new_body = existing_body + occurrence_section + update_result = await github_client.update_issue_body( + owner, repo, issue_number, new_body ) logger.info( - "report_bug commented on duplicate", + "report_bug updated duplicate body", issue=issue_number, - comment_success=comment_result.get("success"), + update_success=update_result.get("success"), ) return { "duplicate": True, "issue_url": issue_url, - "comment_added": comment_result.get("success", False), + "body_updated": update_result.get("success", False), } # No duplicate — create a new issue. @@ -550,7 +590,7 @@ async def _file_or_update_github_issue( "issue_url": create_result.get("url", ""), } except Exception as exc: - logger.error("_file_or_update_github_issue unhandled exception", exc_info=exc) + logger.error("_file_or_update_github_issue unhandled exception", exc_info=True) return {"skipped": True, "reason": f"unexpected error: {exc}"} @@ -568,6 +608,8 @@ async def _run_report_session( expected_output_patterns: list[str] | None = None, write_behavior: Any = None, status_path: Path | None = None, + provider_extras: dict[str, str] | None = None, + profile_name: str = "", ) -> dict[str, Any]: """Run the headless session, write the report, and handle GitHub filing. @@ -584,6 +626,8 @@ async def _run_report_session( timeout=float(cfg.timeout), expected_output_patterns=expected_output_patterns or [], write_behavior=write_behavior, + provider_extras=provider_extras, + profile_name=profile_name, ) report_text = skill_result.result or skill_result.stderr or "No report generated." @@ -629,7 +673,7 @@ async def _run_report_session( "github": github, } except Exception as exc: - logger.error("_run_report_session unhandled exception", exc_info=exc) + logger.error("_run_report_session unhandled exception", exc_info=True) write_status(status_path, "failed", error=str(exc)) return { "success": False, diff --git a/src/autoskillit/server/tools/tools_issue_composite.py b/src/autoskillit/server/tools/tools_issue_composite.py new file mode 100644 index 0000000000..6e1fa4e34c --- /dev/null +++ b/src/autoskillit/server/tools/tools_issue_composite.py @@ -0,0 +1,179 @@ +"""MCP tool handlers: claim_and_resolve_issue (composite).""" + +from __future__ import annotations + +import json +import time +from typing import Any + +import structlog + +from autoskillit.core import ( + _parse_issue_ref, + get_logger, +) +from autoskillit.server import mcp +from autoskillit.server._guards import _require_enabled +from autoskillit.server._notify import track_response_size + +logger = get_logger(__name__) + + +def _extract_label_names(raw_labels: list[Any]) -> list[str]: + return [lbl["name"] if isinstance(lbl, dict) else str(lbl) for lbl in raw_labels if lbl] + + +@mcp.tool(tags={"autoskillit", "kitchen", "github"}, annotations={"readOnlyHint": True}) +@track_response_size("claim_and_resolve_issue") +async def claim_and_resolve_issue( + issue_url: str, + label: str | None = None, + allow_reentry: bool = False, +) -> str: + """Fetch the issue title and claim it with an in-progress label in one turn. + + Combines get_issue_title + claim_issue into a single orchestrator call. + + Returns JSON with: success, claimed (bool), issue_number, issue_title, + issue_slug, timings. When claimed=false, issue_number/title/slug are still + returned so the orchestrator can use them in logging or abort paths. + When allow_reentry=True and the label is already present, returns + claimed=True with reentry=True. + + Args: + issue_url: Full GitHub issue URL or shorthand (owner/repo#42). + label: Label name to apply. Defaults to github.in_progress_label from config. + allow_reentry: When True and label already present, returns claimed=True. + + Never raises. + """ + if (gate := _require_enabled()) is not None: + return gate + try: + structlog.contextvars.clear_contextvars() + structlog.contextvars.bind_contextvars(tool="claim_and_resolve_issue", issue_url=issue_url) + logger.info("claim_and_resolve_issue", issue_url=issue_url) + + from autoskillit.server import _get_ctx + + tool_ctx = _get_ctx() + if tool_ctx.github_client is None: + return json.dumps( + {"success": False, "error": "GitHub token required for label management"} + ) + + effective_label = label or tool_ctx.config.github.in_progress_label + + try: + owner, repo, issue_number = _parse_issue_ref(issue_url) + except ValueError as exc: + return json.dumps({"success": False, "error": str(exc)}) + + if err := tool_ctx.config.github.check_label_allowed(effective_label): + return json.dumps({"success": False, "error": err}) + + _fetch_title_start = time.monotonic() + title_result = await tool_ctx.github_client.fetch_title(issue_url) + fetch_title_ms = int((time.monotonic() - _fetch_title_start) * 1000) + + if not title_result.get("success"): + return json.dumps( + {"success": False, "error": title_result.get("error", "fetch_title failed")} + ) + + issue_title = title_result.get("title", "") + issue_slug = title_result.get("slug", "") + + _claim_start = time.monotonic() + fetch_result = await tool_ctx.github_client.fetch_issue(issue_url, include_comments=False) + if not fetch_result.get("success"): + claim_ms = int((time.monotonic() - _claim_start) * 1000) + return json.dumps( + { + "success": False, + "error": fetch_result.get("error", "fetch_issue failed"), + "issue_number": issue_number, + "issue_title": issue_title, + "issue_slug": issue_slug, + "timings": {"fetch_title_ms": fetch_title_ms, "claim_ms": claim_ms}, + } + ) + + current_labels = _extract_label_names(fetch_result.get("labels", [])) + if effective_label in current_labels: + claim_ms = int((time.monotonic() - _claim_start) * 1000) + if allow_reentry: + return json.dumps( + { + "success": True, + "claimed": True, + "reentry": True, + "issue_number": issue_number, + "issue_title": issue_title, + "issue_slug": issue_slug, + "label": effective_label, + "timings": {"fetch_title_ms": fetch_title_ms, "claim_ms": claim_ms}, + } + ) + return json.dumps( + { + "success": True, + "claimed": False, + "reason": ( + f"Issue #{issue_number} already has '{effective_label}' label" + " — another session may be processing it" + ), + "issue_number": issue_number, + "issue_title": issue_title, + "issue_slug": issue_slug, + "timings": {"fetch_title_ms": fetch_title_ms, "claim_ms": claim_ms}, + } + ) + + ensure_color, ensure_description, remove_labels = ( + tool_ctx.config.github.resolve_label_metadata(effective_label) + ) + + await tool_ctx.github_client.ensure_label( + owner, + repo, + effective_label, + color=ensure_color, + description=ensure_description, + ) + + swap_result = await tool_ctx.github_client.swap_labels( + owner, + repo, + issue_number, + remove_labels=remove_labels, + add_labels=[effective_label], + ) + claim_ms = int((time.monotonic() - _claim_start) * 1000) + + if not swap_result.get("success"): + return json.dumps( + { + "success": False, + "error": swap_result.get("error", "swap_labels failed"), + "issue_number": issue_number, + "issue_title": issue_title, + "issue_slug": issue_slug, + "timings": {"fetch_title_ms": fetch_title_ms, "claim_ms": claim_ms}, + } + ) + + return json.dumps( + { + "success": True, + "claimed": True, + "issue_number": issue_number, + "issue_title": issue_title, + "issue_slug": issue_slug, + "label": effective_label, + "timings": {"fetch_title_ms": fetch_title_ms, "claim_ms": claim_ms}, + } + ) + except Exception as exc: + logger.error("claim_and_resolve_issue unhandled exception", exc_info=True) + return json.dumps({"success": False, "error": f"{type(exc).__name__}: {exc}"}) diff --git a/src/autoskillit/server/tools_issue_lifecycle.py b/src/autoskillit/server/tools/tools_issue_lifecycle.py similarity index 79% rename from src/autoskillit/server/tools_issue_lifecycle.py rename to src/autoskillit/server/tools/tools_issue_lifecycle.py index 5d25b76422..9d79ca85c6 100644 --- a/src/autoskillit/server/tools_issue_lifecycle.py +++ b/src/autoskillit/server/tools/tools_issue_lifecycle.py @@ -10,14 +10,15 @@ from fastmcp import Context from fastmcp.dependencies import CurrentContext -from autoskillit.core import RetryReason, _parse_issue_ref, get_logger -from autoskillit.server import mcp -from autoskillit.server.helpers import ( - _extract_block, - _notify, - _require_enabled, - track_response_size, +from autoskillit.core import ( + RetryReason, + _parse_issue_ref, + get_logger, ) +from autoskillit.server import mcp +from autoskillit.server._guards import _require_enabled +from autoskillit.server._misc import _extract_block +from autoskillit.server._notify import _notify, track_response_size if TYPE_CHECKING: from autoskillit.core import SkillResult, WriteBehaviorSpec @@ -101,8 +102,8 @@ def _build_prepare_skill_command( dry_run: bool, split: bool, ) -> str: - """Assemble the skill_command string for /autoskillit:prepare-issue.""" - parts = [f"/autoskillit:prepare-issue\n\nTitle: {title}\n\nBody:\n{body}"] + """Assemble the skill_command string for /prepare-issue.""" + parts = [f"/prepare-issue\n\nTitle: {title}\n\nBody:\n{body}"] if repo: parts.append(f"--repo {repo}") if labels: @@ -132,8 +133,8 @@ def _build_enrich_skill_command( dry_run: bool, repo: str | None, ) -> str: - """Assemble the skill_command string for /autoskillit:enrich-issues.""" - parts = ["/autoskillit:enrich-issues"] + """Assemble the skill_command string for /enrich-issues.""" + parts = ["/enrich-issues"] if issue_number is not None: parts.append(f"--issue {issue_number}") if batch is not None: @@ -174,7 +175,7 @@ async def prepare_issue( ) -> str: """Create a GitHub issue and immediately triage it with LLM classification. - Launches /autoskillit:prepare-issue in a headless session to perform the + Launches /prepare-issue in a headless session to perform the full triage workflow: dedup check, create or adopt the issue, LLM classification (bug vs enhancement, implementation vs remediation route), mixed-concern detection, and label application. @@ -279,7 +280,7 @@ async def enrich_issues( ) -> str: """Backfill structured requirements on existing recipe:implementation issues. - Launches /autoskillit:enrich-issues in a headless session to scan candidate + Launches /enrich-issues in a headless session to scan candidate issues, filter out already-enriched ones, perform codebase-grounded analysis, and append a Requirements section in REQ-{GRP}-NNN format via gh issue edit. @@ -449,20 +450,28 @@ async def claim_issue( } ) + ensure_color, ensure_description, remove_labels = ( + tool_ctx.config.github.resolve_label_metadata(effective_label) + ) + await tool_ctx.github_client.ensure_label( owner, repo, effective_label, - color="fbca04", - description="Issue is actively being processed by a pipeline session", + color=ensure_color, + description=ensure_description, ) - add_result = await tool_ctx.github_client.add_labels( - owner, repo, issue_number, [effective_label] + swap_result = await tool_ctx.github_client.swap_labels( + owner, + repo, + issue_number, + remove_labels=remove_labels, + add_labels=[effective_label], ) - if not add_result.get("success"): + if not swap_result.get("success"): return json.dumps( - {"success": False, "error": add_result.get("error", "add_labels failed")} + {"success": False, "error": swap_result.get("error", "swap_labels failed")} ) return json.dumps( @@ -485,6 +494,7 @@ async def release_issue( label: str | None = None, target_branch: str | None = None, staged_label: str | None = None, + fail_label: str | None = None, ) -> str: """Remove the in-progress label from a GitHub issue to release it. @@ -494,6 +504,9 @@ async def release_issue( When target_branch is provided and differs from the configured default base branch, also applies a 'staged' label to indicate the work is merged and awaiting promotion. + When fail_label is provided (and target_branch is NOT), swaps in-progress for the + fail label to mark the issue as failed without releasing it back to the queue. + Returns JSON with: success, issue_number, label, staged, staged_label. On gate closed or no token: {success: false, error: "..."}. @@ -502,6 +515,7 @@ async def release_issue( label: Label name to remove. Defaults to github.in_progress_label from config. target_branch: Branch the PR was merged into. When non-default, applies staged label. staged_label: Label name for staged state. Defaults to github.staged_label from config. + fail_label: Label name for failure state. When provided, swaps in-progress for this label. Never raises. """ @@ -527,26 +541,12 @@ async def release_issue( except ValueError as exc: return json.dumps({"success": False, "error": str(exc)}) - result = await tool_ctx.github_client.remove_label( - owner, repo, issue_number, effective_label - ) - if not result.get("success", False): - return json.dumps( - { - "success": False, - "error": result.get("error", "remove_label failed"), - "issue_number": issue_number, - "label": effective_label, - "staged": False, - "staged_label": None, - } - ) - # Determine if staging is needed promotion_target = tool_ctx.config.branching.promotion_target should_stage = target_branch is not None and target_branch != promotion_target staged = False + config_fail_label = tool_ctx.config.github.fail_label effective_staged_label = staged_label or tool_ctx.config.github.staged_label if should_stage: @@ -560,14 +560,27 @@ async def release_issue( } ) + if tool_ctx.config.github.state_for_label(effective_staged_label) is not None: + staged_color, staged_description, remove_labels = ( + tool_ctx.config.github.resolve_label_metadata(effective_staged_label) + ) + else: + staged_color = "0075ca" + staged_description = ( + f"Implementation staged and waiting for promotion to {promotion_target}" + ) + remove_labels = [ + effective_label, + config_fail_label, + tool_ctx.config.github.queued_label, + ] + ensure_result = await tool_ctx.github_client.ensure_label( owner, repo, effective_staged_label, - color="0075ca", - description=( - f"Implementation staged and waiting for promotion to {promotion_target}" - ), + color=staged_color, + description=staged_description, ) if not ensure_result.get("success"): return json.dumps( @@ -581,19 +594,109 @@ async def release_issue( } ) - apply_result = await tool_ctx.github_client.add_labels( - owner, repo, issue_number, [effective_staged_label] + swap_result = await tool_ctx.github_client.swap_labels( + owner, + repo, + issue_number, + remove_labels=remove_labels, + add_labels=[effective_staged_label], ) - if not apply_result.get("success"): + if not swap_result.get("success"): return json.dumps( { "success": False, "issue_number": issue_number, "label": effective_label, - "error": f"Failed to apply staged label: {apply_result.get('error', '?')}", + "error": f"Failed to apply staged label: {swap_result.get('error', '?')}", } ) staged = True + elif fail_label is not None: + if err := tool_ctx.config.github.check_label_allowed(fail_label): + return json.dumps( + { + "success": False, + "issue_number": issue_number, + "label": fail_label, + "error": err, + } + ) + + if tool_ctx.config.github.state_for_label(fail_label) is not None: + fail_color, fail_description, remove_labels = ( + tool_ctx.config.github.resolve_label_metadata(fail_label) + ) + else: + fail_color = "d73a4a" + fail_description = "Recipe execution failed" + remove_labels = [effective_label, tool_ctx.config.github.queued_label] + + ensure_result = await tool_ctx.github_client.ensure_label( + owner, + repo, + fail_label, + color=fail_color, + description=fail_description, + ) + if not ensure_result.get("success"): + return json.dumps( + { + "success": False, + "issue_number": issue_number, + "label": effective_label, + "error": ( + f"Failed to ensure fail label: {ensure_result.get('error', '?')}" + ), + } + ) + + swap_result = await tool_ctx.github_client.swap_labels( + owner, + repo, + issue_number, + remove_labels=remove_labels, + add_labels=[fail_label], + ) + if not swap_result.get("success"): + return json.dumps( + { + "success": False, + "issue_number": issue_number, + "label": effective_label, + "error": f"Failed to apply fail label: {swap_result.get('error', '?')}", + } + ) + + return json.dumps( + { + "success": True, + "issue_number": issue_number, + "label": effective_label, + "failed": True, + "fail_label": fail_label, + } + ) + else: + logger.warning( + "release_issue bare removal (no fail_label or target_branch) for %s", + issue_url, + ) + swap_result = await tool_ctx.github_client.swap_labels( + owner, + repo, + issue_number, + remove_labels=tool_ctx.config.github.all_lifecycle_labels(), + add_labels=[], + ) + if not swap_result.get("success"): + return json.dumps( + { + "success": False, + "issue_number": issue_number, + "label": effective_label, + "error": f"Failed to remove label: {swap_result.get('error', '?')}", + } + ) return json.dumps( { diff --git a/src/autoskillit/server/tools_kitchen.py b/src/autoskillit/server/tools/tools_kitchen.py similarity index 60% rename from src/autoskillit/server/tools_kitchen.py rename to src/autoskillit/server/tools/tools_kitchen.py index 9a7c7c9c3c..ae47c918f6 100644 --- a/src/autoskillit/server/tools_kitchen.py +++ b/src/autoskillit/server/tools/tools_kitchen.py @@ -3,32 +3,39 @@ from __future__ import annotations import json +import os +from datetime import UTC, datetime from pathlib import Path +from typing import TYPE_CHECKING, TypedDict from uuid import uuid4 +if TYPE_CHECKING: + from autoskillit.config.settings import QuotaGuardConfig + from fastmcp import Context from fastmcp.dependencies import CurrentContext from autoskillit import __version__ -from autoskillit.config import resolve_ingredient_defaults +from autoskillit.config import iter_display_categories, resolve_ingredient_defaults from autoskillit.core import ( PIPELINE_FORBIDDEN_TOOLS, + _collect_disabled_feature_tags, atomic_write, + find_latest_session_id, get_logger, pkg_root, ) from autoskillit.pipeline import create_background_task from autoskillit.server import mcp -from autoskillit.server.helpers import ( +from autoskillit.server._guards import _require_orchestrator_exact +from autoskillit.server._misc import ( _apply_triage_gate, _build_hook_diagnostic_warning, - _find_recipe, _hook_config_path, _prime_quota_cache, _quota_refresh_loop, - _require_not_headless, - track_response_size, ) +from autoskillit.server._notify import track_response_size logger = get_logger(__name__) @@ -47,7 +54,8 @@ def _kitchen_failure_envelope( """ msg = user_hint or ( f"open_kitchen failed during {stage}: {type(exc).__name__}. " - f"Run 'autoskillit doctor' to diagnose, or reinstall if the failure persists." + f"Run 'autoskillit doctor' to diagnose, " + f"or run 'autoskillit install' if the failure persists." ) return json.dumps( { @@ -60,66 +68,32 @@ def _kitchen_failure_envelope( ) -_DISPLAY_CATEGORIES: tuple[tuple[str, tuple[str, ...]], ...] = ( - ("Execution", ("run_cmd", "run_python", "run_skill")), - ("Testing & Workspace", ("test_check", "reset_test_dir", "classify_fix", "reset_workspace")), - ( - "Git Operations", - ("merge_worktree", "create_unique_branch", "check_pr_mergeable", "set_commit_status"), - ), - ("Recipes", ("migrate_recipe", "list_recipes", "load_recipe", "validate_recipe")), - ( - "Clone & Remote", - ( - "clone_repo", - "remove_clone", - "push_to_remote", - "register_clone_status", - "batch_cleanup_clones", - ), - ), - ( - "GitHub", - ( - "fetch_github_issue", - "get_issue_title", - "report_bug", - "prepare_issue", - "enrich_issues", - "claim_issue", - "release_issue", - "get_pr_reviews", - "bulk_close_issues", - ), - ), - ( - "CI & Automation", - ( - "wait_for_ci", - "wait_for_merge_queue", - "check_repo_merge_state", - "toggle_auto_merge", - "get_ci_status", - ), - ), - ( - "Telemetry & Diagnostics", - ( - "read_db", - "write_telemetry_files", - "kitchen_status", - "get_pipeline_report", - "get_token_summary", - "get_timing_summary", - "get_quota_events", - ), - ), - ("Kitchen", ("open_kitchen", "close_kitchen", "disable_quota_guard")), -) +class QuotaGuardHookPayload(TypedDict): + cache_max_age: int + cache_path: str + buffer_seconds: int + disabled: bool + + +def _quota_guard_hook_payload(cfg: QuotaGuardConfig) -> QuotaGuardHookPayload: + """Return the quota_guard section of .hook_config.json for a given config. + + This is the single authoritative definition of which QuotaGuardConfig fields + cross the stdlib-only boundary into hook subprocesses. When adding a field to + QuotaHookSettings, add the corresponding source field here AND update + QUOTA_GUARD_HOOK_PAYLOAD_KEYS in _hook_settings.py. The contract test + test_hook_bridge_coverage.py enforces that both stay in sync. + """ + return { + "cache_max_age": cfg.cache_max_age, + "cache_path": cfg.cache_path, + "buffer_seconds": cfg.buffer_seconds, + "disabled": not cfg.enabled, + } def _write_hook_config() -> None: - """Write user-configured quota values to temp/.autoskillit_hook_config.json. + """Write user-configured quota values to .autoskillit/temp/.hook_config.json. The hook subprocess (quota_guard.py) reads this file to apply user settings without importing the autoskillit package. @@ -129,11 +103,7 @@ def _write_hook_config() -> None: ctx = _get_ctx() cfg = ctx.config.quota_guard payload = { - "quota_guard": { - "cache_max_age": cfg.cache_max_age, - "cache_path": cfg.cache_path, - "buffer_seconds": cfg.buffer_seconds, - }, + "quota_guard": _quota_guard_hook_payload(cfg), "kitchen_id": ctx.kitchen_id, } hook_cfg_path = _hook_config_path(Path.cwd()) @@ -153,8 +123,9 @@ async def _open_kitchen_handler() -> str | None: ctx = _get_ctx() ctx.gate.enable() - ctx.kitchen_id = str(uuid4()) + ctx.kitchen_id = os.environ.get("AUTOSKILLIT_CAMPAIGN_ID") or str(uuid4()) ctx.active_recipe_packs = frozenset() + ctx.active_recipe_features = frozenset() logger.info("open_kitchen", gate_state="open", kitchen_id=ctx.kitchen_id) try: @@ -183,19 +154,46 @@ async def _open_kitchen_handler() -> str | None: logger.warning("open_kitchen_failure", stage="start_quota_refresh", exc_info=True) return _kitchen_failure_envelope(exc, stage="start_quota_refresh") + try: + from autoskillit.core import register_active_kitchen # noqa: PLC0415 + + register_active_kitchen(ctx.kitchen_id, os.getpid(), str(Path.cwd())) + except Exception: + logger.warning("open_kitchen_registry_failed", exc_info=True) + return None -async def _redisable_subsets(ctx: Context, disabled: list[str]) -> None: - """Re-disable subset-tagged tools after enabling kitchen. +async def _redisable_subsets( + ctx: Context, + disabled: list[str], + features: dict[str, bool] | None = None, + *, + experimental_enabled: bool = False, +) -> None: + """Re-disable subset-tagged and feature-disabled tools after enabling kitchen. + + Pass 1 (existing): Re-disable config-disabled subset tags so dual-tagged tools + (e.g. kitchen+github) that are server-disabled are not accidentally revealed. - REQ-VIS-008: FastMCP session rules override server rules; enable_components(kitchen) - would otherwise reveal dual-tagged tools (e.g. kitchen+github) that are server-disabled. - Later session rules win, so these disables correctly override the kitchen enable. + Pass 2: Suppress tool tags for disabled features via `_collect_disabled_feature_tags`. + Shared tools with kitchen-core retain visibility via the kitchen-core tag + (FastMCP union model). + + ``features`` defaults to ``None`` (treated as ``{}``, i.e. all features use + ``FeatureDef.default_enabled``). Pass ``config.features`` from the call site. """ + # Pass 1: subset re-disable (existing) for subset in disabled: await ctx.disable_components(tags={subset}) + # Pass 2: feature gate — suppress tool tags for disabled features + _features = features or {} + for tag in _collect_disabled_feature_tags( + _features, experimental_enabled=experimental_enabled + ): + await ctx.disable_components(tags={tag}) + def _close_kitchen_handler() -> None: """Clear the tools-enabled flag. Extracted for testability.""" @@ -206,37 +204,84 @@ def _close_kitchen_handler() -> None: ctx.quota_refresh_task.cancel() ctx.quota_refresh_task = None ctx.gate.disable() + try: + from autoskillit.core import unregister_active_kitchen # noqa: PLC0415 + + unregister_active_kitchen(ctx.kitchen_id) + except Exception: + logger.warning("close_kitchen_registry_failed", exc_info=True) ctx.active_recipe_packs = None + ctx.active_recipe_features = None + ctx.recipe_name = "" + ctx.recipe_content_hash = "" + ctx.recipe_composite_hash = "" + ctx.recipe_version = "" logger.info("close_kitchen", gate_state="closed") hook_cfg_path = _hook_config_path(Path.cwd()) try: hook_cfg_path.unlink(missing_ok=True) except OSError: logger.warning("hook_config_remove_failed", path=str(hook_cfg_path)) + review_gate_path = Path.cwd() / ".autoskillit" / "temp" / "review_gate_state.json" + try: + try: + state = json.loads(review_gate_path.read_text()) + loop_active = ( + isinstance(state, dict) + and state.get("gate") == "LOOP_REQUIRED" + and not state.get("check_review_loop_called", False) + ) + except (json.JSONDecodeError, OSError) as exc: + logger.debug( + "review_gate_state_read_failed", path=str(review_gate_path), error=str(exc) + ) + loop_active = False + if loop_active: + logger.warning( + "close_kitchen_review_gate_preserved", + path=str(review_gate_path), + reason="active_review_loop", + ) + else: + review_gate_path.unlink(missing_ok=True) + except OSError: + logger.warning("review_gate_state_remove_failed", path=str(review_gate_path)) @mcp.resource("recipe://{name}") def get_recipe(name: str) -> str: """Return recipe YAML for the orchestrating agent to follow.""" - match = _find_recipe(name, Path.cwd()) + from autoskillit.server._state import _get_ctx_or_none + + ctx = _get_ctx_or_none() + match = ctx.recipes.find(name, Path.cwd()) if ctx and ctx.recipes else None if match is None: return json.dumps({"error": f"No recipe named '{name}'."}) return match.path.read_text() -def _build_tool_category_listing() -> str: +def _build_tool_category_listing( + features: dict[str, bool], *, experimental_enabled: bool = False +) -> str: """Return a formatted string listing all tool categories.""" lines = [] - for name, tools in _DISPLAY_CATEGORIES: + for name, tools in iter_display_categories( + features, experimental_enabled=experimental_enabled + ): lines.append(f" {name}: {', '.join(tools)}") return "\n".join(lines) -@mcp.tool(tags={"autoskillit"}, annotations={"readOnlyHint": True}) +@mcp.tool( + tags={"autoskillit"}, + annotations={"readOnlyHint": True}, + meta={"anthropic/maxResultSizeChars": 100_000, "anthropic/alwaysLoad": True}, +) @track_response_size("open_kitchen") async def open_kitchen( name: str | None = None, overrides: dict[str, str] | None = None, + ingredients_only: bool = False, ctx: Context = CurrentContext(), ) -> str: """Open the AutoSkillit kitchen for service. @@ -248,12 +293,17 @@ async def open_kitchen( name: Optional recipe name to load immediately after opening. overrides: Optional dict of ingredient name → value to override recipe defaults. Use to activate hidden features (e.g., ``{"sprint_mode": "true"}``). + ingredients_only: When True and name is provided, return only the ingredient + schema (ingredients_table, validity, suggestions) without the full recipe + content, orchestration rules, or sous-chef discipline. Use for dispatch + workflows where the caller needs ingredient discovery but not pipeline + execution context. Never raises. """ try: # Headless guard — wrap denial in envelope shape - if (h := _require_not_headless("open_kitchen")) is not None: + if (h := _require_orchestrator_exact("open_kitchen")) is not None: parsed_h = json.loads(h) return json.dumps( { @@ -283,13 +333,22 @@ async def open_kitchen( return _kitchen_failure_envelope(exc, stage="enable_components") try: - await _redisable_subsets(ctx, disabled_subsets) + _kctx = _get_ctx() + await _redisable_subsets( + ctx, + disabled_subsets, + _kctx.config.features, + experimental_enabled=_kctx.config.experimental_enabled, + ) except Exception as exc: logger.warning("open_kitchen_failure", stage="redisable_subsets", exc_info=True) return _kitchen_failure_envelope(exc, stage="redisable_subsets") _forbidden_list = ", ".join(PIPELINE_FORBIDDEN_TOOLS) - _categories = _build_tool_category_listing() + _ctx = _get_ctx() + _categories = _build_tool_category_listing( + _ctx.config.features, experimental_enabled=_ctx.config.experimental_enabled + ) if name is not None: tool_ctx = _get_ctx() @@ -329,6 +388,18 @@ async def open_kitchen( return _kitchen_failure_envelope(exc, stage="load_and_validate") tool_ctx.active_recipe_packs = frozenset(result.get("requires_packs", [])) + tool_ctx.active_recipe_features = frozenset(result.get("requires_features", [])) + tool_ctx.recipe_name = name + tool_ctx.recipe_content_hash = result.get("content_hash", "") + tool_ctx.recipe_composite_hash = result.get("composite_hash", "") + tool_ctx.recipe_version = result.get("recipe_version") or "" + + composite = result.get("composite_hash", "") + from autoskillit.server._state import _check_rerun # noqa: PLC0415 + + rerun_suggestion = _check_rerun(tool_ctx.config.linux_tracing.log_dir, composite) + if rerun_suggestion: + result.setdefault("suggestions", []).append(rerun_suggestion) try: recipe_info = tool_ctx.recipes.find(name, Path.cwd()) @@ -346,6 +417,11 @@ async def open_kitchen( result["kitchen"] = "open" result["version"] = __version__ + if ingredients_only: + result.pop("content", None) + result.pop("orchestration_rules", None) + result.pop("stop_step_semantics", None) + if "ingredients_table" not in result or not result["ingredients_table"]: result["ingredients_table"] = None @@ -356,6 +432,7 @@ async def open_kitchen( return _kitchen_failure_envelope(exc, stage="hook_diagnostic") if warning: result["hook_warning"] = warning.strip() + return json.dumps(result) text = ( @@ -413,7 +490,9 @@ async def open_kitchen( return _kitchen_failure_envelope(exc, stage="unhandled") -@mcp.tool(tags={"autoskillit"}, annotations={"readOnlyHint": True}) +@mcp.tool( + tags={"autoskillit"}, annotations={"readOnlyHint": True}, meta={"anthropic/alwaysLoad": True} +) @track_response_size("close_kitchen") async def close_kitchen(ctx: Context = CurrentContext()) -> str: """Close the AutoSkillit kitchen. @@ -421,7 +500,7 @@ async def close_kitchen(ctx: Context = CurrentContext()) -> str: Never raises. """ try: - if (h := _require_not_headless("close_kitchen")) is not None: + if (h := _require_orchestrator_exact("close_kitchen")) is not None: return h _close_kitchen_handler() await ctx.reset_visibility() @@ -431,7 +510,9 @@ async def close_kitchen(ctx: Context = CurrentContext()) -> str: return json.dumps({"success": False, "error": f"{type(exc).__name__}: {exc}"}) -@mcp.tool(tags={"autoskillit"}) +@mcp.tool( + tags={"autoskillit"}, annotations={"readOnlyHint": True}, meta={"anthropic/alwaysLoad": True} +) @track_response_size("disable_quota_guard") async def disable_quota_guard() -> str: """Disable the quota guard for the remainder of this kitchen session. @@ -446,7 +527,7 @@ async def disable_quota_guard() -> str: Never raises. """ try: - if (h := _require_not_headless("disable_quota_guard")) is not None: + if (h := _require_orchestrator_exact("disable_quota_guard")) is not None: return h hook_cfg_path = _hook_config_path(Path.cwd()) if not hook_cfg_path.exists(): @@ -481,3 +562,71 @@ async def disable_quota_guard() -> str: except Exception as exc: logger.error("disable_quota_guard unhandled exception", exc_info=True) return json.dumps({"success": False, "error": f"{type(exc).__name__}: {exc}"}) + + +def _find_session_id_for_reload(cwd: Path) -> str | None: + """Return the session_id to use for reload; kitchen marker preferred, mtime fallback.""" + from autoskillit.core import get_state_dir, is_marker_fresh, read_marker + + state_dir = get_state_dir() + if state_dir.is_dir(): + + def _safe_mtime(p: Path) -> float: + try: + return p.stat().st_mtime + except OSError: + return 0.0 + + candidates = sorted(state_dir.glob("*.json"), key=_safe_mtime, reverse=True) + for p in candidates: + marker = read_marker(p.stem) + if marker is not None and is_marker_fresh(marker): + return marker.session_id + return find_latest_session_id(str(cwd)) + + +def _write_reload_sentinel(cwd: Path, session_id: str) -> None: + """Atomically write a reload sentinel file for session_id.""" + sentinel_path = cwd / ".autoskillit" / "temp" / "reload_sentinel" / f"{session_id}.json" + payload = json.dumps({"session_id": session_id, "requested_at": datetime.now(UTC).isoformat()}) + atomic_write(sentinel_path, payload) + + +def _reload_session_handler() -> dict[str, str]: + """Core logic for the reload_session tool — testable without FastMCP.""" + cwd = Path.cwd() + session_id = _find_session_id_for_reload(cwd) + if not session_id: + raise ValueError( + "Cannot determine session ID. Ensure open_kitchen was called, " + "or that a Claude Code session JSONL exists for this project." + ) + _write_reload_sentinel(cwd, session_id) + return { + "status": "reload_requested", + "session_id": session_id, + "next_action": ( + "Run /exit now. The parent autoskillit process will re-launch " + "with --resume and full wrapper environment." + ), + } + + +@mcp.tool( + tags={"autoskillit"}, annotations={"readOnlyHint": True}, meta={"anthropic/alwaysLoad": True} +) +@track_response_size("reload_session") +async def reload_session() -> str: + """Signal the parent autoskillit process to reload this session with the full + wrapper environment intact and resume the conversation. + + After calling this tool, run /exit to allow the parent process to detect the + reload request and re-launch claude with --resume . + + Never raises. + """ + try: + return json.dumps(_reload_session_handler()) + except Exception as exc: + logger.error("reload_session unhandled exception", exc_info=True) + return json.dumps({"status": "error", "error": f"{type(exc).__name__}: {exc}"}) diff --git a/src/autoskillit/server/tools_pr_ops.py b/src/autoskillit/server/tools/tools_pr_ops.py similarity index 74% rename from src/autoskillit/server/tools_pr_ops.py rename to src/autoskillit/server/tools/tools_pr_ops.py index 556273e884..17aaf7452c 100644 --- a/src/autoskillit/server/tools_pr_ops.py +++ b/src/autoskillit/server/tools/tools_pr_ops.py @@ -2,20 +2,20 @@ from __future__ import annotations +import asyncio import json +import time +from pathlib import Path import structlog from fastmcp import Context from fastmcp.dependencies import CurrentContext -from autoskillit.core import get_logger +from autoskillit.core import atomic_write, get_logger from autoskillit.server import mcp -from autoskillit.server.helpers import ( - _notify, - _require_enabled, - _run_subprocess, - track_response_size, -) +from autoskillit.server._guards import _require_enabled +from autoskillit.server._notify import _notify, track_response_size +from autoskillit.server._subprocess import _run_subprocess logger = get_logger(__name__) @@ -52,14 +52,50 @@ async def _close_issues_sequentially( """Run gh issue close for each number; return (closed, failed) lists.""" closed: list[int] = [] failed: list[int] = [] - for num in issue_numbers: - cmd = ["gh", "issue", "close", str(num)] - if comment: - cmd.extend(["--comment", comment]) - rc, _, _ = await _run_subprocess(cmd, cwd=cwd, timeout=30) - if rc == 0: - closed.append(num) - else: + for i, num in enumerate(issue_numbers): + if i > 0: + await asyncio.sleep(1) + try: + if comment: + rc, body_out, _ = await _run_subprocess( + ["gh", "issue", "view", str(num), "--json", "body", "--jq", ".body"], + cwd=cwd, + timeout=30, + ) + if rc != 0: + failed.append(num) + continue + current_body = body_out.strip() + if current_body == "null": + current_body = "" + new_body = current_body + f"\n\n---\n\n## Closing Note\n\n{comment}" + + temp_dir = Path(cwd) / ".autoskillit" / "temp" / "bulk-close-issues" + temp_dir.mkdir(parents=True, exist_ok=True) + temp_file = temp_dir / f"{num}_{int(time.time() * 1000)}_close_body.md" + atomic_write(temp_file, new_body) + + rc2, _, _ = await _run_subprocess( + ["gh", "issue", "edit", str(num), "--body-file", str(temp_file)], + cwd=cwd, + timeout=30, + ) + if rc2 != 0: + failed.append(num) + continue + await asyncio.sleep(1) + + rc3, _, _ = await _run_subprocess( + ["gh", "issue", "close", str(num)], + cwd=cwd, + timeout=30, + ) + if rc3 == 0: + closed.append(num) + else: + failed.append(num) + except Exception: + logger.warning("Failed to close issue %s", num, exc_info=True) failed.append(num) return closed, failed diff --git a/src/autoskillit/server/tools_recipe.py b/src/autoskillit/server/tools/tools_recipe.py similarity index 86% rename from src/autoskillit/server/tools_recipe.py rename to src/autoskillit/server/tools/tools_recipe.py index 4accbf1e3e..234c8959e0 100644 --- a/src/autoskillit/server/tools_recipe.py +++ b/src/autoskillit/server/tools/tools_recipe.py @@ -13,18 +13,18 @@ from autoskillit.core import get_logger, temp_dir_display_str from autoskillit.pipeline import GATED_TOOLS, UNGATED_TOOLS # noqa: F401 from autoskillit.server import mcp +from autoskillit.server._guards import _require_enabled +from autoskillit.server._misc import _apply_triage_gate +from autoskillit.server._notify import _notify, track_response_size from autoskillit.server._state import _get_ctx_or_none -from autoskillit.server.helpers import ( - _apply_triage_gate, - _notify, - _require_enabled, - track_response_size, -) logger = get_logger(__name__) -@mcp.tool(tags={"autoskillit", "kitchen"}, annotations={"readOnlyHint": True}) +@mcp.tool( + tags={"autoskillit", "kitchen-core", "fleet-dispatch"}, + annotations={"readOnlyHint": True}, +) @track_response_size("list_recipes") async def list_recipes() -> str: """List available recipes from .autoskillit/recipes/. @@ -32,8 +32,8 @@ async def list_recipes() -> str: Returns a JSON array of recipes with name, description, and summary. Recipes are YAML workflow definitions that agents follow as orchestration instructions. Use load_recipe to load a specific recipe. - To create a new recipe, use the /autoskillit:write-recipe skill. - To generate recipes as part of project onboarding, use /autoskillit:setup-project. + To create a new recipe, use the /write-recipe skill. + To generate recipes as part of project onboarding, use /setup-project. IMPORTANT: Recipes are NOT slash commands. They cannot be invoked as /autoskillit:. They are loaded via load_recipe and executed @@ -51,17 +51,23 @@ async def list_recipes() -> str: structlog.contextvars.bind_contextvars(tool="list_recipes") tool_ctx = _get_ctx_or_none() if tool_ctx is None or tool_ctx.recipes is None: - return json.dumps([]) - result = tool_ctx.recipes.list_all(Path.cwd()) + return json.dumps({"error": "kitchen not open — call open_kitchen first"}) + result = tool_ctx.recipes.list_all(Path.cwd(), features=tool_ctx.config.features) return json.dumps(result) except Exception: logger.error("list_recipes unhandled exception", exc_info=True) - return json.dumps([]) + return json.dumps({"error": "internal error listing recipes — see server logs"}) -@mcp.tool(tags={"autoskillit", "kitchen"}, annotations={"readOnlyHint": True}) +@mcp.tool( + tags={"autoskillit", "kitchen-core", "fleet-dispatch"}, + annotations={"readOnlyHint": True}, + meta={"anthropic/maxResultSizeChars": 100_000}, +) @track_response_size("load_recipe") -async def load_recipe(name: str, overrides: dict[str, str] | None = None) -> str: +async def load_recipe( + name: str, overrides: dict[str, str] | None = None, ingredients_only: bool = False +) -> str: """Load a recipe by name and return its raw YAML content. The YAML follows the recipe schema (ingredients, steps with tool/action, @@ -86,7 +92,7 @@ async def load_recipe(name: str, overrides: dict[str, str] | None = None) -> str diagram, or invoke the /render-recipe skill. The canonical visual grammar is defined in the render-recipe SKILL.md — do not attempt to render inline. (See: .claude/skills/render-recipe/SKILL.md) - 3. If the user requests changes, use the /autoskillit:write-recipe skill + 3. If the user requests changes, use the /write-recipe skill to apply modifications. That skill has the complete schema, validation rules, and formatting constraints needed for correct changes. Do NOT edit the YAML file directly — always delegate modifications to write-recipe. @@ -158,7 +164,13 @@ async def load_recipe(name: str, overrides: dict[str, str] | None = None) -> str - classify_fix: "error" key present in response - push_to_remote: {"success": false} (also has "error" and "stderr" keys) - To CREATE a new recipe, use the /autoskillit:write-recipe skill. + OPTIONAL STEP SEMANTICS: + - optional: true means the step is SKIPPED when its skip_when_false ingredient + is false. When the ingredient is true (or absent), the step is MANDATORY. + - NEVER skip a step for any other reason (PR size, diff triviality, etc.). + - A running optional step that returns success: false MUST follow on_failure. + + To CREATE a new recipe, use the /write-recipe skill. This tool is for loading and executing existing recipes. IMPORTANT: Recipes are NOT slash commands. They cannot be invoked @@ -196,13 +208,18 @@ async def load_recipe(name: str, overrides: dict[str, str] | None = None) -> str temp_dir_relpath=temp_dir_display_str(tool_ctx.config.workspace.temp_dir), ) recipe_info = tool_ctx.recipes.find(name, Path.cwd()) - return json.dumps(await _apply_triage_gate(result, name, recipe_info=recipe_info)) + result = await _apply_triage_gate(result, name, recipe_info=recipe_info) + if ingredients_only: + result.pop("content", None) + result.pop("orchestration_rules", None) + result.pop("stop_step_semantics", None) + return json.dumps(result) except Exception as exc: logger.error("load_recipe unhandled exception", exc_info=True) return json.dumps({"success": False, "error": f"{type(exc).__name__}: {exc}"}) -@mcp.tool(tags={"autoskillit", "kitchen"}, annotations={"readOnlyHint": True}) +@mcp.tool(tags={"autoskillit", "kitchen", "kitchen-core"}, annotations={"readOnlyHint": True}) @track_response_size("validate_recipe") async def validate_recipe(script_path: str) -> str: """Validate a recipe YAML file against the recipe schema. @@ -210,11 +227,11 @@ async def validate_recipe(script_path: str) -> str: Parses the file, checks all validation rules (name, steps, routing, retry fields, ingredient references), and returns structured results. Use after generating or modifying a recipe (via write-recipe) - to confirm it is valid. The /autoskillit:write-recipe skill + to confirm it is valid. The /write-recipe skill calls this tool automatically after generating a recipe. When validation fails ({"valid": false}), do NOT edit the YAML file - directly to fix errors. Use the /autoskillit:write-recipe skill + directly to fix errors. Use the /write-recipe skill to apply corrections — it has the complete schema, validation rules, and formatting constraints needed for correct modifications. @@ -246,7 +263,7 @@ async def validate_recipe(script_path: str) -> str: return json.dumps({"valid": False, "errors": [f"{type(exc).__name__}: {exc}"]}) -@mcp.tool(tags={"autoskillit", "kitchen"}, annotations={"readOnlyHint": True}) +@mcp.tool(tags={"autoskillit", "kitchen", "kitchen-core"}, annotations={"readOnlyHint": True}) @track_response_size("migrate_recipe") async def migrate_recipe(name: str, ctx: Context = CurrentContext()) -> str: """Apply pending migration notes to a recipe file. diff --git a/src/autoskillit/server/tools_status.py b/src/autoskillit/server/tools/tools_status.py similarity index 79% rename from src/autoskillit/server/tools_status.py rename to src/autoskillit/server/tools/tools_status.py index a2f291c5b9..d061df1e59 100644 --- a/src/autoskillit/server/tools_status.py +++ b/src/autoskillit/server/tools/tools_status.py @@ -14,13 +14,9 @@ from autoskillit.core import atomic_write, get_logger from autoskillit.pipeline import TelemetryFormatter from autoskillit.server import mcp -from autoskillit.server.helpers import ( - _notify, - _require_enabled, - resolve_log_dir, - track_response_size, - write_telemetry_clear_marker, -) +from autoskillit.server._guards import _require_enabled +from autoskillit.server._misc import resolve_log_dir, write_telemetry_clear_marker +from autoskillit.server._notify import _notify, track_response_size logger = get_logger(__name__) @@ -32,7 +28,7 @@ def _get_log_root() -> Path: return resolve_log_dir(_get_ctx().config.linux_tracing.log_dir) -@mcp.tool(tags={"autoskillit", "kitchen"}, annotations={"readOnlyHint": True}) +@mcp.tool(tags={"autoskillit", "kitchen", "kitchen-core"}, annotations={"readOnlyHint": True}) @track_response_size("kitchen_status") async def kitchen_status() -> str: """Return version health and configuration status for the running server. @@ -79,7 +75,10 @@ async def kitchen_status() -> str: return json.dumps({"success": False, "error": f"{type(exc).__name__}: {exc}"}) -@mcp.tool(tags={"autoskillit", "kitchen"}, annotations={"readOnlyHint": True}) +@mcp.tool( + tags={"autoskillit", "kitchen-core", "fleet"}, + annotations={"readOnlyHint": True}, +) @track_response_size("get_pipeline_report") async def get_pipeline_report(clear: bool = False) -> str: """Return accumulated run_skill failures since last clear. @@ -148,7 +147,10 @@ def _merge_wall_clock_seconds(steps: list[dict], timing_report: list[dict]) -> l return steps -@mcp.tool(tags={"autoskillit", "kitchen", "telemetry"}, annotations={"readOnlyHint": True}) +@mcp.tool( + tags={"autoskillit", "kitchen-core", "telemetry", "fleet"}, + annotations={"readOnlyHint": True}, +) @track_response_size("get_token_summary") async def get_token_summary(clear: bool = False, format: str = "json", order_id: str = "") -> str: """Return accumulated run_skill token usage grouped by step name. @@ -193,8 +195,17 @@ async def get_token_summary(clear: bool = False, format: str = "json", order_id: write_telemetry_clear_marker(_get_log_root()) except Exception: logger.debug("write_telemetry_clear_marker failed", exc_info=True) + model_totals = ctx.token_log.compute_model_totals(order_id=order_id) if format == "table": - return TelemetryFormatter.format_token_table(steps, total) + token_table = TelemetryFormatter.format_token_table(steps, total) + efficiency_table = TelemetryFormatter.format_efficiency_table(steps, total) + model_table = TelemetryFormatter.format_model_table(model_totals) + result = token_table + if efficiency_table: + result += "\n\n" + efficiency_table + if model_table: + result += "\n\n" + model_table + return result return json.dumps( { "steps": steps, @@ -203,6 +214,7 @@ async def get_token_summary(clear: bool = False, format: str = "json", order_id: "steps": mcp_report, "total": mcp_total, }, + "model_totals": model_totals, } ) except Exception as exc: @@ -210,7 +222,10 @@ async def get_token_summary(clear: bool = False, format: str = "json", order_id: return json.dumps({"success": False, "error": f"{type(exc).__name__}: {exc}"}) -@mcp.tool(tags={"autoskillit", "kitchen", "telemetry"}, annotations={"readOnlyHint": True}) +@mcp.tool( + tags={"autoskillit", "kitchen-core", "telemetry", "fleet"}, + annotations={"readOnlyHint": True}, +) @track_response_size("get_timing_summary") async def get_timing_summary(clear: bool = False, format: str = "json", order_id: str = "") -> str: """Return accumulated wall-clock timing grouped by step name. @@ -251,6 +266,92 @@ async def get_timing_summary(clear: bool = False, format: str = "json", order_id return json.dumps({"success": False, "error": f"{type(exc).__name__}: {exc}"}) +@mcp.tool( + tags={"autoskillit", "kitchen", "kitchen-core", "telemetry"}, + annotations={"readOnlyHint": True}, +) +@track_response_size("analyze_tool_sequences") +async def analyze_tool_sequences( + recipe: str = "", + format: str = "table", + top_n: int = 20, + min_count: int = 1, +) -> str: + """Analyze cross-session tool call sequences and return a DFG summary. + + Args: + recipe: Filter to a specific recipe name (empty = all recipes). + format: Output format — "table", "mermaid", or "dot". + top_n: Limit rendering to top-N bigrams by frequency. + min_count: Exclude bigrams with count below this threshold. + + Returns JSON with fields: session_count, recipe_count, top_bigrams, + rendering (the formatted DFG string). + Never raises. + """ + if (gate := _require_enabled()) is not None: + return gate + try: + structlog.contextvars.clear_contextvars() + with structlog.contextvars.bound_contextvars(tool="analyze_tool_sequences"): + _valid_formats = {"table", "mermaid", "dot"} + if format not in _valid_formats: + return json.dumps( + { + "success": False, + "error": ( + f"Invalid format '{format}'; must be one of {sorted(_valid_formats)}" + ), + } + ) + if top_n < 1: + return json.dumps({"success": False, "error": f"top_n must be >= 1, got {top_n}"}) + if min_count < 1: + return json.dumps( + {"success": False, "error": f"min_count must be >= 1, got {min_count}"} + ) + from autoskillit.core import ( + compute_analysis, + filter_sessions_by_recipe, + format_top_bigrams, + parse_sessions_from_summary_dir, + render_adjacency_table, + render_dot, + render_mermaid, + ) + + log_root = _get_log_root() + sessions = list(parse_sessions_from_summary_dir(log_root)) + if recipe: + sessions = filter_sessions_by_recipe(sessions, recipe) + result = compute_analysis(sessions) + dfg = ( + result.global_dfg + if not recipe + else result.by_recipe.get(recipe, result.global_dfg) + ) + + if format == "mermaid": + rendering = render_mermaid(dfg, min_count=min_count, top_n=top_n) + elif format == "dot": + rendering = render_dot(dfg, min_count=min_count, top_n=top_n) + else: + rendering = render_adjacency_table(dfg, top_n=top_n) + + return json.dumps( + { + "success": True, + "session_count": result.session_count, + "recipe_count": len(result.by_recipe), + "top_bigrams": format_top_bigrams(dfg, top_n, min_count), + "rendering": rendering, + } + ) + except Exception as exc: + logger.error("analyze_tool_sequences unhandled exception", exc_info=True) + return json.dumps({"success": False, "error": f"{type(exc).__name__}: {exc}"}) + + def _read_quota_events(log_root: Path, n: int) -> tuple[list[dict], int]: """Read the last n events from quota_events.jsonl (most recent first). @@ -275,7 +376,10 @@ def _read_quota_events(log_root: Path, n: int) -> tuple[list[dict], int]: return list(reversed(events))[:n], total # most recent first -@mcp.tool(tags={"autoskillit", "kitchen", "telemetry"}, annotations={"readOnlyHint": True}) +@mcp.tool( + tags={"autoskillit", "kitchen-core", "telemetry", "fleet"}, + annotations={"readOnlyHint": True}, +) @track_response_size("get_quota_events") async def get_quota_events(n: int = 50) -> str: """Return the most recent quota guard events from the diagnostic log. @@ -314,7 +418,10 @@ async def get_quota_events(n: int = 50) -> str: return json.dumps({"success": False, "error": f"{type(exc).__name__}: {exc}"}) -@mcp.tool(tags={"autoskillit", "kitchen", "telemetry"}, annotations={"readOnlyHint": True}) +@mcp.tool( + tags={"autoskillit", "kitchen", "kitchen-core", "telemetry"}, + annotations={"readOnlyHint": True}, +) @track_response_size("write_telemetry_files") async def write_telemetry_files( output_dir: str, @@ -365,9 +472,12 @@ async def write_telemetry_files( token_total = tool_ctx.token_log.compute_total() token_path = out / "token_summary.md" - atomic_write( - token_path, TelemetryFormatter.format_token_table(token_steps, token_total) - ) + token_content = TelemetryFormatter.format_token_table(token_steps, token_total) + model_totals = tool_ctx.token_log.compute_model_totals() + model_table = TelemetryFormatter.format_model_table(model_totals) + if model_table: + token_content += "\n\n" + model_table + atomic_write(token_path, token_content) timing_path = out / "timing_summary.md" atomic_write( @@ -396,7 +506,7 @@ async def write_telemetry_files( return json.dumps({"success": False, "error": f"{type(exc).__name__}: {exc}"}) -@mcp.tool(tags={"autoskillit", "kitchen"}, annotations={"readOnlyHint": True}) +@mcp.tool(tags={"autoskillit", "kitchen", "kitchen-core"}, annotations={"readOnlyHint": True}) @track_response_size("read_db") async def read_db( db_path: str, diff --git a/src/autoskillit/server/tools_workspace.py b/src/autoskillit/server/tools/tools_workspace.py similarity index 84% rename from src/autoskillit/server/tools_workspace.py rename to src/autoskillit/server/tools/tools_workspace.py index 918ee92bff..0822acead0 100644 --- a/src/autoskillit/server/tools_workspace.py +++ b/src/autoskillit/server/tools/tools_workspace.py @@ -13,17 +13,18 @@ from autoskillit.core import get_logger, truncate_text from autoskillit.server import mcp -from autoskillit.server.helpers import ( - _notify, - _require_enabled, - _run_subprocess, - track_response_size, -) +from autoskillit.server._guards import _require_enabled +from autoskillit.server._notify import _notify, track_response_size +from autoskillit.server._subprocess import _run_subprocess logger = get_logger(__name__) -@mcp.tool(tags={"autoskillit", "kitchen", "headless"}, annotations={"readOnlyHint": True}) +@mcp.tool( + tags={"autoskillit", "kitchen", "kitchen-core", "headless"}, annotations={"readOnlyHint": True} +) +# No _require_enabled() — headless skill sessions need test_check without opening kitchen. +# The kitchen tag governs visibility only; the headless tag provides access in SKILL sessions. @track_response_size("test_check") async def test_check( worktree_path: str, @@ -43,6 +44,8 @@ async def test_check( Args: worktree_path: Path to the git worktree to run tests in. step_name: Optional YAML step key for wall-clock timing accumulation. + Has no effect on test filtering — filter behavior is controlled by + config.test_check.filter_mode. Never raises. """ @@ -78,13 +81,22 @@ async def test_check( extra={"worktree": worktree_path}, ) - return json.dumps( - { - "passed": test_result.passed, - "stdout": truncate_text(test_result.stdout), - "stderr": truncate_text(test_result.stderr), - } - ) + response = { + "passed": test_result.passed, + "stdout": truncate_text(test_result.stdout), + "stderr": truncate_text(test_result.stderr), + } + if test_result.duration_seconds is not None: + response["duration_seconds"] = round(test_result.duration_seconds, 2) + if test_result.filter_mode is not None: + response["filter_mode"] = test_result.filter_mode + if test_result.tests_selected is not None: + response["tests_selected"] = test_result.tests_selected + if test_result.tests_deselected is not None: + response["tests_deselected"] = test_result.tests_deselected + if test_result.full_run_reason is not None: + response["full_run_reason"] = test_result.full_run_reason + return json.dumps(response) finally: if step_name: tool_ctx.timing_log.record(step_name, time.monotonic() - _start) @@ -93,7 +105,7 @@ async def test_check( return json.dumps({"passed": False, "error": f"{type(exc).__name__}: {exc}"}) -@mcp.tool(tags={"autoskillit", "kitchen"}, annotations={"readOnlyHint": True}) +@mcp.tool(tags={"autoskillit", "kitchen", "kitchen-core"}, annotations={"readOnlyHint": True}) @track_response_size("reset_test_dir") async def reset_test_dir( test_dir: str, @@ -175,7 +187,7 @@ async def reset_test_dir( return json.dumps({"error": f"{type(exc).__name__}: {exc}"}) -@mcp.tool(tags={"autoskillit", "kitchen"}, annotations={"readOnlyHint": True}) +@mcp.tool(tags={"autoskillit", "kitchen", "kitchen-core"}, annotations={"readOnlyHint": True}) @track_response_size("reset_workspace") async def reset_workspace(test_dir: str, ctx: Context = CurrentContext()) -> str: """Runs a configured reset command then deletes directory contents, diff --git a/src/autoskillit/server/tools_ci.py b/src/autoskillit/server/tools_ci.py deleted file mode 100644 index c221e46350..0000000000 --- a/src/autoskillit/server/tools_ci.py +++ /dev/null @@ -1,587 +0,0 @@ -"""MCP tool handlers: wait_for_ci (gated), get_ci_status (ungated), set_commit_status (gated), -wait_for_merge_queue (gated). -""" - -from __future__ import annotations - -import json -import time -from typing import Literal - -import httpx -import structlog -from fastmcp import Context -from fastmcp.dependencies import CurrentContext - -from autoskillit.core import CIRunScope, get_logger -from autoskillit.server import mcp -from autoskillit.server.helpers import ( - _notify, - _require_enabled, - _run_subprocess, - fetch_repo_merge_state, - infer_repo_from_remote, - track_response_size, -) - -logger = get_logger(__name__) - - -@mcp.tool(tags={"autoskillit", "kitchen", "ci"}, annotations={"readOnlyHint": True}) -@track_response_size("wait_for_ci") -async def wait_for_ci( - branch: str, - repo: str | None = None, - remote_url: str = "", - head_sha: str | None = None, - workflow: str | None = None, - event: str | None = None, - timeout_seconds: int = 300, - cwd: str = ".", - step_name: str = "", - ctx: Context = CurrentContext(), -) -> str: - """Wait for a GitHub Actions CI run to complete on the given branch. - - Uses a three-phase algorithm (look-back, poll, wait) that eliminates - the race condition where CI completes before polling starts. - - Args: - branch: Git branch name to watch CI for. - repo: GitHub owner/repo (e.g. "owner/repo"). If omitted, inferred - from git remote in cwd. - remote_url: Full GitHub remote URL (e.g. "https://github.com/owner/repo.git"). - Parsed to owner/repo before inference. Takes priority over repo - when both are provided. - head_sha: Specific commit SHA to match. If omitted, inferred from - HEAD in cwd. - workflow: Workflow filename to filter runs (e.g. "tests.yml"). If - omitted, falls back to the project-level ci.workflow config. - event: GitHub trigger event to filter runs (e.g. "push", "pull_request"). - If omitted, falls back to the project-level ci.event config. - timeout_seconds: Maximum time to wait (default 300s). - cwd: Working directory for git operations. - step_name: Optional YAML step key for wall-clock timing accumulation. - - Returns: - JSON with run_id, conclusion ("success", "failure", "cancelled", - "action_required", "timed_out", "no_runs", "error", "unknown"), - and failed_jobs list. Billing limit errors surface as - conclusion="action_required" with failed_jobs=[]. - - Never raises. - """ - if (gate := _require_enabled()) is not None: - return gate - try: - _start = time.monotonic() - _timing_ctx = None - structlog.contextvars.clear_contextvars() - structlog.contextvars.bind_contextvars(tool="wait_for_ci") - logger.info("wait_for_ci", branch=branch, repo=repo or "(infer)") - - from autoskillit.server import _get_ctx - - tool_ctx = _get_ctx() - _timing_ctx = tool_ctx - if tool_ctx.ci_watcher is None: - if step_name: - tool_ctx.timing_log.record(step_name, time.monotonic() - _start) - return json.dumps( - { - "run_id": None, - "conclusion": "error", - "failed_jobs": [], - "error": "CI watcher not configured", - } - ) - - # Infer head_sha from cwd if not provided - if head_sha is None and cwd: - try: - rc, stdout, _ = await _run_subprocess( - ["git", "rev-parse", "HEAD"], cwd=cwd, timeout=5.0 - ) - if rc == 0: - head_sha = stdout.strip() - except Exception: - logger.warning("git rev-parse HEAD failed", exc_info=True) - - scope = CIRunScope( - workflow=workflow or tool_ctx.default_ci_scope.workflow, - head_sha=head_sha, - event=event or tool_ctx.default_ci_scope.event, - ) - - resolved_repo = await infer_repo_from_remote(cwd, hint=remote_url or repo or None) - - await _notify( - ctx, - "info", - f"Watching CI for branch {branch}", - "autoskillit.wait_for_ci", - extra={ - "repo": resolved_repo or "(infer)", - "head_sha": scope.head_sha or "(any)", - "workflow": scope.workflow or "(any)", - }, - ) - - try: - result = await tool_ctx.ci_watcher.wait( - branch, - repo=resolved_repo or None, - scope=scope, - timeout_seconds=timeout_seconds, - cwd=cwd, - ) - - # Include head_sha used for this CI check so orchestrators can verify - # CI results correspond to the current HEAD after a force-push. - if scope.head_sha: - result = {**result, "head_sha": scope.head_sha} - - conclusion = result.get("conclusion", "unknown") - level = "info" if conclusion == "success" else "error" - await _notify( - ctx, - level, - f"CI result: {conclusion}", - "autoskillit.wait_for_ci", - extra={"run_id": result.get("run_id")}, - ) - - return json.dumps(result) - except Exception as exc: - logger.error("wait_for_ci failed", exc_info=True) - return json.dumps({"success": False, "error": f"{type(exc).__name__}: {exc}"}) - finally: - if step_name: - tool_ctx.timing_log.record(step_name, time.monotonic() - _start) - except Exception as exc: - logger.error("wait_for_ci unhandled exception", exc_info=True) - if step_name and _timing_ctx is not None: - _timing_ctx.timing_log.record(step_name, time.monotonic() - _start) - return json.dumps({"success": False, "error": f"{type(exc).__name__}: {exc}"}) - - -@mcp.tool(tags={"autoskillit", "kitchen", "github"}, annotations={"readOnlyHint": True}) -@track_response_size("set_commit_status") -async def set_commit_status( - sha: str, - state: Literal["pending", "success", "failure", "error"], - context: str, - description: str = "", - target_url: str = "", - repo: str = "", - cwd: str = "", -) -> str: - """Post a GitHub Commit Status to a commit SHA. - - Use to implement review-first gating: post `pending` when review starts, - then `success` or `failure` when it completes. Combine with a required - status check in branch protection to block merge until the review resolves. - - Args: - sha: Full commit SHA to attach the status to. - state: One of: pending, success, failure, error. - context: Status context label (e.g. "autoskillit/ai-review"). - description: Short human-readable status description (max 140 chars). - target_url: Optional URL linking to review details. - repo: owner/repo format. Inferred from `cwd` git remote if absent. - cwd: Working directory for repo inference. Defaults to plugin_dir. - - Never raises. - """ - if (gate := _require_enabled()) is not None: - return gate - - if not sha: - return json.dumps({"success": False, "error": "sha must not be empty"}) - if not context: - return json.dumps({"success": False, "error": "context must not be empty"}) - if len(description) > 140: - return json.dumps( - { - "success": False, - "error": f"description exceeds 140 chars ({len(description)} chars)", - } - ) - - try: - from autoskillit.server import _get_ctx - - tool_ctx = _get_ctx() - effective_cwd = cwd or tool_ctx.plugin_dir or "." - - # Resolve owner/repo if not provided - owner_repo = repo - if not owner_repo: - owner_repo = await infer_repo_from_remote(effective_cwd) - if not owner_repo: - return json.dumps( - {"success": False, "error": "Could not infer owner/repo from git remote"} - ) - - cmd = [ - "gh", - "api", - "--method", - "POST", - f"/repos/{owner_repo}/statuses/{sha}", - "-f", - f"state={state}", - "-f", - f"context={context}", - "-f", - f"description={description}", - ] - if target_url: - cmd += ["-f", f"target_url={target_url}"] - - rc, _stdout, stderr = await _run_subprocess(cmd, cwd=effective_cwd, timeout=30.0) - if rc != 0: - return json.dumps({"success": False, "error": stderr}) - - return json.dumps({"success": True, "sha": sha, "state": state, "context": context}) - except Exception as exc: - logger.error("set_commit_status unhandled exception", exc_info=True) - return json.dumps({"success": False, "error": f"{type(exc).__name__}: {exc}"}) - - -@mcp.tool(tags={"autoskillit", "kitchen", "ci"}, annotations={"readOnlyHint": True}) -@track_response_size("get_ci_status") -async def get_ci_status( - branch: str | None = None, - run_id: int | None = None, - repo: str | None = None, - workflow: str | None = None, - event: str | None = None, - cwd: str = ".", -) -> str: - """Return current CI status for a branch or specific run without waiting. - - Args: - branch: Git branch name. Required if run_id is not provided. - run_id: Specific run ID to check. If provided, branch is ignored. - repo: GitHub owner/repo. If omitted, inferred from git remote in cwd. - workflow: Workflow filename to filter runs (e.g. "tests.yml"). If - omitted, falls back to the project-level ci.workflow config. - event: GitHub trigger event to filter runs (e.g. "push", "pull_request"). - If omitted, falls back to the project-level ci.event config. - cwd: Working directory for git operations. - - Returns: - JSON with runs list, each containing id, status, conclusion, failed_jobs. - - Never raises. - """ - if (gate := _require_enabled()) is not None: - return gate - try: - from autoskillit.server import _get_ctx - - tool_ctx = _get_ctx() - if tool_ctx.ci_watcher is None: - return json.dumps({"runs": [], "error": "CI watcher not configured"}) - - if branch is None and run_id is None: - return json.dumps({"runs": [], "error": "Provide branch or run_id"}) - - scope = CIRunScope( - workflow=workflow or tool_ctx.default_ci_scope.workflow, - event=event or tool_ctx.default_ci_scope.event, - ) - - result = await tool_ctx.ci_watcher.status( - branch or "", - repo=repo, - run_id=run_id, - scope=scope, - cwd=cwd, - ) - return json.dumps(result) - except Exception as exc: - logger.error("get_ci_status unhandled exception", exc_info=True) - return json.dumps({"runs": [], "error": f"{type(exc).__name__}: {exc}"}) - - -@mcp.tool(tags={"autoskillit", "kitchen", "ci"}, annotations={"readOnlyHint": False}) -@track_response_size("toggle_auto_merge") -async def toggle_auto_merge( - pr_number: int, - target_branch: str, - cwd: str, - repo: str = "", - remote_url: str = "", - ctx: Context = CurrentContext(), -) -> str: - """Disable then re-enable auto-merge for a PR to re-enroll it in the merge queue. - - Uses the same GraphQL mutation path as wait_for_merge_queue's stall recovery. - Call this when wait_for_merge_queue returns pr_state="stalled" and you want - to attempt one additional re-enrollment cycle. - - Args: - pr_number: PR number to re-enroll. - target_branch: Branch the merge queue targets (e.g. "integration"). - cwd: Working directory for git remote resolution when repo is not provided. - repo: Optional "owner/name" string. Inferred from git remote if empty. - remote_url: Full GitHub remote URL. Parsed to owner/repo before inference. - - Returns: - JSON: {"success": bool, "pr_number": int} on success, - {"success": false, "error": str} on failure. - - Never raises. - """ - if (gate := _require_enabled()) is not None: - return gate - try: - structlog.contextvars.clear_contextvars() - structlog.contextvars.bind_contextvars( - tool="toggle_auto_merge", pr_number=pr_number, target_branch=target_branch - ) - await _notify( - ctx, - "info", - f"Toggling auto-merge for PR #{pr_number} on {target_branch!r}", - "autoskillit.toggle_auto_merge", - extra={"pr_number": pr_number, "target_branch": target_branch}, - ) - - from autoskillit.server import _get_ctx - - tool_ctx = _get_ctx() - - if tool_ctx.merge_queue_watcher is None: - return json.dumps( - { - "success": False, - "error": "merge_queue_watcher not configured (missing GITHUB_TOKEN?)", - } - ) - - resolved_repo = await infer_repo_from_remote(cwd, hint=remote_url or repo or None) - - result = await tool_ctx.merge_queue_watcher.toggle( - pr_number=pr_number, - target_branch=target_branch, - repo=resolved_repo or None, - cwd=cwd, - ) - return json.dumps(result) - except Exception as exc: - logger.error("toggle_auto_merge unhandled exception", exc_info=True) - return json.dumps({"success": False, "error": f"{type(exc).__name__}: {exc}"}) - - -@mcp.tool(tags={"autoskillit", "kitchen", "ci"}, annotations={"readOnlyHint": False}) -@track_response_size("wait_for_merge_queue") -async def wait_for_merge_queue( - pr_number: int, - target_branch: str, - cwd: str, - repo: str = "", - remote_url: str = "", - timeout_seconds: int = 600, - poll_interval: int = 15, - stall_grace_period: int = 60, - max_stall_retries: int = 3, - not_in_queue_confirmation_cycles: int = 2, - max_inconclusive_retries: int = 5, - step_name: str = "", - ctx: Context = CurrentContext(), -) -> str: - """Poll a PR's progress through GitHub's merge queue until merged, ejected, or timed out. - - Args: - pr_number: PR number to monitor. - target_branch: Branch the merge queue targets (e.g. "integration"). - cwd: Working directory for git remote resolution when repo is not provided. - repo: Optional "owner/name" string. Inferred from git remote if empty. - remote_url: Full GitHub remote URL (e.g. "https://github.com/owner/repo.git"). - Parsed to owner/repo before inference. Takes priority over repo - when both are provided. - timeout_seconds: Total polling budget (default 600s). - poll_interval: Seconds between polls (default 15s). - stall_grace_period: Seconds after auto-merge is enabled before stall recovery - may trigger. Prevents intervention during normal queue processing - (default 60s). - max_stall_retries: Maximum disable/re-enable toggle attempts before declaring - the PR stalled and returning pr_state="stalled" (default 3). - not_in_queue_confirmation_cycles: Consecutive "not in queue" cycles required - before treating absence as definitive. Guards against race between - queue exit and merged=true propagation (default 2). - max_inconclusive_retries: Maximum NoPositiveSignal cycles (beyond the - confirmation window) before returning pr_state="timeout" (default 5). - step_name: Optional YAML step key for wall-clock timing accumulation. - - Returns: - JSON: { - "success": bool, - "pr_state": "merged"|"ejected"|"ejected_ci_failure"|"stalled"| - "dropped_healthy"|"timeout"|"error", - "reason": str, - "stall_retries_attempted": int, - } - - pr_state values: - merged — PR successfully merged through the queue. - ejected — PR removed from queue (conflict or other non-CI reason). - ejected_ci_failure — PR removed from queue because CI checks failed. - stalled — PR stuck in queue; max stall retries exhausted. - dropped_healthy — auto-merge disabled on a PR with no CI/conflict issues. - timeout — polling budget exhausted before a terminal state was reached. - error — watcher raised an unexpected exception. - - Never raises. - """ - if (gate := _require_enabled()) is not None: - return gate - try: - structlog.contextvars.clear_contextvars() - structlog.contextvars.bind_contextvars( - tool="wait_for_merge_queue", pr_number=pr_number, target_branch=target_branch - ) - await _notify( - ctx, - "info", - f"Waiting for PR #{pr_number} in merge queue on {target_branch!r}", - "autoskillit.wait_for_merge_queue", - extra={"pr_number": pr_number, "target_branch": target_branch}, - ) - - from autoskillit.server import _get_ctx - - tool_ctx = _get_ctx() - - if tool_ctx.merge_queue_watcher is None: - if step_name: - tool_ctx.timing_log.record(step_name, 0.0) - return json.dumps( - { - "success": False, - "pr_state": "error", - "reason": "merge_queue_watcher not configured (missing GITHUB_TOKEN?)", - } - ) - - resolved_repo = await infer_repo_from_remote(cwd, hint=remote_url or repo or None) - - _start = time.monotonic() - try: - result = await tool_ctx.merge_queue_watcher.wait( - pr_number=pr_number, - target_branch=target_branch, - repo=resolved_repo or None, - cwd=cwd, - timeout_seconds=timeout_seconds, - poll_interval=poll_interval, - stall_grace_period=stall_grace_period, - max_stall_retries=max_stall_retries, - not_in_queue_confirmation_cycles=not_in_queue_confirmation_cycles, - max_inconclusive_retries=max_inconclusive_retries, - ) - return json.dumps(result) - except Exception as exc: - logger.error("wait_for_merge_queue ci_watcher error", exc_info=True) - return json.dumps({"success": False, "error": f"{type(exc).__name__}: {exc}"}) - finally: - if step_name: - tool_ctx.timing_log.record(step_name, time.monotonic() - _start) - except Exception as exc: - logger.error("wait_for_merge_queue unhandled exception", exc_info=True) - return json.dumps({"success": False, "error": f"{type(exc).__name__}: {exc}"}) - - -@mcp.tool(tags={"autoskillit", "kitchen", "ci"}, annotations={"readOnlyHint": True}) -@track_response_size("check_repo_merge_state") -async def check_repo_merge_state( - branch: str, - cwd: str = ".", - remote_url: str = "", - step_name: str = "", - ctx: Context = CurrentContext(), -) -> str: - """Single GraphQL round-trip returning queue_available, merge_group_trigger, - auto_merge_available, and ci_event for the given repository branch. - - Consolidates the three former run_cmd shell steps in the pre_queue_gate block - into a single MCP tool call, eliminating N+2 REST/GraphQL round-trips and - making the call budget auditable by block-level semantic rules. - - Args: - branch: Target branch name to check merge queue state for. - cwd: Working directory for git remote resolution. - remote_url: Full GitHub remote URL; parsed to owner/repo if provided. - step_name: Step name for timing telemetry. - - Returns a JSON object with keys: - - ``queue_available``: branch has an active GitHub merge queue. - - ``merge_group_trigger``: a CI workflow declares the merge_group event. - - ``auto_merge_available``: repository has auto-merge enabled. - - ``ci_event``: ``"push"`` | ``"merge_group"`` | ``null`` — recommended - event to use when polling CI via wait_for_ci. - On any error, returns an error field alongside the four boolean/null defaults. - - Never raises. - """ - if (gate := _require_enabled()) is not None: - return gate - try: - from autoskillit.server import _get_ctx - - tool_ctx = _get_ctx() - _start = time.monotonic() - try: - resolved_repo = await infer_repo_from_remote(cwd, hint=remote_url or None) - if not resolved_repo or "/" not in resolved_repo: - return json.dumps( - { - "error": f"Could not resolve owner/repo from cwd={cwd!r}", - "queue_available": False, - "merge_group_trigger": False, - "auto_merge_available": False, - "ci_event": None, - } - ) - owner, repo = resolved_repo.split("/", 1) - resolved_token = ( - tool_ctx.token_factory() - if tool_ctx.token_factory is not None - else tool_ctx.config.github.token - ) - state = await fetch_repo_merge_state( - owner=owner, - repo=repo, - branch=branch, - token=resolved_token, - ) - return json.dumps(state) - except Exception as exc: - logger.error("autoskillit.check_repo_merge_state failed", exc_info=True) - envelope: dict[str, object] = { - "error": f"{type(exc).__name__}: {exc}", - "queue_available": False, - "merge_group_trigger": False, - "auto_merge_available": False, - "ci_event": None, - } - if isinstance(exc, httpx.HTTPStatusError): - envelope["http_status"] = exc.response.status_code - return json.dumps(envelope) - finally: - if step_name: - tool_ctx.timing_log.record(step_name, time.monotonic() - _start) - except Exception as exc: - logger.error("check_repo_merge_state unhandled exception", exc_info=True) - return json.dumps( - { - "error": f"{type(exc).__name__}: {exc}", - "queue_available": False, - "merge_group_trigger": False, - "auto_merge_available": False, - "ci_event": None, - } - ) diff --git a/src/autoskillit/skills/sous-chef/SKILL.md b/src/autoskillit/skills/sous-chef/SKILL.md index d2b6681522..6c30e2aea6 100644 --- a/src/autoskillit/skills/sous-chef/SKILL.md +++ b/src/autoskillit/skills/sous-chef/SKILL.md @@ -27,6 +27,26 @@ responsible for enforcing this regardless of what the plan says. --- +## SKILL_COMMAND FORMATTING — MANDATORY + +When calling `run_skill`, the `skill_command` argument MUST be a space-separated token +string — never a structured document or markdown section list. + +- Substitute `${{ context.* }}` and `${{ inputs.* }}` placeholders with their resolved + values and pass the result **VERBATIM** to `run_skill`. +- **Do NOT** add markdown headers (`##`), labels, notes, or explanatory prose to + `skill_command`. It is not a document — it is a command string. +- Path arguments are single tokens: `/path/to/file.md` — not a labeled section. +- Extra arguments from a step `note:` are appended as space-separated tokens. + +**Wrong:** `/autoskillit:implement-worktree-no-merge\n\n## Plan Path\n/path/plan.md\n\n## Branch\nimpl-926` +**Right:** `/autoskillit:implement-worktree-no-merge /path/plan.md impl-926` + +This applies to ALL skills, including bare-placeholder steps where you supply values +at runtime (`/autoskillit:arch-lens-{slug} {context_path}` → substitute, then pass verbatim). + +--- + ## CONTEXT LIMIT ROUTING — MANDATORY When `run_skill` returns `needs_retry=true` for **any step**: @@ -37,22 +57,57 @@ When `run_skill` returns `needs_retry=true` for **any step**: - **If `retry_reason: resume` AND `subtype≠stale` AND the step defines `on_context_limit`** → follow `on_context_limit`. The worktree or partial state is on disk; route to the designated recovery step (typically `test` or `retry_worktree`) to check whether partial work was sufficient. + API infrastructure errors (overload, 529, ECONNRESET) also produce `retry_reason=resume` + with `infra_exit_category="api_error"` — route them identically to context exhaustion. + The `infra_exit_category` field is informational only + (`"completed"`, `"context_exhausted"`, `"api_error"`, `"process_killed"`). - **If `retry_reason: resume` AND `subtype≠stale` AND the step has no `on_context_limit`** → fall through to `on_failure`. - **If `retry_reason: drain_race` AND the step defines `on_context_limit`** → follow `on_context_limit`. The channel signal confirmed session completion; stdout was not fully flushed before kill. Partial progress is confirmed — treat identically to `resume` for routing purposes. - **If `retry_reason: drain_race` AND the step has no `on_context_limit`** → fall through to `on_failure`. +- **If `retry_reason: completed_no_flush` AND the step defines `on_context_limit`** → follow `on_context_limit`. + The session exited with empty stdout but write evidence confirms files were written to the worktree. + Partial progress is confirmed — treat identically to `resume` for routing purposes. +- **If `retry_reason: completed_no_flush` AND the step has no `on_context_limit`** → fall through to `on_failure`. - **If `retry_reason: empty_output`** → fall through to `on_failure`. The session produced no - output; there is no partial state on disk. Do NOT route to `on_context_limit` even if defined. + output AND no write evidence (no Write/Edit calls, no filesystem writes). Do NOT route to `on_context_limit` even if defined. - **If `retry_reason: path_contamination`** → fall through to `on_failure`. The session wrote files outside its working directory. This is a CWD boundary violation, not a context limit. Do NOT route to `on_context_limit` even if defined. -- **If `retry_reason: early_stop` or `zero_writes`** → fall through to `on_failure`. +- **If `retry_reason: thinking_stall` AND `lifespan_started` is true AND the step defines + `on_context_limit`** → follow `on_context_limit`. The model consumed tokens (thinking + blocks) but produced no final output. Prior tool calls suggest partial progress on disk. +- **If `retry_reason: thinking_stall` AND `lifespan_started` is false** → fall through to `on_failure`. + No progress was made. +- **If `retry_reason: idle_stall` AND `lifespan_started` is true AND the step defines + `on_context_limit`** → follow `on_context_limit`. The idle watchdog killed the session, + but prior tool calls suggest partial progress on disk. +- **If `retry_reason: idle_stall` AND `lifespan_started` is false** → fall through to `on_failure`. + No progress was made. +- **If `retry_reason: early_stop` AND `has_progress_evidence` is true in the result AND the step + defines `on_context_limit`** → follow `on_context_limit`. The model made progress (wrote files + or created a worktree) but stopped before emitting the completion marker. Partial progress + exists on disk. +- **If `retry_reason: early_stop` AND `has_progress_evidence` is false** → fall through to `on_failure`. +- **If `retry_reason: zero_writes` AND `has_progress_evidence` is true in the result AND the step + defines `on_context_limit`** → follow `on_context_limit`. The model made filesystem contact + but made no Write/Edit tool calls. Partial progress may exist on disk. +- **If `retry_reason: zero_writes` AND `has_progress_evidence` is false** → fall through to `on_failure`. - **If `retry_reason: stale`** → decrement the `retries` counter for this step. Re-execute the same step if retries remain. If retries are exhausted, fall through to `on_failure`. Do NOT route to `on_context_limit` — stale is a transient failure, not a context limit. No partial progress is assumed. +**Worktree-stale carve-out:** When a step that invokes a worktree-creating skill +(`implement-worktree-no-merge`, `implement-worktree`, `implement-experiment`) returns +`retry_reason: stale` (or `retry_reason: resume` with `subtype: stale`), re-execute the +step **without consuming the retries budget**. Stale means the session produced nothing +useful — the worktree orphan concern that motivates `retries: 0` does not apply. +This is a one-shot retry: if the retry also goes stale, fall through to `on_failure`. +Before re-executing, if the stale result captured `worktree_path`, remove the empty +worktree (`git worktree remove --force `) to prevent orphaned worktrees. + **For `implement-worktree-no-merge` specifically:** - `on_context_limit` routes to `retry_worktree` in standard recipes. - Use `/autoskillit:retry-worktree` — pass the existing `worktree_path` from the @@ -69,8 +124,20 @@ Summary: `needs_retry=true` + `retry_reason=resume` + `subtype=stale` → re-exe `needs_retry=true` + `retry_reason=resume` + `subtype≠stale` + no `on_context_limit` → `on_failure`. `needs_retry=true` + `retry_reason=drain_race` + step has `on_context_limit` → follow `on_context_limit`. `needs_retry=true` + `retry_reason=drain_race` + no `on_context_limit` → `on_failure`. + `needs_retry=true` + `retry_reason=completed_no_flush` + step has `on_context_limit` → follow `on_context_limit`. + `needs_retry=true` + `retry_reason=completed_no_flush` + no `on_context_limit` → `on_failure`. + `needs_retry=true` + `retry_reason=empty_output` → `on_failure`. + `needs_retry=true` + `retry_reason=path_contamination` → `on_failure`. + `needs_retry=true` + `retry_reason=thinking_stall` + `lifespan_started=true` + step has `on_context_limit` → follow `on_context_limit`. + `needs_retry=true` + `retry_reason=thinking_stall` + `lifespan_started=false` → `on_failure`. + `needs_retry=true` + `retry_reason=idle_stall` + `lifespan_started=true` + step has `on_context_limit` → follow `on_context_limit`. + `needs_retry=true` + `retry_reason=idle_stall` + `lifespan_started=false` → `on_failure`. + `needs_retry=true` + `retry_reason=early_stop` + `has_progress_evidence=true` + step has `on_context_limit` → follow `on_context_limit`. + `needs_retry=true` + `retry_reason=early_stop` + `has_progress_evidence=false` → `on_failure`. + `needs_retry=true` + `retry_reason=zero_writes` + `has_progress_evidence=true` + step has `on_context_limit` → follow `on_context_limit`. + `needs_retry=true` + `retry_reason=zero_writes` + `has_progress_evidence=false` → `on_failure`. `needs_retry=true` + `retry_reason=stale` → decrement retries counter → `on_failure` when exhausted (no partial progress, not a context limit). - `needs_retry=true` + any other `retry_reason` → `on_failure` (no partial progress). + `needs_retry=true` + `retry_reason=stale` + worktree-creating step → one-shot re-execute (bypasses retries budget; on_failure if repeated stale). --- @@ -114,8 +181,28 @@ Act on this list as follows: When the user provides **more than one issue or task** in a single request: 1. **If the user says "parallel"** (or "run in parallel", "simultaneously", "at the - same time", "concurrently") → launch N independent pipeline sessions **immediately**. - No questions, no pushback, no alternative suggestions. + same time", "concurrently"): + + a. **Build execution map first.** Call `run_skill` with `/autoskillit:build-execution-map` + passing all issue numbers. This produces an `execution_map` JSON artifact at the + emitted path. + + b. **Read the execution map.** Parse the JSON to extract `groups` and `merge_order`. + + c. **Dispatch groups in order.** For each group in ascending `group` number: + - If `parallel: true` → launch all issues in the group as independent pipeline + sessions simultaneously, using the wavefront scheduling rule (defined in the section below). + - If `parallel: false` → run the group's issues one at a time in sequence. + + d. **Merge-wait between groups.** Group N+1 must NOT begin cloning until ALL of + Group N's PRs have merged to the base branch. This ensures every group's clones + capture a base SHA that includes all prior groups' changes. Use the MERGE PHASE + rules to merge each group's PRs, following the `merge_order` from the map for + intra-group merge sequencing. + + e. **Fallback.** If `build-execution-map` fails or returns an error, fall back to + launching all N pipelines immediately (current behavior). Do not block dispatch + on map failure. 2. **If the user says "sequential"** (or "one at a time", "in order", "one by one") → run them one at a time without asking. @@ -142,7 +229,8 @@ or user says "parallel"). Within each batched round, pipeline steps have two spe **Fast steps** — MCP tool calls that complete in seconds: `run_cmd`, `clone_repo`, `create_unique_branch`, `fetch_github_issue`, -`claim_issue`, `merge_worktree`, `test_check`, `reset_test_dir`, `classify_fix` +`claim_issue`, `merge_worktree`, `test_check`, `reset_test_dir`, `classify_fix`, +`push_to_remote` **Slow steps** — headless sessions that take minutes: Any `run_skill` invocation (investigate, implement, audit, review, etc.) @@ -162,6 +250,13 @@ Any `run_skill` invocation (investigate, implement, audit, review, etc.) the batch. A fast step launched alongside a slow step completes instantly but sits idle until the slow step finishes — wasting wall-clock time and blocking re-inspection. +4. **Advance every active pipeline in every round.** A pipeline is "active" if it has not + reached `done` or `escalate_stop`. In every batched round, every active pipeline MUST + receive at least one step — either a fast step is drained or a slow step is launched. + Never leave an active pipeline idle for an entire round while sibling pipelines are + progressing. If a pipeline has completed all its `plan_parts` and only has finalization + steps remaining (push, merge, close), it is still active and must be advanced. + ### Rationale Batched rounds wait for the **slowest step** in the batch. If a slow `run_skill` is launched @@ -172,6 +267,132 @@ simultaneously, after which all slow steps run in parallel and their wall-clock --- +## EXECUTION MAP — GROUP DISPATCH — MANDATORY + +When dispatching from an execution map: + +1. **Group iteration is outer loop.** The group number (1, 2, 3, ...) is the primary + ordering. Within each group, the wavefront scheduling rule governs step interleaving. + +2. **Merge-wait is mandatory between groups.** After all pipelines in Group N complete + (including their merge phase), verify all Group N PRs have merged to the base branch + before starting Group N+1. This prevents Group N+1 from cloning a stale base. + +3. **merge_order governs intra-group PR merge sequencing.** Within a parallel group, + merge PRs in the order specified by `merge_order` (not by completion time). This + minimizes merge conflicts by merging simpler changes first. + +4. **Single-issue groups skip wavefront.** If a group has `parallel: false` or contains + only one issue, run it as a single pipeline — no wavefront scheduling needed. + +5. **Do not pause for confirmation between groups.** Once merge-wait verifies all + Group N PRs have merged, dispatch Group N+1 immediately. NEVER use + AskUserQuestion to ask whether to proceed to the next group. + +6. **Handle deferred issues before dispatching any group.** + + After reading the execution map, check for `has_deferred: true`. + + **If `has_deferred` is false** (no deferrals): proceed directly to Group 1 dispatch. + + **If `has_deferred` is true:** + + **6a. Pre-dispatch freshness check.** Before presenting any escalation question, + re-query the current label state of ALL unique blocker issue numbers across all + `deferred_issues` entries in a single batched GraphQL request using aliases: + + ```graphql + query { + i887: repository(owner:"", name:"") { + issue(number:887) { labels(first:20) { nodes { name } } } + } + i912: repository(owner:"", name:"") { + issue(number:912) { labels(first:20) { nodes { name } } } + } + } + ``` + + For each blocker, check whether the `in-progress` label is still present. If a + blocker's label has been removed since the map was built, remove it from that deferred + issue's `blocked_by` list. If all blockers for a given deferred issue have cleared, + that issue is no longer deferred — collect it in an "auto-cleared" list for the + supplementary map (step 6e). **Never issue individual `gh issue view` calls per blocker + — always batch into a single GraphQL request.** + + **6b. Present AskUserQuestion for each still-deferred issue.** For each deferred issue + where at least one blocker's `in-progress` label is still present, call + `AskUserQuestion`: + + > "Issue #N (title) cannot be dispatched safely. It conflicts with in-progress + > issue(s): #M1 (title1), #M2 (title2). + > Conflict: {reasoning} + > + > Choose: + > 1. **Wait** — Hold #N; retry after the blocking issues complete + > 2. **Proceed anyway** — Dispatch #N now, accepting conflict risk + > 3. **Drop** — Remove #N from this session entirely" + + **Headless-mode rule (MANDATORY):** When `AskUserQuestion` is denied by the hook + (the deny message says "proceed without user confirmation" — this refers to general + tool behavior, NOT this decision), treat the response as **Wait**. Do NOT interpret + the deny message as permission to proceed. This is the explicit safe default for + unattended sessions. + + **6c. Route based on user answer (or Wait default):** + + - **Wait**: Hold the issue. At each group-completion barrier (after all `run_skill` + calls in a group return — the natural inter-group barrier in both queue-mode and + classic-mode), re-check ALL outstanding Wait-path blockers via a single batched + GraphQL aliases query. When all blockers for a Wait issue have cleared, move it to + the auto-cleared list for step 6e. After the final group completes, if any Wait + issues remain uncleared, report them as skipped in the session result. + + - **Proceed**: Add the issue to a **new sequential group** inserted at the end of the + dispatch sequence (after all other groups). This is transparent in both merge modes: + in queue-mode (#1268), the pipeline self-merges; in classic mode, `merge-prs` handles + it. Do not create one group per Proceed issue — batch all Proceed issues into the + single final group together. + + - **Drop**: If the issue has already been claimed (its `in-progress` label is set by + this session), call `release_issue` to remove it. Then exclude the issue from all + dispatch, merge, and reporting. Do not mark it as failed — it is not a failure. + + **6d. Zero-non-deferred-groups edge case.** When all target issues are deferred and no + dispatch groups exist (group_count is 0 after removing deferred issues), skip group + dispatch entirely. Enter an explicit poll loop: + + 1. Wait `github.deferred_poll_interval_seconds` (default: 60 seconds between checks). + 2. Re-query ALL outstanding blocker labels via a single batched GraphQL aliases query. + 3. If at least one deferred issue becomes unblocked (all its blockers cleared), proceed + to step 6e for those issues. + 4. Repeat until the first unblocked issue is found OR the elapsed time exceeds + `github.deferred_poll_timeout_seconds` (default: 1800 seconds / 30 minutes). + 5. On timeout: report all deferred issues as skipped. Set session result + `success: false` with `failure_reason: "All target issues deferred due to + in-progress conflicts — human decision required"`. Exit. + + **If in headless mode and all issues default to Wait with zero dispatch groups:** + skip the poll loop and immediately set `success: false` with the above + `failure_reason`. Do not poll indefinitely in unattended sessions. + + **6e. Supplementary map for auto-cleared and freshness-cleared issues.** When one or + more Wait/freshness-cleared issues become eligible: + + 1. Re-run `build-execution-map` for the newly eligible issues **only** — pass their + issue numbers as arguments. This is a full analysis (pairwise + cross-assessment), + not a passthrough. The supplementary map may itself produce new deferrals if + remaining in-progress issues conflict with the eligible set. + 2. If the supplementary map returns `has_deferred=true`, apply steps 6b and 6c to + the newly deferred issues before dispatching the supplementary groups. In headless + mode, treat all new deferrals as Wait (same rule as 6b). Issues that remain deferred + after this re-entry are skipped and reported in the session result. + 3. Dispatch the resulting dispatch groups as additional sequential group(s) appended + after the last completed group. + 4. Apply the same group-boundary merge-wait rule before dispatching the supplementary + groups. + +--- + ## STEP NAME IMMUTABILITY — MANDATORY The `step_name` passed to `run_skill` (and all other recipe-step tools that accept @@ -198,6 +419,12 @@ parallel run of the same step reports under the same canonical step name. --- +## MODEL PROPAGATION — MANDATORY + +**MODEL PROPAGATION** — When the user specifies a model (e.g. "use opus"), apply it to the `model` parameter of ALL `run_skill` calls for steps that declare a `model:` field — including follow-on steps (retry_worktree, fix, resolve_review, resolve_ci, conflict resolution). All `run_skill` steps in orchestrated recipes must declare a `model:` field; steps that omit it are ineligible for propagation and silently bypass user model selection. + +--- + ## MERGE PHASE — MANDATORY This rule applies whenever the orchestrator must merge **one or more open PRs**, whether @@ -211,7 +438,7 @@ headless session): ```bash REPO=$(gh repo view --json nameWithOwner -q .nameWithOwner) && OWNER=${REPO%%/*} && REPO_NAME=${REPO##*/} && -BRANCH="" && # substitute the PR's target branch (e.g. "main", "integration") +BRANCH="" && # substitute the PR's target branch (e.g. "main", "develop") gh api graphql -f query="query { repository(owner:\"$OWNER\", name:\"$REPO_NAME\") { mergeQueue(branch:\"$BRANCH\") { id } @@ -276,8 +503,22 @@ executes synchronously. reports `merge unconfirmed` even though the PR will eventually merge. For ad-hoc (off-recipe) merges: -- If merging multiple PRs collected from parallel pipelines: route through the - `merge-prs` recipe for batch sequential merging. +- When `queue_available=true` (and `sequential_queue` is not `"true"`): each pipeline's + implementation recipe handles its own enqueue via `route_queue_mode` → + `enqueue_to_queue` → `wait_for_queue`. Do NOT invoke `merge-prs`. The orchestrator's + natural parallel-batch join (waiting for all Group N `run_skill` invocations to + return) serves as the inter-group barrier — each pipeline only returns after + `wait_for_queue` reports `merged`, confirming its PR is in the base branch. Advance + to Group N+1 only after all Group N pipelines complete. +- When `queue_available=false` OR `sequential_queue == "true"`: route through the + `merge-prs` recipe for batch sequential merging (unchanged behavior). + +**Hidden ingredient — `sequential_queue`** (default `"false"`): Force the centralized +`merge-prs` path even when `queue_available=true`. Use when batch-level review via an +integration PR is required, or when cross-PR `audit_impl` is needed as a quality gate. +Each pipeline's per-PR review step before enqueue provides per-PR review coverage; the +merge queue itself provides conflict safety. `sequential_queue` is only needed when +consolidated batch-level review is explicitly required. ### 3. NEVER bypass recipe merge steps @@ -327,6 +568,21 @@ When `run_skill` output contains `--- QUOTA WARNING ---`: 3. After sleeping, proceed normally with the next step. 4. Before executing, state aloud: "Quota warning. Sleeping N seconds before next step." +### Budget-Exceeded Denial (quota sleep exceeds session wall-clock budget) + +When a `run_skill` call is DENIED with "QUOTA BUDGET EXCEEDED": + +1. The required quota sleep exceeds the session's remaining wall-clock budget. +2. **MANDATORY**: Do NOT execute the sleep command. +3. Instead, emit your result block immediately with: + - `"success": false` + - `"reason": "fleet_quota_exhausted"` + - `"wait_seconds": ` + - `"summary": "Quota exceeded; session budget insufficient for sleep. Resume after window resets."` +4. Then STOP — do not call any more tools. +5. Do NOT follow `on_failure`. Do NOT report to the user. +6. The fleet dispatcher will handle retry scheduling. + ### Key Rules - Quota denials are **always temporary**. The API enforces multiple rate-limit windows (e.g. one-minute, one-hour, five-hour, one-day). The guard waits for the most constrained window — the one that resets latest among all windows above the threshold — to reset before retrying. @@ -338,6 +594,50 @@ When `run_skill` output contains `--- QUOTA WARNING ---`: --- +## STEP EXECUTION IS NOT DISCRETIONARY — MANDATORY + +You MUST execute every step the pipeline routes you to. The recipe step graph is the +sole authority on what executes and in what order. + +Context management is handled by the system via on_context_limit routing. Execute +every step at full fidelity regardless of session length. + +### 1. Anti-skip rule + +- NEVER skip a step because the PR is small, the diff is trivial, the change looks + simple, or you judge the step unnecessary. +- NEVER skip a step because you believe it has already been done or is redundant. +- The ONLY mechanism for skipping a step is `skip_when_false` evaluating to false. + When `skip_when_false` evaluates to true (or is absent), the step MUST execute. +- Consequence: skipping PR review steps results in unreviewed code, missing diff + annotations, and no architectural lens analysis — code reaches main without + quality gates. Skipping issue lifecycle steps breaks traceability. + +### 2. Anti-improvisation rule + +- NEVER replace recipe steps with manual tool calls. In particular, NEVER use `run_cmd` + with `gh pr create`, `gh pr review`, or `gh api` to substitute for recipe steps. +- All PR creation and review must flow through the recipe's declared step chain + (`prepare_pr`, `run_arch_lenses`, `compose_pr`, `annotate_pr_diff`, `review_pr`). + Bypassing these steps skips diff annotation, architectural lens analysis, and + automated code review. + +### 3. The word "optional" in YAML + +`optional: true` on a recipe step does NOT mean the step is discretionary. It means: +- The step is SKIPPED when its `skip_when_false` ingredient evaluates to false. +- When the ingredient evaluates to true, the step is MANDATORY. +- A running optional step that returns `success: false` MUST follow `on_failure`. + +### 4. Anti-shortcut rule + +- Do not generalize from prior step outcomes. A step that returned a non-branching + result in a previous iteration may return a different result in the next. Every step + must execute on every issue — observed patterns from earlier issues do not make later + executions redundant. + +--- + ## NARRATION SUPPRESSION — MANDATORY Do NOT output prose status text, phase announcements, or progress summaries between diff --git a/src/autoskillit/skills_extended/analyze-prs/SKILL.md b/src/autoskillit/skills_extended/analyze-prs/SKILL.md index 9397cf547d..aef4124f1c 100644 --- a/src/autoskillit/skills_extended/analyze-prs/SKILL.md +++ b/src/autoskillit/skills_extended/analyze-prs/SKILL.md @@ -28,6 +28,7 @@ complexity, and produce machine-readable output for the `merge-prs` recipe. - Merge, close, or modify any PR - Modify any source code files - Create files outside `{{AUTOSKILLIT_TEMP}}/merge-prs/` directory +- Run subagents in the background (`run_in_background: true` is prohibited) **ALWAYS:** - Use subagents to fetch PR data in parallel @@ -140,43 +141,77 @@ is active on `{base_branch}` with `MERGEABLE` entries. building the overlap matrix. PRs that fail either gate are reported in the manifest but excluded from merge ordering. + Use a single GraphQL alias query to fetch CI status and reviews for all PRs at once + (1 API call regardless of PR count, instead of 2N sequential REST calls). Chunk into + batches of 20 PRs to stay within GraphQL complexity limits. + ```bash ELIGIBLE_PRS=() CI_BLOCKED_PRS=() # [{number, title, reason}] REVIEW_BLOCKED_PRS=() # [{number, title, reason}] - for PR in "${ALL_PRS[@]}"; do - PR_NUM=$(echo "$PR" | jq -r .number) - PR_TITLE=$(echo "$PR" | jq -r .title) - - # --- CI Gate --- - CI_CHECKS=$(gh pr checks "$PR_NUM" --json name,status,conclusion 2>/dev/null \ - || echo "[]") - FAILING=$(echo "$CI_CHECKS" | jq '[.[] | select( - .conclusion != null and - .conclusion != "success" and - .conclusion != "skipped" and - .conclusion != "neutral" - )] | length') - IN_PROGRESS=$(echo "$CI_CHECKS" | jq '[.[] | select(.conclusion == null)] | length') - - if [ "$FAILING" -gt 0 ] || [ "$IN_PROGRESS" -gt 0 ]; then - REASON="CI failing: ${FAILING} failed, ${IN_PROGRESS} in-progress" - CI_BLOCKED_PRS+=("{\"number\":${PR_NUM},\"title\":\"${PR_TITLE}\",\"reason\":\"${REASON}\"}") - continue - fi - - # --- Review Gate --- - REVIEWS=$(gh pr view "$PR_NUM" --json reviews -q '.reviews // []') - CHANGES_REQUESTED=$(echo "$REVIEWS" | jq '[.[] | select(.state == "CHANGES_REQUESTED")] | length') - - if [ "$CHANGES_REQUESTED" -gt 0 ]; then - REASON="${CHANGES_REQUESTED} unresolved CHANGES_REQUESTED review(s)" - REVIEW_BLOCKED_PRS+=("{\"number\":${PR_NUM},\"title\":\"${PR_TITLE}\",\"reason\":\"${REASON}\"}") - continue - fi - - ELIGIBLE_PRS+=("$PR") + # Build GraphQL alias query for all PRs in batches of 20 + ALL_PR_NUMS=($(echo "$ALL_PRS" | jq -r '.[].number')) + BATCH_SIZE=20 + for batch_start in $(seq 0 $BATCH_SIZE $((${#ALL_PR_NUMS[@]} - 1))); do + BATCH_NUMS=("${ALL_PR_NUMS[@]:$batch_start:$BATCH_SIZE}") + + QUERY="query { " + for i in $(seq 0 $((${#BATCH_NUMS[@]} - 1))); do + NUM="${BATCH_NUMS[$i]}" + QUERY="${QUERY} pr${i}: repository(owner: \"${OWNER}\", name: \"${REPO}\") { + pullRequest(number: ${NUM}) { + number + reviews(last: 20) { nodes { state } } + commits(last: 1) { nodes { commit { statusCheckRollup { + state + contexts(last: 100) { nodes { ... on CheckRun { name status conclusion } } } + } } } } + } + }" + done + QUERY="${QUERY} }" + BATCH_RESULT=$(gh api graphql -f query="${QUERY}") + + # Parse per-PR results from BATCH_RESULT + for i in $(seq 0 $((${#BATCH_NUMS[@]} - 1))); do + NUM="${BATCH_NUMS[$i]}" + PR_TITLE=$(echo "$ALL_PRS" | jq -r --argjson n "$NUM" '.[] | select(.number == $n) | .title') + PR_DATA=$(echo "$BATCH_RESULT" | jq --arg key "pr${i}" '.data[$key].pullRequest') + + # --- CI Gate --- + FAILING=$(echo "$PR_DATA" | jq ' + [(.commits.nodes[0].commit.statusCheckRollup.contexts.nodes // [])[] | + select(.conclusion != null and + .conclusion != "success" and + .conclusion != "skipped" and + .conclusion != "neutral")] | length') + IN_PROGRESS=$(echo "$PR_DATA" | jq ' + [(.commits.nodes[0].commit.statusCheckRollup.contexts.nodes // [])[] | + select(.conclusion == null)] | length') + + if [ "${FAILING:-0}" -gt 0 ] || [ "${IN_PROGRESS:-0}" -gt 0 ]; then + REASON="CI failing: ${FAILING} failed, ${IN_PROGRESS} in-progress" + CI_BLOCKED_PRS+=("{\"number\":${NUM},\"title\":\"${PR_TITLE}\",\"reason\":\"${REASON}\"}") + continue + fi + + # --- Review Gate --- + CHANGES_REQUESTED=$(echo "$PR_DATA" | jq ' + [.reviews.nodes | + group_by(.author.login)[] | + last | + select(.state == "CHANGES_REQUESTED")] | length') + + if [ "${CHANGES_REQUESTED:-0}" -gt 0 ]; then + REASON="${CHANGES_REQUESTED} unresolved CHANGES_REQUESTED review(s)" + REVIEW_BLOCKED_PRS+=("{\"number\":${NUM},\"title\":\"${PR_TITLE}\",\"reason\":\"${REASON}\"}") + continue + fi + + mapfile -t _pr_entry < <(echo "$ALL_PRS" | jq -c --argjson n "$NUM" '.[] | select(.number == $n)') + ELIGIBLE_PRS+=("${_pr_entry[@]}") + done done ``` @@ -248,7 +283,7 @@ Ensure `{{AUTOSKILLIT_TEMP}}/merge-prs/` exists. ```json { - "integration_branch": "pr-batch/pr-merge-YYYYMMDD-HHMMSS", + "batch_branch": "pr-batch/pr-merge-YYYYMMDD-HHMMSS", "base_branch": "{base_branch}", "generated_at": "{ISO timestamp}", "pr_count": 5, @@ -351,7 +386,7 @@ This file is named `*_plan_*.md` so `audit-impl` can discover it as the baseline Verify: - `pr_order_{ts}.json` is valid JSON and parseable - Every listed PR number appears exactly once -- `integration_branch` field is set +- `batch_branch` field is set Report to terminal: - Order file path @@ -381,7 +416,7 @@ structured output tokens as the very last lines of your text output: ``` pr_order_file = {absolute_path_to_pr_order_json} analysis_file = {absolute_path_to_pr_analysis_plan_md} -integration_branch = {integration_branch_name} +batch_branch = {batch_branch_name} pr_count = {eligible_pr_count} simple_count = {simple_pr_count} needs_check_count = {needs_check_pr_count} diff --git a/src/autoskillit/skills_extended/arch-lens-c4-container/SKILL.md b/src/autoskillit/skills_extended/arch-lens-c4-container/SKILL.md index 4f41bff74e..645cbcb90a 100644 --- a/src/autoskillit/skills_extended/arch-lens-c4-container/SKILL.md +++ b/src/autoskillit/skills_extended/arch-lens-c4-container/SKILL.md @@ -31,6 +31,7 @@ hooks: - Modify any source code files - Include internal implementation details (that's for other lenses) - Show runtime behavior or state transitions +- Run subagents in the background (`run_in_background: true` is prohibited) **ALWAYS:** - Focus on CONTAINERS (deployable units, not classes) diff --git a/src/autoskillit/skills_extended/arch-lens-concurrency/SKILL.md b/src/autoskillit/skills_extended/arch-lens-concurrency/SKILL.md index aade0ee5ad..926c2c6413 100644 --- a/src/autoskillit/skills_extended/arch-lens-concurrency/SKILL.md +++ b/src/autoskillit/skills_extended/arch-lens-concurrency/SKILL.md @@ -31,6 +31,7 @@ hooks: - Modify any source code files - Conflate with general process flow (that's a different lens) - Ignore thread safety implications +- Run subagents in the background (`run_in_background: true` is prohibited) **ALWAYS:** - Focus on PARALLEL execution specifically diff --git a/src/autoskillit/skills_extended/arch-lens-data-lineage/SKILL.md b/src/autoskillit/skills_extended/arch-lens-data-lineage/SKILL.md index 7ec1ee4221..1ff071cf9d 100644 --- a/src/autoskillit/skills_extended/arch-lens-data-lineage/SKILL.md +++ b/src/autoskillit/skills_extended/arch-lens-data-lineage/SKILL.md @@ -31,6 +31,7 @@ hooks: - Modify any source code files - Focus on runtime behavior (that's process flow lens) - Show static structure without data context +- Run subagents in the background (`run_in_background: true` is prohibited) **ALWAYS:** - Trace data from INPUT to STORAGE diff --git a/src/autoskillit/skills_extended/arch-lens-deployment/SKILL.md b/src/autoskillit/skills_extended/arch-lens-deployment/SKILL.md index 9972bf6997..5eda8b998d 100644 --- a/src/autoskillit/skills_extended/arch-lens-deployment/SKILL.md +++ b/src/autoskillit/skills_extended/arch-lens-deployment/SKILL.md @@ -31,6 +31,7 @@ hooks: - Modify any source code files - Include code-level details - Show internal logic +- Run subagents in the background (`run_in_background: true` is prohibited) **ALWAYS:** - Focus on PHYSICAL deployment diff --git a/src/autoskillit/skills_extended/arch-lens-development/SKILL.md b/src/autoskillit/skills_extended/arch-lens-development/SKILL.md index 28b088b5d4..c3c4b48384 100644 --- a/src/autoskillit/skills_extended/arch-lens-development/SKILL.md +++ b/src/autoskillit/skills_extended/arch-lens-development/SKILL.md @@ -31,6 +31,7 @@ hooks: - Modify any source code files - Include runtime architecture details - Show business logic +- Run subagents in the background (`run_in_background: true` is prohibited) **ALWAYS:** - Focus on BUILD and TEST infrastructure diff --git a/src/autoskillit/skills_extended/arch-lens-error-resilience/SKILL.md b/src/autoskillit/skills_extended/arch-lens-error-resilience/SKILL.md index b0b7b0cf14..24024439fd 100644 --- a/src/autoskillit/skills_extended/arch-lens-error-resilience/SKILL.md +++ b/src/autoskillit/skills_extended/arch-lens-error-resilience/SKILL.md @@ -31,6 +31,7 @@ hooks: - Modify any source code files - Show happy path details (that's process flow lens) - Ignore validation and fail-fast patterns +- Run subagents in the background (`run_in_background: true` is prohibited) **ALWAYS:** - Focus on FAILURE paths and recovery diff --git a/src/autoskillit/skills_extended/arch-lens-module-dependency/SKILL.md b/src/autoskillit/skills_extended/arch-lens-module-dependency/SKILL.md index 3284f44e96..bcc4547166 100644 --- a/src/autoskillit/skills_extended/arch-lens-module-dependency/SKILL.md +++ b/src/autoskillit/skills_extended/arch-lens-module-dependency/SKILL.md @@ -31,6 +31,7 @@ hooks: - Modify any source code files - Include runtime behavior details - Show external system integrations in detail +- Run subagents in the background (`run_in_background: true` is prohibited) **ALWAYS:** - Focus on IMPORT relationships between modules diff --git a/src/autoskillit/skills_extended/arch-lens-operational/SKILL.md b/src/autoskillit/skills_extended/arch-lens-operational/SKILL.md index 94b8e22351..e2dceec9ca 100644 --- a/src/autoskillit/skills_extended/arch-lens-operational/SKILL.md +++ b/src/autoskillit/skills_extended/arch-lens-operational/SKILL.md @@ -31,6 +31,7 @@ hooks: - Modify any source code files - Include internal implementation details - Show code-level patterns +- Run subagents in the background (`run_in_background: true` is prohibited) **ALWAYS:** - Focus on OPERATOR experience diff --git a/src/autoskillit/skills_extended/arch-lens-process-flow/SKILL.md b/src/autoskillit/skills_extended/arch-lens-process-flow/SKILL.md index 7a02b64641..87304e9a87 100644 --- a/src/autoskillit/skills_extended/arch-lens-process-flow/SKILL.md +++ b/src/autoskillit/skills_extended/arch-lens-process-flow/SKILL.md @@ -31,6 +31,7 @@ hooks: - Modify any source code files - Include static structure details (that's C4 lens) - Show data storage details (that's data lineage lens) +- Run subagents in the background (`run_in_background: true` is prohibited) **ALWAYS:** - Focus on BEHAVIOR and STATE TRANSITIONS diff --git a/src/autoskillit/skills_extended/arch-lens-repository-access/SKILL.md b/src/autoskillit/skills_extended/arch-lens-repository-access/SKILL.md index e551281c61..edf06a75e0 100644 --- a/src/autoskillit/skills_extended/arch-lens-repository-access/SKILL.md +++ b/src/autoskillit/skills_extended/arch-lens-repository-access/SKILL.md @@ -31,6 +31,7 @@ hooks: - Modify any source code files - Focus on data flow (that's data lineage lens) - Include business logic details +- Run subagents in the background (`run_in_background: true` is prohibited) **ALWAYS:** - Focus on REPOSITORIES and their methods diff --git a/src/autoskillit/skills_extended/arch-lens-scenarios/SKILL.md b/src/autoskillit/skills_extended/arch-lens-scenarios/SKILL.md index 413333ef74..62f9b5fd18 100644 --- a/src/autoskillit/skills_extended/arch-lens-scenarios/SKILL.md +++ b/src/autoskillit/skills_extended/arch-lens-scenarios/SKILL.md @@ -31,6 +31,7 @@ hooks: - Modify any source code files - Show internal component details - Include all possible scenarios (pick key ones) +- Run subagents in the background (`run_in_background: true` is prohibited) **ALWAYS:** - Focus on END-TO-END journeys diff --git a/src/autoskillit/skills_extended/arch-lens-security/SKILL.md b/src/autoskillit/skills_extended/arch-lens-security/SKILL.md index 07935eaadd..e11f635b2a 100644 --- a/src/autoskillit/skills_extended/arch-lens-security/SKILL.md +++ b/src/autoskillit/skills_extended/arch-lens-security/SKILL.md @@ -31,6 +31,7 @@ hooks: - Modify any source code files - Expose actual secrets or credentials - Show implementation details that could aid attacks +- Run subagents in the background (`run_in_background: true` is prohibited) **ALWAYS:** - Focus on TRUST BOUNDARIES diff --git a/src/autoskillit/skills_extended/arch-lens-state-lifecycle/SKILL.md b/src/autoskillit/skills_extended/arch-lens-state-lifecycle/SKILL.md index eaed162994..afce878ed3 100644 --- a/src/autoskillit/skills_extended/arch-lens-state-lifecycle/SKILL.md +++ b/src/autoskillit/skills_extended/arch-lens-state-lifecycle/SKILL.md @@ -31,6 +31,7 @@ hooks: - Modify any source code files - Show business logic details - Focus on data content (focus on mutation rules) +- Run subagents in the background (`run_in_background: true` is prohibited) **ALWAYS:** - Focus on STATE MUTATION RULES diff --git a/src/autoskillit/skills_extended/audit-arch/SKILL.md b/src/autoskillit/skills_extended/audit-arch/SKILL.md index 7c39041ebd..5131e6c4be 100644 --- a/src/autoskillit/skills_extended/audit-arch/SKILL.md +++ b/src/autoskillit/skills_extended/audit-arch/SKILL.md @@ -24,6 +24,7 @@ Audit the codebase for adherence to architectural standards and rules. **NEVER:** - Modify any source code files - Update an existing report - always generate new +- Run subagents in the background (`run_in_background: true` is prohibited) **ALWAYS:** - Use subagents for parallel exploration diff --git a/src/autoskillit/skills_extended/audit-bugs/SKILL.md b/src/autoskillit/skills_extended/audit-bugs/SKILL.md index f3b1acc3f2..315d06987f 100644 --- a/src/autoskillit/skills_extended/audit-bugs/SKILL.md +++ b/src/autoskillit/skills_extended/audit-bugs/SKILL.md @@ -30,6 +30,7 @@ The user may provide a "since" date (e.g., `2/7`, `2026-02-07`, `last week`). If **NEVER:** - Modify any source code files - Create files outside `{{AUTOSKILLIT_TEMP}}/audit-bugs/` directory +- Run subagents in the background (`run_in_background: true` is prohibited) **ALWAYS:** - Use subagents heavily for parallel log analysis diff --git a/src/autoskillit/skills_extended/audit-claims/SKILL.md b/src/autoskillit/skills_extended/audit-claims/SKILL.md index e931666e27..e4b5951360 100644 --- a/src/autoskillit/skills_extended/audit-claims/SKILL.md +++ b/src/autoskillit/skills_extended/audit-claims/SKILL.md @@ -46,6 +46,7 @@ comments and emits a verdict for recipe routing. - Modify any source code - Run deterministic diff annotation (claim positions are report-level, not line-level) - Generate findings for `experimental` claims — they are self-evidencing by definition +- Run subagents in the background (`run_in_background: true` is prohibited) **ALWAYS:** - Use the explicit `pr_url` argument instead of re-discovering via `gh pr list` @@ -75,25 +76,6 @@ fi If `pr_url` is missing or positional args are insufficient, abort with: `"Usage: /autoskillit:audit-claims "` -### Step 0.5 — Code-Index Initialization (required before any code-index tool call) - -Call `set_project_path` with the repo root where this skill was invoked (not a worktree path): - -``` -mcp__code-index__set_project_path(path="{PROJECT_ROOT}") -``` - -Code-index tools require **project-relative paths**. Always use paths like: - - src/mypackage/core/module.py - -NOT absolute paths like: - - /path/to/project/src/mypackage/core/module.py - -Agents launched via `run_skill` inherit no code-index state from the parent session — this -call is mandatory at the start of every headless session that uses code-index tools. - ### Step 1: Use the Explicit PR URL Parse `pr_number` from `pr_url` (last path segment). diff --git a/src/autoskillit/skills_extended/audit-cohesion/SKILL.md b/src/autoskillit/skills_extended/audit-cohesion/SKILL.md index 1b779e6822..54fa2d5b23 100644 --- a/src/autoskillit/skills_extended/audit-cohesion/SKILL.md +++ b/src/autoskillit/skills_extended/audit-cohesion/SKILL.md @@ -29,6 +29,7 @@ Audit the codebase for internal cohesion: how well components integrate and main - Modify any source code files - Update an existing report — always generate new - Duplicate findings that belong in audit-arch (rule violations) +- Run subagents in the background (`run_in_background: true` is prohibited) **ALWAYS:** - Use subagents for parallel exploration (one per cohesion dimension) @@ -369,12 +370,6 @@ Flag duplicates (same name in different agents). ## Audit Workflow -### Step 0: Initialize Code Index - -``` -mcp__code-index__set_project_path(path="{PROJECT_ROOT}") -``` - ### Step 1: Launch Parallel Subagents Spawn subagents for each cohesion dimension. Each subagent MUST be instructed: diff --git a/src/autoskillit/skills_extended/audit-defense-standards/SKILL.md b/src/autoskillit/skills_extended/audit-defense-standards/SKILL.md index db349c9e93..56014d4a4f 100644 --- a/src/autoskillit/skills_extended/audit-defense-standards/SKILL.md +++ b/src/autoskillit/skills_extended/audit-defense-standards/SKILL.md @@ -28,6 +28,7 @@ Standards are added here when `/autoskillit:design-guards` recommends them and t **NEVER:** - Modify any source code files - Update an existing report - always generate new +- Run subagents in the background (`run_in_background: true` is prohibited) **ALWAYS:** - Use subagents for parallel exploration (one per standard or group) diff --git a/src/autoskillit/skills_extended/audit-docs/SKILL.md b/src/autoskillit/skills_extended/audit-docs/SKILL.md new file mode 100644 index 0000000000..1e6c1a6a88 --- /dev/null +++ b/src/autoskillit/skills_extended/audit-docs/SKILL.md @@ -0,0 +1,165 @@ +--- +name: audit-docs +categories: [audit] +description: > + Audit documentation for drift, staleness, and inconsistency against the actual + codebase. Use when user says "audit docs", "check documentation", "docs audit", + or "documentation review". Spawns parallel subagents to explore codebase subsystems, + then cross-references all documentation sources against findings. +tier: 2 +hooks: + PreToolUse: + - matcher: "*" + hooks: + - type: command + command: "echo '[SKILL: audit-docs] Auditing documentation for staleness and drift...'" + once: true +--- + +# Documentation Audit Skill + +Audit all documentation sources for drift, staleness, and inconsistency against actual codebase behavior. + +## When to Use + +- User says "audit docs", "check documentation", "docs audit", "documentation review", "docs drift", "stale docs" + +## Critical Constraints + +**NEVER:** +- Modify any source files +- Update an existing report — always generate a new one +- Compare doc-to-doc without first grounding claims in actual code behavior +- Run subagents in the background (`run_in_background: true` is prohibited) + +**ALWAYS:** +- Ground every cross-reference finding in what the code actually does (not what other docs say) +- Use subagents for parallel exploration +- Write report to `{{AUTOSKILLIT_TEMP}}/audit-docs/docs_audit_{YYYY-MM-DD_HHMMSS}.md` +- Provide file:line references for every finding +- Categorize findings by severity (CRITICAL, HIGH, MEDIUM, LOW) +- Produce a usable report even when one or more subagents fail + +--- + +## Documentation Sources + +Enumerate and audit all of the following: + +- **`CLAUDE.md`** — project instructions, architecture tree, file path references, tool/skill counts, layer descriptions +- **`docs/architecture/**/*.md`** — component names, module paths, layer assignments +- **`docs/requirements/**/*.md`** and **`docs/specs/**/*.md`** — API surface, behavioral contracts +- **All `README.md` files** at any depth in the repository +- **Module/class/function docstrings** in Python files under `src/` +- **Recipe YAML `description`, `summary`, and `note` fields** in `.autoskillit/recipes/` and `src/autoskillit/recipes/` + +--- + +## Inconsistency Categories + +Flag findings in these categories (maps to REQ-SKILL-004): + +- **Stale claims** — doc asserts X, actual code behavior is Y +- **Orphaned references** — doc mentions a file, module, tool, class, or skill that no longer exists +- **Missing docs** — new subsystem or module with no mention in CLAUDE.md tree or arch docs +- **Path/name drift** — hyphen-vs-underscore mismatches, renamed symbols, old module paths +- **Count mismatches** — CLAUDE.md states N tools/skills/hooks but actual count differs +- **Inter-doc contradictions** — CLAUDE.md says A, arch doc says B about the same entity + +--- + +## Audit Workflow + +1. **Pre-flight**: Verify `{{AUTOSKILLIT_TEMP}}/audit-docs/` directory exists; create it if not. + +2. **Familiarization wave** — spawn 6 parallel subagents, one per subsystem group. Each subagent reports: actual module/component names, exported symbols, behavioral summary (2–5 sentences per module). If any subagent fails, record the gap and continue. + + | Agent | Subsystems | Focus | + |---|---|---| + | Agent 1 | `core/`, `config/` | What these modules actually expose and do | + | Agent 2 | `execution/`, `workspace/` | Runtime orchestration and workspace lifecycle | + | Agent 3 | `recipe/`, `migration/` | Recipe schema, rules, validation, migration engine | + | Agent 4 | `server/` | MCP tool surface, gating, lifespan, factory | + | Agent 5 | `cli/`, `hooks/` | CLI commands, hook scripts, what each does | + | Agent 6 | `skills/`, `skills_extended/` | Bundled skills, categories, tiers | + +3. **Doc inventory** — enumerate all documentation sources (list files found under each source category above). + +4. **Cross-reference wave** — spawn 4 parallel subagents, each checking one doc domain against the familiarization findings: + + | Agent | Domain | What to check | + |---|---|---| + | Agent A | `CLAUDE.md` | Architecture tree accuracy, file path references, tool/skill counts, layer descriptions | + | Agent B | `docs/architecture/**` | Component names, module paths, layer assignments | + | Agent C | `docs/requirements/**` and `docs/specs/**` | API surface, behavioral contracts | + | Agent D | Recipe YAML descriptions + docstrings | Step descriptions, ingredient names, parameter docs | + +5. **Consolidate** — merge findings from all 4 agents, deduplicate by file:line, assign severity. + +6. **Self-validation pass** — for every CRITICAL or HIGH finding, re-read the cited file line to confirm the claim; downgrade or remove if not confirmed. + +7. **Write report** to `{{AUTOSKILLIT_TEMP}}/audit-docs/docs_audit_{YYYY-MM-DD_HHMMSS}.md` (relative to the current working directory) using the format below. + +8. **Output summary** — print finding counts by severity to terminal. + +--- + +## Report Format + +``` +# Documentation Audit Report — {YYYY-MM-DD HH:MM} + +## Summary +| Severity | Count | +|----------|-------| +| CRITICAL | N | +| HIGH | N | +| MEDIUM | N | +| LOW | N | + +## Findings + +### CRITICAL + +#### [DOC-001] {Title} +- **File:** `path/to/doc.md:42` +- **Claim:** "..." +- **Actual:** "..." +- **Fix:** ... + +... + +## Coverage Gaps +{If any familiarization subagent failed, list affected subsystems here} +``` + +--- + +## Exclusions + +Do NOT flag: +- Test files (`tests/`) +- Generated files (`{{AUTOSKILLIT_TEMP}}/`, `uv.lock`, `*.pyc`) +- Comment-only files or changelog entries +- External tool output or CI logs +- Doc-to-doc wording differences that don't contradict each other factually + +--- + +## Severity Guidelines + +**CRITICAL:** +- Orphaned references to deleted modules/tools +- CLAUDE.md architecture tree listing a path that doesn't exist + +**HIGH:** +- Stale behavioral claim that would mislead an implementer +- Count mismatch (e.g., CLAUDE.md says N skills, actual is M) + +**MEDIUM:** +- Path/name drift (hyphen vs underscore, module moved) +- Missing doc for a significant new subsystem + +**LOW:** +- Minor wording drift +- Docstring describes a parameter that was renamed diff --git a/src/autoskillit/skills_extended/audit-feature-gates/SKILL.md b/src/autoskillit/skills_extended/audit-feature-gates/SKILL.md new file mode 100644 index 0000000000..f315c5b11d --- /dev/null +++ b/src/autoskillit/skills_extended/audit-feature-gates/SKILL.md @@ -0,0 +1,277 @@ +--- +name: audit-feature-gates +categories: [audit] +description: > + Audit feature flag isolation — traces import chains, runtime gates, tool/skill + tag coverage, UI surfaces, and test markers to detect leakage and miswiring. +hooks: + PreToolUse: + - matcher: "*" + hooks: + - type: command + command: "echo '[SKILL: audit-feature-gates] Read-only audit — no code changes permitted'" + once: true +--- + +# Audit Feature Gates Skill + +Audits feature flag isolation across 6 dimensions: config projection, import chain integrity, +runtime gate consistency, tool/skill tag completeness, boundary coupling, and test marker +coverage. Detects where disabled features leak through import chains, runtime bypasses, UI +surfaces, and ungated callables. + +## When to Use + +- User says "audit feature gates", "check feature isolation", "feature flag audit" +- After adding a new feature to `FEATURE_REGISTRY` to verify isolation is complete +- As part of the `full-audit.yaml` 4th parallel chain + +## Arguments + +No arguments required. Reads `FEATURE_REGISTRY` from `src/autoskillit/core/_type_constants.py` +to enumerate features. + +## Critical Constraints + +**NEVER:** +- Modify any source code files +- Create files outside `{{AUTOSKILLIT_TEMP}}/audit-feature-gates/` +- Issue subagent Task calls sequentially — ALL 6 must be in a single parallel message +- Write output files before synthesizing ALL subagent results +- Run subagents in the background (`run_in_background: true` is prohibited) + +**ALWAYS:** +- Use `model: "sonnet"` when spawning all subagents via the Task tool +- Issue all 6 Task calls in a single message to maximize parallelism +- Subagents must NOT create their own files — they return findings in response text only + +## Severity Semantics + +- **BLOCK** — active leakage that would execute feature code when the feature is disabled +- **WARN** — isolation gap that may cause confusion or is one step from leakage +- **INFO** — coupling assessment, informational only + +--- + +## Workflow + +### Step 0: Pre-flight — Enumerate Features + +Read `src/autoskillit/core/_type_constants.py`. Extract **all** entries from `FEATURE_REGISTRY` exhaustively — do not assume a fixed list. +For each feature, note: `name`, `lifecycle`, `import_package`, `tool_tags`, `skill_categories`, +`default_enabled`. + +### Step 1: Launch 6 Parallel Subagents (SINGLE MESSAGE) + +**Issue ALL 6 Task calls in a single message.** + +--- + +**D1 — Config Projection** (subagent): + +**Finding IDs:** Assign each finding a unique ID using the scheme `FG-D1-{seq:02d}` where +`{seq}` is a zero-padded sequence starting at 01 (e.g., `FG-D1-01`, `FG-D1-02`). Include +the ID in every finding line. For inventory table rows, cite `file:line` for the config +source (e.g., `src/autoskillit/config/defaults.yaml:NN` or +`src/autoskillit/core/_type_constants.py:NN` for the registry entry). + +For each feature in `FEATURE_REGISTRY`: do not output any prose between iterations — process all features and return findings as structured text only. +- Parse `src/autoskillit/config/defaults.yaml` `features:` section +- Parse `.autoskillit/config.yaml` `features:` section (if it exists) +- Compute resolved state: `config_override ?? defaults ?? FeatureDef.default_enabled` +- FLAG (WARN): EXPERIMENTAL feature with `default_enabled=True` in registry +- FLAG (WARN): EXPERIMENTAL feature enabled in project config while on a stable/main branch +- FLAG (WARN): expired `sunset_date` values (compare against today's date, if field present) +- Produce inventory table: `ID | FEATURE | LIFECYCLE | DEFAULT | CONFIG | RESOLVED | RISK | file:line` + +Return findings as structured text. Do NOT create any files. + +--- + +**D2 — Import Chain Integrity** (subagent): + +**Finding IDs:** Assign each finding a unique ID using the scheme `FG-D2-{seq:02d}` (e.g., +`FG-D2-01`). Every BLOCK and WARN finding **must** include `file:line` — this is mandatory, +not optional. + +For each feature's `import_package`: do not output any prose between iterations — return findings as structured text only. +- Grep all `from {package} import` and `import {package}` across `src/` (excluding tests) +- Classify each import site: + - GUARDED: inside a function body, inside `if TYPE_CHECKING:`, or inside `if is_feature_enabled(...):` + - UNGUARDED: top-level import in a module that is not itself the feature's package +- FLAG (BLOCK) all UNGUARDED imports with `file:line` +- Ground-truth targets: `tools_kitchen.py`, `_cook.py`, `_prompts.py`, `_fleet.py` + +Return findings as structured text. Do NOT create any files. + +--- + +**D3 — Runtime Gate Consistency** (subagent): + +**Finding IDs:** Assign each finding a unique ID using the scheme `FG-D3-{seq:02d}` (e.g., +`FG-D3-01`). Every BLOCK and WARN finding **must** include `file:line`. + +For each feature: do not output any prose between iterations — return findings as structured text only. +- Find all `is_feature_enabled("{name}"` call sites across `src/` +- Find all `AUTOSKILLIT_FEATURES__{NAME}` env-var reads (bypass paths) +- FLAG (BLOCK): env-var gate without a corresponding `is_feature_enabled()` in the same code path +- FLAG (WARN): `_fleet_auto_gate_boot()` calling `mcp.enable()` without then calling `_redisable_subsets()` (`server/_lifespan.py`) +- FLAG (BLOCK): tool handlers for feature-tagged tools without an in-handler `is_feature_enabled()` check (e.g., `dispatch_food_truck` in `server/tools_execution.py`) +- FLAG (WARN): session-type checks that enable feature functionality without verifying the feature flag + +Return findings as structured text. Do NOT create any files. + +--- + +**D4 — Tool/Skill Tag Completeness** (subagent): + +**Finding IDs:** Assign each finding a unique ID using the scheme `FG-D4-{seq:02d}` (e.g., +`FG-D4-01`). Every BLOCK and WARN finding **must** include `file:line`. + +For each feature: do not output any prose between iterations — return findings as structured text only. +- Cross-reference `feature_def.tool_tags` against `TOOL_SUBSET_TAGS` in `src/autoskillit/core/_type_constants.py` +- Grep skill bodies in `src/autoskillit/skills_extended/` for feature-specific references +- FLAG (WARN): skills with feature references in body but missing the feature's category in frontmatter +- Verify `_DISPLAY_CATEGORIES` in `cli/_cook.py` applies feature-check filtering before displaying +- Verify `list_recipes` in `server/tools_recipe.py` filters `kind: campaign` when fleet is disabled +- FLAG (WARN): any `run_python` callable in `skill_contracts.yaml` whose package matches `feature_def.import_package` without a feature gate in the execution path + +Return findings as structured text. Do NOT create any files. + +--- + +**D5 — Boundary Coupling** (subagent): + +**Finding IDs:** Assign each finding a unique ID using the scheme `FG-D5-{seq:02d}` (e.g., +`FG-D5-01`). For WARN/BLOCK coupling table rows, cite `file:line` for the import site or +field declaration (e.g., `src/autoskillit/pipeline/context.py:NN`). + +For each feature: +- Grep `src/autoskillit/core/` (IL-0) for feature-specific constants or imports beyond `FeatureDef`/`FEATURE_REGISTRY` +- Check `src/autoskillit/pipeline/context.py` for feature-specific fields unconditionally allocated on `ToolContext` +- Check `src/autoskillit/config/settings.py` for feature-specific config dataclasses parsed without a validation gate +- Check `src/autoskillit/execution/headless.py` for unconditional reads of feature config +- Check `src/autoskillit/server/_factory.py` for unconditional feature-object allocation +- Produce coupling table: `ID | LAYER | FEATURE | COUPLING TYPE | SEVERITY | file:line` + +Return findings as structured text. Do NOT create any files. + +--- + +**D6 — Test Marker Coverage** (subagent): + +**Finding IDs:** Assign each finding a unique ID using the scheme `FG-D6-{seq:02d}` (e.g., +`FG-D6-01`). Every WARN finding **must** include `file:line` (the test file missing the marker). + +For each feature: +- Find test files importing from `feature.import_package` or referencing feature-specific symbols (Grep `tests/` for the import_package name) +- Verify each such test file carries `pytest.mark.feature("{name}")` at file or class level +- Check `tests/arch/test_feature_markers.py` for per-feature enforcement lists +- FLAG (WARN): features with no marker enforcement list in `test_feature_markers.py` +- FLAG (WARN): test files with feature code but no `feature("{name}")` marker +- Ground-truth: check for test files with fleet code but no `feature("fleet")` marker; check if planner marker enforcement exists in `test_feature_markers.py` + +Return findings as structured text. Do NOT create any files. + +--- + +### Step 2: Synthesize and Assemble Report + +After all 6 subagents return, consolidate findings per dimension. Count BLOCK/WARN/INFO totals. + +Ensure `{{AUTOSKILLIT_TEMP}}/audit-feature-gates/` exists (`mkdir -p`). All paths below are +relative to the current working directory. + +Write report to: +`{{AUTOSKILLIT_TEMP}}/audit-feature-gates/feature_gate_audit_{YYYY-MM-DD_HHMMSS}.md` + +Report format: + +```markdown +# Feature Gate Audit + +**Date:** {YYYY-MM-DD HH:MM:SS} **Features audited:** {comma-separated list from FEATURE_REGISTRY} + +## Config Projection + +{D1 findings with BLOCK/WARN/INFO severity badges and FG-D1-NN IDs} + +### D1 Remediation Checklist + +- [ ] [{ID}] {one-line action to resolve the finding} + +## Import Chain Integrity + +{D2 findings with BLOCK/WARN/INFO severity badges and FG-D2-NN IDs} + +### D2 Remediation Checklist + +- [ ] [{ID}] {one-line action to resolve the finding} + +## Runtime Gate Consistency + +{D3 findings with BLOCK/WARN/INFO severity badges and FG-D3-NN IDs} + +### D3 Remediation Checklist + +- [ ] [{ID}] {one-line action to resolve the finding} + +## Tool/Skill Tag Completeness + +{D4 findings with BLOCK/WARN/INFO severity badges and FG-D4-NN IDs} + +### D4 Remediation Checklist + +- [ ] [{ID}] {one-line action to resolve the finding} + +## Boundary Coupling + +{D5 findings with BLOCK/WARN/INFO severity badges and FG-D5-NN IDs} + +### D5 Remediation Checklist + +- [ ] [{ID}] {one-line action to resolve the finding} + +## Test Marker Coverage + +{D6 findings with BLOCK/WARN/INFO severity badges and FG-D6-NN IDs} + +### D6 Remediation Checklist + +- [ ] [{ID}] {one-line action to resolve the finding} + +## Summary + +| Severity | Count | +|----------|-------| +| BLOCK | N | +| WARN | N | +| INFO | N | +| **Total**| N | +``` + +### Step 3: Print Handoff + +Print to terminal: + +``` +[audit-feature-gates] Done. + BLOCK: {N} | WARN: {N} | INFO: {N} + Report: {AUTOSKILLIT_TEMP}/audit-feature-gates/feature_gate_audit_{ts}.md +``` + +--- + +## Output Location + +``` +{{AUTOSKILLIT_TEMP}}/audit-feature-gates/ +└── feature_gate_audit_{YYYY-MM-DD_HHMMSS}.md +``` + +## Related Skills + +- `/autoskillit:validate-audit` — validates the report produced by this skill +- `/autoskillit:audit-arch` — parallel audit chain in `full-audit.yaml` +- `/autoskillit:audit-tests` — parallel audit chain in `full-audit.yaml` +- `/autoskillit:audit-cohesion` — parallel audit chain in `full-audit.yaml` diff --git a/src/autoskillit/skills_extended/audit-friction/SKILL.md b/src/autoskillit/skills_extended/audit-friction/SKILL.md index 4b671b5040..59caccc218 100644 --- a/src/autoskillit/skills_extended/audit-friction/SKILL.md +++ b/src/autoskillit/skills_extended/audit-friction/SKILL.md @@ -32,6 +32,7 @@ The user may provide a "since" date (e.g., `2/7`, `2026-02-07`, `last month`). I - Create files outside `{{AUTOSKILLIT_TEMP}}/audit-friction/` directory - Have subagents write files — they return all findings in response text only - Analyze subagent log subdirectories (`*/subagents/`) — top-level session files only +- Run subagents in the background (`run_in_background: true` is prohibited) **ALWAYS:** - Use subagents heavily for parallel log analysis @@ -61,13 +62,13 @@ Logs are too large to read in full. All analysis uses targeted grep commands to grep -n '"is_error".*true' FILE | head -100 # Error keywords in content -grep -in '"not found\|permission denied\|no such file\|command not found\|ENOENT\|"failed\|"cannot\|"error"\|exit code [1-9]\|Traceback\|AssertionError' FILE | head -200 +rg -i '"not found|permission denied|no such file|command not found|ENOENT|"failed|"cannot|"error"|exit code [1-9]|Traceback|AssertionError' FILE | head -200 # Human correction language (look inside "type":"human" lines) -grep -n '"type":"human"' FILE | grep -i 'wrong\|"no,\|try again\|you already\|that.s not\|incorrect\|revert\|undo\|stop' | head -50 +rg -n '"type":"human"' FILE | rg -i 'wrong|"no,|try again|you already|that.s not|incorrect|revert|undo|stop' | head -50 # Test / build failures -grep -in 'FAILED\|test.*failed\|build.*failed\|compile.*error\|syntax error' FILE | head -100 +rg -i 'FAILED|test.*failed|build.*failed|compile.*error|syntax error' FILE | head -100 # Consecutive tool call loops — extract tool names in document order, show runs of 2+ back-to-back grep -o '"name":"[^"]*"' FILE | awk -F'"' '{print $4}' | uniq -c | awk '$1>=2' | head -30 @@ -76,7 +77,7 @@ grep -o '"name":"[^"]*"' FILE | awk -F'"' '{print $4}' | uniq -c | awk '$1>=2' | grep -n '"name":"TOOL_NAME"' FILE | head -30 # Permission / access blockers -grep -in 'permission denied\|access denied\|not allowed\|forbidden' FILE | head -50 +rg -i 'permission denied|access denied|not allowed|forbidden' FILE | head -50 ``` For each match batch, pull context (`-A 10 -B 10`) to confirm it is a real friction event rather than incidental text, then record the line range and category. diff --git a/src/autoskillit/skills_extended/audit-impl/SKILL.md b/src/autoskillit/skills_extended/audit-impl/SKILL.md index 0b8cb65d66..00bb29b683 100644 --- a/src/autoskillit/skills_extended/audit-impl/SKILL.md +++ b/src/autoskillit/skills_extended/audit-impl/SKILL.md @@ -38,7 +38,7 @@ requirements, scope creep, and unexpected changes. Produces a GO or NO GO verdic stable after merge_worktree destroys named refs); a branch name is also accepted for standalone invocations. A live worktree path is accepted for legacy use (Step 0 extracts the branch name automatically). -- `base_branch` — branch to diff against (default: `integration`) +- `base_branch` — branch to diff against (default: `develop`) - `conflict_report_paths` (optional) — comma-separated list of absolute paths to conflict resolution reports produced by `resolve-merge-conflicts`. When provided and non-empty, cross-reference resolution decisions against plan intent in Step 2.5. @@ -50,6 +50,7 @@ requirements, scope creep, and unexpected changes. Produces a GO or NO GO verdic - Run tests — this skill audits, it does not fix - Create files outside `{{AUTOSKILLIT_TEMP}}/audit-impl/` - Emit a GO verdict when any `MISSING` or `CONFLICT` finding exists +- Run subagents in the background (`run_in_background: true` is prohibited) **ALWAYS:** - Use Explore subagents for all file reads and diff retrieval @@ -116,30 +117,6 @@ exist (e.g., plan file arguments, `{{AUTOSKILLIT_TEMP}}/investigate/` reports, e `Glob` or `ls` to confirm the path exists first. This prevents ENOENT errors that cascade into sibling parallel-call cancellations. -### Step 0.5 — Code-Index Initialization (required before any code-index tool call) - -Call `set_project_path` with the repo root where this skill was invoked (not a worktree path): - -``` -mcp__code-index__set_project_path(path="{PROJECT_ROOT}") -``` - -Code-index tools require **project-relative paths**. Always use paths like: - - src//some_module.py - -NOT absolute paths like: - - /absolute/path/to/src//some_module.py - -> **Note:** Code-index tools (`find_files`, `search_code_advanced`, `get_file_summary`, -> `get_symbol_body`) are only available when the `code-index` MCP server is configured. -> If `set_project_path` returns an error, fall back to native `Glob` and `Grep` tools -> for the same searches — they provide equivalent results without the code-index server. - -Agents launched via `run_skill` inherit no code-index state from the parent session — this -call is mandatory at the start of every headless session that uses code-index tools. - ### Step 1 — Load Plans via Parallel Subagents Launch one Explore subagent per plan file in parallel. Each returns: @@ -330,7 +307,13 @@ Generated by `/autoskillit:audit-impl` after auditing: ## Verification After remediation: -- Run `task test-all` +- Run: + ```bash + CURRENT_BRANCH=$(git rev-parse --abbrev-ref HEAD) && \ + AUTOSKILLIT_TEST_FILTER="${AUTOSKILLIT_TEST_FILTER:-conservative}" \ + AUTOSKILLIT_TEST_BASE_REF=$(cat "{{AUTOSKILLIT_TEMP}}/worktrees/${CURRENT_BRANCH}/base-branch" 2>/dev/null || echo "") \ + task test-all + ``` - Re-run `/autoskillit:audit-impl` to confirm GO ``` diff --git a/src/autoskillit/skills_extended/audit-review-decisions/SKILL.md b/src/autoskillit/skills_extended/audit-review-decisions/SKILL.md new file mode 100644 index 0000000000..93b1c28c07 --- /dev/null +++ b/src/autoskillit/skills_extended/audit-review-decisions/SKILL.md @@ -0,0 +1,328 @@ +--- +name: audit-review-decisions +categories: [audit] +description: > + Audit merged PR review threads for agreed-but-deferred suggestions (design decisions, + future work, out-of-scope items) that were never implemented. Mines REVIEW-FLAG markers + from resolve-review and legacy keyword signals. Produces a structured markdown report + with VALID/RESOLVED/STALE classifications and annotates processed threads with [AUDIT] + markers to prevent re-identification on future runs. +hooks: + PreToolUse: + - matcher: "*" + hooks: + - type: command + command: "echo 'Auditing PR review decisions...'" + once: true +--- + +# Audit Review Decisions Skill + +Mine merged PR review threads for agreed-but-deferred suggestions that were never +implemented. Identify review debt before it compounds. + +## When to Use + +- User says "audit review decisions", "find deferred review items", "surface review + debt", "what did reviewers flag for later" + +## Arguments + +- `$1` — Time period (e.g. `14d`, `30d`, `7d`). Default: `14d`. +- `$2` — Output path. Default: + `${AUTOSKILLIT_TEMP}/audit-review-decisions/review_decisions_audit_$(date +%Y-%m-%d_%H%M%S).md` + +## Critical Constraints + +**NEVER:** +- Create files outside `${AUTOSKILLIT_TEMP}/audit-review-decisions/` +- Have triage or validation subagents make GitHub API calls (local data only for Step 2) +- Post duplicate `[AUDIT]` markers — check for existing marker before posting +- Run subagents in the background (`run_in_background: true` is prohibited) +- Use `gh pr list` without `--limit` to avoid pagination truncation +- Use `\|` in Grep patterns — use `|` for alternation (ERE, not BRE) + +**ALWAYS:** +- Save raw PR JSON to temp before any analysis (Step 1) +- Use GraphQL alias batching (~20 PRs per query) for data collection +- Include `rateLimit { cost remaining resetAt }` in every GraphQL query +- Sleep 1s between consecutive mutating GitHub API calls (Step 5 watermark posts) +- Step 2 triage subagents read local JSON files only — zero API calls +- Step 3 validation subagents grep the actual current codebase +- Skip threads that already contain an `[AUDIT]` comment +- Resolve owner/repo from `git remote get-url origin` — never hardcode +- Use `/autoskillit:` prefix when invoking any other skill + +--- + +## Workflow + +### Step 0: Watermark Resolution + +1. Parse `$1` for time period. Default `14d`. Compute `PERIOD_DAYS`. + +2. Resolve `OWNER` and `REPO` from `git remote get-url origin`. + +3. Query the most recent `[AUDIT]` sentinel comment across recently merged PRs: + ```bash + gh api graphql -f query=' + query($owner:String!, $name:String!) { + rateLimit { cost remaining resetAt } + repository(owner:$owner, name:$name) { + pullRequests(first:500, states:MERGED, orderBy:{field:UPDATED_AT,direction:DESC}) { + nodes { number + reviewThreads(first:50) { + nodes { comments(first:10) { nodes { body createdAt } } } + } + } + } + } + }' -f owner="${OWNER}" -f name="${REPO}" + ``` + Extract the most recent `createdAt` from any comment whose `body` starts with + `[AUDIT]`. Store as `LAST_AUDIT_TS` (empty string if none — first run). + +4. Compute `SCAN_SINCE`: + - If `LAST_AUDIT_TS` is set: `max(LAST_AUDIT_TS, date -d "now - PERIOD_DAYS days")` + - Else: `date -d "now - PERIOD_DAYS days" --iso-8601=seconds` + +5. Log: `Scan window: ${SCAN_SINCE} to now (${PERIOD_DAYS}d configured, last audit: ${LAST_AUDIT_TS:-none})` + +--- + +### Step 1: Data Collection (GraphQL Batch) + +1. List merged PRs in the scan window: + ```bash + SCAN_DATE=$(echo "${SCAN_SINCE}" | cut -c1-10) + PR_NUMS=$(gh pr list --state merged \ + --search "merged:>=${SCAN_DATE}" \ + --json number --limit 500 | jq -r '.[].number') + ``` + +2. Create temp directory: + ```bash + mkdir -p "${AUTOSKILLIT_TEMP}/audit-review-decisions/raw" + ``` + +3. Batch fetch in groups of 20 using GraphQL aliases. For each batch, build a query + with aliased `pr${i}: pullRequest(number: ${NUM})` nodes. Each node fetches: + ```graphql + number title mergedAt + reviews(first: 100) { + nodes { author { login } body state submittedAt } + } + reviewThreads(first: 100) { + pageInfo { hasNextPage endCursor } + nodes { + isResolved + comments(first: 100) { + nodes { databaseId author { login } body path line createdAt } + } + } + } + ``` + Include `rateLimit { cost remaining resetAt }` at query root. + After the initial fetch, for each PR where `reviewThreads.pageInfo.hasNextPage` is + `true`, issue additional aliased queries with `reviewThreads(first:100, after:$endCursor)` + until `hasNextPage` is `false`. Merge the `nodes` arrays across pages before filtering. + +4. For each PR in the batch response: + - Filter out threads whose `comments` list contains any comment with `body` + starting with `[AUDIT]` (already watermarked — skip entirely). + - If the PR has zero remaining threads: skip saving. + - Otherwise: save filtered data to + `${AUTOSKILLIT_TEMP}/audit-review-decisions/raw/pr_${number}.json` + +--- + +### Step 2: Triage (Haiku — Broad Pass) + +1. List all JSON files in `raw/`. Split into batches of ~5 files per agent. + +2. Launch **parallel Haiku subagents** (one per batch, `model: "haiku"`). Each agent: + - Reads its assigned JSON files only (no API calls). + - Flags a thread if it matches any signal: + - ` ``` #### Multiple plans format: @@ -201,6 +200,7 @@ Plan files: - `{plan_path_2}` 🤖 Generated with [Claude Code](https://claude.com/claude-code) via AutoSkillit + ``` ### Step 4: Check GitHub Availability @@ -236,7 +236,7 @@ On success: pr_url = https://github.com/owner/repo/pull/N ``` -On graceful degradation (no `gh` or not authenticated): +When `gh` is not available or not authenticated: ``` pr_url = ``` diff --git a/src/autoskillit/skills_extended/compose-research-pr/SKILL.md b/src/autoskillit/skills_extended/compose-research-pr/SKILL.md index 4a21449b44..3051b1de94 100644 --- a/src/autoskillit/skills_extended/compose-research-pr/SKILL.md +++ b/src/autoskillit/skills_extended/compose-research-pr/SKILL.md @@ -35,10 +35,11 @@ Does NOT invoke lens skills or other sub-skills. **NEVER:** - Invoke any sub-skills or slash commands during execution - Auto-merge or approve the PR — research PRs are for human review only -- Fail the pipeline if `gh` is unavailable — emit `pr_url = ` (empty string) and exit 0 +- Fail the pipeline when `gh` is not accessible — emit `pr_url = ` (empty string) and return - Create files outside `{{AUTOSKILLIT_TEMP}}/compose-research-pr/` (relative to the current working directory) - Invent mermaid classDef colors — when embedding validated diagrams, include them verbatim. Using ONLY classDef styles from the mermaid skill when composing the PR body. +- Run subagents in the background (`run_in_background: true` is prohibited) **ALWAYS:** - Check `gh auth status` before attempting GitHub operations @@ -187,7 +188,7 @@ as your final output: pr_url = https://github.com/owner/repo/pull/N ``` -Where `pr_url` is the absolute GitHub PR URL, or an empty string if GitHub is unavailable: +Where `pr_url` is the absolute GitHub PR URL, or an empty string when GitHub is not accessible: ``` pr_url = diff --git a/src/autoskillit/skills_extended/design-guards/SKILL.md b/src/autoskillit/skills_extended/design-guards/SKILL.md index fe31aa1176..9bdaeeb89b 100644 --- a/src/autoskillit/skills_extended/design-guards/SKILL.md +++ b/src/autoskillit/skills_extended/design-guards/SKILL.md @@ -30,6 +30,7 @@ Path to a bug pattern audit report (typically under `{{AUTOSKILLIT_TEMP}}/audit- - Modify any source code files - Propose bandaid fixes or direct-only patches - Create files outside `{{AUTOSKILLIT_TEMP}}/design-guards/` directory +- Run subagents in the background (`run_in_background: true` is prohibited) **ALWAYS:** - Use subagents for parallel codebase exploration diff --git a/src/autoskillit/skills_extended/diagnose-ci/SKILL.md b/src/autoskillit/skills_extended/diagnose-ci/SKILL.md index 9ef557cb9f..50f79196fa 100644 --- a/src/autoskillit/skills_extended/diagnose-ci/SKILL.md +++ b/src/autoskillit/skills_extended/diagnose-ci/SKILL.md @@ -1,6 +1,7 @@ --- name: diagnose-ci categories: [ci] +description: Diagnostic executor for CI failures. ALWAYS invoke this skill when instructed to diagnose CI failures. Do not fetch CI logs directly — use this skill first to load the diagnosis workflow. hooks: PreToolUse: - matcher: "*" @@ -38,7 +39,6 @@ before routing to `resolve-failures`. - Block on missing `gh` CLI — write a minimal `failure_type=unknown` diagnosis instead **ALWAYS:** -- Initialize code-index: call `set_project_path` to current cwd before any search - Write the diagnosis file before emitting output tokens - Emit the four output tokens (`diagnosis_path`, `failure_type`, `failure_subtype`, `is_fixable`) at the end of the response on their own lines @@ -55,13 +55,7 @@ best-effort with whatever diagnosis was written (or none). ## Workflow -### Step 1: Initialize Code Index - -``` -mcp__code-index__set_project_path(path=) -``` - -### Step 2: Discover Run ID (if not provided) +### Step 1: Discover Run ID (if not provided) If `run_id` is not provided as an argument (or is `-`), construct the `gh run list` command with any provided filters: @@ -84,16 +78,16 @@ gh run list --branch {branch} --workflow {workflow} --event {event} --limit 1 -- Parse the JSON to extract `databaseId` as `run_id`. -If `gh` is unavailable or the command fails, skip to Step 5 (write minimal diagnosis). +When `gh` is not accessible or the command fails, proceed to Step 5 (write diagnosis report). -### Step 3: Fetch Failure Summary +### Step 2: Fetch Failure Summary ```bash gh run view {run_id} --log-failed ``` Capture the output (stdout). This is the primary failure log. -### Step 4: Fetch Per-Job Logs +### Step 3: Fetch Per-Job Logs For each failing job in `ci_failed_jobs` (or all failed jobs from `gh run view` if not provided): ```bash @@ -106,7 +100,7 @@ gh api repos/{owner}/{repo}/actions/jobs/{job_id}/logs Use `gh repo view --json nameWithOwner` to resolve `{owner}/{repo}` if needed. -### Step 5: Classify Failure +### Step 4: Classify Failure Analyze the log output to classify `failure_type` as one of: - `test` — pytest/jest/unit test failures @@ -116,7 +110,7 @@ Analyze the log output to classify `failure_type` as one of: - `env` — missing environment variables, secrets, or infrastructure issues - `unknown` — cannot determine from logs -#### Step 5a: Subtype Classification +#### Step 4a: Subtype Classification After determining `failure_type`, classify `failure_subtype` using the following error-pattern decision tree (first match wins): @@ -139,15 +133,15 @@ local test result): |---|---| | `flaky` or `timing_race` | `flake_suspected` | | `deterministic` | `ci_only_failure` (if local tests pass) or `real_fix` (if fixable locally) | -| `fixture` or `import` | `ci_only_failure` (if local tests pass) | -| `env` | `ci_only_failure` (conservative) | -| `unknown` | `ci_only_failure` (conservative) | +| `fixture` or `import` | `flake_suspected` (if local tests pass) | +| `env` | `flake_suspected` | +| `unknown` | `flake_suspected` | Determine `is_fixable`: - `true` for `test`, `lint`, `build`, `type_check` - `false` for `env`, `unknown` -### Step 6: Write Diagnosis Report +### Step 5: Write Diagnosis Report Create directory `{{AUTOSKILLIT_TEMP}}/diagnose-ci/` if it doesn't exist. Write the diagnosis file: @@ -179,7 +173,7 @@ failure_subtype = {failure_subtype} Save to `{{AUTOSKILLIT_TEMP}}/diagnose-ci/diagnosis_{timestamp}.md`. (relative to the current working directory) -### Step 7: Emit Output Tokens +### Step 6: Emit Output Tokens Emit these tokens on their own lines at the end of your response: @@ -197,7 +191,7 @@ is_fixable = true|false ## gh Unavailable Fallback -If `gh` is unavailable at any step, write a minimal diagnosis: +When `gh` is not accessible at any step, write a minimal diagnosis: - `failure_type = unknown` - `failure_subtype = unknown` - `is_fixable = false` diff --git a/src/autoskillit/skills_extended/dry-walkthrough/SKILL.md b/src/autoskillit/skills_extended/dry-walkthrough/SKILL.md index f31b025e1e..0e87e33657 100644 --- a/src/autoskillit/skills_extended/dry-walkthrough/SKILL.md +++ b/src/autoskillit/skills_extended/dry-walkthrough/SKILL.md @@ -1,6 +1,6 @@ --- name: dry-walkthrough -description: Validate an implementation plan by tracing through each change without implementing. Use when user says dry walkthrough, drywalkthrough, validate plan, or check plan. Identifies gaps, fixes the plan directly, and reports changes to terminal. +description: Plan validation executor. ALWAYS invoke this skill when instructed to validate or dry-walkthrough a plan. Do not read the plan or trace changes directly — use this skill first to load the validation workflow. hooks: PreToolUse: - matcher: "*" @@ -45,6 +45,7 @@ The plan file must remain a **clean, self-contained implementation instruction s - Remove or defer goals or phases from the plan - Reduce the plan's scope to a "simpler fix" - the plan defines the problem scope, not you - Consider effort as a reason for choosing one approach over another +- Run subagents in the background (`run_in_background: true` is prohibited) **ALWAYS:** - Keep the plan as clean implementation instructions only (information/background helpful to implementation is okay) @@ -102,6 +103,13 @@ For each phase, verify using subagents: 6. Does this violate any project rules? 7. Does the implmentation make sense given the reality of the current state of code? 8. Is every new component, class, or function actually wired into the call chain? Nothing should be created but left unconnected. +9. If the plan adds tests that call mutating methods on shared objects (singletons, global + registries, server instances), does the plan account for state restoration? Scan the + plan text for method calls on module-scope objects (e.g., enable(...), disable(...), + register(...), connect(...)). If found, verify the plan specifies cleanup. Inspect + the target test directory's existing isolation pattern (conftest autouse fixtures, + setup/teardown) and confirm the plan's new tests comply. If the plan prescribes + mutating shared state without specifying cleanup, flag it. ``` ### Step 3: Check Cross-Phase Dependencies @@ -121,11 +129,11 @@ PROJECT RULES CHECKLIST: [ ] No PR breakdown sections [ ] Follows existing architectural patterns [ ] Uses existing utilities (not reinventing) unless refactoring is part of plan or provides major improvement -[ ] Test command uses the project's configured `test_check.command` (from `.autoskillit/config.yaml`, default: `task test-check`) — no unconfigured direct test runner invocations (pytest, python -m pytest, etc.) +[ ] Test command uses the project's configured `test_check.commands` (list of commands, if set) or `test_check.command` (from `.autoskillit/config.yaml`, default: `task test-check`) — no unconfigured direct test runner invocations (pytest, python -m pytest, etc.) [ ] Worktree setup uses `worktree_setup.command` or `task install-worktree` — no hardcoded `uv venv`, `pip install`, or direct package manager invocations ``` -**Test command enforcement:** Scan the entire plan for any test invocation. Read the project's configured test command from `test_check.command` in `.autoskillit/config.yaml` (default: `task test-check` if absent or unconfigured). If the plan contains `pytest`, `python -m pytest`, `make test`, or any other unconfigured test runner invocation, replace it with the config-driven command. +**Test command enforcement:** Scan the entire plan for any test invocation. Read the project's configured test commands from `.autoskillit/config.yaml`: check `test_check.commands` first (list of ordered commands, if set); fall back to `test_check.command` (single command, default: `task test-check`). If the plan contains `pytest`, `python -m pytest`, `make test`, or any other unconfigured test runner invocation, replace it with the config-driven command(s). **Worktree setup enforcement:** Scan the plan for any worktree environment setup. The plan should reference the project's configured `worktree_setup.command` or `task install-worktree`. If the plan contains hardcoded `uv venv`, `uv pip install`, `pip install -e`, `npm install` (as worktree setup, not as a configured command), flag it and replace with the config-driven approach. @@ -150,7 +158,7 @@ This is a quick cross-reference sanity check — not a deep audit. 3. For each matching commit, determine signal strength: - **Strong signal:** The plan proposes to add a function or class name that appears in the commit's diff as a deletion — check with: - `git show {hash} | grep "^-def \|^-class \|^-async def "` and compare against + `git show {hash} | rg "^-def |^-class |^-async def "` and compare against function/class names the plan introduces. - **Weak signal:** Same file touched + fix/revert keyword in message, but no symbol-level match. diff --git a/src/autoskillit/skills_extended/elaborate-phase/SKILL.md b/src/autoskillit/skills_extended/elaborate-phase/SKILL.md index 1a943e6df4..2467007d4c 100644 --- a/src/autoskillit/skills_extended/elaborate-phase/SKILL.md +++ b/src/autoskillit/skills_extended/elaborate-phase/SKILL.md @@ -35,6 +35,7 @@ Elaborate a single phase from a high-level migration plan into a complete, self- - Elaborate multiple phases at once - one phase per invocation - Make assumptions about codebase state without verifying - Read previous `Phase#.md` files unless you have a specific question that requires looking up a detail +- Run subagents in the background (`run_in_background: true` is prohibited) **ALWAYS:** - Assess current codebase state with subagents FIRST diff --git a/src/autoskillit/skills_extended/enrich-issues/SKILL.md b/src/autoskillit/skills_extended/enrich-issues/SKILL.md index 749c60ec15..3ab44e29fd 100644 --- a/src/autoskillit/skills_extended/enrich-issues/SKILL.md +++ b/src/autoskillit/skills_extended/enrich-issues/SKILL.md @@ -119,24 +119,31 @@ Classify the issue into one of three categories: **Too vague** — cannot extract clear acceptance criteria (e.g., "improve X", no measurable outcome, contradictory claims): -- Post a clarifying comment: +- Append a `## Triage Note` section to the issue body: ```bash - gh issue comment N --body "This issue needs more detail before requirements - can be generated. Consider: What is the expected outcome? What signals success? - If the goal is unclear, relabeling to \`recipe:remediation\` may be appropriate - for investigation first." + ENRICH_BODY_FILE="{{AUTOSKILLIT_TEMP}}/enrich-issues/enrich_{N}_$(date +%s).md" + mkdir -p "$(dirname "$ENRICH_BODY_FILE")" + gh issue view N --json body --jq '.body' > "$ENRICH_BODY_FILE" + printf '\n\n---\n\n## Triage Note\n\nThis issue needs more detail before requirements can be generated. Consider: What is the expected outcome? What signals success? If the goal is unclear, relabeling to `recipe:remediation` may be appropriate for investigation first.\n' \ + >> "$ENRICH_BODY_FILE" + gh issue edit N --body-file "$ENRICH_BODY_FILE" + sleep 1 ``` -- Add to `skipped_too_vague`. Do not edit the issue body. +- Add to `skipped_too_vague`. **Mixed concerns** — the issue describes two or more independently-completable sub-features or mixes a bug fix with a new feature: -- Post a comment: +- Append a `## Triage Note` section to the issue body: ```bash - gh issue comment N --body "This issue mixes independent concerns. Consider - running \`/autoskillit:issue-splitter\` to split it into focused sub-issues - before enrichment." + ENRICH_BODY_FILE="{{AUTOSKILLIT_TEMP}}/enrich-issues/enrich_{N}_$(date +%s).md" + mkdir -p "$(dirname "$ENRICH_BODY_FILE")" + gh issue view N --json body --jq '.body' > "$ENRICH_BODY_FILE" + printf '\n\n---\n\n## Triage Note\n\nThis issue mixes independent concerns. Consider running `/autoskillit:issue-splitter` to split it into focused sub-issues before enrichment.\n' \ + >> "$ENRICH_BODY_FILE" + gh issue edit N --body-file "$ENRICH_BODY_FILE" + sleep 1 ``` -- Add to `skipped_mixed_concerns`. Do not edit the issue body. +- Add to `skipped_mixed_concerns`. **Well-defined** — a single, coherent goal with extractable acceptance criteria: - Proceed to requirement generation (Step 5d). @@ -190,6 +197,7 @@ The API layer exposes skill execution to MCP clients through the headless execut cat "${REQUIREMENTS_FILE}" >> "${EDIT_BODY_FILE}" gh issue edit N --body-file "${EDIT_BODY_FILE}" + sleep 1 # Rate-limit discipline: throttle within each subagent ``` Always fetch the current body immediately before editing to avoid overwriting @@ -222,6 +230,7 @@ After processing all candidates, emit to stdout: skipped - Use `--body` shell substitution (`--body "$(...)`) for `gh issue edit` — always write to `{{AUTOSKILLIT_TEMP}}/enrich-issues/edit_body_{timestamp}.md` and use `--body-file` +- Run subagents in the background (`run_in_background: true` is prohibited) **ALWAYS:** - Respect `--dry-run`: never call `gh issue edit` when this flag is set diff --git a/src/autoskillit/skills_extended/exp-lens-benchmark-representativeness/SKILL.md b/src/autoskillit/skills_extended/exp-lens-benchmark-representativeness/SKILL.md index f9e3ae6cde..21149056e6 100644 --- a/src/autoskillit/skills_extended/exp-lens-benchmark-representativeness/SKILL.md +++ b/src/autoskillit/skills_extended/exp-lens-benchmark-representativeness/SKILL.md @@ -43,6 +43,7 @@ hooks: - Modify any source code files - Do not litter the codebase with useless comments, TODO markers, or explanatory annotations — the skill output and diagram speak for themselves - Create files outside `{{AUTOSKILLIT_TEMP}}/exp-lens-benchmark-representativeness/` +- Run subagents in the background (`run_in_background: true` is prohibited) **ALWAYS:** - Focus on GENERALIZATION GAP between benchmark coverage and claimed scope diff --git a/src/autoskillit/skills_extended/exp-lens-causal-assumptions/SKILL.md b/src/autoskillit/skills_extended/exp-lens-causal-assumptions/SKILL.md index 174611d0b2..72ea69c427 100644 --- a/src/autoskillit/skills_extended/exp-lens-causal-assumptions/SKILL.md +++ b/src/autoskillit/skills_extended/exp-lens-causal-assumptions/SKILL.md @@ -43,6 +43,7 @@ hooks: - Modify any source code files - Do not litter the codebase with useless comments, TODO markers, or explanatory annotations — the skill output and diagram speak for themselves - Create files outside `{{AUTOSKILLIT_TEMP}}/exp-lens-causal-assumptions/` +- Run subagents in the background (`run_in_background: true` is prohibited) **ALWAYS:** - Classify every variable as Treatment, Outcome, Confounder, Mediator, Collider, Instrument, or Selection variable diff --git a/src/autoskillit/skills_extended/exp-lens-comparator-construction/SKILL.md b/src/autoskillit/skills_extended/exp-lens-comparator-construction/SKILL.md index eeb45adb7b..764fdfd4e4 100644 --- a/src/autoskillit/skills_extended/exp-lens-comparator-construction/SKILL.md +++ b/src/autoskillit/skills_extended/exp-lens-comparator-construction/SKILL.md @@ -44,6 +44,7 @@ hooks: - Do not litter the codebase with useless comments, TODO markers, or explanatory annotations — the skill output and diagram speak for themselves - Accept at face value that baselines received symmetric treatment - Create files outside `{{AUTOSKILLIT_TEMP}}/exp-lens-comparator-construction/` +- Run subagents in the background (`run_in_background: true` is prohibited) **ALWAYS:** - Build a fairness matrix covering all treatment-vs-comparator pairs diff --git a/src/autoskillit/skills_extended/exp-lens-error-budget/SKILL.md b/src/autoskillit/skills_extended/exp-lens-error-budget/SKILL.md index 0616bd48ed..fb18163790 100644 --- a/src/autoskillit/skills_extended/exp-lens-error-budget/SKILL.md +++ b/src/autoskillit/skills_extended/exp-lens-error-budget/SKILL.md @@ -44,6 +44,7 @@ hooks: - Do not litter the codebase with useless comments, TODO markers, or explanatory annotations — the skill output and diagram speak for themselves - Accept default alpha=0.05 without checking whether it is appropriate for the decision context - Create files outside `{{AUTOSKILLIT_TEMP}}/exp-lens-error-budget/` +- Run subagents in the background (`run_in_background: true` is prohibited) **ALWAYS:** - Enumerate every statistical test and account for its error contribution diff --git a/src/autoskillit/skills_extended/exp-lens-estimand-clarity/SKILL.md b/src/autoskillit/skills_extended/exp-lens-estimand-clarity/SKILL.md index 9cfedead24..8c902c070b 100644 --- a/src/autoskillit/skills_extended/exp-lens-estimand-clarity/SKILL.md +++ b/src/autoskillit/skills_extended/exp-lens-estimand-clarity/SKILL.md @@ -43,6 +43,7 @@ hooks: - Modify any source code or experiment files - Do not litter the codebase with useless comments, TODO markers, or explanatory annotations — the skill output and diagram speak for themselves - Create files outside `{{AUTOSKILLIT_TEMP}}/exp-lens-estimand-clarity/` +- Run subagents in the background (`run_in_background: true` is prohibited) **ALWAYS:** - Decompose every stated claim into formal contrast notation (Treatment A vs Treatment B on Outcome Y in Population Z) diff --git a/src/autoskillit/skills_extended/exp-lens-exploratory-confirmatory/SKILL.md b/src/autoskillit/skills_extended/exp-lens-exploratory-confirmatory/SKILL.md index 425aecd444..9e1a6abd33 100644 --- a/src/autoskillit/skills_extended/exp-lens-exploratory-confirmatory/SKILL.md +++ b/src/autoskillit/skills_extended/exp-lens-exploratory-confirmatory/SKILL.md @@ -42,6 +42,7 @@ hooks: **NEVER:** - Modify any source code files - Create files outside `{{AUTOSKILLIT_TEMP}}/exp-lens-exploratory-confirmatory/` +- Run subagents in the background (`run_in_background: true` is prohibited) **ALWAYS:** - Map the full analytic timeline — what was decided before vs. after seeing data diff --git a/src/autoskillit/skills_extended/exp-lens-fair-comparison/SKILL.md b/src/autoskillit/skills_extended/exp-lens-fair-comparison/SKILL.md index 7a6a14b798..464d3bb868 100644 --- a/src/autoskillit/skills_extended/exp-lens-fair-comparison/SKILL.md +++ b/src/autoskillit/skills_extended/exp-lens-fair-comparison/SKILL.md @@ -43,6 +43,7 @@ hooks: - Modify any source code files - Do not litter the codebase with useless comments, TODO markers, or explanatory annotations — the skill output and diagram speak for themselves - Create files outside `{{AUTOSKILLIT_TEMP}}/exp-lens-fair-comparison/` +- Run subagents in the background (`run_in_background: true` is prohibited) **ALWAYS:** - Build the full symmetry matrix — every method against every resource dimension diff --git a/src/autoskillit/skills_extended/exp-lens-governance-risk/SKILL.md b/src/autoskillit/skills_extended/exp-lens-governance-risk/SKILL.md index 84ef227096..cb06b45c8c 100644 --- a/src/autoskillit/skills_extended/exp-lens-governance-risk/SKILL.md +++ b/src/autoskillit/skills_extended/exp-lens-governance-risk/SKILL.md @@ -42,6 +42,7 @@ hooks: **NEVER:** - Modify any source code files - Create files outside `{{AUTOSKILLIT_TEMP}}/exp-lens-governance-risk/` +- Run subagents in the background (`run_in_background: true` is prohibited) **ALWAYS:** - Identify subgroups for whom the experimental evidence may not generalize diff --git a/src/autoskillit/skills_extended/exp-lens-iterative-learning/SKILL.md b/src/autoskillit/skills_extended/exp-lens-iterative-learning/SKILL.md index 6977af37b1..bffc810317 100644 --- a/src/autoskillit/skills_extended/exp-lens-iterative-learning/SKILL.md +++ b/src/autoskillit/skills_extended/exp-lens-iterative-learning/SKILL.md @@ -43,6 +43,7 @@ hooks: - Modify any source code files - Recommend one-factor-at-a-time exploration when interactions are plausible - Create files outside `{{AUTOSKILLIT_TEMP}}/exp-lens-iterative-learning/` +- Run subagents in the background (`run_in_background: true` is prohibited) **ALWAYS:** - Evaluate exploration efficiency against the key uncertainty being reduced diff --git a/src/autoskillit/skills_extended/exp-lens-measurement-validity/SKILL.md b/src/autoskillit/skills_extended/exp-lens-measurement-validity/SKILL.md index 2a401dbc5e..d8ee952612 100644 --- a/src/autoskillit/skills_extended/exp-lens-measurement-validity/SKILL.md +++ b/src/autoskillit/skills_extended/exp-lens-measurement-validity/SKILL.md @@ -43,6 +43,7 @@ hooks: - Modify any source code or experimental artifacts - Do not litter the codebase with useless comments, TODO markers, or explanatory annotations — the skill output and diagram speak for themselves - Create files outside `{{AUTOSKILLIT_TEMP}}/exp-lens-measurement-validity/` +- Run subagents in the background (`run_in_background: true` is prohibited) **ALWAYS:** - Treat every reported metric as a claim requiring a validity argument diff --git a/src/autoskillit/skills_extended/exp-lens-pipeline-integrity/SKILL.md b/src/autoskillit/skills_extended/exp-lens-pipeline-integrity/SKILL.md index c4aebedfd9..cc7d177278 100644 --- a/src/autoskillit/skills_extended/exp-lens-pipeline-integrity/SKILL.md +++ b/src/autoskillit/skills_extended/exp-lens-pipeline-integrity/SKILL.md @@ -43,6 +43,7 @@ hooks: - Modify any source code files - Do not litter the codebase with useless comments, TODO markers, or explanatory annotations — the skill output and diagram speak for themselves - Create files outside `{{AUTOSKILLIT_TEMP}}/exp-lens-pipeline-integrity/` +- Run subagents in the background (`run_in_background: true` is prohibited) **ALWAYS:** - Classify every pipeline stage as pre-split or post-split diff --git a/src/autoskillit/skills_extended/exp-lens-randomization-blocking/SKILL.md b/src/autoskillit/skills_extended/exp-lens-randomization-blocking/SKILL.md index e909984b91..91ecb484b0 100644 --- a/src/autoskillit/skills_extended/exp-lens-randomization-blocking/SKILL.md +++ b/src/autoskillit/skills_extended/exp-lens-randomization-blocking/SKILL.md @@ -43,6 +43,7 @@ hooks: - Modify any source code files - Assume comparability without tracing its source - Create files outside `{{AUTOSKILLIT_TEMP}}/exp-lens-randomization-blocking/` +- Run subagents in the background (`run_in_background: true` is prohibited) **ALWAYS:** - Trace the exact mechanism that creates comparability between treatment groups diff --git a/src/autoskillit/skills_extended/exp-lens-reproducibility-artifacts/SKILL.md b/src/autoskillit/skills_extended/exp-lens-reproducibility-artifacts/SKILL.md index e535ef6f02..39f614aa8f 100644 --- a/src/autoskillit/skills_extended/exp-lens-reproducibility-artifacts/SKILL.md +++ b/src/autoskillit/skills_extended/exp-lens-reproducibility-artifacts/SKILL.md @@ -43,6 +43,7 @@ hooks: - Modify any source code files - Do not litter the codebase with useless comments, TODO markers, or explanatory annotations — the skill output and diagram speak for themselves - Create files outside `{{AUTOSKILLIT_TEMP}}/exp-lens-reproducibility-artifacts/` +- Run subagents in the background (`run_in_background: true` is prohibited) **ALWAYS:** - Trace the full chain from "clone repo" to "reproduce figures" diff --git a/src/autoskillit/skills_extended/exp-lens-sensitivity-robustness/SKILL.md b/src/autoskillit/skills_extended/exp-lens-sensitivity-robustness/SKILL.md index ab912b79b1..4fd412e7e1 100644 --- a/src/autoskillit/skills_extended/exp-lens-sensitivity-robustness/SKILL.md +++ b/src/autoskillit/skills_extended/exp-lens-sensitivity-robustness/SKILL.md @@ -44,6 +44,7 @@ hooks: - Do not litter the codebase with useless comments, TODO markers, or explanatory annotations — the skill output and diagram speak for themselves - Treat "untested" as equivalent to "robust" - Create files outside `{{AUTOSKILLIT_TEMP}}/exp-lens-sensitivity-robustness/` +- Run subagents in the background (`run_in_background: true` is prohibited) **ALWAYS:** - Build a full sensitivity matrix (choices x perturbation types) diff --git a/src/autoskillit/skills_extended/exp-lens-severity-testing/SKILL.md b/src/autoskillit/skills_extended/exp-lens-severity-testing/SKILL.md index 09f4776ad9..2cd1bfdd10 100644 --- a/src/autoskillit/skills_extended/exp-lens-severity-testing/SKILL.md +++ b/src/autoskillit/skills_extended/exp-lens-severity-testing/SKILL.md @@ -42,6 +42,7 @@ hooks: - Modify any source code files - Accept a "pass" result without asking what a false result would have looked like under this design - Create files outside `{{AUTOSKILLIT_TEMP}}/exp-lens-severity-testing/` +- Run subagents in the background (`run_in_background: true` is prohibited) **ALWAYS:** - For every positive claim, identify what error the test was capable of detecting diff --git a/src/autoskillit/skills_extended/exp-lens-unit-interference/SKILL.md b/src/autoskillit/skills_extended/exp-lens-unit-interference/SKILL.md index 4f82797382..0434bdb0ec 100644 --- a/src/autoskillit/skills_extended/exp-lens-unit-interference/SKILL.md +++ b/src/autoskillit/skills_extended/exp-lens-unit-interference/SKILL.md @@ -43,6 +43,7 @@ hooks: - Modify any source code files - Do not litter the codebase with useless comments, TODO markers, or explanatory annotations — the skill output and diagram speak for themselves - Create files outside `{{AUTOSKILLIT_TEMP}}/exp-lens-unit-interference/` +- Run subagents in the background (`run_in_background: true` is prohibited) **ALWAYS:** - Focus on the unit definition and whether SUTVA is plausible diff --git a/src/autoskillit/skills_extended/exp-lens-validity-threats/SKILL.md b/src/autoskillit/skills_extended/exp-lens-validity-threats/SKILL.md index 0ee326958e..b5f16e27c2 100644 --- a/src/autoskillit/skills_extended/exp-lens-validity-threats/SKILL.md +++ b/src/autoskillit/skills_extended/exp-lens-validity-threats/SKILL.md @@ -42,6 +42,7 @@ hooks: **NEVER:** - Modify any source code files - Create files outside `{{AUTOSKILLIT_TEMP}}/exp-lens-validity-threats/` +- Run subagents in the background (`run_in_background: true` is prohibited) **ALWAYS:** - Apply Campbell & Stanley's full threat taxonomy — do not skip threats just because they seem unlikely diff --git a/src/autoskillit/skills_extended/exp-lens-variance-stability/SKILL.md b/src/autoskillit/skills_extended/exp-lens-variance-stability/SKILL.md index 04a18b232c..7dde4e9193 100644 --- a/src/autoskillit/skills_extended/exp-lens-variance-stability/SKILL.md +++ b/src/autoskillit/skills_extended/exp-lens-variance-stability/SKILL.md @@ -43,6 +43,7 @@ hooks: - Modify any source code files - Do not litter the codebase with useless comments, TODO markers, or explanatory annotations — the skill output and diagram speak for themselves - Create files outside `{{AUTOSKILLIT_TEMP}}/exp-lens-variance-stability/` +- Run subagents in the background (`run_in_background: true` is prohibited) **ALWAYS:** - Count the actual number of independent runs — single-run results must be flagged prominently diff --git a/src/autoskillit/skills_extended/generate-report/SKILL.md b/src/autoskillit/skills_extended/generate-report/SKILL.md index 9286e1ef95..4183eae659 100644 --- a/src/autoskillit/skills_extended/generate-report/SKILL.md +++ b/src/autoskillit/skills_extended/generate-report/SKILL.md @@ -29,6 +29,8 @@ results are valid findings, not failures. ``` /autoskillit:generate-report {worktree_path} {results_path} [--inconclusive] +[--output-mode {local|pr}] [--issue-url {url}] +[--experiment-type {type}] [--methodology-traditions {tradition}] ``` - `{worktree_path}` — Absolute path to the worktree (required). First path-like @@ -44,6 +46,15 @@ results are valid findings, not failures. - `--issue-url {url}` — Optional. GitHub issue URL. When output_mode=local and this is supplied, inject a blockquote reference at the top of report.md. In pr mode, skip — the PR body handles the issue link. +- `--experiment-type {type}` — Optional. Experiment type classification (e.g., "benchmark", + "causal_inference"). When absent or "null", write `experiment_type: null` in frontmatter. +- `--methodology-traditions {tradition}` — Optional. Methodology tradition slug from Tier-C + routing (e.g., "controlled_intervention"). When absent or "null", write + `methodology_traditions: []` in frontmatter; otherwise wrap in a YAML list. +- `--design-review-verdict {verdict}` — Optional. GO/REVISE/STOP verdict from design review. +- `--disambiguation-rule-applied {rule}` — Optional. Disambiguation rule applied during Tier-C routing. +- `--tier-c-lens {lens}` — Optional. Tier-C vis-lens selected. +- `--classification-timestamp {timestamp}` — Optional. UTC timestamp of design classification. ## Inputs @@ -61,6 +72,7 @@ In addition to the arguments above, this skill reads from the worktree: - Omit the methodology section — reproducibility requires it - Frame inconclusive results as failures — they are valid findings - Create the report outside the worktree's `research/` directory +- Run subagents in the background (`run_in_background: true` is prohibited) **ALWAYS:** - Use `model: "sonnet"` when spawning all subagents via the Task tool @@ -113,7 +125,7 @@ If `--output-mode local` AND `--issue-url {url}` is present: ``` (Include a blank line after the blockquote before the title.) -In pr mode: skip this step entirely — the PR body contains the issue link. +In pr mode: omit this step entirely — the PR body contains the issue link. ### Step 2 — Determine Report Type @@ -146,7 +158,7 @@ Based on the `--inconclusive` flag and the experiment results status: If `${RESEARCH_DIR}/visualization-plan.md` exists: 1. Read `visualization-plan.md`. If it contains zero figure specs (empty plan), - skip all sub-steps and proceed to Step 3. + omit all sub-steps and proceed to Step 3. 2. Identify the experiment's Docker image. The image tag is `research-{slug}` where `{slug}` is the experiment directory name. Verify the image exists: @@ -215,9 +227,33 @@ research/YYYY-MM-DD-{slug}/ The `{slug}` is a kebab-case summary of the research topic (max 40 chars). +<<<<<<< HEAD +======= +Write a YAML frontmatter block (fenced with `---`) at the very top of report.md, +before the title heading. Use the values from `--experiment-type` and +`--methodology-traditions` flags. If a flag is absent or its value is the literal +string `null`, write the key with value `null` (not omitted). Always include +`generated_at`. If `--output-mode local` with `--issue-url`, the issue blockquote +goes AFTER the frontmatter, before the title. + +>>>>>>> b64b32f7 (fix(review): align --methodology-tradition synopsis to plural form) The report structure: ```markdown +--- +experiment_type: {value from --experiment-type, or null} +methodology_traditions: + - {value from --methodology-traditions, or empty list if null} +disambiguation_rule_applied: {value from --disambiguation-rule-applied, or null} +tier_c_lens: {value from --tier-c-lens, or null} +design_review_verdict: {value from --design-review-verdict, or null} +classification_timestamp: {value from --classification-timestamp, or null} +audit_trail_path: + design_review: research/{slug}/audit/design-review-dashboard.md + visualization_trace: research/{slug}/audit/visualization-plan-trace.md +generated_at: {ISO 8601 timestamp} +--- + # {Research Title} > Research report for [Issue #{N}]({issue_url}) — {date} @@ -348,6 +384,19 @@ analysis if relevant to the experiment type.} {Direct answer to the research question.} +## Design Review Summary + +**Verdict:** {design_review_verdict} + +The design review evaluated the experiment plan against {experiment_type} criteria. +The review confirmed the experimental design meets quality standards. + +For detailed evaluation dimensions, scorecard, and adversarial findings, see +[`audit/design-review-dashboard.md`](audit/design-review-dashboard.md). + +For visualization tier routing decisions and methodology tradition analysis, see +[`audit/visualization-plan-trace.md`](audit/visualization-plan-trace.md). + ## Recommendations *For biology and social science domains, use "Discussion and Future Directions" as the section title instead.* diff --git a/src/autoskillit/skills_extended/implement-experiment/SKILL.md b/src/autoskillit/skills_extended/implement-experiment/SKILL.md index 51bbf16e96..03f439d61b 100644 --- a/src/autoskillit/skills_extended/implement-experiment/SKILL.md +++ b/src/autoskillit/skills_extended/implement-experiment/SKILL.md @@ -50,6 +50,7 @@ tokens after the skill name for the first path-like token (starts with `/`, - Create experiment files outside the planned `research/` subfolder - Execute `git merge` commands (all branch content must be applied via `git cherry-pick` or `git checkout -- `) +- Run subagents in the background (`run_in_background: true` is prohibited) **ALWAYS:** - Create a new worktree from the current branch @@ -61,6 +62,7 @@ tokens after the skill name for the first path-like token (starts with `/`, - Leave the worktree intact when done - Write `tests/test_{script_name}.py` alongside each experiment script created in Step 4 - Run `pytest --collect-only` after creating tests to verify discovery before committing +- **Read before editing**: Before issuing an `Edit` call on any file, ensure you have issued a `Read` on that file earlier in this session. Claude Code rejects `Edit` on unread files — the retry wastes a full API turn at current context size. If you are uncertain whether a file was read, issue a targeted `Read` (offset + limit to the region you plan to edit) rather than risk an error. ## Context Limit Behavior @@ -104,11 +106,12 @@ mkdir -p "{{AUTOSKILLIT_TEMP}}/worktrees/${WORKTREE_NAME}" echo "${CURRENT_BRANCH}" > "{{AUTOSKILLIT_TEMP}}/worktrees/${WORKTREE_NAME}/base-branch" # Set upstream tracking if possible: -if ! git fetch origin "${CURRENT_BRANCH}" 2>/dev/null; then - echo "NOTE: Branch '${CURRENT_BRANCH}' has no remote tracking ref on origin." +REMOTE=$(git remote get-url upstream >/dev/null 2>&1 && echo upstream || echo origin) +if ! git fetch "$REMOTE" "${CURRENT_BRANCH}" 2>/dev/null; then + echo "NOTE: Branch '${CURRENT_BRANCH}' has no remote tracking ref on $REMOTE." fi -if ! git -C "${WORKTREE_PATH}" branch --set-upstream-to="origin/${CURRENT_BRANCH}" "${WORKTREE_NAME}" 2>/dev/null; then - echo "NOTE: Could not set upstream tracking for '${WORKTREE_NAME}' → 'origin/${CURRENT_BRANCH}'." +if ! git -C "${WORKTREE_PATH}" branch --set-upstream-to="${REMOTE}/${CURRENT_BRANCH}" "${WORKTREE_NAME}" 2>/dev/null; then + echo "NOTE: Could not set upstream tracking for '${WORKTREE_NAME}' → '$REMOTE/${CURRENT_BRANCH}'." fi ``` @@ -154,11 +157,13 @@ required** — launch as many additional subagents as needed. - Understanding specific APIs, types, or interfaces the scripts will use - Any other codebase investigation needed to write correct experiment code -### Step 3 — Set Up Container Environment +### Step 3 — Write Environment Artifacts -The research worktree is isolated via Docker. All experiment code runs inside -a container built from the experiment's `environment.yml`. Nothing is installed -on the host. +The environment is already prepared by `setup-environment` (upstream in the +recipe). Read `env_mode` from context to understand what environment was +provisioned (`docker`, `micromamba-host`, or `unavailable`). This step writes +the Dockerfile template and `environment.yml` into the worktree as +reproducibility artifacts — it does NOT build or install anything. **3a — Write the Dockerfile:** @@ -205,12 +210,6 @@ vars: sh: pwd tasks: - build-env: - desc: Build Docker image for this experiment - cmds: - - docker build --build-arg MAMBA_ENV={{.SLUG}} -t {{.IMAGE}} . - dir: "{{.RESEARCH_DIR}}" - run-experiment: desc: Run experiment inside container (volume-mounts research dir) cmds: @@ -226,15 +225,20 @@ tasks: Adjust the `run-experiment` command to match the actual entry-point script from the experiment plan. -**3c — Build the Docker image:** +**3c — Write `environment.yml` and note `env_mode`:** -```bash -cd "${RESEARCH_DIR}" -docker build --build-arg MAMBA_ENV={slug} -t "research-{slug}" . -``` +If the experiment plan specifies an `environment.yml`, write it to +`${RESEARCH_DIR}/environment.yml`. This file is committed to the worktree +as a reproducibility artifact — reviewers can inspect exact dependency +versions. + +Note the `env_mode` from context (set by `setup-environment`): +- `docker` — a container image was pre-built; `run-experiment` will execute inside it +- `micromamba-host` — a host environment was created; `run-experiment` will use `micromamba run` +- `unavailable` — no environment could be provisioned; `run-experiment` will emit `blocked_experiment` +- `none` — standard environment, no special setup needed -Verify the build succeeds before proceeding. If the build fails due to missing -system packages, add them to the `apt-get install` layer and rebuild. +Do NOT invoke any container or environment construction commands. The environment is already ready. **All commands from this point must run from `${WORKTREE_PATH}`.** Use absolute paths to avoid CWD drift across Bash tool calls. diff --git a/src/autoskillit/skills_extended/implement-worktree-no-merge/SKILL.md b/src/autoskillit/skills_extended/implement-worktree-no-merge/SKILL.md index c54865f02c..5b9497848c 100644 --- a/src/autoskillit/skills_extended/implement-worktree-no-merge/SKILL.md +++ b/src/autoskillit/skills_extended/implement-worktree-no-merge/SKILL.md @@ -1,6 +1,7 @@ --- name: implement-worktree-no-merge -description: Implement a plan in an isolated git worktree without merging, testing, or cleaning up. For MCP orchestration use — the orchestrator handles testing and merging separately. +activate_deps: [write-recipe] +description: Implementation executor. ALWAYS invoke this skill when instructed to implement a plan in a worktree. Do not read the plan or edit files directly — use this skill first to load the full implementation workflow. hooks: PreToolUse: - matcher: "*" @@ -38,6 +39,7 @@ The worktree is left intact for the orchestrator to test and merge separately. - Re-run tests just to see failures — grep the saved output file instead - Pipe test output through `tail`, `head`, or other truncation commands - **Execute `git merge` commands** (including `--no-ff`, `--no-commit`, or any variant). All branch content must be applied via `git cherry-pick ` for individual commits or `git checkout -- ` for specific files. `merge_worktree` requires linear commit history — merge commits cannot be rebased and will cause `WORKTREE_INTACT_MERGE_COMMITS_DETECTED` failure. +- Run subagents in the background (`run_in_background: true` is prohibited) **ALWAYS:** - Create a new worktree from the current branch @@ -46,6 +48,8 @@ The worktree is left intact for the orchestrator to test and merge separately. - Implement one phase at a time - Commit per phase with descriptive messages - Leave the worktree intact when done +- **Read before editing**: Before issuing an `Edit` call on any file, ensure you have issued a `Read` on that file earlier in this session. Claude Code rejects `Edit` on unread files — the retry wastes a full API turn at current context size. If you are uncertain whether a file was read, issue a targeted `Read` (offset + limit to the region you plan to edit) rather than risk an error. +- **Read files fully**: When reading a file to understand it in full, read it in a single call without a `limit` parameter. Do not paginate files with sequential offset reads — read once completely. Use `limit`/`offset` only for targeted section reads of files you have already read in full. ## Context Limit Behavior @@ -71,7 +75,7 @@ Correct orchestration on `needs_retry=true`: after the skill name for the first one that starts with `/`, `./`, `{{AUTOSKILLIT_TEMP}}/`, or `.autoskillit/` — that token is the plan path. Ignore any non-path words that appear before it (orchestrators sometimes prepend descriptive text such - as "the verified plan"). If no path-like token is found, treat the entire + as "the verified plan"). When no path-like token is present, treat the entire argument string as pasted plan content. Verify the resolved file exists before proceeding; if it does not, abort with: `"Plan file not found: {path}. Correct format: /autoskillit:implement-worktree-no-merge "` @@ -97,17 +101,28 @@ git worktree add -b "${WORKTREE_NAME}" "${WORKTREE_PATH}" WORKTREE_PATH="$(cd "${WORKTREE_PATH}" && pwd)" # Record the base branch in two ways for reliable discovery by retry-worktree: -# 1) Write an explicit file store (works with any Git version, works offline) -mkdir -p "{{AUTOSKILLIT_TEMP}}/worktrees/${WORKTREE_NAME}" -echo "${CURRENT_BRANCH}" > "{{AUTOSKILLIT_TEMP}}/worktrees/${WORKTREE_NAME}/base-branch" +# 1) Write an explicit file store (stdlib-only, works with any Git version, works offline) +python3 -c " +from pathlib import Path +import tempfile, os +project_root = Path('$(git rev-parse --show-toplevel)') +sidecar_dir = project_root / '.autoskillit' / 'temp' / 'worktrees' / '${WORKTREE_NAME}' +sidecar_dir.mkdir(parents=True, exist_ok=True) +sidecar_file = sidecar_dir / 'base-branch' +fd, tmp = tempfile.mkstemp(dir=str(sidecar_dir)) +os.write(fd, ('${CURRENT_BRANCH}' + '\n').encode()) +os.close(fd) +os.replace(tmp, str(sidecar_file)) +" # 2) Set git upstream tracking (requires remote tracking ref in local fetch cache) -if ! git fetch origin "${CURRENT_BRANCH}" 2>/dev/null; then - echo "NOTE: Branch '${CURRENT_BRANCH}' has no remote tracking ref on origin." - echo " merge_worktree will fail unless you push first: git push -u origin ${CURRENT_BRANCH}" +REMOTE=$(git remote get-url upstream 2>/dev/null | grep -qv "^file://" && echo upstream || echo origin) +if ! git fetch "$REMOTE" "${CURRENT_BRANCH}" 2>/dev/null; then + echo "NOTE: Branch '${CURRENT_BRANCH}' has no remote tracking ref on $REMOTE." + echo " merge_worktree will fail unless you push first: git push -u $REMOTE ${CURRENT_BRANCH}" echo " Continuing — implementation will proceed, but the merge step will be blocked." fi -if ! git -C "${WORKTREE_PATH}" branch --set-upstream-to="origin/${CURRENT_BRANCH}" "${WORKTREE_NAME}" 2>/dev/null; then - echo "NOTE: Could not set upstream tracking for '${WORKTREE_NAME}' → 'origin/${CURRENT_BRANCH}'." +if ! git -C "${WORKTREE_PATH}" branch --set-upstream-to="${REMOTE}/${CURRENT_BRANCH}" "${WORKTREE_NAME}" 2>/dev/null; then + echo "NOTE: Could not set upstream tracking for '${WORKTREE_NAME}' → '$REMOTE/${CURRENT_BRANCH}'." fi ``` @@ -132,16 +147,6 @@ execution layer scans `assistant_messages` for `worktree_path=` and surfaces it as a top-level field in the `run_skill` JSON response. The orchestrator reads this field directly without filesystem discovery heuristics. -### Step 1.5: Initialize Code Index for Original Project - -Set the MCP code-index project path to the **original project directory** so Explore subagents can use code search tools: - -``` -mcp__code-index__set_project_path(path="{ORIGINAL_PROJECT_PATH}") -``` - -This must happen before Step 2 or any code-index search tools will fail with "Project path not set." - ### Step 2: Deep System Understanding (Subagents) Before implementing ANY code, launch parallel Explore subagents to understand affected systems: @@ -164,16 +169,6 @@ task install-worktree # or equivalent for the project type **All commands in Steps 4–5 must run from `${WORKTREE_PATH}`.** Use absolute paths to avoid CWD drift across Bash tool calls. -### Step 3.5: Re-point Code Index to Worktree (REQUIRED) - -**CRITICAL:** After setting up the worktree environment, you **MUST** update the MCP code-index project path to the worktree: - -``` -mcp__code-index__set_project_path(path="${WORKTREE_PATH}") -``` - -**Failure to do this means code-index searches will return results from the original project, not your worktree.** - ### Step 4: Implement Phase by Phase For each phase, begin implementation immediately (no announcement): @@ -238,14 +233,6 @@ created from the post-merge state of the base branch, not from Part N's base commit. This is a global sequencing rule — it applies even when operating off-recipe. -### Step 6.5: Reset Code Index to Original Project (REQUIRED) - -**CRITICAL:** After completion, you **MUST** reset the MCP code-index project path back to the original project directory: - -``` -mcp__code-index__set_project_path(path="{ORIGINAL_PROJECT_PATH}") -``` - ## Error Handling - **Worktree creation fails** — check `git worktree list`, suggest `git worktree prune` diff --git a/src/autoskillit/skills_extended/implement-worktree/SKILL.md b/src/autoskillit/skills_extended/implement-worktree/SKILL.md index a3ac73cedc..5c297be7b1 100644 --- a/src/autoskillit/skills_extended/implement-worktree/SKILL.md +++ b/src/autoskillit/skills_extended/implement-worktree/SKILL.md @@ -1,6 +1,7 @@ --- name: implement-worktree -description: Implement a plan in an isolated git worktree. Use when user says "implement in worktree", "worktree implement", or "isolated implementation". Creates a worktree from current branch, explores affected systems with subagents, then implements phase by phase. +activate_deps: [write-recipe] +description: Worktree implementation executor. ALWAYS invoke this skill when instructed to implement a plan in a worktree with testing and merging. Do not read the plan or edit files directly — use this skill first to load the full implementation workflow. hooks: PreToolUse: - matcher: "*" @@ -32,6 +33,7 @@ Implement a provided plan in an isolated git worktree branched from the current - Consider implementation complete if ANY test fails - Blame test failures on "pre-existing issues" — ALL tests must pass - Re-run tests just to see failures — grep the saved output file instead +- Run subagents in the background (`run_in_background: true` is prohibited) **ALWAYS:** - Create a new worktree from the current branch @@ -40,6 +42,7 @@ Implement a provided plan in an isolated git worktree branched from the current - Implement one phase at a time - Run the project's test suite after implementation - Rebase onto base branch before completion (ready for squash-and-merge) +- **Read before editing**: Before issuing an `Edit` call on any file, ensure you have issued a `Read` on that file earlier in this session. Claude Code rejects `Edit` on unread files — the retry wastes a full API turn at current context size. If you are uncertain whether a file was read, issue a targeted `Read` (offset + limit to the region you plan to edit) rather than risk an error. ## Context Limit Behavior @@ -90,16 +93,6 @@ mkdir -p "{{AUTOSKILLIT_TEMP}}/worktrees/${WORKTREE_NAME}" echo "${CURRENT_BRANCH}" > "{{AUTOSKILLIT_TEMP}}/worktrees/${WORKTREE_NAME}/base-branch" ``` -### Step 1.5: Initialize Code Index for Original Project - -Set the MCP code-index project path to the **original project directory** so Explore subagents can use code search tools: - -``` -mcp__code-index__set_project_path(path="{ORIGINAL_PROJECT_PATH}") -``` - -This must happen before Step 2 or any code-index search tools will fail with "Project path not set." - ### Step 2: Deep System Understanding (Subagents) Before implementing ANY code, launch parallel Explore subagents to understand affected systems: @@ -122,18 +115,11 @@ task install-worktree # or equivalent for the project type **All commands in Steps 4–6 must run from `${WORKTREE_PATH}`.** Use absolute paths to avoid CWD drift across Bash tool calls. -### Step 3.5: Re-point Code Index to Worktree (REQUIRED) - -**CRITICAL:** After setting up the worktree environment, you **MUST** update the MCP code-index project path to the worktree. This ensures all subsequent code searches operate on the worktree's files (which will diverge from the original as implementation proceeds): - -``` -mcp__code-index__set_project_path(path="${WORKTREE_PATH}") -``` - -**Failure to do this means code-index searches will return results from the original project, not your worktree — leading to confusion and incorrect file reads.** - ### Step 4: Implement Phase by Phase +NEVER pause for confirmation between phases. Once the plan is loaded, execute all +phases sequentially without asking the user whether to proceed to the next phase. + For each phase: 1. Announce phase objective and files to modify 2. Implement changes guided by understanding from Step 2 @@ -166,13 +152,16 @@ missed registration generates 5–30 cascading test failures that require a seco ### Step 5: Final Verification -Read the configured test command from `.autoskillit/config.yaml` (key: `test_check.command`). Use this command wherever `{test_command}` appears below. If no config exists, use `task test-check` as the default. +Read the configured test command(s) from `.autoskillit/config.yaml`: check `test_check.commands` (ordered list of commands, if set) or `test_check.command` (single command, default: `task test-check`). The `test_check` MCP tool runs all configured commands automatically. Run the project's code quality checks and test suite from the worktree. ```bash cd "${WORKTREE_PATH}" && pre-commit run --all-files -cd "${WORKTREE_PATH}" && {test_command} +cd "${WORKTREE_PATH}" && \ + AUTOSKILLIT_TEST_FILTER="${AUTOSKILLIT_TEST_FILTER:-conservative}" \ + AUTOSKILLIT_TEST_BASE_REF=$(cat "{{AUTOSKILLIT_TEMP}}/worktrees/${WORKTREE_NAME}/base-branch") \ + {test_command} ``` If tests fail, fix the issue and re-run. @@ -212,16 +201,6 @@ branch_name = ${WORKTREE_NAME} base_branch = ${CURRENT_BRANCH} ``` -### Step 7.5: Reset Code Index to Original Project (REQUIRED) - -**CRITICAL:** After worktree cleanup, you **MUST** reset the MCP code-index project path back to the original project directory: - -``` -mcp__code-index__set_project_path(path="{ORIGINAL_PROJECT_PATH}") -``` - -**Failure to do this leaves code-index pointing at a deleted worktree path, breaking all subsequent code searches in any session.** - ## Error Handling - **Worktree creation fails** — check `git worktree list`, suggest `git worktree prune` diff --git a/src/autoskillit/skills_extended/investigate/SKILL.md b/src/autoskillit/skills_extended/investigate/SKILL.md index a622b700bd..c021ffcbbc 100644 --- a/src/autoskillit/skills_extended/investigate/SKILL.md +++ b/src/autoskillit/skills_extended/investigate/SKILL.md @@ -22,6 +22,26 @@ Perform deep codebase investigation without making any changes. This skill uses - User says "investigate", "explore", "understand", or "analyze" - User explicitly says "do not change any code" +## Deep Analysis Mode + +### Activation + +Deep analysis mode is activated when ANY of the following conditions are met: + +1. The `--depth deep` flag is present in ARGUMENTS +2. The user explicitly says "investigate deeply" or requests "deep analysis" +3. An explicit batch count is requested (e.g., "run 3 batches") +4. The user requests "maximum thoroughness" +5. The recipe passes `--depth deep` via the `skill_command` + +### Model + +When deep analysis mode is active, all subagents spawned via the Task tool always use `model: "sonnet"`. The main skill session model is controlled by the recipe or user — typically `opus[1m]` for the main session in deep mode. + +### When NOT Activated + +Deep analysis mode is never enabled by default. When none of the activation conditions above are met, the skill routes to Standard Mode (Steps 1–4). + ## GitHub Issue Input If the ARGUMENTS contain a GitHub issue reference, call `fetch_github_issue` via the MCP @@ -52,6 +72,8 @@ tool **before** beginning any analysis. Use the returned `content` field as the - Suggest fallbacks that hide errors - Create files outside `{{AUTOSKILLIT_TEMP}}/investigate/` directory - Choose or accept approaches, solutions, and/or fixes that are chosen simply because they are easier +- File GitHub issues automatically (issue filing is always a separate, user-directed action) +- Run subagents in the background (`run_in_background: true` is prohibited) **ALWAYS:** - Use subagents for parallel exploration @@ -68,37 +90,13 @@ tool **before** beginning any analysis. Use the returned `content` field as the - Check for similar existing patterns in codebase - Ensure approaches, solutions, and fixes are the appropriate long-term solutions with proper architecture -## Investigation Workflow +## Standard Mode Workflow (Steps 1–4) **Path-existence guard:** Before issuing a `Read` call on a path that is not guaranteed to exist (e.g., plan file arguments, `{{AUTOSKILLIT_TEMP}}/investigate/` reports, external file references), use `Glob` or `ls` to confirm the path exists first. This prevents ENOENT errors that cascade into sibling parallel-call cancellations. -### Step 0.5 — Code-Index Initialization (required before any code-index tool call) - -Call `set_project_path` with the repo root where this skill was invoked (not a worktree path): - -``` -mcp__code-index__set_project_path(path="{PROJECT_ROOT}") -``` - -Code-index tools require **project-relative paths**. Always use paths like: - - src//some_module.py - -NOT absolute paths like: - - /absolute/path/to/src//some_module.py - -> **Note:** Code-index tools (`find_files`, `search_code_advanced`, `get_file_summary`, -> `get_symbol_body`) are only available when the `code-index` MCP server is configured. -> If `set_project_path` returns an error, fall back to native `Glob` and `Grep` tools -> for the same searches — they provide equivalent results without the code-index server. - -Agents launched via `run_skill` inherit no code-index state from the parent session — this -call is mandatory at the start of every headless session that uses code-index tools. - ### Step 1: Parse the Investigation Target Identify what needs investigation: @@ -147,6 +145,13 @@ Spawn explore subagents to investigate different aspects simultaneously (some as - Check GitHub issues for similar problems - Search for Stack Overflow discussions +**Design Intent** +- Run `git log --follow` on the mechanism's primary file(s) to find the introducing commit +- Read the introducing commit message and diff to extract the stated purpose +- Trace callers and dependents to map what relies on the mechanism +- Check architecture docs (architecture.md, CLAUDE.md, ADRs) for documented design constraints +- Produce a "design intent finding" per mechanism: what it is for, what introduced it, and what depends on it + ### Step 3: Synthesize Findings After subagents complete, consolidate into structured findings: @@ -157,9 +162,12 @@ After subagents complete, consolidate into structured findings: 4. **Data Flow**: How data moves through the system 5. **Test Gap Analysis**: Why tests didn't catch this 6. **Similar Patterns**: How similar issues are handled elsewhere -7. **Historical Context**: Whether this root cause has been investigated or fixed before (populated by Step 3.5) -8. **External Research**: Relevant findings from web search (if applicable) -9. **Recommendations**: Suggested approaches (NOT implementations) +7. **Design Intent Findings**: Per-mechanism analysis — what each mechanism is for, what introduced it, and what depends on it (populated by the Design Intent subagent) +8. **Historical Context**: Whether this root cause has been investigated or fixed before (populated by the historical recurrence check below) +9. **External Research**: Relevant findings from web search (if applicable) +10. **Scope Boundary**: What was investigated vs. what remains unexplored +11. **Confidence Levels**: Per-finding confidence — SUPPORTED (direct code evidence or experimental confirmation), UNSUPPORTED (contradicted by evidence), NEEDS-EVIDENCE (theoretical reasoning, not yet confirmed) +12. **Recommendations**: Suggested approaches (NOT implementations) ### Step 3.5 — Historical Recurrence Check @@ -229,6 +237,7 @@ Report structure: **Date:** {YYYY-MM-DD} **Scope:** {What was investigated} +**Mode:** {Standard | Deep Analysis} ## Summary {One paragraph overview} @@ -237,8 +246,8 @@ Report structure: {If error investigation - the actual source} ## Affected Components -- {file1}: {role} -- {file2}: {role} +- {file1}: {role} [SUPPORTED / NEEDS-EVIDENCE] +- {file2}: {role} [SUPPORTED / NEEDS-EVIDENCE] ## Data Flow {How data moves through the system} @@ -249,6 +258,13 @@ Report structure: ## Similar Patterns {How similar scenarios are handled elsewhere} +## Design Intent Findings +{Per-mechanism analysis from the Design Intent subagent:} +- {mechanism_1}: Introduced in {commit_hash} ({date}) — {stated purpose}. Callers: {caller list}. Documented constraints: {any ADR/architecture.md references}. +- {mechanism_2}: ... +{If no mechanisms were specifically targeted:} +No specific mechanisms flagged for design intent analysis. + ## Historical Context {If prior fixes found:} - Prior investigation dates and report paths @@ -262,8 +278,25 @@ No prior investigations or fixes found for this root cause. {Relevant findings from web search - library bugs, known issues, documentation insights} {Include source URLs for reference} +## Scope Boundary + +**Investigated:** {What was actually explored in this investigation} +**Not yet explored:** {Areas identified but not yet investigated — may warrant follow-up} + ## Recommendations {Suggested approaches - NOT code changes} +{In deep analysis mode: single recommendation, not a menu of options} +{Include killed alternatives with reasons if deep mode} + +## Breakage Analysis {Deep mode only — omit in standard mode} +{Deep mode only — adversarial breakage analysis for removal/change recommendations:} +- Recommendation: {recommendation text} + - Breakage surface: {components that would break, with file paths} + - Prior reverts: {any revert history on this mechanism} + - Downstream contract violations: {contracts that depend on this mechanism} + - Risk level: {LOW / MEDIUM / HIGH} +{If standard mode or no removal/change recommendations:} +{Omit this section entirely.} ``` After writing the report file, emit the structured output token as the very last line @@ -278,9 +311,97 @@ of your text output: investigation_path = {absolute_path_to_investigation_report_file} ``` +## Deep Analysis Mode Workflow (Steps D1–D6) + +When deep analysis mode is activated, Steps D1–D6 replace standard Steps 1–3.5. Step 4 (Write Report) is still executed at the end of D5, and Step D6 adds post-report validation before emitting `investigation_path`. + +### Step D1: Parse Target + Propose Batch Plan + +Parse the investigation target (same as Step 1). Then propose an adaptive batch plan: + +- Minimum 2 batches; typically 3–4 for complex investigations +- State the planned batch count and scope for each batch +- Present as "Proposed investigation plan: Batch 1 — {scope}, Batch 2 — {scope}, ..." + +**Headless auto-approve:** Check the `AUTOSKILLIT_HEADLESS` environment variable. If set to a truthy value (`1`, `true`, `yes`), automatically approve the proposed batch plan without prompting the user. In interactive mode (AUTOSKILLIT_HEADLESS unset or falsy), present the plan and await user confirmation before proceeding to D2. + +### Step D2: Batch 1 — Broad Parallel Exploration + +Launch a minimum of 5 parallel subagents (model: "sonnet") covering: + +- **Code path tracing**: Trace execution paths through the primary affected components +- **Log and history analysis**: Scan session logs and git history for prior occurrences +- **Related component mapping**: Map all components that interact with the target +- **External research**: Web search for known issues, library bugs, documentation +- **Design Intent**: Run `git log --follow`, caller tracing, and architecture doc checks on the mechanisms under investigation — identical to the Standard Mode Design Intent vector but with deep mode evidence standards + +Simultaneously with Batch 1 subagents, run historical recurrence check (Step 3.5 Parts A+B) in parallel. After Batch 1 completes, produce inter-batch synthesis: summarize confirmed findings, open questions, and new investigative leads. + +After inter-batch synthesis, run Part C of Step 3.5 conditionally: if Parts A+B found prior history, spawn the conditional analysis subagent. Otherwise skip and proceed to D3. + +### Step D3: Batch 2+ — Informed Deepening + +For each subsequent batch (Batch 2, Batch 3, ...): + +1. Open with an explicit synthesis from prior batches — what was confirmed, what remains uncertain +2. Each batch must include mandatory code exploration (local code search, file reads, symbol tracing) and mandatory web research (search for external documentation, known issues, library behavior) +3. After each batch completes, produce inter-batch synthesis (confirmed findings, open questions, new leads) +4. If inter-batch synthesis surfaces new mechanisms as change candidates that were not covered in prior batches, re-dispatch the Design Intent subagent targeting those specific mechanisms. Fan-in: append each re-dispatched result as additional bullet points under `## Design Intent Findings` in the final report (do not replace prior findings). + +**Early termination:** When all findings across all open questions are SUPPORTED (backed by direct code evidence) and no new investigative leads have emerged in the last batch, stop iterating and proceed to D4. + +**Empty batch handling:** If a batch produces no new findings (all subagents report the same conclusions as prior batches), treat it as an early termination signal and proceed to D4. + +### Step D4: Challenge Round + +Fires when ANY finding across any batch is marked NEEDS-EVIDENCE. + +Spawn one adversarial subagent (model: "sonnet") whose role is to disconfirm the primary hypothesis: + +- Provide the primary hypothesis and all supporting evidence collected so far +- Task: find counterevidence — code paths, behaviors, or data that contradict the hypothesis +- Task: assess prior-fix falsifiability — if Step 3.5 found prior fix history, determine whether the prior fix actually addressed this root cause or only a surface symptom + +**If counterevidence is found:** Return to D3 for one additional deepening batch focused on reconciling the contradiction. + +**If no counterevidence is found:** The hypothesis stands. Proceed to D5. + +### Step D5: Solution Convergence + +Spawn solution-space subagents to enumerate candidate fixes. For each candidate, spawn one blast radius subagent (model: "sonnet") to assess: + +- Which components would be affected by this fix +- What tests would need to be added or modified +- What risk surface is introduced + +After blast radius analysis, converge to a single recommendation — the highest-confidence, lowest-blast-radius candidate with direct code evidence. Kill alternative options and document why each was rejected. + +**Adversarial Breakage Analysis:** After converging to a single recommendation, assess whether it proposes removal, replacement, or any action whose execution would eliminate, reduce, or supersede the function of an existing mechanism. If so, spawn one adversarial breakage subagent (model: "sonnet") per such recommendation: + +1. Receive the recommendation and the mechanism it targets, along with the Design Intent findings for that mechanism +2. Trace the mechanism's full dependency chain through code: callers, importers, flag consumers +3. Check git history for prior revert patterns on the same mechanism: `git log --oneline --grep="revert" -- {mechanism_files}` +4. Report a breakage surface analysis: what components would break, prior revert history, downstream contract violations, and overall breakage risk (LOW / MEDIUM / HIGH) + +The adversarial breakage subagent is distinct from the D4 challenge round. D4 asks "is the root cause correct?" (epistemological — fires on NEEDS-EVIDENCE findings). D5 breakage asks "what breaks if we follow this recommendation?" (consequentialist — fires on removal/change recommendations). Both are required in deep mode. + +If the recommendation does not propose removal or change of an existing mechanism (e.g., it proposes adding new code only), skip adversarial breakage analysis for that recommendation. + +### Step D6: Post-Report Validation + +After writing the report (Step 4), spawn 2–3 independent validator subagents (model: "sonnet") with distinct roles: + +- **Validator 1 — Factual accuracy**: Cross-check every claim in the report against actual code/evidence. Flag any factual inaccuracy. +- **Validator 2 — Recommendation soundness**: Assess whether the single recommendation is implementable, safe, and correctly scoped. +- **Validator 3 — Gap analysis** (optional, spawn if investigation was complex): Identify what the report does not cover that could be relevant. + +If any validator identifies errors or gaps, apply in-place corrections to the report before emitting `investigation_path`. The structured output token is emitted **after** all validation and correction is complete. + ## Subagent Prompt Template -Use this template for each Explore subagent: +### Standard Mode Template + +Use this template for each Explore subagent in standard mode: ``` Investigate {specific aspect} of {target}. @@ -293,3 +414,138 @@ Focus on: This is a research task - DO NOT modify any code. Report your findings in a structured format. ``` + +### Design Intent Subagent Template + +Use this template for the Design Intent subagent in both standard and deep modes: + +``` +Investigate the design intent and dependency chain of {mechanism} in {target}. + +Investigation tasks: +1. Run `git log --follow` on {file_path(s)} to find the introducing commit +2. Read the introducing commit message and diff to extract the stated purpose +3. Trace all callers and dependents — map every component that relies on this mechanism +4. Check architecture docs (architecture.md, CLAUDE.md, ADRs) for documented design constraints +5. Produce a design intent finding: what this mechanism is for, what introduced it, and what depends on it + +Evidence standards: +- Cite the introducing commit hash and date +- List specific caller file paths and line numbers +- Reference any architecture doc sections that mention this mechanism +- Mark finding as SUPPORTED (direct evidence) or NEEDS-EVIDENCE (inferred) + +This is a research task - DO NOT modify any code. +Report your findings in a structured format with explicit evidence citations. +``` + +### Deep Analysis Mode Template + +Use this template for each subagent in deep mode batches: + +``` +Investigate {specific aspect} of {target}. + +Context from prior batches: +{Summary of confirmed findings and open questions from previous batch inter-batch synthesis} + +Focus on: +1. {Specific question 1} +2. {Specific question 2} +3. {Specific question 3} + +Evidence standards: +- Cite specific file paths and line numbers for all code claims +- Include log timestamps for any log-based findings +- Mark each finding as SUPPORTED (direct evidence), UNSUPPORTED (contradicted), or NEEDS-EVIDENCE (not yet confirmed) + +This is a research task - DO NOT modify any code. +Report your findings in a structured format with explicit evidence citations. +``` + +### Adversarial Subagent Template (Challenge Round) + +Use this template for the D4 challenge round subagent: + +``` +PRIMARY HYPOTHESIS: {primary hypothesis from D3 synthesis} + +SUPPORTING EVIDENCE: +{All evidence collected across all batches that supports the hypothesis} + +YOUR ROLE: You are an adversarial reviewer. Your task is NOT to confirm the hypothesis. +Your task is to find counterevidence — code paths, behaviors, data, or reasoning that +CONTRADICTS or WEAKENS the primary hypothesis. + +Specifically: +1. Search for code paths that would NOT exhibit the described behavior +2. Look for prior fix commits that may have already addressed this root cause +3. Assess prior-fix falsifiability: if prior fixes exist, determine whether they actually + addressed this root cause or only a surface symptom +4. Identify any assumptions in the hypothesis that are not backed by direct code evidence + +If you find counterevidence, report it explicitly with citations. +If you cannot find counterevidence after thorough search, report "No counterevidence found." + +This is a research task - DO NOT modify any code. +``` + +### Validation Subagent Template + +Use this template for D6 validator subagents: + +``` +REPORT PATH: {absolute path to investigation report} + +YOUR ROLE: {Factual Accuracy Validator | Recommendation Soundness Validator | Gap Analysis Validator} + +Factual Accuracy Validator tasks: +- Read the full report +- Cross-check every factual claim against the actual codebase (use file reads, symbol lookups) +- Flag any claim that is incorrect, imprecise, or not backed by direct code evidence +- Propose corrections for any inaccuracies found + +Recommendation Soundness Validator tasks: +- Read the Recommendations section of the report +- Assess whether the recommendation is implementable without introducing new bugs +- Assess whether the scope is correct (not too narrow, not too broad) +- Flag any risk surface the recommendation introduces + +Gap Analysis Validator tasks: +- Read the Scope Boundary section of the report +- Identify any areas listed as "not yet explored" that are actually relevant to the root cause +- Identify any areas NOT listed that should have been explored +- Report whether the investigation coverage is sufficient for the stated findings + +Report your findings in a structured format. If corrections are needed, state them explicitly. +This is a research task - DO NOT modify any code. +``` + +### Adversarial Breakage Subagent Template (Solution Breakage) + +Use this template for the D5 adversarial breakage subagent (deep mode only): + +``` +RECOMMENDATION: {recommendation text} +TARGET MECHANISM: {mechanism being removed/changed/superseded} +DESIGN INTENT: {summary from Design Intent subagent findings for this mechanism} + +YOUR ROLE: You are a breakage analyst. Your task is NOT to evaluate the recommendation's +merit. Your task is to determine what would break if this recommendation is followed. + +Investigation tasks: +1. Trace the mechanism's full dependency chain: callers, importers, flag consumers, + test fixtures that depend on its behavior +2. Check git history for prior revert patterns on this mechanism: + `git log --oneline --grep="revert" -- {mechanism_files}` +3. Identify downstream components whose contracts or invariants rely on this mechanism +4. Assess whether the recommendation accounts for all dependents found in steps 1-3 + +Report a breakage surface analysis: +- Components that would break (with file paths and line numbers) +- Prior reverts of similar changes (with commit hashes) +- Downstream contract violations +- Overall breakage risk: LOW (no dependents) / MEDIUM (dependents exist, manageable) / HIGH (critical path dependents) + +This is a research task - DO NOT modify any code. +``` diff --git a/src/autoskillit/skills_extended/issue-splitter/SKILL.md b/src/autoskillit/skills_extended/issue-splitter/SKILL.md index b40299a40c..6c422f2a28 100644 --- a/src/autoskillit/skills_extended/issue-splitter/SKILL.md +++ b/src/autoskillit/skills_extended/issue-splitter/SKILL.md @@ -143,15 +143,18 @@ For each concern (up to `max_sub_issues`): gh label create "split-from:#N" --force \ --description "Sub-issue created from parent #N" \ --color "e4e669" [--repo {repo}] +sleep 1 # Rate-limit discipline: 1s between mutating calls # Ensure the recipe route label exists (idempotent) gh label create "recipe:implementation" --force \ --description "Route through implementation recipe" \ --color "0E8A16" [--repo {repo}] +sleep 1 # Rate-limit discipline: 1s between mutating calls # or: gh label create "recipe:remediation" --force \ --description "Route through remediation recipe" \ --color "D93F0B" [--repo {repo}] +sleep 1 # Rate-limit discipline: 1s between mutating calls # Create the sub-issue gh issue create \ @@ -160,6 +163,7 @@ gh issue create \ --label "split-from:#N" \ --label "recipe:{route}" \ [--repo {repo}] +sleep 1 # Rate-limit discipline: 1s between mutating calls ``` Capture the new issue URL and number from stdout. @@ -175,16 +179,18 @@ gh label create "split" --force \ # Label the parent as split gh issue edit {N} --add-label "split" [--repo {repo}] -# Add tracking comment with cross-references -gh issue comment {N} \ - --body "## Split into sub-issues - -This issue covers multiple concerns and has been decomposed into focused sub-issues: - -{bullet list of sub-issue links with route labels} - -This issue remains open as a tracking issue." \ - [--repo {repo}] +# Append ## Decomposed section to parent body +SPLIT_BODY_FILE="{{AUTOSKILLIT_TEMP}}/issue-splitter/decomposed_{N}_$(date +%s).md" +mkdir -p "$(dirname "$SPLIT_BODY_FILE")" +gh issue view {N} --json body --jq '.body' [--repo {repo}] > "$SPLIT_BODY_FILE" +printf '\n\n---\n\n## Decomposed\n\nThis issue covers multiple concerns and has been decomposed into focused sub-issues:\n\n' \ + >> "$SPLIT_BODY_FILE" +for sub in $sub_issue_links; do + printf '- %s\n' "$sub" >> "$SPLIT_BODY_FILE" +done +printf '\nThis issue remains open as a tracking issue.\n' >> "$SPLIT_BODY_FILE" +gh issue edit {N} --body-file "$SPLIT_BODY_FILE" [--repo {repo}] +sleep 1 ``` Do **not** close the parent — it serves as a tracking issue for all sub-issues. diff --git a/src/autoskillit/skills_extended/make-campaign/SKILL.md b/src/autoskillit/skills_extended/make-campaign/SKILL.md new file mode 100644 index 0000000000..31b5d3319c --- /dev/null +++ b/src/autoskillit/skills_extended/make-campaign/SKILL.md @@ -0,0 +1,137 @@ +--- +name: make-campaign +description: Interactively author a campaign recipe YAML through a 6-phase guided workflow. Use when user says "make campaign", "create campaign", "author campaign", "new campaign recipe", or wants to decompose a campaign goal into dispatches. +categories: + - orchestration-family + - fleet +hooks: + PreToolUse: + - matcher: "*" + hooks: + - type: command + command: "echo '[SKILL: make-campaign] Authoring campaign recipe...'" + once: true +--- + +# Campaign Recipe Authoring Skill + +Guides users through a 6-phase interactive workflow to decompose a campaign goal into a validated campaign recipe YAML. + +## When to Use + +- User says "make campaign", "create campaign", "author campaign", "new campaign recipe" +- User wants to decompose a campaign goal into dispatches +- User wants to orchestrate multiple recipe runs as a campaign +- User says "decompose into dispatches" + +## Critical Constraints + +**NEVER:** +- Modify or write to any source code files +- Create files outside `.autoskillit/recipes/campaigns/` (final output) or `{{AUTOSKILLIT_TEMP}}/make-campaign/` (temp/validation drafts) +- Write the final campaign YAML without first passing `validate_recipe` in Phase 5 +- Use recipe names not confirmed by `find_recipe_by_name` or `list_recipes` +- Accept ingredient keys that do not exist in the target recipe's declared ingredients + +**ALWAYS:** +- Use `find_recipe_by_name` to confirm dispatch targets exist before accepting a recipe name +- Use `load_recipe` to inspect each target recipe's `ingredients` schema before populating dispatch ingredients +- Validate the campaign YAML with `validate_recipe` before writing the final manifest +- Emit the structured output token `campaign_path` as the **absolute path** to the written campaign YAML: + ``` + campaign_path = /absolute/path/to/.autoskillit/recipes/campaigns/.yaml + ``` + Use `$(pwd)` to resolve the absolute path when writing the output file. +- Run cycle detection on `depends_on` before proceeding to Phase 5 + +## Phase 1 — Goal Clarification + +Prompt the user for the following information interactively: + +1. **Campaign name** (kebab-case, e.g. `my-feature-campaign`) +2. **Description** (1–2 sentences describing the campaign's purpose) +3. **Target outcome** (what the campaign achieves when all dispatches succeed) +4. **Rough dispatch count** (how many dispatch phases the user anticipates) +5. **`continue_on_failure`** — should the campaign proceed if a dispatch fails? (default: `false`) +6. **`categories`** — which recipe family this campaign targets (default: `orchestration-family`) +7. **`requires_recipe_packs`** — which recipe pack(s) the dispatches will draw from (e.g. `implementation-family`) + +Produce a Goal Artifact with: `name`, `description`, `categories`, `requires_recipe_packs`, `continue_on_failure`. + +## Phase 2 — Dispatch Decomposition + +For each dispatch (repeat until the user signals done): + +1. Ask the user to describe the dispatch's task in one sentence. +2. Call `list_recipes` to show available recipes matching the declared packs. +3. Ask the user which recipe this dispatch targets. +4. Verify the recipe exists by calling `find_recipe_by_name` (or confirming from `list_recipes` results). + - If not found: show available options and re-prompt. + - If found: accept the recipe name. +5. Capture: + - `name` — kebab-case dispatch name (e.g. `phase-1-implement`) + - `recipe` — confirmed recipe name + - `task` — non-empty task description +6. Optionally allow the user to add entries to `allowed_recipes` for dispatches that use recipes outside the declared packs. + +Do not proceed to Phase 3 until all dispatch names, recipes, and tasks are confirmed. + +## Phase 3 — Ingredient Population + +For each dispatch captured in Phase 2: + +1. Call `load_recipe` on the target recipe to inspect its `ingredients` schema. +2. Present the ingredient table: name, description, required, default. +3. For each ingredient the user wants to set: + - Ask for the value (string) or a campaign-level pass-through using `${{ inputs. }}` syntax. + - Validate that the key exists in the target recipe's schema — reject unknown keys immediately. +4. Collect all key-value pairs as `ingredients: {key: "value"}` under the dispatch. + +Proceed to Phase 4 only after all dispatches have their ingredients populated (even if empty). + +## Phase 4 — Dependency Ordering + +1. Present all dispatches in capture order with their names. +2. For each dispatch, ask: "Which earlier dispatches does this one depend on?" (comma-separated names, or none). +3. Build the `depends_on` adjacency map. +4. Run cycle detection (DFS): + - If a cycle is found: report the full cycle path (e.g. `a → b → c → a`) and re-prompt affected dispatches. + - Loop until the dependency graph is acyclic. +5. Suggest the topological sort order; ask the user to confirm or adjust dispatch ordering. + +## Phase 5 — Schema Validation + +1. Assemble the complete campaign YAML from all captured data (Phases 1–4). +2. Write a draft file (relative to the current working directory) to `{{AUTOSKILLIT_TEMP}}/make-campaign/_draft.yaml`. +3. Call `validate_recipe` with the draft path. +4. If validation returns errors: + - Display each error with its rule name and message. + - Guide the user through the correction. + - Update the draft and re-call `validate_recipe`. + - Repeat until validation passes (structural + all 14 campaign semantic rules clean). +5. Proceed to Phase 6 only when `validate_recipe` reports no errors. + +## Phase 6 — Manifest Write + +1. Ensure the `.autoskillit/recipes/campaigns/` directory exists (create with `mkdir -p` if absent). +2. Write the validated YAML to `.autoskillit/recipes/campaigns/.yaml`. +3. Print the absolute path of the written file. +4. Print the next-step hint: + + ``` + Campaign recipe written. Use the dispatch_food_truck MCP tool (with kitchen open) + to execute this campaign. + ``` + +## Output + +Emit the structured output token on its own line: + +``` +campaign_path = {absolute_path_to_campaign_yaml} +``` + +Example: +``` +campaign_path = /home/user/project/.autoskillit/recipes/campaigns/my-campaign.yaml +``` diff --git a/src/autoskillit/skills_extended/make-groups/SKILL.md b/src/autoskillit/skills_extended/make-groups/SKILL.md index 3a07de0003..5179797ab9 100644 --- a/src/autoskillit/skills_extended/make-groups/SKILL.md +++ b/src/autoskillit/skills_extended/make-groups/SKILL.md @@ -57,6 +57,7 @@ tool **before** beginning any analysis. Use the returned `content` field as the - Drop, split, or rewrite requirements — reference them by original ID - Create groups that cannot be independently planned - Include implementation steps or technical approach in the group descriptions +- Run subagents in the background (`run_in_background: true` is prohibited) **ALWAYS:** - Use subagents to verify codebase structure before finalizing groups @@ -81,30 +82,6 @@ tool **before** beginning any analysis. Use the returned `content` field as the Read the full document. Inventory every requirement (REQ-*), feature, and deliverable. Build a raw list with original IDs preserved. -### Step 1.5 — Code-Index Initialization (required before any code-index tool call) - -Call `set_project_path` with the repo root where this skill was invoked (not a worktree path): - -``` -mcp__code-index__set_project_path(path="{PROJECT_ROOT}") -``` - -Code-index tools require **project-relative paths**. Always use paths like: - - src//some_module.py - -NOT absolute paths like: - - /absolute/path/to/src//some_module.py - -> **Note:** Code-index tools (`find_files`, `search_code_advanced`, `get_file_summary`, -> `get_symbol_body`) are only available when the `code-index` MCP server is configured. -> If `set_project_path` returns an error, fall back to native `Glob` and `Grep` tools -> for the same searches — they provide equivalent results without the code-index server. - -Agents launched via `run_skill` inherit no code-index state from the parent session — this -call is mandatory at the start of every headless session that uses code-index tools. - ### Step 2: Verify Against Codebase Launch **parallel Explore subagents** to understand: @@ -295,7 +272,7 @@ git checkout -b feature/{topic} Then run each group through the pipeline using the feature branch as `base_branch` for all `merge_worktree` calls. The `/autoskillit:audit-impl` skill accepts the manifest as input -and audits all groups at once as the final gate before merging the feature branch to `integration`. +and audits all groups at once as the final gate before merging the feature branch to `develop`. Use the `group-implementation` bundled workflow to automate this — it creates the feature branch, runs the group loop, and gates on audit before signalling merge-ready. diff --git a/src/autoskillit/skills_extended/make-plan/SKILL.md b/src/autoskillit/skills_extended/make-plan/SKILL.md index 824a6b2925..0c7437b81e 100644 --- a/src/autoskillit/skills_extended/make-plan/SKILL.md +++ b/src/autoskillit/skills_extended/make-plan/SKILL.md @@ -1,7 +1,7 @@ --- name: make-plan -activate_deps: [arch-lens] -description: Create implementation plans through deep codebase understanding. Use when user asks to create, devise, or write a plan. Leverages subagents to explore approaches, understand systems, and design aligned solutions. +activate_deps: [arch-lens, write-recipe] +description: Planning executor. ALWAYS invoke this skill when instructed to create, devise, or write an implementation plan. Do not explore the codebase or draft a plan directly — use this skill first to load the planning workflow. hooks: PreToolUse: - matcher: "*" @@ -66,31 +66,7 @@ tool **before** beginning any analysis. Use the returned `content` field as the ## Planning Steps -**Step 0 — Code-Index Initialization (required before any code-index tool call)** - -Call `set_project_path` with the repo root where this skill was invoked (not a worktree path): - -``` -mcp__code-index__set_project_path(path="{PROJECT_ROOT}") -``` - -Code-index tools require **project-relative paths**. Always use paths like: - - src//some_module.py - -NOT absolute paths like: - - /absolute/path/to/src//some_module.py - -> **Note:** Code-index tools (`find_files`, `search_code_advanced`, `get_file_summary`, -> `get_symbol_body`) are only available when the `code-index` MCP server is configured. -> If `set_project_path` returns an error, fall back to native `Glob` and `Grep` tools -> for the same searches — they provide equivalent results without the code-index server. - -Agents launched via `run_skill` inherit no code-index state from the parent session — this -call is mandatory at the start of every headless session that uses code-index tools. - -1. **Understand related systems and validate details** - Use subagents to study the architecture, how components work together, their purpose, patterns, and standards. Validate any details provided in the task description. +1. **Understand related systems and validate details** - Use subagents to study the architecture, how components work together, their purpose, patterns, and standards. Validate any details provided in the task description. When the plan involves adding tests that call mutating methods on singleton or module-level objects (enable/disable, register/unregister, connect/disconnect), use a subagent to read the target test directory's existing isolation patterns (conftest fixtures, setup_method/teardown_method, autouse fixtures) before proceeding to Step 3. 2. **Explore and design approaches** - Use subagents to investigate different ways to solve the problem. Use subagents with web search to research modern solutions, approaches, designs, and architectures relevant to the problem. For each approach, focus on: - Does it solve the problem correctly? @@ -100,6 +76,8 @@ call is mandatory at the start of every headless session that uses code-index to 3. **Design tests first** - For the chosen approach, define tests that capture the intended behavior. These tests should fail against the current codebase and pass once the implementation is complete. The implementation steps should be ordered to make these tests pass. + **Test isolation contract:** When the plan adds tests that call mutating methods on a singleton or module-level object, the plan must specify the isolation strategy — how state is reset between tests. Ensure new tests either inherit the existing isolation mechanism or explicitly define their own. Plans that prescribe calling mutating methods on shared objects without specifying cleanup are incomplete. + 4. **Evaluate approaches on technical merit only** - Use subagents to assess each approach. Evaluation criteria: - **Correctness**: Does it fully solve the stated problem? - **Design quality**: Is this the right abstraction? Is it clean? @@ -164,7 +142,7 @@ More than one lens diagram is okay if it is complex plan (don't do more than 3, ## Conflict-Resolution Plan Requirements -When the task involves applying changes from a PR branch to an integration branch +When the task involves resolving conflicts to apply changes from one branch onto another (i.e., the input is a conflict report produced by `merge-pr`), the plan **MUST produce a worktree with a linear commit history**. @@ -225,6 +203,7 @@ Before writing the final plan, verify: - Reject an approach because it's harder - Create files outside `{{AUTOSKILLIT_TEMP}}/make-plan/` directory - **Use `git merge` in implementation plans.** When a plan needs to bring in changes from another branch, use `git cherry-pick ` for individual commits or `git checkout -- ` for specific files. `merge_worktree` requires linear commit history — merge commits cannot be rebased and will cause `WORKTREE_INTACT_MERGE_COMMITS_DETECTED` failure. See "Conflict-Resolution Plan Requirements" section for full guidance. +- Run subagents in the background (`run_in_background: true` is prohibited) **ALWAYS:** - Write to `{{AUTOSKILLIT_TEMP}}/make-plan/` directory (relative to the current working directory) diff --git a/src/autoskillit/skills_extended/make-req/SKILL.md b/src/autoskillit/skills_extended/make-req/SKILL.md index 0f9c3c8927..ffcc50ea50 100644 --- a/src/autoskillit/skills_extended/make-req/SKILL.md +++ b/src/autoskillit/skills_extended/make-req/SKILL.md @@ -35,6 +35,7 @@ Decompose a task, plan, roadmap, or feature description into a structured set of - Prescribe implementation approaches or libraries in requirements - Include implementation steps disguised as requirements ("Refactor X to use Y" is an instruction, not a requirement) - Write requirements that can only be verified by reading source code +- Run subagents in the background (`run_in_background: true` is prohibited) **ALWAYS:** - Use subagents to understand the source material and relevant codebase context @@ -143,4 +144,4 @@ Before finalizing, verify: - Every requirement is a testable condition - Groups are cohesive — moving a requirement to a different group would feel wrong - A reader unfamiliar with the codebase could understand what each group demands from the background section alone -- No group depends on understanding another group's requirements to be plannable (cross-references to other groups are allowed for ordering, not comprehension) \ No newline at end of file +- No group depends on understanding another group's requirements to be plannable (cross-references to other groups are allowed for ordering, not comprehension) diff --git a/src/autoskillit/skills_extended/merge-pr/SKILL.md b/src/autoskillit/skills_extended/merge-pr/SKILL.md index 53fb9038e8..ae72b4aa79 100644 --- a/src/autoskillit/skills_extended/merge-pr/SKILL.md +++ b/src/autoskillit/skills_extended/merge-pr/SKILL.md @@ -39,6 +39,7 @@ conflicts from earlier merges in the queue. - Close or comment on the PR - Leave the git working tree in a dirty state - Create files outside `{{AUTOSKILLIT_TEMP}}/merge-prs/` directory +- Run subagents in the background (`run_in_background: true` is prohibited) **ALWAYS:** - Run `git status` before any operation to verify clean state @@ -73,7 +74,8 @@ Extract: Fetch the PR branch locally: ```bash -git fetch origin {pr_branch}:{pr_branch} 2>/dev/null || git fetch origin pull/{pr_number}/head:{pr_branch} +REMOTE=$(git remote get-url upstream >/dev/null 2>&1 && echo upstream || echo origin) +git fetch "$REMOTE" {pr_branch}:{pr_branch} 2>/dev/null || git fetch "$REMOTE" pull/{pr_number}/head:{pr_branch} ``` ### Step 1.5: Deletion Regression Scan @@ -86,18 +88,18 @@ deliberately deleted from the base branch after the PR's branch point. ```bash BASE_BRANCH=$(gh pr view {pr_number} --json baseRefName --jq '.baseRefName') # 1. Find the divergence point between this PR and the base branch -MERGE_BASE=$(git merge-base origin/${BASE_BRANCH} origin/{pr_branch}) +MERGE_BASE=$(git merge-base ${REMOTE}/${BASE_BRANCH} ${REMOTE}/{pr_branch}) # 2. Files deleted from base since the branch point -DELETED_FILES=$(git diff --name-only --diff-filter=D ${MERGE_BASE} origin/${BASE_BRANCH}) +DELETED_FILES=$(git diff --name-only --diff-filter=D ${MERGE_BASE} ${REMOTE}/${BASE_BRANCH}) # 3. Symbols (functions/classes) removed from files this PR modifies, # relative to base (catches deletions in files that still exist) -PR_FILES=$(git diff --name-only ${MERGE_BASE}...origin/{pr_branch}) +PR_FILES=$(git diff --name-only ${MERGE_BASE}...${REMOTE}/{pr_branch}) if [ -n "$PR_FILES" ]; then DELETED_SYMBOLS=$( echo "$PR_FILES" | \ - git diff --diff-filter=M ${MERGE_BASE} origin/${BASE_BRANCH} --pathspecs-from-file=- \ + git diff --diff-filter=M ${MERGE_BASE} ${REMOTE}/${BASE_BRANCH} --pathspecs-from-file=- \ | grep '^-' \ | grep -E '^-(def |class |async def )' \ | sed 's/^-//' \ @@ -108,7 +110,7 @@ else fi # 4. What the PR adds (relative to the merge base) -PR_ADDITIONS=$(git diff ${MERGE_BASE}...origin/{pr_branch} | grep '^+' | grep -v '^+++') +PR_ADDITIONS=$(git diff ${MERGE_BASE}...${REMOTE}/{pr_branch} | grep '^+' | grep -v '^+++') ``` **Detect regressions:** @@ -135,12 +137,12 @@ items must NOT be reintroduced in the implementation. ```bash # For each regressed file: find the commit that deleted it on base -git log --diff-filter=D --oneline --follow -- {file_path} origin/${BASE_BRANCH} | head -1 +git log --diff-filter=D --oneline --follow -- {file_path} ${REMOTE}/${BASE_BRANCH} | head -1 # For each regressed symbol: find the commit that removed it -git log --diff-filter=M --oneline -p -- {file_path} origin/${BASE_BRANCH} \ - | grep -B20 "^-def {symbol_name}\|^-class {symbol_name}" \ - | grep "^[0-9a-f]\{7,\}" \ +git log --diff-filter=M --oneline -p -- {file_path} ${REMOTE}/${BASE_BRANCH} \ + | rg -B 20 "^-def {symbol_name}|^-class {symbol_name}" \ + | rg "^[0-9a-f]{7,}" \ | head -1 ``` @@ -177,6 +179,14 @@ AUTO_MERGE_ALLOWED=$(gh api graphql -f query="query { Capture `AUTO_MERGE_ALLOWED` ("true" or "false"). On gh failure, default to "false" (the safe path: use plain `--squash`, not `--squash --auto`). +### Step 1.9: Pre-flight Mergeability Check + +Run: `gh pr view {pr_number} --json mergeable --jq '.mergeable'` + +- If `MERGEABLE`: proceed to Step 2 +- If `CONFLICTING`: skip Step 2, set `needs_plan=true`, write a conflict report noting the PR has git-level conflicts, proceed to Step 5 output +- If `UNKNOWN`: wait 10 seconds, retry up to 3 times. If still `UNKNOWN` after 3 retries, treat as `CONFLICTING` (sets `needs_plan=true`, same as git-level conflicts — distinct from the poll-loop timeout path which sets `needs_plan=false, timeout_error=true`) + ### Step 2: Simple Path — gh pr merge **When `AUTO_MERGE_ALLOWED == true`** (repo has branch protection with required checks): @@ -252,7 +262,7 @@ Before writing the conflict report, fetch the complete set of files changed on t ```bash # All files changed on the PR branch relative to the base -git diff ${BASE_BRANCH}...origin/{pr_branch} --name-only +git diff ${BASE_BRANCH}...${REMOTE}/{pr_branch} --name-only ``` Classify each file into one of three categories: @@ -291,7 +301,7 @@ semantic conflicts with previously merged PRs. **PR:** #{pr_number} **Branch:** {pr_branch} -**Integration Branch:** {integration_branch (current HEAD)} +**Integration Branch:** {batch_branch (current HEAD)} ## PR Changes Inventory @@ -439,6 +449,20 @@ with `conflict_report_path` set. The pipeline then routes to make-plan → imple } ``` +**On timeout (poll loop exceeded 600s):** +```json +{ + "merged": false, + "needs_plan": false, + "deletion_regression": false, + "escalation_required": false, + "timeout_error": true, + "pr_number": 47, + "pr_branch": "feature/db-refactor", + "pr_title": "Refactor database layer" +} +``` + Exit 0 in all cases — `needs_plan=true` and `escalation_required=true` are routing signals, not failures. After printing the result block, emit the following structured output tokens as the @@ -514,6 +538,18 @@ pr_branch = {pr_branch_name} pr_title = {pr_title} ``` +**On timeout:** +``` +merged = false +needs_plan = false +deletion_regression = false +escalation_required = false +timeout_error = true +pr_number = {pr_number} +pr_branch = {branch_name} +pr_title = {title} +``` + Emit `conflict_report_path=` only when `needs_plan=true` and a conflict plan file was written. Omit the line entirely on a successful direct merge or when `escalation_required=true`. diff --git a/src/autoskillit/skills_extended/open-integration-pr/SKILL.md b/src/autoskillit/skills_extended/open-integration-pr/SKILL.md index 0c3292ce95..99d9b51dc7 100644 --- a/src/autoskillit/skills_extended/open-integration-pr/SKILL.md +++ b/src/autoskillit/skills_extended/open-integration-pr/SKILL.md @@ -28,10 +28,10 @@ PR, and output `pr_url=`. ## Arguments ``` -/autoskillit:open-integration-pr {integration_branch} {base_branch} {pr_order_file} [audit_verdict] [conflict_report_paths] [domain_partitions_path] +/autoskillit:open-integration-pr {batch_branch} {base_branch} {pr_order_file} [audit_verdict] [conflict_report_paths] [domain_partitions_path] ``` -- `integration_branch` — integration branch name (e.g. `pr-batch/pr-merge-20250228-143052`) +- `batch_branch` — integration branch name (e.g. `pr-batch/pr-merge-20250228-143052`) - `base_branch` — PR target branch (e.g. `main`) - `pr_order_file` — absolute path to JSON produced by `analyze-prs` - `audit_verdict` (optional) — `GO`, `NO GO`, or empty string when audit was skipped @@ -50,6 +50,7 @@ PR, and output `pr_url=`. - Modify any source code - Fail the pipeline if `gh` is unavailable or not authenticated — output `pr_url=` (empty) and exit successfully - Close original PRs before the integration PR is successfully created +- Run subagents in the background (`run_in_background: true` is prohibited) **ALWAYS:** - Check `gh auth status` before attempting any GitHub operations @@ -61,7 +62,7 @@ PR, and output `pr_url=`. ### Step 1: Parse Arguments -Parse four positional args: `integration_branch`, `base_branch`, `pr_order_file`, +Parse four positional args: `batch_branch`, `base_branch`, `pr_order_file`, `audit_verdict` (last one may be absent or empty string). Parse the optional fifth positional argument `conflict_report_paths` (may be absent or empty string). Split on `,` to get a list of paths; filter out any empty strings. Store as `conflict_report_path_list`. @@ -76,22 +77,40 @@ Read the JSON file. Extract: `prs` array (each: `number`, `title`, `branch`, ### Step 3: Fetch Closes/Fixes References from Original PR Bodies -For each PR in `pr_list`: +Fetch all PR bodies in a single GraphQL alias query (1 API call instead of N sequential +REST calls). Chunk into batches of 20 to stay within GraphQL complexity limits. ```bash -gh pr view {number} --json body -q .body 2>/dev/null +# Build GraphQL alias query for all PRs +PR_NUMS=($(echo "$pr_list" | jq -r '.[].number')) +BODIES_JSON="{}" +BATCH_SIZE=20 +for batch_start in $(seq 0 $BATCH_SIZE $((${#PR_NUMS[@]} - 1))); do + BATCH_NUMS=("${PR_NUMS[@]:$batch_start:$BATCH_SIZE}") + QUERY="query { " + for i in $(seq 0 $((${#BATCH_NUMS[@]} - 1))); do + NUM="${BATCH_NUMS[$i]}" + QUERY="${QUERY} pr${i}: repository(owner: \"${OWNER}\", name: \"${REPO}\") { + pullRequest(number: ${NUM}) { number body } + }" + done + QUERY="${QUERY} }" + BATCH=$(gh api graphql -f query="${QUERY}" 2>/dev/null || echo "{}") + BODIES_JSON=$(echo "${BODIES_JSON} ${BATCH}" | jq -s 'add // {}') +done ``` -Extract every line matching `(Closes|Fixes|Resolves)\s+#\d+` (case-insensitive). -Deduplicate across all PRs. Store as `closing_refs` (list of strings like `Closes #42`). +Extract every line matching `(Closes|Fixes|Resolves)\s+#\d+` (case-insensitive) from +each PR body in `BODIES_JSON`. Deduplicate across all PRs. Store as `closing_refs` +(list of strings like `Closes #42`). Skip gracefully if `gh` is unavailable — `closing_refs` remains empty. ### Step 4: Get Changed Files ```bash -git diff --name-only {base_branch}..{integration_branch} -git diff --diff-filter=A --name-only {base_branch}..{integration_branch} -git diff --diff-filter=M --name-only {base_branch}..{integration_branch} +git diff --name-only {base_branch}..{batch_branch} +git diff --diff-filter=A --name-only {base_branch}..{batch_branch} +git diff --diff-filter=M --name-only {base_branch}..{batch_branch} ``` Store as `changed_files`, `new_files`, `modified_files`. @@ -129,7 +148,7 @@ For each domain name `D` in `domain_partitions` where `domain_partitions[D]` is run the following in parallel (issue all Bash calls in a single message): ```bash -git diff {base_branch}..{integration_branch} -- {space-separated list of files in domain D} +git diff {base_branch}..{batch_branch} -- {space-separated list of files in domain D} ``` Store results as `domain_diffs: dict[str, str]` mapping domain name → diff text. @@ -161,7 +180,7 @@ For each domain `D` in `domain_diffs` (domains that actually have diff content), parallel (all Bash calls in a single message): ```bash -git log {base_branch}..{integration_branch} --oneline -- {space-separated files in domain D} +git log {base_branch}..{batch_branch} --oneline -- {space-separated files in domain D} ``` Store as `domain_commits: dict[str, list[str]]` (each entry is a list of `"sha message"` strings). @@ -258,7 +277,7 @@ Write to `{{AUTOSKILLIT_TEMP}}/open-integration-pr/pr_body_{timestamp}.md`. (rel ```markdown ## Integration Summary -Collapsed {N} PRs into `{integration_branch}` targeting `{base_branch}`. +Collapsed {N} PRs into `{batch_branch}` targeting `{base_branch}`. ## Merged PRs @@ -323,7 +342,7 @@ If exit code non-zero: output `pr_url=` and exit successfully. ```bash gh pr create \ --base {base_branch} \ - --head {integration_branch} \ + --head {batch_branch} \ --title "Integration: collapsed PRs #{numbers} into {base_branch}" \ --body-file {{AUTOSKILLIT_TEMP}}/open-integration-pr/pr_body_{timestamp}.md ``` @@ -338,6 +357,7 @@ For each PR in `pr_list`: ```bash gh pr close {number} --comment "Collapsed into integration PR #{new_pr_number} ({new_pr_url})" +sleep 1 # Rate-limit discipline: 1s between mutating calls ``` Continue even if individual close operations fail (log warning, do not exit). @@ -346,4 +366,4 @@ Continue even if individual close operations fail (log warning, do not exit). ``` pr_url = {new_pr_url} -``` \ No newline at end of file +``` diff --git a/src/autoskillit/skills_extended/pipeline-summary/SKILL.md b/src/autoskillit/skills_extended/pipeline-summary/SKILL.md index 9b1598fd64..4d991adfaf 100644 --- a/src/autoskillit/skills_extended/pipeline-summary/SKILL.md +++ b/src/autoskillit/skills_extended/pipeline-summary/SKILL.md @@ -18,13 +18,12 @@ and a PR from the feature branch into the target branch. ## Arguments -`/autoskillit:pipeline-summary {bug_report_path} {feature_branch} {target_branch} {workspace} [{token_summary_path}] [{closing_issue}]` +`/autoskillit:pipeline-summary {bug_report_path} {feature_branch} {target_branch} {workspace} [{closing_issue}]` - **bug_report_path** — Path to the JSON file containing bug metadata - **feature_branch** — Name of the branch containing all accumulated fixes - **target_branch** — Branch to create the PR against (e.g., "main") - **workspace** — Path to the git repository workspace -- **token_summary_path** — (Optional) Path to a JSON file with token/timing data written by the orchestrator. When absent or the file does not exist, the skill operates exactly as today with no token table in the PR body. - **closing_issue** — (Optional) GitHub issue number whose `## Requirements` section should be extracted and embedded in the PR body. When absent or empty, requirements extraction is skipped. ## When to Use @@ -49,7 +48,7 @@ and a PR from the feature branch into the target branch. ## Workflow ### Step 1: Parse Arguments -Parse up to six positional arguments from the prompt. The fifth (`token_summary_path`) and sixth (`closing_issue`) are optional. +Parse up to five positional arguments from the prompt. The fifth (`closing_issue`) is optional. ### Step 2: Read Bug Report Read the JSON file at `{bug_report_path}`. Expected structure: @@ -74,46 +73,6 @@ Write a markdown summary to `{workspace}/run-summary.md`: Output: `summary_path={workspace}/run-summary.md` -### Step 3b: Append Token+Timing Table (if token_summary_path provided) - -If `token_summary_path` was provided as a fifth argument and the file exists, read it as JSON. The JSON has the structure: - -```json -{ - "steps": [ - { - "step_name": "string", - "input_tokens": 0, - "output_tokens": 0, - "cache_creation_input_tokens": 0, - "cache_read_input_tokens": 0, - "invocation_count": 0, - "elapsed_seconds": 0.0 - } - ], - "total": { - "input_tokens": 0, - "output_tokens": 0, - "cache_creation_input_tokens": 0, - "cache_read_input_tokens": 0, - "total_elapsed_seconds": 0.0 - } -} -``` - -Append the following two markdown sections to `run-summary.md`: - -```markdown -## Token Usage - -| Step | Input | Output | Cache Write | Cache Read | Calls | Elapsed (s) | -|------|-------|--------|-------------|------------|-------|-------------| -| {step_name} | {input_tokens} | {output_tokens} | {cache_creation_input_tokens} | {cache_read_input_tokens} | {invocation_count} | {elapsed_seconds:.1f} | -| **Total** | {total.input_tokens} | {total.output_tokens} | {total.cache_creation_input_tokens} | {total.cache_read_input_tokens} | — | {total.total_elapsed_seconds:.1f} | -``` - -If `token_summary_path` is absent or the file does not exist, skip this step — no token table is added. - ### Step 4: Check GitHub Availability Run `gh auth status 2>/dev/null`. If exit code is non-zero or `gh` is not found: - Log "GitHub CLI not available or not authenticated — skipping issue/PR creation" @@ -122,7 +81,8 @@ Run `gh auth status 2>/dev/null`. If exit code is non-zero or `gh` is not found: ### Step 5: Push Feature Branch ```bash cd {workspace} -git push -u origin {feature_branch} +REMOTE=$(git remote get-url upstream >/dev/null 2>&1 && echo upstream || echo origin) +git push -u "$REMOTE" {feature_branch} ``` If push fails (no remote, network issue), log the error and exit successfully. @@ -167,7 +127,6 @@ The PR body (`temp_pr_body`) contains: - `## Requirements` (if `requirements_section` is non-empty from Step 5b) - `Closes #{closing_issue}` (if closing_issue was provided) - Bug table from Step 3 -- Token/timing table from Step 3b (if available) Capture the PR URL from stdout. @@ -177,19 +136,3 @@ Output: `pr_url={url}` - Always: `summary_path={workspace}/run-summary.md` - If GitHub available: `issue_url={url}` and `pr_url={url}` -## Orchestrator Calling Convention - -For recipe authors who want to include token/timing data in the PR body: - -1. Call the `get_token_summary` MCP tool to retrieve current pipeline token data. -2. Write the JSON result to `{{AUTOSKILLIT_TEMP}}/token_summary_{timestamp}.json` using a `run_python` step. (relative to the current working directory) - The `run_python` step executes in the MCP server process and has access to the live - `ToolContext` via the server context; call `ctx.token_log.get_report()` and - `ctx.token_log.compute_total()`, then write `{"steps": ..., "total": ...}` as JSON. - Capture the output path via `print(f"token_summary_path={out}")`. -3. Pass the file path as the fifth positional argument to `run_skill pipeline-summary`. - -> Note: The headless session for `pipeline-summary` runs in a separate process with its own -> (empty) token log, which is why the file-based handoff is required. `run_python` steps share -> the live in-process token log with the MCP server, so they can access accumulated timing data -> directly without a network call. \ No newline at end of file diff --git a/src/autoskillit/skills_extended/plan-experiment/SKILL.md b/src/autoskillit/skills_extended/plan-experiment/SKILL.md index 832e4b335e..562e2124ae 100644 --- a/src/autoskillit/skills_extended/plan-experiment/SKILL.md +++ b/src/autoskillit/skills_extended/plan-experiment/SKILL.md @@ -53,6 +53,7 @@ incorporate the feedback before writing the plan. environment is needed or not, and why - Omit YAML frontmatter unless a V1–V4 ERROR is triggered — every plan must have frontmatter - Write frontmatter after the `# Experiment Plan:` heading — it always goes BEFORE +- Run subagents in the background (`run_in_background: true` is prohibited) **ALWAYS:** - Use `model: "sonnet"` when spawning all subagents via the Task tool @@ -81,7 +82,7 @@ Detect and read inputs: 2. If a second path-like token is present and resolves to an existing file, read `{revision_guidance}`. Extract all revision instructions — these take priority over your initial analysis in Step 2. Note which sections of the plan need rework. - When absent or empty, skip this sub-step and proceed normally (first pass). + When absent or empty, omit this sub-step and proceed normally (first pass). ### Step 2 — Explore Feasibility @@ -271,7 +272,7 @@ dependencies: - Run `pytest --collect-only` in the research directory to verify test discovery - Commit the test suite -{Adapt phases as needed — not all experiments require all phases. Add or +{Adapt phases as needed — experiments may not require every phase. Add or remove phases to match the specific experiment. Each phase should list the specific files to create and commands to run.} diff --git a/src/autoskillit/skills_extended/plan-visualization/SKILL.md b/src/autoskillit/skills_extended/plan-visualization/SKILL.md index 03f3ca7ea6..21a4d27419 100644 --- a/src/autoskillit/skills_extended/plan-visualization/SKILL.md +++ b/src/autoskillit/skills_extended/plan-visualization/SKILL.md @@ -36,6 +36,7 @@ outputs, and synthesizes a complete visualization plan. - Run vis-lens calls across multiple assistant messages — all selected lens calls must appear in a SINGLE assistant message to execute in parallel - Write outputs outside `{{AUTOSKILLIT_TEMP}}/plan-visualization/` +- Run subagents in the background (`run_in_background: true` is prohibited) **ALWAYS:** - Use `model: "sonnet"` when spawning all subagents via the Task tool @@ -86,13 +87,43 @@ Experiment-type table (use when no override fires or to fill second Tier-B slot) Cap Tier B at 2 lenses total (overrides count toward this cap). -**Tier C (0–1 based on target_domain):** -| target_domain | Lens | -|---|---| -| nlp | vis-lens-domain-norms | -| cv | vis-lens-domain-norms | -| rl | vis-lens-temporal | -| (others / general) | — (skip Tier C) | +**Tier C (0–1 based on methodology tradition detection):** + +Tier C selects `vis-lens-methodology-norms` when the experiment plan's research +methodology is identifiable from the 12 bundled methodology traditions. + +**Stage 1 — Deterministic keyword match:** +1. Load all methodology traditions from `recipes/methodology-traditions/*.yaml` +2. For each tradition, count how many of its `detection_keywords` appear in the + experiment plan text (case-insensitive, word-boundary matching) +3. Build `candidate_set` = traditions with ≥ 2 keyword matches +4. Branch on `len(candidate_set)`: + +| Candidates | Action | `precedence_trace` | +|---|---|---| +| 0 | Skip Tier C entirely | `stage1_no_match_fallback` | +| 1 | Use that tradition as `primary_tradition` | `stage1_single_match` | +| ≥ 2 | Proceed to Stage 2 | — | + +**Stage 2 — LLM arbitration (only when ≥ 2 candidates):** +1. If any registered `UnionRuleDef` covers the candidate set, apply it: + select `resolved_tradition`, record rule name in `applied_union_rules`, + set `precedence_trace = "stage2_tiebreak_by_rule_{rule_name}"` +2. Otherwise, select among candidates by analyzing the plan's primary research + question and methodology at temperature 0. Prefer the tradition whose + mandatory figures are most relevant to the stated research design. + Set `precedence_trace = "stage2_tiebreak_by_methodology_fit"` + +**Emit routing triple** (include as a fenced yaml block in the vis-lens context file): +```yaml +primary_tradition: +applied_union_rules: [, ...] +precedence_trace: "" +candidate_set: [] +``` + +When `primary_tradition` is set, add `vis-lens-methodology-norms` to `selected_lenses` +and write the `tradition_slug` and routing triple into its context file (Step 2). Only add Tier C lens if it is not already in Tier A or Tier B. @@ -134,6 +165,19 @@ Template for each context file: data_manifest or results/ section if available} ``` +When the context file is for `vis-lens-methodology-norms`, append the following +section to the template above: + +``` +## Methodology Tradition +tradition_slug: {primary_tradition from Tier-C routing triple} +routing_triple: + primary_tradition: {slug} + applied_union_rules: [{rules}] + precedence_trace: {trace} + candidate_set: [{candidates}] +``` + ### Step 3 — Run Vis-Lens Skills in Parallel In a **single assistant message**, invoke all `selected_lenses` as slash commands: @@ -156,14 +200,14 @@ For each figure-spec block where two lenses disagree on chart type, color encodi or layout, apply the conflict resolution hierarchy: ``` -accessibility > anti-pattern > domain-norms > chart-select +accessibility > anti-pattern > methodology-norms > chart-select ``` Resolution rules: - `accessibility` (from `vis-lens-always-on` or `vis-lens-color-access`) wins over all - `anti-pattern` findings (from `vis-lens-antipattern` or always-on pass 1) override - chart-select and domain-norms recommendations -- `domain-norms` (from `vis-lens-domain-norms`) overrides `chart-select` + chart-select and methodology-norms recommendations +- `methodology-norms` (from `vis-lens-methodology-norms`) overrides `chart-select` - `chart-select` (from `vis-lens-chart-select`) is the lowest priority Every resolution must be logged as a row in the Conflict Resolution Log table. @@ -215,9 +259,39 @@ Content structure: | Appendix | all | full captions | ``` +### Step 6.5 — Write visualization-plan-trace.md + +After completing Tier-C routing, write a trace file capturing routing decisions: + +Path: `{{AUTOSKILLIT_TEMP}}/plan-visualization/visualization-plan-trace.md` + +Content structure: +```markdown +# Visualization Plan Trace + +## Tier-C Routing Decision + +- **tier_c_lens**: {selected_lens_name or null} +- **primary_tradition**: {methodology_tradition_slug or null} +- **disambiguation_rule_applied**: {rule_name or null} +- **applied_union_rules**: [{list of union rules applied, or empty if none}] +- **precedence_trace**: [{chain of precedence resolution, or null}] +``` + +Fill in the fields from the Tier-C routing performed in Step 1: +- `tier_c_lens`: the lens selected by Tier-C (0-or-1), or `null` if target_domain was `general`/others +- `primary_tradition`: the `target_domain` value that determined Tier-C selection (e.g., `nlp`, `cv`, `rl`, `general`) +- `disambiguation_rule_applied`: `null` for now (disambiguation rules from #844 not yet implemented) +- `applied_union_rules`: empty list `[]` for now +- `precedence_trace`: `null` for now + ### Step 7 — Emit Structured Tokens ``` visualization_plan_path = {absolute_path_to_visualization-plan.md} report_plan_path = {absolute_path_to_report-plan.md} +disambiguation_rule_applied = {disambiguation_rule_applied or null} +tier_c_lens = {tier_c_lens or null} +methodology_tradition = {primary_tradition or null} +visualization_plan_trace_path = {absolute_path_to_visualization-plan-trace.md} ``` diff --git a/src/autoskillit/skills_extended/planner-analyze/SKILL.md b/src/autoskillit/skills_extended/planner-analyze/SKILL.md new file mode 100644 index 0000000000..b6ea1257fd --- /dev/null +++ b/src/autoskillit/skills_extended/planner-analyze/SKILL.md @@ -0,0 +1,77 @@ +--- +name: planner-analyze +categories: [planner] +description: Analyze project structure for planning decomposition +hooks: + PreToolUse: + - matcher: "*" + hooks: + - type: command + command: "echo '[SKILL: planner-analyze] Analyzing project structure...'" + once: true +--- + +# planner-analyze + +Detect language, framework, test infrastructure, project structure, and existing patterns in the target project. Produces `analysis.json` used by subsequent planner skills. + +## When to Use + +- Invoked by the planner recipe as the first analysis step +- User says "analyze project structure" in a planning context + +## Arguments + +- **$1** — Absolute path to the run-scoped planner directory (e.g., `{{AUTOSKILLIT_TEMP}}/planner/run-YYYYMMDD-HHMMSS`). Created by the `init` step. + +## Critical Constraints + +**NEVER:** +- Modify any target project files +- Write analysis.json outside `$1/` +- Run subagents in the background (`run_in_background: true` is prohibited) + +**ALWAYS:** +- Use Explore subagents for all file reads +- Spawn all 4 subagents in parallel +- Write valid JSON to `analysis.json` + +## Workflow + +### Step 1: Launch 4 parallel Explore subagents + +Spawn all four concurrently with `model: "sonnet"`: + +1. **Languages & Frameworks** — Identify primary language, framework, build system. Look for: `pyproject.toml`, `package.json`, `Cargo.toml`, `go.mod`, `pom.xml`, `build.gradle`, import statements, dependency files. + +2. **Test Infrastructure** — Identify test runner, coverage tools, test directory layout. Look for: `pytest.ini`, `jest.config.*`, `go test`, `cargo test`, test file naming patterns. + +3. **Architecture Patterns** — Identify architecture style (layered, hexagonal, monolithic, microservices, etc.) and count modules. Look for: directory depth, import graphs, layer naming, package boundaries. + +4. **Existing Conventions** — Identify naming conventions, code patterns, and risk areas. Look for: consistent naming in identifiers, repeated structural patterns, areas with high coupling or missing tests. + +### Step 2: Synthesize results + +Merge all four agent outputs into a single `analysis.json` document matching the output schema. + +### Step 3: Write output + +Write to `$1/analysis.json`. The directory was created by the `init` step. + +## Output Schema + +```json +{ + "language": "python", + "framework": "fastapi", + "build_system": "uv", + "test_runner": "pytest", + "architecture_style": "layered", + "module_count": 42, + "key_patterns": ["dependency injection", "protocol-based contracts"], + "conventions": ["snake_case identifiers", "private prefix _"], + "risks": ["high coupling in server layer", "no tests for migration engine"] +} +``` + +All fields are required. Use `null` for fields that cannot be determined. Arrays may be empty but must be present. diff --git a/src/autoskillit/skills_extended/planner-assess-review-approach/SKILL.md b/src/autoskillit/skills_extended/planner-assess-review-approach/SKILL.md new file mode 100644 index 0000000000..cd02fe3afa --- /dev/null +++ b/src/autoskillit/skills_extended/planner-assess-review-approach/SKILL.md @@ -0,0 +1,114 @@ +--- +name: planner-assess-review-approach +categories: [planner] +description: > + Assess each work package for review-approach benefit before implementation. + Writes review_approach_assessment.json; does NOT invoke review-approach. +hooks: + PreToolUse: + - matcher: "*" + hooks: + - type: command + command: "echo '[SKILL: planner-assess-review-approach] Assessing review-approach benefit...'" + once: true +--- + +# planner-assess-review-approach + +Assessment-only pass that evaluates each work package for review-approach benefit. +Reads `refined_wps.json` and `analysis.json`, spawns subagents to evaluate WPs, and +writes `review_approach_assessment.json` to the planner directory. Does NOT invoke +`review-approach` — assessment only. + +## Arguments + +- **$1** — Absolute path to `refined_wps.json` (PlanDocument with `task`, `work_packages[]`) +- **$2** — Absolute path to the planner output directory (for `analysis.json` and output) + +## Critical Constraints + +**NEVER:** +- Invoke `review-approach` — this skill performs assessment only +- Write output outside `$2/` +- Modify input files +- Run subagents in the background (`run_in_background: true` is prohibited) + +**ALWAYS:** +- Read the `review-approach` SKILL.md at `src/autoskillit/skills_extended/review-approach/SKILL.md` before assessing +- Read `$1` to get `task` and `work_packages[]` +- Read `$2/analysis.json` for codebase technology context +- Write `$2/review_approach_assessment.json` +- Emit: `review_approach_assessment_path = ` + +## Workflow + +### Step 1: Ground heuristics + +Read `src/autoskillit/skills_extended/review-approach/SKILL.md`. Understand what +`review-approach` does and when it provides value. Do not rely solely on the hardcoded +signals below — use the SKILL.md as the authoritative source for benefit criteria. + +### Step 2: Read inputs + +Read `$1` to extract the `task` field and `work_packages[]` list. Read `$2/analysis.json` +for codebase technology context: available libraries, architectural patterns in use, and +established conventions. This context informs whether a WP is "following established patterns" +(no-benefit) versus "introducing something new" (benefit signal). + +### Step 3: Evaluate each WP + +Spawn 1–2 subagents (model: "sonnet") to evaluate WPs in parallel batches. For each WP, +evaluate against these signals: + +**Benefit signals (recommend: true):** +- Involves integrating an unfamiliar external library or API +- Proposes a design decision with multiple viable architectural approaches +- References emerging patterns, standards, or technologies not yet in the codebase +- Contains open questions about *how* to approach the problem +- Requires understanding trade-offs between competing solutions + +**No-benefit signals (recommend: false):** +- Well-scoped bug fix with a clear root cause +- Internal refactoring following established codebase patterns +- Adds a feature using patterns already present in the codebase +- Documentation update or configuration change +- Already contains a fully specified implementation approach in the WP body + +Per WP, produce: `review_approach_recommended` (bool) and `review_approach_reasoning` +(one sentence). + +### Step 4: Write output + +Write `$2/review_approach_assessment.json`: + +```json +{ + "schema_version": 1, + "assessments": [ + { + "wp_id": "P1-A1-WP1", + "review_approach_recommended": true, + "review_approach_reasoning": "WP requires evaluating trade-offs between two competing persistence strategies." + } + ] +} +``` + +Example path: `{{AUTOSKILLIT_TEMP}}/planner/run-20260502-120000/review_approach_assessment.json` + +### Step 5: Emit output token + +``` +review_approach_assessment_path = $2/review_approach_assessment.json +``` + +## Context Limit Behavior + +This skill writes `$2/review_approach_assessment.json` before emitting structured output +tokens. If context is exhausted mid-execution: + +1. Before emitting any structured output tokens, verify that `review_approach_assessment.json` + exists in `$2/`. +2. If the file exists, emit the structured token and exit normally. +3. If context exhaustion interrupts before the file is written, the caller's + `on_context_limit` routing handles escalation — do not attempt partial output. diff --git a/src/autoskillit/skills_extended/planner-consolidate-wps/SKILL.md b/src/autoskillit/skills_extended/planner-consolidate-wps/SKILL.md new file mode 100644 index 0000000000..0aa1f13779 --- /dev/null +++ b/src/autoskillit/skills_extended/planner-consolidate-wps/SKILL.md @@ -0,0 +1,185 @@ +--- +name: planner-consolidate-wps +categories: [planner] +description: Analyze WP complexity per phase and emit consolidation group manifests for trivial WP merging (L1+L0 pattern) +hooks: + PreToolUse: + - matcher: "*" + hooks: + - type: command + command: "echo '[SKILL: planner-consolidate-wps] Analyzing WP complexity for consolidation...'" + once: true +--- + +# planner-consolidate-wps + +L1 session that analyzes WP complexity per phase and proposes consolidation groups +for trivial work packages. Dispatches one L0 subagent per phase in parallel. Each +L0 evaluates per-WP complexity, groups trivial WPs that share the same assignment +and have sequential ordering or a direct dependency chain, and writes a +`{phase_id}_consolidation.json` manifest. The L1 validates each response and writes +the manifest files to `{planner_dir}/work_packages/consolidation/`. + +This is an analysis-only step: even if all L0s fail, the downstream +`consolidate_wps_merge` callable performs a no-op passthrough (no manifests = no +merging). + +## When to Use + +- Launched by the L2 planner recipe after `refine_wps`, before `consolidate_wps_merge` +- Receives `refined_wps.json` (full WP set) and the planner run directory +- Produces per-phase `{phase_id}_consolidation.json` files + +## Arguments + +- **$1** — Absolute path to `refined_wps.json` (PlanDocument with `work_packages: list[WPElaborated]`) +- **$2** — Absolute path to the run-scoped planner directory (e.g., `{{AUTOSKILLIT_TEMP}}/planner/run-YYYYMMDD-HHMMSS`) + +## Critical Constraints + +**NEVER:** +- Merge WPs from different assignments unless they have a direct dependency linking them +- Create a merged group that would clearly exceed a medium-complexity PR (use judgment — no hard threshold) +- Skip a WP from a manifest — every WP must appear in exactly one group (singleton = no-op) +- Use a `merged_id` that is not one of the `source_wp_ids` +- Allow an L0 to write files outside `$2/work_packages/consolidation/` +- Run subagents in the background (`run_in_background: true` is prohibited) +- Spawn more than 6 L0s in a single parallel batch + +**ALWAYS:** +- Create `$2/work_packages/consolidation/` before dispatching L0s +- Validate each L0 response before writing its manifest +- Write a manifest for every phase, even if it contains only singleton groups +- Emit: `consolidation_manifest_dir = {planner_dir}/work_packages/consolidation` + +## Workflow + +### Step 1: Parse inputs + +Read `$1` (refined_wps.json). Parse as a PlanDocument. Extract all WP entries from +`work_packages[]`. Fail immediately if the file is malformed: +``` +FATAL: failed to parse {path}: {error_detail} +``` + +Group WPs by `phase_id` (always populated; read directly from the field). +Build a map `phase_id → [WPElaborated, ...]`. + +### Step 2: Create output directory + +```bash +mkdir -p "$2/work_packages/consolidation" +``` + +### Step 3: Build L0 context packets + +For each phase, assemble a context packet containing: +- `phase_id` — the phase being analyzed +- `wps` — the full list of WP objects in this phase (id, name, goal, technical_steps, + files_touched, deliverables, acceptance_criteria, depends_on, estimated_files, scope) +- `all_wp_ids` — WP IDs from every phase (for cross-phase dep awareness) +- Merge guidance: + - **Default: merge.** WPs within the same assignment should be merged unless there is + a clear reason to keep them separate. + - **Merge when:** WPs share files, do closely related work, form a natural unit of + implementation, or would be awkward as separate PRs. + - **Keep separate when:** WPs represent genuinely independent concerns that a developer + would naturally implement and review as separate PRs. + - **Soft upper bound:** Avoid merging into a group that would clearly exceed a + medium-complexity PR (rough guide: >15 substantive technical steps or >10 distinct + files). This is guidance, not a hard gate — use judgment about whether the combined + work forms a coherent unit. + +### Step 4: Dispatch parallel L0 subagents + +If phase count ≤ 6: spawn all in one parallel batch. +If phase count > 6: spawn sequential batches of 6. + +Each L0 receives the phase context packet and must return structured text in this exact format: +``` +phase_id = P1 +groups = [ + { + "merged_id": "P1-A1-WP1", + "source_wp_ids": ["P1-A1-WP1", "P1-A1-WP2"], + "merge_order": ["P1-A1-WP1", "P1-A1-WP2"], + "name": null, + "goal": null + }, + { + "merged_id": "P1-A1-WP3", + "source_wp_ids": ["P1-A1-WP3"], + "merge_order": ["P1-A1-WP3"], + "name": null, + "goal": null + } +] +``` + +L0 grouping rules: +- Assign each WP to exactly one group (solo or merged). +- **Default to merging** WPs within the same assignment. Only keep WPs as singletons when + they represent genuinely distinct concerns. +- Cross-assignment merges are allowed when WPs share a direct dependency AND touch the + same files. +- Use judgment about combined complexity. Avoid creating groups that would clearly be + too large for a single PR, but do not apply hard thresholds on step/file counts. + Read the actual work described and decide: "would a developer naturally do these + together?" +- `merged_id` MUST be the lowest-numbered WP ID in the group (primary WP). +- `merge_order` defines the technical_steps concatenation order; typically + `[primary, ...others_by_id_order]`. +- `name` and `goal` may be null (inherit from primary) or a short override string when + the merged purpose is clearly distinct from the primary WP's name. + +### Step 5: Validate L0 responses + +For each L0 response: +- `phase_id` must be present and match the expected phase ID +- `groups` must be a valid JSON array +- Every WP in the phase must appear in exactly one group's `source_wp_ids` +- `merged_id` must be one of the `source_wp_ids` + +On `phase_id` mismatch: +``` +WARNING: L0 response phase_id mismatch — expected {expected}, got {actual} — skipping +``` + +On validation failure: +``` +WARNING: L0 response for {phase_id} failed validation — skipping +``` + +On complete L0 failure: +``` +CRITICAL: L0 for {phase_id} failed — no manifest written for this phase +``` + +### Step 6: Write manifests + +For each validated L0 response, write the manifest file: +``` +$2/work_packages/consolidation/{phase_id}_consolidation.json +``` + +Manifest format: +```json +{ + "phase_id": "P1", + "groups": [ + { + "merged_id": "P1-A1-WP1", + "source_wp_ids": ["P1-A1-WP1", "P1-A1-WP2"], + "merge_order": ["P1-A1-WP1", "P1-A1-WP2"], + "name": null, + "goal": null + } + ] +} +``` + +### Step 7: Emit output token + +``` +consolidation_manifest_dir = $2/work_packages/consolidation +``` diff --git a/src/autoskillit/skills_extended/planner-elaborate-assignments/SKILL.md b/src/autoskillit/skills_extended/planner-elaborate-assignments/SKILL.md new file mode 100644 index 0000000000..30cbeedbd7 --- /dev/null +++ b/src/autoskillit/skills_extended/planner-elaborate-assignments/SKILL.md @@ -0,0 +1,163 @@ +--- +name: planner-elaborate-assignments +categories: [planner] +description: > + Elaborate all assignments for a target phase via parallel L0 subagents. + One invocation per phase; spawns one L0 per assignment concurrently. (Pass 2 loop body) +--- + +# planner-elaborate-assignments + +Pass 2 loop body. Receives one phase context file (written by `expand_assignments`), spawns one L0 +subagent per assignment in parallel using the native Agent/Task tool, collects results, +and writes per-assignment result files plus a phase sentinel file. The L1 (this skill) +is the sole writer for this phase's assignments — no concurrent write races. + +## When to Use + +- Invoked by the planner recipe when the recipe dispatches parallel elaboration via `dispatch_items` +- One invocation per phase; handles all assignments within the phase in a single session + +## Arguments + +- **$1** — Absolute path to the phase context file (written by `expand_assignments`) (contains `id=`, `metadata.assignment_count`, `metadata.assignment_ids`, `metadata.assignment_names`, `prior_results`) +- **$2** — Absolute path to the run-scoped planner directory (e.g., `{{AUTOSKILLIT_TEMP}}/planner/run-YYYYMMDD-HHMMSS`) + +## Critical Constraints + +**NEVER:** +- Allow L0 subagents to write files directly — L0s return JSON only +- Let an L0 failure abort the phase — always write a stub and continue +- Write output outside `$2/assignments/` +- Spawn L0s sequentially — always in parallel +- Read `{{AUTOSKILLIT_TEMP}}` artifacts outside your designated input files and output directory +- Explore parent directories of your input paths (e.g., `ls $(dirname $1)/..`) +- Read result files from other phases +- Run subagents in the background (`run_in_background: true` is prohibited) + +**ALWAYS:** +- Spawn all L0 subagents in parallel using the native Agent/Task tool (NOT run_skill — skill session guard blocks it) +- Write the phase sentinel file before emitting the output token +- Emit: `phase_assignments_result_dir = ` +- Write a stub result for any L0 that fails or returns invalid JSON + +## Workflow + +### Step 1: Parse context file + +Read the context file at `$1`. Extract: +- `id` — the phase ID (e.g., `"P1"`) +- `metadata.assignment_ids` — list of assignment IDs for this phase +- `metadata.assignment_names` — parallel list of assignment names +- `prior_results` — list of paths to result files from completed prior items + +### Step 2: Load phase context + +Read `$2/phases/{id}_result.json` (the elaborated phase result). Extract: +- `goal` — phase-level goal +- `scope` — phase-level scope list +- `technical_approach` — overall technical approach for the phase +- `assignments` — array of assignment objects with `name`, `goal`, `metadata` + +Read the `task_file_path` field from the context file at $1, then read the task description +from disk at that path. Every assignment elaboration — its `goal`, `scope`, `deliverables`, +and `work_packages_preview` — must serve the stated task. Do not read the full task text +into the L1 context or embed it in the L0 prompt — pass the path reference only and +instruct L0s to read the file from disk for scope creep verification. + +### Step 3: Build per-L0 context packets + +For each assignment in the phase, build a self-contained context packet: +- Assignment ID, name, goal (from the phase result's assignments array) +- All other phase assignments in short-form (id, name, goal only — for overlap detection) +- Prior result file paths (from `prior_results`) for cross-phase dependency detection +- The phase's `technical_approach` and `scope` +- The planner directory path `$2` for reading prior results if needed + +### Step 4: Spawn L0 subagents in PARALLEL + +Use the native Agent/Task tool to spawn one L0 per assignment simultaneously. +All L0s must be launched in a single batch — do NOT wait for one before starting the next. + +Each L0 receives a self-contained prompt that: +1. Identifies the assignment (ID, name, goal) +2. Provides short-form context for all other phase assignments +3. Lists prior result file paths for cross-phase overlap analysis +4. Instructs the L0 to use Grep/Glob/Read for codebase analysis (no sub-subagents) +5. Instructs the L0 to decompose into 1–5 work packages +6. Instructs the L0 to return results as JSON between triple-backtick json fences + +Each L0 MUST: +- Use Grep/Glob/Read for codebase analysis (no sub-subagent spawning — they are actual leaf nodes) +- Scan for dependencies by comparing scope against other phase assignments (short-form provided in prompt) +- Read prior result files (if paths provided) to detect cross-phase overlaps +- Decompose into 1–5 work packages with: `id_suffix` (WP1, WP2, ...), `name`, `scope`, `estimated_files` +- Return structured JSON between ` ```json ` and ` ``` ` delimiters +- Include `dependency_notes` (string) and `overlap_notes` (string) in the JSON + +Expected L0 return schema: +```json +{ + "id": "P1-A2", + "phase_id": "P1", + "name": "", + "goal": "", + "technical_approach": "", + "dependency_notes": "Depends on P1-A1 for ", + "overlap_notes": "No overlap detected with other assignments", + "proposed_work_packages": [ + { + "id_suffix": "WP1", + "name": "", + "scope": "", + "estimated_files": [ + "src//.py", + "src//.py" + ] + } + ] +} +``` + +### Step 5: Collect and validate L0 responses + +For each L0 response: +1. Extract JSON from between ` ```json ` and ` ``` ` delimiters +2. Validate required fields: `id`, `phase_id`, `name`, `goal`, `technical_approach`, `proposed_work_packages` +3. On validation failure OR no valid JSON found: emit `WARNING: L0 elaboration failed for {assignment_id}` and mark for stub creation + +### Step 6: Write per-assignment files + +For each successful L0 result, write `$2/assignments/{assignment_id}_result.json` with the full result JSON. + +For each failed L0, write `$2/assignments/{assignment_id}_result.json` with: +```json +{ + "id": "...", + "phase_id": "...", + "name": "...", + "goal": "...", + "technical_approach": "", + "proposed_work_packages": [], + "elaboration_failed": true +} +``` + +After writing all assignment files, update `$2/work_packages/wp_index.json` by appending compact entries +for all **successful** results only (skip stubs). Read the current index, append, and write back +atomically. L1 is the sole writer for this phase's assignments — no concurrent writes. + +Finally, write the phase sentinel file to `$2/assignments/{phase_id}_result.json`: +```json +{"id": "", "status": "complete", "assignment_count": N, "failed_count": M} +``` + +> The sentinel path MUST be `$2/assignments/{phase_id}_result.json`. The manifest's +> `result_dir` points to `$2/assignments/`, and this path is used to detect phase +> completion. Verify the path before writing. + +### Step 7: Emit output token + +``` +phase_assignments_result_dir = +``` diff --git a/src/autoskillit/skills_extended/planner-elaborate-phase/SKILL.md b/src/autoskillit/skills_extended/planner-elaborate-phase/SKILL.md new file mode 100644 index 0000000000..d8138bc5ab --- /dev/null +++ b/src/autoskillit/skills_extended/planner-elaborate-phase/SKILL.md @@ -0,0 +1,112 @@ +--- +name: planner-elaborate-phase +categories: [planner] +description: Elaborate a single phase into a full result, parallel-safe — receives plan snapshot + target phase ID +hooks: + PreToolUse: + - matcher: "*" + hooks: + - type: command + command: "echo '[SKILL: planner-elaborate-phase] Elaborating phase...'" + once: true +--- + +# planner-elaborate-phase + +Standalone parallel worker for Phase Pass 1. Each instance receives the full plan +snapshot (each phase in condensed form) and a target phase ID. It explores the codebase +independently and writes a single elaborated phase result. No dependency on +`check_remaining` or any shared state machine. + +## When to Use + +- Launched in parallel by the L2 orchestrator (planner recipe, Issue 08) +- One instance per phase ID, all running simultaneously +- Also usable standalone for manual single-phase elaboration + +## Arguments + +- **$1** — Absolute path to `plan_snapshot.json` (every phase as a `PhaseShort` entry) +- **$2** — Phase ID to elaborate (e.g., `"P3"`) +- **$3** — Absolute path to output directory (result written here) + +## Critical Constraints + +**NEVER:** +- Write output outside `$3/` +- Read any `*_result.json` file from other phases (you have only the snapshot) +- Require or read a context file from `check_remaining` +- Communicate with other parallel worker instances +- Read `{{AUTOSKILLIT_TEMP}}` artifacts outside your designated input files and output directory +- Explore parent directories of your input paths (e.g., `ls $(dirname $1)/..`) +- Run subagents in the background (`run_in_background: true` is prohibited) + +**ALWAYS:** +- Derive `relationship_notes` from snapshot context + codebase analysis, NOT from prior result files +- Write result to `$3/{phase_id}_result.json` (keep `_result.json` suffix — downstream consumers glob `*_result.json`) +- Emit: `elab_result_path = ` +- Include all `PhaseElaborated` fields in the result + +## Workflow + +### Step 1: Parse arguments and read snapshot + +Read the plan snapshot at `$1`. It is a `PlanDocument` with a `phases` list of `PhaseShort` objects: +```json +{ + "schema_version": 1, + "task": "...", + "source_dir": "...", + "phases": [ + {"id": "P1", "name": "...", "goal": "...", "scope": [...], "ordering": 1}, + {"id": "P2", "name": "...", "goal": "...", "scope": [...], "ordering": 2}, + ... + ] +} +``` + +Find the entry in `phases` where `id == "$2"` (the target phase). Note its `ordering` to +understand which phases come before and after it. + +After reading `plan_snapshot.json`, extract the `task` field. Every aspect of the elaborated +phase — its `technical_approach`, `scope`, and `assignments[]` — must serve the stated task. +Do not elaborate into work not requested by the task. Flag if the phase goal appears +unrelated to the task. + +### Step 2: Launch parallel codebase exploration subagents + +Spawn up to 5 simultaneous Explore subagents against the codebase in `source_dir`: + +1. **Affected files** — Which files/modules fall within this phase's `scope`? Current state, imports, deviations from conventions. +2. **Dependency analysis** — What imports and consumes the affected modules? Full import graph. +3. **Test coverage** — Which tests cover the affected scope? Gaps in coverage? +4. **Pattern discovery** — What conventions and reusable utilities exist in this scope? +5. **Cross-phase boundaries** — Based on snapshot context (other phases' names/goals/scopes), where do likely dependencies or handoff points exist? + +### Step 3: Write phase result + +Write to `$3/{target_phase_id}_result.json` matching `PhaseElaborated`: +```json +{ + "id": "P3", + "name": "...", + "goal": "...", + "scope": [...], + "ordering": 3, + "technical_approach": "...", + "relationship_notes": "Depends on P1 (...name...) for ...; P5 (...name...) will consume ...", + "assignments_preview": ["Assignment title 1", "Assignment title 2", ...] +} +``` + +For `relationship_notes`: use other phases' `name`, `goal`, and `scope` from the snapshot +(not their result files) combined with codebase evidence to identify real dependencies. + +Do NOT write `phase_number` or `name_slug` — the backend derives these at load time from +`ordering` and `name` respectively. + +### Step 4: Emit output token + +``` +elab_result_path = +``` diff --git a/src/autoskillit/skills_extended/planner-elaborate-wps/SKILL.md b/src/autoskillit/skills_extended/planner-elaborate-wps/SKILL.md new file mode 100644 index 0000000000..aa8a426775 --- /dev/null +++ b/src/autoskillit/skills_extended/planner-elaborate-wps/SKILL.md @@ -0,0 +1,168 @@ +--- +name: planner-elaborate-wps +categories: [planner] +description: > + Elaborate all work packages for a target phase via parallel L0 subagents. + One invocation per phase; spawns one L0 per WP concurrently. (Pass 3 loop body) +--- + +# planner-elaborate-wps + +Pass 3 loop body. Receives one phase context file (written by `expand_wps`), spawns one L0 +subagent per WP in parallel using the native Agent/Task tool, collects results, +and writes per-WP result files plus a phase sentinel file. The L1 (this skill) +is the sole writer for this phase's WPs — no concurrent write races. + +## When to Use + +- Invoked by the planner recipe once per phase via sequential dispatch (`dispatch_items`) +- One invocation per phase; handles all WPs within the phase in a single session + +## Arguments + +- **$1** — Absolute path to the phase context file (written by `expand_wps`) (contains `id=`, `metadata.wp_count`, `metadata.wp_ids`, `metadata.wp_names`, `metadata.wp_scopes`, `metadata.wp_estimated_files`, `prior_results`) +- **$2** — Absolute path to the run-scoped planner directory (e.g., `{{AUTOSKILLIT_TEMP}}/planner/run-YYYYMMDD-HHMMSS`) + +## Critical Constraints + +**NEVER:** +- Allow L0 subagents to write files directly — L0s return JSON only +- Let an L0 failure abort the phase — always write a stub and continue +- Write output outside `$2/work_packages/` +- Spawn L0s sequentially — always in parallel +- Spawn more than 6 L0s in one batch — if WP count exceeds 6, use sequential batches of 6 +- Read `{{AUTOSKILLIT_TEMP}}` artifacts outside your designated input files and output directory +- Explore parent directories of your input paths (e.g., `ls $(dirname $1)/..`) +- Read result files from other phases +- Run subagents in the background (`run_in_background: true` is prohibited) + +**ALWAYS:** +- Spawn all L0 subagents in parallel using the native Agent/Task tool (NOT run_skill — skill session guard blocks it) +- Write the phase sentinel file before emitting the output token +- Emit: `phase_wps_result_dir = ` +- Write a stub result for any L0 that fails or returns invalid JSON + +## Workflow + +### Step 1: Parse context file + +Read the context file at `$1`. Extract: +- `id` — the phase ID (e.g., `"P1"`) +- `metadata.wp_ids` — list of WP IDs for this phase (e.g., `["P1-A1-WP1", "P1-A1-WP2"]`) +- `metadata.wp_names` — parallel list of WP names +- `metadata.wp_scopes` — parallel list of WP scopes +- `metadata.wp_estimated_files` — parallel list of estimated file lists +- `prior_results` — list of paths to result files from completed prior items + +### Step 2: Load phase and assignment context + +Read `$2/phases/{id}_result.json` (the elaborated phase result). Extract: +- `goal` — phase-level goal +- `scope` — phase-level scope list +- `technical_approach` — overall technical approach for the phase + +Read all `$2/assignments/P{N}-A*_result.json` matching this phase to get assignment-level context: +- `goal` — assignment goal +- `technical_approach` — assignment technical approach +- `proposed_work_packages` — the WP decomposition from the assignment pass + +Read the `task_file_path` field from the context file at $1, then read the task description +from disk at that path. Every WP elaboration — its `deliverables`, `acceptance_criteria`, +and `estimated_files` — must serve the stated task. Do not read the full task text into the +L1 context or embed it in the L0 prompt — pass the path reference only and instruct L0s +to read the file from disk for scope creep verification. + +### Step 3: Build per-L0 context packets + +For each WP in the phase, build a self-contained context packet: +- WP ID, name, scope, estimated_files (from manifest metadata) +- Assignment context: the parent assignment's goal, technical_approach +- All sibling WPs for this phase in short-form (id, name, scope only — for dependency detection) +- Phase goal and scope +- Instruct L0 to use Grep/Glob/Read for codebase analysis (no sub-subagents) +- Instruct L0 to return JSON between triple-backtick json fences + +### Step 4: Spawn L0 subagents in PARALLEL + +Use the native Agent/Task tool to spawn one L0 per WP simultaneously. +If WP count > 6, spawn in sequential batches of 6 — await each batch before starting the next. + +Each L0 receives a self-contained prompt that: +1. Identifies the WP (ID, name, scope, estimated_files) +2. Provides assignment context (goal, technical_approach) +3. Lists all sibling WPs in short-form for dependency detection +4. Provides phase goal and scope +5. Instructs the L0 to use Grep/Glob/Read for codebase analysis (no sub-subagents) +6. Instructs the L0 to produce exactly 1–5 deliverables (hard constraint — validate_wp_result rejects out-of-range counts) +7. Instructs the L0 to return results as JSON between triple-backtick json fences + +Each L0 MUST: +- Use Grep/Glob/Read for codebase analysis (no sub-subagent spawning — they are actual leaf nodes) +- Elaborate the WP with all mandatory fields +- Produce exactly 1–5 deliverables — if the WP naturally spans more than 5 files, group related files into logical deliverables (e.g., "test suite for module X" rather than individual test files) +- Return structured JSON between ` ```json ` and ` ``` ` delimiters + +Expected L0 return schema: +```json +{ + "id": "P1-A2-WP1", + "name": "...", + "goal": "...", + "summary": "<=120 chars", + "technical_steps": ["..."], + "files_touched": ["..."], + "apis_defined": ["..."], + "apis_consumed": ["..."], + "depends_on": ["..."], + "deliverables": ["(exactly 1–5 items, hard limit) file_or_logical_group", "..."], + "acceptance_criteria": ["..."] +} +``` + +### Step 5: Collect and validate L0 responses + +For each L0 response: +1. Extract JSON from between ` ```json ` and ` ``` ` delimiters +2. Validate required fields: `id`, `name`, `deliverables` (matching `WP_REQUIRED_KEYS`) +3. On validation failure OR no valid JSON found: emit `WARNING: L0 elaboration failed for {wp_id}` and mark for stub creation. CRITICAL: partial failure must not abort the phase. + +### Step 6: Write per-WP files + +For each successful L0 result, write `$2/work_packages/{wp_id}_result.json` with the full result JSON. + +For each failed L0, write `$2/work_packages/{wp_id}_result.json` with: +```json +{ + "id": "...", + "name": "...", + "deliverables": [], + "elaboration_failed": true, + "summary": "", + "goal": "", + "technical_steps": [], + "files_touched": [], + "apis_defined": [], + "apis_consumed": [], + "depends_on": [], + "acceptance_criteria": [] +} +``` + +After writing all WP files, update `$2/work_packages/wp_index.json` by appending compact entries +for all **successful** results only (skip stubs). Read the current index, append, and write back +atomically. L1 is the sole writer for this phase's WPs — no concurrent writes. + +Finally, write the phase sentinel file to `$2/work_packages/wp_sentinels/{phase_id}_result.json`: +```json +{"id": "", "status": "complete", "wp_count": N, "failed_count": M} +``` + +> The sentinel path MUST be `$2/work_packages/wp_sentinels/{phase_id}_result.json`. The manifest's +> `result_dir` points to `$2/work_packages/wp_sentinels/`, and this path is used to detect +> phase completion. Verify the path before writing. + +### Step 7: Emit output token + +``` +phase_wps_result_dir = +``` diff --git a/src/autoskillit/skills_extended/planner-extract-domain/SKILL.md b/src/autoskillit/skills_extended/planner-extract-domain/SKILL.md new file mode 100644 index 0000000000..511e4d1535 --- /dev/null +++ b/src/autoskillit/skills_extended/planner-extract-domain/SKILL.md @@ -0,0 +1,73 @@ +--- +name: planner-extract-domain +categories: [planner] +description: Extract domain knowledge and naming conventions for planning context +hooks: + PreToolUse: + - matcher: "*" + hooks: + - type: command + command: "echo '[SKILL: planner-extract-domain] Extracting domain knowledge...'" + once: true +--- + +# planner-extract-domain + +Extract domain knowledge, naming conventions, and structural patterns specific to the project. Optional step — failure is non-fatal and the planner recipe continues without domain context. + +## When to Use + +- Invoked by the planner recipe after `planner-analyze` completes +- Provides richer domain context for decomposition planning + +## Arguments + +- **$1** — Absolute path to `analysis.json` produced by `planner-analyze` +- **$2** — Absolute path to a file containing the task description. When provided and non-empty, focus domain extraction on areas relevant to the stated task. When empty, perform a full-codebase survey. + +## Critical Constraints + +**NEVER:** +- Modify any target project files +- Abort the calling recipe on failure — log a warning and return gracefully +- Run subagents in the background (`run_in_background: true` is prohibited) +- If `$1` is empty or the file does not exist, STOP immediately and report failure + +**ALWAYS:** +- Read the analysis file from argument $1 before spawning subagents +- Use Explore subagents for all file reads +- Spawn subagents in parallel + +## Workflow + +### Step 1: Read analysis + +Read the `analysis.json` file from argument $1. Use its `language`, `framework`, `architecture_style`, and `key_patterns` fields to focus subagent queries. + +### Step 2: Launch 3–5 parallel Explore subagents + +Read the task description: if $2 is provided and non-empty, read the file at that path. + +If the task description is available, include it in each subagent's prompt: "Focus exploration on +domain vocabulary, abstractions, and integration points relevant to this task: {task}. +Prioritize areas the task will touch over exhaustive full-codebase coverage." + +Spawn all concurrently with `model: "sonnet"`. Always spawn agents 1–3; spawn agents 4–5 only when the project has >20 modules or architecture_style is layered/hexagonal: + +1. **Domain Vocabulary** — Extract domain-specific terms, entity names, and verb patterns used in identifiers. Look for: class names, function names, docstrings, README files, ADR documents. + +2. **Existing Abstractions** — Identify base classes, protocols, ABCs, and reusable interfaces. Look for: `class * (Protocol)`, `ABC`, the `abstractmethod` decorator, shared base types. + +3. **Integration Points** — Identify external system boundaries, HTTP clients, database adapters, message queues. Look for: import of third-party HTTP/DB libraries, adapter classes, port/adapter naming. + +4. **Cross-cutting Concerns** (deep mode) — Identify async patterns, error handling conventions, logging strategy. Look for: `async def`, custom exception hierarchies, structured logging calls. + +5. **Data Flow Patterns** (deep mode) — Identify pipeline stages, transformation chains, data schemas. Look for: dataclass chains, TypedDict, Pydantic models, transformation functions. + +### Step 3: Synthesize + +Merge all agent outputs into a coherent `domain_knowledge.md` Markdown document with sections: Domain Vocabulary, Key Abstractions, Integration Points, Cross-cutting Concerns, Data Flow Patterns. + +### Step 4: Write output (non-fatal) + +Write to `$(dirname $1)/domain_knowledge.md`. If any step fails, log a warning to stdout and exit with code 0 — do not propagate the error to the recipe. diff --git a/src/autoskillit/skills_extended/planner-generate-phases/SKILL.md b/src/autoskillit/skills_extended/planner-generate-phases/SKILL.md new file mode 100644 index 0000000000..eff3f3433a --- /dev/null +++ b/src/autoskillit/skills_extended/planner-generate-phases/SKILL.md @@ -0,0 +1,146 @@ +--- +name: planner-generate-phases +categories: [planner] +description: Generate high-level phases from project analysis (Pass 1 entry point) +hooks: + PreToolUse: + - matcher: "*" + hooks: + - type: command + command: "echo '[SKILL: planner-generate-phases] Generating phases...'" + once: true +--- + +# planner-generate-phases + +Pass 1 entry point. Read the analysis file (and optionally domain knowledge) and produce +phase definitions in a single session. The number of phases is determined entirely by the +task — each phase is an independently verifiable checkpoint. Write all phase results and +a fully-done `phase_manifest.json` in one shot. + +## When to Use + +- Invoked by the planner recipe as the Pass 1 phase-generation step +- After `planner-analyze` (and optionally `planner-extract-domain`) have completed + +## Arguments + +- **$1** — Absolute path to `analysis.json` produced by `planner-analyze` +- **$2** — (optional) Absolute path to `domain_knowledge.md` produced by `planner-extract-domain` +- **$3** — Absolute path to a file containing the task description + +## Critical Constraints + +**NEVER:** +- Invent phases that do not serve the task description — every phase must map to work requested by the user +- Anchor phase count to a predetermined number — derive it from the task's natural verification boundaries +- Write output outside `$(dirname $1)/phases/` +- Use freeform text instead of the required JSON schema +- Read files outside `$(dirname $1)` or the project's git-tracked source tree +- Explore parent directories of `$(dirname $1)` (e.g., `ls $(dirname $1)/..`) +- Read `{{AUTOSKILLIT_TEMP}}` artifacts from other planner runs or pipeline steps +- If `$3` is empty or the file does not exist, STOP immediately and report failure. Do not attempt to infer the task from the codebase, GitHub issues, or any other source. + +**ALWAYS:** +- Write `$(dirname $1)/phases/{phase_id}_result.json` for every phase +- Write `$(dirname $1)/phases/phase_manifest.json` with every item status=`done` +- Use sequential `ordering` values starting at 1 +- Emit: `phase_manifest_path = `; also emit `phase_count` and `phase_ids` + +## Workflow + +### Step 0: Read task description + +Read the task description from the file at `$3`. This is the user's statement of what +they want planned. Every generated phase MUST serve this task. +Do not generate phases for work not described in the task. If the task asks for specific +deliverables (e.g., "split research.yaml into 4 sub-recipes"), the phases should decompose +that work — not decompose the codebase into architectural layers. + +### Step 1: Read inputs + +Read `analysis.json` from argument $1. If $2 is provided and the file exists, read +`domain_knowledge.md` from $2. Use the domain vocabulary and patterns to inform phase naming +and scope. + +### Step 2: Decompose into phases + +Decompose the implementation work into phases. A phase is an **independently verifiable +checkpoint** — a coherent set of features whose functionality can be fully tested once +implemented. Draw phase boundaries wherever you can stop and validate that everything +up to that point works. + +Phases should be: +- Coherent (each phase has a single, clear goal) +- Ordered by dependency (foundational work first) +- Non-overlapping in scope +- Named to reflect the task's work units, grounded in the codebase's architecture + +The number of phases is determined by the task: +- A single-component change may warrant 1 phase +- A multi-component feature may warrant one phase per independently testable component +- There is no minimum or maximum — let the verification boundaries dictate the count + +Do NOT provide yourself with concrete domain examples of phase decomposition. Concrete +examples anchor the decomposition pattern regardless of the actual task (pink elephant +effect). Derive phase names and boundaries from the task description and codebase analysis. + +For each phase, generate: +- `id`: Sequential `P{N}` identifier (P1, P2, ...) +- `name`: Short, descriptive phase name +- `goal`: One-sentence statement of what the phase achieves +- `scope`: Array of domain areas or component names covered +- `ordering`: Integer sequence position (1-based) +- `relationship_notes`: Description of dependencies on prior phases ("Foundation phase — no prior dependencies" for P1) +- `assignments_preview`: Array of 2–5 short strings naming likely assignments within this phase + +### Step 3: Write phase results + +For each phase, write to `$(dirname $1)/phases/{phase_id}_result.json`: + +```json +{ + "id": "P1", + "name": "", + "goal": "", + "scope": ["", ""], + "ordering": 1, + "relationship_notes": "Foundation phase — no prior dependencies", + "assignments_preview": ["", "", ""] +} +``` + +The backend derives two additional fields at load time — do not write them: +- `phase_number` (integer): derived from `ordering` +- `name_slug` (string): derived by slugifying `name` (e.g., "Component Setup" → "component-setup") + +### Step 4: Write phase manifest + +Write `$(dirname $1)/phases/phase_manifest.json`. Set every item's status to +`done` (Pass 1 is coarse-grained enough to resolve in one shot; the elaborate loop exists +only as a fallback). Set `result_path` to the absolute path of the corresponding result file. + +Manifest structure: +```json +{ + "pass_name": "phases", + "created_at": "", + "items": [ + { + "id": "P1", + "name": "", + "status": "done", + "result_path": "/phases/P1_result.json", + "metadata": {"ordering": 1} + } + ] +} +``` + +### Step 5: Emit output tokens + +``` +phase_manifest_path = +phase_count = +phase_ids = +``` diff --git a/src/autoskillit/skills_extended/planner-reconcile-deps/SKILL.md b/src/autoskillit/skills_extended/planner-reconcile-deps/SKILL.md new file mode 100644 index 0000000000..5fda05ea73 --- /dev/null +++ b/src/autoskillit/skills_extended/planner-reconcile-deps/SKILL.md @@ -0,0 +1,105 @@ +--- +name: planner-reconcile-deps +categories: [planner] +description: Post-Pass-3 global dependency DAG construction — detects implicit dependencies individual WP sessions could not see +hooks: + PreToolUse: + - matcher: "*" + hooks: + - type: command + command: "echo '[SKILL: planner-reconcile-deps] Reconciling dependency graph...'" + once: true +--- + +# planner-reconcile-deps + +Post-Pass-3 global dependency analysis. Reads the full `wp_index.json` (compact, ~12k +tokens for 60 WPs) and produces a corrected global dependency DAG. Detects implicit +dependencies that individual WP sessions could not see — forward references, transitive +chains, and API consumption patterns missed during per-WP elaboration. + +Runs as a single LLM session for holistic cross-WP reasoning that per-WP sessions cannot +perform. The compact index fits entirely in context, enabling global analysis without +spawning parallel sessions. + +## When to Use + +- Invoked by the planner recipe immediately after Pass 3 completes +- One invocation per plan — processes the complete WP set + +## Arguments + +- **$1** — Absolute path to the run-scoped planner directory (e.g., `{{AUTOSKILLIT_TEMP}}/planner/run-YYYYMMDD-HHMMSS`) + +## Critical Constraints + +**NEVER:** +- Load full WP result files — only `wp_index.json` (the compact array) +- Spawn additional sessions — this skill must run as a single session for holistic reasoning +- Write output outside `{$1}/` +- Add transitive deps without a concrete API or file relationship as evidence + +**ALWAYS:** +- Read `wp_index.json` in full before analyzing +- Populate `added_backward_deps` only when an `apis_consumed`/`apis_defined` or `files_touched` match justifies the dependency +- Emit `dep_graph_path` as the output token + +## Workflow + +### Step 1: Load wp_index.json + +Read `{$1}/work_packages/wp_index.json`. This JSON array contains compact entries for +all WPs (~200 bytes each). + +Each entry has: `id`, `name`, `summary`, `phase`, `assignment`, `files_touched`, +`apis_defined`, `apis_consumed`, `depends_on`, `deliverables`, `result_path`. + +### Step 2: Analyze dependency relationships + +Scan the full WP array holistically for: + +1. **Implicit backward dependencies** — A WP has an `apis_consumed` entry that matches + another WP's `apis_defined`, but the consuming WP does not list the defining WP in + `depends_on`. These are missing backward deps that go into `added_backward_deps`. + +2. **File-level implicit deps** — A WP's `files_touched` overlaps with a prior WP's + `deliverables`, but no `depends_on` relationship exists. The later WP implicitly + depends on the earlier. + +3. **Forward dependency map** — For each WP, compute which other WPs depend on it + (transpose of `depends_on`). This produces the `forward_deps` map. + +4. **Reverse dependency consolidation** — Collect the existing `depends_on` arrays into + the `reverse_deps` map for downstream consumers (e.g., `compile_plan`). + +### Step 3: Build dep_graph.json + +```json +{ + "forward_deps": { + "P1-A1-WP1": ["P1-A2-WP1", "P2-A1-WP1"] + }, + "reverse_deps": { + "P1-A2-WP1": ["P1-A1-WP1", "P1-A1-WP2"] + }, + "added_backward_deps": { + "P2-A3-WP1": ["P1-A2-WP2"] + } +} +``` + +- `forward_deps`: For each WP, list all WPs that depend on it (forward in execution order) +- `reverse_deps`: For each WP, list the WPs it currently depends on (mirrors `depends_on` arrays) +- `added_backward_deps`: New deps not declared in original `depends_on` — only when a concrete API/file match justifies the relationship + +If no implicit deps are detected, write `added_backward_deps: {}`. + +### Step 4: Write dep_graph.json + +Write to `{$1}/dep_graph.json`. + +### Step 5: Emit output tokens + +``` +dep_graph_path = +``` diff --git a/src/autoskillit/skills_extended/planner-refine-assignments/SKILL.md b/src/autoskillit/skills_extended/planner-refine-assignments/SKILL.md new file mode 100644 index 0000000000..9b64b5b8f3 --- /dev/null +++ b/src/autoskillit/skills_extended/planner-refine-assignments/SKILL.md @@ -0,0 +1,196 @@ +--- +name: planner-refine-assignments +categories: [planner] +description: Refine elaborated assignments for a single phase via parallel L0 subagents (L1+L0 pattern), using per-phase context file with peer_summaries for cross-phase visibility +hooks: + PreToolUse: + - matcher: "*" + hooks: + - type: command + command: "echo '[SKILL: planner-refine-assignments] Refining phase assignments with cross-visibility...'" + once: true +--- + +# planner-refine-assignments + +L1 session that refines the assignments for a single phase by spawning one L0 subagent per +assignment in parallel (3–5 assignments per phase). Receives a per-phase context file produced +by `merge_tier_results` → `_write_refine_contexts`, which contains only this phase's assignments +and `peer_summaries` (id/name/goal stubs) for all other phases. Each L0 reviews its assignment +in the context of peer_summaries and `refined_plan.json`, returning structured suggestions. L1 +applies suggestions, resolves intra-phase WP ownership conflicts, and writes the phase result +file to `$3/refine_contexts/{phase_id}_result.json`. + +## When to Use + +- Dispatched by the L2 planner recipe in parallel — one session per phase +- Accepts a per-phase context file from `$3/refine_contexts/context_{phase_id}.json` and `refined_plan.json` for phase context +- Produces `{phase_id}_result.json` as input for the downstream `merge_refined_assignments` step + +## Arguments + +- **$1** — Absolute path to the per-phase context file (`$3/refine_contexts/context_{phase_id}.json`). The file contains: + - `phase_id` — identifier for the phase this session processes + - `task_file_path` — path to the task description file (read from disk, not inline) + - `assignments` — full AssignmentElaborated objects for this phase only (3–5 entries) + - `peer_summaries` — `{id, name, goal}` stubs for all assignments in other phases +- **$2** — Absolute path to `refined_plan.json` (PlanDocument with phases as PhaseElaborated, for phase context) +- **$3** — Absolute path to the run-scoped planner directory (e.g., `{{AUTOSKILLIT_TEMP}}/planner/run-YYYYMMDD-HHMMSS`). Output is written to `$3/refine_contexts/{phase_id}_result.json`. + +## Critical Constraints + +**NEVER:** +- Write any file outside `$3/` +- Directly modify the context file ($1) — always write a new result file +- Allow an L0 subagent to write files directly (L0s return structured text only) +- Emit `phase_refined_path` before writing the result file +- Skip emitting `phase_refined_path` even if all L0s fail (write unchanged assignments, still emit) +- Spawn more than 6 L0s in a single parallel batch +- Read `{{AUTOSKILLIT_TEMP}}` artifacts not passed as positional arguments +- Run subagents in the background (`run_in_background: true` is prohibited) +- Write L0 prompts to intermediate `l0_prompts/` files and read them back into the L1 context — spawn L0 subagents directly from in-memory context packets +- Read source code files, test files, or recipe YAML files directly — codebase exploration is the L0 subagents' responsibility +- Run Bash, Grep, or Glob commands for codebase exploration between L0 spawns + +**ALWAYS:** +- Read `phase_id` from the context file to construct the output path +- Validate each L0 response for `assignment_id`, `changes` (array), `dependency_corrections` (array), `wp_proposal_adjustments` (array) +- Log `WARNING` to stdout for any L0 response that fails validation (skip that assignment) +- Log `CRITICAL` to stdout for any L0 subagent that fails entirely (proceed with N-1, partial result) +- When two assignments propose WPs covering the same files, assign ownership to the numerically earlier assignment_id using natural sort on numeric suffixes (e.g. `P1-A1` beats `P1-A2`; `P1-A2` beats `P1-A10`); log each resolution +- Emit: `phase_refined_path = ` + +## Workflow + +### Step 1: Parse input and validate + +Read `$1` (per-phase context file). Extract `phase_id`, `assignments`, `peer_summaries`, and +`task_file_path`. Fail immediately (exit non-zero) if `assignments` is empty or the file is +malformed — do not proceed to spawn L0s. The failure message must include the file path and the +parse/validation error string: +``` +FATAL: failed to parse {path}: {error_detail} +``` + +Read `$2` (refined_plan.json). Build a map `phase_id → PhaseElaborated` for L0 context. + +Input schema for the per-phase context file: +```json +{ + "phase_id": "P2", + "task_file_path": "/path/to/task.md", + "assignments": [ + { + "id": "P2-A1", + "phase_id": "P2", + "name": "...", + "goal": "...", + "technical_approach": "...", + "dependency_notes": "...", + "work_packages": [...] + } + ], + "peer_summaries": [ + {"id": "P1-A1", "name": "...", "goal": "..."}, + {"id": "P3-A1", "name": "...", "goal": "..."} + ] +} +``` + +### Step 2: Build L0 context packets + +For each assignment in `assignments`, build a context packet containing: +- The full serialized assignment object (AssignmentElaborated) +- The `peer_summaries` list for cross-phase dependency detection +- The `PhaseElaborated` entry for the assignment's `phase_id` from `$2` +- The `target_assignment_id` +- `task_file_path` — the path to the task description on disk (pass the path reference only; do NOT read the task text into the L1 context or embed it in the L0 prompt) +- Instructions: review the target assignment in light of peer_summaries; always read the task from disk at `task_file_path` for scope creep verification (flag scope creep if the assignment's goal, scope, or deliverables introduce work not requested by the task); return structured suggestions only — do NOT edit files + +### Step 3: Spawn parallel L0 subagents + +Since each phase has 3–5 assignments (always within the 6-L0 ceiling), spawn all in one +parallel batch via Agent/Task. Do not spawn more than 6 L0s in a single parallel batch: +if assignment count > 6 (unexpected), spawn sequential batches of 6. Between batches, emit +anti-prose guard line: `--- next batch ---`. + +Reading `l0_prompts/*.txt` files back before spawning adds ~15K tokens to the L1 context per L0 with no benefit. + +Each L0 must return structured text in this exact format: +``` +assignment_id = P1-A2 +changes = [ + {"field": "dependency_notes", "new_value": "Requires P1-A1-WP1 interface before starting"}, + {"field": "technical_approach", "new_value": "..."} +] +dependency_corrections = [ + {"missing_dep": "P1-A1-WP2", "reason": "Uses the auth token cache defined there"} +] +wp_proposal_adjustments = [ + {"wp_id_suffix": "WP1", "action": "remove", "reason": "Duplicate of P1-A1-WP3 which owns users.py"} +] +``` + +### Step 4: Validate L0 responses + +For each L0 response: +- `assignment_id` must be present and match the expected assignment ID for that L0 +- `changes` must be a valid JSON array (may be empty `[]`) +- `dependency_corrections` must be a valid JSON array (may be empty `[]`) +- `wp_proposal_adjustments` must be a valid JSON array (may be empty `[]`) + +On `assignment_id` mismatch (field present but does not match expected ID): +``` +WARNING: L0 response assignment_id mismatch — expected {expected}, got {actual} — skipping +``` + +On other validation failure (field absent or array invalid): +``` +WARNING: L0 response for {assignment_id} failed validation — skipping +``` + +On L0 subagent complete failure (no response / timeout): +``` +CRITICAL: L0 for {assignment_id} failed — proceeding with N-1 suggestions +``` + +### Step 5: Resolve WP proposal conflicts + +Collect all `wp_proposal_adjustments` from validated L0 responses. Group by the +set of files each WP proposal covers (use `estimated_files` from the elaborated WP). +When two assignments both claim a file set: assign ownership to the assignment with +the numerically earlier `assignment_id` using natural sort on numeric suffixes +(e.g. `P1-A1` beats `P1-A2`; `P1-A2` beats `P1-A10`). +Log each resolution: +``` +WP CONFLICT: {assignment_id_A} vs {assignment_id_B} — {field} assigned to {winner} +``` +Apply the `remove` action from the losing assignment's `wp_proposal_adjustments`. + +### Step 6: Apply changes + +Apply all validated `changes` to the in-memory assignments document, in assignment +ID order. Apply conflict resolutions before applying changes for affected assignments. +Skip unrecognized field names: +``` +WARNING: Unrecognized field '{field}' in changes for {assignment_id} — skipping +``` +Apply `dependency_corrections` by appending to the `dependency_notes` field. + +### Step 7: Write output + +Write the phase result file to `$3/refine_contexts/{phase_id}_result.json`, where +`phase_id` is read from the context file. The output schema: +```json +{ + "schema_version": 1, + "assignments": [...] +} +``` +The `assignments` list contains only this phase's assignments (3–5 entries) with refinements applied. + +### Step 8: Emit output token + +``` +phase_refined_path = +``` diff --git a/src/autoskillit/skills_extended/planner-refine-phases/SKILL.md b/src/autoskillit/skills_extended/planner-refine-phases/SKILL.md new file mode 100644 index 0000000000..ff0ab75ebc --- /dev/null +++ b/src/autoskillit/skills_extended/planner-refine-phases/SKILL.md @@ -0,0 +1,158 @@ +--- +name: planner-refine-phases +categories: [planner] +description: Refine elaborated phases with cross-phase visibility via parallel L0 subagents (L1+L0 pattern) +hooks: + PreToolUse: + - matcher: "*" + hooks: + - type: command + command: "echo '[SKILL: planner-refine-phases] Refining phases with cross-visibility...'" + once: true +--- + +# planner-refine-phases + +L1 session that refines a `combined_plan.json` (a `PlanDocument` with every phase +in `PhaseElaborated` form) by spawning one L0 subagent per phase in parallel. +Each L0 reviews its assigned phase in the context of all other phases and returns +structured suggestions. L1 collects these suggestions, resolves inter-phase +conflicts, applies field-level edits to the plan, and writes `refined_plan.json`. + +## When to Use + +- Launched by the L2 planner recipe after the parallel phase elaboration merge step +- Accepts the `combined_plan.json` output from the parallel phase elaboration merge step +- Produces `refined_plan.json` as input for downstream planner steps + +## Arguments + +- **$1** — Absolute path to `combined_plan.json` (PlanDocument, every phase as PhaseElaborated) +- **$2** — Absolute path to the run-scoped planner directory (e.g., `{{AUTOSKILLIT_TEMP}}/planner/run-YYYYMMDD-HHMMSS`). Output is written to `$2/refined_plan.json`. + +## Critical Constraints + +**NEVER:** +- Write any file outside `$2/` +- Directly modify the combined_plan.json ($1) — always write a new refined_plan.json +- Allow an L0 subagent to write files directly (L0s return structured text only) +- Emit `refined_plan_path` before writing `refined_plan.json` +- Skip emitting `refined_plan_path` even if all L0s fail (write unchanged plan, still emit) +- Read `{{AUTOSKILLIT_TEMP}}` artifacts not passed as positional arguments +- Run subagents in the background (`run_in_background: true` is prohibited) +- Spawn more than 6 L0s in a single parallel batch + +**ALWAYS:** +- Validate each L0 response for `phase_id`, `changes` (array), `conflicts` (array) +- Log a `WARNING` to stdout for any L0 response that fails validation (skip that phase) +- Log `CRITICAL` to stdout for any L0 subagent that fails entirely (proceed with N-1) +- Log each conflict resolution to stdout before applying it +- Emit: `refined_plan_path = ` + +## Workflow + +### Step 1: Parse input and validate + +Read `$1` (combined_plan.json). Parse as a `PlanDocument`. Extract all phase IDs +from `phases[*].id`. Fail immediately (exit non-zero) if `phases` is empty or the +file is malformed — do not proceed to spawn L0s. + +Input schema (PlanDocument with PhaseElaborated phases): +```json +{ + "schema_version": 1, + "task": "...", + "source_dir": "...", + "phases": [ + { + "id": "P1", + "name": "...", + "goal": "...", + "scope": [...], + "ordering": 1, + "technical_approach": "...", + "relationship_notes": "...", + "assignments_preview": [...] + } + ] +} +``` + +### Step 2: Spawn parallel L0 subagents + +Read the `task` field from the combined plan document. Each L0 subagent reviewing a phase +must verify that the phase's goal and scope serve the stated task. Phases that appear to +address codebase concerns not mentioned in the task should be flagged for scope creep. + +If phase count ≤ 6: spawn one L0 subagent per phase in a single parallel batch. +If phase count > 6: spawn sequential batches of 6 L0 subagents. Wait for each batch +to complete before spawning the next. Process phases in ordering sequence. + +Each L0 receives: +- The full serialized `combined_plan.json` content (pasted inline or via file read) +- Its `target_phase_id` (e.g., `"P2"`) +- Instructions: review the target phase in light of what all other phases committed + to; return structured suggestions only — do NOT edit files + +Each L0 must return structured text in this exact format: +``` +phase_id = P2 +changes = [ + {"field": "relationship_notes", "new_value": "Depends on P1 for the auth client interface defined in P1-A2-WP1"}, + {"field": "technical_approach", "new_value": "...updated approach..."} +] +conflicts = [ + {"with_phase": "P1", "description": "Both P1 and P2 claim ownership of the token refresh module"} +] +``` + +### Step 3: Validate L0 responses + +For each L0 response: +- `phase_id` must be present and match the expected phase ID for that L0 +- `changes` must be a valid JSON array (may be empty `[]`) +- `conflicts` must be a valid JSON array (may be empty `[]`) + +On validation failure: +``` +WARNING: L0 response for {phase_id} failed validation — skipping +``` + +On L0 subagent complete failure (no response / timeout): +``` +CRITICAL: L0 for {phase_id} failed — proceeding with N-1 suggestions +``` + +### Step 4: Collect and resolve conflicts + +Collect all `conflicts` arrays from validated L0 responses. Group conflict reports +by the pair of phases involved. When multiple L0s report conflicts involving the +same resource: +1. Read the conflict descriptions +2. Make an ownership/scope judgment call based on phase goals and scopes +3. Log resolution before applying: + ``` + CONFLICT RESOLUTION: {phase_id} vs {with_phase} — {resolution decision} + ``` + +### Step 5: Apply changes + +Apply all validated `changes` to the in-memory plan document, in phase ID order +(P1, P2, ... PN). Apply conflict resolutions before applying changes for the +affected phases. Only update fields that exist in `PhaseElaborated` — skip +unrecognized field names: +``` +WARNING: Unrecognized field '{field}' in changes for {phase_id} — skipping +``` + +### Step 6: Write output + +Write the updated plan document to `$2/refined_plan.json`. The output schema is +identical to the input `combined_plan.json` (a `PlanDocument` with +`phases: list[PhaseElaborated]`). + +### Step 7: Emit output token + +``` +refined_plan_path = +``` diff --git a/src/autoskillit/skills_extended/planner-refine-wps/SKILL.md b/src/autoskillit/skills_extended/planner-refine-wps/SKILL.md new file mode 100644 index 0000000000..a3a52f73f5 --- /dev/null +++ b/src/autoskillit/skills_extended/planner-refine-wps/SKILL.md @@ -0,0 +1,218 @@ +--- +name: planner-refine-wps +categories: [planner] +description: Refine elaborated work packages with cross-phase visibility via per-phase L0 subagents (L1+L0 pattern) +hooks: + PreToolUse: + - matcher: "*" + hooks: + - type: command + command: "echo '[SKILL: planner-refine-wps] Refining WPs with cross-phase visibility...'" + once: true +--- + +# planner-refine-wps + +L1 session that refines a `combined_wps.json` (a `PlanDocument` with all work +packages in `WPElaborated` form) by spawning one L0 subagent per phase in +parallel (batched to 6). Unlike the per-WP elaboration pass, this skill spawns +one L0 per **phase** (not per WP). Each L0 reviews ALL WPs in its assigned phase +against the full WP set, detecting cross-phase API mismatches, duplicate +deliverables, missing dependencies, and scope overlap. The L1 collects structured +suggestions, resolves conflicts, and writes `refined_wps.json`. + +## When to Use + +- Launched by the L2 planner recipe after the parallel WP elaboration merge step +- Accepts `combined_wps.json`, `refined_plan.json`, and `refined_assignments.json` +- Produces `refined_wps.json` as input for downstream planner steps (e.g. reconcile_deps) + +## Arguments + +- **$1** — Absolute path to `combined_wps.json` (PlanDocument with `work_packages: list[WPElaborated]`) +- **$2** — Absolute path to `refined_plan.json` (PlanDocument with phases as PhaseElaborated) +- **$3** — Absolute path to `refined_assignments.json` (PlanDocument with assignments as AssignmentElaborated) +- **$4** — Absolute path to the run-scoped planner directory (e.g., `{{AUTOSKILLIT_TEMP}}/planner/run-YYYYMMDD-HHMMSS`). Output is written to `$4/refined_wps.json`. + +## Critical Constraints + +**NEVER:** +- Write any file outside `$4/` +- Directly modify `combined_wps.json` ($1) — always write a new `refined_wps.json` +- Allow an L0 subagent to write files directly (L0s return structured text only) +- Emit `refined_wps_path` before writing `refined_wps.json` +- Skip emitting `refined_wps_path` even if all L0s fail (write unchanged WPs, still emit) +- Spawn more than 6 L0s in a single parallel batch +- Spawn one L0 per WP — L0s operate per PHASE +- Read `{{AUTOSKILLIT_TEMP}}` artifacts not passed as positional arguments +- Run subagents in the background (`run_in_background: true` is prohibited) + +**ALWAYS:** +- Spawn one L0 per phase (NOT per WP) — each L0 reviews ALL WPs in its phase against the full WP set +- Validate each L0 response for `phase_id`, `wp_changes` (array), `cross_phase_deps` (array), `deliverable_conflicts` (array), `api_mismatches` (array) +- Log `WARNING` to stdout for any L0 response that fails validation (skip that phase) +- Log `CRITICAL` to stdout for any L0 subagent that fails entirely (proceed with N-1 suggestions) +- When two WPs claim the same deliverable file, assign ownership to the WP with the numerically earlier ID using natural sort (e.g., `P1-A1-WP1` beats `P2-A1-WP1`) +- Emit: `refined_wps_path = ` + +## Workflow + +### Step 1: Parse inputs and validate + +Read `$1` (combined_wps.json). Parse as a `PlanDocument`. Extract all WP entries +from `work_packages[]`. Fail immediately (exit non-zero) if `work_packages` is +empty or the file is malformed — do not proceed to spawn L0s. The failure message +must include the file path and the parse/validation error string: +``` +FATAL: failed to parse {path}: {error_detail} +``` + +Read `$2` (refined_plan.json). Build a map `phase_id → PhaseElaborated` for phase context. +Read `$3` (refined_assignments.json). Build a map `assignment_id → AssignmentElaborated` for assignment context. + +Input schema (PlanDocument with WPElaborated work packages): +```json +{ + "schema_version": 1, + "task": "...", + "source_dir": "...", + "work_packages": [ + { + "id": "P1-A1-WP1", + "assignment_id": "P1-A1", + "phase_id": "P1", + "name": "...", + "scope": "...", + "estimated_files": ["..."], + "goal": "...", + "summary": "...", + "technical_steps": ["..."], + "files_touched": ["..."], + "apis_defined": ["..."], + "apis_consumed": ["..."], + "depends_on": ["..."], + "deliverables": ["..."], + "acceptance_criteria": ["..."] + } + ] +} +``` + +### Step 2: Group WPs by phase and build L0 context packets + +Read the `task` field from the combined WPs document. Each L0 subagent reviewing a phase's +WPs must verify that every WP's deliverables and scope serve the stated task. Flag WPs +whose deliverables address concerns not mentioned in the task as scope creep. + +Group WPs by `phase_id` (always populated; read directly from the field). +For each phase, build a context packet containing: +- The full serialized `combined_wps.json` content (all WPs visible for cross-phase awareness) +- The `PhaseElaborated` entry for the phase from `$2` +- The `AssignmentElaborated` entries for all assignments in this phase from `$3` +- The `target_phase_id` +- The list of WP IDs assigned to this L0 (the WPs in this phase) +- Instructions: review this phase's WPs against the full WP set; return structured suggestions only — do NOT edit files + +### Step 3: Spawn parallel L0 subagents + +If phase count ≤ 6: spawn all in one parallel batch via Agent/Task. +If phase count > 6: spawn sequential batches of 6. Between batches, emit +anti-prose guard line: `--- next batch ---`. + +Each L0 MUST return structured text in this exact format: +``` +phase_id = P1 +wp_changes = [ + {"wp_id": "P1-A1-WP1", "field": "depends_on", "new_value": ["P2-A1-WP3"]}, + {"wp_id": "P1-A1-WP2", "field": "technical_steps", "new_value": ["Step 1...", "Step 2..."]} +] +cross_phase_deps = [ + {"wp_id": "P1-A1-WP2", "missing_dep": "P2-A1-WP1", "reason": "Consumes auth_client API defined by P2-A1-WP1"} +] +deliverable_conflicts = [ + {"wp_id_a": "P1-A1-WP1", "wp_id_b": "P2-A1-WP3", "file": "src/auth/client.py"} +] +api_mismatches = [ + {"consumer_wp": "P2-A1-WP1", "producer_wp": "P1-A1-WP2", "api": "SessionModel.create", "mismatch": "Consumer expects (user_id, token) but producer defines (user_id)"} +] +``` + +Each L0 receives instructions to use Grep/Glob/Read for codebase analysis but NOT +to write files or spawn sub-subagents. + +### Step 4: Validate L0 responses + +For each L0 response: +- `phase_id` must be present and match the expected phase ID +- `wp_changes` must be a valid JSON array (may be empty `[]`) +- `cross_phase_deps` must be a valid JSON array (may be empty `[]`) +- `deliverable_conflicts` must be a valid JSON array (may be empty `[]`) +- `api_mismatches` must be a valid JSON array (may be empty `[]`) + +On `phase_id` mismatch (field present but does not match expected ID): +``` +WARNING: L0 response phase_id mismatch — expected {expected}, got {actual} — skipping +``` + +On other validation failure (field absent or array invalid): +``` +WARNING: L0 response for {phase_id} failed validation — skipping +``` + +On L0 subagent complete failure (no response / timeout): +``` +CRITICAL: L0 for {phase_id} failed — proceeding with N-1 suggestions +``` + +### Step 5: Resolve conflicts + +Collect all `deliverable_conflicts` from validated L0 responses. For each +conflict where two WPs claim the same deliverable file, assign ownership to the +WP with the numerically earlier ID (natural sort: `P1-A1-WP1` < `P1-A2-WP1` < +`P2-A1-WP1`). Log each resolution: +``` +WP CONFLICT: {wp_id_a} vs {wp_id_b} — deliverable {file} assigned to {winner} +``` + +**Post-deduplication orphan check:** After resolving all deliverable conflicts, +scan every losing WP. If any WP now has `deliverables: []`, merge it into the +winning WP (the one that received its deliverables): +1. Promote the orphan's `files_touched` entries as deliverables to the winner WP, + selecting at most `DELIVERABLE_BOUNDS[1] - len(winner.deliverables)` entries + (prefer entries that overlap with the winner's existing scope). Any remaining + entries stay in `files_touched` only. +2. Append the orphan's `technical_steps` and `acceptance_criteria` to the winner +3. Remove the orphan WP from the output and update all `depends_on` references +4. Log: `WP ORPHAN MERGED: {orphan_id} → {winner_id}` +Deliverable count bounds are defined in `schema.py::DELIVERABLE_BOUNDS`. + +Process `api_mismatches`: for each mismatch, add the producer WP's `apis_defined` +signature to the `wp_changes` for the consumer WP to update `apis_consumed` to match. + +Process `cross_phase_deps`: for each missing dependency, append to the target WP's +`depends_on` list. + +### Step 6: Apply changes + +Apply all validated `wp_changes` to the in-memory WPs document, in WP ID order +(P1-A1-WP1 → P1-A1-WP2 → ... → PN-AN-WPN). Apply conflict resolutions and +cross-phase dep corrections before field-level changes. Skip unrecognized field names: +``` +WARNING: Unrecognized field '{field}' in wp_changes for {wp_id} — skipping +``` + +Valid WPElaborated fields for changes: `goal`, `summary`, `technical_steps`, +`files_touched`, `apis_defined`, `apis_consumed`, `depends_on`, `deliverables`, +`acceptance_criteria`, `scope`, `estimated_files`. + +### Step 7: Write output + +Write the updated `PlanDocument` to `$4/refined_wps.json`. The output schema is +identical to the input `combined_wps.json` (a `PlanDocument` with +`work_packages: list[WPElaborated]`). + +### Step 8: Emit output token + +``` +refined_wps_path = +``` diff --git a/src/autoskillit/skills_extended/planner-refine/SKILL.md b/src/autoskillit/skills_extended/planner-refine/SKILL.md new file mode 100644 index 0000000000..c8ecb94768 --- /dev/null +++ b/src/autoskillit/skills_extended/planner-refine/SKILL.md @@ -0,0 +1,167 @@ +--- +name: planner-refine +categories: [planner] +description: Targeted fix of validate_plan findings — re-elaboration, duplicate resolution, dependency corrections +hooks: + PreToolUse: + - matcher: "*" + hooks: + - type: command + command: "echo '[SKILL: planner-refine] Refining plan...'" + once: true +--- + +# planner-refine + +Targeted repair of `validate_plan` findings. Loads `validation.json` and repairs each +finding type: re-elaborates failed WPs, resolves duplicate deliverable ownership, and +fixes dependency reference errors. Sizing violations are escalated as CRITICAL. Writes +corrected artifacts back to the output directory so `validate_plan` can re-run. + +The recipe runs this skill with `retries: 2` — up to 3 total attempts (1 initial + 2 +retries) before escalation. + +## When to Use + +- Invoked by the planner recipe when `validate_plan` returns `verdict: fail` +- One invocation per retry cycle + +## Arguments + +- **$1** — Absolute path to `validation.json` +- **$2** — Absolute path to the run-scoped planner directory (e.g., `{{AUTOSKILLIT_TEMP}}/planner/run-YYYYMMDD-HHMMSS`) + +## Critical Constraints + +**NEVER:** +- Attempt to auto-fix missing assignments or missing WPs — these require human review +- Remove a deliverable without reassigning it to another WP +- Introduce new WP IDs — the skill never creates WPs; it repairs or escalates existing ones + +**ALWAYS:** +- Load `validation.json` before reading any artifact +- Fix all addressable finding types in a single pass +- Update `wp_manifest.json` and `wp_index.json` whenever WP structure changes +- Escalate sizing violations, missing structural elements, malformed WP IDs, and DAG cycles as CRITICAL (write to stdout; do not count toward issues_fixed) +- Emit both `refinement_complete` and `issues_fixed` output tokens + +## Workflow + +### Step 1: Load validation.json + +Read `$1`. Extract the `findings` array (contains only error-severity findings as structured +dicts). Extract the `message` field from each finding for classification. Group by type: + +- **failed_wps**: Findings matching `WP .* has status 'failed'` +- **sizing_violations**: Findings matching `WP .* has \d+ deliverables` +- **duplicate_deliverables**: Findings matching `Deliverable '.*' claimed by multiple WPs` +- **dep_references**: Findings matching `WP .* depends on unknown WP` +- **missing**: Findings matching `Phase .* has no assignments` or `Assignment .* has no work packages` +- **malformed_id**: Findings matching `WP .* has malformed id` +- **dag_cycle**: Findings matching `Cycle detected among WPs` +- **files_touched_overlap** (informational): Findings matching `File '.*' touched by multiple WPs` — these appear in the `warnings` array, not `findings`. No action needed; skip if encountered in `findings`. + +### Step 2: Load required artifacts + +- Always load: `{$2}/work_packages/wp_manifest.json`, `{$2}/work_packages/wp_index.json` +- Load only the `{id}_result.json` files for WPs mentioned in the findings + +### Step 3: Fix each finding type + +**Failed WPs** — re-elaborate: +- For each failed WP ID, read its entry from `wp_manifest.json` (provides `name`, `scope`, + `estimated_files`) +- Spawn a sub-agent with `model: "sonnet"` per failed WP. Provide: WP name, scope, + estimated_files, and the relevant portion of `wp_index.json` for context +- Sub-agent writes a corrected `{$2}/work_packages/{id}_result.json` +- Sub-agent appends corrected compact entry to `wp_index.json` +- Update the WP status in `wp_manifest.json` from `failed` to `done` + +**Sizing violations** — escalate: +- Findings matching `WP .* has \d+ deliverables` indicate WPs outside the 1–5 deliverable + sizing bound. Cannot be auto-corrected — the implementation recipe handles re-splitting + downstream. +``` +CRITICAL: Cannot auto-fix sizing violation: +- {finding text} +Manual review of WP deliverable allocation required. +``` +Write this to stdout. Do NOT attempt WP splitting or merging. + +**Duplicate deliverables** — resolve ownership: +- For each duplicated file path, assign ownership to the WP whose scope most directly + implements that file (strongest semantic claim) +- Remove the duplicate from the lower-priority WP's `deliverables` (keep in `files_touched`) +- **Post-deduplication orphan check:** After resolving all duplicate deliverables, scan every + modified WP. If any WP now has `deliverables: []`, that WP is an orphan. For each orphan: + 1. Identify the WP(s) that received this orphan's former deliverables (the "owner WPs") + 2. Promote the orphan's `files_touched` entries as deliverables to the most relevant owner + WP (the one with the most scope overlap), selecting at most + `DELIVERABLE_BOUNDS[1] - len(owner.deliverables)` entries. Any remaining entries stay + in `files_touched` only. + 3. Merge the orphan's `technical_steps` and `acceptance_criteria` into the owner WP + 4. Remove the orphan WP from the plan and update all `depends_on` references + 5. Update `wp_manifest.json` and `wp_index.json` accordingly + This is a deduplication side-effect resolved in the same step, not a sizing violation. + Deliverable count bounds are defined in `schema.py::DELIVERABLE_BOUNDS`. +- Write updated `_result.json` for the affected WPs + +**Dependency reference errors** — fix broken dep IDs: +- For each `WP X depends on unknown WP Y` finding: + - Search `wp_index.json` for a WP with a similar name or scope to the missing `Y` + (it may have been renamed or split) + - If a valid replacement is found, update `depends_on` in `{$2}/work_packages/{X}_result.json` + - If no valid replacement exists, remove the broken reference from `depends_on` +- If `dep_graph.json` exists, update it to reflect corrected dependency IDs + +**Missing assignments/WPs** — escalate: +``` +CRITICAL: Cannot auto-fix missing structural elements: +- {finding text} +Manual intervention required before validate_plan can pass. +``` +Write this to stdout. Do NOT attempt structural creation. + +**Malformed WP IDs** — escalate: +- Findings matching `WP .* has malformed id (expected PX-AY-WPZ)` indicate a corrupted + `_result.json` or `wp_manifest.json`. Cannot be auto-corrected without understanding the + intended ID. +``` +CRITICAL: Cannot auto-fix malformed WP ID: +- {finding text} +Manual inspection of wp_manifest.json required. +``` +Write this to stdout. Do NOT attempt ID renaming. + +**DAG cycles** — escalate: +- Findings matching `Cycle detected among WPs: ...` indicate a circular dependency in the + WP graph. Resolving cycles requires semantic understanding of the plan structure. +``` +CRITICAL: Cannot auto-fix DAG cycle: +- {finding text} +Manual restructuring of depends_on relationships required. +``` +Write this to stdout. Do NOT attempt cycle-breaking. + +### Step 4: Write corrected artifacts + +Write all modified files back atomically (read current → apply change → write). Modified +files may include: `_result.json` files, `wp_manifest.json`, `wp_index.json`, +`dep_graph.json`. + +> **Note:** Combined documents (`combined_*.json`, `refined_*.json`) are intermediate +> orchestration artifacts and are NOT updated by this skill. Downstream consumers +> (`validate_plan`, `compile_plan`) read from individual `*_result.json` files directly, +> so stale combined documents do not affect pipeline correctness. + +### Step 5: Emit output tokens + +``` +refinement_complete = true +issues_fixed = +``` + +`N` = count of findings addressed from the `findings` array (failed_wps + +duplicate_deliverables + dep_references). Sizing-violation, missing-element, malformed-ID, +and DAG-cycle findings are excluded from the count (they are escalated as critical, not +fixed). Files-touched overlap findings are in the `warnings` array and are not actionable. diff --git a/src/autoskillit/skills_extended/planner-validate-task-alignment/SKILL.md b/src/autoskillit/skills_extended/planner-validate-task-alignment/SKILL.md new file mode 100644 index 0000000000..941eefbec7 --- /dev/null +++ b/src/autoskillit/skills_extended/planner-validate-task-alignment/SKILL.md @@ -0,0 +1,101 @@ +--- +name: planner-validate-task-alignment +categories: [planner] +description: Validate that plan phases and WPs align with the stated task +hooks: + PreToolUse: + - matcher: "*" + hooks: + - type: command + command: "echo '[SKILL: planner-validate-task-alignment] Validating task alignment...'" + once: true +--- + +# planner-validate-task-alignment + +Post-refinement catch-net. Compares the plan's phases, assignments, and WP descriptions +against the original task description. Emits warning-severity findings for misalignment. +This is a safety net — if upstream task injection works correctly, this step should produce +zero findings. + +## Arguments + +- **$1** — Absolute path to `refined_wps.json` (PlanDocument with `task`, `work_packages[]`) +- **$2** — Absolute path to `refined_plan.json` (PlanDocument with `task`, `phases[]`) +- **$3** — Absolute path to output directory for findings + +## Critical Constraints + +**NEVER:** +- Block the pipeline — all findings are `warning` severity +- Write output outside `$3/` +- Read files not passed as arguments +- Modify input files +- Run subagents in the background (`run_in_background: true` is prohibited) + +**ALWAYS:** +- Read the `task` field from $1 or $2 +- Compare every phase goal and every WP description against the task +- Write `$3/task_alignment.json` with findings array +- Emit: `alignment_findings_path = `; also emit `alignment_finding_count` + +## Workflow + +### Step 1: Read inputs + +Read `refined_wps.json` from $1 and `refined_plan.json` from $2. Extract: +- The `task` field (the user's original task description) +- All phase `goal` and `scope` fields from $2 +- All WP `name`, `deliverables`, and `acceptance_criteria` fields from $1 + +### Step 2: Spawn alignment-check subagents + +Spawn 1-2 subagents (model: "sonnet"): + +**Subagent A — Phase alignment:** +Provide the task description and all phase goals/scopes. Ask: "For each phase, does its +goal directly serve the stated task? Rate each phase as 'aligned', 'tangential', or +'unrelated'. A phase is 'aligned' if the task explicitly or implicitly requires the work. +A phase is 'tangential' if it supports the task but was not requested. A phase is 'unrelated' +if the task does not mention or imply this work." + +**Subagent B — WP alignment:** +Provide the task description and all WP names/deliverables. Ask: "For each WP, do its +deliverables serve the stated task? Flag any WP whose deliverables address concerns not +mentioned in the task." + +### Step 3: Collect and write findings + +Parse subagent responses. For each phase or WP rated 'tangential' or 'unrelated', emit +a finding: + +```json +{ + "message": "Phase P2 ('Protocol Sharding') appears unrelated to the stated task", + "severity": "warning", + "check": "task_alignment", + "entity_id": "P2", + "rating": "unrelated" +} +``` + +Write all findings to `$3/task_alignment.json`: +```json +{ + "schema_version": 1, + "task": "", + "findings": [], + "summary": { + "aligned": 4, + "tangential": 1, + "unrelated": 0 + } +} +``` + +### Step 4: Emit output tokens + +``` +alignment_findings_path = +alignment_finding_count = +``` diff --git a/src/autoskillit/skills_extended/prepare-issue/SKILL.md b/src/autoskillit/skills_extended/prepare-issue/SKILL.md index 340d78c8b5..9ffe38e399 100644 --- a/src/autoskillit/skills_extended/prepare-issue/SKILL.md +++ b/src/autoskillit/skills_extended/prepare-issue/SKILL.md @@ -92,50 +92,66 @@ gh repo view --json owner,name ### Step 4: Dedup Check (skip if `--issue N` provided) Extract multiple keyword sets from the description — individual key terms and 2–3 phrase -combinations that capture the core topic. For each keyword set, search open issues: +combinations that capture the core topic. For each keyword set, search all issues: ```bash -gh issue list --state open --search "{keyword-set}" \ - --json number,title,url,body --limit 10 +gh issue list --state all --search "{keyword-set}" \ + --json number,title,url,body,labels,state --limit 15 ``` -Run searches for each keyword set and deduplicate results by issue number. Collect all unique -candidates. +> `labels` is returned as an array of objects `[{"name": "..."}]` by `gh`. +> Extract label names as a flat list when classifying eligibility. -If candidates are found, display them all in a numbered list with number, title, and URL: +Run searches for each keyword set and deduplicate results by issue number, tracking the +`match_count` (number of keyword sets that matched) per candidate. -``` -━━━ Possible Duplicates Found ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ -Found {N} open issue(s) that may be related: +After deduplication, apply a relevance gate to closed issues: include a closed issue as +a candidate only if it was matched by **two or more** keyword sets. Open issues are +included regardless of match count (same threshold as before). Discard single-match +closed results — they are likely false positives. + +Classify each surviving candidate as **extend-eligible** or **informational-only**: + +| Condition | Classification | +|-----------|---------------| +| `state == open` AND no `in-progress` label AND no `staged` label | extend-eligible | +| `state == open` AND has `in-progress` label | informational-only | +| `state == open` AND has `staged` label | informational-only | +| `state == closed` | informational-only | + +Assign sequential numbers `[1]`, `[2]`, … only to extend-eligible candidates. +Informational-only candidates receive a `[—]` marker. + +If candidates are found, display them with state annotations per candidate: - [1] #{number} — {title} +``` +━━━ Possible Duplicates Found ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ + [1] #{number} — {title} (open) {url} - [2] #{number} — {title} + [—] #{number} — {title} (in-progress) {url} - ... + [—] #{number} — {title} (closed) + {url} -━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ Options: - [1]–[{N}] Add to / extend an existing issue (enter a number) + [1]–[{N}] Add to / extend an existing issue (only shown if extend-eligible candidates exist) C Create a new issue anyway Your choice [C]: +━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ ``` +If no extend-eligible candidates exist (all are informational-only), omit the `[N]` +option entirely — only `C` appears. + **If the user enters C (or presses Enter):** Continue to Step 4a. -**If the user enters a number [1]–[N] (extend existing):** +**If the user enters a number corresponding to an extend-eligible candidate (extend existing):** -1. Ask: *"Add your context as a comment or edit the issue body? [comment/edit, default: comment]"* -2. If **comment** (default): - ```bash - gh issue comment {selected_number} --body "{description as additional context}" - ``` -3. If **edit**: +1. Fetch current body and append new context using a temp file to avoid shell injection: ```bash - # Fetch current body and append new context using a temp file to avoid shell injection ts=$(date +%Y-%m-%d_%H%M%S) EDIT_BODY_FILE="{{AUTOSKILLIT_TEMP}}/prepare-issue/edit_body_${ts}.md" mkdir -p "{{AUTOSKILLIT_TEMP}}/prepare-issue" @@ -143,12 +159,12 @@ Your choice [C]: printf '\n## Additional Context\n\n%s' "{description}" >> "${EDIT_BODY_FILE}" gh issue edit {selected_number} --body-file "${EDIT_BODY_FILE}" ``` -4. Set `issue_number = selected_number` (no new issue will be created). -5. Fetch the updated issue for triage: +2. Set `issue_number = selected_number` (no new issue will be created). +3. Fetch the updated issue for triage: ```bash gh issue view {selected_number} --json number,title,body,labels,url ``` -6. **Continue to Step 6 (LLM Classification)** on this existing issue, then proceed through +4. **Continue to Step 6 (LLM Classification)** on this existing issue, then proceed through Steps 7, 7a, 8, and 9 to apply labels and requirements. Emit the result block with the existing issue's number and URL, then exit. @@ -204,6 +220,17 @@ provided (adopting an existing issue) or when `--dry-run` is active. awk '/^## Findings with Exceptions/{skip=1} skip && /^---/{skip=0; next} !skip' \ "${ISSUE_BODY_FILE}" > "${ISSUE_BODY_FILE}.tmp" \ && mv "${ISSUE_BODY_FILE}.tmp" "${ISSUE_BODY_FILE}" + + # Defensive strip: remove any finding detail section that contains an exception note + # (guards against exception-warranted findings leaking inline into ## Validated Findings) + awk ' + /^\*\*Exception note:\*\*/ { in_exception=1; next } + in_exception && /^---$/ { in_exception=0; next } + in_exception && /^## / { in_exception=0 } + !in_exception + END { in_exception=0 } + ' "${ISSUE_BODY_FILE}" > "${ISSUE_BODY_FILE}.tmp" \ + && mv "${ISSUE_BODY_FILE}.tmp" "${ISSUE_BODY_FILE}" ``` **What each transform removes:** @@ -215,6 +242,7 @@ provided (adopting an existing issue) or when `--dry-run` is active. - `| **Contested:** N` and `| **Exception warranted:** N` — count segments from the `**Findings processed:**` summary line; leaves `**Findings processed:** {total} | **Valid:** {N}` - `## Findings with Exceptions` through next `---` — the entire exception-findings section + - `**Exception note:**` blocks — exception-warranted finding detail sections 3. Call `gh issue create` using the temp file: @@ -253,18 +281,30 @@ Use the fetched data as the issue context. ### Step 6: LLM Classification -Analyze the issue title + body using in-context reasoning: +**Shortcut — Validated audit report:** When `is_validated_report = true` (set in Step 1b), +immediately assign: +- `route = implementation` +- `issue_type = enhancement` +- `confidence = high` +- `rationale = "Validated audit report — structural/quality improvement work, not broken behavior"` + +Skip the heuristic table below and proceed directly to Step 7. + +Analyze the issue title + body using in-context reasoning for all other inputs: | Signal | Route | Issue Type | |--------|-------|------------| | Existing behavior broken / error traceback present | `remediation` | `bug` | | New feature / enhancement with clear acceptance criteria | `implementation` | `enhancement` | | "X doesn't support Y" / clearly absent feature | `implementation` | `enhancement` | +| Validated audit report with structural/quality findings (finding IDs present, no tracebacks, no crashes) | `implementation` | `enhancement` | | Large/ambiguous scope / unclear root cause | `remediation` | `enhancement` | Record: `route` (implementation|remediation), `issue_type` (bug|enhancement), `confidence` (high|low), `rationale` (one sentence). +Note: The audit-signal table row is a secondary defense for `--issue N` adoption of pre-existing validated audit issues where `is_validated_report` may not be set. + ### Step 7: Confidence Gate If `confidence == "low"`: @@ -342,20 +382,26 @@ Otherwise: gh label create "recipe:implementation" --force \ --description "Route: proceed directly to implementation" \ --color "0E8A16" +sleep 1 # Rate-limit discipline: 1s between mutating calls gh label create "recipe:remediation" --force \ --description "Route: investigate/decompose before implementation" \ --color "D93F0B" +sleep 1 # Rate-limit discipline: 1s between mutating calls gh label create "bug" --force \ --description "Existing behavior is broken" \ --color "d73a4a" +sleep 1 # Rate-limit discipline: 1s between mutating calls gh label create "enhancement" --force \ --description "New feature or request" \ --color "a2eeef" +sleep 1 # Rate-limit discipline: 1s between mutating calls # Apply triage labels (use the route determined in Step 6) gh issue edit {issue_number} --add-label "recipe:implementation" +sleep 1 # Rate-limit discipline: 1s between mutating calls # or, for remediation route: gh issue edit {issue_number} --add-label "recipe:remediation" +sleep 1 # Rate-limit discipline: 1s between mutating calls gh issue edit {issue_number} --add-label "{issue_type}" ``` @@ -401,3 +447,9 @@ Emit to stdout for recipe capture: } ---/prepare-issue-result--- ``` + +Also emit the issue URL as a standalone structured token for recipe capture: + +``` +issue_url = https://github.com/owner/repo/issues/N +``` diff --git a/src/autoskillit/skills_extended/prepare-pr/SKILL.md b/src/autoskillit/skills_extended/prepare-pr/SKILL.md index 6ee1c43dd0..123ebe6975 100644 --- a/src/autoskillit/skills_extended/prepare-pr/SKILL.md +++ b/src/autoskillit/skills_extended/prepare-pr/SKILL.md @@ -1,10 +1,7 @@ --- name: prepare-pr categories: [github] -description: > - Reads plan(s), runs git diff, classifies changed files, selects 1–3 arch-lens slugs, - writes one context file per lens, and writes a PR prep file. Does NOT invoke arch-lens - skills. Part 1 of 3 in the decomposed PR flow. +description: Preparation executor for pull-request metadata. ALWAYS invoke this skill when instructed to prepare PR metadata. Do not read plans or classify files directly — use this skill first to load the preparation workflow. hooks: PreToolUse: - matcher: "*" @@ -39,6 +36,7 @@ in the decomposed PR flow (prepare → run_arch_lenses → compose). - Invoke arch-lens skills or any other sub-skills - Create files outside `{{AUTOSKILLIT_TEMP}}/prepare-pr/` - Fail if closing_issue is absent or gh is unavailable — degrade gracefully +- Run subagents in the background (`run_in_background: true` is prohibited) **ALWAYS:** - Emit all three output tokens (`prep_path`, `selected_lenses`, `lens_context_paths`) @@ -149,6 +147,8 @@ Output: comma-separated slug list → `selected_lens_slugs`. ### Step 6: Write Context Files per Lens +Use the Write tool (not Bash heredocs or cat) to create each context file. + For each selected slug, write one context file to `{{AUTOSKILLIT_TEMP}}/prepare-pr/pr_arch_lens_context_{slug}_{ts}.md`: @@ -180,6 +180,8 @@ Read `## Summary` from each plan file. Store plan summaries for the prep file. ### Step 8: Write PR Prep File +Use the Write tool (not Bash heredocs or cat) to create the prep file. + Write PR prep file to `{{AUTOSKILLIT_TEMP}}/prepare-pr/pr_prep_{ts}.md`: ```markdown diff --git a/src/autoskillit/skills_extended/prepare-research-pr/SKILL.md b/src/autoskillit/skills_extended/prepare-research-pr/SKILL.md index d74a4354b4..0552f55879 100644 --- a/src/autoskillit/skills_extended/prepare-research-pr/SKILL.md +++ b/src/autoskillit/skills_extended/prepare-research-pr/SKILL.md @@ -36,6 +36,7 @@ Does NOT invoke any exp-lens skills or create a PR. - Invoke exp-lens skills — they are run in separate sessions by the recipe orchestrator - Create files outside `{{AUTOSKILLIT_TEMP}}/prepare-research-pr/` (relative to the current working directory) - Fail silently — always emit all three output tokens as your final output +- Run subagents in the background (`run_in_background: true` is prohibited) **ALWAYS:** - Use Agent subagents (not slash commands) for reading and synthesis diff --git a/src/autoskillit/skills_extended/process-issues/SKILL.md b/src/autoskillit/skills_extended/process-issues/SKILL.md index c45e37594e..c02ce2fee9 100644 --- a/src/autoskillit/skills_extended/process-issues/SKILL.md +++ b/src/autoskillit/skills_extended/process-issues/SKILL.md @@ -32,10 +32,25 @@ issues upfront, load recipe, execute session, collect result, report. - Modify any source code files - Reimplement recipe steps inline — always use `load_recipe` to load the recipe YAML and follow it as an orchestrator +- Run subagents in the background (`run_in_background: true` is prohibited) **ALWAYS:** - Process batches in ascending order: batch 1 before batch 2 before batch 3 - Use `load_recipe` to execute the recipe for each issue +- After loading a recipe via `load_recipe`, execute every step in the recipe's step + graph in sequence. Never skip, replace, or improvise steps. +- `optional: true` means the step is skipped ONLY when its `skip_when_false` ingredient + evaluates to false. When the ingredient is true, the step is mandatory. +- Follow `on_success`, `on_failure`, `on_result`, and `on_context_limit` routing exactly + as declared in the recipe YAML. +- NEVER replace recipe PR steps (`prepare_pr`, `run_arch_lenses`, `compose_pr`, + `annotate_pr_diff`, `review_pr`) with manual `run_cmd` calls such as `gh pr create`. +- Between issues: immediately begin step 1 for the next issue after completing the + previous issue. Do not output prose status between issues — inter-issue text creates + end_turn windows that cause stochastic session termination. +- NEVER use AskUserQuestion to confirm proceeding to the next issue or the next batch. + Once Step 2a's initial confirmation gate passes, all subsequent issue and batch + transitions are fully automated. - Emit `---process-issues-result---` result block on completion (success or failure) - Write the summary report to `{{AUTOSKILLIT_TEMP}}/process-issues/` (relative to the current working directory) - Use `model: "sonnet"` when spawning subagents via the Task tool @@ -47,7 +62,7 @@ issues upfront, load recipe, execute session, collect result, report. - Positional (optional): path to triage manifest JSON - `--batch N` — only process batch N (default: process all batches in order) - `--dry-run` — print the processing plan and exit without launching any recipe sessions -- `--comment` — post a GitHub comment on each issue at pickup and at completion +- `--status-updates` — append body sections to each issue at pickup and at completion - `--merge-batch` — after each batch completes, run `analyze-prs` + `merge-pr` to merge the batch PRs into the integration branch before starting the next batch @@ -59,7 +74,7 @@ Parse arguments: - If a positional path is given, use it as the manifest path. - `--batch N`: record the target batch number; process only that batch. - `--dry-run`: set dry_run flag; print plan then exit after Step 2. -- `--comment`: set comment flag. +- `--status-updates`: set status_updates flag. - `--merge-batch`: set merge_batch flag. ### Step 1: Locate and Read Manifest @@ -147,7 +162,7 @@ completed_urls = [] # issues whose recipe fully returned Collect all issues from all batches that will be processed (respecting `--batch N` filtering). For each issue in the collected list: -1. Call `claim_issue(issue_url=)` — **no** `allow_reentry` (default `False`) +1. Call `claim_issue(issue_url=, label="queued")` — **no** `allow_reentry` (default `False`) 2. If `result.claimed == true`: - append `issue_url` to `pre_claimed_urls` 3. If `result.claimed == false`: @@ -177,13 +192,30 @@ Processing X issues: 1. **Check pre_claimed_urls:** If `issue_url` is NOT in `pre_claimed_urls` → skip (excluded by upfront claim phase — another session holds it). -2. **Optionally post pickup comment** (if `--comment` is active): +2. **Pre-dispatch check:** Before executing the recipe for this issue, check if any + previously completed issue in the CURRENT batch has `status: failure`. If so, skip + this issue with `status: skipped` and continue to the next issue in the batch. + This prevents wasted compute on issues within the same batch after a failure. + Skipped issues are recorded as `status: skipped` (not `status: failure`), so the + batch failure gate below counts only the original failure(s) that triggered the skip. + +3. **Promote queued → in-progress for this issue:** + Call `claim_issue(issue_url=, allow_reentry=true)` + (No label argument — defaults to in-progress; removes the "queued" label + and adds "in-progress".) + +4. **Optionally append pickup status to issue body** (if `--status-updates` is active): ```bash - gh issue comment {number} \ - --body "Processing in batch {N} — recipe: \`{recipe}\`" + PROCESS_BODY_FILE="{{AUTOSKILLIT_TEMP}}/process-issues/status_{number}_$(date +%s).md" + mkdir -p "$(dirname "$PROCESS_BODY_FILE")" + gh issue view {number} --json body --jq '.body' > "$PROCESS_BODY_FILE" + printf '\n\n---\n\n## In Progress\n\nProcessing in batch %s — recipe: `%s`\n' \ + "{N}" "{recipe}" >> "$PROCESS_BODY_FILE" + gh issue edit {number} --body-file "$PROCESS_BODY_FILE" + sleep 1 ``` -3. **Determine recipe name and `run_name`:** +4. **Determine recipe name and `run_name`:** | `recipe` field | Recipe to load | `run_name` | PR Title Prefix | |---------------|----------------|------------|-----------------| @@ -196,7 +228,7 @@ Processing X issues: The `run_name` encodes recipe origin for the `open-pr` skill, which derives the PR title prefix from it by convention (see `open-pr` SKILL.md). -4. **Load the recipe:** +5. **Load the recipe:** ``` load_recipe("{recipe_name}") ``` @@ -204,12 +236,12 @@ Processing X issues: follow each step in the recipe, calling the specified MCP tool with the specified `with:` arguments. -5. **Execute the recipe** with these ingredient values: +6. **Execute the recipe** with these ingredient values: - `task`: the issue title (the recipe's `make-plan` step detects `issue_url` and fetches full content internally) - `issue_url`: the constructed issue URL - `run_name`: `"feature"` (implementation) or `"fix"` (remediation) - - `base_branch`: `"integration"` (or read `git rev-parse --abbrev-ref HEAD`) + - `base_branch`: `"develop"` (or read `git rev-parse --abbrev-ref HEAD`) - `open_pr`: `"true"` - `audit`: `"true"` - `review_approach`: `"false"` @@ -219,7 +251,7 @@ Processing X issues: `upfront_claimed` ingredient) and recognize the pre-existing label as a valid reentry, returning `claimed=true` to proceed normally. -6. **After recipe returns** (any outcome), append to completed_urls: +7. **After recipe returns** (any outcome), append to completed_urls: ``` completed_urls.append(issue_url) ``` @@ -227,9 +259,17 @@ Processing X issues: - On success path (`done` step reached): `{issue_number, recipe, status: success, pr_url}` - On failure path (`escalate_stop` reached): `{issue_number, recipe, status: failure, error}` -7. **Optionally post completion comment** (if `--comment` is active): - - Success: `"✅ Processing complete — PR: {pr_url}"` - - Failure: `"❌ Processing failed — manual intervention required"` +8. **Optionally append completion status to issue body** (if `--status-updates` is active): + ```bash + PROCESS_BODY_FILE="{{AUTOSKILLIT_TEMP}}/process-issues/status_{number}_$(date +%s).md" + mkdir -p "$(dirname "$PROCESS_BODY_FILE")" + gh issue view {number} --json body --jq '.body' > "$PROCESS_BODY_FILE" + printf '\n\n---\n\n## Status\n\n%s\n' \ + "{✅ Processing complete — PR: $pr_url | ❌ Processing failed — manual intervention required}" \ + >> "$PROCESS_BODY_FILE" + gh issue edit {number} --body-file "$PROCESS_BODY_FILE" + sleep 1 + ``` **Fatal failure cleanup** — if any unrecoverable error occurs during recipe dispatch: ``` @@ -240,6 +280,15 @@ Log: "Released N upfront-claimed issues due to fatal failure" Propagate the error ``` +**Batch failure gate (mandatory):** After all issues in the current batch have been +processed, count the number of issues with `status: failure`. If ANY issue in this +batch has `status: failure`: + 1. Log: "Batch {N} had {count} failure(s). Halting batch processing." + 2. Skip all remaining batches. + 3. Proceed directly to Step 4 (Summary Report), recording unprocessed issues + from remaining batches as `status: skipped`. + 4. Call `release_issue()` for each pre-claimed but unprocessed issue URL. + **3c. After all issues in batch complete** (if `--merge-batch` is active): Run the analyze-prs → merge-pr cycle for the batch's PRs: @@ -335,6 +384,12 @@ Print the structured result for pipeline capture: ---end-process-issues-result--- ``` +Also emit the report path as a standalone structured token for recipe capture: + +``` +dispatch_results = {{AUTOSKILLIT_TEMP}}/process-issues/process_report_{ts}.md +``` + ## Output Location ``` diff --git a/src/autoskillit/skills_extended/promote-to-main/SKILL.md b/src/autoskillit/skills_extended/promote-to-main/SKILL.md new file mode 100644 index 0000000000..2b72da3dcb --- /dev/null +++ b/src/autoskillit/skills_extended/promote-to-main/SKILL.md @@ -0,0 +1,472 @@ +--- +name: promote-to-main +categories: [github] +description: > + Promote integration to main with comprehensive changelog and PR creation. Use when + user says "promote to main", "open promotion PR", "integration to main", or + "create release PR to main". Runs pre-flight checks, change inventory, architecture + diagrams, and creates a rich PR with release notes and traceability. +--- + +# Promote to Main + +Orchestrate the integration-to-main promotion workflow. This skill discovers everything +that changed on integration since it diverged from main, runs pre-flight quality checks, +performs change inventory, generates architecture diagrams, synthesizes release notes, +and creates a comprehensive promotion PR. + +## Arguments + +``` +/autoskillit:promote-to-main [batch_branch] [base_branch] [--dry-run] +``` + +- `batch_branch` (optional) — source branch to promote. Defaults to `develop`. +- `base_branch` (optional) — target branch. Defaults to `main`. +- `--dry-run` — generate the full PR body without creating a PR. + +## When to Use + +- When the develop branch is stable and ready to be promoted to main +- After feature PRs, bug fixes, and cleanup work have been collected and tested on develop +- This is the final gate before changes land on main + +## Critical Constraints + +**NEVER:** +- Create files outside `{{AUTOSKILLIT_TEMP}}/promote-to-main/` +- Modify any source code — this skill is read-only analysis + PR creation +- Fail silently when `gh` is unavailable — output `pr_url = ` (empty) and exit successfully +- Push or merge anything — this skill only creates the PR +- Skip pre-flight checks — a failing pre-flight must block PR creation +- Use the Bash tool for file reads — use Read, Grep, Glob for all codebase inspection +- Use `gh pr create --body` inline — always use `--body-file` +- Run subagents in the background (`run_in_background: true` is prohibited) + +**ALWAYS:** +- Run ALL pre-flight checks before any analysis work +- Check `gh auth status` before any GitHub operations +- Output `pr_url = ` as a structured token (empty string when GitHub unavailable or dry-run) +- Output `verdict = ` as a structured token +- Carry forward ALL `Closes #N`, `Fixes #N`, and `Resolves #N` references from merged PR bodies +- Use `gh pr create --body-file` (never inline body via `--body`) +- `pr_body_path` must be an absolute path (prepend CWD) + +## Context Limit Behavior + +When context is exhausted mid-execution, temp files may be written but the PR may +not yet be created. The recipe routes to `on_context_limit` (typically `escalate_stop`), +bypassing the normal completion protocol. + +**Before emitting structured output tokens:** +1. Emit `pr_url = ` (empty) if the PR was not successfully created +2. Emit `verdict = preflight_failed` to signal an incomplete run +3. Emit whatever was completed; the orchestrator handles the context-limit route + +## Workflow + +### Phase 0: Setup + +#### Step 0.1: Parse Arguments + +Parse optional positional arguments and flags: +- `batch_branch` — default `"develop"` if absent or empty +- `base_branch` — default `"main"` if absent or empty +- `dry_run` — `true` if `--dry-run` present in ARGUMENTS + +Validate that both branches exist locally: +```bash +git rev-parse --verify {batch_branch} 2>/dev/null +git rev-parse --verify {base_branch} 2>/dev/null +``` +If either fails, try fetching: +```bash +git fetch origin {batch_branch}:{batch_branch} 2>/dev/null +``` +If still missing, print error to stderr and exit 1. + +#### Step 0.2: Compute Divergence Point + +```bash +git merge-base {base_branch} {batch_branch} +``` + +Store as `$MERGE_BASE_SHA`. + +Get commit count and timestamp: +```bash +git rev-list --count $MERGE_BASE_SHA..{batch_branch} +git show -s --format=%cI $MERGE_BASE_SHA +``` +Store as `$COMMIT_COUNT` and `$MERGE_BASE_DATE`. + +#### Step 0.3: Retrieve Token Summary from Session Logs + +Determine the pipeline working directory and self-retrieve token telemetry from disk. +This aggregates token usage across all constituent PR sessions that ran in this pipeline +working directory. + +```bash +mkdir -p {{AUTOSKILLIT_TEMP}}/promote-to-main +python3 - <<'EOF' > {{AUTOSKILLIT_TEMP}}/promote-to-main/token_summary.md 2>/dev/null || true +import json, pathlib, sys +from autoskillit.pipeline.tokens import DefaultTokenLog +from autoskillit.pipeline.telemetry_fmt import TelemetryFormatter +from autoskillit.execution.session_log import resolve_log_dir + +cfg_path = pathlib.Path(".autoskillit") / "temp" / ".hook_config.json" +kitchen_id = "" +if cfg_path.exists(): + _cfg = json.loads(cfg_path.read_text()) + if isinstance(_cfg, dict): + kitchen_id = _cfg.get("kitchen_id") or _cfg.get("pipeline_id", "") + +log_root = resolve_log_dir("") +tl = DefaultTokenLog() +n = tl.load_from_log_dir(log_root, kitchen_id_filter=kitchen_id) +if n == 0: + sys.exit(0) +steps = tl.get_report() +total = tl.compute_total() +print(TelemetryFormatter.format_token_table(steps, total)) +EOF +``` + +- If `{{AUTOSKILLIT_TEMP}}/promote-to-main/token_summary.md` is non-empty, set `TOKEN_SUMMARY_CONTENT` to its + contents and embed it in the PR body under `## Token Usage Summary`. +- If empty or absent (standalone invocation, no pipeline sessions in this cwd), omit this section. + +### Phase 1: Pre-flight Checks (parallel, blocking) + +Spawn three parallel Task subagents (model: sonnet) to validate promotion readiness. +All three must pass before analysis proceeds. If any fails, report the failure clearly +and exit 1. Do NOT create a PR when pre-flight fails. + +#### Subagent 1A: CI and Branch Status + +Check: +1. CI is green on the integration branch — run `gh pr checks` for any open PR from + integration, or `gh run list --branch {batch_branch} --workflow tests.yml --limit 1 --json conclusion` +2. The integration branch is not behind base — run `git rev-list --count {batch_branch}..{base_branch}` + to check if base has commits not in integration (if > 0, warn that a rebase may be needed) +3. No open PRs targeting integration with failing CI — `gh pr list --base {batch_branch} --state open --json number,title,statusCheckRollup` + +Return JSON: +```json +{ + "ci_status": "pass|fail|unknown", + "ci_details": "description of CI state", + "behind_base_by": 0, + "open_prs_with_failing_ci": [], + "pass": true +} +``` + +#### Subagent 1B: Version Consistency + +Check: +1. `pyproject.toml` version matches `src/autoskillit/.claude-plugin/plugin.json` version +2. `uv lock --check` passes (lockfile consistent) +3. The integration branch version is ahead of the base branch version + +Return JSON: +```json +{ + "pyproject_version": "X.Y.Z", + "plugin_version": "X.Y.Z", + "versions_match": true, + "lockfile_consistent": true, + "version_ahead_of_base": true, + "pass": true +} +``` + +#### Subagent 1C: Outstanding Review Items + +Check: +1. No open PRs targeting integration with `CHANGES_REQUESTED` reviews — + `gh pr list --base {batch_branch} --state open --json number,title,reviews` +2. No `in-progress` labeled issues that might indicate incomplete work — + `gh issue list --label in-progress --state open --json number,title` + +Return JSON: +```json +{ + "prs_with_changes_requested": [], + "in_progress_issues": [], + "pass": true +} +``` + +**Pre-flight gate:** If any subagent returns `"pass": false`, report all failures in a +clear summary table and exit 1 without proceeding to analysis. If `ci_status` is `unknown`, +treat as a warning (non-blocking) and note it in the report. + +### Phase 2: Change Inventory (parallel subagents) + +Spawn four parallel Task subagents (model: sonnet). + +#### Subagent 2A: Commit Categorization + +Receive the output of: +```bash +git log $MERGE_BASE_SHA..{batch_branch} --format="%H %s" +``` + +Categorize each commit into exactly one category based on its subject line: +- `rectify` — subject contains "Rectify:" or "fix:" or "bugfix" (case-insensitive) +- `feature` — subject contains "Implementation Plan:", "feat:", "Add ", or introduces new capability +- `infra` — subject contains CI, workflow, config, build, or infrastructure changes +- `test` — subject only touches test files (infer from "test" in subject or `tests/` paths) +- `docs` — subject contains "docs:", "README", or documentation-only changes + +Extract PR numbers from patterns like `(#123)` in commit subjects. + +Return JSON: +```json +{ + "categories": { + "rectify": [{"sha": "abc123", "title": "...", "pr_number": 495}], + "feature": [], + "infra": [], + "test": [], + "docs": [] + }, + "totals": {"rectify": 14, "feature": 13, "infra": 3, "test": 1, "docs": 1}, + "category_summary": "14 fixes, 13 features, 3 infra" +} +``` + +#### Subagent 2B: PR Discovery and Issue Linkage + +Run: +```bash +gh pr list --base {batch_branch} --state merged --limit 200 --json number,title,author,mergedAt,body,headRefName,additions,deletions,labels,url +``` + +Filter to PRs merged after `$MERGE_BASE_DATE`. If empty, fall back to commit-subject +discovery: +```bash +git log $MERGE_BASE_SHA..{batch_branch} --oneline --grep="(#" --format="%s" +``` + +For each PR, extract `Closes|Fixes|Resolves #N` references (case-insensitive). +Deduplicate across all PRs. For each linked issue number, fetch details: +```bash +gh issue view $ISSUE_NUMBER --json number,title,state,url,labels 2>/dev/null +``` + +Build a traceability matrix: for each issue, identify which PR(s) close it. + +Return JSON: +```json +{ + "prs": [{"number": 1, "title": "...", "author": "...", "labels": [], "url": "...", "additions": 0, "deletions": 0}], + "closing_refs": ["Closes #42", "Fixes #50"], + "linked_issue_numbers": [42, 50], + "issue_details": [{"number": 42, "title": "...", "state": "OPEN", "url": "...", "labels": ["recipe:implementation"]}], + "traceability": [{"issue_number": 42, "issue_title": "...", "pr_numbers": [491], "recipe_route": "implementation"}] +} +``` + +#### Subagent 2C: File Lifecycle Tracking + +Run: +```bash +git diff --name-only {base_branch}..{batch_branch} +git diff --diff-filter=A --name-only {base_branch}..{batch_branch} +git diff --diff-filter=M --name-only {base_branch}..{batch_branch} +git diff --diff-filter=D --name-only {base_branch}..{batch_branch} +git diff --diff-filter=R --name-only {base_branch}..{batch_branch} +git diff --stat {base_branch}..{batch_branch} | tail -1 +``` + +Return JSON: +```json +{ + "changed_files": ["..."], + "new_files": ["..."], + "modified_files": ["..."], + "deleted_files": ["..."], + "renamed_files": ["..."], + "diff_stat_summary": "185 files changed, 10013 insertions(+), 1164 deletions(-)" +} +``` + +#### Subagent 2D: Migration and Schema Detection + +Scan the diff for changes that require manual attention on merge. Check: +1. Changes to `src/autoskillit/migrations/` — new migration YAML notes +2. Changes to `src/autoskillit/recipe/schema.py` — recipe schema modifications +3. Changes to `pyproject.toml` — dependency additions/removals/version bumps +4. Changes to `src/autoskillit/config/defaults.yaml` — config schema changes +5. Changes to `src/autoskillit/hooks/hooks.json` — hook registration changes +6. Changes to `src/autoskillit/.claude-plugin/plugin.json` — plugin metadata changes +7. Changes to `.github/workflows/` — CI workflow modifications + +For each detected change, provide a brief description of what changed and why it +might need attention. + +Return JSON: +```json +{ + "migration_changes": ["Added migration note for v0.3.2"], + "schema_changes": ["New RecipeStep field: optional"], + "dependency_changes": ["Added httpx>=0.27"], + "config_changes": ["New config key: github.staged_label"], + "hook_changes": ["New hook: pretty_output PostToolUse"], + "ci_changes": ["Updated tests.yml matrix"], + "attention_required": true, + "attention_summary": "Brief description of what needs human review" +} +``` + +### Phase 3: Architecture Diagrams + +Architecture diagrams are generated via arch-lens skills. +Using ONLY classDef styles from the mermaid skill (no invented colors). + +#### Step 3.1: Select Arch-Lens Lenses + +Spawn a Task subagent (model: sonnet) with the `changed_files` list and this lens menu: + +``` +c4-container, concurrency, data-lineage, deployment, development, +error-resilience, module-dependency, operational, process-flow, +repository-access, scenarios, security, state-lifecycle +``` + +Return 1-3 lens names. Apply the same selection criteria as open-pr: + +**Development lens guard:** Only select `development` if at least one changed file matches: +`pyproject.toml`, `Taskfile*`, `conftest.py`, `.github/workflows/*`, `Makefile`, +`setup.cfg`, `setup.py`, `tox.ini`, `noxfile.py`, or files under `ci/`. + +For a promotion PR, prefer lenses that show the broadest architectural impact: +- `module-dependency` if changes span multiple packages +- `process-flow` if workflow routing or state transitions changed +- `c4-container` if new services, tools, or integrations were added + +#### Step 3.2: Generate Arch-Lens Diagrams + +For each selected lens, follow this exact sequence: + +**CRITICAL:** Do NOT output any prose status text between lens iterations. +After completing all sub-steps for one lens, immediately begin sub-step 1 for the +next lens. + +**1. Write the PR context to a file using the Write tool:** + +- **Path:** `{{AUTOSKILLIT_TEMP}}/promote-to-main/pr_arch_lens_context_{YYYY-MM-DD_HHMMSS}.md` +- **Content:** + +```markdown +# PR Context — Integration to Main Promotion + +This diagram is for a promotion PR merging the integration branch into main. Focus on the areas of the codebase affected by all accumulated changes. Do not create a generic whole-project diagram. + +## New files (use star prefix on these nodes): +{list of new_files, or "None"} + +## Modified files (use bullet prefix on these nodes): +{list of modified_files, or "None"} + +## Deleted files: +{list of deleted_files, or "None"} + +## Instructions: +- Focus exploration and the diagram on the architectural areas these files belong to +- Use star prefix on nodes representing new files/components +- Use bullet prefix on nodes representing modified files/components +- Mark deleted components with strikethrough or a X prefix +- Leave unchanged components unmarked (include only if needed for context/connectivity) +- This is a promotion PR — show the cumulative architectural impact of all changes +``` + +**2. Immediately call the Skill tool** to load the arch-lens skill (e.g., +`/autoskillit:arch-lens-module-dependency`). + +**3. Follow the loaded skill's instructions** to generate the diagram. + +Read the output from `{{AUTOSKILLIT_TEMP}}/arch-lens-{lens-name}/` and extract the mermaid block(s). + +Validate: if the block contains at least one star marker or bullet marker for +new/modified nodes, add to `validated_diagrams`. Otherwise discard. + +### Phase 4: Compose PR + Create PR + +#### Step 4.1: Release Notes Synthesis + +Spawn one Task subagent (model: sonnet) with results from Phase 2 ONLY (no domain +analysis or quality assessment data). + +The subagent receives: +- Commit categorization from Subagent 2A +- PR and issue data from Subagent 2B +- File lifecycle from Subagent 2C +- Migration/schema detection from Subagent 2D + +Return JSON: +```json +{ + "executive_summary": "3-5 sentence narrative of what this promotion brings, focused on change themes", + "highlights": ["top 3-5 most significant changes"], + "release_notes_md": "### New Features\n- ...\n### Bug Fixes\n- ...\n### Infrastructure\n...\n### Breaking Changes\n...\n### Attention Required\n..." +} +``` + +#### Step 4.2: Write PR Body + +Write to `{{AUTOSKILLIT_TEMP}}/promote-to-main/pr_body_{YYYY-MM-DD_HHMMSS}.md` +(relative to the current working directory). + +Sections in order: +1. `## Promotion: {batch_branch} to {base_branch}` — executive summary + stats (`diff_stat_summary`, `$COMMIT_COUNT`, PR count) +2. `### Highlights` — from synthesis +3. `## Release Notes` — from synthesis +4. `## Merged PRs` — table (PR, Title, Author, Labels) from Subagent 2B +5. `## Linked Issues` — table (Issue, Title, Status, Action) from Subagent 2B +6. `## Attention Required` — from Subagent 2D (only if `attention_required=true`) +7. `## Architecture Impact` — validated mermaid diagrams from Phase 3 +8. Closing references (`Closes #N` lines from Subagent 2B, one per line) +9. `## Token Usage Summary` — if TOKEN_SUMMARY_CONTENT non-empty +10. Footer: `Generated with Claude Code via AutoSkillit` + +#### Step 4.3: Create PR + +If `dry_run` is true: skip to Output section. + +Check GitHub availability: +```bash +gh auth status 2>/dev/null +``` +If exit code non-zero: output `pr_url = ` and exit successfully. + +Construct PR title using actual branch names: +`Promote {batch_branch} to {base_branch} ($PR_COUNT PRs, $ISSUE_COUNT issues, $CATEGORY_SUMMARY)` + +```bash +gh pr create \ + --base {base_branch} \ + --head {batch_branch} \ + --title "$PR_TITLE" \ + --body-file {{AUTOSKILLIT_TEMP}}/promote-to-main/pr_body_$TIMESTAMP.md +``` + +Capture the PR URL as `$PR_URL`. + +Add label (optional, continue if fails): +```bash +gh pr edit $PR_URL --add-label "promotion" 2>/dev/null || true +``` + +### Output + +Always emit these structured output tokens as the final lines: + +``` +pr_body_path = {absolute path to {{AUTOSKILLIT_TEMP}}/promote-to-main/pr_body_{timestamp}.md} +pr_url = {pr_url, empty on dry-run or when GitHub CLI is unreachable} +verdict = {created|dry_run|preflight_failed} +category_summary = {e.g., "14 fixes, 13 features, 3 infra"} +``` diff --git a/src/autoskillit/skills_extended/rectify/SKILL.md b/src/autoskillit/skills_extended/rectify/SKILL.md index 09d42bdc69..6e68c7f140 100644 --- a/src/autoskillit/skills_extended/rectify/SKILL.md +++ b/src/autoskillit/skills_extended/rectify/SKILL.md @@ -38,6 +38,7 @@ Do not change any code. - Propose bandaid fixes, fallbacks, or direct-only fixes - Suggest backward compatibility shims - Create files outside `{{AUTOSKILLIT_TEMP}}/rectify/` directory +- Run subagents in the background (`run_in_background: true` is prohibited) **ALWAYS:** - Use subagents for parallel exploration @@ -71,30 +72,6 @@ exist (e.g., plan file arguments, `{{AUTOSKILLIT_TEMP}}/investigate/` reports, e `Glob` or `ls` to confirm the path exists first. This prevents ENOENT errors that cascade into sibling parallel-call cancellations. -### Step 1.5 — Code-Index Initialization (required before any code-index tool call) - -Call `set_project_path` with the repo root where this skill was invoked (not a worktree path): - -``` -mcp__code-index__set_project_path(path="{PROJECT_ROOT}") -``` - -Code-index tools require **project-relative paths**. Always use paths like: - - src//some_module.py - -NOT absolute paths like: - - /absolute/path/to/src//some_module.py - -> **Note:** Code-index tools (`find_files`, `search_code_advanced`, `get_file_summary`, -> `get_symbol_body`) are only available when the `code-index` MCP server is configured. -> If `set_project_path` returns an error, fall back to native `Glob` and `Grep` tools -> for the same searches — they provide equivalent results without the code-index server. - -Agents launched via `run_skill` inherit no code-index state from the parent session — this -call is mandatory at the start of every headless session that uses code-index tools. - ### Step 2: Deep Exploration with Subagents Launch parallel subagents to investigate (some of the listed aspects may require multiple subagents): @@ -173,7 +150,7 @@ After finalizing the plan, determine which architecture lens best illustrates th | State Lifecycle | `/autoskillit:arch-lens-state-lifecycle` | | Deployment | `/autoskillit:arch-lens-deployment` | -If the Skill tool cannot be used (disable-model-invocation) or refuses this invocation, skip the diagram step and proceed without the architectural diagram. +If the Skill tool cannot be used (disable-model-invocation) or refuses this invocation, omit the diagram and proceed without the architectural diagram. **4d. Create the diagram following the loaded skill's instructions:** - Focus on the PROPOSED changes (use `newComponent` class for new elements) diff --git a/src/autoskillit/skills_extended/reload-session/SKILL.md b/src/autoskillit/skills_extended/reload-session/SKILL.md new file mode 100644 index 0000000000..1a09e54012 --- /dev/null +++ b/src/autoskillit/skills_extended/reload-session/SKILL.md @@ -0,0 +1,28 @@ +--- +name: reload-session +description: Reload the current AutoSkillit session — signals the parent process to re-launch with the full wrapper environment and resume the conversation. +disable-model-invocation: true +--- + +# Reload Session + +Use this skill when the kitchen is disconnected, context is exhausted, or a fresh re-launch +is needed without losing the conversation transcript. + +## Steps + +1. Call `reload_session` with no arguments. +2. Run `/exit` immediately after to allow the parent process to detect the reload request. + +The parent autoskillit process will re-launch claude with `--resume ` and +the original orchestrator environment (system prompt, tools, session type env vars) fully +restored. + +## Critical Constraints + +**NEVER:** +- Skip running `/exit` after `reload_session` — the reload does not happen until claude exits +- Call this skill from a headless or automated session + +**ALWAYS:** +- Call `reload_session` first, then immediately run `/exit` diff --git a/src/autoskillit/skills_extended/resolve-claims-review/SKILL.md b/src/autoskillit/skills_extended/resolve-claims-review/SKILL.md index 080f988b2d..914bdd2d3e 100644 --- a/src/autoskillit/skills_extended/resolve-claims-review/SKILL.md +++ b/src/autoskillit/skills_extended/resolve-claims-review/SKILL.md @@ -45,6 +45,7 @@ Called by the research recipe when `audit_claims` routes `changes_requested` via - Delete or discard the working directory on failure - Modify tests to suppress failures introduced by reviewer fixes - Use file-path-segment grouping — claim comments are grouped by **dimension**, not by file path +- Run subagents in the background (`run_in_background: true` is prohibited) **ALWAYS:** - Find the PR by feature branch at invocation time (not a hardcoded number) @@ -96,25 +97,6 @@ validation_timeout = cr_cfg.get("validation_timeout", 120) If either positional arg is missing, abort with: `"Usage: /autoskillit:resolve-claims-review "` -### Step 0.5 — Code-Index Initialization (required before any code-index tool call) - -Call `set_project_path` with the repo root where this skill was invoked (not a worktree path): - -``` -mcp__code-index__set_project_path(path="{PROJECT_ROOT}") -``` - -Code-index tools require **project-relative paths**. Always use paths like: - - src/mypackage/core/module.py - -NOT absolute paths like: - - /path/to/project/src/mypackage/core/module.py - -Agents launched via `run_skill` inherit no code-index state from the parent session — this -call is mandatory at the start of every headless session that uses code-index tools. - ### Step 1: Find the Open PR ```bash diff --git a/src/autoskillit/skills_extended/resolve-design-review/SKILL.md b/src/autoskillit/skills_extended/resolve-design-review/SKILL.md index ebdfb8ca9c..86e8d236e6 100644 --- a/src/autoskillit/skills_extended/resolve-design-review/SKILL.md +++ b/src/autoskillit/skills_extended/resolve-design-review/SKILL.md @@ -38,6 +38,7 @@ MCP-only — not user-invocable directly. - Create files outside `{{AUTOSKILLIT_TEMP}}/resolve-design-review/` - Modify the evaluation dashboard, experiment plan, or any source file - Apply fixes — this skill triages fixability only +- Run subagents in the background (`run_in_background: true` is prohibited) **ALWAYS:** - Exit 0 in all cases — resolution=revised and resolution=failed are both normal outcomes @@ -60,8 +61,8 @@ abandoning the partial triage. 1. Create `{{AUTOSKILLIT_TEMP}}/resolve-design-review/` if absent 2. Parse two positional path arguments: `evaluation_dashboard_path`, `experiment_plan_path` - - If missing: print `"Error: missing required argument(s) — expected "`, then emit `resolution=failed`, exit 0 - - If file not found: print `"Error: file not found — {missing_path}"`, then emit `resolution=failed`, exit 0 + - If missing: print `"Error: missing required argument(s) — expected "`, then emit `resolution=failed`, and return + - If file not found: print `"Error: file not found — {missing_path}"`, then emit `resolution=failed`, and return 3. Parse optional third argument: `prior_revision_guidance_path` - If present and file exists: read prior revision guidance for theme comparison - If absent or file not found: skip diminishing-return detection (first-round behavior) @@ -129,7 +130,7 @@ When `goalposts_moving: true`, reclassify the finding from ADDRESSABLE to STRUCT with annotation: `"reclassified: goalposts-moving (prior theme: {prior_theme_match})"`. This ensures the fix-and-review cycle terminates for concerns that are not converging. -Fallback: if no prior_revision_guidance_path is provided, skip this step entirely +Fallback: if no prior_revision_guidance_path is provided, omit this step entirely (preserves current first-round behavior unchanged). ### Step 2: Apply Resolution Logic diff --git a/src/autoskillit/skills_extended/resolve-failures/SKILL.md b/src/autoskillit/skills_extended/resolve-failures/SKILL.md index efc3e77b42..b36c120b1c 100644 --- a/src/autoskillit/skills_extended/resolve-failures/SKILL.md +++ b/src/autoskillit/skills_extended/resolve-failures/SKILL.md @@ -1,6 +1,6 @@ --- name: resolve-failures -description: Fix test failures in a worktree without merging. Leaves worktree green and unmerged for the orchestrator's merge gate. Use when tests fail after implementation. Takes worktree path, plan path, and base branch as positional arguments. +description: Failure resolution executor. ALWAYS invoke this skill when instructed to fix test failures in a worktree. Do not read test output or edit code directly — use this skill first to load the failure resolution workflow. hooks: PreToolUse: - matcher: "*" @@ -43,12 +43,30 @@ Fix test failures in a worktree implemented by `/autoskillit:implement-worktree- - Report iteration count and what was fixed - Leave worktree intact on failure for manual inspection - Treat CI as the source of truth: "passes locally" is not a resolution +- **Read before editing**: Before issuing an `Edit` call on any file, ensure you have issued a `Read` on that file earlier in this session. Claude Code rejects `Edit` on unread files — the retry wastes a full API turn at current context size. If you are uncertain whether a file was read, issue a targeted `Read` (offset + limit to the region you plan to edit) rather than risk an error. +- **CWD awareness**: Before running `python3` or other interpreters in the worktree, verify CWD is the worktree root. Use absolute paths or explicit `cd`. Wrong-CWD errors waste a full API turn at current context size. +- **Read files fully**: When reading a file to understand it in full, read it in a single call without a `limit` parameter. Do not paginate files with sequential offset reads — read once completely. Use `limit`/`offset` only for targeted section reads of files you have already read in full. + +**Flaky tests must always be resolved.** A test that failed previously and now passes +is flaky by definition. Investigate timing dependencies, race conditions, insufficient +timeouts, resource contention under parallel execution (pytest-xdist), and +non-deterministic setup/teardown. Apply a stabilizing fix. Never classify a flaky test +as unfixable. Never emit `ci_only_failure` for a test that is merely non-deterministic. + +When a test passes in isolation but fails in the full suite, check whether the test's +shared dependencies (service objects, registries, caches, middleware stacks) accumulate +state across calls — growing lists, maps, or registries rather than toggling flags, +unless accumulation is the intended behavior (e.g., event logs, audit trails). Calling +an inverse method (e.g., `disable()` to undo `enable()`) does not reset +accumulation-based state if the framework appends rather than toggles; every call to +either method appends a new entry. The fix for accumulation-based leakage is full reset: +clear the collection, then re-apply the baseline state — not inverse operations. ## Context Limit Behavior When context is exhausted mid-execution, edits may be on disk but not committed. The recipe routes to `on_context_limit` (typically `test`), bypassing the normal -commit protocol in Step 3. +commit protocol during the fix loop. **Before every test run and before emitting structured output tokens:** 1. Run `git -C {worktree_path} status --porcelain` @@ -60,7 +78,7 @@ edits are committed and the downstream merge gate receives a clean worktree. ## Workflow -Read the configured test command from `.autoskillit/config.yaml` (key: `test_check.command`). Use this command wherever `{test_command}` appears in these instructions. If no config exists, use the `test_check` MCP tool (which resolves the command from the project's config automatically). +Read the configured test command(s) from `.autoskillit/config.yaml`: check `test_check.commands` first (ordered list of commands); fall back to `test_check.command` (single command). Use the `test_check` MCP tool (which runs all configured commands automatically and returns a single pass/fail). ### Step 0: Validate Arguments 1. Parse positional args using **path detection**: scan all tokens after the @@ -93,30 +111,6 @@ Read the configured test command from `.autoskillit/config.yaml` (key: `test_che sibling parallel-call cancellations. 4. Check for development environment in worktree, recreate if missing. Use the project's configured `worktree_setup.command`, or: `cd "${worktree_path}" && task install-worktree` -### Step 0.3 — Code-Index Initialization (required before any code-index tool call) - -Call `set_project_path` with the repo root where this skill was invoked (not a worktree path): - -``` -mcp__code-index__set_project_path(path="{PROJECT_ROOT}") -``` - -Code-index tools require **project-relative paths**. Always use paths like: - - src//some_module.py - -NOT absolute paths like: - - /absolute/path/to/src//some_module.py - -> **Note:** Code-index tools (`find_files`, `search_code_advanced`, `get_file_summary`, -> `get_symbol_body`) are only available when the `code-index` MCP server is configured. -> If `set_project_path` returns an error, fall back to native `Glob` and `Grep` tools -> for the same searches — they provide equivalent results without the code-index server. - -Agents launched via `run_skill` inherit no code-index state from the parent session — this -call is mandatory at the start of every headless session that uses code-index tools. - ### Step 0.5: Commit Uncommitted Files 1. Run `git -C {worktree_path} status --porcelain` 2. If output is non-empty (dirty tree): @@ -125,20 +119,6 @@ call is mandatory at the start of every headless session that uses code-index to - Log: "Committed {N} uncommitted file(s) before test run" 3. If output is empty: continue (worktree is clean) -### Step 0.7: Switch Code-Index to Worktree - -Call `set_project_path` with the worktree path so all subsequent code-index -queries (`find_files`, `search_code_advanced`, `get_file_summary`, `get_symbol_body`) -return worktree-relative paths instead of source-repo paths: - -``` -mcp__code-index__set_project_path(path="{worktree_path}") -``` - -This prevents the model from being exposed to source-repo absolute paths during -investigation and fixing. Note: code-index tools are read-only; this switch does not -affect git operations, which always use `git -C {worktree_path}` explicitly. - ### Step 1: Understand Context 1. Read the plan file to understand what was implemented and why 2. Run `git log --oneline $(git merge-base HEAD origin/{base_branch})..HEAD` @@ -150,7 +130,7 @@ If `diagnosis_path` was provided and the file exists: 1. Open `diagnosis_path` and read its content 2. Find the "Structured Output" section or scan for the line matching `failure_subtype = {value}` 3. Extract the `failure_subtype` value (e.g., `flaky`, `deterministic`, `timing_race`, etc.) -4. Store as `{failure_subtype}` for use in Step 2d +4. Store as `{failure_subtype}` for use in the Verdict Decision Tree If `diagnosis_path` is absent or the file does not exist: - Set `{failure_subtype} = unknown` @@ -174,18 +154,33 @@ This ensures local test results match CI behavior. Skip if no pre-test generatio `test_check` blocks synchronously and returns `passed: true/false` in a single call. 2. Record the result as `{local_result}`: PASS (`passed: true`) or FAIL (`passed: false`) -### Step 2d: Verdict Decision Tree +### Step 2c: Verdict Override Rule + +**This rule takes precedence over the decision table below.** + +If at ANY point during this skill's execution a code change was committed +(i.e., the fix loop was entered and produced at least one commit) AND the final +`test_check` result is PASS: + +→ `verdict = real_fix` — unconditionally, regardless of `failure_subtype`. + +The decision table below applies ONLY when no fix was committed during this +invocation. Once a fix is applied and tests pass, the verdict is `real_fix` +and the table is never consulted. + +### Step 2d: No-Fix Verdict Decision Tree + +**Applies ONLY when no fix was applied (fixes_applied == 0).** If the fix loop was entered +and a commit was made, skip this table — verdict is already `real_fix` per Step 2c. Using `{local_result}` from Step 2 and `{failure_subtype}` from Step 2a, determine `{verdict}`: | Local result | `failure_subtype` | Verdict | |---|---|---| -| FAIL → (fix applied in Step 3) → PASS | any | `real_fix` | -| FAIL → (no fix possible after 3 iterations) | any | proceed to Step 5 | | PASS | `flaky` or `timing_race` | `flake_suspected` | | PASS | `deterministic` | `ci_only_failure` | -| PASS | `fixture` or `import` | `ci_only_failure` | -| PASS | `env` or `unknown` | `ci_only_failure` (conservative) | +| PASS | `fixture` or `import` | `flake_suspected` | +| PASS | `env` or `unknown` | `flake_suspected` | **Note on `already_green`:** This verdict is reserved for the `pre_resolve_rebase` re-entry path — when a sibling pipeline's fix has already landed on integration and @@ -195,10 +190,10 @@ resolve-failures will now emit `real_fix` or another verdict. `already_green` is not emitted by this skill's primary workflow. If local tests PASS (no fix needed): go to Step 2.5 (Validate CI Resolution) before -proceeding to Step 4 — the CI-truth gate may redirect to Step 3 for flakiness +proceeding to Step 4 — the CI-truth gate may redirect to the fix loop for flakiness investigation even when local tests pass. -If local tests FAIL: enter Step 3. +If local tests FAIL: enter the fix loop. ### Step 2.5: Validate CI Resolution @@ -219,10 +214,13 @@ the failure could not be reproduced locally, which is a flaky-test signal. - Do **NOT** proceed to Step 4 - Log: "CI failure on [test name] — local pass is not a resolution (flaky test signal). Entering fix loop to investigate and stabilize." - - Proceed to **Step 3 (Fix Loop)** to investigate the non-determinism, timing + - Proceed to **the Fix Loop** to investigate the non-determinism, timing dependencies, or race conditions that caused the test to pass locally but fail in CI. Apply a stabilizing fix (e.g., increase timeouts, remove timing dependencies, add retry guards, fix resource cleanup). + - When the fix loop applies a stabilizing fix and tests pass, emit `verdict = real_fix` + (per Step 2c override). Do NOT fall back to the Step 2d table — the fix + resolves the CI failure regardless of the original `failure_subtype`. d. If `failure_type` is not "test" (e.g., "lint", "build") and tests pass locally: - Proceed to Step 4 — local pass resolves non-test CI failures (lint/build failures are deterministic; they don't pass locally while failing remotely). @@ -251,7 +249,7 @@ the failure could not be reproduced locally, which is a flaky-test signal. - The specific error message for each failure (first 10–15 lines) Discard the full pytest stdout — do not retain progress dots, install-worktree output, or timing lines. These accumulate across iterations and inflate context. -6. Green → Step 4 (with `verdict = real_fix`); Red and < 3 iterations → repeat; Red and >= 3 → Step 5 +6. Green → `verdict = real_fix` (per Step 2c override — do not re-evaluate Step 2d) → Step 4; Red and < 3 iterations → repeat; Red and >= 3 → Step 5 ### Step 4: Report @@ -275,14 +273,17 @@ fixes_applied = {N} Where: - `{verdict}` is one of: `real_fix`, `flake_suspected`, `ci_only_failure` -- `{N}` is the number of fix iterations performed (0 for non-real_fix verdicts, ≥1 for `real_fix`) +- `{N}` is the number of fix iterations performed (0 for `flake_suspected` or `ci_only_failure` verdicts, ≥1 for `real_fix`) Return control to the orchestrator. The recipe's `on_result:` routing dispatches on `verdict`: - `real_fix` → `re_push` (fix landed, push to remote) -- `flake_suspected` → `release_issue_failure` (human escalation) +- `flake_suspected` → `re_push` (retry via CI, bounded by retries: 2 / on_exhausted: release_issue_failure) - `ci_only_failure` → `release_issue_failure` (human escalation) +**Invariant:** `ci_only_failure` is NEVER emitted when `fixes_applied >= 1`. If a fix was +committed during this invocation and tests pass, the verdict is always `real_fix`. + ### Step 5: Report Failure - Total fix iterations attempted - Remaining test failures (summary) diff --git a/src/autoskillit/skills_extended/resolve-merge-conflicts/SKILL.md b/src/autoskillit/skills_extended/resolve-merge-conflicts/SKILL.md index 40bea5766b..110af0850c 100644 --- a/src/autoskillit/skills_extended/resolve-merge-conflicts/SKILL.md +++ b/src/autoskillit/skills_extended/resolve-merge-conflicts/SKILL.md @@ -41,8 +41,11 @@ hooks: ## When to Use -- Called by the `merge-prs` when `merge_worktree` fails with - `failed_step=rebase` and `state=worktree_intact_rebase_aborted` +- Called by queue-ejection, direct-merge, immediate-merge, and CI-conflict paths + when `merge_worktree` fails with `failed_step=rebase` and `state=worktree_intact_rebase_aborted` +- Called by the `rebase_conflict_fix` step in `implementation.yaml`, + `implementation-groups.yaml`, and `remediation.yaml` when `merge_worktree` returns + `failed_step == 'rebase'` - The worktree must still be intact (rebase aborted cleanly, no partial state) ## Workflow @@ -248,8 +251,34 @@ After a successful `rebase --continue` (no more conflict rounds): cd {worktree_path} && pre-commit run --all-files ``` -Fix any auto-fixable violations (ruff format, ruff check). Re-stage fixed files and -re-run `pre-commit run --all-files` to confirm clean. +If pre-commit fails, apply all auto-fixable hooks in a single pass before re-running: + +1. **ruff format / ruff check**: These hooks auto-apply fixes when they run; check + for modified files after the failure. + +2. **check-version-consistency**: If this hook fails, run: + ```bash + python3 scripts/sync_versions.py + git -C {worktree_path} add -u + ``` + +3. **uv-lock-check**: If this hook fails, run: + ```bash + uv lock + git -C {worktree_path} add -u + ``` + +After applying all applicable fixes, re-stage any remaining modified files with +`git -C {worktree_path} add -u` and re-run `pre-commit run --all-files`. + +If `pre-commit run --all-files` still fails after this second pass, the remaining +failure is from a non-auto-fixable hook (e.g., `mypy`, `gitleaks`, `doc-counts`). +Escalate immediately — do NOT loop: + +``` +escalation_required = true +escalation_reason = pre-commit failed after applying all auto-fixes (ruff, sync_versions, uv lock). Remaining failure from non-fixable hook requires manual remediation: +``` **Do NOT run the full test suite.** Testing is handled by the pipeline's `test` step, which already ran and passed before `merge_to_integration` was first attempted. Running @@ -434,5 +463,5 @@ Omit `conflict_report_path=` line entirely when the rebase was clean (no conflic - **Validation failures**: Emit clear error message and abort before touching git state - **`rebase --continue` failure** (not a conflict): Abort with `git -C {worktree_path} rebase --abort` and escalate -- **`pre-commit` failure**: Fix auto-fixable violations (ruff format/check) and re-run; do not escalate for formatting-only failures +- **`pre-commit` failure**: Apply all auto-fixable hooks in order — ruff format/check (auto-applied by hooks), `check-version-consistency` (run `python3 scripts/sync_versions.py && git add -u`), `uv-lock-check` (run `uv lock && git add -u`) — re-stage, then re-run. If pre-commit still fails after this pass, escalate with `escalation_required=true`; non-fixable hooks (mypy, gitleaks, doc-counts) require manual remediation. - **Unexpected git state**: Run `git -C {worktree_path} rebase --abort` before exiting diff --git a/src/autoskillit/skills_extended/resolve-research-review/SKILL.md b/src/autoskillit/skills_extended/resolve-research-review/SKILL.md index 3075a08375..e30ceb5cdd 100644 --- a/src/autoskillit/skills_extended/resolve-research-review/SKILL.md +++ b/src/autoskillit/skills_extended/resolve-research-review/SKILL.md @@ -44,6 +44,7 @@ Bounded by `retries: 2` — on exhaustion routes to `research_complete`. - Delete or discard the working directory on failure - Modify tests to suppress failures introduced by reviewer fixes - Use file-path-segment grouping — research comments are grouped by **dimension**, not by file path +- Run subagents in the background (`run_in_background: true` is prohibited) **ALWAYS:** - Find the PR by feature branch at invocation time (not a hardcoded number) @@ -142,6 +143,11 @@ From **inline comments**, extract per comment: - `id` — the comment's REST database ID - `thread_node_id` — look up `comment_id_to_thread_id.get(id)` +**File-level comment guard:** If `line` is null (file-level comment posted by +review-research-pr), skip this finding entirely — file-level comments have no code +anchor and cannot be resolved by code changes. Record: `(path, null, reason="file-level +comment — no line anchor")`. Do not add its `thread_node_id` to `addressed_thread_ids`. + **Classify each finding by severity** (same as resolve-review): - `critical` — body contains: "must", "critical", "security", "data loss", "wrong", "broken", "incorrect", "bug", "error", "never" diff --git a/src/autoskillit/skills_extended/resolve-review/SKILL.md b/src/autoskillit/skills_extended/resolve-review/SKILL.md index daa726864a..266a493ccc 100644 --- a/src/autoskillit/skills_extended/resolve-review/SKILL.md +++ b/src/autoskillit/skills_extended/resolve-review/SKILL.md @@ -18,10 +18,13 @@ for actionable findings, commit each fix, and verify tests still pass. ## Arguments -`/autoskillit:resolve-review ` +`/autoskillit:resolve-review [mode=]` - `feature_branch` — The PR's head branch (used to find the open PR) - `base_branch` — The PR's base branch (e.g., "main") +- **mode** (optional, default: `github`) — Controls where findings are read from and how threads are handled: + - `mode=github` (or absent/unrecognized): current behavior — fetch findings from GitHub API, post deferred observations from prior local rounds, resolve threads and post inline replies. + - `mode=local`: read findings from local JSON (written by `review-pr` in `mode=local`), skip all GitHub API fetching, accumulate DISCUSS/REJECT to persistent local files, skip thread resolution and inline reply API calls, still run `task test-check`. The `cwd` is provided by the recipe step's `cwd:` field — the clone with the feature branch already checked out. @@ -41,6 +44,7 @@ branch already checked out. - Exceed 3 fix-and-retest iterations - Delete or discard the working directory on failure - Modify tests to suppress failures introduced by reviewer fixes +- Run subagents in the background (`run_in_background: true` is prohibited) **ALWAYS:** - Find the PR by feature branch at invocation time (not a hardcoded number) @@ -50,6 +54,8 @@ branch already checked out. - Run `{test_command}` (from config, default: `task test-check`) after applying all fixes to catch regressions - Gracefully degrade (exit 0, report skip) if `gh` is unavailable or no PR is found - Report a structured summary: findings fetched, fixes applied, fixes skipped (with reasons) +- **Read before editing**: Before issuing an `Edit` call on any file, ensure you have issued a `Read` on that file earlier in this session. Claude Code rejects `Edit` on unread files — the retry wastes a full API turn at current context size. If you are uncertain whether a file was read, issue a targeted `Read` (offset + limit to the region you plan to edit) rather than risk an error. +- **CWD awareness**: Before running `python3` or other interpreters, verify your current working directory is the worktree root (not the orchestrator's project root). Use absolute paths for imports or `cd` to the worktree first. A wrong-CWD import error wastes a full API turn. ## Context Limit Behavior @@ -67,8 +73,8 @@ review fixes are committed and the downstream push step receives a clean branch. ## Workflow -Read `test_check.command` from `.autoskillit/config.yaml` (default: `task test-check`). -Store the resolved command as `{test_command}` for use in all test-running steps. +Read test configuration from `.autoskillit/config.yaml`: check `test_check.commands` (ordered list, if set) or `test_check.command` (single command, default: `task test-check`). +The `test_check` MCP tool runs all configured commands automatically. ### Step 0: Validate Arguments @@ -77,6 +83,21 @@ Parse two positional arguments: `feature_branch` and `base_branch`. If either is missing, abort with: `"Usage: /autoskillit:resolve-review "` +Parse the optional `mode` keyword argument: + +```bash +# Extract mode from keyword arguments +MODE="github" +for arg in "$@"; do + case "$arg" in + mode=local) MODE="local" ;; + mode=github) MODE="github" ;; + esac +done +``` + +If `mode` is absent or unrecognized, default to `"github"`. + ### Step 1: Find the Open PR ```bash @@ -95,8 +116,66 @@ If `gh` is unavailable or not authenticated, or no PR is found: - Log "No PR found or gh unavailable — skipping review resolution" - Exit 0 (graceful degradation — do not fail the pipeline) +### Step 1.5: Post Accumulated Deferred Observations (github mode only) + +**When `mode=github`:** + +Before fetching current findings from GitHub, check for any deferred observations +accumulated from prior local review rounds: + +```bash +DEFERRED_FILE="{{AUTOSKILLIT_TEMP}}/resolve-review/deferred_observations_${PR_NUMBER}.json" +``` + +If the file exists and contains entries: +1. Load the deferred observations array from the file +2. Post ALL entries as a single batch review via `POST /repos/{owner}/{repo}/pulls/{pr_number}/reviews`: + - `event`: `"COMMENT"` (not requesting changes — these are observations for discussion) + - `body`: `"Observations accumulated from {N} local review rounds:"` + - `commit_id`: current HEAD commit SHA (from `gh pr view {pr_number} --json headRefOid -q .headRefOid`) + - `comments[]` array where each entry has: + - `path`: from the deferred entry + - `line`: from the deferred entry (if `line` is null, omit `line` and use `position: 1` as file-level comment) + - `side`: `"RIGHT"` + - `body`: + ``` + **Observation from local review round {round}:** + + {body} + + **Evidence:** {evidence} + + + ``` + +3. Use the batch review endpoint (never post individual comments unless the batch call fails) +4. **Fallback:** If the batch POST returns HTTP 422 (e.g., stale line numbers), retry by posting each observation individually via `gh api repos/{owner}/{repo}/pulls/{pr_number}/comments --method POST` with 1s delay between calls +5. After all deferred observations are posted successfully, rename the file to `deferred_observations_${PR_NUMBER}_posted.json` to prevent re-posting on retry +6. These review threads are left UNRESOLVED (same behavior as DISCUSS in github mode) + +If the file does not exist or is empty, skip this step and proceed to Step 2. + +**`REVIEW-FLAG` marker format:** `` +Matches the regex `" # REJECT: -BODY="Investigated — this is intentional. ${evidence}" +BODY="Investigated — this is intentional. ${evidence} +" # DISCUSS: -BODY="Valid observation — flagged for design decision. ${evidence}" +BODY="Valid observation — flagged for design decision. ${evidence} + +" +# INFO (auto-classified DISCUSS): +BODY="Acknowledged — minor suggestion noted. + +" gh api repos/{owner}/{repo}/pulls/{pr_number}/comments/{comment_id}/replies \ --method POST \ --field body="${BODY}" +sleep 1 # Rate-limit discipline: 1s between mutating calls ``` For ACCEPT replies, use the `commit_sha` from the most recent commit made in Step 4 @@ -332,11 +618,60 @@ Track: - `reply_posted_count: int` — successfully posted replies - `reply_failed_count: int` — replies that failed (log warning, continue) -This step is best-effort: failure to post any reply must not affect the exit code. - ### Step 6.6: Persist Reject Patterns -After Step 6.5, save all REJECT-classified comments to a JSON file for future analysis: +**MODE BRANCHING:** + +**When `mode=local`:** + +After Step 6.5, save all REJECT-classified comments to a **stable, accumulating** JSON file +(without timestamp — the same file is reused across local rounds): + +```python +import json, pathlib + +reject_file = pathlib.Path("{{AUTOSKILLIT_TEMP}}/resolve-review/reject_patterns_${PR_NUMBER}.json") + +# Load existing entries if file exists +existing = [] +if reject_file.exists(): + existing = json.loads(reject_file.read_text()) + +# Build new entries — use synthetic comment_id for local-mode findings +# Format: "local_{iteration}_{index}" derived from iteration in local_findings + index +reject_entries = [] +for idx, c in enumerate(classification_map.values()): + if c.get("verdict") != "REJECT": + continue + # Build synthetic comment_id from local finding source + comment_id = c.get("comment_id", f"local_unknown_{idx}") + entry = { + "comment_id": comment_id, + "path": c.get("path"), + "line": c.get("line"), + "body": c.get("body"), + "evidence": c.get("evidence", ""), + "category": c.get("category", "other"), + "pr_number": ${PR_NUMBER}, + "feature_branch": "${feature_branch}", + } + reject_entries.append(entry) + +# Deduplicate: skip if (path, line, body) already in existing +seen = {(e["path"], e["line"], e["body"]) for e in existing} +new_entries = [e for e in reject_entries if (e["path"], e["line"], e["body"]) not in seen] + +# Write atomically +all_entries = existing + new_entries +reject_file.write_text(json.dumps(all_entries, indent=2)) +print(f"Accumulated {len(new_entries)} new REJECT patterns ({len(all_entries)} total)") +``` + +**Note:** In local mode, `comment_id` may not be a GitHub database ID. Use whatever ID +is in the classification_map entry. If the finding came from `local_findings_{pr_number}.json` +and has no native ID, use `"local_{iteration}_{index}"` as a synthetic identifier. + +**When `mode=github`:** Execute the following current behavior (timestamped, one-shot write). ```bash ts=$(date +%Y%m%d-%H%M%S) @@ -365,6 +700,9 @@ print(f'Saved {len(reject_entries)} reject patterns') ### Step 7: Report +**MODE INDEPENDENCE:** `task test-check` (Step 5) runs identically in both modes. +Gate token emission is mode-independent. + Print a structured summary to terminal: ``` @@ -373,7 +711,7 @@ PR: #{pr_number} ({feature_branch} → {base_branch}) Findings fetched: {total} - critical: {n} - warning: {n} - - info: {n} (skipped — below threshold) + - info: {n} Intent validation (before code changes): - ACCEPT: {accept_count} - REJECT: {reject_count} @@ -431,6 +769,16 @@ Where `{N}` is the count of ACCEPT findings where code changes were committed. `verdict = real_fix` means fixes were applied; `verdict = already_green` means all review findings were already addressed and no code changes were needed. -When no PR is found (graceful degradation), no structured tokens are emitted. +**Mode-conditional path outputs:** + +When `mode=local`, the following additional tokens are emitted: +``` +deferred_observations_path = {AUTOSKILLIT_TEMP}/resolve-review/deferred_observations_{pr_number}.json +reject_patterns_path = {AUTOSKILLIT_TEMP}/resolve-review/reject_patterns_{pr_number}.json +``` + +When `mode=github` and prior local rounds accumulated observations, these are posted +to GitHub and renamed to `deferred_observations_{pr_number}_posted.json` — no path token +is emitted for the posted state. Summary written to: `{{AUTOSKILLIT_TEMP}}/resolve-review/report_{pr_number}_{ts}.md` (relative to the current working directory) diff --git a/src/autoskillit/skills_extended/retry-worktree/SKILL.md b/src/autoskillit/skills_extended/retry-worktree/SKILL.md index f70ffdff9e..3c7cb4934d 100644 --- a/src/autoskillit/skills_extended/retry-worktree/SKILL.md +++ b/src/autoskillit/skills_extended/retry-worktree/SKILL.md @@ -1,6 +1,6 @@ --- name: retry-worktree -description: Continue implementing a plan in an existing git worktree after context exhaustion. Use when a previous implement-worktree session hit context limits. Takes plan path and worktree path as arguments. +description: Worktree retry executor. ALWAYS invoke this skill when instructed to continue or retry an implementation in an existing worktree. Do not resume editing files directly — use this skill first to load the retry workflow. hooks: PreToolUse: - matcher: "*" @@ -42,6 +42,7 @@ Continue implementing a plan in an **existing** git worktree. This skill is used - Re-run tests just to see failures — grep the saved output file instead - Pipe test output through `tail`, `head`, or other truncation commands — `tail -N` buffers the entire stream and produces no output if the process is killed before EOF - Default to `main` as the base branch — always discover it from git's upstream structure or the explicit base-branch store file +- Run subagents in the background (`run_in_background: true` is prohibited) **ALWAYS:** - Use the provided worktree path (do NOT create a new one) @@ -50,6 +51,7 @@ Continue implementing a plan in an **existing** git worktree. This skill is used - Continue from where the previous session left off - Run the project's test suite from the worktree directory - Rebase onto base branch before completion (ready for squash-and-merge) +- **Read files fully**: When reading a file to understand it in full, read it in a single call without a `limit` parameter. Do not paginate files with sequential offset reads — read once completely. Use `limit`/`offset` only for targeted section reads of files you have already read in full. ## Context Limit Behavior @@ -111,17 +113,16 @@ if [ -z "$BASE_BRANCH" ]; then # Fallback: read explicit file store written by implement-worktree-no-merge MAIN_GIT_DIR=$(git rev-parse --git-common-dir) MAIN_REPO_ROOT=$(dirname "${MAIN_GIT_DIR}") - STORE_FILE="${MAIN_REPO_ROOT}/{{AUTOSKILLIT_TEMP}}/worktrees/${CURRENT_BRANCH}/base-branch" + WORKTREE_DIR_NAME=$(basename "$(pwd)") + STORE_FILE="${MAIN_REPO_ROOT}/{{AUTOSKILLIT_TEMP}}/worktrees/${WORKTREE_DIR_NAME}/base-branch" BASE_BRANCH=$(cat "${STORE_FILE}" 2>/dev/null) fi if [ -z "$BASE_BRANCH" ]; then - echo "ERROR: Cannot determine base branch from git structure." - echo "Both the upstream tracking ref and the explicit base-branch file at" - echo "{{AUTOSKILLIT_TEMP}}/worktrees/${CURRENT_BRANCH}/base-branch are missing." - echo "Ensure the worktree was created by implement-worktree-no-merge," - echo "which writes both stores at worktree creation time." - exit 1 + # Last resort: project-level default from config (always available) + BASE_BRANCH="{{DEFAULT_BASE_BRANCH}}" + echo "WARNING: Could not determine base branch from git upstream or sidecar file." + echo "Falling back to project default: ${BASE_BRANCH}" fi ``` @@ -138,16 +139,6 @@ Then assess what has been implemented: - Which phase is partially complete - Which phases haven't started -### Step 1.5: Initialize Code Index for Worktree - -Set the MCP code-index project path to the worktree so code searches operate on the correct files: - -``` -mcp__code-index__set_project_path(path="{WORKTREE_PATH}") -``` - -This must happen before any code-index searches or Explore subagents. - ### Step 2: Targeted Exploration (Only If Needed) Only explore systems related to the **remaining** phases. Do NOT re-explore already-completed work. Use Explore subagents for: @@ -161,6 +152,9 @@ Only explore systems related to the **remaining** phases. Do NOT re-explore alre Initialize a counter before iterating: `PHASES_IMPLEMENTED=0` +NEVER use AskUserQuestion between phase iterations. Each phase begins immediately +after the previous phase completes. + For each remaining/incomplete phase, begin implementation immediately (no announcement): 1. Implement changes 2. Run per-phase verification if plan specifies it @@ -175,7 +169,10 @@ Run the project's code quality checks and test suite from the worktree. ```bash cd {WORKTREE_PATH} && pre-commit run --all-files -cd {WORKTREE_PATH} && task test-all +cd {WORKTREE_PATH} && \ + AUTOSKILLIT_TEST_FILTER="${AUTOSKILLIT_TEST_FILTER:-conservative}" \ + AUTOSKILLIT_TEST_BASE_REF="${BASE_BRANCH:-}" \ + task test-all ``` If tests fail, fix the issue and re-run. @@ -213,16 +210,6 @@ phases_implemented = ${PHASES_IMPLEMENTED} Where `PHASES_IMPLEMENTED` is the count from Step 3. If Step 3 was skipped entirely (all phases already complete), emit `phases_implemented = 0`. -### Step 6.5: Reset Code Index to Original Project (REQUIRED) - -After worktree cleanup, reset the MCP code-index project path back to the original project directory: - -``` -mcp__code-index__set_project_path(path="{ORIGINAL_PROJECT_PATH}") -``` - -Failure to do this leaves code-index pointing at a deleted worktree path, breaking all subsequent code searches. - ## Error Handling - **Worktree environment missing** — re-create using the project's configured `worktree_setup.command`, or: `task install-worktree` diff --git a/src/autoskillit/skills_extended/review-approach/SKILL.md b/src/autoskillit/skills_extended/review-approach/SKILL.md index f79b62341a..0d1032bb84 100644 --- a/src/autoskillit/skills_extended/review-approach/SKILL.md +++ b/src/autoskillit/skills_extended/review-approach/SKILL.md @@ -26,6 +26,7 @@ Research modern solutions, approaches, and strategies relevant to the issues or **NEVER:** - Modify any source code files - Create files outside `{{AUTOSKILLIT_TEMP}}/review-approach/` directory +- Run subagents in the background (`run_in_background: true` is prohibited) **ALWAYS:** - Use subagents with web search for parallel research diff --git a/src/autoskillit/skills_extended/review-design/SKILL.md b/src/autoskillit/skills_extended/review-design/SKILL.md index 1c3e1c17e2..05cb9e7b95 100644 --- a/src/autoskillit/skills_extended/review-design/SKILL.md +++ b/src/autoskillit/skills_extended/review-design/SKILL.md @@ -43,6 +43,7 @@ best available plan. - Exit non-zero — GO, REVISE, and STOP are all normal outcomes (exit 0 in all cases) - Include code snippets, shell commands, or specific tool invocations in findings or revision guidance — findings describe gaps and risks, not implementation instructions - Prescribe HOW to fix an issue — findings must describe WHAT is lacking or at risk; the fix is the plan author's responsibility +- Run subagents in the background (`run_in_background: true` is prohibited) **ALWAYS:** - Use `model: "sonnet"` when spawning all subagents via the Task tool @@ -77,12 +78,12 @@ absent. The recipe routes to `on_context_limit`, abandoning the partial review. 1. Create `{{AUTOSKILLIT_TEMP}}/review-design/` if absent. 2. Extract `experiment_plan_path` from arguments (first path-like token starting with `/`, `./`, or `.autoskillit/`). - **Error handling:** If no path-like token is found in the arguments, emit - `verdict = STOP` with message "No experiment_plan_path provided" and exit 0 (per + **Error handling:** When no path-like token is present in the arguments, emit + `verdict = STOP` with message "No experiment_plan_path provided" and return (per the NEVER exit-non-zero constraint). 3. Read the plan file. **Error handling:** If the file does not exist or is unreadable at the resolved path, - emit `verdict = STOP` with message "Plan file not found: {path}" and exit 0. + emit `verdict = STOP` with message "Plan file not found: {path}" and return. 4. **Load the experiment type registry:** a. Locate bundled types dir: run `python -c "from autoskillit.core import pkg_root; print(pkg_root() / 'recipes' / 'experiment-types')"` @@ -100,7 +101,7 @@ absent. The recipe routes to `on_context_limit`, abandoning the partial review. (zero LLM tokens). Return present fields and note which are missing. Record `source: frontmatter` for each extracted field. **Error handling:** If the YAML between `---` delimiters is malformed, treat all - fields as missing and fall through to Level 2 for all fields (graceful degradation). + fields as missing and fall through to Level 2 for all fields (fallback handling). - **Level 2 (LLM extraction)**: For each missing field, launch a targeted LLM extraction subagent against the corresponding prose section. All extractions are independent and run in parallel. Record `source: extracted` for each field from @@ -157,6 +158,57 @@ or weight=S (SILENT — dimension not spawned, not mentioned in output). Pass th `dimension_weights` dict to the triage subagent so it can return the complete weight matrix for this plan. +### Silent Type Handling (after Step 1) + +After Step 1 classification, check whether the experiment type is "silent" — an +experiment type where quantitative-audit dimension scoring does not apply. + +**Detection rule:** An experiment type is silent when **>=6 of 8 `dimension_weights`** +are equal to `S`. Use `is_silent_type(spec)` from `experiment_type_registry` (or count +the `S` values directly from the loaded registry entry). + +**Reference:** `docs/research/silent-type-convention.md` defines the shared convention +consumed by both review-design (this skill) and vis-lens-methodology-norms (#846). + +**When `is_silent_type` returns True:** + +1. **Skip Steps 2-7 entirely** — do not launch L1, L2, L3, L4, or red-team subagents. +2. **Emit verdict = GO** with `requires_decision: false`. +3. **Write evaluation_dashboard** with a "Scope Advisory" section containing: + + ```yaml + verdict: GO + advisory_context: + subject_kind: experiment_type + subject_name: {experiment_type} + reasoning: "Quantitative audit framework does not apply to {experiment_type}. Design rigor is assessed via domain-appropriate criteria (e.g., trustworthiness, transferability, dependability, confirmability for qualitative research)." + reference_framework: "SRQR / COREQ" + requires_decision: false + ``` + +4. **Write machine-readable YAML summary** in the dashboard: + + ```yaml + # --- review-design machine summary --- + verdict: GO + experiment_type: {type} + critical_count: 0 + warning_count: 0 + blocking_count: 0 + required_count: 0 + advisory_count: 1 + red_team_count: 0 + active_dimensions: 0 + warning_threshold: 0 + ``` + +5. **Proceed to Step 8** (Emit Output Tokens) — the recipe routes to `plan_visualization`. + +The advisory is appended to the evaluation dashboard file (which is later copied to +`research/{slug}/audit/design-review-dashboard.md` by `create_worktree.sh`). + +**When `is_silent_type` returns False:** Continue with the standard path (Steps 2-7). + ### Subagent Evaluation Scope (applies to ALL dimension subagents) **Include this instruction block in every dimension subagent prompt.** @@ -218,7 +270,7 @@ If ANY Level 1 subagent returns a finding with `"severity": "critical"`: - Collect these as `stop_triggers` - Do not proceed to Level 2, 3, or 4 analysis - Do not start the red-team agent -- Skip directly to Step 7 (Synthesis) with only L1 findings +- Proceed directly to Step 7 (Synthesis) with only L1 findings - The verdict logic will produce STOP; halt and emit tokens **Subagent parse failure:** If a Level 1 subagent returns unparseable output (malformed @@ -459,6 +511,22 @@ One synthesis pass (no subagent — orchestrator synthesizes directly): 5. **Write `evaluation_dashboard_{slug}_{YYYY-MM-DD_HHMMSS}.md`** — always written. Must include: - Verdict banner and classification summary + - **Dimension Rationale** subsection — renders a per-dimension rationale table sourced + from `spec.dimension_weight_rationale` (loaded in Step 0 registry). Table format: + ```markdown + ## Dimension Rationale (why these weights apply to {experiment_type}) + + | Dimension | Weight | Rationale | + |---|---|---| + | {dim} | {weight} | {rationale from spec.dimension_weight_rationale[dim]} | + ``` + Include ALL non-SILENT dimensions (H, M, L). For each dimension: + - If `dimension_weight_rationale[dim]` exists and is non-empty: render the rationale string + - If no rationale entry exists (legacy types with empty `dimension_weight_rationale`): + render the Weight column only, leave Rationale blank + Omit SILENT (S) dimensions from the table (consistent with finding-count suppression). + If `dimension_weight_rationale` is entirely empty (no entries for any dimension), + omit the Dimension Rationale subsection entirely — do not render an empty table. - Dimension scorecard table (dimension → weight → findings count → severity summary) - Adversarial findings section (red-team findings, each marked `requires_decision: true`) - **Cannot Assess** section with at least 2 items (dimensions where evaluation was @@ -502,10 +570,13 @@ Emit these lines as your final output: ``` verdict = GO|REVISE|STOP experiment_type = {experiment_type} +classification_timestamp = {ISO 8601 UTC timestamp} evaluation_dashboard = /absolute/path/{{AUTOSKILLIT_TEMP}}/review-design/evaluation_dashboard_{slug}_{YYYY-MM-DD_HHMMSS}.md revision_guidance = /absolute/path/{{AUTOSKILLIT_TEMP}}/review-design/revision_guidance_{slug}_{YYYY-MM-DD_HHMMSS}.md ``` +`classification_timestamp` is the UTC timestamp (e.g., `2026-04-13T15:32:00Z`) at the moment the experiment-type classification is finalized, before writing the evaluation dashboard. + `revision_guidance` line is emitted ONLY when verdict = REVISE. When verdict is GO or STOP, omit the `revision_guidance` line entirely. diff --git a/src/autoskillit/skills_extended/review-pr/SKILL.md b/src/autoskillit/skills_extended/review-pr/SKILL.md index 7d141ad07e..b8b657ac5d 100644 --- a/src/autoskillit/skills_extended/review-pr/SKILL.md +++ b/src/autoskillit/skills_extended/review-pr/SKILL.md @@ -19,12 +19,16 @@ by the recipe pipeline after `open_pr_step` opens the PR. ## Arguments -`/autoskillit:review-pr [annotated_diff_path=] [hunk_ranges_path=]` +`/autoskillit:review-pr [annotated_diff_path=] [hunk_ranges_path=] [diff_metrics_path=] [mode=]` - **feature-branch** — The feature branch containing the changes to review - **base-branch** — The base branch the PR targets (e.g., "main") - **annotated_diff_path** (optional) — absolute path to a pre-computed annotated diff file (produced by `annotate_pr_diff` run_python step). When provided and present, read from file instead of running python3. - **hunk_ranges_path** (optional) — absolute path to a pre-computed hunk ranges JSON file (produced by `annotate_pr_diff` run_python step). When provided and present, read from file instead of running python3. +- **diff_metrics_path** (optional) — absolute path to a pre-computed diff metrics JSON file (produced by `annotate_pr_diff` run_python step). Contains `dispatch_agents` list that determines which audit dimensions to spawn. When absent, all 6 standard agents are dispatched. +- **mode** (optional, default: `github`) — Controls where findings are written: + - `mode=github` (or absent/unrecognized): current behavior — post findings as GitHub inline review comments via the GitHub Reviews API. + - `mode=local`: write findings to a local JSON file instead of posting to GitHub. Skips all GitHub API calls for comment posting. Still writes `diff_context_{pr_number}.json`, `raw_findings_{pr_number}.json`, and `summary_{pr_number}_{timestamp}.md` as normal. Gate tokens (`%%REVIEW_GATE::*%%`) are emitted identically in both modes. ## When to Use @@ -39,6 +43,7 @@ by the recipe pipeline after `open_pr_step` opens the PR. - Post review comments when `gh` is unavailable — output `verdict=approved` and exit 0 - Review files outside the PR diff — scope all audit to diff content only - Modify any source code +- Run subagents in the background (`run_in_background: true` is prohibited) **ALWAYS:** - Find the PR by feature branch at invocation time (not from a pre-captured URL) @@ -64,24 +69,22 @@ escalation_user=$(gh api user -q .login 2>/dev/null || echo "") If `escalation_user` is non-empty, set `escalation_user_mention="@${escalation_user}"`. If empty (gh unavailable or not authenticated), set `escalation_user_mention=""`. -### Step 0.5 — Code-Index Initialization (required before any code-index tool call) +Parse the optional `mode` keyword argument: -Call `set_project_path` with the repo root where this skill was invoked (not a worktree path): - -``` -mcp__code-index__set_project_path(path="{PROJECT_ROOT}") +```bash +# Extract mode from keyword arguments +MODE="github" +for arg in "$@"; do + case "$arg" in + mode=local) MODE="local" ;; + mode=github) MODE="github" ;; + esac +done ``` -Code-index tools require **project-relative paths**. Always use paths like: - - src/mypackage/core/module.py - -NOT absolute paths like: - - /path/to/project/src/mypackage/core/module.py - -Agents launched via `run_skill` inherit no code-index state from the parent session — this -call is mandatory at the start of every headless session that uses code-index tools. +If `mode` is absent or unrecognized, default to `"github"`. The mode controls where +findings are written — `mode=local` skips all GitHub API posting and writes to a local +JSON file instead. ### Step 1: Find the Open PR @@ -95,6 +98,83 @@ If `gh` is unavailable or not authenticated, or no PR is found: - Output `verdict=approved` - Exit 0 (graceful degradation) +### Step 1.5: Fetch Prior Review Thread Context + +This step is always executed when a PR is found. It builds prior-thread context for +suppressing already-resolved findings on re-reviews and for focusing subagents on +known-unresolved items. + +Fetch all review threads using cursor-based pagination (same GraphQL query as +resolve-review Step 2, but also fetching `comments(first:5)` to see the original +finding and up to 4 replies): + +```graphql +query($owner:String!, $repo:String!, $number:Int!, $after:String) { + repository(owner:$owner, name:$repo) { + pullRequest(number:$number) { + reviewThreads(first:100, after:$after) { + pageInfo { hasNextPage endCursor } + nodes { + isResolved + path + line + originalLine + comments(first:5) { + nodes { databaseId body author { login } } + } + } + } + } + } +} +``` + +Build two lists from the thread nodes. For each thread, resolve line via: +`line = thread.get("line") or thread.get("originalLine")` — `line` is nullable for +outdated threads where new commits have shifted the diff anchor; `originalLine` is +the stable fallback. + +If both `line` and `originalLine` are null (file-level comment thread from a prior review), +skip this thread — do not add it to `prior_resolved_findings` or `prior_unresolved_findings`. +File-level threads have no line anchor and must not suppress line-anchored findings via the +±5 proximity match. + +**`prior_resolved_findings`** — threads meeting EITHER condition, AND where the first comment body +contains `[critical]` or `[warning]` (autoskillit-posted finding): +- `isResolved=true` (ACCEPT/REJECT findings resolved by resolve-review), OR +- Any reply comment (`comments[1:]`) contains `\n" + "\n" + "## my-recipe\n" + "A test recipe\n" + "\n" + "### Graph\n" + "```\n" + "step1 ─── step2 ─── done\n" + "```\n" + "\n" + "### Inputs\n" + "| Name | Default |\n" + "| task | (required) |\n" + ) + result = diagram_to_terminal(md) + assert "step1" in result + assert "step2" in result + + +def test_diagram_to_terminal_suppresses_mermaid(monkeypatch: pytest.MonkeyPatch) -> None: + """T-TERMINAL-MERMAID (b): Mermaid fenced block content must not appear in output.""" + monkeypatch.setenv("NO_COLOR", "1") + md = ( + "\n" + "\n" + "## my-recipe\n" + "A test recipe\n" + "\n" + "```mermaid\n" + "flowchart TD\n" + " S0[clone]\n" + " S1[build]\n" + " S0 --> S1\n" + "```\n" + ) + result = diagram_to_terminal(md) + assert "flowchart TD" not in result + assert "S0[clone]" not in result + assert "S0 --> S1" not in result diff --git a/tests/cli/test_app_main.py b/tests/cli/test_app_main.py new file mode 100644 index 0000000000..6f3e0ed1e7 --- /dev/null +++ b/tests/cli/test_app_main.py @@ -0,0 +1,38 @@ +"""Tests for autoskillit.cli.app.main() entry point behaviour.""" + +from __future__ import annotations + +import importlib +import sys + +import pytest + +pytestmark = [pytest.mark.layer("cli"), pytest.mark.small] + + +def test_main_does_not_call_app_after_update_restart(monkeypatch: pytest.MonkeyPatch) -> None: + """main() must not call app() when run_update_checks triggers a process exit.""" + + app_module = importlib.import_module("autoskillit.cli.app") + + app_called: list[bool] = [] + monkeypatch.setattr(app_module, "app", lambda: app_called.append(True)) + + monkeypatch.setattr( + "autoskillit.cli._init_helpers.evict_direct_mcp_entry", lambda *a, **kw: None + ) + + def fake_run_update_checks(**kwargs: object) -> None: + raise SystemExit(0) + + monkeypatch.setattr( + "autoskillit.cli.update._update_checks.run_update_checks", fake_run_update_checks + ) + monkeypatch.setattr(sys, "argv", ["autoskillit", "order"]) + + try: + app_module.main() + except SystemExit: + pass + + assert not app_called, "app() must not be called when update path exits the process" diff --git a/tests/cli/test_cli_hooks.py b/tests/cli/test_cli_hooks.py index 5e690fdb74..4511340157 100644 --- a/tests/cli/test_cli_hooks.py +++ b/tests/cli/test_cli_hooks.py @@ -5,6 +5,20 @@ import json from pathlib import Path +import pytest + +pytestmark = [pytest.mark.layer("cli"), pytest.mark.small] + + +def _extract_script_from_cmd(cmd: str) -> str: + """Extract the hooks-dir-relative script path from a hook command string.""" + parts = cmd.split() + if "_dispatch.py" in cmd and len(parts) >= 3: + return parts[-1] + ".py" + if "/hooks/" in cmd: + return cmd.split("/hooks/", 1)[1] + return cmd.split("/")[-1] + # HK9 def test_claude_settings_path_user_scope(): @@ -63,7 +77,9 @@ def test_hooks_py_covers_full_registry(tmp_path): for entry in data["hooks"].get("PreToolUse", []) for h in entry.get("hooks", []) ] - registered_pretooluse_scripts = {cmd.split("/")[-1] for cmd in registered_pretooluse} + registered_pretooluse_scripts = { + _extract_script_from_cmd(cmd) for cmd in registered_pretooluse + } assert pretooluse_scripts == registered_pretooluse_scripts, ( f"PreToolUse missing: {pretooluse_scripts - registered_pretooluse_scripts}, " f"Extra: {registered_pretooluse_scripts - pretooluse_scripts}" @@ -78,7 +94,9 @@ def test_hooks_py_covers_full_registry(tmp_path): for entry in data["hooks"].get("PostToolUse", []) for h in entry.get("hooks", []) ] - registered_posttooluse_scripts = {cmd.split("/")[-1] for cmd in registered_posttooluse} + registered_posttooluse_scripts = { + _extract_script_from_cmd(cmd) for cmd in registered_posttooluse + } assert posttooluse_scripts == registered_posttooluse_scripts, ( f"PostToolUse missing: {posttooluse_scripts - registered_posttooluse_scripts}, " f"Extra: {registered_posttooluse_scripts - posttooluse_scripts}" @@ -101,7 +119,10 @@ def test_evict_stale_hooks_removes_legacy_formats(tmp_path): { "matcher": "mcp__.*autoskillit.*__run_skill.*", "hooks": [ - {"type": "command", "command": "python3 -m autoskillit.hooks.quota_guard"}, + { + "type": "command", + "command": "python3 -m autoskillit.hooks.guards.quota_guard", + }, ], }, { @@ -176,8 +197,8 @@ def test_install_production_order_includes_quota_check(tmp_path, monkeypatch): data = json.loads(settings_path.read_text()) pretooluse = data.get("hooks", {}).get("PreToolUse", []) all_commands = [h["command"] for e in pretooluse for h in e.get("hooks", [])] - assert any("quota_guard.py" in c for c in all_commands), ( - "quota_guard.py missing from settings.json after install() — silent drop bug present" + assert any("quota_guard" in c for c in all_commands), ( + "quota_guard missing from settings.json after install() — silent drop bug present" ) @@ -217,7 +238,8 @@ def test_settings_json_matches_hook_registry_after_install(tmp_path, monkeypatch ) entry_commands = [h["command"] for h in matching[0].get("hooks", [])] for script in hook_def.scripts: - assert any(script in c for c in entry_commands), ( + logical_name = script.removesuffix(".py") + assert any(logical_name in c for c in entry_commands), ( f"Script {script!r} missing from matcher {hook_def.matcher!r} " f"in {hook_def.event_type} section of settings.json" ) @@ -234,18 +256,19 @@ def test_sync_hooks_to_settings_writes_all_registry_scripts(tmp_path): data = json.loads(settings.read_text()) - # Verify PreToolUse entry count matches registry - pretooluse_registry_entries = [h for h in HOOK_REGISTRY if h.event_type == "PreToolUse"] + # Verify PreToolUse entry count matches unique (event_type, matcher) pairs. + # HookDef entries sharing a matcher are consolidated into one settings.json entry. + pretooluse_matchers = {h.matcher for h in HOOK_REGISTRY if h.event_type == "PreToolUse"} pretooluse = data["hooks"].get("PreToolUse", []) - assert len(pretooluse) == len(pretooluse_registry_entries), ( - f"Expected {len(pretooluse_registry_entries)} PreToolUse entries, got {len(pretooluse)}" + assert len(pretooluse) == len(pretooluse_matchers), ( + f"Expected {len(pretooluse_matchers)} PreToolUse entries, got {len(pretooluse)}" ) # Verify PostToolUse entries exist - posttooluse_registry_entries = [h for h in HOOK_REGISTRY if h.event_type == "PostToolUse"] + posttooluse_matchers = {h.matcher for h in HOOK_REGISTRY if h.event_type == "PostToolUse"} posttooluse = data["hooks"].get("PostToolUse", []) - assert len(posttooluse) == len(posttooluse_registry_entries), ( - f"Expected {len(posttooluse_registry_entries)} PostToolUse entries, got {len(posttooluse)}" + assert len(posttooluse) == len(posttooluse_matchers), ( + f"Expected {len(posttooluse_matchers)} PostToolUse entries, got {len(posttooluse)}" ) # All scripts from all event types must be present @@ -257,7 +280,8 @@ def test_sync_hooks_to_settings_writes_all_registry_scripts(tmp_path): ] for hook_def in HOOK_REGISTRY: for script in hook_def.scripts: - assert any(script in c for c in all_commands), ( + logical_name = script.removesuffix(".py") + assert any(logical_name in c for c in all_commands), ( f"Script {script!r} missing from settings.json after sync_hooks_to_settings()" ) @@ -276,8 +300,9 @@ def test_sync_hooks_to_settings_is_idempotent(tmp_path): data = json.loads(settings.read_text()) - pretooluse_count = len([h for h in HOOK_REGISTRY if h.event_type == "PreToolUse"]) - posttooluse_count = len([h for h in HOOK_REGISTRY if h.event_type == "PostToolUse"]) + # HookDef entries sharing a matcher are consolidated into one settings.json entry. + pretooluse_count = len({h.matcher for h in HOOK_REGISTRY if h.event_type == "PreToolUse"}) + posttooluse_count = len({h.matcher for h in HOOK_REGISTRY if h.event_type == "PostToolUse"}) pretooluse = data["hooks"].get("PreToolUse", []) posttooluse = data["hooks"].get("PostToolUse", []) @@ -290,6 +315,24 @@ def test_sync_hooks_to_settings_is_idempotent(tmp_path): ) +# T-WT-1: sync_hooks_to_settings rejects worktree pkg_root +def test_sync_hooks_rejects_worktree_pkg_root(tmp_path, monkeypatch): + """sync_hooks_to_settings must raise when pkg_root() is inside a git worktree.""" + from autoskillit.cli._hooks import sync_hooks_to_settings + + fake_pkg = tmp_path / "worktree" / "src" / "autoskillit" + fake_pkg.mkdir(parents=True) + + monkeypatch.setattr("autoskillit.cli._hooks.pkg_root", lambda: fake_pkg) + monkeypatch.setattr("autoskillit.cli._hooks.is_git_worktree", lambda path: True) + + settings_path = tmp_path / "settings.json" + settings_path.write_text("{}") + + with pytest.raises(RuntimeError, match="worktree"): + sync_hooks_to_settings(settings_path) + + # T-CROSS-1 def test_sync_hooks_to_settings_session_start_no_matcher(tmp_path): """sync_hooks_to_settings() must not emit 'matcher' key for SessionStart entries.""" @@ -307,3 +350,27 @@ def test_sync_hooks_to_settings_session_start_no_matcher(tmp_path): assert "matcher" not in entry, ( f"SessionStart entry must not have 'matcher' key, got: {entry}" ) + + +def test_sync_hooks_uses_dispatcher_format(tmp_path, monkeypatch): + """sync_hooks_to_settings() must produce dispatcher-format commands.""" + import autoskillit.cli._hooks as _hooks_mod + + monkeypatch.setattr(_hooks_mod, "is_git_worktree", lambda path: False) + + from autoskillit.cli._hooks import sync_hooks_to_settings + + settings = tmp_path / "settings.json" + sync_hooks_to_settings(settings) + data = json.loads(settings.read_text()) + for event_type, entries in data.get("hooks", {}).items(): + for entry in entries: + for hook in entry.get("hooks", []): + cmd = hook["command"] + assert "_dispatch.py" in cmd, ( + f"{event_type} command does not use dispatcher: {cmd}" + ) + parts = cmd.split() + assert parts[-2].endswith("_dispatch.py"), ( + f"{event_type} dispatcher not in expected position: {cmd}" + ) diff --git a/tests/cli/test_cli_marketplace.py b/tests/cli/test_cli_marketplace.py index f24d715f6a..59bf7ebd5d 100644 --- a/tests/cli/test_cli_marketplace.py +++ b/tests/cli/test_cli_marketplace.py @@ -2,6 +2,10 @@ from __future__ import annotations +import pytest + +pytestmark = [pytest.mark.layer("cli"), pytest.mark.small] + # MK1 def test_marketplace_module_exists(): diff --git a/tests/cli/test_cli_prompts.py b/tests/cli/test_cli_prompts.py index c6d475fe27..bed5a6e517 100644 --- a/tests/cli/test_cli_prompts.py +++ b/tests/cli/test_cli_prompts.py @@ -2,10 +2,14 @@ from __future__ import annotations +from pathlib import Path + import pytest from autoskillit.cli._mcp_names import DIRECT_PREFIX, MARKETPLACE_PREFIX +pytestmark = [pytest.mark.layer("cli"), pytest.mark.small] + # PR1 def test_prompts_module_exists(): @@ -49,6 +53,15 @@ def test_orchestrator_prompt_delegates_ingredient_collection_to_open_kitchen(): ) +def test_build_open_kitchen_prompt_includes_dispatch_routing(): + """_build_open_kitchen_prompt must include dispatch routing instructions.""" + from autoskillit.cli._prompts import _build_open_kitchen_prompt + + prompt = _build_open_kitchen_prompt(DIRECT_PREFIX) + assert "ingredients_only" in prompt + assert "dispatch_food_truck" in prompt + + def test_orchestrator_prompt_documents_confirm_action(): """The orchestrator system prompt must explain how to handle action:confirm steps.""" from autoskillit.cli._prompts import _build_orchestrator_prompt @@ -149,7 +162,7 @@ def test_open_kitchen_prompt_does_not_embed_greetings(): def test_show_cook_preview_prints_table(monkeypatch, tmp_path, capsys): """show_cook_preview prints ingredients table to stdout.""" - from autoskillit.cli._prompts import show_cook_preview + from autoskillit.cli._preview import show_cook_preview from autoskillit.recipe.io import _parse_recipe monkeypatch.setattr( @@ -179,7 +192,7 @@ def test_show_cook_preview_prints_table(monkeypatch, tmp_path, capsys): def test_show_cook_preview_no_diagram(monkeypatch, tmp_path, capsys): """show_cook_preview works when no diagram file exists.""" - from autoskillit.cli._prompts import show_cook_preview + from autoskillit.cli._preview import show_cook_preview from autoskillit.recipe.io import _parse_recipe monkeypatch.setattr( @@ -212,7 +225,7 @@ def test_show_cook_preview_no_diagram(monkeypatch, tmp_path, capsys): def test_show_cook_preview_uses_resolved_base_branch_for_smoke_test(monkeypatch, capsys): """T8: show_cook_preview renders the config-resolved base_branch for smoke-test.""" import autoskillit.recipe._api as api_mod - from autoskillit.cli._prompts import show_cook_preview + from autoskillit.cli._preview import show_cook_preview from autoskillit.core import pkg_root from autoskillit.recipe.io import find_recipe_by_name, load_recipe @@ -220,7 +233,7 @@ def test_show_cook_preview_uses_resolved_base_branch_for_smoke_test(monkeypatch, monkeypatch.setattr(api_mod, "_LOAD_CACHE", {}) monkeypatch.setattr( "autoskillit.config.resolve_ingredient_defaults", - lambda _: {"base_branch": "integration"}, + lambda _: {"base_branch": "develop"}, ) recipe_info = find_recipe_by_name("smoke-test", project_dir) @@ -229,7 +242,7 @@ def test_show_cook_preview_uses_resolved_base_branch_for_smoke_test(monkeypatch, show_cook_preview("smoke-test", recipe, project_dir, project_dir) captured = capsys.readouterr() - assert "integration" in captured.out + assert "develop" in captured.out # base_branch row must NOT show the YAML literal "main" base_branch_lines = [line for line in captured.out.splitlines() if "base_branch" in line] assert base_branch_lines, "base_branch must appear in the rendered table" @@ -327,7 +340,7 @@ def test_show_cook_preview_line_width_bounded_with_implementation_recipe(tmp_pat the real implementation.yaml with its 220-char run_mode description.""" import re - from autoskillit.cli._prompts import show_cook_preview + from autoskillit.cli._preview import show_cook_preview from autoskillit.core import pkg_root from autoskillit.recipe.io import find_recipe_by_name, load_recipe @@ -404,33 +417,378 @@ def test_orchestrator_prompt_contains_quota_routing(): assert "QUOTA DENIAL ROUTING" in prompt -def test_orchestrator_prompt_instructs_toolsearch_when_deferred(): - """PR #750 added DO-NOT-ask; this test pins the recovery path instead.""" +def test_orchestrator_prompt_has_no_server_startup_recovery_block(): + """SERVER-STARTUP RECOVERY block must be removed — it misdiagnoses schema deferral + as server startup latency.""" + from autoskillit.cli._mcp_names import DIRECT_PREFIX + from autoskillit.cli._prompts import _build_open_kitchen_prompt, _build_orchestrator_prompt + + for fn, name in [ + (_build_orchestrator_prompt("test", mcp_prefix=DIRECT_PREFIX), "orchestrator"), + (_build_open_kitchen_prompt(mcp_prefix=DIRECT_PREFIX), "open_kitchen"), + ]: + assert "SERVER-STARTUP RECOVERY" not in fn, ( + f"{name} prompt must not contain SERVER-STARTUP RECOVERY" + ) + assert "Wait 3 seconds" not in fn, f"{name} prompt must not contain 'Wait 3 seconds'" + + +def test_orchestrator_prompt_has_no_deferred_tool_recovery_conditional(): + """The conditional DEFERRED-TOOL RECOVERY block must not be present.""" + from autoskillit.cli._mcp_names import DIRECT_PREFIX + from autoskillit.cli._prompts import _build_orchestrator_prompt + + prompt = _build_orchestrator_prompt("test", mcp_prefix=DIRECT_PREFIX) + assert "schemas NOT loaded — calling directly will fail" not in prompt + + +def test_first_action_no_toolsearch_or_bash(): + """FIRST ACTION must not reference ToolSearch or Bash.""" from autoskillit.cli._mcp_names import DIRECT_PREFIX from autoskillit.cli._prompts import _build_orchestrator_prompt prompt = _build_orchestrator_prompt("my_recipe", mcp_prefix=DIRECT_PREFIX) - assert "ToolSearch" in prompt - assert "deferred" in prompt.lower() - assert "select:" in prompt - assert "open_kitchen" in prompt + start = prompt.index("FIRST ACTION") + end = prompt.index("During pipeline execution", start) + first_action = prompt[start:end] + assert "ToolSearch" not in first_action + assert "Bash" not in first_action + assert "sleep" not in first_action.lower() -def test_open_kitchen_prompt_instructs_toolsearch_when_deferred(): +def test_first_action_opens_with_open_kitchen(): + """FIRST ACTION step 1 must call open_kitchen directly — no preamble step.""" + from autoskillit.cli._mcp_names import DIRECT_PREFIX + from autoskillit.cli._prompts import _build_orchestrator_prompt + + prompt = _build_orchestrator_prompt("my_recipe", mcp_prefix=DIRECT_PREFIX) + start = prompt.index("FIRST ACTION") + end = prompt.index("During pipeline execution", start) + first_action = prompt[start:end] + assert "open_kitchen" in first_action + assert "\n0." not in first_action, "Step 0 must not exist — open_kitchen is step 1" + assert "\n1." in first_action + + +def test_open_kitchen_prompt_no_toolsearch_or_bash(): + """open_kitchen call instruction must not reference ToolSearch or Bash.""" from autoskillit.cli._mcp_names import DIRECT_PREFIX from autoskillit.cli._prompts import _build_open_kitchen_prompt prompt = _build_open_kitchen_prompt(mcp_prefix=DIRECT_PREFIX) - assert "ToolSearch" in prompt - assert "deferred" in prompt.lower() - assert "open_kitchen" in prompt + # Scope to the call instruction before the discipline block + discipline_idx = prompt.index("IMPORTANT") + preamble = prompt[:discipline_idx] + assert "ToolSearch" not in preamble + assert "Bash" not in preamble + assert "sleep" not in preamble.lower() -def test_prompt_builder_includes_startup_retry(): - """Prompt must include retry instruction for 'No such tool available'.""" +def test_open_kitchen_prompt_calls_open_kitchen_directly(): + """_build_open_kitchen_prompt must instruct a direct open_kitchen call.""" from autoskillit.cli._mcp_names import DIRECT_PREFIX + from autoskillit.cli._prompts import _build_open_kitchen_prompt + + prompt = _build_open_kitchen_prompt(mcp_prefix=DIRECT_PREFIX) + assert "open_kitchen" in prompt + + +def test_orchestrator_prompt_contains_anti_skip_rule(): from autoskillit.cli._prompts import _build_orchestrator_prompt prompt = _build_orchestrator_prompt("test", mcp_prefix=DIRECT_PREFIX) - assert "SERVER-STARTUP RECOVERY" in prompt - assert "No such tool available" in prompt + assert "STEP EXECUTION IS NOT DISCRETIONARY" in prompt + assert "NEVER skip a step because" in prompt + + +def test_open_kitchen_prompt_contains_anti_skip_rule(): + from autoskillit.cli._prompts import _build_open_kitchen_prompt + + prompt = _build_open_kitchen_prompt(mcp_prefix=DIRECT_PREFIX) + assert "STEP EXECUTION IS NOT DISCRETIONARY" in prompt + assert "NEVER skip a step because" in prompt + + +def test_orchestrator_prompt_closes_optional_semantics(): + """OPTIONAL STEP SEMANTICS must state that skip_when_false=false is the ONLY skip reason.""" + from autoskillit.cli._prompts import _build_orchestrator_prompt + + prompt = _build_orchestrator_prompt("test", mcp_prefix=DIRECT_PREFIX) + idx = prompt.index("OPTIONAL STEP SEMANTICS") + section = prompt[idx : idx + 500] + assert "ONLY" in section + + +# ING-1 +def test_build_orchestrator_prompt_injects_ingredients_table_when_provided(): + """When ingredients_table is supplied, it appears verbatim in the prompt.""" + from autoskillit.cli._prompts import _build_orchestrator_prompt + + table = "| Name | Description | Default |\n|------|-------------|---------|" + prompt = _build_orchestrator_prompt( + "my-recipe", mcp_prefix=DIRECT_PREFIX, ingredients_table=table + ) + assert table in prompt, "ingredients_table content must appear verbatim in the prompt" + + +# ING-2 +def test_build_orchestrator_prompt_omits_ingredients_section_when_none(): + """When ingredients_table is None (default), no RECIPE INGREDIENTS section is injected.""" + from autoskillit.cli._prompts import _build_orchestrator_prompt + + prompt = _build_orchestrator_prompt("my-recipe", mcp_prefix=DIRECT_PREFIX) + assert "RECIPE INGREDIENTS" not in prompt, ( + "No RECIPE INGREDIENTS section when ingredients_table is None" + ) + + +# ING-3 +def test_build_orchestrator_prompt_first_action_mentions_tool_activation(): + """FIRST ACTION section must clarify open_kitchen is required for tool activation.""" + from autoskillit.cli._prompts import _build_orchestrator_prompt + + prompt = _build_orchestrator_prompt("my-recipe", mcp_prefix=DIRECT_PREFIX) + first_action_start = prompt.index("FIRST ACTION") + first_action_end = prompt.index("During pipeline execution", first_action_start) + section = prompt[first_action_start:first_action_end] + assert "tool" in section.lower() and ( + "activat" in section.lower() or "enable" in section.lower() + ), "FIRST ACTION must clarify open_kitchen is required for tool activation/enabling" + + +# ING-4 +def test_build_orchestrator_prompt_ingredients_section_before_first_action(): + """RECIPE INGREDIENTS section must appear before FIRST ACTION so LLM sees names first.""" + from autoskillit.cli._prompts import _build_orchestrator_prompt + + table = "| task * | What to do | (required) |" + prompt = _build_orchestrator_prompt("impl", mcp_prefix=DIRECT_PREFIX, ingredients_table=table) + ing_idx = prompt.index("RECIPE INGREDIENTS") + first_action_idx = prompt.index("FIRST ACTION") + assert ing_idx < first_action_idx, ( + "RECIPE INGREDIENTS section must appear before FIRST ACTION in the prompt" + ) + + +def test_orchestrator_prompt_contains_skill_command_format_guidance(): + """Orchestrator prompt must instruct the LLM that skill_command is a literal template.""" + from autoskillit.cli._mcp_names import DIRECT_PREFIX + from autoskillit.cli._prompts import _build_orchestrator_prompt + + prompt = _build_orchestrator_prompt("my-recipe", mcp_prefix=DIRECT_PREFIX) + assert "SKILL_COMMAND FORMATTING" in prompt, ( + "Orchestrator prompt must contain a SKILL_COMMAND FORMATTING section. " + "This prevents the LLM from adding markdown headers to skill_command values." + ) + + +def test_orchestrator_prompt_includes_null_context_handling(): + """System prompt must instruct the model on null/None context variable behavior.""" + from autoskillit.cli._mcp_names import DIRECT_PREFIX + from autoskillit.cli._prompts import _build_orchestrator_prompt + + prompt = _build_orchestrator_prompt("my-recipe", mcp_prefix=DIRECT_PREFIX) + assert "NULL/NONE CONTEXT VARIABLES" in prompt, ( + "Orchestrator prompt must contain a NULL/NONE CONTEXT VARIABLES section that " + "instructs the model not to guess or substitute when context values are null." + ) + assert "do not guess" in prompt.lower(), ( + "Null handling section must explicitly instruct the model not to guess values " + "for null/None context variables." + ) + + +def test_campaign_prompt_tool_claim_has_after_startup_qualifier(): + """Fleet campaign prompt must qualify the 6-tool claim as applying after startup only.""" + from unittest.mock import MagicMock + + from autoskillit.cli._prompts import _build_fleet_campaign_prompt + + recipe = MagicMock() + recipe.name = "test-campaign" + recipe.description = "Test" + recipe.dispatches = [MagicMock()] + recipe.continue_on_failure = False + + prompt = _build_fleet_campaign_prompt( + campaign_recipe=recipe, + manifest_yaml="dispatches: []", + completed_dispatches="", + mcp_prefix="mcp__autoskillit__", + campaign_id="abc-123", + ) + + assert "Only these 6 tools are available in this session" not in prompt, ( + "The absolute 'only 6 tools available' claim is factually false during startup " + "— must be qualified as 'after startup' or 'for campaign operations'" + ) + lower = prompt.lower() + assert "after startup" in lower or "startup" in lower or "operational" in lower, ( + "Campaign prompt must qualify the tool list as post-startup / operational tools" + ) + + +def test_orchestrator_prompt_documents_stop_action(): + """The orchestrator system prompt must explain how to handle action:stop steps.""" + from autoskillit.cli._prompts import _build_orchestrator_prompt + + prompt = _build_orchestrator_prompt("my-recipe", mcp_prefix=DIRECT_PREFIX) + assert 'action: "stop"' in prompt or "action: stop" in prompt + assert "TERMINATE" in prompt.upper() or "terminate" in prompt + assert "message" in prompt + + +def test_orchestrator_prompt_documents_route_action(): + """The orchestrator system prompt must explain how to handle action:route steps.""" + from autoskillit.cli._prompts import _build_orchestrator_prompt + + prompt = _build_orchestrator_prompt("my-recipe", mcp_prefix=DIRECT_PREFIX) + assert 'action: "route"' in prompt or "action: route" in prompt + assert "on_result" in prompt + assert "Do NOT call any MCP tool" in prompt or "no tool call" in prompt.lower() + + +def test_orchestrator_prompt_contains_hook_denial_compliance(): + """The orchestrator prompt must teach the model that ALL hook denials are mandatory.""" + from autoskillit.cli._prompts import _build_orchestrator_prompt + + prompt = _build_orchestrator_prompt("my-recipe", mcp_prefix=DIRECT_PREFIX) + assert "HOOK DENIAL" in prompt.upper() + assert "MANDATORY" in prompt.upper() or "mandatory" in prompt + assert "permissionDecision" in prompt or "deny" in prompt.lower() + + +@pytest.mark.parametrize("action_type", ["stop", "confirm", "route"]) +def test_orchestrator_prompt_documents_all_action_types(action_type): + """Every recognized action type must have explicit behavioral semantics.""" + from autoskillit.cli._prompts import _build_orchestrator_prompt + + prompt = _build_orchestrator_prompt("my-recipe", mcp_prefix=DIRECT_PREFIX) + assert f'action: "{action_type}"' in prompt or f"action: {action_type}" in prompt + + +def test_campaign_prompt_tool_list_still_enumerates_six_tools(): + """The 6 operational tools must still be listed in the campaign prompt after the fix.""" + from unittest.mock import MagicMock + + from autoskillit.cli._prompts import _build_fleet_campaign_prompt + + recipe = MagicMock() + recipe.name = "test-campaign" + recipe.description = "Test" + recipe.dispatches = [MagicMock()] + recipe.continue_on_failure = False + + prompt = _build_fleet_campaign_prompt( + campaign_recipe=recipe, + manifest_yaml="dispatches: []", + completed_dispatches="", + mcp_prefix="mcp__autoskillit__", + campaign_id="abc-123", + ) + assert "dispatch_food_truck" in prompt + assert "batch_cleanup_clones" in prompt + assert "get_pipeline_report" in prompt + assert "get_token_summary" in prompt + assert "get_timing_summary" in prompt + assert "get_quota_events" in prompt + + +def test_campaign_prompt_includes_gate_dispatch_handling_section(): + """Campaign prompt includes GATE DISPATCH HANDLING section when a gate dispatch is present.""" + from autoskillit.cli._prompts import _build_fleet_campaign_prompt + from autoskillit.recipe.schema import CampaignDispatch, Recipe, RecipeKind + + dispatch = CampaignDispatch(name="gate-check", gate="confirm", message="Proceed?") + recipe = Recipe( + name="test-campaign", + description="Test", + kind=RecipeKind.CAMPAIGN, + dispatches=[dispatch], + continue_on_failure=False, + ) + + prompt = _build_fleet_campaign_prompt( + campaign_recipe=recipe, + manifest_yaml="...", + completed_dispatches="", + mcp_prefix="mcp__autoskillit__", + campaign_id="abc-123", + ) + assert "GATE DISPATCH HANDLING" in prompt + assert "AskUserQuestion" in prompt + assert "dispatch_food_truck" in prompt + + +def test_show_cook_preview_no_mermaid_in_output( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, + capsys: pytest.CaptureFixture[str], +) -> None: + """T-PREVIEW-FORMAT: show_cook_preview must not leak Mermaid syntax to terminal.""" + from autoskillit.cli._preview import show_cook_preview + from autoskillit.core import pkg_root + from autoskillit.recipe.io import find_recipe_by_name, load_recipe + + monkeypatch.setenv("NO_COLOR", "1") + recipes_dir = pkg_root() / "recipes" + recipe_info = find_recipe_by_name("implementation", recipes_dir) + assert recipe_info is not None + parsed = load_recipe(recipe_info.path) + show_cook_preview("implementation", parsed, recipes_dir, tmp_path) + captured = capsys.readouterr() + assert "flowchart TD" not in captured.out, "Mermaid syntax leaked to terminal" + assert "S0[" not in captured.out, "Mermaid node syntax leaked to terminal" + assert "-->" not in captured.out, "Mermaid arrow syntax leaked to terminal" + + +def test_orchestrator_prompt_no_resume_session_id_in_context_limit_routing(): + """on_context_limit routing must NOT instruct passing resume_session_id. + + Context-exhausted sessions should never be resumed — the retry must start + a fresh session to get a full context window. + """ + from autoskillit.cli._prompts import _build_orchestrator_prompt + + prompt = _build_orchestrator_prompt("implementation", mcp_prefix=DIRECT_PREFIX) + ctx_section_start = prompt.find("CONTEXT LIMIT ROUTING") + assert ctx_section_start != -1 + ctx_section = prompt[ctx_section_start : ctx_section_start + 2000] + assert "resume_session_id" not in ctx_section, ( + "on_context_limit routing must not instruct L2 to pass resume_session_id; " + "context-exhausted sessions must start fresh" + ) + + +class TestPromptsReExporter: + """Guard: _prompts.py re-exports every public symbol from submodules.""" + + def test_reexporter_exposes_all_symbols(self) -> None: + from autoskillit.cli import ( + _prompts, + _prompts_campaign, + _prompts_kitchen, + _prompts_orchestrator, + ) + + for mod in (_prompts_campaign, _prompts_orchestrator, _prompts_kitchen): + for name in mod.__all__: + assert hasattr(_prompts, name), ( + f"_prompts.py missing re-export: {name!r} from {mod.__name__}" + ) + assert getattr(_prompts, name) is getattr(mod, name), ( + f"_prompts.py re-exports {name!r} as a different object " + f"than {mod.__name__}.{name}" + ) + + def test_submodules_importable_independently(self) -> None: + import importlib + + for name in ( + "autoskillit.cli._prompts_campaign", + "autoskillit.cli._prompts_orchestrator", + "autoskillit.cli._prompts_kitchen", + ): + mod = importlib.import_module(name) + assert hasattr(mod, "__all__"), f"{name} missing __all__" + assert len(mod.__all__) > 0, f"{name}.__all__ is empty" diff --git a/tests/cli/test_cli_serve_logging.py b/tests/cli/test_cli_serve_logging.py index e77c359d98..47131f268f 100644 --- a/tests/cli/test_cli_serve_logging.py +++ b/tests/cli/test_cli_serve_logging.py @@ -5,8 +5,11 @@ import logging as _stdlib_logging from unittest.mock import patch +import pytest import structlog.testing +pytestmark = [pytest.mark.layer("cli"), pytest.mark.small] + class TestServeLoggingPhases: """Verify serve() calls configure_logging twice: early init then config-driven.""" diff --git a/tests/cli/test_compute_exit_code.py b/tests/cli/test_compute_exit_code.py new file mode 100644 index 0000000000..cc9bce6450 --- /dev/null +++ b/tests/cli/test_compute_exit_code.py @@ -0,0 +1,57 @@ +"""Tests for _compute_exit_code in cli/_fleet.py (T-RESUMABLE-9).""" + +from __future__ import annotations + +import time + +import pytest + +from autoskillit.fleet import CampaignState, DispatchRecord, DispatchStatus + +pytestmark = [pytest.mark.layer("cli"), pytest.mark.small, pytest.mark.feature("fleet")] + + +def _make_state(*statuses: DispatchStatus) -> CampaignState: + return CampaignState( + schema_version=3, + campaign_id="test-cid", + campaign_name="test", + manifest_path="", + started_at=time.time(), + dispatches=[DispatchRecord(name=f"d{i}", status=s) for i, s in enumerate(statuses)], + ) + + +def test_compute_exit_code_all_success_returns_0() -> None: + from autoskillit.cli.fleet import _compute_exit_code + + state = _make_state(DispatchStatus.SUCCESS, DispatchStatus.SKIPPED) + assert _compute_exit_code(state) == 0 + + +def test_compute_exit_code_resumable_returns_2() -> None: + from autoskillit.cli.fleet import _compute_exit_code + + state = _make_state(DispatchStatus.SUCCESS, DispatchStatus.RESUMABLE) + assert _compute_exit_code(state) == 2 + + +def test_compute_exit_code_failure_beats_resumable() -> None: + from autoskillit.cli.fleet import _compute_exit_code + + state = _make_state(DispatchStatus.FAILURE, DispatchStatus.RESUMABLE) + assert _compute_exit_code(state) == 1 + + +def test_compute_exit_code_interrupted_returns_1() -> None: + from autoskillit.cli.fleet import _compute_exit_code + + state = _make_state(DispatchStatus.SUCCESS, DispatchStatus.INTERRUPTED) + assert _compute_exit_code(state) == 1 + + +def test_compute_exit_code_pending_returns_2() -> None: + from autoskillit.cli.fleet import _compute_exit_code + + state = _make_state(DispatchStatus.SUCCESS, DispatchStatus.PENDING) + assert _compute_exit_code(state) == 2 diff --git a/tests/cli/test_cook.py b/tests/cli/test_cook.py deleted file mode 100644 index bddbf16cb7..0000000000 --- a/tests/cli/test_cook.py +++ /dev/null @@ -1,1429 +0,0 @@ -"""Tests for CLI order, workspace, and skills list commands.""" - -from __future__ import annotations - -import os -import shutil -import subprocess -import time -from pathlib import Path -from unittest.mock import MagicMock, patch - -import pytest - -from autoskillit import cli -from autoskillit.cli._prompts import _OPEN_KITCHEN_CHOICE, _resolve_recipe_input -from autoskillit.cli._workspace import _format_age -from autoskillit.core import ClaudeFlags - -_SCRIPT_YAML = """\ -name: test-script -description: A test script -summary: Test flow -ingredients: - target: - description: Target path - required: true -steps: - do-something: - tool: run_cmd - with: - cmd: echo hello - on_success: done - on_failure: done - done: - action: stop - message: Finished -kitchen_rules: - - Only use AutoSkillit MCP tools during pipeline execution -""" - - -class TestResolveRecipeInput: - """Unit tests for the _resolve_recipe_input picker resolution helper.""" - - def _make_recipe(self, name: str) -> MagicMock: - r = MagicMock() - r.name = name - return r - - def test_zero_returns_open_kitchen_sentinel(self) -> None: - available = [self._make_recipe("smoke-test")] - result = _resolve_recipe_input("0", available) - assert result is _OPEN_KITCHEN_CHOICE - - def test_zero_with_empty_list_returns_open_kitchen_sentinel(self) -> None: - result = _resolve_recipe_input("0", []) - assert result is _OPEN_KITCHEN_CHOICE - - def test_valid_number_first_returns_first_recipe(self) -> None: - r1 = self._make_recipe("implementation") - r2 = self._make_recipe("remediation") - assert _resolve_recipe_input("1", [r1, r2]) is r1 - - def test_valid_number_last_returns_last_recipe(self) -> None: - r1 = self._make_recipe("implementation") - r2 = self._make_recipe("remediation") - assert _resolve_recipe_input("2", [r1, r2]) is r2 - - def test_out_of_range_too_high_returns_none(self) -> None: - available = [self._make_recipe("smoke-test")] - assert _resolve_recipe_input("99", available) is None - - def test_out_of_range_negative_digit_treated_as_name(self) -> None: - # "-1".isdigit() is False in Python — treated as name lookup, returns None - available = [self._make_recipe("smoke-test")] - assert _resolve_recipe_input("-1", available) is None - - def test_name_match_returns_recipe(self) -> None: - r = self._make_recipe("smoke-test") - assert _resolve_recipe_input("smoke-test", [r, self._make_recipe("other")]) is r - - def test_name_no_match_returns_none(self) -> None: - available = [self._make_recipe("smoke-test")] - assert _resolve_recipe_input("nonexistent", available) is None - - def test_empty_string_returns_none(self) -> None: - available = [self._make_recipe("smoke-test")] - assert _resolve_recipe_input("", available) is None - - -class TestCLIOrder: - @pytest.fixture(autouse=True) - def _stub_preview(self, monkeypatch: pytest.MonkeyPatch) -> None: - """Stub terminal preview to avoid subprocess.run collision with git calls.""" - monkeypatch.setattr( - "autoskillit.cli._prompts.show_cook_preview", - lambda *a, **kw: None, - ) - - @pytest.fixture(autouse=True) - def _interactive_stdin(self, monkeypatch: pytest.MonkeyPatch) -> None: - """Most order() paths require an interactive TTY — default to True for this class.""" - monkeypatch.setattr("sys.stdin.isatty", lambda: True) - - @pytest.fixture(autouse=True) - def _stub_is_plugin_installed(self, monkeypatch: pytest.MonkeyPatch) -> None: - """Stub _is_plugin_installed in app.py to False to prevent subprocess.run collision.""" - import importlib - import sys as _sys - - _app_mod = _sys.modules.get("autoskillit.cli.app") or importlib.import_module( - "autoskillit.cli.app" - ) - monkeypatch.setattr(_app_mod, "_is_plugin_installed", lambda: False) - - # --- workspace init --- - - def test_prep_station_init_creates_dir_with_marker( - self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch - ) -> None: - """prep station init creates directory and drops marker file.""" - monkeypatch.chdir(tmp_path) - target = tmp_path / "test-workspace" - cli.workspace_init(str(target)) - assert target.is_dir() - assert (target / ".autoskillit-workspace").is_file() - - def test_prep_station_init_refuses_nonempty_dir( - self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch - ) -> None: - """prep station init refuses to initialize a non-empty directory.""" - monkeypatch.chdir(tmp_path) - target = tmp_path / "existing" - target.mkdir() - (target / "important.txt").touch() - with pytest.raises(SystemExit): - cli.workspace_init(str(target)) - - def test_prep_station_init_idempotent_on_empty_with_marker( - self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch - ) -> None: - """prep station init is safe to re-run on a directory that only has the marker.""" - monkeypatch.chdir(tmp_path) - target = tmp_path / "prep-station" - cli.workspace_init(str(target)) - cli.workspace_init(str(target)) # second run — should not fail - assert (target / ".autoskillit-workspace").is_file() - - def test_prep_station_init_marker_has_content( - self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch - ) -> None: - """Marker file contains human-readable identifying content.""" - monkeypatch.chdir(tmp_path) - target = tmp_path / "prep-station" - cli.workspace_init(str(target)) - content = (target / ".autoskillit-workspace").read_text() - assert "autoskillit" in content - assert "do not delete" in content - - # --- T_WC: workspace clean --- - - def test_workspace_clean_removes_subdirs( - self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture - ) -> None: - """T_WC1: workspace_clean removes stale subdirs of autoskillit-runs/.""" - runs_dir = tmp_path / "autoskillit-runs" - (runs_dir / "run-a").mkdir(parents=True) - (runs_dir / "run-b").mkdir(parents=True) - old_time = time.time() - 6 * 3600 - for d in (runs_dir / "run-a", runs_dir / "run-b"): - os.utime(d, (old_time, old_time)) - cli.workspace_clean(dir=str(tmp_path), force=True) - assert not (runs_dir / "run-a").exists() - assert not (runs_dir / "run-b").exists() - - def test_workspace_clean_reports_nothing_when_no_runs_dir( - self, tmp_path: Path, capsys: pytest.CaptureFixture - ) -> None: - """T_WC2: workspace_clean prints message when autoskillit-runs/ doesn't exist.""" - cli.workspace_clean(dir=str(tmp_path)) - captured = capsys.readouterr() - assert "No autoskillit-runs/" in captured.out - - def test_workspace_clean_defaults_to_parent_of_cwd( - self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch - ) -> None: - """T_WC3: workspace_clean without --dir uses parent of CWD.""" - project_dir = tmp_path / "project" - project_dir.mkdir() - monkeypatch.chdir(project_dir) - runs_dir = tmp_path / "autoskillit-runs" - (runs_dir / "run-x").mkdir(parents=True) - old_time = time.time() - 6 * 3600 - os.utime(runs_dir / "run-x", (old_time, old_time)) - cli.workspace_clean(force=True) - assert not (runs_dir / "run-x").exists() - - def test_workspace_clean_recent_dirs_skipped( - self, tmp_path: Path, capsys: pytest.CaptureFixture - ) -> None: - """T_WC4: recent dirs (<5h) are not deleted, stale dirs (>5h) are.""" - runs_dir = tmp_path / "autoskillit-runs" - (runs_dir / "recent").mkdir(parents=True) - (runs_dir / "stale").mkdir(parents=True) - old_time = time.time() - 6 * 3600 - os.utime(runs_dir / "stale", (old_time, old_time)) - cli.workspace_clean(dir=str(tmp_path), force=True) - assert (runs_dir / "recent").exists() - assert not (runs_dir / "stale").exists() - - def test_workspace_clean_boundary_5h_is_stale( - self, tmp_path: Path, capsys: pytest.CaptureFixture - ) -> None: - """T_WC5: dir with mtime exactly 5h ago is stale (>=5h threshold).""" - runs_dir = tmp_path / "autoskillit-runs" - (runs_dir / "boundary").mkdir(parents=True) - boundary_time = time.time() - 5 * 3600 - os.utime(runs_dir / "boundary", (boundary_time, boundary_time)) - cli.workspace_clean(dir=str(tmp_path), force=True) - assert not (runs_dir / "boundary").exists() - - def test_workspace_clean_skipped_dirs_printed( - self, tmp_path: Path, capsys: pytest.CaptureFixture - ) -> None: - """T_WC6: skipped (recent) dirs are printed with age.""" - runs_dir = tmp_path / "autoskillit-runs" - (runs_dir / "recent-one").mkdir(parents=True) - recent_time = time.time() - 3600 - os.utime(runs_dir / "recent-one", (recent_time, recent_time)) - cli.workspace_clean(dir=str(tmp_path), force=True) - captured = capsys.readouterr() - assert "Skipped" in captured.out - assert "recent-one" in captured.out - - def test_workspace_clean_will_remove_printed( - self, tmp_path: Path, capsys: pytest.CaptureFixture - ) -> None: - """T_WC7: stale dirs are printed as 'Will remove' with age.""" - runs_dir = tmp_path / "autoskillit-runs" - (runs_dir / "old-one").mkdir(parents=True) - old_time = time.time() - 10 * 3600 - os.utime(runs_dir / "old-one", (old_time, old_time)) - cli.workspace_clean(dir=str(tmp_path), force=True) - captured = capsys.readouterr() - assert "Will remove" in captured.out - assert "old-one" in captured.out - - def test_workspace_clean_confirm_defaults_no( - self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch - ) -> None: - """T_WC8: empty input at confirmation prompt defaults to N (no deletion).""" - runs_dir = tmp_path / "autoskillit-runs" - (runs_dir / "stale").mkdir(parents=True) - old_time = time.time() - 6 * 3600 - os.utime(runs_dir / "stale", (old_time, old_time)) - monkeypatch.setattr("builtins.input", lambda _prompt="": "") - cli.workspace_clean(dir=str(tmp_path)) - assert (runs_dir / "stale").exists() - - def test_workspace_clean_confirm_accepts_y( - self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch - ) -> None: - """T_WC9: 'y' at confirmation prompt deletes stale dirs.""" - runs_dir = tmp_path / "autoskillit-runs" - (runs_dir / "stale").mkdir(parents=True) - old_time = time.time() - 6 * 3600 - os.utime(runs_dir / "stale", (old_time, old_time)) - monkeypatch.setattr("builtins.input", lambda _prompt="": "y") - cli.workspace_clean(dir=str(tmp_path)) - assert not (runs_dir / "stale").exists() - - def test_workspace_clean_force_skips_prompt( - self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch - ) -> None: - """T_WC10: --force deletes without calling input().""" - runs_dir = tmp_path / "autoskillit-runs" - (runs_dir / "stale").mkdir(parents=True) - old_time = time.time() - 6 * 3600 - os.utime(runs_dir / "stale", (old_time, old_time)) - monkeypatch.setattr( - "builtins.input", - lambda _prompt="": (_ for _ in ()).throw(AssertionError("input() called")), - ) - cli.workspace_clean(dir=str(tmp_path), force=True) - assert not (runs_dir / "stale").exists() - - def test_workspace_clean_no_stale_nothing_to_clean( - self, tmp_path: Path, capsys: pytest.CaptureFixture - ) -> None: - """T_WC11: only recent dirs prints nothing-to-clean message.""" - runs_dir = tmp_path / "autoskillit-runs" - (runs_dir / "recent").mkdir(parents=True) - cli.workspace_clean(dir=str(tmp_path), force=True) - captured = capsys.readouterr() - assert "Nothing to clean" in captured.out - assert (runs_dir / "recent").exists() - - def test_format_age_various_values(self) -> None: - """T_WC14: _format_age returns human-readable age strings.""" - assert _format_age(30 * 60) == "30m ago" - assert _format_age(2 * 3600 + 14 * 60) == "2h 14m ago" - assert _format_age(3 * 86400) == "3d ago" - assert _format_age(5 * 3600) == "5h ago" - - # --- T6: skills list CLI output --- - - def test_skills_list_shows_all_sources( - self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture - ) -> None: - """skills list outputs skill names with source labels.""" - monkeypatch.chdir(tmp_path) - cli.skills_list() - captured = capsys.readouterr() - assert "investigate" in captured.out - assert "bundled" in captured.out - assert "NAME" in captured.out - - # --- order --- - - def test_order_blocked_inside_claude_session( - self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture - ) -> None: - """order exits 1 when CLAUDECODE env var is set.""" - monkeypatch.chdir(tmp_path) - monkeypatch.setenv("CLAUDECODE", "1") - with pytest.raises(SystemExit) as exc_info: - cli.order("any-script") - assert exc_info.value.code == 1 - captured = capsys.readouterr() - assert "regular terminal" in captured.out.lower() - - def test_order_script_not_found( - self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture - ) -> None: - """order exits 1 when script name doesn't match any entry.""" - monkeypatch.delenv("CLAUDECODE", raising=False) - monkeypatch.chdir(tmp_path) - scripts_dir = tmp_path / ".autoskillit" / "recipes" - scripts_dir.mkdir(parents=True) - - with pytest.raises(SystemExit) as exc_info: - cli.order("nonexistent") - assert exc_info.value.code == 1 - captured = capsys.readouterr() - assert "nonexistent" in captured.out - - def test_order_no_scripts_dir( - self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture - ) -> None: - """order exits 1 with available bundled recipes listed when name not found.""" - monkeypatch.delenv("CLAUDECODE", raising=False) - monkeypatch.chdir(tmp_path) - - with pytest.raises(SystemExit) as exc_info: - cli.order("anything") - assert exc_info.value.code == 1 - captured = capsys.readouterr() - assert "Recipe not found: 'anything'" in captured.out - assert "Available recipes:" in captured.out - assert "implementation" in captured.out - - def test_order_available_scripts_listed( - self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture - ) -> None: - """order lists available scripts when name doesn't match.""" - monkeypatch.delenv("CLAUDECODE", raising=False) - monkeypatch.chdir(tmp_path) - scripts_dir = tmp_path / ".autoskillit" / "recipes" - scripts_dir.mkdir(parents=True) - (scripts_dir / "my-script.yaml").write_text(_SCRIPT_YAML) - - with pytest.raises(SystemExit) as exc_info: - cli.order("nonexistent") - assert exc_info.value.code == 1 - captured = capsys.readouterr() - assert "Available recipes:" in captured.out - assert "test-script" in captured.out - - def test_order_claude_not_on_path( - self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture - ) -> None: - """order exits 1 when claude command is not found.""" - monkeypatch.delenv("CLAUDECODE", raising=False) - monkeypatch.chdir(tmp_path) - scripts_dir = tmp_path / ".autoskillit" / "recipes" - scripts_dir.mkdir(parents=True) - (scripts_dir / "my-script.yaml").write_text(_SCRIPT_YAML) - monkeypatch.setattr(shutil, "which", lambda cmd: None) - monkeypatch.setattr("builtins.input", lambda _prompt="": "") - - with pytest.raises(SystemExit) as exc_info: - cli.order("test-script") - assert exc_info.value.code == 1 - captured = capsys.readouterr() - assert "claude" in captured.out.lower() - - def test_order_invalid_script_exits( - self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture - ) -> None: - """order exits 1 when script YAML fails validation.""" - monkeypatch.delenv("CLAUDECODE", raising=False) - monkeypatch.chdir(tmp_path) - scripts_dir = tmp_path / ".autoskillit" / "recipes" - scripts_dir.mkdir(parents=True) - (scripts_dir / "bad-script.yaml").write_text("name: bad-script\nsteps: {}\n") - - with pytest.raises(SystemExit) as exc_info: - cli.order("bad-script") - assert exc_info.value.code == 1 - captured = capsys.readouterr() - assert "validation" in captured.out.lower() or "error" in captured.out.lower() - - @patch("autoskillit.cli.subprocess.run") - def test_order_builds_correct_command( - self, - mock_run: MagicMock, - tmp_path: Path, - monkeypatch: pytest.MonkeyPatch, - ) -> None: - """order passes correct flags to subprocess.run.""" - monkeypatch.delenv("CLAUDECODE", raising=False) - monkeypatch.chdir(tmp_path) - scripts_dir = tmp_path / ".autoskillit" / "recipes" - scripts_dir.mkdir(parents=True) - (scripts_dir / "my-script.yaml").write_text(_SCRIPT_YAML) - monkeypatch.setattr(shutil, "which", lambda cmd: "/usr/bin/claude") - monkeypatch.setattr("builtins.input", lambda _prompt="": "") - mock_run.return_value = subprocess.CompletedProcess( - args=[], returncode=0, stdout="", stderr="" - ) - - cli.order("test-script") - - mock_run.assert_called_once() - cmd = mock_run.call_args[0][0] - assert cmd[0] == "claude" - assert ClaudeFlags.PLUGIN_DIR in cmd - plugin_dir_idx = cmd.index(ClaudeFlags.PLUGIN_DIR) - plugin_dir_val = Path(cmd[plugin_dir_idx + 1]) - assert plugin_dir_val.is_dir() - assert (plugin_dir_val / ".claude-plugin" / "plugin.json").is_file() - assert ClaudeFlags.TOOLS in cmd - tools_idx = cmd.index(ClaudeFlags.TOOLS) - assert cmd[tools_idx + 1] == "AskUserQuestion" - assert ClaudeFlags.APPEND_SYSTEM_PROMPT in cmd - assert ClaudeFlags.DANGEROUSLY_SKIP_PERMISSIONS in cmd - assert ClaudeFlags.PRINT not in cmd - assert ClaudeFlags.ALLOW_DANGEROUSLY_SKIP_PERMISSIONS not in cmd - kwargs = mock_run.call_args[1] if mock_run.call_args[1] else {} - assert "capture_output" not in kwargs - assert "stdin" not in kwargs - - @patch("autoskillit.cli.subprocess.run") - def test_order_suppresses_plugin_dir_when_plugin_installed( - self, - mock_run: MagicMock, - tmp_path: Path, - monkeypatch: pytest.MonkeyPatch, - ) -> None: - """order omits --plugin-dir when marketplace plugin is installed.""" - import importlib - import sys as _sys - - monkeypatch.delenv("CLAUDECODE", raising=False) - monkeypatch.chdir(tmp_path) - scripts_dir = tmp_path / ".autoskillit" / "recipes" - scripts_dir.mkdir(parents=True) - (scripts_dir / "my-script.yaml").write_text(_SCRIPT_YAML) - monkeypatch.setattr(shutil, "which", lambda cmd: "/usr/bin/claude") - monkeypatch.setattr("builtins.input", lambda _prompt="": "") - _app_mod = _sys.modules.get("autoskillit.cli.app") or importlib.import_module( - "autoskillit.cli.app" - ) - monkeypatch.setattr(_app_mod, "_is_plugin_installed", lambda: True) - mock_run.return_value = subprocess.CompletedProcess( - args=[], returncode=0, stdout="", stderr="" - ) - - cli.order("test-script") - - mock_run.assert_called_once() - cmd = mock_run.call_args[0][0] - assert ClaudeFlags.PLUGIN_DIR not in cmd - - @patch("autoskillit.cli.subprocess.run") - def test_order_includes_plugin_dir_when_no_plugin_installed( - self, - mock_run: MagicMock, - tmp_path: Path, - monkeypatch: pytest.MonkeyPatch, - ) -> None: - """order includes --plugin-dir when marketplace plugin is not installed.""" - import importlib - import sys as _sys - - monkeypatch.delenv("CLAUDECODE", raising=False) - monkeypatch.chdir(tmp_path) - scripts_dir = tmp_path / ".autoskillit" / "recipes" - scripts_dir.mkdir(parents=True) - (scripts_dir / "my-script.yaml").write_text(_SCRIPT_YAML) - monkeypatch.setattr(shutil, "which", lambda cmd: "/usr/bin/claude") - monkeypatch.setattr("builtins.input", lambda _prompt="": "") - _app_mod = _sys.modules.get("autoskillit.cli.app") or importlib.import_module( - "autoskillit.cli.app" - ) - monkeypatch.setattr(_app_mod, "_is_plugin_installed", lambda: False) - mock_run.return_value = subprocess.CompletedProcess( - args=[], returncode=0, stdout="", stderr="" - ) - - cli.order("test-script") - - mock_run.assert_called_once() - cmd = mock_run.call_args[0][0] - assert ClaudeFlags.PLUGIN_DIR in cmd - - @patch("autoskillit.cli.subprocess.run") - def test_order_system_prompt_contains_behavioral_instructions( - self, - mock_run: MagicMock, - tmp_path: Path, - monkeypatch: pytest.MonkeyPatch, - ) -> None: - """order injects recipe name and behavioral instructions into system prompt.""" - monkeypatch.delenv("CLAUDECODE", raising=False) - monkeypatch.chdir(tmp_path) - scripts_dir = tmp_path / ".autoskillit" / "recipes" - scripts_dir.mkdir(parents=True) - (scripts_dir / "my-script.yaml").write_text(_SCRIPT_YAML) - monkeypatch.setattr(shutil, "which", lambda cmd: "/usr/bin/claude") - monkeypatch.setattr("builtins.input", lambda _prompt="": "") - mock_run.return_value = subprocess.CompletedProcess( - args=[], returncode=0, stdout="", stderr="" - ) - - cli.order("test-script") - - cmd = mock_run.call_args[0][0] - prompt_idx = cmd.index(ClaudeFlags.APPEND_SYSTEM_PROMPT) - system_prompt = cmd[prompt_idx + 1] - assert "test-script" in system_prompt - assert "open_kitchen" in system_prompt - assert "ROUTING RULES" in system_prompt - assert "FAILURE PREDICATES" in system_prompt - assert "capture:" in system_prompt - assert "${{ context." in system_prompt - assert "AutoSkillit MCP tools" in system_prompt - assert "--- RECIPE ---" not in system_prompt - assert "do-something" not in system_prompt - - @patch("autoskillit.cli.subprocess.run") - def test_orchestrator_prompt_contains_context_limit_routing( - self, - mock_run: MagicMock, - tmp_path: Path, - monkeypatch: pytest.MonkeyPatch, - ) -> None: - """Orchestrator prompt must instruct routing to on_context_limit when needs_retry=true.""" - monkeypatch.delenv("CLAUDECODE", raising=False) - monkeypatch.chdir(tmp_path) - scripts_dir = tmp_path / ".autoskillit" / "recipes" - scripts_dir.mkdir(parents=True) - (scripts_dir / "my-script.yaml").write_text(_SCRIPT_YAML) - monkeypatch.setattr(shutil, "which", lambda cmd: "/usr/bin/claude") - monkeypatch.setattr("builtins.input", lambda _prompt="": "") - mock_run.return_value = subprocess.CompletedProcess( - args=[], returncode=0, stdout="", stderr="" - ) - - cli.order("test-script") - - cmd = mock_run.call_args[0][0] - prompt_idx = cmd.index(ClaudeFlags.APPEND_SYSTEM_PROMPT) - system_prompt = cmd[prompt_idx + 1] - assert "needs_retry" in system_prompt - assert "on_context_limit" in system_prompt - - @patch("autoskillit.cli.subprocess.run") - def test_order_propagates_exit_code( - self, - mock_run: MagicMock, - tmp_path: Path, - monkeypatch: pytest.MonkeyPatch, - ) -> None: - """order does not raise SystemExit on returncode 0.""" - monkeypatch.delenv("CLAUDECODE", raising=False) - monkeypatch.chdir(tmp_path) - scripts_dir = tmp_path / ".autoskillit" / "recipes" - scripts_dir.mkdir(parents=True) - (scripts_dir / "my-script.yaml").write_text(_SCRIPT_YAML) - monkeypatch.setattr(shutil, "which", lambda cmd: "/usr/bin/claude") - monkeypatch.setattr("builtins.input", lambda _prompt="": "") - mock_run.return_value = subprocess.CompletedProcess( - args=[], returncode=0, stdout="", stderr="" - ) - - cli.order("test-script") # should not raise - mock_run.assert_called_once() - - @patch("autoskillit.cli.subprocess.run") - def test_order_subprocess_failure_propagates( - self, - mock_run: MagicMock, - tmp_path: Path, - monkeypatch: pytest.MonkeyPatch, - ) -> None: - """order propagates non-zero subprocess exit codes.""" - monkeypatch.delenv("CLAUDECODE", raising=False) - monkeypatch.chdir(tmp_path) - scripts_dir = tmp_path / ".autoskillit" / "recipes" - scripts_dir.mkdir(parents=True) - (scripts_dir / "my-script.yaml").write_text(_SCRIPT_YAML) - monkeypatch.setattr(shutil, "which", lambda cmd: "/usr/bin/claude") - monkeypatch.setattr("builtins.input", lambda _prompt="": "") - mock_run.return_value = subprocess.CompletedProcess( - args=[], returncode=42, stdout="", stderr="" - ) - - with pytest.raises(SystemExit) as exc_info: - cli.order("test-script") - assert exc_info.value.code == 42 - - @patch("autoskillit.cli.subprocess.run") - def test_order_uses_dangerously_skip_permissions( - self, - mock_run: MagicMock, - tmp_path: Path, - monkeypatch: pytest.MonkeyPatch, - ) -> None: - """order passes --dangerously-skip-permissions to claude.""" - monkeypatch.delenv("CLAUDECODE", raising=False) - monkeypatch.chdir(tmp_path) - scripts_dir = tmp_path / ".autoskillit" / "recipes" - scripts_dir.mkdir(parents=True) - (scripts_dir / "my-script.yaml").write_text(_SCRIPT_YAML) - monkeypatch.setattr(shutil, "which", lambda cmd: "/usr/bin/claude") - monkeypatch.setattr("builtins.input", lambda _prompt="": "") - mock_run.return_value = subprocess.CompletedProcess( - args=[], returncode=0, stdout="", stderr="" - ) - - cli.order("test-script") - - cmd = mock_run.call_args[0][0] - assert ClaudeFlags.DANGEROUSLY_SKIP_PERMISSIONS in cmd - - def test_order_recipe_not_found_exits( - self, - tmp_path: Path, - monkeypatch: pytest.MonkeyPatch, - capsys: pytest.CaptureFixture, - ) -> None: - """order exits 1 when the given recipe name is not found.""" - monkeypatch.delenv("CLAUDECODE", raising=False) - monkeypatch.chdir(tmp_path) - - with pytest.raises(SystemExit) as exc_info: - cli.order("totally-unknown-recipe-xyz") - assert exc_info.value.code == 1 - captured = capsys.readouterr() - assert "totally-unknown-recipe-xyz" in captured.out - - def test_order_malformed_yaml_exits( - self, - tmp_path: Path, - monkeypatch: pytest.MonkeyPatch, - capsys: pytest.CaptureFixture, - ) -> None: - """order exits 1 with YAML parse error message when load_recipe raises YAMLError.""" - from autoskillit.core import YAMLError - - monkeypatch.delenv("CLAUDECODE", raising=False) - monkeypatch.chdir(tmp_path) - - with ( - patch("autoskillit.recipe.find_recipe_by_name", return_value=MagicMock()), - patch("autoskillit.recipe.load_recipe", side_effect=YAMLError("bad yaml")), - pytest.raises(SystemExit) as exc_info, - ): - cli.order("bad-recipe") - - assert exc_info.value.code == 1 - captured = capsys.readouterr() - assert "YAML parse error" in captured.out - - def test_order_structure_error_exits( - self, - tmp_path: Path, - monkeypatch: pytest.MonkeyPatch, - capsys: pytest.CaptureFixture, - ) -> None: - """order exits 1 with structure error message when load_recipe raises ValueError.""" - monkeypatch.delenv("CLAUDECODE", raising=False) - monkeypatch.chdir(tmp_path) - - with ( - patch("autoskillit.recipe.find_recipe_by_name", return_value=MagicMock()), - patch("autoskillit.recipe.load_recipe", side_effect=ValueError("bad structure")), - pytest.raises(SystemExit) as exc_info, - ): - cli.order("bad-recipe") - - assert exc_info.value.code == 1 - captured = capsys.readouterr() - assert "structure error" in captured.out - - @patch("autoskillit.cli.subprocess.run") - def test_order_named_recipe_only_confirmation_prompt( - self, - mock_run: MagicMock, - tmp_path: Path, - monkeypatch: pytest.MonkeyPatch, - ) -> None: - """order only fires confirmation prompt (no picker).""" - monkeypatch.delenv("CLAUDECODE", raising=False) - monkeypatch.chdir(tmp_path) - scripts_dir = tmp_path / ".autoskillit" / "recipes" - scripts_dir.mkdir(parents=True) - (scripts_dir / "my-script.yaml").write_text(_SCRIPT_YAML) - monkeypatch.setattr(shutil, "which", lambda cmd: "/usr/bin/claude") - mock_run.return_value = subprocess.CompletedProcess( - args=[], returncode=0, stdout="", stderr="" - ) - - input_calls = [] - monkeypatch.setattr("builtins.input", lambda prompt="": input_calls.append(prompt) or "") - - cli.order("test-script") - - assert len(input_calls) == 1, "input() should be called exactly once (confirmation)" - - @patch("autoskillit.cli.subprocess.run") - def test_order_command_includes_positional_greeting( - self, - mock_run: MagicMock, - tmp_path: Path, - monkeypatch: pytest.MonkeyPatch, - ) -> None: - """The order command must pass a greeting as a positional argument.""" - from autoskillit.cli._prompts import _COOK_GREETINGS - - monkeypatch.delenv("CLAUDECODE", raising=False) - monkeypatch.chdir(tmp_path) - scripts_dir = tmp_path / ".autoskillit" / "recipes" - scripts_dir.mkdir(parents=True) - (scripts_dir / "my-script.yaml").write_text(_SCRIPT_YAML) - monkeypatch.setattr(shutil, "which", lambda cmd: "/usr/bin/claude") - monkeypatch.setattr("builtins.input", lambda _prompt="": "") - mock_run.return_value = subprocess.CompletedProcess( - args=[], returncode=0, stdout="", stderr="" - ) - - cli.order("test-script") - - cmd = mock_run.call_args[0][0] - greeting_candidates = [g.format(recipe_name="test-script") for g in _COOK_GREETINGS] - assert any(arg in greeting_candidates for arg in cmd), ( - f"No greeting found as positional arg in: {cmd}" - ) - - @patch("autoskillit.cli.subprocess.run") - def test_order_no_recipe_prompts_user( - self, - mock_run: MagicMock, - tmp_path: Path, - monkeypatch: pytest.MonkeyPatch, - ) -> None: - """order prompts for recipe name when none is provided.""" - monkeypatch.delenv("CLAUDECODE", raising=False) - monkeypatch.chdir(tmp_path) - scripts_dir = tmp_path / ".autoskillit" / "recipes" - scripts_dir.mkdir(parents=True) - (scripts_dir / "my-script.yaml").write_text(_SCRIPT_YAML) - monkeypatch.setattr(shutil, "which", lambda cmd: "/usr/bin/claude") - mock_run.return_value = subprocess.CompletedProcess( - args=[], returncode=0, stdout="", stderr="" - ) - monkeypatch.setattr("builtins.input", lambda _prompt="": "test-script") - - cli.order() # no recipe argument - - mock_run.assert_called_once() - - def test_order_no_recipe_no_available_exits( - self, - tmp_path: Path, - monkeypatch: pytest.MonkeyPatch, - capsys: pytest.CaptureFixture, - ) -> None: - """order exits 1 when no recipe is given and no recipes are available.""" - from unittest.mock import MagicMock as _MagicMock - - import autoskillit.recipe as _recipe_mod - - monkeypatch.delenv("CLAUDECODE", raising=False) - monkeypatch.chdir(tmp_path) - - mock_result = _MagicMock() - mock_result.items = [] - monkeypatch.setattr(_recipe_mod, "list_recipes", lambda _: mock_result) - - with pytest.raises(SystemExit) as exc_info: - cli.order() - assert exc_info.value.code == 1 - - @patch("autoskillit.cli.subprocess.run") - def test_order_picker_shows_zero_option( - self, - mock_run: MagicMock, - tmp_path: Path, - monkeypatch: pytest.MonkeyPatch, - capsys: pytest.CaptureFixture, - ) -> None: - """Picker output includes '0. Open kitchen' line.""" - monkeypatch.delenv("CLAUDECODE", raising=False) - monkeypatch.chdir(tmp_path) - scripts_dir = tmp_path / ".autoskillit" / "recipes" - scripts_dir.mkdir(parents=True) - (scripts_dir / "my-script.yaml").write_text(_SCRIPT_YAML) - monkeypatch.setattr(shutil, "which", lambda cmd: "/usr/bin/claude") - mock_run.return_value = subprocess.CompletedProcess(args=[], returncode=0) - monkeypatch.setattr("builtins.input", lambda _prompt="": "test-script") - - cli.order() - - captured = capsys.readouterr() - assert "0. Open kitchen" in captured.out - - @patch("autoskillit.cli.subprocess.run") - def test_order_picker_zero_launches_open_kitchen( - self, - mock_run: MagicMock, - tmp_path: Path, - monkeypatch: pytest.MonkeyPatch, - ) -> None: - """Typing '0' launches a session without a recipe YAML in the system prompt.""" - monkeypatch.delenv("CLAUDECODE", raising=False) - monkeypatch.chdir(tmp_path) - scripts_dir = tmp_path / ".autoskillit" / "recipes" - scripts_dir.mkdir(parents=True) - (scripts_dir / "my-script.yaml").write_text(_SCRIPT_YAML) - monkeypatch.setattr(shutil, "which", lambda cmd: "/usr/bin/claude") - mock_run.return_value = subprocess.CompletedProcess(args=[], returncode=0) - monkeypatch.setattr("builtins.input", lambda _prompt="": "0") - - cli.order() - - mock_run.assert_called_once() - cmd = mock_run.call_args[0][0] - system_prompt_idx = cmd.index(ClaudeFlags.APPEND_SYSTEM_PROMPT) + 1 - assert "--- RECIPE ---" not in cmd[system_prompt_idx] - - def test_order_picker_out_of_range_exits( - self, - tmp_path: Path, - monkeypatch: pytest.MonkeyPatch, - capsys: pytest.CaptureFixture, - ) -> None: - """Out-of-range numeric input exits 1 with an error message.""" - import autoskillit.recipe as _recipe_mod - - monkeypatch.delenv("CLAUDECODE", raising=False) - monkeypatch.chdir(tmp_path) - - fake_recipe = MagicMock() - fake_recipe.name = "some-recipe" - mock_result = MagicMock() - mock_result.items = [fake_recipe] - monkeypatch.setattr(_recipe_mod, "list_recipes", lambda _: mock_result) - monkeypatch.setattr("builtins.input", lambda _prompt="": "99") - - with pytest.raises(SystemExit) as exc_info: - cli.order() - assert exc_info.value.code == 1 - captured = capsys.readouterr() - assert "Invalid selection" in captured.out - - @patch("autoskillit.cli.subprocess.run") - def test_order_open_kitchen_includes_positional_greeting( - self, - mock_run: MagicMock, - tmp_path: Path, - monkeypatch: pytest.MonkeyPatch, - ) -> None: - """Open-kitchen order sessions also pass a greeting as positional arg.""" - from autoskillit.cli._prompts import _OPEN_KITCHEN_GREETINGS - - monkeypatch.delenv("CLAUDECODE", raising=False) - monkeypatch.chdir(tmp_path) - scripts_dir = tmp_path / ".autoskillit" / "recipes" - scripts_dir.mkdir(parents=True) - (scripts_dir / "my-script.yaml").write_text(_SCRIPT_YAML) - monkeypatch.setattr(shutil, "which", lambda cmd: "/usr/bin/claude") - mock_run.return_value = subprocess.CompletedProcess(args=[], returncode=0) - monkeypatch.setattr("builtins.input", lambda _prompt="": "0") - - cli.order() - - cmd = mock_run.call_args[0][0] - assert any(arg in _OPEN_KITCHEN_GREETINGS for arg in cmd), ( - f"No open-kitchen greeting found as positional arg in: {cmd}" - ) - - @patch("autoskillit.cli.subprocess.run") - def test_order_resume_discovered_session_id_in_subprocess_cmd( - self, - mock_run: MagicMock, - tmp_path: Path, - monkeypatch: pytest.MonkeyPatch, - ) -> None: - """order(resume=True) discovers session via find_latest_session_id and passes --resume.""" - monkeypatch.delenv("CLAUDECODE", raising=False) - monkeypatch.chdir(tmp_path) - scripts_dir = tmp_path / ".autoskillit" / "recipes" - scripts_dir.mkdir(parents=True) - (scripts_dir / "my-script.yaml").write_text(_SCRIPT_YAML) - monkeypatch.setattr(shutil, "which", lambda cmd: "/usr/bin/claude") - monkeypatch.setattr("builtins.input", lambda _prompt="": "") - mock_run.return_value = subprocess.CompletedProcess( - args=[], returncode=0, stdout="", stderr="" - ) - - with patch("autoskillit.core.find_latest_session_id", return_value="sess_abc"): - cli.order("test-script", resume=True) - - cmd = mock_run.call_args[0][0] - assert "--resume" in cmd - assert cmd[cmd.index("--resume") + 1] == "sess_abc" - - @patch("autoskillit.cli.subprocess.run") - def test_order_resume_explicit_session_id_skips_discovery( - self, - mock_run: MagicMock, - tmp_path: Path, - monkeypatch: pytest.MonkeyPatch, - ) -> None: - """order(resume=True, session_id='explicit-abc') uses explicit id; discovery not called.""" - monkeypatch.delenv("CLAUDECODE", raising=False) - monkeypatch.chdir(tmp_path) - scripts_dir = tmp_path / ".autoskillit" / "recipes" - scripts_dir.mkdir(parents=True) - (scripts_dir / "my-script.yaml").write_text(_SCRIPT_YAML) - monkeypatch.setattr(shutil, "which", lambda cmd: "/usr/bin/claude") - monkeypatch.setattr("builtins.input", lambda _prompt="": "") - mock_run.return_value = subprocess.CompletedProcess( - args=[], returncode=0, stdout="", stderr="" - ) - discovery_calls: list = [] - - def fake_discover(cwd=None): - discovery_calls.append(cwd) - return "should-not-be-used" - - with patch("autoskillit.core.find_latest_session_id", side_effect=fake_discover): - cli.order("test-script", resume=True, session_id="explicit-abc") - - cmd = mock_run.call_args[0][0] - assert "--resume" in cmd - assert cmd[cmd.index("--resume") + 1] == "explicit-abc" - assert not discovery_calls, ( - "find_latest_session_id must not be called when session_id is explicit" - ) - - @patch("autoskillit.cli.subprocess.run") - def test_order_resume_no_prior_session_starts_fresh( - self, - mock_run: MagicMock, - tmp_path: Path, - monkeypatch: pytest.MonkeyPatch, - ) -> None: - """order(resume=True) with no prior session omits --resume from subprocess command.""" - monkeypatch.delenv("CLAUDECODE", raising=False) - monkeypatch.chdir(tmp_path) - scripts_dir = tmp_path / ".autoskillit" / "recipes" - scripts_dir.mkdir(parents=True) - (scripts_dir / "my-script.yaml").write_text(_SCRIPT_YAML) - monkeypatch.setattr(shutil, "which", lambda cmd: "/usr/bin/claude") - monkeypatch.setattr("builtins.input", lambda _prompt="": "") - mock_run.return_value = subprocess.CompletedProcess( - args=[], returncode=0, stdout="", stderr="" - ) - - with patch("autoskillit.core.find_latest_session_id", return_value=None): - cli.order("test-script", resume=True) - - cmd = mock_run.call_args[0][0] - assert "--resume" not in cmd - - -class TestOrderDisplayOwnership: - """order() delegates recipe display to the Claude session via load_recipe.""" - - @pytest.fixture(autouse=True) - def _stub_preview(self, monkeypatch: pytest.MonkeyPatch) -> None: - """Stub terminal preview to avoid subprocess.run collision with git calls.""" - monkeypatch.setattr( - "autoskillit.cli._prompts.show_cook_preview", - lambda *a, **kw: None, - ) - - def _setup_recipe(self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Path: - """Write test recipe to scripts_dir and chdir.""" - monkeypatch.delenv("CLAUDECODE", raising=False) - monkeypatch.chdir(tmp_path) - scripts_dir = tmp_path / ".autoskillit" / "recipes" - scripts_dir.mkdir(parents=True) - (scripts_dir / "my-script.yaml").write_text(_SCRIPT_YAML) - monkeypatch.setattr(shutil, "which", lambda cmd: "/usr/bin/claude") - monkeypatch.setattr("builtins.input", lambda _prompt="": "") - monkeypatch.setattr("sys.stdin.isatty", lambda: True) - return scripts_dir - - @patch("autoskillit.cli.subprocess.run") - def test_order_system_prompt_does_not_contain_recipe_yaml( - self, - mock_run: MagicMock, - tmp_path: Path, - monkeypatch: pytest.MonkeyPatch, - ) -> None: - """System prompt must contain recipe NAME but not the raw YAML body.""" - self._setup_recipe(tmp_path, monkeypatch) - mock_run.return_value = subprocess.CompletedProcess( - args=[], returncode=0, stdout="", stderr="" - ) - - cli.order("test-script") - - cmd = mock_run.call_args[0][0] - prompt_idx = cmd.index(ClaudeFlags.APPEND_SYSTEM_PROMPT) - system_prompt = cmd[prompt_idx + 1] - assert "test-script" in system_prompt - assert "--- RECIPE ---" not in system_prompt - assert "--- END RECIPE ---" not in system_prompt - assert "steps:" not in system_prompt - assert "on_success:" not in system_prompt - - @patch("autoskillit.cli.subprocess.run") - def test_order_system_prompt_instructs_open_kitchen_with_recipe( - self, - mock_run: MagicMock, - tmp_path: Path, - monkeypatch: pytest.MonkeyPatch, - ) -> None: - """System prompt must instruct Claude to call open_kitchen(name) as its first action.""" - self._setup_recipe(tmp_path, monkeypatch) - mock_run.return_value = subprocess.CompletedProcess( - args=[], returncode=0, stdout="", stderr="" - ) - - cli.order("test-script") - - cmd = mock_run.call_args[0][0] - prompt_idx = cmd.index(ClaudeFlags.APPEND_SYSTEM_PROMPT) - system_prompt = cmd[prompt_idx + 1] - assert "open_kitchen" in system_prompt - assert "FIRST ACTION" in system_prompt - - -_GITHUB_RECIPE_YAML = """\ -name: github-recipe -description: A recipe using github tools -summary: Fetch an issue -steps: - fetch: - tool: fetch_github_issue - with: - issue_url: https://github.com/example/repo/issues/1 - on_success: done - on_failure: done - done: - action: stop - message: Done -kitchen_rules: - - Only use AutoSkillit MCP tools during pipeline execution -""" - - -class TestOrderSubsetGate: - """Tests for the order-time subset-disabled gate (T-VAL-008..010).""" - - @pytest.fixture(autouse=True) - def _stub_preview(self, monkeypatch: pytest.MonkeyPatch) -> None: - monkeypatch.setattr( - "autoskillit.cli._prompts.show_cook_preview", - lambda *a, **kw: None, - ) - - @pytest.fixture(autouse=True) - def _stub_is_plugin_installed(self, monkeypatch: pytest.MonkeyPatch) -> None: - """Stub _is_plugin_installed in app.py to False to prevent subprocess.run collision.""" - import importlib - import sys as _sys - - _app_mod = _sys.modules.get("autoskillit.cli.app") or importlib.import_module( - "autoskillit.cli.app" - ) - monkeypatch.setattr(_app_mod, "_is_plugin_installed", lambda: False) - - def _make_config_mock(self, disabled: list[str]) -> MagicMock: - mock_cfg = MagicMock() - mock_cfg.subsets.disabled = disabled - return mock_cfg - - def test_order_hard_error_non_interactive_on_disabled_subset( - self, - tmp_path: Path, - monkeypatch: pytest.MonkeyPatch, - capsys: pytest.CaptureFixture, - ) -> None: - """T-VAL-008: order exits 1 non-interactively when recipe references a disabled subset.""" - monkeypatch.delenv("CLAUDECODE", raising=False) - monkeypatch.chdir(tmp_path) - scripts_dir = tmp_path / ".autoskillit" / "recipes" - scripts_dir.mkdir(parents=True) - (scripts_dir / "github-recipe.yaml").write_text(_GITHUB_RECIPE_YAML) - - monkeypatch.setattr( - "autoskillit.config.load_config", lambda *a, **kw: self._make_config_mock(["github"]) - ) - mock_stdin = MagicMock() - mock_stdin.isatty.return_value = False - monkeypatch.setattr("sys.stdin", mock_stdin) - - with pytest.raises(SystemExit) as exc_info: - cli.order("github-recipe") - assert exc_info.value.code == 1 - captured = capsys.readouterr() - assert "requires subset" in captured.out.lower() - - @patch("autoskillit.cli.subprocess.run") - def test_order_enable_temporarily_sets_env_override( - self, - mock_run: MagicMock, - tmp_path: Path, - monkeypatch: pytest.MonkeyPatch, - ) -> None: - """T-VAL-009: order injects AUTOSKILLIT_SUBSETS__DISABLED env override for temp enable.""" - monkeypatch.delenv("CLAUDECODE", raising=False) - monkeypatch.chdir(tmp_path) - scripts_dir = tmp_path / ".autoskillit" / "recipes" - scripts_dir.mkdir(parents=True) - (scripts_dir / "github-recipe.yaml").write_text(_GITHUB_RECIPE_YAML) - - monkeypatch.setattr( - "autoskillit.config.load_config", lambda *a, **kw: self._make_config_mock(["github"]) - ) - mock_stdin = MagicMock() - mock_stdin.isatty.return_value = True - monkeypatch.setattr("sys.stdin", mock_stdin) - monkeypatch.setattr(shutil, "which", lambda cmd: "/usr/bin/claude") - # "1" = enable temporarily, "" = confirm launch - inputs = iter(["1", ""]) - monkeypatch.setattr("builtins.input", lambda _prompt="": next(inputs)) - mock_run.return_value = subprocess.CompletedProcess( - args=[], returncode=0, stdout="", stderr="" - ) - - cli.order("github-recipe") - - mock_run.assert_called_once() - passed_env = mock_run.call_args.kwargs["env"] - assert "AUTOSKILLIT_SUBSETS__DISABLED" in passed_env - assert passed_env["AUTOSKILLIT_SUBSETS__DISABLED"] == "@json []" - - def test_order_enable_permanently_updates_config( - self, - tmp_path: Path, - monkeypatch: pytest.MonkeyPatch, - ) -> None: - """T-VAL-010: order calls _enable_subsets_permanently on enable-permanently choice.""" - import importlib - import sys as _sys - - monkeypatch.delenv("CLAUDECODE", raising=False) - monkeypatch.chdir(tmp_path) - scripts_dir = tmp_path / ".autoskillit" / "recipes" - scripts_dir.mkdir(parents=True) - (scripts_dir / "github-recipe.yaml").write_text(_GITHUB_RECIPE_YAML) - - monkeypatch.setattr( - "autoskillit.config.load_config", lambda *a, **kw: self._make_config_mock(["github"]) - ) - mock_stdin = MagicMock() - mock_stdin.isatty.return_value = True - monkeypatch.setattr("sys.stdin", mock_stdin) - - called_with: list = [] - - def _fake_enable(project_dir, subsets): - called_with.append((project_dir, subsets)) - - _app_mod = _sys.modules.get("autoskillit.cli.app") or importlib.import_module( - "autoskillit.cli.app" - ) - monkeypatch.setattr(_app_mod, "_enable_subsets_permanently", _fake_enable) - # "2" = enable permanently, then "n" = cancel launch (to avoid needing subprocess) - inputs = iter(["2", "n"]) - monkeypatch.setattr("builtins.input", lambda _prompt="": next(inputs)) - monkeypatch.setattr("autoskillit.cli._ansi.permissions_warning", lambda: "") - - cli.order("github-recipe") - - assert called_with, "_enable_subsets_permanently was not called" - _, subsets = called_with[0] - assert "github" in subsets - - -class TestRecipesCLI: - def test_recipes_list_outputs_names(self, capsys: pytest.CaptureFixture) -> None: - """recipes list prints at least one recipe name to stdout.""" - import importlib - import sys - - _app_mod = sys.modules.get("autoskillit.cli.app") or importlib.import_module( - "autoskillit.cli.app" - ) - _app_mod.recipes_list() - captured = capsys.readouterr() - assert captured.out.strip(), "Expected at least one recipe in output" - assert "implementation" in captured.out - - def test_recipes_show_prints_content(self, capsys: pytest.CaptureFixture) -> None: - """recipes show prints the YAML content of a known bundled recipe.""" - import importlib - import sys - - _app_mod = sys.modules.get("autoskillit.cli.app") or importlib.import_module( - "autoskillit.cli.app" - ) - _app_mod.recipes_show("smoke-test") - captured = capsys.readouterr() - assert captured.out.strip(), "Expected YAML content in output" - assert "smoke-test" in captured.out - - def test_recipes_show_unknown_exits(self, capsys: pytest.CaptureFixture) -> None: - """recipes show exits 1 for an unknown recipe name.""" - import importlib - import sys - - _app_mod = sys.modules.get("autoskillit.cli.app") or importlib.import_module( - "autoskillit.cli.app" - ) - with pytest.raises(SystemExit) as exc_info: - _app_mod.recipes_show("nonexistent-recipe-xyz") - assert exc_info.value.code == 1 - - def test_recipes_render_lists_available( - self, - tmp_path: Path, - monkeypatch: pytest.MonkeyPatch, - capsys: pytest.CaptureFixture, - ) -> None: - """DG-22: `autoskillit recipes render` subcommand is registered and lists recipes.""" - from autoskillit import cli - - monkeypatch.chdir(tmp_path) - cli.recipes_render(None) # list all - captured = capsys.readouterr() - assert captured.out.strip(), "Expected recipe names in output" - assert "implementation" in captured.out - - -_PLUGIN_KEY = "autoskillit@autoskillit-local" - - -class TestOrderMcpPrefixSelection: - """order() must embed the resolved MCP prefix in the system prompt.""" - - @pytest.fixture(autouse=True) - def _stub_preview(self, monkeypatch: pytest.MonkeyPatch) -> None: - monkeypatch.setattr("autoskillit.cli._prompts.show_cook_preview", lambda *a, **kw: None) - - @pytest.fixture(autouse=True) - def _interactive_stdin(self, monkeypatch: pytest.MonkeyPatch) -> None: - monkeypatch.setattr("sys.stdin.isatty", lambda: True) - - @pytest.fixture(autouse=True) - def _stub_is_plugin_installed(self, monkeypatch: pytest.MonkeyPatch) -> None: - """Stub _is_plugin_installed in app.py to False to prevent subprocess.run collision.""" - import importlib - import sys as _sys - - _app_mod = _sys.modules.get("autoskillit.cli.app") or importlib.import_module( - "autoskillit.cli.app" - ) - monkeypatch.setattr(_app_mod, "_is_plugin_installed", lambda: False) - - @patch("autoskillit.cli.subprocess.run") - def test_order_prompt_uses_direct_prefix_when_no_marketplace_install( - self, mock_run: MagicMock, tmp_path: Path, monkeypatch: pytest.MonkeyPatch - ) -> None: - """order() builds a prompt with the direct prefix when installed_plugins.json lacks key.""" - from autoskillit.cli._mcp_names import DIRECT_PREFIX - - monkeypatch.delenv("CLAUDECODE", raising=False) - monkeypatch.chdir(tmp_path) - scripts_dir = tmp_path / ".autoskillit" / "recipes" - scripts_dir.mkdir(parents=True) - (scripts_dir / "test-script.yaml").write_text(_SCRIPT_YAML) - monkeypatch.setattr(shutil, "which", lambda cmd: "/usr/bin/claude") - monkeypatch.setattr("builtins.input", lambda _prompt="": "") - plugins_file = tmp_path / "plugins.json" - plugins_file.write_text('{"version": 2, "plugins": {}}') - monkeypatch.setattr( - "autoskillit.cli._mcp_names._installed_plugins_path", lambda: plugins_file - ) - mock_run.return_value = subprocess.CompletedProcess( - args=[], returncode=0, stdout="", stderr="" - ) - - cli.order("test-script") - - mock_run.assert_called_once() - cmd = mock_run.call_args[0][0] - prompt_idx = cmd.index(ClaudeFlags.APPEND_SYSTEM_PROMPT) - captured_prompt = cmd[prompt_idx + 1] - assert f"{DIRECT_PREFIX}open_kitchen" in captured_prompt - - @patch("autoskillit.cli.subprocess.run") - def test_order_prompt_uses_marketplace_prefix_when_plugin_installed( - self, mock_run: MagicMock, tmp_path: Path, monkeypatch: pytest.MonkeyPatch - ) -> None: - """order() uses marketplace prefix when autoskillit is plugin-installed.""" - from autoskillit.cli._mcp_names import MARKETPLACE_PREFIX - - monkeypatch.delenv("CLAUDECODE", raising=False) - monkeypatch.chdir(tmp_path) - scripts_dir = tmp_path / ".autoskillit" / "recipes" - scripts_dir.mkdir(parents=True) - (scripts_dir / "test-script.yaml").write_text(_SCRIPT_YAML) - monkeypatch.setattr(shutil, "which", lambda cmd: "/usr/bin/claude") - monkeypatch.setattr("builtins.input", lambda _prompt="": "") - plugins_file = tmp_path / "plugins.json" - plugins_file.write_text(f'{{"version": 2, "plugins": {{"{_PLUGIN_KEY}": []}}}}') - monkeypatch.setattr( - "autoskillit.cli._mcp_names._installed_plugins_path", lambda: plugins_file - ) - mock_run.return_value = subprocess.CompletedProcess( - args=[], returncode=0, stdout="", stderr="" - ) - - cli.order("test-script") - - mock_run.assert_called_once() - cmd = mock_run.call_args[0][0] - prompt_idx = cmd.index(ClaudeFlags.APPEND_SYSTEM_PROMPT) - captured_prompt = cmd[prompt_idx + 1] - assert f"{MARKETPLACE_PREFIX}open_kitchen" in captured_prompt - - -# SC-B-4: mark_onboarded() must NOT be called when the cook subprocess exits non-zero -def test_cook_mark_onboarded_not_called_on_failure( - monkeypatch: pytest.MonkeyPatch, tmp_path: Path -) -> None: - """mark_onboarded() must not be called when the cook subprocess exits non-zero.""" - import autoskillit.cli._cook as _cook - - onboarded_calls: list[Path] = [] - monkeypatch.setattr( - "autoskillit.cli._onboarding.mark_onboarded", - lambda project_dir: onboarded_calls.append(project_dir), - ) - - def fake_run(cmd: list[str], **kwargs: object) -> object: - return type("R", (), {"returncode": 1})() - - monkeypatch.setattr(_cook.subprocess, "run", fake_run) - - with pytest.raises(SystemExit): - _cook._run_cook_session( - cmd=["claude", "--test"], - env={}, - _first_run=True, - initial_prompt="test", - project_dir=tmp_path, - ) - - assert onboarded_calls == [], ( - "mark_onboarded() must not be called when the subprocess exits non-zero" - ) - - -# REQ-CLI-003 -class TestOrderResumeParsing: - """CLI-level parsing tests for `order --resume [session-id]`.""" - - @pytest.fixture(autouse=True) - def _stub_preview(self, monkeypatch: pytest.MonkeyPatch) -> None: - """Stub terminal preview to avoid subprocess calls.""" - monkeypatch.setattr( - "autoskillit.cli._prompts.show_cook_preview", - lambda *a, **kw: None, - ) - - @pytest.fixture(autouse=True) - def _interactive_stdin(self, monkeypatch: pytest.MonkeyPatch) -> None: - """order() requires an interactive TTY — default to True for this class.""" - monkeypatch.setattr("sys.stdin.isatty", lambda: True) - - def test_order_recipe_resume_with_session_id( - self, monkeypatch: pytest.MonkeyPatch, tmp_path: Path - ) -> None: - """order my-recipe --resume passes session_id to launch — REQ-CLI-003.""" - from autoskillit.cli.app import app - - monkeypatch.delenv("CLAUDECODE", raising=False) - monkeypatch.chdir(tmp_path) - - captured: dict = {} - - def fake_launch(prompt, *, initial_message=None, extra_env=None, resume_session_id=None): - captured["resume_session_id"] = resume_session_id - - with ( - patch("autoskillit.cli.app._launch_cook_session", side_effect=fake_launch), - patch( - "autoskillit.recipe.find_recipe_by_name", - return_value=MagicMock(path=tmp_path / "dummy.yaml", name="my-recipe"), - ), - patch("autoskillit.recipe.load_recipe", return_value=MagicMock()), - patch("autoskillit.recipe.validate_recipe", return_value=[]), - patch("autoskillit.core.find_latest_session_id", return_value=None), - patch("builtins.input", return_value=""), - ): - with pytest.raises(SystemExit) as exc_info: - app(["order", "my-recipe", "--resume", "fa910a41-d1ca-4cae-b878-01028a0c7c1c"]) - assert exc_info.value.code == 0 - - assert captured["resume_session_id"] == "fa910a41-d1ca-4cae-b878-01028a0c7c1c" diff --git a/tests/cli/test_cook_env_scrub.py b/tests/cli/test_cook_env_scrub.py index bc278750b0..4e96dff3d3 100644 --- a/tests/cli/test_cook_env_scrub.py +++ b/tests/cli/test_cook_env_scrub.py @@ -15,6 +15,8 @@ from autoskillit.execution import _MAX_MCP_OUTPUT_TOKENS_VALUE +pytestmark = [pytest.mark.layer("cli"), pytest.mark.small] + def test_launch_cook_session_env_excludes_ide_vars( monkeypatch: pytest.MonkeyPatch, tmp_path: Path @@ -24,19 +26,19 @@ def test_launch_cook_session_env_excludes_ide_vars( monkeypatch.setenv("VSCODE_GIT_ASKPASS_MAIN", "/fake/vscode") monkeypatch.setenv("CLAUDE_CODE_IDE_HOST_OVERRIDE", "host") - from autoskillit.cli.app import _launch_cook_session + from autoskillit.cli.session._session_launch import _launch_cook_session with ( patch("shutil.which", return_value="/usr/bin/claude"), + patch("autoskillit.cli._init_helpers._is_plugin_installed", return_value=False), patch( - "autoskillit.cli.app.subprocess.run", + "autoskillit.cli.session._session_launch.subprocess.run", return_value=MagicMock(returncode=0), ) as mock_run, - patch("autoskillit.cli.app.terminal_guard"), ): _launch_cook_session("system prompt", initial_message="hello") - assert mock_run.call_args is not None + mock_run.assert_called_once() env = mock_run.call_args.kwargs["env"] assert "CLAUDE_CODE_SSE_PORT" not in env assert "ENABLE_IDE_INTEGRATION" not in env @@ -50,15 +52,15 @@ def test_launch_cook_session_extra_env_still_applied( ) -> None: monkeypatch.setenv("CLAUDE_CODE_SSE_PORT", "23270") - from autoskillit.cli.app import _launch_cook_session + from autoskillit.cli.session._session_launch import _launch_cook_session with ( patch("shutil.which", return_value="/usr/bin/claude"), + patch("autoskillit.cli._init_helpers._is_plugin_installed", return_value=False), patch( - "autoskillit.cli.app.subprocess.run", + "autoskillit.cli.session._session_launch.subprocess.run", return_value=MagicMock(returncode=0), ) as mock_run, - patch("autoskillit.cli.app.terminal_guard"), ): _launch_cook_session( "system prompt", @@ -74,15 +76,15 @@ def test_launch_cook_session_env_has_max_mcp_output_tokens( monkeypatch: pytest.MonkeyPatch, ) -> None: """_launch_cook_session (order path) must produce env with MAX_MCP_OUTPUT_TOKENS.""" - from autoskillit.cli.app import _launch_cook_session + from autoskillit.cli.session._session_launch import _launch_cook_session with ( patch("shutil.which", return_value="/usr/bin/claude"), + patch("autoskillit.cli._init_helpers._is_plugin_installed", return_value=False), patch( - "autoskillit.cli.app.subprocess.run", + "autoskillit.cli.session._session_launch.subprocess.run", return_value=MagicMock(returncode=0), ) as mock_run, - patch("autoskillit.cli.app.terminal_guard"), ): _launch_cook_session("system prompt", initial_message="hello") @@ -90,6 +92,26 @@ def test_launch_cook_session_env_has_max_mcp_output_tokens( assert env["MAX_MCP_OUTPUT_TOKENS"] == _MAX_MCP_OUTPUT_TOKENS_VALUE +def test_launch_cook_session_env_has_mcp_connection_nonblocking( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """_launch_cook_session (order path) must produce env with MCP_CONNECTION_NONBLOCKING=0.""" + from autoskillit.cli.session._session_launch import _launch_cook_session + + with ( + patch("shutil.which", return_value="/usr/bin/claude"), + patch("autoskillit.cli._init_helpers._is_plugin_installed", return_value=False), + patch( + "autoskillit.cli.session._session_launch.subprocess.run", + return_value=MagicMock(returncode=0), + ) as mock_run, + ): + _launch_cook_session("system prompt", initial_message="hello") + + env = mock_run.call_args.kwargs["env"] + assert env["MCP_CONNECTION_NONBLOCKING"] == "0" + + def test_cook_command_env_excludes_ide_vars( monkeypatch: pytest.MonkeyPatch, tmp_path: Path ) -> None: @@ -108,16 +130,16 @@ def test_cook_command_env_excludes_ide_vars( patch("sys.stdin.isatty", return_value=True), patch("autoskillit.workspace.DefaultSessionSkillManager", return_value=mock_mgr), patch( - "autoskillit.cli._cook.subprocess.run", + "autoskillit.cli.session._cook.subprocess.run", return_value=MagicMock(returncode=0), ) as mock_run, - patch("autoskillit.cli._cook.terminal_guard"), + patch("autoskillit.cli.session._cook.terminal_guard"), ): - from autoskillit.cli._cook import cook + from autoskillit.cli.session._cook import cook cook() - assert mock_run.call_args is not None + mock_run.assert_called_once() env = mock_run.call_args.kwargs["env"] assert "CLAUDE_CODE_SSE_PORT" not in env assert "ENABLE_IDE_INTEGRATION" not in env @@ -141,12 +163,12 @@ def test_cook_command_env_has_max_mcp_output_tokens( patch("sys.stdin.isatty", return_value=True), patch("autoskillit.workspace.DefaultSessionSkillManager", return_value=mock_mgr), patch( - "autoskillit.cli._cook.subprocess.run", + "autoskillit.cli.session._cook.subprocess.run", return_value=MagicMock(returncode=0), ) as mock_run, - patch("autoskillit.cli._cook.terminal_guard"), + patch("autoskillit.cli.session._cook.terminal_guard"), ): - from autoskillit.cli._cook import cook + from autoskillit.cli.session._cook import cook cook() diff --git a/tests/cli/test_cook_features.py b/tests/cli/test_cook_features.py new file mode 100644 index 0000000000..658847b606 --- /dev/null +++ b/tests/cli/test_cook_features.py @@ -0,0 +1,212 @@ +"""Tests: cook CLI features — subset gate, recipes CLI, fleet display categories.""" + +from __future__ import annotations + +import shutil +import subprocess +from pathlib import Path +from unittest.mock import MagicMock, patch + +import pytest + +from autoskillit import cli +from tests.cli.conftest import _GITHUB_RECIPE_YAML + +pytestmark = [pytest.mark.layer("cli"), pytest.mark.medium] + + +class TestOrderSubsetGate: + """Tests for the order-time subset-disabled gate (T-VAL-008..010).""" + + @pytest.fixture(autouse=True) + def _stub_preview(self, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr("autoskillit.cli._preview.show_cook_preview", lambda *a, **kw: None) + + @pytest.fixture(autouse=True) + def _stub_ingredients_table(self, monkeypatch: pytest.MonkeyPatch) -> None: + import importlib + import sys as _sys + + _app_mod = _sys.modules.get("autoskillit.cli.session._order") or importlib.import_module( + "autoskillit.cli.session._order" + ) + monkeypatch.setattr(_app_mod, "_get_ingredients_table", lambda *a, **kw: "| col | val |") + + def _make_config_mock(self, disabled: list[str]) -> MagicMock: + mock_cfg = MagicMock() + mock_cfg.subsets.disabled = disabled + return mock_cfg + + def test_order_hard_error_non_interactive_on_disabled_subset( + self, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture, + ) -> None: + """T-VAL-008: order exits 1 non-interactively when recipe references a disabled subset.""" + monkeypatch.delenv("CLAUDECODE", raising=False) + monkeypatch.chdir(tmp_path) + scripts_dir = tmp_path / ".autoskillit" / "recipes" + scripts_dir.mkdir(parents=True) + (scripts_dir / "github-recipe.yaml").write_text(_GITHUB_RECIPE_YAML) + + monkeypatch.setattr( + "autoskillit.config.load_config", lambda *a, **kw: self._make_config_mock(["github"]) + ) + mock_stdin = MagicMock() + mock_stdin.isatty.return_value = False + monkeypatch.setattr("sys.stdin", mock_stdin) + + with pytest.raises(SystemExit) as exc_info: + cli.order("github-recipe") + assert exc_info.value.code == 1 + captured = capsys.readouterr() + assert "requires subset" in captured.out.lower() + + @patch("autoskillit.cli.subprocess.run") + def test_order_enable_temporarily_sets_env_override( + self, + mock_run: MagicMock, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + ) -> None: + """T-VAL-009: order injects AUTOSKILLIT_SUBSETS__DISABLED env override for temp enable.""" + monkeypatch.delenv("CLAUDECODE", raising=False) + monkeypatch.chdir(tmp_path) + scripts_dir = tmp_path / ".autoskillit" / "recipes" + scripts_dir.mkdir(parents=True) + (scripts_dir / "github-recipe.yaml").write_text(_GITHUB_RECIPE_YAML) + + monkeypatch.setattr( + "autoskillit.config.load_config", lambda *a, **kw: self._make_config_mock(["github"]) + ) + mock_stdin = MagicMock() + mock_stdin.isatty.return_value = True + monkeypatch.setattr("sys.stdin", mock_stdin) + monkeypatch.setattr(shutil, "which", lambda cmd: "/usr/bin/claude") + inputs = iter(["1", ""]) + monkeypatch.setattr("builtins.input", lambda _prompt="": next(inputs)) + mock_run.return_value = subprocess.CompletedProcess( + args=[], returncode=0, stdout="", stderr="" + ) + + cli.order("github-recipe") + + mock_run.assert_called_once() + passed_env = mock_run.call_args.kwargs["env"] + assert "AUTOSKILLIT_SUBSETS__DISABLED" in passed_env + assert passed_env["AUTOSKILLIT_SUBSETS__DISABLED"] == "@json []" + + def test_order_enable_permanently_updates_config( + self, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + ) -> None: + """T-VAL-010: order calls _enable_subsets_permanently on enable-permanently choice.""" + import importlib + import sys as _sys + + monkeypatch.delenv("CLAUDECODE", raising=False) + monkeypatch.chdir(tmp_path) + scripts_dir = tmp_path / ".autoskillit" / "recipes" + scripts_dir.mkdir(parents=True) + (scripts_dir / "github-recipe.yaml").write_text(_GITHUB_RECIPE_YAML) + + monkeypatch.setattr( + "autoskillit.config.load_config", lambda *a, **kw: self._make_config_mock(["github"]) + ) + mock_stdin = MagicMock() + mock_stdin.isatty.return_value = True + monkeypatch.setattr("sys.stdin", mock_stdin) + + called_with: list = [] + + def _fake_enable(project_dir, subsets): + called_with.append((project_dir, subsets)) + + _app_mod = _sys.modules.get("autoskillit.cli.session._order") or importlib.import_module( + "autoskillit.cli.session._order" + ) + monkeypatch.setattr(_app_mod, "_enable_subsets_permanently", _fake_enable) + inputs = iter(["2", "n"]) + monkeypatch.setattr("builtins.input", lambda _prompt="": next(inputs)) + monkeypatch.setattr("autoskillit.cli.ui._ansi.permissions_warning", lambda: "") + + cli.order("github-recipe") + + assert called_with, "_enable_subsets_permanently was not called" + _, subsets = called_with[0] + assert "github" in subsets + + +class TestRecipesCLI: + def test_recipes_list_outputs_names(self, capsys: pytest.CaptureFixture) -> None: + """recipes list prints at least one recipe name to stdout.""" + import importlib + import sys + + _app_mod = sys.modules.get("autoskillit.cli.app") or importlib.import_module( + "autoskillit.cli.app" + ) + _app_mod.recipes_list() + captured = capsys.readouterr() + assert captured.out.strip(), "Expected at least one recipe in output" + assert "implementation" in captured.out + + def test_recipes_show_prints_content(self, capsys: pytest.CaptureFixture) -> None: + """recipes show prints the YAML content of a known bundled recipe.""" + import importlib + import sys + + _app_mod = sys.modules.get("autoskillit.cli.app") or importlib.import_module( + "autoskillit.cli.app" + ) + _app_mod.recipes_show("smoke-test") + captured = capsys.readouterr() + assert captured.out.strip(), "Expected YAML content in output" + assert "smoke-test" in captured.out + + def test_recipes_show_unknown_exits(self, capsys: pytest.CaptureFixture) -> None: + """recipes show exits 1 for an unknown recipe name.""" + import importlib + import sys + + _app_mod = sys.modules.get("autoskillit.cli.app") or importlib.import_module( + "autoskillit.cli.app" + ) + with pytest.raises(SystemExit) as exc_info: + _app_mod.recipes_show("nonexistent-recipe-xyz") + assert exc_info.value.code == 1 + + def test_recipes_render_lists_available( + self, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture, + ) -> None: + """DG-22: `autoskillit recipes render` subcommand is registered and lists recipes.""" + from autoskillit import cli + + monkeypatch.chdir(tmp_path) + cli.recipes_render(None) + captured = capsys.readouterr() + assert captured.out.strip(), "Expected recipe names in output" + assert "implementation" in captured.out + + +def test_display_categories_omits_fleet_when_disabled() -> None: + """Fleet category must not appear in iter_display_categories output when fleet is disabled.""" + from autoskillit.config import iter_display_categories + + cfg_features: dict[str, bool] = {"fleet": False} + categories = [name for name, _ in iter_display_categories(cfg_features)] + assert "Fleet" not in categories + + +def test_display_categories_includes_fleet_when_enabled() -> None: + """Fleet category must appear in iter_display_categories output when fleet is enabled.""" + from autoskillit.config import iter_display_categories + + cfg_features: dict[str, bool] = {"fleet": True} + categories = [name for name, _ in iter_display_categories(cfg_features)] + assert "Fleet" in categories diff --git a/tests/cli/test_cook_ide_isolation.py b/tests/cli/test_cook_ide_isolation.py index 2c99d37fdf..fecc169c34 100644 --- a/tests/cli/test_cook_ide_isolation.py +++ b/tests/cli/test_cook_ide_isolation.py @@ -19,6 +19,8 @@ import pytest +pytestmark = [pytest.mark.layer("cli"), pytest.mark.small] + def test_cook_session_ignores_ide_lock_file( monkeypatch: pytest.MonkeyPatch, tmp_path: Path @@ -36,15 +38,15 @@ def test_cook_session_ignores_ide_lock_file( monkeypatch.setenv("VSCODE_GIT_ASKPASS_MAIN", "/fake/vscode") monkeypatch.setenv("CLAUDE_CODE_IDE_HOST_OVERRIDE", "localhost") - from autoskillit.cli.app import _launch_cook_session + from autoskillit.cli.session._session_launch import _launch_cook_session with ( patch("shutil.which", return_value="/usr/bin/claude"), + patch("autoskillit.cli._init_helpers._is_plugin_installed", return_value=False), patch( - "autoskillit.cli.app.subprocess.run", + "autoskillit.cli.session._session_launch.subprocess.run", return_value=MagicMock(returncode=0), ) as mock_run, - patch("autoskillit.cli.app.terminal_guard"), ): _launch_cook_session("system prompt", initial_message="hello") diff --git a/tests/cli/test_cook_interactive.py b/tests/cli/test_cook_interactive.py index bd787af9b8..8cf113b2e1 100644 --- a/tests/cli/test_cook_interactive.py +++ b/tests/cli/test_cook_interactive.py @@ -11,6 +11,8 @@ from autoskillit import cli from autoskillit.workspace.session_skills import DefaultSessionSkillManager +pytestmark = [pytest.mark.layer("cli"), pytest.mark.medium] + class TestCookInteractive: @pytest.fixture(autouse=True) @@ -121,7 +123,7 @@ def test_cook_passes_plugin_dir(self, monkeypatch: pytest.MonkeyPatch, tmp_path: patch("autoskillit.workspace.DefaultSessionSkillManager", return_value=mock_mgr), patch("subprocess.run", return_value=MagicMock(returncode=0)) as mock_run, ): - import autoskillit.cli._cook as module + import autoskillit.cli.session._cook as module module.cook() @@ -149,7 +151,7 @@ def test_cook_includes_dangerously_skip_permissions( patch("autoskillit.workspace.DefaultSessionSkillManager", return_value=mock_mgr), patch("subprocess.run", return_value=MagicMock(returncode=0)) as mock_run, ): - import autoskillit.cli._cook as module + import autoskillit.cli.session._cook as module module.cook() @@ -182,7 +184,7 @@ def test_cook_calls_onboarding_menu_on_first_run( patch("autoskillit.workspace.DefaultSessionSkillManager", return_value=mock_mgr), patch("subprocess.run", return_value=MagicMock(returncode=0)), ): - import autoskillit.cli._cook as module + import autoskillit.cli.session._cook as module module.cook() @@ -212,7 +214,7 @@ def test_cook_skips_onboarding_if_not_first_run( patch("autoskillit.workspace.DefaultSessionSkillManager", return_value=mock_mgr), patch("subprocess.run", return_value=MagicMock(returncode=0)), ): - import autoskillit.cli._cook as module + import autoskillit.cli.session._cook as module module.cook() @@ -243,7 +245,7 @@ def test_cook_forwards_initial_prompt_to_build_cmd( patch("autoskillit.workspace.DefaultSessionSkillManager", return_value=mock_mgr), patch("subprocess.run", return_value=MagicMock(returncode=0)) as mock_run, ): - import autoskillit.cli._cook as module + import autoskillit.cli.session._cook as module module.cook() @@ -279,7 +281,7 @@ def test_cook_marks_onboarded_in_finally_when_prompt_set( patch("autoskillit.workspace.DefaultSessionSkillManager", return_value=mock_mgr), patch("subprocess.run", return_value=MagicMock(returncode=0)), ): - import autoskillit.cli._cook as module + import autoskillit.cli.session._cook as module module.cook() @@ -314,7 +316,7 @@ def test_cook_does_not_mark_onboarded_when_prompt_is_none( patch("autoskillit.workspace.DefaultSessionSkillManager", return_value=mock_mgr), patch("subprocess.run", return_value=MagicMock(returncode=0)), ): - import autoskillit.cli._cook as module + import autoskillit.cli.session._cook as module module.cook() @@ -346,21 +348,24 @@ def tracking_rmtree(path, **kw): patch("subprocess.run", return_value=MagicMock(returncode=0)), patch("shutil.rmtree", side_effect=tracking_rmtree), ): - import autoskillit.cli._cook as module + import autoskillit.cli.session._cook as module module.cook() assert not rmtree_calls, "cook() must not rmtree skills_dir on exit" # REQ-CLI-001 + REQ-CLI-002 - def test_cook_resume_passes_resume_flag(self, monkeypatch, tmp_path): - """cook(resume=True) passes --resume to subprocess.""" + def test_cook_resume_bare_flag_produces_bare_resume_and_skips_discovery( + self, monkeypatch, tmp_path + ): + """cook(resume=True) invokes picker; UUID from picker passed as --resume; no discovery.""" from unittest.mock import MagicMock, patch fake_skills_dir = tmp_path / "skills" fake_skills_dir.mkdir() mock_mgr = MagicMock() mock_mgr.init_session.return_value = fake_skills_dir + discovery_calls: list = [] with ( patch("shutil.which", return_value="/usr/bin/claude"), @@ -369,17 +374,24 @@ def test_cook_resume_passes_resume_flag(self, monkeypatch, tmp_path): patch("subprocess.run", return_value=MagicMock(returncode=0)) as mock_run, patch( "autoskillit.core.find_latest_session_id", - return_value="session123", + side_effect=lambda *a, **kw: discovery_calls.append(1) or "latest", + ), + patch( + "autoskillit.cli.session._session_picker.pick_session", + return_value="picker-uuid-abc", ), ): - import autoskillit.cli._cook as module + import autoskillit.cli.session._cook as module module.cook(resume=True) args = mock_run.call_args[0][0] assert "--resume" in args idx = args.index("--resume") - assert args[idx + 1] == "session123" + assert args[idx + 1] == "picker-uuid-abc", ( + "--resume must be followed by the UUID returned by pick_session" + ) + assert not discovery_calls, "find_latest_session_id must not be called for bare --resume" # REQ-CLI-002 def test_cook_resume_explicit_session_id(self, monkeypatch, tmp_path): @@ -403,7 +415,7 @@ def fake_discover(cwd=None): patch("subprocess.run", return_value=MagicMock(returncode=0)) as mock_run, patch("autoskillit.core.find_latest_session_id", side_effect=fake_discover), ): - import autoskillit.cli._cook as module + import autoskillit.cli.session._cook as module module.cook(resume=True, session_id="explicit-abc") @@ -412,9 +424,9 @@ def fake_discover(cwd=None): assert args[args.index("--resume") + 1] == "explicit-abc" assert not discovery_calls, "discovery must not be called when session_id is explicit" - # REQ-CLI-002 — fallback when no session exists - def test_cook_resume_falls_back_to_fresh_when_no_session(self, monkeypatch, tmp_path): - """cook(resume=True) with no prior session starts a fresh session.""" + # REQ-CLI-002 — when picker returns None (no sessions), cook launches a fresh session + def test_cook_resume_bare_flag_no_sessions_starts_fresh(self, monkeypatch, tmp_path): + """cook(resume=True) with picker returning None starts a fresh session (no --resume).""" from unittest.mock import MagicMock, patch fake_skills_dir = tmp_path / "skills" @@ -427,14 +439,16 @@ def test_cook_resume_falls_back_to_fresh_when_no_session(self, monkeypatch, tmp_ patch("builtins.input", return_value=""), patch("autoskillit.workspace.DefaultSessionSkillManager", return_value=mock_mgr), patch("subprocess.run", return_value=MagicMock(returncode=0)) as mock_run, - patch("autoskillit.core.find_latest_session_id", return_value=None), + patch("autoskillit.cli.session._session_picker.pick_session", return_value=None), ): - import autoskillit.cli._cook as module + import autoskillit.cli.session._cook as module module.cook(resume=True) args = mock_run.call_args[0][0] - assert "--resume" not in args + assert "--resume" not in args, ( + "when picker returns None (no sessions), cook must launch fresh (no --resume)" + ) # REQ-CLI-003 def test_cook_cmd_resume_with_session_id_no_error( @@ -448,7 +462,9 @@ def test_cook_cmd_resume_with_session_id_no_error( captured: dict = {} - def fake_cook(*, resume: bool = False, session_id: str | None = None) -> None: + def fake_cook( + *, resume: bool = False, session_id: str | None = None, profile: str | None = None + ) -> None: captured["resume"] = resume captured["session_id"] = session_id @@ -473,7 +489,9 @@ def test_cook_cmd_resume_without_session_id_still_works( captured: dict = {} - def fake_cook(*, resume: bool = False, session_id: str | None = None) -> None: + def fake_cook( + *, resume: bool = False, session_id: str | None = None, profile: str | None = None + ) -> None: captured["resume"] = resume captured["session_id"] = session_id @@ -497,7 +515,9 @@ def test_cook_cmd_positional_session_id_implies_resume( captured: dict = {} - def fake_cook(*, resume: bool = False, session_id: str | None = None) -> None: + def fake_cook( + *, resume: bool = False, session_id: str | None = None, profile: str | None = None + ) -> None: captured["resume"] = resume captured["session_id"] = session_id @@ -508,3 +528,101 @@ def fake_cook(*, resume: bool = False, session_id: str | None = None) -> None: assert captured["session_id"] == "fa910a41-d1ca-4cae-b878-01028a0c7c1c" assert captured["resume"] is True + + def test_bare_resume_invokes_picker( + self, monkeypatch: pytest.MonkeyPatch, tmp_path: Path + ) -> None: + """cook(resume=True, session_id=None) intercepts BareResume via pick_session.""" + from unittest.mock import MagicMock, patch + + fake_skills_dir = tmp_path / "skills" + fake_skills_dir.mkdir() + mock_mgr = MagicMock() + mock_mgr.init_session.return_value = fake_skills_dir + picker_calls: list = [] + + def fake_pick_session(session_type: str, project_dir) -> None: + picker_calls.append((session_type, project_dir)) + return None + + with ( + patch("shutil.which", return_value="/usr/bin/claude"), + patch("builtins.input", return_value=""), + patch("autoskillit.workspace.DefaultSessionSkillManager", return_value=mock_mgr), + patch("subprocess.run", return_value=MagicMock(returncode=0)), + patch("autoskillit.cli.session._session_picker.pick_session", fake_pick_session), + patch("autoskillit.core.write_registry_entry"), + ): + import autoskillit.cli.session._cook as module + + module.cook(resume=True) + + assert picker_calls, "pick_session must be called for BareResume" + assert picker_calls[0][0] == "cook", f"Expected 'cook', got {picker_calls[0][0]!r}" + + def test_cook_launch_sets_session_type_env( + self, monkeypatch: pytest.MonkeyPatch, tmp_path: Path + ) -> None: + """cook() passes AUTOSKILLIT_SESSION_TYPE=cook in env_extras.""" + from unittest.mock import MagicMock, patch + + fake_skills_dir = tmp_path / "skills" + fake_skills_dir.mkdir() + mock_mgr = MagicMock() + mock_mgr.init_session.return_value = fake_skills_dir + captured_env_extras: list = [] + + from autoskillit.execution.commands import ClaudeInteractiveCmd + + def fake_build_interactive_cmd(**kwargs): + captured_env_extras.append(kwargs.get("env_extras", {})) + return ClaudeInteractiveCmd(cmd=["claude", "--dangerously-skip-permissions"], env={}) + + with ( + patch("shutil.which", return_value="/usr/bin/claude"), + patch("builtins.input", return_value=""), + patch("autoskillit.workspace.DefaultSessionSkillManager", return_value=mock_mgr), + patch("subprocess.run", return_value=MagicMock(returncode=0)), + patch("autoskillit.execution.build_interactive_cmd", fake_build_interactive_cmd), + patch("autoskillit.core.write_registry_entry"), + ): + import autoskillit.cli.session._cook as module + + module.cook() + + assert captured_env_extras, "build_interactive_cmd must have been called" + env_extras = captured_env_extras[0] + assert env_extras.get("AUTOSKILLIT_SESSION_TYPE") == "cook" + + def test_cook_launch_sets_launch_id_env( + self, monkeypatch: pytest.MonkeyPatch, tmp_path: Path + ) -> None: + """cook() passes AUTOSKILLIT_LAUNCH_ID in env_extras.""" + from unittest.mock import MagicMock, patch + + fake_skills_dir = tmp_path / "skills" + fake_skills_dir.mkdir() + mock_mgr = MagicMock() + mock_mgr.init_session.return_value = fake_skills_dir + captured_env_extras: list = [] + + from autoskillit.execution.commands import ClaudeInteractiveCmd + + def fake_build_interactive_cmd(**kwargs): + captured_env_extras.append(kwargs.get("env_extras", {})) + return ClaudeInteractiveCmd(cmd=["claude", "--dangerously-skip-permissions"], env={}) + + with ( + patch("shutil.which", return_value="/usr/bin/claude"), + patch("builtins.input", return_value=""), + patch("autoskillit.workspace.DefaultSessionSkillManager", return_value=mock_mgr), + patch("subprocess.run", return_value=MagicMock(returncode=0)), + patch("autoskillit.execution.build_interactive_cmd", fake_build_interactive_cmd), + patch("autoskillit.core.write_registry_entry"), + ): + import autoskillit.cli.session._cook as module + + module.cook() + + assert captured_env_extras, "build_interactive_cmd must have been called" + assert "AUTOSKILLIT_LAUNCH_ID" in captured_env_extras[0] diff --git a/tests/cli/test_cook_order_command.py b/tests/cli/test_cook_order_command.py new file mode 100644 index 0000000000..9c90f27996 --- /dev/null +++ b/tests/cli/test_cook_order_command.py @@ -0,0 +1,447 @@ +"""Tests: cook CLI order command — script validation, command building, env injection.""" + +from __future__ import annotations + +import shutil +import subprocess +from pathlib import Path +from unittest.mock import MagicMock, patch + +import pytest + +from autoskillit import cli +from autoskillit.core import ClaudeFlags +from tests.cli.conftest import _SCRIPT_YAML + +pytestmark = [pytest.mark.layer("cli"), pytest.mark.medium] + + +class TestCLIOrderCommand: + @pytest.fixture(autouse=True) + def _stub_preview(self, monkeypatch: pytest.MonkeyPatch) -> None: + """Stub terminal preview to avoid subprocess.run collision with git calls.""" + monkeypatch.setattr( + "autoskillit.cli._preview.show_cook_preview", + lambda *a, **kw: None, + ) + + @pytest.fixture(autouse=True) + def _interactive_stdin(self, monkeypatch: pytest.MonkeyPatch) -> None: + """Most order() paths require an interactive TTY — default to True for this class.""" + monkeypatch.setattr("sys.stdin.isatty", lambda: True) + + @pytest.fixture(autouse=True) + def _stub_ingredients_table(self, monkeypatch: pytest.MonkeyPatch) -> None: + """Stub _get_ingredients_table in app.py to prevent subprocess.run git calls.""" + import importlib + import sys as _sys + + _app_mod = _sys.modules.get("autoskillit.cli.session._order") or importlib.import_module( + "autoskillit.cli.session._order" + ) + monkeypatch.setattr(_app_mod, "_get_ingredients_table", lambda *a, **kw: "| col | val |") + + def test_order_blocked_inside_claude_session( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture + ) -> None: + """order exits 1 when CLAUDECODE env var is set.""" + monkeypatch.chdir(tmp_path) + monkeypatch.setenv("CLAUDECODE", "1") + with pytest.raises(SystemExit) as exc_info: + cli.order("any-script") + assert exc_info.value.code == 1 + captured = capsys.readouterr() + assert "regular terminal" in captured.out.lower() + + def test_order_script_not_found( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture + ) -> None: + """order exits 1 when script name doesn't match any entry.""" + monkeypatch.delenv("CLAUDECODE", raising=False) + monkeypatch.chdir(tmp_path) + scripts_dir = tmp_path / ".autoskillit" / "recipes" + scripts_dir.mkdir(parents=True) + + with pytest.raises(SystemExit) as exc_info: + cli.order("nonexistent") + assert exc_info.value.code == 1 + captured = capsys.readouterr() + assert "nonexistent" in captured.out + + def test_order_no_scripts_dir( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture + ) -> None: + """order exits 1 with available bundled recipes listed when name not found.""" + monkeypatch.delenv("CLAUDECODE", raising=False) + monkeypatch.chdir(tmp_path) + + with pytest.raises(SystemExit) as exc_info: + cli.order("anything") + assert exc_info.value.code == 1 + captured = capsys.readouterr() + assert "Recipe not found: 'anything'" in captured.out + assert "Available recipes:" in captured.out + assert "implementation" in captured.out + + def test_order_available_scripts_listed( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture + ) -> None: + """order lists available scripts when name doesn't match.""" + monkeypatch.delenv("CLAUDECODE", raising=False) + monkeypatch.chdir(tmp_path) + scripts_dir = tmp_path / ".autoskillit" / "recipes" + scripts_dir.mkdir(parents=True) + (scripts_dir / "my-script.yaml").write_text(_SCRIPT_YAML) + + with pytest.raises(SystemExit) as exc_info: + cli.order("nonexistent") + assert exc_info.value.code == 1 + captured = capsys.readouterr() + assert "Available recipes:" in captured.out + assert "test-script" in captured.out + + def test_order_claude_not_on_path( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture + ) -> None: + """order exits 1 when claude command is not found.""" + monkeypatch.delenv("CLAUDECODE", raising=False) + monkeypatch.chdir(tmp_path) + scripts_dir = tmp_path / ".autoskillit" / "recipes" + scripts_dir.mkdir(parents=True) + (scripts_dir / "my-script.yaml").write_text(_SCRIPT_YAML) + monkeypatch.setattr(shutil, "which", lambda cmd: None) + monkeypatch.setattr("builtins.input", lambda _prompt="": "") + + with pytest.raises(SystemExit) as exc_info: + cli.order("test-script") + assert exc_info.value.code == 1 + captured = capsys.readouterr() + assert "claude" in captured.out.lower() + + def test_order_invalid_script_exits( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture + ) -> None: + """order exits 1 when script YAML fails validation.""" + monkeypatch.delenv("CLAUDECODE", raising=False) + monkeypatch.chdir(tmp_path) + scripts_dir = tmp_path / ".autoskillit" / "recipes" + scripts_dir.mkdir(parents=True) + (scripts_dir / "bad-script.yaml").write_text("name: bad-script\nsteps: {}\n") + + with pytest.raises(SystemExit) as exc_info: + cli.order("bad-script") + assert exc_info.value.code == 1 + captured = capsys.readouterr() + assert "validation" in captured.out.lower() or "error" in captured.out.lower() + + @patch("autoskillit.cli.subprocess.run") + def test_order_builds_correct_command( + self, + mock_run: MagicMock, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + ) -> None: + """order passes correct flags to subprocess.run.""" + monkeypatch.delenv("CLAUDECODE", raising=False) + monkeypatch.chdir(tmp_path) + scripts_dir = tmp_path / ".autoskillit" / "recipes" + scripts_dir.mkdir(parents=True) + (scripts_dir / "my-script.yaml").write_text(_SCRIPT_YAML) + monkeypatch.setattr(shutil, "which", lambda cmd: "/usr/bin/claude") + monkeypatch.setattr("builtins.input", lambda _prompt="": "") + mock_run.return_value = subprocess.CompletedProcess( + args=[], returncode=0, stdout="", stderr="" + ) + + cli.order("test-script") + + mock_run.assert_called_once() + cmd = mock_run.call_args[0][0] + assert cmd[0] == "claude" + assert ClaudeFlags.PLUGIN_DIR in cmd + plugin_dir_idx = cmd.index(ClaudeFlags.PLUGIN_DIR) + plugin_dir_val = Path(cmd[plugin_dir_idx + 1]) + assert plugin_dir_val.is_dir() + assert (plugin_dir_val / ".claude-plugin" / "plugin.json").is_file() + assert ClaudeFlags.TOOLS in cmd + tools_idx = cmd.index(ClaudeFlags.TOOLS) + assert cmd[tools_idx + 1] == "AskUserQuestion" + assert ClaudeFlags.APPEND_SYSTEM_PROMPT in cmd + assert ClaudeFlags.DANGEROUSLY_SKIP_PERMISSIONS in cmd + assert ClaudeFlags.PRINT not in cmd + assert ClaudeFlags.ALLOW_DANGEROUSLY_SKIP_PERMISSIONS not in cmd + kwargs = mock_run.call_args[1] if mock_run.call_args[1] else {} + assert "capture_output" not in kwargs + assert "stdin" not in kwargs + + @patch("autoskillit.cli.subprocess.run") + def test_order_suppresses_plugin_dir_when_plugin_installed( + self, + mock_run: MagicMock, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + ) -> None: + """order omits --plugin-dir when marketplace plugin is installed.""" + monkeypatch.delenv("CLAUDECODE", raising=False) + monkeypatch.chdir(tmp_path) + scripts_dir = tmp_path / ".autoskillit" / "recipes" + scripts_dir.mkdir(parents=True) + (scripts_dir / "my-script.yaml").write_text(_SCRIPT_YAML) + monkeypatch.setattr(shutil, "which", lambda cmd: "/usr/bin/claude") + monkeypatch.setattr("builtins.input", lambda _prompt="": "") + from autoskillit.core._plugin_ids import MARKETPLACE_PREFIX + + monkeypatch.setattr( + "autoskillit.core.detect_autoskillit_mcp_prefix", + lambda: MARKETPLACE_PREFIX, + ) + mock_run.return_value = subprocess.CompletedProcess( + args=[], returncode=0, stdout="", stderr="" + ) + + cli.order("test-script") + + mock_run.assert_called_once() + cmd = mock_run.call_args[0][0] + assert ClaudeFlags.PLUGIN_DIR not in cmd + + @patch("autoskillit.cli.subprocess.run") + def test_order_includes_plugin_dir_when_no_plugin_installed( + self, + mock_run: MagicMock, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + ) -> None: + """order includes --plugin-dir when marketplace plugin is not installed.""" + monkeypatch.delenv("CLAUDECODE", raising=False) + monkeypatch.chdir(tmp_path) + scripts_dir = tmp_path / ".autoskillit" / "recipes" + scripts_dir.mkdir(parents=True) + (scripts_dir / "my-script.yaml").write_text(_SCRIPT_YAML) + monkeypatch.setattr(shutil, "which", lambda cmd: "/usr/bin/claude") + monkeypatch.setattr("builtins.input", lambda _prompt="": "") + from autoskillit.core._plugin_ids import DIRECT_PREFIX + + monkeypatch.setattr( + "autoskillit.core.detect_autoskillit_mcp_prefix", lambda: DIRECT_PREFIX + ) + mock_run.return_value = subprocess.CompletedProcess( + args=[], returncode=0, stdout="", stderr="" + ) + + cli.order("test-script") + + mock_run.assert_called_once() + cmd = mock_run.call_args[0][0] + assert ClaudeFlags.PLUGIN_DIR in cmd + + @patch("autoskillit.cli.subprocess.run") + def test_order_propagates_exit_code( + self, + mock_run: MagicMock, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + ) -> None: + """order does not raise SystemExit on returncode 0.""" + monkeypatch.delenv("CLAUDECODE", raising=False) + monkeypatch.chdir(tmp_path) + scripts_dir = tmp_path / ".autoskillit" / "recipes" + scripts_dir.mkdir(parents=True) + (scripts_dir / "my-script.yaml").write_text(_SCRIPT_YAML) + monkeypatch.setattr(shutil, "which", lambda cmd: "/usr/bin/claude") + monkeypatch.setattr("builtins.input", lambda _prompt="": "") + mock_run.return_value = subprocess.CompletedProcess( + args=[], returncode=0, stdout="", stderr="" + ) + + cli.order("test-script") # should not raise + mock_run.assert_called_once() + + @patch("autoskillit.cli.subprocess.run") + def test_order_subprocess_failure_propagates( + self, + mock_run: MagicMock, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + ) -> None: + """order propagates non-zero subprocess exit codes.""" + monkeypatch.delenv("CLAUDECODE", raising=False) + monkeypatch.chdir(tmp_path) + scripts_dir = tmp_path / ".autoskillit" / "recipes" + scripts_dir.mkdir(parents=True) + (scripts_dir / "my-script.yaml").write_text(_SCRIPT_YAML) + monkeypatch.setattr(shutil, "which", lambda cmd: "/usr/bin/claude") + monkeypatch.setattr("builtins.input", lambda _prompt="": "") + mock_run.return_value = subprocess.CompletedProcess( + args=[], returncode=42, stdout="", stderr="" + ) + + with pytest.raises(SystemExit) as exc_info: + cli.order("test-script") + assert exc_info.value.code == 42 + + @patch("autoskillit.cli.subprocess.run") + def test_order_uses_dangerously_skip_permissions( + self, + mock_run: MagicMock, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + ) -> None: + """order passes --dangerously-skip-permissions to claude.""" + monkeypatch.delenv("CLAUDECODE", raising=False) + monkeypatch.chdir(tmp_path) + scripts_dir = tmp_path / ".autoskillit" / "recipes" + scripts_dir.mkdir(parents=True) + (scripts_dir / "my-script.yaml").write_text(_SCRIPT_YAML) + monkeypatch.setattr(shutil, "which", lambda cmd: "/usr/bin/claude") + monkeypatch.setattr("builtins.input", lambda _prompt="": "") + mock_run.return_value = subprocess.CompletedProcess( + args=[], returncode=0, stdout="", stderr="" + ) + + cli.order("test-script") + + cmd = mock_run.call_args[0][0] + assert ClaudeFlags.DANGEROUSLY_SKIP_PERMISSIONS in cmd + + def test_order_recipe_not_found_exits( + self, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture, + ) -> None: + """order exits 1 when the given recipe name is not found.""" + monkeypatch.delenv("CLAUDECODE", raising=False) + monkeypatch.chdir(tmp_path) + + with pytest.raises(SystemExit) as exc_info: + cli.order("totally-unknown-recipe-xyz") + assert exc_info.value.code == 1 + captured = capsys.readouterr() + assert "totally-unknown-recipe-xyz" in captured.out + + def test_order_malformed_yaml_exits( + self, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture, + ) -> None: + """order exits 1 with YAML parse error message when load_recipe raises YAMLError.""" + from autoskillit.core import YAMLError + + monkeypatch.delenv("CLAUDECODE", raising=False) + monkeypatch.chdir(tmp_path) + + with ( + patch("autoskillit.recipe.find_recipe_by_name", return_value=MagicMock()), + patch("autoskillit.recipe.load_recipe", side_effect=YAMLError("bad yaml")), + pytest.raises(SystemExit) as exc_info, + ): + cli.order("bad-recipe") + + assert exc_info.value.code == 1 + captured = capsys.readouterr() + assert "YAML parse error" in captured.out + + def test_order_structure_error_exits( + self, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture, + ) -> None: + """order exits 1 with structure error message when load_recipe raises ValueError.""" + monkeypatch.delenv("CLAUDECODE", raising=False) + monkeypatch.chdir(tmp_path) + + with ( + patch("autoskillit.recipe.find_recipe_by_name", return_value=MagicMock()), + patch("autoskillit.recipe.load_recipe", side_effect=ValueError("bad structure")), + pytest.raises(SystemExit) as exc_info, + ): + cli.order("bad-recipe") + + assert exc_info.value.code == 1 + captured = capsys.readouterr() + assert "structure error" in captured.out + + @patch("autoskillit.cli.subprocess.run") + def test_order_launch_sets_session_type_order( + self, + mock_run: MagicMock, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + ) -> None: + """order() passes AUTOSKILLIT_SESSION_TYPE=order to subprocess env.""" + monkeypatch.delenv("CLAUDECODE", raising=False) + monkeypatch.chdir(tmp_path) + scripts_dir = tmp_path / ".autoskillit" / "recipes" + scripts_dir.mkdir(parents=True) + (scripts_dir / "test-script.yaml").write_text(_SCRIPT_YAML) + monkeypatch.setattr(shutil, "which", lambda cmd: "/usr/bin/claude") + monkeypatch.setattr("builtins.input", lambda _prompt="": "") + mock_run.return_value = subprocess.CompletedProcess( + args=[], returncode=0, stdout="", stderr="" + ) + + with patch("autoskillit.core.write_registry_entry"): + cli.order("test-script") + + env = mock_run.call_args[1].get("env") or {} + assert env.get("AUTOSKILLIT_SESSION_TYPE") == "order" + + @patch("autoskillit.cli.subprocess.run") + def test_order_launch_sets_launch_id_env( + self, + mock_run: MagicMock, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + ) -> None: + """order() passes AUTOSKILLIT_LAUNCH_ID in subprocess env.""" + monkeypatch.delenv("CLAUDECODE", raising=False) + monkeypatch.chdir(tmp_path) + scripts_dir = tmp_path / ".autoskillit" / "recipes" + scripts_dir.mkdir(parents=True) + (scripts_dir / "test-script.yaml").write_text(_SCRIPT_YAML) + monkeypatch.setattr(shutil, "which", lambda cmd: "/usr/bin/claude") + monkeypatch.setattr("builtins.input", lambda _prompt="": "") + mock_run.return_value = subprocess.CompletedProcess( + args=[], returncode=0, stdout="", stderr="" + ) + + with patch("autoskillit.core.write_registry_entry"): + cli.order("test-script") + + env = mock_run.call_args[1].get("env") or {} + assert "AUTOSKILLIT_LAUNCH_ID" in env + + +# SC-B-4: mark_onboarded() must NOT be called when the cook subprocess exits non-zero +def test_cook_mark_onboarded_not_called_on_failure( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + """mark_onboarded() must not be called when the cook subprocess exits non-zero.""" + import autoskillit.cli.session._cook as _cook + + onboarded_calls: list[Path] = [] + monkeypatch.setattr( + "autoskillit.cli._onboarding.mark_onboarded", + lambda project_dir: onboarded_calls.append(project_dir), + ) + + def fake_run(cmd: list[str], **kwargs: object) -> object: + return type("R", (), {"returncode": 1})() + + monkeypatch.setattr(_cook.subprocess, "run", fake_run) + + with pytest.raises(SystemExit): + _cook._run_cook_session( + cmd=["claude", "--test"], + env={}, + _first_run=True, + initial_prompt="test", + project_dir=tmp_path, + ) + + assert onboarded_calls == [], ( + "mark_onboarded() must not be called when the subprocess exits non-zero" + ) diff --git a/tests/cli/test_cook_order_picker.py b/tests/cli/test_cook_order_picker.py new file mode 100644 index 0000000000..8057c174a8 --- /dev/null +++ b/tests/cli/test_cook_order_picker.py @@ -0,0 +1,612 @@ +"""Tests: cook CLI order command — recipe picker, resume flows, session parsing.""" + +from __future__ import annotations + +import re +import shutil +import subprocess +from pathlib import Path +from unittest.mock import MagicMock, patch + +import pytest + +from autoskillit import cli +from autoskillit.core import ClaudeFlags +from tests.cli.conftest import _SCRIPT_YAML + +pytestmark = [pytest.mark.layer("cli"), pytest.mark.medium] + + +class TestCLIOrderPicker: + @pytest.fixture(autouse=True) + def _stub_preview(self, monkeypatch: pytest.MonkeyPatch) -> None: + """Stub terminal preview to avoid subprocess.run collision with git calls.""" + monkeypatch.setattr( + "autoskillit.cli._preview.show_cook_preview", + lambda *a, **kw: None, + ) + + @pytest.fixture(autouse=True) + def _interactive_stdin(self, monkeypatch: pytest.MonkeyPatch) -> None: + """Most order() paths require an interactive TTY — default to True for this class.""" + monkeypatch.setattr("sys.stdin.isatty", lambda: True) + + @pytest.fixture(autouse=True) + def _stub_ingredients_table(self, monkeypatch: pytest.MonkeyPatch) -> None: + """Stub _get_ingredients_table in app.py to prevent subprocess.run git calls.""" + import importlib + import sys as _sys + + _app_mod = _sys.modules.get("autoskillit.cli.session._order") or importlib.import_module( + "autoskillit.cli.session._order" + ) + monkeypatch.setattr(_app_mod, "_get_ingredients_table", lambda *a, **kw: "| col | val |") + + @patch("autoskillit.cli.subprocess.run") + def test_order_no_recipe_prompts_user( + self, + mock_run: MagicMock, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + ) -> None: + """order prompts for recipe name when none is provided.""" + monkeypatch.delenv("CLAUDECODE", raising=False) + monkeypatch.chdir(tmp_path) + scripts_dir = tmp_path / ".autoskillit" / "recipes" + scripts_dir.mkdir(parents=True) + (scripts_dir / "my-script.yaml").write_text(_SCRIPT_YAML) + monkeypatch.setattr(shutil, "which", lambda cmd: "/usr/bin/claude") + mock_run.return_value = subprocess.CompletedProcess( + args=[], returncode=0, stdout="", stderr="" + ) + monkeypatch.setattr("builtins.input", lambda _prompt="": "test-script") + + cli.order() # no recipe argument + + mock_run.assert_called_once() + + def test_order_no_recipe_no_available_exits( + self, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture, + ) -> None: + """order exits 1 when no recipe is given and no recipes are available.""" + import autoskillit.recipe as _recipe_mod + + monkeypatch.delenv("CLAUDECODE", raising=False) + monkeypatch.chdir(tmp_path) + + mock_result = MagicMock() + mock_result.items = [] + monkeypatch.setattr(_recipe_mod, "list_recipes", lambda *a, **kw: mock_result) + + with pytest.raises(SystemExit) as exc_info: + cli.order() + assert exc_info.value.code == 1 + + @patch("autoskillit.cli.subprocess.run") + def test_order_picker_shows_zero_option( + self, + mock_run: MagicMock, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture, + ) -> None: + """Picker output includes '0. Open kitchen' line.""" + monkeypatch.delenv("CLAUDECODE", raising=False) + monkeypatch.chdir(tmp_path) + scripts_dir = tmp_path / ".autoskillit" / "recipes" + scripts_dir.mkdir(parents=True) + (scripts_dir / "my-script.yaml").write_text(_SCRIPT_YAML) + monkeypatch.setattr(shutil, "which", lambda cmd: "/usr/bin/claude") + mock_run.return_value = subprocess.CompletedProcess(args=[], returncode=0) + monkeypatch.setattr("builtins.input", lambda _prompt="": "test-script") + + cli.order() + + captured = capsys.readouterr() + assert "0. Open kitchen" in captured.out + + @patch("autoskillit.cli.subprocess.run") + def test_order_picker_zero_launches_open_kitchen( + self, + mock_run: MagicMock, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + ) -> None: + """Typing '0' launches a session without a recipe YAML in the system prompt.""" + monkeypatch.delenv("CLAUDECODE", raising=False) + monkeypatch.chdir(tmp_path) + scripts_dir = tmp_path / ".autoskillit" / "recipes" + scripts_dir.mkdir(parents=True) + (scripts_dir / "my-script.yaml").write_text(_SCRIPT_YAML) + monkeypatch.setattr(shutil, "which", lambda cmd: "/usr/bin/claude") + mock_run.return_value = subprocess.CompletedProcess(args=[], returncode=0) + monkeypatch.setattr("builtins.input", lambda _prompt="": "0") + + cli.order() + + mock_run.assert_called_once() + cmd = mock_run.call_args[0][0] + system_prompt_idx = cmd.index(ClaudeFlags.APPEND_SYSTEM_PROMPT) + 1 + assert "--- RECIPE ---" not in cmd[system_prompt_idx] + + def test_order_picker_out_of_range_exits( + self, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture, + ) -> None: + """Out-of-range numeric input exits 1 with an error message.""" + import autoskillit.recipe as _recipe_mod + + monkeypatch.delenv("CLAUDECODE", raising=False) + monkeypatch.chdir(tmp_path) + + fake_recipe = MagicMock() + fake_recipe.name = "some-recipe" + mock_result = MagicMock() + mock_result.items = [fake_recipe] + monkeypatch.setattr(_recipe_mod, "list_recipes", lambda *a, **kw: mock_result) + monkeypatch.setattr("builtins.input", lambda _prompt="": "99") + + with pytest.raises(SystemExit) as exc_info: + cli.order() + assert exc_info.value.code == 1 + captured = capsys.readouterr() + assert "Invalid selection" in captured.out + + @patch("autoskillit.cli.subprocess.run") + def test_order_resume_bare_flag_produces_bare_resume_skips_discovery( + self, + mock_run: MagicMock, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + ) -> None: + """order(resume=True) passes bare --resume; find_latest_session_id must not be called.""" + monkeypatch.delenv("CLAUDECODE", raising=False) + monkeypatch.chdir(tmp_path) + scripts_dir = tmp_path / ".autoskillit" / "recipes" + scripts_dir.mkdir(parents=True) + (scripts_dir / "my-script.yaml").write_text(_SCRIPT_YAML) + monkeypatch.setattr(shutil, "which", lambda cmd: "/usr/bin/claude") + monkeypatch.setattr("builtins.input", lambda _prompt="": "") + mock_run.return_value = subprocess.CompletedProcess( + args=[], returncode=0, stdout="", stderr="" + ) + discovery_calls: list = [] + + with patch( + "autoskillit.core.find_latest_session_id", + side_effect=lambda *a, **kw: discovery_calls.append(1) or "sess_abc", + ): + cli.order("test-script", resume=True) + + cmd = mock_run.call_args[0][0] + assert "--resume" in cmd + idx = cmd.index("--resume") + # bare --resume: next token (if any) must not be a session ID; + # session IDs have no spaces, initial prompts and flags are distinguishable + if idx + 1 < len(cmd): + next_tok = str(cmd[idx + 1]) + assert " " in next_tok or next_tok.startswith("-"), ( + f"bare --resume must not be followed by a session ID, got: {next_tok!r}" + ) + assert not discovery_calls, "find_latest_session_id must not be called for bare --resume" + + @patch("autoskillit.cli.subprocess.run") + def test_order_resume_explicit_session_id_skips_discovery( + self, + mock_run: MagicMock, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + ) -> None: + """order(resume=True, session_id='explicit-abc') uses explicit id; discovery not called.""" + monkeypatch.delenv("CLAUDECODE", raising=False) + monkeypatch.chdir(tmp_path) + scripts_dir = tmp_path / ".autoskillit" / "recipes" + scripts_dir.mkdir(parents=True) + (scripts_dir / "my-script.yaml").write_text(_SCRIPT_YAML) + monkeypatch.setattr(shutil, "which", lambda cmd: "/usr/bin/claude") + monkeypatch.setattr("builtins.input", lambda _prompt="": "") + mock_run.return_value = subprocess.CompletedProcess( + args=[], returncode=0, stdout="", stderr="" + ) + discovery_calls: list = [] + + def fake_discover(cwd=None): + discovery_calls.append(cwd) + return "should-not-be-used" + + with patch("autoskillit.core.find_latest_session_id", side_effect=fake_discover): + cli.order("test-script", resume=True, session_id="explicit-abc") + + cmd = mock_run.call_args[0][0] + assert "--resume" in cmd + assert cmd[cmd.index("--resume") + 1] == "explicit-abc" + assert not discovery_calls, ( + "find_latest_session_id must not be called when session_id is explicit" + ) + + @patch("autoskillit.cli.subprocess.run") + def test_order_resume_bare_flag_always_emits_resume( + self, + mock_run: MagicMock, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + ) -> None: + """order(resume=True) always emits bare --resume; Claude Code handles empty history.""" + monkeypatch.delenv("CLAUDECODE", raising=False) + monkeypatch.chdir(tmp_path) + scripts_dir = tmp_path / ".autoskillit" / "recipes" + scripts_dir.mkdir(parents=True) + (scripts_dir / "my-script.yaml").write_text(_SCRIPT_YAML) + monkeypatch.setattr(shutil, "which", lambda cmd: "/usr/bin/claude") + monkeypatch.setattr("builtins.input", lambda _prompt="": "") + mock_run.return_value = subprocess.CompletedProcess( + args=[], returncode=0, stdout="", stderr="" + ) + + cli.order("test-script", resume=True) + + cmd = mock_run.call_args[0][0] + assert "--resume" in cmd + idx = cmd.index("--resume") + # bare --resume: next token (if any) must not be a session ID; + # session IDs have no spaces, initial prompts and flags are distinguishable + if idx + 1 < len(cmd): + next_tok = str(cmd[idx + 1]) + assert " " in next_tok or next_tok.startswith("-"), ( + f"bare --resume must not be followed by a session ID, got: {next_tok!r}" + ) + + @patch("autoskillit.cli.subprocess.run") + def test_order_bare_resume_no_recipe_invokes_picker( + self, + mock_run: MagicMock, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + ) -> None: + """order(resume=True) with no recipe invokes pick_session for order sessions.""" + monkeypatch.delenv("CLAUDECODE", raising=False) + monkeypatch.chdir(tmp_path) + monkeypatch.setattr(shutil, "which", lambda cmd: "/usr/bin/claude") + monkeypatch.setattr("builtins.input", lambda _prompt="": "") + mock_run.return_value = subprocess.CompletedProcess( + args=[], returncode=0, stdout="", stderr="" + ) + picker_calls: list = [] + + def fake_pick_session(session_type: str, project_dir) -> None: + picker_calls.append(session_type) + return None + + with patch("autoskillit.cli.session._session_picker.pick_session", fake_pick_session): + with patch("autoskillit.core.write_registry_entry"): + cli.order(resume=True) + + assert picker_calls == ["order"], f"Expected ['order'], got {picker_calls}" + + def test_order_picker_renders_family_recipes_header( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] + ) -> None: + """Picker must print 'Family Recipes' header when project recipes exist.""" + from autoskillit.core import RecipeSource + from autoskillit.recipe.schema import RecipeInfo + + project_recipe = RecipeInfo( + name="proj-recipe", + description="d", + source=RecipeSource.PROJECT, + path=tmp_path / "proj.yaml", + experimental=False, + ) + monkeypatch.setattr( + "autoskillit.recipe.list_recipes", + lambda *a, **kw: type("R", (), {"items": [project_recipe]})(), + ) + monkeypatch.setattr("autoskillit.cli.ui._menu.timed_prompt", lambda *a, **kw: "0") + monkeypatch.setattr(shutil, "which", lambda cmd: None) + monkeypatch.delenv("CLAUDECODE", raising=False) + monkeypatch.chdir(tmp_path) + + with pytest.raises(SystemExit): + cli.order() + + out = capsys.readouterr().out + assert "Family Recipes" in out + + def test_order_picker_renders_bundled_recipes_header( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] + ) -> None: + """Picker must print 'Bundled Recipes' header when builtin recipes exist.""" + from autoskillit.core import RecipeSource + from autoskillit.recipe.schema import RecipeInfo + + builtin_recipe = RecipeInfo( + name="impl", + description="d", + source=RecipeSource.BUILTIN, + path=tmp_path / "impl.yaml", + experimental=False, + requires_packs=["github"], + ) + monkeypatch.setattr( + "autoskillit.recipe.list_recipes", + lambda *a, **kw: type("R", (), {"items": [builtin_recipe]})(), + ) + monkeypatch.setattr("autoskillit.cli.ui._menu.timed_prompt", lambda *a, **kw: "0") + monkeypatch.setattr(shutil, "which", lambda cmd: None) + monkeypatch.delenv("CLAUDECODE", raising=False) + monkeypatch.chdir(tmp_path) + + with pytest.raises(SystemExit): + cli.order() + + out = capsys.readouterr().out + assert "Bundled Recipes" in out + + def test_order_picker_renders_experimental_header( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] + ) -> None: + """Picker must print 'Experimental' header when experimental recipes exist.""" + from autoskillit.core import RecipeSource + from autoskillit.recipe.schema import RecipeInfo + + exp_recipe = RecipeInfo( + name="research", + description="d", + source=RecipeSource.BUILTIN, + path=tmp_path / "research.yaml", + experimental=True, + ) + monkeypatch.setattr( + "autoskillit.recipe.list_recipes", + lambda *a, **kw: type("R", (), {"items": [exp_recipe]})(), + ) + monkeypatch.setattr("autoskillit.cli.ui._menu.timed_prompt", lambda *a, **kw: "0") + monkeypatch.setattr(shutil, "which", lambda cmd: None) + monkeypatch.delenv("CLAUDECODE", raising=False) + monkeypatch.chdir(tmp_path) + + with pytest.raises(SystemExit): + cli.order() + + out = capsys.readouterr().out + assert "Experimental" in out + + def test_order_picker_omits_empty_group_header( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] + ) -> None: + """Picker must not print 'Family Recipes' header when no project recipes exist.""" + from autoskillit.core import RecipeSource + from autoskillit.recipe.schema import RecipeInfo + + builtin_recipe = RecipeInfo( + name="impl", + description="d", + source=RecipeSource.BUILTIN, + path=tmp_path / "impl.yaml", + experimental=False, + ) + monkeypatch.setattr( + "autoskillit.recipe.list_recipes", + lambda *a, **kw: type("R", (), {"items": [builtin_recipe]})(), + ) + monkeypatch.setattr("autoskillit.cli.ui._menu.timed_prompt", lambda *a, **kw: "0") + monkeypatch.setattr(shutil, "which", lambda cmd: None) + monkeypatch.delenv("CLAUDECODE", raising=False) + monkeypatch.chdir(tmp_path) + + with pytest.raises(SystemExit): + cli.order() + + out = capsys.readouterr().out + assert "Family Recipes" not in out + + def test_order_picker_renders_bundled_add_ons_header( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] + ) -> None: + """Picker must print 'Bundled Add-ons' header for BUILTIN recipes with non-core packs.""" + from autoskillit.core import RecipeSource + from autoskillit.recipe.schema import RecipeInfo + + addon_recipe = RecipeInfo( + name="planner", + description="d", + source=RecipeSource.BUILTIN, + path=tmp_path / "planner.yaml", + experimental=False, + requires_packs=["kitchen-core"], + ) + monkeypatch.setattr( + "autoskillit.recipe.list_recipes", + lambda *a, **kw: type("R", (), {"items": [addon_recipe]})(), + ) + monkeypatch.setattr("autoskillit.cli.ui._menu.timed_prompt", lambda *a, **kw: "0") + monkeypatch.setattr(shutil, "which", lambda cmd: None) + monkeypatch.delenv("CLAUDECODE", raising=False) + monkeypatch.chdir(tmp_path) + + with pytest.raises(SystemExit): + cli.order() + + out = capsys.readouterr().out + assert "Bundled Add-ons" in out + + def test_order_picker_contiguous_numbering_across_groups( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] + ) -> None: + """Picker numbering must be contiguous across all groups (1, 2, 3...).""" + from autoskillit.core import RecipeSource + from autoskillit.recipe.schema import RecipeInfo + + recipes = [ + RecipeInfo( + name="proj-a", + description="d", + source=RecipeSource.PROJECT, + path=tmp_path / "a.yaml", + experimental=False, + ), + RecipeInfo( + name="impl", + description="d", + source=RecipeSource.BUILTIN, + path=tmp_path / "b.yaml", + experimental=False, + ), + RecipeInfo( + name="exp-r", + description="d", + source=RecipeSource.BUILTIN, + path=tmp_path / "c.yaml", + experimental=True, + ), + ] + monkeypatch.setattr( + "autoskillit.recipe.list_recipes", lambda *a, **kw: type("R", (), {"items": recipes})() + ) + monkeypatch.setattr("autoskillit.cli.ui._menu.timed_prompt", lambda *a, **kw: "0") + monkeypatch.setattr(shutil, "which", lambda cmd: None) + monkeypatch.delenv("CLAUDECODE", raising=False) + monkeypatch.chdir(tmp_path) + + with pytest.raises(SystemExit): + cli.order() + + out = capsys.readouterr().out + numbered = re.findall(r"^\s+([1-9]\d*)\.", out, re.MULTILINE) + numbered_ints = [int(n) for n in numbered] + assert numbered_ints == list(range(1, len(recipes) + 1)) + + +# REQ-CLI-003 +class TestOrderResumeParsing: + """CLI-level parsing tests for `order --resume [session-id]`.""" + + @pytest.fixture(autouse=True) + def _stub_preview(self, monkeypatch: pytest.MonkeyPatch) -> None: + """Stub terminal preview to avoid subprocess calls.""" + monkeypatch.setattr( + "autoskillit.cli._preview.show_cook_preview", + lambda *a, **kw: None, + ) + + @pytest.fixture(autouse=True) + def _interactive_stdin(self, monkeypatch: pytest.MonkeyPatch) -> None: + """order() requires an interactive TTY — default to True for this class.""" + monkeypatch.setattr("sys.stdin.isatty", lambda: True) + + def test_order_recipe_resume_with_session_id( + self, monkeypatch: pytest.MonkeyPatch, tmp_path: Path + ) -> None: + """order my-recipe --resume passes NamedResume to launch — REQ-CLI-003.""" + from autoskillit.cli.app import app + from autoskillit.core import NamedResume, NoResume + + monkeypatch.delenv("CLAUDECODE", raising=False) + monkeypatch.chdir(tmp_path) + + captured: dict = {} + + def fake_launch( + prompt, + *, + initial_message=None, + extra_env=None, + resume_spec=NoResume(), + project_dir=None, + ): + captured["resume_spec"] = resume_spec + + with ( + patch("autoskillit.cli.session._order._launch_cook_session", side_effect=fake_launch), + patch( + "autoskillit.recipe.find_recipe_by_name", + return_value=MagicMock(path=tmp_path / "dummy.yaml", name="my-recipe"), + ), + patch("autoskillit.recipe.load_recipe", return_value=MagicMock()), + patch("autoskillit.recipe.validate_recipe_structure", return_value=[]), + patch("builtins.input", return_value=""), + ): + with pytest.raises(SystemExit) as exc_info: + app(["order", "my-recipe", "--resume", "fa910a41-d1ca-4cae-b878-01028a0c7c1c"]) + assert exc_info.value.code == 0 + + assert captured["resume_spec"] == NamedResume( + session_id="fa910a41-d1ca-4cae-b878-01028a0c7c1c" + ) + + def test_order_resume_uuid_without_recipe_routes_correctly( + self, monkeypatch: pytest.MonkeyPatch, tmp_path: Path + ) -> None: + """order --resume (no recipe name) reroutes UUID to session_id — REQ-CLI-003.""" + from autoskillit.cli.app import app + from autoskillit.core import NamedResume, NoResume + + monkeypatch.delenv("CLAUDECODE", raising=False) + monkeypatch.chdir(tmp_path) + + captured: dict = {} + + def fake_launch(prompt, *, initial_message=None, extra_env=None, resume_spec=NoResume()): + captured["resume_spec"] = resume_spec + + with patch("autoskillit.cli.session._order._launch_cook_session", side_effect=fake_launch): + with pytest.raises(SystemExit) as exc_info: + app(["order", "--resume", "4b581974-1f19-4aec-8405-78c5ede5e233"]) + assert exc_info.value.code == 0 + + assert captured["resume_spec"] == NamedResume( + session_id="4b581974-1f19-4aec-8405-78c5ede5e233" + ) + + def test_order_bare_resume_skips_recipe_validation( + self, monkeypatch: pytest.MonkeyPatch, tmp_path: Path + ) -> None: + """order --resume (no uuid, no recipe) calls _launch_cook_session; no recipe validation.""" + from autoskillit.cli.app import app + from autoskillit.core import NoResume + + monkeypatch.delenv("CLAUDECODE", raising=False) + monkeypatch.chdir(tmp_path) + + captured: dict = {} + + def fake_launch(prompt, *, initial_message=None, extra_env=None, resume_spec=NoResume()): + captured["resume_spec"] = resume_spec + + with ( + patch("autoskillit.cli.session._order._launch_cook_session", side_effect=fake_launch), + patch("autoskillit.cli.session._session_picker.pick_session", return_value=None), + ): + with pytest.raises(SystemExit) as exc_info: + app(["order", "--resume"]) + assert exc_info.value.code == 0 + + assert captured, "fake_launch was never called" + assert captured["resume_spec"] == NoResume() + + def test_order_resume_uuid_does_not_validate_recipe( + self, monkeypatch: pytest.MonkeyPatch, tmp_path: Path + ) -> None: + """order --resume bypasses find_recipe_by_name — UUID is not a recipe name.""" + from autoskillit.cli.app import app + + monkeypatch.delenv("CLAUDECODE", raising=False) + monkeypatch.chdir(tmp_path) + + find_called = [] + monkeypatch.setattr( + "autoskillit.recipe.find_recipe_by_name", + lambda *a, **kw: find_called.append(a) or None, + ) + + with patch("autoskillit.cli.session._order._launch_cook_session"): + with pytest.raises(SystemExit) as exc_info: + app(["order", "--resume", "4b581974-1f19-4aec-8405-78c5ede5e233"]) + assert exc_info.value.code == 0 + + assert find_called == [], "find_recipe_by_name must NOT be called on resume without recipe" diff --git a/tests/cli/test_cook_order_prompt.py b/tests/cli/test_cook_order_prompt.py new file mode 100644 index 0000000000..9ccc44d437 --- /dev/null +++ b/tests/cli/test_cook_order_prompt.py @@ -0,0 +1,391 @@ +"""Tests: cook CLI order command — system prompt content, MCP prefix selection, ownership.""" + +from __future__ import annotations + +import shutil +import subprocess +from pathlib import Path +from unittest.mock import MagicMock, patch + +import pytest + +from autoskillit import cli +from autoskillit.core import ClaudeFlags +from tests.cli.conftest import _SCRIPT_YAML + +pytestmark = [pytest.mark.layer("cli"), pytest.mark.medium] + + +class TestCLIOrderPrompt: + @pytest.fixture(autouse=True) + def _stub_preview(self, monkeypatch: pytest.MonkeyPatch) -> None: + """Stub terminal preview to avoid subprocess.run collision with git calls.""" + monkeypatch.setattr( + "autoskillit.cli._preview.show_cook_preview", + lambda *a, **kw: None, + ) + + @pytest.fixture(autouse=True) + def _interactive_stdin(self, monkeypatch: pytest.MonkeyPatch) -> None: + """Most order() paths require an interactive TTY — default to True for this class.""" + monkeypatch.setattr("sys.stdin.isatty", lambda: True) + + @pytest.fixture(autouse=True) + def _stub_ingredients_table(self, monkeypatch: pytest.MonkeyPatch) -> None: + """Stub _get_ingredients_table in app.py to prevent subprocess.run git calls.""" + import importlib + import sys as _sys + + _app_mod = _sys.modules.get("autoskillit.cli.session._order") or importlib.import_module( + "autoskillit.cli.session._order" + ) + monkeypatch.setattr(_app_mod, "_get_ingredients_table", lambda *a, **kw: "| col | val |") + + @patch("autoskillit.cli.subprocess.run") + def test_order_system_prompt_contains_behavioral_instructions( + self, + mock_run: MagicMock, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + ) -> None: + """order injects recipe name and behavioral instructions into system prompt.""" + monkeypatch.delenv("CLAUDECODE", raising=False) + monkeypatch.chdir(tmp_path) + scripts_dir = tmp_path / ".autoskillit" / "recipes" + scripts_dir.mkdir(parents=True) + (scripts_dir / "my-script.yaml").write_text(_SCRIPT_YAML) + monkeypatch.setattr(shutil, "which", lambda cmd: "/usr/bin/claude") + monkeypatch.setattr("builtins.input", lambda _prompt="": "") + mock_run.return_value = subprocess.CompletedProcess( + args=[], returncode=0, stdout="", stderr="" + ) + + cli.order("test-script") + + cmd = mock_run.call_args[0][0] + prompt_idx = cmd.index(ClaudeFlags.APPEND_SYSTEM_PROMPT) + system_prompt = cmd[prompt_idx + 1] + assert "test-script" in system_prompt + assert "open_kitchen" in system_prompt + assert "ROUTING RULES" in system_prompt + assert "FAILURE PREDICATES" in system_prompt + assert "capture:" in system_prompt + assert "${{ context." in system_prompt + assert "AutoSkillit MCP tools" in system_prompt + assert "--- RECIPE ---" not in system_prompt + assert "do-something" not in system_prompt + + @patch("autoskillit.cli.subprocess.run") + def test_orchestrator_prompt_contains_context_limit_routing( + self, + mock_run: MagicMock, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + ) -> None: + """Orchestrator prompt must instruct routing to on_context_limit when needs_retry=true.""" + monkeypatch.delenv("CLAUDECODE", raising=False) + monkeypatch.chdir(tmp_path) + scripts_dir = tmp_path / ".autoskillit" / "recipes" + scripts_dir.mkdir(parents=True) + (scripts_dir / "my-script.yaml").write_text(_SCRIPT_YAML) + monkeypatch.setattr(shutil, "which", lambda cmd: "/usr/bin/claude") + monkeypatch.setattr("builtins.input", lambda _prompt="": "") + mock_run.return_value = subprocess.CompletedProcess( + args=[], returncode=0, stdout="", stderr="" + ) + + cli.order("test-script") + + cmd = mock_run.call_args[0][0] + prompt_idx = cmd.index(ClaudeFlags.APPEND_SYSTEM_PROMPT) + system_prompt = cmd[prompt_idx + 1] + assert "needs_retry" in system_prompt + assert "on_context_limit" in system_prompt + + @patch("autoskillit.cli.subprocess.run") + def test_order_named_recipe_only_confirmation_prompt( + self, + mock_run: MagicMock, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + ) -> None: + """order only fires confirmation prompt (no picker).""" + monkeypatch.delenv("CLAUDECODE", raising=False) + monkeypatch.chdir(tmp_path) + scripts_dir = tmp_path / ".autoskillit" / "recipes" + scripts_dir.mkdir(parents=True) + (scripts_dir / "my-script.yaml").write_text(_SCRIPT_YAML) + monkeypatch.setattr(shutil, "which", lambda cmd: "/usr/bin/claude") + mock_run.return_value = subprocess.CompletedProcess( + args=[], returncode=0, stdout="", stderr="" + ) + + input_calls = [] + monkeypatch.setattr("builtins.input", lambda prompt="": input_calls.append(prompt) or "") + + cli.order("test-script") + + assert len(input_calls) == 1, "input() should be called exactly once (confirmation)" + + @patch("autoskillit.cli.subprocess.run") + def test_order_command_includes_positional_greeting( + self, + mock_run: MagicMock, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + ) -> None: + """The order command must pass a greeting as a positional argument.""" + from autoskillit.cli._prompts import _COOK_GREETINGS + + monkeypatch.delenv("CLAUDECODE", raising=False) + monkeypatch.chdir(tmp_path) + scripts_dir = tmp_path / ".autoskillit" / "recipes" + scripts_dir.mkdir(parents=True) + (scripts_dir / "my-script.yaml").write_text(_SCRIPT_YAML) + monkeypatch.setattr(shutil, "which", lambda cmd: "/usr/bin/claude") + monkeypatch.setattr("builtins.input", lambda _prompt="": "") + mock_run.return_value = subprocess.CompletedProcess( + args=[], returncode=0, stdout="", stderr="" + ) + + cli.order("test-script") + + cmd = mock_run.call_args[0][0] + greeting_candidates = [g.format(recipe_name="test-script") for g in _COOK_GREETINGS] + assert any(arg in greeting_candidates for arg in cmd), ( + f"No greeting found as positional arg in: {cmd}" + ) + + @patch("autoskillit.cli.subprocess.run") + def test_order_open_kitchen_includes_positional_greeting( + self, + mock_run: MagicMock, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + ) -> None: + """Open-kitchen order sessions also pass a greeting as positional arg.""" + from autoskillit.cli._prompts import _OPEN_KITCHEN_GREETINGS + + monkeypatch.delenv("CLAUDECODE", raising=False) + monkeypatch.chdir(tmp_path) + scripts_dir = tmp_path / ".autoskillit" / "recipes" + scripts_dir.mkdir(parents=True) + (scripts_dir / "my-script.yaml").write_text(_SCRIPT_YAML) + monkeypatch.setattr(shutil, "which", lambda cmd: "/usr/bin/claude") + mock_run.return_value = subprocess.CompletedProcess(args=[], returncode=0) + monkeypatch.setattr("builtins.input", lambda _prompt="": "0") + + cli.order() + + cmd = mock_run.call_args[0][0] + assert any(arg in _OPEN_KITCHEN_GREETINGS for arg in cmd), ( + f"No open-kitchen greeting found as positional arg in: {cmd}" + ) + + +class TestOrderDisplayOwnership: + """order() delegates recipe display to the Claude session via load_recipe.""" + + @pytest.fixture(autouse=True) + def _stub_preview(self, monkeypatch: pytest.MonkeyPatch) -> None: + """Stub terminal preview to avoid subprocess.run collision with git calls.""" + monkeypatch.setattr( + "autoskillit.cli._preview.show_cook_preview", + lambda *a, **kw: None, + ) + + def _setup_recipe(self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Path: + """Write test recipe to scripts_dir and chdir.""" + monkeypatch.delenv("CLAUDECODE", raising=False) + monkeypatch.chdir(tmp_path) + scripts_dir = tmp_path / ".autoskillit" / "recipes" + scripts_dir.mkdir(parents=True) + (scripts_dir / "my-script.yaml").write_text(_SCRIPT_YAML) + monkeypatch.setattr(shutil, "which", lambda cmd: "/usr/bin/claude") + monkeypatch.setattr("builtins.input", lambda _prompt="": "") + monkeypatch.setattr("sys.stdin.isatty", lambda: True) + return scripts_dir + + @patch("autoskillit.cli.subprocess.run") + def test_order_system_prompt_does_not_contain_recipe_yaml( + self, + mock_run: MagicMock, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + ) -> None: + """System prompt must contain recipe NAME but not the raw YAML body.""" + self._setup_recipe(tmp_path, monkeypatch) + mock_run.return_value = subprocess.CompletedProcess( + args=[], returncode=0, stdout="", stderr="" + ) + + cli.order("test-script") + + cmd = mock_run.call_args[0][0] + prompt_idx = cmd.index(ClaudeFlags.APPEND_SYSTEM_PROMPT) + system_prompt = cmd[prompt_idx + 1] + assert "test-script" in system_prompt + assert "--- RECIPE ---" not in system_prompt + assert "--- END RECIPE ---" not in system_prompt + assert "steps:" not in system_prompt + assert "on_success:" not in system_prompt + + @patch("autoskillit.cli.subprocess.run") + def test_order_system_prompt_instructs_open_kitchen_with_recipe( + self, + mock_run: MagicMock, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + ) -> None: + """System prompt must instruct Claude to call open_kitchen(name) as its first action.""" + self._setup_recipe(tmp_path, monkeypatch) + mock_run.return_value = subprocess.CompletedProcess( + args=[], returncode=0, stdout="", stderr="" + ) + + cli.order("test-script") + + cmd = mock_run.call_args[0][0] + prompt_idx = cmd.index(ClaudeFlags.APPEND_SYSTEM_PROMPT) + system_prompt = cmd[prompt_idx + 1] + assert "open_kitchen" in system_prompt + assert "FIRST ACTION" in system_prompt + + +_PLUGIN_KEY = "autoskillit@autoskillit-local" + + +class TestOrderMcpPrefixSelection: + """order() must embed the resolved MCP prefix in the system prompt.""" + + @pytest.fixture(autouse=True) + def _clear_prefix_cache(self): # noqa: ANN204 + from autoskillit.core._plugin_ids import detect_autoskillit_mcp_prefix as _fn + + _fn.cache_clear() + yield + _fn.cache_clear() + + @pytest.fixture(autouse=True) + def _stub_preview(self, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr("autoskillit.cli._preview.show_cook_preview", lambda *a, **kw: None) + + @pytest.fixture(autouse=True) + def _interactive_stdin(self, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr("sys.stdin.isatty", lambda: True) + + @pytest.fixture(autouse=True) + def _stub_ingredients_table(self, monkeypatch: pytest.MonkeyPatch) -> None: + """Stub _get_ingredients_table in app.py to prevent subprocess.run git calls.""" + import importlib + import sys as _sys + + _app_mod = _sys.modules.get("autoskillit.cli.session._order") or importlib.import_module( + "autoskillit.cli.session._order" + ) + monkeypatch.setattr(_app_mod, "_get_ingredients_table", lambda *a, **kw: "| col | val |") + + @patch("autoskillit.cli.subprocess.run") + def test_order_prompt_uses_direct_prefix_when_no_marketplace_install( + self, mock_run: MagicMock, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + """order() builds a prompt with the direct prefix when installed_plugins.json lacks key.""" + from autoskillit.cli._mcp_names import DIRECT_PREFIX + + monkeypatch.delenv("CLAUDECODE", raising=False) + monkeypatch.chdir(tmp_path) + scripts_dir = tmp_path / ".autoskillit" / "recipes" + scripts_dir.mkdir(parents=True) + (scripts_dir / "test-script.yaml").write_text(_SCRIPT_YAML) + monkeypatch.setattr(shutil, "which", lambda cmd: "/usr/bin/claude") + monkeypatch.setattr("builtins.input", lambda _prompt="": "") + plugins_file = tmp_path / "plugins.json" + plugins_file.write_text('{"version": 2, "plugins": {}}') + monkeypatch.setattr( + "autoskillit.core._plugin_ids._installed_plugins_path", lambda: plugins_file + ) + mock_run.return_value = subprocess.CompletedProcess( + args=[], returncode=0, stdout="", stderr="" + ) + + cli.order("test-script") + + mock_run.assert_called_once() + cmd = mock_run.call_args[0][0] + prompt_idx = cmd.index(ClaudeFlags.APPEND_SYSTEM_PROMPT) + captured_prompt = cmd[prompt_idx + 1] + assert f"{DIRECT_PREFIX}open_kitchen" in captured_prompt + + @patch("autoskillit.cli.subprocess.run") + def test_order_prompt_uses_marketplace_prefix_when_plugin_installed( + self, mock_run: MagicMock, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + """order() uses marketplace prefix when autoskillit is plugin-installed.""" + from autoskillit.cli._mcp_names import MARKETPLACE_PREFIX + + monkeypatch.delenv("CLAUDECODE", raising=False) + monkeypatch.chdir(tmp_path) + scripts_dir = tmp_path / ".autoskillit" / "recipes" + scripts_dir.mkdir(parents=True) + (scripts_dir / "test-script.yaml").write_text(_SCRIPT_YAML) + monkeypatch.setattr(shutil, "which", lambda cmd: "/usr/bin/claude") + monkeypatch.setattr("builtins.input", lambda _prompt="": "") + plugins_file = tmp_path / "plugins.json" + plugins_file.write_text(f'{{"version": 2, "plugins": {{"{_PLUGIN_KEY}": []}}}}') + monkeypatch.setattr( + "autoskillit.core._plugin_ids._installed_plugins_path", lambda: plugins_file + ) + mock_run.return_value = subprocess.CompletedProcess( + args=[], returncode=0, stdout="", stderr="" + ) + + cli.order("test-script") + + mock_run.assert_called_once() + cmd = mock_run.call_args[0][0] + prompt_idx = cmd.index(ClaudeFlags.APPEND_SYSTEM_PROMPT) + captured_prompt = cmd[prompt_idx + 1] + assert f"{MARKETPLACE_PREFIX}open_kitchen" in captured_prompt + + @patch("autoskillit.cli.subprocess.run") + def test_cook_passes_ingredients_table_to_orchestrator_prompt( + self, mock_run: MagicMock, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + """cook() must pass the pre-rendered ingredients_table to _build_orchestrator_prompt.""" + captured: list[dict] = [] + + def _capturing_build( + recipe_name: str, mcp_prefix: str, ingredients_table: object = None + ) -> str: + captured.append({"ingredients_table": ingredients_table}) + return "ROUTING RULES\nFIRST ACTION\nopenKitchen\nDuring pipeline execution" + + monkeypatch.delenv("CLAUDECODE", raising=False) + monkeypatch.chdir(tmp_path) + scripts_dir = tmp_path / ".autoskillit" / "recipes" + scripts_dir.mkdir(parents=True) + (scripts_dir / "test-script.yaml").write_text(_SCRIPT_YAML) + monkeypatch.setattr(shutil, "which", lambda cmd: "/usr/bin/claude") + monkeypatch.setattr("builtins.input", lambda _prompt="": "") + plugins_file = tmp_path / "plugins.json" + plugins_file.write_text('{"version": 2, "plugins": {}}') + monkeypatch.setattr( + "autoskillit.core._plugin_ids._installed_plugins_path", lambda: plugins_file + ) + import importlib + import sys as _sys + + _app_mod = _sys.modules.get("autoskillit.cli.session._order") or importlib.import_module( + "autoskillit.cli.session._order" + ) + monkeypatch.setattr(_app_mod, "_build_orchestrator_prompt", _capturing_build) + mock_run.return_value = subprocess.CompletedProcess( + args=[], returncode=0, stdout="", stderr="" + ) + + cli.order("test-script") + + assert captured, "_build_orchestrator_prompt was not called" + assert captured[0]["ingredients_table"] == "| col | val |", ( + "app.cook() must pass pre-rendered ingredients_table to _build_orchestrator_prompt" + ) diff --git a/tests/cli/test_cook_profile.py b/tests/cli/test_cook_profile.py new file mode 100644 index 0000000000..dc3d5bdbe0 --- /dev/null +++ b/tests/cli/test_cook_profile.py @@ -0,0 +1,123 @@ +"""Tests for --profile flag in cook command.""" + +from __future__ import annotations + +from unittest.mock import MagicMock, patch + +import pytest + +import autoskillit.cli.session._cook as cook_module +from autoskillit.execution.commands import ClaudeInteractiveCmd + +pytestmark = [pytest.mark.layer("cli"), pytest.mark.medium] + + +def _fake_build_cmd(): + captured = [] + + def fake(**kwargs): + captured.append(kwargs.get("env_extras", {})) + return ClaudeInteractiveCmd(cmd=["claude"], env={}) + + return fake, captured + + +@pytest.fixture() +def _mock_mgr(): + return MagicMock() + + +def _run_cook(profile, cfg, mock_mgr): + fake_build, captured = _fake_build_cmd() + with ( + patch("shutil.which", return_value="/usr/bin/claude"), + patch("builtins.input", return_value=""), + patch("autoskillit.workspace.DefaultSessionSkillManager", return_value=mock_mgr), + patch("subprocess.run", return_value=MagicMock(returncode=0)), + patch("autoskillit.execution.build_interactive_cmd", fake_build), + patch("autoskillit.core.write_registry_entry"), + patch("autoskillit.config.load_config", return_value=cfg), + patch( + "autoskillit.cli.session._cook.is_feature_enabled", + side_effect=lambda key, *a, **kw: key == "providers", + ), + patch("autoskillit.cli.ui._timed_input.timed_prompt", return_value=""), + ): + cook_module.cook(profile=profile) + return captured + + +def test_profile_valid_injects_provider_env_var(_mock_mgr): + """AUTOSKILLIT_PROVIDER_PROFILE must be in env_extras when --profile is given.""" + cfg = MagicMock() + cfg.experimental_enabled = True + cfg.providers.profiles = {"minimax": {"ANTHROPIC_BASE_URL": "https://minimax.example"}} + captured = _run_cook("minimax", cfg, _mock_mgr) + assert len(captured) >= 1, "build_interactive_cmd was not called" + env = captured[0] + assert env.get("AUTOSKILLIT_PROVIDER_PROFILE") == "minimax" + + +def test_profile_valid_injects_profile_env_vars(_mock_mgr): + """Profile's own env vars (API creds) must be injected into env_extras.""" + cfg = MagicMock() + cfg.experimental_enabled = True + cfg.providers.profiles = { + "minimax": {"ANTHROPIC_BASE_URL": "https://mm.io", "ANTHROPIC_API_KEY": "sk-mm"} + } + captured = _run_cook("minimax", cfg, _mock_mgr) + assert len(captured) >= 1, "build_interactive_cmd was not called" + env = captured[0] + assert env.get("ANTHROPIC_BASE_URL") == "https://mm.io" + assert env.get("ANTHROPIC_API_KEY") == "sk-mm" + + +def test_profile_none_does_not_inject_provider_env(_mock_mgr): + """When profile=None, AUTOSKILLIT_PROVIDER_PROFILE must NOT appear in env_extras.""" + cfg = MagicMock() + cfg.experimental_enabled = True + cfg.providers.profiles = {} + captured = _run_cook(None, cfg, _mock_mgr) + assert len(captured) >= 1, "build_interactive_cmd was not called" + env = captured[0] + assert "AUTOSKILLIT_PROVIDER_PROFILE" not in env + + +def test_profile_feature_disabled_exits(capsys, _mock_mgr): + """SystemExit(1) with informative message when providers feature is not enabled.""" + cfg = MagicMock() + cfg.experimental_enabled = False + cfg.providers.profiles = {"minimax": {}} + fake_build, _ = _fake_build_cmd() + with ( + patch("shutil.which", return_value="/usr/bin/claude"), + patch("autoskillit.workspace.DefaultSessionSkillManager", return_value=_mock_mgr), + patch("autoskillit.execution.build_interactive_cmd", fake_build), + patch("autoskillit.config.load_config", return_value=cfg), + patch("autoskillit.cli.session._cook.is_feature_enabled", return_value=False), + ): + with pytest.raises(SystemExit) as exc_info: + cook_module.cook(profile="minimax") + assert exc_info.value.code == 1 + assert "providers" in capsys.readouterr().err + + +def test_profile_unknown_exits(capsys, _mock_mgr): + """SystemExit(1) with informative message listing known profiles for unknown name.""" + cfg = MagicMock() + cfg.experimental_enabled = True + cfg.providers.profiles = {"anthropic": {}, "openai": {}} + fake_build, _ = _fake_build_cmd() + with ( + patch("shutil.which", return_value="/usr/bin/claude"), + patch("autoskillit.workspace.DefaultSessionSkillManager", return_value=_mock_mgr), + patch("autoskillit.execution.build_interactive_cmd", fake_build), + patch("autoskillit.config.load_config", return_value=cfg), + patch("autoskillit.cli.session._cook.is_feature_enabled", return_value=True), + ): + with pytest.raises(SystemExit) as exc_info: + cook_module.cook(profile="minimax") + assert exc_info.value.code == 1 + err = capsys.readouterr().err + assert "minimax" in err + assert "anthropic" in err or "openai" in err diff --git a/tests/cli/test_cook_workspace.py b/tests/cli/test_cook_workspace.py new file mode 100644 index 0000000000..506a50ade7 --- /dev/null +++ b/tests/cli/test_cook_workspace.py @@ -0,0 +1,231 @@ +"""Tests: cook CLI workspace init and clean commands.""" + +from __future__ import annotations + +import os +import time +from pathlib import Path + +import pytest + +from autoskillit import cli +from autoskillit.cli._workspace import _format_age + +pytestmark = [pytest.mark.layer("cli"), pytest.mark.medium] + + +class TestCLIOrderWorkspace: + @pytest.fixture(autouse=True) + def _stub_preview(self, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr("autoskillit.cli._preview.show_cook_preview", lambda *a, **kw: None) + + @pytest.fixture(autouse=True) + def _interactive_stdin(self, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr("sys.stdin.isatty", lambda: True) + + @pytest.fixture(autouse=True) + def _stub_ingredients_table(self, monkeypatch: pytest.MonkeyPatch) -> None: + import importlib + import sys as _sys + + _app_mod = _sys.modules.get("autoskillit.cli.session._order") or importlib.import_module( + "autoskillit.cli.session._order" + ) + monkeypatch.setattr(_app_mod, "_get_ingredients_table", lambda *a, **kw: "| col | val |") + + def test_prep_station_init_creates_dir_with_marker( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + """prep station init creates directory and drops marker file.""" + monkeypatch.chdir(tmp_path) + target = tmp_path / "test-workspace" + cli.workspace_init(str(target)) + assert target.is_dir() + assert (target / ".autoskillit-workspace").is_file() + + def test_prep_station_init_refuses_nonempty_dir( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + """prep station init refuses to initialize a non-empty directory.""" + monkeypatch.chdir(tmp_path) + target = tmp_path / "existing" + target.mkdir() + (target / "important.txt").touch() + with pytest.raises(SystemExit): + cli.workspace_init(str(target)) + + def test_prep_station_init_idempotent_on_empty_with_marker( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + """prep station init is safe to re-run on a directory that only has the marker.""" + monkeypatch.chdir(tmp_path) + target = tmp_path / "prep-station" + cli.workspace_init(str(target)) + cli.workspace_init(str(target)) + assert (target / ".autoskillit-workspace").is_file() + + def test_prep_station_init_marker_has_content( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + """Marker file contains human-readable identifying content.""" + monkeypatch.chdir(tmp_path) + target = tmp_path / "prep-station" + cli.workspace_init(str(target)) + content = (target / ".autoskillit-workspace").read_text() + assert "autoskillit" in content + assert "do not delete" in content + + def test_workspace_clean_removes_subdirs( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture + ) -> None: + """T_WC1: workspace_clean removes stale subdirs of autoskillit-runs/.""" + runs_dir = tmp_path / "autoskillit-runs" + (runs_dir / "run-a").mkdir(parents=True) + (runs_dir / "run-b").mkdir(parents=True) + old_time = time.time() - 6 * 3600 + for d in (runs_dir / "run-a", runs_dir / "run-b"): + os.utime(d, (old_time, old_time)) + cli.workspace_clean(dir=str(tmp_path), force=True) + assert not (runs_dir / "run-a").exists() + assert not (runs_dir / "run-b").exists() + + def test_workspace_clean_reports_nothing_when_no_runs_dir( + self, tmp_path: Path, capsys: pytest.CaptureFixture + ) -> None: + """T_WC2: workspace_clean prints message when autoskillit-runs/ doesn't exist.""" + cli.workspace_clean(dir=str(tmp_path)) + captured = capsys.readouterr() + assert "No autoskillit-runs/" in captured.out + + def test_workspace_clean_defaults_to_parent_of_cwd( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + """T_WC3: workspace_clean without --dir uses parent of CWD.""" + project_dir = tmp_path / "project" + project_dir.mkdir() + monkeypatch.chdir(project_dir) + runs_dir = tmp_path / "autoskillit-runs" + (runs_dir / "run-x").mkdir(parents=True) + old_time = time.time() - 6 * 3600 + os.utime(runs_dir / "run-x", (old_time, old_time)) + cli.workspace_clean(force=True) + assert not (runs_dir / "run-x").exists() + + def test_workspace_clean_recent_dirs_skipped( + self, tmp_path: Path, capsys: pytest.CaptureFixture + ) -> None: + """T_WC4: recent dirs (<5h) are not deleted, stale dirs (>5h) are.""" + runs_dir = tmp_path / "autoskillit-runs" + (runs_dir / "recent").mkdir(parents=True) + (runs_dir / "stale").mkdir(parents=True) + old_time = time.time() - 6 * 3600 + os.utime(runs_dir / "stale", (old_time, old_time)) + cli.workspace_clean(dir=str(tmp_path), force=True) + assert (runs_dir / "recent").exists() + assert not (runs_dir / "stale").exists() + + def test_workspace_clean_boundary_5h_is_stale( + self, tmp_path: Path, capsys: pytest.CaptureFixture + ) -> None: + """T_WC5: dir with mtime at the 5h threshold is stale (>=5h threshold).""" + runs_dir = tmp_path / "autoskillit-runs" + (runs_dir / "boundary").mkdir(parents=True) + boundary_time = time.time() - 5 * 3600 - 2 + os.utime(runs_dir / "boundary", (boundary_time, boundary_time)) + cli.workspace_clean(dir=str(tmp_path), force=True) + assert not (runs_dir / "boundary").exists() + + def test_workspace_clean_skipped_dirs_printed( + self, tmp_path: Path, capsys: pytest.CaptureFixture + ) -> None: + """T_WC6: skipped (recent) dirs are printed with age.""" + runs_dir = tmp_path / "autoskillit-runs" + (runs_dir / "recent-one").mkdir(parents=True) + recent_time = time.time() - 3600 + os.utime(runs_dir / "recent-one", (recent_time, recent_time)) + cli.workspace_clean(dir=str(tmp_path), force=True) + captured = capsys.readouterr() + assert "Skipped" in captured.out + assert "recent-one" in captured.out + + def test_workspace_clean_will_remove_printed( + self, tmp_path: Path, capsys: pytest.CaptureFixture + ) -> None: + """T_WC7: stale dirs are printed as 'Will remove' with age.""" + runs_dir = tmp_path / "autoskillit-runs" + (runs_dir / "old-one").mkdir(parents=True) + old_time = time.time() - 10 * 3600 + os.utime(runs_dir / "old-one", (old_time, old_time)) + cli.workspace_clean(dir=str(tmp_path), force=True) + captured = capsys.readouterr() + assert "Will remove" in captured.out + assert "old-one" in captured.out + + def test_workspace_clean_confirm_defaults_no( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + """T_WC8: empty input at confirmation prompt defaults to N (no deletion).""" + runs_dir = tmp_path / "autoskillit-runs" + (runs_dir / "stale").mkdir(parents=True) + old_time = time.time() - 6 * 3600 + os.utime(runs_dir / "stale", (old_time, old_time)) + monkeypatch.setattr("builtins.input", lambda _prompt="": "") + cli.workspace_clean(dir=str(tmp_path)) + assert (runs_dir / "stale").exists() + + def test_workspace_clean_confirm_accepts_y( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + """T_WC9: 'y' at confirmation prompt deletes stale dirs.""" + runs_dir = tmp_path / "autoskillit-runs" + (runs_dir / "stale").mkdir(parents=True) + old_time = time.time() - 6 * 3600 + os.utime(runs_dir / "stale", (old_time, old_time)) + monkeypatch.setattr("builtins.input", lambda _prompt="": "y") + cli.workspace_clean(dir=str(tmp_path)) + assert not (runs_dir / "stale").exists() + + def test_workspace_clean_force_skips_prompt( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + """T_WC10: --force deletes without calling input().""" + runs_dir = tmp_path / "autoskillit-runs" + (runs_dir / "stale").mkdir(parents=True) + old_time = time.time() - 6 * 3600 + os.utime(runs_dir / "stale", (old_time, old_time)) + monkeypatch.setattr( + "builtins.input", + lambda _prompt="": (_ for _ in ()).throw(AssertionError("input() called")), + ) + cli.workspace_clean(dir=str(tmp_path), force=True) + assert not (runs_dir / "stale").exists() + + def test_workspace_clean_no_stale_nothing_to_clean( + self, tmp_path: Path, capsys: pytest.CaptureFixture + ) -> None: + """T_WC11: only recent dirs prints nothing-to-clean message.""" + runs_dir = tmp_path / "autoskillit-runs" + (runs_dir / "recent").mkdir(parents=True) + cli.workspace_clean(dir=str(tmp_path), force=True) + captured = capsys.readouterr() + assert "Nothing to clean" in captured.out + assert (runs_dir / "recent").exists() + + def test_format_age_various_values(self) -> None: + """T_WC14: _format_age returns human-readable age strings.""" + assert _format_age(30 * 60) == "30m ago" + assert _format_age(2 * 3600 + 14 * 60) == "2h 14m ago" + assert _format_age(3 * 86400) == "3d ago" + assert _format_age(5 * 3600) == "5h ago" + + +def test_skills_list_shows_all_sources( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture +) -> None: + """skills list outputs skill names with source labels.""" + monkeypatch.chdir(tmp_path) + cli.skills_list() + captured = capsys.readouterr() + assert "investigate" in captured.out + assert "bundled" in captured.out + assert "NAME" in captured.out diff --git a/tests/cli/test_doctor.py b/tests/cli/test_doctor.py index e6b0764115..26d1aec754 100644 --- a/tests/cli/test_doctor.py +++ b/tests/cli/test_doctor.py @@ -11,21 +11,7 @@ from autoskillit import cli -_MINIMAL_SCRIPT_YAML = """\ -name: my-script -description: A test script -steps: - do_it: - tool: run_cmd - with: - cmd: echo hello - on_success: done - done: - action: stop - message: Done -kitchen_rules: - - Only use AutoSkillit MCP tools during pipeline execution -""" +pytestmark = [pytest.mark.layer("cli"), pytest.mark.small] class TestCLIDoctor: @@ -51,7 +37,7 @@ def test_doctor_warns_dead_binary( ) monkeypatch.setattr(Path, "home", lambda: tmp_path) monkeypatch.chdir(tmp_path) - cli.doctor() + cli.doctor_cmd() captured = capsys.readouterr() assert "old-server" in captured.out assert "ERROR" in captured.out @@ -87,10 +73,16 @@ def test_doctor_ignores_healthy_coregistered_servers( (tmp_path / ".pre-commit-config.yaml").write_text( "repos:\n - repo: dummy\n hooks:\n - id: gitleaks\n" ) - # Create plugin cache directory for Check 2c - (tmp_path / ".claude" / "plugins" / "cache" / "autoskillit-local" / "autoskillit").mkdir( - parents=True, exist_ok=True + # Create plugin cache directory for Check 2c + version_consistency plugin.json + import importlib.metadata + + _cache_dir = ( + tmp_path / ".claude" / "plugins" / "cache" / "autoskillit-local" / "autoskillit" ) + _cache_dir.mkdir(parents=True, exist_ok=True) + _plugin_json = _cache_dir / ".claude-plugin" / "plugin.json" + _plugin_json.parent.mkdir(parents=True, exist_ok=True) + _plugin_json.write_text(json.dumps({"version": importlib.metadata.version("autoskillit")})) # Create installed_plugins.json for Check 2d (tmp_path / ".claude" / "plugins" / "installed_plugins.json").write_text( json.dumps({"version": 2, "plugins": {"autoskillit@autoskillit-local": {}}}) @@ -105,6 +97,18 @@ def test_doctor_ignores_healthy_coregistered_servers( settings_path = tmp_path / ".claude" / "settings.json" _evict_stale_autoskillit_hooks(settings_path) sync_hooks_to_settings(settings_path) + # Fleet checks: set SESSION_TYPE to a non-triggering value so ambient + # checks 18-20 all return OK, and stub check 23 directly so it returns OK + # without touching canonical_script_basenames (shared with hook-registration check 4). + monkeypatch.setenv("AUTOSKILLIT_SESSION_TYPE", "worker") + monkeypatch.delenv("AUTOSKILLIT_CAMPAIGN_ID", raising=False) + from autoskillit.cli.doctor import DoctorResult + from autoskillit.core import Severity + + monkeypatch.setattr( + "autoskillit.cli.doctor._check_fleet_dispatch_guard_registered", + lambda: DoctorResult(Severity.OK, "fleet_dispatch_guard_registered", "stubbed"), + ) local_bin = str(tmp_path / ".local" / "bin" / "autoskillit") with ( patch( @@ -116,7 +120,7 @@ def test_doctor_ignores_healthy_coregistered_servers( return_value=type("R", (), {"returncode": 0, "stdout": local_bin})(), ), ): - cli.doctor() + cli.doctor_cmd() captured = capsys.readouterr() assert "WARNING" not in captured.out assert "ERROR" not in captured.out @@ -133,7 +137,7 @@ def test_doctor_warns_missing_config( ) monkeypatch.setattr(Path, "home", lambda: tmp_path) monkeypatch.chdir(tmp_path) - cli.doctor() + cli.doctor_cmd() captured = capsys.readouterr() assert "No project config" in captured.out @@ -149,7 +153,7 @@ def test_doctor_json_output( ) monkeypatch.setattr(Path, "home", lambda: tmp_path) monkeypatch.chdir(tmp_path) - cli.doctor(output_json=True) + cli.doctor_cmd(output_json=True) captured = capsys.readouterr() data = json.loads(captured.out) assert "results" in data @@ -165,19 +169,46 @@ def test_doctor_result_has_severity_tiers( """doctor JSON output uses ok/warning/error severity tiers.""" monkeypatch.setattr(Path, "home", lambda: tmp_path) monkeypatch.chdir(tmp_path) - cli.doctor(output_json=True) + cli.doctor_cmd(output_json=True) captured = capsys.readouterr() data = json.loads(captured.out) severities = {r["severity"] for r in data["results"]} - assert severities <= {"ok", "warning", "error"} + assert severities <= {"ok", "warning", "error", "info"} + + def test_doctor_info_severity_not_treated_as_problem(self) -> None: + """INFO findings must not appear in the problems section.""" + from autoskillit.cli.doctor import _NON_PROBLEM + from autoskillit.core import Severity + + assert Severity.INFO in _NON_PROBLEM, "INFO must be in _NON_PROBLEM" + assert Severity.OK in _NON_PROBLEM, "OK must be in _NON_PROBLEM" + assert Severity.ERROR not in _NON_PROBLEM, "ERROR must not be in _NON_PROBLEM" + assert Severity.WARNING not in _NON_PROBLEM, "WARNING must not be in _NON_PROBLEM" def test_doctor_passes_when_versions_match( - self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture + self, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture, + request: pytest.FixtureRequest, ) -> None: - """doctor reports ok when plugin.json version matches package.""" + """doctor reports ok when cached plugin.json version matches package.""" + import importlib.metadata + + pkg_version = importlib.metadata.version("autoskillit") + cache_dir = ( + tmp_path / ".claude" / "plugins" / "cache" / "autoskillit-local" / "autoskillit" + ) + plugin_json = cache_dir / ".claude-plugin" / "plugin.json" + plugin_json.parent.mkdir(parents=True) + plugin_json.write_text(json.dumps({"version": pkg_version})) monkeypatch.setattr(Path, "home", lambda: tmp_path) monkeypatch.chdir(tmp_path) - cli.doctor(output_json=True) + from autoskillit.version import version_info as _vi + + _vi.cache_clear() + request.addfinalizer(_vi.cache_clear) + cli.doctor_cmd(output_json=True) captured = capsys.readouterr() data = json.loads(captured.out) version_checks = [r for r in data["results"] if r["check"] == "version_consistency"] @@ -190,7 +221,10 @@ def test_doctor_json_output_includes_all_checks( """doctor JSON includes entries for all core check names.""" monkeypatch.setattr(Path, "home", lambda: tmp_path) monkeypatch.chdir(tmp_path) - cli.doctor(output_json=True) + cfg_dir = tmp_path / ".autoskillit" + cfg_dir.mkdir(parents=True, exist_ok=True) + (cfg_dir / "config.yaml").write_text("features:\n fleet: true\n") + cli.doctor_cmd(output_json=True) captured = capsys.readouterr() data = json.loads(captured.out) check_names = {r["check"] for r in data["results"]} @@ -211,6 +245,17 @@ def test_doctor_json_output_includes_all_checks( "dual_mcp_registration", # ★ new "plugin_cache_exists", "installed_plugins_entry", + "ambient_session_type_skill", + "ambient_session_type_orchestrator", + "ambient_session_type_fleet", + "ambient_campaign_id", + "feature_dependencies", + "feature_registry_consistency", + "sous_chef_bundled", + "fleet_dispatch_guard_registered", + "stale_fleet_state", + "campaign_onboarding_hint", + "campaign_manifest_clone_dests", } assert expected <= check_names @@ -238,75 +283,10 @@ def test_doctor_human_output_shows_severity( ) monkeypatch.setattr(Path, "home", lambda: tmp_path) monkeypatch.chdir(tmp_path) - cli.doctor() + cli.doctor_cmd() captured = capsys.readouterr() assert "ERROR:" in captured.out - # DOC-REG-1 - def test_doctor_includes_mcp_server_registered_check( - self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture - ) -> None: - """doctor run_doctor() results include mcp_server_registered check.""" - monkeypatch.setattr(Path, "home", lambda: tmp_path) - monkeypatch.chdir(tmp_path) - cli.doctor(output_json=True) - captured = capsys.readouterr() - data = json.loads(captured.out) - check_names = {r["check"] for r in data["results"]} - assert "mcp_server_registered" in check_names - - # DOC-REG-2 - def test_doctor_includes_hook_registration_check( - self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture - ) -> None: - """doctor run_doctor() results include hook_registration check.""" - monkeypatch.setattr(Path, "home", lambda: tmp_path) - monkeypatch.chdir(tmp_path) - cli.doctor(output_json=True) - captured = capsys.readouterr() - data = json.loads(captured.out) - check_names = {r["check"] for r in data["results"]} - assert "hook_registration" in check_names - - # DOC-REG-3 - def test_doctor_marketplace_freshness_absent( - self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture - ) -> None: - """marketplace_freshness does NOT appear in doctor results.""" - monkeypatch.setattr(Path, "home", lambda: tmp_path) - monkeypatch.chdir(tmp_path) - cli.doctor(output_json=True) - captured = capsys.readouterr() - data = json.loads(captured.out) - check_names = {r["check"] for r in data["results"]} - assert "marketplace_freshness" not in check_names - - # DOC-REG-4 - def test_doctor_plugin_metadata_absent( - self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture - ) -> None: - """plugin_metadata does NOT appear in doctor results.""" - monkeypatch.setattr(Path, "home", lambda: tmp_path) - monkeypatch.chdir(tmp_path) - cli.doctor(output_json=True) - captured = capsys.readouterr() - data = json.loads(captured.out) - check_names = {r["check"] for r in data["results"]} - assert "plugin_metadata" not in check_names - - # DOC-REG-5 - def test_doctor_duplicate_mcp_server_absent( - self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture - ) -> None: - """duplicate_mcp_server does NOT appear in doctor results.""" - monkeypatch.setattr(Path, "home", lambda: tmp_path) - monkeypatch.chdir(tmp_path) - cli.doctor(output_json=True) - captured = capsys.readouterr() - data = json.loads(captured.out) - check_names = {r["check"] for r in data["results"]} - assert "duplicate_mcp_server" not in check_names - # DOC-REG-6 def test_doctor_mcp_server_registered_warns_when_absent( self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture @@ -324,7 +304,7 @@ class _NoPlugin: stdout = "" monkeypatch.setattr(subprocess, "run", lambda *a, **kw: _NoPlugin()) - cli.doctor(output_json=True) + cli.doctor_cmd(output_json=True) captured = capsys.readouterr() data = json.loads(captured.out) mcp_checks = [r for r in data["results"] if r["check"] == "mcp_server_registered"] @@ -339,7 +319,7 @@ def test_doctor_hook_registration_warns_when_scripts_missing( # settings.json does not exist — all hooks missing monkeypatch.setattr(Path, "home", lambda: tmp_path) monkeypatch.chdir(tmp_path) - cli.doctor(output_json=True) + cli.doctor_cmd(output_json=True) captured = capsys.readouterr() data = json.loads(captured.out) hook_checks = [r for r in data["results"] if r["check"] == "hook_registration"] @@ -353,7 +333,7 @@ def test_doctor_json_output_includes_new_checks_not_removed( """Doctor JSON output includes new checks but excludes the three removed checks.""" monkeypatch.setattr(Path, "home", lambda: tmp_path) monkeypatch.chdir(tmp_path) - cli.doctor(output_json=True) + cli.doctor_cmd(output_json=True) captured = capsys.readouterr() data = json.loads(captured.out) check_names = {r["check"] for r in data["results"]} @@ -364,254 +344,25 @@ def test_doctor_json_output_includes_new_checks_not_removed( assert "duplicate_mcp_server" not in check_names -class TestDoctorScriptHealth: - """Doctor check for script version staleness.""" - - # DOC1: No .autoskillit/recipes/ -> OK result - def test_no_scripts_dir_reports_ok( - self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture - ) -> None: - """doctor reports OK for script_version_health when no scripts directory exists.""" - monkeypatch.setattr(Path, "home", lambda: tmp_path) - monkeypatch.chdir(tmp_path) - # No .autoskillit/recipes/ directory created - cli.doctor(output_json=True) - captured = capsys.readouterr() - data = json.loads(captured.out) - script_checks = [r for r in data["results"] if r["check"] == "script_version_health"] - assert len(script_checks) == 1 - assert script_checks[0]["severity"] == "ok" - - # DOC2: All scripts at current version -> OK result - def test_all_scripts_at_current_version_reports_ok( - self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture - ) -> None: - """doctor reports OK when all scripts carry the current installed version.""" - import autoskillit - - current_version = autoskillit.__version__ - monkeypatch.setattr(Path, "home", lambda: tmp_path) - monkeypatch.chdir(tmp_path) - scripts_dir = tmp_path / ".autoskillit" / "recipes" - scripts_dir.mkdir(parents=True) - (scripts_dir / "up-to-date.yaml").write_text( - f"name: up-to-date\ndescription: Current version\n" - f'autoskillit_version: "{current_version}"\n' - + _MINIMAL_SCRIPT_YAML.split("\n", 2)[2] # reuse steps/constraints block - ) - cli.doctor(output_json=True) - captured = capsys.readouterr() - data = json.loads(captured.out) - script_checks = [r for r in data["results"] if r["check"] == "script_version_health"] - assert len(script_checks) == 1 - assert script_checks[0]["severity"] == "ok" - - # DOC3: Scripts below current version -> WARNING result with recipe names - def test_outdated_scripts_reports_warning_with_count( - self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture - ) -> None: - """doctor reports WARNING with recipe names when scripts have an older version.""" - import autoskillit - - monkeypatch.setattr(autoskillit, "__version__", "99.0.0") - monkeypatch.setattr(Path, "home", lambda: tmp_path) - monkeypatch.chdir(tmp_path) - scripts_dir = tmp_path / ".autoskillit" / "recipes" - scripts_dir.mkdir(parents=True) - (scripts_dir / "old-script.yaml").write_text( - 'name: old-script\ndescription: Old\nautoskillit_version: "0.1.0"\n' - ) - (scripts_dir / "also-old.yaml").write_text( - 'name: also-old\ndescription: Also old\nautoskillit_version: "0.1.0"\n' - ) - cli.doctor(output_json=True) - captured = capsys.readouterr() - data = json.loads(captured.out) - script_checks = [r for r in data["results"] if r["check"] == "script_version_health"] - assert len(script_checks) == 1 - assert script_checks[0]["severity"] == "warning" - assert "old-script" in script_checks[0]["message"] - - # DOC4: Scripts with no version field -> WARNING result - def test_scripts_without_version_reports_warning( - self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture - ) -> None: - """doctor reports WARNING when script YAML has no autoskillit_version field.""" - monkeypatch.setattr(Path, "home", lambda: tmp_path) - monkeypatch.chdir(tmp_path) - scripts_dir = tmp_path / ".autoskillit" / "recipes" - scripts_dir.mkdir(parents=True) - (scripts_dir / "no-version.yaml").write_text( - "name: no-version\ndescription: No version field\n" - ) - cli.doctor(output_json=True) - captured = capsys.readouterr() - data = json.loads(captured.out) - script_checks = [r for r in data["results"] if r["check"] == "script_version_health"] - assert len(script_checks) == 1 - assert script_checks[0]["severity"] == "warning" - - def _setup_recipe(self, scripts_dir: Path, name: str, version: str = "0.1.0") -> None: - (scripts_dir / f"{name}.yaml").write_text( - f'name: {name}\ndescription: Test\nautoskillit_version: "{version}"\n' - ) - - def _write_failures_json(self, tmp_path: Path, name: str, retries: int = 3) -> None: - import json as _json - - failures_path = tmp_path / ".autoskillit" / "temp" / "migrations" / "failures.json" - failures_path.parent.mkdir(parents=True, exist_ok=True) - failures_path.write_text( - _json.dumps( - { - name: { - "name": name, - "file_path": f"/fake/{name}.yaml", - "file_type": "recipe", - "timestamp": "2026-01-01T00:00:00+00:00", - "error": "validation failed after retries", - "retries_attempted": retries, - } - } - ) - ) - - # DR1: failures.json has an entry for a recipe -> error severity - def test_doctor_error_on_failed_migration( - self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture - ) -> None: - """doctor reports error severity when failures.json has an entry for a recipe.""" - import autoskillit - - monkeypatch.setattr(autoskillit, "__version__", "99.0.0") - monkeypatch.setattr(Path, "home", lambda: tmp_path) - monkeypatch.chdir(tmp_path) - scripts_dir = tmp_path / ".autoskillit" / "recipes" - scripts_dir.mkdir(parents=True) - self._setup_recipe(scripts_dir, "broken-pipeline") - self._write_failures_json(tmp_path, "broken-pipeline", retries=3) - cli.doctor(output_json=True) - captured = capsys.readouterr() - data = json.loads(captured.out) - script_checks = [r for r in data["results"] if r["check"] == "script_version_health"] - assert len(script_checks) == 1 - assert script_checks[0]["severity"] == "error" - - # DR2: Error message includes retries_attempted value from failure record - def test_doctor_error_message_includes_retry_count( - self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture - ) -> None: - """doctor error message includes retries_attempted value from the failure record.""" - import autoskillit - - monkeypatch.setattr(autoskillit, "__version__", "99.0.0") - monkeypatch.setattr(Path, "home", lambda: tmp_path) - monkeypatch.chdir(tmp_path) - scripts_dir = tmp_path / ".autoskillit" / "recipes" - scripts_dir.mkdir(parents=True) - self._setup_recipe(scripts_dir, "my-pipeline") - self._write_failures_json(tmp_path, "my-pipeline", retries=3) - cli.doctor(output_json=True) - captured = capsys.readouterr() - data = json.loads(captured.out) - script_checks = [r for r in data["results"] if r["check"] == "script_version_health"] - assert len(script_checks) == 1 - assert "3" in script_checks[0]["message"] - - # DR3: Outdated recipe with no failure record -> warning severity - def test_doctor_warning_on_simply_outdated( - self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture - ) -> None: - """doctor reports warning when recipe is outdated but has no failure record.""" - import autoskillit - - monkeypatch.setattr(autoskillit, "__version__", "99.0.0") - monkeypatch.setattr(Path, "home", lambda: tmp_path) - monkeypatch.chdir(tmp_path) - scripts_dir = tmp_path / ".autoskillit" / "recipes" - scripts_dir.mkdir(parents=True) - self._setup_recipe(scripts_dir, "outdated-pipeline") - # No failures.json written - cli.doctor(output_json=True) - captured = capsys.readouterr() - data = json.loads(captured.out) - script_checks = [r for r in data["results"] if r["check"] == "script_version_health"] - assert len(script_checks) == 1 - assert script_checks[0]["severity"] == "warning" - - # DR4: All recipes current, no failures.json -> ok - def test_doctor_ok_when_all_current( - self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture - ) -> None: - """doctor reports ok when all recipes are at current version and no failures.json.""" - import autoskillit - - current_version = autoskillit.__version__ - monkeypatch.setattr(Path, "home", lambda: tmp_path) - monkeypatch.chdir(tmp_path) - scripts_dir = tmp_path / ".autoskillit" / "recipes" - scripts_dir.mkdir(parents=True) - self._setup_recipe(scripts_dir, "current-pipeline", version=current_version) - # No failures.json written - cli.doctor(output_json=True) - captured = capsys.readouterr() - data = json.loads(captured.out) - script_checks = [r for r in data["results"] if r["check"] == "script_version_health"] - assert len(script_checks) == 1 - assert script_checks[0]["severity"] == "ok" - - # DR5: Warning message says "Will be auto-migrated on next load" - def test_doctor_outdated_message_updated( - self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture - ) -> None: - """doctor warning message says 'Will be auto-migrated on next load'.""" - import autoskillit - - monkeypatch.setattr(autoskillit, "__version__", "99.0.0") - monkeypatch.setattr(Path, "home", lambda: tmp_path) - monkeypatch.chdir(tmp_path) - scripts_dir = tmp_path / ".autoskillit" / "recipes" - scripts_dir.mkdir(parents=True) - self._setup_recipe(scripts_dir, "stale-pipeline") - cli.doctor(output_json=True) - captured = capsys.readouterr() - data = json.loads(captured.out) - script_checks = [r for r in data["results"] if r["check"] == "script_version_health"] - assert len(script_checks) == 1 - assert script_checks[0]["severity"] == "warning" - assert "Will be auto-migrated on next load" in script_checks[0]["message"] - - -class TestSyncRemovalCLI: - def test_doctor_has_no_recipe_sync_check( - self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture - ): - """REQ-APP-006: doctor output does not include recipe_sync_status.""" - monkeypatch.chdir(tmp_path) - cli.doctor() - captured = capsys.readouterr() - assert "recipe_sync_status" not in captured.out - - class TestGroupFDoctor: """P8-2, P3-2: CLI refactoring — doctor delegation tests from TestGroupFRefactoring.""" def test_doctor_delegates_to_doctor_module(self, monkeypatch, capsys): - """cli.doctor() must delegate to cli._doctor.run_doctor(), not contain the logic itself.""" - from autoskillit.cli import _doctor + """cli.doctor_cmd() must delegate to cli.doctor.run_doctor(), not contain the logic.""" + import autoskillit.cli.doctor as _doctor_mod called_with: dict = {} def mock_run_doctor(*, output_json: bool = False) -> None: called_with["output_json"] = output_json - monkeypatch.setattr(_doctor, "run_doctor", mock_run_doctor) - cli.doctor(output_json=True) + monkeypatch.setattr(_doctor_mod, "run_doctor", mock_run_doctor) + cli.doctor_cmd(output_json=True) assert called_with == {"output_json": True} def test_severity_and_doctorresult_in_doctor_module(self): - """Severity and DoctorResult must be importable from autoskillit.cli._doctor.""" - from autoskillit.cli._doctor import DoctorResult, Severity + """Severity and DoctorResult must be importable from autoskillit.cli.doctor.""" + from autoskillit.cli.doctor import DoctorResult, Severity r = DoctorResult(severity=Severity.OK, check="test", message="ok") assert r.severity == Severity.OK @@ -624,7 +375,7 @@ def test_doctor_fix_parameter_does_not_exist(): from autoskillit import cli - sig = inspect.signature(cli.doctor) + sig = inspect.signature(cli.doctor_cmd) assert "fix" not in sig.parameters, "doctor --fix is a silent no-op and must be removed" @@ -644,13 +395,32 @@ def test_doctor_does_not_modify_plugin_state(tmp_path, monkeypatch, capsys): ) ) - cli.doctor() + retiring_json = tmp_path / ".autoskillit" / "retiring_cache.json" + retiring_json.parent.mkdir(parents=True, exist_ok=True) + retiring_content = json.dumps( + { + "retiring": [ + { + "version": "0.3.0", + "path": str(cache_dir / "0.3.0"), + "retired_at": "2026-01-01T00:00:00+00:00", + } + ], + "schema_version": 1, + } + ) + retiring_json.write_text(retiring_content) + + cli.doctor_cmd() assert cache_dir.exists(), "Doctor must not delete the plugin cache directory" data = json.loads(plugins_json.read_text()) assert "autoskillit@autoskillit-local" in data.get("plugins", {}), ( "Doctor must not remove installed_plugins.json entries" ) + assert retiring_json.read_text() == retiring_content, ( + "Doctor must not modify retiring_cache.json" + ) def test_doctor_checks_plugin_cache_exists(tmp_path, monkeypatch, capsys): @@ -670,12 +440,12 @@ def test_doctor_checks_plugin_cache_exists(tmp_path, monkeypatch, capsys): editable_source=None, ), ) - cli.doctor(output_json=True) + cli.doctor_cmd(output_json=True) captured = capsys.readouterr() data = json.loads(captured.out) checks = [r for r in data["results"] if r["check"] == "plugin_cache_exists"] assert len(checks) == 1, "Expected a plugin_cache_exists check" - assert checks[0]["severity"] in ("warning", "error") + assert checks[0]["severity"] == "warning" def test_doctor_checks_installed_plugins_entry(tmp_path, monkeypatch, capsys): @@ -686,12 +456,12 @@ def test_doctor_checks_installed_plugins_entry(tmp_path, monkeypatch, capsys): plugins_dir = tmp_path / ".claude" / "plugins" plugins_dir.mkdir(parents=True) (plugins_dir / "installed_plugins.json").write_text("{}") - cli.doctor(output_json=True) + cli.doctor_cmd(output_json=True) captured = capsys.readouterr() data = json.loads(captured.out) checks = [r for r in data["results"] if r["check"] == "installed_plugins_entry"] assert len(checks) == 1, "Expected an installed_plugins_entry check" - assert checks[0]["severity"] in ("warning", "error") + assert checks[0]["severity"] == "warning" def test_stale_gate_check_absent_from_doctor_output(tmp_path, monkeypatch, capsys): @@ -700,7 +470,7 @@ def test_stale_gate_check_absent_from_doctor_output(tmp_path, monkeypatch, capsy monkeypatch.chdir(tmp_path) from autoskillit import cli - cli.doctor(output_json=True) + cli.doctor_cmd(output_json=True) captured = capsys.readouterr() data = json.loads(captured.out) check_names = {r["check"] for r in data["results"]} @@ -713,7 +483,7 @@ def test_doctor_detects_plugin_registration(monkeypatch: pytest.MonkeyPatch) -> import subprocess import tempfile - from autoskillit.cli._doctor import _check_mcp_server_registered + from autoskillit.cli.doctor import _check_mcp_server_registered from autoskillit.core import Severity fake_claude_json_content = _json.dumps({"mcpServers": {}}) # No mcpServers entry @@ -751,7 +521,7 @@ def test_doctor_warns_on_missing_gitignore_entry( (autoskillit_dir / ".secrets.yaml").write_text("github:\n token: ''\n") monkeypatch.chdir(tmp_path) - from autoskillit.cli._doctor import _check_gitignore_completeness + from autoskillit.cli.doctor import _check_gitignore_completeness from autoskillit.core import Severity result = _check_gitignore_completeness(tmp_path) @@ -775,7 +545,7 @@ def test_doctor_gitignore_ok_when_all_covered( (autoskillit_dir / ".secrets.yaml").write_text("github:\n token: ''\n") monkeypatch.chdir(tmp_path) - from autoskillit.cli._doctor import _check_gitignore_completeness + from autoskillit.cli.doctor import _check_gitignore_completeness from autoskillit.core import Severity result = _check_gitignore_completeness(tmp_path) @@ -789,7 +559,7 @@ def test_doctor_includes_secret_scanning_hook_check( """doctor output includes the secret_scanning_hook check.""" monkeypatch.setattr(Path, "home", lambda: tmp_path) monkeypatch.chdir(tmp_path) - cli.doctor(output_json=True) + cli.doctor_cmd(output_json=True) data = json.loads(capsys.readouterr().out) check_names = {r["check"] for r in data["results"]} assert "secret_scanning_hook" in check_names @@ -803,7 +573,7 @@ def test_doctor_error_when_no_scanner_present( monkeypatch.setattr(Path, "home", lambda: tmp_path) monkeypatch.chdir(tmp_path) # No .pre-commit-config.yaml - cli.doctor(output_json=True) + cli.doctor_cmd(output_json=True) data = json.loads(capsys.readouterr().out) checks = [r for r in data["results"] if r["check"] == "secret_scanning_hook"] assert len(checks) == 1 @@ -821,7 +591,7 @@ def test_doctor_ok_when_scanner_present( "repos:\n - repo: https://github.com/gitleaks/gitleaks\n" " hooks:\n - id: gitleaks\n" ) - cli.doctor(output_json=True) + cli.doctor_cmd(output_json=True) data = json.loads(capsys.readouterr().out) checks = [r for r in data["results"] if r["check"] == "secret_scanning_hook"] assert len(checks) == 1 @@ -831,7 +601,7 @@ def test_doctor_ok_when_scanner_present( # SS-DOC-4 (unit test for check function directly) def test_check_secret_scanning_hook_ok_with_gitleaks(tmp_path: Path) -> None: """_check_secret_scanning_hook returns OK when gitleaks hook is present.""" - from autoskillit.cli._doctor import _check_secret_scanning_hook + from autoskillit.cli.doctor import _check_secret_scanning_hook from autoskillit.core import Severity (tmp_path / ".pre-commit-config.yaml").write_text( @@ -844,7 +614,7 @@ def test_check_secret_scanning_hook_ok_with_gitleaks(tmp_path: Path) -> None: # SS-DOC-5 (unit test for check function directly) def test_check_secret_scanning_hook_error_without_scanner(tmp_path: Path) -> None: """_check_secret_scanning_hook returns ERROR when no .pre-commit-config.yaml.""" - from autoskillit.cli._doctor import _check_secret_scanning_hook + from autoskillit.cli.doctor import _check_secret_scanning_hook from autoskillit.core import Severity result = _check_secret_scanning_hook(tmp_path) @@ -859,7 +629,7 @@ def test_doctor_detects_misplaced_token_in_project_config( home has no config so the function must detect the violation via the project path. """ - from autoskillit.cli._doctor import _check_config_layers_for_secrets + from autoskillit.cli.doctor import _check_config_layers_for_secrets from autoskillit.core import Severity home_dir = tmp_path / "home" @@ -886,7 +656,7 @@ def test_doctor_reports_ok_when_no_misplaced_secrets( home has no config; only the project config exists with a clean (non-secret) key. This exercises the project path independently of the home path. """ - from autoskillit.cli._doctor import _check_config_layers_for_secrets + from autoskillit.cli.doctor import _check_config_layers_for_secrets from autoskillit.core import Severity home_dir = tmp_path / "home" @@ -904,8 +674,8 @@ def test_doctor_reports_ok_when_no_misplaced_secrets( # DC-11: _check_hook_registry_drift — deployed matches canonical → OK def test_check_hook_registry_drift_ok(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: - from autoskillit.cli._doctor import _check_hook_registry_drift from autoskillit.cli._hooks import _evict_stale_autoskillit_hooks, sync_hooks_to_settings + from autoskillit.cli.doctor._doctor_hooks import _check_hook_registry_drift from autoskillit.core import Severity settings = tmp_path / "settings.json" @@ -920,7 +690,7 @@ def test_check_hook_registry_drift_ok(tmp_path: Path, monkeypatch: pytest.Monkey def test_check_hook_registry_drift_warning(tmp_path: Path) -> None: import json - from autoskillit.cli._doctor import _check_hook_registry_drift + from autoskillit.cli.doctor._doctor_hooks import _check_hook_registry_drift from autoskillit.core import Severity settings = tmp_path / "settings.json" @@ -941,7 +711,7 @@ def test_doctor_json_output_includes_hook_registry_drift( monkeypatch.setattr(Path, "home", lambda: tmp_path) monkeypatch.chdir(tmp_path) - cli.doctor(output_json=True) + cli.doctor_cmd(output_json=True) data = json.loads(capsys.readouterr().out) drift = next(r for r in data["results"] if r["check"] == "hook_registry_drift") # No settings.json in tmp_path → all canonical hooks missing → WARNING @@ -955,7 +725,7 @@ def test_check_ok_when_not_editable(self, monkeypatch: pytest.MonkeyPatch) -> No """Non-editable install → OK.""" import importlib.metadata as meta - from autoskillit.cli._doctor import _check_editable_install_source_exists + from autoskillit.cli.doctor import _check_editable_install_source_exists class FakeDist: def read_text(self, filename: str) -> str | None: @@ -974,7 +744,7 @@ def test_check_error_when_editable_source_missing( """Editable install pointing to a deleted directory → ERROR.""" import importlib.metadata as meta - from autoskillit.cli._doctor import _check_editable_install_source_exists + from autoskillit.cli.doctor import _check_editable_install_source_exists deleted_path = tmp_path / "deleted-worktree" / "src" # Do NOT create deleted_path — it does not exist @@ -1002,7 +772,7 @@ def test_check_ok_when_editable_source_exists( """Editable install pointing to an existing directory → OK.""" import importlib.metadata as meta - from autoskillit.cli._doctor import _check_editable_install_source_exists + from autoskillit.cli.doctor import _check_editable_install_source_exists existing_path = tmp_path / "src" existing_path.mkdir() @@ -1027,7 +797,7 @@ def test_check_ok_when_not_installed(self, monkeypatch: pytest.MonkeyPatch) -> N """PackageNotFoundError → check returns OK (not installed in this env).""" import importlib.metadata as meta - from autoskillit.cli._doctor import _check_editable_install_source_exists + from autoskillit.cli.doctor import _check_editable_install_source_exists monkeypatch.setattr( meta.Distribution, @@ -1048,7 +818,7 @@ def test_check_ok_when_single_entry_point_in_local_bin( """Single autoskillit binary at ~/.local/bin → OK.""" import subprocess - from autoskillit.cli._doctor import _check_stale_entry_points + from autoskillit.cli.doctor import _check_stale_entry_points local_bin_path = str(Path.home() / ".local/bin/autoskillit") monkeypatch.setattr(shutil, "which", lambda name: local_bin_path) @@ -1067,7 +837,7 @@ def test_check_warning_when_stale_entry_point_outside_local_bin( """autoskillit binary outside ~/.local/bin → WARNING.""" import subprocess - from autoskillit.cli._doctor import _check_stale_entry_points + from autoskillit.cli.doctor import _check_stale_entry_points stale_path = "/usr/local/micromamba/bin/autoskillit" monkeypatch.setattr(shutil, "which", lambda name: stale_path) @@ -1091,7 +861,7 @@ def test_check_warning_when_stale_entry_point_outside_local_bin( def test_doctor_hook_health_checks_all_event_types(tmp_path: Path) -> None: """hook_health must verify PostToolUse and SessionStart scripts exist, not just PreToolUse.""" - from autoskillit.cli._doctor import _check_hook_health + from autoskillit.cli.doctor._doctor_hooks import _check_hook_health from autoskillit.core import Severity # Write a settings.json that includes token_summary_hook (PostToolUse) @@ -1121,12 +891,46 @@ def test_doctor_hook_health_checks_all_event_types(tmp_path: Path) -> None: assert "token_summary_hook" in result.message or "PostToolUse" in result.message +# T-WT-3: _check_hook_health_all_scopes detects broken paths in project scope +def test_check_hook_health_detects_broken_paths_in_project_scope(tmp_path: Path) -> None: + """_check_hook_health_all_scopes must detect broken hooks in project scope, not just user.""" + from autoskillit.cli.doctor import _check_hook_health_all_scopes + from autoskillit.core import Severity + + # Setup project-scope settings with a broken hook path + project_settings = tmp_path / ".claude" / "settings.json" + project_settings.parent.mkdir(parents=True) + project_settings.write_text( + json.dumps( + { + "hooks": { + "PreToolUse": [ + { + "matcher": ".*", + "hooks": [ + { + "type": "command", + "command": "python3 /deleted/worktree/hooks/quota_guard.py", + } + ], + } + ] + } + } + ) + ) + + results = _check_hook_health_all_scopes(tmp_path) + broken = [r for r in results if r.severity != Severity.OK] + assert broken, "Must detect broken hook paths in project scope" + + # T-DRIFT-1: _count_hook_registry_drift() detects orphaned hooks def test_count_hook_registry_drift_detects_orphaned_hooks(tmp_path: Path) -> None: """deployed − canonical must be counted and returned. Orphaned hooks are the fatal failure mode (ENOENT on every tool call). """ - from autoskillit.cli._doctor import _count_hook_registry_drift + from autoskillit.hook_registry import _count_hook_registry_drift settings = tmp_path / ".claude" / "settings.json" settings.parent.mkdir() @@ -1151,7 +955,7 @@ def test_count_hook_registry_drift_detects_orphaned_hooks(tmp_path: Path) -> Non # T-DRIFT-2: _check_hook_registry_drift() returns ERROR for orphaned hooks def test_check_hook_registry_drift_error_on_orphaned_hooks(tmp_path: Path) -> None: - from autoskillit.cli._doctor import _check_hook_registry_drift + from autoskillit.cli.doctor._doctor_hooks import _check_hook_registry_drift from autoskillit.core import Severity settings = tmp_path / ".claude" / "settings.json" @@ -1259,21 +1063,22 @@ def test_check_source_version_drift_ok_outside_source_repo( tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: """GIT_VCS install with empty cache reports OK (no drift observable).""" - from autoskillit.cli._doctor import _check_source_version_drift from autoskillit.cli._install_info import InstallInfo, InstallType + from autoskillit.cli.doctor import _check_source_version_drift from autoskillit.core import Severity info = InstallInfo( install_type=InstallType.GIT_VCS, commit_id="abc1234", - requested_revision="integration", + requested_revision="develop", url=None, editable_source=None, ) monkeypatch.setattr("autoskillit.cli._install_info.detect_install", lambda: info) # Simulate empty cache and no source repo: resolve returns None monkeypatch.setattr( - "autoskillit.cli._update_checks.resolve_reference_sha", lambda info, home, **kw: None + "autoskillit.cli.update._update_checks.resolve_reference_sha", + lambda info, home, **kw: None, ) result = _check_source_version_drift(home=tmp_path) @@ -1284,8 +1089,8 @@ def test_check_source_version_drift_ok_for_editable_install( tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: """LOCAL_EDITABLE installs are under active development — drift check is skipped.""" - from autoskillit.cli._doctor import _check_source_version_drift from autoskillit.cli._install_info import InstallInfo, InstallType + from autoskillit.cli.doctor import _check_source_version_drift from autoskillit.core import Severity info = InstallInfo( @@ -1306,8 +1111,8 @@ def test_check_source_version_drift_ok_for_pinned_sha( tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: """When requested_revision == commit_id, resolve_reference_sha short-circuits → no drift.""" - from autoskillit.cli._doctor import _check_source_version_drift from autoskillit.cli._install_info import InstallInfo, InstallType + from autoskillit.cli.doctor import _check_source_version_drift from autoskillit.core import Severity sha = "abcdef1234567890abcdef1234567890" @@ -1321,7 +1126,7 @@ def test_check_source_version_drift_ok_for_pinned_sha( monkeypatch.setattr("autoskillit.cli._install_info.detect_install", lambda: info) # When requested_revision == commit_id, resolve_reference_sha returns commit_id monkeypatch.setattr( - "autoskillit.cli._update_checks.resolve_reference_sha", lambda info, home, **kw: sha + "autoskillit.cli.update._update_checks.resolve_reference_sha", lambda info, home, **kw: sha ) result = _check_source_version_drift(home=tmp_path) @@ -1332,20 +1137,21 @@ def test_check_source_version_drift_ok_when_cache_empty( tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: """When SHA cannot be resolved (network/cache miss), doctor reports OK.""" - from autoskillit.cli._doctor import _check_source_version_drift from autoskillit.cli._install_info import InstallInfo, InstallType + from autoskillit.cli.doctor import _check_source_version_drift from autoskillit.core import Severity info = InstallInfo( install_type=InstallType.GIT_VCS, commit_id="installed123", - requested_revision="integration", + requested_revision="develop", url=None, editable_source=None, ) monkeypatch.setattr("autoskillit.cli._install_info.detect_install", lambda: info) monkeypatch.setattr( - "autoskillit.cli._update_checks.resolve_reference_sha", lambda info, home, **kw: None + "autoskillit.cli.update._update_checks.resolve_reference_sha", + lambda info, home, **kw: None, ) result = _check_source_version_drift(home=tmp_path) @@ -1360,8 +1166,8 @@ def test_check_source_version_drift_warning_on_drift( tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: """When cache has a different reference SHA than installed, reports WARNING with short SHAs.""" - from autoskillit.cli._doctor import _check_source_version_drift from autoskillit.cli._install_info import InstallInfo, InstallType + from autoskillit.cli.doctor import _check_source_version_drift from autoskillit.core import Severity installed_sha = "installed123abc" @@ -1370,13 +1176,14 @@ def test_check_source_version_drift_warning_on_drift( info = InstallInfo( install_type=InstallType.GIT_VCS, commit_id=installed_sha, - requested_revision="integration", + requested_revision="develop", url=None, editable_source=None, ) monkeypatch.setattr("autoskillit.cli._install_info.detect_install", lambda: info) monkeypatch.setattr( - "autoskillit.cli._update_checks.resolve_reference_sha", lambda info, home, **kw: ref_sha + "autoskillit.cli.update._update_checks.resolve_reference_sha", + lambda info, home, **kw: ref_sha, ) result = _check_source_version_drift(home=tmp_path) @@ -1385,439 +1192,152 @@ def test_check_source_version_drift_warning_on_drift( assert ref_sha[:8] in result.message -# --------------------------------------------------------------------------- -# Check 14: Quota cache schema version (#711 Part B, Phase 4) -# --------------------------------------------------------------------------- - - -class TestCheckQuotaCacheSchema: - """Tests for _check_quota_cache_schema doctor check.""" - - def test_check_quota_cache_schema_ok_when_current(self, tmp_path): - import json - - from autoskillit.cli._doctor import Severity, _check_quota_cache_schema - from autoskillit.execution import QUOTA_CACHE_SCHEMA_VERSION - - cache = tmp_path / "cache.json" - cache.write_text( - json.dumps( - {"schema_version": QUOTA_CACHE_SCHEMA_VERSION, "fetched_at": "2026-01-01T00:00:00"} - ) - ) - result = _check_quota_cache_schema(cache_path=cache) - assert result.severity == Severity.OK - assert f"v{QUOTA_CACHE_SCHEMA_VERSION}" in result.message - - def test_check_quota_cache_schema_ok_when_missing(self, tmp_path): - from autoskillit.cli._doctor import Severity, _check_quota_cache_schema - - cache = tmp_path / "nonexistent.json" - result = _check_quota_cache_schema(cache_path=cache) - assert result.severity == Severity.OK - assert "No quota cache" in result.message - - def test_check_quota_cache_schema_warning_when_no_schema_version_key(self, tmp_path): - import json - - from autoskillit.cli._doctor import Severity, _check_quota_cache_schema - - cache = tmp_path / "cache.json" - cache.write_text(json.dumps({"fetched_at": "2026-01-01T00:00:00"})) - result = _check_quota_cache_schema(cache_path=cache) - assert result.severity == Severity.WARNING - assert "schema drift" in result.message.lower() - - def test_check_quota_cache_schema_warning_when_older_schema_version(self, tmp_path): - import json - - from autoskillit.cli._doctor import Severity, _check_quota_cache_schema - - cache = tmp_path / "cache.json" - cache.write_text(json.dumps({"schema_version": 1, "fetched_at": "2026-01-01T00:00:00"})) - result = _check_quota_cache_schema(cache_path=cache) - assert result.severity == Severity.WARNING - - def test_check_quota_cache_schema_warning_includes_cache_path_and_observed_value( - self, tmp_path - ): - import json - - from autoskillit.cli._doctor import Severity, _check_quota_cache_schema - - cache = tmp_path / "cache.json" - cache.write_text(json.dumps({"schema_version": 1})) - result = _check_quota_cache_schema(cache_path=cache) - assert result.severity == Severity.WARNING - assert str(cache) in result.message - assert "observed=1" in result.message - - -def test_doctor_reports_drift_in_project_scope( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch -) -> None: - """_check_hook_registry_drift must report drift found in project scope.""" +# T-CACHE-INTEGRITY-1: doctor detects plugin cache hooks.json with broken paths +def test_doctor_plugin_cache_integrity(tmp_path: Path) -> None: + """_check_plugin_cache_integrity must return ERROR when cached hooks.json has broken paths.""" import json as _json - from autoskillit.cli._doctor import _check_hook_registry_drift + from autoskillit.cli.doctor._doctor_mcp import _check_plugin_cache_integrity from autoskillit.core import Severity - # Seed a stale pretty_output.py in project scope - project_settings = tmp_path / ".claude" / "settings.json" - project_settings.parent.mkdir(parents=True) - project_settings.write_text( - _json.dumps( - { - "hooks": { - "PostToolUse": [ + fake_cache = tmp_path / "cache" + version_dir = fake_cache / "0.9.347" + version_dir.mkdir(parents=True) + stale_hooks = { + "hooks": { + "PreToolUse": [ + { + "matcher": ".*", + "hooks": [ { - "matcher": "mcp__.*autoskillit.*", - "hooks": [ - {"type": "command", "command": "python3 /stale/pretty_output.py"} - ], + "type": "command", + "command": f"python3 {tmp_path}/hooks/quota_guard.py", } - ] + ], } - } - ) - ) - - result = _check_hook_registry_drift(project_settings, scope_label="project") - assert result.severity == Severity.ERROR - assert "[project]" in result.message - assert "pretty_output.py" in result.message - - -# --------------------------------------------------------------------------- -# REQ-DOCTOR-001 — _check_claude_process_state_breakdown -# --------------------------------------------------------------------------- - - -class TestCheckClaudeProcessStateBreakdown: - """Tests for the claude_process_state doctor check (Check 15).""" - - def _ps_result(self, stdout: str, returncode: int = 0): - return type( - "CompletedProcess", - (), - {"returncode": returncode, "stdout": stdout}, - )() - - def test_ok_when_only_sleeping(self, monkeypatch: pytest.MonkeyPatch) -> None: - """Single sleeping claude process → Severity.OK with state breakdown.""" - import subprocess - - from autoskillit.cli._doctor import Severity, _check_claude_process_state_breakdown - - header = "PID STAT %CPU COMMAND\n" - monkeypatch.setattr( - subprocess, - "run", - lambda *a, **kw: self._ps_result(header + "1234 S 0.5 claude"), - ) - result = _check_claude_process_state_breakdown() - assert result.severity == Severity.OK - assert "S=1" in result.message - - def test_warns_on_d_state(self, monkeypatch: pytest.MonkeyPatch) -> None: - """claude process in D state → Severity.WARNING with pid and pcpu in message.""" - import subprocess - - from autoskillit.cli._doctor import Severity, _check_claude_process_state_breakdown - - header = "PID STAT %CPU COMMAND\n" - monkeypatch.setattr( - subprocess, - "run", - lambda *a, **kw: self._ps_result(header + "1234 D 99.0 claude"), - ) - result = _check_claude_process_state_breakdown() - assert result.severity == Severity.WARNING - assert "D=1" in result.message - assert "99.0" in result.message - - def test_ok_when_no_claude_processes(self, monkeypatch: pytest.MonkeyPatch) -> None: - """Empty ps output (no claude rows) → Severity.OK, 'No claude processes running'.""" - import subprocess - - from autoskillit.cli._doctor import Severity, _check_claude_process_state_breakdown - - header = "PID STAT %CPU COMMAND\n" - monkeypatch.setattr( - subprocess, - "run", - lambda *a, **kw: self._ps_result(header + "5678 S 0.1 python"), - ) - result = _check_claude_process_state_breakdown() - assert result.severity == Severity.OK - assert result.message == "No claude processes running" - - def test_ok_when_ps_missing(self, monkeypatch: pytest.MonkeyPatch) -> None: - """FileNotFoundError from ps → Severity.OK explaining ps unavailability.""" - import subprocess - - from autoskillit.cli._doctor import Severity, _check_claude_process_state_breakdown - - def _raise(*a, **kw): - raise FileNotFoundError("ps") - - monkeypatch.setattr(subprocess, "run", _raise) - result = _check_claude_process_state_breakdown() - assert result.severity == Severity.OK - assert "ps unavailable" in result.message - assert "FileNotFoundError" in result.message - + ] + } + } + (version_dir / "hooks.json").write_text(_json.dumps(stale_hooks)) -class TestDoctorInstallClassification: - """Tests for _check_install_classification doctor check.""" + result = _check_plugin_cache_integrity(cache_dir=fake_cache) - @pytest.mark.parametrize( - "revision,expected_fragment", - [ - ("stable", "stable"), - ("integration", "integration"), - ], + assert result.severity == Severity.ERROR, ( + "_check_plugin_cache_integrity must return ERROR when cached hooks.json has broken paths" ) - def test_doctor_reports_install_classification_git_vcs( - self, monkeypatch: pytest.MonkeyPatch, revision: str, expected_fragment: str - ) -> None: - import json + assert "quota_guard.py" in result.message - from autoskillit.cli._doctor import Severity, _check_install_classification - - fake_direct_url = json.dumps( - { - "url": "https://github.com/TalonT-Org/AutoSkillit.git", - "vcs_info": { - "vcs": "git", - "requested_revision": revision, - "commit_id": "abc123", - }, - } - ) - from unittest.mock import MagicMock - - fake_dist = MagicMock() - fake_dist.read_text.return_value = fake_direct_url - monkeypatch.setattr( - "importlib.metadata.Distribution.from_name", - lambda _name: fake_dist, - ) - result = _check_install_classification() - assert result.severity == Severity.OK - assert expected_fragment in result.message - - def test_doctor_reports_install_classification_unknown( - self, monkeypatch: pytest.MonkeyPatch - ) -> None: - from unittest.mock import MagicMock - from autoskillit.cli._doctor import Severity, _check_install_classification - - fake_dist = MagicMock() - fake_dist.read_text.return_value = None - monkeypatch.setattr( - "importlib.metadata.Distribution.from_name", - lambda _name: fake_dist, - ) - result = _check_install_classification() - assert result.severity == Severity.WARNING - assert "could not be detected" in result.message - - -class TestDoctorUpdateDismissalState: - """Tests for _check_update_dismissal_state doctor check.""" - - def test_doctor_reports_dismissal_state_empty(self, tmp_path: Path) -> None: - from autoskillit.cli._doctor import Severity, _check_update_dismissal_state - - result = _check_update_dismissal_state(home=tmp_path) - assert result.severity == Severity.OK - assert "No active dismissal" in result.message - - def test_doctor_reports_dismissal_state_populated( - self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch - ) -> None: - import json - from datetime import UTC, datetime - from unittest.mock import MagicMock +# T-CACHE-INTEGRITY-2: doctor returns OK when cached hooks.json paths all exist +def test_doctor_plugin_cache_integrity_ok_when_valid(tmp_path: Path) -> None: + """_check_plugin_cache_integrity must return OK when all cached hook paths are valid.""" + import json as _json - from autoskillit.cli._doctor import Severity, _check_update_dismissal_state - from autoskillit.cli._update_checks import _write_dismiss_state + from autoskillit.cli.doctor._doctor_mcp import _check_plugin_cache_integrity + from autoskillit.core import Severity - # Seed state - dismissed_at = datetime.now(UTC).isoformat() - _write_dismiss_state( - tmp_path, - { - "update_prompt": { - "dismissed_at": dismissed_at, - "dismissed_version": "0.7.77", - "conditions": ["binary"], + fake_cache = tmp_path / "cache" + version_dir = fake_cache / "0.9.347" + version_dir.mkdir(parents=True) + valid_script = tmp_path / "hooks" / "guards" / "quota_guard.py" + valid_script.parent.mkdir(parents=True) + valid_script.write_text("# valid") + valid_hooks = { + "hooks": { + "PreToolUse": [ + { + "matcher": ".*", + "hooks": [ + { + "type": "command", + "command": f"python3 {valid_script}", + } + ], } - }, - ) - - # Patch detect_install to return stable GIT_VCS - fake_direct_url = json.dumps( - { - "url": "https://github.com/TalonT-Org/AutoSkillit.git", - "vcs_info": { - "vcs": "git", - "requested_revision": "stable", - "commit_id": "abc123", - }, - } - ) - fake_dist = MagicMock() - fake_dist.read_text.return_value = fake_direct_url - monkeypatch.setattr( - "importlib.metadata.Distribution.from_name", - lambda _name: fake_dist, - ) - - result = _check_update_dismissal_state(home=tmp_path) - assert result.severity == Severity.OK - assert "dismissed until" in result.message - assert "binary" in result.message - - -class TestDoctorSourceVersionDriftUsesNetwork: - """Test that source_version_drift now uses network=True.""" - - def test_doctor_source_version_drift_uses_network_true( - self, monkeypatch: pytest.MonkeyPatch, tmp_path: Path - ) -> None: - """_check_source_version_drift must call resolve_reference_sha with network=True.""" - import json - from unittest.mock import MagicMock - - from autoskillit.cli._doctor import _check_source_version_drift - - fake_direct_url = json.dumps( - { - "url": "https://github.com/TalonT-Org/AutoSkillit.git", - "vcs_info": { - "vcs": "git", - "requested_revision": "stable", - "commit_id": "abc123", - }, - } - ) - fake_dist = MagicMock() - fake_dist.read_text.return_value = fake_direct_url - monkeypatch.setattr( - "importlib.metadata.Distribution.from_name", - lambda _name: fake_dist, - ) + ] + } + } + (version_dir / "hooks.json").write_text(_json.dumps(valid_hooks)) - network_args: list[bool] = [] - monkeypatch.setattr( - "autoskillit.cli._update_checks.resolve_reference_sha", - lambda info, home, **kw: network_args.append(kw.get("network", True)) or None, - ) + result = _check_plugin_cache_integrity(cache_dir=fake_cache) - _check_source_version_drift(home=tmp_path) - assert any(n is True for n in network_args), ( - "_check_source_version_drift must call resolve_reference_sha with network=True" - ) + assert result.severity == Severity.OK - def test_check_source_version_drift_returns_ok_when_network_unavailable( - self, monkeypatch: pytest.MonkeyPatch, tmp_path: Path - ) -> None: - """Network error (resolve_reference_sha returns None) → OK, not hard failure.""" - import json - from unittest.mock import MagicMock - from autoskillit.cli._doctor import _check_source_version_drift +# T-CACHE-VERSION-1: _check_cache_version_mismatch returns ERROR when kitchen open + mismatch +def test_doctor_cache_version_mismatch_with_kitchen_open( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """_check_cache_version_mismatch must return ERROR when kitchen is open and versions differ.""" + import autoskillit.version as _ver + from autoskillit.cli.doctor._doctor_mcp import _check_cache_version_mismatch + from autoskillit.core import Severity - fake_direct_url = json.dumps( - { - "url": "https://github.com/TalonT-Org/AutoSkillit.git", - "vcs_info": { - "vcs": "git", - "requested_revision": "stable", - "commit_id": "abc123", - }, - } - ) - fake_dist = MagicMock() - fake_dist.read_text.return_value = fake_direct_url - monkeypatch.setattr( - "importlib.metadata.Distribution.from_name", - lambda _name: fake_dist, - ) - monkeypatch.setattr( - "autoskillit.cli._update_checks.resolve_reference_sha", - lambda info, home, **kw: None, - ) + monkeypatch.setattr("autoskillit.core.any_kitchen_open", lambda **kw: True) + monkeypatch.setattr( + _ver, + "version_info", + lambda **kw: { + "match": False, + "plugin_json_version": "0.9.347", + "package_version": "0.9.351", + }, + ) - from autoskillit.cli._doctor import Severity + result = _check_cache_version_mismatch() - result = _check_source_version_drift(home=tmp_path) - assert result.severity == Severity.OK, ( - f"Expected OK (fail-open) when network unavailable, " - f"got {result.severity}: {result.message}" - ) - assert "unavailable" in result.message.lower() or "network" in result.message.lower() + assert result.severity == Severity.ERROR, ( + "_check_cache_version_mismatch must return ERROR when kitchen open and version mismatch" + ) + assert "0.9.347" in result.message + assert "0.9.351" in result.message -def test_doctor_dual_mcp_registration_warns( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch +# T-CACHE-VERSION-2: _check_cache_version_mismatch returns WARNING when kitchen closed + mismatch +def test_doctor_cache_version_mismatch_without_kitchen( + monkeypatch: pytest.MonkeyPatch, ) -> None: - """_check_dual_mcp_registration() warns when both direct and marketplace entries exist.""" - from autoskillit.cli._doctor import _check_dual_mcp_registration + """_check_cache_version_mismatch must return WARNING (not ERROR) when kitchen is closed.""" + import autoskillit.version as _ver + from autoskillit.cli.doctor._doctor_mcp import _check_cache_version_mismatch from autoskillit.core import Severity - claude_json = tmp_path / ".claude.json" - claude_json.write_text(json.dumps({"mcpServers": {"autoskillit": {"type": "stdio"}}})) - plugins_dir = tmp_path / ".claude" / "plugins" - plugins_dir.mkdir(parents=True) - (plugins_dir / "installed_plugins.json").write_text( - json.dumps({"plugins": {"autoskillit@autoskillit-local": {"name": "autoskillit"}}}) + monkeypatch.setattr("autoskillit.core.any_kitchen_open", lambda **kw: False) + monkeypatch.setattr( + _ver, + "version_info", + lambda **kw: { + "match": False, + "plugin_json_version": "0.9.347", + "package_version": "0.9.351", + }, ) - monkeypatch.setattr(Path, "home", lambda: tmp_path) - result = _check_dual_mcp_registration() - assert result.severity == Severity.WARNING - assert "autoskillit install" in result.message + result = _check_cache_version_mismatch() -def test_doctor_no_dual_when_only_direct(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: - """_check_dual_mcp_registration() returns OK when only the direct entry exists.""" - from autoskillit.cli._doctor import _check_dual_mcp_registration - from autoskillit.core import Severity - - claude_json = tmp_path / ".claude.json" - claude_json.write_text(json.dumps({"mcpServers": {"autoskillit": {"type": "stdio"}}})) - monkeypatch.setattr(Path, "home", lambda: tmp_path) - result = _check_dual_mcp_registration() - assert result.severity == Severity.OK + assert result.severity == Severity.WARNING -def test_check_installed_plugins_entry_real_structure_is_ok(tmp_path: Path) -> None: - """With the real nested format, the check must report OK.""" - from autoskillit.cli._doctor import _check_installed_plugins_entry +# T-CACHE-VERSION-3: _check_cache_version_mismatch returns OK when versions match +def test_doctor_cache_version_mismatch_ok_when_matching( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """_check_cache_version_mismatch must return OK when versions match.""" + import autoskillit.version as _ver + from autoskillit.cli.doctor._doctor_mcp import _check_cache_version_mismatch from autoskillit.core import Severity - p = tmp_path / "installed_plugins.json" - p.write_text( - json.dumps( - { - "version": 2, - "plugins": {"autoskillit@autoskillit-local": {"name": "autoskillit"}}, - } - ) + monkeypatch.setattr("autoskillit.core.any_kitchen_open", lambda **kw: False) + monkeypatch.setattr( + _ver, + "version_info", + lambda **kw: { + "match": True, + "plugin_json_version": "0.9.351", + "package_version": "0.9.351", + }, ) - result = _check_installed_plugins_entry(plugins_json_path=p) - assert result.severity == Severity.OK - -def test_check_installed_plugins_entry_flat_structure_is_warning(tmp_path: Path) -> None: - """A flat structure (wrong format) must not be silently treated as OK.""" - from autoskillit.cli._doctor import _check_installed_plugins_entry - from autoskillit.core import Severity + result = _check_cache_version_mismatch() - p = tmp_path / "installed_plugins.json" - p.write_text(json.dumps({"autoskillit@autoskillit-local": {}})) - result = _check_installed_plugins_entry(plugins_json_path=p) - assert result.severity == Severity.WARNING + assert result.severity == Severity.OK diff --git a/tests/cli/test_doctor_migration.py b/tests/cli/test_doctor_migration.py new file mode 100644 index 0000000000..6992c00168 --- /dev/null +++ b/tests/cli/test_doctor_migration.py @@ -0,0 +1,1035 @@ +"""Tests for doctor quota cache schema, install classification, version consistency, and drift.""" + +from __future__ import annotations + +import json +import textwrap +from pathlib import Path + +import pytest + +from autoskillit import cli + +pytestmark = [pytest.mark.layer("cli"), pytest.mark.small] + +# --------------------------------------------------------------------------- +# Check 14: Quota cache schema version (#711 Part B, Phase 4) +# --------------------------------------------------------------------------- + + +class TestCheckQuotaCacheSchema: + """Tests for _check_quota_cache_schema doctor check.""" + + def test_check_quota_cache_schema_ok_when_current(self, tmp_path): + from autoskillit.cli.doctor import Severity, _check_quota_cache_schema + from autoskillit.execution import QUOTA_CACHE_SCHEMA_VERSION + + cache = tmp_path / "cache.json" + cache.write_text( + json.dumps( + {"schema_version": QUOTA_CACHE_SCHEMA_VERSION, "fetched_at": "2026-01-01T00:00:00"} + ) + ) + result = _check_quota_cache_schema(cache_path=cache) + assert result.severity == Severity.OK + assert f"v{QUOTA_CACHE_SCHEMA_VERSION}" in result.message + + def test_check_quota_cache_schema_ok_when_missing(self, tmp_path): + from autoskillit.cli.doctor import Severity, _check_quota_cache_schema + + cache = tmp_path / "nonexistent.json" + result = _check_quota_cache_schema(cache_path=cache) + assert result.severity == Severity.OK + assert "No quota cache" in result.message + + def test_check_quota_cache_schema_warning_when_no_schema_version_key(self, tmp_path): + from autoskillit.cli.doctor import Severity, _check_quota_cache_schema + + cache = tmp_path / "cache.json" + cache.write_text(json.dumps({"fetched_at": "2026-01-01T00:00:00"})) + result = _check_quota_cache_schema(cache_path=cache) + assert result.severity == Severity.WARNING + assert "schema drift" in result.message.lower() + + def test_check_quota_cache_schema_warning_includes_cache_path_and_observed_value( + self, tmp_path + ): + from autoskillit.cli.doctor import Severity, _check_quota_cache_schema + + cache = tmp_path / "cache.json" + cache.write_text(json.dumps({"schema_version": 1})) + result = _check_quota_cache_schema(cache_path=cache) + assert result.severity == Severity.WARNING + assert str(cache) in result.message + assert "observed=1" in result.message + + +def test_doctor_reports_drift_in_project_scope( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """_check_hook_registry_drift must report drift found in project scope.""" + from autoskillit.cli.doctor._doctor_hooks import _check_hook_registry_drift + from autoskillit.core import Severity + + # Seed a stale pretty_output.py in project scope + project_settings = tmp_path / ".claude" / "settings.json" + project_settings.parent.mkdir(parents=True) + project_settings.write_text( + json.dumps( + { + "hooks": { + "PostToolUse": [ + { + "matcher": "mcp__.*autoskillit.*", + "hooks": [ + {"type": "command", "command": "python3 /stale/pretty_output.py"} + ], + } + ] + } + } + ) + ) + + result = _check_hook_registry_drift(project_settings, scope_label="project") + assert result.severity == Severity.ERROR + assert "[project]" in result.message + assert "pretty_output.py" in result.message + + +# --------------------------------------------------------------------------- +# REQ-DOCTOR-001 — _check_claude_process_state_breakdown +# --------------------------------------------------------------------------- + + +class TestCheckClaudeProcessStateBreakdown: + """Tests for the claude_process_state doctor check (Check 15).""" + + def _ps_result(self, stdout: str, returncode: int = 0): + return type( + "CompletedProcess", + (), + {"returncode": returncode, "stdout": stdout}, + )() + + def test_ok_when_only_sleeping(self, monkeypatch: pytest.MonkeyPatch) -> None: + """Single sleeping claude process → Severity.OK with state breakdown.""" + import subprocess + + from autoskillit.cli.doctor import Severity, _check_claude_process_state_breakdown + + header = "PID STAT %CPU COMMAND\n" + monkeypatch.setattr( + subprocess, + "run", + lambda *a, **kw: self._ps_result(header + "1234 S 0.5 claude"), + ) + result = _check_claude_process_state_breakdown() + assert result.severity == Severity.OK + assert "S=1" in result.message + + def test_warns_on_d_state(self, monkeypatch: pytest.MonkeyPatch) -> None: + """claude process in D state → Severity.WARNING with pid and pcpu in message.""" + import subprocess + + from autoskillit.cli.doctor import Severity, _check_claude_process_state_breakdown + + header = "PID STAT %CPU COMMAND\n" + monkeypatch.setattr( + subprocess, + "run", + lambda *a, **kw: self._ps_result(header + "1234 D 99.0 claude"), + ) + result = _check_claude_process_state_breakdown() + assert result.severity == Severity.WARNING + assert "D=1" in result.message + assert "99.0" in result.message + + def test_ok_when_no_claude_processes(self, monkeypatch: pytest.MonkeyPatch) -> None: + """Empty ps output (no claude rows) → Severity.OK, 'No claude processes running'.""" + import subprocess + + from autoskillit.cli.doctor import Severity, _check_claude_process_state_breakdown + + header = "PID STAT %CPU COMMAND\n" + monkeypatch.setattr( + subprocess, + "run", + lambda *a, **kw: self._ps_result(header + "5678 S 0.1 python"), + ) + result = _check_claude_process_state_breakdown() + assert result.severity == Severity.OK + assert result.message == "No claude processes running" + + def test_ok_when_ps_missing(self, monkeypatch: pytest.MonkeyPatch) -> None: + """FileNotFoundError from ps → Severity.OK explaining ps unavailability.""" + import subprocess + + from autoskillit.cli.doctor import Severity, _check_claude_process_state_breakdown + + def _raise(*a, **kw): + raise FileNotFoundError("ps") + + monkeypatch.setattr(subprocess, "run", _raise) + result = _check_claude_process_state_breakdown() + assert result.severity == Severity.OK + assert "ps unavailable" in result.message + assert "FileNotFoundError" in result.message + + +class TestDoctorInstallClassification: + """Tests for _check_install_classification doctor check.""" + + @pytest.mark.parametrize( + "revision,expected_fragment", + [ + ("stable", "stable"), + ("develop", "develop"), + ], + ) + def test_doctor_reports_install_classification_git_vcs( + self, monkeypatch: pytest.MonkeyPatch, revision: str, expected_fragment: str + ) -> None: + from autoskillit.cli.doctor import Severity, _check_install_classification + + fake_direct_url = json.dumps( + { + "url": "https://github.com/TalonT-Org/AutoSkillit.git", + "vcs_info": { + "vcs": "git", + "requested_revision": revision, + "commit_id": "abc123", + }, + } + ) + from unittest.mock import MagicMock + + fake_dist = MagicMock() + fake_dist.read_text.return_value = fake_direct_url + monkeypatch.setattr( + "importlib.metadata.Distribution.from_name", + lambda _name: fake_dist, + ) + result = _check_install_classification() + assert result.severity == Severity.OK + assert expected_fragment in result.message + + def test_doctor_reports_install_classification_unknown( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + from unittest.mock import MagicMock + + from autoskillit.cli.doctor import Severity, _check_install_classification + + fake_dist = MagicMock() + fake_dist.read_text.return_value = None + monkeypatch.setattr( + "importlib.metadata.Distribution.from_name", + lambda _name: fake_dist, + ) + result = _check_install_classification() + assert result.severity == Severity.WARNING + assert "could not be detected" in result.message + + +class TestDoctorUpdateDismissalState: + """Tests for _check_update_dismissal_state doctor check.""" + + def test_doctor_reports_dismissal_state_empty(self, tmp_path: Path) -> None: + from autoskillit.cli.doctor import Severity, _check_update_dismissal_state + + result = _check_update_dismissal_state(home=tmp_path) + assert result.severity == Severity.OK + assert "No active dismissal" in result.message + + def test_doctor_reports_dismissal_state_populated( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + from datetime import UTC, datetime + from unittest.mock import MagicMock + + from autoskillit.cli.doctor import Severity, _check_update_dismissal_state + from autoskillit.cli.update._update_checks import _write_dismiss_state + + # Seed state + dismissed_at = datetime.now(UTC).isoformat() + _write_dismiss_state( + tmp_path, + { + "update_prompt": { + "dismissed_at": dismissed_at, + "dismissed_version": "0.7.77", + "conditions": ["binary"], + } + }, + ) + + # Patch detect_install to return stable GIT_VCS + fake_direct_url = json.dumps( + { + "url": "https://github.com/TalonT-Org/AutoSkillit.git", + "vcs_info": { + "vcs": "git", + "requested_revision": "stable", + "commit_id": "abc123", + }, + } + ) + fake_dist = MagicMock() + fake_dist.read_text.return_value = fake_direct_url + monkeypatch.setattr( + "importlib.metadata.Distribution.from_name", + lambda _name: fake_dist, + ) + + result = _check_update_dismissal_state(home=tmp_path) + assert result.severity == Severity.OK + assert "dismissed until" in result.message + assert "binary" in result.message + + +class TestDoctorSourceVersionDriftUsesNetwork: + """Test that source_version_drift now uses network=True.""" + + def test_doctor_source_version_drift_uses_network_true( + self, monkeypatch: pytest.MonkeyPatch, tmp_path: Path + ) -> None: + """_check_source_version_drift must call resolve_reference_sha with network=True.""" + from unittest.mock import MagicMock + + from autoskillit.cli.doctor import _check_source_version_drift + + fake_direct_url = json.dumps( + { + "url": "https://github.com/TalonT-Org/AutoSkillit.git", + "vcs_info": { + "vcs": "git", + "requested_revision": "stable", + "commit_id": "abc123", + }, + } + ) + fake_dist = MagicMock() + fake_dist.read_text.return_value = fake_direct_url + monkeypatch.setattr( + "importlib.metadata.Distribution.from_name", + lambda _name: fake_dist, + ) + + network_args: list[bool] = [] + monkeypatch.setattr( + "autoskillit.cli.update._update_checks.resolve_reference_sha", + lambda info, home, **kw: network_args.append(kw.get("network", True)) or None, + ) + + _check_source_version_drift(home=tmp_path) + assert any(n is True for n in network_args), ( + "_check_source_version_drift must call resolve_reference_sha with network=True" + ) + + def test_check_source_version_drift_returns_ok_when_network_unavailable( + self, monkeypatch: pytest.MonkeyPatch, tmp_path: Path + ) -> None: + """Network error (resolve_reference_sha returns None) → OK, not hard failure.""" + from unittest.mock import MagicMock + + from autoskillit.cli.doctor import _check_source_version_drift + + fake_direct_url = json.dumps( + { + "url": "https://github.com/TalonT-Org/AutoSkillit.git", + "vcs_info": { + "vcs": "git", + "requested_revision": "stable", + "commit_id": "abc123", + }, + } + ) + fake_dist = MagicMock() + fake_dist.read_text.return_value = fake_direct_url + monkeypatch.setattr( + "importlib.metadata.Distribution.from_name", + lambda _name: fake_dist, + ) + monkeypatch.setattr( + "autoskillit.cli.update._update_checks.resolve_reference_sha", + lambda info, home, **kw: None, + ) + + from autoskillit.cli.doctor import Severity + + result = _check_source_version_drift(home=tmp_path) + assert result.severity == Severity.OK, ( + f"Expected OK (fail-open) when network unavailable, " + f"got {result.severity}: {result.message}" + ) + assert "unavailable" in result.message.lower() or "network" in result.message.lower() + + +def test_doctor_dual_mcp_registration_warns( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """_check_dual_mcp_registration() warns when both direct and marketplace entries exist.""" + from autoskillit.cli.doctor import _check_dual_mcp_registration + from autoskillit.core import Severity + + claude_json = tmp_path / ".claude.json" + claude_json.write_text(json.dumps({"mcpServers": {"autoskillit": {"type": "stdio"}}})) + plugins_dir = tmp_path / ".claude" / "plugins" + plugins_dir.mkdir(parents=True) + (plugins_dir / "installed_plugins.json").write_text( + json.dumps({"plugins": {"autoskillit@autoskillit-local": {"name": "autoskillit"}}}) + ) + monkeypatch.setattr(Path, "home", lambda: tmp_path) + result = _check_dual_mcp_registration() + assert result.severity == Severity.WARNING + assert "autoskillit install" in result.message + + +def test_doctor_no_dual_when_only_direct(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + """_check_dual_mcp_registration() returns OK when only the direct entry exists.""" + from autoskillit.cli.doctor import _check_dual_mcp_registration + from autoskillit.core import Severity + + claude_json = tmp_path / ".claude.json" + claude_json.write_text(json.dumps({"mcpServers": {"autoskillit": {"type": "stdio"}}})) + monkeypatch.setattr(Path, "home", lambda: tmp_path) + result = _check_dual_mcp_registration() + assert result.severity == Severity.OK + + +def test_check_installed_plugins_entry_real_structure_is_ok(tmp_path: Path) -> None: + """With the real nested format, the check must report OK.""" + from autoskillit.cli.doctor import _check_installed_plugins_entry + from autoskillit.core import Severity + + p = tmp_path / "installed_plugins.json" + p.write_text( + json.dumps( + { + "version": 2, + "plugins": {"autoskillit@autoskillit-local": {"name": "autoskillit"}}, + } + ) + ) + result = _check_installed_plugins_entry(plugins_json_path=p) + assert result.severity == Severity.OK + + +def test_check_installed_plugins_entry_flat_structure_is_warning(tmp_path: Path) -> None: + """A flat structure (wrong format) must not be silently treated as OK.""" + from autoskillit.cli.doctor import _check_installed_plugins_entry + from autoskillit.core import Severity + + p = tmp_path / "installed_plugins.json" + p.write_text(json.dumps({"autoskillit@autoskillit-local": {}})) + result = _check_installed_plugins_entry(plugins_json_path=p) + assert result.severity == Severity.WARNING + + +@pytest.mark.feature("fleet") +class TestGroupMFranchiseDoctorChecks: + """Group M: Fleet doctor checks (ambient env detection + infra health + campaign ops).""" + + # M1: SESSION_TYPE unset → OK (unset is normal; check fires on 'skill' and deprecated 'leaf') + def test_check_ambient_session_type_skill_ok_when_unset( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + from autoskillit.cli.doctor import _check_ambient_session_type_skill + from autoskillit.core import Severity + + monkeypatch.delenv("AUTOSKILLIT_SESSION_TYPE", raising=False) + result = _check_ambient_session_type_skill() + assert result.severity == Severity.OK + assert result.check == "ambient_session_type_skill" + + # M2: SESSION_TYPE=skill → WARN + def test_check_ambient_session_type_skill_warns_when_skill( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + from autoskillit.cli.doctor import _check_ambient_session_type_skill + from autoskillit.core import Severity + + monkeypatch.setenv("AUTOSKILLIT_SESSION_TYPE", "skill") + result = _check_ambient_session_type_skill() + assert result.severity == Severity.WARNING + assert result.check == "ambient_session_type_skill" + + # M2b: SESSION_TYPE=leaf (deprecated) → WARN with DeprecationWarning + def test_check_ambient_session_type_skill_warns_when_leaf_deprecated( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + from autoskillit.cli.doctor import _check_ambient_session_type_skill + from autoskillit.core import Severity + + monkeypatch.setenv("AUTOSKILLIT_SESSION_TYPE", "leaf") + with pytest.warns(DeprecationWarning, match="deprecated"): + result = _check_ambient_session_type_skill() + assert result.severity == Severity.WARNING + assert result.check == "ambient_session_type_skill" + + # M3: SESSION_TYPE=orchestrator → OK (not this check's concern) + def test_check_ambient_session_type_skill_ok_when_orchestrator( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + from autoskillit.cli.doctor import _check_ambient_session_type_skill + from autoskillit.core import Severity + + monkeypatch.setenv("AUTOSKILLIT_SESSION_TYPE", "orchestrator") + result = _check_ambient_session_type_skill() + assert result.severity == Severity.OK + + # M4: SESSION_TYPE=orchestrator → WARN from orchestrator check + def test_check_ambient_session_type_orchestrator_warns( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + from autoskillit.cli.doctor import _check_ambient_session_type_orchestrator + from autoskillit.core import Severity + + monkeypatch.setenv("AUTOSKILLIT_SESSION_TYPE", "orchestrator") + result = _check_ambient_session_type_orchestrator() + assert result.severity == Severity.WARNING + assert "should only be set by autoskillit CLIs" in result.message + + # M5: SESSION_TYPE=fleet → WARN from fleet check + def test_check_ambient_session_type_fleet_warns(self, monkeypatch: pytest.MonkeyPatch) -> None: + from autoskillit.cli.doctor import _check_ambient_session_type_fleet + from autoskillit.core import Severity + + monkeypatch.setenv("AUTOSKILLIT_SESSION_TYPE", "fleet") + result = _check_ambient_session_type_fleet() + assert result.severity == Severity.WARNING + assert "highest-privilege" in result.message + + # M6: SESSION_TYPE unset → OK for orchestrator and fleet checks + def test_check_ambient_session_type_orchestrator_ok_when_unset( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + from autoskillit.cli.doctor import _check_ambient_session_type_orchestrator + from autoskillit.core import Severity + + monkeypatch.delenv("AUTOSKILLIT_SESSION_TYPE", raising=False) + result = _check_ambient_session_type_orchestrator() + assert result.severity == Severity.OK + + def test_check_ambient_session_type_fleet_ok_when_unset( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + from autoskillit.cli.doctor import _check_ambient_session_type_fleet + from autoskillit.core import Severity + + monkeypatch.delenv("AUTOSKILLIT_SESSION_TYPE", raising=False) + result = _check_ambient_session_type_fleet() + assert result.severity == Severity.OK + + # M7: CAMPAIGN_ID set → WARN + def test_check_ambient_campaign_id_warns_when_set( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + from autoskillit.cli.doctor import _check_ambient_campaign_id + from autoskillit.core import Severity + + monkeypatch.setenv("AUTOSKILLIT_CAMPAIGN_ID", "camp-123") + result = _check_ambient_campaign_id() + assert result.severity == Severity.WARNING + assert "camp-123" in result.message + assert "dispatch_food_truck" in result.message + + # M8: CAMPAIGN_ID unset → OK + def test_check_ambient_campaign_id_ok_when_unset( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + from autoskillit.cli.doctor import _check_ambient_campaign_id + from autoskillit.core import Severity + + monkeypatch.delenv("AUTOSKILLIT_CAMPAIGN_ID", raising=False) + result = _check_ambient_campaign_id() + assert result.severity == Severity.OK + + # M9: sous-chef skill dir exists → OK + def test_check_sous_chef_bundled_ok(self) -> None: + from autoskillit.cli.doctor import _check_sous_chef_bundled + from autoskillit.core import Severity + + result = _check_sous_chef_bundled() + assert result.severity == Severity.OK + + # M10: sous-chef skill dir missing → ERROR + def test_check_sous_chef_bundled_error_when_missing( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + from autoskillit.cli.doctor import _check_sous_chef_bundled + from autoskillit.core import Severity + + monkeypatch.setattr("autoskillit.cli.doctor._doctor_fleet.pkg_root", lambda: tmp_path) + result = _check_sous_chef_bundled() + assert result.severity == Severity.ERROR + assert "sous-chef" in result.message + + # M11: fleet_dispatch_guard registered and exists → OK + def test_check_fleet_dispatch_guard_registered_ok( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + from autoskillit.cli.doctor import _check_fleet_dispatch_guard_registered + from autoskillit.core import Severity + + hooks_dir = tmp_path / "hooks" + (hooks_dir / "guards").mkdir(parents=True) + (hooks_dir / "guards" / "fleet_dispatch_guard.py").write_text("") + monkeypatch.setattr( + "autoskillit.cli.doctor._doctor_fleet.canonical_script_basenames", + lambda: frozenset({"guards/fleet_dispatch_guard.py"}), + ) + monkeypatch.setattr("autoskillit.hook_registry.HOOKS_DIR", hooks_dir) + result = _check_fleet_dispatch_guard_registered() + assert result.severity == Severity.OK + + # M12: fleet_dispatch_guard not registered → ERROR + def test_check_fleet_dispatch_guard_registered_error( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + from autoskillit.cli.doctor import _check_fleet_dispatch_guard_registered + from autoskillit.core import Severity + + monkeypatch.setattr( + "autoskillit.cli.doctor._doctor_fleet.canonical_script_basenames", + lambda: frozenset(), + ) + result = _check_fleet_dispatch_guard_registered() + assert result.severity == Severity.ERROR + assert "sync-hooks" in result.message + + # M13: No state files → OK + def test_check_stale_fleet_state_ok_when_no_state(self, tmp_path: Path) -> None: + from autoskillit.cli.doctor import _check_stale_fleet_state + from autoskillit.core import Severity + + result = _check_stale_fleet_state(project_dir=tmp_path) + assert result.severity == Severity.OK + + # M14: State file with running dispatch and mtime > 7d → WARN + def test_check_stale_fleet_state_warns_on_stale(self, tmp_path: Path) -> None: + import os + import time + + from autoskillit.cli.doctor import _check_stale_fleet_state + from autoskillit.core import Severity + + state_dir = tmp_path / ".autoskillit" / "temp" / "fleet" / "camp-1" + state_dir.mkdir(parents=True) + state_file = state_dir / "state.json" + state_file.write_text( + json.dumps( + { + "schema_version": 1, + "campaign_id": "camp-1", + "campaign_name": "test", + "manifest_path": "", + "started_at": 0, + "dispatches": [{"name": "d1", "status": "running"}], + } + ) + ) + old_time = time.time() - (8 * 86400) + os.utime(state_file, (old_time, old_time)) + result = _check_stale_fleet_state(project_dir=tmp_path) + assert result.severity == Severity.WARNING + assert "camp-1" in result.message or "state.json" in result.message + + # M15: State file with running dispatch and mtime < 7d → OK + def test_check_stale_fleet_state_ok_when_fresh(self, tmp_path: Path) -> None: + from autoskillit.cli.doctor import _check_stale_fleet_state + from autoskillit.core import Severity + + state_dir = tmp_path / ".autoskillit" / "temp" / "fleet" / "camp-1" + state_dir.mkdir(parents=True) + state_file = state_dir / "state.json" + state_file.write_text( + json.dumps( + { + "schema_version": 1, + "campaign_id": "camp-1", + "campaign_name": "test", + "manifest_path": "", + "started_at": 0, + "dispatches": [{"name": "d1", "status": "running"}], + } + ) + ) + result = _check_stale_fleet_state(project_dir=tmp_path) + assert result.severity == Severity.OK + + # M16: No campaigns/ dir → INFO onboarding hint + def test_check_campaign_onboarding_hint_info_when_empty(self, tmp_path: Path) -> None: + from autoskillit.cli.doctor import _check_campaign_onboarding_hint + from autoskillit.core import Severity + + result = _check_campaign_onboarding_hint(project_dir=tmp_path) + assert result.severity == Severity.INFO + assert "make-campaign" in result.message + + # M17: campaigns/ has YAML files → OK + def test_check_campaign_onboarding_hint_ok_when_populated(self, tmp_path: Path) -> None: + from autoskillit.cli.doctor import _check_campaign_onboarding_hint + from autoskillit.core import Severity + + campaigns_dir = tmp_path / ".autoskillit" / "recipes" / "campaigns" + campaigns_dir.mkdir(parents=True) + (campaigns_dir / "my-campaign.yaml").write_text("name: my-campaign\nkind: campaign\n") + result = _check_campaign_onboarding_hint(project_dir=tmp_path) + assert result.severity == Severity.OK + + # M18: Duplicate clone destinations across dispatches → WARN + def test_check_campaign_manifest_clone_dests_warns_on_duplicates(self, tmp_path: Path) -> None: + from autoskillit.cli.doctor import _check_campaign_manifest_clone_dests + from autoskillit.core import Severity + + campaigns_dir = tmp_path / ".autoskillit" / "recipes" / "campaigns" + campaigns_dir.mkdir(parents=True) + recipe_yaml = textwrap.dedent("""\ + name: my-campaign + kind: campaign + dispatches: + - name: task-1 + ingredients: + clone_path: /tmp/shared-clone + - name: task-2 + ingredients: + clone_path: /tmp/shared-clone + """) + (campaigns_dir / "dup-campaign.yaml").write_text(recipe_yaml) + result = _check_campaign_manifest_clone_dests(project_dir=tmp_path) + assert result.severity == Severity.WARNING + assert "/tmp/shared-clone" in result.message + + # M19: Unique clone destinations → OK + def test_check_campaign_manifest_clone_dests_ok_unique(self, tmp_path: Path) -> None: + from autoskillit.cli.doctor import _check_campaign_manifest_clone_dests + from autoskillit.core import Severity + + campaigns_dir = tmp_path / ".autoskillit" / "recipes" / "campaigns" + campaigns_dir.mkdir(parents=True) + recipe_yaml = textwrap.dedent("""\ + name: my-campaign + kind: campaign + dispatches: + - name: task-1 + ingredients: + clone_path: /tmp/clone-1 + - name: task-2 + ingredients: + clone_path: /tmp/clone-2 + """) + (campaigns_dir / "ok-campaign.yaml").write_text(recipe_yaml) + result = _check_campaign_manifest_clone_dests(project_dir=tmp_path) + assert result.severity == Severity.OK + + # M20: All 9 new checks appear in doctor JSON output + def test_doctor_json_output_includes_fleet_checks( + self, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture, + ) -> None: + monkeypatch.setattr(Path, "home", lambda: tmp_path) + monkeypatch.chdir(tmp_path) + monkeypatch.delenv("AUTOSKILLIT_SESSION_TYPE", raising=False) + monkeypatch.delenv("AUTOSKILLIT_CAMPAIGN_ID", raising=False) + cfg_dir = tmp_path / ".autoskillit" + cfg_dir.mkdir(parents=True, exist_ok=True) + (cfg_dir / "config.yaml").write_text("features:\n fleet: true\n") + cli.doctor_cmd(output_json=True) + data = json.loads(capsys.readouterr().out) + check_names = {r["check"] for r in data["results"]} + fleet_checks = { + "ambient_session_type_skill", + "ambient_session_type_orchestrator", + "ambient_session_type_fleet", + "ambient_campaign_id", + "sous_chef_bundled", + "fleet_dispatch_guard_registered", + "stale_fleet_state", + "campaign_onboarding_hint", + "campaign_manifest_clone_dests", + } + assert fleet_checks <= check_names + + +@pytest.mark.feature("fleet") +class TestGroupNFeatureGateDoctorChecks: + """N1–N8: Feature-gate checks and FleetConfig conditional validation.""" + + # N1: Fleet checks skipped when feature disabled + def test_fleet_doctor_checks_skipped_when_disabled( + self, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture, + ) -> None: + from autoskillit.config import AutomationConfig + + mock_cfg = AutomationConfig(features={"fleet": False}) + monkeypatch.setattr("autoskillit.cli.doctor.load_config", lambda _: mock_cfg) + monkeypatch.setattr(Path, "home", lambda: tmp_path) + monkeypatch.chdir(tmp_path) + monkeypatch.delenv("AUTOSKILLIT_SESSION_TYPE", raising=False) + monkeypatch.delenv("AUTOSKILLIT_CAMPAIGN_ID", raising=False) + cli.doctor_cmd(output_json=True) + data = json.loads(capsys.readouterr().out) + check_names = {r["check"] for r in data["results"]} + fleet_infra = { + "sous_chef_bundled", + "fleet_dispatch_guard_registered", + "stale_fleet_state", + "campaign_onboarding_hint", + "campaign_manifest_clone_dests", + } + assert fleet_infra.isdisjoint(check_names), ( + f"Fleet checks must be absent when feature is disabled, " + f"but found: {fleet_infra & check_names}" + ) + + # N2: Fleet checks run when feature enabled + def test_fleet_doctor_checks_run_when_enabled( + self, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture, + ) -> None: + from autoskillit.config import AutomationConfig + + mock_cfg = AutomationConfig(features={"fleet": True}) + monkeypatch.setattr("autoskillit.cli.doctor.load_config", lambda _: mock_cfg) + monkeypatch.setattr(Path, "home", lambda: tmp_path) + monkeypatch.chdir(tmp_path) + monkeypatch.delenv("AUTOSKILLIT_SESSION_TYPE", raising=False) + monkeypatch.delenv("AUTOSKILLIT_CAMPAIGN_ID", raising=False) + cli.doctor_cmd(output_json=True) + data = json.loads(capsys.readouterr().out) + check_names = {r["check"] for r in data["results"]} + fleet_infra = { + "sous_chef_bundled", + "fleet_dispatch_guard_registered", + "stale_fleet_state", + "campaign_onboarding_hint", + "campaign_manifest_clone_dests", + } + assert fleet_infra <= check_names + fleet_results = [r for r in data["results"] if r["check"] in fleet_infra] + assert all(r["severity"] in {"ok", "info"} for r in fleet_results), ( + f"Expected all fleet checks to have non-error severity (ok/info), " + f"got: {[(r['check'], r['severity']) for r in fleet_results]}" + ) + + # N3: Ambient env checks always run even when fleet disabled + def test_ambient_env_checks_always_run_when_fleet_disabled( + self, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture, + ) -> None: + from autoskillit.config import AutomationConfig + + mock_cfg = AutomationConfig(features={"fleet": False}) + monkeypatch.setattr("autoskillit.cli.doctor.load_config", lambda _: mock_cfg) + monkeypatch.setattr(Path, "home", lambda: tmp_path) + monkeypatch.chdir(tmp_path) + monkeypatch.delenv("AUTOSKILLIT_SESSION_TYPE", raising=False) + monkeypatch.delenv("AUTOSKILLIT_CAMPAIGN_ID", raising=False) + cli.doctor_cmd(output_json=True) + data = json.loads(capsys.readouterr().out) + check_names = {r["check"] for r in data["results"]} + ambient_checks = { + "ambient_session_type_skill", + "ambient_session_type_orchestrator", + "ambient_session_type_fleet", + "ambient_campaign_id", + } + assert ambient_checks <= check_names + + # N4: Feature dependency check fires ERROR for unsatisfied dep + def test_feature_dependency_check_fires_on_unsatisfied_dep( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + from autoskillit.cli.doctor import _check_feature_dependencies + from autoskillit.core import Severity + from autoskillit.core.types._type_constants import FeatureDef, FeatureLifecycle + + fake_feature = FeatureDef( + lifecycle=FeatureLifecycle.EXPERIMENTAL, + description="test feature with dep", + tool_tags=frozenset(), + skill_categories=frozenset(), + import_package=None, + default_enabled=False, + depends_on=frozenset({"franchise"}), + ) + monkeypatch.setattr( + "autoskillit.core.FEATURE_REGISTRY", + {"test_feature": fake_feature}, + ) + result = _check_feature_dependencies({"test_feature": True, "franchise": False}) + assert result.severity == Severity.ERROR + assert "test_feature" in result.message + assert "franchise" in result.message + + # N5: Feature dependency check passes when deps satisfied + def test_feature_dependency_check_passes_when_deps_satisfied( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + from autoskillit.cli.doctor import _check_feature_dependencies + from autoskillit.core import Severity + from autoskillit.core.types._type_constants import FeatureDef, FeatureLifecycle + + fake_feature = FeatureDef( + lifecycle=FeatureLifecycle.EXPERIMENTAL, + description="test feature with dep", + tool_tags=frozenset(), + skill_categories=frozenset(), + import_package=None, + default_enabled=False, + depends_on=frozenset({"franchise"}), + ) + monkeypatch.setattr( + "autoskillit.core.FEATURE_REGISTRY", + {"test_feature": fake_feature}, + ) + result = _check_feature_dependencies({"test_feature": True, "franchise": True}) + assert result.severity == Severity.OK + assert result.message == "All feature dependencies satisfied" + + # N6: Feature dependency check passes with empty features + def test_feature_dependency_check_passes_with_empty_features( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + from autoskillit.cli.doctor import _check_feature_dependencies + from autoskillit.core import Severity + + monkeypatch.setattr("autoskillit.core.FEATURE_REGISTRY", {}) + result = _check_feature_dependencies({}) + assert result.severity == Severity.OK + + # N7: Feature registry consistency passes for real registry + def test_feature_registry_consistency_passes(self) -> None: + from autoskillit.cli.doctor import _check_feature_registry_consistency + from autoskillit.core import Severity + + result = _check_feature_registry_consistency() + assert result.severity == Severity.OK + + # N8: Feature registry consistency errors on bad import + def test_feature_registry_consistency_errors_on_bad_import( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + from autoskillit.cli.doctor import _check_feature_registry_consistency + from autoskillit.core import Severity + from autoskillit.core.types._type_constants import FeatureDef, FeatureLifecycle + + bad_feature = FeatureDef( + lifecycle=FeatureLifecycle.EXPERIMENTAL, + description="feature with bad import", + tool_tags=frozenset(), + skill_categories=frozenset(), + import_package="nonexistent.pkg", + ) + monkeypatch.setattr( + "autoskillit.core.FEATURE_REGISTRY", + {"bad_feature": bad_feature}, + ) + result = _check_feature_registry_consistency() + assert result.severity == Severity.ERROR + assert "bad_feature" in result.message + assert "nonexistent.pkg" in result.message + + +# --------------------------------------------------------------------------- +# T3 — version_consistency reads cache dir, not source tree +# --------------------------------------------------------------------------- + + +def test_doctor_version_consistency_detects_stale_cache( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture, + request: pytest.FixtureRequest, +) -> None: + """Check 5 warns when the CACHED plugin.json version is behind the package.""" + import importlib.metadata + + from autoskillit.version import version_info as _vi + + cache_dir = tmp_path / ".claude" / "plugins" / "cache" / "autoskillit-local" / "autoskillit" + plugin_json = cache_dir / ".claude-plugin" / "plugin.json" + plugin_json.parent.mkdir(parents=True) + plugin_json.write_text('{"version": "0.8.0"}') + + monkeypatch.setattr(Path, "home", lambda: tmp_path) + monkeypatch.chdir(tmp_path) + monkeypatch.setattr(importlib.metadata, "version", lambda _: "0.9.0") + _vi.cache_clear() + request.addfinalizer(_vi.cache_clear) + cli.doctor_cmd(output_json=True) + data = json.loads(capsys.readouterr().out) + vc = next((r for r in data["results"] if r["check"] == "version_consistency"), None) + assert vc is not None, "version_consistency check not found in doctor results" + assert vc["severity"] == "warning" + assert "autoskillit install" in vc["message"] + + +def test_doctor_version_consistency_ok_when_cache_matches( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture, + request: pytest.FixtureRequest, +) -> None: + """Check 5 reports OK when cached plugin.json version matches the package.""" + import importlib.metadata + + from autoskillit.version import version_info as _vi + + cache_dir = tmp_path / ".claude" / "plugins" / "cache" / "autoskillit-local" / "autoskillit" + plugin_json = cache_dir / ".claude-plugin" / "plugin.json" + plugin_json.parent.mkdir(parents=True) + plugin_json.write_text('{"version": "0.9.0"}') + + monkeypatch.setattr(Path, "home", lambda: tmp_path) + monkeypatch.chdir(tmp_path) + monkeypatch.setattr(importlib.metadata, "version", lambda _: "0.9.0") + _vi.cache_clear() + request.addfinalizer(_vi.cache_clear) + cli.doctor_cmd(output_json=True) + data = json.loads(capsys.readouterr().out) + vc = next((r for r in data["results"] if r["check"] == "version_consistency"), None) + assert vc is not None, "version_consistency check not found in doctor results" + assert vc["severity"] == "ok" + + +# --------------------------------------------------------------------------- +# T4 — _check_source_version_drift remediation uses specific upgrade command +# --------------------------------------------------------------------------- + + +def test_source_version_drift_remediation_contains_upgrade_command( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """_check_source_version_drift WARNING message contains the install-type-specific command.""" + from autoskillit.cli._install_info import InstallInfo, InstallType + from autoskillit.cli.doctor import _check_source_version_drift + from autoskillit.core import Severity + + info = InstallInfo( + install_type=InstallType.GIT_VCS, + commit_id="aaaa1111bbbb", + requested_revision="stable", + url="https://github.com/TalonT-Org/AutoSkillit.git", + editable_source=None, + ) + monkeypatch.setattr("autoskillit.cli._install_info.detect_install", lambda: info) + monkeypatch.setattr( + "autoskillit.cli.update._update_checks.resolve_reference_sha", + lambda *a, **kw: "bbbb2222cccc", + ) + result = _check_source_version_drift(home=tmp_path) + assert result.severity == Severity.WARNING + assert "uv tool upgrade autoskillit" in result.message + assert "appropriate" not in result.message diff --git a/tests/cli/test_doctor_scripts.py b/tests/cli/test_doctor_scripts.py new file mode 100644 index 0000000000..5955d5032f --- /dev/null +++ b/tests/cli/test_doctor_scripts.py @@ -0,0 +1,255 @@ +"""Tests for doctor script/recipe version health checks.""" + +from __future__ import annotations + +import json +from pathlib import Path + +import pytest + +from autoskillit import cli + +pytestmark = [pytest.mark.layer("cli"), pytest.mark.small] + +_MINIMAL_SCRIPT_YAML = """\ +name: my-script +description: A test script +steps: + do_it: + tool: run_cmd + with: + cmd: echo hello + on_success: done + done: + action: stop + message: Done +kitchen_rules: + - Only use AutoSkillit MCP tools during pipeline execution +""" + + +class TestDoctorScriptHealth: + """Doctor check for script version staleness.""" + + # DOC1: No .autoskillit/recipes/ -> OK result + def test_no_scripts_dir_reports_ok( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture + ) -> None: + """doctor reports OK for script_version_health when no scripts directory exists.""" + monkeypatch.setattr(Path, "home", lambda: tmp_path) + monkeypatch.chdir(tmp_path) + # No .autoskillit/recipes/ directory created + cli.doctor_cmd(output_json=True) + captured = capsys.readouterr() + data = json.loads(captured.out) + script_checks = [r for r in data["results"] if r["check"] == "script_version_health"] + assert len(script_checks) == 1 + assert script_checks[0]["severity"] == "ok" + + # DOC2: All scripts at current version -> OK result + def test_all_scripts_at_current_version_reports_ok( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture + ) -> None: + """doctor reports OK when all scripts carry the current installed version.""" + import autoskillit + + current_version = autoskillit.__version__ + monkeypatch.setattr(Path, "home", lambda: tmp_path) + monkeypatch.chdir(tmp_path) + scripts_dir = tmp_path / ".autoskillit" / "recipes" + scripts_dir.mkdir(parents=True) + (scripts_dir / "up-to-date.yaml").write_text( + f"name: up-to-date\ndescription: Current version\n" + f'autoskillit_version: "{current_version}"\n' + + _MINIMAL_SCRIPT_YAML.split("\n", 2)[2] # reuse steps/constraints block + ) + cli.doctor_cmd(output_json=True) + captured = capsys.readouterr() + data = json.loads(captured.out) + script_checks = [r for r in data["results"] if r["check"] == "script_version_health"] + assert len(script_checks) == 1 + assert script_checks[0]["severity"] == "ok" + + # DOC3: Scripts below current version -> WARNING result with recipe names + def test_outdated_scripts_reports_warning_with_count( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture + ) -> None: + """doctor reports WARNING with recipe names when scripts have an older version.""" + import autoskillit + + monkeypatch.setattr(autoskillit, "__version__", "99.0.0") + monkeypatch.setattr(Path, "home", lambda: tmp_path) + monkeypatch.chdir(tmp_path) + scripts_dir = tmp_path / ".autoskillit" / "recipes" + scripts_dir.mkdir(parents=True) + (scripts_dir / "old-script.yaml").write_text( + 'name: old-script\ndescription: Old\nautoskillit_version: "0.1.0"\n' + ) + (scripts_dir / "also-old.yaml").write_text( + 'name: also-old\ndescription: Also old\nautoskillit_version: "0.1.0"\n' + ) + cli.doctor_cmd(output_json=True) + captured = capsys.readouterr() + data = json.loads(captured.out) + script_checks = [r for r in data["results"] if r["check"] == "script_version_health"] + assert len(script_checks) == 1 + assert script_checks[0]["severity"] == "warning" + assert "old-script" in script_checks[0]["message"] + + # DOC4: Scripts with no version field -> WARNING result + def test_scripts_without_version_reports_warning( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture + ) -> None: + """doctor reports WARNING when script YAML has no autoskillit_version field.""" + monkeypatch.setattr(Path, "home", lambda: tmp_path) + monkeypatch.chdir(tmp_path) + scripts_dir = tmp_path / ".autoskillit" / "recipes" + scripts_dir.mkdir(parents=True) + (scripts_dir / "no-version.yaml").write_text( + "name: no-version\ndescription: No version field\n" + ) + cli.doctor_cmd(output_json=True) + captured = capsys.readouterr() + data = json.loads(captured.out) + script_checks = [r for r in data["results"] if r["check"] == "script_version_health"] + assert len(script_checks) == 1 + assert script_checks[0]["severity"] == "warning" + + def _setup_recipe(self, scripts_dir: Path, name: str, version: str = "0.1.0") -> None: + (scripts_dir / f"{name}.yaml").write_text( + f'name: {name}\ndescription: Test\nautoskillit_version: "{version}"\n' + ) + + def _write_failures_json(self, tmp_path: Path, name: str, retries: int = 3) -> None: + failures_path = tmp_path / ".autoskillit" / "temp" / "migrations" / "failures.json" + failures_path.parent.mkdir(parents=True, exist_ok=True) + failures_path.write_text( + json.dumps( + { + name: { + "name": name, + "file_path": f"/fake/{name}.yaml", + "file_type": "recipe", + "timestamp": "2026-01-01T00:00:00+00:00", + "error": "validation failed after retries", + "retries_attempted": retries, + } + } + ) + ) + + # DR1: failures.json has an entry for a recipe -> error severity + def test_doctor_error_on_failed_migration( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture + ) -> None: + """doctor reports error severity when failures.json has an entry for a recipe.""" + import autoskillit + + monkeypatch.setattr(autoskillit, "__version__", "99.0.0") + monkeypatch.setattr(Path, "home", lambda: tmp_path) + monkeypatch.chdir(tmp_path) + scripts_dir = tmp_path / ".autoskillit" / "recipes" + scripts_dir.mkdir(parents=True) + self._setup_recipe(scripts_dir, "broken-pipeline") + self._write_failures_json(tmp_path, "broken-pipeline", retries=3) + cli.doctor_cmd(output_json=True) + captured = capsys.readouterr() + data = json.loads(captured.out) + script_checks = [r for r in data["results"] if r["check"] == "script_version_health"] + assert len(script_checks) == 1 + assert script_checks[0]["severity"] == "error" + + # DR2: Error message includes retries_attempted value from failure record + def test_doctor_error_message_includes_retry_count( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture + ) -> None: + """doctor error message includes retries_attempted value from the failure record.""" + import autoskillit + + monkeypatch.setattr(autoskillit, "__version__", "99.0.0") + monkeypatch.setattr(Path, "home", lambda: tmp_path) + monkeypatch.chdir(tmp_path) + scripts_dir = tmp_path / ".autoskillit" / "recipes" + scripts_dir.mkdir(parents=True) + self._setup_recipe(scripts_dir, "my-pipeline") + self._write_failures_json(tmp_path, "my-pipeline", retries=3) + cli.doctor_cmd(output_json=True) + captured = capsys.readouterr() + data = json.loads(captured.out) + script_checks = [r for r in data["results"] if r["check"] == "script_version_health"] + assert len(script_checks) == 1 + assert "3" in script_checks[0]["message"] + + # DR3: Outdated recipe with no failure record -> warning severity + def test_doctor_warning_on_simply_outdated( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture + ) -> None: + """doctor reports warning when recipe is outdated but has no failure record.""" + import autoskillit + + monkeypatch.setattr(autoskillit, "__version__", "99.0.0") + monkeypatch.setattr(Path, "home", lambda: tmp_path) + monkeypatch.chdir(tmp_path) + scripts_dir = tmp_path / ".autoskillit" / "recipes" + scripts_dir.mkdir(parents=True) + self._setup_recipe(scripts_dir, "outdated-pipeline") + # No failures.json written + cli.doctor_cmd(output_json=True) + captured = capsys.readouterr() + data = json.loads(captured.out) + script_checks = [r for r in data["results"] if r["check"] == "script_version_health"] + assert len(script_checks) == 1 + assert script_checks[0]["severity"] == "warning" + + # DR4: All recipes current, no failures.json -> ok + def test_doctor_ok_when_all_current( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture + ) -> None: + """doctor reports ok when all recipes are at current version and no failures.json.""" + import autoskillit + + current_version = autoskillit.__version__ + monkeypatch.setattr(Path, "home", lambda: tmp_path) + monkeypatch.chdir(tmp_path) + scripts_dir = tmp_path / ".autoskillit" / "recipes" + scripts_dir.mkdir(parents=True) + self._setup_recipe(scripts_dir, "current-pipeline", version=current_version) + # No failures.json written + cli.doctor_cmd(output_json=True) + captured = capsys.readouterr() + data = json.loads(captured.out) + script_checks = [r for r in data["results"] if r["check"] == "script_version_health"] + assert len(script_checks) == 1 + assert script_checks[0]["severity"] == "ok" + + # DR5: Warning message says "Will be auto-migrated on next load" + def test_doctor_outdated_message_updated( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture + ) -> None: + """doctor warning message says 'Will be auto-migrated on next load'.""" + import autoskillit + + monkeypatch.setattr(autoskillit, "__version__", "99.0.0") + monkeypatch.setattr(Path, "home", lambda: tmp_path) + monkeypatch.chdir(tmp_path) + scripts_dir = tmp_path / ".autoskillit" / "recipes" + scripts_dir.mkdir(parents=True) + self._setup_recipe(scripts_dir, "stale-pipeline") + cli.doctor_cmd(output_json=True) + captured = capsys.readouterr() + data = json.loads(captured.out) + script_checks = [r for r in data["results"] if r["check"] == "script_version_health"] + assert len(script_checks) == 1 + assert script_checks[0]["severity"] == "warning" + assert "Will be auto-migrated on next load" in script_checks[0]["message"] + + +class TestSyncRemovalCLI: + def test_doctor_has_no_recipe_sync_check( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture + ): + """REQ-APP-006: doctor output does not include recipe_sync_status.""" + monkeypatch.chdir(tmp_path) + cli.doctor_cmd() + captured = capsys.readouterr() + assert "recipe_sync_status" not in captured.out diff --git a/tests/cli/test_doctor_split.py b/tests/cli/test_doctor_split.py new file mode 100644 index 0000000000..66eecea881 --- /dev/null +++ b/tests/cli/test_doctor_split.py @@ -0,0 +1,118 @@ +"""Structural guards: test_doctor.py split into three files (P1-F02).""" + +from __future__ import annotations + +import ast +from pathlib import Path + +import pytest + +pytestmark = [pytest.mark.layer("cli"), pytest.mark.small] + +_CLI_TESTS = Path(__file__).parent +_CLI_SRC = Path(__file__).parent.parent.parent / "src" / "autoskillit" / "cli" + + +def _has_pytestmark_cli(path: Path) -> bool: + tree = ast.parse(path.read_text()) + for node in ast.walk(tree): + if isinstance(node, ast.Assign): + for target in node.targets: + if isinstance(target, ast.Name) and target.id == "pytestmark": + src = ast.unparse(node.value) + return 'layer("cli")' in src or "layer('cli')" in src + return False + + +def test_doctor_scripts_file_exists(): + """test_doctor_scripts.py must exist after the split.""" + assert (_CLI_TESTS / "test_doctor_scripts.py").exists() + + +def test_doctor_migration_file_exists(): + """test_doctor_migration.py must exist after the split.""" + assert (_CLI_TESTS / "test_doctor_migration.py").exists() + + +def test_doctor_scripts_has_correct_pytestmark(): + p = _CLI_TESTS / "test_doctor_scripts.py" + assert _has_pytestmark_cli(p), "test_doctor_scripts.py missing layer('cli') pytestmark" + + +def test_doctor_migration_has_correct_pytestmark(): + p = _CLI_TESTS / "test_doctor_migration.py" + assert _has_pytestmark_cli(p), "test_doctor_migration.py missing layer('cli') pytestmark" + + +def test_doctor_scripts_contains_script_health_class(): + p = _CLI_TESTS / "test_doctor_scripts.py" + tree = ast.parse(p.read_text()) + class_names = {n.name for n in ast.walk(tree) if isinstance(n, ast.ClassDef)} + assert "TestDoctorScriptHealth" in class_names + + +def test_doctor_migration_contains_quota_cache_class(): + p = _CLI_TESTS / "test_doctor_migration.py" + tree = ast.parse(p.read_text()) + class_names = {n.name for n in ast.walk(tree) if isinstance(n, ast.ClassDef)} + assert "TestCheckQuotaCacheSchema" in class_names + + +def test_doctor_core_does_not_contain_script_health_class(): + p = _CLI_TESTS / "test_doctor.py" + tree = ast.parse(p.read_text()) + class_names = {n.name for n in ast.walk(tree) if isinstance(n, ast.ClassDef)} + assert "TestDoctorScriptHealth" not in class_names, ( + "TestDoctorScriptHealth must be moved to test_doctor_scripts.py" + ) + + +def test_doctor_core_does_not_contain_quota_cache_class(): + p = _CLI_TESTS / "test_doctor.py" + tree = ast.parse(p.read_text()) + class_names = {n.name for n in ast.walk(tree) if isinstance(n, ast.ClassDef)} + assert "TestCheckQuotaCacheSchema" not in class_names, ( + "TestCheckQuotaCacheSchema must be moved to test_doctor_migration.py" + ) + + +def test_doctor_facade_exports_run_doctor(): + from autoskillit.cli.doctor import run_doctor # noqa: F401 + + +def test_doctor_submodule_types_importable(): + from autoskillit.cli.doctor._doctor_types import _NON_PROBLEM, DoctorResult # noqa: F401 + + +def test_doctor_submodule_mcp_importable(): + from autoskillit.cli.doctor._doctor_mcp import _check_mcp_server_registered # noqa: F401 + + +def test_doctor_submodule_hooks_importable(): + from autoskillit.cli.doctor._doctor_hooks import _check_hook_registry_drift # noqa: F401 + + +def test_doctor_submodule_install_importable(): + from autoskillit.cli.doctor._doctor_install import _check_stale_entry_points # noqa: F401 + + +def test_doctor_submodule_config_importable(): + from autoskillit.cli.doctor._doctor_config import ( + _check_config_layers_for_secrets, # noqa: F401 + ) + + +def test_doctor_submodule_runtime_importable(): + from autoskillit.cli.doctor._doctor_runtime import _check_quota_cache_schema # noqa: F401 + + +def test_doctor_submodule_env_importable(): + from autoskillit.cli.doctor._doctor_env import _check_ambient_session_type_skill # noqa: F401 + + +def test_doctor_submodule_features_importable(): + from autoskillit.cli.doctor._doctor_features import _check_feature_dependencies # noqa: F401 + + +def test_doctor_submodule_fleet_importable(): + from autoskillit.cli.doctor._doctor_fleet import _check_stale_fleet_state # noqa: F401 diff --git a/tests/cli/test_features_cli.py b/tests/cli/test_features_cli.py new file mode 100644 index 0000000000..f02b0f4ac0 --- /dev/null +++ b/tests/cli/test_features_cli.py @@ -0,0 +1,108 @@ +"""Tests for the features CLI subcommand.""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +pytestmark = [pytest.mark.layer("cli"), pytest.mark.medium] + + +# --------------------------------------------------------------------------- +# T_FEATURES_1: features list shows all registered features +# --------------------------------------------------------------------------- + + +def test_features_list_shows_registry( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path, capsys: pytest.CaptureFixture +) -> None: + """features list output includes all registered feature names and lifecycle values.""" + monkeypatch.chdir(tmp_path) + monkeypatch.setattr( + "autoskillit.config.load_config", + lambda path: type("C", (), {"features": {}, "experimental_enabled": False})(), + ) + from autoskillit.cli._features import features_list + + features_list() + out = capsys.readouterr().out + assert "fleet" in out + assert "experimental" in out + assert "FEATURE" in out + assert "LIFECYCLE" in out + assert "EFFECTIVE" in out + fleet_line = next( + line for line in out.splitlines() if "fleet" in line and "FEATURE" not in line + ) + assert len(fleet_line.split()) >= 6 + + +# --------------------------------------------------------------------------- +# T_FEATURES_2: effective column reflects config override +# --------------------------------------------------------------------------- + + +def test_features_list_shows_effective_state( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path, capsys: pytest.CaptureFixture +) -> None: + """features list shows effective=false and source=config when feature is overridden.""" + monkeypatch.chdir(tmp_path) + monkeypatch.setattr( + "autoskillit.config.load_config", + lambda path: type( + "C", (), {"features": {"fleet": False}, "experimental_enabled": False} + )(), + ) + from autoskillit.cli._features import features_list + + features_list() + out = capsys.readouterr().out + # fleet row should show effective=false and source=config + fleet_line = next(line for line in out.splitlines() if "fleet" in line) + assert "false" in fleet_line + assert "config" in fleet_line + + +# --------------------------------------------------------------------------- +# T_FEATURES_3: features status shows all FeatureDef fields +# --------------------------------------------------------------------------- + + +def test_features_status_detail( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path, capsys: pytest.CaptureFixture +) -> None: + """features status fleet shows all FeatureDef fields.""" + monkeypatch.chdir(tmp_path) + monkeypatch.setattr( + "autoskillit.config.load_config", + lambda path: type("C", (), {"features": {}, "experimental_enabled": False})(), + ) + from autoskillit.cli._features import features_status + + features_status("fleet") + out = capsys.readouterr().out + assert "fleet" in out + assert "experimental" in out # lifecycle + tier_line = next(line for line in out.splitlines() if "Tier" in line) + assert "1" in tier_line # tier value on its own line + assert "false" in out # disabled (default_enabled=False for fleet) + + +# --------------------------------------------------------------------------- +# T_FEATURES_4: features status unknown name exits 1 +# --------------------------------------------------------------------------- + + +def test_features_status_unknown(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: + """features status exits 1 for an unknown feature name.""" + monkeypatch.chdir(tmp_path) + monkeypatch.setattr( + "autoskillit.config.load_config", + lambda path: type("C", (), {"features": {}, "experimental_enabled": False})(), + ) + from autoskillit.cli._features import features_status + + with pytest.raises(SystemExit) as exc_info: + features_status("nonexistent-feature") + assert exc_info.value.code == 1 diff --git a/tests/cli/test_fleet_campaign.py b/tests/cli/test_fleet_campaign.py new file mode 100644 index 0000000000..b488856d20 --- /dev/null +++ b/tests/cli/test_fleet_campaign.py @@ -0,0 +1,403 @@ +"""Tests: fleet CLI campaign command.""" + +from __future__ import annotations + +from pathlib import Path +from unittest.mock import MagicMock + +import pytest + +from autoskillit.cli.fleet import fleet_campaign as _fleet_campaign +from tests.cli._fleet_helpers import ( + _capture_subprocess, + _setup_campaign_with_status, + _setup_existing_campaign_state, + _stub_campaign_resolution, + _stub_guards, +) + +pytestmark = [pytest.mark.layer("cli"), pytest.mark.medium, pytest.mark.feature("fleet")] + + +def test_fleet_run_exits_when_campaign_not_found( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + """fleet run exits 1 when campaign name doesn't resolve.""" + monkeypatch.chdir(tmp_path) + _stub_guards(monkeypatch) + with pytest.raises(SystemExit, match="1"): + _fleet_campaign("nonexistent-campaign") + + +def test_fleet_run_sets_session_type_fleet( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + """Subprocess env includes AUTOSKILLIT_SESSION_TYPE=fleet.""" + _stub_guards(monkeypatch) + monkeypatch.chdir(tmp_path) + _stub_campaign_resolution(monkeypatch, tmp_path, "test-campaign") + captured = _capture_subprocess(monkeypatch) + _fleet_campaign("test-campaign") + env = captured["env"] + assert "AUTOSKILLIT_SESSION_TYPE" in env and env["AUTOSKILLIT_SESSION_TYPE"] == "fleet" + assert "AUTOSKILLIT_CAMPAIGN_ID" in env + assert "AUTOSKILLIT_CAMPAIGN_STATE_PATH" in env + assert captured["env"].get("AUTOSKILLIT_FLEET_MODE") == "campaign" + + +def test_fleet_run_writes_initial_state(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: + """state.json exists in fleet temp dir after launch.""" + _stub_guards(monkeypatch) + monkeypatch.chdir(tmp_path) + _stub_campaign_resolution(monkeypatch, tmp_path, "test-campaign") + _capture_subprocess(monkeypatch) + _fleet_campaign("test-campaign") + state_dirs = list((tmp_path / ".autoskillit" / "temp" / "fleet").iterdir()) + assert len(state_dirs) == 1 + state_file = state_dirs[0] / "state.json" + assert state_file.exists() + + +def test_fleet_run_resume_reads_existing_state( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + """--resume-campaign reuses existing campaign_id and passes completed block to prompt.""" + _stub_guards(monkeypatch) + monkeypatch.chdir(tmp_path) + campaign_id = "abc123def456ab12" + _setup_existing_campaign_state(tmp_path, campaign_id, "test-campaign") + _stub_campaign_resolution(monkeypatch, tmp_path, "test-campaign") + captured = _capture_subprocess(monkeypatch) + _fleet_campaign("test-campaign", resume_campaign=campaign_id) + assert captured["env"]["AUTOSKILLIT_CAMPAIGN_ID"] == campaign_id + + +def test_fleet_run_exit_code_passthrough(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: + """fleet run propagates subprocess exit code.""" + import subprocess + + _stub_guards(monkeypatch) + monkeypatch.chdir(tmp_path) + _stub_campaign_resolution(monkeypatch, tmp_path, "test-campaign") + monkeypatch.setattr("autoskillit.cli._init_helpers._is_plugin_installed", lambda: True) + monkeypatch.setattr( + subprocess, + "run", + lambda *a, **kw: type("R", (), {"returncode": 2, "stdout": "", "stderr": ""})(), + ) + with pytest.raises(SystemExit) as exc_info: + _fleet_campaign("test-campaign") + assert exc_info.value.code == 2 + + +def test_fleet_campaign_exits_when_disabled( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + """fleet_campaign exits 1 when fleet feature is disabled via feature gate.""" + monkeypatch.chdir(tmp_path) + _stub_guards(monkeypatch) + checked_features: list[str] = [] + monkeypatch.setattr( + "autoskillit.cli.fleet.is_feature_enabled", + lambda name, features, *, experimental_enabled=False: ( + checked_features.append(name) or False + ), + ) + monkeypatch.setattr( + "autoskillit.config.load_config", + lambda path: type("C", (), {"features": {}, "experimental_enabled": False})(), + ) + with pytest.raises(SystemExit) as exc_info: + _fleet_campaign("any-campaign") + assert exc_info.value.code == 1 + assert "fleet" in checked_features + + +def _stub_list_campaign_recipes(monkeypatch: pytest.MonkeyPatch, names: list[str]) -> list[object]: + """Stub list_campaign_recipes to return mock items with given names.""" + items = [] + for name in names: + r = MagicMock() + r.name = name + r.description = f"Description for {name}" + items.append(r) + + result = MagicMock() + result.items = items + monkeypatch.setattr("autoskillit.recipe.list_campaign_recipes", lambda *a, **kw: result) + return items + + +def test_fleet_campaign_no_name_shows_menu_and_launches( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + """When campaign_name is omitted, menu is shown and selected campaign launches.""" + _stub_guards(monkeypatch) + monkeypatch.chdir(tmp_path) + _stub_list_campaign_recipes(monkeypatch, ["campaign-alpha", "campaign-beta"]) + _stub_campaign_resolution(monkeypatch, tmp_path, "campaign-alpha") + monkeypatch.setattr("autoskillit.cli.ui._menu.timed_prompt", lambda *a, **kw: "1") + captured = _capture_subprocess(monkeypatch) + _fleet_campaign(campaign_name=None) + assert "AUTOSKILLIT_CAMPAIGN_ID" in captured["env"] + assert "AUTOSKILLIT_CAMPAIGN_STATE_PATH" in captured["env"] + from autoskillit.fleet import read_state + + state = read_state(Path(captured["env"]["AUTOSKILLIT_CAMPAIGN_STATE_PATH"])) + assert state is not None + assert state.campaign_name == "campaign-alpha" + out = capsys.readouterr().out + assert "Available campaigns:" in out + assert "campaign-alpha" in out + assert "campaign-beta" in out + + +def test_fleet_campaign_no_name_no_campaigns_exits( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + """When no campaigns exist and name is omitted, exits with message.""" + _stub_guards(monkeypatch) + monkeypatch.chdir(tmp_path) + _stub_list_campaign_recipes(monkeypatch, []) + with pytest.raises(SystemExit, match="1"): + _fleet_campaign(campaign_name=None) + + +def test_fleet_campaign_no_name_invalid_selection_exits( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + """Empty selection from menu exits with code 1.""" + _stub_guards(monkeypatch) + monkeypatch.chdir(tmp_path) + _stub_list_campaign_recipes(monkeypatch, ["campaign-alpha"]) + monkeypatch.setattr("autoskillit.cli.ui._menu.timed_prompt", lambda *a, **kw: "") + with pytest.raises(SystemExit, match="1"): + _fleet_campaign(campaign_name=None) + + +def test_fleet_campaign_resume_no_name_lists_active_campaigns( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + """Resume without name shows only active (non-terminal) campaigns.""" + _stub_guards(monkeypatch) + monkeypatch.chdir(tmp_path) + + _setup_existing_campaign_state(tmp_path, "active-id-1111111", "campaign-active-1") + _setup_existing_campaign_state(tmp_path, "active-id-2222222", "campaign-active-2") + _setup_campaign_with_status( + tmp_path, "terminal-id-333333", "campaign-terminal", status="success" + ) + + _stub_campaign_resolution(monkeypatch, tmp_path, "campaign-active-1") + monkeypatch.setattr("autoskillit.cli.ui._menu.timed_prompt", lambda *a, **kw: "1") + monkeypatch.setattr( + "autoskillit.fleet.resume_campaign_from_state", + lambda *a, **kw: MagicMock( + completed_dispatches_block="", next_dispatch_name="", is_resumable=False + ), + ) + captured = _capture_subprocess(monkeypatch) + + _fleet_campaign(campaign_name=None, resume_campaign="__pick__") + + assert captured["env"].get("AUTOSKILLIT_CAMPAIGN_ID") == "active-id-1111111" + assert "terminal-id-333333" not in captured["env"].get("AUTOSKILLIT_CAMPAIGN_ID", "") + out = capsys.readouterr().out + assert "Active campaigns (resumable):" in out + assert "campaign-active-1" in out + assert "campaign-active-2" in out + assert "campaign-terminal" not in out + + +class TestFleetCampaignResumeHaltedExits: + def test_resume_halted_campaign_exits_with_error( + self, monkeypatch: pytest.MonkeyPatch, tmp_path: Path, capsys: pytest.CaptureFixture[str] + ) -> None: + _stub_guards(monkeypatch) + monkeypatch.chdir(tmp_path) + campaign_id = "halted-campaign-id1" + _setup_existing_campaign_state(tmp_path, campaign_id, "test-campaign") + _stub_campaign_resolution(monkeypatch, tmp_path, "test-campaign") + + from autoskillit.fleet import FLEET_HALTED_SENTINEL + + monkeypatch.setattr( + "autoskillit.fleet.resume_campaign_from_state", + lambda *a, **kw: MagicMock( + completed_dispatches_block=FLEET_HALTED_SENTINEL, + next_dispatch_name="", + is_resumable=False, + ), + ) + with pytest.raises(SystemExit, match="1"): + _fleet_campaign("test-campaign", resume_campaign=campaign_id) + err = capsys.readouterr().err + assert "halted" in err.lower() + assert "continue_on_failure" in err + + def test_resume_halted_does_not_launch_session( + self, monkeypatch: pytest.MonkeyPatch, tmp_path: Path + ) -> None: + _stub_guards(monkeypatch) + monkeypatch.chdir(tmp_path) + campaign_id = "halted-campaign-id2" + _setup_existing_campaign_state(tmp_path, campaign_id, "test-campaign") + _stub_campaign_resolution(monkeypatch, tmp_path, "test-campaign") + + from autoskillit.fleet import FLEET_HALTED_SENTINEL + + monkeypatch.setattr( + "autoskillit.fleet.resume_campaign_from_state", + lambda *a, **kw: MagicMock( + completed_dispatches_block=FLEET_HALTED_SENTINEL, + next_dispatch_name="", + is_resumable=False, + ), + ) + launch_called = False + + def _track_launch(*a: object, **kw: object) -> None: + nonlocal launch_called + launch_called = True + + monkeypatch.setattr("autoskillit.cli.fleet._launch_fleet_session", _track_launch) + with pytest.raises(SystemExit): + _fleet_campaign("test-campaign", resume_campaign=campaign_id) + assert not launch_called + + +def test_fleet_campaign_passes_initial_message( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + """fleet_campaign() passes a non-None initial_message containing the campaign name.""" + _stub_guards(monkeypatch) + monkeypatch.chdir(tmp_path) + _stub_campaign_resolution(monkeypatch, tmp_path, "test-campaign") + captured_kwargs: dict = {} + + def mock_launch_session(*a: object, **kw: object) -> None: + captured_kwargs.update(kw) + + monkeypatch.setattr("autoskillit.cli.fleet._launch_fleet_session", mock_launch_session) + monkeypatch.setattr( + "autoskillit.cli._preview._pre_launch_campaign", + lambda *a, **kw: ("", True), + ) + _fleet_campaign("test-campaign") + assert captured_kwargs.get("initial_message") is not None + assert "test-campaign" in captured_kwargs["initial_message"] + + +def test_fleet_campaign_greetings_have_campaign_name_placeholder() -> None: + """All _FLEET_CAMPAIGN_GREETINGS entries must contain {campaign_name}.""" + from autoskillit.cli.fleet._fleet_preview import _FLEET_CAMPAIGN_GREETINGS + + assert len(_FLEET_CAMPAIGN_GREETINGS) >= 3 + for greeting in _FLEET_CAMPAIGN_GREETINGS: + assert "{campaign_name}" in greeting + + +def test_launch_fleet_session_forwards_initial_message_campaign( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + """_launch_fleet_session in campaign mode forwards initial_message on first launch only.""" + monkeypatch.chdir(tmp_path) + captured_calls: list[dict] = [] + + def mock_run(system_prompt: str, **kwargs: object) -> None: + captured_calls.append(dict(kwargs)) + return None + + monkeypatch.setattr( + "autoskillit.cli.session._session_launch._run_interactive_session", + mock_run, + ) + monkeypatch.setattr( + "autoskillit.cli._prompts._build_fleet_campaign_prompt", + lambda *a, **kw: "campaign-prompt", + ) + monkeypatch.setattr("autoskillit.cli.fleet._fleet_session.dump_yaml_str", lambda *a, **kw: "") + from autoskillit.fleet import ResumeDecision + + monkeypatch.setattr( + "autoskillit.fleet.resume_campaign_from_state", + lambda *a, **kw: ResumeDecision( + completed_dispatches_block="", + next_dispatch_name="", + is_resumable=False, + dispatched_session_id="", + kill_reason="", + ), + ) + from autoskillit.cli.fleet._fleet_session import _launch_fleet_session + + recipe = MagicMock() + recipe.dispatches = [] + recipe.continue_on_failure = False + + _launch_fleet_session( + recipe, + "test-campaign-id", + tmp_path / "state.json", + None, + fleet_mode="campaign", + initial_message="Hello, campaign!", + ) + assert len(captured_calls) == 1 + assert captured_calls[0].get("initial_message") == "Hello, campaign!" + + +def test_launch_fleet_session_clears_initial_message_on_reload_campaign( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + """On reload in campaign mode, initial_message must be None (only injected on first launch).""" + monkeypatch.chdir(tmp_path) + call_count = 0 + captured_messages: list = [] + + def mock_run(system_prompt: str, **kwargs: object) -> object: + nonlocal call_count + captured_messages.append(kwargs.get("initial_message")) + call_count += 1 + return "reload-session-abc" if call_count == 1 else None + + monkeypatch.setattr( + "autoskillit.cli.session._session_launch._run_interactive_session", + mock_run, + ) + monkeypatch.setattr( + "autoskillit.cli._prompts._build_fleet_campaign_prompt", + lambda *a, **kw: "campaign-prompt", + ) + monkeypatch.setattr("autoskillit.cli.fleet._fleet_session.dump_yaml_str", lambda *a, **kw: "") + from autoskillit.fleet import ResumeDecision + + monkeypatch.setattr( + "autoskillit.fleet.resume_campaign_from_state", + lambda *a, **kw: ResumeDecision( + completed_dispatches_block="", + next_dispatch_name="", + is_resumable=False, + dispatched_session_id="", + kill_reason="", + ), + ) + from autoskillit.cli.fleet._fleet_session import _launch_fleet_session + + recipe = MagicMock() + recipe.dispatches = [] + recipe.continue_on_failure = False + + _launch_fleet_session( + recipe, + "test-campaign-id", + tmp_path / "state.json", + None, + fleet_mode="campaign", + initial_message="Hello!", + ) + assert len(captured_messages) >= 2, ( + f"expected reload to fire but got only {len(captured_messages)} call(s)" + ) + assert captured_messages[0] == "Hello!" + assert captured_messages[1] is None diff --git a/tests/cli/test_fleet_campaign_preview.py b/tests/cli/test_fleet_campaign_preview.py new file mode 100644 index 0000000000..9b24771e46 --- /dev/null +++ b/tests/cli/test_fleet_campaign_preview.py @@ -0,0 +1,124 @@ +"""Tests: fleet_campaign shows preview + confirmation before launch.""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from autoskillit.cli.fleet import fleet_campaign as _fleet_campaign +from tests.cli._fleet_helpers import ( + _stub_campaign_resolution, + _stub_guards, +) + +pytestmark = [pytest.mark.layer("cli"), pytest.mark.medium, pytest.mark.feature("fleet")] + + +def _stub_preview_layer( + monkeypatch: pytest.MonkeyPatch, + *, + timed_prompt_return: str = "", + ingredients_table: str | None = None, +) -> dict: + """Stub preview/prompt/launch layer; return captured call tracker.""" + calls: dict = {"preview": [], "launch": [], "timed_prompt": []} + + def _fake_preview(*args: object, **kwargs: object) -> None: + calls["preview"].append(args) + + def _fake_get_itable(*args: object, **kwargs: object) -> str | None: + return ingredients_table + + def _fake_timed_prompt(*args: object, **kwargs: object) -> str: + calls["timed_prompt"].append((args, kwargs)) + return timed_prompt_return + + def _fake_launch(*args: object, **kwargs: object) -> None: + calls["launch"].append({"args": args, "kwargs": kwargs}) + + monkeypatch.setattr("autoskillit.cli._preview.show_campaign_preview", _fake_preview) + monkeypatch.setattr("autoskillit.cli._prompts._get_ingredients_table", _fake_get_itable) + monkeypatch.setattr("autoskillit.cli.ui._timed_input.timed_prompt", _fake_timed_prompt) + monkeypatch.setattr("autoskillit.cli.fleet._launch_fleet_session", _fake_launch) + return calls + + +class TestFleetCampaignPreview: + def test_ingredient_table_displayed_before_launch( + self, capsys: pytest.CaptureFixture[str], monkeypatch: pytest.MonkeyPatch, tmp_path: Path + ) -> None: + _stub_guards(monkeypatch) + monkeypatch.chdir(tmp_path) + _stub_campaign_resolution(monkeypatch, tmp_path, "test-campaign") + calls = _stub_preview_layer( + monkeypatch, ingredients_table="| Name | Desc |\n| task | ... |" + ) + _fleet_campaign("test-campaign") + assert len(calls["preview"]) == 1 + + def test_confirmation_prompt_shown( + self, monkeypatch: pytest.MonkeyPatch, tmp_path: Path + ) -> None: + _stub_guards(monkeypatch) + monkeypatch.chdir(tmp_path) + _stub_campaign_resolution(monkeypatch, tmp_path, "test-campaign") + calls = _stub_preview_layer(monkeypatch) + _fleet_campaign("test-campaign") + assert len(calls["timed_prompt"]) == 1 + args, kwargs = calls["timed_prompt"][0] + assert "campaign" in (args[0] if args else kwargs.get("prompt", "")).lower() + + def test_user_declines_does_not_launch( + self, monkeypatch: pytest.MonkeyPatch, tmp_path: Path + ) -> None: + _stub_guards(monkeypatch) + monkeypatch.chdir(tmp_path) + _stub_campaign_resolution(monkeypatch, tmp_path, "test-campaign") + calls = _stub_preview_layer(monkeypatch, timed_prompt_return="n") + _fleet_campaign("test-campaign") + assert len(calls["launch"]) == 0 + + def test_ingredients_table_passed_to_session( + self, monkeypatch: pytest.MonkeyPatch, tmp_path: Path + ) -> None: + _stub_guards(monkeypatch) + monkeypatch.chdir(tmp_path) + _stub_campaign_resolution(monkeypatch, tmp_path, "test-campaign") + table = "| Name | Value |\n| task | build |" + calls = _stub_preview_layer(monkeypatch, ingredients_table=table) + _fleet_campaign("test-campaign") + assert len(calls["launch"]) == 1 + assert calls["launch"][0]["kwargs"].get("ingredients_table") == table + + def test_resume_skips_preview_and_confirmation( + self, monkeypatch: pytest.MonkeyPatch, tmp_path: Path + ) -> None: + from tests.cli._fleet_helpers import _setup_existing_campaign_state + + _stub_guards(monkeypatch) + monkeypatch.chdir(tmp_path) + campaign_id = "abc123def456ab12" + _setup_existing_campaign_state(tmp_path, campaign_id, "test-campaign") + _stub_campaign_resolution(monkeypatch, tmp_path, "test-campaign") + calls = _stub_preview_layer(monkeypatch) + _fleet_campaign("test-campaign", resume_campaign=campaign_id) + assert len(calls["preview"]) == 0 + assert len(calls["timed_prompt"]) == 0 + assert len(calls["launch"]) == 1 + + def test_resume_still_passes_ingredients_table_to_session( + self, monkeypatch: pytest.MonkeyPatch, tmp_path: Path + ) -> None: + from tests.cli._fleet_helpers import _setup_existing_campaign_state + + _stub_guards(monkeypatch) + monkeypatch.chdir(tmp_path) + campaign_id = "abc123def456ab12" + _setup_existing_campaign_state(tmp_path, campaign_id, "test-campaign") + _stub_campaign_resolution(monkeypatch, tmp_path, "test-campaign") + table = "| Name | Value |\n| task | fix |" + calls = _stub_preview_layer(monkeypatch, ingredients_table=table) + _fleet_campaign("test-campaign", resume_campaign=campaign_id) + assert len(calls["launch"]) == 1 + assert calls["launch"][0]["kwargs"].get("ingredients_table") == table diff --git a/tests/cli/test_fleet_dispatch.py b/tests/cli/test_fleet_dispatch.py new file mode 100644 index 0000000000..2b3af4c36c --- /dev/null +++ b/tests/cli/test_fleet_dispatch.py @@ -0,0 +1,579 @@ +"""Tests: fleet CLI dispatch command.""" + +from __future__ import annotations + +import shutil +from pathlib import Path + +import pytest + +from autoskillit.cli.fleet import fleet_dispatch as _fleet_dispatch +from tests.cli._fleet_helpers import ( + _capture_subprocess, + _stub_guards, +) + +pytestmark = [pytest.mark.layer("cli"), pytest.mark.medium, pytest.mark.feature("fleet")] + + +def _fake_recipe(name: str, source: str, description: str) -> object: + """Create a minimal RecipeInfo-like object for test stubs.""" + return type( + "RecipeInfo", + (), + { + "name": name, + "source": source, + "description": description, + "kind": "standard", + "path": Path(f"/fake/{name}.yaml"), + "summary": "", + "version": None, + "recipe_version": None, + "content_hash": "", + "content": None, + }, + )() + + +def _stub_list_recipes(monkeypatch: pytest.MonkeyPatch, recipes: list) -> None: + """Stub list_recipes to return the given recipe list.""" + result = type("LoadResult", (), {"items": recipes, "errors": []})() + monkeypatch.setattr("autoskillit.recipe.list_recipes", lambda *a, **kw: result) + + +# --------------------------------------------------------------------------- +# T1. fleet dispatch — CLAUDECODE guard +# --------------------------------------------------------------------------- + + +def test_fleet_dispatch_rejects_inside_claude_session(monkeypatch: pytest.MonkeyPatch) -> None: + """fleet dispatch exits 1 when CLAUDECODE env is set.""" + monkeypatch.setenv("CLAUDECODE", "1") + with pytest.raises(SystemExit, match="1"): + _fleet_dispatch() + + +# --------------------------------------------------------------------------- +# T2. fleet dispatch — SESSION_TYPE=skill guard +# --------------------------------------------------------------------------- + + +def test_fleet_dispatch_rejects_skill_session_type(monkeypatch: pytest.MonkeyPatch) -> None: + """fleet dispatch exits 1 when ambient SESSION_TYPE is skill.""" + monkeypatch.setenv("AUTOSKILLIT_SESSION_TYPE", "skill") + monkeypatch.delenv("CLAUDECODE", raising=False) + with pytest.raises(SystemExit, match="1"): + _fleet_dispatch() + + +def test_fleet_dispatch_rejects_deprecated_leaf_session_type( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """fleet dispatch exits 1 when ambient SESSION_TYPE is deprecated 'leaf'.""" + monkeypatch.setenv("AUTOSKILLIT_SESSION_TYPE", "leaf") + monkeypatch.delenv("CLAUDECODE", raising=False) + with pytest.raises(SystemExit, match="1"): + _fleet_dispatch() + + +# --------------------------------------------------------------------------- +# T3. fleet dispatch — claude not on PATH +# --------------------------------------------------------------------------- + + +def test_fleet_dispatch_exits_when_claude_missing( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + """fleet dispatch exits 1 when claude is not on PATH.""" + monkeypatch.chdir(tmp_path) + monkeypatch.delenv("CLAUDECODE", raising=False) + monkeypatch.delenv("AUTOSKILLIT_SESSION_TYPE", raising=False) + monkeypatch.setattr("autoskillit.cli.fleet.is_feature_enabled", lambda *a, **kw: True) + monkeypatch.setattr( + "autoskillit.config.load_config", + lambda path: type("C", (), {"features": {}, "experimental_enabled": False})(), + ) + _stub_list_recipes(monkeypatch, []) + monkeypatch.setattr("autoskillit.cli.ui._timed_input.timed_prompt", lambda *a, **kw: "") + monkeypatch.setattr(shutil, "which", lambda _: None) + with pytest.raises(SystemExit, match="1"): + _fleet_dispatch() + + +# --------------------------------------------------------------------------- +# T_GUARD_1: fleet_dispatch exits 1 when fleet feature disabled +# --------------------------------------------------------------------------- + + +def test_fleet_dispatch_exits_when_disabled( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + """fleet_dispatch exits 1 when fleet feature is disabled via feature gate.""" + monkeypatch.chdir(tmp_path) + _stub_guards(monkeypatch) + checked_features: list[str] = [] + monkeypatch.setattr( + "autoskillit.cli.fleet.is_feature_enabled", + lambda name, features, *, experimental_enabled=False: ( + checked_features.append(name) or False + ), + ) + monkeypatch.setattr( + "autoskillit.config.load_config", + lambda path: type("C", (), {"features": {}, "experimental_enabled": False})(), + ) + with pytest.raises(SystemExit) as exc_info: + _fleet_dispatch() + assert exc_info.value.code == 1 + assert "fleet" in checked_features + + +# --------------------------------------------------------------------------- +# T_GUARD_4: fleet_dispatch proceeds normally when fleet enabled +# --------------------------------------------------------------------------- + + +def test_fleet_dispatch_proceeds_when_enabled( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + """fleet_dispatch passes the feature guard and proceeds to campaign resolution.""" + _stub_guards(monkeypatch) + monkeypatch.chdir(tmp_path) + monkeypatch.setattr( + "autoskillit.cli.fleet.is_feature_enabled", + lambda name, features, *, experimental_enabled=False: True, + ) + monkeypatch.setattr( + "autoskillit.config.load_config", + lambda path: type("C", (), {"features": {}, "experimental_enabled": False})(), + ) + _stub_list_recipes(monkeypatch, []) + monkeypatch.setattr("autoskillit.cli.ui._timed_input.timed_prompt", lambda *a, **kw: "") + captured = _capture_subprocess(monkeypatch) + _fleet_dispatch() + assert captured.get("cmd") is not None, "Expected fleet session subprocess to be invoked" + assert captured["env"].get("AUTOSKILLIT_FLEET_MODE") == "dispatch" + + +# --------------------------------------------------------------------------- +# T_ADHOC. Ad-hoc fleet dispatch mode (campaign_name=None) +# --------------------------------------------------------------------------- + + +def test_fleet_dispatch_sets_fleet_mode_dispatch( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + """fleet_dispatch() must launch an interactive session, not exit 1.""" + _stub_guards(monkeypatch) + monkeypatch.chdir(tmp_path) + _stub_list_recipes(monkeypatch, []) + monkeypatch.setattr("autoskillit.cli.ui._timed_input.timed_prompt", lambda *a, **kw: "") + captured = _capture_subprocess(monkeypatch) + _fleet_dispatch() + assert captured["env"].get("AUTOSKILLIT_FLEET_MODE") == "dispatch" + + +def test_fleet_dispatch_writes_no_state(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: + """fleet_dispatch must not create a state.json under .autoskillit/temp/fleet/.""" + _stub_guards(monkeypatch) + monkeypatch.chdir(tmp_path) + _stub_list_recipes(monkeypatch, []) + monkeypatch.setattr("autoskillit.cli.ui._timed_input.timed_prompt", lambda *a, **kw: "") + _capture_subprocess(monkeypatch) + _fleet_dispatch() + fleet_dir = tmp_path / ".autoskillit" / "temp" / "fleet" + assert not fleet_dir.exists() or not any(fleet_dir.rglob("state.json")) + + +def test_fleet_dispatch_no_campaign_env_vars( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + """Ad-hoc session must not set AUTOSKILLIT_CAMPAIGN_ID or AUTOSKILLIT_CAMPAIGN_STATE_PATH.""" + _stub_guards(monkeypatch) + monkeypatch.chdir(tmp_path) + _stub_list_recipes(monkeypatch, []) + monkeypatch.setattr("autoskillit.cli.ui._timed_input.timed_prompt", lambda *a, **kw: "") + captured = _capture_subprocess(monkeypatch) + _fleet_dispatch() + env = captured["env"] + assert "AUTOSKILLIT_CAMPAIGN_ID" not in env + assert "AUTOSKILLIT_CAMPAIGN_STATE_PATH" not in env + + +def test_build_fleet_dispatch_prompt_no_open_kitchen() -> None: + """Fleet dispatch prompt must NOT instruct calling open_kitchen.""" + from autoskillit.cli._mcp_names import DIRECT_PREFIX + from autoskillit.cli._prompts import _build_fleet_dispatch_prompt + + prompt = _build_fleet_dispatch_prompt(DIRECT_PREFIX) + assert "open_kitchen" not in prompt + + +def test_build_fleet_dispatch_prompt_references_dispatch_tool() -> None: + from autoskillit.cli._mcp_names import DIRECT_PREFIX + from autoskillit.cli._prompts import _build_fleet_dispatch_prompt + + prompt = _build_fleet_dispatch_prompt(DIRECT_PREFIX) + assert "dispatch_food_truck" in prompt + + +def test_build_fleet_dispatch_prompt_no_campaign_manifest() -> None: + from autoskillit.cli._mcp_names import DIRECT_PREFIX + from autoskillit.cli._prompts import _build_fleet_dispatch_prompt + + prompt = _build_fleet_dispatch_prompt(DIRECT_PREFIX) + assert "DISPATCH MANIFEST" not in prompt + assert "CAMPAIGN OVERVIEW" not in prompt + assert "CAMPAIGN DISCIPLINE" not in prompt + + +def test_build_fleet_dispatch_prompt_accepts_marketplace_prefix() -> None: + from autoskillit.cli._mcp_names import MARKETPLACE_PREFIX + from autoskillit.cli._prompts import _build_fleet_dispatch_prompt + + prompt = _build_fleet_dispatch_prompt(MARKETPLACE_PREFIX) + assert MARKETPLACE_PREFIX + "open_kitchen" not in prompt + assert MARKETPLACE_PREFIX + "dispatch_food_truck" in prompt + + +def test_build_fleet_dispatch_prompt_lists_all_10_tools() -> None: + """Dispatch prompt must enumerate all 10 tools in the TOOL SURFACE section.""" + from autoskillit.cli._mcp_names import DIRECT_PREFIX + from autoskillit.cli._prompts import _build_fleet_dispatch_prompt + + prompt = _build_fleet_dispatch_prompt(DIRECT_PREFIX) + assert "TOOL SURFACE" in prompt, "Expected TOOL SURFACE section in dispatch prompt" + tool_surface_start = prompt.index("TOOL SURFACE") + next_section = prompt.find("\n##", tool_surface_start + 1) + tool_surface = ( + prompt[tool_surface_start:next_section] + if next_section != -1 + else prompt[tool_surface_start:] + ) + for tool in ( + "dispatch_food_truck", + "batch_cleanup_clones", + "get_pipeline_report", + "get_token_summary", + "get_timing_summary", + "get_quota_events", + "list_recipes", + "load_recipe", + "fetch_github_issue", + "get_issue_title", + ): + assert tool in tool_surface, ( + f"Expected tool {tool!r} in TOOL SURFACE section of dispatch prompt" + ) + + +def test_build_fleet_dispatch_prompt_includes_admiral_sections() -> None: + """Dispatch prompt must include the 4 L3 admiral sections.""" + from autoskillit.cli._mcp_names import DIRECT_PREFIX + from autoskillit.cli._prompts import _build_fleet_dispatch_prompt + from autoskillit.fleet import _build_admiral_dispatch_block + + admiral_block = _build_admiral_dispatch_block() + assert admiral_block + prompt = _build_fleet_dispatch_prompt(DIRECT_PREFIX) + assert "ADMIRAL DISCIPLINE" in prompt + for section in ( + "CONTEXT LIMIT ROUTING", + "STEP NAME IMMUTABILITY", + "MERGE PHASE", + "QUOTA WAIT PROTOCOL", + ): + assert section in prompt, f"Expected admiral section {section!r} in dispatch prompt" + + +def test_build_fleet_dispatch_prompt_has_recipe_discovery_guidance() -> None: + """Dispatch prompt must guide recipe discovery flow.""" + from autoskillit.cli._mcp_names import DIRECT_PREFIX + from autoskillit.cli._prompts import _build_fleet_dispatch_prompt + + prompt = _build_fleet_dispatch_prompt(DIRECT_PREFIX) + assert "list_recipes" in prompt + assert "load_recipe" in prompt + + +def test_build_fleet_dispatch_prompt_role_text() -> None: + """Dispatch prompt must identify role as fleet dispatcher.""" + from autoskillit.cli._mcp_names import DIRECT_PREFIX + from autoskillit.cli._prompts import _build_fleet_dispatch_prompt + + prompt = _build_fleet_dispatch_prompt(DIRECT_PREFIX) + assert "fleet dispatcher" in prompt.lower() + + +def test_build_fleet_dispatch_prompt_has_cleanup_protocol() -> None: + """Dispatch prompt must include batch_cleanup_clones exit instruction.""" + from autoskillit.cli._mcp_names import DIRECT_PREFIX + from autoskillit.cli._prompts import _build_fleet_dispatch_prompt + + prompt = _build_fleet_dispatch_prompt(DIRECT_PREFIX) + assert "batch_cleanup_clones" in prompt + + +def test_build_fleet_dispatch_prompt_no_sleep_toolsearch_preamble() -> None: + """Dispatch prompt must NOT include sleep/ToolSearch boot sequence.""" + from autoskillit.cli._mcp_names import DIRECT_PREFIX + from autoskillit.cli._prompts import _build_fleet_dispatch_prompt + + prompt = _build_fleet_dispatch_prompt(DIRECT_PREFIX) + assert "sleep 2" not in prompt + + +def test_build_fleet_dispatch_prompt_uses_ingredients_only() -> None: + """_build_fleet_dispatch_prompt recipe discovery must mention ingredients_only.""" + from autoskillit.cli._mcp_names import DIRECT_PREFIX + from autoskillit.cli._prompts import _build_fleet_dispatch_prompt + + prompt = _build_fleet_dispatch_prompt(DIRECT_PREFIX) + assert "ingredients_only" in prompt + + +# --------------------------------------------------------------------------- +# T_PREVIEW: Pre-launch display +# --------------------------------------------------------------------------- + + +def test_fleet_dispatch_prints_recipe_roster( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + """fleet_dispatch prints a food truck roster table before launching.""" + _stub_guards(monkeypatch) + monkeypatch.chdir(tmp_path) + _stub_list_recipes( + monkeypatch, + [ + _fake_recipe("smoke-test", "BUILTIN", "Run smoke tests"), + _fake_recipe("review-pr", "BUILTIN", "Review a pull request"), + ], + ) + monkeypatch.setattr("autoskillit.cli.ui._timed_input.timed_prompt", lambda *a, **kw: "") + _capture_subprocess(monkeypatch) + _fleet_dispatch() + out = capsys.readouterr().out + assert "smoke-test" in out + assert "review-pr" in out + + +def test_fleet_dispatch_shows_permissions_warning( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + """fleet_dispatch prints the permissions_warning text.""" + _stub_guards(monkeypatch) + monkeypatch.chdir(tmp_path) + _stub_list_recipes(monkeypatch, []) + monkeypatch.setattr("autoskillit.cli.ui._timed_input.timed_prompt", lambda *a, **kw: "") + _capture_subprocess(monkeypatch) + _fleet_dispatch() + out = capsys.readouterr().out + assert "dangerously-skip-permissions" in out + + +def test_fleet_dispatch_aborts_on_no(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: + """fleet_dispatch returns without launching when user types 'n'.""" + _stub_guards(monkeypatch) + monkeypatch.chdir(tmp_path) + _stub_list_recipes(monkeypatch, []) + monkeypatch.setattr("autoskillit.cli.ui._timed_input.timed_prompt", lambda *a, **kw: "n") + captured = _capture_subprocess(monkeypatch) + _fleet_dispatch() + assert "cmd" not in captured, "Session should not launch when user aborts" + + +def test_fleet_dispatch_prints_fleet_tool_surface( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + """fleet_dispatch prints the 10 fleet tool names before launching.""" + _stub_guards(monkeypatch) + monkeypatch.chdir(tmp_path) + _stub_list_recipes(monkeypatch, []) + monkeypatch.setattr("autoskillit.cli.ui._timed_input.timed_prompt", lambda *a, **kw: "") + _capture_subprocess(monkeypatch) + _fleet_dispatch() + out = capsys.readouterr().out + assert "dispatch_food_truck" in out + assert "batch_cleanup_clones" in out + assert "list_recipes" in out + + +def test_fleet_dispatch_prints_version_header( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + """fleet_dispatch prints AUTOSKILLIT header line.""" + _stub_guards(monkeypatch) + monkeypatch.chdir(tmp_path) + _stub_list_recipes(monkeypatch, []) + monkeypatch.setattr("autoskillit.cli.ui._timed_input.timed_prompt", lambda *a, **kw: "") + _capture_subprocess(monkeypatch) + _fleet_dispatch() + out = capsys.readouterr().out + assert "AUTOSKILLIT" in out + assert "Fleet dispatcher" in out or "fleet dispatcher" in out.lower() + + +# --------------------------------------------------------------------------- +# T_GREETING: Greeting injection +# --------------------------------------------------------------------------- + + +def test_fleet_dispatch_passes_initial_message( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + """fleet_dispatch passes a non-None initial_message to _run_interactive_session.""" + _stub_guards(monkeypatch) + monkeypatch.chdir(tmp_path) + _stub_list_recipes( + monkeypatch, + [_fake_recipe("smoke-test", "BUILTIN", "Run smoke tests")], + ) + monkeypatch.setattr("autoskillit.cli.ui._timed_input.timed_prompt", lambda *a, **kw: "") + captured_kwargs: dict = {} + + def mock_run_session(system_prompt: str, **kwargs: object) -> None: + captured_kwargs.update(kwargs) + return None + + monkeypatch.setattr( + "autoskillit.cli.session._session_launch._run_interactive_session", + mock_run_session, + ) + _fleet_dispatch() + assert captured_kwargs.get("initial_message") is not None + assert "smoke-test" in captured_kwargs["initial_message"] + + +def test_fleet_dispatch_greeting_contains_recipe_descriptions( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + """The initial_message greeting includes recipe names and descriptions.""" + _stub_guards(monkeypatch) + monkeypatch.chdir(tmp_path) + _stub_list_recipes( + monkeypatch, + [ + _fake_recipe("review-pr", "BUILTIN", "Review a pull request"), + _fake_recipe("implement", "BUILTIN", "Implement a feature"), + ], + ) + monkeypatch.setattr("autoskillit.cli.ui._timed_input.timed_prompt", lambda *a, **kw: "") + captured_kwargs: dict = {} + + def mock_run_session(system_prompt: str, **kwargs: object) -> None: + captured_kwargs.update(kwargs) + return None + + monkeypatch.setattr( + "autoskillit.cli.session._session_launch._run_interactive_session", + mock_run_session, + ) + _fleet_dispatch() + greeting = captured_kwargs["initial_message"] + assert "review-pr" in greeting + assert "implement" in greeting + + +def test_fleet_dispatch_greetings_have_recipe_table_placeholder() -> None: + """All _FLEET_DISPATCH_GREETINGS entries must contain {recipe_table}.""" + from autoskillit.cli.fleet._fleet_preview import _FLEET_DISPATCH_GREETINGS + + assert len(_FLEET_DISPATCH_GREETINGS) >= 3 + for greeting in _FLEET_DISPATCH_GREETINGS: + assert "{recipe_table}" in greeting + + +# --------------------------------------------------------------------------- +# T_PROMPT: recipe_table injection into system prompt +# --------------------------------------------------------------------------- + + +def test_build_fleet_dispatch_prompt_embeds_recipe_table() -> None: + """_build_fleet_dispatch_prompt embeds recipe_table under AVAILABLE FOOD TRUCKS section.""" + from autoskillit.cli._mcp_names import DIRECT_PREFIX + from autoskillit.cli._prompts import _build_fleet_dispatch_prompt + + table = "smoke-test — Run smoke tests\nreview-pr — Review a PR" + prompt = _build_fleet_dispatch_prompt(DIRECT_PREFIX, recipe_table=table) + assert "AVAILABLE FOOD TRUCKS" in prompt + assert "smoke-test" in prompt + assert "review-pr" in prompt + + +def test_build_fleet_dispatch_prompt_no_recipe_table_section_when_none() -> None: + """_build_fleet_dispatch_prompt omits AVAILABLE FOOD TRUCKS when recipe_table is None.""" + from autoskillit.cli._mcp_names import DIRECT_PREFIX + from autoskillit.cli._prompts import _build_fleet_dispatch_prompt + + prompt = _build_fleet_dispatch_prompt(DIRECT_PREFIX) + assert "AVAILABLE FOOD TRUCKS" not in prompt + + +# --------------------------------------------------------------------------- +# T_SESSION: _launch_fleet_session initial_message forwarding +# --------------------------------------------------------------------------- + + +def test_launch_fleet_session_forwards_initial_message_dispatch( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + """_launch_fleet_session passes initial_message to _run_interactive_session.""" + monkeypatch.chdir(tmp_path) + captured_kwargs: dict = {} + + def mock_run(system_prompt: str, **kwargs: object) -> None: + captured_kwargs.update(kwargs) + return None + + monkeypatch.setattr( + "autoskillit.cli.session._session_launch._run_interactive_session", + mock_run, + ) + from autoskillit.cli.fleet._fleet_session import _launch_fleet_session + + _launch_fleet_session( + None, + None, + None, + None, + fleet_mode="dispatch", + initial_message="Hello, dispatcher!", + ) + assert captured_kwargs.get("initial_message") == "Hello, dispatcher!" + + +def test_launch_fleet_session_clears_initial_message_on_reload( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + """On reload, initial_message must be None (only injected on first launch).""" + monkeypatch.chdir(tmp_path) + call_count = 0 + captured_messages: list = [] + + def mock_run(system_prompt: str, **kwargs: object) -> object: + nonlocal call_count + captured_messages.append(kwargs.get("initial_message")) + call_count += 1 + return "reload-session-abc" if call_count == 1 else None + + monkeypatch.setattr( + "autoskillit.cli.session._session_launch._run_interactive_session", + mock_run, + ) + from autoskillit.cli.fleet._fleet_session import _launch_fleet_session + + _launch_fleet_session( + None, + None, + None, + None, + fleet_mode="dispatch", + initial_message="Hello!", + ) + assert len(captured_messages) >= 2, ( + f"expected reload to fire but got only {len(captured_messages)} call(s)" + ) + assert captured_messages[0] == "Hello!" + assert captured_messages[1] is None diff --git a/tests/cli/test_fleet_list.py b/tests/cli/test_fleet_list.py new file mode 100644 index 0000000000..cf6d2af41d --- /dev/null +++ b/tests/cli/test_fleet_list.py @@ -0,0 +1,103 @@ +"""Tests: fleet CLI list command.""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from autoskillit.cli.fleet import fleet_list as _fleet_list +from tests.cli._fleet_helpers import ( + _find_command, + _get_app, + _setup_campaign_recipes, + _subcommand_names, +) + +pytestmark = [pytest.mark.layer("cli"), pytest.mark.medium, pytest.mark.feature("fleet")] + + +def test_fleet_list_no_campaigns( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path, capsys: pytest.CaptureFixture +) -> None: + """fleet list prints 'No campaigns found' when directory is empty and no builtins exist.""" + import autoskillit.recipe.io as _rio + + monkeypatch.chdir(tmp_path) + monkeypatch.setattr(_rio, "pkg_root", lambda: tmp_path) + _fleet_list() + assert "No campaigns found" in capsys.readouterr().out + + +def test_fleet_list_shows_campaigns( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path, capsys: pytest.CaptureFixture +) -> None: + """fleet list shows tabular campaign listing.""" + _setup_campaign_recipes(tmp_path, ["alpha", "beta"]) + monkeypatch.chdir(tmp_path) + _fleet_list() + output = capsys.readouterr().out + assert "alpha" in output + assert "beta" in output + + +def test_fleet_list_exits_when_disabled(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: + """fleet_list exits 1 when fleet feature is disabled via feature gate.""" + monkeypatch.chdir(tmp_path) + checked_features: list[str] = [] + monkeypatch.setattr( + "autoskillit.cli.fleet.is_feature_enabled", + lambda name, features, *, experimental_enabled=False: ( + checked_features.append(name) or False + ), + ) + monkeypatch.setattr( + "autoskillit.config.load_config", + lambda path: type("C", (), {"features": {}, "experimental_enabled": False})(), + ) + with pytest.raises(SystemExit) as exc_info: + _fleet_list() + assert exc_info.value.code == 1 + assert "fleet" in checked_features + + +# --------------------------------------------------------------------------- +# CLI registration tests (fleet surface, not list-specific) +# --------------------------------------------------------------------------- + + +class TestFleetCLIRegistration: + def test_fleet_subcommand_registered(self) -> None: + app = _get_app() + names = _subcommand_names(app) + assert "fleet" in names + + def test_fleet_status_accepts_reap_flag(self) -> None: + from autoskillit.cli.fleet import fleet_app + + status_cmd = _find_command(fleet_app, "status") + assert status_cmd is not None, "fleet status command not found" + + def test_fleet_status_accepts_dry_run_flag(self) -> None: + import inspect + + from autoskillit.cli.fleet import fleet_status + + sig = inspect.signature(fleet_status) + assert "dry_run" in sig.parameters + assert "reap" in sig.parameters + + def test_fleet_dispatch_command_registered(self) -> None: + from autoskillit.cli.fleet import fleet_app + + assert "dispatch" in _subcommand_names(fleet_app) + + def test_fleet_campaign_command_registered(self) -> None: + from autoskillit.cli.fleet import fleet_app + + assert "campaign" in _subcommand_names(fleet_app) + + def test_fleet_run_command_not_registered(self) -> None: + from autoskillit.cli.fleet import fleet_app + + assert "run" not in _subcommand_names(fleet_app) diff --git a/tests/cli/test_fleet_session.py b/tests/cli/test_fleet_session.py new file mode 100644 index 0000000000..a3e2df47be --- /dev/null +++ b/tests/cli/test_fleet_session.py @@ -0,0 +1,465 @@ +"""Tests: _launch_fleet_session forwards ingredients_table to prompt builder.""" + +from __future__ import annotations + +from pathlib import Path +from unittest.mock import MagicMock + +import pytest + +pytestmark = [pytest.mark.layer("cli"), pytest.mark.small, pytest.mark.feature("fleet")] + + +def _make_campaign_recipe(name: str = "test-campaign") -> MagicMock: + recipe = MagicMock() + recipe.name = name + recipe.dispatches = [] + recipe.continue_on_failure = False + recipe.description = f"Test {name}" + return recipe + + +class TestLaunchFleetSessionIngredientsTable: + def _call( + self, + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, + ingredients_table: str | None = None, + ) -> dict: + captured: dict = {} + + def _fake_build( + campaign_recipe: object, + manifest_yaml: str, + completed_dispatches: str, + mcp_prefix: str, + campaign_id: str, + **kwargs: object, + ) -> str: + captured["ingredients_table"] = kwargs.get("ingredients_table") + return "fake-prompt" + + monkeypatch.setattr("autoskillit.cli._prompts._build_fleet_campaign_prompt", _fake_build) + monkeypatch.setattr( + "autoskillit.cli.session._session_launch._run_interactive_session", + lambda *a, **kw: None, + ) + monkeypatch.chdir(tmp_path) + (tmp_path / ".autoskillit" / "temp" / "fleet" / "test-id").mkdir(parents=True) + + state_path = tmp_path / ".autoskillit" / "temp" / "fleet" / "test-id" / "state.json" + state_path.write_text("{}") + + from autoskillit.cli.fleet._fleet_session import _launch_fleet_session + + _launch_fleet_session( + _make_campaign_recipe(), + "test-id", + state_path, + None, + fleet_mode="campaign", + ingredients_table=ingredients_table, + ) + return captured + + def test_ingredients_table_forwarded_to_prompt_builder( + self, monkeypatch: pytest.MonkeyPatch, tmp_path: Path + ) -> None: + table = "| Name | Desc |\n| task | do it |" + captured = self._call(monkeypatch, tmp_path, ingredients_table=table) + assert captured["ingredients_table"] == table + + def test_ingredients_table_none_by_default( + self, monkeypatch: pytest.MonkeyPatch, tmp_path: Path + ) -> None: + captured = self._call(monkeypatch, tmp_path) + assert captured["ingredients_table"] is None + + +class TestLaunchFleetSessionContinueOnFailureEnv: + def _capture_env( + self, monkeypatch: pytest.MonkeyPatch, tmp_path: Path, continue_on_failure: bool + ) -> dict: + captured: dict = {} + + def _fake_run(*args, **kwargs): + captured["extra_env"] = kwargs.get("extra_env", {}) + return None + + monkeypatch.setattr( + "autoskillit.cli.session._session_launch._run_interactive_session", _fake_run + ) + monkeypatch.setattr( + "autoskillit.cli._prompts._build_fleet_campaign_prompt", + lambda *a, **kw: "fake-prompt", + ) + monkeypatch.chdir(tmp_path) + + state_path = tmp_path / "state.json" + state_path.write_text("{}") + + recipe = _make_campaign_recipe() + recipe.continue_on_failure = continue_on_failure + + from autoskillit.cli.fleet._fleet_session import _launch_fleet_session + + _launch_fleet_session(recipe, "test-id", state_path, None, fleet_mode="campaign") + return captured + + def test_continue_on_failure_false_injects_env( + self, monkeypatch: pytest.MonkeyPatch, tmp_path: Path + ) -> None: + captured = self._capture_env(monkeypatch, tmp_path, continue_on_failure=False) + assert captured["extra_env"]["AUTOSKILLIT_CONTINUE_ON_FAILURE"] == "false" + + def test_continue_on_failure_true_injects_env( + self, monkeypatch: pytest.MonkeyPatch, tmp_path: Path + ) -> None: + captured = self._capture_env(monkeypatch, tmp_path, continue_on_failure=True) + assert captured["extra_env"]["AUTOSKILLIT_CONTINUE_ON_FAILURE"] == "true" + + +class TestReloadLoopRefreshesMetadata: + """T3a: Reload loop calls resume_campaign_from_state to refresh metadata.""" + + def _setup_common( + self, + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, + ) -> Path: + monkeypatch.chdir(tmp_path) + state_path = tmp_path / "state.json" + return state_path + + def test_resume_called_on_reload( + self, monkeypatch: pytest.MonkeyPatch, tmp_path: Path + ) -> None: + """resume_campaign_from_state is called exactly once (in reload iteration).""" + state_path = self._setup_common(monkeypatch, tmp_path) + + call_count = {"n": 0} + call_sequence = iter(["reload-id-1", None]) + + def _fake_run_session( + prompt: str, + *, + extra_env: dict, + resume_spec: object, + project_dir: Path, + **kwargs: object, + ) -> str | None: + return next(call_sequence) + + fresh_meta = MagicMock() + fresh_meta.completed_dispatches_block = "- A: success" + fresh_meta.next_dispatch_name = "B" + fresh_meta.is_resumable = False + + def _fake_resume(state_path_arg: Path, continue_on_failure: bool) -> MagicMock: + call_count["n"] += 1 + return fresh_meta + + monkeypatch.setattr( + "autoskillit.cli.session._session_launch._run_interactive_session", + _fake_run_session, + ) + monkeypatch.setattr( + "autoskillit.fleet.resume_campaign_from_state", + _fake_resume, + ) + monkeypatch.setattr( + "autoskillit.cli._prompts._build_fleet_campaign_prompt", + lambda *a, **kw: "fake-prompt", + ) + + from autoskillit.cli.fleet._fleet_session import _launch_fleet_session + + _launch_fleet_session( + _make_campaign_recipe(), + "test-id", + state_path, + None, + fleet_mode="campaign", + ) + + assert call_count["n"] == 1 + + def test_prompt_rebuilt_with_fresh_dispatches( + self, monkeypatch: pytest.MonkeyPatch, tmp_path: Path + ) -> None: + """Second prompt build uses fresh completed_dispatches from resume_campaign_from_state.""" + state_path = self._setup_common(monkeypatch, tmp_path) + + call_sequence = iter(["reload-id-1", None]) + + def _fake_run_session( + prompt: str, + *, + extra_env: dict, + resume_spec: object, + project_dir: Path, + **kwargs: object, + ) -> str | None: + return next(call_sequence) + + fresh_meta = MagicMock() + fresh_meta.completed_dispatches_block = "- A: success" + fresh_meta.next_dispatch_name = "B" + fresh_meta.is_resumable = False + + monkeypatch.setattr( + "autoskillit.fleet.resume_campaign_from_state", + lambda *a, **kw: fresh_meta, + ) + monkeypatch.setattr( + "autoskillit.cli.session._session_launch._run_interactive_session", + _fake_run_session, + ) + + prompt_calls: list[str] = [] + + def _fake_build( + campaign_recipe: object, + manifest_yaml: str, + completed_dispatches: str, + mcp_prefix: str, + campaign_id: str, + **kwargs: object, + ) -> str: + prompt_calls.append(completed_dispatches) + return "fake-prompt" + + monkeypatch.setattr( + "autoskillit.cli._prompts._build_fleet_campaign_prompt", + _fake_build, + ) + + from autoskillit.cli.fleet._fleet_session import _launch_fleet_session + + _launch_fleet_session( + _make_campaign_recipe(), + "test-id", + state_path, + None, + fleet_mode="campaign", + ) + + # First build uses resume_metadata=None -> "", second uses fresh_meta + assert len(prompt_calls) == 2 + assert prompt_calls[1] == "- A: success" + + +class TestReloadLoopSentinelGuard: + """T3b: FLEET_HALTED_SENTINEL in reload metadata breaks the loop cleanly.""" + + def test_sentinel_on_reload_breaks_loop( + self, monkeypatch: pytest.MonkeyPatch, tmp_path: Path + ) -> None: + """When resume_campaign_from_state returns FLEET_HALTED_SENTINEL, loop exits normally.""" + from autoskillit.fleet import FLEET_HALTED_SENTINEL + + monkeypatch.chdir(tmp_path) + state_path = tmp_path / "state.json" + + session_call_count = {"n": 0} + + def _fake_run_session( + prompt: str, + *, + extra_env: dict, + resume_spec: object, + project_dir: Path, + **kwargs: object, + ) -> str | None: + session_call_count["n"] += 1 + return "reload-id-sentinel" + + halted_meta = MagicMock() + halted_meta.completed_dispatches_block = FLEET_HALTED_SENTINEL + halted_meta.next_dispatch_name = "" + halted_meta.is_resumable = False + + monkeypatch.setattr( + "autoskillit.fleet.resume_campaign_from_state", + lambda *a, **kw: halted_meta, + ) + monkeypatch.setattr( + "autoskillit.cli.session._session_launch._run_interactive_session", + _fake_run_session, + ) + monkeypatch.setattr( + "autoskillit.cli._prompts._build_fleet_campaign_prompt", + lambda *a, **kw: "fake-prompt", + ) + + from autoskillit.cli.fleet._fleet_session import _launch_fleet_session + + # Must not raise SystemExit + _launch_fleet_session( + _make_campaign_recipe(), + "test-id", + state_path, + None, + fleet_mode="campaign", + ) + + assert session_call_count["n"] == 1 + + +class TestReloadLoopSafetyGuards: + """T3c: Safety guards — max reload cap and duplicate reload_id detection.""" + + def test_max_reload_guard(self, monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: + """SystemExit raised after _MAX_RELOADS consecutive reloads.""" + monkeypatch.chdir(tmp_path) + state_path = tmp_path / "state.json" + + counter = {"n": 0} + + def _fake_run_session( + prompt: str, + *, + extra_env: dict, + resume_spec: object, + project_dir: Path, + **kwargs: object, + ) -> str: + counter["n"] += 1 + return f"unique-reload-id-{counter['n']}" + + fresh_meta = MagicMock() + fresh_meta.completed_dispatches_block = "- A: success" + fresh_meta.next_dispatch_name = "B" + fresh_meta.is_resumable = False + + monkeypatch.setattr( + "autoskillit.fleet.resume_campaign_from_state", + lambda *a, **kw: fresh_meta, + ) + monkeypatch.setattr( + "autoskillit.cli.session._session_launch._run_interactive_session", + _fake_run_session, + ) + monkeypatch.setattr( + "autoskillit.cli._prompts._build_fleet_campaign_prompt", + lambda *a, **kw: "fake-prompt", + ) + + from autoskillit.cli.fleet._fleet_session import _launch_fleet_session + + with pytest.raises(SystemExit): + _launch_fleet_session( + _make_campaign_recipe(), + "test-id", + state_path, + None, + fleet_mode="campaign", + ) + + from autoskillit.cli.fleet._fleet_session import _MAX_RELOADS + + assert counter["n"] == _MAX_RELOADS + 1 + + def test_duplicate_reload_id_aborts( + self, monkeypatch: pytest.MonkeyPatch, tmp_path: Path + ) -> None: + """SystemExit raised when the same reload_id is returned twice.""" + monkeypatch.chdir(tmp_path) + state_path = tmp_path / "state.json" + + # Always return the same reload_id + def _fake_run_session( + prompt: str, + *, + extra_env: dict, + resume_spec: object, + project_dir: Path, + **kwargs: object, + ) -> str: + return "repeated-reload-id" + + fresh_meta = MagicMock() + fresh_meta.completed_dispatches_block = "- A: success" + fresh_meta.next_dispatch_name = "B" + fresh_meta.is_resumable = False + + monkeypatch.setattr( + "autoskillit.fleet.resume_campaign_from_state", + lambda *a, **kw: fresh_meta, + ) + monkeypatch.setattr( + "autoskillit.cli.session._session_launch._run_interactive_session", + _fake_run_session, + ) + monkeypatch.setattr( + "autoskillit.cli._prompts._build_fleet_campaign_prompt", + lambda *a, **kw: "fake-prompt", + ) + + from autoskillit.cli.fleet._fleet_session import _launch_fleet_session + + with pytest.raises(SystemExit): + _launch_fleet_session( + _make_campaign_recipe(), + "test-id", + state_path, + None, + fleet_mode="campaign", + ) + + +class TestReloadLoopUsesNamedResume: + """T3d: Reload loop passes NoResume on first call, NamedResume on reload.""" + + def test_named_resume_on_reload(self, monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: + """First call uses NoResume(); after reload, NamedResume is used.""" + from autoskillit.core import NamedResume, NoResume + + monkeypatch.chdir(tmp_path) + state_path = tmp_path / "state.json" + + captured_specs: list[object] = [] + call_sequence = iter(["reload-id-abc", None]) + + def _fake_run_session( + prompt: str, + *, + extra_env: dict, + resume_spec: object, + project_dir: Path, + **kwargs: object, + ) -> str | None: + captured_specs.append(resume_spec) + return next(call_sequence) + + fresh_meta = MagicMock() + fresh_meta.completed_dispatches_block = "- A: success" + fresh_meta.next_dispatch_name = "B" + fresh_meta.is_resumable = False + + monkeypatch.setattr( + "autoskillit.fleet.resume_campaign_from_state", + lambda *a, **kw: fresh_meta, + ) + monkeypatch.setattr( + "autoskillit.cli.session._session_launch._run_interactive_session", + _fake_run_session, + ) + monkeypatch.setattr( + "autoskillit.cli._prompts._build_fleet_campaign_prompt", + lambda *a, **kw: "fake-prompt", + ) + + from autoskillit.cli.fleet._fleet_session import _launch_fleet_session + + _launch_fleet_session( + _make_campaign_recipe(), + "test-id", + state_path, + None, + fleet_mode="campaign", + ) + + assert len(captured_specs) == 2 + assert captured_specs[0] == NoResume() + assert captured_specs[1] == NamedResume(session_id="reload-id-abc") diff --git a/tests/cli/test_fleet_split.py b/tests/cli/test_fleet_split.py new file mode 100644 index 0000000000..88bf69c852 --- /dev/null +++ b/tests/cli/test_fleet_split.py @@ -0,0 +1,24 @@ +from __future__ import annotations + +import pytest + +pytestmark = [pytest.mark.layer("cli"), pytest.mark.small] + + +def test_fleet_display_importable(): + from autoskillit.cli.fleet._fleet_display import _build_status_rows, _humanize # noqa: F401 + + +def test_fleet_lifecycle_importable(): + from autoskillit.cli.fleet._fleet_lifecycle import ( # noqa: F401 + _fleet_signal_guard, + _reap_stale_dispatches, + ) + + +def test_fleet_session_importable(): + from autoskillit.cli.fleet._fleet_session import _launch_fleet_session # noqa: F401 + + +def test_fleet_facade_exports_fleet_app(): + from autoskillit.cli.fleet import fleet_app, render_fleet_error # noqa: F401 diff --git a/tests/cli/test_fleet_status.py b/tests/cli/test_fleet_status.py new file mode 100644 index 0000000000..643e66cdb2 --- /dev/null +++ b/tests/cli/test_fleet_status.py @@ -0,0 +1,291 @@ +"""Tests: fleet CLI status command.""" + +from __future__ import annotations + +import io +import json +from pathlib import Path + +import pytest + +from autoskillit.cli.fleet import fleet_status as _fleet_status +from tests.cli._fleet_helpers import ( + DispatchDescriptor, + _make_state, + _make_state_with_tokens, + _mock_batch_delete, + _setup_campaign_with_sessions, + _setup_campaign_with_status, + _setup_campaign_with_tokens, + _setup_existing_campaign_state, +) + +pytestmark = [pytest.mark.layer("cli"), pytest.mark.medium, pytest.mark.feature("fleet")] + + +def test_fleet_status_shows_dispatches( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path, capsys: pytest.CaptureFixture +) -> None: + """fleet status shows dispatch table from state.json.""" + _setup_existing_campaign_state(tmp_path, "abc123", "test-campaign") + monkeypatch.chdir(tmp_path) + with pytest.raises(SystemExit): + _fleet_status("abc123") + output = capsys.readouterr().out + assert "test-campaign" in output + + +def test_fleet_status_json_flag( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path, capsys: pytest.CaptureFixture +) -> None: + """--json produces valid JSON output with campaign_id.""" + _setup_existing_campaign_state(tmp_path, "abc123", "test-campaign") + monkeypatch.chdir(tmp_path) + with pytest.raises(SystemExit): + _fleet_status("abc123", json_output=True) + output = capsys.readouterr().out + data = json.loads(output) + assert data["campaign_id"] == "abc123" + + +def test_fleet_status_cleanup_calls_batch_delete( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + """--cleanup invokes clone_registry.batch_delete with owner filter.""" + _setup_existing_campaign_state(tmp_path, "abc123", "test-campaign") + monkeypatch.chdir(tmp_path) + mock_delete = _mock_batch_delete(monkeypatch) + with pytest.raises(SystemExit): + _fleet_status("abc123", cleanup=True) + assert mock_delete.called + assert mock_delete.call_args.args[0] == "" + assert mock_delete.call_args.kwargs.get("owner") == "abc123" + + +def test_fleet_status_missing_campaign( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path, capsys: pytest.CaptureFixture +) -> None: + """fleet status exits 3 for unknown campaign_id.""" + monkeypatch.chdir(tmp_path) + with pytest.raises(SystemExit, match="3"): + _fleet_status("nonexistent") + + +def test_fleet_status_table_columns( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path, capsys: pytest.CaptureFixture +) -> None: + """fleet status output contains all 8 header labels in order.""" + _setup_campaign_with_tokens(tmp_path, "cid01", "my-campaign") + monkeypatch.chdir(tmp_path) + with pytest.raises(SystemExit): + _fleet_status("cid01") + out = capsys.readouterr().out + assert out.index("NAME") < out.index("STATUS") < out.index("ELAPSED") + assert out.index("INPUT") < out.index("OUTPUT") < out.index("CACHE_RD") + assert "SESSION_LOG" in out + assert "CACHE_WR" in out + + +def test_status_numeric_columns_right_aligned() -> None: + """Numeric column definitions use align='>'.""" + from autoskillit.cli.fleet._fleet_display import _STATUS_COLUMNS + + numeric_labels = {"ELAPSED", "INPUT", "OUTPUT", "CACHE_RD", "CACHE_WR"} + for col in _STATUS_COLUMNS: + if col.label in numeric_labels: + assert col.align == ">", f"{col.label} should be right-aligned" + + +def test_fleet_status_json_includes_totals( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path, capsys: pytest.CaptureFixture +) -> None: + """--json output has totals key with summed token counts.""" + _setup_campaign_with_tokens( + tmp_path, + "cid01", + "test", + token_usage={ + "input_tokens": 100, + "output_tokens": 50, + "cache_creation_input_tokens": 10, + "cache_read_input_tokens": 20, + }, + ) + monkeypatch.chdir(tmp_path) + with pytest.raises(SystemExit): + _fleet_status("cid01", json_output=True) + data = json.loads(capsys.readouterr().out) + assert data["totals"]["input_tokens"] == 100 + assert data["totals"]["output_tokens"] == 50 + assert data["totals"]["cache_read"] == 20 + assert data["totals"]["cache_creation"] == 10 + + +def test_fleet_status_json_no_ansi( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path, capsys: pytest.CaptureFixture +) -> None: + """--json output must not contain ANSI escape sequences.""" + _setup_existing_campaign_state(tmp_path, "cid01", "test") + monkeypatch.chdir(tmp_path) + with pytest.raises(SystemExit): + _fleet_status("cid01", json_output=True) + out = capsys.readouterr().out + assert "\x1b[" not in out + + +def test_exit_code_all_success() -> None: + """_compute_exit_code returns 0 when all dispatches succeed or skip.""" + from autoskillit.cli.fleet import _compute_exit_code + + state = _make_state(statuses=["success", "skipped", "success"]) + assert _compute_exit_code(state) == 0 + + +@pytest.mark.parametrize("bad_status", ["failure", "interrupted", "refused", "released"]) +def test_exit_code_any_failure(bad_status: str) -> None: + """_compute_exit_code returns 1 when any dispatch is in a failure-class status.""" + from autoskillit.cli.fleet import _compute_exit_code + + state = _make_state(statuses=["success", bad_status]) + assert _compute_exit_code(state) == 1 + + +@pytest.mark.parametrize("progress_status", ["running", "pending"]) +def test_exit_code_in_progress(progress_status: str) -> None: + """_compute_exit_code returns 2 when any dispatch is in-progress and none failed.""" + from autoskillit.cli.fleet import _compute_exit_code + + state = _make_state(statuses=["success", progress_status]) + assert _compute_exit_code(state) == 2 + + +def test_exit_code_3_on_missing_state(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: + """fleet status exits 3 when state file is absent.""" + monkeypatch.chdir(tmp_path) + (tmp_path / ".autoskillit" / "temp" / "fleet" / "cid01").mkdir(parents=True) + with pytest.raises(SystemExit) as exc_info: + _fleet_status("cid01") + assert exc_info.value.code == 3 + + +def test_cross_check_warns_on_divergence( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path, capsys: pytest.CaptureFixture +) -> None: + """_cross_check_tokens emits a stderr warning when divergence exceeds 5%.""" + from autoskillit.cli.fleet import _aggregate_totals, _cross_check_tokens + + (tmp_path / "sessions.jsonl").write_text("") + state = _make_state_with_tokens(input_total=10000) + state_totals = _aggregate_totals(state) + + monkeypatch.setattr("autoskillit.execution.resolve_log_dir", lambda *a: tmp_path) + monkeypatch.setattr( + "autoskillit.pipeline.tokens.DefaultTokenLog.load_from_log_dir", + lambda self, *a, **kw: 1, + ) + monkeypatch.setattr( + "autoskillit.pipeline.tokens.DefaultTokenLog.compute_total", + lambda self, **kw: { + "input_tokens": 8000, + "output_tokens": 0, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + "total_elapsed_seconds": 0.0, + }, + ) + _cross_check_tokens(state, state_totals) + assert "diverge" in capsys.readouterr().err.lower() + + +def test_watch_exits_on_terminal_campaign(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: + """--watch detects all-terminal campaign and exits with code 0.""" + _setup_campaign_with_status(tmp_path, "cid01", "test", status="success") + monkeypatch.chdir(tmp_path) + monkeypatch.setattr("sys.stdin", io.StringIO("")) + with pytest.raises(SystemExit) as exc_info: + _fleet_status("cid01", watch=True) + assert exc_info.value.code == 0 + + +def test_cleanup_per_dispatch_session(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: + """--cleanup calls cleanup_session for each dispatch dispatched_session_id.""" + _setup_campaign_with_sessions( + tmp_path, + "cid01", + "test", + dispatches=[ + DispatchDescriptor("d1", "success", "sid-1"), + DispatchDescriptor("d2", "failure", "sid-2"), + ], + ) + monkeypatch.chdir(tmp_path) + _mock_batch_delete(monkeypatch) + cleanup_calls: list[str] = [] + monkeypatch.setattr( + "autoskillit.workspace.DefaultSessionSkillManager.cleanup_session", + lambda self, sid: cleanup_calls.append(sid) or True, + ) + with pytest.raises(SystemExit): + _fleet_status("cid01", cleanup=True) + assert "sid-1" in cleanup_calls + assert "sid-2" in cleanup_calls + + +def test_status_table_no_ansi_when_no_color( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path, capsys: pytest.CaptureFixture +) -> None: + """fleet status table contains no ANSI codes when NO_COLOR is set.""" + _setup_existing_campaign_state(tmp_path, "cid01", "test") + monkeypatch.chdir(tmp_path) + monkeypatch.setenv("NO_COLOR", "1") + with pytest.raises(SystemExit): + _fleet_status("cid01") + out = capsys.readouterr().out + assert "\x1b[" not in out + + +def test_exit_code_failure_over_in_progress() -> None: + """_compute_exit_code returns 1 (failure) when mixed with in-progress.""" + from autoskillit.cli.fleet import _compute_exit_code + + state = _make_state(statuses=["failure", "running"]) + assert _compute_exit_code(state) == 1 + + +def test_status_table_shows_totals_row( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path, capsys: pytest.CaptureFixture +) -> None: + """fleet status table includes a TOTAL row.""" + _setup_campaign_with_tokens( + tmp_path, + "cid01", + "test", + token_usage={"input_tokens": 100, "output_tokens": 50}, + ) + monkeypatch.chdir(tmp_path) + with pytest.raises(SystemExit): + _fleet_status("cid01") + out = capsys.readouterr().out + assert "TOTAL" in out.upper() + assert "100" in out + assert "50" in out + + +def test_fleet_status_exits_when_disabled(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: + """fleet_status exits 1 when fleet feature is disabled via feature gate.""" + monkeypatch.chdir(tmp_path) + checked_features: list[str] = [] + monkeypatch.setattr( + "autoskillit.cli.fleet.is_feature_enabled", + lambda name, features, *, experimental_enabled=False: ( + checked_features.append(name) or False + ), + ) + monkeypatch.setattr( + "autoskillit.config.load_config", + lambda path: type("C", (), {"features": {}, "experimental_enabled": False})(), + ) + with pytest.raises(SystemExit) as exc_info: + _fleet_status(None) + assert exc_info.value.code == 1 + assert "fleet" in checked_features diff --git a/tests/cli/test_food_truck_prompt.py b/tests/cli/test_food_truck_prompt.py new file mode 100644 index 0000000000..d34b940621 --- /dev/null +++ b/tests/cli/test_food_truck_prompt.py @@ -0,0 +1,243 @@ +"""Group E tests: L3 food truck prompt builder — sous-chef subset, inversions, +budget guidance, quota awareness, sentinel format.""" + +from __future__ import annotations + +import json +import pathlib +import re + +import pytest + +pytestmark = [pytest.mark.layer("cli"), pytest.mark.small, pytest.mark.feature("fleet")] + +# --- Fixtures --- + +_RECIPE = "test-recipe" +_TASK = "implement feature X" +_INGREDIENTS = {"branch": "main", "issue_url": "https://github.com/org/repo/issues/1"} +_MCP_PREFIX = "mcp__autoskillit__" +_DISPATCH_ID = "abc12345deadbeef" +_DISPATCH_ID_SHORT = "abc12345" +_CAMPAIGN_ID = "camp-001" +_L3_TIMEOUT = 3600 + + +def _get_prompt() -> str: + from autoskillit.fleet._prompts import _build_food_truck_prompt + + return _build_food_truck_prompt( + recipe=_RECIPE, + task=_TASK, + ingredients=_INGREDIENTS, + mcp_prefix=_MCP_PREFIX, + dispatch_id=_DISPATCH_ID, + campaign_id=_CAMPAIGN_ID, + l3_timeout_sec=_L3_TIMEOUT, + ) + + +def _get_admiral_block() -> str: + from autoskillit.fleet._prompts import _build_admiral_dispatch_block + + return _build_admiral_dispatch_block() + + +# --- Group E-1: Sous-Chef Subset Filter --- + + +class TestAdmiralDispatchBlock: + def test_retains_exactly_four_sections(self) -> None: + block = _get_admiral_block() + for title in ( + "CONTEXT LIMIT ROUTING", + "STEP NAME IMMUTABILITY", + "MERGE PHASE", + "QUOTA WAIT PROTOCOL", + ): + assert title in block, f"Missing retained section: {title}" + assert len(re.findall(r"^## ", block, re.MULTILINE)) == 4 + + def test_excludes_five_sections(self) -> None: + block = _get_admiral_block() + for title in ( + "MULTI-PART PLAN SEQUENCING", + "AUDIT-IMPL ACROSS MULTI-GROUP", + "READING AND ACTING ON", + "MULTIPLE ISSUES", + "PARALLEL STEP SCHEDULING", + ): + assert title not in block, f"Excluded section present: {title}" + + def test_no_dangling_crossrefs(self) -> None: + block = _get_admiral_block() + for phrase in ( + "MULTI-PART PLAN", + "AUDIT-IMPL", + "plan_parts=", + "MULTIPLE ISSUES", + "PARALLEL STEP", + "wavefront", + ): + assert phrase not in block, f"Dangling crossref: {phrase}" + + def test_graceful_degradation( + self, monkeypatch: pytest.MonkeyPatch, tmp_path: pathlib.Path + ) -> None: + from autoskillit.fleet import _prompts as _fleet_prompts + + monkeypatch.setattr(_fleet_prompts, "pkg_root", lambda: tmp_path) + result = _fleet_prompts._build_admiral_dispatch_block() + assert result == "" + + def test_excludes_unretained_sections(self) -> None: + block = _get_admiral_block() + for title in ( + "SKILL_COMMAND FORMATTING", + "STEP EXECUTION IS NOT DISCRETIONARY", + "NARRATION SUPPRESSION", + ): + assert f"## {title}" not in block, f"Unretained section present: {title}" + + def test_l3_sections_constant_matches_build_output(self) -> None: + from autoskillit.core.types._type_constants import ( + ADMIRAL_DISPATCH_SECTIONS, + SOUS_CHEF_MANDATORY_SECTIONS, + ) + + assert set(ADMIRAL_DISPATCH_SECTIONS).issubset(set(SOUS_CHEF_MANDATORY_SECTIONS)) + block = _get_admiral_block() + for header in ADMIRAL_DISPATCH_SECTIONS: + assert f"## {header}" in block, ( + f"_build_admiral_dispatch_block() missing dispatch section: {header!r}. " + "Update ADMIRAL_DISPATCH_SECTIONS or the allowlist in fleet/_prompts.py." + ) + extra = set(SOUS_CHEF_MANDATORY_SECTIONS) - set(ADMIRAL_DISPATCH_SECTIONS) + for header in extra: + assert f"## {header}" not in block, ( + f"_build_admiral_dispatch_block() unexpectedly includes " + f"non-dispatch section: {header!r}" + ) + + +# --- Group E-2: Placeholder Interpolation --- + + +class TestFoodTruckPromptPlaceholders: + def test_interpolates_all_placeholders(self) -> None: + prompt = _get_prompt() + assert _RECIPE in prompt + assert _TASK in prompt + assert _DISPATCH_ID in prompt + assert _DISPATCH_ID_SHORT in prompt + assert _CAMPAIGN_ID in prompt + assert str(_L3_TIMEOUT) in prompt + assert _MCP_PREFIX in prompt + assert json.dumps(_INGREDIENTS) in prompt + assert '"branch": "main"' in prompt + + +# --- Group E-3: Headless Mode Inversions --- + + +class TestHeadlessModeDirectives: + def test_h1_open_kitchen_with_overrides(self) -> None: + prompt = _get_prompt() + assert "open_kitchen" in prompt + assert "overrides=" in prompt + assert f"{_MCP_PREFIX}open_kitchen" in prompt + + def test_h2_no_ingredient_prompting(self) -> None: + prompt = _get_prompt() + assert "DO NOT prompt for ingredient" in prompt + assert "Collect ingredient values" not in prompt + + def test_h3_auto_accept_confirm(self) -> None: + prompt = _get_prompt() + assert "AUTO-ACCEPT CONFIRM STEPS" in prompt + assert "AskUserQuestion with the step's message" not in prompt + + def test_h4_ask_user_question_auto_accepts(self) -> None: + prompt = _get_prompt() + assert "AskUserQuestion" in prompt + assert "auto-accept" in prompt.lower() or "auto_accept" in prompt.lower() + + def test_h5_run_cmd_required(self) -> None: + prompt = _get_prompt() + assert "run_cmd" in prompt + + def test_h6_sentinel_only_output(self) -> None: + prompt = _get_prompt() + assert "SENTINEL-ONLY OUTPUT" in prompt + assert "Display it verbatim" not in prompt + + +# --- Group E-4: Section Content --- + + +class TestFoodTruckPromptSections: + def test_contains_routing_rules(self) -> None: + prompt = _get_prompt() + assert "ROUTING RULES" in prompt + assert "on_failure" in prompt + + def test_contains_failure_predicates(self) -> None: + prompt = _get_prompt() + assert "FAILURE PREDICATES" in prompt + for tool in ("test_check", "merge_worktree", "run_cmd", "run_skill"): + assert tool in prompt + + def test_contains_budget_guidance(self) -> None: + prompt = _get_prompt() + assert "BUDGET" in prompt + assert "120 seconds" in prompt + assert "900 seconds" in prompt + assert "1800 seconds" in prompt + + def test_contains_quota_awareness(self) -> None: + prompt = _get_prompt() + assert "quota_exhausted" in prompt + assert "wait_seconds" in prompt + + def test_contains_campaign_task_block(self) -> None: + prompt = _get_prompt() + assert "CAMPAIGN TASK" in prompt + assert _RECIPE in prompt + assert _TASK in prompt + + def test_contains_ingredient_values_block(self) -> None: + prompt = _get_prompt() + assert "INGREDIENT VALUES" in prompt + assert '"branch"' in prompt + + +# --- Group E-5: Sentinel Format --- + + +class TestSentinelFormat: + def test_sentinel_format_dispatch_id(self) -> None: + prompt = _get_prompt() + assert f"---l3-result::{_DISPATCH_ID}---" in prompt + assert f"---end-l3-result::{_DISPATCH_ID}---" in prompt + + def test_sentinel_done_marker(self) -> None: + prompt = _get_prompt() + assert f"%%L3_DONE::{_DISPATCH_ID_SHORT}%%" in prompt + + def test_sentinel_json_body_contract(self) -> None: + prompt = _get_prompt() + assert '"success"' in prompt + assert '"reason"' in prompt + + +# --- Group E-6: No First-Action Bootstrap --- + + +class TestNoBootstrapSequence: + def test_no_bash_sleep_step(self) -> None: + prompt = _get_prompt() + assert 'Bash(command="sleep 2")' not in prompt + + def test_no_toolsearch_bootstrap(self) -> None: + prompt = _get_prompt() + assert "ToolSearch(query='select:" not in prompt diff --git a/tests/cli/test_init.py b/tests/cli/test_init.py index b59b6e4cd8..6ca3d5d9f8 100644 --- a/tests/cli/test_init.py +++ b/tests/cli/test_init.py @@ -12,6 +12,8 @@ from autoskillit import cli from autoskillit.cli import _generate_config_yaml +pytestmark = [pytest.mark.layer("cli"), pytest.mark.small] + class TestCLIInit: @pytest.fixture(autouse=True) @@ -236,7 +238,10 @@ def test_init_registers_hooks_in_settings_json( ) for hdef in HOOK_REGISTRY: for script in hdef.scripts: - assert script in registered, f"Expected hook script {script!r} to be registered" + logical_name = script.removesuffix(".py") + assert logical_name in registered, ( + f"Expected hook script {script!r} to be registered" + ) # CI-SCOPE-3 def test_init_idempotent_no_duplicates( @@ -803,6 +808,27 @@ def test_init_from_pkg_root_like_cwd_refuses( _register_all(scope="user", project_dir=fake_pkg_root) +# T-WT-2: _register_all rejects worktree pkg_root +def test_register_all_rejects_worktree_pkg_root( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """_register_all must refuse when pkg_root() resolves to a git worktree.""" + import autoskillit.core.paths as _paths + from autoskillit.cli._init_helpers import _register_all + + fake_pkg = tmp_path / "worktree" / "src" / "autoskillit" + fake_pkg.mkdir(parents=True) + + monkeypatch.setattr(_paths, "pkg_root", lambda: fake_pkg) + monkeypatch.setattr(_paths, "is_git_worktree", lambda path: True) + + project_dir = tmp_path / "user_project" + project_dir.mkdir() + + with pytest.raises(Exception, match="worktree"): + _register_all(scope="user", project_dir=project_dir) + + # --- evict_direct_mcp_entry unit tests --- diff --git a/tests/cli/test_input_tty_contracts.py b/tests/cli/test_input_tty_contracts.py index ff583b7dc8..041471fef3 100644 --- a/tests/cli/test_input_tty_contracts.py +++ b/tests/cli/test_input_tty_contracts.py @@ -11,6 +11,8 @@ import pytest +pytestmark = [pytest.mark.layer("cli"), pytest.mark.small] + # Files that are *allowed* to contain raw input() calls. # _timed_input.py is the sole module that wraps input() with timeout/TTY/ANSI. _RAW_INPUT_EXEMPT_FILES: frozenset[str] = frozenset( @@ -127,7 +129,7 @@ def test_prompt_recipe_choice_noninteractive_exits( def test_cook_noninteractive_exits(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: """cook() launch-confirm prompt must raise SystemExit(1) when not interactive.""" - from autoskillit.cli._cook import cook + from autoskillit.cli.session._cook import cook monkeypatch.chdir(tmp_path) monkeypatch.setattr("sys.stdin.isatty", lambda: False) diff --git a/tests/cli/test_install.py b/tests/cli/test_install.py index e42a981d2e..2a02664a17 100644 --- a/tests/cli/test_install.py +++ b/tests/cli/test_install.py @@ -13,6 +13,8 @@ from autoskillit import cli +pytestmark = [pytest.mark.layer("cli"), pytest.mark.medium] + class TestCLIInstall: def test_install_validates_scope(self, capsys: pytest.CaptureFixture) -> None: @@ -375,8 +377,14 @@ def test_install_symlink_target_is_not_inside_git_worktree( """ from autoskillit.core.paths import is_git_worktree, pkg_root - if is_git_worktree(pkg_root()): - pytest.skip("Cannot verify non-worktree install from a worktree environment") + # Check filesystem directly — the cli conftest patches is_git_worktree + # to return False, so we cannot rely on it for the skip guard. + pkg = pkg_root() + for ancestor in [pkg, *pkg.parents]: + if (ancestor / ".git").is_file(): + pytest.skip("Cannot verify non-worktree install from a worktree environment") + if (ancestor / ".git").is_dir(): + break monkeypatch.setattr(Path, "home", lambda: tmp_path) from autoskillit.cli._marketplace import _ensure_marketplace @@ -443,7 +451,7 @@ def test_quota_hook_script_exists(self): import autoskillit pkg_dir = Path(autoskillit.__file__).parent - hook_script = pkg_dir / "hooks" / "quota_guard.py" + hook_script = pkg_dir / "hooks" / "guards" / "quota_guard.py" assert hook_script.exists(), f"Expected hook script at {hook_script}" def test_generate_hooks_json_includes_quota_hook(self): @@ -454,12 +462,12 @@ def test_generate_hooks_json_includes_quota_hook(self): pretooluse_commands = [ hook["command"] for entry in data["hooks"]["PreToolUse"] for hook in entry["hooks"] ] - assert any(cmd.endswith("quota_guard.py") for cmd in pretooluse_commands) + assert any("quota_guard" in cmd for cmd in pretooluse_commands) assert "PostToolUse" in data["hooks"] posttooluse_commands = [ hook["command"] for entry in data["hooks"]["PostToolUse"] for hook in entry["hooks"] ] - assert any(cmd.endswith("pretty_output_hook.py") for cmd in posttooluse_commands) + assert any("pretty_output_hook" in cmd for cmd in posttooluse_commands) def test_install_writes_pretooluse_hooks(self, tmp_path, monkeypatch): """install must register the quota PreToolUse hook in .claude/settings.json.""" @@ -497,7 +505,7 @@ def test_remove_clone_guard_script_exists(self): import autoskillit pkg_dir = Path(autoskillit.__file__).parent - hook_script = pkg_dir / "hooks" / "remove_clone_guard.py" + hook_script = pkg_dir / "hooks" / "guards" / "remove_clone_guard.py" assert hook_script.exists(), f"Expected hook script at {hook_script}" def test_install_registers_remove_clone_guard_hook(self, tmp_path, monkeypatch): @@ -555,10 +563,14 @@ def test_install_remove_clone_guard_hook_idempotent(self, tmp_path, monkeypatch) ) -def test_clear_plugin_cache_removes_nested_entry( +def test_clear_plugin_cache_preserves_plugins_entry( tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: - """_clear_plugin_cache must remove the entry from data['plugins'], not top-level data.""" + """_clear_plugin_cache retires old version dirs but preserves the plugins entry. + + The installed_plugins.json entry is left intact — `claude plugin install` + overwrites it atomically. Old version directories survive under a grace period. + """ from autoskillit.cli._marketplace import _clear_plugin_cache plugins_dir = tmp_path / ".claude" / "plugins" @@ -572,11 +584,28 @@ def test_clear_plugin_cache_removes_nested_entry( } ) ) + # Simulate an old version directory in the plugin cache + cache_dir = tmp_path / ".claude" / "plugins" / "cache" / "autoskillit-local" / "autoskillit" + old_version_dir = cache_dir / "0.9.0" + old_version_dir.mkdir(parents=True) + # Ensure the running __version__ differs from "0.9.0" so retirement applies + import autoskillit as _pkg + + monkeypatch.setattr(_pkg, "__version__", "0.9.99-test") + monkeypatch.setattr(Path, "home", lambda: tmp_path) _clear_plugin_cache() + data = json.loads(installed_json.read_text()) - assert "autoskillit@autoskillit-local" not in data.get("plugins", {}) - assert data["version"] == 2 # other keys preserved + assert "autoskillit@autoskillit-local" in data.get("plugins", {}) + assert data["version"] == 2 + # Old version dir survives under grace period + assert old_version_dir.exists(), "Old version dir must survive under grace period" + # Retiring registry must record the old version + retiring_json = tmp_path / ".autoskillit" / "retiring_cache.json" + assert retiring_json.exists(), "retiring_cache.json must be created" + retiring_data = json.loads(retiring_json.read_text()) + assert any(e["version"] == "0.9.0" for e in retiring_data["retiring"]) def test_clear_plugin_cache_noop_when_entry_absent( diff --git a/tests/cli/test_install_info.py b/tests/cli/test_install_info.py index 0ae8e9b18d..a7e73eb849 100644 --- a/tests/cli/test_install_info.py +++ b/tests/cli/test_install_info.py @@ -10,7 +10,7 @@ import pytest from autoskillit.cli._install_info import ( - _INSTALL_FROM_INTEGRATION, + _INSTALL_FROM_DEVELOP, InstallInfo, InstallType, comparison_branch, @@ -19,6 +19,8 @@ upgrade_command, ) +pytestmark = [pytest.mark.layer("cli"), pytest.mark.small] + # --------------------------------------------------------------------------- # Helpers # --------------------------------------------------------------------------- @@ -58,13 +60,13 @@ def test_detect_install_git_vcs_stable(monkeypatch: pytest.MonkeyPatch) -> None: assert info.editable_source is None -def test_detect_install_git_vcs_integration(monkeypatch: pytest.MonkeyPatch) -> None: +def test_detect_install_git_vcs_develop(monkeypatch: pytest.MonkeyPatch) -> None: payload = json.dumps( { "url": "https://github.com/TalonT-Org/AutoSkillit.git", "vcs_info": { "vcs": "git", - "requested_revision": "integration", + "requested_revision": "develop", "commit_id": "deadbeef0000", }, } @@ -75,7 +77,7 @@ def test_detect_install_git_vcs_integration(monkeypatch: pytest.MonkeyPatch) -> ) info = detect_install() assert info.install_type == InstallType.GIT_VCS - assert info.requested_revision == "integration" + assert info.requested_revision == "develop" assert info.commit_id == "deadbeef0000" @@ -187,15 +189,15 @@ def test_comparison_branch_unknown_type() -> None: assert comparison_branch(info) == "releases/latest" -def test_comparison_branch_integration() -> None: +def test_comparison_branch_develop() -> None: info = InstallInfo( install_type=InstallType.GIT_VCS, commit_id="abc123", - requested_revision="integration", + requested_revision="develop", url=None, editable_source=None, ) - assert comparison_branch(info) == "integration" + assert comparison_branch(info) == "develop" @pytest.mark.parametrize( @@ -243,7 +245,7 @@ def test_dismissal_window_seven_days( @pytest.mark.parametrize( "requested_revision,install_type,editable_source", [ - ("integration", InstallType.GIT_VCS, None), + ("develop", InstallType.GIT_VCS, None), (None, InstallType.LOCAL_EDITABLE, Path("/tmp/repo")), ], ) @@ -300,11 +302,11 @@ def test_upgrade_command_release_tag() -> None: assert upgrade_command(info) == ["uv", "tool", "upgrade", "autoskillit"] -def test_upgrade_command_integration() -> None: +def test_upgrade_command_develop() -> None: info = InstallInfo( install_type=InstallType.GIT_VCS, commit_id="abc123", - requested_revision="integration", + requested_revision="develop", url=None, editable_source=None, ) @@ -313,7 +315,7 @@ def test_upgrade_command_integration() -> None: "tool", "install", "--force", - _INSTALL_FROM_INTEGRATION, + _INSTALL_FROM_DEVELOP, ] @@ -344,3 +346,133 @@ def test_upgrade_command_unknown_and_local_path_returns_none( editable_source=None, ) assert upgrade_command(info) is None + + +# --------------------------------------------------------------------------- +# _is_release_tag — unit tests (1a) +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize("rev", ["v0.7.75", "0.7.75", "v1.0", "1.0.0.0"]) +def test_is_release_tag_positive(rev: str) -> None: + from autoskillit.core._install_detect import _is_release_tag + + assert _is_release_tag(rev) is True + + +@pytest.mark.parametrize("rev", ["develop", "main", "stable", "feature-foo", "v", "v0.7.75-rc1"]) +def test_is_release_tag_negative(rev: str) -> None: + from autoskillit.core._install_detect import _is_release_tag + + assert _is_release_tag(rev) is False + + +# --------------------------------------------------------------------------- +# _is_stable_track — unit tests (1b) +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize("rev", [None, "main", "stable", "v0.7.75", "0.9.300", ""]) +def test_is_stable_track_true(rev: str | None) -> None: + from autoskillit.core._install_detect import _is_stable_track + + assert _is_stable_track(rev) is True + + +@pytest.mark.parametrize( + "rev", + ["develop", "integration", "feature-foo", "staging", "release-candidate", "develops"], +) +def test_is_stable_track_false(rev: str) -> None: + from autoskillit.core._install_detect import _is_stable_track + + assert _is_stable_track(rev) is False + + +# --------------------------------------------------------------------------- +# classify_track — unit tests (1c) +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize( + "install_type,revision,expected_track", + [ + (InstallType.LOCAL_EDITABLE, "develop", "local"), + (InstallType.LOCAL_EDITABLE, None, "local"), + (InstallType.LOCAL_PATH, "stable", "local"), + (InstallType.LOCAL_PATH, None, "local"), + (InstallType.GIT_VCS, "stable", "stable"), + (InstallType.GIT_VCS, "main", "stable"), + (InstallType.GIT_VCS, "v0.7.75", "stable"), + (InstallType.GIT_VCS, None, "stable"), + (InstallType.GIT_VCS, "develop", "dev"), + (InstallType.GIT_VCS, "integration", "dev"), + (InstallType.GIT_VCS, "feature-foo", "dev"), + (InstallType.GIT_VCS, "staging", "dev"), + (InstallType.GIT_VCS, "", "stable"), + (InstallType.UNKNOWN, None, "stable"), + ], +) +def test_classify_track( + install_type: InstallType, revision: str | None, expected_track: str +) -> None: + from autoskillit.cli._install_info import InstallTrack, classify_track + + info = InstallInfo( + install_type=install_type, + commit_id=None, + requested_revision=revision, + url=None, + editable_source=Path("/tmp/repo") if install_type == InstallType.LOCAL_EDITABLE else None, + ) + assert classify_track(info) == InstallTrack(expected_track) + + +# --------------------------------------------------------------------------- +# Policy functions with arbitrary branch names (1d) +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize( + "revision", ["integration", "feature-foo", "staging", "release-candidate", "my-branch"] +) +def test_comparison_branch_arbitrary_dev_revision(revision: str) -> None: + info = InstallInfo( + install_type=InstallType.GIT_VCS, + commit_id="abc123", + requested_revision=revision, + url=None, + editable_source=None, + ) + assert comparison_branch(info) == "develop" + + +@pytest.mark.parametrize( + "revision", ["integration", "feature-foo", "staging", "release-candidate", "my-branch"] +) +def test_dismissal_window_arbitrary_dev_revision(revision: str) -> None: + info = InstallInfo( + install_type=InstallType.GIT_VCS, + commit_id="abc123", + requested_revision=revision, + url=None, + editable_source=None, + ) + assert dismissal_window(info) == timedelta(hours=12) + + +@pytest.mark.parametrize( + "revision", ["integration", "feature-foo", "staging", "release-candidate", "my-branch"] +) +def test_upgrade_command_arbitrary_dev_revision(revision: str) -> None: + info = InstallInfo( + install_type=InstallType.GIT_VCS, + commit_id="abc123", + requested_revision=revision, + url=None, + editable_source=None, + ) + result = upgrade_command(info) + assert result is not None + assert result[:3] == ["uv", "tool", "install"] + assert "--force" in result diff --git a/tests/cli/test_installed_plugins_file.py b/tests/cli/test_installed_plugins_file.py index f7278ef838..f91089f22c 100644 --- a/tests/cli/test_installed_plugins_file.py +++ b/tests/cli/test_installed_plugins_file.py @@ -5,8 +5,12 @@ import json from pathlib import Path +import pytest + from autoskillit.cli._installed_plugins import InstalledPluginsFile +pytestmark = [pytest.mark.layer("cli"), pytest.mark.small] + REAL_STRUCTURE = { "version": 2, "plugins": {"autoskillit@autoskillit-local": {"name": "autoskillit", "version": "0.8.30"}}, @@ -79,3 +83,16 @@ def test_remove_is_noop_when_file_absent(tmp_path: Path) -> None: p = tmp_path / "no_file.json" InstalledPluginsFile(p).remove("autoskillit@autoskillit-local") # should not raise assert not p.exists() + + +def test_installed_plugins_read_logs_on_json_error(tmp_path: Path) -> None: + """_read() emits a WARNING log when installed_plugins.json contains invalid JSON.""" + import structlog.testing + + p = tmp_path / "installed_plugins.json" + p.write_text("{invalid json}") + + with structlog.testing.capture_logs() as cap_logs: + result = InstalledPluginsFile(p).get_plugins() + assert result == {} + assert any("installed_plugins" in entry.get("event", "").lower() for entry in cap_logs) diff --git a/tests/cli/test_interactive_subprocess_contracts.py b/tests/cli/test_interactive_subprocess_contracts.py index 5056dd7eff..fa2e25b6cc 100644 --- a/tests/cli/test_interactive_subprocess_contracts.py +++ b/tests/cli/test_interactive_subprocess_contracts.py @@ -10,6 +10,8 @@ import pytest +pytestmark = [pytest.mark.layer("cli"), pytest.mark.small] + CLI_DIR = Path(__file__).parent.parent.parent / "src" / "autoskillit" / "cli" # Files that contain no subprocess.run calls — skip for clarity @@ -91,7 +93,7 @@ def test_terminal_guard_never_emits_smcup_on_entry() -> None: See: test_interactive_subprocess_calls_wrapped_in_terminal_guard for the analogous structural guard on subprocess call sites. """ - terminal_py = CLI_DIR / "_terminal.py" + terminal_py = CLI_DIR / "ui" / "_terminal.py" source = terminal_py.read_text() assert "?1049h" not in source, ( f"{terminal_py.name} must not emit \\033[?1049h (smcup). " @@ -102,7 +104,7 @@ def test_terminal_guard_never_emits_smcup_on_entry() -> None: ) -@pytest.mark.parametrize("py_file", sorted(CLI_DIR.glob("*.py"))) +@pytest.mark.parametrize("py_file", sorted(CLI_DIR.rglob("*.py"))) def test_interactive_subprocess_calls_wrapped_in_terminal_guard(py_file: Path) -> None: """Every non-capturing subprocess.run call in cli/ must be inside terminal_guard(). @@ -113,7 +115,7 @@ def test_interactive_subprocess_calls_wrapped_in_terminal_guard(py_file: Path) - If this test fails with your change: 1. You added a subprocess.run call in a CLI module without capture_output=True 2. Wrap it: `with terminal_guard(): result = subprocess.run(...)` - 3. Import: `from autoskillit.cli._terminal import terminal_guard` + 3. Import: `from autoskillit.cli.ui._terminal import terminal_guard` """ source = py_file.read_text() if "subprocess.run" not in source: @@ -124,6 +126,6 @@ def test_interactive_subprocess_calls_wrapped_in_terminal_guard(py_file: Path) - f"\n\n{py_file.name}: interactive subprocess.run found at line(s) " f"{violations} without terminal_guard() wrapper.\n\n" f"Fix: wrap with `with terminal_guard():` and import from " - f"`autoskillit.cli._terminal`.\n\n" + f"`autoskillit.cli.ui._terminal`.\n\n" f"See: tests/cli/test_input_tty_contracts.py for the analogous pattern." ) diff --git a/tests/cli/test_l3_orchestrator_prompt.py b/tests/cli/test_l3_orchestrator_prompt.py new file mode 100644 index 0000000000..79a11f118a --- /dev/null +++ b/tests/cli/test_l3_orchestrator_prompt.py @@ -0,0 +1,577 @@ +"""Group K tests: L3 campaign dispatcher prompt builder — 10 sections, tool surface, +sentinel format, progress markers, and negative bootstrap assertions.""" + +from __future__ import annotations + +import pathlib + +import pytest + +from autoskillit.cli._mcp_names import DIRECT_PREFIX, MARKETPLACE_PREFIX +from autoskillit.core.types._type_constants import SOUS_CHEF_MANDATORY_SECTIONS +from autoskillit.recipe.schema import CampaignDispatch, Recipe, RecipeKind + +pytestmark = [pytest.mark.layer("cli"), pytest.mark.small, pytest.mark.feature("fleet")] + +# --- Constants --- + +_CAMPAIGN_ID = "campaign-test-1234-5678" +_MAX_QUOTA_WAIT = 3600 +_MANIFEST_YAML = """\ +dispatches: + - name: phase-1 + recipe: implementation + task: "Build the thing" + - name: phase-2 + recipe: implementation + task: "Test the thing" + depends_on: [phase-1] +""" + + +def _make_recipe() -> Recipe: + return Recipe( + name="test-campaign", + description="A test campaign for validation", + kind=RecipeKind.CAMPAIGN, + dispatches=[ + CampaignDispatch(name="phase-1", recipe="implementation", task="Build the thing"), + CampaignDispatch( + name="phase-2", + recipe="implementation", + task="Test the thing", + depends_on=["phase-1"], + ), + ], + continue_on_failure=False, + ) + + +def _build(**overrides: object) -> str: + from autoskillit.cli._prompts import _build_fleet_campaign_prompt + + defaults: dict[str, object] = { + "campaign_recipe": _make_recipe(), + "manifest_yaml": _MANIFEST_YAML, + "completed_dispatches": "", + "mcp_prefix": DIRECT_PREFIX, + "campaign_id": _CAMPAIGN_ID, + "max_quota_wait_sec": _MAX_QUOTA_WAIT, + } + defaults.update(overrides) + return _build_fleet_campaign_prompt(**defaults) # type: ignore[arg-type] + + +# --- K-1: TestL3PromptPlaceholders --- + + +class TestL3PromptPlaceholders: + def test_all_parameters_interpolated(self) -> None: + prompt = _build() + assert _CAMPAIGN_ID in prompt + assert "test-campaign" in prompt + assert "A test campaign for validation" in prompt + assert "2 dispatches" in prompt + assert DIRECT_PREFIX in prompt + assert str(_MAX_QUOTA_WAIT) in prompt + assert _MANIFEST_YAML.strip() in prompt + + +# --- K-2: TestL3SousChefDiscipline --- + + +class TestL3AdmiralDiscipline: + def test_full_admiral_content_appended(self) -> None: + prompt = _build() + for header in SOUS_CHEF_MANDATORY_SECTIONS: + assert header in prompt, f"Missing admiral section: {header}" + + def test_graceful_degradation_on_missing_skill_md( + self, monkeypatch: pytest.MonkeyPatch, tmp_path: pathlib.Path + ) -> None: + from autoskillit.cli import _prompts + + monkeypatch.setattr(_prompts, "pkg_root", lambda: tmp_path) + result = _build() + assert isinstance(result, str) + assert len(result) > 0 + # Non-sous-chef structural sections must survive degradation + assert "CAMPAIGN OVERVIEW" in result + assert "DISPATCH MANIFEST" in result + assert "FAILURE RECOVERY" in result + assert "QUOTA RETRY" in result + assert "INTERRUPT/CLEANUP SEQUENCE" in result + + +# --- K-3: TestCampaignOverviewSection --- + + +class TestCampaignOverviewSection: + def test_campaign_overview_present(self) -> None: + prompt = _build() + assert "CAMPAIGN OVERVIEW" in prompt + + def test_overview_contains_name_id_description_count(self) -> None: + prompt = _build() + assert "test-campaign" in prompt + assert _CAMPAIGN_ID in prompt + assert "A test campaign for validation" in prompt + assert "2 dispatches" in prompt + + +# --- K-4: TestDispatchManifestSection --- + + +class TestDispatchManifestSection: + def test_manifest_yaml_embedded_verbatim(self) -> None: + prompt = _build() + assert _MANIFEST_YAML.strip() in prompt + + def test_dispatch_manifest_header_present(self) -> None: + prompt = _build() + assert "DISPATCH MANIFEST" in prompt + + +# --- K-5: TestCampaignDisciplineSection --- + + +class TestCampaignDisciplineSection: + def test_dispatch_food_truck_referenced(self) -> None: + prompt = _build() + assert f"{DIRECT_PREFIX}dispatch_food_truck" in prompt + + def test_no_parallel_dispatch(self) -> None: + prompt = _build() + assert "serial" in prompt.lower() or "sequential" in prompt.lower() + + def test_no_cross_dispatch_aggregation(self) -> None: + prompt = _build() + assert "cross-dispatch token aggregation" in prompt or "NO cross-dispatch" in prompt + + def test_fleet_lock_mentioned(self) -> None: + prompt = _build() + assert "fleet_lock" in prompt + + +# --- K-6: TestFailureRecoverySection --- + + +class TestFailureRecoverySection: + def test_failure_decision_tree_header(self) -> None: + prompt = _build() + assert "FAILURE RECOVERY" in prompt + + def test_three_failure_conditions(self) -> None: + prompt = _build() + assert "success=false" in prompt + assert "null" in prompt + assert ".success=false" in prompt + + def test_continue_on_failure_referenced(self) -> None: + prompt = _build() + assert "continue_on_failure" in prompt + + def test_no_retry_in_v1(self) -> None: + prompt = _build() + assert "NEVER retry" in prompt or "never retry" in prompt.lower() + + +# --- K-7: TestQuotaRetrySection --- + + +class TestQuotaRetrySection: + def test_quota_retry_header(self) -> None: + prompt = _build() + assert "QUOTA RETRY" in prompt + + def test_quota_exhausted_trigger(self) -> None: + prompt = _build() + assert "quota_exhausted" in prompt + + def test_max_quota_wait_sec_interpolated(self) -> None: + prompt = _build(max_quota_wait_sec=7200) + assert "7200" in prompt + + def test_retry_once_then_halt(self) -> None: + prompt = _build() + assert "ONCE" in prompt or "once" in prompt.lower() + assert "halt" in prompt.lower() + + +# --- K-8: TestResumeStateInjection --- + + +class TestResumeStateInjection: + def test_completed_dispatches_block_present_when_supplied(self) -> None: + completed = "- phase-1: SUCCESS\n" + prompt = _build(completed_dispatches=completed) + assert "COMPLETED DISPATCHES" in prompt + assert "phase-1: SUCCESS" in prompt + + def test_completed_dispatches_block_absent_when_empty(self) -> None: + prompt = _build(completed_dispatches="") + assert "COMPLETED DISPATCHES" not in prompt + + def test_do_not_re_dispatch_instruction(self) -> None: + completed = "- phase-1: SUCCESS\n" + prompt = _build(completed_dispatches=completed) + assert "DO NOT RE-DISPATCH" in prompt + + +class TestResumableSessionIdInjection: + def test_resumable_section_includes_resume_session_id_instruction(self) -> None: + prompt = _build( + resumable_dispatch_name="impl-1", + resume_session_id="sess-abc-456", + ) + assert "RESUMABLE DISPATCH: impl-1" in prompt + assert 'resume_session_id="sess-abc-456"' in prompt + + def test_resumable_section_without_session_id_omits_instruction(self) -> None: + prompt = _build( + resumable_dispatch_name="impl-1", + resume_session_id="", + ) + assert "RESUMABLE DISPATCH: impl-1" in prompt + assert 'resume_session_id="' not in prompt + + def test_no_resumable_section_when_dispatch_name_empty(self) -> None: + prompt = _build(resumable_dispatch_name="", resume_session_id="sess-abc") + assert "RESUMABLE DISPATCH:" not in prompt + + +# --- K-9: TestInterruptCleanupSection --- + + +class TestInterruptCleanupSection: + def test_interrupt_cleanup_header(self) -> None: + prompt = _build() + assert "INTERRUPT/CLEANUP SEQUENCE" in prompt + + def test_batch_cleanup_clones_called_first(self) -> None: + prompt = _build() + cleanup_idx = prompt.index("batch_cleanup_clones") + summary_idx = prompt.index("campaign summary") + assert cleanup_idx < summary_idx + + def test_cleanup_sequence_order(self) -> None: + prompt = _build() + cleanup_idx = prompt.index("batch_cleanup_clones") + summary_idx = prompt.lower().index("campaign summary") + end_idx = prompt.lower().index("end the session") + assert cleanup_idx < summary_idx < end_idx + + +# --- K-10: TestCampaignSummaryContract --- + + +class TestCampaignSummaryContract: + def test_campaign_summary_sentinels(self) -> None: + prompt = _build() + assert f"---campaign-summary::{_CAMPAIGN_ID}---" in prompt + assert f"---end-campaign-summary::{_CAMPAIGN_ID}---" in prompt + + def test_per_dispatch_field_present(self) -> None: + prompt = _build() + assert "per_dispatch" in prompt + + def test_error_records_field_present(self) -> None: + prompt = _build() + assert "error_records" in prompt + + def test_no_aggregate_fields(self) -> None: + prompt = _build() + # Extract just the campaign summary schema section to avoid false positives + # from sous-chef content that legitimately uses the word "aggregate". + start = prompt.index(f"---campaign-summary::{_CAMPAIGN_ID}---") + end = prompt.index(f"---end-campaign-summary::{_CAMPAIGN_ID}---") + len( + f"---end-campaign-summary::{_CAMPAIGN_ID}---" + ) + summary_section = prompt[start:end] + assert "total_tokens" not in summary_section + assert "aggregate" not in summary_section + assert "total_input_tokens" not in summary_section + assert "total_output_tokens" not in summary_section + + +# --- K-11: TestProgressMarkers --- + + +class TestProgressMarkers: + def test_progress_marker_format(self) -> None: + prompt = _build() + assert f"%%FLEET_PROGRESS::{_CAMPAIGN_ID}::" in prompt + + def test_all_state_transitions_listed(self) -> None: + prompt = _build() + for state in ("queued", "running", "success", "failure", "skipped"): + assert state in prompt, f"Missing progress state: {state}" + + def test_dispatch_index_placeholder(self) -> None: + prompt = _build() + assert "dispatch__of_" in prompt + + +# --- K-12: TestToolSurface --- + + +class TestToolSurface: + @pytest.mark.parametrize("prefix", [DIRECT_PREFIX, MARKETPLACE_PREFIX]) + def test_six_fleet_tools_listed(self, prefix: str) -> None: + prompt = _build(mcp_prefix=prefix) + for tool in ( + "dispatch_food_truck", + "batch_cleanup_clones", + "get_pipeline_report", + "get_token_summary", + "get_timing_summary", + "get_quota_events", + ): + assert f"{prefix}{tool}" in prompt, f"Missing tool: {prefix}{tool}" + + def test_open_kitchen_in_forbidden_list(self) -> None: + """open_kitchen, close_kitchen, and run_skill are all forbidden in fleet sessions.""" + prompt = _build() + forbidden_idx = prompt.index("Explicitly FORBIDDEN") + forbidden_line = prompt[forbidden_idx : prompt.index(chr(10), forbidden_idx)] + assert "open_kitchen" in forbidden_line + assert "close_kitchen" in forbidden_line + assert "run_skill" in forbidden_line + + def test_uses_dispatch_food_truck_not_run_skill(self) -> None: + prompt = _build() + assert f"{DIRECT_PREFIX}dispatch_food_truck" in prompt + # run_skill must not be described as the dispatch mechanism + assert "dispatch via run_skill" not in prompt + assert "dispatch through run_skill" not in prompt + + +# --- K-13: TestL3NoBootstrapSequence --- + + +class TestL3NoBootstrapSequence: + """K-13 (T6): Fleet auto-gate — no open_kitchen startup sequence needed.""" + + def test_no_startup_sequence_section(self) -> None: + prompt = _build() + assert "STARTUP SEQUENCE" not in prompt + + def test_no_bash_sleep(self) -> None: + prompt = _build() + assert 'Bash(command="sleep 2")' not in prompt + + def test_no_open_kitchen_toolsearch(self) -> None: + prompt = _build() + assert "ToolSearch(query='select:" not in prompt + + def test_open_kitchen_not_callable_in_prompt(self) -> None: + prompt = _build() + assert f"{DIRECT_PREFIX}open_kitchen()" not in prompt + + @pytest.mark.parametrize("prefix", [DIRECT_PREFIX, MARKETPLACE_PREFIX]) + def test_open_kitchen_not_callable_for_any_prefix(self, prefix: str) -> None: + prompt = _build(mcp_prefix=prefix) + assert f"{prefix}open_kitchen()" not in prompt + + +# --- T8-12: TestFleetCampaignRoleText --- + + +class TestFleetCampaignRoleText: + def test_role_identifies_as_fleet_campaign_dispatcher(self) -> None: + prompt = _build() + assert "fleet campaign dispatcher" in prompt + + +# --- K-14: TestDynamicDispatchSection --- + + +class TestDynamicDispatchSection: + def _make_dynamic_recipe(self) -> Recipe: + return Recipe( + name="audit-campaign", + description="Full audit and implement campaign", + kind=RecipeKind.CAMPAIGN, + dispatches=[ + CampaignDispatch( + name="build-map", + recipe="bem-wrapper", + task="Build execution map", + capture={ + "execution_map": "${{ result.execution_map }}", + "dispatch_plan": "${{ result.dispatch_plan }}", + }, + ), + ], + continue_on_failure=False, + ) + + def test_dynamic_dispatch_section_present_when_dispatch_plan_captured(self) -> None: + prompt = _build(campaign_recipe=self._make_dynamic_recipe()) + assert "DYNAMIC DISPATCH" in prompt + + def test_dynamic_dispatch_section_absent_for_static_only_campaign(self) -> None: + prompt = _build() + assert "DYNAMIC DISPATCH" not in prompt + + def test_implement_findings_recipe_name_referenced(self) -> None: + prompt = _build(campaign_recipe=self._make_dynamic_recipe()) + assert "implement-findings" in prompt + + def test_dispatch_plan_campaign_ref_referenced(self) -> None: + prompt = _build(campaign_recipe=self._make_dynamic_recipe()) + assert "campaign.dispatch_plan" in prompt + + def test_group_iteration_instruction_present(self) -> None: + prompt = _build(campaign_recipe=self._make_dynamic_recipe()) + assert "For each group (in array order)" in prompt + + def test_naming_convention_instruction_present(self) -> None: + prompt = _build(campaign_recipe=self._make_dynamic_recipe()) + assert "-g" in prompt and "-a" in prompt + + def test_parallel_dispatch_instruction_present(self) -> None: + prompt = _build(campaign_recipe=self._make_dynamic_recipe()) + assert "parallel tool calls" in prompt + + def test_wait_for_group_before_next_group(self) -> None: + prompt = _build(campaign_recipe=self._make_dynamic_recipe()) + assert "Wait for ALL food trucks in this group" in prompt + + def test_max_issues_per_food_truck_split_instruction(self) -> None: + prompt = _build(campaign_recipe=self._make_dynamic_recipe()) + assert "batches of that size" in prompt + + def test_empty_dispatch_plan_handled(self) -> None: + prompt = _build(campaign_recipe=self._make_dynamic_recipe()) + assert "no issues to implement" in prompt + + def test_dynamic_dispatch_absent_when_only_execution_map_captured(self) -> None: + recipe = Recipe( + name="partial-campaign", + description="Only captures execution_map", + kind=RecipeKind.CAMPAIGN, + dispatches=[ + CampaignDispatch( + name="build-map", + recipe="bem-wrapper", + task="Build map", + capture={"execution_map": "${{ result.execution_map }}"}, + ), + ], + continue_on_failure=False, + ) + prompt = _build(campaign_recipe=recipe) + assert "DYNAMIC DISPATCH" not in prompt + + +# --- K-15: TestK15IngredientsTableInjection --- + + +class TestK15IngredientsTableInjection: + _TABLE = "| Name | Description | Default |\n| task | What to fix | — |" + + def test_ingredients_section_present_when_provided(self) -> None: + prompt = _build(ingredients_table=self._TABLE) + assert "RECIPE INGREDIENTS" in prompt + assert self._TABLE in prompt + + def test_ingredients_section_absent_when_none(self) -> None: + prompt = _build(ingredients_table=None) + assert "RECIPE INGREDIENTS" not in prompt + + def test_ask_user_question_instruction_present(self) -> None: + prompt = _build(ingredients_table=self._TABLE) + assert "AskUserQuestion" in prompt + + def test_ingredients_section_between_overview_and_manifest(self) -> None: + prompt = _build(ingredients_table=self._TABLE) + overview_pos = prompt.index("CAMPAIGN OVERVIEW") + ingredients_pos = prompt.index("RECIPE INGREDIENTS") + manifest_pos = prompt.index("DISPATCH MANIFEST") + assert overview_pos < ingredients_pos < manifest_pos + + +class TestK16IngredientDisplayFirstAction: + """K-16: Campaign FIRST ACTION instructs Claude to display the ingredient table.""" + + _TABLE = "| Name | Description | Default |\n| task | What to fix | — |" + + def test_first_action_block_present_when_ingredients_table_provided(self) -> None: + prompt = _build(ingredients_table=self._TABLE) + assert "FIRST ACTION" in prompt + + def test_first_action_instructs_display_verbatim(self) -> None: + prompt = _build(ingredients_table=self._TABLE) + first_action_idx = prompt.index("FIRST ACTION") + dispatch_manifest_idx = prompt.index("DISPATCH MANIFEST") + section = prompt[first_action_idx:dispatch_manifest_idx] + assert "verbatim" in section + + def test_first_action_references_recipe_ingredients_section(self) -> None: + prompt = _build(ingredients_table=self._TABLE) + first_action_idx = prompt.index("FIRST ACTION") + dispatch_manifest_idx = prompt.index("DISPATCH MANIFEST") + section = prompt[first_action_idx:dispatch_manifest_idx] + assert "RECIPE INGREDIENTS" in section + + def test_first_action_absent_when_no_ingredients_table(self) -> None: + prompt = _build(ingredients_table=None) + assert "FIRST ACTION" not in prompt + + def test_first_action_does_not_call_open_kitchen(self) -> None: + prompt = _build(ingredients_table=self._TABLE) + first_action_idx = prompt.index("FIRST ACTION") + dispatch_manifest_idx = prompt.index("DISPATCH MANIFEST") + section = prompt[first_action_idx:dispatch_manifest_idx] + assert "open_kitchen(" not in section + + def test_first_action_appears_before_dispatch_manifest(self) -> None: + prompt = _build(ingredients_table=self._TABLE) + assert prompt.index("FIRST ACTION") < prompt.index("DISPATCH MANIFEST") + + def test_first_action_instructs_ask_user_question(self) -> None: + prompt = _build(ingredients_table=self._TABLE) + first_action_idx = prompt.index("FIRST ACTION") + dispatch_manifest_idx = prompt.index("DISPATCH MANIFEST") + section = prompt[first_action_idx:dispatch_manifest_idx] + assert "AskUserQuestion" in section + + +class TestResumeReasonInPrompt: + def test_idle_stall_resume_includes_reenter_guidance(self) -> None: + prompt = _build( + resumable_dispatch_name="impl-1", + resume_session_id="sess-1", + resume_kill_reason="idle_stall", + ) + assert "RESUMABLE DISPATCH: impl-1" in prompt + assert "idle timeout" in prompt + assert "Resume is safe" in prompt + + def test_context_exhausted_absent_from_resumable(self) -> None: + prompt = _build( + resumable_dispatch_name="impl-1", + resume_kill_reason="context_exhausted", + ) + assert "RESUMABLE DISPATCH: impl-1" in prompt + idx = prompt.index("RESUMABLE DISPATCH: impl-1") + end_idx = prompt.index("## INTERRUPT/CLEANUP", idx) + resumable_section = prompt[idx:end_idx] + assert "context_exhausted" not in resumable_section + assert "Kill reason: unknown" in resumable_section + + def test_api_error_resume_includes_retry_guidance(self) -> None: + prompt = _build( + resumable_dispatch_name="impl-1", + resume_kill_reason="resume", + ) + assert "RESUMABLE DISPATCH: impl-1" in prompt + assert "transient infrastructure failure" in prompt + + def test_resume_discipline_section_present(self) -> None: + prompt = _build() + assert "RESUME DISCIPLINE" in prompt + assert "idle_stall" in prompt + assert "context_exhausted" in prompt + assert "thinking_stall" in prompt diff --git a/tests/cli/test_mcp_names.py b/tests/cli/test_mcp_names.py index 9f8e52c728..85709be154 100644 --- a/tests/cli/test_mcp_names.py +++ b/tests/cli/test_mcp_names.py @@ -13,23 +13,33 @@ detect_autoskillit_mcp_prefix, ) +pytestmark = [pytest.mark.layer("cli"), pytest.mark.small] + _PLUGIN_KEY = "autoskillit@autoskillit-local" class TestDetectAutoskillitMcpPrefix: + @pytest.fixture(autouse=True) + def _clear_prefix_cache(self): # noqa: ANN204 + from autoskillit.core._plugin_ids import detect_autoskillit_mcp_prefix as _fn + + _fn.cache_clear() + yield + _fn.cache_clear() + def test_returns_marketplace_prefix_when_plugin_key_present( self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: f = tmp_path / "installed_plugins.json" f.write_text(json.dumps({"version": 2, "plugins": {_PLUGIN_KEY: []}})) - monkeypatch.setattr("autoskillit.cli._mcp_names._installed_plugins_path", lambda: f) + monkeypatch.setattr("autoskillit.core._plugin_ids._installed_plugins_path", lambda: f) assert detect_autoskillit_mcp_prefix() == MARKETPLACE_PREFIX def test_returns_direct_prefix_when_file_absent( self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: monkeypatch.setattr( - "autoskillit.cli._mcp_names._installed_plugins_path", + "autoskillit.core._plugin_ids._installed_plugins_path", lambda: tmp_path / "no_such_file.json", ) assert detect_autoskillit_mcp_prefix() == DIRECT_PREFIX @@ -39,7 +49,7 @@ def test_returns_direct_prefix_when_autoskillit_key_absent( ) -> None: f = tmp_path / "installed_plugins.json" f.write_text(json.dumps({"version": 2, "plugins": {"other@other-local": []}})) - monkeypatch.setattr("autoskillit.cli._mcp_names._installed_plugins_path", lambda: f) + monkeypatch.setattr("autoskillit.core._plugin_ids._installed_plugins_path", lambda: f) assert detect_autoskillit_mcp_prefix() == DIRECT_PREFIX def test_returns_direct_prefix_on_malformed_json( @@ -47,7 +57,7 @@ def test_returns_direct_prefix_on_malformed_json( ) -> None: f = tmp_path / "installed_plugins.json" f.write_text("not valid json {{{") - monkeypatch.setattr("autoskillit.cli._mcp_names._installed_plugins_path", lambda: f) + monkeypatch.setattr("autoskillit.core._plugin_ids._installed_plugins_path", lambda: f) assert detect_autoskillit_mcp_prefix() == DIRECT_PREFIX def test_direct_prefix_ends_with_double_underscore(self) -> None: diff --git a/tests/cli/test_menu.py b/tests/cli/test_menu.py new file mode 100644 index 0000000000..2e324d123d --- /dev/null +++ b/tests/cli/test_menu.py @@ -0,0 +1,147 @@ +"""Tests: shared selection menu primitive.""" + +from __future__ import annotations + +from unittest.mock import MagicMock + +import pytest + +from autoskillit.cli.ui._menu import ( + SLOT_ZERO_SELECTED, + render_numbered_menu, + resolve_menu_input, + run_selection_menu, +) + +pytestmark = [pytest.mark.layer("cli"), pytest.mark.medium] + + +def _make_item(name: str, tag: str = "") -> MagicMock: + item = MagicMock() + item.name = name + item.tag = tag + return item + + +class TestRenderNumberedMenu: + def test_flat_list_renders_numbered_items(self, capsys: pytest.CaptureFixture[str]) -> None: + items = [_make_item("alpha"), _make_item("beta"), _make_item("gamma")] + render_numbered_menu(items, header="Items:") + out = capsys.readouterr().out + assert "1. alpha" in out + assert "2. beta" in out + assert "3. gamma" in out + assert "Items:" in out + + def test_flat_list_no_group_headers(self, capsys: pytest.CaptureFixture[str]) -> None: + items = [_make_item("alpha"), _make_item("beta"), _make_item("gamma")] + render_numbered_menu(items, header="Items:") + out = capsys.readouterr().out + assert "Primary" not in out + assert "Secondary" not in out + + def test_grouped_list_renders_group_headers(self, capsys: pytest.CaptureFixture[str]) -> None: + items = [_make_item("a"), _make_item("b"), _make_item("c"), _make_item("d")] + classifier = lambda item: 0 if item.name in ("a", "b") else 1 # noqa: E731 + render_numbered_menu( + items, + header="Items:", + group_classifier=classifier, + group_labels={0: "Primary", 1: "Secondary"}, + ) + out = capsys.readouterr().out + assert "Primary" in out + assert "Secondary" in out + primary_pos = out.index("Primary") + secondary_pos = out.index("Secondary") + assert primary_pos < secondary_pos + + def test_slot_zero_renders_when_label_provided( + self, capsys: pytest.CaptureFixture[str] + ) -> None: + items = [_make_item("alpha")] + render_numbered_menu(items, header="Items:", slot_zero_label="Open kitchen") + out = capsys.readouterr().out + assert "0. Open kitchen" in out + assert "1. alpha" in out + + def test_slot_zero_omitted_when_label_is_none( + self, capsys: pytest.CaptureFixture[str] + ) -> None: + items = [_make_item("alpha")] + render_numbered_menu(items, header="Items:", slot_zero_label=None) + out = capsys.readouterr().out + assert "0." not in out + + def test_custom_display_fn(self, capsys: pytest.CaptureFixture[str]) -> None: + items = [_make_item("alpha", tag="fast"), _make_item("beta", tag="slow")] + render_numbered_menu(items, header="Items:", display_fn=lambda x: f"{x.name} ({x.tag})") + out = capsys.readouterr().out + assert "alpha (fast)" in out + assert "beta (slow)" in out + + +class TestResolveMenuInput: + def _items(self) -> list[MagicMock]: + return [_make_item("alpha"), _make_item("beta"), _make_item("gamma")] + + def test_valid_number_returns_item(self) -> None: + items = self._items() + result = resolve_menu_input("2", items) + assert result is items[1] + + def test_slot_zero_returns_sentinel(self) -> None: + items = self._items() + result = resolve_menu_input("0", items, slot_zero=True) + assert result is SLOT_ZERO_SELECTED + + def test_slot_zero_disabled_returns_none(self) -> None: + items = self._items() + result = resolve_menu_input("0", items, slot_zero=False) + assert result is None + + def test_out_of_range_returns_none(self) -> None: + items = self._items() + result = resolve_menu_input("99", items) + assert result is None + + def test_name_match_returns_item(self) -> None: + items = self._items() + result = resolve_menu_input("beta", items, name_key=lambda x: x.name) + assert result is items[1] + + def test_name_no_match_returns_none(self) -> None: + items = self._items() + result = resolve_menu_input("nonexistent", items) + assert result is None + + def test_empty_string_returns_none(self) -> None: + items = self._items() + result = resolve_menu_input("", items) + assert result is None + + +class TestRunSelectionMenu: + def test_end_to_end_numeric_selection( + self, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] + ) -> None: + items = [_make_item("alpha"), _make_item("beta")] + monkeypatch.setattr("autoskillit.cli.ui._menu.timed_prompt", lambda *a, **kw: "1") + result = run_selection_menu(items, header="Items:") + assert result is items[0] + + def test_end_to_end_slot_zero( + self, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] + ) -> None: + items = [_make_item("alpha")] + monkeypatch.setattr("autoskillit.cli.ui._menu.timed_prompt", lambda *a, **kw: "0") + result = run_selection_menu(items, header="Items:", slot_zero_label="Open kitchen") + assert result is SLOT_ZERO_SELECTED + + def test_end_to_end_invalid_returns_none( + self, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] + ) -> None: + items = [_make_item("alpha")] + monkeypatch.setattr("autoskillit.cli.ui._menu.timed_prompt", lambda *a, **kw: "invalid") + result = run_selection_menu(items, header="Items:") + assert result is None diff --git a/tests/cli/test_onboarding.py b/tests/cli/test_onboarding.py index d49bc607a0..5022b01151 100644 --- a/tests/cli/test_onboarding.py +++ b/tests/cli/test_onboarding.py @@ -7,14 +7,13 @@ import pytest from autoskillit.cli._onboarding import ( - OnboardingIntel, - _detect_build_tools, - gather_intel, is_first_run, mark_onboarded, run_onboarding_menu, ) +pytestmark = [pytest.mark.layer("cli"), pytest.mark.small] + def _make_initialized_project(base: Path) -> Path: """Create a minimal initialized project dir (config.yaml present).""" @@ -109,7 +108,6 @@ def test_run_onboarding_menu_skip_e_returns_none_and_marks( inputs = iter(["y", "E"]) monkeypatch.setattr("sys.stdin.isatty", lambda: True) monkeypatch.setattr("builtins.input", lambda _prompt="": next(inputs)) - monkeypatch.setattr("autoskillit.cli._onboarding.gather_intel", lambda _p: OnboardingIntel()) result = run_onboarding_menu(tmp_path, color=False) assert result is None assert (tmp_path / ".autoskillit" / ".onboarded").exists() @@ -124,7 +122,6 @@ def test_run_onboarding_menu_option_a_returns_setup_project( inputs = iter(["y", "A"]) monkeypatch.setattr("sys.stdin.isatty", lambda: True) monkeypatch.setattr("builtins.input", lambda _prompt="": next(inputs)) - monkeypatch.setattr("autoskillit.cli._onboarding.gather_intel", lambda _p: OnboardingIntel()) result = run_onboarding_menu(tmp_path, color=False) assert result is not None assert "/autoskillit:setup-project" in result @@ -140,7 +137,6 @@ def test_run_onboarding_menu_option_b_with_url_returns_prepare_issue( inputs = iter(["y", "B", "https://github.com/org/repo/issues/42"]) monkeypatch.setattr("sys.stdin.isatty", lambda: True) monkeypatch.setattr("builtins.input", lambda _prompt="": next(inputs)) - monkeypatch.setattr("autoskillit.cli._onboarding.gather_intel", lambda _p: OnboardingIntel()) result = run_onboarding_menu(tmp_path, color=False) assert result is not None assert "/autoskillit:prepare-issue" in result @@ -155,7 +151,6 @@ def test_run_onboarding_menu_option_d_returns_write_recipe( inputs = iter(["y", "D"]) monkeypatch.setattr("sys.stdin.isatty", lambda: True) monkeypatch.setattr("builtins.input", lambda _prompt="": next(inputs)) - monkeypatch.setattr("autoskillit.cli._onboarding.gather_intel", lambda _p: OnboardingIntel()) result = run_onboarding_menu(tmp_path, color=False) assert result is not None assert "/autoskillit:write-recipe" in result @@ -170,45 +165,27 @@ def test_run_onboarding_menu_option_c_returns_setup_project_prompt( inputs = iter(["y", "C"]) monkeypatch.setattr("sys.stdin.isatty", lambda: True) monkeypatch.setattr("builtins.input", lambda _prompt="": next(inputs)) - monkeypatch.setattr("autoskillit.cli._onboarding.gather_intel", lambda _p: OnboardingIntel()) result = run_onboarding_menu(tmp_path, color=False) assert result is not None assert result.startswith("/autoskillit:setup-project") -# ON-14 -def test_gather_intel_returns_onboarding_intel_dataclass( +# ON-18 +def test_run_onboarding_menu_e_path_no_executor_import( tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: - """gather_intel(project_dir) returns OnboardingIntel with correct fields.""" - monkeypatch.setattr( - "autoskillit.cli._onboarding._fetch_good_first_issues", - lambda _p: ["#1: some issue"], - ) - intel = gather_intel(tmp_path) - assert isinstance(intel, OnboardingIntel) - assert isinstance(intel.scanner_found, (str, type(None))) - assert isinstance(intel.build_tools, list) - assert intel.github_issues == ["#1: some issue"] - - -# ON-15 -def test_detect_build_tools_finds_taskfile(tmp_path: Path) -> None: - """Project dir has Taskfile.yml → 'Taskfile' in build_tools.""" - (tmp_path / "Taskfile.yml").write_text("version: '3'\n") - tools = _detect_build_tools(tmp_path) - assert "Taskfile" in tools + """_onboarding module must not import ThreadPoolExecutor after dead code removal.""" + import autoskillit.cli._onboarding as _onboarding_module + assert not hasattr(_onboarding_module, "ThreadPoolExecutor"), ( + "ThreadPoolExecutor is still imported in _onboarding — dead code not removed" + ) -# ON-16 -def test_detect_build_tools_finds_makefile(tmp_path: Path) -> None: - """Project dir has Makefile → 'Makefile' in build_tools.""" - (tmp_path / "Makefile").write_text("all:\n\techo hello\n") - tools = _detect_build_tools(tmp_path) - assert "Makefile" in tools - + _make_initialized_project(tmp_path) + inputs = iter(["y", "E"]) + monkeypatch.setattr("sys.stdin.isatty", lambda: True) + monkeypatch.setattr("builtins.input", lambda _prompt="": next(inputs)) -# ON-17 -def test_onboarding_intel_importable_from_cli_onboarding() -> None: - """OnboardingIntel is importable from autoskillit.cli._onboarding.""" - from autoskillit.cli._onboarding import OnboardingIntel # noqa: F401 + result = run_onboarding_menu(tmp_path, color=False) + assert result is None + assert (tmp_path / ".autoskillit" / ".onboarded").exists() diff --git a/tests/cli/test_orchestrator_prompt_contract.py b/tests/cli/test_orchestrator_prompt_contract.py index 69055a6e20..63ea892da1 100644 --- a/tests/cli/test_orchestrator_prompt_contract.py +++ b/tests/cli/test_orchestrator_prompt_contract.py @@ -4,6 +4,10 @@ import re +import pytest + +pytestmark = [pytest.mark.layer("cli"), pytest.mark.small] + def _get_prompt() -> str: """Return the orchestrator prompt for a demo recipe.""" @@ -87,17 +91,119 @@ def test_first_action_prohibits_ask_user_question_before_open_kitchen(self): first_action_section = prompt[first_action_start:first_action_end] assert "DO NOT call AskUserQuestion" in first_action_section + def test_first_action_instructs_retry_on_mcp_unavailable(self): + """FIRST ACTION must contain a retry instruction for 'No such tool available'.""" + from autoskillit.cli._prompts import _build_orchestrator_prompt -class TestFirstActionTimingResilience: - """FIRST ACTION must acknowledge MCP initialization timing.""" + prompt = _build_orchestrator_prompt("demo", "mcp__autoskillit__") + fa_start = prompt.find("FIRST ACTION") + assert fa_start != -1, "'FIRST ACTION' section not found in prompt" + fa_end = prompt.find("During pipeline execution", fa_start) + assert fa_end != -1, "'During pipeline execution' section not found after FIRST ACTION" + first_action = prompt[fa_start:fa_end] + + assert "retry" in first_action.lower(), ( + "FIRST ACTION must contain a retry instruction for MCP tool unavailability" + ) + assert "No such tool" in first_action or "unavailable" in first_action.lower(), ( + "FIRST ACTION retry must reference 'No such tool' or 'unavailable'" + ) + assert "Bash" not in first_action, "Retry instruction must not reference Bash" + assert "ToolSearch" not in first_action, "Retry instruction must not reference ToolSearch" + assert "sleep" not in first_action.lower(), "Retry instruction must not use sleep" + + +class TestOpenKitchenRetryOnUnavailable: + """open_kitchen prompt must instruct the LLM to retry when MCP is not yet ready.""" + + def test_open_kitchen_prompt_instructs_retry_on_mcp_unavailable(self): + """open_kitchen prompt must contain a retry instruction before IMPORTANT section.""" + from autoskillit.cli._prompts import _build_open_kitchen_prompt + + ok_prompt = _build_open_kitchen_prompt("mcp__autoskillit__") + first_section_end = ok_prompt.find("IMPORTANT \u2014 Orchestrator Discipline:") + assert first_section_end != -1, ( + "'IMPORTANT \u2014 Orchestrator Discipline:' section not found in ok_prompt" + ) + first_section = ok_prompt[:first_section_end] + + assert "retry" in first_section.lower(), ( + "_build_open_kitchen_prompt must contain a retry instruction " + "for MCP tool unavailability" + ) + assert "No such tool" in first_section or "unavailable" in first_section.lower(), ( + "_build_open_kitchen_prompt retry must reference 'No such tool' or 'unavailable'" + ) + + def test_mcp_retry_instruction_covers_is_error_block_shape(self) -> None: + """_MCP_RETRY_INSTRUCTION must explicitly cover the is_error:true wire format. + + Claude Code delivers MCP transport failures as is_error: true tool_use_error blocks, + not as string returns. The instruction must cover "any error" or explicitly mention + error blocks — not just the string-return case. + """ + from autoskillit.cli._prompts import _MCP_RETRY_INSTRUCTION + + instr = _MCP_RETRY_INSTRUCTION.lower() + assert any( + phrase in instr for phrase in ("any error", "tool error", "tool_use_error", "is_error") + ), ( + "_MCP_RETRY_INSTRUCTION does not cover is_error:true wire format. " + "Claude Code sends MCP failures as is_error blocks, not string returns." + ) + + def test_mcp_retry_instruction_does_not_gate_on_string_return_only(self) -> None: + """Instruction must not use 'returns' as the sole trigger for retry. + + 'If open_kitchen returns X' is a string-return pattern. The actual error arrives + as is_error: true with a tool_use_error content block — not a function return. + """ + from autoskillit.cli._prompts import _MCP_RETRY_INSTRUCTION - def test_first_action_is_timing_resilient(self): - """FIRST ACTION must not unconditionally command 'MUST be first' - without acknowledging MCP initialization timing.""" - prompt = _get_prompt() - first_action_start = prompt.index("FIRST ACTION") - first_action_end = prompt.index("During pipeline execution", first_action_start) - first_action_section = prompt[first_action_start:first_action_end] assert ( - "retry" in first_action_section.lower() or "available" in first_action_section.lower() - ), "FIRST ACTION must acknowledge MCP init timing with retry/availability guidance" + "returns" not in _MCP_RETRY_INSTRUCTION + or "any error" in _MCP_RETRY_INSTRUCTION.lower() + or "error" in _MCP_RETRY_INSTRUCTION.lower() + ), "_MCP_RETRY_INSTRUCTION uses 'returns' as sole trigger — misses is_error:true shape." + + def test_mcp_retry_instruction_covers_silent_retry(self) -> None: + """Instruction must direct LLM to retry silently without troubleshooting. + + Session 5f08b230 showed LLM gave troubleshooting advice instead of retrying. + Instruction must explicitly prohibit explaining the error to the user. + """ + from autoskillit.cli._prompts import _MCP_RETRY_INSTRUCTION + + instr = _MCP_RETRY_INSTRUCTION.lower() + assert "silently" in instr or "do not explain" in instr or "do not" in instr, ( + "_MCP_RETRY_INSTRUCTION does not prohibit LLM from explaining the error to the user." + ) + + +class TestFirstActionDirectOpenKitchen: + """FIRST ACTION must call open_kitchen directly — no ToolSearch or Bash preamble.""" + + def test_first_action_no_step0(self): + """FIRST ACTION must not contain a step 0.""" + prompt = _get_prompt() + fa_start = prompt.index("FIRST ACTION") + fa_end = prompt.index("During pipeline execution", fa_start) + first_action = prompt[fa_start:fa_end] + assert "\n0." not in first_action + + def test_first_action_no_toolsearch(self): + """FIRST ACTION must not reference ToolSearch.""" + prompt = _get_prompt() + fa_start = prompt.index("FIRST ACTION") + fa_end = prompt.index("During pipeline execution", fa_start) + first_action = prompt[fa_start:fa_end] + assert "ToolSearch" not in first_action + + def test_first_action_no_bash_sleep(self): + """FIRST ACTION must not reference Bash or sleep.""" + prompt = _get_prompt() + fa_start = prompt.index("FIRST ACTION") + fa_end = prompt.index("During pipeline execution", fa_start) + first_action = prompt[fa_start:fa_end] + assert "Bash" not in first_action + assert "sleep" not in first_action.lower() diff --git a/tests/cli/test_order_resume.py b/tests/cli/test_order_resume.py new file mode 100644 index 0000000000..48f0868ab2 --- /dev/null +++ b/tests/cli/test_order_resume.py @@ -0,0 +1,82 @@ +"""T5: Order CLI detects infrastructure exit and auto-resumes.""" + +from __future__ import annotations + +from unittest.mock import patch + +import pytest + +from autoskillit.cli.session._session_launch import ( + _InfraExitSignal, + _launch_cook_session, +) +from autoskillit.core import NamedResume + +pytestmark = [pytest.mark.layer("cli"), pytest.mark.small] + + +class TestLaunchCookSessionInfraResume: + def test_infra_exit_triggers_resume(self) -> None: + call_count = 0 + + def mock_run_interactive(system_prompt, **kwargs): + nonlocal call_count + call_count += 1 + if call_count == 1: + return _InfraExitSignal(session_id="dead-sess", category="context_exhausted") + return None + + with patch( + "autoskillit.cli.session._session_launch._run_interactive_session", + side_effect=mock_run_interactive, + ): + _launch_cook_session("prompt") + + assert call_count == 2 + + def test_infra_exit_uses_named_resume(self) -> None: + resume_specs: list = [] + + def mock_run_interactive(system_prompt, **kwargs): + resume_specs.append(kwargs.get("resume_spec")) + if len(resume_specs) == 1: + return _InfraExitSignal(session_id="sess-42", category="api_error") + return None + + with patch( + "autoskillit.cli.session._session_launch._run_interactive_session", + side_effect=mock_run_interactive, + ): + _launch_cook_session("prompt") + + assert isinstance(resume_specs[1], NamedResume) + assert resume_specs[1].session_id == "sess-42" + + def test_max_infra_resumes_exceeded(self) -> None: + def mock_run_interactive(system_prompt, **kwargs): + return _InfraExitSignal(session_id="sess-loop", category="process_killed") + + with ( + patch( + "autoskillit.cli.session._session_launch._run_interactive_session", + side_effect=mock_run_interactive, + ), + pytest.raises(SystemExit, match="Too many infrastructure resumes"), + ): + _launch_cook_session("prompt") + + def test_no_resume_on_clean_exit(self) -> None: + call_count = 0 + + def mock_run_interactive(system_prompt, **kwargs): + nonlocal call_count + call_count += 1 + return None + + with patch( + "autoskillit.cli.session._session_launch._run_interactive_session", + side_effect=mock_run_interactive, + ): + _launch_cook_session("prompt") + + assert call_count == 1 diff --git a/tests/cli/test_plugin_cache.py b/tests/cli/test_plugin_cache.py new file mode 100644 index 0000000000..0cb115d06a --- /dev/null +++ b/tests/cli/test_plugin_cache.py @@ -0,0 +1,558 @@ +"""Tests for _plugin_cache: retiring cache, install locking, kitchen registry.""" + +from __future__ import annotations + +import fcntl +import json +import os +import subprocess +import threading +from datetime import UTC, datetime, timedelta +from pathlib import Path + +import pytest + +pytestmark = [pytest.mark.layer("cli"), pytest.mark.medium] + + +# --------------------------------------------------------------------------- +# Category A — Grace period on version change +# --------------------------------------------------------------------------- + + +def test_retire_old_versions_registers_different_version( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(Path, "home", lambda: tmp_path) + from autoskillit.core._plugin_cache import _retire_old_versions + + cache_dir = tmp_path / "cache" + old_dir = cache_dir / "0.8.0" + old_dir.mkdir(parents=True) + + _retire_old_versions(cache_dir, "0.9.0") + + assert old_dir.exists(), "0.8.0/ must survive (grace period)" + retiring_json = tmp_path / ".autoskillit" / "retiring_cache.json" + data = json.loads(retiring_json.read_text()) + entries = data["retiring"] + assert len(entries) == 1 + assert entries[0]["version"] == "0.8.0" + assert isinstance(datetime.fromisoformat(entries[0]["retired_at"]), datetime) + + +def test_retire_old_versions_multiple_old_dirs( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(Path, "home", lambda: tmp_path) + from autoskillit.core._plugin_cache import _retire_old_versions + + cache_dir = tmp_path / "cache" + (cache_dir / "0.7.0").mkdir(parents=True) + (cache_dir / "0.8.0").mkdir(parents=True) + + _retire_old_versions(cache_dir, "0.9.0") + + assert (cache_dir / "0.7.0").exists() + assert (cache_dir / "0.8.0").exists() + retiring_json = tmp_path / ".autoskillit" / "retiring_cache.json" + data = json.loads(retiring_json.read_text()) + versions = {e["version"] for e in data["retiring"]} + assert versions == {"0.7.0", "0.8.0"} + + +# --------------------------------------------------------------------------- +# Category B — Same-version reinstall +# --------------------------------------------------------------------------- + + +def test_retire_old_versions_deletes_same_version( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(Path, "home", lambda: tmp_path) + from autoskillit.core._plugin_cache import _retire_old_versions + + cache_dir = tmp_path / "cache" + same_dir = cache_dir / "0.9.0" + same_dir.mkdir(parents=True) + + _retire_old_versions(cache_dir, "0.9.0") + + assert not same_dir.exists(), "Same-version dir must be deleted immediately" + retiring_json = tmp_path / ".autoskillit" / "retiring_cache.json" + assert not retiring_json.exists(), ( + "No retiring entries should be created for a same-version reinstall" + ) + + +def test_retire_old_versions_noop_empty_cache_dir( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(Path, "home", lambda: tmp_path) + from autoskillit.core._plugin_cache import _retire_old_versions + + cache_dir = tmp_path / "cache" + cache_dir.mkdir(parents=True) + + _retire_old_versions(cache_dir, "0.9.0") # must not raise + + retiring_json = tmp_path / ".autoskillit" / "retiring_cache.json" + assert not retiring_json.exists(), ( + "No retiring entries should be created for an empty cache dir" + ) + + +# --------------------------------------------------------------------------- +# Category C — Sweep deletes expired +# --------------------------------------------------------------------------- + + +def test_sweep_deletes_expired_entries(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(Path, "home", lambda: tmp_path) + from autoskillit.core._plugin_cache import sweep_retiring_cache + + old_dir = tmp_path / "cache" / "0.8.0" + old_dir.mkdir(parents=True) + retired_at = (datetime.now(UTC) - timedelta(hours=25)).isoformat() + retiring_json = tmp_path / ".autoskillit" / "retiring_cache.json" + retiring_json.parent.mkdir(parents=True, exist_ok=True) + retiring_json.write_text( + json.dumps( + { + "retiring": [{"version": "0.8.0", "path": str(old_dir), "retired_at": retired_at}], + "schema_version": 1, + } + ) + ) + + count = sweep_retiring_cache(grace_hours=24) + + assert count == 1 + assert not old_dir.exists() + data = json.loads(retiring_json.read_text()) + assert data["retiring"] == [] + + +def test_sweep_deletes_multiple_expired(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(Path, "home", lambda: tmp_path) + from autoskillit.core._plugin_cache import sweep_retiring_cache + + dir1 = tmp_path / "cache" / "0.7.0" + dir2 = tmp_path / "cache" / "0.8.0" + dir1.mkdir(parents=True) + dir2.mkdir(parents=True) + old_ts = (datetime.now(UTC) - timedelta(hours=30)).isoformat() + retiring_json = tmp_path / ".autoskillit" / "retiring_cache.json" + retiring_json.parent.mkdir(parents=True, exist_ok=True) + retiring_json.write_text( + json.dumps( + { + "retiring": [ + {"version": "0.7.0", "path": str(dir1), "retired_at": old_ts}, + {"version": "0.8.0", "path": str(dir2), "retired_at": old_ts}, + ], + "schema_version": 1, + } + ) + ) + + count = sweep_retiring_cache(grace_hours=24) + + assert count == 2 + assert not dir1.exists() + assert not dir2.exists() + + +# --------------------------------------------------------------------------- +# Category D — Sweep preserves fresh +# --------------------------------------------------------------------------- + + +def test_sweep_preserves_fresh_entries(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(Path, "home", lambda: tmp_path) + from autoskillit.core._plugin_cache import sweep_retiring_cache + + fresh_dir = tmp_path / "cache" / "0.8.0" + fresh_dir.mkdir(parents=True) + retired_at = (datetime.now(UTC) - timedelta(hours=1)).isoformat() + retiring_json = tmp_path / ".autoskillit" / "retiring_cache.json" + retiring_json.parent.mkdir(parents=True, exist_ok=True) + retiring_json.write_text( + json.dumps( + { + "retiring": [ + {"version": "0.8.0", "path": str(fresh_dir), "retired_at": retired_at} + ], + "schema_version": 1, + } + ) + ) + + count = sweep_retiring_cache(grace_hours=24) + + assert count == 0 + assert fresh_dir.exists() + data = json.loads(retiring_json.read_text()) + assert len(data["retiring"]) == 1 + + +def test_sweep_mixed_fresh_and_expired(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(Path, "home", lambda: tmp_path) + from autoskillit.core._plugin_cache import sweep_retiring_cache + + expired_dir = tmp_path / "cache" / "0.7.0" + fresh_dir = tmp_path / "cache" / "0.8.0" + expired_dir.mkdir(parents=True) + fresh_dir.mkdir(parents=True) + expired_ts = (datetime.now(UTC) - timedelta(hours=30)).isoformat() + fresh_ts = (datetime.now(UTC) - timedelta(hours=1)).isoformat() + retiring_json = tmp_path / ".autoskillit" / "retiring_cache.json" + retiring_json.parent.mkdir(parents=True, exist_ok=True) + retiring_json.write_text( + json.dumps( + { + "retiring": [ + {"version": "0.7.0", "path": str(expired_dir), "retired_at": expired_ts}, + {"version": "0.8.0", "path": str(fresh_dir), "retired_at": fresh_ts}, + ], + "schema_version": 1, + } + ) + ) + + count = sweep_retiring_cache(grace_hours=24) + + assert count == 1 + assert not expired_dir.exists() + assert fresh_dir.exists() + data = json.loads(retiring_json.read_text()) + assert len(data["retiring"]) == 1 + assert data["retiring"][0]["version"] == "0.8.0" + + +# --------------------------------------------------------------------------- +# Category E — Sweep error handling +# --------------------------------------------------------------------------- + + +def test_sweep_handles_already_deleted_dir( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(Path, "home", lambda: tmp_path) + from autoskillit.core._plugin_cache import sweep_retiring_cache + + missing_dir = tmp_path / "cache" / "0.8.0" + # Deliberately do NOT create the directory + old_ts = (datetime.now(UTC) - timedelta(hours=30)).isoformat() + retiring_json = tmp_path / ".autoskillit" / "retiring_cache.json" + retiring_json.parent.mkdir(parents=True, exist_ok=True) + retiring_json.write_text( + json.dumps( + { + "retiring": [{"version": "0.8.0", "path": str(missing_dir), "retired_at": old_ts}], + "schema_version": 1, + } + ) + ) + + count = sweep_retiring_cache(grace_hours=24) # must not raise + + assert count == 1 + data = json.loads(retiring_json.read_text()) + assert data["retiring"] == [] + + +def test_sweep_noop_when_file_missing(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(Path, "home", lambda: tmp_path) + from autoskillit.core._plugin_cache import sweep_retiring_cache + + count = sweep_retiring_cache() + assert count == 0 + + +def test_sweep_handles_malformed_json(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(Path, "home", lambda: tmp_path) + from autoskillit.core._plugin_cache import sweep_retiring_cache + + retiring_json = tmp_path / ".autoskillit" / "retiring_cache.json" + retiring_json.parent.mkdir(parents=True, exist_ok=True) + retiring_json.write_text("not valid json {{{") + + count = sweep_retiring_cache() # must not raise + + assert count == 0 + + +def test_sweep_handles_missing_retired_at(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(Path, "home", lambda: tmp_path) + from autoskillit.core._plugin_cache import sweep_retiring_cache + + old_dir = tmp_path / "cache" / "0.8.0" + old_dir.mkdir(parents=True) + retiring_json = tmp_path / ".autoskillit" / "retiring_cache.json" + retiring_json.parent.mkdir(parents=True, exist_ok=True) + retiring_json.write_text( + json.dumps({"retiring": [{"version": "0.8.0", "path": str(old_dir)}], "schema_version": 1}) + ) + + sweep_retiring_cache() # must not raise + + data = json.loads(retiring_json.read_text()) + assert len(data["retiring"]) == 1, ( + "Entry with missing retired_at must be preserved in survivors" + ) + assert data["retiring"][0]["version"] == "0.8.0" + + +# --------------------------------------------------------------------------- +# Category F — Install locking +# --------------------------------------------------------------------------- + + +def test_install_lock_creates_lock_file(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(Path, "home", lambda: tmp_path) + from autoskillit.core._plugin_cache import _InstallLock + + lock_path = tmp_path / ".autoskillit" / "install.lock" + with _InstallLock(): + assert lock_path.exists() + # File still exists after release (fcntl lock is released, not deleted) + assert lock_path.exists() + + +def test_install_lock_blocks_concurrent_access( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(Path, "home", lambda: tmp_path) + from autoskillit.core._plugin_cache import _install_lock_path + + lock_file_path = _install_lock_path() + lock_file_path.parent.mkdir(parents=True, exist_ok=True) + + acquired = threading.Event() + release = threading.Event() + + def hold_lock() -> None: + with open(lock_file_path, "w") as fh: + fcntl.flock(fh, fcntl.LOCK_EX) + acquired.set() + release.wait(timeout=5) + + t = threading.Thread(target=hold_lock, daemon=True) + t.start() + assert acquired.wait(timeout=2), "background lock thread did not acquire within 2s" + + # Try non-blocking acquire — must fail while first holder has it + with open(lock_file_path, "w") as fh2: + try: + fcntl.flock(fh2, fcntl.LOCK_EX | fcntl.LOCK_NB) + blocked = False + fcntl.flock(fh2, fcntl.LOCK_UN) + except OSError: + blocked = True + + release.set() + t.join(timeout=2) + assert not t.is_alive(), "lock-holding thread did not exit within 2s after release" + + assert blocked, "Second acquire must be blocked while first holder has the lock" + + +# --------------------------------------------------------------------------- +# Category G — Retiring registry integrity +# --------------------------------------------------------------------------- + + +def test_append_preserves_existing_entries( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(Path, "home", lambda: tmp_path) + from autoskillit.core._plugin_cache import append_retiring_entry + + append_retiring_entry("0.7.0", "/some/path/0.7.0") + append_retiring_entry("0.8.0", "/some/path/0.8.0") + + retiring_json = tmp_path / ".autoskillit" / "retiring_cache.json" + data = json.loads(retiring_json.read_text()) + versions = [e["version"] for e in data["retiring"]] + assert "0.7.0" in versions + assert "0.8.0" in versions + + +def test_registry_path_is_absolute(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(Path, "home", lambda: tmp_path) + from autoskillit.core._plugin_cache import append_retiring_entry + + abs_path = str(tmp_path / "cache" / "0.8.0") + append_retiring_entry("0.8.0", abs_path) + + retiring_json = tmp_path / ".autoskillit" / "retiring_cache.json" + data = json.loads(retiring_json.read_text()) + assert len(data["retiring"]) == 1 + assert Path(data["retiring"][0]["path"]).is_absolute() + + +# --------------------------------------------------------------------------- +# Category K — Kitchen registry +# --------------------------------------------------------------------------- + + +def test_register_creates_entry(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(Path, "home", lambda: tmp_path) + from autoskillit.core._plugin_cache import register_active_kitchen + + kitchen_id = "test-kitchen-001" + pid = os.getpid() + project_path = str(tmp_path) + + register_active_kitchen(kitchen_id, pid, project_path) + + akp = tmp_path / ".autoskillit" / "active_kitchens.json" + data = json.loads(akp.read_text()) + kitchens = data["kitchens"] + assert len(kitchens) == 1 + assert kitchens[0]["kitchen_id"] == kitchen_id + assert kitchens[0]["pid"] == pid + assert kitchens[0]["project_path"] == project_path + assert kitchens[0]["create_time"] is not None + + +def test_unregister_removes_entry(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(Path, "home", lambda: tmp_path) + from autoskillit.core._plugin_cache import register_active_kitchen, unregister_active_kitchen + + kitchen_id = "test-kitchen-002" + register_active_kitchen(kitchen_id, os.getpid(), str(tmp_path)) + unregister_active_kitchen(kitchen_id) + + akp = tmp_path / ".autoskillit" / "active_kitchens.json" + data = json.loads(akp.read_text()) + assert data["kitchens"] == [] + + +def test_any_kitchen_open_false_when_pid_dead( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(Path, "home", lambda: tmp_path) + from autoskillit.core._plugin_cache import any_kitchen_open, register_active_kitchen + + proc = subprocess.Popen(["true"]) + proc.wait() + dead_pid = proc.pid + register_active_kitchen("test-kitchen-003", dead_pid, str(tmp_path)) + + result = any_kitchen_open() + assert result is False + + +def test_any_kitchen_open_sweeps_stale(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(Path, "home", lambda: tmp_path) + from autoskillit.core._plugin_cache import any_kitchen_open, register_active_kitchen + + proc = subprocess.Popen(["true"]) + proc.wait() + dead_pid = proc.pid + register_active_kitchen("test-kitchen-004", dead_pid, str(tmp_path)) + + any_kitchen_open() + + akp = tmp_path / ".autoskillit" / "active_kitchens.json" + data = json.loads(akp.read_text()) + assert data["kitchens"] == [] + + +def test_clear_kitchens_for_pid(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(Path, "home", lambda: tmp_path) + from autoskillit.core._plugin_cache import clear_kitchens_for_pid, register_active_kitchen + + pid = os.getpid() + register_active_kitchen("test-kitchen-005a", pid, str(tmp_path)) + register_active_kitchen("test-kitchen-005b", pid, str(tmp_path)) + + clear_kitchens_for_pid(pid) + + akp = tmp_path / ".autoskillit" / "active_kitchens.json" + data = json.loads(akp.read_text()) + assert data["kitchens"] == [] + + +def test_any_kitchen_open_true_for_live_pid( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(Path, "home", lambda: tmp_path) + from autoskillit.core._plugin_cache import any_kitchen_open, register_active_kitchen + + register_active_kitchen("test-kitchen-006", os.getpid(), str(tmp_path)) + + result = any_kitchen_open() + assert result is True + + +def test_any_kitchen_open_scoped_excludes_other_project( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(Path, "home", lambda: tmp_path) + from autoskillit.core._plugin_cache import any_kitchen_open, register_active_kitchen + + project_a = "/project_A" + project_b = "/project_B" + register_active_kitchen("test-kitchen-007", os.getpid(), project_a) + + assert any_kitchen_open(project_path=project_b) is False + assert any_kitchen_open(project_path=project_a) is True + + +def test_any_kitchen_open_no_project_path_returns_global( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(Path, "home", lambda: tmp_path) + from autoskillit.core._plugin_cache import any_kitchen_open, register_active_kitchen + + register_active_kitchen("test-kitchen-008", os.getpid(), "/project_A") + + assert any_kitchen_open() is True + + +# --------------------------------------------------------------------------- +# Category G — Plugin cache integrity validation +# --------------------------------------------------------------------------- + + +def test_stale_cache_after_reorg_detected(tmp_path: Path) -> None: + """validate_plugin_cache_hooks must detect flat-path commands after directory reorganization. + + Reproduces issue #1740: hooks.json frozen with pre-reorg flat paths while scripts + have moved to subdirectories. + """ + from autoskillit.hook_registry import validate_plugin_cache_hooks + + fake_cache = tmp_path / "cache" + version_dir = fake_cache / "0.9.347" + version_dir.mkdir(parents=True) + + stale_hooks_json = { + "hooks": { + "PreToolUse": [ + { + "matcher": ".*", + "hooks": [ + { + "type": "command", + "command": f"python3 {tmp_path}/pkg/hooks/quota_guard.py", + }, + { + "type": "command", + "command": f"python3 {tmp_path}/pkg/hooks/pretty_output_hook.py", + }, + ], + } + ] + } + } + (version_dir / "hooks.json").write_text(json.dumps(stale_hooks_json)) + + broken = validate_plugin_cache_hooks(cache_dir=fake_cache) + + assert len(broken) == 2 + assert any("quota_guard.py" in cmd for cmd in broken) + assert any("pretty_output_hook.py" in cmd for cmd in broken) diff --git a/tests/cli/test_preview.py b/tests/cli/test_preview.py new file mode 100644 index 0000000000..b04ca69032 --- /dev/null +++ b/tests/cli/test_preview.py @@ -0,0 +1,106 @@ +"""Tests for the shared pre-launch preview module (_preview.py).""" + +from __future__ import annotations + +import inspect +from pathlib import Path + +import pytest + +from autoskillit.cli._preview import ( + _render_pre_launch_preview, + show_campaign_preview, + show_cook_preview, +) + +pytestmark = [pytest.mark.layer("cli"), pytest.mark.small] + + +def _make_recipe(monkeypatch: pytest.MonkeyPatch, *, has_ingredients: bool = True) -> object: + from autoskillit.recipe.io import _parse_recipe + + monkeypatch.setattr( + "autoskillit.config.resolve_ingredient_defaults", + lambda _: {"source_dir": "https://github.com/test/repo"}, + ) + ingredients = ( + {"task": {"description": "What to build", "required": True}} if has_ingredients else {} + ) + return _parse_recipe( + { + "name": "preview-test", + "steps": { + "do": { + "tool": "run_cmd", + "with": {"cmd": "echo hi"}, + "on_success": "done", + "on_failure": "done", + }, + "done": {"action": "stop", "message": "ok"}, + }, + "ingredients": ingredients, + } + ) + + +class TestRenderPreLaunchPreview: + def test_renders_ingredients_when_present( + self, capsys: pytest.CaptureFixture[str], monkeypatch: pytest.MonkeyPatch, tmp_path: Path + ) -> None: + recipe = _make_recipe(monkeypatch) + _render_pre_launch_preview("preview-test", recipe, tmp_path, tmp_path) + out = capsys.readouterr().out + assert "task" in out + assert "(required)" in out + + def test_skips_diagram_when_none( + self, capsys: pytest.CaptureFixture[str], monkeypatch: pytest.MonkeyPatch, tmp_path: Path + ) -> None: + recipe = _make_recipe(monkeypatch) + _render_pre_launch_preview("nonexistent-recipe", recipe, tmp_path, tmp_path) + out = capsys.readouterr().out + assert "task" in out + assert "RECIPE" not in out + + def test_skips_ingredients_when_empty( + self, capsys: pytest.CaptureFixture[str], monkeypatch: pytest.MonkeyPatch, tmp_path: Path + ) -> None: + recipe = _make_recipe(monkeypatch, has_ingredients=False) + _render_pre_launch_preview("nonexistent-recipe", recipe, tmp_path, tmp_path) + out = capsys.readouterr().out + assert out.strip() == "" + + +class TestShowCookPreview: + def test_signature_matches_expected(self) -> None: + sig = inspect.signature(show_cook_preview) + params = list(sig.parameters) + assert params == ["recipe_name", "parsed_recipe", "recipes_dir", "project_dir"] + + def test_produces_terminal_output( + self, capsys: pytest.CaptureFixture[str], monkeypatch: pytest.MonkeyPatch, tmp_path: Path + ) -> None: + recipe = _make_recipe(monkeypatch) + show_cook_preview("preview-test", recipe, tmp_path, tmp_path) + out = capsys.readouterr().out + assert "task" in out + assert "(required)" in out + + +class TestShowCampaignPreview: + def test_renders_ingredients_for_campaign( + self, capsys: pytest.CaptureFixture[str], monkeypatch: pytest.MonkeyPatch, tmp_path: Path + ) -> None: + recipe = _make_recipe(monkeypatch) + show_campaign_preview("preview-test", recipe, tmp_path, tmp_path) + out = capsys.readouterr().out + assert "task" in out + + def test_skips_diagram_gracefully( + self, capsys: pytest.CaptureFixture[str], monkeypatch: pytest.MonkeyPatch, tmp_path: Path + ) -> None: + recipe = _make_recipe(monkeypatch) + show_campaign_preview("no-such-campaign-recipe", recipe, tmp_path, tmp_path) + out = capsys.readouterr().out + assert "task" in out + assert "RECIPE" not in out diff --git a/tests/cli/test_reap.py b/tests/cli/test_reap.py new file mode 100644 index 0000000000..c445f835ad --- /dev/null +++ b/tests/cli/test_reap.py @@ -0,0 +1,227 @@ +"""Tests for _reap_stale_dispatches in cli/_fleet.py (Group J).""" + +from __future__ import annotations + +import json +from pathlib import Path +from unittest.mock import patch + +import pytest + +from autoskillit.fleet import ( + DispatchRecord, + DispatchStatus, + read_state, + write_initial_state, +) + +pytestmark = [pytest.mark.layer("cli"), pytest.mark.small, pytest.mark.feature("fleet")] + +BOOT_ID = "boot-abc-123" +OTHER_BOOT_ID = "boot-xyz-999" + + +def _make_running_state( + tmp_path: Path, + *, + dispatch_name: str = "d1", + dispatched_pid: int = 12345, + dispatched_starttime_ticks: int = 1000, + dispatched_boot_id: str = BOOT_ID, +) -> Path: + """Create a state file with a single RUNNING dispatch.""" + sp = tmp_path / "state.json" + write_initial_state( + sp, "cid-reap", "reap-campaign", "/m.yaml", [DispatchRecord(name=dispatch_name)] + ) + # Inject RUNNING state directly into the JSON to bypass transition validation + raw = json.loads(sp.read_text()) + raw["dispatches"][0].update( + { + "status": "running", + "dispatch_id": "did-reap", + "dispatched_pid": dispatched_pid, + "dispatched_starttime_ticks": dispatched_starttime_ticks, + "dispatched_boot_id": dispatched_boot_id, + "started_at": 1000.0, + } + ) + sp.write_text(json.dumps(raw)) + return sp + + +def _reap(state_path: Path, *, dry_run: bool = False) -> None: + from autoskillit.cli.fleet import _reap_stale_dispatches + + _reap_stale_dispatches(state_path, dry_run=dry_run) + + +class TestReap: + def test_reap_kills_orphan(self, tmp_path: Path) -> None: + sp = _make_running_state(tmp_path, dispatched_pid=12345, dispatched_starttime_ticks=1000) + with ( + patch("psutil.pid_exists", return_value=True), + patch( + "autoskillit.execution.read_starttime_ticks", + return_value=1000, + ), + patch( + "autoskillit.execution.read_boot_id", + return_value=BOOT_ID, + ), + patch("autoskillit.execution.kill_process_tree") as mock_kill, + ): + _reap(sp) + + mock_kill.assert_called_once_with(12345) + state = read_state(sp) + assert state is not None + assert state.dispatches[0].status == DispatchStatus.INTERRUPTED + assert state.dispatches[0].reason == "reaped_orphan" + + def test_reap_skips_recycled_pid(self, tmp_path: Path) -> None: + sp = _make_running_state(tmp_path, dispatched_pid=12345, dispatched_starttime_ticks=1000) + with ( + patch("psutil.pid_exists", return_value=True), + patch( + "autoskillit.execution.read_starttime_ticks", + return_value=9999, # different ticks → recycled + ), + patch( + "autoskillit.execution.read_boot_id", + return_value=BOOT_ID, + ), + patch("autoskillit.execution.kill_process_tree") as mock_kill, + ): + _reap(sp) + + mock_kill.assert_not_called() + state = read_state(sp) + assert state is not None + assert state.dispatches[0].status == DispatchStatus.INTERRUPTED + assert state.dispatches[0].reason == "reaped_pid_recycled" + + def test_reap_marks_dead_pid(self, tmp_path: Path) -> None: + sp = _make_running_state(tmp_path, dispatched_pid=12345) + with ( + patch("psutil.pid_exists", return_value=False), + patch( + "autoskillit.execution.read_boot_id", + return_value=BOOT_ID, + ), + patch("autoskillit.execution.kill_process_tree") as mock_kill, + ): + _reap(sp) + + mock_kill.assert_not_called() + state = read_state(sp) + assert state is not None + assert state.dispatches[0].status == DispatchStatus.INTERRUPTED + assert state.dispatches[0].reason == "reaped_dead_pid" + + def test_reap_idempotent(self, tmp_path: Path) -> None: + sp = _make_running_state(tmp_path, dispatched_pid=12345) + with ( + patch("psutil.pid_exists", return_value=False), + patch( + "autoskillit.execution.read_boot_id", + return_value=BOOT_ID, + ), + ): + _reap(sp) + # Second invocation: dispatch is now INTERRUPTED → no-op + _reap(sp) + + state = read_state(sp) + assert state is not None + assert state.dispatches[0].status == DispatchStatus.INTERRUPTED + + def test_reap_no_running_dispatches(self, tmp_path: Path) -> None: + sp = tmp_path / "state.json" + write_initial_state( + sp, + "cid-done", + "done-campaign", + "/m.yaml", + [DispatchRecord(name="d1")], + ) + # Mark d1 as SUCCESS directly in JSON + raw = json.loads(sp.read_text()) + raw["dispatches"][0]["status"] = "success" + sp.write_text(json.dumps(raw)) + + with ( + patch( + "autoskillit.execution.read_boot_id", + return_value=BOOT_ID, + ), + patch("psutil.pid_exists") as mock_pid, + ): + _reap(sp) + + mock_pid.assert_not_called() + state = read_state(sp) + assert state is not None + assert state.dispatches[0].status == DispatchStatus.SUCCESS + + def test_reap_skips_kill_after_reboot(self, tmp_path: Path) -> None: + """Dispatch with different dispatched_boot_id → machine rebooted, no kill.""" + sp = _make_running_state( + tmp_path, + dispatched_pid=12345, + dispatched_starttime_ticks=1000, + dispatched_boot_id=OTHER_BOOT_ID, + ) + with ( + patch( + "autoskillit.execution.read_boot_id", + return_value=BOOT_ID, # current boot differs from stored + ), + patch("psutil.pid_exists", return_value=True), + patch("autoskillit.execution.kill_process_tree") as mock_kill, + ): + _reap(sp) + + mock_kill.assert_not_called() + state = read_state(sp) + assert state is not None + assert state.dispatches[0].reason == "reaped_pid_recycled" + + def test_reap_dry_run_does_not_modify_state(self, tmp_path: Path) -> None: + sp = _make_running_state(tmp_path, dispatched_pid=12345, dispatched_starttime_ticks=1000) + original_text = sp.read_text() + + with ( + patch("psutil.pid_exists", return_value=True), + patch( + "autoskillit.execution.read_starttime_ticks", + return_value=1000, + ), + patch( + "autoskillit.execution.read_boot_id", + return_value=BOOT_ID, + ), + patch("autoskillit.execution.kill_process_tree"), + ): + _reap(sp, dry_run=True) + + assert sp.read_text() == original_text + + def test_reap_concurrent_flock(self, tmp_path: Path) -> None: + """Two concurrent reap calls — second sees terminal states and skips cleanly.""" + sp = _make_running_state(tmp_path, dispatched_pid=12345) + + with ( + patch("psutil.pid_exists", return_value=False), + patch( + "autoskillit.execution.read_boot_id", + return_value=BOOT_ID, + ), + ): + _reap(sp) + # Second call: dispatch is already INTERRUPTED; _reap must not raise + _reap(sp) + + state = read_state(sp) + assert state is not None + assert state.dispatches[0].status == DispatchStatus.INTERRUPTED diff --git a/tests/cli/test_reap_sidecar_check.py b/tests/cli/test_reap_sidecar_check.py new file mode 100644 index 0000000000..b30fb09446 --- /dev/null +++ b/tests/cli/test_reap_sidecar_check.py @@ -0,0 +1,83 @@ +"""Tests for _reap_stale_dispatches sidecar-aware status transition (T-RESUMABLE-10).""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from autoskillit.fleet import ( + DispatchRecord, + DispatchStatus, + mark_dispatch_running, + read_state, + write_initial_state, +) + +pytestmark = [pytest.mark.layer("cli"), pytest.mark.small, pytest.mark.feature("fleet")] + + +def _state_path(tmp_path: Path) -> Path: + p = tmp_path / "campaign" / "state.json" + p.parent.mkdir(parents=True, exist_ok=True) + return p + + +def test_reap_stale_dispatches_marks_resumable_when_sidecar_exists(tmp_path: Path) -> None: + from autoskillit.cli.fleet import _reap_stale_dispatches + + sp = _state_path(tmp_path) + write_initial_state(sp, "c1", "camp", "manifest.yaml", [DispatchRecord(name="impl")]) + sidecar_file = sp.parent / "d1111_issues.jsonl" + mark_dispatch_running( + sp, "impl", dispatch_id="d1111", dispatched_pid=0, sidecar_path=str(sidecar_file) + ) + sidecar_file.write_text( + '{"issue_url":"https://github.com/o/r/issues/1","status":"completed","ts":"2026-01-01T00:00:00"}\n' + ) + + _reap_stale_dispatches(sp) + + state = read_state(sp) + assert state is not None + matches = [d for d in state.dispatches if d.name == "impl"] + assert matches, "no dispatch named 'impl' found" + assert matches[-1].status == DispatchStatus.RESUMABLE + + +def test_reap_stale_dispatches_marks_interrupted_when_no_sidecar(tmp_path: Path) -> None: + from autoskillit.cli.fleet import _reap_stale_dispatches + + sp = _state_path(tmp_path) + write_initial_state(sp, "c1", "camp", "manifest.yaml", [DispatchRecord(name="impl")]) + mark_dispatch_running(sp, "impl", dispatch_id="d1111", dispatched_pid=0) + + _reap_stale_dispatches(sp) + + state = read_state(sp) + assert state is not None + matches = [d for d in state.dispatches if d.name == "impl"] + assert matches, "no dispatch named 'impl' found" + assert matches[-1].status == DispatchStatus.INTERRUPTED + + +def test_reap_stale_dispatches_dry_run_does_not_modify_state(tmp_path: Path) -> None: + from autoskillit.cli.fleet import _reap_stale_dispatches + + sp = _state_path(tmp_path) + write_initial_state(sp, "c1", "camp", "manifest.yaml", [DispatchRecord(name="impl")]) + sidecar_file = sp.parent / "d1111_issues.jsonl" + mark_dispatch_running( + sp, "impl", dispatch_id="d1111", dispatched_pid=0, sidecar_path=str(sidecar_file) + ) + sidecar_file.write_text( + '{"issue_url":"https://github.com/o/r/issues/1","status":"completed","ts":"2026-01-01T00:00:00"}\n' + ) + + _reap_stale_dispatches(sp, dry_run=True) + + state = read_state(sp) + assert state is not None + matches = [d for d in state.dispatches if d.name == "impl"] + assert matches, "no dispatch named 'impl' found" + assert matches[-1].status == DispatchStatus.RUNNING diff --git a/tests/cli/test_reload_loop.py b/tests/cli/test_reload_loop.py new file mode 100644 index 0000000000..eae47c58f1 --- /dev/null +++ b/tests/cli/test_reload_loop.py @@ -0,0 +1,257 @@ +"""Tests for the session reload sentinel and loop mechanics.""" + +from __future__ import annotations + +import json +import shutil +import subprocess +from contextlib import contextmanager +from pathlib import Path + +import pytest + +pytestmark = [pytest.mark.layer("cli"), pytest.mark.small] + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + + +@contextmanager +def _noop_terminal_guard(): # type: ignore[misc] + yield + + +def _make_result(returncode: int = 0) -> object: + return type("Result", (), {"returncode": returncode})() + + +def _write_sentinel(project_dir: Path, session_id: str) -> Path: + sentinel_dir = project_dir / ".autoskillit" / "temp" / "reload_sentinel" + sentinel_dir.mkdir(parents=True, exist_ok=True) + sentinel = sentinel_dir / f"{session_id}.json" + sentinel.write_text( + json.dumps({"session_id": session_id, "requested_at": "2026-01-01T00:00:00+00:00"}), + encoding="utf-8", + ) + return sentinel + + +# --------------------------------------------------------------------------- +# RL-1 — _run_cook_session returns None on normal exit (no sentinel) +# --------------------------------------------------------------------------- + + +def test_cook_session_no_sentinel_returns_none( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(subprocess, "run", lambda *a, **kw: _make_result(0)) + monkeypatch.setattr("autoskillit.cli.session._cook.terminal_guard", _noop_terminal_guard) + + from autoskillit.cli.session._cook import _run_cook_session + + result = _run_cook_session( + cmd=["claude"], + env={}, + _first_run=False, + initial_prompt=None, + project_dir=tmp_path, + ) + assert result is None + + +# --------------------------------------------------------------------------- +# RL-2 — _run_cook_session returns session_id when sentinel file exists +# --------------------------------------------------------------------------- + + +def test_cook_session_with_sentinel_returns_session_id( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + _write_sentinel(tmp_path, "sess-001") + monkeypatch.setattr(subprocess, "run", lambda *a, **kw: _make_result(0)) + monkeypatch.setattr("autoskillit.cli.session._cook.terminal_guard", _noop_terminal_guard) + + from autoskillit.cli.session._cook import _run_cook_session + + result = _run_cook_session( + cmd=["claude"], + env={}, + _first_run=False, + initial_prompt=None, + project_dir=tmp_path, + ) + assert result == "sess-001" + + +# --------------------------------------------------------------------------- +# RL-3 — _run_cook_session deletes sentinel after reading it +# --------------------------------------------------------------------------- + + +def test_cook_session_sentinel_consumed_after_reload( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + sentinel = _write_sentinel(tmp_path, "sess-del") + assert sentinel.exists() + + monkeypatch.setattr(subprocess, "run", lambda *a, **kw: _make_result(0)) + monkeypatch.setattr("autoskillit.cli.session._cook.terminal_guard", _noop_terminal_guard) + + from autoskillit.cli.session._cook import _run_cook_session + + _run_cook_session( + cmd=["claude"], + env={}, + _first_run=False, + initial_prompt=None, + project_dir=tmp_path, + ) + assert not sentinel.exists() + + +# --------------------------------------------------------------------------- +# RL-4 — cook() reload loop rebuilds with NamedResume on reload +# --------------------------------------------------------------------------- + + +def test_cook_reload_loop_uses_named_resume( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + from autoskillit.execution.commands import ClaudeInteractiveCmd + + run_count = [0] + captured_resume_specs: list = [] + + def fake_run_cook_session(*, cmd, env, _first_run, initial_prompt, project_dir): + run_count[0] += 1 + if run_count[0] == 1: + return "sess-001" + return None + + def fake_build_interactive_cmd(**kwargs): + captured_resume_specs.append(kwargs.get("resume_spec")) + return ClaudeInteractiveCmd(cmd=["claude"], env={}) + + monkeypatch.chdir(tmp_path) + monkeypatch.setattr(shutil, "which", lambda x: "/usr/bin/claude") + monkeypatch.setattr("sys.stdin.isatty", lambda: True) + monkeypatch.setattr("builtins.input", lambda _prompt="": "") + monkeypatch.setattr("autoskillit.cli._onboarding.is_first_run", lambda _: False) + monkeypatch.setattr("autoskillit.cli.session._cook._run_cook_session", fake_run_cook_session) + monkeypatch.setattr("autoskillit.execution.build_interactive_cmd", fake_build_interactive_cmd) + + from autoskillit.workspace.session_skills import DefaultSessionSkillManager + + fake_skills_dir = tmp_path / "fake-skills" + fake_skills_dir.mkdir() + monkeypatch.setattr( + DefaultSessionSkillManager, + "init_session", + lambda self, sid, *, cook_session=False, config=None, project_dir=None: fake_skills_dir, + ) + + from autoskillit import cli + from autoskillit.core import NamedResume + + cli.cook() + + assert run_count[0] == 2 + assert len(captured_resume_specs) == 2 + assert isinstance(captured_resume_specs[1], NamedResume) + assert captured_resume_specs[1].session_id == "sess-001" + + +# --------------------------------------------------------------------------- +# RL-5 — _run_interactive_session returns session_id when sentinel exists +# --------------------------------------------------------------------------- + + +def test_interactive_session_reload_uses_named_resume( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + _write_sentinel(tmp_path, "isess-001") + monkeypatch.setattr(shutil, "which", lambda _: "/usr/bin/claude") + monkeypatch.setattr(subprocess, "run", lambda *a, **kw: _make_result(0)) + monkeypatch.setattr("autoskillit.cli.ui._terminal.terminal_guard", _noop_terminal_guard) + monkeypatch.setattr("autoskillit.cli._init_helpers._is_plugin_installed", lambda: True) + + from autoskillit.cli.session._session_launch import _run_interactive_session + + result = _run_interactive_session(system_prompt="test", project_dir=tmp_path) + assert result == "isess-001" + + +# --------------------------------------------------------------------------- +# RL-6 — Fleet reload re-launches with same system_prompt, no --resume +# --------------------------------------------------------------------------- + + +@pytest.mark.feature("fleet") +def test_fleet_reload_relaunches_without_resume( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + from autoskillit.core import NamedResume, NoResume + + call_count = [0] + captured_resume_specs: list = [] + + def fake_run_interactive_session( + prompt, *, extra_env=None, resume_spec=None, project_dir=None, initial_message=None + ): + call_count[0] += 1 + captured_resume_specs.append(resume_spec) + if call_count[0] == 1: + return "franchise-sess" + return None + + monkeypatch.setattr( + "autoskillit.cli.session._session_launch._run_interactive_session", + fake_run_interactive_session, + ) + monkeypatch.setattr("autoskillit.cli.detect_autoskillit_mcp_prefix", lambda: "autoskillit") + monkeypatch.setattr( + "autoskillit.cli._prompts._build_fleet_dispatch_prompt", + lambda mcp_prefix, recipe_table=None: "test-prompt", + ) + monkeypatch.chdir(tmp_path) + + from autoskillit.cli.fleet import _launch_fleet_session + + _launch_fleet_session( + campaign_recipe=None, + campaign_id=None, + state_path=None, + resume_metadata=None, + fleet_mode="dispatch", + ) + + assert call_count[0] == 2 + assert isinstance(captured_resume_specs[0], NoResume) + assert isinstance(captured_resume_specs[1], NamedResume) + assert captured_resume_specs[1].session_id == "franchise-sess" + + +# --------------------------------------------------------------------------- +# RL-7 — Non-zero exit without sentinel raises SystemExit +# --------------------------------------------------------------------------- + + +def test_non_zero_exit_without_sentinel_raises_system_exit( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(subprocess, "run", lambda *a, **kw: _make_result(42)) + monkeypatch.setattr("autoskillit.cli.session._cook.terminal_guard", _noop_terminal_guard) + + from autoskillit.cli.session._cook import _run_cook_session + + with pytest.raises(SystemExit) as exc_info: + _run_cook_session( + cmd=["claude"], + env={}, + _first_run=False, + initial_prompt=None, + project_dir=tmp_path, + ) + assert exc_info.value.code == 42 diff --git a/tests/cli/test_restart.py b/tests/cli/test_restart.py new file mode 100644 index 0000000000..4ba3595e9a --- /dev/null +++ b/tests/cli/test_restart.py @@ -0,0 +1,44 @@ +"""Tests for cli/_restart.py — NoReturn process restart contract.""" + +from __future__ import annotations + +import os +import sys +from unittest.mock import patch + +import pytest + +pytestmark = [pytest.mark.layer("cli"), pytest.mark.small] + + +def test_perform_restart_sets_skip_env_and_execs() -> None: + """perform_restart must set AUTOSKILLIT_SKIP_UPDATE_CHECK=1 then call os.execv.""" + from autoskillit.cli._restart import perform_restart + + captured: dict[str, object] = {} + + def fake_execv(exe: str, args: list[str]) -> None: + assert os.environ.get("AUTOSKILLIT_SKIP_UPDATE_CHECK") == "1" + captured["exe"] = exe + captured["args"] = args + raise SystemExit(0) + + with patch.dict(os.environ, {}, clear=False): + with patch("autoskillit.cli._restart.os.execv", fake_execv): + try: + perform_restart() + except SystemExit: + pass + + assert captured["exe"] == sys.executable + assert captured["args"] == [sys.executable] + sys.argv + + +def test_perform_restart_is_noreturn_typed() -> None: + """perform_restart must be annotated as -> NoReturn.""" + import typing + + from autoskillit.cli._restart import perform_restart + + hints = typing.get_type_hints(perform_restart) + assert hints.get("return") is typing.NoReturn diff --git a/tests/cli/test_routing_completeness.py b/tests/cli/test_routing_completeness.py new file mode 100644 index 0000000000..143cb18d25 --- /dev/null +++ b/tests/cli/test_routing_completeness.py @@ -0,0 +1,92 @@ +"""Routing completeness guard: every orchestrator-visible RetryReason must have a routing rule. + +Prevents future RetryReason additions from silently missing routing rules in the +orchestrator prompt — the same class of oversight that produced the EMPTY_OUTPUT bug. +""" + +from __future__ import annotations + +import pytest + +from autoskillit.cli._mcp_names import DIRECT_PREFIX +from autoskillit.core.types import RetryReason + +pytestmark = [pytest.mark.layer("cli"), pytest.mark.small] + +# Reasons excluded from orchestrator-prompt routing check: +# - NONE: not a retry scenario, no routing needed +# - BUDGET_EXHAUSTED: caps other reasons; orchestrator never sees it directly +# - CONTRACT_RECOVERY: handled by infrastructure nudge in headless/__init__.py +# - CLONE_CONTAMINATION: handled by clone_guard infrastructure +_ROUTING_EXCLUDED = { + RetryReason.NONE, + RetryReason.BUDGET_EXHAUSTED, + RetryReason.CONTRACT_RECOVERY, + RetryReason.CLONE_CONTAMINATION, +} + +# Routing contract: RetryReason → (expected_route_keyword, evidence_condition_keyword_or_None) +_EXPECTED_ROUTES: dict[RetryReason, tuple[str, str | None]] = { + RetryReason.RESUME: ("on_context_limit", "subtype"), + RetryReason.STALE: ("on_failure", None), + RetryReason.DRAIN_RACE: ("on_context_limit", None), + RetryReason.COMPLETED_NO_FLUSH: ("on_context_limit", None), + RetryReason.EMPTY_OUTPUT: ("on_failure", None), + RetryReason.PATH_CONTAMINATION: ("on_failure", None), + RetryReason.THINKING_STALL: ("on_context_limit", "lifespan_started"), + RetryReason.IDLE_STALL: ("on_context_limit", "lifespan_started"), + RetryReason.EARLY_STOP: ("on_context_limit", "has_progress_evidence"), + RetryReason.ZERO_WRITES: ("on_context_limit", "has_progress_evidence"), +} + + +def test_all_retry_reasons_have_routing_rules() -> None: + """Every orchestrator-visible RetryReason must have an explicit routing rule.""" + from autoskillit.cli._prompts import _build_orchestrator_prompt + + prompt_text = _build_orchestrator_prompt("test-recipe", mcp_prefix=DIRECT_PREFIX) + missing = [] + for reason in RetryReason: + if reason in _ROUTING_EXCLUDED: + continue + if reason.value not in prompt_text: + missing.append(reason.name) + + assert not missing, ( + f"RetryReason values missing routing rules in orchestrator prompt: {missing}" + ) + + +@pytest.mark.parametrize( + "reason,expected", + _EXPECTED_ROUTES.items(), + ids=[r.value for r in _EXPECTED_ROUTES], +) +def test_reason_routes_to_expected_destination( + reason: RetryReason, + expected: tuple[str, str | None], +) -> None: + """Every RetryReason must route to its declared destination in _prompts.py.""" + from autoskillit.cli._prompts import _build_orchestrator_prompt + + prompt_text = _build_orchestrator_prompt("test-recipe", mcp_prefix=DIRECT_PREFIX) + idx = prompt_text.find(reason.value) + assert idx != -1, f"{reason.value} not found in orchestrator prompt" + + window = prompt_text[idx : idx + 600] + route_keyword, evidence_keyword = expected + assert route_keyword in window, ( + f"{reason.value} must reference '{route_keyword}' within 600 chars" + ) + if evidence_keyword: + assert evidence_keyword in window, ( + f"{reason.value} routing must reference evidence signal '{evidence_keyword}'" + ) + + +def test_expected_routes_covers_all_orchestrator_visible_reasons() -> None: + """_EXPECTED_ROUTES must have an entry for every non-excluded RetryReason.""" + missing = [ + r.name for r in RetryReason if r not in _ROUTING_EXCLUDED and r not in _EXPECTED_ROUTES + ] + assert not missing, f"Add routing expectation for: {missing}" diff --git a/tests/cli/test_serve_sigterm.py b/tests/cli/test_serve_sigterm.py index d7395e5a56..3896fe028a 100644 --- a/tests/cli/test_serve_sigterm.py +++ b/tests/cli/test_serve_sigterm.py @@ -12,9 +12,14 @@ from __future__ import annotations +import ast import inspect from unittest.mock import MagicMock +import pytest + +pytestmark = [pytest.mark.layer("cli"), pytest.mark.small] + def test_serve_uses_anyio_run_not_mcp_run(monkeypatch, tmp_path): """serve() routes through anyio.run(), not mcp.run() directly.""" @@ -48,3 +53,81 @@ def capture_anyio_run(coro_fn, *args, **kwargs): assert inspect.iscoroutinefunction(guard_fn), ( f"Expected serve() to pass an async coroutine function to anyio.run(), got {guard_fn!r}" ) + + +def test_serve_passes_uvloop_backend_options(monkeypatch, tmp_path): + """serve() must pass backend='asyncio' and use_uvloop=True (non-Windows) to anyio.run().""" + import sys + + anyio_run_calls: list[dict] = [] + + monkeypatch.chdir(tmp_path) + + mock_cfg = MagicMock() + mock_cfg.logging.level = "INFO" + mock_cfg.logging.json_output = None + mock_cfg.safety.protected_branches = [] + + monkeypatch.setattr("autoskillit.config.load_config", lambda _: mock_cfg) + monkeypatch.setattr("autoskillit.core.configure_logging", lambda **kw: None) + monkeypatch.setattr("autoskillit.server.make_context", lambda *a, **kw: MagicMock()) + monkeypatch.setattr("autoskillit.server._initialize", lambda ctx: None) + monkeypatch.setattr(sys, "platform", "linux") + + def capture_anyio_run(coro_fn, *args, **kwargs): + anyio_run_calls.append({"fn": coro_fn, "kwargs": kwargs}) + + monkeypatch.setattr("anyio.run", capture_anyio_run) + + from autoskillit.cli.app import serve + + serve() + + assert anyio_run_calls, "serve() did not call anyio.run()" + call_kwargs = anyio_run_calls[0]["kwargs"] + assert call_kwargs.get("backend") == "asyncio", ( + f"Expected backend='asyncio', got {call_kwargs.get('backend')!r}. " + "The MCP server must use the asyncio backend with uvloop enabled." + ) + assert call_kwargs.get("backend_options", {}).get("use_uvloop") is True, ( + f"Expected backend_options['use_uvloop']=True, " + f"got {call_kwargs.get('backend_options')!r}. " + "The MCP server must enable uvloop for network I/O throughput." + ) + + +def test_sighup_in_serve_guard_signal_list() -> None: + """_serve_guard.py must pass signal.SIGHUP to open_signal_receiver (AST guard). + + Sending a real SIGHUP in tests is unsafe — before SIGHUP is registered, + the default disposition terminates the process. AST analysis is safe for + both the pre- and post-implementation codebase. + """ + from autoskillit.core.paths import pkg_root + + src_path = pkg_root() / "cli" / "_serve_guard.py" + tree = ast.parse(src_path.read_text(encoding="utf-8")) + + sighup_found = False + for node in ast.walk(tree): + if not isinstance(node, ast.Call): + continue + func = node.func + if not (isinstance(func, ast.Attribute) and func.attr == "open_signal_receiver"): + continue + for arg in node.args: + if ( + isinstance(arg, ast.Attribute) + and arg.attr == "SIGHUP" + and isinstance(arg.value, ast.Name) + and arg.value.id == "signal" + ): + sighup_found = True + break + if sighup_found: + break + + assert sighup_found, ( + "signal.SIGHUP not found in anyio.open_signal_receiver() call in _serve_guard.py. " + "Terminal disconnects (SIGHUP) must trigger graceful shutdown." + ) diff --git a/tests/cli/test_session_launch.py b/tests/cli/test_session_launch.py new file mode 100644 index 0000000000..d9952aeebe --- /dev/null +++ b/tests/cli/test_session_launch.py @@ -0,0 +1,183 @@ +"""Tests for cli/_session_launch.py — _run_interactive_session contract.""" + +from __future__ import annotations + +import shutil +import subprocess + +import pytest + +from autoskillit.cli.session._session_launch import _run_interactive_session +from autoskillit.core import ClaudeFlags + +pytestmark = [pytest.mark.layer("cli"), pytest.mark.small] + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + + +def _capture_subprocess(monkeypatch: pytest.MonkeyPatch) -> dict: + """Replace subprocess.run with a capturing stub. Stubs shutil.which to /usr/bin/claude.""" + captured: dict = {} + monkeypatch.setattr(shutil, "which", lambda _: "/usr/bin/claude") + + def mock_run(cmd, **kwargs): # type: ignore[no-untyped-def] + captured["cmd"] = list(cmd) + captured["env"] = kwargs.get("env", {}) or {} + return type("Result", (), {"returncode": 0, "stdout": "", "stderr": ""})() + + monkeypatch.setattr(subprocess, "run", mock_run) + return captured + + +def _stub_plugin_installed(monkeypatch: pytest.MonkeyPatch, *, installed: bool = True) -> None: + """Stub detect_autoskillit_mcp_prefix to simulate marketplace/direct install.""" + from autoskillit.core._plugin_ids import DIRECT_PREFIX, MARKETPLACE_PREFIX + + prefix = MARKETPLACE_PREFIX if installed else DIRECT_PREFIX + monkeypatch.setattr("autoskillit.core.detect_autoskillit_mcp_prefix", lambda: prefix) + + +# --------------------------------------------------------------------------- +# T13. _session_launch.py — plugin flags when plugin not installed +# --------------------------------------------------------------------------- + + +def test_run_interactive_session_passes_plugin_flags(monkeypatch: pytest.MonkeyPatch) -> None: + """_run_interactive_session adds --plugin-dir when plugin not installed.""" + _stub_plugin_installed(monkeypatch, installed=False) + captured = _capture_subprocess(monkeypatch) + _run_interactive_session(system_prompt="test") + assert ClaudeFlags.PLUGIN_DIR in captured["cmd"] + + +# --------------------------------------------------------------------------- +# T14. _session_launch.py — tool restriction +# --------------------------------------------------------------------------- + + +def test_run_interactive_session_restricts_tools(monkeypatch: pytest.MonkeyPatch) -> None: + """_run_interactive_session passes --tools AskUserQuestion.""" + _stub_plugin_installed(monkeypatch, installed=True) + captured = _capture_subprocess(monkeypatch) + _run_interactive_session(system_prompt="test") + idx = captured["cmd"].index(ClaudeFlags.TOOLS) + assert captured["cmd"][idx + 1] == "AskUserQuestion" + + +# --------------------------------------------------------------------------- +# system prompt appended +# --------------------------------------------------------------------------- + + +def test_run_interactive_session_appends_system_prompt(monkeypatch: pytest.MonkeyPatch) -> None: + """_run_interactive_session appends --append-system-prompt .""" + _stub_plugin_installed(monkeypatch) + captured = _capture_subprocess(monkeypatch) + _run_interactive_session(system_prompt="my-unique-prompt") + assert ClaudeFlags.APPEND_SYSTEM_PROMPT in captured["cmd"] + idx = captured["cmd"].index(ClaudeFlags.APPEND_SYSTEM_PROMPT) + assert captured["cmd"][idx + 1] == "my-unique-prompt" + + +# --------------------------------------------------------------------------- +# env extras passed through +# --------------------------------------------------------------------------- + + +def test_run_interactive_session_extra_env_merged(monkeypatch: pytest.MonkeyPatch) -> None: + """extra_env values appear in the subprocess env.""" + _stub_plugin_installed(monkeypatch) + captured = _capture_subprocess(monkeypatch) + _run_interactive_session(system_prompt="test", extra_env={"MY_UNIQUE_KEY": "MY_VAL"}) + assert captured["env"].get("MY_UNIQUE_KEY") == "MY_VAL" + + +# --------------------------------------------------------------------------- +# exits when claude missing +# --------------------------------------------------------------------------- + + +def test_run_interactive_session_exits_when_claude_missing( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """_run_interactive_session exits 1 when claude is not on PATH.""" + monkeypatch.setattr(shutil, "which", lambda _: None) + with pytest.raises(SystemExit, match="1"): + _run_interactive_session(system_prompt="test") + + +# --------------------------------------------------------------------------- +# no plugin dir when plugin installed +# --------------------------------------------------------------------------- + + +def test_run_interactive_session_no_plugin_dir_when_installed( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """_run_interactive_session omits --plugin-dir when plugin is installed.""" + _stub_plugin_installed(monkeypatch, installed=True) + captured = _capture_subprocess(monkeypatch) + _run_interactive_session(system_prompt="test") + assert ClaudeFlags.PLUGIN_DIR not in captured["cmd"] + + +# --------------------------------------------------------------------------- +# system prompt suppressed for resume sessions +# --------------------------------------------------------------------------- + + +def test_run_interactive_session_suppresses_system_prompt_on_named_resume( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """_run_interactive_session omits --append-system-prompt when resume_spec is NamedResume.""" + from autoskillit.core import NamedResume + + _stub_plugin_installed(monkeypatch) + captured = _capture_subprocess(monkeypatch) + _run_interactive_session( + system_prompt="should-not-appear", + resume_spec=NamedResume(session_id="4b581974-1f19-4aec-8405-78c5ede5e233"), + ) + assert ClaudeFlags.APPEND_SYSTEM_PROMPT not in captured["cmd"] + + +def test_run_interactive_session_suppresses_system_prompt_on_bare_resume( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """_run_interactive_session omits --append-system-prompt when resume_spec is BareResume.""" + from autoskillit.core import BareResume + + _stub_plugin_installed(monkeypatch) + captured = _capture_subprocess(monkeypatch) + _run_interactive_session( + system_prompt="should-not-appear", + resume_spec=BareResume(), + ) + assert ClaudeFlags.APPEND_SYSTEM_PROMPT not in captured["cmd"] + + +def test_session_type_cook_order_in_cli_session() -> None: + from autoskillit.cli.session._constants import SESSION_TYPE_COOK, SESSION_TYPE_ORDER + + assert SESSION_TYPE_COOK == "cook" + assert SESSION_TYPE_ORDER == "order" + + +def test_run_interactive_session_appends_system_prompt_on_fresh_session( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """_run_interactive_session appends --append-system-prompt for fresh (NoResume) sessions.""" + from autoskillit.core import NoResume + + _stub_plugin_installed(monkeypatch) + captured = _capture_subprocess(monkeypatch) + _run_interactive_session( + system_prompt="my-prompt", + resume_spec=NoResume(), + ) + assert ClaudeFlags.APPEND_SYSTEM_PROMPT in captured["cmd"] + idx = captured["cmd"].index(ClaudeFlags.APPEND_SYSTEM_PROMPT) + assert captured["cmd"][idx + 1] == "my-prompt" diff --git a/tests/cli/test_session_picker.py b/tests/cli/test_session_picker.py new file mode 100644 index 0000000000..e2d51e647c --- /dev/null +++ b/tests/cli/test_session_picker.py @@ -0,0 +1,178 @@ +"""Tests for cli/_session_picker.py.""" + +from __future__ import annotations + +import json +from pathlib import Path + +import pytest + +from autoskillit.cli.session._session_picker import ( + _classify_session, + _run_picker, + pick_session, +) +from autoskillit.core.runtime.session_registry import write_registry_entry + +pytestmark = [pytest.mark.layer("cli"), pytest.mark.medium] + + +def _make_index(tmp_path: Path, entries: list[dict]) -> Path: + """Write sessions-index.json into a temp project dir and return that dir.""" + index_path = tmp_path / "sessions-index.json" + index_path.write_text(json.dumps(entries), encoding="utf-8") + return tmp_path + + +def test_pick_session_no_sessions_returns_none( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + project_dir = tmp_path / "project" + project_dir.mkdir() + claude_dir = tmp_path / "claude-projects" + claude_dir.mkdir() + monkeypatch.setattr( + "autoskillit.cli.session._session_picker.claude_code_project_dir", + lambda _: claude_dir, + ) + result = pick_session("cook", project_dir) + assert result is None + + +def test_pick_session_filters_cook( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture +) -> None: + project_dir = tmp_path / "project" + project_dir.mkdir() + claude_dir = tmp_path / "claude" + claude_dir.mkdir() + + entries = [ + {"sessionId": "cook-uuid-1", "firstPrompt": "What's the issue?", "isSidechain": False}, + { + "sessionId": "order-uuid-1", + "firstPrompt": "Kitchen's open! Hello", + "isSidechain": False, + }, + ] + (claude_dir / "sessions-index.json").write_text(json.dumps(entries), encoding="utf-8") + + write_registry_entry(project_dir, "lid-cook", "cook", None) + write_registry_entry(project_dir, "lid-order", "order", None) + + from autoskillit.core.runtime.session_registry import bridge_claude_session_id + + bridge_claude_session_id(project_dir, "lid-cook", "cook-uuid-1") + bridge_claude_session_id(project_dir, "lid-order", "order-uuid-1") + + monkeypatch.setattr( + "autoskillit.cli.session._session_picker.claude_code_project_dir", + lambda _: claude_dir, + ) + monkeypatch.setattr("sys.stdin.isatty", lambda: True) + monkeypatch.setattr("builtins.input", lambda _prompt="": "1") + + result = pick_session("cook", project_dir) + assert result == "cook-uuid-1" + + +def test_pick_session_filters_order(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + project_dir = tmp_path / "project" + project_dir.mkdir() + claude_dir = tmp_path / "claude" + claude_dir.mkdir() + + entries = [ + {"sessionId": "cook-uuid-1", "firstPrompt": "What's the issue?", "isSidechain": False}, + { + "sessionId": "order-uuid-1", + "firstPrompt": "Kitchen's open! Hello", + "isSidechain": False, + }, + ] + (claude_dir / "sessions-index.json").write_text(json.dumps(entries), encoding="utf-8") + + write_registry_entry(project_dir, "lid-cook", "cook", None) + write_registry_entry(project_dir, "lid-order", "order", None) + + from autoskillit.core.runtime.session_registry import bridge_claude_session_id + + bridge_claude_session_id(project_dir, "lid-cook", "cook-uuid-1") + bridge_claude_session_id(project_dir, "lid-order", "order-uuid-1") + + monkeypatch.setattr( + "autoskillit.cli.session._session_picker.claude_code_project_dir", + lambda _: claude_dir, + ) + monkeypatch.setattr("sys.stdin.isatty", lambda: True) + monkeypatch.setattr("builtins.input", lambda _prompt="": "1") + + result = pick_session("order", project_dir) + assert result == "order-uuid-1" + + +def test_greeting_heuristic_order_session() -> None: + entry = {"sessionId": "s1", "firstPrompt": "Order up! Today's special: myrecipe."} + result = _classify_session(entry, {}) + assert result == "order" + + +def test_greeting_heuristic_cook_session() -> None: + entry = {"sessionId": "s1", "firstPrompt": "What's the issue?"} + result = _classify_session(entry, {}) + assert result == "cook" + + +def test_greeting_heuristic_open_kitchen_order() -> None: + entry = {"sessionId": "s1", "firstPrompt": "Kitchen's open! What are we cooking today?"} + result = _classify_session(entry, {}) + assert result == "order" + + +def test_sidechain_sessions_excluded(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + project_dir = tmp_path / "project" + project_dir.mkdir() + claude_dir = tmp_path / "claude" + claude_dir.mkdir() + + entries = [ + {"sessionId": "sidechain-uuid", "firstPrompt": "What's the issue?", "isSidechain": True}, + ] + (claude_dir / "sessions-index.json").write_text(json.dumps(entries), encoding="utf-8") + + monkeypatch.setattr( + "autoskillit.cli.session._session_picker.claude_code_project_dir", + lambda _: claude_dir, + ) + + result = pick_session("cook", project_dir) + assert result is None + + +def test_user_selects_numbered_session(monkeypatch: pytest.MonkeyPatch) -> None: + sessions = [ + {"sessionId": "uuid-1", "firstPrompt": "What's the issue?"}, + {"sessionId": "uuid-2", "firstPrompt": "Fix the bug"}, + ] + inputs = iter(["1"]) + monkeypatch.setattr("sys.stdin.isatty", lambda: True) + monkeypatch.setattr("builtins.input", lambda _prompt="": next(inputs)) + result = _run_picker(sessions, "cook", {}) + assert result == "uuid-1" + + +def test_user_selects_fresh_start(monkeypatch: pytest.MonkeyPatch) -> None: + sessions = [{"sessionId": "uuid-1", "firstPrompt": "Hello"}] + monkeypatch.setattr("sys.stdin.isatty", lambda: True) + monkeypatch.setattr("builtins.input", lambda _prompt="": "0") + result = _run_picker(sessions, "cook", {}) + assert result is None + + +def test_user_selects_out_of_range(monkeypatch: pytest.MonkeyPatch) -> None: + sessions = [{"sessionId": "uuid-1", "firstPrompt": "Hello"}] + inputs = iter(["99", "0"]) + monkeypatch.setattr("sys.stdin.isatty", lambda: True) + monkeypatch.setattr("builtins.input", lambda _prompt="": next(inputs)) + result = _run_picker(sessions, "cook", {}) + assert result is None diff --git a/tests/cli/test_signal_guard.py b/tests/cli/test_signal_guard.py new file mode 100644 index 0000000000..e6bdd55d30 --- /dev/null +++ b/tests/cli/test_signal_guard.py @@ -0,0 +1,339 @@ +"""Tests for _fleet_signal_guard in cli/_fleet.py (Group J).""" + +from __future__ import annotations + +import ast +import json +import os +import signal as _signal +from pathlib import Path +from unittest.mock import MagicMock, patch + +import anyio +import pytest + +from autoskillit.fleet import ( + DispatchRecord, + DispatchStatus, + read_state, + write_initial_state, +) + +pytestmark = [pytest.mark.layer("cli"), pytest.mark.small, pytest.mark.feature("fleet")] + +BOOT_ID = "boot-sg-001" +TICKS = 5000 + + +def _make_running_state( + tmp_path: Path, + *, + dispatch_name: str = "sg-dispatch", + dispatched_pid: int = 0, + dispatched_starttime_ticks: int = TICKS, + dispatched_boot_id: str = BOOT_ID, +) -> tuple[Path, str]: + sp = tmp_path / "state.json" + campaign_id = "camp-sg-001" + write_initial_state( + sp, + campaign_id, + "signal-guard-campaign", + "/m.yaml", + [DispatchRecord(name=dispatch_name)], + ) + raw = json.loads(sp.read_text()) + raw["dispatches"][0].update( + { + "status": "running", + "dispatch_id": "did-sg", + "dispatched_pid": dispatched_pid, + "dispatched_starttime_ticks": dispatched_starttime_ticks, + "dispatched_boot_id": dispatched_boot_id, + "started_at": 1000.0, + } + ) + sp.write_text(json.dumps(raw)) + return sp, campaign_id + + +async def _run_signal_guard( + state_path: Path, + campaign_id: str, + sig: int, + *, + cleanup_on_interrupt: bool = False, + campaign_name: str | None = None, +) -> None: + """Run the signal guard and fire a signal after it's armed.""" + from autoskillit.cli.fleet._fleet_lifecycle import _fleet_signal_guard + + async with anyio.create_task_group() as tg: + + async def _fire_signal() -> None: + await anyio.sleep(0.05) + os.kill(os.getpid(), sig) + + tg.start_soon(_fire_signal) + + async with _fleet_signal_guard( + state_path, + campaign_id, + campaign_name=campaign_name, + cleanup_on_interrupt=cleanup_on_interrupt, + ): + await anyio.sleep(10) # cancelled by signal guard + + +class TestSignalGuard: + async def test_signal_guard_kills_running_dispatch(self, tmp_path: Path) -> None: + state_path, campaign_id = _make_running_state( + tmp_path, dispatched_pid=12345, dispatched_starttime_ticks=TICKS + ) + kill_calls: list[int] = [] + + async def fake_kill(pid: int, timeout: float = 2.0) -> None: + kill_calls.append(pid) + + with ( + patch( + "autoskillit.execution.read_starttime_ticks", + return_value=TICKS, + ), + patch( + "autoskillit.execution.async_kill_process_tree", + side_effect=fake_kill, + ), + ): + await _run_signal_guard(state_path, campaign_id, _signal.SIGTERM) + + assert kill_calls == [12345] + state = read_state(state_path) + assert state is not None + assert state.dispatches[0].status == DispatchStatus.INTERRUPTED + + async def test_signal_guard_reason_contains_signal_name(self, tmp_path: Path) -> None: + state_path, campaign_id = _make_running_state( + tmp_path, + dispatched_pid=0, # zero pid → skips kill, goes straight to mark + ) + + await _run_signal_guard(state_path, campaign_id, _signal.SIGINT) + + state = read_state(state_path) + assert state is not None + assert state.dispatches[0].reason == "signal_SIGINT" + + async def test_signal_guard_skips_zero_pid(self, tmp_path: Path) -> None: + """Dispatch with dispatched_pid=0 → state marked INTERRUPTED without kill attempt.""" + state_path, campaign_id = _make_running_state(tmp_path, dispatched_pid=0) + kill_calls: list[int] = [] + + async def fake_kill(pid: int, timeout: float = 2.0) -> None: + kill_calls.append(pid) + + with patch( + "autoskillit.execution.async_kill_process_tree", + side_effect=fake_kill, + ): + await _run_signal_guard(state_path, campaign_id, _signal.SIGTERM) + + assert kill_calls == [] + state = read_state(state_path) + assert state is not None + assert state.dispatches[0].status == DispatchStatus.INTERRUPTED + + async def test_signal_guard_verifies_pid_identity(self, tmp_path: Path) -> None: + """Dispatch with pid alive but different starttime_ticks → kill NOT called.""" + state_path, campaign_id = _make_running_state( + tmp_path, dispatched_pid=12345, dispatched_starttime_ticks=TICKS + ) + kill_calls: list[int] = [] + + async def fake_kill(pid: int, timeout: float = 2.0) -> None: + kill_calls.append(pid) + + with ( + patch( + "autoskillit.execution.read_starttime_ticks", + return_value=9999, # mismatch → recycled + ), + patch( + "autoskillit.execution.async_kill_process_tree", + side_effect=fake_kill, + ), + ): + await _run_signal_guard(state_path, campaign_id, _signal.SIGTERM) + + assert kill_calls == [] + state = read_state(state_path) + assert state is not None + assert state.dispatches[0].status == DispatchStatus.INTERRUPTED + + async def test_signal_guard_stderr_resume_hint( + self, tmp_path: Path, capsys: pytest.CaptureFixture[str] + ) -> None: + state_path, campaign_id = _make_running_state(tmp_path, dispatched_pid=0) + + await _run_signal_guard(state_path, campaign_id, _signal.SIGTERM) + + captured = capsys.readouterr() + assert campaign_id in captured.err + + async def test_signal_guard_cleanup_on_interrupt_true(self, tmp_path: Path) -> None: + """cleanup_on_interrupt=True → DefaultWorkspaceManager.delete_contents is invoked.""" + state_path, campaign_id = _make_running_state(tmp_path, dispatched_pid=0) + cleanup_calls: list[Path] = [] + + class FakeWorkspaceManager: + def delete_contents(self, directory: Path, preserve: object = None) -> object: + cleanup_calls.append(directory) + return MagicMock() + + with ( + patch("autoskillit.workspace.DefaultWorkspaceManager", FakeWorkspaceManager), + patch("autoskillit.core.ensure_project_temp", return_value=tmp_path / "ws"), + ): + await _run_signal_guard( + state_path, campaign_id, _signal.SIGTERM, cleanup_on_interrupt=True + ) + + assert len(cleanup_calls) == 1 + + async def test_signal_guard_cleanup_on_interrupt_default_false(self, tmp_path: Path) -> None: + """cleanup_on_interrupt=False (default) → workspace cleanup NOT invoked.""" + state_path, campaign_id = _make_running_state(tmp_path, dispatched_pid=0) + cleanup_calls: list[Path] = [] + + class FakeWorkspaceManager: + def delete_contents(self, directory: Path, preserve: object = None) -> object: + cleanup_calls.append(directory) + return MagicMock() + + with ( + patch("autoskillit.workspace.DefaultWorkspaceManager", FakeWorkspaceManager), + patch("autoskillit.core.ensure_project_temp", return_value=tmp_path / "ws"), + ): + await _run_signal_guard(state_path, campaign_id, _signal.SIGTERM) + + assert cleanup_calls == [] + + async def test_signal_guard_cancels_scope_before_state_write(self, tmp_path: Path) -> None: + """Scope is cancelled (guard exits) AND state is written (in shielded section). + + Verifies that scope cancellation happens before the state write by checking + that the guard context manager exited cleanly (scope was cancelled) AND the + dispatch state was updated (shielded cleanup ran). + """ + state_path, campaign_id = _make_running_state(tmp_path, dispatched_pid=0) + + events: list[str] = [] + + from autoskillit.fleet import mark_dispatch_interrupted as _real_mark + + def tracking_mark(sp: Path, name: str, *, reason: str) -> None: + events.append("state_written") + _real_mark(sp, name, reason=reason) + + with patch("autoskillit.fleet.state.mark_dispatch_interrupted", tracking_mark): + await _run_signal_guard(state_path, campaign_id, _signal.SIGTERM) + events.append("guard_exited") + + # Guard exits (scope was cancelled) and state was written in the shielded section + assert "state_written" in events + assert "guard_exited" in events + # state_written happens inside the shielded section; guard_exited happens after + assert events.index("state_written") < events.index("guard_exited") + + async def test_signal_guard_resume_hint_includes_campaign_name( + self, tmp_path: Path, capsys: pytest.CaptureFixture[str] + ) -> None: + state_path, campaign_id = _make_running_state(tmp_path, dispatched_pid=0) + await _run_signal_guard( + state_path, campaign_id, _signal.SIGTERM, campaign_name="my-campaign" + ) + captured = capsys.readouterr() + assert "fleet campaign my-campaign" in captured.err + assert campaign_id in captured.err + + async def test_signal_guard_resume_hint_generic_when_no_campaign_name( + self, tmp_path: Path, capsys: pytest.CaptureFixture[str] + ) -> None: + state_path, campaign_id = _make_running_state(tmp_path, dispatched_pid=0) + await _run_signal_guard(state_path, campaign_id, _signal.SIGTERM) + captured = capsys.readouterr() + assert "fleet campaign " in captured.err + + +def test_sighup_in_fleet_signal_list() -> None: + """_fleet_lifecycle.py must pass signal.SIGHUP to open_signal_receiver (AST guard). + + Sending a real SIGHUP in tests is unsafe — before SIGHUP is registered, + the default disposition terminates the process. AST analysis is safe for + both the pre- and post-implementation codebase. + """ + from autoskillit.core.paths import pkg_root + + src_path = pkg_root() / "cli" / "fleet" / "_fleet_lifecycle.py" + tree = ast.parse(src_path.read_text(encoding="utf-8")) + + sighup_found = False + for node in ast.walk(tree): + if not isinstance(node, ast.Call): + continue + func = node.func + if not (isinstance(func, ast.Attribute) and func.attr == "open_signal_receiver"): + continue + for arg in node.args: + if ( + isinstance(arg, ast.Attribute) + and arg.attr == "SIGHUP" + and isinstance(arg.value, ast.Name) + and arg.value.id == "signal" + ): + sighup_found = True + break + + assert sighup_found, ( + "signal.SIGHUP not found in anyio.open_signal_receiver() call in _fleet_lifecycle.py. " + "Terminal disconnects (SIGHUP) must trigger graceful shutdown." + ) + + +def test_fleet_signame_uses_sig_name_attribute() -> None: + """_fleet_lifecycle.py must use sig.name for signame, not a hardcoded ternary (AST guard). + + Verifies that the old ``"SIGINT" if sig == signal.SIGINT else "SIGTERM"`` + ternary has been replaced with ``sig.name`` so that SIGHUP and any future + signal additions are handled automatically without stale-mapping risk. + + Sending a real SIGHUP to check the actual value at runtime would be unsafe, + so we use AST inspection instead. + """ + from autoskillit.core.paths import pkg_root + + src_path = pkg_root() / "cli" / "fleet" / "_fleet_lifecycle.py" + tree = ast.parse(src_path.read_text(encoding="utf-8")) + + sig_name_attr_found = False + for node in ast.walk(tree): + # Look for: signame = sig.name (Assign with a target named "signame") + if not isinstance(node, ast.Assign): + continue + targets = node.targets + if not targets: + continue + target = targets[0] + if not (isinstance(target, ast.Name) and target.id == "signame"): + continue + # The value should be an attribute access `.name` on any expression + val = node.value + if isinstance(val, ast.Attribute) and val.attr == "name": + sig_name_attr_found = True + break + + assert sig_name_attr_found, ( + "signame assignment in _fleet_lifecycle.py does not use sig.name. " + "Replace the hardcoded ternary with `signame = sig.name` so that " + "SIGHUP and future signals are handled without manual mapping." + ) diff --git a/tests/cli/test_startup_budget.py b/tests/cli/test_startup_budget.py index e06cc0b1ef..91c531115c 100644 --- a/tests/cli/test_startup_budget.py +++ b/tests/cli/test_startup_budget.py @@ -10,10 +10,13 @@ import time from unittest.mock import patch +import pytest import structlog.testing from autoskillit.server import _state +pytestmark = [pytest.mark.layer("cli"), pytest.mark.small] + def test_serve_calls_mcp_run_within_budget(monkeypatch, tmp_path): """serve() pre-anyio.run() synchronous work must complete within 2 seconds.""" diff --git a/tests/cli/test_subprocess_env_contracts.py b/tests/cli/test_subprocess_env_contracts.py index 259b8bf1d3..00a0c92f31 100644 --- a/tests/cli/test_subprocess_env_contracts.py +++ b/tests/cli/test_subprocess_env_contracts.py @@ -27,6 +27,8 @@ import pytest +pytestmark = [pytest.mark.layer("cli"), pytest.mark.small] + CLI_ROOT = Path(__file__).parents[2] / "src" / "autoskillit" / "cli" SRC_ROOT = Path(__file__).parents[2] / "src" / "autoskillit" REQUIRED_GUARD = "AUTOSKILLIT_SKIP_STALE_CHECK" @@ -35,7 +37,7 @@ { "build_interactive_cmd", "build_headless_cmd", - "build_full_headless_cmd", + "build_skill_session_cmd", } ) @@ -343,7 +345,7 @@ class _ClaudeLaunchWalker: kwarg and verifies the ``env=`` kwarg shape is one of: - ``Attribute(value=Name, attr="env")`` where Name was assigned from a - ``build_interactive_cmd``/``build_headless_cmd``/``build_full_headless_cmd`` + ``build_interactive_cmd``/``build_headless_cmd``/``build_skill_session_cmd`` call earlier in the same function body (spec.env pattern). - ``Call(func=Name("build_claude_env") | Attribute(..., "build_claude_env"))`` (direct builder escape hatch). @@ -497,6 +499,16 @@ def _check_call(self, call: ast.Call) -> None: # marketplace registration + install: `claude plugin marketplace add` / # `claude plugin install`. Administrative ops invoked once during setup. ("_marketplace.py", "install"), + # run_headless_core and dispatch_food_truck pass `spec` (ClaudeHeadlessCmd) + # to _execute_claude_headless, which internally uses `spec.env` at the + # runner call site. The checker incorrectly treats _execute_claude_headless(spec, ...) + # as a direct subprocess call because `spec` resolves to a builder return value. + ("__init__.py", "run_headless_core"), + ("__init__.py", "dispatch_food_truck"), + # build_headless_resume_cmd constructs cmd = ["claude", ...] then calls + # _apply_output_format(cmd, ...) — an in-place list mutation, not a subprocess + # launch. The function returns ClaudeHeadlessCmd(cmd=cmd, env=build_claude_env(...)). + ("commands.py", "build_headless_resume_cmd"), } ) diff --git a/tests/cli/test_terminal.py b/tests/cli/test_terminal.py index fb3dc8dc12..038a450732 100644 --- a/tests/cli/test_terminal.py +++ b/tests/cli/test_terminal.py @@ -12,7 +12,9 @@ import pytest -from autoskillit.cli._terminal import _RESET_SPEC +from autoskillit.cli.ui._terminal import _RESET_SPEC + +pytestmark = [pytest.mark.layer("cli"), pytest.mark.small] class TestTerminalGuardTTYRestore: @@ -20,13 +22,13 @@ class TestTerminalGuardTTYRestore: def test_restores_on_normal_exit(self): """tcsetattr(TCSAFLUSH, saved_attrs) is called after normal yield exit.""" - from autoskillit.cli._terminal import terminal_guard + from autoskillit.cli.ui._terminal import terminal_guard fake_attrs = [0, 0, 0, 0, 0, 0, [b"\x00"] * 32] with ( - patch("autoskillit.cli._terminal.sys.stdin") as mock_stdin, - patch("autoskillit.cli._terminal.termios") as mock_termios, - patch("autoskillit.cli._terminal.sys.stdout"), + patch("autoskillit.cli.ui._terminal.sys.stdin") as mock_stdin, + patch("autoskillit.cli.ui._terminal.termios") as mock_termios, + patch("autoskillit.cli.ui._terminal.sys.stdout"), ): mock_stdin.isatty.return_value = True mock_stdin.fileno.return_value = 0 @@ -41,13 +43,13 @@ def test_restores_on_normal_exit(self): def test_restores_on_keyboard_interrupt(self): """tcsetattr is called even when KeyboardInterrupt is raised inside the guard.""" - from autoskillit.cli._terminal import terminal_guard + from autoskillit.cli.ui._terminal import terminal_guard fake_attrs = [0, 0, 0, 0, 0, 0, [b"\x00"] * 32] with ( - patch("autoskillit.cli._terminal.sys.stdin") as mock_stdin, - patch("autoskillit.cli._terminal.termios") as mock_termios, - patch("autoskillit.cli._terminal.sys.stdout"), + patch("autoskillit.cli.ui._terminal.sys.stdin") as mock_stdin, + patch("autoskillit.cli.ui._terminal.termios") as mock_termios, + patch("autoskillit.cli.ui._terminal.sys.stdout"), ): mock_stdin.isatty.return_value = True mock_stdin.fileno.return_value = 0 @@ -63,13 +65,13 @@ def test_restores_on_keyboard_interrupt(self): def test_restores_on_system_exit(self): """tcsetattr is called when SystemExit is raised (non-zero subprocess returncode).""" - from autoskillit.cli._terminal import terminal_guard + from autoskillit.cli.ui._terminal import terminal_guard fake_attrs = [0, 0, 0, 0, 0, 0, [b"\x00"] * 32] with ( - patch("autoskillit.cli._terminal.sys.stdin") as mock_stdin, - patch("autoskillit.cli._terminal.termios") as mock_termios, - patch("autoskillit.cli._terminal.sys.stdout"), + patch("autoskillit.cli.ui._terminal.sys.stdin") as mock_stdin, + patch("autoskillit.cli.ui._terminal.termios") as mock_termios, + patch("autoskillit.cli.ui._terminal.sys.stdout"), ): mock_stdin.isatty.return_value = True mock_stdin.fileno.return_value = 0 @@ -86,12 +88,12 @@ def test_restores_on_system_exit(self): @pytest.mark.parametrize("entry", _RESET_SPEC, ids=lambda e: e.name) def test_emits_all_spec_sequences_on_normal_exit(self, entry): """Every sequence in _RESET_SPEC is emitted by terminal_guard() on normal exit.""" - from autoskillit.cli._terminal import terminal_guard + from autoskillit.cli.ui._terminal import terminal_guard with ( - patch("autoskillit.cli._terminal.sys.stdin") as mock_stdin, - patch("autoskillit.cli._terminal.termios"), - patch("autoskillit.cli._terminal.sys.stdout") as mock_stdout, + patch("autoskillit.cli.ui._terminal.sys.stdin") as mock_stdin, + patch("autoskillit.cli.ui._terminal.termios"), + patch("autoskillit.cli.ui._terminal.sys.stdout") as mock_stdout, ): mock_stdin.isatty.return_value = True mock_stdin.fileno.return_value = 0 @@ -107,12 +109,12 @@ def test_emits_all_spec_sequences_on_normal_exit(self, entry): @pytest.mark.parametrize("entry", _RESET_SPEC, ids=lambda e: e.name) def test_emits_all_spec_sequences_on_exception(self, entry): """Every sequence in _RESET_SPEC is emitted by terminal_guard() on exception.""" - from autoskillit.cli._terminal import terminal_guard + from autoskillit.cli.ui._terminal import terminal_guard with ( - patch("autoskillit.cli._terminal.sys.stdin") as mock_stdin, - patch("autoskillit.cli._terminal.termios"), - patch("autoskillit.cli._terminal.sys.stdout") as mock_stdout, + patch("autoskillit.cli.ui._terminal.sys.stdin") as mock_stdin, + patch("autoskillit.cli.ui._terminal.termios"), + patch("autoskillit.cli.ui._terminal.sys.stdout") as mock_stdout, ): mock_stdin.isatty.return_value = True mock_stdin.fileno.return_value = 0 @@ -128,11 +130,11 @@ def test_emits_all_spec_sequences_on_exception(self, entry): def test_noop_in_non_tty_environment(self): """When stdin is not a TTY, tcgetattr and tcsetattr are never called.""" - from autoskillit.cli._terminal import terminal_guard + from autoskillit.cli.ui._terminal import terminal_guard with ( - patch("autoskillit.cli._terminal.sys.stdin") as mock_stdin, - patch("autoskillit.cli._terminal.termios") as mock_termios, + patch("autoskillit.cli.ui._terminal.sys.stdin") as mock_stdin, + patch("autoskillit.cli.ui._terminal.termios") as mock_termios, ): mock_stdin.isatty.return_value = False @@ -144,12 +146,12 @@ def test_noop_in_non_tty_environment(self): def test_handles_tcgetattr_error_gracefully(self): """termios.error from tcgetattr does not propagate — guard becomes no-op.""" - from autoskillit.cli._terminal import terminal_guard + from autoskillit.cli.ui._terminal import terminal_guard with ( - patch("autoskillit.cli._terminal.sys.stdin") as mock_stdin, - patch("autoskillit.cli._terminal.termios") as mock_termios, - patch("autoskillit.cli._terminal.sys.stdout"), + patch("autoskillit.cli.ui._terminal.sys.stdin") as mock_stdin, + patch("autoskillit.cli.ui._terminal.termios") as mock_termios, + patch("autoskillit.cli.ui._terminal.sys.stdout"), ): mock_stdin.isatty.return_value = True mock_stdin.fileno.return_value = 0 @@ -163,14 +165,14 @@ def test_handles_tcgetattr_error_gracefully(self): def test_stty_fallback_on_tcsetattr_error(self): """os.system('stty sane') is called if tcsetattr raises termios.error.""" - from autoskillit.cli._terminal import terminal_guard + from autoskillit.cli.ui._terminal import terminal_guard fake_attrs = [0, 0, 0, 0, 0, 0, [b"\x00"] * 32] with ( - patch("autoskillit.cli._terminal.sys.stdin") as mock_stdin, - patch("autoskillit.cli._terminal.termios") as mock_termios, - patch("autoskillit.cli._terminal.sys.stdout"), - patch("autoskillit.cli._terminal.os") as mock_os, + patch("autoskillit.cli.ui._terminal.sys.stdin") as mock_stdin, + patch("autoskillit.cli.ui._terminal.termios") as mock_termios, + patch("autoskillit.cli.ui._terminal.sys.stdout"), + patch("autoskillit.cli.ui._terminal.os") as mock_os, ): mock_stdin.isatty.return_value = True mock_stdin.fileno.return_value = 0 @@ -196,15 +198,15 @@ def test_does_not_emit_entry_alt_screen_sequence(self): Regression guard for: investigation_terminal_guard_alt_screen_scrollbar """ - from autoskillit.cli._terminal import terminal_guard + from autoskillit.cli.ui._terminal import terminal_guard writes_before_yield: list[str] = [] all_writes: list[str] = [] with ( - patch("autoskillit.cli._terminal.sys.stdin") as mock_stdin, - patch("autoskillit.cli._terminal.termios"), - patch("autoskillit.cli._terminal.sys.stdout") as mock_stdout, + patch("autoskillit.cli.ui._terminal.sys.stdin") as mock_stdin, + patch("autoskillit.cli.ui._terminal.termios"), + patch("autoskillit.cli.ui._terminal.sys.stdout") as mock_stdout, ): mock_stdin.isatty.return_value = True mock_stdin.fileno.return_value = 0 @@ -221,12 +223,12 @@ def test_does_not_emit_entry_alt_screen_sequence(self): def test_emits_exit_alt_screen_on_system_exit(self): """terminal_guard() emits \\033[?1049l in finally even when SystemExit raised.""" - from autoskillit.cli._terminal import terminal_guard + from autoskillit.cli.ui._terminal import terminal_guard with ( - patch("autoskillit.cli._terminal.sys.stdin") as mock_stdin, - patch("autoskillit.cli._terminal.termios"), - patch("autoskillit.cli._terminal.sys.stdout") as mock_stdout, + patch("autoskillit.cli.ui._terminal.sys.stdin") as mock_stdin, + patch("autoskillit.cli.ui._terminal.termios"), + patch("autoskillit.cli.ui._terminal.sys.stdout") as mock_stdout, ): mock_stdin.isatty.return_value = True mock_stdin.fileno.return_value = 0 @@ -242,12 +244,12 @@ def test_emits_exit_alt_screen_on_system_exit(self): def test_noop_does_not_emit_escape_sequences(self): """When stdin is not a TTY, no VT100 escape sequences are written to stdout.""" - from autoskillit.cli._terminal import terminal_guard + from autoskillit.cli.ui._terminal import terminal_guard with ( - patch("autoskillit.cli._terminal.sys.stdin") as mock_stdin, - patch("autoskillit.cli._terminal.termios"), - patch("autoskillit.cli._terminal.sys.stdout") as mock_stdout, + patch("autoskillit.cli.ui._terminal.sys.stdin") as mock_stdin, + patch("autoskillit.cli.ui._terminal.termios"), + patch("autoskillit.cli.ui._terminal.sys.stdout") as mock_stdout, ): mock_stdin.isatty.return_value = False @@ -258,12 +260,12 @@ def test_noop_does_not_emit_escape_sequences(self): def test_kitty_sequences_emitted_on_supported_terminal(self): """Kitty KBP sequences are emitted when TERM_PROGRAM indicates support.""" - from autoskillit.cli._terminal import terminal_guard + from autoskillit.cli.ui._terminal import terminal_guard with ( - patch("autoskillit.cli._terminal.sys.stdin") as mock_stdin, - patch("autoskillit.cli._terminal.termios"), - patch("autoskillit.cli._terminal.sys.stdout") as mock_stdout, + patch("autoskillit.cli.ui._terminal.sys.stdin") as mock_stdin, + patch("autoskillit.cli.ui._terminal.termios"), + patch("autoskillit.cli.ui._terminal.sys.stdout") as mock_stdout, patch.dict("os.environ", {"TERM_PROGRAM": "kitty"}, clear=False), ): mock_stdin.isatty.return_value = True @@ -281,16 +283,16 @@ def test_kitty_sequences_not_emitted_on_unsupported_terminal(self): JediTerm (JetBrains IDEs) echoes literal garbage from \\033[<99u. """ - from autoskillit.cli._terminal import terminal_guard + from autoskillit.cli.ui._terminal import terminal_guard env = os.environ.copy() env.pop("TERM_PROGRAM", None) env.pop("KITTY_WINDOW_ID", None) with ( - patch("autoskillit.cli._terminal.sys.stdin") as mock_stdin, - patch("autoskillit.cli._terminal.termios"), - patch("autoskillit.cli._terminal.sys.stdout") as mock_stdout, + patch("autoskillit.cli.ui._terminal.sys.stdin") as mock_stdin, + patch("autoskillit.cli.ui._terminal.termios"), + patch("autoskillit.cli.ui._terminal.sys.stdout") as mock_stdout, patch.dict("os.environ", env, clear=True), ): mock_stdin.isatty.return_value = True @@ -312,12 +314,12 @@ def test_kitty_sequences_not_emitted_on_unsupported_terminal(self): def test_kitty_sequences_emitted_via_kitty_window_id(self): """KITTY_WINDOW_ID triggers Kitty KBP sequences regardless of TERM_PROGRAM.""" - from autoskillit.cli._terminal import terminal_guard + from autoskillit.cli.ui._terminal import terminal_guard with ( - patch("autoskillit.cli._terminal.sys.stdin") as mock_stdin, - patch("autoskillit.cli._terminal.termios"), - patch("autoskillit.cli._terminal.sys.stdout") as mock_stdout, + patch("autoskillit.cli.ui._terminal.sys.stdin") as mock_stdin, + patch("autoskillit.cli.ui._terminal.termios"), + patch("autoskillit.cli.ui._terminal.sys.stdout") as mock_stdout, patch.dict("os.environ", {"KITTY_WINDOW_ID": "1"}, clear=False), ): mock_stdin.isatty.return_value = True @@ -331,12 +333,12 @@ def test_kitty_sequences_emitted_via_kitty_window_id(self): def test_kitty_protocol_sequences_emitted_after_decstr(self): """Kitty KBP sequences must follow DECSTR to avoid being reset.""" - from autoskillit.cli._terminal import terminal_guard + from autoskillit.cli.ui._terminal import terminal_guard with ( - patch("autoskillit.cli._terminal.sys.stdin") as mock_stdin, - patch("autoskillit.cli._terminal.termios"), - patch("autoskillit.cli._terminal.sys.stdout") as mock_stdout, + patch("autoskillit.cli.ui._terminal.sys.stdin") as mock_stdin, + patch("autoskillit.cli.ui._terminal.termios"), + patch("autoskillit.cli.ui._terminal.sys.stdout") as mock_stdout, patch.dict("os.environ", {"TERM_PROGRAM": "kitty"}, clear=False), ): mock_stdin.isatty.return_value = True @@ -362,7 +364,7 @@ class TestResetSpecificationCompleteness: def test_reset_spec_covers_all_layers(self): """Every ResetLayer enum member must have >= 1 entry in _RESET_SPEC.""" - from autoskillit.cli._terminal import _RESET_SPEC, ResetLayer + from autoskillit.cli.ui._terminal import _RESET_SPEC, ResetLayer covered_layers = {entry.layer for entry in _RESET_SPEC} missing = set(ResetLayer) - covered_layers @@ -375,7 +377,7 @@ def test_base_reset_matches_spec_bidirectional(self): """_BASE_RESET must contain exactly the sequences in _RESET_SPEC.""" import re - from autoskillit.cli._terminal import _BASE_RESET, _RESET_SPEC + from autoskillit.cli.ui._terminal import _BASE_RESET, _RESET_SPEC spec_sequences = {entry.sequence for entry in _RESET_SPEC} for seq in spec_sequences: @@ -390,7 +392,7 @@ def test_base_reset_matches_spec_bidirectional(self): def test_content_layer_sequences_are_last(self): """CONTENT layer sequences must follow all other layers in _BASE_RESET.""" - from autoskillit.cli._terminal import _BASE_RESET, _RESET_SPEC, ResetLayer + from autoskillit.cli.ui._terminal import _BASE_RESET, _RESET_SPEC, ResetLayer content_seqs = [e.sequence for e in _RESET_SPEC if e.layer == ResetLayer.CONTENT] other_seqs = [e.sequence for e in _RESET_SPEC if e.layer != ResetLayer.CONTENT] @@ -406,7 +408,7 @@ def test_content_layer_sequences_are_last(self): def test_reset_spec_has_no_duplicate_sequences(self): """Each escape sequence must appear exactly once in _RESET_SPEC.""" - from autoskillit.cli._terminal import _RESET_SPEC + from autoskillit.cli.ui._terminal import _RESET_SPEC sequences = [e.sequence for e in _RESET_SPEC] duplicates = [s for s in sequences if sequences.count(s) > 1] @@ -418,23 +420,23 @@ class TestCookTerminalGuard: def test_cook_restores_terminal_on_keyboard_interrupt(self, monkeypatch, tmp_path): """cook() must restore terminal even when subprocess.run raises KeyboardInterrupt.""" - import autoskillit.cli._cook as cook_mod + import autoskillit.cli.session._cook as cook_mod monkeypatch.setattr("sys.stdin.isatty", lambda: True) monkeypatch.setattr("sys.stdin.fileno", lambda: 0) tcsetattr_calls = [] monkeypatch.setattr( - "autoskillit.cli._terminal.termios.tcgetattr", + "autoskillit.cli.ui._terminal.termios.tcgetattr", lambda fd: [0, 0, 0, 0, 0, 0, []], ) monkeypatch.setattr( - "autoskillit.cli._terminal.termios.tcsetattr", + "autoskillit.cli.ui._terminal.termios.tcsetattr", lambda fd, when, attrs: tcsetattr_calls.append(attrs), ) - monkeypatch.setattr("autoskillit.cli._terminal.termios.error", termios.error) + monkeypatch.setattr("autoskillit.cli.ui._terminal.termios.error", termios.error) monkeypatch.setattr( - "subprocess.run", + "autoskillit.cli.session._cook.subprocess.run", lambda *a, **kw: (_ for _ in ()).throw(KeyboardInterrupt()), ) monkeypatch.setattr("shutil.which", lambda cmd: "/usr/bin/claude") @@ -464,27 +466,27 @@ def test_launch_cook_session_restores_terminal_on_keyboard_interrupt(self, monke """_launch_cook_session() must restore terminal on exception.""" import importlib - app_mod = importlib.import_module("autoskillit.cli.app") + app_mod = importlib.import_module("autoskillit.cli.session._session_launch") monkeypatch.setattr("sys.stdin.isatty", lambda: True) monkeypatch.setattr("sys.stdin.fileno", lambda: 0) tcsetattr_calls = [] monkeypatch.setattr( - "autoskillit.cli._terminal.termios.tcgetattr", + "autoskillit.cli.ui._terminal.termios.tcgetattr", lambda fd: [0, 0, 0, 0, 0, 0, []], ) monkeypatch.setattr( - "autoskillit.cli._terminal.termios.tcsetattr", + "autoskillit.cli.ui._terminal.termios.tcsetattr", lambda fd, when, attrs: tcsetattr_calls.append(attrs), ) - monkeypatch.setattr("autoskillit.cli._terminal.termios.error", termios.error) + monkeypatch.setattr("autoskillit.cli.ui._terminal.termios.error", termios.error) + monkeypatch.setattr("autoskillit.cli._init_helpers._is_plugin_installed", lambda: False) monkeypatch.setattr( - "subprocess.run", + "autoskillit.cli.session._session_launch.subprocess.run", lambda *a, **kw: (_ for _ in ()).throw(KeyboardInterrupt()), ) monkeypatch.setattr("shutil.which", lambda cmd: "/usr/bin/claude") - monkeypatch.setattr(app_mod, "_is_plugin_installed", lambda: False) with pytest.raises(KeyboardInterrupt): app_mod._launch_cook_session("system prompt") diff --git a/tests/cli/test_update_checks.py b/tests/cli/test_update_checks.py deleted file mode 100644 index 3d80dfa170..0000000000 --- a/tests/cli/test_update_checks.py +++ /dev/null @@ -1,1577 +0,0 @@ -"""Tests for cli/_update_checks.py — unified update check orchestration. - -Absorbs and extends the behavioral coverage from the deleted -test_stale_check.py and test_source_drift.py files. -""" - -from __future__ import annotations - -import io -import json -import sys -from datetime import UTC, datetime, timedelta -from pathlib import Path -from typing import Any -from unittest.mock import MagicMock, patch - -import pytest - -from autoskillit.cli._install_info import InstallInfo, InstallType -from autoskillit.cli._update_checks import ( - _fetch_with_cache, - _is_dismissed, - _read_dismiss_state, - _write_dismiss_state, - run_update_checks, -) - -# --------------------------------------------------------------------------- -# Helpers -# --------------------------------------------------------------------------- - - -def _make_stable_info(commit_id: str = "abc123", revision: str = "stable") -> InstallInfo: - return InstallInfo( - install_type=InstallType.GIT_VCS, - commit_id=commit_id, - requested_revision=revision, - url="https://github.com/TalonT-Org/AutoSkillit.git", - editable_source=None, - ) - - -def _make_integration_info(commit_id: str = "def456") -> InstallInfo: - return InstallInfo( - install_type=InstallType.GIT_VCS, - commit_id=commit_id, - requested_revision="integration", - url="https://github.com/TalonT-Org/AutoSkillit.git", - editable_source=None, - ) - - -def _make_mock_client( - status_code: int = 200, - json_body: dict[str, Any] | None = None, - etag: str | None = None, - raise_exc: Exception | None = None, -) -> MagicMock: - """Build a MagicMock httpx.Client context manager for fetch cache tests.""" - response = MagicMock() - response.status_code = status_code - response.json.return_value = json_body or {} - response.headers = {"ETag": etag} if etag else {} - - client_instance = MagicMock() - if raise_exc is not None: - client_instance.get.side_effect = raise_exc - else: - client_instance.get.return_value = response - - ctx_manager = MagicMock() - ctx_manager.__enter__.return_value = client_instance - ctx_manager.__exit__.return_value = False - return ctx_manager - - -# --------------------------------------------------------------------------- -# UC-1 Guards -# --------------------------------------------------------------------------- - - -@pytest.mark.parametrize( - "env_var,value", - [ - ("CLAUDECODE", "1"), - ("CI", "1"), - ("AUTOSKILLIT_SKIP_STALE_CHECK", "1"), - ("AUTOSKILLIT_SKIP_SOURCE_DRIFT_CHECK", "1"), - ], -) -def test_run_update_checks_skips_on_guard_env_var( - monkeypatch: pytest.MonkeyPatch, - env_var: str, - value: str, - tmp_path: Path, -) -> None: - monkeypatch.setenv(env_var, value) - # Ensure no other guard vars are set - for other in [ - "CLAUDECODE", - "CI", - "AUTOSKILLIT_SKIP_STALE_CHECK", - "AUTOSKILLIT_SKIP_SOURCE_DRIFT_CHECK", - ]: - if other != env_var: - monkeypatch.delenv(other, raising=False) - fetched: list[str] = [] - monkeypatch.setattr( - "autoskillit.cli._update_checks._fetch_with_cache", - lambda url, **kw: fetched.append(url) or None, - ) - prompted: list[str] = [] - monkeypatch.setattr("builtins.input", lambda _: prompted.append("called") or "n") - run_update_checks(home=tmp_path) - assert not fetched, "No network fetch should occur under guard env" - assert not prompted, "No input() call under guard env" - - -def test_run_update_checks_skips_non_tty_stdin( - monkeypatch: pytest.MonkeyPatch, tmp_path: Path -) -> None: - monkeypatch.delenv("CLAUDECODE", raising=False) - monkeypatch.delenv("CI", raising=False) - monkeypatch.delenv("AUTOSKILLIT_SKIP_STALE_CHECK", raising=False) - monkeypatch.delenv("AUTOSKILLIT_SKIP_SOURCE_DRIFT_CHECK", raising=False) - fake_stdin = io.StringIO() - fake_stdout = io.StringIO() - monkeypatch.setattr(sys, "stdin", fake_stdin) - monkeypatch.setattr(sys, "stdout", fake_stdout) - fetched: list[str] = [] - monkeypatch.setattr( - "autoskillit.cli._update_checks._fetch_with_cache", - lambda url, **kw: fetched.append(url) or None, - ) - run_update_checks(home=tmp_path) - assert not fetched - - -def test_run_update_checks_skips_non_tty_stdout( - monkeypatch: pytest.MonkeyPatch, tmp_path: Path -) -> None: - monkeypatch.delenv("CLAUDECODE", raising=False) - monkeypatch.delenv("CI", raising=False) - monkeypatch.delenv("AUTOSKILLIT_SKIP_STALE_CHECK", raising=False) - monkeypatch.delenv("AUTOSKILLIT_SKIP_SOURCE_DRIFT_CHECK", raising=False) - - fake_stdin = MagicMock() - fake_stdin.isatty.return_value = True - fake_stdout = io.StringIO() - monkeypatch.setattr(sys, "stdin", fake_stdin) - monkeypatch.setattr(sys, "stdout", fake_stdout) - fetched: list[str] = [] - monkeypatch.setattr( - "autoskillit.cli._update_checks._fetch_with_cache", - lambda url, **kw: fetched.append(url) or None, - ) - run_update_checks(home=tmp_path) - assert not fetched - - -@pytest.mark.parametrize( - "install_type", - [InstallType.LOCAL_EDITABLE, InstallType.LOCAL_PATH, InstallType.UNKNOWN], -) -def test_run_update_checks_skips_local_and_unknown_install_types( - monkeypatch: pytest.MonkeyPatch, tmp_path: Path, install_type: InstallType -) -> None: - monkeypatch.delenv("CLAUDECODE", raising=False) - monkeypatch.delenv("CI", raising=False) - monkeypatch.delenv("AUTOSKILLIT_SKIP_STALE_CHECK", raising=False) - monkeypatch.delenv("AUTOSKILLIT_SKIP_SOURCE_DRIFT_CHECK", raising=False) - monkeypatch.delenv("AUTOSKILLIT_FORCE_UPDATE_CHECK", raising=False) - - fake_stdin = MagicMock() - fake_stdin.isatty.return_value = True - fake_stdout = MagicMock() - fake_stdout.isatty.return_value = True - monkeypatch.setattr(sys, "stdin", fake_stdin) - monkeypatch.setattr(sys, "stdout", fake_stdout) - - info = InstallInfo( - install_type=install_type, - commit_id=None, - requested_revision=None, - url=None, - editable_source=Path(tmp_path) if install_type == InstallType.LOCAL_EDITABLE else None, - ) - monkeypatch.setattr("autoskillit.cli._update_checks.detect_install", lambda: info) - fetched: list[str] = [] - monkeypatch.setattr( - "autoskillit.cli._update_checks._fetch_with_cache", - lambda url, **kw: fetched.append(url) or None, - ) - prompted: list[str] = [] - monkeypatch.setattr("builtins.input", lambda _: prompted.append("called") or "n") - run_update_checks(home=tmp_path) - assert not fetched - assert not prompted - - -# --------------------------------------------------------------------------- -# UC-2 Signal gathering -# --------------------------------------------------------------------------- - - -def test_binary_signal_fires_when_newer_version_available( - monkeypatch: pytest.MonkeyPatch, tmp_path: Path -) -> None: - from autoskillit.cli._update_checks import _binary_signal - - info = _make_stable_info() - monkeypatch.setattr( - "autoskillit.cli._update_checks._fetch_latest_version", - lambda target, home: "0.9.0", - ) - sig = _binary_signal(info, tmp_path, "0.7.77") - assert sig is not None - assert sig.kind == "binary" - assert "0.9.0" in sig.message - assert "0.7.77" in sig.message - - -def test_binary_signal_silent_when_same_version( - monkeypatch: pytest.MonkeyPatch, tmp_path: Path -) -> None: - from autoskillit.cli._update_checks import _binary_signal - - info = _make_stable_info() - monkeypatch.setattr( - "autoskillit.cli._update_checks._fetch_latest_version", - lambda target, home: "0.7.77", - ) - assert _binary_signal(info, tmp_path, "0.7.77") is None - - -def test_binary_signal_silent_when_fetch_returns_none( - monkeypatch: pytest.MonkeyPatch, tmp_path: Path -) -> None: - from autoskillit.cli._update_checks import _binary_signal - - info = _make_stable_info() - monkeypatch.setattr( - "autoskillit.cli._update_checks._fetch_latest_version", - lambda target, home: None, - ) - assert _binary_signal(info, tmp_path, "0.7.77") is None - - -def test_binary_signal_uses_releases_url_for_stable( - monkeypatch: pytest.MonkeyPatch, tmp_path: Path -) -> None: - from autoskillit.cli._update_checks import _binary_signal - - info = _make_stable_info() - targets: list[str] = [] - monkeypatch.setattr( - "autoskillit.cli._update_checks._fetch_latest_version", - lambda target, home: targets.append(target) or "0.9.0", - ) - _binary_signal(info, tmp_path, "0.7.77") - assert targets == ["releases/latest"] - - -def test_binary_signal_uses_integration_url_for_integration( - monkeypatch: pytest.MonkeyPatch, tmp_path: Path -) -> None: - from autoskillit.cli._update_checks import _binary_signal - - info = _make_integration_info() - targets: list[str] = [] - monkeypatch.setattr( - "autoskillit.cli._update_checks._fetch_latest_version", - lambda target, home: targets.append(target) or "0.9.0", - ) - _binary_signal(info, tmp_path, "0.7.77") - assert targets == ["integration"] - - -def test_hooks_signal_fires_on_missing_hooks( - monkeypatch: pytest.MonkeyPatch, tmp_path: Path -) -> None: - from autoskillit.cli._update_checks import _hooks_signal - from autoskillit.hook_registry import HookDriftResult - - monkeypatch.setattr( - "autoskillit.cli._update_checks._count_hook_registry_drift", - lambda path: HookDriftResult(missing=3, orphaned=0), - ) - sig = _hooks_signal(tmp_path / "settings.json") - assert sig is not None - assert sig.kind == "hooks" - assert "3" in sig.message - - -def test_hooks_signal_fires_on_orphaned_hooks( - monkeypatch: pytest.MonkeyPatch, tmp_path: Path -) -> None: - from autoskillit.cli._update_checks import _hooks_signal - from autoskillit.hook_registry import HookDriftResult - - monkeypatch.setattr( - "autoskillit.cli._update_checks._count_hook_registry_drift", - lambda path: HookDriftResult(missing=0, orphaned=2), - ) - sig = _hooks_signal(tmp_path / "settings.json") - assert sig is not None - assert sig.kind == "hooks" - assert "2" in sig.message - - -def test_hooks_signal_silent_when_no_drift( - monkeypatch: pytest.MonkeyPatch, tmp_path: Path -) -> None: - from autoskillit.cli._update_checks import _hooks_signal - from autoskillit.hook_registry import HookDriftResult - - monkeypatch.setattr( - "autoskillit.cli._update_checks._count_hook_registry_drift", - lambda path: HookDriftResult(missing=0, orphaned=0), - ) - assert _hooks_signal(tmp_path / "settings.json") is None - - -def test_source_drift_signal_fires_when_commit_lags_ref( - monkeypatch: pytest.MonkeyPatch, tmp_path: Path -) -> None: - from autoskillit.cli._update_checks import _source_drift_signal - - info = _make_stable_info(commit_id="aaaaaa") - monkeypatch.setattr( - "autoskillit.cli._update_checks.resolve_reference_sha", - lambda info, home, **kw: "bbbbbb", - ) - sig = _source_drift_signal(info, tmp_path) - assert sig is not None - assert sig.kind == "source_drift" - # "source drift" must NOT appear in user-visible text (per plan verification item 11) - assert "source drift" not in sig.message.lower() - - -def test_source_drift_signal_silent_when_sha_matches( - monkeypatch: pytest.MonkeyPatch, tmp_path: Path -) -> None: - from autoskillit.cli._update_checks import _source_drift_signal - - info = _make_stable_info(commit_id="aaaaaa") - monkeypatch.setattr( - "autoskillit.cli._update_checks.resolve_reference_sha", - lambda info, home, **kw: "aaaaaa", - ) - assert _source_drift_signal(info, tmp_path) is None - - -def test_source_drift_signal_silent_when_ref_sha_none( - monkeypatch: pytest.MonkeyPatch, tmp_path: Path -) -> None: - from autoskillit.cli._update_checks import _source_drift_signal - - info = _make_stable_info(commit_id="aaaaaa") - monkeypatch.setattr( - "autoskillit.cli._update_checks.resolve_reference_sha", - lambda info, home, **kw: None, - ) - assert _source_drift_signal(info, tmp_path) is None - - -def test_dual_mcp_signal_fires_when_both_registered( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch -) -> None: - from autoskillit.cli._update_checks import _dual_mcp_signal - - monkeypatch.setattr( - "autoskillit.cli._update_checks._is_dual_mcp_registered", - lambda home: True, - ) - sig = _dual_mcp_signal(tmp_path) - assert sig is not None - assert sig.kind == "dual_mcp" - assert "autoskillit install" in sig.message - - -def test_dual_mcp_signal_silent_when_only_one_registered( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch -) -> None: - from autoskillit.cli._update_checks import _dual_mcp_signal - - monkeypatch.setattr( - "autoskillit.cli._update_checks._is_dual_mcp_registered", - lambda home: False, - ) - sig = _dual_mcp_signal(tmp_path) - assert sig is None - - -def test_dual_mcp_signal_silent_on_corrupted_files( - tmp_path: Path, -) -> None: - # _check_dual_mcp_files is fail-open: corrupted JSON → False → no signal - from autoskillit.cli._update_checks import _dual_mcp_signal - - claude_json = tmp_path / ".claude.json" - claude_json.write_text("{invalid json") - sig = _dual_mcp_signal(tmp_path) - assert sig is None - - -# --------------------------------------------------------------------------- -# find_source_repo behavioral tests -# --------------------------------------------------------------------------- - - -def test_find_source_repo_env_var_override_valid( - monkeypatch: pytest.MonkeyPatch, tmp_path: Path -) -> None: - """AUTOSKILLIT_SOURCE_REPO env var pointing to a valid repo root is returned.""" - from autoskillit.cli._update_checks import find_source_repo - - src_dir = tmp_path / "src" / "autoskillit" - src_dir.mkdir(parents=True) - monkeypatch.setenv("AUTOSKILLIT_SOURCE_REPO", str(tmp_path)) - result = find_source_repo() - assert result == tmp_path - - -def test_find_source_repo_env_var_override_invalid_falls_through( - monkeypatch: pytest.MonkeyPatch, tmp_path: Path -) -> None: - """AUTOSKILLIT_SOURCE_REPO pointing to a path without src/autoskillit/ is ignored.""" - from autoskillit.cli._update_checks import find_source_repo - - monkeypatch.setenv("AUTOSKILLIT_SOURCE_REPO", str(tmp_path / "nonexistent")) - monkeypatch.setattr("autoskillit.cli._update_checks.Path.cwd", lambda: tmp_path) - result = find_source_repo() - assert result is None - - -def test_find_source_repo_cwd_walk_finds_pyproject( - monkeypatch: pytest.MonkeyPatch, tmp_path: Path -) -> None: - """CWD walk finds the project root when pyproject.toml has name=autoskillit.""" - from autoskillit.cli._update_checks import find_source_repo - - project_root = tmp_path / "project" - src_dir = project_root / "src" / "autoskillit" - src_dir.mkdir(parents=True) - pyproject = project_root / "pyproject.toml" - pyproject.write_text('[project]\nname = "autoskillit"\n', encoding="utf-8") - - nested_cwd = project_root / "src" / "autoskillit" / "cli" - nested_cwd.mkdir(parents=True) - monkeypatch.delenv("AUTOSKILLIT_SOURCE_REPO", raising=False) - monkeypatch.setattr("autoskillit.cli._update_checks.Path.cwd", lambda: nested_cwd) - result = find_source_repo() - assert result == project_root - - -def test_find_source_repo_cwd_walk_no_match_returns_none( - monkeypatch: pytest.MonkeyPatch, tmp_path: Path -) -> None: - """CWD walk returns None when no autoskillit project is found.""" - from autoskillit.cli._update_checks import find_source_repo - - monkeypatch.delenv("AUTOSKILLIT_SOURCE_REPO", raising=False) - monkeypatch.setattr("autoskillit.cli._update_checks.Path.cwd", lambda: tmp_path) - result = find_source_repo() - assert result is None - - -# --------------------------------------------------------------------------- -# UC-3 Prompt consolidation -# --------------------------------------------------------------------------- - - -def _setup_run_checks( - monkeypatch: pytest.MonkeyPatch, - tmp_path: Path, - *, - info: InstallInfo | None = None, - binary_signal: bool = False, - hooks_signal: bool = False, - source_drift_signal: bool = False, - answer: str = "n", - current_version: str = "0.7.77", - state: dict | None = None, -) -> tuple[list[str], list[str]]: - """Set up mocks for run_update_checks and return (printed_lines, input_calls).""" - import select as _select_mod - - from autoskillit.cli._update_checks import Signal - - monkeypatch.delenv("CLAUDECODE", raising=False) - monkeypatch.delenv("CI", raising=False) - monkeypatch.delenv("AUTOSKILLIT_SKIP_STALE_CHECK", raising=False) - monkeypatch.delenv("AUTOSKILLIT_SKIP_SOURCE_DRIFT_CHECK", raising=False) - monkeypatch.delenv("AUTOSKILLIT_FORCE_UPDATE_CHECK", raising=False) - - fake_stdin = MagicMock() - fake_stdin.isatty.return_value = True - fake_stdout = MagicMock() - fake_stdout.isatty.return_value = True - monkeypatch.setattr(sys, "stdin", fake_stdin) - monkeypatch.setattr(sys, "stdout", fake_stdout) - - # timed_prompt uses select.select to implement timeout; mock it to - # report "stdin is ready" so tests proceed without real file descriptors. - monkeypatch.setattr( - _select_mod, "select", lambda rlist, wlist, xlist, timeout=None: (rlist, [], []) - ) - - _info = info or _make_stable_info() - monkeypatch.setattr("autoskillit.cli._update_checks.detect_install", lambda: _info) - - if state is not None: - (tmp_path / ".autoskillit").mkdir(parents=True, exist_ok=True) - (tmp_path / ".autoskillit" / "update_check.json").write_text( - json.dumps(state), encoding="utf-8" - ) - - import autoskillit as _pkg - - monkeypatch.setattr(_pkg, "__version__", current_version) - - monkeypatch.setattr( - "autoskillit.cli._update_checks._binary_signal", - lambda info, home, current: ( - Signal("binary", "New release: 0.9.0 (you have 0.7.77)") if binary_signal else None - ), - ) - monkeypatch.setattr( - "autoskillit.cli._update_checks._hooks_signal", - lambda settings_path: ( - Signal("hooks", "1 new/changed hook(s) detected") if hooks_signal else None - ), - ) - monkeypatch.setattr( - "autoskillit.cli._update_checks._source_drift_signal", - lambda info, home: ( - Signal("source_drift", "A newer version is available on the stable branch (aaa..bbb)") - if source_drift_signal - else None - ), - ) - monkeypatch.setattr( - "autoskillit.cli._update_checks._claude_settings_path", - lambda scope: tmp_path / "settings.json", - ) - - printed: list[str] = [] - monkeypatch.setattr( - "builtins.print", lambda *args, **kw: printed.append(" ".join(str(a) for a in args)) - ) - - input_calls: list[str] = [] - monkeypatch.setattr("builtins.input", lambda _="": input_calls.append("called") or answer) - - return printed, input_calls - - -def test_no_prompt_when_no_conditions_fire( - monkeypatch: pytest.MonkeyPatch, tmp_path: Path -) -> None: - printed, input_calls = _setup_run_checks(monkeypatch, tmp_path) - run_update_checks(home=tmp_path) - assert not input_calls - - -def test_single_prompt_when_only_binary_fires( - monkeypatch: pytest.MonkeyPatch, tmp_path: Path -) -> None: - printed, input_calls = _setup_run_checks(monkeypatch, tmp_path, binary_signal=True) - run_update_checks(home=tmp_path) - assert len(input_calls) == 1 - - -def test_single_prompt_when_only_hooks_fires( - monkeypatch: pytest.MonkeyPatch, tmp_path: Path -) -> None: - printed, input_calls = _setup_run_checks(monkeypatch, tmp_path, hooks_signal=True) - run_update_checks(home=tmp_path) - assert len(input_calls) == 1 - - -def test_single_prompt_when_only_source_drift_fires( - monkeypatch: pytest.MonkeyPatch, tmp_path: Path -) -> None: - printed, input_calls = _setup_run_checks(monkeypatch, tmp_path, source_drift_signal=True) - run_update_checks(home=tmp_path) - assert len(input_calls) == 1 - - -def test_consolidated_prompt_when_binary_plus_hooks_fire( - monkeypatch: pytest.MonkeyPatch, tmp_path: Path -) -> None: - printed, input_calls = _setup_run_checks( - monkeypatch, tmp_path, binary_signal=True, hooks_signal=True - ) - run_update_checks(home=tmp_path) - assert len(input_calls) == 1 - combined = " ".join(printed) - # Should contain 2 bullet lines - assert combined.count(" - ") == 2 - - -def test_consolidated_prompt_when_all_three_fire( - monkeypatch: pytest.MonkeyPatch, tmp_path: Path -) -> None: - printed, input_calls = _setup_run_checks( - monkeypatch, tmp_path, binary_signal=True, hooks_signal=True, source_drift_signal=True - ) - run_update_checks(home=tmp_path) - assert len(input_calls) == 1 - combined = " ".join(printed) - assert combined.count(" - ") == 3 - - -def test_prompt_never_contains_phrase_source_drift( - monkeypatch: pytest.MonkeyPatch, tmp_path: Path -) -> None: - printed, _ = _setup_run_checks(monkeypatch, tmp_path, source_drift_signal=True) - run_update_checks(home=tmp_path) - combined = " ".join(printed) - assert "source drift" not in combined.lower() - - -def test_prompt_uses_friendly_branch_language( - monkeypatch: pytest.MonkeyPatch, tmp_path: Path -) -> None: - printed, _ = _setup_run_checks(monkeypatch, tmp_path, source_drift_signal=True) - run_update_checks(home=tmp_path) - combined = " ".join(printed) - assert "newer version is available on the stable branch" in combined - - -# --------------------------------------------------------------------------- -# UC-4 Yes path -# --------------------------------------------------------------------------- - - -def test_yes_runs_upgrade_command_from_install_info_not_hardcoded( - monkeypatch: pytest.MonkeyPatch, tmp_path: Path -) -> None: - from autoskillit.cli._install_info import upgrade_command - - info = _make_stable_info() - printed, input_calls = _setup_run_checks( - monkeypatch, tmp_path, binary_signal=True, answer="y", info=info - ) - run_calls: list[list[str]] = [] - monkeypatch.setattr( - "autoskillit.cli._update_checks.subprocess.run", - lambda cmd, **kw: run_calls.append(list(cmd)), - ) - monkeypatch.setattr("autoskillit.cli._update_checks.terminal_guard", MagicMock()) - monkeypatch.setattr( - "autoskillit.cli._update_checks._verify_update_result", lambda *a, **kw: True - ) - monkeypatch.setattr( - "autoskillit.cli._update_checks._fetch_latest_version", lambda *a, **kw: "0.9.0" - ) - expected_cmd = upgrade_command(info) - run_update_checks(home=tmp_path) - assert expected_cmd in run_calls, ( - f"Expected upgrade command {expected_cmd!r} from upgrade_command(info); got {run_calls!r}" - ) - - -def test_yes_runs_autoskillit_install_after_upgrade_command( - monkeypatch: pytest.MonkeyPatch, tmp_path: Path -) -> None: - printed, input_calls = _setup_run_checks(monkeypatch, tmp_path, binary_signal=True, answer="y") - run_calls: list[list[str]] = [] - - class FakeTG: - def __enter__(self): - return self - - def __exit__(self, *a): - return False - - monkeypatch.setattr("autoskillit.cli._update_checks.terminal_guard", FakeTG) - monkeypatch.setattr( - "autoskillit.cli._update_checks.subprocess.run", - lambda cmd, **kw: run_calls.append(list(cmd)), - ) - monkeypatch.setattr( - "autoskillit.cli._update_checks._verify_update_result", lambda *a, **kw: True - ) - run_update_checks(home=tmp_path) - # ["autoskillit", "install"] must be among the calls - assert any(cmd[:2] == ["autoskillit", "install"] for cmd in run_calls) - - -def test_yes_passes_skip_env_to_subprocess( - monkeypatch: pytest.MonkeyPatch, tmp_path: Path -) -> None: - printed, input_calls = _setup_run_checks(monkeypatch, tmp_path, binary_signal=True, answer="y") - env_passed: list[dict] = [] - - class FakeTG: - def __enter__(self): - return self - - def __exit__(self, *a): - return False - - monkeypatch.setattr("autoskillit.cli._update_checks.terminal_guard", FakeTG) - monkeypatch.setattr( - "autoskillit.cli._update_checks.subprocess.run", - lambda cmd, **kw: env_passed.append(kw.get("env", {})), - ) - monkeypatch.setattr( - "autoskillit.cli._update_checks._verify_update_result", lambda *a, **kw: True - ) - run_update_checks(home=tmp_path) - for env in env_passed: - assert env.get("AUTOSKILLIT_SKIP_STALE_CHECK") == "1" - assert env.get("AUTOSKILLIT_SKIP_SOURCE_DRIFT_CHECK") == "1" - - -def test_yes_single_invocation_exits_without_any_other_prompt( - monkeypatch: pytest.MonkeyPatch, tmp_path: Path -) -> None: - printed, input_calls = _setup_run_checks( - monkeypatch, tmp_path, binary_signal=True, hooks_signal=True, answer="y" - ) - - class FakeTG: - def __enter__(self): - return self - - def __exit__(self, *a): - return False - - monkeypatch.setattr("autoskillit.cli._update_checks.terminal_guard", FakeTG) - monkeypatch.setattr("autoskillit.cli._update_checks.subprocess.run", lambda *a, **kw: None) - monkeypatch.setattr( - "autoskillit.cli._update_checks._verify_update_result", lambda *a, **kw: True - ) - run_update_checks(home=tmp_path) - assert len(input_calls) == 1 - - -# --------------------------------------------------------------------------- -# UC-5 No path -# --------------------------------------------------------------------------- - - -def test_no_writes_single_unified_dismissal_entry( - monkeypatch: pytest.MonkeyPatch, tmp_path: Path -) -> None: - printed, input_calls = _setup_run_checks(monkeypatch, tmp_path, binary_signal=True, answer="n") - run_update_checks(home=tmp_path) - state = _read_dismiss_state(tmp_path) - assert "update_prompt" in state - # No legacy sub-keys - assert "binary" not in state - assert "hooks" not in state - assert "source_drift" not in state - - -def test_no_records_conditions_list_in_dismissal_entry( - monkeypatch: pytest.MonkeyPatch, tmp_path: Path -) -> None: - printed, input_calls = _setup_run_checks( - monkeypatch, tmp_path, binary_signal=True, hooks_signal=True, answer="n" - ) - run_update_checks(home=tmp_path) - state = _read_dismiss_state(tmp_path) - entry = state["update_prompt"] - assert isinstance(entry, dict) - conditions = entry["conditions"] - assert "binary" in conditions - assert "hooks" in conditions - - -def test_no_prints_expiry_date_line_with_correct_date( - monkeypatch: pytest.MonkeyPatch, tmp_path: Path -) -> None: - printed, input_calls = _setup_run_checks(monkeypatch, tmp_path, binary_signal=True, answer="n") - run_update_checks(home=tmp_path) - combined = " ".join(printed) - # Should mention "Dismissed until" - assert "Dismissed until" in combined - # And an escape hatch hint - assert "autoskillit update" in combined or "AUTOSKILLIT_SKIP_STALE_CHECK" in combined - - -def test_no_prints_escape_hatch_hint(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: - printed, input_calls = _setup_run_checks(monkeypatch, tmp_path, binary_signal=True, answer="n") - run_update_checks(home=tmp_path) - combined = " ".join(printed) - assert "autoskillit update" in combined - assert "AUTOSKILLIT_SKIP_STALE_CHECK=1" in combined - - -# --------------------------------------------------------------------------- -# UC-6 Branch-aware dismissal windows -# --------------------------------------------------------------------------- - - -def _dismissed_state( - ago: timedelta, - version: str = "0.7.77", - conditions: list[str] | None = None, -) -> dict: - return { - "update_prompt": { - "dismissed_at": (datetime.now(UTC) - ago).isoformat(), - "dismissed_version": version, - "conditions": conditions or ["binary"], - } - } - - -def test_stable_install_dismissal_silent_within_six_days( - monkeypatch: pytest.MonkeyPatch, tmp_path: Path -) -> None: - state = _dismissed_state(ago=timedelta(days=6)) - printed, input_calls = _setup_run_checks( - monkeypatch, tmp_path, binary_signal=True, state=state - ) - run_update_checks(home=tmp_path) - assert not input_calls - - -def test_stable_install_dismissal_reprompts_after_eight_days( - monkeypatch: pytest.MonkeyPatch, tmp_path: Path -) -> None: - state = _dismissed_state(ago=timedelta(days=8)) - printed, input_calls = _setup_run_checks( - monkeypatch, tmp_path, binary_signal=True, state=state - ) - run_update_checks(home=tmp_path) - assert len(input_calls) == 1 - - -def test_integration_install_dismissal_silent_within_eleven_hours( - monkeypatch: pytest.MonkeyPatch, tmp_path: Path -) -> None: - state = _dismissed_state(ago=timedelta(hours=11)) - printed, input_calls = _setup_run_checks( - monkeypatch, - tmp_path, - binary_signal=True, - info=_make_integration_info(), - state=state, - ) - run_update_checks(home=tmp_path) - assert not input_calls - - -def test_integration_install_dismissal_reprompts_after_thirteen_hours( - monkeypatch: pytest.MonkeyPatch, tmp_path: Path -) -> None: - state = _dismissed_state(ago=timedelta(hours=13)) - printed, input_calls = _setup_run_checks( - monkeypatch, - tmp_path, - binary_signal=True, - info=_make_integration_info(), - state=state, - ) - run_update_checks(home=tmp_path) - assert len(input_calls) == 1 - - -def test_dismissal_window_chosen_from_current_install_not_stored( - monkeypatch: pytest.MonkeyPatch, tmp_path: Path -) -> None: - """User dismisses on stable (7d window), then migrates to integration (12h window). - 13 hours later the prompt should re-appear under the integration window.""" - state = _dismissed_state(ago=timedelta(hours=13)) - # Info is now integration — 12h window applies - printed, input_calls = _setup_run_checks( - monkeypatch, - tmp_path, - binary_signal=True, - info=_make_integration_info(), - state=state, - ) - run_update_checks(home=tmp_path) - assert len(input_calls) == 1 - - -# --------------------------------------------------------------------------- -# UC-7 Time-windowed source-drift dismissal (no SHA keying) -# --------------------------------------------------------------------------- - - -def test_source_drift_dismissal_survives_new_upstream_commit_within_window( - monkeypatch: pytest.MonkeyPatch, tmp_path: Path -) -> None: - """Dismissed with ref=B; new check sees ref=C within window → still silent.""" - state = _dismissed_state(ago=timedelta(hours=1), conditions=["source_drift"]) - printed, input_calls = _setup_run_checks( - monkeypatch, tmp_path, source_drift_signal=True, state=state - ) - run_update_checks(home=tmp_path) - assert not input_calls - - -def test_source_drift_dismissal_expires_on_window_boundary( - monkeypatch: pytest.MonkeyPatch, tmp_path: Path -) -> None: - state = _dismissed_state(ago=timedelta(days=8), conditions=["source_drift"]) - printed, input_calls = _setup_run_checks( - monkeypatch, tmp_path, source_drift_signal=True, state=state - ) - run_update_checks(home=tmp_path) - assert len(input_calls) == 1 - - -def test_source_drift_dismissal_expires_on_version_delta( - monkeypatch: pytest.MonkeyPatch, tmp_path: Path -) -> None: - """Version advanced past dismissed_version → re-prompts regardless of time.""" - state = _dismissed_state(ago=timedelta(hours=1), version="0.7.77", conditions=["source_drift"]) - printed, input_calls = _setup_run_checks( - monkeypatch, tmp_path, source_drift_signal=True, state=state, current_version="0.7.78" - ) - run_update_checks(home=tmp_path) - assert len(input_calls) == 1 - - -# --------------------------------------------------------------------------- -# UC-8 Hook dismissal with version-delta -# --------------------------------------------------------------------------- - - -def test_hook_dismissal_expires_when_version_advances( - monkeypatch: pytest.MonkeyPatch, tmp_path: Path -) -> None: - state = _dismissed_state(ago=timedelta(hours=1), version="0.7.77", conditions=["hooks"]) - printed, input_calls = _setup_run_checks( - monkeypatch, tmp_path, hooks_signal=True, state=state, current_version="0.7.78" - ) - run_update_checks(home=tmp_path) - assert len(input_calls) == 1 - - -def test_hook_dismissal_holds_within_window_at_same_version( - monkeypatch: pytest.MonkeyPatch, tmp_path: Path -) -> None: - state = _dismissed_state(ago=timedelta(hours=1), version="0.7.77", conditions=["hooks"]) - printed, input_calls = _setup_run_checks( - monkeypatch, tmp_path, hooks_signal=True, state=state, current_version="0.7.77" - ) - run_update_checks(home=tmp_path) - assert not input_calls - - -# --------------------------------------------------------------------------- -# Dismiss state I/O -# --------------------------------------------------------------------------- - - -def test_read_dismiss_state_empty(tmp_path: Path) -> None: - assert _read_dismiss_state(tmp_path) == {} - - -def test_read_dismiss_state_malformed(tmp_path: Path) -> None: - p = tmp_path / ".autoskillit" / "update_check.json" - p.parent.mkdir(parents=True) - p.write_text("not-json", encoding="utf-8") - assert _read_dismiss_state(tmp_path) == {} - - -def test_write_dismiss_state_roundtrip(tmp_path: Path) -> None: - state = {"update_prompt": {"dismissed_at": "2026-01-01T00:00:00+00:00"}} - _write_dismiss_state(tmp_path, state) - assert _read_dismiss_state(tmp_path) == state - - -def test_read_dismiss_state_non_dict_json_returns_empty(tmp_path: Path) -> None: - p = tmp_path / ".autoskillit" / "update_check.json" - p.parent.mkdir(parents=True) - p.write_text("[1, 2, 3]", encoding="utf-8") - assert _read_dismiss_state(tmp_path) == {} - - -# --------------------------------------------------------------------------- -# _is_dismissed -# --------------------------------------------------------------------------- - - -def test_is_dismissed_within_window() -> None: - state = { - "update_prompt": { - "dismissed_at": (datetime.now(UTC) - timedelta(hours=1)).isoformat(), - "dismissed_version": "0.9.0", - "conditions": ["binary"], - } - } - assert _is_dismissed( - state, window=timedelta(hours=12), current_version="0.7.77", condition="binary" - ) - - -def test_is_dismissed_expired() -> None: - state = { - "update_prompt": { - "dismissed_at": (datetime.now(UTC) - timedelta(days=8)).isoformat(), - "dismissed_version": "0.9.0", - "conditions": ["binary"], - } - } - assert not _is_dismissed( - state, window=timedelta(days=7), current_version="0.7.77", condition="binary" - ) - - -def test_is_dismissed_newer_version_resets() -> None: - state = { - "update_prompt": { - "dismissed_at": (datetime.now(UTC) - timedelta(hours=1)).isoformat(), - "dismissed_version": "0.7.77", - "conditions": ["binary"], - } - } - assert not _is_dismissed( - state, window=timedelta(days=7), current_version="0.7.78", condition="binary" - ) - - -def test_is_dismissed_condition_not_in_list() -> None: - state = { - "update_prompt": { - "dismissed_at": (datetime.now(UTC) - timedelta(hours=1)).isoformat(), - "dismissed_version": "0.9.0", - "conditions": ["binary"], - } - } - # hooks was NOT dismissed — should still fire - assert not _is_dismissed( - state, window=timedelta(days=7), current_version="0.7.77", condition="hooks" - ) - - -def test_is_dismissed_empty_state() -> None: - assert not _is_dismissed( - {}, window=timedelta(days=7), current_version="0.7.77", condition="binary" - ) - - -# --------------------------------------------------------------------------- -# UC-9 Fetch-cache regression coverage -# --------------------------------------------------------------------------- - - -def test_fetch_latest_version_uses_cache_within_ttl(tmp_path: Path) -> None: - # Seed a cache entry that is fresh (1 second old, TTL = 30 min) - import time - - from autoskillit.cli._update_checks import _fetch_latest_version - - cache_data = { - "https://api.github.com/repos/TalonT-Org/AutoSkillit/releases/latest": { - "body": {"tag_name": "v0.9.0"}, - "etag": '"test-etag"', - "cached_at": time.time() - 1, - } - } - (tmp_path / ".autoskillit").mkdir(parents=True, exist_ok=True) - (tmp_path / ".autoskillit" / "github_fetch_cache.json").write_text( - json.dumps(cache_data), encoding="utf-8" - ) - call_count = [0] - - class CountingClient: - def __init__(self, **kw): - pass - - def __enter__(self): - call_count[0] += 1 - return self - - def __exit__(self, *a): - return False - - def get(self, url, **kw): - raise AssertionError("Should not hit network when cache is fresh") - - with patch("httpx.Client", CountingClient): - result = _fetch_latest_version("releases/latest", tmp_path) - - assert result == "0.9.0" - assert call_count[0] == 0 - - -def test_fetch_cache_expires_after_ttl(tmp_path: Path) -> None: - import time - - from autoskillit.cli._update_checks import _fetch_latest_version - - cache_data = { - "https://api.github.com/repos/TalonT-Org/AutoSkillit/releases/latest": { - "body": {"tag_name": "v0.8.0"}, - "etag": '"stale-etag"', - "cached_at": time.time() - 3601, # 1 hour + 1 second old - } - } - (tmp_path / ".autoskillit").mkdir(parents=True, exist_ok=True) - (tmp_path / ".autoskillit" / "github_fetch_cache.json").write_text( - json.dumps(cache_data), encoding="utf-8" - ) - - mock_client = _make_mock_client( - status_code=200, - json_body={"tag_name": "v0.9.0"}, - etag='"new-etag"', - ) - with patch("httpx.Client", return_value=mock_client): - result = _fetch_latest_version("releases/latest", tmp_path) - - assert result == "0.9.0" - - -def test_fetch_cache_respects_env_var_ttl(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: - import time - - from autoskillit.cli._update_checks import _fetch_latest_version - - # Entry is 61 seconds old — older than the custom 60s TTL - cache_data = { - "https://api.github.com/repos/TalonT-Org/AutoSkillit/releases/latest": { - "body": {"tag_name": "v0.8.0"}, - "etag": '"stale-etag"', - "cached_at": time.time() - 61, - } - } - (tmp_path / ".autoskillit").mkdir(parents=True, exist_ok=True) - (tmp_path / ".autoskillit" / "github_fetch_cache.json").write_text( - json.dumps(cache_data), encoding="utf-8" - ) - monkeypatch.setenv("AUTOSKILLIT_FETCH_CACHE_TTL_SECONDS", "60") - - mock_client = _make_mock_client( - status_code=200, - json_body={"tag_name": "v0.9.0"}, - ) - with patch("httpx.Client", return_value=mock_client): - result = _fetch_latest_version("releases/latest", tmp_path) - - assert result == "0.9.0" - - -def test_fetch_sends_github_token_auth_header( - monkeypatch: pytest.MonkeyPatch, tmp_path: Path -) -> None: - - monkeypatch.setenv("GITHUB_TOKEN", "my-secret-token") - (tmp_path / ".autoskillit").mkdir(parents=True, exist_ok=True) - - received_headers: dict = {} - - class CapturingClient: - def __init__(self, **kw): - pass - - def __enter__(self): - return self - - def __exit__(self, *a): - return False - - def get(self, url, headers=None, **kw): - received_headers.update(headers or {}) - r = MagicMock() - r.status_code = 200 - r.json.return_value = {"tag_name": "v0.9.0"} - r.headers = {} - return r - - with patch("httpx.Client", CapturingClient): - _fetch_with_cache( - "https://api.github.com/repos/TalonT-Org/AutoSkillit/releases/latest", - home=tmp_path, - ) - - assert "Authorization" in received_headers - assert received_headers["Authorization"] == "Bearer my-secret-token" - - -# --------------------------------------------------------------------------- -# timed_prompt primitive tests -# --------------------------------------------------------------------------- - - -def test_timed_prompt_returns_default_on_timeout(monkeypatch: pytest.MonkeyPatch) -> None: - """timed_prompt returns the default value when select.select times out.""" - import select as _select_mod - - from autoskillit.cli._timed_input import timed_prompt - - monkeypatch.setattr("sys.stdin.isatty", lambda: True) - monkeypatch.setattr("sys.stdout.isatty", lambda: True) - monkeypatch.delenv("NO_COLOR", raising=False) - - # select.select returns empty list = timeout - monkeypatch.setattr( - _select_mod, "select", lambda rlist, wlist, xlist, timeout=None: ([], [], []) - ) - - printed: list[str] = [] - monkeypatch.setattr("builtins.print", lambda *args, **kw: printed.append(str(args))) - - result = timed_prompt("Test prompt?", default="n", timeout=30, label="test") - assert result == "n" - assert any("timed out" in p for p in printed) - - -def test_timed_prompt_applies_ansi_formatting(monkeypatch: pytest.MonkeyPatch) -> None: - """timed_prompt output includes ANSI escape sequences when color is supported.""" - import select as _select_mod - - from autoskillit.cli._timed_input import timed_prompt - - monkeypatch.setattr("sys.stdin.isatty", lambda: True) - monkeypatch.setattr("sys.stdout.isatty", lambda: True) - monkeypatch.delenv("NO_COLOR", raising=False) - monkeypatch.delenv("TERM", raising=False) - - monkeypatch.setattr( - _select_mod, "select", lambda rlist, wlist, xlist, timeout=None: (rlist, [], []) - ) - monkeypatch.setattr("builtins.input", lambda _="": "y") - - raw_output: list[str] = [] - monkeypatch.setattr( - "builtins.print", - lambda *args, **kw: raw_output.append(" ".join(str(a) for a in args)), - ) - - timed_prompt("Update now? [Y/n]", default="n", timeout=30, label="test") - combined = " ".join(raw_output) - assert "\x1b[" in combined # ANSI escape present - - -def test_timed_prompt_respects_no_color(monkeypatch: pytest.MonkeyPatch) -> None: - """timed_prompt output has no ANSI sequences when NO_COLOR is set.""" - import select as _select_mod - - from autoskillit.cli._timed_input import timed_prompt - - monkeypatch.setattr("sys.stdin.isatty", lambda: True) - monkeypatch.setattr("sys.stdout.isatty", lambda: True) - monkeypatch.setenv("NO_COLOR", "1") - - monkeypatch.setattr( - _select_mod, "select", lambda rlist, wlist, xlist, timeout=None: (rlist, [], []) - ) - monkeypatch.setattr("builtins.input", lambda _="": "y") - - raw_output: list[str] = [] - monkeypatch.setattr( - "builtins.print", - lambda *args, **kw: raw_output.append(" ".join(str(a) for a in args)), - ) - - timed_prompt("Update now? [Y/n]", default="n", timeout=30, label="test") - combined = " ".join(raw_output) - assert "\x1b[" not in combined # No ANSI escapes - - -# --------------------------------------------------------------------------- -# Pre-flight feedback ordering test -# --------------------------------------------------------------------------- - - -def test_run_update_checks_emits_status_before_network( - monkeypatch: pytest.MonkeyPatch, - tmp_path: Path, -) -> None: - """The first print() call in run_update_checks occurs before any signal - gatherer is invoked (the 'time to first output' structural test).""" - import select as _select_mod - - from autoskillit.cli._update_checks import Signal - - monkeypatch.delenv("CLAUDECODE", raising=False) - monkeypatch.delenv("CI", raising=False) - monkeypatch.delenv("AUTOSKILLIT_SKIP_STALE_CHECK", raising=False) - monkeypatch.delenv("AUTOSKILLIT_SKIP_SOURCE_DRIFT_CHECK", raising=False) - monkeypatch.delenv("AUTOSKILLIT_FORCE_UPDATE_CHECK", raising=False) - - fake_stdin = MagicMock() - fake_stdin.isatty.return_value = True - fake_stdout = MagicMock() - fake_stdout.isatty.return_value = True - monkeypatch.setattr(sys, "stdin", fake_stdin) - monkeypatch.setattr(sys, "stdout", fake_stdout) - - monkeypatch.setattr( - _select_mod, - "select", - lambda rlist, wlist, xlist, timeout=None: (rlist, [], []), - ) - - info = _make_stable_info() - monkeypatch.setattr("autoskillit.cli._update_checks.detect_install", lambda: info) - monkeypatch.setattr( - "autoskillit.cli._update_checks._claude_settings_path", - lambda scope: tmp_path / "settings.json", - ) - - import autoskillit as _pkg - - monkeypatch.setattr(_pkg, "__version__", "0.7.77") - - # Track call order: print vs signal gatherers - call_log: list[str] = [] - - def tracking_print(*args: Any, **kw: Any) -> None: - call_log.append("print") - - monkeypatch.setattr("builtins.print", tracking_print) - - monkeypatch.setattr( - "autoskillit.cli._update_checks._binary_signal", - lambda info, home, current: ( - call_log.append("binary_signal") - or Signal("binary", "New release: 0.9.0 (you have 0.7.77)") - ), - ) - monkeypatch.setattr( - "autoskillit.cli._update_checks._hooks_signal", - lambda settings_path: call_log.append("hooks_signal"), - ) - monkeypatch.setattr( - "autoskillit.cli._update_checks._source_drift_signal", - lambda info, home: call_log.append("source_drift_signal"), - ) - - monkeypatch.setattr("builtins.input", lambda _="": "n") - - run_update_checks(home=tmp_path) - - # The first print (status_line) must come before any signal gatherer - assert call_log[0] == "print", ( - f"Expected first call to be print() (status_line), got: {call_log}" - ) - signal_calls = [c for c in call_log if c.endswith("_signal")] - assert signal_calls, f"No signal gatherers were called — call_log: {call_log}" - first_signal_idx = call_log.index(signal_calls[0]) - first_print_idx = call_log.index("print") - assert first_print_idx < first_signal_idx, ( - f"print() must occur before signal gatherers. Order: {call_log}" - ) - - -# --------------------------------------------------------------------------- -# AUTOSKILLIT_FORCE_UPDATE_CHECK override -# --------------------------------------------------------------------------- - - -def test_force_update_check_env_overrides_local_editable( - monkeypatch: pytest.MonkeyPatch, - tmp_path: Path, -) -> None: - """AUTOSKILLIT_FORCE_UPDATE_CHECK=1 bypasses the LOCAL_EDITABLE early return.""" - info = InstallInfo( - install_type=InstallType.LOCAL_EDITABLE, - commit_id=None, - requested_revision=None, - url=None, - editable_source=Path(tmp_path), - ) - printed, input_calls = _setup_run_checks( - monkeypatch, tmp_path, info=info, binary_signal=True, answer="n" - ) - monkeypatch.setenv("AUTOSKILLIT_FORCE_UPDATE_CHECK", "1") - run_update_checks(home=tmp_path) - # The prompt should have been reached (not early-returned) - assert len(input_calls) == 1 - - -def test_fetch_sends_if_none_match_when_cached_etag(tmp_path: Path) -> None: - import time - - cache_data = { - "https://api.github.com/repos/TalonT-Org/AutoSkillit/releases/latest": { - "body": {"tag_name": "v0.8.0"}, - "etag": '"cached-etag"', - "cached_at": time.time() - 3601, # stale, so will hit network - } - } - (tmp_path / ".autoskillit").mkdir(parents=True, exist_ok=True) - (tmp_path / ".autoskillit" / "github_fetch_cache.json").write_text( - json.dumps(cache_data), encoding="utf-8" - ) - - received_headers: dict = {} - - class CapturingClient: - def __init__(self, **kw): - pass - - def __enter__(self): - return self - - def __exit__(self, *a): - return False - - def get(self, url, headers=None, **kw): - received_headers.update(headers or {}) - r = MagicMock() - r.status_code = 200 - r.json.return_value = {"tag_name": "v0.9.0"} - r.headers = {} - return r - - with patch("httpx.Client", CapturingClient): - _fetch_with_cache( - "https://api.github.com/repos/TalonT-Org/AutoSkillit/releases/latest", - home=tmp_path, - ) - - assert received_headers.get("If-None-Match") == '"cached-etag"' - - -def test_fetch_304_response_returns_cached_payload(tmp_path: Path) -> None: - import time - - url = "https://api.github.com/repos/TalonT-Org/AutoSkillit/releases/latest" - cache_data = { - url: { - "body": {"tag_name": "v0.8.5"}, - "etag": '"my-etag"', - "cached_at": time.time() - 3601, # stale - } - } - (tmp_path / ".autoskillit").mkdir(parents=True, exist_ok=True) - (tmp_path / ".autoskillit" / "github_fetch_cache.json").write_text( - json.dumps(cache_data), encoding="utf-8" - ) - - mock_client = _make_mock_client(status_code=304) - with patch("httpx.Client", return_value=mock_client): - result = _fetch_with_cache(url, home=tmp_path) - - assert result == {"tag_name": "v0.8.5"} - - -def test_fetch_uses_correct_timeout(tmp_path: Path) -> None: - from autoskillit.cli._update_checks import _HTTP_TIMEOUT - - assert _HTTP_TIMEOUT.connect == 2.0 - assert _HTTP_TIMEOUT.read == 1.0 - assert _HTTP_TIMEOUT.write == 5.0 - assert _HTTP_TIMEOUT.pool == 1.0 - - -def test_fetch_sends_modern_github_api_version_header(tmp_path: Path) -> None: - (tmp_path / ".autoskillit").mkdir(parents=True, exist_ok=True) - - received_headers: dict = {} - - class CapturingClient: - def __init__(self, **kw): - pass - - def __enter__(self): - return self - - def __exit__(self, *a): - return False - - def get(self, url, headers=None, **kw): - received_headers.update(headers or {}) - r = MagicMock() - r.status_code = 200 - r.json.return_value = {"tag_name": "v0.9.0"} - r.headers = {} - return r - - with patch("httpx.Client", CapturingClient): - _fetch_with_cache( - "https://api.github.com/repos/TalonT-Org/AutoSkillit/releases/latest", - home=tmp_path, - ) - - assert received_headers.get("X-GitHub-Api-Version") == "2022-11-28" - assert received_headers.get("Accept") == "application/vnd.github+json" - - -def test_fetch_sends_user_agent_with_package_version(tmp_path: Path) -> None: - (tmp_path / ".autoskillit").mkdir(parents=True, exist_ok=True) - - received_headers: dict = {} - - class CapturingClient: - def __init__(self, **kw): - pass - - def __enter__(self): - return self - - def __exit__(self, *a): - return False - - def get(self, url, headers=None, **kw): - received_headers.update(headers or {}) - r = MagicMock() - r.status_code = 200 - r.json.return_value = {} - r.headers = {} - return r - - with patch("httpx.Client", CapturingClient): - _fetch_with_cache( - "https://api.github.com/repos/TalonT-Org/AutoSkillit/releases/latest", - home=tmp_path, - ) - - assert received_headers.get("User-Agent", "").startswith("autoskillit/") - - -def test_fetch_scrubs_authorization_header_from_logged_errors( - monkeypatch: pytest.MonkeyPatch, tmp_path: Path, caplog: pytest.LogCaptureFixture -) -> None: - import logging - - monkeypatch.setenv("GITHUB_TOKEN", "super-secret-token-xyz") - (tmp_path / ".autoskillit").mkdir(parents=True, exist_ok=True) - - import httpx as _httpx - - class FailingClient: - def __init__(self, **kw): - pass - - def __enter__(self): - return self - - def __exit__(self, *a): - return False - - def get(self, url, headers=None, **kw): - raise _httpx.ConnectError("Connection refused [super-secret-token-xyz]") - - with caplog.at_level(logging.DEBUG, logger="autoskillit"): - with patch("httpx.Client", FailingClient): - result = _fetch_with_cache( - "https://api.github.com/repos/TalonT-Org/AutoSkillit/releases/latest", - home=tmp_path, - ) - - assert result is None - # The token must not appear in any log record - for record in caplog.records: - assert "super-secret-token-xyz" not in record.getMessage() - - -def test_fetch_fails_fast_offline(tmp_path: Path) -> None: - import httpx as _httpx - - from autoskillit.cli._update_checks import _fetch_latest_version - - (tmp_path / ".autoskillit").mkdir(parents=True, exist_ok=True) - - class OfflineClient: - def __init__(self, **kw): - pass - - def __enter__(self): - return self - - def __exit__(self, *a): - return False - - def get(self, url, **kw): - raise _httpx.ConnectError("Network unreachable") - - with patch("httpx.Client", OfflineClient): - result = _fetch_latest_version("releases/latest", tmp_path) - - assert result is None diff --git a/tests/cli/test_update_checks_fetch.py b/tests/cli/test_update_checks_fetch.py new file mode 100644 index 0000000000..8d12edfcc5 --- /dev/null +++ b/tests/cli/test_update_checks_fetch.py @@ -0,0 +1,1061 @@ +"""Tests for cli/_update_checks.py — UC-9 fetch-cache regression coverage, +UC-11 fetch cache lifecycle, UC-12 state transitions, and T1/T2/T6 update +sequence and verification.""" + +from __future__ import annotations + +import json +import subprocess +from pathlib import Path +from typing import Any +from unittest.mock import MagicMock, patch + +import pytest + +from autoskillit.cli.update._update_checks_fetch import ( + _fetch_with_cache, +) + +from ._update_checks_helpers import ( + _make_develop_info, + _make_mock_client, + _make_stable_info, +) + +pytestmark = [pytest.mark.layer("cli"), pytest.mark.small] + +# --------------------------------------------------------------------------- +# UC-9 Fetch-cache regression coverage +# --------------------------------------------------------------------------- + + +def test_fetch_latest_version_uses_cache_within_ttl(tmp_path: Path) -> None: + # Seed a cache entry that is fresh (1 second old, TTL = 30 min) + import time + + from autoskillit.cli.update._update_checks import _fetch_latest_version + from autoskillit.core import AUTOSKILLIT_INSTALLED_VERSION + + cache_data = { + "https://api.github.com/repos/TalonT-Org/AutoSkillit/releases/latest": { + "body": {"tag_name": "v0.9.0"}, + "etag": '"test-etag"', + "cached_at": time.time() - 1, + "installed_version": AUTOSKILLIT_INSTALLED_VERSION, + } + } + (tmp_path / ".autoskillit").mkdir(parents=True, exist_ok=True) + (tmp_path / ".autoskillit" / "github_fetch_cache.json").write_text( + json.dumps(cache_data), encoding="utf-8" + ) + call_count = [0] + + class CountingClient: + def __init__(self, **kw): + pass + + def __enter__(self): + call_count[0] += 1 + return self + + def __exit__(self, *a): + return False + + def get(self, url, **kw): + raise AssertionError("Should not hit network when cache is fresh") + + with patch("httpx.Client", CountingClient): + result = _fetch_latest_version("releases/latest", tmp_path) + + assert result == "0.9.0" + assert call_count[0] == 0 + + +def test_fetch_cache_expires_after_ttl(tmp_path: Path) -> None: + import time + + from autoskillit.cli.update._update_checks import _fetch_latest_version + + cache_data = { + "https://api.github.com/repos/TalonT-Org/AutoSkillit/releases/latest": { + "body": {"tag_name": "v0.8.0"}, + "etag": '"stale-etag"', + "cached_at": time.time() - 3601, # 1 hour + 1 second old + } + } + (tmp_path / ".autoskillit").mkdir(parents=True, exist_ok=True) + (tmp_path / ".autoskillit" / "github_fetch_cache.json").write_text( + json.dumps(cache_data), encoding="utf-8" + ) + + mock_client = _make_mock_client( + status_code=200, + json_body={"tag_name": "v0.9.0"}, + etag='"new-etag"', + ) + with patch("httpx.Client", return_value=mock_client): + result = _fetch_latest_version("releases/latest", tmp_path) + + assert result == "0.9.0" + + +def test_fetch_cache_respects_env_var_ttl(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: + import time + + from autoskillit.cli.update._update_checks import _fetch_latest_version + + # Entry is 61 seconds old — older than the custom 60s TTL + cache_data = { + "https://api.github.com/repos/TalonT-Org/AutoSkillit/releases/latest": { + "body": {"tag_name": "v0.8.0"}, + "etag": '"stale-etag"', + "cached_at": time.time() - 61, + } + } + (tmp_path / ".autoskillit").mkdir(parents=True, exist_ok=True) + (tmp_path / ".autoskillit" / "github_fetch_cache.json").write_text( + json.dumps(cache_data), encoding="utf-8" + ) + monkeypatch.setenv("AUTOSKILLIT_FETCH_CACHE_TTL_SECONDS", "60") + + mock_client = _make_mock_client( + status_code=200, + json_body={"tag_name": "v0.9.0"}, + ) + with patch("httpx.Client", return_value=mock_client): + result = _fetch_latest_version("releases/latest", tmp_path) + + assert result == "0.9.0" + + +def test_fetch_sends_github_token_auth_header( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + + monkeypatch.setenv("GITHUB_TOKEN", "my-secret-token") + (tmp_path / ".autoskillit").mkdir(parents=True, exist_ok=True) + + received_headers: dict = {} + + class CapturingClient: + def __init__(self, **kw): + pass + + def __enter__(self): + return self + + def __exit__(self, *a): + return False + + def get(self, url, headers=None, **kw): + received_headers.update(headers or {}) + r = MagicMock() + r.status_code = 200 + r.json.return_value = {"tag_name": "v0.9.0"} + r.headers = {} + return r + + with patch("httpx.Client", CapturingClient): + _fetch_with_cache( + "https://api.github.com/repos/TalonT-Org/AutoSkillit/releases/latest", + home=tmp_path, + ) + + assert "Authorization" in received_headers + assert received_headers["Authorization"] == "Bearer my-secret-token" + + +def test_fetch_sends_if_none_match_when_cached_etag(tmp_path: Path) -> None: + import time + + cache_data = { + "https://api.github.com/repos/TalonT-Org/AutoSkillit/releases/latest": { + "body": {"tag_name": "v0.8.0"}, + "etag": '"cached-etag"', + "cached_at": time.time() - 3601, # stale, so will hit network + } + } + (tmp_path / ".autoskillit").mkdir(parents=True, exist_ok=True) + (tmp_path / ".autoskillit" / "github_fetch_cache.json").write_text( + json.dumps(cache_data), encoding="utf-8" + ) + + received_headers: dict = {} + + class CapturingClient: + def __init__(self, **kw): + pass + + def __enter__(self): + return self + + def __exit__(self, *a): + return False + + def get(self, url, headers=None, **kw): + received_headers.update(headers or {}) + r = MagicMock() + r.status_code = 200 + r.json.return_value = {"tag_name": "v0.9.0"} + r.headers = {} + return r + + with patch("httpx.Client", CapturingClient): + _fetch_with_cache( + "https://api.github.com/repos/TalonT-Org/AutoSkillit/releases/latest", + home=tmp_path, + ) + + assert received_headers.get("If-None-Match") == '"cached-etag"' + + +def test_fetch_304_response_returns_cached_payload(tmp_path: Path) -> None: + import time + + url = "https://api.github.com/repos/TalonT-Org/AutoSkillit/releases/latest" + cache_data = { + url: { + "body": {"tag_name": "v0.8.5"}, + "etag": '"my-etag"', + "cached_at": time.time() - 3601, # stale + } + } + (tmp_path / ".autoskillit").mkdir(parents=True, exist_ok=True) + (tmp_path / ".autoskillit" / "github_fetch_cache.json").write_text( + json.dumps(cache_data), encoding="utf-8" + ) + + mock_client = _make_mock_client(status_code=304) + with patch("httpx.Client", return_value=mock_client): + result = _fetch_with_cache(url, home=tmp_path) + + assert result == {"tag_name": "v0.8.5"} + + +def test_fetch_uses_correct_timeout(tmp_path: Path) -> None: + from autoskillit.cli.update._update_checks_fetch import _HTTP_TIMEOUT + + assert _HTTP_TIMEOUT.connect == 2.0 + assert _HTTP_TIMEOUT.read == 1.0 + assert _HTTP_TIMEOUT.write == 5.0 + assert _HTTP_TIMEOUT.pool == 1.0 + + +def test_fetch_sends_modern_github_api_version_header(tmp_path: Path) -> None: + (tmp_path / ".autoskillit").mkdir(parents=True, exist_ok=True) + + received_headers: dict = {} + + class CapturingClient: + def __init__(self, **kw): + pass + + def __enter__(self): + return self + + def __exit__(self, *a): + return False + + def get(self, url, headers=None, **kw): + received_headers.update(headers or {}) + r = MagicMock() + r.status_code = 200 + r.json.return_value = {"tag_name": "v0.9.0"} + r.headers = {} + return r + + with patch("httpx.Client", CapturingClient): + _fetch_with_cache( + "https://api.github.com/repos/TalonT-Org/AutoSkillit/releases/latest", + home=tmp_path, + ) + + assert received_headers.get("X-GitHub-Api-Version") == "2022-11-28" + assert received_headers.get("Accept") == "application/vnd.github+json" + + +def test_fetch_sends_user_agent_with_package_version(tmp_path: Path) -> None: + (tmp_path / ".autoskillit").mkdir(parents=True, exist_ok=True) + + received_headers: dict = {} + + class CapturingClient: + def __init__(self, **kw): + pass + + def __enter__(self): + return self + + def __exit__(self, *a): + return False + + def get(self, url, headers=None, **kw): + received_headers.update(headers or {}) + r = MagicMock() + r.status_code = 200 + r.json.return_value = {} + r.headers = {} + return r + + with patch("httpx.Client", CapturingClient): + _fetch_with_cache( + "https://api.github.com/repos/TalonT-Org/AutoSkillit/releases/latest", + home=tmp_path, + ) + + assert received_headers.get("User-Agent", "").startswith("autoskillit/") + + +def test_fetch_scrubs_authorization_header_from_logged_errors( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path, caplog: pytest.LogCaptureFixture +) -> None: + import logging + + monkeypatch.setenv("GITHUB_TOKEN", "super-secret-token-xyz") + (tmp_path / ".autoskillit").mkdir(parents=True, exist_ok=True) + + import httpx as _httpx + + class FailingClient: + def __init__(self, **kw): + pass + + def __enter__(self): + return self + + def __exit__(self, *a): + return False + + def get(self, url, headers=None, **kw): + raise _httpx.ConnectError("Connection refused [super-secret-token-xyz]") + + with caplog.at_level(logging.DEBUG, logger="autoskillit"): + with patch("httpx.Client", FailingClient): + result = _fetch_with_cache( + "https://api.github.com/repos/TalonT-Org/AutoSkillit/releases/latest", + home=tmp_path, + ) + + assert result is None + # The token must not appear in any log record + for record in caplog.records: + assert "super-secret-token-xyz" not in record.getMessage() + + +def test_fetch_fails_fast_offline(tmp_path: Path) -> None: + import httpx as _httpx + + from autoskillit.cli.update._update_checks import _fetch_latest_version + + (tmp_path / ".autoskillit").mkdir(parents=True, exist_ok=True) + + class OfflineClient: + def __init__(self, **kw): + pass + + def __enter__(self): + return self + + def __exit__(self, *a): + return False + + def get(self, url, **kw): + raise _httpx.ConnectError("Network unreachable") + + with patch("httpx.Client", OfflineClient): + result = _fetch_latest_version("releases/latest", tmp_path) + + assert result is None + + +# --------------------------------------------------------------------------- +# UC-11 Fetch cache lifecycle — version-epoch and invalidation +# --------------------------------------------------------------------------- + + +def test_stale_fetch_cache_after_install_detected_by_epoch( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + """Cache entry with stale installed_version is treated as a miss even within TTL.""" + import time + + from autoskillit.core.types._type_constants import ( + AUTOSKILLIT_INSTALLED_VERSION as _REAL_VERSION, + ) + + old_version = "0.0.0-stale" + assert old_version != _REAL_VERSION + + url = "https://api.github.com/repos/TalonT-Org/AutoSkillit/releases/latest" + cache_data = { + url: { + "body": {"tag_name": "v0.9.170"}, + "etag": '"old-etag"', + "cached_at": time.time() - 1, + "installed_version": old_version, + } + } + (tmp_path / ".autoskillit").mkdir(parents=True, exist_ok=True) + (tmp_path / ".autoskillit" / "github_fetch_cache.json").write_text( + json.dumps(cache_data), encoding="utf-8" + ) + + network_hit = [False] + + class TrackingClient: + def __init__(self, **kw): + pass + + def __enter__(self): + return self + + def __exit__(self, *a): + return False + + def get(self, url, **kw): + network_hit[0] = True + r = MagicMock() + r.status_code = 200 + r.json.return_value = {"tag_name": "v0.9.175"} + r.headers = {"ETag": '"new-etag"'} + return r + + with patch("httpx.Client", TrackingClient): + result = _fetch_with_cache(url, home=tmp_path) + + assert network_hit[0], "Epoch mismatch must force a network fetch" + assert result == {"tag_name": "v0.9.175"} + + +def test_stale_fetch_cache_after_install_resolve_reference_sha_path2( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + """PATH 2 (no source repo, fallback to _api_sha): stale epoch forces fresh fetch.""" + import time + + from autoskillit.cli.update._update_checks import resolve_reference_sha + + url = "https://api.github.com/repos/TalonT-Org/AutoSkillit/git/refs/heads/develop" + stale_sha = "a" * 40 + fresh_sha = "b" * 40 + + cache_data = { + url: { + "body": { + "object": {"sha": stale_sha, "type": "commit"}, + "ref": "refs/heads/develop", + }, + "etag": '"old-etag"', + "cached_at": time.time() - 1, + "installed_version": "0.9.170", + } + } + (tmp_path / ".autoskillit").mkdir(parents=True, exist_ok=True) + (tmp_path / ".autoskillit" / "github_fetch_cache.json").write_text( + json.dumps(cache_data), encoding="utf-8" + ) + + monkeypatch.setattr( + "autoskillit.cli.update._update_checks_source.find_source_repo", lambda: None + ) + + class FreshClient: + def __init__(self, **kw): + pass + + def __enter__(self): + return self + + def __exit__(self, *a): + return False + + def get(self, url, **kw): + r = MagicMock() + r.status_code = 200 + r.json.return_value = { + "object": {"sha": fresh_sha, "type": "commit"}, + "ref": "refs/heads/develop", + } + r.headers = {"ETag": '"fresh-etag"'} + return r + + info = _make_develop_info(commit_id=stale_sha) + with patch("httpx.Client", FreshClient): + result = resolve_reference_sha(info, tmp_path) + + assert result == fresh_sha, f"Expected fresh SHA {fresh_sha!r}, got {result!r}" + + +def test_run_update_sequence_invalidates_fetch_cache( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + """_run_update_sequence must delete github_fetch_cache.json on success.""" + from autoskillit.cli.update._update_checks import _run_update_sequence + + cache_file = tmp_path / ".autoskillit" / "github_fetch_cache.json" + cache_file.parent.mkdir(parents=True, exist_ok=True) + cache_file.write_text('{"some": "data"}', encoding="utf-8") + + class FakeTG: + def __enter__(self): + return self + + def __exit__(self, *a): + return False + + info = _make_stable_info() + upgrade_ok = subprocess.CompletedProcess([], returncode=0) + install_ok = subprocess.CompletedProcess([], returncode=0) + calls = iter([upgrade_ok, install_ok]) + monkeypatch.setattr( + "autoskillit.cli.update._update_checks.subprocess.run", lambda *a, **kw: next(calls) + ) + monkeypatch.setattr("autoskillit.cli.update._update_checks.terminal_guard", FakeTG) + monkeypatch.setattr( + "autoskillit.cli.update._update_checks._fetch_latest_version", lambda *a, **kw: "0.9.1" + ) + monkeypatch.setattr( + "autoskillit.cli.update._update_checks._verify_update_result", lambda *a, **kw: True + ) + monkeypatch.setattr("autoskillit.cli.update._update_checks.perform_restart", lambda: None) + _run_update_sequence(info, "0.9.0", tmp_path, {}, {}) + assert not cache_file.exists(), "Fetch cache must be deleted after successful update" + + +def test_run_update_command_invalidates_fetch_cache( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + """run_update_command must delete github_fetch_cache.json on success.""" + from autoskillit.cli.update._update import run_update_command + + cache_file = tmp_path / ".autoskillit" / "github_fetch_cache.json" + cache_file.parent.mkdir(parents=True, exist_ok=True) + cache_file.write_text('{"some": "data"}', encoding="utf-8") + + class FakeTG: + def __enter__(self): + return self + + def __exit__(self, *a): + return False + + info = _make_stable_info() + monkeypatch.setattr("autoskillit.cli.update._update.detect_install", lambda: info) + monkeypatch.setattr("autoskillit.cli.update._update.terminal_guard", FakeTG) + + upgrade_ok = subprocess.CompletedProcess([], returncode=0) + install_ok = subprocess.CompletedProcess([], returncode=0) + mock_run = MagicMock(side_effect=[upgrade_ok, install_ok]) + monkeypatch.setattr("autoskillit.cli.update._update.subprocess.run", mock_run) + + import autoskillit as _pkg + + monkeypatch.setattr(_pkg, "__version__", "0.9.0") + + import importlib.metadata + + monkeypatch.setattr(importlib.metadata, "version", lambda _: "0.9.1") + monkeypatch.setattr( + "autoskillit.cli.update._update_checks._fetch_latest_version", lambda *a, **kw: "0.9.1" + ) + monkeypatch.setattr("autoskillit.cli.update._update.perform_restart", lambda: None) + + run_update_command(home=tmp_path) + assert not cache_file.exists(), "Fetch cache must be deleted after successful update command" + + +def test_install_invalidates_fetch_cache(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: + """_marketplace.install() must delete github_fetch_cache.json after install.""" + import importlib + + _app_mod = importlib.import_module("autoskillit.cli._marketplace") + + cache_file = tmp_path / ".autoskillit" / "github_fetch_cache.json" + cache_file.parent.mkdir(parents=True, exist_ok=True) + cache_file.write_text('{"some": "data"}', encoding="utf-8") + + monkeypatch.setattr(_app_mod, "is_git_worktree", lambda path: False) + monkeypatch.setattr(Path, "home", lambda: tmp_path) + monkeypatch.setattr(Path, "cwd", lambda: tmp_path) + monkeypatch.delenv("CLAUDECODE", raising=False) + monkeypatch.setattr("shutil.which", lambda _: "/usr/bin/claude") + monkeypatch.setattr( + "subprocess.run", + lambda *a, **kw: type("R", (), {"returncode": 0, "stdout": "", "stderr": ""})(), + ) + monkeypatch.setattr("autoskillit.cli._marketplace.evict_direct_mcp_entry", lambda _: False) + monkeypatch.setattr( + "autoskillit.cli._marketplace.sweep_all_scopes_for_orphans", lambda _: None + ) + monkeypatch.setattr("autoskillit.cli._marketplace.sync_hooks_to_settings", lambda _: None) + monkeypatch.setattr("autoskillit.cli._marketplace.generate_hooks_json", lambda: {}) + monkeypatch.setattr("autoskillit.cli._marketplace.atomic_write", lambda *a, **kw: None) + + from autoskillit.cli._marketplace import install as _install + + _install(scope="user") + assert not cache_file.exists(), "Fetch cache must be deleted after plugin install" + + +def test_api_sha_with_seeded_cache_returns_cached_sha(tmp_path: Path) -> None: + """_api_sha returns cached SHA when cache epoch matches current version.""" + import time + + from autoskillit.cli.update._update_checks_source import _api_sha + from autoskillit.core import AUTOSKILLIT_INSTALLED_VERSION + + sha = "c" * 40 + url = "https://api.github.com/repos/TalonT-Org/AutoSkillit/git/refs/heads/develop" + cache_data = { + url: { + "body": {"object": {"sha": sha, "type": "commit"}, "ref": "refs/heads/develop"}, + "etag": '"test-etag"', + "cached_at": time.time() - 1, + "installed_version": AUTOSKILLIT_INSTALLED_VERSION, + } + } + (tmp_path / ".autoskillit").mkdir(parents=True, exist_ok=True) + (tmp_path / ".autoskillit" / "github_fetch_cache.json").write_text( + json.dumps(cache_data), encoding="utf-8" + ) + + class NoNetworkClient: + def __init__(self, **kw): + pass + + def __enter__(self): + return self + + def __exit__(self, *a): + return False + + def get(self, url, **kw): + raise AssertionError("Should not hit network when epoch matches") + + with patch("httpx.Client", NoNetworkClient): + result = _api_sha("develop", tmp_path) + + assert result == sha + + +def test_api_sha_with_stale_epoch_forces_network( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + """_api_sha issues a network request when cache epoch is stale.""" + import time + + from autoskillit.cli.update._update_checks_source import _api_sha + + stale_sha = "d" * 40 + fresh_sha = "e" * 40 + url = "https://api.github.com/repos/TalonT-Org/AutoSkillit/git/refs/heads/develop" + cache_data = { + url: { + "body": { + "object": {"sha": stale_sha, "type": "commit"}, + "ref": "refs/heads/develop", + }, + "etag": '"old-etag"', + "cached_at": time.time() - 1, + "installed_version": "0.9.170", + } + } + (tmp_path / ".autoskillit").mkdir(parents=True, exist_ok=True) + (tmp_path / ".autoskillit" / "github_fetch_cache.json").write_text( + json.dumps(cache_data), encoding="utf-8" + ) + + network_hit = [False] + + class FreshClient: + def __init__(self, **kw): + pass + + def __enter__(self): + return self + + def __exit__(self, *a): + return False + + def get(self, url, **kw): + network_hit[0] = True + r = MagicMock() + r.status_code = 200 + r.json.return_value = { + "object": {"sha": fresh_sha, "type": "commit"}, + "ref": "refs/heads/develop", + } + r.headers = {"ETag": '"fresh-etag"'} + return r + + with patch("httpx.Client", FreshClient): + result = _api_sha("develop", tmp_path) + + assert network_hit[0], "Stale epoch must force network fetch" + assert result == fresh_sha + + +def test_api_sha_network_false_reads_raw_cache_no_epoch(tmp_path: Path) -> None: + """_api_sha(network=False) reads raw cache regardless of epoch (doctor mode).""" + import time + + from autoskillit.cli.update._update_checks_source import _api_sha + + sha = "f" * 40 + url = "https://api.github.com/repos/TalonT-Org/AutoSkillit/git/refs/heads/develop" + cache_data = { + url: { + "body": {"object": {"sha": sha, "type": "commit"}, "ref": "refs/heads/develop"}, + "etag": '"cached-etag"', + "cached_at": time.time() - 1, + "installed_version": "0.0.0", + } + } + (tmp_path / ".autoskillit").mkdir(parents=True, exist_ok=True) + (tmp_path / ".autoskillit" / "github_fetch_cache.json").write_text( + json.dumps(cache_data), encoding="utf-8" + ) + + result = _api_sha("develop", tmp_path, network=False) + assert result == sha, "Doctor mode must read cache body regardless of epoch" + + +def test_api_sha_tags_url_prefix(tmp_path: Path) -> None: + """_api_sha('v0.9.174', ...) constructs a refs/tags/ URL.""" + from autoskillit.cli.update._update_checks_source import _api_sha + + (tmp_path / ".autoskillit").mkdir(parents=True, exist_ok=True) + captured_urls: list[str] = [] + + class UrlCapturingClient: + def __init__(self, **kw): + pass + + def __enter__(self): + return self + + def __exit__(self, *a): + return False + + def get(self, url, **kw): + captured_urls.append(url) + r = MagicMock() + r.status_code = 200 + r.json.return_value = {"object": {"sha": "a" * 40, "type": "commit"}} + r.headers = {} + return r + + with patch("httpx.Client", UrlCapturingClient): + _api_sha("v0.9.174", tmp_path) + + assert len(captured_urls) == 1 + assert "refs/tags/v0.9.174" in captured_urls[0] + + +# --------------------------------------------------------------------------- +# UC-12 State transitions — cross-hemisphere lifecycle tests +# --------------------------------------------------------------------------- + + +def test_full_lifecycle_install_clears_stale_cache_then_check_detects_new_version( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + """Full lifecycle: install invalidates cache, next binary_signal detects new version.""" + import time + + from autoskillit.cli.update._update_checks import _binary_signal, invalidate_fetch_cache + from autoskillit.core import AUTOSKILLIT_INSTALLED_VERSION + + stale_version = "0.0.0-stale" + newer_version = "99.99.99" + + url = "https://api.github.com/repos/TalonT-Org/AutoSkillit/releases/latest" + cache_data = { + url: { + "body": {"tag_name": f"v{stale_version}"}, + "etag": '"old-etag"', + "cached_at": time.time(), + "installed_version": stale_version, + }, + "https://api.github.com/repos/TalonT-Org/AutoSkillit/git/refs/heads/develop": { + "body": {"object": {"sha": "a" * 40}}, + "etag": '"ref-etag"', + "cached_at": time.time(), + "installed_version": stale_version, + }, + } + cache_file = tmp_path / ".autoskillit" / "github_fetch_cache.json" + cache_file.parent.mkdir(parents=True, exist_ok=True) + cache_file.write_text(json.dumps(cache_data), encoding="utf-8") + + invalidate_fetch_cache(tmp_path) + assert not cache_file.exists(), "invalidate must remove cache file" + + mock_client = _make_mock_client( + status_code=200, + json_body={"tag_name": f"v{newer_version}"}, + etag='"new-etag"', + ) + info = _make_stable_info() + with patch("httpx.Client", return_value=mock_client): + signal = _binary_signal(info, tmp_path, AUTOSKILLIT_INSTALLED_VERSION) + + assert signal is not None, "Binary signal must fire after cache invalidation" + assert newer_version in signal.message + + +@pytest.mark.parametrize( + "entry_kwargs,expect_hit", + [ + pytest.param( + {"installed_version": "_CURRENT_"}, + True, + id="matching-epoch-fresh-ttl", + ), + pytest.param( + {"installed_version": "0.0.1"}, + False, + id="mismatched-epoch-fresh-ttl", + ), + pytest.param( + {}, + False, + id="missing-epoch-fresh-ttl", + ), + pytest.param( + {"installed_version": "_CURRENT_", "cached_at_offset": -3601}, + False, + id="matching-epoch-expired-ttl", + ), + ], +) +def test_fetch_with_cache_epoch_check_contract( + tmp_path: Path, + entry_kwargs: dict, + expect_hit: bool, +) -> None: + """Parametrized contract: epoch + TTL together determine cache hit/miss.""" + import time + + from autoskillit.core import AUTOSKILLIT_INSTALLED_VERSION + + entry_kwargs = dict(entry_kwargs) + cached_at_offset = entry_kwargs.pop("cached_at_offset", -1) + url = "https://api.github.com/repos/TalonT-Org/AutoSkillit/releases/latest" + entry: dict[str, Any] = { + "body": {"tag_name": "v0.8.0"}, + "etag": '"test-etag"', + "cached_at": time.time() + cached_at_offset, + } + for k, v in entry_kwargs.items(): + entry[k] = AUTOSKILLIT_INSTALLED_VERSION if v == "_CURRENT_" else v + + cache_data = {url: entry} + (tmp_path / ".autoskillit").mkdir(parents=True, exist_ok=True) + (tmp_path / ".autoskillit" / "github_fetch_cache.json").write_text( + json.dumps(cache_data), encoding="utf-8" + ) + + network_hit = [False] + + class DetectingClient: + def __init__(self, **kw): + pass + + def __enter__(self): + return self + + def __exit__(self, *a): + return False + + def get(self, url, **kw): + network_hit[0] = True + r = MagicMock() + r.status_code = 200 + r.json.return_value = {"tag_name": "v0.9.0"} + r.headers = {"ETag": '"fresh-etag"'} + return r + + with patch("httpx.Client", DetectingClient): + result = _fetch_with_cache(url, home=tmp_path) + + if expect_hit: + assert not network_hit[0], "Expected cache hit but network was called" + assert result == {"tag_name": "v0.8.0"} + else: + assert network_hit[0], "Expected cache miss but network was not called" + + +# --------------------------------------------------------------------------- +# T1 — _verify_update_result uses install-type-aware upgrade command +# --------------------------------------------------------------------------- + + +def test_verify_update_result_prints_git_vcs_stable_command( + tmp_path: Path, capsys: pytest.CaptureFixture +) -> None: + import importlib.metadata + + from autoskillit.cli.update._update_checks import _verify_update_result + + info = _make_stable_info() + with patch.object(importlib.metadata, "version", return_value="0.9.0"): + result = _verify_update_result(info, "0.9.0", "0.9.1", tmp_path, {}) + assert result is False + out = capsys.readouterr().out + assert "uv tool upgrade autoskillit" in out + assert "autoskillit update" in out + + +def test_verify_update_result_prints_git_vcs_develop_command( + tmp_path: Path, capsys: pytest.CaptureFixture +) -> None: + import importlib.metadata + + from autoskillit.cli.update._update_checks import _verify_update_result + + info = _make_develop_info() + with patch.object(importlib.metadata, "version", return_value="0.9.0"): + result = _verify_update_result(info, "0.9.0", "0.9.1", tmp_path, {}) + assert result is False + out = capsys.readouterr().out + assert "git+" in out + assert "uv tool upgrade autoskillit" not in out + + +# --------------------------------------------------------------------------- +# T2 — _run_update_sequence warns when autoskillit install exits non-zero +# --------------------------------------------------------------------------- + + +def test_run_update_sequence_warns_on_install_failure( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture +) -> None: + from autoskillit.cli.update._update_checks import _run_update_sequence + + class FakeTG: + def __enter__(self): + return self + + def __exit__(self, *a): + return False + + info = _make_stable_info() + upgrade_ok = subprocess.CompletedProcess([], returncode=0) + install_fail = subprocess.CompletedProcess([], returncode=1) + calls = iter([upgrade_ok, install_fail]) + monkeypatch.setattr( + "autoskillit.cli.update._update_checks.subprocess.run", lambda *a, **kw: next(calls) + ) + monkeypatch.setattr("autoskillit.cli.update._update_checks.terminal_guard", FakeTG) + monkeypatch.setattr( + "autoskillit.cli.update._update_checks._fetch_latest_version", lambda *a, **kw: "0.9.1" + ) + monkeypatch.setattr( + "autoskillit.cli.update._update_checks._verify_update_result", lambda *a, **kw: True + ) + monkeypatch.setattr("autoskillit.cli.update._update_checks.perform_restart", lambda: None) + _run_update_sequence(info, "0.9.0", tmp_path, {}, {}) + out = capsys.readouterr().out + assert "autoskillit install" in out + assert "stale" in out.lower() + + +# --------------------------------------------------------------------------- +# T6 — binary_snoozed is never written by _verify_update_result +# --------------------------------------------------------------------------- + + +def test_run_update_sequence_restarts_on_success( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + """After a successful upgrade, _run_update_sequence must call perform_restart.""" + from autoskillit.cli.update._update_checks import _run_update_sequence + + info = _make_stable_info() + upgrade_ok = subprocess.CompletedProcess([], returncode=0) + install_ok = subprocess.CompletedProcess([], returncode=0) + calls = iter([upgrade_ok, install_ok]) + monkeypatch.setattr( + "autoskillit.cli.update._update_checks.subprocess.run", lambda *a, **kw: next(calls) + ) + + class FakeTG: + def __enter__(self): + return self + + def __exit__(self, *a): + return False + + monkeypatch.setattr("autoskillit.cli.update._update_checks.terminal_guard", FakeTG) + monkeypatch.setattr( + "autoskillit.cli.update._update_checks._fetch_latest_version", lambda *a, **kw: "0.9.1" + ) + monkeypatch.setattr( + "autoskillit.cli.update._update_checks._verify_update_result", lambda *a, **kw: True + ) + + restart_called: list[bool] = [] + monkeypatch.setattr( + "autoskillit.cli.update._update_checks.perform_restart", + lambda: restart_called.append(True), + ) + + _run_update_sequence(info, "0.9.0", tmp_path, {}, {}) + assert restart_called, ( + "_run_update_sequence must call perform_restart() after successful upgrade" + ) + + +def test_verify_update_result_does_not_write_binary_snoozed( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + import importlib.metadata + + from autoskillit.cli.update._update_checks import _verify_update_result + + info = _make_stable_info(commit_id="abc") + state: dict = {} + with patch.object(importlib.metadata, "version", return_value="0.9.0"): + _verify_update_result(info, "0.9.0", "0.9.1", tmp_path, state) + assert "binary_snoozed" not in state + + +# --------------------------------------------------------------------------- +# Step 1e — _fetch_latest_version contract: target routing +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize( + "target,expected_url_fragment", + [ + ("develop", "contents/pyproject.toml"), + ("releases/latest", "releases/latest"), + ], +) +def test_fetch_latest_version_routes_by_target( + target: str, expected_url_fragment: str, tmp_path: Path +) -> None: + from autoskillit.cli.update._update_checks import _fetch_latest_version + + fetched_urls: list[str] = [] + + def _mock_fetch(url: str, *, home: Path) -> dict | None: + fetched_urls.append(url) + if "pyproject.toml" in url: + import base64 + + content = base64.b64encode(b'version = "0.9.300"\n').decode() + return {"content": content} + return {"tag_name": "v0.9.300"} + + with patch( + "autoskillit.cli.update._update_checks_fetch._fetch_with_cache", side_effect=_mock_fetch + ): + result = _fetch_latest_version(target, tmp_path) + + assert result is not None + assert fetched_urls, "Expected _fetch_with_cache to be called" + assert any(expected_url_fragment in url for url in fetched_urls), ( + f"Expected URL containing '{expected_url_fragment}', got: {fetched_urls}" + ) diff --git a/tests/cli/test_update_checks_guards.py b/tests/cli/test_update_checks_guards.py new file mode 100644 index 0000000000..493f9e7d18 --- /dev/null +++ b/tests/cli/test_update_checks_guards.py @@ -0,0 +1,414 @@ +"""Tests for cli/_update_checks.py — UC-1 early-return guards, UC-2 signal gatherers, +and find_source_repo behavioral coverage.""" + +from __future__ import annotations + +import io +import sys +from pathlib import Path +from unittest.mock import MagicMock + +import pytest + +from autoskillit.cli._install_info import InstallInfo, InstallType +from autoskillit.cli.update._update_checks import run_update_checks + +from ._update_checks_helpers import _make_develop_info, _make_stable_info + +pytestmark = [pytest.mark.layer("cli"), pytest.mark.small] + +# --------------------------------------------------------------------------- +# UC-1 Guards +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize( + "env_var,value", + [ + ("CLAUDECODE", "1"), + ("CI", "1"), + ("AUTOSKILLIT_SKIP_STALE_CHECK", "1"), + ("AUTOSKILLIT_SKIP_UPDATE_CHECK", "1"), + ("AUTOSKILLIT_SKIP_SOURCE_DRIFT_CHECK", "1"), + ], +) +def test_run_update_checks_skips_on_guard_env_var( + monkeypatch: pytest.MonkeyPatch, + env_var: str, + value: str, + tmp_path: Path, +) -> None: + monkeypatch.setenv(env_var, value) + # Ensure no other guard vars are set + for other in [ + "CLAUDECODE", + "CI", + "AUTOSKILLIT_SKIP_STALE_CHECK", + "AUTOSKILLIT_SKIP_UPDATE_CHECK", + "AUTOSKILLIT_SKIP_SOURCE_DRIFT_CHECK", + ]: + if other != env_var: + monkeypatch.delenv(other, raising=False) + fetched: list[str] = [] + monkeypatch.setattr( + "autoskillit.cli.update._update_checks_fetch._fetch_with_cache", + lambda url, **kw: fetched.append(url) or None, + ) + prompted: list[str] = [] + monkeypatch.setattr("builtins.input", lambda _: prompted.append("called") or "n") + run_update_checks(home=tmp_path) + assert not fetched, "No network fetch should occur under guard env" + assert not prompted, "No input() call under guard env" + + +def test_run_update_checks_skips_non_tty_stdin( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + monkeypatch.delenv("CLAUDECODE", raising=False) + monkeypatch.delenv("CI", raising=False) + monkeypatch.delenv("AUTOSKILLIT_SKIP_STALE_CHECK", raising=False) + monkeypatch.delenv("AUTOSKILLIT_SKIP_UPDATE_CHECK", raising=False) + monkeypatch.delenv("AUTOSKILLIT_SKIP_SOURCE_DRIFT_CHECK", raising=False) + fake_stdin = io.StringIO() + fake_stdout = io.StringIO() + monkeypatch.setattr(sys, "stdin", fake_stdin) + monkeypatch.setattr(sys, "stdout", fake_stdout) + fetched: list[str] = [] + monkeypatch.setattr( + "autoskillit.cli.update._update_checks_fetch._fetch_with_cache", + lambda url, **kw: fetched.append(url) or None, + ) + run_update_checks(home=tmp_path) + assert not fetched + + +def test_run_update_checks_skips_non_tty_stdout( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + monkeypatch.delenv("CLAUDECODE", raising=False) + monkeypatch.delenv("CI", raising=False) + monkeypatch.delenv("AUTOSKILLIT_SKIP_STALE_CHECK", raising=False) + monkeypatch.delenv("AUTOSKILLIT_SKIP_UPDATE_CHECK", raising=False) + monkeypatch.delenv("AUTOSKILLIT_SKIP_SOURCE_DRIFT_CHECK", raising=False) + + fake_stdin = MagicMock() + fake_stdin.isatty.return_value = True + fake_stdout = io.StringIO() + monkeypatch.setattr(sys, "stdin", fake_stdin) + monkeypatch.setattr(sys, "stdout", fake_stdout) + fetched: list[str] = [] + monkeypatch.setattr( + "autoskillit.cli.update._update_checks_fetch._fetch_with_cache", + lambda url, **kw: fetched.append(url) or None, + ) + run_update_checks(home=tmp_path) + assert not fetched + + +@pytest.mark.parametrize( + "install_type", + [InstallType.LOCAL_EDITABLE, InstallType.LOCAL_PATH, InstallType.UNKNOWN], +) +def test_run_update_checks_skips_local_and_unknown_install_types( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path, install_type: InstallType +) -> None: + monkeypatch.delenv("CLAUDECODE", raising=False) + monkeypatch.delenv("CI", raising=False) + monkeypatch.delenv("AUTOSKILLIT_SKIP_STALE_CHECK", raising=False) + monkeypatch.delenv("AUTOSKILLIT_SKIP_UPDATE_CHECK", raising=False) + monkeypatch.delenv("AUTOSKILLIT_SKIP_SOURCE_DRIFT_CHECK", raising=False) + monkeypatch.delenv("AUTOSKILLIT_FORCE_UPDATE_CHECK", raising=False) + + fake_stdin = MagicMock() + fake_stdin.isatty.return_value = True + fake_stdout = MagicMock() + fake_stdout.isatty.return_value = True + monkeypatch.setattr(sys, "stdin", fake_stdin) + monkeypatch.setattr(sys, "stdout", fake_stdout) + + info = InstallInfo( + install_type=install_type, + commit_id=None, + requested_revision=None, + url=None, + editable_source=Path(tmp_path) if install_type == InstallType.LOCAL_EDITABLE else None, + ) + monkeypatch.setattr("autoskillit.cli.update._update_checks.detect_install", lambda: info) + fetched: list[str] = [] + monkeypatch.setattr( + "autoskillit.cli.update._update_checks_fetch._fetch_with_cache", + lambda url, **kw: fetched.append(url) or None, + ) + prompted: list[str] = [] + monkeypatch.setattr("builtins.input", lambda _: prompted.append("called") or "n") + run_update_checks(home=tmp_path) + assert not fetched + assert not prompted + + +# --------------------------------------------------------------------------- +# UC-2 Signal gathering +# --------------------------------------------------------------------------- + + +def test_binary_signal_fires_when_newer_version_available( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + from autoskillit.cli.update._update_checks import _binary_signal + + info = _make_stable_info() + monkeypatch.setattr( + "autoskillit.cli.update._update_checks._fetch_latest_version", + lambda target, home: "0.9.0", + ) + sig = _binary_signal(info, tmp_path, "0.7.77") + assert sig is not None + assert sig.kind == "binary" + assert "0.9.0" in sig.message + assert "0.7.77" in sig.message + + +def test_binary_signal_silent_when_same_version( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + from autoskillit.cli.update._update_checks import _binary_signal + + info = _make_stable_info() + monkeypatch.setattr( + "autoskillit.cli.update._update_checks._fetch_latest_version", + lambda target, home: "0.7.77", + ) + assert _binary_signal(info, tmp_path, "0.7.77") is None + + +def test_binary_signal_silent_when_fetch_returns_none( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + from autoskillit.cli.update._update_checks import _binary_signal + + info = _make_stable_info() + monkeypatch.setattr( + "autoskillit.cli.update._update_checks._fetch_latest_version", + lambda target, home: None, + ) + assert _binary_signal(info, tmp_path, "0.7.77") is None + + +def test_binary_signal_uses_releases_url_for_stable( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + from autoskillit.cli.update._update_checks import _binary_signal + + info = _make_stable_info() + targets: list[str] = [] + monkeypatch.setattr( + "autoskillit.cli.update._update_checks._fetch_latest_version", + lambda target, home: targets.append(target) or "0.9.0", + ) + _binary_signal(info, tmp_path, "0.7.77") + assert targets == ["releases/latest"] + + +def test_binary_signal_uses_develop_url_for_develop( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + from autoskillit.cli.update._update_checks import _binary_signal + + info = _make_develop_info() + targets: list[str] = [] + monkeypatch.setattr( + "autoskillit.cli.update._update_checks._fetch_latest_version", + lambda target, home: targets.append(target) or "0.9.0", + ) + _binary_signal(info, tmp_path, "0.7.77") + assert targets == ["develop"] + + +def test_hooks_signal_fires_on_missing_hooks( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + from autoskillit.cli.update._update_checks import _hooks_signal + from autoskillit.hook_registry import HookDriftResult + + monkeypatch.setattr( + "autoskillit.cli.update._update_checks._count_hook_registry_drift", + lambda path: HookDriftResult(missing=3, orphaned=0), + ) + sig = _hooks_signal(tmp_path / "settings.json") + assert sig is not None + assert sig.kind == "hooks" + assert "3" in sig.message + + +def test_hooks_signal_fires_on_orphaned_hooks( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + from autoskillit.cli.update._update_checks import _hooks_signal + from autoskillit.hook_registry import HookDriftResult + + monkeypatch.setattr( + "autoskillit.cli.update._update_checks._count_hook_registry_drift", + lambda path: HookDriftResult(missing=0, orphaned=2), + ) + sig = _hooks_signal(tmp_path / "settings.json") + assert sig is not None + assert sig.kind == "hooks" + assert "2" in sig.message + + +def test_hooks_signal_silent_when_no_drift( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + from autoskillit.cli.update._update_checks import _hooks_signal + from autoskillit.hook_registry import HookDriftResult + + monkeypatch.setattr( + "autoskillit.cli.update._update_checks._count_hook_registry_drift", + lambda path: HookDriftResult(missing=0, orphaned=0), + ) + assert _hooks_signal(tmp_path / "settings.json") is None + + +def test_source_drift_signal_fires_when_commit_lags_ref( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + from autoskillit.cli.update._update_checks import _source_drift_signal + + info = _make_stable_info(commit_id="aaaaaa") + monkeypatch.setattr( + "autoskillit.cli.update._update_checks.resolve_reference_sha", + lambda info, home, **kw: "bbbbbb", + ) + sig = _source_drift_signal(info, tmp_path) + assert sig is not None + assert sig.kind == "source_drift" + # "source drift" must NOT appear in user-visible text (per plan verification item 11) + assert "source drift" not in sig.message.lower() + + +def test_source_drift_signal_silent_when_sha_matches( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + from autoskillit.cli.update._update_checks import _source_drift_signal + + info = _make_stable_info(commit_id="aaaaaa") + monkeypatch.setattr( + "autoskillit.cli.update._update_checks.resolve_reference_sha", + lambda info, home, **kw: "aaaaaa", + ) + assert _source_drift_signal(info, tmp_path) is None + + +def test_source_drift_signal_silent_when_ref_sha_none( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + from autoskillit.cli.update._update_checks import _source_drift_signal + + info = _make_stable_info(commit_id="aaaaaa") + monkeypatch.setattr( + "autoskillit.cli.update._update_checks.resolve_reference_sha", + lambda info, home, **kw: None, + ) + assert _source_drift_signal(info, tmp_path) is None + + +def test_dual_mcp_signal_fires_when_both_registered( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + from autoskillit.cli.update._update_checks import _dual_mcp_signal + + monkeypatch.setattr( + "autoskillit.cli.update._update_checks._is_dual_mcp_registered", + lambda home: True, + ) + sig = _dual_mcp_signal(tmp_path) + assert sig is not None + assert sig.kind == "dual_mcp" + assert "autoskillit install" in sig.message + + +def test_dual_mcp_signal_silent_when_only_one_registered( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + from autoskillit.cli.update._update_checks import _dual_mcp_signal + + monkeypatch.setattr( + "autoskillit.cli.update._update_checks._is_dual_mcp_registered", + lambda home: False, + ) + sig = _dual_mcp_signal(tmp_path) + assert sig is None + + +def test_dual_mcp_signal_silent_on_corrupted_files( + tmp_path: Path, +) -> None: + # _check_dual_mcp_files is fail-open: corrupted JSON → False → no signal + from autoskillit.cli.update._update_checks import _dual_mcp_signal + + claude_json = tmp_path / ".claude.json" + claude_json.write_text("{invalid json") + sig = _dual_mcp_signal(tmp_path) + assert sig is None + + +# --------------------------------------------------------------------------- +# find_source_repo behavioral tests +# --------------------------------------------------------------------------- + + +def test_find_source_repo_env_var_override_valid( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + """AUTOSKILLIT_SOURCE_REPO env var pointing to a valid repo root is returned.""" + from autoskillit.cli.update._update_checks_source import find_source_repo + + src_dir = tmp_path / "src" / "autoskillit" + src_dir.mkdir(parents=True) + monkeypatch.setenv("AUTOSKILLIT_SOURCE_REPO", str(tmp_path)) + result = find_source_repo() + assert result == tmp_path + + +def test_find_source_repo_env_var_override_invalid_falls_through( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + """AUTOSKILLIT_SOURCE_REPO pointing to a path without src/autoskillit/ is ignored.""" + from autoskillit.cli.update._update_checks_source import find_source_repo + + monkeypatch.setenv("AUTOSKILLIT_SOURCE_REPO", str(tmp_path / "nonexistent")) + monkeypatch.setattr("autoskillit.cli.update._update_checks.Path.cwd", lambda: tmp_path) + result = find_source_repo() + assert result is None + + +def test_find_source_repo_cwd_walk_finds_pyproject( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + """CWD walk finds the project root when pyproject.toml has name=autoskillit.""" + from autoskillit.cli.update._update_checks_source import find_source_repo + + project_root = tmp_path / "project" + src_dir = project_root / "src" / "autoskillit" + src_dir.mkdir(parents=True) + pyproject = project_root / "pyproject.toml" + pyproject.write_text('[project]\nname = "autoskillit"\n', encoding="utf-8") + + nested_cwd = project_root / "src" / "autoskillit" / "cli" + nested_cwd.mkdir(parents=True) + monkeypatch.delenv("AUTOSKILLIT_SOURCE_REPO", raising=False) + monkeypatch.setattr("autoskillit.cli.update._update_checks.Path.cwd", lambda: nested_cwd) + result = find_source_repo() + assert result == project_root + + +def test_find_source_repo_cwd_walk_no_match_returns_none( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + """CWD walk returns None when no autoskillit project is found.""" + from autoskillit.cli.update._update_checks_source import find_source_repo + + monkeypatch.delenv("AUTOSKILLIT_SOURCE_REPO", raising=False) + monkeypatch.setattr("autoskillit.cli.update._update_checks.Path.cwd", lambda: tmp_path) + result = find_source_repo() + assert result is None diff --git a/tests/cli/test_update_checks_prompt.py b/tests/cli/test_update_checks_prompt.py new file mode 100644 index 0000000000..8e2b7dda58 --- /dev/null +++ b/tests/cli/test_update_checks_prompt.py @@ -0,0 +1,817 @@ +"""Tests for cli/_update_checks.py — UC-3 through UC-10: prompt consolidation, +yes/no paths, dismissal windows, timed_prompt primitives, FORCE env, and passive +notification for dismissed signals.""" + +from __future__ import annotations + +import json +import subprocess +import sys +from datetime import UTC, datetime, timedelta +from pathlib import Path +from unittest.mock import MagicMock + +import pytest + +from autoskillit.cli._install_info import InstallInfo, InstallType +from autoskillit.cli.update._update_checks import ( + _is_dismissed, + _read_dismiss_state, + _write_dismiss_state, + run_update_checks, +) + +from ._update_checks_helpers import _make_develop_info, _make_stable_info + +pytestmark = [pytest.mark.layer("cli"), pytest.mark.small] + +# --------------------------------------------------------------------------- +# UC-3 Prompt consolidation +# --------------------------------------------------------------------------- + + +def _setup_run_checks( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, + *, + info: InstallInfo | None = None, + binary_signal: bool = False, + hooks_signal: bool = False, + source_drift_signal: bool = False, + answer: str = "n", + current_version: str = "0.7.77", + state: dict | None = None, +) -> tuple[list[str], list[str]]: + """Set up mocks for run_update_checks and return (printed_lines, input_calls).""" + import select as _select_mod + + from autoskillit.cli.update._update_checks import Signal + + monkeypatch.delenv("CLAUDECODE", raising=False) + monkeypatch.delenv("CI", raising=False) + monkeypatch.delenv("AUTOSKILLIT_SKIP_STALE_CHECK", raising=False) + monkeypatch.delenv("AUTOSKILLIT_SKIP_UPDATE_CHECK", raising=False) + monkeypatch.delenv("AUTOSKILLIT_SKIP_SOURCE_DRIFT_CHECK", raising=False) + monkeypatch.delenv("AUTOSKILLIT_FORCE_UPDATE_CHECK", raising=False) + + fake_stdin = MagicMock() + fake_stdin.isatty.return_value = True + fake_stdout = MagicMock() + fake_stdout.isatty.return_value = True + monkeypatch.setattr(sys, "stdin", fake_stdin) + monkeypatch.setattr(sys, "stdout", fake_stdout) + + # timed_prompt uses select.select to implement timeout; mock it to + # report "stdin is ready" so tests proceed without real file descriptors. + monkeypatch.setattr( + _select_mod, "select", lambda rlist, wlist, xlist, timeout=None: (rlist, [], []) + ) + + _info = info or _make_stable_info() + monkeypatch.setattr("autoskillit.cli.update._update_checks.detect_install", lambda: _info) + + if state is not None: + (tmp_path / ".autoskillit").mkdir(parents=True, exist_ok=True) + (tmp_path / ".autoskillit" / "update_check.json").write_text( + json.dumps(state), encoding="utf-8" + ) + + import autoskillit as _pkg + + monkeypatch.setattr(_pkg, "__version__", current_version) + + monkeypatch.setattr( + "autoskillit.cli.update._update_checks._binary_signal", + lambda info, home, current: ( + Signal("binary", "New release: 0.9.0 (you have 0.7.77)") if binary_signal else None + ), + ) + monkeypatch.setattr( + "autoskillit.cli.update._update_checks._hooks_signal", + lambda settings_path: ( + Signal("hooks", "1 new/changed hook(s) detected") if hooks_signal else None + ), + ) + monkeypatch.setattr( + "autoskillit.cli.update._update_checks._source_drift_signal", + lambda info, home: ( + Signal("source_drift", "A newer version is available on the stable branch (aaa..bbb)") + if source_drift_signal + else None + ), + ) + monkeypatch.setattr( + "autoskillit.cli.update._update_checks._claude_settings_path", + lambda scope: tmp_path / "settings.json", + ) + + printed: list[str] = [] + monkeypatch.setattr( + "builtins.print", lambda *args, **kw: printed.append(" ".join(str(a) for a in args)) + ) + + input_calls: list[str] = [] + monkeypatch.setattr("builtins.input", lambda _="": input_calls.append("called") or answer) + + return printed, input_calls + + +def test_no_prompt_when_no_conditions_fire( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + printed, input_calls = _setup_run_checks(monkeypatch, tmp_path) + run_update_checks(home=tmp_path) + assert not input_calls + assert not printed, f"No output expected when zero signals fire; got: {printed!r}" + + +def test_single_prompt_when_only_binary_fires( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + printed, input_calls = _setup_run_checks(monkeypatch, tmp_path, binary_signal=True) + run_update_checks(home=tmp_path) + assert len(input_calls) == 1 + + +def test_single_prompt_when_only_hooks_fires( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + printed, input_calls = _setup_run_checks(monkeypatch, tmp_path, hooks_signal=True) + run_update_checks(home=tmp_path) + assert len(input_calls) == 1 + + +def test_single_prompt_when_only_source_drift_fires( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + printed, input_calls = _setup_run_checks(monkeypatch, tmp_path, source_drift_signal=True) + run_update_checks(home=tmp_path) + assert len(input_calls) == 1 + + +def test_consolidated_prompt_when_binary_plus_hooks_fire( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + printed, input_calls = _setup_run_checks( + monkeypatch, tmp_path, binary_signal=True, hooks_signal=True + ) + run_update_checks(home=tmp_path) + assert len(input_calls) == 1 + combined = " ".join(printed) + # Should contain 2 bullet lines + assert combined.count(" - ") == 2 + + +def test_consolidated_prompt_when_all_three_fire( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + printed, input_calls = _setup_run_checks( + monkeypatch, tmp_path, binary_signal=True, hooks_signal=True, source_drift_signal=True + ) + run_update_checks(home=tmp_path) + assert len(input_calls) == 1 + combined = " ".join(printed) + assert combined.count(" - ") == 3 + + +def test_prompt_never_contains_phrase_source_drift( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + printed, _ = _setup_run_checks(monkeypatch, tmp_path, source_drift_signal=True) + run_update_checks(home=tmp_path) + combined = " ".join(printed) + assert "source drift" not in combined.lower() + + +def test_prompt_uses_friendly_branch_language( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + printed, _ = _setup_run_checks(monkeypatch, tmp_path, source_drift_signal=True) + run_update_checks(home=tmp_path) + combined = " ".join(printed) + assert "newer version is available on the stable branch" in combined + + +# --------------------------------------------------------------------------- +# UC-4 Yes path +# --------------------------------------------------------------------------- + + +def test_yes_runs_upgrade_command_from_install_info_not_hardcoded( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + from autoskillit.cli._install_info import upgrade_command + + info = _make_stable_info() + printed, input_calls = _setup_run_checks( + monkeypatch, tmp_path, binary_signal=True, answer="y", info=info + ) + run_calls: list[list[str]] = [] + monkeypatch.setattr( + "autoskillit.cli.update._update_checks.subprocess.run", + lambda cmd, **kw: run_calls.append(list(cmd)) or subprocess.CompletedProcess(cmd, 0), + ) + monkeypatch.setattr("autoskillit.cli.update._update_checks.terminal_guard", MagicMock()) + monkeypatch.setattr( + "autoskillit.cli.update._update_checks._verify_update_result", lambda *a, **kw: True + ) + monkeypatch.setattr( + "autoskillit.cli.update._update_checks._fetch_latest_version", lambda *a, **kw: "0.9.0" + ) + monkeypatch.setattr("autoskillit.cli.update._update_checks.perform_restart", lambda: None) + expected_cmd = upgrade_command(info) + run_update_checks(home=tmp_path) + assert expected_cmd in run_calls, ( + f"Expected upgrade command {expected_cmd!r} from upgrade_command(info); got {run_calls!r}" + ) + + +def test_yes_runs_autoskillit_install_after_upgrade_command( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + printed, input_calls = _setup_run_checks(monkeypatch, tmp_path, binary_signal=True, answer="y") + run_calls: list[list[str]] = [] + + class FakeTG: + def __enter__(self): + return self + + def __exit__(self, *a): + return False + + monkeypatch.setattr("autoskillit.cli.update._update_checks.terminal_guard", FakeTG) + monkeypatch.setattr( + "autoskillit.cli.update._update_checks.subprocess.run", + lambda cmd, **kw: run_calls.append(list(cmd)) or subprocess.CompletedProcess(cmd, 0), + ) + monkeypatch.setattr( + "autoskillit.cli.update._update_checks._verify_update_result", lambda *a, **kw: True + ) + monkeypatch.setattr("autoskillit.cli.update._update_checks.perform_restart", lambda: None) + run_update_checks(home=tmp_path) + # ["autoskillit", "install"] must be among the calls + assert any(cmd[:2] == ["autoskillit", "install"] for cmd in run_calls) + + +def test_yes_passes_skip_env_to_subprocess( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + printed, input_calls = _setup_run_checks(monkeypatch, tmp_path, binary_signal=True, answer="y") + env_passed: list[dict] = [] + + class FakeTG: + def __enter__(self): + return self + + def __exit__(self, *a): + return False + + monkeypatch.setattr("autoskillit.cli.update._update_checks.terminal_guard", FakeTG) + monkeypatch.setattr( + "autoskillit.cli.update._update_checks.subprocess.run", + lambda cmd, **kw: ( + env_passed.append(kw.get("env", {})) or subprocess.CompletedProcess(cmd, 0) + ), + ) + monkeypatch.setattr( + "autoskillit.cli.update._update_checks._verify_update_result", lambda *a, **kw: True + ) + monkeypatch.setattr("autoskillit.cli.update._update_checks.perform_restart", lambda: None) + run_update_checks(home=tmp_path) + for env in env_passed: + assert env.get("AUTOSKILLIT_SKIP_STALE_CHECK") == "1" + assert env.get("AUTOSKILLIT_SKIP_UPDATE_CHECK") == "1" + assert env.get("AUTOSKILLIT_SKIP_SOURCE_DRIFT_CHECK") == "1" + + +def test_yes_single_invocation_exits_without_any_other_prompt( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + printed, input_calls = _setup_run_checks( + monkeypatch, tmp_path, binary_signal=True, hooks_signal=True, answer="y" + ) + + class FakeTG: + def __enter__(self): + return self + + def __exit__(self, *a): + return False + + monkeypatch.setattr("autoskillit.cli.update._update_checks.terminal_guard", FakeTG) + monkeypatch.setattr( + "autoskillit.cli.update._update_checks.subprocess.run", + lambda *a, **kw: subprocess.CompletedProcess([], 0), + ) + monkeypatch.setattr( + "autoskillit.cli.update._update_checks._verify_update_result", lambda *a, **kw: True + ) + monkeypatch.setattr("autoskillit.cli.update._update_checks.perform_restart", lambda: None) + run_update_checks(home=tmp_path) + assert len(input_calls) == 1 + + +# --------------------------------------------------------------------------- +# UC-5 No path +# --------------------------------------------------------------------------- + + +def test_no_writes_single_unified_dismissal_entry( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + printed, input_calls = _setup_run_checks(monkeypatch, tmp_path, binary_signal=True, answer="n") + run_update_checks(home=tmp_path) + state = _read_dismiss_state(tmp_path) + assert "update_prompt" in state + # No legacy sub-keys + assert "binary" not in state + assert "hooks" not in state + assert "source_drift" not in state + + +def test_no_records_conditions_list_in_dismissal_entry( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + printed, input_calls = _setup_run_checks( + monkeypatch, tmp_path, binary_signal=True, hooks_signal=True, answer="n" + ) + run_update_checks(home=tmp_path) + state = _read_dismiss_state(tmp_path) + entry = state["update_prompt"] + assert isinstance(entry, dict) + conditions = entry["conditions"] + assert "binary" in conditions + assert "hooks" in conditions + + +def test_no_prints_expiry_date_line_with_correct_date( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + printed, input_calls = _setup_run_checks(monkeypatch, tmp_path, binary_signal=True, answer="n") + run_update_checks(home=tmp_path) + combined = " ".join(printed) + # Should mention "Dismissed until" + assert "Dismissed until" in combined + # And an escape hatch hint + assert "autoskillit update" in combined or "AUTOSKILLIT_SKIP_STALE_CHECK" in combined + + +def test_no_prints_escape_hatch_hint(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: + printed, input_calls = _setup_run_checks(monkeypatch, tmp_path, binary_signal=True, answer="n") + run_update_checks(home=tmp_path) + combined = " ".join(printed) + assert "autoskillit update" in combined + assert "AUTOSKILLIT_SKIP_STALE_CHECK=1" in combined + + +# --------------------------------------------------------------------------- +# UC-6 Branch-aware dismissal windows +# --------------------------------------------------------------------------- + + +def _dismissed_state( + ago: timedelta, + version: str = "0.7.77", + conditions: list[str] | None = None, +) -> dict: + return { + "update_prompt": { + "dismissed_at": (datetime.now(UTC) - ago).isoformat(), + "dismissed_version": version, + "conditions": conditions or ["binary"], + } + } + + +def test_stable_install_dismissal_silent_within_six_days( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + state = _dismissed_state(ago=timedelta(days=6)) + printed, input_calls = _setup_run_checks( + monkeypatch, tmp_path, binary_signal=True, state=state + ) + run_update_checks(home=tmp_path) + assert not input_calls + + +def test_stable_install_dismissal_reprompts_after_eight_days( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + state = _dismissed_state(ago=timedelta(days=8)) + printed, input_calls = _setup_run_checks( + monkeypatch, tmp_path, binary_signal=True, state=state + ) + run_update_checks(home=tmp_path) + assert len(input_calls) == 1 + + +def test_develop_install_dismissal_silent_within_eleven_hours( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + state = _dismissed_state(ago=timedelta(hours=11)) + printed, input_calls = _setup_run_checks( + monkeypatch, + tmp_path, + binary_signal=True, + info=_make_develop_info(), + state=state, + ) + run_update_checks(home=tmp_path) + assert not input_calls + + +def test_develop_install_dismissal_reprompts_after_thirteen_hours( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + state = _dismissed_state(ago=timedelta(hours=13)) + printed, input_calls = _setup_run_checks( + monkeypatch, + tmp_path, + binary_signal=True, + info=_make_develop_info(), + state=state, + ) + run_update_checks(home=tmp_path) + assert len(input_calls) == 1 + + +def test_dismissal_window_chosen_from_current_install_not_stored( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + """User dismisses on stable (7d window), then migrates to integration (12h window). + 13 hours later the prompt should re-appear under the integration window.""" + state = _dismissed_state(ago=timedelta(hours=13)) + # Info is now integration — 12h window applies + printed, input_calls = _setup_run_checks( + monkeypatch, + tmp_path, + binary_signal=True, + info=_make_develop_info(), + state=state, + ) + run_update_checks(home=tmp_path) + assert len(input_calls) == 1 + + +# --------------------------------------------------------------------------- +# UC-7 Time-windowed source-drift dismissal (no SHA keying) +# --------------------------------------------------------------------------- + + +def test_source_drift_dismissal_survives_new_upstream_commit_within_window( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + """Dismissed with ref=B; new check sees ref=C within window → still silent.""" + state = _dismissed_state(ago=timedelta(hours=1), conditions=["source_drift"]) + printed, input_calls = _setup_run_checks( + monkeypatch, tmp_path, source_drift_signal=True, state=state + ) + run_update_checks(home=tmp_path) + assert not input_calls + + +def test_source_drift_dismissal_expires_on_window_boundary( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + state = _dismissed_state(ago=timedelta(days=8), conditions=["source_drift"]) + printed, input_calls = _setup_run_checks( + monkeypatch, tmp_path, source_drift_signal=True, state=state + ) + run_update_checks(home=tmp_path) + assert len(input_calls) == 1 + + +def test_source_drift_dismissal_expires_on_version_delta( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + """Version advanced past dismissed_version → re-prompts regardless of time.""" + state = _dismissed_state(ago=timedelta(hours=1), version="0.7.77", conditions=["source_drift"]) + printed, input_calls = _setup_run_checks( + monkeypatch, tmp_path, source_drift_signal=True, state=state, current_version="0.7.78" + ) + run_update_checks(home=tmp_path) + assert len(input_calls) == 1 + + +# --------------------------------------------------------------------------- +# UC-8 Hook dismissal with version-delta +# --------------------------------------------------------------------------- + + +def test_hook_dismissal_expires_when_version_advances( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + state = _dismissed_state(ago=timedelta(hours=1), version="0.7.77", conditions=["hooks"]) + printed, input_calls = _setup_run_checks( + monkeypatch, tmp_path, hooks_signal=True, state=state, current_version="0.7.78" + ) + run_update_checks(home=tmp_path) + assert len(input_calls) == 1 + + +def test_hook_dismissal_holds_within_window_at_same_version( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + state = _dismissed_state(ago=timedelta(hours=1), version="0.7.77", conditions=["hooks"]) + printed, input_calls = _setup_run_checks( + monkeypatch, tmp_path, hooks_signal=True, state=state, current_version="0.7.77" + ) + run_update_checks(home=tmp_path) + assert not input_calls + + +# --------------------------------------------------------------------------- +# Dismiss state I/O +# --------------------------------------------------------------------------- + + +def test_read_dismiss_state_empty(tmp_path: Path) -> None: + assert _read_dismiss_state(tmp_path) == {} + + +def test_read_dismiss_state_malformed(tmp_path: Path) -> None: + p = tmp_path / ".autoskillit" / "update_check.json" + p.parent.mkdir(parents=True) + p.write_text("not-json", encoding="utf-8") + assert _read_dismiss_state(tmp_path) == {} + + +def test_write_dismiss_state_roundtrip(tmp_path: Path) -> None: + state = {"update_prompt": {"dismissed_at": "2026-01-01T00:00:00+00:00"}} + _write_dismiss_state(tmp_path, state) + assert _read_dismiss_state(tmp_path) == state + + +def test_read_dismiss_state_non_dict_json_returns_empty(tmp_path: Path) -> None: + p = tmp_path / ".autoskillit" / "update_check.json" + p.parent.mkdir(parents=True) + p.write_text("[1, 2, 3]", encoding="utf-8") + assert _read_dismiss_state(tmp_path) == {} + + +# --------------------------------------------------------------------------- +# _is_dismissed +# --------------------------------------------------------------------------- + + +def test_is_dismissed_within_window() -> None: + state = { + "update_prompt": { + "dismissed_at": (datetime.now(UTC) - timedelta(hours=1)).isoformat(), + "dismissed_version": "0.9.0", + "conditions": ["binary"], + } + } + assert _is_dismissed( + state, window=timedelta(hours=12), current_version="0.7.77", condition="binary" + ) + + +def test_is_dismissed_expired() -> None: + state = { + "update_prompt": { + "dismissed_at": (datetime.now(UTC) - timedelta(days=8)).isoformat(), + "dismissed_version": "0.9.0", + "conditions": ["binary"], + } + } + assert not _is_dismissed( + state, window=timedelta(days=7), current_version="0.7.77", condition="binary" + ) + + +def test_is_dismissed_newer_version_resets() -> None: + state = { + "update_prompt": { + "dismissed_at": (datetime.now(UTC) - timedelta(hours=1)).isoformat(), + "dismissed_version": "0.7.77", + "conditions": ["binary"], + } + } + assert not _is_dismissed( + state, window=timedelta(days=7), current_version="0.7.78", condition="binary" + ) + + +def test_is_dismissed_condition_not_in_list() -> None: + state = { + "update_prompt": { + "dismissed_at": (datetime.now(UTC) - timedelta(hours=1)).isoformat(), + "dismissed_version": "0.9.0", + "conditions": ["binary"], + } + } + # hooks was NOT dismissed — should still fire + assert not _is_dismissed( + state, window=timedelta(days=7), current_version="0.7.77", condition="hooks" + ) + + +def test_is_dismissed_empty_state() -> None: + assert not _is_dismissed( + {}, window=timedelta(days=7), current_version="0.7.77", condition="binary" + ) + + +# --------------------------------------------------------------------------- +# timed_prompt primitive tests +# --------------------------------------------------------------------------- + + +def test_timed_prompt_returns_default_on_timeout(monkeypatch: pytest.MonkeyPatch) -> None: + """timed_prompt returns the default value when select.select times out.""" + import select as _select_mod + + from autoskillit.cli.ui._timed_input import timed_prompt + + monkeypatch.setattr("sys.stdin.isatty", lambda: True) + monkeypatch.setattr("sys.stdout.isatty", lambda: True) + monkeypatch.delenv("NO_COLOR", raising=False) + + # select.select returns empty list = timeout + monkeypatch.setattr( + _select_mod, "select", lambda rlist, wlist, xlist, timeout=None: ([], [], []) + ) + + printed: list[str] = [] + monkeypatch.setattr("builtins.print", lambda *args, **kw: printed.append(str(args))) + + result = timed_prompt("Test prompt?", default="n", timeout=30, label="test") + assert result == "n" + assert any("timed out" in p for p in printed) + + +def test_timed_prompt_applies_ansi_formatting(monkeypatch: pytest.MonkeyPatch) -> None: + """timed_prompt output includes ANSI escape sequences when color is supported.""" + import select as _select_mod + + from autoskillit.cli.ui._timed_input import timed_prompt + + monkeypatch.setattr("sys.stdin.isatty", lambda: True) + monkeypatch.setattr("sys.stdout.isatty", lambda: True) + monkeypatch.delenv("NO_COLOR", raising=False) + monkeypatch.delenv("TERM", raising=False) + + monkeypatch.setattr( + _select_mod, "select", lambda rlist, wlist, xlist, timeout=None: (rlist, [], []) + ) + monkeypatch.setattr("builtins.input", lambda _="": "y") + + raw_output: list[str] = [] + monkeypatch.setattr( + "builtins.print", + lambda *args, **kw: raw_output.append(" ".join(str(a) for a in args)), + ) + + timed_prompt("Update now? [Y/n]", default="n", timeout=30, label="test") + combined = " ".join(raw_output) + assert "\x1b[" in combined # ANSI escape present + + +def test_timed_prompt_respects_no_color(monkeypatch: pytest.MonkeyPatch) -> None: + """timed_prompt output has no ANSI sequences when NO_COLOR is set.""" + import select as _select_mod + + from autoskillit.cli.ui._timed_input import timed_prompt + + monkeypatch.setattr("sys.stdin.isatty", lambda: True) + monkeypatch.setattr("sys.stdout.isatty", lambda: True) + monkeypatch.setenv("NO_COLOR", "1") + + monkeypatch.setattr( + _select_mod, "select", lambda rlist, wlist, xlist, timeout=None: (rlist, [], []) + ) + monkeypatch.setattr("builtins.input", lambda _="": "y") + + raw_output: list[str] = [] + monkeypatch.setattr( + "builtins.print", + lambda *args, **kw: raw_output.append(" ".join(str(a) for a in args)), + ) + + timed_prompt("Update now? [Y/n]", default="n", timeout=30, label="test") + combined = " ".join(raw_output) + assert "\x1b[" not in combined # No ANSI escapes + + +# --------------------------------------------------------------------------- +# AUTOSKILLIT_FORCE_UPDATE_CHECK override +# --------------------------------------------------------------------------- + + +def test_force_update_check_env_overrides_local_editable( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + """AUTOSKILLIT_FORCE_UPDATE_CHECK=1 bypasses the LOCAL_EDITABLE early return.""" + info = InstallInfo( + install_type=InstallType.LOCAL_EDITABLE, + commit_id=None, + requested_revision=None, + url=None, + editable_source=Path(tmp_path), + ) + printed, input_calls = _setup_run_checks( + monkeypatch, tmp_path, info=info, binary_signal=True, answer="n" + ) + monkeypatch.setenv("AUTOSKILLIT_FORCE_UPDATE_CHECK", "1") + run_update_checks(home=tmp_path) + # The prompt should have been reached (not early-returned) + assert len(input_calls) == 1 + + +# --------------------------------------------------------------------------- +# UC-10 Passive notification for dismissed signals (REQ-UX-002–006, REQ-FLOW-001–004) +# --------------------------------------------------------------------------- + + +def test_dismissed_signal_prints_passive_notification( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + state = _dismissed_state(ago=timedelta(hours=1), conditions=["binary"]) + printed, input_calls = _setup_run_checks( + monkeypatch, tmp_path, binary_signal=True, state=state + ) + run_update_checks(home=tmp_path) + assert not input_calls, "Dismissed signal must not trigger interactive prompt" + combined = " ".join(printed) + assert "autoskillit update" in combined, ( + "Dismissed signal must produce passive notification containing 'autoskillit update'" + ) + + +def test_passive_notification_contains_version_info( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + state = _dismissed_state(ago=timedelta(hours=1), conditions=["binary"]) + printed, _ = _setup_run_checks(monkeypatch, tmp_path, binary_signal=True, state=state) + run_update_checks(home=tmp_path) + combined = " ".join(printed) + # The binary signal message is "New release: 0.9.0 (you have 0.7.77)" + assert "0.9.0" in combined or "0.7.77" in combined, ( + "Passive notification must include version info" + ) + + +def test_passive_notification_contains_expiry_date( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + dismissed_ago = timedelta(hours=1) + state = _dismissed_state(ago=dismissed_ago, conditions=["binary"]) + # Derive expiry from state to avoid date-boundary races between setup and assertion + dismissed_at = datetime.fromisoformat(state["update_prompt"]["dismissed_at"]) + printed, _ = _setup_run_checks(monkeypatch, tmp_path, binary_signal=True, state=state) + run_update_checks(home=tmp_path) + combined = " ".join(printed) + # Stable install: 7-day window. + expected_expiry = (dismissed_at + timedelta(days=7)).strftime("%Y-%m-%d") + assert expected_expiry in combined, ( + f"Passive notification must include expiry date {expected_expiry!r}; got: {combined!r}" + ) + + +def test_passive_notification_contains_update_command( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + state = _dismissed_state(ago=timedelta(hours=1), conditions=["binary"]) + printed, _ = _setup_run_checks(monkeypatch, tmp_path, binary_signal=True, state=state) + run_update_checks(home=tmp_path) + combined = " ".join(printed) + assert "autoskillit update" in combined, ( + "Passive notification must contain 'autoskillit update'" + ) + + +def test_undismissed_signal_still_gets_interactive_prompt_when_dismissed_also_present( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + # binary is dismissed; hooks is NOT dismissed (not in conditions list) + state = _dismissed_state(ago=timedelta(hours=1), conditions=["binary"]) + printed, input_calls = _setup_run_checks( + monkeypatch, + tmp_path, + binary_signal=True, + hooks_signal=True, + state=state, + ) + run_update_checks(home=tmp_path) + assert len(input_calls) == 1, "Undismissed hooks signal must trigger interactive prompt" + + +def test_all_dismissed_signals_produce_no_interactive_prompt( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + state = _dismissed_state(ago=timedelta(hours=1), conditions=["binary", "hooks"]) + printed, input_calls = _setup_run_checks( + monkeypatch, + tmp_path, + binary_signal=True, + hooks_signal=True, + state=state, + ) + run_update_checks(home=tmp_path) + assert not input_calls, "All-dismissed signals must not trigger interactive prompt" + combined = " ".join(printed) + assert "autoskillit update" in combined, ( + "All-dismissed signals must produce passive notification containing 'autoskillit update'" + ) diff --git a/tests/cli/test_update_checks_split.py b/tests/cli/test_update_checks_split.py new file mode 100644 index 0000000000..f8e0434996 --- /dev/null +++ b/tests/cli/test_update_checks_split.py @@ -0,0 +1,36 @@ +from __future__ import annotations + +from pathlib import Path + +import pytest + +pytestmark = [pytest.mark.layer("cli"), pytest.mark.small] + +_CLI_SRC = Path(__file__).parent.parent.parent / "src" / "autoskillit" / "cli" + + +def test_update_checks_fetch_importable(): + from autoskillit.cli.update._update_checks_fetch import ( # noqa: F401 + _fetch_latest_version, + _fetch_with_cache, + invalidate_fetch_cache, + ) + + +def test_update_checks_source_importable(): + from autoskillit.cli.update._update_checks_source import ( # noqa: F401 + find_source_repo, + resolve_reference_sha, + ) + + +def test_update_checks_facade_public_api(): + from autoskillit.cli.update._update_checks import Signal, run_update_checks # noqa: F401 + + +def test_order_module_file_exists(): + assert (_CLI_SRC / "session" / "_order.py").exists() + + +def test_order_importable_from_submodule(): + from autoskillit.cli.session._order import _get_subsets_needed, order # noqa: F401 diff --git a/tests/cli/test_update_command.py b/tests/cli/test_update_command.py index 8922295608..7386ebe4e7 100644 --- a/tests/cli/test_update_command.py +++ b/tests/cli/test_update_command.py @@ -2,12 +2,16 @@ from __future__ import annotations +import subprocess from pathlib import Path +from unittest.mock import MagicMock import pytest from autoskillit.cli._install_info import InstallInfo, InstallType +pytestmark = [pytest.mark.layer("cli"), pytest.mark.small] + def _make_info( install_type: InstallType, @@ -39,13 +43,13 @@ def _setup_run_update( new_version: str = "0.9.0", ) -> list[list[str]]: """Patch run_update_command dependencies, return captured subprocess.run call args.""" - monkeypatch.setattr("autoskillit.cli._update.detect_install", lambda: info) - monkeypatch.setattr("autoskillit.cli._update.terminal_guard", FakeTG) + monkeypatch.setattr("autoskillit.cli.update._update.detect_install", lambda: info) + monkeypatch.setattr("autoskillit.cli.update._update.terminal_guard", FakeTG) run_calls: list[list[str]] = [] monkeypatch.setattr( - "autoskillit.cli._update.subprocess.run", - lambda cmd, **kw: run_calls.append(list(cmd)), + "autoskillit.cli.update._update.subprocess.run", + lambda cmd, **kw: run_calls.append(list(cmd)) or subprocess.CompletedProcess(cmd, 0), ) import autoskillit as _pkg @@ -55,6 +59,7 @@ def _setup_run_update( import importlib.metadata monkeypatch.setattr(importlib.metadata, "version", lambda _name: new_version) + monkeypatch.setattr("autoskillit.cli.update._update.perform_restart", lambda: None) return run_calls @@ -77,7 +82,7 @@ def test_update_subcommand_registered_in_help() -> None: ("stable", ["uv", "tool", "upgrade", "autoskillit"]), ("main", ["uv", "tool", "upgrade", "autoskillit"]), ("v0.7.75", ["uv", "tool", "upgrade", "autoskillit"]), - ("integration", ["uv", "tool", "install", "--force"]), + ("develop", ["uv", "tool", "install", "--force"]), ], ) def test_update_runs_upgrade_command_for_git_vcs_install( @@ -86,7 +91,7 @@ def test_update_runs_upgrade_command_for_git_vcs_install( revision: str, expected_cmd_prefix: list[str], ) -> None: - from autoskillit.cli._update import run_update_command + from autoskillit.cli.update._update import run_update_command info = _make_info(InstallType.GIT_VCS, revision=revision) run_calls = _setup_run_update(monkeypatch, info, tmp_path) @@ -99,7 +104,7 @@ def test_update_runs_upgrade_command_for_git_vcs_install( def test_update_runs_upgrade_command_for_local_editable( monkeypatch: pytest.MonkeyPatch, tmp_path: Path ) -> None: - from autoskillit.cli._update import run_update_command + from autoskillit.cli.update._update import run_update_command editable_source = tmp_path / "repo" editable_source.mkdir() @@ -114,7 +119,7 @@ def test_update_runs_upgrade_command_for_local_editable( def test_update_runs_autoskillit_install_after_upgrade_command( monkeypatch: pytest.MonkeyPatch, tmp_path: Path ) -> None: - from autoskillit.cli._update import run_update_command + from autoskillit.cli.update._update import run_update_command info = _make_info(InstallType.GIT_VCS, revision="stable") run_calls = _setup_run_update(monkeypatch, info, tmp_path) @@ -125,16 +130,18 @@ def test_update_runs_autoskillit_install_after_upgrade_command( def test_update_passes_skip_env_to_subprocess( monkeypatch: pytest.MonkeyPatch, tmp_path: Path ) -> None: - from autoskillit.cli._update import run_update_command + from autoskillit.cli.update._update import run_update_command info = _make_info(InstallType.GIT_VCS, revision="stable") - monkeypatch.setattr("autoskillit.cli._update.detect_install", lambda: info) - monkeypatch.setattr("autoskillit.cli._update.terminal_guard", FakeTG) + monkeypatch.setattr("autoskillit.cli.update._update.detect_install", lambda: info) + monkeypatch.setattr("autoskillit.cli.update._update.terminal_guard", FakeTG) env_passed: list[dict] = [] monkeypatch.setattr( - "autoskillit.cli._update.subprocess.run", - lambda cmd, **kw: env_passed.append(kw.get("env", {})), + "autoskillit.cli.update._update.subprocess.run", + lambda cmd, **kw: ( + env_passed.append(kw.get("env", {})) or subprocess.CompletedProcess(cmd, 0) + ), ) import autoskillit as _pkg @@ -144,25 +151,29 @@ def test_update_passes_skip_env_to_subprocess( import importlib.metadata monkeypatch.setattr(importlib.metadata, "version", lambda _name: "0.9.0") + monkeypatch.setattr("autoskillit.cli.update._update.perform_restart", lambda: None) run_update_command(home=tmp_path) for env in env_passed: assert env.get("AUTOSKILLIT_SKIP_STALE_CHECK") == "1" + assert env.get("AUTOSKILLIT_SKIP_UPDATE_CHECK") == "1" assert env.get("AUTOSKILLIT_SKIP_SOURCE_DRIFT_CHECK") == "1" def test_update_verifies_version_advance_and_warns_on_failure( monkeypatch: pytest.MonkeyPatch, tmp_path: Path ) -> None: - from autoskillit.cli._update import run_update_command - from autoskillit.cli._update_checks import _read_dismiss_state + from autoskillit.cli.update._update import run_update_command info = _make_info(InstallType.GIT_VCS, revision="stable") - monkeypatch.setattr("autoskillit.cli._update.detect_install", lambda: info) - monkeypatch.setattr("autoskillit.cli._update.terminal_guard", FakeTG) - monkeypatch.setattr("autoskillit.cli._update.subprocess.run", lambda *a, **kw: None) + monkeypatch.setattr("autoskillit.cli.update._update.detect_install", lambda: info) + monkeypatch.setattr("autoskillit.cli.update._update.terminal_guard", FakeTG) + monkeypatch.setattr( + "autoskillit.cli.update._update.subprocess.run", + lambda *a, **kw: subprocess.CompletedProcess([], 0), + ) monkeypatch.setattr( - "autoskillit.cli._update_checks._fetch_latest_version", lambda *a, **kw: "0.9.0" + "autoskillit.cli.update._update_checks._fetch_latest_version", lambda *a, **kw: "0.9.0" ) import autoskillit as _pkg @@ -180,26 +191,14 @@ def test_update_verifies_version_advance_and_warns_on_failure( ) run_update_command(home=tmp_path) combined = " ".join(printed) - assert "still" in combined or "unchanged" in combined or "still 0.7.77" in combined - - # Verify binary_snoozed disk state records the correct attempted_version - state = _read_dismiss_state(tmp_path) - assert "binary_snoozed" in state, ( - f"Expected 'binary_snoozed' key in dismiss state after failed update; got {list(state)}" - ) - snooze = state["binary_snoozed"] - assert isinstance(snooze, dict), f"Expected dict for binary_snoozed; got {type(snooze)}" - assert snooze.get("attempted_version") == "0.9.0", ( - f"Expected attempted_version='0.9.0' (from _fetch_latest_version); " - f"got {snooze.get('attempted_version')!r}" - ) + assert "still 0.7.77" in combined def test_update_clears_dismissal_state_on_success( monkeypatch: pytest.MonkeyPatch, tmp_path: Path ) -> None: - from autoskillit.cli._update import run_update_command - from autoskillit.cli._update_checks import _read_dismiss_state, _write_dismiss_state + from autoskillit.cli.update._update import run_update_command + from autoskillit.cli.update._update_checks import _read_dismiss_state, _write_dismiss_state # Seed dismissal state (tmp_path / ".autoskillit").mkdir(parents=True, exist_ok=True) @@ -215,9 +214,12 @@ def test_update_clears_dismissal_state_on_success( ) info = _make_info(InstallType.GIT_VCS, revision="stable") - monkeypatch.setattr("autoskillit.cli._update.detect_install", lambda: info) - monkeypatch.setattr("autoskillit.cli._update.terminal_guard", FakeTG) - monkeypatch.setattr("autoskillit.cli._update.subprocess.run", lambda *a, **kw: None) + monkeypatch.setattr("autoskillit.cli.update._update.detect_install", lambda: info) + monkeypatch.setattr("autoskillit.cli.update._update.terminal_guard", FakeTG) + monkeypatch.setattr( + "autoskillit.cli.update._update.subprocess.run", + lambda *a, **kw: subprocess.CompletedProcess([], 0), + ) import autoskillit as _pkg @@ -228,6 +230,7 @@ def test_update_clears_dismissal_state_on_success( monkeypatch.setattr(importlib.metadata, "version", lambda _name: "0.9.0") monkeypatch.setattr("builtins.print", lambda *a, **kw: None) + monkeypatch.setattr("autoskillit.cli.update._update.perform_restart", lambda: None) run_update_command(home=tmp_path) state = _read_dismiss_state(tmp_path) assert "update_prompt" not in state @@ -236,10 +239,10 @@ def test_update_clears_dismissal_state_on_success( def test_update_reports_actionable_error_on_unknown_install_type( monkeypatch: pytest.MonkeyPatch, tmp_path: Path ) -> None: - from autoskillit.cli._update import run_update_command + from autoskillit.cli.update._update import run_update_command info = _make_info(InstallType.UNKNOWN) - monkeypatch.setattr("autoskillit.cli._update.detect_install", lambda: info) + monkeypatch.setattr("autoskillit.cli.update._update.detect_install", lambda: info) printed: list[str] = [] monkeypatch.setattr( "builtins.print", lambda *args, **kw: printed.append(" ".join(str(a) for a in args)) @@ -249,3 +252,71 @@ def test_update_reports_actionable_error_on_unknown_install_type( assert exc_info.value.code == 2 combined = " ".join(printed) assert "Unknown install type" in combined or "install.sh" in combined + + +def test_run_update_command_warns_on_install_failure( + monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture, tmp_path: Path +) -> None: + """run_update_command warns user when autoskillit install step exits non-zero.""" + from autoskillit.cli.update._update import run_update_command + + info = _make_info(InstallType.GIT_VCS, revision="stable") + monkeypatch.setattr("autoskillit.cli.update._update.detect_install", lambda: info) + monkeypatch.setattr("autoskillit.cli.update._update.terminal_guard", FakeTG) + + upgrade_ok = subprocess.CompletedProcess([], returncode=0) + install_fail = subprocess.CompletedProcess([], returncode=1) + mock_run = MagicMock(side_effect=[upgrade_ok, install_fail]) + monkeypatch.setattr("autoskillit.cli.update._update.subprocess.run", mock_run) + monkeypatch.setattr( + "autoskillit.cli.update._update_checks._fetch_latest_version", lambda *a, **kw: "0.9.1" + ) + import autoskillit as _pkg + + # Simulate pre-update state: process-cached __version__ is stale (0.9.0), + # but post-install metadata already reflects the new version (0.9.1). + monkeypatch.setattr(_pkg, "__version__", "0.9.0") + import importlib.metadata + + monkeypatch.setattr(importlib.metadata, "version", lambda _: "0.9.1") + monkeypatch.setattr("autoskillit.cli.update._update.perform_restart", lambda: None) + run_update_command(home=tmp_path) + out = capsys.readouterr().out + assert "autoskillit install" in out + + +def test_run_update_command_restarts_on_success( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + """After a successful upgrade, run_update_command must call perform_restart.""" + from autoskillit.cli.update._update import run_update_command + + info = _make_info(InstallType.GIT_VCS, revision="stable") + monkeypatch.setattr("autoskillit.cli.update._update.detect_install", lambda: info) + monkeypatch.setattr("autoskillit.cli.update._update.terminal_guard", FakeTG) + monkeypatch.setattr( + "autoskillit.cli.update._update.subprocess.run", + lambda *a, **kw: subprocess.CompletedProcess([], 0), + ) + monkeypatch.setattr( + "autoskillit.cli.update._update_checks._fetch_latest_version", lambda *a, **kw: "0.9.1" + ) + + import autoskillit as _pkg + + monkeypatch.setattr(_pkg, "__version__", "0.9.0") + + import importlib.metadata + + monkeypatch.setattr(importlib.metadata, "version", lambda _name: "0.9.1") + monkeypatch.setattr("builtins.print", lambda *a, **kw: None) + + restart_called: list[bool] = [] + monkeypatch.setattr( + "autoskillit.cli.update._update.perform_restart", lambda: restart_called.append(True) + ) + + run_update_command(home=tmp_path) + assert restart_called, ( + "run_update_command must call perform_restart() after successful upgrade" + ) diff --git a/tests/cli/test_validate_registries.py b/tests/cli/test_validate_registries.py new file mode 100644 index 0000000000..230ba5a5b6 --- /dev/null +++ b/tests/cli/test_validate_registries.py @@ -0,0 +1,462 @@ +"""Tests for the validate registries CLI subcommand.""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from autoskillit.cli._validate import validate_registries +from autoskillit.recipe import ExperimentTypeSpec + +pytestmark = [pytest.mark.layer("cli"), pytest.mark.medium] + + +def _write_yaml(tmp_path: Path, registry_type: str, filename: str, content: str) -> Path: + """Write a YAML file to the appropriate registry directory.""" + registry_dir = tmp_path / ".autoskillit" / registry_type + registry_dir.mkdir(parents=True, exist_ok=True) + file_path = registry_dir / filename + file_path.write_text(content) + return file_path + + +VALID_EXPERIMENT_TYPE = """\ +name: my-test-type +schema_version: "1.0" +priority: 1 +classification_triggers: + - trigger_alpha + - trigger_beta +applicable_lenses: + primary: exp-lens-estimand-clarity + secondary: null +dimension_weights: + clarity: high + methodological_rigor: medium + external_validity: low + practical_significance: medium + inferential_validity: high + ethical_compliance: medium + reporting_completeness: high + transparency: high +red_team_focus: + priority_area: test_area +l1_severity: + severity_rating: medium +""" + +VALID_METHODOLOGY_TRADITION = """\ +name: my-test-tradition +display_name: My Test Tradition +schema_version: "1.0" +priority: 1 +canonical_guideline: + title: Test Guideline + url: https://example.com +fields_spanned: + - field1 +detection_keywords: + - keyword1 + - keyword2 +mandatory_figures: + - figure_type: table +strongly_expected_figures: + - figure_type: chart +anti_patterns: + - pattern_name: bad_pattern +""" + + +class TestValidateRegistries: + """Tests for validate_registries command.""" + + @pytest.fixture(autouse=True) + def _mock_lens_slugs(self, monkeypatch: pytest.MonkeyPatch) -> None: + """Patch lens slug resolution to return a known set.""" + monkeypatch.setattr( + "autoskillit.cli._validate._get_valid_lens_slugs", + lambda: { + "exp-lens-estimand-clarity", + "exp-lens-error-budget", + "exp-lens-validity-threats", + }, + ) + + @pytest.fixture(autouse=True) + def _mock_bundled_types(self, monkeypatch: pytest.MonkeyPatch) -> None: + """Patch bundled types loading to return empty dict.""" + monkeypatch.setattr( + "autoskillit.cli._validate.load_types_from_dir", + lambda _: {}, + ) + + # T_VAL_1: Valid experiment-type YAML produces ✓ output and exit 0 + def test_valid_experiment_type_valid( + self, + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, + capsys: pytest.CaptureFixture, + ) -> None: + """Valid experiment-type YAML produces ✓ output and exits 0.""" + monkeypatch.chdir(tmp_path) + _write_yaml(tmp_path, "experiment-types", "my_type.yaml", VALID_EXPERIMENT_TYPE) + + validate_registries() + out = capsys.readouterr().out + assert "✓" in out + assert "my_type.yaml" in out + assert not (tmp_path / ".autoskillit" / "validation-errors").exists() + + # T_VAL_2: Schema-version mismatch produces ⚠ warning and exit 0 + def test_schema_version_mismatch_warning( + self, + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, + capsys: pytest.CaptureFixture, + ) -> None: + """Schema-version mismatch produces ⚠ warning and exits 0.""" + monkeypatch.chdir(tmp_path) + yaml_content = VALID_EXPERIMENT_TYPE.replace( + 'schema_version: "1.0"', 'schema_version: "0.9"' + ) + _write_yaml(tmp_path, "experiment-types", "warn_type.yaml", yaml_content) + + validate_registries() + out = capsys.readouterr().out + assert "⚠" in out + assert "schema_version" in out + assert "expected '1.0'" in out + + # T_VAL_3: Missing 'name' field produces ✗ error and exit 1 + def test_missing_name_error( + self, + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, + capsys: pytest.CaptureFixture, + ) -> None: + """Missing 'name' field produces ✗ error and exit 1.""" + monkeypatch.chdir(tmp_path) + yaml_content = VALID_EXPERIMENT_TYPE.replace("name: my-test-type\n", "") + _write_yaml(tmp_path, "experiment-types", "no_name.yaml", yaml_content) + + with pytest.raises(SystemExit) as exc_info: + validate_registries() + assert exc_info.value.code == 1 + out = capsys.readouterr().out + assert "✗" in out + assert "missing" in out + assert (tmp_path / ".autoskillit" / "validation-errors" / "no_name.yaml.error.md").exists() + + # T_VAL_4: Invalid lens reference produces ✗ error and exit 1 + def test_invalid_lens_reference_error( + self, + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, + capsys: pytest.CaptureFixture, + ) -> None: + """Invalid lens reference produces ✗ error and exit 1.""" + monkeypatch.chdir(tmp_path) + yaml_content = VALID_EXPERIMENT_TYPE.replace( + "exp-lens-estimand-clarity", "exp-lens-nonexistent" + ) + _write_yaml(tmp_path, "experiment-types", "bad_lens.yaml", yaml_content) + + with pytest.raises(SystemExit) as exc_info: + validate_registries() + assert exc_info.value.code == 1 + out = capsys.readouterr().out + assert "✗" in out + assert "lens" in out + + # T_VAL_5: Empty classification_triggers produces ✗ error and exit 1 + def test_empty_classification_triggers_error( + self, + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, + capsys: pytest.CaptureFixture, + ) -> None: + """Empty classification_triggers produces ✗ error and exit 1.""" + monkeypatch.chdir(tmp_path) + yaml_content = VALID_EXPERIMENT_TYPE.replace( + "classification_triggers:\n - trigger_alpha\n - trigger_beta", + "classification_triggers: []", + ) + _write_yaml(tmp_path, "experiment-types", "empty_triggers.yaml", yaml_content) + + with pytest.raises(SystemExit) as exc_info: + validate_registries() + assert exc_info.value.code == 1 + out = capsys.readouterr().out + assert "✗" in out + assert "classification_triggers is empty" in out + + # T_VAL_6: Non-positive priority produces ✗ error and exit 1 + def test_non_positive_priority_error( + self, + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, + capsys: pytest.CaptureFixture, + ) -> None: + """Non-positive priority produces ✗ error and exit 1.""" + monkeypatch.chdir(tmp_path) + yaml_content = VALID_EXPERIMENT_TYPE.replace("priority: 1", "priority: 0") + _write_yaml(tmp_path, "experiment-types", "bad_priority.yaml", yaml_content) + + with pytest.raises(SystemExit) as exc_info: + validate_registries() + assert exc_info.value.code == 1 + out = capsys.readouterr().out + assert "✗" in out + assert "priority" in out + + # T_VAL_7: Multiple is_fallback entries produces ✗ error and exit 1 + def test_multiple_fallback_error( + self, + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, + capsys: pytest.CaptureFixture, + ) -> None: + """Multiple is_fallback entries produces ✗ error and exit 1.""" + monkeypatch.chdir(tmp_path) + yaml1 = VALID_EXPERIMENT_TYPE + "\nis_fallback: true\n" + yaml2 = ( + VALID_EXPERIMENT_TYPE.replace("name: my-test-type", "name: my-other-type") + + "\nis_fallback: true\n" + ) + _write_yaml(tmp_path, "experiment-types", "fallback1.yaml", yaml1) + _write_yaml(tmp_path, "experiment-types", "fallback2.yaml", yaml2) + + with pytest.raises(SystemExit) as exc_info: + validate_registries() + assert exc_info.value.code == 1 + out = capsys.readouterr().out + assert "✗" in out + assert "is_fallback" in out + + # T_VAL_8: No user directories — prints info message and exit 0 + def test_no_user_directories( + self, + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, + capsys: pytest.CaptureFixture, + ) -> None: + """No user directories prints info message and exits 0.""" + monkeypatch.chdir(tmp_path) + + validate_registries() + out = capsys.readouterr().out + assert "No user registry directories found" in out + + # T_VAL_9: Valid methodology-tradition YAML produces ✓ output and exit 0 + def test_valid_methodology_tradition_valid( + self, + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, + capsys: pytest.CaptureFixture, + ) -> None: + """Valid methodology-tradition YAML produces ✓ output and exits 0.""" + monkeypatch.chdir(tmp_path) + _write_yaml( + tmp_path, "methodology-traditions", "my_tradition.yaml", VALID_METHODOLOGY_TRADITION + ) + + validate_registries() + out = capsys.readouterr().out + assert "✓" in out + assert "my_tradition.yaml" in out + + # T_VAL_10: Empty detection_keywords in methodology tradition produces ✗ error + def test_empty_detection_keywords_error( + self, + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, + capsys: pytest.CaptureFixture, + ) -> None: + """Empty detection_keywords produces ✗ error and exit 1.""" + monkeypatch.chdir(tmp_path) + yaml_content = VALID_METHODOLOGY_TRADITION.replace( + "detection_keywords:\n - keyword1\n - keyword2", + "detection_keywords: []", + ) + _write_yaml(tmp_path, "methodology-traditions", "empty_keywords.yaml", yaml_content) + + with pytest.raises(SystemExit) as exc_info: + validate_registries() + assert exc_info.value.code == 1 + out = capsys.readouterr().out + assert "✗" in out + assert "detection_keywords is empty" in out + + # T_VAL_11: Error report markdown file has expected structure + def test_error_report_structure( + self, + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, + ) -> None: + """Error report markdown file has expected structure.""" + monkeypatch.chdir(tmp_path) + yaml_content = VALID_EXPERIMENT_TYPE.replace("name: my-test-type\n", "") + _write_yaml(tmp_path, "experiment-types", "parse_error.yaml", yaml_content) + + with pytest.raises(SystemExit) as exc_info: + validate_registries() + assert exc_info.value.code == 1 + + error_file = tmp_path / ".autoskillit" / "validation-errors" / "parse_error.yaml.error.md" + assert error_file.exists() + content = error_file.read_text() + assert "# Validation error:" in content + assert "**File:**" in content + assert "**Validated at:**" in content + assert "## Error" in content + assert "## Expected schema" in content + assert "## Your file" in content + assert "## How to fix" in content + + # T_VAL_12: Exit code 0 when only warnings (no errors) + def test_exit_0_with_only_warnings( + self, + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, + capsys: pytest.CaptureFixture, + ) -> None: + """Exit code 0 when only warnings (no errors).""" + monkeypatch.chdir(tmp_path) + yaml_content = VALID_EXPERIMENT_TYPE.replace( + 'schema_version: "1.0"', 'schema_version: "0.9"' + ) + _write_yaml(tmp_path, "experiment-types", "warn_only.yaml", yaml_content) + + validate_registries() + out = capsys.readouterr().out + assert "⚠" in out + assert "0 error" in out + + # T_VAL_13: Summary line includes correct counts + def test_summary_counts( + self, + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, + capsys: pytest.CaptureFixture, + ) -> None: + """Summary line includes correct counts.""" + monkeypatch.chdir(tmp_path) + _write_yaml(tmp_path, "experiment-types", "valid_type.yaml", VALID_EXPERIMENT_TYPE) + yaml_warn = VALID_EXPERIMENT_TYPE.replace('schema_version: "1.0"', 'schema_version: "0.9"') + _write_yaml(tmp_path, "experiment-types", "warn_type.yaml", yaml_warn) + yaml_err = VALID_EXPERIMENT_TYPE.replace("name: my-test-type\n", "") + _write_yaml(tmp_path, "experiment-types", "error_type.yaml", yaml_err) + + with pytest.raises(SystemExit) as exc_info: + validate_registries() + assert exc_info.value.code == 1 + out = capsys.readouterr().out + assert "1 valid" in out + assert "1 warning" in out + assert "1 error" in out + + # T_VAL_14: is_fallback check allows single fallback entry + def test_single_fallback_allowed( + self, + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, + capsys: pytest.CaptureFixture, + ) -> None: + """Single fallback entry is allowed.""" + monkeypatch.chdir(tmp_path) + yaml_content = VALID_EXPERIMENT_TYPE + "\nis_fallback: true\n" + _write_yaml(tmp_path, "experiment-types", "single_fallback.yaml", yaml_content) + + validate_registries() + out = capsys.readouterr().out + assert "✓" in out + assert "✗" not in out + + # T_VAL_15: Malformed YAML in experiment-type produces YAML parse error and exit 1 + def test_malformed_yaml_experiment_type( + self, + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, + capsys: pytest.CaptureFixture, + ) -> None: + monkeypatch.chdir(tmp_path) + _write_yaml(tmp_path, "experiment-types", "bad_yaml.yaml", "name: [unclosed\n") + + with pytest.raises(SystemExit) as exc_info: + validate_registries() + assert exc_info.value.code == 1 + out = capsys.readouterr().out + assert "✗" in out + assert "YAML parse error" in out + + # T_VAL_16: Malformed YAML in methodology-tradition produces YAML parse error and exit 1 + def test_malformed_yaml_methodology_tradition( + self, + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, + capsys: pytest.CaptureFixture, + ) -> None: + monkeypatch.chdir(tmp_path) + _write_yaml(tmp_path, "methodology-traditions", "bad_yaml.yaml", "name: [unclosed\n") + + with pytest.raises(SystemExit) as exc_info: + validate_registries() + assert exc_info.value.code == 1 + out = capsys.readouterr().out + assert "✗" in out + assert "YAML parse error" in out + + # T_VAL_17: Invalid secondary applicable_lenses produces ✗ error and exit 1 + def test_invalid_secondary_lens_error( + self, + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, + capsys: pytest.CaptureFixture, + ) -> None: + monkeypatch.chdir(tmp_path) + yaml_content = VALID_EXPERIMENT_TYPE.replace( + " secondary: null", " secondary: exp-lens-nonexistent" + ) + _write_yaml(tmp_path, "experiment-types", "bad_secondary.yaml", yaml_content) + + with pytest.raises(SystemExit) as exc_info: + validate_registries() + assert exc_info.value.code == 1 + out = capsys.readouterr().out + assert "✗" in out + assert "secondary" in out + + # T_VAL_18: Duplicate is_fallback between user and bundled type is flagged + def test_duplicate_fallback_with_bundled_type( + self, + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, + capsys: pytest.CaptureFixture, + ) -> None: + """User type with is_fallback=True conflicts with a bundled fallback type.""" + monkeypatch.chdir(tmp_path) + + bundled_fallback = ExperimentTypeSpec( + name="bundled-fallback-type", + classification_triggers=[], + dimension_weights={}, + applicable_lenses={}, + red_team_focus={}, + l1_severity={}, + is_fallback=True, + ) + monkeypatch.setattr( + "autoskillit.cli._validate.load_types_from_dir", + lambda _: {"bundled-fallback-type": bundled_fallback}, + ) + + yaml_content = VALID_EXPERIMENT_TYPE + "\nis_fallback: true\n" + _write_yaml(tmp_path, "experiment-types", "user_fallback.yaml", yaml_content) + + with pytest.raises(SystemExit) as exc_info: + validate_registries() + assert exc_info.value.code == 1 + out = capsys.readouterr().out + assert "✗" in out + assert "is_fallback" in out diff --git a/tests/cli/test_workspace.py b/tests/cli/test_workspace.py index 178668bafc..4382e4b40b 100644 --- a/tests/cli/test_workspace.py +++ b/tests/cli/test_workspace.py @@ -11,6 +11,8 @@ from autoskillit.cli._workspace import _format_age, run_workspace_clean from autoskillit.workspace import CleanupResult +pytestmark = [pytest.mark.layer("cli"), pytest.mark.small] + class TestFormatAge: def test_format_age_under_one_hour_returns_minutes(self) -> None: diff --git a/tests/config/CLAUDE.md b/tests/config/CLAUDE.md new file mode 100644 index 0000000000..85eb1dd0da --- /dev/null +++ b/tests/config/CLAUDE.md @@ -0,0 +1,28 @@ +# config/ + +Configuration loading, defaults, and schema tests. + +## Files + +| File | Purpose | +|------|---------| +| `__init__.py` | empty | +| `test_branching_config.py` | Tests for BranchingConfig and ReleaseReadinessConfig loading and validation | +| `test_config.py` | Tests for configuration loading and resolution | +| `test_config_split.py` | Structural guard for config split | +| `test_defaults.py` | Settings coherence guard: natural_exit_grace_seconds and exit_after_stop_delay_ms coherence | +| `test_fleet_config.py` | Tests for FleetConfig loading and validation | +| `test_helpers.py` | Tests for autoskillit.config resolve_ingredient_defaults | +| `test_linux_tracing_config.py` | Tests for LinuxTracingConfig loading and validation | +| `test_packs_config.py` | Tests for PacksConfig loading and validation | +| `test_providers_config.py` | Tests for ProvidersConfig loading and validation | +| `test_quota_guard_config.py` | Tests for QuotaGuardConfig loading and validation | +| `test_review_config.py` | Tests for ReviewConfig and local_review_rounds wiring | +| `test_settings_allowed_labels.py` | Tests for GitHubConfig.allowed_labels field and check_label_allowed validation | +| `test_settings_staged_label.py` | Tests for GitHubConfig.staged_label field and config layer resolution | +| `test_settings_queued_label.py` | Tests for GitHubConfig.queued_label field and label-state mapping methods | +| `test_skills_config.py` | Tests for SkillsConfig loading and validation | +| `test_timeout_coherence.py` | Tests for idle_output_timeout config coherence validation gate | +| `test_subsets_config.py` | Tests for SubsetsConfig loading and validation | +| `test_workspace_temp_dir_config.py` | Tests for workspace.temp_dir layered config resolution | +| `test_write_config_layer.py` | Tests for write_config_layer atomic write and validation | diff --git a/tests/config/test_branching_config.py b/tests/config/test_branching_config.py new file mode 100644 index 0000000000..cb07d4e6f5 --- /dev/null +++ b/tests/config/test_branching_config.py @@ -0,0 +1,68 @@ +"""Tests for BranchingConfig and ReleaseReadinessConfig loading and validation.""" + +import pytest + +from autoskillit.config import load_config +from autoskillit.config.settings import AutomationConfig, BranchingConfig +from autoskillit.core.io import load_yaml +from autoskillit.core.paths import pkg_root + +pytestmark = [pytest.mark.layer("config"), pytest.mark.small] + + +class TestBranchingConfig: + def test_automation_config_has_branching_field(self) -> None: + """AutomationConfig must expose a BranchingConfig as .branching.""" + cfg = AutomationConfig() + assert cfg.branching.default_base_branch == "main" + + def test_branching_config_is_overridable(self) -> None: + """BranchingConfig.default_base_branch must accept override values.""" + cfg = BranchingConfig(default_base_branch="develop") + assert cfg.default_base_branch == "develop" + + def test_branching_default_base_branch_matches_defaults_yaml(self) -> None: + """BranchingConfig Python default must match defaults.yaml.""" + defaults = load_yaml(pkg_root() / "config" / "defaults.yaml") + yaml_default = defaults["branching"]["default_base_branch"] + python_default = BranchingConfig().default_base_branch + + assert python_default == yaml_default, ( + f"BranchingConfig.default_base_branch Python default ({python_default!r}) " + f"disagrees with defaults.yaml ({yaml_default!r})" + ) + + def test_branching_config_promotion_target_defaults_to_main(self) -> None: + """BranchingConfig.promotion_target defaults to main (package default).""" + assert BranchingConfig().promotion_target == "main" + + def test_automation_config_branching_promotion_target_default(self) -> None: + """AutomationConfig propagates promotion_target default.""" + assert AutomationConfig().branching.promotion_target == "main" + + def test_branching_config_promotion_target_overridable(self) -> None: + """promotion_target can be set independently of default_base_branch.""" + cfg = BranchingConfig(default_base_branch="develop", promotion_target="main") + assert cfg.default_base_branch == "develop" + assert cfg.promotion_target == "main" + + def test_branching_config_promotion_target_defaults_match_yaml(self, tmp_path) -> None: + """Python default for promotion_target matches defaults.yaml.""" + loaded = load_config(tmp_path) + defaults = load_yaml(pkg_root() / "config" / "defaults.yaml") + assert "branching" in defaults, ( + f"Missing 'branching' key in defaults.yaml: {list(defaults.keys())}" + ) + assert "promotion_target" in defaults["branching"], ( + "Missing 'promotion_target' in defaults.yaml['branching']" + ) + expected = defaults["branching"]["promotion_target"] + assert loaded.branching.promotion_target == expected + + def test_branching_config_promotion_target_env_var_override( + self, monkeypatch, tmp_path + ) -> None: + """AUTOSKILLIT_BRANCHING__PROMOTION_TARGET env var overrides promotion_target.""" + monkeypatch.setenv("AUTOSKILLIT_BRANCHING__PROMOTION_TARGET", "stable") + cfg = load_config(tmp_path) + assert cfg.branching.promotion_target == "stable" diff --git a/tests/config/test_config.py b/tests/config/test_config.py index aa296e0056..33d5c9363c 100644 --- a/tests/config/test_config.py +++ b/tests/config/test_config.py @@ -1,12 +1,18 @@ """Tests for configuration loading and resolution.""" from dataclasses import fields as dc_fields -from pathlib import Path import pytest import yaml -from autoskillit.config import AutomationConfig, ConfigSchemaError, RunSkillConfig, load_config +from autoskillit.config import ( + AutomationConfig, + ConfigSchemaError, + RunSkillConfig, + load_config, +) + +pytestmark = [pytest.mark.layer("config"), pytest.mark.small] class TestDefaultConfig: @@ -20,15 +26,15 @@ def test_default_config_matches_current_constants(self): assert cfg.reset_workspace.preserve_dirs == set() assert cfg.implement_gate.marker == "Dry-walkthrough verified = TRUE" assert cfg.implement_gate.skill_names == { - "/autoskillit:implement-worktree", - "/autoskillit:implement-worktree-no-merge", + "/implement-worktree", + "/implement-worktree-no-merge", } assert cfg.safety.reset_guard_marker == ".autoskillit-workspace" assert cfg.safety.require_dry_walkthrough is True assert cfg.safety.test_gate_on_merge is True assert isinstance(cfg.safety.protected_branches, list) assert "main" in cfg.safety.protected_branches - assert "integration" in cfg.safety.protected_branches + assert "develop" in cfg.safety.protected_branches assert "stable" in cfg.safety.protected_branches assert cfg.worktree_setup.command is None @@ -201,11 +207,6 @@ def test_partial_model_config(self, tmp_path): assert cfg.model.override == "haiku" assert cfg.model.default == "sonnet" - def test_loaded_config_has_sonnet_default(self, tmp_path): - """MOD_C4: load_config produces model.default='sonnet'.""" - cfg = load_config(tmp_path) - assert cfg.model.default == "sonnet" - def test_yaml_loads_worktree_setup_config(self, tmp_path): """WS_C2: YAML with worktree_setup section populates WorktreeSetupConfig.""" config_dir = tmp_path / ".autoskillit" @@ -336,144 +337,12 @@ def test_run_skill_config_fields_include_exit_delay(self): assert "exit_after_stop_delay_ms" in names -class TestQuotaGuardConfig: - def test_default_enabled(self): - import pytest - - config = AutomationConfig() - assert config.quota_guard.enabled is True - assert config.quota_guard.short_window_threshold == pytest.approx(85.0) - assert config.quota_guard.long_window_threshold == pytest.approx(98.0) - assert config.quota_guard.buffer_seconds == 60 - assert config.quota_guard.cache_max_age == 300 - - def test_load_quota_guard_from_yaml(self, tmp_path): - import pytest - - config_dir = tmp_path / ".autoskillit" - config_dir.mkdir() - (config_dir / "config.yaml").write_text( - yaml.dump( - { - "quota_guard": { - "enabled": True, - "short_window_threshold": 85.0, - "long_window_threshold": 98.0, - } - } - ) - ) - config = load_config(tmp_path) - assert config.quota_guard.enabled is True - assert config.quota_guard.short_window_threshold == pytest.approx(85.0) - assert config.quota_guard.long_window_threshold == pytest.approx(98.0) - # Unspecified fields keep defaults - assert config.quota_guard.buffer_seconds == 60 - - def test_short_window_threshold_defaults_to_85(self): - from autoskillit.config.settings import QuotaGuardConfig - - assert QuotaGuardConfig().short_window_threshold == 85.0 - - def test_long_window_threshold_defaults_to_98(self): - from autoskillit.config.settings import QuotaGuardConfig - - assert QuotaGuardConfig().long_window_threshold == 98.0 - - def test_long_window_patterns_default(self): - from autoskillit.config.settings import QuotaGuardConfig - - assert QuotaGuardConfig().long_window_patterns == ["weekly", "sonnet", "opus"] - - def test_threshold_field_removed(self): - import dataclasses - - from autoskillit.config.settings import QuotaGuardConfig - - names = {f.name for f in dataclasses.fields(QuotaGuardConfig)} - assert "threshold" not in names - - def test_quota_guard_yaml_round_trip_per_window(self, tmp_path): - import pytest - - config_dir = tmp_path / ".autoskillit" - config_dir.mkdir() - (config_dir / "config.yaml").write_text( - yaml.dump( - { - "quota_guard": { - "short_window_threshold": 85.0, - "long_window_threshold": 98.0, - "long_window_patterns": ["weekly", "sonnet", "opus"], - } - } - ) - ) - config = load_config(tmp_path) - assert config.quota_guard.short_window_threshold == pytest.approx(85.0) - assert config.quota_guard.long_window_threshold == pytest.approx(98.0) - assert config.quota_guard.long_window_patterns == ["weekly", "sonnet", "opus"] - - def test_quota_guard_config_has_cache_refresh_interval(self): - """QG_C3: QuotaGuardConfig has cache_refresh_interval defaulting to 240.""" - from autoskillit.config.settings import QuotaGuardConfig - - config = QuotaGuardConfig() - assert config.cache_refresh_interval == 240 - - def test_defaults_yaml_has_cache_refresh_interval(self): - """QG_C4: defaults.yaml defines quota_guard.cache_refresh_interval < cache_max_age.""" - from autoskillit.core.paths import pkg_root - - defaults = yaml.safe_load((pkg_root() / "config" / "defaults.yaml").read_text()) - assert "cache_refresh_interval" in defaults["quota_guard"] - interval = defaults["quota_guard"]["cache_refresh_interval"] - max_age = defaults["quota_guard"]["cache_max_age"] - assert interval < max_age, ( - f"cache_refresh_interval ({interval}) must be < cache_max_age ({max_age}); " - "otherwise the loop arrives after the cache has already expired" - ) - - def test_quota_guard_per_window_enabled_defaults_true(self): - """Test 14: QuotaGuardConfig() defaults both per-window flags to True.""" - from autoskillit.config.settings import QuotaGuardConfig - - config = QuotaGuardConfig() - assert config.short_window_enabled is True - assert config.long_window_enabled is True - - def test_quota_guard_per_window_enabled_yaml_round_trip(self, tmp_path): - """Test 15: per-window enabled flags survive a YAML round-trip.""" - config_dir = tmp_path / ".autoskillit" - config_dir.mkdir() - (config_dir / "config.yaml").write_text( - yaml.dump({"quota_guard": {"short_window_enabled": False}}) - ) - config = load_config(tmp_path) - assert config.quota_guard.short_window_enabled is False - assert config.quota_guard.long_window_enabled is True - - def test_defaults_yaml_has_per_window_enabled_keys(self): - """Test 16: defaults.yaml has both per-window enabled keys set to True.""" - from autoskillit.core.paths import pkg_root - - defaults = yaml.safe_load((pkg_root() / "config" / "defaults.yaml").read_text()) - assert defaults["quota_guard"]["short_window_enabled"] is True - assert defaults["quota_guard"]["long_window_enabled"] is True - - def test_quota_guard_env_var_override_short_window_enabled(self, monkeypatch, tmp_path): - """Test 17: AUTOSKILLIT_QUOTA_GUARD__SHORT_WINDOW_ENABLED=false is cast to bool False.""" - monkeypatch.setenv("AUTOSKILLIT_QUOTA_GUARD__SHORT_WINDOW_ENABLED", "false") - config = load_config(tmp_path) - assert config.quota_guard.short_window_enabled is False - - class TestLoggingConfig: """LoggingConfig dataclass and YAML loading.""" def test_logging_config_defaults(self, tmp_path): """LOG_C1: LoggingConfig has correct defaults from defaults.yaml.""" - cfg = load_config(tmp_path / "settings.toml") + cfg = load_config(tmp_path) assert cfg.logging.level == "INFO" assert cfg.logging.json_output is None @@ -535,45 +404,6 @@ def test_logging_config_fields(self): assert names == {"level", "json_output"} -class TestLinuxTracingConfig: - """LinuxTracingConfig dataclass and YAML loading.""" - - def test_linux_tracing_config_defaults(self, tmp_path): - """LT_C1: LinuxTracingConfig defaults: enabled, 5s interval, empty log_dir.""" - cfg = load_config(tmp_path / "settings.toml") - assert cfg.linux_tracing.enabled is True - assert cfg.linux_tracing.proc_interval == 5.0 - assert cfg.linux_tracing.log_dir == "" - - def test_linux_tracing_config_from_yaml(self, tmp_path): - """LT_C2: LinuxTracingConfig reads from project config.""" - config_dir = tmp_path / ".autoskillit" - config_dir.mkdir() - (config_dir / "config.yaml").write_text( - "linux_tracing:\n enabled: true\n proc_interval: 2.0\n log_dir: /custom/logs\n" - ) - cfg = load_config(tmp_path) - assert cfg.linux_tracing.enabled is True - assert cfg.linux_tracing.proc_interval == 2.0 - assert cfg.linux_tracing.log_dir == "/custom/logs" - - def test_automation_config_has_linux_tracing_field(self): - """LT_C3: AutomationConfig has linux_tracing sub-config.""" - cfg = AutomationConfig() - assert cfg.linux_tracing.enabled is True - assert cfg.linux_tracing.proc_interval == 5.0 - assert cfg.linux_tracing.log_dir == "" - - def test_linux_tracing_config_fields(self): - """LT_C4: LinuxTracingConfig has exactly the expected fields.""" - from dataclasses import fields as dc_fields - - from autoskillit.config.settings import LinuxTracingConfig - - names = {f.name for f in dc_fields(LinuxTracingConfig)} - assert names == {"enabled", "proc_interval", "log_dir", "tmpfs_path"} - - class TestDynaconfIntegration: def test_env_var_overrides_nested_github_token(self, tmp_path, monkeypatch): """ENV-1: AUTOSKILLIT_GITHUB__TOKEN env var sets config.github.token.""" @@ -723,6 +553,12 @@ def test_github_config_has_in_progress_label(self): cfg = GitHubConfig() assert cfg.in_progress_label == "in-progress" + def test_github_config_has_fail_label(self): + from autoskillit.config.settings import GitHubConfig + + cfg = GitHubConfig() + assert cfg.fail_label == "fail" + def test_secrets_only_keys_covers_all_github_secret_fields() -> None: """_SECRETS_ONLY_KEYS must include every field in GitHubConfig that holds a secret. @@ -747,407 +583,11 @@ def test_secrets_only_keys_covers_all_github_secret_fields() -> None: ) -class TestReleaseReadinessConfig: - def test_branching_default_base_branch_is_main(self): - from autoskillit.core.io import load_yaml - from autoskillit.core.paths import pkg_root - - defaults = load_yaml(pkg_root() / "config" / "defaults.yaml") - assert defaults["branching"]["default_base_branch"] == "main" - - -class TestBranchingConfig: - def test_branching_config_default_base_branch_is_main(self) -> None: - """BranchingConfig must default default_base_branch to 'main'.""" - from autoskillit.config.settings import BranchingConfig - - assert BranchingConfig().default_base_branch == "main" - - def test_automation_config_has_branching_field(self) -> None: - """AutomationConfig must expose a BranchingConfig as .branching.""" - from autoskillit.config.settings import AutomationConfig - - cfg = AutomationConfig() - assert cfg.branching.default_base_branch == "main" - - def test_branching_config_is_overridable(self) -> None: - """BranchingConfig.default_base_branch must accept override values.""" - from autoskillit.config.settings import BranchingConfig - - cfg = BranchingConfig(default_base_branch="develop") - assert cfg.default_base_branch == "develop" - - def test_branching_default_base_branch_matches_defaults_yaml(self) -> None: - """BranchingConfig Python default must match defaults.yaml.""" - from autoskillit.config.settings import BranchingConfig - from autoskillit.core.io import load_yaml - from autoskillit.core.paths import pkg_root - - defaults = load_yaml(pkg_root() / "config" / "defaults.yaml") - yaml_default = defaults["branching"]["default_base_branch"] - python_default = BranchingConfig().default_base_branch - - assert python_default == yaml_default, ( - f"BranchingConfig.default_base_branch Python default ({python_default!r}) " - f"disagrees with defaults.yaml ({yaml_default!r})" - ) - - def test_branching_config_promotion_target_defaults_to_main(self) -> None: - """BranchingConfig.promotion_target defaults to main (package default).""" - from autoskillit.config.settings import BranchingConfig - - assert BranchingConfig().promotion_target == "main" - - def test_automation_config_branching_promotion_target_default(self) -> None: - """AutomationConfig propagates promotion_target default.""" - from autoskillit.config.settings import AutomationConfig - - assert AutomationConfig().branching.promotion_target == "main" - - def test_branching_config_promotion_target_overridable(self) -> None: - """promotion_target can be set independently of default_base_branch.""" - from autoskillit.config.settings import BranchingConfig - - cfg = BranchingConfig(default_base_branch="integration", promotion_target="main") - assert cfg.default_base_branch == "integration" - assert cfg.promotion_target == "main" - - def test_branching_config_promotion_target_defaults_match_yaml(self, tmp_path) -> None: - """Python default for promotion_target matches defaults.yaml.""" - from autoskillit.config import load_config - from autoskillit.core.io import load_yaml - from autoskillit.core.paths import pkg_root - - loaded = load_config(tmp_path / "settings.toml") - defaults = load_yaml(pkg_root() / "config" / "defaults.yaml") - expected = defaults["branching"]["promotion_target"] - assert loaded.branching.promotion_target == expected - - def test_branching_config_promotion_target_env_var_override( - self, monkeypatch, tmp_path - ) -> None: - """AUTOSKILLIT_BRANCHING__PROMOTION_TARGET env var overrides promotion_target.""" - from autoskillit.config import load_config - - monkeypatch.setenv("AUTOSKILLIT_BRANCHING__PROMOTION_TARGET", "stable") - cfg = load_config(tmp_path / "settings.toml") - assert cfg.branching.promotion_target == "stable" - - -class TestBuildSubsetsConfigCustomTagsValidation: - """CC-F2: _build_subsets_config must raise ValueError for non-dict custom_tags.""" - - @pytest.mark.parametrize("bad_value", [["list", "not", "dict"], "oops", 42]) - def test_build_subsets_config_raises_for_non_dict_custom_tags(self, bad_value: object) -> None: - """Non-dict custom_tags must raise ValueError, not silently coerce to {}.""" - from autoskillit.config.settings import _build_subsets_config - - with pytest.raises(ValueError, match="custom_tags"): - _build_subsets_config({"custom_tags": bad_value}) - - def test_build_subsets_config_dict_custom_tags_accepted(self): - """Valid dict custom_tags must not raise.""" - from autoskillit.config.settings import _build_subsets_config - - result = _build_subsets_config({"custom_tags": {"my_tag": ["skill-a"]}}) - assert result.custom_tags == {"my_tag": ["skill-a"]} - - def test_build_subsets_config_empty_dict_custom_tags_accepted(self): - """Empty dict custom_tags must not raise.""" - from autoskillit.config.settings import _build_subsets_config - - result = _build_subsets_config({"custom_tags": {}}) - assert result.custom_tags == {} - - -class TestSkillsConfig: - """SkillsConfig dataclass, tier duplication validation, and AutomationConfig integration.""" - - def test_skills_config_dataclass(self) -> None: - """SkillsConfig has tier1, tier2, tier3 list[str] fields.""" - from autoskillit.config.settings import SkillsConfig - - sc = SkillsConfig(tier1=["a"], tier2=["b"], tier3=["c"]) - assert sc.tier1 == ["a"] and sc.tier2 == ["b"] and sc.tier3 == ["c"] - - def test_skills_config_tier_duplication_raises(self) -> None: - """Skill in multiple tiers raises ValueError at construction (REQ-TIER-009).""" - import pytest - - from autoskillit.config.settings import SkillsConfig - - with pytest.raises(ValueError, match="multiple tiers"): - SkillsConfig(tier1=["open-kitchen"], tier2=["open-kitchen"], tier3=[]) - - def test_automation_config_has_skills_field(self) -> None: - """AutomationConfig.skills is a SkillsConfig with tier lists.""" - from autoskillit.config.settings import AutomationConfig, SkillsConfig - - cfg = AutomationConfig() - assert isinstance(cfg.skills, SkillsConfig) - - def test_defaults_yaml_skills_section(self) -> None: - """defaults.yaml has non-empty skills.tier1/tier2/tier3 lists.""" - from autoskillit.core import load_yaml, pkg_root - - defaults = load_yaml(pkg_root() / "config" / "defaults.yaml") - assert isinstance(defaults, dict) - skills = defaults.get("skills", {}) - assert "open-kitchen" in skills.get("tier1", []) - assert len(skills.get("tier2", [])) >= 20 - assert len(skills.get("tier3", [])) >= 10 - - def test_load_config_populates_skills_tiers(self, tmp_path) -> None: - """load_config() produces an AutomationConfig with tier assignments from defaults.""" - from autoskillit.config import load_config - - cfg = load_config(tmp_path / "settings.toml") - assert "open-kitchen" in cfg.skills.tier1 - assert "make-plan" in cfg.skills.tier2 - assert "compose-pr" in cfg.skills.tier3 - - def test_skills_config_exported_from_config_package(self) -> None: - """SkillsConfig is importable from autoskillit.config and has expected fields.""" - from autoskillit.config import SkillsConfig - - cfg = SkillsConfig() - assert hasattr(cfg, "tier1") - assert hasattr(cfg, "tier2") - assert hasattr(cfg, "tier3") - - -class TestSubsetsConfig: - # T1 — SubsetsConfig defaults - - def test_subsets_config_default_disabled_is_empty_list(self) -> None: - from autoskillit.config import SubsetsConfig - - cfg = SubsetsConfig() - assert cfg.disabled == [] - - def test_subsets_config_default_custom_tags_is_empty_dict(self) -> None: - from autoskillit.config import SubsetsConfig - - cfg = SubsetsConfig() - assert cfg.custom_tags == {} - - def test_automation_config_has_subsets_field(self) -> None: - from autoskillit.config import AutomationConfig, SubsetsConfig - - cfg = AutomationConfig() - assert isinstance(cfg.subsets, SubsetsConfig) - assert cfg.subsets.disabled == [] - assert cfg.subsets.custom_tags == {} - - # T2 — load_config with subsets.disabled - - def test_load_config_subsets_disabled(self, tmp_path) -> None: - config_dir = tmp_path / ".autoskillit" - config_dir.mkdir() - (config_dir / "config.yaml").write_text("subsets:\n disabled:\n - github\n - ci\n") - cfg = load_config(tmp_path) - assert cfg.subsets.disabled == ["github", "ci"] - - def test_load_config_subsets_disabled_absent_means_empty(self, tmp_path) -> None: - cfg = load_config(tmp_path) - assert cfg.subsets.disabled == [] - - # T3 — load_config with subsets.custom_tags - - def test_load_config_subsets_custom_tags(self, tmp_path) -> None: - config_dir = tmp_path / ".autoskillit" - config_dir.mkdir() - yaml_text = ( - "subsets:\n" - " custom_tags:\n" - " my-team-tools:\n" - " - investigate\n" - " - make-plan\n" - ) - (config_dir / "config.yaml").write_text(yaml_text) - cfg = load_config(tmp_path) - assert cfg.subsets.custom_tags == {"my-team-tools": ["investigate", "make-plan"]} - - def test_load_config_subsets_custom_tags_absent_means_empty(self, tmp_path) -> None: - cfg = load_config(tmp_path) - assert cfg.subsets.custom_tags == {} - - # T4 — Unknown category warning, no crash - - def test_load_config_unknown_disabled_category_logs_warning_not_crash(self, tmp_path) -> None: - import logging - - config_dir = tmp_path / ".autoskillit" - config_dir.mkdir() - (config_dir / "config.yaml").write_text( - "subsets:\n disabled:\n - totally-unknown-category\n" - ) - # Attach a handler directly to the logger to capture warnings reliably. - # caplog is unreliable here because the structlog capture_logs() autouse - # fixture can intercept the handler chain under xdist worker ordering. - captured: list[logging.LogRecord] = [] - handler = logging.Handler() - handler.emit = captured.append # type: ignore[assignment] - logger = logging.getLogger("autoskillit.config.settings") # noqa: TID251 - logger.addHandler(handler) - try: - cfg = load_config(tmp_path) - finally: - logger.removeHandler(handler) - assert cfg.subsets.disabled == ["totally-unknown-category"] # preserved as-is - assert any("totally-unknown-category" in r.getMessage() for r in captured) - - def test_load_config_known_disabled_category_no_warning(self, tmp_path, caplog) -> None: - import logging - - config_dir = tmp_path / ".autoskillit" - config_dir.mkdir() - (config_dir / "config.yaml").write_text("subsets:\n disabled:\n - github\n") - with caplog.at_level(logging.WARNING): - load_config(tmp_path) - assert not any("github" in r.message for r in caplog.records) - - def test_load_config_custom_tag_in_disabled_is_valid(self, tmp_path, caplog) -> None: - """Custom tags defined in custom_tags can also appear in disabled.""" - import logging - - config_dir = tmp_path / ".autoskillit" - config_dir.mkdir() - yaml_text = ( - "subsets:\n" - " disabled:\n" - " - experimental\n" - " custom_tags:\n" - " experimental:\n" - " - write-recipe\n" - ) - (config_dir / "config.yaml").write_text(yaml_text) - with caplog.at_level(logging.WARNING): - load_config(tmp_path) - assert not any("experimental" in r.message for r in caplog.records) - - # T5 — SubsetsConfig exported from config package - - def test_subsets_config_importable_from_config_package(self) -> None: - from autoskillit.config import SubsetsConfig - - cfg = SubsetsConfig() - assert hasattr(cfg, "disabled") - assert hasattr(cfg, "custom_tags") - - -class TestPacksConfig: - # REQ-PACK-003: PacksConfig.enabled default - - def test_packs_config_default_enabled_is_empty_list(self) -> None: - from autoskillit.config import PacksConfig - - assert PacksConfig().enabled == [] - - def test_automation_config_has_packs_field(self) -> None: - from autoskillit.config import AutomationConfig, PacksConfig - - cfg = AutomationConfig() - assert isinstance(cfg.packs, PacksConfig) - assert cfg.packs.enabled == [] - - def test_load_config_packs_enabled(self, tmp_path) -> None: - from autoskillit.config import load_config - - config_dir = tmp_path / ".autoskillit" - config_dir.mkdir() - (config_dir / "config.yaml").write_text("packs:\n enabled:\n - research\n") - config = load_config(tmp_path) - assert config.packs.enabled == ["research"] - - def test_load_config_packs_enabled_absent_means_empty(self, tmp_path) -> None: - from autoskillit.config import load_config - - config_dir = tmp_path / ".autoskillit" - config_dir.mkdir() - (config_dir / "config.yaml").write_text("subsets:\n disabled: []\n") - config = load_config(tmp_path) - assert config.packs.enabled == [] - - def test_load_config_unknown_pack_in_packs_enabled_logs_warning(self, tmp_path) -> None: - import logging - - from autoskillit.config import load_config - - config_dir = tmp_path / ".autoskillit" - config_dir.mkdir() - (config_dir / "config.yaml").write_text("packs:\n enabled:\n - nonexistent-pack\n") - captured: list[logging.LogRecord] = [] - handler = logging.Handler() - handler.emit = captured.append # type: ignore[assignment] - logger = logging.getLogger("autoskillit.config.settings") # noqa: TID251 - logger.addHandler(handler) - try: - config = load_config(tmp_path) - finally: - logger.removeHandler(handler) - assert config.packs.enabled == ["nonexistent-pack"] # preserved as-is - assert any("nonexistent-pack" in r.getMessage() for r in captured) - - def test_write_config_layer_accepts_packs_enabled(self, tmp_path) -> None: - """write_config_layer does not raise for valid packs.enabled.""" - from autoskillit.config.settings import write_config_layer - - config_path = tmp_path / "config.yaml" - write_config_layer(config_path, {"packs": {"enabled": ["research"]}}) - assert config_path.exists() - - -class TestWriteConfigLayer: - def test_write_config_layer_rejects_secret_key(self, tmp_path: Path) -> None: - """write_config_layer raises ConfigSchemaError before touching the file.""" - - from autoskillit.config.settings import ConfigSchemaError, write_config_layer - - config_path = tmp_path / "config.yaml" - with pytest.raises(ConfigSchemaError, match="github.token"): - write_config_layer(config_path, {"github": {"token": "ghp_test"}}) - assert not config_path.exists(), "config.yaml must not be written on schema error" - - def test_write_config_layer_rejects_unknown_key(self, tmp_path: Path) -> None: - """write_config_layer raises ConfigSchemaError for unknown section.""" - from autoskillit.config.settings import ConfigSchemaError, write_config_layer - - config_path = tmp_path / "config.yaml" - with pytest.raises(ConfigSchemaError, match="unrecognized key"): - write_config_layer(config_path, {"invented_section": {"foo": "bar"}}) - assert not config_path.exists() - - def test_write_config_layer_writes_valid_content(self, tmp_path: Path) -> None: - """write_config_layer writes valid schema content atomically.""" - import yaml as _yaml - - from autoskillit.config.settings import write_config_layer - - config_path = tmp_path / "config.yaml" - write_config_layer(config_path, {"github": {"default_repo": "owner/repo"}}) - assert config_path.is_file() - data = _yaml.safe_load(config_path.read_text()) - assert data["github"]["default_repo"] == "owner/repo" - - class TestWorkspaceConfig: """WorkspaceConfig section is present in AutomationConfig with correct defaults.""" - def test_workspace_config_exists_on_automation_config(self, tmp_path): - from autoskillit.config import load_config - - cfg = load_config(tmp_path / "settings.toml") + def test_workspace_config_defaults(self, tmp_path): + cfg = load_config(tmp_path) assert hasattr(cfg, "workspace") - - def test_workspace_worktree_root_defaults_to_none(self, tmp_path): - from autoskillit.config import load_config - - cfg = load_config(tmp_path / "settings.toml") assert cfg.workspace.worktree_root is None - - def test_workspace_runs_root_defaults_to_none(self, tmp_path): - from autoskillit.config import load_config - - cfg = load_config(tmp_path / "settings.toml") assert cfg.workspace.runs_root is None diff --git a/tests/config/test_config_split.py b/tests/config/test_config_split.py new file mode 100644 index 0000000000..a4a3bda7c6 --- /dev/null +++ b/tests/config/test_config_split.py @@ -0,0 +1,26 @@ +import pytest + +pytestmark = [pytest.mark.layer("config"), pytest.mark.small] + + +def test_leaf_dataclasses_importable_from_submodule(): + from autoskillit.config._config_dataclasses import ( + RunSkillConfig, + ) + + assert RunSkillConfig is not None + + +def test_loader_importable_from_submodule(): + from autoskillit.config._config_loader import load_config + + assert callable(load_config) + + +def test_public_api_unchanged(): + # config/__init__.py surface is unbroken after split + from autoskillit.config import ( + AutomationConfig, + ) + + assert AutomationConfig is not None diff --git a/tests/config/test_defaults.py b/tests/config/test_defaults.py index 8f02f2814c..7f3ab851c3 100644 --- a/tests/config/test_defaults.py +++ b/tests/config/test_defaults.py @@ -7,10 +7,13 @@ from __future__ import annotations import pytest +import structlog.testing -from autoskillit.config import AutomationConfig +from autoskillit.config import AutomationConfig, load_config from autoskillit.config.settings import RunSkillConfig +pytestmark = [pytest.mark.layer("config"), pytest.mark.small] + class TestGraceWindowCoherence: """natural_exit_grace_seconds must cover exit_after_stop_delay_ms + margin.""" @@ -42,3 +45,44 @@ def test_settings_validator_accepts_coherent_config(self) -> None: ) # 3.0 * 1000 = 3000 >= 2000 + 500 = 2500 ✓ assert cfg.natural_exit_grace_seconds == 3.0 + + +class TestAllDefaultsPassAllGates: + """Universal guard: shipped defaults must produce zero validation warnings.""" + + def test_pure_defaults_emit_no_coherence_warnings(self, tmp_path) -> None: + """load_config with empty user config emits no *_coherence warnings.""" + config_dir = tmp_path / ".autoskillit" + config_dir.mkdir() + (config_dir / "config.yaml").write_text("") + with structlog.testing.capture_logs() as cap_logs: + load_config(tmp_path) + coherence_events = [ + e + for e in cap_logs + if e.get("log_level") == "warning" and "_coherence" in e.get("event", "") + ] + assert coherence_events == [], ( + f"Default config produced coherence warnings: {coherence_events}. " + "A shipped default violates its own validation gate. " + "Fix the default or the gate threshold." + ) + + +class TestDefaultsSyncYamlDataclass: + """Numeric defaults in dataclasses must match defaults.yaml.""" + + def test_run_skill_defaults_match_yaml(self, tmp_path) -> None: + """RunSkillConfig field defaults agree with defaults.yaml values.""" + config_dir = tmp_path / ".autoskillit" + config_dir.mkdir() + (config_dir / "config.yaml").write_text("") + cfg = load_config(tmp_path) + dc = RunSkillConfig() + assert cfg.run_skill.timeout == dc.timeout + assert cfg.run_skill.stale_threshold == dc.stale_threshold + assert cfg.run_skill.idle_output_timeout == dc.idle_output_timeout + assert cfg.run_skill.exit_after_stop_delay_ms == dc.exit_after_stop_delay_ms + assert cfg.run_skill.natural_exit_grace_seconds == dc.natural_exit_grace_seconds + assert cfg.run_skill.max_suppression_seconds == dc.max_suppression_seconds + assert cfg.run_skill.completion_drain_timeout == dc.completion_drain_timeout diff --git a/tests/config/test_fleet_config.py b/tests/config/test_fleet_config.py new file mode 100644 index 0000000000..703660ca7e --- /dev/null +++ b/tests/config/test_fleet_config.py @@ -0,0 +1,235 @@ +"""Tests for FleetConfig loading and validation.""" + +from __future__ import annotations + +from pathlib import Path + +import pytest +import yaml + +from autoskillit.config import AutomationConfig, FleetConfig, load_config + +pytestmark = [pytest.mark.layer("config"), pytest.mark.small] + + +class TestFleetConfig: + def test_automation_config_has_fleet_field(self) -> None: + """AutomationConfig exposes fleet as a FleetConfig.""" + cfg = AutomationConfig() + assert isinstance(cfg.fleet, FleetConfig) + assert cfg.fleet.default_timeout_sec == 3600 + + def test_fleet_l3_timeout_matches_defaults_yaml(self) -> None: + """FleetConfig Python default matches defaults.yaml value.""" + from autoskillit.config.settings import FleetConfig + from autoskillit.core.io import load_yaml + from autoskillit.core.paths import pkg_root + + defaults = load_yaml(pkg_root() / "config" / "defaults.yaml") + assert "fleet" in defaults, "defaults.yaml missing 'fleet' section" + assert "default_timeout_sec" in defaults["fleet"], ( + "defaults.yaml['fleet'] missing 'default_timeout_sec'" + ) + yaml_val = defaults["fleet"]["default_timeout_sec"] + assert FleetConfig().default_timeout_sec == yaml_val + + def test_load_config_fleet_override(self, tmp_path) -> None: + """User config with fleet section overrides the default.""" + import yaml + + from autoskillit.config import load_config + + config_dir = tmp_path / ".autoskillit" + config_dir.mkdir() + config_data = {"fleet": {"default_timeout_sec": 7200}} + (config_dir / "config.yaml").write_text(yaml.dump(config_data)) + cfg = load_config(tmp_path) + assert cfg.fleet.default_timeout_sec == 7200 + + def test_fleet_config_importable_from_config_package(self) -> None: + """FleetConfig is importable from autoskillit.config.""" + import dataclasses + + from autoskillit.config import FleetConfig + + assert dataclasses.is_dataclass(FleetConfig) + + def test_fleet_key_accepted_by_schema_validator(self, tmp_path) -> None: + """User config with fleet: section does not raise ConfigSchemaError.""" + config_dir = tmp_path / ".autoskillit" + config_dir.mkdir() + config_data = {"fleet": {"default_timeout_sec": 1800}} + (config_dir / "config.yaml").write_text(yaml.dump(config_data)) + cfg = load_config(tmp_path) # must not raise ConfigSchemaError + assert cfg.fleet.default_timeout_sec == 1800 + + def test_fleet_config_rejects_zero_timeout(self) -> None: + """FleetConfig raises ValueError when default_timeout_sec is zero.""" + from autoskillit.config.settings import FleetConfig + + with pytest.raises(ValueError, match="default_timeout_sec must be positive"): + FleetConfig(default_timeout_sec=0).validate(True) + + def test_fleet_config_rejects_negative_timeout(self) -> None: + """FleetConfig raises ValueError when default_timeout_sec is negative.""" + from autoskillit.config.settings import FleetConfig + + with pytest.raises(ValueError, match="default_timeout_sec must be positive"): + FleetConfig(default_timeout_sec=-1).validate(True) + + def test_fleet_config_validate_skips_when_feature_disabled(self) -> None: + """FC_NEW_2: validate(False) does NOT raise even for invalid timeout.""" + from autoskillit.config.settings import FleetConfig + + FleetConfig(default_timeout_sec=0).validate(False) # must not raise + + def test_fleet_config_construction_no_longer_raises_for_invalid_timeout(self) -> None: + """FC_NEW_3: FleetConfig(default_timeout_sec=0) constructs without raising.""" + from autoskillit.config.settings import FleetConfig + + cfg = FleetConfig(default_timeout_sec=0) + assert cfg.default_timeout_sec == 0 + + def test_load_config_fleet_invalid_timeout_skips_when_disabled(self, tmp_path: Path) -> None: + """FC_NEW_4: load_config does NOT raise with invalid timeout when fleet disabled.""" + config_dir = tmp_path / ".autoskillit" + config_dir.mkdir() + config_data = { + "fleet": {"default_timeout_sec": -1}, + "features": {"fleet": False}, + } + (config_dir / "config.yaml").write_text(yaml.dump(config_data)) + cfg = load_config(tmp_path) # must not raise + assert cfg.fleet.default_timeout_sec == -1 + + def test_load_config_fleet_invalid_timeout_raises_when_enabled(self, tmp_path: Path) -> None: + """FC_NEW_5: load_config raises ValueError with invalid timeout when fleet enabled.""" + config_dir = tmp_path / ".autoskillit" + config_dir.mkdir() + config_data = { + "fleet": {"default_timeout_sec": -1}, + "features": {"fleet": True}, + } + (config_dir / "config.yaml").write_text(yaml.dump(config_data)) + with pytest.raises(ValueError, match="default_timeout_sec must be positive"): + load_config(tmp_path) + + def test_fleet_config_max_concurrent_dispatches_defaults_to_3(self) -> None: + """FleetConfig.max_concurrent_dispatches defaults to 3 (BEM-gated parallel).""" + from autoskillit.config.settings import FleetConfig + + assert FleetConfig().max_concurrent_dispatches == 3 + + def test_fleet_config_max_concurrent_dispatches_rejects_above_3(self) -> None: + """FleetConfig.validate() rejects max_concurrent_dispatches > 3.""" + from autoskillit.config.settings import FleetConfig + + with pytest.raises(ValueError, match="must be <= 3"): + FleetConfig(max_concurrent_dispatches=4).validate(feature_enabled=True) + + def test_fleet_config_max_concurrent_dispatches_validates_positive(self) -> None: + """FleetConfig raises ValueError when max_concurrent_dispatches is 0.""" + from autoskillit.config.settings import FleetConfig + + with pytest.raises(ValueError, match="max_concurrent_dispatches"): + FleetConfig(max_concurrent_dispatches=0).validate(feature_enabled=True) + + def test_fleet_config_max_total_issues_defaults_to_12(self) -> None: + """FleetConfig.max_total_issues defaults to 12.""" + from autoskillit.config.settings import FleetConfig + + assert FleetConfig().max_total_issues == 12 + + def test_fleet_config_max_total_issues_validates_positive(self) -> None: + """FleetConfig.validate() rejects max_total_issues < 1.""" + from autoskillit.config.settings import FleetConfig + + with pytest.raises(ValueError, match="max_total_issues"): + FleetConfig(max_total_issues=0).validate(feature_enabled=True) + + def test_fleet_config_max_total_issues_matches_defaults_yaml(self) -> None: + """FleetConfig.max_total_issues matches defaults.yaml fleet.max_total_issues.""" + from autoskillit.config.settings import FleetConfig + from autoskillit.core.io import load_yaml + from autoskillit.core.paths import pkg_root + + defaults = load_yaml(pkg_root() / "config" / "defaults.yaml") + assert FleetConfig().max_total_issues == defaults["fleet"]["max_total_issues"] + + def test_fleet_config_max_concurrent_dispatches_matches_defaults_yaml(self) -> None: + """FleetConfig.max_concurrent_dispatches matches defaults.yaml.""" + from autoskillit.config.settings import FleetConfig + from autoskillit.core.io import load_yaml + from autoskillit.core.paths import pkg_root + + defaults = load_yaml(pkg_root() / "config" / "defaults.yaml") + assert ( + FleetConfig().max_concurrent_dispatches + == defaults["fleet"]["max_concurrent_dispatches"] + ) + + def test_load_config_fleet_max_concurrent_override(self, tmp_path: Path) -> None: + """User config can override max_concurrent_dispatches.""" + config_dir = tmp_path / ".autoskillit" + config_dir.mkdir() + config_data = {"fleet": {"max_concurrent_dispatches": 3}} + (config_dir / "config.yaml").write_text(yaml.dump(config_data)) + cfg = load_config(tmp_path) + assert cfg.fleet.max_concurrent_dispatches == 3 + + def test_fleet_config_enable_deadline_extension_defaults_true(self) -> None: + """FleetConfig.enable_deadline_extension defaults to True.""" + from autoskillit.config.settings import FleetConfig + + assert FleetConfig().enable_deadline_extension is True + + def test_fleet_config_max_extension_seconds_defaults_7200(self) -> None: + """FleetConfig.max_extension_seconds defaults to 7200.""" + from autoskillit.config.settings import FleetConfig + + assert FleetConfig().max_extension_seconds == 7200 + + def test_fleet_config_idle_output_timeout_defaults_1800(self) -> None: + """FleetConfig.idle_output_timeout defaults to 1800.""" + from autoskillit.config.settings import FleetConfig + + assert FleetConfig().idle_output_timeout == 1800 + + def test_fleet_config_rejects_zero_max_extension_seconds(self) -> None: + """FleetConfig raises ValueError when max_extension_seconds is zero.""" + from autoskillit.config.settings import FleetConfig + + with pytest.raises(ValueError, match="max_extension_seconds must be positive"): + FleetConfig(max_extension_seconds=0).validate(feature_enabled=True) + + def test_fleet_config_rejects_negative_max_extension_seconds(self) -> None: + """FleetConfig raises ValueError when max_extension_seconds is negative.""" + from autoskillit.config.settings import FleetConfig + + with pytest.raises(ValueError, match="max_extension_seconds must be positive"): + FleetConfig(max_extension_seconds=-1).validate(feature_enabled=True) + + def test_fleet_config_rejects_negative_idle_output_timeout(self) -> None: + """FleetConfig raises ValueError when idle_output_timeout is negative.""" + from autoskillit.config.settings import FleetConfig + + with pytest.raises(ValueError, match="idle_output_timeout must be non-negative"): + FleetConfig(idle_output_timeout=-1).validate(feature_enabled=True) + + +def test_config_resolution_fleet_enabled_via_experimental(tmp_path) -> None: + """Full config resolution enables fleet when experimental_enabled=True in project config.""" + from autoskillit.config.settings import load_config + from autoskillit.core.feature_flags import is_feature_enabled + + config_dir = tmp_path / ".autoskillit" + config_dir.mkdir() + (config_dir / "config.yaml").write_text( + yaml.dump({"features": {"experimental_enabled": True}}) + ) + cfg = load_config(tmp_path) + assert cfg.experimental_enabled is True + assert ( + is_feature_enabled("fleet", cfg.features, experimental_enabled=cfg.experimental_enabled) + is True + ) diff --git a/tests/config/test_helpers.py b/tests/config/test_helpers.py index a3ad849bb0..055e079f65 100644 --- a/tests/config/test_helpers.py +++ b/tests/config/test_helpers.py @@ -4,6 +4,10 @@ import subprocess +import pytest + +pytestmark = [pytest.mark.layer("config"), pytest.mark.medium] + def test_resolve_ingredient_defaults_uses_upstream_when_origin_is_file_url(tmp_path): """resolve_ingredient_defaults must return the upstream URL when origin is file://.""" @@ -38,3 +42,19 @@ def test_resolve_ingredient_defaults_still_works_with_github_origin(tmp_path): ) defaults = resolve_ingredient_defaults(repo) assert defaults.get("source_dir") == "https://github.com/owner/repo.git" + + +def test_resolve_ingredient_defaults_includes_local_review_rounds(tmp_path): + """T2.3: resolve_ingredient_defaults includes local_review_rounds with default value 3.""" + from autoskillit.config import resolve_ingredient_defaults + + repo = tmp_path / "repo" + subprocess.run(["git", "init", str(repo)], check=True) + subprocess.run( + ["git", "remote", "add", "origin", "https://github.com/owner/repo.git"], + cwd=str(repo), + check=True, + ) + + defaults = resolve_ingredient_defaults(repo) + assert defaults.get("local_review_rounds") == "3" diff --git a/tests/config/test_linux_tracing_config.py b/tests/config/test_linux_tracing_config.py new file mode 100644 index 0000000000..58e8e5573d --- /dev/null +++ b/tests/config/test_linux_tracing_config.py @@ -0,0 +1,64 @@ +"""Tests for LinuxTracingConfig loading and validation.""" + +import dataclasses + +import pytest + +from autoskillit.config import AutomationConfig, load_config +from autoskillit.config.settings import LinuxTracingConfig + +pytestmark = [pytest.mark.layer("config"), pytest.mark.small] + + +class TestLinuxTracingConfig: + """LinuxTracingConfig dataclass and YAML loading.""" + + def test_linux_tracing_config_defaults(self, tmp_path): + """LT_C1: LinuxTracingConfig defaults: enabled, 5s interval, empty log_dir.""" + cfg = load_config(tmp_path) + assert cfg.linux_tracing.enabled is True + assert cfg.linux_tracing.proc_interval == 5.0 + assert cfg.linux_tracing.log_dir == "" + + def test_linux_tracing_config_from_yaml(self, tmp_path): + """LT_C2: LinuxTracingConfig reads from project config.""" + config_dir = tmp_path / ".autoskillit" + config_dir.mkdir() + (config_dir / "config.yaml").write_text( + "linux_tracing:\n enabled: true\n proc_interval: 2.0\n log_dir: /custom/logs\n" + ) + cfg = load_config(tmp_path) + assert cfg.linux_tracing.enabled is True + assert cfg.linux_tracing.proc_interval == 2.0 + assert cfg.linux_tracing.log_dir == "/custom/logs" + + def test_automation_config_has_linux_tracing_field(self): + """LT_C3: AutomationConfig has linux_tracing sub-config.""" + cfg = AutomationConfig() + assert cfg.linux_tracing.enabled is True + assert cfg.linux_tracing.proc_interval == 5.0 + assert cfg.linux_tracing.log_dir == "" + + def test_linux_tracing_config_fields(self): + """LT_C4: LinuxTracingConfig has exactly the expected fields.""" + names = {f.name for f in dataclasses.fields(LinuxTracingConfig)} + assert names == {"enabled", "proc_interval", "log_dir", "tmpfs_path", "max_sessions"} + + def test_linux_tracing_max_sessions_default(self): + """LT_C5: LinuxTracingConfig includes max_sessions with default 2000.""" + cfg = LinuxTracingConfig(tmpfs_path="/tmp/test") + assert cfg.max_sessions == 2000 + + def test_linux_tracing_max_sessions_yaml_roundtrip(self, tmp_path): + """LT_C6: max_sessions survives YAML round-trip.""" + (tmp_path / ".autoskillit").mkdir() + (tmp_path / ".autoskillit" / "config.yaml").write_text( + "linux_tracing:\n max_sessions: 750\n" + ) + cfg = load_config(tmp_path) + assert cfg.linux_tracing.max_sessions == 750 + + def test_automation_config_linux_tracing_has_max_sessions(self): + """LT_C3 extension: AutomationConfig().linux_tracing includes max_sessions.""" + cfg = AutomationConfig() + assert cfg.linux_tracing.max_sessions == 2000 diff --git a/tests/config/test_packs_config.py b/tests/config/test_packs_config.py new file mode 100644 index 0000000000..20be1fdbc7 --- /dev/null +++ b/tests/config/test_packs_config.py @@ -0,0 +1,54 @@ +"""Tests for PacksConfig loading and validation.""" + +from __future__ import annotations + +import pytest + +pytestmark = [pytest.mark.layer("config"), pytest.mark.small] + + +class TestPacksConfig: + # REQ-PACK-003: PacksConfig.enabled default + + def test_packs_config_default_enabled_is_empty_list(self) -> None: + from autoskillit.config import PacksConfig + + assert PacksConfig().enabled == [] + + def test_automation_config_has_packs_field(self) -> None: + from autoskillit.config import AutomationConfig, PacksConfig + + cfg = AutomationConfig() + assert isinstance(cfg.packs, PacksConfig) + assert cfg.packs.enabled == [] + + def test_load_config_packs_enabled(self, tmp_path) -> None: + from autoskillit.config import load_config + + config_dir = tmp_path / ".autoskillit" + config_dir.mkdir() + (config_dir / "config.yaml").write_text("packs:\n enabled:\n - research\n") + config = load_config(tmp_path) + assert config.packs.enabled == ["research"] + + def test_load_config_packs_enabled_absent_means_empty(self, tmp_path) -> None: + from autoskillit.config import load_config + + config_dir = tmp_path / ".autoskillit" + config_dir.mkdir() + (config_dir / "config.yaml").write_text("subsets:\n disabled: []\n") + config = load_config(tmp_path) + assert config.packs.enabled == [] + + def test_load_config_unknown_pack_in_packs_enabled_logs_warning(self, tmp_path) -> None: + import structlog.testing + + from autoskillit.config import load_config + + config_dir = tmp_path / ".autoskillit" + config_dir.mkdir() + (config_dir / "config.yaml").write_text("packs:\n enabled:\n - nonexistent-pack\n") + with structlog.testing.capture_logs() as cap_logs: + config = load_config(tmp_path) + assert config.packs.enabled == ["nonexistent-pack"] # preserved as-is + assert any("nonexistent-pack" in entry.get("event", "") for entry in cap_logs) diff --git a/tests/config/test_providers_config.py b/tests/config/test_providers_config.py new file mode 100644 index 0000000000..6b81d1fb29 --- /dev/null +++ b/tests/config/test_providers_config.py @@ -0,0 +1,145 @@ +"""Tests for ProvidersConfig loading and validation.""" + +from __future__ import annotations + +import pytest +import yaml + +pytestmark = [pytest.mark.layer("config"), pytest.mark.small] + + +class TestProvidersConfig: + def test_providers_config_importable_from_settings(self) -> None: + from autoskillit.config import ProvidersConfig + from autoskillit.config.settings import ProvidersConfig as PC + + assert PC is ProvidersConfig + + def test_providers_config_in_settings_all(self) -> None: + import autoskillit.config.settings as m + + assert "ProvidersConfig" in m.__all__ + + def test_automation_config_has_providers_field(self) -> None: + from dataclasses import fields as _dc_fields + + from autoskillit.config import AutomationConfig, ProvidersConfig + + field_names = [f.name for f in _dc_fields(AutomationConfig)] + assert "providers" in field_names + cfg = AutomationConfig() + assert isinstance(cfg.providers, ProvidersConfig) + + def test_providers_field_ordering(self) -> None: + from dataclasses import fields as _dc_fields + + from autoskillit.config import AutomationConfig + + field_names = [f.name for f in _dc_fields(AutomationConfig)] + fleet_idx = field_names.index("fleet") + providers_idx = field_names.index("providers") + features_idx = field_names.index("features") + assert fleet_idx < providers_idx < features_idx + + def test_from_dynaconf_providers_defaults(self, tmp_path) -> None: + from autoskillit.config import load_config + + cfg = load_config(tmp_path) + assert cfg.providers.default_provider is None + assert cfg.providers.profiles == {} + assert cfg.providers.step_overrides == {} + assert cfg.providers.provider_retry_limit == 2 + + def test_providers_config_defaults(self) -> None: + from autoskillit.config.settings import ProvidersConfig + + cfg = ProvidersConfig() + assert cfg.default_provider is None + assert cfg.profiles == {} + assert cfg.step_overrides == {} + assert cfg.provider_retry_limit == 2 + + def test_providers_config_is_mutable(self) -> None: + from autoskillit.config.settings import ProvidersConfig + + cfg = ProvidersConfig() + cfg.default_provider = "openai" + assert cfg.default_provider == "openai" + + def test_providers_config_field_types(self) -> None: + from dataclasses import fields as _dc_fields + + from autoskillit.config.settings import ProvidersConfig + + field_map = {f.name: f for f in _dc_fields(ProvidersConfig)} + assert set(field_map.keys()) == { + "default_provider", + "profiles", + "step_overrides", + "provider_retry_limit", + } + + def test_providers_config_retry_limit_zero_raises(self) -> None: + from autoskillit.config.settings import ProvidersConfig + + with pytest.raises(ValueError, match="provider_retry_limit must be >= 1"): + ProvidersConfig(provider_retry_limit=0) + + def test_providers_config_retry_limit_negative_raises(self) -> None: + from autoskillit.config.settings import ProvidersConfig + + with pytest.raises(ValueError, match="provider_retry_limit must be >= 1"): + ProvidersConfig(provider_retry_limit=-1) + + def test_providers_config_profiles_non_string_value_raises(self) -> None: + from autoskillit.config.settings import ProvidersConfig + + with pytest.raises(ValueError, match=r"profiles\[.+\]\[.+\] must be a string"): + ProvidersConfig(profiles={"my_profile": {"model": 42}}) # type: ignore[arg-type] + + +class TestProvidersConfigYaml: + def test_defaults_yaml_has_providers_section(self) -> None: + from autoskillit.core.io import load_yaml + from autoskillit.core.paths import pkg_root + + defaults = load_yaml(pkg_root() / "config" / "defaults.yaml") + assert "providers" in defaults, "defaults.yaml missing 'providers' section" + assert defaults["providers"]["provider_retry_limit"] == 2 + + def test_load_config_step_overrides_parsing(self, tmp_path) -> None: + from autoskillit.config import load_config + + config_dir = tmp_path / ".autoskillit" + config_dir.mkdir() + config_data = {"providers": {"step_overrides": {"fetch-data": "openai"}}} + (config_dir / "config.yaml").write_text(yaml.dump(config_data)) + cfg = load_config(tmp_path) + assert cfg.providers.step_overrides == {"fetch-data": "openai"} + + def test_load_config_profiles_dict_parsing(self, tmp_path) -> None: + from autoskillit.config import load_config + + config_dir = tmp_path / ".autoskillit" + config_dir.mkdir() + config_data = { + "providers": { + "profiles": { + "fast": {"model": "gpt-4o-mini", "api_base": "https://api.openai.com"}, + } + } + } + (config_dir / "config.yaml").write_text(yaml.dump(config_data)) + cfg = load_config(tmp_path) + assert cfg.providers.profiles == { + "fast": {"model": "gpt-4o-mini", "api_base": "https://api.openai.com"}, + } + + def test_load_config_provider_retry_limit_override(self, tmp_path) -> None: + from autoskillit.config import load_config + + config_dir = tmp_path / ".autoskillit" + config_dir.mkdir() + (config_dir / "config.yaml").write_text("providers:\n provider_retry_limit: 5\n") + cfg = load_config(tmp_path) + assert cfg.providers.provider_retry_limit == 5 diff --git a/tests/config/test_quota_guard_config.py b/tests/config/test_quota_guard_config.py new file mode 100644 index 0000000000..3f85205e18 --- /dev/null +++ b/tests/config/test_quota_guard_config.py @@ -0,0 +1,132 @@ +"""Tests for QuotaGuardConfig loading and validation.""" + +import dataclasses + +import pytest +import yaml + +from autoskillit.config import AutomationConfig, load_config +from autoskillit.config.settings import QuotaGuardConfig +from autoskillit.core.paths import pkg_root + +pytestmark = [pytest.mark.layer("config"), pytest.mark.small] + + +class TestQuotaGuardConfig: + def test_default_enabled(self): + config = AutomationConfig() + assert config.quota_guard.enabled is True + assert config.quota_guard.short_window_threshold == pytest.approx(85.0) + assert config.quota_guard.long_window_threshold == pytest.approx(95.0) + assert config.quota_guard.buffer_seconds == 60 + assert config.quota_guard.cache_max_age == 300 + + def test_load_quota_guard_from_yaml(self, tmp_path): + config_dir = tmp_path / ".autoskillit" + config_dir.mkdir() + (config_dir / "config.yaml").write_text( + yaml.dump( + { + "quota_guard": { + "enabled": True, + "short_window_threshold": 70.0, + "long_window_threshold": 80.0, + } + } + ) + ) + config = load_config(tmp_path) + assert config.quota_guard.enabled is True + assert config.quota_guard.short_window_threshold == pytest.approx(70.0) + assert config.quota_guard.long_window_threshold == pytest.approx(80.0) + # Unspecified fields keep defaults + assert config.quota_guard.buffer_seconds == 60 + + def test_short_window_threshold_defaults_to_85(self): + assert QuotaGuardConfig().short_window_threshold == 85.0 + + def test_long_window_threshold_defaults_to_95(self): + assert QuotaGuardConfig().long_window_threshold == 95.0 + + def test_long_window_patterns_default(self): + defaults = QuotaGuardConfig().long_window_patterns + assert "seven_day" in defaults, ( + f"Default long_window_patterns {defaults!r} must include 'seven_day' " + "(the actual Anthropic API key for the 7-day budget)" + ) + + def test_threshold_field_removed(self): + names = {f.name for f in dataclasses.fields(QuotaGuardConfig)} + assert "threshold" not in names + + def test_quota_guard_yaml_round_trip_per_window(self, tmp_path): + config_dir = tmp_path / ".autoskillit" + config_dir.mkdir() + (config_dir / "config.yaml").write_text( + yaml.dump( + { + "quota_guard": { + "short_window_threshold": 85.0, + "long_window_threshold": 95.0, + "long_window_patterns": ["weekly", "sonnet", "opus"], + } + } + ) + ) + config = load_config(tmp_path) + assert config.quota_guard.short_window_threshold == pytest.approx(85.0) + assert config.quota_guard.long_window_threshold == pytest.approx(95.0) + assert config.quota_guard.long_window_patterns == ["weekly", "sonnet", "opus"] + + def test_quota_guard_config_has_cache_refresh_interval(self): + """QG_C3: QuotaGuardConfig has cache_refresh_interval defaulting to 240.""" + config = QuotaGuardConfig() + assert config.cache_refresh_interval == 240 + + def test_defaults_yaml_has_cache_refresh_interval(self): + """QG_C4: defaults.yaml defines quota_guard.cache_refresh_interval < cache_max_age.""" + defaults = yaml.safe_load((pkg_root() / "config" / "defaults.yaml").read_text()) + assert "quota_guard" in defaults, ( + f"Missing 'quota_guard' key in defaults.yaml: {list(defaults.keys())}" + ) + assert "cache_refresh_interval" in defaults["quota_guard"], ( + "Missing 'cache_refresh_interval' in defaults.yaml['quota_guard']" + ) + assert "cache_max_age" in defaults["quota_guard"], ( + "Missing 'cache_max_age' in defaults.yaml['quota_guard']" + ) + interval = defaults["quota_guard"]["cache_refresh_interval"] + max_age = defaults["quota_guard"]["cache_max_age"] + assert interval < max_age, ( + f"cache_refresh_interval ({interval}) must be < cache_max_age ({max_age}); " + "otherwise the loop arrives after the cache has already expired" + ) + + def test_quota_guard_per_window_enabled_defaults_true(self): + """QG_C5: QuotaGuardConfig() defaults both per-window flags to True.""" + config = QuotaGuardConfig() + assert config.short_window_enabled is True + assert config.long_window_enabled is True + + def test_quota_guard_per_window_enabled_yaml_round_trip(self, tmp_path): + """QG_C6: per-window enabled flags survive a YAML round-trip.""" + config_dir = tmp_path / ".autoskillit" + config_dir.mkdir() + (config_dir / "config.yaml").write_text( + yaml.dump({"quota_guard": {"short_window_enabled": False}}) + ) + config = load_config(tmp_path) + assert config.quota_guard.short_window_enabled is False + assert config.quota_guard.long_window_enabled is True + + def test_defaults_yaml_has_per_window_enabled_keys(self): + """QG_C7: defaults.yaml has both per-window enabled keys set to True.""" + defaults = yaml.safe_load((pkg_root() / "config" / "defaults.yaml").read_text()) + assert defaults["quota_guard"]["short_window_enabled"] is True + assert defaults["quota_guard"]["long_window_enabled"] is True + + def test_quota_guard_env_var_override_short_window_enabled(self, monkeypatch, tmp_path): + """QG_C8: AUTOSKILLIT_QUOTA_GUARD__SHORT_WINDOW_ENABLED=false is cast to bool False.""" + monkeypatch.setenv("AUTOSKILLIT_QUOTA_GUARD__SHORT_WINDOW_ENABLED", "false") + config = load_config(tmp_path) + assert config.quota_guard.short_window_enabled is False diff --git a/tests/config/test_review_config.py b/tests/config/test_review_config.py new file mode 100644 index 0000000000..35744d0c91 --- /dev/null +++ b/tests/config/test_review_config.py @@ -0,0 +1,47 @@ +"""Tests for ReviewConfig and local_review_rounds wiring.""" + +from pathlib import Path + +import pytest +import yaml + +import autoskillit.config +from autoskillit.config import AutomationConfig, load_config + +pytestmark = [pytest.mark.layer("config"), pytest.mark.small] + + +class TestReviewConfigDefaults: + def test_review_config_defaults(self) -> None: + """T1.1: AutomationConfig().review.local_review_rounds == 3.""" + cfg = AutomationConfig() + assert cfg.review.local_review_rounds == 3 + + +class TestReviewConfigFromYaml: + def test_review_config_from_yaml(self, tmp_path) -> None: + """T1.2: YAML with review.local_review_rounds: 5 loads correctly.""" + config_dir = tmp_path / ".autoskillit" + config_dir.mkdir() + (config_dir / "config.yaml").write_text("review:\n local_review_rounds: 5\n") + cfg = load_config(tmp_path) + assert cfg.review.local_review_rounds == 5 + + def test_review_config_env_var_override(self, tmp_path, monkeypatch) -> None: + """T1.3: AUTOSKILLIT_REVIEW__LOCAL_REVIEW_ROUNDS env var overrides default.""" + config_dir = tmp_path / ".autoskillit" + config_dir.mkdir() + (config_dir / "config.yaml").write_text("review:\n local_review_rounds: 5\n") + monkeypatch.setenv("AUTOSKILLIT_REVIEW__LOCAL_REVIEW_ROUNDS", "0") + cfg = load_config(tmp_path) + assert cfg.review.local_review_rounds == 0 + + +class TestReviewConfigDefaultsYaml: + def test_review_config_defaults_yaml_coherence(self) -> None: + """T1.4: defaults.yaml review.local_review_rounds == 3.""" + pkg_root = Path(autoskillit.config.__file__).parent + defaults_file = pkg_root / "defaults.yaml" + with open(defaults_file) as f: + data = yaml.safe_load(f) + assert data["review"]["local_review_rounds"] == 3 diff --git a/tests/config/test_settings_allowed_labels.py b/tests/config/test_settings_allowed_labels.py index 9cf28db5e1..0ee494de9d 100644 --- a/tests/config/test_settings_allowed_labels.py +++ b/tests/config/test_settings_allowed_labels.py @@ -1,7 +1,12 @@ """Tests for GitHubConfig.allowed_labels field and check_label_allowed validation.""" +import pytest + from autoskillit.config import AutomationConfig -from autoskillit.config.settings import GitHubConfig, _make_dynaconf +from autoskillit.config._config_loader import _make_dynaconf +from autoskillit.config.settings import GitHubConfig + +pytestmark = [pytest.mark.layer("config"), pytest.mark.small] class TestGitHubConfigAllowedLabelsField: @@ -74,6 +79,7 @@ def test_load_config_from_defaults_includes_standard_labels(self, tmp_path): expected = { "bug", "enhancement", + "queued", "in-progress", "staged", "autoreported", @@ -81,3 +87,10 @@ def test_load_config_from_defaults_includes_standard_labels(self, tmp_path): "recipe:remediation", } assert expected.issubset(set(cfg.github.allowed_labels)) + + def test_fail_label_in_allowed_labels_default(self, tmp_path): + """defaults.yaml whitelist includes 'fail' label for CI failure marking.""" + from autoskillit.config import load_config + + cfg = load_config(tmp_path) + assert "fail" in cfg.github.allowed_labels diff --git a/tests/config/test_settings_queued_label.py b/tests/config/test_settings_queued_label.py new file mode 100644 index 0000000000..99d49f7377 --- /dev/null +++ b/tests/config/test_settings_queued_label.py @@ -0,0 +1,61 @@ +"""Tests for GitHubConfig.queued_label field and label-state mapping methods.""" + +from __future__ import annotations + +import pytest + +from autoskillit.config import AutomationConfig +from autoskillit.config._config_loader import _make_dynaconf +from autoskillit.config.settings import GitHubConfig +from autoskillit.core.types import IssueLabelState + +pytestmark = [pytest.mark.layer("config"), pytest.mark.small] + + +class TestQueuedLabelField: + def test_default_is_queued(self): + """GitHubConfig has queued_label field with default 'queued'.""" + cfg = GitHubConfig() + assert cfg.queued_label == "queued" + + def test_loads_from_env_var(self, monkeypatch): + """queued_label is loaded from AUTOSKILLIT_GITHUB__QUEUED_LABEL env var.""" + monkeypatch.setenv("AUTOSKILLIT_GITHUB__QUEUED_LABEL", "waiting") + d = _make_dynaconf() + cfg = AutomationConfig.from_dynaconf(d) + assert cfg.github.queued_label == "waiting" + + +class TestLabelStateMapping: + def test_label_for_state_returns_configured_string(self): + """label_for_state returns the configured label string for each IssueLabelState.""" + cfg = GitHubConfig() + assert cfg.label_for_state(IssueLabelState.QUEUED) == "queued" + assert cfg.label_for_state(IssueLabelState.IN_PROGRESS) == "in-progress" + assert cfg.label_for_state(IssueLabelState.STAGED) == "staged" + assert cfg.label_for_state(IssueLabelState.FAIL) == "fail" + + def test_label_for_state_respects_custom_config(self): + """label_for_state uses per-field overrides when set.""" + cfg = GitHubConfig(queued_label="waiting") + assert cfg.label_for_state(IssueLabelState.QUEUED) == "waiting" + + def test_state_for_label_roundtrip(self): + """state_for_label(label_for_state(state)) == state for all states.""" + cfg = GitHubConfig() + for state in IssueLabelState: + label = cfg.label_for_state(state) + assert cfg.state_for_label(label) == state + + def test_state_for_label_returns_none_for_classification_labels(self): + """state_for_label returns None for non-lifecycle labels.""" + cfg = GitHubConfig() + assert cfg.state_for_label("bug") is None + assert cfg.state_for_label("recipe:implementation") is None + + def test_label_for_state_covers_all_members(self): + """label_for_state returns a non-empty string for every IssueLabelState.""" + cfg = GitHubConfig() + for state in IssueLabelState: + label = cfg.label_for_state(state) + assert label diff --git a/tests/config/test_settings_staged_label.py b/tests/config/test_settings_staged_label.py index 2d80a04c44..60feb21d2d 100644 --- a/tests/config/test_settings_staged_label.py +++ b/tests/config/test_settings_staged_label.py @@ -2,8 +2,13 @@ from __future__ import annotations +import pytest + from autoskillit.config import AutomationConfig -from autoskillit.config.settings import GitHubConfig, _make_dynaconf +from autoskillit.config._config_loader import _make_dynaconf +from autoskillit.config.settings import GitHubConfig + +pytestmark = [pytest.mark.layer("config"), pytest.mark.small] class TestGitHubConfigStagedLabel: diff --git a/tests/config/test_skills_config.py b/tests/config/test_skills_config.py new file mode 100644 index 0000000000..e452db3e5a --- /dev/null +++ b/tests/config/test_skills_config.py @@ -0,0 +1,61 @@ +"""Tests for SkillsConfig loading and validation.""" + +from __future__ import annotations + +import pytest + +pytestmark = [pytest.mark.layer("config"), pytest.mark.small] + + +class TestSkillsConfig: + """SkillsConfig dataclass, tier duplication validation, and AutomationConfig integration.""" + + def test_skills_config_dataclass(self) -> None: + """SkillsConfig has tier1, tier2, tier3 list[str] fields.""" + from autoskillit.config.settings import SkillsConfig + + sc = SkillsConfig(tier1=["a"], tier2=["b"], tier3=["c"]) + assert sc.tier1 == ["a"] and sc.tier2 == ["b"] and sc.tier3 == ["c"] + + def test_skills_config_tier_duplication_raises(self) -> None: + """Skill in multiple tiers raises ValueError at construction (REQ-TIER-009).""" + from autoskillit.config.settings import SkillsConfig + + with pytest.raises(ValueError, match="multiple tiers"): + SkillsConfig(tier1=["open-kitchen"], tier2=["open-kitchen"], tier3=[]) + + def test_automation_config_has_skills_field(self) -> None: + """AutomationConfig.skills is a SkillsConfig with tier lists.""" + from autoskillit.config.settings import AutomationConfig, SkillsConfig + + cfg = AutomationConfig() + assert isinstance(cfg.skills, SkillsConfig) + + def test_defaults_yaml_skills_section(self) -> None: + """defaults.yaml has non-empty skills.tier1/tier2/tier3 lists.""" + from autoskillit.core import load_yaml, pkg_root + + defaults = load_yaml(pkg_root() / "config" / "defaults.yaml") + assert isinstance(defaults, dict) + skills = defaults.get("skills", {}) + assert "open-kitchen" in skills.get("tier1", []) + assert len(skills.get("tier2", [])) >= 20 + assert len(skills.get("tier3", [])) >= 10 + + def test_load_config_populates_skills_tiers(self, tmp_path) -> None: + """load_config() produces an AutomationConfig with tier assignments from defaults.""" + from autoskillit.config import load_config + + cfg = load_config(tmp_path) + assert "open-kitchen" in cfg.skills.tier1 + assert "make-plan" in cfg.skills.tier2 + assert "compose-pr" in cfg.skills.tier3 + + def test_skills_config_exported_from_config_package(self) -> None: + """SkillsConfig is importable from autoskillit.config and has expected fields.""" + from autoskillit.config import SkillsConfig + + cfg = SkillsConfig() + assert hasattr(cfg, "tier1") + assert hasattr(cfg, "tier2") + assert hasattr(cfg, "tier3") diff --git a/tests/config/test_subsets_config.py b/tests/config/test_subsets_config.py new file mode 100644 index 0000000000..b9007e729b --- /dev/null +++ b/tests/config/test_subsets_config.py @@ -0,0 +1,145 @@ +"""Tests for SubsetsConfig loading and validation.""" + +from __future__ import annotations + +import pytest + +from autoskillit.config import load_config + +pytestmark = [pytest.mark.layer("config"), pytest.mark.small] + + +class TestBuildSubsetsConfigCustomTagsValidation: + """CC-F2: _build_subsets_config must raise ValueError for non-dict custom_tags.""" + + @pytest.mark.parametrize("bad_value", [["list", "not", "dict"], "oops", 42]) + def test_build_subsets_config_raises_for_non_dict_custom_tags(self, bad_value: object) -> None: + """Non-dict custom_tags must raise ValueError, not silently coerce to {}.""" + from autoskillit.config.settings import _build_subsets_config + + with pytest.raises(ValueError, match="custom_tags"): + _build_subsets_config({"custom_tags": bad_value}) + + def test_build_subsets_config_dict_custom_tags_accepted(self): + """Valid dict custom_tags must not raise.""" + from autoskillit.config.settings import _build_subsets_config + + result = _build_subsets_config({"custom_tags": {"my_tag": ["skill-a"]}}) + assert result.custom_tags == {"my_tag": ["skill-a"]} + + def test_build_subsets_config_empty_dict_custom_tags_accepted(self): + """Empty dict custom_tags must not raise.""" + from autoskillit.config.settings import _build_subsets_config + + result = _build_subsets_config({"custom_tags": {}}) + assert result.custom_tags == {} + + +class TestSubsetsConfig: + # T1 — SubsetsConfig defaults + + def test_subsets_config_default_disabled_is_empty_list(self) -> None: + from autoskillit.config import SubsetsConfig + + cfg = SubsetsConfig() + assert cfg.disabled == [] + + def test_subsets_config_default_custom_tags_is_empty_dict(self) -> None: + from autoskillit.config import SubsetsConfig + + cfg = SubsetsConfig() + assert cfg.custom_tags == {} + + def test_automation_config_has_subsets_field(self) -> None: + from autoskillit.config import AutomationConfig, SubsetsConfig + + cfg = AutomationConfig() + assert isinstance(cfg.subsets, SubsetsConfig) + assert cfg.subsets.disabled == [] + assert cfg.subsets.custom_tags == {} + + # T2 — load_config with subsets.disabled + + def test_load_config_subsets_disabled(self, tmp_path) -> None: + config_dir = tmp_path / ".autoskillit" + config_dir.mkdir() + (config_dir / "config.yaml").write_text("subsets:\n disabled:\n - github\n - ci\n") + cfg = load_config(tmp_path) + assert cfg.subsets.disabled == ["github", "ci"] + + def test_load_config_subsets_disabled_absent_means_empty(self, tmp_path) -> None: + cfg = load_config(tmp_path) + assert cfg.subsets.disabled == [] + + # T3 — load_config with subsets.custom_tags + + def test_load_config_subsets_custom_tags(self, tmp_path) -> None: + config_dir = tmp_path / ".autoskillit" + config_dir.mkdir() + yaml_text = ( + "subsets:\n" + " custom_tags:\n" + " my-team-tools:\n" + " - investigate\n" + " - make-plan\n" + ) + (config_dir / "config.yaml").write_text(yaml_text) + cfg = load_config(tmp_path) + assert cfg.subsets.custom_tags == {"my-team-tools": ["investigate", "make-plan"]} + + def test_load_config_subsets_custom_tags_absent_means_empty(self, tmp_path) -> None: + cfg = load_config(tmp_path) + assert cfg.subsets.custom_tags == {} + + # T4 — Unknown category warning, no crash + + def test_load_config_unknown_disabled_category_logs_warning_not_crash(self, tmp_path) -> None: + import structlog.testing + + config_dir = tmp_path / ".autoskillit" + config_dir.mkdir() + (config_dir / "config.yaml").write_text( + "subsets:\n disabled:\n - totally-unknown-category\n" + ) + with structlog.testing.capture_logs() as cap_logs: + cfg = load_config(tmp_path) + assert cfg.subsets.disabled == ["totally-unknown-category"] # preserved as-is + assert any("totally-unknown-category" in entry.get("event", "") for entry in cap_logs) + + def test_load_config_known_disabled_category_no_warning(self, tmp_path, caplog) -> None: + import logging + + config_dir = tmp_path / ".autoskillit" + config_dir.mkdir() + (config_dir / "config.yaml").write_text("subsets:\n disabled:\n - github\n") + with caplog.at_level(logging.WARNING): + load_config(tmp_path) + assert not any("github" in r.message for r in caplog.records) + + def test_load_config_custom_tag_in_disabled_is_valid(self, tmp_path, caplog) -> None: + """Custom tags defined in custom_tags can also appear in disabled.""" + import logging + + config_dir = tmp_path / ".autoskillit" + config_dir.mkdir() + yaml_text = ( + "subsets:\n" + " disabled:\n" + " - experimental\n" + " custom_tags:\n" + " experimental:\n" + " - write-recipe\n" + ) + (config_dir / "config.yaml").write_text(yaml_text) + with caplog.at_level(logging.WARNING): + load_config(tmp_path) + assert not any("experimental" in r.message for r in caplog.records) + + # T5 — SubsetsConfig exported from config package + + def test_subsets_config_importable_from_config_package(self) -> None: + from autoskillit.config import SubsetsConfig + + cfg = SubsetsConfig() + assert hasattr(cfg, "disabled") + assert hasattr(cfg, "custom_tags") diff --git a/tests/config/test_timeout_coherence.py b/tests/config/test_timeout_coherence.py new file mode 100644 index 0000000000..91befb3138 --- /dev/null +++ b/tests/config/test_timeout_coherence.py @@ -0,0 +1,59 @@ +"""Tests for idle_output_timeout config coherence validation.""" + +import pytest +import structlog.testing + +from autoskillit.config import load_config + +pytestmark = [pytest.mark.layer("config"), pytest.mark.small] + + +class TestTimeoutCoherenceGate: + """Tests for _timeout_coherence_gate warning behavior.""" + + def test_idle_output_timeout_less_than_tool_max_emits_warning(self, tmp_path): + """idle_output_timeout < known tool max triggers coherence warning.""" + config_dir = tmp_path / ".autoskillit" + config_dir.mkdir() + # idle_output_timeout=600, but wait_for_merge_queue recipe override is 900s + (config_dir / "config.yaml").write_text("run_skill:\n idle_output_timeout: 600\n") + with structlog.testing.capture_logs() as cap_logs: + cfg = load_config(tmp_path) + assert cfg.run_skill.idle_output_timeout == 600 + assert any("idle_output_timeout_coherence" in entry.get("event", "") for entry in cap_logs) + + def test_idle_output_timeout_zero_skips_coherence_check(self, tmp_path): + """Disabled watchdog (0) passes coherence check unconditionally.""" + config_dir = tmp_path / ".autoskillit" + config_dir.mkdir() + (config_dir / "config.yaml").write_text("run_skill:\n idle_output_timeout: 0\n") + with structlog.testing.capture_logs() as cap_logs: + cfg = load_config(tmp_path) + assert cfg.run_skill.idle_output_timeout == 0 + assert not any( + "idle_output_timeout_coherence" in entry.get("event", "") for entry in cap_logs + ) + + def test_coherence_gate_warns_on_matched_defaults(self, tmp_path): + """idle_output_timeout == tool_timeout is incoherent (race condition).""" + config_dir = tmp_path / ".autoskillit" + config_dir.mkdir() + # idle_output_timeout=600, wait_for_merge_queue default=600 — exact match + (config_dir / "config.yaml").write_text("run_skill:\n idle_output_timeout: 600\n") + with structlog.testing.capture_logs() as cap_logs: + cfg = load_config(tmp_path) + assert cfg.run_skill.idle_output_timeout == 600 + assert any("idle_output_timeout_coherence" in entry.get("event", "") for entry in cap_logs) + + def test_coherence_gate_passes_when_idle_exceeds_tool_max(self, tmp_path): + """idle_output_timeout > all known tool timeouts passes cleanly.""" + config_dir = tmp_path / ".autoskillit" + config_dir.mkdir() + # idle_output_timeout=1000 > _MERGE_QUEUE_RECIPE_MAX (900) + (config_dir / "config.yaml").write_text("run_skill:\n idle_output_timeout: 1000\n") + with structlog.testing.capture_logs() as cap_logs: + cfg = load_config(tmp_path) + assert cfg.run_skill.idle_output_timeout == 1000 + assert not any( + "idle_output_timeout_coherence" in entry.get("event", "") for entry in cap_logs + ) diff --git a/tests/config/test_workspace_temp_dir_config.py b/tests/config/test_workspace_temp_dir_config.py index 60289c960e..68c739eb19 100644 --- a/tests/config/test_workspace_temp_dir_config.py +++ b/tests/config/test_workspace_temp_dir_config.py @@ -8,6 +8,8 @@ from autoskillit.config import WorkspaceConfig, load_config +pytestmark = [pytest.mark.layer("config"), pytest.mark.small] + def test_workspace_config_has_temp_dir_field() -> None: assert WorkspaceConfig().temp_dir is None diff --git a/tests/config/test_write_config_layer.py b/tests/config/test_write_config_layer.py new file mode 100644 index 0000000000..0dd4b8a51e --- /dev/null +++ b/tests/config/test_write_config_layer.py @@ -0,0 +1,50 @@ +"""Tests for write_config_layer atomic write and validation.""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +pytestmark = [pytest.mark.layer("config"), pytest.mark.small] + + +class TestWriteConfigLayer: + def test_write_config_layer_rejects_secret_key(self, tmp_path: Path) -> None: + """write_config_layer raises ConfigSchemaError before touching the file.""" + + from autoskillit.config.settings import ConfigSchemaError, write_config_layer + + config_path = tmp_path / "config.yaml" + with pytest.raises(ConfigSchemaError, match="github.token"): + write_config_layer(config_path, {"github": {"token": "ghp_test"}}) + assert not config_path.exists(), "config.yaml must not be written on schema error" + + def test_write_config_layer_rejects_unknown_key(self, tmp_path: Path) -> None: + """write_config_layer raises ConfigSchemaError for unknown section.""" + from autoskillit.config.settings import ConfigSchemaError, write_config_layer + + config_path = tmp_path / "config.yaml" + with pytest.raises(ConfigSchemaError, match="unrecognized key"): + write_config_layer(config_path, {"invented_section": {"foo": "bar"}}) + assert not config_path.exists() + + def test_write_config_layer_writes_valid_content(self, tmp_path: Path) -> None: + """write_config_layer writes valid schema content atomically.""" + import yaml as _yaml + + from autoskillit.config.settings import write_config_layer + + config_path = tmp_path / "config.yaml" + write_config_layer(config_path, {"github": {"default_repo": "owner/repo"}}) + assert config_path.is_file() + data = _yaml.safe_load(config_path.read_text()) + assert data["github"]["default_repo"] == "owner/repo" + + def test_write_config_layer_accepts_packs_enabled(self, tmp_path: Path) -> None: + """write_config_layer does not raise for valid packs.enabled.""" + from autoskillit.config.settings import write_config_layer + + config_path = tmp_path / "config.yaml" + write_config_layer(config_path, {"packs": {"enabled": ["research"]}}) + assert config_path.exists() diff --git a/tests/conftest.py b/tests/conftest.py index df37dff378..1a0d55581b 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,86 +1,75 @@ """Shared test fixtures for autoskillit.""" -import sys +import functools +import os from pathlib import Path as _Path +from typing import TYPE_CHECKING import pytest +if TYPE_CHECKING: + from autoskillit.config.settings import AutomationConfig + from autoskillit.core.types import ( ChannelConfirmation, SubprocessResult, - SubprocessRunner, TerminationReason, - TestResult, ) from tests._helpers import _flush_structlog_proxy_caches +_LAYER_DIRS: frozenset[str] = frozenset( + { + "core", + "config", + "pipeline", + "execution", + "workspace", + "recipe", + "migration", + "fleet", + "planner", + "server", + "cli", + } +) -class StatefulMockTester: - """Test double for TestRunner returning pre-configured results on successive calls. - - Enables the scenario: pre-rebase tests pass, post-rebase tests fail. - Falls back to TestResult(passed=True, stdout="", stderr="") for any call beyond the - configured list. - """ - - def __init__(self, results: list[TestResult]) -> None: - self._results = list(results) - self._index = 0 - - async def run(self, cwd: _Path) -> TestResult: - if self._index < len(self._results): - result = self._results[self._index] - else: - result = TestResult(passed=True, stdout="", stderr="") - self._index += 1 - return result +_SIZE_DIRS: frozenset[str] = frozenset( + { + "cli", + "config", + "core", + "execution", + "fleet", + "migration", + "pipeline", + "planner", + "recipe", + "server", + "workspace", + } +) - @property - def call_count(self) -> int: - return self._index +_scope_key = pytest.StashKey[set[_Path] | None]() +_filter_mode_key = pytest.StashKey[str | None]() +_selected_count_key = pytest.StashKey[int | None]() +_deselected_count_key = pytest.StashKey[int | None]() +_full_run_reason_key = pytest.StashKey[str | None]() +# Module-level accumulator for xdist worker-to-controller IPC. +# Populated by pytest_testnodedown (controller); cleared by pytest_configure +# at session start so in-process pytester reruns don't leak stale data. +_worker_filter_counts: dict[str, int | None] = {} -class MockSubprocessRunner(SubprocessRunner): - """Test double for SubprocessRunner. Queues predetermined results. - Inherits from SubprocessRunner (Protocol) so mypy verifies the __call__ - signature matches the protocol at class definition, not just at call sites. +class TimeoutTier: + """Centralized timeout tiers encoding xdist -n 4 budget math. - call_args_list stores (cmd, cwd, timeout, kwargs) tuples. - IMPORTANT: Assert [N][1] (cwd) when testing cwd propagation. + CHANNEL_B minimum: 1s preamble + _phase1_timeout (30s) + drain + jitter > 31.5s. """ - def __init__(self) -> None: - self._queue: list[SubprocessResult] = [] - self._default = SubprocessResult( - returncode=0, - stdout="", - stderr="", - termination=TerminationReason.NATURAL_EXIT, - pid=99999, - ) - self.call_args_list: list[tuple] = [] - - def push(self, result: SubprocessResult) -> None: - """Queue a result to be returned by the next __call__.""" - self._queue.append(result) - - def set_default(self, result: SubprocessResult) -> None: - """Set the result returned when the queue is empty.""" - self._default = result - - async def __call__( - self, - cmd: list[str], - *, - cwd: _Path, - timeout: float, - **kwargs: object, - ) -> SubprocessResult: - self.call_args_list.append((cmd, cwd, timeout, kwargs)) - if self._queue: - return self._queue.pop(0) - return self._default + UNIT = 10 # Pure logic, no I/O + INTEGRATION = 30 # Filesystem/subprocess, no Channel B + CHANNEL_B = 60 # Full session_log_dir + Channel B path @pytest.fixture(autouse=True) @@ -213,15 +202,21 @@ def _clear_headless_env(monkeypatch): """Ensure AUTOSKILLIT_HEADLESS is unset at the start of every test. Tools check this env var to block calls from headless sessions. - Also resets mcp kitchen visibility transforms when the server module is - already imported. + MCP tag resets are handled by tests/server/conftest.py for server tests. """ - monkeypatch.delenv("AUTOSKILLIT_HEADLESS", raising=False) - if "autoskillit.server" in sys.modules: - from autoskillit.server import mcp - mcp.disable(tags={"kitchen"}) + +@pytest.fixture(autouse=True) +def _clear_session_type_env(monkeypatch): + """Prevent SESSION_TYPE leaking between tests.""" + monkeypatch.delenv("AUTOSKILLIT_SESSION_TYPE", raising=False) + + +@pytest.fixture(autouse=True) +def _clear_skill_name_env(monkeypatch): + """Prevent AUTOSKILLIT_SKILL_NAME leaking between tests.""" + monkeypatch.delenv("AUTOSKILLIT_SKILL_NAME", raising=False) @pytest.fixture(autouse=True) @@ -229,31 +224,92 @@ def _clear_skip_stale_check_env(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.delenv("AUTOSKILLIT_SKIP_STALE_CHECK", raising=False) +@pytest.fixture(autouse=True) +def _reset_mcp_visibility(): + import sys + + def _clear_transforms(): + from autoskillit.core import ALL_VISIBILITY_TAGS + from autoskillit.server import mcp + + mcp._transforms.clear() + for tag in sorted(ALL_VISIBILITY_TAGS): + mcp.disable(tags={tag}) + + if "autoskillit.server" in sys.modules: + _clear_transforms() + yield + if "autoskillit.server" in sys.modules: + _clear_transforms() + + @pytest.fixture(scope="function") def anyio_backend(): """Lock all @pytest.mark.anyio tests to the asyncio backend.""" return "asyncio" +@pytest.fixture +def minimal_ctx(tmp_path): + """Lightweight ToolContext using only L0+L1 imports (core, pipeline, config). + + Use for tests that only need gate, audit, token_log, timing_log, or config — + no server factory, no L2/L3 service wiring. Importing this fixture does NOT + pull in autoskillit.server, autoskillit.execution, autoskillit.recipe, + autoskillit.migration, or autoskillit.workspace. + + Tests that need full service wiring (executor, tester, recipes, etc.) should + use tool_ctx instead. + """ + from autoskillit.config import AutomationConfig + from autoskillit.core.types._type_plugin_source import DirectInstall + from autoskillit.pipeline.audit import DefaultAuditLog + from autoskillit.pipeline.context import ToolContext + from autoskillit.pipeline.gate import DefaultGateState + from autoskillit.pipeline.timings import DefaultTimingLog + from autoskillit.pipeline.tokens import DefaultTokenLog + + ctx = ToolContext( + config=AutomationConfig(features={"fleet": True}), + audit=DefaultAuditLog(), + token_log=DefaultTokenLog(), + timing_log=DefaultTimingLog(), + gate=DefaultGateState(enabled=False), + plugin_source=DirectInstall(plugin_dir=tmp_path), + runner=None, + temp_dir=tmp_path / ".autoskillit" / "temp", + project_dir=tmp_path, + ) + return ctx + + @pytest.fixture def tool_ctx(monkeypatch, tmp_path): - """Provide a fully isolated ToolContext for server tests. + """Provide a fully isolated ToolContext for server integration tests. + + Full-stack fixture: calls make_context() from server/_factory.py, which + imports ALL production layers (L0–L3). Use minimal_ctx instead when the + test only needs gate, audit, token_log, timing_log, or config fields. Monkeypatches server._ctx so all server tool calls use this context. - Gate is enabled (open kitchen) by default — tests that need a closed - gate should do: tool_ctx.gate = DefaultGateState(enabled=False) locally. + Gate starts closed (matching production) — use tool_ctx_kitchen_open + when a test needs the gate open. All service fields (executor, tester, db_reader, workspace_mgr, recipes, migrations) are wired via make_context() so routing tests work correctly. """ from autoskillit.config import AutomationConfig - from autoskillit.pipeline.gate import DefaultGateState + from autoskillit.core.types._type_plugin_source import DirectInstall from autoskillit.server import _state from autoskillit.server._factory import make_context + from tests.fakes import MockSubprocessRunner mock_runner = MockSubprocessRunner() - ctx = make_context(AutomationConfig(), runner=mock_runner, plugin_dir=str(tmp_path)) - ctx.gate = DefaultGateState(enabled=True) + ctx = make_context( + AutomationConfig(features={"fleet": True}), + runner=mock_runner, + plugin_source=DirectInstall(plugin_dir=tmp_path), + ) ctx.config.linux_tracing.log_dir = str(tmp_path / "session_logs") ctx.config.linux_tracing.tmpfs_path = str(tmp_path / "shm") # Anchor temp_dir to tmp_path so server tools that read from ctx.temp_dir @@ -263,3 +319,415 @@ def tool_ctx(monkeypatch, tmp_path): monkeypatch.setattr(_state, "_ctx", ctx) monkeypatch.setattr(_state, "_startup_ready", None) return ctx + + +@pytest.fixture +def tool_ctx_marketplace(monkeypatch, tmp_path): + """ToolContext simulating a marketplace install: plugin_source = MarketplaceInstall.""" + from autoskillit.config import AutomationConfig + from autoskillit.core.types._type_plugin_source import MarketplaceInstall + from autoskillit.pipeline.gate import DefaultGateState + from autoskillit.server import _state + from autoskillit.server._factory import make_context + from tests.fakes import MockSubprocessRunner + + fake_cache = tmp_path / "marketplace_cache" + fake_cache.mkdir() + + mock_runner = MockSubprocessRunner() + ctx = make_context( + AutomationConfig(features={"fleet": True}), + runner=mock_runner, + plugin_source=MarketplaceInstall(cache_path=fake_cache), + ) + ctx.gate = DefaultGateState(enabled=False) + ctx.config.linux_tracing.log_dir = str(tmp_path / "session_logs") + ctx.config.linux_tracing.tmpfs_path = str(tmp_path / "shm") + ctx.temp_dir = tmp_path / ".autoskillit" / "temp" + monkeypatch.setattr(_state, "_ctx", ctx) + monkeypatch.setattr(_state, "_startup_ready", None) + return ctx + + +@pytest.fixture +def tool_ctx_kitchen_open(tool_ctx): + """tool_ctx variant with gate explicitly opened. + + Use when the test requires a tool that calls _require_enabled() and + the test is not testing gate-boot behavior itself. This fixture + mirrors the post-lifespan-boot state for interactive sessions. + """ + from autoskillit.pipeline.gate import DefaultGateState + + tool_ctx.gate = DefaultGateState(enabled=True) + return tool_ctx + + +# --------------------------------------------------------------------------- +# Test filter hooks (opt-in via AUTOSKILLIT_TEST_FILTER env var) +# --------------------------------------------------------------------------- + + +def pytest_addoption(parser: pytest.Parser) -> None: + parser.addoption( + "--filter-mode", + default=None, + choices=("none", "conservative", "aggressive"), + help="Test filter mode (overrides AUTOSKILLIT_TEST_FILTER env var).", + ) + parser.addoption( + "--filter-base-ref", + default=None, + help="Git base ref for changed-file detection (overrides AUTOSKILLIT_TEST_BASE_REF).", + ) + + +def pytest_configure(config: pytest.Config) -> None: + """Compute test filter scope from env var + git diff + manifest. + + Opt-in via AUTOSKILLIT_TEST_FILTER env var or --filter-mode CLI flag. + Fail-open: any error sets scope to None (full test run). + """ + import warnings + + # Reset xdist IPC accumulator so in-process pytester reruns don't leak counts. + _worker_filter_counts.clear() + + config.stash[_scope_key] = None + config.stash[_filter_mode_key] = None + config.stash[_full_run_reason_key] = None + + cli_mode = config.getoption("--filter-mode", default=None) + env_val = os.environ.get("AUTOSKILLIT_TEST_FILTER", "") + + if not cli_mode and not env_val: + return + if not cli_mode and env_val.lower() in ("0", "false", "no"): + return + + try: + from tests._test_filter import ( + FilterMode, + FullRunReason, + build_test_scope, + git_changed_files, + load_manifest, + ) + + if cli_mode: + mode = FilterMode(cli_mode) + elif env_val.lower() in ("1", "true", "yes"): + mode = FilterMode.CONSERVATIVE + else: + mode = FilterMode(env_val) + + if mode == FilterMode.NONE: + return + + cli_base_ref = config.getoption("--filter-base-ref", default=None) + changed = git_changed_files(config.rootpath, base_ref=cli_base_ref) + + # Resolve the actual base_ref used (env fallback mirrors git_changed_files logic) + resolved_base_ref = cli_base_ref or os.environ.get( + "AUTOSKILLIT_TEST_BASE_REF", + os.environ.get("GITHUB_BASE_REF"), + ) + + manifest = load_manifest(config.rootpath) + coverage_map_path = config.rootpath / ".autoskillit" / "test-source-map.json" + + scope = build_test_scope( + changed_files=changed, + mode=mode, + manifest=manifest, + tests_root=config.rootpath / "tests", + coverage_map_path=coverage_map_path, + cwd=config.rootpath, + base_ref=resolved_base_ref, + ) + + if isinstance(scope, FullRunReason): + config.stash[_scope_key] = None + config.stash[_full_run_reason_key] = scope.value + else: + config.stash[_scope_key] = scope + config.stash[_filter_mode_key] = mode.value + + except Exception as exc: + warnings.warn( + f"Test filter setup failed, running all tests: {exc}", + stacklevel=1, + ) + + +@functools.lru_cache(maxsize=1) +def _resolve_test_config() -> "AutomationConfig | None": + """Resolve full config for test collection via full config resolution. + + Uses the same dynaconf chain as production: defaults.yaml → project config → env vars. + Returns None on any failure (fail-open: individual features fall back to + FEATURE_REGISTRY[name].default_enabled). + """ + try: + from pathlib import Path + + from autoskillit.config.settings import AutomationConfig as _AutomationConfig + from autoskillit.config.settings import load_config + + # Anchor to repo root via this file's known location (tests/conftest.py) + # rather than Path.cwd(), which varies across IDE runners and monkeypatch.chdir. + repo_root = Path(__file__).resolve().parent.parent + cfg = load_config(repo_root) + if not isinstance(cfg, _AutomationConfig): + raise TypeError(f"load_config returned {type(cfg)!r}, expected AutomationConfig") + return cfg + except Exception as exc: + import warnings + + warnings.warn( + f"Feature flag config resolution failed, falling back to defaults: {exc}", + stacklevel=1, + ) + return None + + +def _is_test_feature_enabled(feature_name: str, *, env_val: str | None) -> bool: + """Return True if feature_name is enabled for this test run. + + Resolution order: + 1. If AUTOSKILLIT_TEST_FEATURES is set (including empty string), parse it + as a comma-separated whitelist. Only listed names are enabled. + 2. If unset, resolve via full config chain (defaults.yaml → project config + → env vars) using load_config(). This respects experimental_enabled and + per-feature config overrides. + 3. If config resolution fails, fall back to FEATURE_REGISTRY[name].default_enabled. + Unknown feature names return True (fail-open). + + Args: + feature_name: The feature name to check. + env_val: Pre-read value of AUTOSKILLIT_TEST_FEATURES (pass ``None`` when unset). + """ + if env_val is not None: + enabled = {f.strip() for f in env_val.split(",") if f.strip()} + return feature_name in enabled + + from autoskillit.core import FEATURE_REGISTRY + from autoskillit.core.feature_flags import is_feature_enabled + + defn = FEATURE_REGISTRY.get(feature_name) + if defn is None: + import warnings + + warnings.warn( + f"pytest.mark.feature({feature_name!r}) references an unknown feature; " + "fail-open assumed (test will run). Check for typos in the marker.", + stacklevel=4, + ) + return True + + cfg = _resolve_test_config() + if cfg is None: + return defn.default_enabled + + return is_feature_enabled( + feature_name, cfg.features, experimental_enabled=cfg.experimental_enabled + ) + + +def pytest_collection_modifyitems( + items: list[pytest.Item], + config: pytest.Config, +) -> None: + """Deselect test items outside the computed filter scope. + + Fail-open: any error leaves all items selected. + """ + import warnings + + # Layer marker mismatch validation (controller-only under xdist) + if not hasattr(config, "workerinput"): + tests_root = config.rootpath / "tests" + for item in items: + try: + rel = item.path.relative_to(tests_root) + except (ValueError, TypeError): + continue + parts = rel.parts + if not parts or parts[0] not in _LAYER_DIRS: + continue + expected_dir = parts[0] + + for mark in item.iter_markers("layer"): + if mark.args and mark.args[0] != expected_dir: + warnings.warn( + f"Layer marker mismatch: {item.nodeid} has layer('{mark.args[0]}') " + f"but lives in tests/{expected_dir}/", + stacklevel=1, + ) + + # Feature gate pass — orthogonal to layer/size, runs on every worker + _test_features_env = os.environ.get("AUTOSKILLIT_TEST_FEATURES") + for item in items: + marker = item.get_closest_marker("feature") + if marker and marker.args: + feature_name = marker.args[0] + if not isinstance(feature_name, str): + warnings.warn( + f"pytest.mark.feature() received a non-string argument {feature_name!r} " + f"on {item.nodeid}; marker will be ignored.", + stacklevel=1, + ) + continue + if not _is_test_feature_enabled(feature_name, env_val=_test_features_env): + env_display = _test_features_env or "" + item.add_marker( + pytest.mark.skip( + reason=( + f"feature '{feature_name}' disabled" + f" (AUTOSKILLIT_TEST_FEATURES='{env_display}'" + f" does not include '{feature_name}')" + ) + ) + ) + + scope: set[_Path] | None = config.stash.get(_scope_key, None) + if scope is None: + return + + try: + root = config.rootpath + scope_abs: set[_Path] = set() + for p in scope: + scope_abs.add(p if p.is_absolute() else root / p) + + selected: list[pytest.Item] = [] + deselected: list[pytest.Item] = [] + + for item in items: + item_path = item.path + matched = False + for sp in scope_abs: + if sp.is_file(): + if item_path == sp: + matched = True + break + else: + try: + item_path.relative_to(sp) + matched = True + break + except ValueError: + continue + if matched: + selected.append(item) + else: + deselected.append(item) + + if deselected: + config.hook.pytest_deselected(items=deselected) + items[:] = selected + warnings.warn( + f"Test filter: {len(selected)} selected, {len(deselected)} deselected " + f"({len(scope)} scope paths)", + stacklevel=1, + ) + + config.stash[_selected_count_key] = len(items) + config.stash[_deselected_count_key] = len(deselected) + + except Exception as exc: + warnings.warn( + f"Test filter deselection failed, running all tests: {exc}", + stacklevel=1, + ) + + # --- Size-based deselection (aggressive mode only) --- + filter_mode = config.stash.get(_filter_mode_key, None) + if filter_mode == "aggressive": + _SIZE_MARKERS = {"small", "medium", "large"} + size_selected: list[pytest.Item] = [] + size_deselected: list[pytest.Item] = [] + + for item in items: + size_marks = [m.name for m in item.iter_markers() if m.name in _SIZE_MARKERS] + effective_size = size_marks[0] if size_marks else "large" + if effective_size in ("small", "medium"): + size_selected.append(item) + else: + size_deselected.append(item) + + if size_deselected: + config.hook.pytest_deselected(items=size_deselected) + items[:] = size_selected + warnings.warn( + f"Size filter (aggressive): {len(size_selected)} selected, " + f"{len(size_deselected)} large/unannotated deselected", + stacklevel=1, + ) + prev_deselected = config.stash.get(_deselected_count_key, None) or 0 + config.stash[_selected_count_key] = len(size_selected) + config.stash[_deselected_count_key] = prev_deselected + len(size_deselected) + + +def pytest_sessionfinish(session, exitstatus): + """Write filter stats sidecar for DefaultTestRunner consumption.""" + if hasattr(session.config, "workerinput"): + # xdist worker: propagate counts to controller via workeroutput IPC channel. + # config.stash is process-local; the controller never sees stash writes from + # workers, so we must transfer the counts explicitly here. + session.config.workeroutput["filter_selected"] = session.config.stash.get( + _selected_count_key, None + ) + session.config.workeroutput["filter_deselected"] = session.config.stash.get( + _deselected_count_key, None + ) + return + out_path = os.environ.get("AUTOSKILLIT_FILTER_STATS_FILE") + if not out_path: + return + filter_mode = session.config.stash.get(_filter_mode_key, None) + selected = session.config.stash.get(_selected_count_key, None) + deselected = session.config.stash.get(_deselected_count_key, None) + # Under xdist the controller never runs pytest_collection_modifyitems, so the + # stash keys are None there. Fall back to counts aggregated by pytest_testnodedown. + if selected is None and _worker_filter_counts: + selected = _worker_filter_counts.get("selected") + if deselected is None and _worker_filter_counts: + deselected = _worker_filter_counts.get("deselected") + if filter_mode is None: + return + import json + + full_run_reason = session.config.stash.get(_full_run_reason_key, None) + _Path(out_path).write_text( + json.dumps( + { + "filter_mode": filter_mode, + "tests_selected": selected, + "tests_deselected": deselected, + "full_run_reason": full_run_reason, + } + ) + ) + + +@pytest.hookimpl(optionalhook=True) +def pytest_testnodedown(node, error): + """Aggregate filter counts from the first xdist worker that reports. + + Called on the controller process by xdist after each worker finishes. + We capture the first worker that reports both counts as non-None; all workers + see the same test set under ``--dist load`` (collection and filtering happen + per-worker before distribution), so any single worker's counts are + representative of the full session. Note: this assumption only holds under + ``--dist load``; under ``--dist loadscope`` or ``--dist loadfile`` different + workers process different subsets and counts may diverge. + """ + if _worker_filter_counts: + return # already captured from the first reporting worker + wo = getattr(node, "workeroutput", {}) + selected = wo.get("filter_selected") + deselected = wo.get("filter_deselected") + if selected is not None and deselected is not None: + _worker_filter_counts["selected"] = selected + _worker_filter_counts["deselected"] = deselected diff --git a/tests/contracts/CLAUDE.md b/tests/contracts/CLAUDE.md new file mode 100644 index 0000000000..6349d02831 --- /dev/null +++ b/tests/contracts/CLAUDE.md @@ -0,0 +1,71 @@ +# contracts/ + +Protocol satisfaction, package gateway, and skill contract compliance tests. + +## Files + +| File | Purpose | +|------|---------| +| `__init__.py` | empty | +| `_anti_confirm_helpers.py` | Shared anti-confirmation regex for contract tests — mirrors production regex | +| `conftest.py` | Shared constants for contract tests — REFUSAL_SIGNALS | +| `test_activate_deps_completeness.py` | Contracts: SKILL.md activate_deps must cover invoked Skill tool calls | +| `test_advisory_coverage.py` | Contracts: SKILL_FILE_ADVISORY_MAP advisory hook coverage | +| `test_api_surface_alignment.py` | REQ-C8-01 / C2-01: API surface alignment tests | +| `test_claim_issue_contracts.py` | Contract tests for claim_issue and release_issue MCP tools | +| `test_claude_code_interface_contracts.py` | Contract tests for Claude Code external interface conventions | +| `test_collapse_issues_contracts.py` | Contract tests for the collapse-issues skill SKILL.md | +| `test_campaign_prompt_accuracy.py` | Contract: campaign prompt does not contain inaccurate semaphore language | +| `test_config_field_coverage.py` | REQ-CONFIG-001: every sub-config dataclass field must be referenced in from_dynaconf | +| `test_core_public_api_surface.py` | Validates that every symbol in autoskillit.core.__all__ is importable via the public gateway | +| `test_diagnose_ci_steps.py` | Contract tests for diagnose-ci SKILL.md step numbering and cross-reference integrity | +| `test_docstring_skill_prefix.py` | Contract: source files must not use /autoskillit: prefix for skills_extended skills | +| `test_environment_setup_design_contracts.py` | Contract tests verifying the environment-setup skill design doc completeness | +| `test_enrich_issues_contracts.py` | Contract tests for the enrich-issues skill SKILL.md | +| `test_execution_map_contracts.py` | Contract tests for the build-execution-map skill SKILL.md | +| `test_exogenous_string_coupling.py` | Exogenous string coupling tests: orchestrator prompt triggers coupled to emitting module | +| `test_filter_env_var_coverage.py` | Tests that retry-worktree and audit-impl skills set filter env vars for test runs | +| `test_fleet_dispatch_bem_gate.py` | Contract: fleet dispatcher prompt contains BEM pre-step gate instructions | +| `test_generate_report_contracts.py` | Contract tests for generate-report SKILL.md — data provenance lifecycle | +| `test_github_ops.py` | Contract tests: GitHub operation semantics in SKILL.md files | +| `test_hook_bridge_coverage.py` | REQ-BRIDGE-001: quota guard hook config bridge must produce exactly the keys that resolve_quota_settings() reads | +| `test_implement_experiment_contracts.py` | Contract tests for implement-experiment SKILL.md — test infrastructure requirements | +| `test_instruction_surface.py` | Contract tests: every instruction surface must carry the pipeline tool restriction | +| `test_issue_content_fidelity.py` | Cross-skill contract: content fidelity for issue body assembly | +| `test_issue_splitter_contracts.py` | Contract tests: issue-splitter skill correctness and triage-issues integration | +| `test_l1_packages.py` | Package export surface tests for the L1 sub-packages | +| `test_make_campaign_skill_contracts.py` | Contract tests: structural invariants for the make-campaign SKILL.md | +| `test_mermaid_palette_contracts.py` | Contract: any SKILL.md that generates mermaid diagrams must embed the canonical 9-class palette | +| `test_no_pagination_file_read.py` | Contract tests for no-pagination file read instruction in high-turn SKILL.md files | +| `test_package_gateways.py` | Tests for Package Gateway API (groupC) — REQ-GWAY-001 through REQ-GWAY-008 | +| `test_plan_experiment_contracts.py` | Contract tests for plan-experiment SKILL.md — data provenance lifecycle | +| `test_pr_traceability_contracts.py` | Cross-skill contract tests for requirement traceability across PR lifecycle skills | +| `test_prepare_compose_pr_contracts.py` | Contract tests for prepare-pr and compose-pr skills | +| `test_prepare_issue_contracts.py` | Contract tests for the prepare-issue SKILL.md | +| `test_process_issues_contracts.py` | Contract tests for the process-issues skill SKILL.md | +| `test_protocol_definitions.py` | Tests for Protocol definitions in core/_type_protocols_*.py shards (REQ-PROTO-007) | +| `test_protocol_satisfaction.py` | Tests for Protocol Contract Layer (GroupB) | +| `test_protocol_satisfaction_five.py` | Protocol satisfaction tests — Group Five (issue #1523) | +| `test_review_local_mode_contracts.py` | Contract tests for skill_contracts.yaml and SKILL.md validation for local review mode (mode=local) | +| `test_review_pr_diff_annotation.py` | C-RPR-1: Contract tests for review-pr diff annotation inputs | +| `test_run_experiment_contracts.py` | Contract tests for run-experiment SKILL.md — data provenance lifecycle | +| `test_scope_contracts.py` | Contract tests for the scope skill's SKILL.md template | +| `test_skill_contracts.py` | Contract tests: every delimiter-emitting skill must be registered in skill_contracts.yaml | +| `test_skill_directive_descriptions.py` | Contract: headless recipe skills must use directive language in SKILL.md descriptions | +| `test_skill_transition_boundaries.py` | Contract tests for anti-confirmation instructions at SKILL.md transition boundaries | +| `test_skill_yaml_validation.py` | Contract: YAML workflow examples embedded in SKILL.md files must be valid recipes | +| `test_sous_chef_quota_protocol.py` | Contract test: sous-chef SKILL.md must contain QUOTA WAIT PROTOCOL section | +| `test_sous_chef_routing.py` | Contract tests for the CONTEXT LIMIT ROUTING section in sous-chef SKILL.md | +| `test_sous_chef_scheduling.py` | Contract tests for the PARALLEL STEP SCHEDULING section in sous-chef SKILL.md | +| `test_stage_data_contracts.py` | Contract tests for stage-data SKILL.md — pre-flight resource feasibility gate | +| `test_sub_skill_refusal_contracts.py` | Cross-skill contract: every SKILL.md that invokes sub-skills must contain explicit refusal handling language | +| `test_target_skill_invocability.py` | Contract: the target skill of a run_skill call must be invocable after session setup | +| `test_token_summary_contracts.py` | Structural contracts for the token summary pipeline | +| `test_tools_recipe_contracts.py` | Contract tests for tools_recipe.py MCP tool docstrings | +| `test_triage_contracts.py` | Contract tests for triage-issues --enrich flag and requirement enrichment behavior | +| `test_triage_issues_contracts.py` | Contract tests for triage-issues body-file safety (gh issue edit --body-file) | +| `test_version_consistency.py` | Cross-file version consistency: pyproject.toml, __init__.__version__, plugin.json, bundled recipe versions | + +## Architecture Notes + +`conftest.py` provides `REFUSAL_SIGNALS` constants shared across many contract tests. `_anti_confirm_helpers.py` mirrors the production anti-confirmation regex for structural contract verification. diff --git a/tests/contracts/_anti_confirm_helpers.py b/tests/contracts/_anti_confirm_helpers.py new file mode 100644 index 0000000000..2def7a635b --- /dev/null +++ b/tests/contracts/_anti_confirm_helpers.py @@ -0,0 +1,16 @@ +"""Shared anti-confirmation regex for contract tests — mirrors production regex.""" + +from __future__ import annotations + +import re + +ANTI_CONFIRM_RE = re.compile( + r"(?:never|do\s+not|must\s+not|prohibited)" + r"[^\n]{0,80}" + r"(?:ask|confirm|pause|AskUserQuestion" + r"|wait\s+for\s+[^\n]{0,30}(?:user|human|permission|approval|confirmation))" + r"|(?:proceed|dispatch|continue|start)" + r"[^\n]{0,60}" + r"(?:immediately|without[^\n]{0,30}(?:asking|confirming|pausing|waiting))", + re.IGNORECASE, +) diff --git a/tests/contracts/test_activate_deps_completeness.py b/tests/contracts/test_activate_deps_completeness.py index 3e42432c9e..cc60ecb328 100644 --- a/tests/contracts/test_activate_deps_completeness.py +++ b/tests/contracts/test_activate_deps_completeness.py @@ -9,14 +9,16 @@ from __future__ import annotations import re +from pathlib import Path import pytest +from autoskillit.core.types._type_constants import PACK_REGISTRY, SKILL_ACTIVATE_DEPS_REQUIRED from autoskillit.workspace.session_skills import ( SkillsDirectoryProvider, compute_skill_closure, ) -from autoskillit.workspace.skills import bundled_skills_extended_dir +from autoskillit.workspace.skills import bundled_skills_dir, bundled_skills_extended_dir _INVOCATION_PATTERNS = ( re.compile( @@ -26,6 +28,7 @@ ) _FM_PATTERN = re.compile(r"^---\n(.*?)\n?---\n?(.*)", re.DOTALL) +_ACTIVATE_DEPS_LINE_RE = re.compile(r"^activate_deps:\s*\[([^\]]*)\]", re.MULTILINE) def _strip_frontmatter(content: str) -> str: @@ -33,6 +36,25 @@ def _strip_frontmatter(content: str) -> str: return m.group(2) if m else content +def _get_activate_deps(skill_md: Path) -> list[str]: + """Parse activate_deps from a SKILL.md frontmatter (bracket list format).""" + content = skill_md.read_text() + m = _FM_PATTERN.match(content) + frontmatter = m.group(1) if m else content + deps_m = _ACTIVATE_DEPS_LINE_RE.search(frontmatter) + if not deps_m: + return [] + return [d.strip() for d in deps_m.group(1).split(",") if d.strip()] + + +def _find_skill_md(skill_name: str) -> Path | None: + for base in (bundled_skills_dir(), bundled_skills_extended_dir()): + candidate = base / skill_name / "SKILL.md" + if candidate.exists(): + return candidate + return None + + @pytest.fixture(scope="module") def provider() -> SkillsDirectoryProvider: return SkillsDirectoryProvider() @@ -87,3 +109,52 @@ def test_all_skills_skill_tool_invocations_match_activate_deps( f"but it is not in compute_skill_closure({skill_name!r})" ) assert not failures, "Undeclared Skill-tool invocations:\n" + "\n".join(failures) + + +def test_required_activate_deps_present() -> None: + """For each (skill, required_deps) in SKILL_ACTIVATE_DEPS_REQUIRED, the skill's + SKILL.md must declare all required deps in activate_deps.""" + failures: list[str] = [] + for skill_name, required_deps in SKILL_ACTIVATE_DEPS_REQUIRED.items(): + skill_md = _find_skill_md(skill_name) + if skill_md is None: + failures.append(f"{skill_name}: SKILL.md not found") + continue + declared = set(_get_activate_deps(skill_md)) + for dep in sorted(required_deps): + if dep not in declared: + failures.append( + f"{skill_name}/SKILL.md: SKILL_ACTIVATE_DEPS_REQUIRED requires " + f"{dep!r} in activate_deps, but only {sorted(declared)!r} declared" + ) + assert not failures, "Missing required activate_deps:\n" + "\n".join(failures) + + +def test_all_activate_deps_resolve() -> None: + """Every name in any SKILL.md's activate_deps must resolve to a real skill or pack. + + No silent drops — a typo in activate_deps is caught at CI time. + """ + pack_keys = set(PACK_REGISTRY.keys()) + resolver_cache: dict[str, bool] = {} + + # Use a fresh provider resolver to check skill resolution + from autoskillit.workspace.skills import DefaultSkillResolver + + resolver = DefaultSkillResolver() + + def _resolves(name: str) -> bool: + if name not in resolver_cache: + resolver_cache[name] = name in pack_keys or resolver.resolve(name) is not None + return resolver_cache[name] + + failures: list[str] = [] + for base in (bundled_skills_dir(), bundled_skills_extended_dir()): + for skill_md in sorted(base.glob("*/SKILL.md")): + for dep in _get_activate_deps(skill_md): + if not _resolves(dep): + failures.append( + f"{skill_md.parent.name}/SKILL.md: activate_deps contains " + f"{dep!r} which does not resolve to a skill or pack" + ) + assert not failures, "Unresolvable activate_deps entries:\n" + "\n".join(failures) diff --git a/tests/contracts/test_advisory_coverage.py b/tests/contracts/test_advisory_coverage.py new file mode 100644 index 0000000000..2943242d33 --- /dev/null +++ b/tests/contracts/test_advisory_coverage.py @@ -0,0 +1,103 @@ +"""Contracts: SKILL_FILE_ADVISORY_MAP advisory hook coverage. + +Guards: +- recipe_write_advisor.py is registered in HOOK_REGISTRY with Write|Edit matcher. +- Advisory hook emits message payload, never permissionDecision. +- All skill names in SKILL_FILE_ADVISORY_MAP resolve to real skills. +- recipe_write_advisor._ADVISORY_PATTERNS stays in sync with SKILL_FILE_ADVISORY_MAP. +""" + +from __future__ import annotations + +import json +import os +import subprocess +import sys + +from autoskillit.core.paths import pkg_root +from autoskillit.core.types._type_constants import SKILL_FILE_ADVISORY_MAP +from autoskillit.hook_registry import HOOK_REGISTRY +from autoskillit.workspace.skills import DefaultSkillResolver + + +def _run_advisor(payload: dict, extra_env: dict[str, str] | None = None) -> tuple[int, str]: + hook_path = pkg_root() / "hooks" / "guards" / "recipe_write_advisor.py" + env = {**os.environ, **(extra_env or {})} + env.pop("AUTOSKILLIT_HEADLESS", None) + result = subprocess.run( + [sys.executable, str(hook_path)], + input=json.dumps(payload), + capture_output=True, + text=True, + env=env, + ) + return result.returncode, result.stdout + + +def test_every_advisory_pattern_has_a_hook() -> None: + """recipe_write_advisor.py is registered in HOOK_REGISTRY under Write|Edit.""" + advisory_scripts = { + script + for hook in HOOK_REGISTRY + if "Write" in hook.matcher and hook.session_scope == "interactive_only" + for script in hook.scripts + } + assert "guards/recipe_write_advisor.py" in advisory_scripts, ( + "guards/recipe_write_advisor.py is not registered in HOOK_REGISTRY with " + "a Write|Edit matcher and session_scope=interactive_only" + ) + + +def test_advisory_hooks_are_non_blocking() -> None: + """Advisory hook emits message payload, never permissionDecision.""" + payload = { + "tool_name": "Write", + "tool_input": {"file_path": ".autoskillit/recipes/test.yaml"}, + } + rc, stdout = _run_advisor(payload) + assert rc == 0 + assert stdout.strip(), "Expected advisory output for a recipe YAML path" + data = json.loads(stdout.strip()) + hook_out = data["hookSpecificOutput"] + assert "message" in hook_out, f"Expected 'message' key in hookSpecificOutput, got {hook_out!r}" + assert "permissionDecision" not in hook_out, "Advisory hook must not emit permissionDecision" + + +def test_advisory_map_skill_names_resolve() -> None: + """Every skill name in SKILL_FILE_ADVISORY_MAP must resolve to a real skill.""" + resolver = DefaultSkillResolver() + unresolvable = [ + skill for skill in SKILL_FILE_ADVISORY_MAP.values() if resolver.resolve(skill) is None + ] + assert not unresolvable, ( + f"SKILL_FILE_ADVISORY_MAP references unresolvable skill names: {unresolvable}" + ) + + +def test_advisory_hook_message_uses_bare_prefix() -> None: + """Advisory message must use /name, not /autoskillit:name, for skills_extended skills.""" + payload = { + "tool_name": "Write", + "tool_input": {"file_path": ".autoskillit/recipes/test.yaml"}, + } + rc, stdout = _run_advisor(payload) + assert rc == 0 + assert stdout.strip() + data = json.loads(stdout.strip()) + message = data["hookSpecificOutput"]["message"] + assert "/autoskillit:" not in message, ( + f"Advisory message uses /autoskillit: prefix for a skills_extended skill: {message}" + ) + assert "/write-recipe" in message + + +def test_hook_patterns_match_type_constants() -> None: + """recipe_write_advisor._ADVISORY_PATTERNS must exactly match SKILL_FILE_ADVISORY_MAP.""" + from autoskillit.hooks.guards.recipe_write_advisor import _ADVISORY_PATTERNS + + assert list(SKILL_FILE_ADVISORY_MAP.items()) == _ADVISORY_PATTERNS, ( + "recipe_write_advisor._ADVISORY_PATTERNS is out of sync with " + "SKILL_FILE_ADVISORY_MAP in _type_constants.py. " + f"Hook patterns: {_ADVISORY_PATTERNS!r}, " + f"Constants map: {list(SKILL_FILE_ADVISORY_MAP.items())!r}" + ) diff --git a/tests/contracts/test_api_surface_alignment.py b/tests/contracts/test_api_surface_alignment.py index 215f3c6887..96511d192c 100644 --- a/tests/contracts/test_api_surface_alignment.py +++ b/tests/contracts/test_api_surface_alignment.py @@ -66,3 +66,24 @@ def test_subprocess_runner_pty_mode_default_is_false(): assert param.default is False, ( f"SubprocessRunner.pty_mode default must be False, got {param.default!r}" ) + + +def test_output_path_tokens_contains_all_file_path_contract_outputs() -> None: + from autoskillit.execution.headless import ( + _INTENTIONALLY_EXCLUDED_PATH_TOKENS, + _OUTPUT_PATH_TOKENS, + ) + from autoskillit.recipe.contracts import load_bundled_manifest + + manifest = load_bundled_manifest() + declared_path_tokens = { + out["name"] + for skill_data in manifest.get("skills", {}).values() + for out in skill_data.get("outputs", []) + if isinstance(out, dict) and out.get("type", "").startswith("file_path") + } + untracked = declared_path_tokens - _OUTPUT_PATH_TOKENS - _INTENTIONALLY_EXCLUDED_PATH_TOKENS + assert not untracked, ( + f"These path tokens are declared in skill_contracts.yaml but missing from " + f"_OUTPUT_PATH_TOKENS or _INTENTIONALLY_EXCLUDED_PATH_TOKENS: {untracked}" + ) diff --git a/tests/contracts/test_campaign_prompt_accuracy.py b/tests/contracts/test_campaign_prompt_accuracy.py new file mode 100644 index 0000000000..f1d04a5cd1 --- /dev/null +++ b/tests/contracts/test_campaign_prompt_accuracy.py @@ -0,0 +1,22 @@ +"""Contract: campaign prompt does not contain inaccurate semaphore language.""" + +from __future__ import annotations + +import pytest + +from autoskillit.cli._prompts_campaign import _build_dynamic_dispatch_section + +MCP_PREFIX = "autoskillit__" + + +@pytest.fixture(scope="module") +def dynamic_dispatch_text() -> str: + return _build_dynamic_dispatch_section(mcp_prefix=MCP_PREFIX) + + +def test_campaign_prompt_does_not_claim_queuing(dynamic_dispatch_text: str) -> None: + """Campaign prompt must NOT claim dispatches queue when semaphore is saturated.""" + assert "calls queue" not in dynamic_dispatch_text, ( + "Campaign prompt must not claim 'calls queue when the semaphore is saturated' — " + "FLEET_PARALLEL_REFUSED is a fast-fail, not a queue. Dispatcher must wait and retry." + ) diff --git a/tests/contracts/test_claim_issue_contracts.py b/tests/contracts/test_claim_issue_contracts.py index 327d0a6060..cc0fdbe8d0 100644 --- a/tests/contracts/test_claim_issue_contracts.py +++ b/tests/contracts/test_claim_issue_contracts.py @@ -17,13 +17,13 @@ def test_release_issue_in_gated_tools() -> None: def test_claim_issue_tool_registered() -> None: """claim_issue must be importable from tools_issue_lifecycle.""" - from autoskillit.server.tools_issue_lifecycle import claim_issue + from autoskillit.server.tools.tools_issue_lifecycle import claim_issue assert callable(claim_issue) def test_release_issue_tool_registered() -> None: """release_issue must be importable from tools_issue_lifecycle.""" - from autoskillit.server.tools_issue_lifecycle import release_issue + from autoskillit.server.tools.tools_issue_lifecycle import release_issue assert callable(release_issue) diff --git a/tests/contracts/test_claude_code_interface_contracts.py b/tests/contracts/test_claude_code_interface_contracts.py index 7429470f31..2aa18ec4ae 100644 --- a/tests/contracts/test_claude_code_interface_contracts.py +++ b/tests/contracts/test_claude_code_interface_contracts.py @@ -240,7 +240,7 @@ def fake_run(cmd: list[str], **kw: object) -> object: monkeypatch.setattr("builtins.input", lambda _prompt="": "") monkeypatch.setattr("sys.stdin.isatty", lambda: True) - from autoskillit.cli._cook import cook + from autoskillit.cli.session._cook import cook # Use a fixed ephemeral root so cleanup is deterministic from autoskillit.workspace.session_skills import ( diff --git a/tests/contracts/test_collapse_issues_contracts.py b/tests/contracts/test_collapse_issues_contracts.py index ef0e8924cc..7893d9b22e 100644 --- a/tests/contracts/test_collapse_issues_contracts.py +++ b/tests/contracts/test_collapse_issues_contracts.py @@ -78,10 +78,18 @@ def test_max_group_flag_documented(): assert "--max-group" in skill_text() -def test_originals_closed_with_comment(): +def test_originals_superseded_body_edit(): text = skill_text() - assert "gh issue comment" in text, "Must document closing comment on originals" - assert "gh issue close" in text, "Must document closing originals" + assert "## Superseded" in text, "Must append ## Superseded section to original body" + assert "gh issue edit" in text, "Must use gh issue edit to update original body" + assert "gh issue close" in text, "Must still close originals" + + +def test_no_gh_issue_comment_in_collapse_issues(): + text = skill_text() + assert "gh issue comment" not in text, ( + "Must not post issue comments — all updates go to issue body" + ) def test_combined_issue_references_originals(): diff --git a/tests/contracts/test_config_field_coverage.py b/tests/contracts/test_config_field_coverage.py index 87adc93255..08837c2a8d 100644 --- a/tests/contracts/test_config_field_coverage.py +++ b/tests/contracts/test_config_field_coverage.py @@ -29,13 +29,23 @@ def _collect_referenced_names(tree: ast.Module) -> set[str]: dedicated _build_* helpers that contain the actual val(...) / .get(...) calls. These helpers are called directly from from_dynaconf and are part of the same construction pipeline — their field references are equally authoritative. + After the config split, _build_* helpers may reside in _config_loader.py + rather than settings.py; both modules are scanned. """ - # Collect all module-level function definitions by name + # Collect all module-level function definitions from settings.py top_level_fns: dict[str, ast.FunctionDef] = {} for node in ast.walk(tree): if isinstance(node, ast.FunctionDef): top_level_fns[node.name] = node + # Also collect from _config_loader.py (split destination for _build_* helpers) + loader_path = Path(settings_mod.__file__).parent / "_config_loader.py" + if loader_path.exists(): + loader_tree = ast.parse(loader_path.read_text()) + for node in ast.walk(loader_tree): + if isinstance(node, ast.FunctionDef) and node.name not in top_level_fns: + top_level_fns[node.name] = node + fn: ast.FunctionDef | None = top_level_fns.get("from_dynaconf") assert fn is not None, "from_dynaconf must exist" diff --git a/tests/contracts/test_core_public_api_surface.py b/tests/contracts/test_core_public_api_surface.py new file mode 100644 index 0000000000..471e4d5dfd --- /dev/null +++ b/tests/contracts/test_core_public_api_surface.py @@ -0,0 +1,20 @@ +"""Validates that every symbol in autoskillit.core.__all__ is importable via the public gateway.""" + +from __future__ import annotations + +import pytest + +pytestmark = [pytest.mark.layer("contracts"), pytest.mark.small] + + +def test_all_public_symbols_importable() -> None: + """Every symbol in autoskillit.core.__all__ must be importable via the public gateway.""" + import autoskillit.core as core_module + + failures: list[str] = [] + for symbol in core_module.__all__: + try: + getattr(core_module, symbol) + except (ImportError, AttributeError) as exc: + failures.append(f"{symbol}: {exc}") + assert not failures, "Public API surface broken:\n" + "\n".join(failures) diff --git a/tests/contracts/test_diagnose_ci_steps.py b/tests/contracts/test_diagnose_ci_steps.py new file mode 100644 index 0000000000..08955d7295 --- /dev/null +++ b/tests/contracts/test_diagnose_ci_steps.py @@ -0,0 +1,50 @@ +"""diagnose-ci SKILL.md step-structure contracts: numbering and cross-reference integrity.""" + +from __future__ import annotations + +import re +from pathlib import Path + +import pytest + +pytestmark = [pytest.mark.layer("infra"), pytest.mark.small] + +_REPO_ROOT = Path(__file__).resolve().parents[2] +_SKILLS_DIR = _REPO_ROOT / "src" / "autoskillit" / "skills_extended" + + +def _get_diagnose_ci_content() -> str: + skill_md = _SKILLS_DIR / "diagnose-ci" / "SKILL.md" + return skill_md.read_text() + + +def test_diagnose_ci_step_numbering_is_sequential() -> None: + """T2: diagnose-ci steps are sequential starting from 1 with no gaps.""" + content = _get_diagnose_ci_content() + step_headings = re.findall(r"^### Step (\d+):", content, re.MULTILINE) + step_numbers = [int(n) for n in step_headings] + expected = list(range(1, len(step_numbers) + 1)) + assert step_numbers == expected, ( + f"diagnose-ci step numbers must be sequential 1..{len(step_numbers)}, got {step_numbers}" + ) + + +def test_diagnose_ci_step_crossref_resolves_correctly() -> None: + """T3: every 'proceed to Step N (...)' cross-reference in diagnose-ci resolves correctly.""" + content = _get_diagnose_ci_content() + matches = re.findall(r"proceed to Step (\d+)\s+\(([^)]+)\)", content) + assert matches, ( + "Could not find any 'proceed to Step N (...)' cross-reference in diagnose-ci/SKILL.md" + ) + for step_num_str, parenthetical in matches: + step_num = int(step_num_str) + parenthetical_lower = parenthetical.lower() + heading_pattern = rf"^### Step {step_num}:\s+(.+)$" + heading_m = re.search(heading_pattern, content, re.MULTILINE) + assert heading_m, f"Step {step_num} heading not found in diagnose-ci/SKILL.md" + heading_title = heading_m.group(1).lower() + assert parenthetical_lower in heading_title, ( + f"Cross-ref 'proceed to Step {step_num} ({parenthetical})' points to " + f"'### Step {step_num}: {heading_m.group(1)}' — " + f"'{parenthetical}' not found as a phrase in title" + ) diff --git a/tests/contracts/test_docstring_skill_prefix.py b/tests/contracts/test_docstring_skill_prefix.py new file mode 100644 index 0000000000..6de0101f73 --- /dev/null +++ b/tests/contracts/test_docstring_skill_prefix.py @@ -0,0 +1,49 @@ +"""Contract: source files must not use /autoskillit: prefix for skills_extended skills. + +The /autoskillit: namespace prefix is reserved for Tier 1 skills delivered via --plugin-dir +(open-kitchen, close-kitchen, sous-chef). Skills in skills_extended/ are registered as bare +names via --add-dir and must be referenced as /name, not /autoskillit:name. +""" + +import re + +from autoskillit.core import SkillSource +from autoskillit.core.paths import pkg_root +from autoskillit.workspace.skills import DefaultSkillResolver + +_PKG = pkg_root() +_SCAN_DIRS_AND_GLOBS: list[tuple[str, list[str]]] = [ + ("server/tools", ["tools_*.py"]), + ("server", ["_guards.py"]), + ("hooks", ["*.py"]), +] +_PREFIX_RE = re.compile(r"/autoskillit:([a-z][\w-]*)") + + +def _collect_prefixed_skill_refs() -> list[tuple[str, int, str]]: + """Return (relative_path, line_number, skill_name) for every /autoskillit:name reference.""" + hits: list[tuple[str, int, str]] = [] + for subdir, globs in _SCAN_DIRS_AND_GLOBS: + scan_dir = _PKG / subdir + for glob in globs: + for path in sorted(scan_dir.glob(glob)): + for i, line in enumerate(path.read_text().splitlines(), 1): + for m in _PREFIX_RE.finditer(line): + hits.append((f"{subdir}/{path.name}", i, m.group(1))) + return hits + + +def test_no_skills_extended_skill_uses_autoskillit_prefix() -> None: + resolver = DefaultSkillResolver() + violations: list[str] = [] + for rel_path, lineno, skill_name in _collect_prefixed_skill_refs(): + info = resolver.resolve(skill_name) + if info is not None and info.source == SkillSource.BUNDLED_EXTENDED: + violations.append( + f"{rel_path}:{lineno}: /autoskillit:{skill_name} " + f"is a skills_extended skill — use /{skill_name} instead" + ) + assert not violations, ( + "Source files use /autoskillit: prefix for skills_extended skills:\n" + + "\n".join(f" - {v}" for v in violations) + ) diff --git a/tests/contracts/test_enrich_issues_contracts.py b/tests/contracts/test_enrich_issues_contracts.py index e3282fa820..e314fd938a 100644 --- a/tests/contracts/test_enrich_issues_contracts.py +++ b/tests/contracts/test_enrich_issues_contracts.py @@ -33,3 +33,15 @@ def test_enrich_issues_body_file_uses_autoskillit_temp(): "enrich-issues must write issue body to {{AUTOSKILLIT_TEMP}}/enrich-issues/ " "before calling gh issue edit --body-file" ) + + +def test_enrich_issues_no_issue_comment(): + text = SKILL_MD.read_text() + assert "gh issue comment" not in text, ( + "Too-vague and mixed-concerns feedback must be appended to body, not posted as comments" + ) + + +def test_enrich_issues_triage_note_section(): + text = SKILL_MD.read_text() + assert "## Triage Note" in text, "Must use ## Triage Note body section for skip-path feedback" diff --git a/tests/contracts/test_environment_setup_design_contracts.py b/tests/contracts/test_environment_setup_design_contracts.py new file mode 100644 index 0000000000..fa09eedcb3 --- /dev/null +++ b/tests/contracts/test_environment_setup_design_contracts.py @@ -0,0 +1,87 @@ +"""Contract tests for the environment-setup skill design document. + +These tests verify the design document exists and contains all required +sections per issue #838. They fail until the design doc is written. +""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +DESIGN_DOC = Path(__file__).resolve().parents[2] / "docs" / "design" / "env-setup-design.md" + + +@pytest.fixture +def design_content() -> str: + assert DESIGN_DOC.is_file(), f"Design doc missing: {DESIGN_DOC}" + return DESIGN_DOC.read_text() + + +class TestDesignDocExists: + def test_design_doc_file_exists(self) -> None: + assert DESIGN_DOC.is_file(), ( + f"Expected design doc at {DESIGN_DOC}. " + "Create docs/design/env-setup-design.md per issue #838." + ) + + +class TestDesignDocCompleteness: + """Verify the design doc addresses all seven design requirements from #838.""" + + def test_defines_env_mode_enum_values(self, design_content: str) -> None: + for value in ("none", "docker", "micromamba-host", "unavailable"): + assert value in design_content, f"Design doc must define env_mode value '{value}'" + + def test_specifies_structured_output_tokens(self, design_content: str) -> None: + for token in ("env_mode", "env_report", "verdict"): + assert token in design_content, ( + f"Design doc must specify structured output token '{token}'" + ) + + def test_specifies_order_up_sentinel(self, design_content: str) -> None: + assert "%%ORDER_UP%%" in design_content + + def test_defines_verdict_routing(self, design_content: str) -> None: + for verdict in ("PASS", "WARN", "FAIL"): + assert verdict in design_content, ( + f"Design doc must define verdict routing for '{verdict}'" + ) + + def test_references_dockerfile_template(self, design_content: str) -> None: + assert "Dockerfile.template" in design_content, ( + "Design doc must reference the canonical Dockerfile template" + ) + + def test_pins_base_image_version(self, design_content: str) -> None: + assert "mambaorg/micromamba:1.0-bullseye-slim" in design_content, ( + "Design doc must pin the Dockerfile base image version" + ) + + def test_defines_recipe_step_yaml(self, design_content: str) -> None: + assert "setup_environment" in design_content, ( + "Design doc must define the setup_environment recipe step" + ) + + def test_defines_viability_rules(self, design_content: str) -> None: + content_lower = design_content.lower() + assert "conda-forge" in content_lower, ( + "Design doc must discuss conda-forge channel viability" + ) + assert "cuda" in content_lower, ( + "Design doc must discuss CUDA as a non-viable fallback case" + ) + + def test_defines_downstream_consumption(self, design_content: str) -> None: + assert "run_experiment" in design_content or "run-experiment" in design_content, ( + "Design doc must explain how run-experiment consumes env_mode" + ) + assert "implement_phase" in design_content or "implement-experiment" in design_content, ( + "Design doc must explain how implement-experiment consumes env_mode" + ) + + def test_skill_name_defined(self, design_content: str) -> None: + assert "setup-environment" in design_content, ( + "Design doc must name the skill 'setup-environment'" + ) diff --git a/tests/contracts/test_execution_map_contracts.py b/tests/contracts/test_execution_map_contracts.py new file mode 100644 index 0000000000..da75723175 --- /dev/null +++ b/tests/contracts/test_execution_map_contracts.py @@ -0,0 +1,342 @@ +"""Contract tests for the build-execution-map skill SKILL.md.""" + +from __future__ import annotations + +import json +import re +from pathlib import Path + +import pytest +import yaml + + +def _skill_md_text() -> str: + skill_md = ( + Path(__file__).resolve().parent.parent.parent + / "src" + / "autoskillit" + / "skills_extended" + / "build-execution-map" + / "SKILL.md" + ) + return skill_md.read_text() + + +def test_execution_map_schema_has_required_fields() -> None: + """SKILL.md output schema section must document required JSON fields.""" + text = _skill_md_text() + # Top-level fields + assert '"groups"' in text, "Schema must document 'groups' array" + assert '"parallel"' in text, "Schema must document 'parallel' bool per group" + assert '"issues"' in text, "Schema must document 'issues' array per group" + assert '"merge_order"' in text, "Schema must document top-level 'merge_order'" + # Per-issue fields + assert '"number"' in text, "Schema must document 'number' per issue" + assert '"title"' in text, "Schema must document 'title' per issue" + # Group-level field + assert '"group"' in text, "Schema must document 'group' integer per group" + # AI assessment fields + assert '"pairwise_assessments"' in text, "Schema must document 'pairwise_assessments' array" + assert '"parallel_safe"' in text, "Schema must document 'parallel_safe' per assessment" + assert '"confidence"' in text, "Schema must document 'confidence' per assessment" + assert '"reasoning"' in text, "Schema must document 'reasoning' per assessment" + + +def test_execution_map_output_tokens_declared() -> None: + """SKILL.md must declare structured output tokens execution_map and execution_map_report.""" + text = _skill_md_text() + # Tokens must appear as 'token_name = value' pattern (spaces around =) + assert re.search(r"execution_map\s*=\s*\S", text), ( + "SKILL.md must declare 'execution_map = ...' output token" + ) + assert re.search(r"execution_map_report\s*=\s*\S", text), ( + "SKILL.md must declare 'execution_map_report = ...' output token" + ) + + +def test_execution_map_never_block_prohibits_code_changes() -> None: + """SKILL.md NEVER block must prohibit modifying source code.""" + text = _skill_md_text() + never_match = re.search(r"\*\*NEVER:\*\*(.*?)(?=\n\*\*|\n##)", text, re.DOTALL) + assert never_match is not None, "SKILL.md must have a **NEVER:** block" + never_block = never_match.group(1).lower() + assert "modify" in never_block and "source code" in never_block, ( + "NEVER block must prohibit modifying source code files" + ) + + +def test_execution_map_always_block_requires_file_output() -> None: + """SKILL.md ALWAYS block must require writing the execution map to temp directory.""" + text = _skill_md_text() + always_match = re.search(r"\*\*ALWAYS:\*\*(.*?)(?=\n\*\*|\n##)", text, re.DOTALL) + assert always_match is not None, "SKILL.md must have an **ALWAYS:** block" + always_block = always_match.group(1) + assert "AUTOSKILLIT_TEMP" in always_block, ( + "ALWAYS block must reference AUTOSKILLIT_TEMP output directory" + ) + + +def test_execution_map_references_ai_assessment() -> None: + """SKILL.md must reference AI-driven pairwise assessment for parallelism decisions.""" + text = _skill_md_text() + lower = text.lower() + assert "pairwise" in lower, "SKILL.md must reference pairwise assessment" + assert "assessment" in lower or "parallel_safe" in lower, ( + "SKILL.md must reference AI-driven assessment or parallel_safe field" + ) + + +def test_execution_map_references_dependency_ordering() -> None: + """SKILL.md must reference dependency-based group ordering.""" + text = _skill_md_text() + lower = text.lower() + assert "dependency" in lower, "SKILL.md must reference dependency-based group ordering" + + +def test_execution_map_review_approach_flag_declared() -> None: + """SKILL.md Arguments section must declare --assess-review-approach flag.""" + text = _skill_md_text() + args_match = re.search(r"## Arguments(.*?)(?=\n##)", text, re.DOTALL) + assert args_match is not None, "SKILL.md must have an ## Arguments section" + args_section = args_match.group(1) + assert "--assess-review-approach" in args_section, ( + "Arguments section must declare --assess-review-approach flag" + ) + + +def test_execution_map_review_approach_schema_fields() -> None: + """SKILL.md output schema must document review-approach fields when flag is active.""" + text = _skill_md_text() + assert '"review_approach_recommended"' in text, ( + "Schema must document 'review_approach_recommended' boolean field" + ) + assert '"review_approach_reasoning"' in text, ( + "Schema must document 'review_approach_reasoning' string field" + ) + + +def test_execution_map_review_approach_candidates_token() -> None: + """SKILL.md must declare review_approach_candidates output token.""" + text = _skill_md_text() + assert re.search(r"review_approach_candidates\s*=\s*\S", text), ( + "SKILL.md must declare 'review_approach_candidates = ...' output token" + ) + + +def test_execution_map_reads_review_approach_skill() -> None: + """SKILL.md must instruct reading review-approach/SKILL.md to ground assessment.""" + text = _skill_md_text() + assert "review-approach" in text.lower() and "SKILL.md" in text, ( + "SKILL.md must reference reading review-approach/SKILL.md for assessment grounding" + ) + + +def test_review_approach_candidates_contract_registered() -> None: + """skill_contracts.yaml must register review_approach_candidates output.""" + from autoskillit.recipe.contracts import get_skill_contract, load_bundled_manifest + + manifest = load_bundled_manifest() + contract = get_skill_contract("build-execution-map", manifest) + assert contract is not None + output_names = [o.name for o in contract.outputs] + assert "review_approach_candidates" in output_names, ( + "build-execution-map contract must register review_approach_candidates output" + ) + + +def test_skill_declares_max_parallel_argument() -> None: + """REQ-MAP-001: SKILL.md must document --max-parallel as an accepted input.""" + skill_md = _skill_md_text() + assert "--max-parallel" in skill_md + + +def test_skill_documents_max_parallel_default() -> None: + """REQ-MAP-002: Default of 6 must appear in the SKILL.md arguments section.""" + skill_md = _skill_md_text() + args_start = skill_md.find("## Arguments") + assert args_start != -1, "SKILL.md must have an ## Arguments section" + next_section = skill_md.find("\n##", args_start + 1) + args_section = ( + skill_md[args_start:next_section] if next_section != -1 else skill_md[args_start:] + ) + assert "default" in args_section.lower() and "6" in args_section, ( + "SKILL.md Arguments section must document the default value of 6 for --max-parallel" + ) + + +def test_output_schema_includes_max_parallel_field() -> None: + """REQ-OUT-001: JSON schema section must include max_parallel field.""" + skill_md = _skill_md_text() + schema_section_start = skill_md.find("## Output JSON Schema") + assert schema_section_start != -1 + schema_section = skill_md[schema_section_start:] + assert '"max_parallel"' in schema_section + + +def test_skill_documents_group_splitting_logic() -> None: + """REQ-MAP-003/004/005/006: Group splitting instructions must appear in Step 3.5.""" + skill_md = _skill_md_text() + step35_start = skill_md.find("Step 3.5") + assert step35_start != -1, "SKILL.md must document Step 3.5 group-splitting logic" + next_heading = skill_md.find("\n###", step35_start + 1) + step35 = skill_md[step35_start:next_heading] if next_heading != -1 else skill_md[step35_start:] + assert "split" in step35.lower(), "Step 3.5 section must describe splitting groups" + assert "sub-group" in step35.lower() or "subgroup" in step35.lower(), ( + "Step 3.5 section must describe sub-groups" + ) + + +def test_build_execution_map_new_tokens_registered() -> None: + """has_deferred, deferred_count, dispatched_count must appear in the BEM contract.""" + from autoskillit.recipe.contracts import get_skill_contract, load_bundled_manifest + + manifest = load_bundled_manifest() + contract = get_skill_contract("build-execution-map", manifest) + assert contract is not None + output_names = [o.name for o in contract.outputs] + assert "has_deferred" in output_names + assert "deferred_count" in output_names + assert "dispatched_count" in output_names + + +@pytest.fixture() +def sample_bem_json_with_deferrals() -> str: + return json.dumps( + { + "total_issues": 3, + "dispatched_count": 2, + "deferred_count": 1, + "has_deferred": True, + "groups": [ + { + "group": 1, + "parallel": True, + "issues": [{"number": 1155, "title": "A"}, {"number": 1156, "title": "B"}], + } + ], + "merge_order": [1155, 1156], + "in_progress_context": [{"number": 887, "title": "X"}], + "cross_assessments": [ + { + "target_issue": 1158, + "in_progress_issue": 887, + "conflict_severity": "critical", + "conflict_type": "undeclared_dependency", + "reasoning": "...", + "recommendation": "defer", + } + ], + "deferred_issues": [ + {"number": 1158, "title": "C", "blocked_by": [887], "reason": "..."} + ], + } + ) + + +def test_bem_count_invariant(sample_bem_json_with_deferrals: str) -> None: + data = json.loads(sample_bem_json_with_deferrals) + assert data["dispatched_count"] + data["deferred_count"] == data["total_issues"] + + +@pytest.fixture() +def sample_bem_json_with_critical_deferral() -> str: + return json.dumps( + { + "total_issues": 2, + "dispatched_count": 1, + "deferred_count": 1, + "has_deferred": True, + "groups": [{"group": 1, "parallel": True, "issues": [{"number": 1155, "title": "A"}]}], + "merge_order": [1155], + "in_progress_context": [{"number": 887, "title": "X"}], + "cross_assessments": [ + { + "target_issue": 1158, + "in_progress_issue": 887, + "conflict_severity": "critical", + "conflict_type": "undeclared_dependency", + "reasoning": "...", + "recommendation": "defer", + } + ], + "deferred_issues": [ + {"number": 1158, "title": "B", "blocked_by": [887], "reason": "..."} + ], + } + ) + + +def test_critical_cross_assessment_defers_issue( + sample_bem_json_with_critical_deferral: str, +) -> None: + data = json.loads(sample_bem_json_with_critical_deferral) + deferred_numbers = {d["number"] for d in data["deferred_issues"]} + grouped_numbers = {i["number"] for g in data["groups"] for i in g["issues"]} + assert deferred_numbers.isdisjoint(grouped_numbers), ( + "Deferred issues must not appear in any dispatch group" + ) + + +@pytest.fixture() +def sample_bem_json_no_in_progress() -> str: + return json.dumps( + { + "total_issues": 2, + "dispatched_count": 2, + "deferred_count": 0, + "has_deferred": False, + "groups": [ + { + "group": 1, + "parallel": True, + "issues": [{"number": 1155, "title": "A"}, {"number": 1156, "title": "B"}], + } + ], + "merge_order": [1155, 1156], + "in_progress_context": [], + "cross_assessments": [], + "deferred_issues": [], + } + ) + + +def test_no_in_progress_produces_no_deferrals(sample_bem_json_no_in_progress: str) -> None: + data = json.loads(sample_bem_json_no_in_progress) + assert data["has_deferred"] is False + assert data["deferred_count"] == 0 + assert data["dispatched_count"] == data["total_issues"] + assert data["deferred_issues"] == [] + assert data["in_progress_context"] == [] + assert data["cross_assessments"] == [] + + +def test_blocked_by_is_array(sample_bem_json_with_deferrals: str) -> None: + data = json.loads(sample_bem_json_with_deferrals) + for deferred in data["deferred_issues"]: + assert isinstance(deferred["blocked_by"], list), ( + f"blocked_by for issue {deferred['number']} must be an array" + ) + + +def test_bem_skillmd_has_cross_assessment_section() -> None: + content = _skill_md_text() + assert "cross_assessments" in content + assert "in_progress_context" in content + assert "deferred_issues" in content + assert "has_deferred" in content + + +def test_bem_contract_pattern_example_includes_new_tokens() -> None: + contracts_yaml = ( + Path(__file__).resolve().parent.parent.parent + / "src" + / "autoskillit" + / "recipe" + / "skill_contracts.yaml" + ) + raw = yaml.safe_load(contracts_yaml.read_text()) + skills = raw.get("skills", {}) + example = skills["build-execution-map"]["pattern_examples"][0] + assert "has_deferred" in example + assert "deferred_count" in example + assert "dispatched_count" in example diff --git a/tests/contracts/test_exogenous_string_coupling.py b/tests/contracts/test_exogenous_string_coupling.py new file mode 100644 index 0000000000..2c736011ed --- /dev/null +++ b/tests/contracts/test_exogenous_string_coupling.py @@ -0,0 +1,101 @@ +"""Exogenous string coupling tests. + +These tests assert that every string referenced in orchestrator prompts as a trigger +or routing condition is coupled to the module that actually emits it. If either side +drifts, these tests fail immediately — preventing silent recovery-path breakage. +""" + +from __future__ import annotations + +import re + + +def test_quota_guard_deny_trigger_coupled_to_prompt(): + """QUOTA_GUARD_DENY_TRIGGER constant in quota_guard must appear verbatim + in the orchestrator prompt's QUOTA DENIAL ROUTING section.""" + from autoskillit.cli._mcp_names import DIRECT_PREFIX + from autoskillit.cli._prompts import _build_orchestrator_prompt + from autoskillit.hooks.guards.quota_guard import QUOTA_GUARD_DENY_TRIGGER + + prompt = _build_orchestrator_prompt("test", mcp_prefix=DIRECT_PREFIX) + assert QUOTA_GUARD_DENY_TRIGGER in prompt, ( + f"Prompt must reference quota_guard.QUOTA_GUARD_DENY_TRIGGER " + f"({QUOTA_GUARD_DENY_TRIGGER!r}) verbatim" + ) + + +def test_quota_post_warning_trigger_coupled_to_prompt(): + """QUOTA_POST_WARNING_TRIGGER constant in quota_post_hook must appear verbatim + in the orchestrator prompt's QUOTA DENIAL ROUTING section.""" + from autoskillit.cli._mcp_names import DIRECT_PREFIX + from autoskillit.cli._prompts import _build_orchestrator_prompt + from autoskillit.hooks.quota_post_hook import QUOTA_POST_WARNING_TRIGGER + + prompt = _build_orchestrator_prompt("test", mcp_prefix=DIRECT_PREFIX) + assert QUOTA_POST_WARNING_TRIGGER in prompt, ( + f"Prompt must reference quota_post_hook.QUOTA_POST_WARNING_TRIGGER " + f"({QUOTA_POST_WARNING_TRIGGER!r}) verbatim" + ) + + +def test_quota_post_warning_trigger_coupled_to_sous_chef_skill(): + """QUOTA_POST_WARNING_TRIGGER constant must also appear in sous-chef SKILL.md + so both prompt surfaces stay in sync with the hook emit string.""" + from autoskillit.core import pkg_root + from autoskillit.hooks.quota_post_hook import QUOTA_POST_WARNING_TRIGGER + + skill_path = pkg_root() / "skills" / "sous-chef" / "SKILL.md" + assert skill_path.exists(), "sous-chef SKILL.md not found" + content = skill_path.read_text() + assert QUOTA_POST_WARNING_TRIGGER in content, ( + f"sous-chef SKILL.md must reference quota_post_hook.QUOTA_POST_WARNING_TRIGGER " + f"({QUOTA_POST_WARNING_TRIGGER!r}) verbatim" + ) + + +class TestPromptToolReachability: + """Tools referenced in FIRST ACTION must be registered in the FastMCP app.""" + + def test_prompt_tool_reachability(self): + """Each MCP tool name in FIRST ACTION must exist in the FastMCP tool registry.""" + from autoskillit.cli._prompts import _build_orchestrator_prompt + from autoskillit.core.types._type_constants import FREE_RANGE_TOOLS, GATED_TOOLS + + registered_tools = {*GATED_TOOLS, *FREE_RANGE_TOOLS} + + prompt = _build_orchestrator_prompt("test", "mcp__autoskillit__") + fa_start = prompt.index("FIRST ACTION") + fa_end = prompt.find("During pipeline execution", fa_start) + assert fa_end != -1, "'During pipeline execution' section not found after FIRST ACTION" + first_action = prompt[fa_start:fa_end] + + tool_names = set(re.findall(r"mcp__autoskillit__(\w+)\(", first_action)) + assert len(tool_names) > 0, "No MCP tool names found in FIRST ACTION" + + unregistered = [t for t in tool_names if t not in registered_tools] + assert unregistered == [], ( + f"FIRST ACTION references tool(s) not in the FastMCP registry: {unregistered}. " + f"Either add them to GATED_TOOLS/FREE_RANGE_TOOLS or remove from the prompt." + ) + + +class TestPromptToolsWhitelistCoupling: + """FIRST ACTION must not reference native tools blocked by --tools AskUserQuestion.""" + + def test_first_action_references_no_blocked_native_tools(self): + """FIRST ACTION must not mention any PIPELINE_FORBIDDEN_TOOLS by name.""" + from autoskillit.cli._mcp_names import DIRECT_PREFIX + from autoskillit.cli._prompts import _build_orchestrator_prompt + from autoskillit.core.types._type_constants import PIPELINE_FORBIDDEN_TOOLS + + prompt = _build_orchestrator_prompt("test", mcp_prefix=DIRECT_PREFIX) + start = prompt.index("FIRST ACTION") + end = prompt.find("During pipeline execution", start) + assert end != -1, "'During pipeline execution' section not found after FIRST ACTION" + first_action = prompt[start:end] + + found = [t for t in PIPELINE_FORBIDDEN_TOOLS if t in first_action] + assert found == [], ( + f"FIRST ACTION references forbidden native tools {found} that are blocked " + f"by --tools AskUserQuestion in _session_launch.py" + ) diff --git a/tests/contracts/test_filter_env_var_coverage.py b/tests/contracts/test_filter_env_var_coverage.py new file mode 100644 index 0000000000..2221b362b9 --- /dev/null +++ b/tests/contracts/test_filter_env_var_coverage.py @@ -0,0 +1,31 @@ +"""Tests that retry-worktree and audit-impl skills set filter env vars for test runs.""" + +from pathlib import Path + +import pytest + +SKILLS_EXTENDED = Path(__file__).resolve().parents[2] / "src" / "autoskillit" / "skills_extended" + + +@pytest.mark.parametrize("env_var", ["AUTOSKILLIT_TEST_FILTER", "AUTOSKILLIT_TEST_BASE_REF"]) +def test_retry_worktree_step4_sets_env_var(env_var: str) -> None: + """retry-worktree Step 4 must set AUTOSKILLIT_TEST_FILTER and AUTOSKILLIT_TEST_BASE_REF.""" + skill_md = (SKILLS_EXTENDED / "retry-worktree" / "SKILL.md").read_text() + assert "### Step 4" in skill_md + assert "### Step 5" in skill_md + step4_start = skill_md.index("### Step 4") + step5_start = skill_md.index("### Step 5") + step4_text = skill_md[step4_start:step5_start] + assert env_var in step4_text + + +@pytest.mark.parametrize("env_var", ["AUTOSKILLIT_TEST_FILTER", "AUTOSKILLIT_TEST_BASE_REF"]) +def test_audit_impl_remediation_template_sets_env_var(env_var: str) -> None: + """audit-impl Verification section must set AUTOSKILLIT_TEST_FILTER and TEST_BASE_REF.""" + skill_md = (SKILLS_EXTENDED / "audit-impl" / "SKILL.md").read_text() + assert "## Verification" in skill_md + verification_start = skill_md.index("## Verification") + assert "Then print:" in skill_md[verification_start:] + verification_end = skill_md.index("Then print:", verification_start) + verification_text = skill_md[verification_start:verification_end] + assert env_var in verification_text diff --git a/tests/contracts/test_fleet_dispatch_bem_gate.py b/tests/contracts/test_fleet_dispatch_bem_gate.py new file mode 100644 index 0000000000..d67cca2be3 --- /dev/null +++ b/tests/contracts/test_fleet_dispatch_bem_gate.py @@ -0,0 +1,62 @@ +"""Contract: fleet dispatcher prompt contains BEM pre-step gate instructions.""" + +from __future__ import annotations + +import pytest + +from autoskillit.cli._prompts_kitchen import _build_fleet_dispatch_prompt + +MCP_PREFIX = "autoskillit__" + + +@pytest.fixture(scope="module") +def fleet_prompt() -> str: + return _build_fleet_dispatch_prompt(mcp_prefix=MCP_PREFIX) + + +def test_fleet_prompt_contains_bem_wrapper_reference(fleet_prompt: str) -> None: + """Fleet dispatcher prompt must reference bem-wrapper for conflict analysis.""" + assert "bem-wrapper" in fleet_prompt, ( + "Fleet dispatcher prompt must instruct the dispatcher to use bem-wrapper " + "as the BEM pre-step for multi-issue dispatches" + ) + + +def test_fleet_prompt_contains_dispatch_plan_reference(fleet_prompt: str) -> None: + """Fleet dispatcher prompt must reference dispatch_plan from l3_payload.""" + assert "dispatch_plan" in fleet_prompt, ( + "Fleet dispatcher prompt must instruct the dispatcher to read dispatch_plan " + "from the bem-wrapper dispatch_food_truck response" + ) + + +def test_fleet_prompt_contains_multi_issue_gate_language(fleet_prompt: str) -> None: + """Fleet dispatcher prompt must contain multi-issue gate language.""" + lower = fleet_prompt.lower() + assert any(phrase in lower for phrase in ("multi-issue", "2 or more", "2+ issues")), ( + "Fleet dispatcher prompt must describe the multi-issue BEM gate trigger condition" + ) + + +def test_fleet_prompt_does_not_contain_stale_serial_guidance(fleet_prompt: str) -> None: + """Fleet dispatcher prompt must NOT contain stale serial-only dispatch guidance.""" + assert "dispatch one food truck at a time" not in fleet_prompt, ( + "Stale 'Serial execution: dispatch one food truck at a time' text must be " + "replaced by BEM-gated parallel dispatch guidance (REQ-BEM-002)" + ) + + +def test_fleet_prompt_references_fallback_to_sequential(fleet_prompt: str) -> None: + """Fleet dispatcher prompt must describe sequential fallback when BEM fails.""" + lower = fleet_prompt.lower() + assert "fallback" in lower or "fall back" in lower or "sequential" in lower, ( + "Fleet dispatcher prompt must describe the sequential fallback path when " + "bem-wrapper fails or returns an empty dispatch_plan" + ) + + +def test_fleet_prompt_references_max_total_issues_cap(fleet_prompt: str) -> None: + """Fleet dispatcher prompt must describe the max_total_issues cap.""" + assert "max_total_issues" in fleet_prompt, ( + "Fleet dispatcher prompt must mention the total issues session cap" + ) diff --git a/tests/contracts/test_hook_bridge_coverage.py b/tests/contracts/test_hook_bridge_coverage.py new file mode 100644 index 0000000000..308c678c9c --- /dev/null +++ b/tests/contracts/test_hook_bridge_coverage.py @@ -0,0 +1,50 @@ +""" +REQ-BRIDGE-001: The quota guard hook config bridge must produce exactly +the keys that resolve_quota_settings() reads. + +Analogous to test_config_field_coverage.py (REQ-CONFIG-001), which guards +dynaconf loading completeness, this test guards bridge serialization completeness. +Prevents silent omission when fields are added to QuotaGuardConfig that need +to cross the stdlib-only boundary. +""" + +from autoskillit.config.settings import QuotaGuardConfig +from autoskillit.hooks._hook_settings import QUOTA_GUARD_HOOK_PAYLOAD_KEYS +from autoskillit.server.tools.tools_kitchen import _quota_guard_hook_payload + + +def test_quota_guard_hook_payload_keys_match_payload_keys() -> None: + """_quota_guard_hook_payload() must produce exactly the keys that + resolve_quota_settings() reads from hook_config['quota_guard']. + + If this test fails after adding a field to QuotaGuardConfig or + QuotaHookSettings, update _quota_guard_hook_payload() and + QUOTA_GUARD_HOOK_PAYLOAD_KEYS together. + """ + cfg = QuotaGuardConfig() + payload = _quota_guard_hook_payload(cfg) + assert set(payload.keys()) == QUOTA_GUARD_HOOK_PAYLOAD_KEYS, ( + f"Bridge payload keys {set(payload.keys())} do not match " + f"payload keys {QUOTA_GUARD_HOOK_PAYLOAD_KEYS}. " + f"Update _quota_guard_hook_payload() and QUOTA_GUARD_HOOK_PAYLOAD_KEYS together." + ) + + +def test_quota_guard_hook_payload_enabled_true_produces_disabled_false() -> None: + cfg = QuotaGuardConfig(enabled=True) + payload = _quota_guard_hook_payload(cfg) + assert payload["disabled"] is False + + +def test_quota_guard_hook_payload_enabled_false_produces_disabled_true() -> None: + cfg = QuotaGuardConfig(enabled=False) + payload = _quota_guard_hook_payload(cfg) + assert payload["disabled"] is True + + +def test_quota_guard_hook_payload_bridges_cache_fields() -> None: + cfg = QuotaGuardConfig(cache_max_age=999, cache_path="/x/y.json", buffer_seconds=42) + payload = _quota_guard_hook_payload(cfg) + assert payload["cache_max_age"] == 999 + assert payload["cache_path"] == "/x/y.json" + assert payload["buffer_seconds"] == 42 diff --git a/tests/contracts/test_implement_experiment_contracts.py b/tests/contracts/test_implement_experiment_contracts.py index 5f2fe2b15c..5654c8e5e0 100644 --- a/tests/contracts/test_implement_experiment_contracts.py +++ b/tests/contracts/test_implement_experiment_contracts.py @@ -45,17 +45,25 @@ def test_implement_experiment_allows_pytest_collect_only() -> None: ) -def test_implement_experiment_step3_references_docker_build() -> None: - """Step 3 must instruct the agent to build a Docker image.""" +def test_implement_experiment_step3_references_env_mode() -> None: + """Step 3 must reference env_mode and write Dockerfile artifact, but NOT build.""" text = SKILL_PATH.read_text() step3_section = ( text.split("### Step 3")[1].split("### Step 4")[0] if "### Step 3" in text and "### Step 4" in text else text ) - assert "docker build" in step3_section.lower(), ( - "implement-experiment/SKILL.md Step 3 must reference 'docker build' — " - "Docker is the primary isolation mechanism for research execution" + assert "env_mode" in step3_section, ( + "implement-experiment/SKILL.md Step 3 must reference 'env_mode' — " + "the environment mode is set by setup-environment upstream" + ) + assert "Dockerfile" in step3_section, ( + "implement-experiment/SKILL.md Step 3 must still write a Dockerfile " + "template as a reproducibility artifact" + ) + assert "docker build" not in step3_section.lower(), ( + "implement-experiment/SKILL.md Step 3 must NOT invoke 'docker build' — " + "environment construction is handled by setup-environment" ) diff --git a/tests/contracts/test_instruction_surface.py b/tests/contracts/test_instruction_surface.py index 602f3bf5d6..9804dc2061 100644 --- a/tests/contracts/test_instruction_surface.py +++ b/tests/contracts/test_instruction_surface.py @@ -63,22 +63,24 @@ class TestServerToolSurfaceContract: """Server tool docstrings and prompts must name all forbidden tools.""" @pytest.fixture(autouse=True) - def _close_kitchen(self, tool_ctx, monkeypatch): + def _close_kitchen(self, minimal_ctx, monkeypatch): from autoskillit.pipeline.gate import DefaultGateState + from autoskillit.server import _state - monkeypatch.setattr(tool_ctx, "gate", DefaultGateState(enabled=False)) + monkeypatch.setattr(minimal_ctx, "gate", DefaultGateState(enabled=False)) + monkeypatch.setattr(_state, "_ctx", minimal_ctx) @pytest.mark.anyio async def test_open_kitchen_prompt_names_all_forbidden_tools(self): """open_kitchen tool text must name every forbidden tool with prohibition framing.""" from unittest.mock import AsyncMock, MagicMock, patch - from autoskillit.server.tools_kitchen import open_kitchen + from autoskillit.server.tools.tools_kitchen import open_kitchen mock_ctx = MagicMock() mock_ctx.enable_components = AsyncMock() - with patch("autoskillit.server.tools_kitchen._prime_quota_cache", new=AsyncMock()): - with patch("autoskillit.server.tools_kitchen._write_hook_config"): + with patch("autoskillit.server.tools.tools_kitchen._prime_quota_cache", new=AsyncMock()): + with patch("autoskillit.server.tools.tools_kitchen._write_hook_config"): text = await open_kitchen(ctx=mock_ctx) missing = [t for t in PIPELINE_FORBIDDEN_TOOLS if t not in text] @@ -89,7 +91,7 @@ async def test_open_kitchen_prompt_names_all_forbidden_tools(self): def test_run_skill_docstring_names_all_forbidden_tools(self): """run_skill docstring must name every forbidden tool.""" - from autoskillit.server.tools_execution import run_skill + from autoskillit.server.tools.tools_execution import run_skill doc = run_skill.__doc__ assert doc, "run_skill has no docstring" @@ -98,7 +100,7 @@ def test_run_skill_docstring_names_all_forbidden_tools(self): def test_load_recipe_docstring_names_all_forbidden_tools(self): """load_recipe docstring must name every forbidden tool.""" - from autoskillit.server.tools_recipe import load_recipe + from autoskillit.server.tools.tools_recipe import load_recipe doc = load_recipe.__doc__ assert doc, "load_skill_script has no docstring" @@ -293,7 +295,7 @@ class TestQuotaGuardStructuralEnforcement: def test_load_recipe_has_no_quota_guard_instructions(self): """Quota guard enforcement is structural (hook), not instructional (docstring).""" - from autoskillit.server.tools_recipe import load_recipe + from autoskillit.server.tools.tools_recipe import load_recipe docstring = load_recipe.__doc__ or "" assert "QUOTA GUARD" not in docstring, ( @@ -317,14 +319,15 @@ def test_clone_using_recipes_have_source_isolation_rule(self): clone_recipes = [] for wf_info in bundled: raw = wf_info.path.read_text() - if "clone_repo" not in raw: + if "clone_repo" not in raw and "bootstrap_clone" not in raw: continue clone_recipes.append(wf_info.name) wf = load_recipe(wf_info.path) assert wf.kitchen_rules, f"{wf_info.name} has no kitchen_rules" all_rules = " ".join(wf.kitchen_rules) assert self._SENTINEL in all_rules, ( - f"{wf_info.name} uses clone_repo but kitchen_rules lack '{self._SENTINEL}'" + f"{wf_info.name} uses clone_repo/bootstrap_clone but " + f"kitchen_rules lack '{self._SENTINEL}'" ) assert "checkout" in all_rules.lower(), ( f"{wf_info.name} SOURCE ISOLATION rule must explicitly mention 'checkout'" @@ -336,7 +339,7 @@ def test_clone_using_recipes_have_source_isolation_rule(self): def test_clone_repo_tool_docstring_has_source_isolation(self): """clone_repo MCP tool docstring must include SOURCE ISOLATION prohibition.""" - from autoskillit.server.tools_clone import clone_repo + from autoskillit.server.tools.tools_clone import clone_repo doc = clone_repo.__doc__ or "" assert self._SENTINEL in doc, "clone_repo docstring must contain 'SOURCE ISOLATION'" @@ -354,10 +357,15 @@ def test_workspace_clone_module_has_source_isolation(self): ) def test_git_mutating_recipes_have_clone_step(self): - """Recipes using MCP git-mutation tools must use clone_repo.""" + """Recipes using MCP git-mutation tools must use clone_repo or bootstrap_clone.""" from autoskillit.recipe.io import list_recipes, load_recipe - GIT_MUTATION_TOOLS = {"create_unique_branch", "push_to_remote"} + CLONE_TOOLS = {"clone_repo", "bootstrap_clone"} + GIT_MUTATION_TOOLS = { + "create_unique_branch", + "push_to_remote", + "create_and_publish_branch", + } workflows = list_recipes(Path("/nonexistent")) bundled = [w for w in workflows.items if w.source.value == "builtin"] for wf_info in bundled: @@ -366,13 +374,14 @@ def test_git_mutating_recipes_have_clone_step(self): if not uses_mutation_tool: continue has_clone = any( - step.tool == "clone_repo" or (step.python and "clone_repo" in step.python) + step.tool in CLONE_TOOLS + or (step.python and any(t in step.python for t in CLONE_TOOLS)) for step in wf.steps.values() ) assert has_clone, ( f"{wf_info.name} uses MCP git-mutation tools " f"({GIT_MUTATION_TOOLS & {s.tool for s in wf.steps.values()}}) " - f"but never calls clone_repo — workspace isolation is missing." + f"but never calls clone_repo/bootstrap_clone — workspace isolation is missing." ) @@ -432,6 +441,15 @@ def test_sous_chef_routes_conflict_to_on_failure(self): "sous-chef/SKILL.md must state that merge conflicts route to on_failure" ) + def test_sous_chef_contains_skill_command_format_guidance(self): + """sous-chef/SKILL.md must contain skill_command format discipline.""" + content = self._sous_chef_text() + assert "SKILL_COMMAND FORMATTING" in content, ( + "sous-chef/SKILL.md must contain a SKILL_COMMAND FORMATTING section. " + "This is the persistent behavioral guard against LLM document-formatting " + "of skill_command." + ) + class TestPathArgSkillsContract: """Path-argument skills must document path-detection parsing in their SKILL.md.""" @@ -458,6 +476,43 @@ def test_path_arg_skills_have_path_detection_instructions(self): ) +class TestSkillCommandParsingContract: + """skill_cmd_guard._PATH_PREFIXES must match core._type_helpers._PATH_PREFIXES.""" + + def _load_hook_module(self): + import importlib.util + import pathlib + + hook_path = ( + pathlib.Path(__file__).parents[2] + / "src" + / "autoskillit" + / "hooks" + / "guards" + / "skill_cmd_guard.py" + ) + spec = importlib.util.spec_from_file_location("skill_cmd_guard", hook_path) + mod = importlib.util.module_from_spec(spec) + spec.loader.exec_module(mod) + return mod + + def test_hook_path_prefixes_match_core(self): + from autoskillit.core.types._type_helpers import _PATH_PREFIXES as core_prefixes + + mod = self._load_hook_module() + assert set(mod._PATH_PREFIXES) == set(core_prefixes), ( + f"skill_cmd_guard._PATH_PREFIXES {set(mod._PATH_PREFIXES)!r} " + f"diverges from core._type_helpers._PATH_PREFIXES {set(core_prefixes)!r}." + ) + + def test_hook_path_arg_skills_matches_contract_list(self): + mod = self._load_hook_module() + assert set(mod.PATH_ARG_SKILLS) == set(TestPathArgSkillsContract.PATH_ARG_SKILLS), ( + "TestPathArgSkillsContract.PATH_ARG_SKILLS is out of sync with " + "skill_cmd_guard.PATH_ARG_SKILLS. Update the hardcoded list." + ) + + class TestResolveFailuresDirtyTreeContract: """resolve-failures SKILL.md must document dirty tree pre-check.""" @@ -573,7 +628,8 @@ def test_claude_md_documents_all_source_modules() -> None: """Every .py file in src/autoskillit/ must appear by name in CLAUDE.md. For __init__.py files, the containing package directory name must appear. - For all other files, the filename must appear somewhere in CLAUDE.md. + For all other files, the filename must appear in CLAUDE.md or the package's + own sub-CLAUDE.md (for collapsed subdirectory listings). """ claude_path = Path(__file__).parent.parent.parent / "CLAUDE.md" content = claude_path.read_text() @@ -591,7 +647,10 @@ def test_claude_md_documents_all_source_modules() -> None: missing.append(str(rel)) else: if py_file.name not in content: - missing.append(str(rel)) + # Accept documentation in a sub-CLAUDE.md alongside the file + pkg_claude = py_file.parent / "CLAUDE.md" + if not (pkg_claude.exists() and py_file.name in pkg_claude.read_text()): + missing.append(str(rel)) assert not missing, ( f"Modules not documented in CLAUDE.md: {', '.join(missing)}. " diff --git a/tests/contracts/test_issue_splitter_contracts.py b/tests/contracts/test_issue_splitter_contracts.py index 34a1a2d6f1..8a75e27d60 100644 --- a/tests/contracts/test_issue_splitter_contracts.py +++ b/tests/contracts/test_issue_splitter_contracts.py @@ -98,6 +98,20 @@ def test_triage_issues_calls_issue_splitter() -> None: assert "issue-splitter" in text +def test_issue_splitter_no_issue_comment() -> None: + text = skill_text("issue-splitter") + assert "gh issue comment" not in text, ( + "Tracking info must be appended to parent body, not posted as a comment" + ) + + +def test_issue_splitter_decomposed_section() -> None: + text = skill_text("issue-splitter") + assert "## Decomposed" in text, ( + "Must append ## Decomposed section listing sub-issues to parent body" + ) + + # 13. Split step appears before classification step in triage-issues def test_split_step_before_classification_in_triage() -> None: text = skill_text("triage-issues") diff --git a/tests/contracts/test_make_campaign_skill_contracts.py b/tests/contracts/test_make_campaign_skill_contracts.py new file mode 100644 index 0000000000..005d34fe88 --- /dev/null +++ b/tests/contracts/test_make_campaign_skill_contracts.py @@ -0,0 +1,120 @@ +"""Contract tests: structural invariants for the make-campaign SKILL.md.""" + +from __future__ import annotations + +import re +from typing import Any + +import yaml + +from autoskillit.core import pkg_root +from autoskillit.workspace.skills import DefaultSkillResolver + +_SKILL_DIR = pkg_root() / "skills_extended" / "make-campaign" +_SKILL_MD = _SKILL_DIR / "SKILL.md" + + +def _read_skill_md() -> str: + return _SKILL_MD.read_text() + + +def _parse_frontmatter(content: str) -> dict[str, Any]: + """Extract YAML frontmatter between --- delimiters.""" + m = re.match(r"^---\n(.+?)\n---\n", content, re.DOTALL) + assert m, "SKILL.md must have YAML frontmatter delimited by ---" + parsed = yaml.safe_load(m.group(1)) + assert isinstance(parsed, dict), "SKILL.md frontmatter must be a YAML mapping" + return parsed + + +def test_make_campaign_skill_discovered(): + """make-campaign appears in DefaultSkillResolver.list_all().""" + resolver = DefaultSkillResolver() + names = [s.name for s in resolver.list_all()] + assert "make-campaign" in names, ( + "make-campaign not discovered by DefaultSkillResolver.list_all(). " + "Ensure the directory is under skills_extended/ and SKILL.md name field matches." + ) + + +def test_make_campaign_frontmatter_valid(): + """Frontmatter has name='make-campaign', categories=['orchestration-family'].""" + content = _read_skill_md() + fm = _parse_frontmatter(content) + assert fm.get("name") == "make-campaign", ( + f"Expected name='make-campaign', got {fm.get('name')!r}" + ) + assert "orchestration-family" in fm.get("categories", []), ( + f"Expected categories to include 'orchestration-family', got {fm.get('categories')!r}" + ) + + +def test_make_campaign_has_six_phases(): + """SKILL.md body contains Phase 1 through Phase 6 headings or numbered sections.""" + content = _read_skill_md() + for phase_num in range(1, 7): + assert re.search(rf"Phase\s+{phase_num}\b", content, re.IGNORECASE), ( + f"SKILL.md missing Phase {phase_num} section" + ) + + +def test_make_campaign_output_path_convention(): + """Output instructions reference .autoskillit/recipes/campaigns/.yaml.""" + content = _read_skill_md() + assert ".autoskillit/recipes/campaigns/" in content, ( + "SKILL.md must reference .autoskillit/recipes/campaigns/ as the output path" + ) + + +def test_make_campaign_declares_structured_output_token(): + """SKILL.md declares a campaign_path structured output token.""" + content = _read_skill_md() + assert "campaign_path" in content, ( + "SKILL.md must declare a campaign_path structured output token" + ) + assert re.search(r"campaign_path\s*=", content), ( + "SKILL.md must show campaign_path = token emission" + ) + + +def test_make_campaign_no_code_modification_instructions(): + """SKILL.md never instructs to modify source code files.""" + content = _read_skill_md() + # Strip frontmatter before checking + body = re.sub(r"^---\n.+?\n---\n", "", content, flags=re.DOTALL) + code_mod_patterns = [ + r"\bmodify\s+source\s+code\b", + r"\bedit\s+\.py\s+files\b", + r"\bwrite\s+to\s+src/\b", + ] + for pattern in code_mod_patterns: + assert not re.search(pattern, body, re.IGNORECASE), ( + f"SKILL.md must not instruct modifying source code. Found pattern: {pattern!r}" + ) + # Critical Constraints section must include NEVER modify source code + constraints_m = re.search( + r"##\s+Critical Constraints\b(.+?)(?=\n##\s|\Z)", body, re.DOTALL | re.IGNORECASE + ) + assert constraints_m, "SKILL.md must have a '## Critical Constraints' section" + constraints_section = constraints_m.group(1) + assert re.search( + r"NEVER\b.{0,300}modify.{0,50}source\s+code", + constraints_section, + re.IGNORECASE | re.DOTALL, + ), "SKILL.md Critical Constraints section must state NEVER modify source code files" + + +def test_make_campaign_invokes_validation(): + """SKILL.md body references recipe validation (validate_recipe or validate_from_path).""" + content = _read_skill_md() + assert re.search(r"validate_recipe|validate_from_path", content), ( + "SKILL.md must reference validate_recipe or validate_from_path for campaign validation" + ) + + +def test_make_campaign_references_recipe_discovery(): + """SKILL.md references list_recipes or find_recipe_by_name for dispatch decomposition.""" + content = _read_skill_md() + assert re.search(r"list_recipes|find_recipe_by_name", content), ( + "SKILL.md must reference list_recipes or find_recipe_by_name for recipe discovery" + ) diff --git a/tests/contracts/test_no_pagination_file_read.py b/tests/contracts/test_no_pagination_file_read.py new file mode 100644 index 0000000000..82f69db8e3 --- /dev/null +++ b/tests/contracts/test_no_pagination_file_read.py @@ -0,0 +1,55 @@ +"""Contract guards for no-pagination file read instruction in high-turn skills.""" + +from pathlib import Path + +import pytest + +SKILLS_ROOT = ( + Path(__file__).resolve().parent.parent.parent / "src" / "autoskillit" / "skills_extended" +) + +TARGET_SKILLS = [ + "implement-worktree-no-merge", + "resolve-failures", + "retry-worktree", +] + + +@pytest.fixture(params=TARGET_SKILLS, ids=TARGET_SKILLS) +def skill_text(request: pytest.FixtureRequest) -> str: + path = SKILLS_ROOT / request.param / "SKILL.md" + assert path.exists(), f"SKILL.md not found at {path}" + return path.read_text() + + +def _extract_always_block(text: str) -> str: + """Extract the ALWAYS block from Critical Constraints, or "" if absent.""" + always_idx = text.find("**ALWAYS:**") + if always_idx == -1: + return "" + next_section = text.find("\n## ", always_idx) + return text[always_idx:next_section] if next_section != -1 else text[always_idx:] + + +def test_no_pagination_instruction_present(skill_text: str) -> None: + """The ALWAYS block must contain the no-pagination file read instruction.""" + always_block = _extract_always_block(skill_text) + assert "single call without a `limit` parameter" in always_block, ( + "ALWAYS block must instruct reading files in a single call without limit" + ) + + +def test_no_pagination_instruction_prohibits_sequential_offset(skill_text: str) -> None: + """The instruction must explicitly prohibit sequential offset reads.""" + always_block = _extract_always_block(skill_text) + assert "Do not paginate" in always_block, ( + "ALWAYS block must explicitly prohibit paginated sequential offset reads" + ) + + +def test_no_pagination_instruction_permits_targeted_reads(skill_text: str) -> None: + """The instruction must permit targeted limit/offset for known files.""" + always_block = _extract_always_block(skill_text) + assert "targeted section reads" in always_block, ( + "ALWAYS block must permit limit/offset for targeted reads of already-read files" + ) diff --git a/tests/contracts/test_package_gateways.py b/tests/contracts/test_package_gateways.py index a28f27a55b..757b47a735 100644 --- a/tests/contracts/test_package_gateways.py +++ b/tests/contracts/test_package_gateways.py @@ -217,7 +217,10 @@ def test_factory_make_context_returns_toolcontext(monkeypatch): assert ctx.gate.enabled is False # starts closed assert isinstance(ctx.audit, DefaultAuditLog) assert ctx.token_log is not None - assert ctx.plugin_dir == str(pkg_root()) + from autoskillit.core.types._type_plugin_source import DirectInstall + + assert isinstance(ctx.plugin_source, DirectInstall) + assert ctx.plugin_source.plugin_dir == pkg_root() def test_factory_make_context_accepts_runner(): @@ -230,10 +233,12 @@ def test_factory_make_context_accepts_runner(): def test_factory_make_context_accepts_plugin_dir(tmp_path): from autoskillit.config import AutomationConfig + from autoskillit.core.types._type_plugin_source import DirectInstall from autoskillit.server._factory import make_context ctx = make_context(AutomationConfig(), plugin_dir=str(tmp_path)) - assert ctx.plugin_dir == str(tmp_path) + assert isinstance(ctx.plugin_source, DirectInstall) + assert ctx.plugin_source.plugin_dir == tmp_path # --------------------------------------------------------------------------- @@ -324,10 +329,12 @@ def test_package_all_matches_exports() -> None: "core", "config", "pipeline", + "planner", "execution", "workspace", "recipe", "migration", + "fleet", "hooks", "cli", "server", @@ -386,6 +393,7 @@ def test_root_module_allowlist() -> None: "__init__.py", "__main__.py", "_llm_triage.py", + "_test_filter.py", "hook_registry.py", "smoke_utils.py", "version.py", diff --git a/tests/contracts/test_prepare_issue_contracts.py b/tests/contracts/test_prepare_issue_contracts.py index 59d7de3442..d3614a8099 100644 --- a/tests/contracts/test_prepare_issue_contracts.py +++ b/tests/contracts/test_prepare_issue_contracts.py @@ -161,7 +161,8 @@ def test_prepare_issue_dedup_shows_all_candidates(): dedup_start = text.find("### Step 4: Dedup Check") assert dedup_start != -1, "SKILL.md must contain Step 4 dedup section" dedup_end = text.find("### Step 4a:", dedup_start) - dedup_section = text[dedup_start:dedup_end] if dedup_end != -1 else text[dedup_start:] + assert dedup_end != -1, "Step 4a section not found" + dedup_section = text[dedup_start:dedup_end] # The option-menu block must use the [1]–[{N}] indexed format within the dedup section assert "[1]" in dedup_section and "[{N}]" in dedup_section, ( "Dedup must display candidates in [1]–[{N}] indexed format within Step 4" @@ -188,7 +189,8 @@ def test_prepare_issue_dedup_extend_runs_triage(): dedup_start = text.find("### Step 4: Dedup Check") assert dedup_start != -1, "SKILL.md must contain Step 4 dedup section" dedup_end = text.find("### Step 4a:", dedup_start) - dedup_section = text[dedup_start:dedup_end] if dedup_end != -1 else text[dedup_start:] + assert dedup_end != -1, "Step 4a section not found" + dedup_section = text[dedup_start:dedup_end] assert "extend" in dedup_section.lower(), "Step 4 dedup section must document the extend path" # The extend path within the dedup section must reference continuing to Step 6 has_triage_ref = ( @@ -371,3 +373,203 @@ def test_prepare_issue_no_tmp_paths(): "prepare-issue uses /tmp/ path(s). All temp files must live in " "{{AUTOSKILLIT_TEMP}}/prepare-issue/ per project convention" ) + + +def test_prepare_issue_no_issue_comment(): + text = SKILL_MD.read_text() + assert "gh issue comment" not in text, ( + "Comment option must be removed; always edit the issue body" + ) + + +# --------------------------------------------------------------------------- +# STATE-AWARE DEDUP: Issue #402 — lifecycle-aware extend eligibility +# --------------------------------------------------------------------------- + + +def test_dedup_searches_all_states(): + """Step 4 dedup must search all issue states, not only open.""" + text = SKILL_MD.read_text() + dedup_start = text.find("### Step 4: Dedup Check") + assert dedup_start != -1, "Step 4 dedup section not found" + dedup_end = text.find("### Step 4a:", dedup_start) + assert dedup_end != -1, "Step 4a section not found" + dedup_section = text[dedup_start:dedup_end] + # --state open must not be the only state directive in the dedup section + assert "--state open" not in dedup_section, ( + "Dedup search must not be limited to --state open; must use --state all " + "or a combined open+closed search to surface closed duplicate candidates" + ) + assert "--state all" in dedup_section, ( + "Dedup search must explicitly use --state all to surface closed duplicate candidates" + ) + + +def test_dedup_gh_issue_list_includes_labels_field(): + """gh issue list in Step 4 must request the labels field.""" + text = SKILL_MD.read_text() + dedup_start = text.find("### Step 4: Dedup Check") + assert dedup_start != -1, "Step 4 dedup section not found" + dedup_end = text.find("### Step 4a:", dedup_start) + assert dedup_end != -1, "Step 4a section not found" + dedup_section = text[dedup_start:dedup_end] + assert "labels" in dedup_section, ( + "gh issue list in Step 4 must include 'labels' in --json fields to enable " + "extend-eligibility classification" + ) + + +def test_dedup_gh_issue_list_includes_state_field(): + """gh issue list in Step 4 must request the state field.""" + text = SKILL_MD.read_text() + dedup_start = text.find("### Step 4: Dedup Check") + assert dedup_start != -1, "Step 4 dedup section not found" + dedup_end = text.find("### Step 4a:", dedup_start) + assert dedup_end != -1, "Step 4a section not found" + dedup_section = text[dedup_start:dedup_end] + # 'state' must appear as a --json field (ordering-independent) + assert re.search(r"--json\s+[^\n]*\bstate\b", dedup_section), ( + "gh issue list in Step 4 must include 'state' in --json fields" + ) + + +def test_dedup_informational_marker_documented(): + """Step 4 dedup display must document the [—] marker for non-extend-eligible candidates.""" + text = SKILL_MD.read_text() + dedup_start = text.find("### Step 4: Dedup Check") + assert dedup_start != -1, "Step 4 dedup section not found" + dedup_end = text.find("### Step 4a:", dedup_start) + assert dedup_end != -1, "Step 4a section not found" + dedup_section = text[dedup_start:dedup_end] + assert "[—]" in dedup_section, ( + "Step 4 dedup display must include the [—] marker for informational-only " + "(non-extend-eligible) candidates" + ) + + +def test_dedup_extend_not_offered_for_in_progress(): + """Step 4 must document that in-progress labeled issues are shown but not extend-eligible.""" + text = SKILL_MD.read_text() + dedup_start = text.find("### Step 4: Dedup Check") + assert dedup_start != -1, "Step 4 dedup section not found" + dedup_end = text.find("### Step 4a:", dedup_start) + assert dedup_end != -1, "Step 4a section not found" + dedup_section = text[dedup_start:dedup_end] + assert "in-progress" in dedup_section, ( + "Step 4 must document that issues with the in-progress label are shown " + "as dedup candidates but are not extend-eligible" + ) + + +def test_dedup_extend_not_offered_for_staged(): + """Step 4 must document that staged labeled issues are shown but not extend-eligible.""" + text = SKILL_MD.read_text() + dedup_start = text.find("### Step 4: Dedup Check") + assert dedup_start != -1, "Step 4 dedup section not found" + dedup_end = text.find("### Step 4a:", dedup_start) + assert dedup_end != -1, "Step 4a section not found" + dedup_section = text[dedup_start:dedup_end] + assert "staged" in dedup_section, ( + "Step 4 must document that issues with the staged label are shown " + "as dedup candidates but are not extend-eligible" + ) + + +def test_dedup_extend_not_offered_for_closed(): + """Step 4 must document that closed issues are shown but not extend-eligible.""" + text = SKILL_MD.read_text() + dedup_start = text.find("### Step 4: Dedup Check") + assert dedup_start != -1, "Step 4 dedup section not found" + dedup_end = text.find("### Step 4a:", dedup_start) + assert dedup_end != -1, "Step 4a section not found" + dedup_section = text[dedup_start:dedup_end] + assert "closed" in dedup_section.lower(), ( + "Step 4 must document that closed issues are shown as dedup candidates " + "but are not extend-eligible" + ) + + +def test_dedup_state_annotation_per_candidate(): + """Step 4 candidate display must show a state annotation per candidate.""" + text = SKILL_MD.read_text() + dedup_start = text.find("### Step 4: Dedup Check") + assert dedup_start != -1, "Step 4 dedup section not found" + dedup_end = text.find("### Step 4a:", dedup_start) + assert dedup_end != -1, "Step 4a section not found" + dedup_section = text[dedup_start:dedup_end] + # The display block must include state annotations like (open), (in-progress), etc. + has_annotation = ( + "(open)" in dedup_section + or "(in-progress)" in dedup_section + or "(closed)" in dedup_section + ) + assert has_annotation, ( + "Step 4 candidate display must show per-candidate state annotations " + "such as (open), (in-progress), or (closed)" + ) + + +def test_dedup_closed_issues_stricter_relevance(): + """Step 4 must document stricter relevance criteria for closed issue candidates.""" + text = SKILL_MD.read_text() + dedup_start = text.find("### Step 4: Dedup Check") + assert dedup_start != -1, "Step 4 dedup section not found" + dedup_end = text.find("### Step 4a:", dedup_start) + assert dedup_end != -1, "Step 4a section not found" + dedup_section = text[dedup_start:dedup_end] + # Must reference stricter matching for closed issues + stricter_signals = ["stricter", "higher threshold", "multiple keyword", "2+", "two or more"] + assert any(s in dedup_section.lower() for s in stricter_signals), ( + "Step 4 must document applying stricter relevance criteria for closed issue " + "candidates before showing them (e.g., require 2+ keyword set matches)" + ) + + +# --------------------------------------------------------------------------- +# TRIAGE MISCLASSIFICATION: Gap 1a — validated report shortcut in Step 6 +# --------------------------------------------------------------------------- + + +def test_step6_validated_report_shortcut_to_implementation(): + """Step 6 must short-circuit to implementation when is_validated_report is true.""" + text = SKILL_MD.read_text(encoding="utf-8") + step6_pos = text.find("### Step 6") + assert step6_pos != -1, "Step 6 not found" + step7_pos = text.find("### Step 7", step6_pos) + step6_section = ( + text[step6_pos:step7_pos] if step7_pos != -1 else text[step6_pos : step6_pos + 2000] + ) + assert "is_validated_report" in step6_section, ( + "Step 6 must check is_validated_report and short-circuit to implementation route" + ) + # Must reference implementation in the validated-report shortcut, not only remediation + shortcut_idx = step6_section.find("is_validated_report") + assert shortcut_idx != -1, "is_validated_report not found in Step 6 section" + next_heading = step6_section.find("###", shortcut_idx) + shortcut_block = ( + step6_section[shortcut_idx:next_heading] + if next_heading != -1 + else step6_section[shortcut_idx:] + ) + assert "implementation" in shortcut_block, ( + "Step 6 is_validated_report shortcut must assign route=implementation" + ) + + +def test_step6_heuristic_table_has_audit_signal_row(): + """Step 6 heuristic table must include an explicit row for validated audit reports.""" + text = SKILL_MD.read_text() + step6_pos = text.find("### Step 6") + assert step6_pos != -1, "Step 6 not found" + step7_pos = text.find("### Step 7", step6_pos) + step6_section = ( + text[step6_pos:step7_pos] if step7_pos != -1 else text[step6_pos : step6_pos + 2000] + ) + # Either the shortcut or an explicit audit row must map validated reports to implementation + has_audit_implementation_signal = ( + "audit" in step6_section.lower() and "implementation" in step6_section + ) + assert has_audit_implementation_signal, ( + "Step 6 must document that validated audit reports (no tracebacks, structural scope) " + "route to implementation — not fall through to the large/ambiguous-scope remediation row" + ) diff --git a/tests/contracts/test_process_issues_contracts.py b/tests/contracts/test_process_issues_contracts.py index bc2dcda139..c144d1dead 100644 --- a/tests/contracts/test_process_issues_contracts.py +++ b/tests/contracts/test_process_issues_contracts.py @@ -80,6 +80,22 @@ def test_process_issues_derives_issue_url(skill_text: str) -> None: assert github_url_ref in skill_text or "default_repo" in skill_text +def test_process_issues_has_step_execution_discipline(skill_text: str) -> None: + assert "execute every step" in skill_text.lower() or "MUST execute" in skill_text + assert "never skip" in skill_text.lower() or "NEVER skip" in skill_text + assert "optional: true" in skill_text + + +def test_process_issues_has_anti_improvisation_rule(skill_text: str) -> None: + assert "run_cmd" in skill_text and "gh pr create" in skill_text + + +def test_process_issues_no_issue_comment(skill_text: str) -> None: + assert "gh issue comment" not in skill_text, ( + "--comment flag must be removed; status updates go to issue body" + ) + + def test_prepare_pr_supports_run_name_title_prefix() -> None: """prepare-pr must derive [FEATURE]/[FIX] PR title prefix from run_name convention.""" content = (bundled_skills_extended_dir() / "prepare-pr" / "SKILL.md").read_text() diff --git a/tests/contracts/test_protocol_definitions.py b/tests/contracts/test_protocol_definitions.py index c60599ed18..877b1f417a 100644 --- a/tests/contracts/test_protocol_definitions.py +++ b/tests/contracts/test_protocol_definitions.py @@ -1,6 +1,6 @@ -"""Tests for Protocol definitions in core/_type_protocols.py. +"""Tests for Protocol definitions in core/_type_protocols_*.py shards. -REQ-PROTO-007: SkillLister Protocol must live in core/_type_protocols.py. +REQ-PROTO-007: SkillLister Protocol must live in core/_type_protocols_workspace.py. """ from __future__ import annotations @@ -8,12 +8,31 @@ import inspect +def test_github_fetcher_has_update_issue_body() -> None: + from autoskillit.core.types._type_protocols_github import GitHubFetcher + + assert hasattr(GitHubFetcher, "update_issue_body") + sig = inspect.signature(GitHubFetcher.update_issue_body) + assert "owner" in sig.parameters + assert "repo" in sig.parameters + assert "issue_number" in sig.parameters + assert "new_body" in sig.parameters + + +def test_github_fetcher_no_add_comment() -> None: + from autoskillit.core.types._type_protocols_github import GitHubFetcher + + assert not hasattr(GitHubFetcher, "add_comment"), ( + "add_comment must be removed after all call sites are migrated" + ) + + def test_skill_lister_protocol_defined() -> None: """REQ-PROTO-007: SkillLister Protocol must live in - core/_type_protocols.py and define a `list_all() -> list[Any]` + core/_type_protocols_workspace.py and define a `list_all() -> list[Any]` method, so L2 recipe code can type-annotate against L0 instead of binding to the L1 workspace concrete class.""" - from autoskillit.core._type_protocols import SkillLister + from autoskillit.core.types._type_protocols_workspace import SkillLister assert hasattr(SkillLister, "list_all") sig = inspect.signature(SkillLister.list_all) @@ -21,8 +40,21 @@ def test_skill_lister_protocol_defined() -> None: def test_skill_resolver_satisfies_skill_lister() -> None: - from autoskillit.core._type_protocols import SkillLister + from autoskillit.core.types._type_protocols_workspace import SkillLister from autoskillit.workspace.skills import DefaultSkillResolver instance: SkillLister = DefaultSkillResolver() assert callable(instance.list_all) + + +def test_recipe_repository_load_and_validate_project_dir_annotation() -> None: + import typing + from pathlib import Path + + from autoskillit.core.types._type_protocols_recipe import RecipeRepository + + hints = typing.get_type_hints(RecipeRepository.load_and_validate) + ann = hints["project_dir"] + args = typing.get_args(ann) + assert Path in args, f"Expected Path in annotation args, got: {ann}" + assert str in args, f"Expected str in annotation args, got: {ann}" diff --git a/tests/contracts/test_protocol_satisfaction.py b/tests/contracts/test_protocol_satisfaction.py index c8442a1453..82f7d695d8 100644 --- a/tests/contracts/test_protocol_satisfaction.py +++ b/tests/contracts/test_protocol_satisfaction.py @@ -175,6 +175,39 @@ def test_default_recipe_repository_satisfies_recipe_repository(): assert isinstance(DefaultRecipeRepository(), RecipeRepository) +def test_recipe_repository_has_apply_triage_gate(): + """RecipeRepository protocol defines apply_triage_gate as an async method.""" + import inspect + + from autoskillit.core import RecipeRepository + + method = getattr(RecipeRepository, "apply_triage_gate", None) + assert method is not None, "RecipeRepository must define apply_triage_gate" + assert inspect.iscoroutinefunction(method), "apply_triage_gate must be async" + sig = inspect.signature(method) + expected_params = [ + "self", + "result", + "recipe_name", + "recipe_info", + "temp_dir", + "logger", + "triage_fn", + ] + assert list(sig.parameters.keys()) == expected_params + + +def test_default_recipe_repository_has_apply_triage_gate(): + """DefaultRecipeRepository implements apply_triage_gate from the protocol.""" + import inspect + + from autoskillit.recipe.repository import DefaultRecipeRepository + + method = getattr(DefaultRecipeRepository, "apply_triage_gate", None) + assert method is not None + assert inspect.iscoroutinefunction(method) + + def test_default_migration_service_satisfies_migration_service(): from autoskillit.core import MigrationService from autoskillit.migration.engine import DefaultMigrationService, MigrationEngine @@ -306,6 +339,9 @@ def test_req_api_001_public_params_present(self): "linux_tracing_config", "idle_output_timeout", "max_suppression_seconds", + "on_pid_resolved", + "enable_deadline_extension", + "max_extension_seconds", } assert expected == public_params, ( f"run_managed_async public params changed.\n" @@ -367,6 +403,9 @@ def test_req_api_002_call_params_match_protocol(self): "env", "idle_output_timeout", "max_suppression_seconds", + "on_pid_resolved", + "enable_deadline_extension", + "max_extension_seconds", } assert expected == actual, ( f"DefaultSubprocessRunner.__call__ params changed.\n" @@ -426,7 +465,7 @@ def test_req_api_004_llm_triage_no_asyncio_module_import(self): def test_req_api_004_headless_subprocess_result_under_type_checking(self): """execution/headless.py must import SubprocessResult only under TYPE_CHECKING.""" - source = (self._pkg_root() / "execution" / "headless.py").read_text() + source = (self._pkg_root() / "execution" / "headless" / "__init__.py").read_text() assert "SubprocessResult" in source, ( "SubprocessResult reference vanished from headless.py entirely" ) @@ -442,20 +481,20 @@ def test_req_api_004_headless_subprocess_result_under_type_checking(self): f"{runtime_import.group()!r}" ) - def test_req_api_004_server_helpers_subprocess_result_under_type_checking(self): - """server/helpers.py must import SubprocessResult only under TYPE_CHECKING.""" - source = (self._pkg_root() / "server" / "helpers.py").read_text() + def test_req_api_004_server_subprocess_result_under_type_checking(self): + """server/_subprocess.py must import SubprocessResult only under TYPE_CHECKING.""" + source = (self._pkg_root() / "server" / "_subprocess.py").read_text() assert "SubprocessResult" in source, ( - "SubprocessResult reference vanished from server/helpers.py entirely" + "SubprocessResult reference vanished from server/_subprocess.py entirely" ) - assert "TYPE_CHECKING" in source, "TYPE_CHECKING guard removed from server/helpers.py" + assert "TYPE_CHECKING" in source, "TYPE_CHECKING guard removed from server/_subprocess.py" runtime_import = re.search( r"^from\s+\S+\s+import\s+.*SubprocessResult", source, re.MULTILINE, ) assert runtime_import is None, ( - f"SubprocessResult has a runtime import in server/helpers.py: " + f"SubprocessResult has a runtime import in server/_subprocess.py: " f"{runtime_import.group()!r}" ) @@ -464,7 +503,7 @@ def test_req_api_004_server_helpers_subprocess_result_under_type_checking(self): # ------------------------------------------------------------------ def test_req_api_005_subprocess_result_field_names(self): - """SubprocessResult must have exactly the 14 canonical fields.""" + """SubprocessResult must have exactly the 15 canonical fields.""" from autoskillit.core.types import SubprocessResult fields = {f.name for f in dataclasses.fields(SubprocessResult)} @@ -483,6 +522,7 @@ def test_req_api_005_subprocess_result_field_names(self): "elapsed_seconds", "kill_reason", "tracked_comm", + "orphaned_tool_result", } assert fields == expected, ( f"SubprocessResult fields changed.\n" @@ -541,3 +581,9 @@ def test_req_api_006_channel_confirmation_string_values(self): assert ChannelConfirmation.CHANNEL_A == "channel_a" assert ChannelConfirmation.CHANNEL_B == "channel_b" assert ChannelConfirmation.UNMONITORED == "unmonitored" + + +# PROTO-1 +def test_supports_logger_importable_from_core(): + """SupportsLogger protocol is exported from the autoskillit.core gateway.""" + from autoskillit.core import SupportsLogger # noqa: F401 diff --git a/tests/contracts/test_protocol_satisfaction_five.py b/tests/contracts/test_protocol_satisfaction_five.py new file mode 100644 index 0000000000..dd5030de81 --- /dev/null +++ b/tests/contracts/test_protocol_satisfaction_five.py @@ -0,0 +1,132 @@ +"""Protocol satisfaction tests — Group Five (issue #1523). + +Covers: OutputPatternResolver, WriteExpectedResolver, QuotaRefreshTask, +TokenFactory, CampaignProtector. + +Tests 7–9 require WP2 (@runtime_checkable on CampaignProtector) to pass. +""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +# ── WP3-1: OutputPatternResolver ────────────────────────────────────────────── + + +def test_output_pattern_resolver_satisfied_by_minimal_callable(): + from autoskillit.core import OutputPatternResolver + + class _Resolver: + def __call__(self, skill_command: str) -> list[str]: + return [] + + assert isinstance(_Resolver(), OutputPatternResolver) + + +# ── WP3-2: WriteExpectedResolver ────────────────────────────────────────────── + + +def test_write_expected_resolver_satisfied_by_minimal_callable(): + from autoskillit.core import WriteExpectedResolver + + class _Resolver: + def __call__(self, skill_command: str) -> None: + pass + + assert isinstance(_Resolver(), WriteExpectedResolver) + + +# ── WP3-3: QuotaRefreshTask — asyncio.Task ──────────────────────────────────── + + +@pytest.mark.anyio +async def test_quota_refresh_task_satisfied_by_asyncio_task(): + """asyncio.Task has a cancel() method — satisfies QuotaRefreshTask.""" + import asyncio + + from autoskillit.core import QuotaRefreshTask + + async def _noop() -> None: + pass + + task = asyncio.ensure_future(_noop()) + try: + assert isinstance(task, QuotaRefreshTask) + finally: + task.cancel() + + +# ── WP3-4: QuotaRefreshTask — minimal cancel class ──────────────────────────── + + +def test_quota_refresh_task_satisfied_by_minimal_cancel_class(): + from autoskillit.core import QuotaRefreshTask + + class _MinimalTask: + def cancel(self, msg: object = None) -> bool: + return True + + assert isinstance(_MinimalTask(), QuotaRefreshTask) + + +# ── WP3-5: TokenFactory — concrete server class ─────────────────────────────── + + +def test_token_factory_satisfied_by_server_concrete_class(): + from autoskillit.core import TokenFactory + from autoskillit.server._factory import _LazyTokenFactory as ConcreteTokenFactory + + instance = ConcreteTokenFactory(lambda: None) + assert isinstance(instance, TokenFactory) + + +# ── WP3-6: TokenFactory — minimal callable ──────────────────────────────────── + + +def test_token_factory_satisfied_by_minimal_callable(): + from autoskillit.core import TokenFactory + + class _MinimalFactory: + def __call__(self) -> str | None: + return None + + assert isinstance(_MinimalFactory(), TokenFactory) + + +# ── WP3-7: CampaignProtector — is runtime_checkable ────────────────────────── + + +def test_campaign_protector_is_runtime_checkable(): + """Requires WP2: @runtime_checkable added to CampaignProtector.""" + from autoskillit.core import CampaignProtector + + assert getattr(CampaignProtector, "_is_runtime_protocol", False), ( + "CampaignProtector must be decorated with @runtime_checkable" + ) + + +# ── WP3-8: CampaignProtector — build_protected_campaign_ids ────────────────── + + +def test_campaign_protector_satisfied_by_build_protected_campaign_ids(): + """Requires WP2: @runtime_checkable added to CampaignProtector.""" + from autoskillit.core import CampaignProtector + from autoskillit.fleet import build_protected_campaign_ids + + assert isinstance(build_protected_campaign_ids, CampaignProtector) + + +# ── WP3-9: CampaignProtector — minimal callable ─────────────────────────────── + + +def test_campaign_protector_satisfied_by_minimal_callable(): + """Requires WP2: @runtime_checkable added to CampaignProtector.""" + from autoskillit.core import CampaignProtector + + class _MinimalProtector: + def __call__(self, project_dir: Path) -> frozenset[str]: + return frozenset() + + assert isinstance(_MinimalProtector(), CampaignProtector) diff --git a/tests/contracts/test_review_local_mode_contracts.py b/tests/contracts/test_review_local_mode_contracts.py new file mode 100644 index 0000000000..e2c7e872f5 --- /dev/null +++ b/tests/contracts/test_review_local_mode_contracts.py @@ -0,0 +1,206 @@ +"""Tests for skill_contracts.yaml and SKILL.md contract validation for local review mode. + +Verifies that: +- skill_contracts.yaml documents the mode parameter for review-pr and resolve-review +- review-pr documents local_findings output when mode=local +- resolve-review documents deferred_observations and reject_patterns outputs +- deferred_observations JSON schema is documented in SKILL.md +""" + +from pathlib import Path + +import yaml + +CONTRACTS_YAML = ( + Path(__file__).parent.parent.parent / "src" / "autoskillit" / "recipe" / "skill_contracts.yaml" +) + +REVIEW_PR_SKILL_PATH = ( + Path(__file__).parent.parent.parent + / "src" + / "autoskillit" + / "skills_extended" + / "review-pr" + / "SKILL.md" +) + +RESOLVE_REVIEW_SKILL_PATH = ( + Path(__file__).parent.parent.parent + / "src" + / "autoskillit" + / "skills_extended" + / "resolve-review" + / "SKILL.md" +) + + +def _contracts() -> dict: + return yaml.safe_load(CONTRACTS_YAML.read_text()) + + +def _review_pr_skill() -> str: + return REVIEW_PR_SKILL_PATH.read_text() + + +def _resolve_review_skill() -> str: + return RESOLVE_REVIEW_SKILL_PATH.read_text() + + +def test_review_pr_contract_has_mode_input(): + """Assert skill_contracts.yaml review-pr inputs includes mode parameter.""" + contracts = _contracts() + inputs = contracts.get("skills", {}).get("review-pr", {}).get("inputs", []) + names = [inp["name"] for inp in inputs] + assert "mode" in names, "skill_contracts.yaml review-pr inputs must include 'mode' parameter" + + +def test_resolve_review_contract_has_mode_input(): + """Assert skill_contracts.yaml resolve-review inputs includes mode parameter.""" + contracts = _contracts() + inputs = contracts.get("skills", {}).get("resolve-review", {}).get("inputs", []) + names = [inp["name"] for inp in inputs] + assert "mode" in names, ( + "skill_contracts.yaml resolve-review inputs must include 'mode' parameter" + ) + + +def test_review_pr_contract_local_findings_output(): + """Assert review-pr outputs includes local_findings_path when mode=local.""" + contracts = _contracts() + outputs = contracts.get("skills", {}).get("review-pr", {}).get("outputs", []) + output_names = [o["name"] for o in outputs] + assert "local_findings_path" in output_names, ( + "skill_contracts.yaml review-pr outputs must include 'local_findings_path' " + "for the local_findings_{pr_number}.json output file" + ) + + +def test_resolve_review_contract_deferred_observations_output(): + """Assert skill_contracts.yaml resolve-review outputs includes deferred_observations_path.""" + contracts = _contracts() + outputs = contracts.get("skills", {}).get("resolve-review", {}).get("outputs", []) + output_names = [o["name"] for o in outputs] + assert "deferred_observations_path" in output_names, ( + "skill_contracts.yaml resolve-review outputs must include 'deferred_observations_path' " + "for the deferred_observations_{pr_number}.json output file" + ) + + +def test_resolve_review_contract_reject_patterns_output(): + """Assert skill_contracts.yaml resolve-review outputs includes reject_patterns_path.""" + contracts = _contracts() + outputs = contracts.get("skills", {}).get("resolve-review", {}).get("outputs", []) + output_names = [o["name"] for o in outputs] + assert "reject_patterns_path" in output_names, ( + "skill_contracts.yaml resolve-review outputs must include 'reject_patterns_path' " + "for the reject_patterns_{pr_number}.json output file" + ) + + +def test_deferred_observations_schema_review_pr_documents_iteration(): + """Verify that review-pr SKILL.md documents the iteration field in local_findings JSON + (which resolve-review uses as the round number).""" + text = _review_pr_skill() + local_findings_idx = text.find("local_findings") + assert local_findings_idx >= 0 + after = text[local_findings_idx : local_findings_idx + 1500] + assert "iteration" in after.lower(), ( + "review-pr SKILL.md must document the 'iteration' field in local_findings JSON " + "so resolve-review can use it as the round number in deferred_observations" + ) + + +def test_deferred_observations_schema_resolve_review_documents_round(): + """Verify resolve-review SKILL.md documents the JSON schema for deferred_observations + with required fields: round, path, line, body, evidence, severity, dimension, verdict, + category.""" + text = _resolve_review_skill() + step36_idx = text.find("### Step 3.6") + assert step36_idx >= 0 + step36_section = text[step36_idx : step36_idx + 2500] + # Should document round field + assert "round" in step36_section.lower(), ( + "resolve-review SKILL.md Step 3.6 must document the 'round' field in " + "deferred_observations JSON schema" + ) + # Should document path field + assert '"path"' in step36_section or "path" in step36_section.lower(), ( + "resolve-review SKILL.md Step 3.6 must document the 'path' field in " + "deferred_observations JSON schema" + ) + # Should document line field + assert '"line"' in step36_section or "line" in step36_section.lower(), ( + "resolve-review SKILL.md Step 3.6 must document the 'line' field in " + "deferred_observations JSON schema" + ) + # Should document body field + assert '"body"' in step36_section or "body" in step36_section.lower(), ( + "resolve-review SKILL.md Step 3.6 must document the 'body' field in " + "deferred_observations JSON schema" + ) + # Should document evidence field + assert '"evidence"' in step36_section or "evidence" in step36_section.lower(), ( + "resolve-review SKILL.md Step 3.6 must document the 'evidence' field in " + "deferred_observations JSON schema" + ) + # Should document severity field + assert "severity" in step36_section.lower(), ( + "resolve-review SKILL.md Step 3.6 must document the 'severity' field in " + "deferred_observations JSON schema" + ) + # Should document dimension field + assert "dimension" in step36_section.lower(), ( + "resolve-review SKILL.md Step 3.6 must document the 'dimension' field in " + "deferred_observations JSON schema" + ) + # Should document verdict field + assert '"verdict"' in step36_section or "verdict" in step36_section.lower(), ( + "resolve-review SKILL.md Step 3.6 must document the 'verdict' field in " + "deferred_observations JSON schema" + ) + # Should document category field + assert "category" in step36_section.lower(), ( + "resolve-review SKILL.md Step 3.6 must document the 'category' field in " + "deferred_observations JSON schema" + ) + + +def test_review_flag_marker_format(): + """Verify the REVIEW-FLAG marker format matches the regex: + """ + text = _resolve_review_skill() + # The marker format should be documented + assert "REVIEW-FLAG" in text, ( + "resolve-review SKILL.md must document the REVIEW-FLAG marker format" + ) + step15_idx = text.find("### Step 1.5") + assert step15_idx >= 0 + step15_section = text[step15_idx : step15_idx + 2500] + # Should have severity= and dimension= in the marker + assert "severity=" in step15_section.lower() and "dimension=" in step15_section.lower(), ( + "resolve-review SKILL.md Step 1.5 must document REVIEW-FLAG marker with " + "severity={severity} dimension={dimension} format" + ) + + +def test_review_pr_mode_default_github(): + """Assert skill_contracts.yaml documents mode default as github.""" + contracts = _contracts() + inputs = contracts.get("skills", {}).get("review-pr", {}).get("inputs", []) + mode_input = next((i for i in inputs if i.get("name") == "mode"), None) + assert mode_input is not None, "mode input must exist in review-pr contract" + # mode should not be marked as required (it's optional with default) + assert mode_input.get("required") is not True, ( + "mode parameter should not be required (has default: github)" + ) + + +def test_resolve_review_mode_default_github(): + """Assert skill_contracts.yaml documents mode default as github.""" + contracts = _contracts() + inputs = contracts.get("skills", {}).get("resolve-review", {}).get("inputs", []) + mode_input = next((i for i in inputs if i.get("name") == "mode"), None) + assert mode_input is not None, "mode input must exist in resolve-review contract" + assert mode_input.get("required") is not True, ( + "mode parameter should not be required (has default: github)" + ) diff --git a/tests/contracts/test_review_pr_diff_annotation.py b/tests/contracts/test_review_pr_diff_annotation.py index d29dd9344d..eaf86139fc 100644 --- a/tests/contracts/test_review_pr_diff_annotation.py +++ b/tests/contracts/test_review_pr_diff_annotation.py @@ -70,3 +70,25 @@ def test_review_pr_skill_reads_hunk_ranges_from_file() -> None: assert "hunk_ranges_path" in skill_text, ( "review-pr SKILL.md must read hunk_ranges_path from disk" ) + + +def test_review_pr_contract_has_diff_metrics_path() -> None: + """annotate_pr_diff callable contract must declare diff_metrics_path output.""" + raw = yaml.safe_load(_CONTRACTS_YAML.read_text()) + outputs = ( + raw.get("callable_contracts", {}) + .get("autoskillit.smoke_utils.annotate_pr_diff", {}) + .get("outputs", []) + ) + names = [o["name"] for o in outputs] + assert "diff_metrics_path" in names, ( + "annotate_pr_diff contract must have a diff_metrics_path output entry" + ) + + +def test_review_pr_skill_reads_diff_metrics_from_file() -> None: + """review-pr SKILL.md must reference diff_metrics_path.""" + skill_text = _SKILL_MD.read_text() + assert "diff_metrics_path" in skill_text, ( + "review-pr SKILL.md must read diff_metrics_path from disk" + ) diff --git a/tests/contracts/test_run_experiment_contracts.py b/tests/contracts/test_run_experiment_contracts.py index c8f2f94661..f9fa473607 100644 --- a/tests/contracts/test_run_experiment_contracts.py +++ b/tests/contracts/test_run_experiment_contracts.py @@ -22,3 +22,23 @@ def test_data_manifest_preflight_check() -> None: lower = text.lower() assert "data manifest" in lower assert "pre-flight" in lower or "preflight" in lower + + +def test_run_experiment_env_mode_dispatch() -> None: + """run-experiment must dispatch execution based on env_mode.""" + text = SKILL_PATH.read_text() + assert "env_mode" in text, ( + "run-experiment/SKILL.md must reference 'env_mode' for execution dispatch" + ) + assert "blocked_experiment" in text, ( + "run-experiment/SKILL.md must emit 'blocked_experiment' token when env_mode is unavailable" + ) + + +def test_run_experiment_micromamba_run_command() -> None: + """run-experiment must include micromamba run command for host fallback.""" + text = SKILL_PATH.read_text() + assert "micromamba run" in text, ( + "run-experiment/SKILL.md must include 'micromamba run' command " + "for the micromamba-host execution path" + ) diff --git a/tests/contracts/test_skill_contracts.py b/tests/contracts/test_skill_contracts.py index 55eaf96e8c..bb9479643b 100644 --- a/tests/contracts/test_skill_contracts.py +++ b/tests/contracts/test_skill_contracts.py @@ -4,6 +4,7 @@ import re from pathlib import Path +from typing import Any, Final import pytest import yaml @@ -160,6 +161,39 @@ def test_all_exp_lens_skills_have_contracts(skills): assert not missing, f"exp-lens skills missing contracts: {sorted(missing)}" +_EXP_LENS_SKILLS: Final[list[str]] = sorted( + name + for name in yaml.safe_load(_CONTRACTS_YAML.read_text()).get("skills", {}) + if name.startswith("exp-lens-") +) + + +@pytest.mark.parametrize("skill_name", _EXP_LENS_SKILLS) +def test_exp_lens_experiment_plan_path_is_required( + skills: dict[str, Any], skill_name: str +) -> None: + """experiment_plan_path must be required: true on all exp-lens contracts.""" + contract = skills[skill_name] + inp = next( + (i for i in contract["inputs"] if i["name"] == "experiment_plan_path"), + None, + ) + assert inp is not None, f"{skill_name}: missing experiment_plan_path input" + assert inp["required"] is True, f"{skill_name}: experiment_plan_path must be required: true" + + +@pytest.mark.parametrize("skill_name", _EXP_LENS_SKILLS) +def test_exp_lens_context_path_remains_optional(skills: dict[str, Any], skill_name: str) -> None: + """context_path must remain required: false on all exp-lens contracts.""" + contract = skills[skill_name] + inp = next( + (i for i in contract["inputs"] if i["name"] == "context_path"), + None, + ) + assert inp is not None, f"{skill_name}: missing context_path input" + assert inp["required"] is False, f"{skill_name}: context_path must remain required: false" + + def test_skill_contracts_yaml_includes_prepare_research_pr(skills): """prepare-research-pr must be registered with prep_path output pattern.""" _assert_skill_has_patterns(skills, "prepare-research-pr", r"prep_path\s*=\s*/.+") @@ -175,3 +209,179 @@ def test_skill_contracts_yaml_includes_compose_research_pr(skills): def test_skill_contracts_yaml_open_research_pr_removed(skills): """open-research-pr must no longer be registered — it has been retired.""" assert "open-research-pr" not in skills + + +def test_review_pr_verdict_allowed_values_includes_approved_with_comments(skills): + """review-pr allowed_values must include approved_with_comments. + + The skill emits 4 distinct verdicts; the contract previously only listed 3. + A missing allowed_value causes unrouted-verdict-value semantic rule failures. + """ + assert "review-pr" in skills + verdict_output = next( + (o for o in skills["review-pr"].get("outputs", []) if o["name"] == "verdict"), + None, + ) + assert verdict_output is not None, "review-pr must declare a verdict output" + allowed = verdict_output.get("allowed_values", []) + assert "approved_with_comments" in allowed, ( + f"review-pr allowed_values must include 'approved_with_comments'; got {allowed!r}" + ) + + +def test_review_pr_pattern_examples_cover_all_verdicts(skills): + """Every allowed verdict value for review-pr must appear in at least one pattern_example. + + Ensures the contract's example set is complete: one example per outcome. + """ + assert "review-pr" in skills + verdict_output = next( + (o for o in skills["review-pr"].get("outputs", []) if o["name"] == "verdict"), + None, + ) + assert verdict_output is not None + allowed = verdict_output.get("allowed_values", []) + examples = skills["review-pr"].get("pattern_examples", []) + missing = [v for v in allowed if not any(f"verdict = {v}" in ex for ex in examples)] + assert not missing, ( + f"review-pr pattern_examples missing examples for verdicts: {missing!r}. " + "Each allowed_value must be represented by at least one pattern_example." + ) + + +def test_every_pattern_example_satisfies_all_patterns(skills): + """For every skill with expected_output_patterns and pattern_examples, + every example must re.search-match ALL patterns (bi-directional check). + + The one-directional check (each pattern matches >=1 example) misses conditional + tokens: a pattern may match *some* examples while failing for valid output that + legitimately omits a conditional token. + """ + failures = [] + for skill_name, contract in skills.items(): + patterns = contract.get("expected_output_patterns", []) + examples = contract.get("pattern_examples", []) + if not patterns or not examples: + continue + for i, example in enumerate(examples): + for pattern in patterns: + if not re.search(pattern, example): + failures.append( + f"Skill '{skill_name}': example[{i}] does not match pattern " + f"{pattern!r}.\n Example: {example!r}" + ) + assert not failures, ( + "Bi-directional pattern/example check failed — " + "conditional tokens cause AND-semantics failures at runtime:\n" + "\n".join(failures) + ) + + +def test_skill_contracts_allowed_values_covers_recipe_routes() -> None: + """Every verdict value routed in recipe on_result blocks must appear in allowed_values. + + Scans implementation.yaml, remediation.yaml, implementation-groups.yaml, and + merge-prs.yaml for result.verdict routing conditions. Any value routed in a recipe + but absent from skill_contracts.yaml allowed_values will trigger the + unrouted-verdict-value semantic rule at recipe-load time. + """ + recipes_dir = Path(__file__).parents[2] / "src/autoskillit/recipes" + target_files = [ + "implementation.yaml", + "remediation.yaml", + "implementation-groups.yaml", + "merge-prs.yaml", + ] + contracts = yaml.safe_load(_CONTRACTS_YAML.read_text()) + verdict_output = next( + (o for o in contracts["skills"]["review-pr"].get("outputs", []) if o["name"] == "verdict"), + None, + ) + assert verdict_output is not None + allowed = set(verdict_output.get("allowed_values", [])) + + # Match only lowercase verdict values (review-pr convention: approved, changes_requested…). + # Excludes all-uppercase review-design verdicts (GO, REVISE, STOP) which appear in the + # same recipe files under different steps. + verdict_route_re = re.compile(r"result\.verdict\s*}}\s*==\s*([a-z][a-z_]*)") + routed_values: set[str] = set() + for filename in target_files: + fpath = recipes_dir / filename + if not fpath.exists(): + continue + for m in verdict_route_re.finditer(fpath.read_text()): + routed_values.add(m.group(1)) + + missing = routed_values - allowed + assert not missing, ( + f"Verdict values routed in recipes but absent from skill_contracts.yaml allowed_values: " + f"{sorted(missing)}. Add them to the review-pr outputs[verdict].allowed_values list." + ) + + +# T3-1 +def test_review_gate_loop_required_pattern_in_review_pr_contracts(skills): + """review-pr gate pattern must use OR-conditional form compatible with approved_with_comments. + + The unconditional %%REVIEW_GATE::(LOOP_REQUIRED|CLEAR)%% pattern causes + CONTRACT_VIOLATION for sessions that legitimately emit no gate tag + (approved_with_comments verdict). The corrected form must be an OR that accepts + either a gate tag or an approved_with_comments verdict. + """ + assert "review-pr" in skills + patterns = skills["review-pr"].get("expected_output_patterns", []) + conditional_pattern = ( + "(?:%%REVIEW_GATE::(LOOP_REQUIRED|CLEAR)%%|verdict\\s*=\\s*approved_with_comments)" + ) + assert conditional_pattern in patterns, ( + f"review-pr gate pattern must use OR-conditional form so that approved_with_comments " + f"sessions succeed without a %%REVIEW_GATE:: tag. " + f"Expected pattern: {conditional_pattern!r}. Got: {patterns!r}" + ) + + +# T3-2 +def test_review_gate_clear_pattern_in_review_pr_contracts(skills): + """review-pr REVIEW_GATE pattern must cover CLEAR and LOOP_REQUIRED; approved_with_comments + example must exist WITHOUT a gate tag.""" + assert "review-pr" in skills + patterns = skills["review-pr"].get("expected_output_patterns", []) + examples = skills["review-pr"].get("pattern_examples", []) + + gate_patterns = [p for p in patterns if "REVIEW_GATE" in p] + assert gate_patterns, "No REVIEW_GATE pattern found for review-pr" + combined = " ".join(gate_patterns) + assert "LOOP_REQUIRED" in combined and "CLEAR" in combined, ( + f"REVIEW_GATE pattern must reference both tags; found: {gate_patterns}" + ) + + awc_examples = [ex for ex in examples if "approved_with_comments" in ex] + assert awc_examples, ( + "No approved_with_comments example found in pattern_examples — " + "add one to document the no-gate-tag path" + ) + for ex in awc_examples: + assert "%%REVIEW_GATE::" not in ex, ( + f"approved_with_comments example must NOT include %%REVIEW_GATE:: tag; found: {ex!r}" + ) + + +def test_skill_contracts_yaml_includes_setup_environment(skills: dict[str, Any]) -> None: + """setup-environment must be registered with env_mode and verdict patterns.""" + _assert_skill_has_patterns( + skills, "setup-environment", r"env_mode\s*=\s*(none|docker|micromamba-host|unavailable)" + ) + contract = skills["setup-environment"] + patterns = contract["expected_output_patterns"] + assert any("verdict" in p for p in patterns), "missing verdict pattern" + assert any("env_report" in p for p in patterns), "missing env_report pattern" + + +def test_setup_environment_env_mode_examples_cover_all_modes( + skills: dict[str, Any], +) -> None: + """Pattern examples must cover all four env_mode values.""" + contract = skills["setup-environment"] + examples = contract.get("pattern_examples", []) + example_text = "\n".join(examples) + for mode in ("none", "docker", "micromamba-host", "unavailable"): + assert f"env_mode = {mode}" in example_text, f"missing example for env_mode={mode}" diff --git a/tests/contracts/test_skill_directive_descriptions.py b/tests/contracts/test_skill_directive_descriptions.py new file mode 100644 index 0000000000..ca7a10c121 --- /dev/null +++ b/tests/contracts/test_skill_directive_descriptions.py @@ -0,0 +1,54 @@ +"""SKILL.md directive description contract: headless recipe skills use directive language.""" + +from __future__ import annotations + +import re +from pathlib import Path + +import pytest + +pytestmark = [pytest.mark.layer("infra"), pytest.mark.small] + +_REPO_ROOT = Path(__file__).resolve().parents[2] +_SKILLS_DIR = _REPO_ROOT / "src" / "autoskillit" / "skills_extended" + +_FM_PATTERN = re.compile(r"^---\n(.*?)\n---", re.DOTALL) + +_HEADLESS_RECIPE_SKILLS = [ + "implement-worktree-no-merge", + "implement-worktree", + "retry-worktree", + "dry-walkthrough", + "make-plan", + "resolve-failures", + "compose-pr", + "prepare-pr", + "diagnose-ci", +] + + +def _get_description(skill_name: str) -> str: + skill_md = _SKILLS_DIR / skill_name / "SKILL.md" + content = skill_md.read_text() + m = _FM_PATTERN.match(content) + assert m, f"{skill_name}/SKILL.md must have YAML frontmatter" + for line in m.group(1).splitlines(): + if line.startswith("description:"): + return line[len("description:") :].strip() + pytest.fail(f"{skill_name}/SKILL.md frontmatter missing description field") + + +@pytest.mark.parametrize("skill_name", _HEADLESS_RECIPE_SKILLS) +def test_headless_recipe_skills_use_directive_descriptions(skill_name: str) -> None: + """T4-1: Each headless recipe skill description uses directive language.""" + desc = _get_description(skill_name) + + assert re.match(r"^[A-Z][a-z]", desc), ( + f"{skill_name}: description must start with a capitalized noun phrase, got: {desc!r}" + ) + assert "ALWAYS invoke this skill" in desc, ( + f"{skill_name}: description must contain 'ALWAYS invoke this skill', got: {desc!r}" + ) + assert "Do not" in desc, ( + f"{skill_name}: description must contain 'Do not' prohibition, got: {desc!r}" + ) diff --git a/tests/contracts/test_skill_transition_boundaries.py b/tests/contracts/test_skill_transition_boundaries.py new file mode 100644 index 0000000000..613be074c6 --- /dev/null +++ b/tests/contracts/test_skill_transition_boundaries.py @@ -0,0 +1,84 @@ +"""Contract tests for anti-confirmation instructions at SKILL.md transition boundaries.""" + +from __future__ import annotations + +import re +from pathlib import Path + +from tests.contracts._anti_confirm_helpers import ANTI_CONFIRM_RE as _ANTI_CONFIRM_RE + + +def _skill_text(skill_name: str) -> str: + base = Path(__file__).resolve().parent.parent.parent / "src" / "autoskillit" + for tier in ("skills", "skills_extended"): + p = base / tier / skill_name / "SKILL.md" + if p.exists(): + return p.read_text() + raise FileNotFoundError(f"SKILL.md not found for skill: {skill_name!r}") + + +_BATCH_ANTI_CONFIRM_RE = re.compile( + r"(?:never|do\s+not|must\s+not)[^\n]{0,80}(?:next\s+batch|batch\s+transition)", + re.IGNORECASE, +) +_INTER_ISSUE_ANTI_CONFIRM_RE = re.compile( + r"between\s+issues[^\n]{0,60}immediately" + r"|(?:never|do\s+not|must\s+not)[^\n]{0,80}next\s+issue", + re.IGNORECASE, +) + + +def test_process_issues_batch_anti_confirmation() -> None: + """Critical Constraints must explicitly prohibit confirmation at batch transitions.""" + text = _skill_text("process-issues") + cc_start = text.find("## Critical Constraints") + assert cc_start != -1, "process-issues must have a Critical Constraints section" + next_section = text.find("\n## ", cc_start + 1) + section = text[cc_start:next_section] if next_section != -1 else text[cc_start:] + assert _BATCH_ANTI_CONFIRM_RE.search(section) is not None, ( + "process-issues Critical Constraints must explicitly prohibit confirmation at " + "batch transitions (e.g., 'NEVER use AskUserQuestion ... next batch')" + ) + + +def test_process_issues_inter_issue_anti_confirmation() -> None: + """Critical Constraints section must explicitly prohibit pausing between issues.""" + text = _skill_text("process-issues") + cc_start = text.find("## Critical Constraints") + assert cc_start != -1, "process-issues must have a Critical Constraints section" + next_section = text.find("\n## ", cc_start + 1) + section = text[cc_start:next_section] if next_section != -1 else text[cc_start:] + assert _INTER_ISSUE_ANTI_CONFIRM_RE.search(section) is not None, ( + "process-issues Critical Constraints must explicitly prohibit pausing between " + "issues (e.g., 'Between issues: immediately...' or 'NEVER...next issue')" + ) + + +def test_implement_worktree_phase_anti_confirmation() -> None: + """Phase iteration section must prohibit pausing for confirmation between phases.""" + text = _skill_text("implement-worktree") + phase_pos = text.find("Phase by Phase") + assert phase_pos != -1, "implement-worktree must have a 'Phase by Phase' section" + next_section = text.find("\n### ", phase_pos + 1) + if next_section == -1: + next_section = text.find("\n## ", phase_pos + 1) + section = text[phase_pos:next_section] if next_section != -1 else text[phase_pos:] + assert _ANTI_CONFIRM_RE.search(section) is not None, ( + "implement-worktree phase iteration section must explicitly prohibit " + "pausing for confirmation between phases" + ) + + +def test_retry_worktree_phase_anti_confirmation() -> None: + """Step 3 loop section must prohibit AskUserQuestion between phase iterations.""" + text = _skill_text("retry-worktree") + step3_pos = text.find("Step 3: Continue Implementation") + assert step3_pos != -1, "retry-worktree must have a 'Step 3: Continue Implementation' section" + next_section = text.find("\n### ", step3_pos + 1) + if next_section == -1: + next_section = text.find("\n## ", step3_pos + 1) + section = text[step3_pos:next_section] if next_section != -1 else text[step3_pos:] + assert _ANTI_CONFIRM_RE.search(section) is not None, ( + "retry-worktree Step 3 loop section must explicitly prohibit " + "AskUserQuestion between phase iterations" + ) diff --git a/tests/contracts/test_skill_yaml_validation.py b/tests/contracts/test_skill_yaml_validation.py new file mode 100644 index 0000000000..40c16670b0 --- /dev/null +++ b/tests/contracts/test_skill_yaml_validation.py @@ -0,0 +1,46 @@ +"""Contract: YAML workflow examples embedded in SKILL.md files must be valid recipes.""" + +from __future__ import annotations + +import re +from pathlib import Path + + +def _all_skill_roots() -> list[Path]: + from autoskillit.workspace.skills import bundled_skills_dir, bundled_skills_extended_dir + + return [bundled_skills_dir(), bundled_skills_extended_dir()] + + +def test_skill_md_yaml_examples_are_valid_workflows() -> None: + """YAML workflow examples embedded in SKILL.md files must pass validation.""" + import yaml as _yaml + + from autoskillit.recipe.io import ( + _parse_recipe as _parse_workflow, + ) + from autoskillit.recipe.validator import ( + validate_recipe_structure as validate_workflow, + ) + + yaml_block_re = re.compile(r"```yaml\n(.*?)```", re.DOTALL) + + for skills_dir in _all_skill_roots(): + for skill_md in skills_dir.rglob("SKILL.md"): + content = skill_md.read_text() + for match in yaml_block_re.finditer(content): + block = match.group(1) + # Only validate blocks that look like full workflow definitions + if "steps:" not in block or "name:" not in block: + continue + # Skip format templates that use {placeholder} syntax + if "{script-name}" in block or "{mcp_tool_name}" in block: + continue + data = _yaml.safe_load(block) + if not isinstance(data, dict) or "steps" not in data: + continue + wf = _parse_workflow(data) + errors = [e for e in validate_workflow(wf) if "kitchen_rules" not in e.lower()] + assert not errors, ( + f"{skill_md.parent.name}/SKILL.md has invalid YAML example:\n {errors}" + ) diff --git a/tests/contracts/test_sous_chef_routing.py b/tests/contracts/test_sous_chef_routing.py index fa07208c3d..1a0f5b2a7d 100644 --- a/tests/contracts/test_sous_chef_routing.py +++ b/tests/contracts/test_sous_chef_routing.py @@ -18,6 +18,23 @@ def _sous_chef_text() -> str: return skill_md.read_text() +def _extract_routing_section(skill_md: str) -> str: + """Extract the full CONTEXT LIMIT ROUTING section.""" + lines = skill_md.splitlines() + in_section = False + extracted: list[str] = [] + for line in lines: + if "CONTEXT LIMIT ROUTING" in line: + in_section = True + extracted.append(line) + continue + if in_section and line.startswith("---"): + break + if in_section: + extracted.append(line) + return "\n".join(extracted) + + def _extract_routing_rule(skill_md: str, retry_reason: str) -> str: """Extract the bullet(s) in CONTEXT LIMIT ROUTING that mention a given retry_reason.""" lines = skill_md.splitlines() @@ -91,6 +108,40 @@ def _extract_merge_phase_section(skill_md: str) -> str: return "\n".join(extracted) +def test_sous_chef_contains_step_execution_obligation() -> None: + text = _sous_chef_text() + assert "STEP EXECUTION IS NOT DISCRETIONARY" in text + assert "MUST execute every step" in text + assert "NEVER skip a step because" in text + + +def test_sous_chef_contains_pr_pipeline_protection() -> None: + text = _sous_chef_text() + assert "review_pr" in text + assert "annotate_pr_diff" in text + assert "compose_pr" in text + # Must follow the pattern of the merge protection: named NEVER rule + idx = text.index("review_pr") + assert "NEVER" in text[max(0, idx - 400) : idx + 200] + + +def test_sous_chef_contains_context_ownership_line() -> None: + """The STEP EXECUTION section must contain a context-ownership assertion. + + The model must be told that context management is handled by the system + so it cannot rationalize step-skipping with context-pressure arguments. + The assertion must appear within 600 characters of the STEP EXECUTION sentinel. + """ + content = _sous_chef_text() + idx = content.find("STEP EXECUTION IS NOT DISCRETIONARY") + assert idx >= 0, "STEP EXECUTION sentinel missing from sous-chef SKILL.md" + section = content[idx : idx + 600] + assert "on_context_limit routing" in section, ( + "Context-ownership line missing from STEP EXECUTION section. " + "The model must be told the system handles context via on_context_limit routing." + ) + + def test_sous_chef_merge_phase_documents_queue_no_auto_path() -> None: """MERGE PHASE section must document the queue_enqueue_no_auto routing cell.""" skill_md = _sous_chef_text() @@ -103,3 +154,269 @@ def test_sous_chef_merge_phase_documents_queue_no_auto_path() -> None: "MERGE PHASE section must document the condition " "'queue_available == true and auto_merge_available == false'" ) + + +def _extract_multiple_issues_section(skill_md: str) -> str: + """Extract text from '## MULTIPLE ISSUES' up to the next top-level '## ' heading.""" + lines = skill_md.splitlines() + in_section = False + extracted: list[str] = [] + for line in lines: + if line.startswith("## MULTIPLE ISSUES"): + in_section = True + extracted.append(line) + continue + if in_section and line.startswith("## ") and "MULTIPLE ISSUES" not in line: + break + if in_section: + extracted.append(line) + return "\n".join(extracted) + + +def test_sous_chef_invokes_execution_map_before_parallel_dispatch() -> None: + """MULTIPLE ISSUES section must invoke /autoskillit:build-execution-map before dispatch.""" + skill_md = _sous_chef_text() + multiple_issues = _extract_multiple_issues_section(skill_md) + assert multiple_issues, "MULTIPLE ISSUES section not found in sous-chef/SKILL.md" + assert "/autoskillit:build-execution-map" in multiple_issues, ( + "MULTIPLE ISSUES section must invoke /autoskillit:build-execution-map before " + "launching parallel pipelines" + ) + # The invocation must come before launching pipelines + map_idx = multiple_issues.find("/autoskillit:build-execution-map") + pipeline_idx = multiple_issues.find("pipeline") + assert map_idx < pipeline_idx, ( + "build-execution-map invocation must appear before parallel pipeline launch " + "in MULTIPLE ISSUES section" + ) + + +def test_sous_chef_group_merge_wait_before_next_group() -> None: + """Sous-chef must require merge-wait between execution map groups.""" + skill_md = _sous_chef_text() + assert "Group N+1" in skill_md or "group N+1" in skill_md.lower(), ( + "sous-chef SKILL.md must contain a Group N+1 merge-wait rule" + ) + # Check the actual merge-wait language is present + lower = skill_md.lower() + assert "merge" in lower and ("wait" in lower or "before" in lower), ( + "sous-chef SKILL.md must instruct that Group N+1 waits for Group N's PRs to merge" + ) + + +def test_merge_phase_skips_merge_prs_for_queue_mode() -> None: + """MERGE PHASE must instruct the orchestrator NOT to invoke merge-prs + when queue_available=true (and sequential_queue is not set to true).""" + merge_phase = _extract_merge_phase_section(_sous_chef_text()) + + assert "queue_available" in merge_phase, ( + "MERGE PHASE must mention queue_available to distinguish queue vs classic routing" + ) + lower = merge_phase.lower() + assert any( + phrase in lower + for phrase in [ + "do not invoke merge-prs", + "do not invoke `merge-prs`", + "skip merge-prs", + "skip `merge-prs`", + "not invoke merge-prs", + "not invoke `merge-prs`", + ] + ), "MERGE PHASE must explicitly say NOT to invoke merge-prs when queue_available=true" + + +def test_merge_phase_documents_sequential_queue_override() -> None: + """MERGE PHASE must document the sequential_queue hidden ingredient as + a force-override that routes through merge-prs even for queue-mode repos.""" + merge_phase = _extract_merge_phase_section(_sous_chef_text()) + + assert "sequential_queue" in merge_phase, ( + "MERGE PHASE must document the sequential_queue hidden ingredient override" + ) + + +def test_merge_phase_preserves_merge_prs_for_non_queue_repos() -> None: + """MERGE PHASE must still route to merge-prs when queue_available=false + (the classic batch-branch path is unchanged).""" + merge_phase = _extract_merge_phase_section(_sous_chef_text()) + + lower = merge_phase.lower() + assert "merge-prs" in lower, ( + "MERGE PHASE must still mention merge-prs for the queue_available=false classic path" + ) + assert "queue_available" in lower and "false" in lower, ( + "MERGE PHASE must distinguish queue_available=false as the condition for merge-prs" + ) + + +def test_sous_chef_no_resume_session_id_in_context_limit_routing() -> None: + """sous-chef SKILL.md must not instruct passing resume_session_id for + on_context_limit routing. + + Context-exhausted sessions must start fresh to get a full context window. + """ + skill_md = _sous_chef_text() + assert "resume_session_id" not in skill_md, ( + "sous-chef SKILL.md must not instruct passing resume_session_id; " + "context-exhausted sessions must start fresh" + ) + + +class TestWorktreeStaleCarveout: + """SKILL.md routing contract for the worktree-stale carve-out.""" + + def test_worktree_stale_carveout_exists(self) -> None: + """SKILL.md must contain a worktree-stale carve-out for retry_reason=stale.""" + skill_md = _sous_chef_text() + routing_section = _extract_routing_section(skill_md) + assert "worktree" in routing_section.lower() and "stale" in routing_section.lower(), ( + "CONTEXT LIMIT ROUTING must contain a worktree-stale carve-out" + ) + assert "implement-worktree-no-merge" in routing_section, ( + "Worktree-stale carve-out must explicitly name implement-worktree-no-merge" + ) + + def test_worktree_stale_carveout_does_not_route_to_on_context_limit(self) -> None: + """Worktree-stale carve-out must NOT route to on_context_limit.""" + skill_md = _sous_chef_text() + routing_section = _extract_routing_section(skill_md) + lines = routing_section.splitlines() + carveout_lines: list[str] = [] + in_carveout = False + for line in lines: + if "worktree" in line.lower() and "stale" in line.lower() and "carve" in line.lower(): + in_carveout = True + carveout_lines.append(line) + continue + if in_carveout: + if line.startswith("**") or line.startswith("---"): + break + carveout_lines.append(line) + carveout_text = "\n".join(carveout_lines) + assert carveout_text, "Expected to find a worktree-stale carve-out subsection" + assert "on_context_limit" not in carveout_text, ( + "Worktree-stale carve-out must not route to on_context_limit — " + "stale is not a context limit" + ) + + def test_worktree_stale_carveout_bypasses_retries_budget(self) -> None: + """Worktree-stale carve-out must bypass the retries budget.""" + skill_md = _sous_chef_text() + lower = skill_md.lower() + assert any( + phrase in lower + for phrase in [ + "without consuming the retries budget", + "without decrementing the retries counter", + "does not consume the retries budget", + "does not decrement the retries", + "bypass the retries", + "independent of the retries counter", + "regardless of the retries counter", + ] + ), "Worktree-stale carve-out must explicitly state it bypasses the retries budget" + + def test_worktree_stale_carveout_is_one_shot(self) -> None: + """Worktree-stale carve-out must be limited to a single retry.""" + skill_md = _sous_chef_text() + lower = skill_md.lower() + assert any( + phrase in lower + for phrase in [ + "one-shot", + "once", + "single retry", + "maximum once", + "at most once", + "if the retry also goes stale", + ] + ), "Worktree-stale carve-out must be one-shot (limited to a single stale retry)" + + +def test_prompts_worktree_stale_carveout() -> None: + """_prompts_orchestrator.py orchestrator prompt must contain the worktree-stale carve-out.""" + prompts_path = ( + Path(__file__).resolve().parent.parent.parent + / "src" + / "autoskillit" + / "cli" + / "_prompts_orchestrator.py" + ) + prompts_text = prompts_path.read_text().lower() + assert "worktree" in prompts_text and "stale" in prompts_text, ( + "_prompts_orchestrator.py must contain a worktree-stale carve-out matching SKILL.md" + ) + + +def test_skill_md_covers_all_prompts_routing_reasons() -> None: + """Every RetryReason with a routing rule in _prompts.py must also appear in SKILL.md.""" + from autoskillit.cli._mcp_names import DIRECT_PREFIX + from autoskillit.cli._prompts import _build_orchestrator_prompt + from autoskillit.core.types import RetryReason + + prompt_text = _build_orchestrator_prompt("test-recipe", mcp_prefix=DIRECT_PREFIX) + skill_md = _sous_chef_text() + routing_section = _extract_routing_section(skill_md) + + for reason in RetryReason: + if reason in { + RetryReason.NONE, + RetryReason.BUDGET_EXHAUSTED, + RetryReason.CONTRACT_RECOVERY, + RetryReason.CLONE_CONTAMINATION, + }: + continue + if reason.value in prompt_text: + assert reason.value in routing_section, ( + f"{reason.value} has a routing rule in _prompts.py but is missing " + f"from SKILL.md CONTEXT LIMIT ROUTING section" + ) + + +class TestWorktreeEarlyStopCarveout: + """SKILL.md routing contract: early_stop must check has_progress_evidence.""" + + def test_early_stop_references_worktree_path(self) -> None: + skill_md = _sous_chef_text() + routing_section = _extract_routing_section(skill_md) + idx = routing_section.find("early_stop") + assert idx != -1 + window = routing_section[idx : idx + 500] + assert "has_progress_evidence" in window, ( + "early_stop routing rule must reference has_progress_evidence" + ) + + def test_early_stop_with_worktree_routes_to_on_context_limit(self) -> None: + skill_md = _sous_chef_text() + routing_section = _extract_routing_section(skill_md) + idx = routing_section.find("early_stop") + assert idx != -1 + window = routing_section[idx : idx + 500] + assert "on_context_limit" in window, ( + "early_stop with worktree_path must route to on_context_limit" + ) + + +class TestWorktreeZeroWritesCarveout: + """SKILL.md routing contract: zero_writes must check has_progress_evidence.""" + + def test_zero_writes_references_worktree_path(self) -> None: + skill_md = _sous_chef_text() + routing_section = _extract_routing_section(skill_md) + idx = routing_section.find("zero_writes") + assert idx != -1 + window = routing_section[idx : idx + 500] + assert "has_progress_evidence" in window, ( + "zero_writes routing rule must reference has_progress_evidence" + ) + + def test_zero_writes_with_worktree_routes_to_on_context_limit(self) -> None: + skill_md = _sous_chef_text() + routing_section = _extract_routing_section(skill_md) + idx = routing_section.find("zero_writes") + assert idx != -1 + window = routing_section[idx : idx + 500] + assert "on_context_limit" in window, ( + "zero_writes with worktree_path must route to on_context_limit" + ) diff --git a/tests/contracts/test_sous_chef_scheduling.py b/tests/contracts/test_sous_chef_scheduling.py index fe715a54f3..77388aa262 100644 --- a/tests/contracts/test_sous_chef_scheduling.py +++ b/tests/contracts/test_sous_chef_scheduling.py @@ -6,6 +6,8 @@ import pytest +from tests.contracts._anti_confirm_helpers import ANTI_CONFIRM_RE as _ANTI_CONFIRM_RE + def _sous_chef_text() -> str: skill_md = ( @@ -29,6 +31,7 @@ def _sous_chef_text() -> str: "test_check", "reset_test_dir", "classify_fix", + "push_to_remote", ] @@ -119,6 +122,42 @@ def test_sous_chef_parallel_scheduling_explains_wall_clock_rationale() -> None: ) +def test_sous_chef_wavefront_has_positive_advancement_mandate() -> None: + """REQ-PROMPT-009: Wavefront must mandate advancing every active pipeline each round.""" + text = _sous_chef_text() + section_start = text.find("PARALLEL STEP SCHEDULING") + assert section_start != -1 + next_section = text.find("\n## ", section_start + 1) + section_text = text[section_start:next_section] if next_section != -1 else text[section_start:] + lower = section_text.lower() + has_advancement = ( + ("every" in lower or "all" in lower) + and "active" in lower + and ("must" in lower or "advance" in lower or "idle" in lower) + ) + assert has_advancement, ( + "PARALLEL STEP SCHEDULING section must contain a positive advancement mandate " + "requiring every active pipeline to be advanced each round" + ) + + +def test_sous_chef_wavefront_has_four_rules() -> None: + """REQ-PROMPT-010: Wavefront Scheduling Rule must contain exactly 4 numbered rules.""" + import re + + text = _sous_chef_text() + section_start = text.find("### Wavefront Scheduling Rule") + assert section_start != -1, "Wavefront Scheduling Rule subsection must exist" + next_subsection = text.find("\n### ", section_start + 1) + subsection = ( + text[section_start:next_subsection] if next_subsection != -1 else text[section_start:] + ) + rules = re.findall(r"^\d+\.\s", subsection, re.MULTILINE) + assert len(rules) == 4, ( + f"Wavefront Scheduling Rule must have exactly 4 numbered rules, found {len(rules)}" + ) + + def test_sous_chef_scheduling_section_placement() -> None: """REQ-PROMPT-008: PARALLEL STEP SCHEDULING section must appear after MULTIPLE ISSUES.""" text = _sous_chef_text() @@ -129,3 +168,35 @@ def test_sous_chef_scheduling_section_placement() -> None: assert scheduling_pos > multiple_issues_pos, ( "PARALLEL STEP SCHEDULING section must appear after MULTIPLE ISSUES section" ) + + +def test_sous_chef_execution_map_group_iteration() -> None: + """EXECUTION MAP section must contain instruction to iterate groups in topological order.""" + text = _sous_chef_text() + assert "EXECUTION MAP" in text, ( + "sous-chef SKILL.md must contain an EXECUTION MAP section for group dispatch" + ) + # Find EXECUTION MAP section + exec_map_pos = text.find("EXECUTION MAP") + next_section = text.find("\n## ", exec_map_pos + 1) + section_text = text[exec_map_pos:next_section] if next_section != -1 else text[exec_map_pos:] + lower = section_text.lower() + assert "group" in lower and ("order" in lower or "iteration" in lower or "topolog" in lower), ( + "EXECUTION MAP section must instruct iterating groups in topological order" + ) + assert "merge" in lower and ("wait" in lower or "before" in lower or "mandatory" in lower), ( + "EXECUTION MAP section must contain mandatory merge-wait rule between groups" + ) + + +def test_sous_chef_execution_map_has_anti_confirmation() -> None: + """EXECUTION MAP section must contain explicit anti-confirmation instruction between groups.""" + text = _sous_chef_text() + exec_map_pos = text.find("EXECUTION MAP") + assert exec_map_pos != -1, "sous-chef SKILL.md must contain an EXECUTION MAP section" + next_section = text.find("\n## ", exec_map_pos + 1) + section_text = text[exec_map_pos:next_section] if next_section != -1 else text[exec_map_pos:] + assert _ANTI_CONFIRM_RE.search(section_text) is not None, ( + "EXECUTION MAP section must contain an explicit anti-confirmation instruction " + "(e.g., 'NEVER use AskUserQuestion to ask whether to proceed to the next group')" + ) diff --git a/tests/contracts/test_stage_data_contracts.py b/tests/contracts/test_stage_data_contracts.py index 2bb4d8cba4..f69ec1dd31 100644 --- a/tests/contracts/test_stage_data_contracts.py +++ b/tests/contracts/test_stage_data_contracts.py @@ -67,3 +67,48 @@ def test_stage_data_documents_fail_verdict(skill_text: str) -> None: def test_stage_data_categories_include_research(skill_text: str) -> None: assert "categories" in skill_text assert "research" in skill_text + + +def test_stage_data_probes_allen_brain_atlas(skill_text: str) -> None: + assert "brain-map.org" in skill_text + + +def test_stage_data_probes_cellxgene(skill_text: str) -> None: + assert "cellxgene" in skill_text.lower() + + +def test_stage_data_probes_expression_atlas(skill_text: str) -> None: + assert "gxa" in skill_text + + +def test_stage_data_probes_human_protein_atlas(skill_text: str) -> None: + assert "proteinatlas.org" in skill_text + + +def test_stage_data_probes_string_db(skill_text: str) -> None: + assert "string-db.org" in skill_text + + +def test_stage_data_probes_jaspar(skill_text: str) -> None: + assert "jaspar" in skill_text.lower() + + +def test_stage_data_documents_rate_limit_behavior(skill_text: str) -> None: + lower = skill_text.lower() + assert "rate" in lower and "limit" in lower + + +def test_stage_data_probe_count_minimum(skill_text: str) -> None: + known_sources = [ + "GEO / NCBI", + "ENCODE", + "UniProt", + "Allen Brain Atlas", + "CellxGene", + "Expression Atlas", + "Human Protein Atlas", + "STRING", + "JASPAR", + ] + count = sum(1 for src in known_sources if src.lower() in skill_text.lower()) + assert count >= 9 diff --git a/tests/contracts/test_token_summary_contracts.py b/tests/contracts/test_token_summary_contracts.py new file mode 100644 index 0000000000..0e43b9ea2d --- /dev/null +++ b/tests/contracts/test_token_summary_contracts.py @@ -0,0 +1,190 @@ +"""Structural contracts for the token summary pipeline.""" + +from __future__ import annotations + +import ast +import inspect +import json +from pathlib import Path + +import pytest + +from autoskillit.recipe._cmd_rpc import batch_create_issues +from autoskillit.smoke_utils import patch_pr_token_summary + +pytestmark = [pytest.mark.layer("contracts"), pytest.mark.small] + + +# ── Step 1a: Bridge contract: TOKEN_USAGE_FILE_KEYS must match TokenUsageFileEntry ── + + +def test_token_usage_file_keys_match_typed_dict(): + """Bridge contract: TOKEN_USAGE_FILE_KEYS must equal TokenUsageFileEntry annotations. + + Analogous to test_hook_bridge_coverage.py for the quota guard. + If this fails, update TOKEN_USAGE_FILE_KEYS and TokenUsageFileEntry together. + """ + from autoskillit.core.types import TokenUsageFileEntry + from autoskillit.hooks._hook_settings import TOKEN_USAGE_FILE_KEYS + + assert TOKEN_USAGE_FILE_KEYS == set(TokenUsageFileEntry.__annotations__.keys()) + + +# ── Step 1b: AST contract: hook reads only keys from TOKEN_USAGE_FILE_KEYS ── + + +def test_hook_load_sessions_reads_only_declared_keys(): + """AST contract: every data.get() key in _load_sessions must be in TOKEN_USAGE_FILE_KEYS. + + Follows test_config_field_coverage.py pattern. Prevents silent key drift + when fields are renamed in the TypedDict but not in the hook. + """ + from autoskillit.hooks._hook_settings import TOKEN_USAGE_FILE_KEYS + + hook_path = ( + Path(__file__).resolve().parents[2] + / "src" + / "autoskillit" + / "hooks" + / "token_summary_hook.py" + ) + tree = ast.parse(hook_path.read_text()) + + # Find _load_sessions function + for node in ast.walk(tree): + if isinstance(node, ast.FunctionDef) and node.name == "_load_sessions": + # Collect all data.get("key") string arguments + disk_keys: set[str] = set() + for call in ast.walk(node): + if ( + isinstance(call, ast.Call) + and isinstance(call.func, ast.Attribute) + and call.func.attr == "get" + and isinstance(call.func.value, ast.Name) + and call.func.value.id == "data" + and call.args + and isinstance(call.args[0], ast.Constant) + and isinstance(call.args[0].value, str) + ): + disk_keys.add(call.args[0].value) + + assert disk_keys, "No data.get() calls found in _load_sessions" + unexpected = disk_keys - TOKEN_USAGE_FILE_KEYS + assert not unexpected, ( + f"_load_sessions reads keys not in TOKEN_USAGE_FILE_KEYS: {unexpected}. " + f"Update the hook or TOKEN_USAGE_FILE_KEYS." + ) + break + else: + raise AssertionError("_load_sessions function not found in token_summary_hook.py") + + +# ── Step 1c: Cross-seam integration test ── + + +def test_flush_to_hook_cross_seam(tmp_path): + """Cross-seam: flush_session_log output must produce non-empty hook aggregation. + + This is the missing link between producer tests (test_session_log_flush.py) + and consumer tests (test_token_summary_appender.py). Proves the hook can + actually read what the producer writes. + """ + # Simulate what flush_session_log (session_log.py:384-399) writes to token_usage.json + session_dir_name = "test-session-abc123" + # log_root is the sessions/ dir — _load_sessions reads sessions.jsonl from here + # and looks for sessions/{dir_name}/token_usage.json + log_root = tmp_path / "sessions" + log_root.mkdir(parents=True) + + # Write sessions.jsonl index entry so _load_sessions discovers the session dir + sessions_jsonl = log_root / "sessions.jsonl" + sessions_jsonl.write_text( + json.dumps( + {"session_id": "abc123", "dir_name": session_dir_name, "order_id": "test-order"} + ) + + "\n" + ) + + # Write the token_usage.json that flush_session_log produces + session_dir = log_root / "sessions" / session_dir_name + session_dir.mkdir(parents=True) + tu_data = { + "session_label": "plan", + "input_tokens": 100, + "output_tokens": 200, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + "timing_seconds": 1.5, + "order_id": "test-order", + "loc_insertions": 10, + "loc_deletions": 5, + "peak_context": 50, + "turn_count": 3, + "provider_used": "test-provider", + "model_identifier": "test-model", + "dispatch_id": "test-dispatch", + "campaign_id": "test-campaign", + } + (session_dir / "token_usage.json").write_text(json.dumps(tu_data)) + + # Feed the file through the hook's _load_sessions — proves cross-seam coherence + from autoskillit.hooks import token_summary_hook + + aggregated = token_summary_hook._load_sessions(log_root, "", order_id="test-order") + assert aggregated, ( + "_load_sessions returned empty dict — hook cannot read what flush_session_log wrote. " + "Check key name: flush writes 'session_label' but hook may read 'step_name'." + ) + key = token_summary_hook._canonical("plan") + assert key in aggregated, f"Expected key 'plan' in aggregated, got: {list(aggregated.keys())}" + entry = aggregated[key] + assert entry["input_tokens"] == 100 + assert entry["output_tokens"] == 200 + + +# ── Step 1d: patch_pr_token_summary accepts timeout ── + + +def test_patch_pr_token_summary_accepts_timeout(): + """patch_pr_token_summary must accept timeout kwarg to absorb LLM arg-routing variance.""" + sig = inspect.signature(patch_pr_token_summary) + assert "timeout" in sig.parameters, ( + "patch_pr_token_summary must accept 'timeout' parameter. " + "Fleet LLMs may route the recipe with: timeout into args." + ) + + +# ── Step 1e: batch_create_issues accepts timeout ── + + +def test_batch_create_issues_accepts_timeout(): + """batch_create_issues must accept timeout kwarg to absorb LLM arg-routing variance.""" + sig = inspect.signature(batch_create_issues) + assert "timeout" in sig.parameters + + +# ── Existing structural contracts ── + + +def test_patch_pr_token_summary_uses_order_id_not_cwd_filter(): + """Structural contract: patch_pr_token_summary must accept order_id parameter. + + The presence of order_id in the signature is the canonical guard against regression + to cwd_filter-based scoping. If someone removes it, this test fails immediately. + """ + sig = inspect.signature(patch_pr_token_summary) + assert "order_id" in sig.parameters, ( + "patch_pr_token_summary must have an 'order_id' parameter. " + "This is the canonical scoping key for multi-clone pipelines. " + "Do not remove it or replace it with cwd_filter." + ) + + +def test_patch_pr_token_summary_cwd_is_optional(): + """cwd must have a default value so callers can omit it when using order_id.""" + sig = inspect.signature(patch_pr_token_summary) + assert "cwd" in sig.parameters + cwd_param = sig.parameters["cwd"] + assert cwd_param.default == "", ( + "cwd must default to '' so fleet callers can omit it and rely on order_id scoping." + ) diff --git a/tests/contracts/test_tools_recipe_contracts.py b/tests/contracts/test_tools_recipe_contracts.py index 039ecc034d..0fa19ec670 100644 --- a/tests/contracts/test_tools_recipe_contracts.py +++ b/tests/contracts/test_tools_recipe_contracts.py @@ -10,7 +10,7 @@ def test_load_recipe_instructs_step_name_exact_yaml_key(): """ import inspect - from autoskillit.server.tools_recipe import load_recipe + from autoskillit.server.tools.tools_recipe import load_recipe doc = inspect.getdoc(load_recipe) or "" assert "must match the yaml step key exactly" in doc.lower(), ( @@ -19,6 +19,22 @@ def test_load_recipe_instructs_step_name_exact_yaml_key(): ) +def test_load_recipe_docstring_contains_optional_step_semantics(): + import inspect + + from autoskillit.server.tools.tools_recipe import load_recipe + + doc = inspect.getdoc(load_recipe) or "" + assert "OPTIONAL STEP SEMANTICS" in doc + + +def test_load_recipe_response_includes_orchestration_rules(): + """load_and_validate result TypedDict must declare orchestration_rules.""" + from autoskillit.recipe._api import LoadRecipeResult + + assert "orchestration_rules" in LoadRecipeResult.__annotations__ + + def test_load_recipe_does_not_instruct_get_token_summary_pre_staging(): """ The load_recipe docstring must not instruct the orchestrator to call @@ -29,7 +45,7 @@ def test_load_recipe_does_not_instruct_get_token_summary_pre_staging(): """ import inspect - from autoskillit.server.tools_recipe import load_recipe + from autoskillit.server.tools.tools_recipe import load_recipe doc = inspect.getdoc(load_recipe) or "" # Check that the active invocation form is absent. After Step 2b, the replacement diff --git a/tests/contracts/test_triage_contracts.py b/tests/contracts/test_triage_contracts.py index 4b815bbb23..360056ae0a 100644 --- a/tests/contracts/test_triage_contracts.py +++ b/tests/contracts/test_triage_contracts.py @@ -91,3 +91,48 @@ def test_triage_enrich_no_subagents(): # (other steps like split analysis do use subagents — we can't forbid globally) # Check that the enrich step description does not mention spawning/launching subagents assert "spawn" not in enrich_section.lower() or "enrich" not in enrich_section.lower() + + +# --------------------------------------------------------------------------- +# TRIAGE MISCLASSIFICATION: Gap 1b — validated audit report signal in Step 3 +# --------------------------------------------------------------------------- + + +def test_triage_step3_validated_audit_report_signal(): + """Step 3 must have an explicit signal for Validated Audit Reports → implementation.""" + text = TRIAGE_SKILL.read_text() + step3_pos = text.find("### Step 3:") + assert step3_pos != -1, "Step 3 Recipe Classification not found" + step3b_pos = text.find("### Step 3b:", step3_pos) + step3_section = ( + text[step3_pos:step3b_pos] if step3b_pos != -1 else text[step3_pos : step3_pos + 3000] + ) + has_audit_signal = ( + "Validated Audit Report" in step3_section or "validated audit" in step3_section.lower() + ) + assert has_audit_signal, ( + "triage-issues Step 3 must have an explicit signal for issues with 'Validated Audit " + "Report' in the title routing to implementation — not fall through to ambiguous-scope " + "fallback" + ) + + +def test_triage_step3_scope_alone_not_remediation(): + """Step 3 must document that scope alone must not override behavioral signals.""" + text = TRIAGE_SKILL.read_text() + step3_pos = text.find("### Step 3:") + assert step3_pos != -1 + step3b_pos = text.find("### Step 3b:", step3_pos) + step3_section = ( + text[step3_pos:step3b_pos] if step3b_pos != -1 else text[step3_pos : step3_pos + 3000] + ) + # Must explicitly call out that scope/volume alone does not trigger remediation + scope_guarded = ( + "scope alone" in step3_section.lower() + or "number of findings" in step3_section.lower() + or "large scope" in step3_section.lower() + ) + assert scope_guarded, ( + "Step 3 must clarify that issue scope (high finding count) alone must not route " + "a validated audit report to remediation" + ) diff --git a/tests/contracts/test_version_consistency.py b/tests/contracts/test_version_consistency.py index e25ecac2dd..589c6fca6b 100644 --- a/tests/contracts/test_version_consistency.py +++ b/tests/contracts/test_version_consistency.py @@ -9,6 +9,8 @@ from pathlib import Path from unittest.mock import patch +import yaml + import autoskillit @@ -52,6 +54,20 @@ def counting_read_text(self, *args, **kwargs): assert read_count == 1, f"plugin.json should be read once (got {read_count})" version_info.cache_clear() + def test_bundled_recipes_do_not_have_autoskillit_version(self): + recipes_dir = Path(autoskillit.__file__).parent / "recipes" + has_field = {} + for recipe_path in sorted(recipes_dir.rglob("*.yaml")): + data = yaml.safe_load(recipe_path.read_text()) + if not isinstance(data, dict): + continue + if "autoskillit_version" in data: + has_field[str(recipe_path.relative_to(recipes_dir))] = data["autoskillit_version"] + assert not has_field, ( + f"Bundled recipes must NOT declare autoskillit_version " + f"(field is only for project-local recipes): {has_field}" + ) + def test_marketplace_json_version_field(self, tmp_path, monkeypatch): import importlib as _importlib diff --git a/tests/core/CLAUDE.md b/tests/core/CLAUDE.md new file mode 100644 index 0000000000..8bdef540d4 --- /dev/null +++ b/tests/core/CLAUDE.md @@ -0,0 +1,40 @@ +# core/ + +Core layer (IL-0) unit tests — paths, IO, types, feature flags. + +## Files + +| File | Purpose | +|------|---------| +| `__init__.py` | empty | +| `test_add_dir_validation.py` | Tests for ValidatedAddDir and validate_add_dir | +| `test_branch_guard.py` | Tests for core.branch_guard — protected-branch validation | +| `test_claude_env.py` | Unit tests for build_claude_env() — IDE env scrubbing at the subprocess launch boundary | +| `test_core.py` | Tests for the core/ sub-package foundation layer | +| `test_core_terminal_table.py` | Tests for core/_terminal_table.py — the L0 shared table primitive | +| `test_ensure_project_temp_with_config.py` | Tests for ensure_project_temp with configurable override | +| `test_feature_flags.py` | Tests for core/feature_flags.py — _collect_disabled_feature_tags helper | +| `test_github_url.py` | Unit tests for core.github_url.parse_github_repo | +| `test_import_isolation.py` | Fleet package must not be imported at server startup via lazy-import structure | +| `test_install_detect.py` | Tests for core/_install_detect.py — install-type detection | +| `test_io.py` | Extended YAML I/O tests for core/io.py consolidation | +| `test_json.py` | Tests for autoskillit.core._json — fast_loads and fast_dumps | +| `test_kitchen_state.py` | Tests for KitchenMarker hash field support | +| `test_label_lifecycle.py` | Tests for IssueLabelState, LabelDef, LABEL_LIFECYCLE_REGISTRY, LABEL_TRANSITIONS, and validate_label_transition | +| `test_logging.py` | Tests for autoskillit.core.logging — centralized structlog configuration | +| `test_paths.py` | Tests for autoskillit.core.paths — is_git_worktree and pkg_root | +| `test_resolve_temp_dir.py` | Tests for autoskillit.core.io.resolve_temp_dir | +| `test_session_provenance.py` | Tests for core/runtime/session_provenance.py — provenance store for L2 sessions | +| `test_session_checkpoint.py` | Tests for SessionCheckpoint schema validation and compute_remaining | +| `test_session_index_schema.py` | Tests for SessionIndexEntry TypedDict field completeness | +| `test_session_liveness.py` | Tests for is_session_alive generalized liveness triple-check | +| `test_session_registry.py` | Tests for core/session_registry.py | +| `test_session_type.py` | Tests for SessionType resolver and constants | +| `test_skill_command_parsing.py` | Unit tests for extract_path_arg in core._type_helpers | +| `test_tool_sequence_analysis.py` | Tool sequence analysis tests | +| `test_type_constants.py` | Tests for PACK_REGISTRY and related constants in core._type_constants | +| `test_type_protocol_shards.py` | Type protocol shards guard | +| `test_types.py` | Tests for shared type contracts — enum exhaustiveness | +| `test_types_structure.py` | Tests for core/types.py split into focused sub-modules (P8-F2) | +| `test_type_results_execution.py` | Tests for _type_results_execution.py — execution-scoped types | +| `test_version_snapshot.py` | Tests for core/_version_snapshot.py | diff --git a/tests/core/test_add_dir_validation.py b/tests/core/test_add_dir_validation.py index 002a7e239a..aaafc45a73 100644 --- a/tests/core/test_add_dir_validation.py +++ b/tests/core/test_add_dir_validation.py @@ -10,6 +10,8 @@ from autoskillit.core import ValidatedAddDir from autoskillit.core.claude_conventions import LayoutError, validate_add_dir +pytestmark = [pytest.mark.layer("core"), pytest.mark.small] + class TestValidatedAddDir: """ValidatedAddDir is an opaque wrapper with str/fspath support.""" diff --git a/tests/core/test_branch_guard.py b/tests/core/test_branch_guard.py index 0fd9370694..7354a53784 100644 --- a/tests/core/test_branch_guard.py +++ b/tests/core/test_branch_guard.py @@ -4,12 +4,14 @@ from autoskillit.core.branch_guard import is_protected_branch -_DEFAULTS = ["main", "integration", "stable"] +pytestmark = [pytest.mark.layer("core"), pytest.mark.small] + +_DEFAULTS = ["main", "develop", "stable"] # ---------- standard protected list ---------- -@pytest.mark.parametrize("branch", ["main", "integration", "stable"]) +@pytest.mark.parametrize("branch", ["main", "develop", "stable"]) def test_default_protected_branches_are_rejected(branch: str) -> None: assert is_protected_branch(branch, protected=_DEFAULTS) is True diff --git a/tests/core/test_claude_env.py b/tests/core/test_claude_env.py index b34112cb31..392217fbda 100644 --- a/tests/core/test_claude_env.py +++ b/tests/core/test_claude_env.py @@ -2,6 +2,7 @@ from __future__ import annotations +import subprocess from types import MappingProxyType import pytest @@ -12,6 +13,9 @@ IDE_ENV_DENYLIST, IDE_ENV_PREFIX_DENYLIST, ) +from autoskillit.core.types._type_constants import AUTOSKILLIT_PRIVATE_ENV_VARS + +pytestmark = [pytest.mark.layer("core"), pytest.mark.small] def test_build_claude_env_strips_sse_port() -> None: @@ -130,3 +134,32 @@ def test_build_claude_env_strips_max_mcp_output_tokens() -> None: result = build_claude_env(base={"MAX_MCP_OUTPUT_TOKENS": "99999", "HOME": "/tmp"}) assert "MAX_MCP_OUTPUT_TOKENS" not in result assert result["HOME"] == "/tmp" + + +@pytest.mark.parametrize("var", sorted(AUTOSKILLIT_PRIVATE_ENV_VARS)) +def test_build_claude_env_scrubs_autoskillit_private_vars(var: str) -> None: + result = build_claude_env(base={var: "leaked", "HOME": "/tmp"}) + assert var not in result + assert result["HOME"] == "/tmp" + + +def test_build_claude_env_extras_override_scrubbed_private_vars() -> None: + result = build_claude_env( + base={"AUTOSKILLIT_SESSION_TYPE": "franchise", "HOME": "/tmp"}, + extras={"AUTOSKILLIT_SESSION_TYPE": "skill"}, + ) + assert result["AUTOSKILLIT_SESSION_TYPE"] == "skill" + + +def test_build_claude_env_subprocess_compatible() -> None: + """build_claude_env() result must be accepted by subprocess.Popen.""" + env = build_claude_env() + proc = subprocess.Popen( + ["echo", "ok"], + env=dict(env), + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + stdout, _ = proc.communicate(timeout=10) + assert proc.returncode == 0 + assert b"ok" in stdout diff --git a/tests/core/test_core.py b/tests/core/test_core.py index e46198fb01..d1ee04c9e6 100644 --- a/tests/core/test_core.py +++ b/tests/core/test_core.py @@ -2,6 +2,10 @@ from __future__ import annotations +import pytest + +pytestmark = [pytest.mark.layer("core"), pytest.mark.small] + def test_atomic_write_docstring_contains_atomic_keyword(): import autoskillit.core.io as m @@ -28,3 +32,87 @@ def test_t_typevar_not_in_core_all(): import autoskillit.core as core assert "T" not in core.__all__ + + +def test_core_init_uses_lazy_getattr(): + import autoskillit.core as core + + assert hasattr(core, "__getattr__") + assert callable(core.__getattr__) + + +def test_core_init_has_no_eager_submodule_imports(): + import ast + + from autoskillit.core.paths import pkg_root + + init_path = pkg_root() / "core" / "__init__.py" + tree = ast.parse(init_path.read_text()) + for node in ast.walk(tree): + if isinstance(node, ast.ImportFrom) and node.level == 1: + pytest.fail(f"Eager relative import found: from .{node.module} import ...") + + +def test_core_pyi_stub_exists(): + from autoskillit.core.paths import pkg_root + + pyi = pkg_root() / "core" / "__init__.pyi" + assert pyi.exists(), f"Missing stub: {pyi}" + + +def test_core_pyi_stub_consistent_with_all(): + import ast + + import autoskillit.core as core + from autoskillit.core.paths import pkg_root + + pyi = pkg_root() / "core" / "__init__.pyi" + tree = ast.parse(pyi.read_text()) + stub_names = set() + for node in ast.walk(tree): + if isinstance(node, ast.ImportFrom): + for alias in node.names: + stub_names.add(alias.asname or alias.name) + + all_set = set(core.__all__) + private_reexports = core._PRIVATE_REEXPORTS + assert all_set <= stub_names, f"Names in __all__ missing from stub: {all_set - stub_names}" + assert stub_names - all_set == private_reexports, ( + f"Unexpected stub-only names: {(stub_names - all_set) - private_reexports}" + ) + + +def test_core_pyi_uses_as_form(): + import ast + + from autoskillit.core.paths import pkg_root + + pyi = pkg_root() / "core" / "__init__.pyi" + tree = ast.parse(pyi.read_text()) + for node in ast.walk(tree): + if isinstance(node, ast.ImportFrom): + for alias in node.names: + assert alias.asname == alias.name, ( + f"Missing 'as' form: {alias.name} from .{node.module}" + ) + + +def test_closure_walk_does_not_detect_lazy_init(): + import sys + from pathlib import Path + + from autoskillit.core.paths import pkg_root + + tests_dir = Path(pkg_root()).parent.parent / "tests" + sys.path.insert(0, str(tests_dir)) + try: + from _test_filter import _expand_reexport_closure + + src_root = pkg_root() + changed = {"core/paths.py"} + expanded = _expand_reexport_closure(changed, src_root) + assert "core/__init__.py" not in expanded, ( + "Closure walk detected lazy __init__.py — PEP 562 migration incomplete" + ) + finally: + sys.path.remove(str(tests_dir)) diff --git a/tests/core/test_core_terminal_table.py b/tests/core/test_core_terminal_table.py index bdb1a4fdc5..53f9d588e5 100644 --- a/tests/core/test_core_terminal_table.py +++ b/tests/core/test_core_terminal_table.py @@ -2,6 +2,10 @@ from __future__ import annotations +import pytest + +pytestmark = [pytest.mark.layer("core"), pytest.mark.small] + def test_compute_col_widths_is_module_level() -> None: """_compute_col_widths must be importable as a module-level function. @@ -63,14 +67,3 @@ def test_core_terminal_table_is_importable() -> None: col = TerminalColumn(label="STEP", max_width=40, align="<") result = _render_terminal_table([col], [("a very long step name " * 5,)]) assert "…" in result, "Long values must be truncated with ellipsis" - - -def test_cli_terminal_table_reexports_from_core() -> None: - """cli/_terminal_table.py must re-export from core, not define its own copy.""" - from autoskillit.cli._terminal_table import TerminalColumn as CliTC - from autoskillit.core._terminal_table import TerminalColumn as CoreTC - - assert CliTC is CoreTC, ( - "cli._terminal_table.TerminalColumn must be the same object as " - "core._terminal_table.TerminalColumn (re-export, not a copy)" - ) diff --git a/tests/core/test_ensure_project_temp_with_config.py b/tests/core/test_ensure_project_temp_with_config.py index 7652f02a79..a81336482c 100644 --- a/tests/core/test_ensure_project_temp_with_config.py +++ b/tests/core/test_ensure_project_temp_with_config.py @@ -4,8 +4,12 @@ from pathlib import Path +import pytest + from autoskillit.core.io import ensure_project_temp +pytestmark = [pytest.mark.layer("core"), pytest.mark.small] + def test_ensure_project_temp_default_writes_self_gitignore(tmp_path: Path) -> None: result = ensure_project_temp(tmp_path) diff --git a/tests/core/test_feature_flags.py b/tests/core/test_feature_flags.py new file mode 100644 index 0000000000..cc06e1e227 --- /dev/null +++ b/tests/core/test_feature_flags.py @@ -0,0 +1,97 @@ +"""Tests for core/feature_flags.py — _collect_disabled_feature_tags helper.""" + +from __future__ import annotations + +import pytest + +pytestmark = [pytest.mark.layer("core"), pytest.mark.small] + + +class TestCollectDisabledFeatureTags: + def test_returns_frozenset(self): + from autoskillit.core.feature_flags import _collect_disabled_feature_tags + + result = _collect_disabled_feature_tags({}) + assert isinstance(result, frozenset) + + def test_disabled_fleet_returns_fleet_tag(self): + from autoskillit.core.feature_flags import _collect_disabled_feature_tags + + result = _collect_disabled_feature_tags({"fleet": False}) + assert "fleet" in result + + def test_enabled_fleet_not_in_result(self): + from autoskillit.core.feature_flags import _collect_disabled_feature_tags + + result = _collect_disabled_feature_tags({"fleet": True}) + assert "fleet" not in result + + def test_planner_skipped_empty_tool_tags(self): + from autoskillit.core.feature_flags import _collect_disabled_feature_tags + + # "planner" FeatureDef has tool_tags=frozenset() — must be skipped + result = _collect_disabled_feature_tags({"fleet": False, "planner": False}) + assert result == frozenset({"fleet"}) + + def test_default_disabled_fleet(self): + from autoskillit.core.feature_flags import _collect_disabled_feature_tags + + # fleet.default_enabled is False → fleet disabled when key absent + result = _collect_disabled_feature_tags({}) + assert "fleet" in result + + def test_hypothetical_third_feature_auto_discovered(self, monkeypatch): + from autoskillit.core import feature_flags as ff + from autoskillit.core.feature_flags import _collect_disabled_feature_tags + from autoskillit.core.types._type_constants import ( + FEATURE_REGISTRY, + FeatureDef, + FeatureLifecycle, + ) + + fake_def = FeatureDef( + lifecycle=FeatureLifecycle.EXPERIMENTAL, + description="Test-only gate", + tool_tags=frozenset({"testgate-tool"}), + skill_categories=frozenset(), + import_package=None, + default_enabled=False, + ) + patched_registry = {**FEATURE_REGISTRY, "testgate": fake_def} + monkeypatch.setattr(ff, "FEATURE_REGISTRY", patched_registry) + + result = _collect_disabled_feature_tags({}) + assert "testgate-tool" in result + + def test_union_model_tag_claimed_by_enabled_feature(self, monkeypatch): + from autoskillit.core import feature_flags as ff + from autoskillit.core.feature_flags import _collect_disabled_feature_tags + from autoskillit.core.types._type_constants import ( + FEATURE_REGISTRY, + FeatureDef, + FeatureLifecycle, + ) + + shared_tag = frozenset({"shared-tag"}) + def_a = FeatureDef( + lifecycle=FeatureLifecycle.EXPERIMENTAL, + description="A", + tool_tags=shared_tag, + skill_categories=frozenset(), + import_package=None, + default_enabled=False, + ) + def_b = FeatureDef( + lifecycle=FeatureLifecycle.EXPERIMENTAL, + description="B", + tool_tags=shared_tag, + skill_categories=frozenset(), + import_package=None, + default_enabled=True, + ) + patched = {**FEATURE_REGISTRY, "feat_a": def_a, "feat_b": def_b} + monkeypatch.setattr(ff, "FEATURE_REGISTRY", patched) + + result = _collect_disabled_feature_tags({"feat_a": False, "feat_b": True}) + # feat_b enabled claims the tag → must not appear in result + assert "shared-tag" not in result diff --git a/tests/core/test_github_url.py b/tests/core/test_github_url.py index c6152d6563..14b88f8c9f 100644 --- a/tests/core/test_github_url.py +++ b/tests/core/test_github_url.py @@ -6,6 +6,8 @@ from autoskillit.core import parse_github_repo +pytestmark = [pytest.mark.layer("core"), pytest.mark.small] + @pytest.mark.parametrize( "url,expected", diff --git a/tests/core/test_import_isolation.py b/tests/core/test_import_isolation.py new file mode 100644 index 0000000000..9ad6d8d308 --- /dev/null +++ b/tests/core/test_import_isolation.py @@ -0,0 +1,40 @@ +"""Fleet package must not be imported at server startup via lazy-import structure.""" + +from __future__ import annotations + +import subprocess +import sys + +import pytest + +pytestmark = [pytest.mark.layer("core"), pytest.mark.medium] + + +def test_server_import_does_not_load_fleet_package_lazily() -> None: + """ + Fleet module loading is deferred to lifespan/tool-call time via lazy imports. + A bare `import autoskillit.server` must not pull in autoskillit.fleet.* — + not because of an env-var gate, but because no top-level fleet import exists. + """ + code = ( + "import sys; import autoskillit.server; " + "fleet_modules = [k for k in sys.modules if k.startswith('autoskillit.fleet')]; " + "print(fleet_modules)" + ) + result = subprocess.run( + [sys.executable, "-c", code], capture_output=True, text=True, check=True + ) + assert result.stdout.strip() == "[]", f"Fleet modules leaked: {result.stdout}" + + +def test_cli_app_import_does_not_load_fleet_package() -> None: + """Importing autoskillit.cli.app must not trigger fleet package init.""" + code = ( + "import sys; import autoskillit.cli.app; " + "fleet_modules = [k for k in sys.modules if k.startswith('autoskillit.fleet')]; " + "print(fleet_modules)" + ) + result = subprocess.run( + [sys.executable, "-c", code], capture_output=True, text=True, check=True + ) + assert result.stdout.strip() == "[]", f"Fleet modules leaked: {result.stdout}" diff --git a/tests/core/test_install_detect.py b/tests/core/test_install_detect.py new file mode 100644 index 0000000000..6326e0e5e3 --- /dev/null +++ b/tests/core/test_install_detect.py @@ -0,0 +1,117 @@ +"""Tests for core/_install_detect.py — install-type detection.""" + +from __future__ import annotations + +import json +from unittest.mock import MagicMock + +import pytest + +pytestmark = [pytest.mark.layer("core"), pytest.mark.small] + + +def _fake_dist(direct_url_json: str | None) -> MagicMock: + dist = MagicMock() + dist.read_text.return_value = direct_url_json + return dist + + +def test_is_dev_install_editable_returns_true(monkeypatch: pytest.MonkeyPatch) -> None: + payload = json.dumps({"url": "file:///home/user/autoskillit", "dir_info": {"editable": True}}) + monkeypatch.setattr( + "importlib.metadata.Distribution.from_name", + lambda _name: _fake_dist(payload), + ) + from autoskillit.core._install_detect import is_dev_install + + assert is_dev_install() is True + + +def test_is_dev_install_git_vcs_stable_returns_false(monkeypatch: pytest.MonkeyPatch) -> None: + payload = json.dumps( + { + "url": "https://github.com/TalonT-Org/AutoSkillit.git", + "vcs_info": {"vcs": "git", "requested_revision": "stable", "commit_id": "abc123"}, + } + ) + monkeypatch.setattr( + "importlib.metadata.Distribution.from_name", + lambda _name: _fake_dist(payload), + ) + from autoskillit.core._install_detect import is_dev_install + + assert is_dev_install() is False + + +@pytest.mark.parametrize( + "revision, expected", + [ + ("develop", True), + ("feature-foo", True), + ("integration", True), + ("main", False), + ("stable", False), + ("v1.0.0", False), + ("v0.9.300", False), + (None, False), + ], +) +def test_is_dev_install_git_vcs_revision_matrix( + monkeypatch: pytest.MonkeyPatch, revision: str | None, expected: bool +) -> None: + vcs_info: dict = {"vcs": "git", "commit_id": "abc123"} + if revision is not None: + vcs_info["requested_revision"] = revision + payload = json.dumps( + {"url": "https://github.com/TalonT-Org/AutoSkillit.git", "vcs_info": vcs_info} + ) + monkeypatch.setattr( + "importlib.metadata.Distribution.from_name", + lambda _name: _fake_dist(payload), + ) + from autoskillit.core._install_detect import is_dev_install + + assert is_dev_install() is expected + + +def test_is_dev_install_local_path_returns_false(monkeypatch: pytest.MonkeyPatch) -> None: + payload = json.dumps({"url": "file:///home/user/autoskillit", "dir_info": {}}) + monkeypatch.setattr( + "importlib.metadata.Distribution.from_name", + lambda _name: _fake_dist(payload), + ) + from autoskillit.core._install_detect import is_dev_install + + assert is_dev_install() is False + + +def test_is_dev_install_unknown_returns_false(monkeypatch: pytest.MonkeyPatch) -> None: + import importlib.metadata + + def _raise(_name: str) -> None: + raise importlib.metadata.PackageNotFoundError("autoskillit") + + monkeypatch.setattr("importlib.metadata.Distribution.from_name", _raise) + from autoskillit.core._install_detect import is_dev_install + + assert is_dev_install() is False + + +def test_is_dev_install_no_direct_url_returns_false(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr( + "importlib.metadata.Distribution.from_name", + lambda _name: _fake_dist(None), + ) + from autoskillit.core._install_detect import is_dev_install + + assert is_dev_install() is False + + +def test_is_dev_install_malformed_json_returns_false(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr( + "importlib.metadata.Distribution.from_name", + lambda _name: _fake_dist("not-valid-json{{{"), + ) + from autoskillit.core._install_detect import is_dev_install + + assert is_dev_install() is False diff --git a/tests/core/test_io.py b/tests/core/test_io.py index f8ba57ea85..48bfa3061a 100644 --- a/tests/core/test_io.py +++ b/tests/core/test_io.py @@ -2,8 +2,14 @@ from __future__ import annotations +import json + import pytest +from autoskillit.core.io import write_versioned_json + +pytestmark = [pytest.mark.layer("core"), pytest.mark.small] + class TestLoadYamlExtended: def test_accepts_path(self, tmp_path): @@ -43,6 +49,48 @@ def test_load_yaml_str_raises_yaml_error_on_invalid(self): with pytest.raises(YAMLError): load_yaml("{bad yaml: [unclosed") + @pytest.mark.parametrize( + "input_kind", + ["string", "path"], + ids=["str-input", "path-input"], + ) + def test_load_yaml_uses_c_loader_when_available(self, tmp_path, monkeypatch, input_kind): + import yaml as _yaml + + from autoskillit.core.io import load_yaml + + if not getattr(_yaml, "__with_libyaml__", False): + pytest.skip("LibYAML not available") + captured: dict[str, object] = {} + original_load = _yaml.load + + def spy(data, *, Loader=None, **kw): + assert not kw, f"unexpected kwargs passed to yaml.load: {kw}" + captured["Loader"] = Loader + return original_load(data, Loader=Loader, **kw) + + monkeypatch.setattr("autoskillit.core.io.yaml.load", spy) + + if input_kind == "path": + p = tmp_path / "t.yaml" + p.write_text("a: 1\n") + result = load_yaml(p) + assert result == {"a": 1} + else: + load_yaml("key: val") + + assert captured["Loader"] is _yaml.CSafeLoader + + def test_loader_is_csafe_or_safe(self): + import yaml as _yaml + + from autoskillit.core.io import _Loader + + if getattr(_yaml, "__with_libyaml__", False): + assert _Loader is _yaml.CSafeLoader + else: + assert _Loader is _yaml.SafeLoader + class TestDumpYamlStr: def test_roundtrip_with_load_yaml(self): @@ -71,6 +119,34 @@ def test_default_flow_style_false_honored(self): # Block style: items on separate lines, no inline [...] for lists assert "[1, 2, 3]" not in result + def test_dumper_is_cdumper_or_dumper(self): + import yaml as _yaml + + from autoskillit.core.io import _Dumper + + if getattr(_yaml, "__with_libyaml__", False): + assert _Dumper is _yaml.CDumper + else: + assert _Dumper is _yaml.Dumper + + def test_dump_yaml_str_uses_c_dumper_when_available(self, monkeypatch): + import yaml as _yaml + + from autoskillit.core.io import dump_yaml_str + + if not getattr(_yaml, "__with_libyaml__", False): + pytest.skip("LibYAML not available") + captured: dict[str, object] = {} + original_dump = _yaml.dump + + def spy(data, **kw): + captured["Dumper"] = kw.get("Dumper") + return original_dump(data, **kw) + + monkeypatch.setattr("autoskillit.core.io.yaml.dump", spy) + dump_yaml_str({"x": 1}) + assert captured["Dumper"] is _yaml.CDumper + class TestYamlConsolidationArchitecture: def test_only_yaml_imports_yaml_directly(self): @@ -126,7 +202,6 @@ def test_atomic_write_private_alias_removed(): def test_write_versioned_json_enriches_payload_with_schema_version(tmp_path): - import json from autoskillit.core.io import write_versioned_json @@ -138,7 +213,6 @@ def test_write_versioned_json_enriches_payload_with_schema_version(tmp_path): def test_write_versioned_json_preserves_existing_keys_atomically(tmp_path, monkeypatch): """Asserts the helper routes through ``atomic_write`` (no partial-file fallout on a simulated mid-write crash).""" - import json from autoskillit.core import io as io_mod from autoskillit.core.io import write_versioned_json @@ -165,9 +239,16 @@ def spy(path, content): def test_write_versioned_json_rejects_non_dict_payload(tmp_path): import pytest - from autoskillit.core.io import write_versioned_json - target = tmp_path / "bad.json" with pytest.raises(TypeError, match="dict payload"): write_versioned_json(target, [1, 2, 3], schema_version=1) # type: ignore[arg-type] assert not target.exists() + + +def test_write_versioned_json_produces_indented_output(tmp_path): + target = tmp_path / "f.json" + write_versioned_json(target, {"a": 1, "b": [2, 3]}, schema_version=1) + raw = target.read_text(encoding="utf-8") + lines = raw.strip().splitlines() + assert len(lines) > 1, "Output must be multi-line (indented)" + assert json.loads(raw) == {"a": 1, "b": [2, 3], "schema_version": 1} diff --git a/tests/core/test_json.py b/tests/core/test_json.py new file mode 100644 index 0000000000..7d2c7d7d92 --- /dev/null +++ b/tests/core/test_json.py @@ -0,0 +1,98 @@ +"""Tests for autoskillit.core._json fast JSON wrapper.""" + +from __future__ import annotations + +import importlib +import json +import sys + +import pytest + +from autoskillit.core._json import fast_dumps, fast_loads + +pytestmark = [pytest.mark.layer("core"), pytest.mark.small] + + +def test_fast_loads_dict() -> None: + assert fast_loads('{"a": 1}') == {"a": 1} + + +def test_fast_loads_bytes_input() -> None: + assert fast_loads(b'{"x": true}') == {"x": True} + + +def test_fast_loads_list() -> None: + assert fast_loads("[1, 2, 3]") == [1, 2, 3] + + +def test_fast_loads_returns_standard_python_types() -> None: + result = fast_loads('{"a": 1, "b": [true, null]}') + assert isinstance(result, dict) + assert isinstance(result["b"], list) + + +def test_fast_loads_raises_json_decode_error_on_invalid() -> None: + with pytest.raises(json.JSONDecodeError): + fast_loads("{not valid}") + + +def test_fast_dumps_returns_str() -> None: + result = fast_dumps({"a": 1}) + assert isinstance(result, str) + assert json.loads(result) == {"a": 1} + + +def test_fast_dumps_sort_keys_orders_keys() -> None: + result = fast_dumps({"z": 1, "a": 2}, sort_keys=True) + assert result.index('"a"') < result.index('"z"') + + +def test_fast_dumps_indent_produces_multiline() -> None: + result = fast_dumps({"a": 1}, indent=True) + assert ( + ' "a"' in result + ) # both orjson (OPT_INDENT_2) and stdlib (indent=2) produce 2-space indent + + +def test_fast_dumps_default_handler_handles_set() -> None: + result = fast_dumps({"s": {"a", "b"}}, default=list) + data = json.loads(result) + assert set(data["s"]) == {"a", "b"} + + +def test_fast_dumps_roundtrip_with_fast_loads() -> None: + original = {"key": [1, 2, 3], "flag": True, "n": None} + assert fast_loads(fast_dumps(original)) == original + + +@pytest.fixture() +def stdlib_json_mod(monkeypatch: pytest.MonkeyPatch): + """Provides _json reimported with orjson blocked to exercise the stdlib fallback. + + Scope note: this patch only affects the re-imported module object returned here. + Other modules that already imported `from autoskillit.core._json import fast_dumps` + at collection time retain the orjson-backed callable; those callables are not patched. + """ + monkeypatch.setitem(sys.modules, "orjson", None) # type: ignore[arg-type] + monkeypatch.delitem(sys.modules, "autoskillit.core._json", raising=False) + return importlib.import_module("autoskillit.core._json") + + +def test_stdlib_fallback_fast_loads(stdlib_json_mod) -> None: + assert not hasattr(stdlib_json_mod, "_orjson"), ( + "orjson must not be accessible in the stdlib fallback module" + ) + assert stdlib_json_mod._USE_ORJSON is False, ( + "_USE_ORJSON sentinel must be False when orjson is blocked" + ) + assert stdlib_json_mod.fast_loads('{"a": 1}') == {"a": 1} + assert stdlib_json_mod.fast_loads(b'{"x": true}') == {"x": True} + with pytest.raises(json.JSONDecodeError): + stdlib_json_mod.fast_loads("{not valid}") + + +def test_stdlib_fallback_fast_dumps(stdlib_json_mod) -> None: + result = stdlib_json_mod.fast_dumps({"a": 1}) + assert isinstance(result, str) + assert json.loads(result) == {"a": 1} + assert ' "a"' in stdlib_json_mod.fast_dumps({"a": 1}, indent=True) diff --git a/tests/core/test_kitchen_state.py b/tests/core/test_kitchen_state.py new file mode 100644 index 0000000000..e9fb728da5 --- /dev/null +++ b/tests/core/test_kitchen_state.py @@ -0,0 +1,285 @@ +"""Tests for KitchenMarker hash field support.""" + +from __future__ import annotations + +import json +from datetime import UTC, datetime +from pathlib import Path + +import pytest + +pytestmark = [pytest.mark.layer("core"), pytest.mark.small] + + +def test_kitchen_marker_has_hash_fields(): + from autoskillit.core.runtime.kitchen_state import KitchenMarker + + marker = KitchenMarker( + session_id="s", + opened_at=datetime.now(UTC), + recipe_name="r", + content_hash="sha256:abc", + composite_hash="sha256:def", + ) + assert marker.content_hash == "sha256:abc" + assert marker.composite_hash == "sha256:def" + + +def test_marker_roundtrip_with_hashes(tmp_path, monkeypatch): + from autoskillit.core.runtime.kitchen_state import read_marker, write_marker + + monkeypatch.setenv("AUTOSKILLIT_STATE_DIR", str(tmp_path)) + write_marker("sess1", "recipe", content_hash="sha256:a", composite_hash="sha256:b") + marker = read_marker("sess1") + assert marker is not None + assert marker.content_hash == "sha256:a" + assert marker.composite_hash == "sha256:b" + + +def test_marker_backward_compat_no_hashes(tmp_path, monkeypatch): + from autoskillit.core.runtime.kitchen_state import read_marker + + monkeypatch.setenv("AUTOSKILLIT_STATE_DIR", str(tmp_path)) + state_dir = tmp_path / "kitchen_state" + state_dir.mkdir() + (state_dir / "old.json").write_text( + json.dumps( + { + "session_id": "old", + "opened_at": datetime.now(UTC).isoformat(), + "recipe_name": "r", + "marker_version": 1, + } + ) + ) + marker = read_marker("old") + assert marker is not None + assert marker.content_hash == "" + assert marker.composite_hash == "" + + +def test_kitchen_marker_hash_defaults(): + from autoskillit.core.runtime.kitchen_state import KitchenMarker + + marker = KitchenMarker( + session_id="s", + opened_at=datetime.now(UTC), + recipe_name="r", + ) + assert marker.content_hash == "" + assert marker.composite_hash == "" + + +# --- Group P-1: Kitchen state namespacing --- + + +def test_get_state_dir_namespaces_by_campaign_id(tmp_path, monkeypatch): + """get_state_dir returns campaign-scoped subdirectory when AUTOSKILLIT_CAMPAIGN_ID is set.""" + from autoskillit.core.runtime.kitchen_state import get_state_dir + + monkeypatch.delenv("AUTOSKILLIT_STATE_DIR", raising=False) + monkeypatch.setenv("AUTOSKILLIT_CAMPAIGN_ID", "camp-42") + monkeypatch.setattr(Path, "cwd", lambda: tmp_path) + result = get_state_dir() + assert result == tmp_path / ".autoskillit" / "temp" / "kitchen_state" / "camp-42" + + +def test_get_state_dir_no_campaign_id_flat(tmp_path, monkeypatch): + """get_state_dir returns flat path when AUTOSKILLIT_CAMPAIGN_ID is absent.""" + from autoskillit.core.runtime.kitchen_state import get_state_dir + + monkeypatch.setenv("AUTOSKILLIT_STATE_DIR", str(tmp_path)) + monkeypatch.delenv("AUTOSKILLIT_CAMPAIGN_ID", raising=False) + result = get_state_dir() + assert result == tmp_path / "kitchen_state" + + +def test_concurrent_campaigns_disjoint_dirs(tmp_path, monkeypatch): + """Two different campaign_ids produce completely disjoint marker directories.""" + from autoskillit.core.runtime.kitchen_state import get_state_dir + + monkeypatch.delenv("AUTOSKILLIT_STATE_DIR", raising=False) + monkeypatch.setattr(Path, "cwd", lambda: tmp_path) + + monkeypatch.setenv("AUTOSKILLIT_CAMPAIGN_ID", "campaign-a") + dir_a = get_state_dir() + + monkeypatch.setenv("AUTOSKILLIT_CAMPAIGN_ID", "campaign-b") + dir_b = get_state_dir() + + assert dir_a != dir_b + assert "campaign-a" in str(dir_a) + assert "campaign-b" in str(dir_b) + + +def test_state_dir_override_takes_precedence_over_campaign(tmp_path, monkeypatch): + """AUTOSKILLIT_STATE_DIR override still takes priority even when campaign_id is set.""" + from autoskillit.core.runtime.kitchen_state import get_state_dir + + monkeypatch.setenv("AUTOSKILLIT_STATE_DIR", str(tmp_path)) + monkeypatch.setenv("AUTOSKILLIT_CAMPAIGN_ID", "camp-42") + result = get_state_dir() + # Override path does NOT include campaign_id — it's a test-isolation override + assert result == tmp_path / "kitchen_state" + + +def test_marker_roundtrip_with_campaign_namespace(tmp_path, monkeypatch): + """write_marker + read_marker work correctly under campaign namespacing.""" + from autoskillit.core.runtime.kitchen_state import read_marker, write_marker + + monkeypatch.setenv("AUTOSKILLIT_STATE_DIR", str(tmp_path)) + monkeypatch.setenv("AUTOSKILLIT_CAMPAIGN_ID", "camp-99") + write_marker("sess-1", "my-recipe") + marker = read_marker("sess-1") + assert marker is not None + assert marker.session_id == "sess-1" + assert marker.recipe_name == "my-recipe" + + +def test_sweep_stale_markers_scoped_to_namespace(tmp_path, monkeypatch): + """sweep_stale_markers only sweeps within its own namespace.""" + import json + from datetime import UTC, datetime, timedelta + + from autoskillit.core.runtime.kitchen_state import sweep_stale_markers + + monkeypatch.delenv("AUTOSKILLIT_STATE_DIR", raising=False) + monkeypatch.chdir(tmp_path) + + # Create stale marker in campaign-a namespace + monkeypatch.setenv("AUTOSKILLIT_CAMPAIGN_ID", "campaign-a") + from autoskillit.core.runtime.kitchen_state import get_state_dir + + dir_a = get_state_dir() + dir_a.mkdir(parents=True, exist_ok=True) + stale_time = (datetime.now(UTC) - timedelta(hours=25)).isoformat() + (dir_a / "stale.json").write_text( + json.dumps( + { + "session_id": "stale", + "opened_at": stale_time, + "recipe_name": None, + "marker_version": 1, + } + ) + ) + + # Create fresh marker in campaign-b namespace + monkeypatch.setenv("AUTOSKILLIT_CAMPAIGN_ID", "campaign-b") + dir_b = get_state_dir() + dir_b.mkdir(parents=True, exist_ok=True) + fresh_time = datetime.now(UTC).isoformat() + (dir_b / "fresh.json").write_text( + json.dumps( + { + "session_id": "fresh", + "opened_at": fresh_time, + "recipe_name": None, + "marker_version": 1, + } + ) + ) + + # Sweep in campaign-a scope — should only delete from campaign-a + monkeypatch.setenv("AUTOSKILLIT_CAMPAIGN_ID", "campaign-a") + deleted = sweep_stale_markers() + assert deleted == 1 + + # campaign-b marker untouched + assert (dir_b / "fresh.json").exists() + + +# --- find_caller_session_id --- + + +def test_find_caller_session_id_returns_empty_when_no_dir(tmp_path, monkeypatch): + from autoskillit.core.runtime.kitchen_state import find_caller_session_id + + monkeypatch.setenv("AUTOSKILLIT_STATE_DIR", str(tmp_path)) + result = find_caller_session_id() + assert result == "" + + +def test_find_caller_session_id_returns_empty_when_dir_empty(tmp_path, monkeypatch): + from autoskillit.core.runtime.kitchen_state import find_caller_session_id + + monkeypatch.setenv("AUTOSKILLIT_STATE_DIR", str(tmp_path)) + (tmp_path / "kitchen_state").mkdir() + result = find_caller_session_id() + assert result == "" + + +def test_find_caller_session_id_returns_fresh_marker(tmp_path, monkeypatch): + from autoskillit.core.runtime.kitchen_state import find_caller_session_id, write_marker + + monkeypatch.setenv("AUTOSKILLIT_STATE_DIR", str(tmp_path)) + write_marker("sess-abc", "recipe-1") + result = find_caller_session_id() + assert result == "sess-abc" + + +def test_find_caller_session_id_picks_freshest_by_mtime(tmp_path, monkeypatch): + import time + + from autoskillit.core.runtime.kitchen_state import find_caller_session_id, write_marker + + monkeypatch.setenv("AUTOSKILLIT_STATE_DIR", str(tmp_path)) + write_marker("sess-old", "recipe-1") + time.sleep(0.05) + write_marker("sess-new", "recipe-2") + result = find_caller_session_id() + assert result == "sess-new" + + +def test_find_caller_session_id_skips_stale_markers(tmp_path, monkeypatch): + from datetime import UTC, datetime, timedelta + + from autoskillit.core.runtime.kitchen_state import find_caller_session_id + + monkeypatch.setenv("AUTOSKILLIT_STATE_DIR", str(tmp_path)) + state_dir = tmp_path / "kitchen_state" + state_dir.mkdir() + stale_time = (datetime.now(UTC) - timedelta(hours=25)).isoformat() + (state_dir / "stale-sess.json").write_text( + json.dumps( + { + "session_id": "stale-sess", + "opened_at": stale_time, + "recipe_name": "r", + "marker_version": 1, + } + ) + ) + result = find_caller_session_id() + assert result == "" + + +def test_find_caller_session_id_uses_state_dir_override(tmp_path, monkeypatch): + from autoskillit.core.runtime.kitchen_state import find_caller_session_id, write_marker + + monkeypatch.setenv("AUTOSKILLIT_STATE_DIR", str(tmp_path)) + write_marker("sess-override", "recipe-1") + result = find_caller_session_id() + assert result == "sess-override" + + +def test_find_caller_session_id_with_campaign_id(tmp_path, monkeypatch): + from autoskillit.core.runtime.kitchen_state import find_caller_session_id, write_marker + + monkeypatch.delenv("AUTOSKILLIT_STATE_DIR", raising=False) + monkeypatch.setenv("AUTOSKILLIT_CAMPAIGN_ID", "camp-x") + monkeypatch.setattr(Path, "cwd", lambda: tmp_path) + write_marker("sess-camp", "recipe-1") + result = find_caller_session_id() + assert result == "sess-camp" + + +def test_find_caller_session_id_project_dir_argument(tmp_path, monkeypatch): + from autoskillit.core.runtime.kitchen_state import find_caller_session_id, write_marker + + monkeypatch.delenv("AUTOSKILLIT_STATE_DIR", raising=False) + monkeypatch.delenv("AUTOSKILLIT_CAMPAIGN_ID", raising=False) + monkeypatch.setattr(Path, "cwd", lambda: tmp_path) + write_marker("sess-proj", "recipe-1") + result = find_caller_session_id(project_dir=tmp_path) + assert result == "sess-proj" diff --git a/tests/core/test_label_lifecycle.py b/tests/core/test_label_lifecycle.py new file mode 100644 index 0000000000..e0a26c0e99 --- /dev/null +++ b/tests/core/test_label_lifecycle.py @@ -0,0 +1,160 @@ +"""Tests for IssueLabelState, LabelDef, LABEL_LIFECYCLE_REGISTRY, and LABEL_TRANSITIONS.""" + +from __future__ import annotations + +import pytest + +from autoskillit.core.types import ( + LABEL_LIFECYCLE_REGISTRY, + LABEL_TRANSITIONS, + IssueLabelState, + validate_label_transition, +) + +pytestmark = [pytest.mark.layer("core"), pytest.mark.small] + + +class TestIssueLabelState: + def test_all_members_have_registry_entry(self): + """Every IssueLabelState member exists as a key in LABEL_LIFECYCLE_REGISTRY.""" + for state in IssueLabelState: + assert state in LABEL_LIFECYCLE_REGISTRY + + def test_registry_has_no_extra_keys(self): + """Every key in LABEL_LIFECYCLE_REGISTRY is a valid IssueLabelState member.""" + for key in LABEL_LIFECYCLE_REGISTRY: + assert isinstance(key, IssueLabelState) + + def test_queued_value(self): + assert IssueLabelState.QUEUED == "queued" + + def test_in_progress_value(self): + assert IssueLabelState.IN_PROGRESS == "in-progress" + + def test_staged_value(self): + assert IssueLabelState.STAGED == "staged" + + def test_fail_value(self): + assert IssueLabelState.FAIL == "fail" + + +class TestLabelDef: + def test_all_defs_have_nonempty_color(self): + """Every LabelDef in the registry has a truthy .color string.""" + for defn in LABEL_LIFECYCLE_REGISTRY.values(): + assert defn.color + + def test_all_defs_have_nonempty_description(self): + """Every LabelDef in the registry has a truthy .description string.""" + for defn in LABEL_LIFECYCLE_REGISTRY.values(): + assert defn.description + + def test_removes_on_entry_reference_valid_states(self): + """Every state in every LabelDef.removes_on_entry is a valid IssueLabelState member.""" + for defn in LABEL_LIFECYCLE_REGISTRY.values(): + for removed in defn.removes_on_entry: + assert isinstance(removed, IssueLabelState) + + def test_queued_color(self): + assert LABEL_LIFECYCLE_REGISTRY[IssueLabelState.QUEUED].color == "c2e0c6" + + def test_in_progress_color(self): + assert LABEL_LIFECYCLE_REGISTRY[IssueLabelState.IN_PROGRESS].color == "fbca04" + + def test_staged_color(self): + assert LABEL_LIFECYCLE_REGISTRY[IssueLabelState.STAGED].color == "0075ca" + + def test_fail_color(self): + assert LABEL_LIFECYCLE_REGISTRY[IssueLabelState.FAIL].color == "d73a4a" + + def test_in_progress_removes_queued_and_fail(self): + assert LABEL_LIFECYCLE_REGISTRY[IssueLabelState.IN_PROGRESS].removes_on_entry == frozenset( + {IssueLabelState.QUEUED, IssueLabelState.FAIL} + ) + + def test_queued_removes_fail(self): + assert LABEL_LIFECYCLE_REGISTRY[IssueLabelState.QUEUED].removes_on_entry == frozenset( + {IssueLabelState.FAIL} + ) + + def test_staged_removes_in_progress_fail_and_queued(self): + assert LABEL_LIFECYCLE_REGISTRY[IssueLabelState.STAGED].removes_on_entry == frozenset( + {IssueLabelState.IN_PROGRESS, IssueLabelState.FAIL, IssueLabelState.QUEUED} + ) + + def test_fail_removes_in_progress_and_queued(self): + assert LABEL_LIFECYCLE_REGISTRY[IssueLabelState.FAIL].removes_on_entry == frozenset( + {IssueLabelState.IN_PROGRESS, IssueLabelState.QUEUED} + ) + + +class TestLabelTransitions: + def test_unlabeled_to_queued(self): + """validate_label_transition accepts unlabeled -> queued.""" + validate_label_transition(None, IssueLabelState.QUEUED) + + def test_unlabeled_to_in_progress(self): + """validate_label_transition accepts unlabeled -> in-progress.""" + validate_label_transition(None, IssueLabelState.IN_PROGRESS) + + def test_queued_to_in_progress(self): + """validate_label_transition accepts queued -> in-progress.""" + validate_label_transition(IssueLabelState.QUEUED, IssueLabelState.IN_PROGRESS) + + def test_queued_to_none(self): + """validate_label_transition accepts queued -> None (release/unlabel).""" + validate_label_transition(IssueLabelState.QUEUED, None) + + def test_in_progress_to_staged(self): + """validate_label_transition accepts in-progress -> staged.""" + validate_label_transition(IssueLabelState.IN_PROGRESS, IssueLabelState.STAGED) + + def test_in_progress_to_fail(self): + """validate_label_transition accepts in-progress -> fail.""" + validate_label_transition(IssueLabelState.IN_PROGRESS, IssueLabelState.FAIL) + + def test_in_progress_to_none(self): + """validate_label_transition accepts in-progress -> None (bare removal).""" + validate_label_transition(IssueLabelState.IN_PROGRESS, None) + + def test_fail_to_queued(self): + """validate_label_transition accepts fail -> queued (retry reclaim).""" + validate_label_transition(IssueLabelState.FAIL, IssueLabelState.QUEUED) + + def test_fail_to_in_progress(self): + """validate_label_transition accepts fail -> in-progress (retry reclaim).""" + validate_label_transition(IssueLabelState.FAIL, IssueLabelState.IN_PROGRESS) + + def test_invalid_in_progress_to_queued(self): + """validate_label_transition rejects in-progress -> queued.""" + with pytest.raises(ValueError, match="Invalid label transition"): + validate_label_transition(IssueLabelState.IN_PROGRESS, IssueLabelState.QUEUED) + + def test_invalid_staged_to_in_progress(self): + """validate_label_transition rejects staged -> in-progress.""" + with pytest.raises(ValueError, match="Invalid label transition"): + validate_label_transition(IssueLabelState.STAGED, IssueLabelState.IN_PROGRESS) + + def test_invalid_staged_to_queued(self): + """validate_label_transition rejects staged -> queued.""" + with pytest.raises(ValueError, match="Invalid label transition"): + validate_label_transition(IssueLabelState.STAGED, IssueLabelState.QUEUED) + + def test_invalid_queued_to_staged(self): + """validate_label_transition rejects queued -> staged.""" + with pytest.raises(ValueError, match="Invalid label transition"): + validate_label_transition(IssueLabelState.QUEUED, IssueLabelState.STAGED) + + def test_invalid_queued_to_fail(self): + """validate_label_transition rejects queued -> fail.""" + with pytest.raises(ValueError, match="Invalid label transition"): + validate_label_transition(IssueLabelState.QUEUED, IssueLabelState.FAIL) + + def test_every_enum_member_in_transition_table(self): + """Every IssueLabelState member appears as a key in LABEL_TRANSITIONS.""" + for state in IssueLabelState: + assert state in LABEL_TRANSITIONS + + def test_none_in_transition_table(self): + """None (unlabeled) appears as a key in LABEL_TRANSITIONS.""" + assert None in LABEL_TRANSITIONS diff --git a/tests/core/test_logging.py b/tests/core/test_logging.py index be4012f5f9..6aed1bba09 100644 --- a/tests/core/test_logging.py +++ b/tests/core/test_logging.py @@ -11,6 +11,8 @@ from tests._helpers import _flush_structlog_proxy_caches as _flush_logger_proxy_caches +pytestmark = [pytest.mark.layer("core"), pytest.mark.small] + class TestGetLogger: def test_returns_bound_logger(self): diff --git a/tests/core/test_paths.py b/tests/core/test_paths.py index 1dfb09c676..8cb129f413 100644 --- a/tests/core/test_paths.py +++ b/tests/core/test_paths.py @@ -5,6 +5,10 @@ import importlib.resources as ir from pathlib import Path +import pytest + +pytestmark = [pytest.mark.layer("core"), pytest.mark.small] + class TestWorktreeDetection: def test_detects_main_checkout_as_not_worktree(self, tmp_path: Path) -> None: diff --git a/tests/core/test_resolve_temp_dir.py b/tests/core/test_resolve_temp_dir.py index 62e5acc35d..7f389cbffb 100644 --- a/tests/core/test_resolve_temp_dir.py +++ b/tests/core/test_resolve_temp_dir.py @@ -9,6 +9,8 @@ from autoskillit.core.io import resolve_temp_dir +pytestmark = [pytest.mark.layer("core"), pytest.mark.medium] + def test_resolve_temp_dir_default_returns_project_relative() -> None: assert resolve_temp_dir(Path("/proj"), None) == Path("/proj/.autoskillit/temp") diff --git a/tests/core/test_session_checkpoint.py b/tests/core/test_session_checkpoint.py new file mode 100644 index 0000000000..19157ec37b --- /dev/null +++ b/tests/core/test_session_checkpoint.py @@ -0,0 +1,68 @@ +"""T1: SessionCheckpoint schema validation.""" + +from __future__ import annotations + +import pytest + +from autoskillit.core.types._type_checkpoint import SessionCheckpoint, compute_remaining + +pytestmark = [pytest.mark.layer("core"), pytest.mark.small] + + +class TestSessionCheckpoint: + def test_round_trip_json(self) -> None: + cp = SessionCheckpoint( + completed_items=["item_a", "item_b"], + step_name="step_1", + progress_pct=50.0, + ts="2026-05-04T12:00:00+00:00", + ) + d = cp.to_dict() + restored = SessionCheckpoint.from_dict(d) + assert restored == cp + + def test_defaults(self) -> None: + cp = SessionCheckpoint() + assert cp.completed_items == [] + assert cp.step_name == "" + assert cp.progress_pct == 0.0 + assert cp.ts == "" + + def test_from_dict_missing_fields_uses_defaults(self) -> None: + cp = SessionCheckpoint.from_dict({}) + assert cp.completed_items == [] + assert cp.step_name == "" + + def test_now_factory(self) -> None: + cp = SessionCheckpoint.now(["a", "b"], step_name="s1", progress_pct=75.0) + assert cp.completed_items == ["a", "b"] + assert cp.step_name == "s1" + assert cp.progress_pct == 75.0 + assert cp.ts != "" + + def test_frozen(self) -> None: + cp = SessionCheckpoint(completed_items=["x"]) + with pytest.raises(AttributeError): + cp.step_name = "mutated" # type: ignore[misc] + + +class TestComputeRemaining: + def test_subtracts_completed(self) -> None: + cp = SessionCheckpoint(completed_items=["a", "c"]) + remaining = compute_remaining(["a", "b", "c", "d"], cp) + assert remaining == ["b", "d"] + + def test_preserves_order(self) -> None: + cp = SessionCheckpoint(completed_items=["z"]) + remaining = compute_remaining(["z", "y", "x"], cp) + assert remaining == ["y", "x"] + + def test_empty_checkpoint(self) -> None: + cp = SessionCheckpoint() + remaining = compute_remaining(["a", "b"], cp) + assert remaining == ["a", "b"] + + def test_all_completed(self) -> None: + cp = SessionCheckpoint(completed_items=["a", "b"]) + remaining = compute_remaining(["a", "b"], cp) + assert remaining == [] diff --git a/tests/core/test_session_index_schema.py b/tests/core/test_session_index_schema.py new file mode 100644 index 0000000000..bdd84a9e52 --- /dev/null +++ b/tests/core/test_session_index_schema.py @@ -0,0 +1,27 @@ +"""Tests verifying SessionIndexEntry TypedDict matches the actual JSONL output.""" + +from __future__ import annotations + +import pytest + +pytestmark = [pytest.mark.layer("core"), pytest.mark.small] + +_REQUIRED_INDEX_FIELDS = { + "provider_used", + "provider_fallback", + "recipe_name", + "recipe_content_hash", + "recipe_composite_hash", + "recipe_version", +} + + +class TestSessionIndexEntryCompleteness: + """SessionIndexEntry TypedDict must declare every field written to sessions.jsonl.""" + + def test_required_fields_declared(self): + from autoskillit.core.types._type_results import SessionIndexEntry + + declared = set(SessionIndexEntry.__annotations__) + missing = _REQUIRED_INDEX_FIELDS - declared + assert not missing, f"SessionIndexEntry missing fields: {missing}" diff --git a/tests/core/test_session_liveness.py b/tests/core/test_session_liveness.py new file mode 100644 index 0000000000..c7edead4dc --- /dev/null +++ b/tests/core/test_session_liveness.py @@ -0,0 +1,48 @@ +"""T2: is_session_alive generalized liveness.""" + +from __future__ import annotations + +import os + +import pytest + +from autoskillit.core.runtime._linux_proc import ( + is_session_alive, + read_boot_id, + read_starttime_ticks, +) + +pytestmark = [pytest.mark.layer("core"), pytest.mark.small] + + +class TestIsSessionAlive: + def test_zero_pid_not_alive(self) -> None: + assert is_session_alive(0, "some-boot-id", 12345) is False + + def test_empty_boot_id_not_alive(self) -> None: + assert is_session_alive(os.getpid(), "", 12345) is False + + def test_different_boot_id_not_alive(self) -> None: + boot_id = read_boot_id() + if boot_id is None: + pytest.skip("Not on Linux") + assert is_session_alive(os.getpid(), "wrong-boot-id-xxx", 12345) is False + + def test_nonexistent_pid_not_alive(self) -> None: + boot_id = read_boot_id() + if boot_id is None: + pytest.skip("Not on Linux") + assert is_session_alive(999999999, boot_id, 12345) is False + + def test_ticks_mismatch_not_alive(self) -> None: + boot_id = read_boot_id() + if boot_id is None: + pytest.skip("Not on Linux") + assert is_session_alive(os.getpid(), boot_id, -1) is False + + def test_current_process_is_alive(self) -> None: + boot_id = read_boot_id() + ticks = read_starttime_ticks(os.getpid()) + if boot_id is None or ticks is None: + pytest.skip("Not on Linux") + assert is_session_alive(os.getpid(), boot_id, ticks) is True diff --git a/tests/core/test_session_provenance.py b/tests/core/test_session_provenance.py new file mode 100644 index 0000000000..6ac3c1b67e --- /dev/null +++ b/tests/core/test_session_provenance.py @@ -0,0 +1,136 @@ +"""Tests for core/runtime/session_provenance.py — provenance store for L2 sessions.""" + +from __future__ import annotations + +import json +from pathlib import Path + +import pytest + +from autoskillit.core.runtime.session_provenance import ( + ProvenanceRecord, + provenance_path, + read_provenance_for_session, + write_provenance_record, +) + +pytestmark = [pytest.mark.layer("core"), pytest.mark.small] + + +def _make_record(**overrides: str) -> ProvenanceRecord: + defaults = { + "session_id": "sess-1", + "caller_session_id": "caller-1", + "kitchen_id": "kitchen-1", + "dispatch_id": "dispatch-1", + "recipe_name": "test-recipe", + "step_name": "step-a", + "timestamp": "2026-01-01T00:00:00Z", + } + defaults.update(overrides) + return ProvenanceRecord(**defaults) + + +class TestProvenancePath: + def test_default_resolution(self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.delenv("AUTOSKILLIT_STATE_DIR", raising=False) + monkeypatch.delenv("AUTOSKILLIT_CAMPAIGN_ID", raising=False) + monkeypatch.setattr(Path, "cwd", lambda: tmp_path) + result = provenance_path() + assert result == tmp_path / ".autoskillit" / "temp" / "session_provenance.jsonl" + + def test_state_dir_override(self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("AUTOSKILLIT_STATE_DIR", str(tmp_path)) + result = provenance_path() + assert result == tmp_path / "session_provenance.jsonl" + + def test_campaign_id_namespacing( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + monkeypatch.delenv("AUTOSKILLIT_STATE_DIR", raising=False) + monkeypatch.setenv("AUTOSKILLIT_CAMPAIGN_ID", "camp-42") + monkeypatch.setattr(Path, "cwd", lambda: tmp_path) + result = provenance_path() + assert ( + result == tmp_path / ".autoskillit" / "temp" / "camp-42" / "session_provenance.jsonl" + ) + + def test_project_dir_argument(self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.delenv("AUTOSKILLIT_STATE_DIR", raising=False) + monkeypatch.delenv("AUTOSKILLIT_CAMPAIGN_ID", raising=False) + result = provenance_path(project_dir=tmp_path) + assert result == tmp_path / ".autoskillit" / "temp" / "session_provenance.jsonl" + + +class TestWriteProvenanceRecord: + def test_creates_parent_dirs_and_appends( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + monkeypatch.setenv("AUTOSKILLIT_STATE_DIR", str(tmp_path)) + rec = _make_record() + write_provenance_record(rec) + path = tmp_path / "session_provenance.jsonl" + assert path.exists() + lines = path.read_text().strip().splitlines() + assert len(lines) == 1 + data = json.loads(lines[0]) + assert data["session_id"] == "sess-1" + assert data["caller_session_id"] == "caller-1" + + def test_appends_multiple_records( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + monkeypatch.setenv("AUTOSKILLIT_STATE_DIR", str(tmp_path)) + write_provenance_record(_make_record(session_id="a")) + write_provenance_record(_make_record(session_id="b")) + path = tmp_path / "session_provenance.jsonl" + lines = path.read_text().strip().splitlines() + assert len(lines) == 2 + assert json.loads(lines[0])["session_id"] == "a" + assert json.loads(lines[1])["session_id"] == "b" + + def test_oserror_is_swallowed(self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("AUTOSKILLIT_STATE_DIR", str(tmp_path / "no-write")) + read_only = tmp_path / "no-write" + read_only.mkdir() + read_only.chmod(0o444) + write_provenance_record(_make_record()) + read_only.chmod(0o755) + + +class TestReadProvenanceForSession: + def test_finds_matching_record(self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("AUTOSKILLIT_STATE_DIR", str(tmp_path)) + write_provenance_record(_make_record(session_id="target")) + write_provenance_record(_make_record(session_id="other")) + result = read_provenance_for_session("target") + assert result is not None + assert result["session_id"] == "target" + + def test_returns_none_for_missing_session( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + monkeypatch.setenv("AUTOSKILLIT_STATE_DIR", str(tmp_path)) + write_provenance_record(_make_record(session_id="other")) + assert read_provenance_for_session("nonexistent") is None + + def test_returns_none_when_file_missing( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + monkeypatch.setenv("AUTOSKILLIT_STATE_DIR", str(tmp_path)) + assert read_provenance_for_session("any") is None + + def test_skips_malformed_lines(self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("AUTOSKILLIT_STATE_DIR", str(tmp_path)) + path = tmp_path / "session_provenance.jsonl" + path.write_text("not-json\n" + json.dumps({"session_id": "found"}) + "\n") + result = read_provenance_for_session("found") + assert result is not None + assert result["session_id"] == "found" + + def test_skips_blank_lines(self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("AUTOSKILLIT_STATE_DIR", str(tmp_path)) + path = tmp_path / "session_provenance.jsonl" + path.write_text("\n\n" + json.dumps({"session_id": "found"}) + "\n\n") + result = read_provenance_for_session("found") + assert result is not None diff --git a/tests/core/test_session_registry.py b/tests/core/test_session_registry.py new file mode 100644 index 0000000000..6ce0d297ce --- /dev/null +++ b/tests/core/test_session_registry.py @@ -0,0 +1,70 @@ +"""Tests for core/session_registry.py.""" + +from __future__ import annotations + +import json +from pathlib import Path + +import pytest + +from autoskillit.core.runtime.session_registry import ( + bridge_claude_session_id, + read_registry, + registry_path, + write_registry_entry, +) + +pytestmark = [pytest.mark.layer("core"), pytest.mark.small] + + +def test_write_and_read_entry(tmp_path: Path) -> None: + write_registry_entry(tmp_path, "abc", "cook", None) + reg = read_registry(tmp_path) + assert "abc" in reg + assert reg["abc"]["session_type"] == "cook" + assert reg["abc"]["claude_session_id"] is None + + +def test_write_with_recipe_name(tmp_path: Path) -> None: + write_registry_entry(tmp_path, "xyz", "order", "my-recipe") + reg = read_registry(tmp_path) + assert reg["xyz"]["recipe_name"] == "my-recipe" + + +def test_write_is_atomic(tmp_path: Path) -> None: + write_registry_entry(tmp_path, "id1", "cook", None) + path = registry_path(tmp_path) + data = json.loads(path.read_text(encoding="utf-8")) + assert isinstance(data, dict) + assert "id1" in data + + +def test_read_returns_empty_on_missing_file(tmp_path: Path) -> None: + assert read_registry(tmp_path) == {} + + +def test_read_returns_empty_on_corrupt_json(tmp_path: Path) -> None: + path = registry_path(tmp_path) + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text("not valid json", encoding="utf-8") + assert read_registry(tmp_path) == {} + + +def test_bridge_claude_session_id(tmp_path: Path) -> None: + write_registry_entry(tmp_path, "abc", "cook", None) + bridge_claude_session_id(tmp_path, "abc", "claude-uuid-123") + reg = read_registry(tmp_path) + assert reg["abc"]["claude_session_id"] == "claude-uuid-123" + + +def test_bridge_noop_on_missing_launch_id(tmp_path: Path) -> None: + write_registry_entry(tmp_path, "abc", "cook", None) + bridge_claude_session_id(tmp_path, "unknown-id", "claude-uuid-123") + reg = read_registry(tmp_path) + assert reg["abc"]["claude_session_id"] is None + + +def test_registry_path_in_project_temp(tmp_path: Path) -> None: + path = registry_path(tmp_path) + assert ".autoskillit" in str(path) + assert "temp" in str(path) diff --git a/tests/core/test_session_type.py b/tests/core/test_session_type.py new file mode 100644 index 0000000000..6baa2329e1 --- /dev/null +++ b/tests/core/test_session_type.py @@ -0,0 +1,189 @@ +"""Tests for SessionType resolver and constants.""" + +from __future__ import annotations + +import warnings + +import pytest + +pytestmark = [pytest.mark.layer("core"), pytest.mark.small] + + +# --------------------------------------------------------------------------- +# Group A — SessionType resolver +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize( + "env_val, expected, suppress_deprecation, strict_no_deprecation", + [ + ("orchestrator", "ORCHESTRATOR", False, False), + ("skill", "SKILL", False, False), + ("ORCHESTRATOR", "ORCHESTRATOR", False, False), + (None, "SKILL", False, False), + ("bogus", "SKILL", True, False), + ("fleet", "FLEET", False, False), + ("FLEET", "FLEET", False, False), + ("fleet", "FLEET", False, True), + ("leaf", "SKILL", True, False), + ], + ids=[ + "orchestrator", + "skill", + "case-insensitive", + "unset", + "invalid-defaults-skill", + "fleet", + "fleet-case-insensitive", + "fleet-no-warning", + "leaf-deprecated-suppressed", + ], +) +def test_session_type_resolver( + monkeypatch, env_val, expected, suppress_deprecation, strict_no_deprecation +): + from autoskillit.core import SessionType, session_type + + if env_val is None: + monkeypatch.delenv("AUTOSKILLIT_SESSION_TYPE", raising=False) + else: + monkeypatch.setenv("AUTOSKILLIT_SESSION_TYPE", env_val) + monkeypatch.delenv("AUTOSKILLIT_HEADLESS", raising=False) + + if strict_no_deprecation: + with warnings.catch_warnings(): + warnings.simplefilter("error", DeprecationWarning) + result = session_type() + elif suppress_deprecation: + with warnings.catch_warnings(): + warnings.simplefilter("ignore", DeprecationWarning) + result = session_type() + else: + result = session_type() + + assert result is SessionType[expected] + + +def test_session_type_invalid_emits_deprecation_warning(monkeypatch): + from autoskillit.core import session_type + + monkeypatch.setenv("AUTOSKILLIT_SESSION_TYPE", "bogus") + with pytest.warns(DeprecationWarning, match="Invalid AUTOSKILLIT_SESSION_TYPE"): + session_type() + + +def test_transitional_bridge_headless_without_type_warns(monkeypatch): + from autoskillit.core import SessionType, session_type + + monkeypatch.delenv("AUTOSKILLIT_SESSION_TYPE", raising=False) + monkeypatch.setenv("AUTOSKILLIT_HEADLESS", "1") + with pytest.warns(DeprecationWarning, match="AUTOSKILLIT_HEADLESS=1 without"): + result = session_type() + assert result is SessionType.SKILL + + +def test_no_warning_when_both_unset(monkeypatch): + from autoskillit.core import SessionType, session_type + + monkeypatch.delenv("AUTOSKILLIT_SESSION_TYPE", raising=False) + monkeypatch.delenv("AUTOSKILLIT_HEADLESS", raising=False) + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + result = session_type() + deprecation_warnings = [w for w in caught if issubclass(w.category, DeprecationWarning)] + assert result is SessionType.SKILL + assert len(deprecation_warnings) == 0 + + +# --------------------------------------------------------------------------- +# Group C (partial) — Constants +# --------------------------------------------------------------------------- + + +def test_session_type_enum_values_match_constants(): + from autoskillit.core import ( + SESSION_TYPE_ORCHESTRATOR, + SESSION_TYPE_SKILL, + SessionType, + ) + + assert SessionType.ORCHESTRATOR.value == SESSION_TYPE_ORCHESTRATOR + assert SessionType.SKILL.value == SESSION_TYPE_SKILL + + +def test_session_type_env_var_constant(): + from autoskillit.core import SESSION_TYPE_ENV_VAR + + assert SESSION_TYPE_ENV_VAR == "AUTOSKILLIT_SESSION_TYPE" + + +# --------------------------------------------------------------------------- +# Deprecation compat — legacy "leaf" value +# --------------------------------------------------------------------------- + + +def test_leaf_value_maps_to_skill_with_deprecation_warning(monkeypatch): + from autoskillit.core import SessionType, session_type + + monkeypatch.setenv("AUTOSKILLIT_SESSION_TYPE", "leaf") + with pytest.warns(DeprecationWarning, match="leaf") as warning_list: + result = session_type() + assert result is SessionType.SKILL + assert "SKILL" in str(warning_list[0].message) + + +# --------------------------------------------------------------------------- +# Fleet alias tests — T1 shims +# --------------------------------------------------------------------------- + + +def test_session_type_enum_fleet_value(): + from autoskillit.core import SessionType + + assert SessionType.FLEET.value == "fleet" + + +def test_session_type_fleet_constant_matches_enum(): + from autoskillit.core import SessionType + from autoskillit.core.types._type_constants import SESSION_TYPE_FLEET + + assert SessionType.FLEET.value == SESSION_TYPE_FLEET + + +# --------------------------------------------------------------------------- +# Group D — Module placement +# --------------------------------------------------------------------------- + + +def test_session_type_is_defined_in_type_helpers(): + """session_type() must live in _type_helpers, not _type_enums.""" + import importlib + import inspect + + from autoskillit.core.types._type_helpers import session_type as fn_helpers + + type_enums = importlib.import_module("autoskillit.core.types._type_enums") + importlib.reload(type_enums) + assert not hasattr(type_enums, "session_type"), ( + "session_type must not be defined in _type_enums after relocation" + ) + + assert inspect.getmodule(fn_helpers).__name__ == "autoskillit.core.types._type_helpers" + assert inspect.getfile(fn_helpers).endswith("_type_helpers.py") + + +def test_session_type_docstring_references_orchestration_levels(): + """SessionType docstring must reference L1/L2/L3 level labels.""" + from autoskillit.core import SessionType + + doc = SessionType.__doc__ or "" + assert "L3" in doc + assert "L2" in doc + assert "L1" in doc + assert "L0" in doc + assert "FLEET" in doc + assert "ORCHESTRATOR" in doc + assert "SKILL" in doc + assert "mid-tier" not in doc + assert "bottom-tier" not in doc + assert "Tier discriminator" not in doc diff --git a/tests/core/test_skill_command_parsing.py b/tests/core/test_skill_command_parsing.py new file mode 100644 index 0000000000..6b2539f0b5 --- /dev/null +++ b/tests/core/test_skill_command_parsing.py @@ -0,0 +1,64 @@ +"""Unit tests for extract_path_arg in core._type_helpers.""" + +from __future__ import annotations + +import pytest + +from autoskillit.core.types._type_helpers import _PATH_PREFIXES, extract_path_arg + +pytestmark = [pytest.mark.layer("core"), pytest.mark.small] + + +class TestExtractPathArg: + def test_clean_input_returns_path(self): + result = extract_path_arg("/autoskillit:implement-worktree-no-merge /path/to/plan.md") + assert result == "/path/to/plan.md" + + def test_trailing_markdown_header_ignored(self): + cmd = "/autoskillit:implement-worktree-no-merge /path/plan.md\n\n## Base Branch\nimpl-926" + assert extract_path_arg(cmd) == "/path/plan.md" + + def test_trailing_extra_token_ignored(self): + assert ( + extract_path_arg("/autoskillit:implement-worktree-no-merge /path/plan.md impl-926") + == "/path/plan.md" + ) + + def test_autoskillit_temp_prefix(self): + cmd = "/autoskillit:implement-worktree-no-merge .autoskillit/temp/rectify/plan.md" + assert extract_path_arg(cmd) == ".autoskillit/temp/rectify/plan.md" + + def test_dotslash_prefix(self): + assert ( + extract_path_arg("/autoskillit:implement-worktree-no-merge ./path/plan.md") + == "./path/plan.md" + ) + + def test_returns_none_no_path_token(self): + assert extract_path_arg("/autoskillit:implement-worktree-no-merge impl-926") is None + + def test_strips_double_quotes(self): + assert ( + extract_path_arg('/autoskillit:implement-worktree-no-merge "/path/plan.md"') + == "/path/plan.md" + ) + + def test_strips_single_quotes(self): + assert ( + extract_path_arg("/autoskillit:implement-worktree-no-merge '/path/plan.md'") + == "/path/plan.md" + ) + + def test_returns_none_no_args(self): + assert extract_path_arg("/autoskillit:implement-worktree-no-merge") is None + + +class TestPathPrefixesConstant: + def test_is_tuple(self): + assert isinstance(_PATH_PREFIXES, tuple) + + def test_contains_slash(self): + assert "/" in _PATH_PREFIXES + + def test_contains_dotautoskillit(self): + assert ".autoskillit/" in _PATH_PREFIXES diff --git a/tests/core/test_tool_sequence_analysis.py b/tests/core/test_tool_sequence_analysis.py new file mode 100644 index 0000000000..47f194b5da --- /dev/null +++ b/tests/core/test_tool_sequence_analysis.py @@ -0,0 +1,464 @@ +from __future__ import annotations + +import json +import pathlib +from collections import Counter + +import pytest + +from autoskillit.core.tool_sequence_analysis import ( + DFG, + AssistantTurn, + TurnSequence, + build_dfg, + build_dfg_by_recipe, + compute_gap_stats, + iter_merged_assistant_turns, + parse_raw_cc_jsonl, + parse_sessions_from_summary_dir, + render_adjacency_table, + render_dot, + render_mermaid, +) + +pytestmark = [pytest.mark.layer("core"), pytest.mark.medium] + + +# --------------------------------------------------------------------------- +# TestParseRawCCJsonl +# --------------------------------------------------------------------------- + + +class TestParseRawCCJsonl: + def test_extracts_tool_names_from_assistant_content(self, tmp_path: pathlib.Path) -> None: + record = { + "type": "assistant", + "requestId": "r1", + "message": { + "content": [ + {"type": "text", "text": "hello"}, + {"type": "tool_use", "name": "ToolA"}, + {"type": "tool_use", "name": "ToolB"}, + ] + }, + } + log = tmp_path / "session.jsonl" + log.write_text(json.dumps(record) + "\n") + result = parse_raw_cc_jsonl(log) + assert result == [("ToolA", "ToolB")] + + def test_deduplicates_same_request_id(self, tmp_path: pathlib.Path) -> None: + record = { + "type": "assistant", + "requestId": "dup", + "message": {"content": [{"type": "tool_use", "name": "ToolA"}]}, + } + log = tmp_path / "session.jsonl" + log.write_text(json.dumps(record) + "\n" + json.dumps(record) + "\n") + result = parse_raw_cc_jsonl(log) + assert len(result) == 1 + + def test_cap_applied_at_n_8(self, tmp_path: pathlib.Path) -> None: + tools = [{"type": "tool_use", "name": f"Tool{i}"} for i in range(12)] + record = { + "type": "assistant", + "requestId": "r1", + "message": {"content": tools}, + } + log = tmp_path / "session.jsonl" + log.write_text(json.dumps(record) + "\n") + result = parse_raw_cc_jsonl(log) + assert len(result[0]) == 8 + + def test_skips_records_not_of_type_assistant(self, tmp_path: pathlib.Path) -> None: + records = [ + { + "type": "user", + "requestId": "r1", + "message": {"content": [{"type": "tool_use", "name": "X"}]}, + }, + { + "type": "tool_result", + "requestId": "r2", + "message": {"content": [{"type": "tool_use", "name": "Y"}]}, + }, + ] + log = tmp_path / "session.jsonl" + log.write_text("\n".join(json.dumps(r) for r in records) + "\n") + result = parse_raw_cc_jsonl(log) + assert result == [] + + def test_skips_tool_use_blocks_with_empty_name(self, tmp_path: pathlib.Path) -> None: + record = { + "type": "assistant", + "requestId": "r1", + "message": { + "content": [ + {"type": "tool_use", "name": ""}, + {"type": "tool_use"}, + {"type": "tool_use", "name": "ValidTool"}, + ] + }, + } + log = tmp_path / "session.jsonl" + log.write_text(json.dumps(record) + "\n") + result = parse_raw_cc_jsonl(log) + assert result == [("ValidTool",)] + + def test_empty_jsonl_returns_empty_list(self, tmp_path: pathlib.Path) -> None: + log = tmp_path / "session.jsonl" + log.write_text("") + result = parse_raw_cc_jsonl(log) + assert result == [] + + def test_malformed_json_line_skipped(self, tmp_path: pathlib.Path) -> None: + good = json.dumps( + { + "type": "assistant", + "requestId": "r1", + "message": {"content": [{"type": "tool_use", "name": "GoodTool"}]}, + } + ) + log = tmp_path / "session.jsonl" + log.write_text("NOT_JSON\n" + good + "\n") + result = parse_raw_cc_jsonl(log) + assert result == [("GoodTool",)] + + def test_message_field_absent_returns_empty_turn(self, tmp_path: pathlib.Path) -> None: + record = {"type": "assistant", "requestId": "r1"} + log = tmp_path / "session.jsonl" + log.write_text(json.dumps(record) + "\n") + result = parse_raw_cc_jsonl(log) + assert result == [()] + + +# --------------------------------------------------------------------------- +# TestBuildDFG +# --------------------------------------------------------------------------- + + +class TestBuildDFG: + def _make_seq(self, turns: list[list[str]], recipe: str = "") -> TurnSequence: + return TurnSequence(session_id="s1", recipe_name=recipe, turns=turns) + + def test_bigram_counts_across_turns(self) -> None: + sessions = [self._make_seq([["A", "B"], ["B", "C"]])] + dfg = build_dfg(sessions) + assert dfg.bigrams[("A", "B")] == 1 + assert dfg.bigrams[("B", "B")] == 1 + assert dfg.bigrams[("B", "C")] == 1 + + def test_ngrams_mined_within_single_turn(self) -> None: + sessions = [self._make_seq([["A", "B", "C"]])] + dfg = build_dfg(sessions) + assert ("A", "B") in dfg.ngrams + assert ("B", "C") in dfg.ngrams + assert ("A", "B", "C") in dfg.ngrams + + def test_ngrams_max_length_5(self) -> None: + sessions = [self._make_seq([["A", "B", "C", "D", "E", "F"]])] + dfg = build_dfg(sessions) + for key in dfg.ngrams: + assert len(key) <= 5 + + def test_gap_analysis_correct_turn_distance(self) -> None: + sessions = [self._make_seq([["A"], [], [], ["B"]])] + dfg = build_dfg(sessions) + assert ("A", "B") in dfg.pair_gaps + assert 3 in dfg.pair_gaps[("A", "B")] + + def test_gap_analysis_multiple_sessions_aggregated(self) -> None: + s1 = TurnSequence(session_id="s1", recipe_name="", turns=[["A"], [], ["B"]]) + s2 = TurnSequence(session_id="s2", recipe_name="", turns=[["A"], [], ["B"]]) + dfg = build_dfg([s1, s2]) + assert dfg.pair_gaps[("A", "B")] == [2, 2] + + def test_empty_sessions_list_returns_zero_dfg(self) -> None: + dfg = build_dfg([]) + assert len(dfg.bigrams) == 0 + assert len(dfg.ngrams) == 0 + assert len(dfg.pair_gaps) == 0 + assert dfg.total_turns == 0 + + +class TestBuildDFGByRecipe: + def test_stratification_separates_recipes(self) -> None: + s1 = TurnSequence(session_id="s1", recipe_name="lint", turns=[["A", "B"]]) + s2 = TurnSequence(session_id="s2", recipe_name="review", turns=[["C", "D"]]) + result = build_dfg_by_recipe([s1, s2]) + assert "lint" in result + assert "review" in result + assert ("A", "B") in result["lint"].bigrams + assert ("A", "B") not in result["review"].bigrams + + def test_global_aggregate_includes_all_sessions(self) -> None: + s1 = TurnSequence(session_id="s1", recipe_name="lint", turns=[["A", "B"]]) + s2 = TurnSequence(session_id="s2", recipe_name="review", turns=[["C", "D"]]) + from autoskillit.core.tool_sequence_analysis import compute_analysis + + result = compute_analysis([s1, s2]) + assert ("A", "B") in result.global_dfg.bigrams + assert ("C", "D") in result.global_dfg.bigrams + + +class TestComputeGapStats: + def test_median_and_percentiles_correct(self) -> None: + stats = compute_gap_stats([1, 2, 3, 4, 5]) + assert stats.median == 3.0 + assert stats.p25 == 2.0 + assert stats.p75 == 4.0 + assert stats.maximum == 5 + + def test_single_gap_all_stats_equal(self) -> None: + stats = compute_gap_stats([7]) + assert stats.median == 7.0 + assert stats.p25 == 7.0 + assert stats.p75 == 7.0 + assert stats.maximum == 7 + + +# --------------------------------------------------------------------------- +# TestRenderMermaid +# --------------------------------------------------------------------------- + + +class TestRenderMermaid: + def _make_dfg(self, bigrams: dict[tuple[str, str], int]) -> DFG: + return DFG(bigrams=Counter(bigrams), ngrams=Counter(), pair_gaps={}, total_turns=1) + + def test_output_starts_with_flowchart_lr(self) -> None: + dfg = self._make_dfg({("A", "B"): 1}) + output = render_mermaid(dfg) + first_line = output.splitlines()[0] + assert first_line.startswith("flowchart LR") + + def test_bigram_edges_present_in_output(self) -> None: + dfg = self._make_dfg({("A", "B"): 5}) + output = render_mermaid(dfg) + assert "A" in output + assert "B" in output + assert "5" in output + + def test_empty_dfg_renders_without_error(self) -> None: + dfg = DFG(bigrams=Counter(), ngrams=Counter(), pair_gaps={}, total_turns=0) + output = render_mermaid(dfg) + assert len(output) > 0 + + +# --------------------------------------------------------------------------- +# TestRenderAdjacencyTable +# --------------------------------------------------------------------------- + + +class TestRenderAdjacencyTable: + def _make_dfg(self, bigrams: dict[tuple[str, str], int]) -> DFG: + return DFG(bigrams=Counter(bigrams), ngrams=Counter(), pair_gaps={}, total_turns=1) + + def test_table_has_header_row(self) -> None: + dfg = self._make_dfg({("A", "B"): 1}) + output = render_adjacency_table(dfg) + lower = output.lower() + assert "tool_a" in lower + assert "tool_b" in lower + + def test_top_n_limits_rows(self) -> None: + bigrams = {(f"A{i}", f"B{i}"): i for i in range(10, 0, -1)} + dfg = self._make_dfg(bigrams) + output = render_adjacency_table(dfg, top_n=3) + # header + sep + 3 data rows = 5 lines + data_lines = [line for line in output.splitlines() if line and not line.startswith("-")] + # subtract 1 for header + assert len(data_lines) - 1 <= 3 + + +# --------------------------------------------------------------------------- +# TestRenderDot +# --------------------------------------------------------------------------- + + +class TestRenderDot: + def _make_dfg(self, bigrams: dict[tuple[str, str], int]) -> DFG: + return DFG(bigrams=Counter(bigrams), ngrams=Counter(), pair_gaps={}, total_turns=1) + + def test_dot_output_starts_with_digraph(self) -> None: + dfg = self._make_dfg({("A", "B"): 1}) + output = render_dot(dfg) + assert output.startswith("digraph") + + def test_dot_edge_correct_format(self) -> None: + dfg = self._make_dfg({("A", "B"): 3}) + output = render_dot(dfg) + assert '"A" -> "B"' in output + assert "3" in output + + +# --------------------------------------------------------------------------- +# TestParseSessionsFromSummaryDir +# --------------------------------------------------------------------------- + + +class TestParseSessionsFromSummaryDir: + def test_yields_turn_sequences_from_summary_json(self, tmp_path: pathlib.Path) -> None: + session_dir = tmp_path / "sessions" / "abc123" + session_dir.mkdir(parents=True) + summary = { + "turn_tool_calls": [["ToolA", "ToolB"], ["ToolC"]], + "recipe_name": "test-recipe", + "turn_timestamps": ["2026-01-01T00:00:00Z", "2026-01-01T00:00:01Z"], + } + (session_dir / "summary.json").write_text(json.dumps(summary)) + sessions = list(parse_sessions_from_summary_dir(tmp_path)) + assert len(sessions) == 1 + assert sessions[0].session_id == "abc123" + assert sessions[0].recipe_name == "test-recipe" + assert sessions[0].turns == [["ToolA", "ToolB"], ["ToolC"]] + + def test_skips_summary_without_turn_tool_calls(self, tmp_path: pathlib.Path) -> None: + session_dir = tmp_path / "sessions" / "old" + session_dir.mkdir(parents=True) + (session_dir / "summary.json").write_text(json.dumps({"recipe_name": "r"})) + sessions = list(parse_sessions_from_summary_dir(tmp_path)) + assert sessions == [] + + def test_skips_malformed_summary_json(self, tmp_path: pathlib.Path) -> None: + session_dir = tmp_path / "sessions" / "bad" + session_dir.mkdir(parents=True) + (session_dir / "summary.json").write_text("NOT_JSON") + sessions = list(parse_sessions_from_summary_dir(tmp_path)) + assert sessions == [] + + +class TestParseRawCCJsonlExtendedThinking: + def _write(self, tmp_path: pathlib.Path, *records: dict) -> pathlib.Path: + log = tmp_path / "session.jsonl" + log.write_text("\n".join(json.dumps(r) for r in records) + "\n") + return log + + def test_merges_tool_calls_across_thinking_and_tool_records( + self, tmp_path: pathlib.Path + ) -> None: + # thinking-only record first, tool-bearing record second — same requestId + r1 = { + "type": "assistant", + "requestId": "req-X", + "message": {"content": [{"type": "thinking", "thinking": "reasoning..."}]}, + } + r2 = { + "type": "assistant", + "requestId": "req-X", + "message": { + "content": [ + {"type": "thinking", "thinking": "more..."}, + {"type": "tool_use", "name": "Bash"}, + ] + }, + } + result = parse_raw_cc_jsonl(self._write(tmp_path, r1, r2)) + assert result == [("Bash",)] + + def test_accumulates_tools_from_multiple_records_same_request_id( + self, tmp_path: pathlib.Path + ) -> None: + r1 = { + "type": "assistant", + "requestId": "req-X", + "message": {"content": [{"type": "tool_use", "name": "Read"}]}, + } + r2 = { + "type": "assistant", + "requestId": "req-X", + "message": {"content": [{"type": "tool_use", "name": "Edit"}]}, + } + result = parse_raw_cc_jsonl(self._write(tmp_path, r1, r2)) + assert result == [("Read", "Edit")] + + def test_cap_applied_after_accumulation_across_records(self, tmp_path: pathlib.Path) -> None: + tools5 = [{"type": "tool_use", "name": f"T{i}"} for i in range(5)] + tools5b = [{"type": "tool_use", "name": f"U{i}"} for i in range(5)] + r1 = {"type": "assistant", "requestId": "req-X", "message": {"content": tools5}} + r2 = {"type": "assistant", "requestId": "req-X", "message": {"content": tools5b}} + result = parse_raw_cc_jsonl(self._write(tmp_path, r1, r2)) + assert len(result[0]) == 8 + assert result[0] == ("T0", "T1", "T2", "T3", "T4", "U0", "U1", "U2") + + +class TestIterMergedAssistantTurns: + def _make_record( + self, + *, + rid: str = "", + ts: str = "", + tools: list[str] | None = None, + thinking: bool = False, + ) -> str: + content: list[dict] = [] + if thinking: + content.append({"type": "thinking", "thinking": "..."}) + for name in tools or []: + content.append({"type": "tool_use", "name": name}) + rec: dict = {"type": "assistant", "message": {"content": content}} + if rid: + rec["requestId"] = rid + if ts: + rec["timestamp"] = ts + return json.dumps(rec) + + def _parse(self, *lines: str) -> list[AssistantTurn]: + text = "\n".join(lines) + "\n" + return list(iter_merged_assistant_turns(text)) + + def test_single_record_yields_one_turn(self) -> None: + line = self._make_record(rid="r1", ts="ts1", tools=["Bash"]) + turns = self._parse(line) + assert len(turns) == 1 + assert turns[0].request_id == "r1" + assert turns[0].timestamp == "ts1" + assert turns[0].tool_names == ("Bash",) + + def test_multiple_distinct_request_ids_yield_separate_turns(self) -> None: + l1 = self._make_record(rid="r1", tools=["A"]) + l2 = self._make_record(rid="r2", tools=["B"]) + turns = self._parse(l1, l2) + assert len(turns) == 2 + assert turns[0].tool_names == ("A",) + assert turns[1].tool_names == ("B",) + + def test_no_request_id_records_not_deduplicated(self) -> None: + l1 = self._make_record(tools=["A"]) + l2 = self._make_record(tools=["B"]) + turns = self._parse(l1, l2) + assert len(turns) == 2 + assert turns[0].tool_names == ("A",) + assert turns[1].tool_names == ("B",) + + def test_records_without_request_id_included(self) -> None: + line = self._make_record(tools=["Bash"]) + turns = self._parse(line) + assert len(turns) == 1 + assert turns[0].request_id == "" + assert turns[0].tool_names == ("Bash",) + + def test_preserves_insertion_order(self) -> None: + l1 = self._make_record(rid="r1", tools=["A"]) + l2 = self._make_record(rid="r2", tools=["B"]) + l3 = self._make_record(rid="r3", tools=["C"]) + turns = self._parse(l1, l2, l3) + assert [t.request_id for t in turns] == ["r1", "r2", "r3"] + + def test_no_rid_turns_interleaved_in_file_order(self) -> None: + l_a = self._make_record(rid="r-A", tools=["A"]) + l_no = self._make_record(tools=["NoRid"]) + l_b = self._make_record(rid="r-B", tools=["B"]) + turns = self._parse(l_a, l_no, l_b) + assert len(turns) == 3 + assert turns[0].request_id == "r-A" + assert turns[1].request_id == "" + assert turns[2].request_id == "r-B" + + def test_timestamp_from_first_record_preferred(self) -> None: + l1 = self._make_record(rid="r1", ts="first-ts", tools=[]) + l2 = self._make_record(rid="r1", ts="second-ts", tools=["Bash"]) + turns = self._parse(l1, l2) + assert len(turns) == 1 + assert turns[0].timestamp == "first-ts" + assert turns[0].tool_names == ("Bash",) diff --git a/tests/core/test_type_constants.py b/tests/core/test_type_constants.py index 5c46dd9f3c..9ac12eb9bb 100644 --- a/tests/core/test_type_constants.py +++ b/tests/core/test_type_constants.py @@ -2,8 +2,20 @@ from __future__ import annotations +import pytest + +pytestmark = [pytest.mark.layer("core"), pytest.mark.small] + # REQ-PACK-001: PACK_REGISTRY defines all packs with default_enabled +def test_core_packs_constant_defined() -> None: + """CORE_PACKS must be a frozenset defined in _type_constants and exported via core.""" + from autoskillit.core.types._type_constants import CORE_PACKS + + assert isinstance(CORE_PACKS, frozenset) + assert CORE_PACKS == frozenset({"github", "ci", "clone", "telemetry"}) + + def test_pack_registry_contains_all_packs() -> None: from autoskillit.core import PACK_REGISTRY @@ -47,3 +59,248 @@ def test_pack_registry_importable_from_core() -> None: assert isinstance(PACK_REGISTRY, dict) assert all(isinstance(v, PackDef) for v in PACK_REGISTRY.values()) + + +def test_kitchen_core_in_pack_registry() -> None: + """kitchen-core is a registered pack with default_enabled=True.""" + from autoskillit.core.types import PACK_REGISTRY + + assert "kitchen-core" in PACK_REGISTRY + assert PACK_REGISTRY["kitchen-core"].default_enabled is True + + +def test_private_env_vars_includes_franchise_tier_vars() -> None: + from autoskillit.core import AUTOSKILLIT_PRIVATE_ENV_VARS + + expected = { + "AUTOSKILLIT_SESSION_TYPE", + "AUTOSKILLIT_CAMPAIGN_ID", + "AUTOSKILLIT_KITCHEN_SESSION_ID", + "AUTOSKILLIT_CAMPAIGN_STATE_PATH", + "AUTOSKILLIT_PROJECT_DIR", + "AUTOSKILLIT_FOOD_TRUCK_TOOL_TAGS", + } + assert expected <= AUTOSKILLIT_PRIVATE_ENV_VARS + assert "AUTOSKILLIT_L3_TOOL_TAGS" not in AUTOSKILLIT_PRIVATE_ENV_VARS + + +def test_campaign_id_env_var_and_kitchen_session_id_env_var_exported_from_core() -> None: + """CAMPAIGN_ID_ENV_VAR and KITCHEN_SESSION_ID_ENV_VAR are re-exported from autoskillit.core.""" + from autoskillit.core import CAMPAIGN_ID_ENV_VAR, KITCHEN_SESSION_ID_ENV_VAR + + assert CAMPAIGN_ID_ENV_VAR == "AUTOSKILLIT_CAMPAIGN_ID" + assert KITCHEN_SESSION_ID_ENV_VAR == "AUTOSKILLIT_KITCHEN_SESSION_ID" + + +# --------------------------------------------------------------------------- +# RECIPE_PACK_REGISTRY and RecipePackDef (franchise schema extension) +# --------------------------------------------------------------------------- + + +def test_recipe_pack_def_namedtuple_fields() -> None: + from autoskillit.core import RecipePackDef + + rpd = RecipePackDef(default_enabled=True, description="test pack") + assert rpd.default_enabled is True + assert rpd.description == "test pack" + + +def test_recipe_pack_registry_has_three_entries() -> None: + from autoskillit.core import RECIPE_PACK_REGISTRY, RecipePackDef + + assert len(RECIPE_PACK_REGISTRY) == 3 + assert all(isinstance(v, RecipePackDef) for v in RECIPE_PACK_REGISTRY.values()) + + +def test_recipe_pack_registry_implementation_family() -> None: + from autoskillit.core import RECIPE_PACK_REGISTRY + + assert "implementation-family" in RECIPE_PACK_REGISTRY + assert RECIPE_PACK_REGISTRY["implementation-family"].default_enabled is True + assert ( + RECIPE_PACK_REGISTRY["implementation-family"].description + == "Implementation and refactoring recipes" + ) + + +def test_recipe_pack_registry_research_family() -> None: + from autoskillit.core import RECIPE_PACK_REGISTRY + + assert "research-family" in RECIPE_PACK_REGISTRY + assert RECIPE_PACK_REGISTRY["research-family"].default_enabled is False + assert ( + RECIPE_PACK_REGISTRY["research-family"].description == "Research and exploration recipes" + ) + + +def test_recipe_pack_registry_orchestration_family() -> None: + from autoskillit.core import RECIPE_PACK_REGISTRY + + assert "orchestration-family" in RECIPE_PACK_REGISTRY + assert RECIPE_PACK_REGISTRY["orchestration-family"].default_enabled is True + assert ( + RECIPE_PACK_REGISTRY["orchestration-family"].description + == "Campaign orchestration and automation" + ) + + +def test_recipe_pack_def_importable_from_core() -> None: + from autoskillit.core import RECIPE_PACK_REGISTRY, RecipePackDef + + assert isinstance(RECIPE_PACK_REGISTRY, dict) + assert all(isinstance(v, RecipePackDef) for v in RECIPE_PACK_REGISTRY.values()) + + +def test_feature_reveal_tags_removed() -> None: + """FEATURE_REVEAL_TAGS was intentionally removed in #1141.""" + with pytest.raises(ImportError): + from autoskillit.core import FEATURE_REVEAL_TAGS # noqa: F401 + + +def test_exclusive_feature_tools_removed() -> None: + """EXCLUSIVE_FEATURE_TOOLS was removed (issue #1150) — must not be importable.""" + import autoskillit.core as core + + assert not hasattr(core, "EXCLUSIVE_FEATURE_TOOLS") + + +def test_exclusive_feature_tools_not_in_all() -> None: + """EXCLUSIVE_FEATURE_TOOLS must not appear in _type_constants.__all__.""" + from autoskillit.core.types import _type_constants + + assert "EXCLUSIVE_FEATURE_TOOLS" not in _type_constants.__all__ + + +def test_fleet_default_enabled_is_false() -> None: + """Fleet is gated off by default — enabled only via project config.""" + from autoskillit.core import FEATURE_REGISTRY + + assert FEATURE_REGISTRY["fleet"].default_enabled is False + + +def test_is_feature_enabled_fleet_defaults_false() -> None: + """Without explicit config, fleet resolves to disabled when experimental_enabled=False.""" + from autoskillit.core.feature_flags import is_feature_enabled + + assert is_feature_enabled("fleet", {}, experimental_enabled=False) is False + # fleet is EXPERIMENTAL, so blanket enables it + assert is_feature_enabled("fleet", {}, experimental_enabled=True) is True + + +def test_fleet_dispatch_tools_constant_exists() -> None: + """FLEET_DISPATCH_TOOLS is a frozenset of exactly the 4 fleet-dispatch discovery tools.""" + from autoskillit.core import FLEET_DISPATCH_TOOLS + + assert FLEET_DISPATCH_TOOLS == frozenset( + { + "list_recipes", + "load_recipe", + "fetch_github_issue", + "get_issue_title", + } + ) + + +def test_fleet_menu_tools_in_type_constants() -> None: + """FLEET_MENU_TOOLS must live in core._type_constants, not fleet.__init__.""" + from autoskillit.core.types._type_constants import FLEET_MENU_TOOLS + + assert isinstance(FLEET_MENU_TOOLS, tuple) + assert "dispatch_food_truck" in FLEET_MENU_TOOLS + + +def test_fleet_menu_tools_not_in_fleet_init() -> None: + """FLEET_MENU_TOOLS must no longer be exported from fleet.__init__.""" + import subprocess + import sys + + result = subprocess.run( + [ + sys.executable, + "-c", + "import autoskillit.fleet as f; print(hasattr(f, 'FLEET_MENU_TOOLS'))", + ], + capture_output=True, + text=True, + ) + assert result.returncode == 0, f"Subprocess failed:\n{result.stderr}" + assert result.stdout.strip() == "False", ( + "FLEET_MENU_TOOLS still lives in fleet.__init__; move it to core._type_constants" + ) + + +# --------------------------------------------------------------------------- +# T1: FeatureDef has no redundant name field (Finding 3) +# --------------------------------------------------------------------------- + + +def test_feature_def_has_no_name_field() -> None: + """FeatureDef.name is redundant with the FEATURE_REGISTRY dict key and must not exist.""" + import dataclasses + + from autoskillit.core.types._type_constants import FeatureDef + + field_names = {f.name for f in dataclasses.fields(FeatureDef)} + assert "name" not in field_names, "FeatureDef.name is redundant with FEATURE_REGISTRY dict key" + + +def test_fleet_tools_matches_expected() -> None: + """FLEET_TOOLS must match a hardcoded expected set — not derived from tags.""" + from autoskillit.core import FLEET_TOOLS + + expected = frozenset( + { + "batch_cleanup_clones", + "get_pipeline_report", + "get_token_summary", + "get_timing_summary", + "get_quota_events", + "dispatch_food_truck", + "record_gate_dispatch", + } + ) + assert FLEET_TOOLS == expected, "Update expected set when FLEET_TOOLS changes" + + +def test_skill_tools_matches_expected() -> None: + """SKILL_TOOLS must match a hardcoded expected set.""" + from autoskillit.core import SKILL_TOOLS + + assert SKILL_TOOLS == frozenset({"run_skill"}) + + +def test_provider_profile_in_private_env_vars() -> None: + """AUTOSKILLIT_PROVIDER_PROFILE must be in AUTOSKILLIT_PRIVATE_ENV_VARS.""" + from autoskillit.core import AUTOSKILLIT_PRIVATE_ENV_VARS + + assert "AUTOSKILLIT_PROVIDER_PROFILE" in AUTOSKILLIT_PRIVATE_ENV_VARS + + +def test_headless_tools_contains_expected_names(): + from autoskillit.core.types import HEADLESS_TOOLS + + assert HEADLESS_TOOLS == {"test_check"} + + +def test_free_range_tools_contains_expected_names(): + from autoskillit.core.types import FREE_RANGE_TOOLS + + assert FREE_RANGE_TOOLS == { + "open_kitchen", + "close_kitchen", + "disable_quota_guard", + "reload_session", + } + + +# --------------------------------------------------------------------------- +# T4: SESSION_TYPE_COOK/ORDER moved to cli/session/_constants.py +# --------------------------------------------------------------------------- + + +def test_session_type_cook_order_not_in_core_types() -> None: + """SESSION_TYPE_COOK/ORDER are CLI labels, not core type constants.""" + from autoskillit.core.types import _type_constants + + assert not hasattr(_type_constants, "SESSION_TYPE_COOK") + assert not hasattr(_type_constants, "SESSION_TYPE_ORDER") diff --git a/tests/core/test_type_protocol_shards.py b/tests/core/test_type_protocol_shards.py new file mode 100644 index 0000000000..3f1c6e5556 --- /dev/null +++ b/tests/core/test_type_protocol_shards.py @@ -0,0 +1,105 @@ +import pytest + +pytestmark = [pytest.mark.layer("core"), pytest.mark.small] + + +def test_logging_shard_all(): + from autoskillit.core.types._type_protocols_logging import __all__ + + assert set(__all__) == { + "AuditLog", + "TokenLog", + "TimingLog", + "McpResponseLog", + "GitHubApiLog", + "SupportsDebug", + "SupportsLogger", + } + + +def test_execution_shard_all(): + from autoskillit.core.types._type_protocols_execution import __all__ + + assert set(__all__) == { + "TestRunner", + "HeadlessExecutor", + "OutputPatternResolver", + "WriteExpectedResolver", + } + + +def test_github_shard_all(): + from autoskillit.core.types._type_protocols_github import __all__ + + assert set(__all__) == {"GitHubFetcher", "CIWatcher", "MergeQueueWatcher"} + + +def test_workspace_shard_all(): + from autoskillit.core.types._type_protocols_workspace import __all__ + + assert set(__all__) == { + "WorkspaceManager", + "CloneManager", + "SessionSkillManager", + "SkillLister", + "SkillResolver", + } + + +def test_recipe_shard_all(): + from autoskillit.core.types._type_protocols_recipe import __all__ + + assert set(__all__) == { + "RecipeRepository", + "MigrationService", + "DatabaseReader", + "ReadOnlyResolver", + } + + +def test_infra_shard_all(): + from autoskillit.core.types._type_protocols_infra import __all__ + + assert set(__all__) == { + "GateState", + "BackgroundSupervisor", + "FleetLock", + "QuotaRefreshTask", + "TokenFactory", + "CampaignProtector", + } + + +def test_all_protocols_reachable_via_types(): + from autoskillit.core import types + + for name in [ + "GateState", + "AuditLog", + "HeadlessExecutor", + "GitHubFetcher", + "RecipeRepository", + "WorkspaceManager", + "CampaignProtector", + ]: + assert hasattr(types, name), f"Missing from types: {name}" + + +def test_pyi_stub_exports_skill_constants(): + import autoskillit.core as core + + assert hasattr(core, "SKILL_FILE_ADVISORY_MAP"), ( + "SKILL_FILE_ADVISORY_MAP must be exported from autoskillit.core" + ) + assert hasattr(core, "SKILL_ACTIVATE_DEPS_REQUIRED"), ( + "SKILL_ACTIVATE_DEPS_REQUIRED must be exported from autoskillit.core" + ) + + +def test_runtime_checkable_flags(): + from autoskillit.core.types._type_protocols_logging import SupportsDebug, SupportsLogger + + for proto in (SupportsDebug, SupportsLogger): + assert not getattr(proto, "_is_runtime_protocol", False), ( + f"{proto.__name__} must not be @runtime_checkable" + ) diff --git a/tests/core/test_type_results_execution.py b/tests/core/test_type_results_execution.py new file mode 100644 index 0000000000..8abf398de5 --- /dev/null +++ b/tests/core/test_type_results_execution.py @@ -0,0 +1,67 @@ +"""Tests for core/types/_type_results_execution.py — execution-scoped type module.""" + +import pytest + +pytestmark = [pytest.mark.layer("core"), pytest.mark.small] + + +class TestExecutionTypesImport: + """Each execution-scoped type is importable from the new module.""" + + def test_session_telemetry_importable(self): + from autoskillit.core.types._type_results_execution import SessionTelemetry + + assert hasattr(SessionTelemetry, "empty") + + def test_recipe_identity_importable(self): + from autoskillit.core.types._type_results_execution import RecipeIdentity + + assert hasattr(RecipeIdentity, "empty") + + def test_ci_run_scope_importable(self): + from autoskillit.core.types._type_results_execution import CIRunScope + + assert CIRunScope().workflow is None + + def test_backward_compat_via_core_gateway(self): + """All moved types are still importable from autoskillit.core.""" + import inspect + + from autoskillit.core import ( + CIRunScope, + RecipeIdentity, + SessionTelemetry, + ) + + assert all(inspect.isclass(cls) for cls in [SessionTelemetry, RecipeIdentity, CIRunScope]) + + +class TestExecutionTypesNotInResults: + """Moved types must no longer appear in _type_results.__all__.""" + + def test_moved_types_absent_from_results_all(self): + from autoskillit.core.types._type_results import __all__ as results_all + + moved = {"SessionTelemetry", "RecipeIdentity", "CIRunScope"} + overlap = moved & set(results_all) + assert not overlap, f"Types still in _type_results.__all__: {overlap}" + + def test_provider_outcome_in_results_all(self): + from autoskillit.core.types._type_results import __all__ as results_all + + assert "ProviderOutcome" in results_all + + def test_present_in_execution_all(self): + from autoskillit.core.types._type_results_execution import ( + __all__ as exec_all, + ) + + expected = {"SessionTelemetry", "RecipeIdentity", "CIRunScope"} + assert expected == set(exec_all) + + def test_skill_result_still_uses_provider_outcome(self): + """SkillResult.provider field default_factory references ProviderOutcome.""" + from autoskillit.core import ProviderOutcome, SkillResult + + sr = SkillResult.crashed(Exception("test")) + assert isinstance(sr.provider, ProviderOutcome) diff --git a/tests/core/test_types.py b/tests/core/test_types.py index 49f9d7c276..cb32c69166 100644 --- a/tests/core/test_types.py +++ b/tests/core/test_types.py @@ -2,20 +2,45 @@ import dataclasses import json +from dataclasses import FrozenInstanceError +from typing import Any, ClassVar import pytest from autoskillit.core.types import ( ChannelConfirmation, CIRunScope, + InfraOutcome, MergeFailedStep, MergeState, + ProviderOutcome, RestartScope, RetryReason, SessionOutcome, SkillResult, ) +pytestmark = [pytest.mark.layer("core"), pytest.mark.small] + + +@pytest.mark.parametrize( + ("raw", "expected_value"), + [ + ("text", "text"), + ("tool_use", "tool_use"), + ("thinking", "thinking"), + ("redacted_thinking", "redacted_thinking"), + ("future_new_type", "unknown"), + ("image", "image"), + ("tool_result", "tool_result"), + ], +) +def test_claude_content_block_type_from_api(raw: str, expected_value: str) -> None: + from autoskillit.core.types import ClaudeContentBlockType + + block_type = ClaudeContentBlockType.from_api(raw) + assert block_type.value == expected_value + def test_retry_reason_values(): """RetryReason enum has exactly the expected members.""" @@ -26,11 +51,14 @@ def test_retry_reason_values(): RetryReason.EARLY_STOP, RetryReason.ZERO_WRITES, RetryReason.EMPTY_OUTPUT, + RetryReason.COMPLETED_NO_FLUSH, RetryReason.DRAIN_RACE, RetryReason.PATH_CONTAMINATION, RetryReason.CONTRACT_RECOVERY, RetryReason.STALE, RetryReason.CLONE_CONTAMINATION, + RetryReason.THINKING_STALL, + RetryReason.IDLE_STALL, } assert RetryReason.NONE.value == "none" @@ -42,6 +70,7 @@ def test_merge_failed_step_values(): MergeFailedStep.PROTECTED_BRANCH, MergeFailedStep.BRANCH_DETECTION, MergeFailedStep.DIRTY_TREE, + MergeFailedStep.DIRTY_MAIN_REPO, MergeFailedStep.TEST_GATE, MergeFailedStep.FETCH, MergeFailedStep.PRE_REBASE_CHECK, @@ -126,55 +155,58 @@ def test_session_outcome_is_str_enum_with_expected_values(): assert SessionOutcome.FAILED == "failed" -def test_skill_result_outcome_succeeded(): - """SkillResult with success=True, needs_retry=False → outcome is SUCCEEDED.""" - sr = SkillResult( - success=True, - result="ok", - session_id="s1", - subtype="success", - is_error=False, - exit_code=0, - needs_retry=False, - retry_reason=RetryReason.NONE, - stderr="", - ) - assert sr.outcome is SessionOutcome.SUCCEEDED - assert sr.outcome == "succeeded" - - -def test_skill_result_outcome_retriable(): - """SkillResult with success=False, needs_retry=True → outcome is RETRIABLE.""" - sr = SkillResult( - success=False, - result="partial", - session_id="s1", - subtype="error_max_turns", - is_error=False, - exit_code=1, - needs_retry=True, - retry_reason=RetryReason.RESUME, - stderr="", - ) - assert sr.outcome is SessionOutcome.RETRIABLE - assert sr.outcome == "retriable" - - -def test_skill_result_outcome_failed(): - """SkillResult with success=False, needs_retry=False → outcome is FAILED.""" - sr = SkillResult( - success=False, - result="", - session_id="s1", - subtype="timeout", - is_error=True, - exit_code=-1, - needs_retry=False, - retry_reason=RetryReason.NONE, - stderr="", - ) - assert sr.outcome is SessionOutcome.FAILED - assert sr.outcome == "failed" +@pytest.mark.parametrize( + "kwargs, expected_outcome", + [ + ( + dict( + success=True, + result="ok", + session_id="s1", + subtype="success", + is_error=False, + exit_code=0, + needs_retry=False, + retry_reason=RetryReason.NONE, + stderr="", + ), + SessionOutcome.SUCCEEDED, + ), + ( + dict( + success=False, + result="partial", + session_id="s1", + subtype="error_max_turns", + is_error=False, + exit_code=1, + needs_retry=True, + retry_reason=RetryReason.RESUME, + stderr="", + ), + SessionOutcome.RETRIABLE, + ), + ( + dict( + success=False, + result="", + session_id="s1", + subtype="timeout", + is_error=True, + exit_code=-1, + needs_retry=False, + retry_reason=RetryReason.NONE, + stderr="", + ), + SessionOutcome.FAILED, + ), + ], + ids=["succeeded", "retriable", "failed"], +) +def test_skill_result_outcome(kwargs, expected_outcome): + sr = SkillResult(**kwargs) + assert sr.outcome is expected_outcome + assert sr.outcome == expected_outcome.value def test_skill_result_to_json_excludes_outcome(): @@ -214,7 +246,8 @@ def test_severity_has_ok_member(): assert Severity.OK == "ok" assert Severity.ERROR == "error" assert Severity.WARNING == "warning" - assert set(Severity) == {Severity.OK, Severity.ERROR, Severity.WARNING} + assert Severity.INFO == "info" + assert set(Severity) == {Severity.OK, Severity.ERROR, Severity.WARNING, Severity.INFO} def test_github_fetcher_protocol_has_label_methods(): @@ -324,24 +357,6 @@ def test_subprocess_runner_protocol_pty_mode_default_false(): assert sig.parameters["pty_mode"].default is False -def test_default_subprocess_runner_pty_mode_default_false(): - import inspect - - from autoskillit.execution.process import DefaultSubprocessRunner - - sig = inspect.signature(DefaultSubprocessRunner.__call__) - assert sig.parameters["pty_mode"].default is False - - -def test_run_managed_async_pty_mode_default_false(): - import inspect - - from autoskillit.execution.process import run_managed_async - - sig = inspect.signature(run_managed_async) - assert sig.parameters["pty_mode"].default is False - - # --------------------------------------------------------------------------- # CIRunScope event field # --------------------------------------------------------------------------- @@ -371,10 +386,13 @@ def test_pr_state_enum_members_are_locked(): PRState.EJECTED_CI_FAILURE, PRState.STALLED, PRState.DROPPED_HEALTHY, + PRState.DROPPED_MERGE_GROUP_CI, + PRState.NOT_ENROLLED, PRState.TIMEOUT, PRState.ERROR, } assert PRState.DROPPED_HEALTHY.value == "dropped_healthy" + assert PRState.DROPPED_MERGE_GROUP_CI.value == "dropped_merge_group_ci" class TestSkillResultCrashedFactory: @@ -403,3 +421,174 @@ def test_crashed_to_json_produces_valid_envelope(self): assert "session_id" in data assert "subtype" in data assert data["subtype"] == "crashed" + + def test_crashed_sets_provider_used_empty_string(self): + result = SkillResult.crashed(exception=RuntimeError("boom")) + assert result.provider.provider_used == "" + + def test_crashed_sets_provider_fallback_false(self): + result = SkillResult.crashed(exception=RuntimeError("boom")) + assert result.provider.fallback_activated is False + + +class TestSkillResultProviderFields: + _BASE_KWARGS: ClassVar[dict[str, Any]] = { + "success": True, + "result": "ok", + "session_id": "s1", + "subtype": "success", + "is_error": False, + "exit_code": 0, + "needs_retry": False, + "retry_reason": RetryReason.NONE, + "stderr": "", + } + + def test_provider_used_defaults_to_empty_string(self): + sr = SkillResult(**self._BASE_KWARGS) + assert sr.provider.provider_used == "" + + def test_provider_fallback_defaults_to_false(self): + sr = SkillResult(**self._BASE_KWARGS) + assert sr.provider.fallback_activated is False + + def test_to_json_includes_provider_used_when_non_empty(self): + sr = SkillResult( + **self._BASE_KWARGS, + provider=ProviderOutcome(provider_used="anthropic-vertex", fallback_activated=True), + ) + data = json.loads(sr.to_json()) + assert data["provider_used"] == "anthropic-vertex" + assert data["provider_fallback"] is True + + def test_to_json_includes_provider_used_as_empty_string_when_unset(self): + sr = SkillResult(**self._BASE_KWARGS) + data = json.loads(sr.to_json()) + assert "provider_used" in data + assert data["provider_used"] == "" + + def test_to_json_includes_provider_fallback_when_true(self): + sr = SkillResult( + **self._BASE_KWARGS, + provider=ProviderOutcome(provider_used="", fallback_activated=True), + ) + data = json.loads(sr.to_json()) + assert "provider_fallback" in data + assert data["provider_fallback"] is True + + def test_provider_used_round_trips_via_json(self): + sr = SkillResult( + **self._BASE_KWARGS, + provider=ProviderOutcome(provider_used="bedrock-us", fallback_activated=False), + ) + data = json.loads(sr.to_json()) + assert data["provider_used"] == "bedrock-us" + + +class TestInfraOutcome: + def test_default_exit_category_is_empty(self): + outcome = InfraOutcome() + assert outcome.exit_category == "" + + def test_custom_exit_category(self): + outcome = InfraOutcome(exit_category="context_exhausted") + assert outcome.exit_category == "context_exhausted" + + def test_frozen_rejects_mutation(self): + outcome = InfraOutcome(exit_category="completed") + with pytest.raises(FrozenInstanceError): + outcome.exit_category = "api_error" + + +class TestSkillResultExtensionBundles: + _BASE_KWARGS: ClassVar[dict[str, Any]] = { + "success": True, + "result": "ok", + "session_id": "", + "subtype": "success", + "is_error": False, + "exit_code": 0, + "needs_retry": False, + "retry_reason": RetryReason.NONE, + "stderr": "", + } + + def test_provider_bundle_defaults_to_none_used(self): + sr = SkillResult(**self._BASE_KWARGS) + assert sr.provider == ProviderOutcome.none_used() + assert sr.provider.provider_used == "" + assert sr.provider.fallback_activated is False + + def test_infra_bundle_defaults_to_empty(self): + sr = SkillResult(**self._BASE_KWARGS) + assert sr.infra == InfraOutcome() + assert sr.infra.exit_category == "" + + def test_provider_bundle_accepts_custom_value(self): + sr = SkillResult( + **self._BASE_KWARGS, + provider=ProviderOutcome(provider_used="anthropic", fallback_activated=True), + ) + assert sr.provider.provider_used == "anthropic" + assert sr.provider.fallback_activated is True + + def test_infra_bundle_accepts_custom_value(self): + sr = SkillResult( + **self._BASE_KWARGS, + infra=InfraOutcome(exit_category="api_error"), + ) + assert sr.infra.exit_category == "api_error" + + def test_flat_provider_used_removed(self): + assert "provider_used" not in [f.name for f in dataclasses.fields(SkillResult)] + + def test_flat_provider_fallback_removed(self): + assert "provider_fallback" not in [f.name for f in dataclasses.fields(SkillResult)] + + def test_flat_infra_exit_category_removed(self): + assert "infra_exit_category" not in [f.name for f in dataclasses.fields(SkillResult)] + + def test_to_json_emits_flat_provider_used_key(self): + sr = SkillResult( + **self._BASE_KWARGS, + provider=ProviderOutcome(provider_used="vertex", fallback_activated=False), + ) + data = json.loads(sr.to_json()) + assert data["provider_used"] == "vertex" + + def test_to_json_emits_flat_provider_fallback_key(self): + sr = SkillResult( + **self._BASE_KWARGS, + provider=ProviderOutcome(provider_used="anthropic", fallback_activated=True), + ) + data = json.loads(sr.to_json()) + assert data["provider_fallback"] is True + + def test_to_json_emits_flat_infra_exit_category_key(self): + sr = SkillResult( + **self._BASE_KWARGS, + infra=InfraOutcome(exit_category="context_exhausted"), + ) + data = json.loads(sr.to_json()) + assert data["infra_exit_category"] == "context_exhausted" + + def test_to_json_provider_empty_defaults(self): + sr = SkillResult(**self._BASE_KWARGS) + data = json.loads(sr.to_json()) + assert data["provider_used"] == "" + assert data["provider_fallback"] is False + assert data["infra_exit_category"] == "" + + def test_replace_infra_bundle(self): + sr = SkillResult(**self._BASE_KWARGS) + sr2 = dataclasses.replace(sr, infra=InfraOutcome(exit_category="api_error")) + assert sr2.infra.exit_category == "api_error" + assert sr.infra.exit_category == "" + + def test_replace_provider_bundle(self): + sr = SkillResult(**self._BASE_KWARGS) + sr2 = dataclasses.replace( + sr, provider=ProviderOutcome(provider_used="vertex", fallback_activated=True) + ) + assert sr2.provider.provider_used == "vertex" + assert sr2.provider.fallback_activated is True diff --git a/tests/core/test_types_structure.py b/tests/core/test_types_structure.py index a7b967a579..24356f9d91 100644 --- a/tests/core/test_types_structure.py +++ b/tests/core/test_types_structure.py @@ -2,9 +2,13 @@ from __future__ import annotations +import pytest + +pytestmark = [pytest.mark.layer("core"), pytest.mark.small] + def test_enums_importable_from_sub_module(): - from autoskillit.core._type_enums import ( + from autoskillit.core.types._type_enums import ( RetryReason, ) @@ -12,12 +16,13 @@ def test_enums_importable_from_sub_module(): def test_protocols_importable_from_sub_module(): - from autoskillit.core._type_protocols import GateState, HeadlessExecutor + from autoskillit.core.types._type_protocols_execution import HeadlessExecutor + from autoskillit.core.types._type_protocols_infra import GateState assert callable(GateState) - assert GateState.__module__ == "autoskillit.core._type_protocols" + assert GateState.__module__ == "autoskillit.core.types._type_protocols_infra" assert callable(HeadlessExecutor) - assert HeadlessExecutor.__module__ == "autoskillit.core._type_protocols" + assert HeadlessExecutor.__module__ == "autoskillit.core.types._type_protocols_execution" def test_types_hub_backward_compat(): @@ -47,8 +52,8 @@ def test_types_hub_backward_compat(): assert dataclasses.is_dataclass(LoadResult) # _type_results assert dataclasses.is_dataclass(SkillResult) # _type_results assert dataclasses.is_dataclass(FailureRecord) # _type_results - assert callable(GateState) # _type_protocols — Protocol - assert callable(HeadlessExecutor) # _type_protocols — Protocol + assert callable(GateState) # _type_protocols_infra — Protocol + assert callable(HeadlessExecutor) # _type_protocols_execution — Protocol assert callable(extract_skill_name) # _type_helpers — function @@ -56,6 +61,39 @@ def test_types_hub_line_count_under_threshold(): """After split, core/types.py must be under 200 lines (re-export hub only).""" from autoskillit.core import paths - types_path = paths.pkg_root() / "core" / "types.py" + types_path = paths.pkg_root() / "core" / "types" / "__init__.py" lines = types_path.read_text().splitlines() assert len(lines) < 200, f"types.py has {len(lines)} lines; expected re-export hub only" + + +def test_launch_id_env_var_in_private_vars() -> None: + from autoskillit.core import AUTOSKILLIT_PRIVATE_ENV_VARS, LAUNCH_ID_ENV_VAR + + assert LAUNCH_ID_ENV_VAR in AUTOSKILLIT_PRIVATE_ENV_VARS + + +def test_supports_debug_importable_from_core() -> None: + from typing import Protocol + + from autoskillit.core import SupportsDebug + + assert issubclass(SupportsDebug, Protocol) + + +def test_supports_debug_in_core_all() -> None: + import autoskillit.core as core_mod + + assert hasattr(core_mod, "SupportsDebug") + + +def test_subprocess_shard_all() -> None: + from autoskillit.core.types._type_subprocess import __all__ + + assert set(__all__) == {"SubprocessResult", "SubprocessRunner"} + assert "_TERMINATION_CONTRACT" not in __all__ + + +def test_subprocess_termination_contract_variable_still_defined() -> None: + import autoskillit.core.types._type_subprocess as m + + assert hasattr(m, "_TERMINATION_CONTRACT") diff --git a/tests/core/test_version_snapshot.py b/tests/core/test_version_snapshot.py new file mode 100644 index 0000000000..6c9c68632c --- /dev/null +++ b/tests/core/test_version_snapshot.py @@ -0,0 +1,111 @@ +"""Tests for core/_version_snapshot.py.""" + +from __future__ import annotations + +import json +import subprocess + +import pytest + +from autoskillit.core._version_snapshot import collect_version_snapshot + +pytestmark = [pytest.mark.layer("core"), pytest.mark.small] + + +@pytest.fixture(autouse=True) +def _clear_snapshot_cache(): + collect_version_snapshot.cache_clear() + yield + collect_version_snapshot.cache_clear() + + +def test_collect_version_snapshot_returns_required_keys(): + result = collect_version_snapshot() + assert set(result.keys()) == { + "autoskillit_version", + "install_type", + "commit_id", + "claude_code_version", + "plugins", + } + + +def test_collect_version_snapshot_is_cached(): + first = collect_version_snapshot() + second = collect_version_snapshot() + assert first is second + + +def test_collect_version_snapshot_cache_clear_works(): + first = collect_version_snapshot() + collect_version_snapshot.cache_clear() + second = collect_version_snapshot() + assert first is not second + + +def test_autoskillit_version_is_nonempty_string(): + result = collect_version_snapshot() + assert isinstance(result["autoskillit_version"], str) + assert result["autoskillit_version"] != "" + + +def test_claude_code_version_graceful_on_subprocess_error(monkeypatch): + import autoskillit.core._version_snapshot as mod + + def _raise(*args, **kwargs): + raise FileNotFoundError("claude not found") + + monkeypatch.setattr(mod.subprocess, "run", _raise) + result = collect_version_snapshot() + assert result["claude_code_version"] == "" + + +def test_claude_code_version_graceful_on_timeout(monkeypatch): + import autoskillit.core._version_snapshot as mod + + def _raise(*args, **kwargs): + raise subprocess.TimeoutExpired("claude", 5) + + monkeypatch.setattr(mod.subprocess, "run", _raise) + result = collect_version_snapshot() + assert result["claude_code_version"] == "" + + +def test_plugins_graceful_when_file_absent(monkeypatch, tmp_path): + import autoskillit.core._version_snapshot as mod + + monkeypatch.setattr(mod.Path, "home", classmethod(lambda cls: tmp_path)) + result = collect_version_snapshot() + assert result["plugins"] == [] + + +def test_plugins_reads_version(monkeypatch, tmp_path): + import autoskillit.core._version_snapshot as mod + + plugins_dir = tmp_path / ".claude" / "plugins" + plugins_dir.mkdir(parents=True) + plugin_data = { + "version": 2, + "plugins": { + "ref": [{"version": "1.0", "extra": "ignored"}], + }, + } + (plugins_dir / "installed_plugins.json").write_text(json.dumps(plugin_data), encoding="utf-8") + monkeypatch.setattr(mod.Path, "home", classmethod(lambda cls: tmp_path)) + result = collect_version_snapshot() + assert len(result["plugins"]) == 1 + entry = result["plugins"][0] + assert entry["ref"] == "ref" + assert entry["version"] == "1.0" + assert "name" not in entry + + +def test_plugins_graceful_on_corrupt_json(monkeypatch, tmp_path): + import autoskillit.core._version_snapshot as mod + + plugins_dir = tmp_path / ".claude" / "plugins" + plugins_dir.mkdir(parents=True) + (plugins_dir / "installed_plugins.json").write_text("not valid json", encoding="utf-8") + monkeypatch.setattr(mod.Path, "home", classmethod(lambda cls: tmp_path)) + result = collect_version_snapshot() + assert result["plugins"] == [] diff --git a/tests/docs/CLAUDE.md b/tests/docs/CLAUDE.md new file mode 100644 index 0000000000..b51e4497c8 --- /dev/null +++ b/tests/docs/CLAUDE.md @@ -0,0 +1,24 @@ +# docs/ + +Documentation integrity, link validity, and naming convention tests. + +## Files + +| File | Purpose | +|------|---------| +| `__init__.py` | empty | +| `test_banned_phrases.py` | Reject AI-tone banned phrases in every doc (derived from REQ-DOC-070) | +| `test_claude_md_structure.py` | Validate CLAUDE.md post-reorganization content accuracy | +| `test_doc_counts.py` | Verify every numerical claim in every doc file matches source of truth | +| `test_doc_index.py` | Verify every doc is reachable from docs/README.md and every subdir has a README | +| `test_doc_links.py` | Verify every local markdown link resolves and no old flat-layout link survives | +| `test_filename_naming.py` | Encode the 7 naming rules from REQ-DOC-085 as predicates over docs/ filenames | +| `test_glossary_spelling.py` | Reject banned variants of glossary terms across every doc | +| `test_no_franchise_in_docs.py` | Guard against franchise references in docs | +| `test_orchestration_levels.py` | Orchestration levels doc validation | +| `test_no_synthetic_citation_markers.py` | Guard against synthetic deep-research citation markers in tracked files | +| `test_rationale_document_completeness.py` | Validate experiment-type rationale document completeness | +| `test_sub_claude_md_completeness.py` | Structural tests for per-subfolder CLAUDE.md files under src/autoskillit/ | +| `test_tests_sub_claude_md_completeness.py` | Structural tests for per-subfolder CLAUDE.md files under tests/ | +| `test_guard_fail_mode_docs.py` | Verify guard fail-mode matrix documentation accuracy | +| `test_check_sub_claude_md_script.py` | Unit and integration tests for the check_sub_claude_md.py pre-commit hook script | diff --git a/tests/docs/test_check_sub_claude_md_script.py b/tests/docs/test_check_sub_claude_md_script.py new file mode 100644 index 0000000000..cb76279aa6 --- /dev/null +++ b/tests/docs/test_check_sub_claude_md_script.py @@ -0,0 +1,131 @@ +"""Unit and integration tests for scripts/check_sub_claude_md.py.""" + +from __future__ import annotations + +import importlib.util +import sys +from pathlib import Path + +import pytest + +pytestmark = pytest.mark.medium + +REPO_ROOT = Path(__file__).parent.parent.parent +_SCRIPT = REPO_ROOT / "scripts" / "check_sub_claude_md.py" + + +@pytest.fixture(scope="module") +def script_mod(): + spec = importlib.util.spec_from_file_location("check_sub_claude_md", _SCRIPT) + assert spec is not None and spec.loader is not None + mod = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = mod + spec.loader.exec_module(mod) + yield mod + sys.modules.pop(spec.name, None) + + +class TestCheckCoverage: + def test_check_coverage_all_files_mentioned(self, script_mod, tmp_path): + """Returns empty list when CLAUDE.md mentions every .py file in the directory.""" + subdir = tmp_path / "mypackage" + subdir.mkdir() + (subdir / "CLAUDE.md").write_text( + "| File | Purpose |\n|------|----------|\n" + "| `__init__.py` | init |\n| `foo.py` | foo |\n", + encoding="utf-8", + ) + (subdir / "__init__.py").touch() + (subdir / "foo.py").touch() + result = script_mod.check_coverage(tmp_path, ["mypackage/CLAUDE.md"]) + assert result == [] + + def test_check_coverage_missing_regular_py_file(self, script_mod, tmp_path): + """Returns failure string containing the missing filename.""" + subdir = tmp_path / "mypackage" + subdir.mkdir() + (subdir / "CLAUDE.md").write_text( + "| File | Purpose |\n|------|----------|\n| `__init__.py` | init |\n", + encoding="utf-8", + ) + (subdir / "__init__.py").touch() + (subdir / "bar.py").touch() + result = script_mod.check_coverage(tmp_path, ["mypackage/CLAUDE.md"]) + assert result == ["mypackage/CLAUDE.md: missing bar.py"] + + def test_check_coverage_missing_init_py_backtick(self, script_mod, tmp_path): + """Returns failure when __init__.py exists but CLAUDE.md lacks backtick-wrapped mention.""" + subdir = tmp_path / "mypackage" + subdir.mkdir() + (subdir / "CLAUDE.md").write_text( + "| File | Purpose |\n|------|----------|\n| `foo.py` | foo |\n", + encoding="utf-8", + ) + (subdir / "__init__.py").touch() + (subdir / "foo.py").touch() + result = script_mod.check_coverage(tmp_path, ["mypackage/CLAUDE.md"]) + assert result == ["mypackage/CLAUDE.md: missing `__init__.py` in file table"] + + def test_check_coverage_init_py_without_backticks_fails(self, script_mod, tmp_path): + """Returns failure when CLAUDE.md mentions __init__.py without backtick wrapping.""" + subdir = tmp_path / "mypackage" + subdir.mkdir() + (subdir / "CLAUDE.md").write_text( + "| File | Purpose |\n|------|----------|\n| __init__.py | init |\n", + encoding="utf-8", + ) + (subdir / "__init__.py").touch() + result = script_mod.check_coverage(tmp_path, ["mypackage/CLAUDE.md"]) + assert result == ["mypackage/CLAUDE.md: missing `__init__.py` in file table"] + + def test_check_coverage_flags_missing_claude_md(self, script_mod, tmp_path): + """Returns failure when expected CLAUDE.md path does not exist on disk.""" + result = script_mod.check_coverage(tmp_path, ["nonexistent/CLAUDE.md"]) + assert result == ["nonexistent/CLAUDE.md: CLAUDE.md not found"] + + def test_check_coverage_multiple_missing_files(self, script_mod, tmp_path): + """Returns one failure entry per missing file.""" + subdir = tmp_path / "mypackage" + subdir.mkdir() + (subdir / "CLAUDE.md").write_text( + "| File | Purpose |\n|------|----------|\n", + encoding="utf-8", + ) + (subdir / "__init__.py").touch() + (subdir / "a.py").touch() + (subdir / "b.py").touch() + result = sorted(script_mod.check_coverage(tmp_path, ["mypackage/CLAUDE.md"])) + assert result == [ + "mypackage/CLAUDE.md: missing `__init__.py` in file table", + "mypackage/CLAUDE.md: missing a.py", + "mypackage/CLAUDE.md: missing b.py", + ] + + +class TestExpectedListsSync: + def test_src_expected_matches_test_file(self, script_mod): + """SRC_EXPECTED matches EXPECTED_SUB_CLAUDE_MDS in test_sub_claude_md_completeness.""" + test_module_name = "test_sub_claude_md_completeness" + test_file = REPO_ROOT / "tests" / "docs" / f"{test_module_name}.py" + spec = importlib.util.spec_from_file_location(test_module_name, test_file) + assert spec is not None and spec.loader is not None + test_mod = importlib.util.module_from_spec(spec) + spec.loader.exec_module(test_mod) + assert sorted(script_mod.SRC_EXPECTED) == sorted(test_mod.EXPECTED_SUB_CLAUDE_MDS) + + def test_tests_expected_matches_test_file(self, script_mod): + """TESTS_EXPECTED list matches that in test_tests_sub_claude_md_completeness.""" + test_module_name = "test_tests_sub_claude_md_completeness" + test_file = REPO_ROOT / "tests" / "docs" / f"{test_module_name}.py" + spec = importlib.util.spec_from_file_location(test_module_name, test_file) + assert spec is not None and spec.loader is not None + test_mod = importlib.util.module_from_spec(spec) + spec.loader.exec_module(test_mod) + assert sorted(script_mod.TESTS_EXPECTED) == sorted(test_mod.EXPECTED_SUB_CLAUDE_MDS) + + +class TestMain: + def test_main_returns_zero_on_live_repo(self, script_mod): + """main() returns 0 against the actual project (integration guard).""" + result = script_mod.main() + assert result == 0, "main() should return 0 on a clean repo" diff --git a/tests/docs/test_claude_md_structure.py b/tests/docs/test_claude_md_structure.py new file mode 100644 index 0000000000..eecbc84fdf --- /dev/null +++ b/tests/docs/test_claude_md_structure.py @@ -0,0 +1,123 @@ +"""Validate CLAUDE.md post-reorganization content accuracy.""" + +from __future__ import annotations + +import re +from pathlib import Path + +CLAUDE_MD = Path(__file__).resolve().parents[2] / "CLAUDE.md" +_SERVER_DIR = CLAUDE_MD.parent / "src" / "autoskillit" / "server" + + +def test_claude_md_architecture_tree_has_subpackages() -> None: + content = CLAUDE_MD.read_text() + for subpkg in [ + "types/", + "runtime/", + "headless/", + "process/", + "merge_queue/", + "rules/", + "tools/", + "doctor/", + "fleet/", + "session/", + "ui/", + "update/", + "guards/", + "formatters/", + ]: + assert subpkg in content, f"CLAUDE.md tree missing {subpkg}" + + +def test_claude_md_dataclass_count_is_24() -> None: + content = CLAUDE_MD.read_text() + assert "24 leaf dataclasses" in content + assert "22 leaf dataclasses" not in content + + +def test_claude_md_has_lsp_section() -> None: + content = CLAUDE_MD.read_text() + assert "Pyright LSP" in content or "LSP" in content + assert "goToDefinition" in content + assert "findReferences" in content + + +def test_claude_md_no_precommit_install_line() -> None: + content = CLAUDE_MD.read_text() + assert "Install hooks after cloning: `pre-commit install`" not in content + + +def test_claude_md_def_spec_location_qualified() -> None: + content = CLAUDE_MD.read_text() + lines = content.splitlines() + for line in lines: + if "*Def*" in line or "`*Def`" in line: + if "Lives in `core/`" in line and "Typically" not in line: + raise AssertionError( + "CLAUDE.md *Def rule has unqualified 'Lives in core/' — " + "should say 'Typically lives in core/'" + ) + + +def test_claude_md_mentions_write_guard() -> None: + # write_guard is documented in hooks/guards/CLAUDE.md — accept either location + main_content = CLAUDE_MD.read_text() + guards_claude = CLAUDE_MD.parent / "src" / "autoskillit" / "hooks" / "guards" / "CLAUDE.md" + assert "write_guard" in main_content or ( + guards_claude.exists() and "write_guard" in guards_claude.read_text() + ) + + +def test_claude_md_mentions_dispatch_food_truck() -> None: + # dispatch_food_truck is documented in server/tools/CLAUDE.md — accept either location + main_content = CLAUDE_MD.read_text() + tools_claude = CLAUDE_MD.parent / "src" / "autoskillit" / "server" / "tools" / "CLAUDE.md" + assert "dispatch_food_truck" in main_content or ( + tools_claude.exists() and "dispatch_food_truck" in tools_claude.read_text() + ) + + +def test_claude_md_defines_channel_b() -> None: + content = CLAUDE_MD.read_text() + assert "Channel B" in content + assert re.search(r"Channel B[^.]*JSONL", content), ( + "CLAUDE.md references 'Channel B' without an inline definition mentioning JSONL" + ) + + +def test_server_claude_md_has_tool_gating_section() -> None: + """server/CLAUDE.md documents the two-layer tool gating architecture.""" + text = (_SERVER_DIR / "CLAUDE.md").read_text() + assert "## Tool Gating Architecture" in text + assert "### Tag-Visibility" in text + assert "### Application-Gate" in text + + +def test_server_claude_md_gating_matrix_covers_categories() -> None: + """server/CLAUDE.md gating matrix includes all five tool categories.""" + text = (_SERVER_DIR / "CLAUDE.md").read_text() + for category in [ + "Standard kitchen", + "Fleet tool", + "Fleet-dispatch tool", + "Headless-exempt", + "Free-range", + ]: + assert category in text, f"Missing tool category {category!r} in gating matrix" + + +def test_test_check_has_require_enabled_exception_comment() -> None: + """test_check has an inline comment explaining why it skips _require_enabled().""" + ws_file = _SERVER_DIR / "tools" / "tools_workspace.py" + src = ws_file.read_text() + lines = src.splitlines() + for i, line in enumerate(lines): + if "async def test_check(" in line: + region = lines[max(0, i - 10) : i] + comment_lines = [ln for ln in region if ln.lstrip().startswith("#")] + assert any("_require_enabled" in c for c in comment_lines), ( + "test_check decorator region should contain a comment about _require_enabled" + ) + return + raise AssertionError("async def test_check not found in tools_workspace.py") diff --git a/tests/docs/test_doc_counts.py b/tests/docs/test_doc_counts.py index 7746ba43fd..7fa14aa832 100644 --- a/tests/docs/test_doc_counts.py +++ b/tests/docs/test_doc_counts.py @@ -39,38 +39,59 @@ def _docs_containing(pattern: str) -> list[Path]: # ----- canonical source-of-truth getters -------------------------------------- +def _extract_tool_decorators(text: str) -> list[str]: + """Extract full @mcp.tool(...) decorator text, handling multi-line decorators.""" + decorators: list[str] = [] + lines = text.splitlines() + i = 0 + while i < len(lines): + stripped = lines[i].strip() + if stripped.startswith("@mcp.tool"): + # Collect the full decorator (may span multiple lines) + parts = [stripped] + if ")" not in stripped: + i += 1 + while i < len(lines): + part = lines[i].strip() + parts.append(part) + if ")" in part: + break + i += 1 + decorators.append(" ".join(parts)) + i += 1 + return decorators + + def _count_mcp_tools() -> int: total = 0 - for f in (SRC_DIR / "server").glob("tools_*.py"): - total += sum(1 for line in _read(f).splitlines() if line.strip().startswith("@mcp.tool")) + for f in (SRC_DIR / "server" / "tools").glob("tools_*.py"): + total += len(_extract_tool_decorators(_read(f))) return total def _count_kitchen_tools() -> int: total = 0 - for f in (SRC_DIR / "server").glob("tools_*.py"): - text = _read(f) - for line in text.splitlines(): - if line.strip().startswith("@mcp.tool") and '"kitchen"' in line: + for f in (SRC_DIR / "server" / "tools").glob("tools_*.py"): + for dec in _extract_tool_decorators(_read(f)): + if '"kitchen"' in dec: total += 1 return total def _count_free_range_tools() -> int: total = 0 - for f in (SRC_DIR / "server").glob("tools_*.py"): - for line in _read(f).splitlines(): - stripped = line.strip() - if stripped.startswith("@mcp.tool") and '"kitchen"' not in stripped: + for f in (SRC_DIR / "server" / "tools").glob("tools_*.py"): + for dec in _extract_tool_decorators(_read(f)): + if '"kitchen"' not in dec: total += 1 return total def _count_headless_tools() -> int: total = 0 - for f in (SRC_DIR / "server").glob("tools_*.py"): - for line in _read(f).splitlines(): - if line.strip().startswith("@mcp.tool") and '"headless"' in line: + for f in (SRC_DIR / "server" / "tools").glob("tools_*.py"): + for dec in _extract_tool_decorators(_read(f)): + if '"headless"' in dec: total += 1 return total @@ -106,7 +127,7 @@ def _count_vis_lens_skills() -> int: def _hook_files() -> list[Path]: - return sorted(f for f in (SRC_DIR / "hooks").glob("*.py") if f.name not in {"__init__.py"}) + return sorted(f for f in (SRC_DIR / "hooks").rglob("*.py") if f.name not in {"__init__.py"}) def _count_hooks_by_event() -> dict[str, int]: @@ -151,9 +172,9 @@ def _count_doctor_checks() -> int: style for their internal sub-steps; we restrict the count to the body of ``run_doctor`` so those comments do not double-count. """ - text = _read(SRC_DIR / "cli" / "_doctor.py") + text = _read(SRC_DIR / "cli" / "doctor" / "__init__.py") body = re.search(r"def run_doctor\(.*?\n((?: .*\n|\n)+)", text, re.DOTALL) - assert body, "run_doctor not found in _doctor.py" + assert body, "run_doctor not found in cli/doctor/__init__.py" body_text = body.group(1) numbered = len(re.findall(r"# Check \d+:", body_text)) lettered = len(re.findall(r"# Check \d+[a-z]:", body_text)) @@ -165,28 +186,27 @@ def _bundled_recipes() -> list[str]: def _retry_reason_values() -> list[str]: - text = _read(SRC_DIR / "core" / "_type_enums.py") + text = _read(SRC_DIR / "core" / "types" / "_type_enums.py") block = re.search(r"class RetryReason\(StrEnum\):(.*?)\nclass ", text, re.DOTALL) assert block, "RetryReason enum not found" return re.findall(r'"([a-z_]+)"', block.group(1)) def _count_semantic_rule_files() -> int: - return sum(1 for p in (SRC_DIR / "recipe").glob("rules_*.py")) + return sum(1 for p in (SRC_DIR / "recipe" / "rules").glob("rules_*.py")) # ----- tests ------------------------------------------------------------------ -def test_kitchen_tagged_tool_count_is_41() -> None: - assert _count_kitchen_tools() == 41, ( - f"Expected 41 kitchen-tagged tools; found {_count_kitchen_tools()}" - ) +def test_kitchen_tagged_tool_count_is_37() -> None: + count = _count_kitchen_tools() + assert count == 37, f"Expected 37 kitchen-tagged tools; found {count}" -def test_free_range_tool_count_is_3() -> None: - assert _count_free_range_tools() == 3, ( - f"Expected 3 free-range tools; found {_count_free_range_tools()}" +def test_free_range_tool_count_is_15() -> None: + assert _count_free_range_tools() == 15, ( + f"Expected 15 free-range tools; found {_count_free_range_tools()}" ) @@ -211,36 +231,54 @@ def test_vis_lens_count_is_12() -> None: def test_quota_thresholds_defaults() -> None: short, long_ = _quota_thresholds_default() assert short == pytest.approx(85.0) - assert long_ == pytest.approx(98.0) + assert long_ == pytest.approx(95.0) -def test_doctor_check_count_is_22() -> None: +def test_doctor_check_count_is_31() -> None: # _count_doctor_checks() counts every "# Check N:" and "# Check Nb:" marker - # in run_doctor() — 17 numbered base markers + 5 lettered sub-check markers - # (2b, 2c, 2d, 4b, 7b) = 22 total. test_installation_states_17_doctor_checks - # checks the *user-visible* count from docs/installation.md ("15 numbered + 2 - # lettered sub-checks 4b and 7b = 17"). The gap of 5 is intentional: Check 2, - # Check 4, and Check 7 each appear as separate implementation markers but the - # docs present them as single numbered entries that subsume their sub-variants. + # in run_doctor() — 17 numbered base markers + 6 lettered sub-check markers + # (2b, 2c, 2d, 2e, 4b, 7b) + 4 ambient env checks (18–21) + # + 2 new unconditional feature checks (22–23) + # + 5 gated franchise checks (24–28) = 34 total. + # test_installation_states_17_doctor_checks checks the *user-visible* count + # from docs/installation.md ("15 numbered + 2 lettered sub-checks 4b and 7b + # = 17"). The gap of 5 is intentional: Check 2, Check 4, and Check 7 each + # appear as separate implementation markers but the docs present them as single + # numbered entries that subsume their sub-variants. # Update both tests whenever a new doctor check is added. - assert _count_doctor_checks() == 22, ( - f"Expected 22 doctor checks; found {_count_doctor_checks()}" + assert _count_doctor_checks() == 34, ( + f"Expected 34 doctor checks; found {_count_doctor_checks()}" ) -def test_bundled_recipe_count_is_5() -> None: +def test_bundled_recipe_count_is_14() -> None: recipes = _bundled_recipes() - expected = ["implementation", "implementation-groups", "merge-prs", "remediation", "research"] + expected = [ + "bem-wrapper", + "full-audit", + "implement-findings", + "implementation", + "implementation-groups", + "merge-prs", + "planner", + "promote-to-main-wrapper", + "remediation", + "research", + "research-archive", + "research-design", + "research-implement", + "research-review", + ] assert recipes == expected, f"Recipes drifted: {recipes}" -def test_retry_reason_value_count_is_11() -> None: +def test_retry_reason_value_count_is_14() -> None: values = _retry_reason_values() - assert len(values) == 11, f"RetryReason has {len(values)} values: {values}" + assert len(values) == 14, f"RetryReason has {len(values)} values: {values}" -def test_semantic_rule_family_count_is_20() -> None: - assert _count_semantic_rule_files() == 20 +def test_semantic_rule_family_count_is_25() -> None: + assert _count_semantic_rule_files() == 28 # ----- per-doc count assertions (run once docs exist) ------------------------- @@ -261,8 +299,8 @@ def _assert_doc_states_number(doc: Path, label: str, expected: int) -> None: DOCS_DIR / "execution" / "tool-access.md", ], ) -def test_docs_state_43_mcp_tools(doc_path: Path) -> None: - _assert_doc_states_number(doc_path, "MCP tools", 43) +def test_docs_state_52_mcp_tools(doc_path: Path) -> None: + _assert_doc_states_number(doc_path, "MCP tools", 52) @pytest.mark.parametrize( @@ -272,16 +310,18 @@ def test_docs_state_43_mcp_tools(doc_path: Path) -> None: DOCS_DIR / "execution" / "tool-access.md", ], ) -def test_docs_state_41_kitchen_tools(doc_path: Path) -> None: - _assert_doc_states_number(doc_path, "kitchen tools", 41) +def test_docs_state_37_kitchen_tools(doc_path: Path) -> None: + _assert_doc_states_number(doc_path, "kitchen tools", 37) -def test_skill_visibility_states_109_skills() -> None: - _assert_doc_states_number(DOCS_DIR / "skills" / "visibility.md", "skills total", 109) +def test_skill_visibility_states_133_skills() -> None: + # 133 = 3 Tier-1 (open-kitchen, close-kitchen, sous-chef) + 130 extended. + # DefaultSkillResolver.list_all() returns 132 (excludes sous-chef from public surface). + _assert_doc_states_number(DOCS_DIR / "skills" / "visibility.md", "skills total", 133) -def test_safety_hooks_states_13_hooks() -> None: - _assert_doc_states_number(DOCS_DIR / "safety" / "hooks.md", "hooks total", 13) +def test_safety_hooks_states_21_hooks() -> None: + _assert_doc_states_number(DOCS_DIR / "safety" / "hooks.md", "hooks total", 21) def test_configuration_states_quota_thresholds() -> None: @@ -292,8 +332,8 @@ def test_configuration_states_quota_thresholds() -> None: assert "85.0" in text, ( "docs/configuration.md does not state quota_guard.short_window_threshold = 85.0" ) - assert "98.0" in text, ( - "docs/configuration.md does not state quota_guard.long_window_threshold = 98.0" + assert "95.0" in text, ( + "docs/configuration.md does not state quota_guard.long_window_threshold = 95.0" ) @@ -301,16 +341,16 @@ def test_installation_states_17_doctor_checks() -> None: _assert_doc_states_number(DOCS_DIR / "installation.md", "doctor checks", 17) -def test_recipes_overview_states_5_recipes() -> None: - _assert_doc_states_number(DOCS_DIR / "recipes" / "overview.md", "bundled recipes", 5) +def test_recipes_overview_states_6_recipes() -> None: + _assert_doc_states_number(DOCS_DIR / "recipes" / "overview.md", "bundled recipes", 6) -def test_orchestration_states_11_retry_reasons() -> None: - _assert_doc_states_number(DOCS_DIR / "execution" / "orchestration.md", "retry reasons", 11) +def test_orchestration_states_13_retry_reasons() -> None: + _assert_doc_states_number(DOCS_DIR / "execution" / "orchestration.md", "retry reasons", 13) -def test_authoring_states_17_rule_families() -> None: - _assert_doc_states_number(DOCS_DIR / "recipes" / "authoring.md", "rule families", 17) +def test_authoring_states_24_rule_families() -> None: + _assert_doc_states_number(DOCS_DIR / "recipes" / "authoring.md", "rule families", 24) def test_catalog_states_arch_and_exp_lens_counts() -> None: @@ -328,3 +368,69 @@ def test_catalog_states_vis_lens_count_is_12() -> None: pytest.skip("docs/skills/catalog.md not present") text = _read(catalog) assert "12" in text, "skills/catalog.md does not state 12 vis-lens skills" + + +def test_catalog_count_matches_filesystem() -> None: + """Header in catalog.md must match the actual skill-dir count.""" + catalog = (DOCS_DIR / "skills" / "catalog.md").read_text(encoding="utf-8") + skills_dir = SRC_DIR / "skills" + extended_dir = SRC_DIR / "skills_extended" + tier1_count = sum(1 for p in skills_dir.iterdir() if p.is_dir() and p.name != "__pycache__") + extended_count = sum( + 1 for p in extended_dir.iterdir() if p.is_dir() and p.name != "__pycache__" + ) + total = tier1_count + extended_count + assert f"{total} total" in catalog, f"catalog.md header should claim {total} total skills" + + +def test_catalog_does_not_reference_open_pr() -> None: + """open-pr was decomposed into prepare-pr + compose-pr in PR #659.""" + catalog = (DOCS_DIR / "skills" / "catalog.md").read_text() + assert "`open-pr`" not in catalog + + +def test_catalog_lists_all_skills_in_extended_dir() -> None: + """Every directory in skills_extended/ must appear at least once in catalog.md.""" + catalog = (DOCS_DIR / "skills" / "catalog.md").read_text() + extended_dir = SRC_DIR / "skills_extended" + missing = [ + p.name + for p in sorted(extended_dir.iterdir()) + if p.is_dir() and f"`{p.name}`" not in catalog + ] + assert missing == [], f"Skills in skills_extended/ not listed in catalog.md: {missing}" + + +def test_authoring_bundled_recipe_count_mentions_6() -> None: + """authoring.md prose must reference 6 bundled recipes (planner was missing).""" + authoring = (DOCS_DIR / "recipes" / "authoring.md").read_text() + assert "6 today" in authoring and "planner" in authoring, ( + "authoring.md should mention 6 bundled recipes including planner" + ) + + +def test_authoring_recipe_step_fields_match_schema() -> None: + """authoring.md field summary must use actual RecipeStep field names.""" + authoring = (DOCS_DIR / "recipes" / "authoring.md").read_text() + assert "`name`" in authoring + assert "`with_args`" in authoring + assert "`on_result`" in authoring + assert "`retries`" in authoring + field_summary_line = next( + (line for line in authoring.splitlines() if "with_args" in line or "id`," in line), + None, + ) + assert field_summary_line is not None, ( + "authoring.md must contain a line listing RecipeStep fields (with_args or id`,)" + ) + assert "id`," not in field_summary_line + assert "params`," not in field_summary_line + assert "verdict_routes`" not in field_summary_line + assert "retry`." not in field_summary_line + + +def test_subsets_lists_required_packs() -> None: + """subsets.md must document all four missing built-in pack categories.""" + subsets = (DOCS_DIR / "skills" / "subsets.md").read_text(encoding="utf-8") + for pack in ("kitchen-core", "research", "exp-lens", "vis-lens"): + assert pack in subsets, f"subsets.md missing pack category: {pack}" diff --git a/tests/docs/test_filename_naming.py b/tests/docs/test_filename_naming.py index 413bbd0edd..3ea8866566 100644 --- a/tests/docs/test_filename_naming.py +++ b/tests/docs/test_filename_naming.py @@ -15,7 +15,13 @@ # Files allow-listed against specific rules. Each entry: rule -> {filenames}. ALLOWLIST = { "ing_suffix": {"getting-started.md", "contributing.md", "authoring.md"}, - "segment_count": {"getting-started.md", "end-turn-hazards.md", "research-pipeline.md"}, + "segment_count": { + "getting-started.md", + "end-turn-hazards.md", + "research-pipeline.md", + "0001-prohibit-background-subagent-execution.md", + "0002-ban-inline-shell-scripts-from-cmd.md", + }, } # overrides.md is a list-like file (collection of override sites) and is exempt; diff --git a/tests/docs/test_guard_fail_mode_docs.py b/tests/docs/test_guard_fail_mode_docs.py new file mode 100644 index 0000000000..7d2e15cc96 --- /dev/null +++ b/tests/docs/test_guard_fail_mode_docs.py @@ -0,0 +1,76 @@ +"""Verify guard fail-mode matrix documentation accuracy.""" + +from __future__ import annotations + +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parents[2] +GUARDS_CLAUDE = REPO_ROOT / "src/autoskillit/hooks/guards/CLAUDE.md" +SAFETY_HOOKS = REPO_ROOT / "docs/safety/hooks.md" + +RETIRED_GUARD = "leaf_orchestration_guard.py" + + +def _extract_section(content: str, heading: str) -> str: + import re + + level = heading.index(" ") + pattern = re.compile( + rf"^{re.escape(heading)}\n(.+?)(?=^#{{1,{level}}} |\Z)", + re.MULTILINE | re.DOTALL, + ) + m = pattern.search(content) + return m.group(1) if m else "" + + +class TestGuardsClaudeMd: + def test_guards_claude_md_contains_fail_mode_contract_section(self) -> None: + content = GUARDS_CLAUDE.read_text() + assert "### Fail-Mode Contract" in content + + def test_guards_claude_md_fail_mode_matrix_lists_three_fail_closed_guards(self) -> None: + content = GUARDS_CLAUDE.read_text() + section = _extract_section(content, "### Fail-Mode Contract") + for guard in [ + "skill_command_guard.py", + "open_kitchen_guard.py", + "skill_orchestration_guard.py", + ]: + assert guard in section, f"{guard} not in Fail-Mode Contract section" + assert RETIRED_GUARD not in section + + def test_guards_claude_md_documents_design_principle(self) -> None: + content = GUARDS_CLAUDE.read_text() + assert "Garbage-in" in content + assert "Unknown-tier" in content + + def test_old_fail_mode_sentence_replaced(self) -> None: + content = GUARDS_CLAUDE.read_text() + old = ( + "Guards fail-open for malformed input." + " `skill_command_guard.py` has split error handling" + ) + assert old not in content + + +class TestSafetyHooksMd: + def test_safety_hooks_md_contains_fail_modes_section(self) -> None: + content = SAFETY_HOOKS.read_text() + assert "## Fail Modes" in content + + def test_safety_hooks_md_fail_mode_matrix_lists_three_fail_closed_guards(self) -> None: + content = SAFETY_HOOKS.read_text() + section = _extract_section(content, "## Fail Modes") + for guard in [ + "skill_command_guard.py", + "open_kitchen_guard.py", + "skill_orchestration_guard.py", + ]: + assert guard in section, f"{guard} not in Fail Modes section" + assert RETIRED_GUARD not in section + + def test_safety_hooks_md_documents_design_principle(self) -> None: + content = SAFETY_HOOKS.read_text() + section = _extract_section(content, "## Fail Modes") + assert "Garbage-in" in section + assert "Unknown-tier" in section diff --git a/tests/docs/test_no_franchise_in_docs.py b/tests/docs/test_no_franchise_in_docs.py new file mode 100644 index 0000000000..42eecbdcc0 --- /dev/null +++ b/tests/docs/test_no_franchise_in_docs.py @@ -0,0 +1,22 @@ +from pathlib import Path + +import pytest + +pytestmark = [pytest.mark.layer("docs"), pytest.mark.small] + +DOCS_ROOT = Path(__file__).parents[2] / "docs" + + +def test_docs_no_franchise_references(): + """All docs/*.md files must use fleet terminology, not franchise.""" + hits = [] + for md_file in DOCS_ROOT.rglob("*.md"): + text = md_file.read_text(encoding="utf-8", errors="ignore") + if "franchise" in text.lower(): + lines = [ + f" {md_file.relative_to(DOCS_ROOT)}:{i + 1}: {line.rstrip()}" + for i, line in enumerate(text.splitlines()) + if "franchise" in line.lower() + ] + hits.extend(lines) + assert not hits, "Found franchise references in docs:\n" + "\n".join(hits) diff --git a/tests/docs/test_no_synthetic_citation_markers.py b/tests/docs/test_no_synthetic_citation_markers.py new file mode 100644 index 0000000000..1a609824ef --- /dev/null +++ b/tests/docs/test_no_synthetic_citation_markers.py @@ -0,0 +1,40 @@ +"""Guard: no synthetic deep-research citation markers in the codebase.""" + +from __future__ import annotations + +import subprocess + +import pytest + +pytestmark = [pytest.mark.layer("docs"), pytest.mark.small] + +# Pattern matches deep-research tool citation markers. +# chr() is used intentionally so the guard does not trigger itself. +SYNTHETIC_MARKER_PATTERNS = [ + chr(0x3010), # U+3010 LEFT BLACK LENTICULAR BRACKET (unique to deep-research markers) + r"†L\d", # Dagger + L + digit (line reference fragment) +] + + +@pytest.mark.parametrize("pattern", SYNTHETIC_MARKER_PATTERNS) +def test_no_synthetic_citation_markers_in_tracked_files(pattern: str) -> None: + """No tracked file contains synthetic deep-research citation markers.""" + result = subprocess.run( + [ + "git", + "grep", + "-n", + "-P", + pattern, + "--", + "*.md", + "*.yaml", + "*.py", + ], + capture_output=True, + text=True, + ) + matches = result.stdout.strip() + assert not matches, ( + f"Synthetic citation marker pattern {pattern!r} found in tracked files:\n{matches}" + ) diff --git a/tests/docs/test_orchestration_levels.py b/tests/docs/test_orchestration_levels.py new file mode 100644 index 0000000000..3e49dbfe0b --- /dev/null +++ b/tests/docs/test_orchestration_levels.py @@ -0,0 +1,80 @@ +import re +from pathlib import Path + +REPO_ROOT = Path(__file__).parents[2] +assert (REPO_ROOT / "pyproject.toml").exists(), "REPO_ROOT detection broken" +ORCH_DOC = REPO_ROOT / "docs" / "orchestration-levels.md" +GLOSSARY = REPO_ROOT / "docs" / "glossary.md" + + +def test_orchestration_levels_doc_exists(): + assert ORCH_DOC.exists(), "docs/orchestration-levels.md must be created" + + +def test_orchestration_levels_has_required_sections(): + text = ORCH_DOC.read_text() + for heading in [ + "## Level Definitions", + "## Mapping Table", + "## Key Rules", + "## Disambiguation", + ]: + assert heading in text, f"Missing section: {heading}" + + +def test_orchestration_levels_has_l_identifiers(): + text = ORCH_DOC.read_text() + for level in ["L0", "L1", "L2", "L3"]: + assert level in text + + +def test_glossary_has_orchestration_level_entries(): + text = GLOSSARY.read_text() + for term in ["### L0", "### L1", "### L2", "### L3", "### food truck", "### Ghost Kitchen"]: + assert term in text, f"Glossary missing entry: {term}" + + +def test_glossary_orchestrator_entry_uses_l2(): + text = GLOSSARY.read_text().replace("\r\n", "\n") + match = re.search(r"### orchestrator\n(.+?)(?=\n###|\Z)", text, re.DOTALL) + assert match, "Glossary missing ### orchestrator entry" + section = match.group(1) + assert "L2" in section, "'orchestrator' glossary entry must reference L2" + assert "Tier 1" not in section, "'orchestrator' entry must not use 'Tier 1' language" + + +def test_glossary_worker_entry_uses_l1(): + text = GLOSSARY.read_text().replace("\r\n", "\n") + match = re.search(r"### worker\n(.+?)(?=\n###|\Z)", text, re.DOTALL) + assert match, "Glossary missing ### worker entry" + section = match.group(1) + assert "L1" in section, "'worker' glossary entry must reference L1" + assert "Tier 2" not in section, "'worker' entry must not use 'Tier 2' language" + + +def test_orchestration_doc_cross_references_levels(): + orch_exec = REPO_ROOT / "docs" / "execution" / "orchestration.md" + assert orch_exec.exists(), "docs/execution/orchestration.md must exist" + text = orch_exec.read_text() + assert "orchestration-levels.md" in text, ( + "docs/execution/orchestration.md must cross-reference orchestration-levels.md" + ) + + +def test_claude_md_has_il_disambiguation(): + claude_md = REPO_ROOT / "CLAUDE.md" + text = claude_md.read_text() + assert "| IL-N" in text, "CLAUDE.md must contain the '| IL-N' row in the disambiguation table" + assert "| IL-NNN" in text, ( + "CLAUDE.md must contain the '| IL-NNN' row in the disambiguation table" + ) + assert "| L-N" in text, "CLAUDE.md must contain the '| L-N' row in the disambiguation table" + assert "Import layer level" in text, ( + "Disambiguation table must explain IL-N as 'Import layer level'" + ) + assert "Import-linter contract ID" in text, ( + "Disambiguation table must explain IL-NNN as 'Import-linter contract ID'" + ) + assert "Orchestration level" in text or "orchestration level" in text, ( + "Disambiguation table must explain L-N as orchestration level" + ) diff --git a/tests/docs/test_rationale_document_completeness.py b/tests/docs/test_rationale_document_completeness.py new file mode 100644 index 0000000000..619ad6a8ea --- /dev/null +++ b/tests/docs/test_rationale_document_completeness.py @@ -0,0 +1,57 @@ +"""Validate experiment-type rationale document completeness.""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +pytestmark = [pytest.mark.layer("docs"), pytest.mark.small] + +RATIONALE_PATH = ( + Path(__file__).resolve().parents[2] / "docs" / "research" / "experiment-type-rationale.md" +) + +EXPECTED_TYPE_SECTIONS = [ + "evidence_synthesis", + "observational_correlational", + "instrument_validation", + "simulation_modeling", + "factorial_design", + "single_subject", + "qualitative_interpretive", +] + +_LENTICULAR = chr(0x3010) # U+3010 LEFT BLACK LENTICULAR BRACKET; chr() avoids literal in source + + +@pytest.fixture() +def rationale_text() -> str: + """Read the rationale document.""" + assert RATIONALE_PATH.exists(), f"Rationale document not found at {RATIONALE_PATH}" + return RATIONALE_PATH.read_text() + + +def test_rationale_document_exists() -> None: + """docs/research/experiment-type-rationale.md exists.""" + assert RATIONALE_PATH.exists() + + +@pytest.mark.parametrize("type_name", EXPECTED_TYPE_SECTIONS) +def test_rationale_covers_experiment_type(rationale_text: str, type_name: str) -> None: + """Each new experiment type has a section in the rationale document.""" + assert f"`{type_name}`" in rationale_text, ( + f"Experiment type {type_name!r} not found as a backtick-delimited section heading" + ) + + +def test_rationale_has_reference_list(rationale_text: str) -> None: + """Rationale document contains a Reference List section.""" + assert "## Reference List" in rationale_text + + +def test_rationale_has_no_synthetic_markers(rationale_text: str) -> None: + """The rationale document itself contains no synthetic citation markers.""" + assert _LENTICULAR not in rationale_text, ( + "Rationale document contains synthetic citation marker" + ) diff --git a/tests/docs/test_sub_claude_md_completeness.py b/tests/docs/test_sub_claude_md_completeness.py new file mode 100644 index 0000000000..4a752324dd --- /dev/null +++ b/tests/docs/test_sub_claude_md_completeness.py @@ -0,0 +1,106 @@ +"""Structural tests for per-subfolder CLAUDE.md documentation files.""" + +import re +from pathlib import Path + +import pytest + +pytestmark = pytest.mark.small + +SRC_ROOT = Path(__file__).resolve().parents[2] / "src" / "autoskillit" + +EXPECTED_SUB_CLAUDE_MDS = [ + "core/types/CLAUDE.md", + "core/runtime/CLAUDE.md", + "execution/headless/CLAUDE.md", + "execution/process/CLAUDE.md", + "execution/session/CLAUDE.md", + "execution/merge_queue/CLAUDE.md", + "recipe/rules/CLAUDE.md", + "server/tools/CLAUDE.md", + "cli/doctor/CLAUDE.md", + "cli/fleet/CLAUDE.md", + "cli/session/CLAUDE.md", + "cli/ui/CLAUDE.md", + "cli/update/CLAUDE.md", + "hooks/guards/CLAUDE.md", + "hooks/formatters/CLAUDE.md", + "CLAUDE.md", # src/autoskillit/ package root + "core/CLAUDE.md", + "config/CLAUDE.md", + "pipeline/CLAUDE.md", + "execution/CLAUDE.md", + "workspace/CLAUDE.md", + "planner/CLAUDE.md", + "recipe/CLAUDE.md", + "migration/CLAUDE.md", + "fleet/CLAUDE.md", + "cli/CLAUDE.md", + "hooks/CLAUDE.md", +] + + +def test_all_27_sub_claude_md_files_exist(): + assert len(EXPECTED_SUB_CLAUDE_MDS) == 27, ( + f"Expected 27 entries, got {len(EXPECTED_SUB_CLAUDE_MDS)}" + ) + assert len(EXPECTED_SUB_CLAUDE_MDS) == len(set(EXPECTED_SUB_CLAUDE_MDS)), ( + "Duplicate entries in EXPECTED_SUB_CLAUDE_MDS" + ) + missing = [p for p in EXPECTED_SUB_CLAUDE_MDS if not (SRC_ROOT / p).is_file()] + assert not missing, f"Missing sub-CLAUDE.md files: {missing}" + + +def test_sub_claude_md_covers_all_py_files(): + failures = [] + for rel_path in EXPECTED_SUB_CLAUDE_MDS: + claude_md = SRC_ROOT / rel_path + if not claude_md.is_file(): + failures.append(f"{rel_path}: file does not exist") + continue + content = claude_md.read_text() + directory = claude_md.parent + py_files = sorted(f.name for f in directory.glob("*.py")) + for py_file in py_files: + if py_file == "__init__.py": + if "`__init__.py`" not in content: + failures.append(f"{rel_path}: missing `__init__.py` in file table") + else: + if py_file not in content: + failures.append(f"{rel_path}: missing {py_file}") + assert not failures, "Sub-CLAUDE.md coverage gaps:\n" + "\n".join(failures) + + +def test_sub_claude_md_has_file_table(): + failures = [] + for rel_path in EXPECTED_SUB_CLAUDE_MDS: + claude_md = SRC_ROOT / rel_path + if not claude_md.is_file(): + failures.append(f"{rel_path}: file does not exist") + continue + content = claude_md.read_text() + if "| File | Purpose |" not in content: + failures.append(f"{rel_path}: missing '| File | Purpose |' table header") + assert not failures, "Sub-CLAUDE.md template violations:\n" + "\n".join(failures) + + +def test_channel_b_defined_in_process_claude_md(): + process_md = SRC_ROOT / "execution" / "process" / "CLAUDE.md" + assert process_md.is_file(), "execution/process/CLAUDE.md does not exist" + content = process_md.read_text() + assert "Channel B" in content, "execution/process/CLAUDE.md must define Channel B" + assert "JSONL" in content, "Channel B definition must mention JSONL" + + +def test_sub_claude_md_no_main_claude_md_duplication(): + numbered_section_re = re.compile(r"^## \*{0,2}\d+\.", re.MULTILINE) + failures = [] + for rel_path in EXPECTED_SUB_CLAUDE_MDS: + claude_md = SRC_ROOT / rel_path + if not claude_md.is_file(): + continue + content = claude_md.read_text() + match = numbered_section_re.search(content) + if match: + failures.append(f"{rel_path}: contains '{match.group()}' (main CLAUDE.md section)") + assert not failures, "Sub-CLAUDE.md files duplicate main sections:\n" + "\n".join(failures) diff --git a/tests/docs/test_tests_sub_claude_md_completeness.py b/tests/docs/test_tests_sub_claude_md_completeness.py new file mode 100644 index 0000000000..8ca2832a78 --- /dev/null +++ b/tests/docs/test_tests_sub_claude_md_completeness.py @@ -0,0 +1,123 @@ +"""Structural tests for per-subfolder CLAUDE.md documentation files under tests/.""" + +import re +from pathlib import Path + +import pytest + +pytestmark = pytest.mark.small + +TESTS_ROOT = Path(__file__).resolve().parents[1] + +EXPECTED_SUB_CLAUDE_MDS = [ + "arch/CLAUDE.md", + "assets/CLAUDE.md", + "cli/CLAUDE.md", + "config/CLAUDE.md", + "contracts/CLAUDE.md", + "core/CLAUDE.md", + "docs/CLAUDE.md", + "execution/CLAUDE.md", + "fleet/CLAUDE.md", + "hooks/CLAUDE.md", + "infra/CLAUDE.md", + "migration/CLAUDE.md", + "pipeline/CLAUDE.md", + "planner/CLAUDE.md", + "recipe/CLAUDE.md", + "server/CLAUDE.md", + "skills/CLAUDE.md", + "skills_extended/CLAUDE.md", + "workspace/CLAUDE.md", +] + + +def test_all_tests_sub_claude_md_files_exist(): + """Every test subdirectory has a CLAUDE.md file.""" + missing = [p for p in EXPECTED_SUB_CLAUDE_MDS if not (TESTS_ROOT / p).is_file()] + assert not missing, f"Missing tests/ sub-CLAUDE.md files: {missing}" + + +def test_tests_sub_claude_md_covers_all_py_files(): + """Each sub-CLAUDE.md mentions every .py file in its directory.""" + failures = [] + for rel_path in EXPECTED_SUB_CLAUDE_MDS: + claude_md = TESTS_ROOT / rel_path + if not claude_md.is_file(): + failures.append(f"{rel_path}: file does not exist") + continue + content = claude_md.read_text() + directory = claude_md.parent + py_files = sorted(f.name for f in directory.glob("*.py")) + for py_file in py_files: + if py_file == "__init__.py": + if "`__init__.py`" not in content: + failures.append(f"{rel_path}: missing `__init__.py` in file table") + else: + if py_file not in content: + failures.append(f"{rel_path}: missing {py_file}") + assert not failures, "tests/ sub-CLAUDE.md coverage gaps:\n" + "\n".join(failures) + + +def test_tests_sub_claude_md_has_file_table(): + """Each sub-CLAUDE.md uses the standard file table template.""" + failures = [] + for rel_path in EXPECTED_SUB_CLAUDE_MDS: + claude_md = TESTS_ROOT / rel_path + if not claude_md.is_file(): + failures.append(f"{rel_path}: file does not exist") + continue + content = claude_md.read_text() + if "| File | Purpose |" not in content: + failures.append(f"{rel_path}: missing '| File | Purpose |' table header") + assert not failures, "tests/ sub-CLAUDE.md template violations:\n" + "\n".join(failures) + + +def test_tests_sub_claude_md_no_main_claude_md_duplication(): + """Sub-CLAUDE.md files must not duplicate numbered sections from the main tests/CLAUDE.md.""" + numbered_section_re = re.compile(r"^## \*{0,2}\d+\.", re.MULTILINE) + failures = [] + for rel_path in EXPECTED_SUB_CLAUDE_MDS: + claude_md = TESTS_ROOT / rel_path + if not claude_md.is_file(): + failures.append(f"{rel_path}: file does not exist") + continue + content = claude_md.read_text() + match = numbered_section_re.search(content) + if match: + failures.append(f"{rel_path}: contains '{match.group()}' (main CLAUDE.md section)") + assert not failures, "tests/ sub-CLAUDE.md files duplicate main sections:\n" + "\n".join( + failures + ) + + +def test_top_level_tests_claude_md_references_all_subdirs(): + """The top-level tests/CLAUDE.md references each subdirectory's CLAUDE.md.""" + top_claude_md = TESTS_ROOT / "CLAUDE.md" + if not top_claude_md.is_file(): + pytest.fail("tests/CLAUDE.md does not exist") + content = top_claude_md.read_text() + failures = [] + for rel_path in EXPECTED_SUB_CLAUDE_MDS: + subdir = rel_path.split("/")[0] + marker = f"see {subdir}/CLAUDE.md" + if marker not in content: + failures.append(f"tests/CLAUDE.md missing reference: '{marker}'") + assert not failures, "Top-level tree missing sub-CLAUDE.md references:\n" + "\n".join(failures) + + +def test_top_level_tests_claude_md_no_per_file_subdir_listings(): + """The top-level tests/CLAUDE.md must not contain per-file listings for subdirectories.""" + top_claude_md = TESTS_ROOT / "CLAUDE.md" + if not top_claude_md.is_file(): + pytest.fail("tests/CLAUDE.md does not exist") + content = top_claude_md.read_text() + # Per-file listings look like: │ ├── test_*.py lines inside subdir blocks. + # After decomposition, subdirectory blocks should be single lines only. + # Check that no test_*.py filename appears indented under a subdirectory marker. + indented_test_file_re = re.compile(r"│\s+[├└]── test_\w+\.py") + matches = indented_test_file_re.findall(content) + assert not matches, ( + f"tests/CLAUDE.md still contains {len(matches)} per-file listings " + f"for subdirectories (should be in sub-CLAUDE.md files): {matches[:5]}" + ) diff --git a/tests/execution/CLAUDE.md b/tests/execution/CLAUDE.md new file mode 100644 index 0000000000..c703a43e4c --- /dev/null +++ b/tests/execution/CLAUDE.md @@ -0,0 +1,102 @@ +# execution/ + +Subprocess integration, headless session, process lifecycle, and session result tests. + +## Files + +| File | Purpose | +|------|---------| +| `__init__.py` | empty | +| `_merge_queue_helpers.py` | Merge-queue test helper factories — _make_watcher, _queue_state | +| `conftest.py` | Shared fixtures and helpers for tests/execution/ | +| `test_anomaly_detection.py` | Tests for post-hoc anomaly detection over ProcSnapshot data | +| `test_check_repo_merge_state.py` | Round-trip budget tests for fetch_repo_merge_state | +| `test_ci.py` | L1 unit tests for execution/ci.py — CIWatcher service | +| `test_ci_params.py` | Tests for CIRunScope query param composition and workflow scoping | +| `test_clone_guard.py` | Tests for clone contamination guard — detect and revert direct changes | +| `test_conftest_import_guard.py` | Structural guard: conftest.py must not import merge_queue at module level | +| `test_commands.py` | Tests for execution/commands.py — ClaudeInteractiveCmd / ClaudeHeadlessCmd builders | +| `test_db.py` | L1 unit tests for execution/db.py — SQL validation and authorizer | +| `test_diff_annotator.py` | Behavioral tests for execution/diff_annotator.py | +| `test_exit_classification.py` | Unit tests for classify_infra_exit and InfraExitCategory enum | +| `test_flag_contracts.py` | Contract tests for Claude CLI flags | +| `test_flush_completeness_guard.py` | Structural guard: every required-container field must appear in flush output | +| `test_flush_provider_integration.py` | Integration seam tests: provider fields forwarded from _execute_claude_headless to flush_session_log | +| `test_github.py` | L1 unit tests for execution/github.py | +| `test_github_api_tracking_http.py` | GitHub API tracking HTTP tests | +| `test_github_headers.py` | Tests for the shared github_headers helper and its adoption by all three classes | +| `test_headless_add_dirs.py` | Tests for run_headless_core multi-path --add-dir support (T-OVR-012..013) | +| `test_headless_core.py` | Tests for headless_runner.py extracted helpers | +| `test_headless_debug_logging.py` | Tests for debug logging instrumentation in headless.py | +| `test_headless_dispatch.py` | Tests for headless.py dispatch flow: food truck dispatch, pack injection, executor protocol | +| `test_headless_env_injection.py` | Phase 2 tests: AUTOSKILLIT_HEADLESS=1 env var injection in headless.py | +| `test_headless_env_scrub.py` | Launch-site env-scrub contract test for run_headless_core | +| `test_headless_ordering.py` | AST-based structural test for post-session operation ordering in headless.py | +| `test_headless_path_validation.py` | Tests for headless.py: _build_skill_result, path validation, synthesis, and contract gates | +| `test_headless_provider_fallback.py` | Tests for the provider fallback loop in _execute_claude_headless — STALE and BUDGET_EXHAUSTED trigger provider switch | +| `test_headless_provider_forwarding.py` | Tests verifying provider_extras and profile_name forwarding through the headless call chain | +| `test_headless_result_write_reconciliation.py` | Integration tests for EMPTY_OUTPUT + write-evidence reconciliation gate in _build_skill_result | +| `test_headless_synthesis.py` | Tests for headless.py synthesis helpers: output path extraction, validation, contamination | +| `test_headless_result.py` | Tests for _build_skill_result idle_stall lifespan_started propagation | +| `test_idle_output_env.py` | Group G (execution part): AUTOSKILLIT_IDLE_OUTPUT_TIMEOUT env variable injection tests | +| `test_linux_tracing.py` | Tests for Linux-only process tracing via psutil and /proc filesystem | +| `test_linux_tracing_pty_integration.py` | Integration test: PTY-wrapped command is traced at the workload level, not the wrapper | +| `test_loc_capture.py` | Tests for LoC capture helpers in execution.headless (T-GIT-1..T-GIT-6) | +| `test_merge_queue_classifier.py` | Tests for merge queue classifier: PendingCIGuard, InconclusiveBudget, ClassifierImmunity, VocabularyContract | +| `test_merge_queue_ejection.py` | Tests for merge queue ejection: RelatedCoverage, EjectionEnrichment, FetchRepoMergeStateRetry | +| `test_merge_queue_polling.py` | Tests for DefaultMergeQueueWatcher polling state machine | +| `test_normalize_subtype.py` | Unit tests for ClaudeSessionResult.normalize_subtype() normalization gate | +| `test_on_spawn_timing.py` | Tests for on_pid_resolved callback timing in run_managed_async (Group J) | +| `test_output_format_contract.py` | Contract tests binding output format to data availability | +| `test_pr_analysis.py` | Tests for execution/pr_analysis.py | +| `test_process_channel_b.py` | Integration tests for Channel B drain-race and COMPLETED pipeline adjudication | +| `test_process_debug_logging.py` | Tests for debug logging instrumentation in process.py | +| `test_process_env_boundary.py` | Boundary contract tests: MappingProxyType env coercion through subprocess execution | +| `test_process_deadline_extension.py` | Tests for _watch_child_activity coroutine and deadline extension behavior | +| `test_process_heartbeat.py` | Unit tests for _heartbeat, _has_active_api_connection, _has_active_child_processes, orphaned tool result detection | +| `test_process_idle_watchdog.py` | Tests for the stdout idle watchdog coroutine (_watch_stdout_idle) | +| `test_process_jsonl.py` | Tests for JSONL marker detection utilities | +| `test_process_kill.py` | Integration tests for process tree kill and async cancellation | +| `test_process_pty.py` | Tests for PTY wrapping and pipeline adjudication boundary tests | +| `test_process_race.py` | Unit tests for _process_race.py: resolve_termination and ChannelBStatus | +| `test_process_run.py` | Integration tests for normal subprocess run, stdin, timeout, temp I/O, and logging | +| `test_process_session_log_monitor.py` | Unit tests for _session_log_monitor and related session log monitoring behavior | +| `test_process_submodules.py` | Tests verifying process.py decomposition into focused sub-modules (P8-2) | +| `test_provider_outcome_container.py` | Tests for ProviderOutcome typed container construction — required fields and TypeError on omission | +| `test_push_trigger_applies.py` | Unit tests for _push_trigger_applies_to_branch and _has_merge_group_trigger | +| `test_quota_binding.py` | Tests for execution/quota.py — multi-window selection, per-window thresholds, cache refresh | +| `test_quota_http.py` | End-to-end HTTP tests for quota guard using api-simulator mock_http_server | +| `test_quota_io.py` | Tests for execution/quota.py — credential reading, cache I/O, dataclass validation | +| `test_quota_sleep.py` | Tests for execution/quota.py — check_and_sleep_if_needed, resets_at-None blocking | +| `test_readiness_helper_contract.py` | AST lint guard: no inline stderr/stdout readline loops used as subprocess readiness polls | +| `test_recording.py` | Tests for RecordingSubprocessRunner and related helpers | +| `test_resume_concurrency.py` | Tests for file lock preventing concurrent resume of same session | +| `test_resume_prompt.py` | Tests for _build_resume_context and build_skill_session_cmd resume integration | +| `test_recording_sigterm.py` | Integration test: autoskillit serve subprocess receives SIGTERM and writes scenario.json | +| `test_recording_sigterm_early_term.py` | Edge case: SIGTERM sent to subprocess before readiness sentinel appears | +| `test_recording_skills.py` | Tests for _recording_skills snapshot/restore helpers | +| `test_remote_resolver.py` | Unit tests for execution.remote_resolver.resolve_remote_repo and resolve_remote_name | +| `test_session_adjudication_outcome.py` | Tests for _compute_outcome, content state evaluation, and session adjudication consistency | +| `test_session_adjudication_retry.py` | Tests for _compute_retry, _is_kill_anomaly, and related retry adjudication logic | +| `test_session_adjudication_success.py` | Tests for _compute_success adjudication logic | +| `test_session_content.py` | Tests for session content validation and token normalization (`_strip_markdown_from_tokens`) | +| `test_session_debug_logging.py` | Tests for debug logging instrumentation in session.py | +| `test_session_state_persistence.py` | Tests for persist_session_state, read_session_state, and clear_session_state | +| `test_session_log_fields.py` | Tests for flush_session_log field coverage: write warnings, kitchen/order IDs, crash exception, raw stdout, per-turn fields | +| `test_session_log_flush.py` | Tests for flush_session_log: directory structure, proc-trace, summary/index, resolve_log_dir, temporal fields | +| `test_session_log_integration.py` | Integration tests: full tracing pipeline (accumulation + flush) end-to-end | +| `test_session_log_retention.py` | Tests for recover_crashed_sessions and retention/campaign-protection logic | +| `test_session_model_peak_context.py` | Tests for peak_context and turn_count extraction from extract_token_usage | +| `test_session_parsing.py` | L1 unit tests for execution/session.py — token extraction, parsing, and SkillResult | +| `test_session_result.py` | L1 unit tests for ClaudeSessionResult and parse_session_result — result types and policies | +| `test_termination_action.py` | Unit tests for decide_termination_action — pure decision function | +| `test_termination_executor.py` | Integration tests for execute_termination_action | +| `test_testing.py` | L1 unit tests for execution/testing.py — pytest output parsing | +| `test_trace_target_resolver.py` | Tests for resolve_trace_target — descendant-walk and basename-match contract | +| `test_write_evidence.py` | Write evidence: multi-directory fs snapshot and write_watch_dirs plumbing | +| `test_write_evidence_invariants.py` | Write-evidence invariants: 'no work done' retry reasons must be overridden by write evidence | +| `test_zero_write_detection.py` | Contract: sessions expected to write must actually write (behavioral write-count gate) | + +## Architecture Notes + +`conftest.py` provides shared fixtures for the execution test suite. The headless tests are split across multiple files by concern (dispatch, synthesis, path validation, env injection, ordering) following the P1-F01 audit fix. diff --git a/tests/execution/_merge_queue_helpers.py b/tests/execution/_merge_queue_helpers.py new file mode 100644 index 0000000000..809b8a3c8b --- /dev/null +++ b/tests/execution/_merge_queue_helpers.py @@ -0,0 +1,40 @@ +"""Merge-queue test helper factories used by merge_queue test modules.""" + +from __future__ import annotations + +from datetime import datetime + +from autoskillit.execution.merge_queue import DefaultMergeQueueWatcher, PRFetchState + + +def _make_watcher() -> DefaultMergeQueueWatcher: + return DefaultMergeQueueWatcher(token=None) + + +def _queue_state( + *, + merged: bool = False, + state: str = "OPEN", + mergeable: str = "MERGEABLE", + merge_state_status: str = "CLEAN", + auto_merge_present: bool = False, + auto_merge_enabled_at: datetime | None = None, + pr_node_id: str = "PR_kwDO_test", + in_queue: bool = False, + queue_state: str | None = None, + checks_state: str | None = None, + merge_group_checks_state: str | None = None, +) -> PRFetchState: + return { + "merged": merged, + "state": state, + "mergeable": mergeable, + "merge_state_status": merge_state_status, + "auto_merge_present": auto_merge_present, + "auto_merge_enabled_at": auto_merge_enabled_at, + "pr_node_id": pr_node_id, + "in_queue": in_queue, + "queue_state": queue_state, + "checks_state": checks_state, + "merge_group_checks_state": merge_group_checks_state, + } diff --git a/tests/execution/conftest.py b/tests/execution/conftest.py index 921d173ec8..2d5ffe63fa 100644 --- a/tests/execution/conftest.py +++ b/tests/execution/conftest.py @@ -2,13 +2,113 @@ from __future__ import annotations +import json import pathlib import textwrap +from collections.abc import Callable +from pathlib import Path from typing import Any import pytest -from autoskillit.config.settings import LinuxTracingConfig +from autoskillit.core.types import SubprocessResult, TerminationReason +from autoskillit.execution.session import ClaudeSessionResult +from tests._helpers import make_tracing_config + + +def _success_session_json(result_text: str) -> str: + return json.dumps( + { + "type": "result", + "subtype": "success", + "result": result_text, + "session_id": "test-session", + "is_error": False, + } + ) + + +def _sr( + returncode=0, + stdout="", + stderr="", + termination=TerminationReason.NATURAL_EXIT, + session_id: str = "", + channel_b_session_id: str = "", +): + """Build a minimal SubprocessResult for _build_skill_result tests.""" + return SubprocessResult( + returncode, + stdout, + stderr, + termination, + pid=12345, + session_id=session_id, + channel_b_session_id=channel_b_session_id, + ) + + +WRITE_TOOL_LINE = json.dumps( + { + "type": "assistant", + "message": { + "content": [ + { + "type": "tool_use", + "name": "Write", + "id": "w1", + "input": {"file_path": "/worktree/src/foo.py"}, + } + ] + }, + } +) +EMPTY_OUTPUT_RESULT_LINE = json.dumps( + { + "type": "result", + "subtype": "empty_output", + "is_error": True, + "result": "", + "session_id": "", + } +) + + +def _make_tool_use_line(name: str, input_dict: dict) -> str: + return json.dumps( + { + "type": "assistant", + "message": { + "content": [{"type": "tool_use", "name": name, "id": "x", "input": input_dict}] + }, + } + ) + + +def _make_cc_jsonl_record( + *, + request_id: str = "", + timestamp: str = "", + content: list[dict] | None = None, + record_type: str = "assistant", +) -> str: + rec: dict[str, object] = {"type": record_type} + if request_id: + rec["requestId"] = request_id + if timestamp: + rec["timestamp"] = timestamp + if content is not None: + rec["message"] = {"content": content} + return json.dumps(rec) + + +def _make_thinking_block(text: str = "reasoning...") -> dict[str, str]: + return {"type": "thinking", "thinking": text} + + +def _make_tool_block(name: str) -> dict[str, object]: + return {"type": "tool_use", "name": name, "id": "x", "input": {}} + # Simulates Claude CLI process that writes a result line then hangs. # Used by test_process_channel_b.py and test_process_monitor.py. @@ -31,13 +131,163 @@ @pytest.fixture -def isolated_tracing_config(tmp_path: pathlib.Path) -> LinuxTracingConfig: +def isolated_tracing_config(tmp_path: pathlib.Path): """Pre-isolated LinuxTracingConfig for tracing tests. Always writes to a tmp_path subdir, never to the real /dev/shm. Use this fixture for all new tests that need a LinuxTracingConfig.""" shm = tmp_path / "shm" shm.mkdir(parents=True, exist_ok=True) - return LinuxTracingConfig(enabled=True, proc_interval=0.05, tmpfs_path=str(shm)) + return make_tracing_config(enabled=True, proc_interval=0.05, tmpfs_path=str(shm)) + + +def _snap( + *, + captured_at: str | None = "2026-03-03T12:00:00+00:00", + state: str = "sleeping", + vm_rss_kb: int = 100000, + oom_score: int = 50, + fd_count: int = 10, + fd_soft_limit: int = 1024, + sig_pnd: str = "0000000000000000", + sig_blk: str = "0000000000000000", + sig_cgt: str = "0000000000000000", + threads: int = 4, + wchan: str = "", + ctx_switches_voluntary: int = 500, + ctx_switches_involuntary: int = 20, + cpu_percent: float = 0.0, +) -> dict[str, object]: + d: dict[str, object] = { + "state": state, + "vm_rss_kb": vm_rss_kb, + "oom_score": oom_score, + "fd_count": fd_count, + "fd_soft_limit": fd_soft_limit, + "sig_pnd": sig_pnd, + "sig_blk": sig_blk, + "sig_cgt": sig_cgt, + "threads": threads, + "wchan": wchan, + "ctx_switches_voluntary": ctx_switches_voluntary, + "ctx_switches_involuntary": ctx_switches_involuntary, + "cpu_percent": cpu_percent, + } + if captured_at is not None: + d["captured_at"] = captured_at + return d + + +def _result_ndjson( + result_text: str = "done", + subtype: str = "success", + is_error: bool = False, + session_id: str = "s1", + errors: list | None = None, + usage: dict | None = None, +) -> str: + obj: dict = { + "type": "result", + "subtype": subtype, + "is_error": is_error, + "result": result_text, + "session_id": session_id, + "errors": errors or [], + } + if usage: + obj["usage"] = usage + return json.dumps(obj) + + +def _assistant_ndjson( + model: str = "claude-sonnet-4-6", + input_tokens: int = 100, + output_tokens: int = 50, + cache_create: int = 0, + cache_read: int = 0, +) -> str: + return json.dumps( + { + "type": "assistant", + "message": { + "model": model, + "usage": { + "input_tokens": input_tokens, + "output_tokens": output_tokens, + "cache_creation_input_tokens": cache_create, + "cache_read_input_tokens": cache_read, + }, + }, + } + ) + + +def _flush(tmp_path: Path, **overrides) -> None: + from autoskillit.core.types._type_results import ProviderOutcome + from autoskillit.core.types._type_results_execution import ( + RecipeIdentity, + SessionTelemetry, + ) + from autoskillit.execution.session_log import flush_session_log + + defaults: dict = { + "log_dir": str(tmp_path), + "cwd": "/home/test/project", + "session_id": "test-session-001", + "pid": 12345, + "skill_command": "/autoskillit:investigate some error", + "success": True, + "subtype": "completed", + "exit_code": 0, + "start_ts": "2026-03-03T12:00:00+00:00", + "proc_snapshots": [_snap(), _snap(), _snap()], + "dispatch_id": "", + "caller_session_id": "", + "github_api_log": None, + "token_usage": None, + "timing_seconds": None, + "audit_record": None, + "loc_insertions": 0, + "loc_deletions": 0, + } + defaults.update(overrides) + + # Extract telemetry kwargs and build SessionTelemetry before forwarding + _github_api_log = defaults.pop("github_api_log") + _session_id = defaults.get("session_id", "") + _api_usage = _github_api_log.drain(_session_id) if _github_api_log is not None else None + telemetry = SessionTelemetry( + token_usage=defaults.pop("token_usage"), + timing_seconds=defaults.pop("timing_seconds"), + audit_record=defaults.pop("audit_record"), + github_api_usage=_api_usage, + github_api_requests=_api_usage.get("total_requests", 0) if _api_usage else 0, + loc_insertions=defaults.pop("loc_insertions"), + loc_deletions=defaults.pop("loc_deletions"), + ) + + # Extract provider/recipe kwargs and build typed containers + _provider_used = defaults.pop("provider_used", "") + _provider_fallback = defaults.pop("provider_fallback", False) + _recipe_name = defaults.pop("recipe_name", "") + _recipe_content_hash = defaults.pop("recipe_content_hash", "") + _recipe_composite_hash = defaults.pop("recipe_composite_hash", "") + _recipe_version = defaults.pop("recipe_version", "") + provider_outcome = ProviderOutcome( + provider_used=_provider_used, + fallback_activated=_provider_fallback, + ) + recipe_identity = RecipeIdentity( + name=_recipe_name, + content_hash=_recipe_content_hash, + composite_hash=_recipe_composite_hash, + version=_recipe_version, + ) + flush_session_log( + **defaults, + telemetry=telemetry, + provider_outcome=provider_outcome, + recipe_identity=recipe_identity, + ) @pytest.fixture @@ -67,3 +317,31 @@ def merge_group_only_repo_state() -> dict[str, Any]: "rest_completed_runs": {"workflow_runs": []}, "rest_active_runs": {"workflow_runs": []}, } + + +def _make_success_session(result: str = "done") -> ClaudeSessionResult: + return ClaudeSessionResult( + subtype="success", + is_error=False, + result=result, + session_id="s1", + ) + + +@pytest.fixture +def make_session() -> Callable[..., ClaudeSessionResult]: + def _factory( + subtype: str = "success", + is_error: bool = False, + result: str = "", + assistant_messages: list[str] | None = None, + ) -> ClaudeSessionResult: + return ClaudeSessionResult( + subtype=subtype, + is_error=is_error, + result=result, + session_id="test-session", + assistant_messages=assistant_messages or [], + ) + + return _factory diff --git a/tests/execution/test_anomaly_detection.py b/tests/execution/test_anomaly_detection.py index ff7bdcd4d1..d2e84a8b98 100644 --- a/tests/execution/test_anomaly_detection.py +++ b/tests/execution/test_anomaly_detection.py @@ -2,6 +2,8 @@ from __future__ import annotations +from functools import partial + import pytest from autoskillit.execution.anomaly_detection import ( @@ -10,39 +12,11 @@ AnomalySeverity, detect_anomalies, ) +from tests.execution.conftest import _snap as _snap_full +pytestmark = [pytest.mark.layer("execution"), pytest.mark.small] -def _snap( - *, - state: str = "sleeping", - vm_rss_kb: int = 100000, - oom_score: int = 50, - fd_count: int = 10, - fd_soft_limit: int = 1024, - sig_pnd: str = "0000000000000000", - sig_blk: str = "0000000000000000", - sig_cgt: str = "0000000000000000", - threads: int = 4, - wchan: str = "", - ctx_switches_voluntary: int = 500, - ctx_switches_involuntary: int = 20, - cpu_percent: float = 0.0, -) -> dict[str, object]: - return { - "state": state, - "vm_rss_kb": vm_rss_kb, - "oom_score": oom_score, - "fd_count": fd_count, - "fd_soft_limit": fd_soft_limit, - "sig_pnd": sig_pnd, - "sig_blk": sig_blk, - "sig_cgt": sig_cgt, - "threads": threads, - "wchan": wchan, - "ctx_switches_voluntary": ctx_switches_voluntary, - "ctx_switches_involuntary": ctx_switches_involuntary, - "cpu_percent": cpu_percent, - } +_snap = partial(_snap_full, captured_at=None) def test_detect_oom_spike(): @@ -331,3 +305,98 @@ def test_identity_drift_kind_exists(): "AnomalyKind must have IDENTITY_DRIFT member for PTY wrapper drift detection" ) assert AnomalyKind.IDENTITY_DRIFT == "identity_drift" + + +# --------------------------------------------------------------------------- +# Outcome anomaly detection tests +# --------------------------------------------------------------------------- + + +def test_empty_result_with_tokens_kind_exists(): + """AnomalyKind.EMPTY_RESULT_WITH_TOKENS is defined with the correct string value.""" + assert AnomalyKind.EMPTY_RESULT_WITH_TOKENS == "empty_result_with_tokens" + + +def test_detect_outcome_anomalies_fires_for_empty_result_with_tokens(): + """output_tokens > 0 and subtype == 'empty_result' → EMPTY_RESULT_WITH_TOKENS anomaly.""" + from autoskillit.execution.anomaly_detection import detect_outcome_anomalies + + anomalies = detect_outcome_anomalies({"output_tokens": 945}, "empty_result") + kinds = [a["kind"] for a in anomalies] + assert "empty_result_with_tokens" in kinds + + +def test_detect_outcome_anomalies_no_fire_when_zero_tokens(): + """output_tokens == 0 must NOT fire even when subtype matches.""" + from autoskillit.execution.anomaly_detection import detect_outcome_anomalies + + assert detect_outcome_anomalies({"output_tokens": 0}, "empty_result") == [] + + +def test_detect_outcome_anomalies_no_fire_when_wrong_subtype(): + """output_tokens > 0 but subtype != 'empty_result' must NOT fire.""" + from autoskillit.execution.anomaly_detection import detect_outcome_anomalies + + assert detect_outcome_anomalies({"output_tokens": 100}, "completed") == [] + + +def test_detect_outcome_anomalies_record_structure(): + """Each outcome anomaly record has the required fields.""" + from autoskillit.execution.anomaly_detection import detect_outcome_anomalies + + anomalies = detect_outcome_anomalies({"output_tokens": 100}, "empty_result") + assert len(anomalies) == 1 + a = anomalies[0] + assert a["event"] == "anomaly" + assert "ts" in a + assert "kind" in a + assert "severity" in a + assert "detail" in a + assert a["detail"]["output_tokens"] == 100 + + +def test_thinking_only_final_turn_kind_exists(): + assert AnomalyKind.THINKING_ONLY_FINAL_TURN == "thinking_only_final_turn" + + +def test_detect_outcome_anomalies_thinking_only_fires(): + """THINKING_ONLY_FINAL_TURN anomaly fires when has_thinking_only_turn is True.""" + from autoskillit.execution.anomaly_detection import detect_outcome_anomalies + + anomalies = detect_outcome_anomalies( + {"output_tokens": 500}, "empty_result", has_thinking_only_turn=True + ) + kinds = [a["kind"] for a in anomalies] + assert "thinking_only_final_turn" in kinds + + +def test_detect_outcome_anomalies_thinking_only_not_empty_result_with_tokens(): + """When has_thinking_only_turn=True, EMPTY_RESULT_WITH_TOKENS is suppressed.""" + from autoskillit.execution.anomaly_detection import detect_outcome_anomalies + + anomalies = detect_outcome_anomalies( + {"output_tokens": 500}, "empty_result", has_thinking_only_turn=True + ) + kinds = [a["kind"] for a in anomalies] + assert "empty_result_with_tokens" not in kinds + + +def test_detect_outcome_anomalies_thinking_only_no_fire_wrong_subtype(): + """THINKING_ONLY_FINAL_TURN does not fire when subtype is not 'empty_result'.""" + from autoskillit.execution.anomaly_detection import detect_outcome_anomalies + + anomalies = detect_outcome_anomalies( + {"output_tokens": 100}, "success", has_thinking_only_turn=True + ) + assert anomalies == [] + + +def test_detect_outcome_anomalies_thinking_only_false_falls_back_to_empty_result(): + """When has_thinking_only_turn=False, EMPTY_RESULT_WITH_TOKENS still fires.""" + from autoskillit.execution.anomaly_detection import detect_outcome_anomalies + + anomalies = detect_outcome_anomalies( + {"output_tokens": 200}, "empty_result", has_thinking_only_turn=False + ) + kinds = [a["kind"] for a in anomalies] + assert "empty_result_with_tokens" in kinds diff --git a/tests/execution/test_check_repo_merge_state.py b/tests/execution/test_check_repo_merge_state.py index 4bef40b87c..18b1899952 100644 --- a/tests/execution/test_check_repo_merge_state.py +++ b/tests/execution/test_check_repo_merge_state.py @@ -7,10 +7,14 @@ from __future__ import annotations +import textwrap + import pytest from autoskillit.execution.merge_queue import fetch_repo_merge_state +pytestmark = [pytest.mark.layer("execution"), pytest.mark.small] + # Reminder: fetch_repo_merge_state now returns ci_event in addition to the # three boolean fields. Tests below check for the ci_event field explicitly. @@ -118,15 +122,14 @@ def test_repo_state_query_is_distinct_module_constant_from_pr_state_query(): async def test_check_repo_merge_state_returns_merge_group_as_ci_event( httpx_mock, merge_group_only_repo_state ): - """check_repo_merge_state must derive ci_event='merge_group' when the only - workflow trigger is merge_group. This capture is what ci_watch reads to avoid - the event='push' timeout on repos that only trigger on merge_group.""" + """merge_group-only repo: ci_event must be None (match-any via head_sha), not 'merge_group'. + merge_group_trigger remains True for queue routing.""" httpx_mock.add_response( url="https://api.github.com/graphql", json=merge_group_only_repo_state["graphql_response"], ) result = await fetch_repo_merge_state(owner="o", repo="r", branch="main", token=None) - assert result["ci_event"] == "merge_group" + assert result["ci_event"] is None assert result["merge_group_trigger"] is True @@ -155,3 +158,182 @@ async def test_check_repo_merge_state_returns_null_ci_event_for_no_trigger(httpx ) result = await fetch_repo_merge_state(owner="o", repo="r", branch="main", token=None) assert result["ci_event"] is None + + +def _make_repo_state_response(workflow_entries): + """Helper: build a mock graphql response with given workflow YAML entries.""" + return { + "data": { + "repository": { + "mergeQueue": None, + "autoMergeAllowed": True, + "object": { + "entries": [ + {"name": name, "object": {"text": text}} for name, text in workflow_entries + ] + }, + } + } + } + + +@pytest.mark.anyio +async def test_push_trigger_branch_filter_excludes_feature_branch(httpx_mock): + """Feature branch not in push.branches must not produce ci_event='push'.""" + workflow_text = textwrap.dedent("""\ + on: + push: + branches: + - main + - stable + pull_request: + branches: [main, develop, stable] + """) + httpx_mock.add_response( + url="https://api.github.com/graphql", + json=_make_repo_state_response([("tests.yml", workflow_text)]), + ) + result = await fetch_repo_merge_state( + owner="org", + repo="repo", + branch="feature/impl-20260401-123456", + token=None, + ) + assert result["ci_event"] is None + + +@pytest.mark.anyio +async def test_push_excluded_with_merge_group_present_returns_none(httpx_mock): + """When push excludes the branch but merge_group exists, ci_event must be None (match-any).""" + workflow_text = textwrap.dedent("""\ + on: + push: + branches: + - main + - stable + pull_request: + branches: [main, develop, stable] + merge_group: + """) + httpx_mock.add_response( + url="https://api.github.com/graphql", + json=_make_repo_state_response([("tests.yml", workflow_text)]), + ) + result = await fetch_repo_merge_state( + owner="org", + repo="repo", + branch="feature/impl-20260401-123456", + token=None, + ) + assert result["ci_event"] is None + assert result["merge_group_trigger"] is True + + +@pytest.mark.anyio +@pytest.mark.parametrize( + "workflow_text, branch, expected_ci_event", + [ + ("on: [push, merge_group]", "main", "push"), + ("on:\n push:\n branches: [main]\n merge_group:", "main", "push"), + ("on:\n push:\n branches: [main]\n merge_group:", "feature/x", None), + ("on: [merge_group]", "feature/x", None), + ("on: [pull_request]", "feature/x", None), + ("on:\n schedule:\n - cron: '0 0 * * *'", "main", None), + ("on: push", "any-branch", "push"), + ], + ids=[ + "push_and_mg_push_applies", + "push_branches_match_with_mg", + "push_excluded_mg_present", + "mg_only", + "pr_only", + "schedule_only", + "push_no_filter", + ], +) +async def test_ci_event_is_always_push_or_none( + httpx_mock, workflow_text, branch, expected_ci_event +): + """ci_event must only ever be 'push' or None — never 'merge_group' or any other value.""" + httpx_mock.add_response( + url="https://api.github.com/graphql", + json=_make_repo_state_response([("tests.yml", workflow_text)]), + ) + result = await fetch_repo_merge_state(owner="o", repo="r", branch=branch, token=None) + assert result["ci_event"] == expected_ci_event + + +@pytest.mark.anyio +async def test_merge_group_in_comment_is_not_detected(httpx_mock): + """A YAML comment containing 'merge_group' must not set merge_group_trigger.""" + workflow_text = textwrap.dedent("""\ + # merge_group is not enabled for this repo + on: + push: + branches: [main] + """) + httpx_mock.add_response( + url="https://api.github.com/graphql", + json=_make_repo_state_response([("tests.yml", workflow_text)]), + ) + result = await fetch_repo_merge_state(owner="o", repo="r", branch="main", token=None) + assert result["merge_group_trigger"] is False + + +@pytest.mark.anyio +async def test_push_trigger_no_branch_filter_applies_to_all_branches(httpx_mock): + """push trigger with no branches: filter → ci_event='push' for any branch.""" + workflow_text = "on:\n push:\n pull_request:\n branches: [main]\n" + httpx_mock.add_response( + url="https://api.github.com/graphql", + json=_make_repo_state_response([("tests.yml", workflow_text)]), + ) + result = await fetch_repo_merge_state( + owner="org", + repo="repo", + branch="feature/anything", + token=None, + ) + assert result["ci_event"] == "push" + + +@pytest.mark.anyio +async def test_push_trigger_branch_filter_matches_target_branch(httpx_mock): + """When queried branch IS in push.branches → ci_event='push'.""" + workflow_text = textwrap.dedent("""\ + on: + push: + branches: [main, stable] + """) + httpx_mock.add_response( + url="https://api.github.com/graphql", + json=_make_repo_state_response([("tests.yml", workflow_text)]), + ) + result = await fetch_repo_merge_state( + owner="org", + repo="repo", + branch="main", + token=None, + ) + assert result["ci_event"] == "push" + + +@pytest.mark.anyio +async def test_push_trigger_branches_ignore_allows_feature_branch(httpx_mock): + """branches-ignore: [main, stable] → push fires on feature branches → ci_event='push'.""" + workflow_text = textwrap.dedent("""\ + on: + push: + branches-ignore: [main, stable] + """) + httpx_mock.add_response( + url="https://api.github.com/graphql", + json=_make_repo_state_response([("tests.yml", workflow_text)]), + ) + result = await fetch_repo_merge_state( + owner="org", + repo="repo", + branch="feature/impl-20260401", + token=None, + ) + assert result["ci_event"] == "push" diff --git a/tests/execution/test_ci.py b/tests/execution/test_ci.py index 4b62e2ddc9..32b17355e7 100644 --- a/tests/execution/test_ci.py +++ b/tests/execution/test_ci.py @@ -2,9 +2,10 @@ from __future__ import annotations -from datetime import UTC, datetime +from datetime import UTC, datetime, timedelta from unittest.mock import AsyncMock, patch +import httpx import pytest from autoskillit.core import CIRunScope, CIWatcher @@ -13,6 +14,8 @@ _jittered_sleep, ) +pytestmark = [pytest.mark.layer("execution"), pytest.mark.small] + # --------------------------------------------------------------------------- # Helpers # --------------------------------------------------------------------------- @@ -53,10 +56,22 @@ def _jobs_response(*jobs: tuple[str, str]) -> dict: # --------------------------------------------------------------------------- -@pytest.mark.parametrize("attempt", range(10)) -def test_jittered_sleep_bounded(attempt: int) -> None: - val = _jittered_sleep(attempt) - assert 0 <= val <= 30 # cap is 30 +@pytest.mark.parametrize( + ("attempt", "expected_floor", "expected_ceiling"), + [ + (0, 5, 10), + (1, 10, 20), + (2, 15, 30), + (3, 15, 30), + (9, 15, 30), + ], +) +def test_jittered_sleep_bounded( + attempt: int, expected_floor: float, expected_ceiling: float +) -> None: + for _ in range(50): + val = _jittered_sleep(attempt) + assert expected_floor <= val <= expected_ceiling def test_jittered_sleep_variance(): @@ -120,7 +135,7 @@ async def test_lookback_finds_completed_failed_run(): @pytest.mark.anyio async def test_lookback_filters_by_head_sha(): - """When scope.head_sha is provided, the API filters server-side.""" + """Client-side scope.head_sha still filters via _validate_run_matches_scope.""" watcher = DefaultCIWatcher(token="tok") matching_run = _run(run_id=222, head_sha="abc123") watcher._fetch_completed_runs = AsyncMock( # type: ignore[method-assign] @@ -135,7 +150,6 @@ async def test_lookback_filters_by_head_sha(): timeout_seconds=60, ) assert result["run_id"] == 222 - # Verify scope.head_sha was passed to the API call call_kwargs = watcher._fetch_completed_runs.call_args assert call_kwargs[0][4].head_sha == "abc123" # positional arg: scope @@ -154,6 +168,33 @@ async def test_lookback_without_head_sha_matches_any(): assert result["conclusion"] == "success" +@pytest.mark.anyio +async def test_scope_mismatch_log_includes_found_shas(): + """ci_watcher_scope_mismatch warning includes expected_sha and found_shas.""" + watcher = DefaultCIWatcher(token="tok") + stale_run = _run(run_id=999, head_sha="old-sha", conclusion="success") + watcher._fetch_completed_runs = AsyncMock( # type: ignore[method-assign] + return_value=[stale_run] + ) + watcher._fetch_active_runs = AsyncMock(return_value=[]) # type: ignore[method-assign] + + import structlog.testing + + with structlog.testing.capture_logs() as cap: + await watcher.wait( + "feature-x", + repo="owner/repo", + scope=CIRunScope(head_sha="new-sha"), + timeout_seconds=1, + ) + + mismatch_events = [e for e in cap if e.get("event") == "ci_watcher_scope_mismatch"] + assert mismatch_events, "Expected ci_watcher_scope_mismatch warning to be logged" + evt = mismatch_events[0] + assert evt.get("expected_sha") == "new-sha" + assert "old-sha" in (evt.get("found_shas") or []) + + @pytest.mark.anyio async def test_wait_returns_no_runs_when_fetch_returns_empty(): """wait() returns no_runs or timed_out when _fetch_completed_runs returns []. @@ -224,6 +265,31 @@ async def test_no_runs_at_all_returns_no_runs(): assert result["failed_jobs"] == [] +@pytest.mark.anyio +async def test_no_runs_includes_diagnostic_fields(): + """no_runs return must include scope_used, branch, and poll_duration_s.""" + watcher = DefaultCIWatcher(token="tok") + watcher._fetch_completed_runs = AsyncMock(return_value=[]) # type: ignore[method-assign] + watcher._fetch_active_runs = AsyncMock(return_value=[]) # type: ignore[method-assign] + watcher._fetch_failed_jobs = AsyncMock(return_value=[]) # type: ignore[method-assign] + + with patch("autoskillit.execution.ci.asyncio.sleep", new_callable=AsyncMock): + result = await watcher.wait( + "feature-branch", + repo="owner/repo", + scope=CIRunScope(event="push", workflow="tests.yml"), + timeout_seconds=1, + ) + + assert result["conclusion"] == "no_runs" + assert "scope_used" in result, "no_runs must include scope_used for diagnostics" + assert result["scope_used"]["event"] == "push" + assert result["scope_used"]["workflow"] == "tests.yml" + assert result["branch"] == "feature-branch" + assert "poll_duration_s" in result + assert isinstance(result["poll_duration_s"], float) + + @pytest.mark.anyio async def test_timeout_exceeded(): watcher = DefaultCIWatcher(token="tok") @@ -242,6 +308,9 @@ async def test_timeout_exceeded(): assert result["run_id"] == 666 assert result["conclusion"] == "timed_out" assert result["failed_jobs"] == [] + assert result["run_status"] == "in_progress" + assert "still in progress" in result["hint"] + assert "666" in result["hint"] @pytest.mark.anyio @@ -271,9 +340,9 @@ async def test_exponential_backoff_with_jitter(): result = await watcher.wait("main", repo="owner/repo", timeout_seconds=600) assert result["conclusion"] == "success" - # All sleep durations should be bounded by [0, 30] + # All sleep durations should respect backoff band floors (min 5s) for d in sleep_durations: - assert 0 <= d <= 30 + assert 5 <= d <= 30 @pytest.mark.anyio @@ -405,3 +474,174 @@ async def test_wait_billing_error_surfaced_distinctly(httpx_mock): requests = httpx_mock.get_requests() assert len(requests) == 1 assert "/jobs" not in str(requests[0].url) + + +# --------------------------------------------------------------------------- +# Vocabulary contract +# --------------------------------------------------------------------------- + + +class TestCIVocabularyContract: + """FAILED_CONCLUSIONS and related sets must be declared as named constants, + and those constants must be consistent with each other.""" + + def test_failed_conclusions_constant_exists(self): + """FAILED_CONCLUSIONS must be exported as a module-level constant.""" + from autoskillit.execution import ci + + assert hasattr(ci, "FAILED_CONCLUSIONS") + assert isinstance(ci.FAILED_CONCLUSIONS, frozenset) + assert "failure" in ci.FAILED_CONCLUSIONS + + def test_known_ci_conclusions_constant_exists(self): + """KNOWN_CI_CONCLUSIONS must be exported and cover all values ci.py tests for.""" + from autoskillit.execution import ci + + assert hasattr(ci, "KNOWN_CI_CONCLUSIONS") + assert isinstance(ci.KNOWN_CI_CONCLUSIONS, frozenset) + assert "success" in ci.KNOWN_CI_CONCLUSIONS + + def test_failed_conclusions_subset_of_known(self): + """FAILED_CONCLUSIONS must be a subset of KNOWN_CI_CONCLUSIONS.""" + from autoskillit.execution.ci import FAILED_CONCLUSIONS, KNOWN_CI_CONCLUSIONS + + assert FAILED_CONCLUSIONS.issubset(KNOWN_CI_CONCLUSIONS), ( + f"FAILED_CONCLUSIONS contains values not in KNOWN_CI_CONCLUSIONS: " + f"{FAILED_CONCLUSIONS - KNOWN_CI_CONCLUSIONS}" + ) + + +# --------------------------------------------------------------------------- +# T6 — CI ETag caching returns cached body on 304 +# --------------------------------------------------------------------------- + + +@pytest.mark.anyio +async def test_ci_etag_returns_cached_on_304(httpx_mock): + """When server returns 304, _fetch_completed_runs returns previously cached body.""" + httpx_mock.add_response( + json=_runs_response(_run(run_id=42, conclusion="success")), + headers={"ETag": '"abc123"'}, + ) + httpx_mock.add_response(status_code=304, content=b"") + + watcher = DefaultCIWatcher(token="tok") + result1 = await watcher.wait("feature-x", repo="owner/repo", timeout_seconds=60) + result2 = await watcher.wait("feature-x", repo="owner/repo", timeout_seconds=60) + + assert result1["run_id"] == 42 + assert result2["run_id"] == 42 + assert result1["conclusion"] == "success" + assert result2["conclusion"] == "success" + + requests = httpx_mock.get_requests() + assert len(requests) == 2 + assert requests[1].headers.get("if-none-match") == '"abc123"' + + +# --------------------------------------------------------------------------- +# T7 — CI ETag stores new ETag on 200 +# --------------------------------------------------------------------------- + + +@pytest.mark.anyio +async def test_ci_etag_stores_on_200(httpx_mock): + """ETag from a 200 response is sent as If-None-Match on the next request.""" + httpx_mock.add_response( + json=_runs_response(_run(run_id=99, conclusion="success")), + headers={"ETag": '"abc123"'}, + ) + httpx_mock.add_response( + json=_runs_response(_run(run_id=99, conclusion="success")), + ) + watcher = DefaultCIWatcher(token="tok") + await watcher.wait("feature-x", repo="owner/repo", timeout_seconds=60) + await watcher.wait("feature-x", repo="owner/repo", timeout_seconds=60) + + requests = httpx_mock.get_requests() + assert len(requests) == 2 + assert requests[1].headers.get("if-none-match") == '"abc123"' + + +@pytest.mark.anyio +async def test_fetch_completed_runs_no_sha_rejects_stale(httpx_mock): + """When scope.head_sha is NOT set, stale runs are filtered out.""" + stale_time = (datetime.now(UTC) - timedelta(minutes=30)).isoformat() + httpx_mock.add_response( + json=_runs_response(_run(run_id=42, updated_at=stale_time)), + ) + watcher = DefaultCIWatcher(token="tok") + + async with httpx.AsyncClient() as client: + runs = await watcher._fetch_completed_runs( + client, + watcher._headers(), + "owner/repo", + "main", + scope=CIRunScope(), + cutoff_dt=datetime.now(UTC) - timedelta(seconds=120), + ) + assert len(runs) == 0 + + +@pytest.mark.anyio +async def test_phase2_recheck_uses_anchored_cutoff(): + """Phase 2 re-check uses anchored cutoff — clock advancing doesn't lose runs.""" + watcher = DefaultCIWatcher(token="tok") + # Phase 1: no completed runs + call_count = [0] + borderline_time = (datetime.now(UTC) - timedelta(seconds=119)).isoformat() + + async def mock_fetch_completed(client, headers, repo, branch, scope, cutoff_dt): + call_count[0] += 1 + if call_count[0] == 1: + return [] + return [_run(run_id=99, conclusion="success", updated_at=borderline_time)] + + watcher._fetch_completed_runs = mock_fetch_completed # type: ignore[method-assign] + watcher._fetch_active_runs = AsyncMock(return_value=[]) # type: ignore[method-assign] + + with patch("autoskillit.execution.ci.asyncio.sleep", new_callable=AsyncMock): + result = await watcher.wait( + "main", repo="owner/repo", timeout_seconds=60, lookback_seconds=120 + ) + + assert result["run_id"] == 99 + assert result["conclusion"] == "success" + + +@pytest.mark.anyio +async def test_phase2_recheck_finds_late_completing_run(): + """Phase 2 re-check finds a run that completed between Phase 1 and Phase 2.""" + watcher = DefaultCIWatcher(token="tok") + call_count = [0] + + async def mock_fetch_completed(client, headers, repo, branch, scope, cutoff_dt): + call_count[0] += 1 + if call_count[0] == 1: + return [] + return [_run(run_id=77, conclusion="failure")] + + watcher._fetch_completed_runs = mock_fetch_completed # type: ignore[method-assign] + watcher._fetch_active_runs = AsyncMock(return_value=[]) # type: ignore[method-assign] + watcher._fetch_failed_jobs = AsyncMock(return_value=["build"]) # type: ignore[method-assign] + + with patch("autoskillit.execution.ci.asyncio.sleep", new_callable=AsyncMock): + result = await watcher.wait("main", repo="owner/repo", timeout_seconds=60) + + assert result["run_id"] == 77 + assert result["conclusion"] == "failure" + assert "build" in result["failed_jobs"] + + +def test_known_ci_events_frozenset_exists() -> None: + from autoskillit.core.types._type_constants import KNOWN_CI_EVENTS + + assert isinstance(KNOWN_CI_EVENTS, frozenset) + assert len(KNOWN_CI_EVENTS) >= 4 + assert all(isinstance(e, str) for e in KNOWN_CI_EVENTS) + assert all(e == e.lower() for e in KNOWN_CI_EVENTS) + assert "push" in KNOWN_CI_EVENTS + assert "pull_request" in KNOWN_CI_EVENTS + assert "merge_group" in KNOWN_CI_EVENTS + assert "workflow_dispatch" in KNOWN_CI_EVENTS diff --git a/tests/execution/test_ci_params.py b/tests/execution/test_ci_params.py index 7a64ce3055..c838e81292 100644 --- a/tests/execution/test_ci_params.py +++ b/tests/execution/test_ci_params.py @@ -7,7 +7,7 @@ from __future__ import annotations -from datetime import UTC, datetime +from datetime import UTC, datetime, timedelta import httpx import pytest @@ -15,6 +15,8 @@ from autoskillit.core import CIRunScope from autoskillit.execution.ci import DefaultCIWatcher, _validate_run_matches_scope +pytestmark = [pytest.mark.layer("execution"), pytest.mark.small] + def _now() -> str: return datetime.now(UTC).isoformat() @@ -39,7 +41,7 @@ async def test_completed_runs_includes_workflow_id(httpx_mock): "owner/repo", "main", scope=CIRunScope(workflow="tests.yml"), - lookback_seconds=300, + cutoff_dt=datetime.now(UTC) - timedelta(seconds=300), ) req = httpx_mock.get_requests()[0] assert httpx.URL(str(req.url)).params["workflow_id"] == "tests.yml" @@ -59,7 +61,7 @@ async def test_completed_runs_omits_workflow_id_when_none(httpx_mock): "owner/repo", "main", scope=CIRunScope(), - lookback_seconds=300, + cutoff_dt=datetime.now(UTC) - timedelta(seconds=300), ) req = httpx_mock.get_requests()[0] assert "workflow_id" not in str(req.url) @@ -79,30 +81,51 @@ async def test_completed_runs_always_sends_branch(httpx_mock): "owner/repo", "main", scope=CIRunScope(), - lookback_seconds=300, + cutoff_dt=datetime.now(UTC) - timedelta(seconds=300), ) req = httpx_mock.get_requests()[0] assert httpx.URL(str(req.url)).params["branch"] == "main" @pytest.mark.anyio -async def test_completed_runs_sends_head_sha(httpx_mock): - """When scope.head_sha is set, head_sha must appear in API params.""" - import httpx - - httpx_mock.add_response(json={"workflow_runs": []}) +async def test_completed_runs_omits_head_sha_from_params(httpx_mock): + """_fetch_completed_runs must NOT include head_sha as a query param.""" + httpx_mock.add_response( + json={"workflow_runs": []}, + headers={"ETag": '"abc"'}, + ) watcher = DefaultCIWatcher(token="tok") async with httpx.AsyncClient() as client: await watcher._fetch_completed_runs( client, watcher._headers(), "owner/repo", - "main", - scope=CIRunScope(head_sha="abc123"), - lookback_seconds=300, + "feature-x", + CIRunScope(head_sha="abc123"), + cutoff_dt=None, + ) + req = httpx_mock.get_requests()[0] + assert "head_sha" not in req.url.params + + +@pytest.mark.anyio +async def test_active_runs_omits_head_sha_from_params(httpx_mock): + """_fetch_active_runs must NOT include head_sha as a query param.""" + httpx_mock.add_response( + json={"workflow_runs": []}, + headers={"ETag": '"xyz"'}, + ) + watcher = DefaultCIWatcher(token="tok") + async with httpx.AsyncClient() as client: + await watcher._fetch_active_runs( + client, + watcher._headers(), + "owner/repo", + "feature-x", + CIRunScope(head_sha="abc123"), ) req = httpx_mock.get_requests()[0] - assert httpx.URL(str(req.url)).params["head_sha"] == "abc123" + assert "head_sha" not in req.url.params # --------------------------------------------------------------------------- @@ -251,7 +274,7 @@ async def test_completed_runs_includes_event(httpx_mock): "owner/repo", "main", scope=CIRunScope(event="push"), - lookback_seconds=300, + cutoff_dt=datetime.now(UTC) - timedelta(seconds=300), ) req = httpx_mock.get_requests()[0] assert httpx.URL(str(req.url)).params["event"] == "push" @@ -269,7 +292,7 @@ async def test_completed_runs_omits_event_when_none(httpx_mock): "owner/repo", "main", scope=CIRunScope(), - lookback_seconds=300, + cutoff_dt=datetime.now(UTC) - timedelta(seconds=300), ) req = httpx_mock.get_requests()[0] assert "event" not in httpx.URL(str(req.url)).params diff --git a/tests/execution/test_clone_guard.py b/tests/execution/test_clone_guard.py index 55d853261f..63ef447bcf 100644 --- a/tests/execution/test_clone_guard.py +++ b/tests/execution/test_clone_guard.py @@ -22,7 +22,9 @@ ) from autoskillit.execution.headless import _build_skill_result from autoskillit.pipeline.audit import DefaultAuditLog -from tests.conftest import MockSubprocessRunner +from tests.fakes import MockSubprocessRunner + +pytestmark = [pytest.mark.layer("execution"), pytest.mark.small] def _git_result(stdout: str = "", returncode: int = 0) -> SubprocessResult: @@ -401,3 +403,104 @@ async def test_audit_log_records_contamination(tmp_path): assert "post_sha=def456" in record.stderr assert "files=2" in record.stderr assert "direct_commits=True" in record.stderr + + +# --------------------------------------------------------------------------- +# T18: readonly check fires on success +# --------------------------------------------------------------------------- +@pytest.mark.anyio +async def test_readonly_check_fires_on_success(tmp_path): + runner = MockSubprocessRunner() + runner.push(_git_result(stdout="abc123\n")) # detect: rev-parse (same sha) + runner.push(_git_result(stdout=" M dirty.py\n")) # detect: status (dirty) + runner.push(_git_result()) # revert: checkout -- . + runner.push(_git_result()) # revert: clean -fd --exclude + + snapshot = CloneSnapshot(head_sha="abc123") + skill_result = _make_skill_result(success=True, worktree_path=None) + + result, reverted = await check_and_revert_clone_contamination( + snapshot, + skill_result, + str(tmp_path), + runner, + None, + skill_command="/autoskillit:investigate foo", + readonly_skill=True, + ) + assert reverted + assert result is skill_result + + +# --------------------------------------------------------------------------- +# T19: selective revert uses checkout and clean --exclude +# --------------------------------------------------------------------------- +@pytest.mark.anyio +async def test_readonly_selective_revert_checkout_and_clean(tmp_path): + runner = MockSubprocessRunner() + + snapshot = CloneSnapshot(head_sha="abc123") + from autoskillit.execution.clone_guard import ContaminationReport + + report = ContaminationReport( + pre_sha="abc123", + post_sha="abc123", + uncommitted_files=[" M src/main.py"], + direct_commits=False, + reverted=False, + ) + result = await revert_contamination(snapshot, report, str(tmp_path), runner, selective=True) + assert result.reverted + cmds = [call[0] for call in runner.call_args_list] + assert ["git", "checkout", "--", "."] in cmds + assert ["git", "clean", "-fd", "--exclude=.autoskillit/"] in cmds + assert ["git", "reset", "--hard", "abc123"] not in cmds + + +# --------------------------------------------------------------------------- +# T20: selective revert with direct commits includes reset +# --------------------------------------------------------------------------- +@pytest.mark.anyio +async def test_readonly_selective_revert_with_commits(tmp_path): + runner = MockSubprocessRunner() + + snapshot = CloneSnapshot(head_sha="abc123") + from autoskillit.execution.clone_guard import ContaminationReport + + report = ContaminationReport( + pre_sha="abc123", + post_sha="def456", + uncommitted_files=[], + direct_commits=True, + reverted=False, + ) + result = await revert_contamination(snapshot, report, str(tmp_path), runner, selective=True) + assert result.reverted + cmds = [call[0] for call in runner.call_args_list] + assert ["git", "reset", "--hard", "abc123"] in cmds + assert ["git", "checkout", "--", "."] in cmds + assert ["git", "clean", "-fd", "--exclude=.autoskillit/"] in cmds + + +# --------------------------------------------------------------------------- +# T21: worktree skill still uses nuclear revert +# --------------------------------------------------------------------------- +@pytest.mark.anyio +async def test_worktree_skill_still_uses_nuclear_revert(tmp_path): + runner = MockSubprocessRunner() + + snapshot = CloneSnapshot(head_sha="abc123") + from autoskillit.execution.clone_guard import ContaminationReport + + report = ContaminationReport( + pre_sha="abc123", + post_sha="abc123", + uncommitted_files=[" M src/main.py"], + direct_commits=False, + reverted=False, + ) + result = await revert_contamination(snapshot, report, str(tmp_path), runner, selective=False) + assert result.reverted + cmds = [call[0] for call in runner.call_args_list] + assert ["git", "reset", "--hard", "abc123"] in cmds + assert ["git", "clean", "-fd"] in cmds diff --git a/tests/execution/test_commands.py b/tests/execution/test_commands.py index 26d43b8fdb..2b0016c5fa 100644 --- a/tests/execution/test_commands.py +++ b/tests/execution/test_commands.py @@ -6,17 +6,29 @@ import pytest -from autoskillit.core import ClaudeFlags +from autoskillit.core import ( + BareResume, + ClaudeFlags, + DirectInstall, + MarketplaceInstall, + NamedResume, + NoResume, + OutputFormat, +) from autoskillit.execution.commands import ( + _HEADLESS_EXCLUSIVE_VARS, _MAX_MCP_OUTPUT_TOKENS_VALUE, ClaudeHeadlessCmd, ClaudeInteractiveCmd, - build_full_headless_cmd, + build_food_truck_cmd, build_headless_cmd, build_headless_resume_cmd, build_interactive_cmd, + build_skill_session_cmd, ) +pytestmark = [pytest.mark.layer("execution"), pytest.mark.small] + class TestBuildInteractiveCmd: def test_returns_correct_type(self) -> None: @@ -68,18 +80,26 @@ def test_omits_prompt_when_initial_prompt_is_none(self) -> None: assert len(result.cmd) == 2 # REQ-CMD-001 - def test_resume_session_id_appended(self) -> None: - result = build_interactive_cmd(resume_session_id="abc123") + def test_named_resume_appends_session_id(self) -> None: + result = build_interactive_cmd(resume_spec=NamedResume(session_id="abc123")) assert "--resume" in result.cmd idx = result.cmd.index("--resume") assert result.cmd[idx + 1] == "abc123" - def test_no_resume_flag_when_none(self) -> None: - result = build_interactive_cmd() + def test_bare_resume_produces_bare_flag_no_id(self) -> None: + result = build_interactive_cmd(resume_spec=BareResume()) + assert "--resume" in result.cmd + idx = result.cmd.index("--resume") + assert idx == len(result.cmd) - 1 + + def test_no_resume_spec_emits_no_flag(self) -> None: + result = build_interactive_cmd(resume_spec=NoResume()) assert "--resume" not in result.cmd def test_resume_placed_before_initial_prompt(self) -> None: - result = build_interactive_cmd(resume_session_id="abc123", initial_prompt="hello") + result = build_interactive_cmd( + resume_spec=NamedResume(session_id="abc123"), initial_prompt="hello" + ) resume_idx = result.cmd.index("--resume") prompt_idx = result.cmd.index("hello") assert resume_idx < prompt_idx @@ -96,13 +116,13 @@ def test_caller_extras_override_baseline(self) -> None: class TestBuildInteractiveCmdExtended: - def test_accepts_plugin_dir(self, tmp_path: Path) -> None: - """build_interactive_cmd with plugin_dir includes --plugin-dir flag.""" - plugin_dir = Path(tmp_path) - result = build_interactive_cmd(plugin_dir=plugin_dir) + def test_accepts_plugin_source_direct_install(self, tmp_path: Path) -> None: + """build_interactive_cmd with DirectInstall includes --plugin-dir flag.""" + plugin_source = DirectInstall(plugin_dir=tmp_path) + result = build_interactive_cmd(plugin_source=plugin_source) assert "--plugin-dir" in result.cmd idx = result.cmd.index("--plugin-dir") - assert result.cmd[idx + 1] == str(plugin_dir) + assert result.cmd[idx + 1] == str(tmp_path) def test_accepts_add_dirs(self, tmp_path: Path) -> None: """build_interactive_cmd with add_dirs includes --add-dir for each entry.""" @@ -110,8 +130,13 @@ def test_accepts_add_dirs(self, tmp_path: Path) -> None: result = build_interactive_cmd(add_dirs=[d1, d2]) assert result.cmd.count("--add-dir") == 2 - def test_plugin_dir_none_omits_flag(self) -> None: - """build_interactive_cmd with plugin_dir=None does not emit --plugin-dir.""" + def test_marketplace_install_omits_plugin_dir_flag(self, tmp_path: Path) -> None: + """build_interactive_cmd with MarketplaceInstall does not emit --plugin-dir.""" + result = build_interactive_cmd(plugin_source=MarketplaceInstall(cache_path=tmp_path)) + assert "--plugin-dir" not in result.cmd + + def test_no_plugin_source_omits_plugin_dir_flag(self) -> None: + """build_interactive_cmd with no plugin_source does not emit --plugin-dir.""" result = build_interactive_cmd() assert "--plugin-dir" not in result.cmd @@ -133,13 +158,13 @@ def test_cook_uses_builder_output(self, tmp_path: Path) -> None: patch("autoskillit.workspace.DefaultSessionSkillManager", return_value=mock_mgr), patch("subprocess.run", return_value=MagicMock(returncode=0)) as mock_run, ): - import autoskillit.cli._cook as module + import autoskillit.cli.session._cook as module module.cook() actual_cmd = mock_run.call_args[0][0] expected_prefix = build_interactive_cmd( - plugin_dir=pkg_root(), add_dirs=[fake_skills_dir] + plugin_source=DirectInstall(plugin_dir=pkg_root()), add_dirs=[fake_skills_dir] ).cmd assert actual_cmd == expected_prefix @@ -206,34 +231,35 @@ def test_no_plugin_dir_by_default(self) -> None: result = build_headless_resume_cmd(resume_session_id="abc-123", prompt="Emit token") assert ClaudeFlags.PLUGIN_DIR not in result.cmd - def test_with_plugin_dir(self) -> None: + def test_with_plugin_source_direct_install(self) -> None: result = build_headless_resume_cmd( - resume_session_id="abc-123", prompt="Emit token", plugin_dir=Path("/tmp/plugin") + resume_session_id="abc-123", + prompt="Emit token", + plugin_source=DirectInstall(plugin_dir=Path("/tmp/plugin")), ) assert ClaudeFlags.PLUGIN_DIR in result.cmd idx = result.cmd.index(ClaudeFlags.PLUGIN_DIR) assert result.cmd[idx + 1] == "/tmp/plugin" -class TestBuildFullHeadlessCmd: +class TestBuildSkillSessionCmd: BASE = dict( cwd="/repo", completion_marker="DONE", model=None, - plugin_dir="/plugins", - output_format_value="stream-json", - output_format_required_flags=["--verbose"], + plugin_source=DirectInstall(plugin_dir=Path("/plugins")), + output_format=OutputFormat.STREAM_JSON, add_dirs=[], exit_after_stop_delay_ms=2000, ) def test_returns_claude_headless_cmd(self): - spec = build_full_headless_cmd("/investigate foo", **self.BASE) + spec = build_skill_session_cmd("/investigate foo", **self.BASE) assert isinstance(spec, ClaudeHeadlessCmd) def test_cmd_starts_with_claude_not_env(self): """Argv no longer carries a leading ['env', ...] prefix.""" - spec = build_full_headless_cmd("/investigate foo", **self.BASE) + spec = build_skill_session_cmd("/investigate foo", **self.BASE) assert spec.cmd[0] == "claude" assert "env" != spec.cmd[0] assert not any(tok.startswith("AUTOSKILLIT_HEADLESS=") for tok in spec.cmd) @@ -242,21 +268,21 @@ def test_cmd_starts_with_claude_not_env(self): def test_env_has_autoskillit_headless(self): """AUTOSKILLIT_HEADLESS=1 now lives on spec.env, not in argv.""" - spec = build_full_headless_cmd("/investigate foo", **self.BASE) + spec = build_skill_session_cmd("/investigate foo", **self.BASE) assert spec.env["AUTOSKILLIT_HEADLESS"] == "1" def test_env_has_exit_delay_when_positive(self): - spec = build_full_headless_cmd("/investigate foo", **self.BASE) + spec = build_skill_session_cmd("/investigate foo", **self.BASE) assert spec.env["CLAUDE_CODE_EXIT_AFTER_STOP_DELAY"] == "2000" def test_env_omits_exit_delay_when_zero(self): params = {**self.BASE, "exit_after_stop_delay_ms": 0} - spec = build_full_headless_cmd("/investigate foo", **params) + spec = build_skill_session_cmd("/investigate foo", **params) assert "CLAUDE_CODE_EXIT_AFTER_STOP_DELAY" not in spec.env def test_env_strips_sse_port(self, monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setenv("CLAUDE_CODE_SSE_PORT", "23270") - spec = build_full_headless_cmd("/investigate foo", **self.BASE) + spec = build_skill_session_cmd("/investigate foo", **self.BASE) assert "CLAUDE_CODE_SSE_PORT" not in spec.env def test_headless_exclusive_vars_stripped_from_host_env_exit_delay( @@ -265,7 +291,7 @@ def test_headless_exclusive_vars_stripped_from_host_env_exit_delay( """CLAUDE_CODE_EXIT_AFTER_STOP_DELAY in host env must be stripped even when ms=0.""" monkeypatch.setenv("CLAUDE_CODE_EXIT_AFTER_STOP_DELAY", "99999") params = {**self.BASE, "exit_after_stop_delay_ms": 0} - spec = build_full_headless_cmd("/investigate foo", **params) + spec = build_skill_session_cmd("/investigate foo", **params) assert "CLAUDE_CODE_EXIT_AFTER_STOP_DELAY" not in spec.env def test_headless_exclusive_vars_stripped_from_host_env_scenario_step( @@ -274,96 +300,127 @@ def test_headless_exclusive_vars_stripped_from_host_env_scenario_step( """SCENARIO_STEP_NAME in host env must be stripped even when no step name is given.""" monkeypatch.setenv("SCENARIO_STEP_NAME", "outer-step") params = {**self.BASE, "scenario_step_name": ""} - spec = build_full_headless_cmd("/investigate foo", **params) + spec = build_skill_session_cmd("/investigate foo", **params) assert "SCENARIO_STEP_NAME" not in spec.env def test_env_has_auto_connect_off(self): - spec = build_full_headless_cmd("/investigate foo", **self.BASE) + spec = build_skill_session_cmd("/investigate foo", **self.BASE) assert spec.env["CLAUDE_CODE_AUTO_CONNECT_IDE"] == "0" - def test_plugin_dir_present(self): - spec = build_full_headless_cmd("/investigate foo", **self.BASE) + def test_plugin_source_direct_install_present(self): + spec = build_skill_session_cmd("/investigate foo", **self.BASE) assert "--plugin-dir" in spec.cmd idx = spec.cmd.index("--plugin-dir") assert spec.cmd[idx + 1] == "/plugins" + def test_marketplace_install_omits_plugin_dir(self, tmp_path: Path): + params = {**self.BASE, "plugin_source": MarketplaceInstall(cache_path=tmp_path)} + spec = build_skill_session_cmd("/investigate foo", **params) + assert "--plugin-dir" not in spec.cmd + def test_output_format_present(self): - spec = build_full_headless_cmd("/investigate foo", **self.BASE) + spec = build_skill_session_cmd("/investigate foo", **self.BASE) assert "--output-format" in spec.cmd idx = spec.cmd.index("--output-format") assert spec.cmd[idx + 1] == "stream-json" def test_output_format_required_flags_appended(self): - spec = build_full_headless_cmd("/investigate foo", **self.BASE) + spec = build_skill_session_cmd("/investigate foo", **self.BASE) assert "--verbose" in spec.cmd def test_output_format_required_flags_not_duplicated(self): - """If a required flag is already present it must not be added twice.""" - params = {**self.BASE, "output_format_required_flags": ["--output-format"]} - spec = build_full_headless_cmd("/investigate foo", **params) - assert spec.cmd.count("--output-format") == 1 + """Required flags must not appear twice even if already present.""" + spec = build_skill_session_cmd("/investigate foo", **self.BASE) + assert spec.cmd.count("--verbose") == 1 def test_add_dirs_injected(self): from autoskillit.core import ValidatedAddDir d = ValidatedAddDir(path="/skills/custom") params = {**self.BASE, "add_dirs": [d]} - spec = build_full_headless_cmd("/investigate foo", **params) + spec = build_skill_session_cmd("/investigate foo", **params) assert "--add-dir" in spec.cmd idx = spec.cmd.index("--add-dir") assert spec.cmd[idx + 1] == "/skills/custom" def test_no_add_dirs_emits_no_flag(self): - spec = build_full_headless_cmd("/investigate foo", **self.BASE) + spec = build_skill_session_cmd("/investigate foo", **self.BASE) assert "--add-dir" not in spec.cmd def test_skill_prefix_injected(self): - """Slash commands must be prefixed with 'Use '.""" - spec = build_full_headless_cmd("/investigate foo", **self.BASE) + """Slash commands must be prefixed with 'Use the ... skill'.""" + spec = build_skill_session_cmd("/investigate foo", **self.BASE) cmd = spec.cmd prompt_idx = cmd.index("-p") + 1 if "-p" in cmd else cmd.index("--print") + 1 - assert cmd[prompt_idx].startswith("Use /investigate") + assert cmd[prompt_idx].startswith("Use the /investigate skill") def test_completion_marker_appended(self): """Completion directive must appear in the prompt.""" - spec = build_full_headless_cmd("/investigate foo", **self.BASE) + spec = build_skill_session_cmd("/investigate foo", **self.BASE) cmd = spec.cmd prompt_idx = cmd.index("-p") + 1 if "-p" in cmd else cmd.index("--print") + 1 assert "DONE" in cmd[prompt_idx] + def test_completion_marker_is_last_instruction_in_prompt(self): + """The completion marker value must appear in the final line of the assembled prompt.""" + params = { + **self.BASE, + "completion_marker": "%%ORDER_UP::abc12345%%", + "profile_name": "minimax", + } + spec = build_skill_session_cmd("/autoskillit:make-plan arg", **params) + prompt_idx = spec.cmd.index(ClaudeFlags.PRINT) + 1 + prompt = spec.cmd[prompt_idx] + last_block = prompt.split("\n\n")[-1] + assert "%%ORDER_UP::abc12345%%" in last_block + + def test_completion_reminder_follows_efficiency_directive(self): + """The end-of-prompt marker reminder must be the last directive.""" + params = { + **self.BASE, + "completion_marker": "%%ORDER_UP::abc12345%%", + "profile_name": "minimax", + } + spec = build_skill_session_cmd("/autoskillit:make-plan arg", **params) + prompt_idx = spec.cmd.index(ClaudeFlags.PRINT) + 1 + prompt = spec.cmd[prompt_idx] + eff_pos = prompt.index("EFFICIENCY DIRECTIVE") + reminder_pos = prompt.index("Remember: end your final response with") + assert reminder_pos > eff_pos + def test_cwd_anchor_appended(self): """Working-directory anchor must appear in the prompt.""" - spec = build_full_headless_cmd("/investigate foo", **self.BASE) + spec = build_skill_session_cmd("/investigate foo", **self.BASE) cmd = spec.cmd prompt_idx = cmd.index("-p") + 1 if "-p" in cmd else cmd.index("--print") + 1 assert "/repo" in cmd[prompt_idx] def test_model_injected_when_provided(self): params = {**self.BASE, "model": "claude-opus-4-6"} - spec = build_full_headless_cmd("/investigate foo", **params) + spec = build_skill_session_cmd("/investigate foo", **params) assert "--model" in spec.cmd idx = spec.cmd.index("--model") assert spec.cmd[idx + 1] == "claude-opus-4-6" def test_model_omitted_when_none(self): - spec = build_full_headless_cmd("/investigate foo", **self.BASE) + spec = build_skill_session_cmd("/investigate foo", **self.BASE) assert "--model" not in spec.cmd def test_narration_suppression_directive_in_prompt(self): """EFFICIENCY DIRECTIVE must appear in the assembled prompt.""" - spec = build_full_headless_cmd("/investigate foo", **self.BASE) + spec = build_skill_session_cmd("/investigate foo", **self.BASE) cmd = spec.cmd prompt_idx = cmd.index("-p") + 1 if "-p" in cmd else cmd.index("--print") + 1 assert "EFFICIENCY DIRECTIVE" in cmd[prompt_idx] def test_env_has_max_mcp_output_tokens(self): """MAX_MCP_OUTPUT_TOKENS=50000 must be present in headless session env.""" - spec = build_full_headless_cmd("/investigate foo", **self.BASE) + spec = build_skill_session_cmd("/investigate foo", **self.BASE) assert spec.env["MAX_MCP_OUTPUT_TOKENS"] == _MAX_MCP_OUTPUT_TOKENS_VALUE def test_max_mcp_output_tokens_not_in_argv(self): """MAX_MCP_OUTPUT_TOKENS must live in spec.env, not in argv.""" - spec = build_full_headless_cmd("/investigate foo", **self.BASE) + spec = build_skill_session_cmd("/investigate foo", **self.BASE) assert not any(tok.startswith("MAX_MCP_OUTPUT_TOKENS=") for tok in spec.cmd) def test_headless_exclusive_vars_strips_host_max_mcp_output_tokens( @@ -371,9 +428,390 @@ def test_headless_exclusive_vars_strips_host_max_mcp_output_tokens( ) -> None: """Host-env MAX_MCP_OUTPUT_TOKENS must be stripped and replaced by the hardcoded value.""" monkeypatch.setenv("MAX_MCP_OUTPUT_TOKENS", "99999") - spec = build_full_headless_cmd("/investigate foo", **self.BASE) + spec = build_skill_session_cmd("/investigate foo", **self.BASE) + assert spec.env["MAX_MCP_OUTPUT_TOKENS"] == _MAX_MCP_OUTPUT_TOKENS_VALUE + + def test_env_has_session_type_skill(self): + spec = build_skill_session_cmd("/investigate foo", **self.BASE) + assert spec.env["AUTOSKILLIT_SESSION_TYPE"] == "skill" + + def test_env_overrides_ambient_session_type(self, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("AUTOSKILLIT_SESSION_TYPE", "franchise") + spec = build_skill_session_cmd("/investigate foo", **self.BASE) + assert spec.env["AUTOSKILLIT_SESSION_TYPE"] == "skill" + + def test_env_forwards_campaign_id(self, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("AUTOSKILLIT_CAMPAIGN_ID", "camp-42") + spec = build_skill_session_cmd("/investigate foo", **self.BASE) + assert spec.env["AUTOSKILLIT_CAMPAIGN_ID"] == "camp-42" + + def test_env_omits_campaign_id_when_absent(self, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.delenv("AUTOSKILLIT_CAMPAIGN_ID", raising=False) + spec = build_skill_session_cmd("/investigate foo", **self.BASE) + assert "AUTOSKILLIT_CAMPAIGN_ID" not in spec.env + + def test_env_forwards_kitchen_session_id(self, monkeypatch: pytest.MonkeyPatch) -> None: + """T32 — AUTOSKILLIT_KITCHEN_SESSION_ID forwarded into spec.env when set.""" + monkeypatch.setenv("AUTOSKILLIT_KITCHEN_SESSION_ID", "kit-77") + spec = build_skill_session_cmd("/investigate foo", **self.BASE) + assert spec.env["AUTOSKILLIT_KITCHEN_SESSION_ID"] == "kit-77" + + def test_env_omits_kitchen_session_id_when_absent( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + """T33 — AUTOSKILLIT_KITCHEN_SESSION_ID absent from spec.env when not set.""" + monkeypatch.delenv("AUTOSKILLIT_KITCHEN_SESSION_ID", raising=False) + spec = build_skill_session_cmd("/investigate foo", **self.BASE) + assert "AUTOSKILLIT_KITCHEN_SESSION_ID" not in spec.env + + def test_private_vars_scrubbed_except_explicit_forwards( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + monkeypatch.setenv("AUTOSKILLIT_CAMPAIGN_STATE_PATH", "/tmp/state") + monkeypatch.setenv("AUTOSKILLIT_PROJECT_DIR", "/tmp/proj") + monkeypatch.setenv("AUTOSKILLIT_FOOD_TRUCK_TOOL_TAGS", "kitchen") + spec = build_skill_session_cmd("/investigate foo", **self.BASE) + assert "AUTOSKILLIT_CAMPAIGN_STATE_PATH" not in spec.env + assert "AUTOSKILLIT_PROJECT_DIR" not in spec.env + assert "AUTOSKILLIT_FOOD_TRUCK_TOOL_TAGS" not in spec.env + + def test_provider_extras_injected_into_env(self) -> None: + spec = build_skill_session_cmd( + "/investigate foo", + **self.BASE, + provider_extras={ + "ANTHROPIC_BASE_URL": "https://custom.example.com", + "ANTHROPIC_API_KEY": "sk-test", + }, + ) + assert spec.env["ANTHROPIC_BASE_URL"] == "https://custom.example.com" + assert spec.env["ANTHROPIC_API_KEY"] == "sk-test" + + def test_provider_extras_cannot_override_session_type(self) -> None: + spec = build_skill_session_cmd( + "/investigate foo", + **self.BASE, + provider_extras={"AUTOSKILLIT_SESSION_TYPE": "franchise"}, + ) + assert spec.env["AUTOSKILLIT_SESSION_TYPE"] == "skill" + + def test_provider_extras_cannot_override_headless(self) -> None: + spec = build_skill_session_cmd( + "/investigate foo", + **self.BASE, + provider_extras={"AUTOSKILLIT_HEADLESS": "0"}, + ) + assert spec.env["AUTOSKILLIT_HEADLESS"] == "1" + + def test_host_anthropic_base_url_stripped_when_in_exclusive_vars( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + # Depends on P2-A1 (#1751) having added ANTHROPIC_BASE_URL to + # _HEADLESS_EXCLUSIVE_VARS. + monkeypatch.setenv("ANTHROPIC_BASE_URL", "https://host.example.com") + spec = build_skill_session_cmd("/investigate foo", **self.BASE) + assert "ANTHROPIC_BASE_URL" not in spec.env + + def test_provider_extras_none_changes_nothing(self) -> None: + baseline = build_skill_session_cmd("/investigate foo", **self.BASE) + spec = build_skill_session_cmd("/investigate foo", **self.BASE, provider_extras=None) + assert spec.env == baseline.env + + def test_profile_name_injects_provider_profile_env_var(self) -> None: + spec = build_skill_session_cmd("/investigate foo", **self.BASE, profile_name="minimax") + assert spec.env["AUTOSKILLIT_PROVIDER_PROFILE"] == "minimax" + + def test_empty_profile_name_omits_provider_profile(self) -> None: + spec = build_skill_session_cmd("/investigate foo", **self.BASE, profile_name="") + assert "AUTOSKILLIT_PROVIDER_PROFILE" not in spec.env + + def test_no_first_action_without_profile_name(self): + spec = build_skill_session_cmd("/investigate foo", **self.BASE) + prompt = spec.cmd[spec.cmd.index("-p") + 1] + assert "FIRST ACTION" not in prompt + assert "After loading" not in prompt + + def test_first_action_with_profile_name(self): + spec = build_skill_session_cmd( + "/autoskillit:investigate foo", **self.BASE, profile_name="minimax" + ) + prompt = spec.cmd[spec.cmd.index("-p") + 1] + assert "FIRST ACTION" in prompt + assert "After loading the skill instructions" in prompt + + def test_after_loading_only_with_profile_name(self): + spec = build_skill_session_cmd("/investigate foo", **self.BASE, profile_name="") + prompt = spec.cmd[spec.cmd.index("-p") + 1] + assert "After loading" not in prompt + + def test_no_after_loading_for_plain_prompt_with_profile(self): + spec = build_skill_session_cmd("Fix the bug", **self.BASE, profile_name="minimax") + prompt = spec.cmd[spec.cmd.index("-p") + 1] + assert "FIRST ACTION" not in prompt + assert "After loading" not in prompt + + +class TestBuildFoodTruckCmd: + BASE = dict( + orchestrator_prompt="You are an L3 food truck orchestrator...", + plugin_source=DirectInstall(plugin_dir=Path("/plugins")), + cwd="/repo", + completion_marker="%%L3_DONE::abc12345%%", + model=None, + env_extras=None, + output_format=OutputFormat.STREAM_JSON, + ) + + def test_returns_claude_headless_cmd(self): + spec = build_food_truck_cmd(**self.BASE) + assert isinstance(spec, ClaudeHeadlessCmd) + + def test_cmd_starts_with_claude(self): + spec = build_food_truck_cmd(**self.BASE) + assert spec.cmd[0] == "claude" + + def test_env_has_session_type_orchestrator(self): + spec = build_food_truck_cmd(**self.BASE) + assert spec.env["AUTOSKILLIT_SESSION_TYPE"] == "orchestrator" + + def test_env_has_autoskillit_headless(self): + spec = build_food_truck_cmd(**self.BASE) + assert spec.env["AUTOSKILLIT_HEADLESS"] == "1" + + def test_env_has_max_mcp_output_tokens(self): + spec = build_food_truck_cmd(**self.BASE) assert spec.env["MAX_MCP_OUTPUT_TOKENS"] == _MAX_MCP_OUTPUT_TOKENS_VALUE + def test_env_has_mcp_connection_nonblocking(self): + spec = build_food_truck_cmd(**self.BASE) + assert spec.env["MCP_CONNECTION_NONBLOCKING"] == "0" + + def test_does_not_call_ensure_skill_prefix(self): + """Prompt passed through verbatim — no 'Use ' prefix injected.""" + spec = build_food_truck_cmd(**self.BASE) + prompt_idx = spec.cmd.index(ClaudeFlags.PRINT) + 1 + prompt = spec.cmd[prompt_idx] + assert not prompt.startswith("Use ") + assert "You are an L3 food truck orchestrator" in prompt + + def test_tools_flag_restricts_to_ask_user_question(self): + spec = build_food_truck_cmd(**self.BASE) + assert ClaudeFlags.TOOLS in spec.cmd + idx = spec.cmd.index(ClaudeFlags.TOOLS) + assert spec.cmd[idx + 1] == "AskUserQuestion" + + def test_plugin_dir_present(self): + spec = build_food_truck_cmd(**self.BASE) + assert ClaudeFlags.PLUGIN_DIR in spec.cmd + idx = spec.cmd.index(ClaudeFlags.PLUGIN_DIR) + assert spec.cmd[idx + 1] == "/plugins" + + def test_build_food_truck_cmd_marketplace_uses_cache_path(self, tmp_path: Path): + """build_food_truck_cmd with MarketplaceInstall uses cache_path for --plugin-dir.""" + cache = tmp_path / "marketplace_cache" + cache.mkdir() + cmd = build_food_truck_cmd( + **{**self.BASE, "plugin_source": MarketplaceInstall(cache_path=cache)} + ) + idx = cmd.cmd.index("--plugin-dir") + assert cmd.cmd[idx + 1] == str(cache) + + def test_build_food_truck_cmd_direct_uses_plugin_dir(self, tmp_path: Path): + """build_food_truck_cmd with DirectInstall uses plugin_dir for --plugin-dir.""" + cmd = build_food_truck_cmd( + **{**self.BASE, "plugin_source": DirectInstall(plugin_dir=tmp_path)} + ) + idx = cmd.cmd.index("--plugin-dir") + assert cmd.cmd[idx + 1] == str(tmp_path) + + def test_output_format_present(self): + spec = build_food_truck_cmd(**self.BASE) + assert ClaudeFlags.OUTPUT_FORMAT in spec.cmd + idx = spec.cmd.index(ClaudeFlags.OUTPUT_FORMAT) + assert spec.cmd[idx + 1] == "stream-json" + + def test_output_format_required_flags_appended(self): + spec = build_food_truck_cmd(**self.BASE) + assert "--verbose" in spec.cmd + + def test_output_format_required_flags_not_duplicated(self): + spec = build_food_truck_cmd(**self.BASE) + assert spec.cmd.count("--verbose") == 1 + + def test_completion_marker_in_prompt(self): + spec = build_food_truck_cmd(**self.BASE) + prompt_idx = spec.cmd.index(ClaudeFlags.PRINT) + 1 + assert "%%L3_DONE::abc12345%%" in spec.cmd[prompt_idx] + + def test_cwd_anchor_in_prompt(self): + spec = build_food_truck_cmd(**self.BASE) + prompt_idx = spec.cmd.index(ClaudeFlags.PRINT) + 1 + assert "/repo" in spec.cmd[prompt_idx] + + def test_narration_suppression_in_prompt(self): + spec = build_food_truck_cmd(**self.BASE) + prompt_idx = spec.cmd.index(ClaudeFlags.PRINT) + 1 + assert "EFFICIENCY DIRECTIVE" in spec.cmd[prompt_idx] + + def test_env_extras_layered(self): + params = {**self.BASE, "env_extras": {"AUTOSKILLIT_CAMPAIGN_ID": "camp-1"}} + spec = build_food_truck_cmd(**params) + assert spec.env["AUTOSKILLIT_CAMPAIGN_ID"] == "camp-1" + + def test_env_extras_do_not_override_session_type(self): + params = {**self.BASE, "env_extras": {"AUTOSKILLIT_SESSION_TYPE": "skill"}} + spec = build_food_truck_cmd(**params) + assert spec.env["AUTOSKILLIT_SESSION_TYPE"] == "orchestrator" + + def test_env_overrides_ambient_session_type(self, monkeypatch: pytest.MonkeyPatch): + monkeypatch.setenv("AUTOSKILLIT_SESSION_TYPE", "franchise") + spec = build_food_truck_cmd(**self.BASE) + assert spec.env["AUTOSKILLIT_SESSION_TYPE"] == "orchestrator" + + def test_private_vars_scrubbed_from_host_env(self, monkeypatch: pytest.MonkeyPatch): + monkeypatch.setenv("AUTOSKILLIT_CAMPAIGN_STATE_PATH", "/tmp/state") + monkeypatch.setenv("AUTOSKILLIT_FOOD_TRUCK_TOOL_TAGS", "kitchen") + monkeypatch.setenv("AUTOSKILLIT_PROJECT_DIR", "/tmp/proj") + spec = build_food_truck_cmd(**self.BASE) + assert "AUTOSKILLIT_CAMPAIGN_STATE_PATH" not in spec.env + assert "AUTOSKILLIT_FOOD_TRUCK_TOOL_TAGS" not in spec.env + assert "AUTOSKILLIT_PROJECT_DIR" not in spec.env + + def test_model_injected_when_provided(self): + params = {**self.BASE, "model": "claude-opus-4-6"} + spec = build_food_truck_cmd(**params) + assert ClaudeFlags.MODEL in spec.cmd + idx = spec.cmd.index(ClaudeFlags.MODEL) + assert spec.cmd[idx + 1] == "claude-opus-4-6" + + def test_model_omitted_when_none(self): + spec = build_food_truck_cmd(**self.BASE) + assert ClaudeFlags.MODEL not in spec.cmd + + def test_env_strips_sse_port(self, monkeypatch: pytest.MonkeyPatch): + monkeypatch.setenv("CLAUDE_CODE_SSE_PORT", "23270") + spec = build_food_truck_cmd(**self.BASE) + assert "CLAUDE_CODE_SSE_PORT" not in spec.env + + def test_no_first_action_in_food_truck(self): + spec = build_food_truck_cmd( + orchestrator_prompt="Run the campaign", + plugin_source=DirectInstall(plugin_dir=Path("/plugins")), + cwd="/repo", + completion_marker="DONE", + ) + prompt = spec.cmd[spec.cmd.index("-p") + 1] + assert "FIRST ACTION" not in prompt + assert "After loading" not in prompt + + +class TestBuildFoodTruckCmdPackTags: + def test_env_extras_with_l3_tool_tags_passes_through(self): + """env_extras containing AUTOSKILLIT_FOOD_TRUCK_TOOL_TAGS reaches subprocess env.""" + spec = build_food_truck_cmd( + orchestrator_prompt="...", + plugin_source=DirectInstall(plugin_dir=Path("/plugins")), + cwd="/repo", + completion_marker="%%DONE%%", + env_extras={"AUTOSKILLIT_FOOD_TRUCK_TOOL_TAGS": "github,ci,clone,telemetry"}, + ) + assert spec.env["AUTOSKILLIT_FOOD_TRUCK_TOOL_TAGS"] == "github,ci,clone,telemetry" + + +class TestBuildFoodTruckCmdFeatureParity: + """Tests for features ported from build_skill_session_cmd (issue #1656).""" + + BASE = dict( + orchestrator_prompt="You are an L3 food truck orchestrator...", + plugin_source=DirectInstall(plugin_dir=Path("/plugins")), + cwd="/repo", + completion_marker="%%L3_DONE::abc12345%%", + model=None, + env_extras=None, + output_format=OutputFormat.STREAM_JSON, + exit_after_stop_delay_ms=0, + scenario_step_name="", + temp_dir_relpath=None, + allowed_write_prefix="", + ) + + def test_env_has_exit_delay_when_positive(self): + spec = build_food_truck_cmd(**{**self.BASE, "exit_after_stop_delay_ms": 2000}) + assert spec.env["CLAUDE_CODE_EXIT_AFTER_STOP_DELAY"] == "2000" + + def test_env_omits_exit_delay_when_zero(self): + spec = build_food_truck_cmd(**{**self.BASE, "exit_after_stop_delay_ms": 0}) + assert "CLAUDE_CODE_EXIT_AFTER_STOP_DELAY" not in spec.env + + def test_env_has_scenario_step_name_when_set(self): + spec = build_food_truck_cmd(**{**self.BASE, "scenario_step_name": "cook-recipe"}) + assert spec.env["SCENARIO_STEP_NAME"] == "cook-recipe" + + def test_env_omits_scenario_step_name_when_empty(self): + spec = build_food_truck_cmd(**self.BASE) + assert "SCENARIO_STEP_NAME" not in spec.env + + def test_env_forwards_kitchen_session_id(self, monkeypatch: pytest.MonkeyPatch): + monkeypatch.setenv("AUTOSKILLIT_KITCHEN_SESSION_ID", "ks-abc") + spec = build_food_truck_cmd(**self.BASE) + assert spec.env["AUTOSKILLIT_KITCHEN_SESSION_ID"] == "ks-abc" + + def test_env_omits_kitchen_session_id_when_absent(self, monkeypatch: pytest.MonkeyPatch): + monkeypatch.delenv("AUTOSKILLIT_KITCHEN_SESSION_ID", raising=False) + spec = build_food_truck_cmd(**self.BASE) + assert "AUTOSKILLIT_KITCHEN_SESSION_ID" not in spec.env + + def test_allowed_write_prefix_in_env(self): + spec = build_food_truck_cmd(**{**self.BASE, "allowed_write_prefix": "/tmp/foo/"}) + assert spec.env["AUTOSKILLIT_ALLOWED_WRITE_PREFIX"] == "/tmp/foo/" + + def test_allowed_write_prefix_absent_when_empty(self): + spec = build_food_truck_cmd(**{**self.BASE, "allowed_write_prefix": ""}) + assert "AUTOSKILLIT_ALLOWED_WRITE_PREFIX" not in spec.env + + def test_allowed_write_prefix_exclusive(self, monkeypatch: pytest.MonkeyPatch): + monkeypatch.setenv("AUTOSKILLIT_ALLOWED_WRITE_PREFIX", "old") + spec = build_food_truck_cmd(**{**self.BASE, "allowed_write_prefix": "new"}) + assert spec.env["AUTOSKILLIT_ALLOWED_WRITE_PREFIX"] == "new" + + def test_temp_dir_relpath_in_prompt(self): + spec = build_food_truck_cmd(**{**self.BASE, "temp_dir_relpath": ".autoskillit/temp"}) + prompt_text = spec.cmd[2] + assert ".autoskillit/temp" in prompt_text + + def test_headless_exclusive_vars_stripped_exit_delay(self, monkeypatch: pytest.MonkeyPatch): + monkeypatch.setenv("CLAUDE_CODE_EXIT_AFTER_STOP_DELAY", "99999") + spec = build_food_truck_cmd(**{**self.BASE, "exit_after_stop_delay_ms": 0}) + assert "CLAUDE_CODE_EXIT_AFTER_STOP_DELAY" not in spec.env + + def test_headless_exclusive_vars_stripped_scenario_step(self, monkeypatch: pytest.MonkeyPatch): + monkeypatch.setenv("SCENARIO_STEP_NAME", "outer-step") + spec = build_food_truck_cmd(**{**self.BASE, "scenario_step_name": ""}) + assert "SCENARIO_STEP_NAME" not in spec.env + + +class TestBuildFoodTruckCmdResume: + BASE = dict( + orchestrator_prompt="You are an L3 food truck orchestrator...", + plugin_source=DirectInstall(plugin_dir=Path("/plugins")), + cwd="/repo", + completion_marker="%%L3_DONE::abc12345%%", + ) + + def test_resume_session_id_adds_resume_flag(self): + spec = build_food_truck_cmd(**self.BASE, resume_session_id="abc-123") + assert "--resume" in spec.cmd + idx = spec.cmd.index("--resume") + assert spec.cmd[idx + 1] == "abc-123" + + def test_no_resume_session_id_omits_resume_flag(self): + spec = build_food_truck_cmd(**self.BASE) + assert "--resume" not in spec.cmd + + def test_none_resume_session_id_omits_resume_flag(self): + spec = build_food_truck_cmd(**self.BASE, resume_session_id=None) + assert "--resume" not in spec.cmd + def test_headless_exclusive_vars_contains_max_mcp_output_tokens() -> None: """MAX_MCP_OUTPUT_TOKENS must be in _HEADLESS_EXCLUSIVE_VARS.""" @@ -389,20 +827,279 @@ def test_headless_exclusive_vars_contains_max_mcp_output_tokens() -> None: "builder_call", [ lambda: build_interactive_cmd(), - lambda: build_full_headless_cmd( + lambda: build_skill_session_cmd( "/investigate foo", cwd="/tmp", completion_marker="%%DONE%%", model=None, - plugin_dir=None, - output_format_value="stream-json", + plugin_source=None, + output_format=OutputFormat.STREAM_JSON, ), lambda: build_headless_resume_cmd(resume_session_id="abc", prompt="Emit"), + lambda: build_food_truck_cmd( + orchestrator_prompt="You are an L3 orchestrator", + plugin_source=DirectInstall(plugin_dir=Path("/plugins")), + cwd="/tmp", + completion_marker="%%DONE%%", + ), ], - ids=["interactive", "full_headless", "headless_resume"], + ids=["interactive", "skill_headless", "headless_resume", "food_truck"], ) def test_all_session_builders_inject_max_mcp_output_tokens(builder_call) -> None: """Every session command builder must produce env with MAX_MCP_OUTPUT_TOKENS.""" spec = builder_call() assert "MAX_MCP_OUTPUT_TOKENS" in spec.env assert spec.env["MAX_MCP_OUTPUT_TOKENS"] == _MAX_MCP_OUTPUT_TOKENS_VALUE + + +def test_session_baseline_env_contains_mcp_connection_nonblocking() -> None: + from autoskillit.execution.commands import _SESSION_BASELINE_ENV + + assert "MCP_CONNECTION_NONBLOCKING" in _SESSION_BASELINE_ENV + assert _SESSION_BASELINE_ENV["MCP_CONNECTION_NONBLOCKING"] == "0" + + +def test_interactive_cmd_env_has_mcp_connection_nonblocking() -> None: + spec = build_interactive_cmd() + assert spec.env.get("MCP_CONNECTION_NONBLOCKING") == "0" + + +@pytest.mark.parametrize( + "builder_call", + [ + lambda: build_interactive_cmd(), + lambda: build_skill_session_cmd( + "/investigate foo", + cwd="/tmp", + completion_marker="%%DONE%%", + model=None, + plugin_source=None, + output_format=OutputFormat.STREAM_JSON, + ), + lambda: build_headless_resume_cmd(resume_session_id="abc", prompt="Emit"), + lambda: build_food_truck_cmd( + orchestrator_prompt="You are an L3 orchestrator", + plugin_source=DirectInstall(plugin_dir=Path("/plugins")), + cwd="/tmp", + completion_marker="%%DONE%%", + ), + ], + ids=["interactive", "skill_headless", "headless_resume", "food_truck"], +) +def test_all_session_builders_inject_mcp_connection_nonblocking(builder_call) -> None: + """Every session command builder must produce env with MCP_CONNECTION_NONBLOCKING=0.""" + spec = builder_call() + assert "MCP_CONNECTION_NONBLOCKING" in spec.env + assert spec.env["MCP_CONNECTION_NONBLOCKING"] == "0" + + +def test_launch_id_in_headless_exclusive_vars() -> None: + assert "AUTOSKILLIT_LAUNCH_ID" in _HEADLESS_EXCLUSIVE_VARS + + +def test_allowed_write_prefix_in_headless_exclusive_vars() -> None: + assert "AUTOSKILLIT_ALLOWED_WRITE_PREFIX" in _HEADLESS_EXCLUSIVE_VARS + + +def test_skill_name_in_headless_exclusive_vars() -> None: + assert "AUTOSKILLIT_SKILL_NAME" in _HEADLESS_EXCLUSIVE_VARS + + +def test_skill_cmd_includes_skill_name() -> None: + spec = build_skill_session_cmd( + "/autoskillit:planner-analyze some task", + cwd="/repo", + completion_marker="DONE", + model=None, + plugin_source=DirectInstall(plugin_dir=Path("/plugins")), + output_format=OutputFormat.STREAM_JSON, + add_dirs=[], + exit_after_stop_delay_ms=2000, + ) + assert spec.env["AUTOSKILLIT_SKILL_NAME"] == "planner-analyze" + + +def test_skill_cmd_skill_name_strips_namespace() -> None: + spec = build_skill_session_cmd( + "/autoskillit:investigate some issue", + cwd="/repo", + completion_marker="DONE", + model=None, + plugin_source=DirectInstall(plugin_dir=Path("/plugins")), + output_format=OutputFormat.STREAM_JSON, + add_dirs=[], + exit_after_stop_delay_ms=2000, + ) + assert spec.env["AUTOSKILLIT_SKILL_NAME"] == "investigate" + + +def test_skill_cmd_skill_name_empty_for_non_slash() -> None: + spec = build_skill_session_cmd( + "some prompt without slash", + cwd="/repo", + completion_marker="DONE", + model=None, + plugin_source=DirectInstall(plugin_dir=Path("/plugins")), + output_format=OutputFormat.STREAM_JSON, + add_dirs=[], + exit_after_stop_delay_ms=2000, + ) + assert spec.env["AUTOSKILLIT_SKILL_NAME"] == "" + + +class TestBuildSkillAllowedWritePrefix: + BASE = dict( + cwd="/repo", + completion_marker="DONE", + model=None, + plugin_source=DirectInstall(plugin_dir=Path("/plugins")), + output_format=OutputFormat.STREAM_JSON, + add_dirs=[], + exit_after_stop_delay_ms=2000, + ) + + def test_allowed_write_prefix_in_env(self): + spec = build_skill_session_cmd( + "/investigate foo", **self.BASE, allowed_write_prefix="/tmp/foo/" + ) + assert spec.env["AUTOSKILLIT_ALLOWED_WRITE_PREFIX"] == "/tmp/foo/" + + def test_allowed_write_prefix_absent_when_empty(self): + spec = build_skill_session_cmd("/investigate foo", **self.BASE, allowed_write_prefix="") + assert "AUTOSKILLIT_ALLOWED_WRITE_PREFIX" not in spec.env + + def test_allowed_write_prefix_exclusive(self, monkeypatch: pytest.MonkeyPatch): + monkeypatch.setenv("AUTOSKILLIT_ALLOWED_WRITE_PREFIX", "old") + spec = build_skill_session_cmd("/investigate foo", **self.BASE, allowed_write_prefix="new") + assert spec.env["AUTOSKILLIT_ALLOWED_WRITE_PREFIX"] == "new" + + +def test_provider_profile_in_headless_exclusive_vars() -> None: + """AUTOSKILLIT_PROVIDER_PROFILE must be headless-exclusive.""" + assert "AUTOSKILLIT_PROVIDER_PROFILE" in _HEADLESS_EXCLUSIVE_VARS + + +def test_anthropic_base_url_in_headless_exclusive_vars() -> None: + """ANTHROPIC_BASE_URL must be headless-exclusive.""" + assert "ANTHROPIC_BASE_URL" in _HEADLESS_EXCLUSIVE_VARS + + +def test_anthropic_api_key_in_headless_exclusive_vars() -> None: + """ANTHROPIC_API_KEY must be headless-exclusive.""" + assert "ANTHROPIC_API_KEY" in _HEADLESS_EXCLUSIVE_VARS + + +def test_anthropic_auth_token_in_headless_exclusive_vars() -> None: + """ANTHROPIC_AUTH_TOKEN must be headless-exclusive.""" + assert "ANTHROPIC_AUTH_TOKEN" in _HEADLESS_EXCLUSIVE_VARS + + +# --------------------------------------------------------------------------- +# T3: build_skill_session_cmd resume_session_id support +# --------------------------------------------------------------------------- + + +class TestBuildSkillSessionCmdResume: + BASE = dict( + cwd="/repo", + completion_marker="%%ORDER_UP::abc%%", + model=None, + plugin_source=DirectInstall(plugin_dir=Path("/plugins")), + output_format=OutputFormat.STREAM_JSON, + add_dirs=[], + ) + + def test_resume_flag_present_when_session_id_set(self): + """--resume is in the command when resume_session_id is set.""" + spec = build_skill_session_cmd( + "/implement fix the bug", **self.BASE, resume_session_id="sess-12345" + ) + assert "--resume" in spec.cmd + idx = spec.cmd.index("--resume") + assert spec.cmd[idx + 1] == "sess-12345" + + def test_no_resume_flag_when_empty(self): + """--resume is absent when resume_session_id is empty.""" + spec = build_skill_session_cmd("/implement fix the bug", **self.BASE) + assert "--resume" not in spec.cmd + + def test_resume_prompt_wraps_with_continuation_context(self): + """When resuming, the prompt includes continuation instructions.""" + spec = build_skill_session_cmd( + "/implement fix the bug", **self.BASE, resume_session_id="sess-12345" + ) + prompt = spec.cmd[spec.cmd.index("-p") + 1] + assert "resume" in prompt.lower() or "continue" in prompt.lower() + assert "%%ORDER_UP::abc%%" in prompt + + def test_resume_flag_appended_after_add_dirs(self): + """--resume flag is appended after all --add-dir entries.""" + from autoskillit.core import ValidatedAddDir + + spec = build_skill_session_cmd( + "/implement fix the bug", + **{**self.BASE, "add_dirs": [ValidatedAddDir(path="/extra")]}, + resume_session_id="sess-99", + ) + # --resume must appear after --add-dir in argv + assert "--resume" in spec.cmd + assert "--add-dir" in spec.cmd + resume_idx = spec.cmd.index("--resume") + add_dir_idx = spec.cmd.index("--add-dir") + assert resume_idx > add_dir_idx + + +class TestCompletionReminderPositionInvariant: + """Parametrized invariant: completion marker must appear in final prompt blocks. + + This guards against any future refactoring that buries the reminder in the + middle of the prompt, and verifies the invariant across all builder functions. + """ + + @pytest.mark.parametrize( + "builder", + [ + "skill_session_non_resume", + "skill_session_resume", + "food_truck_non_resume", + ], + ) + def test_marker_in_final_two_blocks(self, builder: str) -> None: + """Completion marker must appear in the last two \\n\\n-delimited blocks.""" + marker = "%%ORDER_UP::test123%%" + if builder == "skill_session_non_resume": + spec = build_skill_session_cmd( + "/autoskillit:make-plan arg", + cwd="/repo", + completion_marker=marker, + model=None, + plugin_source=DirectInstall(plugin_dir=Path("/plugins")), + output_format=OutputFormat.JSON, + profile_name="minimax", + ) + elif builder == "skill_session_resume": + spec = build_skill_session_cmd( + "/autoskillit:make-plan arg", + cwd="/repo", + completion_marker=marker, + model=None, + plugin_source=DirectInstall(plugin_dir=Path("/plugins")), + output_format=OutputFormat.JSON, + profile_name="minimax", + resume_session_id="sess-abc", + ) + else: # food_truck_non_resume + spec = build_food_truck_cmd( + orchestrator_prompt="Run the campaign", + plugin_source=DirectInstall(plugin_dir=Path("/plugins")), + cwd="/repo", + completion_marker=marker, + ) + prompt_idx = spec.cmd.index(ClaudeFlags.PRINT) + 1 + prompt = spec.cmd[prompt_idx] + blocks = prompt.split("\n\n") + last_two = "\n\n".join(blocks[-2:]) + assert marker in last_two, ( + f"[{builder}] Completion marker must appear in the last two prompt blocks. " + f"Last block: {blocks[-1][:100]!r}" + ) diff --git a/tests/execution/test_conftest_import_guard.py b/tests/execution/test_conftest_import_guard.py new file mode 100644 index 0000000000..810f1f391b --- /dev/null +++ b/tests/execution/test_conftest_import_guard.py @@ -0,0 +1,23 @@ +"""Structural guard: conftest.py must not import merge_queue at module level.""" + +from __future__ import annotations + +import ast +from pathlib import Path + +import pytest + +pytestmark = [pytest.mark.layer("execution"), pytest.mark.small] + + +def test_execution_conftest_has_no_merge_queue_imports(): + """conftest.py must not import merge_queue symbols at module level.""" + src = (Path(__file__).parent / "conftest.py").read_text() + tree = ast.parse(src) + for node in tree.body: + if isinstance(node, (ast.Import, ast.ImportFrom)): + module = getattr(node, "module", "") or "" + assert "merge_queue" not in module, ( + f"conftest.py imports from merge_queue at line {node.lineno}; " + "move these to tests/execution/_merge_queue_helpers.py" + ) diff --git a/tests/execution/test_db.py b/tests/execution/test_db.py index 93d8b60864..a84e1599b7 100644 --- a/tests/execution/test_db.py +++ b/tests/execution/test_db.py @@ -15,6 +15,8 @@ _validate_select_only, ) +pytestmark = [pytest.mark.layer("execution"), pytest.mark.small] + class TestValidateSelectOnly: """SQL validation: pure function _validate_select_only.""" diff --git a/tests/execution/test_diff_annotator.py b/tests/execution/test_diff_annotator.py index fdb30af082..972e2c083e 100644 --- a/tests/execution/test_diff_annotator.py +++ b/tests/execution/test_diff_annotator.py @@ -2,12 +2,19 @@ from __future__ import annotations +import pytest + from autoskillit.execution.diff_annotator import ( + DiffMetrics, annotate_diff, + compute_diff_metrics, filter_findings, parse_hunk_ranges, + select_review_agents, ) +pytestmark = [pytest.mark.layer("execution"), pytest.mark.small] + # --- parse_hunk_ranges --- @@ -240,3 +247,198 @@ def test_annotate_and_filter(self): assert len(result.unpostable) == 1 assert result.unpostable[0]["message"] == "stream offset" assert result.all_unpostable is False + + +# --- compute_diff_metrics --- + + +class TestComputeDiffMetrics: + def test_counts_added_lines(self): + diff = ( + "diff --git a/f.py b/f.py\n" + "--- a/f.py\n" + "+++ b/f.py\n" + "@@ -1,2 +1,4 @@\n" + " existing\n" + "+new_line_1\n" + "+new_line_2\n" + " more\n" + ) + metrics = compute_diff_metrics(diff) + assert metrics.added_lines == 2 + + def test_counts_removed_lines(self): + diff = ( + "diff --git a/f.py b/f.py\n" + "--- a/f.py\n" + "+++ b/f.py\n" + "@@ -1,3 +1,1 @@\n" + "-old_1\n" + "-old_2\n" + " kept\n" + ) + metrics = compute_diff_metrics(diff) + assert metrics.removed_lines == 2 + + def test_counts_changed_files(self): + diff = ( + "diff --git a/a.py b/a.py\n--- a/a.py\n+++ b/a.py\n" + "@@ -1,1 +1,2 @@\n+x\n" + "diff --git a/b.py b/b.py\n--- a/b.py\n+++ b/b.py\n" + "@@ -1,1 +1,2 @@\n+y\n" + "diff --git a/c.py b/c.py\n--- a/c.py\n+++ b/c.py\n" + "@@ -1,1 +1,2 @@\n+z\n" + ) + metrics = compute_diff_metrics(diff) + assert metrics.changed_files == 3 + assert set(metrics.file_paths) == {"a.py", "b.py", "c.py"} + + def test_extracts_file_paths(self): + diff = ( + "diff --git a/src/app.py b/src/app.py\n" + "--- a/src/app.py\n" + "+++ b/src/app.py\n" + "@@ -1,1 +1,2 @@\n" + "+x\n" + ) + metrics = compute_diff_metrics(diff) + assert metrics.file_paths == ["src/app.py"] + + def test_empty_diff(self): + metrics = compute_diff_metrics("") + assert metrics.added_lines == 0 + assert metrics.removed_lines == 0 + assert metrics.changed_files == 0 + assert metrics.file_paths == [] + + def test_ignores_diff_headers(self): + diff = "diff --git a/f.py b/f.py\n--- a/f.py\n+++ b/f.py\n@@ -1,1 +1,1 @@\n context\n" + metrics = compute_diff_metrics(diff) + assert metrics.added_lines == 0 + assert metrics.removed_lines == 0 + + +# --- select_review_agents --- + + +class TestSelectReviewAgents: + def test_small_diff_returns_core_agents(self): + metrics = DiffMetrics( + added_lines=50, + removed_lines=10, + changed_files=2, + file_paths=["src/foo.py", "tests/test_foo.py"], + ) + agents = select_review_agents(metrics) + assert agents == ["tests", "cohesion"] + + def test_small_diff_with_structural_adds_arch(self): + metrics = DiffMetrics( + added_lines=30, + removed_lines=5, + changed_files=2, + file_paths=["src/pkg/__init__.py", "src/pkg/mod.py"], + ) + agents = select_review_agents(metrics) + assert "arch" in agents + assert "tests" in agents + assert "cohesion" in agents + assert len(agents) == 3 + + def test_medium_diff_returns_all_agents(self): + metrics = DiffMetrics( + added_lines=200, removed_lines=50, changed_files=3, file_paths=["a.py", "b.py", "c.py"] + ) + agents = select_review_agents(metrics) + assert set(agents) == {"arch", "tests", "defense", "bugs", "cohesion", "slop"} + + def test_many_files_returns_all_agents(self): + fps = [f"f{i}.py" for i in range(5)] + metrics = DiffMetrics(added_lines=50, removed_lines=0, changed_files=5, file_paths=fps) + agents = select_review_agents(metrics) + assert set(agents) == {"arch", "tests", "defense", "bugs", "cohesion", "slop"} + + def test_custom_thresholds(self): + metrics = DiffMetrics( + added_lines=150, removed_lines=0, changed_files=3, file_paths=["a.py", "b.py", "c.py"] + ) + assert len(select_review_agents(metrics)) < 6 + agents = select_review_agents(metrics, loc_threshold=100) + assert set(agents) == {"arch", "tests", "defense", "bugs", "cohesion", "slop"} + + def test_deletion_regression_never_included(self): + metrics = DiffMetrics( + added_lines=500, + removed_lines=200, + changed_files=10, + file_paths=[f"f{i}.py" for i in range(10)], + ) + agents = select_review_agents(metrics) + assert "deletion_regression" not in agents + + +class TestExtractCodeRegion: + """Tests for extract_code_region() — deterministic code region extraction.""" + + ANNOTATED_DIFF = ( + "+++ b/src/app.py\n" + "@@ -38,10 +38,12 @@ def main():\n" + "[L38] existing_line_38\n" + "[L39] existing_line_39\n" + "[L40]+new_import\n" + "[L41]+another_import\n" + "[L42] existing_42\n" + "[L43] existing_43\n" + "[L44]+added_44\n" + "[L45] existing_45\n" + "+++ b/tests/test_app.py\n" + "@@ -1,3 +1,5 @@\n" + "[L1] import pytest\n" + "[L2]+from app import main\n" + "[L3]+from app import helper\n" + "[L4] \n" + ) + + def test_basic_extraction(self): + from autoskillit.execution.diff_annotator import extract_code_region + + region = extract_code_region(self.ANNOTATED_DIFF, "src/app.py", 42) + assert "[L42]" in region + assert "[L40]" in region + assert "[L38]" in region + + def test_file_not_in_diff(self): + from autoskillit.execution.diff_annotator import extract_code_region + + region = extract_code_region(self.ANNOTATED_DIFF, "src/missing.py", 10) + assert region == "" + + def test_line_outside_window(self): + from autoskillit.execution.diff_annotator import extract_code_region + + region = extract_code_region(self.ANNOTATED_DIFF, "src/app.py", 500) + assert region == "" + + def test_multiple_files(self): + from autoskillit.execution.diff_annotator import extract_code_region + + app_region = extract_code_region(self.ANNOTATED_DIFF, "src/app.py", 42) + test_region = extract_code_region(self.ANNOTATED_DIFF, "tests/test_app.py", 2) + assert "[L42]" in app_region + assert "tests/test_app.py" not in app_region + assert "[L2]" in test_region + assert "src/app.py" not in test_region + + def test_custom_context_lines(self): + from autoskillit.execution.diff_annotator import extract_code_region + + region = extract_code_region(self.ANNOTATED_DIFF, "src/app.py", 42, context_lines=2) + assert "[L42]" in region + assert "[L40]" in region + assert "[L44]" in region + assert "[L38]" not in region + + def test_empty_annotated_diff(self): + from autoskillit.execution.diff_annotator import extract_code_region + + assert extract_code_region("", "src/app.py", 42) == "" diff --git a/tests/execution/test_exit_classification.py b/tests/execution/test_exit_classification.py new file mode 100644 index 0000000000..2ab9e5d866 --- /dev/null +++ b/tests/execution/test_exit_classification.py @@ -0,0 +1,140 @@ +"""Tests for classify_infra_exit and InfraExitCategory (T1, T7).""" + +from __future__ import annotations + +import pytest + +from autoskillit.core.types import ( + ChannelConfirmation, + InfraExitCategory, + SubprocessResult, + TerminationReason, +) +from autoskillit.execution.session._exit_classification import classify_infra_exit +from autoskillit.execution.session._session_model import ClaudeSessionResult + +pytestmark = [pytest.mark.layer("execution"), pytest.mark.small] + + +def _sr( + returncode: int = 0, + stderr: str = "", + termination: TerminationReason = TerminationReason.NATURAL_EXIT, +) -> SubprocessResult: + return SubprocessResult( + returncode=returncode, + stdout="", + stderr=stderr, + termination=termination, + pid=12345, + channel_confirmation=ChannelConfirmation.UNMONITORED, + ) + + +class TestClassifyInfraExit: + def test_context_exhausted_from_jsonl_flag(self): + """jsonl_context_exhausted=True → CONTEXT_EXHAUSTED.""" + session = ClaudeSessionResult( + subtype="success", + is_error=True, + result="prompt is too long", + session_id="s1", + jsonl_context_exhausted=True, + ) + result = _sr(returncode=1, stderr="") + assert classify_infra_exit(session, result) == InfraExitCategory.CONTEXT_EXHAUSTED + + def test_api_error_overloaded_in_stderr(self): + """stderr contains 'overloaded' → API_ERROR.""" + session = ClaudeSessionResult( + subtype="empty_output", + is_error=True, + result="", + session_id="", + ) + result = _sr(returncode=1, stderr="Error: API is overloaded") + assert classify_infra_exit(session, result) == InfraExitCategory.API_ERROR + + def test_api_error_529_in_stderr(self): + """stderr contains HTTP 529 → API_ERROR.""" + session = ClaudeSessionResult( + subtype="empty_output", + is_error=True, + result="", + session_id="", + ) + result = _sr(returncode=1, stderr="HTTP Error 529: Service Overloaded") + assert classify_infra_exit(session, result) == InfraExitCategory.API_ERROR + + def test_api_error_connection_reset(self): + """stderr contains ECONNRESET → API_ERROR.""" + session = ClaudeSessionResult( + subtype="empty_output", + is_error=True, + result="", + session_id="", + ) + result = _sr(returncode=1, stderr="Error: read ECONNRESET") + assert classify_infra_exit(session, result) == InfraExitCategory.API_ERROR + + def test_process_killed_sigkill(self): + """returncode=-9 (SIGKILL) → PROCESS_KILLED.""" + session = ClaudeSessionResult( + subtype="empty_output", + is_error=True, + result="", + session_id="", + ) + result = _sr(returncode=-9, stderr="Killed") + assert classify_infra_exit(session, result) == InfraExitCategory.PROCESS_KILLED + + def test_process_killed_sigterm(self): + """returncode=-15 (SIGTERM, NOT from autoskillit kill) → PROCESS_KILLED.""" + session = ClaudeSessionResult( + subtype="empty_output", + is_error=True, + result="", + session_id="", + ) + result = _sr(returncode=-15, stderr="") + assert classify_infra_exit(session, result) == InfraExitCategory.PROCESS_KILLED + + def test_completed_success(self): + """Normal success → COMPLETED.""" + session = ClaudeSessionResult( + subtype="success", + is_error=False, + result="Done.", + session_id="s1", + ) + result = _sr(returncode=0, stderr="") + assert classify_infra_exit(session, result) == InfraExitCategory.COMPLETED + + def test_completed_logical_failure(self): + """Agent failure (success=false, explicit error) → COMPLETED (not infra).""" + session = ClaudeSessionResult( + subtype="error", + is_error=True, + result="Could not find file", + session_id="s1", + ) + result = _sr(returncode=1, stderr="") + assert classify_infra_exit(session, result) == InfraExitCategory.COMPLETED + + def test_context_exhaustion_takes_precedence_over_api_error(self): + """Both signals present → CONTEXT_EXHAUSTED wins (more specific).""" + session = ClaudeSessionResult( + subtype="success", + is_error=True, + result="prompt is too long", + session_id="s1", + jsonl_context_exhausted=True, + ) + result = _sr(returncode=1, stderr="overloaded") + assert classify_infra_exit(session, result) == InfraExitCategory.CONTEXT_EXHAUSTED + + +@pytest.mark.parametrize("category", list(InfraExitCategory)) +def test_all_infra_categories_handled(category: InfraExitCategory) -> None: + """Every InfraExitCategory value has a distinct test above.""" + assert category.value in {"completed", "context_exhausted", "api_error", "process_killed"} diff --git a/tests/execution/test_flag_contracts.py b/tests/execution/test_flag_contracts.py index 8b5ba9251a..a7974b2239 100644 --- a/tests/execution/test_flag_contracts.py +++ b/tests/execution/test_flag_contracts.py @@ -7,12 +7,16 @@ one place, and all tests catch the ripple. """ +import pytest + from autoskillit.core import ClaudeFlags from autoskillit.execution.commands import ( build_headless_cmd, build_interactive_cmd, ) +pytestmark = [pytest.mark.layer("execution"), pytest.mark.small] + # --------------------------------------------------------------------------- # Part 1: Constant value contracts — each assertion is the ground truth. # Any typo or wrong value in ClaudeFlags.* fails here immediately. diff --git a/tests/execution/test_flush_completeness_guard.py b/tests/execution/test_flush_completeness_guard.py new file mode 100644 index 0000000000..5610a5a516 --- /dev/null +++ b/tests/execution/test_flush_completeness_guard.py @@ -0,0 +1,118 @@ +"""Structural guards preventing regression to bare-kwarg provider/recipe omission.""" + +from __future__ import annotations + +import inspect +import json + +import pytest + +from autoskillit.core.types._type_results import ProviderOutcome +from autoskillit.core.types._type_results_execution import RecipeIdentity + +pytestmark = [pytest.mark.layer("execution"), pytest.mark.small] + + +class TestFlushSignatureGuard: + """flush_session_log must not accept bare provider or recipe string kwargs.""" + + def test_no_bare_provider_used_kwarg(self): + from autoskillit.execution.session_log import flush_session_log + + sig = inspect.signature(flush_session_log) + assert "provider_used" not in sig.parameters + + def test_no_bare_provider_fallback_kwarg(self): + from autoskillit.execution.session_log import flush_session_log + + sig = inspect.signature(flush_session_log) + assert "provider_fallback" not in sig.parameters + + def test_no_bare_recipe_name_kwarg(self): + from autoskillit.execution.session_log import flush_session_log + + sig = inspect.signature(flush_session_log) + assert "recipe_name" not in sig.parameters + + def test_no_bare_recipe_content_hash_kwarg(self): + from autoskillit.execution.session_log import flush_session_log + + sig = inspect.signature(flush_session_log) + assert "recipe_content_hash" not in sig.parameters + + def test_provider_outcome_is_required_parameter(self): + from autoskillit.execution.session_log import flush_session_log + + sig = inspect.signature(flush_session_log) + assert "provider_outcome" in sig.parameters + param = sig.parameters["provider_outcome"] + assert param.default is inspect.Parameter.empty + + def test_recipe_identity_is_required_parameter(self): + from autoskillit.execution.session_log import flush_session_log + + sig = inspect.signature(flush_session_log) + assert "recipe_identity" in sig.parameters + param = sig.parameters["recipe_identity"] + assert param.default is inspect.Parameter.empty + + +class TestFlushOutputCompleteness: + """Every ProviderOutcome field must appear in flush output.""" + + def test_provider_outcome_fields_written_to_summary(self, tmp_path): + from autoskillit.core.types._type_results_execution import SessionTelemetry + from autoskillit.execution.session_log import flush_session_log + + outcome = ProviderOutcome(provider_used="test-provider", fallback_activated=True) + flush_session_log( + log_dir=str(tmp_path), + cwd="/tmp", + session_id="completeness-test-001", + pid=1, + skill_command="/test", + success=True, + subtype="completed", + exit_code=0, + start_ts="2026-05-05T00:00:00+00:00", + proc_snapshots=None, + provider_outcome=outcome, + recipe_identity=RecipeIdentity.empty(), + telemetry=SessionTelemetry.empty(), + ) + summary_path = tmp_path / "sessions" / "completeness-test-001" / "summary.json" + summary = json.loads(summary_path.read_text()) + assert summary["provider_used"] == "test-provider" + assert summary["provider_fallback"] is True + + def test_recipe_identity_fields_written_to_index(self, tmp_path): + from autoskillit.core.types._type_results_execution import SessionTelemetry + from autoskillit.execution.session_log import flush_session_log + + identity = RecipeIdentity( + name="my-recipe", + content_hash="abc123", + composite_hash="def456", + version="1.0", + ) + flush_session_log( + log_dir=str(tmp_path), + cwd="/tmp", + session_id="recipe-completeness-001", + pid=1, + skill_command="/test", + success=True, + subtype="completed", + exit_code=0, + start_ts="2026-05-05T00:00:00+00:00", + proc_snapshots=None, + provider_outcome=ProviderOutcome.none_used(), + recipe_identity=identity, + telemetry=SessionTelemetry.empty(), + ) + index_path = tmp_path / "sessions.jsonl" + entry = json.loads(index_path.read_text().strip()) + assert entry["recipe_name"] == "my-recipe" + assert entry["recipe_content_hash"] == "abc123" + assert entry["recipe_composite_hash"] == "def456" + assert entry["recipe_version"] == "1.0" diff --git a/tests/execution/test_flush_provider_integration.py b/tests/execution/test_flush_provider_integration.py new file mode 100644 index 0000000000..35038615f0 --- /dev/null +++ b/tests/execution/test_flush_provider_integration.py @@ -0,0 +1,248 @@ +"""Integration seam tests: provider fields forwarded from _execute_claude_headless to flush.""" + +from __future__ import annotations + +import pytest + +from autoskillit.core.types import RetryReason, SkillResult + +pytestmark = [pytest.mark.layer("execution"), pytest.mark.small] + + +_SUCCESS_RESULT = SkillResult( + success=True, + result="done", + session_id="s1", + subtype="success", + is_error=False, + exit_code=0, + needs_retry=False, + retry_reason=RetryReason.NONE, + stderr="", +) + + +def _patch_common(monkeypatch, tmp_path, skill_result, ctx): + import autoskillit.execution.session_log as _sl_mod + from autoskillit.execution.headless import PostSessionMetrics + from tests.execution.conftest import _sr + + _sub_result = _sr() + + async def fake_runner(cmd, **kwargs): # noqa: ARG001 + return _sub_result + + monkeypatch.setattr( + "autoskillit.execution.headless._build_skill_result", + lambda *a, **kw: skill_result, # noqa: ARG005 + ) + monkeypatch.setattr( + "autoskillit.execution.headless._compute_post_session_metrics", + lambda *a, **kw: PostSessionMetrics(0, 0, str(tmp_path)), # noqa: ARG005 + ) + monkeypatch.setattr( + "autoskillit.execution.headless._capture_git_head_sha", + lambda *a: "", # noqa: ARG005 + ) + monkeypatch.setattr( + "autoskillit.execution.headless.collect_version_snapshot", + lambda: {}, + ) + + flush_calls: list[dict] = [] + + def capture_flush(**kwargs): + flush_calls.append(kwargs) + + monkeypatch.setattr(_sl_mod, "flush_session_log", capture_flush) + return fake_runner, flush_calls + + +class TestProviderFieldsReachFlush: + """Verify provider fields are forwarded from _execute_claude_headless to flush_session_log.""" + + @pytest.mark.anyio + async def test_normal_path_provider_used_in_flush_kwargs( + self, minimal_ctx, tmp_path, monkeypatch + ): + from autoskillit.execution.commands import ClaudeHeadlessCmd + from autoskillit.execution.headless import _execute_claude_headless + + fake_runner, flush_calls = _patch_common( + monkeypatch, tmp_path, _SUCCESS_RESULT, minimal_ctx + ) + minimal_ctx.runner = fake_runner # type: ignore[assignment] + + await _execute_claude_headless( + ClaudeHeadlessCmd(cmd=["echo", "test"], env={}), + str(tmp_path), + minimal_ctx, + timeout=30.0, + stale_threshold=5.0, + provider_name="minimax", + step_name="implement", + ) + + assert len(flush_calls) == 1 + outcome = flush_calls[0]["provider_outcome"] + assert outcome.provider_used == "minimax" + assert outcome.fallback_activated is False + + @pytest.mark.anyio + async def test_normal_path_provider_fallback_in_flush_kwargs( + self, minimal_ctx, tmp_path, monkeypatch + ): + from autoskillit.core.types import RetryReason, SkillResult + from autoskillit.execution.commands import ClaudeHeadlessCmd + from autoskillit.execution.headless import _execute_claude_headless + + stale = SkillResult( + success=False, + result="", + session_id="s0", + subtype="stale", + is_error=False, + exit_code=1, + needs_retry=True, + retry_reason=RetryReason.STALE, + stderr="", + ) + results = [stale, _SUCCESS_RESULT] + call_count = [0] + + import autoskillit.execution.session_log as _sl_mod + from autoskillit.execution.headless import PostSessionMetrics + from tests.execution.conftest import _sr + + _sub_result = _sr() + + async def fake_runner(cmd, **kwargs): # noqa: ARG001 + return _sub_result + + def build_result(*a, **kw): # noqa: ARG001 + r = results[min(call_count[0], len(results) - 1)] + call_count[0] += 1 + return r + + monkeypatch.setattr("autoskillit.execution.headless._build_skill_result", build_result) + monkeypatch.setattr( + "autoskillit.execution.headless._compute_post_session_metrics", + lambda *a, **kw: PostSessionMetrics(0, 0, str(tmp_path)), # noqa: ARG005 + ) + monkeypatch.setattr( + "autoskillit.execution.headless._capture_git_head_sha", + lambda *a: "", # noqa: ARG005 + ) + monkeypatch.setattr( + "autoskillit.execution.headless.collect_version_snapshot", + lambda: {}, + ) + monkeypatch.setattr(minimal_ctx.config.providers, "provider_retry_limit", 2) + monkeypatch.setattr( + "autoskillit.execution.headless.is_feature_enabled", + lambda name, *a, **kw: name == "providers", # noqa: ARG005 + ) + + flush_calls: list[dict] = [] + monkeypatch.setattr(_sl_mod, "flush_session_log", lambda **kw: flush_calls.append(kw)) + + minimal_ctx.runner = fake_runner # type: ignore[assignment] + + result = await _execute_claude_headless( + ClaudeHeadlessCmd(cmd=["echo", "test"], env={}), + str(tmp_path), + minimal_ctx, + timeout=30.0, + stale_threshold=5.0, + provider_name="minimax", + provider_fallback_env={"ANTHROPIC_API_KEY": "sk-test"}, + provider_fallback_name="anthropic", + step_name="implement", + ) + + assert result.provider.fallback_activated is True + assert result.provider.provider_used == "anthropic" + assert len(flush_calls) == 1 + outcome = flush_calls[0]["provider_outcome"] + assert outcome.provider_used == "anthropic" + assert outcome.fallback_activated is True + + @pytest.mark.anyio + async def test_crash_path_provider_used_in_flush_kwargs( + self, minimal_ctx, tmp_path, monkeypatch + ): + from autoskillit.execution.commands import ClaudeHeadlessCmd + from autoskillit.execution.headless import _execute_claude_headless + + monkeypatch.setattr( + "autoskillit.execution.headless.collect_version_snapshot", + lambda: {}, + ) + + flush_calls: list[dict] = [] + + def capture_flush(**kwargs): + flush_calls.append(kwargs) + + monkeypatch.setattr("autoskillit.execution.flush_session_log", capture_flush) + + async def raising_runner(cmd, **kwargs): # noqa: ARG001 + raise RuntimeError("disk crash") + + minimal_ctx.runner = raising_runner # type: ignore[assignment] + + result = await _execute_claude_headless( + ClaudeHeadlessCmd(cmd=["echo", "test"], env={}), + str(tmp_path), + minimal_ctx, + timeout=30.0, + stale_threshold=5.0, + provider_name="minimax", + ) + + assert result.subtype == "crashed" + crashed_calls = [f for f in flush_calls if f.get("termination_reason") == "CRASHED"] + assert len(crashed_calls) == 1 + outcome = crashed_calls[0]["provider_outcome"] + assert outcome.provider_used == "minimax" + + @pytest.mark.anyio + async def test_cancel_path_provider_used_in_flush_kwargs( + self, minimal_ctx, tmp_path, monkeypatch + ): + import anyio + + from autoskillit.execution.commands import ClaudeHeadlessCmd + from autoskillit.execution.headless import _execute_claude_headless + + monkeypatch.setattr( + "autoskillit.execution.headless.collect_version_snapshot", + lambda: {}, + ) + + flush_calls: list[dict] = [] + + def capture_flush(**kwargs): + flush_calls.append(kwargs) + + monkeypatch.setattr("autoskillit.execution.flush_session_log", capture_flush) + + async def cancelling_runner(cmd, **kwargs): # noqa: ARG001 + raise anyio.get_cancelled_exc_class()() + + minimal_ctx.runner = cancelling_runner # type: ignore[assignment] + + with pytest.raises(anyio.get_cancelled_exc_class()): + await _execute_claude_headless( + ClaudeHeadlessCmd(cmd=["echo", "test"], env={}), + str(tmp_path), + minimal_ctx, + timeout=30.0, + stale_threshold=5.0, + provider_name="openai", + ) + + cancelled_calls = [f for f in flush_calls if f.get("termination_reason") == "CANCELLED"] + assert len(cancelled_calls) == 1 + outcome = cancelled_calls[0]["provider_outcome"] + assert outcome.provider_used == "openai" diff --git a/tests/execution/test_github.py b/tests/execution/test_github.py index 26f3689a67..5136be46c9 100644 --- a/tests/execution/test_github.py +++ b/tests/execution/test_github.py @@ -2,6 +2,11 @@ from __future__ import annotations +import asyncio +import json +import time +from unittest.mock import AsyncMock, patch + import httpx import pytest @@ -12,6 +17,8 @@ parse_merge_queue_response, ) +pytestmark = [pytest.mark.layer("execution"), pytest.mark.small] + # --------------------------------------------------------------------------- # _parse_issue_ref unit tests # --------------------------------------------------------------------------- @@ -316,60 +323,53 @@ async def test_create_issue_request_error(httpx_mock): # --------------------------------------------------------------------------- -# add_comment +# update_issue_body # --------------------------------------------------------------------------- -_COMMENT_JSON = { - "id": 99, - "html_url": "https://github.com/owner/repo/issues/7#issuecomment-99", +_UPDATE_BODY_JSON = { + "number": 7, + "html_url": "https://github.com/owner/repo/issues/7", + "body": "updated body", } @pytest.mark.anyio -async def test_add_comment_success(httpx_mock): +async def test_update_issue_body_success(httpx_mock): httpx_mock.add_response( - url="https://api.github.com/repos/owner/repo/issues/7/comments", - method="POST", - json=_COMMENT_JSON, + url="https://api.github.com/repos/owner/repo/issues/7", + method="PATCH", + json=_UPDATE_BODY_JSON, ) fetcher = DefaultGitHubFetcher(token="tok") - result = await fetcher.add_comment("owner", "repo", 7, "New occurrence details") + result = await fetcher.update_issue_body( + "owner", "repo", 7, "original body\n\n## New Occurrence\n\ndetails" + ) assert result["success"] is True - assert result["comment_id"] == 99 + assert result["issue_url"] == "https://github.com/owner/repo/issues/7" @pytest.mark.anyio -async def test_add_comment_http_error(httpx_mock): +async def test_update_issue_body_http_error(httpx_mock): httpx_mock.add_response( - url="https://api.github.com/repos/owner/repo/issues/7/comments", - method="POST", + url="https://api.github.com/repos/owner/repo/issues/7", + method="PATCH", status_code=404, json={"message": "Not Found"}, ) fetcher = DefaultGitHubFetcher(token="tok") - result = await fetcher.add_comment("owner", "repo", 7, "body") + result = await fetcher.update_issue_body("owner", "repo", 7, "new body") assert result["success"] is False assert "404" in result["error"] @pytest.mark.anyio -async def test_add_comment_request_error(httpx_mock): +async def test_update_issue_body_request_error(httpx_mock): httpx_mock.add_exception(httpx.ConnectError("down")) fetcher = DefaultGitHubFetcher(token="tok") - result = await fetcher.add_comment("owner", "repo", 7, "body") + result = await fetcher.update_issue_body("owner", "repo", 7, "new body") assert result["success"] is False -# --------------------------------------------------------------------------- -# Protocol conformance -# --------------------------------------------------------------------------- - - -def test_github_fetcher_protocol_includes_write_methods(): - fetcher = DefaultGitHubFetcher(token=None) - assert isinstance(fetcher, GitHubFetcher) - - # --------------------------------------------------------------------------- # DefaultGitHubFetcher — fetch_title # --------------------------------------------------------------------------- @@ -464,10 +464,6 @@ async def test_fetch_title_network_error(self, httpx_mock): assert result["success"] is False assert "error" in result - def test_protocol_conformance(self): - """DefaultGitHubFetcher satisfies GitHubFetcher protocol (has fetch_title).""" - assert isinstance(DefaultGitHubFetcher(), GitHubFetcher) - # --------------------------------------------------------------------------- # T2 — _parse_issue_ref source is in core, not github.py @@ -603,7 +599,7 @@ def test_default_github_fetcher_implements_full_protocol(): "fetch_issue", "search_issues", "create_issue", - "add_comment", + "update_issue_body", "add_labels", "remove_label", "ensure_label", @@ -750,6 +746,68 @@ def test_parse_merge_queue_response_bad_node_skips_not_drops_rest(): assert entries[1]["pr_number"] == 30 +# --------------------------------------------------------------------------- +# swap_labels +# --------------------------------------------------------------------------- + + +@pytest.mark.anyio +async def test_swap_labels_replaces_atomically(httpx_mock): + """swap_labels GETs current labels then PUTs the computed target set atomically.""" + httpx_mock.add_response( + url="https://api.github.com/repos/owner/repo/issues/42/labels", + method="GET", + json=[{"name": "in-progress"}, {"name": "bug"}], + ) + httpx_mock.add_response( + url="https://api.github.com/repos/owner/repo/issues/42/labels", + method="PUT", + json=[{"name": "bug"}, {"name": "staged"}], + ) + client = DefaultGitHubFetcher(token="tok") + + with patch("autoskillit.execution.github.asyncio.sleep", new_callable=AsyncMock): + result = await client.swap_labels( + "owner", "repo", 42, remove_labels=["in-progress"], add_labels=["staged"] + ) + + assert result["success"] is True + assert set(result["labels"]) == {"bug", "staged"} + + requests = httpx_mock.get_requests() + put_request = next(r for r in requests if r.method == "PUT") + body = json.loads(put_request.content) + assert set(body["labels"]) == {"bug", "staged"} + + +@pytest.mark.anyio +async def test_swap_labels_remove_only(httpx_mock): + """swap_labels with empty add_labels removes the label and leaves the rest.""" + httpx_mock.add_response( + url="https://api.github.com/repos/owner/repo/issues/42/labels", + method="GET", + json=[{"name": "in-progress"}, {"name": "bug"}], + ) + httpx_mock.add_response( + url="https://api.github.com/repos/owner/repo/issues/42/labels", + method="PUT", + json=[{"name": "bug"}], + ) + client = DefaultGitHubFetcher(token="tok") + + with patch("autoskillit.execution.github.asyncio.sleep", new_callable=AsyncMock): + result = await client.swap_labels( + "owner", "repo", 42, remove_labels=["in-progress"], add_labels=[] + ) + + assert result["success"] is True + + requests = httpx_mock.get_requests() + put_request = next(r for r in requests if r.method == "PUT") + body = json.loads(put_request.content) + assert body["labels"] == ["bug"] + + def test_parse_merge_queue_response_missing_position_sorts_last(): """Entries without a position key sort after entries with explicit positions.""" data = { @@ -777,3 +835,165 @@ def test_parse_merge_queue_response_missing_position_sorts_last(): assert len(entries) == 2 assert entries[0]["pr_number"] == 1 assert entries[1]["pr_number"] == 99 + + +# --------------------------------------------------------------------------- +# T1 — Mutating throttle enforces 1s gap +# --------------------------------------------------------------------------- + + +@pytest.mark.anyio +async def test_mutating_throttle_enforces_delay(httpx_mock): + """Two consecutive mutating calls sleep to enforce 1s gap.""" + httpx_mock.add_response( + url="https://api.github.com/repos/owner/repo/issues", + method="POST", + json={"number": 1, "html_url": "https://github.com/owner/repo/issues/1"}, + ) + httpx_mock.add_response( + url="https://api.github.com/repos/owner/repo/issues", + method="POST", + json={"number": 2, "html_url": "https://github.com/owner/repo/issues/2"}, + ) + fetcher = DefaultGitHubFetcher(token="test") + with patch("autoskillit.execution.github.asyncio.sleep", new_callable=AsyncMock) as mock_sleep: + await fetcher.create_issue("owner", "repo", "Title 1", "body") + await fetcher.create_issue("owner", "repo", "Title 2", "body") + + assert mock_sleep.call_count == 1 + sleep_duration = mock_sleep.call_args[0][0] + assert sleep_duration == pytest.approx(1.0, abs=0.1) + + +# --------------------------------------------------------------------------- +# T2 — Read-only methods bypass throttle +# --------------------------------------------------------------------------- + + +@pytest.mark.anyio +async def test_read_methods_bypass_throttle(httpx_mock): + """fetch_issue does not trigger throttle delay after a mutating call.""" + httpx_mock.add_response( + url="https://api.github.com/repos/owner/repo/issues", + method="POST", + json={"number": 1, "html_url": "https://github.com/owner/repo/issues/1"}, + ) + httpx_mock.add_response( + url="https://api.github.com/repos/owner/repo/issues/1", + json=_ISSUE_NO_COMMENTS_JSON, + ) + fetcher = DefaultGitHubFetcher(token="test") + with patch("autoskillit.execution.github.asyncio.sleep", new_callable=AsyncMock) as mock_sleep: + await fetcher.create_issue("owner", "repo", "Title", "body") + await fetcher.fetch_issue("owner/repo#1", include_comments=False) + + mock_sleep.assert_not_called() + + +# --------------------------------------------------------------------------- +# T3 — ensure_label cache hit skips API call +# --------------------------------------------------------------------------- + + +@pytest.mark.anyio +async def test_ensure_label_cache_hit(httpx_mock): + """Second ensure_label for same owner/repo/label returns cached result without API call.""" + httpx_mock.add_response( + url="https://api.github.com/repos/owner/repo/labels", + method="POST", + status_code=201, + json={"name": "bug", "color": "ededed"}, + ) + fetcher = DefaultGitHubFetcher(token="tok") + with patch("autoskillit.execution.github.asyncio.sleep", new_callable=AsyncMock): + result1 = await fetcher.ensure_label("owner", "repo", "bug") + result2 = await fetcher.ensure_label("owner", "repo", "bug") + + assert result1 == {"success": True, "created": True} + assert result2 == {"success": True, "created": False} + assert len(httpx_mock.get_requests()) == 1 + + +# --------------------------------------------------------------------------- +# T4 — ensure_label cache is scoped to owner/repo/label triple +# --------------------------------------------------------------------------- + + +@pytest.mark.anyio +async def test_ensure_label_cache_different_repos(httpx_mock): + """ensure_label for different repos hits API separately.""" + httpx_mock.add_response( + url="https://api.github.com/repos/owner/repo1/labels", + method="POST", + status_code=201, + json={"name": "bug", "color": "ededed"}, + ) + httpx_mock.add_response( + url="https://api.github.com/repos/owner/repo2/labels", + method="POST", + status_code=201, + json={"name": "bug", "color": "ededed"}, + ) + fetcher = DefaultGitHubFetcher(token="tok") + with patch("autoskillit.execution.github.asyncio.sleep", new_callable=AsyncMock): + result1 = await fetcher.ensure_label("owner", "repo1", "bug") + result2 = await fetcher.ensure_label("owner", "repo2", "bug") + + assert len(httpx_mock.get_requests()) == 2 + assert result1 == {"success": True, "created": True} + assert result2 == {"success": True, "created": True} + + +# --------------------------------------------------------------------------- +# T8 — Throttle lock serializes concurrent mutating calls +# --------------------------------------------------------------------------- + + +@pytest.mark.anyio +async def test_throttle_serializes_concurrent_mutating_calls(httpx_mock): + """Lock must block the second call until the first fully exits _throttle_mutating. + + Seeds _last_mutating_ts so both coroutines need to sleep, then uses a + recording side_effect that yields to the event loop inside the lock region. + The assertion verifies strict sequential ordering of sleeps: second sleep must + start only after the first sleep has ended — impossible without the lock. + """ + httpx_mock.add_response( + url="https://api.github.com/repos/owner/repo/issues/42/labels", + method="POST", + json=[{"name": "bug"}], + ) + httpx_mock.add_response( + url="https://api.github.com/repos/owner/repo/issues", + method="POST", + json={"number": 2, "html_url": "https://github.com/owner/repo/issues/2"}, + ) + fetcher = DefaultGitHubFetcher(token="test") + # Seed _last_mutating_ts so both coroutines will need to sleep, + # placing both in the lock-held region where ordering matters. + fetcher._last_mutating_ts = time.monotonic() + + events: list[str] = [] + # Capture the real sleep before the patch replaces it; asyncio.sleep is patched + # globally via the module attribute, so recording_sleep must use this reference + # directly to avoid infinite recursion when yielding to the event loop. + _real_sleep = asyncio.sleep + + async def recording_sleep(_delay: float) -> None: + events.append("sleep_start") + await _real_sleep(0) # yield to event loop while lock is held + events.append("sleep_end") + + with patch("autoskillit.execution.github.asyncio.sleep", side_effect=recording_sleep): + await asyncio.gather( + fetcher.add_labels("owner", "repo", 42, ["bug"]), + fetcher.create_issue("owner", "repo", "Title", "body"), + ) + + starts = [i for i, e in enumerate(events) if e == "sleep_start"] + ends = [i for i, e in enumerate(events) if e == "sleep_end"] + assert len(starts) == 2 and len(ends) == 2 + # Second sleep must start after the first sleep ends — only possible with the lock. + assert starts[1] > ends[0], ( + "Second sleep started before first sleep ended: lock is not serializing coroutines" + ) diff --git a/tests/execution/test_github_api_tracking_http.py b/tests/execution/test_github_api_tracking_http.py new file mode 100644 index 0000000000..f9fca70422 --- /dev/null +++ b/tests/execution/test_github_api_tracking_http.py @@ -0,0 +1,118 @@ +import sys + +import pytest + +pytestmark = [ + pytest.mark.skipif(sys.platform != "linux", reason="api-simulator Linux-only"), + pytest.mark.layer("execution"), + pytest.mark.anyio, + pytest.mark.medium, +] + +_api_sim_http = pytest.importorskip("api_simulator.http") +PyResponseSpec = _api_sim_http.MockResponseSpec + +from autoskillit.execution.github import ( # noqa: E402 + DefaultGitHubFetcher, + make_tracked_httpx_client, +) +from tests.fakes import InMemoryGitHubApiLog # noqa: E402 + + +@pytest.fixture +def _reset_mock(mock_http_server): + mock_http_server.reset() + yield + + +async def test_tracking_transport_records_request(mock_http_server, _reset_mock): + mock_http_server.register( + "GET", + "/repos/owner/repo/issues/1", + PyResponseSpec( + body={"number": 1, "title": "T", "body": "", "labels": [], "state": "open"}, + headers=[ + ("X-RateLimit-Remaining", "4900"), + ("X-RateLimit-Used", "100"), + ("X-RateLimit-Reset", "1714000000"), + ], + ), + ) + log = InMemoryGitHubApiLog() + fetcher = DefaultGitHubFetcher(token="test-token", tracker=log, base_url=mock_http_server.url) + await fetcher.fetch_issue("owner", "repo", 1) + assert len(log.httpx_calls) == 1 + call = log.httpx_calls[0] + assert call["path"] == "/repos/owner/repo/issues/1" + assert call["status_code"] == 200 + + +async def test_tracking_transport_captures_rate_limit_headers(mock_http_server, _reset_mock): + mock_http_server.register( + "GET", + "/repos/owner/repo/issues/1", + PyResponseSpec( + body={"number": 1, "title": "T", "body": "", "labels": [], "state": "open"}, + headers=[ + ("X-RateLimit-Remaining", "42"), + ("X-RateLimit-Used", "4958"), + ("X-RateLimit-Reset", "1714000000"), + ], + ), + ) + log = InMemoryGitHubApiLog() + fetcher = DefaultGitHubFetcher(token="test-token", tracker=log, base_url=mock_http_server.url) + await fetcher.fetch_issue("owner", "repo", 1) + assert len(log.httpx_calls) == 1 + assert log.httpx_calls[0]["rate_limit_remaining"] == 42 + + +async def test_tracking_transport_records_4xx_error(mock_http_server, _reset_mock): + mock_http_server.register( + "GET", + "/repos/owner/repo/issues/999", + PyResponseSpec(status=404, body={"message": "Not Found"}), + ) + log = InMemoryGitHubApiLog() + fetcher = DefaultGitHubFetcher(token="test-token", tracker=log, base_url=mock_http_server.url) + # fetch_issue never raises — 404 is returned as {"success": False, ...} + await fetcher.fetch_issue("owner", "repo", 999) + assert len(log.httpx_calls) == 1 + assert log.httpx_calls[0]["status_code"] == 404 + + +async def test_tracking_transport_records_latency(mock_http_server, _reset_mock): + mock_http_server.register( + "GET", + "/repos/owner/repo/issues/1", + PyResponseSpec( + body={"number": 1, "title": "T", "body": "", "labels": [], "state": "open"}, + delay_ms=50, + ), + ) + log = InMemoryGitHubApiLog() + fetcher = DefaultGitHubFetcher(token="test-token", tracker=log, base_url=mock_http_server.url) + await fetcher.fetch_issue("owner", "repo", 1) + assert len(log.httpx_calls) == 1 + assert log.httpx_calls[0]["latency_ms"] >= 50.0 + + +async def test_no_tracker_still_works(mock_http_server, _reset_mock): + mock_http_server.register( + "GET", + "/repos/owner/repo/issues/1", + PyResponseSpec( + body={"number": 1, "title": "T", "body": "", "labels": [], "state": "open"} + ), + ) + fetcher = DefaultGitHubFetcher(token="test-token", tracker=None, base_url=mock_http_server.url) + result = await fetcher.fetch_issue("owner", "repo", 1) + assert result is not None + + +async def test_make_tracked_httpx_client_without_tracker_is_normal_client(): + import httpx + + client = make_tracked_httpx_client(None, timeout=httpx.Timeout(10.0)) + assert isinstance(client, httpx.AsyncClient) + await client.aclose() diff --git a/tests/execution/test_github_headers.py b/tests/execution/test_github_headers.py index fb52a8d855..ee236973b9 100644 --- a/tests/execution/test_github_headers.py +++ b/tests/execution/test_github_headers.py @@ -11,6 +11,8 @@ from autoskillit.execution.github import github_headers from autoskillit.execution.merge_queue import DefaultMergeQueueWatcher +pytestmark = [pytest.mark.layer("execution"), pytest.mark.small] + # --------------------------------------------------------------------------- # github_headers — unit tests # --------------------------------------------------------------------------- diff --git a/tests/execution/test_headless_add_dirs.py b/tests/execution/test_headless_add_dirs.py index c1f0e7aec9..42a03fc757 100644 --- a/tests/execution/test_headless_add_dirs.py +++ b/tests/execution/test_headless_add_dirs.py @@ -1,4 +1,4 @@ -"""Tests for run_headless_core multi-path --add-dir support (T-OVR-012..014).""" +"""Tests for run_headless_core multi-path --add-dir support (T-OVR-012..013).""" from __future__ import annotations @@ -6,27 +6,11 @@ from autoskillit.core import ValidatedAddDir - -@pytest.fixture -def make_ctx(tmp_path): - from autoskillit.config import AutomationConfig - from autoskillit.server._factory import make_context - from tests.conftest import MockSubprocessRunner - - def factory(runner=None): - ctx = make_context( - AutomationConfig(), - runner=runner or MockSubprocessRunner(), - plugin_dir=str(tmp_path), - ) - ctx.config.linux_tracing.log_dir = str(tmp_path / "session_logs") - return ctx - - return factory +pytestmark = [pytest.mark.layer("execution"), pytest.mark.small] @pytest.mark.anyio -async def test_run_headless_core_no_add_dir_when_empty(make_ctx, tmp_path): +async def test_run_headless_core_no_add_dir_when_empty(minimal_ctx, tmp_path): """T-OVR-012: run_headless_core with empty add_dirs emits no --add-dir flags.""" from autoskillit.execution.headless import run_headless_core from tests.conftest import _make_result @@ -37,15 +21,15 @@ async def mock_runner(cmd, **kwargs): captured_cmd.extend(cmd) return _make_result() - ctx = make_ctx(runner=mock_runner) + minimal_ctx.runner = mock_runner proj = tmp_path / "proj" proj.mkdir() - await run_headless_core("/autoskillit:investigate foo", str(proj), ctx, add_dirs=()) + await run_headless_core("/autoskillit:investigate foo", str(proj), minimal_ctx, add_dirs=()) assert "--add-dir" not in captured_cmd @pytest.mark.anyio -async def test_run_headless_core_two_add_dirs(make_ctx, tmp_path): +async def test_run_headless_core_two_add_dirs(minimal_ctx, tmp_path): """T-OVR-013: run_headless_core with two ValidatedAddDir paths emits two --add-dir flags.""" from autoskillit.execution.headless import run_headless_core from tests.conftest import _make_result @@ -65,13 +49,13 @@ async def mock_runner(cmd, **kwargs): captured_cmd.extend(cmd) return _make_result() - ctx = make_ctx(runner=mock_runner) + minimal_ctx.runner = mock_runner proj = tmp_path / "proj" proj.mkdir() await run_headless_core( "/autoskillit:investigate foo", str(proj), - ctx, + minimal_ctx, add_dirs=[dir_a, dir_b], ) add_dir_positions = [i for i, x in enumerate(captured_cmd) if x == "--add-dir"] @@ -79,60 +63,3 @@ async def mock_runner(cmd, **kwargs): dirs_passed = [captured_cmd[i + 1] for i in add_dir_positions] assert str(tmp_path / "a") in dirs_passed assert str(tmp_path / "b") in dirs_passed - - -@pytest.mark.anyio -async def test_raw_skills_extended_excluded_from_run_skill_add_dirs(tool_ctx, monkeypatch): - """T-OVR-014: run_skill passes ephemeral session dir (not raw skills_extended/) as add_dirs.""" - from autoskillit.core import SkillResult - from autoskillit.server import _state - - captured: dict = {} - - class MockExecutor: - async def run(self, skill_command, cwd, *, add_dirs=(), **kwargs): - captured["add_dirs"] = add_dirs - captured["cwd"] = cwd - return SkillResult( - success=True, - result="ok", - session_id="", - subtype="success", - is_error=False, - exit_code=0, - needs_retry=False, - retry_reason="none", - stderr="", - token_usage=None, - ) - - tool_ctx.executor = MockExecutor() - monkeypatch.setattr(_state, "_ctx", tool_ctx) - - from autoskillit.server.tools_execution import run_skill - - await run_skill("/autoskillit:investigate foo", "/some/cwd") - - # All add_dirs entries must be ValidatedAddDir instances - assert all(isinstance(d, ValidatedAddDir) for d in captured["add_dirs"]) - - # At least one add_dir must have .claude/skills/*/SKILL.md layout - from pathlib import Path - - has_skills = False - for d in captured["add_dirs"]: - p = Path(d.path) - if list(p.glob(".claude/skills/*/SKILL.md")): - has_skills = True - break - assert has_skills, "run_skill must pass an --add-dir with discoverable skills" - - # Must NOT include raw skills_extended/ path - from autoskillit.workspace.skills import bundled_skills_extended_dir - - skills_ext = str(bundled_skills_extended_dir()) - add_dir_paths = [d.path for d in captured["add_dirs"]] - assert skills_ext not in add_dir_paths, ( - "run_skill must not pass skills_extended/ directly — " - "it should route through DefaultSessionSkillManager" - ) diff --git a/tests/execution/test_headless.py b/tests/execution/test_headless_core.py similarity index 56% rename from tests/execution/test_headless.py rename to tests/execution/test_headless_core.py index 83605b28f0..286089612b 100644 --- a/tests/execution/test_headless.py +++ b/tests/execution/test_headless_core.py @@ -17,11 +17,12 @@ from autoskillit.execution.commands import _ensure_skill_prefix from autoskillit.execution.headless import ( _build_skill_result, - _extract_missing_token_hints, _extract_worktree_path, - _scan_jsonl_write_paths, ) from tests.conftest import _make_result, _make_timeout_result +from tests.execution.conftest import _sr, _success_session_json + +pytestmark = [pytest.mark.layer("execution"), pytest.mark.small] def test_inject_completion_directive_appends_marker(): @@ -33,18 +34,6 @@ def test_inject_completion_directive_appends_marker(): assert "ORCHESTRATION DIRECTIVE" in result -def _success_session_json(result_text: str) -> str: - return json.dumps( - { - "type": "result", - "subtype": "success", - "result": result_text, - "session_id": "test-session", - "is_error": False, - } - ) - - def _failed_session_json() -> str: return json.dumps( { @@ -71,30 +60,10 @@ def _context_exhausted_session_json() -> str: ) -def _sr( - returncode=0, - stdout="", - stderr="", - termination=TerminationReason.NATURAL_EXIT, - session_id: str = "", - channel_b_session_id: str = "", -): - """Build a minimal SubprocessResult for _build_skill_result tests.""" - return SubprocessResult( - returncode, - stdout, - stderr, - termination, - pid=12345, - session_id=session_id, - channel_b_session_id=channel_b_session_id, - ) - - class TestSessionLogDir: """Unit tests for _session_log_dir — path derivation and log emission.""" - # --- path derivation (from test_tools_execution.py TestSessionLogDir) --- + # --- path derivation (from test_tools_execution_command.py TestSessionLogDir) --- def test_replaces_slashes(self): from autoskillit.execution.headless import _session_log_dir @@ -116,7 +85,7 @@ def test_replaces_both_slashes_and_underscores(self): result == Path.home() / ".claude" / "projects" / "-home-user-name-my-project-sub-dir" ) - # --- log behavior (from test_server_init.py TestGateTransitionLogs) --- + # --- log behavior (from test_server_init_gate.py TestGateTransitionLogs) --- def test_warns_when_dir_missing(self, tmp_path, monkeypatch): import structlog.testing @@ -201,6 +170,76 @@ def test_session_log_dir_creates_missing_directory(self, tmp_path, monkeypatch): assert result.exists() assert result.is_dir() + def test_session_log_dir_mkdir_oserror_reraises( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + from unittest.mock import Mock + + from autoskillit.execution.headless import _session_log_dir + + monkeypatch.setattr(Path, "home", lambda: tmp_path / "nonexistent_home") + monkeypatch.setattr(Path, "mkdir", Mock(side_effect=OSError("disk full"))) + + with pytest.raises(OSError, match="disk full"): + _session_log_dir(str(tmp_path / "some_project")) + + +class TestRunHeadlessCoreFilesystemWrites: + """Tests that fs_writes_detected reflects filesystem changes in the skill temp dir.""" + + @pytest.mark.anyio + async def test_fs_writes_detected_true_when_file_created( + self, minimal_ctx, tmp_path: Path + ) -> None: + from autoskillit.execution.headless import run_headless_core + from tests.fakes import MockSubprocessRunner + + skill_temp = tmp_path / ".autoskillit" / "temp" / "probe" + skill_temp.mkdir(parents=True) + + class _FileSideEffectRunner: + def __init__(self, inner: MockSubprocessRunner, write_file: Path) -> None: + self._inner = inner + self._write_file = write_file + + async def __call__(self, cmd, *, cwd, timeout, **kwargs): + self._write_file.touch() + return await self._inner(cmd, cwd=cwd, timeout=timeout, **kwargs) + + base_runner = MockSubprocessRunner() + base_runner.set_default(_sr(0, _success_session_json("done"), "")) + minimal_ctx.runner = _FileSideEffectRunner(base_runner, skill_temp / "output.txt") + + result = await run_headless_core( + "/autoskillit:probe", + str(tmp_path), + minimal_ctx, + ) + + assert result.fs_writes_detected is True + + @pytest.mark.anyio + async def test_fs_writes_detected_false_when_no_file_created( + self, minimal_ctx, tmp_path: Path + ) -> None: + from autoskillit.execution.headless import run_headless_core + from tests.fakes import MockSubprocessRunner + + skill_temp = tmp_path / ".autoskillit" / "temp" / "probe" + skill_temp.mkdir(parents=True) + + runner = MockSubprocessRunner() + runner.set_default(_sr(0, _success_session_json("done"), "")) + minimal_ctx.runner = runner + + result = await run_headless_core( + "/autoskillit:probe", + str(tmp_path), + minimal_ctx, + ) + + assert result.fs_writes_detected is False + class TestResolveSessionId: """Unit tests for _resolve_skill_session_id — the session UUID resolution helper.""" @@ -792,22 +831,61 @@ class TestEnsureSkillPrefix: """Unit tests for _ensure_skill_prefix helper.""" def test_adds_use_to_slash_command(self): - assert _ensure_skill_prefix("/investigate error") == "Use /investigate error" + assert _ensure_skill_prefix("/investigate error") == "Use the /investigate skill error" def test_adds_use_to_namespaced_skill(self): assert ( _ensure_skill_prefix("/autoskillit:investigate error") - == "Use /autoskillit:investigate error" + == "Use the /autoskillit:investigate skill error" ) def test_no_double_prefix(self): - assert _ensure_skill_prefix("Use /investigate error") == "Use /investigate error" + assert ( + _ensure_skill_prefix("Use the /investigate skill error") + == "Use the /investigate skill error" + ) def test_ignores_plain_prompts(self): assert _ensure_skill_prefix("Fix the bug in main.py") == "Fix the bug in main.py" def test_handles_leading_whitespace(self): - assert _ensure_skill_prefix(" /investigate error") == "Use /investigate error" + assert _ensure_skill_prefix(" /investigate error") == "Use the /investigate skill error" + + def test_bare_slash_command_no_args(self): + assert _ensure_skill_prefix("/investigate") == "Use the /investigate skill" + + def test_no_first_action_without_provider_profile(self): + result = _ensure_skill_prefix("/investigate error") + assert "FIRST ACTION" not in result + + def test_no_first_action_with_empty_provider_profile(self): + result = _ensure_skill_prefix("/investigate error", provider_profile="") + assert "FIRST ACTION" not in result + + def test_first_action_prepended_for_non_anthropic_provider(self): + result = _ensure_skill_prefix( + "/autoskillit:implement-worktree-no-merge /path", provider_profile="minimax" + ) + assert result.startswith("FIRST ACTION:") + assert 'skill="implement-worktree-no-merge"' in result + + def test_first_action_includes_fallback(self): + result = _ensure_skill_prefix("/autoskillit:investigate error", provider_profile="minimax") + assert "SKILL.md" in result + + def test_first_action_uses_extract_skill_name(self): + result = _ensure_skill_prefix("/autoskillit:make-plan task", provider_profile="minimax") + assert 'skill="make-plan"' in result + assert "autoskillit:" not in result.split('"')[1] + + def test_first_action_preserves_use_the_line(self): + result = _ensure_skill_prefix("/investigate error", provider_profile="minimax") + assert "Use the /investigate skill error" in result + + def test_plain_prompt_unchanged_with_provider_profile(self): + result = _ensure_skill_prefix("Fix the bug", provider_profile="minimax") + assert "FIRST ACTION" not in result + assert result == "Fix the bug" class TestStalenessReturnsNeedsRetry: @@ -841,14 +919,30 @@ class TestBuildSkillResultCrossValidation: "is_error", "exit_code", "kill_reason", + "last_stop_reason", + "lifespan_started", + "provider_used", + "provider_fallback", "needs_retry", "retry_reason", + "order_id", "stderr", "token_usage", "write_path_warnings", "write_call_count", + "fs_writes_detected", + "has_progress_evidence", + "infra_exit_category", } + def test_expected_skill_keys_includes_provider(self): + import dataclasses + + from autoskillit.core.types import SkillResult + + skill_result_field_names = {f.name for f in dataclasses.fields(SkillResult)} + assert "provider" in skill_result_field_names + def test_empty_stdout_exit_zero_is_failure(self): """Exit 0 with empty stdout is NOT success — output was lost.""" result_obj = SubprocessResult( @@ -921,59 +1015,91 @@ def test_nonzero_exit_overrides_is_error_false(self): response = json.loads(_build_skill_result(result_obj).to_json()) assert response["success"] is False - def test_gate_disabled_schema(self, tool_ctx): - """Gate-disabled response has standard keys.""" - from autoskillit.pipeline.gate import DefaultGateState - from autoskillit.server.helpers import _require_enabled - - tool_ctx.gate = DefaultGateState(enabled=False) - response = json.loads(_require_enabled()) - assert set(response.keys()) == self.EXPECTED_SKILL_KEYS - - def test_stale_schema(self): - """Stale response has standard keys.""" - result_obj = SubprocessResult( - returncode=-1, stdout="", stderr="", termination=TerminationReason.STALE, pid=1 - ) + @pytest.mark.parametrize( + "result_obj", + [ + SubprocessResult( + returncode=-1, stdout="", stderr="", termination=TerminationReason.STALE, pid=1 + ), + SubprocessResult( + returncode=-1, stdout="", stderr="", termination=TerminationReason.TIMED_OUT, pid=1 + ), + SubprocessResult( + returncode=0, + stdout=json.dumps( + { + "type": "result", + "subtype": "success", + "is_error": False, + "result": "Done.", + "session_id": "s1", + } + ), + stderr="", + termination=TerminationReason.NATURAL_EXIT, + pid=1, + ), + SubprocessResult( + returncode=0, + stdout="", + stderr="", + termination=TerminationReason.NATURAL_EXIT, + pid=1, + ), + ], + ids=["stale", "timeout", "normal_success", "empty_stdout"], + ) + def test_schema_keys(self, result_obj: SubprocessResult): + """Response always exposes the full standard key set.""" response = json.loads(_build_skill_result(result_obj).to_json()) assert set(response.keys()) == self.EXPECTED_SKILL_KEYS - def test_timeout_schema(self): - """Timeout response has standard keys.""" - result_obj = SubprocessResult( - returncode=-1, stdout="", stderr="", termination=TerminationReason.TIMED_OUT, pid=1 - ) - response = json.loads(_build_skill_result(result_obj).to_json()) - assert set(response.keys()) == self.EXPECTED_SKILL_KEYS - def test_normal_success_schema(self): - """Normal success response has standard keys.""" - valid_json = json.dumps( +def _make_subprocess_result_with_tool_uses( + tool_use_names: list[str] | None = None, +) -> SubprocessResult: + """Build a SubprocessResult whose stdout includes optional tool_use NDJSON blocks.""" + records = [] + if tool_use_names: + content = [ + {"type": "tool_use", "name": name, "id": f"tu-{i}"} + for i, name in enumerate(tool_use_names) + ] + records.append(json.dumps({"type": "assistant", "message": {"content": content}})) + records.append( + json.dumps( { "type": "result", "subtype": "success", "is_error": False, - "result": "Done.", - "session_id": "s1", + "result": "done", + "session_id": "test-session", } ) - result_obj = SubprocessResult( - returncode=0, - stdout=valid_json, - stderr="", - termination=TerminationReason.NATURAL_EXIT, - pid=1, - ) - response = json.loads(_build_skill_result(result_obj).to_json()) - assert set(response.keys()) == self.EXPECTED_SKILL_KEYS + ) + return SubprocessResult( + returncode=0, + stdout="\n".join(records), + stderr="", + termination=TerminationReason.NATURAL_EXIT, + pid=12345, + ) - def test_empty_stdout_schema(self): - """Empty stdout response has standard keys.""" - result_obj = SubprocessResult( - returncode=0, stdout="", stderr="", termination=TerminationReason.NATURAL_EXIT, pid=1 - ) - response = json.loads(_build_skill_result(result_obj).to_json()) - assert set(response.keys()) == self.EXPECTED_SKILL_KEYS + +class TestBuildSkillResultLifespanStarted: + """_build_skill_result sets lifespan_started based on tool_uses in stdout.""" + + def test_build_skill_result_sets_lifespan_started_true(self): + """Non-empty tool_uses in stdout → _build_skill_result produces lifespan_started=True.""" + result = _make_subprocess_result_with_tool_uses(tool_use_names=["Write", "Edit"]) + sr = _build_skill_result(result) + assert sr.lifespan_started is True + + def test_build_skill_result_sets_lifespan_started_false_no_tool_uses(self): + """Empty tool_uses in stdout → _build_skill_result produces lifespan_started=False.""" + result = _make_subprocess_result_with_tool_uses(tool_use_names=None) + sr = _build_skill_result(result) + assert sr.lifespan_started is False class TestBuildSkillResultStderr: @@ -1953,8 +2079,7 @@ def test_captured_record_has_timestamp(self, tool_ctx): ) _build_skill_result(result, skill_command="/test", audit=tool_ctx.audit) record = tool_ctx.audit.get_report()[0] - assert record.timestamp # non-empty ISO timestamp - datetime.fromisoformat(record.timestamp) # must parse as ISO + assert datetime.fromisoformat(record.timestamp) # valid ISO 8601 format def test_stale_termination_is_captured(self, tool_ctx): result = _make_result(returncode=0, termination_reason=TerminationReason.STALE) @@ -2209,6 +2334,83 @@ async def raising_runner(*args: object, **kwargs: object) -> None: assert result.success is False +class TestCancelledSessionLog: + """flush_session_log is called with success=False when runner raises CancelledError.""" + + @pytest.mark.anyio + async def test_run_headless_core_cancellation_still_flushes_session_log( + self, monkeypatch, tool_ctx, tmp_path + ): + """flush_session_log is called with CANCELLED termination_reason on cancellation.""" + import anyio + + from autoskillit.execution.headless import run_headless_core + + flushed: list[dict] = [] + + def fake_flush(**kwargs: object) -> None: + flushed.append(dict(kwargs)) + + monkeypatch.setattr("autoskillit.execution.flush_session_log", fake_flush) + + async def cancelled_runner(*args: object, **kwargs: object) -> None: + raise anyio.get_cancelled_exc_class()() + + tool_ctx.runner = cancelled_runner # type: ignore[assignment] + + with pytest.raises(anyio.get_cancelled_exc_class()): + await run_headless_core("/investigate test", cwd=str(tmp_path), ctx=tool_ctx) + + cancel_calls = [f for f in flushed if f.get("termination_reason") == "CANCELLED"] + assert len(cancel_calls) == 1 + assert cancel_calls[0]["success"] is False + assert cancel_calls[0]["subtype"] == "cancelled" + assert cancel_calls[0]["exit_code"] == -1 + assert cancel_calls[0]["session_id"] == "" + assert cancel_calls[0]["pid"] == 0 + + @pytest.mark.anyio + async def test_run_headless_core_cancelled_error_propagates_after_flush( + self, monkeypatch, tool_ctx, tmp_path + ): + """CancelledError re-raises after flush — never swallowed.""" + import anyio + + from autoskillit.execution.headless import run_headless_core + + monkeypatch.setattr("autoskillit.execution.flush_session_log", lambda **kw: None) + + async def cancelled_runner(*args: object, **kwargs: object) -> None: + raise anyio.get_cancelled_exc_class()() + + tool_ctx.runner = cancelled_runner # type: ignore[assignment] + + with pytest.raises(anyio.get_cancelled_exc_class()): + await run_headless_core("/investigate test", cwd=str(tmp_path), ctx=tool_ctx) + + @pytest.mark.anyio + async def test_run_headless_core_flush_failure_does_not_suppress_cancellation( + self, monkeypatch, tool_ctx, tmp_path + ): + """Even if flush_session_log raises, cancellation still propagates.""" + import anyio + + from autoskillit.execution.headless import run_headless_core + + def exploding_flush(**kwargs: object) -> None: + raise OSError("log disk full") + + monkeypatch.setattr("autoskillit.execution.flush_session_log", exploding_flush) + + async def cancelled_runner(*args: object, **kwargs: object) -> None: + raise anyio.get_cancelled_exc_class()() + + tool_ctx.runner = cancelled_runner # type: ignore[assignment] + + with pytest.raises(anyio.get_cancelled_exc_class()): + await run_headless_core("/investigate test", cwd=str(tmp_path), ctx=tool_ctx) + + class TestRetryBudgetEnforcement: """_build_skill_result enforces max_consecutive_retries budget.""" @@ -2325,12 +2527,11 @@ def test_preserves_original_command(self): result = _inject_cwd_anchor(original, "/clone/dir") assert result.startswith(original) - def test_directive_mentions_temp_and_read_only(self): + def test_directive_mentions_temp(self): from autoskillit.execution.commands import _inject_cwd_anchor result = _inject_cwd_anchor("cmd", "/wd") assert ".autoskillit/temp/" in result - assert "READ-ONLY" in result def test_skips_when_cwd_empty(self): from autoskillit.execution.commands import _inject_cwd_anchor @@ -2378,1711 +2579,27 @@ def test_directive_exempts_final_response(self): # Must not suppress the final response where structured output tokens live assert "final response" in result + def test_no_after_loading_by_default(self): + from autoskillit.execution.commands import _inject_narration_suppression -# --------------------------------------------------------------------------- -# Test: _extract_output_paths (Step 1b) -# --------------------------------------------------------------------------- - - -class TestExtractOutputPaths: - def test_extracts_single_path(self): - from autoskillit.execution.headless import _extract_output_paths - - msgs = ["plan_path = /correct/path/.autoskillit/temp/make-plan/foo.md"] - result = _extract_output_paths(msgs) - assert result == {"plan_path": "/correct/path/.autoskillit/temp/make-plan/foo.md"} - - def test_extracts_multiple_tokens(self): - from autoskillit.execution.headless import _extract_output_paths - - msg = ( - "plan_path = /clone/.autoskillit/temp/make-plan/plan.md\n" - "summary_path = /clone/.autoskillit/temp/report/summary.md\n" - "investigation_path = /clone/.autoskillit/temp/investigate/inv.md" - ) - result = _extract_output_paths([msg]) - assert result == { - "plan_path": "/clone/.autoskillit/temp/make-plan/plan.md", - "summary_path": "/clone/.autoskillit/temp/report/summary.md", - "investigation_path": "/clone/.autoskillit/temp/investigate/inv.md", - } - - def test_returns_empty_when_no_tokens(self): - from autoskillit.execution.headless import _extract_output_paths - - result = _extract_output_paths(["no tokens here", "just regular text"]) - assert result == {} - - def test_ignores_non_absolute_paths(self): - from autoskillit.execution.headless import _extract_output_paths - - result = _extract_output_paths(["plan_path = .autoskillit/temp/make-plan/foo.md"]) - assert result == {} - - def test_extracts_from_multiple_messages(self): - from autoskillit.execution.headless import _extract_output_paths - - msgs = [ - "plan_path = /first/path", - "investigation_path = /second/path", - ] - result = _extract_output_paths(msgs) - assert result == { - "plan_path": "/first/path", - "investigation_path": "/second/path", - } - - def test_last_occurrence_wins(self): - from autoskillit.execution.headless import _extract_output_paths - - msgs = [ - "plan_path = /first/path", - "plan_path = /second/path", - ] - result = _extract_output_paths(msgs) - assert result == {"plan_path": "/second/path"} - - -# --------------------------------------------------------------------------- -# Test: _validate_output_paths (Step 1c) -# --------------------------------------------------------------------------- - - -class TestValidateOutputPaths: - def test_returns_none_when_all_paths_under_cwd(self): - from autoskillit.execution.headless import _validate_output_paths - - paths = { - "plan_path": "/clone/path/.autoskillit/temp/make-plan/foo.md", - "report_path": "/clone/path/.autoskillit/temp/report/bar.md", - } - result = _validate_output_paths(paths, "/clone/path") - assert result is None - - def test_returns_diagnostic_when_path_outside_cwd(self): - from autoskillit.execution.headless import _validate_output_paths - - paths = { - "plan_path": "/source/repo/.autoskillit/temp/make-plan/foo.md", - } - result = _validate_output_paths(paths, "/clone/path") - assert result is not None - assert "plan_path" in result - assert "/source/repo/.autoskillit/temp/make-plan/foo.md" in result - assert "/clone/path" in result - - def test_returns_none_for_empty_paths(self): - from autoskillit.execution.headless import _validate_output_paths - - result = _validate_output_paths({}, "/clone/path") - assert result is None - - def test_cwd_trailing_slash_handling(self): - from autoskillit.execution.headless import _validate_output_paths - - paths = {"plan_path": "/clone/path/.autoskillit/temp/foo.md"} - assert _validate_output_paths(paths, "/clone/path/") is None - assert _validate_output_paths(paths, "/clone/path") is None - - def test_multiple_violations(self): - from autoskillit.execution.headless import _validate_output_paths - - paths = { - "plan_path": "/wrong/.autoskillit/temp/plan.md", - "report_path": "/wrong/.autoskillit/temp/report.md", - } - result = _validate_output_paths(paths, "/clone") - assert result is not None - assert "plan_path" in result - assert "report_path" in result - - -# --------------------------------------------------------------------------- -# Test: _build_skill_result path contamination detection (Step 1d) -# --------------------------------------------------------------------------- - - -class TestBuildSkillResultPathContamination: - @staticmethod - def _assistant_ndjson(text: str) -> str: - """Build an NDJSON assistant message line.""" - return json.dumps( - { - "type": "assistant", - "message": {"content": [{"text": text}]}, - } - ) - - def test_path_contamination_detected(self): - """Output paths outside cwd override success to False.""" - path = "/wrong/source/repo/.autoskillit/temp/make-plan/foo.md" - stdout = ( - self._assistant_ndjson(f"plan_path = {path}") - + "\n" - + _success_session_json("Plan created.") - ) - result = _sr(0, stdout, "", TerminationReason.NATURAL_EXIT) - sr = _build_skill_result(result, cwd="/correct/clone") - assert sr.success is False - assert sr.subtype == "path_contamination" - assert sr.needs_retry is True - assert sr.retry_reason == RetryReason.PATH_CONTAMINATION - - def test_path_contamination_emits_path_contamination_reason(self): - """Path contamination must emit PATH_CONTAMINATION, not RESUME. - - PATH_CONTAMINATION is not a context limit — it is a CWD boundary violation. - The orchestrator must route to on_failure, not on_context_limit. - """ - path = "/wrong/source/repo/.autoskillit/temp/make-plan/bar.md" - stdout = ( - self._assistant_ndjson(f"plan_path = {path}") - + "\n" - + _success_session_json("Plan created.") - ) - result = _sr(0, stdout, "", TerminationReason.NATURAL_EXIT) - sr = _build_skill_result(result, cwd="/correct/clone") - assert sr.retry_reason == RetryReason.PATH_CONTAMINATION - assert sr.needs_retry is True - # Confirm it is NOT RESUME — routing must not go to on_context_limit - - def test_no_contamination_when_paths_under_cwd(self): - """All output paths under cwd yields normal result.""" - path = "/correct/clone/.autoskillit/temp/make-plan/foo.md" - stdout = ( - self._assistant_ndjson(f"plan_path = {path}") - + "\n" - + _success_session_json("Plan created.") - ) - result = _sr(0, stdout, "", TerminationReason.NATURAL_EXIT) - sr = _build_skill_result(result, cwd="/correct/clone") - assert sr.success is True - assert sr.subtype != "path_contamination" + result = _inject_narration_suppression("cmd") + assert "After loading" not in result - def test_no_contamination_when_cwd_empty(self): - """Empty cwd skips path validation — direct callers without clone context.""" - path = "/any/path/.autoskillit/temp/foo.md" - stdout = ( - self._assistant_ndjson(f"plan_path = {path}") + "\n" + _success_session_json("Done.") - ) - result = _sr(0, stdout, "", TerminationReason.NATURAL_EXIT) - sr = _build_skill_result(result, cwd="") - assert sr.success is True + def test_no_after_loading_when_false(self): + from autoskillit.execution.commands import _inject_narration_suppression - def test_no_contamination_when_no_path_tokens(self): - """No output path tokens means validation passes.""" - stdout = _success_session_json("Done with no file output.") - result = _sr(0, stdout, "", TerminationReason.NATURAL_EXIT) - sr = _build_skill_result(result, cwd="/clone/path") - assert sr.success is True + result = _inject_narration_suppression("cmd", has_skill_prefix=False) + assert "After loading" not in result + def test_after_loading_when_has_skill_prefix(self): + from autoskillit.execution.commands import _inject_narration_suppression -# --------------------------------------------------------------------------- -# Test: run_headless_core passes cwd (Step 1f) -# --------------------------------------------------------------------------- + result = _inject_narration_suppression("cmd", has_skill_prefix=True) + assert "After loading the skill instructions" in result + def test_still_contains_efficiency_directive_with_prefix(self): + from autoskillit.execution.commands import _inject_narration_suppression -class TestRunHeadlessCorePassesCwd: - @pytest.mark.anyio - async def test_cwd_anchor_injected_into_prompt(self, tool_ctx): - """Verify run_headless_core injects cwd anchor into the skill prompt.""" - from autoskillit.execution.headless import run_headless_core - - payload = _success_session_json("Result text %%ORDER_UP%%") - tool_ctx.runner.push( - SubprocessResult(0, payload, "", TerminationReason.NATURAL_EXIT, pid=1) - ) - await run_headless_core( - "/autoskillit:investigate test", - "/some/test/cwd", - tool_ctx, - ) - assert tool_ctx.runner.call_args_list, "Runner was never called" - last_cmd = tool_ctx.runner.call_args_list[-1][0] - # cmd is ["env", ...vars, "claude", "-p", , ...] - p_idx = last_cmd.index("-p") - prompt_arg = last_cmd[p_idx + 1] - assert "WORKING DIRECTORY ANCHOR" in prompt_arg - assert "/some/test/cwd" in prompt_arg - - -def _make_tool_use_line(name: str, input_dict: dict) -> str: - return json.dumps( - { - "type": "assistant", - "message": { - "content": [{"type": "tool_use", "name": name, "id": "x", "input": input_dict}] - }, - } - ) - - -class TestScanJsonlWritePaths: - CWD = "/clone/worktree" - - def test_returns_empty_for_clean_write_inside_cwd(self): - line = _make_tool_use_line( - "Write", {"file_path": f"{self.CWD}/.autoskillit/temp/out.md", "content": "x"} - ) - assert _scan_jsonl_write_paths(line, self.CWD) == [] - - def test_detects_write_outside_cwd(self): - line = _make_tool_use_line( - "Write", {"file_path": "/source/repo/.autoskillit/temp/stolen.md", "content": "x"} - ) - warnings = _scan_jsonl_write_paths(line, self.CWD) - assert len(warnings) == 1 - assert "/source/repo/.autoskillit/temp/stolen.md" in warnings[0] - - def test_detects_edit_outside_cwd(self): - line = _make_tool_use_line( - "Edit", - { - "file_path": "/source/repo/src/autoskillit/file.py", - "old_string": "a", - "new_string": "b", - }, - ) - warnings = _scan_jsonl_write_paths(line, self.CWD) - assert len(warnings) == 1 - assert "Edit" in warnings[0] - - def test_detects_bash_with_absolute_path_outside_cwd(self): - line = _make_tool_use_line( - "Bash", {"command": "cat /source/repo/README.md > /tmp/out.txt"} - ) - warnings = _scan_jsonl_write_paths(line, self.CWD) - assert len(warnings) >= 1 - assert any("/source/repo" in w for w in warnings) - - def test_no_warnings_for_empty_stdout(self): - assert _scan_jsonl_write_paths("", self.CWD) == [] - - def test_no_warnings_for_malformed_jsonl(self): - assert _scan_jsonl_write_paths("not json at all\n{broken", self.CWD) == [] - - def test_no_warnings_for_read_only_tool_calls(self): - line = _make_tool_use_line("Read", {"file_path": "/source/repo/some_file.py"}) - assert _scan_jsonl_write_paths(line, self.CWD) == [] - - def test_multiple_violations_in_one_session(self): - lines = "\n".join( - [ - _make_tool_use_line("Write", {"file_path": "/source/repo/a.md", "content": "x"}), - _make_tool_use_line( - "Edit", - {"file_path": "/source/repo/b.py", "old_string": "a", "new_string": "b"}, - ), - json.dumps( - { - "type": "result", - "subtype": "success", - "result": "done", - "session_id": "", - "is_error": False, - } - ), - ] - ) - warnings = _scan_jsonl_write_paths(lines, self.CWD) - assert len(warnings) == 2 - - def test_no_warning_when_cwd_empty(self): - line = _make_tool_use_line("Write", {"file_path": "/any/path/file.md", "content": "x"}) - assert _scan_jsonl_write_paths(line, "") == [] - - def test_no_warning_when_cwd_is_relative(self): - line = _make_tool_use_line("Write", {"file_path": "/any/path/file.md", "content": "x"}) - assert _scan_jsonl_write_paths(line, "relative/path") == [] - - -class TestBuildSkillResultWritePathWarnings: - def test_write_path_warnings_empty_for_clean_session(self): - stdout = ( - _make_tool_use_line( - "Write", {"file_path": "/clone/worktree/.autoskillit/temp/out.md", "content": "x"} - ) - + "\n" - + _success_session_json("Done %%DONE%%") - ) - result = _sr(0, stdout, "", TerminationReason.NATURAL_EXIT) - sr = _build_skill_result(result, cwd="/clone/worktree") - assert sr.write_path_warnings == [] - - def test_write_path_warnings_populated_for_contaminated_session(self): - stdout = ( - _make_tool_use_line( - "Write", {"file_path": "/source/repo/.autoskillit/temp/stolen.md", "content": "x"} - ) - + "\n" - + _success_session_json("Done %%DONE%%") - ) - result = _sr(0, stdout, "", TerminationReason.NATURAL_EXIT) - sr = _build_skill_result(result, cwd="/clone/worktree") - assert len(sr.write_path_warnings) == 1 - assert "/source/repo/.autoskillit/temp/stolen.md" in sr.write_path_warnings[0] - - def test_write_path_warnings_appear_in_to_json(self): - stdout = ( - _make_tool_use_line("Write", {"file_path": "/source/repo/bad.md", "content": "x"}) - + "\n" - + _success_session_json("Done %%DONE%%") - ) - result = _sr(0, stdout, "", TerminationReason.NATURAL_EXIT) - sr = _build_skill_result(result, cwd="/clone/worktree") - data = json.loads(sr.to_json()) - assert "write_path_warnings" in data - assert len(data["write_path_warnings"]) == 1 - - def test_write_path_warnings_independent_of_output_token_contamination(self): - """Warnings are populated even when _validate_output_paths also fires.""" - # plan_path token must appear in an assistant text message for - # _validate_output_paths to detect it (it scans assistant_messages, - # not the final result record). - path_token_line = json.dumps( - { - "type": "assistant", - "message": { - "content": [ - { - "type": "text", - "text": "plan_path = /source/repo/.autoskillit/temp/plan.md", - } - ] - }, - } - ) - stdout = ( - _make_tool_use_line("Write", {"file_path": "/source/repo/bad.md", "content": "x"}) - + "\n" - + path_token_line - + "\n" - + _success_session_json("Done %%DONE%%") - ) - result = _sr(0, stdout, "", TerminationReason.NATURAL_EXIT) - sr = _build_skill_result(result, cwd="/clone/worktree") - # subtype is path_contamination (from _validate_output_paths) - assert sr.subtype == "path_contamination" - # write_path_warnings also populated from JSONL scan - assert len(sr.write_path_warnings) >= 1 - - -class TestBuildSkillResultChannelBPatternRecovery: - """When Channel B wins and pattern is absent from result but present in - assistant_messages, _build_skill_result should recover and produce success=True. - """ - - def test_build_skill_result_channel_b_recovers_pattern_from_assistant_messages( - self, - ) -> None: - """Channel B wins before stdout drain; pattern found in assistant_messages - → recovery produces success=True with block in result. - """ - block = "---prepare-issue-result---\n{}\n---/prepare-issue-result---" - # Assistant message has the block but NOT a standalone %%ORDER_UP%% marker, - # so _recover_from_separate_marker will not activate. The new - # _recover_block_from_assistant_messages must find the block here instead. - assistant_line = json.dumps( - { - "type": "assistant", - "message": { - "content": [{"type": "text", "text": block}], - }, - } - ) - result_line = json.dumps( - { - "type": "result", - "subtype": "success", - "is_error": False, - "result": "", # stdout not yet drained — Channel B won the race - "session_id": "s1", - "errors": [], - } - ) - stdout = assistant_line + "\n" + result_line - - sub_result = SubprocessResult( - returncode=0, - stdout=stdout, - stderr="", - termination=TerminationReason.COMPLETED, - pid=12345, - channel_confirmation=ChannelConfirmation.CHANNEL_B, - ) - sr = _build_skill_result( - sub_result, - completion_marker="%%ORDER_UP%%", - expected_output_patterns=["---prepare-issue-result---"], - ) - assert sr.success is True - assert "---prepare-issue-result---" in sr.result - - def test_synthesis_not_run_for_unparseable_channel_b_with_write_evidence(self) -> None: - """CHANNEL_B + UNPARSEABLE + write evidence must not produce success=True. - - Recovery (and CHANNEL_B bypass) must be blocked when session.session_complete - is False. Synthesis must not inject the file path into result. - """ - - tool_use_line = json.dumps( - { - "type": "tool_use", - "id": "t1", - "name": "Write", - "input": { - "file_path": "/cwd/.autoskillit/temp/make-plan/arch_lens_selection_2026-01-01.md" # noqa: E501 - }, - } - ) - # UNPARSEABLE result (no proper result record) - raw_stdout = tool_use_line + "\ngarbage ndjson partial" - - sub_result = SubprocessResult( - returncode=1, - stdout=raw_stdout, - stderr="", - termination=TerminationReason.COMPLETED, - pid=12345, - channel_confirmation=ChannelConfirmation.CHANNEL_B, - ) - sr = _build_skill_result( - sub_result, - expected_output_patterns=[r"plan_path\s*=\s*/.+"], - ) - assert sr.success is False - assert sr.subtype not in {"success"} - # Synthesis must not have injected the structured token (raw NDJSON may contain - # the file path string, but the plan_path = ... assignment must be absent) - assert "plan_path =" not in sr.result - - def test_synthesis_not_run_for_timeout_channel_b_with_write_evidence(self) -> None: - """CHANNEL_B + TIMEOUT + write evidence must not produce success=True.""" - tool_use_line = json.dumps( - { - "type": "tool_use", - "id": "t1", - "name": "Write", - "input": { - "file_path": "/cwd/.autoskillit/temp/make-plan/arch_lens_selection_2026-01-01.md" # noqa: E501 - }, - } - ) - result_line = json.dumps( - { - "type": "result", - "subtype": "success", - "is_error": False, - "result": "", - "session_id": "s1", - "errors": [], - } - ) - stdout = tool_use_line + "\n" + result_line - - sub_result = SubprocessResult( - returncode=-1, - stdout=stdout, - stderr="", - termination=TerminationReason.TIMED_OUT, - pid=12345, - channel_confirmation=ChannelConfirmation.CHANNEL_B, - ) - sr = _build_skill_result( - sub_result, - expected_output_patterns=[r"plan_path\s*=\s*/.+"], - ) - assert sr.success is False - assert "arch_lens_selection" not in sr.result - - def test_synthesis_skipped_for_channel_b_session_complete(self) -> None: - """CHANNEL_B + SUCCESS + write evidence but no pattern in assistant_messages. - - Synthesis must NOT fabricate the token — if pattern is absent from - assistant_messages, the agent never emitted it. - """ - tool_use_line = json.dumps( - { - "type": "tool_use", - "id": "t1", - "name": "Write", - "input": {"file_path": "/cwd/.autoskillit/temp/make-plan/task_plan.md"}, - } - ) - result_line = json.dumps( - { - "type": "result", - "subtype": "success", - "is_error": False, - "result": "done", - "session_id": "s1", - "errors": [], - } - ) - stdout = tool_use_line + "\n" + result_line - - sub_result = SubprocessResult( - returncode=0, - stdout=stdout, - stderr="", - termination=TerminationReason.COMPLETED, - pid=12345, - channel_confirmation=ChannelConfirmation.CHANNEL_B, - ) - sr = _build_skill_result( - sub_result, - expected_output_patterns=[r"plan_path\s*=\s*/.+"], - ) - assert sr.success is False - assert "plan_path" not in sr.result - - -class TestBuildSkillResultChannelAPatternRecovery: - """_build_skill_result must extend pattern recovery to CHANNEL_A wins, not just CHANNEL_B.""" - - def test_build_skill_result_channel_a_recovers_from_assistant_messages(self) -> None: - """_build_skill_result must attempt pattern recovery for CHANNEL_A, not just CHANNEL_B.""" - block = "verdict = GO\n%%ORDER_UP%%" - assistant_line = json.dumps( - { - "type": "assistant", - "message": { - "content": [{"type": "text", "text": block}], - }, - } - ) - result_line = json.dumps( - { - "type": "result", - "subtype": "success", - "is_error": False, - "result": "Done. %%ORDER_UP%%", - "session_id": "s1", - "errors": [], - } - ) - stdout = assistant_line + "\n" + result_line - sub_result = SubprocessResult( - returncode=0, - stdout=stdout, - stderr="", - termination=TerminationReason.NATURAL_EXIT, - pid=12345, - channel_confirmation=ChannelConfirmation.CHANNEL_A, - ) - sr = _build_skill_result( - sub_result, - completion_marker="%%ORDER_UP%%", - expected_output_patterns=["verdict\\s*=\\s*(GO|NO GO)"], - ) - assert sr.success is True, ( - "CHANNEL_A should recover patterns from assistant_messages, same as CHANNEL_B." - ) - - def test_build_skill_result_channel_a_pattern_contract_violation_is_terminal( - self, - ) -> None: - """After failed CHANNEL_A recovery, contract violation must be FAILED, not RETRIABLE.""" - assistant_line = json.dumps( - { - "type": "assistant", - "message": { - "content": [{"type": "text", "text": "No verdict here. %%ORDER_UP%%"}], - }, - } - ) - result_line = json.dumps( - { - "type": "result", - "subtype": "success", - "is_error": False, - "result": "Done. %%ORDER_UP%%", - "session_id": "s1", - "errors": [], - } - ) - stdout = assistant_line + "\n" + result_line - sub_result = SubprocessResult( - returncode=0, - stdout=stdout, - stderr="", - termination=TerminationReason.NATURAL_EXIT, - pid=12345, - channel_confirmation=ChannelConfirmation.CHANNEL_A, - ) - sr = _build_skill_result( - sub_result, - completion_marker="%%ORDER_UP%%", - expected_output_patterns=["verdict\\s*=\\s*(GO|NO GO)"], - ) - assert sr.success is False - assert sr.needs_retry is False - assert sr.subtype == "adjudicated_failure" - - def test_bold_wrapped_plan_path_returns_success_not_adjudicated_failure( - self, - ) -> None: - """Regression test for issue #462: model emitting **plan_path** = /abs/path - must produce success=True, not adjudicated_failure.""" - result_text = ( - "The implementation plan has been written.\n\n" - "**plan_path** = /abs/path/plan.md\n" - "**plan_parts** = /abs/path/plan.md\n" - "%%ORDER_UP%%" - ) - sub_result = SubprocessResult( - returncode=0, - stdout=_success_session_json(result_text), - stderr="", - termination=TerminationReason.NATURAL_EXIT, - pid=12345, - ) - sr = _build_skill_result( - sub_result, - completion_marker="%%ORDER_UP%%", - expected_output_patterns=["plan_path\\s*=\\s*/.+"], - ) - assert sr.success is True - assert sr.subtype != "adjudicated_failure" - assert sr.needs_retry is False - - def test_bold_wrapped_tokens_in_assistant_messages_recovery_succeeds( - self, - ) -> None: - """Recovery path: bold-wrapped tokens in assistant_messages must also be found. - The pattern_recovery block in _build_skill_result requires channel_confirmation - != UNMONITORED to trigger; use CHANNEL_A to activate the recovery path.""" - token_block = "**plan_path** = /abs/path/plan.md\n**plan_parts** = /abs/path/plan.md" - assistant_line = json.dumps( - { - "type": "assistant", - "message": { - "content": [{"type": "text", "text": token_block}], - }, - } - ) - result_line = json.dumps( - { - "type": "result", - "subtype": "success", - "is_error": False, - "result": "The plan is complete. %%ORDER_UP%%", - "session_id": "s1", - "errors": [], - } - ) - stdout = assistant_line + "\n" + result_line - sub_result = SubprocessResult( - returncode=0, - stdout=stdout, - stderr="", - termination=TerminationReason.NATURAL_EXIT, - pid=12345, - channel_confirmation=ChannelConfirmation.CHANNEL_A, - ) - sr = _build_skill_result( - sub_result, - completion_marker="%%ORDER_UP%%", - expected_output_patterns=["plan_path\\s*=\\s*/.+"], - ) - assert sr.success is True - assert sr.subtype != "adjudicated_failure" - assert sr.needs_retry is False - - -class TestBuildSkillResultDirMissingRecovery: - """DIR_MISSING channel confirmation does not silently pass — it attempts - late-bind recovery when conditions allow.""" - - def test_dir_missing_with_recoverable_subtype_attempts_recovery(self): - """When channel_confirmation is DIR_MISSING and subtype is recoverable, - the recovery gate must attempt marker-based recovery. - - Setup: termination=COMPLETED (bypasses stale branch), subtype=empty_output - (in _CHANNEL_B_RECOVERABLE_SUBTYPES), marker on a standalone line in the - assistant message (required by _marker_is_standalone). - """ - import structlog.testing - - marker = "===DONE===" - # Marker must appear as a standalone line for _marker_is_standalone to match - assistant_line = json.dumps( - { - "type": "assistant", - "message": { - "content": [{"type": "text", "text": f"Task complete.\n\n{marker}"}], - }, - } - ) - # subtype=empty_output places the session in _CHANNEL_B_RECOVERABLE_SUBTYPES, - # triggering the DIR_MISSING recovery guard in _build_skill_result - result_line = json.dumps( - { - "type": "result", - "subtype": "empty_output", - "is_error": True, - "result": "", - "session_id": "s1", - "errors": [], - } - ) - stdout = assistant_line + "\n" + result_line - # termination=COMPLETED skips the stale-branch early return so execution - # reaches the Channel B / DIR_MISSING recovery gate - sub_result = SubprocessResult( - returncode=0, - stdout=stdout, - stderr="", - termination=TerminationReason.COMPLETED, - pid=12345, - channel_confirmation=ChannelConfirmation.DIR_MISSING, - ) - with structlog.testing.capture_logs() as logs: - skill = _build_skill_result(sub_result, completion_marker=marker) - # Recovery should succeed — marker found as standalone line in assistant_messages - assert skill.success is True - # Verify the recovery code path was taken, not just the outcome - assert any(e.get("event") == "dir_missing_late_bind_recovery" for e in logs) - - def test_dir_missing_without_marker_does_not_recover(self): - """DIR_MISSING with no completion marker must not silently succeed.""" - sub_result = SubprocessResult( - returncode=0, - stdout="", - stderr="", - termination=TerminationReason.STALE, - pid=12345, - channel_confirmation=ChannelConfirmation.DIR_MISSING, - ) - skill = _build_skill_result(sub_result) - assert skill.success is False - - -class TestTimedOutSessionPreservesState: - """TIMED_OUT branch must parse stdout to preserve tool_uses and assistant_messages.""" - - def test_timed_out_with_writes_preserves_write_call_count(self): - """Timed-out session with Write/Edit tool_use blocks must report write_call_count > 0.""" - from autoskillit.execution.headless import _build_skill_result - - ndjson = "\n".join( - [ - _make_tool_use_line("Write", {"file_path": "/tmp/a.py", "content": "x"}), - _make_tool_use_line( - "Edit", {"file_path": "/tmp/b.py", "old_string": "a", "new_string": "b"} - ), - ] - ) - sub_result = SubprocessResult( - returncode=-1, - stdout=ndjson, - stderr="", - termination=TerminationReason.TIMED_OUT, - pid=12345, - ) - sr = _build_skill_result(sub_result) - assert sr.write_call_count == 2 - assert sr.success is False - assert sr.needs_retry is False - - def test_timed_out_with_empty_stdout_uses_timeout_subtype(self): - """Timed-out session with no stdout uses TIMEOUT subtype.""" - from autoskillit.execution.headless import _build_skill_result - - sub_result = SubprocessResult( - returncode=-1, - stdout="", - stderr="", - termination=TerminationReason.TIMED_OUT, - pid=12345, - ) - sr = _build_skill_result(sub_result) - assert sr.success is False - assert sr.write_call_count == 0 - - def test_timed_out_with_success_result_overrides_to_timeout(self): - """When timed-out stdout has a success result, subtype is overridden to timeout.""" - from autoskillit.execution.headless import _build_skill_result - - ndjson = "\n".join( - [ - _make_tool_use_line("Write", {"file_path": "/tmp/a.py", "content": "x"}), - json.dumps( - { - "type": "result", - "subtype": "success", - "is_error": False, - "result": "Task completed.", - "session_id": "s1", - } - ), - ] - ) - sub_result = SubprocessResult( - returncode=-1, - stdout=ndjson, - stderr="", - termination=TerminationReason.TIMED_OUT, - pid=12345, - ) - sr = _build_skill_result(sub_result) - assert sr.cli_subtype == "timeout" - assert sr.write_call_count == 1 - - -class TestOutputPathTokensDerivedFromContracts: - # Hardcoded fixture — update when skill_contracts.yaml gains new file_path outputs. - # Using a fixed set (rather than re-deriving) ensures bugs in the derivation formula - # cause test failures rather than silent agreement between production and test code. - _EXPECTED_OUTPUT_PATH_TOKENS = frozenset( - { - "analysis_file", - "conflict_report_path", - "diagnosis_path", - "diagram_path", - "evaluation_dashboard", - "experiment_plan", - "group_files", - "groups_path", - "investigation_path", - "manifest_path", - "plan_parts", - "plan_path", - "pr_order_file", - "prep_path", - "recipe_path", - "remediation_path", - "report_path", - "report_plan_path", - "results_path", - "review_path", - "revision_guidance", - "scope_report", - "summary_path", - "triage_manifest", - "triage_report", - "visualization_plan_path", - "html_path", - "resource_report", - } - ) - - def test_output_path_tokens_contains_all_file_path_contract_outputs(self) -> None: - """Every skill output declared with type=file_path or type=file_path_list in - skill_contracts.yaml must appear in _OUTPUT_PATH_TOKENS (or be documented as - intentionally excluded).""" - from autoskillit.execution.headless import ( - _INTENTIONALLY_EXCLUDED_PATH_TOKENS, - _OUTPUT_PATH_TOKENS, - ) - from autoskillit.recipe.contracts import load_bundled_manifest - - manifest = load_bundled_manifest() - declared_path_tokens = { - out["name"] - for skill_data in manifest.get("skills", {}).values() - for out in skill_data.get("outputs", []) - if isinstance(out, dict) and out.get("type", "").startswith("file_path") - } - untracked = ( - declared_path_tokens - _OUTPUT_PATH_TOKENS - _INTENTIONALLY_EXCLUDED_PATH_TOKENS - ) - assert not untracked, ( - f"These path tokens are declared in skill_contracts.yaml but missing from " - f"_OUTPUT_PATH_TOKENS or _INTENTIONALLY_EXCLUDED_PATH_TOKENS: {untracked}" - ) - - def test_output_path_tokens_matches_expected_fixture(self) -> None: - """_OUTPUT_PATH_TOKENS must exactly match the known fixture set. - - This guards against bugs in the derivation formula: if _build_path_token_set() - is broken, both the production frozenset and a re-derived set would agree, but - this hardcoded fixture would not. - """ - from autoskillit.execution.headless import _OUTPUT_PATH_TOKENS - - extra = _OUTPUT_PATH_TOKENS - self._EXPECTED_OUTPUT_PATH_TOKENS - missing = self._EXPECTED_OUTPUT_PATH_TOKENS - _OUTPUT_PATH_TOKENS - assert _OUTPUT_PATH_TOKENS == self._EXPECTED_OUTPUT_PATH_TOKENS, ( - f"_OUTPUT_PATH_TOKENS diverged from expected fixture.\n" - f"Extra (in production, not in fixture): {extra}\n" - f"Missing (in fixture, not in production): {missing}" - ) - - -# --------------------------------------------------------------------------- -# T2: _synthesize_from_write_artifacts -# --------------------------------------------------------------------------- - -from autoskillit.execution.session import ClaudeSessionResult # noqa: E402 - - -@pytest.fixture -def make_session(): - """Build a ClaudeSessionResult with configurable result and tool_uses (parsed form).""" - - def _factory( - result: str = "", - tool_uses: list[dict] | None = None, - subtype: str = "success", - is_error: bool = False, - ) -> ClaudeSessionResult: - return ClaudeSessionResult( - subtype=subtype, - is_error=is_error, - result=result, - session_id="test-session", - tool_uses=tool_uses or [], - ) - - return _factory - - -class TestSynthesizeFromWriteArtifacts: - def test_synthesizes_plan_path_from_write_tool_use(self, make_session): - """When Write file_path is absolute and pattern is plan_path=, token is injected.""" - from autoskillit.execution.headless import _synthesize_from_write_artifacts - - session = make_session( - result="plan summary\n%%ORDER_UP%%", - tool_uses=[ - {"name": "Write", "id": "t1", "file_path": "/abs/temp/make-plan/my_plan.md"} - ], - ) - patterns = [r"plan_path\s*=\s*/.+"] - result = _synthesize_from_write_artifacts(session, patterns, write_call_count=1) - assert result is not None - assert "plan_path = /abs/temp/make-plan/my_plan.md" in result.result - - def test_returns_none_when_no_write_tool_uses(self, make_session): - """No synthesis when write_call_count == 0.""" - from autoskillit.execution.headless import _synthesize_from_write_artifacts - - session = make_session(result="plan summary\n%%ORDER_UP%%", tool_uses=[]) - result = _synthesize_from_write_artifacts( - session, [r"plan_path\s*=\s*/.+"], write_call_count=0 - ) - assert result is None - - def test_returns_none_when_no_absolute_file_path(self, make_session): - """Relative file_path in Write tool_use is not used for synthesis.""" - from autoskillit.execution.headless import _synthesize_from_write_artifacts - - session = make_session( - result="plan summary\n%%ORDER_UP%%", - tool_uses=[{"name": "Write", "id": "t1", "file_path": "temp/make-plan/plan.md"}], - ) - result = _synthesize_from_write_artifacts( - session, [r"plan_path\s*=\s*/.+"], write_call_count=1 - ) - assert result is None - - def test_returns_none_when_pattern_already_satisfied(self, make_session): - """No synthesis when the pattern is already in session.result.""" - from autoskillit.execution.headless import _synthesize_from_write_artifacts - - session = make_session( - result="plan_path = /abs/plan.md\n%%ORDER_UP%%", - tool_uses=[{"name": "Write", "id": "t1", "file_path": "/abs/plan.md"}], - ) - result = _synthesize_from_write_artifacts( - session, [r"plan_path\s*=\s*/.+"], write_call_count=1 - ) - assert result is None - - def test_returns_none_when_no_path_capture_patterns(self, make_session): - """Non-path patterns (verdict= etc) are not attempted for synthesis.""" - from autoskillit.execution.headless import _synthesize_from_write_artifacts - - session = make_session( - result="%%ORDER_UP%%", - tool_uses=[{"name": "Write", "id": "t1", "file_path": "/abs/plan.md"}], - ) - result = _synthesize_from_write_artifacts( - session, [r"verdict\s*=\s*(GO|NO GO)"], write_call_count=1 - ) - assert result is None - - def test_synthesis_uses_last_write_not_first(self, make_session): - """When multiple Write tool_uses exist, synthesis must use the LAST absolute path. - - Multi-artifact skills write intermediate files first, final deliverable last. - Synthesis must inject the final deliverable (last path), not the intermediate. - """ - from autoskillit.execution.headless import _synthesize_from_write_artifacts - - session = make_session( - result="", - tool_uses=[ - { - "name": "Write", - "id": "t1", - "file_path": "/cwd/.autoskillit/temp/make-plan/arch_lens_selection.md", - }, - { - "name": "Write", - "id": "t2", - "file_path": "/cwd/.autoskillit/temp/make-plan/task_plan_2026-01-01.md", - }, - ], - ) - result = _synthesize_from_write_artifacts( - session, [r"plan_path\s*=\s*/.+"], write_call_count=2 - ) - assert result is not None - assert ( - "plan_path = /cwd/.autoskillit/temp/make-plan/task_plan_2026-01-01.md" in result.result - ) - assert "arch_lens_selection" not in result.result - - -# --------------------------------------------------------------------------- -# T3: _build_skill_result integration — CONTRACT_RECOVERY gate -# --------------------------------------------------------------------------- - -from autoskillit.pipeline.audit import DefaultAuditLog, FailureRecord # noqa: E402 - - -@pytest.fixture -def make_build_skill_result_kwargs(): - """Build a dict of kwargs for _build_skill_result from high-level parameters. - - tool_uses entries use the parsed form: 'file_path' at the top level (not nested - in 'input'). The fixture converts them to raw NDJSON tool_use content blocks - with 'input': {'file_path': ...} so parse_session_result (after Step 3) preserves them. - - consecutive_failures_over_budget=True creates an audit with 4 recorded failures - (> max_consecutive_retries=3 default) so the budget guard fires. - """ - from datetime import UTC, datetime - - def _factory( - result_text: str = "done", - completion_marker: str = "", - expected_output_patterns: list[str] | None = None, - tool_uses: list[dict] | None = None, - consecutive_failures_over_budget: bool = False, - ) -> dict: - records = [] - if tool_uses: - content = [] - for tu in tool_uses: - block: dict = {"type": "tool_use", "name": tu["name"], "id": tu.get("id", "")} - fp = tu.get("file_path") - if fp: - block["input"] = {"file_path": fp} - content.append(block) - records.append(json.dumps({"type": "assistant", "message": {"content": content}})) - records.append( - json.dumps( - { - "type": "result", - "subtype": "success", - "is_error": False, - "result": result_text, - "session_id": "test-session", - } - ) - ) - ndjson = "\n".join(records) - result = SubprocessResult( - returncode=0, - stdout=ndjson, - stderr="", - termination=TerminationReason.NATURAL_EXIT, - pid=12345, - ) - audit = None - skill_command = "" - if consecutive_failures_over_budget: - skill_command = "/autoskillit:make-plan" - audit = DefaultAuditLog() - for _ in range(4): # 4 > max_consecutive_retries=3 → budget exhausted - audit.record_failure( - FailureRecord( # type: ignore[arg-type] - timestamp=datetime.now(UTC).isoformat(), - skill_command=skill_command, - exit_code=-1, - subtype="stale", - needs_retry=True, - retry_reason="stale", - stderr="", - ) - ) - return { - "result": result, - "completion_marker": completion_marker, - "expected_output_patterns": expected_output_patterns or [], - "skill_command": skill_command, - "audit": audit, - } - - return _factory - - -class TestContractRecoveryGate: - def test_adjudicated_failure_with_write_evidence_becomes_retriable( - self, make_build_skill_result_kwargs - ): - """ - COMPLETED + marker present + plan_path absent + Write call with no file_path - → synthesis finds no usable path → CONTRACT_RECOVERY gate fires - → needs_retry=True, retry_reason=contract_recovery. - """ - kwargs = make_build_skill_result_kwargs( - result_text="plan summary\n%%ORDER_UP%%", - completion_marker="%%ORDER_UP%%", - expected_output_patterns=[r"plan_path\s*=\s*/.+"], - tool_uses=[ - # Write call exists (write evidence present) but no file_path key - # → synthesis has nothing to inject → CONTRACT_RECOVERY gate fires - {"name": "Write", "id": "t1"} - ], - ) - sr = _build_skill_result(**kwargs) - assert sr.needs_retry is True - assert sr.retry_reason == RetryReason.CONTRACT_RECOVERY - assert sr.success is False - - def test_adjudicated_failure_without_write_evidence_stays_terminal( - self, make_build_skill_result_kwargs - ): - """ - COMPLETED + marker present + plan_path absent + write_call_count == 0 - → success=False, needs_retry=False, subtype=adjudicated_failure (unchanged behavior). - """ - kwargs = make_build_skill_result_kwargs( - result_text="plan summary\n%%ORDER_UP%%", - completion_marker="%%ORDER_UP%%", - expected_output_patterns=[r"plan_path\s*=\s*/.+"], - tool_uses=[], - ) - sr = _build_skill_result(**kwargs) - assert sr.success is False - assert sr.needs_retry is False - assert sr.subtype == "adjudicated_failure" - - def test_synthesis_succeeds_plan_path_token_from_write_file_path( - self, make_build_skill_result_kwargs - ): - """ - COMPLETED + marker present + plan_path absent + Write file_path is absolute - → synthesis injects token → success=True (no CONTRACT_RECOVERY needed). - """ - kwargs = make_build_skill_result_kwargs( - result_text="plan summary\n%%ORDER_UP%%", - completion_marker="%%ORDER_UP%%", - expected_output_patterns=[r"plan_path\s*=\s*/.+"], - tool_uses=[ - {"name": "Write", "id": "t1", "file_path": "/abs/temp/make-plan/my_plan.md"} - ], - ) - sr = _build_skill_result(**kwargs) - assert sr.success is True - assert "plan_path = /abs/temp/make-plan/my_plan.md" in sr.result - - def test_contract_recovery_respects_budget_guard(self, make_build_skill_result_kwargs): - """ - When budget is exhausted, budget guard must override CONTRACT_RECOVERY. - Write call with no file_path → synthesis fails → CONTRACT_RECOVERY promotes - to retriable → budget guard fires (second call inside gate) → needs_retry=False. - """ - kwargs = make_build_skill_result_kwargs( - result_text="plan summary\n%%ORDER_UP%%", - completion_marker="%%ORDER_UP%%", - expected_output_patterns=[r"plan_path\s*=\s*/.+"], - tool_uses=[{"name": "Write", "id": "t1"}], # no file_path → synthesis fails - consecutive_failures_over_budget=True, - ) - sr = _build_skill_result(**kwargs) - assert sr.needs_retry is False - assert sr.retry_reason == RetryReason.BUDGET_EXHAUSTED - - -class TestBuildSkillResultSessionIdFromSubprocess: - """_build_skill_result propagates result.session_id on all paths.""" - - def test_stale_path_session_id_from_subprocess_result(self) -> None: - """Stale path: SkillResult.session_id == result.session_id (not hardcoded '').""" - result = SubprocessResult( - returncode=-1, - stdout="", - stderr="", - termination=TerminationReason.STALE, - pid=1, - session_id="real-uuid-from-channel-b", - ) - sr = _build_skill_result(result) - assert sr.session_id == "real-uuid-from-channel-b" - - def test_timeout_empty_stdout_session_id_from_subprocess_result(self) -> None: - """TIMED_OUT with empty stdout: SkillResult.session_id == result.session_id.""" - result = SubprocessResult( - returncode=-1, - stdout="", - stderr="", - termination=TerminationReason.TIMED_OUT, - pid=1, - session_id="real-uuid-from-channel-b", - ) - sr = _build_skill_result(result) - assert sr.session_id == "real-uuid-from-channel-b" - - def test_channel_a_session_id_takes_precedence(self) -> None: - """When stdout has a result record with session_id, it wins over result.session_id.""" - stdout = json.dumps( - { - "type": "result", - "subtype": "success", - "result": "done", - "session_id": "stdout-uuid", - "is_error": False, - } - ) - result = SubprocessResult( - returncode=0, - stdout=stdout, - stderr="", - termination=TerminationReason.NATURAL_EXIT, - pid=1, - session_id="ch-b-uuid", - ) - sr = _build_skill_result(result) - assert sr.session_id == "stdout-uuid" - - def test_context_exhaustion_session_id_from_subprocess_result(self) -> None: - """Context-exhaustion (no result record): uses result.session_id as fallback.""" - partial_assistant_stdout = json.dumps( - { - "type": "assistant", - "message": {"content": [{"type": "text", "text": "Doing work..."}]}, - } - ) - result = SubprocessResult( - returncode=1, - stdout=partial_assistant_stdout, - stderr="", - termination=TerminationReason.NATURAL_EXIT, - pid=1, - session_id="ch-b-uuid-5678", - ) - sr = _build_skill_result(result) - assert sr.session_id == "ch-b-uuid-5678" - - -class TestHeadlessExecutorCompletionMarker: - """Protocol conformance: DefaultHeadlessExecutor.run accepts completion_marker.""" - - def test_headless_executor_accepts_completion_marker(self) -> None: - import inspect - - from autoskillit.execution.headless import DefaultHeadlessExecutor - - sig = inspect.signature(DefaultHeadlessExecutor.run) - assert "completion_marker" in sig.parameters - param = sig.parameters["completion_marker"] - assert param.default == "" - - -class TestHeadlessExecutorIdleOutputTimeout: - """Protocol conformance and resolution logic for idle_output_timeout.""" - - def test_headless_executor_accepts_idle_output_timeout(self) -> None: - import inspect - - from autoskillit.execution.headless import DefaultHeadlessExecutor - - sig = inspect.signature(DefaultHeadlessExecutor.run) - assert "idle_output_timeout" in sig.parameters - param = sig.parameters["idle_output_timeout"] - assert param.default is None - - def _success_payload(self, marker: str) -> SubprocessResult: - payload = json.dumps( - { - "type": "result", - "subtype": "success", - "is_error": False, - "result": f"Done. {marker}", - "session_id": "sess-iot", - } - ) - return SubprocessResult(0, payload, "", TerminationReason.NATURAL_EXIT, pid=1) - - @pytest.mark.anyio - async def test_default_headless_executor_uses_per_step_idle_output_timeout( - self, tool_ctx - ) -> None: - """idle_output_timeout=120 is converted to float and passed to the runner.""" - from autoskillit.execution.headless import run_headless_core - - marker = tool_ctx.config.run_skill.completion_marker - tool_ctx.runner.push(self._success_payload(marker)) - await run_headless_core( - "/investigate foo", cwd="/tmp", ctx=tool_ctx, idle_output_timeout=120.0 - ) - _, _cwd, _timeout, kwargs = tool_ctx.runner.call_args_list[0] - assert kwargs["idle_output_timeout"] == 120.0 - - @pytest.mark.anyio - async def test_default_headless_executor_converts_zero_to_none(self, tool_ctx) -> None: - """idle_output_timeout=0 is converted to None (disabled) before passing to runner.""" - from autoskillit.execution.headless import run_headless_core - - marker = tool_ctx.config.run_skill.completion_marker - tool_ctx.runner.push(self._success_payload(marker)) - await run_headless_core( - "/investigate foo", cwd="/tmp", ctx=tool_ctx, idle_output_timeout=0.0 - ) - _, _cwd, _timeout, kwargs = tool_ctx.runner.call_args_list[0] - assert kwargs["idle_output_timeout"] is None - - @pytest.mark.anyio - async def test_default_headless_executor_falls_back_to_cfg_idle_output_timeout( - self, tool_ctx - ) -> None: - """idle_output_timeout=None falls back to float(cfg.idle_output_timeout).""" - from autoskillit.execution.headless import run_headless_core - - marker = tool_ctx.config.run_skill.completion_marker - tool_ctx.runner.push(self._success_payload(marker)) - await run_headless_core( - "/investigate foo", cwd="/tmp", ctx=tool_ctx, idle_output_timeout=None - ) - _, _cwd, _timeout, kwargs = tool_ctx.runner.call_args_list[0] - assert kwargs["idle_output_timeout"] == 600.0 - - -def _ndjson_with_write(result_text: str, file_paths: list[str], session_id: str = "test-session"): - """Build NDJSON stdout with Write tool_use entries and a result record.""" - records = [] - if file_paths: - content = [ - { - "type": "tool_use", - "name": "Write", - "id": f"t{i}", - "input": {"file_path": fp}, - } - for i, fp in enumerate(file_paths) - ] - records.append(json.dumps({"type": "assistant", "message": {"content": content}})) - records.append( - json.dumps( - { - "type": "result", - "subtype": "success", - "is_error": False, - "result": result_text, - "session_id": session_id, - } - ) - ) - return "\n".join(records) - - -class TestExtractMissingTokenHints: - def test_extracts_token_and_path(self): - stdout = _ndjson_with_write("plan summary\n%%ORDER_UP%%", ["/tmp/out.md"]) - hints = _extract_missing_token_hints(stdout, [r"plan_path\s*=\s*/.+"]) - assert hints == [("plan_path", "/tmp/out.md")] - - def test_returns_empty_when_pattern_satisfied(self): - stdout = _ndjson_with_write("plan_path = /tmp/out.md\n%%ORDER_UP%%", ["/tmp/out.md"]) - hints = _extract_missing_token_hints(stdout, [r"plan_path\s*=\s*/.+"]) - assert hints == [] - - def test_returns_empty_for_non_path_patterns(self): - stdout = _ndjson_with_write("%%ORDER_UP%%", ["/tmp/out.md"]) - hints = _extract_missing_token_hints(stdout, [r"verdict\s*=\s*\w+"]) - assert hints == [] - - def test_uses_last_write_path(self): - stdout = _ndjson_with_write("%%ORDER_UP%%", ["/tmp/first.md", "/tmp/final.md"]) - hints = _extract_missing_token_hints(stdout, [r"plan_path\s*=\s*/.+"]) - assert hints == [("plan_path", "/tmp/final.md")] - - -class TestContractNudge: - """Integration tests for the contract recovery nudge in run_headless_core. - - The nudge fires when CONTRACT_RECOVERY is triggered with a valid session_id. - CONTRACT_RECOVERY requires synthesis to have failed first. For CHANNEL_A/B - sessions, synthesis is skipped (gated on UNMONITORED), so Write evidence - with file_path triggers CONTRACT_RECOVERY while _extract_missing_token_hints - can still find the file_path for hints. - """ - - def _main_session_ndjson( - self, marker: str, *, include_token: bool = False, session_id: str = "sess-main" - ) -> str: - """Build NDJSON for a CHANNEL_A-confirmed session with Write evidence.""" - result_text = "plan summary\n" - if include_token: - result_text += "plan_path = /tmp/out.md\n" - result_text += marker - return _ndjson_with_write(result_text, ["/tmp/out.md"], session_id=session_id) - - def _main_subprocess_result( - self, marker: str, *, session_id: str = "sess-main" - ) -> SubprocessResult: - """Build a SubprocessResult with CHANNEL_A confirmation to bypass synthesis.""" - return SubprocessResult( - returncode=0, - stdout=self._main_session_ndjson(marker, session_id=session_id), - stderr="", - termination=TerminationReason.NATURAL_EXIT, - pid=1, - channel_confirmation=ChannelConfirmation.CHANNEL_A, - ) - - def _nudge_response_ndjson(self, marker: str, *, include_token: bool = True) -> str: - """Build NDJSON for a nudge response.""" - if include_token: - result_text = f"plan_path = /tmp/out.md\n{marker}" - else: - result_text = f"I cannot do that.\n{marker}" - return json.dumps( - { - "type": "result", - "subtype": "success", - "is_error": False, - "result": result_text, - "session_id": "sess-main", - "usage": {"input_tokens": 100, "output_tokens": 50}, - } - ) - - @pytest.mark.anyio - async def test_nudge_fires_on_contract_recovery(self, tool_ctx): - from autoskillit.execution.headless import run_headless_core - - marker = tool_ctx.config.run_skill.completion_marker - tool_ctx.runner.push(self._main_subprocess_result(marker)) - tool_ctx.runner.push( - SubprocessResult( - 0, self._nudge_response_ndjson(marker), "", TerminationReason.NATURAL_EXIT, pid=2 - ) - ) - result = await run_headless_core( - "/autoskillit:make-plan foo", - cwd="/tmp", - ctx=tool_ctx, - expected_output_patterns=[r"plan_path\s*=\s*/.+"], - ) - assert result.success is True - assert result.needs_retry is False - assert "plan_path = /tmp/out.md" in result.result - - @pytest.mark.anyio - async def test_nudge_failure_falls_through(self, tool_ctx): - from autoskillit.execution.headless import run_headless_core - - marker = tool_ctx.config.run_skill.completion_marker - tool_ctx.runner.push(self._main_subprocess_result(marker)) - tool_ctx.runner.push( - SubprocessResult( - 0, - self._nudge_response_ndjson(marker, include_token=False), - "", - TerminationReason.NATURAL_EXIT, - pid=2, - ) - ) - result = await run_headless_core( - "/autoskillit:make-plan foo", - cwd="/tmp", - ctx=tool_ctx, - expected_output_patterns=[r"plan_path\s*=\s*/.+"], - ) - assert result.retry_reason == RetryReason.CONTRACT_RECOVERY - assert result.needs_retry is True - - @pytest.mark.anyio - async def test_nudge_timeout_falls_through(self, tool_ctx): - from autoskillit.execution.headless import run_headless_core - - marker = tool_ctx.config.run_skill.completion_marker - tool_ctx.runner.push(self._main_subprocess_result(marker)) - tool_ctx.runner.push( - SubprocessResult(1, "", "timeout", TerminationReason.TIMED_OUT, pid=2) - ) - result = await run_headless_core( - "/autoskillit:make-plan foo", - cwd="/tmp", - ctx=tool_ctx, - expected_output_patterns=[r"plan_path\s*=\s*/.+"], - ) - assert result.retry_reason == RetryReason.CONTRACT_RECOVERY - assert result.needs_retry is True - - @pytest.mark.anyio - async def test_nudge_skips_when_budget_exhausted(self, tool_ctx): - from datetime import UTC, datetime - - from autoskillit.core import FailureRecord - from autoskillit.execution.headless import run_headless_core - from autoskillit.pipeline.audit import DefaultAuditLog - - marker = tool_ctx.config.run_skill.completion_marker - audit = DefaultAuditLog() - for _ in range(4): - audit.record_failure( - FailureRecord( # type: ignore[arg-type] - timestamp=datetime.now(UTC).isoformat(), - skill_command="/autoskillit:make-plan foo", - exit_code=-1, - subtype="stale", - needs_retry=True, - retry_reason="stale", - stderr="", - ) - ) - tool_ctx.audit = audit - tool_ctx.runner.push(self._main_subprocess_result(marker)) - result = await run_headless_core( - "/autoskillit:make-plan foo", - cwd="/tmp", - ctx=tool_ctx, - expected_output_patterns=[r"plan_path\s*=\s*/.+"], - ) - assert result.retry_reason == RetryReason.BUDGET_EXHAUSTED - assert len(tool_ctx.runner.call_args_list) == 1 - - @pytest.mark.anyio - async def test_nudge_skips_when_no_session_id(self, tool_ctx): - from autoskillit.execution.headless import run_headless_core - - marker = tool_ctx.config.run_skill.completion_marker - ndjson = _ndjson_with_write(f"plan summary\n{marker}", ["/tmp/out.md"], session_id="") - tool_ctx.runner.push( - SubprocessResult( - 0, - ndjson, - "", - TerminationReason.NATURAL_EXIT, - pid=1, - channel_confirmation=ChannelConfirmation.CHANNEL_A, - ) - ) - result = await run_headless_core( - "/autoskillit:make-plan foo", - cwd="/tmp", - ctx=tool_ctx, - expected_output_patterns=[r"plan_path\s*=\s*/.+"], - ) - assert result.retry_reason == RetryReason.CONTRACT_RECOVERY - assert len(tool_ctx.runner.call_args_list) == 1 - - @pytest.mark.anyio - async def test_nudge_resume_cmd_uses_correct_session_id(self, tool_ctx): - from autoskillit.execution.headless import run_headless_core - - marker = tool_ctx.config.run_skill.completion_marker - tool_ctx.runner.push(self._main_subprocess_result(marker, session_id="sess-abc-123")) - tool_ctx.runner.push( - SubprocessResult( - 0, self._nudge_response_ndjson(marker), "", TerminationReason.NATURAL_EXIT, pid=2 - ) - ) - await run_headless_core( - "/autoskillit:make-plan foo", - cwd="/tmp", - ctx=tool_ctx, - expected_output_patterns=[r"plan_path\s*=\s*/.+"], - ) - assert len(tool_ctx.runner.call_args_list) == 2 - nudge_cmd = tool_ctx.runner.call_args_list[1][0] - assert "--resume" in nudge_cmd - resume_idx = nudge_cmd.index("--resume") - assert nudge_cmd[resume_idx + 1] == "sess-abc-123" - - @pytest.mark.anyio - async def test_nudge_token_usage_merged(self, tool_ctx): - from autoskillit.execution.headless import run_headless_core - - marker = tool_ctx.config.run_skill.completion_marker - main_ndjson_records = [ - json.dumps( - { - "type": "assistant", - "message": { - "content": [ - { - "type": "tool_use", - "name": "Write", - "id": "t0", - "input": {"file_path": "/tmp/out.md"}, - } - ] - }, - } - ), - json.dumps( - { - "type": "result", - "subtype": "success", - "is_error": False, - "result": f"plan summary\n{marker}", - "session_id": "sess-tok", - "usage": {"input_tokens": 1000, "output_tokens": 500}, - } - ), - ] - main_ndjson = "\n".join(main_ndjson_records) - - nudge_ndjson = json.dumps( - { - "type": "result", - "subtype": "success", - "is_error": False, - "result": f"plan_path = /tmp/out.md\n{marker}", - "session_id": "sess-tok", - "usage": {"input_tokens": 200, "output_tokens": 100}, - } - ) - - tool_ctx.runner.push( - SubprocessResult( - 0, - main_ndjson, - "", - TerminationReason.NATURAL_EXIT, - pid=1, - channel_confirmation=ChannelConfirmation.CHANNEL_A, - ) - ) - tool_ctx.runner.push( - SubprocessResult(0, nudge_ndjson, "", TerminationReason.NATURAL_EXIT, pid=2) - ) - - result = await run_headless_core( - "/autoskillit:make-plan foo", - cwd="/tmp", - ctx=tool_ctx, - expected_output_patterns=[r"plan_path\s*=\s*/.+"], - step_name="test-step", - ) - assert result.success is True - assert result.token_usage is not None - assert result.token_usage.get("input_tokens", 0) >= 1200 - assert result.token_usage.get("output_tokens", 0) >= 600 - - @pytest.mark.anyio - async def test_nudge_exception_falls_through(self, tool_ctx): - from autoskillit.execution.headless import run_headless_core - - marker = tool_ctx.config.run_skill.completion_marker - tool_ctx.runner.push(self._main_subprocess_result(marker)) - # Empty stdout nudge → patterns not found → falls through - tool_ctx.runner.push( - SubprocessResult(1, "", "RuntimeError", TerminationReason.NATURAL_EXIT, pid=2) - ) - result = await run_headless_core( - "/autoskillit:make-plan foo", - cwd="/tmp", - ctx=tool_ctx, - expected_output_patterns=[r"plan_path\s*=\s*/.+"], - ) - assert result.retry_reason == RetryReason.CONTRACT_RECOVERY - assert result.needs_retry is True + result = _inject_narration_suppression("cmd", has_skill_prefix=True) + assert "EFFICIENCY DIRECTIVE" in result + assert "between tool calls" in result diff --git a/tests/execution/test_headless_debug_logging.py b/tests/execution/test_headless_debug_logging.py index b7dce26af8..073bf0d310 100644 --- a/tests/execution/test_headless_debug_logging.py +++ b/tests/execution/test_headless_debug_logging.py @@ -4,11 +4,13 @@ import json +import pytest import structlog.testing -from autoskillit.config import AutomationConfig, ModelConfig from autoskillit.core.types import SubprocessResult, TerminationReason +pytestmark = [pytest.mark.layer("execution"), pytest.mark.small] + def _sr(returncode=0, stdout="", stderr="", termination=TerminationReason.NATURAL_EXIT): """Build a minimal SubprocessResult for _build_skill_result tests.""" @@ -85,8 +87,10 @@ class TestResolveModelLogging: """Verify _resolve_model logs which priority tier resolved the model.""" def _make_config(self, *, override=None, default=None): - cfg = AutomationConfig() - cfg.model = ModelConfig(default=default, override=override) + from tests._helpers import make_model_config, make_test_config + + cfg = make_test_config() + cfg.model = make_model_config(default=default, override=override) return cfg def test_logs_override_tier(self): diff --git a/tests/execution/test_headless_dispatch.py b/tests/execution/test_headless_dispatch.py new file mode 100644 index 0000000000..6a63ac66c8 --- /dev/null +++ b/tests/execution/test_headless_dispatch.py @@ -0,0 +1,373 @@ +"""Tests for headless.py dispatch flow: food truck dispatch, pack injection, executor protocol.""" + +import json +from pathlib import Path + +import pytest + +from autoskillit.core.types._type_plugin_source import DirectInstall, MarketplaceInstall + +pytestmark = [pytest.mark.layer("execution"), pytest.mark.small] + + +def _make_success_stdout(marker: str = "%%FT_DONE%%") -> str: + return json.dumps( + { + "type": "result", + "subtype": "success", + "result": f"L3 done {marker}", + "session_id": "ft-session", + "is_error": False, + } + ) + + +class TestDispatchFoodTruck: + """Tests for DefaultHeadlessExecutor.dispatch_food_truck.""" + + @pytest.mark.anyio + async def test_dispatch_food_truck_calls_runner(self, minimal_ctx, tmp_path: Path): + from autoskillit.core.types import SubprocessResult, TerminationReason + from autoskillit.execution.headless import DefaultHeadlessExecutor + from tests.fakes import MockSubprocessRunner + + runner = MockSubprocessRunner() + runner.set_default( + SubprocessResult( + returncode=0, + stdout=_make_success_stdout(), + stderr="", + termination=TerminationReason.NATURAL_EXIT, + pid=55555, + ) + ) + minimal_ctx.runner = runner + minimal_ctx.plugin_source = DirectInstall(plugin_dir=tmp_path) + + executor = DefaultHeadlessExecutor(minimal_ctx) + await executor.dispatch_food_truck( + "You are an L3 orchestrator", + str(tmp_path), + completion_marker="%%FT_DONE%%", + ) + + assert runner.call_args_list, "runner was never called" + cmd, _cwd, _timeout, kwargs = runner.call_args_list[0] + env = kwargs.get("env") + assert env is not None + assert env["AUTOSKILLIT_SESSION_TYPE"] == "orchestrator" + assert env["AUTOSKILLIT_HEADLESS"] == "1" + assert "--tools" in cmd + assert "AskUserQuestion" in cmd + + @pytest.mark.anyio + async def test_dispatch_food_truck_returns_skill_result(self, minimal_ctx, tmp_path: Path): + from autoskillit.core.types import SkillResult, SubprocessResult, TerminationReason + from autoskillit.execution.headless import DefaultHeadlessExecutor + from tests.fakes import MockSubprocessRunner + + runner = MockSubprocessRunner() + runner.set_default( + SubprocessResult( + returncode=0, + stdout=_make_success_stdout(), + stderr="", + termination=TerminationReason.NATURAL_EXIT, + pid=55555, + ) + ) + minimal_ctx.runner = runner + minimal_ctx.plugin_source = DirectInstall(plugin_dir=tmp_path) + + executor = DefaultHeadlessExecutor(minimal_ctx) + result = await executor.dispatch_food_truck( + "You are an L3 orchestrator", + str(tmp_path), + completion_marker="%%FT_DONE%%", + ) + + assert isinstance(result, SkillResult) + assert result.success is True + + @pytest.mark.anyio + async def test_dispatch_food_truck_on_spawn_receives_pid(self, minimal_ctx, tmp_path: Path): + from autoskillit.core.types import SubprocessResult, TerminationReason + from autoskillit.execution.headless import DefaultHeadlessExecutor + from tests.fakes import MockSubprocessRunner + + runner = MockSubprocessRunner() + runner.set_default( + SubprocessResult( + returncode=0, + stdout=_make_success_stdout(), + stderr="", + termination=TerminationReason.NATURAL_EXIT, + pid=55555, + ) + ) + minimal_ctx.runner = runner + minimal_ctx.plugin_source = DirectInstall(plugin_dir=tmp_path) + + spawned_pids: list[int] = [] + + executor = DefaultHeadlessExecutor(minimal_ctx) + await executor.dispatch_food_truck( + "You are an L3 orchestrator", + str(tmp_path), + completion_marker="%%FT_DONE%%", + on_spawn=lambda pid, ticks: spawned_pids.append(pid), + ) + + assert spawned_pids == [55555] + + @pytest.mark.anyio + async def test_dispatch_food_truck_on_spawn_not_required(self, minimal_ctx, tmp_path: Path): + from autoskillit.core.types import SkillResult, SubprocessResult, TerminationReason + from autoskillit.execution.headless import DefaultHeadlessExecutor + from tests.fakes import MockSubprocessRunner + + runner = MockSubprocessRunner() + runner.set_default( + SubprocessResult( + returncode=0, + stdout=_make_success_stdout(), + stderr="", + termination=TerminationReason.NATURAL_EXIT, + pid=55555, + ) + ) + minimal_ctx.runner = runner + minimal_ctx.plugin_source = DirectInstall(plugin_dir=tmp_path) + + executor = DefaultHeadlessExecutor(minimal_ctx) + result = await executor.dispatch_food_truck( + "You are an L3 orchestrator", + str(tmp_path), + completion_marker="%%FT_DONE%%", + on_spawn=None, + ) + + assert isinstance(result, SkillResult) + assert result.success is True + + @pytest.mark.anyio + async def test_dispatch_food_truck_marketplace_install_does_not_raise( + self, minimal_ctx, tmp_path: Path + ): + """dispatch_food_truck with MarketplaceInstall resolves cache_path — no ValueError.""" + from autoskillit.core.types import SubprocessResult, TerminationReason + from autoskillit.execution.headless import DefaultHeadlessExecutor + from tests.fakes import MockSubprocessRunner + + cache = tmp_path / "marketplace_cache" + cache.mkdir() + runner = MockSubprocessRunner() + runner.set_default( + SubprocessResult( + returncode=0, + stdout=_make_success_stdout(), + stderr="", + termination=TerminationReason.NATURAL_EXIT, + pid=55555, + ) + ) + minimal_ctx.runner = runner + minimal_ctx.plugin_source = MarketplaceInstall(cache_path=cache) + + executor = DefaultHeadlessExecutor(minimal_ctx) + await executor.dispatch_food_truck( + "You are an L3 orchestrator", + str(tmp_path), + completion_marker="%%FT_DONE%%", + ) + cmd, _cwd, _timeout, _kwargs = runner.call_args_list[0] + assert "--plugin-dir" in cmd + assert str(cache) in cmd + + @pytest.mark.anyio + async def test_dispatch_food_truck_passes_resume_session_id_to_cmd_builder( + self, minimal_ctx, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ): + """resume_session_id is forwarded from dispatch_food_truck to build_food_truck_cmd.""" + from autoskillit.core.types import SubprocessResult, TerminationReason + from autoskillit.execution.headless import DefaultHeadlessExecutor + from tests.fakes import MockSubprocessRunner + + captured_kwargs: list[dict] = [] + + import autoskillit.execution.headless as _headless_mod + + original_build = _headless_mod.build_food_truck_cmd + + def _spy(**kwargs): + captured_kwargs.append(kwargs) + return original_build(**kwargs) + + monkeypatch.setattr(_headless_mod, "build_food_truck_cmd", _spy) + + runner = MockSubprocessRunner() + runner.set_default( + SubprocessResult( + returncode=0, + stdout=_make_success_stdout(), + stderr="", + termination=TerminationReason.NATURAL_EXIT, + pid=55555, + ) + ) + minimal_ctx.runner = runner + minimal_ctx.plugin_source = DirectInstall(plugin_dir=tmp_path) + + executor = DefaultHeadlessExecutor(minimal_ctx) + await executor.dispatch_food_truck( + "You are an L3 orchestrator", + str(tmp_path), + completion_marker="%%FT_DONE%%", + resume_session_id="abc-123", + ) + + assert captured_kwargs, "build_food_truck_cmd was never called" + assert captured_kwargs[0]["resume_session_id"] == "abc-123" + + +class TestDispatchFoodTruckPackInjection: + """Tests that dispatch_food_truck correctly injects AUTOSKILLIT_FOOD_TRUCK_TOOL_TAGS.""" + + @pytest.mark.anyio + async def test_requires_packs_injected_as_l3_tool_tags(self, minimal_ctx, tmp_path: Path): + """dispatch_food_truck with requires_packs injects sorted comma-joined env var.""" + from autoskillit.core.types import SubprocessResult, TerminationReason + from autoskillit.execution.headless import DefaultHeadlessExecutor + from tests.fakes import MockSubprocessRunner + + runner = MockSubprocessRunner() + runner.set_default( + SubprocessResult( + returncode=0, + stdout=_make_success_stdout(), + stderr="", + termination=TerminationReason.NATURAL_EXIT, + pid=12345, + ) + ) + minimal_ctx.runner = runner + minimal_ctx.plugin_source = DirectInstall(plugin_dir=tmp_path) + + executor = DefaultHeadlessExecutor(minimal_ctx) + await executor.dispatch_food_truck( + "You are an L3 orchestrator", + str(tmp_path), + completion_marker="%%FT_DONE%%", + requires_packs=["ci", "github", "clone"], + ) + + assert runner.call_args_list, "runner was never called" + _cmd, _cwd, _timeout, kwargs = runner.call_args_list[0] + env = kwargs.get("env") + assert env is not None + assert env["AUTOSKILLIT_FOOD_TRUCK_TOOL_TAGS"] == "ci,clone,github" + + @pytest.mark.anyio + async def test_requires_packs_empty_omits_l3_tool_tags(self, minimal_ctx, tmp_path: Path): + """dispatch_food_truck with empty requires_packs does not inject FOOD_TRUCK_TOOL_TAGS.""" + from autoskillit.core.types import SubprocessResult, TerminationReason + from autoskillit.execution.headless import DefaultHeadlessExecutor + from tests.fakes import MockSubprocessRunner + + runner = MockSubprocessRunner() + runner.set_default( + SubprocessResult( + returncode=0, + stdout=_make_success_stdout(), + stderr="", + termination=TerminationReason.NATURAL_EXIT, + pid=12345, + ) + ) + minimal_ctx.runner = runner + minimal_ctx.plugin_source = DirectInstall(plugin_dir=tmp_path) + + executor = DefaultHeadlessExecutor(minimal_ctx) + await executor.dispatch_food_truck( + "You are an L3 orchestrator", + str(tmp_path), + completion_marker="%%FT_DONE%%", + requires_packs=[], + ) + + assert runner.call_args_list, "runner was never called" + _cmd, _cwd, _timeout, kwargs = runner.call_args_list[0] + env = kwargs.get("env") + assert env is not None + assert "AUTOSKILLIT_FOOD_TRUCK_TOOL_TAGS" not in env + + +class TestDispatchFoodTruckGuards: + """Guard-path tests for dispatch_food_truck: conflict detection and skip_clone_guard.""" + + @pytest.mark.anyio + async def test_dispatch_food_truck_l3_tool_tags_conflict_raises( + self, minimal_ctx, tmp_path: Path + ) -> None: + from autoskillit.core.types._type_plugin_source import DirectInstall + from autoskillit.execution.headless import DefaultHeadlessExecutor + + minimal_ctx.plugin_source = DirectInstall(plugin_dir=tmp_path) + executor = DefaultHeadlessExecutor(minimal_ctx) + + with pytest.raises(ValueError, match="AUTOSKILLIT_FOOD_TRUCK_TOOL_TAGS"): + await executor.dispatch_food_truck( + "some prompt", + str(tmp_path), + completion_marker="DONE", + requires_packs=["ci"], + env_extras={"AUTOSKILLIT_FOOD_TRUCK_TOOL_TAGS": "ci"}, + ) + + @pytest.mark.anyio + async def test_dispatch_food_truck_skip_clone_guard_prevents_snapshot( + self, minimal_ctx, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + from unittest.mock import AsyncMock + + from autoskillit.core.types import SubprocessResult, TerminationReason + from autoskillit.core.types._type_plugin_source import DirectInstall + from autoskillit.execution.headless import DefaultHeadlessExecutor + from tests.fakes import MockSubprocessRunner + + mock_snapshot = AsyncMock() + monkeypatch.setattr( + "autoskillit.execution.headless.snapshot_clone_state", + mock_snapshot, + ) + + runner = MockSubprocessRunner() + runner.set_default( + SubprocessResult( + returncode=0, + stdout=_make_success_stdout(), + stderr="", + termination=TerminationReason.NATURAL_EXIT, + pid=12345, + ) + ) + minimal_ctx.runner = runner + minimal_ctx.plugin_source = DirectInstall(plugin_dir=tmp_path) + executor = DefaultHeadlessExecutor(minimal_ctx) + + await executor.dispatch_food_truck( + "some prompt", + str(tmp_path), + completion_marker="DONE", + ) + + assert mock_snapshot.call_count == 0 + + +def test_default_executor_satisfies_protocol_with_dispatch(minimal_ctx) -> None: + """DefaultHeadlessExecutor satisfies HeadlessExecutor protocol with dispatch_food_truck.""" + from autoskillit.core import HeadlessExecutor + from autoskillit.execution.headless import DefaultHeadlessExecutor + + executor = DefaultHeadlessExecutor(minimal_ctx) + assert isinstance(executor, HeadlessExecutor) diff --git a/tests/execution/test_headless_env_injection.py b/tests/execution/test_headless_env_injection.py index c1d3dade64..41c0c51599 100644 --- a/tests/execution/test_headless_env_injection.py +++ b/tests/execution/test_headless_env_injection.py @@ -8,17 +8,17 @@ import pytest from autoskillit.core.types import SubprocessResult, TerminationReason -from tests.conftest import MockSubprocessRunner +from tests.fakes import MockSubprocessRunner + +pytestmark = [pytest.mark.layer("execution"), pytest.mark.small] @pytest.mark.anyio -async def test_headless_command_includes_headless_env_var(tmp_path: Path) -> None: +async def test_headless_command_includes_headless_env_var(minimal_ctx, tmp_path: Path) -> None: """run_headless_core must inject AUTOSKILLIT_HEADLESS=1 into the subprocess command.""" - from autoskillit.config import AutomationConfig from autoskillit.execution.headless import run_headless_core - from autoskillit.pipeline.gate import DefaultGateState - from autoskillit.server._factory import make_context + minimal_ctx.runner = MockSubprocessRunner() success_result = json.dumps( { "type": "result", @@ -28,8 +28,7 @@ async def test_headless_command_includes_headless_env_var(tmp_path: Path) -> Non "is_error": False, } ) - mock_runner = MockSubprocessRunner() - mock_runner.set_default( + minimal_ctx.runner.set_default( SubprocessResult( returncode=0, stdout=success_result, @@ -39,14 +38,10 @@ async def test_headless_command_includes_headless_env_var(tmp_path: Path) -> Non ) ) - ctx = make_context(AutomationConfig(), runner=mock_runner, plugin_dir=str(tmp_path)) - ctx.gate = DefaultGateState(enabled=True) - ctx.config.linux_tracing.log_dir = str(tmp_path / "logs") - - await run_headless_core("/investigate foo", str(tmp_path), ctx) + await run_headless_core("/investigate foo", str(tmp_path), minimal_ctx) - assert mock_runner.call_args_list, "runner was never called" - cmd, _cwd, _timeout, kwargs = mock_runner.call_args_list[0] + assert minimal_ctx.runner.call_args_list, "runner was never called" + cmd, _cwd, _timeout, kwargs = minimal_ctx.runner.call_args_list[0] env = kwargs.get("env") assert env is not None assert env["AUTOSKILLIT_HEADLESS"] == "1", ( diff --git a/tests/execution/test_headless_env_scrub.py b/tests/execution/test_headless_env_scrub.py index 4f31ee397b..1d0214d316 100644 --- a/tests/execution/test_headless_env_scrub.py +++ b/tests/execution/test_headless_env_scrub.py @@ -11,31 +11,27 @@ import pytest -from tests.conftest import MockSubprocessRunner, _make_result +from tests.conftest import _make_result + +pytestmark = [pytest.mark.layer("execution"), pytest.mark.small] @pytest.mark.anyio async def test_run_headless_core_env_excludes_ide_vars( - monkeypatch: pytest.MonkeyPatch, tmp_path: Path + tool_ctx, monkeypatch: pytest.MonkeyPatch, tmp_path: Path ) -> None: monkeypatch.setenv("CLAUDE_CODE_SSE_PORT", "23270") monkeypatch.setenv("ENABLE_IDE_INTEGRATION", "true") monkeypatch.setenv("CLAUDE_CODE_IDE_HOST_OVERRIDE", "host") - from autoskillit.config import AutomationConfig from autoskillit.execution.headless import run_headless_core - from autoskillit.pipeline import DefaultGateState - from autoskillit.server._factory import make_context - mock_runner = MockSubprocessRunner() - mock_runner.set_default(_make_result()) - ctx = make_context(AutomationConfig(), runner=mock_runner, plugin_dir=str(tmp_path)) - ctx.gate = DefaultGateState(enabled=True) + tool_ctx.runner.set_default(_make_result()) - await run_headless_core("/investigate foo", str(tmp_path), ctx) + await run_headless_core("/investigate foo", str(tmp_path), tool_ctx) - assert mock_runner.call_args_list, "runner was never called" - _cmd, _cwd, _timeout, kwargs = mock_runner.call_args_list[0] + assert tool_ctx.runner.call_args_list, "runner was never called" + _cmd, _cwd, _timeout, kwargs = tool_ctx.runner.call_args_list[0] env = kwargs.get("env") assert env is not None assert "CLAUDE_CODE_SSE_PORT" not in env diff --git a/tests/execution/test_headless_ordering.py b/tests/execution/test_headless_ordering.py new file mode 100644 index 0000000000..e9944292f6 --- /dev/null +++ b/tests/execution/test_headless_ordering.py @@ -0,0 +1,103 @@ +"""AST-based structural test for post-session operation ordering in headless.py. + +T-ORD-1: _compute_loc_changed must not be called before _build_skill_result + in _execute_claude_headless. After the fix, the call is wrapped in + _compute_post_session_metrics which is invoked after _build_skill_result. +""" + +from __future__ import annotations + +import ast +from pathlib import Path + +import pytest + +pytestmark = [pytest.mark.layer("execution"), pytest.mark.small] + +_HEADLESS_PATH = ( + Path(__file__).parent.parent.parent + / "src" + / "autoskillit" + / "execution" + / "headless" + / "__init__.py" +) + + +def _find_first_call_line(func_body: list[ast.stmt], call_name: str) -> int | None: + """Return the first line number of a direct Call to call_name in func_body. + + Uses a source-order DFS (children sorted by lineno) so nested calls are + visited in the order they appear in the source file rather than BFS order. + """ + + def _dfs(node: ast.AST) -> int | None: + if isinstance(node, ast.Call): + func = node.func + name = None + if isinstance(func, ast.Name): + name = func.id + elif isinstance(func, ast.Attribute): + name = func.attr + if name == call_name: + return node.lineno + children = sorted( + ast.iter_child_nodes(node), + key=lambda n: getattr(n, "lineno", 0), + ) + for child in children: + result = _dfs(child) + if result is not None: + return result + return None + + return _dfs(ast.Module(body=func_body, type_ignores=[])) + + +# T-ORD-1 +def test_compute_loc_changed_called_after_build_skill_result(): + """_compute_loc_changed must not appear before _build_skill_result in _execute_claude_headless. + + After the fix the direct call to _compute_loc_changed is replaced by + _compute_post_session_metrics, which must itself appear after _build_skill_result. + """ + source = _HEADLESS_PATH.read_text() + tree = ast.parse(source) + + target_func: ast.AsyncFunctionDef | None = None + for node in ast.walk(tree): + if isinstance(node, ast.AsyncFunctionDef) and node.name == "_execute_claude_headless": + target_func = node + break + + assert target_func is not None, "_execute_claude_headless not found in headless.py" + + build_result_line = _find_first_call_line(target_func.body, "_build_skill_result") + assert build_result_line is not None, ( + "_build_skill_result call not found in _execute_claude_headless" + ) + + # Direct call to _compute_loc_changed must not appear before _build_skill_result. + # After the fix, it should not appear at all in _execute_claude_headless (it is + # delegated to _compute_post_session_metrics). + loc_changed_line = _find_first_call_line(target_func.body, "_compute_loc_changed") + # Expected post-fix state: _compute_loc_changed is absent from _execute_claude_headless + # (delegated to _compute_post_session_metrics). If it reappears, the ordering + # assertion below catches it. If it is absent, that is intentional and correct. + if loc_changed_line is not None: + assert loc_changed_line > build_result_line, ( + f"_compute_loc_changed (line {loc_changed_line}) must appear after " + f"_build_skill_result (line {build_result_line}) in _execute_claude_headless. " + "LoC measurement must use the constructed SkillResult to resolve effective_cwd." + ) + + # _compute_post_session_metrics must appear after _build_skill_result. + metrics_line = _find_first_call_line(target_func.body, "_compute_post_session_metrics") + assert metrics_line is not None, ( + "_compute_post_session_metrics not found in _execute_claude_headless — " + "the post-session metrics factory must be wired up." + ) + assert metrics_line > build_result_line, ( + f"_compute_post_session_metrics (line {metrics_line}) must appear after " + f"_build_skill_result (line {build_result_line}) in _execute_claude_headless." + ) diff --git a/tests/execution/test_headless_path_validation.py b/tests/execution/test_headless_path_validation.py new file mode 100644 index 0000000000..077fe5e929 --- /dev/null +++ b/tests/execution/test_headless_path_validation.py @@ -0,0 +1,1451 @@ +"""Tests for headless.py: _build_skill_result, path validation, synthesis, and contract gates.""" + +import json + +import pytest + +from autoskillit.core.types import ( + ChannelConfirmation, + RetryReason, + SubprocessResult, + TerminationReason, +) +from autoskillit.execution.headless import ( + _build_skill_result, + _extract_missing_token_hints, +) +from autoskillit.execution.session import ClaudeSessionResult +from autoskillit.pipeline.audit import DefaultAuditLog, FailureRecord +from tests.conftest import _make_result +from tests.execution.conftest import _make_tool_use_line, _success_session_json + +pytestmark = [pytest.mark.layer("execution"), pytest.mark.small] + + +class TestBuildSkillResultChannelAPatternRecovery: + """_build_skill_result must extend pattern recovery to CHANNEL_A wins, not just CHANNEL_B.""" + + def test_build_skill_result_channel_a_recovers_from_assistant_messages(self) -> None: + """_build_skill_result must attempt pattern recovery for CHANNEL_A, not just CHANNEL_B.""" + block = "verdict = GO\n%%ORDER_UP%%" + assistant_line = json.dumps( + { + "type": "assistant", + "message": { + "content": [{"type": "text", "text": block}], + }, + } + ) + result_line = json.dumps( + { + "type": "result", + "subtype": "success", + "is_error": False, + "result": "Done. %%ORDER_UP%%", + "session_id": "s1", + "errors": [], + } + ) + stdout = assistant_line + "\n" + result_line + sub_result = SubprocessResult( + returncode=0, + stdout=stdout, + stderr="", + termination=TerminationReason.NATURAL_EXIT, + pid=12345, + channel_confirmation=ChannelConfirmation.CHANNEL_A, + ) + sr = _build_skill_result( + sub_result, + completion_marker="%%ORDER_UP%%", + expected_output_patterns=["verdict\\s*=\\s*(GO|NO GO)"], + ) + assert sr.success is True, ( + "CHANNEL_A should recover patterns from assistant_messages, same as CHANNEL_B." + ) + + def test_build_skill_result_channel_a_pattern_contract_violation_is_terminal( + self, + ) -> None: + """After failed CHANNEL_A recovery, contract violation must be FAILED, not RETRIABLE.""" + assistant_line = json.dumps( + { + "type": "assistant", + "message": { + "content": [{"type": "text", "text": "No verdict here. %%ORDER_UP%%"}], + }, + } + ) + result_line = json.dumps( + { + "type": "result", + "subtype": "success", + "is_error": False, + "result": "Done. %%ORDER_UP%%", + "session_id": "s1", + "errors": [], + } + ) + stdout = assistant_line + "\n" + result_line + sub_result = SubprocessResult( + returncode=0, + stdout=stdout, + stderr="", + termination=TerminationReason.NATURAL_EXIT, + pid=12345, + channel_confirmation=ChannelConfirmation.CHANNEL_A, + ) + sr = _build_skill_result( + sub_result, + completion_marker="%%ORDER_UP%%", + expected_output_patterns=["verdict\\s*=\\s*(GO|NO GO)"], + ) + assert sr.success is False + assert sr.needs_retry is False + assert sr.subtype == "adjudicated_failure" + + def test_bold_wrapped_plan_path_returns_success_not_adjudicated_failure( + self, + ) -> None: + """Regression test for issue #462: model emitting **plan_path** = /abs/path + must produce success=True, not adjudicated_failure.""" + result_text = ( + "The implementation plan has been written.\n\n" + "**plan_path** = /abs/path/plan.md\n" + "**plan_parts** = /abs/path/plan.md\n" + "%%ORDER_UP%%" + ) + sub_result = SubprocessResult( + returncode=0, + stdout=_success_session_json(result_text), + stderr="", + termination=TerminationReason.NATURAL_EXIT, + pid=12345, + ) + sr = _build_skill_result( + sub_result, + completion_marker="%%ORDER_UP%%", + expected_output_patterns=["plan_path\\s*=\\s*/.+"], + ) + assert sr.success is True + assert sr.subtype != "adjudicated_failure" + assert sr.needs_retry is False + + def test_bold_wrapped_tokens_in_assistant_messages_recovery_succeeds( + self, + ) -> None: + """Recovery path: bold-wrapped tokens in assistant_messages must also be found. + The pattern_recovery block in _build_skill_result requires channel_confirmation + != UNMONITORED to trigger; use CHANNEL_A to activate the recovery path.""" + token_block = "**plan_path** = /abs/path/plan.md\n**plan_parts** = /abs/path/plan.md" + assistant_line = json.dumps( + { + "type": "assistant", + "message": { + "content": [{"type": "text", "text": token_block}], + }, + } + ) + result_line = json.dumps( + { + "type": "result", + "subtype": "success", + "is_error": False, + "result": "The plan is complete. %%ORDER_UP%%", + "session_id": "s1", + "errors": [], + } + ) + stdout = assistant_line + "\n" + result_line + sub_result = SubprocessResult( + returncode=0, + stdout=stdout, + stderr="", + termination=TerminationReason.NATURAL_EXIT, + pid=12345, + channel_confirmation=ChannelConfirmation.CHANNEL_A, + ) + sr = _build_skill_result( + sub_result, + completion_marker="%%ORDER_UP%%", + expected_output_patterns=["plan_path\\s*=\\s*/.+"], + ) + assert sr.success is True + assert sr.subtype != "adjudicated_failure" + assert sr.needs_retry is False + + +class TestBuildSkillResultDirMissingRecovery: + """DIR_MISSING channel confirmation does not silently pass — it attempts + late-bind recovery when conditions allow.""" + + def test_dir_missing_with_recoverable_subtype_attempts_recovery(self): + """When channel_confirmation is DIR_MISSING and subtype is recoverable, + the recovery gate must attempt marker-based recovery. + + Setup: termination=COMPLETED (bypasses stale branch), subtype=empty_output + (in _CHANNEL_B_RECOVERABLE_SUBTYPES), marker on a standalone line in the + assistant message (required by _marker_is_standalone). + """ + import structlog.testing + + marker = "===DONE===" + # Marker must appear as a standalone line for _marker_is_standalone to match + assistant_line = json.dumps( + { + "type": "assistant", + "message": { + "content": [{"type": "text", "text": f"Task complete.\n\n{marker}"}], + }, + } + ) + # subtype=empty_output places the session in _CHANNEL_B_RECOVERABLE_SUBTYPES, + # triggering the DIR_MISSING recovery guard in _build_skill_result + result_line = json.dumps( + { + "type": "result", + "subtype": "empty_output", + "is_error": True, + "result": "", + "session_id": "s1", + "errors": [], + } + ) + stdout = assistant_line + "\n" + result_line + # termination=COMPLETED skips the stale-branch early return so execution + # reaches the Channel B / DIR_MISSING recovery gate + sub_result = SubprocessResult( + returncode=0, + stdout=stdout, + stderr="", + termination=TerminationReason.COMPLETED, + pid=12345, + channel_confirmation=ChannelConfirmation.DIR_MISSING, + ) + with structlog.testing.capture_logs() as logs: + skill = _build_skill_result(sub_result, completion_marker=marker) + # Recovery should succeed — marker found as standalone line in assistant_messages + assert skill.success is True + # Verify the recovery code path was taken, not just the outcome + assert any(e.get("event") == "dir_missing_late_bind_recovery" for e in logs) + + def test_dir_missing_without_marker_does_not_recover(self): + """DIR_MISSING with no completion marker must not silently succeed.""" + sub_result = SubprocessResult( + returncode=0, + stdout="", + stderr="", + termination=TerminationReason.STALE, + pid=12345, + channel_confirmation=ChannelConfirmation.DIR_MISSING, + ) + skill = _build_skill_result(sub_result) + assert skill.success is False + + +class TestTimedOutSessionPreservesState: + """TIMED_OUT branch must parse stdout to preserve tool_uses and assistant_messages.""" + + def test_timed_out_with_writes_preserves_write_call_count(self): + """Timed-out session with Write/Edit tool_use blocks must report write_call_count > 0.""" + from autoskillit.execution.headless import _build_skill_result + + ndjson = "\n".join( + [ + _make_tool_use_line("Write", {"file_path": "/tmp/a.py", "content": "x"}), + _make_tool_use_line( + "Edit", {"file_path": "/tmp/b.py", "old_string": "a", "new_string": "b"} + ), + ] + ) + sub_result = SubprocessResult( + returncode=-1, + stdout=ndjson, + stderr="", + termination=TerminationReason.TIMED_OUT, + pid=12345, + ) + sr = _build_skill_result(sub_result) + assert sr.write_call_count == 2 + assert sr.success is False + assert sr.needs_retry is False + + def test_timed_out_with_empty_stdout_uses_timeout_subtype(self): + """Timed-out session with no stdout uses TIMEOUT subtype.""" + from autoskillit.execution.headless import _build_skill_result + + sub_result = SubprocessResult( + returncode=-1, + stdout="", + stderr="", + termination=TerminationReason.TIMED_OUT, + pid=12345, + ) + sr = _build_skill_result(sub_result) + assert sr.success is False + assert sr.write_call_count == 0 + + def test_timed_out_with_success_result_overrides_to_timeout(self): + """When timed-out stdout has a success result, subtype is overridden to timeout.""" + from autoskillit.execution.headless import _build_skill_result + + ndjson = "\n".join( + [ + _make_tool_use_line("Write", {"file_path": "/tmp/a.py", "content": "x"}), + json.dumps( + { + "type": "result", + "subtype": "success", + "is_error": False, + "result": "Task completed.", + "session_id": "s1", + } + ), + ] + ) + sub_result = SubprocessResult( + returncode=-1, + stdout=ndjson, + stderr="", + termination=TerminationReason.TIMED_OUT, + pid=12345, + ) + sr = _build_skill_result(sub_result) + assert sr.cli_subtype == "timeout" + assert sr.write_call_count == 1 + + +class TestOutputPathTokensDerivedFromContracts: + # Hardcoded fixture — update when skill_contracts.yaml gains new file_path outputs. + # Using a fixed set (rather than re-deriving) ensures bugs in the derivation formula + # cause test failures rather than silent agreement between production and test code. + _EXPECTED_OUTPUT_PATH_TOKENS = frozenset( + { + "analysis_file", + "campaign_path", + "conflict_report_path", + "diagnosis_path", + "diagram_path", + "evaluation_dashboard", + "experiment_plan", + "group_files", + "groups_path", + "investigation_path", + "manifest_path", + "plan_parts", + "plan_path", + "pr_order_file", + "prep_path", + "recipe_path", + "remediation_path", + "report_path", + "report_plan_path", + "results_path", + "review_path", + "revision_guidance", + "scope_report", + "summary_path", + "triage_manifest", + "triage_report", + "visualization_plan_path", + "visualization_plan_trace_path", + "html_path", + "resource_report", + "execution_map", + "execution_map_report", + # planner-generate-phases output (planner recipe) + "phase_manifest_path", + # planner-elaborate-phase output (parallel worker) + "elab_result_path", + # planner-refine-phases output + "refined_plan_path", + # planner-refine-assignments output + "phase_refined_path", + # planner-refine-wps output + "refined_wps_path", + # planner-validate-task-alignment output + "alignment_findings_path", + # planner-assess-review-approach output + "review_approach_assessment_path", + # audit-tests output (bundled full-audit recipe) + "audit_report_path", + # validate-audit output (bundled full-audit recipe) + "validated_report_path", + # promote-to-main skill output (promote-to-main-wrapper recipe) + "pr_body_path", + # setup-environment skill output (research recipe pre-flight gate) + "env_report", + } + ) + + def test_output_path_tokens_contains_all_file_path_contract_outputs(self) -> None: + """Every skill output declared with type=file_path or type=file_path_list in + skill_contracts.yaml must appear in _OUTPUT_PATH_TOKENS (or be documented as + intentionally excluded).""" + from autoskillit.execution.headless import ( + _INTENTIONALLY_EXCLUDED_PATH_TOKENS, + _OUTPUT_PATH_TOKENS, + ) + from autoskillit.recipe.contracts import load_bundled_manifest + + manifest = load_bundled_manifest() + declared_path_tokens = { + out["name"] + for skill_data in manifest.get("skills", {}).values() + for out in skill_data.get("outputs", []) + if isinstance(out, dict) and out.get("type", "").startswith("file_path") + } + untracked = ( + declared_path_tokens - _OUTPUT_PATH_TOKENS - _INTENTIONALLY_EXCLUDED_PATH_TOKENS + ) + assert not untracked, ( + f"These path tokens are declared in skill_contracts.yaml but missing from " + f"_OUTPUT_PATH_TOKENS or _INTENTIONALLY_EXCLUDED_PATH_TOKENS: {untracked}" + ) + + def test_output_path_tokens_matches_expected_fixture(self) -> None: + """_OUTPUT_PATH_TOKENS must exactly match the known fixture set. + + This guards against bugs in the derivation formula: if _build_path_token_set() + is broken, both the production frozenset and a re-derived set would agree, but + this hardcoded fixture would not. + """ + from autoskillit.execution.headless import _OUTPUT_PATH_TOKENS + + extra = _OUTPUT_PATH_TOKENS - self._EXPECTED_OUTPUT_PATH_TOKENS + missing = self._EXPECTED_OUTPUT_PATH_TOKENS - _OUTPUT_PATH_TOKENS + assert _OUTPUT_PATH_TOKENS == self._EXPECTED_OUTPUT_PATH_TOKENS, ( + f"_OUTPUT_PATH_TOKENS diverged from expected fixture.\n" + f"Extra (in production, not in fixture): {extra}\n" + f"Missing (in fixture, not in production): {missing}" + ) + + +@pytest.fixture +def make_headless_session(): + """Build a ClaudeSessionResult with configurable result and tool_uses (parsed form).""" + + def _factory( + result: str = "", + tool_uses: list[dict] | None = None, + subtype: str = "success", + is_error: bool = False, + ) -> ClaudeSessionResult: + return ClaudeSessionResult( + subtype=subtype, + is_error=is_error, + result=result, + session_id="test-session", + tool_uses=tool_uses or [], + ) + + return _factory + + +class TestSynthesizeFromWriteArtifacts: + def test_synthesizes_plan_path_from_write_tool_use(self, make_headless_session): + """When Write file_path is absolute and pattern is plan_path=, token is injected.""" + from autoskillit.execution.headless import _synthesize_from_write_artifacts + + session = make_headless_session( + result="plan summary\n%%ORDER_UP%%", + tool_uses=[ + {"name": "Write", "id": "t1", "file_path": "/abs/temp/make-plan/my_plan.md"} + ], + ) + patterns = [r"plan_path\s*=\s*/.+"] + result = _synthesize_from_write_artifacts(session, patterns, write_call_count=1) + assert result is not None + assert "plan_path = /abs/temp/make-plan/my_plan.md" in result.result + + def test_returns_none_when_no_write_tool_uses(self, make_headless_session): + """No synthesis when write_call_count == 0.""" + from autoskillit.execution.headless import _synthesize_from_write_artifacts + + session = make_headless_session(result="plan summary\n%%ORDER_UP%%", tool_uses=[]) + result = _synthesize_from_write_artifacts( + session, [r"plan_path\s*=\s*/.+"], write_call_count=0 + ) + assert result is None + + def test_returns_none_when_no_absolute_file_path(self, make_headless_session): + """Relative file_path in Write tool_use is not used for synthesis.""" + from autoskillit.execution.headless import _synthesize_from_write_artifacts + + session = make_headless_session( + result="plan summary\n%%ORDER_UP%%", + tool_uses=[{"name": "Write", "id": "t1", "file_path": "temp/make-plan/plan.md"}], + ) + result = _synthesize_from_write_artifacts( + session, [r"plan_path\s*=\s*/.+"], write_call_count=1 + ) + assert result is None + + def test_returns_none_when_pattern_already_satisfied(self, make_headless_session): + """No synthesis when the pattern is already in session.result.""" + from autoskillit.execution.headless import _synthesize_from_write_artifacts + + session = make_headless_session( + result="plan_path = /abs/plan.md\n%%ORDER_UP%%", + tool_uses=[{"name": "Write", "id": "t1", "file_path": "/abs/plan.md"}], + ) + result = _synthesize_from_write_artifacts( + session, [r"plan_path\s*=\s*/.+"], write_call_count=1 + ) + assert result is None + + def test_returns_none_when_no_path_capture_patterns(self, make_headless_session): + """Non-path patterns (verdict= etc) are not attempted for synthesis.""" + from autoskillit.execution.headless import _synthesize_from_write_artifacts + + session = make_headless_session( + result="%%ORDER_UP%%", + tool_uses=[{"name": "Write", "id": "t1", "file_path": "/abs/plan.md"}], + ) + result = _synthesize_from_write_artifacts( + session, [r"verdict\s*=\s*(GO|NO GO)"], write_call_count=1 + ) + assert result is None + + def test_synthesis_uses_last_write_not_first(self, make_headless_session): + """When multiple Write tool_uses exist, synthesis must use the LAST absolute path. + + Multi-artifact skills write intermediate files first, final deliverable last. + Synthesis must inject the final deliverable (last path), not the intermediate. + """ + from autoskillit.execution.headless import _synthesize_from_write_artifacts + + session = make_headless_session( + result="", + tool_uses=[ + { + "name": "Write", + "id": "t1", + "file_path": "/cwd/.autoskillit/temp/make-plan/arch_lens_selection.md", + }, + { + "name": "Write", + "id": "t2", + "file_path": "/cwd/.autoskillit/temp/make-plan/task_plan_2026-01-01.md", + }, + ], + ) + result = _synthesize_from_write_artifacts( + session, [r"plan_path\s*=\s*/.+"], write_call_count=2 + ) + assert result is not None + assert ( + "plan_path = /cwd/.autoskillit/temp/make-plan/task_plan_2026-01-01.md" in result.result + ) + assert "arch_lens_selection" not in result.result + + def test_synthesizes_path_from_backslash_s_plus_pattern(self, make_headless_session): + """Synthesis must work for \\S+-terminated patterns, not only /.+-terminated ones.""" + from autoskillit.execution.headless import _synthesize_from_write_artifacts + + session = make_headless_session( + result="", + tool_uses=[{"name": "Write", "id": "t1", "file_path": "/abs/planner/elab.json"}], + ) + result = _synthesize_from_write_artifacts( + session, [r"elab_result_path\s*=\s*\S+"], write_call_count=1 + ) + assert result is not None + assert "elab_result_path = /abs/planner/elab.json" in result.result + + def test_synthesizes_path_from_dot_plus_pattern(self, make_headless_session): + """Synthesis must work for .+-terminated patterns (no leading /).""" + from autoskillit.execution.headless import _synthesize_from_write_artifacts + + session = make_headless_session( + result="", + tool_uses=[{"name": "Write", "id": "t1", "file_path": "/abs/triage.md"}], + ) + result = _synthesize_from_write_artifacts( + session, [r"triage_report\s*=\s*.+"], write_call_count=1 + ) + assert result is not None + assert "triage_report = /abs/triage.md" in result.result + + +@pytest.fixture +def make_build_skill_result_kwargs(): + """Build a dict of kwargs for _build_skill_result from high-level parameters. + + tool_uses entries use the parsed form: 'file_path' at the top level (not nested + in 'input'). The fixture converts them to raw NDJSON tool_use content blocks + with 'input': {'file_path': ...} so parse_session_result (after Step 3) preserves them. + + consecutive_failures_over_budget=True creates an audit with 4 recorded failures + (> max_consecutive_retries=3 default) so the budget guard fires. + """ + from datetime import UTC, datetime + + def _factory( + result_text: str = "done", + completion_marker: str = "", + expected_output_patterns: list[str] | None = None, + tool_uses: list[dict] | None = None, + consecutive_failures_over_budget: bool = False, + ) -> dict: + records = [] + if tool_uses: + content = [] + for tu in tool_uses: + block: dict = {"type": "tool_use", "name": tu["name"], "id": tu.get("id", "")} + fp = tu.get("file_path") + if fp: + block["input"] = {"file_path": fp} + content.append(block) + records.append(json.dumps({"type": "assistant", "message": {"content": content}})) + records.append( + json.dumps( + { + "type": "result", + "subtype": "success", + "is_error": False, + "result": result_text, + "session_id": "test-session", + } + ) + ) + ndjson = "\n".join(records) + result = SubprocessResult( + returncode=0, + stdout=ndjson, + stderr="", + termination=TerminationReason.NATURAL_EXIT, + pid=12345, + ) + audit = None + skill_command = "" + if consecutive_failures_over_budget: + skill_command = "/autoskillit:make-plan" + audit = DefaultAuditLog() + for _ in range(4): # 4 > max_consecutive_retries=3 → budget exhausted + audit.record_failure( + FailureRecord( # type: ignore[arg-type] + timestamp=datetime.now(UTC).isoformat(), + skill_command=skill_command, + exit_code=-1, + subtype="stale", + needs_retry=True, + retry_reason="stale", + stderr="", + ) + ) + return { + "result": result, + "completion_marker": completion_marker, + "expected_output_patterns": expected_output_patterns or [], + "skill_command": skill_command, + "audit": audit, + } + + return _factory + + +class TestContractRecoveryGate: + def test_adjudicated_failure_with_write_evidence_becomes_retriable( + self, make_build_skill_result_kwargs + ): + """ + COMPLETED + marker present + plan_path absent + Write call with no file_path + → synthesis finds no usable path → CONTRACT_RECOVERY gate fires + → needs_retry=True, retry_reason=contract_recovery. + """ + kwargs = make_build_skill_result_kwargs( + result_text="plan summary\n%%ORDER_UP%%", + completion_marker="%%ORDER_UP%%", + expected_output_patterns=[r"plan_path\s*=\s*/.+"], + tool_uses=[ + # Write call exists (write evidence present) but no file_path key + # → synthesis has nothing to inject → CONTRACT_RECOVERY gate fires + {"name": "Write", "id": "t1"} + ], + ) + sr = _build_skill_result(**kwargs) + assert sr.needs_retry is True + assert sr.retry_reason == RetryReason.CONTRACT_RECOVERY + assert sr.success is False + + def test_adjudicated_failure_without_write_evidence_stays_terminal( + self, make_build_skill_result_kwargs + ): + """ + COMPLETED + marker present + plan_path absent + write_call_count == 0 + → success=False, needs_retry=False, subtype=adjudicated_failure (unchanged behavior). + """ + kwargs = make_build_skill_result_kwargs( + result_text="plan summary\n%%ORDER_UP%%", + completion_marker="%%ORDER_UP%%", + expected_output_patterns=[r"plan_path\s*=\s*/.+"], + tool_uses=[], + ) + sr = _build_skill_result(**kwargs) + assert sr.success is False + assert sr.needs_retry is False + assert sr.subtype == "adjudicated_failure" + + def test_synthesis_succeeds_plan_path_token_from_write_file_path( + self, make_build_skill_result_kwargs + ): + """ + COMPLETED + marker present + plan_path absent + Write file_path is absolute + → synthesis injects token → success=True (no CONTRACT_RECOVERY needed). + """ + kwargs = make_build_skill_result_kwargs( + result_text="plan summary\n%%ORDER_UP%%", + completion_marker="%%ORDER_UP%%", + expected_output_patterns=[r"plan_path\s*=\s*/.+"], + tool_uses=[ + {"name": "Write", "id": "t1", "file_path": "/abs/temp/make-plan/my_plan.md"} + ], + ) + sr = _build_skill_result(**kwargs) + assert sr.success is True + assert "plan_path = /abs/temp/make-plan/my_plan.md" in sr.result + + def test_contract_recovery_respects_budget_guard(self, make_build_skill_result_kwargs): + """ + When budget is exhausted, budget guard must override CONTRACT_RECOVERY. + Write call with no file_path → synthesis fails → CONTRACT_RECOVERY promotes + to retriable → budget guard fires (second call inside gate) → needs_retry=False. + """ + kwargs = make_build_skill_result_kwargs( + result_text="plan summary\n%%ORDER_UP%%", + completion_marker="%%ORDER_UP%%", + expected_output_patterns=[r"plan_path\s*=\s*/.+"], + tool_uses=[{"name": "Write", "id": "t1"}], # no file_path → synthesis fails + consecutive_failures_over_budget=True, + ) + sr = _build_skill_result(**kwargs) + assert sr.needs_retry is False + assert sr.retry_reason == RetryReason.BUDGET_EXHAUSTED + + +class TestBuildSkillResultSessionIdFromSubprocess: + """_build_skill_result propagates result.session_id on all paths.""" + + def test_stale_path_session_id_from_subprocess_result(self) -> None: + """Stale path: SkillResult.session_id == result.session_id (not hardcoded '').""" + result = SubprocessResult( + returncode=-1, + stdout="", + stderr="", + termination=TerminationReason.STALE, + pid=1, + session_id="real-uuid-from-channel-b", + ) + sr = _build_skill_result(result) + assert sr.session_id == "real-uuid-from-channel-b" + + def test_timeout_empty_stdout_session_id_from_subprocess_result(self) -> None: + """TIMED_OUT with empty stdout: SkillResult.session_id == result.session_id.""" + result = SubprocessResult( + returncode=-1, + stdout="", + stderr="", + termination=TerminationReason.TIMED_OUT, + pid=1, + session_id="real-uuid-from-channel-b", + ) + sr = _build_skill_result(result) + assert sr.session_id == "real-uuid-from-channel-b" + + def test_channel_a_session_id_takes_precedence(self) -> None: + """When stdout has a result record with session_id, it wins over result.session_id.""" + stdout = json.dumps( + { + "type": "result", + "subtype": "success", + "result": "done", + "session_id": "stdout-uuid", + "is_error": False, + } + ) + result = SubprocessResult( + returncode=0, + stdout=stdout, + stderr="", + termination=TerminationReason.NATURAL_EXIT, + pid=1, + session_id="ch-b-uuid", + ) + sr = _build_skill_result(result) + assert sr.session_id == "stdout-uuid" + + def test_context_exhaustion_session_id_from_subprocess_result(self) -> None: + """Context-exhaustion (no result record): uses result.session_id as fallback.""" + partial_assistant_stdout = json.dumps( + { + "type": "assistant", + "message": {"content": [{"type": "text", "text": "Doing work..."}]}, + } + ) + result = SubprocessResult( + returncode=1, + stdout=partial_assistant_stdout, + stderr="", + termination=TerminationReason.NATURAL_EXIT, + pid=1, + session_id="ch-b-uuid-5678", + ) + sr = _build_skill_result(result) + assert sr.session_id == "ch-b-uuid-5678" + + +class TestHeadlessExecutorCompletionMarker: + """Protocol conformance: DefaultHeadlessExecutor.run accepts completion_marker.""" + + def test_headless_executor_accepts_completion_marker(self) -> None: + import inspect + + from autoskillit.execution.headless import DefaultHeadlessExecutor + + sig = inspect.signature(DefaultHeadlessExecutor.run) + assert "completion_marker" in sig.parameters + param = sig.parameters["completion_marker"] + assert param.default == "" + + +class TestHeadlessExecutorIdleOutputTimeout: + """Protocol conformance and resolution logic for idle_output_timeout.""" + + def test_headless_executor_accepts_idle_output_timeout(self) -> None: + import inspect + + from autoskillit.execution.headless import DefaultHeadlessExecutor + + sig = inspect.signature(DefaultHeadlessExecutor.run) + assert "idle_output_timeout" in sig.parameters + param = sig.parameters["idle_output_timeout"] + assert param.default is None + + def _success_payload(self, marker: str) -> SubprocessResult: + payload = json.dumps( + { + "type": "result", + "subtype": "success", + "is_error": False, + "result": f"Done. {marker}", + "session_id": "sess-iot", + } + ) + return SubprocessResult(0, payload, "", TerminationReason.NATURAL_EXIT, pid=1) + + @pytest.mark.anyio + async def test_default_headless_executor_uses_per_step_idle_output_timeout( + self, tool_ctx + ) -> None: + """idle_output_timeout=120 is converted to float and passed to the runner.""" + from autoskillit.execution.headless import run_headless_core + + marker = tool_ctx.config.run_skill.completion_marker + tool_ctx.runner.push(self._success_payload(marker)) + await run_headless_core( + "/investigate foo", cwd="/tmp", ctx=tool_ctx, idle_output_timeout=120.0 + ) + _, _cwd, _timeout, kwargs = tool_ctx.runner.call_args_list[0] + assert kwargs["idle_output_timeout"] == 120.0 + + @pytest.mark.anyio + async def test_default_headless_executor_converts_zero_to_none(self, tool_ctx) -> None: + """idle_output_timeout=0 is converted to None (disabled) before passing to runner.""" + from autoskillit.execution.headless import run_headless_core + + marker = tool_ctx.config.run_skill.completion_marker + tool_ctx.runner.push(self._success_payload(marker)) + await run_headless_core( + "/investigate foo", cwd="/tmp", ctx=tool_ctx, idle_output_timeout=0.0 + ) + _, _cwd, _timeout, kwargs = tool_ctx.runner.call_args_list[0] + assert kwargs["idle_output_timeout"] is None + + @pytest.mark.anyio + async def test_default_headless_executor_falls_back_to_cfg_idle_output_timeout( + self, tool_ctx + ) -> None: + """idle_output_timeout=None falls back to float(cfg.idle_output_timeout).""" + from autoskillit.execution.headless import run_headless_core + + marker = tool_ctx.config.run_skill.completion_marker + tool_ctx.runner.push(self._success_payload(marker)) + await run_headless_core( + "/investigate foo", cwd="/tmp", ctx=tool_ctx, idle_output_timeout=None + ) + _, _cwd, _timeout, kwargs = tool_ctx.runner.call_args_list[0] + assert kwargs["idle_output_timeout"] == 1000.0 + + +def _ndjson_with_write(result_text: str, file_paths: list[str], session_id: str = "test-session"): + """Build NDJSON stdout with Write tool_use entries and a result record.""" + records = [] + if file_paths: + content = [ + { + "type": "tool_use", + "name": "Write", + "id": f"t{i}", + "input": {"file_path": fp}, + } + for i, fp in enumerate(file_paths) + ] + records.append(json.dumps({"type": "assistant", "message": {"content": content}})) + records.append( + json.dumps( + { + "type": "result", + "subtype": "success", + "is_error": False, + "result": result_text, + "session_id": session_id, + } + ) + ) + return "\n".join(records) + + +class TestExtractMissingTokenHints: + def test_extracts_token_and_path(self): + stdout = _ndjson_with_write("plan summary\n%%ORDER_UP%%", ["/tmp/out.md"]) + hints = _extract_missing_token_hints(stdout, [r"plan_path\s*=\s*/.+"]) + assert hints == [("plan_path", "/tmp/out.md")] + + def test_returns_empty_when_pattern_satisfied(self): + stdout = _ndjson_with_write("plan_path = /tmp/out.md\n%%ORDER_UP%%", ["/tmp/out.md"]) + hints = _extract_missing_token_hints(stdout, [r"plan_path\s*=\s*/.+"]) + assert hints == [] + + def test_returns_empty_for_non_path_patterns(self): + stdout = _ndjson_with_write("%%ORDER_UP%%", ["/tmp/out.md"]) + hints = _extract_missing_token_hints(stdout, [r"verdict\s*=\s*\w+"]) + assert hints == [] + + def test_uses_last_write_path(self): + stdout = _ndjson_with_write("%%ORDER_UP%%", ["/tmp/first.md", "/tmp/final.md"]) + hints = _extract_missing_token_hints(stdout, [r"plan_path\s*=\s*/.+"]) + assert hints == [("plan_path", "/tmp/final.md")] + + def test_extracts_hints_for_backslash_s_plus_pattern(self): + """_extract_missing_token_hints must work for \\S+-terminated patterns.""" + stdout = _ndjson_with_write("%%ORDER_UP%%", ["/tmp/out.md"]) + hints = _extract_missing_token_hints(stdout, [r"elab_result_path\s*=\s*\S+"]) + assert hints == [("elab_result_path", "/tmp/out.md")] + + +class TestContractNudge: + """Integration tests for the contract recovery nudge in run_headless_core. + + The nudge fires when CONTRACT_RECOVERY is triggered with a valid session_id. + CONTRACT_RECOVERY requires synthesis to have failed first. For CHANNEL_A/B + sessions, synthesis is skipped (gated on UNMONITORED), so Write evidence + with file_path triggers CONTRACT_RECOVERY while _extract_missing_token_hints + can still find the file_path for hints. + """ + + def _main_session_ndjson( + self, marker: str, *, include_token: bool = False, session_id: str = "sess-main" + ) -> str: + """Build NDJSON for a CHANNEL_A-confirmed session with Write evidence.""" + result_text = "plan summary\n" + if include_token: + result_text += "plan_path = /tmp/out.md\n" + result_text += marker + return _ndjson_with_write(result_text, ["/tmp/out.md"], session_id=session_id) + + def _main_subprocess_result( + self, marker: str, *, session_id: str = "sess-main" + ) -> SubprocessResult: + """Build a SubprocessResult with CHANNEL_A confirmation to bypass synthesis.""" + return SubprocessResult( + returncode=0, + stdout=self._main_session_ndjson(marker, session_id=session_id), + stderr="", + termination=TerminationReason.NATURAL_EXIT, + pid=1, + channel_confirmation=ChannelConfirmation.CHANNEL_A, + ) + + def _nudge_response_ndjson(self, marker: str, *, include_token: bool = True) -> str: + """Build NDJSON for a nudge response.""" + if include_token: + result_text = f"plan_path = /tmp/out.md\n{marker}" + else: + result_text = f"I cannot do that.\n{marker}" + return json.dumps( + { + "type": "result", + "subtype": "success", + "is_error": False, + "result": result_text, + "session_id": "sess-main", + "usage": {"input_tokens": 100, "output_tokens": 50}, + } + ) + + @pytest.mark.anyio + async def test_nudge_fires_on_contract_recovery(self, tool_ctx): + from autoskillit.execution.headless import run_headless_core + + marker = tool_ctx.config.run_skill.completion_marker + tool_ctx.runner.push(self._main_subprocess_result(marker)) + tool_ctx.runner.push( + SubprocessResult( + 0, self._nudge_response_ndjson(marker), "", TerminationReason.NATURAL_EXIT, pid=2 + ) + ) + result = await run_headless_core( + "/autoskillit:make-plan foo", + cwd="/tmp", + ctx=tool_ctx, + expected_output_patterns=[r"plan_path\s*=\s*/.+"], + ) + assert result.success is True + assert result.needs_retry is False + assert "plan_path = /tmp/out.md" in result.result + + @pytest.mark.anyio + async def test_nudge_failure_falls_through(self, tool_ctx): + from autoskillit.execution.headless import run_headless_core + + marker = tool_ctx.config.run_skill.completion_marker + tool_ctx.runner.push(self._main_subprocess_result(marker)) + tool_ctx.runner.push( + SubprocessResult( + 0, + self._nudge_response_ndjson(marker, include_token=False), + "", + TerminationReason.NATURAL_EXIT, + pid=2, + ) + ) + result = await run_headless_core( + "/autoskillit:make-plan foo", + cwd="/tmp", + ctx=tool_ctx, + expected_output_patterns=[r"plan_path\s*=\s*/.+"], + ) + assert result.retry_reason == RetryReason.CONTRACT_RECOVERY + assert result.needs_retry is True + + @pytest.mark.anyio + async def test_nudge_timeout_falls_through(self, tool_ctx): + from autoskillit.execution.headless import run_headless_core + + marker = tool_ctx.config.run_skill.completion_marker + tool_ctx.runner.push(self._main_subprocess_result(marker)) + tool_ctx.runner.push( + SubprocessResult(1, "", "timeout", TerminationReason.TIMED_OUT, pid=2) + ) + result = await run_headless_core( + "/autoskillit:make-plan foo", + cwd="/tmp", + ctx=tool_ctx, + expected_output_patterns=[r"plan_path\s*=\s*/.+"], + ) + assert result.retry_reason == RetryReason.CONTRACT_RECOVERY + assert result.needs_retry is True + + @pytest.mark.anyio + async def test_nudge_skips_when_budget_exhausted(self, tool_ctx): + from datetime import UTC, datetime + + from autoskillit.core import FailureRecord + from autoskillit.execution.headless import run_headless_core + from autoskillit.pipeline.audit import DefaultAuditLog + + marker = tool_ctx.config.run_skill.completion_marker + audit = DefaultAuditLog() + for _ in range(4): + audit.record_failure( + FailureRecord( # type: ignore[arg-type] + timestamp=datetime.now(UTC).isoformat(), + skill_command="/autoskillit:make-plan foo", + exit_code=-1, + subtype="stale", + needs_retry=True, + retry_reason="stale", + stderr="", + ) + ) + tool_ctx.audit = audit + tool_ctx.runner.push(self._main_subprocess_result(marker)) + result = await run_headless_core( + "/autoskillit:make-plan foo", + cwd="/tmp", + ctx=tool_ctx, + expected_output_patterns=[r"plan_path\s*=\s*/.+"], + ) + assert result.retry_reason == RetryReason.BUDGET_EXHAUSTED + assert len(tool_ctx.runner.call_args_list) == 1 + + @pytest.mark.anyio + async def test_nudge_skips_when_no_session_id(self, tool_ctx): + from autoskillit.execution.headless import run_headless_core + + marker = tool_ctx.config.run_skill.completion_marker + ndjson = _ndjson_with_write(f"plan summary\n{marker}", ["/tmp/out.md"], session_id="") + tool_ctx.runner.push( + SubprocessResult( + 0, + ndjson, + "", + TerminationReason.NATURAL_EXIT, + pid=1, + channel_confirmation=ChannelConfirmation.CHANNEL_A, + ) + ) + result = await run_headless_core( + "/autoskillit:make-plan foo", + cwd="/tmp", + ctx=tool_ctx, + expected_output_patterns=[r"plan_path\s*=\s*/.+"], + ) + assert result.retry_reason == RetryReason.CONTRACT_RECOVERY + assert len(tool_ctx.runner.call_args_list) == 1 + + @pytest.mark.anyio + async def test_nudge_resume_cmd_uses_correct_session_id(self, tool_ctx): + from autoskillit.execution.headless import run_headless_core + + marker = tool_ctx.config.run_skill.completion_marker + tool_ctx.runner.push(self._main_subprocess_result(marker, session_id="sess-abc-123")) + tool_ctx.runner.push( + SubprocessResult( + 0, self._nudge_response_ndjson(marker), "", TerminationReason.NATURAL_EXIT, pid=2 + ) + ) + await run_headless_core( + "/autoskillit:make-plan foo", + cwd="/tmp", + ctx=tool_ctx, + expected_output_patterns=[r"plan_path\s*=\s*/.+"], + ) + assert len(tool_ctx.runner.call_args_list) == 2 + nudge_cmd = tool_ctx.runner.call_args_list[1][0] + assert "--resume" in nudge_cmd + resume_idx = nudge_cmd.index("--resume") + assert nudge_cmd[resume_idx + 1] == "sess-abc-123" + + @pytest.mark.anyio + async def test_nudge_token_usage_merged(self, tool_ctx): + from autoskillit.execution.headless import run_headless_core + + marker = tool_ctx.config.run_skill.completion_marker + main_ndjson_records = [ + json.dumps( + { + "type": "assistant", + "message": { + "content": [ + { + "type": "tool_use", + "name": "Write", + "id": "t0", + "input": {"file_path": "/tmp/out.md"}, + } + ] + }, + } + ), + json.dumps( + { + "type": "result", + "subtype": "success", + "is_error": False, + "result": f"plan summary\n{marker}", + "session_id": "sess-tok", + "usage": {"input_tokens": 1000, "output_tokens": 500}, + } + ), + ] + main_ndjson = "\n".join(main_ndjson_records) + + nudge_ndjson = json.dumps( + { + "type": "result", + "subtype": "success", + "is_error": False, + "result": f"plan_path = /tmp/out.md\n{marker}", + "session_id": "sess-tok", + "usage": {"input_tokens": 200, "output_tokens": 100}, + } + ) + + tool_ctx.runner.push( + SubprocessResult( + 0, + main_ndjson, + "", + TerminationReason.NATURAL_EXIT, + pid=1, + channel_confirmation=ChannelConfirmation.CHANNEL_A, + ) + ) + tool_ctx.runner.push( + SubprocessResult(0, nudge_ndjson, "", TerminationReason.NATURAL_EXIT, pid=2) + ) + + result = await run_headless_core( + "/autoskillit:make-plan foo", + cwd="/tmp", + ctx=tool_ctx, + expected_output_patterns=[r"plan_path\s*=\s*/.+"], + step_name="test-step", + ) + assert result.success is True + assert result.token_usage is not None + assert result.token_usage.get("input_tokens", 0) >= 1200 + assert result.token_usage.get("output_tokens", 0) >= 600 + + @pytest.mark.anyio + async def test_nudge_exception_falls_through(self, tool_ctx): + from autoskillit.execution.headless import run_headless_core + + marker = tool_ctx.config.run_skill.completion_marker + tool_ctx.runner.push(self._main_subprocess_result(marker)) + # Empty stdout nudge → patterns not found → falls through + tool_ctx.runner.push( + SubprocessResult(1, "", "RuntimeError", TerminationReason.NATURAL_EXIT, pid=2) + ) + result = await run_headless_core( + "/autoskillit:make-plan foo", + cwd="/tmp", + ctx=tool_ctx, + expected_output_patterns=[r"plan_path\s*=\s*/.+"], + ) + assert result.retry_reason == RetryReason.CONTRACT_RECOVERY + assert result.needs_retry is True + + +class TestPathCaptureCoversAllContractPatterns: + """_is_path_capture_pattern must correctly classify all patterns in skill_contracts.yaml.""" + + def test_classifies_all_path_output_patterns_as_path_capture(self) -> None: + import re + + from autoskillit.execution.headless import _is_path_capture_pattern + from autoskillit.execution.headless._headless_path_tokens import ( + _INTENTIONALLY_EXCLUDED_PATH_TOKENS, + ) + from autoskillit.recipe.contracts import load_bundled_manifest + + manifest = load_bundled_manifest() + skills_data = manifest.get("skills", {}) + + path_token_names: set[str] = set() + non_path_token_names: set[str] = set() + pattern_token_map: list[tuple[str, str]] = [] + + for skill_data in skills_data.values(): + if not isinstance(skill_data, dict): + continue + for out in skill_data.get("outputs", []): + if not isinstance(out, dict): + continue + t = out.get("type", "") + name = out.get("name", "") + if t.startswith("file_path") or t == "directory_path": + path_token_names.add(name) + elif t: + non_path_token_names.add(name) + for pattern in skill_data.get("expected_output_patterns", []): + m = re.match(r"^(\w+)", pattern) + if m and "=" in pattern[m.end() :]: + pattern_token_map.append((m.group(1), pattern)) + + assert pattern_token_map, "No patterns found in skill_contracts.yaml — test is vacuous" + + recoverable_path_token_names = path_token_names - _INTENTIONALLY_EXCLUDED_PATH_TOKENS + + misclassified_path: list[str] = [] + for token_name, pattern in pattern_token_map: + if token_name in recoverable_path_token_names: + result = _is_path_capture_pattern(pattern) + if result != token_name: + misclassified_path.append( + f"Pattern {pattern!r} (token={token_name!r}) classified as {result!r}, " + f"expected {token_name!r}" + ) + + misclassified_non_path: list[str] = [] + for token_name, pattern in pattern_token_map: + # Skip tokens that appear as path-type in some skill and non-path-type in another; + # cross-skill token name reuse is intentional and not a classification error. + if token_name in non_path_token_names and token_name not in path_token_names: + result = _is_path_capture_pattern(pattern) + if result is not None: + misclassified_non_path.append( + f"Pattern {pattern!r} (token={token_name!r}) incorrectly classified as " + f"path-capture (returned {result!r})" + ) + + errors = misclassified_path + misclassified_non_path + assert not errors, "Pattern classification errors:\n" + "\n".join( + f" - {e}" for e in errors + ) + + +class TestRecoverablePathTokensCoverage: + def test_excluded_tokens_match_intentionally_excluded(self) -> None: + """_PATH_RECOVERY_EXCLUDED_TOKENS must equal _INTENTIONALLY_EXCLUDED_PATH_TOKENS. + + Both constants exclude the same token names but live in different layers + (recipe IL-2 vs execution IL-1) and cannot be imported across. + """ + from autoskillit.execution.headless._headless_path_tokens import ( + _INTENTIONALLY_EXCLUDED_PATH_TOKENS, + ) + from autoskillit.recipe.rules.rules_contracts import _PATH_RECOVERY_EXCLUDED_TOKENS + + assert _PATH_RECOVERY_EXCLUDED_TOKENS == _INTENTIONALLY_EXCLUDED_PATH_TOKENS, ( + f"Constants diverged.\n" + f"_PATH_RECOVERY_EXCLUDED_TOKENS: {_PATH_RECOVERY_EXCLUDED_TOKENS}\n" + f"_INTENTIONALLY_EXCLUDED_PATH_TOKENS: {_INTENTIONALLY_EXCLUDED_PATH_TOKENS}" + ) + + def test_recoverable_path_tokens_includes_directory_path_outputs(self) -> None: + from autoskillit.execution.headless import _RECOVERABLE_PATH_TOKENS + from autoskillit.recipe.contracts import load_bundled_manifest + + manifest = load_bundled_manifest() + skills_data = manifest.get("skills", {}) + declared: set[str] = set() + for skill_data in skills_data.values(): + if not isinstance(skill_data, dict): + continue + for out in skill_data.get("outputs", []): + if not isinstance(out, dict): + continue + t = out.get("type", "") + name = out.get("name", "").strip() + if name and (t.startswith("file_path") or t == "directory_path"): + declared.add(name) + + from autoskillit.execution.headless._headless_path_tokens import ( + _INTENTIONALLY_EXCLUDED_PATH_TOKENS, + ) + + expected = declared - _INTENTIONALLY_EXCLUDED_PATH_TOKENS + normalized_actual = {t.strip() for t in _RECOVERABLE_PATH_TOKENS} + assert normalized_actual == expected, ( + f"_RECOVERABLE_PATH_TOKENS diverged from derived set.\n" + f"Extra (in production, not in derived): " + f"{normalized_actual - expected}\n" + f"Missing (in derived, not in production): " + f"{expected - normalized_actual}" + ) + + +def test_build_skill_result_surfaces_last_stop_reason(): + ndjson = "\n".join( + [ + json.dumps( + { + "type": "assistant", + "message": {"stop_reason": "end_turn", "content": [], "usage": {}}, + } + ), + json.dumps( + { + "type": "result", + "subtype": "success", + "is_error": False, + "result": "done", + "session_id": "s1", + } + ), + ] + ) + result = _make_result(returncode=0, stdout=ndjson) + sr = _build_skill_result(result) + assert sr.last_stop_reason == "end_turn" + + +class TestEarlyStopRecovery: + """EARLY_STOP must trigger nudge recovery, not fall through to on_failure.""" + + def _early_stop_subprocess_result( + self, result_text: str, session_id: str = "sess-123" + ) -> SubprocessResult: + """Build a SubprocessResult that produces EARLY_STOP: substantive output, no marker.""" + return SubprocessResult( + returncode=0, + stdout=_success_session_json(result_text), + stderr="", + termination=TerminationReason.NATURAL_EXIT, + pid=12345, + session_id=session_id, + ) + + def _nudge_response_with_marker(self, marker: str) -> SubprocessResult: + """Build a nudge response that emits the marker.""" + result_text = f"completion confirmed\n{marker}" + return SubprocessResult( + returncode=0, + stdout=_success_session_json(result_text), + stderr="", + termination=TerminationReason.NATURAL_EXIT, + pid=2, + ) + + @pytest.mark.anyio + async def test_nudge_fires_on_early_stop(self, tool_ctx): + """_attempt_contract_nudge fires when retry_reason is EARLY_STOP.""" + from autoskillit.execution.headless import run_headless_core + + marker = tool_ctx.config.run_skill.completion_marker + substantive_result = "## Summary\nPlan complete." + tool_ctx.runner.push( + self._early_stop_subprocess_result(substantive_result, session_id="sess-early") + ) + tool_ctx.runner.push(self._nudge_response_with_marker(marker)) + result = await run_headless_core( + "/autoskillit:make-plan foo", + cwd="/tmp", + ctx=tool_ctx, + provider_extras={"ANTHROPIC_BASE_URL": "https://minimax.example"}, + ) + assert result.success is True + assert len(tool_ctx.runner.call_args_list) == 2 + + @pytest.mark.anyio + async def test_early_stop_nudge_bypasses_hints_guard(self, tool_ctx): + """EARLY_STOP nudge does not require _extract_missing_token_hints to find hints.""" + from autoskillit.execution.headless import run_headless_core + + marker = tool_ctx.config.run_skill.completion_marker + substantive_result = "## Summary\nPlan complete written to /tmp/plan.md" + tool_ctx.runner.push( + self._early_stop_subprocess_result(substantive_result, session_id="sess-no-hints") + ) + tool_ctx.runner.push(self._nudge_response_with_marker(marker)) + result = await run_headless_core( + "/autoskillit:make-plan foo", + cwd="/tmp", + ctx=tool_ctx, + provider_extras={"ANTHROPIC_BASE_URL": "https://minimax.example"}, + ) + assert result.success is True + + +class TestEarlyStopDetection: + """MiniMax-shaped result (substantive output, no marker) produces EARLY_STOP, not success.""" + + def test_minimax_shaped_result_produces_early_stop(self): + """A non-empty result with no marker from NATURAL_EXIT produces EARLY_STOP, not success.""" + substantive_result = "## Summary\nPlan written to /tmp/plan.md\n\nplan_path = /tmp/plan.md" + sub_result = SubprocessResult( + returncode=0, + stdout=_success_session_json(substantive_result), + stderr="", + termination=TerminationReason.NATURAL_EXIT, + pid=12345, + ) + sr = _build_skill_result( + sub_result, + completion_marker="%%ORDER_UP::abc%%", + ) + assert sr.needs_retry is True + assert sr.retry_reason == RetryReason.EARLY_STOP diff --git a/tests/execution/test_headless_provider_fallback.py b/tests/execution/test_headless_provider_fallback.py new file mode 100644 index 0000000000..3f603b2fbf --- /dev/null +++ b/tests/execution/test_headless_provider_fallback.py @@ -0,0 +1,212 @@ +"""Tests verifying the provider fallback loop in _execute_claude_headless. + +Covers: STALE triggers fallback, BUDGET_EXHAUSTED triggers fallback, +no fallback_env suppresses retry, and empty provider (Anthropic) never falls back. +""" + +from __future__ import annotations + +from collections import deque + +import pytest + +from autoskillit.core.types import RetryReason, SkillResult + +pytestmark = [pytest.mark.layer("execution"), pytest.mark.small] + +_PROVIDER_RETRY_LIMIT = 2 + +_STALE_RESULT = SkillResult( + success=False, + result="", + session_id="s1", + subtype="stale", + is_error=False, + exit_code=1, + needs_retry=True, + retry_reason=RetryReason.STALE, + stderr="", +) + +_BUDGET_EXHAUSTED_RESULT = SkillResult( + success=False, + result="", + session_id="s1", + subtype="budget_exhausted", + is_error=False, + exit_code=1, + needs_retry=False, + retry_reason=RetryReason.BUDGET_EXHAUSTED, + stderr="", +) + +_SUCCESS_RESULT = SkillResult( + success=True, + result="done", + session_id="s2", + subtype="success", + is_error=False, + exit_code=0, + needs_retry=False, + retry_reason=RetryReason.NONE, + stderr="", +) + + +def _make_queued_build_result(*results: SkillResult): + q: deque[SkillResult] = deque(results) + + def _build(*args, **kwargs): # noqa: ARG001 + return q.popleft() + + return _build + + +class TestProviderFallbackLoop: + def _patch_common(self, monkeypatch, tmp_path, build_result_fn, ctx=None): + import autoskillit.execution.session_log as _sl_mod + from autoskillit.execution.headless import PostSessionMetrics + from tests.execution.conftest import _sr + + _sub_result = _sr() + call_count: list[int] = [0] + + async def fake_runner(cmd, **kwargs): # noqa: ARG001 + call_count[0] += 1 + return _sub_result + + if ctx is not None: + monkeypatch.setattr( + ctx.config.providers, "provider_retry_limit", _PROVIDER_RETRY_LIMIT + ) + + monkeypatch.setattr( + "autoskillit.execution.headless._build_skill_result", + build_result_fn, + ) + monkeypatch.setattr( + "autoskillit.execution.headless._compute_post_session_metrics", + lambda *a, **kw: PostSessionMetrics(0, 0, str(tmp_path)), # noqa: ARG005 + ) + monkeypatch.setattr( + "autoskillit.execution.headless._capture_git_head_sha", + lambda *a: "", # noqa: ARG005 + ) + monkeypatch.setattr( + "autoskillit.execution.headless.is_feature_enabled", + lambda name, *a, **kw: name == "providers", # noqa: ARG005 + ) + monkeypatch.setattr( + "autoskillit.execution.headless.collect_version_snapshot", + lambda: {}, + ) + monkeypatch.setattr(_sl_mod, "flush_session_log", lambda **kw: None) # noqa: ARG005 + + return fake_runner, call_count + + @pytest.mark.anyio + async def test_stale_triggers_fallback(self, minimal_ctx, tmp_path, monkeypatch): + from autoskillit.execution.commands import ClaudeHeadlessCmd + from autoskillit.execution.headless import _execute_claude_headless + + fake_runner, call_count = self._patch_common( + monkeypatch, + tmp_path, + _make_queued_build_result(_STALE_RESULT, _SUCCESS_RESULT), + ctx=minimal_ctx, + ) + minimal_ctx.runner = fake_runner + + result = await _execute_claude_headless( + ClaudeHeadlessCmd(cmd=["echo", "test"], env={}), + str(tmp_path), + minimal_ctx, + timeout=30.0, + stale_threshold=5.0, + provider_name="minimax", + provider_fallback_env={"ANTHROPIC_API_KEY": "sk-test"}, + provider_fallback_name="anthropic", + ) + + assert call_count[0] == 2 + assert result.provider.fallback_activated is True + assert result.provider.provider_used == "anthropic" + + @pytest.mark.anyio + async def test_budget_exhausted_triggers_fallback(self, minimal_ctx, tmp_path, monkeypatch): + from autoskillit.execution.commands import ClaudeHeadlessCmd + from autoskillit.execution.headless import _execute_claude_headless + + fake_runner, call_count = self._patch_common( + monkeypatch, + tmp_path, + _make_queued_build_result(_BUDGET_EXHAUSTED_RESULT, _SUCCESS_RESULT), + ctx=minimal_ctx, + ) + minimal_ctx.runner = fake_runner + + result = await _execute_claude_headless( + ClaudeHeadlessCmd(cmd=["echo", "test"], env={}), + str(tmp_path), + minimal_ctx, + timeout=30.0, + stale_threshold=5.0, + provider_name="minimax", + provider_fallback_env={"ANTHROPIC_API_KEY": "sk-test"}, + provider_fallback_name="anthropic", + ) + + assert call_count[0] == 2 + assert result.provider.fallback_activated is True + assert result.provider.provider_used == "anthropic" + + @pytest.mark.anyio + async def test_no_fallback_env_suppresses_retry(self, minimal_ctx, tmp_path, monkeypatch): + from autoskillit.execution.commands import ClaudeHeadlessCmd + from autoskillit.execution.headless import _execute_claude_headless + + fake_runner, call_count = self._patch_common( + monkeypatch, + tmp_path, + _make_queued_build_result(_STALE_RESULT), + ) + minimal_ctx.runner = fake_runner + + result = await _execute_claude_headless( + ClaudeHeadlessCmd(cmd=["echo", "test"], env={}), + str(tmp_path), + minimal_ctx, + timeout=30.0, + stale_threshold=5.0, + provider_name="minimax", + ) + + assert call_count[0] == 1 + assert result.provider.fallback_activated is False + + @pytest.mark.anyio + async def test_anthropic_provider_never_falls_back(self, minimal_ctx, tmp_path, monkeypatch): + from autoskillit.execution.commands import ClaudeHeadlessCmd + from autoskillit.execution.headless import _execute_claude_headless + + fake_runner, call_count = self._patch_common( + monkeypatch, + tmp_path, + _make_queued_build_result(_STALE_RESULT), + ctx=minimal_ctx, + ) + minimal_ctx.runner = fake_runner + + result = await _execute_claude_headless( + ClaudeHeadlessCmd(cmd=["echo", "test"], env={}), + str(tmp_path), + minimal_ctx, + timeout=30.0, + stale_threshold=5.0, + provider_name="", + provider_fallback_env={"ANTHROPIC_API_KEY": "sk-test"}, + provider_fallback_name="anthropic", + ) + + assert call_count[0] == 1 + assert result.provider.fallback_activated is False diff --git a/tests/execution/test_headless_provider_forwarding.py b/tests/execution/test_headless_provider_forwarding.py new file mode 100644 index 0000000000..1a01c5b14b --- /dev/null +++ b/tests/execution/test_headless_provider_forwarding.py @@ -0,0 +1,348 @@ +"""Tests verifying provider_extras, profile_name, provider_name, and provider_fallback_env +forwarding through the headless call chain.""" + +from __future__ import annotations + +import pytest + +from autoskillit.core.types import RetryReason, SkillResult + +pytestmark = [pytest.mark.layer("execution"), pytest.mark.small] + +_STUB_RESULT = SkillResult( + success=True, + result="done", + session_id="s1", + subtype="success", + is_error=False, + exit_code=0, + needs_retry=False, + retry_reason=RetryReason.NONE, + stderr="", +) + + +@pytest.mark.anyio +async def test_run_headless_core_forwards_provider_extras_to_build_cmd( + minimal_ctx, tmp_path, monkeypatch +) -> None: + from autoskillit.execution.headless import run_headless_core + + captured: dict = {} + execute_kwargs: dict = {} + + def fake_build(skill_command, **kwargs): + captured.update(kwargs) + return object() + + async def fake_execute(spec, cwd, ctx, **kwargs): + execute_kwargs.update(kwargs) + return _STUB_RESULT + + monkeypatch.setattr("autoskillit.execution.headless.build_skill_session_cmd", fake_build) + monkeypatch.setattr("autoskillit.execution.headless._execute_claude_headless", fake_execute) + + await run_headless_core( + "/autoskillit:probe", + str(tmp_path), + minimal_ctx, + provider_extras={"AWS_REGION": "us-east-1"}, + profile_name="bedrock", + ) + + assert captured["provider_extras"] == {"AWS_REGION": "us-east-1"} + assert captured["profile_name"] == "bedrock" + assert execute_kwargs.get("provider_extras") == {"AWS_REGION": "us-east-1"} + assert "profile_name" not in execute_kwargs + + +@pytest.mark.anyio +async def test_run_headless_core_defaults_provider_extras_none( + minimal_ctx, tmp_path, monkeypatch +) -> None: + from autoskillit.execution.headless import run_headless_core + + captured: dict = {} + + def fake_build(skill_command, **kwargs): + captured.update(kwargs) + return object() + + async def fake_execute(spec, cwd, ctx, **kwargs): + return _STUB_RESULT + + monkeypatch.setattr("autoskillit.execution.headless.build_skill_session_cmd", fake_build) + monkeypatch.setattr("autoskillit.execution.headless._execute_claude_headless", fake_execute) + + await run_headless_core("/autoskillit:probe", str(tmp_path), minimal_ctx) + + assert captured["provider_extras"] is None + assert captured["profile_name"] == "" + + +@pytest.mark.anyio +async def test_default_executor_run_forwards_provider_extras( + minimal_ctx, tmp_path, monkeypatch +) -> None: + import autoskillit.execution.headless as _headless_mod + from autoskillit.execution.headless import DefaultHeadlessExecutor + + captured: dict = {} + + async def fake_core(skill_command, cwd, ctx, **kwargs): + captured.update(kwargs) + return _STUB_RESULT + + monkeypatch.setattr(_headless_mod, "run_headless_core", fake_core) + + executor = DefaultHeadlessExecutor(minimal_ctx) + await executor.run( + "/autoskillit:probe", + str(tmp_path), + provider_extras={"KEY": "val"}, + profile_name="vertex", + ) + + assert captured["provider_extras"] == {"KEY": "val"} + assert captured["profile_name"] == "vertex" + + +@pytest.mark.anyio +async def test_default_executor_run_defaults_provider_extras( + minimal_ctx, tmp_path, monkeypatch +) -> None: + import autoskillit.execution.headless as _headless_mod + from autoskillit.execution.headless import DefaultHeadlessExecutor + + captured: dict = {} + + async def fake_core(skill_command, cwd, ctx, **kwargs): + captured.update(kwargs) + return _STUB_RESULT + + monkeypatch.setattr(_headless_mod, "run_headless_core", fake_core) + + executor = DefaultHeadlessExecutor(minimal_ctx) + await executor.run("/autoskillit:probe", str(tmp_path)) + + assert captured["provider_extras"] is None + assert captured["profile_name"] == "" + + +def test_execute_claude_headless_accepts_provider_name_and_fallback_env() -> None: + import inspect + + from autoskillit.execution.headless import _execute_claude_headless + + sig = inspect.signature(_execute_claude_headless) + assert sig.parameters["provider_name"].default == "" + assert sig.parameters["provider_fallback_env"].default is None + + +def test_run_headless_core_accepts_provider_name_and_fallback_env() -> None: + import inspect + + from autoskillit.execution.headless import run_headless_core + + sig = inspect.signature(run_headless_core) + assert sig.parameters["provider_name"].default == "" + assert sig.parameters["provider_fallback_env"].default is None + + +def test_default_executor_run_accepts_provider_name_and_fallback_env() -> None: + import inspect + + from autoskillit.execution.headless import DefaultHeadlessExecutor + + sig = inspect.signature(DefaultHeadlessExecutor.run) + assert sig.parameters["provider_name"].default == "" + assert sig.parameters["provider_fallback_env"].default is None + + +@pytest.mark.anyio +async def test_run_headless_core_forwards_provider_name_and_fallback_env( + minimal_ctx, tmp_path, monkeypatch +) -> None: + from autoskillit.execution.headless import run_headless_core + + execute_kwargs: dict = {} + + def fake_build(skill_command, **kwargs): # noqa: ARG001 + return object() + + async def fake_execute(spec, cwd, ctx, **kwargs): # noqa: ARG001 + execute_kwargs.update(kwargs) + return _STUB_RESULT + + monkeypatch.setattr("autoskillit.execution.headless.build_skill_session_cmd", fake_build) + monkeypatch.setattr("autoskillit.execution.headless._execute_claude_headless", fake_execute) + + await run_headless_core( + "/autoskillit:probe", + str(tmp_path), + minimal_ctx, + provider_name="bedrock", + provider_fallback_env={"KEY": "val"}, + ) + + assert execute_kwargs["provider_name"] == "bedrock" + assert execute_kwargs["provider_fallback_env"] == {"KEY": "val"} + + +@pytest.mark.anyio +async def test_default_executor_run_forwards_provider_name_and_fallback_env( + minimal_ctx, tmp_path, monkeypatch +) -> None: + import autoskillit.execution.headless as _headless_mod + from autoskillit.execution.headless import DefaultHeadlessExecutor + + captured: dict = {} + + async def fake_core(skill_command, cwd, ctx, **kwargs): # noqa: ARG001 + captured.update(kwargs) + return _STUB_RESULT + + monkeypatch.setattr(_headless_mod, "run_headless_core", fake_core) + + executor = DefaultHeadlessExecutor(minimal_ctx) + await executor.run( + "/autoskillit:probe", + str(tmp_path), + provider_name="vertex", + provider_fallback_env={"K": "v"}, + ) + + assert captured["provider_name"] == "vertex" + assert captured["provider_fallback_env"] == {"K": "v"} + + +@pytest.mark.anyio +async def test_no_fallback_env_returns_empty_provider_used( + minimal_ctx, tmp_path, monkeypatch +) -> None: + from autoskillit.execution.commands import ClaudeHeadlessCmd + from autoskillit.execution.headless import PostSessionMetrics, _execute_claude_headless + from tests.execution.conftest import _sr + + _spec = ClaudeHeadlessCmd(cmd=["echo", "test"], env={}) + _sub_result = _sr() + + async def fake_runner(cmd, **kwargs): # noqa: ARG001 + return _sub_result + + minimal_ctx.runner = fake_runner + + monkeypatch.setattr( + "autoskillit.execution.headless._build_skill_result", + lambda *a, **kw: _STUB_RESULT, # noqa: ARG005 + ) + monkeypatch.setattr( + "autoskillit.execution.headless._compute_post_session_metrics", + lambda *a, **kw: PostSessionMetrics(0, 0, str(tmp_path)), # noqa: ARG005 + ) + monkeypatch.setattr( + "autoskillit.execution.headless._capture_git_head_sha", + lambda *a: "", # noqa: ARG005 + ) + + result = await _execute_claude_headless( + _spec, + str(tmp_path), + minimal_ctx, + timeout=30.0, + stale_threshold=5.0, + ) + + assert result.provider.provider_used == "" + assert result.provider.fallback_activated is False + + +@pytest.mark.anyio +async def test_provider_name_stamps_provider_used_on_result( + minimal_ctx, tmp_path, monkeypatch +) -> None: + from autoskillit.execution.commands import ClaudeHeadlessCmd + from autoskillit.execution.headless import PostSessionMetrics, _execute_claude_headless + from tests.execution.conftest import _sr + + _spec = ClaudeHeadlessCmd(cmd=["echo", "test"], env={}) + _sub_result = _sr() + + async def fake_runner(cmd, **kwargs): # noqa: ARG001 + return _sub_result + + minimal_ctx.runner = fake_runner + + monkeypatch.setattr( + "autoskillit.execution.headless._build_skill_result", + lambda *a, **kw: _STUB_RESULT, # noqa: ARG005 + ) + monkeypatch.setattr( + "autoskillit.execution.headless._compute_post_session_metrics", + lambda *a, **kw: PostSessionMetrics(0, 0, str(tmp_path)), # noqa: ARG005 + ) + monkeypatch.setattr( + "autoskillit.execution.headless._capture_git_head_sha", + lambda *a: "", # noqa: ARG005 + ) + + result = await _execute_claude_headless( + _spec, + str(tmp_path), + minimal_ctx, + timeout=30.0, + stale_threshold=5.0, + provider_name="bedrock", + ) + + assert result.provider.provider_used == "bedrock" + assert result.provider.fallback_activated is False + + +def test_headless_executor_protocol_includes_provider_params() -> None: + import inspect + + from autoskillit.core.types import HeadlessExecutor + + sig = inspect.signature(HeadlessExecutor.run) + assert sig.parameters["provider_name"].default == "" + assert sig.parameters["provider_fallback_env"].default is None + + +def test_build_skill_result_accepts_provider_used_kwarg() -> None: + import inspect + + from autoskillit.execution.headless._headless_result import _build_skill_result + + sig = inspect.signature(_build_skill_result) + param = sig.parameters["provider_used"] + assert param.default == "" + assert param.kind == inspect.Parameter.KEYWORD_ONLY + + +def test_build_skill_result_stamps_provider_used_on_result() -> None: + from autoskillit.execution.headless._headless_result import _build_skill_result + from tests.execution.conftest import _sr + + sub_result = _sr(stdout="result: done\n", returncode=0) + sr = _build_skill_result(sub_result, provider_used="vertex") + assert sr.provider.provider_used == "vertex" + sr_default = _build_skill_result(sub_result) + assert sr_default.provider.provider_used == "" + + +def test_build_skill_result_provider_used_survives_budget_guard() -> None: + from autoskillit.execution.headless._headless_result import _build_skill_result + from autoskillit.pipeline.audit import DefaultAuditLog + from tests.execution.conftest import _sr + + audit = DefaultAuditLog() + sub_result = _sr(stdout="", returncode=1) + sr = _build_skill_result( + sub_result, + skill_command="/test:cmd", + audit=audit, + max_consecutive_retries=3, + provider_used="bedrock", + ) + assert sr.provider.provider_used == "bedrock" diff --git a/tests/execution/test_headless_result.py b/tests/execution/test_headless_result.py new file mode 100644 index 0000000000..e04b9d673c --- /dev/null +++ b/tests/execution/test_headless_result.py @@ -0,0 +1,56 @@ +"""Tests for _build_skill_result idle_stall lifespan_started propagation.""" + +from __future__ import annotations + +import json + +import pytest + +from autoskillit.core.types import SubprocessResult, TerminationReason +from autoskillit.execution.headless import _build_skill_result + +pytestmark = [pytest.mark.layer("execution"), pytest.mark.small, pytest.mark.feature("fleet")] + + +def _idle_stall_result(stdout: str) -> SubprocessResult: + """Build a SubprocessResult with IDLE_STALL termination.""" + return SubprocessResult( + returncode=-1, + stdout=stdout, + stderr="", + termination=TerminationReason.IDLE_STALL, + pid=12345, + session_id="sess-idle-1", + channel_b_session_id="", + ) + + +def _tool_use_ndjson(tool_name: str = "Write", **input_kwargs: object) -> str: + return json.dumps( + { + "type": "assistant", + "message": { + "content": [ + { + "type": "tool_use", + "name": tool_name, + "id": "tool-1", + "input": input_kwargs, + } + ] + }, + } + ) + + +class TestIdleStallLifespanStarted: + def test_idle_stall_failure_preserves_lifespan_started_true(self): + stdout = _tool_use_ndjson("Write", file_path="/worktree/src/foo.py") + result = _idle_stall_result(stdout) + skill_result = _build_skill_result(result) + assert skill_result.lifespan_started is True + + def test_idle_stall_failure_preserves_lifespan_started_false(self): + result = _idle_stall_result("") + skill_result = _build_skill_result(result) + assert skill_result.lifespan_started is False diff --git a/tests/execution/test_headless_result_write_reconciliation.py b/tests/execution/test_headless_result_write_reconciliation.py new file mode 100644 index 0000000000..ee9d254e00 --- /dev/null +++ b/tests/execution/test_headless_result_write_reconciliation.py @@ -0,0 +1,95 @@ +"""Integration tests for EMPTY_OUTPUT + write-evidence reconciliation gate. + +Verifies that _build_skill_result() reclassifies EMPTY_OUTPUT to COMPLETED_NO_FLUSH +when write evidence exists, preserving partial worktree progress instead of discarding it. +""" + +from __future__ import annotations + +import json +from datetime import UTC, datetime + +import pytest + +from autoskillit.core.types import ( + RetryReason, +) +from autoskillit.execution.headless import _build_skill_result +from autoskillit.pipeline.audit import DefaultAuditLog, FailureRecord +from tests.execution.conftest import EMPTY_OUTPUT_RESULT_LINE, WRITE_TOOL_LINE, _sr + +pytestmark = [pytest.mark.layer("execution"), pytest.mark.small] + +_SUCCESS_EMPTY_RESULT_LINE = json.dumps( + { + "type": "result", + "subtype": "success", + "is_error": False, + "result": "", + "session_id": "s1", + } +) + + +class TestEmptyOutputWriteReconciliation: + def test_empty_output_with_write_tool_use_becomes_completed_no_flush(self) -> None: + stdout = "\n".join([WRITE_TOOL_LINE, EMPTY_OUTPUT_RESULT_LINE]) + sr = _build_skill_result(_sr(stdout=stdout)) + + assert sr.needs_retry is True + assert sr.retry_reason == RetryReason.COMPLETED_NO_FLUSH + assert sr.retry_reason != RetryReason.EMPTY_OUTPUT + assert sr.write_call_count > 0 + + def test_empty_output_with_fs_writes_only_becomes_completed_no_flush(self) -> None: + """fs_writes_detected alone (no tool_uses) triggers reclassification.""" + stdout = EMPTY_OUTPUT_RESULT_LINE + sr = _build_skill_result(_sr(stdout=stdout), fs_writes_detected=True) + + assert sr.needs_retry is True + assert sr.retry_reason == RetryReason.COMPLETED_NO_FLUSH + assert sr.retry_reason != RetryReason.EMPTY_OUTPUT + + def test_success_empty_result_with_write_evidence_becomes_completed_no_flush(self) -> None: + """success subtype with empty result + write evidence → COMPLETED_NO_FLUSH.""" + stdout = "\n".join([WRITE_TOOL_LINE, _SUCCESS_EMPTY_RESULT_LINE]) + sr = _build_skill_result(_sr(stdout=stdout)) + + assert sr.needs_retry is True + assert sr.retry_reason == RetryReason.COMPLETED_NO_FLUSH + assert sr.retry_reason != RetryReason.EMPTY_OUTPUT + + def test_empty_output_without_write_evidence_stays_empty_output(self) -> None: + """EMPTY_OUTPUT with no writes preserves original reason — correct behavior unchanged.""" + stdout = EMPTY_OUTPUT_RESULT_LINE + sr = _build_skill_result(_sr(stdout=stdout), fs_writes_detected=False) + + assert sr.needs_retry is True + assert sr.retry_reason == RetryReason.EMPTY_OUTPUT + + def test_budget_guard_caps_completed_no_flush(self) -> None: + """Budget exhaustion overrides COMPLETED_NO_FLUSH → BUDGET_EXHAUSTED.""" + stdout = "\n".join([WRITE_TOOL_LINE, EMPTY_OUTPUT_RESULT_LINE]) + skill_command = "/autoskillit:implement-worktree-no-merge" + audit = DefaultAuditLog() + for _ in range(3): # 3 pre-existing + 1 from _capture_failure = 4 > max=3 + audit.record_failure( + FailureRecord( # type: ignore[arg-type] + timestamp=datetime.now(UTC).isoformat(), + skill_command=skill_command, + exit_code=-1, + subtype="empty_output", + needs_retry=True, + retry_reason="empty_output", + stderr="", + ) + ) + + sr = _build_skill_result( + _sr(stdout=stdout), + skill_command=skill_command, + audit=audit, + ) + + assert sr.retry_reason == RetryReason.BUDGET_EXHAUSTED + assert sr.needs_retry is False diff --git a/tests/execution/test_headless_synthesis.py b/tests/execution/test_headless_synthesis.py new file mode 100644 index 0000000000..d759608253 --- /dev/null +++ b/tests/execution/test_headless_synthesis.py @@ -0,0 +1,542 @@ +"""Tests for headless.py synthesis helpers: output path extraction, validation, contamination.""" + +import json + +import pytest + +from autoskillit.core.types import ( + ChannelConfirmation, + RetryReason, + SubprocessResult, + TerminationReason, +) +from autoskillit.execution.headless import _build_skill_result, _scan_jsonl_write_paths +from tests.execution.conftest import _make_tool_use_line, _sr, _success_session_json + +pytestmark = [pytest.mark.layer("execution"), pytest.mark.small] + + +class TestExtractOutputPaths: + def test_extracts_single_path(self): + from autoskillit.execution.headless import _extract_output_paths + + msgs = ["plan_path = /correct/path/.autoskillit/temp/make-plan/foo.md"] + result = _extract_output_paths(msgs) + assert result == {"plan_path": "/correct/path/.autoskillit/temp/make-plan/foo.md"} + + def test_extracts_multiple_tokens(self): + from autoskillit.execution.headless import _extract_output_paths + + msg = ( + "plan_path = /clone/.autoskillit/temp/make-plan/plan.md\n" + "summary_path = /clone/.autoskillit/temp/report/summary.md\n" + "investigation_path = /clone/.autoskillit/temp/investigate/inv.md" + ) + result = _extract_output_paths([msg]) + assert result == { + "plan_path": "/clone/.autoskillit/temp/make-plan/plan.md", + "summary_path": "/clone/.autoskillit/temp/report/summary.md", + "investigation_path": "/clone/.autoskillit/temp/investigate/inv.md", + } + + def test_returns_empty_when_no_tokens(self): + from autoskillit.execution.headless import _extract_output_paths + + result = _extract_output_paths(["no tokens here", "just regular text"]) + assert result == {} + + def test_ignores_non_absolute_paths(self): + from autoskillit.execution.headless import _extract_output_paths + + result = _extract_output_paths(["plan_path = .autoskillit/temp/make-plan/foo.md"]) + assert result == {} + + def test_extracts_from_multiple_messages(self): + from autoskillit.execution.headless import _extract_output_paths + + msgs = [ + "plan_path = /first/path", + "investigation_path = /second/path", + ] + result = _extract_output_paths(msgs) + assert result == { + "plan_path": "/first/path", + "investigation_path": "/second/path", + } + + def test_last_occurrence_wins(self): + from autoskillit.execution.headless import _extract_output_paths + + msgs = [ + "plan_path = /first/path", + "plan_path = /second/path", + ] + result = _extract_output_paths(msgs) + assert result == {"plan_path": "/second/path"} + + +class TestValidateOutputPaths: + def test_returns_none_when_all_paths_under_cwd(self): + from autoskillit.execution.headless import _validate_output_paths + + paths = { + "plan_path": "/clone/path/.autoskillit/temp/make-plan/foo.md", + "report_path": "/clone/path/.autoskillit/temp/report/bar.md", + } + result = _validate_output_paths(paths, "/clone/path") + assert result is None + + def test_returns_diagnostic_when_path_outside_cwd(self): + from autoskillit.execution.headless import _validate_output_paths + + paths = { + "plan_path": "/source/repo/.autoskillit/temp/make-plan/foo.md", + } + result = _validate_output_paths(paths, "/clone/path") + assert result is not None + assert "plan_path" in result + assert "/source/repo/.autoskillit/temp/make-plan/foo.md" in result + assert "/clone/path" in result + + def test_returns_none_for_empty_paths(self): + from autoskillit.execution.headless import _validate_output_paths + + result = _validate_output_paths({}, "/clone/path") + assert result is None + + def test_cwd_trailing_slash_handling(self): + from autoskillit.execution.headless import _validate_output_paths + + paths = {"plan_path": "/clone/path/.autoskillit/temp/foo.md"} + assert _validate_output_paths(paths, "/clone/path/") is None + assert _validate_output_paths(paths, "/clone/path") is None + + def test_multiple_violations(self): + from autoskillit.execution.headless import _validate_output_paths + + paths = { + "plan_path": "/wrong/.autoskillit/temp/plan.md", + "report_path": "/wrong/.autoskillit/temp/report.md", + } + result = _validate_output_paths(paths, "/clone") + assert result is not None + assert "plan_path" in result + assert "report_path" in result + + +class TestBuildSkillResultPathContamination: + @staticmethod + def _assistant_ndjson(text: str) -> str: + """Build an NDJSON assistant message line.""" + return json.dumps( + { + "type": "assistant", + "message": {"content": [{"type": "text", "text": text}]}, + } + ) + + def test_path_contamination_detected(self): + """Output paths outside cwd override success to False.""" + path = "/wrong/source/repo/.autoskillit/temp/make-plan/foo.md" + stdout = ( + self._assistant_ndjson(f"plan_path = {path}") + + "\n" + + _success_session_json("Plan created.") + ) + result = _sr(0, stdout, "", TerminationReason.NATURAL_EXIT) + sr = _build_skill_result(result, cwd="/correct/clone") + assert sr.success is False + assert sr.subtype == "path_contamination" + assert sr.needs_retry is True + assert sr.retry_reason == RetryReason.PATH_CONTAMINATION + + def test_path_contamination_emits_path_contamination_reason(self): + """Path contamination must emit PATH_CONTAMINATION, not RESUME. + + PATH_CONTAMINATION is not a context limit — it is a CWD boundary violation. + The orchestrator must route to on_failure, not on_context_limit. + """ + path = "/wrong/source/repo/.autoskillit/temp/make-plan/bar.md" + stdout = ( + self._assistant_ndjson(f"plan_path = {path}") + + "\n" + + _success_session_json("Plan created.") + ) + result = _sr(0, stdout, "", TerminationReason.NATURAL_EXIT) + sr = _build_skill_result(result, cwd="/correct/clone") + assert sr.retry_reason == RetryReason.PATH_CONTAMINATION + assert sr.needs_retry is True + + def test_no_contamination_when_paths_under_cwd(self): + """All output paths under cwd yields normal result.""" + path = "/correct/clone/.autoskillit/temp/make-plan/foo.md" + stdout = ( + self._assistant_ndjson(f"plan_path = {path}") + + "\n" + + _success_session_json("Plan created.") + ) + result = _sr(0, stdout, "", TerminationReason.NATURAL_EXIT) + sr = _build_skill_result(result, cwd="/correct/clone") + assert sr.success is True + assert sr.subtype != "path_contamination" + + def test_no_contamination_when_cwd_empty(self): + """Empty cwd skips path validation — direct callers without clone context.""" + path = "/any/path/.autoskillit/temp/foo.md" + stdout = ( + self._assistant_ndjson(f"plan_path = {path}") + "\n" + _success_session_json("Done.") + ) + result = _sr(0, stdout, "", TerminationReason.NATURAL_EXIT) + sr = _build_skill_result(result, cwd="") + assert sr.success is True + + def test_no_contamination_when_no_path_tokens(self): + """No output path tokens means validation passes.""" + stdout = _success_session_json("Done with no file output.") + result = _sr(0, stdout, "", TerminationReason.NATURAL_EXIT) + sr = _build_skill_result(result, cwd="/clone/path") + assert sr.success is True + + +class TestRunHeadlessCorePassesCwd: + @pytest.mark.anyio + async def test_cwd_anchor_injected_into_prompt(self, tool_ctx): + """Verify run_headless_core injects cwd anchor into the skill prompt.""" + from autoskillit.execution.headless import run_headless_core + + payload = _success_session_json("Result text %%ORDER_UP%%") + tool_ctx.runner.push( + SubprocessResult(0, payload, "", TerminationReason.NATURAL_EXIT, pid=1) + ) + await run_headless_core( + "/autoskillit:investigate test", + "/some/test/cwd", + tool_ctx, + ) + assert tool_ctx.runner.call_args_list, "Runner was never called" + last_cmd = tool_ctx.runner.call_args_list[-1][0] + # cmd is ["env", ...vars, "claude", "-p", , ...] + p_idx = last_cmd.index("-p") + prompt_arg = last_cmd[p_idx + 1] + assert "WORKING DIRECTORY ANCHOR" in prompt_arg + assert "/some/test/cwd" in prompt_arg + + +class TestScanJsonlWritePaths: + CWD = "/clone/worktree" + + def test_returns_empty_for_clean_write_inside_cwd(self): + line = _make_tool_use_line( + "Write", {"file_path": f"{self.CWD}/.autoskillit/temp/out.md", "content": "x"} + ) + assert _scan_jsonl_write_paths(line, self.CWD) == [] + + def test_detects_write_outside_cwd(self): + line = _make_tool_use_line( + "Write", {"file_path": "/source/repo/.autoskillit/temp/stolen.md", "content": "x"} + ) + warnings = _scan_jsonl_write_paths(line, self.CWD) + assert len(warnings) == 1 + assert "/source/repo/.autoskillit/temp/stolen.md" in warnings[0] + + def test_detects_edit_outside_cwd(self): + line = _make_tool_use_line( + "Edit", + { + "file_path": "/source/repo/src/autoskillit/file.py", + "old_string": "a", + "new_string": "b", + }, + ) + warnings = _scan_jsonl_write_paths(line, self.CWD) + assert len(warnings) == 1 + assert "Edit" in warnings[0] + + def test_detects_bash_with_absolute_path_outside_cwd(self): + line = _make_tool_use_line( + "Bash", {"command": "cat /source/repo/README.md > /tmp/out.txt"} + ) + warnings = _scan_jsonl_write_paths(line, self.CWD) + assert len(warnings) >= 1 + assert any("/source/repo" in w for w in warnings) + + def test_no_warnings_for_empty_stdout(self): + assert _scan_jsonl_write_paths("", self.CWD) == [] + + def test_no_warnings_for_malformed_jsonl(self): + assert _scan_jsonl_write_paths("not json at all\n{broken", self.CWD) == [] + + def test_no_warnings_for_read_only_tool_calls(self): + line = _make_tool_use_line("Read", {"file_path": "/source/repo/some_file.py"}) + assert _scan_jsonl_write_paths(line, self.CWD) == [] + + def test_multiple_violations_in_one_session(self): + lines = "\n".join( + [ + _make_tool_use_line("Write", {"file_path": "/source/repo/a.md", "content": "x"}), + _make_tool_use_line( + "Edit", + {"file_path": "/source/repo/b.py", "old_string": "a", "new_string": "b"}, + ), + json.dumps( + { + "type": "result", + "subtype": "success", + "result": "done", + "session_id": "", + "is_error": False, + } + ), + ] + ) + warnings = _scan_jsonl_write_paths(lines, self.CWD) + assert len(warnings) == 2 + + def test_no_warning_when_cwd_empty(self): + line = _make_tool_use_line("Write", {"file_path": "/any/path/file.md", "content": "x"}) + assert _scan_jsonl_write_paths(line, "") == [] + + def test_no_warning_when_cwd_is_relative(self): + line = _make_tool_use_line("Write", {"file_path": "/any/path/file.md", "content": "x"}) + assert _scan_jsonl_write_paths(line, "relative/path") == [] + + +class TestBuildSkillResultWritePathWarnings: + def test_write_path_warnings_empty_for_clean_session(self): + stdout = ( + _make_tool_use_line( + "Write", {"file_path": "/clone/worktree/.autoskillit/temp/out.md", "content": "x"} + ) + + "\n" + + _success_session_json("Done %%DONE%%") + ) + result = _sr(0, stdout, "", TerminationReason.NATURAL_EXIT) + sr = _build_skill_result(result, cwd="/clone/worktree") + assert sr.write_path_warnings == [] + + def test_write_path_warnings_populated_for_contaminated_session(self): + stdout = ( + _make_tool_use_line( + "Write", {"file_path": "/source/repo/.autoskillit/temp/stolen.md", "content": "x"} + ) + + "\n" + + _success_session_json("Done %%DONE%%") + ) + result = _sr(0, stdout, "", TerminationReason.NATURAL_EXIT) + sr = _build_skill_result(result, cwd="/clone/worktree") + assert len(sr.write_path_warnings) == 1 + assert "/source/repo/.autoskillit/temp/stolen.md" in sr.write_path_warnings[0] + + def test_write_path_warnings_appear_in_to_json(self): + stdout = ( + _make_tool_use_line("Write", {"file_path": "/source/repo/bad.md", "content": "x"}) + + "\n" + + _success_session_json("Done %%DONE%%") + ) + result = _sr(0, stdout, "", TerminationReason.NATURAL_EXIT) + sr = _build_skill_result(result, cwd="/clone/worktree") + data = json.loads(sr.to_json()) + assert "write_path_warnings" in data + assert len(data["write_path_warnings"]) == 1 + + def test_write_path_warnings_independent_of_output_token_contamination(self): + """Warnings are populated even when _validate_output_paths also fires.""" + # plan_path token must appear in an assistant text message for + # _validate_output_paths to detect it (it scans assistant_messages, + # not the final result record). + path_token_line = json.dumps( + { + "type": "assistant", + "message": { + "content": [ + { + "type": "text", + "text": "plan_path = /source/repo/.autoskillit/temp/plan.md", + } + ] + }, + } + ) + stdout = ( + _make_tool_use_line("Write", {"file_path": "/source/repo/bad.md", "content": "x"}) + + "\n" + + path_token_line + + "\n" + + _success_session_json("Done %%DONE%%") + ) + result = _sr(0, stdout, "", TerminationReason.NATURAL_EXIT) + sr = _build_skill_result(result, cwd="/clone/worktree") + # subtype is path_contamination (from _validate_output_paths) + assert sr.subtype == "path_contamination" + # write_path_warnings also populated from JSONL scan + assert len(sr.write_path_warnings) >= 1 + + +class TestBuildSkillResultChannelBPatternRecovery: + """When Channel B wins and pattern is absent from result but present in + assistant_messages, _build_skill_result should recover and produce success=True. + """ + + def test_build_skill_result_channel_b_recovers_pattern_from_assistant_messages( + self, + ) -> None: + """Channel B wins before stdout drain; pattern found in assistant_messages + → recovery produces success=True with block in result. + """ + block = "---prepare-issue-result---\n{}\n---/prepare-issue-result---" + # Assistant message has the block but NOT a standalone %%ORDER_UP%% marker, + # so _recover_from_separate_marker will not activate. The new + # _recover_block_from_assistant_messages must find the block here instead. + assistant_line = json.dumps( + { + "type": "assistant", + "message": { + "content": [{"type": "text", "text": block}], + }, + } + ) + result_line = json.dumps( + { + "type": "result", + "subtype": "success", + "is_error": False, + "result": "", # stdout not yet drained — Channel B won the race + "session_id": "s1", + "errors": [], + } + ) + stdout = assistant_line + "\n" + result_line + + sub_result = SubprocessResult( + returncode=0, + stdout=stdout, + stderr="", + termination=TerminationReason.COMPLETED, + pid=12345, + channel_confirmation=ChannelConfirmation.CHANNEL_B, + ) + sr = _build_skill_result( + sub_result, + completion_marker="%%ORDER_UP%%", + expected_output_patterns=["---prepare-issue-result---"], + ) + assert sr.success is True + assert "---prepare-issue-result---" in sr.result + + def test_synthesis_not_run_for_unparseable_channel_b_with_write_evidence(self) -> None: + """CHANNEL_B + UNPARSEABLE + write evidence must not produce success=True. + + Recovery (and CHANNEL_B bypass) must be blocked when session.session_complete + is False. Synthesis must not inject the file path into result. + """ + + tool_use_line = json.dumps( + { + "type": "tool_use", + "id": "t1", + "name": "Write", + "input": { + "file_path": "/cwd/.autoskillit/temp/make-plan/arch_lens_selection_2026-01-01.md" # noqa: E501 + }, + } + ) + # UNPARSEABLE result (no proper result record) + raw_stdout = tool_use_line + "\ngarbage ndjson partial" + + sub_result = SubprocessResult( + returncode=1, + stdout=raw_stdout, + stderr="", + termination=TerminationReason.COMPLETED, + pid=12345, + channel_confirmation=ChannelConfirmation.CHANNEL_B, + ) + sr = _build_skill_result( + sub_result, + expected_output_patterns=[r"plan_path\s*=\s*/.+"], + ) + assert sr.success is False + assert sr.subtype not in {"success"} + # Synthesis must not have injected the structured token (raw NDJSON may contain + # the file path string, but the plan_path = ... assignment must be absent) + assert "plan_path =" not in sr.result + + def test_synthesis_not_run_for_timeout_channel_b_with_write_evidence(self) -> None: + """CHANNEL_B + TIMEOUT + write evidence must not produce success=True.""" + tool_use_line = json.dumps( + { + "type": "tool_use", + "id": "t1", + "name": "Write", + "input": { + "file_path": "/cwd/.autoskillit/temp/make-plan/arch_lens_selection_2026-01-01.md" # noqa: E501 + }, + } + ) + result_line = json.dumps( + { + "type": "result", + "subtype": "success", + "is_error": False, + "result": "", + "session_id": "s1", + "errors": [], + } + ) + stdout = tool_use_line + "\n" + result_line + + sub_result = SubprocessResult( + returncode=-1, + stdout=stdout, + stderr="", + termination=TerminationReason.TIMED_OUT, + pid=12345, + channel_confirmation=ChannelConfirmation.CHANNEL_B, + ) + sr = _build_skill_result( + sub_result, + expected_output_patterns=[r"plan_path\s*=\s*/.+"], + ) + assert sr.success is False + assert "arch_lens_selection" not in sr.result + + def test_synthesis_skipped_for_channel_b_session_complete(self) -> None: + """CHANNEL_B + SUCCESS + write evidence but no pattern in assistant_messages. + + Synthesis must NOT fabricate the token — if pattern is absent from + assistant_messages, the agent never emitted it. + """ + tool_use_line = json.dumps( + { + "type": "tool_use", + "id": "t1", + "name": "Write", + "input": {"file_path": "/cwd/.autoskillit/temp/make-plan/task_plan.md"}, + } + ) + result_line = json.dumps( + { + "type": "result", + "subtype": "success", + "is_error": False, + "result": "done", + "session_id": "s1", + "errors": [], + } + ) + stdout = tool_use_line + "\n" + result_line + + sub_result = SubprocessResult( + returncode=0, + stdout=stdout, + stderr="", + termination=TerminationReason.COMPLETED, + pid=12345, + channel_confirmation=ChannelConfirmation.CHANNEL_B, + ) + sr = _build_skill_result( + sub_result, + expected_output_patterns=[r"plan_path\s*=\s*/.+"], + ) + assert sr.success is False + assert "plan_path" not in sr.result diff --git a/tests/execution/test_idle_output_env.py b/tests/execution/test_idle_output_env.py new file mode 100644 index 0000000000..62b477d7e1 --- /dev/null +++ b/tests/execution/test_idle_output_env.py @@ -0,0 +1,219 @@ +"""Group G (execution part): AUTOSKILLIT_IDLE_OUTPUT_TIMEOUT env variable injection tests.""" + +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any + +import pytest + +from autoskillit.core.types import SubprocessResult, TerminationReason +from tests.fakes import MockSubprocessRunner + +pytestmark = [pytest.mark.layer("execution"), pytest.mark.small] + + +def _success_result() -> SubprocessResult: + """Build a minimal successful SubprocessResult for MockSubprocessRunner.""" + return SubprocessResult( + returncode=0, + stdout=json.dumps( + { + "type": "result", + "subtype": "success", + "is_error": False, + "result": "done", + "session_id": "sess-idle-test", + } + ), + stderr="", + termination=TerminationReason.NATURAL_EXIT, + pid=12345, + ) + + +class TestExecuteClaudeHeadlessIdleEnv: + @pytest.mark.anyio + async def test_execute_claude_headless_reads_idle_output_env( + self, minimal_ctx, tmp_path: Path, monkeypatch + ) -> None: + """When idle_output_timeout=None and AUTOSKILLIT_IDLE_OUTPUT_TIMEOUT is set in env, + run_headless_core uses the env value as the effective idle timeout.""" + from autoskillit.execution.headless import run_headless_core + + monkeypatch.setenv("AUTOSKILLIT_IDLE_OUTPUT_TIMEOUT", "30") + minimal_ctx.runner = MockSubprocessRunner() + minimal_ctx.runner.set_default(_success_result()) + + await run_headless_core("/investigate foo", str(tmp_path), minimal_ctx) + + assert minimal_ctx.runner.call_args_list, "runner was never called" + _cmd, _cwd, _timeout, kwargs = minimal_ctx.runner.call_args_list[0] + assert kwargs.get("idle_output_timeout") == 30.0, ( + f"Expected idle_output_timeout=30.0, got {kwargs.get('idle_output_timeout')!r}" + ) + + @pytest.mark.anyio + async def test_idle_output_timeout_priority_chain( + self, minimal_ctx, tmp_path: Path, monkeypatch + ) -> None: + """Priority chain: per-step arg > AUTOSKILLIT_IDLE_OUTPUT_TIMEOUT env > cfg. + + Level 1: per-step arg beats env and cfg. + Level 2: env beats cfg when per-step arg is None. + Level 3: cfg is used when both arg and env are absent. + """ + from autoskillit.execution.headless import run_headless_core + + # Level 1: per-step arg takes priority over env and cfg + monkeypatch.setenv("AUTOSKILLIT_IDLE_OUTPUT_TIMEOUT", "30") + minimal_ctx.config.run_skill.idle_output_timeout = 60 + minimal_ctx.runner = MockSubprocessRunner() + minimal_ctx.runner.set_default(_success_result()) + + await run_headless_core( + "/investigate foo", str(tmp_path), minimal_ctx, idle_output_timeout=15.0 + ) + _, _, _, kwargs1 = minimal_ctx.runner.call_args_list[0] + assert kwargs1.get("idle_output_timeout") == 15.0, ( + f"Level 1 (per-step arg): expected 15.0, got {kwargs1.get('idle_output_timeout')!r}" + ) + + # Level 2: env beats cfg when per-step arg is None + minimal_ctx.runner = MockSubprocessRunner() + minimal_ctx.runner.set_default(_success_result()) + + await run_headless_core( + "/investigate foo", str(tmp_path), minimal_ctx, idle_output_timeout=None + ) + _, _, _, kwargs2 = minimal_ctx.runner.call_args_list[0] + assert kwargs2.get("idle_output_timeout") == 30.0, ( + f"Level 2 (env): expected 30.0, got {kwargs2.get('idle_output_timeout')!r}" + ) + + # Level 3: cfg when env is absent and arg is None + monkeypatch.delenv("AUTOSKILLIT_IDLE_OUTPUT_TIMEOUT") + minimal_ctx.config.run_skill.idle_output_timeout = 45 + minimal_ctx.runner = MockSubprocessRunner() + minimal_ctx.runner.set_default(_success_result()) + + await run_headless_core( + "/investigate foo", str(tmp_path), minimal_ctx, idle_output_timeout=None + ) + _, _, _, kwargs3 = minimal_ctx.runner.call_args_list[0] + assert kwargs3.get("idle_output_timeout") == 45.0, ( + f"Level 3 (cfg): expected 45.0, got {kwargs3.get('idle_output_timeout')!r}" + ) + + @pytest.mark.anyio + async def test_idle_output_env_zero_means_disabled( + self, minimal_ctx, tmp_path: Path, monkeypatch + ) -> None: + """AUTOSKILLIT_IDLE_OUTPUT_TIMEOUT=0 → effective idle is None (feature disabled).""" + from autoskillit.execution.headless import run_headless_core + + monkeypatch.setenv("AUTOSKILLIT_IDLE_OUTPUT_TIMEOUT", "0") + minimal_ctx.config.run_skill.idle_output_timeout = 0 # cfg also 0 + minimal_ctx.runner = MockSubprocessRunner() + minimal_ctx.runner.set_default(_success_result()) + + await run_headless_core("/investigate foo", str(tmp_path), minimal_ctx) + + assert minimal_ctx.runner.call_args_list, "runner was never called" + _, _, _, kwargs = minimal_ctx.runner.call_args_list[0] + actual = kwargs.get("idle_output_timeout") + assert actual is None, f"Expected idle_output_timeout=None when env=0, got {actual!r}" + + @pytest.mark.anyio + async def test_idle_output_env_invalid_float_falls_back_to_config( + self, minimal_ctx, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + from autoskillit.execution.headless import run_headless_core + + monkeypatch.setenv("AUTOSKILLIT_IDLE_OUTPUT_TIMEOUT", "not-a-number") + + async def _no_nudge(*_a, **_kw): + return None + + monkeypatch.setattr( + "autoskillit.execution.headless._attempt_contract_nudge", + _no_nudge, + ) + minimal_ctx.config.run_skill.idle_output_timeout = 45 + + runner = MockSubprocessRunner() + runner.set_default(_success_result()) + minimal_ctx.runner = runner + + await run_headless_core( + "/autoskillit:some-skill", + str(tmp_path), + minimal_ctx, + ) + idle = runner.call_args_list[-1][3].get("idle_output_timeout") + assert idle == 45.0 + + +class TestDispatchFoodTruckIdleEnvInjection: + @pytest.mark.anyio + async def test_dispatch_food_truck_injects_idle_output_timeout_env( + self, minimal_ctx, tmp_path: Path, monkeypatch + ) -> None: + """dispatch_food_truck adds AUTOSKILLIT_IDLE_OUTPUT_TIMEOUT to env_extras + based on fleet config idle_output_timeout (priority: caller > fleet > run_skill).""" + from autoskillit.core.types import SkillResult as _SkillResult + from autoskillit.execution.headless import DefaultHeadlessExecutor + + minimal_ctx.config.fleet.idle_output_timeout = 120 + + captured_env_extras: list[dict[str, str] | None] = [] + + def _capture_build(**kwargs: Any): + captured_env_extras.append(kwargs.get("env_extras")) + from autoskillit.execution.commands import ClaudeHeadlessCmd + + return ClaudeHeadlessCmd(cmd=["echo", "done"], env={}) + + async def _fake_execute(*_args: Any, **_kwargs: Any) -> _SkillResult: + from autoskillit.core.types import KillReason, RetryReason + + return _SkillResult( + success=True, + result="done", + session_id="", + subtype="success", + is_error=False, + exit_code=0, + needs_retry=False, + retry_reason=RetryReason.NONE, + stderr="", + kill_reason=KillReason.NATURAL_EXIT, + ) + + monkeypatch.setattr( + "autoskillit.execution.headless.build_food_truck_cmd", + _capture_build, + ) + monkeypatch.setattr( + "autoskillit.execution.headless._execute_claude_headless", + _fake_execute, + ) + + from autoskillit.core.types._type_plugin_source import DirectInstall + + minimal_ctx.plugin_source = DirectInstall(plugin_dir=tmp_path / "plugin") + executor = DefaultHeadlessExecutor(minimal_ctx) + await executor.dispatch_food_truck( + orchestrator_prompt="test", + cwd=str(tmp_path), + completion_marker="%%DONE%%", + env_extras=None, + ) + + assert captured_env_extras, "build_food_truck_cmd was not called" + env = captured_env_extras[0] or {} + assert "AUTOSKILLIT_IDLE_OUTPUT_TIMEOUT" in env, ( + f"Expected AUTOSKILLIT_IDLE_OUTPUT_TIMEOUT in env_extras, got {env!r}" + ) + assert env["AUTOSKILLIT_IDLE_OUTPUT_TIMEOUT"] == "120" diff --git a/tests/execution/test_linux_tracing.py b/tests/execution/test_linux_tracing.py index 45b3c3f501..4a22c07373 100644 --- a/tests/execution/test_linux_tracing.py +++ b/tests/execution/test_linux_tracing.py @@ -10,10 +10,14 @@ import anyio import pytest -pytestmark = pytest.mark.skipif( - sys.platform != "linux", - reason="Linux-only tracing tests", -) +pytestmark = [ + pytest.mark.layer("execution"), + pytest.mark.medium, + pytest.mark.skipif( + sys.platform != "linux", + reason="Linux-only tracing tests", + ), +] PROC_STATUS_FIXTURE = """\ Name:\tclaude @@ -84,11 +88,11 @@ async def test_tracing_handle_accumulates_snapshots(tmp_path): import anyio - from autoskillit.config import LinuxTracingConfig from autoskillit.execution.linux_tracing import start_linux_tracing, trace_target_from_pid + from tests._helpers import make_tracing_config proc = subprocess.Popen(["sleep", "2"]) - cfg = LinuxTracingConfig(enabled=True, proc_interval=0.1, tmpfs_path=str(tmp_path)) + cfg = make_tracing_config(enabled=True, proc_interval=0.1, tmpfs_path=str(tmp_path)) async with anyio.create_task_group() as tg: handle = start_linux_tracing(target=trace_target_from_pid(proc.pid), config=cfg, tg=tg) @@ -110,10 +114,10 @@ async def test_tracing_handle_stop_returns_snapshots(tmp_path): import anyio - from autoskillit.config import LinuxTracingConfig from autoskillit.execution.linux_tracing import start_linux_tracing, trace_target_from_pid + from tests._helpers import make_tracing_config - cfg = LinuxTracingConfig(enabled=True, proc_interval=0.1, tmpfs_path=str(tmp_path)) + cfg = make_tracing_config(enabled=True, proc_interval=0.1, tmpfs_path=str(tmp_path)) async with anyio.create_task_group() as tg: handle = start_linux_tracing(target=trace_target_from_pid(os.getpid()), config=cfg, tg=tg) @@ -144,11 +148,11 @@ def test_noop_on_non_linux(monkeypatch, tmp_path): """start_linux_tracing is a no-op when LINUX_TRACING_AVAILABLE is False.""" import os - from autoskillit.config import LinuxTracingConfig from autoskillit.execution import linux_tracing + from tests._helpers import make_tracing_config monkeypatch.setattr(linux_tracing, "LINUX_TRACING_AVAILABLE", False) - cfg = LinuxTracingConfig(enabled=True, proc_interval=1.0, tmpfs_path=str(tmp_path)) + cfg = make_tracing_config(enabled=True, proc_interval=1.0, tmpfs_path=str(tmp_path)) result = linux_tracing.start_linux_tracing( target=linux_tracing.trace_target_from_pid(os.getpid()), config=cfg, tg=None ) @@ -191,10 +195,10 @@ async def test_proc_monitor_stamps_unique_captured_at(): @pytest.mark.anyio async def test_start_linux_tracing_creates_trace_file(tmp_path): """When tmpfs_path is configured, start_linux_tracing opens a trace file.""" - from autoskillit.config import LinuxTracingConfig from autoskillit.execution.linux_tracing import start_linux_tracing, trace_target_from_pid + from tests._helpers import make_tracing_config - config = LinuxTracingConfig(enabled=True, proc_interval=0.01, tmpfs_path=str(tmp_path)) + config = make_tracing_config(enabled=True, proc_interval=0.01, tmpfs_path=str(tmp_path)) async with anyio.create_task_group() as tg: handle = start_linux_tracing(trace_target_from_pid(os.getpid()), config, tg) assert handle is not None @@ -211,11 +215,11 @@ async def test_streaming_writes_each_snapshot_as_jsonl(tmp_path): """Each yielded snapshot appears as a JSONL line in the trace file.""" import subprocess - from autoskillit.config import LinuxTracingConfig from autoskillit.execution.linux_tracing import start_linux_tracing, trace_target_from_pid + from tests._helpers import make_tracing_config proc = subprocess.Popen(["sleep", "2"]) - config = LinuxTracingConfig(enabled=True, proc_interval=0.05, tmpfs_path=str(tmp_path)) + config = make_tracing_config(enabled=True, proc_interval=0.05, tmpfs_path=str(tmp_path)) async with anyio.create_task_group() as tg: handle = start_linux_tracing(trace_target_from_pid(proc.pid), config, tg) @@ -271,10 +275,10 @@ def test_stop_idempotent(tmp_path): @pytest.mark.anyio async def test_streaming_graceful_when_tmpfs_missing(tmp_path): """If tmpfs_path does not exist, tracing still works in-memory.""" - from autoskillit.config import LinuxTracingConfig from autoskillit.execution.linux_tracing import start_linux_tracing, trace_target_from_pid + from tests._helpers import make_tracing_config - config = LinuxTracingConfig( + config = make_tracing_config( enabled=True, proc_interval=0.01, tmpfs_path=str(tmp_path / "nonexistent"), @@ -301,7 +305,6 @@ def test_proc_snapshot_has_captured_at_field(): snap = read_proc_snapshot(os.getpid()) assert snap is not None assert hasattr(snap, "captured_at") - assert snap.captured_at # non-empty # Must be UTC-aware ISO 8601 dt = datetime.fromisoformat(snap.captured_at) assert dt.tzinfo is not None @@ -314,10 +317,10 @@ async def test_proc_monitor_snapshots_have_distinct_timestamps(tmp_path): import anyio - from autoskillit.config import LinuxTracingConfig from autoskillit.execution.linux_tracing import start_linux_tracing, trace_target_from_pid + from tests._helpers import make_tracing_config - config = LinuxTracingConfig(proc_interval=0.05, tmpfs_path=str(tmp_path)) + config = make_tracing_config(proc_interval=0.05, tmpfs_path=str(tmp_path)) async with anyio.create_task_group() as tg: handle = start_linux_tracing(trace_target_from_pid(os.getpid()), config, tg) await anyio.sleep(0.2) @@ -359,15 +362,76 @@ async def test_proc_monitor_persists_psutil_process_for_cpu_percent(): proc.wait() +def test_parse_net_tcp_established_port_443(): + """_parse_net_tcp counts ESTABLISHED connections to port 443.""" + from autoskillit.execution.linux_tracing import _parse_net_tcp + + fixture = ( + " sl local_address rem_address st tx_queue rx_queue tr...\n" + " 0: 0100007F:0035 0101A8C0:01BB 01 00000000:00000000 ...\n" # ESTABLISHED to :443 + " 1: 0100007F:0035 0101A8C0:0050 01 00000000:00000000 ...\n" # :80 excluded + ) + result = _parse_net_tcp(fixture) + assert result.get("ESTABLISHED") == 1 + + +def test_parse_net_tcp_empty_returns_empty_dict(): + from autoskillit.execution.linux_tracing import _parse_net_tcp + + assert _parse_net_tcp("") == {} + assert _parse_net_tcp(" sl local_address rem_address st\n") == {} + + +def test_parse_proc_io_extracts_bytes(): + from autoskillit.execution.linux_tracing import _parse_proc_io + + fixture = "rchar: 1000\nwchar: 2000\nread_bytes: 4096\nwrite_bytes: 8192\n" + read_b, write_b = _parse_proc_io(fixture) + assert read_b == 4096 + assert write_b == 8192 + + +def test_read_proc_snapshot_has_network_fields(): + """read_proc_snapshot includes api_connection_established on Linux.""" + from autoskillit.execution.linux_tracing import read_proc_snapshot + + snap = read_proc_snapshot(os.getpid()) + assert snap is not None + # Field exists and is either None (no port-443 connections from test process) or int >= 0 + assert snap.api_connection_established is None or isinstance( + snap.api_connection_established, int + ) + + +def test_read_proc_snapshot_network_graceful_on_missing_proc_net(monkeypatch): + """Fields are None, no exception, when /proc/{pid}/net/tcp is unavailable.""" + from pathlib import Path + + from autoskillit.execution.linux_tracing import read_proc_snapshot + + original_read_text = Path.read_text + + def patched_read_text(self, *args, **kwargs): + if "net/tcp" in str(self): + raise FileNotFoundError("mocked") + return original_read_text(self, *args, **kwargs) + + monkeypatch.setattr(Path, "read_text", patched_read_text) + snap = read_proc_snapshot(os.getpid()) + assert snap is not None + assert snap.api_connection_established is None + assert snap.api_connection_states is None + + @pytest.mark.anyio async def test_start_linux_tracing_writes_enrollment_sidecar(tmp_path): """start_linux_tracing must write autoskillit_enrollment_{pid}.json immediately.""" import anyio - from autoskillit.config import LinuxTracingConfig from autoskillit.execution.linux_tracing import start_linux_tracing, trace_target_from_pid + from tests._helpers import make_tracing_config - cfg = LinuxTracingConfig(enabled=True, proc_interval=0.1, tmpfs_path=str(tmp_path)) + cfg = make_tracing_config(enabled=True, proc_interval=0.1, tmpfs_path=str(tmp_path)) async with anyio.create_task_group() as tg: proc = await anyio.open_process(["sleep", "2"]) handle = start_linux_tracing(trace_target_from_pid(proc.pid), cfg, tg) @@ -390,34 +454,34 @@ async def test_start_linux_tracing_writes_enrollment_sidecar(tmp_path): def test_tracing_config_rejects_dev_shm_in_test_env(monkeypatch): """LinuxTracingConfig.__post_init__ must raise when tmpfs_path is /dev/shm and PYTEST_CURRENT_TEST env var is set.""" - from autoskillit.config import LinuxTracingConfig + from tests._helpers import make_tracing_config monkeypatch.setenv( "PYTEST_CURRENT_TEST", "tests/execution/test_linux_tracing.py::fake_test", ) with pytest.raises(RuntimeError, match="tmpfs_path|PYTEST_CURRENT_TEST"): - LinuxTracingConfig(tmpfs_path="/dev/shm") + make_tracing_config(tmpfs_path="/dev/shm") def test_tracing_config_allows_custom_tmpfs_in_test_env(monkeypatch, tmp_path): """LinuxTracingConfig must not raise when a non-/dev/shm path is provided.""" - from autoskillit.config import LinuxTracingConfig + from tests._helpers import make_tracing_config monkeypatch.setenv( "PYTEST_CURRENT_TEST", "tests/execution/test_linux_tracing.py::fake_test", ) - cfg = LinuxTracingConfig(tmpfs_path=str(tmp_path)) # must not raise + cfg = make_tracing_config(tmpfs_path=str(tmp_path)) # must not raise assert cfg.tmpfs_path == str(tmp_path) def test_tracing_config_allows_dev_shm_outside_test_env(monkeypatch): """LinuxTracingConfig must not raise for /dev/shm when not running under pytest.""" - from autoskillit.config import LinuxTracingConfig + from tests._helpers import make_tracing_config monkeypatch.delenv("PYTEST_CURRENT_TEST", raising=False) - cfg = LinuxTracingConfig(tmpfs_path="/dev/shm") # production path — must not raise + cfg = make_tracing_config(tmpfs_path="/dev/shm") # production path — must not raise assert cfg.tmpfs_path == "/dev/shm" @@ -486,10 +550,10 @@ async def test_stop_unlinks_trace_and_enrollment(tmp_path): """stop() must delete both trace JSONL and enrollment sidecar on clean exit.""" import anyio - from autoskillit.config import LinuxTracingConfig from autoskillit.execution.linux_tracing import start_linux_tracing, trace_target_from_pid + from tests._helpers import make_tracing_config - cfg = LinuxTracingConfig(enabled=True, proc_interval=0.1, tmpfs_path=str(tmp_path)) + cfg = make_tracing_config(enabled=True, proc_interval=0.1, tmpfs_path=str(tmp_path)) async with anyio.create_task_group() as tg: proc = await anyio.open_process(["sleep", "2"]) handle = start_linux_tracing(trace_target_from_pid(proc.pid), cfg, tg) diff --git a/tests/execution/test_linux_tracing_pty_integration.py b/tests/execution/test_linux_tracing_pty_integration.py index 4fef51cad2..66f52559c0 100644 --- a/tests/execution/test_linux_tracing_pty_integration.py +++ b/tests/execution/test_linux_tracing_pty_integration.py @@ -19,10 +19,14 @@ from tests.execution.conftest import _ALLOCATE_60MB_SCRIPT -pytestmark = pytest.mark.skipif( - sys.platform != "linux", - reason="Linux only — tests PTY wrapping + /proc tracing", -) +pytestmark = [ + pytest.mark.layer("execution"), + pytest.mark.medium, + pytest.mark.skipif( + sys.platform != "linux", + reason="Linux only — tests PTY wrapping + /proc tracing", + ), +] # Skip the entire module when script(1) is absent; no stub needed. pytestmark_script = pytest.mark.skipif( @@ -95,6 +99,11 @@ async def test_pty_wrapped_tracing_produces_no_script_snapshots_in_proc_trace_js Test 1.10 (partial): after the fix, every row in proc_trace.jsonl self-identifies the tracked process, and 'script' must never appear there. """ + from autoskillit.core.types._type_results import ProviderOutcome + from autoskillit.core.types._type_results_execution import ( + RecipeIdentity, + SessionTelemetry, + ) from autoskillit.execution.process import run_managed_async from autoskillit.execution.session_log import flush_session_log @@ -122,6 +131,9 @@ async def test_pty_wrapped_tracing_produces_no_script_snapshots_in_proc_trace_js exit_code=result.returncode if result.returncode is not None else -1, start_ts=result.start_ts or "2026-01-01T00:00:00+00:00", proc_snapshots=result.proc_snapshots, + telemetry=SessionTelemetry.empty(), + provider_outcome=ProviderOutcome.none_used(), + recipe_identity=RecipeIdentity.empty(), ) trace_path = tmp_path / "logs" / "sessions" / "pty-trace-test-001" / "proc_trace.jsonl" diff --git a/tests/execution/test_loc_capture.py b/tests/execution/test_loc_capture.py new file mode 100644 index 0000000000..dc9dcab6ce --- /dev/null +++ b/tests/execution/test_loc_capture.py @@ -0,0 +1,225 @@ +"""Tests for LoC capture helpers in execution.headless. + +T-GIT-1..T-GIT-4: unit tests for _parse_numstat and _compute_loc_changed. +T-GIT-5: real git repo + worktree baseline (validates primitive with real git). +T-GIT-6: PostSessionMetrics resolves effective_cwd from worktree_path. +T-GIT-7: LoC is non-zero when worktree has commits. +""" + +from __future__ import annotations + +import subprocess +from pathlib import Path +from unittest.mock import MagicMock, patch + +import pytest + +pytestmark = [pytest.mark.layer("execution"), pytest.mark.medium] + + +def _parse_numstat(numstat_output: str): + from autoskillit.execution.headless._headless_git import _parse_numstat + + return _parse_numstat(numstat_output) + + +def _compute_loc_changed(cwd: str, pre_sha: str): + from autoskillit.execution.headless._headless_git import _compute_loc_changed + + return _compute_loc_changed(cwd, pre_sha) + + +def _capture_git_head_sha(cwd: str): + from autoskillit.execution.headless._headless_git import _capture_git_head_sha + + return _capture_git_head_sha(cwd) + + +def _compute_post_session_metrics(cwd, pre_session_sha, skill_result): + from autoskillit.execution.headless import _compute_post_session_metrics + + return _compute_post_session_metrics(cwd, pre_session_sha, skill_result) + + +# T-GIT-1 +def test_compute_loc_changed_parses_numstat_output(): + """_parse_numstat correctly sums numstat lines: insertions, deletions.""" + numstat_output = "10\t5\tfile.py\n3\t1\tsrc/foo.py\n" + result = _parse_numstat(numstat_output) + assert result == (13, 6) + + +# T-GIT-2 +def test_compute_loc_changed_returns_zero_on_subprocess_error(tmp_path: Path): + """When git subprocess fails (non-git dir), loc capture returns (0, 0).""" + result = _compute_loc_changed(str(tmp_path), "abc1234") + assert result == (0, 0) + + +# T-GIT-3 +def test_compute_loc_changed_handles_empty_diff(): + """Empty numstat output (no changes) → (0, 0).""" + assert _parse_numstat("") == (0, 0) + + +# T-GIT-4 +def test_compute_loc_changed_handles_binary_files(): + """Binary file lines (-\t-\tbinary.png) are ignored without error.""" + numstat_output = "-\t-\timage.png\n5\t2\tfile.py\n" + assert _parse_numstat(numstat_output) == (5, 2) + + +def test_capture_git_head_sha_returns_empty_on_non_git_dir(tmp_path: Path): + """_capture_git_head_sha returns '' when directory is not a git repo.""" + result = _capture_git_head_sha(str(tmp_path)) + assert result == "" + + +def test_capture_git_head_sha_returns_empty_string_on_exception(): + """_capture_git_head_sha returns '' on any subprocess exception.""" + with patch("subprocess.run", side_effect=OSError("mocked error")): + result = _capture_git_head_sha("/any/path") + assert result == "" + + +def test_compute_loc_changed_returns_zero_for_empty_pre_sha(tmp_path: Path): + """_compute_loc_changed returns (0, 0) when pre_sha is empty.""" + result = _compute_loc_changed(str(tmp_path), "") + assert result == (0, 0) + + +def _setup_git_repo(repo_path: Path) -> None: + repo_path.mkdir(parents=True, exist_ok=True) + subprocess.run(["git", "init"], cwd=repo_path, check=True, capture_output=True) + subprocess.run( + ["git", "config", "user.email", "test@test.com"], + cwd=repo_path, + check=True, + capture_output=True, + ) + subprocess.run( + ["git", "config", "user.name", "Test"], + cwd=repo_path, + check=True, + capture_output=True, + ) + (repo_path / "init.txt").write_text("initial\n") + subprocess.run(["git", "add", "."], cwd=repo_path, check=True, capture_output=True) + subprocess.run( + ["git", "commit", "-m", "init"], + cwd=repo_path, + check=True, + capture_output=True, + ) + + +# T-GIT-5 +def test_compute_loc_changed_clone_root_vs_worktree(tmp_path: Path): + """Clone root returns (0,0); worktree with new commits returns non-zero LoC.""" + repo_path = tmp_path / "repo" + _setup_git_repo(repo_path) + + pre_sha = _capture_git_head_sha(str(repo_path)) + assert pre_sha, "expected non-empty SHA from real git repo" + + worktree_path = tmp_path / "worktree" + subprocess.run( + ["git", "worktree", "add", "-b", "wt-branch", str(worktree_path)], + cwd=repo_path, + check=True, + capture_output=True, + ) + subprocess.run( + ["git", "config", "user.email", "test@test.com"], + cwd=worktree_path, + check=True, + capture_output=True, + ) + subprocess.run( + ["git", "config", "user.name", "Test"], + cwd=worktree_path, + check=True, + capture_output=True, + ) + (worktree_path / "feature.txt").write_text("new content\n" * 5) + subprocess.run(["git", "add", "."], cwd=worktree_path, check=True, capture_output=True) + subprocess.run( + ["git", "commit", "-m", "add feature"], + cwd=worktree_path, + check=True, + capture_output=True, + ) + + # Clone root is untouched — LoC should be (0, 0) + assert _compute_loc_changed(str(repo_path), pre_sha) == (0, 0) + + # Worktree has new commits — LoC should be non-zero insertions, zero deletions + wt_ins, wt_del = _compute_loc_changed(str(worktree_path), pre_sha) + assert wt_ins > 0 + assert wt_del == 0 + + +# T-GIT-6 +def test_post_session_metrics_effective_cwd_resolution(tmp_path: Path): + """PostSessionMetrics resolves effective_cwd from worktree_path when set.""" + cwd = str(tmp_path / "clone") + worktree = str(tmp_path / "worktree") + + mock_result_no_worktree = MagicMock() + mock_result_no_worktree.worktree_path = None + + mock_result_with_worktree = MagicMock() + mock_result_with_worktree.worktree_path = worktree + + with patch("autoskillit.execution.headless._compute_loc_changed", return_value=(0, 0)): + metrics_no_wt = _compute_post_session_metrics(cwd, "abc123", mock_result_no_worktree) + metrics_with_wt = _compute_post_session_metrics(cwd, "abc123", mock_result_with_worktree) + + assert metrics_no_wt.effective_cwd == cwd + assert metrics_with_wt.effective_cwd == worktree + + +# T-GIT-7 +def test_post_session_metrics_non_zero_loc_for_worktree(tmp_path: Path): + """PostSessionMetrics returns non-zero LoC when worktree has new commits.""" + repo_path = tmp_path / "repo" + _setup_git_repo(repo_path) + + pre_sha = _capture_git_head_sha(str(repo_path)) + assert pre_sha + + worktree_path = tmp_path / "worktree" + subprocess.run( + ["git", "worktree", "add", "-b", "wt-branch2", str(worktree_path)], + cwd=repo_path, + check=True, + capture_output=True, + ) + subprocess.run( + ["git", "config", "user.email", "test@test.com"], + cwd=worktree_path, + check=True, + capture_output=True, + ) + subprocess.run( + ["git", "config", "user.name", "Test"], + cwd=worktree_path, + check=True, + capture_output=True, + ) + (worktree_path / "impl.py").write_text("x = 1\n" * 10) + subprocess.run(["git", "add", "."], cwd=worktree_path, check=True, capture_output=True) + subprocess.run( + ["git", "commit", "-m", "implement"], + cwd=worktree_path, + check=True, + capture_output=True, + ) + + mock_skill_result = MagicMock() + mock_skill_result.worktree_path = str(worktree_path) + + metrics = _compute_post_session_metrics(str(repo_path), pre_sha, mock_skill_result) + + assert metrics.loc_insertions > 0 + assert metrics.effective_cwd == str(worktree_path) diff --git a/tests/execution/test_merge_queue.py b/tests/execution/test_merge_queue.py deleted file mode 100644 index 0537898a3e..0000000000 --- a/tests/execution/test_merge_queue.py +++ /dev/null @@ -1,1510 +0,0 @@ -"""Tests for DefaultMergeQueueWatcher polling state machine.""" - -from __future__ import annotations - -from datetime import UTC, datetime, timedelta -from unittest.mock import AsyncMock, patch - -import httpx -import pytest - -import autoskillit.execution.merge_queue as _mq -from autoskillit.core.types import PRState -from autoskillit.execution.merge_queue import ( - ClassifierInconclusive, - DefaultMergeQueueWatcher, - PRFetchState, -) - - -def _make_watcher() -> DefaultMergeQueueWatcher: - return DefaultMergeQueueWatcher(token=None) - - -def _queue_state( - *, - merged: bool = False, - state: str = "OPEN", - mergeable: str = "MERGEABLE", - merge_state_status: str = "CLEAN", - auto_merge_present: bool = False, - auto_merge_enabled_at: datetime | None = None, - pr_node_id: str = "PR_kwDO_test", - in_queue: bool = False, - queue_state: str | None = None, - checks_state: str | None = None, -) -> PRFetchState: - return { - "merged": merged, - "state": state, - "mergeable": mergeable, - "merge_state_status": merge_state_status, - "auto_merge_present": auto_merge_present, - "auto_merge_enabled_at": auto_merge_enabled_at, - "pr_node_id": pr_node_id, - "in_queue": in_queue, - "queue_state": queue_state, - "checks_state": checks_state, - } - - -class TestDefaultMergeQueueWatcher: - """Tests for DefaultMergeQueueWatcher polling state machine.""" - - @pytest.mark.anyio - async def test_returns_merged_on_first_pr_state_check(self): - watcher = _make_watcher() - watcher._fetch_pr_and_queue_state = AsyncMock( # type: ignore[method-assign] - return_value=_queue_state(merged=True) - ) - result = await watcher.wait( - pr_number=42, target_branch="main", repo="owner/repo", poll_interval=1 - ) - assert result["success"] is True - assert result["pr_state"] == "merged" - - @pytest.mark.anyio - async def test_returns_merged_when_pr_closed_and_merged(self): - """merged=True takes priority over state=CLOSED.""" - watcher = _make_watcher() - watcher._fetch_pr_and_queue_state = AsyncMock( # type: ignore[method-assign] - return_value=_queue_state(state="CLOSED", merged=True) - ) - result = await watcher.wait( - pr_number=42, target_branch="main", repo="owner/repo", poll_interval=1 - ) - assert result["success"] is True - assert result["pr_state"] == "merged" - - @pytest.mark.anyio - async def test_returns_ejected_when_mergeable_conflicting_and_not_in_queue(self): - """mergeable=CONFLICTING + not in queue → ejected via positive conflicting signal.""" - watcher = _make_watcher() - watcher._fetch_pr_and_queue_state = AsyncMock( # type: ignore[method-assign] - return_value=_queue_state( - merged=False, - in_queue=False, - mergeable="CONFLICTING", - merge_state_status="BLOCKED", - ) - ) - with patch("autoskillit.execution.merge_queue.asyncio.sleep", new_callable=AsyncMock): - result = await watcher.wait( - pr_number=42, target_branch="main", repo="owner/repo", poll_interval=1 - ) - assert result["success"] is False - assert result["pr_state"] == "ejected" - - @pytest.mark.anyio - async def test_keeps_polling_while_pr_in_queue(self): - watcher = _make_watcher() - call_count = 0 - - async def _fetch_side(*_a: object, **_kw: object) -> dict: - nonlocal call_count - call_count += 1 - if call_count >= 3: - return _queue_state(merged=True) - return _queue_state(in_queue=True, queue_state="AWAITING_CHECKS") - - watcher._fetch_pr_and_queue_state = _fetch_side # type: ignore[method-assign] - - with patch( - "autoskillit.execution.merge_queue.asyncio.sleep", new_callable=AsyncMock - ) as mock_sleep: - result = await watcher.wait( - pr_number=42, target_branch="main", repo="owner/repo", poll_interval=1 - ) - - assert result["success"] is True - assert result["pr_state"] == "merged" - assert mock_sleep.call_count >= 2 - - @pytest.mark.anyio - async def test_stuck_detection_triggers_toggle_once(self): - """Stalled PR (auto_merge set, grace expired) triggers toggle and then merges.""" - watcher = _make_watcher() - toggle_calls: list[int] = [] - call_count = 0 - enabled_at = datetime.now(UTC) - timedelta(seconds=120) - - async def _fetch_side(*_a: object, **_kw: object) -> dict: - nonlocal call_count - call_count += 1 - if call_count >= 4: - return _queue_state(merged=True) - return _queue_state( - auto_merge_enabled_at=enabled_at, - merge_state_status="CLEAN", - in_queue=False, - ) - - async def _toggle_side(*_a: object, **_kw: object) -> None: - toggle_calls.append(1) - - watcher._fetch_pr_and_queue_state = _fetch_side # type: ignore[method-assign] - watcher._toggle_auto_merge = _toggle_side # type: ignore[method-assign] - - with patch("autoskillit.execution.merge_queue.asyncio.sleep", new_callable=AsyncMock): - result = await watcher.wait( - pr_number=42, - target_branch="main", - repo="owner/repo", - poll_interval=1, - stall_grace_period=60, - ) - - assert result["success"] is True - assert result["pr_state"] == "merged" - assert len(toggle_calls) == 1 - - @pytest.mark.anyio - async def test_returns_timeout_when_deadline_exceeded(self): - watcher = _make_watcher() - watcher._fetch_pr_and_queue_state = AsyncMock( # type: ignore[method-assign] - return_value=_queue_state(in_queue=True, queue_state="AWAITING_CHECKS") - ) - - with patch("autoskillit.execution.merge_queue.asyncio.sleep", new_callable=AsyncMock): - with patch("autoskillit.execution.merge_queue.time.monotonic") as mock_time: - # deadline=0.0+1000=1000.0; loop enters (1.0<1000); after sleep exceeds (1001.0) - mock_time.side_effect = [0.0, 1.0, 1001.0] - result = await watcher.wait( - pr_number=42, - target_branch="main", - repo="owner/repo", - timeout_seconds=1000, - poll_interval=1, - ) - - assert result["success"] is False - assert result["pr_state"] == "timeout" - assert "Timed out" in result["reason"] - - @pytest.mark.anyio - async def test_returns_ejected_when_pr_closed_not_merged(self): - watcher = _make_watcher() - watcher._fetch_pr_and_queue_state = AsyncMock( # type: ignore[method-assign] - return_value=_queue_state(state="CLOSED", merged=False) - ) - result = await watcher.wait( - pr_number=42, target_branch="main", repo="owner/repo", poll_interval=1 - ) - assert result["success"] is False - assert result["pr_state"] == "ejected" - - @pytest.mark.anyio - async def test_http_error_in_pr_state_logs_and_retries(self): - """Fetch error is caught and retried; subsequent success returns merged.""" - watcher = _make_watcher() - call_count = 0 - - async def _fetch_side(*_a: object, **_kw: object) -> dict: - nonlocal call_count - call_count += 1 - if call_count == 1: - raise httpx.HTTPStatusError( - "500", - request=httpx.Request("GET", "http://x"), - response=httpx.Response(500), - ) - return _queue_state(merged=True) - - watcher._fetch_pr_and_queue_state = _fetch_side # type: ignore[method-assign] - - with patch("autoskillit.execution.merge_queue.asyncio.sleep", new_callable=AsyncMock): - result = await watcher.wait( - pr_number=42, target_branch="main", repo="owner/repo", poll_interval=1 - ) - - assert result["success"] is True - assert result["pr_state"] == "merged" - - @pytest.mark.anyio - async def test_graphql_error_falls_through_as_retry(self): - """GraphQL fetch error is caught and retried; subsequent success returns merged.""" - watcher = _make_watcher() - call_count = 0 - - async def _fetch_side(*_a: object, **_kw: object) -> dict: - nonlocal call_count - call_count += 1 - if call_count == 1: - raise RuntimeError("GraphQL error: connection error") - return _queue_state(merged=True) - - watcher._fetch_pr_and_queue_state = _fetch_side # type: ignore[method-assign] - - with patch("autoskillit.execution.merge_queue.asyncio.sleep", new_callable=AsyncMock): - result = await watcher.wait( - pr_number=42, target_branch="main", repo="owner/repo", poll_interval=1 - ) - - assert result["success"] is True - assert result["pr_state"] == "merged" - - @pytest.mark.anyio - async def test_graphql_errors_raise(self): - """GraphQL error responses from _fetch_pr_and_queue_state should raise.""" - watcher = _make_watcher() - - async def _mock_post(*args, **kwargs): - resp = httpx.Response( - 200, - json={"errors": [{"message": "some error"}]}, - request=httpx.Request("POST", "http://x"), - ) - return resp - - watcher._client.post = _mock_post # type: ignore[method-assign] - with pytest.raises(RuntimeError, match="GraphQL error"): - await watcher._fetch_pr_and_queue_state(42, "owner", "repo", "main") - - -class TestMergeQueueReliability: - """Tests for confirmation-window ejection guard and stall detection/retry logic.""" - - # Group 1: Confirmation-window ejection guard - - @pytest.mark.anyio - async def test_no_false_ejection_when_pr_merges_between_cycles(self): - """Cycle 1 sees 'not in queue'; cycle 2 sees merged=True → return merged.""" - watcher = _make_watcher() - responses = [ - _queue_state(merged=False, in_queue=False), - _queue_state(merged=True), - ] - watcher._fetch_pr_and_queue_state = AsyncMock(side_effect=responses) # type: ignore[method-assign] - with patch("autoskillit.execution.merge_queue.asyncio.sleep", new_callable=AsyncMock): - result = await watcher.wait( - pr_number=42, target_branch="main", repo="owner/repo", poll_interval=1 - ) - assert result["success"] is True - assert result["pr_state"] == "merged" - - @pytest.mark.anyio - async def test_confirmation_window_delays_ejection_by_one_cycle(self): - """Two consecutive not-in-queue cycles required before acting on CONFLICTING ejection.""" - watcher = _make_watcher() - state = _queue_state( - merged=False, in_queue=False, mergeable="CONFLICTING", merge_state_status="BLOCKED" - ) - watcher._fetch_pr_and_queue_state = AsyncMock(return_value=state) # type: ignore[method-assign] - with patch("autoskillit.execution.merge_queue.asyncio.sleep", new_callable=AsyncMock): - result = await watcher.wait( - pr_number=42, target_branch="main", repo="owner/repo", poll_interval=1 - ) - assert result["success"] is False - assert result["pr_state"] == "ejected" - # Confirmation window requires exactly 2 cycles before ejection - assert watcher._fetch_pr_and_queue_state.call_count == 2 # type: ignore[union-attr] - - @pytest.mark.anyio - async def test_single_graphql_call_per_poll_cycle(self): - """Each poll cycle makes exactly one call to _fetch_pr_and_queue_state.""" - watcher = _make_watcher() - call_count = 0 - - async def _fetch_side(*_a: object, **_kw: object) -> dict: - nonlocal call_count - call_count += 1 - if call_count >= 3: - return _queue_state(merged=True) - return _queue_state(in_queue=True, queue_state="AWAITING_CHECKS") - - watcher._fetch_pr_and_queue_state = _fetch_side # type: ignore[method-assign] - with patch("autoskillit.execution.merge_queue.asyncio.sleep", new_callable=AsyncMock): - result = await watcher.wait( - pr_number=42, target_branch="main", repo="owner/repo", poll_interval=1 - ) - assert result["success"] is True - assert call_count == 3 # exactly 3 fetch calls (not 6 as with the old 2-call approach) - - # Group 2: Stall detection and multi-retry - - @pytest.mark.anyio - async def test_stall_grace_period_prevents_premature_toggle(self): - """PR stalled 30s ago with grace=60s → toggle must NOT be called within grace.""" - watcher = _make_watcher() - toggle_calls: list[object] = [] - call_count = 0 - enabled_at = datetime.now(UTC) - timedelta(seconds=30) - - async def _fetch_side(*_a: object, **_kw: object) -> dict: - nonlocal call_count - call_count += 1 - if call_count >= 4: - return _queue_state(merged=True) - return _queue_state( - auto_merge_enabled_at=enabled_at, - merge_state_status="CLEAN", - in_queue=False, - ) - - async def _toggle_side(*_a: object, **_kw: object) -> None: - toggle_calls.append(1) - - watcher._fetch_pr_and_queue_state = _fetch_side # type: ignore[method-assign] - watcher._toggle_auto_merge = _toggle_side # type: ignore[method-assign] - with patch("autoskillit.execution.merge_queue.asyncio.sleep", new_callable=AsyncMock): - result = await watcher.wait( - pr_number=42, - target_branch="main", - repo="owner/repo", - poll_interval=1, - stall_grace_period=60, - ) - assert len(toggle_calls) == 0, "Toggle must NOT be called within grace period" - assert result["success"] is True - - @pytest.mark.anyio - async def test_stall_toggle_triggered_after_grace_period_expires(self): - """PR stalled 90s ago with grace=60s → toggle IS called after grace expires.""" - watcher = _make_watcher() - toggle_calls: list[object] = [] - call_count = 0 - enabled_at = datetime.now(UTC) - timedelta(seconds=90) - - async def _fetch_side(*_a: object, **_kw: object) -> dict: - nonlocal call_count - call_count += 1 - if call_count >= 4: - return _queue_state(merged=True) - return _queue_state( - auto_merge_enabled_at=enabled_at, - merge_state_status="CLEAN", - in_queue=False, - ) - - async def _toggle_side(*_a: object, **_kw: object) -> None: - toggle_calls.append(1) - - watcher._fetch_pr_and_queue_state = _fetch_side # type: ignore[method-assign] - watcher._toggle_auto_merge = _toggle_side # type: ignore[method-assign] - with patch("autoskillit.execution.merge_queue.asyncio.sleep", new_callable=AsyncMock): - await watcher.wait( - pr_number=42, - target_branch="main", - repo="owner/repo", - poll_interval=1, - stall_grace_period=60, - ) - assert len(toggle_calls) >= 1, "Toggle must be called after grace period expires" - - @pytest.mark.anyio - async def test_max_stall_retries_exhausted_returns_stalled_not_ejected(self): - """After max_stall_retries=2 toggle attempts, returns pr_state='stalled' not 'ejected'.""" - watcher = _make_watcher() - enabled_at = datetime.now(UTC) - timedelta(seconds=120) - watcher._fetch_pr_and_queue_state = AsyncMock( # type: ignore[method-assign] - return_value=_queue_state( - merged=False, - in_queue=False, - auto_merge_enabled_at=enabled_at, - merge_state_status="CLEAN", - ) - ) - watcher._toggle_auto_merge = AsyncMock() # type: ignore[method-assign] - with patch("autoskillit.execution.merge_queue.asyncio.sleep", new_callable=AsyncMock): - result = await watcher.wait( - pr_number=42, - target_branch="main", - repo="owner/repo", - poll_interval=1, - stall_grace_period=60, - max_stall_retries=2, - ) - assert result["success"] is False - assert result["pr_state"] == "stalled" - assert result["stall_retries_attempted"] == 2 - - @pytest.mark.anyio - async def test_exponential_backoff_between_stall_retries(self): - """Backoff durations must be [30, 60, 120] seconds for retries 0, 1, 2.""" - watcher = _make_watcher() - enabled_at = datetime.now(UTC) - timedelta(seconds=120) - watcher._fetch_pr_and_queue_state = AsyncMock( # type: ignore[method-assign] - return_value=_queue_state( - merged=False, - in_queue=False, - auto_merge_enabled_at=enabled_at, - merge_state_status="CLEAN", - ) - ) - watcher._toggle_auto_merge = AsyncMock() # type: ignore[method-assign] - sleep_calls: list[float] = [] - - async def _capture_sleep(seconds: float) -> None: - sleep_calls.append(seconds) - - with patch("autoskillit.execution.merge_queue.asyncio.sleep", side_effect=_capture_sleep): - result = await watcher.wait( - pr_number=42, - target_branch="main", - repo="owner/repo", - poll_interval=1, - stall_grace_period=60, - max_stall_retries=3, - ) - assert result["pr_state"] == "stalled" - backoff_sleeps = [s for s in sleep_calls if s > 1] - assert backoff_sleeps == [30, 60, 120] - - @pytest.mark.anyio - async def test_graphql_mutation_used_for_toggle(self): - """Toggle must use disablePullRequestAutoMerge and enablePullRequestAutoMerge mutations.""" - watcher = _make_watcher() - enabled_at = datetime.now(UTC) - timedelta(seconds=120) - call_count = 0 - - async def _fetch_side(*_a: object, **_kw: object) -> dict: - nonlocal call_count - call_count += 1 - if call_count >= 4: - return _queue_state(merged=True) - return _queue_state( - merged=False, - in_queue=False, - auto_merge_enabled_at=enabled_at, - merge_state_status="CLEAN", - ) - - watcher._fetch_pr_and_queue_state = _fetch_side # type: ignore[method-assign] - captured_bodies: list[dict] = [] - - async def _mock_post(url: str, **kwargs: object) -> httpx.Response: - body = dict(kwargs.get("json", {}) or {}) # type: ignore[arg-type] - captured_bodies.append(body) - query = body.get("query", "") - if "disablePullRequestAutoMerge" in query: - data: dict = { - "data": {"disablePullRequestAutoMerge": {"pullRequest": {"number": 42}}} - } - elif "enablePullRequestAutoMerge" in query: - data = {"data": {"enablePullRequestAutoMerge": {"pullRequest": {"number": 42}}}} - else: - data = {"data": {}} - return httpx.Response(200, json=data, request=httpx.Request("POST", url)) - - watcher._client.post = _mock_post # type: ignore[method-assign] - with patch("autoskillit.execution.merge_queue.asyncio.sleep", new_callable=AsyncMock): - await watcher.wait( - pr_number=42, - target_branch="main", - repo="owner/repo", - poll_interval=1, - stall_grace_period=60, - max_stall_retries=1, - ) - queries = [b.get("query", "") for b in captured_bodies] - assert any("disablePullRequestAutoMerge" in q for q in queries), ( - "Expected disable mutation" - ) - assert any("enablePullRequestAutoMerge" in q for q in queries), "Expected enable mutation" - - @pytest.mark.anyio - async def test_stall_recovery_success_after_toggle(self): - """After 1 toggle PR re-enters queue then merges → success, stall_retries_attempted=1.""" - watcher = _make_watcher() - toggle_calls: list[object] = [] - call_count = 0 - enabled_at = datetime.now(UTC) - timedelta(seconds=120) - - async def _fetch_side(*_a: object, **_kw: object) -> dict: - nonlocal call_count - call_count += 1 - if call_count <= 2: - return _queue_state( - merged=False, - in_queue=False, - auto_merge_enabled_at=enabled_at, - merge_state_status="CLEAN", - ) - elif call_count == 3: - return _queue_state(in_queue=True, queue_state="AWAITING_CHECKS") - else: - return _queue_state(merged=True) - - async def _toggle_side(*_a: object, **_kw: object) -> None: - toggle_calls.append(1) - - watcher._fetch_pr_and_queue_state = _fetch_side # type: ignore[method-assign] - watcher._toggle_auto_merge = _toggle_side # type: ignore[method-assign] - with patch("autoskillit.execution.merge_queue.asyncio.sleep", new_callable=AsyncMock): - result = await watcher.wait( - pr_number=42, - target_branch="main", - repo="owner/repo", - poll_interval=1, - stall_grace_period=60, - max_stall_retries=3, - ) - assert result["success"] is True - assert result["pr_state"] == "merged" - assert result["stall_retries_attempted"] == 1 - - # Group 3: Tool Interface - - @pytest.mark.anyio - async def test_stall_retries_attempted_present_in_all_terminal_responses(self): - """stall_retries_attempted key must be present in all four terminal responses.""" - watcher = _make_watcher() - - # merged - watcher._fetch_pr_and_queue_state = AsyncMock( # type: ignore[method-assign] - return_value=_queue_state(merged=True) - ) - result = await watcher.wait(pr_number=1, target_branch="main", repo="owner/repo") - assert "stall_retries_attempted" in result, ( - "merged response missing stall_retries_attempted" - ) - assert result["stall_retries_attempted"] == 0, "merged: no retries expected" - - # ejected (closed) - watcher._fetch_pr_and_queue_state = AsyncMock( # type: ignore[method-assign] - return_value=_queue_state(state="CLOSED", merged=False) - ) - result = await watcher.wait(pr_number=1, target_branch="main", repo="owner/repo") - assert "stall_retries_attempted" in result, ( - "ejected response missing stall_retries_attempted" - ) - assert result["stall_retries_attempted"] == 0, "ejected: no retries expected" - - # timeout - with patch("autoskillit.execution.merge_queue.asyncio.sleep", new_callable=AsyncMock): - with patch("autoskillit.execution.merge_queue.time.monotonic") as mock_time: - mock_time.side_effect = [0.0, 1.0, 1001.0] - watcher._fetch_pr_and_queue_state = AsyncMock( # type: ignore[method-assign] - return_value=_queue_state(in_queue=True, queue_state="AWAITING_CHECKS") - ) - result = await watcher.wait( - pr_number=1, - target_branch="main", - repo="owner/repo", - timeout_seconds=1000, - ) - assert "stall_retries_attempted" in result, ( - "timeout response missing stall_retries_attempted" - ) - assert result["stall_retries_attempted"] == 0, "timeout: no retries expected" - assert result["pr_state"] == "timeout" - - # stalled - enabled_at = datetime.now(UTC) - timedelta(seconds=120) - watcher._fetch_pr_and_queue_state = AsyncMock( # type: ignore[method-assign] - return_value=_queue_state( - merged=False, - in_queue=False, - auto_merge_enabled_at=enabled_at, - merge_state_status="CLEAN", - ) - ) - watcher._toggle_auto_merge = AsyncMock() # type: ignore[method-assign] - with patch("autoskillit.execution.merge_queue.asyncio.sleep", new_callable=AsyncMock): - result = await watcher.wait( - pr_number=1, - target_branch="main", - repo="owner/repo", - stall_grace_period=60, - max_stall_retries=1, - ) - assert "stall_retries_attempted" in result, ( - "stalled response missing stall_retries_attempted" - ) - assert result["stall_retries_attempted"] == 1, "stalled: expected max_stall_retries=1" - assert result["pr_state"] == "stalled" - - @pytest.mark.anyio - async def test_wait_accepts_stall_grace_period_and_max_stall_retries_params(self): - """wait() accepts stall_grace_period and max_stall_retries without TypeError.""" - watcher = _make_watcher() - watcher._fetch_pr_and_queue_state = AsyncMock( # type: ignore[method-assign] - return_value=_queue_state(merged=True) - ) - result = await watcher.wait( - pr_number=42, - target_branch="main", - repo="owner/repo", - stall_grace_period=120, - max_stall_retries=5, - ) - assert result["success"] is True - assert result["pr_state"] == "merged" - - -class TestPendingCIGuard: - """Tests for the pending-CI guard that prevents false ejection when checks are running.""" - - @pytest.mark.anyio - async def test_continues_polling_when_checks_pending_and_not_in_queue(self): - """Core bug: auto-merge + BLOCKED + PENDING checks must keep polling, not eject.""" - watcher = _make_watcher() - call_count = 0 - - async def _fetch(*_a, **_kw): - nonlocal call_count - call_count += 1 - if call_count < 4: - return _queue_state( - merged=False, - in_queue=False, - auto_merge_enabled_at=datetime.now(UTC), - merge_state_status="BLOCKED", - checks_state="PENDING", - ) - return _queue_state(merged=True) - - watcher._fetch_pr_and_queue_state = _fetch # type: ignore[method-assign] - with patch("autoskillit.execution.merge_queue.asyncio.sleep", new_callable=AsyncMock): - result = await watcher.wait(pr_number=1, target_branch="main", repo="owner/repo") - - assert result["success"] is True - assert result["pr_state"] == "merged" - assert call_count >= 4 - - @pytest.mark.anyio - async def test_continues_polling_when_checks_expected_and_not_in_queue(self): - """EXPECTED means 'check not yet started' — same as PENDING: keep polling.""" - watcher = _make_watcher() - call_count = 0 - - async def _fetch(*_a, **_kw): - nonlocal call_count - call_count += 1 - if call_count < 3: - return _queue_state( - in_queue=False, - auto_merge_enabled_at=datetime.now(UTC), - merge_state_status="BLOCKED", - checks_state="EXPECTED", - ) - return _queue_state(merged=True) - - watcher._fetch_pr_and_queue_state = _fetch # type: ignore[method-assign] - with patch("autoskillit.execution.merge_queue.asyncio.sleep", new_callable=AsyncMock): - result = await watcher.wait(pr_number=1, target_branch="main", repo="owner/repo") - - assert result["success"] is True - assert result["pr_state"] == "merged" - assert call_count >= 3 - - @pytest.mark.anyio - async def test_returns_dropped_healthy_when_checks_success_and_mergeable(self): - """Healthy PR (SUCCESS + MERGEABLE + CLEAN + auto_merge cleared) → dropped_healthy. - - This is the exact issue #802 failure mode: the old elimination classifier returned - 'ejected' for this state because no other gate matched. The new classifier returns - DROPPED_HEALTHY via positive signal — auto_merge was cleared while the PR was healthy. - """ - watcher = _make_watcher() - watcher._fetch_pr_and_queue_state = AsyncMock( # type: ignore[method-assign] - return_value=_queue_state( - in_queue=False, - mergeable="MERGEABLE", - merge_state_status="CLEAN", - checks_state="SUCCESS", - auto_merge_present=False, - ) - ) - with patch("autoskillit.execution.merge_queue.asyncio.sleep", new_callable=AsyncMock): - result = await watcher.wait(pr_number=1, target_branch="main", repo="owner/repo") - - assert result["success"] is False - assert result["pr_state"] == "dropped_healthy" - - @pytest.mark.anyio - async def test_returns_ejected_when_checks_success_but_mergeable_conflicting(self): - """checks_state=SUCCESS + mergeable=CONFLICTING → ejected via positive CONFLICTING gate.""" - watcher = _make_watcher() - watcher._fetch_pr_and_queue_state = AsyncMock( # type: ignore[method-assign] - return_value=_queue_state( - in_queue=False, - mergeable="CONFLICTING", - merge_state_status="BLOCKED", - checks_state="SUCCESS", - ) - ) - with patch("autoskillit.execution.merge_queue.asyncio.sleep", new_callable=AsyncMock): - result = await watcher.wait(pr_number=1, target_branch="main", repo="owner/repo") - - assert result["success"] is False - assert result["pr_state"] == "ejected" - - @pytest.mark.anyio - async def test_returns_ejected_when_no_checks_configured_but_conflicting(self): - """checks_state=None + mergeable=CONFLICTING → ejected via positive CONFLICTING signal. - - For repos without required CI checks, ejection requires a positive mergeable=CONFLICTING - signal rather than the absence of checks. ClassifierInconclusive handles the ambiguous - case (no CI, no conflict signal) by continuing to poll. - """ - watcher = _make_watcher() - watcher._fetch_pr_and_queue_state = AsyncMock( # type: ignore[method-assign] - return_value=_queue_state( - in_queue=False, - mergeable="CONFLICTING", - merge_state_status="BLOCKED", - checks_state=None, - ) - ) - with patch("autoskillit.execution.merge_queue.asyncio.sleep", new_callable=AsyncMock): - result = await watcher.wait(pr_number=1, target_branch="main", repo="owner/repo") - - assert result["success"] is False - assert result["pr_state"] == "ejected" - - @pytest.mark.anyio - async def test_fetch_extracts_checks_state_from_graphql_response(self): - """_fetch_pr_and_queue_state must extract statusCheckRollup.state into checks_state.""" - watcher = _make_watcher() - - async def _mock_post(url, *, json, headers=None, **_kw): - return httpx.Response( - 200, - json={ - "data": { - "repository": { - "pullRequest": { - "id": "PR_kwDO_test", - "merged": False, - "state": "OPEN", - "mergeStateStatus": "BLOCKED", - "autoMergeRequest": {"enabledAt": "2026-03-26T12:00:00Z"}, - "statusCheckRollup": {"state": "PENDING"}, - }, - "mergeQueue": {"entries": {"nodes": []}}, - } - } - }, - request=httpx.Request("POST", url), - ) - - watcher._client.post = _mock_post # type: ignore[method-assign] - state = await watcher._fetch_pr_and_queue_state(1, "owner", "repo", "main") - - assert state["checks_state"] == "PENDING" - assert state["in_queue"] is False - assert state["merged"] is False - - def test_implements_merge_queue_watcher_protocol(self): - from autoskillit.core import MergeQueueWatcher - - assert isinstance(_make_watcher(), MergeQueueWatcher) - - -class TestInconclusiveBudget: - """Tests for the split inconclusive budget: CIStillRunning vs NoPositiveSignal.""" - - @pytest.mark.anyio - async def test_pending_ci_does_not_exhaust_inconclusive_budget(self): - """CIStillRunning must not consume inconclusive_count. - Six PENDING cycles with budget=3 must NOT trigger budget ceiling. - timeout_seconds=99999 ensures outer deadline is not the cause. - """ - watcher = DefaultMergeQueueWatcher(token=None) - call_count = 0 - - async def _fetch(*_a, **_kw): - nonlocal call_count - call_count += 1 - if call_count > 6: - return _queue_state(merged=True) - return _queue_state( - merged=False, - in_queue=False, - checks_state="PENDING", - merge_state_status="BLOCKED", - auto_merge_enabled_at=None, - ) - - watcher._fetch_pr_and_queue_state = _fetch # type: ignore[method-assign] - with patch("autoskillit.execution.merge_queue.asyncio.sleep", new_callable=AsyncMock): - result = await watcher.wait( - pr_number=42, - target_branch="main", - repo="owner/repo", - poll_interval=1, - timeout_seconds=99999, - ) - - assert result["success"] is True - assert result["pr_state"] == "merged" - - @pytest.mark.anyio - async def test_expected_ci_does_not_exhaust_inconclusive_budget(self): - """CIStillRunning (EXPECTED) must not consume inconclusive_count. - Six EXPECTED cycles with budget=3 must NOT trigger budget ceiling. - """ - watcher = DefaultMergeQueueWatcher(token=None) - call_count = 0 - - async def _fetch(*_a, **_kw): - nonlocal call_count - call_count += 1 - if call_count > 6: - return _queue_state(merged=True) - return _queue_state( - merged=False, - in_queue=False, - checks_state="EXPECTED", - merge_state_status="BLOCKED", - auto_merge_enabled_at=None, - ) - - watcher._fetch_pr_and_queue_state = _fetch # type: ignore[method-assign] - with patch("autoskillit.execution.merge_queue.asyncio.sleep", new_callable=AsyncMock): - result = await watcher.wait( - pr_number=42, - target_branch="main", - repo="owner/repo", - poll_interval=1, - timeout_seconds=99999, - ) - - assert result["success"] is True - assert result["pr_state"] == "merged" - - @pytest.mark.anyio - async def test_no_positive_signal_exhausts_budget_returns_timeout(self): - """NoPositiveSignal must still exhaust the bounded budget. - Unknown state with no positive classifier match × (window + N) cycles - → pr_state='timeout'. Reason must contain 'Inconclusive after'. - """ - watcher = DefaultMergeQueueWatcher(token=None) - watcher._fetch_pr_and_queue_state = AsyncMock( # type: ignore[method-assign] - return_value=_queue_state( - merged=False, - in_queue=False, - checks_state="SUCCESS", - mergeable="UNKNOWN", - merge_state_status="UNKNOWN", - auto_merge_enabled_at=None, - ) - ) - with patch("autoskillit.execution.merge_queue.asyncio.sleep", new_callable=AsyncMock): - result = await watcher.wait( - pr_number=42, - target_branch="main", - repo="owner/repo", - poll_interval=1, - timeout_seconds=99999, - ) - - assert result["success"] is False - assert result["pr_state"] == "timeout" - assert "Inconclusive after" in result["reason"] - - @pytest.mark.anyio - async def test_inconclusive_count_resets_on_queue_reentry(self): - """inconclusive_count must reset when in_queue becomes True. - Scenario: out(PENDING×5) → in_queue → out(PENDING×5) → merged. - Budget=3. Without reset: second phase exhausts budget immediately. - With reset: both phases are tolerated independently. - """ - watcher = DefaultMergeQueueWatcher(token=None) - call_count = 0 - - async def _fetch(*_a, **_kw): - nonlocal call_count - call_count += 1 - if call_count <= 5: - return _queue_state( - merged=False, - in_queue=False, - checks_state="PENDING", - merge_state_status="BLOCKED", - auto_merge_enabled_at=None, - ) - if call_count == 6: - return _queue_state(merged=False, in_queue=True) - if call_count <= 11: - return _queue_state( - merged=False, - in_queue=False, - checks_state="PENDING", - merge_state_status="BLOCKED", - auto_merge_enabled_at=None, - ) - return _queue_state(merged=True) - - watcher._fetch_pr_and_queue_state = _fetch # type: ignore[method-assign] - with patch("autoskillit.execution.merge_queue.asyncio.sleep", new_callable=AsyncMock): - result = await watcher.wait( - pr_number=42, - target_branch="main", - repo="owner/repo", - poll_interval=1, - timeout_seconds=99999, - ) - - assert result["success"] is True - assert result["pr_state"] == "merged" - - @pytest.mark.anyio - async def test_confirmation_window_does_not_consume_inconclusive_budget(self): - """CIStillRunning (PENDING) must never consume the inconclusive budget at any cycle. - confirmation_cycles=4, budget=3: all 7 PENDING cycles complete without budget exhaustion - because PENDING raises CIStillRunning (exempt from budget). Merged on call 8. - """ - watcher = DefaultMergeQueueWatcher(token=None) - call_count = 0 - - async def _fetch(*_a, **_kw): - nonlocal call_count - call_count += 1 - if call_count > 7: - return _queue_state(merged=True) - return _queue_state( - merged=False, - in_queue=False, - checks_state="PENDING", - merge_state_status="BLOCKED", - auto_merge_enabled_at=None, - ) - - watcher._fetch_pr_and_queue_state = _fetch # type: ignore[method-assign] - with patch("autoskillit.execution.merge_queue.asyncio.sleep", new_callable=AsyncMock): - result = await watcher.wait( - pr_number=42, - target_branch="main", - repo="owner/repo", - poll_interval=1, - timeout_seconds=99999, - not_in_queue_confirmation_cycles=4, - ) - - assert result["success"] is True - assert result["pr_state"] == "merged" - assert call_count == 8 - - @pytest.mark.anyio - async def test_wait_accepts_max_inconclusive_retries_per_call_param(self): - """max_inconclusive_retries must be a wait() parameter, not constructor-only. - Passing it must not raise TypeError. Passing a small value (1) must cause - budget exhaustion after 1 NoPositiveSignal cycle beyond the confirmation window. - """ - watcher = _make_watcher() - # State: unknown — no positive classifier fires → NoPositiveSignal - watcher._fetch_pr_and_queue_state = AsyncMock( # type: ignore[method-assign] - return_value=_queue_state( - merged=False, - in_queue=False, - checks_state="SUCCESS", - mergeable="UNKNOWN", - merge_state_status="UNKNOWN", - auto_merge_enabled_at=None, - ) - ) - with patch("autoskillit.execution.merge_queue.asyncio.sleep", new_callable=AsyncMock): - result = await watcher.wait( - pr_number=42, - target_branch="main", - repo="owner/repo", - poll_interval=1, - timeout_seconds=99999, - max_inconclusive_retries=1, # ← per-call override - ) - - assert result["success"] is False - assert result["pr_state"] == "timeout" - assert "Inconclusive after 1 retries" in result["reason"] - - @pytest.mark.anyio - async def test_per_call_max_inconclusive_retries_overrides_constructor_default(self): - """Per-call value must take precedence over the constructor default of 5. - budget=2 per call, constructor default would be 5. Budget exhaustion must - occur after 2 ambiguous cycles (+ confirmation window), not 5. - """ - watcher = _make_watcher() # constructor default = 5 - call_count = 0 - - async def _fetch(*_a, **_kw): - nonlocal call_count - call_count += 1 - return _queue_state( - merged=False, - in_queue=False, - checks_state="SUCCESS", - mergeable="UNKNOWN", - merge_state_status="UNKNOWN", - auto_merge_enabled_at=None, - ) - - watcher._fetch_pr_and_queue_state = _fetch # type: ignore[method-assign] - with patch("autoskillit.execution.merge_queue.asyncio.sleep", new_callable=AsyncMock): - result = await watcher.wait( - pr_number=42, - target_branch="main", - repo="owner/repo", - poll_interval=1, - timeout_seconds=99999, - max_inconclusive_retries=2, # ← per-call override - ) - - assert result["success"] is False - assert result["pr_state"] == "timeout" - # confirmation_cycles=2 (default): cycle 1 is window, cycles 2-3 consume budget - # budget=2: exhausted on call 3 → total calls = 3 - assert call_count == 3 - - -class TestRelatedCoverage: - """Coverage for related untested paths found during investigation.""" - - @pytest.mark.anyio - async def test_returns_ejected_when_unmergeable_in_queue(self): - """in_queue=True, queue_state=UNMERGEABLE → ejected immediately.""" - watcher = _make_watcher() - watcher._fetch_pr_and_queue_state = AsyncMock( # type: ignore[method-assign] - return_value=_queue_state(in_queue=True, queue_state="UNMERGEABLE") - ) - with patch("autoskillit.execution.merge_queue.asyncio.sleep", new_callable=AsyncMock): - result = await watcher.wait(pr_number=1, target_branch="main", repo="owner/repo") - - assert result["success"] is False - assert result["pr_state"] == "ejected" - - @pytest.mark.anyio - async def test_is_stall_candidate_when_has_hooks(self): - """merge_state_status=HAS_HOOKS + auto-merge enabled → stall detection fires.""" - watcher = _make_watcher() - enabled_at = datetime.now(UTC) - timedelta(seconds=120) - watcher._fetch_pr_and_queue_state = AsyncMock( # type: ignore[method-assign] - return_value=_queue_state( - merged=False, - in_queue=False, - auto_merge_enabled_at=enabled_at, - merge_state_status="HAS_HOOKS", - ) - ) - watcher._toggle_auto_merge = AsyncMock() # type: ignore[method-assign] - with patch("autoskillit.execution.merge_queue.asyncio.sleep", new_callable=AsyncMock): - result = await watcher.wait( - pr_number=1, - target_branch="main", - repo="owner/repo", - stall_grace_period=60, - max_stall_retries=1, - ) - - assert result["pr_state"] == "stalled" - assert watcher._toggle_auto_merge.call_count == 1 # type: ignore[union-attr] - - @pytest.mark.anyio - async def test_returns_error_when_repo_has_no_slash(self): - """repo='noslash' → pr_state='error', no polling.""" - watcher = _make_watcher() - result = await watcher.wait(pr_number=1, target_branch="main", repo="noslash") - - assert result["success"] is False - assert result["pr_state"] == "error" - assert "Invalid repo format" in result["reason"] - - -class TestEjectionEnrichment: - """Tests ejection response enrichment with CI failure cause.""" - - @pytest.mark.anyio - async def test_ejected_ci_failure_when_checks_state_is_failure(self): - """When checks_state=FAILURE, wait() returns pr_state='ejected_ci_failure'.""" - watcher = _make_watcher() - watcher._fetch_pr_and_queue_state = AsyncMock( # type: ignore[method-assign] - return_value=_queue_state( - merged=False, - in_queue=False, - checks_state="FAILURE", - merge_state_status="BLOCKED", - auto_merge_enabled_at=None, - ) - ) - with patch("autoskillit.execution.merge_queue.asyncio.sleep", new_callable=AsyncMock): - result = await watcher.wait( - pr_number=42, - target_branch="main", - repo="owner/repo", - poll_interval=1, - not_in_queue_confirmation_cycles=2, - ) - assert result["success"] is False - assert result["pr_state"] == "ejected_ci_failure" - assert result.get("ejection_cause") == "ci_failure" - - @pytest.mark.anyio - async def test_ejected_when_checks_state_is_none(self): - """checks_state=None + mergeable=CONFLICTING → pr_state='ejected', no ejection_cause.""" - watcher = _make_watcher() - watcher._fetch_pr_and_queue_state = AsyncMock( # type: ignore[method-assign] - return_value=_queue_state( - merged=False, - in_queue=False, - checks_state=None, - merge_state_status="BLOCKED", - mergeable="CONFLICTING", - auto_merge_enabled_at=None, - ) - ) - with patch("autoskillit.execution.merge_queue.asyncio.sleep", new_callable=AsyncMock): - result = await watcher.wait( - pr_number=42, - target_branch="main", - repo="owner/repo", - poll_interval=1, - not_in_queue_confirmation_cycles=2, - ) - assert result["success"] is False - assert result["pr_state"] == "ejected" - assert "ejection_cause" not in result - - @pytest.mark.anyio - async def test_ejected_when_checks_state_is_success(self): - """checks_state=SUCCESS + mergeable=CONFLICTING → pr_state='ejected', no enrichment.""" - watcher = _make_watcher() - watcher._fetch_pr_and_queue_state = AsyncMock( # type: ignore[method-assign] - return_value=_queue_state( - merged=False, - in_queue=False, - checks_state="SUCCESS", - merge_state_status="BLOCKED", - mergeable="CONFLICTING", - auto_merge_enabled_at=None, - ) - ) - with patch("autoskillit.execution.merge_queue.asyncio.sleep", new_callable=AsyncMock): - result = await watcher.wait( - pr_number=42, - target_branch="main", - repo="owner/repo", - poll_interval=1, - not_in_queue_confirmation_cycles=2, - ) - assert result["success"] is False - assert result["pr_state"] == "ejected" - assert "ejection_cause" not in result - - @pytest.mark.anyio - async def test_ejected_ci_failure_on_closed_pr_with_failure_checks(self): - """CLOSED state with checks_state=FAILURE → ejected_ci_failure.""" - watcher = _make_watcher() - watcher._fetch_pr_and_queue_state = AsyncMock( # type: ignore[method-assign] - return_value=_queue_state( - state="CLOSED", - merged=False, - checks_state="FAILURE", - ) - ) - result = await watcher.wait( - pr_number=42, target_branch="main", repo="owner/repo", poll_interval=1 - ) - assert result["success"] is False - assert result["pr_state"] == "ejected_ci_failure" - assert result.get("ejection_cause") == "ci_failure" - - @pytest.mark.anyio - async def test_ejected_ci_failure_after_in_queue_to_not_in_queue_transition(self): - """in_queue=True on first poll, then in_queue=False+FAILURE → ejected_ci_failure.""" - watcher = _make_watcher() - watcher._fetch_pr_and_queue_state = AsyncMock( # type: ignore[method-assign] - side_effect=[ - _queue_state(in_queue=True, checks_state=None), - _queue_state( - in_queue=False, - checks_state="FAILURE", - merge_state_status="BLOCKED", - auto_merge_enabled_at=None, - ), - _queue_state( - in_queue=False, - checks_state="FAILURE", - merge_state_status="BLOCKED", - auto_merge_enabled_at=None, - ), - ] - ) - with patch("autoskillit.execution.merge_queue.asyncio.sleep", new_callable=AsyncMock): - result = await watcher.wait( - pr_number=42, - target_branch="main", - repo="owner/repo", - poll_interval=1, - not_in_queue_confirmation_cycles=2, - ) - assert result["success"] is False - assert result["pr_state"] == "ejected_ci_failure" - assert result.get("ejection_cause") == "ci_failure" - - -# --------------------------------------------------------------------------- -# Part A: New classifier immunity tests (T1–T6) -# --------------------------------------------------------------------------- - - -class TestClassifierImmunity: - """Positive-signal immunity tests for the extracted _classify_pr_state function.""" - - # --- T1: Reproduces the reported bug --- - - def test_classifier_returns_dropped_healthy_when_auto_merge_cleared_on_healthy_pr(self): - """Exact issue #802 state: healthy PR with auto_merge cleared → DROPPED_HEALTHY.""" - state = _queue_state( - merged=False, - state="OPEN", - mergeable="MERGEABLE", - merge_state_status="CLEAN", - checks_state="SUCCESS", - in_queue=False, - auto_merge_present=False, - ) - result = _mq._classify_pr_state(state) - assert result.terminal == PRState.DROPPED_HEALTHY - - # --- T2: Positive-signal contract: EJECTED requires positive signal --- - - @pytest.mark.parametrize( - "state", - [ - # open + MERGEABLE + CLEAN + PENDING checks - _queue_state( - mergeable="MERGEABLE", - merge_state_status="CLEAN", - checks_state="PENDING", - auto_merge_present=True, - ), - # open + UNKNOWN mergeable + CLEAN + SUCCESS - _queue_state( - mergeable="UNKNOWN", - merge_state_status="CLEAN", - checks_state="SUCCESS", - ), - # open + MERGEABLE + HAS_HOOKS + SUCCESS + auto_merge_present=True - _queue_state( - mergeable="MERGEABLE", - merge_state_status="HAS_HOOKS", - checks_state="SUCCESS", - auto_merge_present=True, - ), - ], - ) - def test_classifier_never_returns_ejected_without_positive_signal(self, state): - """For ambiguous-but-healthy states, classifier must NOT return EJECTED.""" - try: - result = _mq._classify_pr_state(state) - assert result.terminal != PRState.EJECTED, ( - f"Classifier returned EJECTED without a positive signal for state: {state}" - ) - except ClassifierInconclusive: - pass # expected — no positive signal matched - - # --- T3: Classifier raises ClassifierInconclusive — no silent fall-through --- - - def test_classifier_raises_inconclusive_when_no_positive_signal_matches(self): - """ClassifierInconclusive raised when no positive gate matches; .state exposes fields.""" - state = _queue_state( - merged=False, - state="OPEN", - mergeable="UNKNOWN", - merge_state_status="BEHIND", - checks_state=None, - auto_merge_present=True, - in_queue=False, - ) - with pytest.raises(ClassifierInconclusive) as exc_info: - _mq._classify_pr_state(state) - assert exc_info.value.state is state - assert exc_info.value.reason - - # --- T4: Exhaustive PRState coverage --- - - @pytest.mark.parametrize( - "expected_terminal,state", - [ - ( - PRState.MERGED, - _queue_state(merged=True), - ), - ( - PRState.EJECTED, - _queue_state( - merged=False, - state="OPEN", - mergeable="CONFLICTING", - in_queue=False, - ), - ), - ( - PRState.EJECTED_CI_FAILURE, - _queue_state( - merged=False, - state="OPEN", - checks_state="FAILURE", - in_queue=False, - ), - ), - ( - PRState.STALLED, - _queue_state( - merged=False, - state="OPEN", - mergeable="MERGEABLE", - merge_state_status="CLEAN", - auto_merge_enabled_at=datetime.now(UTC) - timedelta(seconds=120), - auto_merge_present=True, - in_queue=False, - ), - ), - ( - PRState.DROPPED_HEALTHY, - _queue_state( - merged=False, - state="OPEN", - mergeable="MERGEABLE", - merge_state_status="CLEAN", - checks_state="SUCCESS", - auto_merge_present=False, - in_queue=False, - ), - ), - ], - ) - def test_classifier_returns_expected_terminal_for_canonical_fixture( - self, expected_terminal, state - ): - """Every classifier-reachable PRState terminal has at least one positive fixture.""" - result = _mq._classify_pr_state(state) - assert result.terminal == expected_terminal - - # --- T5: Import-time schema round-trip guard --- - - def test_query_field_map_matches_prfetchstate_required_keys(self): - """_QUERY_FIELD_MAP keys must exactly match PRFetchState required+optional keys.""" - all_keys = PRFetchState.__required_keys__ | PRFetchState.__optional_keys__ - assert set(_mq._QUERY_FIELD_MAP) == all_keys, ( - f"Mismatch — missing from map: {all_keys - set(_mq._QUERY_FIELD_MAP)}, " - f"extra in map: {set(_mq._QUERY_FIELD_MAP) - all_keys}" - ) - - # --- T6: Fixture immunity --- - - def test_queue_state_fixture_populates_all_prfetchstate_fields(self): - """_queue_state() must cover every PRFetchState key so new fields are caught at once.""" - all_keys = PRFetchState.__required_keys__ | PRFetchState.__optional_keys__ - fixture_keys = set(_queue_state().keys()) - assert fixture_keys == all_keys, ( - f"_queue_state() missing keys: {all_keys - fixture_keys}, " - f"extra keys: {fixture_keys - all_keys}" - ) - - -# --------------------------------------------------------------------------- -# fetch_repo_merge_state rate-limit retry -# --------------------------------------------------------------------------- - - -_SUCCESS_BODY = { - "data": { - "repository": { - "mergeQueue": None, - "autoMergeAllowed": False, - "object": None, - } - } -} - - -class TestFetchRepoMergeStateRetry: - """Tests for HTTP 429 / secondary-rate-limit 403 retry behaviour.""" - - @pytest.mark.anyio - async def test_retries_on_429(self, httpx_mock, monkeypatch): - """Retries on HTTP 429, succeeds on second attempt.""" - sleep_calls: list[float] = [] - - async def fake_sleep(secs: float) -> None: - sleep_calls.append(secs) - - monkeypatch.setattr(_mq.asyncio, "sleep", fake_sleep) - - httpx_mock.add_response( - url="https://api.github.com/graphql", - status_code=429, - headers={"Retry-After": "1"}, - ) - httpx_mock.add_response( - url="https://api.github.com/graphql", - json=_SUCCESS_BODY, - ) - - from autoskillit.execution.merge_queue import fetch_repo_merge_state - - result = await fetch_repo_merge_state(owner="o", repo="r", branch="main", token=None) - assert result["queue_available"] is False - assert len(sleep_calls) == 1 - assert sleep_calls[0] == 1.0 # Retry-After header value is '1' - - @pytest.mark.anyio - async def test_retries_on_secondary_rate_limit_403(self, httpx_mock, monkeypatch): - """Retries on 403 whose body contains 'secondary rate limit'.""" - sleep_calls: list[float] = [] - - async def fake_sleep(secs: float) -> None: - sleep_calls.append(secs) - - monkeypatch.setattr(_mq.asyncio, "sleep", fake_sleep) - monkeypatch.setattr(_mq.random, "uniform", lambda a, b: 0.42) - - httpx_mock.add_response( - url="https://api.github.com/graphql", - status_code=403, - text="You have exceeded a secondary rate limit", - ) - httpx_mock.add_response( - url="https://api.github.com/graphql", - json=_SUCCESS_BODY, - ) - - from autoskillit.execution.merge_queue import fetch_repo_merge_state - - result = await fetch_repo_merge_state(owner="o", repo="r", branch="main", token=None) - assert result["queue_available"] is False - assert len(sleep_calls) == 1 - assert sleep_calls[0] == 0.42 # jitter backoff via patched random.uniform - - @pytest.mark.anyio - async def test_raises_on_non_rate_limit_403(self, httpx_mock): - """Non-secondary-rate-limit 403 propagates immediately without retry.""" - httpx_mock.add_response( - url="https://api.github.com/graphql", - status_code=403, - text="Bad credentials", - ) - - from autoskillit.execution.merge_queue import fetch_repo_merge_state - - with pytest.raises(httpx.HTTPStatusError) as exc_info: - await fetch_repo_merge_state(owner="o", repo="r", branch="main", token=None) - assert exc_info.value.response.status_code == 403 - # Only one request was made (no retry) - assert len(httpx_mock.get_requests()) == 1 - - @pytest.mark.anyio - async def test_exhausts_retries_and_raises(self, httpx_mock, monkeypatch): - """After max_attempts of 429, raises HTTPStatusError.""" - sleep_calls: list[float] = [] - - async def fake_sleep(secs: float) -> None: - sleep_calls.append(secs) - - monkeypatch.setattr(_mq.asyncio, "sleep", fake_sleep) - - for _ in range(_mq._RATE_LIMIT_MAX_ATTEMPTS): - httpx_mock.add_response( - url="https://api.github.com/graphql", - status_code=429, - headers={"Retry-After": "1"}, - ) - - from autoskillit.execution.merge_queue import fetch_repo_merge_state - - with pytest.raises(httpx.HTTPStatusError) as exc_info: - await fetch_repo_merge_state(owner="o", repo="r", branch="main", token=None) - assert exc_info.value.response.status_code == 429 - assert len(httpx_mock.get_requests()) == _mq._RATE_LIMIT_MAX_ATTEMPTS diff --git a/tests/execution/test_merge_queue_classifier.py b/tests/execution/test_merge_queue_classifier.py new file mode 100644 index 0000000000..277b5e2029 --- /dev/null +++ b/tests/execution/test_merge_queue_classifier.py @@ -0,0 +1,698 @@ +"""Tests for merge queue classifier: PendingCIGuard, InconclusiveBudget, +ClassifierImmunity, and VocabularyContract.""" + +from __future__ import annotations + +from datetime import UTC, datetime, timedelta +from unittest.mock import AsyncMock, patch + +import httpx +import pytest + +import autoskillit.execution.merge_queue as _mq +from autoskillit.core.types import PRState +from autoskillit.execution.merge_queue import ( + ClassifierInconclusive, + DefaultMergeQueueWatcher, + PRFetchState, +) +from tests.execution._merge_queue_helpers import _make_watcher, _queue_state + +pytestmark = [pytest.mark.layer("execution"), pytest.mark.small] + + +class TestPendingCIGuard: + """Tests for the pending-CI guard that prevents false ejection when checks are running.""" + + @pytest.mark.anyio + async def test_continues_polling_when_checks_pending_and_not_in_queue(self): + """Core bug: auto-merge + BLOCKED + PENDING checks must keep polling, not eject.""" + watcher = _make_watcher() + call_count = 0 + + async def _fetch(*_a, **_kw): + nonlocal call_count + call_count += 1 + if call_count < 4: + return _queue_state( + merged=False, + in_queue=False, + auto_merge_enabled_at=datetime.now(UTC), + merge_state_status="BLOCKED", + checks_state="PENDING", + ) + return _queue_state(merged=True) + + watcher._fetch_pr_and_queue_state = _fetch # type: ignore[method-assign] + with patch("autoskillit.execution.merge_queue.asyncio.sleep", new_callable=AsyncMock): + result = await watcher.wait(pr_number=1, target_branch="main", repo="owner/repo") + + assert result["success"] is True + assert result["pr_state"] == "merged" + assert call_count >= 4 + + @pytest.mark.anyio + async def test_continues_polling_when_checks_expected_and_not_in_queue(self): + """EXPECTED means 'check not yet started' — same as PENDING: keep polling.""" + watcher = _make_watcher() + call_count = 0 + + async def _fetch(*_a, **_kw): + nonlocal call_count + call_count += 1 + if call_count < 3: + return _queue_state( + in_queue=False, + auto_merge_enabled_at=datetime.now(UTC), + merge_state_status="BLOCKED", + checks_state="EXPECTED", + ) + return _queue_state(merged=True) + + watcher._fetch_pr_and_queue_state = _fetch # type: ignore[method-assign] + with patch("autoskillit.execution.merge_queue.asyncio.sleep", new_callable=AsyncMock): + result = await watcher.wait(pr_number=1, target_branch="main", repo="owner/repo") + + assert result["success"] is True + assert result["pr_state"] == "merged" + assert call_count >= 3 + + @pytest.mark.anyio + async def test_returns_dropped_healthy_when_checks_success_and_mergeable(self): + """Healthy PR (SUCCESS + MERGEABLE + CLEAN + auto_merge cleared) → dropped_healthy. + + This is the exact issue #802 failure mode: the old elimination classifier returned + 'ejected' for this state because no other gate matched. The new classifier returns + DROPPED_HEALTHY via positive signal — auto_merge was cleared while the PR was healthy. + Requires prior enrollment evidence (auto_merge_present=True in cycle 1). + """ + watcher = _make_watcher() + responses = [ + _queue_state( + in_queue=False, + mergeable="MERGEABLE", + merge_state_status="CLEAN", + checks_state="SUCCESS", + auto_merge_present=True, + ), + _queue_state( + in_queue=False, + mergeable="MERGEABLE", + merge_state_status="CLEAN", + checks_state="SUCCESS", + auto_merge_present=False, + ), + _queue_state( + in_queue=False, + mergeable="MERGEABLE", + merge_state_status="CLEAN", + checks_state="SUCCESS", + auto_merge_present=False, + ), + ] + watcher._fetch_pr_and_queue_state = AsyncMock(side_effect=responses) # type: ignore[method-assign] + with patch("autoskillit.execution.merge_queue.asyncio.sleep", new_callable=AsyncMock): + result = await watcher.wait(pr_number=1, target_branch="main", repo="owner/repo") + + assert result["success"] is False + assert result["pr_state"] == "dropped_healthy" + + @pytest.mark.anyio + async def test_returns_ejected_when_checks_success_but_mergeable_conflicting(self): + """checks_state=SUCCESS + mergeable=CONFLICTING → ejected via positive CONFLICTING gate.""" + watcher = _make_watcher() + watcher._fetch_pr_and_queue_state = AsyncMock( # type: ignore[method-assign] + return_value=_queue_state( + in_queue=False, + mergeable="CONFLICTING", + merge_state_status="BLOCKED", + checks_state="SUCCESS", + ) + ) + with patch("autoskillit.execution.merge_queue.asyncio.sleep", new_callable=AsyncMock): + result = await watcher.wait(pr_number=1, target_branch="main", repo="owner/repo") + + assert result["success"] is False + assert result["pr_state"] == "ejected" + + @pytest.mark.anyio + async def test_returns_ejected_when_no_checks_configured_but_conflicting(self): + """checks_state=None + mergeable=CONFLICTING → ejected via positive CONFLICTING signal. + + For repos without required CI checks, ejection requires a positive mergeable=CONFLICTING + signal rather than the absence of checks. ClassifierInconclusive handles the ambiguous + case (no CI, no conflict signal) by continuing to poll. + """ + watcher = _make_watcher() + watcher._fetch_pr_and_queue_state = AsyncMock( # type: ignore[method-assign] + return_value=_queue_state( + in_queue=False, + mergeable="CONFLICTING", + merge_state_status="BLOCKED", + checks_state=None, + ) + ) + with patch("autoskillit.execution.merge_queue.asyncio.sleep", new_callable=AsyncMock): + result = await watcher.wait(pr_number=1, target_branch="main", repo="owner/repo") + + assert result["success"] is False + assert result["pr_state"] == "ejected" + + @pytest.mark.anyio + async def test_fetch_extracts_checks_state_from_graphql_response(self): + """_fetch_pr_and_queue_state must extract statusCheckRollup.state into checks_state.""" + watcher = _make_watcher() + + async def _mock_post(url, *, json, headers=None, **_kw): + return httpx.Response( + 200, + json={ + "data": { + "repository": { + "pullRequest": { + "id": "PR_kwDO_test", + "merged": False, + "state": "OPEN", + "mergeStateStatus": "BLOCKED", + "autoMergeRequest": {"enabledAt": "2026-03-26T12:00:00Z"}, + "statusCheckRollup": {"state": "PENDING"}, + }, + "mergeQueue": {"entries": {"nodes": []}}, + } + } + }, + request=httpx.Request("POST", url), + ) + + watcher._client.post = _mock_post # type: ignore[method-assign] + state = await watcher._fetch_pr_and_queue_state(1, "owner", "repo", "main") + + assert state["checks_state"] == "PENDING" + assert state["in_queue"] is False + assert state["merged"] is False + + def test_implements_merge_queue_watcher_protocol(self): + from autoskillit.core import MergeQueueWatcher + + assert isinstance(_make_watcher(), MergeQueueWatcher) + + +class TestInconclusiveBudget: + """Tests for the split inconclusive budget: CIStillRunning vs NoPositiveSignal.""" + + @pytest.mark.anyio + async def test_pending_ci_does_not_exhaust_inconclusive_budget(self): + """CIStillRunning must not consume inconclusive_count. + Six PENDING cycles with budget=3 must NOT trigger budget ceiling. + timeout_seconds=99999 ensures outer deadline is not the cause. + """ + watcher = DefaultMergeQueueWatcher(token=None) + call_count = 0 + + async def _fetch(*_a, **_kw): + nonlocal call_count + call_count += 1 + if call_count > 6: + return _queue_state(merged=True) + return _queue_state( + merged=False, + in_queue=False, + checks_state="PENDING", + merge_state_status="BLOCKED", + auto_merge_enabled_at=None, + ) + + watcher._fetch_pr_and_queue_state = _fetch # type: ignore[method-assign] + with patch("autoskillit.execution.merge_queue.asyncio.sleep", new_callable=AsyncMock): + result = await watcher.wait( + pr_number=42, + target_branch="main", + repo="owner/repo", + poll_interval=1, + timeout_seconds=99999, + ) + + assert result["success"] is True + assert result["pr_state"] == "merged" + + @pytest.mark.anyio + async def test_expected_ci_does_not_exhaust_inconclusive_budget(self): + """CIStillRunning (EXPECTED) must not consume inconclusive_count. + Six EXPECTED cycles with budget=3 must NOT trigger budget ceiling. + """ + watcher = DefaultMergeQueueWatcher(token=None) + call_count = 0 + + async def _fetch(*_a, **_kw): + nonlocal call_count + call_count += 1 + if call_count > 6: + return _queue_state(merged=True) + return _queue_state( + merged=False, + in_queue=False, + checks_state="EXPECTED", + merge_state_status="BLOCKED", + auto_merge_enabled_at=None, + ) + + watcher._fetch_pr_and_queue_state = _fetch # type: ignore[method-assign] + with patch("autoskillit.execution.merge_queue.asyncio.sleep", new_callable=AsyncMock): + result = await watcher.wait( + pr_number=42, + target_branch="main", + repo="owner/repo", + poll_interval=1, + timeout_seconds=99999, + ) + + assert result["success"] is True + assert result["pr_state"] == "merged" + + @pytest.mark.anyio + async def test_no_positive_signal_exhausts_budget_returns_timeout(self): + """NoPositiveSignal must still exhaust the bounded budget. + Unknown state with no positive classifier match × (window + N) cycles + → pr_state='timeout'. Reason must contain 'Inconclusive after'. + """ + watcher = DefaultMergeQueueWatcher(token=None) + watcher._fetch_pr_and_queue_state = AsyncMock( # type: ignore[method-assign] + return_value=_queue_state( + merged=False, + in_queue=False, + checks_state="SUCCESS", + mergeable="UNKNOWN", + merge_state_status="UNKNOWN", + auto_merge_enabled_at=None, + ) + ) + with patch("autoskillit.execution.merge_queue.asyncio.sleep", new_callable=AsyncMock): + result = await watcher.wait( + pr_number=42, + target_branch="main", + repo="owner/repo", + poll_interval=1, + timeout_seconds=99999, + ) + + assert result["success"] is False + assert result["pr_state"] == "timeout" + assert "Inconclusive after" in result["reason"] + + @pytest.mark.anyio + async def test_inconclusive_count_resets_on_queue_reentry(self): + """inconclusive_count must reset when in_queue becomes True. + Scenario: out(PENDING×5) → in_queue → out(PENDING×5) → merged. + Budget=3. Without reset: second phase exhausts budget immediately. + With reset: both phases are tolerated independently. + """ + watcher = DefaultMergeQueueWatcher(token=None) + call_count = 0 + + async def _fetch(*_a, **_kw): + nonlocal call_count + call_count += 1 + if call_count <= 5: + return _queue_state( + merged=False, + in_queue=False, + checks_state="PENDING", + merge_state_status="BLOCKED", + auto_merge_enabled_at=None, + ) + if call_count == 6: + return _queue_state(merged=False, in_queue=True) + if call_count <= 11: + return _queue_state( + merged=False, + in_queue=False, + checks_state="PENDING", + merge_state_status="BLOCKED", + auto_merge_enabled_at=None, + ) + return _queue_state(merged=True) + + watcher._fetch_pr_and_queue_state = _fetch # type: ignore[method-assign] + with patch("autoskillit.execution.merge_queue.asyncio.sleep", new_callable=AsyncMock): + result = await watcher.wait( + pr_number=42, + target_branch="main", + repo="owner/repo", + poll_interval=1, + timeout_seconds=99999, + ) + + assert result["success"] is True + assert result["pr_state"] == "merged" + + @pytest.mark.anyio + async def test_confirmation_window_does_not_consume_inconclusive_budget(self): + """CIStillRunning (PENDING) must never consume the inconclusive budget at any cycle. + confirmation_cycles=4, budget=3: all 7 PENDING cycles complete without budget exhaustion + because PENDING raises CIStillRunning (exempt from budget). Merged on call 8. + """ + watcher = DefaultMergeQueueWatcher(token=None) + call_count = 0 + + async def _fetch(*_a, **_kw): + nonlocal call_count + call_count += 1 + if call_count > 7: + return _queue_state(merged=True) + return _queue_state( + merged=False, + in_queue=False, + checks_state="PENDING", + merge_state_status="BLOCKED", + auto_merge_enabled_at=None, + ) + + watcher._fetch_pr_and_queue_state = _fetch # type: ignore[method-assign] + with patch("autoskillit.execution.merge_queue.asyncio.sleep", new_callable=AsyncMock): + result = await watcher.wait( + pr_number=42, + target_branch="main", + repo="owner/repo", + poll_interval=1, + timeout_seconds=99999, + not_in_queue_confirmation_cycles=4, + ) + + assert result["success"] is True + assert result["pr_state"] == "merged" + assert call_count == 8 + + @pytest.mark.anyio + async def test_wait_accepts_max_inconclusive_retries_per_call_param(self): + """max_inconclusive_retries must be a wait() parameter, not constructor-only. + Passing it must not raise TypeError. Passing a small value (1) must cause + budget exhaustion after 1 NoPositiveSignal cycle beyond the confirmation window. + """ + watcher = _make_watcher() + # State: unknown — no positive classifier fires → NoPositiveSignal + watcher._fetch_pr_and_queue_state = AsyncMock( # type: ignore[method-assign] + return_value=_queue_state( + merged=False, + in_queue=False, + checks_state="SUCCESS", + mergeable="UNKNOWN", + merge_state_status="UNKNOWN", + auto_merge_enabled_at=None, + ) + ) + with patch("autoskillit.execution.merge_queue.asyncio.sleep", new_callable=AsyncMock): + result = await watcher.wait( + pr_number=42, + target_branch="main", + repo="owner/repo", + poll_interval=1, + timeout_seconds=99999, + max_inconclusive_retries=1, # ← per-call override + ) + + assert result["success"] is False + assert result["pr_state"] == "timeout" + assert "Inconclusive after 1 retries" in result["reason"] + + @pytest.mark.anyio + async def test_per_call_max_inconclusive_retries_overrides_constructor_default(self): + """Per-call value must take precedence over the constructor default of 5. + budget=2 per call, constructor default would be 5. Budget exhaustion must + occur after 2 ambiguous cycles (+ confirmation window), not 5. + """ + watcher = _make_watcher() # constructor default = 5 + call_count = 0 + + async def _fetch(*_a, **_kw): + nonlocal call_count + call_count += 1 + return _queue_state( + merged=False, + in_queue=False, + checks_state="SUCCESS", + mergeable="UNKNOWN", + merge_state_status="UNKNOWN", + auto_merge_enabled_at=None, + ) + + watcher._fetch_pr_and_queue_state = _fetch # type: ignore[method-assign] + with patch("autoskillit.execution.merge_queue.asyncio.sleep", new_callable=AsyncMock): + result = await watcher.wait( + pr_number=42, + target_branch="main", + repo="owner/repo", + poll_interval=1, + timeout_seconds=99999, + max_inconclusive_retries=2, # ← per-call override + ) + + assert result["success"] is False + assert result["pr_state"] == "timeout" + # confirmation_cycles=2 (default): cycle 1 is window, cycles 2-3 consume budget + # budget=2: exhausted on call 3 → total calls = 3 + assert call_count == 3 + + +class TestClassifierImmunity: + """Positive-signal immunity tests for the extracted _classify_pr_state function.""" + + # --- T1: Reproduces the reported bug --- + + def test_classifier_returns_dropped_healthy_when_auto_merge_cleared_on_healthy_pr(self): + """Exact issue #802 state: healthy PR with auto_merge cleared → DROPPED_HEALTHY.""" + state = _queue_state( + merged=False, + state="OPEN", + mergeable="MERGEABLE", + merge_state_status="CLEAN", + checks_state="SUCCESS", + in_queue=False, + auto_merge_present=False, + ) + result = _mq._classify_pr_state(state, ever_enrolled=True) + assert result.terminal == PRState.DROPPED_HEALTHY + + # --- T2: Positive-signal contract: EJECTED requires positive signal --- + + @pytest.mark.parametrize( + "state", + [ + # open + MERGEABLE + CLEAN + PENDING checks + _queue_state( + mergeable="MERGEABLE", + merge_state_status="CLEAN", + checks_state="PENDING", + auto_merge_present=True, + ), + # open + UNKNOWN mergeable + CLEAN + SUCCESS + _queue_state( + mergeable="UNKNOWN", + merge_state_status="CLEAN", + checks_state="SUCCESS", + ), + # open + MERGEABLE + HAS_HOOKS + SUCCESS + auto_merge_present=True + _queue_state( + mergeable="MERGEABLE", + merge_state_status="HAS_HOOKS", + checks_state="SUCCESS", + auto_merge_present=True, + ), + ], + ) + def test_classifier_never_returns_ejected_without_positive_signal(self, state): + """For ambiguous-but-healthy states, classifier must NOT return EJECTED.""" + try: + result = _mq._classify_pr_state(state, ever_enrolled=True) + assert result.terminal != PRState.EJECTED, ( + f"Classifier returned EJECTED without a positive signal for state: {state}" + ) + except ClassifierInconclusive: + pass # expected — no positive signal matched + + # --- T3: Classifier raises ClassifierInconclusive — no silent fall-through --- + + def test_classifier_raises_inconclusive_when_no_positive_signal_matches(self): + """ClassifierInconclusive raised when no positive gate matches; .state exposes fields.""" + state = _queue_state( + merged=False, + state="OPEN", + mergeable="UNKNOWN", + merge_state_status="BEHIND", + checks_state=None, + auto_merge_present=True, + in_queue=False, + ) + with pytest.raises(ClassifierInconclusive) as exc_info: + _mq._classify_pr_state(state, ever_enrolled=True) + assert exc_info.value.state is state + assert exc_info.value.reason + + # --- T4: Exhaustive PRState coverage --- + + @pytest.mark.parametrize( + "expected_terminal,state", + [ + ( + PRState.MERGED, + _queue_state(merged=True), + ), + ( + PRState.EJECTED, + _queue_state( + merged=False, + state="OPEN", + mergeable="CONFLICTING", + in_queue=False, + ), + ), + ( + PRState.EJECTED_CI_FAILURE, + _queue_state( + merged=False, + state="OPEN", + checks_state="FAILURE", + in_queue=False, + ), + ), + ( + PRState.STALLED, + _queue_state( + merged=False, + state="OPEN", + mergeable="MERGEABLE", + merge_state_status="CLEAN", + auto_merge_enabled_at=datetime.now(UTC) - timedelta(seconds=120), + auto_merge_present=True, + in_queue=False, + ), + ), + ( + PRState.DROPPED_HEALTHY, + _queue_state( + merged=False, + state="OPEN", + mergeable="MERGEABLE", + merge_state_status="CLEAN", + checks_state="SUCCESS", + auto_merge_present=False, + in_queue=False, + merge_group_checks_state=None, + ), + ), + ( + PRState.DROPPED_MERGE_GROUP_CI, + _queue_state( + merged=False, + state="OPEN", + mergeable="MERGEABLE", + merge_state_status="CLEAN", + checks_state="SUCCESS", + auto_merge_present=False, + in_queue=False, + merge_group_checks_state="FAILURE", + ), + ), + ], + ) + def test_classifier_returns_expected_terminal_for_canonical_fixture( + self, expected_terminal, state + ): + """Every classifier-reachable PRState terminal has at least one positive fixture.""" + result = _mq._classify_pr_state(state, ever_enrolled=True) + assert result.terminal == expected_terminal + + # --- T5: Import-time schema round-trip guard --- + + def test_query_field_map_matches_prfetchstate_required_keys(self): + """_QUERY_FIELD_MAP keys must exactly match PRFetchState required+optional keys.""" + all_keys = PRFetchState.__required_keys__ | PRFetchState.__optional_keys__ + assert set(_mq._QUERY_FIELD_MAP) == all_keys, ( + f"Mismatch — missing from map: {all_keys - set(_mq._QUERY_FIELD_MAP)}, " + f"extra in map: {set(_mq._QUERY_FIELD_MAP) - all_keys}" + ) + + # --- T6: Fixture immunity --- + + def test_queue_state_fixture_populates_all_prfetchstate_fields(self): + """_queue_state() must cover every PRFetchState key so new fields are caught at once.""" + all_keys = PRFetchState.__required_keys__ | PRFetchState.__optional_keys__ + fixture_keys = set(_queue_state().keys()) + assert fixture_keys == all_keys, ( + f"_queue_state() missing keys: {all_keys - fixture_keys}, " + f"extra keys: {fixture_keys - all_keys}" + ) + + +class TestMergeQueueVocabularyContract: + """KNOWN_MQ_MERGE_STATE_STATUSES must be declared as a named frozenset constant.""" + + def test_known_mq_merge_state_statuses_constant_exists(self): + """KNOWN_MQ_MERGE_STATE_STATUSES must be exported as a module-level frozenset.""" + from autoskillit.execution import merge_queue + + assert hasattr(merge_queue, "KNOWN_MQ_MERGE_STATE_STATUSES") + assert isinstance(merge_queue.KNOWN_MQ_MERGE_STATE_STATUSES, frozenset) + assert "CLEAN" in merge_queue.KNOWN_MQ_MERGE_STATE_STATUSES + + def test_positive_stall_statuses_subset_of_known(self): + """The positive-stall statuses used in _is_positive_stall must be known.""" + from autoskillit.execution.merge_queue import KNOWN_MQ_MERGE_STATE_STATUSES + + positive_stall_statuses = frozenset({"CLEAN", "HAS_HOOKS"}) + assert positive_stall_statuses.issubset(KNOWN_MQ_MERGE_STATE_STATUSES), ( + f"Positive stall statuses not in KNOWN_MQ_MERGE_STATE_STATUSES: " + f"{positive_stall_statuses - KNOWN_MQ_MERGE_STATE_STATUSES}" + ) + + +class TestDroppedMergeGroupCI: + """Tests for the DROPPED_MERGE_GROUP_CI classifier state (merge-group CI blind spot fix).""" + + def test_dropped_healthy_not_returned_when_merge_group_ci_failed(self): + """When a PR exits the queue with PR-branch CI SUCCESS but merge-group CI FAILURE, + the classifier must return DROPPED_MERGE_GROUP_CI, not DROPPED_HEALTHY.""" + state = _queue_state( + merged=False, + state="OPEN", + mergeable="MERGEABLE", + merge_state_status="CLEAN", + auto_merge_present=False, + in_queue=False, + checks_state="SUCCESS", + merge_group_checks_state="FAILURE", + ) + result = _mq._classify_pr_state(state, ever_enrolled=True) + assert result.terminal == PRState.DROPPED_MERGE_GROUP_CI + assert result.terminal != PRState.DROPPED_HEALTHY + + def test_dropped_healthy_fires_when_merge_group_ci_unknown(self): + """When merge-group CI result is unknown (None), DROPPED_HEALTHY is still valid — + we cannot confirm the ejection was CI-caused, so the conservative classification + (re-enroll) is correct.""" + state = _queue_state( + merged=False, + state="OPEN", + mergeable="MERGEABLE", + merge_state_status="CLEAN", + auto_merge_present=False, + in_queue=False, + checks_state="SUCCESS", + merge_group_checks_state=None, + ) + result = _mq._classify_pr_state(state, ever_enrolled=True) + assert result.terminal == PRState.DROPPED_HEALTHY + + def test_dropped_merge_group_ci_also_fires_on_error_conclusion(self): + """merge_group_checks_state='ERROR' is treated the same as 'FAILURE'.""" + state = _queue_state( + merged=False, + state="OPEN", + mergeable="MERGEABLE", + merge_state_status="CLEAN", + auto_merge_present=False, + in_queue=False, + checks_state="SUCCESS", + merge_group_checks_state="ERROR", + ) + result = _mq._classify_pr_state(state, ever_enrolled=True) + assert result.terminal == PRState.DROPPED_MERGE_GROUP_CI diff --git a/tests/execution/test_merge_queue_ejection.py b/tests/execution/test_merge_queue_ejection.py new file mode 100644 index 0000000000..d5b039057b --- /dev/null +++ b/tests/execution/test_merge_queue_ejection.py @@ -0,0 +1,567 @@ +"""Tests for merge queue ejection: RelatedCoverage, EjectionEnrichment, +FetchRepoMergeStateRetry, NotEnrolledState, and EnqueueMethod.""" + +from __future__ import annotations + +from datetime import UTC, datetime, timedelta +from unittest.mock import AsyncMock, patch + +import httpx +import pytest + +import autoskillit.execution.merge_queue as _mq +from autoskillit.core.types import PRState +from tests.execution._merge_queue_helpers import _make_watcher, _queue_state + +pytestmark = [pytest.mark.layer("execution"), pytest.mark.small] + + +class TestRelatedCoverage: + """Coverage for related untested paths found during investigation.""" + + @pytest.mark.anyio + async def test_unmergeable_waits_for_dequeue_before_returning_ejected(self): + """UNMERGEABLE must wait for in_queue=False before returning EJECTED.""" + watcher = _make_watcher() + watcher._fetch_pr_and_queue_state = AsyncMock( # type: ignore[method-assign] + side_effect=[ + _queue_state(in_queue=True, queue_state="UNMERGEABLE"), + _queue_state(in_queue=True, queue_state="UNMERGEABLE"), + _queue_state(in_queue=False, state="CLOSED"), + _queue_state(in_queue=False, state="CLOSED"), + ] + ) + with patch("autoskillit.execution.merge_queue.asyncio.sleep", new_callable=AsyncMock): + result = await watcher.wait( + pr_number=1, + target_branch="main", + repo="owner/repo", + poll_interval=1, + not_in_queue_confirmation_cycles=2, + ) + + assert result["success"] is False + assert result["pr_state"] == "ejected" + assert watcher._fetch_pr_and_queue_state.call_count == 4 # type: ignore[union-attr] + + @pytest.mark.anyio + async def test_unmergeable_dequeue_wait_respects_timeout(self): + """UNMERGEABLE with perpetual in_queue=True times out instead of returning ejected.""" + watcher = _make_watcher() + watcher._fetch_pr_and_queue_state = AsyncMock( # type: ignore[method-assign] + return_value=_queue_state(in_queue=True, queue_state="UNMERGEABLE") + ) + with patch("autoskillit.execution.merge_queue.asyncio.sleep", new_callable=AsyncMock): + result = await watcher.wait( + pr_number=1, + target_branch="main", + repo="owner/repo", + poll_interval=1, + timeout_seconds=0, + ) + + assert result["success"] is False + assert result["pr_state"] == "timeout" + + @pytest.mark.anyio + async def test_unmergeable_respects_confirmation_cycles(self): + """UNMERGEABLE with confirmation_cycles=2 requires 2 cycles after in_queue=False.""" + watcher = _make_watcher() + watcher._fetch_pr_and_queue_state = AsyncMock( # type: ignore[method-assign] + side_effect=[ + _queue_state(in_queue=True, queue_state="UNMERGEABLE"), + _queue_state(in_queue=False, state="CLOSED"), + _queue_state(in_queue=False, state="CLOSED"), + ] + ) + with patch("autoskillit.execution.merge_queue.asyncio.sleep", new_callable=AsyncMock): + result = await watcher.wait( + pr_number=1, + target_branch="main", + repo="owner/repo", + poll_interval=1, + not_in_queue_confirmation_cycles=2, + ) + + assert result["success"] is False + assert result["pr_state"] == "ejected" + assert watcher._fetch_pr_and_queue_state.call_count == 3 # type: ignore[union-attr] + + @pytest.mark.anyio + async def test_is_stall_candidate_when_has_hooks(self): + """merge_state_status=HAS_HOOKS + auto-merge enabled → stall detection fires.""" + watcher = _make_watcher() + enabled_at = datetime.now(UTC) - timedelta(seconds=120) + watcher._fetch_pr_and_queue_state = AsyncMock( # type: ignore[method-assign] + return_value=_queue_state( + merged=False, + in_queue=False, + auto_merge_enabled_at=enabled_at, + merge_state_status="HAS_HOOKS", + ) + ) + watcher._toggle_auto_merge = AsyncMock() # type: ignore[method-assign] + with patch("autoskillit.execution.merge_queue.asyncio.sleep", new_callable=AsyncMock): + result = await watcher.wait( + pr_number=1, + target_branch="main", + repo="owner/repo", + stall_grace_period=60, + max_stall_retries=1, + ) + + assert result["pr_state"] == "stalled" + assert watcher._toggle_auto_merge.call_count == 1 # type: ignore[union-attr] + + @pytest.mark.anyio + async def test_returns_error_when_repo_has_no_slash(self): + """repo='noslash' → pr_state='error', no polling.""" + watcher = _make_watcher() + result = await watcher.wait(pr_number=1, target_branch="main", repo="noslash") + + assert result["success"] is False + assert result["pr_state"] == "error" + assert "Invalid repo format" in result["reason"] + + +class TestEjectionEnrichment: + """Tests ejection response enrichment with CI failure cause.""" + + @pytest.mark.anyio + async def test_ejected_ci_failure_when_checks_state_is_failure(self): + """When checks_state=FAILURE, wait() returns pr_state='ejected_ci_failure'.""" + watcher = _make_watcher() + watcher._fetch_pr_and_queue_state = AsyncMock( # type: ignore[method-assign] + return_value=_queue_state( + merged=False, + in_queue=False, + checks_state="FAILURE", + merge_state_status="BLOCKED", + auto_merge_enabled_at=None, + ) + ) + with patch("autoskillit.execution.merge_queue.asyncio.sleep", new_callable=AsyncMock): + result = await watcher.wait( + pr_number=42, + target_branch="main", + repo="owner/repo", + poll_interval=1, + not_in_queue_confirmation_cycles=2, + ) + assert result["success"] is False + assert result["pr_state"] == "ejected_ci_failure" + assert result.get("ejection_cause") == "ci_failure" + + @pytest.mark.anyio + async def test_ejected_when_checks_state_is_none(self): + """checks_state=None + mergeable=CONFLICTING → pr_state='ejected', no ejection_cause.""" + watcher = _make_watcher() + watcher._fetch_pr_and_queue_state = AsyncMock( # type: ignore[method-assign] + return_value=_queue_state( + merged=False, + in_queue=False, + checks_state=None, + merge_state_status="BLOCKED", + mergeable="CONFLICTING", + auto_merge_enabled_at=None, + ) + ) + with patch("autoskillit.execution.merge_queue.asyncio.sleep", new_callable=AsyncMock): + result = await watcher.wait( + pr_number=42, + target_branch="main", + repo="owner/repo", + poll_interval=1, + not_in_queue_confirmation_cycles=2, + ) + assert result["success"] is False + assert result["pr_state"] == "ejected" + assert "ejection_cause" not in result + + @pytest.mark.anyio + async def test_ejected_when_checks_state_is_success(self): + """checks_state=SUCCESS + mergeable=CONFLICTING → pr_state='ejected', no enrichment.""" + watcher = _make_watcher() + watcher._fetch_pr_and_queue_state = AsyncMock( # type: ignore[method-assign] + return_value=_queue_state( + merged=False, + in_queue=False, + checks_state="SUCCESS", + merge_state_status="BLOCKED", + mergeable="CONFLICTING", + auto_merge_enabled_at=None, + ) + ) + with patch("autoskillit.execution.merge_queue.asyncio.sleep", new_callable=AsyncMock): + result = await watcher.wait( + pr_number=42, + target_branch="main", + repo="owner/repo", + poll_interval=1, + not_in_queue_confirmation_cycles=2, + ) + assert result["success"] is False + assert result["pr_state"] == "ejected" + assert "ejection_cause" not in result + + @pytest.mark.anyio + async def test_ejected_ci_failure_on_closed_pr_with_failure_checks(self): + """CLOSED state with checks_state=FAILURE → ejected_ci_failure.""" + watcher = _make_watcher() + watcher._fetch_pr_and_queue_state = AsyncMock( # type: ignore[method-assign] + return_value=_queue_state( + state="CLOSED", + merged=False, + checks_state="FAILURE", + ) + ) + result = await watcher.wait( + pr_number=42, target_branch="main", repo="owner/repo", poll_interval=1 + ) + assert result["success"] is False + assert result["pr_state"] == "ejected_ci_failure" + assert result.get("ejection_cause") == "ci_failure" + + @pytest.mark.anyio + async def test_ejected_ci_failure_after_in_queue_to_not_in_queue_transition(self): + """in_queue=True on first poll, then in_queue=False+FAILURE → ejected_ci_failure.""" + watcher = _make_watcher() + watcher._fetch_pr_and_queue_state = AsyncMock( # type: ignore[method-assign] + side_effect=[ + _queue_state(in_queue=True, checks_state=None), + _queue_state( + in_queue=False, + checks_state="FAILURE", + merge_state_status="BLOCKED", + auto_merge_enabled_at=None, + ), + _queue_state( + in_queue=False, + checks_state="FAILURE", + merge_state_status="BLOCKED", + auto_merge_enabled_at=None, + ), + ] + ) + with patch("autoskillit.execution.merge_queue.asyncio.sleep", new_callable=AsyncMock): + result = await watcher.wait( + pr_number=42, + target_branch="main", + repo="owner/repo", + poll_interval=1, + not_in_queue_confirmation_cycles=2, + ) + assert result["success"] is False + assert result["pr_state"] == "ejected_ci_failure" + assert result.get("ejection_cause") == "ci_failure" + + +# --------------------------------------------------------------------------- +# fetch_repo_merge_state rate-limit retry +# --------------------------------------------------------------------------- + + +_SUCCESS_BODY = { + "data": { + "repository": { + "mergeQueue": None, + "autoMergeAllowed": False, + "object": None, + } + } +} + + +class TestFetchRepoMergeStateRetry: + """Tests for HTTP 429 / secondary-rate-limit 403 retry behaviour.""" + + @pytest.mark.anyio + async def test_retries_on_429(self, httpx_mock, monkeypatch): + """Retries on HTTP 429, succeeds on second attempt.""" + sleep_calls: list[float] = [] + + async def fake_sleep(secs: float) -> None: + sleep_calls.append(secs) + + monkeypatch.setattr(_mq.asyncio, "sleep", fake_sleep) + + httpx_mock.add_response( + url="https://api.github.com/graphql", + status_code=429, + headers={"Retry-After": "1"}, + ) + httpx_mock.add_response( + url="https://api.github.com/graphql", + json=_SUCCESS_BODY, + ) + + from autoskillit.execution.merge_queue import fetch_repo_merge_state + + result = await fetch_repo_merge_state(owner="o", repo="r", branch="main", token=None) + assert result["queue_available"] is False + assert len(sleep_calls) == 1 + assert sleep_calls[0] == 1.0 # Retry-After header value is '1' + + @pytest.mark.anyio + async def test_retries_on_secondary_rate_limit_403(self, httpx_mock, monkeypatch): + """Retries on 403 whose body contains 'secondary rate limit'.""" + sleep_calls: list[float] = [] + + async def fake_sleep(secs: float) -> None: + sleep_calls.append(secs) + + monkeypatch.setattr(_mq.asyncio, "sleep", fake_sleep) + monkeypatch.setattr(_mq.random, "uniform", lambda a, b: 0.42) + + httpx_mock.add_response( + url="https://api.github.com/graphql", + status_code=403, + text="You have exceeded a secondary rate limit", + ) + httpx_mock.add_response( + url="https://api.github.com/graphql", + json=_SUCCESS_BODY, + ) + + from autoskillit.execution.merge_queue import fetch_repo_merge_state + + result = await fetch_repo_merge_state(owner="o", repo="r", branch="main", token=None) + assert result["queue_available"] is False + assert len(sleep_calls) == 1 + assert sleep_calls[0] == 0.42 # jitter backoff via patched random.uniform + + @pytest.mark.anyio + async def test_raises_on_non_rate_limit_403(self, httpx_mock): + """Non-secondary-rate-limit 403 propagates immediately without retry.""" + httpx_mock.add_response( + url="https://api.github.com/graphql", + status_code=403, + text="Bad credentials", + ) + + from autoskillit.execution.merge_queue import fetch_repo_merge_state + + with pytest.raises(httpx.HTTPStatusError) as exc_info: + await fetch_repo_merge_state(owner="o", repo="r", branch="main", token=None) + assert exc_info.value.response.status_code == 403 + # Only one request was made (no retry) + assert len(httpx_mock.get_requests()) == 1 + + @pytest.mark.anyio + async def test_exhausts_retries_and_raises(self, httpx_mock, monkeypatch): + """After max_attempts of 429, raises HTTPStatusError.""" + sleep_calls: list[float] = [] + + async def fake_sleep(secs: float) -> None: + sleep_calls.append(secs) + + monkeypatch.setattr(_mq.asyncio, "sleep", fake_sleep) + + for _ in range(_mq._RATE_LIMIT_MAX_ATTEMPTS): + httpx_mock.add_response( + url="https://api.github.com/graphql", + status_code=429, + headers={"Retry-After": "1"}, + ) + + from autoskillit.execution.merge_queue import fetch_repo_merge_state + + with pytest.raises(httpx.HTTPStatusError) as exc_info: + await fetch_repo_merge_state(owner="o", repo="r", branch="main", token=None) + assert exc_info.value.response.status_code == 429 + assert len(httpx_mock.get_requests()) == _mq._RATE_LIMIT_MAX_ATTEMPTS + + +class TestNotEnrolledState: + """Tests for the NOT_ENROLLED classifier state and enrollment tracking.""" + + def test_pr_state_has_not_enrolled(self): + """PRState must include NOT_ENROLLED value.""" + assert hasattr(PRState, "NOT_ENROLLED") + assert PRState.NOT_ENROLLED == "not_enrolled" + + def test_classifier_returns_not_enrolled_when_never_in_queue_and_healthy(self): + """A healthy PR with ever_enrolled=False must classify as NOT_ENROLLED.""" + state = _queue_state( + merged=False, + state="OPEN", + mergeable="MERGEABLE", + merge_state_status="CLEAN", + checks_state="SUCCESS", + in_queue=False, + auto_merge_present=False, + ) + result = _mq._classify_pr_state(state, ever_enrolled=False) + assert result.terminal == PRState.NOT_ENROLLED + + def test_classifier_returns_dropped_healthy_only_when_ever_enrolled(self): + """DROPPED_HEALTHY requires ever_enrolled=True.""" + state = _queue_state( + merged=False, + state="OPEN", + mergeable="MERGEABLE", + merge_state_status="CLEAN", + checks_state="SUCCESS", + in_queue=False, + auto_merge_present=False, + ) + result = _mq._classify_pr_state(state, ever_enrolled=True) + assert result.terminal == PRState.DROPPED_HEALTHY + + @pytest.mark.anyio + async def test_wait_tracks_ever_enrolled_from_in_queue(self): + """wait() sets ever_enrolled=True when in_queue=True is observed.""" + watcher = _make_watcher() + responses = [ + _queue_state(in_queue=True, queue_state="QUEUED"), + _queue_state( + in_queue=False, + auto_merge_present=False, + mergeable="MERGEABLE", + merge_state_status="CLEAN", + checks_state="SUCCESS", + ), + _queue_state( + in_queue=False, + auto_merge_present=False, + mergeable="MERGEABLE", + merge_state_status="CLEAN", + checks_state="SUCCESS", + ), + ] + watcher._fetch_pr_and_queue_state = AsyncMock(side_effect=responses) # type: ignore[method-assign] + with patch("autoskillit.execution.merge_queue.asyncio.sleep", new_callable=AsyncMock): + result = await watcher.wait( + pr_number=42, + target_branch="main", + repo="owner/repo", + poll_interval=1, + not_in_queue_confirmation_cycles=2, + ) + assert result["pr_state"] == "dropped_healthy" + + @pytest.mark.anyio + async def test_wait_tracks_ever_enrolled_from_auto_merge_present(self): + """wait() sets ever_enrolled=True when auto_merge_present=True is observed.""" + watcher = _make_watcher() + responses = [ + _queue_state(in_queue=False, auto_merge_present=True), + _queue_state( + in_queue=False, + auto_merge_present=False, + mergeable="MERGEABLE", + merge_state_status="CLEAN", + checks_state="SUCCESS", + ), + _queue_state( + in_queue=False, + auto_merge_present=False, + mergeable="MERGEABLE", + merge_state_status="CLEAN", + checks_state="SUCCESS", + ), + ] + watcher._fetch_pr_and_queue_state = AsyncMock(side_effect=responses) # type: ignore[method-assign] + with patch("autoskillit.execution.merge_queue.asyncio.sleep", new_callable=AsyncMock): + result = await watcher.wait( + pr_number=42, + target_branch="main", + repo="owner/repo", + poll_interval=1, + not_in_queue_confirmation_cycles=2, + ) + assert result["pr_state"] == "dropped_healthy" + + @pytest.mark.anyio + async def test_wait_returns_not_enrolled_when_enrollment_never_observed(self): + """wait() returns NOT_ENROLLED when healthy PR never shows enrollment evidence.""" + watcher = _make_watcher() + healthy_state = _queue_state( + in_queue=False, + auto_merge_present=False, + mergeable="MERGEABLE", + merge_state_status="CLEAN", + checks_state="SUCCESS", + ) + watcher._fetch_pr_and_queue_state = AsyncMock( # type: ignore[method-assign] + return_value=healthy_state, + ) + with patch("autoskillit.execution.merge_queue.asyncio.sleep", new_callable=AsyncMock): + result = await watcher.wait( + pr_number=42, + target_branch="main", + repo="owner/repo", + poll_interval=1, + not_in_queue_confirmation_cycles=2, + ) + assert result["pr_state"] == "not_enrolled" + assert result["success"] is False + + +# --------------------------------------------------------------------------- +# enqueue() method tests +# --------------------------------------------------------------------------- + + +class TestEnqueueMethod: + """Tests for DefaultMergeQueueWatcher.enqueue() enrollment strategy.""" + + @pytest.mark.anyio + async def test_enqueue_uses_enqueue_pr_mutation_when_auto_merge_unavailable(self): + """When auto_merge_available=False, enqueue() must call enqueuePullRequest.""" + watcher = _make_watcher() + watcher._fetch_pr_and_queue_state = AsyncMock( # type: ignore[method-assign] + return_value=_queue_state(pr_node_id="PR_kwDO_test123"), + ) + posted_bodies: list[str] = [] + + async def _mock_post(*args, **kwargs): + body = kwargs.get("json", {}) + posted_bodies.append(body.get("query", "")) + return httpx.Response( + 200, + json={"data": {"enqueuePullRequest": {"mergeQueueEntry": {"id": "MQE_1"}}}}, + request=httpx.Request("POST", "http://x"), + ) + + watcher._client.post = _mock_post # type: ignore[method-assign] + result = await watcher.enqueue( + pr_number=42, + target_branch="main", + repo="owner/repo", + auto_merge_available=False, + ) + assert result["success"] is True + assert result["enrollment_method"] == "direct_enqueue" + assert any("enqueuePullRequest" in b for b in posted_bodies) + assert not any("enablePullRequestAutoMerge" in b for b in posted_bodies) + + @pytest.mark.anyio + async def test_enqueue_uses_auto_merge_mutation_when_auto_merge_available(self): + """When auto_merge_available=True, enqueue() must call enablePullRequestAutoMerge.""" + watcher = _make_watcher() + watcher._fetch_pr_and_queue_state = AsyncMock( # type: ignore[method-assign] + return_value=_queue_state(pr_node_id="PR_kwDO_test123"), + ) + posted_bodies: list[str] = [] + + async def _mock_post(*args, **kwargs): + body = kwargs.get("json", {}) + posted_bodies.append(body.get("query", "")) + return httpx.Response( + 200, + json={"data": {"enablePullRequestAutoMerge": {"pullRequest": {"number": 42}}}}, + request=httpx.Request("POST", "http://x"), + ) + + watcher._client.post = _mock_post # type: ignore[method-assign] + result = await watcher.enqueue( + pr_number=42, + target_branch="main", + repo="owner/repo", + auto_merge_available=True, + ) + assert result["success"] is True + assert result["enrollment_method"] == "auto_merge" + assert any("enablePullRequestAutoMerge" in b for b in posted_bodies) + assert not any("enqueuePullRequest" in b for b in posted_bodies) diff --git a/tests/execution/test_merge_queue_polling.py b/tests/execution/test_merge_queue_polling.py new file mode 100644 index 0000000000..83082f13db --- /dev/null +++ b/tests/execution/test_merge_queue_polling.py @@ -0,0 +1,733 @@ +"""Tests for DefaultMergeQueueWatcher polling state machine.""" + +from __future__ import annotations + +from datetime import UTC, datetime, timedelta +from unittest.mock import AsyncMock, MagicMock, patch + +import httpx +import pytest + +from tests.execution._merge_queue_helpers import _make_watcher, _queue_state + +pytestmark = [pytest.mark.layer("execution"), pytest.mark.small] + + +class TestDefaultMergeQueueWatcher: + """Tests for DefaultMergeQueueWatcher polling state machine.""" + + @pytest.mark.anyio + async def test_returns_merged_on_first_pr_state_check(self): + watcher = _make_watcher() + watcher._fetch_pr_and_queue_state = AsyncMock( # type: ignore[method-assign] + return_value=_queue_state(merged=True) + ) + result = await watcher.wait( + pr_number=42, target_branch="main", repo="owner/repo", poll_interval=1 + ) + assert result["success"] is True + assert result["pr_state"] == "merged" + + @pytest.mark.anyio + async def test_returns_merged_when_pr_closed_and_merged(self): + """merged=True takes priority over state=CLOSED.""" + watcher = _make_watcher() + watcher._fetch_pr_and_queue_state = AsyncMock( # type: ignore[method-assign] + return_value=_queue_state(state="CLOSED", merged=True) + ) + result = await watcher.wait( + pr_number=42, target_branch="main", repo="owner/repo", poll_interval=1 + ) + assert result["success"] is True + assert result["pr_state"] == "merged" + + @pytest.mark.anyio + async def test_returns_ejected_when_mergeable_conflicting_and_not_in_queue(self): + """mergeable=CONFLICTING + not in queue → ejected via positive conflicting signal.""" + watcher = _make_watcher() + watcher._fetch_pr_and_queue_state = AsyncMock( # type: ignore[method-assign] + return_value=_queue_state( + merged=False, + in_queue=False, + mergeable="CONFLICTING", + merge_state_status="BLOCKED", + ) + ) + with patch("autoskillit.execution.merge_queue.asyncio.sleep", new_callable=AsyncMock): + result = await watcher.wait( + pr_number=42, target_branch="main", repo="owner/repo", poll_interval=1 + ) + assert result["success"] is False + assert result["pr_state"] == "ejected" + + @pytest.mark.anyio + async def test_keeps_polling_while_pr_in_queue(self): + watcher = _make_watcher() + call_count = 0 + + async def _fetch_side(*_a: object, **_kw: object) -> dict: + nonlocal call_count + call_count += 1 + if call_count >= 3: + return _queue_state(merged=True) + return _queue_state(in_queue=True, queue_state="AWAITING_CHECKS") + + watcher._fetch_pr_and_queue_state = _fetch_side # type: ignore[method-assign] + + with patch( + "autoskillit.execution.merge_queue.asyncio.sleep", new_callable=AsyncMock + ) as mock_sleep: + result = await watcher.wait( + pr_number=42, target_branch="main", repo="owner/repo", poll_interval=1 + ) + + assert result["success"] is True + assert result["pr_state"] == "merged" + assert mock_sleep.call_count >= 2 + + @pytest.mark.anyio + async def test_stuck_detection_triggers_toggle_once(self): + """Stalled PR (auto_merge set, grace expired) triggers toggle and then merges.""" + watcher = _make_watcher() + toggle_calls: list[int] = [] + call_count = 0 + enabled_at = datetime.now(UTC) - timedelta(seconds=120) + + async def _fetch_side(*_a: object, **_kw: object) -> dict: + nonlocal call_count + call_count += 1 + if call_count >= 4: + return _queue_state(merged=True) + return _queue_state( + auto_merge_enabled_at=enabled_at, + merge_state_status="CLEAN", + in_queue=False, + ) + + async def _toggle_side(*_a: object, **_kw: object) -> None: + toggle_calls.append(1) + + watcher._fetch_pr_and_queue_state = _fetch_side # type: ignore[method-assign] + watcher._toggle_auto_merge = _toggle_side # type: ignore[method-assign] + + with patch("autoskillit.execution.merge_queue.asyncio.sleep", new_callable=AsyncMock): + result = await watcher.wait( + pr_number=42, + target_branch="main", + repo="owner/repo", + poll_interval=1, + stall_grace_period=60, + ) + + assert result["success"] is True + assert result["pr_state"] == "merged" + assert len(toggle_calls) == 1 + + @pytest.mark.anyio + async def test_returns_timeout_when_deadline_exceeded(self): + watcher = _make_watcher() + watcher._fetch_pr_and_queue_state = AsyncMock( # type: ignore[method-assign] + return_value=_queue_state(in_queue=True, queue_state="AWAITING_CHECKS") + ) + + with patch("autoskillit.execution.merge_queue.asyncio.sleep", new_callable=AsyncMock): + with patch("autoskillit.execution.merge_queue.time.monotonic") as mock_time: + # deadline=0.0+1000=1000.0; loop enters (1.0<1000); after sleep exceeds (1001.0) + mock_time.side_effect = [0.0, 1.0, 1001.0] + result = await watcher.wait( + pr_number=42, + target_branch="main", + repo="owner/repo", + timeout_seconds=1000, + poll_interval=1, + ) + + assert result["success"] is False + assert result["pr_state"] == "timeout" + assert "Timed out" in result["reason"] + + @pytest.mark.anyio + async def test_returns_ejected_when_pr_closed_not_merged(self): + watcher = _make_watcher() + watcher._fetch_pr_and_queue_state = AsyncMock( # type: ignore[method-assign] + return_value=_queue_state(state="CLOSED", merged=False) + ) + result = await watcher.wait( + pr_number=42, target_branch="main", repo="owner/repo", poll_interval=1 + ) + assert result["success"] is False + assert result["pr_state"] == "ejected" + + @pytest.mark.anyio + async def test_http_error_in_pr_state_logs_and_retries(self): + """Fetch error is caught and retried; subsequent success returns merged.""" + watcher = _make_watcher() + call_count = 0 + + async def _fetch_side(*_a: object, **_kw: object) -> dict: + nonlocal call_count + call_count += 1 + if call_count == 1: + raise httpx.HTTPStatusError( + "500", + request=httpx.Request("GET", "http://x"), + response=httpx.Response(500), + ) + return _queue_state(merged=True) + + watcher._fetch_pr_and_queue_state = _fetch_side # type: ignore[method-assign] + + with patch("autoskillit.execution.merge_queue.asyncio.sleep", new_callable=AsyncMock): + result = await watcher.wait( + pr_number=42, target_branch="main", repo="owner/repo", poll_interval=1 + ) + + assert result["success"] is True + assert result["pr_state"] == "merged" + + @pytest.mark.anyio + async def test_graphql_error_falls_through_as_retry(self): + """GraphQL fetch error is caught and retried; subsequent success returns merged.""" + watcher = _make_watcher() + call_count = 0 + + async def _fetch_side(*_a: object, **_kw: object) -> dict: + nonlocal call_count + call_count += 1 + if call_count == 1: + raise RuntimeError("GraphQL error: connection error") + return _queue_state(merged=True) + + watcher._fetch_pr_and_queue_state = _fetch_side # type: ignore[method-assign] + + with patch("autoskillit.execution.merge_queue.asyncio.sleep", new_callable=AsyncMock): + result = await watcher.wait( + pr_number=42, target_branch="main", repo="owner/repo", poll_interval=1 + ) + + assert result["success"] is True + assert result["pr_state"] == "merged" + + @pytest.mark.anyio + async def test_graphql_errors_raise(self): + """GraphQL error responses from _fetch_pr_and_queue_state should raise.""" + watcher = _make_watcher() + + async def _mock_post(*args, **kwargs): + resp = httpx.Response( + 200, + json={"errors": [{"message": "some error"}]}, + request=httpx.Request("POST", "http://x"), + ) + return resp + + watcher._client.post = _mock_post # type: ignore[method-assign] + with pytest.raises(RuntimeError, match="GraphQL error"): + await watcher._fetch_pr_and_queue_state(42, "owner", "repo", "main") + + +class TestMergeQueueReliability: + """Tests for confirmation-window ejection guard and stall detection/retry logic.""" + + # Group 1: Confirmation-window ejection guard + + @pytest.mark.anyio + async def test_no_false_ejection_when_pr_merges_between_cycles(self): + """Cycle 1 sees 'not in queue'; cycle 2 sees merged=True → return merged.""" + watcher = _make_watcher() + responses = [ + _queue_state(merged=False, in_queue=False), + _queue_state(merged=True), + ] + watcher._fetch_pr_and_queue_state = AsyncMock(side_effect=responses) # type: ignore[method-assign] + with patch("autoskillit.execution.merge_queue.asyncio.sleep", new_callable=AsyncMock): + result = await watcher.wait( + pr_number=42, target_branch="main", repo="owner/repo", poll_interval=1 + ) + assert result["success"] is True + assert result["pr_state"] == "merged" + + @pytest.mark.anyio + async def test_confirmation_window_delays_ejection_by_one_cycle(self): + """Two consecutive not-in-queue cycles required before acting on CONFLICTING ejection.""" + watcher = _make_watcher() + state = _queue_state( + merged=False, in_queue=False, mergeable="CONFLICTING", merge_state_status="BLOCKED" + ) + watcher._fetch_pr_and_queue_state = AsyncMock(return_value=state) # type: ignore[method-assign] + with patch("autoskillit.execution.merge_queue.asyncio.sleep", new_callable=AsyncMock): + result = await watcher.wait( + pr_number=42, target_branch="main", repo="owner/repo", poll_interval=1 + ) + assert result["success"] is False + assert result["pr_state"] == "ejected" + # Confirmation window requires exactly 2 cycles before ejection + assert watcher._fetch_pr_and_queue_state.call_count == 2 # type: ignore[union-attr] + + @pytest.mark.anyio + async def test_single_graphql_call_per_poll_cycle(self): + """Each poll cycle makes exactly one call to _fetch_pr_and_queue_state.""" + watcher = _make_watcher() + call_count = 0 + + async def _fetch_side(*_a: object, **_kw: object) -> dict: + nonlocal call_count + call_count += 1 + if call_count >= 3: + return _queue_state(merged=True) + return _queue_state(in_queue=True, queue_state="AWAITING_CHECKS") + + watcher._fetch_pr_and_queue_state = _fetch_side # type: ignore[method-assign] + with patch("autoskillit.execution.merge_queue.asyncio.sleep", new_callable=AsyncMock): + result = await watcher.wait( + pr_number=42, target_branch="main", repo="owner/repo", poll_interval=1 + ) + assert result["success"] is True + assert call_count == 3 # exactly 3 fetch calls (not 6 as with the old 2-call approach) + + # Group 2: Stall detection and multi-retry + + @pytest.mark.anyio + async def test_stall_grace_period_prevents_premature_toggle(self): + """PR stalled 30s ago with grace=60s → toggle must NOT be called within grace.""" + watcher = _make_watcher() + toggle_calls: list[object] = [] + call_count = 0 + enabled_at = datetime.now(UTC) - timedelta(seconds=30) + + async def _fetch_side(*_a: object, **_kw: object) -> dict: + nonlocal call_count + call_count += 1 + if call_count >= 4: + return _queue_state(merged=True) + return _queue_state( + auto_merge_enabled_at=enabled_at, + merge_state_status="CLEAN", + in_queue=False, + ) + + async def _toggle_side(*_a: object, **_kw: object) -> None: + toggle_calls.append(1) + + watcher._fetch_pr_and_queue_state = _fetch_side # type: ignore[method-assign] + watcher._toggle_auto_merge = _toggle_side # type: ignore[method-assign] + with patch("autoskillit.execution.merge_queue.asyncio.sleep", new_callable=AsyncMock): + result = await watcher.wait( + pr_number=42, + target_branch="main", + repo="owner/repo", + poll_interval=1, + stall_grace_period=60, + ) + assert len(toggle_calls) == 0, "Toggle must NOT be called within grace period" + assert result["success"] is True + + @pytest.mark.anyio + async def test_stall_toggle_triggered_after_grace_period_expires(self): + """PR stalled 90s ago with grace=60s → toggle IS called after grace expires.""" + watcher = _make_watcher() + toggle_calls: list[object] = [] + call_count = 0 + enabled_at = datetime.now(UTC) - timedelta(seconds=90) + + async def _fetch_side(*_a: object, **_kw: object) -> dict: + nonlocal call_count + call_count += 1 + if call_count >= 4: + return _queue_state(merged=True) + return _queue_state( + auto_merge_enabled_at=enabled_at, + merge_state_status="CLEAN", + in_queue=False, + ) + + async def _toggle_side(*_a: object, **_kw: object) -> None: + toggle_calls.append(1) + + watcher._fetch_pr_and_queue_state = _fetch_side # type: ignore[method-assign] + watcher._toggle_auto_merge = _toggle_side # type: ignore[method-assign] + with patch("autoskillit.execution.merge_queue.asyncio.sleep", new_callable=AsyncMock): + await watcher.wait( + pr_number=42, + target_branch="main", + repo="owner/repo", + poll_interval=1, + stall_grace_period=60, + ) + assert len(toggle_calls) >= 1, "Toggle must be called after grace period expires" + + @pytest.mark.anyio + async def test_max_stall_retries_exhausted_returns_stalled_not_ejected(self): + """After max_stall_retries=2 toggle attempts, returns pr_state='stalled' not 'ejected'.""" + watcher = _make_watcher() + enabled_at = datetime.now(UTC) - timedelta(seconds=120) + watcher._fetch_pr_and_queue_state = AsyncMock( # type: ignore[method-assign] + return_value=_queue_state( + merged=False, + in_queue=False, + auto_merge_enabled_at=enabled_at, + merge_state_status="CLEAN", + ) + ) + watcher._toggle_auto_merge = AsyncMock() # type: ignore[method-assign] + with patch("autoskillit.execution.merge_queue.asyncio.sleep", new_callable=AsyncMock): + result = await watcher.wait( + pr_number=42, + target_branch="main", + repo="owner/repo", + poll_interval=1, + stall_grace_period=60, + max_stall_retries=2, + ) + assert result["success"] is False + assert result["pr_state"] == "stalled" + assert result["stall_retries_attempted"] == 2 + + @pytest.mark.anyio + async def test_exponential_backoff_between_stall_retries(self): + """Backoff durations must be [30, 60, 120] seconds for retries 0, 1, 2.""" + watcher = _make_watcher() + enabled_at = datetime.now(UTC) - timedelta(seconds=120) + watcher._fetch_pr_and_queue_state = AsyncMock( # type: ignore[method-assign] + return_value=_queue_state( + merged=False, + in_queue=False, + auto_merge_enabled_at=enabled_at, + merge_state_status="CLEAN", + ) + ) + watcher._toggle_auto_merge = AsyncMock() # type: ignore[method-assign] + sleep_calls: list[float] = [] + + async def _capture_sleep(seconds: float) -> None: + sleep_calls.append(seconds) + + with patch("autoskillit.execution.merge_queue.asyncio.sleep", side_effect=_capture_sleep): + result = await watcher.wait( + pr_number=42, + target_branch="main", + repo="owner/repo", + poll_interval=1, + stall_grace_period=60, + max_stall_retries=3, + ) + assert result["pr_state"] == "stalled" + backoff_sleeps = [s for s in sleep_calls if s > 1] + assert backoff_sleeps == [30, 60, 120] + + @pytest.mark.anyio + async def test_graphql_mutation_used_for_toggle(self): + """Toggle must use disablePullRequestAutoMerge and enablePullRequestAutoMerge mutations.""" + watcher = _make_watcher() + enabled_at = datetime.now(UTC) - timedelta(seconds=120) + call_count = 0 + + async def _fetch_side(*_a: object, **_kw: object) -> dict: + nonlocal call_count + call_count += 1 + if call_count >= 4: + return _queue_state(merged=True) + return _queue_state( + merged=False, + in_queue=False, + auto_merge_enabled_at=enabled_at, + merge_state_status="CLEAN", + ) + + watcher._fetch_pr_and_queue_state = _fetch_side # type: ignore[method-assign] + captured_bodies: list[dict] = [] + + async def _mock_post(url: str, **kwargs: object) -> httpx.Response: + body = dict(kwargs.get("json", {}) or {}) # type: ignore[arg-type] + captured_bodies.append(body) + query = body.get("query", "") + if "disablePullRequestAutoMerge" in query: + data: dict = { + "data": {"disablePullRequestAutoMerge": {"pullRequest": {"number": 42}}} + } + elif "enablePullRequestAutoMerge" in query: + data = {"data": {"enablePullRequestAutoMerge": {"pullRequest": {"number": 42}}}} + else: + data = {"data": {}} + return httpx.Response(200, json=data, request=httpx.Request("POST", url)) + + watcher._client.post = _mock_post # type: ignore[method-assign] + with patch("autoskillit.execution.merge_queue.asyncio.sleep", new_callable=AsyncMock): + await watcher.wait( + pr_number=42, + target_branch="main", + repo="owner/repo", + poll_interval=1, + stall_grace_period=60, + max_stall_retries=1, + ) + queries = [b.get("query", "") for b in captured_bodies] + assert any("disablePullRequestAutoMerge" in q for q in queries), ( + "Expected disable mutation" + ) + assert any("enablePullRequestAutoMerge" in q for q in queries), "Expected enable mutation" + + @pytest.mark.anyio + async def test_stall_recovery_success_after_toggle(self): + """After 1 toggle PR re-enters queue then merges → success, stall_retries_attempted=1.""" + watcher = _make_watcher() + toggle_calls: list[object] = [] + call_count = 0 + enabled_at = datetime.now(UTC) - timedelta(seconds=120) + + async def _fetch_side(*_a: object, **_kw: object) -> dict: + nonlocal call_count + call_count += 1 + if call_count <= 2: + return _queue_state( + merged=False, + in_queue=False, + auto_merge_enabled_at=enabled_at, + merge_state_status="CLEAN", + ) + elif call_count == 3: + return _queue_state(in_queue=True, queue_state="AWAITING_CHECKS") + else: + return _queue_state(merged=True) + + async def _toggle_side(*_a: object, **_kw: object) -> None: + toggle_calls.append(1) + + watcher._fetch_pr_and_queue_state = _fetch_side # type: ignore[method-assign] + watcher._toggle_auto_merge = _toggle_side # type: ignore[method-assign] + with patch("autoskillit.execution.merge_queue.asyncio.sleep", new_callable=AsyncMock): + result = await watcher.wait( + pr_number=42, + target_branch="main", + repo="owner/repo", + poll_interval=1, + stall_grace_period=60, + max_stall_retries=3, + ) + assert result["success"] is True + assert result["pr_state"] == "merged" + assert result["stall_retries_attempted"] == 1 + + # Group 3: Tool Interface + + @pytest.mark.anyio + async def test_stall_retries_attempted_present_in_all_terminal_responses(self): + """stall_retries_attempted key must be present in all four terminal responses.""" + watcher = _make_watcher() + + # merged + watcher._fetch_pr_and_queue_state = AsyncMock( # type: ignore[method-assign] + return_value=_queue_state(merged=True) + ) + result = await watcher.wait(pr_number=1, target_branch="main", repo="owner/repo") + assert "stall_retries_attempted" in result, ( + "merged response missing stall_retries_attempted" + ) + assert result["stall_retries_attempted"] == 0, "merged: no retries expected" + + # ejected (closed) + watcher._fetch_pr_and_queue_state = AsyncMock( # type: ignore[method-assign] + return_value=_queue_state(state="CLOSED", merged=False) + ) + result = await watcher.wait(pr_number=1, target_branch="main", repo="owner/repo") + assert "stall_retries_attempted" in result, ( + "ejected response missing stall_retries_attempted" + ) + assert result["stall_retries_attempted"] == 0, "ejected: no retries expected" + + # timeout + with patch("autoskillit.execution.merge_queue.asyncio.sleep", new_callable=AsyncMock): + with patch("autoskillit.execution.merge_queue.time.monotonic") as mock_time: + mock_time.side_effect = [0.0, 1.0, 1001.0] + watcher._fetch_pr_and_queue_state = AsyncMock( # type: ignore[method-assign] + return_value=_queue_state(in_queue=True, queue_state="AWAITING_CHECKS") + ) + result = await watcher.wait( + pr_number=1, + target_branch="main", + repo="owner/repo", + timeout_seconds=1000, + ) + assert "stall_retries_attempted" in result, ( + "timeout response missing stall_retries_attempted" + ) + assert result["stall_retries_attempted"] == 0, "timeout: no retries expected" + assert result["pr_state"] == "timeout" + + # stalled + enabled_at = datetime.now(UTC) - timedelta(seconds=120) + watcher._fetch_pr_and_queue_state = AsyncMock( # type: ignore[method-assign] + return_value=_queue_state( + merged=False, + in_queue=False, + auto_merge_enabled_at=enabled_at, + merge_state_status="CLEAN", + ) + ) + watcher._toggle_auto_merge = AsyncMock() # type: ignore[method-assign] + with patch("autoskillit.execution.merge_queue.asyncio.sleep", new_callable=AsyncMock): + result = await watcher.wait( + pr_number=1, + target_branch="main", + repo="owner/repo", + stall_grace_period=60, + max_stall_retries=1, + ) + assert "stall_retries_attempted" in result, ( + "stalled response missing stall_retries_attempted" + ) + assert result["stall_retries_attempted"] == 1, "stalled: expected max_stall_retries=1" + assert result["pr_state"] == "stalled" + + @pytest.mark.anyio + async def test_wait_accepts_stall_grace_period_and_max_stall_retries_params(self): + """wait() accepts stall_grace_period and max_stall_retries without TypeError.""" + watcher = _make_watcher() + watcher._fetch_pr_and_queue_state = AsyncMock( # type: ignore[method-assign] + return_value=_queue_state(merged=True) + ) + result = await watcher.wait( + pr_number=42, + target_branch="main", + repo="owner/repo", + stall_grace_period=120, + max_stall_retries=5, + ) + assert result["success"] is True + assert result["pr_state"] == "merged" + + @pytest.mark.anyio + async def test_transition_gate_queries_merge_group_ci_on_in_to_out_with_success_checks(self): + """When the watcher observes in_queue True→False with checks_state=SUCCESS, + it must call _query_merge_group_ci and surface DROPPED_MERGE_GROUP_CI if CI failed.""" + import autoskillit.execution.merge_queue as _mq + + watcher = _make_watcher() + call_count = 0 + + async def _fetch(*_a, **_kw): + nonlocal call_count + call_count += 1 + if call_count == 1: + return _queue_state(in_queue=True, checks_state="SUCCESS") + return _queue_state( + in_queue=False, + checks_state="SUCCESS", + state="OPEN", + mergeable="MERGEABLE", + merge_state_status="CLEAN", + auto_merge_present=False, + merge_group_checks_state=None, + ) + + watcher._fetch_pr_and_queue_state = _fetch # type: ignore[method-assign] + with ( + patch("autoskillit.execution.merge_queue.asyncio.sleep", new_callable=AsyncMock), + patch.object( + _mq, "_query_merge_group_ci", new_callable=AsyncMock, return_value="FAILURE" + ) as mock_query, + ): + result = await watcher.wait(pr_number=7, target_branch="main", repo="owner/repo") + + mock_query.assert_called_once() + assert result["pr_state"] == "dropped_merge_group_ci" + + @pytest.mark.anyio + async def test_transition_gate_not_triggered_when_checks_already_failed(self): + """When checks_state is FAILURE on exit, the transition gate must NOT query + merge-group CI — the classifier already handles this as EJECTED_CI_FAILURE.""" + import autoskillit.execution.merge_queue as _mq + + watcher = _make_watcher() + call_count = 0 + + async def _fetch(*_a, **_kw): + nonlocal call_count + call_count += 1 + if call_count == 1: + return _queue_state(in_queue=True, checks_state="SUCCESS") + return _queue_state(in_queue=False, checks_state="FAILURE", state="OPEN") + + watcher._fetch_pr_and_queue_state = _fetch # type: ignore[method-assign] + with ( + patch("autoskillit.execution.merge_queue.asyncio.sleep", new_callable=AsyncMock), + patch.object( + _mq, "_query_merge_group_ci", new_callable=AsyncMock, return_value="FAILURE" + ) as mock_query, + ): + await watcher.wait(pr_number=7, target_branch="main", repo="owner/repo") + + mock_query.assert_not_called() + + +class TestToggleAutoMergeConfirmation: + """Tests for _toggle_auto_merge confirmation polling instead of hardcoded sleep.""" + + @pytest.mark.anyio + async def test_toggle_polls_for_disable_confirmation(self): + """_toggle_auto_merge polls until auto-merge is confirmed disabled.""" + watcher = _make_watcher() + watcher._client = AsyncMock() + + disable_resp = MagicMock() + disable_resp.status_code = 200 + disable_resp.raise_for_status = lambda: None + disable_resp.json.return_value = {"data": {"disablePullRequestAutoMerge": {}}} + + poll_resp_still_active = MagicMock() + poll_resp_still_active.status_code = 200 + poll_resp_still_active.json.return_value = { + "data": {"node": {"autoMergeRequest": {"enabledAt": "2026-01-01T00:00:00Z"}}} + } + + poll_resp_disabled = MagicMock() + poll_resp_disabled.status_code = 200 + poll_resp_disabled.json.return_value = {"data": {"node": {"autoMergeRequest": None}}} + + enable_resp = MagicMock() + enable_resp.status_code = 200 + enable_resp.raise_for_status = lambda: None + enable_resp.json.return_value = {"data": {"enablePullRequestAutoMerge": {}}} + + watcher._client.post = AsyncMock( + side_effect=[disable_resp, poll_resp_still_active, poll_resp_disabled, enable_resp] + ) + watcher._ensure_client = lambda: watcher._client + + with patch("autoskillit.execution.merge_queue.asyncio.sleep", new_callable=AsyncMock): + await watcher._toggle_auto_merge("PR_node123") + + assert watcher._client.post.call_count == 4 + + @pytest.mark.anyio + async def test_toggle_does_not_use_hardcoded_sleep_2(self): + """_toggle_auto_merge no longer uses asyncio.sleep(2).""" + watcher = _make_watcher() + watcher._client = AsyncMock() + + disable_resp = MagicMock() + disable_resp.status_code = 200 + disable_resp.raise_for_status = lambda: None + disable_resp.json.return_value = {"data": {"disablePullRequestAutoMerge": {}}} + + poll_resp = MagicMock() + poll_resp.status_code = 200 + poll_resp.json.return_value = {"data": {"node": {"autoMergeRequest": None}}} + + enable_resp = MagicMock() + enable_resp.status_code = 200 + enable_resp.raise_for_status = lambda: None + enable_resp.json.return_value = {"data": {"enablePullRequestAutoMerge": {}}} + + watcher._client.post = AsyncMock(side_effect=[disable_resp, poll_resp, enable_resp]) + watcher._ensure_client = lambda: watcher._client + + sleep_durations: list[float] = [] + + async def _capture_sleep(seconds: float) -> None: + sleep_durations.append(seconds) + + with patch("autoskillit.execution.merge_queue.asyncio.sleep", side_effect=_capture_sleep): + await watcher._toggle_auto_merge("PR_node123") + + assert 2 not in sleep_durations diff --git a/tests/execution/test_normalize_subtype.py b/tests/execution/test_normalize_subtype.py index d3a400f245..7de5132c73 100644 --- a/tests/execution/test_normalize_subtype.py +++ b/tests/execution/test_normalize_subtype.py @@ -1,11 +1,15 @@ -"""Unit tests for the _normalize_subtype() normalization gate.""" +"""Unit tests for ClaudeSessionResult.normalize_subtype() normalization gate.""" from __future__ import annotations +import pytest + from autoskillit.core.types import ( SessionOutcome, ) -from autoskillit.execution.session import ClaudeSessionResult, _normalize_subtype +from autoskillit.execution.session import ClaudeSessionResult + +pytestmark = [pytest.mark.layer("execution"), pytest.mark.small] def _session( @@ -27,14 +31,14 @@ def _session( def test_normalize_succeeded_empty_output_returns_success(): """SUCCEEDED + 'empty_output' → 'success' (Class 2 fix).""" session = _session(subtype="empty_output", result="", is_error=False) - result = _normalize_subtype("empty_output", SessionOutcome.SUCCEEDED, session, "") + result = session.normalize_subtype(SessionOutcome.SUCCEEDED, "") assert result == "success" def test_normalize_succeeded_success_passthrough(): """SUCCEEDED + 'success' → 'success' (passthrough).""" session = _session(subtype="success", result="output") - result = _normalize_subtype("success", SessionOutcome.SUCCEEDED, session, "") + result = session.normalize_subtype(SessionOutcome.SUCCEEDED, "") assert result == "success" @@ -46,14 +50,14 @@ def test_normalize_succeeded_success_passthrough(): def test_normalize_failed_error_during_execution_passthrough(): """FAILED + 'error_during_execution' → 'error_during_execution' (passthrough).""" session = _session(subtype="error_during_execution", result="", is_error=True) - result = _normalize_subtype("error_during_execution", SessionOutcome.FAILED, session, "") + result = session.normalize_subtype(SessionOutcome.FAILED, "") assert result == "error_during_execution" def test_normalize_retriable_error_max_turns_passthrough(): """RETRIABLE + 'error_max_turns' → 'error_max_turns' (passthrough).""" session = _session(subtype="error_max_turns", result="partial") - result = _normalize_subtype("error_max_turns", SessionOutcome.RETRIABLE, session, "") + result = session.normalize_subtype(SessionOutcome.RETRIABLE, "") assert result == "error_max_turns" @@ -65,21 +69,21 @@ def test_normalize_retriable_error_max_turns_passthrough(): def test_normalize_failed_success_empty_result_returns_empty_result(): """FAILED + 'success' + empty result → 'empty_result' (Class 1 fix).""" session = _session(subtype="success", result="", is_error=False) - result = _normalize_subtype("success", SessionOutcome.FAILED, session, "") + result = session.normalize_subtype(SessionOutcome.FAILED, "") assert result == "empty_result" def test_normalize_failed_success_missing_marker_returns_missing_completion_marker(): """FAILED + 'success' + non-empty result + missing marker → 'missing_completion_marker'.""" session = _session(subtype="success", result="I did the work.", is_error=False) - result = _normalize_subtype("success", SessionOutcome.FAILED, session, "%%ORDER_UP%%") + result = session.normalize_subtype(SessionOutcome.FAILED, "%%ORDER_UP%%") assert result == "missing_completion_marker" def test_normalize_retriable_success_empty_result_returns_empty_result(): """RETRIABLE + 'success' + empty result → 'empty_result'.""" session = _session(subtype="success", result="", is_error=False) - result = _normalize_subtype("success", SessionOutcome.RETRIABLE, session, "") + result = session.normalize_subtype(SessionOutcome.RETRIABLE, "") assert result == "empty_result" @@ -96,8 +100,61 @@ def test_normalize_retriable_success_jsonl_exhausted_returns_context_exhausted() session_id="s1", jsonl_context_exhausted=True, ) - result = _normalize_subtype("success", SessionOutcome.RETRIABLE, session, "%%ORDER_UP%%") + result = session.normalize_subtype(SessionOutcome.RETRIABLE, "%%ORDER_UP%%") assert result == "context_exhausted", ( f"Expected 'context_exhausted', got '{result}'. " "JSONL-detected context exhaustion must not be masked as missing_completion_marker." ) + + +# --------------------------------------------------------------------------- +# Path 2: is_error + errors list contains marker → 'context_exhausted' +# --------------------------------------------------------------------------- + + +def test_normalize_retriable_success_context_exhausted_via_errors_list(): + """Path 2: is_error=True + errors list contains marker → 'context_exhausted'.""" + from autoskillit.core.types._type_constants import CONTEXT_EXHAUSTION_MARKER + + session = ClaudeSessionResult( + subtype="success", + result="partial work but no completion marker", + is_error=True, + session_id="s1", + errors=[f"Request failed: {CONTEXT_EXHAUSTION_MARKER}"], + jsonl_context_exhausted=False, + ) + assert session._is_context_exhausted(), "Path 2 must fire on this fixture" + result = session.normalize_subtype(SessionOutcome.RETRIABLE, "%%ORDER_UP%%") + assert result == "context_exhausted" + + +# --------------------------------------------------------------------------- +# Path 3: is_error + result text contains marker → 'context_exhausted' +# --------------------------------------------------------------------------- + + +def test_normalize_retriable_success_context_exhausted_via_result_text(): + """Path 3: is_error=True + result contains marker + subtype=success → 'context_exhausted'. + + This is the bug scenario: jsonl_context_exhausted is False (wrapped message format + bypassed the flat-record scan), but _is_context_exhausted() fires via Path 3. + The old code checked jsonl_context_exhausted directly and returned 'missing_completion_marker'. + The new code delegates to _is_context_exhausted() and returns 'context_exhausted'. + """ + from autoskillit.core.types._type_constants import CONTEXT_EXHAUSTION_MARKER + + session = ClaudeSessionResult( + subtype="success", + result=CONTEXT_EXHAUSTION_MARKER.capitalize(), # "Prompt is too long" + is_error=True, + session_id="s1", + jsonl_context_exhausted=False, # Path 1 deliberately absent + ) + assert session._is_context_exhausted(), "Path 3 must fire on this fixture" + result = session.normalize_subtype(SessionOutcome.RETRIABLE, "%%ORDER_UP%%") + assert result == "context_exhausted", ( + f"Expected 'context_exhausted', got '{result}'. " + "normalize_subtype must delegate to _is_context_exhausted(), not read " + "jsonl_context_exhausted directly." + ) diff --git a/tests/execution/test_on_spawn_timing.py b/tests/execution/test_on_spawn_timing.py new file mode 100644 index 0000000000..fdb4b320d7 --- /dev/null +++ b/tests/execution/test_on_spawn_timing.py @@ -0,0 +1,81 @@ +"""Tests for on_pid_resolved callback timing in run_managed_async (Group J).""" + +from __future__ import annotations + +import sys +from pathlib import Path +from unittest.mock import AsyncMock, MagicMock, patch + +import psutil +import pytest + +pytestmark = [pytest.mark.layer("execution"), pytest.mark.small] + + +class TestOnPidResolvedTiming: + async def test_on_pid_resolved_fires_during_execution(self, tmp_path: Path) -> None: + """Callback fires while the subprocess is still alive (spawn-time, not post-completion).""" + from autoskillit.execution.process import run_managed_async + + alive_at_callback: list[bool] = [] + + def callback(pid: int, ticks: int) -> None: + alive_at_callback.append(psutil.pid_exists(pid)) + + await run_managed_async( + [sys.executable, "-c", "import time; time.sleep(0.3)"], + cwd=tmp_path, + timeout=5.0, + on_pid_resolved=callback, + ) + + assert len(alive_at_callback) == 1, "Callback should fire exactly once" + assert alive_at_callback[0] is True, "Process must be alive when callback fires" + + async def test_on_pid_resolved_provides_starttime_ticks(self, tmp_path: Path) -> None: + """On Linux, starttime_ticks is > 0; non-Linux 0 is accepted.""" + from autoskillit.execution.process import run_managed_async + + received_ticks: list[int] = [] + + def callback(pid: int, ticks: int) -> None: + received_ticks.append(ticks) + + await run_managed_async( + [sys.executable, "-c", "import time; time.sleep(0.3)"], + cwd=tmp_path, + timeout=5.0, + on_pid_resolved=callback, + ) + + assert len(received_ticks) == 1 + if sys.platform == "linux": + assert received_ticks[0] > 0, "starttime_ticks must be > 0 on Linux" + else: + assert received_ticks[0] >= 0, "Non-Linux: 0 is accepted" + + async def test_on_pid_resolved_not_called_on_zero_pid(self, tmp_path: Path) -> None: + """Guard: on_pid_resolved is NOT invoked when _observed_pid is 0.""" + from autoskillit.execution.process import run_managed_async + + called_with: list[tuple[int, int]] = [] + + mock_proc = MagicMock() + mock_proc.pid = 0 + + with patch( + "autoskillit.execution.process.anyio.open_process", + new_callable=AsyncMock, + return_value=mock_proc, + ): + try: + await run_managed_async( + [sys.executable, "-c", "pass"], + cwd=tmp_path, + timeout=1.0, + on_pid_resolved=lambda pid, ticks: called_with.append((pid, ticks)), + ) + except Exception: + pass # Mock artifacts expected — only the callback check matters + + assert called_with == [], "on_pid_resolved must NOT be called when pid == 0" diff --git a/tests/execution/test_output_format_contract.py b/tests/execution/test_output_format_contract.py index cac0e53bba..672c346855 100644 --- a/tests/execution/test_output_format_contract.py +++ b/tests/execution/test_output_format_contract.py @@ -12,13 +12,22 @@ from __future__ import annotations import json +from pathlib import Path + +import pytest from autoskillit.core.types import ( ChannelConfirmation, + DirectInstall, OutputFormat, SubprocessResult, TerminationReason, ) +from autoskillit.execution.commands import ( + build_food_truck_cmd, + build_headless_resume_cmd, + build_skill_session_cmd, +) from autoskillit.execution.headless import _build_skill_result from autoskillit.execution.session import ( ClaudeSessionResult, @@ -26,6 +35,8 @@ parse_session_result, ) +pytestmark = [pytest.mark.layer("execution"), pytest.mark.small] + class TestOutputFormatDataContract: """Contract tests binding output format to data availability.""" @@ -180,16 +191,16 @@ def test_format_enum_has_capability_declarations(self): def test_run_skill_config_derives_format(self): """RunSkillConfig.output_format derives STREAM_JSON for default config.""" - from autoskillit.config.settings import RunSkillConfig + from tests._helpers import make_run_skill_config - cfg = RunSkillConfig() # default completion_marker is set + cfg = make_run_skill_config() # default completion_marker is set assert cfg.output_format == OutputFormat.STREAM_JSON def test_run_skill_config_derives_json_when_no_marker(self): """RunSkillConfig.output_format derives JSON when completion_marker is empty.""" - from autoskillit.config.settings import RunSkillConfig + from tests._helpers import make_run_skill_config - cfg = RunSkillConfig(completion_marker="") + cfg = make_run_skill_config(completion_marker="") assert cfg.output_format == OutputFormat.JSON @@ -211,22 +222,83 @@ def test_all_formats_declare_required_flags(self): assert isinstance(flags, tuple), f"{fmt.name} returned {type(flags)}, expected tuple" def test_command_assembly_satisfies_format_requirements(self): - """Assembled command must contain all flags required by the chosen format. + """Actual builders must produce commands containing all required flags.""" + fmt = OutputFormat.STREAM_JSON + spec = build_skill_session_cmd( + "/investigate foo", + cwd="/tmp", + completion_marker="%%DONE%%", + model=None, + plugin_source=None, + output_format=fmt, + ) + for flag in fmt.required_cli_flags: + assert flag in spec.cmd, f"Missing required flag {flag} in skill builder" - Builds a command the same way run_headless_core does, then asserts - every flag from the format's required_cli_flags is present. - """ - from autoskillit.execution.commands import build_headless_cmd - fmt = OutputFormat.STREAM_JSON - spec = build_headless_cmd("Use /investigate test", model=None) - cmd = spec.cmd + ["--plugin-dir", "/fake", "--output-format", fmt.value] - # Apply required flags (mirrors run_headless_core logic) +class TestAllBuildersEnforceOutputFormatFlags: + """Every cmd builder that accepts OutputFormat must self-apply required_cli_flags.""" + + @pytest.mark.parametrize("fmt", list(OutputFormat)) + def test_skill_builder_satisfies_format_requirements(self, fmt: OutputFormat): + spec = build_skill_session_cmd( + "/investigate foo", + cwd="/tmp", + completion_marker="%%DONE%%", + model=None, + plugin_source=None, + output_format=fmt, + ) for flag in fmt.required_cli_flags: - if flag not in cmd: - cmd.append(flag) + assert flag in spec.cmd + + @pytest.mark.parametrize("fmt", list(OutputFormat)) + def test_food_truck_builder_satisfies_format_requirements(self, fmt: OutputFormat): + spec = build_food_truck_cmd( + orchestrator_prompt="Orchestrator", + plugin_source=DirectInstall(plugin_dir=Path("/p")), + cwd="/tmp", + completion_marker="%%DONE%%", + output_format=fmt, + ) + for flag in fmt.required_cli_flags: + assert flag in spec.cmd + + @pytest.mark.parametrize("fmt", list(OutputFormat)) + def test_resume_builder_satisfies_format_requirements(self, fmt: OutputFormat): + spec = build_headless_resume_cmd( + resume_session_id="abc", + prompt="Emit", + output_format=fmt, + ) for flag in fmt.required_cli_flags: - assert flag in cmd, f"Missing required flag {flag} in assembled command" + assert flag in spec.cmd + + +def test_all_headless_builders_handle_output_format(): + """Every build_*_cmd function accepting output_format must produce correct flags.""" + import ast + import importlib + import inspect + + mod = importlib.import_module("autoskillit.execution.commands") + source = inspect.getsource(mod) + tree = ast.parse(source) + + builders_with_format = [] + for node in ast.walk(tree): + if ( + isinstance(node, ast.FunctionDef) + and node.name.startswith("build_") + and node.name.endswith("_cmd") + ): + params = [a.arg for a in node.args.args + node.args.kwonlyargs] + if "output_format" in params: + builders_with_format.append(node.name) + + assert len(builders_with_format) >= 3, ( + f"Expected at least 3 builders with output_format, found {builders_with_format}" + ) class TestChannelDefaultCoverage: diff --git a/tests/execution/test_pr_analysis.py b/tests/execution/test_pr_analysis.py index 47b41b328e..be916e5242 100644 --- a/tests/execution/test_pr_analysis.py +++ b/tests/execution/test_pr_analysis.py @@ -2,6 +2,8 @@ from __future__ import annotations +import pytest + from autoskillit.execution.pr_analysis import ( DOMAIN_PATHS, extract_linked_issues, @@ -9,6 +11,8 @@ partition_files_by_domain, ) +pytestmark = [pytest.mark.layer("execution"), pytest.mark.small] + # --------------------------------------------------------------------------- # extract_linked_issues # --------------------------------------------------------------------------- diff --git a/tests/execution/test_process_channel_b.py b/tests/execution/test_process_channel_b.py index bc153460a3..65f6eacf05 100644 --- a/tests/execution/test_process_channel_b.py +++ b/tests/execution/test_process_channel_b.py @@ -9,8 +9,11 @@ from autoskillit.core.types import ChannelConfirmation, SubprocessResult, TerminationReason from autoskillit.execution.process import run_managed_async +from tests.conftest import TimeoutTier from tests.execution.conftest import WRITE_RESULT_THEN_HANG_SCRIPT +pytestmark = [pytest.mark.layer("execution"), pytest.mark.medium] + # Script that: # (1) writes %%ORDER_UP%% to a JSONL session file (Channel B fires) # (2) writes type=result to stdout after a delay (Channel A confirms within drain window) @@ -20,6 +23,8 @@ import sys, time, json, os session_dir = sys.argv[1] os.makedirs(session_dir, exist_ok=True) + sys.stdout.write(json.dumps({"type": "system", "session_id": "session"}) + "\\n") + sys.stdout.flush() # Small delay to ensure file ctime > spawn_time recorded in run_managed_async time.sleep(0.1) with open(os.path.join(session_dir, "session.jsonl"), "w") as f: @@ -50,7 +55,7 @@ "content": "%%ORDER_UP%%"}} f.write(json.dumps(record) + "\\n") f.flush() - time.sleep(3600) + time.sleep(300) """) # Script that: @@ -64,6 +69,8 @@ import sys, time, json, os session_dir = sys.argv[1] os.makedirs(session_dir, exist_ok=True) + sys.stdout.write(json.dumps({"type": "system", "session_id": "session"}) + "\\n") + sys.stdout.flush() # Small delay to ensure file ctime > spawn_time recorded in run_managed_async time.sleep(0.1) with open(os.path.join(session_dir, "session.jsonl"), "w") as f: @@ -88,6 +95,8 @@ import sys, json, os, time session_dir = sys.argv[1] os.makedirs(session_dir, exist_ok=True) + sys.stdout.write(json.dumps({"type": "system", "session_id": "session"}) + "\\n") + sys.stdout.flush() # Small delay ensures file ctime > spawn_time recorded in run_managed_async time.sleep(0.1) with open(os.path.join(session_dir, "session.jsonl"), "w") as f: @@ -106,6 +115,7 @@ class TestChannelBDrainWait: """Channel B (session monitor) winning before Channel A triggers bounded drain wait.""" + @pytest.mark.timeout(90) @pytest.mark.anyio async def test_channel_b_wins_then_channel_a_confirms_within_drain(self, tmp_path): """Channel B fires first; drain wait allows Channel A to confirm stdout data. @@ -127,19 +137,22 @@ async def test_channel_b_wins_then_channel_a_confirms_within_drain(self, tmp_pat result = await run_managed_async( [sys.executable, str(script), str(session_dir), "0.15"], cwd=tmp_path, - timeout=30, + timeout=TimeoutTier.CHANNEL_B, session_log_dir=session_dir, completion_marker="%%ORDER_UP%%", completion_drain_timeout=5.0, + _phase1_timeout=120, _phase1_poll=0.01, _phase2_poll=0.05, _heartbeat_poll=0.05, + _session_id_timeout=0.01, ) assert result.termination == TerminationReason.COMPLETED # Drain wait confirmed Channel A fired: stdout is non-empty assert result.stdout.strip() + @pytest.mark.timeout(150) @pytest.mark.anyio async def test_channel_b_wins_drain_timeout_still_kills(self, tmp_path): """Channel B fires; Channel A never fires; drain times out and process is killed. @@ -150,10 +163,14 @@ async def test_channel_b_wins_drain_timeout_still_kills(self, tmp_path): t=0.66s drain times out (script never wrote to stdout) t=0.66s process killed with empty stdout - timeout=60s: guards against the outer wall-clock expiring under xdist -n 4 load. - _watch_session_log waits up to 1s for stdout_session_id_ready before Phase 1 starts; + timeout=120s: guards against the outer wall-clock expiring under xdist -n 4 load. + _watch_session_log waits up to _session_id_timeout (default 1.0s, tests pass 0.01s) + for stdout_session_id_ready before Phase 1 starts; under CI load both the preamble and Phase 1 polls can overrun, so the outer - timeout must exceed 1s + _phase1_timeout (30s default) + drain (0.5s) = 31.5s. + timeout must exceed _session_id_timeout + _phase1_timeout (30s default) + drain (0.5s) = 31.5s. + _phase1_timeout=250: must exceed outer timeout (120s) so that Phase 1 never fires + first with STALE when subprocess startup is slow under WSL2 + xdist load; the + outer 120s guard cancels all tasks before Phase 1 can timeout independently. """ session_dir = tmp_path / "session" session_dir.mkdir() @@ -163,18 +180,21 @@ async def test_channel_b_wins_drain_timeout_still_kills(self, tmp_path): result = await run_managed_async( [sys.executable, str(script), str(session_dir)], cwd=tmp_path, - timeout=60, + timeout=120, session_log_dir=session_dir, completion_marker="%%ORDER_UP%%", completion_drain_timeout=0.5, _phase1_poll=0.01, _phase2_poll=0.05, + _session_id_timeout=0.01, + _phase1_timeout=250, ) assert result.termination == TerminationReason.COMPLETED # Drain timed out: CLI hung and never flushed its result record assert not result.stdout.strip() + @pytest.mark.timeout(90) @pytest.mark.anyio async def test_channel_a_wins_unchanged_behavior(self, tmp_path): """Channel A (heartbeat) wins before any session monitor: no drain wait needed. @@ -190,7 +210,7 @@ async def test_channel_a_wins_unchanged_behavior(self, tmp_path): result = await run_managed_async( [sys.executable, str(script)], cwd=tmp_path, - timeout=30, + timeout=60, # No session_log_dir: Channel B cannot fire _heartbeat_poll=0.05, ) @@ -198,12 +218,20 @@ async def test_channel_a_wins_unchanged_behavior(self, tmp_path): assert result.termination == TerminationReason.COMPLETED assert result.stdout.strip() # Channel A confirmed: stdout is non-empty + @pytest.mark.timeout(90) @pytest.mark.anyio async def test_data_confirmed_false_set_on_drain_timeout(self, tmp_path): """Channel B wins the race; drain timeout expires without Channel A confirming. Verifies that SubprocessResult.data_confirmed is False when the bounded drain wait times out — i.e. Channel A never confirmed stdout data. + _phase1_timeout=120: must exceed outer timeout (60s) to prevent Phase 1 from + firing STALE before the outer guard when subprocess startup is slow under load. + completion_drain_timeout=0.5: 0.1s was too tight under xdist -n 4 load; the + event loop may not process the drain callback before pytest-timeout fires. + natural_exit_grace_seconds=0.1: script never exits naturally (time.sleep(3600)), + so shorten grace window to reduce total test time and avoid asyncio-waitpid + thread contention under CI load (default 3.0s grace + 3.0s kill = 6s total). """ session_dir = tmp_path / "session" session_dir.mkdir() @@ -213,17 +241,21 @@ async def test_data_confirmed_false_set_on_drain_timeout(self, tmp_path): result = await run_managed_async( [sys.executable, str(script), str(session_dir)], cwd=tmp_path, - timeout=30, + timeout=TimeoutTier.CHANNEL_B, session_log_dir=session_dir, completion_marker="%%ORDER_UP%%", - completion_drain_timeout=0.1, + completion_drain_timeout=0.5, + natural_exit_grace_seconds=0.1, _phase1_poll=0.01, _phase2_poll=0.05, + _session_id_timeout=0.01, + _phase1_timeout=120, ) assert result.termination == TerminationReason.COMPLETED assert result.channel_confirmation == ChannelConfirmation.CHANNEL_B + @pytest.mark.timeout(90) @pytest.mark.anyio async def test_data_confirmed_true_when_channel_a_wins(self, tmp_path): """Channel A (heartbeat) wins; data_confirmed must be True. @@ -237,7 +269,7 @@ async def test_data_confirmed_true_when_channel_a_wins(self, tmp_path): result = await run_managed_async( [sys.executable, str(script)], cwd=tmp_path, - timeout=30, + timeout=60, # No session_log_dir: Channel B cannot fire _heartbeat_poll=0.05, ) @@ -245,6 +277,7 @@ async def test_data_confirmed_true_when_channel_a_wins(self, tmp_path): assert result.termination == TerminationReason.COMPLETED assert result.channel_confirmation == ChannelConfirmation.CHANNEL_A + @pytest.mark.timeout(90) @pytest.mark.anyio async def test_channel_b_then_a_empty_result_data_confirmed_is_false(self, tmp_path): """Channel B fires (%%ORDER_UP%% in JSONL). @@ -261,13 +294,15 @@ async def test_channel_b_then_a_empty_result_data_confirmed_is_false(self, tmp_p result = await run_managed_async( [sys.executable, str(script), str(session_dir)], cwd=tmp_path, - timeout=30, + timeout=TimeoutTier.CHANNEL_B, session_log_dir=session_dir, completion_marker="%%ORDER_UP%%", completion_drain_timeout=2.0, _phase1_poll=0.01, _phase2_poll=0.05, _heartbeat_poll=0.05, + _session_id_timeout=0.01, + _phase1_timeout=120, ) assert result.termination == TerminationReason.COMPLETED assert ( @@ -275,6 +310,7 @@ async def test_channel_b_then_a_empty_result_data_confirmed_is_false(self, tmp_p ) # FAILS before fix: True +@pytest.mark.timeout(180) class TestChannelBFullPipelineAdjudication: """Full end-to-end adjudication for Channel B drain-race scenarios.""" @@ -289,9 +325,9 @@ async def test_channel_b_then_a_empty_result_produces_success(self, tmp_path): - completion_drain_timeout=0.5s: the heartbeat has already seen the empty result and failed to confirm by the time Channel B fires (~1s after task group start), so 0.5s of additional drain time is more than sufficient semantically. - - timeout=60s: guards against the outer wall-clock expiring under xdist -n 4 load. - Under heavy load the stdout_session_id_ready wait (1.0s) and inner drain (0.5s) - can each overrun 10x, giving a worst-case total of ~15s well inside 60s. + - timeout=TimeoutTier.CHANNEL_B (60s): guards against the outer wall-clock expiring under xdist -n 4 load. + Under heavy load the stdout_session_id_ready wait (_session_id_timeout, default 1.0s, tests pass 0.01s) + and inner drain (0.5s) can each overrun 10x, giving a worst-case total of ~15s well inside 60s. """ from autoskillit.execution.headless import _build_skill_result @@ -303,13 +339,16 @@ async def test_channel_b_then_a_empty_result_produces_success(self, tmp_path): result = await run_managed_async( [sys.executable, str(script), str(session_dir)], cwd=tmp_path, - timeout=60, + timeout=TimeoutTier.CHANNEL_B, session_log_dir=session_dir, completion_marker="%%ORDER_UP%%", completion_drain_timeout=0.5, + natural_exit_grace_seconds=0.5, + _phase1_timeout=120, _phase1_poll=0.01, _phase2_poll=0.05, _heartbeat_poll=0.05, + _session_id_timeout=0.5, ) skill_result = _build_skill_result( result, @@ -346,12 +385,14 @@ async def test_channel_b_drain_timeout_produces_success_skill_result(self, tmp_p result = await run_managed_async( [sys.executable, str(script), str(session_dir)], cwd=tmp_path, - timeout=30, + timeout=TimeoutTier.CHANNEL_B, session_log_dir=session_dir, completion_marker="%%ORDER_UP%%", completion_drain_timeout=0.5, + _phase1_timeout=120, _phase1_poll=0.01, _phase2_poll=0.05, + _session_id_timeout=0.01, ) assert result.termination == TerminationReason.COMPLETED @@ -402,6 +443,7 @@ def test_natural_exit_channel_b_empty_stdout_is_success(self): class TestPostExitDrainWindow: """Symmetric drain window: process exits first, Channel B gets a bounded window to deposit.""" + @pytest.mark.timeout(120) @pytest.mark.anyio async def test_drain_window_allows_channel_b_to_deposit(self, tmp_path): """Process exits before Phase 1 polls; drain window lets Channel B detect marker. @@ -413,7 +455,7 @@ async def test_drain_window_allows_channel_b_to_deposit(self, tmp_path): Timing rationale for completion_drain_timeout=30.0: - Before channel_b_ready can be set, _watch_session_log must: - 1. Wait up to 1.0s for stdout_session_id_ready (move_on_after(1.0)) + 1. Wait up to _session_id_timeout for stdout_session_id_ready (default 1.0s, tests pass 0.01s) 2. Sleep _phase1_poll=1.0s before Phase 1's first check 3. Sleep _phase2_poll=0.05s before Phase 2's first check Total minimum: ~2.05s under normal conditions. @@ -431,17 +473,20 @@ async def test_drain_window_allows_channel_b_to_deposit(self, tmp_path): result = await run_managed_async( [sys.executable, str(script), str(session_dir)], cwd=tmp_path, - timeout=60, + timeout=TimeoutTier.CHANNEL_B, session_log_dir=session_dir, completion_marker="%%ORDER_UP%%", completion_drain_timeout=30.0, + _phase1_timeout=120, _phase1_poll=1.0, _phase2_poll=0.05, _heartbeat_poll=0.05, + _session_id_timeout=0.01, ) assert result.channel_confirmation == ChannelConfirmation.CHANNEL_B + @pytest.mark.timeout(30) @pytest.mark.anyio async def test_drain_window_times_out_when_no_session_jsonl(self, tmp_path): """Process exits with no session JSONL; drain window times out, UNMONITORED preserved. @@ -474,6 +519,7 @@ async def test_drain_window_times_out_when_no_session_jsonl(self, tmp_path): _phase1_poll=0.05, _phase2_poll=0.05, _heartbeat_poll=0.05, + _session_id_timeout=0.01, ) assert result.channel_confirmation == ChannelConfirmation.UNMONITORED @@ -490,6 +536,8 @@ async def test_drain_window_times_out_when_no_session_jsonl(self, tmp_path): session_dir = sys.argv[1] unique_marker = sys.argv[2] os.makedirs(session_dir, exist_ok=True) + sys.stdout.write(json.dumps({"type": "system", "session_id": "session"}) + "\\n") + sys.stdout.flush() time.sleep(0.1) with open(os.path.join(session_dir, "session.jsonl"), "w") as f: # Sub-skill emits static marker — should NOT trigger completion @@ -517,6 +565,14 @@ class TestChannelBSubSkillCollision: @pytest.mark.anyio async def test_channel_b_ignores_sub_skill_marker(self, tmp_path): + """Channel B must not trigger on a sub-skill's static %%ORDER_UP%% marker. + + timeout=TimeoutTier.CHANNEL_B (60s): guards against the outer wall-clock + expiring under xdist -n 4 load. + _session_id_timeout=2.0 gives the stdout reader enough headroom under + heavy parallel load so Channel B monitoring always starts before the + JSONL markers are written. + """ session_dir = tmp_path / "session" session_dir.mkdir() unique_marker = "%%ORDER_UP::test1234%%" @@ -526,13 +582,15 @@ async def test_channel_b_ignores_sub_skill_marker(self, tmp_path): result = await run_managed_async( [sys.executable, str(script), str(session_dir), unique_marker], cwd=tmp_path, - timeout=15, + timeout=TimeoutTier.CHANNEL_B, session_log_dir=session_dir, completion_marker=unique_marker, - completion_drain_timeout=2.0, + completion_drain_timeout=5.0, + _phase1_timeout=120, _phase1_poll=0.05, _phase2_poll=0.05, _heartbeat_poll=0.05, + _session_id_timeout=5.0, ) assert result.termination == TerminationReason.COMPLETED diff --git a/tests/execution/test_process_deadline_extension.py b/tests/execution/test_process_deadline_extension.py new file mode 100644 index 0000000000..aadbae6336 --- /dev/null +++ b/tests/execution/test_process_deadline_extension.py @@ -0,0 +1,167 @@ +"""Tests for _watch_child_activity coroutine and deadline extension behavior.""" + +from __future__ import annotations + +import anyio +import pytest + +from autoskillit.execution.process._process_race import _watch_child_activity + +pytestmark = [pytest.mark.layer("execution"), pytest.mark.medium] + + +@pytest.mark.anyio +async def test_extends_deadline_when_children_active(monkeypatch) -> None: + """Deadline is extended when _has_active_child_processes returns True.""" + monkeypatch.setattr( + "autoskillit.execution.process._process_race._has_active_child_processes", + lambda pid: True, + ) + monkeypatch.setattr( + "autoskillit.execution.process._process_race._has_active_api_connection", + lambda pid: False, + ) + trigger = anyio.Event() + scope_ref: list[anyio.CancelScope | None] = [None] + original_deadline_ref: list[float] = [] + + async with anyio.create_task_group() as tg: + tg.start_soon(_watch_child_activity, 1, scope_ref, 7200.0, trigger, 0.05) + with anyio.move_on_after(0.1) as scope: + scope_ref[0] = scope + original_deadline_ref.append(scope.deadline) + await anyio.sleep(0.3) + trigger.set() + tg.cancel_scope.cancel() + + assert scope_ref[0] is not None + assert scope_ref[0].deadline > original_deadline_ref[0] + + +@pytest.mark.anyio +async def test_no_extension_when_inactive(monkeypatch) -> None: + """Deadline is NOT extended when both probes return False.""" + monkeypatch.setattr( + "autoskillit.execution.process._process_race._has_active_child_processes", + lambda pid: False, + ) + monkeypatch.setattr( + "autoskillit.execution.process._process_race._has_active_api_connection", + lambda pid: False, + ) + trigger = anyio.Event() + scope_ref: list[anyio.CancelScope | None] = [None] + original_deadline_ref: list[float] = [] + + async with anyio.create_task_group() as tg: + tg.start_soon(_watch_child_activity, 1, scope_ref, 7200.0, trigger, 0.05) + with anyio.move_on_after(2.0) as scope: + scope_ref[0] = scope + original_deadline_ref.append(scope.deadline) + await anyio.sleep(0.5) + trigger.set() + tg.cancel_scope.cancel() + + assert scope_ref[0] is not None + assert scope_ref[0].deadline == original_deadline_ref[0] + + +@pytest.mark.anyio +async def test_max_extension_cap_enforced(monkeypatch) -> None: + """Extension is capped at max_extension_seconds beyond original deadline.""" + monkeypatch.setattr( + "autoskillit.execution.process._process_race._has_active_child_processes", + lambda pid: True, + ) + monkeypatch.setattr( + "autoskillit.execution.process._process_race._has_active_api_connection", + lambda pid: False, + ) + trigger = anyio.Event() + scope_ref: list[anyio.CancelScope | None] = [None] + original_deadline_ref: list[float] = [] + + async with anyio.create_task_group() as tg: + tg.start_soon(_watch_child_activity, 1, scope_ref, 0.2, trigger, 0.05) + with anyio.move_on_after(0.1) as scope: + scope_ref[0] = scope + original_deadline_ref.append(scope.deadline) + await anyio.sleep(0.8) + trigger.set() + tg.cancel_scope.cancel() + + assert scope_ref[0] is not None + assert scope_ref[0].deadline <= original_deadline_ref[0] + 0.2 + + +@pytest.mark.anyio +async def test_terminates_on_trigger(monkeypatch) -> None: + """Watcher exits cleanly when trigger fires immediately.""" + monkeypatch.setattr( + "autoskillit.execution.process._process_race._has_active_child_processes", + lambda pid: True, + ) + monkeypatch.setattr( + "autoskillit.execution.process._process_race._has_active_api_connection", + lambda pid: True, + ) + trigger = anyio.Event() + scope_ref: list[anyio.CancelScope | None] = [None] + + trigger.set() + + with anyio.fail_after(2.0): + await _watch_child_activity(1, scope_ref, 7200.0, trigger, 0.05) + + +@pytest.mark.anyio +async def test_api_connection_also_extends(monkeypatch) -> None: + """Deadline is extended when _has_active_api_connection returns True (children inactive).""" + monkeypatch.setattr( + "autoskillit.execution.process._process_race._has_active_child_processes", + lambda pid: False, + ) + monkeypatch.setattr( + "autoskillit.execution.process._process_race._has_active_api_connection", + lambda pid: True, + ) + trigger = anyio.Event() + scope_ref: list[anyio.CancelScope | None] = [None] + original_deadline_ref: list[float] = [] + + async with anyio.create_task_group() as tg: + tg.start_soon(_watch_child_activity, 1, scope_ref, 7200.0, trigger, 0.05) + with anyio.move_on_after(0.1) as scope: + scope_ref[0] = scope + original_deadline_ref.append(scope.deadline) + await anyio.sleep(0.3) + trigger.set() + tg.cancel_scope.cancel() + + assert scope_ref[0] is not None + assert scope_ref[0].deadline > original_deadline_ref[0] + + +@pytest.mark.anyio +async def test_scope_ref_none_polling(monkeypatch) -> None: + """Watcher polls harmlessly when scope_ref is None (before scope binding).""" + monkeypatch.setattr( + "autoskillit.execution.process._process_race._has_active_child_processes", + lambda pid: True, + ) + monkeypatch.setattr( + "autoskillit.execution.process._process_race._has_active_api_connection", + lambda pid: True, + ) + trigger = anyio.Event() + scope_ref: list[anyio.CancelScope | None] = [None] # intentionally empty + + with anyio.fail_after(2.0): + # Run watcher with empty scope_ref for 3 poll cycles, then set scope and trigger + async with anyio.create_task_group() as tg: + tg.start_soon(_watch_child_activity, 1, scope_ref, 7200.0, trigger, 0.05) + await anyio.sleep(0.2) + with anyio.move_on_after(1.0) as scope: + scope_ref[0] = scope + trigger.set() + tg.cancel_scope.cancel() diff --git a/tests/execution/test_process_debug_logging.py b/tests/execution/test_process_debug_logging.py index 95cb3e648d..399109fc2c 100644 --- a/tests/execution/test_process_debug_logging.py +++ b/tests/execution/test_process_debug_logging.py @@ -10,6 +10,8 @@ from autoskillit.core.types import TerminationReason from autoskillit.execution.process import RaceAccumulator, RaceSignals +pytestmark = [pytest.mark.layer("execution"), pytest.mark.medium] + @pytest.mark.anyio async def test_run_managed_async_logs_entry(tmp_path): diff --git a/tests/execution/test_process_env_boundary.py b/tests/execution/test_process_env_boundary.py new file mode 100644 index 0000000000..bef50541c1 --- /dev/null +++ b/tests/execution/test_process_env_boundary.py @@ -0,0 +1,106 @@ +"""Boundary contract tests: MappingProxyType env through subprocess execution. + +Validates that run_managed_async() and run_managed_sync() coerce any +Mapping env to a plain dict before passing it to the external subprocess API, +so that uvloop's strict ``type(env) is dict`` check is satisfied. +""" + +from __future__ import annotations + +import os +import subprocess +from types import MappingProxyType +from typing import Any + +import anyio +import pytest + +from autoskillit.execution.process import run_managed_async, run_managed_sync + +pytestmark = [pytest.mark.layer("execution"), pytest.mark.medium] + + +@pytest.mark.anyio +async def test_mappingproxy_env_through_run_managed_async(tmp_path: Any) -> None: + """MappingProxyType env must not cause TypeError in run_managed_async.""" + env = MappingProxyType({"PATH": os.environ["PATH"]}) + result = await run_managed_async( + cmd=["echo", "ok"], + cwd=tmp_path, + timeout=10.0, + env=env, + ) + assert result.returncode == 0 + assert "ok" in result.stdout + + +@pytest.mark.anyio +async def test_run_managed_async_coerces_env_to_dict( + tmp_path: Any, monkeypatch: pytest.MonkeyPatch +) -> None: + """run_managed_async must coerce env to dict before anyio.open_process.""" + captured_env: dict[str, Any] = {} + + real_open_process = anyio.open_process + + async def spy_open_process(*args: Any, **kwargs: Any) -> Any: + captured_env["env"] = kwargs.get("env") + return await real_open_process(*args, **kwargs) + + monkeypatch.setattr("autoskillit.execution.process.anyio.open_process", spy_open_process) + env = MappingProxyType({"PATH": os.environ["PATH"], "FOO": "bar"}) + await run_managed_async( + cmd=["echo", "ok"], + cwd=tmp_path, + timeout=10.0, + env=env, + ) + assert type(captured_env["env"]) is dict + assert captured_env["env"]["FOO"] == "bar" + + +@pytest.mark.anyio +async def test_run_managed_async_none_env_passthrough(tmp_path: Any) -> None: + """env=None must remain None (not coerced to empty dict).""" + result = await run_managed_async( + cmd=["echo", "ok"], + cwd=tmp_path, + timeout=10.0, + env=None, + ) + assert result.returncode == 0 + + +def test_run_managed_sync_coerces_env_to_dict( + tmp_path: Any, monkeypatch: pytest.MonkeyPatch +) -> None: + """run_managed_sync must coerce env to dict before subprocess.Popen.""" + captured_env: dict[str, Any] = {} + + real_popen = subprocess.Popen + + class SpyPopen(real_popen): # type: ignore[misc] + def __init__(self, *args: Any, **kwargs: Any) -> None: + captured_env["env"] = kwargs.get("env") + super().__init__(*args, **kwargs) + + monkeypatch.setattr("subprocess.Popen", SpyPopen) + env = MappingProxyType({"PATH": os.environ["PATH"], "FOO": "bar"}) + run_managed_sync( + cmd=["echo", "ok"], + cwd=tmp_path, + timeout=10.0, + env=env, + ) + assert type(captured_env["env"]) is dict + assert captured_env["env"]["FOO"] == "bar" + + +def test_headless_cmd_with_mappingproxy_env() -> None: + """ClaudeHeadlessCmd must accept MappingProxyType env without error.""" + from autoskillit.execution.commands import ClaudeHeadlessCmd + + env = MappingProxyType({"PATH": "/usr/bin"}) + cmd = ClaudeHeadlessCmd(cmd=["echo", "test"], env=env) + assert cmd.env["PATH"] == "/usr/bin" + assert isinstance(cmd.env, MappingProxyType) diff --git a/tests/execution/test_process_heartbeat.py b/tests/execution/test_process_heartbeat.py new file mode 100644 index 0000000000..6176dd0e26 --- /dev/null +++ b/tests/execution/test_process_heartbeat.py @@ -0,0 +1,595 @@ +"""Unit tests for _heartbeat, _has_active_api_connection, _has_active_child_processes, +and orphaned tool result detection.""" + +from __future__ import annotations + +import sys +import time +from unittest.mock import MagicMock, patch + +import anyio +import psutil +import pytest + +from autoskillit.core.types import ChannelBStatus, TerminationReason +from autoskillit.execution.process import ( + _has_active_api_connection, + _has_active_child_processes, + _heartbeat, + _session_log_monitor, + run_managed_async, +) +from tests.execution.conftest import WRITE_RESULT_THEN_HANG_SCRIPT + +pytestmark = [pytest.mark.layer("execution"), pytest.mark.medium] + + +class TestHeartbeatDetectsCompletion: + """Stdout heartbeat detects completion and triggers kill.""" + + @pytest.mark.anyio + async def test_heartbeat_detects_completion_and_kills(self, tmp_path): + """Script writes result JSON then hangs — heartbeat detects and returns.""" + script = tmp_path / "result_hang.py" + script.write_text(WRITE_RESULT_THEN_HANG_SCRIPT) + + start = time.monotonic() + result = await run_managed_async( + [sys.executable, str(script)], + cwd=tmp_path, + timeout=30, + ) + elapsed = time.monotonic() - start + + assert result.termination != TerminationReason.TIMED_OUT, ( + "Heartbeat should fire before wall-clock timeout" + ) + assert elapsed < 10, f"Heartbeat should detect within ~5s, took {elapsed:.1f}s" + assert '"type": "result"' in result.stdout or '"type":"result"' in result.stdout + + @pytest.mark.anyio + async def test_heartbeat_ignores_non_matching_output(self, tmp_path): + """Script writes non-matching output — heartbeat doesn't fire, backstop does.""" + partial_output_then_hang_script = ( + "import sys, time\n" + 'sys.stdout.write("partial output\\n")\n' + "sys.stdout.flush()\n" + "time.sleep(3600)\n" + ) + script = tmp_path / "partial_hang.py" + script.write_text(partial_output_then_hang_script) + + result = await run_managed_async( + [sys.executable, str(script)], + cwd=tmp_path, + timeout=3, + ) + + assert result.termination == TerminationReason.TIMED_OUT, ( + "Non-matching output should not trigger heartbeat" + ) + + +class TestNoHeartbeatPreservesExistingBehavior: + """Without heartbeat, behavior matches original blind-wait.""" + + @pytest.mark.anyio + async def test_no_heartbeat_preserves_existing_behavior(self, tmp_path): + """No heartbeat marker — same hanging script, same timeout behavior.""" + import textwrap + + hang_script = textwrap.dedent("""\ + import sys, time + sys.stdout.write("before hang\\n") + sys.stdout.flush() + time.sleep(3600) + """) + script = tmp_path / "hang.py" + script.write_text(hang_script) + + result = await run_managed_async( + [sys.executable, str(script)], + cwd=tmp_path, + timeout=3, + ) + + assert result.termination == TerminationReason.TIMED_OUT + + +class TestHeartbeatStructuredParsing: + """Heartbeat uses structured parsing to avoid false-fires.""" + + @pytest.mark.anyio + async def test_heartbeat_ignores_marker_in_string_values(self, tmp_path): + """Heartbeat must not fire when the marker text appears as a string + value inside a non-result record (e.g., model discussing JSON formats). + """ + import json + + stdout_path = tmp_path / "stdout.tmp" + # Write an assistant message that CONTAINS the marker text as a value. + # Use separators=(",", ":") to match Claude CLI's compact JSON format. + assistant_msg = json.dumps( + { + "type": "assistant", + "message": {"content": 'The output format uses "type":"result" for completion'}, + }, + separators=(",", ":"), + ) + stdout_path.write_text(assistant_msg + "\n") + + poll_count = 0 + polls_done = anyio.Event() + + def on_poll() -> None: + nonlocal poll_count + poll_count += 1 + if poll_count >= 5: + polls_done.set() + + heartbeat_result: list[str] = [] + + async with anyio.create_task_group() as tg: + + async def _run_heartbeat() -> None: + heartbeat_result.append( + await _heartbeat( + stdout_path, '"type":"result"', _poll_interval=0.05, _on_poll=on_poll + ) + ) + + tg.start_soon(_run_heartbeat) + with anyio.fail_after(10.0): + await polls_done.wait() + assert not heartbeat_result, ( + "Heartbeat fired on non-result record containing marker text" + ) + + # Now write an actual result record (compact format matching Claude CLI) + result_record = json.dumps( + { + "type": "result", + "subtype": "success", + "is_error": False, + "result": "done", + "session_id": "s1", + }, + separators=(",", ":"), + ) + with stdout_path.open("a") as f: + f.write(result_record + "\n") + # task group awaits _run_heartbeat to detect result record and complete + + assert heartbeat_result[0] == "completion" + + +class TestHeartbeatTerminationReason: + """Heartbeat kill produces COMPLETED termination reason.""" + + @pytest.mark.anyio + async def test_heartbeat_kill_sets_completed_termination(self, tmp_path): + """When heartbeat detects result and kills the process, termination is COMPLETED.""" + script = tmp_path / "result_hang.py" + script.write_text(WRITE_RESULT_THEN_HANG_SCRIPT) + + result = await run_managed_async( + [sys.executable, str(script)], + cwd=tmp_path, + timeout=30, + _heartbeat_poll=0.05, + ) + + assert result.termination == TerminationReason.COMPLETED + + +class TestHasActiveApiConnection: + """Unit tests for _has_active_api_connection.""" + + def _make_conn(self, port: int, status: str = "ESTABLISHED"): + from unittest.mock import Mock + + conn = Mock() + conn.status = status + conn.raddr = Mock() + conn.raddr.port = port + return conn + + def _patch_psutil(self, parent_conns, child_conns_list=None): + """Returns a context manager patching psutil in process_lifecycle.""" + from unittest.mock import Mock + + mock_parent = Mock() + mock_parent.net_connections.return_value = parent_conns + children = [] + for child_conns in child_conns_list or []: + mock_child = Mock() + mock_child.net_connections.return_value = child_conns + children.append(mock_child) + mock_parent.children.return_value = children + return patch( + "autoskillit.execution.process._process_monitor.psutil.Process", + return_value=mock_parent, + ) + + def test_returns_true_when_parent_has_established_port_443(self): + with self._patch_psutil([self._make_conn(443)]): + assert _has_active_api_connection(12345) is True + + def test_returns_true_when_child_has_established_port_443(self): + with self._patch_psutil( + parent_conns=[self._make_conn(80)], + child_conns_list=[[self._make_conn(443)]], + ): + assert _has_active_api_connection(12345) is True + + def test_returns_false_when_no_connections(self): + with self._patch_psutil([]): + assert _has_active_api_connection(12345) is False + + def test_returns_false_when_all_connections_non_443(self): + conns = [self._make_conn(80), self._make_conn(8080), self._make_conn(22)] + with self._patch_psutil(conns): + assert _has_active_api_connection(12345) is False + + def test_returns_false_when_443_is_not_established(self): + conns = [ + self._make_conn(443, status="TIME_WAIT"), + self._make_conn(443, status="CLOSE_WAIT"), + ] + with self._patch_psutil(conns): + assert _has_active_api_connection(12345) is False + + def test_returns_false_when_no_raddr(self): + from unittest.mock import Mock + + conn = Mock() + conn.status = "ESTABLISHED" + conn.raddr = None + with self._patch_psutil([conn]): + assert _has_active_api_connection(12345) is False + + def test_returns_false_on_nosuchprocess(self): + import psutil as _psutil + + with patch( + "autoskillit.execution.process._process_monitor.psutil.Process", + side_effect=_psutil.NoSuchProcess(12345), + ): + assert _has_active_api_connection(12345) is False + + def test_skips_dead_child_gracefully(self): + from unittest.mock import Mock + + import psutil as _psutil + + mock_parent = Mock() + mock_parent.net_connections.return_value = [] + mock_dead_child = Mock() + mock_dead_child.net_connections.side_effect = _psutil.NoSuchProcess(99999) + mock_live_child = Mock() + mock_live_child.net_connections.return_value = [self._make_conn(443)] + mock_parent.children.return_value = [mock_dead_child, mock_live_child] + with patch( + "autoskillit.execution.process._process_monitor.psutil.Process", + return_value=mock_parent, + ): + assert _has_active_api_connection(12345) is True + + def test_skips_zombie_child_gracefully(self): + from unittest.mock import Mock + + import psutil as _psutil + + mock_parent = Mock() + mock_parent.net_connections.return_value = [] + mock_zombie = Mock() + mock_zombie.net_connections.side_effect = _psutil.ZombieProcess(99998) + mock_live_child = Mock() + mock_live_child.net_connections.return_value = [self._make_conn(443)] + mock_parent.children.return_value = [mock_zombie, mock_live_child] + with patch( + "autoskillit.execution.process._process_monitor.psutil.Process", + return_value=mock_parent, + ): + assert _has_active_api_connection(12345) is True + + +class TestHasActiveChildProcesses: + """Unit tests for _has_active_child_processes. + + The function caches psutil.Process objects so cpu_percent(interval=0) + returns meaningful deltas on the second call. First call primes the + baseline (always returns False); second call with the same child PIDs + uses the cached objects. + """ + + @pytest.fixture(autouse=True) + def _clear_cache(self, monkeypatch): + """Reset the module-level Process cache between tests.""" + from autoskillit.execution.process import _process_monitor + + monkeypatch.setattr(_process_monitor, "_child_process_cache", {}) + + def _make_child(self, cpu: float | type[Exception], *, pid: int = 999) -> MagicMock: + """Build a mock psutil child process.""" + child = MagicMock() + child.pid = pid + if isinstance(cpu, type) and issubclass(cpu, Exception): + child.cpu_percent.side_effect = cpu(pid=pid) + else: + child.cpu_percent.return_value = cpu + return child + + def _patch_children(self, children, monkeypatch, parent_raises=None): + mock_proc = MagicMock() + if parent_raises: + monkeypatch.setattr( + "autoskillit.execution.process._process_monitor.psutil.Process", + MagicMock(side_effect=parent_raises(pid=1234)), + ) + return + mock_proc.children.return_value = children + monkeypatch.setattr( + "autoskillit.execution.process._process_monitor.psutil.Process", + MagicMock(return_value=mock_proc), + ) + + def test_returns_true_when_child_exceeds_threshold(self, monkeypatch): + child = self._make_child(15.0, pid=100) + self._patch_children([child], monkeypatch) + # First call primes baseline; second call returns meaningful delta. + _has_active_child_processes(1234) + # Cache now holds the child object; second call uses cached.cpu_percent. + from autoskillit.execution.process._process_monitor import _child_process_cache + + _child_process_cache[100] = child + assert _has_active_child_processes(1234) is True + + def test_returns_false_when_all_children_below_threshold(self, monkeypatch): + children = [ + self._make_child(0.0, pid=100), + self._make_child(5.0, pid=101), + self._make_child(9.9, pid=102), + ] + self._patch_children(children, monkeypatch) + # Prime baseline + _has_active_child_processes(1234) + from autoskillit.execution.process._process_monitor import _child_process_cache + + for c in children: + _child_process_cache[c.pid] = c + assert _has_active_child_processes(1234) is False + + def test_returns_false_when_no_children(self, monkeypatch): + self._patch_children([], monkeypatch) + assert _has_active_child_processes(1234) is False + + def test_returns_false_on_parent_nosuchprocess(self, monkeypatch): + self._patch_children([], monkeypatch, parent_raises=psutil.NoSuchProcess) + assert _has_active_child_processes(1234) is False + + def test_skips_dead_child_gracefully(self, monkeypatch): + children = [ + self._make_child(psutil.NoSuchProcess, pid=100), + self._make_child(5.0, pid=101), + ] + self._patch_children(children, monkeypatch) + _has_active_child_processes(1234) + from autoskillit.execution.process._process_monitor import _child_process_cache + + for c in children: + _child_process_cache[c.pid] = c + assert _has_active_child_processes(1234) is False + + def test_skips_zombie_child_then_finds_active(self, monkeypatch): + children = [ + self._make_child(psutil.ZombieProcess, pid=100), + self._make_child(20.0, pid=101), + ] + self._patch_children(children, monkeypatch) + # Prime baseline + _has_active_child_processes(1234) + from autoskillit.execution.process._process_monitor import _child_process_cache + + for c in children: + _child_process_cache[c.pid] = c + assert _has_active_child_processes(1234) is True + + def test_skips_access_denied_gracefully(self, monkeypatch): + self._patch_children( + [self._make_child(psutil.AccessDenied, pid=100)], + monkeypatch, + ) + _has_active_child_processes(1234) + from autoskillit.execution.process._process_monitor import _child_process_cache + + _child_process_cache[100] = self._make_child(psutil.AccessDenied, pid=100) + assert _has_active_child_processes(1234) is False + + +class TestHeartbeatMarkerAwareness: + """_heartbeat respects completion_marker when configured.""" + + @pytest.mark.anyio + async def test_heartbeat_does_not_fire_without_marker_when_configured(self, tmp_path): + """Channel A must NOT fire on a result missing the marker when marker is configured.""" + import json + + stdout_path = tmp_path / "stdout.tmp" + result_record = json.dumps( + { + "type": "result", + "subtype": "success", + "is_error": False, + "result": "Partial output", + "session_id": "s1", + }, + separators=(",", ":"), + ) + stdout_path.write_text(result_record + "\n") + + with pytest.raises(TimeoutError): + with anyio.fail_after(0.3): + await _heartbeat( + stdout_path, + completion_marker="%%ORDER_UP%%", + _poll_interval=0.05, + ) + + @pytest.mark.anyio + async def test_heartbeat_fires_with_marker_when_configured(self, tmp_path): + """Channel A fires when the result contains the marker as a standalone line.""" + import json + + stdout_path = tmp_path / "stdout.tmp" + result_record = json.dumps( + { + "type": "result", + "subtype": "success", + "is_error": False, + "result": "Done.\n%%ORDER_UP%%", + "session_id": "s1", + }, + separators=(",", ":"), + ) + stdout_path.write_text(result_record + "\n") + + with anyio.fail_after(5.0): + result = await _heartbeat( + stdout_path, + completion_marker="%%ORDER_UP%%", + _poll_interval=0.05, + ) + assert result == "completion" + + +class TestHeartbeatScanPosition: + """_heartbeat uses byte-safe scan position — regression test for multi-byte content.""" + + @pytest.mark.anyio + async def test_heartbeat_detects_record_after_multibyte_content(self, tmp_path): + """Heartbeat correctly scans past multi-byte UTF-8 content to detect a result record. + + Regression test: ensures the byte-offset refactor (scan_pos tracks bytes, not chars) + does not break detection when prior content contains multi-byte characters. + """ + import json + + stdout_path = tmp_path / "stdout.tmp" + # Write an assistant message with multi-byte UTF-8 content (CJK characters), + # followed by a result record on a new line. + assistant_msg = json.dumps( + {"type": "assistant", "message": "こんにちは"}, + separators=(",", ":"), + ) + result_record = json.dumps( + { + "type": "result", + "subtype": "success", + "is_error": False, + "result": "done", + "session_id": "s1", + }, + separators=(",", ":"), + ) + stdout_path.write_text(assistant_msg + "\n" + result_record + "\n", encoding="utf-8") + + with anyio.fail_after(10): + result = await _heartbeat(stdout_path, '"type":"result"') + assert result == "completion" + + +class TestOrphanedToolResultDetection: + """Channel B detects orphaned tool results when last JSONL record is type=user.""" + + @pytest.mark.anyio + async def test_orphaned_tool_result_detected_when_last_record_is_user_type(self, tmp_path): + """STALE result has orphaned_tool_result=True when last JSONL record is type=user.""" + import json + + log_dir = tmp_path / "session_logs" + log_dir.mkdir() + spawn_time = time.time() - 1 + + session_file = log_dir / "sess_orphaned.jsonl" + session_file.write_text( + json.dumps({"type": "assistant", "message": {"content": "working..."}}) + + "\n" + + json.dumps({"type": "user", "message": {"content": [{"type": "tool_result"}]}}) + + "\n" + ) + + result = await _session_log_monitor( + log_dir, + "%%AUTOSKILLIT_COMPLETE%%", + stale_threshold=0.15, + spawn_time=spawn_time, + _phase1_poll=0.01, + _phase2_poll=0.04, + ) + + assert result.status == ChannelBStatus.STALE + assert result.orphaned_tool_result is True + + @pytest.mark.anyio + async def test_no_false_positive_when_last_record_is_assistant(self, tmp_path): + """STALE result has orphaned_tool_result=False when last JSONL record is type=assistant.""" + import json + + log_dir = tmp_path / "session_logs" + log_dir.mkdir() + spawn_time = time.time() - 1 + + session_file = log_dir / "sess_assistant.jsonl" + session_file.write_text( + json.dumps({"type": "user", "message": {"content": "do something"}}) + + "\n" + + json.dumps({"type": "assistant", "message": {"content": "thinking..."}}) + + "\n" + ) + + result = await _session_log_monitor( + log_dir, + "%%AUTOSKILLIT_COMPLETE%%", + stale_threshold=0.15, + spawn_time=spawn_time, + _phase1_poll=0.01, + _phase2_poll=0.04, + ) + + assert result.status == ChannelBStatus.STALE + assert result.orphaned_tool_result is False + + @pytest.mark.anyio + async def test_orphaned_false_on_completion(self, tmp_path): + """COMPLETION result always has orphaned_tool_result=False.""" + import json + + log_dir = tmp_path / "session_logs" + log_dir.mkdir() + spawn_time = time.time() - 1 + + session_file = log_dir / "sess_complete.jsonl" + session_file.write_text( + json.dumps({"type": "user", "message": {"content": "do something"}}) + + "\n" + + json.dumps( + { + "type": "assistant", + "message": {"content": "done\n%%AUTOSKILLIT_COMPLETE%%"}, + } + ) + + "\n" + ) + + result = await _session_log_monitor( + log_dir, + "%%AUTOSKILLIT_COMPLETE%%", + stale_threshold=5.0, + spawn_time=spawn_time, + _phase1_poll=0.01, + _phase2_poll=0.04, + ) + + assert result.status == ChannelBStatus.COMPLETION + assert result.orphaned_tool_result is False diff --git a/tests/execution/test_process_idle_watchdog.py b/tests/execution/test_process_idle_watchdog.py index 7ae6c34905..239a4fa3ab 100644 --- a/tests/execution/test_process_idle_watchdog.py +++ b/tests/execution/test_process_idle_watchdog.py @@ -9,11 +9,13 @@ import anyio import pytest -from autoskillit.execution._process_race import ( +from autoskillit.execution.process._process_race import ( RaceAccumulator, _watch_stdout_idle, ) +pytestmark = [pytest.mark.layer("execution"), pytest.mark.medium] + WRITE_BURST_THEN_STALL_SCRIPT = textwrap.dedent("""\ import sys, time, json for i in range(3): diff --git a/tests/execution/test_process_jsonl.py b/tests/execution/test_process_jsonl.py index 6f100420b1..401a11aed8 100644 --- a/tests/execution/test_process_jsonl.py +++ b/tests/execution/test_process_jsonl.py @@ -9,12 +9,17 @@ import json +import pytest + from autoskillit.execution.process import ( _jsonl_contains_marker, _jsonl_has_record_type, + _jsonl_last_record_type, _marker_is_standalone, ) +pytestmark = [pytest.mark.layer("execution"), pytest.mark.small] + class TestJsonlContainsMarker: """_jsonl_contains_marker performs structured record filtering.""" @@ -361,3 +366,71 @@ def test_different_session_markers_dont_collide(self): + "\n" ) assert not _jsonl_contains_marker(content, "%%ORDER_UP::bbb%%", frozenset({"assistant"})) + + +class TestJsonlLastRecordType: + """_jsonl_last_record_type returns the type of the last parseable JSONL record.""" + + def test_returns_last_type_in_content(self): + content = '{"type": "assistant", "message": {}}\n{"type": "user", "message": {}}\n' + assert _jsonl_last_record_type(content) == "user" + + def test_returns_none_for_empty_content(self): + assert _jsonl_last_record_type("") is None + + def test_skips_unparseable_lines(self): + content = '{"type": "assistant"}\nnot-json\n' + assert _jsonl_last_record_type(content) == "assistant" + + def test_ignores_records_without_type_field(self): + content = '{"other": "field"}\n{"type": "result"}\n' + assert _jsonl_last_record_type(content) == "result" + + +class TestJsonlContainsMarkerThinkingBlocks: + """_jsonl_contains_marker must not match markers inside thinking blocks.""" + + def test_marker_in_thinking_block_not_detected(self): + import json + + marker = "%%COMPLETE%%" + content = json.dumps( + { + "type": "assistant", + "message": { + "content": [ + {"type": "thinking", "thinking": f"I will emit {marker} when done."} + ] + }, + } + ) + assert not _jsonl_contains_marker(content, marker, frozenset({"assistant"})) + + def test_marker_in_text_block_is_detected(self): + import json + + marker = "%%COMPLETE%%" + content = json.dumps( + { + "type": "assistant", + "message": {"content": [{"type": "text", "text": marker}]}, + } + ) + assert _jsonl_contains_marker(content, marker, frozenset({"assistant"})) + + def test_marker_in_text_block_despite_thinking_block_present(self): + import json + + marker = "%%COMPLETE%%" + content = json.dumps( + { + "type": "assistant", + "message": { + "content": [ + {"type": "thinking", "thinking": "Internal reasoning."}, + {"type": "text", "text": marker}, + ] + }, + } + ) + assert _jsonl_contains_marker(content, marker, frozenset({"assistant"})) diff --git a/tests/execution/test_process_kill.py b/tests/execution/test_process_kill.py index a3c6cc1c34..8a03e8e231 100644 --- a/tests/execution/test_process_kill.py +++ b/tests/execution/test_process_kill.py @@ -24,6 +24,8 @@ run_managed_async, ) +pytestmark = [pytest.mark.layer("execution"), pytest.mark.medium] + # --------------------------------------------------------------------------- # Helper scripts — small Python programs that reproduce specific scenarios # --------------------------------------------------------------------------- @@ -220,7 +222,7 @@ async def test_pid_passed_to_session_monitor(self, tmp_path): captured = {} async def capturing_monitor(*args, **kwargs): - from autoskillit.execution._process_monitor import SessionMonitorResult + from autoskillit.execution.process._process_monitor import SessionMonitorResult captured["pid"] = kwargs.get("pid") captured["positional_pid"] = args[5] if len(args) > 5 else None @@ -229,7 +231,9 @@ async def capturing_monitor(*args, **kwargs): session_file = tmp_path / "fake_session.jsonl" session_file.write_text("") - with patch("autoskillit.execution._process_race._session_log_monitor", capturing_monitor): + with patch( + "autoskillit.execution.process._process_race._session_log_monitor", capturing_monitor + ): result = await run_managed_async( ["sleep", "5"], cwd=tmp_path, diff --git a/tests/execution/test_process_pty.py b/tests/execution/test_process_pty.py index 5ea77ce86e..393960dc0c 100644 --- a/tests/execution/test_process_pty.py +++ b/tests/execution/test_process_pty.py @@ -29,6 +29,9 @@ run_managed_async, ) from autoskillit.execution.session import ClaudeSessionResult +from tests.execution.conftest import _result_ndjson + +pytestmark = [pytest.mark.layer("execution"), pytest.mark.medium] # --------------------------------------------------------------------------- # Helper scripts — small Python programs that reproduce specific scenarios @@ -193,7 +196,7 @@ def test_pty_wrap_command_linux_uses_gnu_flags(self) -> None: cmd = ["claude", "--no-color", "do something"] fake_script = "/usr/bin/script" with ( - patch("autoskillit.execution._process_pty.sys.platform", "linux"), + patch("autoskillit.execution.process._process_pty.sys.platform", "linux"), patch("shutil.which", return_value=fake_script), ): result = pty_wrap_command(cmd) @@ -211,7 +214,7 @@ def test_pty_wrap_command_macos_uses_bsd_flags(self) -> None: cmd = ["claude", "--no-color", "do something"] fake_script = "/usr/bin/script" with ( - patch("autoskillit.execution._process_pty.sys.platform", "darwin"), + patch("autoskillit.execution.process._process_pty.sys.platform", "darwin"), patch("shutil.which", return_value=fake_script), ): result = pty_wrap_command(cmd) @@ -374,7 +377,7 @@ async def test_stale_with_valid_result_recovers_to_success(self, tmp_path): result = await run_managed_async( [sys.executable, str(script), str(session_dir)], cwd=tmp_path, - timeout=10, + timeout=20, session_log_dir=session_dir, completion_marker="%%NONEXISTENT%%", stale_threshold=0.3, @@ -491,25 +494,6 @@ def test_all_termination_reasons_have_integration_coverage(self): # --------------------------------------------------------------------------- -def _result_ndjson( - result: str = "done", - subtype: str = "success", - is_error: bool = False, - session_id: str = "s1", -) -> str: - """Build a minimal NDJSON string with a single type=result record.""" - return json.dumps( - { - "type": "result", - "subtype": subtype, - "is_error": is_error, - "result": result, - "session_id": session_id, - "errors": [], - } - ) - - class TestResolveTerminationMatrix: """Pure-function unit tests for resolve_termination. @@ -725,7 +709,7 @@ def test_channel_b_drain_race_recovery_empty_output( Monkeypatches parse_session_result to inject assistant_messages since an empty stdout produces no NDJSON records to accumulate from. """ - from autoskillit.execution import headless as headless_mod + import autoskillit.execution.headless._headless_result as headless_result_mod fake_session = ClaudeSessionResult( subtype=CliSubtype.EMPTY_OUTPUT, @@ -734,7 +718,7 @@ def test_channel_b_drain_race_recovery_empty_output( session_id="test", assistant_messages=["Done.\n%%ORDER_UP%%"], ) - monkeypatch.setattr(headless_mod, "parse_session_result", lambda _: fake_session) + monkeypatch.setattr(headless_result_mod, "parse_session_result", lambda _: fake_session) result = SubprocessResult( returncode=0, stdout="", @@ -833,7 +817,7 @@ def test_channel_a_empty_result_not_dead_end(self) -> None: """CHANNEL_A + empty result: dead-end guard escalates to retriable.""" result = SubprocessResult( returncode=0, - stdout=_result_ndjson(subtype="success", result=""), + stdout=_result_ndjson(subtype="success", result_text=""), stderr="", termination=TerminationReason.NATURAL_EXIT, pid=12345, @@ -856,7 +840,7 @@ def test_channel_b_max_turns_no_contradiction(self) -> None: """CHANNEL_B + error_max_turns: contradiction guard resolves to retriable.""" result = SubprocessResult( returncode=0, - stdout=_result_ndjson(subtype="error_max_turns", result="partial", is_error=True), + stdout=_result_ndjson(subtype="error_max_turns", result_text="partial", is_error=True), stderr="", termination=TerminationReason.NATURAL_EXIT, pid=12345, @@ -880,7 +864,7 @@ def test_completed_channel_b_max_turns_no_contradiction(self) -> None: """COMPLETED + CHANNEL_B + error_max_turns: contradiction guard resolves to retriable.""" result = SubprocessResult( returncode=-15, - stdout=_result_ndjson(subtype="error_max_turns", result="partial", is_error=True), + stdout=_result_ndjson(subtype="error_max_turns", result_text="partial", is_error=True), stderr="", termination=TerminationReason.COMPLETED, pid=12345, diff --git a/tests/execution/test_process_race.py b/tests/execution/test_process_race.py index 9543a4b9d0..d4f7495e1c 100644 --- a/tests/execution/test_process_race.py +++ b/tests/execution/test_process_race.py @@ -11,12 +11,14 @@ ChannelConfirmation, TerminationReason, ) -from autoskillit.execution._process_race import ( +from autoskillit.execution.process._process_race import ( RaceAccumulator, RaceSignals, resolve_termination, ) +pytestmark = [pytest.mark.layer("execution"), pytest.mark.medium] + class TestChannelBStatusExhaustiveCoverage: """Every ChannelBStatus member maps to a defined termination pair.""" @@ -244,12 +246,75 @@ class TestRaceSignalsFieldCount: """Sentinel test: breaks when RaceSignals fields change.""" def test_race_signals_field_count(self) -> None: - assert len(dataclasses.fields(RaceSignals)) == 8, ( - f"RaceSignals has {len(dataclasses.fields(RaceSignals))} fields (expected 8). " + assert len(dataclasses.fields(RaceSignals)) == 10, ( + f"RaceSignals has {len(dataclasses.fields(RaceSignals))} fields (expected 10). " "Update tests to cover the new field." ) +class TestExitSnapshot: + """exit_snapshot field on RaceAccumulator and RaceSignals.""" + + def test_exit_snapshot_defaults_to_none(self) -> None: + """RaceAccumulator.exit_snapshot defaults to None.""" + acc = RaceAccumulator() + assert acc.exit_snapshot is None + + def test_exit_snapshot_propagates_to_signals(self) -> None: + """exit_snapshot stored on accumulator is visible in RaceSignals.""" + acc = RaceAccumulator() + acc.exit_snapshot = {"event": "exit_snapshot", "vm_rss_kb": 1024} + signals = acc.to_race_signals() + assert signals.exit_snapshot is not None + assert signals.exit_snapshot["event"] == "exit_snapshot" + + @pytest.mark.anyio + async def test_watch_process_captures_exit_snapshot(self) -> None: + """_watch_process populates acc.exit_snapshot after process exits.""" + import sys + + import anyio + + from autoskillit.execution.process._process_race import _watch_process + + acc = RaceAccumulator() + trigger = anyio.Event() + async with await anyio.open_process( + [sys.executable, "-c", "import time; time.sleep(0.2)"], + start_new_session=True, + ) as proc: + async with anyio.create_task_group() as tg: + tg.start_soon(_watch_process, proc, acc, trigger) + await trigger.wait() + tg.cancel_scope.cancel() + + # exit_snapshot may be None if read_proc_snapshot failed (race — process gone) + # but the attribute must exist (not missing) + assert hasattr(acc, "exit_snapshot") + + @pytest.mark.anyio + async def test_watch_process_exit_snapshot_has_event_marker(self) -> None: + """If exit_snapshot was captured, it carries event='exit_snapshot'.""" + import sys + + import anyio + + from autoskillit.execution.process._process_race import _watch_process + + acc = RaceAccumulator() + trigger = anyio.Event() + async with await anyio.open_process( + [sys.executable, "-c", "pass"], # instant exit — maximises snapshot chance + start_new_session=True, + ) as proc: + async with anyio.create_task_group() as tg: + tg.start_soon(_watch_process, proc, acc, trigger) + await trigger.wait() + tg.cancel_scope.cancel() + + assert acc.exit_snapshot is None or acc.exit_snapshot.get("event") == "exit_snapshot" + + class TestProcessExitedEvent: """process_exited_event on RaceAccumulator / RaceSignals (1h).""" @@ -260,7 +325,7 @@ async def test_watch_process_sets_both_event_and_flag(self, tmp_path) -> None: import anyio - from autoskillit.execution._process_race import _watch_process + from autoskillit.execution.process._process_race import _watch_process acc = RaceAccumulator() trigger = anyio.Event() @@ -285,7 +350,7 @@ async def test_process_exited_event_fires_before_trigger(self, tmp_path) -> None import anyio - from autoskillit.execution._process_race import _watch_process + from autoskillit.execution.process._process_race import _watch_process acc = RaceAccumulator() trigger = anyio.Event() diff --git a/tests/execution/test_process_run.py b/tests/execution/test_process_run.py index f85eb9be89..55dc1c7733 100644 --- a/tests/execution/test_process_run.py +++ b/tests/execution/test_process_run.py @@ -26,6 +26,8 @@ run_managed_sync, ) +pytestmark = [pytest.mark.layer("execution"), pytest.mark.medium] + # --------------------------------------------------------------------------- # Helper scripts — small Python programs that reproduce specific scenarios # --------------------------------------------------------------------------- @@ -221,8 +223,8 @@ def _reset_structlog_config(self): """ import structlog - import autoskillit.execution._process_io as io_mod import autoskillit.execution.process as proc_mod + import autoskillit.execution.process._process_io as io_mod structlog.reset_defaults() current_procs = structlog.get_config()["processors"] @@ -294,8 +296,8 @@ async def test_tracing_stop_called_on_task_group_exception(self, monkeypatch, tm """tracing_handle.stop() is called in except BaseException even when task group raises.""" import subprocess - from autoskillit.config import LinuxTracingConfig from autoskillit.execution.linux_tracing import LinuxTracingHandle + from tests._helpers import make_tracing_config stop_called: list[bool] = [] original_stop = LinuxTracingHandle.stop @@ -308,7 +310,7 @@ def patched_stop(self_handle: LinuxTracingHandle) -> list: # Use a real process with tracing enabled; cancel mid-run to trigger BaseException path proc = subprocess.Popen(["sleep", "2"]) - cfg = LinuxTracingConfig(enabled=True, proc_interval=0.05, tmpfs_path=str(tmp_path)) + cfg = make_tracing_config(enabled=True, proc_interval=0.05, tmpfs_path=str(tmp_path)) import anyio @@ -382,7 +384,7 @@ async def test_run_managed_async_idle_stall_kills_hanging_process(self, tmp_path ) monkeypatch.setattr( - "autoskillit.execution._process_monitor._has_active_api_connection", + "autoskillit.execution.process._process_monitor._has_active_api_connection", lambda pid: True, ) diff --git a/tests/execution/test_process_monitor.py b/tests/execution/test_process_session_log_monitor.py similarity index 59% rename from tests/execution/test_process_monitor.py rename to tests/execution/test_process_session_log_monitor.py index 99d0349826..cd0ce0f7db 100644 --- a/tests/execution/test_process_monitor.py +++ b/tests/execution/test_process_session_log_monitor.py @@ -1,26 +1,21 @@ -"""Unit tests for _session_log_monitor, _heartbeat, and _has_active_api_connection.""" +"""Unit tests for _session_log_monitor and related session log monitoring behavior.""" from __future__ import annotations -import sys import time -from unittest.mock import MagicMock, patch +from unittest.mock import patch import anyio -import psutil import pytest -from autoskillit.core.types import ChannelBStatus, TerminationReason +from autoskillit.core.types import ChannelBStatus from autoskillit.execution.process import ( RaceAccumulator, - _has_active_api_connection, - _has_active_child_processes, - _heartbeat, _session_log_monitor, _watch_session_log, - run_managed_async, ) -from tests.execution.conftest import WRITE_RESULT_THEN_HANG_SCRIPT + +pytestmark = [pytest.mark.layer("execution"), pytest.mark.medium] # Script that writes non-matching output then hangs PARTIAL_OUTPUT_THEN_HANG_SCRIPT = ( @@ -52,7 +47,7 @@ async def test_session_log_monitor_detects_completion(self, tmp_path): ) async def append_marker(): - await anyio.sleep(1.0) + await anyio.sleep(0.5) with session_file.open("a") as f: f.write( json.dumps( @@ -72,7 +67,12 @@ async def append_marker(): async def _run_monitor() -> None: monitor_result.append( await _session_log_monitor( - log_dir, "%%AUTOSKILLIT_COMPLETE%%", stale_threshold=30, spawn_time=spawn_time + log_dir, + "%%AUTOSKILLIT_COMPLETE%%", + stale_threshold=30, + spawn_time=spawn_time, + _phase1_poll=0.01, + _phase2_poll=0.05, ) ) @@ -324,382 +324,6 @@ async def _run_monitor() -> None: assert monitor_result[0].status == "completion" -class TestHeartbeatDetectsCompletion: - """Stdout heartbeat detects completion and triggers kill.""" - - @pytest.mark.anyio - async def test_heartbeat_detects_completion_and_kills(self, tmp_path): - """Script writes result JSON then hangs — heartbeat detects and returns.""" - script = tmp_path / "result_hang.py" - script.write_text(WRITE_RESULT_THEN_HANG_SCRIPT) - - start = time.monotonic() - result = await run_managed_async( - [sys.executable, str(script)], - cwd=tmp_path, - timeout=30, - ) - elapsed = time.monotonic() - start - - assert result.termination != TerminationReason.TIMED_OUT, ( - "Heartbeat should fire before wall-clock timeout" - ) - assert elapsed < 10, f"Heartbeat should detect within ~5s, took {elapsed:.1f}s" - assert '"type": "result"' in result.stdout or '"type":"result"' in result.stdout - - @pytest.mark.anyio - async def test_heartbeat_ignores_non_matching_output(self, tmp_path): - """Script writes non-matching output — heartbeat doesn't fire, backstop does.""" - script = tmp_path / "partial_hang.py" - script.write_text(PARTIAL_OUTPUT_THEN_HANG_SCRIPT) - - result = await run_managed_async( - [sys.executable, str(script)], - cwd=tmp_path, - timeout=3, - ) - - assert result.termination == TerminationReason.TIMED_OUT, ( - "Non-matching output should not trigger heartbeat" - ) - - -class TestNoHeartbeatPreservesExistingBehavior: - """Without heartbeat, behavior matches original blind-wait.""" - - @pytest.mark.anyio - async def test_no_heartbeat_preserves_existing_behavior(self, tmp_path): - """No heartbeat marker — same hanging script, same timeout behavior.""" - import textwrap - - hang_script = textwrap.dedent("""\ - import sys, time - sys.stdout.write("before hang\\n") - sys.stdout.flush() - time.sleep(3600) - """) - script = tmp_path / "hang.py" - script.write_text(hang_script) - - result = await run_managed_async( - [sys.executable, str(script)], - cwd=tmp_path, - timeout=3, - ) - - assert result.termination == TerminationReason.TIMED_OUT - - -class TestHeartbeatStructuredParsing: - """Heartbeat uses structured parsing to avoid false-fires.""" - - @pytest.mark.anyio - async def test_heartbeat_ignores_marker_in_string_values(self, tmp_path): - """Heartbeat must not fire when the marker text appears as a string - value inside a non-result record (e.g., model discussing JSON formats). - """ - import json - - stdout_path = tmp_path / "stdout.tmp" - # Write an assistant message that CONTAINS the marker text as a value. - # Use separators=(",", ":") to match Claude CLI's compact JSON format. - assistant_msg = json.dumps( - { - "type": "assistant", - "message": {"content": 'The output format uses "type":"result" for completion'}, - }, - separators=(",", ":"), - ) - stdout_path.write_text(assistant_msg + "\n") - - poll_count = 0 - polls_done = anyio.Event() - - def on_poll() -> None: - nonlocal poll_count - poll_count += 1 - if poll_count >= 5: - polls_done.set() - - heartbeat_result: list[str] = [] - - async with anyio.create_task_group() as tg: - - async def _run_heartbeat() -> None: - heartbeat_result.append( - await _heartbeat( - stdout_path, '"type":"result"', _poll_interval=0.05, _on_poll=on_poll - ) - ) - - tg.start_soon(_run_heartbeat) - with anyio.fail_after(10.0): - await polls_done.wait() - assert not heartbeat_result, ( - "Heartbeat fired on non-result record containing marker text" - ) - - # Now write an actual result record (compact format matching Claude CLI) - result_record = json.dumps( - { - "type": "result", - "subtype": "success", - "is_error": False, - "result": "done", - "session_id": "s1", - }, - separators=(",", ":"), - ) - with stdout_path.open("a") as f: - f.write(result_record + "\n") - # task group awaits _run_heartbeat to detect result record and complete - - assert heartbeat_result[0] == "completion" - - -class TestHeartbeatTerminationReason: - """Heartbeat kill produces COMPLETED termination reason.""" - - @pytest.mark.anyio - async def test_heartbeat_kill_sets_completed_termination(self, tmp_path): - """When heartbeat detects result and kills the process, termination is COMPLETED.""" - script = tmp_path / "result_hang.py" - script.write_text(WRITE_RESULT_THEN_HANG_SCRIPT) - - result = await run_managed_async( - [sys.executable, str(script)], - cwd=tmp_path, - timeout=30, - _heartbeat_poll=0.05, - ) - - assert result.termination == TerminationReason.COMPLETED - - -class TestHasActiveApiConnection: - """Unit tests for _has_active_api_connection.""" - - def _make_conn(self, port: int, status: str = "ESTABLISHED"): - from unittest.mock import Mock - - conn = Mock() - conn.status = status - conn.raddr = Mock() - conn.raddr.port = port - return conn - - def _patch_psutil(self, parent_conns, child_conns_list=None): - """Returns a context manager patching psutil in process_lifecycle.""" - from unittest.mock import Mock - - mock_parent = Mock() - mock_parent.net_connections.return_value = parent_conns - children = [] - for child_conns in child_conns_list or []: - mock_child = Mock() - mock_child.net_connections.return_value = child_conns - children.append(mock_child) - mock_parent.children.return_value = children - return patch( - "autoskillit.execution._process_monitor.psutil.Process", return_value=mock_parent - ) - - def test_returns_true_when_parent_has_established_port_443(self): - with self._patch_psutil([self._make_conn(443)]): - assert _has_active_api_connection(12345) is True - - def test_returns_true_when_child_has_established_port_443(self): - with self._patch_psutil( - parent_conns=[self._make_conn(80)], - child_conns_list=[[self._make_conn(443)]], - ): - assert _has_active_api_connection(12345) is True - - def test_returns_false_when_no_connections(self): - with self._patch_psutil([]): - assert _has_active_api_connection(12345) is False - - def test_returns_false_when_all_connections_non_443(self): - conns = [self._make_conn(80), self._make_conn(8080), self._make_conn(22)] - with self._patch_psutil(conns): - assert _has_active_api_connection(12345) is False - - def test_returns_false_when_443_is_not_established(self): - conns = [ - self._make_conn(443, status="TIME_WAIT"), - self._make_conn(443, status="CLOSE_WAIT"), - ] - with self._patch_psutil(conns): - assert _has_active_api_connection(12345) is False - - def test_returns_false_when_no_raddr(self): - from unittest.mock import Mock - - conn = Mock() - conn.status = "ESTABLISHED" - conn.raddr = None - with self._patch_psutil([conn]): - assert _has_active_api_connection(12345) is False - - def test_returns_false_on_nosuchprocess(self): - import psutil as _psutil - - with patch( - "autoskillit.execution._process_monitor.psutil.Process", - side_effect=_psutil.NoSuchProcess(12345), - ): - assert _has_active_api_connection(12345) is False - - def test_skips_dead_child_gracefully(self): - from unittest.mock import Mock - - import psutil as _psutil - - mock_parent = Mock() - mock_parent.net_connections.return_value = [] - mock_dead_child = Mock() - mock_dead_child.net_connections.side_effect = _psutil.NoSuchProcess(99999) - mock_live_child = Mock() - mock_live_child.net_connections.return_value = [self._make_conn(443)] - mock_parent.children.return_value = [mock_dead_child, mock_live_child] - with patch( - "autoskillit.execution._process_monitor.psutil.Process", return_value=mock_parent - ): - assert _has_active_api_connection(12345) is True - - def test_skips_zombie_child_gracefully(self): - from unittest.mock import Mock - - import psutil as _psutil - - mock_parent = Mock() - mock_parent.net_connections.return_value = [] - mock_zombie = Mock() - mock_zombie.net_connections.side_effect = _psutil.ZombieProcess(99998) - mock_live_child = Mock() - mock_live_child.net_connections.return_value = [self._make_conn(443)] - mock_parent.children.return_value = [mock_zombie, mock_live_child] - with patch( - "autoskillit.execution._process_monitor.psutil.Process", return_value=mock_parent - ): - assert _has_active_api_connection(12345) is True - - -class TestHasActiveChildProcesses: - """Unit tests for _has_active_child_processes. - - The function caches psutil.Process objects so cpu_percent(interval=0) - returns meaningful deltas on the second call. First call primes the - baseline (always returns False); second call with the same child PIDs - uses the cached objects. - """ - - @pytest.fixture(autouse=True) - def _clear_cache(self, monkeypatch): - """Reset the module-level Process cache between tests.""" - from autoskillit.execution import _process_monitor - - monkeypatch.setattr(_process_monitor, "_child_process_cache", {}) - - def _make_child(self, cpu: float | type[Exception], *, pid: int = 999) -> MagicMock: - """Build a mock psutil child process.""" - child = MagicMock() - child.pid = pid - if isinstance(cpu, type) and issubclass(cpu, Exception): - child.cpu_percent.side_effect = cpu(pid=pid) - else: - child.cpu_percent.return_value = cpu - return child - - def _patch_children(self, children, monkeypatch, parent_raises=None): - mock_proc = MagicMock() - if parent_raises: - monkeypatch.setattr( - "autoskillit.execution._process_monitor.psutil.Process", - MagicMock(side_effect=parent_raises(pid=1234)), - ) - return - mock_proc.children.return_value = children - monkeypatch.setattr( - "autoskillit.execution._process_monitor.psutil.Process", - MagicMock(return_value=mock_proc), - ) - - def test_returns_true_when_child_exceeds_threshold(self, monkeypatch): - child = self._make_child(15.0, pid=100) - self._patch_children([child], monkeypatch) - # First call primes baseline; second call returns meaningful delta. - _has_active_child_processes(1234) - # Cache now holds the child object; second call uses cached.cpu_percent. - from autoskillit.execution._process_monitor import _child_process_cache - - _child_process_cache[100] = child - assert _has_active_child_processes(1234) is True - - def test_returns_false_when_all_children_below_threshold(self, monkeypatch): - children = [ - self._make_child(0.0, pid=100), - self._make_child(5.0, pid=101), - self._make_child(9.9, pid=102), - ] - self._patch_children(children, monkeypatch) - # Prime baseline - _has_active_child_processes(1234) - from autoskillit.execution._process_monitor import _child_process_cache - - for c in children: - _child_process_cache[c.pid] = c - assert _has_active_child_processes(1234) is False - - def test_returns_false_when_no_children(self, monkeypatch): - self._patch_children([], monkeypatch) - assert _has_active_child_processes(1234) is False - - def test_returns_false_on_parent_nosuchprocess(self, monkeypatch): - self._patch_children([], monkeypatch, parent_raises=psutil.NoSuchProcess) - assert _has_active_child_processes(1234) is False - - def test_skips_dead_child_gracefully(self, monkeypatch): - children = [ - self._make_child(psutil.NoSuchProcess, pid=100), - self._make_child(5.0, pid=101), - ] - self._patch_children(children, monkeypatch) - _has_active_child_processes(1234) - from autoskillit.execution._process_monitor import _child_process_cache - - for c in children: - if c.pid in _child_process_cache: - pass # Already cached by prime call - _child_process_cache[c.pid] = c - assert _has_active_child_processes(1234) is False - - def test_skips_zombie_child_then_finds_active(self, monkeypatch): - children = [ - self._make_child(psutil.ZombieProcess, pid=100), - self._make_child(20.0, pid=101), - ] - self._patch_children(children, monkeypatch) - # Prime baseline - _has_active_child_processes(1234) - from autoskillit.execution._process_monitor import _child_process_cache - - for c in children: - _child_process_cache[c.pid] = c - assert _has_active_child_processes(1234) is True - - def test_skips_access_denied_gracefully(self, monkeypatch): - self._patch_children( - [self._make_child(psutil.AccessDenied, pid=100)], - monkeypatch, - ) - _has_active_child_processes(1234) - from autoskillit.execution._process_monitor import _child_process_cache - - _child_process_cache[100] = self._make_child(psutil.AccessDenied, pid=100) - assert _has_active_child_processes(1234) is False - - class TestSessionLogMonitorStaleSuppressionGate: """_session_log_monitor suppresses stale when process has an active port-443 connection.""" @@ -721,7 +345,7 @@ def side_effect(pid): return call_count["n"] == 1 # True on first call, False on second with patch( - "autoskillit.execution._process_monitor._has_active_api_connection", + "autoskillit.execution.process._process_monitor._has_active_api_connection", side_effect=side_effect, ): with anyio.fail_after(5.0): @@ -763,7 +387,7 @@ async def test_fires_stale_when_pid_is_none_regardless_of_tcp(self, tmp_path): spawn_time = time.time() - 10 with patch( - "autoskillit.execution._process_monitor._has_active_api_connection" + "autoskillit.execution.process._process_monitor._has_active_api_connection" ) as mock_tcp: with anyio.fail_after(2.0): result = await _session_log_monitor( @@ -794,7 +418,7 @@ def side_effect(pid): return calls["n"] == 1 with patch( - "autoskillit.execution._process_monitor._has_active_api_connection", + "autoskillit.execution.process._process_monitor._has_active_api_connection", side_effect=side_effect, ): with structlog.testing.capture_logs() as logs: @@ -839,11 +463,11 @@ def fake_child_cpu(pid): return call_count["cpu"] == 1 # True first, False second monkeypatch.setattr( - "autoskillit.execution._process_monitor._has_active_api_connection", + "autoskillit.execution.process._process_monitor._has_active_api_connection", fake_api_conn, ) monkeypatch.setattr( - "autoskillit.execution._process_monitor._has_active_child_processes", + "autoskillit.execution.process._process_monitor._has_active_child_processes", fake_child_cpu, ) result = await _session_log_monitor( @@ -870,7 +494,7 @@ async def test_stale_suppression_bounded_by_max_duration(self, tmp_path, monkeyp spawn_time = time.time() - 10 monkeypatch.setattr( - "autoskillit.execution._process_monitor._has_active_api_connection", + "autoskillit.execution.process._process_monitor._has_active_api_connection", lambda pid: True, ) @@ -895,7 +519,7 @@ async def test_stale_suppression_resets_on_genuine_activity(self, tmp_path, monk spawn_time = time.time() - 10 monkeypatch.setattr( - "autoskillit.execution._process_monitor._has_active_api_connection", + "autoskillit.execution.process._process_monitor._has_active_api_connection", lambda pid: True, ) @@ -937,7 +561,7 @@ async def test_stale_suppression_logs_warning_on_bounded_kill( spawn_time = time.time() - 10 monkeypatch.setattr( - "autoskillit.execution._process_monitor._has_active_api_connection", + "autoskillit.execution.process._process_monitor._has_active_api_connection", lambda pid: True, ) @@ -960,98 +584,6 @@ async def test_stale_suppression_logs_warning_on_bounded_kill( assert bounded_in_logs or bounded_in_stdout -class TestHeartbeatMarkerAwareness: - """_heartbeat respects completion_marker when configured.""" - - @pytest.mark.anyio - async def test_heartbeat_does_not_fire_without_marker_when_configured(self, tmp_path): - """Channel A must NOT fire on a result missing the marker when marker is configured.""" - import json - - stdout_path = tmp_path / "stdout.tmp" - result_record = json.dumps( - { - "type": "result", - "subtype": "success", - "is_error": False, - "result": "Partial output", - "session_id": "s1", - }, - separators=(",", ":"), - ) - stdout_path.write_text(result_record + "\n") - - with pytest.raises(TimeoutError): - with anyio.fail_after(0.3): - await _heartbeat( - stdout_path, - completion_marker="%%ORDER_UP%%", - _poll_interval=0.05, - ) - - @pytest.mark.anyio - async def test_heartbeat_fires_with_marker_when_configured(self, tmp_path): - """Channel A fires when the result contains the marker as a standalone line.""" - import json - - stdout_path = tmp_path / "stdout.tmp" - result_record = json.dumps( - { - "type": "result", - "subtype": "success", - "is_error": False, - "result": "Done.\n%%ORDER_UP%%", - "session_id": "s1", - }, - separators=(",", ":"), - ) - stdout_path.write_text(result_record + "\n") - - with anyio.fail_after(5.0): - result = await _heartbeat( - stdout_path, - completion_marker="%%ORDER_UP%%", - _poll_interval=0.05, - ) - assert result == "completion" - - -class TestHeartbeatScanPosition: - """_heartbeat uses byte-safe scan position — regression test for multi-byte content.""" - - @pytest.mark.anyio - async def test_heartbeat_detects_record_after_multibyte_content(self, tmp_path): - """Heartbeat correctly scans past multi-byte UTF-8 content to detect a result record. - - Regression test: ensures the byte-offset refactor (scan_pos tracks bytes, not chars) - does not break detection when prior content contains multi-byte characters. - """ - import json - - stdout_path = tmp_path / "stdout.tmp" - # Write an assistant message with multi-byte UTF-8 content (CJK characters), - # followed by a result record on a new line. - assistant_msg = json.dumps( - {"type": "assistant", "message": "こんにちは"}, - separators=(",", ":"), - ) - result_record = json.dumps( - { - "type": "result", - "subtype": "success", - "is_error": False, - "result": "done", - "session_id": "s1", - }, - separators=(",", ":"), - ) - stdout_path.write_text(assistant_msg + "\n" + result_record + "\n", encoding="utf-8") - - with anyio.fail_after(10): - result = await _heartbeat(stdout_path, '"type":"result"') - assert result == "completion" - - class TestSessionLogMonitorSessionId: """_session_log_monitor returns SessionMonitorResult with session ID from filename.""" diff --git a/tests/execution/test_process_submodules.py b/tests/execution/test_process_submodules.py index dcfbfaebba..ef30e03d52 100644 --- a/tests/execution/test_process_submodules.py +++ b/tests/execution/test_process_submodules.py @@ -6,89 +6,152 @@ from __future__ import annotations +import inspect + +import pytest + +pytestmark = [pytest.mark.layer("execution"), pytest.mark.small] + +_EXPECTED_PROCESS_SYMBOLS: frozenset[str] = frozenset( + { + "DefaultSubprocessRunner", + "_extract_stdout_session_id", + "_resolve_session_id", + "RaceAccumulator", + "RaceSignals", + "_has_active_api_connection", + "_has_active_child_processes", + "_heartbeat", + "_jsonl_contains_marker", + "_jsonl_has_record_type", + "_jsonl_last_record_type", + "_marker_is_standalone", + "_session_log_monitor", + "_wait_process_dead", + "_watch_heartbeat", + "_watch_process", + "_watch_session_log", + "async_kill_process_tree", + "create_temp_io", + "decide_termination_action", + "execute_termination_action", + "kill_process_tree", + "pty_wrap_command", + "read_temp_output", + "resolve_termination", + "run_managed_async", + "run_managed_sync", + } +) + def test_process_kill_exports(): - """_process_kill.py exports kill/async_kill functions.""" - from autoskillit.execution._process_kill import async_kill_process_tree, kill_process_tree + """kill_process_tree and async_kill_process_tree are defined in _process_kill submodule.""" + from autoskillit.execution.process._process_kill import ( + async_kill_process_tree, + kill_process_tree, + ) assert callable(kill_process_tree) - assert kill_process_tree.__module__ == "autoskillit.execution._process_kill" + assert kill_process_tree.__module__ == "autoskillit.execution.process._process_kill" assert callable(async_kill_process_tree) - assert async_kill_process_tree.__module__ == "autoskillit.execution._process_kill" + assert async_kill_process_tree.__module__ == "autoskillit.execution.process._process_kill" def test_process_pty_exports(): - """_process_pty.py exports pty_wrap_command.""" - from autoskillit.execution._process_pty import pty_wrap_command + """pty_wrap_command is defined in _process_pty submodule.""" + from autoskillit.execution.process._process_pty import pty_wrap_command assert callable(pty_wrap_command) - assert pty_wrap_command.__module__ == "autoskillit.execution._process_pty" + assert pty_wrap_command.__module__ == "autoskillit.execution.process._process_pty" def test_process_jsonl_exports(): - """_process_jsonl.py exports JSONL parsing helpers.""" - from autoskillit.execution._process_jsonl import ( + """_jsonl_contains_marker, _jsonl_has_record_type, and _marker_is_standalone + are defined in _process_jsonl submodule.""" + from autoskillit.execution.process._process_jsonl import ( _jsonl_contains_marker, _jsonl_has_record_type, _marker_is_standalone, ) assert callable(_jsonl_contains_marker) - assert _jsonl_contains_marker.__module__ == "autoskillit.execution._process_jsonl" + assert _jsonl_contains_marker.__module__ == "autoskillit.execution.process._process_jsonl" assert callable(_jsonl_has_record_type) - assert _jsonl_has_record_type.__module__ == "autoskillit.execution._process_jsonl" + assert _jsonl_has_record_type.__module__ == "autoskillit.execution.process._process_jsonl" assert callable(_marker_is_standalone) - assert _marker_is_standalone.__module__ == "autoskillit.execution._process_jsonl" + assert _marker_is_standalone.__module__ == "autoskillit.execution.process._process_jsonl" def test_process_io_exports(): - """_process_io.py exports temp I/O helpers.""" - from autoskillit.execution._process_io import create_temp_io, read_temp_output + """create_temp_io and read_temp_output are defined in _process_io submodule.""" + from autoskillit.execution.process._process_io import create_temp_io, read_temp_output assert callable(create_temp_io) - assert create_temp_io.__module__ == "autoskillit.execution._process_io" + assert create_temp_io.__module__ == "autoskillit.execution.process._process_io" assert callable(read_temp_output) - assert read_temp_output.__module__ == "autoskillit.execution._process_io" + assert read_temp_output.__module__ == "autoskillit.execution.process._process_io" def test_process_monitor_exports(): - """_process_monitor.py exports monitoring functions.""" - from autoskillit.execution._process_monitor import ( + """_heartbeat, _session_log_monitor, and _has_active_api_connection + are defined in _process_monitor submodule.""" + from autoskillit.execution.process._process_monitor import ( _has_active_api_connection, _heartbeat, _session_log_monitor, ) assert callable(_heartbeat) - assert _heartbeat.__module__ == "autoskillit.execution._process_monitor" + assert _heartbeat.__module__ == "autoskillit.execution.process._process_monitor" assert callable(_session_log_monitor) - assert _session_log_monitor.__module__ == "autoskillit.execution._process_monitor" + assert _session_log_monitor.__module__ == "autoskillit.execution.process._process_monitor" assert callable(_has_active_api_connection) - assert _has_active_api_connection.__module__ == "autoskillit.execution._process_monitor" + assert ( + _has_active_api_connection.__module__ == "autoskillit.execution.process._process_monitor" + ) def test_process_race_exports(): - """_process_race.py exports race coordination types and functions.""" - from autoskillit.execution._process_race import ( + """Race types/functions are defined in _process_race submodule.""" + from autoskillit.execution.process._process_race import ( RaceAccumulator, RaceSignals, _watch_heartbeat, resolve_termination, ) - assert RaceAccumulator.__module__ == "autoskillit.execution._process_race" - assert RaceSignals.__module__ == "autoskillit.execution._process_race" + assert RaceAccumulator.__module__ == "autoskillit.execution.process._process_race" + assert RaceSignals.__module__ == "autoskillit.execution.process._process_race" assert callable(resolve_termination) - assert resolve_termination.__module__ == "autoskillit.execution._process_race" + assert resolve_termination.__module__ == "autoskillit.execution.process._process_race" assert callable(_watch_heartbeat) - assert _watch_heartbeat.__module__ == "autoskillit.execution._process_race" + assert _watch_heartbeat.__module__ == "autoskillit.execution.process._process_race" def test_process_facade_reexports_all_public_symbols(): - """process.py facade re-exports at least 21 public symbols.""" + """process.py facade re-exports exactly the expected 27 public symbols.""" from autoskillit.execution import process assert hasattr(process, "__all__") - assert len(process.__all__) >= 21, ( - f"process.py __all__ has {len(process.__all__)} symbols, expected at least 21" + assert set(process.__all__) == _EXPECTED_PROCESS_SYMBOLS, ( + f"process.__all__ mismatch.\n" + f" Extra : {set(process.__all__) - _EXPECTED_PROCESS_SYMBOLS}\n" + f" Missing : {_EXPECTED_PROCESS_SYMBOLS - set(process.__all__)}" ) + + +def test_default_subprocess_runner_pty_mode_default_false(): + + from autoskillit.execution.process import DefaultSubprocessRunner + + sig = inspect.signature(DefaultSubprocessRunner.__call__) + assert sig.parameters["pty_mode"].default is False + + +def test_run_managed_async_pty_mode_default_false(): + + from autoskillit.execution.process import run_managed_async + + sig = inspect.signature(run_managed_async) + assert sig.parameters["pty_mode"].default is False diff --git a/tests/execution/test_provider_outcome_container.py b/tests/execution/test_provider_outcome_container.py new file mode 100644 index 0000000000..2df76d5962 --- /dev/null +++ b/tests/execution/test_provider_outcome_container.py @@ -0,0 +1,38 @@ +"""Tests for ProviderOutcome typed container construction contract.""" + +from __future__ import annotations + +import pytest + +from autoskillit.core.types._type_results import ProviderOutcome + +pytestmark = [pytest.mark.layer("execution"), pytest.mark.small] + + +class TestProviderOutcomeConstruction: + """ProviderOutcome must require all fields — no silent defaults.""" + + def test_missing_provider_used_raises_type_error(self): + with pytest.raises(TypeError): + ProviderOutcome(fallback_activated=False) # type: ignore[call-arg] + + def test_missing_fallback_activated_raises_type_error(self): + with pytest.raises(TypeError): + ProviderOutcome(provider_used="minimax") # type: ignore[call-arg] + + def test_complete_construction_succeeds(self): + outcome = ProviderOutcome(provider_used="minimax", fallback_activated=False) + assert outcome.provider_used == "minimax" + assert outcome.fallback_activated is False + + def test_none_used_sentinel(self): + outcome = ProviderOutcome.none_used() + assert outcome.provider_used == "" + assert outcome.fallback_activated is False + + def test_frozen_rejects_mutation(self): + import dataclasses + + outcome = ProviderOutcome(provider_used="minimax", fallback_activated=False) + with pytest.raises(dataclasses.FrozenInstanceError): + outcome.provider_used = "other" # type: ignore[misc] diff --git a/tests/execution/test_push_trigger_applies.py b/tests/execution/test_push_trigger_applies.py new file mode 100644 index 0000000000..2754e5be4b --- /dev/null +++ b/tests/execution/test_push_trigger_applies.py @@ -0,0 +1,97 @@ +"""Unit tests for _push_trigger_applies_to_branch and _has_merge_group_trigger.""" + +import pytest + +from autoskillit.execution.merge_queue import ( + _has_merge_group_trigger, + _push_trigger_applies_to_branch, +) + +pytestmark = [pytest.mark.layer("execution"), pytest.mark.small] + + +def test_branch_filter_excludes_feature(): + text = "on:\n push:\n branches:\n - main\n - stable\n" + assert _push_trigger_applies_to_branch(text, "feature/foo") is False + + +def test_branch_filter_matches_listed_branch(): + text = "on:\n push:\n branches:\n - main\n - stable\n" + assert _push_trigger_applies_to_branch(text, "main") is True + + +def test_no_branch_filter_matches_all(): + text = "on:\n push:\n" + assert _push_trigger_applies_to_branch(text, "feature/anything") is True + + +def test_scalar_push_form_matches_all(): + assert _push_trigger_applies_to_branch("on: push\n", "feature/x") is True + + +def test_list_push_form_matches_all(): + assert _push_trigger_applies_to_branch("on: [push, pull_request]\n", "feature/x") is True + + +def test_branches_ignore_excludes_listed(): + text = "on:\n push:\n branches-ignore:\n - main\n - stable\n" + assert _push_trigger_applies_to_branch(text, "main") is False + + +def test_branches_ignore_allows_unlisted(): + text = "on:\n push:\n branches-ignore:\n - main\n - stable\n" + assert _push_trigger_applies_to_branch(text, "feature/foo") is True + + +def test_glob_pattern_matches(): + text = "on:\n push:\n branches:\n - 'feature/**'\n" + assert _push_trigger_applies_to_branch(text, "feature/impl-20260401") is True + + +def test_glob_pattern_excludes_non_matching(): + text = "on:\n push:\n branches:\n - 'feature/**'\n" + assert _push_trigger_applies_to_branch(text, "main") is False + + +def test_no_push_trigger_returns_false(): + text = "on:\n pull_request:\n branches: [main]\n" + assert _push_trigger_applies_to_branch(text, "main") is False + + +def test_yaml_parse_failure_falls_back_to_heuristic_safe(): + # Invalid YAML that doesn't contain any push trigger substrings + assert _push_trigger_applies_to_branch(": [\n", "main") is False + + +# --------------------------------------------------------------------------- +# _has_merge_group_trigger tests +# --------------------------------------------------------------------------- + + +def test_merge_group_in_on_list(): + assert _has_merge_group_trigger("on: [push, merge_group]") is True + + +def test_merge_group_as_dict_key(): + assert _has_merge_group_trigger("on:\n merge_group:\n push:") is True + + +def test_merge_group_scalar(): + assert _has_merge_group_trigger("on: merge_group") is True + + +def test_no_merge_group_trigger(): + assert _has_merge_group_trigger("on: [push, pull_request]") is False + + +def test_merge_group_in_comment_only(): + assert _has_merge_group_trigger("# merge_group\non: push") is False + + +def test_merge_group_in_shell_string(): + text = "on: push\njobs:\n test:\n steps:\n - run: echo merge_group" + assert _has_merge_group_trigger(text) is False + + +def test_yaml_parse_failure_falls_back_to_heuristic(): + assert _has_merge_group_trigger("{invalid yaml merge_group") is True diff --git a/tests/execution/test_quota.py b/tests/execution/test_quota.py deleted file mode 100644 index 54519be171..0000000000 --- a/tests/execution/test_quota.py +++ /dev/null @@ -1,1727 +0,0 @@ -"""Tests for execution/quota.py — credential reading, cache I/O, and check_and_sleep_if_needed.""" - -from __future__ import annotations - -import json -import time -from datetime import UTC, datetime, timedelta -from unittest.mock import patch - -import pytest - -from autoskillit.config.settings import QuotaGuardConfig - - -class TestReadCredentials: - def test_reads_access_token(self, tmp_path): - expires_ms = int(time.time() * 1000) + 3_600_000 - creds = {"claudeAiOauth": {"accessToken": "tok-abc", "expiresAt": expires_ms}} - path = tmp_path / ".credentials.json" - path.write_text(json.dumps(creds)) - from autoskillit.execution.quota import _read_credentials - - assert _read_credentials(str(path)) == "tok-abc" - - def test_expired_token_raises(self, tmp_path): - expires_ms = int(time.time() * 1000) - 1000 # already expired - creds = {"claudeAiOauth": {"accessToken": "tok-abc", "expiresAt": expires_ms}} - path = tmp_path / ".credentials.json" - path.write_text(json.dumps(creds)) - from autoskillit.execution.quota import _read_credentials - - with pytest.raises(PermissionError): - _read_credentials(str(path)) - - def test_missing_file_raises(self, tmp_path): - from autoskillit.execution.quota import _read_credentials - - with pytest.raises(FileNotFoundError): - _read_credentials(str(tmp_path / "nonexistent.json")) - - def test_missing_key_raises(self, tmp_path): - path = tmp_path / "bad.json" - path.write_text('{"other": "data"}') - from autoskillit.execution.quota import _read_credentials - - with pytest.raises(KeyError): - _read_credentials(str(path)) - - -class TestReadCache: - def test_fresh_cache_returns_status(self, tmp_path): - from autoskillit.execution.quota import _read_cache - - now = datetime.now(UTC) - cache_data = { - "schema_version": 3, - "fetched_at": (now - timedelta(seconds=30)).isoformat(), - "windows": { - "five_hour": { - "utilization": 87.3, - "resets_at": "2026-02-27T20:15:00+00:00", - } - }, - "binding": { - "window_name": "five_hour", - "utilization": 87.3, - "resets_at": "2026-02-27T20:15:00+00:00", - "should_block": True, - "effective_threshold": 85.0, - }, - } - cache_path = tmp_path / "usage_cache.json" - cache_path.write_text(json.dumps(cache_data)) - status = _read_cache(str(cache_path), max_age=120) - assert status is not None - assert status.utilization == pytest.approx(87.3) - - def test_stale_cache_returns_none(self, tmp_path): - from autoskillit.execution.quota import _read_cache - - now = datetime.now(UTC) - cache_data = { - "fetched_at": (now - timedelta(seconds=300)).isoformat(), - "five_hour": { - "utilization": 50.0, - "resets_at": "2026-02-27T20:15:00+00:00", - }, - } - cache_path = tmp_path / "usage_cache.json" - cache_path.write_text(json.dumps(cache_data)) - assert _read_cache(str(cache_path), max_age=120) is None - - def test_missing_cache_returns_none(self, tmp_path): - from autoskillit.execution.quota import _read_cache - - assert _read_cache(str(tmp_path / "nonexistent.json"), max_age=120) is None - - def test_corrupted_cache_returns_none(self, tmp_path): - from autoskillit.execution.quota import _read_cache - - cache_path = tmp_path / "usage_cache.json" - cache_path.write_text("not valid json{{") - assert _read_cache(str(cache_path), max_age=120) is None - - def test_read_cache_null_utilization_returns_none(self, tmp_path): - """Cache with utilization: null in binding must return None (fail-open).""" - from autoskillit.execution.quota import _read_cache - - cache_path = tmp_path / "usage_cache.json" - payload = { - "fetched_at": datetime.now(UTC).isoformat(), - "binding": {"utilization": None, "resets_at": None, "window_name": "five_hour"}, - } - cache_path.write_text(json.dumps(payload)) - assert _read_cache(str(cache_path), max_age=300) is None - - def test_read_cache_typeerror_returns_none(self, tmp_path): - """_read_cache must catch TypeError (e.g., float(None)) and return None.""" - from autoskillit.execution.quota import _read_cache - - cache_path = tmp_path / "usage_cache.json" - payload = { - "fetched_at": datetime.now(UTC).isoformat(), - "binding": {"utilization": None, "resets_at": None}, - } - cache_path.write_text(json.dumps(payload)) - # With __post_init__, constructing QuotaStatus with None raises TypeError. - # _read_cache must catch it and return None. - assert _read_cache(str(cache_path), max_age=300) is None - - -class TestQuotaDataclassPostInit: - """QuotaStatus and QuotaWindowEntry must enforce types at construction.""" - - def test_quota_window_entry_rejects_none_utilization(self): - from autoskillit.execution.quota import QuotaWindowEntry - - with pytest.raises(TypeError, match="utilization"): - QuotaWindowEntry(utilization=None, resets_at=None) - - def test_quota_status_rejects_none_utilization(self): - from autoskillit.execution.quota import QuotaStatus - - with pytest.raises(TypeError, match="utilization"): - QuotaStatus(utilization=None, resets_at=None) - - def test_quota_window_entry_coerces_string_utilization(self): - from autoskillit.execution.quota import QuotaWindowEntry - - entry = QuotaWindowEntry(utilization="85.5", resets_at=None) - assert entry.utilization == 85.5 - assert isinstance(entry.utilization, float) - - def test_quota_status_coerces_string_utilization(self): - from autoskillit.execution.quota import QuotaStatus - - status = QuotaStatus(utilization="42.0", resets_at=None) - assert status.utilization == 42.0 - assert isinstance(status.utilization, float) - - -class TestWriteCache: - def test_writes_readable_cache(self, tmp_path): - from autoskillit.execution.quota import ( - QuotaFetchResult, - QuotaStatus, - QuotaWindowEntry, - _read_cache, - _write_cache, - ) - - resets_at = datetime(2026, 2, 27, 20, 15, tzinfo=UTC) - result = QuotaFetchResult( - windows={"five_hour": QuotaWindowEntry(utilization=75.0, resets_at=resets_at)}, - binding=QuotaStatus(utilization=75.0, resets_at=resets_at, window_name="five_hour"), - ) - cache_path = tmp_path / "usage_cache.json" - _write_cache(str(cache_path), result) - recovered = _read_cache(str(cache_path), max_age=60) - assert recovered is not None - assert recovered.utilization == pytest.approx(75.0) - - def test_write_failure_does_not_raise(self, tmp_path): - from autoskillit.execution.quota import ( - QuotaFetchResult, - QuotaStatus, - QuotaWindowEntry, - _write_cache, - ) - - resets_at = datetime(2026, 2, 27, 20, 15, tzinfo=UTC) - result = QuotaFetchResult( - windows={"five_hour": QuotaWindowEntry(utilization=50.0, resets_at=resets_at)}, - binding=QuotaStatus(utilization=50.0, resets_at=resets_at, window_name="five_hour"), - ) - # Nonexistent parent directory — should log warning, not raise - _write_cache("/nonexistent/dir/cache.json", result) - - -class TestCheckAndSleepIfNeeded: - @pytest.mark.anyio - async def test_disabled_returns_immediately_no_io(self, monkeypatch): - from autoskillit.config.settings import QuotaGuardConfig - from autoskillit.execution.quota import check_and_sleep_if_needed - - config = QuotaGuardConfig(enabled=False) - fetch_called = [] - - async def mock_fetch(*a, **kw): - fetch_called.append(1) - raise AssertionError("should not fetch") - - monkeypatch.setattr("autoskillit.execution.quota._fetch_quota", mock_fetch) - result = await check_and_sleep_if_needed(config) - assert result["should_sleep"] is False - assert fetch_called == [] - - @pytest.mark.anyio - async def test_below_threshold_returns_should_sleep_false(self, monkeypatch, tmp_path): - from autoskillit.config.settings import QuotaGuardConfig - from autoskillit.execution.quota import ( - QuotaFetchResult, - QuotaStatus, - QuotaWindowEntry, - check_and_sleep_if_needed, - ) - - resets_at = datetime.now(UTC) + timedelta(hours=2) - config = QuotaGuardConfig( - enabled=True, - credentials_path=str(tmp_path / ".credentials.json"), - cache_path=str(tmp_path / "cache.json"), - ) - - async def mock_fetch(path, **kwargs): - return QuotaFetchResult( - windows={"five_hour": QuotaWindowEntry(utilization=50.0, resets_at=resets_at)}, - binding=QuotaStatus( - utilization=50.0, resets_at=resets_at, window_name="five_hour" - ), - ) - - monkeypatch.setattr("autoskillit.execution.quota._fetch_quota", mock_fetch) - result = await check_and_sleep_if_needed(config) - assert result["should_sleep"] is False - assert result["utilization"] == pytest.approx(50.0) - - @pytest.mark.anyio - async def test_above_threshold_returns_should_sleep_true(self, monkeypatch, tmp_path): - from unittest.mock import AsyncMock - - from autoskillit.config.settings import QuotaGuardConfig - from autoskillit.execution.quota import ( - QuotaFetchResult, - QuotaStatus, - QuotaWindowEntry, - check_and_sleep_if_needed, - ) - - resets_at = datetime.now(UTC) + timedelta(hours=2) - config = QuotaGuardConfig( - enabled=True, - buffer_seconds=0, - credentials_path=str(tmp_path / ".credentials.json"), - cache_path=str(tmp_path / "cache.json"), - ) - - def _make_result(util: float) -> QuotaFetchResult: - return QuotaFetchResult( - windows={"five_hour": QuotaWindowEntry(utilization=util, resets_at=resets_at)}, - binding=QuotaStatus( - utilization=util, - resets_at=resets_at, - window_name="five_hour", - should_block=util >= 85.0, - effective_threshold=85.0, - ), - ) - - mock_fetch = AsyncMock(side_effect=[_make_result(90.0), _make_result(91.0)]) - monkeypatch.setattr("autoskillit.execution.quota._fetch_quota", mock_fetch) - result = await check_and_sleep_if_needed(config) - assert result["should_sleep"] is True - assert mock_fetch.call_count == 2 - assert result["sleep_seconds"] == pytest.approx(7200, abs=60) - - @pytest.mark.anyio - async def test_uses_fresh_cache_skips_fetch(self, monkeypatch, tmp_path): - from autoskillit.config.settings import QuotaGuardConfig - from autoskillit.execution.quota import ( - QuotaFetchResult, - QuotaStatus, - QuotaWindowEntry, - _write_cache, - check_and_sleep_if_needed, - ) - - resets_at = datetime.now(UTC) + timedelta(hours=1) - cache_path = tmp_path / "cache.json" - _write_cache( - str(cache_path), - QuotaFetchResult( - windows={"five_hour": QuotaWindowEntry(utilization=40.0, resets_at=resets_at)}, - binding=QuotaStatus( - utilization=40.0, resets_at=resets_at, window_name="five_hour" - ), - ), - ) - config = QuotaGuardConfig( - enabled=True, - cache_max_age=120, - credentials_path=str(tmp_path / ".credentials.json"), - cache_path=str(cache_path), - ) - fetch_called = [] - - async def mock_fetch(path): - fetch_called.append(1) - raise AssertionError("should not fetch when cache is fresh") - - monkeypatch.setattr("autoskillit.execution.quota._fetch_quota", mock_fetch) - result = await check_and_sleep_if_needed(config) - assert fetch_called == [] - assert result["should_sleep"] is False - assert result["utilization"] == pytest.approx(40.0) - - @pytest.mark.anyio - async def test_credentials_failure_returns_error_dict(self, tmp_path): - from autoskillit.config.settings import QuotaGuardConfig - from autoskillit.execution.quota import check_and_sleep_if_needed - - config = QuotaGuardConfig( - enabled=True, - credentials_path=str(tmp_path / "nonexistent.json"), - cache_path=str(tmp_path / "cache.json"), - ) - result = await check_and_sleep_if_needed(config) - assert result["should_sleep"] is False - assert "error" in result - - @pytest.mark.anyio - async def test_network_error_returns_error_dict(self, monkeypatch, tmp_path): - from autoskillit.config.settings import QuotaGuardConfig - from autoskillit.execution.quota import check_and_sleep_if_needed - - config = QuotaGuardConfig( - enabled=True, - credentials_path=str(tmp_path / ".credentials.json"), - cache_path=str(tmp_path / "cache.json"), - ) - expires_ms = int(time.time() * 1000) + 3_600_000 - (tmp_path / ".credentials.json").write_text( - json.dumps({"claudeAiOauth": {"accessToken": "tok", "expiresAt": expires_ms}}) - ) - - async def mock_fetch(path, **kwargs): - raise OSError("network down") - - monkeypatch.setattr("autoskillit.execution.quota._fetch_quota", mock_fetch) - result = await check_and_sleep_if_needed(config) - assert result["should_sleep"] is False - assert "error" in result - - @pytest.mark.anyio - async def test_resets_at_none_after_refetch_logs_warning_with_fallback( - self, monkeypatch, tmp_path - ): - """Second resets_at-is-None guard (after re-fetch) must emit - the 'blocking with fallback' warning.""" - from unittest.mock import AsyncMock - - import structlog.testing - - from autoskillit.config.settings import QuotaGuardConfig - from autoskillit.execution.quota import ( - QuotaFetchResult, - QuotaStatus, - QuotaWindowEntry, - check_and_sleep_if_needed, - ) - - config = QuotaGuardConfig( - enabled=True, - credentials_path=str(tmp_path / ".credentials.json"), - cache_path=str(tmp_path / "cache.json"), - ) - resets = datetime.now(UTC) + timedelta(hours=1) - # First fetch: above threshold, has resets_at so Gate 1 passes - first_result = QuotaFetchResult( - windows={"five_hour": QuotaWindowEntry(utilization=90.0, resets_at=resets)}, - binding=QuotaStatus( - utilization=90.0, - resets_at=resets, - window_name="five_hour", - should_block=True, - effective_threshold=85.0, - ), - ) - # Second fetch (re-fetch): above threshold, resets_at is None → Gate 2 fires - second_result = QuotaFetchResult( - windows={"five_hour": QuotaWindowEntry(utilization=90.0, resets_at=None)}, - binding=QuotaStatus( - utilization=90.0, - resets_at=None, - window_name="five_hour", - should_block=True, - effective_threshold=85.0, - ), - ) - - mock_fetch = AsyncMock(side_effect=[first_result, second_result]) - monkeypatch.setattr("autoskillit.execution.quota._fetch_quota", mock_fetch) - - with structlog.testing.capture_logs() as cap: - result = await check_and_sleep_if_needed(config) - - assert result["should_sleep"] is True - assert result["sleep_seconds"] > 0 - assert mock_fetch.call_count == 2 - # Exact message required by the rectify plan - assert any( - "quota above threshold but resets_at is None after re-fetch — blocking with fallback" - in rec.get("event", "") - for rec in cap - ) - - -class TestCheckAndSleepResetAtNoneBlocks: - @pytest.mark.anyio - async def test_above_threshold_resets_at_none_first_fetch_blocks(self, monkeypatch, tmp_path): - from autoskillit.config.settings import QuotaGuardConfig - from autoskillit.execution.quota import ( - QuotaFetchResult, - QuotaStatus, - QuotaWindowEntry, - check_and_sleep_if_needed, - ) - - config = QuotaGuardConfig( - enabled=True, - buffer_seconds=60, - credentials_path=str(tmp_path / ".credentials.json"), - cache_path=str(tmp_path / "cache.json"), - ) - - async def mock_fetch(path, **kwargs): - return QuotaFetchResult( - windows={"five_hour": QuotaWindowEntry(utilization=90.0, resets_at=None)}, - binding=QuotaStatus( - utilization=90.0, - resets_at=None, - window_name="five_hour", - should_block=True, - effective_threshold=85.0, - ), - ) - - monkeypatch.setattr("autoskillit.execution.quota._fetch_quota", mock_fetch) - result = await check_and_sleep_if_needed(config) - assert result["should_sleep"] is True - assert result["sleep_seconds"] > 0 - - @pytest.mark.anyio - async def test_above_threshold_resets_at_none_second_fetch_blocks(self, monkeypatch, tmp_path): - from unittest.mock import AsyncMock - - from autoskillit.config.settings import QuotaGuardConfig - from autoskillit.execution.quota import ( - QuotaFetchResult, - QuotaStatus, - QuotaWindowEntry, - check_and_sleep_if_needed, - ) - - resets_at = datetime.now(UTC) + timedelta(hours=2) - config = QuotaGuardConfig( - enabled=True, - buffer_seconds=60, - credentials_path=str(tmp_path / ".credentials.json"), - cache_path=str(tmp_path / "cache.json"), - ) - # First fetch has resets_at (passes first None guard), second fetch has None - first_result = QuotaFetchResult( - windows={"five_hour": QuotaWindowEntry(utilization=90.0, resets_at=resets_at)}, - binding=QuotaStatus( - utilization=90.0, - resets_at=resets_at, - window_name="five_hour", - should_block=True, - effective_threshold=85.0, - ), - ) - second_result = QuotaFetchResult( - windows={"five_hour": QuotaWindowEntry(utilization=90.0, resets_at=None)}, - binding=QuotaStatus( - utilization=90.0, - resets_at=None, - window_name="five_hour", - should_block=True, - effective_threshold=85.0, - ), - ) - mock_fetch = AsyncMock(side_effect=[first_result, second_result]) - monkeypatch.setattr("autoskillit.execution.quota._fetch_quota", mock_fetch) - result = await check_and_sleep_if_needed(config) - assert result["should_sleep"] is True - assert result["sleep_seconds"] > 0 - - @pytest.mark.anyio - async def test_cache_hit_resets_at_none_above_threshold_blocks(self, monkeypatch, tmp_path): - from autoskillit.config.settings import QuotaGuardConfig - from autoskillit.execution.quota import ( - QuotaFetchResult, - QuotaStatus, - QuotaWindowEntry, - _write_cache, - check_and_sleep_if_needed, - ) - - cache_path = tmp_path / "cache.json" - # Write a cache entry with resets_at=None and above-threshold utilization - _write_cache( - str(cache_path), - QuotaFetchResult( - windows={"five_hour": QuotaWindowEntry(utilization=90.0, resets_at=None)}, - binding=QuotaStatus( - utilization=90.0, - resets_at=None, - window_name="five_hour", - should_block=True, - effective_threshold=85.0, - ), - ), - ) - config = QuotaGuardConfig( - enabled=True, - buffer_seconds=60, - cache_max_age=120, - credentials_path=str(tmp_path / ".credentials.json"), - cache_path=str(cache_path), - ) - fetch_called = [] - - async def mock_fetch(path): - fetch_called.append(1) - raise AssertionError("should not reach re-fetch when first branch blocks") - - monkeypatch.setattr("autoskillit.execution.quota._fetch_quota", mock_fetch) - result = await check_and_sleep_if_needed(config) - assert result["should_sleep"] is True - assert fetch_called == [] - - @pytest.mark.anyio - async def test_fallback_sleep_uses_at_least_buffer_seconds(self, monkeypatch, tmp_path): - from autoskillit.config.settings import QuotaGuardConfig - from autoskillit.execution.quota import ( - QuotaFetchResult, - QuotaStatus, - QuotaWindowEntry, - check_and_sleep_if_needed, - ) - - config = QuotaGuardConfig( - enabled=True, - buffer_seconds=120, - credentials_path=str(tmp_path / ".credentials.json"), - cache_path=str(tmp_path / "cache.json"), - ) - - async def mock_fetch(path, **kwargs): - return QuotaFetchResult( - windows={"five_hour": QuotaWindowEntry(utilization=90.0, resets_at=None)}, - binding=QuotaStatus( - utilization=90.0, - resets_at=None, - window_name="five_hour", - should_block=True, - effective_threshold=85.0, - ), - ) - - monkeypatch.setattr("autoskillit.execution.quota._fetch_quota", mock_fetch) - result = await check_and_sleep_if_needed(config) - assert result["should_sleep"] is True - assert result["sleep_seconds"] >= 120 - - @pytest.mark.anyio - async def test_above_threshold_with_buffer_seconds_default(self, monkeypatch, tmp_path): - from unittest.mock import AsyncMock - - from autoskillit.config.settings import QuotaGuardConfig - from autoskillit.execution.quota import ( - QuotaFetchResult, - QuotaStatus, - QuotaWindowEntry, - check_and_sleep_if_needed, - ) - - resets_at = datetime.now(UTC) + timedelta(hours=2) - # Do NOT override buffer_seconds — exercises the real default (60) - config = QuotaGuardConfig( - enabled=True, - credentials_path=str(tmp_path / ".credentials.json"), - cache_path=str(tmp_path / "cache.json"), - ) - - def _r(util: float) -> QuotaFetchResult: - return QuotaFetchResult( - windows={"five_hour": QuotaWindowEntry(utilization=util, resets_at=resets_at)}, - binding=QuotaStatus( - utilization=util, - resets_at=resets_at, - window_name="five_hour", - should_block=util >= 85.0, - effective_threshold=85.0, - ), - ) - - mock_fetch = AsyncMock(side_effect=[_r(90.0), _r(90.0)]) - monkeypatch.setattr("autoskillit.execution.quota._fetch_quota", mock_fetch) - result = await check_and_sleep_if_needed(config) - assert result["should_sleep"] is True - assert result["sleep_seconds"] > 0 - - -class TestIntegration: - """Integration tests: write/read contract between execution.quota and hooks.quota_guard.""" - - def test_write_cache_then_quota_check_main_reads_it(self, tmp_path, monkeypatch): - """Cache written by _write_cache must be readable and actionable by quota_guard.main(). - - T-INT-1: Catches format drift between _write_cache (execution layer) and - _read_quota_cache (hook subprocess layer). - """ - import io - from contextlib import redirect_stdout - - from autoskillit.execution.quota import ( - QuotaFetchResult, - QuotaStatus, - QuotaWindowEntry, - _write_cache, - ) - from autoskillit.hooks.quota_guard import main - - cache_path = tmp_path / "quota_cache.json" - _write_cache( - str(cache_path), - QuotaFetchResult( - windows={"five_hour": QuotaWindowEntry(utilization=95.0, resets_at=None)}, - binding=QuotaStatus( - utilization=95.0, - resets_at=None, - window_name="five_hour", - should_block=True, - effective_threshold=85.0, - ), - ), - ) - - stdin_text = json.dumps({"tool_name": "run_skill"}) - buf = io.StringIO() - with patch("sys.stdin", io.StringIO(stdin_text)): - with redirect_stdout(buf): - try: - main(cache_path_override=str(cache_path)) - except SystemExit: - pass - - out = buf.getvalue() - data = json.loads(out) - assert data["hookSpecificOutput"]["permissionDecision"] == "deny" - - -class TestMultiWindowSelection: - """Verifies _compute_binding selects the worst-case window and full-snapshot cache I/O.""" - - # T-MW-1: one_hour exhausted, five_hour fine → binding = one_hour - def test_binding_window_one_hour_exhausted(self): - from autoskillit.execution.quota import QuotaWindowEntry, _compute_binding - - now = datetime.now(UTC) - resets_one_hour = now + timedelta(minutes=45) - windows = { - "one_hour": QuotaWindowEntry(utilization=91.0, resets_at=resets_one_hour), - "five_hour": QuotaWindowEntry(utilization=35.0, resets_at=now + timedelta(hours=4)), - } - binding = _compute_binding( - windows, - short_threshold=85.0, - long_threshold=98.0, - long_patterns=["weekly", "sonnet", "opus"], - ) - assert binding.utilization == pytest.approx(91.0) - assert binding.resets_at == resets_one_hour - assert binding.window_name == "one_hour" - - # T-MW-2: multiple windows exhausted → binding = window with latest resets_at - def test_binding_window_latest_resets_at_governs(self): - from autoskillit.execution.quota import QuotaWindowEntry, _compute_binding - - now = datetime.now(UTC) - resets_one_hour = now + timedelta(minutes=45) - resets_one_day = now + timedelta(hours=18) - windows = { - "one_hour": QuotaWindowEntry(utilization=91.0, resets_at=resets_one_hour), - "one_day": QuotaWindowEntry(utilization=97.0, resets_at=resets_one_day), - "five_hour": QuotaWindowEntry(utilization=35.0, resets_at=now + timedelta(hours=4)), - } - binding = _compute_binding( - windows, - short_threshold=85.0, - long_threshold=98.0, - long_patterns=["weekly", "sonnet", "opus"], - ) - assert binding.window_name == "one_day" - assert binding.resets_at == resets_one_day - - # T-MW-3: all windows fine → should_sleep=False, binding = highest utilization - def test_all_windows_fine_binding_is_highest_utilization(self): - from autoskillit.execution.quota import QuotaWindowEntry, _compute_binding - - now = datetime.now(UTC) - five_hour_resets = now + timedelta(hours=4) - windows = { - "one_hour": QuotaWindowEntry(utilization=40.0, resets_at=now + timedelta(hours=1)), - "five_hour": QuotaWindowEntry(utilization=60.0, resets_at=five_hour_resets), - "one_day": QuotaWindowEntry(utilization=30.0, resets_at=now + timedelta(days=1)), - } - binding = _compute_binding( - windows, - short_threshold=85.0, - long_threshold=98.0, - long_patterns=["weekly", "sonnet", "opus"], - ) - assert binding.utilization == pytest.approx(60.0) - assert binding.window_name == "five_hour" - assert binding.resets_at == five_hour_resets - - # T-MW-3b: empty windows dict → returns zero-utilization sentinel, no ValueError - def test_empty_windows_returns_zero_sentinel(self): - from autoskillit.execution.quota import _compute_binding - - binding = _compute_binding( - {}, - short_threshold=85.0, - long_threshold=98.0, - long_patterns=["weekly", "sonnet", "opus"], - ) - assert binding.utilization == pytest.approx(0.0) - assert binding.resets_at is None - - # T-MW-4: _write_cache stores full windows dict + binding key - def test_write_cache_stores_all_windows(self, tmp_path): - from autoskillit.execution.quota import ( - QuotaFetchResult, - QuotaStatus, - QuotaWindowEntry, - _write_cache, - ) - - now = datetime.now(UTC) - resets = now + timedelta(hours=1) - windows = { - "one_hour": QuotaWindowEntry(utilization=91.0, resets_at=resets), - "five_hour": QuotaWindowEntry(utilization=35.0, resets_at=now + timedelta(hours=4)), - } - binding = QuotaStatus(utilization=91.0, resets_at=resets, window_name="one_hour") - result = QuotaFetchResult(windows=windows, binding=binding) - cache_path = tmp_path / "cache.json" - _write_cache(str(cache_path), result) - data = json.loads(cache_path.read_text()) - assert "windows" in data - assert "one_hour" in data["windows"] - assert "five_hour" in data["windows"] - assert "binding" in data - assert data["binding"]["window_name"] == "one_hour" - assert data["binding"]["utilization"] == pytest.approx(91.0) - - # T-MW-5: _read_cache with old-format cache (missing "binding") returns None - def test_read_cache_old_format_returns_none(self, tmp_path): - from autoskillit.execution.quota import _read_cache - - old_cache = { - "fetched_at": datetime.now(UTC).isoformat(), - "five_hour": {"utilization": 87.0, "resets_at": None}, - } - cache_path = tmp_path / "old_cache.json" - cache_path.write_text(json.dumps(old_cache)) - result = _read_cache(str(cache_path), max_age=120) - assert result is None - - # T-MW-6: _read_cache with new format returns QuotaStatus from binding - def test_read_cache_new_format_returns_binding(self, tmp_path): - from autoskillit.execution.quota import _read_cache - - new_cache = { - "schema_version": 3, - "fetched_at": datetime.now(UTC).isoformat(), - "windows": { - "one_hour": { - "utilization": 91.0, - "resets_at": "2026-04-10T09:45:00+00:00", - }, - "five_hour": { - "utilization": 35.0, - "resets_at": "2026-04-10T13:00:00+00:00", - }, - }, - "binding": { - "window_name": "one_hour", - "utilization": 91.0, - "resets_at": "2026-04-10T09:45:00+00:00", - "should_block": True, - "effective_threshold": 85.0, - }, - } - cache_path = tmp_path / "new_cache.json" - cache_path.write_text(json.dumps(new_cache)) - status = _read_cache(str(cache_path), max_age=120) - assert status is not None - assert status.utilization == pytest.approx(91.0) - assert status.window_name == "one_hour" - - -class TestPerWindowThresholds: - """Per-window threshold classification: short windows block at 85%, long at 98%.""" - - _LONG_PATTERNS = ["weekly", "sonnet", "opus"] - - def test_threshold_for_window_short_default(self): - from autoskillit.execution.quota import _threshold_for_window - - assert ( - _threshold_for_window( - "five_hour", - short_threshold=85.0, - long_threshold=98.0, - long_patterns=self._LONG_PATTERNS, - ) - == 85.0 - ) - - def test_threshold_for_window_long_weekly(self): - from autoskillit.execution.quota import _threshold_for_window - - assert ( - _threshold_for_window( - "weekly", - short_threshold=85.0, - long_threshold=98.0, - long_patterns=self._LONG_PATTERNS, - ) - == 98.0 - ) - - def test_threshold_for_window_long_sonnet_substring(self): - from autoskillit.execution.quota import _threshold_for_window - - assert ( - _threshold_for_window( - "weekly_sonnet", - short_threshold=85.0, - long_threshold=98.0, - long_patterns=self._LONG_PATTERNS, - ) - == 98.0 - ) - - def test_threshold_for_window_case_insensitive(self): - from autoskillit.execution.quota import _threshold_for_window - - assert ( - _threshold_for_window( - "Weekly", - short_threshold=85.0, - long_threshold=98.0, - long_patterns=self._LONG_PATTERNS, - ) - == 98.0 - ) - - def test_threshold_for_window_unknown_uses_short(self): - from autoskillit.execution.quota import _threshold_for_window - - assert ( - _threshold_for_window( - "context", - short_threshold=85.0, - long_threshold=98.0, - long_patterns=self._LONG_PATTERNS, - ) - == 85.0 - ) - - def test_compute_binding_picks_short_when_only_short_exhausted(self): - from autoskillit.execution.quota import QuotaWindowEntry, _compute_binding - - now = datetime.now(UTC) - windows = { - "five_hour": QuotaWindowEntry(utilization=90.0, resets_at=now + timedelta(hours=1)), - "weekly": QuotaWindowEntry(utilization=80.0, resets_at=now + timedelta(days=4)), - } - binding = _compute_binding( - windows, - short_threshold=85.0, - long_threshold=98.0, - long_patterns=self._LONG_PATTERNS, - ) - assert binding.window_name == "five_hour" - assert binding.should_block is True - assert binding.effective_threshold == pytest.approx(85.0) - - def test_compute_binding_does_not_block_weekly_below_long_threshold(self): - """Regression test for issue #721: weekly at 86% must not block. - - Long-window quotas (weekly, sonnet, opus) reset across multi-day windows, - so 14% remaining headroom is comfortable, not exhausted. The pre-fix - behaviour blocked the pipeline for ~4 days at 86% weekly. - """ - from autoskillit.execution.quota import QuotaWindowEntry, _compute_binding - - now = datetime.now(UTC) - windows = { - "weekly": QuotaWindowEntry(utilization=86.0, resets_at=now + timedelta(days=4)), - "five_hour": QuotaWindowEntry(utilization=50.0, resets_at=now + timedelta(hours=1)), - } - binding = _compute_binding( - windows, - short_threshold=85.0, - long_threshold=98.0, - long_patterns=self._LONG_PATTERNS, - ) - assert binding.should_block is False - assert binding.effective_threshold == pytest.approx(98.0) - - def test_compute_binding_blocks_weekly_at_99_percent(self): - from autoskillit.execution.quota import QuotaWindowEntry, _compute_binding - - now = datetime.now(UTC) - windows = { - "weekly": QuotaWindowEntry(utilization=99.0, resets_at=now + timedelta(days=4)), - } - binding = _compute_binding( - windows, - short_threshold=85.0, - long_threshold=98.0, - long_patterns=self._LONG_PATTERNS, - ) - assert binding.should_block is True - assert binding.effective_threshold == pytest.approx(98.0) - - def test_compute_binding_picks_latest_resets_among_exhausted(self): - """Among exhausted windows, the one with the latest reset wins.""" - from autoskillit.execution.quota import QuotaWindowEntry, _compute_binding - - now = datetime.now(UTC) - weekly_resets = now + timedelta(days=4) - windows = { - "five_hour": QuotaWindowEntry(utilization=90.0, resets_at=now + timedelta(hours=1)), - "weekly": QuotaWindowEntry(utilization=99.0, resets_at=weekly_resets), - } - binding = _compute_binding( - windows, - short_threshold=85.0, - long_threshold=98.0, - long_patterns=self._LONG_PATTERNS, - ) - assert binding.window_name == "weekly" - assert binding.resets_at == weekly_resets - assert binding.should_block is True - - def test_write_cache_includes_should_block_and_effective_threshold(self, tmp_path): - from autoskillit.execution.quota import ( - QuotaFetchResult, - QuotaStatus, - QuotaWindowEntry, - _write_cache, - ) - - now = datetime.now(UTC) - weekly_resets = now + timedelta(days=4) - windows = { - "weekly": QuotaWindowEntry(utilization=86.0, resets_at=weekly_resets), - } - binding = QuotaStatus( - utilization=86.0, - resets_at=weekly_resets, - window_name="weekly", - should_block=False, - effective_threshold=98.0, - ) - result = QuotaFetchResult(windows=windows, binding=binding) - cache_path = tmp_path / "cache.json" - _write_cache(str(cache_path), result) - data = json.loads(cache_path.read_text()) - assert data["binding"]["should_block"] is False - assert data["binding"]["effective_threshold"] == pytest.approx(98.0) - - def test_read_cache_round_trip_should_block(self, tmp_path): - from autoskillit.execution.quota import ( - QuotaFetchResult, - QuotaStatus, - QuotaWindowEntry, - _read_cache, - _write_cache, - ) - - now = datetime.now(UTC) - weekly_resets = now + timedelta(days=4) - windows = { - "weekly": QuotaWindowEntry(utilization=99.0, resets_at=weekly_resets), - } - binding = QuotaStatus( - utilization=99.0, - resets_at=weekly_resets, - window_name="weekly", - should_block=True, - effective_threshold=98.0, - ) - cache_path = tmp_path / "cache.json" - _write_cache(str(cache_path), QuotaFetchResult(windows=windows, binding=binding)) - status = _read_cache(str(cache_path), max_age=300) - assert status is not None - assert status.should_block is True - assert status.effective_threshold == pytest.approx(98.0) - assert status.window_name == "weekly" - - @pytest.mark.anyio - async def test_check_and_sleep_returns_false_for_weekly_at_86_percent( - self, monkeypatch, tmp_path - ): - """End-to-end regression test for #721 — weekly at 86% must not sleep.""" - from autoskillit.execution.quota import ( - QuotaFetchResult, - QuotaWindowEntry, - _compute_binding, - check_and_sleep_if_needed, - ) - - now = datetime.now(UTC) - windows = { - "weekly": QuotaWindowEntry(utilization=86.0, resets_at=now + timedelta(days=4)), - "five_hour": QuotaWindowEntry(utilization=2.0, resets_at=now + timedelta(hours=1)), - } - binding = _compute_binding( - windows, - short_threshold=85.0, - long_threshold=98.0, - long_patterns=self._LONG_PATTERNS, - ) - - async def fake_fetch(credentials_path, **kwargs): - return QuotaFetchResult(windows=windows, binding=binding) - - monkeypatch.setattr("autoskillit.execution.quota._fetch_quota", fake_fetch) - config = QuotaGuardConfig( - cache_path=str(tmp_path / "cache.json"), - credentials_path=str(tmp_path / "creds.json"), - ) - result = await check_and_sleep_if_needed(config) - assert result["should_sleep"] is False - assert result["window_name"] == "weekly" - - @pytest.mark.anyio - async def test_check_and_sleep_returns_true_for_weekly_at_99_percent( - self, monkeypatch, tmp_path - ): - from autoskillit.execution.quota import ( - QuotaFetchResult, - QuotaWindowEntry, - _compute_binding, - check_and_sleep_if_needed, - ) - - now = datetime.now(UTC) - weekly_resets = now + timedelta(days=4) - windows = { - "weekly": QuotaWindowEntry(utilization=99.0, resets_at=weekly_resets), - "five_hour": QuotaWindowEntry(utilization=2.0, resets_at=now + timedelta(hours=1)), - } - binding = _compute_binding( - windows, - short_threshold=85.0, - long_threshold=98.0, - long_patterns=self._LONG_PATTERNS, - ) - - async def fake_fetch(credentials_path, **kwargs): - return QuotaFetchResult(windows=windows, binding=binding) - - monkeypatch.setattr("autoskillit.execution.quota._fetch_quota", fake_fetch) - config = QuotaGuardConfig( - cache_path=str(tmp_path / "cache.json"), - credentials_path=str(tmp_path / "creds.json"), - ) - result = await check_and_sleep_if_needed(config) - assert result["should_sleep"] is True - assert result["window_name"] == "weekly" - assert result["sleep_seconds"] > 0 - - -class TestRefreshQuotaCache: - """Tests for _refresh_quota_cache: unconditional fetch-and-write behavior.""" - - @pytest.mark.anyio - async def test_refresh_quota_cache_fetches_even_when_cache_is_fresh( - self, tmp_path, monkeypatch - ): - """_refresh_quota_cache calls _fetch_quota unconditionally, even if cache is fresh.""" - from autoskillit.execution.quota import ( - QuotaFetchResult, - QuotaStatus, - QuotaWindowEntry, - _refresh_quota_cache, - ) - - fresh_cache = tmp_path / "cache.json" - # Write a 10-second-old cache (well within 300s max_age) - fresh_cache.write_text( - json.dumps( - { - "fetched_at": (datetime.now(UTC) - timedelta(seconds=10)).isoformat(), - "five_hour": {"utilization": 0.3, "resets_at": None}, - } - ) - ) - fetch_called = [] - - async def fake_fetch(credentials_path, **kwargs): - fetch_called.append(True) - return QuotaFetchResult( - windows={"five_hour": QuotaWindowEntry(utilization=0.35, resets_at=None)}, - binding=QuotaStatus(utilization=0.35, resets_at=None, window_name="five_hour"), - ) - - monkeypatch.setattr("autoskillit.execution.quota._fetch_quota", fake_fetch) - config = QuotaGuardConfig(cache_path=str(fresh_cache)) - await _refresh_quota_cache(config) - assert len(fetch_called) == 1 # must have fetched even though cache was fresh - - @pytest.mark.anyio - async def test_refresh_quota_cache_writes_new_cache(self, tmp_path, monkeypatch): - """_refresh_quota_cache writes a new cache file after fetching.""" - from autoskillit.execution.quota import ( - QuotaFetchResult, - QuotaStatus, - QuotaWindowEntry, - _refresh_quota_cache, - ) - - cache_path = tmp_path / "cache.json" - - async def fake_fetch(credentials_path, **kwargs): - return QuotaFetchResult( - windows={"five_hour": QuotaWindowEntry(utilization=0.5, resets_at=None)}, - binding=QuotaStatus(utilization=0.5, resets_at=None, window_name="five_hour"), - ) - - monkeypatch.setattr("autoskillit.execution.quota._fetch_quota", fake_fetch) - config = QuotaGuardConfig(cache_path=str(cache_path)) - await _refresh_quota_cache(config) - assert cache_path.exists() - data = json.loads(cache_path.read_text()) - assert data["binding"]["utilization"] == pytest.approx(0.5) - - -class TestCacheSchemaVersion: - """Phase 4 (#711 Part B): quota cache schema versioning tests.""" - - def setup_method(self): - from autoskillit.execution.quota import _reset_schema_drift_logged_for_tests - - _reset_schema_drift_logged_for_tests() - - def test_write_cache_embeds_schema_version(self, tmp_path): - from autoskillit.execution.quota import ( - QuotaFetchResult, - QuotaStatus, - QuotaWindowEntry, - _write_cache, - ) - - result = QuotaFetchResult( - windows={"five_hour": QuotaWindowEntry(utilization=50.0, resets_at=None)}, - binding=QuotaStatus(utilization=50.0, resets_at=None, window_name="five_hour"), - ) - cache_path = tmp_path / "cache.json" - _write_cache(str(cache_path), result) - raw = json.loads(cache_path.read_text()) - assert raw["schema_version"] == 3 - - def test_write_cache_uses_write_versioned_json(self, tmp_path, monkeypatch): - from autoskillit.execution.quota import ( - QuotaFetchResult, - QuotaStatus, - QuotaWindowEntry, - _write_cache, - ) - - calls = [] - - def spy(path, payload, schema_version): - calls.append({"path": path, "schema_version": schema_version}) - # Call through to real impl - from autoskillit.core import write_versioned_json as real - - real(path, payload, schema_version) - - monkeypatch.setattr("autoskillit.execution.quota.write_versioned_json", spy) - - result = QuotaFetchResult( - windows={"five_hour": QuotaWindowEntry(utilization=50.0, resets_at=None)}, - binding=QuotaStatus(utilization=50.0, resets_at=None, window_name="five_hour"), - ) - cache_path = tmp_path / "cache.json" - _write_cache(str(cache_path), result) - assert len(calls) == 1 - assert calls[0]["schema_version"] == 3 - - def test_read_cache_schema_round_trip_returns_status(self, tmp_path): - from autoskillit.execution.quota import ( - QuotaFetchResult, - QuotaStatus, - QuotaWindowEntry, - _read_cache, - _write_cache, - ) - - resets_at = datetime(2026, 2, 27, 20, 15, tzinfo=UTC) - result = QuotaFetchResult( - windows={"five_hour": QuotaWindowEntry(utilization=60.0, resets_at=resets_at)}, - binding=QuotaStatus(utilization=60.0, resets_at=resets_at, window_name="five_hour"), - ) - cache_path = tmp_path / "cache.json" - _write_cache(str(cache_path), result) - status = _read_cache(str(cache_path), max_age=60) - assert status is not None - assert status.utilization == pytest.approx(60.0) - - def test_read_cache_missing_schema_version_returns_none(self, tmp_path): - from autoskillit.execution.quota import _read_cache - - cache_path = tmp_path / "cache.json" - old_data = { - "fetched_at": datetime.now(UTC).isoformat(), - "windows": {}, - "binding": {"utilization": 50.0, "resets_at": None, "window_name": "five_hour"}, - } - cache_path.write_text(json.dumps(old_data)) - assert _read_cache(str(cache_path), max_age=60) is None - - def test_read_cache_wrong_schema_version_returns_none(self, tmp_path): - from autoskillit.execution.quota import _read_cache - - cache_path = tmp_path / "cache.json" - old_data = { - "schema_version": 1, - "fetched_at": datetime.now(UTC).isoformat(), - "windows": {}, - "binding": {"utilization": 50.0, "resets_at": None, "window_name": "five_hour"}, - } - cache_path.write_text(json.dumps(old_data)) - assert _read_cache(str(cache_path), max_age=60) is None - - def test_read_cache_logs_drift_warning_on_schema_mismatch(self, tmp_path): - import structlog.testing - - from autoskillit.execution.quota import _read_cache - - cache_path = tmp_path / "cache.json" - old_data = { - "fetched_at": datetime.now(UTC).isoformat(), - "windows": {}, - "binding": {"utilization": 50.0, "resets_at": None, "window_name": "five_hour"}, - } - cache_path.write_text(json.dumps(old_data)) - - with structlog.testing.capture_logs() as cap: - _read_cache(str(cache_path), max_age=60) - - drift_logs = [r for r in cap if "quota_cache_schema_drift" in r.get("event", "")] - assert len(drift_logs) == 1 - assert "cache_path" in drift_logs[0] - assert drift_logs[0]["observed"] is None - - def test_read_cache_logs_drift_exactly_once_per_path_per_process(self, tmp_path): - import structlog.testing - - from autoskillit.execution.quota import _read_cache - - cache_path = tmp_path / "cache.json" - old_data = { - "fetched_at": datetime.now(UTC).isoformat(), - "windows": {}, - "binding": {"utilization": 50.0, "resets_at": None, "window_name": "five_hour"}, - } - cache_path.write_text(json.dumps(old_data)) - - with structlog.testing.capture_logs() as cap: - for _ in range(5): - _read_cache(str(cache_path), max_age=60) - - drift_logs = [r for r in cap if "quota_cache_schema_drift" in r.get("event", "")] - assert len(drift_logs) == 1 - - def test_read_cache_logs_drift_once_per_path_not_globally(self, tmp_path): - import structlog.testing - - from autoskillit.execution.quota import _read_cache - - for name in ("cache_a.json", "cache_b.json"): - path = tmp_path / name - path.write_text( - json.dumps( - { - "fetched_at": datetime.now(UTC).isoformat(), - "windows": {}, - "binding": {"utilization": 50.0}, - } - ) - ) - - with structlog.testing.capture_logs() as cap: - _read_cache(str(tmp_path / "cache_a.json"), max_age=60) - _read_cache(str(tmp_path / "cache_b.json"), max_age=60) - - drift_logs = [r for r in cap if "quota_cache_schema_drift" in r.get("event", "")] - assert len(drift_logs) == 2 - - def test_read_cache_drift_set_is_module_scoped_and_reset_helper_works(self, tmp_path): - import structlog.testing - - from autoskillit.execution.quota import ( - _read_cache, - _reset_schema_drift_logged_for_tests, - ) - - cache_path = tmp_path / "cache.json" - cache_path.write_text( - json.dumps( - { - "fetched_at": datetime.now(UTC).isoformat(), - "windows": {}, - "binding": {"utilization": 50.0}, - } - ) - ) - - with structlog.testing.capture_logs() as cap1: - _read_cache(str(cache_path), max_age=60) - assert len([r for r in cap1 if "quota_cache_schema_drift" in r.get("event", "")]) == 1 - - _reset_schema_drift_logged_for_tests() - - with structlog.testing.capture_logs() as cap2: - _read_cache(str(cache_path), max_age=60) - assert len([r for r in cap2 if "quota_cache_schema_drift" in r.get("event", "")]) == 1 - - @pytest.mark.anyio - async def test_old_format_cache_gets_rewritten_with_new_format_on_next_fetch( - self, tmp_path, monkeypatch - ): - from autoskillit.execution.quota import ( - QuotaFetchResult, - QuotaStatus, - QuotaWindowEntry, - check_and_sleep_if_needed, - ) - - cache_path = tmp_path / "cache.json" - # Write old-format cache (no schema_version) - old_data = { - "fetched_at": datetime.now(UTC).isoformat(), - "windows": {}, - "binding": {"utilization": 50.0, "resets_at": None, "window_name": "five_hour"}, - } - cache_path.write_text(json.dumps(old_data)) - - async def fake_fetch(credentials_path, **kwargs): - return QuotaFetchResult( - windows={"five_hour": QuotaWindowEntry(utilization=30.0, resets_at=None)}, - binding=QuotaStatus(utilization=30.0, resets_at=None, window_name="five_hour"), - ) - - monkeypatch.setattr("autoskillit.execution.quota._fetch_quota", fake_fetch) - config = QuotaGuardConfig( - cache_path=str(cache_path), - credentials_path=str(tmp_path / "fake_creds.json"), - ) - await check_and_sleep_if_needed(config) - new_data = json.loads(cache_path.read_text()) - assert new_data["schema_version"] == 3 - - -class TestPerWindowToggles: - """Per-window enable/disable toggles for _compute_binding.""" - - _LONG_PATTERNS = ["weekly", "sonnet", "opus"] - - def _windows(self, five_hour_util: float, weekly_util: float) -> dict: - from autoskillit.execution.quota import QuotaWindowEntry - - now = datetime.now(UTC) - return { - "five_hour": QuotaWindowEntry( - utilization=five_hour_util, resets_at=now + timedelta(hours=3) - ), - "weekly": QuotaWindowEntry(utilization=weekly_util, resets_at=now + timedelta(days=4)), - } - - def test_compute_binding_defaults_both_enabled_preserves_behavior(self): - """Test 1: default call (no toggle kwargs) — five_hour at 90% is binding and blocks.""" - from autoskillit.execution.quota import _compute_binding - - binding = _compute_binding( - self._windows(90.0, 80.0), - short_threshold=85.0, - long_threshold=98.0, - long_patterns=self._LONG_PATTERNS, - ) - assert binding.window_name == "five_hour" - assert binding.should_block is True - assert binding.effective_threshold == pytest.approx(85.0) - - def test_compute_binding_short_disabled_drops_five_hour(self): - """Test 2: short_enabled=False — five_hour is dropped, weekly survives.""" - from autoskillit.execution.quota import _compute_binding - - binding = _compute_binding( - self._windows(90.0, 80.0), - short_threshold=85.0, - long_threshold=98.0, - long_patterns=self._LONG_PATTERNS, - short_enabled=False, - long_enabled=True, - ) - assert binding.window_name == "weekly" - assert binding.should_block is False - assert binding.effective_threshold == pytest.approx(98.0) - - def test_compute_binding_short_disabled_and_only_short_windows_present(self): - """Test 3: all windows are short class → all dropped → empty sentinel.""" - from autoskillit.execution.quota import QuotaWindowEntry, _compute_binding - - now = datetime.now(UTC) - windows = { - "one_hour": QuotaWindowEntry(utilization=95.0, resets_at=now + timedelta(minutes=45)), - "five_hour": QuotaWindowEntry(utilization=90.0, resets_at=now + timedelta(hours=3)), - } - binding = _compute_binding( - windows, - short_threshold=85.0, - long_threshold=98.0, - long_patterns=self._LONG_PATTERNS, - short_enabled=False, - ) - assert binding.utilization == pytest.approx(0.0) - assert binding.resets_at is None - assert binding.should_block is False - assert binding.effective_threshold == pytest.approx(100.0) - assert binding.window_name == "unknown" - - def test_compute_binding_long_disabled_drops_weekly(self): - """Test 4: long_enabled=False — weekly dropped, five_hour survives.""" - from autoskillit.execution.quota import _compute_binding - - binding = _compute_binding( - self._windows(80.0, 99.0), - short_threshold=85.0, - long_threshold=98.0, - long_patterns=self._LONG_PATTERNS, - long_enabled=False, - ) - assert binding.window_name == "five_hour" - assert binding.should_block is False - assert binding.effective_threshold == pytest.approx(85.0) - - def test_compute_binding_long_disabled_suppresses_weekly_block(self): - """Test 5: long_enabled=False — weekly at 99% is ignored, five_hour at 2% passes.""" - from autoskillit.execution.quota import _compute_binding - - binding = _compute_binding( - self._windows(2.0, 99.0), - short_threshold=85.0, - long_threshold=98.0, - long_patterns=self._LONG_PATTERNS, - long_enabled=False, - ) - assert binding.window_name == "five_hour" - assert binding.should_block is False - - def test_compute_binding_both_disabled_returns_sentinel(self): - """Test 6: both disabled → empty sentinel with should_block=False.""" - from autoskillit.execution.quota import _compute_binding - - binding = _compute_binding( - self._windows(90.0, 99.0), - short_threshold=85.0, - long_threshold=98.0, - long_patterns=self._LONG_PATTERNS, - short_enabled=False, - long_enabled=False, - ) - assert binding.utilization == pytest.approx(0.0) - assert binding.resets_at is None - assert binding.should_block is False - assert binding.effective_threshold == pytest.approx(100.0) - - def test_compute_binding_substring_classification_respects_long_patterns(self): - """Test 7: weekly_sonnet is classified as long via substring match and dropped.""" - from autoskillit.execution.quota import QuotaWindowEntry, _compute_binding - - now = datetime.now(UTC) - windows = { - "weekly_sonnet": QuotaWindowEntry(utilization=99.0, resets_at=now + timedelta(days=6)), - "five_hour": QuotaWindowEntry(utilization=90.0, resets_at=now + timedelta(hours=3)), - } - binding = _compute_binding( - windows, - short_threshold=85.0, - long_threshold=98.0, - long_patterns=self._LONG_PATTERNS, - long_enabled=False, - ) - assert binding.window_name == "five_hour" - - def test_compute_binding_both_enabled_matches_legacy_keyword_signature(self): - """Test 8: explicit True values produce identical output to implicit defaults.""" - from autoskillit.execution.quota import _compute_binding - - windows = self._windows(90.0, 80.0) - implicit = _compute_binding( - windows, - short_threshold=85.0, - long_threshold=98.0, - long_patterns=self._LONG_PATTERNS, - ) - explicit = _compute_binding( - windows, - short_threshold=85.0, - long_threshold=98.0, - long_patterns=self._LONG_PATTERNS, - short_enabled=True, - long_enabled=True, - ) - assert explicit.window_name == implicit.window_name - assert explicit.should_block == implicit.should_block - assert explicit.effective_threshold == pytest.approx(implicit.effective_threshold) - assert explicit.utilization == pytest.approx(implicit.utilization) - - @pytest.mark.anyio - async def test_check_and_sleep_respects_short_window_disabled(self, monkeypatch, tmp_path): - """Test 9: short_window_enabled=False — five_hour at 90% is ignored.""" - from autoskillit.execution.quota import ( - QuotaFetchResult, - QuotaStatus, - QuotaWindowEntry, - check_and_sleep_if_needed, - ) - - resets_at = datetime.now(UTC) + timedelta(hours=3) - config = QuotaGuardConfig( - short_window_enabled=False, - credentials_path=str(tmp_path / ".credentials.json"), - cache_path=str(tmp_path / "cache.json"), - ) - - captured_kwargs: dict = {} - - async def fake_fetch(credentials_path, **kwargs): - captured_kwargs.update(kwargs) - return QuotaFetchResult( - windows={ - "five_hour": QuotaWindowEntry(utilization=90.0, resets_at=resets_at), - "weekly": QuotaWindowEntry( - utilization=80.0, resets_at=datetime.now(UTC) + timedelta(days=4) - ), - }, - binding=QuotaStatus( - utilization=80.0, - resets_at=datetime.now(UTC) + timedelta(days=4), - window_name="weekly", - should_block=False, - effective_threshold=98.0, - ), - ) - - monkeypatch.setattr("autoskillit.execution.quota._fetch_quota", fake_fetch) - result = await check_and_sleep_if_needed(config) - assert result["should_sleep"] is False - assert result["window_name"] == "weekly" - assert captured_kwargs.get("short_enabled") is False - - @pytest.mark.anyio - async def test_check_and_sleep_respects_long_window_disabled(self, monkeypatch, tmp_path): - """Test 10: long_window_enabled=False — weekly at 99% is ignored.""" - from autoskillit.execution.quota import ( - QuotaFetchResult, - QuotaStatus, - QuotaWindowEntry, - check_and_sleep_if_needed, - ) - - config = QuotaGuardConfig( - long_window_enabled=False, - credentials_path=str(tmp_path / ".credentials.json"), - cache_path=str(tmp_path / "cache.json"), - ) - - captured_kwargs: dict = {} - - async def fake_fetch(credentials_path, **kwargs): - captured_kwargs.update(kwargs) - return QuotaFetchResult( - windows={ - "weekly": QuotaWindowEntry( - utilization=99.0, resets_at=datetime.now(UTC) + timedelta(days=6) - ), - "five_hour": QuotaWindowEntry( - utilization=2.0, resets_at=datetime.now(UTC) + timedelta(hours=3) - ), - }, - binding=QuotaStatus( - utilization=2.0, - resets_at=datetime.now(UTC) + timedelta(hours=3), - window_name="five_hour", - should_block=False, - effective_threshold=85.0, - ), - ) - - monkeypatch.setattr("autoskillit.execution.quota._fetch_quota", fake_fetch) - result = await check_and_sleep_if_needed(config) - assert result["should_sleep"] is False - assert result["window_name"] == "five_hour" - assert captured_kwargs.get("long_enabled") is False - - @pytest.mark.anyio - async def test_check_and_sleep_both_disabled_returns_sentinel_no_sleep( - self, monkeypatch, tmp_path - ): - """Test 11: both flags False → sentinel, no sleep, no error.""" - from autoskillit.execution.quota import ( - QuotaFetchResult, - QuotaStatus, - QuotaWindowEntry, - check_and_sleep_if_needed, - ) - - config = QuotaGuardConfig( - short_window_enabled=False, - long_window_enabled=False, - credentials_path=str(tmp_path / ".credentials.json"), - cache_path=str(tmp_path / "cache.json"), - ) - - async def fake_fetch(credentials_path, **kwargs): - return QuotaFetchResult( - windows={ - "weekly": QuotaWindowEntry( - utilization=99.0, resets_at=datetime.now(UTC) + timedelta(days=6) - ), - "five_hour": QuotaWindowEntry( - utilization=99.0, resets_at=datetime.now(UTC) + timedelta(hours=3) - ), - }, - binding=QuotaStatus(utilization=0.0, resets_at=None, effective_threshold=100.0), - ) - - monkeypatch.setattr("autoskillit.execution.quota._fetch_quota", fake_fetch) - result = await check_and_sleep_if_needed(config) - assert result["should_sleep"] is False - assert result["utilization"] == pytest.approx(0.0) - assert result["resets_at"] is None - assert "error" not in result - - @pytest.mark.anyio - async def test_check_and_sleep_global_enabled_false_still_short_circuits( - self, monkeypatch, tmp_path - ): - """Test 12: global enabled=False short-circuits before _fetch_quota is called.""" - fetch_called = [] - - async def sentinel_fetch(*a, **kw): - fetch_called.append(1) - raise AssertionError("_fetch_quota must not be called when enabled=False") - - monkeypatch.setattr("autoskillit.execution.quota._fetch_quota", sentinel_fetch) - from autoskillit.execution.quota import check_and_sleep_if_needed - - config = QuotaGuardConfig( - enabled=False, - short_window_enabled=True, - long_window_enabled=True, - ) - result = await check_and_sleep_if_needed(config) - assert result["should_sleep"] is False - assert fetch_called == [] - - @pytest.mark.anyio - async def test_refresh_quota_cache_forwards_per_window_toggles(self, monkeypatch, tmp_path): - """Test 13: _refresh_quota_cache passes short_enabled/long_enabled to _fetch_quota.""" - from autoskillit.execution.quota import ( - QuotaFetchResult, - QuotaStatus, - QuotaWindowEntry, - _refresh_quota_cache, - ) - - captured_kwargs: dict = {} - - async def fake_fetch(credentials_path, **kwargs): - captured_kwargs.update(kwargs) - return QuotaFetchResult( - windows={"five_hour": QuotaWindowEntry(utilization=0.1, resets_at=None)}, - binding=QuotaStatus(utilization=0.1, resets_at=None, window_name="five_hour"), - ) - - monkeypatch.setattr("autoskillit.execution.quota._fetch_quota", fake_fetch) - config = QuotaGuardConfig( - short_window_enabled=False, - cache_path=str(tmp_path / "cache.json"), - ) - await _refresh_quota_cache(config) - assert captured_kwargs["short_enabled"] is False - assert captured_kwargs["long_enabled"] is True diff --git a/tests/execution/test_quota_binding.py b/tests/execution/test_quota_binding.py new file mode 100644 index 0000000000..cd49289763 --- /dev/null +++ b/tests/execution/test_quota_binding.py @@ -0,0 +1,900 @@ +"""Tests for execution/quota.py — multi-window selection, per-window thresholds, +cache refresh, per-window toggles, and API vocabulary contract.""" + +from __future__ import annotations + +import json +from datetime import UTC, datetime, timedelta + +import pytest + +from tests._helpers import make_quota_guard_config + +pytestmark = [pytest.mark.layer("execution"), pytest.mark.medium] + + +class TestMultiWindowSelection: + """Verifies _compute_binding selects the worst-case window and full-snapshot cache I/O.""" + + # T-MW-1: one_hour exhausted, five_hour fine → binding = one_hour + def test_binding_window_one_hour_exhausted(self): + from autoskillit.execution.quota import QuotaWindowEntry, _compute_binding + + now = datetime.now(UTC) + resets_one_hour = now + timedelta(minutes=45) + windows = { + "one_hour": QuotaWindowEntry(utilization=91.0, resets_at=resets_one_hour), + "five_hour": QuotaWindowEntry(utilization=35.0, resets_at=now + timedelta(hours=4)), + } + binding = _compute_binding( + windows, + short_threshold=85.0, + long_threshold=98.0, + long_patterns=["weekly", "sonnet", "opus"], + ) + assert binding.utilization == pytest.approx(91.0) + assert binding.resets_at == resets_one_hour + assert binding.window_name == "one_hour" + + # T-MW-2: multiple windows exhausted → binding = window with latest resets_at + def test_binding_window_latest_resets_at_governs(self): + from autoskillit.execution.quota import QuotaWindowEntry, _compute_binding + + now = datetime.now(UTC) + resets_one_hour = now + timedelta(minutes=45) + resets_one_day = now + timedelta(hours=18) + windows = { + "one_hour": QuotaWindowEntry(utilization=91.0, resets_at=resets_one_hour), + "one_day": QuotaWindowEntry(utilization=97.0, resets_at=resets_one_day), + "five_hour": QuotaWindowEntry(utilization=35.0, resets_at=now + timedelta(hours=4)), + } + binding = _compute_binding( + windows, + short_threshold=85.0, + long_threshold=98.0, + long_patterns=["weekly", "sonnet", "opus"], + ) + assert binding.window_name == "one_day" + assert binding.resets_at == resets_one_day + + # T-MW-3: all windows fine → should_sleep=False, binding = highest utilization + def test_all_windows_fine_binding_is_highest_utilization(self): + from autoskillit.execution.quota import QuotaWindowEntry, _compute_binding + + now = datetime.now(UTC) + five_hour_resets = now + timedelta(hours=4) + windows = { + "one_hour": QuotaWindowEntry(utilization=40.0, resets_at=now + timedelta(hours=1)), + "five_hour": QuotaWindowEntry(utilization=60.0, resets_at=five_hour_resets), + "one_day": QuotaWindowEntry(utilization=30.0, resets_at=now + timedelta(days=1)), + } + binding = _compute_binding( + windows, + short_threshold=85.0, + long_threshold=98.0, + long_patterns=["weekly", "sonnet", "opus"], + ) + assert binding.utilization == pytest.approx(60.0) + assert binding.window_name == "five_hour" + assert binding.resets_at == five_hour_resets + + # T-MW-3b: empty windows dict → returns zero-utilization sentinel, no ValueError + def test_empty_windows_returns_zero_sentinel(self): + from autoskillit.execution.quota import _compute_binding + + binding = _compute_binding( + {}, + short_threshold=85.0, + long_threshold=98.0, + long_patterns=["weekly", "sonnet", "opus"], + ) + assert binding.utilization == pytest.approx(0.0) + assert binding.resets_at is None + + # T-MW-4: _write_cache stores full windows dict + binding key + def test_write_cache_stores_all_windows(self, tmp_path): + from autoskillit.execution.quota import ( + QuotaFetchResult, + QuotaStatus, + QuotaWindowEntry, + _write_cache, + ) + + now = datetime.now(UTC) + resets = now + timedelta(hours=1) + windows = { + "one_hour": QuotaWindowEntry(utilization=91.0, resets_at=resets), + "five_hour": QuotaWindowEntry(utilization=35.0, resets_at=now + timedelta(hours=4)), + } + binding = QuotaStatus(utilization=91.0, resets_at=resets, window_name="one_hour") + result = QuotaFetchResult(windows=windows, binding=binding) + cache_path = tmp_path / "cache.json" + _write_cache(str(cache_path), result) + data = json.loads(cache_path.read_text()) + assert "windows" in data + assert "one_hour" in data["windows"] + assert "five_hour" in data["windows"] + assert "binding" in data + assert data["binding"]["window_name"] == "one_hour" + assert data["binding"]["utilization"] == pytest.approx(91.0) + + # T-MW-5: _read_cache with old-format cache (missing "binding") returns None + def test_read_cache_old_format_returns_none(self, tmp_path): + from autoskillit.execution.quota import _read_cache + + old_cache = { + "fetched_at": datetime.now(UTC).isoformat(), + "five_hour": {"utilization": 87.0, "resets_at": None}, + } + cache_path = tmp_path / "old_cache.json" + cache_path.write_text(json.dumps(old_cache)) + result = _read_cache(str(cache_path), max_age=120) + assert result is None + + # T-MW-6: _read_cache with new format returns QuotaStatus from binding + def test_read_cache_new_format_returns_binding(self, tmp_path): + from autoskillit.execution.quota import _read_cache + + new_cache = { + "schema_version": 3, + "fetched_at": datetime.now(UTC).isoformat(), + "windows": { + "one_hour": { + "utilization": 91.0, + "resets_at": "2026-04-10T09:45:00+00:00", + }, + "five_hour": { + "utilization": 35.0, + "resets_at": "2026-04-10T13:00:00+00:00", + }, + }, + "binding": { + "window_name": "one_hour", + "utilization": 91.0, + "resets_at": "2026-04-10T09:45:00+00:00", + "should_block": True, + "effective_threshold": 85.0, + }, + } + cache_path = tmp_path / "new_cache.json" + cache_path.write_text(json.dumps(new_cache)) + status = _read_cache(str(cache_path), max_age=120) + assert status is not None + assert status.utilization == pytest.approx(91.0) + assert status.window_name == "one_hour" + + def test_compute_binding_blocks_seven_day_above_long_threshold(self): + """Full _compute_binding path: seven_day at 99% must block at long threshold.""" + from autoskillit.config.settings import QuotaGuardConfig + from autoskillit.execution.quota import QuotaWindowEntry, _compute_binding + + cfg = QuotaGuardConfig() + now = datetime.now(UTC) + windows = { + "seven_day": QuotaWindowEntry( + utilization=99.0, + resets_at=now + timedelta(days=3), + ), + "five_hour": QuotaWindowEntry( + utilization=10.0, + resets_at=now + timedelta(hours=3), + ), + } + result = _compute_binding( + windows, + short_threshold=cfg.short_window_threshold, + long_threshold=cfg.long_window_threshold, + long_patterns=cfg.long_window_patterns, + short_enabled=cfg.short_window_enabled, + long_enabled=cfg.long_window_enabled, + ) + assert result.should_block is True + assert result.window_name == "seven_day" + assert result.effective_threshold == 95.0 + + +class TestPerWindowThresholds: + """Per-window threshold classification: short windows block at 85%, long at 98%.""" + + _LONG_PATTERNS = ["seven_day", "sonnet", "opus"] + + def test_threshold_for_window_short_default(self): + from autoskillit.execution.quota import _threshold_for_window + + assert ( + _threshold_for_window( + "five_hour", + short_threshold=85.0, + long_threshold=98.0, + long_patterns=self._LONG_PATTERNS, + ) + == 85.0 + ) + + def test_threshold_for_window_long_seven_day(self): + from autoskillit.execution.quota import _threshold_for_window + + assert ( + _threshold_for_window( + "seven_day", + short_threshold=85.0, + long_threshold=98.0, + long_patterns=self._LONG_PATTERNS, + ) + == 98.0 + ) + + def test_threshold_for_window_long_sonnet_substring(self): + from autoskillit.execution.quota import _threshold_for_window + + assert ( + _threshold_for_window( + "weekly_sonnet", + short_threshold=85.0, + long_threshold=98.0, + long_patterns=self._LONG_PATTERNS, + ) + == 98.0 + ) + + def test_threshold_for_window_case_insensitive(self): + from autoskillit.execution.quota import _threshold_for_window + + assert ( + _threshold_for_window( + "Seven_Day", + short_threshold=85.0, + long_threshold=98.0, + long_patterns=self._LONG_PATTERNS, + ) + == 98.0 + ) + + def test_threshold_for_window_unknown_uses_short(self): + from autoskillit.execution.quota import _threshold_for_window + + assert ( + _threshold_for_window( + "context", + short_threshold=85.0, + long_threshold=98.0, + long_patterns=self._LONG_PATTERNS, + ) + == 85.0 + ) + + def test_compute_binding_picks_short_when_only_short_exhausted(self): + from autoskillit.execution.quota import QuotaWindowEntry, _compute_binding + + now = datetime.now(UTC) + windows = { + "five_hour": QuotaWindowEntry(utilization=90.0, resets_at=now + timedelta(hours=1)), + "weekly": QuotaWindowEntry(utilization=80.0, resets_at=now + timedelta(days=4)), + } + binding = _compute_binding( + windows, + short_threshold=85.0, + long_threshold=98.0, + long_patterns=self._LONG_PATTERNS, + ) + assert binding.window_name == "five_hour" + assert binding.should_block is True + assert binding.effective_threshold == pytest.approx(85.0) + + def test_compute_binding_does_not_block_seven_day_below_long_threshold(self): + """Regression test for issue #721: seven_day at 86% must not block. + + Long-window quotas (seven_day, sonnet, opus) reset across multi-day windows, + so 14% remaining headroom is comfortable, not exhausted. The pre-fix + behaviour blocked the pipeline for ~4 days at 86% seven_day. + """ + from autoskillit.execution.quota import QuotaWindowEntry, _compute_binding + + now = datetime.now(UTC) + windows = { + "seven_day": QuotaWindowEntry(utilization=86.0, resets_at=now + timedelta(days=4)), + "five_hour": QuotaWindowEntry(utilization=50.0, resets_at=now + timedelta(hours=1)), + } + binding = _compute_binding( + windows, + short_threshold=85.0, + long_threshold=98.0, + long_patterns=self._LONG_PATTERNS, + ) + assert binding.should_block is False + assert binding.effective_threshold == pytest.approx(98.0) + + def test_compute_binding_blocks_seven_day_at_99_percent(self): + from autoskillit.execution.quota import QuotaWindowEntry, _compute_binding + + now = datetime.now(UTC) + windows = { + "seven_day": QuotaWindowEntry(utilization=99.0, resets_at=now + timedelta(days=4)), + } + binding = _compute_binding( + windows, + short_threshold=85.0, + long_threshold=98.0, + long_patterns=self._LONG_PATTERNS, + ) + assert binding.should_block is True + assert binding.effective_threshold == pytest.approx(98.0) + + def test_compute_binding_picks_latest_resets_among_exhausted(self): + """Among exhausted windows, the one with the latest reset wins.""" + from autoskillit.execution.quota import QuotaWindowEntry, _compute_binding + + now = datetime.now(UTC) + weekly_resets = now + timedelta(days=4) + windows = { + "five_hour": QuotaWindowEntry(utilization=90.0, resets_at=now + timedelta(hours=1)), + "weekly": QuotaWindowEntry(utilization=99.0, resets_at=weekly_resets), + } + binding = _compute_binding( + windows, + short_threshold=85.0, + long_threshold=98.0, + long_patterns=self._LONG_PATTERNS, + ) + assert binding.window_name == "weekly" + assert binding.resets_at == weekly_resets + assert binding.should_block is True + + def test_write_cache_includes_should_block_and_effective_threshold(self, tmp_path): + from autoskillit.execution.quota import ( + QuotaFetchResult, + QuotaStatus, + QuotaWindowEntry, + _write_cache, + ) + + now = datetime.now(UTC) + weekly_resets = now + timedelta(days=4) + windows = { + "weekly": QuotaWindowEntry(utilization=86.0, resets_at=weekly_resets), + } + binding = QuotaStatus( + utilization=86.0, + resets_at=weekly_resets, + window_name="weekly", + should_block=False, + effective_threshold=98.0, + ) + result = QuotaFetchResult(windows=windows, binding=binding) + cache_path = tmp_path / "cache.json" + _write_cache(str(cache_path), result) + data = json.loads(cache_path.read_text()) + assert data["binding"]["should_block"] is False + assert data["binding"]["effective_threshold"] == pytest.approx(98.0) + + def test_read_cache_round_trip_should_block(self, tmp_path): + from autoskillit.execution.quota import ( + QuotaFetchResult, + QuotaStatus, + QuotaWindowEntry, + _read_cache, + _write_cache, + ) + + now = datetime.now(UTC) + weekly_resets = now + timedelta(days=4) + windows = { + "seven_day": QuotaWindowEntry(utilization=99.0, resets_at=weekly_resets), + } + binding = QuotaStatus( + utilization=99.0, + resets_at=weekly_resets, + window_name="seven_day", + should_block=True, + effective_threshold=98.0, + ) + cache_path = tmp_path / "cache.json" + _write_cache(str(cache_path), QuotaFetchResult(windows=windows, binding=binding)) + status = _read_cache(str(cache_path), max_age=300) + assert status is not None + assert status.should_block is True + assert status.effective_threshold == pytest.approx(98.0) + assert status.window_name == "seven_day" + + @pytest.mark.anyio + async def test_check_and_sleep_returns_false_for_seven_day_at_86_percent( + self, monkeypatch, tmp_path + ): + """End-to-end regression test for #721 — seven_day at 86% must not sleep.""" + from autoskillit.execution.quota import ( + QuotaFetchResult, + QuotaWindowEntry, + _compute_binding, + check_and_sleep_if_needed, + ) + + now = datetime.now(UTC) + windows = { + "seven_day": QuotaWindowEntry(utilization=86.0, resets_at=now + timedelta(days=4)), + "five_hour": QuotaWindowEntry(utilization=2.0, resets_at=now + timedelta(hours=1)), + } + binding = _compute_binding( + windows, + short_threshold=85.0, + long_threshold=98.0, + long_patterns=self._LONG_PATTERNS, + ) + + async def fake_fetch(credentials_path, **kwargs): + return QuotaFetchResult(windows=windows, binding=binding) + + monkeypatch.setattr("autoskillit.execution.quota._fetch_quota", fake_fetch) + config = make_quota_guard_config( + cache_path=str(tmp_path / "cache.json"), + credentials_path=str(tmp_path / "creds.json"), + ) + result = await check_and_sleep_if_needed(config) + assert result["should_sleep"] is False + assert result["window_name"] == "seven_day" + + @pytest.mark.anyio + async def test_check_and_sleep_returns_true_for_seven_day_at_99_percent( + self, monkeypatch, tmp_path + ): + from autoskillit.execution.quota import ( + QuotaFetchResult, + QuotaWindowEntry, + _compute_binding, + check_and_sleep_if_needed, + ) + + now = datetime.now(UTC) + weekly_resets = now + timedelta(days=4) + windows = { + "seven_day": QuotaWindowEntry(utilization=99.0, resets_at=weekly_resets), + "five_hour": QuotaWindowEntry(utilization=2.0, resets_at=now + timedelta(hours=1)), + } + binding = _compute_binding( + windows, + short_threshold=85.0, + long_threshold=98.0, + long_patterns=self._LONG_PATTERNS, + ) + + async def fake_fetch(credentials_path, **kwargs): + return QuotaFetchResult(windows=windows, binding=binding) + + monkeypatch.setattr("autoskillit.execution.quota._fetch_quota", fake_fetch) + config = make_quota_guard_config( + cache_path=str(tmp_path / "cache.json"), + credentials_path=str(tmp_path / "creds.json"), + ) + result = await check_and_sleep_if_needed(config) + assert result["should_sleep"] is True + assert result["window_name"] == "seven_day" + assert result["sleep_seconds"] > 0 + + def test_seven_day_window_classified_as_long_with_default_patterns(self): + """seven_day is the actual Anthropic API key for the weekly budget. + It must be classified as a long window by the default long_window_patterns.""" + from autoskillit.config.settings import QuotaGuardConfig + from autoskillit.execution.quota import _is_long_window + + defaults = QuotaGuardConfig().long_window_patterns + assert _is_long_window("seven_day", defaults), ( + f"seven_day not classified as long by default patterns {defaults!r}. " + "Update long_window_patterns to include a pattern that matches 'seven_day'." + ) + + def test_threshold_for_window_seven_day_returns_long_threshold(self): + """seven_day must yield the long threshold (95.0) with default config.""" + from autoskillit.config.settings import QuotaGuardConfig + from autoskillit.execution.quota import _threshold_for_window + + cfg = QuotaGuardConfig() + result = _threshold_for_window( + "seven_day", + short_threshold=cfg.short_window_threshold, + long_threshold=cfg.long_window_threshold, + long_patterns=cfg.long_window_patterns, + ) + assert result == 95.0, ( + f"seven_day returned threshold {result}, expected 95.0. " + "The Anthropic API uses 'seven_day' for the 7-day rate-limit window." + ) + + +class TestRefreshQuotaCache: + """Tests for _refresh_quota_cache: unconditional fetch-and-write behavior.""" + + @pytest.mark.anyio + async def test_refresh_quota_cache_fetches_even_when_cache_is_fresh( + self, tmp_path, monkeypatch + ): + """_refresh_quota_cache calls _fetch_quota unconditionally, even if cache is fresh.""" + from autoskillit.execution.quota import ( + QuotaFetchResult, + QuotaStatus, + QuotaWindowEntry, + _refresh_quota_cache, + ) + + fresh_cache = tmp_path / "cache.json" + # Write a 10-second-old cache (well within 300s max_age) + fresh_cache.write_text( + json.dumps( + { + "fetched_at": (datetime.now(UTC) - timedelta(seconds=10)).isoformat(), + "five_hour": {"utilization": 0.3, "resets_at": None}, + } + ) + ) + fetch_called = [] + + async def fake_fetch(credentials_path, **kwargs): + fetch_called.append(True) + return QuotaFetchResult( + windows={"five_hour": QuotaWindowEntry(utilization=0.35, resets_at=None)}, + binding=QuotaStatus(utilization=0.35, resets_at=None, window_name="five_hour"), + ) + + monkeypatch.setattr("autoskillit.execution.quota._fetch_quota", fake_fetch) + config = make_quota_guard_config(cache_path=str(fresh_cache)) + await _refresh_quota_cache(config) + assert len(fetch_called) == 1 # must have fetched even though cache was fresh + + @pytest.mark.anyio + async def test_refresh_quota_cache_writes_new_cache(self, tmp_path, monkeypatch): + """_refresh_quota_cache writes a new cache file after fetching.""" + from autoskillit.execution.quota import ( + QuotaFetchResult, + QuotaStatus, + QuotaWindowEntry, + _refresh_quota_cache, + ) + + cache_path = tmp_path / "cache.json" + + async def fake_fetch(credentials_path, **kwargs): + return QuotaFetchResult( + windows={"five_hour": QuotaWindowEntry(utilization=0.5, resets_at=None)}, + binding=QuotaStatus(utilization=0.5, resets_at=None, window_name="five_hour"), + ) + + monkeypatch.setattr("autoskillit.execution.quota._fetch_quota", fake_fetch) + config = make_quota_guard_config(cache_path=str(cache_path)) + await _refresh_quota_cache(config) + assert cache_path.exists() + data = json.loads(cache_path.read_text()) + assert data["binding"]["utilization"] == pytest.approx(0.5) + + +class TestPerWindowToggles: + """Per-window enable/disable toggles for _compute_binding.""" + + _LONG_PATTERNS = ["seven_day", "sonnet", "opus"] + + def _windows(self, five_hour_util: float, seven_day_util: float) -> dict: + from autoskillit.execution.quota import QuotaWindowEntry + + now = datetime.now(UTC) + return { + "five_hour": QuotaWindowEntry( + utilization=five_hour_util, resets_at=now + timedelta(hours=3) + ), + "seven_day": QuotaWindowEntry( + utilization=seven_day_util, resets_at=now + timedelta(days=4) + ), + } + + def test_compute_binding_defaults_both_enabled_preserves_behavior(self): + """Test 1: default call (no toggle kwargs) — five_hour at 90% is binding and blocks.""" + from autoskillit.execution.quota import _compute_binding + + binding = _compute_binding( + self._windows(90.0, 80.0), + short_threshold=85.0, + long_threshold=98.0, + long_patterns=self._LONG_PATTERNS, + ) + assert binding.window_name == "five_hour" + assert binding.should_block is True + assert binding.effective_threshold == pytest.approx(85.0) + + def test_compute_binding_short_disabled_drops_five_hour(self): + """Test 2: short_enabled=False — five_hour is dropped, seven_day survives.""" + from autoskillit.execution.quota import _compute_binding + + binding = _compute_binding( + self._windows(90.0, 80.0), + short_threshold=85.0, + long_threshold=98.0, + long_patterns=self._LONG_PATTERNS, + short_enabled=False, + long_enabled=True, + ) + assert binding.window_name == "seven_day" + assert binding.should_block is False + assert binding.effective_threshold == pytest.approx(98.0) + + def test_compute_binding_short_disabled_and_only_short_windows_present(self): + """Test 3: all windows are short class → all dropped → empty sentinel.""" + from autoskillit.execution.quota import QuotaWindowEntry, _compute_binding + + now = datetime.now(UTC) + windows = { + "one_hour": QuotaWindowEntry(utilization=95.0, resets_at=now + timedelta(minutes=45)), + "five_hour": QuotaWindowEntry(utilization=90.0, resets_at=now + timedelta(hours=3)), + } + binding = _compute_binding( + windows, + short_threshold=85.0, + long_threshold=98.0, + long_patterns=self._LONG_PATTERNS, + short_enabled=False, + ) + assert binding.utilization == pytest.approx(0.0) + assert binding.resets_at is None + assert binding.should_block is False + assert binding.effective_threshold == pytest.approx(100.0) + assert binding.window_name == "unknown" + + def test_compute_binding_long_disabled_drops_weekly(self): + """Test 4: long_enabled=False — weekly dropped, five_hour survives.""" + from autoskillit.execution.quota import _compute_binding + + binding = _compute_binding( + self._windows(80.0, 99.0), + short_threshold=85.0, + long_threshold=98.0, + long_patterns=self._LONG_PATTERNS, + long_enabled=False, + ) + assert binding.window_name == "five_hour" + assert binding.should_block is False + assert binding.effective_threshold == pytest.approx(85.0) + + def test_compute_binding_long_disabled_suppresses_weekly_block(self): + """Test 5: long_enabled=False — weekly at 99% is ignored, five_hour at 2% passes.""" + from autoskillit.execution.quota import _compute_binding + + binding = _compute_binding( + self._windows(2.0, 99.0), + short_threshold=85.0, + long_threshold=98.0, + long_patterns=self._LONG_PATTERNS, + long_enabled=False, + ) + assert binding.window_name == "five_hour" + assert binding.should_block is False + + def test_compute_binding_both_disabled_returns_sentinel(self): + """Test 6: both disabled → empty sentinel with should_block=False.""" + from autoskillit.execution.quota import _compute_binding + + binding = _compute_binding( + self._windows(90.0, 99.0), + short_threshold=85.0, + long_threshold=98.0, + long_patterns=self._LONG_PATTERNS, + short_enabled=False, + long_enabled=False, + ) + assert binding.utilization == pytest.approx(0.0) + assert binding.resets_at is None + assert binding.should_block is False + assert binding.effective_threshold == pytest.approx(100.0) + + def test_compute_binding_substring_classification_respects_long_patterns(self): + """Test 7: weekly_sonnet is classified as long via substring match and dropped.""" + from autoskillit.execution.quota import QuotaWindowEntry, _compute_binding + + now = datetime.now(UTC) + windows = { + "weekly_sonnet": QuotaWindowEntry(utilization=99.0, resets_at=now + timedelta(days=6)), + "five_hour": QuotaWindowEntry(utilization=90.0, resets_at=now + timedelta(hours=3)), + } + binding = _compute_binding( + windows, + short_threshold=85.0, + long_threshold=98.0, + long_patterns=self._LONG_PATTERNS, + long_enabled=False, + ) + assert binding.window_name == "five_hour" + + def test_compute_binding_both_enabled_matches_legacy_keyword_signature(self): + """Test 8: explicit True values produce identical output to implicit defaults.""" + from autoskillit.execution.quota import _compute_binding + + windows = self._windows(90.0, 80.0) + implicit = _compute_binding( + windows, + short_threshold=85.0, + long_threshold=98.0, + long_patterns=self._LONG_PATTERNS, + ) + explicit = _compute_binding( + windows, + short_threshold=85.0, + long_threshold=98.0, + long_patterns=self._LONG_PATTERNS, + short_enabled=True, + long_enabled=True, + ) + assert explicit.window_name == implicit.window_name + assert explicit.should_block == implicit.should_block + assert explicit.effective_threshold == pytest.approx(implicit.effective_threshold) + assert explicit.utilization == pytest.approx(implicit.utilization) + + @pytest.mark.anyio + async def test_check_and_sleep_respects_short_window_disabled(self, monkeypatch, tmp_path): + """Test 9: short_window_enabled=False — five_hour at 90% is ignored.""" + from autoskillit.execution.quota import ( + QuotaFetchResult, + QuotaStatus, + QuotaWindowEntry, + check_and_sleep_if_needed, + ) + + resets_at = datetime.now(UTC) + timedelta(hours=3) + config = make_quota_guard_config( + short_window_enabled=False, + credentials_path=str(tmp_path / ".credentials.json"), + cache_path=str(tmp_path / "cache.json"), + ) + + captured_kwargs: dict = {} + + async def fake_fetch(credentials_path, **kwargs): + captured_kwargs.update(kwargs) + return QuotaFetchResult( + windows={ + "five_hour": QuotaWindowEntry(utilization=90.0, resets_at=resets_at), + "weekly": QuotaWindowEntry( + utilization=80.0, resets_at=datetime.now(UTC) + timedelta(days=4) + ), + }, + binding=QuotaStatus( + utilization=80.0, + resets_at=datetime.now(UTC) + timedelta(days=4), + window_name="weekly", + should_block=False, + effective_threshold=98.0, + ), + ) + + monkeypatch.setattr("autoskillit.execution.quota._fetch_quota", fake_fetch) + result = await check_and_sleep_if_needed(config) + assert result["should_sleep"] is False + assert result["window_name"] == "weekly" + assert captured_kwargs.get("short_enabled") is False + + @pytest.mark.anyio + async def test_check_and_sleep_respects_long_window_disabled(self, monkeypatch, tmp_path): + """Test 10: long_window_enabled=False — weekly at 99% is ignored.""" + from autoskillit.execution.quota import ( + QuotaFetchResult, + QuotaStatus, + QuotaWindowEntry, + check_and_sleep_if_needed, + ) + + config = make_quota_guard_config( + long_window_enabled=False, + credentials_path=str(tmp_path / ".credentials.json"), + cache_path=str(tmp_path / "cache.json"), + ) + + captured_kwargs: dict = {} + + async def fake_fetch(credentials_path, **kwargs): + captured_kwargs.update(kwargs) + return QuotaFetchResult( + windows={ + "weekly": QuotaWindowEntry( + utilization=99.0, resets_at=datetime.now(UTC) + timedelta(days=6) + ), + "five_hour": QuotaWindowEntry( + utilization=2.0, resets_at=datetime.now(UTC) + timedelta(hours=3) + ), + }, + binding=QuotaStatus( + utilization=2.0, + resets_at=datetime.now(UTC) + timedelta(hours=3), + window_name="five_hour", + should_block=False, + effective_threshold=85.0, + ), + ) + + monkeypatch.setattr("autoskillit.execution.quota._fetch_quota", fake_fetch) + result = await check_and_sleep_if_needed(config) + assert result["should_sleep"] is False + assert result["window_name"] == "five_hour" + assert captured_kwargs.get("long_enabled") is False + + @pytest.mark.anyio + async def test_check_and_sleep_both_disabled_returns_sentinel_no_sleep( + self, monkeypatch, tmp_path + ): + """Test 11: both flags False → sentinel, no sleep, no error.""" + from autoskillit.execution.quota import ( + QuotaFetchResult, + QuotaStatus, + QuotaWindowEntry, + check_and_sleep_if_needed, + ) + + config = make_quota_guard_config( + short_window_enabled=False, + long_window_enabled=False, + credentials_path=str(tmp_path / ".credentials.json"), + cache_path=str(tmp_path / "cache.json"), + ) + + async def fake_fetch(credentials_path, **kwargs): + return QuotaFetchResult( + windows={ + "weekly": QuotaWindowEntry( + utilization=99.0, resets_at=datetime.now(UTC) + timedelta(days=6) + ), + "five_hour": QuotaWindowEntry( + utilization=99.0, resets_at=datetime.now(UTC) + timedelta(hours=3) + ), + }, + binding=QuotaStatus(utilization=0.0, resets_at=None, effective_threshold=100.0), + ) + + monkeypatch.setattr("autoskillit.execution.quota._fetch_quota", fake_fetch) + result = await check_and_sleep_if_needed(config) + assert result["should_sleep"] is False + assert result["utilization"] == pytest.approx(0.0) + assert result["resets_at"] is None + assert "error" not in result + + @pytest.mark.anyio + async def test_check_and_sleep_global_enabled_false_still_short_circuits( + self, monkeypatch, tmp_path + ): + """Test 12: global enabled=False short-circuits before _fetch_quota is called.""" + fetch_called = [] + + async def sentinel_fetch(*a, **kw): + fetch_called.append(1) + raise AssertionError("_fetch_quota must not be called when enabled=False") + + monkeypatch.setattr("autoskillit.execution.quota._fetch_quota", sentinel_fetch) + from autoskillit.execution.quota import check_and_sleep_if_needed + + config = make_quota_guard_config( + enabled=False, + short_window_enabled=True, + long_window_enabled=True, + ) + result = await check_and_sleep_if_needed(config) + assert result["should_sleep"] is False + assert fetch_called == [] + + @pytest.mark.anyio + async def test_refresh_quota_cache_forwards_per_window_toggles(self, monkeypatch, tmp_path): + """Test 13: _refresh_quota_cache passes short_enabled/long_enabled to _fetch_quota.""" + from autoskillit.execution.quota import ( + QuotaFetchResult, + QuotaStatus, + QuotaWindowEntry, + _refresh_quota_cache, + ) + + captured_kwargs: dict = {} + + async def fake_fetch(credentials_path, **kwargs): + captured_kwargs.update(kwargs) + return QuotaFetchResult( + windows={"five_hour": QuotaWindowEntry(utilization=0.1, resets_at=None)}, + binding=QuotaStatus(utilization=0.1, resets_at=None, window_name="five_hour"), + ) + + monkeypatch.setattr("autoskillit.execution.quota._fetch_quota", fake_fetch) + config = make_quota_guard_config( + short_window_enabled=False, + cache_path=str(tmp_path / "cache.json"), + ) + await _refresh_quota_cache(config) + assert captured_kwargs["short_enabled"] is False + assert captured_kwargs["long_enabled"] is True diff --git a/tests/execution/test_quota_http.py b/tests/execution/test_quota_http.py index dbce6a2e28..6ffdc3a4c7 100644 --- a/tests/execution/test_quota_http.py +++ b/tests/execution/test_quota_http.py @@ -7,16 +7,24 @@ from __future__ import annotations import json +import sys import time from datetime import UTC, datetime, timedelta from types import SimpleNamespace import pytest -from api_simulator.http import MockResponseSpec as PyResponseSpec -from autoskillit.execution.quota import check_and_sleep_if_needed +_api_sim_http = pytest.importorskip("api_simulator.http") +PyResponseSpec = _api_sim_http.MockResponseSpec -pytestmark = pytest.mark.anyio +from autoskillit.execution.quota import check_and_sleep_if_needed # noqa: E402 + +pytestmark = [ + pytest.mark.layer("execution"), + pytest.mark.medium, + pytest.mark.skipif(sys.platform != "linux", reason="api-simulator is Linux-only"), + pytest.mark.anyio, +] QUOTA_ENDPOINT = "/api/oauth/usage" @@ -50,7 +58,7 @@ def quota_config(credentials, tmp_path): cache_max_age=120, short_window_threshold=80.0, long_window_threshold=98.0, - long_window_patterns=["weekly", "sonnet", "opus"], + long_window_patterns=["seven_day", "sonnet", "opus"], buffer_seconds=60, ) @@ -254,3 +262,48 @@ async def test_unknown_window_keys_tolerated(mock_http_server, quota_config): result = await check_and_sleep_if_needed(quota_config, base_url=mock_http_server.url) assert result["should_sleep"] is False assert result["utilization"] == pytest.approx(50.0) + + +async def test_seven_day_raw_api_response_blocks_at_long_threshold(mock_http_server, credentials): + """A realistic /api/oauth/usage response with seven_day at 99% must produce a block at 98%. + + Exercises the full path: raw API JSON → _fetch_quota parsing loop → + _compute_binding classification → QuotaFetchResult. Unlike unit tests that mock + _fetch_quota itself, this test validates the HTTP parsing layer end-to-end. + """ + from autoskillit.config.settings import QuotaGuardConfig + from autoskillit.execution.quota import _fetch_quota + + resets_at = (datetime.now(UTC) + timedelta(days=3)).isoformat() + mock_http_server.register( + "GET", + QUOTA_ENDPOINT, + PyResponseSpec( + body={ + "seven_day": {"utilization": 99.0, "resets_at": resets_at}, + "five_hour": { + "utilization": 10.0, + "resets_at": (datetime.now(UTC) + timedelta(hours=3)).isoformat(), + }, + } + ), + ) + cfg = QuotaGuardConfig() + + result = await _fetch_quota( + credentials, + short_threshold=cfg.short_window_threshold, + long_threshold=cfg.long_window_threshold, + long_patterns=cfg.long_window_patterns, + short_enabled=cfg.short_window_enabled, + long_enabled=cfg.long_window_enabled, + base_url=mock_http_server.url, + ) + + assert result.binding.should_block is True, ( + f"Expected should_block=True for seven_day at 99% with long_threshold=95.0, " + f"got effective_threshold={result.binding.effective_threshold}, " + f"window_name={result.binding.window_name!r}" + ) + assert result.binding.window_name == "seven_day" + assert result.binding.effective_threshold == 95.0 diff --git a/tests/execution/test_quota_io.py b/tests/execution/test_quota_io.py new file mode 100644 index 0000000000..f6dc536a2e --- /dev/null +++ b/tests/execution/test_quota_io.py @@ -0,0 +1,537 @@ +"""Tests for execution/quota.py — credential reading, cache I/O, +dataclass validation, and cache schema versioning.""" + +from __future__ import annotations + +import json +import time +from datetime import UTC, datetime, timedelta + +import pytest + +from tests._helpers import make_quota_guard_config + +pytestmark = [pytest.mark.layer("execution"), pytest.mark.medium] + + +class TestReadCredentials: + def test_reads_access_token(self, tmp_path): + expires_ms = int(time.time() * 1000) + 3_600_000 + creds = {"claudeAiOauth": {"accessToken": "tok-abc", "expiresAt": expires_ms}} + path = tmp_path / ".credentials.json" + path.write_text(json.dumps(creds)) + from autoskillit.execution.quota import _read_credentials + + assert _read_credentials(str(path)) == "tok-abc" + + def test_expired_token_raises(self, tmp_path): + expires_ms = int(time.time() * 1000) - 1000 # already expired + creds = {"claudeAiOauth": {"accessToken": "tok-abc", "expiresAt": expires_ms}} + path = tmp_path / ".credentials.json" + path.write_text(json.dumps(creds)) + from autoskillit.execution.quota import _read_credentials + + with pytest.raises(PermissionError): + _read_credentials(str(path)) + + def test_missing_file_raises(self, tmp_path): + from autoskillit.execution.quota import _read_credentials + + with pytest.raises(FileNotFoundError): + _read_credentials(str(tmp_path / "nonexistent.json")) + + def test_missing_key_raises(self, tmp_path): + path = tmp_path / "bad.json" + path.write_text('{"other": "data"}') + from autoskillit.execution.quota import _read_credentials + + with pytest.raises(KeyError): + _read_credentials(str(path)) + + +class TestReadCache: + def test_fresh_cache_returns_status(self, tmp_path): + from autoskillit.execution.quota import _read_cache + + now = datetime.now(UTC) + cache_data = { + "schema_version": 3, + "fetched_at": (now - timedelta(seconds=30)).isoformat(), + "windows": { + "five_hour": { + "utilization": 87.3, + "resets_at": "2026-02-27T20:15:00+00:00", + } + }, + "binding": { + "window_name": "five_hour", + "utilization": 87.3, + "resets_at": "2026-02-27T20:15:00+00:00", + "should_block": True, + "effective_threshold": 85.0, + }, + } + cache_path = tmp_path / "usage_cache.json" + cache_path.write_text(json.dumps(cache_data)) + status = _read_cache(str(cache_path), max_age=120) + assert status is not None + assert status.utilization == pytest.approx(87.3) + + def test_stale_cache_returns_none(self, tmp_path): + from autoskillit.execution.quota import _read_cache + + now = datetime.now(UTC) + cache_data = { + "fetched_at": (now - timedelta(seconds=300)).isoformat(), + "five_hour": { + "utilization": 50.0, + "resets_at": "2026-02-27T20:15:00+00:00", + }, + } + cache_path = tmp_path / "usage_cache.json" + cache_path.write_text(json.dumps(cache_data)) + assert _read_cache(str(cache_path), max_age=120) is None + + def test_missing_cache_returns_none(self, tmp_path): + from autoskillit.execution.quota import _read_cache + + assert _read_cache(str(tmp_path / "nonexistent.json"), max_age=120) is None + + def test_corrupted_cache_returns_none(self, tmp_path): + from autoskillit.execution.quota import _read_cache + + cache_path = tmp_path / "usage_cache.json" + cache_path.write_text("not valid json{{") + assert _read_cache(str(cache_path), max_age=120) is None + + def test_read_cache_null_utilization_returns_none(self, tmp_path): + """Cache with utilization: null in binding must return None (fail-open).""" + from autoskillit.execution.quota import _read_cache + + cache_path = tmp_path / "usage_cache.json" + payload = { + "fetched_at": datetime.now(UTC).isoformat(), + "binding": {"utilization": None, "resets_at": None, "window_name": "five_hour"}, + } + cache_path.write_text(json.dumps(payload)) + assert _read_cache(str(cache_path), max_age=300) is None + + def test_read_cache_typeerror_returns_none(self, tmp_path): + """_read_cache must catch TypeError (e.g., float(None)) and return None.""" + from autoskillit.execution.quota import _read_cache + + cache_path = tmp_path / "usage_cache.json" + payload = { + "fetched_at": datetime.now(UTC).isoformat(), + "binding": {"utilization": None, "resets_at": None}, + } + cache_path.write_text(json.dumps(payload)) + # With __post_init__, constructing QuotaStatus with None raises TypeError. + # _read_cache must catch it and return None. + assert _read_cache(str(cache_path), max_age=300) is None + + +class TestQuotaDataclassPostInit: + """QuotaStatus and QuotaWindowEntry must enforce types at construction.""" + + def test_quota_window_entry_rejects_none_utilization(self): + from autoskillit.execution.quota import QuotaWindowEntry + + with pytest.raises(TypeError, match="utilization"): + QuotaWindowEntry(utilization=None, resets_at=None) + + def test_quota_status_rejects_none_utilization(self): + from autoskillit.execution.quota import QuotaStatus + + with pytest.raises(TypeError, match="utilization"): + QuotaStatus(utilization=None, resets_at=None) + + def test_quota_window_entry_coerces_string_utilization(self): + from autoskillit.execution.quota import QuotaWindowEntry + + entry = QuotaWindowEntry(utilization="85.5", resets_at=None) + assert entry.utilization == 85.5 + assert isinstance(entry.utilization, float) + + def test_quota_status_coerces_string_utilization(self): + from autoskillit.execution.quota import QuotaStatus + + status = QuotaStatus(utilization="42.0", resets_at=None) + assert status.utilization == 42.0 + assert isinstance(status.utilization, float) + + +class TestWriteCache: + def test_writes_readable_cache(self, tmp_path): + from autoskillit.execution.quota import ( + QuotaFetchResult, + QuotaStatus, + QuotaWindowEntry, + _read_cache, + _write_cache, + ) + + resets_at = datetime(2026, 2, 27, 20, 15, tzinfo=UTC) + result = QuotaFetchResult( + windows={"five_hour": QuotaWindowEntry(utilization=75.0, resets_at=resets_at)}, + binding=QuotaStatus(utilization=75.0, resets_at=resets_at, window_name="five_hour"), + ) + cache_path = tmp_path / "usage_cache.json" + _write_cache(str(cache_path), result) + recovered = _read_cache(str(cache_path), max_age=60) + assert recovered is not None + assert recovered.utilization == pytest.approx(75.0) + + def test_write_failure_does_not_raise(self, tmp_path): + from autoskillit.execution.quota import ( + QuotaFetchResult, + QuotaStatus, + QuotaWindowEntry, + _write_cache, + ) + + resets_at = datetime(2026, 2, 27, 20, 15, tzinfo=UTC) + result = QuotaFetchResult( + windows={"five_hour": QuotaWindowEntry(utilization=50.0, resets_at=resets_at)}, + binding=QuotaStatus(utilization=50.0, resets_at=resets_at, window_name="five_hour"), + ) + # Nonexistent parent directory — should log warning, not raise + _write_cache("/nonexistent/dir/cache.json", result) + + +class TestCacheSchemaVersion: + """Phase 4 (#711 Part B): quota cache schema versioning tests.""" + + def setup_method(self): + from autoskillit.execution.quota import _reset_schema_drift_logged_for_tests + + _reset_schema_drift_logged_for_tests() + + def test_write_cache_embeds_schema_version(self, tmp_path): + from autoskillit.execution.quota import ( + QuotaFetchResult, + QuotaStatus, + QuotaWindowEntry, + _write_cache, + ) + + result = QuotaFetchResult( + windows={"five_hour": QuotaWindowEntry(utilization=50.0, resets_at=None)}, + binding=QuotaStatus(utilization=50.0, resets_at=None, window_name="five_hour"), + ) + cache_path = tmp_path / "cache.json" + _write_cache(str(cache_path), result) + raw = json.loads(cache_path.read_text()) + assert raw["schema_version"] == 3 + + def test_write_cache_uses_write_versioned_json(self, tmp_path, monkeypatch): + from autoskillit.execution.quota import ( + QuotaFetchResult, + QuotaStatus, + QuotaWindowEntry, + _write_cache, + ) + + calls = [] + + def spy(path, payload, schema_version): + calls.append({"path": path, "schema_version": schema_version}) + # Call through to real impl + from autoskillit.core import write_versioned_json as real + + real(path, payload, schema_version) + + monkeypatch.setattr("autoskillit.execution.quota.write_versioned_json", spy) + + result = QuotaFetchResult( + windows={"five_hour": QuotaWindowEntry(utilization=50.0, resets_at=None)}, + binding=QuotaStatus(utilization=50.0, resets_at=None, window_name="five_hour"), + ) + cache_path = tmp_path / "cache.json" + _write_cache(str(cache_path), result) + assert len(calls) == 1 + assert calls[0]["schema_version"] == 3 + + def test_read_cache_schema_round_trip_returns_status(self, tmp_path): + from autoskillit.execution.quota import ( + QuotaFetchResult, + QuotaStatus, + QuotaWindowEntry, + _read_cache, + _write_cache, + ) + + resets_at = datetime(2026, 2, 27, 20, 15, tzinfo=UTC) + result = QuotaFetchResult( + windows={"five_hour": QuotaWindowEntry(utilization=60.0, resets_at=resets_at)}, + binding=QuotaStatus(utilization=60.0, resets_at=resets_at, window_name="five_hour"), + ) + cache_path = tmp_path / "cache.json" + _write_cache(str(cache_path), result) + status = _read_cache(str(cache_path), max_age=60) + assert status is not None + assert status.utilization == pytest.approx(60.0) + + def test_read_cache_missing_schema_version_returns_none(self, tmp_path): + from autoskillit.execution.quota import _read_cache + + cache_path = tmp_path / "cache.json" + old_data = { + "fetched_at": datetime.now(UTC).isoformat(), + "windows": {}, + "binding": {"utilization": 50.0, "resets_at": None, "window_name": "five_hour"}, + } + cache_path.write_text(json.dumps(old_data)) + assert _read_cache(str(cache_path), max_age=60) is None + + def test_read_cache_wrong_schema_version_returns_none(self, tmp_path): + from autoskillit.execution.quota import _read_cache + + cache_path = tmp_path / "cache.json" + old_data = { + "schema_version": 1, + "fetched_at": datetime.now(UTC).isoformat(), + "windows": {}, + "binding": {"utilization": 50.0, "resets_at": None, "window_name": "five_hour"}, + } + cache_path.write_text(json.dumps(old_data)) + assert _read_cache(str(cache_path), max_age=60) is None + + def test_read_cache_logs_drift_warning_on_schema_mismatch(self, tmp_path): + import structlog.testing + + from autoskillit.execution.quota import _read_cache + + cache_path = tmp_path / "cache.json" + old_data = { + "fetched_at": datetime.now(UTC).isoformat(), + "windows": {}, + "binding": {"utilization": 50.0, "resets_at": None, "window_name": "five_hour"}, + } + cache_path.write_text(json.dumps(old_data)) + + with structlog.testing.capture_logs() as cap: + _read_cache(str(cache_path), max_age=60) + + drift_logs = [r for r in cap if "quota_cache_schema_drift" in r.get("event", "")] + assert len(drift_logs) == 1 + assert "cache_path" in drift_logs[0] + assert drift_logs[0]["observed"] is None + + def test_read_cache_logs_drift_exactly_once_per_path_per_process(self, tmp_path): + import structlog.testing + + from autoskillit.execution.quota import _read_cache + + cache_path = tmp_path / "cache.json" + old_data = { + "fetched_at": datetime.now(UTC).isoformat(), + "windows": {}, + "binding": {"utilization": 50.0, "resets_at": None, "window_name": "five_hour"}, + } + cache_path.write_text(json.dumps(old_data)) + + with structlog.testing.capture_logs() as cap: + for _ in range(5): + _read_cache(str(cache_path), max_age=60) + + drift_logs = [r for r in cap if "quota_cache_schema_drift" in r.get("event", "")] + assert len(drift_logs) == 1 + + def test_read_cache_logs_drift_once_per_path_not_globally(self, tmp_path): + import structlog.testing + + from autoskillit.execution.quota import _read_cache + + for name in ("cache_a.json", "cache_b.json"): + path = tmp_path / name + path.write_text( + json.dumps( + { + "fetched_at": datetime.now(UTC).isoformat(), + "windows": {}, + "binding": {"utilization": 50.0}, + } + ) + ) + + with structlog.testing.capture_logs() as cap: + _read_cache(str(tmp_path / "cache_a.json"), max_age=60) + _read_cache(str(tmp_path / "cache_b.json"), max_age=60) + + drift_logs = [r for r in cap if "quota_cache_schema_drift" in r.get("event", "")] + assert len(drift_logs) == 2 + + def test_read_cache_drift_set_is_module_scoped_and_reset_helper_works(self, tmp_path): + import structlog.testing + + from autoskillit.execution.quota import ( + _read_cache, + _reset_schema_drift_logged_for_tests, + ) + + cache_path = tmp_path / "cache.json" + cache_path.write_text( + json.dumps( + { + "fetched_at": datetime.now(UTC).isoformat(), + "windows": {}, + "binding": {"utilization": 50.0}, + } + ) + ) + + with structlog.testing.capture_logs() as cap1: + _read_cache(str(cache_path), max_age=60) + assert len([r for r in cap1 if "quota_cache_schema_drift" in r.get("event", "")]) == 1 + + _reset_schema_drift_logged_for_tests() + + with structlog.testing.capture_logs() as cap2: + _read_cache(str(cache_path), max_age=60) + assert len([r for r in cap2 if "quota_cache_schema_drift" in r.get("event", "")]) == 1 + + @pytest.mark.anyio + async def test_old_format_cache_gets_rewritten_with_new_format_on_next_fetch( + self, tmp_path, monkeypatch + ): + from autoskillit.execution.quota import ( + QuotaFetchResult, + QuotaStatus, + QuotaWindowEntry, + check_and_sleep_if_needed, + ) + + cache_path = tmp_path / "cache.json" + # Write old-format cache (no schema_version) + old_data = { + "fetched_at": datetime.now(UTC).isoformat(), + "windows": {}, + "binding": {"utilization": 50.0, "resets_at": None, "window_name": "five_hour"}, + } + cache_path.write_text(json.dumps(old_data)) + + async def fake_fetch(credentials_path, **kwargs): + return QuotaFetchResult( + windows={"five_hour": QuotaWindowEntry(utilization=30.0, resets_at=None)}, + binding=QuotaStatus(utilization=30.0, resets_at=None, window_name="five_hour"), + ) + + monkeypatch.setattr("autoskillit.execution.quota._fetch_quota", fake_fetch) + config = make_quota_guard_config( + cache_path=str(cache_path), + credentials_path=str(tmp_path / "fake_creds.json"), + ) + await check_and_sleep_if_needed(config) + new_data = json.loads(cache_path.read_text()) + assert new_data["schema_version"] == 3 + + +class TestInvalidateCache: + def test_invalidate_cache_removes_existing_file(self, tmp_path): + from autoskillit.execution.quota import invalidate_cache + + cache_file = tmp_path / "quota_cache.json" + cache_file.write_text('{"version": 3}') + assert cache_file.exists() + + invalidate_cache(str(cache_file)) + + assert not cache_file.exists() + + def test_invalidate_cache_tolerates_missing_file(self, tmp_path): + from autoskillit.execution.quota import invalidate_cache + + missing_path = tmp_path / "nonexistent_cache.json" + invalidate_cache(str(missing_path)) # must not raise + + def test_invalidate_cache_logs_warning_on_permission_error(self, monkeypatch): + import structlog.testing + + from autoskillit.execution.quota import invalidate_cache + + def _raise_permission_error(self, missing_ok=False): + raise PermissionError("denied") + + from pathlib import Path + + monkeypatch.setattr(Path, "unlink", _raise_permission_error) + + with structlog.testing.capture_logs() as cap: + invalidate_cache("/some/path/cache.json") + + assert any("quota cache invalidation failed" in rec.get("event", "") for rec in cap) + + def test_invalidate_cache_expands_user_tilde(self, monkeypatch): + from pathlib import Path + + from autoskillit.execution.quota import invalidate_cache + + unlinked_paths: list[str] = [] + + def _capture_unlink(self, missing_ok=False): + unlinked_paths.append(str(self)) + + monkeypatch.setattr(Path, "unlink", _capture_unlink) + + invalidate_cache("~/some/cache.json") + + assert len(unlinked_paths) == 1 + assert not unlinked_paths[0].startswith("~") + assert "cache.json" in unlinked_paths[0] + + +class TestAPIWindowVocabularyContract: + """Contract tests: LONG_WINDOW_NAMES × default long_window_patterns → correct classification. + + These tests bind the vocabulary constants in quota.py to the config defaults in settings.py. + Any change to either that breaks this invariant will fail here immediately. + """ + + def test_long_window_names_all_match_default_patterns(self): + """Every name in LONG_WINDOW_NAMES must be classified as long by the default patterns.""" + from autoskillit.config.settings import QuotaGuardConfig + from autoskillit.execution.quota import LONG_WINDOW_NAMES, _is_long_window + + defaults = QuotaGuardConfig().long_window_patterns + for name in sorted(LONG_WINDOW_NAMES): + assert _is_long_window(name, defaults), ( + f"Known long window {name!r} not matched by default patterns {defaults!r}. " + "Either add a matching pattern to long_window_patterns defaults, " + "or remove the name from LONG_WINDOW_NAMES if it is no longer a long window." + ) + + def test_known_short_windows_not_classified_as_long_by_default(self): + """Windows not in LONG_WINDOW_NAMES must not match long patterns.""" + from autoskillit.config.settings import QuotaGuardConfig + from autoskillit.execution.quota import ( + KNOWN_QUOTA_WINDOW_NAMES, + LONG_WINDOW_NAMES, + _is_long_window, + ) + + defaults = QuotaGuardConfig().long_window_patterns + pure_short = KNOWN_QUOTA_WINDOW_NAMES - LONG_WINDOW_NAMES + for name in sorted(pure_short): + assert not _is_long_window(name, defaults), ( + f"Known short window {name!r} is being classified as long " + f"by patterns {defaults!r}. " + "This would apply the long threshold to a short window." + ) + + def test_long_window_names_constant_exists_and_is_nonempty(self): + """LONG_WINDOW_NAMES must be a non-empty frozenset exported from quota.py.""" + from autoskillit.execution import quota + + assert hasattr(quota, "LONG_WINDOW_NAMES") + assert isinstance(quota.LONG_WINDOW_NAMES, frozenset) + assert len(quota.LONG_WINDOW_NAMES) > 0 + + def test_known_quota_window_names_constant_exists_and_contains_long_names(self): + """KNOWN_QUOTA_WINDOW_NAMES must include all entries from LONG_WINDOW_NAMES.""" + from autoskillit.execution.quota import KNOWN_QUOTA_WINDOW_NAMES, LONG_WINDOW_NAMES + + assert LONG_WINDOW_NAMES.issubset(KNOWN_QUOTA_WINDOW_NAMES), ( + f"LONG_WINDOW_NAMES contains names not in KNOWN_QUOTA_WINDOW_NAMES: " + f"{LONG_WINDOW_NAMES - KNOWN_QUOTA_WINDOW_NAMES}" + ) diff --git a/tests/execution/test_quota_sleep.py b/tests/execution/test_quota_sleep.py new file mode 100644 index 0000000000..c67a92dedf --- /dev/null +++ b/tests/execution/test_quota_sleep.py @@ -0,0 +1,592 @@ +"""Tests for execution/quota.py — check_and_sleep_if_needed, resets_at-None blocking, +integration, and novel window warnings.""" + +from __future__ import annotations + +import json +import time +from datetime import UTC, datetime, timedelta +from unittest.mock import patch + +import pytest + +from tests._helpers import make_quota_guard_config + +pytestmark = [pytest.mark.layer("execution"), pytest.mark.medium] + + +class TestCheckAndSleepIfNeeded: + @pytest.mark.anyio + async def test_disabled_returns_immediately_no_io(self, monkeypatch): + from autoskillit.execution.quota import check_and_sleep_if_needed + + config = make_quota_guard_config(enabled=False) + fetch_called = [] + + async def mock_fetch(*a, **kw): + fetch_called.append(1) + raise AssertionError("should not fetch") + + monkeypatch.setattr("autoskillit.execution.quota._fetch_quota", mock_fetch) + result = await check_and_sleep_if_needed(config) + assert result["should_sleep"] is False + assert fetch_called == [] + + @pytest.mark.anyio + async def test_below_threshold_returns_should_sleep_false(self, monkeypatch, tmp_path): + from autoskillit.execution.quota import ( + QuotaFetchResult, + QuotaStatus, + QuotaWindowEntry, + check_and_sleep_if_needed, + ) + + resets_at = datetime.now(UTC) + timedelta(hours=2) + config = make_quota_guard_config( + enabled=True, + credentials_path=str(tmp_path / ".credentials.json"), + cache_path=str(tmp_path / "cache.json"), + ) + + async def mock_fetch(path, **kwargs): + return QuotaFetchResult( + windows={"five_hour": QuotaWindowEntry(utilization=50.0, resets_at=resets_at)}, + binding=QuotaStatus( + utilization=50.0, resets_at=resets_at, window_name="five_hour" + ), + ) + + monkeypatch.setattr("autoskillit.execution.quota._fetch_quota", mock_fetch) + result = await check_and_sleep_if_needed(config) + assert result["should_sleep"] is False + assert result["utilization"] == pytest.approx(50.0) + + @pytest.mark.anyio + async def test_above_threshold_returns_should_sleep_true(self, monkeypatch, tmp_path): + from unittest.mock import AsyncMock + + from autoskillit.execution.quota import ( + QuotaFetchResult, + QuotaStatus, + QuotaWindowEntry, + check_and_sleep_if_needed, + ) + + resets_at = datetime.now(UTC) + timedelta(hours=2) + config = make_quota_guard_config( + enabled=True, + buffer_seconds=0, + credentials_path=str(tmp_path / ".credentials.json"), + cache_path=str(tmp_path / "cache.json"), + ) + + def _make_result(util: float) -> QuotaFetchResult: + return QuotaFetchResult( + windows={"five_hour": QuotaWindowEntry(utilization=util, resets_at=resets_at)}, + binding=QuotaStatus( + utilization=util, + resets_at=resets_at, + window_name="five_hour", + should_block=util >= 85.0, + effective_threshold=85.0, + ), + ) + + mock_fetch = AsyncMock(side_effect=[_make_result(90.0), _make_result(91.0)]) + monkeypatch.setattr("autoskillit.execution.quota._fetch_quota", mock_fetch) + result = await check_and_sleep_if_needed(config) + assert result["should_sleep"] is True + assert mock_fetch.call_count == 2 + assert result["sleep_seconds"] == pytest.approx(7200, abs=60) + + @pytest.mark.anyio + async def test_uses_fresh_cache_skips_fetch(self, monkeypatch, tmp_path): + from autoskillit.execution.quota import ( + QuotaFetchResult, + QuotaStatus, + QuotaWindowEntry, + _write_cache, + check_and_sleep_if_needed, + ) + + resets_at = datetime.now(UTC) + timedelta(hours=1) + cache_path = tmp_path / "cache.json" + _write_cache( + str(cache_path), + QuotaFetchResult( + windows={"five_hour": QuotaWindowEntry(utilization=40.0, resets_at=resets_at)}, + binding=QuotaStatus( + utilization=40.0, resets_at=resets_at, window_name="five_hour" + ), + ), + ) + config = make_quota_guard_config( + enabled=True, + cache_max_age=120, + credentials_path=str(tmp_path / ".credentials.json"), + cache_path=str(cache_path), + ) + fetch_called = [] + + async def mock_fetch(path): + fetch_called.append(1) + raise AssertionError("should not fetch when cache is fresh") + + monkeypatch.setattr("autoskillit.execution.quota._fetch_quota", mock_fetch) + result = await check_and_sleep_if_needed(config) + assert fetch_called == [] + assert result["should_sleep"] is False + assert result["utilization"] == pytest.approx(40.0) + + @pytest.mark.anyio + async def test_credentials_failure_returns_error_dict(self, tmp_path): + from autoskillit.execution.quota import check_and_sleep_if_needed + + config = make_quota_guard_config( + enabled=True, + credentials_path=str(tmp_path / "nonexistent.json"), + cache_path=str(tmp_path / "cache.json"), + ) + result = await check_and_sleep_if_needed(config) + assert result["should_sleep"] is False + assert "error" in result + + @pytest.mark.anyio + async def test_network_error_returns_error_dict(self, monkeypatch, tmp_path): + from autoskillit.execution.quota import check_and_sleep_if_needed + + config = make_quota_guard_config( + enabled=True, + credentials_path=str(tmp_path / ".credentials.json"), + cache_path=str(tmp_path / "cache.json"), + ) + expires_ms = int(time.time() * 1000) + 3_600_000 + (tmp_path / ".credentials.json").write_text( + json.dumps({"claudeAiOauth": {"accessToken": "tok", "expiresAt": expires_ms}}) + ) + + async def mock_fetch(path, **kwargs): + raise OSError("network down") + + monkeypatch.setattr("autoskillit.execution.quota._fetch_quota", mock_fetch) + result = await check_and_sleep_if_needed(config) + assert result["should_sleep"] is False + assert "error" in result + + @pytest.mark.anyio + async def test_resets_at_none_after_refetch_logs_warning_with_fallback( + self, monkeypatch, tmp_path + ): + """Second resets_at-is-None guard (after re-fetch) must emit + the 'blocking with fallback' warning.""" + from unittest.mock import AsyncMock + + import structlog.testing + + from autoskillit.execution.quota import ( + QuotaFetchResult, + QuotaStatus, + QuotaWindowEntry, + check_and_sleep_if_needed, + ) + + config = make_quota_guard_config( + enabled=True, + credentials_path=str(tmp_path / ".credentials.json"), + cache_path=str(tmp_path / "cache.json"), + ) + resets = datetime.now(UTC) + timedelta(hours=1) + # First fetch: above threshold, has resets_at so Gate 1 passes + first_result = QuotaFetchResult( + windows={"five_hour": QuotaWindowEntry(utilization=90.0, resets_at=resets)}, + binding=QuotaStatus( + utilization=90.0, + resets_at=resets, + window_name="five_hour", + should_block=True, + effective_threshold=85.0, + ), + ) + # Second fetch (re-fetch): above threshold, resets_at is None → Gate 2 fires + second_result = QuotaFetchResult( + windows={"five_hour": QuotaWindowEntry(utilization=90.0, resets_at=None)}, + binding=QuotaStatus( + utilization=90.0, + resets_at=None, + window_name="five_hour", + should_block=True, + effective_threshold=85.0, + ), + ) + + mock_fetch = AsyncMock(side_effect=[first_result, second_result]) + monkeypatch.setattr("autoskillit.execution.quota._fetch_quota", mock_fetch) + + with structlog.testing.capture_logs() as cap: + result = await check_and_sleep_if_needed(config) + + assert result["should_sleep"] is True + assert result["sleep_seconds"] > 0 + assert mock_fetch.call_count == 2 + # Exact message required by the rectify plan + assert any( + "quota above threshold but resets_at is None after re-fetch — blocking with fallback" + in rec.get("event", "") + for rec in cap + ) + + +class TestCheckAndSleepResetAtNoneBlocks: + @pytest.mark.anyio + async def test_above_threshold_resets_at_none_first_fetch_blocks(self, monkeypatch, tmp_path): + from autoskillit.execution.quota import ( + QuotaFetchResult, + QuotaStatus, + QuotaWindowEntry, + check_and_sleep_if_needed, + ) + + config = make_quota_guard_config( + enabled=True, + buffer_seconds=60, + credentials_path=str(tmp_path / ".credentials.json"), + cache_path=str(tmp_path / "cache.json"), + ) + + async def mock_fetch(path, **kwargs): + return QuotaFetchResult( + windows={"five_hour": QuotaWindowEntry(utilization=90.0, resets_at=None)}, + binding=QuotaStatus( + utilization=90.0, + resets_at=None, + window_name="five_hour", + should_block=True, + effective_threshold=85.0, + ), + ) + + monkeypatch.setattr("autoskillit.execution.quota._fetch_quota", mock_fetch) + result = await check_and_sleep_if_needed(config) + assert result["should_sleep"] is True + assert result["sleep_seconds"] > 0 + + @pytest.mark.anyio + async def test_above_threshold_resets_at_none_second_fetch_blocks(self, monkeypatch, tmp_path): + from unittest.mock import AsyncMock + + from autoskillit.execution.quota import ( + QuotaFetchResult, + QuotaStatus, + QuotaWindowEntry, + check_and_sleep_if_needed, + ) + + resets_at = datetime.now(UTC) + timedelta(hours=2) + config = make_quota_guard_config( + enabled=True, + buffer_seconds=60, + credentials_path=str(tmp_path / ".credentials.json"), + cache_path=str(tmp_path / "cache.json"), + ) + # First fetch has resets_at (passes first None guard), second fetch has None + first_result = QuotaFetchResult( + windows={"five_hour": QuotaWindowEntry(utilization=90.0, resets_at=resets_at)}, + binding=QuotaStatus( + utilization=90.0, + resets_at=resets_at, + window_name="five_hour", + should_block=True, + effective_threshold=85.0, + ), + ) + second_result = QuotaFetchResult( + windows={"five_hour": QuotaWindowEntry(utilization=90.0, resets_at=None)}, + binding=QuotaStatus( + utilization=90.0, + resets_at=None, + window_name="five_hour", + should_block=True, + effective_threshold=85.0, + ), + ) + mock_fetch = AsyncMock(side_effect=[first_result, second_result]) + monkeypatch.setattr("autoskillit.execution.quota._fetch_quota", mock_fetch) + result = await check_and_sleep_if_needed(config) + assert result["should_sleep"] is True + assert result["sleep_seconds"] > 0 + + @pytest.mark.anyio + async def test_cache_hit_resets_at_none_above_threshold_blocks(self, monkeypatch, tmp_path): + from autoskillit.execution.quota import ( + QuotaFetchResult, + QuotaStatus, + QuotaWindowEntry, + _write_cache, + check_and_sleep_if_needed, + ) + + cache_path = tmp_path / "cache.json" + # Write a cache entry with resets_at=None and above-threshold utilization + _write_cache( + str(cache_path), + QuotaFetchResult( + windows={"five_hour": QuotaWindowEntry(utilization=90.0, resets_at=None)}, + binding=QuotaStatus( + utilization=90.0, + resets_at=None, + window_name="five_hour", + should_block=True, + effective_threshold=85.0, + ), + ), + ) + config = make_quota_guard_config( + enabled=True, + buffer_seconds=60, + cache_max_age=120, + credentials_path=str(tmp_path / ".credentials.json"), + cache_path=str(cache_path), + ) + fetch_called = [] + + async def mock_fetch(path): + fetch_called.append(1) + raise AssertionError("should not reach re-fetch when first branch blocks") + + monkeypatch.setattr("autoskillit.execution.quota._fetch_quota", mock_fetch) + result = await check_and_sleep_if_needed(config) + assert result["should_sleep"] is True + assert fetch_called == [] + + @pytest.mark.anyio + async def test_fallback_sleep_uses_at_least_buffer_seconds(self, monkeypatch, tmp_path): + from autoskillit.execution.quota import ( + QuotaFetchResult, + QuotaStatus, + QuotaWindowEntry, + check_and_sleep_if_needed, + ) + + config = make_quota_guard_config( + enabled=True, + buffer_seconds=120, + credentials_path=str(tmp_path / ".credentials.json"), + cache_path=str(tmp_path / "cache.json"), + ) + + async def mock_fetch(path, **kwargs): + return QuotaFetchResult( + windows={"five_hour": QuotaWindowEntry(utilization=90.0, resets_at=None)}, + binding=QuotaStatus( + utilization=90.0, + resets_at=None, + window_name="five_hour", + should_block=True, + effective_threshold=85.0, + ), + ) + + monkeypatch.setattr("autoskillit.execution.quota._fetch_quota", mock_fetch) + result = await check_and_sleep_if_needed(config) + assert result["should_sleep"] is True + assert result["sleep_seconds"] >= 120 + + @pytest.mark.anyio + async def test_above_threshold_with_buffer_seconds_default(self, monkeypatch, tmp_path): + from unittest.mock import AsyncMock + + from autoskillit.execution.quota import ( + QuotaFetchResult, + QuotaStatus, + QuotaWindowEntry, + check_and_sleep_if_needed, + ) + + resets_at = datetime.now(UTC) + timedelta(hours=2) + # Do NOT override buffer_seconds — exercises the real default (60) + config = make_quota_guard_config( + enabled=True, + credentials_path=str(tmp_path / ".credentials.json"), + cache_path=str(tmp_path / "cache.json"), + ) + + def _r(util: float) -> QuotaFetchResult: + return QuotaFetchResult( + windows={"five_hour": QuotaWindowEntry(utilization=util, resets_at=resets_at)}, + binding=QuotaStatus( + utilization=util, + resets_at=resets_at, + window_name="five_hour", + should_block=util >= 85.0, + effective_threshold=85.0, + ), + ) + + mock_fetch = AsyncMock(side_effect=[_r(90.0), _r(90.0)]) + monkeypatch.setattr("autoskillit.execution.quota._fetch_quota", mock_fetch) + result = await check_and_sleep_if_needed(config) + assert result["should_sleep"] is True + assert result["sleep_seconds"] > 0 + + +class TestIntegration: + """Integration tests: write/read contract between execution.quota and hooks.quota_guard.""" + + def test_write_cache_then_quota_check_main_reads_it(self, tmp_path, monkeypatch): + """Cache written by _write_cache must be readable and actionable by quota_guard.main(). + + T-INT-1: Catches format drift between _write_cache (execution layer) and + _read_quota_cache (hook subprocess layer). + """ + import io + from contextlib import redirect_stdout + + from autoskillit.execution.quota import ( + QuotaFetchResult, + QuotaStatus, + QuotaWindowEntry, + _write_cache, + ) + from autoskillit.hooks.guards.quota_guard import main + + cache_path = tmp_path / "quota_cache.json" + _write_cache( + str(cache_path), + QuotaFetchResult( + windows={"five_hour": QuotaWindowEntry(utilization=95.0, resets_at=None)}, + binding=QuotaStatus( + utilization=95.0, + resets_at=None, + window_name="five_hour", + should_block=True, + effective_threshold=85.0, + ), + ), + ) + + stdin_text = json.dumps({"tool_name": "run_skill"}) + buf = io.StringIO() + with patch("sys.stdin", io.StringIO(stdin_text)): + with redirect_stdout(buf): + try: + main(cache_path_override=str(cache_path)) + except SystemExit: + pass + + out = buf.getvalue() + data = json.loads(out) + assert data["hookSpecificOutput"]["permissionDecision"] == "deny" + + +class TestFetchQuotaNovelWindowWarning: + """_fetch_quota must log a warning when it encounters a window name not in + KNOWN_QUOTA_WINDOW_NAMES. This surfaces Anthropic API vocabulary drift in + operator logs without disrupting the pipeline.""" + + @staticmethod + def _make_fake_httpx_client(api_response: dict): + """Return a fake httpx.AsyncClient instance that serves api_response for GET requests.""" + + class FakeResponse: + status_code = 200 + + def json(self): + return api_response + + def raise_for_status(self): + pass + + class FakeClient: + async def __aenter__(self): + return self + + async def __aexit__(self, *a): + pass + + async def get(self, *a, **kw): + return FakeResponse() + + return FakeClient() + + @pytest.mark.anyio + async def test_novel_window_name_logs_warning(self, monkeypatch): + """An unknown window name in the API response must produce a warning log entry.""" + import structlog.testing + + from autoskillit.config.settings import QuotaGuardConfig + from autoskillit.execution.quota import _fetch_quota + + resets_at = (datetime.now(UTC) + timedelta(hours=3)).isoformat() + api_response = { + "five_hour": {"utilization": 10.0, "resets_at": resets_at}, + "fortnight": {"utilization": 50.0, "resets_at": resets_at}, # unknown + } + cfg = QuotaGuardConfig() + + monkeypatch.setattr( + "httpx.AsyncClient", lambda **kw: self._make_fake_httpx_client(api_response) + ) + monkeypatch.setattr( + "autoskillit.execution.quota._read_credentials", + lambda path: "fake-token", + ) + + with structlog.testing.capture_logs() as cap: + await _fetch_quota( + cfg.credentials_path, + short_threshold=cfg.short_window_threshold, + long_threshold=cfg.long_window_threshold, + long_patterns=cfg.long_window_patterns, + short_enabled=cfg.short_window_enabled, + long_enabled=cfg.long_window_enabled, + ) + + novel_warning_records = [ + rec for rec in cap if rec.get("log_level") == "warning" and "novel_windows" in rec + ] + assert any( + "fortnight" in str(rec.get("novel_windows", [])) for rec in novel_warning_records + ), ( + f"Expected a warning with 'fortnight' in novel_windows for unknown window name. " + f"Got all captured records: {cap}" + ) + + @pytest.mark.anyio + async def test_all_known_windows_do_not_log_warning(self, monkeypatch): + """No warning must be logged when all API window names are in KNOWN_QUOTA_WINDOW_NAMES.""" + import structlog.testing + + from autoskillit.config.settings import QuotaGuardConfig + from autoskillit.execution.quota import KNOWN_QUOTA_WINDOW_NAMES, _fetch_quota + + resets_at = (datetime.now(UTC) + timedelta(hours=3)).isoformat() + api_response = { + name: {"utilization": 10.0, "resets_at": resets_at} + for name in KNOWN_QUOTA_WINDOW_NAMES + } + cfg = QuotaGuardConfig() + + monkeypatch.setattr( + "httpx.AsyncClient", lambda **kw: self._make_fake_httpx_client(api_response) + ) + monkeypatch.setattr( + "autoskillit.execution.quota._read_credentials", + lambda path: "fake-token", + ) + + with structlog.testing.capture_logs() as cap: + await _fetch_quota( + cfg.credentials_path, + short_threshold=cfg.short_window_threshold, + long_threshold=cfg.long_window_threshold, + long_patterns=cfg.long_window_patterns, + short_enabled=cfg.short_window_enabled, + long_enabled=cfg.long_window_enabled, + ) + + novel_warnings = [ + rec for rec in cap if rec.get("log_level") == "warning" and "novel_windows" in rec + ] + assert not novel_warnings, ( + f"Unexpected novel-window warnings for known windows: {novel_warnings}" + ) diff --git a/tests/execution/test_readiness_helper_contract.py b/tests/execution/test_readiness_helper_contract.py index 0c917d374f..fd1d81993a 100644 --- a/tests/execution/test_readiness_helper_contract.py +++ b/tests/execution/test_readiness_helper_contract.py @@ -21,6 +21,10 @@ import ast from pathlib import Path +import pytest + +pytestmark = [pytest.mark.layer("execution"), pytest.mark.small] + _TESTS_ROOT = Path(__file__).parent.parent diff --git a/tests/execution/test_recording.py b/tests/execution/test_recording.py index 1aefd09ecd..79bbce1987 100644 --- a/tests/execution/test_recording.py +++ b/tests/execution/test_recording.py @@ -9,15 +9,18 @@ import pytest -from autoskillit.core.types import SubprocessRunner, TerminationReason -from autoskillit.execution.commands import build_full_headless_cmd +from autoskillit.core.types import DirectInstall, OutputFormat, SubprocessRunner, TerminationReason +from autoskillit.execution.commands import build_skill_session_cmd from autoskillit.execution.recording import ( RecordingSubprocessRunner, ReplayingSubprocessRunner, ScenarioReplayError, _extract_model, ) -from tests.conftest import MockSubprocessRunner, _make_result +from tests.conftest import _make_result +from tests.fakes import MockSubprocessRunner + +pytestmark = [pytest.mark.layer("execution"), pytest.mark.small] @dataclass @@ -166,19 +169,19 @@ def test_extract_model_missing(): assert _extract_model(args) == "" -# --- T7: SCENARIO_STEP_NAME in cmd from build_full_headless_cmd --- +# --- T7: SCENARIO_STEP_NAME in cmd from build_skill_session_cmd --- _BASE_CMD_ARGS = dict( cwd="/tmp", completion_marker="DONE", model=None, - plugin_dir="/plugins", - output_format_value="stream-json", + plugin_source=DirectInstall(plugin_dir=Path("/plugins")), + output_format=OutputFormat.STREAM_JSON, ) -def test_build_full_headless_cmd_injects_scenario_step_name(): - spec = build_full_headless_cmd( +def test_build_skill_session_cmd_injects_scenario_step_name(): + spec = build_skill_session_cmd( "/investigate foo", scenario_step_name="investigate", **_BASE_CMD_ARGS, @@ -187,11 +190,11 @@ def test_build_full_headless_cmd_injects_scenario_step_name(): assert not any("SCENARIO_STEP_NAME" in tok for tok in spec.cmd) -# --- T8: build_full_headless_cmd without scenario_step_name --- +# --- T8: build_skill_session_cmd without scenario_step_name --- -def test_build_full_headless_cmd_no_scenario_step_name(): - spec = build_full_headless_cmd( +def test_build_skill_session_cmd_no_scenario_step_name(): + spec = build_skill_session_cmd( "/investigate foo", **_BASE_CMD_ARGS, ) @@ -199,31 +202,6 @@ def test_build_full_headless_cmd_no_scenario_step_name(): assert not any("SCENARIO_STEP_NAME" in tok for tok in spec.cmd) -# --- T9: run_headless_core passes scenario_step_name through --- - - -@pytest.mark.anyio -async def test_run_headless_core_injects_scenario_step_name(tmp_path): - """run_headless_core passes step_name as scenario_step_name and routes it via env kwarg.""" - from autoskillit.config import AutomationConfig - from autoskillit.execution.headless import run_headless_core - from autoskillit.pipeline import DefaultGateState - from autoskillit.server._factory import make_context - - mock_runner = MockSubprocessRunner() - mock_runner.set_default(_make_result()) - ctx = make_context(AutomationConfig(), runner=mock_runner, plugin_dir=str(tmp_path)) - ctx.gate = DefaultGateState(enabled=True) - - await run_headless_core("/investigate foo", str(tmp_path), ctx, step_name="investigate") - - cmd, _cwd, _timeout, kwargs = mock_runner.call_args_list[0] - assert cmd[0] != "env" - env = kwargs.get("env") - assert env is not None - assert env["SCENARIO_STEP_NAME"] == "investigate" - - # --- T-DERIVE: _derive_step_name_from_skill_command --- @@ -248,84 +226,6 @@ def test_derive_step_name_from_plain_skill(): assert _derive_step_name_from_skill_command("") == "" -# --- T-AUTO-DERIVE: run_headless_core auto-derives step name --- - - -@pytest.mark.anyio -async def test_run_headless_core_auto_derives_step_name_when_recording(tmp_path): - """When runner is RecordingSubprocessRunner and step_name is empty, - run_headless_core auto-derives step_name from the skill command.""" - from autoskillit.config import AutomationConfig - from autoskillit.execution.headless import run_headless_core - from autoskillit.pipeline import DefaultGateState - from autoskillit.server._factory import make_context - - mock_recorder = Mock() - mock_recorder.record_step.return_value = FakeStepResult( - cassette_exit_code=0, - cassette_path="", - cassette_duration_ms=100, - ) - inner = MockSubprocessRunner() - inner.set_default(_make_result()) - recording_runner = RecordingSubprocessRunner(recorder=mock_recorder, inner=inner) - - ctx = make_context(AutomationConfig(), runner=recording_runner, plugin_dir=str(tmp_path)) - ctx.gate = DefaultGateState(enabled=True) - ctx.config.linux_tracing.log_dir = str(tmp_path / "logs") - - # Call WITHOUT step_name — auto-derivation should kick in - await run_headless_core("/autoskillit:smoke-task", str(tmp_path), ctx) - - # record_step must be called with the derived step name - mock_recorder.record_step.assert_called_once() - call_kwargs = mock_recorder.record_step.call_args.kwargs - assert call_kwargs["step_name"] == "smoke-task", ( - f"Expected derived step_name 'smoke-task', got {call_kwargs['step_name']!r}" - ) - - -# --- T10: make_context wraps runner when RECORD_SCENARIO set --- - - -def test_make_context_wraps_runner_when_record_scenario(monkeypatch, tmp_path): - scenario_dir = tmp_path / "scenario" - scenario_dir.mkdir() - monkeypatch.setenv("RECORD_SCENARIO", "1") - monkeypatch.setenv("RECORD_SCENARIO_DIR", str(scenario_dir)) - monkeypatch.setenv("RECORD_SCENARIO_RECIPE", "smoke-test") - mock_recorder = Mock() - import api_simulator.claude as _api_sim_claude - - monkeypatch.setattr( - _api_sim_claude, "make_scenario_recorder", Mock(return_value=mock_recorder), raising=False - ) - mock_atexit = Mock() - monkeypatch.setattr("atexit.register", mock_atexit) - - from autoskillit.config import AutomationConfig - from autoskillit.server._factory import make_context - - ctx = make_context(AutomationConfig(), plugin_dir=str(tmp_path)) - assert isinstance(ctx.runner, RecordingSubprocessRunner) - mock_atexit.assert_not_called() - - -# --- T11: make_context default runner unchanged without env var --- - - -def test_make_context_default_runner_without_record_scenario(monkeypatch, tmp_path): - monkeypatch.delenv("RECORD_SCENARIO", raising=False) - monkeypatch.delenv("REPLAY_SCENARIO", raising=False) - - from autoskillit.config import AutomationConfig - from autoskillit.execution.process import DefaultSubprocessRunner - from autoskillit.server._factory import make_context - - ctx = make_context(AutomationConfig(), plugin_dir=str(tmp_path)) - assert isinstance(ctx.runner, DefaultSubprocessRunner) - - # --- T12: Protocol conformance --- @@ -477,78 +377,6 @@ async def test_sequencing_exhausted_session_falls_to_non_session(tmp_path): assert result.stdout == "non-session result" -# --- T20: make_context wires ReplayingSubprocessRunner when REPLAY_SCENARIO set --- - - -def test_make_context_wires_sequencing_runner_when_replay_scenario(monkeypatch, tmp_path): - """REPLAY_SCENARIO=1 + valid dir → ctx.runner is ReplayingSubprocessRunner.""" - replay_dir = tmp_path / "replay" - replay_dir.mkdir() - monkeypatch.setenv("REPLAY_SCENARIO", "1") - monkeypatch.setenv("REPLAY_SCENARIO_DIR", str(replay_dir)) - monkeypatch.delenv("RECORD_SCENARIO", raising=False) - - mock_scenario = Mock() - mock_scenario.step_sequence = [] - mock_player = Mock() - mock_player.scenario.return_value = mock_scenario - mock_player.build_session_map.return_value = {} - mock_make_player = Mock(return_value=mock_player) - - import api_simulator.claude as _api_sim_claude - - monkeypatch.setattr(_api_sim_claude, "make_scenario_player", mock_make_player, raising=False) - - from autoskillit.config import AutomationConfig - from autoskillit.server._factory import make_context - - ctx = make_context(AutomationConfig(), plugin_dir=str(tmp_path)) - assert isinstance(ctx.runner, ReplayingSubprocessRunner) - mock_make_player.assert_called_once() - call_kwargs = mock_make_player.call_args.kwargs - assert call_kwargs.get("scenario_dir") == str(replay_dir) - - -# --- T21: REPLAY_SCENARIO takes precedence over RECORD_SCENARIO --- - - -def test_replay_takes_precedence_over_record(monkeypatch, tmp_path): - """When both REPLAY and RECORD env vars set, REPLAY wins.""" - replay_dir = tmp_path / "replay" - replay_dir.mkdir() - monkeypatch.setenv("REPLAY_SCENARIO", "1") - monkeypatch.setenv("REPLAY_SCENARIO_DIR", str(replay_dir)) - monkeypatch.setenv("RECORD_SCENARIO", "1") - monkeypatch.setenv("RECORD_SCENARIO_DIR", str(replay_dir)) - - mock_scenario = Mock() - mock_scenario.step_sequence = [] - mock_player = Mock() - mock_player.scenario.return_value = mock_scenario - mock_player.build_session_map.return_value = {} - mock_recorder = Mock() - mock_make_recorder = Mock(return_value=mock_recorder) - - import api_simulator.claude as _api_sim_claude - - monkeypatch.setattr( - _api_sim_claude, "make_scenario_player", Mock(return_value=mock_player), raising=False - ) - monkeypatch.setattr( - _api_sim_claude, "make_scenario_recorder", mock_make_recorder, raising=False - ) - mock_atexit = Mock() - monkeypatch.setattr("atexit.register", mock_atexit) - - from autoskillit.config import AutomationConfig - from autoskillit.server._factory import make_context - - ctx = make_context(AutomationConfig(), plugin_dir=str(tmp_path)) - assert isinstance(ctx.runner, ReplayingSubprocessRunner) - mock_make_recorder.assert_not_called() # REPLAY takes precedence over RECORD - mock_atexit.assert_not_called() - - # --- T22: Cross-scenario session override (integration, requires api-simulator) --- @@ -619,9 +447,119 @@ def test_build_replay_runner_stores_player_on_runner(tmp_path, monkeypatch): monkeypatch.setattr( _api_sim_claude, "make_scenario_player", Mock(return_value=mock_player), raising=False ) + import weakref + + # weakref.finalize registers _exitfunc with atexit on first use in a process. + # Pre-set the class flag so that registration doesn't happen under the mock. + monkeypatch.setattr(weakref.finalize, "_registered_with_atexit", True) mock_atexit = Mock() monkeypatch.setattr("atexit.register", mock_atexit) result = build_replay_runner(str(tmp_path)) assert result.player is mock_player mock_atexit.assert_not_called() + + +# --- T-REC-SNAP: RecordingSubprocessRunner snapshots skill dir after recording --- + + +@pytest.mark.anyio +async def test_recording_runner_snapshots_skill_dir(tmp_path): + """After _record_session, skill-snapshots/{step_name}/ is written under scenario_dir.""" + ephemeral_dir = tmp_path / "autoskillit-sessions" / "headless-snap01" + skill_file = ephemeral_dir / ".claude" / "skills" / "investigate" / "SKILL.md" + skill_file.parent.mkdir(parents=True) + skill_file.write_text("# investigate\n", encoding="utf-8") + + cassette_path = tmp_path / "sessions" / "investigate" + cassette_path.mkdir(parents=True) + + mock_recorder = Mock() + mock_recorder.record_step.return_value = FakeStepResult( + cassette_exit_code=0, + cassette_path=str(cassette_path), + cassette_duration_ms=1000, + ) + + runner = RecordingSubprocessRunner(recorder=mock_recorder, inner=Mock()) + cmd = [ + "claude", + "--add-dir", + str(ephemeral_dir), + "--model", + "sonnet", + "--print", + "go", + ] + env = {"SCENARIO_STEP_NAME": "investigate"} + await runner(cmd, cwd=tmp_path, timeout=60, env=env, pty_mode=True) + + snapshot_dir = tmp_path / "skill-snapshots" / "investigate" + assert snapshot_dir.exists(), "skill-snapshots/investigate/ not created" + assert (snapshot_dir / ".claude" / "skills" / "investigate" / "SKILL.md").exists() + assert (snapshot_dir / "manifest.json").exists() + + +# --- T-REC-SNAP-SKIP: no --add-dir → no skill_snapshots created --- + + +@pytest.mark.anyio +async def test_recording_runner_no_ephemeral_dir_skips_snapshot(tmp_path): + """When cmd has no --add-dir, no skill-snapshots/ directory is created.""" + cassette_path = tmp_path / "sessions" / "investigate" + cassette_path.mkdir(parents=True) + + mock_recorder = Mock() + mock_recorder.record_step.return_value = FakeStepResult( + cassette_exit_code=0, + cassette_path=str(cassette_path), + cassette_duration_ms=500, + ) + + runner = RecordingSubprocessRunner(recorder=mock_recorder, inner=Mock()) + cmd = ["claude", "--model", "sonnet", "--print", "go"] + env = {"SCENARIO_STEP_NAME": "investigate"} + await runner(cmd, cwd=tmp_path, timeout=60, env=env, pty_mode=True) + + assert not (tmp_path / "skill-snapshots").exists() + + +# --- T-REPLAY-SNAP: ReplayingSubprocessRunner stores skill_snapshots --- + + +def test_replaying_runner_stores_skill_snapshots(tmp_path): + """ReplayingSubprocessRunner.skill_snapshots is populated when provided.""" + snap_path = tmp_path / "skill-snapshots" / "investigate" + snap_path.mkdir(parents=True) + skill_snapshots = {"investigate": snap_path} + + runner = ReplayingSubprocessRunner({}, {}, skill_snapshots=skill_snapshots) + + assert runner.skill_snapshots == {"investigate": snap_path} + + +# --- T-REPLAY-RESTORE: ReplayingSubprocessRunner.restore_skill_snapshot delegates correctly --- + + +def test_replaying_runner_restore_delegates_correctly(tmp_path): + """restore_skill_snapshot returns ValidatedAddDir when snapshot exists and files are copied.""" + snap_dir = tmp_path / "snapshot" + skill_md = snap_dir / ".claude" / "skills" / "investigate" / "SKILL.md" + skill_md.parent.mkdir(parents=True) + skill_md.write_text("# investigate\n", encoding="utf-8") + + runner = ReplayingSubprocessRunner({}, {}, skill_snapshots={"investigate": snap_dir}) + + ephemeral_root = tmp_path / "sessions" + result = runner.restore_skill_snapshot("investigate", ephemeral_root, "headless-xyz") + + assert result is not None + session_dir = ephemeral_root / "headless-xyz" + assert (session_dir / ".claude" / "skills" / "investigate" / "SKILL.md").exists() + + +def test_replaying_runner_restore_missing_step_returns_none(tmp_path): + """restore_skill_snapshot returns None when no snapshot for step_name.""" + runner = ReplayingSubprocessRunner({}, {}, skill_snapshots={}) + result = runner.restore_skill_snapshot("missing", tmp_path / "sessions", "headless-abc") + assert result is None diff --git a/tests/execution/test_recording_sigterm.py b/tests/execution/test_recording_sigterm.py index 0aaa21276c..d27d925006 100644 --- a/tests/execution/test_recording_sigterm.py +++ b/tests/execution/test_recording_sigterm.py @@ -15,9 +15,11 @@ import pytest -from autoskillit.core.readiness import readiness_sentinel_path +from autoskillit.core.runtime.readiness import readiness_sentinel_path from tests._subprocess_ready import wait_for_subprocess_ready +pytestmark = [pytest.mark.layer("execution"), pytest.mark.medium] + @pytest.mark.integration def test_sigterm_writes_scenario_json(tmp_path): diff --git a/tests/execution/test_recording_sigterm_early_term.py b/tests/execution/test_recording_sigterm_early_term.py index 643fbeeaa0..eb047e0759 100644 --- a/tests/execution/test_recording_sigterm_early_term.py +++ b/tests/execution/test_recording_sigterm_early_term.py @@ -24,6 +24,8 @@ import pytest +pytestmark = [pytest.mark.layer("execution"), pytest.mark.medium] + @pytest.mark.integration def test_sigterm_during_startup_no_hang(tmp_path): diff --git a/tests/execution/test_recording_skills.py b/tests/execution/test_recording_skills.py new file mode 100644 index 0000000000..5c5828db25 --- /dev/null +++ b/tests/execution/test_recording_skills.py @@ -0,0 +1,254 @@ +"""Tests for _recording_skills snapshot/restore helpers.""" + +from __future__ import annotations + +import json +from pathlib import Path + +import pytest + +from autoskillit.execution._recording_skills import ( + _extract_ephemeral_add_dir, + build_skills_manifest, + restore_skill_snapshot, + scan_skill_snapshots, + snapshot_skill_dir, +) + +pytestmark = [pytest.mark.layer("execution"), pytest.mark.small] + + +def _make_skill(skills_dir: Path, name: str, content: str = "# SKILL\n") -> Path: + skill_dir = skills_dir / name + skill_dir.mkdir(parents=True, exist_ok=True) + (skill_dir / "SKILL.md").write_text(content, encoding="utf-8") + return skill_dir + + +def _make_ephemeral_dir(tmp_path: Path, *skill_names: str) -> Path: + add_dir = tmp_path / "autoskillit-sessions" / "headless-abc123" + skills_dir = add_dir / ".claude" / "skills" + skills_dir.mkdir(parents=True, exist_ok=True) + for name in skill_names: + _make_skill(skills_dir, name) + return add_dir + + +# --- T-SNAP-1: snapshot copies full skill tree --- + + +def test_snapshot_skill_dir_copies_tree(tmp_path: Path) -> None: + add_dir = _make_ephemeral_dir(tmp_path, "investigate", "implement") + scenario_dir = tmp_path / "scenario" + scenario_dir.mkdir() + + result = snapshot_skill_dir(scenario_dir, "investigate", add_dir) + + assert result is not None + assert (result / ".claude" / "skills" / "investigate" / "SKILL.md").exists() + assert (result / ".claude" / "skills" / "implement" / "SKILL.md").exists() + + +# --- T-SNAP-2: snapshot writes manifest.json --- + + +def test_snapshot_skill_dir_writes_manifest(tmp_path: Path) -> None: + add_dir = _make_ephemeral_dir(tmp_path, "investigate") + scenario_dir = tmp_path / "scenario" + scenario_dir.mkdir() + + result = snapshot_skill_dir(scenario_dir, "investigate", add_dir) + + assert result is not None + manifest_path = result / "manifest.json" + assert manifest_path.exists() + manifest = json.loads(manifest_path.read_text()) + assert manifest["schema_version"] == 1 + assert "captured_at" in manifest + assert manifest["skill_count"] == 1 + assert "investigate" in manifest["skills"] + entry = manifest["skills"]["investigate"] + assert "content_sha256" in entry + assert "size_bytes" in entry + + +# --- T-SNAP-3: empty skills dir returns None --- + + +def test_snapshot_skill_dir_empty_skills_returns_none(tmp_path: Path) -> None: + add_dir = tmp_path / "autoskillit-sessions" / "headless-abc" + (add_dir / ".claude" / "skills").mkdir(parents=True) + scenario_dir = tmp_path / "scenario" + scenario_dir.mkdir() + + result = snapshot_skill_dir(scenario_dir, "step", add_dir) + + assert result is None + + +# --- T-SNAP-4: missing .claude/skills returns None --- + + +def test_snapshot_skill_dir_no_skills_subdir_returns_none(tmp_path: Path) -> None: + add_dir = tmp_path / "autoskillit-sessions" / "headless-abc" + add_dir.mkdir(parents=True) + scenario_dir = tmp_path / "scenario" + scenario_dir.mkdir() + + result = snapshot_skill_dir(scenario_dir, "step", add_dir) + + assert result is None + + +# --- T-REST-1: restore populates target dir and returns ValidatedAddDir --- + + +def test_restore_skill_snapshot_populates_target(tmp_path: Path) -> None: + snapshot_dir = tmp_path / "snapshot" + skills_src = snapshot_dir / ".claude" / "skills" / "investigate" + skills_src.mkdir(parents=True) + (skills_src / "SKILL.md").write_text("# investigate\n", encoding="utf-8") + + ephemeral_root = tmp_path / "sessions" + result = restore_skill_snapshot(snapshot_dir, ephemeral_root, "headless-test123") + + assert result is not None + session_dir = ephemeral_root / "headless-test123" + assert (session_dir / ".claude" / "skills" / "investigate" / "SKILL.md").exists() + assert result.path == str(session_dir) + + +# --- T-REST-2: snapshot path with no .claude/skills returns None --- + + +def test_restore_skill_snapshot_no_snapshot_returns_none(tmp_path: Path) -> None: + snapshot_dir = tmp_path / "empty-snapshot" + snapshot_dir.mkdir() + ephemeral_root = tmp_path / "sessions" + + result = restore_skill_snapshot(snapshot_dir, ephemeral_root, "headless-missing") + + assert result is None + + +# --- T-REST-3: restore preserves gated and ungated skill content byte-for-byte --- + + +def test_restore_preserves_gated_state(tmp_path: Path) -> None: + snapshot_dir = tmp_path / "snapshot" + skills_src = snapshot_dir / ".claude" / "skills" + skills_src.mkdir(parents=True) + + ungated_dir = skills_src / "open-skill" + ungated_dir.mkdir() + ungated_content = "# open skill\nNormal skill." + (ungated_dir / "SKILL.md").write_text(ungated_content, encoding="utf-8") + + gated_dir = skills_src / "gated-skill" + gated_dir.mkdir() + gated_content = "# gated skill\ndisable-model-invocation: true\n" + (gated_dir / "SKILL.md").write_text(gated_content, encoding="utf-8") + + ephemeral_root = tmp_path / "sessions" + result = restore_skill_snapshot(snapshot_dir, ephemeral_root, "headless-abc") + + assert result is not None + session_dir = Path(result.path) + restored_open = (session_dir / ".claude" / "skills" / "open-skill" / "SKILL.md").read_text() + restored_gated = (session_dir / ".claude" / "skills" / "gated-skill" / "SKILL.md").read_text() + assert restored_open == ungated_content + assert restored_gated == gated_content + + +# --- T-EXTRACT-1: extracts ephemeral --add-dir path --- + + +def test_extract_ephemeral_add_dir_finds_shm_path() -> None: + cmd = ["claude", "--add-dir", "/dev/shm/autoskillit-sessions/headless-abc123", "--print", "go"] + result = _extract_ephemeral_add_dir(cmd) + assert result == Path("/dev/shm/autoskillit-sessions/headless-abc123") + + +# --- T-EXTRACT-2: skips non-ephemeral --add-dir --- + + +def test_extract_ephemeral_add_dir_skips_non_ephemeral() -> None: + cmd = ["claude", "--add-dir", "/home/user/project", "--print", "go"] + result = _extract_ephemeral_add_dir(cmd) + assert result is None + + +# --- T-EXTRACT-3: multiple --add-dir, returns only ephemeral --- + + +def test_extract_ephemeral_add_dir_multiple_dirs() -> None: + cmd = [ + "claude", + "--add-dir", + "/home/user/project", + "--add-dir", + "/tmp/autoskillit-sessions/headless-xyz", + "--print", + "go", + ] + result = _extract_ephemeral_add_dir(cmd) + assert result == Path("/tmp/autoskillit-sessions/headless-xyz") + + +# --- T-MANIFEST-1: build_skills_manifest structure --- + + +def test_build_skills_manifest_structure(tmp_path: Path) -> None: + skills_dir = tmp_path / "skills" + _make_skill(skills_dir, "investigate") + _make_skill(skills_dir, "implement") + + manifest = build_skills_manifest(skills_dir) + + assert manifest["schema_version"] == 1 + assert manifest["skill_count"] == 2 + assert "investigate" in manifest["skills"] + assert "implement" in manifest["skills"] + assert "content_sha256" in manifest["skills"]["investigate"] + assert "size_bytes" in manifest["skills"]["investigate"] + + +# --- T-MANIFEST-2: manifest detects gated skills --- + + +def test_manifest_detects_gated_skills(tmp_path: Path) -> None: + skills_dir = tmp_path / "skills" + _make_skill(skills_dir, "normal", "# normal skill") + _make_skill(skills_dir, "gated", "# gated\ndisable-model-invocation: true\n") + + manifest = build_skills_manifest(skills_dir) + + assert manifest["skills"]["normal"]["gated"] is False + assert manifest["skills"]["gated"]["gated"] is True + + +# --- T-SCAN-1: scan finds step subdirectories --- + + +def test_scan_skill_snapshots_finds_steps(tmp_path: Path) -> None: + scenario_dir = tmp_path / "scenario" + (scenario_dir / "skill-snapshots" / "step_a").mkdir(parents=True) + (scenario_dir / "skill-snapshots" / "step_b").mkdir(parents=True) + + result = scan_skill_snapshots(scenario_dir) + + assert "step_a" in result + assert "step_b" in result + assert result["step_a"] == scenario_dir / "skill-snapshots" / "step_a" + + +# --- T-SCAN-2: scan returns empty dict when no skill_snapshots dir --- + + +def test_scan_skill_snapshots_empty_returns_empty_dict(tmp_path: Path) -> None: + scenario_dir = tmp_path / "scenario" + scenario_dir.mkdir() + + result = scan_skill_snapshots(scenario_dir) + + assert result == {} diff --git a/tests/execution/test_remote_resolver.py b/tests/execution/test_remote_resolver.py index 49ff97782f..cdcc676156 100644 --- a/tests/execution/test_remote_resolver.py +++ b/tests/execution/test_remote_resolver.py @@ -1,4 +1,4 @@ -"""Unit tests for execution.remote_resolver.resolve_remote_repo.""" +"""Unit tests for execution.remote_resolver.resolve_remote_repo and resolve_remote_name.""" from __future__ import annotations @@ -8,6 +8,9 @@ import pytest from autoskillit.execution import resolve_remote_repo +from autoskillit.execution.remote_resolver import resolve_remote_name + +pytestmark = [pytest.mark.layer("execution"), pytest.mark.medium] # --------------------------------------------------------------------------- # Hint-path tests (no subprocess calls) @@ -140,3 +143,60 @@ def test_resolve_remote_has_timeout_in_source() -> None: for node in ast.walk(tree) ) assert has_wait_for, "resolve_remote_repo should use asyncio.wait_for for timeout protection" + + +@pytest.mark.anyio +async def test_resolve_remote_name_upstream_exists(tmp_path: Path) -> None: + """upstream remote with HTTPS URL → returns 'upstream'.""" + repo = _make_repo_with_remotes( + tmp_path, + origin="file:///local/clone/path", + upstream="https://github.com/org/repo.git", + ) + result = await resolve_remote_name(str(repo)) + assert result == "upstream" + + +@pytest.mark.anyio +async def test_resolve_remote_name_upstream_missing_falls_back_to_origin(tmp_path: Path) -> None: + """No upstream remote → falls back to 'origin'.""" + repo = _make_repo_with_remotes( + tmp_path, + origin="https://github.com/org/repo.git", + ) + result = await resolve_remote_name(str(repo)) + assert result == "origin" + + +@pytest.mark.anyio +async def test_resolve_remote_name_upstream_is_file_url_falls_back_to_origin( + tmp_path: Path, +) -> None: + """upstream has file:// URL (clone isolation) → rejected; falls back to 'origin'.""" + repo = _make_repo_with_remotes( + tmp_path, + origin="https://github.com/org/repo.git", + upstream="file:///local/worktree/clone", + ) + result = await resolve_remote_name(str(repo)) + assert result == "origin" + + +@pytest.mark.anyio +async def test_resolve_remote_name_both_missing_returns_origin(tmp_path: Path) -> None: + """No remotes at all → safe default 'origin'.""" + repo = _make_repo_with_remotes(tmp_path) + result = await resolve_remote_name(str(repo)) + assert result == "origin" + + +@pytest.mark.anyio +async def test_resolve_remote_name_upstream_ssh_url(tmp_path: Path) -> None: + """upstream with SSH URL is valid (not file://) → returns 'upstream'.""" + repo = _make_repo_with_remotes( + tmp_path, + origin="file:///local/clone", + upstream="git@github.com:org/repo.git", + ) + result = await resolve_remote_name(str(repo)) + assert result == "upstream" diff --git a/tests/execution/test_resume_concurrency.py b/tests/execution/test_resume_concurrency.py new file mode 100644 index 0000000000..878dc7fcab --- /dev/null +++ b/tests/execution/test_resume_concurrency.py @@ -0,0 +1,41 @@ +"""T6: Concurrency guard for resume.""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from autoskillit.execution.session._session_state import SessionStateLock + +pytestmark = [pytest.mark.layer("execution"), pytest.mark.small] + + +class TestResumeFileLock: + def test_concurrent_lock_blocked(self, tmp_path: Path) -> None: + lock_a = SessionStateLock(tmp_path) + lock_b = SessionStateLock(tmp_path) + assert lock_a.acquire() is True + assert lock_b.acquire() is False + lock_a.release() + + def test_lock_released_allows_next(self, tmp_path: Path) -> None: + lock_a = SessionStateLock(tmp_path) + lock_b = SessionStateLock(tmp_path) + assert lock_a.acquire() is True + lock_a.release() + assert lock_b.acquire() is True + lock_b.release() + + def test_context_manager_releases(self, tmp_path: Path) -> None: + with SessionStateLock(tmp_path) as acquired: + assert acquired is True + lock_b = SessionStateLock(tmp_path) + assert lock_b.acquire() is True + lock_b.release() + + def test_double_release_is_safe(self, tmp_path: Path) -> None: + lock = SessionStateLock(tmp_path) + lock.acquire() + lock.release() + lock.release() diff --git a/tests/execution/test_resume_prompt.py b/tests/execution/test_resume_prompt.py new file mode 100644 index 0000000000..c9d4e90307 --- /dev/null +++ b/tests/execution/test_resume_prompt.py @@ -0,0 +1,75 @@ +"""T4: Resume prompt injection includes completed work.""" + +from __future__ import annotations + +import pytest + +from autoskillit.core.types._type_checkpoint import SessionCheckpoint +from autoskillit.core.types._type_enums import OutputFormat +from autoskillit.execution.commands import _build_resume_context + +pytestmark = [pytest.mark.layer("execution"), pytest.mark.small] + + +class TestBuildResumeContext: + def test_lists_completed_items(self) -> None: + cp = SessionCheckpoint( + completed_items=["issue/1", "issue/2"], + step_name="fleet_dispatch", + ) + ctx = _build_resume_context(cp) + assert "COMPLETED: issue/1" in ctx + assert "COMPLETED: issue/2" in ctx + + def test_includes_step_name(self) -> None: + cp = SessionCheckpoint(completed_items=["x"], step_name="my_step") + ctx = _build_resume_context(cp) + assert "Last active step: my_step" in ctx + + def test_skip_instruction(self) -> None: + cp = SessionCheckpoint(completed_items=["a"]) + ctx = _build_resume_context(cp) + assert "MUST be skipped" in ctx + + def test_no_step_name_omits_line(self) -> None: + cp = SessionCheckpoint(completed_items=["a"], step_name="") + ctx = _build_resume_context(cp) + assert "Last active step" not in ctx + + +class TestBuildSkillSessionCmdResume: + BASE = dict( + skill_command="Use /test-skill", + cwd="/tmp/test", + completion_marker="%%DONE%%", + model=None, + plugin_source=None, + output_format=OutputFormat.STREAM_JSON, + resume_session_id="sess-123", + ) + + def test_resume_without_checkpoint(self) -> None: + from autoskillit.execution.commands import build_skill_session_cmd + + spec = build_skill_session_cmd(**self.BASE) + prompt = spec.cmd[spec.cmd.index("-p") + 1] if "-p" in spec.cmd else "" + assert "interrupted before completion" in prompt + assert "RESUME CONTEXT" not in prompt + + def test_resume_with_checkpoint(self) -> None: + from autoskillit.execution.commands import build_skill_session_cmd + + cp = SessionCheckpoint(completed_items=["done_a", "done_b"], step_name="step_x") + spec = build_skill_session_cmd(**self.BASE, resume_checkpoint=cp) + prompt = spec.cmd[spec.cmd.index("-p") + 1] if "-p" in spec.cmd else "" + assert "RESUME CONTEXT" in prompt + assert "COMPLETED: done_a" in prompt + assert "COMPLETED: done_b" in prompt + + def test_resume_with_empty_checkpoint_no_context(self) -> None: + from autoskillit.execution.commands import build_skill_session_cmd + + cp = SessionCheckpoint(completed_items=[], step_name="") + spec = build_skill_session_cmd(**self.BASE, resume_checkpoint=cp) + prompt = spec.cmd[spec.cmd.index("-p") + 1] if "-p" in spec.cmd else "" + assert "RESUME CONTEXT" not in prompt diff --git a/tests/execution/test_session_adjudication.py b/tests/execution/test_session_adjudication.py deleted file mode 100644 index e43f61f02a..0000000000 --- a/tests/execution/test_session_adjudication.py +++ /dev/null @@ -1,1817 +0,0 @@ -"""Tests for _compute_success, _compute_retry, _is_kill_anomaly, and adjudication consistency.""" - -from __future__ import annotations - -from collections.abc import Callable - -import pytest - -from autoskillit.core.types import ( - ChannelConfirmation, - RetryReason, - SessionOutcome, - TerminationReason, -) -from autoskillit.execution.session import ( - ClaudeSessionResult, - ContentState, - _check_expected_patterns, - _check_session_content, - _compute_outcome, - _compute_retry, - _compute_success, - _evaluate_content_state, - _is_kill_anomaly, - parse_session_result, -) - - -def _make_success_session(result: str = "done") -> ClaudeSessionResult: - return ClaudeSessionResult( - subtype="success", - is_error=False, - result=result, - session_id="s1", - ) - - -@pytest.fixture -def make_session() -> Callable[..., ClaudeSessionResult]: - def _factory( - subtype: str = "success", - is_error: bool = False, - result: str = "", - assistant_messages: list[str] | None = None, - ) -> ClaudeSessionResult: - return ClaudeSessionResult( - subtype=subtype, - is_error=is_error, - result=result, - session_id="test-session", - assistant_messages=assistant_messages or [], - ) - - return _factory - - -class TestComputeSuccess: - """_compute_success cross-validates all signals for unambiguous success.""" - - def test_all_good_is_success(self): - session = ClaudeSessionResult( - subtype="success", is_error=False, result="Done.", session_id="s1" - ) - assert ( - _compute_success(session, returncode=0, termination=TerminationReason.NATURAL_EXIT) - is True - ) - - def test_nonzero_exit_is_failure(self): - session = ClaudeSessionResult( - subtype="success", is_error=False, result="Done.", session_id="s1" - ) - assert ( - _compute_success(session, returncode=1, termination=TerminationReason.NATURAL_EXIT) - is False - ) - - def test_is_error_true_is_failure(self): - session = ClaudeSessionResult( - subtype="success", is_error=True, result="Error occurred", session_id="s1" - ) - assert ( - _compute_success(session, returncode=0, termination=TerminationReason.NATURAL_EXIT) - is False - ) - - def test_timed_out_is_failure(self): - session = ClaudeSessionResult( - subtype="success", is_error=False, result="Done.", session_id="s1" - ) - assert ( - _compute_success(session, returncode=0, termination=TerminationReason.TIMED_OUT) - is False - ) - - def test_stale_is_failure(self): - session = ClaudeSessionResult( - subtype="success", is_error=False, result="Done.", session_id="s1" - ) - assert ( - _compute_success(session, returncode=0, termination=TerminationReason.STALE) is False - ) - - def test_unknown_subtype_is_failure(self): - session = ClaudeSessionResult( - subtype="unknown", is_error=False, result="Done.", session_id="s1" - ) - assert ( - _compute_success(session, returncode=0, termination=TerminationReason.NATURAL_EXIT) - is False - ) - - def test_channel_b_bypasses_content_check(self): - """CHANNEL_B + COMPLETED + empty result → True (provenance bypass).""" - session = ClaudeSessionResult( - subtype="success", is_error=False, result="", session_id="s1" - ) - assert ( - _compute_success( - session, - returncode=0, - termination=TerminationReason.COMPLETED, - channel_confirmation=ChannelConfirmation.CHANNEL_B, - ) - is True - ) - - def test_channel_a_falls_through_to_content_check(self): - """CHANNEL_A + COMPLETED + empty result → False (content check required).""" - session = ClaudeSessionResult( - subtype="success", is_error=False, result="", session_id="s1" - ) - assert ( - _compute_success( - session, - returncode=0, - termination=TerminationReason.COMPLETED, - channel_confirmation=ChannelConfirmation.CHANNEL_A, - ) - is False - ) - - def test_unmonitored_falls_through_to_content_check(self): - """UNMONITORED delegates to normal gates.""" - session = ClaudeSessionResult( - subtype="success", is_error=False, result="", session_id="s1" - ) - assert ( - _compute_success( - session, - returncode=0, - termination=TerminationReason.NATURAL_EXIT, - channel_confirmation=ChannelConfirmation.UNMONITORED, - ) - is False - ) - - def test_natural_exit_channel_b_empty_result_true(self): - """NATURAL_EXIT + CHANNEL_B bypass fires before termination dispatch.""" - session = ClaudeSessionResult( - subtype="success", is_error=False, result="", session_id="s1" - ) - assert ( - _compute_success( - session, - returncode=0, - termination=TerminationReason.NATURAL_EXIT, - channel_confirmation=ChannelConfirmation.CHANNEL_B, - ) - is True - ) - - def test_natural_exit_channel_a_valid_result_true(self): - """NATURAL_EXIT + CHANNEL_A + valid content → True.""" - session = ClaudeSessionResult( - subtype="success", is_error=False, result="Done.", session_id="s1" - ) - assert ( - _compute_success( - session, - returncode=0, - termination=TerminationReason.NATURAL_EXIT, - channel_confirmation=ChannelConfirmation.CHANNEL_A, - ) - is True - ) - - -class TestComputeSuccessNaturalExitNonZero: - """NATURAL_EXIT with non-zero returncode. - - Default: failure (no bypass). Exception: post-completion kill bypass fires when - the session envelope is SUCCESS, result is non-empty, and the completion marker - is present — the signal is a teardown artifact, not a genuine CLI error. - """ - - def test_natural_exit_nonzero_returncode_without_marker_returns_false(self): - """NATURAL_EXIT + non-zero returncode + no completion marker → failure. - - Without a completion marker, we cannot distinguish a post-completion kill - from a genuine crash. Fail conservatively (PTY masking cannot be ruled out). - """ - session = ClaudeSessionResult( - subtype="success", is_error=False, result="done", session_id="s1" - ) - assert ( - _compute_success(session, returncode=1, termination=TerminationReason.NATURAL_EXIT) - is False - ) - - def test_completed_and_natural_exit_same_outcome_when_returncode_zero(self): - """COMPLETED and NATURAL_EXIT agree when returncode=0 (no PTY masking issue). - - Documents the symmetric case: asymmetry only matters when returncode != 0. - """ - session = ClaudeSessionResult( - subtype="success", is_error=False, result="done", session_id="s1" - ) - result_completed = _compute_success( - session, returncode=-15, termination=TerminationReason.COMPLETED - ) - result_natural = _compute_success( - session, returncode=0, termination=TerminationReason.NATURAL_EXIT - ) - assert result_completed is True - assert result_natural is True - - -class TestComputeSuccessNaturalExitPostCompletionKill: - """NATURAL_EXIT with non-zero returncode is success when completion marker is present.""" - - def test_natural_exit_sigkill_with_marker_is_success(self): - """NATURAL_EXIT + returncode -9 + SUCCESS subtype + completion marker → success. - - Documents the post-completion kill bypass: a process killed by an external - watchdog AFTER completing its work produces -9 on NATURAL_EXIT. The completion - marker in the result confirms the session finished before the kill. - """ - marker = "%%ORDER_UP%%" - session = ClaudeSessionResult( - subtype="success", - is_error=False, - result=f"fixes_applied = 2\n{marker}", - session_id="s1", - ) - assert ( - _compute_success( - session, - returncode=-9, - termination=TerminationReason.NATURAL_EXIT, - completion_marker=marker, - ) - is True - ) - - def test_natural_exit_sigkill_without_marker_remains_failure(self): - """No bypass when no completion_marker is configured — conservative path unchanged.""" - session = ClaudeSessionResult( - subtype="success", - is_error=False, - result="fixes_applied = 2", - session_id="s1", - ) - assert ( - _compute_success( - session, - returncode=-9, - termination=TerminationReason.NATURAL_EXIT, - completion_marker="", # no marker configured - ) - is False - ) - - def test_natural_exit_sigkill_marker_absent_from_result_remains_failure(self): - """Bypass does not fire when marker is configured but missing from result.""" - marker = "%%ORDER_UP%%" - session = ClaudeSessionResult( - subtype="success", - is_error=False, - result="partial output without marker", - session_id="s1", - ) - assert ( - _compute_success( - session, - returncode=-9, - termination=TerminationReason.NATURAL_EXIT, - completion_marker=marker, - ) - is False - ) - - def test_natural_exit_nonzero_non_success_subtype_remains_failure(self): - """Bypass does not fire for non-success subtypes even with marker present.""" - marker = "%%ORDER_UP%%" - session = ClaudeSessionResult( - subtype="unparseable", - is_error=False, - result=f"some output {marker}", - session_id="s1", - ) - assert ( - _compute_success( - session, - returncode=-9, - termination=TerminationReason.NATURAL_EXIT, - completion_marker=marker, - ) - is False - ) - - -class TestComputeRetry: - """_compute_retry cross-validates all signals for retry eligibility.""" - - def test_success_not_retriable(self): - session = ClaudeSessionResult( - subtype="success", is_error=False, result="Done.", session_id="s1" - ) - needs, reason = _compute_retry( - session, returncode=0, termination=TerminationReason.NATURAL_EXIT - ) - assert needs is False - assert reason == RetryReason.NONE - - def test_max_turns_is_retriable(self): - session = ClaudeSessionResult( - subtype="error_max_turns", is_error=False, result="", session_id="s1" - ) - needs, reason = _compute_retry( - session, returncode=1, termination=TerminationReason.NATURAL_EXIT - ) - assert needs is True - assert reason == RetryReason.RESUME - - def test_context_exhaustion_is_retriable(self): - session = ClaudeSessionResult( - subtype="success", is_error=True, result="Prompt is too long", session_id="s1" - ) - needs, reason = _compute_retry( - session, returncode=1, termination=TerminationReason.NATURAL_EXIT - ) - assert needs is True - assert reason == RetryReason.RESUME - - def test_empty_output_exit_zero_is_retriable(self): - """Infrastructure failure: session never ran, CLI exited cleanly. - - NATURAL_EXIT + rc=0 + empty_output subtype means the session process - exited before writing any output — NOT a context exhaustion. The retry - reason must be EMPTY_OUTPUT, not RESUME, because no partial progress - exists on disk. RESUME is reserved for context/turn limit cases. - """ - session = ClaudeSessionResult( - subtype="empty_output", is_error=True, result="", session_id="" - ) - needs, reason = _compute_retry( - session, returncode=0, termination=TerminationReason.NATURAL_EXIT - ) - assert needs is True - assert reason == RetryReason.EMPTY_OUTPUT - - def test_empty_output_exit_one_not_retriable(self): - """Real failure: CLI crashed with empty output.""" - session = ClaudeSessionResult( - subtype="empty_output", is_error=True, result="", session_id="" - ) - needs, reason = _compute_retry( - session, returncode=1, termination=TerminationReason.NATURAL_EXIT - ) - assert needs is False - assert reason == RetryReason.NONE - - def test_timeout_not_retriable(self): - session = ClaudeSessionResult(subtype="timeout", is_error=True, result="", session_id="") - needs, reason = _compute_retry( - session, returncode=-1, termination=TerminationReason.TIMED_OUT - ) - assert needs is False - assert reason == RetryReason.NONE - - def test_unparseable_not_retriable(self): - session = ClaudeSessionResult( - subtype="unparseable", is_error=True, result="crash", session_id="" - ) - needs, reason = _compute_retry( - session, returncode=1, termination=TerminationReason.NATURAL_EXIT - ) - assert needs is False - assert reason == RetryReason.NONE - - def test_execution_error_not_retriable(self): - session = ClaudeSessionResult( - subtype="error_during_execution", - is_error=True, - result="tool error", - session_id="s1", - ) - needs, reason = _compute_retry( - session, returncode=1, termination=TerminationReason.NATURAL_EXIT - ) - assert needs is False - assert reason == RetryReason.NONE - - def test_channel_b_completed_no_retry(self): - """CHANNEL_B is authoritative on COMPLETED — no retry.""" - session = ClaudeSessionResult( - subtype="success", is_error=False, result="", session_id="s1" - ) - needs, reason = _compute_retry( - session, - returncode=0, - termination=TerminationReason.COMPLETED, - channel_confirmation=ChannelConfirmation.CHANNEL_B, - ) - assert needs is False - assert reason == RetryReason.NONE - - def test_channel_a_completed_kill_anomaly_retries(self): - """CHANNEL_A + empty result → retry RESUME (kill anomaly suspected).""" - session = ClaudeSessionResult( - subtype="success", is_error=False, result="", session_id="s1" - ) - needs, reason = _compute_retry( - session, - returncode=0, - termination=TerminationReason.COMPLETED, - channel_confirmation=ChannelConfirmation.CHANNEL_A, - ) - assert needs is True - assert reason == RetryReason.RESUME - - def test_natural_exit_channel_b_no_retry(self): - """CHANNEL_B confirmation skips kill-anomaly check on NATURAL_EXIT.""" - session = ClaudeSessionResult( - subtype="success", is_error=False, result="", session_id="s1" - ) - needs, reason = _compute_retry( - session, - returncode=0, - termination=TerminationReason.NATURAL_EXIT, - channel_confirmation=ChannelConfirmation.CHANNEL_B, - ) - assert needs is False - assert reason == RetryReason.NONE - - def test_natural_exit_channel_a_no_retry(self): - """CHANNEL_A confirmation skips kill-anomaly check on NATURAL_EXIT.""" - session = ClaudeSessionResult( - subtype="success", is_error=False, result="Done.", session_id="s1" - ) - needs, reason = _compute_retry( - session, - returncode=0, - termination=TerminationReason.NATURAL_EXIT, - channel_confirmation=ChannelConfirmation.CHANNEL_A, - ) - assert needs is False - assert reason == RetryReason.NONE - - -class TestComputeRetryUnparseable: - """_compute_retry distinguishes unparseable under COMPLETED vs NATURAL_EXIT.""" - - def test_unparseable_subtype_with_nonzero_returncode_should_retry(self): - """unparseable under COMPLETED means process was killed mid-write. - - The drain timeout expired before the result record was fully flushed. - The session likely completed; retry with resume. - """ - session = ClaudeSessionResult( - subtype="unparseable", is_error=True, result="partial", session_id="" - ) - needs, reason = _compute_retry( - session, returncode=-15, termination=TerminationReason.COMPLETED - ) - assert needs is True - assert reason == RetryReason.RESUME - - def test_unparseable_subtype_natural_exit_no_retry(self): - """unparseable under NATURAL_EXIT is a content failure, not retryable. - - The process exited cleanly with malformed output — this is not a timing issue. - """ - session = ClaudeSessionResult( - subtype="unparseable", is_error=True, result="", session_id="" - ) - needs, reason = _compute_retry( - session, returncode=1, termination=TerminationReason.NATURAL_EXIT - ) - assert needs is False - assert reason == RetryReason.NONE - - -class TestIsKillAnomaly: - """_is_kill_anomaly covers exactly the subtypes that represent kill artifacts.""" - - @pytest.mark.parametrize( - "subtype,result,expected", - [ - ("unparseable", "", True), # killed mid-write → partial NDJSON - ("empty_output", "", True), # killed before any stdout written - ("success", "", True), # killed after result record, content empty - ("success", "x", False), # success with content → NOT an anomaly - ("error_during_execution", "", False), # explicit API error, not a kill artifact - ("timeout", "", False), # timeout is a separate terminal state - ], - ) - def test_anomaly_classification(self, subtype: str, result: str, expected: bool) -> None: - session = ClaudeSessionResult( - subtype=subtype, - is_error=(subtype != "success"), - result=result, - session_id="", - errors=[], - token_usage=None, - ) - assert _is_kill_anomaly(session) is expected - - -class TestComputeRetrySuccessEmptyResult: - """_compute_retry for success subtype with empty result under COMPLETED termination.""" - - def test_success_empty_result_completed_rc0_is_retriable(self) -> None: - """success + "" + COMPLETED + rc=0 must be retriable (drain-race glitch).""" - session = ClaudeSessionResult( - subtype="success", - is_error=False, - result="", - session_id="s1", - errors=[], - token_usage=None, - ) - retriable, reason = _compute_retry( - session, returncode=0, termination=TerminationReason.COMPLETED - ) - assert retriable is True - assert reason == RetryReason.RESUME - - def test_success_empty_result_completed_negative_rc_is_retriable(self) -> None: - """success + "" + COMPLETED + rc=-15 (SIGTERM kill) must also be retriable.""" - session = ClaudeSessionResult( - subtype="success", - is_error=False, - result="", - session_id="s1", - errors=[], - token_usage=None, - ) - retriable, reason = _compute_retry( - session, returncode=-15, termination=TerminationReason.COMPLETED - ) - assert retriable is True - assert reason == RetryReason.RESUME - - def test_success_nonempty_result_completed_is_not_retriable(self) -> None: - """success + non-empty result + COMPLETED must NOT be retriable (genuine success).""" - session = ClaudeSessionResult( - subtype="success", - is_error=False, - result="Done. %%ORDER_UP%%", - session_id="s1", - errors=[], - token_usage=None, - ) - retriable, _ = _compute_retry( - session, returncode=-15, termination=TerminationReason.COMPLETED - ) - assert retriable is False - - def test_success_empty_result_natural_exit_zero_rc_is_retriable(self) -> None: - """Transient empty result: session exited cleanly but result field is empty. - - NATURAL_EXIT + rc=0 + SUCCESS subtype + empty result covers any case where - the Claude process self-exited cleanly without writing output — including - CLAUDE_CODE_EXIT_AFTER_STOP_DELAY timer-based exits AND transient API failures. - This is NOT a context exhaustion (no jsonl marker, no error markers). The retry - reason must be EMPTY_OUTPUT, not RESUME: no partial progress on disk. - """ - session = ClaudeSessionResult( - subtype="success", - is_error=False, - result="", - session_id="s1", - errors=[], - token_usage=None, - ) - retriable, reason = _compute_retry( - session, returncode=0, termination=TerminationReason.NATURAL_EXIT - ) - assert retriable is True - assert reason == RetryReason.EMPTY_OUTPUT - - def test_empty_output_completed_negative_rc_is_retriable(self) -> None: - """empty_output + COMPLETED + rc=-15 must be retriable. - - Process was killed by infrastructure before writing any stdout. - """ - session = ClaudeSessionResult( - subtype="empty_output", - is_error=True, - result="", - session_id="", - errors=[], - token_usage=None, - ) - retriable, reason = _compute_retry( - session, returncode=-15, termination=TerminationReason.COMPLETED - ) - assert retriable is True - assert reason == RetryReason.RESUME - - def test_context_exhausted_natural_exit_zero_rc_emits_resume(self) -> None: - """Context exhaustion on NATURAL_EXIT still emits RESUME. - - When _is_context_exhausted() returns True (jsonl_context_exhausted=True), - NATURAL_EXIT + kill_anomaly must still emit RESUME — partial context - progress exists on disk and is safe to resume. - """ - session = ClaudeSessionResult( - subtype="success", - is_error=False, - result="", - session_id="", - jsonl_context_exhausted=True, - ) - needs, reason = _compute_retry( - session, returncode=0, termination=TerminationReason.NATURAL_EXIT - ) - assert needs is True - assert reason == RetryReason.RESUME # context exhausted = partial progress exists - - def test_completed_kill_anomaly_still_emits_resume(self) -> None: - """Infrastructure kill (COMPLETED) + kill anomaly → RESUME unchanged. - - COMPLETED termination means infrastructure killed the process after channel - confirmation — the session made progress, partial results exist. RESUME - is correct here regardless of _is_context_exhausted(). - """ - session = ClaudeSessionResult(subtype="success", is_error=False, result="", session_id="") - needs, reason = _compute_retry( - session, - returncode=0, - termination=TerminationReason.COMPLETED, - channel_confirmation=ChannelConfirmation.CHANNEL_A, - ) - assert needs is True - assert reason == RetryReason.RESUME # infrastructure kill — progress existed - - -class TestComputeSuccessCompletionMarker: - """_compute_success tests unique to test_session_result.py (completion_marker path).""" - - def test_nonzero_returncode_recoverable_path(self): - s = ClaudeSessionResult( - subtype="success", is_error=False, result="great output", session_id="s1" - ) - assert _compute_success(s, 1, TerminationReason.COMPLETED) is True - - def test_failure_subtype_false_for_more_subtypes(self): - for subtype in ("empty_output", "unparseable", "timeout"): - s = ClaudeSessionResult( - subtype=subtype, is_error=False, result="some text", session_id="s1" - ) - assert _compute_success(s, 0, TerminationReason.NATURAL_EXIT) is False - - def test_missing_completion_marker_false(self): - s = _make_success_session("result without marker") - assert ( - _compute_success(s, 0, TerminationReason.NATURAL_EXIT, completion_marker="%%DONE%%") - is False - ) - - def test_result_is_only_marker_false(self): - marker = "%%DONE%%" - s = _make_success_session(marker) - assert ( - _compute_success(s, 0, TerminationReason.NATURAL_EXIT, completion_marker=marker) - is False - ) - - -class TestComputeSuccessRealisticInputs: - """_compute_success using parse_session_result() as input constructor.""" - - def test_empty_stdout_parses_to_empty_output_adjudicates_false(self): - session = parse_session_result("") - assert session.subtype == "empty_output" - assert session.is_error is True - assert _compute_success(session, 0, TerminationReason.NATURAL_EXIT) is False - - def test_garbled_stdout_parses_to_unparseable_adjudicates_false(self): - session = parse_session_result("Traceback (most recent call last):\n boom\n") - assert session.subtype == "unparseable" - assert session.is_error is True - assert _compute_success(session, 0, TerminationReason.NATURAL_EXIT) is False - - def test_empty_stdout_not_bypassed_by_completed_path(self): - session = parse_session_result("") - assert _compute_success(session, -15, TerminationReason.COMPLETED) is False - - def test_unparseable_not_bypassed_by_completed_path(self): - session = parse_session_result("garbled output not json\n") - assert _compute_success(session, -15, TerminationReason.COMPLETED) is False - - -class TestComputeRetryCompletedPath: - """_compute_retry unique test: COMPLETED termination + unparseable.""" - - def test_unparseable_on_completed_returns_resume(self): - s = ClaudeSessionResult( - subtype="unparseable", is_error=True, result="garbled", session_id="" - ) - needs, reason = _compute_retry(s, -15, TerminationReason.COMPLETED) - assert needs is True - assert reason == RetryReason.RESUME - - -@pytest.mark.parametrize("termination", list(TerminationReason)) -def test_compute_retry_handles_all_termination_reasons_without_raising( - termination: TerminationReason, -) -> None: - """Exhaustiveness guard: _compute_retry must return valid (bool, RetryReason) - for every TerminationReason value.""" - session = ClaudeSessionResult( - subtype="success", - result="done. %%ORDER_UP%%", - is_error=False, - session_id="abc", - errors=[], - ) - result = _compute_retry(session, returncode=0, termination=termination) - assert isinstance(result, tuple) and len(result) == 2 - assert isinstance(result[0], bool) - assert isinstance(result[1], RetryReason) - - -@pytest.mark.parametrize("termination", list(TerminationReason)) -def test_compute_success_handles_all_termination_reasons_without_raising( - termination: TerminationReason, -) -> None: - """Exhaustiveness guard for _compute_success: must return bool for every TerminationReason.""" - session = ClaudeSessionResult( - subtype="success", - result="done. %%ORDER_UP%%", - is_error=False, - session_id="abc", - errors=[], - ) - result = _compute_success( - session, - returncode=0, - termination=termination, - completion_marker="%%ORDER_UP%%", - ) - assert isinstance(result, bool) - - -def test_is_kill_anomaly_returns_true_for_interrupted_subtype(): - """'interrupted' subtype under COMPLETED termination must be retriable.""" - session = ClaudeSessionResult( - subtype="interrupted", - result="", - is_error=True, - session_id="abc", - errors=[], - ) - assert _is_kill_anomaly(session) is True - - -class TestAdjudicationConsistency: - """Contract documentation for known-impossible adjudication states. - - These tests intentionally cover (returncode, termination) combinations that - cannot occur in production. They serve as specification — documenting that the - adjudicator is deterministic and exhaustive, not that these paths are reachable. - """ - - @pytest.mark.parametrize( - "termination,channel,result_content,returncode,subtype,is_error," - "expected_success,expected_retry,completion_marker", - [ - # NATURAL_EXIT + CHANNEL_A: dead-end state - # (known bug, corrected by guard in _build_skill_result) - ( - TerminationReason.NATURAL_EXIT, - ChannelConfirmation.CHANNEL_A, - "", - 0, - "success", - False, - False, - False, - "", - ), - # NATURAL_EXIT + CHANNEL_A: valid success with content - ( - TerminationReason.NATURAL_EXIT, - ChannelConfirmation.CHANNEL_A, - "done", - 0, - "success", - False, - True, - False, - "", - ), - # NATURAL_EXIT + CHANNEL_B: contradiction - # (known bug, corrected by guard in _build_skill_result) - ( - TerminationReason.NATURAL_EXIT, - ChannelConfirmation.CHANNEL_B, - "", - 0, - "error_max_turns", - True, - True, - True, - "", - ), - # NATURAL_EXIT + CHANNEL_B: valid success - ( - TerminationReason.NATURAL_EXIT, - ChannelConfirmation.CHANNEL_B, - "", - 0, - "success", - False, - True, - False, - "", - ), - ( - TerminationReason.NATURAL_EXIT, - ChannelConfirmation.CHANNEL_B, - "done", - 0, - "success", - False, - True, - False, - "", - ), - # COMPLETED + CHANNEL_A: valid retriable (kill anomaly) - ( - TerminationReason.COMPLETED, - ChannelConfirmation.CHANNEL_A, - "", - -15, - "success", - False, - False, - True, - "", - ), - # COMPLETED + CHANNEL_B: valid success - ( - TerminationReason.COMPLETED, - ChannelConfirmation.CHANNEL_B, - "", - -15, - "success", - False, - True, - False, - "", - ), - # COMPLETED + CHANNEL_B: contradiction - # (known bug, corrected by guard in _build_skill_result) - ( - TerminationReason.COMPLETED, - ChannelConfirmation.CHANNEL_B, - "", - -15, - "error_max_turns", - True, - True, - True, - "", - ), - # UNMONITORED baselines — all should already be valid states - ( - TerminationReason.NATURAL_EXIT, - ChannelConfirmation.UNMONITORED, - "", - 0, - "success", - False, - False, - True, - "", - ), - ( - TerminationReason.NATURAL_EXIT, - ChannelConfirmation.UNMONITORED, - "done", - 0, - "success", - False, - True, - False, - "", - ), - ( - TerminationReason.TIMED_OUT, - ChannelConfirmation.UNMONITORED, - "", - -1, - "timeout", - True, - False, - False, - "", - ), - ( - TerminationReason.STALE, - ChannelConfirmation.UNMONITORED, - "", - 0, - "success", - False, - False, - False, - "", - ), - # NATURAL_EXIT + UNMONITORED: substantive result without marker - # (premature exit / early stop — retriable via EARLY_STOP) - pytest.param( - TerminationReason.NATURAL_EXIT, - ChannelConfirmation.UNMONITORED, - "Here is the PR context block with substantive content...", - 0, - "success", - False, - False, - True, - "%%ORDER_UP%%", - id="natural_exit-unmonitored-substantive_result_no_marker", - ), - ], - ) - def test_raw_adjudication_pair( - self, - termination: TerminationReason, - channel: ChannelConfirmation, - result_content: str, - returncode: int, - subtype: str, - is_error: bool, - expected_success: bool, - expected_retry: bool, - completion_marker: str, - ) -> None: - """Document exact raw outputs of the individual adjudication functions. - - Known bad states (dead end, contradiction) are documented as expected values - rather than as invariant failures — the guards in _build_skill_result correct - those states before they reach the orchestrator. - """ - session = ClaudeSessionResult( - subtype=subtype, - result=result_content, - is_error=is_error, - session_id="cross-val", - errors=[], - ) - success = _compute_success( - session, - returncode, - termination, - channel_confirmation=channel, - completion_marker=completion_marker, - ) - needs_retry, _ = _compute_retry( - session, - returncode, - termination, - channel_confirmation=channel, - completion_marker=completion_marker, - ) - assert success == expected_success - assert needs_retry == expected_retry - - def test_premature_exit_substantive_result_no_marker_is_early_stop(self) -> None: - """NATURAL_EXIT + UNMONITORED + substantive result without marker → EARLY_STOP. - - When the model produces substantive output but stops before emitting - the completion marker, the session is classified as retriable with - EARLY_STOP reason. This is the text-then-tool boundary fix. - """ - session = ClaudeSessionResult( - subtype="success", - result="Here is the PR context block with substantive content...", - is_error=False, - session_id="premature", - errors=[], - ) - success = _compute_success( - session, - returncode=0, - termination=TerminationReason.NATURAL_EXIT, - channel_confirmation=ChannelConfirmation.UNMONITORED, - completion_marker="%%ORDER_UP%%", - ) - needs_retry, reason = _compute_retry( - session, - returncode=0, - termination=TerminationReason.NATURAL_EXIT, - channel_confirmation=ChannelConfirmation.UNMONITORED, - completion_marker="%%ORDER_UP%%", - ) - assert success is False - assert needs_retry is True - assert reason == RetryReason.EARLY_STOP - - def test_channel_a_empty_result_raw_dead_end(self) -> None: - """Document: NATURAL_EXIT + CHANNEL_A + empty result is a dead end at raw function level. - - Both _compute_success and _compute_retry return False for this combination: - - _compute_success: CHANNEL_A falls through content check, empty result → False - - _compute_retry: NATURAL_EXIT + CHANNEL_A confirmation suppresses retry → False - - The composition guard in _build_skill_result escalates this to retriable. - See TestAdjudicationGuards.test_channel_a_empty_result_not_dead_end. - """ - session = ClaudeSessionResult( - subtype="success", - result="", - is_error=False, - session_id="regression", - errors=[], - ) - success = _compute_success( - session, - returncode=0, - termination=TerminationReason.NATURAL_EXIT, - channel_confirmation=ChannelConfirmation.CHANNEL_A, - ) - needs_retry, _ = _compute_retry( - session, - returncode=0, - termination=TerminationReason.NATURAL_EXIT, - channel_confirmation=ChannelConfirmation.CHANNEL_A, - ) - # Raw functions produce the dead-end state — this is the known bug at the - # individual function level, corrected at the _build_skill_result boundary. - assert success is False - assert needs_retry is False - - -class TestEarlyStop: - """Tests for EARLY_STOP retry classification.""" - - def test_natural_exit_substantive_content_no_marker_is_retriable(self) -> None: - """A session that exits cleanly with substantive output but without - the completion marker should be classified as RETRIABLE with EARLY_STOP, - because the model may have stopped early at a text-then-tool boundary. - """ - session = ClaudeSessionResult( - subtype="success", - result="Here is the PR context block with substantive content...", - is_error=False, - session_id="early-stop", - errors=[], - ) - outcome, reason = _compute_outcome( - session, - returncode=0, - termination=TerminationReason.NATURAL_EXIT, - completion_marker="%%ORDER_UP%%", - channel_confirmation=ChannelConfirmation.UNMONITORED, - ) - assert outcome == SessionOutcome.RETRIABLE - assert reason == RetryReason.EARLY_STOP - - def test_early_stop_not_triggered_without_marker(self) -> None: - """When no completion_marker is configured, EARLY_STOP should NOT fire. - - Sessions without a marker are not subject to early-stop detection. - """ - session = ClaudeSessionResult( - subtype="success", - result="Some output without any marker", - is_error=False, - session_id="no-marker", - errors=[], - ) - needs_retry, reason = _compute_retry( - session, - returncode=0, - termination=TerminationReason.NATURAL_EXIT, - channel_confirmation=ChannelConfirmation.UNMONITORED, - completion_marker="", - ) - assert needs_retry is False - assert reason == RetryReason.NONE - - def test_early_stop_not_triggered_when_marker_present(self) -> None: - """When the completion marker IS present, EARLY_STOP should NOT fire.""" - session = ClaudeSessionResult( - subtype="success", - result="Result with %%ORDER_UP%% marker present", - is_error=False, - session_id="has-marker", - errors=[], - ) - needs_retry, reason = _compute_retry( - session, - returncode=0, - termination=TerminationReason.NATURAL_EXIT, - channel_confirmation=ChannelConfirmation.UNMONITORED, - completion_marker="%%ORDER_UP%%", - ) - assert needs_retry is False - assert reason == RetryReason.NONE - - def test_early_stop_not_triggered_for_errors(self) -> None: - """Error sessions should not be classified as EARLY_STOP.""" - session = ClaudeSessionResult( - subtype="error_during_execution", - result="Error occurred but substantive output...", - is_error=True, - session_id="error-session", - errors=["something broke"], - ) - needs_retry, reason = _compute_retry( - session, - returncode=0, - termination=TerminationReason.NATURAL_EXIT, - channel_confirmation=ChannelConfirmation.UNMONITORED, - completion_marker="%%ORDER_UP%%", - ) - # is_error sessions may trigger API-level retry via needs_retry property, - # but EARLY_STOP specifically should not fire for non-success subtypes - assert reason != RetryReason.EARLY_STOP - - def test_early_stop_not_triggered_for_empty_result(self) -> None: - """Empty result should be classified as kill anomaly, not EARLY_STOP.""" - session = ClaudeSessionResult( - subtype="success", - result="", - is_error=False, - session_id="empty", - errors=[], - ) - needs_retry, reason = _compute_retry( - session, - returncode=0, - termination=TerminationReason.NATURAL_EXIT, - channel_confirmation=ChannelConfirmation.UNMONITORED, - completion_marker="%%ORDER_UP%%", - ) - # Empty result triggers kill_anomaly path (EMPTY_OUTPUT), not EARLY_STOP. - # No context exhaustion detected → EMPTY_OUTPUT, not RESUME. - assert needs_retry is True - assert reason == RetryReason.EMPTY_OUTPUT - - -class TestArtifactValidation: - """Tests for expected_output_patterns artifact validation.""" - - def test_check_session_content_validates_expected_artifacts(self) -> None: - """When expected_output_patterns are configured, _check_session_content - must verify that the session result contains at least one match. - """ - session = ClaudeSessionResult( - subtype="success", - result="Done! %%ORDER_UP%%", - is_error=False, - session_id="no-artifact", - errors=[], - ) - # With marker present but no matching artifact - result = _check_session_content( - session, - completion_marker="%%ORDER_UP%%", - expected_output_patterns=[r"https://github\.com/.*/pull/\d+"], - ) - assert result is False - - def test_check_session_content_passes_with_matching_artifact(self) -> None: - """When artifacts match, content check passes.""" - session = ClaudeSessionResult( - subtype="success", - result="PR created: https://github.com/user/repo/pull/42 %%ORDER_UP%%", - is_error=False, - session_id="has-artifact", - errors=[], - ) - result = _check_session_content( - session, - completion_marker="%%ORDER_UP%%", - expected_output_patterns=[r"https://github\.com/.*/pull/\d+"], - ) - assert result is True - - def test_check_session_content_no_patterns_skips_validation(self) -> None: - """When no patterns are provided, artifact validation is skipped.""" - session = ClaudeSessionResult( - subtype="success", - result="Done! %%ORDER_UP%%", - is_error=False, - session_id="no-patterns", - errors=[], - ) - result = _check_session_content( - session, - completion_marker="%%ORDER_UP%%", - expected_output_patterns=[], - ) - assert result is True - - def test_compute_outcome_threads_expected_output_patterns(self) -> None: - """_compute_outcome must thread expected_output_patterns through to - _compute_success and _check_session_content.""" - session = ClaudeSessionResult( - subtype="success", - result="Done! %%ORDER_UP%%", - is_error=False, - session_id="threaded", - errors=[], - ) - # Without patterns: success - outcome_no_patterns, _ = _compute_outcome( - session, - returncode=0, - termination=TerminationReason.NATURAL_EXIT, - completion_marker="%%ORDER_UP%%", - channel_confirmation=ChannelConfirmation.UNMONITORED, - expected_output_patterns=[], - ) - assert outcome_no_patterns == SessionOutcome.SUCCEEDED - - # With patterns that don't match: failed (EARLY_STOP since marker - # absent from _compute_success perspective when artifact check fails) - outcome_with_patterns, _ = _compute_outcome( - session, - returncode=0, - termination=TerminationReason.NATURAL_EXIT, - completion_marker="%%ORDER_UP%%", - channel_confirmation=ChannelConfirmation.UNMONITORED, - expected_output_patterns=[r"https://github\.com/.*/pull/\d+"], - ) - assert outcome_with_patterns != SessionOutcome.SUCCEEDED - - -class TestToolUseParsing: - """Tests for tool_use NDJSON record extraction.""" - - @staticmethod - def _result_line() -> str: - import json - - return json.dumps( - { - "type": "result", - "subtype": "success", - "is_error": False, - "result": "done", - "session_id": "s1", - } - ) - - @staticmethod - def _assistant_line(*content_blocks: dict) -> str: # type: ignore[type-arg] - import json - - return json.dumps( - { - "type": "assistant", - "message": {"content": list(content_blocks)}, - } - ) - - def test_parse_session_result_captures_tool_uses(self) -> None: - """parse_session_result must extract tool_use records.""" - ndjson = "\n".join( - [ - self._assistant_line( - {"type": "tool_use", "name": "Skill", "id": "tu_1"}, - {"type": "text", "text": "loading skill"}, - ), - self._result_line(), - ] - ) - session = parse_session_result(ndjson) - assert len(session.tool_uses) == 1 - assert session.tool_uses[0]["name"] == "Skill" - assert session.tool_uses[0]["id"] == "tu_1" - - def test_parse_session_result_no_tool_uses(self) -> None: - """Sessions without tool_use records have an empty list.""" - ndjson = "\n".join( - [ - self._assistant_line( - {"type": "text", "text": "just text"}, - ), - self._result_line(), - ] - ) - session = parse_session_result(ndjson) - assert session.tool_uses == [] - - def test_parse_session_result_multiple_tool_uses(self) -> None: - """Multiple tool_use records across messages are captured.""" - ndjson = "\n".join( - [ - self._assistant_line( - {"type": "tool_use", "name": "Write", "id": "tu_1"}, - ), - self._assistant_line( - {"type": "tool_use", "name": "Skill", "id": "tu_2"}, - ), - self._result_line(), - ] - ) - session = parse_session_result(ndjson) - assert len(session.tool_uses) == 2 - assert session.tool_uses[0]["name"] == "Write" - assert session.tool_uses[1]["name"] == "Skill" - - -class TestCheckExpectedPatterns: - """Unit tests for the standalone _check_expected_patterns function.""" - - def test_check_expected_patterns_present(self) -> None: - assert ( - _check_expected_patterns( - result="some text ---my-block--- more text", - patterns=["---my-block---"], - ) - is True - ) - - def test_check_expected_patterns_absent(self) -> None: - assert ( - _check_expected_patterns( - result="some text without the block", - patterns=["---my-block---"], - ) - is False - ) - - def test_check_expected_patterns_empty_patterns_always_true(self) -> None: - assert _check_expected_patterns(result="anything", patterns=[]) is True - - def test_check_expected_patterns_bold_wrapped_token_matches(self) -> None: - """Bold-wrapped token name must match after normalization.""" - result = "**plan_path** = /abs/path/plan.md\n%%ORDER_UP%%" - assert _check_expected_patterns(result, ["plan_path\\s*=\\s*/.+"]) is True - - def test_check_expected_patterns_italic_wrapped_token_matches(self) -> None: - """Italic-wrapped token name must match after normalization.""" - result = "*plan_path* = /abs/path/plan.md\n%%ORDER_UP%%" - assert _check_expected_patterns(result, ["plan_path\\s*=\\s*/.+"]) is True - - def test_check_expected_patterns_bold_verdict_matches(self) -> None: - """Bold-wrapped verdict token must match after normalization.""" - result = "**verdict** = GO\n%%ORDER_UP%%" - assert _check_expected_patterns(result, ["verdict\\s*=\\s*(GO|NO GO)"]) is True - - def test_check_expected_patterns_multiple_bold_tokens_all_match(self) -> None: - """Multiple bold-wrapped tokens must all match (AND semantics preserved).""" - result = ( - "**plan_path** = /abs/path/plan.md\n**plan_parts** = /abs/path/plan.md\n%%ORDER_UP%%" - ) - assert ( - _check_expected_patterns(result, ["plan_path\\s*=\\s*/.+", "plan_parts\\s*=\\s*/.+"]) - is True - ) - - def test_check_expected_patterns_bold_relative_path_still_fails(self) -> None: - """Bold wrapping on a relative path must still fail — normalization must not - mask a genuine contract violation (wrong value type).""" - result = "**worktree_path** = ../worktrees/impl\n%%ORDER_UP%%" - assert _check_expected_patterns(result, ["worktree_path\\s*=\\s*/.+"]) is False - - def test_check_expected_patterns_bold_absent_value_still_fails(self) -> None: - """Bold key with no value must still fail — semantic content must be present.""" - result = "**plan_path** =\n%%ORDER_UP%%" - assert _check_expected_patterns(result, ["plan_path\\s*=\\s*/.+"]) is False - - -class TestComputeSuccessChannelBPatterns: - """Channel B bypass must not skip expected_output_patterns validation.""" - - def test_compute_success_channel_b_bypass_with_missing_patterns_returns_false( - self, - ) -> None: - """Channel B bypass must not skip expected_output_patterns validation. - - When patterns are configured and absent from session.result, success must - be False (not True) even when channel_confirmation is CHANNEL_B. - """ - session = ClaudeSessionResult( - subtype="success", - result="Final message without the block.", # pattern absent - is_error=False, - session_id="s1", - assistant_messages=["Final message without the block."], - ) - success = _compute_success( - session=session, - returncode=0, - termination=TerminationReason.COMPLETED, - channel_confirmation=ChannelConfirmation.CHANNEL_B, - completion_marker="%%ORDER_UP%%", - expected_output_patterns=["---prepare-issue-result---"], - ) - assert success is False, "Channel B bypass must not skip expected_output_patterns check" - - def test_channel_b_bypass_fails_for_failure_subtype_with_injected_pattern( - self, - ) -> None: - """CHANNEL_B bypass must not return True for failure-subtype sessions. - - Even when synthesis has already run and injected a matching line into - session.result, CHANNEL_B bypass must refuse because session.session_complete - is False. - """ - from autoskillit.core.types import CliSubtype - - # Simulate: synthesis already ran and injected the line into result - session = ClaudeSessionResult( - subtype=CliSubtype.UNPARSEABLE, - is_error=True, - result="plan_path = /cwd/.autoskillit/temp/make-plan/arch_lens_2026.md\n", - session_id="s1", - ) - success = _compute_success( - session=session, - returncode=0, - termination=TerminationReason.COMPLETED, - channel_confirmation=ChannelConfirmation.CHANNEL_B, - expected_output_patterns=[r"plan_path\s*=\s*/.+"], - ) - assert success is False, "CHANNEL_B bypass must not accept failure-subtype sessions" - - -class TestSessionCompleteProperty: - """session_complete property must encode the canonical completion invariant.""" - - @pytest.mark.parametrize( - "subtype, is_error, expected", - [ - ("success", False, True), - ("error_max_turns", False, True), - ("error_during_execution", False, True), - ("context_exhaustion", False, True), - ("interrupted", False, True), - ("unknown", True, False), - ("empty_output", True, False), - ("unparseable", True, False), - ("timeout", True, False), - ], - ) - def test_session_complete_property_values( - self, subtype: str, is_error: bool, expected: bool - ) -> None: - """session_complete must be True iff not is_error AND subtype not in FAILURE_SUBTYPES.""" - session = ClaudeSessionResult( - subtype=subtype, - is_error=is_error, - result="", - session_id="test", - ) - assert session.session_complete is expected - - -class TestContentStateEnum: - """ContentState enum exposes the four expected variants.""" - - def test_content_state_enum_variants(self) -> None: - """ContentState exposes the four expected variants.""" - assert ContentState.COMPLETE == "complete" - assert ContentState.ABSENT == "absent" - assert ContentState.CONTRACT_VIOLATION == "contract_violation" - assert ContentState.SESSION_ERROR == "session_error" - - @pytest.mark.parametrize( - "result, is_error, completion_marker, patterns, expected_state", - [ - # COMPLETE: result present, marker present, patterns match - ( - "verdict = GO\n%%ORDER_UP%%", - False, - "%%ORDER_UP%%", - ["verdict\\s*=\\s*(GO|NO GO)"], - "complete", - ), - # ABSENT: empty result - ("", False, "%%ORDER_UP%%", ["verdict\\s*=\\s*(GO|NO GO)"], "absent"), - # ABSENT: marker missing from non-empty result - ("Some output, no marker", False, "%%ORDER_UP%%", [], "absent"), - # CONTRACT_VIOLATION: result + marker present, patterns fail - ( - "Done. %%ORDER_UP%%", - False, - "%%ORDER_UP%%", - ["verdict\\s*=\\s*(GO|NO GO)"], - "contract_violation", - ), - # SESSION_ERROR: is_error=True - ("Error: max turns exceeded", True, "%%ORDER_UP%%", [], "session_error"), - ], - ) - def test_evaluate_content_state( - self, - result: str, - is_error: bool, - completion_marker: str, - patterns: list[str], - expected_state: str, - make_session: Callable[..., ClaudeSessionResult], - ) -> None: - session = make_session(subtype="success", is_error=is_error, result=result) - state = _evaluate_content_state(session, completion_marker, patterns) - assert state.value == expected_state - - -class TestDeadEndGuardContentState: - """Dead-end guard must distinguish drain-race artifacts from terminal failures.""" - - def test_compute_outcome_channel_b_pattern_contract_violation_is_terminal( - self, - make_session: Callable[..., ClaudeSessionResult], - ) -> None: - """Dead-end guard must NOT promote to RETRIABLE when session has content + marker - but expected_output_patterns are absent — contract violation, not a drain race.""" - session = make_session( - subtype="success", - is_error=False, - result="Investigation complete. %%ORDER_UP%%", - assistant_messages=[], # no assistant_messages to recover from - ) - outcome, retry_reason = _compute_outcome( - session=session, - returncode=0, - termination=TerminationReason.NATURAL_EXIT, - completion_marker="%%ORDER_UP%%", - channel_confirmation=ChannelConfirmation.CHANNEL_B, - expected_output_patterns=["investigation_path\\s*=\\s*/.+"], - ) - # Contract violation: result is non-empty, marker is present, but patterns are absent. - # The dead-end guard must NOT promote this to RETRIABLE — retrying will never help. - assert outcome == SessionOutcome.FAILED, ( - f"Expected FAILED for pattern contract violation, got {outcome}. " - "Dead-end guard is incorrectly treating contract violations as drain-race artifacts." - ) - assert retry_reason == RetryReason.NONE - - def test_compute_outcome_completed_channel_b_pattern_contract_violation_is_terminal( - self, - make_session: Callable[..., ClaudeSessionResult], - ) -> None: - """Same contract-violation guard check for COMPLETED + CHANNEL_B termination path.""" - session = make_session( - subtype="success", - is_error=False, - result="Done. %%ORDER_UP%%", - assistant_messages=[], - ) - outcome, retry_reason = _compute_outcome( - session=session, - returncode=0, - termination=TerminationReason.COMPLETED, - completion_marker="%%ORDER_UP%%", - channel_confirmation=ChannelConfirmation.CHANNEL_B, - expected_output_patterns=["verdict\\s*=\\s*(GO|NO GO)"], - ) - assert outcome == SessionOutcome.FAILED - assert retry_reason == RetryReason.NONE - - def test_compute_outcome_channel_b_empty_result_is_still_retriable( - self, - make_session: Callable[..., ClaudeSessionResult], - ) -> None: - """Regression test: drain-race rescue (empty result) must still be promoted to RETRIABLE. - The ContentState fix must NOT break the existing drain-race handling.""" - session = make_session( - subtype="success", - is_error=False, - result="", # empty — drain race candidate - assistant_messages=[], - ) - outcome, retry_reason = _compute_outcome( - session=session, - returncode=0, - termination=TerminationReason.NATURAL_EXIT, - completion_marker="%%ORDER_UP%%", - channel_confirmation=ChannelConfirmation.CHANNEL_B, - expected_output_patterns=["investigation_path\\s*=\\s*/.+"], - ) - assert outcome == SessionOutcome.RETRIABLE, ( - "Empty result with channel confirmation must remain RETRIABLE (drain-race rescue)." - ) - assert retry_reason == RetryReason.DRAIN_RACE - - def test_compute_outcome_channel_b_missing_marker_is_still_retriable( - self, - make_session: Callable[..., ClaudeSessionResult], - ) -> None: - """Regression: result present but marker absent is still RETRIABLE (partial drain).""" - session = make_session( - subtype="success", - is_error=False, - result="Some output without the marker", - assistant_messages=[], - ) - outcome, retry_reason = _compute_outcome( - session=session, - returncode=0, - termination=TerminationReason.NATURAL_EXIT, - completion_marker="%%ORDER_UP%%", - channel_confirmation=ChannelConfirmation.CHANNEL_B, - expected_output_patterns=["investigation_path\\s*=\\s*/.+"], - ) - assert outcome == SessionOutcome.RETRIABLE, ( - "Missing completion marker with channel confirmation must remain RETRIABLE." - ) - assert retry_reason == RetryReason.DRAIN_RACE - - def test_dead_end_guard_channel_confirmed_absent_emits_drain_race( - self, - make_session: Callable[..., ClaudeSessionResult], - ) -> None: - """Channel confirmed + content ABSENT → DRAIN_RACE, not RESUME. - - DRAIN_RACE distinguishes "infrastructure confirmed completion, stdout not - fully flushed" from "session hit context limit." Both route to on_context_limit - because progress was confirmed, but the provenance is now explicit. - """ - session = make_session( - subtype="success", - is_error=False, - result="", # empty — drain race candidate - assistant_messages=[], - ) - outcome, reason = _compute_outcome( - session=session, - returncode=0, - termination=TerminationReason.NATURAL_EXIT, - completion_marker="%%ORDER_UP%%", - channel_confirmation=ChannelConfirmation.CHANNEL_A, - expected_output_patterns=(), - ) - assert outcome == SessionOutcome.RETRIABLE - assert reason == RetryReason.DRAIN_RACE # not RESUME - - def test_compute_outcome_bold_wrapped_token_is_success_not_violation( - self, - make_session: Callable[..., ClaudeSessionResult], - ) -> None: - """A session with bold-wrapped structured output tokens must succeed, - not be classified as CONTRACT_VIOLATION and returned as adjudicated_failure.""" - session = make_session(result="**plan_path** = /abs/path/plan.md\n%%ORDER_UP%%") - outcome, reason = _compute_outcome( - session, - returncode=0, - termination=TerminationReason.NATURAL_EXIT, - completion_marker="%%ORDER_UP%%", - channel_confirmation=ChannelConfirmation.CHANNEL_B, - expected_output_patterns=["plan_path\\s*=\\s*/.+"], - ) - assert outcome == SessionOutcome.SUCCEEDED - assert reason == RetryReason.NONE - - -# --------------------------------------------------------------------------- -# T1: parse_session_result preserves file_path from Write/Edit tool_use input -# --------------------------------------------------------------------------- - -import json # noqa: E402 — imported here to keep T1 tests self-contained - - -@pytest.fixture -def make_ndjson(): - """Build a minimal NDJSON string with assistant tool_use records and a result record. - - tool_uses entries use the raw NDJSON form: each dict must have 'name', 'id', and - optionally 'input' (a dict whose 'file_path' key will be preserved by Step 3's changes). - """ - - def _factory( - tool_uses: list[dict] | None = None, - result_text: str = "done", - ) -> str: - records = [] - if tool_uses: - content = [ - { - "type": "tool_use", - "name": tu["name"], - "id": tu["id"], - "input": tu.get("input", {}), - } - for tu in tool_uses - ] - records.append(json.dumps({"type": "assistant", "message": {"content": content}})) - records.append( - json.dumps( - { - "type": "result", - "subtype": "success", - "is_error": False, - "result": result_text, - "session_id": "test-session", - } - ) - ) - return "\n".join(records) - - return _factory - - -def test_parse_session_result_preserves_write_file_path(make_ndjson): - """Write tool_use input.file_path must be preserved in tool_uses entries.""" - ndjson = make_ndjson( - tool_uses=[{"name": "Write", "id": "tu1", "input": {"file_path": "/abs/plan.md"}}] - ) - session = parse_session_result(ndjson) - assert session.tool_uses == [{"name": "Write", "id": "tu1", "file_path": "/abs/plan.md"}] - - -def test_parse_session_result_preserves_edit_file_path(make_ndjson): - """Edit tool_use input.file_path must be preserved in tool_uses entries.""" - ndjson = make_ndjson( - tool_uses=[{"name": "Edit", "id": "tu2", "input": {"file_path": "/abs/file.py"}}] - ) - session = parse_session_result(ndjson) - assert session.tool_uses == [{"name": "Edit", "id": "tu2", "file_path": "/abs/file.py"}] - - -def test_parse_session_result_non_write_tools_no_file_path(make_ndjson): - """Non-Write/Edit tool_uses must not gain a file_path key.""" - ndjson = make_ndjson(tool_uses=[{"name": "Bash", "id": "tu3", "input": {"command": "ls"}}]) - session = parse_session_result(ndjson) - assert session.tool_uses == [{"name": "Bash", "id": "tu3"}] - assert "file_path" not in session.tool_uses[0] - - -class TestMinus9ReturncodeCoverage: - """Specification tests: returncode=-9 under COMPLETED is a valid success (1c). - - Before the fix, channel_won unconditionally sent SIGKILL (-9) to every - COMPLETED session. These tests document that the adjudicator correctly - classifies -9 under COMPLETED as successful — the bug was in the kill logic, - not the adjudicator. - """ - - def test_compute_success_completed_returncode_minus_9_is_success(self) -> None: - """COMPLETED + CHANNEL_A + returncode=-9 + content → success=True.""" - session = _make_success_session("ok") - result = _compute_success( - session, - returncode=-9, - termination=TerminationReason.COMPLETED, - channel_confirmation=ChannelConfirmation.CHANNEL_A, - ) - assert result is True - - def test_compute_success_completed_minus_9_channel_b_is_success(self) -> None: - """COMPLETED + CHANNEL_B + returncode=-9 → success=True (bypass).""" - session = ClaudeSessionResult( - subtype="success", - result="", - is_error=False, - session_id="b9", - ) - result = _compute_success( - session, - returncode=-9, - termination=TerminationReason.COMPLETED, - channel_confirmation=ChannelConfirmation.CHANNEL_B, - ) - assert result is True - - @pytest.mark.parametrize("returncode", [0, -15, -9]) - @pytest.mark.parametrize("termination", list(TerminationReason)) - def test_compute_retry_handles_minus_9_for_all_termination_reasons( - self, - termination: TerminationReason, - returncode: int, - ) -> None: - """_compute_retry must not raise for any (TerminationReason, returncode) pair.""" - session = ClaudeSessionResult( - subtype="success", - result="done. %%ORDER_UP%%", - is_error=False, - session_id="rc9", - errors=[], - ) - result = _compute_retry(session, returncode=returncode, termination=termination) - assert isinstance(result, tuple) and len(result) == 2 - - @pytest.mark.parametrize("returncode", [0, -15, -9]) - @pytest.mark.parametrize("termination", list(TerminationReason)) - def test_compute_success_handles_minus_9_for_all_termination_reasons( - self, - termination: TerminationReason, - returncode: int, - ) -> None: - """_compute_success must not raise for any (TerminationReason, returncode) pair.""" - session = ClaudeSessionResult( - subtype="success", - result="done. %%ORDER_UP%%", - is_error=False, - session_id="rc9", - errors=[], - ) - result = _compute_success( - session, - returncode=returncode, - termination=termination, - completion_marker="%%ORDER_UP%%", - ) - assert isinstance(result, bool) diff --git a/tests/execution/test_session_adjudication_outcome.py b/tests/execution/test_session_adjudication_outcome.py new file mode 100644 index 0000000000..f6e4b20c7a --- /dev/null +++ b/tests/execution/test_session_adjudication_outcome.py @@ -0,0 +1,854 @@ +"""Tests for _compute_outcome, content state evaluation, and session adjudication consistency.""" + +from __future__ import annotations + +import json +from collections.abc import Callable + +import pytest + +from autoskillit.core.types import ( + ChannelConfirmation, + RetryReason, + SessionOutcome, + TerminationReason, +) +from autoskillit.execution.session import ( + ClaudeSessionResult, + _check_expected_patterns, + _check_session_content, + _compute_outcome, + _compute_retry, + _compute_success, + parse_session_result, +) + +pytestmark = [pytest.mark.layer("execution"), pytest.mark.small] + + +class TestAdjudicationConsistency: + """Contract documentation for known-impossible adjudication states. + + These tests intentionally cover (returncode, termination) combinations that + cannot occur in production. They serve as specification — documenting that the + adjudicator is deterministic and exhaustive, not that these paths are reachable. + """ + + @pytest.mark.parametrize( + "termination,channel,result_content,returncode,subtype,is_error," + "expected_success,expected_retry,completion_marker", + [ + # NATURAL_EXIT + CHANNEL_A: dead-end state + # (known bug, corrected by guard in _build_skill_result) + ( + TerminationReason.NATURAL_EXIT, + ChannelConfirmation.CHANNEL_A, + "", + 0, + "success", + False, + False, + False, + "", + ), + # NATURAL_EXIT + CHANNEL_A: valid success with content + ( + TerminationReason.NATURAL_EXIT, + ChannelConfirmation.CHANNEL_A, + "done", + 0, + "success", + False, + True, + False, + "", + ), + # NATURAL_EXIT + CHANNEL_B: contradiction + # (known bug, corrected by guard in _build_skill_result) + ( + TerminationReason.NATURAL_EXIT, + ChannelConfirmation.CHANNEL_B, + "", + 0, + "error_max_turns", + True, + True, + True, + "", + ), + # NATURAL_EXIT + CHANNEL_B: valid success + ( + TerminationReason.NATURAL_EXIT, + ChannelConfirmation.CHANNEL_B, + "", + 0, + "success", + False, + True, + False, + "", + ), + ( + TerminationReason.NATURAL_EXIT, + ChannelConfirmation.CHANNEL_B, + "done", + 0, + "success", + False, + True, + False, + "", + ), + # COMPLETED + CHANNEL_A: valid retriable (kill anomaly) + ( + TerminationReason.COMPLETED, + ChannelConfirmation.CHANNEL_A, + "", + -15, + "success", + False, + False, + True, + "", + ), + # COMPLETED + CHANNEL_B: valid success + ( + TerminationReason.COMPLETED, + ChannelConfirmation.CHANNEL_B, + "", + -15, + "success", + False, + True, + False, + "", + ), + # COMPLETED + CHANNEL_B: contradiction + # (known bug, corrected by guard in _build_skill_result) + ( + TerminationReason.COMPLETED, + ChannelConfirmation.CHANNEL_B, + "", + -15, + "error_max_turns", + True, + True, + True, + "", + ), + # UNMONITORED baselines — all should already be valid states + ( + TerminationReason.NATURAL_EXIT, + ChannelConfirmation.UNMONITORED, + "", + 0, + "success", + False, + False, + True, + "", + ), + ( + TerminationReason.NATURAL_EXIT, + ChannelConfirmation.UNMONITORED, + "done", + 0, + "success", + False, + True, + False, + "", + ), + ( + TerminationReason.TIMED_OUT, + ChannelConfirmation.UNMONITORED, + "", + -1, + "timeout", + True, + False, + False, + "", + ), + ( + TerminationReason.STALE, + ChannelConfirmation.UNMONITORED, + "", + 0, + "success", + False, + False, + False, + "", + ), + # NATURAL_EXIT + UNMONITORED: substantive result without marker + # (premature exit / early stop — retriable via EARLY_STOP) + pytest.param( + TerminationReason.NATURAL_EXIT, + ChannelConfirmation.UNMONITORED, + "Here is the PR context block with substantive content...", + 0, + "success", + False, + False, + True, + "%%ORDER_UP%%", + id="natural_exit-unmonitored-substantive_result_no_marker", + ), + ], + ) + def test_raw_adjudication_pair( + self, + termination: TerminationReason, + channel: ChannelConfirmation, + result_content: str, + returncode: int, + subtype: str, + is_error: bool, + expected_success: bool, + expected_retry: bool, + completion_marker: str, + ) -> None: + """Document exact raw outputs of the individual adjudication functions. + + Known bad states (dead end, contradiction) are documented as expected values + rather than as invariant failures — the guards in _build_skill_result correct + those states before they reach the orchestrator. + """ + session = ClaudeSessionResult( + subtype=subtype, + result=result_content, + is_error=is_error, + session_id="cross-val", + errors=[], + ) + success = _compute_success( + session, + returncode, + termination, + channel_confirmation=channel, + completion_marker=completion_marker, + ) + needs_retry, _ = _compute_retry( + session, + returncode, + termination, + channel_confirmation=channel, + completion_marker=completion_marker, + ) + assert success == expected_success + assert needs_retry == expected_retry + + def test_premature_exit_substantive_result_no_marker_is_early_stop(self) -> None: + """NATURAL_EXIT + UNMONITORED + substantive result without marker → EARLY_STOP. + + When the model produces substantive output but stops before emitting + the completion marker, the session is classified as retriable with + EARLY_STOP reason. This is the text-then-tool boundary fix. + """ + session = ClaudeSessionResult( + subtype="success", + result="Here is the PR context block with substantive content...", + is_error=False, + session_id="premature", + errors=[], + ) + success = _compute_success( + session, + returncode=0, + termination=TerminationReason.NATURAL_EXIT, + channel_confirmation=ChannelConfirmation.UNMONITORED, + completion_marker="%%ORDER_UP%%", + ) + needs_retry, reason = _compute_retry( + session, + returncode=0, + termination=TerminationReason.NATURAL_EXIT, + channel_confirmation=ChannelConfirmation.UNMONITORED, + completion_marker="%%ORDER_UP%%", + ) + assert success is False + assert needs_retry is True + assert reason == RetryReason.EARLY_STOP + + def test_channel_a_empty_result_raw_dead_end(self) -> None: + """Document: NATURAL_EXIT + CHANNEL_A + empty result is a dead end at raw function level. + + Both _compute_success and _compute_retry return False for this combination: + - _compute_success: CHANNEL_A falls through content check, empty result → False + - _compute_retry: NATURAL_EXIT + CHANNEL_A confirmation suppresses retry → False + + The composition guard in _build_skill_result escalates this to retriable. + See TestAdjudicationGuards.test_channel_a_empty_result_not_dead_end. + """ + session = ClaudeSessionResult( + subtype="success", + result="", + is_error=False, + session_id="regression", + errors=[], + ) + success = _compute_success( + session, + returncode=0, + termination=TerminationReason.NATURAL_EXIT, + channel_confirmation=ChannelConfirmation.CHANNEL_A, + ) + needs_retry, _ = _compute_retry( + session, + returncode=0, + termination=TerminationReason.NATURAL_EXIT, + channel_confirmation=ChannelConfirmation.CHANNEL_A, + ) + # Raw functions produce the dead-end state — this is the known bug at the + # individual function level, corrected at the _build_skill_result boundary. + assert success is False + assert needs_retry is False + + +class TestEarlyStop: + """Tests for EARLY_STOP retry classification.""" + + def test_natural_exit_substantive_content_no_marker_is_retriable(self) -> None: + """A session that exits cleanly with substantive output but without + the completion marker should be classified as RETRIABLE with EARLY_STOP, + because the model may have stopped early at a text-then-tool boundary. + """ + session = ClaudeSessionResult( + subtype="success", + result="Here is the PR context block with substantive content...", + is_error=False, + session_id="early-stop", + errors=[], + ) + outcome, reason = _compute_outcome( + session, + returncode=0, + termination=TerminationReason.NATURAL_EXIT, + completion_marker="%%ORDER_UP%%", + channel_confirmation=ChannelConfirmation.UNMONITORED, + ) + assert outcome == SessionOutcome.RETRIABLE + assert reason == RetryReason.EARLY_STOP + + def test_early_stop_not_triggered_without_marker(self) -> None: + """When no completion_marker is configured, EARLY_STOP should NOT fire. + + Sessions without a marker are not subject to early-stop detection. + """ + session = ClaudeSessionResult( + subtype="success", + result="Some output without any marker", + is_error=False, + session_id="no-marker", + errors=[], + ) + needs_retry, reason = _compute_retry( + session, + returncode=0, + termination=TerminationReason.NATURAL_EXIT, + channel_confirmation=ChannelConfirmation.UNMONITORED, + completion_marker="", + ) + assert needs_retry is False + assert reason == RetryReason.NONE + + def test_early_stop_not_triggered_when_marker_present(self) -> None: + """When the completion marker IS present, EARLY_STOP should NOT fire.""" + session = ClaudeSessionResult( + subtype="success", + result="Result with %%ORDER_UP%% marker present", + is_error=False, + session_id="has-marker", + errors=[], + ) + needs_retry, reason = _compute_retry( + session, + returncode=0, + termination=TerminationReason.NATURAL_EXIT, + channel_confirmation=ChannelConfirmation.UNMONITORED, + completion_marker="%%ORDER_UP%%", + ) + assert needs_retry is False + assert reason == RetryReason.NONE + + def test_early_stop_not_triggered_for_errors(self) -> None: + """Error sessions should not be classified as EARLY_STOP.""" + session = ClaudeSessionResult( + subtype="error_during_execution", + result="Error occurred but substantive output...", + is_error=True, + session_id="error-session", + errors=["something broke"], + ) + needs_retry, reason = _compute_retry( + session, + returncode=0, + termination=TerminationReason.NATURAL_EXIT, + channel_confirmation=ChannelConfirmation.UNMONITORED, + completion_marker="%%ORDER_UP%%", + ) + # is_error sessions may trigger API-level retry via needs_retry property, + # but EARLY_STOP specifically should not fire for non-success subtypes + assert reason != RetryReason.EARLY_STOP + + def test_early_stop_not_triggered_for_empty_result(self) -> None: + """Empty result should be classified as kill anomaly, not EARLY_STOP.""" + session = ClaudeSessionResult( + subtype="success", + result="", + is_error=False, + session_id="empty", + errors=[], + ) + needs_retry, reason = _compute_retry( + session, + returncode=0, + termination=TerminationReason.NATURAL_EXIT, + channel_confirmation=ChannelConfirmation.UNMONITORED, + completion_marker="%%ORDER_UP%%", + ) + # Empty result triggers kill_anomaly path (EMPTY_OUTPUT), not EARLY_STOP. + # No context exhaustion detected → EMPTY_OUTPUT, not RESUME. + assert needs_retry is True + assert reason == RetryReason.EMPTY_OUTPUT + + +class TestArtifactValidation: + """Tests for expected_output_patterns artifact validation.""" + + def test_check_session_content_validates_expected_artifacts(self) -> None: + """When expected_output_patterns are configured, _check_session_content + must verify that the session result contains at least one match. + """ + session = ClaudeSessionResult( + subtype="success", + result="Done! %%ORDER_UP%%", + is_error=False, + session_id="no-artifact", + errors=[], + ) + # With marker present but no matching artifact + result = _check_session_content( + session, + completion_marker="%%ORDER_UP%%", + expected_output_patterns=[r"https://github\.com/.*/pull/\d+"], + ) + assert result is False + + def test_check_session_content_passes_with_matching_artifact(self) -> None: + """When artifacts match, content check passes.""" + session = ClaudeSessionResult( + subtype="success", + result="PR created: https://github.com/user/repo/pull/42 %%ORDER_UP%%", + is_error=False, + session_id="has-artifact", + errors=[], + ) + result = _check_session_content( + session, + completion_marker="%%ORDER_UP%%", + expected_output_patterns=[r"https://github\.com/.*/pull/\d+"], + ) + assert result is True + + def test_check_session_content_no_patterns_skips_validation(self) -> None: + """When no patterns are provided, artifact validation is skipped.""" + session = ClaudeSessionResult( + subtype="success", + result="Done! %%ORDER_UP%%", + is_error=False, + session_id="no-patterns", + errors=[], + ) + result = _check_session_content( + session, + completion_marker="%%ORDER_UP%%", + expected_output_patterns=[], + ) + assert result is True + + def test_compute_outcome_threads_expected_output_patterns(self) -> None: + """_compute_outcome must thread expected_output_patterns through to + _compute_success and _check_session_content.""" + session = ClaudeSessionResult( + subtype="success", + result="Done! %%ORDER_UP%%", + is_error=False, + session_id="threaded", + errors=[], + ) + # Without patterns: success + outcome_no_patterns, _ = _compute_outcome( + session, + returncode=0, + termination=TerminationReason.NATURAL_EXIT, + completion_marker="%%ORDER_UP%%", + channel_confirmation=ChannelConfirmation.UNMONITORED, + expected_output_patterns=[], + ) + assert outcome_no_patterns == SessionOutcome.SUCCEEDED + + # With patterns that don't match: failed (EARLY_STOP since marker + # absent from _compute_success perspective when artifact check fails) + outcome_with_patterns, _ = _compute_outcome( + session, + returncode=0, + termination=TerminationReason.NATURAL_EXIT, + completion_marker="%%ORDER_UP%%", + channel_confirmation=ChannelConfirmation.UNMONITORED, + expected_output_patterns=[r"https://github\.com/.*/pull/\d+"], + ) + assert outcome_with_patterns != SessionOutcome.SUCCEEDED + + +class TestToolUseParsing: + """Tests for tool_use NDJSON record extraction.""" + + @staticmethod + def _result_line() -> str: + return json.dumps( + { + "type": "result", + "subtype": "success", + "is_error": False, + "result": "done", + "session_id": "s1", + } + ) + + @staticmethod + def _assistant_line(*content_blocks: dict) -> str: # type: ignore[type-arg] + return json.dumps( + { + "type": "assistant", + "message": {"content": list(content_blocks)}, + } + ) + + def test_parse_session_result_captures_tool_uses(self) -> None: + """parse_session_result must extract tool_use records.""" + ndjson = "\n".join( + [ + self._assistant_line( + {"type": "tool_use", "name": "Skill", "id": "tu_1"}, + {"type": "text", "text": "loading skill"}, + ), + self._result_line(), + ] + ) + session = parse_session_result(ndjson) + assert len(session.tool_uses) == 1 + assert session.tool_uses[0]["name"] == "Skill" + assert session.tool_uses[0]["id"] == "tu_1" + + def test_parse_session_result_no_tool_uses(self) -> None: + """Sessions without tool_use records have an empty list.""" + ndjson = "\n".join( + [ + self._assistant_line( + {"type": "text", "text": "just text"}, + ), + self._result_line(), + ] + ) + session = parse_session_result(ndjson) + assert session.tool_uses == [] + + def test_parse_session_result_multiple_tool_uses(self) -> None: + """Multiple tool_use records across messages are captured.""" + ndjson = "\n".join( + [ + self._assistant_line( + {"type": "tool_use", "name": "Write", "id": "tu_1"}, + ), + self._assistant_line( + {"type": "tool_use", "name": "Skill", "id": "tu_2"}, + ), + self._result_line(), + ] + ) + session = parse_session_result(ndjson) + assert len(session.tool_uses) == 2 + assert session.tool_uses[0]["name"] == "Write" + assert session.tool_uses[1]["name"] == "Skill" + + +class TestCheckExpectedPatterns: + """Unit tests for the standalone _check_expected_patterns function.""" + + def test_check_expected_patterns_present(self) -> None: + assert ( + _check_expected_patterns( + result="some text ---my-block--- more text", + patterns=["---my-block---"], + ) + is True + ) + + def test_check_expected_patterns_absent(self) -> None: + assert ( + _check_expected_patterns( + result="some text without the block", + patterns=["---my-block---"], + ) + is False + ) + + def test_check_expected_patterns_empty_patterns_always_true(self) -> None: + assert _check_expected_patterns(result="anything", patterns=[]) is True + + def test_check_expected_patterns_bold_wrapped_token_matches(self) -> None: + """Bold-wrapped token name must match after normalization.""" + result = "**plan_path** = /abs/path/plan.md\n%%ORDER_UP%%" + assert _check_expected_patterns(result, ["plan_path\\s*=\\s*/.+"]) is True + + def test_check_expected_patterns_italic_wrapped_token_matches(self) -> None: + """Italic-wrapped token name must match after normalization.""" + result = "*plan_path* = /abs/path/plan.md\n%%ORDER_UP%%" + assert _check_expected_patterns(result, ["plan_path\\s*=\\s*/.+"]) is True + + def test_check_expected_patterns_bold_verdict_matches(self) -> None: + """Bold-wrapped verdict token must match after normalization.""" + result = "**verdict** = GO\n%%ORDER_UP%%" + assert _check_expected_patterns(result, ["verdict\\s*=\\s*(GO|NO GO)"]) is True + + def test_check_expected_patterns_multiple_bold_tokens_all_match(self) -> None: + """Multiple bold-wrapped tokens must all match (AND semantics preserved).""" + result = ( + "**plan_path** = /abs/path/plan.md\n**plan_parts** = /abs/path/plan.md\n%%ORDER_UP%%" + ) + assert ( + _check_expected_patterns(result, ["plan_path\\s*=\\s*/.+", "plan_parts\\s*=\\s*/.+"]) + is True + ) + + def test_check_expected_patterns_bold_relative_path_still_fails(self) -> None: + """Bold wrapping on a relative path must still fail — normalization must not + mask a genuine contract violation (wrong value type).""" + result = "**worktree_path** = ../worktrees/impl\n%%ORDER_UP%%" + assert _check_expected_patterns(result, ["worktree_path\\s*=\\s*/.+"]) is False + + def test_check_expected_patterns_bold_absent_value_still_fails(self) -> None: + """Bold key with no value must still fail — semantic content must be present.""" + result = "**plan_path** =\n%%ORDER_UP%%" + assert _check_expected_patterns(result, ["plan_path\\s*=\\s*/.+"]) is False + + +class TestDeadEndGuardContentState: + """Dead-end guard must distinguish drain-race artifacts from terminal failures.""" + + def test_compute_outcome_channel_b_pattern_contract_violation_is_terminal( + self, + make_session: Callable[..., ClaudeSessionResult], + ) -> None: + """Dead-end guard must NOT promote to RETRIABLE when session has content + marker + but expected_output_patterns are absent — contract violation, not a drain race.""" + session = make_session( + subtype="success", + is_error=False, + result="Investigation complete. %%ORDER_UP%%", + assistant_messages=[], # no assistant_messages to recover from + ) + outcome, retry_reason = _compute_outcome( + session=session, + returncode=0, + termination=TerminationReason.NATURAL_EXIT, + completion_marker="%%ORDER_UP%%", + channel_confirmation=ChannelConfirmation.CHANNEL_B, + expected_output_patterns=["investigation_path\\s*=\\s*/.+"], + ) + # Contract violation: result is non-empty, marker is present, but patterns are absent. + # The dead-end guard must NOT promote this to RETRIABLE — retrying will never help. + assert outcome == SessionOutcome.FAILED, ( + f"Expected FAILED for pattern contract violation, got {outcome}. " + "Dead-end guard is incorrectly treating contract violations as drain-race artifacts." + ) + assert retry_reason == RetryReason.NONE + + def test_compute_outcome_completed_channel_b_pattern_contract_violation_is_terminal( + self, + make_session: Callable[..., ClaudeSessionResult], + ) -> None: + """Same contract-violation guard check for COMPLETED + CHANNEL_B termination path.""" + session = make_session( + subtype="success", + is_error=False, + result="Done. %%ORDER_UP%%", + assistant_messages=[], + ) + outcome, retry_reason = _compute_outcome( + session=session, + returncode=0, + termination=TerminationReason.COMPLETED, + completion_marker="%%ORDER_UP%%", + channel_confirmation=ChannelConfirmation.CHANNEL_B, + expected_output_patterns=["verdict\\s*=\\s*(GO|NO GO)"], + ) + assert outcome == SessionOutcome.FAILED + assert retry_reason == RetryReason.NONE + + def test_compute_outcome_channel_b_empty_result_is_still_retriable( + self, + make_session: Callable[..., ClaudeSessionResult], + ) -> None: + """Regression test: drain-race rescue (empty result) must still be promoted to RETRIABLE. + The ContentState fix must NOT break the existing drain-race handling.""" + session = make_session( + subtype="success", + is_error=False, + result="", # empty — drain race candidate + assistant_messages=[], + ) + outcome, retry_reason = _compute_outcome( + session=session, + returncode=0, + termination=TerminationReason.NATURAL_EXIT, + completion_marker="%%ORDER_UP%%", + channel_confirmation=ChannelConfirmation.CHANNEL_B, + expected_output_patterns=["investigation_path\\s*=\\s*/.+"], + ) + assert outcome == SessionOutcome.RETRIABLE, ( + "Empty result with channel confirmation must remain RETRIABLE (drain-race rescue)." + ) + assert retry_reason == RetryReason.DRAIN_RACE + + def test_compute_outcome_channel_b_missing_marker_is_still_retriable( + self, + make_session: Callable[..., ClaudeSessionResult], + ) -> None: + """Regression: result present but marker absent is still RETRIABLE (partial drain).""" + session = make_session( + subtype="success", + is_error=False, + result="Some output without the marker", + assistant_messages=[], + ) + outcome, retry_reason = _compute_outcome( + session=session, + returncode=0, + termination=TerminationReason.NATURAL_EXIT, + completion_marker="%%ORDER_UP%%", + channel_confirmation=ChannelConfirmation.CHANNEL_B, + expected_output_patterns=["investigation_path\\s*=\\s*/.+"], + ) + assert outcome == SessionOutcome.RETRIABLE, ( + "Missing completion marker with channel confirmation must remain RETRIABLE." + ) + assert retry_reason == RetryReason.DRAIN_RACE + + def test_dead_end_guard_channel_confirmed_absent_emits_drain_race( + self, + make_session: Callable[..., ClaudeSessionResult], + ) -> None: + """Channel confirmed + content ABSENT → DRAIN_RACE, not RESUME. + + DRAIN_RACE distinguishes "infrastructure confirmed completion, stdout not + fully flushed" from "session hit context limit." Both route to on_context_limit + because progress was confirmed, but the provenance is now explicit. + """ + session = make_session( + subtype="success", + is_error=False, + result="", # empty — drain race candidate + assistant_messages=[], + ) + outcome, reason = _compute_outcome( + session=session, + returncode=0, + termination=TerminationReason.NATURAL_EXIT, + completion_marker="%%ORDER_UP%%", + channel_confirmation=ChannelConfirmation.CHANNEL_A, + expected_output_patterns=(), + ) + assert outcome == SessionOutcome.RETRIABLE + assert reason == RetryReason.DRAIN_RACE # not RESUME + + def test_compute_outcome_bold_wrapped_token_is_success_not_violation( + self, + make_session: Callable[..., ClaudeSessionResult], + ) -> None: + """A session with bold-wrapped structured output tokens must succeed, + not be classified as CONTRACT_VIOLATION and returned as adjudicated_failure.""" + session = make_session(result="**plan_path** = /abs/path/plan.md\n%%ORDER_UP%%") + outcome, reason = _compute_outcome( + session, + returncode=0, + termination=TerminationReason.NATURAL_EXIT, + completion_marker="%%ORDER_UP%%", + channel_confirmation=ChannelConfirmation.CHANNEL_B, + expected_output_patterns=["plan_path\\s*=\\s*/.+"], + ) + assert outcome == SessionOutcome.SUCCEEDED + assert reason == RetryReason.NONE + + +# --------------------------------------------------------------------------- +# T1: parse_session_result preserves file_path from Write/Edit tool_use input +# --------------------------------------------------------------------------- + + +@pytest.fixture +def make_ndjson(): + """Build a minimal NDJSON string with assistant tool_use records and a result record. + + tool_uses entries use the raw NDJSON form: each dict must have 'name', 'id', and + optionally 'input' (a dict whose 'file_path' key will be preserved by Step 3's changes). + """ + + def _factory( + tool_uses: list[dict] | None = None, + result_text: str = "done", + ) -> str: + records = [] + if tool_uses: + content = [ + { + "type": "tool_use", + "name": tu["name"], + "id": tu["id"], + "input": tu.get("input", {}), + } + for tu in tool_uses + ] + records.append(json.dumps({"type": "assistant", "message": {"content": content}})) + records.append( + json.dumps( + { + "type": "result", + "subtype": "success", + "is_error": False, + "result": result_text, + "session_id": "test-session", + } + ) + ) + return "\n".join(records) + + return _factory + + +def test_parse_session_result_preserves_write_file_path(make_ndjson): + """Write tool_use input.file_path must be preserved in tool_uses entries.""" + ndjson = make_ndjson( + tool_uses=[{"name": "Write", "id": "tu1", "input": {"file_path": "/abs/plan.md"}}] + ) + session = parse_session_result(ndjson) + assert session.tool_uses == [{"name": "Write", "id": "tu1", "file_path": "/abs/plan.md"}] + + +def test_parse_session_result_preserves_edit_file_path(make_ndjson): + """Edit tool_use input.file_path must be preserved in tool_uses entries.""" + ndjson = make_ndjson( + tool_uses=[{"name": "Edit", "id": "tu2", "input": {"file_path": "/abs/file.py"}}] + ) + session = parse_session_result(ndjson) + assert session.tool_uses == [{"name": "Edit", "id": "tu2", "file_path": "/abs/file.py"}] + + +def test_parse_session_result_non_write_tools_no_file_path(make_ndjson): + """Non-Write/Edit tool_uses must not gain a file_path key.""" + ndjson = make_ndjson(tool_uses=[{"name": "Bash", "id": "tu3", "input": {"command": "ls"}}]) + session = parse_session_result(ndjson) + assert session.tool_uses == [{"name": "Bash", "id": "tu3"}] + assert "file_path" not in session.tool_uses[0] diff --git a/tests/execution/test_session_adjudication_retry.py b/tests/execution/test_session_adjudication_retry.py new file mode 100644 index 0000000000..cc5c12907b --- /dev/null +++ b/tests/execution/test_session_adjudication_retry.py @@ -0,0 +1,576 @@ +"""Tests for _compute_retry, _is_kill_anomaly, and related retry adjudication logic.""" + +from __future__ import annotations + +from collections.abc import Callable + +import pytest + +from autoskillit.core.types import ( + ChannelConfirmation, + RetryReason, + TerminationReason, +) +from autoskillit.execution.session import ( + ClaudeSessionResult, + ContentState, + _compute_retry, + _evaluate_content_state, + _is_kill_anomaly, +) + +pytestmark = [pytest.mark.layer("execution"), pytest.mark.small] + + +class TestComputeRetry: + """_compute_retry cross-validates all signals for retry eligibility.""" + + def test_success_not_retriable(self): + session = ClaudeSessionResult( + subtype="success", is_error=False, result="Done.", session_id="s1" + ) + needs, reason = _compute_retry( + session, returncode=0, termination=TerminationReason.NATURAL_EXIT + ) + assert needs is False + assert reason == RetryReason.NONE + + def test_max_turns_is_retriable(self): + session = ClaudeSessionResult( + subtype="error_max_turns", is_error=False, result="", session_id="s1" + ) + needs, reason = _compute_retry( + session, returncode=1, termination=TerminationReason.NATURAL_EXIT + ) + assert needs is True + assert reason == RetryReason.RESUME + + def test_context_exhaustion_is_retriable(self): + session = ClaudeSessionResult( + subtype="success", is_error=True, result="Prompt is too long", session_id="s1" + ) + needs, reason = _compute_retry( + session, returncode=1, termination=TerminationReason.NATURAL_EXIT + ) + assert needs is True + assert reason == RetryReason.RESUME + + def test_thinking_stall_is_distinct_from_empty_output(self): + """Thinking-only final turn produces THINKING_STALL, not EMPTY_OUTPUT. + + When the model's final turn contained only thinking blocks (has_thinking_only_turn=True) + and the session has no text result and no tool calls, the retry reason must be + THINKING_STALL rather than EMPTY_OUTPUT to allow callers to distinguish between + "model produced nothing" and "model was actively reasoning." + """ + session = ClaudeSessionResult( + subtype="success", + is_error=False, + result="", + session_id="s1", + has_thinking_only_turn=True, + ) + needs, reason = _compute_retry( + session, returncode=0, termination=TerminationReason.NATURAL_EXIT + ) + assert needs is True + assert reason == RetryReason.THINKING_STALL + + def test_empty_output_without_thinking_flag_stays_empty_output(self): + """EMPTY_OUTPUT retry reason requires has_thinking_only_turn=False.""" + session = ClaudeSessionResult( + subtype="success", + is_error=False, + result="", + session_id="s1", + has_thinking_only_turn=False, + ) + needs, reason = _compute_retry( + session, returncode=0, termination=TerminationReason.NATURAL_EXIT + ) + assert needs is True + assert reason == RetryReason.EMPTY_OUTPUT + + def test_empty_output_exit_zero_is_retriable(self): + """Infrastructure failure: session never ran, CLI exited cleanly. + + NATURAL_EXIT + rc=0 + empty_output subtype means the session process + exited before writing any output — NOT a context exhaustion. The retry + reason must be EMPTY_OUTPUT, not RESUME, because no partial progress + exists on disk. RESUME is reserved for context/turn limit cases. + """ + session = ClaudeSessionResult( + subtype="empty_output", is_error=True, result="", session_id="" + ) + needs, reason = _compute_retry( + session, returncode=0, termination=TerminationReason.NATURAL_EXIT + ) + assert needs is True + assert reason == RetryReason.EMPTY_OUTPUT + + def test_empty_output_exit_one_not_retriable(self): + """Real failure: CLI crashed with empty output.""" + session = ClaudeSessionResult( + subtype="empty_output", is_error=True, result="", session_id="" + ) + needs, reason = _compute_retry( + session, returncode=1, termination=TerminationReason.NATURAL_EXIT + ) + assert needs is False + assert reason == RetryReason.NONE + + def test_timeout_not_retriable(self): + session = ClaudeSessionResult(subtype="timeout", is_error=True, result="", session_id="") + needs, reason = _compute_retry( + session, returncode=-1, termination=TerminationReason.TIMED_OUT + ) + assert needs is False + assert reason == RetryReason.NONE + + def test_unparseable_not_retriable(self): + session = ClaudeSessionResult( + subtype="unparseable", is_error=True, result="crash", session_id="" + ) + needs, reason = _compute_retry( + session, returncode=1, termination=TerminationReason.NATURAL_EXIT + ) + assert needs is False + assert reason == RetryReason.NONE + + def test_execution_error_not_retriable(self): + session = ClaudeSessionResult( + subtype="error_during_execution", + is_error=True, + result="tool error", + session_id="s1", + ) + needs, reason = _compute_retry( + session, returncode=1, termination=TerminationReason.NATURAL_EXIT + ) + assert needs is False + assert reason == RetryReason.NONE + + def test_channel_b_completed_no_retry(self): + """CHANNEL_B is authoritative on COMPLETED — no retry.""" + session = ClaudeSessionResult( + subtype="success", is_error=False, result="", session_id="s1" + ) + needs, reason = _compute_retry( + session, + returncode=0, + termination=TerminationReason.COMPLETED, + channel_confirmation=ChannelConfirmation.CHANNEL_B, + ) + assert needs is False + assert reason == RetryReason.NONE + + def test_channel_a_completed_kill_anomaly_retries(self): + """CHANNEL_A + empty result → retry RESUME (kill anomaly suspected).""" + session = ClaudeSessionResult( + subtype="success", is_error=False, result="", session_id="s1" + ) + needs, reason = _compute_retry( + session, + returncode=0, + termination=TerminationReason.COMPLETED, + channel_confirmation=ChannelConfirmation.CHANNEL_A, + ) + assert needs is True + assert reason == RetryReason.RESUME + + def test_natural_exit_channel_b_no_retry(self): + """CHANNEL_B confirmation skips kill-anomaly check on NATURAL_EXIT.""" + session = ClaudeSessionResult( + subtype="success", is_error=False, result="", session_id="s1" + ) + needs, reason = _compute_retry( + session, + returncode=0, + termination=TerminationReason.NATURAL_EXIT, + channel_confirmation=ChannelConfirmation.CHANNEL_B, + ) + assert needs is False + assert reason == RetryReason.NONE + + def test_natural_exit_channel_a_no_retry(self): + """CHANNEL_A confirmation skips kill-anomaly check on NATURAL_EXIT.""" + session = ClaudeSessionResult( + subtype="success", is_error=False, result="Done.", session_id="s1" + ) + needs, reason = _compute_retry( + session, + returncode=0, + termination=TerminationReason.NATURAL_EXIT, + channel_confirmation=ChannelConfirmation.CHANNEL_A, + ) + assert needs is False + assert reason == RetryReason.NONE + + +class TestComputeRetryUnparseable: + """_compute_retry distinguishes unparseable under COMPLETED vs NATURAL_EXIT.""" + + def test_unparseable_subtype_with_nonzero_returncode_should_retry(self): + """unparseable under COMPLETED means process was killed mid-write. + + The drain timeout expired before the result record was fully flushed. + The session likely completed; retry with resume. + """ + session = ClaudeSessionResult( + subtype="unparseable", is_error=True, result="partial", session_id="" + ) + needs, reason = _compute_retry( + session, returncode=-15, termination=TerminationReason.COMPLETED + ) + assert needs is True + assert reason == RetryReason.RESUME + + def test_unparseable_subtype_natural_exit_no_retry(self): + """unparseable under NATURAL_EXIT is a content failure, not retryable. + + The process exited cleanly with malformed output — this is not a timing issue. + """ + session = ClaudeSessionResult( + subtype="unparseable", is_error=True, result="", session_id="" + ) + needs, reason = _compute_retry( + session, returncode=1, termination=TerminationReason.NATURAL_EXIT + ) + assert needs is False + assert reason == RetryReason.NONE + + +class TestIsKillAnomaly: + """_is_kill_anomaly covers exactly the subtypes that represent kill artifacts.""" + + @pytest.mark.parametrize( + "subtype,result,expected", + [ + ("unparseable", "", True), # killed mid-write → partial NDJSON + ("empty_output", "", True), # killed before any stdout written + ("success", "", True), # killed after result record, content empty + ("success", "x", False), # success with content → NOT an anomaly + ("error_during_execution", "", False), # explicit API error, not a kill artifact + ("timeout", "", False), # timeout is a separate terminal state + ], + ) + def test_anomaly_classification(self, subtype: str, result: str, expected: bool) -> None: + session = ClaudeSessionResult( + subtype=subtype, + is_error=(subtype != "success"), + result=result, + session_id="", + errors=[], + token_usage=None, + ) + assert _is_kill_anomaly(session) is expected + + +class TestComputeRetrySuccessEmptyResult: + """_compute_retry for success subtype with empty result under COMPLETED termination.""" + + def test_success_empty_result_completed_rc0_is_retriable(self) -> None: + """success + "" + COMPLETED + rc=0 must be retriable (drain-race glitch).""" + session = ClaudeSessionResult( + subtype="success", + is_error=False, + result="", + session_id="s1", + errors=[], + token_usage=None, + ) + retriable, reason = _compute_retry( + session, returncode=0, termination=TerminationReason.COMPLETED + ) + assert retriable is True + assert reason == RetryReason.RESUME + + def test_success_empty_result_completed_negative_rc_is_retriable(self) -> None: + """success + "" + COMPLETED + rc=-15 (SIGTERM kill) must also be retriable.""" + session = ClaudeSessionResult( + subtype="success", + is_error=False, + result="", + session_id="s1", + errors=[], + token_usage=None, + ) + retriable, reason = _compute_retry( + session, returncode=-15, termination=TerminationReason.COMPLETED + ) + assert retriable is True + assert reason == RetryReason.RESUME + + def test_success_nonempty_result_completed_is_not_retriable(self) -> None: + """success + non-empty result + COMPLETED must NOT be retriable (genuine success).""" + session = ClaudeSessionResult( + subtype="success", + is_error=False, + result="Done. %%ORDER_UP%%", + session_id="s1", + errors=[], + token_usage=None, + ) + retriable, _ = _compute_retry( + session, returncode=-15, termination=TerminationReason.COMPLETED + ) + assert retriable is False + + def test_success_empty_result_natural_exit_zero_rc_is_retriable(self) -> None: + """Transient empty result: session exited cleanly but result field is empty. + + NATURAL_EXIT + rc=0 + SUCCESS subtype + empty result covers any case where + the Claude process self-exited cleanly without writing output — including + CLAUDE_CODE_EXIT_AFTER_STOP_DELAY timer-based exits AND transient API failures. + This is NOT a context exhaustion (no jsonl marker, no error markers). The retry + reason must be EMPTY_OUTPUT, not RESUME: no partial progress on disk. + """ + session = ClaudeSessionResult( + subtype="success", + is_error=False, + result="", + session_id="s1", + errors=[], + token_usage=None, + ) + retriable, reason = _compute_retry( + session, returncode=0, termination=TerminationReason.NATURAL_EXIT + ) + assert retriable is True + assert reason == RetryReason.EMPTY_OUTPUT + + def test_empty_output_completed_negative_rc_is_retriable(self) -> None: + """empty_output + COMPLETED + rc=-15 must be retriable. + + Process was killed by infrastructure before writing any stdout. + """ + session = ClaudeSessionResult( + subtype="empty_output", + is_error=True, + result="", + session_id="", + errors=[], + token_usage=None, + ) + retriable, reason = _compute_retry( + session, returncode=-15, termination=TerminationReason.COMPLETED + ) + assert retriable is True + assert reason == RetryReason.RESUME + + def test_context_exhausted_natural_exit_zero_rc_emits_resume(self) -> None: + """Context exhaustion on NATURAL_EXIT still emits RESUME. + + When _is_context_exhausted() returns True (jsonl_context_exhausted=True), + NATURAL_EXIT + kill_anomaly must still emit RESUME — partial context + progress exists on disk and is safe to resume. + """ + session = ClaudeSessionResult( + subtype="success", + is_error=False, + result="", + session_id="", + jsonl_context_exhausted=True, + ) + needs, reason = _compute_retry( + session, returncode=0, termination=TerminationReason.NATURAL_EXIT + ) + assert needs is True + assert reason == RetryReason.RESUME # context exhausted = partial progress exists + + def test_completed_kill_anomaly_still_emits_resume(self) -> None: + """Infrastructure kill (COMPLETED) + kill anomaly → RESUME unchanged. + + COMPLETED termination means infrastructure killed the process after channel + confirmation — the session made progress, partial results exist. RESUME + is correct here regardless of _is_context_exhausted(). + """ + session = ClaudeSessionResult(subtype="success", is_error=False, result="", session_id="") + needs, reason = _compute_retry( + session, + returncode=0, + termination=TerminationReason.COMPLETED, + channel_confirmation=ChannelConfirmation.CHANNEL_A, + ) + assert needs is True + assert reason == RetryReason.RESUME # infrastructure kill — progress existed + + +class TestComputeRetryCompletedPath: + """_compute_retry unique test: COMPLETED termination + unparseable.""" + + def test_unparseable_on_completed_returns_resume(self): + s = ClaudeSessionResult( + subtype="unparseable", is_error=True, result="garbled", session_id="" + ) + needs, reason = _compute_retry(s, -15, TerminationReason.COMPLETED) + assert needs is True + assert reason == RetryReason.RESUME + + +@pytest.mark.parametrize("termination", list(TerminationReason)) +def test_compute_retry_handles_all_termination_reasons_without_raising( + termination: TerminationReason, +) -> None: + """Exhaustiveness guard: _compute_retry must return valid (bool, RetryReason) + for every TerminationReason value.""" + session = ClaudeSessionResult( + subtype="success", + result="done. %%ORDER_UP%%", + is_error=False, + session_id="abc", + errors=[], + ) + result = _compute_retry(session, returncode=0, termination=termination) + assert isinstance(result, tuple) and len(result) == 2 + assert isinstance(result[0], bool) + assert isinstance(result[1], RetryReason) + + +def test_is_kill_anomaly_returns_true_for_interrupted_subtype(): + """'interrupted' subtype under COMPLETED termination must be retriable.""" + session = ClaudeSessionResult( + subtype="interrupted", + result="", + is_error=True, + session_id="abc", + errors=[], + ) + assert _is_kill_anomaly(session) is True + + +class TestSessionCompleteProperty: + """session_complete property must encode the canonical completion invariant.""" + + @pytest.mark.parametrize( + "subtype, is_error, expected", + [ + ("success", False, True), + ("error_max_turns", False, True), + ("error_during_execution", False, True), + ("context_exhaustion", False, True), + ("interrupted", False, True), + ("unknown", True, False), + ("empty_output", True, False), + ("unparseable", True, False), + ("timeout", True, False), + ], + ) + def test_session_complete_property_values( + self, subtype: str, is_error: bool, expected: bool + ) -> None: + """session_complete must be True iff not is_error AND subtype not in FAILURE_SUBTYPES.""" + session = ClaudeSessionResult( + subtype=subtype, + is_error=is_error, + result="", + session_id="test", + ) + assert session.session_complete is expected + + +class TestContentStateEnum: + """ContentState enum exposes the four expected variants.""" + + def test_content_state_enum_variants(self) -> None: + """ContentState exposes the four expected variants.""" + assert ContentState.COMPLETE == "complete" + assert ContentState.ABSENT == "absent" + assert ContentState.CONTRACT_VIOLATION == "contract_violation" + assert ContentState.SESSION_ERROR == "session_error" + + @pytest.mark.parametrize( + "result, is_error, completion_marker, patterns, expected_state", + [ + # COMPLETE: result present, marker present, patterns match + ( + "verdict = GO\n%%ORDER_UP%%", + False, + "%%ORDER_UP%%", + ["verdict\\s*=\\s*(GO|NO GO)"], + "complete", + ), + # ABSENT: empty result + ("", False, "%%ORDER_UP%%", ["verdict\\s*=\\s*(GO|NO GO)"], "absent"), + # ABSENT: marker missing from non-empty result + ("Some output, no marker", False, "%%ORDER_UP%%", [], "absent"), + # CONTRACT_VIOLATION: result + marker present, patterns fail + ( + "Done. %%ORDER_UP%%", + False, + "%%ORDER_UP%%", + ["verdict\\s*=\\s*(GO|NO GO)"], + "contract_violation", + ), + # SESSION_ERROR: is_error=True + ("Error: max turns exceeded", True, "%%ORDER_UP%%", [], "session_error"), + ], + ) + def test_evaluate_content_state( + self, + result: str, + is_error: bool, + completion_marker: str, + patterns: list[str], + expected_state: str, + make_session: Callable[..., ClaudeSessionResult], + ) -> None: + session = make_session(subtype="success", is_error=is_error, result=result) + state = _evaluate_content_state(session, completion_marker, patterns) + assert state.value == expected_state + + +# --------------------------------------------------------------------------- +# T2: API errors route to RetryReason.RESUME (not EMPTY_OUTPUT) +# --------------------------------------------------------------------------- + + +class TestApiErrorRetryRouting: + """API errors (529/overload/disconnect) must route to RESUME, not EMPTY_OUTPUT.""" + + def _make_result( + self, + stderr: str = "", + returncode: int = 1, + ): + from autoskillit.core.types import ChannelConfirmation, SubprocessResult, TerminationReason + + return SubprocessResult( + returncode=returncode, + stdout="", + stderr=stderr, + termination=TerminationReason.NATURAL_EXIT, + pid=12345, + channel_confirmation=ChannelConfirmation.UNMONITORED, + ) + + def test_api_error_overrides_empty_output_to_resume(self): + """When _build_skill_result detects API error in stderr, override + EMPTY_OUTPUT → RESUME for on_context_limit routing.""" + from autoskillit.core.types import RetryReason + from autoskillit.execution.headless._headless_result import _build_skill_result + + result = self._make_result(stderr="Error: API is overloaded", returncode=1) + sr = _build_skill_result(result) + assert sr.needs_retry is True + assert sr.retry_reason == RetryReason.RESUME + + def test_api_error_503_overrides_to_resume(self): + """503 Service Unavailable in stderr → RESUME.""" + from autoskillit.core.types import RetryReason + from autoskillit.execution.headless._headless_result import _build_skill_result + + result = self._make_result(stderr="503 Service Unavailable", returncode=1) + sr = _build_skill_result(result) + assert sr.needs_retry is True + assert sr.retry_reason == RetryReason.RESUME + + def test_api_error_econnreset_overrides_to_resume(self): + """ECONNRESET in stderr → RESUME.""" + from autoskillit.core.types import RetryReason + from autoskillit.execution.headless._headless_result import _build_skill_result + + result = self._make_result(stderr="read ECONNRESET", returncode=1) + sr = _build_skill_result(result) + assert sr.needs_retry is True + assert sr.retry_reason == RetryReason.RESUME diff --git a/tests/execution/test_session_adjudication_success.py b/tests/execution/test_session_adjudication_success.py new file mode 100644 index 0000000000..6409d768da --- /dev/null +++ b/tests/execution/test_session_adjudication_success.py @@ -0,0 +1,486 @@ +"""Tests for _compute_success adjudication logic.""" + +from __future__ import annotations + +import pytest + +from autoskillit.core.types import ( + ChannelConfirmation, + TerminationReason, +) +from autoskillit.execution.session import ( + ClaudeSessionResult, + _compute_retry, + _compute_success, + parse_session_result, +) +from tests.execution.conftest import _make_success_session + +pytestmark = [pytest.mark.layer("execution"), pytest.mark.small] + + +class TestComputeSuccess: + """_compute_success cross-validates all signals for unambiguous success.""" + + def test_all_good_is_success(self): + session = ClaudeSessionResult( + subtype="success", is_error=False, result="Done.", session_id="s1" + ) + assert ( + _compute_success(session, returncode=0, termination=TerminationReason.NATURAL_EXIT) + is True + ) + + def test_nonzero_exit_is_failure(self): + session = ClaudeSessionResult( + subtype="success", is_error=False, result="Done.", session_id="s1" + ) + assert ( + _compute_success(session, returncode=1, termination=TerminationReason.NATURAL_EXIT) + is False + ) + + def test_is_error_true_is_failure(self): + session = ClaudeSessionResult( + subtype="success", is_error=True, result="Error occurred", session_id="s1" + ) + assert ( + _compute_success(session, returncode=0, termination=TerminationReason.NATURAL_EXIT) + is False + ) + + def test_timed_out_is_failure(self): + session = ClaudeSessionResult( + subtype="success", is_error=False, result="Done.", session_id="s1" + ) + assert ( + _compute_success(session, returncode=0, termination=TerminationReason.TIMED_OUT) + is False + ) + + def test_stale_is_failure(self): + session = ClaudeSessionResult( + subtype="success", is_error=False, result="Done.", session_id="s1" + ) + assert ( + _compute_success(session, returncode=0, termination=TerminationReason.STALE) is False + ) + + def test_unknown_subtype_is_failure(self): + session = ClaudeSessionResult( + subtype="unknown", is_error=False, result="Done.", session_id="s1" + ) + assert ( + _compute_success(session, returncode=0, termination=TerminationReason.NATURAL_EXIT) + is False + ) + + def test_channel_b_bypasses_content_check(self): + """CHANNEL_B + COMPLETED + empty result → True (provenance bypass).""" + session = ClaudeSessionResult( + subtype="success", is_error=False, result="", session_id="s1" + ) + assert ( + _compute_success( + session, + returncode=0, + termination=TerminationReason.COMPLETED, + channel_confirmation=ChannelConfirmation.CHANNEL_B, + ) + is True + ) + + def test_channel_a_falls_through_to_content_check(self): + """CHANNEL_A + COMPLETED + empty result → False (content check required).""" + session = ClaudeSessionResult( + subtype="success", is_error=False, result="", session_id="s1" + ) + assert ( + _compute_success( + session, + returncode=0, + termination=TerminationReason.COMPLETED, + channel_confirmation=ChannelConfirmation.CHANNEL_A, + ) + is False + ) + + def test_unmonitored_falls_through_to_content_check(self): + """UNMONITORED delegates to normal gates.""" + session = ClaudeSessionResult( + subtype="success", is_error=False, result="", session_id="s1" + ) + assert ( + _compute_success( + session, + returncode=0, + termination=TerminationReason.NATURAL_EXIT, + channel_confirmation=ChannelConfirmation.UNMONITORED, + ) + is False + ) + + def test_natural_exit_channel_b_empty_result_true(self): + """NATURAL_EXIT + CHANNEL_B bypass fires before termination dispatch.""" + session = ClaudeSessionResult( + subtype="success", is_error=False, result="", session_id="s1" + ) + assert ( + _compute_success( + session, + returncode=0, + termination=TerminationReason.NATURAL_EXIT, + channel_confirmation=ChannelConfirmation.CHANNEL_B, + ) + is True + ) + + def test_natural_exit_channel_a_valid_result_true(self): + """NATURAL_EXIT + CHANNEL_A + valid content → True.""" + session = ClaudeSessionResult( + subtype="success", is_error=False, result="Done.", session_id="s1" + ) + assert ( + _compute_success( + session, + returncode=0, + termination=TerminationReason.NATURAL_EXIT, + channel_confirmation=ChannelConfirmation.CHANNEL_A, + ) + is True + ) + + +class TestComputeSuccessNaturalExitNonZero: + """NATURAL_EXIT with non-zero returncode. + + Default: failure (no bypass). Exception: post-completion kill bypass fires when + the session envelope is SUCCESS, result is non-empty, and the completion marker + is present — the signal is a teardown artifact, not a genuine CLI error. + """ + + def test_natural_exit_nonzero_returncode_without_marker_returns_false(self): + """NATURAL_EXIT + non-zero returncode + no completion marker → failure. + + Without a completion marker, we cannot distinguish a post-completion kill + from a genuine crash. Fail conservatively (PTY masking cannot be ruled out). + """ + session = ClaudeSessionResult( + subtype="success", is_error=False, result="done", session_id="s1" + ) + assert ( + _compute_success(session, returncode=1, termination=TerminationReason.NATURAL_EXIT) + is False + ) + + def test_completed_and_natural_exit_same_outcome_when_returncode_zero(self): + """COMPLETED and NATURAL_EXIT agree when returncode=0 (no PTY masking issue). + + Documents the symmetric case: asymmetry only matters when returncode != 0. + """ + session = ClaudeSessionResult( + subtype="success", is_error=False, result="done", session_id="s1" + ) + result_completed = _compute_success( + session, returncode=-15, termination=TerminationReason.COMPLETED + ) + result_natural = _compute_success( + session, returncode=0, termination=TerminationReason.NATURAL_EXIT + ) + assert result_completed is True + assert result_natural is True + + +class TestComputeSuccessNaturalExitPostCompletionKill: + """NATURAL_EXIT with non-zero returncode is success when completion marker is present.""" + + def test_natural_exit_sigkill_with_marker_is_success(self): + """NATURAL_EXIT + returncode -9 + SUCCESS subtype + completion marker → success. + + Documents the post-completion kill bypass: a process killed by an external + watchdog AFTER completing its work produces -9 on NATURAL_EXIT. The completion + marker in the result confirms the session finished before the kill. + """ + marker = "%%ORDER_UP%%" + session = ClaudeSessionResult( + subtype="success", + is_error=False, + result=f"fixes_applied = 2\n{marker}", + session_id="s1", + ) + assert ( + _compute_success( + session, + returncode=-9, + termination=TerminationReason.NATURAL_EXIT, + completion_marker=marker, + ) + is True + ) + + def test_natural_exit_sigkill_without_marker_remains_failure(self): + """No bypass when no completion_marker is configured — conservative path unchanged.""" + session = ClaudeSessionResult( + subtype="success", + is_error=False, + result="fixes_applied = 2", + session_id="s1", + ) + assert ( + _compute_success( + session, + returncode=-9, + termination=TerminationReason.NATURAL_EXIT, + completion_marker="", # no marker configured + ) + is False + ) + + def test_natural_exit_sigkill_marker_absent_from_result_remains_failure(self): + """Bypass does not fire when marker is configured but missing from result.""" + marker = "%%ORDER_UP%%" + session = ClaudeSessionResult( + subtype="success", + is_error=False, + result="partial output without marker", + session_id="s1", + ) + assert ( + _compute_success( + session, + returncode=-9, + termination=TerminationReason.NATURAL_EXIT, + completion_marker=marker, + ) + is False + ) + + def test_natural_exit_nonzero_non_success_subtype_remains_failure(self): + """Bypass does not fire for non-success subtypes even with marker present.""" + marker = "%%ORDER_UP%%" + session = ClaudeSessionResult( + subtype="unparseable", + is_error=False, + result=f"some output {marker}", + session_id="s1", + ) + assert ( + _compute_success( + session, + returncode=-9, + termination=TerminationReason.NATURAL_EXIT, + completion_marker=marker, + ) + is False + ) + + +class TestComputeSuccessCompletionMarker: + """_compute_success tests unique to test_session_result.py (completion_marker path).""" + + def test_nonzero_returncode_recoverable_path(self): + s = ClaudeSessionResult( + subtype="success", is_error=False, result="great output", session_id="s1" + ) + assert _compute_success(s, 1, TerminationReason.COMPLETED) is True + + def test_failure_subtype_false_for_more_subtypes(self): + for subtype in ("empty_output", "unparseable", "timeout"): + s = ClaudeSessionResult( + subtype=subtype, is_error=False, result="some text", session_id="s1" + ) + assert _compute_success(s, 0, TerminationReason.NATURAL_EXIT) is False + + def test_missing_completion_marker_false(self): + s = _make_success_session("result without marker") + assert ( + _compute_success(s, 0, TerminationReason.NATURAL_EXIT, completion_marker="%%DONE%%") + is False + ) + + def test_result_is_only_marker_false(self): + marker = "%%DONE%%" + s = _make_success_session(marker) + assert ( + _compute_success(s, 0, TerminationReason.NATURAL_EXIT, completion_marker=marker) + is False + ) + + +class TestComputeSuccessRealisticInputs: + """_compute_success using parse_session_result() as input constructor.""" + + def test_empty_stdout_parses_to_empty_output_adjudicates_false(self): + session = parse_session_result("") + assert session.subtype == "empty_output" + assert session.is_error is True + assert _compute_success(session, 0, TerminationReason.NATURAL_EXIT) is False + + def test_garbled_stdout_parses_to_unparseable_adjudicates_false(self): + session = parse_session_result("Traceback (most recent call last):\n boom\n") + assert session.subtype == "unparseable" + assert session.is_error is True + assert _compute_success(session, 0, TerminationReason.NATURAL_EXIT) is False + + def test_empty_stdout_not_bypassed_by_completed_path(self): + session = parse_session_result("") + assert _compute_success(session, -15, TerminationReason.COMPLETED) is False + + def test_unparseable_not_bypassed_by_completed_path(self): + session = parse_session_result("garbled output not json\n") + assert _compute_success(session, -15, TerminationReason.COMPLETED) is False + + +class TestComputeSuccessChannelBPatterns: + """Channel B bypass must not skip expected_output_patterns validation.""" + + def test_compute_success_channel_b_bypass_with_missing_patterns_returns_false( + self, + ) -> None: + """Channel B bypass must not skip expected_output_patterns validation. + + When patterns are configured and absent from session.result, success must + be False (not True) even when channel_confirmation is CHANNEL_B. + """ + session = ClaudeSessionResult( + subtype="success", + result="Final message without the block.", # pattern absent + is_error=False, + session_id="s1", + assistant_messages=["Final message without the block."], + ) + success = _compute_success( + session=session, + returncode=0, + termination=TerminationReason.COMPLETED, + channel_confirmation=ChannelConfirmation.CHANNEL_B, + completion_marker="%%ORDER_UP%%", + expected_output_patterns=["---prepare-issue-result---"], + ) + assert success is False, "Channel B bypass must not skip expected_output_patterns check" + + def test_channel_b_bypass_fails_for_failure_subtype_with_injected_pattern( + self, + ) -> None: + """CHANNEL_B bypass must not return True for failure-subtype sessions. + + Even when synthesis has already run and injected a matching line into + session.result, CHANNEL_B bypass must refuse because session.session_complete + is False. + """ + from autoskillit.core.types import CliSubtype + + # Simulate: synthesis already ran and injected the line into result + session = ClaudeSessionResult( + subtype=CliSubtype.UNPARSEABLE, + is_error=True, + result="plan_path = /cwd/.autoskillit/temp/make-plan/arch_lens_2026.md\n", + session_id="s1", + ) + success = _compute_success( + session=session, + returncode=0, + termination=TerminationReason.COMPLETED, + channel_confirmation=ChannelConfirmation.CHANNEL_B, + expected_output_patterns=[r"plan_path\s*=\s*/.+"], + ) + assert success is False, "CHANNEL_B bypass must not accept failure-subtype sessions" + + +class TestMinus9ReturncodeCoverage: + """Specification tests: returncode=-9 under COMPLETED is a valid success (1c). + + Before the fix, channel_won unconditionally sent SIGKILL (-9) to every + COMPLETED session. These tests document that the adjudicator correctly + classifies -9 under COMPLETED as successful — the bug was in the kill logic, + not the adjudicator. + """ + + def test_compute_success_completed_returncode_minus_9_is_success(self) -> None: + """COMPLETED + CHANNEL_A + returncode=-9 + content → success=True.""" + session = _make_success_session("ok") + result = _compute_success( + session, + returncode=-9, + termination=TerminationReason.COMPLETED, + channel_confirmation=ChannelConfirmation.CHANNEL_A, + ) + assert result is True + + def test_compute_success_completed_minus_9_channel_b_is_success(self) -> None: + """COMPLETED + CHANNEL_B + returncode=-9 → success=True (bypass).""" + session = ClaudeSessionResult( + subtype="success", + result="", + is_error=False, + session_id="b9", + ) + result = _compute_success( + session, + returncode=-9, + termination=TerminationReason.COMPLETED, + channel_confirmation=ChannelConfirmation.CHANNEL_B, + ) + assert result is True + + @pytest.mark.parametrize("returncode", [0, -15, -9]) + @pytest.mark.parametrize("termination", list(TerminationReason)) + def test_compute_retry_handles_minus_9_for_all_termination_reasons( + self, + termination: TerminationReason, + returncode: int, + ) -> None: + """_compute_retry must not raise for any (TerminationReason, returncode) pair.""" + session = ClaudeSessionResult( + subtype="success", + result="done. %%ORDER_UP%%", + is_error=False, + session_id="rc9", + errors=[], + ) + result = _compute_retry(session, returncode=returncode, termination=termination) + assert isinstance(result, tuple) and len(result) == 2 + + @pytest.mark.parametrize("returncode", [0, -15, -9]) + @pytest.mark.parametrize("termination", list(TerminationReason)) + def test_compute_success_handles_minus_9_for_all_termination_reasons( + self, + termination: TerminationReason, + returncode: int, + ) -> None: + """_compute_success must not raise for any (TerminationReason, returncode) pair.""" + session = ClaudeSessionResult( + subtype="success", + result="done. %%ORDER_UP%%", + is_error=False, + session_id="rc9", + errors=[], + ) + result = _compute_success( + session, + returncode=returncode, + termination=termination, + completion_marker="%%ORDER_UP%%", + ) + assert isinstance(result, bool) + + +@pytest.mark.parametrize("termination", list(TerminationReason)) +def test_compute_success_handles_all_termination_reasons_without_raising( + termination: TerminationReason, +) -> None: + """Exhaustiveness guard for _compute_success: must return bool for every TerminationReason.""" + session = ClaudeSessionResult( + subtype="success", + result="done. %%ORDER_UP%%", + is_error=False, + session_id="abc", + errors=[], + ) + result = _compute_success( + session, + returncode=0, + termination=termination, + completion_marker="%%ORDER_UP%%", + ) + assert isinstance(result, bool) diff --git a/tests/execution/test_session_content.py b/tests/execution/test_session_content.py new file mode 100644 index 0000000000..dbaa463905 --- /dev/null +++ b/tests/execution/test_session_content.py @@ -0,0 +1,67 @@ +"""Tests for session content validation and token normalization.""" + +from __future__ import annotations + +import pytest + +from autoskillit.execution.session._session_content import _strip_markdown_from_tokens + +pytestmark = [pytest.mark.layer("execution"), pytest.mark.small] + + +class TestStripMarkdownFromTokens: + """_strip_markdown_from_tokens normalizes model output decorators to canonical form.""" + + def test_bold_equals_lowercase(self) -> None: + """**plan_path** = /abs/path → plan_path = /abs/path""" + result = _strip_markdown_from_tokens("**plan_path** = /abs/path/plan.md") + assert result == "plan_path = /abs/path/plan.md" + + def test_bold_equals_uppercase_key_lowercased(self) -> None: + """**Plan_Path** = /abs/path → plan_path = /abs/path""" + result = _strip_markdown_from_tokens("**Plan_Path** = /abs/path/plan.md") + assert result == "plan_path = /abs/path/plan.md" + + def test_italic_equals(self) -> None: + """*plan_path* = /abs/path → plan_path = /abs/path""" + result = _strip_markdown_from_tokens("*plan_path* = /abs/path/plan.md") + assert result == "plan_path = /abs/path/plan.md" + + def test_bold_colon_separator(self) -> None: + """**Plan_Path:** /abs/path → plan_path = /abs/path""" + result = _strip_markdown_from_tokens("**Plan_Path:** /abs/path/plan.md") + assert result == "plan_path = /abs/path/plan.md" + + def test_backtick_equals(self) -> None: + """`plan_path` = /abs/path → plan_path = /abs/path""" + result = _strip_markdown_from_tokens("`plan_path` = /abs/path/plan.md") + assert result == "plan_path = /abs/path/plan.md" + + def test_backtick_equals_uppercase_key_lowercased(self) -> None: + """`Plan_Path` = /abs/path → plan_path = /abs/path""" + result = _strip_markdown_from_tokens("`Plan_Path` = /abs/path/plan.md") + assert result == "plan_path = /abs/path/plan.md" + + def test_canonical_no_change(self) -> None: + """Already-canonical token is unchanged.""" + result = _strip_markdown_from_tokens("plan_path = /abs/path/plan.md") + assert result == "plan_path = /abs/path/plan.md" + + def test_non_adjacent_bold_untouched(self) -> None: + """Bold decorators not adjacent to = or : are left unchanged.""" + result = _strip_markdown_from_tokens("Some text with **bold** words here") + assert result == "Some text with **bold** words here" + + def test_multiple_tokens_all_normalized(self) -> None: + """Multiple decorated tokens in one string are all normalized.""" + result = _strip_markdown_from_tokens( + "**worktree_path** = /tmp/wt\n`branch_name` = impl-xyz\n*status* = running\n" + ) + assert "worktree_path = /tmp/wt" in result + assert "branch_name = impl-xyz" in result + assert "status = running" in result + + def test_worktree_path_bold_colon_variant(self) -> None: + """The colon-decorated variant that caused #1716 contract failures.""" + result = _strip_markdown_from_tokens("**worktree_path:** /tmp/wt") + assert result == "worktree_path = /tmp/wt" diff --git a/tests/execution/test_session_debug_logging.py b/tests/execution/test_session_debug_logging.py index 918f685bc6..8e43b0305f 100644 --- a/tests/execution/test_session_debug_logging.py +++ b/tests/execution/test_session_debug_logging.py @@ -2,6 +2,7 @@ from __future__ import annotations +import pytest import structlog.testing from autoskillit.core.types import ChannelConfirmation, TerminationReason @@ -13,6 +14,8 @@ _compute_success, ) +pytestmark = [pytest.mark.layer("execution"), pytest.mark.small] + class TestCheckSessionContentLogging: """Verify _check_session_content logs exact failure reason.""" diff --git a/tests/execution/test_session_log_fields.py b/tests/execution/test_session_log_fields.py new file mode 100644 index 0000000000..9e76b927b0 --- /dev/null +++ b/tests/execution/test_session_log_fields.py @@ -0,0 +1,855 @@ +"""Tests for flush_session_log field coverage: write warnings, call counts, +kitchen/order IDs, crash exception, raw stdout, per-turn fields, tool calls, +silent gap, proc-trace exit snapshot, versions block, and recipe identity.""" + +from __future__ import annotations + +import json +import os +from datetime import UTC, datetime + +import pytest + +from autoskillit.core.types._type_results import ProviderOutcome +from autoskillit.core.types._type_results_execution import ( + RecipeIdentity, + SessionTelemetry, +) +from autoskillit.execution.session_log import ( + flush_session_log, +) +from tests.execution.conftest import ( + _flush, + _make_cc_jsonl_record, + _make_thinking_block, + _make_tool_block, + _snap, +) + +pytestmark = [pytest.mark.layer("execution"), pytest.mark.medium] + + +def test_flush_session_log_includes_write_path_warnings_in_summary(tmp_path): + """summary.json records write_path_warnings list.""" + warnings = [ + "Write tool wrote to /source/repo/.autoskillit/temp/foo.md (outside cwd /clone)", + "Edit tool wrote to /source/repo/src/file.py (outside cwd /clone)", + ] + _flush(tmp_path, session_id="warn-session", write_path_warnings=warnings, proc_snapshots=None) + summary = json.loads((tmp_path / "sessions" / "warn-session" / "summary.json").read_text()) + assert summary["write_path_warnings"] == warnings + + +def test_flush_session_log_empty_warnings_produce_empty_list(tmp_path): + """No warnings → write_path_warnings is [] in summary.""" + _flush(tmp_path, session_id="clean-session", write_path_warnings=[], proc_snapshots=None) + summary = json.loads((tmp_path / "sessions" / "clean-session" / "summary.json").read_text()) + assert summary["write_path_warnings"] == [] + + +def test_flush_session_log_none_warnings_treated_as_empty(tmp_path): + """write_path_warnings=None (default) produces empty list in summary.""" + _flush(tmp_path, session_id="default-warn", proc_snapshots=None) # no write_path_warnings arg + summary = json.loads((tmp_path / "sessions" / "default-warn" / "summary.json").read_text()) + assert summary["write_path_warnings"] == [] + + +def test_flush_session_log_includes_write_call_count_in_summary(tmp_path): + """summary.json records write_call_count.""" + _flush(tmp_path, session_id="wc-session", write_call_count=5, proc_snapshots=None) + summary = json.loads((tmp_path / "sessions" / "wc-session" / "summary.json").read_text()) + assert summary["write_call_count"] == 5 + + +def test_flush_session_log_write_call_count_in_index(tmp_path): + """sessions.jsonl index includes write_call_count.""" + _flush(tmp_path, session_id="wc-idx", write_call_count=3, proc_snapshots=None) + index_path = tmp_path / "sessions.jsonl" + entry = json.loads(index_path.read_text().strip().split("\n")[-1]) + assert entry["write_call_count"] == 3 + + +def test_flush_session_log_write_call_count_defaults_to_zero(tmp_path): + """write_call_count defaults to 0 when not specified.""" + _flush(tmp_path, session_id="wc-default", proc_snapshots=None) + summary = json.loads((tmp_path / "sessions" / "wc-default" / "summary.json").read_text()) + assert summary["write_call_count"] == 0 + + +def test_flush_session_log_writes_kitchen_id(tmp_path): + """kitchen_id parameter is written to sessions.jsonl index entry.""" + flush_session_log( + log_dir=str(tmp_path), + cwd="/some/worktree", + kitchen_id="my-pipeline-123", + session_id="sess-001", + pid=12345, + skill_command="/autoskillit:implement", + success=True, + subtype="completed", + exit_code=0, + start_ts="2026-03-27T08:00:00", + proc_snapshots=None, + telemetry=SessionTelemetry.empty(), + provider_outcome=ProviderOutcome.none_used(), + recipe_identity=RecipeIdentity.empty(), + ) + + index = (tmp_path / "sessions.jsonl").read_text() + entry = json.loads(index.strip()) + assert entry["kitchen_id"] == "my-pipeline-123" + + +def test_flush_session_log_writes_order_id_to_index(tmp_path): + """order_id is written to sessions.jsonl index entry when provided.""" + flush_session_log( + log_dir=str(tmp_path), + cwd="/some/worktree", + kitchen_id="kitchen-abc", + order_id="issue-185", + session_id="sess-002", + pid=12345, + skill_command="/autoskillit:implement", + success=True, + subtype="completed", + exit_code=0, + start_ts="2026-03-27T08:00:00", + proc_snapshots=None, + telemetry=SessionTelemetry.empty(), + provider_outcome=ProviderOutcome.none_used(), + recipe_identity=RecipeIdentity.empty(), + ) + + entry = json.loads((tmp_path / "sessions.jsonl").read_text().strip()) + assert entry["order_id"] == "issue-185" + + +def test_flush_session_log_order_id_defaults_to_empty(tmp_path): + """order_id defaults to empty string when not supplied.""" + flush_session_log( + log_dir=str(tmp_path), + cwd="/some/worktree", + kitchen_id="kitchen-abc", + session_id="sess-003", + pid=12345, + skill_command="/autoskillit:implement", + success=True, + subtype="completed", + exit_code=0, + start_ts="2026-03-27T08:00:00", + proc_snapshots=None, + telemetry=SessionTelemetry.empty(), + provider_outcome=ProviderOutcome.none_used(), + recipe_identity=RecipeIdentity.empty(), + ) + + entry = json.loads((tmp_path / "sessions.jsonl").read_text().strip()) + assert "order_id" in entry + assert entry["order_id"] == "" + + +def test_flush_writes_crash_exception_file(tmp_path): + """When exception_text is provided, flush_session_log writes crash_exception.txt.""" + flush_session_log( + log_dir=str(tmp_path), + cwd="/tmp", + session_id="test-session", + pid=1234, + skill_command="/test", + success=False, + subtype="crashed", + exit_code=-1, + start_ts=datetime.now(UTC).isoformat(), + proc_snapshots=None, + termination_reason="CRASHED", + exception_text="RuntimeError: boom\n at headless.py:1023", + telemetry=SessionTelemetry.empty(), + provider_outcome=ProviderOutcome.none_used(), + recipe_identity=RecipeIdentity.empty(), + ) + session_dir = tmp_path / "sessions" / "test-session" + crash_file = session_dir / "crash_exception.txt" + assert crash_file.exists() + assert "RuntimeError: boom" in crash_file.read_text() + + +# --------------------------------------------------------------------------- +# raw_stdout and per-turn field tests +# --------------------------------------------------------------------------- + + +def test_flush_session_log_writes_raw_stdout_on_failure(tmp_path): + raw = '{"type": "assistant"}\n{"type": "result"}\n' + flush_session_log( + log_dir=str(tmp_path), + cwd="/tmp", + session_id="test-session", + pid=1, + skill_command="test", + success=False, + subtype="empty_output", + exit_code=-1, + start_ts="2026-04-15T07:00:00Z", + proc_snapshots=None, + raw_stdout=raw, + telemetry=SessionTelemetry.empty(), + provider_outcome=ProviderOutcome.none_used(), + recipe_identity=RecipeIdentity.empty(), + ) + raw_file = tmp_path / "sessions" / "test-session" / "raw_stdout.jsonl" + assert raw_file.exists() + assert raw_file.read_text() == raw + + +def test_flush_session_log_no_raw_stdout_on_success(tmp_path): + flush_session_log( + log_dir=str(tmp_path), + cwd="/tmp", + session_id="ok-session", + pid=1, + skill_command="test", + success=True, + subtype="success", + exit_code=0, + start_ts="2026-04-15T07:00:00Z", + proc_snapshots=None, + raw_stdout='{"type": "result"}', + telemetry=SessionTelemetry.empty(), + provider_outcome=ProviderOutcome.none_used(), + recipe_identity=RecipeIdentity.empty(), + ) + raw_file = tmp_path / "sessions" / "ok-session" / "raw_stdout.jsonl" + assert not raw_file.exists() + + +def test_flush_session_log_summary_contains_per_turn_fields(tmp_path, monkeypatch): + cb_log = tmp_path / "s.jsonl" + cb_log.write_text( + json.dumps( + {"type": "assistant", "requestId": "req-001", "timestamp": "2026-04-15T07:00:00Z"} + ) + + "\n" + + json.dumps( + {"type": "assistant", "requestId": "req-002", "timestamp": "2026-04-15T07:00:05Z"} + ) + + "\n" + ) + import autoskillit.execution.session_log as sl_mod + + monkeypatch.setattr(sl_mod, "claude_code_log_path", lambda cwd, sid: cb_log) + + flush_session_log( + log_dir=str(tmp_path), + cwd="/tmp", + session_id="s", + pid=1, + skill_command="test", + success=False, + subtype="empty_output", + exit_code=-1, + start_ts="2026-04-15T07:00:00Z", + proc_snapshots=None, + last_stop_reason="end_turn", + telemetry=SessionTelemetry.empty(), + provider_outcome=ProviderOutcome.none_used(), + recipe_identity=RecipeIdentity.empty(), + ) + summary = json.loads((tmp_path / "sessions" / "s" / "summary.json").read_text()) + assert summary["last_stop_reason"] == "end_turn" + assert summary["request_ids"] == ["req-001", "req-002"] + assert summary["turn_timestamps"] == ["2026-04-15T07:00:00Z", "2026-04-15T07:00:05Z"] + + +# turn_tool_calls + + +def test_flush_session_log_summary_contains_turn_tool_calls(tmp_path, monkeypatch): + cb_log = tmp_path / "s.jsonl" + cb_log.write_text( + json.dumps( + { + "type": "assistant", + "requestId": "req-001", + "message": { + "content": [ + {"type": "tool_use", "name": "ToolA"}, + {"type": "tool_use", "name": "ToolB"}, + ] + }, + } + ) + + "\n" + ) + import autoskillit.execution.session_log as sl_mod + + monkeypatch.setattr(sl_mod, "claude_code_log_path", lambda cwd, sid: cb_log) + + flush_session_log( + log_dir=str(tmp_path), + cwd="/tmp", + session_id="s", + pid=1, + skill_command="test", + success=True, + subtype="completed", + exit_code=0, + start_ts="2026-04-15T07:00:00Z", + proc_snapshots=None, + telemetry=SessionTelemetry.empty(), + provider_outcome=ProviderOutcome.none_used(), + recipe_identity=RecipeIdentity.empty(), + ) + summary = json.loads((tmp_path / "sessions" / "s" / "summary.json").read_text()) + assert summary["turn_tool_calls"] == [["ToolA", "ToolB"]] + + +def test_turn_tool_calls_capped_at_8_per_turn(tmp_path, monkeypatch): + tools = [{"type": "tool_use", "name": f"Tool{i}"} for i in range(10)] + cb_log = tmp_path / "s.jsonl" + cb_log.write_text( + json.dumps( + { + "type": "assistant", + "requestId": "req-001", + "message": {"content": tools}, + } + ) + + "\n" + ) + import autoskillit.execution.session_log as sl_mod + + monkeypatch.setattr(sl_mod, "claude_code_log_path", lambda cwd, sid: cb_log) + flush_session_log( + log_dir=str(tmp_path), + cwd="/tmp", + session_id="s", + pid=1, + skill_command="test", + success=True, + subtype="completed", + exit_code=0, + start_ts="2026-04-15T07:00:00Z", + proc_snapshots=None, + telemetry=SessionTelemetry.empty(), + provider_outcome=ProviderOutcome.none_used(), + recipe_identity=RecipeIdentity.empty(), + ) + summary = json.loads((tmp_path / "sessions" / "s" / "summary.json").read_text()) + assert len(summary["turn_tool_calls"][0]) == 8 + assert summary["turn_tool_calls"][0] == [f"Tool{i}" for i in range(8)] + + +def test_turn_tool_calls_empty_for_text_only_turn(tmp_path, monkeypatch): + cb_log = tmp_path / "s.jsonl" + cb_log.write_text( + json.dumps( + { + "type": "assistant", + "requestId": "req-001", + "message": {"content": [{"type": "text", "text": "hello"}]}, + } + ) + + "\n" + ) + import autoskillit.execution.session_log as sl_mod + + monkeypatch.setattr(sl_mod, "claude_code_log_path", lambda cwd, sid: cb_log) + flush_session_log( + log_dir=str(tmp_path), + cwd="/tmp", + session_id="s", + pid=1, + skill_command="test", + success=True, + subtype="completed", + exit_code=0, + start_ts="2026-04-15T07:00:00Z", + proc_snapshots=None, + telemetry=SessionTelemetry.empty(), + provider_outcome=ProviderOutcome.none_used(), + recipe_identity=RecipeIdentity.empty(), + ) + summary = json.loads((tmp_path / "sessions" / "s" / "summary.json").read_text()) + assert summary["turn_tool_calls"] == [[]] + + +def test_turn_tool_calls_parallel_to_request_ids(tmp_path, monkeypatch): + records = [ + json.dumps( + { + "type": "assistant", + "requestId": f"req-{i}", + "message": {"content": [{"type": "tool_use", "name": f"Tool{i}"}]}, + } + ) + for i in range(3) + ] + cb_log = tmp_path / "s.jsonl" + cb_log.write_text("\n".join(records) + "\n") + import autoskillit.execution.session_log as sl_mod + + monkeypatch.setattr(sl_mod, "claude_code_log_path", lambda cwd, sid: cb_log) + flush_session_log( + log_dir=str(tmp_path), + cwd="/tmp", + session_id="s", + pid=1, + skill_command="test", + success=True, + subtype="completed", + exit_code=0, + start_ts="2026-04-15T07:00:00Z", + proc_snapshots=None, + telemetry=SessionTelemetry.empty(), + provider_outcome=ProviderOutcome.none_used(), + recipe_identity=RecipeIdentity.empty(), + ) + summary = json.loads((tmp_path / "sessions" / "s" / "summary.json").read_text()) + assert len(summary["turn_tool_calls"]) == len(summary["request_ids"]) == 3 + + +# --------------------------------------------------------------------------- +# Silent gap, outcome anomaly, and exit snapshot tests +# --------------------------------------------------------------------------- + + +def test_summary_includes_silent_gap_seconds(tmp_path, monkeypatch): + """silent_gap_seconds computed from cc_log mtime vs end_ts — approx 5.0s.""" + import autoskillit.execution.session_log as sl_mod + + cb_log = tmp_path / "session.jsonl" + cb_log.write_text("") + end_ts = "2026-04-15T07:00:10+00:00" + end_dt = datetime.fromisoformat(end_ts) + os.utime(cb_log, (end_dt.timestamp() - 5.0,) * 2) + monkeypatch.setattr(sl_mod, "claude_code_log_path", lambda cwd, sid: cb_log) + _flush(tmp_path, session_id="gap-test", end_ts=end_ts, proc_snapshots=None) + summary = json.loads((tmp_path / "sessions" / "gap-test" / "summary.json").read_text()) + assert "silent_gap_seconds" in summary + assert summary["silent_gap_seconds"] == pytest.approx(5.0, abs=0.5) + + +def test_summary_silent_gap_seconds_null_when_no_end_ts(tmp_path, monkeypatch): + """silent_gap_seconds is null when end_ts is not provided.""" + import autoskillit.execution.session_log as sl_mod + + cb_log = tmp_path / "session.jsonl" + cb_log.write_text("") + monkeypatch.setattr(sl_mod, "claude_code_log_path", lambda cwd, sid: cb_log) + _flush(tmp_path, session_id="no-end-ts", end_ts="", proc_snapshots=None) + summary = json.loads((tmp_path / "sessions" / "no-end-ts" / "summary.json").read_text()) + assert summary["silent_gap_seconds"] is None + + +def test_summary_silent_gap_seconds_null_when_cc_log_missing(tmp_path): + """silent_gap_seconds is null when claude_code_log cannot be resolved.""" + _flush( + tmp_path, + session_id="no-cc-log", + end_ts="2026-04-15T07:00:10+00:00", + proc_snapshots=None, + cwd="/nonexistent/path", + ) + summary = json.loads((tmp_path / "sessions" / "no-cc-log" / "summary.json").read_text()) + assert summary["silent_gap_seconds"] is None + + +def test_flush_outcome_anomaly_included_in_anomaly_count(tmp_path, monkeypatch): + """empty_result + output_tokens > 0 increments anomaly_count in summary and index.""" + import autoskillit.execution.session_log as sl_mod + + monkeypatch.setattr(sl_mod, "claude_code_log_path", lambda cwd, sid: None) + _flush( + tmp_path, + session_id="outcome-anomaly", + subtype="empty_result", + success=False, + token_usage={"output_tokens": 945, "input_tokens": 500}, + proc_snapshots=None, + ) + summary = json.loads((tmp_path / "sessions" / "outcome-anomaly" / "summary.json").read_text()) + assert summary["anomaly_count"] >= 1 + anomalies_path = tmp_path / "sessions" / "outcome-anomaly" / "anomalies.jsonl" + assert anomalies_path.exists() + kinds = [json.loads(line)["kind"] for line in anomalies_path.read_text().splitlines() if line] + assert "empty_result_with_tokens" in kinds + index_entry = json.loads((tmp_path / "sessions.jsonl").read_text().strip()) + assert index_entry["anomaly_count"] >= 1 + + +def test_proc_trace_preserves_exit_snapshot_event(tmp_path): + """proc_trace.jsonl rows with event='exit_snapshot' preserve the marker.""" + exit_snap = {**_snap(), "event": "exit_snapshot"} + _flush( + tmp_path, + session_id="exit-snap-test", + proc_snapshots=[_snap(), _snap(), exit_snap], + ) + rows = [ + json.loads(line) + for line in (tmp_path / "sessions" / "exit-snap-test" / "proc_trace.jsonl") + .read_text() + .splitlines() + ] + assert rows[0]["event"] == "snapshot" + assert rows[1]["event"] == "snapshot" + assert rows[2]["event"] == "exit_snapshot" + + +# --- Versions block tests --- + +_VERSIONS = { + "autoskillit_version": "1.2.3", + "install_type": "local-editable", + "commit_id": None, + "claude_code_version": "1.0.5", + "plugins": [], +} + + +def test_summary_json_includes_versions_block(tmp_path): + _flush(tmp_path, session_id="vs-001", versions=_VERSIONS) + summary = json.loads((tmp_path / "sessions" / "vs-001" / "summary.json").read_text()) + assert "versions" in summary + assert summary["versions"]["autoskillit_version"] == "1.2.3" + assert summary["versions"]["claude_code_version"] == "1.0.5" + + +def test_summary_json_versions_includes_model_identifier(tmp_path): + _flush(tmp_path, session_id="vs-002", versions=_VERSIONS, model_identifier="claude-opus-4") + summary = json.loads((tmp_path / "sessions" / "vs-002" / "summary.json").read_text()) + assert summary["versions"]["model_identifier"] == "claude-opus-4" + + +def test_summary_json_omits_versions_when_not_passed(tmp_path): + _flush(tmp_path, session_id="vs-003") + summary = json.loads((tmp_path / "sessions" / "vs-003" / "summary.json").read_text()) + assert "versions" not in summary + + +def test_sessions_jsonl_includes_autoskillit_version(tmp_path): + _flush(tmp_path, session_id="vs-004", versions=_VERSIONS) + entries = [ + json.loads(line) + for line in (tmp_path / "sessions.jsonl").read_text().splitlines() + if line.strip() + ] + entry = next(e for e in entries if e["session_id"] == "vs-004") + assert entry["autoskillit_version"] == "1.2.3" + + +def test_sessions_jsonl_includes_claude_code_version(tmp_path): + _flush(tmp_path, session_id="vs-005", versions=_VERSIONS) + entries = [ + json.loads(line) + for line in (tmp_path / "sessions.jsonl").read_text().splitlines() + if line.strip() + ] + entry = next(e for e in entries if e["session_id"] == "vs-005") + assert entry["claude_code_version"] == "1.0.5" + + +def test_sessions_jsonl_autoskillit_version_empty_when_no_versions(tmp_path): + _flush(tmp_path, session_id="vs-006") + entries = [ + json.loads(line) + for line in (tmp_path / "sessions.jsonl").read_text().splitlines() + if line.strip() + ] + entry = next(e for e in entries if e["session_id"] == "vs-006") + assert entry["autoskillit_version"] == "" + + +def test_session_log_includes_recipe_name(tmp_path): + _flush(tmp_path, session_id="rp-001", recipe_name="impl") + entries = [ + json.loads(line) + for line in (tmp_path / "sessions.jsonl").read_text().splitlines() + if line.strip() + ] + entry = next(e for e in entries if e["session_id"] == "rp-001") + assert entry["recipe_name"] == "impl" + + +def test_session_log_includes_recipe_hashes(tmp_path): + _flush( + tmp_path, + session_id="rp-002", + recipe_content_hash="sha256:abc", + recipe_composite_hash="sha256:def", + ) + entries = [ + json.loads(line) + for line in (tmp_path / "sessions.jsonl").read_text().splitlines() + if line.strip() + ] + entry = next(e for e in entries if e["session_id"] == "rp-002") + assert entry["recipe_content_hash"] == "sha256:abc" + assert entry["recipe_composite_hash"] == "sha256:def" + + +def test_summary_includes_recipe_provenance(tmp_path): + _flush( + tmp_path, + session_id="rp-003", + recipe_name="impl", + recipe_content_hash="sha256:abc", + recipe_composite_hash="sha256:def", + recipe_version="1.0.0", + ) + session_dir = tmp_path / "sessions" / "rp-003" + summary = json.loads((session_dir / "summary.json").read_text()) + assert "recipe_provenance" in summary + assert summary["recipe_provenance"]["schema_version"] == 1 + assert summary["recipe_provenance"]["name"] == "impl" + assert summary["recipe_provenance"]["content_hash"] == "sha256:abc" + assert summary["recipe_provenance"]["composite_hash"] == "sha256:def" + assert summary["recipe_provenance"]["version"] == "1.0.0" + + +def test_session_log_empty_recipe_identity(tmp_path): + _flush(tmp_path, session_id="rp-004") + entries = [ + json.loads(line) + for line in (tmp_path / "sessions.jsonl").read_text().splitlines() + if line.strip() + ] + entry = next(e for e in entries if e["session_id"] == "rp-004") + assert entry["recipe_name"] == "" + assert entry["recipe_content_hash"] == "" + assert entry["recipe_composite_hash"] == "" + + +def test_summary_no_recipe_provenance_when_empty(tmp_path): + _flush(tmp_path, session_id="rp-005") + session_dir = tmp_path / "sessions" / "rp-005" + summary = json.loads((session_dir / "summary.json").read_text()) + assert "recipe_provenance" not in summary + + +def test_flush_index_includes_duration_seconds(tmp_path): + """sessions.jsonl index entry includes duration_seconds.""" + _flush(tmp_path, elapsed_seconds=42.5) + index = (tmp_path / "sessions.jsonl").read_text().strip() + entry = json.loads(index) + assert entry["duration_seconds"] == pytest.approx(42.5) + + +# --------------------------------------------------------------------------- +# Provider field tests +# --------------------------------------------------------------------------- + + +def test_flush_session_log_provider_used_in_summary(tmp_path): + _flush(tmp_path, session_id="prov-sum", provider_used="minimax", proc_snapshots=None) + summary = json.loads((tmp_path / "sessions" / "prov-sum" / "summary.json").read_text()) + assert summary["provider_used"] == "minimax" + + +def test_flush_session_log_provider_fallback_in_summary(tmp_path): + _flush(tmp_path, session_id="fb-sum", provider_fallback=True, proc_snapshots=None) + summary = json.loads((tmp_path / "sessions" / "fb-sum" / "summary.json").read_text()) + assert summary["provider_fallback"] is True + + +def test_flush_session_log_provider_used_defaults_empty_in_summary(tmp_path): + _flush(tmp_path, session_id="prov-def", proc_snapshots=None) + summary = json.loads((tmp_path / "sessions" / "prov-def" / "summary.json").read_text()) + assert summary["provider_used"] == "" + + +def test_flush_session_log_provider_fallback_defaults_false_in_summary(tmp_path): + _flush(tmp_path, session_id="fb-def", proc_snapshots=None) + summary = json.loads((tmp_path / "sessions" / "fb-def" / "summary.json").read_text()) + assert summary["provider_fallback"] is False + + +def test_flush_session_log_provider_used_in_index(tmp_path): + _flush(tmp_path, session_id="prov-idx", provider_used="openai", proc_snapshots=None) + entry = json.loads((tmp_path / "sessions.jsonl").read_text().strip().split("\n")[-1]) + assert entry["provider_used"] == "openai" + + +def test_flush_session_log_provider_fallback_in_index(tmp_path): + _flush(tmp_path, session_id="fb-idx", provider_fallback=True, proc_snapshots=None) + entry = json.loads((tmp_path / "sessions.jsonl").read_text().strip().split("\n")[-1]) + assert entry["provider_fallback"] is True + + +def test_flush_session_log_kill_reason_absent_from_index(tmp_path): + _flush(tmp_path, session_id="kr-idx", kill_reason="timeout", proc_snapshots=None) + entry = json.loads((tmp_path / "sessions.jsonl").read_text().strip().split("\n")[-1]) + assert "kill_reason" not in entry + + +def test_flush_session_log_provider_used_in_token_usage(tmp_path): + _flush( + tmp_path, + session_id="prov-tu", + step_name="implement", + provider_used="minimax", + token_usage={"input_tokens": 100, "output_tokens": 50}, + proc_snapshots=None, + ) + tu = json.loads((tmp_path / "sessions" / "prov-tu" / "token_usage.json").read_text()) + assert tu["provider_used"] == "minimax" + + +def test_flush_session_log_provider_fallback_absent_from_token_usage(tmp_path): + _flush( + tmp_path, + session_id="fb-tu", + step_name="implement", + provider_fallback=True, + token_usage={"input_tokens": 100, "output_tokens": 50}, + proc_snapshots=None, + ) + tu = json.loads((tmp_path / "sessions" / "fb-tu" / "token_usage.json").read_text()) + assert "provider_fallback" not in tu + + +def test_flush_session_log_provider_used_defaults_empty_in_token_usage(tmp_path): + _flush( + tmp_path, + session_id="prov-tu-def", + step_name="implement", + token_usage={"input_tokens": 100, "output_tokens": 50}, + proc_snapshots=None, + ) + tu = json.loads((tmp_path / "sessions" / "prov-tu-def" / "token_usage.json").read_text()) + assert tu["provider_used"] == "" + + +def test_turn_tool_calls_merged_across_thinking_and_tool_records(tmp_path, monkeypatch): + cb_log = tmp_path / "s.jsonl" + cb_log.write_text( + _make_cc_jsonl_record( + request_id="req-001", + timestamp="2026-05-04T00:00:00Z", + content=[_make_thinking_block()], + ) + + "\n" + + _make_cc_jsonl_record( + request_id="req-001", + content=[_make_tool_block("Bash")], + ) + + "\n" + ) + import autoskillit.execution.session_log as sl_mod + + monkeypatch.setattr(sl_mod, "claude_code_log_path", lambda cwd, sid: cb_log) + flush_session_log( + log_dir=str(tmp_path), + cwd="/tmp", + session_id="s", + pid=1, + skill_command="test", + success=True, + subtype="completed", + exit_code=0, + start_ts="2026-05-04T00:00:00Z", + proc_snapshots=None, + telemetry=SessionTelemetry.empty(), + provider_outcome=ProviderOutcome.none_used(), + recipe_identity=RecipeIdentity.empty(), + ) + summary = json.loads((tmp_path / "sessions" / "s" / "summary.json").read_text()) + assert summary["turn_tool_calls"] == [["Bash"]] + assert summary["request_ids"] == ["req-001"] + + +def test_parallel_lists_aligned_when_timestamp_missing(tmp_path, monkeypatch): + cb_log = tmp_path / "s.jsonl" + cb_log.write_text( + _make_cc_jsonl_record( + request_id="req-001", + timestamp="2026-05-04T00:00:00Z", + content=[_make_tool_block("Read")], + ) + + "\n" + + _make_cc_jsonl_record( + request_id="req-002", + content=[_make_tool_block("Edit")], + ) + + "\n" + ) + import autoskillit.execution.session_log as sl_mod + + monkeypatch.setattr(sl_mod, "claude_code_log_path", lambda cwd, sid: cb_log) + flush_session_log( + log_dir=str(tmp_path), + cwd="/tmp", + session_id="s", + pid=1, + skill_command="test", + success=True, + subtype="completed", + exit_code=0, + start_ts="2026-05-04T00:00:00Z", + proc_snapshots=None, + telemetry=SessionTelemetry.empty(), + provider_outcome=ProviderOutcome.none_used(), + recipe_identity=RecipeIdentity.empty(), + ) + summary = json.loads((tmp_path / "sessions" / "s" / "summary.json").read_text()) + assert ( + len(summary["request_ids"]) + == len(summary["turn_timestamps"]) + == len(summary["turn_tool_calls"]) + == 2 + ) + assert summary["request_ids"] == ["req-001", "req-002"] + assert summary["turn_timestamps"][0] == "2026-05-04T00:00:00Z" + assert summary["turn_timestamps"][1] == "" + assert summary["turn_tool_calls"] == [["Read"], ["Edit"]] + + +def test_flush_session_log_provider_used_defaults_to_empty_string(tmp_path): + _flush(tmp_path, session_id="prov-def-001", proc_snapshots=None) + entry = json.loads((tmp_path / "sessions.jsonl").read_text().strip().split("\n")[-1]) + assert entry["provider_used"] == "" + + +def test_flush_session_log_provider_fallback_defaults_to_false(tmp_path): + _flush(tmp_path, session_id="prov-def-002", proc_snapshots=None) + entry = json.loads((tmp_path / "sessions.jsonl").read_text().strip().split("\n")[-1]) + assert entry["provider_fallback"] is False + + +def test_token_usage_json_includes_model_identifier(tmp_path): + """flush_session_log writes model_identifier to token_usage.json.""" + _flush( + tmp_path, + session_id="model-id-001", + proc_snapshots=None, + token_usage={ + "input_tokens": 100, + "output_tokens": 50, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + "model_breakdown": {"claude-sonnet-4-6": {"input_tokens": 100, "output_tokens": 50}}, + }, + ) + tu_path = tmp_path / "sessions" / "model-id-001" / "token_usage.json" + assert tu_path.exists() + data = json.loads(tu_path.read_text()) + assert "model_identifier" in data + assert data["model_identifier"] == "claude-sonnet-4-6" + + +def test_token_usage_json_model_identifier_empty_when_no_breakdown(tmp_path): + """flush_session_log writes empty model_identifier when no model_breakdown.""" + _flush( + tmp_path, + session_id="model-id-002", + proc_snapshots=None, + token_usage={ + "input_tokens": 100, + "output_tokens": 50, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + }, + ) + tu_path = tmp_path / "sessions" / "model-id-002" / "token_usage.json" + data = json.loads(tu_path.read_text()) + assert data.get("model_identifier", "") == "" diff --git a/tests/execution/test_session_log.py b/tests/execution/test_session_log_flush.py similarity index 56% rename from tests/execution/test_session_log.py rename to tests/execution/test_session_log_flush.py index 88f53a15ee..57b34911b1 100644 --- a/tests/execution/test_session_log.py +++ b/tests/execution/test_session_log_flush.py @@ -1,67 +1,30 @@ -"""Tests for the session diagnostics log writer.""" +"""Tests for flush_session_log: directory structure, proc-trace, summary/index, +resolve_log_dir, temporal fields, duration/elapsed time, sidecars, schemas, +proc-trace comm, and clear-marker roundtrip.""" from __future__ import annotations import json import os -import sys -import time from datetime import UTC, datetime from pathlib import Path import pytest -from autoskillit.execution.linux_tracing import read_boot_id, read_starttime_ticks +from autoskillit.core.types._type_results import ProviderOutcome +from autoskillit.core.types._type_results_execution import ( + RecipeIdentity, + SessionTelemetry, +) from autoskillit.execution.session_log import ( flush_session_log, read_telemetry_clear_marker, - recover_crashed_sessions, resolve_log_dir, write_telemetry_clear_marker, ) +from tests.execution.conftest import _flush, _snap - -def _snap( - *, - captured_at: str = "2026-03-03T12:00:00+00:00", - vm_rss_kb: int = 100000, - oom_score: int = 50, - fd_count: int = 10, - fd_soft_limit: int = 1024, - state: str = "sleeping", -) -> dict[str, object]: - return { - "captured_at": captured_at, - "state": state, - "vm_rss_kb": vm_rss_kb, - "oom_score": oom_score, - "fd_count": fd_count, - "fd_soft_limit": fd_soft_limit, - "sig_pnd": "0000000000000000", - "sig_blk": "0000000000000000", - "sig_cgt": "0000000000000000", - "threads": 4, - "wchan": "", - "ctx_switches_voluntary": 500, - "ctx_switches_involuntary": 20, - } - - -def _flush(tmp_path: Path, **overrides) -> None: - defaults = { - "log_dir": str(tmp_path), - "cwd": "/home/test/project", - "session_id": "test-session-001", - "pid": 12345, - "skill_command": "/autoskillit:investigate some error", - "success": True, - "subtype": "completed", - "exit_code": 0, - "start_ts": "2026-03-03T12:00:00+00:00", - "proc_snapshots": [_snap(), _snap(), _snap()], - } - defaults.update(overrides) - flush_session_log(**defaults) +pytestmark = [pytest.mark.layer("execution"), pytest.mark.medium] def test_flush_session_log_creates_directory_structure(tmp_path): @@ -219,14 +182,13 @@ def test_flush_session_log_retention_purges_oldest(tmp_path): (d / "summary.json").write_text("{}") # Set mtime to ensure ordering mtime = 1000000000 + i - import os os.utime(d, (mtime, mtime)) with index_path.open("a") as f: f.write(json.dumps({"session_id": dir_name, "dir_name": dir_name}) + "\n") - # Flush a 503rd session - _flush(tmp_path, session_id="session-0502") + # Flush a 503rd session with explicit cap + _flush(tmp_path, session_id="session-0502", max_sessions=500) remaining = list(sessions_dir.iterdir()) assert len(remaining) == 500 @@ -357,122 +319,6 @@ def test_proc_trace_falls_back_to_start_ts_when_no_captured_at(tmp_path): assert records[0]["ts"] == "2026-03-03T09:00:00+00:00" -# --- recover_crashed_sessions tests --- - - -def test_recover_crashed_sessions_noop_when_no_orphans(tmp_path): - """Returns 0 when tmpfs is empty.""" - tmpfs = tmp_path / "shm" - tmpfs.mkdir() - count = recover_crashed_sessions(tmpfs_path=str(tmpfs), log_dir=str(tmp_path / "logs")) - assert count == 0 - - -def test_recover_crashed_sessions_noop_when_tmpfs_missing(tmp_path): - """Returns 0 when tmpfs_path does not exist.""" - count = recover_crashed_sessions( - tmpfs_path=str(tmp_path / "nonexistent"), log_dir=str(tmp_path / "logs") - ) - assert count == 0 - - -def test_recover_crashed_sessions_skips_recent_files(tmp_path): - """Files modified within the last 30 seconds are skipped (may be active).""" - - tmpfs = tmp_path / "shm" - tmpfs.mkdir() - trace = tmpfs / "autoskillit_trace_12345.jsonl" - trace.write_text( - json.dumps({"vm_rss_kb": 500, "captured_at": "2026-03-03T10:00:00+00:00"}) + "\n" - ) - # Leave the file fresh (mtime = now) - count = recover_crashed_sessions(tmpfs_path=str(tmpfs), log_dir=str(tmp_path / "logs")) - assert count == 0 - - -def _write_old_trace(tmpfs: Path, filename: str, content: str) -> Path: - """Write a trace file (backdated 60s) and its enrollment sidecar. - - The enrollment sidecar uses the current boot_id so Gate 2 passes. - The PID embedded in the filename is expected to be dead (so Gate 3 passes). - """ - trace = tmpfs / filename - trace.write_text(content) - old_mtime = time.time() - 60 - os.utime(trace, (old_mtime, old_mtime)) - - # Write companion enrollment sidecar so Gate 1 passes - try: - pid = int(Path(filename).stem.split("_")[-1]) - except (ValueError, IndexError): - pid = 0 - enrollment = tmpfs / f"autoskillit_enrollment_{pid}.json" - enrollment.write_text( - json.dumps( - { - "schema_version": 1, - "pid": pid, - "boot_id": read_boot_id() or "", - "starttime_ticks": None, - "session_id": "", - "enrolled_at": "2026-01-01T00:00:00+00:00", - "kitchen_id": "", - "order_id": "", - } - ) - ) - return trace - - -def test_recover_crashed_sessions_finalizes_orphaned_file(tmp_path): - """recover_crashed_sessions reads tmpfs file and writes permanent session dir.""" - tmpfs = tmp_path / "shm" - tmpfs.mkdir() - _write_old_trace( - tmpfs, - "autoskillit_trace_12345.jsonl", - json.dumps({"vm_rss_kb": 500, "captured_at": "2026-03-03T10:00:00+00:00"}) + "\n", - ) - count = recover_crashed_sessions(tmpfs_path=str(tmpfs), log_dir=str(tmp_path / "logs")) - assert count == 1 - sessions = list((tmp_path / "logs" / "sessions").iterdir()) - assert len(sessions) == 1 - assert "crashed" in sessions[0].name - assert (sessions[0] / "summary.json").exists() - summary = json.loads((sessions[0] / "summary.json").read_text()) - assert summary["termination_reason"] == "CRASHED" - assert summary["success"] is False - - -def test_recover_crashed_sessions_deletes_tmpfs_file(tmp_path): - """Trace file is removed from tmpfs after recovery.""" - tmpfs = tmp_path / "shm" - tmpfs.mkdir() - trace = _write_old_trace( - tmpfs, - "autoskillit_trace_99999.jsonl", - json.dumps({"vm_rss_kb": 300, "captured_at": "2026-03-03T10:00:00+00:00"}) + "\n", - ) - recover_crashed_sessions(tmpfs_path=str(tmpfs), log_dir=str(tmp_path / "logs")) - assert not trace.exists() - - -def test_recover_crashed_sessions_handles_multiple_files(tmp_path): - """Multiple orphaned trace files are all recovered.""" - tmpfs = tmp_path / "shm" - tmpfs.mkdir() - for pid in [111, 222, 333]: - _write_old_trace( - tmpfs, - f"autoskillit_trace_{pid}.jsonl", - json.dumps({"vm_rss_kb": 100, "captured_at": "2026-03-03T10:00:00+00:00"}) + "\n", - ) - count = recover_crashed_sessions(tmpfs_path=str(tmpfs), log_dir=str(tmp_path / "logs")) - assert count == 3 - sessions = list((tmp_path / "logs" / "sessions").iterdir()) - assert len(sessions) == 3 - - def test_flush_session_log_backward_clock_produces_non_negative_duration(tmp_path): """duration_seconds must never be negative, even if end_ts precedes start_ts.""" start_ts = "2026-01-01T12:05:00+00:00" # later @@ -491,6 +337,9 @@ def test_flush_session_log_backward_clock_produces_non_negative_duration(tmp_pat proc_snapshots=[], termination_reason="completed", snapshot_interval_seconds=5.0, + telemetry=SessionTelemetry.empty(), + provider_outcome=ProviderOutcome.none_used(), + recipe_identity=RecipeIdentity.empty(), ) session_dir = tmp_path / "sessions" / "backward-clock-test" summary = json.loads((session_dir / "summary.json").read_text()) @@ -518,6 +367,9 @@ def test_flush_session_log_uses_elapsed_seconds_over_iso_subtraction(tmp_path): proc_snapshots=[], termination_reason="completed", snapshot_interval_seconds=5.0, + telemetry=SessionTelemetry.empty(), + provider_outcome=ProviderOutcome.none_used(), + recipe_identity=RecipeIdentity.empty(), ) session_dir = tmp_path / "sessions" / "elapsed-seconds-test" summary = json.loads((session_dir / "summary.json").read_text()) @@ -548,6 +400,9 @@ def test_flush_session_log_zero_elapsed_seconds_is_valid(tmp_path): proc_snapshots=[], termination_reason="completed", snapshot_interval_seconds=5.0, + telemetry=SessionTelemetry.empty(), + provider_outcome=ProviderOutcome.none_used(), + recipe_identity=RecipeIdentity.empty(), ) session_dir = tmp_path / "sessions" / "zero-elapsed-test" summary = json.loads((session_dir / "summary.json").read_text()) @@ -570,17 +425,27 @@ def test_flush_writes_token_usage_json_when_step_provided(tmp_path): assert (session_dir / "token_usage.json").is_file() -def test_flush_omits_token_usage_json_when_no_step_name(tmp_path): - """token_usage.json is NOT written when step_name is empty, even if token_usage provided.""" +def test_flush_writes_token_usage_json_when_step_name_empty(tmp_path): + """token_usage.json IS written when step_name is empty, as long as token_usage is present.""" _flush( tmp_path, step_name="", - token_usage={"input_tokens": 100}, + token_usage={ + "input_tokens": 100, + "output_tokens": 50, + "cache_creation_input_tokens": 20, + "cache_read_input_tokens": 80, + }, proc_snapshots=None, success=False, ) session_dir = tmp_path / "sessions" / "test-session-001" - assert not (session_dir / "token_usage.json").exists() + assert (session_dir / "token_usage.json").exists() + data = json.loads((session_dir / "token_usage.json").read_text()) + assert data["session_label"] == "(ad-hoc)" + assert "step_name" not in data + assert data["cache_creation_input_tokens"] == 20 + assert data["cache_read_input_tokens"] == 80 def test_flush_writes_step_timing_json(tmp_path): @@ -622,92 +487,6 @@ def test_proc_trace_jsonl_rows_include_comm(tmp_path): assert row["comm"] == "claude", f"Expected comm='claude' in row {i}, got {row['comm']!r}" -# --------------------------------------------------------------------------- -# Test 1.11 — recovery path reads comm and excludes alien files -# --------------------------------------------------------------------------- - - -def _write_old_trace_with_comm(tmpfs: Path, pid: int, comm: str, *, n_snaps: int = 2) -> Path: - """Write a backdated trace file with snapshots that have a specific comm.""" - filename = f"autoskillit_trace_{pid}.jsonl" - trace = tmpfs / filename - snaps = [] - for _ in range(n_snaps): - snap_dict = {**_snap(), "comm": comm} - snaps.append(json.dumps(snap_dict)) - trace.write_text("\n".join(snaps) + "\n") - - # Backdate so Gate 1 (age > 30s) passes - old_mtime = time.time() - 60 - os.utime(trace, (old_mtime, old_mtime)) - - # Write enrollment sidecar so Gate 1 (sidecar present) passes. - # Use schema_version=2 with comm='claude' — autoskillit always enrolls its own - # binary as 'claude'. Snapshots whose first comm != enrollment.comm are alien. - enrollment = tmpfs / f"autoskillit_enrollment_{pid}.json" - enrollment.write_text( - json.dumps( - { - "schema_version": 2, - "pid": pid, - "boot_id": read_boot_id() or "", - "starttime_ticks": None, - "session_id": "", - "enrolled_at": "2026-01-01T00:00:00+00:00", - "kitchen_id": "", - "order_id": "", - "comm": "claude", - } - ) - ) - return trace - - -def test_recover_crashed_sessions_excludes_non_claude_trace_files(tmp_path): - """recover_crashed_sessions tags alien trace files (non-claude comm) and excludes them. - - Test 1.11: place two trace files — one with comm='claude' and one with - comm='sleep'. The 'sleep' file is an alien artifact (test pollution or a - non-autoskillit process). After the fix, recovery recognises it via comm - and either skips it or marks it with alien=true in the recovered record. - """ - tmpfs = tmp_path / "shm" - tmpfs.mkdir() - log_dir = tmp_path / "logs" - - # One legitimate claude trace - _write_old_trace_with_comm(tmpfs, pid=20001, comm="claude") - # One alien trace (e.g., leftover from a test or wrong process) - _write_old_trace_with_comm(tmpfs, pid=20002, comm="sleep") - - recover_crashed_sessions(tmpfs_path=str(tmpfs), log_dir=str(log_dir)) - - sessions_dir = log_dir / "sessions" - recovered = list(sessions_dir.iterdir()) if sessions_dir.exists() else [] - session_summaries = [] - for s in recovered: - summary_file = s / "summary.json" - if summary_file.exists(): - session_summaries.append(json.loads(summary_file.read_text())) - - claude_sessions = [s for s in session_summaries if "20001" in s.get("session_id", "")] - - # The claude trace must be recovered (not excluded) - assert claude_sessions, "The claude trace file must be recovered by recover_crashed_sessions" - - # The alien trace should not produce a normal session — it should be excluded - # or marked as alien so it doesn't pollute capacity planning / anomaly analytics - alien_included_normally = [ - s - for s in session_summaries - if "20002" in s.get("session_id", "") and not s.get("alien") and not s.get("pre_fix_data") - ] - assert not alien_included_normally, ( - "Alien trace (comm='sleep') must not be recovered as a normal session. " - "It should be skipped or marked alien=true to prevent #771-style mis-attribution." - ) - - def test_flush_writes_audit_log_json(tmp_path): """audit_log.json written to session dir when step_name and audit_record dict provided.""" record = { @@ -734,19 +513,27 @@ def test_flush_omits_audit_log_when_no_record(tmp_path): def test_flush_index_includes_step_name_and_token_fields(tmp_path): - """sessions.jsonl entry has step_name, input_tokens, output_tokens fields.""" + """sessions.jsonl entry has step_name, input_tokens, output_tokens, and cache fields.""" _flush( tmp_path, step_name="implement", - token_usage={"input_tokens": 100, "output_tokens": 50}, + token_usage={ + "input_tokens": 100, + "output_tokens": 50, + "cache_creation_input_tokens": 20, + "cache_read_input_tokens": 80, + }, proc_snapshots=None, success=False, ) lines = (tmp_path / "sessions.jsonl").read_text().strip().split("\n") + assert len(lines) == 1 entry = json.loads(lines[-1]) assert entry["step_name"] == "implement" assert entry["input_tokens"] == 100 assert entry["output_tokens"] == 50 + assert entry["cache_creation_input_tokens"] == 20 + assert entry["cache_read_input_tokens"] == 80 def test_flush_index_token_fields_zero_when_no_step(tmp_path): @@ -757,6 +544,8 @@ def test_flush_index_token_fields_zero_when_no_step(tmp_path): assert entry["step_name"] == "" assert entry["input_tokens"] == 0 assert entry["output_tokens"] == 0 + assert entry["cache_creation_input_tokens"] == 0 + assert entry["cache_read_input_tokens"] == 0 def test_token_usage_json_schema(tmp_path): @@ -773,14 +562,42 @@ def test_token_usage_json_schema(tmp_path): timing_seconds=15.0, proc_snapshots=None, success=False, + dispatch_id="disp-abc", + campaign_id="camp-xyz", ) tu = json.loads((tmp_path / "sessions" / "test-session-001" / "token_usage.json").read_text()) - assert tu["step_name"] == "plan" + assert tu["session_label"] == "plan" assert tu["input_tokens"] == 10 assert tu["output_tokens"] == 5 assert tu["cache_creation_input_tokens"] == 2 assert tu["cache_read_input_tokens"] == 1 assert tu["timing_seconds"] == 15.0 + assert tu["peak_context"] == 0 + assert tu["turn_count"] == 0 + assert tu["dispatch_id"] == "disp-abc" + assert tu["campaign_id"] == "camp-xyz" + + +def test_token_usage_json_includes_peak_context_and_turn_count(tmp_path): + """flush_session_log writes peak_context and turn_count to token_usage.json.""" + _flush( + tmp_path, + step_name="implement", + token_usage={ + "input_tokens": 100, + "output_tokens": 50, + "cache_creation_input_tokens": 10, + "cache_read_input_tokens": 200, + "peak_context": 75000, + "turn_count": 14, + }, + timing_seconds=30.0, + proc_snapshots=None, + success=True, + ) + tu = json.loads((tmp_path / "sessions" / "test-session-001" / "token_usage.json").read_text()) + assert tu["peak_context"] == 75000 + assert tu["turn_count"] == 14 def test_step_timing_json_schema(tmp_path): @@ -844,213 +661,176 @@ def test_write_clear_marker_is_atomic(tmp_path): assert t2 >= t1 -def test_flush_session_log_includes_write_path_warnings_in_summary(tmp_path): - """summary.json records write_path_warnings list.""" - warnings = [ - "Write tool wrote to /source/repo/.autoskillit/temp/foo.md (outside cwd /clone)", - "Edit tool wrote to /source/repo/src/file.py (outside cwd /clone)", - ] - _flush(tmp_path, session_id="warn-session", write_path_warnings=warnings, proc_snapshots=None) - summary = json.loads((tmp_path / "sessions" / "warn-session" / "summary.json").read_text()) - assert summary["write_path_warnings"] == warnings +# --- github_api_log integration tests --- -def test_flush_session_log_empty_warnings_produce_empty_list(tmp_path): - """No warnings → write_path_warnings is [] in summary.""" - _flush(tmp_path, session_id="clean-session", write_path_warnings=[], proc_snapshots=None) - summary = json.loads((tmp_path / "sessions" / "clean-session" / "summary.json").read_text()) - assert summary["write_path_warnings"] == [] +@pytest.mark.anyio +async def test_flush_writes_github_api_usage_from_populated_log(tmp_path): + """flush_session_log writes github_api_usage.json and non-zero github_api_requests.""" + from autoskillit.pipeline.github_api_log import DefaultGitHubApiLog + log = DefaultGitHubApiLog() + await log.record_gh_cli( + subcommand="gh issue list", + exit_code=0, + latency_ms=50.0, + timestamp="2026-05-02T10:00:00Z", + ) -def test_flush_session_log_none_warnings_treated_as_empty(tmp_path): - """write_path_warnings=None (default) produces empty list in summary.""" - _flush(tmp_path, session_id="default-warn", proc_snapshots=None) # no write_path_warnings arg - summary = json.loads((tmp_path / "sessions" / "default-warn" / "summary.json").read_text()) - assert summary["write_path_warnings"] == [] + _flush(tmp_path, github_api_log=log) + usage_file = tmp_path / "sessions" / "test-session-001" / "github_api_usage.json" + assert usage_file.exists(), "github_api_usage.json must be written when log has entries" + data = json.loads(usage_file.read_text()) + assert data["total_requests"] == 1 -def test_flush_session_log_includes_write_call_count_in_summary(tmp_path): - """summary.json records write_call_count.""" - _flush(tmp_path, session_id="wc-session", write_call_count=5, proc_snapshots=None) - summary = json.loads((tmp_path / "sessions" / "wc-session" / "summary.json").read_text()) - assert summary["write_call_count"] == 5 + summary = json.loads((tmp_path / "sessions" / "test-session-001" / "summary.json").read_text()) + assert summary["github_api_requests"] == 1 -def test_flush_session_log_write_call_count_in_index(tmp_path): - """sessions.jsonl index includes write_call_count.""" - _flush(tmp_path, session_id="wc-idx", write_call_count=3, proc_snapshots=None) - index_path = tmp_path / "sessions.jsonl" - entry = json.loads(index_path.read_text().strip().split("\n")[-1]) - assert entry["write_call_count"] == 3 +def test_flush_helper_builds_and_passes_session_telemetry(): + """The _flush() helper builds a SessionTelemetry and passes it to flush_session_log. + When a new field is added to SessionTelemetry without updating _flush()'s defaults, + the SessionTelemetry construction in _flush() will raise TypeError — caught immediately + by any test that uses _flush(). + """ + import tempfile + import unittest.mock as mock -def test_flush_session_log_write_call_count_defaults_to_zero(tmp_path): - """write_call_count defaults to 0 when not specified.""" - _flush(tmp_path, session_id="wc-default", proc_snapshots=None) - summary = json.loads((tmp_path / "sessions" / "wc-default" / "summary.json").read_text()) - assert summary["write_call_count"] == 0 + from autoskillit.core.types._type_results_execution import SessionTelemetry + captured: dict = {} -def test_flush_session_log_writes_kitchen_id(tmp_path): - """kitchen_id parameter is written to sessions.jsonl index entry.""" - flush_session_log( - log_dir=str(tmp_path), - cwd="/some/worktree", - kitchen_id="my-pipeline-123", - session_id="sess-001", - pid=12345, - skill_command="/autoskillit:implement", - success=True, - subtype="completed", - exit_code=0, - start_ts="2026-03-27T08:00:00", - proc_snapshots=None, - ) + def _capture(**kwargs): + captured.update(kwargs) - index = (tmp_path / "sessions.jsonl").read_text() - entry = json.loads(index.strip()) - assert entry["kitchen_id"] == "my-pipeline-123" + # _flush() does a local import from autoskillit.execution.session_log. + # Patch the function at its source module so the local import picks up the mock. + with mock.patch("autoskillit.execution.session_log.flush_session_log", side_effect=_capture): + with tempfile.TemporaryDirectory() as td: + _flush(Path(td)) + assert "telemetry" in captured, "_flush() must forward telemetry= to flush_session_log" + telemetry = captured["telemetry"] + assert isinstance(telemetry, SessionTelemetry), "telemetry must be a SessionTelemetry instance" + assert telemetry.github_api_requests == 0 + assert telemetry.github_api_usage is None + assert telemetry.loc_insertions == 0 + assert telemetry.loc_deletions == 0 -def test_flush_session_log_writes_order_id_to_index(tmp_path): - """order_id is written to sessions.jsonl index entry when provided.""" - flush_session_log( - log_dir=str(tmp_path), - cwd="/some/worktree", - kitchen_id="kitchen-abc", - order_id="issue-185", - session_id="sess-002", - pid=12345, - skill_command="/autoskillit:implement", - success=True, - subtype="completed", - exit_code=0, - start_ts="2026-03-27T08:00:00", + +def test_flush_writes_token_usage_with_dispatch_label(tmp_path): + """token_usage.json uses dispatch:{id} label when step_name is empty and dispatch_id set.""" + _flush( + tmp_path, + step_name="", + dispatch_id="abc-123", + token_usage={ + "input_tokens": 100, + "output_tokens": 50, + "cache_creation_input_tokens": 20, + "cache_read_input_tokens": 80, + }, proc_snapshots=None, + success=False, ) + session_dir = tmp_path / "sessions" / "test-session-001" + assert (session_dir / "token_usage.json").exists() + data = json.loads((session_dir / "token_usage.json").read_text()) + assert data["session_label"] == "dispatch:abc-123" + assert data["cache_creation_input_tokens"] == 20 + assert data["cache_read_input_tokens"] == 80 - entry = json.loads((tmp_path / "sessions.jsonl").read_text().strip()) - assert entry["order_id"] == "issue-185" +def test_token_usage_file_entry_type_matches_written_fields(tmp_path): + """token_usage.json keys match exactly the fields defined in TokenUsageFileEntry TypedDict.""" + from autoskillit.core.types import TokenUsageFileEntry -def test_flush_session_log_order_id_defaults_to_empty(tmp_path): - """order_id defaults to empty string when not supplied.""" - flush_session_log( - log_dir=str(tmp_path), - cwd="/some/worktree", - kitchen_id="kitchen-abc", - session_id="sess-003", - pid=12345, - skill_command="/autoskillit:implement", - success=True, - subtype="completed", - exit_code=0, - start_ts="2026-03-27T08:00:00", + _flush( + tmp_path, + step_name="implement", + token_usage={ + "input_tokens": 100, + "output_tokens": 50, + "cache_creation_input_tokens": 20, + "cache_read_input_tokens": 80, + "peak_context": 1000, + "turn_count": 5, + }, + timing_seconds=42.0, proc_snapshots=None, + success=False, ) - - entry = json.loads((tmp_path / "sessions.jsonl").read_text().strip()) - assert "order_id" in entry - assert entry["order_id"] == "" - - -@pytest.mark.skipif(sys.platform != "linux", reason="Linux-only: uses /proc and boot_id") -def test_recover_crashed_sessions_skips_live_pid(tmp_path): - """A trace file whose enrolled PID is still alive must not be recovered.""" - tmpfs = tmp_path / "shm" - tmpfs.mkdir() - pid = os.getpid() - trace = tmpfs / f"autoskillit_trace_{pid}.jsonl" - enrollment = tmpfs / f"autoskillit_enrollment_{pid}.json" - trace.write_text("") - enrollment.write_text( - json.dumps( - { - "schema_version": 1, - "pid": pid, - "boot_id": read_boot_id() or "", - "starttime_ticks": read_starttime_ticks(pid), - "session_id": "", - "enrolled_at": datetime.now(UTC).isoformat(), - "kitchen_id": "", - "order_id": "", - } - ) - ) - os.utime(trace, (time.time() - 60,) * 2) - - count = recover_crashed_sessions(tmpfs_path=str(tmpfs), log_dir=str(tmp_path)) - - assert count == 0 - assert trace.exists(), "Trace file for alive PID must not be deleted" - assert enrollment.exists(), "Enrollment sidecar for alive PID must not be deleted" - - -def test_recover_crashed_sessions_skips_file_without_enrollment(tmp_path): - """A trace file with no enrollment sidecar must be skipped — it is not - an autoskillit-owned trace (e.g. a test artifact or alien file).""" - tmpfs = tmp_path / "shm" - tmpfs.mkdir() - trace = tmpfs / "autoskillit_trace_99997.jsonl" - trace.write_text("") - os.utime(trace, (time.time() - 60,) * 2) - - count = recover_crashed_sessions(tmpfs_path=str(tmpfs), log_dir=str(tmp_path)) - - assert count == 0 - assert trace.exists(), "Alien trace file must not be deleted" + session_dir = tmp_path / "sessions" / "test-session-001" + data = json.loads((session_dir / "token_usage.json").read_text()) + expected_keys = set(TokenUsageFileEntry.__annotations__.keys()) + assert set(data.keys()) == expected_keys -def test_recover_crashed_sessions_skips_wrong_boot_id(tmp_path, monkeypatch): - """An enrollment sidecar with a different boot_id must be rejected.""" - monkeypatch.setattr( - "autoskillit.execution.session_log.read_boot_id", - lambda: "current-boot-id", - ) - tmpfs = tmp_path / "shm" - tmpfs.mkdir() - pid = 99996 - trace = tmpfs / f"autoskillit_trace_{pid}.jsonl" - enrollment = tmpfs / f"autoskillit_enrollment_{pid}.json" - trace.write_text("") - enrollment.write_text( - json.dumps( - { - "schema_version": 1, - "pid": pid, - "boot_id": "stale-boot-id", - "starttime_ticks": 1234, - "session_id": "", - "enrolled_at": "2026-01-01T00:00:00+00:00", - "kitchen_id": "", - "order_id": "", - } - ) +def test_token_usage_json_includes_dispatch_and_campaign_ids(tmp_path): + """flush_session_log writes dispatch_id and campaign_id to token_usage.json.""" + _flush( + tmp_path, + step_name="implement", + dispatch_id="d-abc-123", + campaign_id="c-xyz-789", + token_usage={ + "input_tokens": 100, + "output_tokens": 50, + "cache_creation_input_tokens": 10, + "cache_read_input_tokens": 20, + }, + timing_seconds=5.0, + proc_snapshots=None, + success=True, ) - os.utime(trace, (time.time() - 60,) * 2) + tu = json.loads((tmp_path / "sessions" / "test-session-001" / "token_usage.json").read_text()) + assert tu["dispatch_id"] == "d-abc-123" + assert tu["campaign_id"] == "c-xyz-789" - count = recover_crashed_sessions(tmpfs_path=str(tmpfs), log_dir=str(tmp_path)) - assert count == 0 +def test_token_usage_json_dispatch_campaign_default_empty(tmp_path): + """dispatch_id and campaign_id present as empty strings when omitted.""" + _flush( + tmp_path, + step_name="plan", + token_usage={ + "input_tokens": 10, + "output_tokens": 5, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + }, + timing_seconds=2.0, + proc_snapshots=None, + success=True, + ) + tu = json.loads((tmp_path / "sessions" / "test-session-001" / "token_usage.json").read_text()) + assert tu["dispatch_id"] == "" + assert tu["campaign_id"] == "" -def test_flush_writes_crash_exception_file(tmp_path): - """When exception_text is provided, flush_session_log writes crash_exception.txt.""" - flush_session_log( - log_dir=str(tmp_path), - cwd="/tmp", - session_id="test-session", - pid=1234, - skill_command="/test", - success=False, - subtype="crashed", - exit_code=-1, - start_ts=datetime.now(UTC).isoformat(), - proc_snapshots=None, - termination_reason="CRASHED", - exception_text="RuntimeError: boom\n at headless.py:1023", +def test_flush_co_writes_provenance_record(tmp_path, monkeypatch): + """flush_session_log writes a provenance record when cwd is non-empty.""" + monkeypatch.delenv("AUTOSKILLIT_STATE_DIR", raising=False) + monkeypatch.delenv("AUTOSKILLIT_CAMPAIGN_ID", raising=False) + project = tmp_path / "project" + project.mkdir() + _flush( + tmp_path, + cwd=str(project), + caller_session_id="caller-abc", + kitchen_id="kitchen-1", + dispatch_id="dispatch-1", + recipe_name="test-recipe", + step_name="implement", ) - session_dir = tmp_path / "sessions" / "test-session" - crash_file = session_dir / "crash_exception.txt" - assert crash_file.exists() - assert "RuntimeError: boom" in crash_file.read_text() + prov_path = project / ".autoskillit" / "temp" / "session_provenance.jsonl" + assert prov_path.is_file() + record = json.loads(prov_path.read_text().strip().splitlines()[0]) + assert record["session_id"] == "test-session-001" + assert record["caller_session_id"] == "caller-abc" + assert record["kitchen_id"] == "kitchen-1" + assert record["dispatch_id"] == "dispatch-1" + assert record["recipe_name"] == "test-recipe" + assert record["step_name"] == "implement" diff --git a/tests/execution/test_session_log_integration.py b/tests/execution/test_session_log_integration.py index c9311aec2c..62dab15343 100644 --- a/tests/execution/test_session_log_integration.py +++ b/tests/execution/test_session_log_integration.py @@ -6,24 +6,34 @@ import os import sys import time +from dataclasses import asdict from datetime import UTC, datetime, timedelta import anyio import pytest +from autoskillit.core.types._type_results import ProviderOutcome +from autoskillit.core.types._type_results_execution import ( + RecipeIdentity, + SessionTelemetry, +) from tests.execution.conftest import _ALLOCATE_60MB_SCRIPT -pytestmark = pytest.mark.skipif(sys.platform != "linux", reason="Linux only") +pytestmark = [ + pytest.mark.layer("execution"), + pytest.mark.medium, + pytest.mark.skipif(sys.platform != "linux", reason="Linux only"), +] @pytest.mark.anyio async def test_full_tracing_pipeline_writes_distinct_timestamps(tmp_path): """End-to-end: snapshot accumulation + flush produces unique ts per record.""" - from autoskillit.config import LinuxTracingConfig from autoskillit.execution.linux_tracing import start_linux_tracing, trace_target_from_pid from autoskillit.execution.session_log import flush_session_log + from tests._helpers import make_tracing_config - config = LinuxTracingConfig(proc_interval=0.05, tmpfs_path=str(tmp_path)) + config = make_tracing_config(proc_interval=0.05, tmpfs_path=str(tmp_path)) start_ts = datetime.now(UTC).isoformat() start_mono = time.monotonic() async with anyio.create_task_group() as tg: @@ -37,7 +47,7 @@ async def test_full_tracing_pipeline_writes_distinct_timestamps(tmp_path): elapsed = time.monotonic() - start_mono end_ts = (datetime.fromisoformat(start_ts) + timedelta(seconds=elapsed)).isoformat() assert len(snaps) >= 2, "Need at least 2 snapshots for timestamp variance test" - snap_dicts = [s.__dict__ for s in snaps] + snap_dicts = [asdict(s) for s in snaps] flush_session_log( log_dir=str(tmp_path), @@ -54,6 +64,9 @@ async def test_full_tracing_pipeline_writes_distinct_timestamps(tmp_path): termination_reason="natural_exit", snapshot_interval_seconds=0.05, proc_snapshots=snap_dicts, + telemetry=SessionTelemetry.empty(), + provider_outcome=ProviderOutcome.none_used(), + recipe_identity=RecipeIdentity.empty(), ) session_dir = tmp_path / "sessions" / "integration-test-001" @@ -108,6 +121,9 @@ def _flush_with_snaps(tmp_path, session_id: str, snaps: list[dict]) -> None: exit_code=0, start_ts="2026-01-01T00:00:00+00:00", proc_snapshots=snaps, + telemetry=SessionTelemetry.empty(), + provider_outcome=ProviderOutcome.none_used(), + recipe_identity=RecipeIdentity.empty(), ) @@ -155,11 +171,11 @@ async def test_peak_rss_kb_above_sanity_floor(tmp_path): Test 1.6: a deliberate lower bound that script(1) (~2 MB) cannot satisfy. If this ever fails, the test name points directly at the PTY wrapper tracer bug class. """ - from autoskillit.config import LinuxTracingConfig from autoskillit.execution.process import run_managed_async from autoskillit.execution.session_log import flush_session_log + from tests._helpers import make_tracing_config - cfg = LinuxTracingConfig(proc_interval=0.1, tmpfs_path=str(tmp_path / "shm")) + cfg = make_tracing_config(proc_interval=0.1, tmpfs_path=str(tmp_path / "shm")) (tmp_path / "shm").mkdir(parents=True) helper = tmp_path / "alloc.py" @@ -186,6 +202,9 @@ async def test_peak_rss_kb_above_sanity_floor(tmp_path): exit_code=0, start_ts=result.start_ts or "2026-01-01T00:00:00+00:00", proc_snapshots=result.proc_snapshots, + telemetry=SessionTelemetry.empty(), + provider_outcome=ProviderOutcome.none_used(), + recipe_identity=RecipeIdentity.empty(), ) summary_path = tmp_path / "logs" / "sessions" / "sanity-floor-001" / "summary.json" diff --git a/tests/execution/test_session_log_retention.py b/tests/execution/test_session_log_retention.py new file mode 100644 index 0000000000..c28886338e --- /dev/null +++ b/tests/execution/test_session_log_retention.py @@ -0,0 +1,909 @@ +"""Tests for recover_crashed_sessions and retention/campaign-protection logic.""" + +from __future__ import annotations + +import json +import os +import sys +import time +from datetime import UTC, datetime +from pathlib import Path + +import pytest + +from autoskillit.core.types._type_results import ProviderOutcome +from autoskillit.core.types._type_results_execution import ( + RecipeIdentity, + SessionTelemetry, +) +from autoskillit.execution.linux_tracing import read_boot_id, read_starttime_ticks +from autoskillit.execution.session_log import ( + flush_session_log, + recover_crashed_sessions, +) +from autoskillit.fleet import build_protected_campaign_ids +from tests.execution.conftest import _flush, _snap + +pytestmark = [pytest.mark.layer("execution"), pytest.mark.medium] + + +# --- recover_crashed_sessions tests --- + + +def test_recover_crashed_sessions_noop_when_no_orphans(tmp_path): + """Returns 0 when tmpfs is empty.""" + tmpfs = tmp_path / "shm" + tmpfs.mkdir() + count = recover_crashed_sessions(tmpfs_path=str(tmpfs), log_dir=str(tmp_path / "logs")) + assert count == 0 + + +def test_recover_crashed_sessions_noop_when_tmpfs_missing(tmp_path): + """Returns 0 when tmpfs_path does not exist.""" + count = recover_crashed_sessions( + tmpfs_path=str(tmp_path / "nonexistent"), log_dir=str(tmp_path / "logs") + ) + assert count == 0 + + +def test_recover_crashed_sessions_skips_recent_files(tmp_path): + """Files modified within the last 30 seconds are skipped (may be active).""" + + tmpfs = tmp_path / "shm" + tmpfs.mkdir() + trace = tmpfs / "autoskillit_trace_12345.jsonl" + trace.write_text( + json.dumps({"vm_rss_kb": 500, "captured_at": "2026-03-03T10:00:00+00:00"}) + "\n" + ) + # Leave the file fresh (mtime = now) + count = recover_crashed_sessions(tmpfs_path=str(tmpfs), log_dir=str(tmp_path / "logs")) + assert count == 0 + + +def _write_old_trace(tmpfs: Path, filename: str, content: str) -> Path: + """Write a trace file (backdated 60s) and its enrollment sidecar. + + The enrollment sidecar uses the current boot_id so Gate 2 passes. + The PID embedded in the filename is expected to be dead (so Gate 3 passes). + """ + trace = tmpfs / filename + trace.write_text(content) + old_mtime = time.time() - 60 + os.utime(trace, (old_mtime, old_mtime)) + + # Write companion enrollment sidecar so Gate 1 passes + try: + pid = int(Path(filename).stem.split("_")[-1]) + except (ValueError, IndexError): + pid = 0 + enrollment = tmpfs / f"autoskillit_enrollment_{pid}.json" + enrollment.write_text( + json.dumps( + { + "schema_version": 1, + "pid": pid, + "boot_id": read_boot_id() or "", + "starttime_ticks": None, + "session_id": "", + "enrolled_at": "2026-01-01T00:00:00+00:00", + "kitchen_id": "", + "order_id": "", + } + ) + ) + return trace + + +def test_recover_crashed_sessions_finalizes_orphaned_file(tmp_path): + """recover_crashed_sessions reads tmpfs file and writes permanent session dir.""" + tmpfs = tmp_path / "shm" + tmpfs.mkdir() + _write_old_trace( + tmpfs, + "autoskillit_trace_12345.jsonl", + json.dumps({"vm_rss_kb": 500, "captured_at": "2026-03-03T10:00:00+00:00"}) + "\n", + ) + count = recover_crashed_sessions(tmpfs_path=str(tmpfs), log_dir=str(tmp_path / "logs")) + assert count == 1 + sessions = list((tmp_path / "logs" / "sessions").iterdir()) + assert len(sessions) == 1 + assert "crashed" in sessions[0].name + assert (sessions[0] / "summary.json").exists() + summary = json.loads((sessions[0] / "summary.json").read_text()) + assert summary["termination_reason"] == "CRASHED" + assert summary["success"] is False + + +def test_recover_crashed_sessions_deletes_tmpfs_file(tmp_path): + """Trace file is removed from tmpfs after recovery.""" + tmpfs = tmp_path / "shm" + tmpfs.mkdir() + trace = _write_old_trace( + tmpfs, + "autoskillit_trace_99999.jsonl", + json.dumps({"vm_rss_kb": 300, "captured_at": "2026-03-03T10:00:00+00:00"}) + "\n", + ) + recover_crashed_sessions(tmpfs_path=str(tmpfs), log_dir=str(tmp_path / "logs")) + assert not trace.exists() + + +def test_recover_crashed_sessions_handles_multiple_files(tmp_path): + """Multiple orphaned trace files are all recovered.""" + tmpfs = tmp_path / "shm" + tmpfs.mkdir() + for pid in [111, 222, 333]: + _write_old_trace( + tmpfs, + f"autoskillit_trace_{pid}.jsonl", + json.dumps({"vm_rss_kb": 100, "captured_at": "2026-03-03T10:00:00+00:00"}) + "\n", + ) + count = recover_crashed_sessions(tmpfs_path=str(tmpfs), log_dir=str(tmp_path / "logs")) + assert count == 3 + sessions = list((tmp_path / "logs" / "sessions").iterdir()) + assert len(sessions) == 3 + + +def _write_old_trace_with_comm(tmpfs: Path, pid: int, comm: str, *, n_snaps: int = 2) -> Path: + """Write a backdated trace file with snapshots that have a specific comm.""" + filename = f"autoskillit_trace_{pid}.jsonl" + trace = tmpfs / filename + snaps = [] + for _ in range(n_snaps): + snap_dict = {**_snap(), "comm": comm} + snaps.append(json.dumps(snap_dict)) + trace.write_text("\n".join(snaps) + "\n") + + # Backdate so Gate 1 (age > 30s) passes + old_mtime = time.time() - 60 + os.utime(trace, (old_mtime, old_mtime)) + + # Write enrollment sidecar so Gate 1 (sidecar present) passes. + # Use schema_version=2 with comm='claude' — autoskillit always enrolls its own + # binary as 'claude'. Snapshots whose first comm != enrollment.comm are alien. + enrollment = tmpfs / f"autoskillit_enrollment_{pid}.json" + enrollment.write_text( + json.dumps( + { + "schema_version": 2, + "pid": pid, + "boot_id": read_boot_id() or "", + "starttime_ticks": None, + "session_id": "", + "enrolled_at": "2026-01-01T00:00:00+00:00", + "kitchen_id": "", + "order_id": "", + "comm": "claude", + } + ) + ) + return trace + + +def test_recover_crashed_sessions_excludes_non_claude_trace_files(tmp_path): + """recover_crashed_sessions tags alien trace files (non-claude comm) and excludes them. + + Test 1.11: place two trace files — one with comm='claude' and one with + comm='sleep'. The 'sleep' file is an alien artifact (test pollution or a + non-autoskillit process). After the fix, recovery recognises it via comm + and either skips it or marks it with alien=true in the recovered record. + """ + tmpfs = tmp_path / "shm" + tmpfs.mkdir() + log_dir = tmp_path / "logs" + + # One legitimate claude trace + _write_old_trace_with_comm(tmpfs, pid=20001, comm="claude") + # One alien trace (e.g., leftover from a test or wrong process) + _write_old_trace_with_comm(tmpfs, pid=20002, comm="sleep") + + recover_crashed_sessions(tmpfs_path=str(tmpfs), log_dir=str(log_dir)) + + sessions_dir = log_dir / "sessions" + recovered = list(sessions_dir.iterdir()) if sessions_dir.exists() else [] + session_summaries = [] + for s in recovered: + summary_file = s / "summary.json" + if summary_file.exists(): + session_summaries.append(json.loads(summary_file.read_text())) + + claude_sessions = [s for s in session_summaries if "20001" in s.get("session_id", "")] + + # The claude trace must be recovered (not excluded) + assert claude_sessions, "The claude trace file must be recovered by recover_crashed_sessions" + + # The alien trace should not produce a normal session — it should be excluded + # or marked as alien so it doesn't pollute capacity planning / anomaly analytics + alien_included_normally = [ + s + for s in session_summaries + if "20002" in s.get("session_id", "") and not s.get("alien") and not s.get("pre_fix_data") + ] + assert not alien_included_normally, ( + "Alien trace (comm='sleep') must not be recovered as a normal session. " + "It should be skipped or marked alien=true to prevent #771-style mis-attribution." + ) + + +@pytest.mark.skipif(sys.platform != "linux", reason="Linux-only: uses /proc and boot_id") +def test_recover_crashed_sessions_skips_live_pid(tmp_path): + """A trace file whose enrolled PID is still alive must not be recovered.""" + tmpfs = tmp_path / "shm" + tmpfs.mkdir() + pid = os.getpid() + trace = tmpfs / f"autoskillit_trace_{pid}.jsonl" + enrollment = tmpfs / f"autoskillit_enrollment_{pid}.json" + trace.write_text("") + enrollment.write_text( + json.dumps( + { + "schema_version": 1, + "pid": pid, + "boot_id": read_boot_id() or "", + "starttime_ticks": read_starttime_ticks(pid), + "session_id": "", + "enrolled_at": datetime.now(UTC).isoformat(), + "kitchen_id": "", + "order_id": "", + } + ) + ) + os.utime(trace, (time.time() - 60,) * 2) + + count = recover_crashed_sessions(tmpfs_path=str(tmpfs), log_dir=str(tmp_path)) + + assert count == 0 + assert trace.exists(), "Trace file for alive PID must not be deleted" + assert enrollment.exists(), "Enrollment sidecar for alive PID must not be deleted" + + +def test_recover_crashed_sessions_skips_file_without_enrollment(tmp_path): + """A trace file with no enrollment sidecar must be skipped — it is not + an autoskillit-owned trace (e.g. a test artifact or alien file).""" + tmpfs = tmp_path / "shm" + tmpfs.mkdir() + trace = tmpfs / "autoskillit_trace_99997.jsonl" + trace.write_text("") + os.utime(trace, (time.time() - 60,) * 2) + + count = recover_crashed_sessions(tmpfs_path=str(tmpfs), log_dir=str(tmp_path)) + + assert count == 0 + assert trace.exists(), "Alien trace file must not be deleted" + + +def test_recover_crashed_sessions_skips_wrong_boot_id(tmp_path, monkeypatch): + """An enrollment sidecar with a different boot_id must be rejected.""" + monkeypatch.setattr( + "autoskillit.execution.session_log.read_boot_id", + lambda: "current-boot-id", + ) + tmpfs = tmp_path / "shm" + tmpfs.mkdir() + pid = 99996 + trace = tmpfs / f"autoskillit_trace_{pid}.jsonl" + enrollment = tmpfs / f"autoskillit_enrollment_{pid}.json" + trace.write_text("") + enrollment.write_text( + json.dumps( + { + "schema_version": 1, + "pid": pid, + "boot_id": "stale-boot-id", + "starttime_ticks": 1234, + "session_id": "", + "enrolled_at": "2026-01-01T00:00:00+00:00", + "kitchen_id": "", + "order_id": "", + } + ) + ) + os.utime(trace, (time.time() - 60,) * 2) + + count = recover_crashed_sessions(tmpfs_path=str(tmpfs), log_dir=str(tmp_path)) + + assert count == 0 + + +# --- Group H: campaign_id / dispatch_id schema tests --- + + +def test_flush_writes_campaign_id_to_summary(tmp_path): + """summary.json contains campaign_id field when kwarg passed.""" + _flush(tmp_path, session_id="gh-001", campaign_id="camp-abc") + summary = json.loads((tmp_path / "sessions" / "gh-001" / "summary.json").read_text()) + assert summary["campaign_id"] == "camp-abc" + + +def test_flush_writes_dispatch_id_to_summary(tmp_path): + """summary.json contains dispatch_id field when kwarg passed.""" + _flush(tmp_path, session_id="gh-002", campaign_id="camp-abc", dispatch_id="disp-xyz") + summary = json.loads((tmp_path / "sessions" / "gh-002" / "summary.json").read_text()) + assert summary["dispatch_id"] == "disp-xyz" + + +def test_flush_writes_campaign_id_to_index(tmp_path): + """sessions.jsonl entry contains campaign_id.""" + _flush(tmp_path, session_id="gh-003", campaign_id="camp-abc") + entry = json.loads((tmp_path / "sessions.jsonl").read_text().strip()) + assert entry["campaign_id"] == "camp-abc" + + +def test_flush_writes_dispatch_id_to_index(tmp_path): + """sessions.jsonl entry contains dispatch_id.""" + _flush(tmp_path, session_id="gh-004", campaign_id="camp-abc", dispatch_id="disp-xyz") + entry = json.loads((tmp_path / "sessions.jsonl").read_text().strip()) + assert entry["dispatch_id"] == "disp-xyz" + + +def test_flush_writes_meta_json_sidecar(tmp_path): + """meta.json written with campaign_id and dispatch_id when campaign_id non-empty.""" + _flush(tmp_path, session_id="gh-005", campaign_id="c1", dispatch_id="d1") + meta_path = tmp_path / "sessions" / "gh-005" / "meta.json" + assert meta_path.exists() + meta = json.loads(meta_path.read_text()) + assert meta == {"campaign_id": "c1", "dispatch_id": "d1"} + + +def test_flush_writes_meta_json_sidecar_campaign_only(tmp_path): + """meta.json written with empty dispatch_id when only campaign_id is provided.""" + _flush(tmp_path, session_id="gh-005b", campaign_id="c1") + meta_path = tmp_path / "sessions" / "gh-005b" / "meta.json" + assert meta_path.exists() + meta = json.loads(meta_path.read_text()) + assert meta == {"campaign_id": "c1", "dispatch_id": ""} + + +def test_flush_omits_meta_json_when_no_campaign(tmp_path): + """No meta.json written when campaign_id is empty (default).""" + _flush(tmp_path, session_id="gh-006") + meta_path = tmp_path / "sessions" / "gh-006" / "meta.json" + assert not meta_path.exists() + + +def test_flush_defaults_campaign_dispatch_empty(tmp_path): + """Existing callers without new kwargs produce empty-string fields.""" + _flush(tmp_path, session_id="gh-007") + summary = json.loads((tmp_path / "sessions" / "gh-007" / "summary.json").read_text()) + assert summary["campaign_id"] == "" + assert summary["dispatch_id"] == "" + entry = json.loads((tmp_path / "sessions.jsonl").read_text().strip()) + assert entry["campaign_id"] == "" + assert entry["dispatch_id"] == "" + + +# --- Group M: retention protection tests --- + + +def _make_sessions(tmp_path, count, start_mtime=1_000_000_000, campaign_id=""): + """Create `count` session directories with staggered mtimes. + + Returns list of dir_names in mtime order (oldest first). + Seeds meta.json with campaign_id if provided. + """ + sessions_dir = tmp_path / "sessions" + sessions_dir.mkdir(parents=True, exist_ok=True) + index_path = tmp_path / "sessions.jsonl" + dir_names = [] + for i in range(count): + dir_name = f"session-{i:04d}" + d = sessions_dir / dir_name + d.mkdir(exist_ok=True) + mtime = start_mtime + i + os.utime(d, (mtime, mtime)) + if campaign_id: + (d / "meta.json").write_text( + json.dumps({"campaign_id": campaign_id, "dispatch_id": f"d-{i}"}) + ) + with index_path.open("a") as f: + f.write( + json.dumps( + {"session_id": dir_name, "dir_name": dir_name, "campaign_id": campaign_id} + ) + + "\n" + ) + dir_names.append(dir_name) + return dir_names + + +def _make_state_file(project_dir, campaign_id, status): + """Create a fleet dispatch state file.""" + dispatches_dir = project_dir / ".autoskillit" / "temp" / "dispatches" + dispatches_dir.mkdir(parents=True, exist_ok=True) + state_path = dispatches_dir / "d1.json" + state_path.write_text( + json.dumps( + { + "campaign_id": campaign_id, + "dispatches": [{"name": "truck-1", "status": status}], + } + ) + ) + return state_path + + +def test_retention_protects_active_campaign_sessions(tmp_path, monkeypatch): + """Sessions belonging to an active campaign survive retention even when expired.""" + import autoskillit.execution.session_log as sl_module + + monkeypatch.setattr(sl_module, "_MAX_SESSIONS", 5) + + project_dir = tmp_path / "project" + project_dir.mkdir() + + # Create 5 "non-campaign" sessions + 2 "active campaign" sessions at the oldest positions + sessions_dir = tmp_path / "sessions" + sessions_dir.mkdir(parents=True) + index_path = tmp_path / "sessions.jsonl" + + # Oldest 2 dirs: campaign sessions (will be "expired" if not protected) + for i in range(2): + dir_name = f"session-{i:04d}" + d = sessions_dir / dir_name + d.mkdir() + os.utime(d, (1_000_000_000 + i, 1_000_000_000 + i)) + (d / "meta.json").write_text( + json.dumps({"campaign_id": "active-campaign", "dispatch_id": f"d{i}"}) + ) + with index_path.open("a") as f: + f.write( + json.dumps( + { + "session_id": dir_name, + "dir_name": dir_name, + "campaign_id": "active-campaign", + } + ) + + "\n" + ) + + # Next 4 dirs: non-campaign sessions + for i in range(2, 6): + dir_name = f"session-{i:04d}" + d = sessions_dir / dir_name + d.mkdir() + os.utime(d, (1_000_000_000 + i, 1_000_000_000 + i)) + with index_path.open("a") as f: + f.write( + json.dumps({"session_id": dir_name, "dir_name": dir_name, "campaign_id": ""}) + + "\n" + ) + + _make_state_file(project_dir, "active-campaign", "running") + + # Flush a 7th session to trigger retention (5 max, so 2 should expire) + flush_session_log( + log_dir=str(tmp_path), + cwd="/some/project", + project_dir=str(project_dir), + build_protected_campaign_ids=build_protected_campaign_ids, + session_id="session-0006", + pid=12345, + skill_command="/autoskillit:implement", + success=True, + subtype="completed", + exit_code=0, + start_ts="2026-04-20T10:00:00+00:00", + proc_snapshots=None, + telemetry=SessionTelemetry.empty(), + provider_outcome=ProviderOutcome.none_used(), + recipe_identity=RecipeIdentity.empty(), + ) + + # Protected sessions survive — campaign meta.json writes update their mtime so they + # end up in the "surviving" window; even if they landed in expired, protection saves them. + assert (sessions_dir / "session-0000").exists(), "active campaign session must survive" + assert (sessions_dir / "session-0001").exists(), "active campaign session must survive" + # The 2 non-campaign sessions with oldest mtimes (0002, 0003) are deleted. + # session-0002 and session-0003 retain the manually-set Sept-2001 mtimes (no file writes + # update their directory mtime) so they are the oldest dirs overall. + assert not (sessions_dir / "session-0002").exists(), ( + "oldest non-campaign session must be deleted" + ) + assert not (sessions_dir / "session-0003").exists(), ( + "second oldest non-campaign session must be deleted" + ) + # Newer non-campaign sessions survive (they are in the top-5 window) + assert (sessions_dir / "session-0004").exists(), "session-0004 must survive" + assert (sessions_dir / "session-0005").exists(), "session-0005 must survive" + # Newly flushed session must be present + assert (sessions_dir / "session-0006").exists() + + +def test_retention_deletes_released_campaign_sessions(tmp_path, monkeypatch): + """Sessions whose campaign is in a terminal state are eligible for deletion.""" + import autoskillit.execution.session_log as sl_module + + monkeypatch.setattr(sl_module, "_MAX_SESSIONS", 5) + + project_dir = tmp_path / "project" + project_dir.mkdir() + sessions_dir = tmp_path / "sessions" + sessions_dir.mkdir(parents=True) + index_path = tmp_path / "sessions.jsonl" + + # Create 6 dirs — oldest 2 have meta.json but campaign is released + for i in range(6): + dir_name = f"session-{i:04d}" + d = sessions_dir / dir_name + d.mkdir() + if i < 2: + (d / "meta.json").write_text( + json.dumps({"campaign_id": "done-campaign", "dispatch_id": f"d{i}"}) + ) + with index_path.open("a") as f: + cid = "done-campaign" if i < 2 else "" + f.write( + json.dumps({"session_id": dir_name, "dir_name": dir_name, "campaign_id": cid}) + + "\n" + ) + # Set mtime AFTER all writes inside the dir to get the intended ordering + os.utime(d, (1_000_000_000 + i, 1_000_000_000 + i)) + + _make_state_file(project_dir, "done-campaign", "released") + + flush_session_log( + log_dir=str(tmp_path), + cwd="/some/project", + project_dir=str(project_dir), + build_protected_campaign_ids=build_protected_campaign_ids, + session_id="session-0006", + pid=12345, + skill_command="/autoskillit:implement", + success=True, + subtype="completed", + exit_code=0, + start_ts="2026-04-20T10:00:00+00:00", + proc_snapshots=None, + telemetry=SessionTelemetry.empty(), + provider_outcome=ProviderOutcome.none_used(), + recipe_identity=RecipeIdentity.empty(), + ) + + # Released campaign sessions are NOT protected — oldest 2 should be deleted + assert not (sessions_dir / "session-0000").exists(), ( + "released campaign session must be deleted" + ) + assert not (sessions_dir / "session-0001").exists(), ( + "released campaign session must be deleted" + ) + + +def test_retention_preserves_index_for_protected(tmp_path, monkeypatch): + """Protected sessions' entries survive the sessions.jsonl rewrite.""" + import autoskillit.execution.session_log as sl_module + + monkeypatch.setattr(sl_module, "_MAX_SESSIONS", 5) + + project_dir = tmp_path / "project" + project_dir.mkdir() + sessions_dir = tmp_path / "sessions" + sessions_dir.mkdir(parents=True) + index_path = tmp_path / "sessions.jsonl" + + for i in range(6): + dir_name = f"session-{i:04d}" + d = sessions_dir / dir_name + d.mkdir() + os.utime(d, (1_000_000_000 + i, 1_000_000_000 + i)) + if i < 2: + (d / "meta.json").write_text( + json.dumps({"campaign_id": "live-campaign", "dispatch_id": f"d{i}"}) + ) + cid = "live-campaign" if i < 2 else "" + with index_path.open("a") as f: + f.write( + json.dumps({"session_id": dir_name, "dir_name": dir_name, "campaign_id": cid}) + + "\n" + ) + + _make_state_file(project_dir, "live-campaign", "pending") + + flush_session_log( + log_dir=str(tmp_path), + cwd="/some/project", + project_dir=str(project_dir), + build_protected_campaign_ids=build_protected_campaign_ids, + session_id="session-0006", + pid=12345, + skill_command="/autoskillit:implement", + success=True, + subtype="completed", + exit_code=0, + start_ts="2026-04-20T10:00:00+00:00", + proc_snapshots=None, + telemetry=SessionTelemetry.empty(), + provider_outcome=ProviderOutcome.none_used(), + recipe_identity=RecipeIdentity.empty(), + ) + + index_lines = [ln for ln in index_path.read_text().strip().split("\n") if ln.strip()] + dir_names_in_index = {json.loads(ln)["dir_name"] for ln in index_lines} + assert "session-0000" in dir_names_in_index, "protected session index entry must be preserved" + assert "session-0001" in dir_names_in_index, "protected session index entry must be preserved" + + +def test_retention_handles_missing_meta_json(tmp_path, monkeypatch): + """Session dirs without meta.json are not protected (normal deletion).""" + import autoskillit.execution.session_log as sl_module + + monkeypatch.setattr(sl_module, "_MAX_SESSIONS", 5) + + project_dir = tmp_path / "project" + project_dir.mkdir() + sessions_dir = tmp_path / "sessions" + sessions_dir.mkdir(parents=True) + index_path = tmp_path / "sessions.jsonl" + + _make_state_file(project_dir, "active-campaign", "running") + + for i in range(6): + dir_name = f"session-{i:04d}" + d = sessions_dir / dir_name + d.mkdir() + os.utime(d, (1_000_000_000 + i, 1_000_000_000 + i)) + # No meta.json written — sessions are not linked to any campaign + with index_path.open("a") as f: + f.write(json.dumps({"session_id": dir_name, "dir_name": dir_name}) + "\n") + + flush_session_log( + log_dir=str(tmp_path), + cwd="/some/project", + project_dir=str(project_dir), + build_protected_campaign_ids=build_protected_campaign_ids, + session_id="session-0006", + pid=12345, + skill_command="/autoskillit:implement", + success=True, + subtype="completed", + exit_code=0, + start_ts="2026-04-20T10:00:00+00:00", + proc_snapshots=None, + telemetry=SessionTelemetry.empty(), + provider_outcome=ProviderOutcome.none_used(), + recipe_identity=RecipeIdentity.empty(), + ) + + # Oldest dirs with no meta.json are deleted normally + assert not (sessions_dir / "session-0000").exists() + assert not (sessions_dir / "session-0001").exists() + + +def test_retention_handles_missing_franchise_state_dir(tmp_path, monkeypatch): + """No franchise state files → normal retention behavior (no crash).""" + import autoskillit.execution.session_log as sl_module + + monkeypatch.setattr(sl_module, "_MAX_SESSIONS", 5) + + project_dir = tmp_path / "project" + project_dir.mkdir() + # No dispatches dir created — build_protected_campaign_ids returns empty frozenset + sessions_dir = tmp_path / "sessions" + sessions_dir.mkdir(parents=True) + index_path = tmp_path / "sessions.jsonl" + + for i in range(6): + dir_name = f"session-{i:04d}" + d = sessions_dir / dir_name + d.mkdir() + (d / "meta.json").write_text( + json.dumps({"campaign_id": "some-campaign", "dispatch_id": f"d{i}"}) + ) + with index_path.open("a") as f: + f.write(json.dumps({"session_id": dir_name, "dir_name": dir_name}) + "\n") + # Set mtime AFTER all writes inside the dir to get the intended ordering + os.utime(d, (1_000_000_000 + i, 1_000_000_000 + i)) + + # Must not crash even though project_dir exists but has no dispatches dir + flush_session_log( + log_dir=str(tmp_path), + cwd="/some/project", + project_dir=str(project_dir), + build_protected_campaign_ids=build_protected_campaign_ids, + session_id="session-0006", + pid=12345, + skill_command="/autoskillit:implement", + success=True, + subtype="completed", + exit_code=0, + start_ts="2026-04-20T10:00:00+00:00", + proc_snapshots=None, + telemetry=SessionTelemetry.empty(), + provider_outcome=ProviderOutcome.none_used(), + recipe_identity=RecipeIdentity.empty(), + ) + + # Normal retention applies — oldest dirs deleted + assert not (sessions_dir / "session-0000").exists() + assert not (sessions_dir / "session-0001").exists() + + +def test_retention_handles_corrupt_meta_json(tmp_path, monkeypatch): + """Malformed meta.json → session not protected (graceful degradation).""" + import autoskillit.execution.session_log as sl_module + + monkeypatch.setattr(sl_module, "_MAX_SESSIONS", 5) + + project_dir = tmp_path / "project" + project_dir.mkdir() + _make_state_file(project_dir, "active-campaign", "running") + sessions_dir = tmp_path / "sessions" + sessions_dir.mkdir(parents=True) + index_path = tmp_path / "sessions.jsonl" + + for i in range(6): + dir_name = f"session-{i:04d}" + d = sessions_dir / dir_name + d.mkdir() + if i < 2: + # Write corrupt JSON so meta.json is unreadable + (d / "meta.json").write_text("not valid json {{{{") + with index_path.open("a") as f: + f.write(json.dumps({"session_id": dir_name, "dir_name": dir_name}) + "\n") + # Set mtime AFTER all writes inside the dir to get the intended ordering + os.utime(d, (1_000_000_000 + i, 1_000_000_000 + i)) + + flush_session_log( + log_dir=str(tmp_path), + cwd="/some/project", + project_dir=str(project_dir), + build_protected_campaign_ids=build_protected_campaign_ids, + session_id="session-0006", + pid=12345, + skill_command="/autoskillit:implement", + success=True, + subtype="completed", + exit_code=0, + start_ts="2026-04-20T10:00:00+00:00", + proc_snapshots=None, + telemetry=SessionTelemetry.empty(), + provider_outcome=ProviderOutcome.none_used(), + recipe_identity=RecipeIdentity.empty(), + ) + + # Corrupt meta.json → not protected → deleted normally + assert not (sessions_dir / "session-0000").exists() + assert not (sessions_dir / "session-0001").exists() + + +def test_session_log_removed_build_protected_function() -> None: + """SL_CB_1: _build_protected_campaign_ids must not exist on session_log module.""" + import autoskillit.execution.session_log as sl_module + + assert not hasattr(sl_module, "_build_protected_campaign_ids") + + +def test_session_log_removed_terminal_statuses_constant() -> None: + """SL_CB_2: TERMINAL_DISPATCH_STATUSES must not exist on session_log module.""" + import autoskillit.execution.session_log as sl_module + + assert not hasattr(sl_module, "TERMINAL_DISPATCH_STATUSES") + + +def test_retention_no_protection_when_callback_is_none(tmp_path: Path, monkeypatch) -> None: + """SL_CB_6: build_protected_campaign_ids=None with active campaign → no protection applied.""" + import autoskillit.execution.session_log as sl_module + + monkeypatch.setattr(sl_module, "_MAX_SESSIONS", 5) + + project_dir = tmp_path / "project" + project_dir.mkdir() + _make_state_file(project_dir, "active-campaign", "running") + + sessions_dir = tmp_path / "sessions" + sessions_dir.mkdir(parents=True) + index_path = tmp_path / "sessions.jsonl" + + for i in range(6): + dir_name = f"session-{i:04d}" + d = sessions_dir / dir_name + d.mkdir() + os.utime(d, (1_000_000_000 + i, 1_000_000_000 + i)) + if i < 2: + (d / "meta.json").write_text( + json.dumps({"campaign_id": "active-campaign", "dispatch_id": f"d{i}"}) + ) + # Reset mtime after meta.json write (writing a file bumps directory mtime) + os.utime(d, (1_000_000_000 + i, 1_000_000_000 + i)) + with index_path.open("a") as f: + f.write(json.dumps({"session_id": dir_name, "dir_name": dir_name}) + "\n") + + flush_session_log( + log_dir=str(tmp_path), + cwd="/some/project", + project_dir=str(project_dir), + build_protected_campaign_ids=None, + session_id="session-0006", + pid=12345, + skill_command="/autoskillit:implement", + success=True, + subtype="completed", + exit_code=0, + start_ts="2026-04-20T10:00:00+00:00", + proc_snapshots=None, + telemetry=SessionTelemetry.empty(), + provider_outcome=ProviderOutcome.none_used(), + recipe_identity=RecipeIdentity.empty(), + ) + + # No protection applied — oldest sessions deleted even though campaign is active + assert not (sessions_dir / "session-0000").exists() + assert not (sessions_dir / "session-0001").exists() + + +def test_flush_session_log_passes_callback_to_enforce_retention( + tmp_path: Path, monkeypatch +) -> None: + """SL_CB_7: flush_session_log forwards build_protected_campaign_ids to _enforce_retention.""" + import autoskillit.execution.session_log as sl_module + + captured: list = [] + + def fake_enforce_retention( + log_root, project_dir="", build_protected_campaign_ids=None, **kwargs + ) -> None: + captured.append(build_protected_campaign_ids) + + monkeypatch.setattr(sl_module, "_enforce_retention", fake_enforce_retention) + + sentinel = build_protected_campaign_ids + flush_session_log( + log_dir=str(tmp_path), + cwd="/some/project", + project_dir=str(tmp_path), + build_protected_campaign_ids=sentinel, + session_id="session-cb7", + pid=12345, + skill_command="/autoskillit:implement", + success=True, + subtype="completed", + exit_code=0, + start_ts="2026-04-20T10:00:00+00:00", + proc_snapshots=None, + telemetry=SessionTelemetry.empty(), + provider_outcome=ProviderOutcome.none_used(), + recipe_identity=RecipeIdentity.empty(), + ) + + assert (tmp_path / "sessions.jsonl").exists() + assert len(captured) == 1 + assert captured[0] is sentinel + + +# --- max_sessions configurability tests --- + + +def test_max_sessions_constant_is_2000(): + """T5: _MAX_SESSIONS equals 2000.""" + from autoskillit.execution.session_log import _MAX_SESSIONS + + assert _MAX_SESSIONS == 2000 + + +def test_enforce_retention_respects_max_sessions_param(tmp_path): + """T3: Passing max_sessions overrides the module-level _MAX_SESSIONS.""" + import autoskillit.execution.session_log as sl_module + + _make_sessions(tmp_path, count=10) + sl_module._enforce_retention(tmp_path, max_sessions=7) + remaining = list((tmp_path / "sessions").iterdir()) + assert len(remaining) == 7 + + +def test_flush_threads_max_sessions_to_enforce_retention(tmp_path, monkeypatch): + """T4: flush_session_log passes max_sessions through to _enforce_retention.""" + import autoskillit.execution.session_log as sl_module + + captured: dict = {} + + def fake_enforce( + log_root, + project_dir="", + build_protected_campaign_ids=None, + *, + max_sessions=sl_module._MAX_SESSIONS, + ) -> None: + captured["max_sessions"] = max_sessions + + monkeypatch.setattr(sl_module, "_enforce_retention", fake_enforce) + _flush(tmp_path, max_sessions=999) + assert captured["max_sessions"] == 999 diff --git a/tests/execution/test_session_model_peak_context.py b/tests/execution/test_session_model_peak_context.py new file mode 100644 index 0000000000..d1057ff781 --- /dev/null +++ b/tests/execution/test_session_model_peak_context.py @@ -0,0 +1,120 @@ +"""Tests for peak_context and turn_count extraction from extract_token_usage.""" + +from __future__ import annotations + +import json + +import pytest + +from autoskillit.execution.session._session_model import extract_token_usage + +pytestmark = [pytest.mark.layer("execution"), pytest.mark.medium] + + +def _assistant(cache_read: int = 0) -> str: + return json.dumps( + { + "type": "assistant", + "message": { + "model": "claude-sonnet-4-6", + "usage": { + "input_tokens": 100, + "output_tokens": 50, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": cache_read, + }, + }, + } + ) + + +def _result(cache_read: int = 0) -> str: + return json.dumps( + { + "type": "result", + "subtype": "success", + "is_error": False, + "result": "done", + "session_id": "s1", + "errors": [], + "usage": { + "input_tokens": 100, + "output_tokens": 50, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": cache_read, + }, + } + ) + + +def _build_ndjson(lines: list[str]) -> str: + return "\n".join(lines) + + +def test_extract_peak_context_from_multi_turn_stdout(): + stdout = _build_ndjson( + [ + _assistant(cache_read=10000), + _assistant(cache_read=50000), + _assistant(cache_read=30000), + ] + ) + result = extract_token_usage(stdout) + assert result is not None + assert result["peak_context"] == 50000 + + +def test_extract_turn_count_from_multi_turn_stdout(): + stdout = _build_ndjson( + [ + _assistant(cache_read=10000), + _assistant(cache_read=20000), + _assistant(cache_read=30000), + ] + ) + result = extract_token_usage(stdout) + assert result is not None + assert result["turn_count"] == 3 + + +def test_extract_peak_context_single_turn(): + stdout = _build_ndjson([_assistant(cache_read=42000)]) + result = extract_token_usage(stdout) + assert result is not None + assert result["peak_context"] == 42000 + assert result["turn_count"] == 1 + + +def test_extract_peak_context_with_result_record(): + stdout = _build_ndjson( + [ + _assistant(cache_read=60000), + _assistant(cache_read=80000), + _result(cache_read=140000), + ] + ) + result = extract_token_usage(stdout) + assert result is not None + assert result["peak_context"] == 80000 + assert result["turn_count"] == 2 + + +def test_extract_peak_context_no_assistant_records(): + stdout = _build_ndjson([_result(cache_read=100000)]) + result = extract_token_usage(stdout) + assert result is not None + assert result["peak_context"] == 0 + assert result["turn_count"] == 0 + + +def test_extract_peak_context_zero_cache_read(): + stdout = _build_ndjson( + [ + _assistant(cache_read=0), + _assistant(cache_read=0), + ] + ) + result = extract_token_usage(stdout) + assert result is not None + assert result["peak_context"] == 0 + assert result["turn_count"] == 2 diff --git a/tests/execution/test_session.py b/tests/execution/test_session_parsing.py similarity index 51% rename from tests/execution/test_session.py rename to tests/execution/test_session_parsing.py index b15a60866f..b0604e71a7 100644 --- a/tests/execution/test_session.py +++ b/tests/execution/test_session_parsing.py @@ -1,4 +1,4 @@ -"""L1 unit tests for execution/session.py — session result parsing and policy functions.""" +"""L1 unit tests for execution/session.py — token extraction, parsing, and SkillResult.""" from __future__ import annotations @@ -8,20 +8,22 @@ import structlog from autoskillit.core.types import ( - CONTEXT_EXHAUSTION_MARKER, ChannelConfirmation, + InfraExitCategory, + InfraOutcome, RetryReason, SubprocessResult, TerminationReason, ) from autoskillit.execution.session import ( - ClaudeSessionResult, SkillResult, extract_token_usage, parse_session_result, ) -from autoskillit.server.tools_execution import run_skill from tests._helpers import _flush_structlog_proxy_caches as _flush_logger_proxy_caches +from tests.execution.conftest import _assistant_ndjson, _result_ndjson + +pytestmark = [pytest.mark.layer("execution"), pytest.mark.small] def _make_session_result( @@ -42,328 +44,6 @@ def _make_session_result( ) -class TestClaudeSessionResult: - """ClaudeSessionResult correctly parses Claude Code JSON output.""" - - def test_parses_success_result(self): - """Normal completion extracts result and session_id.""" - raw = { - "type": "result", - "subtype": "success", - "is_error": False, - "result": "Done.", - "session_id": "abc-123", - } - parsed = parse_session_result(json.dumps(raw)) - assert parsed.subtype == "success" - assert parsed.is_error is False - assert parsed.result == "Done." - assert parsed.session_id == "abc-123" - assert parsed.needs_retry is False - assert parsed.retry_reason == RetryReason.NONE - - def test_parses_error_max_turns(self): - """Turn limit produces needs_retry=True with reason=RESUME.""" - raw = { - "type": "result", - "subtype": "error_max_turns", - "is_error": False, - "session_id": "abc-123", - "errors": ["Max turns reached"], - } - parsed = parse_session_result(json.dumps(raw)) - assert parsed.subtype == "error_max_turns" - assert parsed.needs_retry is True - assert parsed.retry_reason == RetryReason.RESUME - assert parsed.result == "" - - def test_parses_prompt_too_long(self): - """Context exhaustion produces needs_retry=True with reason=RESUME.""" - raw = { - "type": "result", - "subtype": "success", - "is_error": True, - "result": CONTEXT_EXHAUSTION_MARKER.capitalize(), - "session_id": "abc-123", - } - parsed = parse_session_result(json.dumps(raw)) - assert parsed.is_error is True - assert parsed.needs_retry is True - assert parsed.retry_reason == RetryReason.RESUME - - def test_parses_execution_error_not_retriable(self): - """Runtime errors are not automatically retriable.""" - raw = { - "type": "result", - "subtype": "error_during_execution", - "is_error": True, - "session_id": "abc-123", - "errors": ["Tool execution failed"], - } - parsed = parse_session_result(json.dumps(raw)) - assert parsed.subtype == "error_during_execution" - assert parsed.needs_retry is False - assert parsed.retry_reason == RetryReason.NONE - - def test_non_json_stdout_is_error(self): - """Non-JSON output (crashes, tracebacks) is always an error.""" - parsed = parse_session_result("Traceback (most recent call last):\n File...") - assert parsed.is_error is True - assert parsed.subtype == "unparseable" - assert "Traceback" in parsed.result - assert parsed.needs_retry is False - - def test_empty_stdout_is_error(self): - """Empty stdout means the session produced no output — always an error.""" - parsed = parse_session_result("") - assert parsed.is_error is True - assert parsed.subtype == "empty_output" - assert parsed.result == "" - assert parsed.needs_retry is False - - def test_whitespace_only_stdout_is_error(self): - """Whitespace-only stdout is treated as empty.""" - parsed = parse_session_result(" \n \t ") - assert parsed.is_error is True - assert parsed.subtype == "empty_output" - - def test_json_without_type_result_is_error(self): - """JSON that isn't a Claude result object is rejected by fallback.""" - parsed = parse_session_result('{"some": "random", "json": true}') - assert parsed.is_error is True - assert parsed.subtype == "unparseable" - - def test_non_dict_json_is_error(self): - """Non-dict JSON (list, string, number) is unparseable.""" - parsed = parse_session_result("[1, 2, 3]") - assert parsed.is_error is True - assert parsed.subtype == "unparseable" - - def test_handles_ndjson_with_multiple_lines(self): - """Parser finds type=result in multi-line NDJSON output.""" - lines = [ - json.dumps({"type": "assistant", "message": "working..."}), - json.dumps( - { - "type": "result", - "subtype": "success", - "is_error": False, - "result": "Done.", - "session_id": "s1", - } - ), - ] - parsed = parse_session_result("\n".join(lines)) - assert parsed.subtype == "success" - assert parsed.result == "Done." - assert parsed.session_id == "s1" - - def test_needs_retry_and_retry_reason_are_consistent(self): - """retry_reason is RESUME iff needs_retry is True, NONE otherwise.""" - cases = [ - ("success", False, "Done."), - ("error_max_turns", False, ""), - ("success", True, "Prompt is too long"), - ("error_during_execution", True, "crashed"), - ("unknown", False, ""), - ] - for subtype, is_error, result_text in cases: - session = ClaudeSessionResult( - subtype=subtype, - is_error=is_error, - result=result_text, - session_id="s1", - ) - if session.needs_retry: - assert session.retry_reason == RetryReason.RESUME, ( - f"needs_retry=True but retry_reason={session.retry_reason!r} " - f"for subtype={subtype}, is_error={is_error}" - ) - else: - assert session.retry_reason == RetryReason.NONE, ( - f"needs_retry=False but retry_reason={session.retry_reason!r} " - f"for subtype={subtype}, is_error={is_error}" - ) - - def test_all_retriable_cases_produce_same_retry_reason(self): - """Phase 1 (ClaudeSessionResult.retry_reason property) always returns RESUME - when needs_retry is True. - - This property is only triggered by context/turn limit signals (error_max_turns - and context_exhaustion). Both conditions always have partial progress on disk - and always return RESUME. - - Note: _compute_retry() can return RetryReason.EMPTY_OUTPUT for NATURAL_EXIT + - kill_anomaly cases with no context exhaustion. That is Phase 2 behavior and is - not tested here. - """ - max_turns_case = ClaudeSessionResult( - subtype="error_max_turns", - is_error=False, - result="", - session_id="s1", - ) - context_case = ClaudeSessionResult( - subtype="success", - is_error=True, - result="Prompt is too long", - session_id="s2", - ) - assert max_turns_case.needs_retry is True - assert context_case.needs_retry is True - assert max_turns_case.retry_reason == context_case.retry_reason - - -class TestAgentResult: - """agent_result produces actionable text for LLM callers.""" - - def test_rewrites_context_exhaustion(self): - """Context exhaustion result must NOT contain 'Prompt is too long'.""" - session = ClaudeSessionResult( - subtype="success", - is_error=True, - result="Prompt is too long", - session_id="s1", - ) - assert "prompt is too long" not in session.agent_result.lower() - assert "context" in session.agent_result.lower() - assert "continue" in session.agent_result.lower() - - def test_rewrites_max_turns(self): - """Max turns result must describe the situation, not pass through empty string.""" - session = ClaudeSessionResult( - subtype="error_max_turns", - is_error=False, - result="", - session_id="s1", - ) - assert ( - "turn limit" in session.agent_result.lower() - or "resume" in session.agent_result.lower() - ) - - def test_preserves_normal_result(self): - """Normal success result passes through unchanged.""" - session = ClaudeSessionResult( - subtype="success", - is_error=False, - result="Task completed. Created 3 files.", - session_id="s1", - ) - assert session.agent_result == "Task completed. Created 3 files." - - def test_preserves_error_result_when_not_retriable(self): - """Non-retriable errors pass through unchanged.""" - session = ClaudeSessionResult( - subtype="error_during_execution", - is_error=True, - result="Tool execution failed: permission denied", - session_id="s1", - ) - assert session.agent_result == "Tool execution failed: permission denied" - - -class TestResponseFieldsAreTypeSafe: - """Every discriminator field in MCP tool responses uses enum values.""" - - @pytest.mark.anyio - async def test_retry_reason_is_enum_value(self, tool_ctx): - stdout = json.dumps( - { - "type": "result", - "subtype": "error_max_turns", - "is_error": False, - "session_id": "s1", - "num_turns": 200, - "errors": [], - } - ) - tool_ctx.runner.push(_make_session_result(1, stdout, "")) - result = json.loads(await run_skill("/retry-worktree plan.md", "/tmp")) - assert result["retry_reason"] in {e.value for e in RetryReason} - - @pytest.mark.anyio - async def test_retry_reason_none_is_enum_value(self, tool_ctx): - stdout = json.dumps( - { - "type": "result", - "subtype": "success", - "is_error": False, - "result": "Done.", - "session_id": "s1", - "num_turns": 50, - } - ) - tool_ctx.runner.push(_make_session_result(0, stdout, "")) - result = json.loads(await run_skill("/retry-worktree plan.md", "/tmp")) - assert result["retry_reason"] in {e.value for e in RetryReason} - - -class TestClaudeSessionResultTypeEnforcement: - """ClaudeSessionResult.__post_init__ enforces field types.""" - - def test_null_result_becomes_empty_string(self): - session = ClaudeSessionResult(subtype="error", is_error=True, result=None, session_id="s1") - assert session.result == "" - - def test_null_errors_becomes_empty_list(self): - session = ClaudeSessionResult( - subtype="error", is_error=True, result="err", session_id="s1", errors=None - ) - assert session.errors == [] - - def test_null_subtype_becomes_unknown(self): - session = ClaudeSessionResult(subtype=None, is_error=False, result="ok", session_id="s1") - assert session.subtype == "unknown" - - def test_null_session_id_becomes_empty(self): - session = ClaudeSessionResult( - subtype="success", is_error=False, result="ok", session_id=None - ) - assert session.session_id == "" - - def test_is_context_exhausted_with_null_safe_fields(self): - session = ClaudeSessionResult( - subtype="error", is_error=True, result=None, session_id="s1", errors=None - ) - assert session._is_context_exhausted() is False - - def test_list_content_result_becomes_string(self): - blocks = [{"type": "text", "text": "Task completed."}] - session = ClaudeSessionResult( - subtype="success", is_error=False, result=blocks, session_id="s1" - ) - assert session.result == "Task completed." - assert isinstance(session.result, str) - - -class TestParseSessionResultNullFields: - """parse_session_result handles null JSON values correctly.""" - - def test_null_result_field(self): - raw = { - "type": "result", - "subtype": "error_during_execution", - "is_error": True, - "result": None, - "session_id": "s1", - } - parsed = parse_session_result(json.dumps(raw)) - assert parsed.result == "" - - def test_null_errors_field(self): - raw = { - "type": "result", - "subtype": "success", - "is_error": False, - "result": "done", - "session_id": "s1", - "errors": None, - } - parsed = parse_session_result(json.dumps(raw)) - assert parsed.errors == [] - - class TestExtractTokenUsage: """Tests for extract_token_usage().""" @@ -599,260 +279,6 @@ def test_ignores_non_assistant_non_result_records(self): assert result is None -class TestClaudeSessionResultTokenUsage: - """Token usage field on ClaudeSessionResult.""" - - def test_default_is_none(self): - """token_usage defaults to None when not provided.""" - session = ClaudeSessionResult( - subtype="success", is_error=False, result="Done.", session_id="s1" - ) - assert session.token_usage is None - - def test_preserves_token_usage_dict(self): - """token_usage dict is stored and accessible.""" - usage = { - "input_tokens": 100, - "output_tokens": 50, - "cache_creation_input_tokens": 10, - "cache_read_input_tokens": 5, - "model_breakdown": {"claude-sonnet-4-6": {"input_tokens": 100, "output_tokens": 50}}, - } - session = ClaudeSessionResult( - subtype="success", - is_error=False, - result="Done.", - session_id="s1", - token_usage=usage, - ) - assert session.token_usage is usage - assert session.token_usage["input_tokens"] == 100 - assert "model_breakdown" in session.token_usage - - -def _make_success_session(result: str = "done") -> ClaudeSessionResult: - return ClaudeSessionResult( - subtype="success", - is_error=False, - result=result, - session_id="s1", - ) - - -def _make_error_session( - subtype: str = "error_during_execution", - result: str = "failed", - errors: list[str] | None = None, -) -> ClaudeSessionResult: - return ClaudeSessionResult( - subtype=subtype, - is_error=True, - result=result, - session_id="s1", - errors=errors or [], - ) - - -def _result_ndjson( - result_text: str = "done", - subtype: str = "success", - is_error: bool = False, - session_id: str = "s1", - errors: list | None = None, - usage: dict | None = None, -) -> str: - obj: dict = { - "type": "result", - "subtype": subtype, - "is_error": is_error, - "result": result_text, - "session_id": session_id, - "errors": errors or [], - } - if usage: - obj["usage"] = usage - return json.dumps(obj) - - -def _assistant_ndjson( - model: str = "claude-sonnet-4-6", - input_tokens: int = 100, - output_tokens: int = 50, - cache_create: int = 0, - cache_read: int = 0, -) -> str: - return json.dumps( - { - "type": "assistant", - "message": { - "model": model, - "usage": { - "input_tokens": input_tokens, - "output_tokens": output_tokens, - "cache_creation_input_tokens": cache_create, - "cache_read_input_tokens": cache_read, - }, - }, - } - ) - - -class TestClaudeSessionResultBasic: - def test_post_init_coerces_none_result_to_empty_str(self): - s = ClaudeSessionResult( - subtype="success", - is_error=False, - result=None, - session_id="s1", # type: ignore[arg-type] - ) - assert s.result == "" - - def test_post_init_coerces_non_list_errors(self): - s = ClaudeSessionResult( - subtype="success", - is_error=False, - result="ok", - session_id="s1", - errors=None, # type: ignore[arg-type] - ) - assert s.errors == [] - - def test_post_init_list_with_non_dict_element(self): - s = ClaudeSessionResult( - subtype="success", - is_error=False, - result=["plain string", {"type": "text", "text": "extra"}], - session_id="s1", - ) - assert s.result == "plain string\nextra" - - -class TestClaudeSessionResultContextExhausted: - def test_is_context_exhausted_via_errors_list(self): - s = ClaudeSessionResult( - subtype="error", - is_error=True, - result="", - session_id="s1", - errors=[f"Request failed: {CONTEXT_EXHAUSTION_MARKER}"], - ) - assert s._is_context_exhausted() is True - - def test_is_context_exhausted_via_result_text(self): - s = ClaudeSessionResult( - subtype="success", - is_error=True, - result=f"Something: {CONTEXT_EXHAUSTION_MARKER}", - session_id="s1", - ) - assert s._is_context_exhausted() is True - - def test_is_context_exhausted_false_when_no_error(self): - s = ClaudeSessionResult( - subtype="success", - is_error=False, - result=f"contains {CONTEXT_EXHAUSTION_MARKER} text", - session_id="s1", - ) - assert s._is_context_exhausted() is False - - def test_jsonl_flat_record_sets_context_exhausted_flag(self): - """parse_session_result sets jsonl_context_exhausted=True for flat assistant record.""" - import json - - flat_record = json.dumps( - { - "type": "assistant", - "content": [{"type": "text", "text": "Prompt is too long"}], - "output_tokens": 0, - "input_tokens": 0, - "cache_read_input_tokens": 0, - } - ) - result_record = json.dumps( - { - "type": "result", - "subtype": "success", - "is_error": False, - "result": "", - "session_id": "sess-1", - } - ) - session = parse_session_result(flat_record + "\n" + result_record) - assert session.jsonl_context_exhausted is True - - def test_jsonl_flat_record_is_context_exhausted_bypasses_is_error_guard(self): - """_is_context_exhausted() returns True via JSONL flag even when is_error=False.""" - s = ClaudeSessionResult( - subtype="success", - is_error=False, - result="", - session_id="s1", - jsonl_context_exhausted=True, - ) - assert s._is_context_exhausted() is True - assert s.needs_retry is True - - def test_jsonl_flat_record_nonzero_output_tokens_not_detected(self): - """Flat assistant record with output_tokens > 0 does NOT trigger detection. - - The content deliberately contains CONTEXT_EXHAUSTION_MARKER so that only - the output_tokens != 0 guard suppresses detection — isolating the gate - under test. - """ - flat_record = json.dumps( - { - "type": "assistant", - "content": [{"type": "text", "text": CONTEXT_EXHAUSTION_MARKER}], - "output_tokens": 42, - "input_tokens": 100, - } - ) - result_record = json.dumps( - { - "type": "result", - "subtype": "success", - "is_error": False, - "result": "done %%ORDER_UP%%", - "session_id": "sess-2", - } - ) - session = parse_session_result(flat_record + "\n" + result_record) - assert session.jsonl_context_exhausted is False - - -class TestClaudeSessionResultNeedsRetry: - def test_needs_retry_true_for_max_turns(self): - s = ClaudeSessionResult( - subtype="error_max_turns", is_error=False, result="r", session_id="s1" - ) - assert s.needs_retry is True - - def test_needs_retry_true_for_context_exhausted(self): - s = ClaudeSessionResult( - subtype="error", - is_error=True, - result="", - session_id="s1", - errors=[CONTEXT_EXHAUSTION_MARKER], - ) - assert s.needs_retry is True - - def test_needs_retry_false_for_success(self): - s = _make_success_session() - assert s.needs_retry is False - - def test_retry_reason_resume_when_needs_retry(self): - s = ClaudeSessionResult( - subtype="error_max_turns", is_error=False, result="r", session_id="s1" - ) - assert s.retry_reason == RetryReason.RESUME - - def test_retry_reason_none_when_no_retry(self): - s = _make_success_session() - assert s.retry_reason == RetryReason.NONE - - class TestParseSessionResult: @pytest.fixture(autouse=True) def _reset_structlog(self): @@ -1068,12 +494,20 @@ def test_to_json_produces_all_required_keys(self): "is_error", "exit_code", "kill_reason", + "last_stop_reason", + "lifespan_started", + "provider_fallback", + "provider_used", "needs_retry", "retry_reason", + "order_id", "stderr", "token_usage", "write_path_warnings", "write_call_count", + "fs_writes_detected", + "has_progress_evidence", + "infra_exit_category", } assert set(parsed.keys()) == expected @@ -1099,100 +533,6 @@ def test_to_json_token_usage_none(self): assert parsed["token_usage"] is None -# --------------------------------------------------------------------------- -# CliSubtype sealed enum tests -# --------------------------------------------------------------------------- - - -class TestCliSubtypeExhaustiveCoverage: - """Every subtype returned by parse_session_result is a CliSubtype member.""" - - def test_success_subtype_is_cli_subtype(self): - from autoskillit.core.types import CliSubtype - - ndjson = json.dumps( - { - "type": "result", - "subtype": "success", - "is_error": False, - "result": "ok", - "session_id": "s1", - } - ) - session = parse_session_result(ndjson) - assert isinstance(session.subtype, CliSubtype) - assert session.subtype == CliSubtype.SUCCESS - - def test_empty_output_subtype_is_cli_subtype(self): - from autoskillit.core.types import CliSubtype - - session = parse_session_result("") - assert isinstance(session.subtype, CliSubtype) - assert session.subtype == CliSubtype.EMPTY_OUTPUT - - def test_unparseable_subtype_is_cli_subtype(self): - from autoskillit.core.types import CliSubtype - - session = parse_session_result("not json at all") - assert isinstance(session.subtype, CliSubtype) - assert session.subtype == CliSubtype.UNPARSEABLE - - def test_unknown_cli_subtype_maps_to_unknown(self): - from autoskillit.core.types import CliSubtype - - ndjson = json.dumps( - { - "type": "result", - "subtype": "some_future_subtype", - "is_error": False, - "result": "ok", - "session_id": "s1", - } - ) - session = parse_session_result(ndjson) - assert isinstance(session.subtype, CliSubtype) - assert session.subtype == CliSubtype.UNKNOWN - - @pytest.mark.parametrize( - "raw_subtype", - [ - "success", - "error_max_turns", - "error_during_execution", - "unknown", - "empty_output", - "unparseable", - "timeout", - ], - ) - def test_all_known_subtypes_are_cli_subtype_members(self, raw_subtype: str): - from autoskillit.core.types import CliSubtype - - ndjson = json.dumps( - { - "type": "result", - "subtype": raw_subtype, - "is_error": False, - "result": "ok", - "session_id": "s1", - } - ) - session = parse_session_result(ndjson) - assert isinstance(session.subtype, CliSubtype) - - def test_string_subtype_coerced_in_post_init(self): - from autoskillit.core.types import CliSubtype - - session = ClaudeSessionResult( - subtype="success", # type: ignore[arg-type] - is_error=False, - result="ok", - session_id="s1", - ) - assert isinstance(session.subtype, CliSubtype) - assert session.subtype == CliSubtype.SUCCESS - - # --------------------------------------------------------------------------- # Accumulator pattern tests: state preserved on fallback paths # --------------------------------------------------------------------------- @@ -1283,6 +623,173 @@ def test_context_exhaustion_via_flat_record_without_result(self): # --------------------------------------------------------------------------- +class TestThinkingBlockParsing: + """parse_session_result handles thinking content blocks correctly.""" + + def test_thinking_only_turn_sets_flag(self): + ndjson = "\n".join( + [ + json.dumps( + { + "type": "assistant", + "message": { + "content": [ + { + "type": "thinking", + "thinking": "Let me re-run pre-commit...", + } + ], + "stop_reason": "end_turn", + }, + } + ), + json.dumps( + { + "type": "result", + "subtype": "success", + "is_error": False, + "result": "", + "session_id": "s1", + } + ), + ] + ) + session = parse_session_result(ndjson) + assert session.has_thinking_only_turn is True + assert session.assistant_messages == [] + + def test_thinking_text_is_not_stored_as_assistant_message(self): + ndjson = "\n".join( + [ + json.dumps( + { + "type": "assistant", + "message": { + "content": [{"type": "thinking", "thinking": "Reasoning here..."}], + }, + } + ), + json.dumps( + { + "type": "result", + "subtype": "success", + "is_error": False, + "result": "", + "session_id": "s1", + } + ), + ] + ) + session = parse_session_result(ndjson) + assert session.assistant_messages == [] + + def test_redacted_thinking_also_sets_flag(self): + ndjson = "\n".join( + [ + json.dumps( + { + "type": "assistant", + "message": { + "content": [{"type": "redacted_thinking", "data": "REDACTED"}], + }, + } + ), + json.dumps( + { + "type": "result", + "subtype": "success", + "is_error": False, + "result": "", + "session_id": "s1", + } + ), + ] + ) + session = parse_session_result(ndjson) + assert session.has_thinking_only_turn is True + + def test_thinking_plus_text_turn_is_not_thinking_only(self): + ndjson = "\n".join( + [ + json.dumps( + { + "type": "assistant", + "message": { + "content": [ + {"type": "thinking", "thinking": "First I'll check..."}, + {"type": "text", "text": "I found the issue."}, + ], + }, + } + ), + json.dumps( + { + "type": "result", + "subtype": "success", + "is_error": False, + "result": "done", + "session_id": "s1", + } + ), + ] + ) + session = parse_session_result(ndjson) + assert session.has_thinking_only_turn is False + assert session.assistant_messages == ["I found the issue."] + + def test_thinking_with_tool_use_is_not_thinking_only(self): + ndjson = "\n".join( + [ + json.dumps( + { + "type": "assistant", + "message": { + "content": [ + {"type": "thinking", "thinking": "I should read the file."}, + {"type": "tool_use", "name": "Read", "id": "tu_1"}, + ], + }, + } + ), + ] + ) + session = parse_session_result(ndjson) + assert session.has_thinking_only_turn is False + assert len(session.tool_uses) == 1 + + def test_no_thinking_blocks_leaves_flag_false(self): + ndjson = json.dumps( + { + "type": "result", + "subtype": "success", + "is_error": False, + "result": "done", + "session_id": "s1", + } + ) + session = parse_session_result(ndjson) + assert session.has_thinking_only_turn is False + + @pytest.mark.parametrize( + "block_type", + ["text", "tool_use", "thinking", "redacted_thinking", "image", "tool_result", "unknown"], + ) + def test_all_content_block_types_handled_without_error(self, block_type: str): + block: dict = {"type": block_type} + if block_type == "text": + block["text"] = "hello" + elif block_type == "tool_use": + block["name"] = "Read" + block["id"] = "tu_x" + elif block_type == "thinking": + block["thinking"] = "reasoning" + elif block_type == "redacted_thinking": + block["data"] = "REDACTED" + ndjson = json.dumps({"type": "assistant", "message": {"content": [block]}}) + session = parse_session_result(ndjson) + assert session is not None + + class TestFullChainZeroWriteGate: """Full chain: NDJSON → parse_session_result → write_call_count.""" @@ -1306,3 +813,45 @@ def test_write_call_count_survives_unparseable(self): session = parse_session_result(ndjson) write_call_count = sum(1 for t in session.tool_uses if t.get("name") in {"Write", "Edit"}) assert write_call_count == 2 + + +# --------------------------------------------------------------------------- +# T6: SkillResult.to_json() includes infra_exit_category +# --------------------------------------------------------------------------- + + +def test_skill_result_to_json_includes_infra_exit_category(): + """infra_exit_category is serialized when non-empty.""" + + sr = SkillResult( + success=False, + result="", + session_id="s1", + subtype="context_exhaustion", + is_error=True, + exit_code=1, + needs_retry=True, + retry_reason=RetryReason.RESUME, + stderr="", + infra=InfraOutcome(exit_category=InfraExitCategory.CONTEXT_EXHAUSTED), + ) + data = json.loads(sr.to_json()) + assert data["infra_exit_category"] == "context_exhausted" + + +def test_skill_result_to_json_infra_exit_category_always_present(): + """infra_exit_category is always present in JSON; defaults to empty string.""" + sr = SkillResult( + success=True, + result="done", + session_id="s1", + subtype="success", + is_error=False, + exit_code=0, + needs_retry=False, + retry_reason=RetryReason.NONE, + stderr="", + ) + data = json.loads(sr.to_json()) + assert "infra_exit_category" in data + assert data["infra_exit_category"] == "" diff --git a/tests/execution/test_session_result.py b/tests/execution/test_session_result.py new file mode 100644 index 0000000000..18a9eca0e1 --- /dev/null +++ b/tests/execution/test_session_result.py @@ -0,0 +1,727 @@ +"""L1 unit tests for ClaudeSessionResult and parse_session_result — result types and policies.""" + +from __future__ import annotations + +import json + +import pytest + +from autoskillit.core.types import ( + CONTEXT_EXHAUSTION_MARKER, + RetryReason, +) +from autoskillit.execution.session import ClaudeSessionResult, parse_session_result +from tests.execution.conftest import _make_success_session + +pytestmark = [pytest.mark.layer("execution"), pytest.mark.small] + + +def _make_error_session( + subtype: str = "error_during_execution", + result: str = "failed", + errors: list[str] | None = None, +) -> ClaudeSessionResult: + return ClaudeSessionResult( + subtype=subtype, + is_error=True, + result=result, + session_id="s1", + errors=errors or [], + ) + + +class TestClaudeSessionResult: + """ClaudeSessionResult correctly parses Claude Code JSON output.""" + + def test_parses_success_result(self): + """Normal completion extracts result and session_id.""" + raw = { + "type": "result", + "subtype": "success", + "is_error": False, + "result": "Done.", + "session_id": "abc-123", + } + parsed = parse_session_result(json.dumps(raw)) + assert parsed.subtype == "success" + assert parsed.is_error is False + assert parsed.result == "Done." + assert parsed.session_id == "abc-123" + assert parsed.needs_retry is False + assert parsed.retry_reason == RetryReason.NONE + + def test_parses_error_max_turns(self): + """Turn limit produces needs_retry=True with reason=RESUME.""" + raw = { + "type": "result", + "subtype": "error_max_turns", + "is_error": False, + "session_id": "abc-123", + "errors": ["Max turns reached"], + } + parsed = parse_session_result(json.dumps(raw)) + assert parsed.subtype == "error_max_turns" + assert parsed.needs_retry is True + assert parsed.retry_reason == RetryReason.RESUME + assert parsed.result == "" + + def test_parses_prompt_too_long(self): + """Context exhaustion produces needs_retry=True with reason=RESUME.""" + raw = { + "type": "result", + "subtype": "success", + "is_error": True, + "result": CONTEXT_EXHAUSTION_MARKER.capitalize(), + "session_id": "abc-123", + } + parsed = parse_session_result(json.dumps(raw)) + assert parsed.is_error is True + assert parsed.needs_retry is True + assert parsed.retry_reason == RetryReason.RESUME + + def test_parses_execution_error_not_retriable(self): + """Runtime errors are not automatically retriable.""" + raw = { + "type": "result", + "subtype": "error_during_execution", + "is_error": True, + "session_id": "abc-123", + "errors": ["Tool execution failed"], + } + parsed = parse_session_result(json.dumps(raw)) + assert parsed.subtype == "error_during_execution" + assert parsed.needs_retry is False + assert parsed.retry_reason == RetryReason.NONE + + def test_non_json_stdout_is_error(self): + """Non-JSON output (crashes, tracebacks) is always an error.""" + parsed = parse_session_result("Traceback (most recent call last):\n File...") + assert parsed.is_error is True + assert parsed.subtype == "unparseable" + assert "Traceback" in parsed.result + assert parsed.needs_retry is False + + def test_empty_stdout_is_error(self): + """Empty stdout means the session produced no output — always an error.""" + parsed = parse_session_result("") + assert parsed.is_error is True + assert parsed.subtype == "empty_output" + assert parsed.result == "" + assert parsed.needs_retry is False + + def test_whitespace_only_stdout_is_error(self): + """Whitespace-only stdout is treated as empty.""" + parsed = parse_session_result(" \n \t ") + assert parsed.is_error is True + assert parsed.subtype == "empty_output" + + def test_json_without_type_result_is_error(self): + """JSON that isn't a Claude result object is rejected by fallback.""" + parsed = parse_session_result('{"some": "random", "json": true}') + assert parsed.is_error is True + assert parsed.subtype == "unparseable" + + def test_non_dict_json_is_error(self): + """Non-dict JSON (list, string, number) is unparseable.""" + parsed = parse_session_result("[1, 2, 3]") + assert parsed.is_error is True + assert parsed.subtype == "unparseable" + + def test_handles_ndjson_with_multiple_lines(self): + """Parser finds type=result in multi-line NDJSON output.""" + lines = [ + json.dumps({"type": "assistant", "message": "working..."}), + json.dumps( + { + "type": "result", + "subtype": "success", + "is_error": False, + "result": "Done.", + "session_id": "s1", + } + ), + ] + parsed = parse_session_result("\n".join(lines)) + assert parsed.subtype == "success" + assert parsed.result == "Done." + assert parsed.session_id == "s1" + + def test_needs_retry_and_retry_reason_are_consistent(self): + """retry_reason is RESUME iff needs_retry is True, NONE otherwise.""" + cases = [ + ("success", False, "Done."), + ("error_max_turns", False, ""), + ("success", True, "Prompt is too long"), + ("error_during_execution", True, "crashed"), + ("unknown", False, ""), + ] + for subtype, is_error, result_text in cases: + session = ClaudeSessionResult( + subtype=subtype, + is_error=is_error, + result=result_text, + session_id="s1", + ) + if session.needs_retry: + assert session.retry_reason == RetryReason.RESUME, ( + f"needs_retry=True but retry_reason={session.retry_reason!r} " + f"for subtype={subtype}, is_error={is_error}" + ) + else: + assert session.retry_reason == RetryReason.NONE, ( + f"needs_retry=False but retry_reason={session.retry_reason!r} " + f"for subtype={subtype}, is_error={is_error}" + ) + + def test_all_retriable_cases_produce_same_retry_reason(self): + """Phase 1 (ClaudeSessionResult.retry_reason property) always returns RESUME + when needs_retry is True. + + This property is only triggered by context/turn limit signals (error_max_turns + and context_exhaustion). Both conditions always have partial progress on disk + and always return RESUME. + + Note: _compute_retry() can return RetryReason.EMPTY_OUTPUT for NATURAL_EXIT + + kill_anomaly cases with no context exhaustion. That is Phase 2 behavior and is + not tested here. + """ + max_turns_case = ClaudeSessionResult( + subtype="error_max_turns", + is_error=False, + result="", + session_id="s1", + ) + context_case = ClaudeSessionResult( + subtype="success", + is_error=True, + result="Prompt is too long", + session_id="s2", + ) + assert max_turns_case.needs_retry is True + assert context_case.needs_retry is True + assert max_turns_case.retry_reason == context_case.retry_reason + + +class TestAgentResult: + """agent_result produces actionable text for LLM callers.""" + + def test_rewrites_context_exhaustion(self): + """Context exhaustion result must NOT contain 'Prompt is too long'.""" + session = ClaudeSessionResult( + subtype="success", + is_error=True, + result="Prompt is too long", + session_id="s1", + ) + assert "prompt is too long" not in session.agent_result.lower() + assert "context" in session.agent_result.lower() + assert "continue" in session.agent_result.lower() + + def test_rewrites_max_turns(self): + """Max turns result must describe the situation, not pass through empty string.""" + session = ClaudeSessionResult( + subtype="error_max_turns", + is_error=False, + result="", + session_id="s1", + ) + assert ( + "turn limit" in session.agent_result.lower() + or "resume" in session.agent_result.lower() + ) + + def test_preserves_normal_result(self): + """Normal success result passes through unchanged.""" + session = ClaudeSessionResult( + subtype="success", + is_error=False, + result="Task completed. Created 3 files.", + session_id="s1", + ) + assert session.agent_result == "Task completed. Created 3 files." + + def test_preserves_error_result_when_not_retriable(self): + """Non-retriable errors pass through unchanged.""" + session = ClaudeSessionResult( + subtype="error_during_execution", + is_error=True, + result="Tool execution failed: permission denied", + session_id="s1", + ) + assert session.agent_result == "Tool execution failed: permission denied" + + +class TestClaudeSessionResultTypeEnforcement: + """ClaudeSessionResult.__post_init__ enforces field types.""" + + def test_null_result_becomes_empty_string(self): + session = ClaudeSessionResult(subtype="error", is_error=True, result=None, session_id="s1") + assert session.result == "" + + def test_null_errors_becomes_empty_list(self): + session = ClaudeSessionResult( + subtype="error", is_error=True, result="err", session_id="s1", errors=None + ) + assert session.errors == [] + + def test_null_subtype_becomes_unknown(self): + session = ClaudeSessionResult(subtype=None, is_error=False, result="ok", session_id="s1") + assert session.subtype == "unknown" + + def test_null_session_id_becomes_empty(self): + session = ClaudeSessionResult( + subtype="success", is_error=False, result="ok", session_id=None + ) + assert session.session_id == "" + + def test_is_context_exhausted_with_null_safe_fields(self): + session = ClaudeSessionResult( + subtype="error", is_error=True, result=None, session_id="s1", errors=None + ) + assert session._is_context_exhausted() is False + + def test_list_content_result_becomes_string(self): + blocks = [{"type": "text", "text": "Task completed."}] + session = ClaudeSessionResult( + subtype="success", is_error=False, result=blocks, session_id="s1" + ) + assert session.result == "Task completed." + assert isinstance(session.result, str) + + def test_thinking_block_in_list_result_yields_empty_string(self): + blocks = [{"type": "thinking", "thinking": "reasoning here..."}] + session = ClaudeSessionResult( + subtype="success", is_error=False, result=blocks, session_id="s1" + ) + assert session.result == "" + assert isinstance(session.result, str) + + def test_mixed_text_and_thinking_blocks_only_extracts_text(self): + blocks = [ + {"type": "thinking", "thinking": "internal reasoning"}, + {"type": "text", "text": "Final answer."}, + ] + session = ClaudeSessionResult( + subtype="success", is_error=False, result=blocks, session_id="s1" + ) + assert session.result == "Final answer." + + +class TestParseSessionResultNullFields: + """parse_session_result handles null JSON values correctly.""" + + def test_null_result_field(self): + raw = { + "type": "result", + "subtype": "error_during_execution", + "is_error": True, + "result": None, + "session_id": "s1", + } + parsed = parse_session_result(json.dumps(raw)) + assert parsed.result == "" + + def test_null_errors_field(self): + raw = { + "type": "result", + "subtype": "success", + "is_error": False, + "result": "done", + "session_id": "s1", + "errors": None, + } + parsed = parse_session_result(json.dumps(raw)) + assert parsed.errors == [] + + +class TestClaudeSessionResultTokenUsage: + """Token usage field on ClaudeSessionResult.""" + + def test_default_is_none(self): + """token_usage defaults to None when not provided.""" + session = ClaudeSessionResult( + subtype="success", is_error=False, result="Done.", session_id="s1" + ) + assert session.token_usage is None + + def test_preserves_token_usage_dict(self): + """token_usage dict is stored and accessible.""" + usage = { + "input_tokens": 100, + "output_tokens": 50, + "cache_creation_input_tokens": 10, + "cache_read_input_tokens": 5, + "model_breakdown": {"claude-sonnet-4-6": {"input_tokens": 100, "output_tokens": 50}}, + } + session = ClaudeSessionResult( + subtype="success", + is_error=False, + result="Done.", + session_id="s1", + token_usage=usage, + ) + assert session.token_usage is usage + assert session.token_usage["input_tokens"] == 100 + assert "model_breakdown" in session.token_usage + + +class TestClaudeSessionResultBasic: + def test_post_init_list_with_non_dict_element(self): + s = ClaudeSessionResult( + subtype="success", + is_error=False, + result=["plain string", {"type": "text", "text": "extra"}], + session_id="s1", + ) + assert s.result == "plain string\nextra" + + +class TestClaudeSessionResultContextExhausted: + def test_is_context_exhausted_via_errors_list(self): + s = ClaudeSessionResult( + subtype="error", + is_error=True, + result="", + session_id="s1", + errors=[f"Request failed: {CONTEXT_EXHAUSTION_MARKER}"], + ) + assert s._is_context_exhausted() is True + + def test_is_context_exhausted_via_result_text(self): + s = ClaudeSessionResult( + subtype="success", + is_error=True, + result=f"Something: {CONTEXT_EXHAUSTION_MARKER}", + session_id="s1", + ) + assert s._is_context_exhausted() is True + + def test_is_context_exhausted_false_when_no_error(self): + s = ClaudeSessionResult( + subtype="success", + is_error=False, + result=f"contains {CONTEXT_EXHAUSTION_MARKER} text", + session_id="s1", + ) + assert s._is_context_exhausted() is False + + def test_jsonl_flat_record_sets_context_exhausted_flag(self): + """parse_session_result sets jsonl_context_exhausted=True for flat assistant record.""" + import json + + flat_record = json.dumps( + { + "type": "assistant", + "content": [{"type": "text", "text": "Prompt is too long"}], + "output_tokens": 0, + "input_tokens": 0, + "cache_read_input_tokens": 0, + } + ) + result_record = json.dumps( + { + "type": "result", + "subtype": "success", + "is_error": False, + "result": "", + "session_id": "sess-1", + } + ) + session = parse_session_result(flat_record + "\n" + result_record) + assert session.jsonl_context_exhausted is True + + def test_jsonl_flat_record_is_context_exhausted_bypasses_is_error_guard(self): + """_is_context_exhausted() returns True via JSONL flag even when is_error=False.""" + s = ClaudeSessionResult( + subtype="success", + is_error=False, + result="", + session_id="s1", + jsonl_context_exhausted=True, + ) + assert s._is_context_exhausted() is True + assert s.needs_retry is True + + def test_jsonl_flat_record_nonzero_output_tokens_not_detected(self): + """Flat assistant record with output_tokens > 0 does NOT trigger detection. + + The content deliberately contains CONTEXT_EXHAUSTION_MARKER so that only + the output_tokens != 0 guard suppresses detection — isolating the gate + under test. + """ + flat_record = json.dumps( + { + "type": "assistant", + "content": [{"type": "text", "text": CONTEXT_EXHAUSTION_MARKER}], + "output_tokens": 42, + "input_tokens": 100, + } + ) + result_record = json.dumps( + { + "type": "result", + "subtype": "success", + "is_error": False, + "result": "done %%ORDER_UP%%", + "session_id": "sess-2", + } + ) + session = parse_session_result(flat_record + "\n" + result_record) + assert session.jsonl_context_exhausted is False + + +class TestClaudeSessionResultNeedsRetry: + def test_needs_retry_true_for_max_turns(self): + s = ClaudeSessionResult( + subtype="error_max_turns", is_error=False, result="r", session_id="s1" + ) + assert s.needs_retry is True + + def test_needs_retry_true_for_context_exhausted(self): + s = ClaudeSessionResult( + subtype="error", + is_error=True, + result="", + session_id="s1", + errors=[CONTEXT_EXHAUSTION_MARKER], + ) + assert s.needs_retry is True + + def test_needs_retry_false_for_success(self): + s = _make_success_session() + assert s.needs_retry is False + + def test_retry_reason_resume_when_needs_retry(self): + s = ClaudeSessionResult( + subtype="error_max_turns", is_error=False, result="r", session_id="s1" + ) + assert s.retry_reason == RetryReason.RESUME + + def test_retry_reason_none_when_no_retry(self): + s = _make_success_session() + assert s.retry_reason == RetryReason.NONE + + +# --------------------------------------------------------------------------- +# CliSubtype sealed enum tests +# --------------------------------------------------------------------------- + + +class TestCliSubtypeExhaustiveCoverage: + """Every subtype returned by parse_session_result is a CliSubtype member.""" + + def test_success_subtype_is_cli_subtype(self): + from autoskillit.core.types import CliSubtype + + ndjson = json.dumps( + { + "type": "result", + "subtype": "success", + "is_error": False, + "result": "ok", + "session_id": "s1", + } + ) + session = parse_session_result(ndjson) + assert isinstance(session.subtype, CliSubtype) + assert session.subtype == CliSubtype.SUCCESS + + def test_empty_output_subtype_is_cli_subtype(self): + from autoskillit.core.types import CliSubtype + + session = parse_session_result("") + assert isinstance(session.subtype, CliSubtype) + assert session.subtype == CliSubtype.EMPTY_OUTPUT + + def test_unparseable_subtype_is_cli_subtype(self): + from autoskillit.core.types import CliSubtype + + session = parse_session_result("not json at all") + assert isinstance(session.subtype, CliSubtype) + assert session.subtype == CliSubtype.UNPARSEABLE + + def test_unknown_cli_subtype_maps_to_unknown(self): + from autoskillit.core.types import CliSubtype + + ndjson = json.dumps( + { + "type": "result", + "subtype": "some_future_subtype", + "is_error": False, + "result": "ok", + "session_id": "s1", + } + ) + session = parse_session_result(ndjson) + assert isinstance(session.subtype, CliSubtype) + assert session.subtype == CliSubtype.UNKNOWN + + @pytest.mark.parametrize( + "raw_subtype", + [ + "success", + "error_max_turns", + "error_during_execution", + "unknown", + "empty_output", + "unparseable", + "timeout", + ], + ) + def test_all_known_subtypes_are_cli_subtype_members(self, raw_subtype: str): + from autoskillit.core.types import CliSubtype + + ndjson = json.dumps( + { + "type": "result", + "subtype": raw_subtype, + "is_error": False, + "result": "ok", + "session_id": "s1", + } + ) + session = parse_session_result(ndjson) + assert isinstance(session.subtype, CliSubtype) + + def test_string_subtype_coerced_in_post_init(self): + from autoskillit.core.types import CliSubtype + + session = ClaudeSessionResult( + subtype="success", # type: ignore[arg-type] + is_error=False, + result="ok", + session_id="s1", + ) + assert isinstance(session.subtype, CliSubtype) + assert session.subtype == CliSubtype.SUCCESS + + +class TestLifespanStarted: + def test_lifespan_started_true_when_tool_uses_present(self): + from autoskillit.execution.session import ClaudeSessionResult + + session = ClaudeSessionResult( + subtype="success", + is_error=False, + result="ok", + session_id="s1", + tool_uses=[{"name": "open_kitchen", "id": "t1"}], + ) + assert session.lifespan_started is True + + def test_lifespan_started_false_when_no_tool_uses(self): + from autoskillit.execution.session import ClaudeSessionResult + + session = ClaudeSessionResult( + subtype="success", + is_error=False, + result="ok", + session_id="s1", + tool_uses=[], + ) + assert session.lifespan_started is False + + +# --------------------------------------------------------------------------- +# stop_reasons extraction tests +# --------------------------------------------------------------------------- + + +def test_parse_session_result_extracts_stop_reasons(): + ndjson = "\n".join( + [ + json.dumps( + { + "type": "assistant", + "message": {"stop_reason": "tool_use", "content": [], "usage": {}}, + } + ), + json.dumps( + { + "type": "assistant", + "message": {"stop_reason": "end_turn", "content": [], "usage": {}}, + } + ), + json.dumps( + { + "type": "result", + "subtype": "success", + "is_error": False, + "result": "Done.", + "session_id": "abc", + } + ), + ] + ) + parsed = parse_session_result(ndjson) + assert parsed.stop_reasons == ["tool_use", "end_turn"] + assert parsed.last_stop_reason == "end_turn" + + +def test_parse_session_result_empty_stop_reasons_when_absent(): + ndjson = json.dumps( + { + "type": "result", + "subtype": "success", + "is_error": False, + "result": "Done.", + "session_id": "abc", + } + ) + parsed = parse_session_result(ndjson) + assert parsed.stop_reasons == [] + assert parsed.last_stop_reason == "" + + +def test_last_stop_reason_is_last_turn(): + ndjson = "\n".join( + [ + json.dumps( + { + "type": "assistant", + "requestId": "r1", + "timestamp": "t1", + "message": {"stop_reason": "tool_use", "content": [], "usage": {}}, + } + ), + json.dumps( + { + "type": "assistant", + "requestId": "r2", + "timestamp": "t2", + "message": {"stop_reason": "max_tokens", "content": [], "usage": {}}, + } + ), + json.dumps( + { + "type": "result", + "subtype": "success", + "is_error": False, + "result": "x", + "session_id": "s", + } + ), + ] + ) + parsed = parse_session_result(ndjson) + assert parsed.last_stop_reason == "max_tokens" + + +def test_parse_session_result_missing_stop_reason_skipped(): + ndjson = "\n".join( + [ + json.dumps({"type": "assistant", "message": {"content": [], "usage": {}}}), + json.dumps( + { + "type": "result", + "subtype": "success", + "is_error": False, + "result": "ok", + "session_id": "s", + } + ), + ] + ) + parsed = parse_session_result(ndjson) + assert parsed.stop_reasons == [] + assert parsed.last_stop_reason == "" diff --git a/tests/execution/test_session_state_persistence.py b/tests/execution/test_session_state_persistence.py new file mode 100644 index 0000000000..a5da6407db --- /dev/null +++ b/tests/execution/test_session_state_persistence.py @@ -0,0 +1,75 @@ +"""T3: Early session ID persistence.""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from autoskillit.execution.session._session_state import ( + SessionState, + clear_session_state, + persist_session_state, + read_session_state, +) + +pytestmark = [pytest.mark.layer("execution"), pytest.mark.small] + + +class TestPersistSessionState: + def test_write_and_read(self, tmp_path: Path) -> None: + state = SessionState( + session_id="sess-abc", + pid=1234, + boot_id="boot-xyz", + starttime_ticks=99999, + ) + persist_session_state(state, tmp_path) + restored = read_session_state(tmp_path) + assert restored is not None + assert restored.session_id == "sess-abc" + assert restored.pid == 1234 + assert restored.boot_id == "boot-xyz" + assert restored.starttime_ticks == 99999 + + def test_optional_fields(self, tmp_path: Path) -> None: + state = SessionState( + session_id="s1", + pid=1, + boot_id="b", + starttime_ticks=0, + checkpoint_path="/tmp/cp.json", + infra_exit_category="context_exhausted", + ) + persist_session_state(state, tmp_path) + restored = read_session_state(tmp_path) + assert restored is not None + assert restored.checkpoint_path == "/tmp/cp.json" + assert restored.infra_exit_category == "context_exhausted" + + def test_read_missing_returns_none(self, tmp_path: Path) -> None: + assert read_session_state(tmp_path) is None + + def test_read_corrupt_returns_none(self, tmp_path: Path) -> None: + (tmp_path / "dispatch_session_state.json").write_text("not json") + assert read_session_state(tmp_path) is None + + def test_read_malformed_returns_none(self, tmp_path: Path) -> None: + (tmp_path / "dispatch_session_state.json").write_text('{"session_id": "x"}') + assert read_session_state(tmp_path) is None + + def test_clear_removes_file(self, tmp_path: Path) -> None: + state = SessionState(session_id="s", pid=1, boot_id="b", starttime_ticks=0) + persist_session_state(state, tmp_path) + assert read_session_state(tmp_path) is not None + clear_session_state(tmp_path) + assert read_session_state(tmp_path) is None + + def test_clear_missing_is_noop(self, tmp_path: Path) -> None: + clear_session_state(tmp_path) + + def test_creates_parent_dirs(self, tmp_path: Path) -> None: + nested = tmp_path / "deep" / "nested" / "dir" + state = SessionState(session_id="s", pid=1, boot_id="b", starttime_ticks=0) + persist_session_state(state, nested) + assert read_session_state(nested) is not None diff --git a/tests/execution/test_termination_action.py b/tests/execution/test_termination_action.py index 30b1a54bf6..8b5b77bedc 100644 --- a/tests/execution/test_termination_action.py +++ b/tests/execution/test_termination_action.py @@ -12,6 +12,8 @@ from autoskillit.core.types import TerminationAction, TerminationReason from autoskillit.execution.process import decide_termination_action +pytestmark = [pytest.mark.layer("execution"), pytest.mark.small] + @pytest.mark.parametrize( "termination,timeout_fired,process_exited,expected", diff --git a/tests/execution/test_termination_executor.py b/tests/execution/test_termination_executor.py index 8df59755c2..f49366821d 100644 --- a/tests/execution/test_termination_executor.py +++ b/tests/execution/test_termination_executor.py @@ -17,6 +17,8 @@ from autoskillit.core.types import KillReason, TerminationAction from autoskillit.execution.process import execute_termination_action +pytestmark = [pytest.mark.layer("execution"), pytest.mark.medium] + # --------------------------------------------------------------------------- # Helper scripts # --------------------------------------------------------------------------- diff --git a/tests/execution/test_testing.py b/tests/execution/test_testing.py index 7cf13d719f..d1904aeea7 100644 --- a/tests/execution/test_testing.py +++ b/tests/execution/test_testing.py @@ -6,7 +6,6 @@ import pytest -from autoskillit.config import AutomationConfig from autoskillit.core.types import ( AUTOSKILLIT_PRIVATE_ENV_VARS, SubprocessResult, @@ -14,11 +13,16 @@ ) from autoskillit.execution.testing import ( DefaultTestRunner, + _read_sidecar_base_branch, + _resolve_base_ref, build_sanitized_env, ) from autoskillit.execution.testing import ( parse_pytest_summary as _parse_pytest_summary, ) +from tests._helpers import make_test_check_config, make_test_config + +pytestmark = [pytest.mark.layer("execution"), pytest.mark.medium] def test_build_sanitized_env_strips_private_env_vars(monkeypatch): @@ -66,7 +70,7 @@ async def capturing_runner(cmd, *, cwd, timeout, **kwargs): pid=12345, ) - runner = DefaultTestRunner(config=AutomationConfig(), runner=capturing_runner) + runner = DefaultTestRunner(config=make_test_config(), runner=capturing_runner) await runner.run(cwd=tmp_path) assert "env" in captured_kwargs, "DefaultTestRunner must pass env= to its runner" @@ -305,14 +309,14 @@ def test_check_test_passed_parses_pytest_summary_in_stderr() -> None: @pytest.mark.anyio async def test_default_test_runner_returns_test_result_with_stderr(tmp_path: Path) -> None: - from autoskillit.config import AutomationConfig from autoskillit.core import TestResult from autoskillit.execution.testing import DefaultTestRunner - from tests.conftest import MockSubprocessRunner, _make_result + from tests.conftest import _make_result + from tests.fakes import MockSubprocessRunner runner = MockSubprocessRunner() runner.push(_make_result(0, "", stderr="PASSED [0.5s] all tests")) - config = AutomationConfig() + config = make_test_config() tester = DefaultTestRunner(config=config, runner=runner) result = await tester.run(tmp_path) assert isinstance(result, TestResult) @@ -327,3 +331,593 @@ def test_test_result_dataclass_fields() -> None: assert r.passed is True assert r.stdout == "out" assert r.stderr == "err" + + +# ---------- TestCheckConfig filter_mode / base_ref ---------- + + +def test_test_check_config_has_filter_mode_and_base_ref_fields(): + cfg = make_test_check_config() + assert cfg.filter_mode is None + assert cfg.base_ref is None + + +def test_from_dynaconf_reads_filter_mode_and_base_ref(monkeypatch): + from tests._helpers import make_dynaconf_and_automation_config + + _make_dynaconf, AutomationConfig = make_dynaconf_and_automation_config() + monkeypatch.setenv("AUTOSKILLIT_TEST_CHECK__FILTER_MODE", "conservative") + monkeypatch.setenv("AUTOSKILLIT_TEST_CHECK__BASE_REF", "origin/main") + d = _make_dynaconf() + cfg = AutomationConfig.from_dynaconf(d) + assert cfg.test_check.filter_mode == "conservative" + assert cfg.test_check.base_ref == "origin/main" + + +@pytest.mark.anyio +async def test_default_test_runner_injects_filter_mode_env_var(monkeypatch, tmp_path): + captured_kwargs: dict = {} + + async def capturing_runner(cmd, *, cwd, timeout, **kwargs): + captured_kwargs.update(kwargs) + return SubprocessResult( + returncode=0, + stdout="1 passed", + stderr="", + termination=TerminationReason.NATURAL_EXIT, + pid=12345, + ) + + config = make_test_config(test_check=make_test_check_config(filter_mode="conservative")) + runner = DefaultTestRunner(config=config, runner=capturing_runner) + await runner.run(cwd=tmp_path) + assert captured_kwargs["env"]["AUTOSKILLIT_TEST_FILTER"] == "conservative" + + +@pytest.mark.anyio +async def test_default_test_runner_omits_filter_env_when_none(monkeypatch, tmp_path): + captured_kwargs: dict = {} + + async def capturing_runner(cmd, *, cwd, timeout, **kwargs): + captured_kwargs.update(kwargs) + return SubprocessResult( + returncode=0, + stdout="1 passed", + stderr="", + termination=TerminationReason.NATURAL_EXIT, + pid=12345, + ) + + monkeypatch.delenv("AUTOSKILLIT_TEST_FILTER", raising=False) + config = make_test_config(test_check=make_test_check_config()) + runner = DefaultTestRunner(config=config, runner=capturing_runner) + await runner.run(cwd=tmp_path) + assert "AUTOSKILLIT_TEST_FILTER" not in captured_kwargs["env"] + + +@pytest.mark.anyio +async def test_default_test_runner_injects_base_ref_from_config(monkeypatch, tmp_path): + captured_kwargs: dict = {} + + async def capturing_runner(cmd, *, cwd, timeout, **kwargs): + captured_kwargs.update(kwargs) + return SubprocessResult( + returncode=0, + stdout="1 passed", + stderr="", + termination=TerminationReason.NATURAL_EXIT, + pid=12345, + ) + + config = make_test_config(test_check=make_test_check_config(base_ref="origin/main")) + runner = DefaultTestRunner(config=config, runner=capturing_runner) + await runner.run(cwd=tmp_path) + assert captured_kwargs["env"]["AUTOSKILLIT_TEST_BASE_REF"] == "origin/main" + + +# ---------- _resolve_base_ref ---------- + + +@pytest.mark.anyio +async def test_resolve_base_ref_config_override_wins(tmp_path): + result = await _resolve_base_ref("origin/main", tmp_path) + assert result == "origin/main" + + +@pytest.mark.anyio +async def test_resolve_base_ref_returns_none_when_no_source(tmp_path): + result = await _resolve_base_ref(None, tmp_path) + assert result is None + + +@pytest.mark.anyio +async def test_resolve_base_ref_git_upstream_fallback(tmp_path): + import subprocess + + repo = tmp_path / "repo" + repo.mkdir() + subprocess.run(["git", "init", str(repo)], check=True, capture_output=True) + subprocess.run( + ["git", "-C", str(repo), "config", "user.email", "test@test.com"], + check=True, + capture_output=True, + ) + subprocess.run( + ["git", "-C", str(repo), "config", "user.name", "test"], + check=True, + capture_output=True, + ) + subprocess.run( + ["git", "-C", str(repo), "commit", "--allow-empty", "-m", "init"], + check=True, + capture_output=True, + ) + subprocess.run( + ["git", "-C", str(repo), "branch", "upstream-branch"], + check=True, + capture_output=True, + ) + subprocess.run( + ["git", "-C", str(repo), "branch", "--set-upstream-to=upstream-branch"], + check=True, + capture_output=True, + ) + result = await _resolve_base_ref(None, repo) + assert result == "upstream-branch" + + +# ---------- _read_sidecar_base_branch ---------- + + +def test_read_sidecar_base_branch_returns_branch(tmp_path): + wt_dir = tmp_path / "my-worktree" + wt_dir.mkdir() + main_git = tmp_path / "main-repo" / ".git" + main_git.mkdir(parents=True) + worktrees_gitdir = main_git / "worktrees" / "my-worktree" + worktrees_gitdir.mkdir(parents=True) + (wt_dir / ".git").write_text(f"gitdir: {worktrees_gitdir}\n") + sidecar = tmp_path / "main-repo" / ".autoskillit" / "temp" / "worktrees" / "my-worktree" + sidecar.mkdir(parents=True) + (sidecar / "base-branch").write_text("impl-934\n") + assert _read_sidecar_base_branch(wt_dir) == "impl-934" + + +def test_read_sidecar_base_branch_returns_none_for_regular_dir(tmp_path): + assert _read_sidecar_base_branch(tmp_path) is None + + +def test_read_sidecar_base_branch_returns_none_for_main_checkout(tmp_path): + (tmp_path / ".git").mkdir() + assert _read_sidecar_base_branch(tmp_path) is None + + +# ---------- env var passthrough ---------- + + +def test_filter_env_vars_not_in_private_set(): + assert "AUTOSKILLIT_TEST_FILTER" not in AUTOSKILLIT_PRIVATE_ENV_VARS + assert "AUTOSKILLIT_TEST_BASE_REF" not in AUTOSKILLIT_PRIVATE_ENV_VARS + + +def test_defaults_yaml_has_filter_mode_and_base_ref(): + from autoskillit.core import load_yaml, pkg_root + + defaults = load_yaml(pkg_root() / "config" / "defaults.yaml") + tc = defaults["test_check"] + assert "filter_mode" in tc + assert "base_ref" in tc + assert tc["filter_mode"] is None + assert tc["base_ref"] is None + + +def test_test_result_has_duration_and_filter_fields(): + """TestResult accepts optional duration_seconds and filter stat fields.""" + from autoskillit.core import TestResult + + tr = TestResult( + passed=True, + stdout="out", + stderr="err", + duration_seconds=3.14, + tests_selected=73, + tests_deselected=275, + filter_mode="aggressive", + ) + assert tr.duration_seconds == 3.14 + assert tr.tests_selected == 73 + assert tr.tests_deselected == 275 + assert tr.filter_mode == "aggressive" + + +def test_test_result_new_fields_default_to_none(): + """Existing callers with positional args still work.""" + from autoskillit.core import TestResult + + tr = TestResult(passed=True, stdout="", stderr="") + assert tr.duration_seconds is None + assert tr.tests_selected is None + assert tr.tests_deselected is None + assert tr.filter_mode is None + + +@pytest.mark.anyio +async def test_default_test_runner_measures_duration(tmp_path: Path) -> None: + """run() populates duration_seconds on the returned TestResult.""" + from tests.conftest import _make_result + from tests.fakes import MockSubprocessRunner + + runner = MockSubprocessRunner() + runner.push(_make_result(0, "= 10 passed =\n", "")) + tester = DefaultTestRunner(config=make_test_config(), runner=runner) + result = await tester.run(tmp_path) + assert result.duration_seconds is not None + assert result.duration_seconds >= 0.0 + + +@pytest.mark.anyio +async def test_default_test_runner_populates_filter_stats_from_sidecar(tmp_path: Path) -> None: + """run() populates filter fields when sidecar JSON file is written.""" + import json + + sidecar_data = {"filter_mode": "conservative", "tests_selected": 50, "tests_deselected": 100} + + async def fake_runner(command, *, cwd, timeout, env, **kwargs): + sidecar_path = env.get("AUTOSKILLIT_FILTER_STATS_FILE") + assert sidecar_path, "AUTOSKILLIT_FILTER_STATS_FILE must be set in env" + Path(sidecar_path).write_text(json.dumps(sidecar_data)) + return SubprocessResult( + returncode=0, + stdout="= 50 passed =\n", + stderr="", + termination=TerminationReason.NATURAL_EXIT, + pid=12345, + ) + + tester = DefaultTestRunner(config=make_test_config(), runner=fake_runner) + result = await tester.run(tmp_path) + assert result.filter_mode == "conservative" + assert result.tests_selected == 50 + assert result.tests_deselected == 100 + + +@pytest.mark.anyio +async def test_default_test_runner_null_sidecar_values_become_none(tmp_path: Path) -> None: + """run() maps JSON null for selected/deselected counts to None in TestResult. + + Regression guard for the isinstance(ts, int) guard at testing.py:244-245. + When the sidecar contains null (as happens under xdist before the workeroutput + fix), DefaultTestRunner must return None — not crash or coerce to 0. + """ + import json + + sidecar_data = { + "filter_mode": "conservative", + "tests_selected": None, + "tests_deselected": None, + } + + async def fake_runner(command, *, cwd, timeout, env, **kwargs): + sidecar_path = env.get("AUTOSKILLIT_FILTER_STATS_FILE") + assert sidecar_path, "AUTOSKILLIT_FILTER_STATS_FILE must be set in env" + Path(sidecar_path).write_text(json.dumps(sidecar_data)) + return SubprocessResult( + returncode=0, + stdout="= 1 passed =\n", + stderr="", + termination=TerminationReason.NATURAL_EXIT, + pid=12345, + ) + + tester = DefaultTestRunner(config=make_test_config(), runner=fake_runner) + result = await tester.run(tmp_path) + assert result.filter_mode == "conservative" + assert result.tests_selected is None + assert result.tests_deselected is None + + +@pytest.mark.anyio +async def test_default_test_runner_no_sidecar_means_no_filter_stats(tmp_path: Path) -> None: + """run() leaves filter fields as None when no sidecar is written.""" + from tests.conftest import _make_result + from tests.fakes import MockSubprocessRunner + + runner = MockSubprocessRunner() + runner.push(_make_result(0, "= 10 passed =\n", "")) + tester = DefaultTestRunner(config=make_test_config(), runner=runner) + result = await tester.run(tmp_path) + assert result.filter_mode is None + assert result.tests_selected is None + assert result.tests_deselected is None + + +@pytest.mark.anyio +async def test_resolve_base_ref_default_base_branch_fallback(tmp_path: Path) -> None: + """AC5: When no config, sidecar, or upstream, default_base_branch is used.""" + result = await _resolve_base_ref(None, tmp_path, default_base_branch="develop") + assert result == "develop" + + +@pytest.mark.anyio +async def test_resolve_base_ref_config_wins_over_default_base_branch(tmp_path: Path) -> None: + """Config base_ref takes precedence over default_base_branch.""" + result = await _resolve_base_ref("origin/main", tmp_path, default_base_branch="develop") + assert result == "origin/main" + + +@pytest.mark.anyio +async def test_resolve_base_ref_none_default_base_branch_still_returns_none( + tmp_path: Path, +) -> None: + """When default_base_branch is also None, returns None.""" + result = await _resolve_base_ref(None, tmp_path, default_base_branch=None) + assert result is None + + +@pytest.mark.anyio +async def test_default_test_runner_passes_default_base_branch( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + """AC5: DefaultTestRunner.run() passes branching.default_base_branch to _resolve_base_ref.""" + captured_kwargs: dict = {} + + async def capturing_runner(cmd, *, cwd, timeout, env): + captured_kwargs["env"] = env + return SubprocessResult( + returncode=0, + stdout="1 passed", + stderr="", + termination=TerminationReason.NATURAL_EXIT, + pid=12345, + ) + + config = make_test_config( + test_check=make_test_check_config(filter_mode="conservative"), + ) + config.branching.default_base_branch = "develop" + runner = DefaultTestRunner(config=config, runner=capturing_runner) + await runner.run(cwd=tmp_path) + assert captured_kwargs["env"].get("AUTOSKILLIT_TEST_BASE_REF") == "develop" + + +# --------------------------------------------------------------------------- +# Multi-command (commands: list[list[str]]) tests +# --------------------------------------------------------------------------- + + +@pytest.mark.anyio +async def test_multi_command_sequential_pass(tmp_path: Path) -> None: + from tests.conftest import _make_result + from tests.fakes import MockSubprocessRunner + + runner = MockSubprocessRunner() + runner.push(_make_result(returncode=0, stdout="cmd1 output")) + runner.push(_make_result(returncode=0, stdout="cmd2 output")) + cfg = make_test_config( + test_check=make_test_check_config( + commands=[["cargo", "test"], ["task", "e2e"]], + ) + ) + result = await DefaultTestRunner(cfg, runner).run(tmp_path) + assert result.passed is True + assert "cmd1 output" in result.stdout + assert "cmd2 output" in result.stdout + + +@pytest.mark.anyio +async def test_multi_command_fail_fast(tmp_path: Path) -> None: + from tests.conftest import _make_result + from tests.fakes import MockSubprocessRunner + + runner = MockSubprocessRunner() + runner.push(_make_result(returncode=1, stdout="cmd1 failed")) + cfg = make_test_config( + test_check=make_test_check_config( + commands=[["cargo", "test"], ["task", "e2e"]], + ) + ) + result = await DefaultTestRunner(cfg, runner).run(tmp_path) + assert result.passed is False + assert len(runner.call_args_list) == 1 + + +@pytest.mark.anyio +async def test_multi_command_second_fails(tmp_path: Path) -> None: + from tests.conftest import _make_result + from tests.fakes import MockSubprocessRunner + + runner = MockSubprocessRunner() + runner.push(_make_result(returncode=0, stdout="cmd1 ok")) + runner.push(_make_result(returncode=1, stdout="cmd2 failed")) + cfg = make_test_config( + test_check=make_test_check_config( + commands=[["cargo", "test"], ["task", "e2e"]], + ) + ) + result = await DefaultTestRunner(cfg, runner).run(tmp_path) + assert result.passed is False + assert "cmd1 ok" in result.stdout + assert "cmd2 failed" in result.stdout + + +@pytest.mark.anyio +async def test_multi_command_timeout_ceiling( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + import autoskillit.execution.testing as testing_mod + from tests.conftest import _make_result + from tests.fakes import MockSubprocessRunner + + # Simulate 3 seconds elapsed between start and the second command's remaining check. + # monotonic() call sequence: start, loop1-remaining, loop2-remaining, elapsed + _times = iter([0.0, 0.0, 3.0, 3.0]) + monkeypatch.setattr( + testing_mod, + "time", + type("_FakeTime", (), {"monotonic": staticmethod(lambda: next(_times))})(), + ) + + runner = MockSubprocessRunner() + runner.push(_make_result(returncode=0, stdout="cmd1")) + runner.push(_make_result(returncode=0, stdout="cmd2")) + cfg = make_test_config( + test_check=make_test_check_config( + commands=[["a"], ["b"]], + timeout=10, + ) + ) + await DefaultTestRunner(cfg, runner).run(tmp_path) + t1 = runner.call_args_list[0][2] + t2 = runner.call_args_list[1][2] + assert t1 == pytest.approx(10.0) + assert t2 == pytest.approx(7.0) + assert t2 < t1 + + +@pytest.mark.anyio +async def test_multi_command_empty_commands_falls_back(tmp_path: Path) -> None: + from tests.conftest import _make_result + from tests.fakes import MockSubprocessRunner + + runner = MockSubprocessRunner() + runner.push(_make_result(returncode=0, stdout="fallback")) + cfg = make_test_config( + test_check=make_test_check_config( + command=["task", "test-check"], + commands=None, + ) + ) + result = await DefaultTestRunner(cfg, runner).run(tmp_path) + assert result.passed is True + assert runner.call_args_list[0][0] == ["task", "test-check"] + + +@pytest.mark.anyio +async def test_multi_command_single_entry_equivalent(tmp_path: Path) -> None: + from tests.conftest import _make_result + from tests.fakes import MockSubprocessRunner + + runner_a = MockSubprocessRunner() + runner_a.push(_make_result(returncode=0, stdout="out")) + runner_b = MockSubprocessRunner() + runner_b.push(_make_result(returncode=0, stdout="out")) + cfg_single = make_test_config(test_check=make_test_check_config(command=["task", "t"])) + cfg_multi = make_test_config(test_check=make_test_check_config(commands=[["task", "t"]])) + r1 = await DefaultTestRunner(cfg_single, runner_a).run(tmp_path) + r2 = await DefaultTestRunner(cfg_multi, runner_b).run(tmp_path) + assert r1.passed == r2.passed + + +def test_commands_and_command_mutual_exclusion() -> None: + with pytest.raises(ValueError, match="mutually exclusive"): + make_test_check_config( + command=["cargo", "test"], + commands=[["cargo", "test"], ["task", "e2e"]], + ) + + +@pytest.mark.anyio +async def test_multi_command_env_vars_propagated(tmp_path: Path) -> None: + captured_envs: list[dict] = [] + + async def capturing_runner(cmd, *, cwd, timeout, env, **kwargs): + captured_envs.append(dict(env)) + from tests.conftest import _make_result + + return _make_result(returncode=0, stdout="") + + cfg = make_test_config( + test_check=make_test_check_config( + commands=[["a"], ["b"]], + filter_mode="conservative", + ) + ) + await DefaultTestRunner(cfg, capturing_runner).run(tmp_path) + assert all("AUTOSKILLIT_TEST_FILTER" in e for e in captured_envs) + + +@pytest.mark.anyio +async def test_multi_command_section_headers(tmp_path: Path) -> None: + from tests.conftest import _make_result + from tests.fakes import MockSubprocessRunner + + runner = MockSubprocessRunner() + runner.push(_make_result(returncode=0, stdout="out1")) + runner.push(_make_result(returncode=0, stdout="out2")) + cfg = make_test_config( + test_check=make_test_check_config( + commands=[["cargo", "test"], ["task", "e2e"]], + ) + ) + result = await DefaultTestRunner(cfg, runner).run(tmp_path) + assert "=== [1/2]" in result.stdout + assert "=== [2/2]" in result.stdout + + +def test_defaults_yaml_has_commands_null() -> None: + import yaml + + from autoskillit.core.paths import pkg_root + + defaults = yaml.safe_load((pkg_root() / "config" / "defaults.yaml").read_text()) + assert "commands" in defaults["test_check"] + assert defaults["test_check"]["commands"] is None + + +def test_test_check_config_has_commands_field() -> None: + cfg = make_test_check_config() + assert hasattr(cfg, "commands") + assert cfg.commands is None + + +@pytest.mark.anyio +async def test_default_test_runner_reads_full_run_reason_from_sidecar(tmp_path: Path) -> None: + """run() populates full_run_reason when sidecar includes the field.""" + import json + + sidecar_data = { + "filter_mode": "conservative", + "tests_selected": None, + "tests_deselected": None, + "full_run_reason": "bucket_a", + } + + async def fake_runner(command, *, cwd, timeout, env, **kwargs): + sidecar_path = env.get("AUTOSKILLIT_FILTER_STATS_FILE") + assert sidecar_path, "AUTOSKILLIT_FILTER_STATS_FILE must be set in env" + Path(sidecar_path).write_text(json.dumps(sidecar_data)) + return SubprocessResult( + returncode=0, + stdout="= 100 passed =\n", + stderr="", + termination=TerminationReason.NATURAL_EXIT, + pid=12345, + ) + + tester = DefaultTestRunner(config=make_test_config(), runner=fake_runner) + result = await tester.run(tmp_path) + assert result.full_run_reason == "bucket_a" + + +@pytest.mark.anyio +async def test_default_test_runner_full_run_reason_none_when_absent(tmp_path: Path) -> None: + """run() leaves full_run_reason as None when sidecar omits the field.""" + import json + + sidecar_data = {"filter_mode": "conservative", "tests_selected": 50, "tests_deselected": 10} + + async def fake_runner(command, *, cwd, timeout, env, **kwargs): + sidecar_path = env.get("AUTOSKILLIT_FILTER_STATS_FILE") + assert sidecar_path, "AUTOSKILLIT_FILTER_STATS_FILE must be set in env" + Path(sidecar_path).write_text(json.dumps(sidecar_data)) + return SubprocessResult( + returncode=0, + stdout="= 50 passed =\n", + stderr="", + termination=TerminationReason.NATURAL_EXIT, + pid=12345, + ) + + tester = DefaultTestRunner(config=make_test_config(), runner=fake_runner) + result = await tester.run(tmp_path) + assert result.full_run_reason is None diff --git a/tests/execution/test_trace_target_resolver.py b/tests/execution/test_trace_target_resolver.py index 9cff1f98b7..31d40f1461 100644 --- a/tests/execution/test_trace_target_resolver.py +++ b/tests/execution/test_trace_target_resolver.py @@ -13,14 +13,19 @@ import pytest -pytestmark = pytest.mark.skipif( - sys.platform != "linux", - reason="Linux only — tests /proc descendant walking", -) +pytestmark = [ + pytest.mark.layer("execution"), + pytest.mark.medium, + pytest.mark.skipif( + sys.platform != "linux", + reason="Linux only — tests /proc descendant walking", + ), +] @pytest.mark.skipif(shutil.which("script") is None, reason="script(1) not available") -def test_resolve_trace_target_walks_from_wrapper_to_workload(): +@pytest.mark.anyio +async def test_resolve_trace_target_walks_from_wrapper_to_workload(): """resolve_trace_target resolves from script(1) wrapper to the workload process. Test 1.3: lock in the descendant-walk + basename-match contract. @@ -33,7 +38,9 @@ def test_resolve_trace_target_walks_from_wrapper_to_workload(): stderr=subprocess.DEVNULL, ) try: - target = resolve_trace_target(root_pid=proc.pid, expected_basename="python3", timeout=2.0) + target = await resolve_trace_target( + root_pid=proc.pid, expected_basename="python3", timeout=2.0 + ) assert target.pid != proc.pid, ( "Resolved PID must not be the wrapper (script) PID — " "resolver must walk children to find the workload" @@ -48,7 +55,8 @@ def test_resolve_trace_target_walks_from_wrapper_to_workload(): proc.wait() -def test_resolve_trace_target_raises_on_miss(): +@pytest.mark.anyio +async def test_resolve_trace_target_raises_on_miss(): """resolve_trace_target raises TraceTargetResolutionError when workload never appears. Test 1.4: failure must be loud, not a silent fall-back to wrapper PID. @@ -61,7 +69,7 @@ def test_resolve_trace_target_raises_on_miss(): proc = subprocess.Popen(["sleep", "5"]) try: with pytest.raises(TraceTargetResolutionError) as exc_info: - resolve_trace_target( + await resolve_trace_target( root_pid=proc.pid, expected_basename="definitely_not_there", timeout=0.5, diff --git a/tests/execution/test_write_evidence.py b/tests/execution/test_write_evidence.py new file mode 100644 index 0000000000..9fa7992979 --- /dev/null +++ b/tests/execution/test_write_evidence.py @@ -0,0 +1,311 @@ +"""Write evidence: multi-directory fs snapshot and write_watch_dirs plumbing. + +Tests for Part B of write-detection architectural immunity: +- Multi-dir filesystem snapshot via write_watch_dirs +- output_dir → write_watch_dirs plumbing in run_skill +- HeadlessExecutor protocol includes write_watch_dirs +- Planner skill end-to-end write detection +""" + +from __future__ import annotations + +import inspect +import json +import os +from pathlib import Path + +import pytest + +from autoskillit.core import HeadlessExecutor, WriteBehaviorSpec +from tests.conftest import _make_result + +pytestmark = [pytest.mark.layer("execution"), pytest.mark.small] + + +class TestMultiDirFsSnapshot: + """write_watch_dirs enables multi-directory filesystem snapshot.""" + + def test_fs_snapshot_watches_multiple_dirs(self, tmp_path: Path) -> None: + """When write_watch_dirs contains multiple paths, fs_writes_detected + is True if ANY directory has new files.""" + dir_a = tmp_path / "dir_a" + dir_b = tmp_path / "dir_b" + dir_a.mkdir() + dir_b.mkdir() + + pre_a = {e.name for e in os.scandir(dir_a)} + pre_b = {e.name for e in os.scandir(dir_b)} + assert pre_a == set() + assert pre_b == set() + + (dir_b / "output.json").write_text("{}") + + post_a = {e.name for e in os.scandir(dir_a)} + post_b = {e.name for e in os.scandir(dir_b)} + + fs_writes_detected = any( + bool(post - pre) for post, pre in [(post_a, pre_a), (post_b, pre_b)] + ) + assert fs_writes_detected is True + + @pytest.mark.anyio + async def test_fs_snapshot_watches_explicit_dir_not_skill_name( + self, tmp_path: Path, minimal_ctx, monkeypatch + ) -> None: + """When write_watch_dirs is provided, _resolve_skill_temp_dir is NOT called.""" + import autoskillit.execution.headless as headless_mod + from autoskillit.execution.headless import run_headless_core + + resolver_calls: list[str] = [] + original = headless_mod._resolve_skill_temp_dir + + def recording_resolver(cwd: str, skill_command: str) -> Path | None: + resolver_calls.append(skill_command) + return original(cwd, skill_command) + + monkeypatch.setattr(headless_mod, "_resolve_skill_temp_dir", recording_resolver) + + explicit_dir = tmp_path / "planner" / "run-20260502" + explicit_dir.mkdir(parents=True) + + async def mock_runner(cmd, **kwargs): + return _make_result() + + minimal_ctx.runner = mock_runner + proj = tmp_path / "proj" + proj.mkdir() + + await run_headless_core( + "/autoskillit:planner-refine-phases arg", + str(proj), + minimal_ctx, + write_watch_dirs=[explicit_dir], + ) + + assert resolver_calls == [], ( + "_resolve_skill_temp_dir must not be called when write_watch_dirs is provided" + ) + + +class TestHeadlessExecutorProtocol: + """HeadlessExecutor protocol includes write_watch_dirs.""" + + def test_headless_executor_protocol_has_write_watch_dirs(self) -> None: + """HeadlessExecutor.run() accepts write_watch_dirs parameter.""" + sig = inspect.signature(HeadlessExecutor.run) + assert "write_watch_dirs" in sig.parameters + + +class TestUnifiedSkillNameResolution: + """resolve_skill_name in core handles both /name and /autoskillit:name forms.""" + + @pytest.mark.parametrize( + ("command", "expected"), + [ + ("/autoskillit:make-plan arg1", "make-plan"), + ("/make-plan arg1", "make-plan"), + ("/autoskillit:planner-refine-phases ...", "planner-refine-phases"), + ( + "/autoskillit:exp-lens-{slug}", + None, + ), # regex stops at '{'; remainder.startswith("{") → None + ( + "/autoskillit:foo-${{ var }}", + None, + ), # regex stops at '$'; remainder.startswith("${{") → None + ("not a skill command", None), + ], + ) + def test_resolve_skill_name_handles_both_forms( + self, command: str, expected: str | None + ) -> None: + from autoskillit.core import resolve_skill_name + + assert resolve_skill_name(command) == expected + + def test_no_duplicate_skill_name_regexes(self) -> None: + """No module in recipe/ defines its own _SKILL_NAME_RE regex — all use core.""" + import ast + + from autoskillit.core import pkg_root + + recipe_dir = pkg_root() / "recipe" + py_files = sorted(recipe_dir.glob("*.py")) + assert len(py_files) > 0, ( + f"No .py files found in {recipe_dir} — pkg_root() may have resolved incorrectly" + ) + violations: list[str] = [] + for py_file in py_files: + try: + tree = ast.parse(py_file.read_text()) + except SyntaxError: + continue + for node in ast.walk(tree): + if ( + isinstance(node, ast.Assign) + and len(node.targets) == 1 + and isinstance(node.targets[0], ast.Name) + and node.targets[0].id == "_SKILL_NAME_RE" + ): + violations.append(py_file.name) + assert violations == [], ( + f"_SKILL_NAME_RE defined in recipe/ modules {violations} — " + "all should import resolve_skill_name from core" + ) + + +class TestBashFilePathEnrichment: + """parse_session_result extracts absolute paths from Bash tool_use commands.""" + + def test_bash_tool_use_has_bash_paths(self) -> None: + from autoskillit.execution.session import parse_session_result + + bash_block = { + "type": "tool_use", + "name": "Bash", + "id": "tu_1", + "input": {"command": "cat /home/user/project/file.txt && ls /tmp/output/"}, + } + assistant = { + "type": "assistant", + "message": {"content": [bash_block]}, + } + result_record = { + "type": "result", + "subtype": "success", + "is_error": False, + "result": "done", + "session_id": "test-sess", + } + stdout = json.dumps(assistant) + "\n" + json.dumps(result_record) + session = parse_session_result(stdout) + assert len(session.tool_uses) == 1 + entry = session.tool_uses[0] + assert "bash_paths" in entry + paths = entry["bash_paths"] + assert "/home/user/project/file.txt" in paths + assert "/tmp/output/" in paths + + +class TestRecursiveSnapshot: + """Recursive snapshot detects writes in pre-existing subdirectories.""" + + def test_detects_write_in_preexisting_subdir(self, tmp_path: Path) -> None: + from autoskillit.execution.headless import _recursive_snapshot + + watch_dir = tmp_path / "planner_run" + sub_dir = watch_dir / "refine_contexts" + sub_dir.mkdir(parents=True) + (sub_dir / "context_P1.json").write_text("{}") + + pre = _recursive_snapshot(watch_dir) + + (sub_dir / "P1_result.json").write_text('{"assignments": []}') + + post = _recursive_snapshot(watch_dir) + + assert post - pre == {"refine_contexts/P1_result.json"} + + def test_detects_write_in_nested_subdir(self, tmp_path: Path) -> None: + from autoskillit.execution.headless import _recursive_snapshot + + watch_dir = tmp_path / "planner_run" + nested = watch_dir / "work_packages" / "wp_sentinels" + nested.mkdir(parents=True) + + pre = _recursive_snapshot(watch_dir) + + (nested / "P1_result.json").write_text("{}") + + post = _recursive_snapshot(watch_dir) + + assert post - pre == {"work_packages/wp_sentinels/P1_result.json"} + + @pytest.mark.anyio + async def test_run_headless_core_detects_subdir_write( + self, tmp_path: Path, minimal_ctx + ) -> None: + from autoskillit.execution.headless import run_headless_core + from tests.execution.conftest import _success_session_json + + watch_dir = tmp_path / "output" + sub_dir = watch_dir / "results" + sub_dir.mkdir(parents=True) + (sub_dir / "existing.json").write_text("{}") + + async def mock_runner(cmd, **kwargs): + (sub_dir / "new_output.json").write_text('{"data": true}') + return _make_result(returncode=0, stdout=_success_session_json("done")) + + minimal_ctx.runner = mock_runner + proj = tmp_path / "proj" + proj.mkdir() + + result = await run_headless_core( + "/autoskillit:test-skill", + str(proj), + minimal_ctx, + write_watch_dirs=[watch_dir], + ) + assert result.fs_writes_detected is True + + @pytest.mark.anyio + async def test_run_headless_core_empty_subdir_no_false_positive( + self, tmp_path: Path, minimal_ctx + ) -> None: + from autoskillit.execution.headless import run_headless_core + from tests.execution.conftest import _success_session_json + + watch_dir = tmp_path / "output" + sub_dir = watch_dir / "results" + sub_dir.mkdir(parents=True) + (sub_dir / "existing.json").write_text("{}") + + async def mock_runner(cmd, **kwargs): + return _make_result(returncode=0, stdout=_success_session_json("done")) + + minimal_ctx.runner = mock_runner + proj = tmp_path / "proj" + proj.mkdir() + + result = await run_headless_core( + "/autoskillit:test-skill", + str(proj), + minimal_ctx, + write_watch_dirs=[watch_dir], + ) + assert result.fs_writes_detected is False + + +class TestPlannerSkillEndToEnd: + """Planner skill that writes via Bash to run dir detected via write_watch_dirs.""" + + @pytest.mark.anyio + async def test_planner_skill_bash_write_to_run_dir_detected( + self, tmp_path: Path, minimal_ctx + ) -> None: + """write_watch_dirs detection fires when the skill writes to run_dir during the session.""" + from autoskillit.execution.headless import run_headless_core + from tests.execution.conftest import _success_session_json + + run_dir = tmp_path / ".autoskillit" / "temp" / "planner" / "run-20260502" + run_dir.mkdir(parents=True) + # Pre-snapshot: run_dir is empty at session start + + async def mock_runner(cmd, **kwargs): + # Simulate the skill writing to run_dir during the session + (run_dir / "refined_plan.json").write_text("{}") + return _make_result(returncode=0, stdout=_success_session_json("done")) + + minimal_ctx.runner = mock_runner + proj = tmp_path / "proj" + proj.mkdir() + + sr = await run_headless_core( + "/autoskillit:planner-refine-phases arg", + str(proj), + minimal_ctx, + write_watch_dirs=[run_dir], + write_behavior=WriteBehaviorSpec(mode="always"), + ) + assert sr.fs_writes_detected is True diff --git a/tests/execution/test_write_evidence_invariants.py b/tests/execution/test_write_evidence_invariants.py new file mode 100644 index 0000000000..ae0dab7de1 --- /dev/null +++ b/tests/execution/test_write_evidence_invariants.py @@ -0,0 +1,48 @@ +"""Write-evidence invariants: 'no work done' retry reasons must be overridden by write evidence. + +Architectural immunity test: any RetryReason asserting 'no work done' must be reclassified +when write evidence contradicts the assertion. Adding a new 'no work' reason without a +corresponding gate causes this test to fail immediately. +""" + +from __future__ import annotations + +import pytest + +from autoskillit.core.types import ( + RetryReason, + SubprocessResult, + TerminationReason, +) +from autoskillit.execution.headless import _build_skill_result +from tests.execution.conftest import EMPTY_OUTPUT_RESULT_LINE, WRITE_TOOL_LINE + +pytestmark = [pytest.mark.layer("execution"), pytest.mark.small] + +# Reasons that carry a "no work done" semantic assertion. +# Any reason in this set must be reclassified when write evidence is present. +NO_WORK_REASONS = {RetryReason.EMPTY_OUTPUT} + +# Reasons that correctly reflect write-aware reclassification. +WRITE_AWARE_REASONS = {RetryReason.COMPLETED_NO_FLUSH} + + +@pytest.mark.parametrize("reason", sorted(NO_WORK_REASONS, key=lambda r: r.value)) +def test_no_work_reasons_are_overridden_by_write_evidence(reason: RetryReason) -> None: + stdout = "\n".join([WRITE_TOOL_LINE, EMPTY_OUTPUT_RESULT_LINE]) + result = SubprocessResult( + returncode=0, + stdout=stdout, + stderr="", + termination=TerminationReason.NATURAL_EXIT, + pid=12345, + ) + sr = _build_skill_result(result, fs_writes_detected=True) + + assert sr.retry_reason != reason, ( + f"RetryReason.{reason.name} was not overridden despite write evidence. " + f"Add a write-evidence reconciliation gate for this reason." + ) + assert sr.retry_reason in WRITE_AWARE_REASONS, ( + f"Expected reclassification to one of {WRITE_AWARE_REASONS!r}, got {sr.retry_reason!r}" + ) diff --git a/tests/execution/test_zero_write_detection.py b/tests/execution/test_zero_write_detection.py index 4f6120ae69..aa0dc18e20 100644 --- a/tests/execution/test_zero_write_detection.py +++ b/tests/execution/test_zero_write_detection.py @@ -8,11 +8,18 @@ from __future__ import annotations import json +from pathlib import Path + +import pytest from autoskillit.core import RetryReason, WriteBehaviorSpec, extract_skill_name from autoskillit.execution.headless import _build_skill_result from tests.conftest import _make_result +pytestmark = [pytest.mark.layer("execution"), pytest.mark.small] + +FIXTURE_CONDITIONAL_PATTERN = r"verdict\s*=\s*real_fix" + def _ndjson_with_tool_uses(tool_names: list[str]) -> str: """Build NDJSON stdout with assistant tool_use blocks and a success result.""" @@ -317,10 +324,7 @@ def test_conditional_no_pr_found_not_demoted(self) -> None: assert sr.subtype != "zero_writes" def test_resolve_claims_review_no_fixes_applied_not_demoted(self) -> None: - """resolve-claims-review: 'Fixes applied: 0' → gate inactive → success preserved. - - The pattern [1-9][0-9]* excludes zero, so zero-fix runs are not demoted. - """ + """resolve-claims-review: no 'verdict = real_fix' token → gate inactive → success.""" result_record = { "type": "result", "subtype": "success", @@ -334,11 +338,11 @@ def test_resolve_claims_review_no_fixes_applied_not_demoted(self) -> None: skill_command="/autoskillit:resolve-claims-review /tmp/wt main", write_behavior=WriteBehaviorSpec( mode="conditional", - expected_when=(r"Fixes applied:\s*[1-9][0-9]*",), + expected_when=(FIXTURE_CONDITIONAL_PATTERN,), ), ) assert sr.success is True, ( - "'Fixes applied: 0' must NOT be demoted — pattern [1-9][0-9]* excludes zero." + "No 'verdict = real_fix' token must NOT trigger zero_writes gate." ) assert sr.subtype != "zero_writes" @@ -346,13 +350,13 @@ def test_resolve_claims_review_all_escalations_not_demoted(self) -> None: """resolve-claims-review: all-escalations path → success preserved. When all accepted findings are rerun_required/design_flaw, no code is written - and 'Fixes applied: 0' is emitted. The gate must not fire. + and 'verdict = already_green' is emitted. The gate must not fire. """ result_record = { "type": "result", "subtype": "success", "is_error": False, - "result": "Fixes applied: 0\nneeds_rerun = true\n%%ORDER_UP%%", + "result": "verdict = already_green\nfixes_applied = 0\nneeds_rerun = true\n%%ORDER_UP%%", # noqa: E501 "session_id": "test-sess", } stdout = json.dumps(result_record) @@ -361,18 +365,18 @@ def test_resolve_claims_review_all_escalations_not_demoted(self) -> None: skill_command="/autoskillit:resolve-claims-review /tmp/wt main", write_behavior=WriteBehaviorSpec( mode="conditional", - expected_when=(r"Fixes applied:\s*[1-9][0-9]*",), + expected_when=(FIXTURE_CONDITIONAL_PATTERN,), ), ) assert sr.success is True, ( - "All-escalations path (Fixes applied: 0, needs_rerun=true) must NOT be demoted." + "All-escalations path (verdict = already_green) must NOT be demoted." ) assert sr.subtype != "zero_writes" def test_resolve_claims_review_graceful_degradation_not_demoted(self) -> None: """resolve-claims-review: no-PR graceful degradation → success preserved. - When no PR is found, the skill exits without a 'Fixes applied' line. + When no PR is found, the skill exits without a 'verdict = real_fix' token. No pattern match → gate inactive → success preserved. """ result_record = { @@ -388,25 +392,25 @@ def test_resolve_claims_review_graceful_degradation_not_demoted(self) -> None: skill_command="/autoskillit:resolve-claims-review /tmp/wt main", write_behavior=WriteBehaviorSpec( mode="conditional", - expected_when=(r"Fixes applied:\s*[1-9][0-9]*",), + expected_when=(FIXTURE_CONDITIONAL_PATTERN,), ), ) assert sr.success is True, ( - "Graceful degradation (no 'Fixes applied' line) must NOT be demoted." + "Graceful degradation (no 'verdict = real_fix' token) must NOT be demoted." ) assert sr.subtype != "zero_writes" def test_resolve_claims_review_fixes_applied_zero_writes_demoted(self) -> None: - """resolve-claims-review: 'Fixes applied: 3' + 0 writes → demoted to zero_writes. + """resolve-claims-review: 'verdict = real_fix' + 0 writes → demoted to zero_writes. - When the skill claims fixes were applied but produced no Edit/Write calls, + When the skill emits verdict = real_fix but produced no Edit/Write calls, this indicates silent degradation. The gate must fire. """ result_record = { "type": "result", "subtype": "success", "is_error": False, - "result": "Fixes applied: 3\nneeds_rerun = false\n%%ORDER_UP%%", + "result": "verdict = real_fix\nfixes_applied = 3\n%%ORDER_UP%%", "session_id": "test-sess", } stdout = json.dumps(result_record) @@ -415,17 +419,17 @@ def test_resolve_claims_review_fixes_applied_zero_writes_demoted(self) -> None: skill_command="/autoskillit:resolve-claims-review /tmp/wt main", write_behavior=WriteBehaviorSpec( mode="conditional", - expected_when=(r"Fixes applied:\s*[1-9][0-9]*",), + expected_when=(FIXTURE_CONDITIONAL_PATTERN,), ), ) assert sr.success is False, ( - "'Fixes applied: 3' with 0 writes must be demoted — silent degradation detected." + "'verdict = real_fix' with 0 writes must be demoted — silent degradation detected." ) assert sr.subtype == "zero_writes" assert sr.retry_reason == RetryReason.ZERO_WRITES def test_resolve_claims_review_fixes_applied_with_writes_passes(self) -> None: - """resolve-claims-review: 'Fixes applied: 3' + 3 writes → success preserved. + """resolve-claims-review: 'verdict = real_fix' + 3 writes → success preserved. The gate short-circuits at write_call_count > 0 before evaluating result text. """ @@ -435,19 +439,19 @@ def test_resolve_claims_review_fixes_applied_with_writes_passes(self) -> None: skill_command="/autoskillit:resolve-claims-review /tmp/wt main", write_behavior=WriteBehaviorSpec( mode="conditional", - expected_when=(r"Fixes applied:\s*[1-9][0-9]*",), + expected_when=(FIXTURE_CONDITIONAL_PATTERN,), ), ) - assert sr.success is True, "Fixes applied with actual writes must NOT be demoted." + assert sr.success is True, "Real fix with actual writes must NOT be demoted." assert sr.subtype != "zero_writes" def test_resolve_research_review_no_fixes_applied_not_demoted(self) -> None: - """resolve-research-review: 'Fixes applied: 0' → gate inactive → success preserved.""" + """resolve-research-review: no 'verdict = real_fix' token → gate inactive → success.""" result_record = { "type": "result", "subtype": "success", "is_error": False, - "result": "Fixes applied: 0\nneeds_rerun = false\n%%ORDER_UP%%", + "result": "verdict = already_green\nfixes_applied = 0\nneeds_rerun = false\n%%ORDER_UP%%", # noqa: E501 "session_id": "test-sess", } stdout = json.dumps(result_record) @@ -456,11 +460,11 @@ def test_resolve_research_review_no_fixes_applied_not_demoted(self) -> None: skill_command="/autoskillit:resolve-research-review /tmp/wt main", write_behavior=WriteBehaviorSpec( mode="conditional", - expected_when=(r"Fixes applied:\s*[1-9][0-9]*",), + expected_when=(FIXTURE_CONDITIONAL_PATTERN,), ), ) assert sr.success is True, ( - "'Fixes applied: 0' must NOT be demoted — pattern [1-9][0-9]* excludes zero." + "No 'verdict = real_fix' token must NOT trigger zero_writes gate." ) assert sr.subtype != "zero_writes" @@ -470,7 +474,7 @@ def test_resolve_research_review_all_escalations_not_demoted(self) -> None: "type": "result", "subtype": "success", "is_error": False, - "result": "Fixes applied: 0\nneeds_rerun = true\n%%ORDER_UP%%", + "result": "verdict = already_green\nfixes_applied = 0\nneeds_rerun = true\n%%ORDER_UP%%", # noqa: E501 "session_id": "test-sess", } stdout = json.dumps(result_record) @@ -479,11 +483,11 @@ def test_resolve_research_review_all_escalations_not_demoted(self) -> None: skill_command="/autoskillit:resolve-research-review /tmp/wt main", write_behavior=WriteBehaviorSpec( mode="conditional", - expected_when=(r"Fixes applied:\s*[1-9][0-9]*",), + expected_when=(FIXTURE_CONDITIONAL_PATTERN,), ), ) assert sr.success is True, ( - "All-escalations path (Fixes applied: 0, needs_rerun=true) must NOT be demoted." + "All-escalations path (verdict = already_green) must NOT be demoted." ) assert sr.subtype != "zero_writes" @@ -502,21 +506,21 @@ def test_resolve_research_review_graceful_degradation_not_demoted(self) -> None: skill_command="/autoskillit:resolve-research-review /tmp/wt main", write_behavior=WriteBehaviorSpec( mode="conditional", - expected_when=(r"Fixes applied:\s*[1-9][0-9]*",), + expected_when=(FIXTURE_CONDITIONAL_PATTERN,), ), ) assert sr.success is True, ( - "Graceful degradation (no 'Fixes applied' line) must NOT be demoted." + "Graceful degradation (no 'verdict = real_fix' token) must NOT be demoted." ) assert sr.subtype != "zero_writes" def test_resolve_research_review_fixes_applied_zero_writes_demoted(self) -> None: - """resolve-research-review: 'Fixes applied: 3' + 0 writes → demoted to zero_writes.""" + """resolve-research-review: 'verdict = real_fix' + 0 writes → demoted to zero_writes.""" result_record = { "type": "result", "subtype": "success", "is_error": False, - "result": "Fixes applied: 3\nneeds_rerun = false\n%%ORDER_UP%%", + "result": "verdict = real_fix\nfixes_applied = 3\n%%ORDER_UP%%", "session_id": "test-sess", } stdout = json.dumps(result_record) @@ -525,27 +529,27 @@ def test_resolve_research_review_fixes_applied_zero_writes_demoted(self) -> None skill_command="/autoskillit:resolve-research-review /tmp/wt main", write_behavior=WriteBehaviorSpec( mode="conditional", - expected_when=(r"Fixes applied:\s*[1-9][0-9]*",), + expected_when=(FIXTURE_CONDITIONAL_PATTERN,), ), ) assert sr.success is False, ( - "'Fixes applied: 3' with 0 writes must be demoted — silent degradation detected." + "'verdict = real_fix' with 0 writes must be demoted — silent degradation detected." ) assert sr.subtype == "zero_writes" assert sr.retry_reason == RetryReason.ZERO_WRITES def test_resolve_research_review_fixes_applied_with_writes_passes(self) -> None: - """resolve-research-review: 'Fixes applied: 3' + 3 writes → success preserved.""" + """resolve-research-review: 'verdict = real_fix' + 3 writes → success preserved.""" stdout = _ndjson_with_tool_uses(["Edit", "Edit", "Edit"]) sr = _build_skill_result( _make_result(returncode=0, stdout=stdout), skill_command="/autoskillit:resolve-research-review /tmp/wt main", write_behavior=WriteBehaviorSpec( mode="conditional", - expected_when=(r"Fixes applied:\s*[1-9][0-9]*",), + expected_when=(FIXTURE_CONDITIONAL_PATTERN,), ), ) - assert sr.success is True, "Fixes applied with actual writes must NOT be demoted." + assert sr.success is True, "Real fix with actual writes must NOT be demoted." assert sr.subtype != "zero_writes" @@ -592,3 +596,164 @@ def test_no_slash_returns_none(self) -> None: def test_leading_whitespace(self) -> None: assert extract_skill_name(" /investigate error") == "investigate" + + +class TestFilesystemWriteDetection: + """Filesystem-level write detection suppresses false positives.""" + + def test_bash_heredoc_write_not_counted_by_mcp(self) -> None: + """Bash tool_use blocks are not counted by write_call_count.""" + stdout = _ndjson_with_tool_uses(["Bash"]) + sr = _build_skill_result( + _make_result(returncode=0, stdout=stdout), + skill_command="/autoskillit:prepare-pr args", + write_behavior=WriteBehaviorSpec(mode="always"), + ) + assert sr.write_call_count == 0 + + def test_fs_writes_detected_suppresses_zero_writes_gate(self) -> None: + """When fs_writes_detected=True, zero_writes gate must NOT fire + even when write_call_count == 0.""" + stdout = _ndjson_with_tool_uses(["Bash"]) + sr = _build_skill_result( + _make_result(returncode=0, stdout=stdout), + skill_command="/autoskillit:prepare-pr args", + write_behavior=WriteBehaviorSpec(mode="always"), + fs_writes_detected=True, + ) + assert sr.success is True + assert sr.subtype != "zero_writes" + assert sr.fs_writes_detected is True + + def test_fs_writes_detected_enables_contract_recovery(self) -> None: + """When fs_writes_detected=True, CONTRACT_RECOVERY gate must fire + for adjudicated_failure even when write_call_count == 0.""" + result_record = { + "type": "result", + "subtype": "success", + "is_error": False, + "result": "done", + "session_id": "test-sess", + } + stdout = json.dumps(result_record) + sr_without_fs = _build_skill_result( + _make_result(returncode=0, stdout=stdout), + skill_command="/autoskillit:prepare-pr args", + write_behavior=WriteBehaviorSpec(mode="always"), + expected_output_patterns=["prep_path\\s*=\\s*/.+"], + fs_writes_detected=False, + ) + assert sr_without_fs.subtype == "adjudicated_failure" + assert sr_without_fs.needs_retry is False + + sr_with_fs = _build_skill_result( + _make_result(returncode=0, stdout=stdout), + skill_command="/autoskillit:prepare-pr args", + write_behavior=WriteBehaviorSpec(mode="always"), + expected_output_patterns=["prep_path\\s*=\\s*/.+"], + fs_writes_detected=True, + ) + assert sr_with_fs.needs_retry is True + assert sr_with_fs.retry_reason == RetryReason.CONTRACT_RECOVERY + + def test_fs_writes_false_no_suppression(self) -> None: + """When fs_writes_detected=False and write_call_count == 0, + zero_writes gate must fire as before.""" + stdout = _ndjson_with_tool_uses(["Read", "Grep"]) + sr = _build_skill_result( + _make_result(returncode=0, stdout=stdout), + skill_command="/autoskillit:make-plan task", + write_behavior=WriteBehaviorSpec(mode="always"), + fs_writes_detected=False, + ) + assert sr.success is False + assert sr.subtype == "zero_writes" + + def test_fs_writes_detected_in_to_json_included(self) -> None: + """fs_writes_detected must appear in to_json() output.""" + stdout = _ndjson_with_tool_uses(["Edit"]) + sr = _build_skill_result( + _make_result(returncode=0, stdout=stdout), + skill_command="/autoskillit:make-plan task", + write_behavior=WriteBehaviorSpec(mode="always"), + fs_writes_detected=True, + ) + json_data = json.loads(sr.to_json()) + assert json_data["fs_writes_detected"] is True + + +class TestTempDirSnapshot: + """AUTOSKILLIT_TEMP snapshot detects Bash-created files.""" + + def test_snapshot_detects_new_file(self, tmp_path: Path) -> None: + """A file created between pre and post snapshot is detected.""" + from autoskillit.execution.headless import _recursive_snapshot + + temp_dir = tmp_path / ".autoskillit" / "temp" / "prepare-pr" + temp_dir.mkdir(parents=True) + pre = _recursive_snapshot(temp_dir) + (temp_dir / "pr_prep_20260425.md").write_text("content") + post = _recursive_snapshot(temp_dir) + assert len(post - pre) > 0 + + def test_snapshot_empty_when_no_new_files(self, tmp_path: Path) -> None: + """No new files between snapshots yields empty diff.""" + from autoskillit.execution.headless import _recursive_snapshot + + temp_dir = tmp_path / ".autoskillit" / "temp" / "prepare-pr" + temp_dir.mkdir(parents=True) + (temp_dir / "existing.md").write_text("content") + pre = _recursive_snapshot(temp_dir) + post = _recursive_snapshot(temp_dir) + assert post - pre == set() + + +class TestWriteExpectedWhenPatternSync: + """write_expected_when patterns in tests must match skill_contracts.yaml.""" + + @pytest.mark.parametrize( + "skill_name", + [ + "resolve-review", + "resolve-claims-review", + "resolve-research-review", + "resolve-failures", + ], + ) + def test_conditional_pattern_matches_production(self, skill_name: str) -> None: + """FIXTURE_CONDITIONAL_PATTERN must equal skill_contracts.yaml value.""" + from autoskillit.recipe.contracts import get_skill_contract, load_bundled_manifest + + manifest = load_bundled_manifest() + contract = get_skill_contract(skill_name, manifest) + assert contract is not None, f"{skill_name} not in manifest" + assert contract.write_behavior == "conditional", ( + f"{skill_name} expected conditional, got {contract.write_behavior}" + ) + assert contract.write_expected_when, ( + f"{skill_name} is conditional but has empty write_expected_when" + ) + assert contract.write_expected_when[0] == FIXTURE_CONDITIONAL_PATTERN, ( + f"{skill_name}: production pattern is {contract.write_expected_when[0]!r} " + f"but FIXTURE_CONDITIONAL_PATTERN is {FIXTURE_CONDITIONAL_PATTERN!r}. " + f"Update the constant and all WriteBehaviorSpec(expected_when=...) calls." + ) + + +class TestHelperDiversityGuard: + """Ensure test helper covers all write-relevant tool names.""" + + def test_ndjson_helper_includes_bash_scenario(self) -> None: + """At least one test in this module must use Bash tool_use blocks.""" + import ast + import inspect + + source = inspect.getsource(TestFilesystemWriteDetection) + tree = ast.parse(source) + bash_found = any( + isinstance(node, ast.Constant) and node.value == "Bash" for node in ast.walk(tree) + ) + assert bash_found, ( + "TestFilesystemWriteDetection must include at least one test " + "with 'Bash' tool_use to prevent the MCP-only counting blindspot" + ) diff --git a/tests/fakes.py b/tests/fakes.py new file mode 100644 index 0000000000..a24045a8af --- /dev/null +++ b/tests/fakes.py @@ -0,0 +1,692 @@ +"""Protocol-based test fakes for autoskillit. + +Single authoritative module for in-memory test doubles that satisfy protocols +defined in ``core/_type_protocols_*.py`` shards. Imports only from L0 (``autoskillit.core``). +""" + +from __future__ import annotations + +import asyncio +import dataclasses +from collections import deque +from collections.abc import Awaitable, Callable, Mapping, Sequence +from pathlib import Path +from typing import Any + +from autoskillit.core.types import ( + CIRunScope, + CIWatcher, + DatabaseReader, + HeadlessExecutor, + MergeQueueWatcher, + RecipeRepository, + SessionCheckpoint, # noqa: F401, TC001 + SkillResult, + SubprocessResult, + SubprocessRunner, + TerminationReason, + TestResult, + TestRunner, + WriteBehaviorSpec, +) + +# --------------------------------------------------------------------------- +# Shared side-effect resolution helper +# --------------------------------------------------------------------------- + + +def _resolve_side_effect(effect: Any) -> Any: + """Resolve a side-effect value: raise exceptions, call callables, or return as-is. + + Handles async callables by running them on the current event loop. + """ + if isinstance(effect, BaseException): + raise effect + if isinstance(effect, type) and issubclass(effect, BaseException): + raise effect() + if callable(effect): + result = effect() + if asyncio.iscoroutine(result): + return asyncio.get_event_loop().run_until_complete(result) + return result + return effect + + +# --------------------------------------------------------------------------- +# HeadlessExecutor fake +# --------------------------------------------------------------------------- + + +@dataclasses.dataclass +class ExecutorCall: + """Record of a single ``HeadlessExecutor.run()`` invocation.""" + + skill_command: str + cwd: str + model: str = "" + step_name: str = "" + kitchen_id: str = "" + order_id: str = "" + add_dirs: tuple[Any, ...] = () + timeout: float | None = None + stale_threshold: float | None = None + idle_output_timeout: float | None = None + expected_output_patterns: tuple[str, ...] = () + write_behavior: Any | None = None + completion_marker: str = "" + recipe_name: str = "" + recipe_content_hash: str = "" + recipe_composite_hash: str = "" + recipe_version: str | None = None + allowed_write_prefix: str = "" + readonly_skill: bool = False + write_watch_dirs: tuple[Any, ...] = () + provider_extras: Mapping[str, str] | None = None + profile_name: str = "" + resume_session_id: str = "" + resume_checkpoint: SessionCheckpoint | None = None + + +@dataclasses.dataclass +class DispatchFoodTruckCall: + """Record of a single ``HeadlessExecutor.dispatch_food_truck()`` invocation.""" + + orchestrator_prompt: str + cwd: str + completion_marker: str = "" + resume_session_id: str | None = None + resume_checkpoint: SessionCheckpoint | None = None + model: str = "" + step_name: str = "" + kitchen_id: str = "" + order_id: str = "" + campaign_id: str = "" + dispatch_id: str = "" + caller_session_id: str = "" + project_dir: str = "" + timeout: float | None = None + stale_threshold: float | None = None + idle_output_timeout: float | None = None + env_extras: Mapping[str, str] | None = None + requires_packs: Sequence[str] = () + on_spawn: Callable[[int, int], None] | None = None + allowed_write_prefix: str = "" + + +_DEFAULT_SKILL_RESULT = SkillResult( + success=True, + result="ok", + session_id="", + subtype="success", + is_error=False, + exit_code=0, + needs_retry=False, + retry_reason="none", + stderr="", + token_usage=None, +) + + +class InMemoryHeadlessExecutor(HeadlessExecutor): + """In-memory test double for :class:`HeadlessExecutor`. + + Supports a FIFO queue via :meth:`push` and records every call in + :attr:`calls`. + """ + + def __init__(self, default_result: SkillResult | None = None) -> None: + self._default = default_result or _DEFAULT_SKILL_RESULT + self._queue: deque[SkillResult] = deque() + self.calls: list[ExecutorCall] = [] + self.dispatch_calls: list[DispatchFoodTruckCall] = [] + + def push(self, result: SkillResult) -> None: + """Enqueue a result to be returned by the next :meth:`run` call.""" + self._queue.append(result) + + async def run( + self, + skill_command: str, + cwd: str, + *, + model: str = "", + step_name: str = "", + kitchen_id: str = "", + order_id: str = "", + add_dirs: Sequence[Any] = (), + timeout: float | None = None, + stale_threshold: float | None = None, + idle_output_timeout: float | None = None, + expected_output_patterns: Sequence[str] = (), + write_behavior: WriteBehaviorSpec | None = None, + completion_marker: str = "", + recipe_name: str = "", + recipe_content_hash: str = "", + recipe_composite_hash: str = "", + recipe_version: str | None = None, + allowed_write_prefix: str = "", + readonly_skill: bool = False, + write_watch_dirs: Sequence[Any] = (), + provider_extras: Mapping[str, str] | None = None, + profile_name: str = "", + resume_session_id: str = "", + resume_checkpoint: SessionCheckpoint | None = None, + ) -> SkillResult: + self.calls.append( + ExecutorCall( + skill_command=skill_command, + cwd=cwd, + model=model, + step_name=step_name, + kitchen_id=kitchen_id, + order_id=order_id, + add_dirs=tuple(add_dirs), + timeout=timeout, + stale_threshold=stale_threshold, + idle_output_timeout=idle_output_timeout, + expected_output_patterns=tuple(expected_output_patterns), + write_behavior=write_behavior, + completion_marker=completion_marker, + recipe_name=recipe_name, + recipe_content_hash=recipe_content_hash, + recipe_composite_hash=recipe_composite_hash, + recipe_version=recipe_version, + allowed_write_prefix=allowed_write_prefix, + readonly_skill=readonly_skill, + write_watch_dirs=tuple(write_watch_dirs), + provider_extras=provider_extras, + profile_name=profile_name, + resume_session_id=resume_session_id, + resume_checkpoint=resume_checkpoint, + ) + ) + if self._queue: + return dataclasses.replace(self._queue.popleft()) + # Return a defensive copy so callers mutating fields (e.g. run_skill + # setting order_id) don't pollute the shared default across tests. + return dataclasses.replace(self._default) + + async def dispatch_food_truck( + self, + orchestrator_prompt: str, + cwd: str, + *, + completion_marker: str, + resume_session_id: str | None = None, + resume_checkpoint: SessionCheckpoint | None = None, + model: str = "", + step_name: str = "", + kitchen_id: str = "", + order_id: str = "", + campaign_id: str = "", + dispatch_id: str = "", + caller_session_id: str = "", + project_dir: str = "", + timeout: float | None = None, + stale_threshold: float | None = None, + idle_output_timeout: float | None = None, + env_extras: Mapping[str, str] | None = None, + requires_packs: Sequence[str] = (), + on_spawn: Callable[[int, int], None] | None = None, + allowed_write_prefix: str = "", + ) -> SkillResult: + self.dispatch_calls.append( + DispatchFoodTruckCall( + orchestrator_prompt=orchestrator_prompt, + cwd=cwd, + completion_marker=completion_marker, + resume_session_id=resume_session_id, + resume_checkpoint=resume_checkpoint, + model=model, + step_name=step_name, + kitchen_id=kitchen_id, + order_id=order_id, + campaign_id=campaign_id, + dispatch_id=dispatch_id, + caller_session_id=caller_session_id, + project_dir=project_dir, + timeout=timeout, + stale_threshold=stale_threshold, + idle_output_timeout=idle_output_timeout, + env_extras=env_extras, + requires_packs=requires_packs, + on_spawn=on_spawn, + allowed_write_prefix=allowed_write_prefix, + ) + ) + if self._queue: + return dataclasses.replace(self._queue.popleft()) + return dataclasses.replace(self._default) + + +# --------------------------------------------------------------------------- +# TestRunner fake +# --------------------------------------------------------------------------- + + +class InMemoryTestRunner(TestRunner): + """In-memory test double for :class:`TestRunner`. + + Pops pre-configured results from a deque; falls back to a passing + ``TestResult`` when the deque is exhausted. + """ + + def __init__(self, results: list[TestResult] | None = None) -> None: + self._results: deque[TestResult] = deque(results or []) + self._call_count = 0 + self.calls: list[Path] = [] + + async def run(self, cwd: Path) -> TestResult: + self._call_count += 1 + self.calls.append(cwd) + if self._results: + return self._results.popleft() + return TestResult(passed=True, stdout="", stderr="") + + @property + def call_count(self) -> int: + return self._call_count + + +# --------------------------------------------------------------------------- +# RecipeRepository fake +# --------------------------------------------------------------------------- + + +class InMemoryRecipeRepository(RecipeRepository): + """In-memory test double for :class:`RecipeRepository`.""" + + def __init__(self) -> None: + self._recipes: dict[str, Any] = {} + self._full_recipes: dict[Any, Any] = {} + self._validated: dict[str, dict[str, Any]] = {} + self._path_validated: dict[str, dict[str, Any]] = {} + self._all_recipes: dict[str, Any] = {} + self.calls: list[dict[str, Any]] = [] + + # -- test setup helpers -- + + def add_full_recipe(self, path: Any, recipe: Any) -> None: + self._full_recipes[path] = recipe + + def add_recipe(self, name: str, data: Any) -> None: + from autoskillit.recipe.schema import RecipeInfo # noqa: PLC0415 + + if not isinstance(data, RecipeInfo): + raise TypeError( + f"InMemoryRecipeRepository.add_recipe expects RecipeInfo, " + f"got {type(data).__name__}. " + "Production DefaultRecipeRepository.find() returns RecipeInfo, not Recipe." + ) + self._recipes[name] = data + + def set_validated(self, name: str, result: dict[str, Any]) -> None: + self._validated[name] = result + + def set_path_validated(self, path: str, result: dict[str, Any]) -> None: + self._path_validated[path] = result + + def set_all(self, data: dict[str, Any]) -> None: + self._all_recipes = data + + # -- protocol methods -- + + def find(self, name: str, project_dir: Path) -> Any: + self.calls.append({"method": "find", "name": name, "project_dir": project_dir}) + return self._recipes.get(name) + + def load(self, path: Path) -> Any: + self.calls.append({"method": "load", "path": path}) + if path in self._full_recipes: + return self._full_recipes[path] + raise FileNotFoundError( + f"No mocked full Recipe for path {path!r}. " + "Call add_full_recipe(path, recipe) in the test to configure it." + ) + + def list(self, project_dir: Path) -> Any: + self.calls.append({"method": "list", "project_dir": project_dir}) + return list(self._recipes.keys()) + + def load_and_validate( + self, + name: str, + project_dir: Any, + *, + suppressed: Sequence[str] | None = None, + resolved_defaults: dict[str, str] | None = None, + ingredient_overrides: dict[str, str] | None = None, + temp_dir: Path | None = None, + temp_dir_relpath: str | None = None, + ) -> dict[str, Any]: + self.calls.append( + { + "method": "load_and_validate", + "name": name, + "project_dir": project_dir, + "suppressed": suppressed, + "resolved_defaults": resolved_defaults, + "ingredient_overrides": ingredient_overrides, + "temp_dir": temp_dir, + "temp_dir_relpath": temp_dir_relpath, + } + ) + return self._validated.get(name, {"valid": False, "error": "not configured"}) + + def validate_from_path( + self, script_path: Any, temp_dir_relpath: str = ".autoskillit/temp" + ) -> dict[str, Any]: + self.calls.append( + { + "method": "validate_from_path", + "script_path": script_path, + "temp_dir_relpath": temp_dir_relpath, + } + ) + key = str(script_path) + return self._path_validated.get(key, {"valid": False, "error": "not configured"}) + + def list_all(self, project_dir: Any | None = None) -> dict[str, Any]: + self.calls.append({"method": "list_all", "project_dir": project_dir}) + return self._all_recipes + + async def apply_triage_gate( + self, + result: dict[str, Any], + recipe_name: str, + recipe_info: Any, + temp_dir: Path, + logger: Any, + triage_fn: Callable[..., Awaitable[Sequence[dict[str, Any]]]] | None = None, + ) -> dict[str, Any]: + return result + + +# --------------------------------------------------------------------------- +# CIWatcher fake +# --------------------------------------------------------------------------- + + +class InMemoryCIWatcher(CIWatcher): + """In-memory test double for :class:`CIWatcher`.""" + + def __init__( + self, + wait_result: dict[str, Any] | None = None, + status_result: dict[str, Any] | None = None, + wait_results: list[dict[str, Any]] | None = None, + ) -> None: + self._wait_result = wait_result or { + "run_id": 0, + "conclusion": "success", + "failed_jobs": [], + } + self._wait_results_queue: list[dict[str, Any]] = list(wait_results) if wait_results else [] + self._status_result = status_result or {"runs": []} + self.wait_calls: list[dict[str, Any]] = [] + self.status_calls: list[dict[str, Any]] = [] + self.wait_side_effect: Any | None = None + self.status_side_effect: Any | None = None + + async def wait( + self, + branch: str, + *, + repo: str | None = None, + scope: CIRunScope = CIRunScope(), + timeout_seconds: int = 300, + lookback_seconds: int = 120, + cwd: str = "", + ) -> dict[str, Any]: + self.wait_calls.append( + { + "branch": branch, + "repo": repo, + "scope": scope, + "timeout_seconds": timeout_seconds, + "lookback_seconds": lookback_seconds, + "cwd": cwd, + } + ) + if self.wait_side_effect is not None: + return _resolve_side_effect(self.wait_side_effect) + if self._wait_results_queue: + return self._wait_results_queue.pop(0) + return self._wait_result + + async def status( + self, + branch: str, + *, + repo: str | None = None, + run_id: int | None = None, + scope: CIRunScope = CIRunScope(), + cwd: str = "", + ) -> dict[str, Any]: + self.status_calls.append( + { + "branch": branch, + "repo": repo, + "run_id": run_id, + "scope": scope, + "cwd": cwd, + } + ) + if self.status_side_effect is not None: + return _resolve_side_effect(self.status_side_effect) + return self._status_result + + +# --------------------------------------------------------------------------- +# MergeQueueWatcher fake +# --------------------------------------------------------------------------- + + +class InMemoryMergeQueueWatcher(MergeQueueWatcher): + """In-memory test double for :class:`MergeQueueWatcher`.""" + + def __init__( + self, + wait_result: dict[str, Any] | None = None, + toggle_result: dict[str, Any] | None = None, + enqueue_result: dict[str, Any] | None = None, + ) -> None: + self._wait_result = wait_result or { + "success": True, + "pr_state": "merged", + "reason": "PR merged", + } + self._toggle_result = toggle_result or {"success": True, "toggled": True} + self._enqueue_result = enqueue_result or { + "success": True, + "pr_number": 0, + "enrollment_method": "auto_merge", + } + self.wait_calls: list[dict[str, Any]] = [] + self.toggle_calls: list[dict[str, Any]] = [] + self.enqueue_calls: list[dict[str, Any]] = [] + self.wait_side_effect: Any | None = None + self.toggle_side_effect: Any | None = None + self.enqueue_side_effect: Any | None = None + + async def wait( + self, + pr_number: int, + target_branch: str, + repo: str | None = None, + cwd: str = ".", + timeout_seconds: int = 600, + poll_interval: int = 15, + stall_grace_period: int = 60, + max_stall_retries: int = 3, + not_in_queue_confirmation_cycles: int = 2, + max_inconclusive_retries: int = 5, + auto_merge_available: bool = True, + ) -> dict[str, Any]: + self.wait_calls.append( + { + "pr_number": pr_number, + "target_branch": target_branch, + "repo": repo, + "cwd": cwd, + "timeout_seconds": timeout_seconds, + "poll_interval": poll_interval, + "auto_merge_available": auto_merge_available, + } + ) + if self.wait_side_effect is not None: + return _resolve_side_effect(self.wait_side_effect) + return self._wait_result + + async def toggle( + self, + pr_number: int, + target_branch: str, + repo: str | None = None, + cwd: str = ".", + ) -> dict[str, Any]: + self.toggle_calls.append( + { + "pr_number": pr_number, + "target_branch": target_branch, + "repo": repo, + "cwd": cwd, + } + ) + if self.toggle_side_effect is not None: + return _resolve_side_effect(self.toggle_side_effect) + return self._toggle_result + + async def enqueue( + self, + pr_number: int, + target_branch: str, + repo: str | None = None, + cwd: str = ".", + auto_merge_available: bool = True, + ) -> dict[str, Any]: + self.enqueue_calls.append( + { + "pr_number": pr_number, + "target_branch": target_branch, + "repo": repo, + "cwd": cwd, + "auto_merge_available": auto_merge_available, + } + ) + if self.enqueue_side_effect is not None: + return _resolve_side_effect(self.enqueue_side_effect) + return self._enqueue_result + + +# --------------------------------------------------------------------------- +# DatabaseReader fake +# --------------------------------------------------------------------------- + + +class InMemoryDatabaseReader(DatabaseReader): + """In-memory test double for :class:`DatabaseReader`.""" + + def __init__(self, query_result: dict[str, Any] | None = None) -> None: + self._query_result = query_result or { + "columns": [], + "rows": [], + "row_count": 0, + } + self.calls: list[dict[str, Any]] = [] + self.side_effect: Any | None = None + + def query( + self, + db_path: str, + sql: str, + params: list | dict, # type: ignore[type-arg] + timeout_sec: int, + max_rows: int, + ) -> dict[str, Any]: + self.calls.append( + { + "db_path": db_path, + "sql": sql, + "params": params, + "timeout_sec": timeout_sec, + "max_rows": max_rows, + } + ) + if self.side_effect is not None: + return _resolve_side_effect(self.side_effect) + return self._query_result + + +# --------------------------------------------------------------------------- +# SubprocessRunner fake (moved from conftest.py) +# --------------------------------------------------------------------------- + + +class InMemoryGitHubApiLog: + """In-memory fake for the GitHubApiLog protocol. Records calls for assertions.""" + + def __init__(self) -> None: + self.httpx_calls: list[dict[str, Any]] = [] + self.gh_cli_calls: list[dict[str, Any]] = [] + + async def record_httpx(self, **kwargs: Any) -> None: + self.httpx_calls.append(kwargs) + + async def record_gh_cli(self, **kwargs: Any) -> None: + self.gh_cli_calls.append(kwargs) + + def to_usage(self, session_id: str) -> dict[str, Any] | None: + total = len(self.httpx_calls) + len(self.gh_cli_calls) + if total == 0: + return None + return {"session_id": session_id, "total_requests": total} + + def clear(self) -> None: + self.httpx_calls.clear() + self.gh_cli_calls.clear() + + +class MockSubprocessRunner(SubprocessRunner): + """Test double for SubprocessRunner. Queues predetermined results. + + Inherits from SubprocessRunner (Protocol) so mypy verifies the __call__ + signature matches the protocol at class definition, not just at call sites. + + call_args_list stores (cmd, cwd, timeout, kwargs) tuples. + IMPORTANT: Assert [N][1] (cwd) when testing cwd propagation. + """ + + def __init__(self) -> None: + self._queue: deque[SubprocessResult] = deque() + self._default = SubprocessResult( + returncode=0, + stdout="", + stderr="", + termination=TerminationReason.NATURAL_EXIT, + pid=99999, + ) + self.call_args_list: list[tuple] = [] # type: ignore[type-arg] + + def push(self, result: SubprocessResult) -> None: + """Queue a result to be returned by the next __call__.""" + self._queue.append(result) + + def set_default(self, result: SubprocessResult) -> None: + """Set the result returned when the queue is empty.""" + self._default = result + + async def __call__( + self, + cmd: list[str], + *, + cwd: Path, + timeout: float, + **kwargs: object, + ) -> SubprocessResult: + self.call_args_list.append((cmd, cwd, timeout, kwargs)) + result = self._queue.popleft() if self._queue else self._default + on_pid_resolved = kwargs.get("on_pid_resolved") + if callable(on_pid_resolved) and result.pid > 0: + on_pid_resolved(result.pid, 0) + return result diff --git a/tests/fleet/CLAUDE.md b/tests/fleet/CLAUDE.md new file mode 100644 index 0000000000..1576a69e08 --- /dev/null +++ b/tests/fleet/CLAUDE.md @@ -0,0 +1,41 @@ +# fleet/ + +Fleet campaign dispatch, state persistence, and sidecar tests. + +## Files + +| File | Purpose | +|------|---------| +| `__init__.py` | empty | +| `_helpers.py` | Shared helpers for tests/fleet/ test modules | +| `conftest.py` | Shared fixtures for tests/fleet/ | +| `test_api.py` | Tests for fleet._api module (Group J) | +| `test_campaign_capture.py` | Tests for campaign capture extraction and ingredient interpolation (Group J) | +| `test_checkpoint_bridge.py` | Tests for checkpoint_from_sidecar converting IssueSidecarEntry to SessionCheckpoint | +| `test_dispatch_failure_semantics.py` | Group F: Timeout + No-Result-Block failure semantics for fleet dispatch | +| `test_dispatch_lifespan.py` | Group G (fleet part): lifespan_started surface + envelope propagation | +| `test_dispatch_outcome_classifier.py` | Tests for classify_dispatch_outcome() pure classification function | +| `test_error_envelope.py` | Tests for fleet error envelope registry and constructor (Group R) | +| `test_findings_rpc.py` | Tests for autoskillit.fleet._findings_rpc (T15–T21) | +| `test_fleet.py` | Tests for fleet package | +| `test_fleet_e2e.py` | Fleet Group O: end-to-end test suite for fleet dispatch loop | +| `test_fleet_rename_integrity.py` | Fleet rename integrity guard | +| `test_fleet_semaphore.py` | Unit tests for FleetSemaphore (FleetLock semaphore implementation) | +| `test_food_truck_prompt.py` | Tests for fleet/_prompts.py: _build_food_truck_prompt behavioral semantics | +| `test_gate_state_persistence.py` | Tests for gate dispatch state persistence and campaign state writes | +| `test_helpers_exports.py` | Tests that shared helpers are importable from tests.fleet._helpers | +| `test_liveness.py` | Liveness tests for Linux proc helpers | +| `test_pack_enforcement.py` | Fleet per-recipe tool-surface enforcement tests | +| `test_pack_enforcement_e2e.py` | Fleet per-recipe tool-surface e2e tests using a real MCP server subprocess | +| `test_research_campaign_dispatch.py` | Tests for multi-recipe research campaign dispatch capture propagation (Group J) | +| `test_result_parser.py` | Tests for fleet.result_parser — L2 result block parsing | +| `test_retry_failed_dispatch.py` | Tests for explicit retry of failed campaign dispatches via FAILURE → PENDING state transition | +| `test_sidecar.py` | Sidecar tests | +| `test_state.py` | Tests for fleet state module (Group J) | +| `test_state_protection.py` | Tests for fleet.state.build_protected_campaign_ids (PROT_1–PROT_9) | +| `test_state_schema.py` | Tests for DispatchRecord schema v2 fields and backward compatibility (Group J) | +| `test_summary.py` | Tests for fleet campaign summary schema v1 (Group S) | + +## Architecture Notes + +`conftest.py` and `_helpers.py` provide shared fixtures and helper factories for fleet tests. `test_helpers_exports.py` guards that `_helpers` is importable from other test modules. diff --git a/tests/fleet/__init__.py b/tests/fleet/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/fleet/_helpers.py b/tests/fleet/_helpers.py new file mode 100644 index 0000000000..740a6fab7b --- /dev/null +++ b/tests/fleet/_helpers.py @@ -0,0 +1,92 @@ +"""Shared helpers for tests/fleet/ test modules.""" + +from __future__ import annotations + +from autoskillit.core.types._type_constants import PACK_REGISTRY, TOOL_SUBSET_TAGS + +# --------------------------------------------------------------------------- +# Module-level constants — derived from the authoritative TOOL_SUBSET_TAGS map +# --------------------------------------------------------------------------- + +_tools_by_pack: dict[str, set[str]] = {} +for _tool, _tags in TOOL_SUBSET_TAGS.items(): + for _tag in _tags: + if _tag in PACK_REGISTRY: + _tools_by_pack.setdefault(_tag, set()).add(_tool) + +TOOLS_BY_PACK: dict[str, frozenset[str]] = {k: frozenset(v) for k, v in _tools_by_pack.items()} + +KITCHEN_CORE_TOOLS = TOOLS_BY_PACK["kitchen-core"] + + +def compute_food_truck_tool_surface(recipe_name: str) -> frozenset[str]: + """Compute the expected tool surface for a food truck running the given recipe.""" + from autoskillit.recipe.io import builtin_recipes_dir, load_recipe + + path = builtin_recipes_dir() / f"{recipe_name}.yaml" + recipe = load_recipe(path) + expected: set[str] = set(KITCHEN_CORE_TOOLS) + for pack in recipe.requires_packs or []: + expected |= TOOLS_BY_PACK.get(pack, frozenset()) + return frozenset(expected) + + +def _simple_prompt_builder(**kwargs) -> str: + return f"prompt-for-{kwargs.get('recipe', 'unknown')}" + + +async def _no_sleep_quota_checker(config, **kwargs) -> dict: + return { + "should_sleep": False, + "sleep_seconds": 0, + "utilization": None, + "resets_at": None, + "window_name": None, + } + + +async def _noop_quota_refresher(config, **kwargs) -> None: + pass + + +def _make_recipe_info(name: str = "test-recipe", path_prefix: str = "/fake/"): + from pathlib import Path + + from autoskillit.recipe.schema import RecipeInfo, RecipeSource + + return RecipeInfo( + name=name, + description="test", + source=RecipeSource.PROJECT, + path=Path(f"{path_prefix}{name}.yaml"), + ) + + +def _setup_dispatch( + tool_ctx, + monkeypatch, + recipe_name: str = "test-recipe", + *, + requires_packs: list[str] | None = None, +): + """Wire tool_ctx for dispatch tests.""" + from autoskillit.fleet import FleetSemaphore + from autoskillit.recipe.schema import Recipe, RecipeKind + from tests.fakes import InMemoryHeadlessExecutor, InMemoryRecipeRepository + + tool_ctx.fleet_lock = FleetSemaphore(max_concurrent=1) + repo = InMemoryRecipeRepository() + recipe_info = _make_recipe_info(recipe_name) + repo.add_recipe(recipe_name, recipe_info) + repo.add_full_recipe( + recipe_info.path, + Recipe( + name=recipe_name, + description="test", + kind=RecipeKind.STANDARD, + ingredients={}, + requires_packs=requires_packs if requires_packs is not None else [], + ), + ) + tool_ctx.recipes = repo + tool_ctx.executor = InMemoryHeadlessExecutor() diff --git a/tests/fleet/conftest.py b/tests/fleet/conftest.py new file mode 100644 index 0000000000..9a46ed4456 --- /dev/null +++ b/tests/fleet/conftest.py @@ -0,0 +1 @@ +"""Shared fixtures for tests/fleet/.""" diff --git a/tests/fleet/test_api.py b/tests/fleet/test_api.py new file mode 100644 index 0000000000..8a467e862e --- /dev/null +++ b/tests/fleet/test_api.py @@ -0,0 +1,187 @@ +"""Tests for fleet._api module (Group J).""" + +from __future__ import annotations + +import asyncio +import json +from pathlib import Path + +import pytest +import structlog.testing + +from autoskillit.fleet import ( + DispatchRecord, + _write_pid, + write_initial_state, +) +from tests.fleet._helpers import _setup_dispatch + +pytestmark = [pytest.mark.layer("fleet"), pytest.mark.small, pytest.mark.feature("fleet")] + + +def _state_path(tmp_path: Path) -> Path: + return tmp_path / "campaign" / "state.json" + + +def _make_dispatches(*names: str) -> list[DispatchRecord]: + return [DispatchRecord(name=n) for n in names] + + +class TestWritePidExceptionSwallow: + def test_nonexistent_state_logs_warning(self, tmp_path: Path) -> None: + bogus = tmp_path / "nope" / "state.json" + with structlog.testing.capture_logs() as logs: + _write_pid(bogus, "d1", "id1", 123, 0) + assert any( + "_write_pid" in entry.get("event", "") + for entry in logs + if entry.get("log_level") == "warning" + ) + + def test_runtime_error_logs_warning( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + sp = _state_path(tmp_path) + write_initial_state(sp, "cid", "camp", "/m.yaml", _make_dispatches("d1")) + monkeypatch.setattr( + "autoskillit.fleet.mark_dispatch_running", + lambda *a, **kw: (_ for _ in ()).throw(RuntimeError("boom")), + ) + with structlog.testing.capture_logs() as logs: + _write_pid(sp, "d1", "id1", 123, 0) + assert any( + "_write_pid" in entry.get("event", "") + for entry in logs + if entry.get("log_level") == "warning" + ) + + +class TestExecuteDispatchCancelledErrorLockRelease: + @pytest.mark.anyio + async def test_cancelled_error_propagates_and_releases_lock( + self, tool_ctx, monkeypatch + ) -> None: + from tests.fleet._helpers import _setup_dispatch + + _setup_dispatch(tool_ctx, monkeypatch) + fleet_lock = tool_ctx.fleet_lock + active_count_at_cancel: list[int] = [] + + async def _raise_cancelled(**_kwargs): + active_count_at_cancel.append(fleet_lock.active_count) + raise asyncio.CancelledError + + monkeypatch.setattr("autoskillit.fleet._api._run_dispatch", _raise_cancelled) + + with pytest.raises(asyncio.CancelledError): + from autoskillit.fleet import execute_dispatch + + await execute_dispatch( + tool_ctx=tool_ctx, + recipe="test-recipe", + task="do something", + ingredients=None, + dispatch_name="test-dispatch", + timeout_sec=None, + prompt_builder=lambda *a, **kw: "prompt", + quota_checker=lambda *a, **kw: None, + quota_refresher=lambda *a, **kw: None, + ) + + assert active_count_at_cancel == [1], "lock must be held when _run_dispatch is called" + assert fleet_lock.active_count == 0 + + @pytest.mark.anyio + async def test_execute_dispatch_passes_resume_session_id_to_run_dispatch( + self, tool_ctx, monkeypatch + ) -> None: + """Verify resume_session_id is forwarded to _run_dispatch. + + Raises CancelledError from the _run_dispatch mock intentionally: this + short-circuits execute_dispatch after kwarg capture, avoiding the need + to wire a full dispatch chain while still asserting kwarg forwarding. + """ + from tests.fleet._helpers import _setup_dispatch + + _setup_dispatch(tool_ctx, monkeypatch) + + captured_kwargs: list[dict] = [] + + async def _capture(**kwargs): + captured_kwargs.append(kwargs) + raise asyncio.CancelledError + + monkeypatch.setattr("autoskillit.fleet._api._run_dispatch", _capture) + + with pytest.raises(asyncio.CancelledError): + from autoskillit.fleet import execute_dispatch + + await execute_dispatch( + tool_ctx=tool_ctx, + recipe="test-recipe", + task="do something", + ingredients=None, + dispatch_name="test-dispatch", + timeout_sec=None, + prompt_builder=lambda *a, **kw: "prompt", + quota_checker=lambda *a, **kw: None, + quota_refresher=lambda *a, **kw: None, + resume_session_id="abc-123", + ) + + assert captured_kwargs, "_run_dispatch was never called" + assert captured_kwargs[0].get("resume_session_id") == "abc-123" + + +# --------------------------------------------------------------------------- +# requires_packs forwarding helpers +# --------------------------------------------------------------------------- + + +async def _no_sleep_quota_checker(config, **kwargs) -> dict: + return { + "should_sleep": False, + "sleep_seconds": 0, + "utilization": None, + "resets_at": None, + "window_name": None, + } + + +async def _noop_quota_refresher(config, **kwargs) -> None: + pass + + +async def _run(tool_ctx, recipe="test-recipe", ingredients=None): + from autoskillit.fleet._api import execute_dispatch + + raw = await execute_dispatch( + tool_ctx=tool_ctx, + recipe=recipe, + task="t", + ingredients=ingredients, + dispatch_name=None, + timeout_sec=None, + prompt_builder=lambda **kwargs: f"prompt-for-{kwargs.get('recipe', 'unknown')}", + quota_checker=_no_sleep_quota_checker, + quota_refresher=_noop_quota_refresher, + ) + return json.loads(raw) + + +class TestRequiresPacksForwarding: + @pytest.mark.anyio + async def test_run_dispatch_forwards_requires_packs_to_executor(self, tool_ctx, monkeypatch): + _setup_dispatch(tool_ctx, monkeypatch, requires_packs=["github", "ci"]) + await _run(tool_ctx) + call = tool_ctx.executor.dispatch_calls[0] + assert list(call.requires_packs) == ["github", "ci"] + + @pytest.mark.anyio + async def test_run_dispatch_defaults_kitchen_core_when_requires_packs_empty( + self, tool_ctx, monkeypatch + ): + _setup_dispatch(tool_ctx, monkeypatch) + await _run(tool_ctx) + call = tool_ctx.executor.dispatch_calls[0] + assert list(call.requires_packs) == ["kitchen-core"] diff --git a/tests/fleet/test_campaign_capture.py b/tests/fleet/test_campaign_capture.py new file mode 100644 index 0000000000..fe9e364cba --- /dev/null +++ b/tests/fleet/test_campaign_capture.py @@ -0,0 +1,546 @@ +"""Tests for campaign capture extraction and ingredient interpolation (Group J).""" + +from __future__ import annotations + +import json +from pathlib import Path + +import pytest + +pytestmark = [pytest.mark.layer("fleet"), pytest.mark.small, pytest.mark.feature("fleet")] + + +# --------------------------------------------------------------------------- +# Capture extraction tests +# --------------------------------------------------------------------------- + + +def test_extract_captures_from_payload(): + from autoskillit.fleet._api import _extract_captures + + result = _extract_captures( + {"sources_manifest": "${{ result.sources_manifest }}"}, + {"sources_manifest": "/tmp/sources.json", "extra": "ignored"}, + ) + assert result == {"sources_manifest": "/tmp/sources.json"} + + +def test_extract_captures_missing_field_skipped(): + from autoskillit.fleet._api import _extract_captures + + result = _extract_captures( + {"missing_key": "${{ result.missing_key }}"}, + {"other": "value"}, + ) + assert "missing_key" not in result + + +def test_extract_captures_non_result_template_skipped(): + from autoskillit.fleet._api import _extract_captures + + result = _extract_captures( + {"a": "plain_string", "b": "${{ inputs.x }}"}, + {"plain_string": "val", "x": "val2"}, + ) + assert result == {} + + +def test_extract_captures_converts_value_to_str(): + from autoskillit.fleet._api import _extract_captures + + result = _extract_captures( + {"count": "${{ result.count }}"}, + {"count": 42}, + ) + assert result == {"count": "42"} + + +def test_extract_captures_list_value_uses_json_dumps(): + """list payload value must be JSON-serialized, not Python repr.""" + from autoskillit.fleet._api import _extract_captures + + result = _extract_captures( + {"issue_urls": "${{ result.issue_urls }}"}, + { + "issue_urls": [ + "https://github.com/org/repo/issues/1", + "https://github.com/org/repo/issues/2", + ] + }, + ) + assert result == { + "issue_urls": '["https://github.com/org/repo/issues/1", "https://github.com/org/repo/issues/2"]' + } + assert "'" not in result["issue_urls"] + + +def test_extract_captures_dict_value_uses_json_dumps(): + """dict payload value must be JSON-serialized, not Python repr.""" + from autoskillit.fleet._api import _extract_captures + + result = _extract_captures( + {"meta": "${{ result.meta }}"}, + {"meta": {"count": 3, "status": "ok"}}, + ) + assert json.loads(result["meta"]) == {"count": 3, "status": "ok"} + + +# --------------------------------------------------------------------------- +# Ingredient interpolation tests +# --------------------------------------------------------------------------- + + +def test_interpolate_campaign_refs_basic(): + from autoskillit.fleet._api import _interpolate_campaign_refs + + result = _interpolate_campaign_refs({"k": "${{ campaign.v }}"}, {"v": "resolved"}) + assert result == {"k": "resolved"} + + +def test_interpolate_unresolved_ref_raises_value_error(): + from autoskillit.fleet._api import _interpolate_campaign_refs + + with pytest.raises(ValueError, match="missing"): + _interpolate_campaign_refs({"k": "${{ campaign.missing }}"}, {}) + + +def test_interpolate_passthrough_non_campaign_values(): + from autoskillit.fleet._api import _interpolate_campaign_refs + + result = _interpolate_campaign_refs( + {"a": "${{ inputs.x }}", "b": "plain"}, + {}, + ) + assert result == {"a": "${{ inputs.x }}", "b": "plain"} + + +def test_interpolate_multiple_refs_in_one_value(): + from autoskillit.fleet._api import _interpolate_campaign_refs + + result = _interpolate_campaign_refs( + {"path": "${{ campaign.a }}/${{ campaign.b }}"}, + {"a": "foo", "b": "bar"}, + ) + assert result == {"path": "foo/bar"} + + +def test_extract_captures_bool_value_uses_json_dumps(): + from autoskillit.fleet._api import _extract_captures + + result = _extract_captures( + {"flag": "${{ result.flag }}"}, + {"flag": True}, + ) + assert result == {"flag": "true"} + assert result["flag"] != "True" + + +def test_extract_captures_none_value_uses_json_dumps(): + from autoskillit.fleet._api import _extract_captures + + result = _extract_captures( + {"val": "${{ result.val }}"}, + {"val": None}, + ) + assert result == {"val": "null"} + assert result["val"] != "None" + + +def test_extract_captures_float_value_uses_json_dumps(): + from autoskillit.fleet._api import _extract_captures + + result = _extract_captures( + {"pi": "${{ result.pi }}"}, + {"pi": 3.14}, + ) + assert result == {"pi": "3.14"} + + +def test_extract_captures_empty_spec_returns_empty(): + from autoskillit.fleet._api import _extract_captures + + result = _extract_captures({}, {"anything": "value"}) + assert result == {} + + +def test_interpolate_campaign_refs_empty_ingredients(): + from autoskillit.fleet._api import _interpolate_campaign_refs + + result = _interpolate_campaign_refs({}, {"k": "v"}) + assert result == {} + + +# --------------------------------------------------------------------------- +# Research-campaign capture field tests +# --------------------------------------------------------------------------- + + +def test_extract_research_design_captures(): + from autoskillit.fleet._api import _extract_captures + + spec = { + "worktree_path": "${{ result.worktree_path }}", + "research_dir": "${{ result.research_dir }}", + "experiment_plan": "${{ result.experiment_plan }}", + "visualization_plan_path": "${{ result.visualization_plan_path }}", + } + payload = { + "worktree_path": "/tmp/wt-123", + "research_dir": "/tmp/wt-123/research", + "experiment_plan": "plan.md", + "visualization_plan_path": "/tmp/wt-123/viz.yaml", + } + result = _extract_captures(spec, payload) + assert result == { + "worktree_path": "/tmp/wt-123", + "research_dir": "/tmp/wt-123/research", + "experiment_plan": "plan.md", + "visualization_plan_path": "/tmp/wt-123/viz.yaml", + } + + +def test_extract_research_implement_capture(): + from autoskillit.fleet._api import _extract_captures + + result = _extract_captures( + {"report_path": "${{ result.report_path }}"}, + {"report_path": "/tmp/wt-123/report.md"}, + ) + assert result == {"report_path": "/tmp/wt-123/report.md"} + + +def test_extract_research_review_captures(): + from autoskillit.fleet._api import _extract_captures + + spec = { + "pr_url": "${{ result.pr_url }}", + "all_diagram_paths": "${{ result.all_diagram_paths }}", + "report_path_after_finalize": "${{ result.report_path_after_finalize }}", + } + payload = { + "pr_url": "https://github.com/org/repo/pull/42", + "all_diagram_paths": "/tmp/diagrams/arch.png", + "report_path_after_finalize": "/tmp/wt-123/report-final.md", + } + result = _extract_captures(spec, payload) + assert result == { + "pr_url": "https://github.com/org/repo/pull/42", + "all_diagram_paths": "/tmp/diagrams/arch.png", + "report_path_after_finalize": "/tmp/wt-123/report-final.md", + } + + +def test_extract_research_review_all_diagram_paths_list(): + from autoskillit.fleet._api import _extract_captures + + result = _extract_captures( + {"all_diagram_paths": "${{ result.all_diagram_paths }}"}, + {"all_diagram_paths": ["arch.png", "flow.png"]}, + ) + assert result == {"all_diagram_paths": '["arch.png", "flow.png"]'} + assert "'" not in result["all_diagram_paths"] + + +def test_extract_research_partial_payload_skips_missing(): + from autoskillit.fleet._api import _extract_captures + + spec = { + "worktree_path": "${{ result.worktree_path }}", + "research_dir": "${{ result.research_dir }}", + "experiment_plan": "${{ result.experiment_plan }}", + "visualization_plan_path": "${{ result.visualization_plan_path }}", + } + payload = { + "worktree_path": "/tmp/wt-123", + "experiment_plan": "plan.md", + } + result = _extract_captures(spec, payload) + assert result == { + "worktree_path": "/tmp/wt-123", + "experiment_plan": "plan.md", + } + assert "research_dir" not in result + assert "visualization_plan_path" not in result + + +def test_extract_empty_string_path_is_captured(): + from autoskillit.fleet._api import _extract_captures + + result = _extract_captures( + {"p": "${{ result.p }}"}, + {"p": ""}, + ) + assert result == {"p": ""} + + +def test_extract_research_capture_all_fields_combined(): + from autoskillit.fleet._api import _extract_captures + + spec = { + "worktree_path": "${{ result.worktree_path }}", + "research_dir": "${{ result.research_dir }}", + "experiment_plan": "${{ result.experiment_plan }}", + "visualization_plan_path": "${{ result.visualization_plan_path }}", + "report_path": "${{ result.report_path }}", + "pr_url": "${{ result.pr_url }}", + "all_diagram_paths": "${{ result.all_diagram_paths }}", + } + payload = { + "worktree_path": "/tmp/wt-123", + "research_dir": "/tmp/wt-123/research", + "experiment_plan": "plan.md", + "visualization_plan_path": "/tmp/wt-123/viz.yaml", + "report_path": "/tmp/wt-123/report.md", + "pr_url": "https://github.com/org/repo/pull/42", + "all_diagram_paths": ["a.png", "b.png"], + } + result = _extract_captures(spec, payload) + assert len(result) == 7 + assert result["worktree_path"] == "/tmp/wt-123" + assert result["all_diagram_paths"] == '["a.png", "b.png"]' + + +def test_interpolate_campaign_worktree_and_report_path(): + from autoskillit.fleet._api import _interpolate_campaign_refs + + result = _interpolate_campaign_refs( + { + "wt": "${{ campaign.worktree_path }}", + "rpt": "${{ campaign.report_path }}", + }, + {"worktree_path": "/tmp/wt-123", "report_path": "/tmp/wt-123/report.md"}, + ) + assert result == {"wt": "/tmp/wt-123", "rpt": "/tmp/wt-123/report.md"} + + +def test_interpolate_campaign_unresolved_research_ref_raises(): + import pytest + + from autoskillit.fleet._api import _interpolate_campaign_refs + + with pytest.raises(ValueError, match="has not been captured"): + _interpolate_campaign_refs( + {"plan": "${{ campaign.experiment_plan }}"}, + {}, + ) + + +# --------------------------------------------------------------------------- +# Integration path via execute_dispatch +# --------------------------------------------------------------------------- + + +def _make_recipe_info(name: str = "test-recipe"): + from autoskillit.recipe.schema import RecipeInfo, RecipeSource + + return RecipeInfo( + name=name, + description="test", + source=RecipeSource.PROJECT, + path=Path(f"/fake/{name}.yaml"), + ) + + +def _simple_prompt_builder(**kwargs) -> str: + return f"prompt-for-{kwargs.get('recipe', 'unknown')}" + + +async def _no_sleep_quota_checker(config, **kwargs) -> dict: + return { + "should_sleep": False, + "sleep_seconds": 0, + "utilization": None, + "resets_at": None, + "window_name": None, + } + + +async def _noop_quota_refresher(config, **kwargs) -> None: + pass + + +def _setup_dispatch(tool_ctx, recipe_name: str = "test-recipe", ingredients: dict | None = None): + from autoskillit.fleet import FleetSemaphore + from autoskillit.recipe.schema import Recipe, RecipeKind + from tests.fakes import InMemoryHeadlessExecutor, InMemoryRecipeRepository + + tool_ctx.fleet_lock = FleetSemaphore(max_concurrent=1) + repo = InMemoryRecipeRepository() + recipe_info = _make_recipe_info(recipe_name) + repo.add_recipe(recipe_name, recipe_info) + repo.add_full_recipe( + recipe_info.path, + Recipe( + name=recipe_name, + description="test", + kind=RecipeKind.STANDARD, + ingredients=ingredients or {}, + ), + ) + tool_ctx.recipes = repo + tool_ctx.executor = InMemoryHeadlessExecutor() + + +def _make_success_result(payload: dict): + + from autoskillit.core.types import SkillResult + + body = json.dumps(payload) + sentinel_id_placeholder = "PLACEHOLDER" + stdout = ( + f"%%L3_DONE::{sentinel_id_placeholder}%%\n---l2-result---\n{body}\n---end-l2-result---" + ) + return SkillResult( + success=True, + result=stdout, + session_id="sess-123", + subtype="success", + is_error=False, + exit_code=0, + needs_retry=False, + retry_reason="none", + stderr="", + token_usage=None, + ) + + +def _read_state_file(tool_ctx) -> dict: + state_files = list((tool_ctx.temp_dir / "dispatches").glob("*.json")) + return json.loads(state_files[0].read_text()) + + +@pytest.mark.anyio +async def test_dispatch_captures_extracted_and_written_to_state(tool_ctx, monkeypatch): + """After a successful dispatch with capture spec, state file has captured_values.""" + + from autoskillit.fleet._api import execute_dispatch + + _setup_dispatch(tool_ctx) + + payload = {"success": True, "reason": "", "out": "hello"} + + # The actual dispatch ID isn't known ahead of time; we patch parse_l3_result_block + # to return a clean result with the payload. + from autoskillit.fleet.result_parser import L3ParseResult + + monkeypatch.setattr( + "autoskillit.fleet._api.parse_l3_result_block", + lambda **kwargs: L3ParseResult( + outcome="completed_clean", + payload=payload, + raw_body=None, + parse_error=None, + source="stdout", + ), + ) + + raw = await execute_dispatch( + tool_ctx=tool_ctx, + recipe="test-recipe", + task="t", + ingredients=None, + dispatch_name=None, + timeout_sec=None, + capture={"out": "${{ result.out }}"}, + prompt_builder=_simple_prompt_builder, + quota_checker=_no_sleep_quota_checker, + quota_refresher=_noop_quota_refresher, + ) + + result = json.loads(raw) + assert result["success"] is True + + dispatch_files = list((tool_ctx.temp_dir / "dispatches").glob("*.json")) + assert len(dispatch_files) == 1, f"Expected 1 state file, found {len(dispatch_files)}" + state_data = _read_state_file(tool_ctx) + assert state_data.get("captured_values") == {"out": "hello"} + + +@pytest.mark.anyio +async def test_dispatch_ingredients_interpolated_from_captured_values(tool_ctx, monkeypatch): + """Prior captured_values in state file are resolved into ingredients before dispatch.""" + + from autoskillit.fleet._api import execute_dispatch + from autoskillit.fleet.result_parser import L3ParseResult + from autoskillit.fleet.state import DispatchRecord, write_captured_values, write_initial_state + + # Pre-create a state file for the same campaign_id with captured_values + campaign_id = tool_ctx.kitchen_id + dispatches_dir = tool_ctx.temp_dir / "dispatches" + dispatches_dir.mkdir(parents=True, exist_ok=True) + prior_state_path = dispatches_dir / "prior.json" + write_initial_state( + prior_state_path, + campaign_id=campaign_id, + campaign_name="prior-dispatch", + manifest_path="", + dispatches=[DispatchRecord(name="prior-dispatch")], + ) + from autoskillit.fleet.state import DispatchStatus, append_dispatch_record + + append_dispatch_record( + prior_state_path, + DispatchRecord(name="prior-dispatch", status=DispatchStatus.SUCCESS), + ) + write_captured_values(prior_state_path, {"v": "injected"}) + + _setup_dispatch(tool_ctx, ingredients={"x": ""}) + + received_ingredients: list[dict] = [] + + def _capturing_prompt_builder(**kwargs): + received_ingredients.append(kwargs.get("ingredients", {})) + return "prompt" + + monkeypatch.setattr( + "autoskillit.fleet._api.parse_l3_result_block", + lambda **kwargs: L3ParseResult( + outcome="completed_clean", + payload={"success": True, "reason": ""}, + raw_body=None, + parse_error=None, + source="stdout", + ), + ) + + await execute_dispatch( + tool_ctx=tool_ctx, + recipe="test-recipe", + task="t", + ingredients={"x": "${{ campaign.v }}"}, + dispatch_name=None, + timeout_sec=None, + capture=None, + prompt_builder=_capturing_prompt_builder, + quota_checker=_no_sleep_quota_checker, + quota_refresher=_noop_quota_refresher, + ) + + assert received_ingredients, "prompt_builder was not called" + assert len(received_ingredients) == 1 + assert received_ingredients[0].get("x") == "injected" + + +@pytest.mark.anyio +async def test_unresolved_campaign_ref_in_ingredients_returns_fleet_error(tool_ctx, monkeypatch): + """Dispatch with ${{ campaign.missing }} and no prior captures returns fleet_error.""" + from autoskillit.fleet._api import execute_dispatch + + _setup_dispatch(tool_ctx, ingredients={"x": ""}) + + raw = await execute_dispatch( + tool_ctx=tool_ctx, + recipe="test-recipe", + task="t", + ingredients={"x": "${{ campaign.missing }}"}, + dispatch_name=None, + timeout_sec=None, + capture=None, + prompt_builder=_simple_prompt_builder, + quota_checker=_no_sleep_quota_checker, + quota_refresher=_noop_quota_refresher, + ) + + result = json.loads(raw) + assert result["success"] is False + assert "error" in result diff --git a/tests/fleet/test_checkpoint_bridge.py b/tests/fleet/test_checkpoint_bridge.py new file mode 100644 index 0000000000..003a841dc6 --- /dev/null +++ b/tests/fleet/test_checkpoint_bridge.py @@ -0,0 +1,55 @@ +"""T1 (cont.): checkpoint_from_sidecar bridge.""" + +from __future__ import annotations + +import pytest + +from autoskillit.fleet._checkpoint_bridge import checkpoint_from_sidecar +from autoskillit.fleet.sidecar import IssueSidecarEntry + +pytestmark = [pytest.mark.layer("fleet"), pytest.mark.small, pytest.mark.feature("fleet")] + + +class TestCheckpointFromSidecar: + def test_extracts_completed_urls(self) -> None: + entries = [ + IssueSidecarEntry( + issue_url="https://github.com/o/r/issues/1", status="completed", ts="t1" + ), + IssueSidecarEntry( + issue_url="https://github.com/o/r/issues/2", status="failed", ts="t2" + ), + IssueSidecarEntry( + issue_url="https://github.com/o/r/issues/3", status="completed", ts="t3" + ), + ] + cp = checkpoint_from_sidecar(entries) + assert cp.completed_items == [ + "https://github.com/o/r/issues/1", + "https://github.com/o/r/issues/3", + ] + assert cp.step_name == "fleet_dispatch" + + def test_empty_entries(self) -> None: + cp = checkpoint_from_sidecar([]) + assert cp.completed_items == [] + assert cp.ts == "" + + def test_uses_last_entry_ts(self) -> None: + entries = [ + IssueSidecarEntry(issue_url="u1", status="completed", ts="2026-01-01"), + IssueSidecarEntry(issue_url="u2", status="completed", ts="2026-05-04"), + ] + cp = checkpoint_from_sidecar(entries) + assert cp.ts == "2026-05-04" + + def test_produces_valid_checkpoint(self) -> None: + entries = [ + IssueSidecarEntry(issue_url="u1", status="completed", ts="t"), + ] + cp = checkpoint_from_sidecar(entries) + d = cp.to_dict() + from autoskillit.core.types._type_checkpoint import SessionCheckpoint + + restored = SessionCheckpoint.from_dict(d) + assert restored == cp diff --git a/tests/fleet/test_dispatch_failure_semantics.py b/tests/fleet/test_dispatch_failure_semantics.py new file mode 100644 index 0000000000..09a9215a7c --- /dev/null +++ b/tests/fleet/test_dispatch_failure_semantics.py @@ -0,0 +1,620 @@ +"""Group F: Timeout + No-Result-Block failure semantics for fleet dispatch.""" + +from __future__ import annotations + +import json + +import pytest + +from autoskillit.fleet.result_parser import L3ParseResult +from tests.fakes import InMemoryHeadlessExecutor +from tests.fleet._helpers import _setup_dispatch + +pytestmark = [pytest.mark.layer("fleet"), pytest.mark.small, pytest.mark.feature("fleet")] + + +def _simple_prompt_builder(**kwargs) -> str: + return f"prompt-for-{kwargs.get('recipe', 'unknown')}" + + +async def _no_sleep_quota_checker(config, **kwargs) -> dict: + return { + "should_sleep": False, + "sleep_seconds": 0, + "utilization": None, + "resets_at": None, + "window_name": None, + } + + +async def _noop_quota_refresher(config, **kwargs) -> None: + pass + + +async def _run( + tool_ctx, recipe: str = "test-recipe", ingredients: dict[str, str] | None = None +) -> dict: + from autoskillit.fleet._api import execute_dispatch + + raw = await execute_dispatch( + tool_ctx=tool_ctx, + recipe=recipe, + task="t", + ingredients=ingredients, + dispatch_name=None, + timeout_sec=None, + prompt_builder=_simple_prompt_builder, + quota_checker=_no_sleep_quota_checker, + quota_refresher=_noop_quota_refresher, + ) + return json.loads(raw) + + +def _read_dispatch_record(tool_ctx) -> dict: + """Read the single dispatch record written to the state file.""" + state_files = list((tool_ctx.temp_dir / "dispatches").glob("*.json")) + assert len(state_files) == 1, f"Expected 1 state file, found {len(state_files)}" + state = json.loads(state_files[0].read_text()) + return state["dispatches"][0] + + +def _make_no_sentinel(): + + return L3ParseResult( + outcome="no_sentinel", + payload=None, + raw_body=None, + parse_error=None, + source="stdout", + ) + + +def _make_completed_dirty(): + + return L3ParseResult( + outcome="completed_dirty", + payload=None, + raw_body="garbled", + parse_error="json decode error", + source="stdout", + ) + + +def _make_completed_clean(success: bool, reason: str = ""): + + payload: dict = {"success": success} + if reason: + payload["reason"] = reason + return L3ParseResult( + outcome="completed_clean", + payload=payload, + raw_body=None, + parse_error=None, + source="stdout", + ) + + +class TestTimeoutPath: + @pytest.mark.anyio + async def test_timeout_returns_fleet_error_envelope(self, tool_ctx, monkeypatch): + """skill_result.subtype == 'timeout' → fleet_error envelope with error='l3_timeout'.""" + import dataclasses + + from tests.fakes import _DEFAULT_SKILL_RESULT + + _setup_dispatch(tool_ctx, monkeypatch) + tool_ctx.executor = InMemoryHeadlessExecutor( + default_result=dataclasses.replace(_DEFAULT_SKILL_RESULT, subtype="timeout") + ) + + result = await _run(tool_ctx) + assert result["success"] is False + assert result["error"] == "fleet_l3_timeout" + + @pytest.mark.anyio + async def test_timeout_writes_state_with_reason_l3_timeout(self, tool_ctx, monkeypatch): + """Timeout path writes DispatchRecord with status=failure and reason=l3_timeout.""" + import dataclasses + + from tests.fakes import _DEFAULT_SKILL_RESULT + + _setup_dispatch(tool_ctx, monkeypatch) + tool_ctx.executor = InMemoryHeadlessExecutor( + default_result=dataclasses.replace(_DEFAULT_SKILL_RESULT, subtype="timeout") + ) + + await _run(tool_ctx) + + record = _read_dispatch_record(tool_ctx) + assert record["status"] == "failure" + assert record["reason"] == "fleet_l3_timeout" + + @pytest.mark.anyio + async def test_timeout_skips_parse_l3_result_block(self, tool_ctx, monkeypatch): + """Timeout path must not call parse_l3_result_block.""" + import dataclasses + + from tests.fakes import _DEFAULT_SKILL_RESULT + + _setup_dispatch(tool_ctx, monkeypatch) + tool_ctx.executor = InMemoryHeadlessExecutor( + default_result=dataclasses.replace(_DEFAULT_SKILL_RESULT, subtype="timeout") + ) + + def _should_not_be_called(**_kwargs): + raise AssertionError("parse_l3_result_block called on timeout path") + + monkeypatch.setattr( + "autoskillit.fleet._api.parse_l3_result_block", + _should_not_be_called, + ) + + # Should succeed (return l3_timeout error envelope) without raising + result = await _run(tool_ctx) + assert result["error"] == "fleet_l3_timeout" + + @pytest.mark.anyio + async def test_timeout_envelope_includes_dispatch_metadata(self, tool_ctx, monkeypatch): + """Timeout envelope includes dispatch_id, dispatched_session_id, and token_usage.""" + import dataclasses + + from tests.fakes import _DEFAULT_SKILL_RESULT + + _setup_dispatch(tool_ctx, monkeypatch) + tool_ctx.executor = InMemoryHeadlessExecutor( + default_result=dataclasses.replace( + _DEFAULT_SKILL_RESULT, + subtype="timeout", + session_id="sess-timeout-123", + token_usage={"input_tokens": 50}, + ) + ) + + result = await _run(tool_ctx) + details = result.get("details", {}) + assert "dispatch_id" in details + assert details["dispatched_session_id"] == "sess-timeout-123" + assert details["token_usage"] == {"input_tokens": 50} + + @pytest.mark.anyio + async def test_idle_stall_falls_through_to_parse(self, tool_ctx, monkeypatch): + """idle_stall subtype must NOT trigger the timeout pre-check; parse is called.""" + import dataclasses + + from tests.fakes import _DEFAULT_SKILL_RESULT + + _setup_dispatch(tool_ctx, monkeypatch) + tool_ctx.executor = InMemoryHeadlessExecutor( + default_result=dataclasses.replace( + _DEFAULT_SKILL_RESULT, + subtype="idle_stall", + success=False, + ) + ) + + parse_called = [] + + def _recording_parse(**kwargs): + parse_called.append(True) + return _make_no_sentinel() + + monkeypatch.setattr("autoskillit.fleet._api.parse_l3_result_block", _recording_parse) + + await _run(tool_ctx) + assert parse_called, "parse_l3_result_block was not called for idle_stall" + + +class TestNoSentinelPath: + @pytest.mark.anyio + async def test_no_sentinel_writes_state_with_reason_l3_no_result_block( + self, tool_ctx, monkeypatch + ): + """no_sentinel outcome → DispatchRecord.reason = 'l3_no_result_block'.""" + _setup_dispatch(tool_ctx, monkeypatch) + monkeypatch.setattr( + "autoskillit.fleet._api.parse_l3_result_block", + lambda **_: _make_no_sentinel(), + ) + + await _run(tool_ctx) + + record = _read_dispatch_record(tool_ctx) + assert record["reason"] == "fleet_l3_no_result_block" + + @pytest.mark.anyio + async def test_no_sentinel_clean_exit_is_not_success(self, tool_ctx, monkeypatch): + """no_sentinel outcome → envelope.success=False even when SkillResult.success=True.""" + import dataclasses + + from tests.fakes import _DEFAULT_SKILL_RESULT + + _setup_dispatch(tool_ctx, monkeypatch) + tool_ctx.executor = InMemoryHeadlessExecutor( + default_result=dataclasses.replace( + _DEFAULT_SKILL_RESULT, + success=True, + exit_code=0, + ) + ) + monkeypatch.setattr( + "autoskillit.fleet._api.parse_l3_result_block", + lambda **_: _make_no_sentinel(), + ) + + result = await _run(tool_ctx) + assert result["success"] is False + + +class TestCompletedDirtyPath: + @pytest.mark.anyio + async def test_completed_dirty_writes_state_with_reason_l3_parse_failed( + self, tool_ctx, monkeypatch + ): + """completed_dirty outcome → DispatchRecord.reason = 'l3_parse_failed'.""" + _setup_dispatch(tool_ctx, monkeypatch) + monkeypatch.setattr( + "autoskillit.fleet._api.parse_l3_result_block", + lambda **_: _make_completed_dirty(), + ) + + await _run(tool_ctx) + + record = _read_dispatch_record(tool_ctx) + assert record["reason"] == "fleet_l3_parse_failed" + + +class TestDispatchStatusEnvelopeField: + @pytest.mark.anyio + async def test_envelope_includes_dispatch_status_on_success(self, tool_ctx, monkeypatch): + """Envelope from _run_dispatch includes dispatch_status matching state-file status.""" + _setup_dispatch(tool_ctx, monkeypatch) + monkeypatch.setattr( + "autoskillit.fleet._api.parse_l3_result_block", + lambda **_: _make_completed_clean(success=True), + ) + + result = await _run(tool_ctx) + assert "dispatch_status" in result + assert result["dispatch_status"] == "success" + + @pytest.mark.anyio + async def test_envelope_includes_dispatch_status_on_failure(self, tool_ctx, monkeypatch): + """Envelope includes dispatch_status='failure' when outcome is completed_dirty.""" + _setup_dispatch(tool_ctx, monkeypatch) + monkeypatch.setattr( + "autoskillit.fleet._api.parse_l3_result_block", + lambda **_: _make_completed_dirty(), + ) + + result = await _run(tool_ctx) + assert "dispatch_status" in result + assert result["dispatch_status"] == "failure" + + @pytest.mark.anyio + async def test_envelope_includes_dispatch_status_on_no_sentinel(self, tool_ctx, monkeypatch): + """Envelope includes dispatch_status='failure' for no_sentinel without session signal.""" + _setup_dispatch(tool_ctx, monkeypatch) + monkeypatch.setattr( + "autoskillit.fleet._api.parse_l3_result_block", + lambda **_: _make_no_sentinel(), + ) + + result = await _run(tool_ctx) + assert "dispatch_status" in result + assert result["dispatch_status"] == "failure" + + @pytest.mark.anyio + async def test_envelope_includes_dispatch_status_on_no_sentinel_resumable( + self, tool_ctx, monkeypatch + ): + """no_sentinel + session_id + lifespan_started + sidecar → dispatch_status='resumable'.""" + import dataclasses + from uuid import UUID + + from tests.fakes import _DEFAULT_SKILL_RESULT, InMemoryHeadlessExecutor + + _setup_dispatch(tool_ctx, monkeypatch) + + fixed_dispatch_id = "aaaabbbb-cccc-dddd-eeee-ffffffffffff" + monkeypatch.setattr("autoskillit.fleet._api.uuid4", lambda: UUID(fixed_dispatch_id)) + + from autoskillit.fleet.sidecar import sidecar_path + + sidecar_file = sidecar_path(fixed_dispatch_id, tool_ctx.project_dir) + sidecar_file.parent.mkdir(parents=True, exist_ok=True) + sidecar_file.touch() + + resumable_result = dataclasses.replace( + _DEFAULT_SKILL_RESULT, + session_id="sess-resumable-abc", + lifespan_started=True, + ) + + class _SpawningExecutor(InMemoryHeadlessExecutor): + """Calls on_spawn with a fake PID to drive PENDING → RUNNING before returning.""" + + async def dispatch_food_truck(self, orchestrator_prompt, cwd, *, on_spawn=None, **kw): + if on_spawn is not None: + on_spawn(12345, 1000) + return await super().dispatch_food_truck( + orchestrator_prompt, cwd, on_spawn=on_spawn, **kw + ) + + tool_ctx.executor = _SpawningExecutor(default_result=resumable_result) + + monkeypatch.setattr( + "autoskillit.fleet._api.parse_l3_result_block", + lambda **_: _make_no_sentinel(), + ) + + result = await _run(tool_ctx) + assert "dispatch_status" in result + assert result["dispatch_status"] == "resumable" + + +class TestCompletedCleanPath: + @pytest.mark.anyio + async def test_completed_clean_success_writes_empty_reason(self, tool_ctx, monkeypatch): + """completed_clean with success=True → DispatchRecord.reason = ''.""" + _setup_dispatch(tool_ctx, monkeypatch) + monkeypatch.setattr( + "autoskillit.fleet._api.parse_l3_result_block", + lambda **_: _make_completed_clean(success=True), + ) + + await _run(tool_ctx) + + record = _read_dispatch_record(tool_ctx) + assert record["reason"] == "" + + @pytest.mark.anyio + async def test_completed_clean_failure_writes_reason_from_payload(self, tool_ctx, monkeypatch): + """completed_clean success=False: payload.reason → DispatchRecord.reason.""" + _setup_dispatch(tool_ctx, monkeypatch) + monkeypatch.setattr( + "autoskillit.fleet._api.parse_l3_result_block", + lambda **_: _make_completed_clean(success=False, reason="my-failure-reason"), + ) + + await _run(tool_ctx) + + record = _read_dispatch_record(tool_ctx) + assert record["reason"] == "my-failure-reason" + + +# --------------------------------------------------------------------------- +# Group: Missing required ingredient validation +# --------------------------------------------------------------------------- + + +def _setup_dispatch_with_ingredients(tool_ctx, monkeypatch, ingredients: dict): + """Wire tool_ctx with a recipe that has specific ingredients.""" + from autoskillit.fleet import FleetSemaphore + from autoskillit.recipe.schema import Recipe, RecipeIngredient, RecipeKind + from tests.fakes import InMemoryHeadlessExecutor, InMemoryRecipeRepository + + tool_ctx.fleet_lock = FleetSemaphore(max_concurrent=1) + repo = InMemoryRecipeRepository() + recipe_info = _make_recipe_info("test-recipe") + repo.add_recipe("test-recipe", recipe_info) + repo.add_full_recipe( + recipe_info.path, + Recipe( + name="test-recipe", + description="test", + kind=RecipeKind.STANDARD, + ingredients={ + k: RecipeIngredient(description=f"desc-{k}", **v) for k, v in ingredients.items() + }, + ), + ) + tool_ctx.recipes = repo + tool_ctx.executor = InMemoryHeadlessExecutor() + + +def _make_recipe_info(name: str = "test-recipe"): + from tests.fleet._helpers import _make_recipe_info as _base + + return _base(name) + + +class TestMissingRequiredIngredient: + @pytest.mark.anyio + async def test_dispatch_rejects_missing_required_ingredient(self, tool_ctx, monkeypatch): + """Required ingredient with no default → FLEET_MISSING_INGREDIENT.""" + _setup_dispatch_with_ingredients( + tool_ctx, monkeypatch, {"api_key": {"required": True, "default": None}} + ) + + result = await _run(tool_ctx, ingredients={}) + assert result["success"] is False + assert result["error"] == "fleet_missing_ingredient" + + @pytest.mark.anyio + async def test_dispatch_allows_required_ingredient_when_supplied(self, tool_ctx, monkeypatch): + """A required ingredient that IS supplied passes validation.""" + _setup_dispatch_with_ingredients( + tool_ctx, monkeypatch, {"api_key": {"required": True, "default": None}} + ) + + result = await _run(tool_ctx, ingredients={"api_key": "secret"}) + assert result.get("error") != "fleet_missing_ingredient" + + @pytest.mark.anyio + async def test_dispatch_allows_required_ingredient_with_default(self, tool_ctx, monkeypatch): + """A required ingredient with a non-None default passes even when not supplied.""" + _setup_dispatch_with_ingredients( + tool_ctx, monkeypatch, {"api_key": {"required": True, "default": "fallback"}} + ) + + result = await _run(tool_ctx, ingredients={}) + assert result.get("error") != "fleet_missing_ingredient" + + @pytest.mark.anyio + async def test_dispatch_lists_all_missing_required_ingredients(self, tool_ctx, monkeypatch): + """When multiple required ingredients are missing, all are listed.""" + _setup_dispatch_with_ingredients( + tool_ctx, + monkeypatch, + { + "key_a": {"required": True, "default": None}, + "key_b": {"required": True, "default": None}, + }, + ) + + result = await _run(tool_ctx, ingredients={}) + assert result["success"] is False + assert result["error"] == "fleet_missing_ingredient" + assert "key_a" in result["user_visible_message"] + assert "key_b" in result["user_visible_message"] + + @pytest.mark.anyio + async def test_dispatch_ignores_optional_missing_ingredients(self, tool_ctx, monkeypatch): + """Optional ingredients (required=False) don't trigger missing-ingredient errors.""" + _setup_dispatch_with_ingredients( + tool_ctx, monkeypatch, {"optional_key": {"required": False, "default": None}} + ) + + result = await _run(tool_ctx, ingredients={}) + assert result.get("error") != "fleet_missing_ingredient" + + @pytest.mark.anyio + async def test_task_auto_injected_from_top_level_param(self, tool_ctx, monkeypatch): + """top-level task param auto-injects into effective_ingredients when recipe declares it.""" + _setup_dispatch_with_ingredients( + tool_ctx, monkeypatch, {"task": {"required": True, "default": None}} + ) + + result = await _run(tool_ctx, ingredients={}) + assert result.get("error") != "fleet_missing_ingredient" + + @pytest.mark.anyio + async def test_explicit_ingredient_task_overrides_top_level(self, tool_ctx, monkeypatch): + """Explicit ingredients['task'] takes precedence over top-level task param.""" + _setup_dispatch_with_ingredients( + tool_ctx, monkeypatch, {"task": {"required": True, "default": None}} + ) + + captured = {} + + def _capture_prompt_builder(**kwargs): + captured.update(kwargs) + return "prompt" + + from autoskillit.fleet._api import execute_dispatch + + await execute_dispatch( + tool_ctx=tool_ctx, + recipe="test-recipe", + task="top-level-value", + ingredients={"task": "override-value"}, + dispatch_name=None, + timeout_sec=None, + prompt_builder=_capture_prompt_builder, + quota_checker=_no_sleep_quota_checker, + quota_refresher=_noop_quota_refresher, + ) + + assert captured["ingredients"]["task"] == "override-value" + + @pytest.mark.anyio + async def test_task_not_injected_when_not_declared_ingredient(self, tool_ctx, monkeypatch): + """Top-level task is NOT injected when recipe has no 'task' ingredient key.""" + _setup_dispatch_with_ingredients( + tool_ctx, monkeypatch, {"other_key": {"required": False, "default": "x"}} + ) + + captured = {} + + def _capture_prompt_builder(**kwargs): + captured.update(kwargs) + return "prompt" + + from autoskillit.fleet._api import execute_dispatch + + await execute_dispatch( + tool_ctx=tool_ctx, + recipe="test-recipe", + task="some-task", + ingredients={}, + dispatch_name=None, + timeout_sec=None, + prompt_builder=_capture_prompt_builder, + quota_checker=_no_sleep_quota_checker, + quota_refresher=_noop_quota_refresher, + ) + + assert "task" not in captured["ingredients"] + + +# --------------------------------------------------------------------------- +# Group: Recipe kind dispatch gate +# --------------------------------------------------------------------------- + + +class TestRecipeKindDispatchGate: + """Verify that dispatch gate accepts/rejects by RecipeKind.""" + + def _setup_food_truck_recipe(self, tool_ctx): + """Wire tool_ctx with a food-truck kind recipe.""" + from autoskillit.fleet import FleetSemaphore + from autoskillit.recipe.schema import Recipe, RecipeKind + from tests.fakes import InMemoryHeadlessExecutor, InMemoryRecipeRepository + + tool_ctx.fleet_lock = FleetSemaphore(max_concurrent=1) + repo = InMemoryRecipeRepository() + recipe_info = _make_recipe_info("test-recipe") + repo.add_recipe("test-recipe", recipe_info) + repo.add_full_recipe( + recipe_info.path, + Recipe( + name="test-recipe", + description="test", + kind=RecipeKind.FOOD_TRUCK, + ingredients={}, + requires_packs=[], + ), + ) + tool_ctx.recipes = repo + tool_ctx.executor = InMemoryHeadlessExecutor() + + def _setup_campaign_recipe(self, tool_ctx): + """Wire tool_ctx with a campaign kind recipe.""" + from autoskillit.fleet import FleetSemaphore + from autoskillit.recipe.schema import Recipe, RecipeKind + from tests.fakes import InMemoryHeadlessExecutor, InMemoryRecipeRepository + + tool_ctx.fleet_lock = FleetSemaphore(max_concurrent=1) + repo = InMemoryRecipeRepository() + recipe_info = _make_recipe_info("test-recipe") + repo.add_recipe("test-recipe", recipe_info) + repo.add_full_recipe( + recipe_info.path, + Recipe( + name="test-recipe", + description="test", + kind=RecipeKind.CAMPAIGN, + ingredients={}, + requires_packs=[], + ), + ) + tool_ctx.recipes = repo + tool_ctx.executor = InMemoryHeadlessExecutor() + + @pytest.mark.anyio + async def test_food_truck_dispatchable(self, tool_ctx, monkeypatch): + """T4: FOOD_TRUCK kind is accepted by the dispatch gate (not rejected).""" + self._setup_food_truck_recipe(tool_ctx) + + result = await _run(tool_ctx) + # Any error OTHER than fleet_invalid_recipe_kind means the gate passed + assert result.get("error") != "fleet_invalid_recipe_kind" + + @pytest.mark.anyio + async def test_campaign_kind_still_rejected_by_dispatch(self, tool_ctx, monkeypatch): + """T5: CAMPAIGN kind is still rejected by the dispatch gate.""" + self._setup_campaign_recipe(tool_ctx) + + result = await _run(tool_ctx) + assert result["success"] is False + assert result["error"] == "fleet_invalid_recipe_kind" diff --git a/tests/fleet/test_dispatch_lifespan.py b/tests/fleet/test_dispatch_lifespan.py new file mode 100644 index 0000000000..4b788ac39d --- /dev/null +++ b/tests/fleet/test_dispatch_lifespan.py @@ -0,0 +1,161 @@ +"""Group G (fleet part): lifespan_started surface + envelope propagation.""" + +from __future__ import annotations + +import json + +import pytest + +from tests.fakes import InMemoryHeadlessExecutor +from tests.fleet._helpers import _setup_dispatch + +pytestmark = [pytest.mark.layer("fleet"), pytest.mark.small, pytest.mark.feature("fleet")] + + +def _simple_prompt_builder(**kwargs) -> str: + return f"prompt-for-{kwargs.get('recipe', 'unknown')}" + + +async def _no_sleep_quota_checker(config, **kwargs) -> dict: + return { + "should_sleep": False, + "sleep_seconds": 0, + "utilization": None, + "resets_at": None, + "window_name": None, + } + + +async def _noop_quota_refresher(config, **kwargs) -> None: + pass + + +async def _run(tool_ctx, recipe: str = "test-recipe") -> dict: + from autoskillit.fleet._api import execute_dispatch + + raw = await execute_dispatch( + tool_ctx=tool_ctx, + recipe=recipe, + task="t", + ingredients=None, + dispatch_name=None, + timeout_sec=None, + prompt_builder=_simple_prompt_builder, + quota_checker=_no_sleep_quota_checker, + quota_refresher=_noop_quota_refresher, + ) + return json.loads(raw) + + +def _make_completed_clean(success: bool): + from autoskillit.fleet.result_parser import L3ParseResult + + return L3ParseResult( + outcome="completed_clean", + payload={"success": success}, + raw_body=None, + parse_error=None, + source="stdout", + ) + + +def _make_no_sentinel(): + from autoskillit.fleet.result_parser import L3ParseResult + + return L3ParseResult( + outcome="no_sentinel", + payload=None, + raw_body=None, + parse_error=None, + source="stdout", + ) + + +def _make_completed_dirty(): + from autoskillit.fleet.result_parser import L3ParseResult + + return L3ParseResult( + outcome="completed_dirty", + payload=None, + raw_body="bad", + parse_error="json error", + source="stdout", + ) + + +class TestLifespanStartedField: + def test_skill_result_lifespan_started_field_exists(self): + """SkillResult has a lifespan_started: bool field defaulting to False.""" + from autoskillit.core import SkillResult + from autoskillit.core.types import RetryReason + + sr = SkillResult( + success=True, + result="ok", + session_id="", + subtype="success", + is_error=False, + exit_code=0, + needs_retry=False, + retry_reason=RetryReason.NONE, + stderr="", + ) + assert hasattr(sr, "lifespan_started") + assert sr.lifespan_started is False + + +class TestLifespanStartedInEnvelopes: + @pytest.mark.anyio + async def test_success_envelope_includes_lifespan_started(self, tool_ctx, monkeypatch): + """completed_clean success envelope includes 'lifespan_started' key.""" + import dataclasses + + from tests.fakes import _DEFAULT_SKILL_RESULT + + _setup_dispatch(tool_ctx, monkeypatch) + tool_ctx.executor = InMemoryHeadlessExecutor( + default_result=dataclasses.replace( + _DEFAULT_SKILL_RESULT, + lifespan_started=True, + ) + ) + monkeypatch.setattr( + "autoskillit.fleet._api.parse_l3_result_block", + lambda **_: _make_completed_clean(success=True), + ) + + result = await _run(tool_ctx) + assert result["success"] is True + assert "lifespan_started" in result + assert result["lifespan_started"] is True + + @pytest.mark.anyio + async def test_failure_envelope_includes_lifespan_started_no_sentinel( + self, tool_ctx, monkeypatch + ): + """no_sentinel envelope includes 'lifespan_started' field.""" + _setup_dispatch(tool_ctx, monkeypatch) + monkeypatch.setattr( + "autoskillit.fleet._api.parse_l3_result_block", + lambda **_: _make_no_sentinel(), + ) + + result = await _run(tool_ctx) + assert result["success"] is False + assert "lifespan_started" in result + assert result["lifespan_started"] is False + + @pytest.mark.anyio + async def test_failure_envelope_includes_lifespan_started_completed_dirty( + self, tool_ctx, monkeypatch + ): + """completed_dirty envelope includes 'lifespan_started' field.""" + _setup_dispatch(tool_ctx, monkeypatch) + monkeypatch.setattr( + "autoskillit.fleet._api.parse_l3_result_block", + lambda **_: _make_completed_dirty(), + ) + + result = await _run(tool_ctx) + assert result["success"] is False + assert "lifespan_started" in result diff --git a/tests/fleet/test_dispatch_outcome_classifier.py b/tests/fleet/test_dispatch_outcome_classifier.py new file mode 100644 index 0000000000..751dda6eb0 --- /dev/null +++ b/tests/fleet/test_dispatch_outcome_classifier.py @@ -0,0 +1,232 @@ +"""Tests for classify_dispatch_outcome() pure classification function.""" + +from __future__ import annotations + +import dataclasses + +import pytest + +from autoskillit.core import FleetErrorCode, InfraOutcome, SkillResult +from autoskillit.fleet import DispatchStatus +from autoskillit.fleet._api import classify_dispatch_outcome +from autoskillit.fleet.result_parser import L3ParseResult +from tests.fakes import _DEFAULT_SKILL_RESULT + +pytestmark = [pytest.mark.layer("fleet"), pytest.mark.small, pytest.mark.feature("fleet")] + + +def _no_sentinel( + session_id: str = "", lifespan_started: bool = False +) -> tuple[L3ParseResult, SkillResult]: + parsed = L3ParseResult( + outcome="no_sentinel", + payload=None, + raw_body=None, + parse_error=None, + source="stdout", + ) + skill_result = dataclasses.replace( + _DEFAULT_SKILL_RESULT, + session_id=session_id, + lifespan_started=lifespan_started, + ) + return parsed, skill_result + + +class TestClassifyDispatchOutcomeNoSentinel: + def test_no_sentinel_with_session_and_sidecar_is_resumable(self): + parsed, skill_result = _no_sentinel(session_id="sess-abc", lifespan_started=True) + status, reason = classify_dispatch_outcome(parsed, skill_result, sidecar_exists=True) + assert status == DispatchStatus.RESUMABLE + assert reason == FleetErrorCode.FLEET_L3_NO_RESULT_BLOCK + + def test_no_sentinel_without_session_is_failure(self): + parsed, skill_result = _no_sentinel(session_id="", lifespan_started=True) + status, reason = classify_dispatch_outcome(parsed, skill_result, sidecar_exists=True) + assert status == DispatchStatus.FAILURE + assert reason == FleetErrorCode.FLEET_L3_NO_RESULT_BLOCK + + def test_no_sentinel_lifespan_not_started_is_failure(self): + parsed, skill_result = _no_sentinel(session_id="sess-abc", lifespan_started=False) + status, reason = classify_dispatch_outcome(parsed, skill_result, sidecar_exists=True) + assert status == DispatchStatus.FAILURE + assert reason == FleetErrorCode.FLEET_L3_NO_RESULT_BLOCK + + def test_no_sentinel_without_sidecar_is_failure(self): + parsed, skill_result = _no_sentinel(session_id="sess-abc", lifespan_started=True) + status, reason = classify_dispatch_outcome(parsed, skill_result, sidecar_exists=False) + assert status == DispatchStatus.FAILURE + assert reason == FleetErrorCode.FLEET_L3_NO_RESULT_BLOCK + + +class TestClassifyDispatchOutcomeCompletedClean: + def test_completed_clean_success(self): + parsed = L3ParseResult( + outcome="completed_clean", + payload={"success": True}, + raw_body=None, + parse_error=None, + source="stdout", + ) + skill_result = dataclasses.replace(_DEFAULT_SKILL_RESULT) + status, reason = classify_dispatch_outcome(parsed, skill_result, sidecar_exists=False) + assert status == DispatchStatus.SUCCESS + assert reason == "" + + def test_completed_clean_failure(self): + parsed = L3ParseResult( + outcome="completed_clean", + payload={"success": False, "reason": "my-error"}, + raw_body=None, + parse_error=None, + source="stdout", + ) + skill_result = dataclasses.replace(_DEFAULT_SKILL_RESULT) + status, reason = classify_dispatch_outcome(parsed, skill_result, sidecar_exists=False) + assert status == DispatchStatus.FAILURE + assert reason == "my-error" + + +class TestQuotaExhaustedOutcome: + def test_quota_exhausted_is_failure(self): + """completed_clean + success=false + reason=fleet_quota_exhausted → FAILURE.""" + parsed = L3ParseResult( + outcome="completed_clean", + payload={"success": False, "reason": FleetErrorCode.FLEET_QUOTA_EXHAUSTED}, + raw_body=None, + parse_error=None, + source="stdout", + ) + skill_result = dataclasses.replace(_DEFAULT_SKILL_RESULT) + status, reason = classify_dispatch_outcome(parsed, skill_result, sidecar_exists=False) + assert status == DispatchStatus.FAILURE + assert reason == FleetErrorCode.FLEET_QUOTA_EXHAUSTED + + +class TestInfrastructureFailureReasons: + def test_has_failed_dispatch_returns_false_for_quota_exhaustion(self, tmp_path): + """Quota exhaustion is infrastructure failure — does not halt campaign.""" + from autoskillit.core import FleetErrorCode as FEC + from autoskillit.fleet.state import _write_state as write_state + from autoskillit.fleet.state_recovery import has_failed_dispatch + from autoskillit.fleet.state_types import CampaignState, DispatchRecord + + dispatches = [ + DispatchRecord( + name="step_1", + status=DispatchStatus.FAILURE, + reason=FEC.FLEET_QUOTA_EXHAUSTED, + ), + ] + state = CampaignState( + schema_version=4, + campaign_id="test-id", + campaign_name="test", + manifest_path="manifest.yaml", + started_at=0.0, + dispatches=dispatches, + ) + write_state(tmp_path / "state.json", state) + assert not has_failed_dispatch(tmp_path / "state.json") + + +class TestClassifyDispatchOutcomeCompletedDirty: + def test_completed_dirty_is_failure(self): + parsed = L3ParseResult( + outcome="completed_dirty", + payload=None, + raw_body="garbled", + parse_error="json decode error", + source="stdout", + ) + skill_result = dataclasses.replace(_DEFAULT_SKILL_RESULT) + status, reason = classify_dispatch_outcome(parsed, skill_result, sidecar_exists=False) + assert status == DispatchStatus.FAILURE + assert reason == FleetErrorCode.FLEET_L3_PARSE_FAILED + + +class TestReasonAwareResumePolicy: + def test_idle_stall_with_progress_is_resumable(self): + parsed, skill_result = _no_sentinel(session_id="sess-abc", lifespan_started=True) + skill_result = dataclasses.replace( + skill_result, + retry_reason="idle_stall", + ) + status, reason = classify_dispatch_outcome(parsed, skill_result, sidecar_exists=True) + assert status == DispatchStatus.RESUMABLE + assert reason == FleetErrorCode.FLEET_L3_NO_RESULT_BLOCK + + def test_context_exhausted_with_progress_is_failure(self): + parsed, skill_result = _no_sentinel(session_id="sess-abc", lifespan_started=True) + skill_result = dataclasses.replace( + skill_result, + retry_reason="resume", + infra=InfraOutcome(exit_category="context_exhausted"), + ) + status, reason = classify_dispatch_outcome(parsed, skill_result, sidecar_exists=True) + assert status == DispatchStatus.FAILURE + assert reason == FleetErrorCode.FLEET_L3_NO_RESULT_BLOCK + + def test_api_error_with_progress_is_resumable(self): + parsed, skill_result = _no_sentinel(session_id="sess-abc", lifespan_started=True) + skill_result = dataclasses.replace( + skill_result, + retry_reason="resume", + infra=InfraOutcome(exit_category="api_error"), + ) + status, reason = classify_dispatch_outcome(parsed, skill_result, sidecar_exists=True) + assert status == DispatchStatus.RESUMABLE + assert reason == FleetErrorCode.FLEET_L3_NO_RESULT_BLOCK + + def test_process_killed_with_progress_is_resumable(self): + parsed, skill_result = _no_sentinel(session_id="sess-abc", lifespan_started=True) + skill_result = dataclasses.replace( + skill_result, + retry_reason="resume", + infra=InfraOutcome(exit_category="process_killed"), + ) + status, reason = classify_dispatch_outcome(parsed, skill_result, sidecar_exists=True) + assert status == DispatchStatus.RESUMABLE + assert reason == FleetErrorCode.FLEET_L3_NO_RESULT_BLOCK + + def test_thinking_stall_with_progress_is_failure(self): + parsed, skill_result = _no_sentinel(session_id="sess-abc", lifespan_started=True) + skill_result = dataclasses.replace( + skill_result, + retry_reason="thinking_stall", + ) + status, reason = classify_dispatch_outcome(parsed, skill_result, sidecar_exists=True) + assert status == DispatchStatus.FAILURE + assert reason == FleetErrorCode.FLEET_L3_NO_RESULT_BLOCK + + def test_stale_with_progress_is_failure(self): + parsed, skill_result = _no_sentinel(session_id="sess-abc", lifespan_started=True) + skill_result = dataclasses.replace( + skill_result, + retry_reason="stale", + ) + status, reason = classify_dispatch_outcome(parsed, skill_result, sidecar_exists=True) + assert status == DispatchStatus.FAILURE + assert reason == FleetErrorCode.FLEET_L3_NO_RESULT_BLOCK + + def test_no_progress_still_failure_regardless_of_reason(self): + parsed, skill_result = _no_sentinel(session_id="sess-abc", lifespan_started=True) + skill_result = dataclasses.replace( + skill_result, + retry_reason="idle_stall", + ) + status, reason = classify_dispatch_outcome( + parsed, skill_result, sidecar_exists=False, checkpoint=None + ) + assert status == DispatchStatus.FAILURE + assert reason == FleetErrorCode.FLEET_L3_NO_RESULT_BLOCK + + def test_no_lifespan_still_failure_regardless_of_reason(self): + parsed, skill_result = _no_sentinel(session_id="sess-abc", lifespan_started=False) + skill_result = dataclasses.replace( + skill_result, + retry_reason="idle_stall", + ) + status, reason = classify_dispatch_outcome(parsed, skill_result, sidecar_exists=True) + assert status == DispatchStatus.FAILURE + assert reason == FleetErrorCode.FLEET_L3_NO_RESULT_BLOCK diff --git a/tests/fleet/test_error_envelope.py b/tests/fleet/test_error_envelope.py new file mode 100644 index 0000000000..5b32b1bb4f --- /dev/null +++ b/tests/fleet/test_error_envelope.py @@ -0,0 +1,200 @@ +"""Tests for fleet error envelope registry and constructor. + +Group R: FleetErrorCode enum, FLEET_ERROR_CODES frozenset, and fleet_error() helper. +""" + +from __future__ import annotations + +import ast +import json +import re +from pathlib import Path + +import pytest + +pytestmark = [pytest.mark.layer("fleet"), pytest.mark.small, pytest.mark.feature("fleet")] + +SRC_ROOT = Path(__file__).parent.parent.parent / "src" / "autoskillit" + +# Matches error code strings in the fleet namespace +_FLEET_CODE_PATTERN = re.compile(r"^fleet_") + + +class TestFleetErrorCodeEnum: + def test_fleet_error_code_enum_has_expected_codes(self): + from autoskillit.core import FleetErrorCode + + expected_values = { + "fleet_parallel_refused", + "fleet_unknown_ingredient", + "fleet_recipe_not_found", + "fleet_invalid_recipe_kind", + "fleet_hard_refusal_headless", + "fleet_feature_disabled", + "fleet_manifest_missing", + "fleet_manifest_corrupted", + "fleet_lock_not_initialized", + "fleet_l3_timeout", + "fleet_l3_no_result_block", + "fleet_l3_parse_failed", + "fleet_l3_startup_or_crash", + "fleet_budget_exceeded", + "fleet_quota_exhausted", + "fleet_cleanup_failed", + "fleet_gate_unknown_dispatch", + "fleet_gate_already_recorded", + "fleet_gate_no_campaign", + "fleet_missing_ingredient", + "fleet_campaign_halted", + } + assert {c.value for c in FleetErrorCode} == expected_values + + def test_fleet_error_codes_frozenset_matches_enum(self): + from autoskillit.core import FLEET_ERROR_CODES, FleetErrorCode + + assert FLEET_ERROR_CODES == frozenset(FleetErrorCode) + + def test_fleet_error_code_values_are_snake_case(self): + from autoskillit.core import FleetErrorCode + + snake_case = re.compile(r"^[a-z][a-z0-9_]*$") + for code in FleetErrorCode: + assert snake_case.match(code.value), f"{code!r} is not snake_case" + + def test_fleet_error_code_enum_re_exported_from_core(self): + from autoskillit.core import FleetErrorCode # noqa: F401 + + assert FleetErrorCode is not None + + def test_fleet_error_code_all_values_have_fleet_prefix(self): + from autoskillit.core import FleetErrorCode + + for code in FleetErrorCode: + assert code.value.startswith("fleet_"), ( + f"Code {code.name!r} has value {code.value!r} without fleet_ prefix" + ) + + def test_fleet_error_code_old_unprefixed_values_removed(self): + from autoskillit.core import FleetErrorCode + + old_values = { + "l2_timeout", + "l2_no_result_block", + "l2_parse_failed", + "l2_startup_or_crash", + "l3_timeout", + "l3_no_result_block", + "l3_parse_failed", + "l3_startup_or_crash", + "dispatch_budget_exceeded", + "quota_exhausted", + "cleanup_failed", + } + current_values = {c.value for c in FleetErrorCode} + assert not (old_values & current_values), ( + f"Old unprefixed values still present: {old_values & current_values}" + ) + + +class TestFleetErrorHelper: + def test_fleet_error_rejects_unregistered_code(self): + from autoskillit.core import fleet_error + + with pytest.raises(ValueError, match="Unregistered fleet error code"): + fleet_error("bogus_code", "msg") + + def test_fleet_error_returns_valid_json_envelope(self): + from autoskillit.core import FleetErrorCode, fleet_error + + result = fleet_error(FleetErrorCode.FLEET_L3_TIMEOUT, "timed out") + data = json.loads(result) + assert set(data.keys()) >= {"success", "error", "user_visible_message", "details"} + assert data["success"] is False + assert data["error"] == "fleet_l3_timeout" + assert data["user_visible_message"] == "timed out" + + def test_fleet_error_details_default_none(self): + from autoskillit.core import FleetErrorCode, fleet_error + + result = json.loads(fleet_error(FleetErrorCode.FLEET_L3_TIMEOUT, "msg")) + assert result["details"] is None + + def test_fleet_error_details_json_serializable(self): + from autoskillit.core import FleetErrorCode, fleet_error + + result = json.loads( + fleet_error( + FleetErrorCode.FLEET_L3_TIMEOUT, + "msg", + details={"key": [1, 2]}, + ) + ) + assert result["details"] == {"key": [1, 2]} + + with pytest.raises(TypeError): + fleet_error( + FleetErrorCode.FLEET_L3_TIMEOUT, + "msg", + details={"fn": lambda: 0}, # type: ignore[arg-type] + ) + + def test_fleet_error_re_exported_from_pipeline(self): + import importlib + + pipeline = importlib.import_module("autoskillit.pipeline") + assert hasattr(pipeline, "fleet_error") + assert pipeline.fleet_error is not None + + +class TestFleetErrorASTScan: + def _find_raw_fleet_error_dumps(self, path: Path) -> list[str]: + """Find raw json.dumps({..., 'error': ''}) calls in a file. + + Returns a list of violation descriptions. Empty = clean. + """ + src = path.read_text() + try: + tree = ast.parse(src) + except SyntaxError: + return [] + + violations = [] + for node in ast.walk(tree): + if not isinstance(node, ast.Call): + continue + # Check if this is a json.dumps(...) call + func = node.func + is_json_dumps = ( + isinstance(func, ast.Attribute) + and func.attr == "dumps" + and isinstance(func.value, ast.Name) + and func.value.id == "json" + ) + if not is_json_dumps: + continue + if not node.args: + continue + dict_arg = node.args[0] + if not isinstance(dict_arg, ast.Dict): + continue + # Look for "error" key with a franchise-style string literal value + for key, value in zip(dict_arg.keys, dict_arg.values): + if not isinstance(key, ast.Constant) or key.value != "error": + continue + if not isinstance(value, ast.Constant): + continue + error_val = str(value.value) + if _FLEET_CODE_PATTERN.match(error_val): + violations.append( + f"{path.name}:{node.lineno}: raw json.dumps with error={error_val!r}" + ) + return violations + + def test_all_fleet_errors_use_registered_code(self): + all_violations: list[str] = [] + for f in sorted(SRC_ROOT.rglob("*.py")): + all_violations.extend(self._find_raw_fleet_error_dumps(f)) + assert not all_violations, ( + "Raw json.dumps fleet error patterns found. " + "Use fleet_error() instead:\n" + "\n".join(all_violations) + ) diff --git a/tests/fleet/test_findings_rpc.py b/tests/fleet/test_findings_rpc.py new file mode 100644 index 0000000000..e81ed1fbfb --- /dev/null +++ b/tests/fleet/test_findings_rpc.py @@ -0,0 +1,112 @@ +"""Tests for autoskillit.fleet._findings_rpc (T15–T21).""" + +from __future__ import annotations + +import json +from pathlib import Path + +import pytest + +from autoskillit.fleet._findings_rpc import load_execution_map, parse_and_resume +from autoskillit.fleet._sidecar_rpc import write_sidecar_entry + +pytestmark = [pytest.mark.layer("fleet"), pytest.mark.small, pytest.mark.feature("fleet")] + + +# T15 +def test_parse_and_resume_empty_sidecar(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("AUTOSKILLIT_DISPATCH_ID", "test-dispatch-001") + result = parse_and_resume( + "https://github.com/o/r/issues/1,https://github.com/o/r/issues/2", + project_dir=str(tmp_path), + ) + assert json.loads(result["remaining_urls_json"]) == [ + "https://github.com/o/r/issues/1", + "https://github.com/o/r/issues/2", + ] + assert result["completed_count"] == "0" + + +# T16 +def test_parse_and_resume_with_completed_sidecar( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + dispatch_id = "test-dispatch-002" + monkeypatch.setenv("AUTOSKILLIT_DISPATCH_ID", dispatch_id) + write_sidecar_entry( + dispatch_id, + "https://github.com/o/r/issues/1", + "completed", + project_dir=str(tmp_path), + ) + result = parse_and_resume( + "https://github.com/o/r/issues/1,https://github.com/o/r/issues/2", + project_dir=str(tmp_path), + ) + remaining = json.loads(result["remaining_urls_json"]) + assert remaining == ["https://github.com/o/r/issues/2"] + assert result["completed_count"] == "1" + + +# T17 +def test_parse_and_resume_single_url(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("AUTOSKILLIT_DISPATCH_ID", "test-dispatch-003") + result = parse_and_resume( + "https://github.com/o/r/issues/5", + project_dir=str(tmp_path), + ) + remaining = json.loads(result["remaining_urls_json"]) + assert remaining == ["https://github.com/o/r/issues/5"] + + +# T18 +def test_load_execution_map_valid(tmp_path: Path) -> None: + map_data = { + "groups": [ + { + "name": "g1", + "parallel": True, + "issues": ["https://github.com/o/r/issues/1"], + }, + ] + } + map_file = tmp_path / "bem_map.json" + map_file.write_text(json.dumps(map_data)) + result = load_execution_map(str(map_file)) + groups = json.loads(result["groups_json"]) + assert len(groups) == 1 + assert groups[0]["name"] == "g1" + assert groups[0]["parallel"] is True + assert result["total_groups"] == "1" + + +# T19 +def test_load_execution_map_missing_file(tmp_path: Path) -> None: + result = load_execution_map(str(tmp_path / "nonexistent.json")) + assert "error" in result + + +# T20 +def test_load_execution_map_invalid_json(tmp_path: Path) -> None: + bad_file = tmp_path / "bad.json" + bad_file.write_text("not json {{{") + result = load_execution_map(str(bad_file)) + assert "error" in result + + +# T21 +def test_parse_and_resume_reads_dispatch_id_from_env( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + dispatch_id = "env-dispatch-99" + monkeypatch.setenv("AUTOSKILLIT_DISPATCH_ID", dispatch_id) + write_sidecar_entry( + dispatch_id, "https://github.com/o/r/issues/7", "completed", project_dir=str(tmp_path) + ) + result = parse_and_resume( + "https://github.com/o/r/issues/7", + project_dir=str(tmp_path), + ) + # Proves the env dispatch_id was used: the pre-completed issue is filtered out + assert json.loads(result["remaining_urls_json"]) == [] + assert result["completed_count"] == "1" diff --git a/tests/fleet/test_fleet.py b/tests/fleet/test_fleet.py new file mode 100644 index 0000000000..5fc92496aa --- /dev/null +++ b/tests/fleet/test_fleet.py @@ -0,0 +1,54 @@ +"""Tests for fleet package.""" + +from __future__ import annotations + +import inspect + +import pytest + +pytestmark = [pytest.mark.layer("fleet"), pytest.mark.small, pytest.mark.feature("fleet")] + + +def test_fleet_package_importable() -> None: + """fleet package can be imported without error.""" + import autoskillit.fleet # noqa: F401 + + # Verify gateway exports are accessible + from autoskillit.fleet import CampaignState, DispatchRecord, read_state # noqa: F401 + + +def test_fleet_semaphore_satisfies_fleet_lock() -> None: + """FleetSemaphore is a structural match for FleetLock at runtime.""" + from autoskillit.core import FleetLock + from autoskillit.fleet import FleetSemaphore + + s = FleetSemaphore(max_concurrent=1) + assert isinstance(s, FleetLock) + assert inspect.iscoroutinefunction(s.acquire) + + +def test_fleet_lock_protocol_has_required_methods() -> None: + """FleetLock exposes at_capacity, acquire, release, active_count, max_concurrent.""" + from autoskillit.core import FleetLock + + members = {name for name, _ in inspect.getmembers(FleetLock) if not name.startswith("_")} + assert "at_capacity" in members + assert "acquire" in members + assert "release" in members + assert "active_count" in members + assert "max_concurrent" in members + assert "locked" not in members + + +def test_fleet_lock_acquire_is_coroutine() -> None: + """acquire() must be async — verifies FleetSemaphore.acquire is a coroutine.""" + from autoskillit.fleet import FleetSemaphore + + assert inspect.iscoroutinefunction(FleetSemaphore(max_concurrent=1).acquire) + + +def test_headless_executor_protocol_has_dispatch_food_truck() -> None: + """HeadlessExecutor protocol exposes dispatch_food_truck.""" + from autoskillit.core import HeadlessExecutor + + assert hasattr(HeadlessExecutor, "dispatch_food_truck") diff --git a/tests/fleet/test_fleet_e2e.py b/tests/fleet/test_fleet_e2e.py new file mode 100644 index 0000000000..255b07b81c --- /dev/null +++ b/tests/fleet/test_fleet_e2e.py @@ -0,0 +1,1006 @@ +"""Fleet Group O: end-to-end test suite for fleet dispatch loop. + +Exercises the full execute_dispatch → DefaultHeadlessExecutor.dispatch_food_truck +→ build_food_truck_cmd → _execute_claude_headless pipeline, substituting only +the claude binary with a parameterized Python shim. +""" + +from __future__ import annotations + +import asyncio +import json +import os +import signal +import subprocess +import threading +import time +from collections.abc import Generator +from pathlib import Path +from typing import Any +from uuid import uuid4 + +import anyio +import psutil +import pytest + +pytestmark = [ + pytest.mark.layer("fleet"), + pytest.mark.medium, + pytest.mark.integration, + pytest.mark.feature("fleet"), +] + + +# --------------------------------------------------------------------------- +# Claude shim script +# --------------------------------------------------------------------------- + +_SHIM_SCRIPT = """\ +#!/usr/bin/env python3 +\"\"\"Parameterized claude shim for fleet E2E tests.\"\"\" +import json +import os +import sys +import time + +dispatch_id = os.environ.get("AUTOSKILLIT_DISPATCH_ID", "unknown") +mode = os.environ.get("CLAUDE_SHIM_MODE", "success") +sleep_sec = float(os.environ.get("CLAUDE_SHIM_SLEEP_SEC", "10")) + + +def _sentinel(payload: str) -> str: + return ( + f"---l3-result::{dispatch_id}---\\n" + f"{payload}\\n" + f"---end-l3-result::{dispatch_id}---" + ) + + +if mode == "exit_nonzero": + sys.exit(1) +elif mode == "success": + text = _sentinel('{"success": true, "reason": ""}') +elif mode == "malformed_sentinel": + text = _sentinel("NOT VALID JSON") +elif mode == "no_sentinel": + text = "Task completed without sentinel." +elif mode == "sleep_then_exit": + time.sleep(sleep_sec) + text = _sentinel('{"success": true, "reason": ""}') +else: + text = "" + +envelope = { + "type": "result", + "subtype": "success", + "is_error": False, + "result": text, + "session_id": "test-session-id", + "errors": [], + "usage": {"input_tokens": 0, "output_tokens": 0}, +} +print(json.dumps(envelope), flush=True) +""" + + +def _write_claude_shim(bin_dir: Path) -> Path: + """Write a Python shim to bin_dir/claude and make it executable.""" + bin_dir.mkdir(parents=True, exist_ok=True) + shim_path = bin_dir / "claude" + shim_path.write_text(_SHIM_SCRIPT, encoding="utf-8") + shim_path.chmod(0o755) + return shim_path + + +# --------------------------------------------------------------------------- +# Stub callables (module-level, not fixtures) +# --------------------------------------------------------------------------- + + +def _simple_prompt_builder(**kwargs: Any) -> str: + return f"dispatch {kwargs.get('recipe', 'unknown')} for {kwargs.get('task', 'test')}" + + +async def _no_sleep_quota_checker(config: Any, **kwargs: Any) -> dict[str, Any]: + return { + "should_sleep": False, + "sleep_seconds": 0, + "utilization": None, + "resets_at": None, + "window_name": None, + } + + +async def _noop_quota_refresher(config: Any, **kwargs: Any) -> None: + pass + + +# --------------------------------------------------------------------------- +# FleetTestRunner +# --------------------------------------------------------------------------- + + +class FleetTestRunner: + """SubprocessRunner-conforming runner that spawns real subprocesses. + + Used by FleetRuntime to exercise the full dispatch pipeline with + the claude shim binary, avoiding any mocking of the headless executor. + """ + + def __init__(self) -> None: + self.call_count: int = 0 + self.last_pid: int = 0 + + async def __call__( + self, + cmd: list[str], + *, + cwd: Any, + timeout: float, + env: Any = None, + on_pid_resolved: Any = None, + **kwargs: Any, + ) -> Any: + from autoskillit.core.runtime._linux_proc import read_starttime_ticks + from autoskillit.core.types._type_enums import ( + ChannelConfirmation, + KillReason, + TerminationReason, + ) + from autoskillit.core.types._type_subprocess import SubprocessResult + from autoskillit.execution import kill_process_tree + + self.call_count += 1 + proc = subprocess.Popen( + cmd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + env=env, + cwd=str(cwd), + ) + self.last_pid = proc.pid + if on_pid_resolved is not None: + ticks = read_starttime_ticks(proc.pid) or 0 + on_pid_resolved(proc.pid, ticks) + + try: + stdout_b, stderr_b = await asyncio.to_thread(lambda: proc.communicate(timeout=timeout)) + except subprocess.TimeoutExpired: + await asyncio.to_thread(kill_process_tree, proc.pid) + await asyncio.to_thread(proc.wait) + return SubprocessResult( + returncode=-9, + stdout="", + stderr="", + termination=TerminationReason.TIMED_OUT, + pid=proc.pid, + channel_confirmation=ChannelConfirmation.UNMONITORED, + kill_reason=KillReason.INFRA_KILL, + ) + + return SubprocessResult( + returncode=proc.returncode, + stdout=stdout_b.decode("utf-8", errors="replace"), + stderr=stderr_b.decode("utf-8", errors="replace"), + termination=TerminationReason.NATURAL_EXIT, + pid=proc.pid, + channel_confirmation=ChannelConfirmation.UNMONITORED, + kill_reason=KillReason.NATURAL_EXIT, + ) + + +# --------------------------------------------------------------------------- +# FleetRuntime helper +# --------------------------------------------------------------------------- + + +class FleetRuntime: + """Test harness for end-to-end fleet dispatch tests. + + Wires FleetTestRunner into ToolContext, provides helper methods + for configuring shim behavior and reading per-dispatch state files. + """ + + def __init__( + self, + tool_ctx: Any, + dispatches_dir: Path, + shim_dir: Path, + runner: FleetTestRunner, + recipes: Any, + monkeypatch: Any, + ) -> None: + self.tool_ctx = tool_ctx + self.dispatches_dir = dispatches_dir + self.shim_dir = shim_dir + self.runner = runner + self.recipes = recipes + self._monkeypatch = monkeypatch + + def configure_shim(self, mode: str, sleep_sec: float | None = None) -> None: + """Set CLAUDE_SHIM_MODE (and optionally CLAUDE_SHIM_SLEEP_SEC) for next dispatch.""" + self._monkeypatch.setenv("CLAUDE_SHIM_MODE", mode) + if sleep_sec is not None: + self._monkeypatch.setenv("CLAUDE_SHIM_SLEEP_SEC", str(sleep_sec)) + + def add_recipe(self, name: str) -> None: + """Register a minimal standard recipe.""" + from autoskillit.recipe.schema import Recipe, RecipeInfo, RecipeKind, RecipeSource + + info = RecipeInfo( + name=name, + description="test", + source=RecipeSource.PROJECT, + path=Path(f"/fake/{name}.yaml"), + ) + self.recipes.add_recipe(name, info) + self.recipes.add_full_recipe( + info.path, + Recipe(name=name, description="test", kind=RecipeKind.STANDARD, ingredients={}), + ) + + async def dispatch( + self, + recipe: str, + task: str = "test-task", + *, + ingredients: dict[str, str] | None = None, + dispatch_name: str | None = None, + timeout_sec: int | None = None, + shim_mode: str = "success", + sleep_sec: float | None = None, + quota_checker: Any = None, + ) -> dict[str, Any]: + """Run execute_dispatch and return the parsed JSON envelope.""" + from autoskillit.fleet._api import execute_dispatch + + self.configure_shim(shim_mode, sleep_sec=sleep_sec) + raw = await execute_dispatch( + tool_ctx=self.tool_ctx, + recipe=recipe, + task=task, + ingredients=ingredients, # type: ignore[arg-type] + dispatch_name=dispatch_name, + timeout_sec=timeout_sec, + prompt_builder=_simple_prompt_builder, + quota_checker=quota_checker if quota_checker is not None else _no_sleep_quota_checker, + quota_refresher=_noop_quota_refresher, + ) + return json.loads(raw) # type: ignore[no-any-return] + + def dispatch_state_path(self, dispatch_id: str) -> Path: + """Path to per-dispatch state file created by execute_dispatch.""" + return self.dispatches_dir / f"{dispatch_id}.json" + + def read_dispatch_state(self, dispatch_id: str) -> Any: + """Read per-dispatch CampaignState (or None if missing/corrupt).""" + from autoskillit.fleet.state import read_state + + return read_state(self.dispatch_state_path(dispatch_id)) + + +# --------------------------------------------------------------------------- +# State manipulation helpers (bypass transition validation) +# --------------------------------------------------------------------------- + + +def _force_running_state( + state_path: Path, + name: str, + pid: int, + ticks: int, + boot_id: str, +) -> None: + """Directly set a dispatch to RUNNING with PID identity fields in the JSON.""" + data = json.loads(state_path.read_text(encoding="utf-8")) + for d in data["dispatches"]: + if d["name"] == name: + d["status"] = "running" + d["dispatched_pid"] = pid + d["dispatched_starttime_ticks"] = ticks + d["dispatched_boot_id"] = boot_id + d["started_at"] = time.time() + break + state_path.write_text(json.dumps(data), encoding="utf-8") + + +def _force_state_statuses(state_path: Path, overrides: dict[str, str]) -> None: + """Bulk override dispatch statuses, bypassing transition validation.""" + data = json.loads(state_path.read_text(encoding="utf-8")) + for d in data["dispatches"]: + if d["name"] in overrides: + d["status"] = overrides[d["name"]] + state_path.write_text(json.dumps(data), encoding="utf-8") + + +# --------------------------------------------------------------------------- +# fleet_runtime fixture (local — avoids conflict with test_pack_enforcement_e2e.py) +# --------------------------------------------------------------------------- + + +@pytest.fixture +def fleet_runtime( + tmp_path: Path, monkeypatch: Any, tool_ctx: Any +) -> Generator[FleetRuntime, None, None]: + from autoskillit.execution.headless import DefaultHeadlessExecutor + from tests.fakes import InMemoryRecipeRepository + + shim_dir = tmp_path / "bin" + _write_claude_shim(shim_dir) + monkeypatch.setenv("PATH", f"{shim_dir}:{os.environ['PATH']}") + + from autoskillit.fleet import FleetSemaphore + + runner = FleetTestRunner() + tool_ctx.runner = runner + tool_ctx.executor = DefaultHeadlessExecutor(tool_ctx) + tool_ctx.fleet_lock = FleetSemaphore(max_concurrent=1) + recipes = InMemoryRecipeRepository() + tool_ctx.recipes = recipes + tool_ctx.kitchen_id = uuid4().hex[:16] + tool_ctx.project_dir = tmp_path + + dispatches_dir = tool_ctx.temp_dir / "dispatches" + dispatches_dir.mkdir(parents=True, exist_ok=True) + + pre_children = {c.pid for c in psutil.Process(os.getpid()).children(recursive=True)} + + rt = FleetRuntime( + tool_ctx=tool_ctx, + dispatches_dir=dispatches_dir, + shim_dir=shim_dir, + runner=runner, + recipes=recipes, + monkeypatch=monkeypatch, + ) + + yield rt + + # Process leak detection + post_children = psutil.Process(os.getpid()).children(recursive=True) + leaked = [] + for c in post_children: + if c.pid not in pre_children: + try: + if c.is_running() and c.status() not in ( + psutil.STATUS_ZOMBIE, + psutil.STATUS_DEAD, + ): + leaked.append(c) + except (psutil.NoSuchProcess, psutil.AccessDenied): + pass + for c in leaked: + try: + c.kill() + c.wait(timeout=2) + except (psutil.NoSuchProcess, psutil.AccessDenied): + pass + assert not leaked, f"Test leaked processes: {[c.pid for c in leaked]}" + + +@pytest.fixture +def fleet_runtime_factory(fleet_runtime: FleetRuntime): + """Factory variant of fleet_runtime that accepts max_concurrent_dispatches.""" + + def _factory(max_concurrent_dispatches: int = 1) -> FleetRuntime: + from autoskillit.fleet import FleetSemaphore + + fleet_runtime.tool_ctx.fleet_lock = FleetSemaphore( + max_concurrent=max_concurrent_dispatches + ) + return fleet_runtime + + return _factory + + +# --------------------------------------------------------------------------- +# Tests 1–3, 5, 9: dispatch-pipeline (happy path, failure, continue-on-failure) +# --------------------------------------------------------------------------- + + +@pytest.mark.anyio +async def test_two_dispatch_happy_path(fleet_runtime: FleetRuntime) -> None: + """Full dispatch pipeline runs twice and both succeed.""" + rt = fleet_runtime + rt.add_recipe("recipe-a") + rt.add_recipe("recipe-b") + + result_a = await rt.dispatch("recipe-a", shim_mode="success") + assert result_a["success"] is True + + result_b = await rt.dispatch("recipe-b", shim_mode="success") + assert result_b["success"] is True + + for result in [result_a, result_b]: + state = rt.read_dispatch_state(result["dispatch_id"]) + assert state is not None + d = state.dispatches[0] + from autoskillit.fleet.state import DispatchStatus + + assert d.status == DispatchStatus.SUCCESS + assert d.dispatched_pid > 0 + assert d.ended_at > d.started_at + + +@pytest.mark.anyio +async def test_halt_on_first_failure_default(fleet_runtime: FleetRuntime) -> None: + """Failure detection + campaign halt with continue_on_failure=False.""" + from autoskillit.fleet.state import DispatchStatus, resume_campaign_from_state + + rt = fleet_runtime + rt.add_recipe("recipe-a") + + result = await rt.dispatch("recipe-a", shim_mode="exit_nonzero") + assert result["success"] is False + + state_path = rt.dispatch_state_path(result["dispatch_id"]) + decision = resume_campaign_from_state(state_path, continue_on_failure=False) + assert decision is not None + assert decision.next_dispatch_name == "" + assert decision.completed_dispatches_block == "fleet_halted_on_failure" + + state = rt.read_dispatch_state(result["dispatch_id"]) + assert state is not None + assert state.dispatches[0].status == DispatchStatus.FAILURE + + +@pytest.mark.anyio +async def test_continue_on_failure_when_flagged(fleet_runtime: FleetRuntime) -> None: + """continue_on_failure=True does not halt; FAILURE dispatches excluded from next selection.""" + from autoskillit.fleet.state import ( + FLEET_HALTED_SENTINEL, + DispatchStatus, + resume_campaign_from_state, + ) + + rt = fleet_runtime + rt.add_recipe("recipe-a") + rt.add_recipe("recipe-b") + + result_a = await rt.dispatch("recipe-a", shim_mode="no_sentinel") + assert result_a["success"] is False + assert result_a["reason"] == "fleet_l3_no_result_block" + + state_path = rt.dispatch_state_path(result_a["dispatch_id"]) + decision = resume_campaign_from_state(state_path, continue_on_failure=True) + assert decision is not None + # Campaign did not halt (FLEET_HALTED_SENTINEL not emitted) + assert decision.completed_dispatches_block != FLEET_HALTED_SENTINEL + # FAILURE dispatches are excluded from next_dispatch_name selection; + # with only one failed dispatch and no pending ones, next_name is empty. + assert decision.next_dispatch_name == "" + + result_b = await rt.dispatch("recipe-b", shim_mode="success") + assert result_b["success"] is True + + state_b = rt.read_dispatch_state(result_b["dispatch_id"]) + assert state_b is not None + assert state_b.dispatches[0].status == DispatchStatus.SUCCESS + + +@pytest.mark.anyio +async def test_malformed_l3_result_surfaces_warning(fleet_runtime: FleetRuntime) -> None: + """Malformed sentinel body produces l3_parse_failed failure with diagnostic fields.""" + from autoskillit.fleet.state import DispatchStatus + + rt = fleet_runtime + rt.add_recipe("recipe-a") + + result = await rt.dispatch("recipe-a", shim_mode="malformed_sentinel") + assert result["success"] is False + assert result["reason"] == "fleet_l3_parse_failed" + assert "l3_raw_body" in result + assert "l3_parse_error" in result + + state = rt.read_dispatch_state(result["dispatch_id"]) + assert state is not None + assert state.dispatches[0].status == DispatchStatus.FAILURE + assert state.dispatches[0].reason == "fleet_l3_parse_failed" + + +@pytest.mark.anyio +async def test_l3_halts_on_missing_result_block_when_continue_on_failure_false( + fleet_runtime: FleetRuntime, +) -> None: + """No-sentinel failure + continue_on_failure=False yields fleet_halted_on_failure.""" + from autoskillit.fleet.state import DispatchStatus, resume_campaign_from_state + + rt = fleet_runtime + rt.add_recipe("recipe-a") + + result = await rt.dispatch("recipe-a", shim_mode="no_sentinel") + assert result["success"] is False + assert result["reason"] == "fleet_l3_no_result_block" + + state = rt.read_dispatch_state(result["dispatch_id"]) + assert state is not None + assert state.dispatches[0].status == DispatchStatus.FAILURE + assert state.dispatches[0].reason == "fleet_l3_no_result_block" + + state_path = rt.dispatch_state_path(result["dispatch_id"]) + decision = resume_campaign_from_state(state_path, continue_on_failure=False) + assert decision is not None + assert decision.next_dispatch_name == "" + assert decision.completed_dispatches_block == "fleet_halted_on_failure" + + +# --------------------------------------------------------------------------- +# Tests 4, 10, 12, 13: concurrency and validation +# --------------------------------------------------------------------------- + + +@pytest.mark.anyio +async def test_parallel_dispatch_refused_mid_campaign( + fleet_runtime: FleetRuntime, +) -> None: + """Lock guard rejects second concurrent dispatch with fleet_parallel_refused.""" + rt = fleet_runtime + rt.add_recipe("slow-recipe") + + results: list[dict[str, Any] | None] = [None, None] + + async def _first() -> None: + results[0] = await rt.dispatch( + "slow-recipe", + shim_mode="sleep_then_exit", + sleep_sec=2.0, + timeout_sec=10, + ) + + async def _second() -> None: + await anyio.sleep(0.3) + results[1] = await rt.dispatch("slow-recipe", shim_mode="success") + + async with anyio.create_task_group() as tg: + tg.start_soon(_first) + tg.start_soon(_second) + + assert results[0] is not None + assert results[0]["success"] is True + + assert results[1] is not None + assert results[1]["error"] == "fleet_parallel_refused" + assert results[1]["success"] is False + + +@pytest.mark.anyio +async def test_state_json_atomic_under_concurrent_read( + fleet_runtime: FleetRuntime, tmp_path: Path +) -> None: + """atomic_write guarantees readers never observe corrupted partial JSON.""" + from autoskillit.fleet.state import DispatchRecord, write_initial_state + + state_path = tmp_path / "atomic-test-state.json" + write_initial_state(state_path, "cid-0", "cn", str(state_path), [DispatchRecord(name="d0")]) + + json_errors: list[Exception] = [] + stop_event = threading.Event() + + def _writer() -> None: + i = 0 + while not stop_event.is_set(): + dispatches = [DispatchRecord(name=f"d{i % 5}")] + write_initial_state(state_path, f"cid-{i % 100}", "cn", str(state_path), dispatches) + i += 1 + + writer_thread = threading.Thread(target=_writer, daemon=True) + writer_thread.start() + + deadline = time.monotonic() + 2.0 + while time.monotonic() < deadline: + try: + json.loads(state_path.read_text(encoding="utf-8")) + except json.JSONDecodeError as exc: + json_errors.append(exc) + + stop_event.set() + writer_thread.join(timeout=5.0) + + assert not json_errors, f"JSONDecodeError under concurrent writes: {json_errors[:3]}" + + +@pytest.mark.anyio +async def test_ingredient_type_validation(fleet_runtime: FleetRuntime) -> None: + """Non-string ingredient values are rejected before any subprocess is spawned.""" + rt = fleet_runtime + rt.add_recipe("recipe-a") + + result = await rt.dispatch("recipe-a", ingredients={"key": 123}) # type: ignore[arg-type] + assert result["success"] is False + assert result["error"] == "fleet_unknown_ingredient" + assert rt.runner.call_count == 0 + + +@pytest.mark.anyio +async def test_quota_exhausted_mid_campaign_sleeps_and_retries_once( + fleet_runtime: FleetRuntime, +) -> None: + """Quota sleep is honored before dispatch proceeds to completion.""" + rt = fleet_runtime + rt.add_recipe("recipe-a") + + call_count = [0] + + async def _stateful_quota_checker(config: Any, **kwargs: Any) -> dict[str, Any]: + call_count[0] += 1 + if call_count[0] == 1: + return { + "should_sleep": True, + "sleep_seconds": 0.1, + "utilization": None, + "resets_at": None, + "window_name": None, + } + return { + "should_sleep": False, + "sleep_seconds": 0, + "utilization": None, + "resets_at": None, + "window_name": None, + } + + t0 = time.monotonic() + result = await rt.dispatch( + "recipe-a", + shim_mode="success", + quota_checker=_stateful_quota_checker, + ) + elapsed = time.monotonic() - t0 + + assert result["success"] is True + assert elapsed >= 0.1, f"Expected quota sleep ≥ 0.1s, got {elapsed:.3f}s" + assert call_count[0] >= 1 + + +# --------------------------------------------------------------------------- +# Tests 6, 8, 11: process lifecycle +# --------------------------------------------------------------------------- + + +@pytest.mark.anyio +async def test_l3_killed_mid_dispatch_records_failure( + fleet_runtime: FleetRuntime, +) -> None: + """L3 process killed mid-dispatch produces l3_no_result_block failure (not crash).""" + from autoskillit.fleet.state import DispatchStatus + + rt = fleet_runtime + rt.add_recipe("sleepy-recipe") + + dispatch_result: dict[str, Any] | None = None + + async def _dispatch() -> None: + nonlocal dispatch_result + dispatch_result = await rt.dispatch( + "sleepy-recipe", + shim_mode="sleep_then_exit", + sleep_sec=30, + timeout_sec=60, + ) + + async def _killer() -> None: + await anyio.sleep(0.5) + pid = rt.runner.last_pid + if pid > 0: + os.kill(pid, signal.SIGKILL) + + async with anyio.create_task_group() as tg: + tg.start_soon(_dispatch) + tg.start_soon(_killer) + + assert dispatch_result is not None + assert dispatch_result["success"] is False + assert dispatch_result["reason"] == "fleet_l3_no_result_block" + + state = rt.read_dispatch_state(dispatch_result["dispatch_id"]) + assert state is not None + assert state.dispatches[0].status == DispatchStatus.FAILURE + assert state.dispatches[0].reason == "fleet_l3_no_result_block" + + killed_pid = rt.runner.last_pid + assert killed_pid > 0 + assert not psutil.pid_exists(killed_pid) or _is_zombie(killed_pid) + + +def _is_zombie(pid: int) -> bool: + """Return True if the process is a zombie (already dead, reap pending).""" + try: + return psutil.Process(pid).status() == psutil.STATUS_ZOMBIE + except (psutil.NoSuchProcess, psutil.AccessDenied): + return True + + +@pytest.mark.anyio +async def test_orphan_l3_reaping(fleet_runtime: FleetRuntime, tmp_path: Path) -> None: + """_reap_stale_dispatches kills a real orphan process and marks it interrupted.""" + from autoskillit.cli.fleet import _reap_stale_dispatches + from autoskillit.core.runtime._linux_proc import read_boot_id, read_starttime_ticks + from autoskillit.fleet.state import ( + DispatchRecord, + DispatchStatus, + read_state, + write_initial_state, + ) + + orphan = subprocess.Popen(["sleep", "999"]) + orphan_pid = orphan.pid + orphan_ticks = read_starttime_ticks(orphan_pid) or 0 + boot_id = read_boot_id() or "" + + try: + state_path = tmp_path / "orphan-test-state.json" + write_initial_state( + state_path, + "test-campaign", + "test", + str(state_path), + [DispatchRecord(name="orphaned")], + ) + _force_running_state(state_path, "orphaned", orphan_pid, orphan_ticks, boot_id) + + _reap_stale_dispatches(state_path, dry_run=False) + + assert not psutil.pid_exists(orphan_pid) or _is_zombie(orphan_pid) + + state = read_state(state_path) + assert state is not None + d = next(d for d in state.dispatches if d.name == "orphaned") + assert d.status == DispatchStatus.INTERRUPTED + assert d.reason == "reaped_orphan" + finally: + if orphan.poll() is None: + orphan.kill() + orphan.wait() + + +@pytest.mark.anyio +async def test_l3_timeout_enforced(fleet_runtime: FleetRuntime) -> None: + """timeout_sec=1 kills a sleeping L3 process and returns l3_timeout fleet_error.""" + from autoskillit.fleet.state import DispatchStatus + + rt = fleet_runtime + rt.add_recipe("slow-recipe") + + result = await rt.dispatch( + "slow-recipe", + shim_mode="sleep_then_exit", + sleep_sec=10, + timeout_sec=1, + ) + + assert result["success"] is False + assert result["error"] == "fleet_l3_timeout" + assert "details" in result + assert "dispatch_id" in result["details"] + assert "dispatched_session_id" in result["details"] + + state = rt.read_dispatch_state(result["details"]["dispatch_id"]) + assert state is not None + assert state.dispatches[0].status == DispatchStatus.FAILURE + assert state.dispatches[0].reason == "fleet_l3_timeout" + + killed_pid = rt.runner.last_pid + assert killed_pid > 0 + assert not psutil.pid_exists(killed_pid) or _is_zombie(killed_pid) + + +# --------------------------------------------------------------------------- +# Tests 7, 14, 15: state and manifest edge cases +# --------------------------------------------------------------------------- + + +@pytest.mark.anyio +async def test_resume_after_l3_crash(fleet_runtime: FleetRuntime, tmp_path: Path) -> None: + """resume_campaign_from_state marks stale RUNNING as interrupted and returns next pending.""" + from autoskillit.fleet.state import ( + DispatchRecord, + DispatchStatus, + read_state, + resume_campaign_from_state, + write_initial_state, + ) + + state_path = tmp_path / "crash-test-state.json" + dispatches = [ + DispatchRecord(name="completed-one"), + DispatchRecord(name="crashed-one"), + DispatchRecord(name="pending-one"), + ] + write_initial_state( + state_path, "test-campaign-id", "test-campaign", str(state_path), dispatches + ) + _force_state_statuses(state_path, {"completed-one": "success", "crashed-one": "running"}) + + decision = resume_campaign_from_state(state_path, continue_on_failure=True) + + assert decision is not None + assert decision.next_dispatch_name == "pending-one" + + state = read_state(state_path) + assert state is not None + crashed = next(d for d in state.dispatches if d.name == "crashed-one") + assert crashed.status == DispatchStatus.INTERRUPTED + assert crashed.reason == "stale_running_on_resume" + + +@pytest.mark.anyio +async def test_manifest_corrupted_yaml(fleet_runtime: FleetRuntime) -> None: + """Recipe with wrong kind returns fleet_invalid_recipe_kind without spawning.""" + from autoskillit.recipe.schema import Recipe, RecipeInfo, RecipeKind, RecipeSource + + rt = fleet_runtime + recipe_info = RecipeInfo( + name="bad-recipe", + description="bad", + source=RecipeSource.PROJECT, + path=Path("/fake/bad-recipe.yaml"), + ) + rt.recipes.add_recipe("bad-recipe", recipe_info) + rt.recipes.add_full_recipe( + recipe_info.path, + Recipe(name="bad-recipe", description="bad", kind=RecipeKind.CAMPAIGN, ingredients={}), + ) + + result = await rt.dispatch("bad-recipe") + assert result["success"] is False + assert result["error"] == "fleet_invalid_recipe_kind" + assert rt.runner.call_count == 0 + + +@pytest.mark.anyio +async def test_manifest_mid_campaign_deletion(fleet_runtime: FleetRuntime, tmp_path: Path) -> None: + """resume_campaign_from_state returns None when state file is missing.""" + from autoskillit.fleet.state import ( + DispatchRecord, + resume_campaign_from_state, + write_initial_state, + ) + + state_path = tmp_path / "missing-state.json" + write_initial_state( + state_path, + "test-campaign", + "test", + str(state_path), + [DispatchRecord(name="d1")], + ) + state_path.unlink() + + decision = resume_campaign_from_state(state_path, continue_on_failure=True) + assert decision is None + + +# --------------------------------------------------------------------------- +# Parallel dispatch tests — FleetSemaphore with max > 1 +# --------------------------------------------------------------------------- + + +@pytest.mark.anyio +async def test_two_concurrent_dispatches_allowed_with_max2( + fleet_runtime_factory, +) -> None: + """FleetSemaphore(max=2) allows both L3 dispatches to complete successfully.""" + rt = fleet_runtime_factory(max_concurrent_dispatches=2) + rt.add_recipe("slow-recipe") + results: list[dict | None] = [None, None] + + async def _dispatch(idx: int) -> None: + results[idx] = await rt.dispatch( + "slow-recipe", shim_mode="sleep_then_exit", sleep_sec=1.0, timeout_sec=10 + ) + + async with anyio.create_task_group() as tg: + tg.start_soon(_dispatch, 0) + tg.start_soon(_dispatch, 1) + + assert results[0] is not None and results[0]["success"] is True + assert results[1] is not None and results[1]["success"] is True + + +@pytest.mark.anyio +async def test_third_concurrent_dispatch_refused_with_max2( + fleet_runtime_factory, +) -> None: + """FleetSemaphore(max=2) rejects a third concurrent dispatch immediately.""" + rt = fleet_runtime_factory(max_concurrent_dispatches=2) + rt.add_recipe("slow-recipe") + results: list[dict | None] = [None, None, None] + + async def _slow(idx: int) -> None: + results[idx] = await rt.dispatch( + "slow-recipe", shim_mode="sleep_then_exit", sleep_sec=3.0, timeout_sec=15 + ) + + async def _fast() -> None: + await anyio.sleep(0.3) + results[2] = await rt.dispatch("slow-recipe", shim_mode="success") + + async with anyio.create_task_group() as tg: + tg.start_soon(_slow, 0) + tg.start_soon(_slow, 1) + tg.start_soon(_fast) + + assert results[0] is not None and results[0]["success"] is True + assert results[1] is not None and results[1]["success"] is True + assert results[2] is not None + assert results[2]["error"] == "fleet_parallel_refused" + assert results[2]["success"] is False + + +@pytest.mark.anyio +async def test_fourth_concurrent_dispatch_refused_with_max3( + fleet_runtime_factory, +) -> None: + """FleetSemaphore(max=3) rejects a fourth concurrent dispatch immediately.""" + rt = fleet_runtime_factory(max_concurrent_dispatches=3) + rt.add_recipe("slow-recipe") + results: list[dict | None] = [None, None, None, None] + + async def _slow(idx: int) -> None: + results[idx] = await rt.dispatch( + "slow-recipe", shim_mode="sleep_then_exit", sleep_sec=3.0, timeout_sec=15 + ) + + async def _fast() -> None: + await anyio.sleep(0.3) + results[3] = await rt.dispatch("slow-recipe", shim_mode="success") + + async with anyio.create_task_group() as tg: + tg.start_soon(_slow, 0) + tg.start_soon(_slow, 1) + tg.start_soon(_slow, 2) + tg.start_soon(_fast) + + assert results[0] is not None and results[0]["success"] is True + assert results[1] is not None and results[1]["success"] is True + assert results[2] is not None and results[2]["success"] is True + assert results[3] is not None + assert results[3]["error"] == "fleet_parallel_refused" + assert results[3]["success"] is False + + +# --------------------------------------------------------------------------- +# Test: resume_session_id threads through to the subprocess cmd +# --------------------------------------------------------------------------- + + +@pytest.mark.anyio +async def test_end_to_end_resumable_dispatch_uses_resume_flag( + fleet_runtime: FleetRuntime, monkeypatch: Any +) -> None: + """resume_session_id reaches build_food_truck_cmd and produces --resume in the cmd.""" + import autoskillit.execution.headless as _headless_mod + from autoskillit.fleet._api import execute_dispatch + + rt = fleet_runtime + rt.add_recipe("recipe-a") + + captured_cmds: list[list[str]] = [] + original_build = _headless_mod.build_food_truck_cmd + + def _spy_build(**kwargs): + spec = original_build(**kwargs) + captured_cmds.append(list(spec.cmd)) + return spec + + monkeypatch.setattr(_headless_mod, "build_food_truck_cmd", _spy_build) + rt.configure_shim("success") + + await execute_dispatch( + tool_ctx=rt.tool_ctx, + recipe="recipe-a", + task="test-task", + ingredients=None, + dispatch_name=None, + timeout_sec=None, + prompt_builder=_simple_prompt_builder, + quota_checker=_no_sleep_quota_checker, + quota_refresher=_noop_quota_refresher, + resume_session_id="test-session-id", + ) + + assert captured_cmds, "build_food_truck_cmd was never called" + cmd = captured_cmds[0] + assert "--resume" in cmd + idx = cmd.index("--resume") + assert cmd[idx + 1] == "test-session-id" diff --git a/tests/fleet/test_fleet_rename_integrity.py b/tests/fleet/test_fleet_rename_integrity.py new file mode 100644 index 0000000000..0561b0e025 --- /dev/null +++ b/tests/fleet/test_fleet_rename_integrity.py @@ -0,0 +1,109 @@ +import pytest + +pytestmark = [pytest.mark.layer("fleet"), pytest.mark.small, pytest.mark.feature("fleet")] + + +def test_fleet_menu_tools_importable_from_core() -> None: + from autoskillit.core import FLEET_MENU_TOOLS + + assert FLEET_MENU_TOOLS == ("dispatch_food_truck", "record_gate_dispatch") + + +def test_campaign_summary_importable_from_fleet() -> None: + from autoskillit.fleet import CampaignSummary, parse_campaign_summary + + assert CampaignSummary is not None + assert callable(parse_campaign_summary) + + +def test_fleet_error_code_class_exists() -> None: + from autoskillit.core import FleetErrorCode + + assert "fleet_parallel_refused" in FleetErrorCode._value2member_map_ + + +def test_fleet_error_code_class_gone() -> None: + with pytest.raises(ImportError): + from autoskillit.core import FranchiseErrorCode # noqa: F401 + + +def test_fleet_lock_protocol_exists() -> None: + from autoskillit.core import FleetLock + + assert FleetLock is not None + + +def test_fleet_error_helper_exists() -> None: + from autoskillit.core import fleet_error + + assert callable(fleet_error) + + +def test_fleet_error_helper_gone() -> None: + with pytest.raises(ImportError): + from autoskillit.core import franchise_error # noqa: F401 + + +def test_feature_registry_fleet_tool_tags() -> None: + from autoskillit.core import FEATURE_REGISTRY + + assert FEATURE_REGISTRY["fleet"].tool_tags == frozenset({"fleet"}) + + +def test_feature_registry_fleet_import_package() -> None: + from autoskillit.core import FEATURE_REGISTRY + + assert FEATURE_REGISTRY["fleet"].import_package == "autoskillit.fleet" + + +def test_feature_registry_franchise_entry_gone() -> None: + from autoskillit.core import FEATURE_REGISTRY + + assert "franchise" not in FEATURE_REGISTRY + + +def test_session_type_no_franchise() -> None: + from autoskillit.core.types._type_enums import SessionType + + assert not hasattr(SessionType, "FRANCHISE") + + +def test_session_type_fleet_canonical() -> None: + from autoskillit.core.types._type_enums import SessionType + + assert SessionType.FLEET.value == "fleet" + + +def test_hook_registry_fleet_dispatch_guard() -> None: + from autoskillit.hook_registry import HOOK_REGISTRY + + scripts = [s for hd in HOOK_REGISTRY for s in hd.scripts] + assert "guards/fleet_dispatch_guard.py" in scripts + assert "franchise_dispatch_guard.py" not in scripts + + +def test_retired_basenames_includes_franchise_guard() -> None: + from autoskillit.hook_registry import RETIRED_SCRIPT_BASENAMES + + assert "franchise_dispatch_guard.py" in RETIRED_SCRIPT_BASENAMES + + +def test_fleet_config_class_exists() -> None: + from autoskillit.config import FleetConfig + + assert FleetConfig is not None + + +def test_franchise_config_class_gone() -> None: + with pytest.raises(ImportError): + from autoskillit.config import FranchiseConfig # noqa: F401 + + +def test_tool_context_fleet_lock_field() -> None: + from dataclasses import fields + + from autoskillit.pipeline.context import ToolContext + + field_names = {f.name for f in fields(ToolContext)} + assert "fleet_lock" in field_names + assert "franchise_lock" not in field_names diff --git a/tests/fleet/test_fleet_semaphore.py b/tests/fleet/test_fleet_semaphore.py new file mode 100644 index 0000000000..94cc30ba0a --- /dev/null +++ b/tests/fleet/test_fleet_semaphore.py @@ -0,0 +1,79 @@ +"""Unit tests for FleetSemaphore (FleetLock semaphore implementation).""" + +from __future__ import annotations + +import pytest + +from autoskillit.core import FleetLock +from autoskillit.fleet import FleetSemaphore + +pytestmark = [pytest.mark.layer("fleet"), pytest.mark.small, pytest.mark.feature("fleet")] + + +@pytest.mark.anyio +async def test_fleet_semaphore_max1_at_capacity_after_acquire(): + """max=1 semaphore reports at_capacity True after single acquire.""" + s = FleetSemaphore(max_concurrent=1) + assert not s.at_capacity() + await s.acquire() + assert s.at_capacity() + s.release() + assert not s.at_capacity() + + +@pytest.mark.anyio +async def test_fleet_semaphore_max3_allows_three_concurrent(): + """max=3 semaphore allows 3 acquires without blocking.""" + s = FleetSemaphore(max_concurrent=3) + await s.acquire() + await s.acquire() + await s.acquire() + assert s.at_capacity() + assert s.active_count == 3 + s.release() + assert not s.at_capacity() + assert s.active_count == 2 + + +@pytest.mark.anyio +async def test_fleet_semaphore_active_count_tracks_correctly(): + """active_count increments on acquire and decrements on release.""" + s = FleetSemaphore(max_concurrent=5) + assert s.active_count == 0 + await s.acquire() + await s.acquire() + assert s.active_count == 2 + s.release() + assert s.active_count == 1 + + +def test_fleet_semaphore_max_concurrent_property(): + """max_concurrent returns the constructor argument.""" + s = FleetSemaphore(max_concurrent=7) + assert s.max_concurrent == 7 + + +def test_fleet_semaphore_satisfies_fleet_lock_protocol(): + """FleetSemaphore is a structural FleetLock at runtime.""" + s = FleetSemaphore(max_concurrent=1) + assert isinstance(s, FleetLock) + + +@pytest.mark.anyio +async def test_fleet_semaphore_max1_equivalent_to_serial(): + """FleetSemaphore(1) rejects a second concurrent caller (same as asyncio.Lock).""" + s = FleetSemaphore(max_concurrent=1) + await s.acquire() + assert s.at_capacity() # second would be refused at call site + s.release() + assert not s.at_capacity() + + +class TestFleetSemaphoreConstructorGuard: + def test_max_concurrent_zero_raises(self) -> None: + with pytest.raises(ValueError): + FleetSemaphore(max_concurrent=0) + + def test_max_concurrent_negative_raises(self) -> None: + with pytest.raises(ValueError): + FleetSemaphore(max_concurrent=-1) diff --git a/tests/fleet/test_food_truck_prompt.py b/tests/fleet/test_food_truck_prompt.py new file mode 100644 index 0000000000..dc6d4ffa1f --- /dev/null +++ b/tests/fleet/test_food_truck_prompt.py @@ -0,0 +1,54 @@ +"""Tests for fleet/_prompts.py: _build_food_truck_prompt behavioral semantics.""" + +from __future__ import annotations + +import pytest + +from autoskillit.core._plugin_ids import DIRECT_PREFIX +from autoskillit.fleet._prompts import _build_food_truck_prompt + +pytestmark = [pytest.mark.layer("fleet"), pytest.mark.small, pytest.mark.feature("fleet")] + + +def test_food_truck_prompt_documents_stop_action(): + """L3 food truck prompt must explain how to handle action:stop steps.""" + prompt = _build_food_truck_prompt( + recipe="test-recipe", + task="Test task", + ingredients={}, + mcp_prefix=DIRECT_PREFIX, + dispatch_id="test-dispatch", + campaign_id="test-campaign", + l3_timeout_sec=300, + ) + assert 'action: "stop"' in prompt or "action: stop" in prompt + assert "TERMINATE" in prompt.upper() or "terminate" in prompt + + +def test_food_truck_prompt_contains_hook_denial_compliance(): + """L3 food truck prompt must teach the model that ALL hook denials are mandatory.""" + prompt = _build_food_truck_prompt( + recipe="test-recipe", + task="Test task", + ingredients={}, + mcp_prefix=DIRECT_PREFIX, + dispatch_id="test-dispatch", + campaign_id="test-campaign", + l3_timeout_sec=300, + ) + assert "HOOK DENIAL" in prompt.upper() + + +def test_fleet_prompt_contains_budget_exceeded_routing(): + """L3 food truck prompt must contain QUOTA WAIT REQUIRED and QUOTA BUDGET EXCEEDED routing.""" + prompt = _build_food_truck_prompt( + recipe="test-recipe", + task="Test task", + ingredients={}, + mcp_prefix=DIRECT_PREFIX, + dispatch_id="test-dispatch", + campaign_id="test-campaign", + l3_timeout_sec=300, + ) + assert "QUOTA WAIT REQUIRED" in prompt + assert "QUOTA BUDGET EXCEEDED" in prompt diff --git a/tests/fleet/test_gate_state_persistence.py b/tests/fleet/test_gate_state_persistence.py new file mode 100644 index 0000000000..e082f2559c --- /dev/null +++ b/tests/fleet/test_gate_state_persistence.py @@ -0,0 +1,305 @@ +"""Tests for gate dispatch state persistence and campaign state writes.""" + +from __future__ import annotations + +import json +from pathlib import Path + +import pytest + +from autoskillit.fleet import ( + DispatchRecord, + DispatchStatus, + read_state, + resume_campaign_from_state, + write_initial_state, +) + +pytestmark = [pytest.mark.layer("fleet"), pytest.mark.small, pytest.mark.feature("fleet")] + + +def _state_path(tmp_path: Path) -> Path: + return tmp_path / "campaign" / "state.json" + + +def _init_state(tmp_path: Path, *names: str) -> Path: + sp = _state_path(tmp_path) + write_initial_state( + sp, "cid", "test-campaign", "/m.yaml", [DispatchRecord(name=n) for n in names] + ) + return sp + + +# --------------------------------------------------------------------------- +# Tests 1-5: record_gate_dispatch MCP tool +# --------------------------------------------------------------------------- + + +class TestRecordGateDispatch: + @pytest.fixture(autouse=True) + def _set_fleet_session(self, monkeypatch): + monkeypatch.setenv("AUTOSKILLIT_SESSION_TYPE", "fleet") + + @pytest.mark.anyio + async def test_record_gate_dispatch_writes_success( + self, tool_ctx_kitchen_open, monkeypatch, tmp_path + ): + sp = _init_state(tmp_path, "gate-check", "phase-one") + monkeypatch.setenv("AUTOSKILLIT_CAMPAIGN_STATE_PATH", str(sp)) + + from autoskillit.server.tools.tools_fleet_dispatch import record_gate_dispatch + + raw = await record_gate_dispatch(dispatch_name="gate-check", approved=True) + result = json.loads(raw) + assert result["success"] is True + assert result["status"] == "success" + + state = read_state(sp) + assert state is not None + gate = next(d for d in state.dispatches if d.name == "gate-check") + assert gate.status == DispatchStatus.SUCCESS + phase = next(d for d in state.dispatches if d.name == "phase-one") + assert phase.status == DispatchStatus.PENDING + + @pytest.mark.anyio + async def test_record_gate_dispatch_writes_failure( + self, tool_ctx_kitchen_open, monkeypatch, tmp_path + ): + sp = _init_state(tmp_path, "gate-check", "phase-one") + monkeypatch.setenv("AUTOSKILLIT_CAMPAIGN_STATE_PATH", str(sp)) + + from autoskillit.server.tools.tools_fleet_dispatch import record_gate_dispatch + + raw = await record_gate_dispatch(dispatch_name="gate-check", approved=False) + result = json.loads(raw) + assert result["success"] is True + assert result["status"] == "failure" + + state = read_state(sp) + assert state is not None + gate = next(d for d in state.dispatches if d.name == "gate-check") + assert gate.status == DispatchStatus.FAILURE + + @pytest.mark.anyio + async def test_record_gate_dispatch_rejects_unknown_dispatch( + self, tool_ctx_kitchen_open, monkeypatch, tmp_path + ): + sp = _init_state(tmp_path, "full-audit", "review-gate") + monkeypatch.setenv("AUTOSKILLIT_CAMPAIGN_STATE_PATH", str(sp)) + + from autoskillit.server.tools.tools_fleet_dispatch import record_gate_dispatch + + raw = await record_gate_dispatch(dispatch_name="nonexistent", approved=True) + result = json.loads(raw) + assert result["success"] is False + assert result["error"] == "fleet_gate_unknown_dispatch" + + @pytest.mark.anyio + async def test_record_gate_dispatch_rejects_non_pending( + self, tool_ctx_kitchen_open, monkeypatch, tmp_path + ): + from autoskillit.fleet.state import append_dispatch_record + + sp = _init_state(tmp_path, "gate-check", "phase-one") + append_dispatch_record( + sp, + DispatchRecord( + name="gate-check", status=DispatchStatus.SUCCESS, reason="gate_approved" + ), + ) + monkeypatch.setenv("AUTOSKILLIT_CAMPAIGN_STATE_PATH", str(sp)) + + from autoskillit.server.tools.tools_fleet_dispatch import record_gate_dispatch + + raw = await record_gate_dispatch(dispatch_name="gate-check", approved=True) + result = json.loads(raw) + assert result["success"] is False + assert result["error"] == "fleet_gate_already_recorded" + + @pytest.mark.anyio + async def test_record_gate_dispatch_requires_campaign_state_path( + self, tool_ctx_kitchen_open, monkeypatch + ): + monkeypatch.delenv("AUTOSKILLIT_CAMPAIGN_STATE_PATH", raising=False) + + from autoskillit.server.tools.tools_fleet_dispatch import record_gate_dispatch + + raw = await record_gate_dispatch(dispatch_name="gate-check", approved=True) + result = json.loads(raw) + assert result["success"] is False + assert result["error"] == "fleet_gate_no_campaign" + + +# --------------------------------------------------------------------------- +# Tests 6-9: dispatch_food_truck campaign state persistence +# --------------------------------------------------------------------------- + + +class TestDispatchFoodTruckCampaignState: + @pytest.fixture(autouse=True) + def _set_fleet_session(self, monkeypatch): + monkeypatch.setenv("AUTOSKILLIT_SESSION_TYPE", "fleet") + + @pytest.mark.anyio + async def test_dispatch_food_truck_updates_campaign_state_on_success( + self, tool_ctx_kitchen_open, monkeypatch, tmp_path + ): + sp = _init_state(tmp_path, "full-audit") + monkeypatch.setenv("AUTOSKILLIT_CAMPAIGN_STATE_PATH", str(sp)) + monkeypatch.delenv("AUTOSKILLIT_HEADLESS", raising=False) + + success_envelope = json.dumps( + { + "success": True, + "dispatch_status": "success", + "dispatch_id": "d1", + "dispatched_session_id": "s1", + "reason": "", + "token_usage": {}, + } + ) + + async def _fake_execute(**kwargs): + return success_envelope + + import autoskillit.fleet + + monkeypatch.setattr(autoskillit.fleet, "execute_dispatch", _fake_execute) + + from autoskillit.server.tools.tools_fleet_dispatch import dispatch_food_truck + + await dispatch_food_truck(recipe="full-audit", task="audit", dispatch_name="full-audit") + + state = read_state(sp) + assert state is not None + d = next(d for d in state.dispatches if d.name == "full-audit") + assert d.status == DispatchStatus.SUCCESS + + @pytest.mark.anyio + async def test_dispatch_food_truck_updates_campaign_state_on_failure( + self, tool_ctx_kitchen_open, monkeypatch, tmp_path + ): + sp = _init_state(tmp_path, "full-audit") + monkeypatch.setenv("AUTOSKILLIT_CAMPAIGN_STATE_PATH", str(sp)) + monkeypatch.delenv("AUTOSKILLIT_HEADLESS", raising=False) + + failure_envelope = json.dumps( + { + "success": False, + "dispatch_status": "failure", + "dispatch_id": "d1", + "dispatched_session_id": "s1", + "reason": "l2_crashed", + "token_usage": {}, + } + ) + + async def _fake_execute(**kwargs): + return failure_envelope + + import autoskillit.fleet + + monkeypatch.setattr(autoskillit.fleet, "execute_dispatch", _fake_execute) + + from autoskillit.server.tools.tools_fleet_dispatch import dispatch_food_truck + + await dispatch_food_truck(recipe="full-audit", task="audit", dispatch_name="full-audit") + + state = read_state(sp) + assert state is not None + d = next(d for d in state.dispatches if d.name == "full-audit") + assert d.status == DispatchStatus.FAILURE + + @pytest.mark.anyio + async def test_dispatch_food_truck_skips_campaign_state_without_env( + self, tool_ctx_kitchen_open, monkeypatch, tmp_path + ): + monkeypatch.delenv("AUTOSKILLIT_CAMPAIGN_STATE_PATH", raising=False) + monkeypatch.delenv("AUTOSKILLIT_HEADLESS", raising=False) + + async def _fake_execute(**kwargs): + return json.dumps( + { + "success": True, + "dispatch_id": "d1", + "dispatched_session_id": "s1", + "reason": "", + "token_usage": {}, + } + ) + + import autoskillit.fleet + + monkeypatch.setattr(autoskillit.fleet, "execute_dispatch", _fake_execute) + + from autoskillit.server.tools.tools_fleet_dispatch import dispatch_food_truck + + raw = await dispatch_food_truck( + recipe="full-audit", task="audit", dispatch_name="full-audit" + ) + result = json.loads(raw) + assert result["success"] is True + # No state file should exist + assert not (tmp_path / "campaign" / "state.json").exists() + + @pytest.mark.anyio + async def test_dispatch_food_truck_skips_campaign_state_without_dispatch_name( + self, tool_ctx_kitchen_open, monkeypatch, tmp_path + ): + sp = _init_state(tmp_path, "full-audit") + monkeypatch.setenv("AUTOSKILLIT_CAMPAIGN_STATE_PATH", str(sp)) + monkeypatch.delenv("AUTOSKILLIT_HEADLESS", raising=False) + + async def _fake_execute(**kwargs): + return json.dumps( + { + "success": True, + "dispatch_id": "d1", + "dispatched_session_id": "s1", + "reason": "", + "token_usage": {}, + } + ) + + import autoskillit.fleet + + monkeypatch.setattr(autoskillit.fleet, "execute_dispatch", _fake_execute) + + from autoskillit.server.tools.tools_fleet_dispatch import dispatch_food_truck + + await dispatch_food_truck(recipe="full-audit", task="audit", dispatch_name=None) + + state = read_state(sp) + assert state is not None + d = next(d for d in state.dispatches if d.name == "full-audit") + assert d.status == DispatchStatus.PENDING + + +# --------------------------------------------------------------------------- +# Test 11: resume chain for promote-to-main shape +# --------------------------------------------------------------------------- + + +class TestCampaignResumeChain: + def test_campaign_dispatch_chain_resume_after_two_successes(self, tmp_path): + from autoskillit.fleet.state import append_dispatch_record + + sp = _init_state( + tmp_path, "full-audit", "review-gate", "build-map", "implement-findings", "promote" + ) + append_dispatch_record( + sp, + DispatchRecord(name="full-audit", status=DispatchStatus.SUCCESS, reason="completed"), + ) + append_dispatch_record( + sp, + DispatchRecord( + name="review-gate", status=DispatchStatus.SUCCESS, reason="gate_approved" + ), + ) + + decision = resume_campaign_from_state(sp, continue_on_failure=False) + assert decision is not None + assert decision.next_dispatch_name == "build-map" + assert "full-audit" in decision.completed_dispatches_block + assert "review-gate" in decision.completed_dispatches_block diff --git a/tests/fleet/test_helpers_exports.py b/tests/fleet/test_helpers_exports.py new file mode 100644 index 0000000000..ef7a5f4bb7 --- /dev/null +++ b/tests/fleet/test_helpers_exports.py @@ -0,0 +1,28 @@ +"""Tests that shared helpers are importable from tests.fleet._helpers.""" + +from __future__ import annotations + +import pytest + +pytestmark = [pytest.mark.layer("fleet"), pytest.mark.small, pytest.mark.feature("fleet")] + + +def test_make_recipe_info_exported_from_fleet_helpers(): + from tests.fleet._helpers import _make_recipe_info + + info = _make_recipe_info("my-recipe") + assert info.name == "my-recipe" + assert str(info.path) == "/fake/my-recipe.yaml" + + +def test_make_recipe_info_custom_prefix(): + from tests.fleet._helpers import _make_recipe_info + + info = _make_recipe_info("my-recipe", path_prefix="/fake/recipes/") + assert str(info.path) == "/fake/recipes/my-recipe.yaml" + + +def test_setup_dispatch_exported_from_fleet_helpers(): + from tests.fleet._helpers import _setup_dispatch + + assert callable(_setup_dispatch) diff --git a/tests/fleet/test_liveness.py b/tests/fleet/test_liveness.py new file mode 100644 index 0000000000..7c6d2f8155 --- /dev/null +++ b/tests/fleet/test_liveness.py @@ -0,0 +1,86 @@ +import os + +import pytest + +from autoskillit.core.runtime._linux_proc import read_boot_id, read_starttime_ticks +from autoskillit.fleet import DispatchRecord +from autoskillit.fleet._liveness import is_dispatch_session_alive + +pytestmark = [pytest.mark.layer("fleet"), pytest.mark.small, pytest.mark.feature("fleet")] + + +class TestIsDispatchSessionAlive: + def test_unstarted_dispatch_not_alive(self) -> None: + record = DispatchRecord(name="test") # dispatched_pid defaults to 0 + assert not is_dispatch_session_alive(record) + + def test_different_boot_id_not_alive(self) -> None: + record = DispatchRecord( + name="test", + dispatched_pid=os.getpid(), + dispatched_boot_id="different-boot-id-xyz", + dispatched_starttime_ticks=999, + ) + assert not is_dispatch_session_alive(record) + + def test_nonexistent_pid_not_alive(self) -> None: + boot_id = read_boot_id() + if boot_id is None: + pytest.skip("Not on Linux") + record = DispatchRecord( + name="test", + dispatched_pid=999999999, + dispatched_boot_id=boot_id, + dispatched_starttime_ticks=999, + ) + assert not is_dispatch_session_alive(record) + + def test_current_process_is_alive(self) -> None: + pid = os.getpid() + ticks = read_starttime_ticks(pid) + boot_id = read_boot_id() + if ticks is None or boot_id is None: + pytest.skip("Not on Linux") + record = DispatchRecord( + name="test", + dispatched_pid=pid, + dispatched_boot_id=boot_id, + dispatched_starttime_ticks=ticks, + ) + assert is_dispatch_session_alive(record) + + def test_ticks_mismatch_not_alive(self) -> None: + pid = os.getpid() + boot_id = read_boot_id() + if boot_id is None: + pytest.skip("Not on Linux") + record = DispatchRecord( + name="test", + dispatched_pid=pid, + dispatched_boot_id=boot_id, + dispatched_starttime_ticks=-1, + ) + assert not is_dispatch_session_alive(record) + + def test_missing_boot_id_on_record_not_alive(self) -> None: + record = DispatchRecord( + name="test", + dispatched_pid=os.getpid(), + dispatched_boot_id="", + dispatched_starttime_ticks=999, + ) + assert not is_dispatch_session_alive(record) + + def test_delegates_to_core_is_session_alive(self) -> None: + from unittest.mock import patch + + record = DispatchRecord( + name="test", + dispatched_pid=42, + dispatched_boot_id="boot-id", + dispatched_starttime_ticks=100, + ) + with patch("autoskillit.fleet._liveness.is_session_alive", return_value=True) as mock: + result = is_dispatch_session_alive(record) + mock.assert_called_once_with(42, "boot-id", 100) + assert result is True diff --git a/tests/fleet/test_pack_enforcement.py b/tests/fleet/test_pack_enforcement.py new file mode 100644 index 0000000000..0733f5c6ce --- /dev/null +++ b/tests/fleet/test_pack_enforcement.py @@ -0,0 +1,328 @@ +"""Fleet per-recipe tool-surface enforcement tests. + +Validates that food trucks launched for each bundled recipe see exactly the +expected tool surface under the pack enforcement model: kitchen-core tools +plus only the tools from packs declared in the recipe's requires_packs field. +""" + +from __future__ import annotations + +import pytest + +from tests.fleet._helpers import ( + KITCHEN_CORE_TOOLS, + TOOLS_BY_PACK, + compute_food_truck_tool_surface, +) + +pytestmark = [pytest.mark.layer("fleet"), pytest.mark.small, pytest.mark.feature("fleet")] + +# --- MCP state isolation (module-level; only this file mutates MCP tags) --- + + +@pytest.fixture(autouse=True) +def _reset_mcp_tags(): + from autoskillit.core import ALL_VISIBILITY_TAGS + from autoskillit.server import mcp + + mcp._transforms.clear() + for tag in sorted(ALL_VISIBILITY_TAGS): + mcp.disable(tags={tag}) + yield + mcp._transforms.clear() + for tag in sorted(ALL_VISIBILITY_TAGS): + mcp.disable(tags={tag}) + + +@pytest.fixture(autouse=True) +def _reset_server_state(monkeypatch): + from autoskillit.server import _state + + monkeypatch.setattr(_state, "_ctx", None) + + +# --------------------------------------------------------------------------- +# Module-level constants — additional packs used only in this test module +# --------------------------------------------------------------------------- + +GITHUB_PACK_TOOLS = TOOLS_BY_PACK["github"] +CI_PACK_TOOLS = TOOLS_BY_PACK["ci"] +CLONE_PACK_TOOLS = TOOLS_BY_PACK["clone"] +TELEMETRY_PACK_TOOLS = TOOLS_BY_PACK["telemetry"] + + +def _simulate_food_truck(monkeypatch: pytest.MonkeyPatch, packs: str) -> None: + """Set env vars to simulate a food truck session, then apply visibility.""" + monkeypatch.setenv("AUTOSKILLIT_SESSION_TYPE", "orchestrator") + monkeypatch.setenv("AUTOSKILLIT_HEADLESS", "1") + monkeypatch.setenv("AUTOSKILLIT_FOOD_TRUCK_TOOL_TAGS", packs) + from autoskillit.server import _apply_session_type_visibility + + _apply_session_type_visibility() + + +# --------------------------------------------------------------------------- +# Group A: 8 per-recipe tests +# --------------------------------------------------------------------------- + +_STANDARD_PACKS = ",".join(sorted(["github", "ci", "clone", "telemetry"])) + + +class TestPerRecipeToolSurface: + @pytest.mark.anyio + async def test_implementation_food_truck_sees_expected_tools(self, monkeypatch): + from fastmcp.client import Client + + from autoskillit.server import mcp + + expected = compute_food_truck_tool_surface("implementation") + _simulate_food_truck(monkeypatch, _STANDARD_PACKS) + + async with Client(mcp) as client: + tools = await client.list_tools() + visible = {t.name for t in tools} + + for name in expected: + assert name in visible, f"{name} should be visible for implementation food truck" + + @pytest.mark.anyio + async def test_implementation_food_truck_hides_research_tools(self, monkeypatch): + from fastmcp.client import Client + + from autoskillit.core import UNGATED_TOOLS + from autoskillit.server import mcp + + expected = compute_food_truck_tool_surface("implementation") + _simulate_food_truck(monkeypatch, _STANDARD_PACKS) + + async with Client(mcp) as client: + tools = await client.list_tools() + visible = {t.name for t in tools} + + extras = visible - expected - UNGATED_TOOLS + assert not extras, f"Unexpected tools visible for implementation food truck: {extras}" + + @pytest.mark.anyio + async def test_merge_prs_food_truck_sees_expected_tools(self, monkeypatch): + from fastmcp.client import Client + + from autoskillit.server import mcp + + expected = compute_food_truck_tool_surface("merge-prs") + _simulate_food_truck(monkeypatch, _STANDARD_PACKS) + + async with Client(mcp) as client: + tools = await client.list_tools() + visible = {t.name for t in tools} + + for name in expected: + assert name in visible, f"{name} should be visible for merge-prs food truck" + + @pytest.mark.anyio + async def test_merge_prs_food_truck_hides_research_tools(self, monkeypatch): + from fastmcp.client import Client + + from autoskillit.core import UNGATED_TOOLS + from autoskillit.server import mcp + + expected = compute_food_truck_tool_surface("merge-prs") + _simulate_food_truck(monkeypatch, _STANDARD_PACKS) + + async with Client(mcp) as client: + tools = await client.list_tools() + visible = {t.name for t in tools} + + extras = visible - expected - UNGATED_TOOLS + assert not extras, f"Unexpected tools visible for merge-prs food truck: {extras}" + + @pytest.mark.anyio + async def test_remediation_food_truck_sees_expected_tools(self, monkeypatch): + from fastmcp.client import Client + + from autoskillit.server import mcp + + expected = compute_food_truck_tool_surface("remediation") + _simulate_food_truck(monkeypatch, _STANDARD_PACKS) + + async with Client(mcp) as client: + tools = await client.list_tools() + visible = {t.name for t in tools} + + for name in expected: + assert name in visible, f"{name} should be visible for remediation food truck" + + @pytest.mark.anyio + async def test_remediation_food_truck_hides_research_tools(self, monkeypatch): + from fastmcp.client import Client + + from autoskillit.core import UNGATED_TOOLS + from autoskillit.server import mcp + + expected = compute_food_truck_tool_surface("remediation") + _simulate_food_truck(monkeypatch, _STANDARD_PACKS) + + async with Client(mcp) as client: + tools = await client.list_tools() + visible = {t.name for t in tools} + + extras = visible - expected - UNGATED_TOOLS + assert not extras, f"Unexpected tools visible for remediation food truck: {extras}" + + @pytest.mark.anyio + async def test_implementation_groups_food_truck_sees_expected_tools(self, monkeypatch): + from fastmcp.client import Client + + from autoskillit.server import mcp + + expected = compute_food_truck_tool_surface("implementation-groups") + _simulate_food_truck(monkeypatch, _STANDARD_PACKS) + + async with Client(mcp) as client: + tools = await client.list_tools() + visible = {t.name for t in tools} + + for name in expected: + assert name in visible, ( + f"{name} should be visible for implementation-groups food truck" + ) + + @pytest.mark.anyio + async def test_implementation_groups_food_truck_hides_research_tools(self, monkeypatch): + from fastmcp.client import Client + + from autoskillit.core import UNGATED_TOOLS + from autoskillit.server import mcp + + expected = compute_food_truck_tool_surface("implementation-groups") + _simulate_food_truck(monkeypatch, _STANDARD_PACKS) + + async with Client(mcp) as client: + tools = await client.list_tools() + visible = {t.name for t in tools} + + extras = visible - expected - UNGATED_TOOLS + assert not extras, ( + f"Unexpected tools visible for implementation-groups food truck: {extras}" + ) + + +# --------------------------------------------------------------------------- +# Group B: 3 synthetic tests +# --------------------------------------------------------------------------- + + +class TestSyntheticPackScenarios: + @pytest.mark.anyio + async def test_food_truck_with_empty_requires_packs_sees_full_kitchen_fallback( + self, monkeypatch + ): + from fastmcp.client import Client + + from autoskillit.core import FLEET_DISPATCH_TOOLS, FLEET_TOOLS, GATED_TOOLS + from autoskillit.server import _apply_session_type_visibility, mcp + + monkeypatch.setenv("AUTOSKILLIT_SESSION_TYPE", "orchestrator") + monkeypatch.setenv("AUTOSKILLIT_HEADLESS", "1") + monkeypatch.setenv("AUTOSKILLIT_FOOD_TRUCK_TOOL_TAGS", "") + _apply_session_type_visibility() + + async with Client(mcp) as client: + tools = await client.list_tools() + visible = {t.name for t in tools} + + for name in GATED_TOOLS - FLEET_TOOLS - FLEET_DISPATCH_TOOLS: + assert name in visible, ( + f"{name} should be visible under full kitchen fallback" + " (empty FOOD_TRUCK_TOOL_TAGS)" + ) + + @pytest.mark.anyio + async def test_food_truck_with_single_pack_sees_kitchen_core_plus_pack(self, monkeypatch): + from fastmcp.client import Client + + from autoskillit.server import mcp + + _simulate_food_truck(monkeypatch, "github") + + async with Client(mcp) as client: + tools = await client.list_tools() + visible = {t.name for t in tools} + + for name in KITCHEN_CORE_TOOLS: + assert name in visible, f"{name} (kitchen-core) should be visible" + for name in GITHUB_PACK_TOOLS: + assert name in visible, f"{name} (github pack) should be visible" + for name in CI_PACK_TOOLS: + assert name not in visible, f"{name} (ci pack) should be hidden" + + def test_food_truck_startup_enables_tags_in_order(self, monkeypatch): + from unittest.mock import MagicMock, call, patch + + monkeypatch.setenv("AUTOSKILLIT_SESSION_TYPE", "orchestrator") + monkeypatch.setenv("AUTOSKILLIT_HEADLESS", "1") + monkeypatch.setenv("AUTOSKILLIT_FOOD_TRUCK_TOOL_TAGS", "github,ci,telemetry") + + mock_mcp = MagicMock() + with patch("autoskillit.server.mcp", mock_mcp): + from autoskillit.server._session_type import _apply_session_type_visibility + + _apply_session_type_visibility() + + enable_calls = mock_mcp.enable.call_args_list + assert len(enable_calls) == 4, ( + f"Expected 4 enable calls, got {len(enable_calls)}: {enable_calls}" + ) + assert enable_calls[0] == call(tags={"kitchen-core"}), "kitchen-core must be first" + assert enable_calls[1] == call(tags={"github"}) + assert enable_calls[2] == call(tags={"ci"}) + assert enable_calls[3] == call(tags={"telemetry"}) + + +# --------------------------------------------------------------------------- +# Group C: 3 regression guards (module-level functions) +# --------------------------------------------------------------------------- + + +def test_every_bundled_recipe_declares_requires_packs() -> None: + from autoskillit.recipe.io import builtin_recipes_dir, load_recipe + + MCP_TOOL_PACKS = {"github", "ci", "clone", "telemetry", "kitchen-core"} + RESEARCH_PACKS = {"research", "exp-lens", "vis-lens"} + + for path in sorted(builtin_recipes_dir().glob("*.yaml")): + recipe = load_recipe(path) + assert recipe.requires_packs, f"{path.name} does not declare requires_packs" + pack_set = set(recipe.requires_packs) + if pack_set & RESEARCH_PACKS: + continue + assert pack_set & MCP_TOOL_PACKS, ( + f"{path.name} declares packs {pack_set} but none are MCP tool packs" + ) + + +@pytest.mark.anyio +async def test_no_tool_has_bare_kitchen_tag_only() -> None: + from autoskillit.core.types._type_constants import CATEGORY_TAGS + from autoskillit.server import mcp + + all_tools = {t.name: t for t in await mcp.list_tools()} + for name, tool in all_tools.items(): + if "kitchen" in tool.tags and "autoskillit" in tool.tags: + pack_tags = tool.tags & CATEGORY_TAGS + assert pack_tags, ( + f"{name} has 'kitchen' tag but no pack tag from CATEGORY_TAGS. Tags: {tool.tags}" + ) + + +def test_kitchen_core_and_packs_partition_kitchen_gated_tools() -> None: + from autoskillit.core.types._type_constants import TOOL_SUBSET_TAGS + + all_tools_in_subsets = set(TOOL_SUBSET_TAGS.keys()) + union_of_packs: set[str] = set() + for tools in TOOLS_BY_PACK.values(): + union_of_packs |= tools + + orphaned = all_tools_in_subsets - union_of_packs + assert not orphaned, f"Tools not in any pack: {orphaned}" + extra = union_of_packs - all_tools_in_subsets + assert not extra, f"Tools in packs but not in TOOL_SUBSET_TAGS: {extra}" diff --git a/tests/fleet/test_pack_enforcement_e2e.py b/tests/fleet/test_pack_enforcement_e2e.py new file mode 100644 index 0000000000..3884d6e2ab --- /dev/null +++ b/tests/fleet/test_pack_enforcement_e2e.py @@ -0,0 +1,84 @@ +"""Fleet per-recipe tool-surface e2e tests. + +Validates the tool surface using a real MCP server subprocess — no monkeypatching. +Marked integration + medium to allow subprocess spawning. +""" + +from __future__ import annotations + +import os +import sys + +import pytest + +from tests.fleet._helpers import compute_food_truck_tool_surface + +pytestmark = [pytest.mark.layer("fleet"), pytest.mark.medium, pytest.mark.feature("fleet")] + +# --------------------------------------------------------------------------- +# Fixture +# --------------------------------------------------------------------------- + + +@pytest.fixture +def fleet_runtime(): + async def _get_surface(recipe_name: str) -> set[str]: + from fastmcp.client import Client + from fastmcp.client.transports import StdioTransport + + from autoskillit.recipe.io import builtin_recipes_dir, load_recipe + + path = builtin_recipes_dir() / f"{recipe_name}.yaml" + recipe = load_recipe(path) + packs = ",".join(sorted(recipe.requires_packs or [])) + + env = { + **os.environ, + "AUTOSKILLIT_SESSION_TYPE": "orchestrator", + "AUTOSKILLIT_HEADLESS": "1", + "AUTOSKILLIT_FOOD_TRUCK_TOOL_TAGS": packs, + } + + transport = StdioTransport( + command=sys.executable, + args=["-m", "autoskillit"], + env=env, + ) + async with Client(transport) as client: + tools = await client.list_tools() + return {t.name for t in tools} + + return _get_surface + + +# --------------------------------------------------------------------------- +# E2E tests +# --------------------------------------------------------------------------- + + +@pytest.mark.integration +@pytest.mark.anyio +async def test_implementation_food_truck_real_subprocess_tool_surface(fleet_runtime): + from autoskillit.core import UNGATED_TOOLS + + visible = await fleet_runtime("implementation") + expected = compute_food_truck_tool_surface("implementation") + + extras = visible - expected - UNGATED_TOOLS + assert not extras, f"Unexpected tools visible for implementation food truck: {extras}" + for name in expected: + assert name in visible, f"{name} should be visible for implementation food truck" + + +@pytest.mark.integration +@pytest.mark.anyio +async def test_merge_prs_food_truck_real_subprocess_tool_surface(fleet_runtime): + from autoskillit.core import UNGATED_TOOLS + + visible = await fleet_runtime("merge-prs") + expected = compute_food_truck_tool_surface("merge-prs") + + extras = visible - expected - UNGATED_TOOLS + assert not extras, f"Unexpected tools visible for merge-prs food truck: {extras}" + for name in expected: + assert name in visible, f"{name} should be visible for merge-prs food truck" diff --git a/tests/fleet/test_research_campaign_dispatch.py b/tests/fleet/test_research_campaign_dispatch.py new file mode 100644 index 0000000000..628bb00754 --- /dev/null +++ b/tests/fleet/test_research_campaign_dispatch.py @@ -0,0 +1,232 @@ +"""Tests for multi-recipe research campaign dispatch capture propagation (Group J).""" + +from __future__ import annotations + +import json + +import pytest + +from tests.fleet._helpers import ( + _make_recipe_info, + _no_sleep_quota_checker, + _noop_quota_refresher, + _simple_prompt_builder, +) + +pytestmark = [pytest.mark.layer("fleet"), pytest.mark.small, pytest.mark.feature("fleet")] + + +def _read_state_file(tool_ctx) -> dict: + state_files = list((tool_ctx.temp_dir / "dispatches").glob("*.json")) + assert len(state_files) == 1, f"Expected 1 state file, found {len(state_files)}: {state_files}" + return json.loads(state_files[0].read_text()) + + +def _setup_research_campaign(tool_ctx): + from autoskillit.fleet import FleetSemaphore + from autoskillit.recipe.schema import Recipe, RecipeIngredient, RecipeKind + from tests.fakes import InMemoryHeadlessExecutor, InMemoryRecipeRepository + + tool_ctx.fleet_lock = FleetSemaphore(max_concurrent=1) + repo = InMemoryRecipeRepository() + + design_info = _make_recipe_info("research-design") + repo.add_recipe("research-design", design_info) + repo.add_full_recipe( + design_info.path, + Recipe( + name="research-design", + description="Research design phase", + kind=RecipeKind.STANDARD, + ingredients={}, + ), + ) + + impl_info = _make_recipe_info("research-implement") + repo.add_recipe("research-implement", impl_info) + repo.add_full_recipe( + impl_info.path, + Recipe( + name="research-implement", + description="Research implement phase", + kind=RecipeKind.STANDARD, + ingredients={ + "worktree_path": RecipeIngredient(description="Path to worktree"), + "research_dir": RecipeIngredient(description="Path to research dir"), + }, + ), + ) + + tool_ctx.recipes = repo + tool_ctx.executor = InMemoryHeadlessExecutor() + + +@pytest.mark.anyio +async def test_design_captured_values_propagate_to_implement_dispatch(tool_ctx, monkeypatch): + """End-to-end: design dispatch captures propagate to implement dispatch via state file.""" + from autoskillit.fleet._api import execute_dispatch + from autoskillit.fleet.result_parser import L3ParseResult + + _setup_research_campaign(tool_ctx) + + monkeypatch.setattr( + "autoskillit.fleet._api.parse_l3_result_block", + lambda **_: L3ParseResult( + outcome="completed_clean", + payload={ + "success": True, + "worktree_path": "/tmp/wt/proj", + "research_dir": "/tmp/wt/proj/.research", + }, + raw_body=None, + parse_error=None, + source="stdout", + ), + ) + + raw = await execute_dispatch( + tool_ctx=tool_ctx, + recipe="research-design", + task="Design the research", + ingredients=None, + dispatch_name=None, + timeout_sec=None, + capture={ + "worktree_path": "${{ result.worktree_path }}", + "research_dir": "${{ result.research_dir }}", + }, + prompt_builder=_simple_prompt_builder, + quota_checker=_no_sleep_quota_checker, + quota_refresher=_noop_quota_refresher, + ) + + result = json.loads(raw) + assert result["success"] is True + + state_data = _read_state_file(tool_ctx) + assert state_data.get("captured_values") == { + "worktree_path": "/tmp/wt/proj", + "research_dir": "/tmp/wt/proj/.research", + } + + received_ingredients: list[dict] = [] + + def _capturing_prompt_builder(**kwargs): + received_ingredients.append(kwargs.get("ingredients", {})) + return "prompt" + + monkeypatch.setattr( + "autoskillit.fleet._api.parse_l3_result_block", + lambda **kwargs: L3ParseResult( + outcome="completed_clean", + payload={"success": True}, + raw_body=None, + parse_error=None, + source="stdout", + ), + ) + + raw = await execute_dispatch( + tool_ctx=tool_ctx, + recipe="research-implement", + task="Implement the research", + ingredients={ + "worktree_path": "${{ campaign.worktree_path }}", + "research_dir": "${{ campaign.research_dir }}", + }, + dispatch_name=None, + timeout_sec=None, + capture=None, + prompt_builder=_capturing_prompt_builder, + quota_checker=_no_sleep_quota_checker, + quota_refresher=_noop_quota_refresher, + ) + + assert received_ingredients, "prompt_builder was not called" + assert received_ingredients[0] == { + "worktree_path": "/tmp/wt/proj", + "research_dir": "/tmp/wt/proj/.research", + } + + +@pytest.mark.anyio +async def test_missing_campaign_ref_returns_fleet_error(tool_ctx): + """Dispatch with ${{ campaign.worktree_path }} and no prior captures returns fleet_error.""" + from autoskillit.fleet._api import execute_dispatch + + _setup_research_campaign(tool_ctx) + + raw = await execute_dispatch( + tool_ctx=tool_ctx, + recipe="research-implement", + task="Implement with missing ref", + ingredients={"worktree_path": "${{ campaign.worktree_path }}"}, + dispatch_name=None, + timeout_sec=None, + capture=None, + prompt_builder=_simple_prompt_builder, + quota_checker=_no_sleep_quota_checker, + quota_refresher=_noop_quota_refresher, + ) + + result = json.loads(raw) + assert result["success"] is False + assert result["error"] == "fleet_unknown_ingredient" + assert "campaign.worktree_path" in result["user_visible_message"] + + +@pytest.mark.anyio +async def test_partial_capture_propagates_only_captured_keys(tool_ctx, monkeypatch): + """Design captured only worktree_path; implement requests uncaptured research_dir.""" + from autoskillit.fleet._api import execute_dispatch + from autoskillit.fleet.result_parser import L3ParseResult + + _setup_research_campaign(tool_ctx) + + monkeypatch.setattr( + "autoskillit.fleet._api.parse_l3_result_block", + lambda **kwargs: L3ParseResult( + outcome="completed_clean", + payload={"success": True, "worktree_path": "/tmp/wt/proj"}, + raw_body=None, + parse_error=None, + source="stdout", + ), + ) + + raw = await execute_dispatch( + tool_ctx=tool_ctx, + recipe="research-design", + task="Design with partial capture", + ingredients=None, + dispatch_name=None, + timeout_sec=None, + capture={"worktree_path": "${{ result.worktree_path }}"}, + prompt_builder=_simple_prompt_builder, + quota_checker=_no_sleep_quota_checker, + quota_refresher=_noop_quota_refresher, + ) + + result = json.loads(raw) + assert result["success"] is True + + raw = await execute_dispatch( + tool_ctx=tool_ctx, + recipe="research-implement", + task="Implement with missing research_dir", + ingredients={ + "worktree_path": "${{ campaign.worktree_path }}", + "research_dir": "${{ campaign.research_dir }}", + }, + dispatch_name=None, + timeout_sec=None, + capture=None, + prompt_builder=_simple_prompt_builder, + quota_checker=_no_sleep_quota_checker, + quota_refresher=_noop_quota_refresher, + ) + + result = json.loads(raw) + assert result["success"] is False + assert "error" in result + assert "campaign.research_dir" in result["user_visible_message"] diff --git a/tests/fleet/test_result_parser.py b/tests/fleet/test_result_parser.py new file mode 100644 index 0000000000..d2a7b9b04d --- /dev/null +++ b/tests/fleet/test_result_parser.py @@ -0,0 +1,330 @@ +"""Tests for fleet.result_parser — L3 result block parsing.""" + +from __future__ import annotations + +import json +from pathlib import Path + +import pytest + +from autoskillit.fleet.result_parser import parse_l3_result_block + +pytestmark = [pytest.mark.layer("fleet"), pytest.mark.small, pytest.mark.feature("fleet")] + +DISPATCH_ID = "aaaabbbb-cccc-dddd-eeee-ffffaaaabbbb" + + +def _open(dispatch_id: str = DISPATCH_ID) -> str: + return f"---l3-result::{dispatch_id}---" + + +def _close(dispatch_id: str = DISPATCH_ID) -> str: + return f"---end-l3-result::{dispatch_id}---" + + +def make_stdout(payload_json: str, dispatch_id: str = DISPATCH_ID) -> str: + """Build a well-formed stdout string with sentinel block.""" + return ( + f"some prefix output\n{_open(dispatch_id)}\n" + f"{payload_json}\n{_close(dispatch_id)}\nsome suffix" + ) + + +def make_jsonl_file(tmp_path, messages: list[str]) -> Path: + """Write a JSONL file with type=assistant records containing given text.""" + path: Path = tmp_path / "session.jsonl" + lines = [] + for text in messages: + record = { + "type": "assistant", + "message": { + "content": [{"type": "text", "text": text}], + }, + } + lines.append(json.dumps(record)) + path.write_text("\n".join(lines), encoding="utf-8") + return path + + +def test_clean_parse_from_stdout() -> None: + """Parse valid JSON body between properly formed sentinels.""" + + payload = {"success": True, "value": 42} + stdout = make_stdout(json.dumps(payload)) + + result = parse_l3_result_block(stdout=stdout, expected_dispatch_id=DISPATCH_ID) + + assert result.outcome == "completed_clean" + assert result.payload == payload + assert result.source == "stdout" + assert result.parse_error is None + + +def test_last_occurrence_wins() -> None: + """Parser must use the LAST occurrence of the sentinel block (rfind).""" + + first_payload = {"success": False, "value": "first"} + second_payload = {"success": True, "value": "second"} + stdout = ( + make_stdout(json.dumps(first_payload)) + "\n" + make_stdout(json.dumps(second_payload)) + ) + + result = parse_l3_result_block(stdout=stdout, expected_dispatch_id=DISPATCH_ID) + + assert result.outcome == "completed_clean" + assert result.payload == second_payload + + +def test_mismatched_dispatch_id_rejected() -> None: + """Sentinels with a different UUID must not be matched.""" + + wrong_id = "00000000-0000-0000-0000-000000000000" + stdout = make_stdout(json.dumps({"success": True}), dispatch_id=wrong_id) + + result = parse_l3_result_block(stdout=stdout, expected_dispatch_id=DISPATCH_ID) + + assert result.outcome == "no_sentinel" + assert result.payload is None + assert result.source == "stdout" + + +def test_ansi_codes_stripped_before_scan() -> None: + """ANSI escape sequences around sentinel markers are stripped before scanning.""" + + payload = {"success": True, "ansi": "ok"} + raw_open = f"\x1b[1m{_open()}\x1b[0m" + raw_close = f"\x1b[1m{_close()}\x1b[0m" + stdout = f"prefix\n{raw_open}\n{json.dumps(payload)}\n{raw_close}\nsuffix" + + result = parse_l3_result_block(stdout=stdout, expected_dispatch_id=DISPATCH_ID) + + assert result.outcome == "completed_clean" + assert result.payload == payload + assert result.source == "stdout" + + +def test_channel_b_fallback_recovers_truncated_stdout(tmp_path) -> None: + """Channel B JSONL fallback recovers payload when stdout is truncated.""" + + payload = {"success": True, "recovered": True} + sentinel_text = f"{_open()}\n{json.dumps(payload)}\n{_close()}" + jsonl_path = make_jsonl_file(tmp_path, [sentinel_text]) + + result = parse_l3_result_block( + stdout="truncated output with no sentinel", + expected_dispatch_id=DISPATCH_ID, + assistant_messages_path=jsonl_path, + ) + + assert result.outcome == "completed_clean" + assert result.payload == payload + assert result.source == "assistant_messages_jsonl" + + +def test_empty_body_between_sentinels() -> None: + """Opening and closing sentinels with no content yields completed_dirty.""" + + stdout = f"prefix\n{_open()}\n{_close()}\nsuffix" + + result = parse_l3_result_block(stdout=stdout, expected_dispatch_id=DISPATCH_ID) + + assert result.outcome == "completed_dirty" + assert result.raw_body == "" + assert result.parse_error is not None + assert "empty" in result.parse_error + + +def test_invalid_json_body() -> None: + """Sentinels present but body is malformed JSON yields completed_dirty.""" + + stdout = make_stdout("not valid json {{{") + + result = parse_l3_result_block(stdout=stdout, expected_dispatch_id=DISPATCH_ID) + + assert result.outcome == "completed_dirty" + assert result.raw_body == "not valid json {{{" + assert result.parse_error is not None + + +def test_no_sentinel_at_all() -> None: + """Stdout with no sentinel markers and no JSONL path yields no_sentinel.""" + + result = parse_l3_result_block( + stdout="This output has absolutely no sentinel markers.", + expected_dispatch_id=DISPATCH_ID, + ) + + assert result.outcome == "no_sentinel" + assert result.payload is None + assert result.source == "stdout" + + +def test_closing_before_opening_rejected() -> None: + """Closing sentinel appearing before opening yields no_sentinel.""" + + stdout = f"prefix\n{_close()}\nsome content\n{_open()}\nsuffix" + + result = parse_l3_result_block(stdout=stdout, expected_dispatch_id=DISPATCH_ID) + + assert result.outcome == "no_sentinel" + + +def test_bare_sentinel_without_id_ignored() -> None: + """A sentinel like ---l3-result--- (no ::dispatch_id) must not be matched.""" + + stdout = "prefix\n---l3-result---\n{}\n---end-l3-result---\nsuffix" + + result = parse_l3_result_block(stdout=stdout, expected_dispatch_id=DISPATCH_ID) + + assert result.outcome == "no_sentinel" + + +def test_multiple_occurrences_uses_last() -> None: + """Three sentinel blocks — only the last payload is returned.""" + + payloads = [ + {"order": 1, "success": False}, + {"order": 2, "success": False}, + {"order": 3, "success": True}, + ] + stdout = "\n".join(make_stdout(json.dumps(p)) for p in payloads) + + result = parse_l3_result_block(stdout=stdout, expected_dispatch_id=DISPATCH_ID) + + assert result.outcome == "completed_clean" + assert result.payload == payloads[2] + + +def test_unicode_content_in_payload() -> None: + """JSON body with Unicode characters (emoji, CJK) is preserved.""" + + payload = {"success": True, "emoji": "🚀", "cjk": "日本語"} + stdout = make_stdout(json.dumps(payload, ensure_ascii=False)) + + result = parse_l3_result_block(stdout=stdout, expected_dispatch_id=DISPATCH_ID) + + assert result.outcome == "completed_clean" + assert result.payload == payload + + +def test_nested_triple_dashes_in_json_value() -> None: + """JSON body with --- sequences inside string values does not confuse the parser.""" + + payload = {"success": True, "value": "---this has --- dashes --- in it---"} + stdout = make_stdout(json.dumps(payload)) + + result = parse_l3_result_block(stdout=stdout, expected_dispatch_id=DISPATCH_ID) + + assert result.outcome == "completed_clean" + assert result.payload == payload + + +def test_source_field_tracks_origin(tmp_path) -> None: + """source field is 'stdout' when found in stdout, 'assistant_messages_jsonl' via JSONL.""" + + payload = {"success": True} + + # (a) found in stdout + stdout_result = parse_l3_result_block( + stdout=make_stdout(json.dumps(payload)), + expected_dispatch_id=DISPATCH_ID, + ) + assert stdout_result.source == "stdout" + + # (b) found via JSONL fallback + sentinel_text = f"{_open()}\n{json.dumps(payload)}\n{_close()}" + jsonl_path = make_jsonl_file(tmp_path, [sentinel_text]) + jsonl_result = parse_l3_result_block( + stdout="no sentinel here", + expected_dispatch_id=DISPATCH_ID, + assistant_messages_path=jsonl_path, + ) + assert jsonl_result.source == "assistant_messages_jsonl" + + +def test_channel_b_jsonl_file_missing(tmp_path) -> None: + """Non-existent assistant_messages_path yields no_sentinel gracefully.""" + + missing_path = tmp_path / "does_not_exist.jsonl" + + result = parse_l3_result_block( + stdout="no sentinel here", + expected_dispatch_id=DISPATCH_ID, + assistant_messages_path=missing_path, + ) + + assert result.outcome == "no_sentinel" + + +def test_channel_b_jsonl_empty_file(tmp_path) -> None: + """Empty assistant_messages_path file yields no_sentinel.""" + + empty_path = tmp_path / "empty.jsonl" + empty_path.write_text("", encoding="utf-8") + + result = parse_l3_result_block( + stdout="no sentinel here", + expected_dispatch_id=DISPATCH_ID, + assistant_messages_path=empty_path, + ) + + assert result.outcome == "no_sentinel" + + +def test_channel_b_jsonl_no_assistant_records(tmp_path) -> None: + """JSONL with only system/result records (no assistant) yields no_sentinel.""" + + path = tmp_path / "session.jsonl" + records = [ + {"type": "system", "message": "some system message"}, + {"type": "result", "subtype": "success", "result": "done"}, + ] + path.write_text("\n".join(json.dumps(r) for r in records), encoding="utf-8") + + result = parse_l3_result_block( + stdout="no sentinel here", + expected_dispatch_id=DISPATCH_ID, + assistant_messages_path=path, + ) + + assert result.outcome == "no_sentinel" + + +def test_extract_text_from_jsonl_ignores_thinking_blocks(tmp_path: Path) -> None: + """Thinking blocks do not contribute to sentinel search text.""" + from autoskillit.fleet.result_parser import _extract_text_from_jsonl + + path = tmp_path / "session.jsonl" + records = [ + { + "type": "assistant", + "message": {"content": [{"type": "thinking", "thinking": "Internal reasoning only."}]}, + }, + ] + path.write_text("\n".join(json.dumps(r) for r in records), encoding="utf-8") + + text = _extract_text_from_jsonl(path) + assert text == "" + + +def test_extract_text_from_jsonl_extracts_text_blocks_only(tmp_path: Path) -> None: + """Only text blocks are returned; thinking blocks are excluded.""" + from autoskillit.fleet.result_parser import _extract_text_from_jsonl + + path = tmp_path / "session.jsonl" + records = [ + { + "type": "assistant", + "message": { + "content": [ + {"type": "thinking", "thinking": "Private reasoning."}, + {"type": "text", "text": "Final result here."}, + ] + }, + }, + ] + path.write_text("\n".join(json.dumps(r) for r in records), encoding="utf-8") + + text = _extract_text_from_jsonl(path) + assert text == "Final result here." + assert "Private reasoning" not in text diff --git a/tests/fleet/test_retry_failed_dispatch.py b/tests/fleet/test_retry_failed_dispatch.py new file mode 100644 index 0000000000..b35e4d0376 --- /dev/null +++ b/tests/fleet/test_retry_failed_dispatch.py @@ -0,0 +1,241 @@ +"""Tests for retry of failed campaign dispatches (Group J extension).""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from autoskillit.fleet import ( + FLEET_HALTED_SENTINEL, + DispatchRecord, + DispatchStatus, + append_dispatch_record, + read_state, + reset_failed_dispatch, + resume_campaign_from_state, + write_initial_state, +) +from autoskillit.fleet.state_types import _validate_transition + +pytestmark = [pytest.mark.layer("fleet"), pytest.mark.small, pytest.mark.feature("fleet")] + + +def _make_dispatches(*names: str) -> list[DispatchRecord]: + return [DispatchRecord(name=n) for n in names] + + +def _state_path(tmp_path: Path) -> Path: + return tmp_path / "campaign" / "state.json" + + +# --- Transition validity --- + + +class TestFailureToPendingTransition: + def test_failure_to_pending_is_valid(self): + """_validate_transition accepts FAILURE → PENDING.""" + _validate_transition(DispatchStatus.FAILURE, DispatchStatus.PENDING, "d1") + + def test_failure_to_running_still_invalid(self): + """FAILURE → RUNNING is still rejected (only PENDING is allowed).""" + with pytest.raises(ValueError, match="Invalid transition"): + _validate_transition(DispatchStatus.FAILURE, DispatchStatus.RUNNING, "d1") + + +# --- reset_failed_dispatch --- + + +class TestResetFailedDispatch: + def test_resets_failure_to_pending(self, tmp_path: Path): + """FAILURE dispatch is reset to PENDING with cleared metadata.""" + sp = _state_path(tmp_path) + write_initial_state(sp, "cid", "camp", "/m.yaml", _make_dispatches("d1", "d2")) + append_dispatch_record( + sp, + DispatchRecord( + name="d1", + status=DispatchStatus.SUCCESS, + ), + ) + append_dispatch_record( + sp, + DispatchRecord( + name="d2", + status=DispatchStatus.FAILURE, + reason="task_failed", + dispatch_id="old-dispatch-id", + dispatched_session_id="old-session", + dispatched_pid=12345, + dispatched_starttime_ticks=999, + dispatched_boot_id="old-boot", + token_usage={"prompt_tokens": 100}, + started_at=1000.0, + ended_at=2000.0, + sidecar_path="/old/sidecar", + ), + ) + + result = reset_failed_dispatch(sp, "d2") + + assert result is True + state = read_state(sp) + assert state is not None + d2 = next(d for d in state.dispatches if d.name == "d2") + assert d2.status == DispatchStatus.PENDING + assert d2.reason == "" + assert d2.dispatch_id == "" + assert d2.dispatched_session_id == "" + assert d2.dispatched_session_log_dir == "" + assert d2.dispatched_pid == 0 + assert d2.dispatched_starttime_ticks == 0 + assert d2.dispatched_boot_id == "" + assert d2.token_usage == {} + assert d2.started_at == 0.0 + assert d2.ended_at == 0.0 + assert d2.sidecar_path is None + + def test_returns_true_on_success(self, tmp_path: Path): + """Returns True when the dispatch was actually reset.""" + sp = _state_path(tmp_path) + write_initial_state(sp, "cid", "camp", "/m.yaml", _make_dispatches("d1")) + append_dispatch_record(sp, DispatchRecord(name="d1", status=DispatchStatus.FAILURE)) + + result = reset_failed_dispatch(sp, "d1") + + assert result is True + + def test_returns_false_for_non_failure_dispatch(self, tmp_path: Path): + """Returns False when dispatch is PENDING/SUCCESS/etc.""" + sp = _state_path(tmp_path) + write_initial_state(sp, "cid", "camp", "/m.yaml", _make_dispatches("d1")) + append_dispatch_record(sp, DispatchRecord(name="d1", status=DispatchStatus.SUCCESS)) + + result = reset_failed_dispatch(sp, "d1") + + assert result is False + + def test_returns_false_for_unknown_dispatch_name(self, tmp_path: Path): + """Returns False when dispatch_name not found in state.""" + sp = _state_path(tmp_path) + write_initial_state(sp, "cid", "camp", "/m.yaml", _make_dispatches("d1")) + append_dispatch_record(sp, DispatchRecord(name="d1", status=DispatchStatus.FAILURE)) + + result = reset_failed_dispatch(sp, "nonexistent") + + assert result is False + + def test_returns_false_for_missing_state_file(self, tmp_path: Path): + """Returns False when state file does not exist (fail-safe).""" + sp = tmp_path / "nonexistent" / "state.json" + + result = reset_failed_dispatch(sp, "d1") + + assert result is False + + def test_preserves_other_dispatches(self, tmp_path: Path): + """Resetting d2 does not affect d1's state.""" + sp = _state_path(tmp_path) + write_initial_state(sp, "cid", "camp", "/m.yaml", _make_dispatches("d1", "d2")) + append_dispatch_record( + sp, + DispatchRecord( + name="d1", + status=DispatchStatus.SUCCESS, + dispatch_id="d1-dispatch", + reason="done", + ), + ) + append_dispatch_record( + sp, + DispatchRecord( + name="d2", + status=DispatchStatus.FAILURE, + reason="task_failed", + ), + ) + + reset_failed_dispatch(sp, "d2") + + state = read_state(sp) + assert state is not None + d1 = next(d for d in state.dispatches if d.name == "d1") + assert d1.status == DispatchStatus.SUCCESS + assert d1.dispatch_id == "d1-dispatch" + assert d1.reason == "done" + + +# --- resume_campaign_from_state with reset_on_retry --- + + +class TestResumeResetOnRetry: + def test_reset_on_retry_resets_failure_and_selects_as_next(self, tmp_path: Path): + """reset_on_retry=True resets FAILURE dispatch to PENDING, selects as next.""" + sp = _state_path(tmp_path) + write_initial_state(sp, "cid", "camp", "/m.yaml", _make_dispatches("d1", "d2", "d3")) + append_dispatch_record(sp, DispatchRecord(name="d1", status=DispatchStatus.SUCCESS)) + append_dispatch_record( + sp, + DispatchRecord( + name="d2", + status=DispatchStatus.FAILURE, + dispatch_id="d2-uuid", + dispatched_session_id="d2-sess", + ), + ) + + decision = resume_campaign_from_state(sp, continue_on_failure=False, reset_on_retry=True) + + assert decision is not None + assert decision.next_dispatch_name == "d2" + assert decision.completed_dispatches_block != FLEET_HALTED_SENTINEL + state = read_state(sp) + assert state is not None + d2 = next(d for d in state.dispatches if d.name == "d2") + assert d2.status == DispatchStatus.PENDING + assert d2.dispatch_id == "" + assert d2.dispatched_session_id == "" + d3 = next(d for d in state.dispatches if d.name == "d3") + assert d3.status == DispatchStatus.PENDING + assert d3.dispatch_id == "" + + def test_reset_on_retry_false_still_halts(self, tmp_path: Path): + """reset_on_retry=False preserves existing halt behavior.""" + sp = _state_path(tmp_path) + write_initial_state(sp, "cid", "camp", "/m.yaml", _make_dispatches("d1", "d2")) + append_dispatch_record(sp, DispatchRecord(name="d1", status=DispatchStatus.SUCCESS)) + append_dispatch_record(sp, DispatchRecord(name="d2", status=DispatchStatus.FAILURE)) + + decision = resume_campaign_from_state(sp, continue_on_failure=False, reset_on_retry=False) + + assert decision is not None + assert decision.completed_dispatches_block == FLEET_HALTED_SENTINEL + + def test_reset_on_retry_with_continue_on_failure_true_is_noop(self, tmp_path: Path): + """When continue_on_failure=True, reset_on_retry has no effect on FAILURE dispatches.""" + sp = _state_path(tmp_path) + write_initial_state(sp, "cid", "camp", "/m.yaml", _make_dispatches("d1", "d2", "d3")) + append_dispatch_record(sp, DispatchRecord(name="d1", status=DispatchStatus.SUCCESS)) + append_dispatch_record(sp, DispatchRecord(name="d2", status=DispatchStatus.FAILURE)) + + decision = resume_campaign_from_state(sp, continue_on_failure=True, reset_on_retry=True) + + assert decision is not None + assert decision.next_dispatch_name == "d3" + assert decision.completed_dispatches_block != FLEET_HALTED_SENTINEL + state = read_state(sp) + assert state is not None + d2 = next(d for d in state.dispatches if d.name == "d2") + assert d2.status == DispatchStatus.FAILURE + + def test_reset_on_retry_clears_dispatch_from_completed_block(self, tmp_path: Path): + """Reset dispatch does NOT appear in completed_dispatches_block.""" + sp = _state_path(tmp_path) + write_initial_state(sp, "cid", "camp", "/m.yaml", _make_dispatches("d1", "d2")) + append_dispatch_record(sp, DispatchRecord(name="d1", status=DispatchStatus.SUCCESS)) + append_dispatch_record(sp, DispatchRecord(name="d2", status=DispatchStatus.FAILURE)) + + decision = resume_campaign_from_state(sp, continue_on_failure=False, reset_on_retry=True) + + assert decision is not None + assert decision.completed_dispatches_block == "- d1: success" diff --git a/tests/fleet/test_sidecar.py b/tests/fleet/test_sidecar.py new file mode 100644 index 0000000000..6bf2afc513 --- /dev/null +++ b/tests/fleet/test_sidecar.py @@ -0,0 +1,179 @@ +import json +from pathlib import Path + +import pytest + +from autoskillit.fleet.sidecar import ( + IssueSidecarEntry, + append_sidecar_entry, + compute_remaining_issues, + read_sidecar, + read_sidecar_from_path, + sidecar_path, +) + +pytestmark = [pytest.mark.layer("fleet"), pytest.mark.small, pytest.mark.feature("fleet")] + +URL1 = "https://github.com/org/repo/issues/101" +URL2 = "https://github.com/org/repo/issues/102" +URL3 = "https://github.com/org/repo/issues/103" +TS = "2026-04-27T12:00:00Z" + + +class TestSidecarPath: + def test_deterministic(self, tmp_path: Path) -> None: + assert sidecar_path("abc-123", tmp_path) == sidecar_path("abc-123", tmp_path) + + def test_contains_dispatch_id(self, tmp_path: Path) -> None: + p = sidecar_path("my-dispatch", tmp_path) + assert "my-dispatch_issues.jsonl" in p.name + + def test_in_dispatches_dir(self, tmp_path: Path) -> None: + p = sidecar_path("d1", tmp_path) + assert p.parent.name == "dispatches" + + +class TestAppendSidecarEntry: + def test_creates_file_on_first_append(self, tmp_path: Path) -> None: + entry = IssueSidecarEntry(issue_url=URL1, status="completed", ts=TS) + append_sidecar_entry("d1", entry, tmp_path) + assert sidecar_path("d1", tmp_path).exists() + + def test_multiple_appends_produce_multiple_lines(self, tmp_path: Path) -> None: + for url in [URL1, URL2, URL3]: + append_sidecar_entry( + "d2", IssueSidecarEntry(issue_url=url, status="completed", ts=TS), tmp_path + ) + lines = [ln for ln in sidecar_path("d2", tmp_path).read_text().splitlines() if ln.strip()] + assert len(lines) == 3 + + def test_appended_line_is_valid_json(self, tmp_path: Path) -> None: + entry = IssueSidecarEntry(issue_url=URL1, status="failed", reason="tests failed", ts=TS) + append_sidecar_entry("d3", entry, tmp_path) + parsed = json.loads(sidecar_path("d3", tmp_path).read_text().strip()) + assert parsed["issue_url"] == URL1 + assert parsed["status"] == "failed" + assert parsed["reason"] == "tests failed" + + def test_optional_fields_omitted_when_none(self, tmp_path: Path) -> None: + entry = IssueSidecarEntry(issue_url=URL1, status="completed", ts=TS) + append_sidecar_entry("d4", entry, tmp_path) + parsed = json.loads(sidecar_path("d4", tmp_path).read_text().strip()) + assert "pr_url" not in parsed or parsed.get("pr_url") is None + assert "reason" not in parsed or parsed.get("reason") is None + + def test_pr_url_written_for_completed(self, tmp_path: Path) -> None: + entry = IssueSidecarEntry( + issue_url=URL1, + status="completed", + pr_url="https://github.com/org/repo/pull/200", + ts=TS, + ) + append_sidecar_entry("d5", entry, tmp_path) + parsed = json.loads(sidecar_path("d5", tmp_path).read_text().strip()) + assert parsed["pr_url"] == "https://github.com/org/repo/pull/200" + + +class TestReadSidecar: + def test_missing_file_returns_empty(self, tmp_path: Path) -> None: + assert read_sidecar("nonexistent", tmp_path) == [] + + def test_empty_file_returns_empty(self, tmp_path: Path) -> None: + p = sidecar_path("d6", tmp_path) + p.parent.mkdir(parents=True, exist_ok=True) + p.write_text("") + assert read_sidecar("d6", tmp_path) == [] + + def test_truncated_line_skipped(self, tmp_path: Path) -> None: + p = sidecar_path("d7", tmp_path) + p.parent.mkdir(parents=True, exist_ok=True) + good = json.dumps({"issue_url": URL1, "status": "completed", "ts": TS}) + p.write_text(f'{good}\n{{"issue_url":"truncated-bad-json\n') + entries = read_sidecar("d7", tmp_path) + assert len(entries) == 1 + assert entries[0].issue_url == URL1 + + def test_blank_lines_skipped(self, tmp_path: Path) -> None: + p = sidecar_path("d8", tmp_path) + p.parent.mkdir(parents=True, exist_ok=True) + good = json.dumps({"issue_url": URL1, "status": "completed", "ts": TS}) + p.write_text(f"{good}\n\n\n") + assert len(read_sidecar("d8", tmp_path)) == 1 + + def test_round_trip_fidelity(self, tmp_path: Path) -> None: + entry = IssueSidecarEntry( + issue_url=URL1, + status="completed", + pr_url="https://github.com/org/repo/pull/200", + ts=TS, + ) + append_sidecar_entry("d9", entry, tmp_path) + [result] = read_sidecar("d9", tmp_path) + assert result.issue_url == entry.issue_url + assert result.status == entry.status + assert result.pr_url == entry.pr_url + assert result.ts == entry.ts + + +class TestComputeRemainingIssues: + def test_empty_sidecar_returns_all(self, tmp_path: Path) -> None: + originals = [URL1, URL2, URL3] + assert compute_remaining_issues("d10", originals, tmp_path) == originals + + def test_completed_removed(self, tmp_path: Path) -> None: + append_sidecar_entry( + "d11", IssueSidecarEntry(issue_url=URL1, status="completed", ts=TS), tmp_path + ) + assert compute_remaining_issues("d11", [URL1, URL2], tmp_path) == [URL2] + + def test_failed_removed(self, tmp_path: Path) -> None: + append_sidecar_entry( + "d12", IssueSidecarEntry(issue_url=URL1, status="failed", reason="x", ts=TS), tmp_path + ) + assert compute_remaining_issues("d12", [URL1, URL2], tmp_path) == [URL2] + + def test_preserves_original_order(self, tmp_path: Path) -> None: + originals = [URL1, URL2, URL3] + append_sidecar_entry( + "d13", IssueSidecarEntry(issue_url=URL2, status="completed", ts=TS), tmp_path + ) + assert compute_remaining_issues("d13", originals, tmp_path) == [URL1, URL3] + + def test_all_done_returns_empty(self, tmp_path: Path) -> None: + for url in [URL1, URL2]: + append_sidecar_entry( + "d14", IssueSidecarEntry(issue_url=url, status="completed", ts=TS), tmp_path + ) + assert compute_remaining_issues("d14", [URL1, URL2], tmp_path) == [] + + def test_crash_scenario_2_of_4_done(self, tmp_path: Path) -> None: + originals = [URL1, URL2, URL3, "https://github.com/org/repo/issues/104"] + for url in originals[:2]: + append_sidecar_entry( + "d15", IssueSidecarEntry(issue_url=url, status="completed", ts=TS), tmp_path + ) + remaining = compute_remaining_issues("d15", originals, tmp_path) + assert remaining == originals[2:] + + +class TestReadSidecarFromPath: + def test_nonexistent_parent_returns_empty(self) -> None: + result = read_sidecar_from_path(Path("/nonexistent/dir/issues.jsonl")) + assert result == [] + + def test_valid_jsonl_parsed_correctly(self, tmp_path: Path) -> None: + p = tmp_path / "issues.jsonl" + lines = [ + json.dumps({"issue_url": URL1, "status": "completed", "ts": TS}), + json.dumps( + {"issue_url": URL2, "status": "failed", "reason": "tests failed", "ts": TS} + ), + ] + p.write_text("\n".join(lines) + "\n", encoding="utf-8") + + entries = read_sidecar_from_path(p) + + assert len(entries) == 2 + assert all(isinstance(e, IssueSidecarEntry) for e in entries) + assert entries[0].issue_url == URL1 + assert entries[1].issue_url == URL2 diff --git a/tests/fleet/test_state.py b/tests/fleet/test_state.py new file mode 100644 index 0000000000..77a49539db --- /dev/null +++ b/tests/fleet/test_state.py @@ -0,0 +1,1027 @@ +"""Tests for fleet state module (Group J).""" + +from __future__ import annotations + +import errno +import json +import os +import threading +from pathlib import Path +from unittest.mock import patch + +import pytest + +from autoskillit.fleet import ( + FLEET_HALTED_SENTINEL, + DispatchRecord, + DispatchStatus, + append_dispatch_record, + crash_recover_dispatch, + has_failed_dispatch, + mark_dispatch_resumable, + mark_dispatch_running, + read_all_campaign_captures, + read_state, + resume_campaign_from_state, + write_captured_values, + write_initial_state, +) + +pytestmark = [pytest.mark.layer("fleet"), pytest.mark.small, pytest.mark.feature("fleet")] + + +def _make_dispatches(*names: str) -> list[DispatchRecord]: + return [DispatchRecord(name=n) for n in names] + + +def _state_path(tmp_path: Path) -> Path: + return tmp_path / "campaign" / "state.json" + + +class TestInitialState: + def test_initial_state_file_has_all_dispatches_pending(self, tmp_path: Path) -> None: + sp = _state_path(tmp_path) + dispatches = _make_dispatches("a", "b", "c") + write_initial_state(sp, "cid-1", "my-campaign", "/m.yaml", dispatches) + + state = read_state(sp) + assert state is not None + assert state.schema_version == 4 + assert state.campaign_id == "cid-1" + assert state.campaign_name == "my-campaign" + assert state.manifest_path == "/m.yaml" + assert len(state.dispatches) == 3 + for d in state.dispatches: + assert d.status == DispatchStatus.PENDING + + def test_read_state_round_trips_through_from_dict(self, tmp_path: Path) -> None: + sp = _state_path(tmp_path) + dispatches = [ + DispatchRecord( + name="a", + status=DispatchStatus.RUNNING, + dispatch_id="d1", + caller_session_id="c1", + dispatched_session_id="s1", + dispatched_session_log_dir="/log", + dispatched_pid=1234, + dispatched_starttime_ticks=9999, + dispatched_boot_id="b1", + reason="started", + kill_reason="", + infra_exit_category="", + token_usage={"input": 100}, + started_at=1.0, + ended_at=2.0, + sidecar_path="/s.jsonl", + ), + DispatchRecord(name="b"), + ] + write_initial_state(sp, "cid", "cname", "/m.yaml", dispatches) + state = read_state(sp) + assert state is not None + assert [d.name for d in state.dispatches] == ["a", "b"] + a = state.dispatches[0] + assert a.status == DispatchStatus.RUNNING + assert a.dispatch_id == "d1" + assert a.caller_session_id == "c1" + assert a.dispatched_session_id == "s1" + assert a.dispatched_session_log_dir == "/log" + assert a.dispatched_pid == 1234 + assert a.dispatched_starttime_ticks == 9999 + assert a.dispatched_boot_id == "b1" + assert a.reason == "started" + assert a.token_usage == {"input": 100} + assert a.started_at == 1.0 + assert a.ended_at == 2.0 + assert a.sidecar_path == "/s.jsonl" + assert state.dispatches[1].status == DispatchStatus.PENDING + + +class TestDispatchRecordFromDict: + def test_from_dict_canonical_fields(self) -> None: + d = { + "name": "build-docs", + "status": "running", + "dispatch_id": "abc", + "caller_session_id": "c1", + "dispatched_session_id": "s1", + "dispatched_session_log_dir": "/log", + "dispatched_pid": 1234, + "dispatched_starttime_ticks": 9999, + "dispatched_boot_id": "b1", + "reason": "started", + "kill_reason": "", + "infra_exit_category": "", + "token_usage": {"input": 100}, + "started_at": 1.0, + "ended_at": 2.0, + "sidecar_path": "/s.jsonl", + } + rec = DispatchRecord.from_dict(d) + assert rec.name == "build-docs" + assert rec.status == DispatchStatus.RUNNING + assert rec.dispatch_id == "abc" + assert rec.caller_session_id == "c1" + assert rec.dispatched_session_id == "s1" + assert rec.dispatched_session_log_dir == "/log" + assert rec.dispatched_pid == 1234 + assert rec.dispatched_starttime_ticks == 9999 + assert rec.dispatched_boot_id == "b1" + assert rec.reason == "started" + assert rec.kill_reason == "" + assert rec.infra_exit_category == "" + assert rec.token_usage == {"input": 100} + assert rec.started_at == 1.0 + assert rec.ended_at == 2.0 + assert rec.sidecar_path == "/s.jsonl" + + def test_from_dict_legacy_l2_fields(self) -> None: + d = { + "name": "t", + "l2_session_id": "l2s", + "l2_session_log_dir": "/l2", + "l2_pid": 555, + "l2_starttime_ticks": 111, + "l2_boot_id": "l2b", + } + rec = DispatchRecord.from_dict(d) + assert rec.dispatched_session_id == "l2s" + assert rec.dispatched_session_log_dir == "/l2" + assert rec.dispatched_pid == 555 + assert rec.dispatched_starttime_ticks == 111 + assert rec.dispatched_boot_id == "l2b" + + def test_from_dict_l3_takes_priority_over_l2(self) -> None: + d = { + "name": "t", + "l3_session_id": "l3s", + "l2_session_id": "l2s", + "l3_session_log_dir": "/l3", + "l2_session_log_dir": "/l2", + "l3_pid": 333, + "l2_pid": 222, + "l3_starttime_ticks": 300, + "l2_starttime_ticks": 200, + "l3_boot_id": "l3b", + "l2_boot_id": "l2b", + } + rec = DispatchRecord.from_dict(d) + assert rec.dispatched_session_id == "l3s" + assert rec.dispatched_session_log_dir == "/l3" + assert rec.dispatched_pid == 333 + assert rec.dispatched_starttime_ticks == 300 + assert rec.dispatched_boot_id == "l3b" + + def test_from_dict_canonical_takes_priority_over_l3(self) -> None: + d = {"name": "t", "dispatched_session_id": "canon", "l3_session_id": "l3s"} + rec = DispatchRecord.from_dict(d) + assert rec.dispatched_session_id == "canon" + + def test_from_dict_minimal_dict(self) -> None: + rec = DispatchRecord.from_dict({"name": "minimal"}) + assert rec.name == "minimal" + assert rec.status == DispatchStatus.PENDING + assert rec.dispatched_pid == 0 + assert rec.token_usage == {} + assert rec.sidecar_path is None + + def test_from_dict_pid_zero_not_falsy(self) -> None: + d = {"name": "t", "dispatched_pid": 0, "l3_pid": 999} + rec = DispatchRecord.from_dict(d) + assert rec.dispatched_pid == 0 + + def test_from_dict_starttime_ticks_zero_not_falsy(self) -> None: + d = {"name": "t", "dispatched_starttime_ticks": 0, "l3_starttime_ticks": 888} + rec = DispatchRecord.from_dict(d) + assert rec.dispatched_starttime_ticks == 0 + + +class TestStateDecompositionImports: + def test_state_types_importable(self) -> None: + from autoskillit.fleet.state_types import ( + DispatchRecord, + DispatchStatus, + ) + + assert DispatchStatus.PENDING == "pending" + assert hasattr(DispatchRecord, "from_dict") + + def test_state_gates_importable(self) -> None: + from autoskillit.fleet.state_gates import record_gate_outcome + + assert callable(record_gate_outcome) + + def test_state_recovery_importable(self) -> None: + from autoskillit.fleet.state_recovery import ( + resume_campaign_from_state, + ) + + assert callable(resume_campaign_from_state) + + def test_backward_compat_from_state_module(self) -> None: + from autoskillit.fleet.state import ( + read_state, + ) + + assert callable(read_state) + + +class TestAppendDispatchRecord: + def test_append_dispatch_record_updates_status(self, tmp_path: Path) -> None: + sp = _state_path(tmp_path) + write_initial_state(sp, "cid", "camp", "/m.yaml", _make_dispatches("x", "y")) + + updated = DispatchRecord(name="x", status=DispatchStatus.SUCCESS) + append_dispatch_record(sp, updated) + + state = read_state(sp) + assert state is not None + assert len(state.dispatches) == 2 + assert state.dispatches[0].name == "x" + assert state.dispatches[0].status == DispatchStatus.SUCCESS + + +class TestAtomicWriteSurvivesPartialTmp: + def test_atomic_write_survives_partial_tmp_file(self, tmp_path: Path) -> None: + sp = _state_path(tmp_path) + write_initial_state(sp, "cid", "camp", "/m.yaml", _make_dispatches("a")) + + original = sp.read_text(encoding="utf-8") + real_replace = os.replace + + call_count = 0 + + def failing_replace(src, dst): + nonlocal call_count + call_count += 1 + if call_count == 1: + raise OSError("simulated crash") + return real_replace(src, dst) + + with patch("autoskillit.core.io.os.replace", side_effect=failing_replace): + with pytest.raises(OSError, match="simulated crash"): + mark_dispatch_running(sp, "a", dispatch_id="d1", dispatched_pid=42) + + assert sp.read_text(encoding="utf-8") == original + + # Retry succeeds + mark_dispatch_running(sp, "a", dispatch_id="d1", dispatched_pid=42) + state = read_state(sp) + assert state is not None + assert state.dispatches[0].status == DispatchStatus.RUNNING + + +class TestResumeSkipsSuccessful: + def test_resume_skips_successful_dispatches(self, tmp_path: Path) -> None: + sp = _state_path(tmp_path) + write_initial_state(sp, "cid", "camp", "/m.yaml", _make_dispatches("A", "B", "C")) + append_dispatch_record(sp, DispatchRecord(name="A", status=DispatchStatus.SUCCESS)) + + decision = resume_campaign_from_state(sp, continue_on_failure=True) + assert decision is not None + assert decision.next_dispatch_name == "B" + assert "A" in decision.completed_dispatches_block + + +class TestResumeMarksRunningInterrupted: + def test_resume_marks_running_as_interrupted(self, tmp_path: Path) -> None: + sp = _state_path(tmp_path) + write_initial_state(sp, "cid", "camp", "/m.yaml", _make_dispatches("A", "B", "C")) + append_dispatch_record(sp, DispatchRecord(name="A", status=DispatchStatus.SUCCESS)) + mark_dispatch_running(sp, "B", dispatch_id="d-b", dispatched_pid=99) + + decision = resume_campaign_from_state(sp, continue_on_failure=True) + assert decision is not None + + # B should now be interrupted on disk + state = read_state(sp) + assert state is not None + b = next(d for d in state.dispatches if d.name == "B") + assert b.status == DispatchStatus.INTERRUPTED + + # INTERRUPTED dispatches are skipped when searching for next_dispatch_name; + # the first PENDING dispatch (C) is returned, not the interrupted one (B). + assert decision.next_dispatch_name == "C" + + +class TestResumeRejectsHaltedOnFailure: + def test_resume_rejects_if_halted_on_failure_with_no_continue_on_failure( + self, tmp_path: Path + ) -> None: + sp = _state_path(tmp_path) + write_initial_state(sp, "cid", "camp", "/m.yaml", _make_dispatches("A", "B", "C")) + append_dispatch_record(sp, DispatchRecord(name="A", status=DispatchStatus.SUCCESS)) + append_dispatch_record(sp, DispatchRecord(name="B", status=DispatchStatus.FAILURE)) + + decision = resume_campaign_from_state(sp, continue_on_failure=False) + assert decision is not None + assert decision.next_dispatch_name == "" + assert decision.completed_dispatches_block == FLEET_HALTED_SENTINEL + + +class TestAtomicUnderConcurrentRead: + def test_state_json_atomic_under_concurrent_read(self, tmp_path: Path) -> None: + sp = _state_path(tmp_path) + write_initial_state(sp, "cid", "camp", "/m.yaml", _make_dispatches("a")) + + barrier = threading.Barrier(2, timeout=5) + errors: list[str] = [] + + def writer(): + barrier.wait() + mark_dispatch_running(sp, "a", dispatch_id="d1", dispatched_pid=42) + + def reader(): + barrier.wait() + for _ in range(50): + state = read_state(sp) + if state is None: + errors.append("read_state returned None (corrupted)") + break + + t_write = threading.Thread(target=writer) + t_read = threading.Thread(target=reader) + t_write.start() + t_read.start() + t_write.join(timeout=5) + t_read.join(timeout=5) + + assert not errors, f"Concurrent read errors: {errors}" + + +class TestWriteDiskFull: + def test_state_write_disk_full(self, tmp_path: Path) -> None: + sp = _state_path(tmp_path) + write_initial_state(sp, "cid", "camp", "/m.yaml", _make_dispatches("a")) + + original = sp.read_text(encoding="utf-8") + + def enospc_replace(src, dst): + raise OSError(errno.ENOSPC, "No space left on device") + + with patch("autoskillit.core.io.os.replace", side_effect=enospc_replace): + with pytest.raises(OSError): + mark_dispatch_running(sp, "a", dispatch_id="d1", dispatched_pid=42) + + assert sp.read_text(encoding="utf-8") == original + + +class TestReadStateRejectsCorrupted: + def test_read_state_rejects_corrupted_json(self, tmp_path: Path) -> None: + sp = _state_path(tmp_path) + sp.parent.mkdir(parents=True, exist_ok=True) + sp.write_bytes(b"not valid json {{{") + + result = read_state(sp) + assert result is None + + +class TestCapturedValuesRoundTrip: + def test_captured_values_round_trip(self, tmp_path: Path) -> None: + sp = _state_path(tmp_path) + write_initial_state(sp, "cid", "camp", "/m.yaml", _make_dispatches("a")) + write_captured_values(sp, {"k": "v"}) + + state = read_state(sp) + assert state is not None + assert state.captured_values == {"k": "v"} + + +class TestReadV2StateFileDefaultsCapturedValues: + def test_read_v2_state_file_defaults_captured_values(self, tmp_path: Path) -> None: + sp = _state_path(tmp_path) + sp.parent.mkdir(parents=True, exist_ok=True) + v2_data = { + "schema_version": 2, + "campaign_id": "cid", + "campaign_name": "camp", + "manifest_path": "/m.yaml", + "started_at": 0.0, + "dispatches": [], + } + sp.write_text(json.dumps(v2_data), encoding="utf-8") + + state = read_state(sp) + assert state is not None + assert state.captured_values == {} + + +class TestReadAllCampaignCaptures: + def test_read_all_campaign_captures_merges_across_dispatches(self, tmp_path: Path) -> None: + dispatches_dir = tmp_path / "dispatches" + dispatches_dir.mkdir() + + for i, (key, val) in enumerate([("a", "1"), ("b", "2")]): + sp = dispatches_dir / f"state{i}.json" + write_initial_state(sp, "cid-merge", "camp", "/m.yaml", _make_dispatches(f"d{i}")) + append_dispatch_record(sp, DispatchRecord(name=f"d{i}", status=DispatchStatus.SUCCESS)) + write_captured_values(sp, {key: val}) + + result = read_all_campaign_captures(dispatches_dir, "cid-merge") + assert result == {"a": "1", "b": "2"} + + def test_read_all_campaign_captures_ignores_non_success_dispatches( + self, tmp_path: Path + ) -> None: + dispatches_dir = tmp_path / "dispatches" + dispatches_dir.mkdir() + + sp = dispatches_dir / "failure.json" + write_initial_state(sp, "cid-fail", "camp", "/m.yaml", _make_dispatches("d1")) + append_dispatch_record(sp, DispatchRecord(name="d1", status=DispatchStatus.FAILURE)) + write_captured_values(sp, {"k": "should-not-appear"}) + + result = read_all_campaign_captures(dispatches_dir, "cid-fail") + assert result == {} + + def test_read_all_campaign_captures_empty_dir(self, tmp_path: Path) -> None: + result = read_all_campaign_captures(tmp_path / "nonexistent", "any-id") + assert result == {} + + +class TestGateDispatchSuccessIsSkippedOnResume: + def test_gate_dispatch_success_is_skipped_on_resume(self, tmp_path: Path) -> None: + sp = _state_path(tmp_path) + write_initial_state( + sp, "cid", "camp", "/m.yaml", _make_dispatches("gate-check", "phase-one") + ) + append_dispatch_record( + sp, DispatchRecord(name="gate-check", status=DispatchStatus.SUCCESS) + ) + + decision = resume_campaign_from_state(sp, continue_on_failure=False) + assert decision is not None + assert decision.next_dispatch_name == "phase-one" + assert "gate-check" in decision.completed_dispatches_block + + +class TestGateDispatchFailureHaltsCampaign: + def test_gate_dispatch_failure_halts_campaign(self, tmp_path: Path) -> None: + sp = _state_path(tmp_path) + write_initial_state( + sp, "cid", "camp", "/m.yaml", _make_dispatches("gate-check", "phase-one") + ) + append_dispatch_record( + sp, DispatchRecord(name="gate-check", status=DispatchStatus.FAILURE) + ) + + decision = resume_campaign_from_state(sp, continue_on_failure=False) + assert decision is not None + assert decision.completed_dispatches_block == FLEET_HALTED_SENTINEL + assert decision.next_dispatch_name == "" + + +class TestResumeSkipsAliveRunningDispatch: + def test_resume_skips_running_dispatch_when_alive(self, tmp_path: Path, monkeypatch) -> None: + """RUNNING dispatch with live process is NOT interrupted on resume.""" + sp = _state_path(tmp_path) + record = DispatchRecord( + name="issue-1", + status=DispatchStatus.RUNNING, + dispatched_pid=12345, + dispatched_boot_id="abc", + dispatched_starttime_ticks=999, + ) + monkeypatch.setattr( + "autoskillit.fleet.is_dispatch_session_alive", + lambda r: True, + ) + write_initial_state(sp, "c1", "test", "", [record]) + + decision = resume_campaign_from_state(sp, continue_on_failure=False) + + state = read_state(sp) + assert state is not None + assert state.dispatches[0].status == DispatchStatus.RUNNING + assert decision is not None + assert decision.next_dispatch_name == "" + + +class TestResumeInterruptsStaleRunningDispatch: + def test_resume_interrupts_stale_running_dispatch(self, tmp_path: Path, monkeypatch) -> None: + """resume_campaign_from_state marks RUNNING as INTERRUPTED when process is dead.""" + sp = _state_path(tmp_path) + record = DispatchRecord( + name="issue-1", + status=DispatchStatus.RUNNING, + dispatched_pid=0, + dispatched_boot_id="", + dispatched_starttime_ticks=0, + ) + monkeypatch.setattr( + "autoskillit.fleet.is_dispatch_session_alive", + lambda r: False, + ) + write_initial_state(sp, "c1", "test", "", [record]) + + resume_campaign_from_state(sp, continue_on_failure=False) + + state = read_state(sp) + assert state is not None + assert state.dispatches[0].status == DispatchStatus.INTERRUPTED + + +class TestResumeLockPreventsDoubleInterrupt: + def test_resume_lock_prevents_concurrent_mutation(self, tmp_path: Path, monkeypatch) -> None: + """Two concurrent resume_campaign_from_state calls serialize — no double-interrupt.""" + sp = _state_path(tmp_path) + record = DispatchRecord( + name="issue-1", + status=DispatchStatus.RUNNING, + dispatched_pid=0, + dispatched_boot_id="", + dispatched_starttime_ticks=0, + ) + monkeypatch.setattr( + "autoskillit.fleet.is_dispatch_session_alive", + lambda r: False, + ) + write_initial_state(sp, "c1", "test", "", [record]) + + results: list[object] = [] + + def _call() -> None: + results.append(resume_campaign_from_state(sp, continue_on_failure=False)) + + t1 = threading.Thread(target=_call) + t2 = threading.Thread(target=_call) + t1.start() + t2.start() + t1.join() + t2.join() + + state = read_state(sp) + assert state is not None + assert state.dispatches[0].status == DispatchStatus.INTERRUPTED + assert len(results) == 2 + + +class TestResumeTransitionsRunningToResumable: + def test_resume_marks_running_as_resumable_when_sidecar_exists( + self, tmp_path: Path, monkeypatch + ) -> None: + sp = _state_path(tmp_path) + write_initial_state(sp, "c1", "myCampaign", "manifest.yaml", _make_dispatches("impl")) + sidecar_file = sp.parent / "d1111_issues.jsonl" + mark_dispatch_running( + sp, "impl", dispatch_id="d1111", dispatched_pid=999, sidecar_path=str(sidecar_file) + ) + sidecar_file.write_text( + '{"issue_url":"https://github.com/o/r/issues/1","status":"completed","ts":"2026-01-01T00:00:00"}\n' + ) + monkeypatch.setattr("autoskillit.fleet.is_dispatch_session_alive", lambda _: False) + + decision = resume_campaign_from_state(sp, continue_on_failure=False) + + state = read_state(sp) + assert state is not None + latest = next(d for d in reversed(state.dispatches) if d.name == "impl") + assert latest.status == DispatchStatus.RESUMABLE + assert latest.sidecar_path is not None + assert decision is not None + assert decision.is_resumable is True + assert decision.next_dispatch_name == "impl" + + +class TestResumeTransitionsRunningToInterruptedNoSidecar: + def test_resume_marks_running_as_interrupted_when_no_sidecar( + self, tmp_path: Path, monkeypatch + ) -> None: + sp = _state_path(tmp_path) + write_initial_state(sp, "c1", "myCampaign", "manifest.yaml", _make_dispatches("impl")) + mark_dispatch_running(sp, "impl", dispatch_id="d1111", dispatched_pid=999) + monkeypatch.setattr("autoskillit.fleet.is_dispatch_session_alive", lambda _: False) + + resume_campaign_from_state(sp, continue_on_failure=False) + + state = read_state(sp) + assert state is not None + latest = next(d for d in reversed(state.dispatches) if d.name == "impl") + assert latest.status == DispatchStatus.INTERRUPTED + + +class TestResumeTransitionsRunningToInterruptedCorruptSidecar: + def test_resume_marks_running_as_interrupted_when_sidecar_corrupt( + self, tmp_path: Path, monkeypatch + ) -> None: + sp = _state_path(tmp_path) + write_initial_state(sp, "c1", "myCampaign", "manifest.yaml", _make_dispatches("impl")) + sidecar_file = sp.parent / "d1111_issues.jsonl" + mark_dispatch_running( + sp, "impl", dispatch_id="d1111", dispatched_pid=999, sidecar_path=str(sidecar_file) + ) + sidecar_file.write_text("{not valid json{{{\n") + monkeypatch.setattr("autoskillit.fleet.is_dispatch_session_alive", lambda _: False) + + resume_campaign_from_state(sp, continue_on_failure=False) + + state = read_state(sp) + assert state is not None + latest = next(d for d in reversed(state.dispatches) if d.name == "impl") + assert latest.status == DispatchStatus.INTERRUPTED + + +class TestResumeEmptySidecarIsResumable: + def test_resume_marks_running_as_resumable_when_sidecar_empty( + self, tmp_path: Path, monkeypatch + ) -> None: + sp = _state_path(tmp_path) + write_initial_state(sp, "c1", "myCampaign", "manifest.yaml", _make_dispatches("impl")) + sidecar_file = sp.parent / "d1111_issues.jsonl" + mark_dispatch_running( + sp, "impl", dispatch_id="d1111", dispatched_pid=999, sidecar_path=str(sidecar_file) + ) + sidecar_file.write_text("") + monkeypatch.setattr("autoskillit.fleet.is_dispatch_session_alive", lambda _: False) + + decision = resume_campaign_from_state(sp, continue_on_failure=False) + + state = read_state(sp) + assert state is not None + latest = next(d for d in reversed(state.dispatches) if d.name == "impl") + assert latest.status == DispatchStatus.RESUMABLE + assert decision is not None + assert decision.is_resumable is True + + +class TestResumableSelectedBeforePending: + def test_resumable_selected_as_next_before_pending(self, tmp_path: Path) -> None: + sp = _state_path(tmp_path) + write_initial_state( + sp, "c1", "myCampaign", "manifest.yaml", _make_dispatches("impl-1", "impl-2") + ) + mark_dispatch_running(sp, "impl-1", dispatch_id="d1111", dispatched_pid=999) + # Non-existent sidecar is intentional: test covers selection ordering only, + # not the sidecar-existence branch in crash_recover_dispatch. + mark_dispatch_resumable(sp, "impl-1", sidecar_path=str(sp.parent / "d1111_issues.jsonl")) + + decision = resume_campaign_from_state(sp, continue_on_failure=False) + + assert decision is not None + assert decision.next_dispatch_name == "impl-1" + assert decision.is_resumable is True + + def test_resume_decision_carries_dispatched_session_id(self, tmp_path: Path) -> None: + sp = _state_path(tmp_path) + write_initial_state(sp, "c1", "myCampaign", "manifest.yaml", _make_dispatches("impl-1")) + mark_dispatch_running(sp, "impl-1", dispatch_id="d2222", dispatched_pid=888) + append_dispatch_record( + sp, + DispatchRecord( + name="impl-1", + status=DispatchStatus.RESUMABLE, + dispatch_id="d2222", + dispatched_session_id="sess-xyz-test", + sidecar_path=str(sp.parent / "d2222_issues.jsonl"), + ), + ) + + decision = resume_campaign_from_state(sp, continue_on_failure=False) + + assert decision is not None + assert decision.is_resumable is True + assert decision.dispatched_session_id == "sess-xyz-test" + + +class TestResumableStateTransitionsValid: + def test_resumable_valid_transitions(self, tmp_path: Path) -> None: + for next_status in [ + DispatchStatus.RUNNING, + DispatchStatus.SUCCESS, + DispatchStatus.FAILURE, + DispatchStatus.INTERRUPTED, + ]: + sp = _state_path(tmp_path / next_status.value) + write_initial_state(sp, "c1", "camp", "m.yaml", _make_dispatches("impl")) + mark_dispatch_running(sp, "impl", dispatch_id="d1", dispatched_pid=1) + mark_dispatch_resumable(sp, "impl", sidecar_path=str(tmp_path / "s.jsonl")) + append_dispatch_record(sp, DispatchRecord(name="impl", status=next_status)) + state = read_state(sp) + assert state is not None + matches = [d for d in reversed(state.dispatches) if d.name == "impl"] + assert matches, f"no dispatch named 'impl' found for status {next_status}" + assert matches[0].status == next_status + + +class TestMarkDispatchResumable: + def test_mark_dispatch_resumable_sets_sidecar_path(self, tmp_path: Path) -> None: + sp = _state_path(tmp_path) + write_initial_state(sp, "c1", "myCampaign", "manifest.yaml", _make_dispatches("impl")) + expected_sidecar = str(sp.parent / "d1111_issues.jsonl") + mark_dispatch_running(sp, "impl", dispatch_id="d1111", dispatched_pid=999) + + mark_dispatch_resumable(sp, "impl", sidecar_path=expected_sidecar) + + state = read_state(sp) + assert state is not None + latest = next(d for d in reversed(state.dispatches) if d.name == "impl") + assert latest.status == DispatchStatus.RESUMABLE + assert latest.sidecar_path == expected_sidecar + + +class TestSidecarPathSetOnMarkRunning: + def test_sidecar_path_set_when_mark_dispatch_running(self, tmp_path: Path) -> None: + sp = _state_path(tmp_path) + write_initial_state(sp, "c1", "myCampaign", "manifest.yaml", _make_dispatches("impl")) + expected_sidecar = str(sp.parent / "d1111_issues.jsonl") + + mark_dispatch_running( + sp, + "impl", + dispatch_id="d1111", + dispatched_pid=999, + sidecar_path=expected_sidecar, + ) + + state = read_state(sp) + assert state is not None + latest = next(d for d in reversed(state.dispatches) if d.name == "impl") + assert latest.sidecar_path == expected_sidecar + + +class TestAppendDispatchRecordIllegalTransition: + def test_success_to_running_raises_valueerror(self, tmp_path: Path) -> None: + sp = _state_path(tmp_path) + write_initial_state(sp, "cid", "camp", "/m.yaml", _make_dispatches("A")) + append_dispatch_record(sp, DispatchRecord(name="A", status=DispatchStatus.SUCCESS)) + with pytest.raises(ValueError, match="Invalid transition"): + append_dispatch_record(sp, DispatchRecord(name="A", status=DispatchStatus.RUNNING)) + state = read_state(sp) + assert state is not None + assert state.dispatches[0].status == DispatchStatus.SUCCESS + + def test_pending_to_interrupted_raises_valueerror(self, tmp_path: Path) -> None: + sp = _state_path(tmp_path) + write_initial_state(sp, "cid", "camp", "/m.yaml", _make_dispatches("A")) + with pytest.raises(ValueError, match="Invalid transition"): + append_dispatch_record(sp, DispatchRecord(name="A", status=DispatchStatus.INTERRUPTED)) + + def test_running_to_success_succeeds(self, tmp_path: Path) -> None: + sp = _state_path(tmp_path) + write_initial_state(sp, "cid", "camp", "/m.yaml", _make_dispatches("A")) + mark_dispatch_running(sp, "A", dispatch_id="d-a", dispatched_pid=99) + append_dispatch_record(sp, DispatchRecord(name="A", status=DispatchStatus.SUCCESS)) + state = read_state(sp) + assert state is not None + latest = next(d for d in reversed(state.dispatches) if d.name == "A") + assert latest.status == DispatchStatus.SUCCESS + + +class TestResumeShowsRefusedInBlock: + def test_refused_dispatch_visible_next_is_b(self, tmp_path: Path) -> None: + sp = _state_path(tmp_path) + write_initial_state(sp, "cid", "camp", "/m.yaml", _make_dispatches("A", "B")) + append_dispatch_record(sp, DispatchRecord(name="A", status=DispatchStatus.REFUSED)) + decision = resume_campaign_from_state(sp, continue_on_failure=True) + assert decision is not None + assert decision.next_dispatch_name == "B" + assert "A" in decision.completed_dispatches_block + assert "refused" in decision.completed_dispatches_block.lower() + + +class TestResumeShowsReleasedInBlock: + def test_released_dispatch_visible_next_is_b(self, tmp_path: Path) -> None: + sp = _state_path(tmp_path) + write_initial_state(sp, "cid", "camp", "/m.yaml", _make_dispatches("A", "B")) + append_dispatch_record(sp, DispatchRecord(name="A", status=DispatchStatus.RELEASED)) + decision = resume_campaign_from_state(sp, continue_on_failure=True) + assert decision is not None + assert decision.next_dispatch_name == "B" + assert "A" in decision.completed_dispatches_block + assert "released" in decision.completed_dispatches_block.lower() + + +class TestResumeIncludesInterruptedInBlock: + def test_interrupted_dispatch_visible_in_completed_block(self, tmp_path: Path) -> None: + sp = _state_path(tmp_path) + write_initial_state(sp, "cid", "camp", "/m.yaml", _make_dispatches("A", "B", "C")) + append_dispatch_record(sp, DispatchRecord(name="A", status=DispatchStatus.SUCCESS)) + mark_dispatch_running(sp, "B", dispatch_id="d-b", dispatched_pid=99) + append_dispatch_record(sp, DispatchRecord(name="B", status=DispatchStatus.INTERRUPTED)) + decision = resume_campaign_from_state(sp, continue_on_failure=True) + assert decision is not None + assert "B" in decision.completed_dispatches_block + assert "interrupted" in decision.completed_dispatches_block.lower() + assert decision.next_dispatch_name == "C" + + +class TestResumeIncludesRunningAliveInBlock: + def test_running_alive_dispatch_visible_in_completed_block( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + sp = _state_path(tmp_path) + record_a = DispatchRecord(name="A", status=DispatchStatus.SUCCESS) + record_b = DispatchRecord( + name="B", + status=DispatchStatus.RUNNING, + dispatched_pid=12345, + dispatched_boot_id="abc", + dispatched_starttime_ticks=999, + ) + record_c = DispatchRecord(name="C", status=DispatchStatus.PENDING) + monkeypatch.setattr( + "autoskillit.fleet.is_dispatch_session_alive", + lambda r: True, + ) + write_initial_state(sp, "cid", "camp", "/m.yaml", [record_a, record_b, record_c]) + decision = resume_campaign_from_state(sp, continue_on_failure=True) + assert decision is not None + assert "B" in decision.completed_dispatches_block + assert "running" in decision.completed_dispatches_block.lower() + assert decision.next_dispatch_name == "C" + + +class TestWriteCapturedValuesCorruptStateNoOp: + def test_invalid_json_returns_none_file_unchanged(self, tmp_path: Path) -> None: + sp = _state_path(tmp_path) + sp.parent.mkdir(parents=True, exist_ok=True) + sp.write_text("not-valid-json{{", encoding="utf-8") + original = sp.read_text(encoding="utf-8") + write_captured_values(sp, {"key": "val"}) + assert sp.read_text(encoding="utf-8") == original + + +class TestReadAllCampaignCapturesMixedSuccessFailure: + def test_mixed_success_failure_returns_empty(self, tmp_path: Path) -> None: + d = tmp_path / "dispatches" + d.mkdir() + state = { + "campaign_id": "c1", + "captured_values": {"k": "v"}, + "dispatches": [ + {"name": "A", "status": DispatchStatus.SUCCESS}, + {"name": "B", "status": DispatchStatus.FAILURE}, + ], + } + (d / "state.json").write_text(json.dumps(state), encoding="utf-8") + result = read_all_campaign_captures(d, "c1") + assert result == {} + + +class TestCrashRecoverDispatchSidecarVanished: + def test_sidecar_oserror_yields_interrupted( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + sp = _state_path(tmp_path) + write_initial_state(sp, "cid", "camp", "/m.yaml", _make_dispatches("impl")) + sidecar = tmp_path / "sidecar.jsonl" + sidecar.write_text('{"issue_url":"x","status":"completed"}\n', encoding="utf-8") + mark_dispatch_running( + sp, "impl", dispatch_id="d-1", dispatched_pid=99, sidecar_path=str(sidecar) + ) + + record = read_state(sp).dispatches[0] + + original_read_text = Path.read_text + + def _oserror_read_text(self_path, *a, **kw): + if str(self_path) == str(sidecar): + raise OSError("TOCTOU race") + return original_read_text(self_path, *a, **kw) + + monkeypatch.setattr(Path, "read_text", _oserror_read_text) + + result = crash_recover_dispatch(sp, record) + assert result == DispatchStatus.INTERRUPTED + assert read_state(sp).dispatches[0].status == DispatchStatus.INTERRUPTED + + +class TestHasFailedDispatchReasonAware: + def test_no_result_block_failure_does_not_halt_campaign(self, tmp_path: Path) -> None: + """has_failed_dispatch returns False when only FAILURE is fleet_l3_no_result_block.""" + sp = _state_path(tmp_path) + write_initial_state(sp, "cid", "camp", "/m.yaml", _make_dispatches("d1")) + append_dispatch_record( + sp, + DispatchRecord( + name="d1", + status=DispatchStatus.FAILURE, + reason="fleet_l3_no_result_block", + ), + ) + assert has_failed_dispatch(sp) is False + + def test_logic_failure_halts_campaign(self, tmp_path: Path) -> None: + """has_failed_dispatch returns True for a completed_clean-based FAILURE.""" + sp = _state_path(tmp_path) + write_initial_state(sp, "cid", "camp", "/m.yaml", _make_dispatches("d1")) + append_dispatch_record( + sp, + DispatchRecord( + name="d1", + status=DispatchStatus.FAILURE, + reason="task-failed", + ), + ) + assert has_failed_dispatch(sp) is True + + def test_mixed_infrastructure_and_logic_failure_halts(self, tmp_path: Path) -> None: + """has_failed_dispatch returns True if ANY non-infrastructure FAILURE exists.""" + sp = _state_path(tmp_path) + write_initial_state(sp, "cid", "camp", "/m.yaml", _make_dispatches("d1", "d2")) + append_dispatch_record( + sp, + DispatchRecord( + name="d1", status=DispatchStatus.FAILURE, reason="fleet_l3_no_result_block" + ), + ) + append_dispatch_record( + sp, + DispatchRecord(name="d2", status=DispatchStatus.FAILURE, reason="task-failed"), + ) + assert has_failed_dispatch(sp) is True + + +class TestKillReasonPropagation: + def test_kill_reason_stored_in_dispatch_record(self, tmp_path: Path) -> None: + sp = _state_path(tmp_path) + write_initial_state(sp, "cid", "camp", "/m.yaml", _make_dispatches("x")) + record = DispatchRecord( + name="x", + status=DispatchStatus.FAILURE, + kill_reason="idle_stall", + ) + append_dispatch_record(sp, record) + state = read_state(sp) + assert state is not None + assert state.dispatches[0].kill_reason == "idle_stall" + + def test_kill_reason_defaults_empty(self) -> None: + record = DispatchRecord(name="x") + assert record.kill_reason == "" + + def test_kill_reason_round_trips_through_json(self, tmp_path: Path) -> None: + sp = _state_path(tmp_path) + write_initial_state(sp, "cid", "camp", "/m.yaml", _make_dispatches("x")) + record = DispatchRecord( + name="x", + status=DispatchStatus.FAILURE, + kill_reason="context_exhausted", + infra_exit_category="context_exhausted", + ) + append_dispatch_record(sp, record) + raw = sp.read_text(encoding="utf-8") + assert '"kill_reason"' in raw + assert '"infra_exit_category"' in raw + state = read_state(sp) + assert state is not None + assert state.dispatches[0].kill_reason == "context_exhausted" + assert state.dispatches[0].infra_exit_category == "context_exhausted" + + def test_crash_recover_abandons_context_exhausted(self, tmp_path: Path) -> None: + sp = _state_path(tmp_path) + sidecar = tmp_path / "sidecar.jsonl" + sidecar.write_text("") + write_initial_state(sp, "cid", "camp", "/m.yaml", _make_dispatches("x")) + mark_dispatch_running(sp, "x", dispatch_id="d1", dispatched_pid=42) + from autoskillit.fleet import is_dispatch_session_alive + + with patch( + f"{is_dispatch_session_alive.__module__}.is_dispatch_session_alive", return_value=False + ): + state = read_state(sp) + record = next(d for d in state.dispatches if d.name == "x") + record.kill_reason = "resume" + record.infra_exit_category = "context_exhausted" + record.sidecar_path = str(sidecar) + result = crash_recover_dispatch(sp, record) + assert result == DispatchStatus.INTERRUPTED + + def test_crash_recover_resumes_idle_stall(self, tmp_path: Path) -> None: + sp = _state_path(tmp_path) + sidecar = tmp_path / "sidecar.jsonl" + sidecar.write_text("") + write_initial_state(sp, "cid", "camp", "/m.yaml", _make_dispatches("x")) + mark_dispatch_running(sp, "x", dispatch_id="d1", dispatched_pid=42) + from autoskillit.fleet import is_dispatch_session_alive + + with patch( + f"{is_dispatch_session_alive.__module__}.is_dispatch_session_alive", return_value=False + ): + state = read_state(sp) + record = next(d for d in state.dispatches if d.name == "x") + record.kill_reason = "idle_stall" + record.infra_exit_category = "" + record.sidecar_path = str(sidecar) + result = crash_recover_dispatch(sp, record) + assert result == DispatchStatus.RESUMABLE + + def test_resume_decision_includes_kill_reason(self, tmp_path: Path) -> None: + sp = _state_path(tmp_path) + sidecar = tmp_path / "sidecar.jsonl" + sidecar.write_text("") + write_initial_state(sp, "cid", "camp", "/m.yaml", _make_dispatches("x")) + mark_dispatch_running(sp, "x", dispatch_id="d1", dispatched_pid=42) + mark_dispatch_resumable(sp, "x", sidecar_path=str(sidecar)) + data = json.loads(sp.read_text()) + for d in data["dispatches"]: + if d["name"] == "x": + d["kill_reason"] = "idle_stall" + d["infra_exit_category"] = "" + d["dispatched_session_id"] = "sess-1" + sp.write_text(json.dumps(data)) + decision = resume_campaign_from_state(sp, continue_on_failure=False) + assert decision is not None + assert decision.is_resumable is True + assert decision.kill_reason == "idle_stall" diff --git a/tests/fleet/test_state_protection.py b/tests/fleet/test_state_protection.py new file mode 100644 index 0000000000..1820685615 --- /dev/null +++ b/tests/fleet/test_state_protection.py @@ -0,0 +1,132 @@ +"""Tests for fleet.state.build_protected_campaign_ids (PROT_1–PROT_9).""" + +from __future__ import annotations + +import json +from pathlib import Path + +import pytest + +from autoskillit.fleet.state import build_protected_campaign_ids + +pytestmark = [pytest.mark.layer("fleet"), pytest.mark.feature("fleet"), pytest.mark.small] + + +def _write_dispatch_file(dispatches_dir: Path, filename: str, data: dict) -> None: + dispatches_dir.mkdir(parents=True, exist_ok=True) + (dispatches_dir / filename).write_text(json.dumps(data), encoding="utf-8") + + +def test_build_protected_ids_missing_dispatches_dir(tmp_path: Path) -> None: + """PROT_1: Returns frozenset() when dispatches dir does not exist.""" + result = build_protected_campaign_ids(tmp_path) + assert result == frozenset() + + +def test_build_protected_ids_pending_dispatch(tmp_path: Path) -> None: + """PROT_2: Campaign with a pending dispatch is protected.""" + dispatches_dir = tmp_path / ".autoskillit" / "temp" / "dispatches" + _write_dispatch_file( + dispatches_dir, + "campaign-abc.json", + { + "campaign_id": "campaign-abc", + "dispatches": [{"name": "d1", "status": "pending"}], + }, + ) + result = build_protected_campaign_ids(tmp_path) + assert "campaign-abc" in result + + +def test_build_protected_ids_running_dispatch(tmp_path: Path) -> None: + """PROT_3: Campaign with a running dispatch is protected.""" + dispatches_dir = tmp_path / ".autoskillit" / "temp" / "dispatches" + _write_dispatch_file( + dispatches_dir, + "campaign-run.json", + { + "campaign_id": "campaign-run", + "dispatches": [{"name": "d1", "status": "running"}], + }, + ) + result = build_protected_campaign_ids(tmp_path) + assert "campaign-run" in result + + +def test_build_protected_ids_interrupted_dispatch(tmp_path: Path) -> None: + """PROT_4: Campaign with an interrupted dispatch is protected.""" + dispatches_dir = tmp_path / ".autoskillit" / "temp" / "dispatches" + _write_dispatch_file( + dispatches_dir, + "campaign-int.json", + { + "campaign_id": "campaign-int", + "dispatches": [{"name": "d1", "status": "interrupted"}], + }, + ) + result = build_protected_campaign_ids(tmp_path) + assert "campaign-int" in result + + +def test_build_protected_ids_all_terminal_not_protected(tmp_path: Path) -> None: + """PROT_5: Campaign where all dispatches are terminal is NOT protected.""" + dispatches_dir = tmp_path / ".autoskillit" / "temp" / "dispatches" + _write_dispatch_file( + dispatches_dir, + "campaign-done.json", + { + "campaign_id": "campaign-done", + "dispatches": [ + {"name": "d1", "status": "success"}, + {"name": "d2", "status": "failure"}, + {"name": "d3", "status": "skipped"}, + {"name": "d4", "status": "released"}, + ], + }, + ) + result = build_protected_campaign_ids(tmp_path) + assert "campaign-done" not in result + + +def test_build_protected_ids_no_dispatches_list_protects(tmp_path: Path) -> None: + """PROT_6: State file with campaign_id but empty dispatches list → protected conservatively.""" + dispatches_dir = tmp_path / ".autoskillit" / "temp" / "dispatches" + _write_dispatch_file( + dispatches_dir, + "campaign-empty.json", + { + "campaign_id": "campaign-empty", + "dispatches": [], + }, + ) + result = build_protected_campaign_ids(tmp_path) + assert "campaign-empty" in result + + +def test_build_protected_ids_corrupt_file_skipped(tmp_path: Path) -> None: + """PROT_7: Corrupt JSON file is skipped without exception.""" + dispatches_dir = tmp_path / ".autoskillit" / "temp" / "dispatches" + dispatches_dir.mkdir(parents=True, exist_ok=True) + (dispatches_dir / "corrupt.json").write_text("{not valid json", encoding="utf-8") + result = build_protected_campaign_ids(tmp_path) + assert result == frozenset() + + +def test_build_protected_ids_missing_campaign_id_skipped(tmp_path: Path) -> None: + """PROT_8: File with no campaign_id key is skipped.""" + dispatches_dir = tmp_path / ".autoskillit" / "temp" / "dispatches" + _write_dispatch_file( + dispatches_dir, + "no-cid.json", + {"dispatches": [{"name": "d1", "status": "running"}]}, + ) + result = build_protected_campaign_ids(tmp_path) + assert result == frozenset() + + +def test_build_protected_ids_exported_from_fleet(tmp_path: Path) -> None: + """PROT_9: build_protected_campaign_ids is importable from autoskillit.fleet.""" + from autoskillit.fleet import build_protected_campaign_ids as fn + + assert callable(fn) + assert fn(tmp_path) == frozenset() diff --git a/tests/fleet/test_state_schema.py b/tests/fleet/test_state_schema.py new file mode 100644 index 0000000000..7bce1a7be7 --- /dev/null +++ b/tests/fleet/test_state_schema.py @@ -0,0 +1,388 @@ +"""Tests for DispatchRecord schema v2 fields and backward compatibility (Group J).""" + +from __future__ import annotations + +import json +from pathlib import Path + +import pytest + +from autoskillit.fleet import ( + DispatchRecord, + DispatchStatus, + DispatchTokenUsage, + mark_dispatch_running, + normalize_dispatch_token_usage, + read_state, + write_initial_state, +) +from autoskillit.fleet.state import _SCHEMA_VERSION + +pytestmark = [pytest.mark.layer("fleet"), pytest.mark.small, pytest.mark.feature("fleet")] + + +def _make_state(tmp_path: Path, dispatch_name: str = "a") -> Path: + sp = tmp_path / "state.json" + write_initial_state( + sp, "cid-schema", "test-campaign", "/m.yaml", [DispatchRecord(name=dispatch_name)] + ) + return sp + + +class TestDispatchRecordSchemaV2: + def test_dispatch_record_has_dispatched_starttime_ticks(self) -> None: + d = DispatchRecord(name="x") + assert hasattr(d, "dispatched_starttime_ticks") + assert d.dispatched_starttime_ticks == 0 + assert isinstance(d.dispatched_starttime_ticks, int) + + def test_dispatch_record_has_dispatched_boot_id(self) -> None: + d = DispatchRecord(name="x") + assert hasattr(d, "dispatched_boot_id") + assert d.dispatched_boot_id == "" + assert isinstance(d.dispatched_boot_id, str) + + def test_mark_dispatch_running_stores_starttime_ticks(self, tmp_path: Path) -> None: + sp = _make_state(tmp_path, "a") + mark_dispatch_running( + sp, + "a", + dispatch_id="did-1", + dispatched_pid=1234, + starttime_ticks=42, + boot_id="abc-boot", + ) + state = read_state(sp) + assert state is not None + d = state.dispatches[0] + assert d.status == DispatchStatus.RUNNING + assert d.dispatched_pid == 1234 + assert d.dispatched_starttime_ticks == 42 + assert d.dispatched_boot_id == "abc-boot" + + raw = json.loads(sp.read_text()) + dispatch_raw = raw["dispatches"][0] + assert dispatch_raw["dispatched_starttime_ticks"] == 42 + assert dispatch_raw["dispatched_boot_id"] == "abc-boot" + + def test_schema_version_is_4(self) -> None: + assert _SCHEMA_VERSION == 4 + + def test_read_state_accepts_legacy_l2_field_names(self, tmp_path: Path) -> None: + """read_state must parse schema v3 state files that use old l2_* field names.""" + legacy_payload = { + "schema_version": 3, + "campaign_id": "cmp-legacy", + "campaign_name": "legacy", + "manifest_path": "/tmp/m.yaml", + "started_at": 1.0, + "dispatches": [ + { + "name": "d1", + "status": "running", + "dispatch_id": "did-1", + "l2_session_id": "sess-old", + "l2_session_log_dir": "/old/logs", + "l2_pid": 1234, + "l2_starttime_ticks": 5678, + "l2_boot_id": "boot-old", + } + ], + } + state_path = tmp_path / "state.json" + state_path.write_text(json.dumps(legacy_payload)) + state = read_state(state_path) + assert state is not None + d = state.dispatches[0] + assert d.dispatched_session_id == "sess-old" + assert d.dispatched_session_log_dir == "/old/logs" + assert d.dispatched_pid == 1234 + assert d.dispatched_starttime_ticks == 5678 + assert d.dispatched_boot_id == "boot-old" + + def test_dispatch_record_serializes_dispatched_field_names(self) -> None: + """DispatchRecord.to_dict() must use dispatched_* field names.""" + d = DispatchRecord(name="x", dispatched_pid=42, dispatched_session_id="sess-new") + raw = d.to_dict() + assert "dispatched_pid" in raw + assert "dispatched_session_id" in raw + assert "l2_pid" not in raw + assert "l2_session_id" not in raw + + def test_read_state_accepts_legacy_l3_field_names(self, tmp_path: Path) -> None: + """read_state must parse schema v3 state files that use old l3_* field names.""" + legacy_payload = { + "schema_version": 3, + "campaign_id": "cmp-legacy-l3", + "campaign_name": "legacy-l3", + "manifest_path": "/tmp/m.yaml", + "started_at": 1.0, + "dispatches": [ + { + "name": "d1", + "status": "running", + "dispatch_id": "did-1", + "l3_session_id": "sess-old-l3", + "l3_session_log_dir": "/old/l3/logs", + "l3_pid": 1234, + "l3_starttime_ticks": 5678, + "l3_boot_id": "boot-old-l3", + } + ], + } + state_path = tmp_path / "state.json" + state_path.write_text(json.dumps(legacy_payload)) + state = read_state(state_path) + assert state is not None + d = state.dispatches[0] + assert d.dispatched_session_id == "sess-old-l3" + assert d.dispatched_session_log_dir == "/old/l3/logs" + assert d.dispatched_pid == 1234 + assert d.dispatched_starttime_ticks == 5678 + assert d.dispatched_boot_id == "boot-old-l3" + assert d.caller_session_id == "" + + def test_read_state_handles_v1_without_ticks(self, tmp_path: Path) -> None: + """read_state on a v1 file missing dispatched_starttime_ticks or dispatched_boot_id.""" + sp = tmp_path / "state_v1.json" + v1_payload = { + "schema_version": 1, + "campaign_id": "cid-v1", + "campaign_name": "old-campaign", + "manifest_path": "/m.yaml", + "started_at": 0.0, + "dispatches": [ + { + "name": "dispatch-a", + "status": "running", + "dispatch_id": "did-old", + "l2_session_id": "", + "l2_session_log_dir": "", + "l2_pid": 9999, + "reason": "", + "token_usage": {}, + "started_at": 0.0, + "ended_at": 0.0, + } + ], + } + sp.write_text(json.dumps(v1_payload)) + state = read_state(sp) + assert state is not None + d = state.dispatches[0] + assert d.dispatched_pid == 9999 + assert d.dispatched_starttime_ticks == 0 + assert d.dispatched_boot_id == "" + + +class TestCampaignIdField: + def test_dispatch_record_has_campaign_id_default_empty(self) -> None: + d = DispatchRecord(name="x") + assert d.campaign_id == "" + + def test_dispatch_record_campaign_id_in_to_dict(self) -> None: + d = DispatchRecord(name="x", campaign_id="cmp-42") + raw = d.to_dict() + assert raw["campaign_id"] == "cmp-42" + + def test_read_state_deserializes_campaign_id(self, tmp_path: Path) -> None: + sp = tmp_path / "state.json" + write_initial_state( + sp, "cmp-99", "test", "/m.yaml", [DispatchRecord(name="a", campaign_id="cmp-99")] + ) + state = read_state(sp) + assert state is not None + assert state.dispatches[0].campaign_id == "cmp-99" + + def test_read_state_defaults_campaign_id_when_absent(self, tmp_path: Path) -> None: + sp = tmp_path / "state.json" + payload = { + "schema_version": 4, + "campaign_id": "cmp-old", + "campaign_name": "test", + "manifest_path": "/m.yaml", + "started_at": 1.0, + "dispatches": [{"name": "a", "status": "pending"}], + } + sp.write_text(json.dumps(payload)) + state = read_state(sp) + assert state is not None + assert state.dispatches[0].campaign_id == "" + + def test_campaign_id_round_trip(self, tmp_path: Path) -> None: + sp = tmp_path / "state.json" + write_initial_state( + sp, "cmp-rt", "test", "/m.yaml", [DispatchRecord(name="a", campaign_id="cmp-rt")] + ) + state = read_state(sp) + assert state is not None + assert state.dispatches[0].campaign_id == "cmp-rt" + + +class TestNormalizeDispatchTokenUsage: + def test_full_mapping(self) -> None: + result = normalize_dispatch_token_usage( + { + "input_tokens": 10, + "output_tokens": 5, + "cache_creation_input_tokens": 2, + "cache_read_input_tokens": 3, + } + ) + assert result == {"input": 10, "output": 5, "cache_creation": 2, "cache_read": 3} + + def test_empty_dict_returns_zeros(self) -> None: + result = normalize_dispatch_token_usage({}) + assert result == {"input": 0, "output": 0, "cache_creation": 0, "cache_read": 0} + + def test_string_values_coerced_to_int(self) -> None: + result = normalize_dispatch_token_usage( + { + "input_tokens": "7", + "output_tokens": "3", + "cache_creation_input_tokens": "1", + "cache_read_input_tokens": "2", + } + ) + assert result == {"input": 7, "output": 3, "cache_creation": 1, "cache_read": 2} + + def test_result_unpacks_into_dispatch_token_usage(self) -> None: + dtu = DispatchTokenUsage( + **normalize_dispatch_token_usage( + { + "input_tokens": 10, + "output_tokens": 5, + "cache_creation_input_tokens": 2, + "cache_read_input_tokens": 3, + } + ) + ) + assert dtu.input == 10 + assert dtu.output == 5 + assert dtu.cache_creation == 2 + assert dtu.cache_read == 3 + + def test_importable_from_fleet_package(self) -> None: + from autoskillit.fleet import normalize_dispatch_token_usage as imported + + assert callable(imported) + + def test_in_fleet_all(self) -> None: + import autoskillit.fleet + + assert "normalize_dispatch_token_usage" in autoskillit.fleet.__all__ + + +class TestDispatchRecordToDict: + def test_dispatch_record_to_dict_all_fields(self) -> None: + """DispatchRecord.to_dict() must return all expected keys.""" + from autoskillit.fleet import DispatchRecord + + record = DispatchRecord(name="test-job") + d = record.to_dict() + assert set(d.keys()) == { + "name", + "status", + "dispatch_id", + "campaign_id", + "caller_session_id", + "dispatched_session_id", + "dispatched_session_log_dir", + "dispatched_pid", + "dispatched_starttime_ticks", + "dispatched_boot_id", + "reason", + "kill_reason", + "infra_exit_category", + "token_usage", + "started_at", + "ended_at", + "sidecar_path", + } + + def test_dispatch_record_to_dict_token_usage_is_shallow_copy(self) -> None: + """DispatchRecord.to_dict() token_usage is a shallow copy, not a deep copy.""" + from autoskillit.fleet import DispatchRecord + + inner = {"nested": "value"} + record = DispatchRecord(name="test-job", token_usage={"key": inner}) + d = record.to_dict() + assert d["token_usage"] is not record.token_usage + assert d["token_usage"]["key"] is inner + + def test_dispatch_record_to_dict_is_json_serializable(self) -> None: + """DispatchRecord.to_dict() output must be JSON-serializable.""" + import json + + from autoskillit.fleet import DispatchRecord + + record = DispatchRecord(name="test-job", token_usage={"x": 1}) + d = record.to_dict() + roundtripped = json.loads(json.dumps(d)) + assert roundtripped["name"] == "test-job" + assert roundtripped["token_usage"] == {"x": 1} + + +class TestDispatchRecordSchemaV3: + def test_normalize_maps_cache_keys_to_canonical_names(self) -> None: + result = normalize_dispatch_token_usage( + { + "input_tokens": 10, + "output_tokens": 5, + "cache_creation_input_tokens": 200, + "cache_read_input_tokens": 300, + } + ) + assert result["cache_creation"] == 200 + assert result["cache_read"] == 300 + assert "cache_creation_input_tokens" not in result + assert "cache_read_input_tokens" not in result + + def test_normalize_defaults_missing_cache_keys_to_zero(self) -> None: + result = normalize_dispatch_token_usage({"input_tokens": 10, "output_tokens": 5}) + assert result["cache_creation"] == 0 + assert result["cache_read"] == 0 + + def test_campaign_id_roundtrip_via_dispatch_record(self, tmp_path: Path) -> None: + sp = tmp_path / "state.json" + write_initial_state( + sp, + "camp-xyz", + "test", + "/m.yaml", + [DispatchRecord(name="a", campaign_id="camp-xyz")], + ) + state = read_state(sp) + assert state is not None + assert state.dispatches[0].campaign_id == "camp-xyz" + assert json.loads(sp.read_text())["dispatches"][0]["campaign_id"] == "camp-xyz" + + def test_backward_compat_missing_campaign_id_defaults_to_empty_string( + self, tmp_path: Path + ) -> None: + sp = tmp_path / "state_v1.json" + v1_payload = { + "schema_version": 1, + "campaign_id": "cid-v1", + "campaign_name": "old-campaign", + "manifest_path": "/m.yaml", + "started_at": 0.0, + "dispatches": [ + { + "name": "dispatch-a", + "status": "running", + "dispatch_id": "did-old", + "l2_session_id": "", + "l2_session_log_dir": "", + "l2_pid": 9999, + "reason": "", + "token_usage": {}, + "started_at": 0.0, + "ended_at": 0.0, + } + ], + } + sp.write_text(json.dumps(v1_payload)) + state = read_state(sp) + assert state is not None + assert state.dispatches[0].campaign_id == "" diff --git a/tests/fleet/test_summary.py b/tests/fleet/test_summary.py new file mode 100644 index 0000000000..e7bd07c093 --- /dev/null +++ b/tests/fleet/test_summary.py @@ -0,0 +1,294 @@ +"""Tests for fleet campaign summary schema v1. + +Group S: CampaignSummary dataclasses, parse_campaign_summary, validate_campaign_summary, +and serialize_campaign_summary. +""" + +from __future__ import annotations + +import json + +import pytest + +pytestmark = [pytest.mark.layer("fleet"), pytest.mark.small, pytest.mark.feature("fleet")] + +_VALID_CAMPAIGN_ID = "test-cid-123" + +_VALID_SUMMARY_DICT: dict = { + "schema_version": 1, + "campaign_id": _VALID_CAMPAIGN_ID, + "campaign_name": "Test Campaign", + "dispatch_count": 2, + "completed_count": 1, + "failure_count": 1, + "skipped_count": 0, + "per_dispatch": [ + { + "name": "dispatch-1", + "status": "success", + "elapsed_seconds": 10.5, + "token_usage": { + "input": 1000, + "output": 200, + "cache_read": 50, + "cache_creation": 10, + }, + "dispatched_session_id": "sess-abc", + "dispatch_id": "disp-001", + }, + { + "name": "dispatch-2", + "status": "failure", + "elapsed_seconds": 5.0, + "token_usage": { + "input": 500, + "output": 100, + "cache_read": 0, + "cache_creation": 0, + }, + "dispatched_session_id": "sess-def", + "dispatch_id": "disp-002", + }, + ], + "error_records": [ + { + "dispatch_name": "dispatch-2", + "code": "fleet_l3_timeout", + "message": "Timed out after 300s", + "dispatched_session_id": "sess-def", + } + ], +} + + +def _make_sentinel_text(data: dict, campaign_id: str = _VALID_CAMPAIGN_ID) -> str: + body = json.dumps(data, indent=2) + return ( + f"---campaign-summary::{campaign_id}---\n{body}\n---end-campaign-summary::{campaign_id}---" + ) + + +_VALID_SENTINEL_TEXT = _make_sentinel_text(_VALID_SUMMARY_DICT) + + +class TestCampaignSummarySchema: + def test_campaign_summary_schema_valid_example(self): + from autoskillit.fleet import CampaignSummary, parse_campaign_summary + + result = parse_campaign_summary(_VALID_SENTINEL_TEXT, _VALID_CAMPAIGN_ID) + assert isinstance(result, CampaignSummary) + assert result.schema_version == 1 + assert result.campaign_id == _VALID_CAMPAIGN_ID + assert result.campaign_name == "Test Campaign" + assert result.dispatch_count == 2 + assert result.completed_count == 1 + assert result.failure_count == 1 + assert result.skipped_count == 0 + assert len(result.per_dispatch) == 2 + assert len(result.error_records) == 1 + + def test_campaign_summary_rejects_aggregate_fields(self): + from autoskillit.fleet import validate_campaign_summary + + data = {**_VALID_SUMMARY_DICT, "total_input_tokens": 999} + errors = validate_campaign_summary(data) + assert any("total_input_tokens" in e for e in errors) + + data2 = {**_VALID_SUMMARY_DICT, "total_output_tokens": 1} + errors2 = validate_campaign_summary(data2) + assert any("total_output_tokens" in e for e in errors2) + + data3 = {**_VALID_SUMMARY_DICT, "total_duration": 60} + errors3 = validate_campaign_summary(data3) + assert any("total_duration" in e for e in errors3) + + def test_campaign_summary_status_enum_strict(self): + from autoskillit.fleet import CampaignSummaryStatus, validate_campaign_summary + + assert set(m.value for m in CampaignSummaryStatus) == {"success", "failure", "skipped"} + assert len(CampaignSummaryStatus) == 3 + + bad_entry = {**_VALID_SUMMARY_DICT["per_dispatch"][0], "status": "running"} + data = {**_VALID_SUMMARY_DICT, "per_dispatch": [bad_entry]} + errors = validate_campaign_summary(data) + assert any("status" in e for e in errors) + + def test_per_dispatch_token_usage_exactly_4_keys(self): + import dataclasses + + from autoskillit.fleet import DispatchTokenUsage + + fields = {f.name for f in dataclasses.fields(DispatchTokenUsage)} + assert fields == {"input", "output", "cache_read", "cache_creation"} + + def test_sentinel_anchored_to_campaign_id(self): + from autoskillit.fleet import ParseFailure, ParseFailureKind, parse_campaign_summary + + result = parse_campaign_summary(_VALID_SENTINEL_TEXT, "wrong-id") + assert isinstance(result, ParseFailure) + assert result.kind == ParseFailureKind.CAMPAIGN_ID_MISMATCH + + def test_sentinel_parse_missing_end_marker(self): + from autoskillit.fleet import ParseFailure, ParseFailureKind, parse_campaign_summary + + text_no_end = f"---campaign-summary::{_VALID_CAMPAIGN_ID}---\n{{}}\n" + result = parse_campaign_summary(text_no_end, _VALID_CAMPAIGN_ID) + assert isinstance(result, ParseFailure) + assert result.kind == ParseFailureKind.SENTINEL_MISSING + + def test_campaign_summary_schema_version_is_1(self): + from autoskillit.fleet import ( + CampaignSummary, + parse_campaign_summary, + validate_campaign_summary, + ) + + result = parse_campaign_summary(_VALID_SENTINEL_TEXT, _VALID_CAMPAIGN_ID) + assert isinstance(result, CampaignSummary) + assert result.schema_version == 1 + + data_v2 = {**_VALID_SUMMARY_DICT, "schema_version": 2} + errors = validate_campaign_summary(data_v2) + assert any("schema_version" in e for e in errors) + + def test_campaign_summary_count_fields_required(self): + from autoskillit.fleet import validate_campaign_summary + + for required_field in ( + "dispatch_count", + "completed_count", + "failure_count", + "skipped_count", + ): + data = {k: v for k, v in _VALID_SUMMARY_DICT.items() if k != required_field} + errors = validate_campaign_summary(data) + assert errors, f"Expected error for missing {required_field}" + + def test_campaign_summary_roundtrip(self): + from autoskillit.fleet import ( + CampaignSummary, + parse_campaign_summary, + serialize_campaign_summary, + ) + + original = parse_campaign_summary(_VALID_SENTINEL_TEXT, _VALID_CAMPAIGN_ID) + assert isinstance(original, CampaignSummary) + text = serialize_campaign_summary(original) + restored = parse_campaign_summary(text, _VALID_CAMPAIGN_ID) + assert isinstance(restored, CampaignSummary) + assert restored.campaign_id == original.campaign_id + assert restored.dispatch_count == original.dispatch_count + assert restored.per_dispatch[0].name == original.per_dispatch[0].name + assert ( + restored.per_dispatch[0].token_usage.input + == original.per_dispatch[0].token_usage.input + ) + assert restored.error_records[0].code == original.error_records[0].code + + def test_campaign_summary_error_records_have_code_field(self): + from autoskillit.fleet import CampaignSummary, SummaryErrorRecord, parse_campaign_summary + + result = parse_campaign_summary(_VALID_SENTINEL_TEXT, _VALID_CAMPAIGN_ID) + assert isinstance(result, CampaignSummary) + assert len(result.error_records) == 1 + rec = result.error_records[0] + assert isinstance(rec, SummaryErrorRecord) + assert rec.dispatch_name == "dispatch-2" + assert rec.code == "fleet_l3_timeout" + assert rec.message == "Timed out after 300s" + assert rec.dispatched_session_id == "sess-def" + + def test_campaign_summary_no_cross_dispatch_aggregates(self): + from autoskillit.fleet import validate_campaign_summary + + for forbidden_key in ("total_anything", "total_cost", "total_tokens"): + data = {**_VALID_SUMMARY_DICT, forbidden_key: 0} + errors = validate_campaign_summary(data) + assert any(forbidden_key in e for e in errors), ( + f"Expected rejection of key {forbidden_key!r}" + ) + + def test_parse_failure_sentinel_missing(self): + from autoskillit.fleet import ParseFailure, ParseFailureKind, parse_campaign_summary + + result = parse_campaign_summary("no sentinel block here", _VALID_CAMPAIGN_ID) + assert isinstance(result, ParseFailure) + assert result.kind == ParseFailureKind.SENTINEL_MISSING + assert isinstance(result.message, str) and result.message + + def test_parse_failure_campaign_id_mismatch(self): + from autoskillit.fleet import ParseFailure, ParseFailureKind, parse_campaign_summary + + result = parse_campaign_summary(_VALID_SENTINEL_TEXT, "wrong-campaign-id") + assert isinstance(result, ParseFailure) + assert result.kind == ParseFailureKind.CAMPAIGN_ID_MISMATCH + + def test_parse_failure_json_decode_error(self): + from autoskillit.fleet import ParseFailure, ParseFailureKind, parse_campaign_summary + + text = ( + f"---campaign-summary::{_VALID_CAMPAIGN_ID}---\n" + "not valid json {{{\n" + f"---end-campaign-summary::{_VALID_CAMPAIGN_ID}---" + ) + result = parse_campaign_summary(text, _VALID_CAMPAIGN_ID) + assert isinstance(result, ParseFailure) + assert result.kind == ParseFailureKind.JSON_DECODE_ERROR + + def test_parse_failure_schema_validation_error(self): + from autoskillit.fleet import ParseFailure, ParseFailureKind, parse_campaign_summary + + bad_schema = {**_VALID_SUMMARY_DICT, "schema_version": 99} + text = _make_sentinel_text(bad_schema) + result = parse_campaign_summary(text, _VALID_CAMPAIGN_ID) + assert isinstance(result, ParseFailure) + assert result.kind == ParseFailureKind.SCHEMA_VALIDATION_ERROR + + def test_parse_failure_field_error(self): + from autoskillit.fleet import ParseFailure, ParseFailureKind, parse_campaign_summary + + bad_entry = { + "name": "d1", + "status": "success", + # elapsed_seconds omitted → KeyError + "token_usage": {"input": 1, "output": 1, "cache_read": 0, "cache_creation": 0}, + "dispatched_session_id": "s", + } + data = {**_VALID_SUMMARY_DICT, "per_dispatch": [bad_entry]} + text = _make_sentinel_text(data) + result = parse_campaign_summary(text, _VALID_CAMPAIGN_ID) + assert isinstance(result, ParseFailure) + assert result.kind == ParseFailureKind.FIELD_ERROR + assert "elapsed_seconds" in result.message + + def test_per_dispatch_entry_dispatch_id_roundtrip(self) -> None: + from autoskillit.fleet import ( + CampaignSummary, + parse_campaign_summary, + serialize_campaign_summary, + ) + + result = parse_campaign_summary(_VALID_SENTINEL_TEXT, _VALID_CAMPAIGN_ID) + assert isinstance(result, CampaignSummary) + assert result.per_dispatch[0].dispatch_id == "disp-001" + assert result.per_dispatch[1].dispatch_id == "disp-002" + + serialized = serialize_campaign_summary(result) + restored = parse_campaign_summary(serialized, _VALID_CAMPAIGN_ID) + assert isinstance(restored, CampaignSummary) + assert restored.per_dispatch[0].dispatch_id == "disp-001" + assert restored.per_dispatch[1].dispatch_id == "disp-002" + + def test_per_dispatch_entry_dispatch_id_defaults_to_empty_string(self) -> None: + import copy + + from autoskillit.fleet import CampaignSummary, parse_campaign_summary + + d = copy.deepcopy(_VALID_SUMMARY_DICT) + for entry in d["per_dispatch"]: + entry.pop("dispatch_id", None) + text = _make_sentinel_text(d) + result = parse_campaign_summary(text, _VALID_CAMPAIGN_ID) + assert isinstance(result, CampaignSummary) + assert result.per_dispatch[0].dispatch_id == "" + assert result.per_dispatch[1].dispatch_id == "" diff --git a/tests/hooks/CLAUDE.md b/tests/hooks/CLAUDE.md new file mode 100644 index 0000000000..ef2915312e --- /dev/null +++ b/tests/hooks/CLAUDE.md @@ -0,0 +1,31 @@ +# hooks/ + +Hook script behavior, registration, and bridge tests. + +## Files + +| File | Purpose | +|------|---------| +| `__init__.py` | empty | +| `test_fmt_status.py` | Tests for autoskillit.hooks.formatters._fmt_status | +| `test_hook_dispatch.py` | Tests for the stable hook dispatcher (_dispatch.py) — resolution, retired mapping, graceful degrade | +| `test_hook_config_bridge.py` | Regression tests for the quota_guard.py → .hook_config.json bridge | +| `test_hook_executability.py` | Tests for hook command executability — validates invocation path via subprocess | +| `test_hook_registration_coverage.py` | Structural test: every hook script is registered in HOOK_REGISTRY | +| `test_hook_registry.py` | Tests for hook_registry.py — L0 hook identity model | +| `test_hook_settings.py` | Tests for the shared stdlib-only quota hook settings resolver | +| `test_hook_sync.py` | Sync tests: verify parallel stdlib-only hook scripts stay aligned with server code | +| `test_lint_after_edit_hook.py` | Tests for lint_after_edit_hook.py PostToolUse hook | +| `test_quota_check.py` | Tests for the quota_check PreToolUse hook | +| `test_quota_post_check.py` | Tests for the quota_post_check PostToolUse hook | +| `test_recipe_write_advisor.py` | Tests for autoskillit.hooks.guards.recipe_write_advisor | +| `test_review_gate_post_hook.py` | Tests for the review_gate_post_hook PostToolUse hook | +| `test_review_loop_gate.py` | Tests for the review_loop_gate PreToolUse hook | +| `test_session_start_reminder.py` | Tests for the SessionStart hook — session_start_hook.py | +| `test_skill_load_post_hook.py` | Tests for skill_load_post_hook.py PostToolUse hook | +| `test_token_summary_appender.py` | Tests for the token_summary_appender PostToolUse hook | +| `test_write_guard.py` | Tests for write_guard.py PreToolUse hook | + +## Architecture Notes + +`test_token_summary_appender.py` covers the hook registration and end-to-end behavior. The related split test files (`test_token_summary_core.py`, `test_token_summary_filters.py`) and shared helpers (`_token_summary_helpers.py`) live in `tests/infra/` — see `tests/infra/CLAUDE.md`. diff --git a/tests/hooks/test_fmt_status.py b/tests/hooks/test_fmt_status.py index d1b6e459f5..c10303bb56 100644 --- a/tests/hooks/test_fmt_status.py +++ b/tests/hooks/test_fmt_status.py @@ -1,4 +1,4 @@ -"""Tests for autoskillit.hooks._fmt_status.""" +"""Tests for autoskillit.hooks.formatters._fmt_status.""" from __future__ import annotations @@ -8,7 +8,7 @@ class TestFmtCloneRepo: def test_fmt_clone_repo_renders_source_type_key(self) -> None: """New clone_source_type key is rendered in the flat-KV output.""" - from autoskillit.hooks._fmt_status import _fmt_clone_repo + from autoskillit.hooks.formatters._fmt_status import _fmt_clone_repo result = _fmt_clone_repo( { @@ -24,7 +24,7 @@ def test_fmt_clone_repo_renders_source_type_key(self) -> None: def test_fmt_clone_repo_renders_local_source_type(self) -> None: """clone_source_type=local is rendered correctly.""" - from autoskillit.hooks._fmt_status import _fmt_clone_repo + from autoskillit.hooks.formatters._fmt_status import _fmt_clone_repo result = _fmt_clone_repo( { diff --git a/tests/hooks/test_hook_config_bridge.py b/tests/hooks/test_hook_config_bridge.py new file mode 100644 index 0000000000..19ca981b12 --- /dev/null +++ b/tests/hooks/test_hook_config_bridge.py @@ -0,0 +1,327 @@ +"""Regression tests for the quota_guard.py → .hook_config.json bridge. + +Tests the end-to-end path from _quota_guard_hook_payload serialization through +.hook_config.json to resolve_quota_settings() consumption by quota_guard.py. +No pytestmark — hooks/ is out of scope for layer markers. +""" + +from __future__ import annotations + +import io +import json +import time +from contextlib import redirect_stdout +from datetime import UTC, datetime, timedelta +from pathlib import Path +from unittest.mock import patch + +import pytest + +_ENV_VARS = ( + "AUTOSKILLIT_QUOTA_GUARD__CACHE_PATH", + "AUTOSKILLIT_QUOTA_GUARD__CACHE_MAX_AGE", + "AUTOSKILLIT_QUOTA_GUARD__BUFFER_SECONDS", + "AUTOSKILLIT_QUOTA_GUARD__DISABLED", +) + + +def _clear_env(monkeypatch: pytest.MonkeyPatch) -> None: + for name in _ENV_VARS: + monkeypatch.delenv(name, raising=False) + + +def _write_hook_config(tmp_path: Path, quota_guard: dict) -> None: + hook_cfg = tmp_path / ".autoskillit" / "temp" / ".hook_config.json" + hook_cfg.parent.mkdir(parents=True, exist_ok=True) + hook_cfg.write_text(json.dumps({"quota_guard": quota_guard})) + + +def _write_blocking_cache(cache_path: Path, *, fetched_at: str | None = None) -> None: + """Write a quota cache with should_block=True. No resets_at → sleep = buffer_seconds.""" + payload = { + "fetched_at": fetched_at or datetime.now(UTC).isoformat(), + "binding": { + "utilization": 95.0, + "should_block": True, + "effective_threshold": 85.0, + "window_name": "five_hour", + }, + } + cache_path.parent.mkdir(parents=True, exist_ok=True) + cache_path.write_text(json.dumps(payload)) + + +def _run_hook(event: dict | None = None) -> tuple[str, int]: + """Run quota_guard.main() without cache_path_override — exercises the bridge path.""" + from autoskillit.hooks.guards.quota_guard import main + + buf = io.StringIO() + exit_code = 0 + with patch("sys.stdin", io.StringIO(json.dumps(event or {}))): + with redirect_stdout(buf): + try: + main() + except SystemExit as e: + exit_code = e.code if isinstance(e.code, int) else 0 + return buf.getvalue(), exit_code + + +# T-BRIDGE-1 +def test_hook_reads_cache_path_from_hook_config_and_denies(tmp_path, monkeypatch): + """Hook reads cache from payload-written path and denies when should_block=True.""" + cache = tmp_path / "quota_cache.json" + _write_blocking_cache(cache) + _write_hook_config( + tmp_path, + { + "cache_path": str(cache), + "cache_max_age": 300, + "buffer_seconds": 60, + "disabled": False, + }, + ) + monkeypatch.chdir(tmp_path) + _clear_env(monkeypatch) + + out, _ = _run_hook(event={"tool_name": "run_skill"}) + + assert out != "", "hook failed-open unexpectedly (empty output)" + data = json.loads(out) + assert data["hookSpecificOutput"]["permissionDecision"] == "deny" + + +# T-BRIDGE-2 +def test_deny_message_contains_sleep_from_payload_buffer_seconds(tmp_path, monkeypatch): + """Deny message contains time.sleep(77) and Sleeping 77s from hook config buffer_seconds=77.""" + cache = tmp_path / "quota_cache.json" + _write_blocking_cache(cache) # no resets_at → n = buffer_seconds exactly + _write_hook_config( + tmp_path, + { + "cache_path": str(cache), + "cache_max_age": 300, + "buffer_seconds": 77, + "disabled": False, + }, + ) + monkeypatch.chdir(tmp_path) + _clear_env(monkeypatch) + + out, _ = _run_hook(event={"tool_name": "run_skill"}) + + assert out != "", "hook failed-open unexpectedly (empty output)" + data = json.loads(out) + reason = data["hookSpecificOutput"]["permissionDecisionReason"] + assert "time.sleep(77)" in reason + assert "Sleeping 77s" in reason + + +# T-BRIDGE-3 +def test_stale_cache_fails_open_with_hook_config_max_age(tmp_path, monkeypatch): + """Cache older than max_age from hook config treated as stale → fail-open approve.""" + cache = tmp_path / "quota_cache.json" + old_fetched_at = (datetime.now(UTC) - timedelta(seconds=65)).isoformat() + _write_blocking_cache(cache, fetched_at=old_fetched_at) + _write_hook_config( + tmp_path, + { + "cache_path": str(cache), + "cache_max_age": 30, + "buffer_seconds": 60, + "disabled": False, + }, + ) + monkeypatch.chdir(tmp_path) + _clear_env(monkeypatch) + + out, exit_code = _run_hook(event={"tool_name": "run_skill"}) + + assert out == "" # no deny JSON → approve (fail-open on stale cache) + assert exit_code == 0 + + +# T-BRIDGE-4 +def test_disabled_true_unconditionally_approves(tmp_path, monkeypatch): + """disabled=True in hook config → unconditional approve even with a blocking cache.""" + cache = tmp_path / "quota_cache.json" + _write_blocking_cache(cache) + _write_hook_config( + tmp_path, + { + "cache_path": str(cache), + "cache_max_age": 300, + "buffer_seconds": 60, + "disabled": True, + }, + ) + monkeypatch.chdir(tmp_path) + _clear_env(monkeypatch) + + out, exit_code = _run_hook(event={"tool_name": "run_skill"}) + + assert out == "" # guard bypassed entirely — no cache logic runs + assert exit_code == 0 + + +# T-BRIDGE-5 +def test_disabled_false_blocks_normally(tmp_path, monkeypatch): + """enabled=True → disabled=False in hook config → hook blocks on should_block=True.""" + cache = tmp_path / "quota_cache.json" + _write_blocking_cache(cache) + _write_hook_config( + tmp_path, + { + "cache_path": str(cache), + "cache_max_age": 300, + "buffer_seconds": 60, + "disabled": False, + }, + ) + monkeypatch.chdir(tmp_path) + _clear_env(monkeypatch) + + out, _ = _run_hook(event={"tool_name": "run_skill"}) + + assert out != "", "hook failed-open unexpectedly (empty output)" + data = json.loads(out) + assert data["hookSpecificOutput"]["permissionDecision"] == "deny" + + +# T-BRIDGE-6 +def test_hook_config_quota_guard_keys_match_payload_keys(tmp_path): + """set(data['quota_guard'].keys()) == QUOTA_GUARD_HOOK_PAYLOAD_KEYS.""" + from autoskillit.config.settings import QuotaGuardConfig + from autoskillit.hooks._hook_settings import QUOTA_GUARD_HOOK_PAYLOAD_KEYS + from autoskillit.server.tools.tools_kitchen import _quota_guard_hook_payload + + cfg = QuotaGuardConfig() + payload = {"quota_guard": _quota_guard_hook_payload(cfg)} + hook_cfg = tmp_path / ".autoskillit" / "temp" / ".hook_config.json" + hook_cfg.parent.mkdir(parents=True, exist_ok=True) + hook_cfg.write_text(json.dumps(payload)) + + data = json.loads(hook_cfg.read_text()) + assert set(data["quota_guard"].keys()) == QUOTA_GUARD_HOOK_PAYLOAD_KEYS + + +# T-BRIDGE-7 +def test_write_hook_config_round_trip_via_resolve_quota_settings(tmp_path, monkeypatch): + """_write_hook_config round-trip: payload written by _quota_guard_hook_payload + is correctly read back by resolve_quota_settings().""" + from autoskillit.config.settings import QuotaGuardConfig + from autoskillit.hooks._hook_settings import resolve_quota_settings + from autoskillit.server.tools.tools_kitchen import _quota_guard_hook_payload + + monkeypatch.chdir(tmp_path) + _clear_env(monkeypatch) + + cfg = QuotaGuardConfig( + cache_max_age=999, + buffer_seconds=42, + cache_path="/round/trip.json", + enabled=True, + ) + hook_cfg = tmp_path / ".autoskillit" / "temp" / ".hook_config.json" + hook_cfg.parent.mkdir(parents=True, exist_ok=True) + hook_cfg.write_text(json.dumps({"quota_guard": _quota_guard_hook_payload(cfg)})) + + settings = resolve_quota_settings() + + assert settings.cache_max_age == 999 + assert settings.buffer_seconds == 42 + assert settings.cache_path == "/round/trip.json" + assert settings.disabled is False # enabled=True → disabled=False + + +# T-BUDGET-1 +def test_quota_guard_budget_exceeded_exit(tmp_path, monkeypatch): + """When sleep exceeds remaining budget, deny message instructs clean exit.""" + cache = tmp_path / "quota_cache.json" + monkeypatch.setenv("AUTOSKILLIT_SESSION_DEADLINE", str(time.time() + 60)) + _write_blocking_cache( + cache, + fetched_at=datetime.now(UTC).isoformat(), + ) + _write_hook_config( + tmp_path, + { + "cache_path": str(cache), + "cache_max_age": 300, + "buffer_seconds": 3600, + "disabled": False, + }, + ) + monkeypatch.chdir(tmp_path) + _clear_env(monkeypatch) + + out, _ = _run_hook(event={"tool_name": "run_skill"}) + + assert out != "", "hook failed-open unexpectedly (empty output)" + data = json.loads(out) + assert data["hookSpecificOutput"]["permissionDecision"] == "deny" + reason = data["hookSpecificOutput"]["permissionDecisionReason"] + assert "QUOTA BUDGET EXCEEDED" in reason + assert "fleet_quota_exhausted" in reason + assert "run_cmd" not in reason # no sleep command + + +# T-BUDGET-2 +def test_quota_guard_normal_deny_when_budget_sufficient(tmp_path, monkeypatch): + """Normal deny when deadline is far in the future (sleep fits in budget).""" + cache = tmp_path / "quota_cache.json" + monkeypatch.setenv("AUTOSKILLIT_SESSION_DEADLINE", str(time.time() + 7200)) + _write_blocking_cache( + cache, + fetched_at=datetime.now(UTC).isoformat(), + ) + _write_hook_config( + tmp_path, + { + "cache_path": str(cache), + "cache_max_age": 300, + "buffer_seconds": 300, + "disabled": False, + }, + ) + monkeypatch.chdir(tmp_path) + _clear_env(monkeypatch) + + out, _ = _run_hook(event={"tool_name": "run_skill"}) + + assert out != "", "hook failed-open unexpectedly (empty output)" + data = json.loads(out) + assert data["hookSpecificOutput"]["permissionDecision"] == "deny" + reason = data["hookSpecificOutput"]["permissionDecisionReason"] + assert "QUOTA WAIT REQUIRED" in reason + assert "run_cmd" in reason or "time.sleep" in reason # contains sleep command + + +# T-BUDGET-3 +def test_quota_guard_normal_deny_when_no_deadline(tmp_path, monkeypatch): + """Normal deny when AUTOSKILLIT_SESSION_DEADLINE is not set (backward compatible).""" + cache = tmp_path / "quota_cache.json" + _write_blocking_cache( + cache, + fetched_at=datetime.now(UTC).isoformat(), + ) + _write_hook_config( + tmp_path, + { + "cache_path": str(cache), + "cache_max_age": 300, + "buffer_seconds": 300, + "disabled": False, + }, + ) + monkeypatch.chdir(tmp_path) + _clear_env(monkeypatch) + monkeypatch.delenv("AUTOSKILLIT_SESSION_DEADLINE", raising=False) + + out, _ = _run_hook(event={"tool_name": "run_skill"}) + + assert out != "", "hook failed-open unexpectedly (empty output)" + data = json.loads(out) + assert data["hookSpecificOutput"]["permissionDecision"] == "deny" + reason = data["hookSpecificOutput"]["permissionDecisionReason"] + assert "QUOTA WAIT REQUIRED" in reason + assert "run_cmd" in reason or "time.sleep" in reason diff --git a/tests/hooks/test_hook_dispatch.py b/tests/hooks/test_hook_dispatch.py new file mode 100644 index 0000000000..0c847d7e08 --- /dev/null +++ b/tests/hooks/test_hook_dispatch.py @@ -0,0 +1,237 @@ +"""Tests for the stable hook dispatcher (_dispatch.py).""" + +from __future__ import annotations + +import json +import subprocess +import sys +import textwrap +import time +from pathlib import Path + +import pytest + +from autoskillit.hook_registry import HOOKS_DIR, generate_hooks_json + +DISPATCH_SCRIPT = HOOKS_DIR / "_dispatch.py" + + +@pytest.fixture() +def hook_env(tmp_path: Path) -> Path: + """Create a temporary hooks directory with _dispatch.py and a test target.""" + hooks_dir = tmp_path / "hooks" + guards_dir = hooks_dir / "guards" + guards_dir.mkdir(parents=True) + + dispatch_src = DISPATCH_SCRIPT.read_text() + (hooks_dir / "_dispatch.py").write_text(dispatch_src) + + target_script = guards_dir / "quota_guard.py" + target_script.write_text( + textwrap.dedent("""\ + import sys + print("HOOK_OK") + sys.exit(0) + """) + ) + + return hooks_dir + + +@pytest.fixture() +def stdin_echo_env(tmp_path: Path) -> Path: + """Create a hooks directory with a target that echoes stdin to stdout.""" + hooks_dir = tmp_path / "hooks" + guards_dir = hooks_dir / "guards" + guards_dir.mkdir(parents=True) + + dispatch_src = DISPATCH_SCRIPT.read_text() + (hooks_dir / "_dispatch.py").write_text(dispatch_src) + + target_script = guards_dir / "echo_hook.py" + target_script.write_text( + textwrap.dedent("""\ + import sys + data = sys.stdin.read() + sys.stdout.write(data) + sys.exit(0) + """) + ) + + return hooks_dir + + +class TestDispatchResolution: + def test_dispatch_resolves_current_script(self, hook_env: Path) -> None: + result = subprocess.run( + [sys.executable, str(hook_env / "_dispatch.py"), "guards/quota_guard"], + input=b'{"tool_name": "Read", "tool_input": {}}', + capture_output=True, + timeout=10, + ) + assert result.returncode == 0 + assert b"HOOK_OK" in result.stdout + + def test_dispatch_resolves_retired_name(self, tmp_path: Path) -> None: + hooks_dir = tmp_path / "hooks" + guards_dir = hooks_dir / "guards" + guards_dir.mkdir(parents=True) + + dispatch_src = DISPATCH_SCRIPT.read_text() + (hooks_dir / "_dispatch.py").write_text(dispatch_src) + + target = guards_dir / "skill_orchestration_guard.py" + target.write_text( + textwrap.dedent("""\ + import sys + print("RENAMED_OK") + sys.exit(0) + """) + ) + + result = subprocess.run( + [ + sys.executable, + str(hooks_dir / "_dispatch.py"), + "guards/leaf_orchestration_guard", + ], + input=b'{"tool_name": "Read", "tool_input": {}}', + capture_output=True, + timeout=10, + ) + assert result.returncode == 0 + assert b"RENAMED_OK" in result.stdout + + def test_dispatch_unknown_hook_exits_zero(self, hook_env: Path) -> None: + result = subprocess.run( + [sys.executable, str(hook_env / "_dispatch.py"), "guards/nonexistent_hook"], + input=b'{"tool_name": "Read", "tool_input": {}}', + capture_output=True, + timeout=10, + ) + assert result.returncode == 0 + assert b"unknown hook" in result.stderr + + def test_dispatch_buffers_stdin(self, stdin_echo_env: Path) -> None: + payload = json.dumps({"tool_name": "Bash", "tool_input": {"command": "ls"}}) + result = subprocess.run( + [sys.executable, str(stdin_echo_env / "_dispatch.py"), "guards/echo_hook"], + input=payload.encode(), + capture_output=True, + timeout=10, + ) + assert result.returncode == 0 + assert payload.encode() in result.stdout + + +class TestRetiredMappingIntegrity: + def test_retired_mapping_targets_exist(self) -> None: + from autoskillit.hook_registry import canonical_script_basenames + from autoskillit.hooks._dispatch import _RETIRED_MAPPING + + canonical = canonical_script_basenames() + canonical_logical = {s.removesuffix(".py") for s in canonical} + + for old_name, new_name in _RETIRED_MAPPING.items(): + target_path = HOOKS_DIR / (new_name + ".py") + assert target_path.is_file(), ( + f"_RETIRED_MAPPING target missing: {old_name} -> {new_name} " + f"(expected {target_path})" + ) + assert old_name != new_name, f"Self-referencing entry: {old_name}" + assert old_name not in canonical_logical, ( + f"Retired name '{old_name}' is still a live canonical name" + ) + + +class TestCrossVersionSimulation: + def test_stale_command_still_resolves_after_rename(self, tmp_path: Path) -> None: + """Simulates the temporal bug: hook installed at version N, script renamed at N+1.""" + hooks_dir = tmp_path / "hooks" + guards_dir = hooks_dir / "guards" + guards_dir.mkdir(parents=True) + + dispatch_src = DISPATCH_SCRIPT.read_text() + (hooks_dir / "_dispatch.py").write_text(dispatch_src) + + target = guards_dir / "skill_orchestration_guard.py" + target.write_text( + textwrap.dedent("""\ + import sys + print("RESOLVED_VIA_RETIRED_MAPPING") + sys.exit(0) + """) + ) + + stale_command = f"python3 {hooks_dir / '_dispatch.py'} guards/leaf_orchestration_guard" + parts = stale_command.split() + + result = subprocess.run( + parts, + input=b'{"tool_name": "mcp__autoskillit__run_skill", "tool_input": {}}', + capture_output=True, + timeout=10, + ) + assert result.returncode == 0 + assert b"RESOLVED_VIA_RETIRED_MAPPING" in result.stdout + + +class TestGenerateHooksJsonFormat: + def test_generate_hooks_json_uses_dispatcher(self) -> None: + hooks_json = generate_hooks_json() + for event_type, entries in hooks_json["hooks"].items(): + for entry in entries: + for hook in entry.get("hooks", []): + cmd = hook["command"] + assert "_dispatch.py" in cmd, f"Command does not use dispatcher: {cmd}" + parts = cmd.split() + assert parts[-2].endswith("_dispatch.py"), ( + f"Dispatcher not in expected position: {cmd}" + ) + logical_name = parts[-1] + assert "/" in logical_name or logical_name in ( + "session_start_hook", + "lint_after_edit_hook", + "token_summary_hook", + "quota_post_hook", + "review_gate_post_hook", + "skill_load_post_hook", + ), f"Unexpected logical name format: {logical_name}" + + +@pytest.mark.medium +class TestDispatchOverhead: + def test_dispatch_overhead_acceptable(self, hook_env: Path) -> None: + direct_script = hook_env / "guards" / "quota_guard.py" + dispatch_script = hook_env / "_dispatch.py" + stdin_data = b'{"tool_name": "Read", "tool_input": {}}' + runs = 10 + + direct_times = [] + for _ in range(runs): + start = time.perf_counter() + subprocess.run( + [sys.executable, str(direct_script)], + input=stdin_data, + capture_output=True, + ) + direct_times.append(time.perf_counter() - start) + + dispatch_times = [] + for _ in range(runs): + start = time.perf_counter() + subprocess.run( + [sys.executable, str(dispatch_script), "guards/quota_guard"], + input=stdin_data, + capture_output=True, + ) + dispatch_times.append(time.perf_counter() - start) + + direct_median = sorted(direct_times)[runs // 2] + dispatch_median = sorted(dispatch_times)[runs // 2] + overhead = dispatch_median - direct_median + + assert overhead < 0.100, ( + f"Dispatch overhead {overhead:.3f}s exceeds 100ms threshold " + f"(direct={direct_median:.3f}s, dispatch={dispatch_median:.3f}s)" + ) diff --git a/tests/infra/test_hook_executability.py b/tests/hooks/test_hook_executability.py similarity index 90% rename from tests/infra/test_hook_executability.py rename to tests/hooks/test_hook_executability.py index 97a8d87190..6ae82b72c6 100644 --- a/tests/infra/test_hook_executability.py +++ b/tests/hooks/test_hook_executability.py @@ -60,7 +60,12 @@ def test_hook_registry_matches_generated_hooks_json() -> None: matcher = entry.get("matcher", "") for hook in entry["hooks"]: cmd = hook["command"] - script_name = cmd.split("/")[-1] + parts = cmd.split() + if "_dispatch.py" in cmd and len(parts) >= 3: + script_name = parts[-1] + ".py" + else: + path_parts = cmd.split("/hooks/", 1) + script_name = path_parts[1] if len(path_parts) == 2 else cmd.split("/")[-1] generated_pairs.add((matcher, script_name)) registry_pairs: set[tuple[str, str]] = set() @@ -98,6 +103,8 @@ def test_hook_registry_scripts_exist_on_disk() -> None: for script in hook_def.scripts: script_path = hooks_dir / script assert script_path.is_file(), f"Registry script not found on disk: {script_path}" + dispatch_path = hooks_dir / "_dispatch.py" + assert dispatch_path.is_file(), f"Stable dispatcher not found on disk: {dispatch_path}" # REQ-HOOK-001 @@ -115,11 +122,15 @@ def test_generate_hooks_json_session_start_no_matcher() -> None: # T-CROSS-2 -def test_generate_hooks_json_and_sync_produce_equivalent_entries(tmp_path) -> None: +def test_generate_hooks_json_and_sync_produce_equivalent_entries(tmp_path, monkeypatch) -> None: """Both generation paths must produce structurally identical hook entries for every event type. Verifies that _build_hook_entry() is shared, preventing path A/B divergence. """ + import autoskillit.cli._hooks as _hooks_mod + + monkeypatch.setattr(_hooks_mod, "is_git_worktree", lambda path: False) + from autoskillit.cli._hooks import _evict_stale_autoskillit_hooks, sync_hooks_to_settings settings = tmp_path / ".claude" / "settings.json" @@ -166,7 +177,11 @@ def test_generated_hooks_json_embeds_registry_hash() -> None: assert h.get("_autoskillit_registry_hash") == HOOK_REGISTRY_HASH -def test_synced_settings_json_embeds_registry_hash(tmp_path: Path) -> None: +def test_synced_settings_json_embeds_registry_hash(tmp_path: Path, monkeypatch) -> None: + import autoskillit.cli._hooks as _hooks_mod + + monkeypatch.setattr(_hooks_mod, "is_git_worktree", lambda path: False) + from autoskillit.cli._hooks import sync_hooks_to_settings from autoskillit.hook_registry import HOOK_REGISTRY_HASH diff --git a/tests/infra/test_hook_registration_coverage.py b/tests/hooks/test_hook_registration_coverage.py similarity index 53% rename from tests/infra/test_hook_registration_coverage.py rename to tests/hooks/test_hook_registration_coverage.py index 0d9d0b131c..f464b3d4d1 100644 --- a/tests/infra/test_hook_registration_coverage.py +++ b/tests/hooks/test_hook_registration_coverage.py @@ -7,6 +7,20 @@ HOOKS_DIR = Path(__file__).resolve().parents[2] / "src" / "autoskillit" / "hooks" +def _all_hook_script_relpaths() -> set[str]: + """Return all non-private, non-init .py files under HOOKS_DIR as HOOKS_DIR-relative paths. + + Uses rglob to find scripts in subdirectories (guards/, formatters/) as well + as the flat hooks root. Underscore-prefixed and ``__init__.py`` files are + excluded because they are private helper modules, not standalone scripts. + """ + return { + str(p.relative_to(HOOKS_DIR)) + for p in HOOKS_DIR.rglob("*.py") + if p.name != "__init__.py" and not p.name.startswith("_") + } + + def test_all_pretooluse_hook_scripts_are_registered() -> None: """Every hook .py (excl. PostToolUse, SessionStart, __init__, private helpers) is PreToolUse-registered. @@ -14,6 +28,10 @@ def test_all_pretooluse_hook_scripts_are_registered() -> None: Underscore-prefixed modules (e.g. ``_fmt_primitives.py``) are private helper modules imported by hook scripts, not standalone hook scripts themselves, and are excluded from registration coverage. + + After the guards/formatters subdirectory reorganization, HOOK_REGISTRY entries + carry subfolder-prefixed paths (e.g. ``"guards/quota_guard.py"``). This test + compares HOOKS_DIR-relative paths so the comparison is consistent. """ post_or_session_registered = { script @@ -22,11 +40,9 @@ def test_all_pretooluse_hook_scripts_are_registered() -> None: for script in hd.scripts } hook_files = { - f.name - for f in HOOKS_DIR.glob("*.py") - if f.name != "__init__.py" - and not f.name.startswith("_") - and f.name not in post_or_session_registered + relpath + for relpath in _all_hook_script_relpaths() + if relpath not in post_or_session_registered } registered_scripts: set[str] = set() for hook_def in HOOK_REGISTRY: @@ -47,11 +63,7 @@ def test_all_posttooluse_hook_scripts_are_registered() -> None: registered_post = { script for hd in HOOK_REGISTRY if hd.event_type == "PostToolUse" for script in hd.scripts } - all_scripts = { - p.name - for p in HOOKS_DIR.glob("*.py") - if p.name != "__init__.py" and not p.name.startswith("_") - } + all_scripts = _all_hook_script_relpaths() pre_registered = { script for hd in HOOK_REGISTRY if hd.event_type == "PreToolUse" for script in hd.scripts } @@ -71,10 +83,10 @@ def test_all_session_start_hook_scripts_are_registered() -> None: # T-GUARD-2 def test_generated_file_write_guard_registered() -> None: - """generated_file_write_guard.py must be registered in HOOK_REGISTRY.""" + """guards/generated_file_write_guard.py must be registered in HOOK_REGISTRY.""" all_scripts = {s for h in HOOK_REGISTRY for s in h.scripts} - assert "generated_file_write_guard.py" in all_scripts, ( - "generated_file_write_guard.py must be registered in HOOK_REGISTRY" + assert "guards/generated_file_write_guard.py" in all_scripts, ( + "guards/generated_file_write_guard.py must be registered in HOOK_REGISTRY" ) @@ -85,7 +97,7 @@ def test_registry_has_ask_user_question_pre_kitchen_gate(): h for h in HOOK_REGISTRY if h.event_type == "PreToolUse" and h.matcher == "AskUserQuestion" ] assert len(ask_matchers) == 1 - assert ask_matchers[0].scripts == ["ask_user_question_guard.py"] + assert ask_matchers[0].scripts == ["guards/ask_user_question_guard.py"] def test_retired_script_basenames_exists_and_complete() -> None: @@ -98,6 +110,7 @@ def test_retired_script_basenames_exists_and_complete() -> None: "pretty_output.py", "token_summary_appender.py", "session_start_reminder.py", + "headless_orchestration_guard.py", } missing = required - set(RETIRED_SCRIPT_BASENAMES) assert not missing, f"RETIRED_SCRIPT_BASENAMES is missing: {missing}" @@ -166,3 +179,87 @@ def test_hook_registry_hash_changes_on_retired_mutation() -> None: mutated_retired = frozenset(RETIRED_SCRIPT_BASENAMES | {"extra_retired.py"}) assert compute_registry_hash(HOOK_REGISTRY, mutated_retired) != HOOK_REGISTRY_HASH + + +# T4-1 +def test_review_gate_post_hook_registered_in_hook_registry() -> None: + """review_gate_post_hook.py must be registered as a PostToolUse hook.""" + post_scripts = { + s for hd in HOOK_REGISTRY if hd.event_type == "PostToolUse" for s in hd.scripts + } + assert "review_gate_post_hook.py" in post_scripts, ( + "review_gate_post_hook.py must be registered as a PostToolUse hook in HOOK_REGISTRY" + ) + + +# T4-2 +def test_review_loop_gate_registered_in_hook_registry() -> None: + """guards/review_loop_gate.py must be registered as a PreToolUse hook.""" + pre_scripts = {s for hd in HOOK_REGISTRY if hd.event_type == "PreToolUse" for s in hd.scripts} + assert "guards/review_loop_gate.py" in pre_scripts, ( + "guards/review_loop_gate.py must be registered as a PreToolUse hook in HOOK_REGISTRY" + ) + + +# T4-3 +def test_review_loop_gate_matcher_covers_wait_for_ci_and_enqueue_pr() -> None: + """guards/review_loop_gate.py matcher must cover both wait_for_ci and enqueue_pr.""" + import re + + gate_entries = [ + hd + for hd in HOOK_REGISTRY + if hd.event_type == "PreToolUse" and "guards/review_loop_gate.py" in hd.scripts + ] + assert gate_entries, "No PreToolUse entry found for guards/review_loop_gate.py" + matcher = gate_entries[0].matcher + assert re.search(r"wait_for_ci", matcher), f"Matcher must cover wait_for_ci; got: {matcher!r}" + assert re.search(r"enqueue_pr", matcher), f"Matcher must cover enqueue_pr; got: {matcher!r}" + + +def test_skill_load_post_hook_registered_in_hook_registry() -> None: + """T3-1: skill_load_post_hook.py must be registered as a PostToolUse hook.""" + post_scripts = { + s for hd in HOOK_REGISTRY if hd.event_type == "PostToolUse" for s in hd.scripts + } + assert "skill_load_post_hook.py" in post_scripts, ( + "skill_load_post_hook.py must be registered as a PostToolUse hook in HOOK_REGISTRY" + ) + + +def test_skill_load_guard_registered_in_hook_registry() -> None: + """T3-2: guards/skill_load_guard.py must be registered as a PreToolUse hook.""" + pre_scripts = {s for hd in HOOK_REGISTRY if hd.event_type == "PreToolUse" for s in hd.scripts} + assert "guards/skill_load_guard.py" in pre_scripts, ( + "guards/skill_load_guard.py must be registered as a PreToolUse hook in HOOK_REGISTRY" + ) + + +def test_skill_load_guard_matcher_covers_native_tools() -> None: + """T3-3: guards/skill_load_guard.py matcher must cover all native tools.""" + import re + + guard_entries = [ + hd + for hd in HOOK_REGISTRY + if hd.event_type == "PreToolUse" and "guards/skill_load_guard.py" in hd.scripts + ] + assert guard_entries, "No PreToolUse entry found for guards/skill_load_guard.py" + matcher = guard_entries[0].matcher + for tool in ("Read", "Write", "Edit", "Bash", "Grep", "Glob"): + assert re.search(tool, matcher), f"Matcher must cover {tool}; got: {matcher!r}" + + +def test_skill_load_post_hook_matcher_is_skill() -> None: + """T3-4: skill_load_post_hook.py matcher must match Skill.""" + import re + + entries = [ + hd + for hd in HOOK_REGISTRY + if hd.event_type == "PostToolUse" and "skill_load_post_hook.py" in hd.scripts + ] + assert entries, "No PostToolUse entry found for skill_load_post_hook.py" + assert re.search("Skill", entries[0].matcher), ( + f"Matcher must match Skill; got: {entries[0].matcher!r}" + ) diff --git a/tests/hooks/test_hook_registry.py b/tests/hooks/test_hook_registry.py new file mode 100644 index 0000000000..0ea29a8cb2 --- /dev/null +++ b/tests/hooks/test_hook_registry.py @@ -0,0 +1,352 @@ +"""Tests for hook_registry.py — L0 hook identity model.""" + +from __future__ import annotations + +import json +from pathlib import Path + +from autoskillit.hook_registry import ( + HOOK_REGISTRY, + HOOKS_DIR, + _extract_script_basenames, + _is_own_hook, + canonical_script_basenames, + find_broken_hook_scripts, + generate_hooks_json, +) + + +# HR-FILTER-1: command with "autoskillit" substring -> True +def test_is_own_hook_autoskillit_substring() -> None: + assert _is_own_hook("python3 /path/to/autoskillit/hooks/quota_guard.py") is True + + +# HR-FILTER-2: command ending with known script basename -> True +def test_is_own_hook_known_basename() -> None: + assert _is_own_hook("python3 /some/other/path/quota_guard.py") is True + + +# HR-FILTER-3: unrelated command -> False +def test_is_own_hook_unrelated_command() -> None: + assert _is_own_hook('wsl-notify-send.exe "Done!"') is False + assert _is_own_hook("python3 /home/user/my_guard.py") is False + + +# HR-FILTER-4: known basename with different path prefix -> True +def test_is_own_hook_different_prefix() -> None: + assert ( + _is_own_hook("python3 /home/user/.local/share/uv/tools/lib/hooks/skill_cmd_guard.py") + is True + ) + + +# HR-BASENAME-1: canonical hooks dict -> returns set of bare filenames +def test_extract_script_basenames_canonical() -> None: + canonical = generate_hooks_json() + result = _extract_script_basenames(canonical.get("hooks", {})) + expected = canonical_script_basenames() + assert result == expected + + +# HR-BASENAME-2: different path prefixes -> returns same basenames +def test_extract_script_basenames_different_prefix() -> None: + foreign_dir = ( + "/home/user/.local/share/uv/tools/autoskillit/lib/python3.13" + "/site-packages/autoskillit/hooks" + ) + by_event: dict[str, list[dict]] = {} + for hdef in HOOK_REGISTRY: + hook_commands = [ + {"type": "command", "command": f"python3 {foreign_dir}/{script}"} + for script in hdef.scripts + ] + entry: dict = {"hooks": hook_commands} + if hdef.event_type != "SessionStart": + entry["matcher"] = hdef.matcher + by_event.setdefault(hdef.event_type, []).append(entry) + + result = _extract_script_basenames(by_event) + assert result == canonical_script_basenames() + + +# HR-BASENAME-3: mixed autoskillit + user hooks -> returns only autoskillit basenames +def test_extract_script_basenames_filters_user_hooks() -> None: + canonical = generate_hooks_json() + hooks = canonical["hooks"] + hooks.setdefault("PreToolUse", []).append( + { + "matcher": ".*", + "hooks": [ + {"type": "command", "command": "python3 /home/user/my_guard.py"}, + {"type": "command", "command": 'wsl-notify-send.exe "Done!"'}, + ], + } + ) + result = _extract_script_basenames(hooks) + assert result == canonical_script_basenames() + assert "my_guard.py" not in result + + +# --------------------------------------------------------------------------- +# T7 — Hook file renames: new names exist, old names removed +# --------------------------------------------------------------------------- + +_HOOKS_DIR = Path(__file__).parent.parent.parent / "src" / "autoskillit" / "hooks" + + +def test_renamed_hook_files_exist() -> None: + """New hook filenames must exist on disk after rename.""" + for expected in [ + "guards/quota_guard.py", + "quota_post_hook.py", + "guards/skill_cmd_guard.py", + "formatters/pretty_output_hook.py", + "session_start_hook.py", + "token_summary_hook.py", + ]: + assert (_HOOKS_DIR / expected).exists(), f"Missing renamed hook: {expected}" + + +def test_old_hook_names_removed() -> None: + """Old hook filenames must NOT exist on disk after rename.""" + for old in [ + "quota_check.py", + "quota_post_check.py", + "skill_cmd_check.py", + "pretty_output.py", + "session_start_reminder.py", + "token_summary_appender.py", + ]: + assert not (_HOOKS_DIR / old).exists(), f"Old hook file still present: {old}" + + +def test_hook_registry_uses_new_filenames() -> None: + """HOOK_REGISTRY must reference only the new filenames, not the old ones.""" + all_scripts = {s for h in HOOK_REGISTRY for s in h.scripts} + old_names = { + "quota_check.py", + "quota_post_check.py", + "skill_cmd_check.py", + "pretty_output.py", + "session_start_reminder.py", + "token_summary_appender.py", + } + for old in old_names: + assert old not in all_scripts, f"HOOK_REGISTRY still references old name: {old}" + + +def test_find_broken_hook_scripts_ignores_user_inline_hooks(tmp_path: Path) -> None: + """User inline hooks like 'wsl-notify-send.exe Done' must not be flagged as broken.""" + settings = tmp_path / "settings.json" + settings.write_text( + json.dumps( + { + "hooks": { + "PostToolUse": [{"hooks": [{"command": 'wsl-notify-send.exe "Build done"'}]}] + } + } + ) + ) + broken = find_broken_hook_scripts(settings) + assert broken == [] + + +def test_find_broken_hook_scripts_flags_missing_autoskillit_script(tmp_path: Path) -> None: + """A python3 /abs/path/autoskillit/hooks/script.py that doesn't exist must be flagged.""" + missing = str(HOOKS_DIR / "nonexistent_guard.py") + settings = tmp_path / "settings.json" + settings.write_text( + json.dumps( + { + "hooks": { + "PreToolUse": [{"matcher": ".*", "hooks": [{"command": f"python3 {missing}"}]}] + } + } + ) + ) + broken = find_broken_hook_scripts(settings) + assert any("nonexistent_guard.py" in b for b in broken) + + +# T-WT-5: find_broken_hook_scripts detects deleted worktree directory (regression gate) +def test_find_broken_hook_scripts_detects_deleted_worktree_dir(tmp_path: Path) -> None: + """find_broken_hook_scripts must flag hooks where the entire directory tree is gone.""" + deleted_worktree = "/tmp/deleted_worktree_12345/src/autoskillit/hooks" + settings = { + "hooks": { + "PreToolUse": [ + { + "matcher": ".*", + "hooks": [ + { + "type": "command", + "command": f"python3 {deleted_worktree}/quota_guard.py", + }, + { + "type": "command", + "command": f"python3 {deleted_worktree}/branch_protection_guard.py", + }, + ], + } + ] + } + } + settings_path = tmp_path / "settings.json" + settings_path.write_text(json.dumps(settings)) + + broken = find_broken_hook_scripts(settings_path) + assert len(broken) == 2 + assert all(deleted_worktree in b for b in broken) + + +def test_find_broken_hook_scripts_does_not_flag_user_python_scripts(tmp_path: Path) -> None: + """A user's python3 script outside the autoskillit hooks dir must not be flagged.""" + user_script = "/home/user/my_custom_guard.py" + settings = tmp_path / "settings.json" + settings.write_text( + json.dumps( + { + "hooks": { + "PreToolUse": [ + {"matcher": ".*", "hooks": [{"command": f"python3 {user_script}"}]} + ] + } + } + ) + ) + broken = find_broken_hook_scripts(settings) + assert broken == [] + + +# Hooks excluded from the deny-trigger requirement: purely advisory (non-blocking) +# or auto-correcting (not a hard deny). +_ADVISORY_HOOKS: frozenset[str] = frozenset( + { + "guards/recipe_write_advisor.py", + "guards/grep_pattern_lint_guard.py", + "guards/mcp_health_advisor.py", + } +) + + +def test_deny_path_pretooluse_hooks_export_deny_trigger() -> None: + """Every non-advisory PreToolUse hook with a deny path must export a *_DENY_TRIGGER constant. + + This structural test ensures that prompt builders can reference deny triggers + programmatically rather than hardcoding strings. New hooks added to HOOK_REGISTRY + must include a DENY_TRIGGER constant or be added to _ADVISORY_HOOKS above. + """ + import importlib + + deny_path_scripts: set[str] = set() + for hook_def in HOOK_REGISTRY: + if hook_def.event_type != "PreToolUse": + continue + for script in hook_def.scripts: + if script not in _ADVISORY_HOOKS: + deny_path_scripts.add(script) + + missing: list[str] = [] + for script in sorted(deny_path_scripts): + module_name = script.removesuffix(".py").replace("/", ".") + module = importlib.import_module(f"autoskillit.hooks.{module_name}") + has_trigger = any(name.endswith("_DENY_TRIGGER") for name in dir(module)) + if not has_trigger: + missing.append(script) + + assert not missing, ( + f"These PreToolUse hooks have no *_DENY_TRIGGER constant: {missing}. " + "Add a DENY_TRIGGER constant or add the hook to _ADVISORY_HOOKS if it is non-blocking." + ) + + +# HR-RETIRED-1: scripts moved to subdirectories must have flat basenames in RETIRED +def test_moved_scripts_must_be_in_retired() -> None: + """Any script moved from flat hooks/ to a subdirectory must have its flat basename in RETIRED. + + Auto-enforces the CLAUDE.md rule: renaming a hook script must update RETIRED_SCRIPT_BASENAMES + in the same commit. Future directory reorganizations will fail this test automatically. + """ + from autoskillit.hook_registry import ( + HOOK_REGISTRY, + NEW_SUBDIR_BASENAMES, + RETIRED_SCRIPT_BASENAMES, + ) + + flat_scripts = {s for h in HOOK_REGISTRY for s in h.scripts if "/" not in s} + subdir_scripts = {s for h in HOOK_REGISTRY for s in h.scripts if "/" in s} + + missing: list[str] = [] + for subdir_script in subdir_scripts: + bare = subdir_script.rsplit("/", 1)[-1] + if ( + bare not in flat_scripts + and bare not in RETIRED_SCRIPT_BASENAMES + and bare not in NEW_SUBDIR_BASENAMES + ): + missing.append( + f"{subdir_script!r} → {bare!r} not in RETIRED_SCRIPT_BASENAMES" + " or NEW_SUBDIR_BASENAMES" + ) + + assert not missing, ( + "Subdir scripts must have their basename in RETIRED_SCRIPT_BASENAMES (moved)\n" + "or in NEW_SUBDIR_BASENAMES (newly added to a subdir):\n" + + "\n".join(f" {m}" for m in missing) + ) + + +def test_find_broken_hook_scripts_dispatcher_format_valid(tmp_path: Path) -> None: + """Dispatcher-format commands are not flagged when _dispatch.py exists.""" + dispatch = tmp_path / "_dispatch.py" + dispatch.write_text("# dispatcher stub") + settings = tmp_path / "settings.json" + settings.write_text( + json.dumps( + { + "hooks": { + "PreToolUse": [ + { + "matcher": ".*", + "hooks": [ + { + "type": "command", + "command": f"python3 {dispatch} guards/quota_guard", + } + ], + } + ] + } + } + ) + ) + broken = find_broken_hook_scripts(settings) + assert broken == [] + + +def test_find_broken_hook_scripts_dispatcher_missing(tmp_path: Path) -> None: + """Dispatcher-format commands are flagged when _dispatch.py itself is missing.""" + missing_dispatch = tmp_path / "nonexistent" / "_dispatch.py" + settings = tmp_path / "settings.json" + settings.write_text( + json.dumps( + { + "hooks": { + "PreToolUse": [ + { + "matcher": ".*", + "hooks": [ + { + "type": "command", + "command": f"python3 {missing_dispatch} guards/quota_guard", + } + ], + } + ] + } + } + ) + ) + broken = find_broken_hook_scripts(settings) + assert len(broken) == 1 + assert "_dispatch.py" in broken[0] diff --git a/tests/infra/test_hook_settings.py b/tests/hooks/test_hook_settings.py similarity index 100% rename from tests/infra/test_hook_settings.py rename to tests/hooks/test_hook_settings.py diff --git a/tests/infra/test_hook_sync.py b/tests/hooks/test_hook_sync.py similarity index 62% rename from tests/infra/test_hook_sync.py rename to tests/hooks/test_hook_sync.py index 1c5b867f65..0b7238ca2e 100644 --- a/tests/infra/test_hook_sync.py +++ b/tests/hooks/test_hook_sync.py @@ -7,13 +7,13 @@ def test_hook_config_path_components_in_sync(): """pretty_output._HOOK_CONFIG_PATH_COMPONENTS must resolve to the same path as - server/helpers._HOOK_DIR_COMPONENTS + _HOOK_CONFIG_FILENAME. + server/_misc._HOOK_DIR_COMPONENTS + _HOOK_CONFIG_FILENAME. Both scripts must address the same config file. This test guards against independent evolution of the two constant sets. """ - from autoskillit.hooks.pretty_output_hook import _HOOK_CONFIG_PATH_COMPONENTS - from autoskillit.server.helpers import _HOOK_CONFIG_FILENAME, _HOOK_DIR_COMPONENTS + from autoskillit.hooks.formatters.pretty_output_hook import _HOOK_CONFIG_PATH_COMPONENTS + from autoskillit.server._misc import _HOOK_CONFIG_FILENAME, _HOOK_DIR_COMPONENTS path_from_pretty = Path(*_HOOK_CONFIG_PATH_COMPONENTS) path_from_helpers = Path(*_HOOK_DIR_COMPONENTS) / _HOOK_CONFIG_FILENAME @@ -21,7 +21,7 @@ def test_hook_config_path_components_in_sync(): assert path_from_pretty == path_from_helpers, ( f"Hook config path mismatch:\n" f" pretty_output: {path_from_pretty}\n" - f" server/helpers: {path_from_helpers}\n" + f" server/_misc: {path_from_helpers}\n" "Update the constants to point to the same file." ) @@ -34,9 +34,20 @@ def test_hook_config_path_single_source_of_truth(): _hook_settings.HOOK_DIR_COMPONENTS + HOOK_CONFIG_FILENAME reconstructs _fmt_primitives._HOOK_CONFIG_PATH_COMPONENTS exactly. """ - from autoskillit.hooks._fmt_primitives import _HOOK_CONFIG_PATH_COMPONENTS from autoskillit.hooks._hook_settings import HOOK_CONFIG_FILENAME, HOOK_DIR_COMPONENTS + from autoskillit.hooks.formatters._fmt_primitives import _HOOK_CONFIG_PATH_COMPONENTS assert (*HOOK_DIR_COMPONENTS, HOOK_CONFIG_FILENAME) == _HOOK_CONFIG_PATH_COMPONENTS, ( "_hook_settings path constants must match _fmt_primitives._HOOK_CONFIG_PATH_COMPONENTS" ) + + +def test_quota_budget_exceeded_trigger_consistency(): + """QUOTA_BUDGET_EXCEEDED_TRIGGER must be identical in quota_guard and autoskillit.hooks.""" + from autoskillit.hooks import QUOTA_BUDGET_EXCEEDED_TRIGGER + from autoskillit.hooks.guards.quota_guard import QUOTA_BUDGET_EXCEEDED_TRIGGER as _DIRECT + + assert QUOTA_BUDGET_EXCEEDED_TRIGGER == _DIRECT, ( + f"QUOTA_BUDGET_EXCEEDED_TRIGGER mismatch: " + f"quota_guard={_DIRECT!r} vs hooks re-export={QUOTA_BUDGET_EXCEEDED_TRIGGER!r}" + ) diff --git a/tests/hooks/test_lint_after_edit_hook.py b/tests/hooks/test_lint_after_edit_hook.py new file mode 100644 index 0000000000..9af77587bf --- /dev/null +++ b/tests/hooks/test_lint_after_edit_hook.py @@ -0,0 +1,271 @@ +"""Tests for lint_after_edit_hook.py PostToolUse hook.""" + +from __future__ import annotations + +import io +import json +import subprocess as sp +from contextlib import redirect_stdout +from unittest.mock import patch + +import pytest + +pytestmark = [pytest.mark.layer("infra"), pytest.mark.small] + + +def _build_event( + tool_name: str, + file_path: str, + tool_response: str = "The file was edited successfully.", +) -> dict: + return { + "tool_name": tool_name, + "tool_input": {"file_path": file_path}, + "tool_response": tool_response, + } + + +def _run_hook( + event: dict | str, + *, + headless: bool = False, + skill_name: str = "", + monkeypatch: pytest.MonkeyPatch, +) -> tuple[str, int]: + from autoskillit.hooks.lint_after_edit_hook import main # noqa: PLC0415 + + if headless: + monkeypatch.setenv("AUTOSKILLIT_HEADLESS", "1") + else: + monkeypatch.delenv("AUTOSKILLIT_HEADLESS", raising=False) + + if skill_name: + monkeypatch.setenv("AUTOSKILLIT_SKILL_NAME", skill_name) + else: + monkeypatch.delenv("AUTOSKILLIT_SKILL_NAME", raising=False) + + stdin_text = json.dumps(event) if isinstance(event, dict) else event + buf = io.StringIO() + exit_code = 0 + with ( + patch("sys.stdin", io.StringIO(stdin_text)), + redirect_stdout(buf), + ): + try: + main() + except SystemExit as exc: + exit_code = int(exc.code) if exc.code is not None else 0 + return buf.getvalue(), exit_code + + +class TestScopingGates: + """Hook must be silent outside its activation scope.""" + + def test_no_op_when_interactive(self, tmp_path, monkeypatch): + f = tmp_path / "x.py" + f.write_text("x = 1\n") + out, code = _run_hook( + _build_event("Edit", str(f)), + headless=False, + monkeypatch=monkeypatch, + ) + assert out == "" and code == 0 + + def test_no_op_when_non_implement_skill(self, tmp_path, monkeypatch): + f = tmp_path / "x.py" + f.write_text("x = 1\n") + out, code = _run_hook( + _build_event("Edit", str(f)), + headless=True, + skill_name="review-pr", + monkeypatch=monkeypatch, + ) + assert out == "" and code == 0 + + def test_no_op_for_non_python_file(self, tmp_path, monkeypatch): + f = tmp_path / "data.yaml" + f.write_text("key: value\n") + out, code = _run_hook( + _build_event("Edit", str(f)), + headless=True, + skill_name="implement-worktree", + monkeypatch=monkeypatch, + ) + assert out == "" and code == 0 + + def test_no_op_for_nonexistent_file(self, monkeypatch): + out, code = _run_hook( + _build_event("Edit", "/nonexistent/path.py"), + headless=True, + skill_name="implement-worktree", + monkeypatch=monkeypatch, + ) + assert out == "" and code == 0 + + @pytest.mark.parametrize( + "skill", + [ + "implement-worktree-no-merge", + "implement-worktree", + "implement-experiment", + ], + ) + def test_activates_for_implement_skills(self, tmp_path, monkeypatch, skill): + f = tmp_path / "clean.py" + f.write_text("x = 1\n") + _out, code = _run_hook( + _build_event("Edit", str(f)), + headless=True, + skill_name=skill, + monkeypatch=monkeypatch, + ) + assert code == 0 + + @pytest.mark.parametrize( + "skill", + [ + "resolve-review", + "resolve-failures", + "resolve-research-review", + ], + ) + def test_activates_for_resolve_skills(self, tmp_path, monkeypatch, skill): + f = tmp_path / "clean.py" + f.write_text("x = 1\n") + _out, code = _run_hook( + _build_event("Edit", str(f)), + headless=True, + skill_name=skill, + monkeypatch=monkeypatch, + ) + assert code == 0 + + def test_write_tool_also_triggers(self, tmp_path, monkeypatch): + f = tmp_path / "clean.py" + f.write_text("x = 1\n") + _out, code = _run_hook( + _build_event("Write", str(f)), + headless=True, + skill_name="implement-worktree", + monkeypatch=monkeypatch, + ) + assert code == 0 + + +class TestFailOpen: + """Hook must fail-open on malformed input or missing tools.""" + + def test_malformed_stdin(self, monkeypatch): + out, code = _run_hook( + "not json at all", + headless=True, + skill_name="implement-worktree", + monkeypatch=monkeypatch, + ) + assert out == "" and code == 0 + + def test_missing_tool_input(self, monkeypatch): + out, code = _run_hook( + {}, + headless=True, + skill_name="implement-worktree", + monkeypatch=monkeypatch, + ) + assert out == "" and code == 0 + + def test_ruff_not_found(self, tmp_path, monkeypatch): + f = tmp_path / "x.py" + f.write_text("x=1\n") + with patch( + "autoskillit.hooks.lint_after_edit_hook.subprocess.run", + side_effect=FileNotFoundError("ruff not found"), + ): + out, code = _run_hook( + _build_event("Edit", str(f)), + headless=True, + skill_name="implement-worktree", + monkeypatch=monkeypatch, + ) + assert out == "" and code == 0 + + def test_subprocess_timeout(self, tmp_path, monkeypatch): + f = tmp_path / "x.py" + f.write_text("x=1\n") + with patch( + "autoskillit.hooks.lint_after_edit_hook.subprocess.run", + side_effect=sp.TimeoutExpired("ruff", 15), + ): + out, code = _run_hook( + _build_event("Edit", str(f)), + headless=True, + skill_name="implement-worktree", + monkeypatch=monkeypatch, + ) + assert out == "" and code == 0 + + +class TestLintBehavior: + """Hook must detect and report lint issues via ruff.""" + + def test_clean_file_silent(self, tmp_path, monkeypatch): + f = tmp_path / "clean.py" + f.write_text("x = 1\n") + out, code = _run_hook( + _build_event("Edit", str(f)), + headless=True, + skill_name="implement-worktree", + monkeypatch=monkeypatch, + ) + assert out == "" and code == 0 + + def test_autofix_signals_reread(self, tmp_path, monkeypatch): + f = tmp_path / "bad_fmt.py" + f.write_text("x=1\n") + out, code = _run_hook( + _build_event("Edit", str(f)), + headless=True, + skill_name="implement-worktree", + monkeypatch=monkeypatch, + ) + assert code == 0 + from autoskillit.hooks.lint_after_edit_hook import ( + LINT_AUTOFIX_TRIGGER, + ) + + assert out != "", "ruff should auto-format x=1 to x = 1" + parsed = json.loads(out) + updated = parsed["hookSpecificOutput"]["updatedToolResult"] + assert LINT_AUTOFIX_TRIGGER in updated + assert "re-read" in updated.lower() + + def test_unfixable_error_reported(self, tmp_path, monkeypatch): + f = tmp_path / "long_line.py" + long_var = "a" * 200 + f.write_text(f'{long_var} = "x"\n') + out, code = _run_hook( + _build_event("Edit", str(f)), + headless=True, + skill_name="implement-worktree", + monkeypatch=monkeypatch, + ) + assert code == 0 + from autoskillit.hooks.lint_after_edit_hook import LINT_ERROR_TRIGGER + + assert out != "", "E501 should be reported for 200-char variable" + parsed = json.loads(out) + updated = parsed["hookSpecificOutput"]["updatedToolResult"] + assert LINT_ERROR_TRIGGER in updated + + +class TestRegistration: + def test_hook_registered_in_registry(self): + from autoskillit.hook_registry import HOOK_REGISTRY + + post_write_edit = [ + h + for h in HOOK_REGISTRY + if h.event_type == "PostToolUse" and h.matcher == r"Write|Edit" + ] + assert len(post_write_edit) == 1 + assert "lint_after_edit_hook.py" in post_write_edit[0].scripts + assert post_write_edit[0].session_scope == "headless_only" diff --git a/tests/infra/test_quota_check.py b/tests/hooks/test_quota_check.py similarity index 90% rename from tests/infra/test_quota_check.py rename to tests/hooks/test_quota_check.py index cb7777a19c..eb32b758dd 100644 --- a/tests/infra/test_quota_check.py +++ b/tests/hooks/test_quota_check.py @@ -12,9 +12,9 @@ from pathlib import Path from unittest.mock import patch -from autoskillit.hooks._fmt_primitives import _HOOK_CONFIG_PATH_COMPONENTS +from autoskillit.hooks.formatters._fmt_primitives import _HOOK_CONFIG_PATH_COMPONENTS -_LONG_PATTERNS = ("weekly", "sonnet", "opus") +_LONG_PATTERNS = ("seven_day", "sonnet", "opus") def _classify_threshold(window_name: str) -> float: @@ -68,7 +68,7 @@ def _run_hook( Returns (stdout, exit_code). stdout empty = approve, JSON string = deny. Uses cache_path_override parameter for DI — no module-level patching needed. """ - from autoskillit.hooks.quota_guard import main + from autoskillit.hooks.guards.quota_guard import main stdin_text = raw_stdin if raw_stdin is not None else json.dumps(event or {}) @@ -447,18 +447,18 @@ def test_defaults_yaml_cache_max_age_is_300(): assert defaults["quota_guard"]["cache_max_age"] == 300 -# T-HOOK-PWT-1: regression test for #721 — weekly at 86% must approve. -def test_hook_approves_weekly_at_86_percent_should_block_false(tmp_path): - """Weekly window at 86% utilisation must NOT be blocked. +# T-HOOK-PWT-1: regression test for #721 — seven_day at 86% must approve. +def test_hook_approves_seven_day_at_86_percent_should_block_false(tmp_path): + """seven_day window at 86% utilisation must NOT be blocked. - Regression test for issue #721: long-window quotas (weekly/sonnet/opus) + Regression test for issue #721: long-window quotas (seven_day/sonnet/opus) use a higher threshold (98%), so 86% is comfortable headroom — not exhaustion. """ cache = tmp_path / "quota_cache.json" _write_cache( cache, utilization=86.0, - window_name="weekly", + window_name="seven_day", should_block=False, effective_threshold=98.0, ) @@ -466,13 +466,13 @@ def test_hook_approves_weekly_at_86_percent_should_block_false(tmp_path): assert out.strip() == "" -# T-HOOK-PWT-2: weekly above 98% must still deny. -def test_hook_blocks_weekly_above_long_threshold(tmp_path): +# T-HOOK-PWT-2: seven_day above 98% must still deny. +def test_hook_blocks_seven_day_above_long_threshold(tmp_path): cache = tmp_path / "quota_cache.json" _write_cache( cache, utilization=99.0, - window_name="weekly", + window_name="seven_day", should_block=True, effective_threshold=98.0, ) @@ -480,7 +480,7 @@ def test_hook_blocks_weekly_above_long_threshold(tmp_path): data = json.loads(out) assert data["hookSpecificOutput"]["permissionDecision"] == "deny" reason = data["hookSpecificOutput"]["permissionDecisionReason"] - assert "weekly" in reason + assert "seven_day" in reason assert "98" in reason @@ -578,7 +578,7 @@ def test_resolve_quota_log_dir_and_resolve_log_dir_in_sync(monkeypatch): and self-contained. This test is the canonical drift guard. """ from autoskillit.execution.session_log import resolve_log_dir - from autoskillit.hooks.quota_guard import _resolve_quota_log_dir + from autoskillit.hooks.guards.quota_guard import _resolve_quota_log_dir # Case 1: platform default (no env overrides) monkeypatch.delenv("AUTOSKILLIT_LOG_DIR", raising=False) @@ -639,3 +639,42 @@ def test_hook_still_blocks_without_disabled_flag(tmp_path, monkeypatch): out, _ = _run_hook(event={"tool_name": "run_skill"}) data = json.loads(out) assert data["hookSpecificOutput"]["permissionDecision"] == "deny" + + +def test_provider_bypass_skips_quota_check(tmp_path, monkeypatch): + """Non-anthropic AUTOSKILLIT_PROVIDER_PROFILE bypasses quota check entirely.""" + monkeypatch.setenv("AUTOSKILLIT_PROVIDER_PROFILE", "minimax") + cache = tmp_path / "quota_cache.json" + _write_cache(cache, utilization=99.0, should_block=True) + out, exit_code = _run_hook(event={"tool_name": "run_skill"}, cache_path=cache) + assert out.strip() == "" + assert exit_code == 0 + + +def test_provider_bypass_logs_event(tmp_path, monkeypatch): + """Provider bypass writes a 'provider_bypass' event to quota_events.jsonl.""" + monkeypatch.setenv("AUTOSKILLIT_PROVIDER_PROFILE", "minimax") + log_dir = tmp_path / "logs" + monkeypatch.setenv("AUTOSKILLIT_LOG_DIR", str(log_dir)) + cache = tmp_path / "quota_cache.json" + _write_cache(cache, utilization=99.0, should_block=True) + _run_hook(event={"tool_name": "run_skill"}, cache_path=cache) + events = [ + json.loads(line) for line in (log_dir / "quota_events.jsonl").read_text().splitlines() + ] + assert len(events) == 1 + ev = events[0] + assert ev["event"] == "provider_bypass" + assert ev["profile"] == "minimax" + assert "ts" in ev + assert "cache_path" in ev + + +def test_anthropic_profile_does_not_bypass(tmp_path, monkeypatch): + """AUTOSKILLIT_PROVIDER_PROFILE=anthropic still enforces quota check.""" + monkeypatch.setenv("AUTOSKILLIT_PROVIDER_PROFILE", "anthropic") + cache = tmp_path / "quota_cache.json" + _write_cache(cache, utilization=99.0, should_block=True) + out, _ = _run_hook(event={"tool_name": "run_skill"}, cache_path=cache) + data = json.loads(out) + assert data["hookSpecificOutput"]["permissionDecision"] == "deny" diff --git a/tests/infra/test_quota_post_check.py b/tests/hooks/test_quota_post_check.py similarity index 83% rename from tests/infra/test_quota_post_check.py rename to tests/hooks/test_quota_post_check.py index e0aa2cc27f..29ab04f26c 100644 --- a/tests/infra/test_quota_post_check.py +++ b/tests/hooks/test_quota_post_check.py @@ -7,12 +7,15 @@ import io import json +import pathlib from contextlib import redirect_stdout from datetime import UTC, datetime, timedelta from pathlib import Path from unittest.mock import patch -from autoskillit.hooks._fmt_primitives import _HOOK_CONFIG_PATH_COMPONENTS +import pytest + +from autoskillit.hooks.formatters._fmt_primitives import _HOOK_CONFIG_PATH_COMPONENTS _LONG_PATTERNS = ("weekly", "sonnet", "opus") @@ -415,3 +418,79 @@ def test_qpc_cache_max_age_env_var_override(tmp_path, monkeypatch): out, exit_code = _run_hook(event=event, cache_path=cache) assert out.strip() == "" assert exit_code == 0 + + +def test_resolve_quota_log_dir_prints_to_stderr_on_exception( + monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] +) -> None: + """_resolve_quota_log_dir() bare except must print warning to stderr, not silently swallow.""" + from autoskillit.hooks.quota_post_hook import _resolve_quota_log_dir + + def _raise() -> None: + raise OSError("boom") + + monkeypatch.delenv("AUTOSKILLIT_LOG_DIR", raising=False) + monkeypatch.delenv("XDG_DATA_HOME", raising=False) + monkeypatch.setattr(pathlib.Path, "home", staticmethod(_raise)) + + result = _resolve_quota_log_dir() + + assert result is None + captured = capsys.readouterr() + assert "quota_post_hook" in captured.err + + +def test_write_quota_log_event_prints_to_stderr_on_write_failure( + tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + """_write_quota_log_event() bare except must print warning to stderr, not silently swallow.""" + from autoskillit.hooks.quota_post_hook import _write_quota_log_event + + with patch("builtins.open", side_effect=OSError("disk full")): + _write_quota_log_event({}, tmp_path) + + captured = capsys.readouterr() + assert "quota_post_hook" in captured.err + + +def test_post_provider_bypass_skips_warning(tmp_path, monkeypatch): + """Non-anthropic AUTOSKILLIT_PROVIDER_PROFILE bypasses post-check warning.""" + monkeypatch.setenv("AUTOSKILLIT_PROVIDER_PROFILE", "minimax") + cache = tmp_path / "quota_cache.json" + _write_cache(cache, utilization=99.0, should_block=True) + event = _build_event() + out, exit_code = _run_hook(event=event, cache_path=cache) + assert out.strip() == "" + assert exit_code == 0 + + +def test_post_provider_bypass_logs_event(tmp_path, monkeypatch): + """Provider bypass writes a 'post_provider_bypass' event to quota_events.jsonl.""" + monkeypatch.setenv("AUTOSKILLIT_PROVIDER_PROFILE", "minimax") + log_dir = tmp_path / "logs" + monkeypatch.setenv("AUTOSKILLIT_LOG_DIR", str(log_dir)) + cache = tmp_path / "quota_cache.json" + _write_cache(cache, utilization=99.0, should_block=True) + event = _build_event() + _run_hook(event=event, cache_path=cache) + events = [ + json.loads(line) for line in (log_dir / "quota_events.jsonl").read_text().splitlines() + ] + assert len(events) == 1 + ev = events[0] + assert ev["event"] == "post_provider_bypass" + assert ev["profile"] == "minimax" + assert "ts" in ev + assert "tool_name" in ev + assert "cache_path" in ev + + +def test_post_anthropic_profile_does_not_bypass(tmp_path, monkeypatch): + """AUTOSKILLIT_PROVIDER_PROFILE=anthropic still enforces post-check warning.""" + monkeypatch.setenv("AUTOSKILLIT_PROVIDER_PROFILE", "anthropic") + cache = tmp_path / "quota_cache.json" + _write_cache(cache, utilization=99.0, should_block=True) + event = _build_event() + out, _ = _run_hook(event=event, cache_path=cache) + data = json.loads(out) + assert "QUOTA WARNING" in data["hookSpecificOutput"]["updatedMCPToolOutput"] diff --git a/tests/hooks/test_recipe_write_advisor.py b/tests/hooks/test_recipe_write_advisor.py new file mode 100644 index 0000000000..a0c1f1c0ef --- /dev/null +++ b/tests/hooks/test_recipe_write_advisor.py @@ -0,0 +1,140 @@ +"""Tests for autoskillit.hooks.guards.recipe_write_advisor.""" + +from __future__ import annotations + +import io +import json +import os +import subprocess +import sys +from contextlib import redirect_stdout +from unittest.mock import patch + +from autoskillit.core.paths import pkg_root + + +def _run_advisor(payload: dict, extra_env: dict[str, str] | None = None) -> tuple[int, str]: + hook_path = pkg_root() / "hooks" / "guards" / "recipe_write_advisor.py" + env = {k: v for k, v in os.environ.items() if k != "AUTOSKILLIT_HEADLESS"} + env.update(extra_env or {}) + result = subprocess.run( + [sys.executable, str(hook_path)], + input=json.dumps(payload), + capture_output=True, + text=True, + env=env, + ) + return result.returncode, result.stdout + + +class TestRecipeWriteAdvisor: + def test_recipe_yaml_write_triggers_advisory(self) -> None: + """Write to .autoskillit/recipes/foo.yaml triggers write-recipe advisory.""" + payload = { + "tool_name": "Write", + "tool_input": {"file_path": ".autoskillit/recipes/foo.yaml"}, + } + rc, stdout = _run_advisor(payload) + assert rc == 0 + assert stdout.strip(), "Expected advisory output for recipe YAML" + data = json.loads(stdout.strip()) + assert "write-recipe" in data["hookSpecificOutput"]["message"] + + def test_non_recipe_yaml_write_is_silent(self) -> None: + """Write to a non-recipe path produces no output (exit 0).""" + payload = { + "tool_name": "Write", + "tool_input": {"file_path": "/home/user/project/config.py"}, + } + rc, stdout = _run_advisor(payload) + assert rc == 0 + assert not stdout.strip() + + def test_campaign_yaml_suggests_make_campaign(self) -> None: + """Write to campaigns/ subdir suggests make-campaign instead of write-recipe.""" + payload = { + "tool_name": "Write", + "tool_input": {"file_path": ".autoskillit/recipes/campaigns/my_campaign.yaml"}, + } + rc, stdout = _run_advisor(payload) + assert rc == 0 + assert stdout.strip(), "Expected advisory output for campaign YAML" + data = json.loads(stdout.strip()) + msg = data["hookSpecificOutput"]["message"] + assert "make-campaign" in msg + assert "write-recipe" not in msg + + def test_headless_session_skips_advisory(self) -> None: + """When AUTOSKILLIT_HEADLESS=1, no advisory is emitted.""" + payload = { + "tool_name": "Write", + "tool_input": {"file_path": ".autoskillit/recipes/foo.yaml"}, + } + rc, stdout = _run_advisor(payload, extra_env={"AUTOSKILLIT_HEADLESS": "1"}) + assert rc == 0 + assert not stdout.strip(), "Advisory must be suppressed in headless sessions" + + def test_edit_tool_also_triggers_advisory(self) -> None: + """Edit (not just Write) to a recipe YAML also emits the advisory.""" + payload = { + "tool_name": "Edit", + "tool_input": {"file_path": "src/autoskillit/recipes/my_recipe.yaml"}, + } + rc, stdout = _run_advisor(payload) + assert rc == 0 + assert stdout.strip() + data = json.loads(stdout.strip()) + assert "write-recipe" in data["hookSpecificOutput"]["message"] + + def test_non_write_edit_tool_is_silent(self) -> None: + """Tools other than Write/Edit produce no output even for recipe paths.""" + payload = { + "tool_name": "Read", + "tool_input": {"file_path": ".autoskillit/recipes/foo.yaml"}, + } + rc, stdout = _run_advisor(payload) + assert rc == 0 + assert not stdout.strip() + + +# --------------------------------------------------------------------------- +# In-process session-scope enforcement tests (satisfies +# test_scoped_guard_has_both_session_type_test_cases contract) +# --------------------------------------------------------------------------- + + +def _run_advisor_inprocess( + tool_name: str, + file_path: str, + *, + headless: bool = False, +) -> str: + from autoskillit.hooks.guards.recipe_write_advisor import main + + payload = json.dumps({"tool_name": tool_name, "tool_input": {"file_path": file_path}}) + env_clean = {"AUTOSKILLIT_HEADLESS": "1"} if headless else {} + with ( + patch.dict(os.environ, env_clean, clear=True), + patch("sys.stdin", io.StringIO(payload)), + ): + buf = io.StringIO() + with redirect_stdout(buf): + try: + main() + except SystemExit: + pass + return buf.getvalue() + + +def test_recipe_advisor_emits_advisory_when_headless_false() -> None: + """Non-headless session: advisory message is emitted for recipe YAML writes.""" + out = _run_advisor_inprocess("Write", ".autoskillit/recipes/foo.yaml", headless=False) + assert out.strip(), "Expected advisory output in interactive session" + data = json.loads(out.strip()) + assert "write-recipe" in data["hookSpecificOutput"]["message"] + + +def test_recipe_advisor_suppressed_when_headless_true() -> None: + """Headless session: advisory is suppressed (AUTOSKILLIT_HEADLESS=1).""" + out = _run_advisor_inprocess("Write", ".autoskillit/recipes/foo.yaml", headless=True) + assert not out.strip(), "Advisory must be suppressed in headless sessions" diff --git a/tests/hooks/test_review_gate_post_hook.py b/tests/hooks/test_review_gate_post_hook.py new file mode 100644 index 0000000000..f85eaccc1f --- /dev/null +++ b/tests/hooks/test_review_gate_post_hook.py @@ -0,0 +1,315 @@ +"""Tests for the review_gate_post_hook PostToolUse hook. + +Captures %%REVIEW_GATE:: tags from run_skill output and tracks +check_review_loop calls from run_python. + +Pattern mirrors test_quota_post_check.py. +""" + +from __future__ import annotations + +import contextlib +import io +import json +import unittest.mock + +import pytest + +pytestmark = [pytest.mark.layer("infra"), pytest.mark.small] + +_STATE_FILE_RELPATH = ".autoskillit/temp/review_gate_state.json" +_TOOL_RUN_SKILL = "mcp__plugin_autoskillit_autoskillit__run_skill" +_TOOL_RUN_PYTHON = "mcp__plugin_autoskillit_autoskillit__run_python" +_CALLABLE_CHECK_REVIEW_LOOP = "autoskillit.smoke_utils.check_review_loop" +_CALLABLE_OTHER = "autoskillit.smoke_utils.annotate_pr_diff" + +_TAG_LOOP_REQUIRED = "%%REVIEW_GATE::LOOP_REQUIRED%%" +_TAG_CLEAR = "%%REVIEW_GATE::CLEAR%%" + + +def _build_run_skill_event(tool_response_text: str) -> dict: + """Build a PostToolUse event for run_skill with given response text.""" + return { + "tool_name": _TOOL_RUN_SKILL, + "tool_input": {"skill_command": "/review-pr feat/1290 main", "cwd": "/tmp/work"}, + "tool_response": json.dumps( + { + "result": json.dumps( + { + "success": True, + "result": tool_response_text, + } + ) + } + ), + } + + +def _build_run_python_event(callable_name: str, args: dict | None = None) -> dict: + """Build a PostToolUse event for run_python.""" + return { + "tool_name": _TOOL_RUN_PYTHON, + "tool_input": {"callable": callable_name, "args": args or {}}, + "tool_response": json.dumps({"success": True, "result": "{}"}), + } + + +def _run_hook( + event: dict | None = None, raw_stdin: str | None = None, tmp_dir=None +) -> tuple[str, int]: + """Run review_gate_post_hook.main() and return (stdout, exit_code).""" + from autoskillit.hooks.review_gate_post_hook import main # noqa: PLC0415 + + stdin_text = raw_stdin if raw_stdin is not None else json.dumps(event or {}) + + buf = io.StringIO() + exit_code = 0 + with contextlib.redirect_stdout(buf): + with unittest.mock.patch("sys.stdin", io.StringIO(stdin_text)): + with unittest.mock.patch( + "autoskillit.hooks.review_gate_post_hook.Path.cwd", return_value=tmp_dir + ): + try: + main() + except SystemExit as exc: + exit_code = exc.code if exc.code is not None else 0 + + return buf.getvalue(), exit_code + + +def _read_state(tmp_dir) -> dict | None: + state_path = tmp_dir / _STATE_FILE_RELPATH + if not state_path.exists(): + return None + return json.loads(state_path.read_text()) + + +# --------------------------------------------------------------------------- +# T2-1: LOOP_REQUIRED tag in run_skill output → state file written +# --------------------------------------------------------------------------- + + +def test_loop_required_tag_writes_state_file(tmp_path): + """T2-1: State file written with gate=LOOP_REQUIRED when tag present.""" + event = _build_run_skill_event( + f"verdict = changes_requested\n{_TAG_LOOP_REQUIRED}\n%%ORDER_UP%%" + ) + _run_hook(event, tmp_dir=tmp_path) + + state = _read_state(tmp_path) + assert state is not None, "State file must be written" + assert state["gate"] == "LOOP_REQUIRED" + assert state["check_review_loop_called"] is False + assert state["review_verdict"] == "changes_requested" + + +# --------------------------------------------------------------------------- +# T2-2: CLEAR tag in run_skill output → state file unlinked +# --------------------------------------------------------------------------- + + +def test_clear_tag_removes_state_file(tmp_path): + """T2-2: State file unlinked when CLEAR tag detected.""" + event = _build_run_skill_event(f"verdict = approved\n{_TAG_CLEAR}\n%%ORDER_UP%%") + _run_hook(event, tmp_dir=tmp_path) + + state = _read_state(tmp_path) + assert state is None, "State file must be removed on CLEAR" + + +# --------------------------------------------------------------------------- +# T2-3: CLEAR tag clears existing LOOP_REQUIRED state +# --------------------------------------------------------------------------- + + +def test_clear_tag_removes_existing_loop_required_state(tmp_path): + """T2-3: Transition from LOOP_REQUIRED to CLEAR removes state file.""" + # Pre-write existing state + state_path = tmp_path / _STATE_FILE_RELPATH + state_path.parent.mkdir(parents=True, exist_ok=True) + state_path.write_text( + json.dumps( + { + "gate": "LOOP_REQUIRED", + "review_verdict": "changes_requested", + "check_review_loop_called": False, + "pr_number": "1290", + "set_at": "2026-04-26T04:30:00+00:00", + } + ) + ) + + event = _build_run_skill_event(f"verdict = approved\n{_TAG_CLEAR}\n%%ORDER_UP%%") + _run_hook(event, tmp_dir=tmp_path) + + assert _read_state(tmp_path) is None + + +# --------------------------------------------------------------------------- +# T2-4: check_review_loop callable detected → state updated called=true +# --------------------------------------------------------------------------- + + +def test_check_review_loop_callable_marks_called(tmp_path): + """T2-4: Callable detection updates check_review_loop_called=True.""" + state_path = tmp_path / _STATE_FILE_RELPATH + state_path.parent.mkdir(parents=True, exist_ok=True) + state_path.write_text( + json.dumps( + { + "gate": "LOOP_REQUIRED", + "review_verdict": "changes_requested", + "check_review_loop_called": False, + "pr_number": "1290", + "set_at": "2026-04-26T04:30:00+00:00", + } + ) + ) + + event = _build_run_python_event(_CALLABLE_CHECK_REVIEW_LOOP) + _run_hook(event, tmp_dir=tmp_path) + + state = _read_state(tmp_path) + assert state is not None + assert state["check_review_loop_called"] is True + + +# --------------------------------------------------------------------------- +# T2-5: Non-review tool call → no state file created +# --------------------------------------------------------------------------- + + +def test_non_review_skill_call_creates_no_state_file(tmp_path): + """T2-5: No false positives — non-review tool call creates no state file.""" + event = _build_run_skill_event("plan_path = /tmp/plan.md\n%%ORDER_UP%%") + _run_hook(event, tmp_dir=tmp_path) + + assert _read_state(tmp_path) is None + + +# --------------------------------------------------------------------------- +# T2-6: Non-check_review_loop callable → no state change +# --------------------------------------------------------------------------- + + +def test_non_review_loop_callable_does_not_change_state(tmp_path): + """T2-6: Callable filtering — only check_review_loop triggers state update.""" + state_path = tmp_path / _STATE_FILE_RELPATH + state_path.parent.mkdir(parents=True, exist_ok=True) + original_state = json.dumps( + { + "gate": "LOOP_REQUIRED", + "review_verdict": "changes_requested", + "check_review_loop_called": False, + "pr_number": "1290", + "set_at": "2026-04-26T04:30:00+00:00", + } + ) + state_path.write_text(original_state) + + event = _build_run_python_event(_CALLABLE_OTHER) + _run_hook(event, tmp_dir=tmp_path) + + state = _read_state(tmp_path) + assert state is not None + assert state["check_review_loop_called"] is False + + +# --------------------------------------------------------------------------- +# T2-7: Malformed stdin → exit 0, no state file +# --------------------------------------------------------------------------- + + +def test_malformed_stdin_exits_cleanly_and_creates_no_state_file(tmp_path): + """T2-7: Fail-open on invalid JSON input.""" + _, exit_code = _run_hook(raw_stdin="not-json{{{", tmp_dir=tmp_path) + assert exit_code == 0 + assert _read_state(tmp_path) is None + + +# --------------------------------------------------------------------------- +# T2-8: Empty tool_response → no state file change +# --------------------------------------------------------------------------- + + +def test_empty_tool_response_creates_no_state_file(tmp_path): + """T2-8: Graceful handling of empty tool_response.""" + event = { + "tool_name": _TOOL_RUN_SKILL, + "tool_input": {"skill_command": "/review-pr feat/1290 main"}, + "tool_response": "", + } + _run_hook(event, tmp_dir=tmp_path) + + assert _read_state(tmp_path) is None + + +# --------------------------------------------------------------------------- +# T2-9: PR number extracted from run_python args +# --------------------------------------------------------------------------- + + +def test_pr_number_extracted_from_run_python_args(tmp_path): + """T2-9: pr_number captured from check_review_loop args.pr_number.""" + state_path = tmp_path / _STATE_FILE_RELPATH + state_path.parent.mkdir(parents=True, exist_ok=True) + state_path.write_text( + json.dumps( + { + "gate": "LOOP_REQUIRED", + "review_verdict": "changes_requested", + "check_review_loop_called": False, + "pr_number": "", + "set_at": "2026-04-26T04:30:00+00:00", + } + ) + ) + + event = _build_run_python_event( + _CALLABLE_CHECK_REVIEW_LOOP, + args={"pr_number": 1290, "cwd": "/tmp/work"}, + ) + _run_hook(event, tmp_dir=tmp_path) + + state = _read_state(tmp_path) + assert state is not None + assert state["pr_number"] == "1290" + + +# --------------------------------------------------------------------------- +# T2-10: approved_with_comments with no gate tag → no state write, no clear +# --------------------------------------------------------------------------- + + +def test_approved_with_comments_no_gate_tag_produces_no_state(tmp_path): + """T2-10: approved_with_comments emits no %%REVIEW_GATE:: tag — hook must be a no-op.""" + event = _build_run_skill_event("verdict = approved_with_comments\n%%ORDER_UP::abc%%") + + # Part 1: no state file is created when none existed + _run_hook(event, tmp_dir=tmp_path) + assert _read_state(tmp_path) is None, ( + "approved_with_comments with no gate tag must not write review_gate_state.json" + ) + + # Part 2: pre-existing state is not cleared (no gate tag means no state transition) + state_path = tmp_path / _STATE_FILE_RELPATH + state_path.parent.mkdir(parents=True, exist_ok=True) + existing = { + "gate": "LOOP_REQUIRED", + "review_verdict": "changes_requested", + "check_review_loop_called": False, + "pr_number": "1290", + "set_at": "2026-04-26T04:30:00+00:00", + } + state_path.write_text(json.dumps(existing)) + + _run_hook(event, tmp_dir=tmp_path) + state = _read_state(tmp_path) + assert state is not None, ( + "approved_with_comments must not clear an existing LOOP_REQUIRED state — " + "only an explicit %%REVIEW_GATE::CLEAR%% tag should do that" + ) + assert state == existing, ( + "approved_with_comments must leave the full gate state unchanged — " + f"expected {existing!r}, got {state!r}" + ) diff --git a/tests/hooks/test_review_loop_gate.py b/tests/hooks/test_review_loop_gate.py new file mode 100644 index 0000000000..a95f2a7fe5 --- /dev/null +++ b/tests/hooks/test_review_loop_gate.py @@ -0,0 +1,154 @@ +"""Tests for the review_loop_gate PreToolUse hook. + +Blocks wait_for_ci and enqueue_pr when review_pr returned changes_requested +but check_review_loop has not yet been called. + +Pattern mirrors test_pr_create_guard.py exactly. +""" + +from __future__ import annotations + +import contextlib +import io +import json +import unittest.mock + +import pytest + +pytestmark = [pytest.mark.layer("infra"), pytest.mark.small] + +_STATE_FILE_RELPATH = ".autoskillit/temp/review_gate_state.json" +_TOOL_WAIT_FOR_CI = "mcp__plugin_autoskillit_autoskillit__wait_for_ci" +_TOOL_ENQUEUE_PR = "mcp__plugin_autoskillit_autoskillit__enqueue_pr" + + +def _write_state(tmp_dir, gate: str, called: bool, pr_number: str = "1290") -> None: + state_path = tmp_dir / _STATE_FILE_RELPATH + state_path.parent.mkdir(parents=True, exist_ok=True) + state_path.write_text( + json.dumps( + { + "gate": gate, + "review_verdict": "changes_requested", + "check_review_loop_called": called, + "pr_number": pr_number, + "set_at": "2026-04-26T04:30:00+00:00", + } + ) + ) + + +def _run_gate(tool_name: str, tmp_dir, raw_stdin: str | None = None) -> str: + """Invoke review_loop_gate.main() and return captured stdout.""" + from autoskillit.hooks.guards.review_loop_gate import main # noqa: PLC0415 + + if raw_stdin is not None: + stdin_content = raw_stdin + else: + stdin_content = json.dumps({"tool_name": tool_name, "tool_input": {}}) + + buf = io.StringIO() + with contextlib.redirect_stdout(buf): + with unittest.mock.patch("sys.stdin", io.StringIO(stdin_content)): + with unittest.mock.patch( + "autoskillit.hooks.guards.review_loop_gate.Path.cwd", return_value=tmp_dir + ): + try: + main() + except SystemExit as exc: + assert exc.code == 0, f"Gate exited non-zero: {exc.code!r}" + + return buf.getvalue() + + +def _is_denied(output: str) -> bool: + if not output: + return False + data = json.loads(output) + return data.get("hookSpecificOutput", {}).get("permissionDecision") == "deny" + + +# --------------------------------------------------------------------------- +# T1-1: No state file → allow +# --------------------------------------------------------------------------- + + +class TestReviewLoopGateNoStateFile: + def test_allows_when_no_state_file(self, tmp_path): + """T1-1: Fail-open when state file does not exist.""" + out = _run_gate(_TOOL_WAIT_FOR_CI, tmp_path) + assert out.strip() == "", "No output means allow" + + +# --------------------------------------------------------------------------- +# T1-2 through T1-4: State file present +# --------------------------------------------------------------------------- + + +class TestReviewLoopGateWithStateFile: + def test_allows_when_gate_is_clear(self, tmp_path): + """T1-2: CLEAR gate state passes through.""" + _write_state(tmp_path, gate="CLEAR", called=False) + out = _run_gate(_TOOL_WAIT_FOR_CI, tmp_path) + assert out.strip() == "" + + def test_denies_when_loop_required_and_not_called(self, tmp_path): + """T1-3: Core enforcement — blocks merge path when loop not executed.""" + _write_state(tmp_path, gate="LOOP_REQUIRED", called=False) + out = _run_gate(_TOOL_WAIT_FOR_CI, tmp_path) + assert _is_denied(out) + + def test_allows_when_loop_required_and_called(self, tmp_path): + """T1-4: Loop execution satisfies gate.""" + _write_state(tmp_path, gate="LOOP_REQUIRED", called=True) + out = _run_gate(_TOOL_WAIT_FOR_CI, tmp_path) + assert out.strip() == "" + + +# --------------------------------------------------------------------------- +# T1-5 through T1-6: Edge cases +# --------------------------------------------------------------------------- + + +class TestReviewLoopGateEdgeCases: + def test_fails_open_on_malformed_state_file(self, tmp_path): + """T1-5: Fail-open on corrupted state JSON.""" + state_path = tmp_path / _STATE_FILE_RELPATH + state_path.parent.mkdir(parents=True, exist_ok=True) + state_path.write_text("not-json{{{") + out = _run_gate(_TOOL_WAIT_FOR_CI, tmp_path) + assert out.strip() == "" + + def test_deny_message_mentions_check_review_loop(self, tmp_path): + """T1-6: Deny reason contains actionable callable reference.""" + _write_state(tmp_path, gate="LOOP_REQUIRED", called=False) + out = _run_gate(_TOOL_WAIT_FOR_CI, tmp_path) + assert _is_denied(out) + data = json.loads(out) + reason = data["hookSpecificOutput"]["permissionDecisionReason"] + assert "check_review_loop" in reason + + def test_fails_open_on_malformed_stdin(self, tmp_path): + """T1-9: Fail-open when stdin is not valid JSON.""" + _write_state(tmp_path, gate="LOOP_REQUIRED", called=False) + out = _run_gate(_TOOL_WAIT_FOR_CI, tmp_path, raw_stdin="not-json{{{") + assert out.strip() == "" + + +# --------------------------------------------------------------------------- +# T1-7 through T1-8: Matcher coverage +# --------------------------------------------------------------------------- + + +class TestReviewLoopGateMatcherCoverage: + def test_fires_on_wait_for_ci(self, tmp_path): + """T1-7: Gate fires when tool_name matches wait_for_ci.""" + _write_state(tmp_path, gate="LOOP_REQUIRED", called=False) + out = _run_gate(_TOOL_WAIT_FOR_CI, tmp_path) + assert _is_denied(out) + + def test_fires_on_enqueue_pr(self, tmp_path): + """T1-8: Gate fires when tool_name matches enqueue_pr.""" + _write_state(tmp_path, gate="LOOP_REQUIRED", called=False) + out = _run_gate(_TOOL_ENQUEUE_PR, tmp_path) + assert _is_denied(out) diff --git a/tests/hooks/test_session_start_reminder.py b/tests/hooks/test_session_start_reminder.py new file mode 100644 index 0000000000..c18ca80a4b --- /dev/null +++ b/tests/hooks/test_session_start_reminder.py @@ -0,0 +1,293 @@ +"""Tests for the SessionStart hook — session_start_hook.py.""" + +from __future__ import annotations + +import json +import os +import subprocess +import sys +from datetime import UTC, datetime, timedelta +from pathlib import Path + +SCRIPT = Path(__file__).resolve().parents[2] / "src/autoskillit/hooks/session_start_hook.py" + + +def _run(stdin_data: str, env: dict | None = None) -> tuple[int, str]: + result = subprocess.run( + [sys.executable, str(SCRIPT)], + input=stdin_data, + capture_output=True, + text=True, + env=env, + ) + return result.returncode, result.stdout + + +def _write_marker( + marker_dir: Path, session_id: str, recipe_name: object, *, fresh: bool = True +) -> None: + opened_at = datetime.now(UTC) if fresh else datetime.now(UTC) - timedelta(hours=25) + (marker_dir / f"{session_id}.json").write_text( + json.dumps( + { + "session_id": session_id, + "opened_at": opened_at.isoformat(), + "recipe_name": recipe_name, + "marker_version": 1, + } + ) + ) + + +# REQ-HOOK-002, REQ-HOOK-003 +def test_fresh_session_no_output(tmp_path: Path) -> None: + """Empty transcript_path → no additionalContext injected.""" + transcript = tmp_path / "session.jsonl" + transcript.write_text("") + payload = json.dumps({"session_id": "abc", "transcript_path": str(transcript)}) + rc, out = _run(payload) + assert rc == 0 + assert "additionalContext" not in out + + +def test_resumed_session_injects_context(tmp_path: Path) -> None: + """Non-empty transcript → additionalContext with open-kitchen reminder.""" + transcript = tmp_path / "session.jsonl" + transcript.write_text('{"type":"say","text":"hello"}\n') + payload = json.dumps({"session_id": "abc", "transcript_path": str(transcript)}) + env = {**os.environ, "AUTOSKILLIT_STATE_DIR": str(tmp_path / "empty_state")} + rc, out = _run(payload, env=env) + assert rc == 0 + data = json.loads(out.strip()) + assert "additionalContext" in data + assert "open-kitchen" in data["additionalContext"] + + +def test_missing_transcript_path_no_crash(tmp_path: Path) -> None: + """Missing transcript_path key → fail-open, no output, exit 0.""" + payload = json.dumps({"session_id": "abc"}) + rc, out = _run(payload) + assert rc == 0 + + +def test_nonexistent_transcript_no_crash(tmp_path: Path) -> None: + """transcript_path pointing to non-existent file → fail-open, no output.""" + payload = json.dumps({"session_id": "abc", "transcript_path": "/nonexistent/path.jsonl"}) + rc, out = _run(payload) + assert rc == 0 + assert "additionalContext" not in out + + +def test_session_start_sweeps_stale_markers(tmp_path: Path) -> None: + import json + from datetime import datetime + + marker_dir = tmp_path / "kitchen_state" + marker_dir.mkdir(parents=True) + stale = marker_dir / "old-session.json" + stale.write_text( + json.dumps( + { + "session_id": "old-session", + "opened_at": "2020-01-01T00:00:00+00:00", + "recipe_name": None, + "marker_version": 1, + } + ) + ) + fresh = marker_dir / "new-session.json" + fresh.write_text( + json.dumps( + { + "session_id": "new-session", + "opened_at": datetime.now(UTC).isoformat(), + "recipe_name": None, + "marker_version": 1, + } + ) + ) + transcript = tmp_path / "transcript.jsonl" + transcript.write_text("") + payload = json.dumps( + { + "session_id": "new-session", + "transcript_path": str(transcript), + "autoskillit_state_dir": str(tmp_path), + } + ) + import subprocess + import sys + + result = subprocess.run( + [sys.executable, str(SCRIPT)], + input=payload, + capture_output=True, + text=True, + env={**__import__("os").environ, "AUTOSKILLIT_STATE_DIR": str(tmp_path)}, + ) + assert result.returncode == 0 + assert not stale.exists(), "Stale marker should have been swept" + assert fresh.exists(), "Fresh marker must survive" + + +def test_resumed_session_includes_recipe_name_from_fresh_marker(tmp_path: Path) -> None: + """Fresh marker with recipe_name → name appears in additionalContext.""" + marker_dir = tmp_path / "state" / "kitchen_state" + marker_dir.mkdir(parents=True) + _write_marker(marker_dir, "sess-1", "my-recipe", fresh=True) + transcript = tmp_path / "transcript.jsonl" + transcript.write_text('{"type":"say","text":"hello"}\n') + payload = json.dumps({"session_id": "sess-1", "transcript_path": str(transcript)}) + env = {**os.environ, "AUTOSKILLIT_STATE_DIR": str(tmp_path / "state")} + rc, out = _run(payload, env=env) + assert rc == 0 + data = json.loads(out.strip()) + assert "my-recipe" in data["additionalContext"] + + +def test_resumed_session_no_recipe_name_when_marker_has_none(tmp_path: Path) -> None: + """Fresh marker with recipe_name=None → generic reminder, no recipe name.""" + marker_dir = tmp_path / "state" / "kitchen_state" + marker_dir.mkdir(parents=True) + _write_marker(marker_dir, "sess-2", None, fresh=True) + transcript = tmp_path / "transcript.jsonl" + transcript.write_text('{"type":"say","text":"hello"}\n') + payload = json.dumps({"session_id": "sess-2", "transcript_path": str(transcript)}) + env = {**os.environ, "AUTOSKILLIT_STATE_DIR": str(tmp_path / "state")} + rc, out = _run(payload, env=env) + assert rc == 0 + data = json.loads(out.strip()) + assert "recipe" not in data["additionalContext"] + + +def test_resumed_session_no_recipe_name_when_no_markers_exist(tmp_path: Path) -> None: + """No marker files → additionalContext present but no recipe name.""" + (tmp_path / "state" / "kitchen_state").mkdir(parents=True) + transcript = tmp_path / "transcript.jsonl" + transcript.write_text('{"type":"say","text":"hello"}\n') + payload = json.dumps({"session_id": "sess-3", "transcript_path": str(transcript)}) + env = {**os.environ, "AUTOSKILLIT_STATE_DIR": str(tmp_path / "state")} + rc, out = _run(payload, env=env) + assert rc == 0 + data = json.loads(out.strip()) + assert "additionalContext" in data + assert "recipe" not in data["additionalContext"] + + +def test_resumed_session_picks_most_recent_fresh_marker(tmp_path: Path) -> None: + """Two fresh markers → most recent recipe_name wins.""" + marker_dir = tmp_path / "state" / "kitchen_state" + marker_dir.mkdir(parents=True) + # Write older marker first + older_at = datetime.now(UTC) - timedelta(seconds=10) + (marker_dir / "old-sess.json").write_text( + json.dumps( + { + "session_id": "old-sess", + "opened_at": older_at.isoformat(), + "recipe_name": "old-recipe", + "marker_version": 1, + } + ) + ) + # Write newer marker + newer_at = datetime.now(UTC) + (marker_dir / "new-sess.json").write_text( + json.dumps( + { + "session_id": "new-sess", + "opened_at": newer_at.isoformat(), + "recipe_name": "new-recipe", + "marker_version": 1, + } + ) + ) + transcript = tmp_path / "transcript.jsonl" + transcript.write_text('{"type":"say","text":"hello"}\n') + payload = json.dumps({"session_id": "new-sess", "transcript_path": str(transcript)}) + env = {**os.environ, "AUTOSKILLIT_STATE_DIR": str(tmp_path / "state")} + rc, out = _run(payload, env=env) + assert rc == 0 + data = json.loads(out.strip()) + assert "new-recipe" in data["additionalContext"] + assert "old-recipe" not in data["additionalContext"] + + +def test_resumed_session_ignores_stale_marker_recipe_name(tmp_path: Path) -> None: + """Stale marker recipe_name must not appear in additionalContext.""" + marker_dir = tmp_path / "state" / "kitchen_state" + marker_dir.mkdir(parents=True) + _write_marker(marker_dir, "stale-sess", "stale-recipe", fresh=False) + transcript = tmp_path / "transcript.jsonl" + transcript.write_text('{"type":"say","text":"hello"}\n') + payload = json.dumps({"session_id": "stale-sess", "transcript_path": str(transcript)}) + env = {**os.environ, "AUTOSKILLIT_STATE_DIR": str(tmp_path / "state")} + rc, out = _run(payload, env=env) + assert rc == 0 + data = json.loads(out.strip()) + assert "stale-recipe" not in data["additionalContext"] + + +def test_fresh_session_not_affected_by_markers(tmp_path: Path) -> None: + """Fresh session (empty transcript) stays silent even with a marker present.""" + marker_dir = tmp_path / "state" / "kitchen_state" + marker_dir.mkdir(parents=True) + _write_marker(marker_dir, "sess-6", "some-recipe", fresh=True) + transcript = tmp_path / "transcript.jsonl" + transcript.write_text("") + payload = json.dumps({"session_id": "sess-6", "transcript_path": str(transcript)}) + env = {**os.environ, "AUTOSKILLIT_STATE_DIR": str(tmp_path / "state")} + rc, out = _run(payload, env=env) + assert rc == 0 + assert "additionalContext" not in out + + +def test_resumed_session_marker_dir_missing_no_crash(tmp_path: Path) -> None: + """Missing AUTOSKILLIT_STATE_DIR path → exit 0, generic reminder, no crash.""" + transcript = tmp_path / "transcript.jsonl" + transcript.write_text('{"type":"say","text":"hello"}\n') + payload = json.dumps({"session_id": "sess-7", "transcript_path": str(transcript)}) + env = {**os.environ, "AUTOSKILLIT_STATE_DIR": str(tmp_path / "nonexistent")} + rc, out = _run(payload, env=env) + assert rc == 0 + data = json.loads(out.strip()) + assert "additionalContext" in data + assert "recipe" not in data["additionalContext"] + + +# --- Group P-3: Hook namespacing --- + + +def test_session_start_hook_resolves_campaign_namespace(tmp_path: Path, monkeypatch) -> None: + """session_start_hook inline path logic includes AUTOSKILLIT_CAMPAIGN_ID.""" + monkeypatch.delenv("AUTOSKILLIT_STATE_DIR", raising=False) + monkeypatch.setenv("AUTOSKILLIT_CAMPAIGN_ID", "camp-88") + + # Create a marker in the campaign-namespaced directory + state_dir = tmp_path / ".autoskillit" / "temp" / "kitchen_state" / "camp-88" + state_dir.mkdir(parents=True) + marker = { + "session_id": "sess-88", + "opened_at": datetime.now(UTC).isoformat(), + "recipe_name": None, + "marker_version": 1, + } + (state_dir / "sess-88.json").write_text(json.dumps(marker)) + + # Run the hook with the campaign-namespaced marker and a resumed transcript + transcript = tmp_path / "transcript.jsonl" + transcript.write_text('{"type":"say","text":"hello"}\n') + payload = json.dumps({"session_id": "sess-88", "transcript_path": str(transcript)}) + env = { + **os.environ, + "AUTOSKILLIT_CAMPAIGN_ID": "camp-88", + } + # Remove AUTOSKILLIT_STATE_DIR so the hook uses the campaign-namespaced path + env.pop("AUTOSKILLIT_STATE_DIR", None) + # Use monkeypatch chdir so the hook resolves CWD correctly + monkeypatch.chdir(tmp_path) + rc, out = _run(payload, env=env) + assert rc == 0 + # The hook should find the marker and produce a resume reminder + data = json.loads(out.strip()) + assert "additionalContext" in data diff --git a/tests/hooks/test_skill_load_post_hook.py b/tests/hooks/test_skill_load_post_hook.py new file mode 100644 index 0000000000..68d8583ebe --- /dev/null +++ b/tests/hooks/test_skill_load_post_hook.py @@ -0,0 +1,216 @@ +"""Tests for skill_load_post_hook.py PostToolUse hook.""" + +from __future__ import annotations + +import contextlib +import io +import json +import os +import unittest.mock +from pathlib import Path +from unittest.mock import patch + +import pytest + +pytestmark = [pytest.mark.layer("infra"), pytest.mark.small] + +_FLAG_RELPATH = ".autoskillit/temp/skill_guard_abc123.flag" + + +def _run_hook( + *, + stdin_data: dict | str, + tmp_dir: Path, + provider_profile: str | None = None, +) -> tuple[str, int]: + """Run skill_load_post_hook.main(), return (stdout, exit_code).""" + from autoskillit.hooks.skill_load_post_hook import main # noqa: PLC0415 + + stdin_content = stdin_data if isinstance(stdin_data, str) else json.dumps(stdin_data) + + env_base = {k: v for k, v in os.environ.items() if k != "AUTOSKILLIT_PROVIDER_PROFILE"} + if provider_profile is not None: + env_base["AUTOSKILLIT_PROVIDER_PROFILE"] = provider_profile + + buf = io.StringIO() + exit_code = 0 + with ( + patch.dict(os.environ, env_base, clear=True), + contextlib.redirect_stdout(buf), + unittest.mock.patch("sys.stdin", io.StringIO(stdin_content)), + unittest.mock.patch( + "autoskillit.hooks.skill_load_post_hook.Path.cwd", return_value=tmp_dir + ), + ): + try: + main() + except SystemExit as exc: + exit_code = int(exc.code) if exc.code is not None else 0 + + return buf.getvalue(), exit_code + + +def _make_skill_event( + session_id: str = "abc123", + skill: str = "implement-worktree-no-merge", + agent_id: str | None = None, +) -> dict: + event = { + "tool_name": "Skill", + "tool_input": {"skill": skill}, + "session_id": session_id, + } + if agent_id is not None: + event["agent_id"] = agent_id + return event + + +def test_writes_flag_when_provider_profile_set(tmp_path: Path) -> None: + """T1-1: Flag file written with skill name when provider profile is set.""" + (tmp_path / ".autoskillit").mkdir(parents=True) + _run_hook( + stdin_data=_make_skill_event(), + tmp_dir=tmp_path, + provider_profile="minimax", + ) + flag = tmp_path / _FLAG_RELPATH + assert flag.exists(), "Flag file must be written" + assert "implement-worktree-no-merge" in flag.read_text() + + +def test_skips_when_provider_profile_empty(tmp_path: Path) -> None: + """T1-2: No flag file when provider profile is not set.""" + _run_hook( + stdin_data=_make_skill_event(), + tmp_dir=tmp_path, + provider_profile=None, + ) + flag = tmp_path / _FLAG_RELPATH + assert not flag.exists(), "Flag file must NOT be created when provider profile is empty" + + +def test_skips_for_non_skill_tool(tmp_path: Path) -> None: + """T1-3: No flag file for non-Skill tool.""" + event = { + "tool_name": "Read", + "tool_input": {"file_path": "/foo"}, + "session_id": "abc123", + } + _run_hook( + stdin_data=event, + tmp_dir=tmp_path, + provider_profile="minimax", + ) + flag = tmp_path / _FLAG_RELPATH + assert not flag.exists() + + +def test_survives_malformed_stdin(tmp_path: Path) -> None: + """T1-4: Exit 0 on malformed JSON.""" + _, exit_code = _run_hook( + stdin_data="not valid json", + tmp_dir=tmp_path, + provider_profile="minimax", + ) + assert exit_code == 0 + + +def test_skips_when_session_id_absent(tmp_path: Path) -> None: + """T1-5: No flag file when session_id is missing.""" + event = { + "tool_name": "Skill", + "tool_input": {"skill": "make-plan"}, + } + _run_hook( + stdin_data=event, + tmp_dir=tmp_path, + provider_profile="minimax", + ) + flag_dir = tmp_path / ".autoskillit" / "temp" + if flag_dir.exists(): + flags = list(flag_dir.glob("skill_guard_*.flag")) + assert not flags, "No flag file should be created when session_id is absent" + + +def _run_hook_with_marker( + *, + stdin_data: dict | str, + tmp_dir: Path, + provider_profile: str | None = None, + completion_marker: str | None = None, +) -> tuple[str, int]: + """Run skill_load_post_hook.main() with AUTOSKILLIT_COMPLETION_MARKER support.""" + from autoskillit.hooks.skill_load_post_hook import main # noqa: PLC0415 + + stdin_content = stdin_data if isinstance(stdin_data, str) else json.dumps(stdin_data) + + env_base = { + k: v + for k, v in os.environ.items() + if k not in ("AUTOSKILLIT_PROVIDER_PROFILE", "AUTOSKILLIT_COMPLETION_MARKER") + } + if provider_profile is not None: + env_base["AUTOSKILLIT_PROVIDER_PROFILE"] = provider_profile + if completion_marker is not None: + env_base["AUTOSKILLIT_COMPLETION_MARKER"] = completion_marker + + buf = io.StringIO() + exit_code = 0 + with ( + patch.dict(os.environ, env_base, clear=True), + contextlib.redirect_stdout(buf), + unittest.mock.patch("sys.stdin", io.StringIO(stdin_content)), + unittest.mock.patch( + "autoskillit.hooks.skill_load_post_hook.Path.cwd", return_value=tmp_dir + ), + ): + try: + main() + except SystemExit as exc: + exit_code = int(exc.code) if exc.code is not None else 0 + + return buf.getvalue(), exit_code + + +def test_emits_additional_context_when_completion_marker_set(tmp_path: Path) -> None: + """T1-6: When AUTOSKILLIT_COMPLETION_MARKER is set, hook emits additionalContext JSON.""" + marker = "%%ORDER_UP::abc12345%%" + stdout, exit_code = _run_hook_with_marker( + stdin_data=_make_skill_event(), + tmp_dir=tmp_path, + provider_profile="minimax", + completion_marker=marker, + ) + assert exit_code == 0 + assert stdout.strip(), "Hook must emit additionalContext to stdout" + payload = json.loads(stdout) + assert "additionalContext" in payload + assert marker in payload["additionalContext"] + + +def test_skips_flag_write_when_agent_id_present(tmp_path: Path) -> None: + """T1-7: No flag written when agent_id is present (subagent context).""" + _run_hook( + stdin_data=_make_skill_event(agent_id="agent-uuid-123"), + tmp_dir=tmp_path, + provider_profile="minimax", + ) + flag_dir = tmp_path / ".autoskillit" / "temp" + flags = list(flag_dir.glob("skill_guard_*.flag")) + assert not flags, "No flag file should be created in subagent context" + + +def test_writes_flag_to_project_root_via_ancestor_walk(tmp_path: Path) -> None: + """T1-8: Flag written to project root when CWD is a subdirectory.""" + project = tmp_path / "project" + (project / ".autoskillit").mkdir(parents=True) + + deep_cwd = project / "sub" / "deep" + _run_hook( + stdin_data=_make_skill_event(), + tmp_dir=deep_cwd, + provider_profile="minimax", + ) + flag = project / ".autoskillit" / "temp" / "skill_guard_abc123.flag" + assert flag.exists(), "Flag must be written to project root, not CWD" + assert "implement-worktree-no-merge" in flag.read_text() diff --git a/tests/infra/test_token_summary_appender.py b/tests/hooks/test_token_summary_appender.py similarity index 75% rename from tests/infra/test_token_summary_appender.py rename to tests/hooks/test_token_summary_appender.py index 0ae7465395..5d4a0c793f 100644 --- a/tests/infra/test_token_summary_appender.py +++ b/tests/hooks/test_token_summary_appender.py @@ -6,94 +6,18 @@ from __future__ import annotations -import io import json import subprocess import sys -from contextlib import ExitStack, redirect_stdout from pathlib import Path from unittest.mock import MagicMock, patch -from autoskillit.hooks._fmt_primitives import _HOOK_CONFIG_PATH_COMPONENTS - - -def _run_hook( - event: dict | None = None, - raw_stdin: str | None = None, - log_root: Path | None = None, - hook_config_path: Path | None = None, -) -> tuple[str, int]: - """Run token_summary_appender.main() with synthetic stdin. - - Returns (stdout_output, exit_code). - - Args: - hook_config_path: Path to a hook config JSON file containing ``kitchen_id`` - (or legacy ``pipeline_id``). - When provided, patches ``_read_kitchen_id`` to return the ``kitchen_id`` - value from that file. When absent, ``_read_kitchen_id`` reads from the - real filesystem (returns '' if no file present in the test CWD). - """ - from autoskillit.hooks.token_summary_hook import main - - stdin_text = raw_stdin if raw_stdin is not None else json.dumps(event or {}) - exit_code = 0 - buf = io.StringIO() - - with ExitStack() as stack: - stack.enter_context(patch("sys.stdin", io.StringIO(stdin_text))) - stack.enter_context(redirect_stdout(buf)) - if log_root is not None: - stack.enter_context( - patch( - "autoskillit.hooks.token_summary_hook._log_root", - return_value=log_root, - ) - ) - if hook_config_path is not None: - cfg_data = json.loads(hook_config_path.read_text(encoding="utf-8")) - kitchen_id = cfg_data.get("kitchen_id") or cfg_data.get("pipeline_id", "") - stack.enter_context( - patch( - "autoskillit.hooks.token_summary_hook._read_kitchen_id", - return_value=kitchen_id, - ) - ) - try: - main() - except SystemExit as e: - exit_code = e.code if e.code is not None else 0 - - return buf.getvalue(), exit_code - - -def _make_run_skill_event(result_text: str = "Done.\n%%ORDER_UP%%") -> dict: - """Create a double-wrapped PostToolUse event for run_skill.""" - inner = {"result": result_text, "success": True} - outer = {"result": json.dumps(inner)} - return { - "tool_name": "mcp__autoskillit_server__run_skill", - "tool_response": json.dumps(outer), - } - - -def _write_sessions(log_root: Path, entries: list[dict]) -> None: - """Write sessions.jsonl and token_usage.json files for test setup.""" - (log_root / "sessions.jsonl").write_text("\n".join(json.dumps(e) for e in entries) + "\n") - for entry in entries: - dir_name = entry["dir_name"] - session_dir = log_root / "sessions" / dir_name - session_dir.mkdir(parents=True, exist_ok=True) - token_data = { - "step_name": entry.get("step_name", "unknown"), - "input_tokens": entry.get("input_tokens", 1000), - "output_tokens": entry.get("output_tokens", 500), - "cache_creation_input_tokens": entry.get("cache_creation_input_tokens", 100), - "cache_read_input_tokens": entry.get("cache_read_input_tokens", 200), - "timing_seconds": entry.get("timing_seconds", 10.0), - } - (session_dir / "token_usage.json").write_text(json.dumps(token_data)) - +from autoskillit.hooks.formatters._fmt_primitives import _HOOK_CONFIG_PATH_COMPONENTS +from tests.infra._token_summary_helpers import ( + _make_run_skill_event, + _run_hook, + _write_sessions, +) # --------------------------------------------------------------------------- # TSA-1: hook script exists on disk @@ -664,7 +588,7 @@ def test_e1_order_id_isolation_multi_issue_session(tmp_path: Path) -> None: (d / "token_usage.json").write_text( json.dumps( { - "step_name": step, + "session_label": step, "input_tokens": 100, "output_tokens": 50, "cache_creation_input_tokens": 0, @@ -777,7 +701,7 @@ def test_e3_backward_compat_sessions_without_order_id_field(tmp_path: Path) -> N (d / "token_usage.json").write_text( json.dumps( { - "step_name": "plan", + "session_label": "plan", "input_tokens": 100, "output_tokens": 50, "cache_creation_input_tokens": 0, @@ -889,6 +813,89 @@ def test_e4_kitchen_id_renamed_in_hook_config(tmp_path: Path) -> None: assert result == "legacy-pipeline-uuid" +# T9 +def test_non_anthropic_step_marked_in_format_table() -> None: + """Hook _format_table marks non-Anthropic step names with * and adds footnote.""" + from autoskillit.hooks.token_summary_hook import _format_table + + aggregated = { + "implement": { + "step_name": "implement", + "model": "MiniMax-M2.7-highspeed", + "input_tokens": 307600, + "output_tokens": 3400, + "cache_read_input_tokens": 0, + "cache_creation_input_tokens": 0, + "invocation_count": 1, + "elapsed_seconds": 60.0, + "peak_context": 0, + "turn_count": 0, + }, + } + result = _format_table(aggregated) + assert "| implement* |" in result + assert "non-Anthropic provider" in result + + +# --------------------------------------------------------------------------- +# _unwrap_mcp_response unit tests +# --------------------------------------------------------------------------- + + +class TestUnwrapMcpResponse: + """Unit tests for the shared double-unwrap helper.""" + + def _call(self, tool_name: str, raw: str): + from autoskillit.hooks.token_summary_hook import _unwrap_mcp_response + + return _unwrap_mcp_response(tool_name, raw) + + def test_invalid_json_returns_none(self): + assert self._call("mcp__x__y", "not json") is None + + def test_json_array_returns_none(self): + import json + + assert self._call("mcp__x__y", json.dumps([1, 2, 3])) is None + + def test_non_mcp_tool_returns_outer_dict(self): + import json + + payload = {"result": "some text", "success": True} + result = self._call("run_python", json.dumps(payload)) + assert result == payload + + def test_mcp_tool_double_wrapped_returns_inner(self): + import json + + inner = {"result": "https://github.com/o/r/pull/1", "order_id": "abc"} + outer = {"result": json.dumps(inner)} + result = self._call("mcp__autoskillit__run_skill", json.dumps(outer)) + assert result == inner + + def test_mcp_tool_inner_parse_fails_returns_outer(self): + import json + + outer = {"result": "not-json-inner"} + result = self._call("mcp__autoskillit__run_skill", json.dumps(outer)) + assert result == outer + + def test_mcp_tool_inner_not_dict_returns_outer(self): + import json + + outer = {"result": json.dumps([1, 2, 3])} + result = self._call("mcp__autoskillit__run_skill", json.dumps(outer)) + assert result == outer + + def test_mcp_tool_extra_keys_skips_double_unwrap(self): + """Outer dict with more than just 'result' key → not a double-wrapped response.""" + import json + + outer = {"result": json.dumps({"foo": "bar"}), "extra": "key"} + result = self._call("mcp__autoskillit__run_skill", json.dumps(outer)) + assert result == outer + + def test_hook_subprocess_calls_have_timeout() -> None: """All subprocess.run() calls in token_summary_hook.py must have timeout=.""" import ast @@ -905,3 +912,185 @@ def test_hook_subprocess_calls_have_timeout() -> None: assert "timeout" in kw_names, ( f"subprocess.run() at line {node.lineno} in token_summary_hook.py missing timeout=" ) + + +# --------------------------------------------------------------------------- +# T-HOOK-EFF: Token Efficiency table in hook output +# --------------------------------------------------------------------------- + + +# T-HOOK-EFF-1 +def test_efficiency_table_appended_when_loc_present(tmp_path: Path) -> None: + """Hook appends efficiency table after token table when LoC data exists.""" + log_root = tmp_path / "logs" + log_root.mkdir() + kitchen_id = "kitchen-eff-1" + + _write_sessions( + log_root, + [ + { + "dir_name": "s1", + "cwd": "/w", + "kitchen_id": kitchen_id, + "step_name": "implement", + "input_tokens": 1000, + "output_tokens": 500, + "cache_creation_input_tokens": 100, + "cache_read_input_tokens": 200, + "timing_seconds": 10.0, + "loc_insertions": 80, + "loc_deletions": 20, + } + ], + ) + + hook_config = tmp_path / ".autoskillit_hook_config.json" + hook_config.write_text(json.dumps({"kitchen_id": kitchen_id})) + pr_url = "https://github.com/owner/repo/pull/42" + event = _make_run_skill_event(f"pr_url={pr_url}\n%%ORDER_UP%%") + + view_result = MagicMock(returncode=0, stdout="Existing PR body.") + edit_calls: list = [] + + def run_side(args, **kwargs): + if "api" in args and "--method" not in args: + return view_result + if "api" in args and "--method" in args: + edit_calls.append(args) + return MagicMock(returncode=0) + return MagicMock(returncode=0) + + with patch("subprocess.run", side_effect=run_side): + _, exit_code = _run_hook(event, log_root=log_root, hook_config_path=hook_config) + + assert exit_code == 0 + assert len(edit_calls) == 1 + body_arg = edit_calls[0][edit_calls[0].index("--raw-field") + 1] + body_content = body_arg[len("body=") :] + assert "## Token Efficiency" in body_content + assert "cache_read/LoC" in body_content + assert "cache_write/LoC" in body_content + assert "output/LoC" in body_content + assert "peak_ctx/LoC" not in body_content + eff_section = body_content[body_content.index("## Token Efficiency") :] + eff_header = eff_section.split("\n")[2] + assert eff_header.count("|") == 6 # 5 columns + outer pipes + + +# T-HOOK-EFF-2 +def test_efficiency_table_omitted_when_no_loc_data(tmp_path: Path) -> None: + """Hook omits efficiency table when all sessions have loc_insertions=loc_deletions=0.""" + log_root = tmp_path / "logs" + log_root.mkdir() + kitchen_id = "kitchen-eff-2" + + _write_sessions( + log_root, + [ + { + "dir_name": "s1", + "cwd": "/w", + "kitchen_id": kitchen_id, + "step_name": "plan", + "input_tokens": 1000, + "output_tokens": 500, + "cache_creation_input_tokens": 100, + "cache_read_input_tokens": 200, + "timing_seconds": 10.0, + "loc_insertions": 0, + "loc_deletions": 0, + } + ], + ) + + hook_config = tmp_path / ".autoskillit_hook_config.json" + hook_config.write_text(json.dumps({"kitchen_id": kitchen_id})) + pr_url = "https://github.com/owner/repo/pull/42" + event = _make_run_skill_event(f"pr_url={pr_url}\n%%ORDER_UP%%") + + view_result = MagicMock(returncode=0, stdout="Existing PR body.") + edit_calls: list = [] + + def run_side(args, **kwargs): + if "api" in args and "--method" not in args: + return view_result + if "api" in args and "--method" in args: + edit_calls.append(args) + return MagicMock(returncode=0) + return MagicMock(returncode=0) + + with patch("subprocess.run", side_effect=run_side): + _, exit_code = _run_hook(event, log_root=log_root, hook_config_path=hook_config) + + assert exit_code == 0 + assert len(edit_calls) == 1 + body_arg = edit_calls[0][edit_calls[0].index("--raw-field") + 1] + body_content = body_arg[len("body=") :] + assert "## Token Efficiency" not in body_content + + +# T-HOOK-EFF-3 +def test_efficiency_table_zero_loc_step_shows_dash(tmp_path: Path) -> None: + """Steps with zero LoC changed show — in the hook-generated efficiency table.""" + log_root = tmp_path / "logs" + log_root.mkdir() + kitchen_id = "kitchen-eff-3" + + _write_sessions( + log_root, + [ + { + "dir_name": "s1", + "cwd": "/w", + "kitchen_id": kitchen_id, + "step_name": "plan", + "input_tokens": 500, + "output_tokens": 100, + "cache_creation_input_tokens": 50, + "cache_read_input_tokens": 100, + "timing_seconds": 5.0, + "loc_insertions": 0, + "loc_deletions": 0, + }, + { + "dir_name": "s2", + "cwd": "/w", + "kitchen_id": kitchen_id, + "step_name": "implement", + "input_tokens": 1000, + "output_tokens": 500, + "cache_creation_input_tokens": 100, + "cache_read_input_tokens": 200, + "timing_seconds": 10.0, + "loc_insertions": 50, + "loc_deletions": 10, + }, + ], + ) + + hook_config = tmp_path / ".autoskillit_hook_config.json" + hook_config.write_text(json.dumps({"kitchen_id": kitchen_id})) + pr_url = "https://github.com/owner/repo/pull/42" + event = _make_run_skill_event(f"pr_url={pr_url}\n%%ORDER_UP%%") + + view_result = MagicMock(returncode=0, stdout="Existing PR body.") + edit_calls: list = [] + + def run_side(args, **kwargs): + if "api" in args and "--method" not in args: + return view_result + if "api" in args and "--method" in args: + edit_calls.append(args) + return MagicMock(returncode=0) + return MagicMock(returncode=0) + + with patch("subprocess.run", side_effect=run_side): + _, exit_code = _run_hook(event, log_root=log_root, hook_config_path=hook_config) + + assert exit_code == 0 + assert len(edit_calls) == 1 + body_arg = edit_calls[0][edit_calls[0].index("--raw-field") + 1] + body_content = body_arg[len("body=") :] + assert "## Token Efficiency" in body_content + assert "—" in body_content diff --git a/tests/hooks/test_write_guard.py b/tests/hooks/test_write_guard.py new file mode 100644 index 0000000000..615b5aff66 --- /dev/null +++ b/tests/hooks/test_write_guard.py @@ -0,0 +1,130 @@ +"""Tests for write_guard.py PreToolUse hook.""" + +from __future__ import annotations + +import io +import json +from contextlib import redirect_stdout +from unittest.mock import patch + +import pytest + +pytestmark = [pytest.mark.layer("infra"), pytest.mark.small] + + +def _build_event(tool_name: str, file_path: str) -> dict: + return {"tool_name": tool_name, "tool_input": {"file_path": file_path}} + + +def _run_hook(event: dict | str) -> str: + from autoskillit.hooks.guards.write_guard import main + + stdin_text = json.dumps(event) if isinstance(event, dict) else event + buf = io.StringIO() + with ( + patch("sys.stdin", io.StringIO(stdin_text)), + redirect_stdout(buf), + ): + try: + main() + except SystemExit: + pass + return buf.getvalue() + + +def _set_headless(monkeypatch: pytest.MonkeyPatch, *, headless: bool) -> None: + if headless: + monkeypatch.setenv("AUTOSKILLIT_HEADLESS", "1") + else: + monkeypatch.delenv("AUTOSKILLIT_HEADLESS", raising=False) + + +class TestWriteGuardNoHeadless: + def test_no_headless_env_allows_all_writes(self, monkeypatch: pytest.MonkeyPatch): + _set_headless(monkeypatch, headless=False) + monkeypatch.setenv("AUTOSKILLIT_ALLOWED_WRITE_PREFIX", "/clone/.autoskillit/temp/") + result = _run_hook(_build_event("Write", "/clone/src/foo.py")) + assert result == "" + + +class TestWriteGuardNoEnv: + def test_no_env_var_allows_all_writes(self, monkeypatch: pytest.MonkeyPatch): + _set_headless(monkeypatch, headless=True) + monkeypatch.delenv("AUTOSKILLIT_ALLOWED_WRITE_PREFIX", raising=False) + result = _run_hook(_build_event("Write", "/src/foo.py")) + assert result == "" + + def test_no_json_allows_when_no_prefix(self, monkeypatch: pytest.MonkeyPatch): + _set_headless(monkeypatch, headless=True) + monkeypatch.delenv("AUTOSKILLIT_ALLOWED_WRITE_PREFIX", raising=False) + result = _run_hook("not json at all") + assert result == "" + + +class TestWriteGuardWithPrefix: + PREFIX = "/clone/.autoskillit/temp/investigate/" + + @pytest.fixture(autouse=True) + def _enable_headless(self, monkeypatch: pytest.MonkeyPatch): + _set_headless(monkeypatch, headless=True) + + def test_write_within_prefix_allowed(self, monkeypatch: pytest.MonkeyPatch): + monkeypatch.setenv("AUTOSKILLIT_ALLOWED_WRITE_PREFIX", self.PREFIX) + event = _build_event("Write", "/clone/.autoskillit/temp/investigate/report.md") + result = _run_hook(event) + assert result == "" + + def test_write_outside_prefix_denied(self, monkeypatch: pytest.MonkeyPatch): + monkeypatch.setenv("AUTOSKILLIT_ALLOWED_WRITE_PREFIX", self.PREFIX) + event = _build_event("Write", "/clone/src/autoskillit/foo.py") + result = _run_hook(event) + parsed = json.loads(result) + decision = parsed["hookSpecificOutput"]["permissionDecision"] + reason = parsed["hookSpecificOutput"]["permissionDecisionReason"] + assert decision == "deny" + assert "read-only skill session" in reason + + def test_edit_outside_prefix_denied(self, monkeypatch: pytest.MonkeyPatch): + monkeypatch.setenv("AUTOSKILLIT_ALLOWED_WRITE_PREFIX", self.PREFIX) + event = _build_event("Edit", "/clone/tests/test_foo.py") + result = _run_hook(event) + parsed = json.loads(result) + assert parsed["hookSpecificOutput"]["permissionDecision"] == "deny" + + def test_malformed_json_denies_in_readonly_session(self, monkeypatch: pytest.MonkeyPatch): + monkeypatch.setenv("AUTOSKILLIT_ALLOWED_WRITE_PREFIX", self.PREFIX) + result = _run_hook("not valid json") + parsed = json.loads(result) + assert parsed["hookSpecificOutput"]["permissionDecision"] == "deny" + assert "malformed" in parsed["hookSpecificOutput"]["permissionDecisionReason"] + + def test_missing_file_path_denies(self, monkeypatch: pytest.MonkeyPatch): + monkeypatch.setenv("AUTOSKILLIT_ALLOWED_WRITE_PREFIX", self.PREFIX) + event = {"tool_name": "Write", "tool_input": {}} + result = _run_hook(event) + parsed = json.loads(result) + assert parsed["hookSpecificOutput"]["permissionDecision"] == "deny" + assert "no file_path" in parsed["hookSpecificOutput"]["permissionDecisionReason"] + + def test_non_write_tool_allowed(self, monkeypatch: pytest.MonkeyPatch): + monkeypatch.setenv("AUTOSKILLIT_ALLOWED_WRITE_PREFIX", self.PREFIX) + event = {"tool_name": "Read", "tool_input": {"file_path": "/clone/src/foo.py"}} + result = _run_hook(event) + assert result == "" + + def test_symlink_resolved(self, monkeypatch: pytest.MonkeyPatch, tmp_path): + real_dir = tmp_path / "real" + real_dir.mkdir() + target_file = real_dir / "secret.py" + target_file.write_text("x") + + allowed = tmp_path / "allowed" + allowed.mkdir() + link = allowed / "link.py" + link.symlink_to(target_file) + + monkeypatch.setenv("AUTOSKILLIT_ALLOWED_WRITE_PREFIX", str(allowed) + "/") + event = _build_event("Write", str(link)) + result = _run_hook(event) + parsed = json.loads(result) + assert parsed["hookSpecificOutput"]["permissionDecision"] == "deny" diff --git a/tests/infra/CLAUDE.md b/tests/infra/CLAUDE.md new file mode 100644 index 0000000000..a4a32a9c99 --- /dev/null +++ b/tests/infra/CLAUDE.md @@ -0,0 +1,61 @@ +# infra/ + +CI/CD configuration, security, guard coverage, and release sanity tests. + +## Files + +| File | Purpose | +|------|---------| +| `__init__.py` | empty | +| `_pretty_output_helpers.py` | Shared helpers for pretty_output hook tests | +| `_token_summary_helpers.py` | Shared helpers for token_summary_appender hook tests | +| `conftest.py` | FormatterCoverageDef NamedTuple and _FORMATTER_COVERAGE_REGISTRY — maps all 11 _FORMATTERS dispatch keys to TypedDict + frozenset pairs for meta-test enforcement | +| `test_anyio_infra.py` | REQ-DEP-001 through REQ-DEP-004: anyio declared as direct dependency | +| `test_ask_user_question_guard.py` | Tests for the ask_user_question_guard PreToolUse hook | +| `test_branch_protection_guard.py` | Tests for hooks/branch_protection_guard.py — PreToolUse branch protection | +| `test_ci_dev_config.py` | Structural enforcement: CI workflow and pre-commit configuration must contain required quality gates | +| `test_ci_workflow.py` | CI workflow structural tests | +| `test_claude_md_critical_rules.py` | Tests that CLAUDE.md contains required critical rules (FRICT-1B-3, FRICT-3A-1) | +| `test_command_guard_completeness.py` | Structural meta-test: command-inspecting guards must cover all command-executing tools | +| `test_coverage_audit.py` | Tests for scripts/compare-coverage-ast.py — AST extraction and coverage comparison | +| `test_dependency_pins.py` | Dependency pin guards (REQ-DEP-001, REQ-DEP-002) — pytest 9.x, igraph bounds | +| `test_docstring_labels.py` | Tests for correct docstring layer labels across the codebase | +| `test_filter_activation.py` | Infrastructure tests: verify test path filtering is activated in project config | +| `test_fleet_dispatch_guard.py` | Tests for fleet_dispatch_guard.py PreToolUse hook | +| `test_generated_file_write_guard.py` | Tests for generated_file_write_guard.py PreToolUse hook | +| `test_generated_files.py` | Tests that generated files with machine-local paths are not tracked in git | +| `test_gitattributes.py` | REQ-R741-A03: .gitattributes must exist and mark vendored JS as binary | +| `test_grep_pattern_lint_guard.py` | Tests for grep_pattern_lint_guard.py — PreToolUse hook for Grep tool pattern syntax | +| `test_guard_coverage.py` | Structural test: destructive tools have PreToolUse hook coverage | +| `test_skill_orchestration_guard.py` | Tests for skill_orchestration_guard.py PreToolUse hook | +| `test_manifest_completeness.py` | Manifest completeness and orphan detection tests for the test-filter manifest | +| `test_mcp_health_advisor.py` | Tests for mcp_health_advisor PreToolUse hook | +| `test_open_kitchen_guard.py` | Phase 2 tests: open_kitchen_guard PreToolUse hook | +| `test_planner_gh_discovery_guard.py` | Tests for the planner_gh_discovery_guard PreToolUse hook | +| `test_pr_create_guard.py` | Tests for the pr_create_guard PreToolUse hook | +| `test_pretty_output_formatters.py` | Tests: pretty_output per-tool named formatters | +| `test_pretty_output_generic_and_wrap.py` | Tests: pretty_output generic formatter and Claude Code double-wrap | +| `test_pretty_output_hook_infra.py` | Tests: pretty_output hook infrastructure, fail-open, and coverage contracts | +| `test_pretty_output_integration.py` | End-to-end schema consistency tests for the pretty_output hook | +| `test_pretty_output_recipe.py` | Tests: pretty_output token/timing, load_recipe, list_recipes, open_kitchen, deduplication | +| `test_pyproject_bounds.py` | Tests for pyproject.toml version lower bounds | +| `test_pyproject_metadata.py` | Verify pyproject.toml contains required public release metadata | +| `test_resume_ownership_guard.py` | Tests for resume_ownership_guard.py PreToolUse hook — ownership validation at resume | +| `test_release_sanity.py` | Release-readiness sanity checks | +| `test_release_workflows.py` | Structural contract tests for the release CI workflows | +| `test_remove_clone_guard.py` | Tests for the remove_clone_guard PreToolUse hook | +| `test_schema_version_convention.py` | Allowlist ratchet: enforce that new JSON dict write sites use write_versioned_json | +| `test_security_config.py` | Structural tests for security configuration integrity | +| `test_session_scope_enforcement.py` | Structural enforcement tests: session-scope metadata on HookDef | +| `test_skill_cmd_check.py` | Unit tests for the skill_cmd_check PreToolUse hook | +| `test_skill_load_guard.py` | Tests for guards/skill_load_guard.py PreToolUse hook — denies native tools until Skill called | +| `test_skill_command_guard.py` | Tests for the skill_command_guard PreToolUse hook | +| `test_taskfile.py` | Taskfile structural tests | +| `test_testmon_eval.py` | Testmon eval tests | +| `test_token_summary_core.py` | Tests: token_summary_appender core — existence, early-exit, happy path, session filtering | +| `test_token_summary_filters.py` | Tests: token_summary_appender unit helpers and order_id isolation | +| `test_unsafe_install_guard.py` | Tests for the unsafe_install_guard PreToolUse hook | + +## Architecture Notes + +`_pretty_output_helpers.py` and `_token_summary_helpers.py` provide shared helper factories used across the split pretty_output and token_summary test files respectively. diff --git a/tests/infra/_pretty_output_helpers.py b/tests/infra/_pretty_output_helpers.py new file mode 100644 index 0000000000..b0d39aafc7 --- /dev/null +++ b/tests/infra/_pretty_output_helpers.py @@ -0,0 +1,120 @@ +"""Shared helpers for pretty_output hook tests.""" + +from __future__ import annotations + +import io +import json +from contextlib import ExitStack, redirect_stdout +from pathlib import Path +from unittest.mock import patch + +# Realistic recipe YAML that mirrors actual bundled recipes. +# Must include an `ingredients:` block to reproduce the raw/derived duplication scenario. +# All formatter tests using `content` should reference this constant. +REALISTIC_RECIPE_YAML = """\ +name: implementation +description: Full implementation pipeline +autoskillit_version: "0.3.0" +ingredients: + task: + description: What to implement + required: true + source_dir: + description: Path to source directory + default: "" + review_approach: + description: Run review-approach before planning + default: "false" +kitchen_rules: + - Always commit before merging +steps: + implement: + tool: run_skill + skill_command: /implement +""" + + +def _run_hook( + event: dict | None = None, + raw_stdin: str | None = None, + cwd: Path | None = None, +) -> tuple[str, int]: + """Run pretty_output.main() with synthetic stdin. + + Returns (stdout_output, exit_code). + """ + from autoskillit.hooks.formatters.pretty_output_hook import main + + stdin_text = raw_stdin if raw_stdin is not None else json.dumps(event or {}) + + exit_code = 0 + buf = io.StringIO() + + with ExitStack() as stack: + stack.enter_context(patch("sys.stdin", io.StringIO(stdin_text))) + stack.enter_context(redirect_stdout(buf)) + if cwd is not None: + stack.enter_context( + patch("autoskillit.hooks.formatters.pretty_output_hook.Path.cwd", return_value=cwd) + ) + try: + main() + except SystemExit as e: + exit_code = e.code if e.code is not None else 0 + + return buf.getvalue(), exit_code + + +def _make_run_skill_event( + success: bool = True, + result: str = "Done.", + session_id: str = "abc", + subtype: str = "end_turn", + is_error: bool = False, + exit_code: int = 0, + needs_retry: bool = False, + retry_reason: str = "none", + stderr: str = "", + token_usage: dict | None = None, + worktree_path: str = "", + provider_used: str = "", + provider_fallback: bool = False, +) -> dict: + return { + "tool_name": "mcp__plugin_autoskillit_autoskillit__run_skill", + "tool_response": json.dumps( + { + "success": success, + "result": result, + "session_id": session_id, + "subtype": subtype, + "is_error": is_error, + "exit_code": exit_code, + "needs_retry": needs_retry, + "retry_reason": retry_reason, + "stderr": stderr, + "token_usage": token_usage, + "worktree_path": worktree_path, + "provider_used": provider_used, + "provider_fallback": provider_fallback, + } + ), + } + + +def _wrap_for_claude_code(payload: dict) -> str: + """Simulate Claude Code's PostToolUse wrapping of MCP text content.""" + return json.dumps({"result": json.dumps(payload)}) + + +def _wrap_plain_str_for_claude_code(text: str) -> str: + """Simulate Claude Code's PostToolUse wrapping of a plain-text MCP response.""" + return json.dumps({"result": text}) + + +def _make_event(tool_name: str, payload: dict) -> dict: + """Build a minimal PostToolUse hook event for a given tool and payload dict.""" + return { + "tool_name": f"mcp__autoskillit__{tool_name}", + "tool_response": json.dumps(payload), + } diff --git a/tests/infra/_token_summary_helpers.py b/tests/infra/_token_summary_helpers.py new file mode 100644 index 0000000000..2b1004c6ad --- /dev/null +++ b/tests/infra/_token_summary_helpers.py @@ -0,0 +1,90 @@ +"""Shared helpers for token_summary_appender hook tests.""" + +from __future__ import annotations + +import io +import json +from contextlib import ExitStack, redirect_stdout +from pathlib import Path +from unittest.mock import patch + + +def _run_hook( + event: dict | None = None, + raw_stdin: str | None = None, + log_root: Path | None = None, + hook_config_path: Path | None = None, +) -> tuple[str, int]: + """Run token_summary_appender.main() with synthetic stdin.""" + from autoskillit.hooks.token_summary_hook import main + + stdin_text = raw_stdin if raw_stdin is not None else json.dumps(event or {}) + exit_code = 0 + buf = io.StringIO() + + with ExitStack() as stack: + stack.enter_context(patch("sys.stdin", io.StringIO(stdin_text))) + stack.enter_context(redirect_stdout(buf)) + if log_root is not None: + stack.enter_context( + patch( + "autoskillit.hooks.token_summary_hook._log_root", + return_value=log_root, + ) + ) + if hook_config_path is not None: + cfg_data = json.loads(hook_config_path.read_text(encoding="utf-8")) + kitchen_id = cfg_data.get("kitchen_id") or cfg_data.get("pipeline_id", "") + stack.enter_context( + patch( + "autoskillit.hooks.token_summary_hook._read_kitchen_id", + return_value=kitchen_id, + ) + ) + try: + main() + except SystemExit as e: + exit_code = e.code if e.code is not None else 0 + + return buf.getvalue(), exit_code + + +def _make_run_skill_event(result_text: str = "Done.\n%%ORDER_UP%%") -> dict: + """Create a double-wrapped PostToolUse event for run_skill.""" + inner = {"result": result_text, "success": True} + outer = {"result": json.dumps(inner)} + return { + "tool_name": "mcp__autoskillit_server__run_skill", + "tool_response": json.dumps(outer), + } + + +def _resolve_session_label(entry: dict) -> str: + """Resolve session_label from entry, falling back to step_name for old test data.""" + return entry.get("session_label") or entry.get("step_name", "unknown") + + +def _write_sessions(log_root: Path, entries: list[dict]) -> None: + """Write sessions.jsonl and token_usage.json files for test setup.""" + (log_root / "sessions.jsonl").write_text("\n".join(json.dumps(e) for e in entries) + "\n") + for entry in entries: + dir_name = entry["dir_name"] + session_dir = log_root / "sessions" / dir_name + session_dir.mkdir(parents=True, exist_ok=True) + token_data: dict = { + "session_label": _resolve_session_label(entry), + "input_tokens": entry.get("input_tokens", 1000), + "output_tokens": entry.get("output_tokens", 500), + "cache_creation_input_tokens": entry.get("cache_creation_input_tokens", 100), + "cache_read_input_tokens": entry.get("cache_read_input_tokens", 200), + "timing_seconds": entry.get("timing_seconds", 10.0), + "loc_insertions": entry.get("loc_insertions", 0), + "loc_deletions": entry.get("loc_deletions", 0), + "peak_context": entry.get("peak_context", 0), + "turn_count": entry.get("turn_count", 0), + "dispatch_id": entry.get("dispatch_id", ""), + "campaign_id": entry.get("campaign_id", ""), + } + if "model_identifier" in entry: + token_data["model_identifier"] = entry["model_identifier"] + (session_dir / "token_usage.json").write_text(json.dumps(token_data)) diff --git a/tests/infra/conftest.py b/tests/infra/conftest.py new file mode 100644 index 0000000000..f8702ec8f0 --- /dev/null +++ b/tests/infra/conftest.py @@ -0,0 +1,116 @@ +"""Formatter field coverage registry for infra test enforcement. + +Defines FormatterCoverageDef and _FORMATTER_COVERAGE_REGISTRY — used by +test_all_formatters_have_coverage_contracts and test_coverage_registry_entries_are_valid +in test_pretty_output_hook_infra.py. +""" + +from __future__ import annotations + +from typing import NamedTuple + + +class FormatterCoverageDef(NamedTuple): + typed_dict: type + rendered: frozenset[str] + suppressed: frozenset[str] + + +def _build_registry() -> dict[str, FormatterCoverageDef]: + from autoskillit.core.types._type_results import CloneSuccessResult + from autoskillit.hooks.formatters.pretty_output_hook import ( + _FMT_CLONE_REPO_RENDERED, + _FMT_CLONE_REPO_SUPPRESSED, + _FMT_KITCHEN_STATUS_RENDERED, + _FMT_KITCHEN_STATUS_SUPPRESSED, + _FMT_LIST_RECIPES_RENDERED, + _FMT_LIST_RECIPES_SUPPRESSED, + _FMT_LOAD_RECIPE_RENDERED, + _FMT_LOAD_RECIPE_SUPPRESSED, + _FMT_MERGE_WORKTREE_RENDERED, + _FMT_MERGE_WORKTREE_SUPPRESSED, + _FMT_OPEN_KITCHEN_RENDERED, + _FMT_OPEN_KITCHEN_SUPPRESSED, + _FMT_RUN_CMD_RENDERED, + _FMT_RUN_CMD_SUPPRESSED, + _FMT_RUN_SKILL_RENDERED, + _FMT_RUN_SKILL_SUPPRESSED, + _FMT_TEST_CHECK_RENDERED, + _FMT_TEST_CHECK_SUPPRESSED, + _FMT_TIMING_SUMMARY_RENDERED, + _FMT_TIMING_SUMMARY_SUPPRESSED, + _FMT_TOKEN_SUMMARY_RENDERED, + _FMT_TOKEN_SUMMARY_SUPPRESSED, + ) + from autoskillit.recipe._api import ListRecipesResult, LoadRecipeResult + from autoskillit.recipe._recipe_ingredients import OpenKitchenResult + from autoskillit.server.tools._types import ( + KitchenStatusResult, + MergeWorktreeResult, + RunCmdResult, + RunSkillResult, + TestCheckResult, + TimingSummaryResult, + TokenSummaryResult, + ) + + return { + "run_skill": FormatterCoverageDef( + typed_dict=RunSkillResult, + rendered=_FMT_RUN_SKILL_RENDERED, + suppressed=_FMT_RUN_SKILL_SUPPRESSED, + ), + "run_cmd": FormatterCoverageDef( + typed_dict=RunCmdResult, + rendered=_FMT_RUN_CMD_RENDERED, + suppressed=_FMT_RUN_CMD_SUPPRESSED, + ), + "test_check": FormatterCoverageDef( + typed_dict=TestCheckResult, + rendered=_FMT_TEST_CHECK_RENDERED, + suppressed=_FMT_TEST_CHECK_SUPPRESSED, + ), + "merge_worktree": FormatterCoverageDef( + typed_dict=MergeWorktreeResult, + rendered=_FMT_MERGE_WORKTREE_RENDERED, + suppressed=_FMT_MERGE_WORKTREE_SUPPRESSED, + ), + "get_token_summary": FormatterCoverageDef( + typed_dict=TokenSummaryResult, + rendered=_FMT_TOKEN_SUMMARY_RENDERED, + suppressed=_FMT_TOKEN_SUMMARY_SUPPRESSED, + ), + "get_timing_summary": FormatterCoverageDef( + typed_dict=TimingSummaryResult, + rendered=_FMT_TIMING_SUMMARY_RENDERED, + suppressed=_FMT_TIMING_SUMMARY_SUPPRESSED, + ), + "kitchen_status": FormatterCoverageDef( + typed_dict=KitchenStatusResult, + rendered=_FMT_KITCHEN_STATUS_RENDERED, + suppressed=_FMT_KITCHEN_STATUS_SUPPRESSED, + ), + "clone_repo": FormatterCoverageDef( + typed_dict=CloneSuccessResult, + rendered=_FMT_CLONE_REPO_RENDERED, + suppressed=_FMT_CLONE_REPO_SUPPRESSED, + ), + "load_recipe": FormatterCoverageDef( + typed_dict=LoadRecipeResult, + rendered=_FMT_LOAD_RECIPE_RENDERED, + suppressed=_FMT_LOAD_RECIPE_SUPPRESSED, + ), + "open_kitchen": FormatterCoverageDef( + typed_dict=OpenKitchenResult, + rendered=_FMT_OPEN_KITCHEN_RENDERED, + suppressed=_FMT_OPEN_KITCHEN_SUPPRESSED, + ), + "list_recipes": FormatterCoverageDef( + typed_dict=ListRecipesResult, + rendered=_FMT_LIST_RECIPES_RENDERED, + suppressed=_FMT_LIST_RECIPES_SUPPRESSED, + ), + } + + +_FORMATTER_COVERAGE_REGISTRY: dict[str, FormatterCoverageDef] = _build_registry() diff --git a/tests/infra/test_ask_user_question_guard.py b/tests/infra/test_ask_user_question_guard.py index 72c28142ca..e7c78b7d4d 100644 --- a/tests/infra/test_ask_user_question_guard.py +++ b/tests/infra/test_ask_user_question_guard.py @@ -22,7 +22,8 @@ def run_guard( hook_input: dict, env_extra: dict | None = None, *, headless: bool = False ) -> RunResult: hook_path = ( - Path(__file__).resolve().parents[2] / "src/autoskillit/hooks/ask_user_question_guard.py" + Path(__file__).resolve().parents[2] + / "src/autoskillit/hooks/guards/ask_user_question_guard.py" ) extra = dict(env_extra or {}) if headless: diff --git a/tests/infra/test_branch_protection_guard.py b/tests/infra/test_branch_protection_guard.py index 7d7a3d8bba..92014a5f6b 100644 --- a/tests/infra/test_branch_protection_guard.py +++ b/tests/infra/test_branch_protection_guard.py @@ -10,6 +10,7 @@ / "src" / "autoskillit" / "hooks" + / "guards" / "branch_protection_guard.py" ) @@ -22,7 +23,7 @@ def _decision(out: dict) -> str | None: def _run_hook(tool_name: str, tool_input: dict, env_override: dict | None = None) -> dict: """Run the hook script with a tool call on stdin, return parsed JSON.""" payload = json.dumps({"tool_name": tool_name, "tool_input": tool_input}) - env = {**os.environ, "AUTOSKILLIT_PROTECTED_BRANCHES": "main,integration,stable"} + env = {**os.environ, "AUTOSKILLIT_PROTECTED_BRANCHES": "main,develop,stable"} if env_override: env.update(env_override) result = subprocess.run( @@ -51,10 +52,10 @@ def test_allows_merge_into_feature_branch(self) -> None: ) assert _decision(out) != "deny" - def test_denies_merge_into_integration(self) -> None: + def test_denies_merge_into_develop(self) -> None: out = _run_hook( "mcp__autoskillit__merge_worktree", - {"worktree_path": "/tmp/wt", "base_branch": "integration"}, + {"worktree_path": "/tmp/wt", "base_branch": "develop"}, ) assert _decision(out) == "deny" @@ -102,10 +103,10 @@ def test_hook_respects_custom_protected_branches_env_var(self) -> None: assert _decision(out) == "deny" def test_hook_allows_branch_not_in_custom_list(self) -> None: - """Hook allows write to integration when it is not in the injected list.""" + """Hook allows write to develop when it is not in the injected list.""" out = _run_hook( "mcp__autoskillit__merge_worktree", - {"worktree_path": "/tmp/wt", "base_branch": "integration"}, + {"worktree_path": "/tmp/wt", "base_branch": "develop"}, env_override={"AUTOSKILLIT_PROTECTED_BRANCHES": "main,release"}, ) assert _decision(out) != "deny" diff --git a/tests/infra/test_ci_dev_config.py b/tests/infra/test_ci_dev_config.py index 03c9b42a51..eb9c50a3f9 100644 --- a/tests/infra/test_ci_dev_config.py +++ b/tests/infra/test_ci_dev_config.py @@ -48,6 +48,25 @@ def test_ruff_tid251_configured(self): "test_architecture.py relies on this rule being enforced by ruff at pre-commit time" ) + def test_sub_claude_md_hook_present(self): + """pre-commit config must include a sub-CLAUDE.md completeness check hook. + + Without this, agents can commit new .py files without updating the + directory's CLAUDE.md file table, causing CI test failures. + """ + config = yaml.safe_load(PRECOMMIT_CONFIG.read_text()) + hooks = [hook for repo in config.get("repos", []) for hook in repo.get("hooks", [])] + sub_claude_hooks = [h for h in hooks if "check_sub_claude_md" in h.get("entry", "")] + assert sub_claude_hooks, ( + "Missing 'check-sub-claude-md' hook in .pre-commit-config.yaml — " + "add it to catch sub-CLAUDE.md coverage gaps before CI" + ) + hook = sub_claude_hooks[0] + assert hook.get("pass_filenames") is False, ( + "check-sub-claude-md hook must use pass_filenames: false — " + "the script does its own filesystem scan" + ) + class TestCIWorkflow: def test_lockfile_check_present_in_workflow(self): @@ -123,14 +142,16 @@ def test_install_step_includes_dev_extras(self): ) def test_setup_uv_action_has_version_pin(self): - """All setup-uv action usages must specify a uv-version pin. + """All setup-uv action usages must specify a version pin. - Without uv-version, astral-sh/setup-uv calls the GitHub API to resolve the + Without version, astral-sh/setup-uv calls the GitHub API to resolve the latest release on every cache miss. On macOS runners, cache misses are frequent (the cache key includes the Python version), causing network timeout failures before any uv command runs. - If this test fails, add 'uv-version: "X.Y.Z"' to all setup-uv steps in tests.yml. + If this test fails, add 'version: "X.Y.Z"' to all setup-uv steps in tests.yml. + Note: the correct input parameter is 'version', not 'uv-version' — 'uv-version' + is an output of the action and is silently ignored as an input. """ workflow = yaml.safe_load(CI_WORKFLOW.read_text()) for job_name, job in workflow.get("jobs", {}).items(): @@ -138,9 +159,9 @@ def test_setup_uv_action_has_version_pin(self): uses = step.get("uses", "") if "setup-uv" in uses: with_block = step.get("with", {}) or {} - assert "uv-version" in with_block, ( - f"CI job '{job_name}' uses {uses!r} without a uv-version pin — " - "add 'uv-version: \"X.Y.Z\"' to prevent GitHub API network failures" + assert "version" in with_block, ( + f"CI job '{job_name}' uses {uses!r} without a version pin — " + "add 'version: \"X.Y.Z\"' to prevent GitHub API network failures" " on macOS runner cache misses" ) @@ -165,24 +186,24 @@ def test_setup_task_action_has_version_pin(self): " from minor releases" ) - def test_ci_push_trigger_excludes_integration(self) -> None: - """Push trigger must NOT include integration — PRs from integration already + def test_ci_push_trigger_excludes_develop(self) -> None: + """Push trigger must NOT include develop — PRs from develop already get CI via pull_request trigger, and including it in push causes duplicate checks.""" workflow = yaml.safe_load(CI_WORKFLOW.read_text()) # PyYAML parses the YAML 'on:' key as Python True (boolean) triggers = workflow.get(True, workflow.get("on", {})) push_branches = triggers["push"]["branches"] - assert "integration" not in push_branches, ( - "integration must not be in push branches — " - "it causes duplicate CI checks when a PR is open from integration" + assert "develop" not in push_branches, ( + "develop must not be in push branches — " + "it causes duplicate CI checks when a PR is open from develop" ) - def test_ci_pull_request_trigger_includes_integration(self) -> None: - """PR trigger must include integration so PRs targeting it get CI.""" + def test_ci_pull_request_trigger_includes_develop(self) -> None: + """PR trigger must include develop so PRs targeting it get CI.""" workflow = yaml.safe_load(CI_WORKFLOW.read_text()) triggers = workflow.get(True, workflow.get("on", {})) pr_branches = triggers["pull_request"]["branches"] - assert "integration" in pr_branches, "CI must trigger on PRs targeting integration branch" + assert "develop" in pr_branches, "CI must trigger on PRs targeting develop branch" def test_ci_preflight_outputs_os_matrix(self) -> None: """preflight job must export an os-matrix output computed from base_ref.""" diff --git a/tests/infra/test_claude_md_critical_rules.py b/tests/infra/test_claude_md_critical_rules.py index 3ecdbe01ea..de36ab9953 100644 --- a/tests/infra/test_claude_md_critical_rules.py +++ b/tests/infra/test_claude_md_critical_rules.py @@ -21,26 +21,6 @@ def claude_md() -> str: return (REPO_ROOT / "CLAUDE.md").read_text() -def test_claude_md_code_index_requires_set_project_path(claude_md: str) -> None: - """§3.3 must instruct agents to call set_project_path before using code-index tools. - - Without this call every code-index tool fails with 'Project path not set', - cascading into parallel call cancellations (FRICT-1B-3). - """ - # Find the §3.3 section using the full heading to avoid false matches - assert "### **3.3" in claude_md, "§3.3 section not found in CLAUDE.md" - section_start = claude_md.index("### **3.3") - # Find the next top-level section heading (### 3.) to bound the search - next_section = claude_md.find("### **3.", section_start + 1) - section_text = ( - claude_md[section_start:next_section] if next_section != -1 else claude_md[section_start:] - ) - assert "set_project_path" in section_text, ( - "CLAUDE.md §3.3 (Code Index MCP Usage) must instruct agents to call " - "set_project_path before using any code-index tool in a new session (FRICT-1B-3)." - ) - - def test_claude_md_critical_rules_require_precommit(claude_md: str) -> None: """§3.1 (Code and Implementation) must include a pre-commit critical rule. @@ -107,3 +87,9 @@ def test_claude_md_no_stale_fidelity_reference(claude_md: str) -> None: "The helpers extract_linked_issues and is_valid_fidelity_finding live in " "execution/pr_analysis.py. Remove the stale pipeline/fidelity.py entry (P2-5)." ) + + +def test_claude_md_has_github_api_discipline(claude_md: str) -> None: + """CLAUDE.md must include §3.5 GitHub API Call Discipline rule.""" + assert "GitHub API Call Discipline" in claude_md + assert "sleep 1" in claude_md or "asyncio.sleep(1)" in claude_md diff --git a/tests/infra/test_command_guard_completeness.py b/tests/infra/test_command_guard_completeness.py new file mode 100644 index 0000000000..98292a0489 --- /dev/null +++ b/tests/infra/test_command_guard_completeness.py @@ -0,0 +1,122 @@ +"""Structural enforcement: command-inspecting guards must cover all command-executing tools. + +This meta-test ensures that any guard which inspects shell command text: +1. Is invoked for ALL tools that can execute shell commands (Bash, run_cmd) +2. Reads command text from BOTH key formats (command, cmd) +3. Covers all relevant deny patterns + +Without this structural test, a guard can be written that reads ``tool_input.cmd`` +but not ``tool_input.command``, and it will silently bypass the Bash tool. +""" + +from __future__ import annotations + +import re +from pathlib import Path + +import pytest + +pytestmark = [pytest.mark.layer("infra"), pytest.mark.small] + +# The canonical set of tool patterns that can execute shell commands. +# When Claude Code calls the native Bash tool: tool_name="Bash", command in tool_input +# When Claude Code calls run_cmd MCP tool: tool_name="mcp__...__run_cmd", cmd in tool_input +COMMAND_EXECUTING_TOOLS = {"Bash", "mcp__autoskillit__local__autoskillit__run_cmd"} + + +def _find_command_inspecting_guards() -> list[tuple[str, Path]]: + """Discover guards that inspect shell command text. + + Returns list of (guard_name, guard_script_path) tuples. + """ + guards_dir = Path(__file__).parent.parent.parent / "src" / "autoskillit" / "hooks" / "guards" + found = [] + for script_path in sorted(guards_dir.glob("*.py")): + if script_path.name.startswith("_"): + continue + source = script_path.read_text() + if ( + '.get("cmd"' in source + or ".get('cmd'" in source + or '.get("command"' in source + or ".get('command'" in source + ): + guard_name = script_path.stem + found.append((guard_name, script_path)) + return found + + +COMMAND_INSPECTING_GUARDS = _find_command_inspecting_guards() + + +def _get_hookdef_matcher(guard_name: str) -> str | None: + """Get the matcher regex for a guard from HOOK_REGISTRY.""" + from autoskillit.hook_registry import HOOK_REGISTRY + + for hook_def in HOOK_REGISTRY: + if any(script.endswith(f"/{guard_name}.py") for script in hook_def.scripts): + return hook_def.matcher + return None + + +class TestCommandGuardStructuralEnforcement: + """Structural tests ensuring command-inspecting guards cover all command-executing tools.""" + + def test_command_inspecting_guards_cover_all_command_tools(self) -> None: + """Every command-inspecting guard must have a matcher that covers Bash AND run_cmd. + + If a guard reads command text, it must fire for ALL tools that send command text. + A matcher that only covers run_cmd will silently miss Bash tool invocations. + """ + for guard_name, _ in COMMAND_INSPECTING_GUARDS: + matcher = _get_hookdef_matcher(guard_name) + assert matcher is not None, f"Guard {guard_name} not found in HOOK_REGISTRY" + + pattern = re.compile(matcher) + + # Check that the matcher covers Bash + bash_matches = pattern.fullmatch("Bash") + # Check that the matcher covers a representative run_cmd FQN + run_cmd_matches = pattern.fullmatch("mcp__autoskillit__local__autoskillit__run_cmd") + + assert bash_matches is not None, ( + f"Guard {guard_name} reads command text but its matcher " + f"{matcher!r} does not match 'Bash'. " + f"Command-inspecting guards must fire for ALL command-executing tools." + ) + assert run_cmd_matches is not None, ( + f"Guard {guard_name} reads command text but its matcher " + f"{matcher!r} does not match 'mcp__autoskillit__local__autoskillit__run_cmd'. " + f"Command-inspecting guards must fire for ALL command-executing tools." + ) + + def test_command_inspecting_guards_read_both_key_formats(self) -> None: + """Every command-inspecting guard must read BOTH 'command' and 'cmd' keys. + + Bash tool sends: tool_input.command + run_cmd MCP tool sends: tool_input.cmd + + A guard that only reads one key will silently miss the other tool. + The correct pattern is: tool_input.get("command", "") or tool_input.get("cmd", "") + """ + for guard_name, script_path in COMMAND_INSPECTING_GUARDS: + source = script_path.read_text() + + has_command_get = ( + '.get("command"' in source + or ".get('command'" in source + or 'tool_input.get("command"' in source + or "tool_input.get('command'" in source + ) + has_cmd_get = ( + '.get("cmd"' in source + or ".get('cmd'" in source + or 'tool_input.get("cmd"' in source + or "tool_input.get('cmd'" in source + ) + + assert has_command_get and has_cmd_get, ( + f"Guard {guard_name} must read both 'command' and 'cmd' keys from tool_input. " + f"Found command get: {has_command_get}, Found cmd get: {has_cmd_get}. " + f"Use: tool_input.get('command', '') or tool_input.get('cmd', '')" + ) diff --git a/tests/infra/test_coverage_audit.py b/tests/infra/test_coverage_audit.py new file mode 100644 index 0000000000..7c6ca6644e --- /dev/null +++ b/tests/infra/test_coverage_audit.py @@ -0,0 +1,317 @@ +"""Tests for scripts/compare-coverage-ast.py — AST extraction and coverage comparison.""" + +from __future__ import annotations + +import importlib.util +import sys +from pathlib import Path + +import pytest + +REPO_ROOT = Path(__file__).parent.parent.parent +_SCRIPT = REPO_ROOT / "scripts" / "compare-coverage-ast.py" + + +@pytest.fixture(scope="module") +def cov_ast(): + """Import compare-coverage-ast.py as a module without polluting sys.path.""" + spec = importlib.util.spec_from_file_location("compare_coverage_ast", _SCRIPT) + assert spec is not None and spec.loader is not None + mod = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = mod + spec.loader.exec_module(mod) + yield mod + sys.modules.pop(spec.name, None) + + +# ── T1a: extract_functions finds all definitions ── + + +def test_extract_functions_finds_all_definitions(cov_ast, tmp_path: Path): + """extract_functions() returns FuncInfo for every def/async def.""" + src = tmp_path / "sample.py" + src.write_text( + """\ +def top_level(): + pass + +async def async_top(): + return 1 + +class Foo: + def method(self): + pass +""", + encoding="utf-8", + ) + funcs = cov_ast.extract_functions(src) + names = [f.qualname for f in funcs] + assert "top_level" in names + assert "async_top" in names + assert "Foo.method" in names + async_func = next(f for f in funcs if f.qualname == "async_top") + assert async_func.is_async is True + + +# ── T1b: nested classes ── + + +def test_extract_functions_handles_nested_classes(cov_ast, tmp_path: Path): + """Nested class methods get qualified names like 'Outer.Inner.method'.""" + src = tmp_path / "nested.py" + src.write_text( + """\ +class Outer: + class Inner: + def method(self): + pass +""", + encoding="utf-8", + ) + funcs = cov_ast.extract_functions(src) + qualnames = [f.qualname for f in funcs] + assert "Outer.Inner.method" in qualnames + + +# ── T1c: syntax errors ── + + +def test_extract_functions_skips_syntax_errors(cov_ast, tmp_path: Path, capsys): + """Files with syntax errors are skipped with a warning, not a crash.""" + src = tmp_path / "bad.py" + src.write_text("def broken(\n", encoding="utf-8") + funcs = cov_ast.extract_functions(src) + assert funcs == [] + captured = capsys.readouterr() + assert "SyntaxError" in captured.err + + +# ── T1d: uncovered functions ── + + +def test_compare_finds_uncovered_functions(cov_ast): + """Functions whose line ranges have zero intersection with covered lines are reported.""" + func = cov_ast.FuncInfo( + name="orphan", + qualname="orphan", + filepath="mod.py", + lineno=10, + end_lineno=20, + is_async=False, + ) + ast_map = {"mod.py": [func]} + coverage_map: dict[str, set[int]] = {} + report = cov_ast.compare(ast_map, coverage_map) + assert report.uncovered == 1 + assert report.covered == 0 + + +# ── T1e: covered functions ── + + +def test_compare_marks_covered_functions(cov_ast): + """Functions whose line ranges overlap with covered lines are marked as covered.""" + func = cov_ast.FuncInfo( + name="tested", + qualname="tested", + filepath="mod.py", + lineno=1, + end_lineno=3, + is_async=False, + ) + ast_map = {"mod.py": [func]} + coverage_map = {"mod.py": {1, 2, 3}} + report = cov_ast.compare(ast_map, coverage_map) + assert report.covered == 1 + assert report.uncovered == 0 + + +# ── T1f: partial coverage ── + + +def test_compare_detects_partial_coverage(cov_ast): + """Functions where some lines are covered but not all are marked as partially covered.""" + func = cov_ast.FuncInfo( + name="half", + qualname="half", + filepath="mod.py", + lineno=1, + end_lineno=10, + is_async=False, + ) + ast_map = {"mod.py": [func]} + coverage_map = {"mod.py": {1, 2, 3}} + report = cov_ast.compare(ast_map, coverage_map) + assert report.partial == 1 + assert report.covered == 0 + assert report.uncovered == 0 + + +# ── TestBuildTestSourceMap ── + + +class TestBuildTestSourceMap: + def test_query_contexts_map_inverts_correctly(self, cov_ast, tmp_path, monkeypatch): + """query_contexts_map inverts {line: {ctx}} to {src_file: {test_file}}. + + Uses MagicMock to simulate CoverageData. + Context names use |run suffix. + """ + from unittest.mock import MagicMock + + import coverage as coverage_mod + + src_file = str(cov_ast.PROJECT_ROOT / "src" / "autoskillit" / "core" / "io.py") + + mock_data = MagicMock() + mock_data.measured_files.return_value = [src_file] + mock_data.contexts_by_lineno.return_value = { + 1: ["tests/core/test_io.py::TestIO::test_write|run"], + 2: ["tests/core/test_io.py::TestIO::test_write|run"], + } + monkeypatch.setattr(coverage_mod, "CoverageData", MagicMock(return_value=mock_data)) + + result = cov_ast.query_contexts_map(tmp_path / ".coverage") + assert "src/autoskillit/core/io.py" in result + assert "tests/core/test_io.py" in result["src/autoskillit/core/io.py"] + + def test_setup_and_teardown_contexts_excluded(self, cov_ast, tmp_path, monkeypatch): + """query_contexts_map excludes |setup and |teardown contexts. + + Only |run phase entries map source files to tests. A source file touched + only during |setup or |teardown must NOT appear in the result. + """ + from unittest.mock import MagicMock + + import coverage as coverage_mod + + src_file = str(cov_ast.PROJECT_ROOT / "src" / "autoskillit" / "core" / "io.py") + + mock_data = MagicMock() + mock_data.measured_files.return_value = [src_file] + mock_data.contexts_by_lineno.return_value = { + 1: [ + "tests/core/test_io.py::TestIO::test_write|setup", + "tests/core/test_io.py::TestIO::test_write|teardown", + ], + } + monkeypatch.setattr(coverage_mod, "CoverageData", MagicMock(return_value=mock_data)) + + result = cov_ast.query_contexts_map(tmp_path / ".coverage") + assert "src/autoskillit/core/io.py" not in result + + def test_build_test_source_map_writes_json(self, cov_ast, tmp_path, monkeypatch): + """build_test_source_map() writes a valid JSON file to the output path.""" + import json + from unittest.mock import MagicMock + + import coverage as coverage_mod + + src_file = str(cov_ast.PROJECT_ROOT / "src" / "autoskillit" / "core" / "io.py") + mock_data = MagicMock() + mock_data.measured_files.return_value = [src_file] + mock_data.contexts_by_lineno.return_value = { + 1: ["tests/core/test_io.py::TestIO::test_write|run"], + } + monkeypatch.setattr(coverage_mod, "CoverageData", MagicMock(return_value=mock_data)) + + db_path = tmp_path / ".coverage" + db_path.touch() + output_path = tmp_path / "test-source-map.json" + cov_ast.build_test_source_map(db_path, output_path) + + assert output_path.exists() + parsed = json.loads(output_path.read_text()) + assert isinstance(parsed, dict) + expected_key = "src/autoskillit/core/io.py" + assert expected_key in parsed + assert "tests/core/test_io.py" in parsed[expected_key] + + def test_main_routes_build_test_source_map_mode(self, cov_ast, tmp_path, monkeypatch): + """main() with --mode build-test-source-map calls build_test_source_map().""" + + called_with: dict = {} + + def fake_build(db_path, output_path): + called_with["db_path"] = db_path + called_with["output_path"] = output_path + return True + + monkeypatch.setattr(cov_ast, "build_test_source_map", fake_build) + monkeypatch.setattr( + "sys.argv", + ["compare-coverage-ast.py", "--mode", "build-test-source-map"], + ) + result = cov_ast.main() + assert result == 0 + assert "output_path" in called_with + assert called_with["db_path"] == cov_ast.PROJECT_ROOT / ".coverage" + + def test_map_json_values_are_lists(self, cov_ast, tmp_path, monkeypatch): + """The written JSON has list values (not sets), loadable as JSON.""" + import json + from unittest.mock import MagicMock + + import coverage as coverage_mod + + src_file = str(cov_ast.PROJECT_ROOT / "src" / "autoskillit" / "core" / "io.py") + mock_data = MagicMock() + mock_data.measured_files.return_value = [src_file] + mock_data.contexts_by_lineno.return_value = { + 1: ["tests/core/test_io.py::TestIO::test_a|run"], + 2: ["tests/core/test_io.py::TestIO::test_b|run"], + } + monkeypatch.setattr(coverage_mod, "CoverageData", MagicMock(return_value=mock_data)) + + db_path = tmp_path / ".coverage" + db_path.touch() + output_path = tmp_path / "test-source-map.json" + cov_ast.build_test_source_map(db_path, output_path) + + parsed = json.loads(output_path.read_text()) + for v in parsed.values(): + assert isinstance(v, list) + + def test_taskfile_coverage_audit_invokes_map_mode(self): + """Taskfile.yml coverage-audit task includes --mode build-test-source-map.""" + taskfile = Path(__file__).parent.parent.parent / "Taskfile.yml" + content = taskfile.read_text() + assert "build-test-source-map" in content + + +@pytest.mark.small +def test_test_source_map_is_committed(): + """Sentinel: .autoskillit/test-source-map.json must be committed to version control. + + Fails if the file is missing — run 'task coverage-audit' and commit the output. + """ + import json + + map_path = Path(__file__).parent.parent.parent / ".autoskillit" / "test-source-map.json" + assert map_path.exists(), ( + ".autoskillit/test-source-map.json is missing. " + "Run 'task coverage-audit' and commit the output to activate the coverage oracle." + ) + data = json.loads(map_path.read_text(encoding="utf-8")) + assert isinstance(data, dict) + + +@pytest.mark.small +def test_load_coverage_map_reads_committed_file(): + """load_coverage_map() returns a populated dict from the committed oracle file. + + Verifies that the file is also parseable and fresh (not older than 30 days). + """ + from tests._test_filter import load_coverage_map + + map_path = Path(__file__).parent.parent.parent / ".autoskillit" / "test-source-map.json" + assert map_path.exists(), ( + ".autoskillit/test-source-map.json is missing. " + "Run 'task coverage-audit' and commit the output to activate the coverage oracle." + ) + result = load_coverage_map(map_path) + assert result is not None, ( + "load_coverage_map() returned None — oracle file is older than 30 days or malformed. " + "Run 'task coverage-audit' and commit the result." + ) + assert len(result) >= 50, f"Oracle returned only {len(result)} entries" diff --git a/tests/infra/test_docstring_labels.py b/tests/infra/test_docstring_labels.py index 4b622c00b7..3031548653 100644 --- a/tests/infra/test_docstring_labels.py +++ b/tests/infra/test_docstring_labels.py @@ -7,16 +7,16 @@ _REPO_ROOT = Path(__file__).resolve().parent.parent.parent -def test_session_py_docstring_says_l2(): - """session.py docstring must carry an L2 label.""" - src = (_REPO_ROOT / "src/autoskillit/execution/session.py").read_text() - assert "L2" in src.split('"""')[1], "session.py docstring must say L2" +def test_session_py_docstring_says_il1(): + """session.py docstring must carry an IL-1 label.""" + src = (_REPO_ROOT / "src/autoskillit/execution/session/__init__.py").read_text() + assert "IL-1" in src.split('"""')[1], "session.py docstring must say IL-1" -def test_headless_py_docstring_says_l1(): - """headless.py docstring must carry an L1 label.""" - src = (_REPO_ROOT / "src/autoskillit/execution/headless.py").read_text() - assert "L1" in src.split('"""')[1], "headless.py docstring must say L1" +def test_headless_py_docstring_says_il1(): + """headless.py docstring must carry an IL-1 label.""" + src = (_REPO_ROOT / "src/autoskillit/execution/headless/__init__.py").read_text() + assert "IL-1" in src.split('"""')[1], "headless.py docstring must say IL-1" def test_smoke_utils_documents_limitation(): @@ -33,3 +33,12 @@ def test_claude_md_no_stale_l3_headless_label(): assert "L3 service module" not in src, ( "CLAUDE.md still contains stale 'L3 service module' label for headless" ) + + +def test_planner_init_has_docstring(): + """planner/__init__.py must have a module-level docstring like other IL-1 packages.""" + import autoskillit.planner as planner_pkg + + assert planner_pkg.__doc__ is not None and planner_pkg.__doc__.strip(), ( + "planner/__init__.py is missing a module-level docstring" + ) diff --git a/tests/infra/test_filter_activation.py b/tests/infra/test_filter_activation.py new file mode 100644 index 0000000000..e6f83e8f94 --- /dev/null +++ b/tests/infra/test_filter_activation.py @@ -0,0 +1,92 @@ +"""Infrastructure tests: verify test path filtering is activated in project config.""" + +from __future__ import annotations + +from pathlib import Path + +REPO_ROOT = Path(__file__).parent.parent.parent + + +def test_project_config_has_filter_mode_conservative(): + """AC1: .autoskillit/config.yaml must set filter_mode to conservative.""" + from autoskillit.core.io import load_yaml + + cfg = load_yaml(REPO_ROOT / ".autoskillit/config.yaml") + assert cfg["test_check"]["filter_mode"] == "conservative" + + +def test_project_config_has_base_ref(): + """AC1: .autoskillit/config.yaml must set base_ref to develop.""" + from autoskillit.core.io import load_yaml + + cfg = load_yaml(REPO_ROOT / ".autoskillit/config.yaml") + assert cfg["test_check"]["base_ref"] == "develop" + + +def test_hook_registry_tests_in_infra(): + """AC4: test_hook_registry.py must live in tests/hooks/.""" + assert (REPO_ROOT / "tests/hooks/test_hook_registry.py").is_file() + assert not (REPO_ROOT / "tests/test_hook_registry.py").is_file() + + +def test_phase2_skills_in_skills(): + """AC4: test_phase2_skills.py must live in tests/skills/.""" + assert (REPO_ROOT / "tests/skills/test_phase2_skills.py").is_file() + assert not (REPO_ROOT / "tests/test_phase2_skills.py").is_file() + + +def test_skill_preambles_in_skills(): + """AC4: test_skill_preambles.py must live in tests/skills/.""" + assert (REPO_ROOT / "tests/skills/test_skill_preambles.py").is_file() + assert not (REPO_ROOT / "tests/test_skill_preambles.py").is_file() + + +def test_ci_filter_codepath_produces_scope(): + """CI codepath: conservative mode + known source file -> non-None scope.""" + from tests._test_filter import FilterMode, FullRunReason, build_test_scope, load_manifest + + manifest = load_manifest(REPO_ROOT) + scope = build_test_scope( + changed_files={"src/autoskillit/core/paths.py"}, + mode=FilterMode.CONSERVATIVE, + manifest=manifest, + tests_root=REPO_ROOT / "tests", + cwd=REPO_ROOT, + base_ref="develop", + ) + assert not isinstance(scope, FullRunReason), ( + "build_test_scope must return a set scope for conservative mode " + "with a known source file — FullRunReason means silent fallback to full run" + ) + assert len(scope) > 0, "Scope must contain at least one test path" + + +def test_build_test_scope_returns_full_run_reason_for_large_changeset(): + from tests._test_filter import FilterMode, FullRunReason, build_test_scope + + changed = {f"src/autoskillit/fake_{i}.py" for i in range(35)} + result = build_test_scope(changed_files=changed, mode=FilterMode.CONSERVATIVE) + assert result is FullRunReason.LARGE_CHANGESET + + +def test_build_test_scope_returns_full_run_reason_for_git_unavailable(): + from tests._test_filter import FilterMode, FullRunReason, build_test_scope + + result = build_test_scope(changed_files=None, mode=FilterMode.CONSERVATIVE) + assert result is FullRunReason.GIT_UNAVAILABLE + + +def test_build_test_scope_returns_full_run_reason_for_bucket_a(): + from tests._test_filter import FilterMode, FullRunReason, build_test_scope + + changed = {"pyproject.toml"} + result = build_test_scope(changed_files=changed, mode=FilterMode.CONSERVATIVE) + assert result is FullRunReason.BUCKET_A + + +def test_build_test_scope_returns_full_run_reason_for_unmapped(): + from tests._test_filter import FilterMode, FullRunReason, build_test_scope + + changed = {"scripts/random_script.py"} + result = build_test_scope(changed_files=changed, mode=FilterMode.CONSERVATIVE) + assert result is FullRunReason.UNMAPPED_FILE diff --git a/tests/infra/test_fleet_dispatch_guard.py b/tests/infra/test_fleet_dispatch_guard.py new file mode 100644 index 0000000000..577dbebf16 --- /dev/null +++ b/tests/infra/test_fleet_dispatch_guard.py @@ -0,0 +1,92 @@ +"""Tests for fleet_dispatch_guard.py PreToolUse hook.""" + +from __future__ import annotations + +import io +import json +import os +from contextlib import redirect_stdout +from unittest.mock import patch + +import pytest + + +def _run_guard( + tool_input: dict, + *, + headless: bool = False, + session_type: str | None = None, + raw_stdin: str | None = None, +) -> str: + """Run main() with the given event payload.""" + from autoskillit.hooks.guards.fleet_dispatch_guard import main + + stdin_content = raw_stdin if raw_stdin is not None else json.dumps(tool_input) + env_updates: dict[str, str] = {} + if headless: + env_updates["AUTOSKILLIT_HEADLESS"] = "1" + if session_type is not None: + env_updates["AUTOSKILLIT_SESSION_TYPE"] = session_type + with ( + patch.dict(os.environ, env_updates, clear=False), + patch("sys.stdin", io.StringIO(stdin_content)), + ): + buf = io.StringIO() + with redirect_stdout(buf): + try: + main() + except SystemExit as exc: + if exc.code not in (None, 0): + raise + return buf.getvalue() + + +# --- M1: Interactive callers always permitted --- + + +def test_guard_permits_interactive_caller(): + out = _run_guard( + {"tool_name": "mcp__plugin_autoskillit_autoskillit__dispatch_food_truck"}, + headless=False, + ) + assert not out.strip() + + +# --- M2–M5: Headless callers always denied --- + + +@pytest.mark.parametrize( + "session_type", + ["orchestrator", "fleet", "skill", None], + ids=["orchestrator", "fleet", "skill", "unset"], +) +def test_guard_denies_headless_regardless_of_session_type(session_type): + out = _run_guard( + {"tool_name": "mcp__plugin_autoskillit_autoskillit__dispatch_food_truck"}, + headless=True, + session_type=session_type, + ) + response = json.loads(out) + assert response["hookSpecificOutput"]["permissionDecision"] == "deny" + reason = response["hookSpecificOutput"]["permissionDecisionReason"].lower() + assert "headless" in reason or "fleet session" in reason + + +# --- M7: Unrelated tool passes through --- + + +def test_guard_ignores_unrelated_tool(): + out = _run_guard( + {"tool_name": "Bash"}, + headless=True, + session_type="fleet", + ) + assert not out.strip() + + +# --- M8: Malformed input --- + + +def test_guard_fails_open_on_malformed_input(): + out = _run_guard({}, headless=True, raw_stdin="not-json") + assert not out.strip() diff --git a/tests/infra/test_generated_file_write_guard.py b/tests/infra/test_generated_file_write_guard.py index b0acb87257..a3897d6a96 100644 --- a/tests/infra/test_generated_file_write_guard.py +++ b/tests/infra/test_generated_file_write_guard.py @@ -8,7 +8,7 @@ def _run_guard(event: dict) -> dict | None: - script = pkg_root() / "hooks" / "generated_file_write_guard.py" + script = pkg_root() / "hooks" / "guards" / "generated_file_write_guard.py" result = subprocess.run( [sys.executable, str(script)], input=json.dumps(event), @@ -57,7 +57,7 @@ def test_write_guard_allows_other_files(): def test_write_guard_fail_open_on_invalid_json(): - script = pkg_root() / "hooks" / "generated_file_write_guard.py" + script = pkg_root() / "hooks" / "guards" / "generated_file_write_guard.py" result = subprocess.run( [sys.executable, str(script)], input="not json", diff --git a/tests/infra/test_grep_pattern_lint_guard.py b/tests/infra/test_grep_pattern_lint_guard.py new file mode 100644 index 0000000000..cfc90737f9 --- /dev/null +++ b/tests/infra/test_grep_pattern_lint_guard.py @@ -0,0 +1,143 @@ +"""Tests for grep_pattern_lint_guard.py — PreToolUse hook for Grep tool pattern syntax.""" + +import json +from io import StringIO +from unittest.mock import patch + +from autoskillit.hooks.guards.grep_pattern_lint_guard import main + + +def _run_hook(tool_name: str, pattern: str) -> dict | None: + """Run the guard with a synthetic PreToolUse event; return parsed stdout or None.""" + event = {"tool_name": tool_name, "tool_input": {"pattern": pattern}} + captured = StringIO() + with patch("sys.stdin", StringIO(json.dumps(event))), patch("sys.stdout", captured): + try: + main() + except SystemExit: + pass + output = captured.getvalue().strip() + return json.loads(output) if output else None + + +def _decision(result: dict | None) -> str: + if result is None: + return "allow" + return result.get("hookSpecificOutput", {}).get("permissionDecision", "allow") + + +def _reason(result: dict | None) -> str: + if result is None: + return "" + return result.get("hookSpecificOutput", {}).get("permissionDecisionReason", "") + + +# --- Core deny cases --- + + +def test_bre_alternation_single_is_denied(): + """Single \\| alternation operator must be denied.""" + result = _run_hook("Grep", r"foo\|bar") + assert _decision(result) == "deny" + + +def test_bre_alternation_multiple_is_denied(): + """Multiple \\| operators in one pattern must be denied.""" + result = _run_hook("Grep", r"foo\|bar\|baz") + assert _decision(result) == "deny" + + +def test_bre_alternation_in_complex_pattern_is_denied(): + """\\| embedded in a longer regex must be denied.""" + result = _run_hook("Grep", r"def.*generate\|def.*regenerate\|build.*contract") + assert _decision(result) == "deny" + + +# --- Allow cases --- + + +def test_plain_pipe_alternation_is_allowed(): + """Bare | alternation (ripgrep ERE) must be allowed.""" + result = _run_hook("Grep", r"foo|bar") + assert _decision(result) == "allow" + + +def test_empty_pattern_is_allowed(): + """Empty pattern must be allowed (no \\| present).""" + result = _run_hook("Grep", "") + assert _decision(result) == "allow" + + +def test_pattern_without_alternation_is_allowed(): + """Plain word pattern without any | must be allowed.""" + result = _run_hook("Grep", r"def.*generate") + assert _decision(result) == "allow" + + +# --- Non-Grep tool passthrough --- + + +def test_bash_tool_passthrough(): + """Hook must not fire for the Bash tool (guard is Grep-specific).""" + result = _run_hook("Bash", r"grep 'foo\|bar'") + assert _decision(result) == "allow" + + +def test_read_tool_passthrough(): + """Hook must not fire for Read tool.""" + result = _run_hook("Read", r"some\|pattern") + assert _decision(result) == "allow" + + +# --- Deny message quality --- + + +def test_deny_reason_includes_corrected_pattern(): + """permissionDecisionReason must include the corrected pattern with | replacing \\|.""" + result = _run_hook("Grep", r"foo\|bar") + reason = _reason(result) + assert "foo|bar" in reason + + +def test_deny_reason_explains_ripgrep_syntax(): + """permissionDecisionReason must mention ripgrep or ERE/alternation context.""" + result = _run_hook("Grep", r"foo\|bar") + reason = _reason(result) + assert ( + "ripgrep" in reason.lower() or "alternation" in reason.lower() or "ere" in reason.lower() + ) + + +def test_deny_reason_for_multiple_replacements(): + """All \\| occurrences must appear corrected in the reason.""" + result = _run_hook("Grep", r"foo\|bar\|baz") + reason = _reason(result) + assert "foo|bar|baz" in reason + + +# --- Malformed input fail-open --- + + +def test_malformed_json_falls_through(): + """Non-JSON stdin must not crash — hook exits silently (allow).""" + captured = StringIO() + with patch("sys.stdin", StringIO("not json")), patch("sys.stdout", captured): + try: + main() + except SystemExit: + pass + assert captured.getvalue().strip() == "" + + +def test_missing_pattern_field_falls_through(): + """tool_input without pattern key must not deny (allow).""" + event = {"tool_name": "Grep", "tool_input": {}} + captured = StringIO() + with patch("sys.stdin", StringIO(json.dumps(event))), patch("sys.stdout", captured): + try: + main() + except SystemExit: + pass + result = captured.getvalue().strip() + output = json.loads(result) if result else None + assert _decision(output) == "allow" diff --git a/tests/infra/test_headless_orchestration_guard.py b/tests/infra/test_headless_orchestration_guard.py deleted file mode 100644 index 440bb72114..0000000000 --- a/tests/infra/test_headless_orchestration_guard.py +++ /dev/null @@ -1,71 +0,0 @@ -"""Tests for headless_orchestration_guard.py PreToolUse hook.""" - -from __future__ import annotations - -import io -import json -import os -from contextlib import redirect_stdout -from unittest.mock import patch - -import pytest - -_ORCHESTRATION_TOOLS = ["run_skill", "run_cmd", "run_python"] - - -def _run_guard(tool_input: dict, *, headless: bool = False, raw_stdin: str | None = None) -> str: - """Run main() with the given event, optionally setting AUTOSKILLIT_HEADLESS=1. - - raw_stdin: if provided, passed directly to stdin instead of json.dumps(tool_input). - Use this to test malformed-input paths without duplicating setup boilerplate. - """ - from autoskillit.hooks.headless_orchestration_guard import main - - stdin_content = raw_stdin if raw_stdin is not None else json.dumps(tool_input) - env_updates = {"AUTOSKILLIT_HEADLESS": "1"} if headless else {} - with ( - patch.dict(os.environ, env_updates, clear=False), - patch("sys.stdin", io.StringIO(stdin_content)), - ): - buf = io.StringIO() - with redirect_stdout(buf): - try: - main() - except SystemExit: - pass - return buf.getvalue() - - -def _run_guard_headless(tool_input: dict) -> dict: - """Run guard in headless mode and parse output JSON.""" - out = _run_guard(tool_input, headless=True) - return json.loads(out) if out.strip() else {} - - -@pytest.mark.parametrize("tool_name", _ORCHESTRATION_TOOLS) -def test_guard_denies_orchestration_tools_when_headless(tool_name): - response = _run_guard_headless({"tool_name": f"mcp__autoskillit__{tool_name}"}) - assert response["hookSpecificOutput"]["permissionDecision"] == "deny" - assert "headless" in response["hookSpecificOutput"]["permissionDecisionReason"].lower() - - -def test_guard_allows_non_headless(): - out = _run_guard({"tool_name": "mcp__autoskillit__run_skill"}, headless=False) - assert not out.strip() - - -def test_guard_allows_non_orchestration_tool_when_headless(): - response = _run_guard_headless({"tool_name": "mcp__autoskillit__fetch_github_issue"}) - assert response == {} - - -def test_guard_allows_merge_worktree_when_headless(): - """merge_worktree is not an orchestration tool — workers can still call it.""" - response = _run_guard_headless({"tool_name": "mcp__autoskillit__merge_worktree"}) - assert response == {} - - -def test_guard_fails_open_on_malformed_input(): - """Malformed stdin must not raise — hook exits 0 silently.""" - out = _run_guard({}, headless=True, raw_stdin="not-json") - assert not out.strip() diff --git a/tests/infra/test_manifest_completeness.py b/tests/infra/test_manifest_completeness.py new file mode 100644 index 0000000000..0aad895a5e --- /dev/null +++ b/tests/infra/test_manifest_completeness.py @@ -0,0 +1,147 @@ +"""Manifest completeness and orphan detection tests for the test-filter manifest. + +Completeness test: every non-Python tracked file (outside Bucket A and the ignore +list) must match at least one pattern in .autoskillit/test-filter-manifest.yaml. + +Orphan detection test: every manifest pattern must match at least one currently +tracked non-Python file (no dead/stale patterns). +""" + +from __future__ import annotations + +import functools +import subprocess +from pathlib import Path + +import pathspec +import pytest + +from autoskillit._test_filter import load_manifest + +pytestmark = [pytest.mark.layer("infra")] + +_REPO_ROOT = Path(__file__).resolve().parents[2] +_MANIFEST_PATH = _REPO_ROOT / ".autoskillit" / "test-filter-manifest.yaml" + +# Files that trigger a full test run (Bucket A) — excluded from manifest coverage check. +# These are already handled before the manifest is consulted. +_BUCKET_A_FILES: frozenset[str] = frozenset( + { + "pyproject.toml", + "uv.lock", + ".pre-commit-config.yaml", + } +) + +# Files legitimately absent from the manifest (no test directory mapping needed). +# Extend this set when new unmapped file types are added to the repo. +_IGNORE_FILES: frozenset[str] = frozenset( + { + "LICENSE", + "tests/CLAUDE.md", + # Generated/state files in .autoskillit/ that carry no test-routing signal. + ".autoskillit/.gitignore", + ".autoskillit/.onboarded", + ".autoskillit/sync_manifest.json", + } +) + +# Glob patterns for file types legitimately absent from the manifest. +_IGNORE_PATTERNS: tuple[str, ...] = ( + "*.gif", + "**/.gitkeep", + "tests/*/CLAUDE.md", # Per-subdirectory documentation (no test-routing signal) +) + +# Combined PathSpec built once from _IGNORE_PATTERNS to avoid O(files×patterns) +# construction inside _should_be_covered(). +_IGNORE_SPEC: pathspec.PathSpec = pathspec.PathSpec.from_lines( + "gitwildmatch", list(_IGNORE_PATTERNS) +) + + +@functools.cache +def _tracked_non_python_files() -> tuple[str, ...]: + """Return all non-Python files currently tracked by git. + + Cached to avoid repeated subprocess calls within a single worker process: + both _files_to_check() (collection time) and test_manifest_pattern_matches_real_file + (test execution) call this function. + """ + try: + result = subprocess.run( + ["git", "ls-files", "--cached"], + capture_output=True, + text=True, + cwd=_REPO_ROOT, + check=True, + ) + except subprocess.CalledProcessError as e: + msg = f"git ls-files failed in {_REPO_ROOT}: {e.stderr.strip()}" + raise RuntimeError(msg) from e + return tuple(f for f in result.stdout.splitlines() if not f.endswith(".py")) + + +def _should_be_covered(file_path: str) -> bool: + """Return True if file_path must be covered by a manifest pattern.""" + if file_path in _BUCKET_A_FILES: + return False + if file_path in _IGNORE_FILES: + return False + if _IGNORE_SPEC.match_file(file_path): + return False + return True + + +def _files_to_check() -> list[str]: + """Filtered list of non-Python tracked files that must appear in the manifest.""" + return [f for f in _tracked_non_python_files() if _should_be_covered(f)] + + +# Manifest patterns loaded once at module level; shared by _MANIFEST_SPEC and +# _PER_PATTERN_SPECS to avoid redundant YAML loads across the two parametrize sites. +_MANIFEST_PATTERNS: list[str] = list(load_manifest(_MANIFEST_PATH).keys()) + +# Combined spec precomputed once; used by test_file_covered_by_manifest. +_MANIFEST_SPEC: pathspec.PathSpec = pathspec.PathSpec.from_lines( + "gitwildmatch", _MANIFEST_PATTERNS +) + +# Per-pattern specs precomputed once; used by test_manifest_pattern_matches_real_file +# to avoid O(N) PathSpec constructions across parametrized invocations. +_PER_PATTERN_SPECS: dict[str, pathspec.PathSpec] = { + p: pathspec.PathSpec.from_lines("gitwildmatch", [p]) for p in _MANIFEST_PATTERNS +} + + +@pytest.mark.parametrize("file_path", _files_to_check()) +def test_file_covered_by_manifest(file_path: str) -> None: + """Every non-Python tracked file (outside Bucket A + ignore list) must match + at least one manifest pattern. + + Failure means a new non-Python file was added without a manifest entry. + Fix: add a glob pattern to .autoskillit/test-filter-manifest.yaml, or add + the file to _IGNORE_FILES / _IGNORE_PATTERNS if no test mapping is needed. + """ + assert _MANIFEST_SPEC.match_file(file_path), ( + f"File {file_path!r} is not covered by any manifest pattern. " + "Add an entry to .autoskillit/test-filter-manifest.yaml " + "or add to _IGNORE_FILES / _IGNORE_PATTERNS in this test file." + ) + + +@pytest.mark.parametrize("pattern", _MANIFEST_PATTERNS) +def test_manifest_pattern_matches_real_file(pattern: str) -> None: + """Every manifest pattern must match at least one currently tracked non-Python file. + + Failure means the pattern is orphaned — either the files it matched were deleted, + renamed, or the pattern was misspelled from the start. + Fix: remove the stale pattern from .autoskillit/test-filter-manifest.yaml or + correct its path. + """ + all_tracked = _tracked_non_python_files() + matched_files = [f for f in all_tracked if _PER_PATTERN_SPECS[pattern].match_file(f)] + assert matched_files, ( + f"Manifest pattern {pattern!r} matches no tracked files — it may be stale " + "or misspelled. Remove it from .autoskillit/test-filter-manifest.yaml." + ) diff --git a/tests/infra/test_mcp_health_advisor.py b/tests/infra/test_mcp_health_advisor.py new file mode 100644 index 0000000000..08f501cefb --- /dev/null +++ b/tests/infra/test_mcp_health_advisor.py @@ -0,0 +1,173 @@ +"""Tests for mcp_health_advisor PreToolUse hook.""" + +from __future__ import annotations + +import json +import os +import subprocess +import sys +from pathlib import Path + +from autoskillit.core.paths import pkg_root + + +def _run_guard( + tmp_path: Path, + env_extra: dict, + tool_name: str = "Read", + kitchens: list[dict] | None = None, + cwd: Path | str | None = None, + headless: bool = False, +) -> tuple[int, dict]: + """Run mcp_health_advisor.py as a subprocess, return (returncode, parsed_stdout).""" + hook_path = pkg_root() / "hooks" / "guards" / "mcp_health_advisor.py" + home = tmp_path / "fakehome" + home.mkdir(exist_ok=True) + ak_dir = home / ".autoskillit" + ak_dir.mkdir(exist_ok=True) + if kitchens is not None: + (ak_dir / "active_kitchens.json").write_text( + json.dumps({"kitchens": kitchens, "schema_version": 1}) + ) + env = {k: v for k, v in os.environ.items() if k != "AUTOSKILLIT_HEADLESS"} + env["HOME"] = str(home) + if headless: + env["AUTOSKILLIT_HEADLESS"] = "1" + env.update(env_extra) + result = subprocess.run( + [sys.executable, str(hook_path)], + input=json.dumps({"tool_name": tool_name}), + capture_output=True, + text=True, + env=env, + timeout=10, + cwd=str(cwd) if cwd else None, + ) + parsed = json.loads(result.stdout) if result.stdout.strip() else {} + return result.returncode, parsed + + +def _dead_pid(tmp_path: Path) -> int: + """Return a PID that is guaranteed to not be running. + + Uses ``pid_max + 1`` — a value the Linux kernel can never assign — so no + PID-reuse race is possible regardless of xdist parallelism or process churn. + Falls back to spawn-and-reap on non-Linux platforms where ``/proc`` is absent. + """ + try: + return int(Path("/proc/sys/kernel/pid_max").read_text()) + 1 + except OSError: + proc = subprocess.Popen([sys.executable, "-c", "pass"]) + proc.wait() + return proc.pid + + +def test_mcp_health_advisor_dead_pid_injects_message(tmp_path: Path) -> None: + """Dead PID entry for current project → inject reconnection message.""" + dead_pid = _dead_pid(tmp_path) + kitchens = [ + { + "kitchen_id": "k1", + "pid": dead_pid, + "project_path": str(tmp_path), + "opened_at": "2026-01-01T00:00:00+00:00", + } + ] + returncode, payload = _run_guard( + tmp_path, {}, tool_name="Read", kitchens=kitchens, cwd=tmp_path, headless=False + ) + assert returncode == 0 + hook_out = payload.get("hookSpecificOutput", {}) + assert "/MCP" in hook_out.get("message", ""), ( + f"Expected /MCP reconnect hint in message, got: {hook_out}" + ) + + +def test_mcp_health_advisor_no_kitchens_silent(tmp_path: Path) -> None: + """No active_kitchens.json at all → silent exit 0.""" + returncode, payload = _run_guard(tmp_path, {}, tool_name="Bash") + assert returncode == 0 + assert payload == {}, f"Expected empty output, got: {payload}" + + +def test_mcp_health_advisor_alive_pid_silent(tmp_path: Path) -> None: + """Kitchen entry with alive PID → silent exit 0.""" + kitchens = [ + { + "kitchen_id": "k-alive", + "pid": os.getpid(), + "project_path": str(tmp_path), + "opened_at": "2026-01-01T00:00:00+00:00", + } + ] + returncode, payload = _run_guard( + tmp_path, {}, tool_name="Read", kitchens=kitchens, cwd=tmp_path + ) + assert returncode == 0 + assert payload == {}, f"Expected empty output for alive PID, got: {payload}" + + +def test_mcp_health_advisor_headless_bypass(tmp_path: Path) -> None: + """AUTOSKILLIT_HEADLESS=1 → message suppressed even with dead PID.""" + dead_pid = _dead_pid(tmp_path) + kitchens = [ + { + "kitchen_id": "k-headless", + "pid": dead_pid, + "project_path": str(tmp_path), + "opened_at": "2026-01-01T00:00:00+00:00", + } + ] + returncode, payload = _run_guard( + tmp_path, + {}, + tool_name="Read", + kitchens=kitchens, + cwd=tmp_path, + headless=True, + ) + assert returncode == 0 + assert payload == {}, f"Expected empty output in headless mode, got: {payload}" + + +def test_mcp_health_advisor_no_matching_project(tmp_path: Path) -> None: + """Kitchen entry for a different project_path → silent exit 0.""" + dead_pid = _dead_pid(tmp_path) + kitchens = [ + { + "kitchen_id": "k-other", + "pid": dead_pid, + "project_path": "/some/other/project", + "opened_at": "2026-01-01T00:00:00+00:00", + } + ] + # Run from tmp_path — project_path mismatch → no message + returncode, payload = _run_guard( + tmp_path, {}, tool_name="Read", kitchens=kitchens, cwd=tmp_path + ) + assert returncode == 0 + assert payload == {}, f"Expected empty output for non-matching project, got: {payload}" + + +def test_mcp_health_advisor_malformed_json_failopen(tmp_path: Path) -> None: + """Malformed active_kitchens.json → fail-open (exit 0, empty output).""" + home = tmp_path / "fakehome" + home.mkdir(exist_ok=True) + ak_dir = home / ".autoskillit" + ak_dir.mkdir(exist_ok=True) + (ak_dir / "active_kitchens.json").write_text("this is not valid JSON {{{{") + + hook_path = pkg_root() / "hooks" / "guards" / "mcp_health_advisor.py" + env = {**os.environ, "HOME": str(home)} + result = subprocess.run( + [sys.executable, str(hook_path)], + input=json.dumps({"tool_name": "Read"}), + capture_output=True, + text=True, + env=env, + timeout=10, + ) + assert result.returncode == 0 + assert not result.stdout.strip(), ( + f"Expected empty output on malformed JSON, got: {result.stdout!r}" + ) diff --git a/tests/infra/test_open_kitchen_guard.py b/tests/infra/test_open_kitchen_guard.py index 19ca2a6a9c..3f3daab14f 100644 --- a/tests/infra/test_open_kitchen_guard.py +++ b/tests/infra/test_open_kitchen_guard.py @@ -12,7 +12,7 @@ def _run_guard(env_extra: dict, tool_input: dict) -> dict: - hook_path = pkg_root() / "hooks" / "open_kitchen_guard.py" + hook_path = pkg_root() / "hooks" / "guards" / "open_kitchen_guard.py" stdin_payload = json.dumps({"tool_input": tool_input}) env = {**os.environ, **env_extra} result = subprocess.run( @@ -25,15 +25,51 @@ def _run_guard(env_extra: dict, tool_input: dict) -> dict: return json.loads(result.stdout) if result.stdout.strip() else {} -def test_open_kitchen_guard_denies_headless() -> None: - response = _run_guard({"AUTOSKILLIT_HEADLESS": "1"}, {}) +def test_open_kitchen_guard_denies_skill_tier() -> None: + response = _run_guard({"AUTOSKILLIT_HEADLESS": "1", "AUTOSKILLIT_SESSION_TYPE": "skill"}, {}) assert response["hookSpecificOutput"]["permissionDecision"] == "deny" - assert "headless" in response["hookSpecificOutput"]["permissionDecisionReason"].lower() + assert "skill" in response["hookSpecificOutput"]["permissionDecisionReason"].lower() + + +def test_open_kitchen_guard_denies_fleet_tier() -> None: + response = _run_guard({"AUTOSKILLIT_HEADLESS": "1", "AUTOSKILLIT_SESSION_TYPE": "fleet"}, {}) + assert response["hookSpecificOutput"]["permissionDecision"] == "deny" + assert "fleet" in response["hookSpecificOutput"]["permissionDecisionReason"].lower() + + +def test_open_kitchen_guard_permits_headless_orchestrator(tmp_path: Path) -> None: + hook_path = pkg_root() / "hooks" / "guards" / "open_kitchen_guard.py" + hook_input = { + "tool_name": "mcp__autoskillit__open_kitchen", + "tool_input": {"name": "my_recipe"}, + "session_id": "session-orch", + "hook_event_name": "PreToolUse", + } + env = { + **os.environ, + "AUTOSKILLIT_HEADLESS": "1", + "AUTOSKILLIT_SESSION_TYPE": "orchestrator", + "AUTOSKILLIT_STATE_DIR": str(tmp_path), + } + result = subprocess.run( + [sys.executable, str(hook_path)], + input=json.dumps(hook_input), + capture_output=True, + text=True, + env=env, + ) + assert result.returncode == 0 + if result.stdout.strip(): + payload = json.loads(result.stdout) + hook_out = payload.get("hookSpecificOutput", {}) + assert hook_out.get("permissionDecision") != "deny" + marker_path = tmp_path / "kitchen_state" / "session-orch.json" + assert marker_path.exists(), f"Marker not written at {marker_path}" def test_open_kitchen_guard_allows_human_session() -> None: env_without_headless = {k: v for k, v in os.environ.items() if k != "AUTOSKILLIT_HEADLESS"} - hook_path = pkg_root() / "hooks" / "open_kitchen_guard.py" + hook_path = pkg_root() / "hooks" / "guards" / "open_kitchen_guard.py" stdin_payload = json.dumps({"tool_input": {}}) result = subprocess.run( [sys.executable, str(hook_path)], @@ -59,7 +95,7 @@ def test_open_kitchen_guard_writes_marker_on_permit(tmp_path: Path, monkeypatch) "session_id": "session-abc", "hook_event_name": "PreToolUse", } - hook_path = pkg_root() / "hooks" / "open_kitchen_guard.py" + hook_path = pkg_root() / "hooks" / "guards" / "open_kitchen_guard.py" env = {k: v for k, v in os.environ.items() if k != "AUTOSKILLIT_HEADLESS"} env["AUTOSKILLIT_STATE_DIR"] = str(tmp_path) result = subprocess.run( @@ -85,8 +121,13 @@ def test_open_kitchen_guard_writes_marker_on_permit(tmp_path: Path, monkeypatch) def test_open_kitchen_guard_no_marker_on_deny(tmp_path: Path, monkeypatch) -> None: """When headless, the guard denies; no marker should be written.""" monkeypatch.setenv("AUTOSKILLIT_STATE_DIR", str(tmp_path)) - hook_path = pkg_root() / "hooks" / "open_kitchen_guard.py" - env = {**os.environ, "AUTOSKILLIT_HEADLESS": "1", "AUTOSKILLIT_STATE_DIR": str(tmp_path)} + hook_path = pkg_root() / "hooks" / "guards" / "open_kitchen_guard.py" + env = { + **os.environ, + "AUTOSKILLIT_HEADLESS": "1", + "AUTOSKILLIT_SESSION_TYPE": "skill", + "AUTOSKILLIT_STATE_DIR": str(tmp_path), + } result = subprocess.run( [sys.executable, str(hook_path)], input=json.dumps( @@ -105,3 +146,92 @@ def test_open_kitchen_guard_no_marker_on_deny(tmp_path: Path, monkeypatch) -> No payload = json.loads(result.stdout) assert payload["hookSpecificOutput"]["permissionDecision"] == "deny" assert not (tmp_path / "kitchen_state" / "session-abc.json").exists() + + +# --- Group P-3: Hook namespacing --- + + +def test_open_kitchen_guard_uses_campaign_namespace(tmp_path: Path, monkeypatch) -> None: + """open_kitchen_guard writes marker to campaign-namespaced directory.""" + monkeypatch.delenv("AUTOSKILLIT_STATE_DIR", raising=False) + monkeypatch.setenv("AUTOSKILLIT_CAMPAIGN_ID", "camp-77") + monkeypatch.chdir(tmp_path) + from autoskillit.hooks.guards.open_kitchen_guard import _write_kitchen_marker + + _write_kitchen_marker("sess-test", "my-recipe") + expected = tmp_path / ".autoskillit" / "temp" / "kitchen_state" / "camp-77" / "sess-test.json" + assert expected.exists() + + +def test_open_kitchen_guard_denies_fleet_headless() -> None: + response = _run_guard({"AUTOSKILLIT_HEADLESS": "1", "AUTOSKILLIT_SESSION_TYPE": "fleet"}, {}) + assert response["hookSpecificOutput"]["permissionDecision"] == "deny" + + +def test_open_kitchen_guard_fleet_denial_has_specific_message() -> None: + """Fleet denial message must mention fleet or franchise, not the generic skill message.""" + response = _run_guard({"AUTOSKILLIT_HEADLESS": "1", "AUTOSKILLIT_SESSION_TYPE": "fleet"}, {}) + reason = response["hookSpecificOutput"]["permissionDecisionReason"].lower() + assert "fleet" in reason or "franchise" in reason + + +def test_guard_bridges_launch_id_to_registry(tmp_path: Path) -> None: + """open_kitchen_guard bridges AUTOSKILLIT_LAUNCH_ID to claude_session_id in registry.""" + from autoskillit.core.runtime.session_registry import read_registry, write_registry_entry + + project_dir = tmp_path + write_registry_entry(project_dir, "abc", "cook", None) + assert read_registry(project_dir)["abc"]["claude_session_id"] is None + + hook_path = pkg_root() / "hooks" / "guards" / "open_kitchen_guard.py" + hook_input = { + "tool_name": "mcp__autoskillit__open_kitchen", + "tool_input": {}, + "session_id": "claude-xyz", + "hook_event_name": "PreToolUse", + } + env_without_headless = {k: v for k, v in os.environ.items() if k != "AUTOSKILLIT_HEADLESS"} + env_without_headless["AUTOSKILLIT_LAUNCH_ID"] = "abc" + result = subprocess.run( + [sys.executable, str(hook_path)], + input=json.dumps(hook_input), + capture_output=True, + text=True, + env=env_without_headless, + cwd=str(project_dir), + ) + assert result.returncode == 0, f"Hook failed: {result.stderr}" + registry = read_registry(project_dir) + assert registry["abc"]["claude_session_id"] == "claude-xyz" + + +def test_guard_bridge_no_op_when_no_launch_id(tmp_path: Path) -> None: + """open_kitchen_guard bridge is a no-op when AUTOSKILLIT_LAUNCH_ID is not set.""" + from autoskillit.core.runtime.session_registry import read_registry, write_registry_entry + + project_dir = tmp_path + write_registry_entry(project_dir, "abc", "cook", None) + + hook_path = pkg_root() / "hooks" / "guards" / "open_kitchen_guard.py" + hook_input = { + "tool_name": "mcp__autoskillit__open_kitchen", + "tool_input": {}, + "session_id": "claude-xyz", + "hook_event_name": "PreToolUse", + } + env_without = { + k: v + for k, v in os.environ.items() + if k not in ("AUTOSKILLIT_HEADLESS", "AUTOSKILLIT_LAUNCH_ID") + } + result = subprocess.run( + [sys.executable, str(hook_path)], + input=json.dumps(hook_input), + capture_output=True, + text=True, + env=env_without, + cwd=str(project_dir), + ) + assert result.returncode == 0, f"Hook failed: {result.stderr}" + registry = read_registry(project_dir) + assert registry["abc"]["claude_session_id"] is None diff --git a/tests/infra/test_planner_gh_discovery_guard.py b/tests/infra/test_planner_gh_discovery_guard.py new file mode 100644 index 0000000000..38c2de0309 --- /dev/null +++ b/tests/infra/test_planner_gh_discovery_guard.py @@ -0,0 +1,223 @@ +"""Tests for the planner_gh_discovery_guard PreToolUse hook. + +Guards against GitHub discovery commands (gh issue list, gh pr list, gh search, +gh api listing endpoints) in planner skill sessions while allowing targeted +reads (gh issue view , gh api .../issues/). +""" + +from __future__ import annotations + +import contextlib +import io +import json +import os +import unittest.mock + +import pytest + +pytestmark = [pytest.mark.layer("infra"), pytest.mark.small] + +_BASH_TOOL = "Bash" +_RUN_CMD_TOOL = "mcp__autoskillit__local__autoskillit__run_cmd" + + +def _run_guard( + cmd: str, + *, + tool_type: str = "bash", + skill_name: str = "planner-generate-phases", + headless: bool = True, + raw_stdin: str | None = None, +) -> str: + """Invoke planner_gh_discovery_guard.main() and return captured stdout.""" + from autoskillit.hooks.guards.planner_gh_discovery_guard import main # noqa: PLC0415 + + if raw_stdin is not None: + stdin_content = raw_stdin + else: + if tool_type == "bash": + tool_name = _BASH_TOOL + tool_input = {"command": cmd} + else: + tool_name = _RUN_CMD_TOOL + tool_input = {"cmd": cmd} + stdin_content = json.dumps({"tool_name": tool_name, "tool_input": tool_input}) + + env_patch: dict[str, str] = {} + env_remove: list[str] = [] + if headless: + env_patch["AUTOSKILLIT_HEADLESS"] = "1" + else: + env_remove.append("AUTOSKILLIT_HEADLESS") + if skill_name: + env_patch["AUTOSKILLIT_SKILL_NAME"] = skill_name + else: + env_remove.append("AUTOSKILLIT_SKILL_NAME") + + clean_env = {k: v for k, v in os.environ.items() if k not in env_remove} + clean_env.update(env_patch) + + buf = io.StringIO() + with contextlib.redirect_stdout(buf): + with unittest.mock.patch("sys.stdin", io.StringIO(stdin_content)): + with unittest.mock.patch.dict(os.environ, clean_env, clear=True): + try: + main() + except SystemExit as exc: + assert exc.code == 0, f"Guard exited non-zero: {exc.code!r}" + + return buf.getvalue() + + +def _is_denied(output: str) -> bool: + if not output: + return False + data = json.loads(output) + return data.get("hookSpecificOutput", {}).get("permissionDecision") == "deny" + + +# --------------------------------------------------------------------------- +# Group 1: Planner session — discovery commands DENIED +# --------------------------------------------------------------------------- + + +class TestPlannerDiscoveryDenied: + def test_denies_gh_issue_list(self): + out = _run_guard("gh issue list") + assert _is_denied(out) + + def test_denies_gh_issue_list_with_flags(self): + out = _run_guard("gh issue list --state open --label bug") + assert _is_denied(out) + + def test_denies_gh_pr_list(self): + out = _run_guard("gh pr list") + assert _is_denied(out) + + def test_denies_gh_search_issues(self): + out = _run_guard('gh search issues "query"') + assert _is_denied(out) + + def test_denies_gh_search_prs(self): + out = _run_guard('gh search prs "query"') + assert _is_denied(out) + + def test_denies_gh_api_issues_listing(self): + out = _run_guard("gh api /repos/owner/repo/issues") + assert _is_denied(out) + + def test_denies_gh_api_pulls_listing(self): + out = _run_guard("gh api /repos/owner/repo/pulls") + assert _is_denied(out) + + def test_denies_via_run_cmd(self): + out = _run_guard("gh issue list", tool_type="run_cmd") + assert _is_denied(out) + + def test_denies_gh_in_pipeline(self): + out = _run_guard("echo foo && gh issue list") + assert _is_denied(out) + + @pytest.mark.parametrize( + "skill", + ["planner-analyze", "planner-elaborate-phase", "planner-refine"], + ) + def test_denies_all_planner_prefixes(self, skill: str): + out = _run_guard("gh issue list", skill_name=skill) + assert _is_denied(out) + + +# --------------------------------------------------------------------------- +# Group 2: Planner session — targeted reads ALLOWED +# --------------------------------------------------------------------------- + + +class TestPlannerTargetedAllowed: + def test_allows_gh_issue_view(self): + out = _run_guard("gh issue view 1625") + assert out.strip() == "" + + def test_allows_gh_pr_view(self): + out = _run_guard("gh pr view 42") + assert out.strip() == "" + + def test_allows_gh_api_specific_issue(self): + out = _run_guard("gh api /repos/owner/repo/issues/123") + assert out.strip() == "" + + def test_allows_gh_api_specific_pull(self): + out = _run_guard("gh api /repos/owner/repo/pulls/456") + assert out.strip() == "" + + def test_allows_gh_issue_view_with_json(self): + out = _run_guard("gh issue view 42 --json title,body") + assert out.strip() == "" + + +# --------------------------------------------------------------------------- +# Group 3: Non-planner session — everything ALLOWED +# --------------------------------------------------------------------------- + + +class TestNonPlannerAllowed: + def test_allows_when_skill_name_unset(self): + out = _run_guard("gh issue list", skill_name="", headless=True) + assert out.strip() == "" + + def test_allows_when_skill_is_investigate(self): + out = _run_guard("gh issue list", skill_name="investigate", headless=True) + assert out.strip() == "" + + def test_allows_when_skill_is_triage_issues(self): + out = _run_guard("gh issue list", skill_name="triage-issues", headless=True) + assert out.strip() == "" + + +# --------------------------------------------------------------------------- +# Group 4: Edge cases +# --------------------------------------------------------------------------- + + +class TestEdgeCases: + def test_allows_non_gh_command(self): + out = _run_guard("ls -la") + assert out.strip() == "" + + def test_allows_malformed_stdin(self): + out = _run_guard("", raw_stdin="not-json{{{") + assert out.strip() == "" + + def test_allows_empty_command(self): + out = _run_guard("") + assert out.strip() == "" + + def test_denies_gh_api_issues_with_query_params(self): + out = _run_guard("gh api /repos/o/r/issues?state=open") + assert _is_denied(out) + + def test_allows_gh_api_non_issues_endpoint(self): + out = _run_guard("gh api /repos/o/r/commits") + assert out.strip() == "" + + +# --------------------------------------------------------------------------- +# Group 5: Headless gate +# --------------------------------------------------------------------------- + + +class TestHeadlessGate: + def test_allows_when_not_headless(self): + out = _run_guard( + "gh issue list", + skill_name="planner-generate-phases", + headless=False, + ) + assert out.strip() == "" + + def test_denies_when_headless_and_planner(self): + out = _run_guard( + "gh issue list", + skill_name="planner-generate-phases", + headless=True, + ) + assert _is_denied(out) diff --git a/tests/infra/test_pr_create_guard.py b/tests/infra/test_pr_create_guard.py new file mode 100644 index 0000000000..560439add8 --- /dev/null +++ b/tests/infra/test_pr_create_guard.py @@ -0,0 +1,186 @@ +"""Tests for the pr_create_guard PreToolUse hook. + +Guards against `gh pr create` being issued via run_cmd when the kitchen is +open, enforcing the mandatory prepare_pr → compose_pr pipeline. + +Pattern mirrors test_unsafe_install_guard.py exactly. +""" + +from __future__ import annotations + +import contextlib +import io +import json +import unittest.mock + +import pytest + +pytestmark = [pytest.mark.layer("infra"), pytest.mark.small] + +_TOOL_NAME = "mcp__autoskillit__local__autoskillit__run_cmd" +_BASH_TOOL_NAME = "Bash" +_HOOK_CONFIG_RELPATH = ".autoskillit/temp/.hook_config.json" + + +def _run_guard(cmd: str, kitchen_open: bool, tmpdir, raw_stdin: str | None = None) -> str: + """Invoke pr_create_guard.main() and return captured stdout.""" + from autoskillit.hooks.guards.pr_create_guard import main # noqa: PLC0415 + + if raw_stdin is not None: + stdin_content = raw_stdin + else: + tool_input = {"cmd": cmd, "cwd": str(tmpdir)} + stdin_payload = {"tool_name": _TOOL_NAME, "tool_input": tool_input} + stdin_content = json.dumps(stdin_payload) + + if kitchen_open: + hook_cfg = tmpdir / _HOOK_CONFIG_RELPATH + hook_cfg.parent.mkdir(parents=True, exist_ok=True) + hook_cfg.write_text(json.dumps({"kitchen": "open"})) + + buf = io.StringIO() + with contextlib.redirect_stdout(buf): + with unittest.mock.patch("sys.stdin", io.StringIO(stdin_content)): + with unittest.mock.patch("pathlib.Path.cwd", return_value=tmpdir): + try: + main() + except SystemExit as exc: + assert exc.code == 0, f"Guard exited non-zero: {exc.code!r}" + + return buf.getvalue() + + +def _run_bash_guard(cmd: str, kitchen_open: bool, tmpdir, raw_stdin: str | None = None) -> str: + """Invoke pr_create_guard.main() with Bash tool format and return captured stdout.""" + from autoskillit.hooks.guards.pr_create_guard import main # noqa: PLC0415 + + if raw_stdin is not None: + stdin_content = raw_stdin + else: + tool_input = {"command": cmd, "cwd": str(tmpdir)} + stdin_payload = {"tool_name": _BASH_TOOL_NAME, "tool_input": tool_input} + stdin_content = json.dumps(stdin_payload) + + if kitchen_open: + hook_cfg = tmpdir / _HOOK_CONFIG_RELPATH + hook_cfg.parent.mkdir(parents=True, exist_ok=True) + hook_cfg.write_text(json.dumps({"kitchen": "open"})) + + buf = io.StringIO() + with contextlib.redirect_stdout(buf): + with unittest.mock.patch("sys.stdin", io.StringIO(stdin_content)): + with unittest.mock.patch("pathlib.Path.cwd", return_value=tmpdir): + try: + main() + except SystemExit as exc: + assert exc.code == 0, f"Guard exited non-zero: {exc.code!r}" + + return buf.getvalue() + + +def _is_denied(output: str) -> bool: + if not output: + return False + data = json.loads(output) + return data.get("hookSpecificOutput", {}).get("permissionDecision") == "deny" + + +# --------------------------------------------------------------------------- +# Denied cases +# --------------------------------------------------------------------------- + + +class TestPrCreateGuardDenied: + def test_denies_gh_pr_create_when_kitchen_open(self, tmp_path): + out = _run_guard("gh pr create --title foo --body bar", kitchen_open=True, tmpdir=tmp_path) + assert _is_denied(out) + + def test_deny_reason_mentions_pipeline(self, tmp_path): + out = _run_guard("gh pr create --title foo", kitchen_open=True, tmpdir=tmp_path) + data = json.loads(out) + reason = data["hookSpecificOutput"]["permissionDecisionReason"] + assert "prepare_pr" in reason, "Deny reason must name the mandatory pipeline step" + + def test_denies_gh_pr_create_with_extra_flags(self, tmp_path): + out = _run_guard( + "gh pr create -t 'fix' -b 'desc' --base main", + kitchen_open=True, + tmpdir=tmp_path, + ) + assert _is_denied(out) + + def test_denies_gh_pr_create_with_leading_whitespace(self, tmp_path): + out = _run_guard(" gh pr create --title x", kitchen_open=True, tmpdir=tmp_path) + assert _is_denied(out) + + +# --------------------------------------------------------------------------- +# Allowed cases +# --------------------------------------------------------------------------- + + +class TestBashToolDenyPath: + """Bash tool sends command via 'command' key — guard must handle this format.""" + + def test_denies_gh_pr_create_via_bash_tool(self, tmp_path): + out = _run_bash_guard( + "gh pr create --title foo --body bar", kitchen_open=True, tmpdir=tmp_path + ) + assert _is_denied(out) + + def test_denies_gh_pr_create_with_flags_via_bash_tool(self, tmp_path): + out = _run_bash_guard( + "gh pr create -t 'fix' -b 'desc'", kitchen_open=True, tmpdir=tmp_path + ) + assert _is_denied(out) + + def test_allows_non_pr_create_via_bash_tool(self, tmp_path): + out = _run_bash_guard("gh pr list", kitchen_open=True, tmpdir=tmp_path) + assert out.strip() == "" + + def test_allows_when_kitchen_closed_via_bash_tool(self, tmp_path): + out = _run_bash_guard("gh pr create --title foo", kitchen_open=False, tmpdir=tmp_path) + assert out.strip() == "" + + +class TestPrCreateGuardAllowed: + def test_allows_when_kitchen_closed(self, tmp_path): + out = _run_guard("gh pr create --title foo", kitchen_open=False, tmpdir=tmp_path) + assert out.strip() == "", "No output means allow" + + def test_allows_non_pr_create_commands(self, tmp_path): + out = _run_guard("gh pr list", kitchen_open=True, tmpdir=tmp_path) + assert out.strip() == "" + + def test_allows_gh_pr_merge_unchanged(self, tmp_path): + # gh pr merge is a separate concern; this guard must not over-block + out = _run_guard("gh pr merge --squash 42", kitchen_open=True, tmpdir=tmp_path) + assert out.strip() == "" + + def test_allows_gh_pr_view(self, tmp_path): + out = _run_guard("gh pr view 99", kitchen_open=True, tmpdir=tmp_path) + assert out.strip() == "" + + def test_allows_unrelated_run_cmd(self, tmp_path): + out = _run_guard("npm run build", kitchen_open=True, tmpdir=tmp_path) + assert out.strip() == "" + + def test_allows_unrelated_git_command(self, tmp_path): + out = _run_guard("git status", kitchen_open=True, tmpdir=tmp_path) + assert out.strip() == "" + + +# --------------------------------------------------------------------------- +# Edge cases: fail-open on malformed input +# --------------------------------------------------------------------------- + + +class TestPrCreateGuardEdgeCases: + def test_fails_open_on_malformed_stdin(self, tmp_path): + out = _run_guard("", kitchen_open=False, tmpdir=tmp_path, raw_stdin="not-json{{{") + assert out.strip() == "", "Malformed JSON must fail open (no output = allow)" + + def test_fails_open_on_missing_cmd_field(self, tmp_path): + stdin = json.dumps({"tool_name": _TOOL_NAME, "tool_input": {}}) + out = _run_guard("", kitchen_open=False, tmpdir=tmp_path, raw_stdin=stdin) + assert out.strip() == "" diff --git a/tests/infra/test_pretty_output.py b/tests/infra/test_pretty_output.py deleted file mode 100644 index 87373758ed..0000000000 --- a/tests/infra/test_pretty_output.py +++ /dev/null @@ -1,1871 +0,0 @@ -"""Tests for the pretty_output PostToolUse hook. - -The hook reformats raw MCP tool JSON responses into Markdown-KV format -before Claude consumes them. Fails open on any error. -""" - -from __future__ import annotations - -import io -import json -from contextlib import ExitStack, redirect_stdout -from pathlib import Path -from unittest.mock import patch - -from autoskillit.core.types import ChannelConfirmation, TerminationReason -from autoskillit.execution.headless import _build_skill_result -from autoskillit.hooks._fmt_primitives import _HOOK_CONFIG_PATH_COMPONENTS -from autoskillit.hooks.pretty_output_hook import _format_response -from tests.conftest import _make_result - -# Realistic recipe YAML that mirrors actual bundled recipes. -# Must include an `ingredients:` block to reproduce the raw/derived duplication scenario. -# All formatter tests using `content` should reference this constant. -REALISTIC_RECIPE_YAML = """\ -name: implementation -description: Full implementation pipeline -autoskillit_version: "0.3.0" -ingredients: - task: - description: What to implement - required: true - source_dir: - description: Path to source directory - default: "" - review_approach: - description: Run review-approach before planning - default: "false" -kitchen_rules: - - Always commit before merging -steps: - implement: - tool: run_skill - skill_command: /implement -""" - - -def _run_hook( - event: dict | None = None, - raw_stdin: str | None = None, - cwd: Path | None = None, -) -> tuple[str, int]: - """Run pretty_output.main() with synthetic stdin. - - Returns (stdout_output, exit_code). - """ - from autoskillit.hooks.pretty_output_hook import main - - stdin_text = raw_stdin if raw_stdin is not None else json.dumps(event or {}) - - exit_code = 0 - buf = io.StringIO() - - with ExitStack() as stack: - stack.enter_context(patch("sys.stdin", io.StringIO(stdin_text))) - stack.enter_context(redirect_stdout(buf)) - if cwd is not None: - stack.enter_context( - patch("autoskillit.hooks.pretty_output_hook.Path.cwd", return_value=cwd) - ) - try: - main() - except SystemExit as e: - exit_code = e.code if e.code is not None else 0 - - return buf.getvalue(), exit_code - - -def _make_run_skill_event( - success: bool = True, - result: str = "Done.", - session_id: str = "abc", - subtype: str = "end_turn", - is_error: bool = False, - exit_code: int = 0, - needs_retry: bool = False, - retry_reason: str = "none", - stderr: str = "", - token_usage: dict | None = None, - worktree_path: str = "", -) -> dict: - return { - "tool_name": "mcp__plugin_autoskillit_autoskillit__run_skill", - "tool_response": json.dumps( - { - "success": success, - "result": result, - "session_id": session_id, - "subtype": subtype, - "is_error": is_error, - "exit_code": exit_code, - "needs_retry": needs_retry, - "retry_reason": retry_reason, - "stderr": stderr, - "token_usage": token_usage, - "worktree_path": worktree_path, - } - ), - } - - -# PHK-1 -def test_hook_script_exists(): - """pretty_output.py must exist in the hooks directory.""" - from autoskillit.core.paths import pkg_root - - assert (pkg_root() / "hooks" / "pretty_output_hook.py").exists() - - -# PHK-2 -def test_hook_emits_posttooluse_event_name(): - """Hook output JSON must have hookSpecificOutput.hookEventName == 'PostToolUse'.""" - event = { - "tool_name": "mcp__plugin_autoskillit_autoskillit__run_cmd", - "tool_response": json.dumps( - {"success": True, "exit_code": 0, "stdout": "hi", "stderr": ""} - ), - } - out, _ = _run_hook(event=event) - assert out.strip(), "Expected non-empty output" - data = json.loads(out) - assert data["hookSpecificOutput"]["hookEventName"] == "PostToolUse" - - -# PHK-3 -def test_hook_emits_updated_mcp_tool_output_field(): - """Hook output must have non-empty hookSpecificOutput.updatedMCPToolOutput.""" - event = { - "tool_name": "mcp__plugin_autoskillit_autoskillit__run_cmd", - "tool_response": json.dumps( - {"success": True, "exit_code": 0, "stdout": "hi", "stderr": ""} - ), - } - out, _ = _run_hook(event=event) - data = json.loads(out) - assert data["hookSpecificOutput"]["updatedMCPToolOutput"] - - -# PHK-4 -def test_hook_fail_open_on_invalid_json_stdin(): - """Non-JSON stdin → exit 0, no stdout output.""" - out, code = _run_hook(raw_stdin="not valid json {{{{") - assert code == 0 - assert out.strip() == "" - - -# PHK-5 -def test_hook_fail_open_on_missing_tool_response(): - """Valid JSON but missing tool_response key → exit 0, no stdout.""" - event = {"tool_name": "mcp__plugin_autoskillit_autoskillit__run_cmd"} - out, code = _run_hook(event=event) - assert code == 0 - assert out.strip() == "" - - -# PHK-6 -def test_format_run_skill_success(tmp_path): - """run_skill success response must contain tool name, checkmark, and success field.""" - event = _make_run_skill_event(success=True) - out, _ = _run_hook(event=event, cwd=tmp_path) - text = json.loads(out)["hookSpecificOutput"]["updatedMCPToolOutput"] - assert "run_skill" in text - assert "\u2713" in text # ✓ - assert "success: True" in text - - -# PHK-7 -def test_format_run_skill_failure_with_retry(tmp_path): - """run_skill failure with retry must show cross, retry fields, and worktree_path.""" - event = _make_run_skill_event( - success=False, - needs_retry=True, - retry_reason="budget_exhausted", - worktree_path="/tmp/wt/fix-abc", - ) - out, _ = _run_hook(event=event, cwd=tmp_path) - text = json.loads(out)["hookSpecificOutput"]["updatedMCPToolOutput"] - assert "\u2717" in text # ✗ - assert "needs_retry: True" in text - assert "retry_reason: budget_exhausted" in text - assert "worktree_path: /tmp/wt/fix-abc" in text - - -# PHK-8 -def test_format_run_skill_gate_error(): - """run_skill gate_error subtype must show 'Gate Error' and 'gate_error'.""" - event = { - "tool_name": "mcp__plugin_autoskillit_autoskillit__run_skill", - "tool_response": json.dumps( - { - "success": False, - "subtype": "gate_error", - "is_error": True, - "result": "Kitchen is closed.", - } - ), - } - out, _ = _run_hook(event=event) - text = json.loads(out)["hookSpecificOutput"]["updatedMCPToolOutput"] - assert "Gate Error" in text - assert "gate_error" in text - - -# PHK-9 -def test_format_run_cmd_success_shows_stdout(tmp_path): - """run_cmd success must show tool name, checkmark, stdout content.""" - event = { - "tool_name": "mcp__plugin_autoskillit_autoskillit__run_cmd", - "tool_response": json.dumps( - {"success": True, "exit_code": 0, "stdout": "hello\nworld\n", "stderr": ""} - ), - } - out, _ = _run_hook(event=event, cwd=tmp_path) - text = json.loads(out)["hookSpecificOutput"]["updatedMCPToolOutput"] - assert "run_cmd" in text - assert "\u2713" in text # ✓ - assert "stdout" in text - assert "hello" in text - assert "world" in text - - -# PHK-10 -def test_format_run_cmd_failure_shows_stderr(tmp_path): - """run_cmd failure must show cross and stderr content.""" - event = { - "tool_name": "mcp__plugin_autoskillit_autoskillit__run_cmd", - "tool_response": json.dumps( - {"success": False, "exit_code": 1, "stdout": "", "stderr": "No such file"} - ), - } - out, _ = _run_hook(event=event, cwd=tmp_path) - text = json.loads(out)["hookSpecificOutput"]["updatedMCPToolOutput"] - assert "\u2717" in text # ✗ - assert "stderr" in text - assert "No such file" in text - - -# PHK-11 -def test_format_test_check_pass(): - """test_check pass must show tool name, checkmark, PASS, passed: True.""" - event = { - "tool_name": "mcp__plugin_autoskillit_autoskillit__test_check", - "tool_response": json.dumps({"passed": True, "stdout": "...245 passed..."}), - } - out, _ = _run_hook(event=event) - text = json.loads(out)["hookSpecificOutput"]["updatedMCPToolOutput"] - assert "test_check" in text - assert "\u2713" in text # ✓ - assert "PASS" in text - assert "passed: True" in text - - -# PHK-12 -def test_format_test_check_fail_shows_failure_lines(): - """test_check fail must show failure lines and filter pytest boilerplate.""" - pytest_output = ( - "platform linux -- Python 3.13.2\n" - "rootdir: /repo\n" - "collecting ...\n" - "collected 100 items\n" - "FAILED tests/foo.py::test_a - AssertionError: 1 != 2\n" - "FAILED tests/bar.py::test_b - RuntimeError: oops\n" - "2 failed, 98 passed\n" - ) - event = { - "tool_name": "mcp__plugin_autoskillit_autoskillit__test_check", - "tool_response": json.dumps({"passed": False, "stdout": pytest_output}), - } - out, _ = _run_hook(event=event) - text = json.loads(out)["hookSpecificOutput"]["updatedMCPToolOutput"] - assert "\u2717" in text # ✗ - assert "FAIL" in text - assert "FAILED tests/foo.py::test_a" in text - assert "FAILED tests/bar.py::test_b" in text - assert "platform linux" not in text - assert "collecting ..." not in text - - -# PHK-13 -def test_format_merge_worktree_failure(): - """merge_worktree failure must include all key fields.""" - event = { - "tool_name": "mcp__plugin_autoskillit_autoskillit__merge_worktree", - "tool_response": json.dumps( - { - "error": "Rebase failed", - "failed_step": "rebase", - "state": "worktree_dirty", - "worktree_path": "/tmp/wt/fix", - "stderr": "CONFLICT (content): Merge conflict in foo.py", - } - ), - } - out, _ = _run_hook(event=event) - text = json.loads(out)["hookSpecificOutput"]["updatedMCPToolOutput"] - assert "merge_worktree" in text - assert "\u2717" in text # ✗ - assert "error: Rebase failed" in text - assert "failed_step: rebase" in text - assert "state: worktree_dirty" in text - assert "CONFLICT" in text - - -# PHK-14 -def test_format_merge_worktree_success(): - """merge_worktree success must show checkmark and merge_succeeded: True.""" - event = { - "tool_name": "mcp__plugin_autoskillit_autoskillit__merge_worktree", - "tool_response": json.dumps( - { - "merge_succeeded": True, - "merged_branch": "impl-fix-20260101", - "into_branch": "main", - "worktree_removed": True, - "branch_deleted": True, - } - ), - } - out, _ = _run_hook(event=event) - text = json.loads(out)["hookSpecificOutput"]["updatedMCPToolOutput"] - assert "merge_worktree" in text - assert "\u2713" in text # ✓ - assert "merge_succeeded: True" in text - - -# PHK-15 -def test_format_get_token_summary_compact(): - """get_token_summary must show compact per-step lines and totals.""" - event = { - "tool_name": "mcp__plugin_autoskillit_autoskillit__get_token_summary", - "tool_response": json.dumps( - { - "steps": [ - { - "step_name": "investigate", - "invocation_count": 1, - "input_tokens": 45200, - "output_tokens": 12800, - "cache_read_input_tokens": 1200000, - "cache_creation_input_tokens": 0, - }, - { - "step_name": "make_plan", - "invocation_count": 2, - "input_tokens": 30000, - "output_tokens": 8000, - "cache_read_input_tokens": 0, - "cache_creation_input_tokens": 500000, - }, - { - "step_name": "implement", - "invocation_count": 1, - "input_tokens": 60000, - "output_tokens": 15000, - "cache_read_input_tokens": 2000000, - "cache_creation_input_tokens": 0, - }, - ], - "total": { - "input_tokens": 135200, - "output_tokens": 35800, - "cache_read_input_tokens": 3200000, - "cache_creation_input_tokens": 500000, - }, - } - ), - } - out, _ = _run_hook(event=event) - text = json.loads(out)["hookSpecificOutput"]["updatedMCPToolOutput"] - assert "token_summary" in text - # Assert specific compact line format: name xN [uc:Xk out:Xk cr:XM cw:XM t:Xs] - assert "investigate x1 [uc:45.2k out:12.8k cr:1.2M cw:0 t:0.0s]" in text - assert "make_plan x2 [uc:30.0k out:8.0k cr:0 cw:500.0k t:0.0s]" in text - assert "implement x1 [uc:60.0k out:15.0k cr:2.0M cw:0 t:0.0s]" in text - assert "total_uncached:" in text - assert "total_out:" in text - assert "total_cache_read:" in text - assert "total_cache_write:" in text - - -# PHK-16 -def test_format_kitchen_status(): - """kitchen_status must show package_version and tools_enabled.""" - event = { - "tool_name": "mcp__plugin_autoskillit_autoskillit__kitchen_status", - "tool_response": json.dumps( - { - "package_version": "0.2.0", - "plugin_json_version": "0.2.0", - "versions_match": True, - "tools_enabled": True, - "token_usage_verbosity": "summary", - "quota_guard_enabled": True, - "github_token_configured": True, - "github_default_repo": "acme/myrepo", - } - ), - } - out, _ = _run_hook(event=event) - text = json.loads(out)["hookSpecificOutput"]["updatedMCPToolOutput"] - assert "kitchen_status" in text - assert "package_version: 0.2.0" in text - assert "tools_enabled: True" in text - - -# PHK-17 -def test_pipeline_mode_compact_run_skill(tmp_path): - """In pipeline mode (hook config present), run_skill output uses compact format.""" - # Create the hook config file to signal pipeline mode - config_path = tmp_path.joinpath(*_HOOK_CONFIG_PATH_COMPONENTS) - config_path.parent.mkdir(parents=True, exist_ok=True) - config_path.write_text('{"quota_guard": {}}') - - event = _make_run_skill_event(success=False, needs_retry=True, retry_reason="budget_exhausted") - - out, _ = _run_hook(event=event, cwd=tmp_path) - - text = json.loads(out)["hookSpecificOutput"]["updatedMCPToolOutput"] - # Compact format: single-line header, not full ## header - assert "run_skill:" in text - assert "FAIL" in text - # Should not have the full interactive ## header format - assert "## run_skill" not in text - - -# PHK-18 -def test_unknown_tool_passes_through(): - """Unknown tool name gets generic key-value rendering.""" - event = { - "tool_name": "mcp__plugin_autoskillit_autoskillit__some_unknown_tool", - "tool_response": json.dumps({"result": "ok", "status": "done"}), - } - out, code = _run_hook(event=event) - assert code == 0 - # Should produce output (generic formatter) - assert out.strip() - text = json.loads(out)["hookSpecificOutput"]["updatedMCPToolOutput"] - assert "some_unknown_tool" in text - - -# PHK-19 -def test_fmt_run_cmd_tool_exception_shows_diagnostic(): - """run_cmd tool_exception subtype must show error, not FAIL [].""" - event = { - "tool_name": "mcp__plugin_autoskillit_autoskillit__run_cmd", - "tool_response": json.dumps( - { - "error": "RuntimeError: boom", - "exit_code": -1, - "subtype": "tool_exception", - "success": False, - } - ), - } - out, _ = _run_hook(event=event) - text = json.loads(out)["hookSpecificOutput"]["updatedMCPToolOutput"] - assert "tool exception" in text.lower() - assert "RuntimeError: boom" in text - assert "FAIL []" not in text - - -# PHK-20 -def test_fmt_clone_repo_uncommitted_changes_warning(): - """clone_repo uncommitted_changes must show WARNING, not OK.""" - event = { - "tool_name": "mcp__plugin_autoskillit_autoskillit__clone_repo", - "tool_response": json.dumps( - { - "uncommitted_changes": "true", - "source_dir": "/src", - "branch": "main", - "changed_files": "M file.py", - "total_changed": "1", - } - ), - } - out, _ = _run_hook(event=event) - text = json.loads(out)["hookSpecificOutput"]["updatedMCPToolOutput"] - assert "uncommitted_changes" in text - assert "changed_files" in text - assert "\u2713 OK" not in text - - -# PHK-21 -def test_fmt_clone_repo_unpublished_branch_warning(): - """clone_repo unpublished_branch must show WARNING, not OK.""" - event = { - "tool_name": "mcp__plugin_autoskillit_autoskillit__clone_repo", - "tool_response": json.dumps( - { - "unpublished_branch": "true", - "branch": "feat/x", - "source_dir": "/src", - } - ), - } - out, _ = _run_hook(event=event) - text = json.loads(out)["hookSpecificOutput"]["updatedMCPToolOutput"] - assert "unpublished_branch" in text - assert "branch" in text - assert "\u2713 OK" not in text - - -# PHK-22 -def test_fmt_clone_repo_success_includes_clone_path(): - """clone_repo success must include clone_path, source_dir, remote_url.""" - event = { - "tool_name": "mcp__plugin_autoskillit_autoskillit__clone_repo", - "tool_response": json.dumps( - { - "clone_path": "/tmp/clone", - "source_dir": "/src", - "remote_url": "https://example.com", - } - ), - } - out, _ = _run_hook(event=event) - text = json.loads(out)["hookSpecificOutput"]["updatedMCPToolOutput"] - assert "/tmp/clone" in text - assert "/src" in text - assert "https://example.com" in text - - -# PHK-23 -def test_fmt_clone_repo_remote_url_failure_includes_stderr(): - """clone_repo remote_url failure must include stderr.""" - event = { - "tool_name": "mcp__plugin_autoskillit_autoskillit__clone_repo", - "tool_response": json.dumps( - { - "error": "remote_url_rewrite_failed", - "clone_path": "/tmp/clone", - "remote_url": "https://example.com", - "stderr": "fatal: unable to set url", - } - ), - } - out, _ = _run_hook(event=event) - text = json.loads(out)["hookSpecificOutput"]["updatedMCPToolOutput"] - assert "fatal: unable to set url" in text - - -# PHK-24 -def test_fmt_merge_worktree_success_shows_metadata(): - """merge_worktree success must show merge_succeeded and merged_branch.""" - event = { - "tool_name": "mcp__plugin_autoskillit_autoskillit__merge_worktree", - "tool_response": json.dumps( - { - "merge_succeeded": True, - "merged_branch": "feat/x", - "into_branch": "main", - "worktree_removed": True, - "branch_deleted": True, - } - ), - } - out, _ = _run_hook(event=event) - text = json.loads(out)["hookSpecificOutput"]["updatedMCPToolOutput"] - assert "merge_succeeded" in text - assert "merged_branch" in text - assert "\u2713" in text - - -# PHK-25 -def test_fmt_merge_worktree_dirty_tree_shows_files(): - """merge_worktree dirty tree must show dirty_files content.""" - event = { - "tool_name": "mcp__plugin_autoskillit_autoskillit__merge_worktree", - "tool_response": json.dumps( - { - "error": "dirty working tree", - "state": "DIRTY_TREE", - "dirty_files": ["M a.py", "M b.py"], - } - ), - } - out, _ = _run_hook(event=event) - text = json.loads(out)["hookSpecificOutput"]["updatedMCPToolOutput"] - assert "M a.py" in text - assert "M b.py" in text - - -# PHK-26 -def test_fmt_run_skill_pipeline_includes_stderr(tmp_path): - """run_skill pipeline mode must include stderr.""" - config_path = tmp_path.joinpath(*_HOOK_CONFIG_PATH_COMPONENTS) - config_path.parent.mkdir(parents=True, exist_ok=True) - config_path.write_text('{"quota_guard": {}}') - - event = _make_run_skill_event( - success=False, - subtype="execution_failed", - stderr="ImportError: no module named foo", - exit_code=1, - result="", - ) - out, _ = _run_hook(event=event, cwd=tmp_path) - text = json.loads(out)["hookSpecificOutput"]["updatedMCPToolOutput"] - assert "ImportError: no module named foo" in text - - -# PHK-27 -def test_fmt_test_check_error_key_visible(): - """test_check error key must be visible in output.""" - event = { - "tool_name": "mcp__plugin_autoskillit_autoskillit__test_check", - "tool_response": json.dumps({"passed": False, "error": "Test runner not configured"}), - } - out, _ = _run_hook(event=event) - text = json.loads(out)["hookSpecificOutput"]["updatedMCPToolOutput"] - assert "error: Test runner not configured" in text - - -# PHK-28 -def test_tool_exception_subtype_routed_before_formatter(): - """tool_exception subtype must be handled before per-tool dispatch.""" - event = { - "tool_name": "mcp__plugin_autoskillit_autoskillit__merge_worktree", - "tool_response": json.dumps( - { - "success": False, - "error": "OSError: disk full", - "exit_code": -1, - "subtype": "tool_exception", - } - ), - } - out, _ = _run_hook(event=event) - text = json.loads(out)["hookSpecificOutput"]["updatedMCPToolOutput"] - assert "tool exception" in text.lower() - assert "OSError: disk full" in text - - -# PHK-29 -def test_fmt_generic_preserves_list_values(): - """Generic formatter must render list values, not silently drop them.""" - event = { - "tool_name": "mcp__plugin_autoskillit_autoskillit__some_tool", - "tool_response": json.dumps({"total": 3, "items": ["a", "b", "c"]}), - } - out, _ = _run_hook(event=event) - text = json.loads(out)["hookSpecificOutput"]["updatedMCPToolOutput"] - assert "items" in text - assert "a" in text - assert "b" in text - assert "c" in text - - -# PHK-30 -def test_fmt_generic_preserves_dict_values(): - """Generic formatter must render dict values, not silently drop them.""" - event = { - "tool_name": "mcp__plugin_autoskillit_autoskillit__some_tool", - "tool_response": json.dumps( - {"status": "ok", "metadata": {"version": "1.0", "author": "test"}} - ), - } - out, _ = _run_hook(event=event) - text = json.loads(out)["hookSpecificOutput"]["updatedMCPToolOutput"] - assert "metadata" in text - assert "version" in text - assert "1.0" in text - assert "author" in text - - -# PHK-31 -def test_fmt_generic_read_db_rows_visible(): - """read_db rows and columns must be visible through generic formatter.""" - event = { - "tool_name": "mcp__plugin_autoskillit_autoskillit__read_db", - "tool_response": json.dumps( - { - "rows": [["id1", "value1"], ["id2", "value2"]], - "columns": ["id", "val"], - "row_count": 2, - "truncated": False, - } - ), - } - out, _ = _run_hook(event=event) - text = json.loads(out)["hookSpecificOutput"]["updatedMCPToolOutput"] - assert "rows" in text - assert "id1" in text - assert "value2" in text - assert "columns" in text - assert "id" in text - - -# PHK-32 -def test_fmt_generic_pipeline_report_failures_visible(): - """get_pipeline_report failures must be visible through generic formatter.""" - event = { - "tool_name": "mcp__plugin_autoskillit_autoskillit__get_pipeline_report", - "tool_response": json.dumps( - { - "total_failures": 2, - "failures": [ - {"step": "test", "error": "assertion failed"}, - {"step": "build", "error": "compile error"}, - ], - } - ), - } - out, _ = _run_hook(event=event) - text = json.loads(out)["hookSpecificOutput"]["updatedMCPToolOutput"] - assert "failures" in text - assert "assertion failed" in text - assert "compile error" in text - - -# PHK-33 -def test_fmt_generic_deeply_nested_truncated(): - """Deeply nested structures must be rendered as truncated compact JSON.""" - deep = {"a": {"b": {"c": {"d": {"e": "deep"}}}}} - event = { - "tool_name": "mcp__plugin_autoskillit_autoskillit__some_tool", - "tool_response": json.dumps({"info": "top", "nested": deep}), - } - out, _ = _run_hook(event=event) - text = json.loads(out)["hookSpecificOutput"]["updatedMCPToolOutput"] - assert "nested" in text - assert "deep" in text - - -# --------------------------------------------------------------------------- -# Production-realistic tests: Claude Code wraps MCP text in {"result": "..."} -# --------------------------------------------------------------------------- - - -def _wrap_for_claude_code(payload: dict) -> str: - """Simulate Claude Code's PostToolUse wrapping of MCP text content. - - Claude Code takes the MCP text response and nests it inside - {"result": ""} before passing to PostToolUse hooks. - """ - return json.dumps({"result": json.dumps(payload)}) - - -def _wrap_plain_str_for_claude_code(text: str) -> str: - """Simulate Claude Code's PostToolUse wrapping of a plain-text MCP response. - - Claude Code wraps the tool's return value as {"result": ""}. When the - tool returns a pre-formatted string (not JSON), this is the resulting shape. - """ - return json.dumps({"result": text}) - - -# PHK-34 -def test_wrapped_run_cmd_success(tmp_path): - """Wrapped run_cmd success must unwrap and show checkmark, exit_code, stdout.""" - event = { - "tool_name": "mcp__plugin_autoskillit_autoskillit__run_cmd", - "tool_response": _wrap_for_claude_code( - {"success": True, "exit_code": 0, "stdout": "file1.py\nfile2.py\n", "stderr": ""} - ), - } - out, _ = _run_hook(event=event, cwd=tmp_path) - text = json.loads(out)["hookSpecificOutput"]["updatedMCPToolOutput"] - assert "\u2713" in text - assert "success: True" in text - assert "exit_code: 0" in text - assert "file1.py" in text - assert "file2.py" in text - - -# PHK-35 -def test_wrapped_run_cmd_failure(tmp_path): - """Wrapped run_cmd failure must unwrap and show cross, exit_code, stderr.""" - event = { - "tool_name": "mcp__plugin_autoskillit_autoskillit__run_cmd", - "tool_response": _wrap_for_claude_code( - {"success": False, "exit_code": 127, "stdout": "", "stderr": "command not found"} - ), - } - out, _ = _run_hook(event=event, cwd=tmp_path) - text = json.loads(out)["hookSpecificOutput"]["updatedMCPToolOutput"] - assert "\u2717" in text - assert "FAIL" in text - assert "exit_code: 127" in text - assert "command not found" in text - - -# PHK-36 -def test_wrapped_run_skill_success(tmp_path): - """Wrapped run_skill success must unwrap and show all fields.""" - payload = { - "success": True, - "result": "Implementation complete.", - "session_id": "abc123", - "subtype": "end_turn", - "is_error": False, - "exit_code": 0, - "needs_retry": False, - "retry_reason": "none", - "stderr": "", - "token_usage": None, - "worktree_path": "", - } - event = { - "tool_name": "mcp__plugin_autoskillit_autoskillit__run_skill", - "tool_response": _wrap_for_claude_code(payload), - } - out, _ = _run_hook(event=event, cwd=tmp_path) - text = json.loads(out)["hookSpecificOutput"]["updatedMCPToolOutput"] - assert "\u2713" in text - assert "success: True" in text - assert "session_id: abc123" in text - - -# PHK-37 -def test_wrapped_kitchen_status(tmp_path): - """Wrapped kitchen_status must unwrap and show status fields.""" - payload = { - "package_version": "0.4.0", - "plugin_json_version": "0.4.0", - "versions_match": True, - "tools_enabled": True, - "token_usage_verbosity": "summary", - "quota_guard_enabled": True, - "github_token_configured": True, - "github_default_repo": "acme/myrepo", - } - event = { - "tool_name": "mcp__plugin_autoskillit_autoskillit__kitchen_status", - "tool_response": _wrap_for_claude_code(payload), - } - out, _ = _run_hook(event=event, cwd=tmp_path) - text = json.loads(out)["hookSpecificOutput"]["updatedMCPToolOutput"] - assert "kitchen_status" in text - assert "package_version" in text - assert "tools_enabled" in text - - -# PHK-38 -def test_wrapped_plain_text_result_passes_through(tmp_path): - """Plain text result envelope must not crash — passes through to generic.""" - event = { - "tool_name": "mcp__plugin_autoskillit_autoskillit__open_kitchen", - "tool_response": json.dumps( - {"result": "Kitchen is open. AutoSkillit tools are ready for service."} - ), - } - out, _ = _run_hook(event=event, cwd=tmp_path) - text = json.loads(out)["hookSpecificOutput"]["updatedMCPToolOutput"] - assert "open_kitchen" in text - assert "Kitchen is open" in text - - -# PHK-50: open_kitchen combined formatter -def test_fmt_open_kitchen_combined_response(): - """open_kitchen with recipe returns combined kitchen+recipe format.""" - payload = { - "content": "name: my-recipe\nsteps:\n do:\n tool: run_cmd\n", - "valid": True, - "suggestions": [], - "kitchen": "open", - "version": "1.2.3", - } - formatted = _format_response( - "mcp__autoskillit__open_kitchen", - json.dumps(payload), - pipeline=False, - ) - assert formatted is not None - assert "open_kitchen" in formatted - assert "v1.2.3" in formatted - assert "--- RECIPE ---" in formatted - assert "my-recipe" in formatted - - -def test_fmt_open_kitchen_combined_includes_ingredients_table(): - """Combined open_kitchen response includes pre-formatted ingredients table.""" - payload = { - "content": "name: my-recipe\nsteps: ...", - "valid": True, - "suggestions": [], - "kitchen": "open", - "version": "1.0.0", - "ingredients_table": " Name Description Default\n task The task (required)", - } - formatted = _format_response( - "mcp__autoskillit__open_kitchen", - json.dumps(payload), - pipeline=False, - ) - assert formatted is not None - assert "--- INGREDIENTS TABLE" in formatted - assert "The task" in formatted - - -def test_fmt_open_kitchen_combined_error(): - """Combined open_kitchen with recipe error shows error with kitchen status.""" - payload = { - "error": "No recipe named 'bad' found", - "kitchen": "open", - "version": "1.0.0", - } - formatted = _format_response( - "mcp__autoskillit__open_kitchen", - json.dumps(payload), - pipeline=False, - ) - assert formatted is not None - assert "open_kitchen" in formatted - assert "No recipe named 'bad' found" in formatted - assert "\u2717" in formatted - - -def test_fmt_open_kitchen_plain_text(): - """open_kitchen without recipe returns plain text format.""" - formatted = _format_response( - "mcp__autoskillit__open_kitchen", - json.dumps({"result": "Kitchen is open. AutoSkillit 1.2.3."}), - pipeline=False, - ) - assert formatted is not None - assert "open_kitchen" in formatted - assert "Kitchen is open" in formatted - - -# PHK-39 -def test_wrapped_gate_error_still_detected(tmp_path): - """Wrapped gate_error subtype must unwrap and route to gate_error formatter.""" - payload = { - "subtype": "gate_error", - "result": "Kitchen is closed.", - "success": False, - "is_error": True, - } - event = { - "tool_name": "mcp__plugin_autoskillit_autoskillit__run_skill", - "tool_response": _wrap_for_claude_code(payload), - } - out, _ = _run_hook(event=event, cwd=tmp_path) - text = json.loads(out)["hookSpecificOutput"]["updatedMCPToolOutput"] - assert "Gate Error" in text - - -# PHK-40 -def test_wrapped_tool_exception_still_detected(tmp_path): - """Wrapped tool_exception subtype must unwrap and route to exception formatter.""" - payload = { - "subtype": "tool_exception", - "error": "TimeoutError: process hung", - "exit_code": -1, - "success": False, - } - event = { - "tool_name": "mcp__plugin_autoskillit_autoskillit__run_cmd", - "tool_response": _wrap_for_claude_code(payload), - } - out, _ = _run_hook(event=event, cwd=tmp_path) - text = json.loads(out)["hookSpecificOutput"]["updatedMCPToolOutput"] - assert "tool exception" in text.lower() - assert "TimeoutError: process hung" in text - - -# T7 -def test_fmt_get_token_summary_prefers_wall_clock_seconds(): - """_fmt_get_token_summary prefers wall_clock_seconds over elapsed_seconds.""" - event = { - "tool_name": "mcp__plugin_autoskillit_autoskillit__get_token_summary", - "tool_response": json.dumps( - { - "steps": [ - { - "step_name": "implement", - "input_tokens": 5000, - "output_tokens": 1200, - "cache_creation_input_tokens": 200, - "cache_read_input_tokens": 3000, - "invocation_count": 2, - "wall_clock_seconds": 150.0, - "elapsed_seconds": 123.4, - } - ], - "total": { - "input_tokens": 5000, - "output_tokens": 1200, - "cache_creation_input_tokens": 200, - "cache_read_input_tokens": 3000, - "total_elapsed_seconds": 123.4, - }, - "mcp_responses": {"steps": [], "total": {}}, - } - ), - } - out, _ = _run_hook(event=event) - data = json.loads(out) - rendered = data["hookSpecificOutput"]["updatedMCPToolOutput"] - assert "implement" in rendered - # Should use wall_clock_seconds (150.0), not elapsed_seconds (123.4) - assert "t:150.0s" in rendered - - -# T7b -def test_fmt_get_token_summary_falls_back_to_elapsed(): - """_fmt_get_token_summary falls back to elapsed_seconds when no wall_clock.""" - event = { - "tool_name": "mcp__plugin_autoskillit_autoskillit__get_token_summary", - "tool_response": json.dumps( - { - "steps": [ - { - "step_name": "plan", - "input_tokens": 100, - "output_tokens": 50, - "cache_creation_input_tokens": 0, - "cache_read_input_tokens": 0, - "invocation_count": 1, - "elapsed_seconds": 42.5, - } - ], - "total": {"input_tokens": 100, "output_tokens": 50}, - "mcp_responses": {"steps": [], "total": {}}, - } - ), - } - out, _ = _run_hook(event=event) - data = json.loads(out) - rendered = data["hookSpecificOutput"]["updatedMCPToolOutput"] - assert "t:42.5s" in rendered - - -# --------------------------------------------------------------------------- -# Helper: generic event factory for new formatter tests -# --------------------------------------------------------------------------- - - -def _make_event(tool_name: str, payload: dict) -> dict: - """Build a minimal PostToolUse hook event for a given tool and payload dict.""" - return { - "tool_name": f"mcp__autoskillit__{tool_name}", - "tool_response": json.dumps(payload), - } - - -# --------------------------------------------------------------------------- -# PHK-41: Formatter coverage contract -# --------------------------------------------------------------------------- - - -def test_formatter_coverage_contract(): - """PHK-41: Every MCP tool is either in _FORMATTERS or explicitly in _UNFORMATTED_TOOLS. - - This prevents silent fallthrough to _fmt_generic for tools that need dedicated - formatters, and forces an explicit choice when adding new tools. - """ - from autoskillit.core.types import GATED_TOOLS, UNGATED_TOOLS - from autoskillit.hooks.pretty_output_hook import _FORMATTERS, _UNFORMATTED_TOOLS - - all_tools = GATED_TOOLS | UNGATED_TOOLS - covered = set(_FORMATTERS.keys()) | _UNFORMATTED_TOOLS - uncovered = all_tools - covered - assert uncovered == set(), ( - f"Tools have no formatter and are not in _UNFORMATTED_TOOLS: {sorted(uncovered)}. " - "Either add a dedicated formatter or add to _UNFORMATTED_TOOLS." - ) - - -# --------------------------------------------------------------------------- -# PHK-42/43/44: _fmt_load_recipe tests -# --------------------------------------------------------------------------- - - -def test_fmt_load_recipe_suppresses_diagram(): - """Diagram is suppressed — user sees it in terminal, agent doesn't need it.""" - formatted = _format_response( - "mcp__autoskillit__load_recipe", - json.dumps( - { - "content": "name: my-recipe\nsteps: ...", - "diagram": "## my-recipe\nsome diagram graph", - "valid": True, - "suggestions": [], - } - ), - pipeline=False, - ) - assert formatted is not None - assert "some diagram graph" not in formatted - assert "--- RECIPE ---" in formatted - - -def test_fmt_load_recipe_includes_raw_yaml_content(): - """load_recipe response includes the raw YAML so the agent can execute steps.""" - yaml_content = "name: my-recipe\nsteps:\n do_thing:\n tool: run_cmd\n" - formatted = _format_response( - "mcp__autoskillit__load_recipe", - json.dumps( - { - "content": yaml_content, - "diagram": "## my-recipe\nsome diagram", - "valid": True, - "suggestions": [], - } - ), - pipeline=False, - ) - assert formatted is not None - assert "do_thing" in formatted - assert "--- RECIPE ---" in formatted - - -def test_fmt_load_recipe_shows_finding_count(): - """Findings are summarized as a count, not individual bullets.""" - formatted = _format_response( - "mcp__autoskillit__load_recipe", - json.dumps( - { - "content": "name: x", - "diagram": "## x", - "valid": False, - "suggestions": [ - { - "rule": "missing-step", - "message": "Step 'done' not found", - "severity": "error", - }, - {"rule": "unknown-tool", "message": "Tool 'badtool'", "severity": "warning"}, - ], - } - ), - pipeline=False, - ) - assert formatted is not None - assert "2 finding(s)" in formatted - - -# --------------------------------------------------------------------------- -# PHK-45/46: _fmt_list_recipes tests -# --------------------------------------------------------------------------- - - -def test_fmt_list_recipes_shows_all_names(): - """PHK-45: list_recipes response renders all recipe names — no 500-char truncation.""" - recipes = [ - {"name": f"recipe-{i:02d}", "description": f"Description for recipe {i}", "summary": "..."} - for i in range(10) - ] - event = _make_event("list_recipes", {"recipes": recipes, "count": 10}) - out, _ = _run_hook(event) - text = json.loads(out)["hookSpecificOutput"]["updatedMCPToolOutput"] - for i in range(10): - assert f"recipe-{i:02d}" in text, f"recipe-{i:02d} missing from output" - assert '{"name"' not in text - assert "10" in text - - -def test_fmt_list_recipes_compact_representation(): - """PHK-46: list_recipes renders one line per recipe in 'name: description' format.""" - event = _make_event( - "list_recipes", - { - "recipes": [ - { - "name": "implementation", - "description": "Implement a plan in a worktree", - "summary": "...", - }, - { - "name": "smoke-test", - "description": "Run a smoke-test pipeline", - "summary": "...", - }, - ], - "count": 2, - }, - ) - out, _ = _run_hook(event) - text = json.loads(out)["hookSpecificOutput"]["updatedMCPToolOutput"] - assert "implementation" in text - assert "Implement a plan in a worktree" in text - assert "smoke-test" in text - assert "Run a smoke-test pipeline" in text - - -# --------------------------------------------------------------------------- -# PHK-47/48: _fmt_generic list-of-dicts hardening tests -# --------------------------------------------------------------------------- - - -def test_fmt_generic_list_of_dicts_renders_per_item_not_blob(): - """PHK-47: Generic formatter renders list-of-dicts per item, not a truncated JSON blob.""" - failures = [ - {"step": f"step_{i}", "error": f"Error message {i}", "exit_code": 1} for i in range(10) - ] - event = _make_event("some_tool", {"failures": failures}) - out, _ = _run_hook(event) - text = json.loads(out)["hookSpecificOutput"]["updatedMCPToolOutput"] - for i in range(10): - assert f"step_{i}" in text, f"step_{i} missing — possible truncation" - assert "... and" not in text, "Output was truncated" - - -def test_fmt_generic_list_of_dicts_caps_at_20_items(): - """PHK-48: Generic formatter caps list-of-dicts at 20 items with overflow note.""" - items = [{"key": f"item-{i}", "val": f"value-{i}"} for i in range(25)] - event = _make_event("some_tool", {"items": items}) - out, _ = _run_hook(event) - text = json.loads(out)["hookSpecificOutput"]["updatedMCPToolOutput"] - assert "item-0" in text - assert "item-19" in text - assert "item-20" not in text - assert "and 5 more" in text - - -# --------------------------------------------------------------------------- -# Output-equivalence: hook inline formatter ≡ TelemetryFormatter.format_compact_kv -# --------------------------------------------------------------------------- - - -def test_hook_token_summary_output_equivalent_to_canonical(): - """1g: Hook inline _fmt_get_token_summary produces identical output to - TelemetryFormatter.format_compact_kv for the same input data.""" - from autoskillit.hooks.pretty_output_hook import _fmt_get_token_summary - from autoskillit.pipeline.telemetry_fmt import TelemetryFormatter - - data = { - "steps": [ - { - "step_name": "investigate", - "input_tokens": 7000, - "output_tokens": 5939, - "cache_creation_input_tokens": 8495, - "cache_read_input_tokens": 252179, - "invocation_count": 1, - "wall_clock_seconds": 45.0, - "elapsed_seconds": 40.0, - }, - { - "step_name": "implement", - "input_tokens": 2031000, - "output_tokens": 122306, - "cache_creation_input_tokens": 280601, - "cache_read_input_tokens": 19071323, - "invocation_count": 3, - "wall_clock_seconds": 492.0, - "elapsed_seconds": 480.0, - }, - ], - "total": { - "input_tokens": 2038000, - "output_tokens": 128245, - "cache_creation_input_tokens": 289096, - "cache_read_input_tokens": 19323502, - "total_elapsed_seconds": 537.0, - }, - "mcp_responses": { - "steps": [], - "total": {"total_invocations": 42, "total_estimated_response_tokens": 5000}, - }, - } - - hook_output = _fmt_get_token_summary(data, _pipeline=False) - canonical_output = TelemetryFormatter.format_compact_kv( - data["steps"], data["total"], mcp_responses=data["mcp_responses"] - ) - assert hook_output == canonical_output, ( - f"Hook and canonical formatter produce different output:\n" - f"HOOK:\n{hook_output}\n\nCANONICAL:\n{canonical_output}" - ) - - -def test_fmt_run_skill_interactive_shows_four_token_fields(): - """_fmt_run_skill interactive mode shows all 4 token fields.""" - data = { - "success": True, - "subtype": "COMPLETED", - "exit_code": 0, - "needs_retry": False, - "result": "done", - "token_usage": { - "input_tokens": 5000, - "output_tokens": 3000, - "cache_read_input_tokens": 200000, - "cache_creation_input_tokens": 8000, - }, - } - rendered = _format_response( - "mcp__plugin_autoskillit_autoskillit__run_skill", - json.dumps({"result": json.dumps(data)}), - pipeline=False, - ) - assert rendered is not None - assert "tokens_uncached:" in rendered - assert "tokens_out:" in rendered - assert "tokens_cache_read:" in rendered - assert "tokens_cache_write:" in rendered - - -def test_fmt_run_skill_suppresses_zero_cache_fields(): - """_fmt_run_skill suppresses tokens_cache_read and tokens_cache_write when both are 0.""" - data = { - "success": True, - "subtype": "COMPLETED", - "exit_code": 0, - "needs_retry": False, - "result": "done", - "token_usage": { - "input_tokens": 5000, - "output_tokens": 3000, - "cache_read_input_tokens": 0, - "cache_creation_input_tokens": 0, - }, - } - rendered = _format_response( - "mcp__plugin_autoskillit_autoskillit__run_skill", - json.dumps({"result": json.dumps(data)}), - pipeline=False, - ) - assert rendered is not None - assert "tokens_uncached:" in rendered - assert "tokens_out:" in rendered - assert "tokens_cache_read:" not in rendered - assert "tokens_cache_write:" not in rendered - - -# --------------------------------------------------------------------------- -# Timing summary dedicated formatter -# --------------------------------------------------------------------------- - - -def test_fmt_get_timing_summary_renders_compact(): - """get_timing_summary dedicated formatter renders compact Markdown-KV.""" - event = { - "tool_name": "mcp__autoskillit__get_timing_summary", - "tool_response": json.dumps( - { - "steps": [ - {"step_name": "clone", "total_seconds": 4.0, "invocation_count": 1}, - {"step_name": "implement", "total_seconds": 492.0, "invocation_count": 3}, - ], - "total": {"total_seconds": 496.0}, - } - ), - } - out, _ = _run_hook(event=event) - data = json.loads(out) - rendered = data["hookSpecificOutput"]["updatedMCPToolOutput"] - assert "## timing_summary" in rendered - assert "clone x1" in rendered - assert "implement x3" in rendered - assert "dur:4s" in rendered - assert "dur:8m 12s" in rendered - assert "total:" in rendered - - -# Issue #346 -def test_fmt_run_skill_contradictory_subtype_never_renders_fail_success(): - """Test A: full pipeline — COMPLETED+empty never renders 'FAIL [success]'. - - This captures Issue #346: when adjudication overrides the CLI's reported - outcome (CLI says 'success', but result is empty → failure), the rendered - status tag must reflect the adjudicated subtype, not the raw CLI value. - - Before the fix: sr.subtype = "success" → hook renders "FAIL [success]". - After the fix: sr.subtype = "empty_result" → hook renders "FAIL [empty_result]". - """ - - # Build a SubprocessResult that triggers the COMPLETED+empty-result path. - # The CLI reports subtype="success" but result is empty — adjudication sees failure. - stdout = json.dumps( - { - "type": "result", - "subtype": "success", - "is_error": False, - "result": "", - "session_id": "s1", - } - ) - result = _make_result( - returncode=0, - stdout=stdout, - termination_reason=TerminationReason.COMPLETED, - channel_confirmation=ChannelConfirmation.UNMONITORED, - ) - sr = _build_skill_result(result, completion_marker="", skill_command="/test") - assert sr.success is False, "Precondition: this path must produce a failure" - - payload = json.loads(sr.to_json()) - - # Pipeline mode: must NOT render "FAIL [success]" - pipeline_out = _format_response( - "mcp__plugin_autoskillit_autoskillit__run_skill", - json.dumps(payload), - pipeline=True, - ) - assert pipeline_out is not None - assert "FAIL [success]" not in pipeline_out, ( - f"Pipeline mode rendered contradictory 'FAIL [success]': {pipeline_out!r}" - ) - # Must render the normalized subtype instead - assert "FAIL [empty_result]" in pipeline_out, ( - f"Expected 'FAIL [empty_result]' in pipeline output: {pipeline_out!r}" - ) - - # Interactive mode: cross mark must not be paired with "success" as status - interactive_out = _format_response( - "mcp__plugin_autoskillit_autoskillit__run_skill", - json.dumps(payload), - pipeline=False, - ) - assert interactive_out is not None - cross = "\u2717" - assert f"{cross} success" not in interactive_out, ( - f"Interactive mode rendered contradictory '{cross} success': {interactive_out!r}" - ) - - -# --------------------------------------------------------------------------- -# Field coverage contract: LoadRecipeResult -# --------------------------------------------------------------------------- - - -def test_fmt_load_recipe_field_coverage(): - """Every LoadRecipeResult field must be in RENDERED or SUPPRESSED.""" - from autoskillit.hooks.pretty_output_hook import ( - _FMT_LOAD_RECIPE_RENDERED, - _FMT_LOAD_RECIPE_SUPPRESSED, - ) - from autoskillit.recipe._api import LoadRecipeResult - - all_fields = set(LoadRecipeResult.__annotations__) - covered = _FMT_LOAD_RECIPE_RENDERED | _FMT_LOAD_RECIPE_SUPPRESSED - uncovered = all_fields - covered - assert uncovered == set(), ( - f"LoadRecipeResult fields have no coverage decision: {sorted(uncovered)}. " - "Add each to _FMT_LOAD_RECIPE_RENDERED or _FMT_LOAD_RECIPE_SUPPRESSED." - ) - extra = covered - all_fields - assert extra == set(), ( - f"Coverage registry references non-existent fields: {sorted(extra)}. Remove stale entries." - ) - - -def test_fmt_load_recipe_derivation_map_coverage(): - """Every key/value in _LOAD_RECIPE_CONTENT_DERIVED_FROM must be in _FMT_LOAD_RECIPE_RENDERED. - Catches when a new derived field is added without declaring its source relationship.""" - from autoskillit.hooks.pretty_output_hook import ( - _FMT_LOAD_RECIPE_RENDERED, - _LOAD_RECIPE_CONTENT_DERIVED_FROM, - ) - - for derived_field, source_field in _LOAD_RECIPE_CONTENT_DERIVED_FROM.items(): - assert derived_field in _FMT_LOAD_RECIPE_RENDERED, ( - f"Derived field '{derived_field}' must be in _FMT_LOAD_RECIPE_RENDERED. " - f"If it was moved to SUPPRESSED, remove it from _LOAD_RECIPE_CONTENT_DERIVED_FROM." - ) - assert source_field in _FMT_LOAD_RECIPE_RENDERED, ( - f"Source field '{source_field}' must be in _FMT_LOAD_RECIPE_RENDERED " - f"(it is the source for derived field '{derived_field}'). " - f"If the source is suppressed, the derivation map entry is stale." - ) - - -def test_fmt_load_recipe_renders_error(): - """When error is present, it appears in output with cross mark.""" - formatted = _format_response( - "mcp__autoskillit__load_recipe", - json.dumps({"error": "Recipe 'x' not found", "valid": False}), - pipeline=False, - ) - assert formatted is not None - assert "\u2717" in formatted - assert "Recipe 'x' not found" in formatted - - -def test_fmt_load_recipe_suppresses_kitchen_rules(): - """Kitchen rules are suppressed — they're in the YAML content.""" - formatted = _format_response( - "mcp__autoskillit__load_recipe", - json.dumps( - { - "valid": True, - "diagram": "## test diagram", - "suggestions": [], - "kitchen_rules": ["no raw SQL", "use run_cmd for shell"], - } - ), - pipeline=False, - ) - assert formatted is not None - assert "no raw SQL" not in formatted - - -def test_fmt_load_recipe_suppresses_greeting(): - """Greeting is suppressed — delivered via positional CLI arg instead.""" - formatted = _format_response( - "mcp__autoskillit__load_recipe", - json.dumps( - { - "valid": True, - "diagram": "## test diagram", - "suggestions": [], - "greeting": "Welcome to Good Burger!", - } - ), - pipeline=False, - ) - assert formatted is not None - assert "Welcome to Good Burger!" not in formatted - - -# --------------------------------------------------------------------------- -# Field coverage contract: ListRecipesResult + RecipeListItem -# --------------------------------------------------------------------------- - - -def test_fmt_list_recipes_field_coverage(): - """Every ListRecipesResult field must be in RENDERED or SUPPRESSED.""" - from autoskillit.hooks.pretty_output_hook import ( - _FMT_LIST_RECIPES_RENDERED, - _FMT_LIST_RECIPES_SUPPRESSED, - ) - from autoskillit.recipe._api import ListRecipesResult - - all_fields = set(ListRecipesResult.__annotations__) - covered = _FMT_LIST_RECIPES_RENDERED | _FMT_LIST_RECIPES_SUPPRESSED - uncovered = all_fields - covered - assert uncovered == set(), ( - f"ListRecipesResult fields have no coverage decision: {sorted(uncovered)}. " - "Add each to _FMT_LIST_RECIPES_RENDERED or _FMT_LIST_RECIPES_SUPPRESSED." - ) - extra = covered - all_fields - assert extra == set(), ( - f"Coverage registry references non-existent fields: {sorted(extra)}. Remove stale entries." - ) - - -def test_fmt_recipe_list_item_field_coverage(): - """Every RecipeListItem field must be in RENDERED or SUPPRESSED.""" - from autoskillit.hooks.pretty_output_hook import ( - _FMT_RECIPE_LIST_ITEM_RENDERED, - _FMT_RECIPE_LIST_ITEM_SUPPRESSED, - ) - from autoskillit.recipe._api import RecipeListItem - - all_fields = set(RecipeListItem.__annotations__) - covered = _FMT_RECIPE_LIST_ITEM_RENDERED | _FMT_RECIPE_LIST_ITEM_SUPPRESSED - uncovered = all_fields - covered - assert uncovered == set(), ( - f"RecipeListItem fields have no coverage decision: {sorted(uncovered)}. " - "Add each to _FMT_RECIPE_LIST_ITEM_RENDERED or _FMT_RECIPE_LIST_ITEM_SUPPRESSED." - ) - extra = covered - all_fields - assert extra == set(), ( - f"Coverage registry references non-existent fields: {sorted(extra)}. Remove stale entries." - ) - - -def test_fmt_list_recipes_renders_summary(): - """Recipe summary appears on the line below each recipe name.""" - formatted = _format_response( - "mcp__autoskillit__list_recipes", - json.dumps( - { - "recipes": [ - {"name": "test-recipe", "description": "A test", "summary": "step1 -> done"}, - ], - "count": 1, - } - ), - pipeline=False, - ) - assert formatted is not None - assert "test-recipe" in formatted - assert "step1 -> done" in formatted - - -# --------------------------------------------------------------------------- -# T-1/T-2: Typed payload dispatch — plain-text envelope must pass through -# --------------------------------------------------------------------------- - - -def test_get_token_summary_format_table_passes_through_unmodified(tmp_path): - """get_token_summary(format='table') returns pre-formatted markdown; - the hook must pass it through unchanged, not eat it via the named formatter.""" - table = ( - "## Token Usage Summary\n\n| Step | input | output |\n|---|---|---|\n| impl | 45k | 12k |" - ) - event = { - "tool_name": "mcp__plugin_autoskillit_autoskillit__get_token_summary", - "tool_response": json.dumps({"result": table}), - } - out, code = _run_hook(event=event, cwd=tmp_path) - assert code == 0 - text = json.loads(out)["hookSpecificOutput"]["updatedMCPToolOutput"] - assert "## Token Usage Summary" in text - assert "| impl |" in text - # Must NOT produce the empty named-formatter output - assert text.strip() != "## token_summary" - - -def test_get_timing_summary_format_table_passes_through_unmodified(tmp_path): - """get_timing_summary(format='table') returns pre-formatted markdown; - the hook must pass it through unchanged, not eat it via the named formatter.""" - table = ( - "## Step Timing Summary\n\n" - "| Step | Duration | Invocations |\n|---|---|---|\n| clone | 4s | 1 |" - ) - event = { - "tool_name": "mcp__plugin_autoskillit_autoskillit__get_timing_summary", - "tool_response": json.dumps({"result": table}), - } - out, code = _run_hook(event=event, cwd=tmp_path) - assert code == 0 - text = json.loads(out)["hookSpecificOutput"]["updatedMCPToolOutput"] - assert "## Step Timing Summary" in text - assert "| clone |" in text - assert text.strip() != "## timing_summary" - - -# --------------------------------------------------------------------------- -# T-3: _wrap_plain_str_for_claude_code helper shape -# --------------------------------------------------------------------------- - - -def test_wrap_plain_str_helper_produces_correct_shape(): - """_wrap_plain_str_for_claude_code produces the real hook event shape - for tools returning pre-formatted strings.""" - raw = _wrap_plain_str_for_claude_code("hello world") - parsed = json.loads(raw) - assert parsed == {"result": "hello world"} - - -# --------------------------------------------------------------------------- -# T-4/T-5: _UNFORMATTED_TOOLS behavioral gate -# --------------------------------------------------------------------------- - - -def test_unformatted_tools_and_formatters_are_disjoint(): - """_UNFORMATTED_TOOLS and _FORMATTERS must be mutually exclusive.""" - from autoskillit.hooks.pretty_output_hook import _FORMATTERS, _UNFORMATTED_TOOLS - - overlap = set(_FORMATTERS) & _UNFORMATTED_TOOLS - assert not overlap, f"Tools in both dispatch tables: {overlap}" - - -def test_unformatted_tool_routes_to_generic_not_named_formatter(tmp_path): - """A tool in _UNFORMATTED_TOOLS must reach _fmt_generic even if - coincidentally also added to _FORMATTERS (enforcement test).""" - # Verify get_pipeline_report (in _UNFORMATTED_TOOLS) uses generic rendering - payload = {"total_failures": 1, "failures": [{"step": "impl", "reason": "red"}]} - event = { - "tool_name": "mcp__plugin_autoskillit_autoskillit__get_pipeline_report", - "tool_response": _wrap_for_claude_code(payload), - } - out, code = _run_hook(event=event, cwd=tmp_path) - assert code == 0 - text = json.loads(out)["hookSpecificOutput"]["updatedMCPToolOutput"] - assert "get_pipeline_report" in text - - -# --------------------------------------------------------------------------- -# Raw/derived field deduplication: ingredients_table vs content -# --------------------------------------------------------------------------- - - -def test_fmt_recipe_body_ingredients_not_duplicated_when_table_present(): - """When ingredients_table is present, ingredient names must not appear - in the --- RECIPE --- section. This is the canonical test for the - raw/derived field duplication bug.""" - from autoskillit.hooks.pretty_output_hook import _fmt_recipe_body - - data = { - "content": REALISTIC_RECIPE_YAML, - "ingredients_table": ( - "| Name | Description | Default |\n" - "| task | What to implement | (required) |\n" - "| review_approach | Run review-approach before planning | false |\n" - ), - "valid": True, - "suggestions": [], - } - result = "\n".join(_fmt_recipe_body(data)) - assert "--- INGREDIENTS TABLE" in result, ( - "_fmt_recipe_body did not emit the INGREDIENTS TABLE header — " - "deduplication guarantee cannot be tested." - ) - recipe_section = result.split("--- INGREDIENTS TABLE")[0] - # Ingredient names must appear ONLY in the table, not in the raw YAML block. - # `task:` and `review_approach:` as YAML keys indicate the ingredients: block. - assert "review_approach:" not in recipe_section - assert " task:" not in recipe_section - # But the steps and kitchen_rules sections should remain in the RECIPE block. - assert "implement" in recipe_section - assert "kitchen_rules" in recipe_section - # And the table section must contain the ingredient data. - table_section = result.split("--- INGREDIENTS TABLE")[1] - assert "review_approach" in table_section - assert "task" in table_section - - -def test_strip_yaml_ingredients_block_removes_ingredients_section(): - from autoskillit.hooks.pretty_output_hook import _strip_yaml_ingredients_block - - yaml = "name: test\ningredients:\n task:\n description: a task\nsteps:\n do: {}\n" - result = _strip_yaml_ingredients_block(yaml) - assert "ingredients:" not in result - assert " task:" not in result - assert "steps:" in result - assert "name: test" in result - - -def test_strip_yaml_ingredients_block_noop_when_no_ingredients_key(): - from autoskillit.hooks.pretty_output_hook import _strip_yaml_ingredients_block - - yaml = "name: test\nsteps:\n do: {}\n" - result = _strip_yaml_ingredients_block(yaml) - assert result == yaml - - -def test_strip_yaml_ingredients_block_at_end_of_file(): - from autoskillit.hooks.pretty_output_hook import _strip_yaml_ingredients_block - - yaml = "name: test\nsteps:\n do: {}\ningredients:\n foo:\n description: bar\n" - result = _strip_yaml_ingredients_block(yaml) - assert "ingredients:" not in result - assert "steps:" in result - - -def test_strip_yaml_ingredients_block_multiline_description(): - from autoskillit.hooks.pretty_output_hook import _strip_yaml_ingredients_block - - yaml = ( - "name: test\n" - "ingredients:\n" - " task:\n" - " description: >\n" - " Long description\n" - " spanning multiple lines\n" - " required: true\n" - "steps:\n" - " do: {}\n" - ) - result = _strip_yaml_ingredients_block(yaml) - assert "ingredients:" not in result - assert "steps:" in result - - -def test_fmt_open_kitchen_ingredients_not_duplicated_when_table_present(): - """open_kitchen routes through _fmt_recipe_body — verify same deduplication applies.""" - from autoskillit.hooks.pretty_output_hook import _fmt_open_kitchen - - data = { - "content": REALISTIC_RECIPE_YAML, - "ingredients_table": "| task | What to implement | (required) |", - "valid": True, - "suggestions": [], - "kitchen": "open", - "version": "0.6.0", - } - # _fmt_open_kitchen returns str (not list[str]); no join needed. - result = _fmt_open_kitchen(data, pipeline=False) - assert "--- INGREDIENTS TABLE" in result, ( - "_fmt_open_kitchen did not emit the INGREDIENTS TABLE header — " - "deduplication guarantee cannot be tested." - ) - recipe_section = result.split("--- INGREDIENTS TABLE")[0] - assert " task:" not in recipe_section - assert "review_approach:" not in recipe_section - - -def test_pretty_output_public_surface_unchanged() -> None: - """T-5 (audit finding 8.3): the hook entrypoint and the format router are - the only public surface; the four-way split must preserve them.""" - import autoskillit.hooks.pretty_output_hook as p - - assert callable(p.main) - assert callable(p._format_response) - - -# PHK-kill-reason: formatter branches on kill_reason (1e) - - -def test_fmt_run_skill_success_with_kill_after_completion_annotates_exit_code() -> None: - """kill_reason=kill_after_completion must annotate exit_code line.""" - from autoskillit.hooks._fmt_execution import _fmt_run_skill - - data = { - "success": True, - "result": "Implementation complete.", - "session_id": "abc123", - "subtype": "end_turn", - "is_error": False, - "exit_code": -9, - "kill_reason": "kill_after_completion", - "needs_retry": False, - "retry_reason": "none", - "stderr": "", - } - text = _fmt_run_skill(data, pipeline=False) - assert "exit_code: -9 (infra-terminated after completion" in text, ( - f"Expected kill_after_completion annotation in exit_code line, got: {text!r}" - ) - - -def test_fmt_run_skill_success_with_natural_exit_shows_bare_exit_code() -> None: - """kill_reason=natural_exit must render bare exit_code without annotation.""" - from autoskillit.hooks._fmt_execution import _fmt_run_skill - - data = { - "success": True, - "result": "Done.", - "session_id": "abc123", - "subtype": "end_turn", - "is_error": False, - "exit_code": 0, - "kill_reason": "natural_exit", - "needs_retry": False, - "retry_reason": "none", - "stderr": "", - } - text = _fmt_run_skill(data, pipeline=False) - assert "exit_code: 0" in text - assert "infra-terminated" not in text - assert "infra-killed" not in text - - -def test_fmt_run_skill_infra_kill_annotates_reason() -> None: - """kill_reason=infra_kill must annotate exit_code with infra-killed.""" - from autoskillit.hooks._fmt_execution import _fmt_run_skill - - data = { - "success": False, - "result": "", - "session_id": "abc123", - "subtype": "timeout", - "is_error": True, - "exit_code": -9, - "kill_reason": "infra_kill", - "needs_retry": False, - "retry_reason": "none", - "stderr": "", - } - text = _fmt_run_skill(data, pipeline=False) - assert "infra-killed" in text, ( - f"Expected 'infra-killed' annotation for infra_kill reason, got: {text!r}" - ) - - -def test_fmt_run_skill_legacy_payload_without_kill_reason_renders_bare() -> None: - """Payload without kill_reason field must render bare exit_code (backward compat).""" - from autoskillit.hooks._fmt_execution import _fmt_run_skill - - data = { - "success": True, - "result": "Done.", - "session_id": "abc123", - "subtype": "end_turn", - "is_error": False, - "exit_code": 0, - # no kill_reason key - "needs_retry": False, - "retry_reason": "none", - "stderr": "", - } - text = _fmt_run_skill(data, pipeline=False) - assert "exit_code: 0" in text - assert "infra" not in text diff --git a/tests/infra/test_pretty_output_formatters.py b/tests/infra/test_pretty_output_formatters.py new file mode 100644 index 0000000000..9032e53a56 --- /dev/null +++ b/tests/infra/test_pretty_output_formatters.py @@ -0,0 +1,827 @@ +"""Tests: pretty_output per-tool named formatters.""" + +from __future__ import annotations + +import json + +import pytest + +from autoskillit.hooks.formatters._fmt_primitives import _HOOK_CONFIG_PATH_COMPONENTS +from tests.infra._pretty_output_helpers import _make_run_skill_event, _run_hook + +pytestmark = [pytest.mark.layer("infra"), pytest.mark.medium] + + +# PHK-6 +def test_format_run_skill_success(tmp_path): + """run_skill success response must contain tool name, checkmark, and success field.""" + event = _make_run_skill_event(success=True) + out, _ = _run_hook(event=event, cwd=tmp_path) + text = json.loads(out)["hookSpecificOutput"]["updatedMCPToolOutput"] + assert "run_skill" in text + assert "\u2713" in text # ✓ + assert "success: True" in text + + +# PHK-7 +def test_format_run_skill_failure_with_retry(tmp_path): + """run_skill failure with retry must show cross, retry fields, and worktree_path.""" + event = _make_run_skill_event( + success=False, + needs_retry=True, + retry_reason="budget_exhausted", + worktree_path="/tmp/wt/fix-abc", + ) + out, _ = _run_hook(event=event, cwd=tmp_path) + text = json.loads(out)["hookSpecificOutput"]["updatedMCPToolOutput"] + assert "\u2717" in text # ✗ + assert "needs_retry: True" in text + assert "retry_reason: budget_exhausted" in text + assert "worktree_path: /tmp/wt/fix-abc" in text + + +# PHK-8 +def test_format_run_skill_gate_error(): + """run_skill gate_error subtype must show 'Gate Error' and 'gate_error'.""" + event = { + "tool_name": "mcp__plugin_autoskillit_autoskillit__run_skill", + "tool_response": json.dumps( + { + "success": False, + "subtype": "gate_error", + "is_error": True, + "result": "Kitchen is closed.", + } + ), + } + out, _ = _run_hook(event=event) + text = json.loads(out)["hookSpecificOutput"]["updatedMCPToolOutput"] + assert "Gate Error" in text + assert "gate_error" in text + + +# PHK-9 +def test_format_run_cmd_success_shows_stdout(tmp_path): + """run_cmd success must show tool name, checkmark, stdout content.""" + event = { + "tool_name": "mcp__plugin_autoskillit_autoskillit__run_cmd", + "tool_response": json.dumps( + {"success": True, "exit_code": 0, "stdout": "hello\nworld\n", "stderr": ""} + ), + } + out, _ = _run_hook(event=event, cwd=tmp_path) + text = json.loads(out)["hookSpecificOutput"]["updatedMCPToolOutput"] + assert "run_cmd" in text + assert "\u2713" in text # ✓ + assert "stdout" in text + assert "hello" in text + assert "world" in text + + +# PHK-10 +def test_format_run_cmd_failure_shows_stderr(tmp_path): + """run_cmd failure must show cross and stderr content.""" + event = { + "tool_name": "mcp__plugin_autoskillit_autoskillit__run_cmd", + "tool_response": json.dumps( + {"success": False, "exit_code": 1, "stdout": "", "stderr": "No such file"} + ), + } + out, _ = _run_hook(event=event, cwd=tmp_path) + text = json.loads(out)["hookSpecificOutput"]["updatedMCPToolOutput"] + assert "\u2717" in text # ✗ + assert "stderr" in text + assert "No such file" in text + + +# PHK-11 +def test_format_test_check_pass(): + """test_check pass must show tool name, checkmark, PASS, passed: True.""" + event = { + "tool_name": "mcp__plugin_autoskillit_autoskillit__test_check", + "tool_response": json.dumps({"passed": True, "stdout": "...245 passed..."}), + } + out, _ = _run_hook(event=event) + text = json.loads(out)["hookSpecificOutput"]["updatedMCPToolOutput"] + assert "test_check" in text + assert "\u2713" in text # ✓ + assert "PASS" in text + assert "passed: True" in text + + +# PHK-12 +def test_format_test_check_fail_shows_failure_lines(): + """test_check fail must show failure lines and filter pytest boilerplate.""" + pytest_output = ( + "platform linux -- Python 3.13.2\n" + "rootdir: /repo\n" + "collecting ...\n" + "collected 100 items\n" + "FAILED tests/foo.py::test_a - AssertionError: 1 != 2\n" + "FAILED tests/bar.py::test_b - RuntimeError: oops\n" + "2 failed, 98 passed\n" + ) + event = { + "tool_name": "mcp__plugin_autoskillit_autoskillit__test_check", + "tool_response": json.dumps({"passed": False, "stdout": pytest_output}), + } + out, _ = _run_hook(event=event) + text = json.loads(out)["hookSpecificOutput"]["updatedMCPToolOutput"] + assert "\u2717" in text # ✗ + assert "FAIL" in text + assert "FAILED tests/foo.py::test_a" in text + assert "FAILED tests/bar.py::test_b" in text + assert "platform linux" not in text + assert "collecting ..." not in text + + +# PHK-13 +def test_format_merge_worktree_failure(): + """merge_worktree failure must include all key fields.""" + event = { + "tool_name": "mcp__plugin_autoskillit_autoskillit__merge_worktree", + "tool_response": json.dumps( + { + "error": "Rebase failed", + "failed_step": "rebase", + "state": "worktree_dirty", + "worktree_path": "/tmp/wt/fix", + "stderr": "CONFLICT (content): Merge conflict in foo.py", + } + ), + } + out, _ = _run_hook(event=event) + text = json.loads(out)["hookSpecificOutput"]["updatedMCPToolOutput"] + assert "merge_worktree" in text + assert "\u2717" in text # ✗ + assert "error: Rebase failed" in text + assert "failed_step: rebase" in text + assert "state: worktree_dirty" in text + assert "CONFLICT" in text + + +# PHK-14 +def test_format_merge_worktree_success(): + """merge_worktree success must show checkmark and merge_succeeded: True.""" + event = { + "tool_name": "mcp__plugin_autoskillit_autoskillit__merge_worktree", + "tool_response": json.dumps( + { + "merge_succeeded": True, + "merged_branch": "impl-fix-20260101", + "into_branch": "main", + "worktree_removed": True, + "branch_deleted": True, + } + ), + } + out, _ = _run_hook(event=event) + text = json.loads(out)["hookSpecificOutput"]["updatedMCPToolOutput"] + assert "merge_worktree" in text + assert "\u2713" in text # ✓ + assert "merge_succeeded: True" in text + + +# PHK-16 +def test_format_kitchen_status(): + """kitchen_status must show package_version and tools_enabled.""" + event = { + "tool_name": "mcp__plugin_autoskillit_autoskillit__kitchen_status", + "tool_response": json.dumps( + { + "package_version": "0.2.0", + "plugin_json_version": "0.2.0", + "versions_match": True, + "tools_enabled": True, + "token_usage_verbosity": "summary", + "quota_guard_enabled": True, + "github_token_configured": True, + "github_default_repo": "acme/myrepo", + } + ), + } + out, _ = _run_hook(event=event) + text = json.loads(out)["hookSpecificOutput"]["updatedMCPToolOutput"] + assert "kitchen_status" in text + assert "package_version: 0.2.0" in text + assert "tools_enabled: True" in text + + +# PHK-17 +def test_pipeline_mode_compact_run_skill(tmp_path): + """In pipeline mode (hook config present), run_skill output uses compact format.""" + config_path = tmp_path.joinpath(*_HOOK_CONFIG_PATH_COMPONENTS) + config_path.parent.mkdir(parents=True, exist_ok=True) + config_path.write_text('{"quota_guard": {}}') + + event = _make_run_skill_event(success=False, needs_retry=True, retry_reason="budget_exhausted") + + out, _ = _run_hook(event=event, cwd=tmp_path) + + text = json.loads(out)["hookSpecificOutput"]["updatedMCPToolOutput"] + assert "run_skill:" in text + assert "FAIL" in text + assert "## run_skill" not in text + + +# PHK-18 +def test_unknown_tool_passes_through(): + """Unknown tool name gets generic key-value rendering.""" + event = { + "tool_name": "mcp__plugin_autoskillit_autoskillit__some_unknown_tool", + "tool_response": json.dumps({"result": "ok", "status": "done"}), + } + out, code = _run_hook(event=event) + assert code == 0 + assert out.strip() + text = json.loads(out)["hookSpecificOutput"]["updatedMCPToolOutput"] + assert "some_unknown_tool" in text + + +# PHK-19 +def test_fmt_run_cmd_tool_exception_shows_diagnostic(): + """run_cmd tool_exception subtype must show error, not FAIL [].""" + event = { + "tool_name": "mcp__plugin_autoskillit_autoskillit__run_cmd", + "tool_response": json.dumps( + { + "error": "RuntimeError: boom", + "exit_code": -1, + "subtype": "tool_exception", + "success": False, + } + ), + } + out, _ = _run_hook(event=event) + text = json.loads(out)["hookSpecificOutput"]["updatedMCPToolOutput"] + assert "tool exception" in text.lower() + assert "RuntimeError: boom" in text + assert "FAIL []" not in text + + +# PHK-20 +def test_fmt_clone_repo_uncommitted_changes_warning(): + """clone_repo uncommitted_changes must show WARNING, not OK.""" + event = { + "tool_name": "mcp__plugin_autoskillit_autoskillit__clone_repo", + "tool_response": json.dumps( + { + "uncommitted_changes": "true", + "source_dir": "/src", + "branch": "main", + "changed_files": "M file.py", + "total_changed": "1", + } + ), + } + out, _ = _run_hook(event=event) + text = json.loads(out)["hookSpecificOutput"]["updatedMCPToolOutput"] + assert "uncommitted_changes" in text + assert "changed_files" in text + assert "\u2713 OK" not in text + + +# PHK-21 +def test_fmt_clone_repo_unpublished_branch_warning(): + """clone_repo unpublished_branch must show WARNING, not OK.""" + event = { + "tool_name": "mcp__plugin_autoskillit_autoskillit__clone_repo", + "tool_response": json.dumps( + { + "unpublished_branch": "true", + "branch": "feat/x", + "source_dir": "/src", + } + ), + } + out, _ = _run_hook(event=event) + text = json.loads(out)["hookSpecificOutput"]["updatedMCPToolOutput"] + assert "unpublished_branch" in text + assert "branch" in text + assert "\u2713 OK" not in text + + +# PHK-22 +def test_fmt_clone_repo_success_includes_clone_path(): + """clone_repo success must include clone_path, source_dir, remote_url.""" + event = { + "tool_name": "mcp__plugin_autoskillit_autoskillit__clone_repo", + "tool_response": json.dumps( + { + "clone_path": "/tmp/clone", + "source_dir": "/src", + "remote_url": "https://example.com", + } + ), + } + out, _ = _run_hook(event=event) + text = json.loads(out)["hookSpecificOutput"]["updatedMCPToolOutput"] + assert "/tmp/clone" in text + assert "/src" in text + assert "https://example.com" in text + + +# PHK-23 +def test_fmt_clone_repo_remote_url_failure_includes_stderr(): + """clone_repo remote_url failure must include stderr.""" + event = { + "tool_name": "mcp__plugin_autoskillit_autoskillit__clone_repo", + "tool_response": json.dumps( + { + "error": "remote_url_rewrite_failed", + "clone_path": "/tmp/clone", + "remote_url": "https://example.com", + "stderr": "fatal: unable to set url", + } + ), + } + out, _ = _run_hook(event=event) + text = json.loads(out)["hookSpecificOutput"]["updatedMCPToolOutput"] + assert "fatal: unable to set url" in text + + +# PHK-24 +def test_fmt_merge_worktree_success_shows_metadata(): + """merge_worktree success must show merge_succeeded and merged_branch.""" + event = { + "tool_name": "mcp__plugin_autoskillit_autoskillit__merge_worktree", + "tool_response": json.dumps( + { + "merge_succeeded": True, + "merged_branch": "feat/x", + "into_branch": "main", + "worktree_removed": True, + "branch_deleted": True, + } + ), + } + out, _ = _run_hook(event=event) + text = json.loads(out)["hookSpecificOutput"]["updatedMCPToolOutput"] + assert "merge_succeeded" in text + assert "merged_branch" in text + assert "\u2713" in text + + +# PHK-25 +def test_fmt_merge_worktree_dirty_tree_shows_files(): + """merge_worktree dirty tree must show dirty_files content.""" + event = { + "tool_name": "mcp__plugin_autoskillit_autoskillit__merge_worktree", + "tool_response": json.dumps( + { + "error": "dirty working tree", + "state": "DIRTY_TREE", + "dirty_files": ["M a.py", "M b.py"], + } + ), + } + out, _ = _run_hook(event=event) + text = json.loads(out)["hookSpecificOutput"]["updatedMCPToolOutput"] + assert "M a.py" in text + assert "M b.py" in text + + +# PHK-26 +def test_fmt_run_skill_pipeline_includes_stderr(tmp_path): + """run_skill pipeline mode must include stderr.""" + config_path = tmp_path.joinpath(*_HOOK_CONFIG_PATH_COMPONENTS) + config_path.parent.mkdir(parents=True, exist_ok=True) + config_path.write_text('{"quota_guard": {}}') + + event = _make_run_skill_event( + success=False, + subtype="execution_failed", + stderr="ImportError: no module named foo", + exit_code=1, + result="", + ) + out, _ = _run_hook(event=event, cwd=tmp_path) + text = json.loads(out)["hookSpecificOutput"]["updatedMCPToolOutput"] + assert "ImportError: no module named foo" in text + + +# PHK-27 +def test_fmt_test_check_error_key_visible(): + """test_check error key must be visible in output.""" + event = { + "tool_name": "mcp__plugin_autoskillit_autoskillit__test_check", + "tool_response": json.dumps({"passed": False, "error": "Test runner not configured"}), + } + out, _ = _run_hook(event=event) + text = json.loads(out)["hookSpecificOutput"]["updatedMCPToolOutput"] + assert "error: Test runner not configured" in text + + +# PHK-28 +def test_tool_exception_subtype_routed_before_formatter(): + """tool_exception subtype must be handled before per-tool dispatch.""" + event = { + "tool_name": "mcp__plugin_autoskillit_autoskillit__merge_worktree", + "tool_response": json.dumps( + { + "success": False, + "error": "OSError: disk full", + "exit_code": -1, + "subtype": "tool_exception", + } + ), + } + out, _ = _run_hook(event=event) + text = json.loads(out)["hookSpecificOutput"]["updatedMCPToolOutput"] + assert "tool exception" in text.lower() + assert "OSError: disk full" in text + + +# PHK-kill-reason: formatter branches on kill_reason (1e) + + +def test_fmt_run_skill_success_with_kill_after_completion_annotates_exit_code() -> None: + """kill_reason=kill_after_completion must annotate exit_code line.""" + from autoskillit.hooks.formatters._fmt_execution import _fmt_run_skill + + data = { + "success": True, + "result": "Implementation complete.", + "session_id": "abc123", + "subtype": "end_turn", + "is_error": False, + "exit_code": -9, + "kill_reason": "kill_after_completion", + "needs_retry": False, + "retry_reason": "none", + "stderr": "", + } + text = _fmt_run_skill(data, pipeline=False) + assert "exit_code: -9 (infra-terminated after completion" in text, ( + f"Expected kill_after_completion annotation in exit_code line, got: {text!r}" + ) + + +def test_fmt_run_skill_success_with_natural_exit_shows_bare_exit_code() -> None: + """kill_reason=natural_exit must render bare exit_code without annotation.""" + from autoskillit.hooks.formatters._fmt_execution import _fmt_run_skill + + data = { + "success": True, + "result": "Done.", + "session_id": "abc123", + "subtype": "end_turn", + "is_error": False, + "exit_code": 0, + "kill_reason": "natural_exit", + "needs_retry": False, + "retry_reason": "none", + "stderr": "", + } + text = _fmt_run_skill(data, pipeline=False) + assert "exit_code: 0" in text + assert "infra-terminated" not in text + assert "infra-killed" not in text + + +def test_fmt_run_skill_infra_kill_annotates_reason() -> None: + """kill_reason=infra_kill must annotate exit_code with infra-killed.""" + from autoskillit.hooks.formatters._fmt_execution import _fmt_run_skill + + data = { + "success": False, + "result": "", + "session_id": "abc123", + "subtype": "timeout", + "is_error": True, + "exit_code": -9, + "kill_reason": "infra_kill", + "needs_retry": False, + "retry_reason": "none", + "stderr": "", + } + text = _fmt_run_skill(data, pipeline=False) + assert "infra-killed" in text, ( + f"Expected 'infra-killed' annotation for infra_kill reason, got: {text!r}" + ) + + +def test_fmt_run_skill_legacy_payload_without_kill_reason_renders_bare() -> None: + """Payload without kill_reason field must render bare exit_code (backward compat).""" + from autoskillit.hooks.formatters._fmt_execution import _fmt_run_skill + + data = { + "success": True, + "result": "Done.", + "session_id": "abc123", + "subtype": "end_turn", + "is_error": False, + "exit_code": 0, + "needs_retry": False, + "retry_reason": "none", + "stderr": "", + } + text = _fmt_run_skill(data, pipeline=False) + assert "exit_code: 0" in text + assert "infra" not in text + + +def test_fmt_test_check_displays_duration(): + """Pretty output shows duration when present.""" + from autoskillit.hooks.formatters._fmt_execution import _fmt_test_check + + data = {"passed": True, "stdout": "= 10 passed =", "duration_seconds": 12.34} + out = _fmt_test_check(data, False) + assert "12.3s" in out or "12.34" in out + + +def test_fmt_test_check_displays_filter_stats(): + """Pretty output shows filter stats when present.""" + from autoskillit.hooks.formatters._fmt_execution import _fmt_test_check + + data = { + "passed": True, + "stdout": "", + "filter_mode": "conservative", + "tests_selected": 50, + "tests_deselected": 100, + } + out = _fmt_test_check(data, False) + assert "conservative" in out + assert "50" in out + assert "100" in out + + +def test_fmt_test_check_displays_full_run_reason(): + """Pretty output shows full run reason instead of selected/deselected counts.""" + from autoskillit.hooks.formatters._fmt_execution import _fmt_test_check + + data = { + "passed": True, + "stdout": "", + "filter_mode": "conservative", + "full_run_reason": "bucket_a", + } + out = _fmt_test_check(data, False) + assert "full run" in out + assert "bucket" in out.lower() + assert "?" not in out + + +def test_fmt_test_check_shows_question_mark_only_when_stats_truly_unavailable(): + """When filter_mode is set but no counts or reason, output shows '?'.""" + from autoskillit.hooks.formatters._fmt_execution import _fmt_test_check + + data = {"passed": True, "stdout": "", "filter_mode": "conservative"} + out = _fmt_test_check(data, False) + assert "?" in out + assert "full run" not in out + + +def test_fmt_test_check_emits_filter_off_when_no_filter_mode(): + """When filter_mode is absent, output contains 'filter: off'.""" + from autoskillit.hooks.formatters._fmt_execution import _fmt_test_check + + data = {"passed": True, "stdout": "= 10 passed ="} + out = _fmt_test_check(data, False) + assert "filter: off" in out + + +def test_fmt_test_check_emits_filter_off_when_filter_mode_none(): + """Explicit None filter_mode also produces 'filter: off'.""" + from autoskillit.hooks.formatters._fmt_execution import _fmt_test_check + + data = {"passed": True, "stdout": "", "filter_mode": None} + out = _fmt_test_check(data, False) + assert "filter: off" in out + + +def test_fmt_test_check_emits_filter_off_when_filter_mode_empty_string(): + """Empty string filter_mode also produces 'filter: off'.""" + from autoskillit.hooks.formatters._fmt_execution import _fmt_test_check + + data = {"passed": True, "stdout": "", "filter_mode": ""} + out = _fmt_test_check(data, False) + assert "filter: off" in out + + +# --- Provider line rendering --- + + +def test_fmt_run_skill_interactive_no_provider_field_no_provider_line() -> None: + """Legacy payloads without provider_used produce no provider: line.""" + from autoskillit.hooks.formatters._fmt_execution import _fmt_run_skill + + data = { + "success": True, + "result": "Done.", + "session_id": "abc", + "subtype": "end_turn", + "is_error": False, + "exit_code": 0, + "needs_retry": False, + "retry_reason": "none", + "stderr": "", + } + text = _fmt_run_skill(data, pipeline=False) + assert "provider:" not in text + + +def test_fmt_run_skill_interactive_anthropic_provider_suppressed() -> None: + """provider_used='anthropic' must not render a provider line.""" + from autoskillit.hooks.formatters._fmt_execution import _fmt_run_skill + + data = { + "success": True, + "result": "Done.", + "session_id": "abc", + "subtype": "end_turn", + "is_error": False, + "exit_code": 0, + "needs_retry": False, + "retry_reason": "none", + "stderr": "", + "provider_used": "anthropic", + "provider_fallback": False, + } + text = _fmt_run_skill(data, pipeline=False) + assert "provider:" not in text + + +def test_fmt_run_skill_interactive_non_anthropic_provider_rendered() -> None: + """Non-anthropic provider_used renders 'provider: ' line.""" + from autoskillit.hooks.formatters._fmt_execution import _fmt_run_skill + + data = { + "success": True, + "result": "Done.", + "session_id": "abc", + "subtype": "end_turn", + "is_error": False, + "exit_code": 0, + "needs_retry": False, + "retry_reason": "none", + "stderr": "", + "worktree_path": "/tmp/wt", + "provider_used": "bedrock", + "provider_fallback": False, + } + text = _fmt_run_skill(data, pipeline=False) + lines = text.splitlines() + wt_idx = next(i for i, line in enumerate(lines) if "worktree_path:" in line) + provider_idx = next(i for i, line in enumerate(lines) if "provider:" in line) + assert provider_idx == wt_idx + 1 + assert "provider: bedrock" in text + assert "[FALLBACK]" not in text + + +def test_fmt_run_skill_interactive_provider_fallback_annotation() -> None: + """provider_fallback=True appends [FALLBACK] to the provider line.""" + from autoskillit.hooks.formatters._fmt_execution import _fmt_run_skill + + data = { + "success": True, + "result": "Done.", + "session_id": "abc", + "subtype": "end_turn", + "is_error": False, + "exit_code": 0, + "needs_retry": False, + "retry_reason": "none", + "stderr": "", + "provider_used": "vertex", + "provider_fallback": True, + } + text = _fmt_run_skill(data, pipeline=False) + assert "provider: vertex [FALLBACK]" in text + + +def test_fmt_run_skill_interactive_provider_before_token_usage() -> None: + """Provider line appears before the blank line that opens token_usage.""" + from autoskillit.hooks.formatters._fmt_execution import _fmt_run_skill + + data = { + "success": True, + "result": "Done.", + "session_id": "abc", + "subtype": "end_turn", + "is_error": False, + "exit_code": 0, + "needs_retry": False, + "retry_reason": "none", + "stderr": "", + "provider_used": "bedrock", + "provider_fallback": False, + "token_usage": {"input_tokens": 1000, "output_tokens": 500}, + } + text = _fmt_run_skill(data, pipeline=False) + lines = text.splitlines() + provider_idx = next(i for i, line in enumerate(lines) if "provider:" in line) + tokens_idx = next(i for i, line in enumerate(lines) if "tokens_uncached:" in line) + assert provider_idx < tokens_idx + + +def test_fmt_run_skill_pipeline_non_anthropic_provider_rendered() -> None: + """Pipeline mode renders provider line for non-anthropic providers.""" + from autoskillit.hooks.formatters._fmt_execution import _fmt_run_skill + + data = { + "success": True, + "result": "Done.", + "session_id": "abc", + "subtype": "end_turn", + "is_error": False, + "exit_code": 0, + "needs_retry": False, + "retry_reason": "none", + "stderr": "", + "worktree_path": "/tmp/wt", + "provider_used": "bedrock", + "provider_fallback": False, + } + text = _fmt_run_skill(data, pipeline=True) + assert "provider: bedrock" in text + assert "[FALLBACK]" not in text + + +def test_fmt_run_skill_pipeline_provider_fallback_annotation() -> None: + """Pipeline mode appends [FALLBACK] when provider_fallback is True.""" + from autoskillit.hooks.formatters._fmt_execution import _fmt_run_skill + + data = { + "success": True, + "result": "Done.", + "session_id": "abc", + "subtype": "end_turn", + "is_error": False, + "exit_code": 0, + "needs_retry": False, + "retry_reason": "none", + "stderr": "", + "provider_used": "bedrock", + "provider_fallback": True, + } + text = _fmt_run_skill(data, pipeline=True) + assert "provider: bedrock [FALLBACK]" in text + + +def test_fmt_run_skill_pipeline_anthropic_suppressed() -> None: + """Pipeline mode suppresses provider line for anthropic.""" + from autoskillit.hooks.formatters._fmt_execution import _fmt_run_skill + + data = { + "success": True, + "result": "Done.", + "session_id": "abc", + "subtype": "end_turn", + "is_error": False, + "exit_code": 0, + "needs_retry": False, + "retry_reason": "none", + "stderr": "", + "provider_used": "anthropic", + "provider_fallback": False, + } + text = _fmt_run_skill(data, pipeline=True) + assert "provider:" not in text + + +def test_fmt_run_skill_pipeline_provider_position() -> None: + """Pipeline provider line appears after worktree_path and before result.""" + from autoskillit.hooks.formatters._fmt_execution import _fmt_run_skill + + data = { + "success": True, + "result": "Implementation done.", + "session_id": "abc", + "subtype": "end_turn", + "is_error": False, + "exit_code": 0, + "needs_retry": False, + "retry_reason": "none", + "stderr": "", + "worktree_path": "/tmp/wt", + "provider_used": "bedrock", + "provider_fallback": False, + } + text = _fmt_run_skill(data, pipeline=True) + lines = text.splitlines() + wt_idx = next(i for i, line in enumerate(lines) if "worktree_path:" in line) + provider_idx = next(i for i, line in enumerate(lines) if "provider:" in line) + result_idx = next(i for i, line in enumerate(lines) if line.startswith("result:")) + assert wt_idx < provider_idx < result_idx + + +def test_make_run_skill_event_default_includes_provider_fields() -> None: + """_make_run_skill_event with no args includes provider_used='' and provider_fallback=False.""" + from tests.infra._pretty_output_helpers import _make_run_skill_event + + event = _make_run_skill_event() + payload = json.loads(event["tool_response"]) + assert payload["provider_used"] == "" + assert payload["provider_fallback"] is False + + +def test_make_run_skill_event_custom_provider_values() -> None: + """_make_run_skill_event passes through custom provider values.""" + from tests.infra._pretty_output_helpers import _make_run_skill_event + + event = _make_run_skill_event(provider_used="anthropic", provider_fallback=True) + payload = json.loads(event["tool_response"]) + assert payload["provider_used"] == "anthropic" + assert payload["provider_fallback"] is True diff --git a/tests/infra/test_pretty_output_generic_and_wrap.py b/tests/infra/test_pretty_output_generic_and_wrap.py new file mode 100644 index 0000000000..4859717c4a --- /dev/null +++ b/tests/infra/test_pretty_output_generic_and_wrap.py @@ -0,0 +1,275 @@ +"""Tests: pretty_output generic formatter and Claude Code double-wrap.""" + +from __future__ import annotations + +import json + +import pytest + +from tests.infra._pretty_output_helpers import ( + _run_hook, + _wrap_for_claude_code, +) + +pytestmark = [pytest.mark.layer("infra"), pytest.mark.medium] + + +# PHK-29 +def test_fmt_generic_preserves_list_values(): + """Generic formatter must render list values, not silently drop them.""" + event = { + "tool_name": "mcp__plugin_autoskillit_autoskillit__some_tool", + "tool_response": json.dumps({"total": 3, "items": ["a", "b", "c"]}), + } + out, _ = _run_hook(event=event) + text = json.loads(out)["hookSpecificOutput"]["updatedMCPToolOutput"] + assert "items" in text + assert "a" in text + assert "b" in text + assert "c" in text + + +# PHK-30 +def test_fmt_generic_preserves_dict_values(): + """Generic formatter must render dict values, not silently drop them.""" + event = { + "tool_name": "mcp__plugin_autoskillit_autoskillit__some_tool", + "tool_response": json.dumps( + {"status": "ok", "metadata": {"version": "1.0", "author": "test"}} + ), + } + out, _ = _run_hook(event=event) + text = json.loads(out)["hookSpecificOutput"]["updatedMCPToolOutput"] + assert "metadata" in text + assert "version" in text + assert "1.0" in text + assert "author" in text + + +# PHK-31 +def test_fmt_generic_read_db_rows_visible(): + """read_db rows and columns must be visible through generic formatter.""" + event = { + "tool_name": "mcp__plugin_autoskillit_autoskillit__read_db", + "tool_response": json.dumps( + { + "rows": [["id1", "value1"], ["id2", "value2"]], + "columns": ["id", "val"], + "row_count": 2, + "truncated": False, + } + ), + } + out, _ = _run_hook(event=event) + text = json.loads(out)["hookSpecificOutput"]["updatedMCPToolOutput"] + assert "rows" in text + assert "id1" in text + assert "value2" in text + assert "columns" in text + assert "id" in text + + +# PHK-32 +def test_fmt_generic_pipeline_report_failures_visible(): + """get_pipeline_report failures must be visible through generic formatter.""" + event = { + "tool_name": "mcp__plugin_autoskillit_autoskillit__get_pipeline_report", + "tool_response": json.dumps( + { + "total_failures": 2, + "failures": [ + {"step": "test", "error": "assertion failed"}, + {"step": "build", "error": "compile error"}, + ], + } + ), + } + out, _ = _run_hook(event=event) + text = json.loads(out)["hookSpecificOutput"]["updatedMCPToolOutput"] + assert "failures" in text + assert "assertion failed" in text + assert "compile error" in text + + +# PHK-33 +def test_fmt_generic_deeply_nested_truncated(): + """Deeply nested structures must be rendered as truncated compact JSON.""" + deep = {"a": {"b": {"c": {"d": {"e": "deep"}}}}} + event = { + "tool_name": "mcp__plugin_autoskillit_autoskillit__some_tool", + "tool_response": json.dumps({"info": "top", "nested": deep}), + } + out, _ = _run_hook(event=event) + text = json.loads(out)["hookSpecificOutput"]["updatedMCPToolOutput"] + assert "nested" in text + assert "deep" in text + + +# PHK-34 +def test_wrapped_run_cmd_success(tmp_path): + """Wrapped run_cmd success must unwrap and show checkmark, exit_code, stdout.""" + event = { + "tool_name": "mcp__plugin_autoskillit_autoskillit__run_cmd", + "tool_response": _wrap_for_claude_code( + {"success": True, "exit_code": 0, "stdout": "file1.py\nfile2.py\n", "stderr": ""} + ), + } + out, _ = _run_hook(event=event, cwd=tmp_path) + text = json.loads(out)["hookSpecificOutput"]["updatedMCPToolOutput"] + assert "\u2713" in text + assert "success: True" in text + assert "exit_code: 0" in text + assert "file1.py" in text + assert "file2.py" in text + + +# PHK-35 +def test_wrapped_run_cmd_failure(tmp_path): + """Wrapped run_cmd failure must unwrap and show cross, exit_code, stderr.""" + event = { + "tool_name": "mcp__plugin_autoskillit_autoskillit__run_cmd", + "tool_response": _wrap_for_claude_code( + {"success": False, "exit_code": 127, "stdout": "", "stderr": "command not found"} + ), + } + out, _ = _run_hook(event=event, cwd=tmp_path) + text = json.loads(out)["hookSpecificOutput"]["updatedMCPToolOutput"] + assert "\u2717" in text + assert "FAIL" in text + assert "exit_code: 127" in text + assert "command not found" in text + + +# PHK-36 +def test_wrapped_run_skill_success(tmp_path): + """Wrapped run_skill success must unwrap and show all fields.""" + payload = { + "success": True, + "result": "Implementation complete.", + "session_id": "abc123", + "subtype": "end_turn", + "is_error": False, + "exit_code": 0, + "needs_retry": False, + "retry_reason": "none", + "stderr": "", + "token_usage": None, + "worktree_path": "", + } + event = { + "tool_name": "mcp__plugin_autoskillit_autoskillit__run_skill", + "tool_response": _wrap_for_claude_code(payload), + } + out, _ = _run_hook(event=event, cwd=tmp_path) + text = json.loads(out)["hookSpecificOutput"]["updatedMCPToolOutput"] + assert "\u2713" in text + assert "success: True" in text + assert "session_id: abc123" in text + + +# PHK-37 +def test_wrapped_kitchen_status(tmp_path): + """Wrapped kitchen_status must unwrap and show status fields.""" + payload = { + "package_version": "0.4.0", + "plugin_json_version": "0.4.0", + "versions_match": True, + "tools_enabled": True, + "token_usage_verbosity": "summary", + "quota_guard_enabled": True, + "github_token_configured": True, + "github_default_repo": "acme/myrepo", + } + event = { + "tool_name": "mcp__plugin_autoskillit_autoskillit__kitchen_status", + "tool_response": _wrap_for_claude_code(payload), + } + out, _ = _run_hook(event=event, cwd=tmp_path) + text = json.loads(out)["hookSpecificOutput"]["updatedMCPToolOutput"] + assert "kitchen_status" in text + assert "package_version" in text + assert "tools_enabled" in text + + +# PHK-38 +def test_wrapped_plain_text_result_passes_through(tmp_path): + """Plain text result envelope must not crash — passes through to generic.""" + event = { + "tool_name": "mcp__plugin_autoskillit_autoskillit__open_kitchen", + "tool_response": json.dumps( + {"result": "Kitchen is open. AutoSkillit tools are ready for service."} + ), + } + out, _ = _run_hook(event=event, cwd=tmp_path) + text = json.loads(out)["hookSpecificOutput"]["updatedMCPToolOutput"] + assert "open_kitchen" in text + assert "Kitchen is open" in text + + +# PHK-39 +def test_wrapped_gate_error_still_detected(tmp_path): + """Wrapped gate_error subtype must unwrap and route to gate_error formatter.""" + payload = { + "subtype": "gate_error", + "result": "Kitchen is closed.", + "success": False, + "is_error": True, + } + event = { + "tool_name": "mcp__plugin_autoskillit_autoskillit__run_skill", + "tool_response": _wrap_for_claude_code(payload), + } + out, _ = _run_hook(event=event, cwd=tmp_path) + text = json.loads(out)["hookSpecificOutput"]["updatedMCPToolOutput"] + assert "Gate Error" in text + + +# PHK-40 +def test_wrapped_tool_exception_still_detected(tmp_path): + """Wrapped tool_exception subtype must unwrap and route to exception formatter.""" + payload = { + "subtype": "tool_exception", + "error": "TimeoutError: process hung", + "exit_code": -1, + "success": False, + } + event = { + "tool_name": "mcp__plugin_autoskillit_autoskillit__run_cmd", + "tool_response": _wrap_for_claude_code(payload), + } + out, _ = _run_hook(event=event, cwd=tmp_path) + text = json.loads(out)["hookSpecificOutput"]["updatedMCPToolOutput"] + assert "tool exception" in text.lower() + assert "TimeoutError: process hung" in text + + +# PHK-47/48: _fmt_generic list-of-dicts hardening tests + + +def test_fmt_generic_list_of_dicts_renders_per_item_not_blob(): + """PHK-47: Generic formatter renders list-of-dicts per item, not a truncated JSON blob.""" + from tests.infra._pretty_output_helpers import _make_event + + failures = [ + {"step": f"step_{i}", "error": f"Error message {i}", "exit_code": 1} for i in range(10) + ] + event = _make_event("some_tool", {"failures": failures}) + out, _ = _run_hook(event) + text = json.loads(out)["hookSpecificOutput"]["updatedMCPToolOutput"] + for i in range(10): + assert f"step_{i}" in text, f"step_{i} missing — possible truncation" + assert "... and" not in text, "Output was truncated" + + +def test_fmt_generic_list_of_dicts_caps_at_20_items(): + """PHK-48: Generic formatter caps list-of-dicts at 20 items with overflow note.""" + from tests.infra._pretty_output_helpers import _make_event + + items = [{"key": f"item-{i}", "val": f"value-{i}"} for i in range(25)] + event = _make_event("some_tool", {"items": items}) + out, _ = _run_hook(event) + text = json.loads(out)["hookSpecificOutput"]["updatedMCPToolOutput"] + assert "item-0" in text + assert "item-19" in text + assert "item-20" not in text + assert "and 5 more" in text diff --git a/tests/infra/test_pretty_output_hook_infra.py b/tests/infra/test_pretty_output_hook_infra.py new file mode 100644 index 0000000000..93b8fda1eb --- /dev/null +++ b/tests/infra/test_pretty_output_hook_infra.py @@ -0,0 +1,221 @@ +"""Tests: pretty_output hook infrastructure, fail-open, and coverage contracts.""" + +from __future__ import annotations + +import json + +import pytest + +from autoskillit.core.types import ChannelConfirmation, TerminationReason +from autoskillit.execution.headless import _build_skill_result +from autoskillit.hooks.formatters.pretty_output_hook import _format_response +from tests.conftest import _make_result +from tests.infra._pretty_output_helpers import ( + _run_hook, + _wrap_for_claude_code, + _wrap_plain_str_for_claude_code, +) + +pytestmark = [pytest.mark.layer("infra"), pytest.mark.medium] + + +# PHK-1 +def test_hook_script_exists(): + """pretty_output.py must exist in the hooks directory.""" + from autoskillit.core.paths import pkg_root + + assert (pkg_root() / "hooks" / "formatters" / "pretty_output_hook.py").exists() + + +# PHK-2 +def test_hook_emits_posttooluse_event_name(): + """Hook output JSON must have hookSpecificOutput.hookEventName == 'PostToolUse'.""" + event = { + "tool_name": "mcp__plugin_autoskillit_autoskillit__run_cmd", + "tool_response": json.dumps( + {"success": True, "exit_code": 0, "stdout": "hi", "stderr": ""} + ), + } + out, _ = _run_hook(event=event) + assert out.strip(), "Expected non-empty output" + data = json.loads(out) + assert data["hookSpecificOutput"]["hookEventName"] == "PostToolUse" + + +# PHK-3 +def test_hook_emits_updated_mcp_tool_output_field(): + """Hook output must have non-empty hookSpecificOutput.updatedMCPToolOutput.""" + event = { + "tool_name": "mcp__plugin_autoskillit_autoskillit__run_cmd", + "tool_response": json.dumps( + {"success": True, "exit_code": 0, "stdout": "hi", "stderr": ""} + ), + } + out, _ = _run_hook(event=event) + data = json.loads(out) + assert data["hookSpecificOutput"]["updatedMCPToolOutput"] + + +# PHK-4 +def test_hook_fail_open_on_invalid_json_stdin(): + """Non-JSON stdin → exit 0, no stdout output.""" + out, code = _run_hook(raw_stdin="not valid json {{{{") + assert code == 0 + assert out.strip() == "" + + +# PHK-5 +def test_hook_fail_open_on_missing_tool_response(): + """Valid JSON but missing tool_response key → exit 0, no stdout.""" + event = {"tool_name": "mcp__plugin_autoskillit_autoskillit__run_cmd"} + out, code = _run_hook(event=event) + assert code == 0 + assert out.strip() == "" + + +# --------------------------------------------------------------------------- +# PHK-41: Formatter coverage contract +# --------------------------------------------------------------------------- + + +def test_formatter_coverage_contract(): + """PHK-41: Every MCP tool is either in _FORMATTERS or explicitly in _UNFORMATTED_TOOLS.""" + from autoskillit.core.types import GATED_TOOLS, UNGATED_TOOLS + from autoskillit.hooks.formatters.pretty_output_hook import _FORMATTERS, _UNFORMATTED_TOOLS + + all_tools = GATED_TOOLS | UNGATED_TOOLS + covered = set(_FORMATTERS.keys()) | _UNFORMATTED_TOOLS + uncovered = all_tools - covered + assert uncovered == set(), ( + f"Tools have no formatter and are not in _UNFORMATTED_TOOLS: {sorted(uncovered)}. " + "Either add a dedicated formatter or add to _UNFORMATTED_TOOLS." + ) + + +def test_all_formatters_have_coverage_contracts(): + """Every dedicated formatter must have a registered field coverage contract.""" + from autoskillit.hooks.formatters.pretty_output_hook import _FORMATTERS + from tests.infra.conftest import _FORMATTER_COVERAGE_REGISTRY + + uncovered = set(_FORMATTERS.keys()) - set(_FORMATTER_COVERAGE_REGISTRY.keys()) + assert uncovered == set(), ( + f"Formatters without coverage contracts: {sorted(uncovered)}. " + "Define a TypedDict for the tool result, add RENDERED/SUPPRESSED frozensets, " + "and register in _FORMATTER_COVERAGE_REGISTRY." + ) + + +def test_coverage_registry_entries_are_valid(): + """Every registry entry's frozensets must exactly cover its TypedDict annotations.""" + from tests.infra.conftest import _FORMATTER_COVERAGE_REGISTRY + + for name, entry in _FORMATTER_COVERAGE_REGISTRY.items(): + import typing + + all_fields = set(typing.get_type_hints(entry.typed_dict)) + covered = entry.rendered | entry.suppressed + overlap = entry.rendered & entry.suppressed + uncovered = all_fields - covered + extra = covered - all_fields + + assert overlap == set(), ( + f"{name}: fields in both RENDERED and SUPPRESSED: {sorted(overlap)}" + ) + assert uncovered == set(), ( + f"{name}: TypedDict fields without coverage: {sorted(uncovered)}" + ) + assert extra == set(), f"{name}: frozenset entries not in TypedDict: {sorted(extra)}" + + +# --------------------------------------------------------------------------- +# T-3: _wrap_plain_str_for_claude_code helper shape +# --------------------------------------------------------------------------- + + +def test_wrap_plain_str_helper_produces_correct_shape(): + """_wrap_plain_str_for_claude_code produces the real hook event shape.""" + raw = _wrap_plain_str_for_claude_code("hello world") + parsed = json.loads(raw) + assert parsed == {"result": "hello world"} + + +# --------------------------------------------------------------------------- +# T-4/T-5: _UNFORMATTED_TOOLS behavioral gate +# --------------------------------------------------------------------------- + + +def test_unformatted_tools_and_formatters_are_disjoint(): + """_UNFORMATTED_TOOLS and _FORMATTERS must be mutually exclusive.""" + from autoskillit.hooks.formatters.pretty_output_hook import _FORMATTERS, _UNFORMATTED_TOOLS + + overlap = set(_FORMATTERS) & _UNFORMATTED_TOOLS + assert not overlap, f"Tools in both dispatch tables: {overlap}" + + +def test_unformatted_tool_routes_to_generic_not_named_formatter(tmp_path): + """A tool in _UNFORMATTED_TOOLS must reach _fmt_generic.""" + payload = {"total_failures": 1, "failures": [{"step": "impl", "reason": "red"}]} + event = { + "tool_name": "mcp__plugin_autoskillit_autoskillit__get_pipeline_report", + "tool_response": _wrap_for_claude_code(payload), + } + out, code = _run_hook(event=event, cwd=tmp_path) + assert code == 0 + text = json.loads(out)["hookSpecificOutput"]["updatedMCPToolOutput"] + assert "get_pipeline_report" in text + + +def test_pretty_output_public_surface_unchanged() -> None: + """T-5 (audit finding 8.3): the hook entrypoint and the format router are public surface.""" + import autoskillit.hooks.formatters.pretty_output_hook as p + + assert callable(p.main) + assert callable(p._format_response) + + +# Issue #346 +def test_fmt_run_skill_contradictory_subtype_never_renders_fail_success(): + """Test A: full pipeline — COMPLETED+empty never renders 'FAIL [success]'.""" + stdout = json.dumps( + { + "type": "result", + "subtype": "success", + "is_error": False, + "result": "", + "session_id": "s1", + } + ) + result = _make_result( + returncode=0, + stdout=stdout, + termination_reason=TerminationReason.COMPLETED, + channel_confirmation=ChannelConfirmation.UNMONITORED, + ) + sr = _build_skill_result(result, completion_marker="", skill_command="/test") + assert sr.success is False, "Precondition: this path must produce a failure" + + payload = json.loads(sr.to_json()) + + pipeline_out = _format_response( + "mcp__plugin_autoskillit_autoskillit__run_skill", + json.dumps(payload), + pipeline=True, + ) + assert pipeline_out is not None + assert "FAIL [success]" not in pipeline_out, ( + f"Pipeline mode rendered contradictory 'FAIL [success]': {pipeline_out!r}" + ) + assert "FAIL [empty_result]" in pipeline_out, ( + f"Expected 'FAIL [empty_result]' in pipeline output: {pipeline_out!r}" + ) + + interactive_out = _format_response( + "mcp__plugin_autoskillit_autoskillit__run_skill", + json.dumps(payload), + pipeline=False, + ) + assert interactive_out is not None + cross = "\u2717" + assert f"{cross} success" not in interactive_out, ( + f"Interactive mode rendered contradictory '{cross} success': {interactive_out!r}" + ) diff --git a/tests/infra/test_pretty_output_integration.py b/tests/infra/test_pretty_output_integration.py index 215dccff15..6e046183b3 100644 --- a/tests/infra/test_pretty_output_integration.py +++ b/tests/infra/test_pretty_output_integration.py @@ -28,8 +28,8 @@ class TestFormatterSchemaConsistency: @pytest.mark.anyio async def test_list_recipes_tool_output_through_hook(self, tool_ctx): """PHK-E1: list_recipes real output contains all recipe names through hook.""" - from autoskillit.hooks.pretty_output_hook import _format_response - from autoskillit.server.tools_recipe import list_recipes + from autoskillit.hooks.formatters.pretty_output_hook import _format_response + from autoskillit.server.tools.tools_recipe import list_recipes result_json = await list_recipes() output = _format_response("mcp__autoskillit__list_recipes", result_json, pipeline=False) @@ -41,10 +41,10 @@ async def test_list_recipes_tool_output_through_hook(self, tool_ctx): ) @pytest.mark.anyio - async def test_kitchen_status_tool_output_through_hook(self, tool_ctx): + async def test_kitchen_status_tool_output_through_hook(self, tool_ctx_kitchen_open): """PHK-E2: kitchen_status real output contains all key fields through hook.""" - from autoskillit.hooks.pretty_output_hook import _format_response - from autoskillit.server.tools_status import kitchen_status + from autoskillit.hooks.formatters.pretty_output_hook import _format_response + from autoskillit.server.tools.tools_status import kitchen_status result_json = await kitchen_status() output = _format_response("mcp__autoskillit__kitchen_status", result_json, pipeline=False) @@ -58,7 +58,7 @@ async def test_kitchen_status_tool_output_through_hook(self, tool_ctx): @pytest.mark.anyio async def test_integration_get_token_summary_table_not_eaten(self, tool_ctx): """T-6: Real get_token_summary(format='table') response must survive _format_response.""" - from autoskillit.hooks.pretty_output_hook import _format_response + from autoskillit.hooks.formatters.pretty_output_hook import _format_response from autoskillit.pipeline.telemetry_fmt import TelemetryFormatter # Simulate what the real tool handler returns for format="table" diff --git a/tests/infra/test_pretty_output_recipe.py b/tests/infra/test_pretty_output_recipe.py new file mode 100644 index 0000000000..c7dfdbc2c4 --- /dev/null +++ b/tests/infra/test_pretty_output_recipe.py @@ -0,0 +1,819 @@ +"""Tests: pretty_output token/timing, load_recipe, list_recipes, open_kitchen, deduplication.""" + +from __future__ import annotations + +import json + +import pytest + +from autoskillit.hooks.formatters.pretty_output_hook import _format_response +from tests.infra._pretty_output_helpers import ( + REALISTIC_RECIPE_YAML, + _make_event, + _run_hook, +) + +pytestmark = [pytest.mark.layer("infra"), pytest.mark.medium] + + +# PHK-15 +def test_format_get_token_summary_compact(): + """get_token_summary must show compact per-step lines and totals.""" + event = { + "tool_name": "mcp__plugin_autoskillit_autoskillit__get_token_summary", + "tool_response": json.dumps( + { + "steps": [ + { + "step_name": "investigate", + "invocation_count": 1, + "input_tokens": 45200, + "output_tokens": 12800, + "cache_read_input_tokens": 1200000, + "cache_creation_input_tokens": 0, + }, + { + "step_name": "make_plan", + "invocation_count": 2, + "input_tokens": 30000, + "output_tokens": 8000, + "cache_read_input_tokens": 0, + "cache_creation_input_tokens": 500000, + }, + { + "step_name": "implement", + "invocation_count": 1, + "input_tokens": 60000, + "output_tokens": 15000, + "cache_read_input_tokens": 2000000, + "cache_creation_input_tokens": 0, + }, + ], + "total": { + "input_tokens": 135200, + "output_tokens": 35800, + "cache_read_input_tokens": 3200000, + "cache_creation_input_tokens": 500000, + }, + } + ), + } + out, _ = _run_hook(event=event) + text = json.loads(out)["hookSpecificOutput"]["updatedMCPToolOutput"] + assert "token_summary" in text + assert "investigate x1 [uc:45.2k out:12.8k cr:1.2M pk:0 cw:0 turns:0 t:0.0s]" in text + assert "make_plan x2 [uc:30.0k out:8.0k cr:0 pk:0 cw:500.0k turns:0 t:0.0s]" in text + assert "implement x1 [uc:60.0k out:15.0k cr:2.0M pk:0 cw:0 turns:0 t:0.0s]" in text + assert "total_uncached:" in text + assert "total_out:" in text + assert "total_cache_read:" in text + assert "total_peak_context:" in text + assert "total_cache_write:" in text + + +# T7 +def test_fmt_get_token_summary_prefers_wall_clock_seconds(): + """_fmt_get_token_summary prefers wall_clock_seconds over elapsed_seconds.""" + event = { + "tool_name": "mcp__plugin_autoskillit_autoskillit__get_token_summary", + "tool_response": json.dumps( + { + "steps": [ + { + "step_name": "implement", + "input_tokens": 5000, + "output_tokens": 1200, + "cache_creation_input_tokens": 200, + "cache_read_input_tokens": 3000, + "invocation_count": 2, + "wall_clock_seconds": 150.0, + "elapsed_seconds": 123.4, + } + ], + "total": { + "input_tokens": 5000, + "output_tokens": 1200, + "cache_creation_input_tokens": 200, + "cache_read_input_tokens": 3000, + "total_elapsed_seconds": 123.4, + }, + "mcp_responses": {"steps": [], "total": {}}, + } + ), + } + out, _ = _run_hook(event=event) + data = json.loads(out) + rendered = data["hookSpecificOutput"]["updatedMCPToolOutput"] + assert "implement" in rendered + assert "t:150.0s" in rendered + + +# T7b +def test_fmt_get_token_summary_falls_back_to_elapsed(): + """_fmt_get_token_summary falls back to elapsed_seconds when no wall_clock.""" + event = { + "tool_name": "mcp__plugin_autoskillit_autoskillit__get_token_summary", + "tool_response": json.dumps( + { + "steps": [ + { + "step_name": "plan", + "input_tokens": 100, + "output_tokens": 50, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + "invocation_count": 1, + "elapsed_seconds": 42.5, + } + ], + "total": {"input_tokens": 100, "output_tokens": 50}, + "mcp_responses": {"steps": [], "total": {}}, + } + ), + } + out, _ = _run_hook(event=event) + data = json.loads(out) + rendered = data["hookSpecificOutput"]["updatedMCPToolOutput"] + assert "t:42.5s" in rendered + + +def test_hook_token_summary_output_equivalent_to_canonical(): + """1g: Hook inline _fmt_get_token_summary produces identical output to + TelemetryFormatter.format_compact_kv for the same input data.""" + from autoskillit.hooks.formatters.pretty_output_hook import _fmt_get_token_summary + from autoskillit.pipeline.telemetry_fmt import TelemetryFormatter + + data = { + "steps": [ + { + "step_name": "investigate", + "input_tokens": 7000, + "output_tokens": 5939, + "cache_creation_input_tokens": 8495, + "cache_read_input_tokens": 252179, + "invocation_count": 1, + "wall_clock_seconds": 45.0, + "elapsed_seconds": 40.0, + }, + { + "step_name": "implement", + "input_tokens": 2031000, + "output_tokens": 122306, + "cache_creation_input_tokens": 280601, + "cache_read_input_tokens": 19071323, + "invocation_count": 3, + "wall_clock_seconds": 492.0, + "elapsed_seconds": 480.0, + }, + ], + "total": { + "input_tokens": 2038000, + "output_tokens": 128245, + "cache_creation_input_tokens": 289096, + "cache_read_input_tokens": 19323502, + "total_elapsed_seconds": 537.0, + }, + "mcp_responses": { + "steps": [], + "total": {"total_invocations": 42, "total_estimated_response_tokens": 5000}, + }, + } + + hook_output = _fmt_get_token_summary(data, _pipeline=False) + canonical_output = TelemetryFormatter.format_compact_kv( + data["steps"], data["total"], mcp_responses=data["mcp_responses"] + ) + assert hook_output == canonical_output, ( + f"Hook and canonical formatter produce different output:\n" + f"HOOK:\n{hook_output}\n\nCANONICAL:\n{canonical_output}" + ) + + +def test_hook_token_summary_non_anthropic_equivalent_to_canonical(): + """1g-ext: Hook and canonical produce identical * annotation and footnote.""" + from autoskillit.hooks.formatters.pretty_output_hook import _fmt_get_token_summary + from autoskillit.pipeline.telemetry_fmt import TelemetryFormatter + + data = { + "steps": [ + { + "step_name": "plan", + "model": "claude-sonnet-4-6", + "input_tokens": 7000, + "output_tokens": 5939, + "cache_creation_input_tokens": 8495, + "cache_read_input_tokens": 252179, + "invocation_count": 1, + "wall_clock_seconds": 45.0, + "elapsed_seconds": 40.0, + }, + { + "step_name": "implement", + "model": "MiniMax-M2.7-highspeed", + "input_tokens": 2031000, + "output_tokens": 122306, + "cache_creation_input_tokens": 280601, + "cache_read_input_tokens": 19071323, + "invocation_count": 3, + "wall_clock_seconds": 492.0, + "elapsed_seconds": 480.0, + }, + ], + "total": { + "input_tokens": 2038000, + "output_tokens": 128245, + "cache_creation_input_tokens": 289096, + "cache_read_input_tokens": 19323502, + "total_elapsed_seconds": 537.0, + }, + "mcp_responses": { + "total": {"total_invocations": 42, "total_estimated_response_tokens": 5000} + }, + } + hook_output = _fmt_get_token_summary(data, _pipeline=False) + canonical_output = TelemetryFormatter.format_compact_kv( + data["steps"], data["total"], mcp_responses=data["mcp_responses"] + ) + assert hook_output == canonical_output + + +def test_fmt_run_skill_interactive_shows_four_token_fields(): + """_fmt_run_skill interactive mode shows all 4 token fields.""" + data = { + "success": True, + "subtype": "COMPLETED", + "exit_code": 0, + "needs_retry": False, + "result": "done", + "token_usage": { + "input_tokens": 5000, + "output_tokens": 3000, + "cache_read_input_tokens": 200000, + "cache_creation_input_tokens": 8000, + }, + } + rendered = _format_response( + "mcp__plugin_autoskillit_autoskillit__run_skill", + json.dumps({"result": json.dumps(data)}), + pipeline=False, + ) + assert rendered is not None + assert "tokens_uncached:" in rendered + assert "tokens_out:" in rendered + assert "tokens_cache_read:" in rendered + assert "tokens_cache_write:" in rendered + + +def test_fmt_run_skill_suppresses_zero_cache_fields(): + """_fmt_run_skill suppresses tokens_cache_read and tokens_cache_write when both are 0.""" + data = { + "success": True, + "subtype": "COMPLETED", + "exit_code": 0, + "needs_retry": False, + "result": "done", + "token_usage": { + "input_tokens": 5000, + "output_tokens": 3000, + "cache_read_input_tokens": 0, + "cache_creation_input_tokens": 0, + }, + } + rendered = _format_response( + "mcp__plugin_autoskillit_autoskillit__run_skill", + json.dumps({"result": json.dumps(data)}), + pipeline=False, + ) + assert rendered is not None + assert "tokens_uncached:" in rendered + assert "tokens_out:" in rendered + assert "tokens_cache_read:" not in rendered + assert "tokens_cache_write:" not in rendered + + +def test_fmt_get_timing_summary_renders_compact(): + """get_timing_summary dedicated formatter renders compact Markdown-KV.""" + event = { + "tool_name": "mcp__autoskillit__get_timing_summary", + "tool_response": json.dumps( + { + "steps": [ + {"step_name": "clone", "total_seconds": 4.0, "invocation_count": 1}, + {"step_name": "implement", "total_seconds": 492.0, "invocation_count": 3}, + ], + "total": {"total_seconds": 496.0}, + } + ), + } + out, _ = _run_hook(event=event) + data = json.loads(out) + rendered = data["hookSpecificOutput"]["updatedMCPToolOutput"] + assert "## timing_summary" in rendered + assert "clone x1" in rendered + assert "implement x3" in rendered + assert "dur:4s" in rendered + assert "dur:8m 12s" in rendered + assert "total:" in rendered + + +def test_get_token_summary_format_table_passes_through_unmodified(tmp_path): + """get_token_summary(format='table') returns pre-formatted markdown; hook passes it through.""" + table = ( + "## Token Usage Summary\n\n| Step | input | output |\n|---|---|---|\n| impl | 45k | 12k |" + ) + event = { + "tool_name": "mcp__plugin_autoskillit_autoskillit__get_token_summary", + "tool_response": json.dumps({"result": table}), + } + out, code = _run_hook(event=event, cwd=tmp_path) + assert code == 0 + text = json.loads(out)["hookSpecificOutput"]["updatedMCPToolOutput"] + assert "## Token Usage Summary" in text + assert "| impl |" in text + assert text.strip() != "## token_summary" + + +def test_get_timing_summary_format_table_passes_through_unmodified(tmp_path): + """get_timing_summary(format='table') returns pre-formatted markdown; hook passes through.""" + table = ( + "## Step Timing Summary\n\n" + "| Step | Duration | Invocations |\n|---|---|---|\n| clone | 4s | 1 |" + ) + event = { + "tool_name": "mcp__plugin_autoskillit_autoskillit__get_timing_summary", + "tool_response": json.dumps({"result": table}), + } + out, code = _run_hook(event=event, cwd=tmp_path) + assert code == 0 + text = json.loads(out)["hookSpecificOutput"]["updatedMCPToolOutput"] + assert "## Step Timing Summary" in text + assert "| clone |" in text + assert text.strip() != "## timing_summary" + + +# PHK-50: open_kitchen combined formatter +def test_fmt_open_kitchen_combined_response(): + """open_kitchen with recipe returns combined kitchen+recipe format.""" + payload = { + "content": "name: my-recipe\nsteps:\n do:\n tool: run_cmd\n", + "valid": True, + "suggestions": [], + "kitchen": "open", + "version": "1.2.3", + } + formatted = _format_response( + "mcp__autoskillit__open_kitchen", + json.dumps(payload), + pipeline=False, + ) + assert formatted is not None + assert "open_kitchen" in formatted + assert "v1.2.3" in formatted + assert "--- RECIPE ---" in formatted + assert "my-recipe" in formatted + + +def test_fmt_open_kitchen_combined_includes_ingredients_table(): + """Combined open_kitchen response includes pre-formatted ingredients table.""" + payload = { + "content": "name: my-recipe\nsteps: ...", + "valid": True, + "suggestions": [], + "kitchen": "open", + "version": "1.0.0", + "ingredients_table": " Name Description Default\n task The task (required)", + } + formatted = _format_response( + "mcp__autoskillit__open_kitchen", + json.dumps(payload), + pipeline=False, + ) + assert formatted is not None + assert "--- INGREDIENTS TABLE" in formatted + assert "The task" in formatted + + +def test_fmt_open_kitchen_combined_error(): + """Combined open_kitchen with recipe error shows error with kitchen status.""" + payload = { + "error": "No recipe named 'bad' found", + "kitchen": "open", + "version": "1.0.0", + } + formatted = _format_response( + "mcp__autoskillit__open_kitchen", + json.dumps(payload), + pipeline=False, + ) + assert formatted is not None + assert "open_kitchen" in formatted + assert "No recipe named 'bad' found" in formatted + assert "\u2717" in formatted + + +def test_fmt_open_kitchen_plain_text(): + """open_kitchen without recipe returns plain text format.""" + formatted = _format_response( + "mcp__autoskillit__open_kitchen", + json.dumps({"result": "Kitchen is open. AutoSkillit 1.2.3."}), + pipeline=False, + ) + assert formatted is not None + assert "open_kitchen" in formatted + assert "Kitchen is open" in formatted + + +# PHK-42/43/44: _fmt_load_recipe tests + + +def test_fmt_load_recipe_suppresses_diagram(): + """Diagram is suppressed — user sees it in terminal, agent doesn't need it.""" + formatted = _format_response( + "mcp__autoskillit__load_recipe", + json.dumps( + { + "content": "name: my-recipe\nsteps: ...", + "diagram": "## my-recipe\nsome diagram graph", + "valid": True, + "suggestions": [], + } + ), + pipeline=False, + ) + assert formatted is not None + assert "some diagram graph" not in formatted + assert "--- RECIPE ---" in formatted + + +def test_fmt_load_recipe_includes_raw_yaml_content(): + """load_recipe response includes the raw YAML so the agent can execute steps.""" + yaml_content = "name: my-recipe\nsteps:\n do_thing:\n tool: run_cmd\n" + formatted = _format_response( + "mcp__autoskillit__load_recipe", + json.dumps( + { + "content": yaml_content, + "diagram": "## my-recipe\nsome diagram", + "valid": True, + "suggestions": [], + } + ), + pipeline=False, + ) + assert formatted is not None + assert "do_thing" in formatted + assert "--- RECIPE ---" in formatted + + +def test_fmt_load_recipe_shows_finding_count(): + """Findings are summarized as a count, not individual bullets.""" + formatted = _format_response( + "mcp__autoskillit__load_recipe", + json.dumps( + { + "content": "name: x", + "diagram": "## x", + "valid": False, + "suggestions": [ + { + "rule": "missing-step", + "message": "Step 'done' not found", + "severity": "error", + }, + {"rule": "unknown-tool", "message": "Tool 'badtool'", "severity": "warning"}, + ], + } + ), + pipeline=False, + ) + assert formatted is not None + assert "2 finding(s)" in formatted + + +# PHK-45/46: _fmt_list_recipes tests + + +def test_fmt_list_recipes_shows_all_names(): + """PHK-45: list_recipes response renders all recipe names.""" + recipes = [ + {"name": f"recipe-{i:02d}", "description": f"Description for recipe {i}", "summary": "..."} + for i in range(10) + ] + event = _make_event("list_recipes", {"recipes": recipes, "count": 10}) + out, _ = _run_hook(event) + text = json.loads(out)["hookSpecificOutput"]["updatedMCPToolOutput"] + for i in range(10): + assert f"recipe-{i:02d}" in text, f"recipe-{i:02d} missing from output" + assert '{"name"' not in text + assert "10" in text + + +def test_fmt_list_recipes_compact_representation(): + """PHK-46: list_recipes renders one line per recipe in 'name: description' format.""" + event = _make_event( + "list_recipes", + { + "recipes": [ + { + "name": "implementation", + "description": "Implement a plan in a worktree", + "summary": "...", + }, + { + "name": "smoke-test", + "description": "Run a smoke-test pipeline", + "summary": "...", + }, + ], + "count": 2, + }, + ) + out, _ = _run_hook(event) + text = json.loads(out)["hookSpecificOutput"]["updatedMCPToolOutput"] + assert "implementation" in text + assert "Implement a plan in a worktree" in text + assert "smoke-test" in text + assert "Run a smoke-test pipeline" in text + + +def test_fmt_load_recipe_field_coverage(): + """Every LoadRecipeResult field must be in RENDERED or SUPPRESSED.""" + from autoskillit.hooks.formatters.pretty_output_hook import ( + _FMT_LOAD_RECIPE_RENDERED, + _FMT_LOAD_RECIPE_SUPPRESSED, + ) + from autoskillit.recipe._api import LoadRecipeResult + + all_fields = set(LoadRecipeResult.__annotations__) + covered = _FMT_LOAD_RECIPE_RENDERED | _FMT_LOAD_RECIPE_SUPPRESSED + uncovered = all_fields - covered + assert uncovered == set(), ( + f"LoadRecipeResult fields have no coverage decision: {sorted(uncovered)}. " + "Add each to _FMT_LOAD_RECIPE_RENDERED or _FMT_LOAD_RECIPE_SUPPRESSED." + ) + extra = covered - all_fields + assert extra == set(), ( + f"Coverage registry references non-existent fields: {sorted(extra)}. Remove stale entries." + ) + + +def test_fmt_load_recipe_derivation_map_coverage(): + """Every key/value in _LOAD_RECIPE_CONTENT_DERIVED_FROM must be in _FMT_LOAD_RECIPE_RENDERED.""" # noqa: E501 + from autoskillit.hooks.formatters.pretty_output_hook import ( + _FMT_LOAD_RECIPE_RENDERED, + _LOAD_RECIPE_CONTENT_DERIVED_FROM, + ) + + for derived_field, source_field in _LOAD_RECIPE_CONTENT_DERIVED_FROM.items(): + assert derived_field in _FMT_LOAD_RECIPE_RENDERED, ( + f"Derived field '{derived_field}' must be in _FMT_LOAD_RECIPE_RENDERED." + ) + assert source_field in _FMT_LOAD_RECIPE_RENDERED, ( + f"Source field '{source_field}' must be in _FMT_LOAD_RECIPE_RENDERED." + ) + + +def test_fmt_load_recipe_renders_error(): + """When error is present, it appears in output with cross mark.""" + formatted = _format_response( + "mcp__autoskillit__load_recipe", + json.dumps({"error": "Recipe 'x' not found", "valid": False}), + pipeline=False, + ) + assert formatted is not None + assert "\u2717" in formatted + assert "Recipe 'x' not found" in formatted + + +def test_fmt_load_recipe_suppresses_kitchen_rules(): + """Kitchen rules are suppressed — they're in the YAML content.""" + formatted = _format_response( + "mcp__autoskillit__load_recipe", + json.dumps( + { + "valid": True, + "diagram": "## test diagram", + "suggestions": [], + "kitchen_rules": ["no raw SQL", "use run_cmd for shell"], + } + ), + pipeline=False, + ) + assert formatted is not None + assert "no raw SQL" not in formatted + + +def test_fmt_load_recipe_suppresses_greeting(): + """Greeting is suppressed — delivered via positional CLI arg instead.""" + formatted = _format_response( + "mcp__autoskillit__load_recipe", + json.dumps( + { + "valid": True, + "diagram": "## test diagram", + "suggestions": [], + "greeting": "Welcome to Good Burger!", + } + ), + pipeline=False, + ) + assert formatted is not None + assert "Welcome to Good Burger!" not in formatted + + +def test_fmt_list_recipes_field_coverage(): + """Every ListRecipesResult field must be in RENDERED or SUPPRESSED.""" + from autoskillit.hooks.formatters.pretty_output_hook import ( + _FMT_LIST_RECIPES_RENDERED, + _FMT_LIST_RECIPES_SUPPRESSED, + ) + from autoskillit.recipe._api import ListRecipesResult + + all_fields = set(ListRecipesResult.__annotations__) + covered = _FMT_LIST_RECIPES_RENDERED | _FMT_LIST_RECIPES_SUPPRESSED + uncovered = all_fields - covered + assert uncovered == set(), ( + f"ListRecipesResult fields have no coverage decision: {sorted(uncovered)}." + ) + extra = covered - all_fields + assert extra == set(), ( + f"Coverage registry references non-existent fields: {sorted(extra)}. Remove stale entries." + ) + + +def test_fmt_recipe_list_item_field_coverage(): + """Every RecipeListItem field must be in RENDERED or SUPPRESSED.""" + from autoskillit.hooks.formatters.pretty_output_hook import ( + _FMT_RECIPE_LIST_ITEM_RENDERED, + _FMT_RECIPE_LIST_ITEM_SUPPRESSED, + ) + from autoskillit.recipe._api import RecipeListItem + + all_fields = set(RecipeListItem.__annotations__) + covered = _FMT_RECIPE_LIST_ITEM_RENDERED | _FMT_RECIPE_LIST_ITEM_SUPPRESSED + uncovered = all_fields - covered + assert uncovered == set(), ( + f"RecipeListItem fields have no coverage decision: {sorted(uncovered)}." + ) + extra = covered - all_fields + assert extra == set(), ( + f"Coverage registry references non-existent fields: {sorted(extra)}. Remove stale entries." + ) + + +def test_fmt_open_kitchen_field_coverage(): + """Every OpenKitchenResult field must be in RENDERED or SUPPRESSED.""" + import typing + + from autoskillit.hooks.formatters.pretty_output_hook import ( + _FMT_OPEN_KITCHEN_RENDERED, + _FMT_OPEN_KITCHEN_SUPPRESSED, + ) + from autoskillit.recipe._recipe_ingredients import OpenKitchenResult + + all_fields = set(typing.get_type_hints(OpenKitchenResult)) + covered = _FMT_OPEN_KITCHEN_RENDERED | _FMT_OPEN_KITCHEN_SUPPRESSED + uncovered = all_fields - covered + assert uncovered == set(), ( + f"OpenKitchenResult fields have no coverage decision: {sorted(uncovered)}. " + "Add each to _FMT_OPEN_KITCHEN_RENDERED or _FMT_OPEN_KITCHEN_SUPPRESSED." + ) + extra = covered - all_fields + assert extra == set(), ( + f"Coverage registry references non-existent fields: {sorted(extra)}. Remove stale entries." + ) + + +def test_fmt_list_recipes_renders_summary(): + """Recipe summary appears on the line below each recipe name.""" + formatted = _format_response( + "mcp__autoskillit__list_recipes", + json.dumps( + { + "recipes": [ + {"name": "test-recipe", "description": "A test", "summary": "step1 -> done"}, + ], + "count": 1, + } + ), + pipeline=False, + ) + assert formatted is not None + assert "test-recipe" in formatted + assert "step1 -> done" in formatted + + +# Raw/derived field deduplication: ingredients_table vs content + + +def test_fmt_recipe_body_ingredients_not_duplicated_when_table_present(): + """When ingredients_table is present, ingredient names must not appear in RECIPE section.""" + from autoskillit.hooks.formatters.pretty_output_hook import _fmt_recipe_body + + data = { + "content": REALISTIC_RECIPE_YAML, + "ingredients_table": ( + "| Name | Description | Default |\n" + "| task | What to implement | (required) |\n" + "| review_approach | Run review-approach before planning | false |\n" + ), + "valid": True, + "suggestions": [], + } + result = "\n".join(_fmt_recipe_body(data)) + assert "--- INGREDIENTS TABLE" in result, ( + "_fmt_recipe_body did not emit the INGREDIENTS TABLE header." + ) + recipe_section = result.split("--- INGREDIENTS TABLE")[0] + assert "review_approach:" not in recipe_section + assert " task:" not in recipe_section + assert "implement" in recipe_section + assert "kitchen_rules" in recipe_section + table_section = result.split("--- INGREDIENTS TABLE")[1] + assert "review_approach" in table_section + assert "task" in table_section + + +def test_strip_yaml_ingredients_block_removes_ingredients_section(): + from autoskillit.hooks.formatters.pretty_output_hook import _strip_yaml_ingredients_block + + yaml = "name: test\ningredients:\n task:\n description: a task\nsteps:\n do: {}\n" + result = _strip_yaml_ingredients_block(yaml) + assert "ingredients:" not in result + assert " task:" not in result + assert "steps:" in result + assert "name: test" in result + + +def test_strip_yaml_ingredients_block_noop_when_no_ingredients_key(): + from autoskillit.hooks.formatters.pretty_output_hook import _strip_yaml_ingredients_block + + yaml = "name: test\nsteps:\n do: {}\n" + result = _strip_yaml_ingredients_block(yaml) + assert result == yaml + + +def test_strip_yaml_ingredients_block_at_end_of_file(): + from autoskillit.hooks.formatters.pretty_output_hook import _strip_yaml_ingredients_block + + yaml = "name: test\nsteps:\n do: {}\ningredients:\n foo:\n description: bar\n" + result = _strip_yaml_ingredients_block(yaml) + assert "ingredients:" not in result + assert "steps:" in result + + +def test_strip_yaml_ingredients_block_multiline_description(): + from autoskillit.hooks.formatters.pretty_output_hook import _strip_yaml_ingredients_block + + yaml = ( + "name: test\n" + "ingredients:\n" + " task:\n" + " description: >\n" + " Long description\n" + " spanning multiple lines\n" + " required: true\n" + "steps:\n" + " do: {}\n" + ) + result = _strip_yaml_ingredients_block(yaml) + assert "ingredients:" not in result + assert "steps:" in result + + +def test_fmt_open_kitchen_ingredients_not_duplicated_when_table_present(): + """open_kitchen routes through _fmt_recipe_body — verify same deduplication applies.""" + from autoskillit.hooks.formatters.pretty_output_hook import _fmt_open_kitchen + + data = { + "content": REALISTIC_RECIPE_YAML, + "ingredients_table": "| task | What to implement | (required) |", + "valid": True, + "suggestions": [], + "kitchen": "open", + "version": "0.6.0", + } + result = _fmt_open_kitchen(data, pipeline=False) + assert "--- INGREDIENTS TABLE" in result, ( + "_fmt_open_kitchen did not emit the INGREDIENTS TABLE header." + ) + recipe_section = result.split("--- INGREDIENTS TABLE")[0] + assert " task:" not in recipe_section + assert "review_approach:" not in recipe_section + + +def test_fmt_open_kitchen_ingredients_only_no_recipe_block(): + """Formatter must not render RECIPE block when content is absent (ingredients_only).""" + from autoskillit.hooks.formatters.pretty_output_hook import _fmt_open_kitchen + + data = { + "success": True, + "kitchen": "open", + "version": "0.9.372", + "valid": True, + "ingredients_table": "| Name | Description | Default |", + "suggestions": [], + } + output = _fmt_open_kitchen(data, pipeline=False) + assert "--- RECIPE ---" not in output + assert "INGREDIENTS TABLE" in output diff --git a/tests/infra/test_pyproject_bounds.py b/tests/infra/test_pyproject_bounds.py index 94ae2d48b6..459ceb3ef2 100644 --- a/tests/infra/test_pyproject_bounds.py +++ b/tests/infra/test_pyproject_bounds.py @@ -39,3 +39,18 @@ def test_pytest_timeout_no_upper_bound(dev_deps: list[str]): def test_ruff_lower_bound(dev_deps: list[str]): dep = next(d for d in dev_deps if d.startswith("ruff")) assert ">=0.15.0" in dep, f"Expected ruff>=0.15.0, got: {dep!r}" + + +def test_pytest_cov_in_dev_deps(dev_deps: list[str]): + """pytest-cov is present in dev dependencies.""" + assert any(d.startswith("pytest-cov") for d in dev_deps), ( + "pytest-cov must be in dev dependencies" + ) + + +def test_httpx_upper_bound_not_tight() -> None: + data = tomllib.loads(_PYPROJECT.read_text()) + deps = data["project"]["dependencies"] + httpx_dep = next(d for d in deps if d.startswith("httpx")) + assert "<0.29" not in httpx_dep, f"httpx upper bound is too tight: {httpx_dep!r}" + assert "<1.0" in httpx_dep, f"httpx should have <1.0 upper bound: {httpx_dep!r}" diff --git a/tests/infra/test_pyproject_metadata.py b/tests/infra/test_pyproject_metadata.py index ac55a09d86..27b20ea7e9 100644 --- a/tests/infra/test_pyproject_metadata.py +++ b/tests/infra/test_pyproject_metadata.py @@ -44,3 +44,13 @@ def test_markdown_it_py_in_dependencies() -> None: assert any("markdown-it-py" in d for d in deps), ( "markdown-it-py not found in [project].dependencies in pyproject.toml" ) + + +def test_api_simulator_dev_dep_is_linux_only() -> None: + """api-simulator must carry sys_platform == 'linux' so it is not installed on macOS/Windows.""" + deps: list[str] = _project()["project"]["optional-dependencies"]["dev"] + api_sim = next((d for d in deps if d.startswith("api-simulator")), None) + assert api_sim is not None, "api-simulator not found in dev dependencies" + assert "sys_platform" in api_sim and "linux" in api_sim, ( + f"api-simulator dev dep missing sys_platform == 'linux' marker: {api_sim!r}" + ) diff --git a/tests/infra/test_release_workflows.py b/tests/infra/test_release_workflows.py index 1f6f473bed..62ec256bbe 100644 --- a/tests/infra/test_release_workflows.py +++ b/tests/infra/test_release_workflows.py @@ -1,6 +1,6 @@ """Structural contract tests for the release CI workflows. -Ensures version-bump.yml, patch-bump-integration.yml, and release.yml are +Ensures version-bump.yml, patch-bump-develop.yml, and release.yml are correctly shaped, guarded, and consistent with repo conventions. """ @@ -12,9 +12,7 @@ REPO_ROOT = Path(__file__).parent.parent.parent VERSION_BUMP_WORKFLOW = REPO_ROOT / ".github" / "workflows" / "version-bump.yml" -PATCH_BUMP_INTEGRATION_WORKFLOW = ( - REPO_ROOT / ".github" / "workflows" / "patch-bump-integration.yml" -) +PATCH_BUMP_DEVELOP_WORKFLOW = REPO_ROOT / ".github" / "workflows" / "patch-bump-develop.yml" RELEASE_WORKFLOW = REPO_ROOT / ".github" / "workflows" / "release.yml" # ── helpers ─────────────────────────────────────────────────────────────────── @@ -25,13 +23,13 @@ def _load(path: Path) -> dict: def _uv_version_pins(workflow: dict) -> list[str]: - """Return all uv-version values declared in setup-uv steps.""" + """Return all version values declared in setup-uv steps.""" pins = [] for job in workflow.get("jobs", {}).values(): for step in job.get("steps", []): uses = step.get("uses", "") if "setup-uv" in uses: - pins.append(step.get("with", {}).get("uv-version", "")) + pins.append(step.get("with", {}).get("version", "")) return pins @@ -43,13 +41,13 @@ def _find_step(job: dict, name_fragment: str) -> dict | None: ) -def _find_integration_commit_step(job: dict) -> dict | None: - """Return the integration version commit/push step.""" +def _find_develop_commit_step(job: dict) -> dict | None: + """Return the develop version commit/push step.""" return next( ( s for s in job.get("steps", []) - if "integration version" in s.get("name", "").lower() + if "develop version" in s.get("name", "").lower() and ("commit" in s.get("name", "").lower() or "push" in s.get("name", "").lower()) ), None, @@ -80,15 +78,15 @@ def test_not_triggered_on_push(self): "version-bump.yml must not have a push trigger — it is PR-event-based" ) - def test_job_has_integration_branch_guard(self): - """Job must only run when head.ref == 'integration'.""" + def test_job_has_develop_branch_guard(self): + """Job must only run when head.ref == 'develop'.""" wf = _load(VERSION_BUMP_WORKFLOW) jobs = wf.get("jobs", {}) assert len(jobs) >= 1 job = next(iter(jobs.values())) condition = job.get("if", "") assert "merged" in condition, "Job must check github.event.pull_request.merged" - assert "integration" in condition, "Job must guard on head.ref == 'integration'" + assert "develop" in condition, "Job must guard on head.ref == 'develop'" def test_contents_write_permission(self): wf = _load(VERSION_BUMP_WORKFLOW) @@ -136,6 +134,16 @@ def test_plugin_json_is_updated(self): text = VERSION_BUMP_WORKFLOW.read_text() assert "plugin.json" in text + def test_version_bump_workflow_uses_sync_versions_script(self): + """The version-bump workflow must call the unified sync script.""" + wf = _load(VERSION_BUMP_WORKFLOW) + job = next(iter(wf.get("jobs", {}).values())) + sync_step = _find_step(job, "sync version") + assert sync_step is not None, "version-bump.yml must have a 'Sync version artifacts' step" + assert "sync_versions" in sync_step.get("run", ""), ( + "Sync step must call scripts/sync_versions.py" + ) + def test_uv_lock_is_regenerated(self): """Workflow must run uv lock.""" text = VERSION_BUMP_WORKFLOW.read_text() @@ -158,32 +166,30 @@ def test_checkout_uses_main_ref(self): ref = checkout_step.get("with", {}).get("ref", "") assert "main" in ref, "Checkout must use ref: main (not the default detached PR merge ref)" - def test_integration_checkout_step_exists(self): + def test_develop_checkout_step_exists(self): wf = _load(VERSION_BUMP_WORKFLOW) job = next(iter(wf.get("jobs", {}).values())) - assert _find_step(job, "Checkout integration branch") is not None + assert _find_step(job, "Checkout develop branch") is not None - def test_integration_read_version_step_exists(self): + def test_develop_read_version_step_exists(self): wf = _load(VERSION_BUMP_WORKFLOW) job = next(iter(wf.get("jobs", {}).values())) - assert _find_step(job, "Read integration current version") is not None + assert _find_step(job, "Read develop current version") is not None - def test_integration_commit_push_step_exists(self): + def test_develop_commit_push_step_exists(self): wf = _load(VERSION_BUMP_WORKFLOW) job = next(iter(wf.get("jobs", {}).values())) - assert _find_integration_commit_step(job) is not None + assert _find_develop_commit_step(job) is not None - def test_integration_push_is_not_force_push(self): + def test_develop_push_is_not_force_push(self): wf = _load(VERSION_BUMP_WORKFLOW) job = next(iter(wf.get("jobs", {}).values())) - int_commit_step = _find_integration_commit_step(job) - assert int_commit_step is not None, ( - "Workflow must have an integration version commit/push step" - ) + int_commit_step = _find_develop_commit_step(job) + assert int_commit_step is not None, "Workflow must have a develop version commit/push step" run_script = int_commit_step.get("run", "") assert "push --force" not in run_script assert "push -f " not in run_script - assert "integration" in run_script + assert "develop" in run_script def test_minor_version_bump_on_main(self): """version-bump.yml must increment MINOR and reset PATCH to 0 for main.""" @@ -195,14 +201,14 @@ def test_minor_version_bump_on_main(self): assert "$((MINOR + 1))" in run_block, "Must increment MINOR for main" assert ".$((MINOR + 1)).0" in run_block, "main version must end in .0" - def test_integration_reset_to_patch_one(self): - """version-bump.yml must set integration to X.(Y+1).1 after promotion.""" + def test_develop_reset_to_patch_one(self): + """version-bump.yml must set develop to X.(Y+1).1 after promotion.""" wf = _load(VERSION_BUMP_WORKFLOW) job = next(iter(wf.get("jobs", {}).values())) step = _find_step(job, "Compute new minor version") assert step is not None, "Workflow must have a 'Compute new minor version' step" run_block = step.get("run", "") - assert ".$((MINOR + 1)).1" in run_block, "integration version must end in .1" + assert ".$((MINOR + 1)).1" in run_block, "develop version must end in .1" def test_no_force_push(self): """version-bump.yml must not contain any force-push.""" @@ -214,89 +220,93 @@ def test_no_force_push(self): def test_no_branch_protection_api_calls(self): """version-bump.yml must not call the GitHub branch protection API.""" text = VERSION_BUMP_WORKFLOW.read_text() - assert "branches/integration/protection" not in text, ( + assert "branches/develop/protection" not in text, ( "version-bump.yml must not manipulate branch protection — " "no force-push means no protection changes are needed" ) -# ── patch-bump-integration.yml ──────────────────────────────────────────────── +# ── patch-bump-develop.yml ──────────────────────────────────────────────────── -class TestPatchBumpIntegrationWorkflow: +class TestPatchBumpDevelopWorkflow: def test_workflow_file_exists(self): - assert PATCH_BUMP_INTEGRATION_WORKFLOW.exists(), ( - f"patch-bump-integration workflow not found at {PATCH_BUMP_INTEGRATION_WORKFLOW}" + assert PATCH_BUMP_DEVELOP_WORKFLOW.exists(), ( + f"patch-bump-develop workflow not found at {PATCH_BUMP_DEVELOP_WORKFLOW}" ) - def test_triggered_on_pull_request_closed_to_integration(self): - wf = _load(PATCH_BUMP_INTEGRATION_WORKFLOW) + def test_triggered_on_pull_request_closed_to_develop(self): + wf = _load(PATCH_BUMP_DEVELOP_WORKFLOW) pr_trigger = wf.get(True, {}).get("pull_request", {}) assert "closed" in pr_trigger.get("types", []) - assert "integration" in pr_trigger.get("branches", []) + assert "develop" in pr_trigger.get("branches", []) def test_not_triggered_on_push(self): - wf = _load(PATCH_BUMP_INTEGRATION_WORKFLOW) + wf = _load(PATCH_BUMP_DEVELOP_WORKFLOW) assert "push" not in wf.get(True, {}), ( - "patch-bump-integration.yml must not have a push trigger" + "patch-bump-develop.yml must not have a push trigger" ) def test_job_has_merged_guard(self): - wf = _load(PATCH_BUMP_INTEGRATION_WORKFLOW) + wf = _load(PATCH_BUMP_DEVELOP_WORKFLOW) job = next(iter(wf.get("jobs", {}).values())) condition = job.get("if", "") assert "merged" in condition, "Job must check github.event.pull_request.merged" def test_contents_write_permission(self): - wf = _load(PATCH_BUMP_INTEGRATION_WORKFLOW) + wf = _load(PATCH_BUMP_DEVELOP_WORKFLOW) job = next(iter(wf.get("jobs", {}).values())) perms = job.get("permissions", {}) assert perms.get("contents") == "write" def test_setup_uv_has_version_pin(self): - wf = _load(PATCH_BUMP_INTEGRATION_WORKFLOW) + wf = _load(PATCH_BUMP_DEVELOP_WORKFLOW) pins = _uv_version_pins(wf) - assert pins, "patch-bump-integration.yml must use astral-sh/setup-uv with a version pin" + assert pins, "patch-bump-develop.yml must use astral-sh/setup-uv with a version pin" assert all(p for p in pins) def test_uv_version_consistent_with_tests_yml(self): tests_wf = yaml.safe_load((REPO_ROOT / ".github" / "workflows" / "tests.yml").read_text()) - bump_wf = _load(PATCH_BUMP_INTEGRATION_WORKFLOW) + bump_wf = _load(PATCH_BUMP_DEVELOP_WORKFLOW) tests_pins = _uv_version_pins(tests_wf) bump_pins = _uv_version_pins(bump_wf) assert tests_pins and bump_pins assert bump_pins[0] == tests_pins[0], ( - f"uv-version in patch-bump-integration.yml ({bump_pins[0]}) must match " + f"uv-version in patch-bump-develop.yml ({bump_pins[0]}) must match " f"tests.yml ({tests_pins[0]})" ) def test_pyproject_toml_is_updated(self): - wf = _load(PATCH_BUMP_INTEGRATION_WORKFLOW) + wf = _load(PATCH_BUMP_DEVELOP_WORKFLOW) job = next(iter(wf.get("jobs", {}).values())) step = _find_step(job, "Update pyproject.toml") assert step is not None, "Workflow must have an 'Update pyproject.toml' step" assert "pyproject.toml" in step.get("run", ""), "Update step must reference pyproject.toml" - def test_plugin_json_is_updated(self): - wf = _load(PATCH_BUMP_INTEGRATION_WORKFLOW) + def test_patch_bump_workflow_uses_sync_versions_script(self): + wf = _load(PATCH_BUMP_DEVELOP_WORKFLOW) job = next(iter(wf.get("jobs", {}).values())) - step = _find_step(job, "Update plugin.json") - assert step is not None, "Workflow must have an 'Update plugin.json' step" - assert "plugin.json" in step.get("run", ""), "Update step must reference plugin.json" + sync_step = _find_step(job, "sync version") + assert sync_step is not None, ( + "patch-bump-develop.yml must have a 'Sync version artifacts' step" + ) + assert "sync_versions" in sync_step.get("run", ""), ( + "Sync step must call scripts/sync_versions.py" + ) def test_uv_lock_is_regenerated(self): - wf = _load(PATCH_BUMP_INTEGRATION_WORKFLOW) + wf = _load(PATCH_BUMP_DEVELOP_WORKFLOW) job = next(iter(wf.get("jobs", {}).values())) step = _find_step(job, "Regenerate uv.lock") assert step is not None, "Workflow must have a 'Regenerate uv.lock' step" assert "uv lock" in step.get("run", ""), "Regenerate step must run 'uv lock'" def test_uses_github_actions_bot_identity(self): - text = PATCH_BUMP_INTEGRATION_WORKFLOW.read_text() + text = PATCH_BUMP_DEVELOP_WORKFLOW.read_text() assert "github-actions[bot]" in text - def test_checkout_uses_integration_ref(self): - wf = _load(PATCH_BUMP_INTEGRATION_WORKFLOW) + def test_checkout_uses_develop_ref(self): + wf = _load(PATCH_BUMP_DEVELOP_WORKFLOW) job = next(iter(wf.get("jobs", {}).values())) checkout_step = next( (s for s in job.get("steps", []) if "actions/checkout" in s.get("uses", "")), @@ -304,11 +314,11 @@ def test_checkout_uses_integration_ref(self): ) assert checkout_step is not None ref = checkout_step.get("with", {}).get("ref", "") - assert "integration" in ref + assert "develop" in ref def test_patch_increment_logic(self): """Patch bump uses $((PATCH + 1)) arithmetic.""" - wf = _load(PATCH_BUMP_INTEGRATION_WORKFLOW) + wf = _load(PATCH_BUMP_DEVELOP_WORKFLOW) job = next(iter(wf.get("jobs", {}).values())) step = _find_step(job, "Compute new patch version") assert step is not None, "Workflow must have a 'Compute new patch version' step" @@ -317,7 +327,7 @@ def test_patch_increment_logic(self): def test_patch_increment_does_not_overflow_minor(self): """Patch bump must not touch MINOR.""" - wf = _load(PATCH_BUMP_INTEGRATION_WORKFLOW) + wf = _load(PATCH_BUMP_DEVELOP_WORKFLOW) job = next(iter(wf.get("jobs", {}).values())) step = _find_step(job, "Compute new patch version") assert step is not None @@ -325,32 +335,30 @@ def test_patch_increment_does_not_overflow_minor(self): assert "$((MINOR + 1))" not in run_block def test_push_is_not_force_push(self): - text = PATCH_BUMP_INTEGRATION_WORKFLOW.read_text() - assert "--force" not in text, ( - "patch-bump-integration.yml must not force-push to integration" - ) + text = PATCH_BUMP_DEVELOP_WORKFLOW.read_text() + assert "--force" not in text, "patch-bump-develop.yml must not force-push to develop" def test_has_concurrency_group(self): """Workflow must declare a concurrency group to serialize merge-queue batches.""" - wf = _load(PATCH_BUMP_INTEGRATION_WORKFLOW) + wf = _load(PATCH_BUMP_DEVELOP_WORKFLOW) assert wf.get("concurrency") is not None, ( - "patch-bump-integration.yml must declare a concurrency group — " + "patch-bump-develop.yml must declare a concurrency group — " "without it, batched merge queue PRs race and silently skip version bumps" ) def test_concurrency_group_name(self): """Concurrency group name must be a fixed string (not PR-number-scoped).""" - wf = _load(PATCH_BUMP_INTEGRATION_WORKFLOW) + wf = _load(PATCH_BUMP_DEVELOP_WORKFLOW) concurrency = wf.get("concurrency", {}) group = concurrency.get("group", "") - assert group == "patch-bump-integration", ( - f"concurrency.group must be 'patch-bump-integration' (got {group!r}) — " + assert group == "patch-bump-develop", ( + f"concurrency.group must be 'patch-bump-develop' (got {group!r}) — " "a fixed group name ensures all simultaneous batch runs queue behind one another" ) def test_concurrency_cancel_in_progress_is_false(self): """cancel-in-progress must be false so every queued run executes its bump.""" - wf = _load(PATCH_BUMP_INTEGRATION_WORKFLOW) + wf = _load(PATCH_BUMP_DEVELOP_WORKFLOW) concurrency = wf.get("concurrency", {}) cancel = concurrency.get("cancel-in-progress", None) assert cancel is False, ( diff --git a/tests/infra/test_remove_clone_guard.py b/tests/infra/test_remove_clone_guard.py index fcca78250a..a5971b238d 100644 --- a/tests/infra/test_remove_clone_guard.py +++ b/tests/infra/test_remove_clone_guard.py @@ -16,7 +16,7 @@ def _make_proc(returncode: int, stdout: str) -> subprocess.CompletedProcess: # def _run_hook_mocked(event: dict, side_effects: list) -> str: """Run main() with the given event and subprocess.run side effects.""" - from autoskillit.hooks.remove_clone_guard import main + from autoskillit.hooks.guards.remove_clone_guard import main with ( patch("subprocess.run", side_effect=side_effects), @@ -39,7 +39,7 @@ def _run_hook_with_git(event: dict, git_responses: list[tuple[int, str]]) -> str def _run_hook(event: dict) -> str: """Run main() with no subprocess calls (keep=true or no clone_path path).""" - from autoskillit.hooks.remove_clone_guard import main + from autoskillit.hooks.guards.remove_clone_guard import main with patch("sys.stdin", io.StringIO(json.dumps(event))): buf = io.StringIO() @@ -62,7 +62,7 @@ def test_approve_silently_when_keep_true(): def test_approve_on_malformed_json(): """Any parse error silently approves.""" - from autoskillit.hooks.remove_clone_guard import main + from autoskillit.hooks.guards.remove_clone_guard import main with patch("sys.stdin", io.StringIO("not-json")): buf = io.StringIO() diff --git a/tests/infra/test_resume_ownership_guard.py b/tests/infra/test_resume_ownership_guard.py new file mode 100644 index 0000000000..4afb86fc45 --- /dev/null +++ b/tests/infra/test_resume_ownership_guard.py @@ -0,0 +1,201 @@ +"""Tests for resume_ownership_guard.py PreToolUse hook.""" + +from __future__ import annotations + +import io +import json +import os +from contextlib import redirect_stdout +from pathlib import Path +from unittest.mock import patch + +import pytest + +pytestmark = [pytest.mark.layer("infra"), pytest.mark.small] + + +def _run_guard( + tool_input: dict | None = None, + *, + headless: bool = False, + raw_stdin: str | None = None, + session_id: str = "", + campaign_id: str = "", + state_dir: str = "", +) -> str: + from autoskillit.hooks.guards.resume_ownership_guard import main + + payload: dict[str, object] = {"tool_input": tool_input or {}} + if session_id: + payload["session_id"] = session_id + stdin_content = raw_stdin if raw_stdin is not None else json.dumps(payload) + + env_updates: dict[str, str] = {} + if headless: + env_updates["AUTOSKILLIT_HEADLESS"] = "1" + if campaign_id: + env_updates["AUTOSKILLIT_CAMPAIGN_ID"] = campaign_id + if state_dir: + env_updates["AUTOSKILLIT_STATE_DIR"] = state_dir + + env_removals = [] + if not headless: + env_removals.append("AUTOSKILLIT_HEADLESS") + + with ( + patch.dict(os.environ, env_updates, clear=False), + patch("sys.stdin", io.StringIO(stdin_content)), + ): + for key in env_removals: + os.environ.pop(key, None) + buf = io.StringIO() + with redirect_stdout(buf): + try: + main() + except SystemExit as exc: + if exc.code not in (None, 0): + raise + return buf.getvalue() + + +def _write_provenance(path, records: list[dict]) -> None: + lines = [json.dumps(r) for r in records] + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text("\n".join(lines) + "\n") + + +class TestResumeOwnershipGuard: + def test_non_headless_passes_through(self) -> None: + out = _run_guard( + {"resume_session_id": "sess-1"}, + headless=False, + ) + assert not out.strip() + + def test_no_resume_session_id_passes_through(self) -> None: + out = _run_guard( + {"some_other_field": "value"}, + headless=True, + ) + assert not out.strip() + + def test_empty_resume_session_id_passes_through(self) -> None: + out = _run_guard( + {"resume_session_id": ""}, + headless=True, + ) + assert not out.strip() + + def test_no_provenance_record_passes_through(self, tmp_path: Path) -> None: + out = _run_guard( + {"resume_session_id": "unknown-sess"}, + headless=True, + state_dir=str(tmp_path), + ) + assert not out.strip() + + def test_non_food_truck_session_denied(self, tmp_path: Path) -> None: + prov_path = tmp_path / "session_provenance.jsonl" + _write_provenance(prov_path, [{"session_id": "sess-1", "recipe_name": ""}]) + out = _run_guard( + {"resume_session_id": "sess-1"}, + headless=True, + state_dir=str(tmp_path), + ) + response = json.loads(out) + assert response["hookSpecificOutput"]["permissionDecision"] == "deny" + assert "not a food truck" in response["hookSpecificOutput"]["permissionDecisionReason"] + + def test_matching_caller_session_id_passes(self, tmp_path: Path) -> None: + prov_path = tmp_path / "session_provenance.jsonl" + _write_provenance( + prov_path, + [ + { + "session_id": "sess-target", + "recipe_name": "test-recipe", + "caller_session_id": "my-caller", + "kitchen_id": "kitchen-1", + } + ], + ) + out = _run_guard( + {"resume_session_id": "sess-target"}, + headless=True, + state_dir=str(tmp_path), + session_id="my-caller", + ) + assert not out.strip() + + def test_matching_kitchen_id_passes(self, tmp_path: Path) -> None: + prov_path = tmp_path / "session_provenance.jsonl" + _write_provenance( + prov_path, + [ + { + "session_id": "sess-target", + "recipe_name": "test-recipe", + "caller_session_id": "other-caller", + "kitchen_id": "my-kitchen", + } + ], + ) + out = _run_guard( + {"resume_session_id": "sess-target"}, + headless=True, + state_dir=str(tmp_path), + session_id="different-caller", + campaign_id="my-kitchen", + ) + assert not out.strip() + + def test_non_matching_ownership_denied(self, tmp_path: Path) -> None: + prov_path = tmp_path / "session_provenance.jsonl" + _write_provenance( + prov_path, + [ + { + "session_id": "sess-target", + "recipe_name": "test-recipe", + "caller_session_id": "owner-caller", + "kitchen_id": "owner-kitchen", + } + ], + ) + out = _run_guard( + {"resume_session_id": "sess-target"}, + headless=True, + state_dir=str(tmp_path), + session_id="wrong-caller", + campaign_id="wrong-kitchen", + ) + response = json.loads(out) + assert response["hookSpecificOutput"]["permissionDecision"] == "deny" + assert "not owned" in response["hookSpecificOutput"]["permissionDecisionReason"] + + def test_malformed_input_fails_open(self) -> None: + out = _run_guard(raw_stdin="not-json", headless=True) + assert not out.strip() + + def test_malformed_provenance_lines_skipped(self, tmp_path: Path) -> None: + prov_path = tmp_path / "session_provenance.jsonl" + prov_path.parent.mkdir(parents=True, exist_ok=True) + prov_path.write_text( + "not-json\n" + + json.dumps( + { + "session_id": "sess-1", + "recipe_name": "r", + "caller_session_id": "my-caller", + "kitchen_id": "k", + } + ) + + "\n" + ) + out = _run_guard( + {"resume_session_id": "sess-1"}, + headless=True, + state_dir=str(tmp_path), + session_id="my-caller", + ) + assert not out.strip() diff --git a/tests/infra/test_schema_version_convention.py b/tests/infra/test_schema_version_convention.py index 0053786840..bf272435b3 100644 --- a/tests/infra/test_schema_version_convention.py +++ b/tests/infra/test_schema_version_convention.py @@ -105,53 +105,55 @@ def _is_yaml_dump(node: ast.expr) -> bool: # Hard-curated allowlist of existing atomic_write + json.dumps sites detected by AST scan. -# Quota cache is NOT here because it's migrated to write_versioned_json in Phase 4. # Any new site that writes a dict payload SHOULD use write_versioned_json. _LEGACY_JSON_WRITES: set[tuple[str, int]] = { - # core/io.py — write_versioned_json itself (the blessed helper) uses atomic_write+json.dumps - ("src/autoskillit/core/io.py", 118), - # session_log.py — summary dict, token_usage list, audit_log list - ("src/autoskillit/execution/session_log.py", 244), - ("src/autoskillit/execution/session_log.py", 260), - ("src/autoskillit/execution/session_log.py", 263), # migration/store.py — failure store dicts ("src/autoskillit/migration/store.py", 54), ("src/autoskillit/migration/store.py", 64), # clone_registry.py — clones dict (CloneRegistry.__exit__ atomic write-back) - ("src/autoskillit/workspace/clone_registry.py", 82), + ("src/autoskillit/workspace/clone_registry.py", 89), # staleness_cache.py — cache dict ("src/autoskillit/recipe/staleness_cache.py", 67), - # background.py — payload dict - ("src/autoskillit/pipeline/background.py", 132), # _lifespan.py — hooks.json self-heal on startup drift (co-owned with Claude plugin system) - ("src/autoskillit/server/_lifespan.py", 55), + ("src/autoskillit/server/_lifespan.py", 62), # tools_kitchen.py — hook config dict - ("src/autoskillit/server/tools_kitchen.py", 142), - ("src/autoskillit/server/tools_kitchen.py", 471), + ("src/autoskillit/server/tools/tools_kitchen.py", 112), + ("src/autoskillit/server/tools/tools_kitchen.py", 552), # tools_status.py — mcp_data dict - ("src/autoskillit/server/tools_status.py", 385), + ("src/autoskillit/server/tools/tools_status.py", 495), # tools_github.py — bug report dict - ("src/autoskillit/server/tools_github.py", 268), + ("src/autoskillit/server/tools/tools_github.py", 304), # _hooks.py — settings.json dict (co-owned with Claude CLI) - ("src/autoskillit/cli/_hooks.py", 23), + ("src/autoskillit/cli/_hooks.py", 24), # _init_helpers.py — ~/.claude.json (co-owned) ("src/autoskillit/cli/_init_helpers.py", 376), # _init_helpers.py — evict_direct_mcp_entry write-back to ~/.claude.json (co-owned) ("src/autoskillit/cli/_init_helpers.py", 395), # _installed_plugins.py — installed_plugins.json (co-owned with Claude plugin system) - ("src/autoskillit/cli/_installed_plugins.py", 65), + ("src/autoskillit/cli/_installed_plugins.py", 71), # _marketplace.py — marketplace.json (co-owned) - ("src/autoskillit/cli/_marketplace.py", 85), + ("src/autoskillit/cli/_marketplace.py", 100), # _marketplace.py — hooks.json (co-owned) - ("src/autoskillit/cli/_marketplace.py", 161), + ("src/autoskillit/cli/_marketplace.py", 182), # _update_checks.py — dismissal state file - ("src/autoskillit/cli/_update_checks.py", 102), - # _update_checks.py — fetch cache - ("src/autoskillit/cli/_update_checks.py", 129), - # smoke_utils.py — domain partitions dict, hunk ranges list, merge queue list - ("src/autoskillit/smoke_utils.py", 53), - ("src/autoskillit/smoke_utils.py", 83), - ("src/autoskillit/smoke_utils.py", 138), + ("src/autoskillit/cli/update/_update_checks.py", 77), + # _update_checks_fetch.py — fetch cache (extracted from _update_checks.py) + ("src/autoskillit/cli/update/_update_checks_fetch.py", 58), + # smoke_utils.py — partitions, ranges, diff metrics, queue, enriched handoff + ("src/autoskillit/smoke_utils.py", 57), + ("src/autoskillit/smoke_utils.py", 126), + # Lines 143 and 378 are list-payload write sites (dual membership: also in list_sites + # in test_allowlist_includes_list_payloads_as_documented). The AST scanner catches + # them because it cannot distinguish list vs dict return types — intentional. + ("src/autoskillit/smoke_utils.py", 143), + ("src/autoskillit/smoke_utils.py", 378), + ("src/autoskillit/smoke_utils.py", 426), + # planner/consolidation.py — write-back of merged WP dicts to per-file results + ("src/autoskillit/planner/consolidation.py", 308), + # planner/manifests.py — finalize_wp_manifest: wp_index.json rebuild (list payload) + ("src/autoskillit/planner/manifests.py", 255), + # _cmd_rpc.py — emit_fallback_map: BEM fallback execution map (recipe-internal) + ("src/autoskillit/recipe/_cmd_rpc.py", 445), } @@ -212,10 +214,8 @@ def test_allowlist_includes_list_payloads_as_documented(self): """List-payload sites are included since the AST scanner can't distinguish return types.""" # These sites write list payloads through function calls but are caught by the scanner list_sites = [ - ("src/autoskillit/execution/session_log.py", 260), - ("src/autoskillit/execution/session_log.py", 263), - ("src/autoskillit/smoke_utils.py", 83), - ("src/autoskillit/smoke_utils.py", 138), + ("src/autoskillit/smoke_utils.py", 143), + ("src/autoskillit/smoke_utils.py", 378), ] for site in list_sites: assert site in _LEGACY_JSON_WRITES, ( diff --git a/tests/infra/test_session_scope_enforcement.py b/tests/infra/test_session_scope_enforcement.py index fe84c1a0ec..73b85ab85d 100644 --- a/tests/infra/test_session_scope_enforcement.py +++ b/tests/infra/test_session_scope_enforcement.py @@ -73,11 +73,14 @@ def _scoped_guard_names() -> list[str]: def _find_test_file(guard_script: str) -> Path | None: - """Locate the test file for a guard script in tests/infra/.""" + """Locate the test file for a guard script in tests/infra/ or tests/hooks/.""" stem = Path(guard_script).stem # e.g. "ask_user_question_guard" tests_infra = Path(__file__).resolve().parent - candidate = tests_infra / f"test_{stem}.py" - return candidate if candidate.exists() else None + for directory in (tests_infra, tests_infra.parent / "hooks"): + candidate = directory / f"test_{stem}.py" + if candidate.exists(): + return candidate + return None @pytest.mark.parametrize("guard_script", _scoped_guard_names()) @@ -93,7 +96,11 @@ def test_scoped_guard_has_both_session_type_test_cases(guard_script: str) -> Non line for line in source.splitlines() if not line.lstrip().startswith("#") ) has_headless_true = "headless=True" in code_lines or "AUTOSKILLIT_HEADLESS" in code_lines - has_headless_false = "headless=False" in code_lines + # Accept explicit headless=False OR the env-strip pattern used by subprocess-style hook tests + # (k != "AUTOSKILLIT_HEADLESS" strips the var to exercise the non-headless code path) + has_headless_false = ( + "headless=False" in code_lines or '!= "AUTOSKILLIT_HEADLESS"' in code_lines + ) assert has_headless_true, ( f"{test_file.name} must test the headless=True path for scoped guard '{guard_script}'." ) diff --git a/tests/infra/test_session_start_reminder.py b/tests/infra/test_session_start_reminder.py deleted file mode 100644 index 1d19a160e3..0000000000 --- a/tests/infra/test_session_start_reminder.py +++ /dev/null @@ -1,111 +0,0 @@ -"""Tests for the SessionStart hook — session_start_hook.py.""" - -from __future__ import annotations - -import json -import subprocess -import sys -from datetime import UTC -from pathlib import Path - -SCRIPT = Path(__file__).resolve().parents[2] / "src/autoskillit/hooks/session_start_hook.py" - - -def _run(stdin_data: str) -> tuple[int, str]: - result = subprocess.run( - [sys.executable, str(SCRIPT)], - input=stdin_data, - capture_output=True, - text=True, - ) - return result.returncode, result.stdout - - -# REQ-HOOK-002, REQ-HOOK-003 -def test_fresh_session_no_output(tmp_path: Path) -> None: - """Empty transcript_path → no additionalContext injected.""" - transcript = tmp_path / "session.jsonl" - transcript.write_text("") - payload = json.dumps({"session_id": "abc", "transcript_path": str(transcript)}) - rc, out = _run(payload) - assert rc == 0 - assert "additionalContext" not in out - - -def test_resumed_session_injects_context(tmp_path: Path) -> None: - """Non-empty transcript → additionalContext with open-kitchen reminder.""" - transcript = tmp_path / "session.jsonl" - transcript.write_text('{"type":"say","text":"hello"}\n') - payload = json.dumps({"session_id": "abc", "transcript_path": str(transcript)}) - rc, out = _run(payload) - assert rc == 0 - data = json.loads(out.strip()) - assert "additionalContext" in data - assert "open-kitchen" in data["additionalContext"] - - -def test_missing_transcript_path_no_crash(tmp_path: Path) -> None: - """Missing transcript_path key → fail-open, no output, exit 0.""" - payload = json.dumps({"session_id": "abc"}) - rc, out = _run(payload) - assert rc == 0 - - -def test_nonexistent_transcript_no_crash(tmp_path: Path) -> None: - """transcript_path pointing to non-existent file → fail-open, no output.""" - payload = json.dumps({"session_id": "abc", "transcript_path": "/nonexistent/path.jsonl"}) - rc, out = _run(payload) - assert rc == 0 - assert "additionalContext" not in out - - -def test_session_start_sweeps_stale_markers(tmp_path: Path) -> None: - import json - from datetime import datetime - - marker_dir = tmp_path / "kitchen_state" - marker_dir.mkdir(parents=True) - stale = marker_dir / "old-session.json" - stale.write_text( - json.dumps( - { - "session_id": "old-session", - "opened_at": "2020-01-01T00:00:00+00:00", - "recipe_name": None, - "marker_version": 1, - } - ) - ) - fresh = marker_dir / "new-session.json" - fresh.write_text( - json.dumps( - { - "session_id": "new-session", - "opened_at": datetime.now(UTC).isoformat(), - "recipe_name": None, - "marker_version": 1, - } - ) - ) - transcript = tmp_path / "transcript.jsonl" - transcript.write_text("") - payload = json.dumps( - { - "session_id": "new-session", - "transcript_path": str(transcript), - "autoskillit_state_dir": str(tmp_path), - } - ) - import subprocess - import sys - - result = subprocess.run( - [sys.executable, str(SCRIPT)], - input=payload, - capture_output=True, - text=True, - env={**__import__("os").environ, "AUTOSKILLIT_STATE_DIR": str(tmp_path)}, - ) - assert result.returncode == 0 - assert not stale.exists(), "Stale marker should have been swept" - assert fresh.exists(), "Fresh marker must survive" diff --git a/tests/infra/test_skill_cmd_check.py b/tests/infra/test_skill_cmd_check.py index 08ac6c6c04..664ffb8f6b 100644 --- a/tests/infra/test_skill_cmd_check.py +++ b/tests/infra/test_skill_cmd_check.py @@ -4,7 +4,7 @@ from io import StringIO from unittest.mock import patch -from autoskillit.hooks.skill_cmd_guard import _looks_like_path, main +from autoskillit.hooks.guards.skill_cmd_guard import _looks_like_path, main def _run_hook(tool_input: dict) -> dict | None: @@ -172,9 +172,7 @@ def test_deny_message_suggests_correct_format(self): result = _run_hook({"skill_command": _CANONICAL_BUG}) reason = result["hookSpecificOutput"]["permissionDecisionReason"] # Should show the extracted path in the corrected format - assert ( - "/autoskillit:implement-worktree-no-merge .autoskillit/temp/rectify/plan.md" in reason - ) + assert "/implement-worktree-no-merge .autoskillit/temp/rectify/plan.md" in reason def test_extra_words_before_absolute_path(self): result = _run_hook({"skill_command": _WORKTREE_ABSOLUTE}) diff --git a/tests/infra/test_skill_command_guard.py b/tests/infra/test_skill_command_guard.py index 664ace95de..10634fd300 100644 --- a/tests/infra/test_skill_command_guard.py +++ b/tests/infra/test_skill_command_guard.py @@ -8,7 +8,7 @@ def _run_hook(event: dict) -> str: """Run main() with the given event JSON, return captured stdout.""" - from autoskillit.hooks.skill_command_guard import main + from autoskillit.hooks.guards.skill_command_guard import main with patch("sys.stdin", io.StringIO(json.dumps(event))): buf = io.StringIO() @@ -87,7 +87,7 @@ def test_approve_multiline_slash_command(): def test_approve_on_malformed_json(): """Malformed stdin → fail-open (empty stdout, no crash).""" - from autoskillit.hooks.skill_command_guard import main + from autoskillit.hooks.guards.skill_command_guard import main with patch("sys.stdin", io.StringIO("not-json")): buf = io.StringIO() @@ -109,7 +109,7 @@ def test_hook_uses_skill_command_prefix_constant(): """Hook must define SKILL_COMMAND_PREFIX, not hardcode '/' inline.""" import inspect - import autoskillit.hooks.skill_command_guard as mod + import autoskillit.hooks.guards.skill_command_guard as mod src = inspect.getsource(mod) assert "SKILL_COMMAND_PREFIX" in src diff --git a/tests/infra/test_skill_load_guard.py b/tests/infra/test_skill_load_guard.py new file mode 100644 index 0000000000..4a6a33d20e --- /dev/null +++ b/tests/infra/test_skill_load_guard.py @@ -0,0 +1,270 @@ +"""Tests for guards/skill_load_guard.py PreToolUse hook.""" + +from __future__ import annotations + +import io +import json +import os +from contextlib import redirect_stdout +from pathlib import Path +from unittest.mock import patch + +import pytest + +pytestmark = [pytest.mark.layer("infra"), pytest.mark.small] + +_GUARDED_TOOLS = ["Read", "Write", "Edit", "Bash", "Grep", "Glob"] + + +def _run_guard( + stdin_data: dict | str, + *, + tmp_dir: Path, + provider_profile: str | None = None, + headless: bool = False, + session_type: str | None = None, +) -> str: + """Run skill_load_guard.main(), return stdout.""" + from autoskillit.hooks.guards.skill_load_guard import main + + stdin_content = stdin_data if isinstance(stdin_data, str) else json.dumps(stdin_data) + + env_updates: dict[str, str] = {} + env_removals: list[str] = [] + + if provider_profile is not None: + env_updates["AUTOSKILLIT_PROVIDER_PROFILE"] = provider_profile + else: + env_removals.append("AUTOSKILLIT_PROVIDER_PROFILE") + + if headless: + env_updates["AUTOSKILLIT_HEADLESS"] = "1" + else: + env_removals.append("AUTOSKILLIT_HEADLESS") + + if session_type is not None: + env_updates["AUTOSKILLIT_SESSION_TYPE"] = session_type + else: + env_removals.append("AUTOSKILLIT_SESSION_TYPE") + + base_env = {k: v for k, v in os.environ.items() if k not in env_removals} + base_env.update(env_updates) + + with ( + patch.dict(os.environ, base_env, clear=True), + patch("sys.stdin", io.StringIO(stdin_content)), + patch("autoskillit.hooks.guards.skill_load_guard.Path.cwd", return_value=tmp_dir), + ): + buf = io.StringIO() + with redirect_stdout(buf): + try: + main() + except SystemExit: + pass + return buf.getvalue() + + +def _make_event( + tool_name: str = "Read", session_id: str = "abc123", agent_id: str | None = None +) -> dict: + event = { + "tool_name": tool_name, + "tool_input": {"file_path": "/foo"}, + "session_id": session_id, + } + if agent_id is not None: + event["agent_id"] = agent_id + return event + + +def _create_flag( + tmp_dir: Path, session_id: str = "abc123", content: str = "implement-worktree-no-merge" +) -> None: + flag = tmp_dir / ".autoskillit" / "temp" / f"skill_guard_{session_id}.flag" + flag.parent.mkdir(parents=True, exist_ok=True) + flag.write_text(content) + + +def test_denies_read_when_flag_absent_and_non_anthropic_headless_skill(tmp_path): + """T2-1: Deny when all gate conditions met and no flag file.""" + out = _run_guard( + _make_event("Read"), + tmp_dir=tmp_path, + provider_profile="minimax", + headless=True, + session_type="skill", + ) + response = json.loads(out) + assert response["hookSpecificOutput"]["permissionDecision"] == "deny" + assert "SKILL LOADING REQUIRED" in response["hookSpecificOutput"]["permissionDecisionReason"] + + +def test_allows_read_when_flag_exists(tmp_path): + """T2-2: Allow when flag file exists.""" + _create_flag(tmp_path) + out = _run_guard( + _make_event("Read"), + tmp_dir=tmp_path, + provider_profile="minimax", + headless=True, + session_type="skill", + ) + assert not out.strip() + + +def test_allows_silently_when_provider_profile_empty(tmp_path): + """T2-3: Allow when provider profile is not set.""" + out = _run_guard( + _make_event("Read"), + tmp_dir=tmp_path, + provider_profile=None, + headless=True, + session_type="skill", + ) + assert not out.strip() + + +def test_allows_silently_when_provider_is_anthropic(tmp_path): + """T2-4: Allow when provider is anthropic.""" + out = _run_guard( + _make_event("Read"), + tmp_dir=tmp_path, + provider_profile="anthropic", + headless=True, + session_type="skill", + ) + assert not out.strip() + + +def test_allows_silently_when_not_headless(tmp_path): + """T2-5: Allow when not headless.""" + out = _run_guard( + _make_event("Read"), + tmp_dir=tmp_path, + provider_profile="minimax", + headless=False, + session_type="skill", + ) + assert not out.strip() + + +def test_allows_silently_when_session_type_not_skill(tmp_path): + """T2-6: Allow when session type is not skill.""" + out = _run_guard( + _make_event("Read"), + tmp_dir=tmp_path, + provider_profile="minimax", + headless=True, + session_type="orchestrator", + ) + assert not out.strip() + + +@pytest.mark.parametrize("tool_name", _GUARDED_TOOLS) +def test_denies_all_guarded_tools(tmp_path, tool_name): + """T2-7: Deny for each guarded native tool.""" + out = _run_guard( + _make_event(tool_name), + tmp_dir=tmp_path, + provider_profile="minimax", + headless=True, + session_type="skill", + ) + response = json.loads(out) + assert response["hookSpecificOutput"]["permissionDecision"] == "deny" + + +def test_survives_malformed_stdin(tmp_path): + """T2-8: Fail-open on malformed JSON.""" + out = _run_guard( + "not valid json", + tmp_dir=tmp_path, + provider_profile="minimax", + headless=True, + session_type="skill", + ) + assert not out.strip() + + +def test_deny_message_contains_directive_keywords(tmp_path): + """T2-9: Deny message contains directive keywords.""" + out = _run_guard( + _make_event("Read"), + tmp_dir=tmp_path, + provider_profile="minimax", + headless=True, + session_type="skill", + ) + reason = json.loads(out)["hookSpecificOutput"]["permissionDecisionReason"] + assert "MANDATORY" in reason + assert "Skill tool" in reason + assert "Do NOT" in reason + + +def test_allows_silently_for_anthropic_case_insensitive(tmp_path): + """T2-10: Case-insensitive bypass for Anthropic.""" + out = _run_guard( + _make_event("Read"), + tmp_dir=tmp_path, + provider_profile="Anthropic", + headless=True, + session_type="skill", + ) + assert not out.strip() + + +def test_allows_when_agent_id_present(tmp_path): + """T2-11: Subagent exemption — allow when agent_id is in payload.""" + out = _run_guard( + _make_event("Read", agent_id="agent-uuid-123"), + tmp_dir=tmp_path, + provider_profile="minimax", + headless=True, + session_type="skill", + ) + assert not out.strip() + + +def test_denies_when_agent_id_is_empty_string(tmp_path): + """T2-12: Empty agent_id is falsy — guard proceeds normally.""" + out = _run_guard( + _make_event("Read", agent_id=""), + tmp_dir=tmp_path, + provider_profile="minimax", + headless=True, + session_type="skill", + ) + response = json.loads(out) + assert response["hookSpecificOutput"]["permissionDecision"] == "deny" + + +def test_flag_found_via_ancestor_walk_when_cwd_is_subdirectory(tmp_path): + """T2-13: Flag at project root found when CWD is a subdirectory.""" + project = tmp_path / "project" + flag_dir = project / ".autoskillit" / "temp" + flag_dir.mkdir(parents=True) + (flag_dir / "skill_guard_abc123.flag").write_text("make-plan") + + deep_cwd = project / "sub" / "deep" + out = _run_guard( + _make_event("Read"), + tmp_dir=deep_cwd, + provider_profile="minimax", + headless=True, + session_type="skill", + ) + assert not out.strip() + + +def test_denies_when_no_autoskillit_dir_in_ancestors(tmp_path): + """T2-14: Deny when no .autoskillit/ found in any ancestor (fallback to CWD).""" + bare_dir = tmp_path / "bare" / "dir" + out = _run_guard( + _make_event("Read"), + tmp_dir=bare_dir, + provider_profile="minimax", + headless=True, + session_type="skill", + ) + response = json.loads(out) + assert response["hookSpecificOutput"]["permissionDecision"] == "deny" diff --git a/tests/infra/test_skill_orchestration_guard.py b/tests/infra/test_skill_orchestration_guard.py new file mode 100644 index 0000000000..3a2e3ea31d --- /dev/null +++ b/tests/infra/test_skill_orchestration_guard.py @@ -0,0 +1,154 @@ +"""Tests for skill_orchestration_guard.py PreToolUse hook.""" + +from __future__ import annotations + +import io +import json +import os +from contextlib import redirect_stdout +from unittest.mock import patch + +import pytest + +_ORCHESTRATION_TOOLS = ["run_skill", "run_cmd", "run_python"] + + +def _run_guard( + tool_input: dict, + *, + headless: bool = False, + session_type: str | None = None, + raw_stdin: str | None = None, +) -> str: + """Run main() with the given event, optionally setting headless/session_type env vars. + + raw_stdin: if provided, passed directly to stdin instead of json.dumps(tool_input). + Use this to test malformed-input paths without duplicating setup boilerplate. + """ + from autoskillit.hooks.guards.skill_orchestration_guard import main + + stdin_content = raw_stdin if raw_stdin is not None else json.dumps(tool_input) + env_updates: dict[str, str] = {} + if headless: + env_updates["AUTOSKILLIT_HEADLESS"] = "1" + if session_type is not None: + env_updates["AUTOSKILLIT_SESSION_TYPE"] = session_type + with ( + patch.dict(os.environ, env_updates, clear=False), + patch("sys.stdin", io.StringIO(stdin_content)), + ): + buf = io.StringIO() + with redirect_stdout(buf): + try: + main() + except SystemExit: + pass + return buf.getvalue() + + +def _run_guard_headless(tool_input: dict, session_type: str = "skill") -> dict: + """Run guard in headless mode and parse output JSON.""" + out = _run_guard(tool_input, headless=True, session_type=session_type) + return json.loads(out) if out.strip() else {} + + +@pytest.mark.parametrize("tool_name", _ORCHESTRATION_TOOLS) +def test_guard_denies_skill_tier(tool_name): + response = _run_guard_headless( + {"tool_name": f"mcp__autoskillit__{tool_name}"}, session_type="skill" + ) + assert response["hookSpecificOutput"]["permissionDecision"] == "deny" + assert "skill" in response["hookSpecificOutput"]["permissionDecisionReason"].lower() + + +@pytest.mark.parametrize("tool_name", _ORCHESTRATION_TOOLS) +def test_guard_denies_deprecated_leaf_tier(tool_name): + """Backward compat: SESSION_TYPE=leaf (old value) is also denied.""" + response = _run_guard_headless( + {"tool_name": f"mcp__autoskillit__{tool_name}"}, session_type="leaf" + ) + assert response["hookSpecificOutput"]["permissionDecision"] == "deny" + assert "skill" in response["hookSpecificOutput"]["permissionDecisionReason"].lower() + + +def test_guard_permits_orchestrator_tier(): + out = _run_guard( + {"tool_name": "mcp__autoskillit__run_skill"}, + headless=True, + session_type="orchestrator", + ) + assert not out.strip() + + +def test_guard_permits_fleet_tier(): + out = _run_guard( + {"tool_name": "mcp__autoskillit__run_skill"}, + headless=True, + session_type="fleet", + ) + assert not out.strip() + + +def test_guard_denies_when_session_type_unset(): + """Fail-closed: headless with no SESSION_TYPE is treated as skill session → deny.""" + from autoskillit.hooks.guards.skill_orchestration_guard import main + + stdin_content = json.dumps({"tool_name": "mcp__autoskillit__run_skill"}) + env_without_session_type = { + k: v for k, v in os.environ.items() if k != "AUTOSKILLIT_SESSION_TYPE" + } + env_without_session_type["AUTOSKILLIT_HEADLESS"] = "1" + with ( + patch.dict(os.environ, env_without_session_type, clear=True), + patch("sys.stdin", io.StringIO(stdin_content)), + ): + buf = io.StringIO() + with redirect_stdout(buf): + try: + main() + except SystemExit: + pass + out = buf.getvalue() + response = json.loads(out) if out.strip() else {} + assert response["hookSpecificOutput"]["permissionDecision"] == "deny" + + +def test_guard_allows_non_headless(): + out = _run_guard({"tool_name": "mcp__autoskillit__run_skill"}, headless=False) + assert not out.strip() + + +def test_guard_allows_non_orchestration_tool_when_headless(): + response = _run_guard_headless({"tool_name": "mcp__autoskillit__fetch_github_issue"}) + assert response == {} + + +def test_guard_allows_merge_worktree_when_headless(): + """merge_worktree is not an orchestration tool — workers can still call it.""" + response = _run_guard_headless({"tool_name": "mcp__autoskillit__merge_worktree"}) + assert response == {} + + +def test_guard_fails_open_on_malformed_input(): + """Malformed stdin must not raise — hook exits 0 silently.""" + out = _run_guard({}, headless=True, raw_stdin="not-json") + assert not out.strip() + + +@pytest.mark.parametrize("tool_name", _ORCHESTRATION_TOOLS) +def test_skill_orchestration_guard_permits_fleet_session(tool_name): + response = _run_guard_headless( + {"tool_name": f"mcp__autoskillit__{tool_name}"}, session_type="fleet" + ) + assert response == {}, f"Expected fleet to be permitted for {tool_name}" + + +def test_guard_denies_unrecognized_tier_with_diagnostic_note(): + """Unrecognized non-empty session type is denied with a diagnostic note naming the value.""" + response = _run_guard_headless( + {"tool_name": "mcp__autoskillit__run_skill"}, session_type="orchestratr" + ) + assert response["hookSpecificOutput"]["permissionDecision"] == "deny" + reason = response["hookSpecificOutput"]["permissionDecisionReason"] + assert "orchestratr" in reason + assert "not a recognized tier" in reason diff --git a/tests/infra/test_taskfile.py b/tests/infra/test_taskfile.py index 03ff88f9b7..29254616dd 100644 --- a/tests/infra/test_taskfile.py +++ b/tests/infra/test_taskfile.py @@ -54,12 +54,12 @@ def test_install_dev_task_exists(self): data = self._load() assert "install-dev" in data["tasks"], "install-dev task missing from Taskfile.yml" - def test_install_dev_task_uses_integration_branch(self): - """TF-7 — install-dev installs from @integration and runs autoskillit install.""" + def test_install_dev_task_uses_develop_branch(self): + """TF-7 — install-dev installs from @develop and runs autoskillit install.""" data = self._load() task = data["tasks"]["install-dev"] cmds = " ".join(str(c) for c in task.get("cmds", [])) - assert "@integration" in cmds, "install-dev must install from @integration branch" + assert "@develop" in cmds, "install-dev must install from @develop branch" assert "autoskillit install" in cmds, "install-dev must run autoskillit install after uv" def test_vendor_mermaid_task_exists(self) -> None: @@ -77,6 +77,33 @@ def test_vendor_mermaid_task_targets_v11(self) -> None: "vendor-mermaid must write to src/autoskillit/assets/mermaid/mermaid.min.js" ) + def test_test_filtered_task_exists(self): + """TF-8 — test-filtered task exists in Taskfile.yml.""" + data = self._load() + assert "test-filtered" in data["tasks"], "test-filtered task missing from Taskfile.yml" + + def test_test_filtered_delegates_to_test_check(self): + """TF-9 — test-filtered delegates to test-check.""" + data = self._load() + cmds = " ".join(str(c) for c in data["tasks"]["test-filtered"].get("cmds", [])) + assert "test-check" in cmds, "test-filtered must delegate to test-check" + + def test_test_filtered_sets_filter_env_default(self): + """TF-10 — test-filtered defaults AUTOSKILLIT_TEST_FILTER to conservative.""" + data = self._load() + cmds = " ".join(str(c) for c in data["tasks"]["test-filtered"].get("cmds", [])) + assert "AUTOSKILLIT_TEST_FILTER" in cmds, ( + "test-filtered must reference AUTOSKILLIT_TEST_FILTER" + ) + assert "conservative" in cmds, ( + "test-filtered must default AUTOSKILLIT_TEST_FILTER to conservative" + ) + + def test_coverage_audit_task_exists(self): + """TF-11 — coverage-audit task exists in Taskfile.yml.""" + data = self._load() + assert "coverage-audit" in data["tasks"], "coverage-audit task missing from Taskfile.yml" + def test_taskfile_pytest_paths_exist() -> None: """All pytest file paths in Taskfile.yml must exist.""" diff --git a/tests/infra/test_testmon_eval.py b/tests/infra/test_testmon_eval.py new file mode 100644 index 0000000000..6f5527b96b --- /dev/null +++ b/tests/infra/test_testmon_eval.py @@ -0,0 +1,65 @@ +from __future__ import annotations + +import ast +import importlib +import importlib.util +from pathlib import Path + +import pytest + +pytestmark = [pytest.mark.small] + +REPO_ROOT = Path(__file__).resolve().parents[2] + + +def test_testmon_importable(): + """pytest-testmon must be importable when installed with dev extras.""" + mod = importlib.import_module("testmon") + assert hasattr(mod, "TESTMON_VERSION") + + +def test_testmondata_gitignored(): + """The .testmondata SQLite DB must be gitignored.""" + gitignore = (REPO_ROOT / ".gitignore").read_text() + assert ".testmondata" in gitignore + + +def test_testmon_not_active_by_default(pytestconfig: pytest.Config): + """testmon must not be active unless explicitly opted in via --testmon.""" + plugin = pytestconfig.pluginmanager.get_plugin("pytest-testmon") + if plugin is None: + pytest.skip("testmon plugin not registered") + assert "--testmon" not in pytestconfig.getini("addopts") + + +def test_benchmark_script_exists(): + """The testmon benchmark script must exist and be valid Python.""" + script = REPO_ROOT / "scripts" / "benchmark-testmon.py" + assert script.exists(), f"Missing: {script}" + ast.parse(script.read_text()) + + +def test_benchmark_compute_metrics(): + """compute_selection_metrics returns correct precision/recall/f1.""" + script = REPO_ROOT / "scripts" / "benchmark-testmon.py" + spec = importlib.util.spec_from_file_location("benchmark_testmon", script) + assert spec is not None and spec.loader is not None + mod = importlib.util.module_from_spec(spec) + spec.loader.exec_module(mod) + testmon_set = {"test_a.py", "test_b.py", "test_c.py"} + cascade_set = {"test_b.py", "test_c.py", "test_d.py", "test_e.py"} + metrics = mod.compute_selection_metrics(testmon_set, cascade_set) + assert metrics["testmon_only"] == {"test_a.py"} + assert metrics["cascade_only"] == {"test_d.py", "test_e.py"} + assert metrics["overlap"] == {"test_b.py", "test_c.py"} + assert metrics["testmon_count"] == 3 + assert metrics["cascade_count"] == 4 + assert metrics["overlap_count"] == 2 + assert metrics["jaccard_similarity"] == 2 / 5 + + +def test_taskfile_has_testmon_tasks(): + """Taskfile must define testmon evaluation tasks.""" + taskfile = (REPO_ROOT / "Taskfile.yml").read_text() + for task_name in ("testmon-build", "testmon-run", "testmon-benchmark"): + assert f" {task_name}:" in taskfile, f"Missing task: {task_name}" diff --git a/tests/infra/test_token_summary_core.py b/tests/infra/test_token_summary_core.py new file mode 100644 index 0000000000..5f9b6f1276 --- /dev/null +++ b/tests/infra/test_token_summary_core.py @@ -0,0 +1,814 @@ +"""Tests: token_summary_appender core — existence, early-exit, happy path, session filtering.""" + +from __future__ import annotations + +import json +import subprocess +import sys +from pathlib import Path +from unittest.mock import MagicMock, patch + +import pytest + +from tests.infra._token_summary_helpers import _make_run_skill_event, _run_hook, _write_sessions + +pytestmark = [pytest.mark.layer("infra"), pytest.mark.medium] + + +def test_tsa1_token_summary_appender_script_exists() -> None: + """token_summary_hook.py must exist in hooks/ on disk.""" + from autoskillit.core.paths import pkg_root + + assert (pkg_root() / "hooks" / "token_summary_hook.py").exists() + + +def test_tsa_rest_api_no_gh_pr_commands() -> None: + """Hook source must not contain 'gh pr edit' or 'gh pr view' subprocess calls.""" + from autoskillit.core.paths import pkg_root + + source = (pkg_root() / "hooks" / "token_summary_hook.py").read_text(encoding="utf-8") + assert "gh pr edit" not in source, ( + "gh pr edit found in hook — must be replaced with " + "gh api repos/.../pulls/{N} --method PATCH --field body=..." + ) + assert "gh pr view" not in source, ( + "gh pr view found in hook — must be replaced with gh api repos/.../pulls/{N} --jq '.body'" + ) + + +def test_tsa2_no_pr_url_exits_zero() -> None: + """No GitHub PR URL in tool result → exits 0, no gh subprocess.""" + from autoskillit.core.paths import pkg_root + + hook_path = pkg_root() / "hooks" / "token_summary_hook.py" + event = _make_run_skill_event("done.\n%%ORDER_UP%%") + + proc = subprocess.run( + [sys.executable, str(hook_path)], + input=json.dumps(event), + capture_output=True, + text=True, + timeout=10, + ) + assert proc.returncode == 0 + assert "gh" not in proc.stdout + + +def test_tsa3_no_sessions_jsonl_exits_zero(tmp_path: Path) -> None: + """Missing sessions.jsonl → exits 0 (valid: no sessions yet).""" + pr_url = "https://github.com/owner/repo/pull/42" + event = _make_run_skill_event(f"pr_url={pr_url}\n%%ORDER_UP%%") + + _, exit_code = _run_hook(event, log_root=tmp_path / "nonexistent") + assert exit_code == 0 + + +def test_tsa4_no_pipeline_id_sessions_exits_zero(tmp_path: Path) -> None: + """sessions.jsonl exists but no pipeline_id set → hook skips all sessions → exits 0.""" + log_root = tmp_path / "logs" + log_root.mkdir() + + _write_sessions( + log_root, + [ + {"dir_name": "s1", "cwd": "/some/other/pipeline", "step_name": "plan"}, + ], + ) + + pr_url = "https://github.com/owner/repo/pull/42" + event = _make_run_skill_event(f"pr_url={pr_url}\n%%ORDER_UP%%") + + _, exit_code = _run_hook(event, log_root=log_root) + assert exit_code == 0 + + +def test_tsa5_matching_sessions_formats_table_and_edits_pr(tmp_path: Path) -> None: + """Matching sessions → aggregate token data and append ## Token Usage Summary.""" + + log_root = tmp_path / "logs" + log_root.mkdir() + pipeline_id = "test-pipeline-tsa5" + + _write_sessions( + log_root, + [ + { + "dir_name": "session-1", + "cwd": "/some/worktree", + "kitchen_id": pipeline_id, + "step_name": "plan-1", + "input_tokens": 1000, + "output_tokens": 500, + "cache_creation_input_tokens": 100, + "cache_read_input_tokens": 200, + "timing_seconds": 10.0, + }, + { + "dir_name": "session-2", + "cwd": "/some/worktree", + "kitchen_id": pipeline_id, + "step_name": "plan-2", + "input_tokens": 1000, + "output_tokens": 500, + "cache_creation_input_tokens": 100, + "cache_read_input_tokens": 200, + "timing_seconds": 10.0, + }, + { + "dir_name": "session-3", + "cwd": "/some/worktree", + "kitchen_id": pipeline_id, + "step_name": "open-pr", + "input_tokens": 500, + "output_tokens": 250, + "cache_creation_input_tokens": 50, + "cache_read_input_tokens": 100, + "timing_seconds": 5.0, + }, + ], + ) + + hook_config = tmp_path / ".autoskillit_hook_config.json" + hook_config.write_text(json.dumps({"kitchen_id": pipeline_id})) + + pr_url = "https://github.com/owner/repo/pull/42" + event = _make_run_skill_event(f"pr_url={pr_url}\n%%ORDER_UP%%") + + view_result = MagicMock() + view_result.returncode = 0 + view_result.stdout = "Existing PR body without summary." + + edit_calls: list[list[str]] = [] + + def subprocess_side_effect(args: list[str], **kwargs: object) -> MagicMock: + if "api" in args and "--method" not in args: + return view_result + if "api" in args and "--method" in args: + edit_calls.append(list(args)) + return MagicMock(returncode=0) + return MagicMock(returncode=0) + + with patch("subprocess.run", side_effect=subprocess_side_effect): + _, exit_code = _run_hook(event, log_root=log_root, hook_config_path=hook_config) + + assert exit_code == 0 + assert len(edit_calls) == 1 + field_idx = edit_calls[0].index("--raw-field") + body_arg = edit_calls[0][field_idx + 1] + assert body_arg.startswith("body=") + body_content = body_arg[len("body=") :] + assert "## Token Usage Summary" in body_content + assert "plan" in body_content + assert "open-pr" in body_content + assert "**Total**" in body_content + assert "uncached" in body_content + assert "cache_read" in body_content + assert "cache_write" in body_content + + +def test_tsa6_idempotency_skips_if_summary_present(tmp_path: Path) -> None: + """PR body already contains ## Token Usage Summary → gh pr edit NOT called.""" + + log_root = tmp_path / "logs" + log_root.mkdir() + pipeline_id = "test-pipeline-tsa6" + + _write_sessions( + log_root, + [ + { + "dir_name": "session-1", + "cwd": "/some/worktree", + "kitchen_id": pipeline_id, + "step_name": "plan", + "input_tokens": 1000, + "output_tokens": 500, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + "timing_seconds": 10.0, + }, + ], + ) + + hook_config = tmp_path / ".autoskillit_hook_config.json" + hook_config.write_text(json.dumps({"kitchen_id": pipeline_id})) + + pr_url = "https://github.com/owner/repo/pull/42" + event = _make_run_skill_event(f"pr_url={pr_url}\n%%ORDER_UP%%") + + view_result = MagicMock() + view_result.returncode = 0 + view_result.stdout = "## Token Usage Summary\n\n| Step | input |...\n" + + edit_calls: list = [] + + def subprocess_side_effect(args: list[str], **kwargs: object) -> MagicMock: + if "api" in args and "--method" in args: + edit_calls.append(args) + return view_result + + with patch("subprocess.run", side_effect=subprocess_side_effect): + _, exit_code = _run_hook(event, log_root=log_root, hook_config_path=hook_config) + + assert exit_code == 0 + assert len(edit_calls) == 0 + + +def test_tsa_kitchen_id_match_despite_cwd_mismatch(tmp_path: Path) -> None: + """kitchen_id match + CWD mismatch → sessions FOUND → hook appends table.""" + + log_root = tmp_path / "logs" + log_root.mkdir() + kitchen_id = "test-kitchen-abc123" + + _write_sessions( + log_root, + [ + { + "dir_name": "s1", + "cwd": "/worktrees/impl-fix", + "kitchen_id": kitchen_id, + "step_name": "implement", + "input_tokens": 1000, + "output_tokens": 500, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + "timing_seconds": 20.0, + } + ], + ) + + hook_config = tmp_path / ".autoskillit_hook_config.json" + hook_config.write_text(json.dumps({"kitchen_id": kitchen_id})) + pr_url = "https://github.com/owner/repo/pull/42" + event = _make_run_skill_event(f"pr_url={pr_url}\n%%ORDER_UP%%") + view_result = MagicMock(returncode=0, stdout="Existing PR body.") + edit_calls: list = [] + + def run_side(args, **kwargs): + if "api" in args and "--method" not in args: + return view_result + if "api" in args and "--method" in args: + edit_calls.append(args) + return MagicMock(returncode=0) + return MagicMock(returncode=0) + + with patch("subprocess.run", side_effect=run_side): + _, exit_code = _run_hook( + event, + log_root=log_root, + hook_config_path=hook_config, + ) + assert exit_code == 0 + assert len(edit_calls) == 1, "gh api PATCH must be called when kitchen_id matches" + + +def test_tsa_kitchen_id_mismatch_exits_zero(tmp_path: Path) -> None: + """Wrong kitchen_id → no sessions found → exits 0, no gh pr edit.""" + + log_root = tmp_path / "logs" + log_root.mkdir() + _write_sessions( + log_root, + [ + { + "dir_name": "s1", + "cwd": "/worktree", + "kitchen_id": "kitchen-A", + "step_name": "implement", + "input_tokens": 1000, + "output_tokens": 500, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + "timing_seconds": 10.0, + } + ], + ) + hook_config = tmp_path / ".autoskillit_hook_config.json" + hook_config.write_text(json.dumps({"kitchen_id": "kitchen-B"})) + event = _make_run_skill_event("pr_url=https://github.com/owner/repo/pull/99\n%%ORDER_UP%%") + _, exit_code = _run_hook(event, log_root=log_root, hook_config_path=hook_config) + assert exit_code == 0 + + +def test_tsa8_gh_pr_edit_failure_exits_nonzero(tmp_path: Path) -> None: + """gh pr edit returning non-zero → hook exits 0 (fail-open).""" + + log_root = tmp_path / "logs" + log_root.mkdir() + pipeline_id = "test-pipeline-tsa8" + + _write_sessions( + log_root, + [ + { + "dir_name": "session-1", + "cwd": "/some/worktree", + "kitchen_id": pipeline_id, + "step_name": "plan", + "input_tokens": 1000, + "output_tokens": 500, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + "timing_seconds": 5.0, + }, + ], + ) + + hook_config = tmp_path / ".autoskillit_hook_config.json" + hook_config.write_text(json.dumps({"kitchen_id": pipeline_id})) + + pr_url = "https://github.com/owner/repo/pull/42" + event = _make_run_skill_event(f"pr_url={pr_url}\n%%ORDER_UP%%") + + view_result = MagicMock() + view_result.returncode = 0 + view_result.stdout = "Existing body without summary." + + def subprocess_side_effect(args: list[str], **kwargs: object) -> MagicMock: + if "api" in args and "--method" not in args: + return view_result + if "api" in args and "--method" in args: + raise subprocess.CalledProcessError(1, args) + return MagicMock(returncode=0) + + with patch("subprocess.run", side_effect=subprocess_side_effect): + _, exit_code = _run_hook(event, log_root=log_root, hook_config_path=hook_config) + + assert exit_code == 0 + + +def test_tsa_gh_pr_edit_stderr_captured(tmp_path: Path) -> None: + """gh pr edit failure includes stderr in the logged error message.""" + + log_root = tmp_path / "logs" + log_root.mkdir() + pipeline_id = "pipe-edit-test" + + _write_sessions( + log_root, + [ + { + "dir_name": "s1", + "cwd": "/w", + "kitchen_id": pipeline_id, + "step_name": "plan", + "input_tokens": 100, + "output_tokens": 50, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + "timing_seconds": 5.0, + } + ], + ) + + hook_config = tmp_path / ".autoskillit_hook_config.json" + hook_config.write_text(json.dumps({"kitchen_id": pipeline_id})) + + pr_url = "https://github.com/owner/repo/pull/1" + event = _make_run_skill_event(f"pr_url={pr_url}\n%%ORDER_UP%%") + + view_ok = MagicMock(returncode=0, stdout="Some body.") + error = subprocess.CalledProcessError(1, ["gh", "api", "repos/owner/repo/pulls/1"]) + error.stderr = "authentication required" + + def run_side(args, **kwargs): + if "api" in args and "--method" not in args: + return view_ok + if "api" in args and "--method" in args: + raise error + return MagicMock(returncode=0) + + stderr_output: list[str] = [] + with patch("subprocess.run", side_effect=run_side): + with patch("sys.stderr") as mock_stderr: + mock_stderr.write = lambda s: stderr_output.append(s) + _, exit_code = _run_hook(event, log_root=log_root, hook_config_path=hook_config) + + assert exit_code == 0 + combined = "".join(stderr_output) + assert "authentication required" in combined, ( + "stderr from CalledProcessError must appear in diagnostic output" + ) + + +def test_tsa_gh_pr_view_failure_emits_diagnostic(tmp_path: Path) -> None: + """gh pr view non-zero exit emits a stderr message before exiting 0.""" + + log_root = tmp_path / "logs" + log_root.mkdir() + pipeline_id = "pipe-view-fail" + + _write_sessions( + log_root, + [ + { + "dir_name": "s1", + "cwd": "/w", + "kitchen_id": pipeline_id, + "step_name": "plan", + "input_tokens": 100, + "output_tokens": 50, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + "timing_seconds": 5.0, + } + ], + ) + + hook_config = tmp_path / ".autoskillit_hook_config.json" + hook_config.write_text(json.dumps({"kitchen_id": pipeline_id})) + + pr_url = "https://github.com/owner/repo/pull/2" + event = _make_run_skill_event(f"pr_url={pr_url}\n%%ORDER_UP%%") + + view_fail = MagicMock(returncode=1, stderr="HTTP 401 Unauthorized", stdout="") + + stderr_output: list[str] = [] + with patch("subprocess.run", return_value=view_fail): + with patch("sys.stderr") as mock_stderr: + mock_stderr.write = lambda s: stderr_output.append(s) + _, exit_code = _run_hook(event, log_root=log_root, hook_config_path=hook_config) + + assert exit_code == 0 + combined = "".join(stderr_output) + assert combined.strip(), "gh pr view failure must emit a diagnostic to stderr" + + +def test_token_summary_hook_patch_failure_exits_zero(tmp_path: Path) -> None: + """CalledProcessError on PATCH must exit 0, not 1 (fail-open hook).""" + + event = { + "tool_name": "mcp__autoskillit_server__run_skill", + "tool_response": json.dumps({"result": json.dumps({"success": True})}), + } + pr_event = { + **event, + "tool_response": json.dumps( + { + "result": json.dumps( + { + "success": True, + "pr_url": "https://github.com/owner/repo/pull/1", + } + ) + } + ), + } + + original_run = subprocess.run + + def failing_run(cmd, **kwargs): + if "PATCH" in (cmd if isinstance(cmd, str) else " ".join(str(c) for c in cmd)): + raise subprocess.CalledProcessError(1, cmd, stderr="API error") + return original_run(cmd, **kwargs) + + with patch("autoskillit.hooks.token_summary_hook.subprocess.run", failing_run): + _, exit_code = _run_hook( + event=pr_event, + log_root=tmp_path, + ) + assert exit_code == 0 + + +def test_token_summary_hook_unexpected_error_exits_zero(monkeypatch: object) -> None: + """Unhandled exception in outer except must exit 0 (fail-open).""" + + original_loads = json.loads + call_count = [0] + + def bomb_loads(s: str) -> object: + call_count[0] += 1 + if call_count[0] == 1: + raise RuntimeError("injected failure") + return original_loads(s) + + with patch("autoskillit.hooks.token_summary_hook.json.loads", bomb_loads): + _, exit_code = _run_hook(event={"tool_name": "any", "tool_response": "{}"}) + + assert exit_code == 0 + + +def test_efficiency_table_equivalence() -> None: + """format_efficiency_table and hook _format_efficiency_table produce identical output.""" + from autoskillit.hooks.token_summary_hook import _format_efficiency_table + from autoskillit.pipeline.telemetry_fmt import TelemetryFormatter + + steps_data = [ + { + "step_name": "investigate", + "peak_context": 500, + "cache_read_input_tokens": 8000, + "cache_creation_input_tokens": 2000, + "output_tokens": 1000, + "loc_insertions": 30, + "loc_deletions": 10, + }, + { + "step_name": "implement", + "peak_context": 12000, + "cache_read_input_tokens": 50000, + "cache_creation_input_tokens": 15000, + "output_tokens": 8000, + "loc_insertions": 200, + "loc_deletions": 50, + }, + ] + + canonical_total = { + "loc_insertions": sum(s["loc_insertions"] for s in steps_data), + "loc_deletions": sum(s["loc_deletions"] for s in steps_data), + "peak_context": max(s["peak_context"] for s in steps_data), + "cache_read_input_tokens": sum(s["cache_read_input_tokens"] for s in steps_data), + "cache_creation_input_tokens": sum(s["cache_creation_input_tokens"] for s in steps_data), + "output_tokens": sum(s["output_tokens"] for s in steps_data), + } + aggregated = {s["step_name"]: dict(s) for s in steps_data} + + canonical_output = TelemetryFormatter.format_efficiency_table( + list(steps_data), canonical_total + ) + hook_output = _format_efficiency_table(aggregated) + + assert canonical_output == hook_output, ( + f"Canonical and hook efficiency tables differ:\n" + f"CANONICAL:\n{canonical_output}\n\nHOOK:\n{hook_output}" + ) + + +def test_token_table_equivalence_non_anthropic() -> None: + """Token table equivalence holds when non-Anthropic models are present.""" + from autoskillit.hooks.token_summary_hook import _format_table + from autoskillit.pipeline.telemetry_fmt import TelemetryFormatter + + steps_data = [ + { + "step_name": "plan", + "model": "claude-sonnet-4-6", + "input_tokens": 7000, + "output_tokens": 5939, + "cache_creation_input_tokens": 8495, + "cache_read_input_tokens": 252179, + "peak_context": 45000, + "turn_count": 8, + "invocation_count": 1, + "elapsed_seconds": 45.0, + }, + { + "step_name": "implement", + "model": "MiniMax-M2.7-highspeed", + "input_tokens": 2031000, + "output_tokens": 122306, + "cache_creation_input_tokens": 280601, + "cache_read_input_tokens": 19071323, + "peak_context": 890000, + "turn_count": 42, + "invocation_count": 3, + "elapsed_seconds": 492.0, + }, + ] + + canonical_total = { + "input_tokens": sum(s["input_tokens"] for s in steps_data), + "output_tokens": sum(s["output_tokens"] for s in steps_data), + "cache_creation_input_tokens": sum(s["cache_creation_input_tokens"] for s in steps_data), + "cache_read_input_tokens": sum(s["cache_read_input_tokens"] for s in steps_data), + "peak_context": max(s["peak_context"] for s in steps_data), + "total_elapsed_seconds": sum(s["elapsed_seconds"] for s in steps_data), + } + aggregated = {s["step_name"]: dict(s) for s in steps_data} + + canonical_output = TelemetryFormatter.format_token_table(list(steps_data), canonical_total) + hook_output = _format_table(aggregated) + + assert canonical_output == hook_output + assert "implement*" in canonical_output + assert "non-Anthropic provider" in canonical_output + assert "plan*" not in canonical_output + + +def test_token_table_equivalence() -> None: + """Canonical format_token_table and hook _format_table produce identical output.""" + from autoskillit.hooks.token_summary_hook import _format_table + from autoskillit.pipeline.telemetry_fmt import TelemetryFormatter + + steps_data = [ + { + "step_name": "investigate", + "model": "claude-sonnet-4-6", + "input_tokens": 7000, + "output_tokens": 5939, + "cache_creation_input_tokens": 8495, + "cache_read_input_tokens": 252179, + "peak_context": 45000, + "turn_count": 8, + "invocation_count": 1, + "elapsed_seconds": 45.0, + }, + { + "step_name": "implement", + "model": "claude-sonnet-4-6", + "input_tokens": 2031000, + "output_tokens": 122306, + "cache_creation_input_tokens": 280601, + "cache_read_input_tokens": 19071323, + "peak_context": 890000, + "turn_count": 42, + "invocation_count": 3, + "elapsed_seconds": 492.0, + }, + ] + + canonical_total = { + "input_tokens": sum(s["input_tokens"] for s in steps_data), + "output_tokens": sum(s["output_tokens"] for s in steps_data), + "cache_creation_input_tokens": sum(s["cache_creation_input_tokens"] for s in steps_data), + "cache_read_input_tokens": sum(s["cache_read_input_tokens"] for s in steps_data), + "peak_context": max(s["peak_context"] for s in steps_data), + "total_elapsed_seconds": sum(s["elapsed_seconds"] for s in steps_data), + } + aggregated = {s["step_name"]: dict(s) for s in steps_data} + + canonical_output = TelemetryFormatter.format_token_table(list(steps_data), canonical_total) + hook_output = _format_table(aggregated) + + assert canonical_output == hook_output, ( + f"Canonical and hook token tables differ:\n" + f"CANONICAL:\n{canonical_output}\n\nHOOK:\n{hook_output}" + ) + + +def test_format_table_includes_model_column() -> None: + """Hook _format_table includes Model column and value.""" + from autoskillit.hooks.token_summary_hook import _format_table + + aggregated = { + "plan": { + "step_name": "plan", + "model": "claude-sonnet-4-6", + "input_tokens": 1000, + "output_tokens": 500, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 200, + "elapsed_seconds": 60.0, + "invocation_count": 1, + "loc_insertions": 0, + "loc_deletions": 0, + "peak_context": 0, + "turn_count": 5, + } + } + table = _format_table(aggregated) + assert "| Model |" in table + assert "claude-sonnet-4-6" in table + + +def test_hook_format_model_table() -> None: + """Hook _format_model_table produces per-model aggregate table.""" + from autoskillit.hooks.token_summary_hook import _format_model_table + + aggregated = { + "plan": { + "step_name": "plan", + "model": "claude-sonnet-4-6", + "input_tokens": 100, + "output_tokens": 50, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + "elapsed_seconds": 30.0, + "invocation_count": 1, + }, + "implement": { + "step_name": "implement", + "model": "MiniMax-M2.7", + "input_tokens": 500, + "output_tokens": 200, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + "elapsed_seconds": 60.0, + "invocation_count": 1, + }, + } + table = _format_model_table(aggregated) + assert "## Model Usage Breakdown" in table + assert "| Model | steps | uncached | output | cache_read | cache_write | time |" in table + assert "| claude-sonnet-4-6 | 1 | 100 | 50 | 0 | 0 | 30s |" in table + assert "| MiniMax-M2.7 | 1 | 500 | 200 | 0 | 0 | 1m 0s |" in table + table_lines = [ln for ln in table.splitlines() if ln.startswith("|")] + assert len(table_lines) == 4 # header + separator + 2 model rows + + +def test_hook_format_model_table_no_model_returns_empty() -> None: + """Hook _format_model_table returns '' when all entries have no model.""" + from autoskillit.hooks.token_summary_hook import _format_model_table + + aggregated = { + "plan": { + "step_name": "plan", + "model": "", + "input_tokens": 100, + "output_tokens": 50, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + "elapsed_seconds": 30.0, + } + } + assert _format_model_table(aggregated) == "" + + +def test_load_sessions_reads_model_identifier(tmp_path: Path) -> None: + """Hook _load_sessions populates model field from model_identifier in token_usage.json.""" + from autoskillit.hooks.token_summary_hook import _load_sessions + + kitchen_id = "test-kitchen-t10" + log_root = tmp_path / "logs" + log_root.mkdir() + + sessions_dir = log_root / "sessions" / "s1" + sessions_dir.mkdir(parents=True) + (sessions_dir / "token_usage.json").write_text( + json.dumps( + { + "session_label": "plan", + "input_tokens": 100, + "output_tokens": 50, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + "timing_seconds": 10.0, + "model_identifier": "claude-sonnet-4-6", + } + ) + ) + (log_root / "sessions.jsonl").write_text( + json.dumps( + { + "session_id": "s1", + "dir_name": "s1", + "kitchen_id": kitchen_id, + "timestamp": "2026-01-01T00:00:00Z", + } + ) + + "\n" + ) + + aggregated = _load_sessions(log_root, kitchen_id) + assert "plan" in aggregated + assert aggregated["plan"]["model"] == "claude-sonnet-4-6" + + +def test_model_table_equivalence() -> None: + """_format_model_table (hook) and format_model_table (canonical) produce equivalent output.""" + from autoskillit.hooks.token_summary_hook import _format_model_table + from autoskillit.pipeline.telemetry_fmt import TelemetryFormatter + + aggregated = { + "plan": { + "step_name": "plan", + "model": "claude-sonnet-4-6", + "input_tokens": 1000, + "output_tokens": 500, + "cache_creation_input_tokens": 100, + "cache_read_input_tokens": 200, + "elapsed_seconds": 60.0, + "invocation_count": 1, + }, + "implement": { + "step_name": "implement", + "model": "MiniMax-M2.7", + "input_tokens": 5000, + "output_tokens": 2000, + "cache_creation_input_tokens": 500, + "cache_read_input_tokens": 1000, + "elapsed_seconds": 120.0, + "invocation_count": 2, + }, + } + + hook_table = _format_model_table(aggregated) + + model_totals = [ + { + "model": "claude-sonnet-4-6", + "step_count": 1, + "input_tokens": 1000, + "output_tokens": 500, + "cache_creation_input_tokens": 100, + "cache_read_input_tokens": 200, + "elapsed_seconds": 60.0, + }, + { + "model": "MiniMax-M2.7", + "step_count": 1, + "input_tokens": 5000, + "output_tokens": 2000, + "cache_creation_input_tokens": 500, + "cache_read_input_tokens": 1000, + "elapsed_seconds": 120.0, + }, + ] + canonical_table = TelemetryFormatter.format_model_table(model_totals) + + assert hook_table == canonical_table, ( + f"Hook and canonical model tables diverged:" + f"\nHOOK:\n{hook_table}\n\nCANONICAL:\n{canonical_table}" + ) diff --git a/tests/infra/test_token_summary_filters.py b/tests/infra/test_token_summary_filters.py new file mode 100644 index 0000000000..a7484dd208 --- /dev/null +++ b/tests/infra/test_token_summary_filters.py @@ -0,0 +1,309 @@ +"""Tests: token_summary_appender unit helpers and order_id isolation.""" + +from __future__ import annotations + +import ast +import json +from pathlib import Path +from unittest.mock import MagicMock, patch + +import pytest + +from autoskillit.core.paths import pkg_root +from autoskillit.hooks.formatters._fmt_primitives import _HOOK_CONFIG_PATH_COMPONENTS +from tests.infra._token_summary_helpers import _run_hook, _write_sessions + +pytestmark = [pytest.mark.layer("infra"), pytest.mark.medium] + + +def test_tsa7_canonical_step_name() -> None: + """_canonical strips trailing -N suffix; non-digit suffixes are preserved.""" + from autoskillit.hooks.token_summary_hook import _canonical + + assert _canonical("plan-30") == "plan" + assert _canonical("open-pr-2") == "open-pr" + assert _canonical("open-pr") == "open-pr" + assert _canonical("plan-1") == "plan" + assert _canonical("") == "" + assert _canonical("implement") == "implement" + + +def test_tsa_humanize_preserves_decimal() -> None: + """_humanize must use str(n) not str(int(n)) for sub-1000 values.""" + from autoskillit.hooks.token_summary_hook import _humanize + + assert _humanize(999) == "999" + assert _humanize(0) == "0" + assert _humanize(None) == "0" + assert _humanize(45.7) == "45.7", ( + "str(int(n)) truncates decimals — must be str(n) to match telemetry_fmt.py" + ) + + +def _make_run_skill_event_with_order_id( + result_text: str = "Done.\n%%ORDER_UP%%", order_id: str = "" +) -> dict: + """Create a double-wrapped PostToolUse event for run_skill with optional order_id.""" + inner: dict = {"result": result_text, "success": True} + if order_id: + inner["order_id"] = order_id + outer = {"result": json.dumps(inner)} + return { + "tool_name": "mcp__autoskillit_server__run_skill", + "tool_response": json.dumps(outer), + } + + +def test_e1_order_id_isolation_multi_issue_session(tmp_path: Path) -> None: + """E-1: Three issues sharing kitchen_id; hook with order_id='B' includes only B's sessions.""" + log_root = tmp_path / "logs" + log_root.mkdir() + kitchen_id = "shared-kitchen" + + (log_root / "sessions.jsonl").write_text( + "\n".join( + json.dumps(e) + for e in [ + { + "dir_name": "sess-A", + "cwd": "/w", + "kitchen_id": kitchen_id, + "order_id": "issue-A", + "step_name": "plan", + }, + { + "dir_name": "sess-B", + "cwd": "/w", + "kitchen_id": kitchen_id, + "order_id": "issue-B", + "step_name": "implement", + }, + { + "dir_name": "sess-C", + "cwd": "/w", + "kitchen_id": kitchen_id, + "order_id": "issue-C", + "step_name": "review", + }, + ] + ) + + "\n" + ) + for dir_name, step in [("sess-A", "plan"), ("sess-B", "implement"), ("sess-C", "review")]: + d = log_root / "sessions" / dir_name + d.mkdir(parents=True) + (d / "token_usage.json").write_text( + json.dumps( + { + "session_label": step, + "input_tokens": 100, + "output_tokens": 50, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + "timing_seconds": 5.0, + "order_id": dir_name.replace("sess-", "issue-"), + } + ) + ) + + pr_url = "https://github.com/owner/repo/pull/42" + event = _make_run_skill_event_with_order_id( + f"pr_url={pr_url}\n%%ORDER_UP%%", order_id="issue-B" + ) + view_result = MagicMock(returncode=0, stdout="Existing PR body.") + edit_calls: list = [] + + def run_side(args, **kwargs): + if "api" in args and "--method" not in args: + return view_result + if "api" in args and "--method" in args: + edit_calls.append(args) + return MagicMock(returncode=0) + return MagicMock(returncode=0) + + hook_config = tmp_path / ".autoskillit_hook_config.json" + hook_config.write_text(json.dumps({"kitchen_id": kitchen_id})) + + with patch("subprocess.run", side_effect=run_side): + _, exit_code = _run_hook(event, log_root=log_root, hook_config_path=hook_config) + + assert exit_code == 0 + assert len(edit_calls) == 1, "gh api PATCH must be called once" + body_arg = edit_calls[0][edit_calls[0].index("--raw-field") + 1] + body_content = body_arg[len("body=") :] + assert "implement" in body_content, "issue-B's step 'implement' must be in table" + assert "plan" not in body_content, "issue-A's step 'plan' must NOT be in table" + assert "review" not in body_content, "issue-C's step 'review' must NOT be in table" + + +def test_e2_fallback_to_kitchen_id_when_no_order_id(tmp_path: Path) -> None: + """E-2: When run_skill result lacks 'order_id', hook falls back to kitchen_id filtering.""" + from tests.infra._token_summary_helpers import _make_run_skill_event + + log_root = tmp_path / "logs" + log_root.mkdir() + kitchen_id = "my-kitchen" + + _write_sessions( + log_root, + [ + { + "dir_name": "sess-1", + "cwd": "/w", + "kitchen_id": kitchen_id, + "step_name": "plan", + "input_tokens": 100, + "output_tokens": 50, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + "timing_seconds": 5.0, + } + ], + ) + + pr_url = "https://github.com/owner/repo/pull/10" + event = _make_run_skill_event(f"pr_url={pr_url}\n%%ORDER_UP%%") + + hook_config = tmp_path / ".autoskillit_hook_config.json" + hook_config.write_text(json.dumps({"kitchen_id": kitchen_id})) + + view_result = MagicMock(returncode=0, stdout="Existing body.") + edit_calls: list = [] + + def run_side(args, **kwargs): + if "api" in args and "--method" not in args: + return view_result + if "api" in args and "--method" in args: + edit_calls.append(args) + return MagicMock(returncode=0) + return MagicMock(returncode=0) + + with patch("subprocess.run", side_effect=run_side): + _, exit_code = _run_hook(event, log_root=log_root, hook_config_path=hook_config) + + assert exit_code == 0 + assert len(edit_calls) == 1, "gh api PATCH must be called when kitchen_id matches (fallback)" + + +def test_e3_backward_compat_sessions_without_order_id_field(tmp_path: Path) -> None: + """E-3: Sessions without 'order_id' key are skipped when order_id filter is active.""" + log_root = tmp_path / "logs" + log_root.mkdir() + kitchen_id = "kitchen-xyz" + + (log_root / "sessions.jsonl").write_text( + json.dumps( + { + "dir_name": "old-sess", + "cwd": "/w", + "kitchen_id": kitchen_id, + "step_name": "plan", + } + ) + + "\n" + ) + d = log_root / "sessions" / "old-sess" + d.mkdir(parents=True) + (d / "token_usage.json").write_text( + json.dumps( + { + "session_label": "plan", + "input_tokens": 100, + "output_tokens": 50, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + "timing_seconds": 5.0, + } + ) + ) + + pr_url = "https://github.com/owner/repo/pull/20" + event = _make_run_skill_event_with_order_id( + f"pr_url={pr_url}\n%%ORDER_UP%%", order_id="issue-X" + ) + + hook_config = tmp_path / ".autoskillit_hook_config.json" + hook_config.write_text(json.dumps({"kitchen_id": kitchen_id})) + + with patch("subprocess.run") as mock_run: + _, exit_code = _run_hook(event, log_root=log_root, hook_config_path=hook_config) + + assert exit_code == 0 + patch_calls = [ + call for call in mock_run.call_args_list if "--method" in (call[0][0] if call[0] else []) + ] + assert len(patch_calls) == 0, "Old sessions without order_id should be skipped" + + +def test_e4_kitchen_id_renamed_in_hook_config(tmp_path: Path) -> None: + """E-4: _read_kitchen_id reads 'kitchen_id' key; falls back to 'pipeline_id' for old.""" + from autoskillit.hooks.token_summary_hook import _read_kitchen_id + + cfg_path = tmp_path.joinpath(*_HOOK_CONFIG_PATH_COMPONENTS) + cfg_path.parent.mkdir(parents=True) + cfg_path.write_text(json.dumps({"kitchen_id": "new-kitchen-uuid"})) + + result = _read_kitchen_id(base=tmp_path) + assert result == "new-kitchen-uuid" + + cfg_path.write_text(json.dumps({"pipeline_id": "legacy-pipeline-uuid"})) + result = _read_kitchen_id(base=tmp_path) + assert result == "legacy-pipeline-uuid" + + +class TestUnwrapMcpResponse: + """Unit tests for the shared double-unwrap helper.""" + + def _call(self, tool_name: str, raw: str): + from autoskillit.hooks.token_summary_hook import _unwrap_mcp_response + + return _unwrap_mcp_response(tool_name, raw) + + def test_invalid_json_returns_none(self): + assert self._call("mcp__x__y", "not json") is None + + def test_json_array_returns_none(self): + assert self._call("mcp__x__y", json.dumps([1, 2, 3])) is None + + def test_non_mcp_tool_returns_outer_dict(self): + payload = {"result": "some text", "success": True} + result = self._call("run_python", json.dumps(payload)) + assert result == payload + + def test_mcp_tool_double_wrapped_returns_inner(self): + inner = {"result": "https://github.com/o/r/pull/1", "order_id": "abc"} + outer = {"result": json.dumps(inner)} + result = self._call("mcp__autoskillit__run_skill", json.dumps(outer)) + assert result == inner + + def test_mcp_tool_inner_parse_fails_returns_outer(self): + outer = {"result": "not-json-inner"} + result = self._call("mcp__autoskillit__run_skill", json.dumps(outer)) + assert result == outer + + def test_mcp_tool_inner_not_dict_returns_outer(self): + outer = {"result": json.dumps([1, 2, 3])} + result = self._call("mcp__autoskillit__run_skill", json.dumps(outer)) + assert result == outer + + def test_mcp_tool_extra_keys_skips_double_unwrap(self): + """Outer dict with more than just 'result' key → not a double-wrapped response.""" + outer = {"result": json.dumps({"foo": "bar"}), "extra": "key"} + result = self._call("mcp__autoskillit__run_skill", json.dumps(outer)) + assert result == outer + + +def test_hook_subprocess_calls_have_timeout() -> None: + """All subprocess.run() calls in token_summary_hook.py must have timeout=.""" + src = (pkg_root() / "hooks" / "token_summary_hook.py").read_text() + tree = ast.parse(src) + for node in ast.walk(tree): + if ( + isinstance(node, ast.Call) + and isinstance(node.func, ast.Attribute) + and node.func.attr == "run" + ): + kw_names = {kw.arg for kw in node.keywords} + assert "timeout" in kw_names, ( + f"subprocess.run() at line {node.lineno} in token_summary_hook.py missing timeout=" + ) diff --git a/tests/infra/test_unsafe_install_guard.py b/tests/infra/test_unsafe_install_guard.py index a2990a22fe..5f598e9533 100644 --- a/tests/infra/test_unsafe_install_guard.py +++ b/tests/infra/test_unsafe_install_guard.py @@ -8,7 +8,7 @@ def _run_guard(cmd: str, raw_stdin: str | None = None) -> str: """Run the guard's main() in-process and return captured stdout.""" - from autoskillit.hooks.unsafe_install_guard import main + from autoskillit.hooks.guards.unsafe_install_guard import main tool_input = {"cmd": cmd, "cwd": "/some/path"} stdin_content = ( @@ -31,6 +31,31 @@ def _run_guard(cmd: str, raw_stdin: str | None = None) -> str: return buf.getvalue() +def _run_bash_guard(cmd: str, raw_stdin: str | None = None) -> str: + """Run the guard's main() in-process with Bash tool format, return captured stdout.""" + from autoskillit.hooks.guards.unsafe_install_guard import main + + tool_input = {"command": cmd, "cwd": "/some/path"} + stdin_content = ( + raw_stdin + if raw_stdin is not None + else json.dumps( + { + "tool_name": "Bash", + "tool_input": tool_input, + } + ) + ) + buf = io.StringIO() + with patch("sys.stdin", io.StringIO(stdin_content)): + with redirect_stdout(buf): + try: + main() + except SystemExit: + pass + return buf.getvalue() + + def _is_denied(output: str) -> bool: if not output: return False @@ -38,6 +63,51 @@ def _is_denied(output: str) -> bool: return data.get("hookSpecificOutput", {}).get("permissionDecision") == "deny" +class TestBashToolDenyPath: + """Bash tool sends command via 'command' key — guard must handle this format.""" + + def test_pip_install_editable_via_bash_tool(self): + assert _is_denied(_run_bash_guard("pip install -e .")) + + def test_uv_pip_install_editable_via_bash_tool(self): + assert _is_denied(_run_bash_guard("uv pip install -e .")) + + def test_pip_install_editable_long_flag_via_bash_tool(self): + assert _is_denied(_run_bash_guard("pip install --editable .")) + + def test_uv_pip_install_editable_long_flag_via_bash_tool(self): + assert _is_denied(_run_bash_guard("uv pip install --editable .")) + + +class TestMaturinDevelopDenied: + """maturin develop performs editable installs and must be blocked.""" + + def test_maturin_develop_denied(self): + assert _is_denied(_run_guard("maturin develop")) + + def test_maturin_develop_with_args_denied(self): + assert _is_denied(_run_guard("maturin develop --release")) + + def test_maturin_develop_via_bash_tool(self): + assert _is_denied(_run_bash_guard("maturin develop")) + + +class TestSystemFlagDenied: + """--system flag installs into global environment — must be blocked for pip/uv.""" + + def test_uv_pip_install_system_flag_denied(self): + assert _is_denied(_run_guard("uv pip install foo --system")) + + def test_pip_install_system_flag_denied(self): + assert _is_denied(_run_guard("pip install foo --system")) + + def test_uv_pip_install_editable_with_system_denied(self): + assert _is_denied(_run_guard("uv pip install -e . --system")) + + def test_system_flag_via_bash_tool(self): + assert _is_denied(_run_bash_guard("uv pip install foo --system")) + + class TestUnsafeInstallGuardDenied: """Commands that should be blocked.""" diff --git a/tests/integration/__init__.py b/tests/integration/__init__.py new file mode 100644 index 0000000000..b3bc429983 --- /dev/null +++ b/tests/integration/__init__.py @@ -0,0 +1 @@ +"""Integration test suite for autoskillit.""" diff --git a/tests/integration/test_silent_type_flows.py b/tests/integration/test_silent_type_flows.py new file mode 100644 index 0000000000..eda2998535 --- /dev/null +++ b/tests/integration/test_silent_type_flows.py @@ -0,0 +1,50 @@ +"""Integration tests for silent-type handling across review-design and vis-lens. + +Shared between Work Items 2.3 (#835) and 4.7 (#846). +""" + +from pathlib import Path + +import pytest + +pytestmark = [pytest.mark.small] + +REPO_ROOT = Path(__file__).resolve().parents[2] +CONVENTION_PATH = REPO_ROOT / "docs" / "research" / "silent-type-convention.md" +REVIEW_DESIGN_SKILL = ( + REPO_ROOT / "src" / "autoskillit" / "skills_extended" / "review-design" / "SKILL.md" +) + +ADVISORY_REQUIRED_KEYS = {"verdict", "advisory_context", "requires_decision"} +ADVISORY_CONTEXT_KEYS = {"subject_kind", "subject_name", "reasoning", "reference_framework"} + + +def test_silent_type_advisory_schema_matches_convention() -> None: + convention = CONVENTION_PATH.read_text() + for key in ADVISORY_REQUIRED_KEYS: + assert key in convention, f"Convention doc missing '{key}'" + for key in ADVISORY_CONTEXT_KEYS: + assert key in convention, f"Convention doc missing '{key}'" + + +def test_review_design_advisory_write_target() -> None: + convention = CONVENTION_PATH.read_text() + assert "design-review-dashboard.md" in convention + + +def test_is_silent_type_aligns_with_convention_threshold() -> None: + convention = CONVENTION_PATH.read_text() + assert "6 of 8" in convention or "≥6 of 8" in convention + + +def test_review_design_skill_references_silent_type_handling() -> None: + content = REVIEW_DESIGN_SKILL.read_text() + assert "Silent Type Handling" in content + assert "is_silent_type" in content + assert "advisory_context" in content + + +def test_advisory_schema_in_review_design_skill() -> None: + content = REVIEW_DESIGN_SKILL.read_text() + assert "subject_kind: experiment_type" in content + assert "requires_decision: false" in content diff --git a/tests/migration/CLAUDE.md b/tests/migration/CLAUDE.md new file mode 100644 index 0000000000..7bea21fd20 --- /dev/null +++ b/tests/migration/CLAUDE.md @@ -0,0 +1,16 @@ +# migration/ + +Migration engine, store, and loader tests. + +## Files + +| File | Purpose | +|------|---------| +| `__init__.py` | empty | +| `test_api.py` | Tests for migration/_api.py | +| `test_api_integration.py` | Integration tests for migration/_api.py — no mocking of recipe lookup or engine | +| `test_engine.py` | Tests for migration_engine.py — ME1 through ME21 | +| `test_fleet_migration_note.py` | Fleet migration note tests | +| `test_loader.py` | Tests for migration note discovery and version chaining | +| `test_store.py` | Tests for failure_store.py — FS1 through FS11 and FS-IM1 through FS-IM4 | +| `test_vis_lens_rename_migration_note.py` | Vis-lens skill rename migration note tests | diff --git a/tests/migration/test_api.py b/tests/migration/test_api.py index a04e23d74d..e575158a4c 100644 --- a/tests/migration/test_api.py +++ b/tests/migration/test_api.py @@ -10,6 +10,8 @@ from autoskillit.migration._api import check_and_migrate from autoskillit.migration.engine import MigrationResult +pytestmark = [pytest.mark.layer("migration"), pytest.mark.small] + # --------------------------------------------------------------------------- # T6 — migration/_api.py recipe imports are deferred # --------------------------------------------------------------------------- diff --git a/tests/migration/test_api_integration.py b/tests/migration/test_api_integration.py new file mode 100644 index 0000000000..04d4827594 --- /dev/null +++ b/tests/migration/test_api_integration.py @@ -0,0 +1,26 @@ +"""Integration tests for migration/_api.py — no mocking of recipe lookup or engine.""" + +from __future__ import annotations + +import pytest + +from autoskillit.migration import check_and_migrate + +pytestmark = [pytest.mark.layer("migration"), pytest.mark.medium] + + +@pytest.mark.anyio +async def test_check_and_migrate_integration_known_recipe_up_to_date() -> None: + """Integration: no mocks — check_and_migrate with real bundled recipe returns up_to_date. + + Uses the real find_recipe_by_name and applicable_migrations. At the current installed + version, no migrations are pending for a bundled recipe, so the engine is never entered. + """ + import autoskillit + from autoskillit.core import pkg_root + + result = await check_and_migrate("implementation", pkg_root(), autoskillit.__version__) + + assert result.get("status") == "up_to_date" + assert result.get("name") == "implementation" + assert "error" not in result diff --git a/tests/migration/test_engine.py b/tests/migration/test_engine.py index d4df822a5d..d7b79095e2 100644 --- a/tests/migration/test_engine.py +++ b/tests/migration/test_engine.py @@ -13,6 +13,8 @@ from autoskillit.execution.session import SkillResult from autoskillit.migration.engine import ( MIGRATE_RECIPES_MAX_RETRIES, + AdvisoryMigrationAdapter, + AdvisoryResult, ContractMigrationAdapter, DeterministicMigrationAdapter, DiagramMigrationAdapter, @@ -24,6 +26,8 @@ ) from autoskillit.migration.loader import MigrationChange, MigrationNote +pytestmark = [pytest.mark.layer("migration"), pytest.mark.small] + PROJECT_ROOT = Path(__file__).resolve().parent.parent.parent # --------------------------------------------------------------------------- @@ -463,12 +467,6 @@ async def test_engine_returns_failure_when_temp_output_missing( assert result.error is not None assert "output" in result.error.lower() - # ME21 - def test_default_engine_has_both_adapters(self) -> None: - engine = default_migration_engine() - assert engine.get_adapter("recipe") is not None - assert engine.get_adapter("contract") is not None - class TestAdapterHierarchy: # ME-ADP1 @@ -503,6 +501,11 @@ class BrokenAdapter(HeadlessMigrationAdapter): with pytest.raises(TypeError): BrokenAdapter() + # ME-ADP5 + def test_diagram_adapter_is_advisory_not_deterministic(self) -> None: + assert isinstance(DiagramMigrationAdapter(), AdvisoryMigrationAdapter) + assert not isinstance(DiagramMigrationAdapter(), DeterministicMigrationAdapter) + # --------------------------------------------------------------------------- # DiagramMigrationAdapter tests (DG-16 through DG-20) @@ -584,29 +587,20 @@ def test_diagram_adapter_needs_migration_stale( ) assert DiagramMigrationAdapter().needs_migration(file) is True - # DG-19 - @pytest.mark.anyio - async def test_diagram_adapter_migrate_writes_file( - self, tmp_path: Path, sample_recipe_yaml_for_diagram: Path - ) -> None: - """DG-19: DiagramMigrationAdapter.migrate() writes diagram file.""" - import shutil - - recipes_dir = tmp_path / ".autoskillit" / "recipes" - diagrams_dir = recipes_dir / "diagrams" - diagrams_dir.mkdir(parents=True) - shutil.copy2(sample_recipe_yaml_for_diagram, recipes_dir / "my-recipe.yaml") - + # DG-19 (replaced: advisory path instead of destructive migrate) + def test_diagram_adapter_check_staleness_returns_advisory(self) -> None: + """DG-19: DiagramMigrationAdapter.check_staleness() returns AdvisoryResult.""" file = MigrationFile( name="my-recipe", - path=diagrams_dir / "my-recipe.md", + path=Path("/fake/diagrams/my-recipe.md"), file_type="diagram", current_version=None, ) - result = await DiagramMigrationAdapter().migrate(file, temp_dir=tmp_path / "temp") - assert result.success is True + result = DiagramMigrationAdapter().check_staleness(file) + assert isinstance(result, AdvisoryResult) + assert "/render-recipe" in result.suggestion + assert "my-recipe" in result.suggestion - # DG-20 def test_diagram_adapter_validate_passes_when_hash_present(self, tmp_path: Path) -> None: """DG-20: DiagramMigrationAdapter.validate() passes when hash comment present.""" md = tmp_path / "test.md" @@ -631,10 +625,47 @@ def test_default_engine_includes_diagram_adapter(self) -> None: assert isinstance(engine.get_adapter("diagram"), DiagramMigrationAdapter) -class TestMigrateRecipesConstant: - def test_constant_value(self) -> None: - assert MIGRATE_RECIPES_MAX_RETRIES == 3 +def test_diagram_adapter_type_is_not_deterministic() -> None: + """T-ADAPTER-TYPE: DiagramMigrationAdapter must NOT be a DeterministicMigrationAdapter.""" + assert not isinstance(DiagramMigrationAdapter(), DeterministicMigrationAdapter) + +@pytest.mark.anyio +async def test_advisory_dispatch_does_not_write_file(tmp_path: Path) -> None: + """T-ADVISORY-DISPATCH: MigrationEngine returns advisory for stale diagrams without writing.""" + recipes_dir = tmp_path / ".autoskillit" / "recipes" + diagrams_dir = recipes_dir / "diagrams" + diagrams_dir.mkdir(parents=True) + recipe_yaml = recipes_dir / "my-recipe.yaml" + recipe_yaml.write_text(_SAMPLE_RECIPE_YAML_FOR_DIAG) + + original_content = ( + "\n## my-recipe\nASCII art here\n" + ) + diagram_md = diagrams_dir / "my-recipe.md" + diagram_md.write_text(original_content) + + file = MigrationFile( + name="my-recipe", + path=diagram_md, + file_type="diagram", + current_version=None, + ) + from autoskillit.migration.engine import MigrationEngine + + engine = MigrationEngine([DiagramMigrationAdapter()]) + result = await engine.migrate_file( + file, + run_headless=AsyncMock(), + temp_dir=tmp_path / "temp", + ) + assert result.success is True + assert result.advisory is not None + assert "/render-recipe" in result.advisory + assert diagram_md.read_text() == original_content + + +class TestMigrateRecipesConstant: @pytest.mark.anyio async def test_failed_headless_retries_match_constant( self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch diff --git a/tests/migration/test_fleet_migration_note.py b/tests/migration/test_fleet_migration_note.py new file mode 100644 index 0000000000..a2aa7f6de6 --- /dev/null +++ b/tests/migration/test_fleet_migration_note.py @@ -0,0 +1,44 @@ +from pathlib import Path + +import pytest +import yaml + +pytestmark = [pytest.mark.layer("migration"), pytest.mark.small] + +MIGRATIONS_DIR = Path(__file__).parents[2] / "src" / "autoskillit" / "migrations" + + +def test_fleet_rename_migration_note_exists(): + """A migration note must cover the features.franchise → features.fleet rename.""" + notes = list(MIGRATIONS_DIR.glob("*.yaml")) + detect_keys = [] + for note_path in notes: + try: + data = yaml.safe_load(note_path.read_text()) + except yaml.YAMLError as exc: + pytest.fail(f"Malformed YAML in {note_path.name}: {exc}") + for change in data.get("changes", []): + detect = change.get("detect", {}) + if detect.get("key") == "features.franchise": + detect_keys.append(note_path.name) + assert detect_keys, ( + "No migration note found with detect.key == 'features.franchise'. " + "Create src/autoskillit/migrations/0.9.134-to-0.9.135.yaml." + ) + + +def test_fleet_rename_migration_note_is_valid(): + """Migration note for fleet rename must have correct required fields.""" + notes = list(MIGRATIONS_DIR.glob("*.yaml")) + for note_path in notes: + try: + data = yaml.safe_load(note_path.read_text()) + except yaml.YAMLError as exc: + pytest.fail(f"Malformed YAML in {note_path.name}: {exc}") + for change in data.get("changes", []): + if change.get("detect", {}).get("key") == "features.franchise": + assert "instruction" in change, "Migration note missing 'instruction'" + assert "example_before" in change, "Migration note missing 'example_before'" + assert "example_after" in change, "Migration note missing 'example_after'" + return + pytest.fail("No matching migration change found") diff --git a/tests/migration/test_loader.py b/tests/migration/test_loader.py index 923b8d958b..8ce3c0fb27 100644 --- a/tests/migration/test_loader.py +++ b/tests/migration/test_loader.py @@ -2,6 +2,7 @@ from __future__ import annotations +import os from pathlib import Path import pytest @@ -13,6 +14,8 @@ list_migrations, ) +pytestmark = [pytest.mark.layer("migration"), pytest.mark.small] + # --------------------------------------------------------------------------- # Helpers # --------------------------------------------------------------------------- @@ -258,10 +261,10 @@ def test_script_at_to_version_does_not_match( assert result == [] # ML9 - def test_none_version_matches_from_zero( + def test_none_version_not_migratable( self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: - """Scripts with no version (None) match from_version="0.0.0".""" + """Recipes with no version (None) are not subject to migration.""" self._setup_migrations( tmp_path, monkeypatch, @@ -274,8 +277,7 @@ def test_none_version_matches_from_zero( ], ) result = applicable_migrations(None, "0.1.0") - assert len(result) == 1 - assert result[0].from_version == "0.0.0" + assert result == [] # ML10 def test_script_mid_range_matches_enclosing_migration( @@ -342,24 +344,6 @@ def test_gap_in_chain_returns_empty( result = applicable_migrations("0.1.0", "0.3.0") assert result == [] - def test_already_at_installed_version_returns_empty( - self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch - ) -> None: - """Script already at installed_version returns empty list.""" - self._setup_migrations( - tmp_path, - monkeypatch, - [ - { - "from_version": "0.1.0", - "to_version": "0.2.0", - "description": "Never needed", - } - ], - ) - result = applicable_migrations("0.2.0", "0.2.0") - assert result == [] - def test_no_migrations_available_returns_empty( self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: @@ -386,3 +370,45 @@ def test_none_version_with_no_matching_migration_returns_empty( ) result = applicable_migrations(None, "0.2.0") assert result == [] + + +# --------------------------------------------------------------------------- +# ML13: Caching tests +# --------------------------------------------------------------------------- + + +class TestListMigrationsCaching: + def test_result_is_cached_on_repeat_call( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + mig_dir = _make_migrations_dir(tmp_path) + _write_migration( + mig_dir / "001_init.yaml", + {"from_version": "0.1.0", "to_version": "0.2.0", "changes": []}, + ) + monkeypatch.setattr("autoskillit.migration.loader._migrations_dir", lambda: mig_dir) + r1 = list_migrations() + r2 = list_migrations() + assert r1 == r2 + assert r1 is not r2 + + def test_mtime_change_invalidates_cache( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + mig_dir = _make_migrations_dir(tmp_path) + _write_migration( + mig_dir / "001_init.yaml", + {"from_version": "0.1.0", "to_version": "0.2.0", "changes": []}, + ) + monkeypatch.setattr("autoskillit.migration.loader._migrations_dir", lambda: mig_dir) + r1 = list_migrations() + + _write_migration( + mig_dir / "002_add.yaml", + {"from_version": "0.2.0", "to_version": "0.3.0", "changes": []}, + ) + old_mt = mig_dir.stat().st_mtime + os.utime(mig_dir, (old_mt + 2, old_mt + 2)) + r2 = list_migrations() + assert r1 is not r2 + assert len(r2) == len(r1) + 1 diff --git a/tests/migration/test_store.py b/tests/migration/test_store.py index b45dfee0b3..0642f0273c 100644 --- a/tests/migration/test_store.py +++ b/tests/migration/test_store.py @@ -15,6 +15,8 @@ record_from_skill, ) +pytestmark = [pytest.mark.layer("migration"), pytest.mark.small] + # --------------------------------------------------------------------------- # FS1: load() returns {} when failures.json absent diff --git a/tests/migration/test_vis_lens_rename_migration_note.py b/tests/migration/test_vis_lens_rename_migration_note.py new file mode 100644 index 0000000000..da2657b4c2 --- /dev/null +++ b/tests/migration/test_vis_lens_rename_migration_note.py @@ -0,0 +1,50 @@ +from pathlib import Path + +import pytest +import yaml + +pytestmark = [pytest.mark.layer("migration"), pytest.mark.small] + +MIGRATIONS_DIR = Path(__file__).parents[2] / "src" / "autoskillit" / "migrations" + + +def test_vis_lens_rename_migration_note_exists(): + """Migration note covers vis-lens-domain-norms → vis-lens-methodology-norms rename.""" + notes = list(MIGRATIONS_DIR.glob("*.yaml")) + matches = [] + for note_path in notes: + try: + data = yaml.safe_load(note_path.read_text()) + except yaml.YAMLError as exc: + pytest.fail(f"Malformed YAML in {note_path.name}: {exc}") + for change in data.get("changes", []): + detect = change.get("detect", {}) + if detect.get("skill_pattern") == "vis-lens-domain-norms": + matches.append(note_path.name) + assert matches, ( + "No migration note found with detect.skill_pattern == 'vis-lens-domain-norms'. " + "Create src/autoskillit/migrations/-to-.yaml with a " + "vis-lens-methodology-norms-rename change entry." + ) + + +def test_vis_lens_rename_migration_note_is_valid(): + """Migration note for vis-lens rename must have instruction and examples.""" + notes = list(MIGRATIONS_DIR.glob("*.yaml")) + for note_path in notes: + try: + data = yaml.safe_load(note_path.read_text()) + except yaml.YAMLError as exc: + pytest.fail(f"Malformed YAML in {note_path.name}: {exc}") + for change in data.get("changes", []): + if change.get("detect", {}).get("skill_pattern") == "vis-lens-domain-norms": + assert "instruction" in change, "Migration note missing 'instruction'" + assert change["instruction"].strip(), "Migration note 'instruction' is empty" + assert "example_before" in change, "Migration note missing 'example_before'" + assert change["example_before"].strip(), "Migration note 'example_before' is empty" + assert "example_after" in change, "Migration note missing 'example_after'" + assert change["example_after"].strip(), "Migration note 'example_after' is empty" + return + pytest.fail( + "No matching migration change found with detect.skill_pattern == 'vis-lens-domain-norms'" + ) diff --git a/tests/pipeline/CLAUDE.md b/tests/pipeline/CLAUDE.md new file mode 100644 index 0000000000..0a79a8a572 --- /dev/null +++ b/tests/pipeline/CLAUDE.md @@ -0,0 +1,21 @@ +# pipeline/ + +Audit log, gate state, token tracking, and PR-gate tests. + +## Files + +| File | Purpose | +|------|---------| +| `__init__.py` | empty | +| `test_audit.py` | Tests for autoskillit.pipeline.audit — pipeline failure tracking | +| `test_background_supervisor.py` | Unit tests for DefaultBackgroundSupervisor | +| `test_context.py` | Tests for ToolContext dependency injection container | +| `test_gate.py` | Unit tests for _gate.py constants and functions | +| `test_github_api_log.py` | GitHub API log tests | +| `test_mcp_response.py` | Tests for autoskillit.pipeline.mcp_response — MCP tool response size tracking | +| `test_pr_gates.py` | Tests for analyze-prs PR eligibility gate logic (CI gate + review gate) | +| `test_telemetry_formatter.py` | Tests for TelemetryFormatter — canonical telemetry formatting | +| `test_timings.py` | Tests for autoskillit.pipeline.timings — pipeline step timing | +| `test_tokens_core.py` | Tests for pipeline.tokens — TokenEntry, DefaultTokenLog core, and log-dir loading | +| `test_tokens_filters.py` | Tests for pipeline.tokens — cwd filter, step name normalization, order/campaign ID scoping | +| `test_tokens_model.py` | Tests for pipeline.tokens — model field capture, compute_model_totals, and load_from_log_dir model reading | diff --git a/tests/pipeline/test_audit.py b/tests/pipeline/test_audit.py index c8f5bf3fac..e98f431d1d 100644 --- a/tests/pipeline/test_audit.py +++ b/tests/pipeline/test_audit.py @@ -5,12 +5,33 @@ import json from pathlib import Path +import pytest + from autoskillit.pipeline.audit import ( DefaultAuditLog, FailureRecord, _validate_failure_record_dict, ) +pytestmark = [pytest.mark.layer("pipeline"), pytest.mark.small] + + +def _valid_failure_record_dict(**overrides: object) -> dict: + """Module-level factory for valid failure record dicts. + + Used by TestValidateFailureRecordDict and TestLoadFromLogDirTypeValidation. + """ + base: dict = { + "timestamp": "2026-03-28T00:00:00Z", + "skill_command": "/autoskillit:implement-worktree", + "exit_code": 1, + "subtype": "error", + "needs_retry": False, + "retry_reason": "none", + "stderr": "oops", + } + return {**base, **overrides} + def _make_record(**overrides: object) -> FailureRecord: defaults = dict( @@ -31,11 +52,28 @@ def test_to_dict_is_json_serializable(self): d = record.to_dict() assert json.loads(json.dumps(d)) == d - def test_to_dict_contains_all_fields(self): - record = _make_record(skill_command="/test:cmd", exit_code=42) + def test_to_dict_contains_all_fields(self) -> None: + record = _make_record( + timestamp="2026-01-01T00:00:00", + skill_command="/foo", + exit_code=1, + subtype="timeout", + needs_retry=False, + retry_reason="", + stderr="err", + ) d = record.to_dict() - assert d["skill_command"] == "/test:cmd" - assert d["exit_code"] == 42 + assert set(d.keys()) == { + "timestamp", + "skill_command", + "exit_code", + "subtype", + "needs_retry", + "retry_reason", + "stderr", + } + assert d["skill_command"] == "/foo" + assert d["exit_code"] == 1 class TestDefaultAuditLog: @@ -414,62 +452,42 @@ def test_iter_session_log_entries_kitchen_id_backward_compat(tmp_path): class TestValidateFailureRecordDict: - def _valid_dict(self, **overrides) -> dict: - base = { - "timestamp": "2026-03-28T00:00:00Z", - "skill_command": "/autoskillit:implement-worktree", - "exit_code": 1, - "subtype": "error", - "needs_retry": False, - "retry_reason": "none", - "stderr": "oops", - } - return {**base, **overrides} - def test_valid_dict_returns_true(self): - assert _validate_failure_record_dict(self._valid_dict()) is True + assert _validate_failure_record_dict(_valid_failure_record_dict()) is True def test_missing_key_returns_false(self): - d = self._valid_dict() + d = _valid_failure_record_dict() del d["stderr"] assert _validate_failure_record_dict(d) is False def test_wrong_type_exit_code_returns_false(self): - assert _validate_failure_record_dict(self._valid_dict(exit_code="bad")) is False + assert _validate_failure_record_dict(_valid_failure_record_dict(exit_code="bad")) is False def test_wrong_type_needs_retry_returns_false(self): # "true" is a str, not bool - assert _validate_failure_record_dict(self._valid_dict(needs_retry="true")) is False + assert ( + _validate_failure_record_dict(_valid_failure_record_dict(needs_retry="true")) is False + ) def test_wrong_type_timestamp_returns_false(self): - assert _validate_failure_record_dict(self._valid_dict(timestamp=12345)) is False + assert _validate_failure_record_dict(_valid_failure_record_dict(timestamp=12345)) is False def test_int_for_bool_field_returns_false(self): # 0 and 1 are int, not bool — must be rejected for needs_retry: bool - assert _validate_failure_record_dict(self._valid_dict(needs_retry=1)) is False + assert _validate_failure_record_dict(_valid_failure_record_dict(needs_retry=1)) is False def test_extra_keys_are_ignored(self): - d = self._valid_dict() + d = _valid_failure_record_dict() d["extra_unexpected_key"] = "ignored" assert _validate_failure_record_dict(d) is True class TestLoadFromLogDirTypeValidation: - def _valid_record(self, **overrides) -> dict: - base = { - "timestamp": "2026-03-28T00:00:00Z", - "skill_command": "/autoskillit:implement-worktree", - "exit_code": 1, - "subtype": "error", - "needs_retry": False, - "retry_reason": "none", - "stderr": "oops", - } - return {**base, **overrides} - def test_wrong_type_exit_code_is_skipped(self, tmp_path): """record_dict with exit_code as str is skipped, not silently accepted.""" - _write_audit_session(tmp_path, "s001", [self._valid_record(exit_code="not-an-int")]) + _write_audit_session( + tmp_path, "s001", [_valid_failure_record_dict(exit_code="not-an-int")] + ) log = DefaultAuditLog() n = log.load_from_log_dir(tmp_path) assert n == 0 @@ -477,14 +495,14 @@ def test_wrong_type_exit_code_is_skipped(self, tmp_path): def test_wrong_type_needs_retry_is_skipped(self, tmp_path): """record_dict with needs_retry as str is skipped.""" - _write_audit_session(tmp_path, "s001", [self._valid_record(needs_retry="true")]) + _write_audit_session(tmp_path, "s001", [_valid_failure_record_dict(needs_retry="true")]) log = DefaultAuditLog() n = log.load_from_log_dir(tmp_path) assert n == 0 def test_missing_field_is_skipped(self, tmp_path): """record_dict missing a required field is skipped.""" - bad = self._valid_record() + bad = _valid_failure_record_dict() del bad["retry_reason"] _write_audit_session(tmp_path, "s001", [bad]) log = DefaultAuditLog() @@ -494,11 +512,321 @@ def test_missing_field_is_skipped(self, tmp_path): def test_valid_record_alongside_invalid_is_preserved(self, tmp_path): """A valid record in the same session file is loaded despite invalid siblings.""" records = [ - self._valid_record(exit_code="bad"), # skipped - self._valid_record(skill_command="/ok", exit_code=2), # kept + _valid_failure_record_dict(exit_code="bad"), # skipped + _valid_failure_record_dict(skill_command="/ok", exit_code=2), # kept ] _write_audit_session(tmp_path, "s001", records) log = DefaultAuditLog() n = log.load_from_log_dir(tmp_path) assert n == 1 assert log.get_report()[0].skill_command == "/ok" + + +# --- Group N: campaign_id_filter tests --- + + +def _write_audit_session_cid( + log_root: Path, + dir_name: str, + records: list, + campaign_id: str = "", + kitchen_id: str = "", + timestamp: str = "2026-04-20T00:00:00+00:00", +) -> None: + """Write audit session with campaign_id in the index entry.""" + session_dir = log_root / "sessions" / dir_name + session_dir.mkdir(parents=True, exist_ok=True) + (session_dir / "audit_log.json").write_text(json.dumps(records)) + index_entry = { + "dir_name": dir_name, + "timestamp": timestamp, + "campaign_id": campaign_id, + "kitchen_id": kitchen_id, + } + with (log_root / "sessions.jsonl").open("a") as f: + f.write(json.dumps(index_entry) + "\n") + + +def test_iter_entries_campaign_id_filter_matches(tmp_path): + """Only entries with matching campaign_id are yielded.""" + from autoskillit.pipeline.audit import _iter_session_log_entries + + _write_audit_session_cid(tmp_path, "c1-a", [_valid_failure_record_dict()], campaign_id="c1") + _write_audit_session_cid(tmp_path, "c1-b", [_valid_failure_record_dict()], campaign_id="c1") + _write_audit_session_cid(tmp_path, "c2-a", [_valid_failure_record_dict()], campaign_id="c2") + + results = list( + _iter_session_log_entries( + tmp_path, since="", filename="audit_log.json", campaign_id_filter="c1" + ) + ) + assert len(results) == 2 + dir_names = {p.parent.name for p in results} + assert dir_names == {"c1-a", "c1-b"} + + +def test_iter_entries_campaign_id_filter_empty_passes_all(tmp_path): + """Empty campaign_id_filter passes all entries.""" + from autoskillit.pipeline.audit import _iter_session_log_entries + + _write_audit_session_cid(tmp_path, "c1-a", [_valid_failure_record_dict()], campaign_id="c1") + _write_audit_session_cid(tmp_path, "c2-a", [_valid_failure_record_dict()], campaign_id="c2") + _write_audit_session_cid(tmp_path, "no-cid", [_valid_failure_record_dict()], campaign_id="") + + results = list( + _iter_session_log_entries( + tmp_path, since="", filename="audit_log.json", campaign_id_filter="" + ) + ) + assert len(results) == 3 + + +def test_iter_entries_campaign_id_filter_combined_with_kitchen_id(tmp_path): + """Both campaign_id_filter and kitchen_id_filter applied as AND logic.""" + from autoskillit.pipeline.audit import _iter_session_log_entries + + _write_audit_session_cid( + tmp_path, "match", [_valid_failure_record_dict()], campaign_id="c1", kitchen_id="k1" + ) + _write_audit_session_cid( + tmp_path, "wrong-cid", [_valid_failure_record_dict()], campaign_id="c2", kitchen_id="k1" + ) + _write_audit_session_cid( + tmp_path, "wrong-kid", [_valid_failure_record_dict()], campaign_id="c1", kitchen_id="k2" + ) + + results = list( + _iter_session_log_entries( + tmp_path, + since="", + filename="audit_log.json", + kitchen_id_filter="k1", + campaign_id_filter="c1", + ) + ) + assert len(results) == 1 + assert results[0].parent.name == "match" + + +def test_audit_load_campaign_id_filter(tmp_path): + """DefaultAuditLog.load_from_log_dir respects campaign_id_filter.""" + _write_audit_session_cid( + tmp_path, "c1-a", [_valid_failure_record_dict(skill_command="/camp1")], campaign_id="c1" + ) + _write_audit_session_cid( + tmp_path, "c2-a", [_valid_failure_record_dict(skill_command="/camp2")], campaign_id="c2" + ) + + log = DefaultAuditLog() + n = log.load_from_log_dir(tmp_path, campaign_id_filter="c1") + assert n == 1 + assert log.get_report()[0].skill_command == "/camp1" + + +def _write_audit_session_order_id( + log_root: Path, + dir_name: str, + records: list, + order_id: str = "", + timestamp: str = "2026-05-01T00:00:00+00:00", +) -> None: + """Write audit session with order_id in the index entry.""" + from pathlib import Path as _Path + + session_dir = _Path(log_root) / "sessions" / dir_name + session_dir.mkdir(parents=True, exist_ok=True) + (session_dir / "audit_log.json").write_text(json.dumps(records)) + index_entry = { + "dir_name": dir_name, + "timestamp": timestamp, + "session_id": dir_name, + "order_id": order_id, + } + with (_Path(log_root) / "sessions.jsonl").open("a") as f: + f.write(json.dumps(index_entry) + "\n") + + +def test_iter_session_log_entries_order_id_filter(tmp_path): + """order_id_filter selects only sessions with matching order_id.""" + from autoskillit.pipeline.audit import _iter_session_log_entries + + _write_audit_session_order_id( + tmp_path, "oid-a1", [_valid_failure_record_dict()], order_id="abc" + ) + _write_audit_session_order_id( + tmp_path, "oid-a2", [_valid_failure_record_dict()], order_id="abc" + ) + _write_audit_session_order_id( + tmp_path, "oid-xyz", [_valid_failure_record_dict()], order_id="xyz" + ) + + results = list( + _iter_session_log_entries( + tmp_path, since="", filename="audit_log.json", order_id_filter="abc" + ) + ) + assert len(results) == 2 + dir_names = {p.parent.name for p in results} + assert dir_names == {"oid-a1", "oid-a2"} + + +def _write_audit_session_dispatch_id( + log_root: Path, + dir_name: str, + records: list, + dispatch_id: str = "", + timestamp: str = "2026-05-01T00:00:00+00:00", +) -> None: + """Write audit session with dispatch_id in the index entry.""" + from pathlib import Path as _Path + + session_dir = _Path(log_root) / "sessions" / dir_name + session_dir.mkdir(parents=True, exist_ok=True) + (session_dir / "audit_log.json").write_text(json.dumps(records)) + index_entry = { + "dir_name": dir_name, + "timestamp": timestamp, + "session_id": dir_name, + "dispatch_id": dispatch_id, + } + with (_Path(log_root) / "sessions.jsonl").open("a") as f: + f.write(json.dumps(index_entry) + "\n") + + +def test_iter_session_log_entries_dispatch_id_filter(tmp_path): + """dispatch_id_filter selects only sessions with matching dispatch_id.""" + from autoskillit.pipeline.audit import _iter_session_log_entries + + _write_audit_session_dispatch_id( + tmp_path, "did-a1", [_valid_failure_record_dict()], dispatch_id="d-abc" + ) + _write_audit_session_dispatch_id( + tmp_path, "did-a2", [_valid_failure_record_dict()], dispatch_id="d-abc" + ) + _write_audit_session_dispatch_id( + tmp_path, "did-xyz", [_valid_failure_record_dict()], dispatch_id="d-xyz" + ) + + results = list( + _iter_session_log_entries( + tmp_path, since="", filename="audit_log.json", dispatch_id_filter="d-abc" + ) + ) + assert len(results) == 2 + dir_names = {p.parent.name for p in results} + assert dir_names == {"did-a1", "did-a2"} + + +def test_iter_session_log_entries_dispatch_id_filter_empty_passes_all(tmp_path): + """Empty dispatch_id_filter passes all sessions through.""" + from autoskillit.pipeline.audit import _iter_session_log_entries + + _write_audit_session_dispatch_id( + tmp_path, "did-d1", [_valid_failure_record_dict()], dispatch_id="d1" + ) + _write_audit_session_dispatch_id( + tmp_path, "did-d2", [_valid_failure_record_dict()], dispatch_id="d2" + ) + _write_audit_session_dispatch_id( + tmp_path, "did-empty", [_valid_failure_record_dict()], dispatch_id="" + ) + + results = list( + _iter_session_log_entries( + tmp_path, since="", filename="audit_log.json", dispatch_id_filter="" + ) + ) + assert len(results) == 3 + + +def test_iter_entries_dispatch_id_filter_combined_with_campaign_id(tmp_path): + """dispatch_id_filter AND campaign_id_filter apply as AND logic.""" + from pathlib import Path as _Path + + from autoskillit.pipeline.audit import _iter_session_log_entries + + for dir_name, dispatch_id, campaign_id in [ + ("match", "d1", "c1"), + ("wrong-did", "d2", "c1"), + ("wrong-cid", "d1", "c2"), + ]: + session_dir = _Path(tmp_path) / "sessions" / dir_name + session_dir.mkdir(parents=True, exist_ok=True) + (session_dir / "audit_log.json").write_text(json.dumps([_valid_failure_record_dict()])) + index_entry = { + "dir_name": dir_name, + "timestamp": "2026-05-01T00:00:00+00:00", + "session_id": dir_name, + "dispatch_id": dispatch_id, + "campaign_id": campaign_id, + } + with (_Path(tmp_path) / "sessions.jsonl").open("a") as f: + f.write(json.dumps(index_entry) + "\n") + + results = list( + _iter_session_log_entries( + tmp_path, + since="", + filename="audit_log.json", + dispatch_id_filter="d1", + campaign_id_filter="c1", + ) + ) + assert len(results) == 1 + assert results[0].parent.name == "match" + + +def test_iter_session_log_entries_dispatch_id_combined_with_order_id(tmp_path): + """dispatch_id_filter AND order_id_filter apply as AND logic.""" + from pathlib import Path as _Path + + from autoskillit.pipeline.audit import _iter_session_log_entries + + for dir_name, dispatch_id, order_id in [ + ("both-match", "d1", "o1"), + ("dispatch-only", "d1", "o2"), + ("order-only", "d2", "o1"), + ]: + session_dir = _Path(tmp_path) / "sessions" / dir_name + session_dir.mkdir(parents=True, exist_ok=True) + (session_dir / "audit_log.json").write_text(json.dumps([_valid_failure_record_dict()])) + index_entry = { + "dir_name": dir_name, + "timestamp": "2026-05-01T00:00:00+00:00", + "session_id": dir_name, + "dispatch_id": dispatch_id, + "order_id": order_id, + } + with (_Path(tmp_path) / "sessions.jsonl").open("a") as f: + f.write(json.dumps(index_entry) + "\n") + + results = list( + _iter_session_log_entries( + tmp_path, + since="", + filename="audit_log.json", + dispatch_id_filter="d1", + order_id_filter="o1", + ) + ) + assert len(results) == 1 + assert results[0].parent.name == "both-match" + + +def test_audit_load_dispatch_id_filter(tmp_path): + """DefaultAuditLog.load_from_log_dir respects dispatch_id_filter.""" + from autoskillit.pipeline.audit import DefaultAuditLog + + _write_audit_session_dispatch_id( + tmp_path, "d1-a", [_valid_failure_record_dict()], dispatch_id="d1" + ) + _write_audit_session_dispatch_id( + tmp_path, "d1-b", [_valid_failure_record_dict()], dispatch_id="d1" + ) + _write_audit_session_dispatch_id( + tmp_path, "d2-a", [_valid_failure_record_dict()], dispatch_id="d2" + ) + + log = DefaultAuditLog() + n = log.load_from_log_dir(tmp_path, dispatch_id_filter="d1") + assert n == 2 diff --git a/tests/pipeline/test_background_supervisor.py b/tests/pipeline/test_background_supervisor.py index 178e270196..b51cc41ed8 100644 --- a/tests/pipeline/test_background_supervisor.py +++ b/tests/pipeline/test_background_supervisor.py @@ -6,6 +6,8 @@ from autoskillit.pipeline.background import DefaultBackgroundSupervisor +pytestmark = [pytest.mark.layer("pipeline"), pytest.mark.small] + @pytest.mark.anyio async def test_supervisor_captures_exception(): diff --git a/tests/pipeline/test_context.py b/tests/pipeline/test_context.py index f34a5e94d2..a5a9bea36d 100644 --- a/tests/pipeline/test_context.py +++ b/tests/pipeline/test_context.py @@ -9,12 +9,15 @@ from autoskillit.config import AutomationConfig from autoskillit.core import GitHubFetcher +from autoskillit.core.types._type_plugin_source import DirectInstall from autoskillit.pipeline.audit import DefaultAuditLog, FailureRecord from autoskillit.pipeline.context import ToolContext from autoskillit.pipeline.gate import DefaultGateState from autoskillit.pipeline.timings import DefaultTimingLog from autoskillit.pipeline.tokens import DefaultTokenLog +pytestmark = [pytest.mark.layer("pipeline"), pytest.mark.small] + def test_tool_context_fields_accessible(tmp_path): """ToolContext exposes all expected fields.""" @@ -24,14 +27,17 @@ def test_tool_context_fields_accessible(tmp_path): token_log=DefaultTokenLog(), timing_log=DefaultTimingLog(), gate=DefaultGateState(enabled=True), - plugin_dir=str(tmp_path), + plugin_source=DirectInstall(plugin_dir=tmp_path), runner=None, + temp_dir=tmp_path / ".autoskillit" / "temp", + project_dir=tmp_path, ) assert ctx.gate.enabled is True - assert ctx.plugin_dir == str(tmp_path) + assert isinstance(ctx.plugin_source, DirectInstall) + assert ctx.plugin_source.plugin_dir == tmp_path -def test_tool_context_audit_isolation(): +def test_tool_context_audit_isolation(tmp_path): """Two ToolContext instances have independent AuditLog instances.""" ctx_a = ToolContext( config=AutomationConfig(), @@ -39,8 +45,10 @@ def test_tool_context_audit_isolation(): token_log=DefaultTokenLog(), timing_log=DefaultTimingLog(), gate=DefaultGateState(), - plugin_dir="/a", + plugin_source=DirectInstall(plugin_dir=tmp_path / "a"), runner=None, + temp_dir=tmp_path / "a" / ".autoskillit" / "temp", + project_dir=tmp_path / "a", ) ctx_b = ToolContext( config=AutomationConfig(), @@ -48,8 +56,10 @@ def test_tool_context_audit_isolation(): token_log=DefaultTokenLog(), timing_log=DefaultTimingLog(), gate=DefaultGateState(), - plugin_dir="/b", + plugin_source=DirectInstall(plugin_dir=tmp_path / "b"), runner=None, + temp_dir=tmp_path / "b" / ".autoskillit" / "temp", + project_dir=tmp_path / "b", ) ctx_a.audit.record_failure( FailureRecord( @@ -66,7 +76,7 @@ def test_tool_context_audit_isolation(): assert len(ctx_b.audit.get_report()) == 0 -def test_gate_state_replacement(): +def test_gate_state_replacement(tmp_path): """ToolContext allows gate field replacement via plain assignment.""" ctx = ToolContext( config=AutomationConfig(), @@ -74,8 +84,10 @@ def test_gate_state_replacement(): token_log=DefaultTokenLog(), timing_log=DefaultTimingLog(), gate=DefaultGateState(enabled=False), - plugin_dir="/x", + plugin_source=DirectInstall(plugin_dir=tmp_path), runner=None, + temp_dir=tmp_path / ".autoskillit" / "temp", + project_dir=tmp_path, ) assert ctx.gate.enabled is False ctx.gate = DefaultGateState(enabled=True) @@ -90,8 +102,10 @@ def test_toolcontext_new_optional_fields_default_none(tmp_path): token_log=DefaultTokenLog(), timing_log=DefaultTimingLog(), gate=DefaultGateState(enabled=True), - plugin_dir=str(tmp_path), + plugin_source=DirectInstall(plugin_dir=tmp_path), runner=None, + temp_dir=tmp_path / ".autoskillit" / "temp", + project_dir=tmp_path, ) assert ctx.executor is None assert ctx.tester is None @@ -182,15 +196,16 @@ def test_recipe_repository_protocol_has_rich_methods() -> None: def _make_ctx(tmp_path: Path) -> ToolContext: """Helper: minimal ToolContext with no optional fields.""" - plugin_dir = str(tmp_path) return ToolContext( config=AutomationConfig(), audit=DefaultAuditLog(), token_log=DefaultTokenLog(), timing_log=DefaultTimingLog(), gate=DefaultGateState(enabled=True), - plugin_dir=plugin_dir, + plugin_source=DirectInstall(plugin_dir=tmp_path), runner=None, + temp_dir=tmp_path / ".autoskillit" / "temp", + project_dir=tmp_path, ) @@ -240,8 +255,10 @@ async def test_toolcontext_default_background_wired_with_audit(tmp_path): token_log=DefaultTokenLog(), timing_log=DefaultTimingLog(), gate=DefaultGateState(), - plugin_dir=str(tmp_path), + plugin_source=DirectInstall(plugin_dir=tmp_path), runner=None, + temp_dir=tmp_path / ".autoskillit" / "temp", + project_dir=tmp_path, ) assert isinstance(ctx.background, DefaultBackgroundSupervisor) @@ -279,3 +296,79 @@ def test_tool_context_has_token_factory_field(): assert TokenFactory in args, ( f"token_factory type hint {hints['token_factory']} does not include TokenFactory Protocol" ) + + +def test_tool_context_has_fleet_lock_field(): + """ToolContext has a fleet_lock field defaulting to None.""" + from autoskillit.pipeline.context import ToolContext + + field_info = ToolContext.__dataclass_fields__["fleet_lock"] + assert field_info.default is None + + +def test_tool_context_recipe_identity_defaults(tmp_path): + ctx = _make_ctx(tmp_path) + assert ctx.recipe_name == "" + assert ctx.recipe_content_hash == "" + assert ctx.recipe_composite_hash == "" + assert ctx.recipe_version == "" + + +# --- Group P-2: project_dir env inheritance --- + + +def test_toolcontext_has_project_dir_field(): + """ToolContext dataclass has a project_dir field of type Path.""" + import dataclasses + + from autoskillit.pipeline.context import ToolContext + + field_names = {f.name for f in dataclasses.fields(ToolContext)} + assert "project_dir" in field_names + + +# --- Sentinel guard tests --- + + +def test_toolcontext_raises_typeerror_when_temp_dir_unset(tmp_path): + with pytest.raises(TypeError, match="temp_dir"): + ToolContext( + config=AutomationConfig(), + audit=DefaultAuditLog(), + token_log=DefaultTokenLog(), + timing_log=DefaultTimingLog(), + gate=DefaultGateState(), + plugin_source=DirectInstall(plugin_dir=tmp_path), + runner=None, + project_dir=tmp_path, + ) + + +def test_toolcontext_raises_typeerror_when_project_dir_unset(tmp_path): + with pytest.raises(TypeError, match="project_dir"): + ToolContext( + config=AutomationConfig(), + audit=DefaultAuditLog(), + token_log=DefaultTokenLog(), + timing_log=DefaultTimingLog(), + gate=DefaultGateState(), + plugin_source=DirectInstall(plugin_dir=tmp_path), + runner=None, + temp_dir=tmp_path / ".autoskillit" / "temp", + ) + + +def test_toolcontext_accepts_explicit_path_fields(tmp_path): + ctx = ToolContext( + config=AutomationConfig(), + audit=DefaultAuditLog(), + token_log=DefaultTokenLog(), + timing_log=DefaultTimingLog(), + gate=DefaultGateState(), + plugin_source=DirectInstall(plugin_dir=tmp_path), + runner=None, + temp_dir=tmp_path / ".autoskillit" / "temp", + project_dir=tmp_path, + ) + assert ctx.temp_dir == tmp_path / ".autoskillit" / "temp" + assert ctx.project_dir == tmp_path diff --git a/tests/pipeline/test_gate.py b/tests/pipeline/test_gate.py index 6a18a4af13..2aa81f48b1 100644 --- a/tests/pipeline/test_gate.py +++ b/tests/pipeline/test_gate.py @@ -1,5 +1,9 @@ # test_gate.py — unit tests for _gate.py constants and functions +import pytest + +pytestmark = [pytest.mark.layer("pipeline"), pytest.mark.small] + def test_gated_tools_contains_expected_names(): from autoskillit.pipeline.gate import GATED_TOOLS @@ -32,6 +36,7 @@ def test_gated_tools_contains_expected_names(): "wait_for_merge_queue", "check_repo_merge_state", "toggle_auto_merge", + "enqueue_pr", # formerly ungated — now kitchen-gated: "fetch_github_issue", "get_issue_title", @@ -46,6 +51,12 @@ def test_gated_tools_contains_expected_names(): "validate_recipe", "register_clone_status", "batch_cleanup_clones", + "dispatch_food_truck", + "record_gate_dispatch", + "analyze_tool_sequences", + "bootstrap_clone", + "claim_and_resolve_issue", + "create_and_publish_branch", } assert GATED_TOOLS == expected @@ -62,7 +73,7 @@ def test_check_quota_not_in_ungated_tools(): def test_ungated_tools_contains_expected_names(): from autoskillit.pipeline.gate import UNGATED_TOOLS - expected = {"open_kitchen", "close_kitchen", "disable_quota_guard"} + expected = {"open_kitchen", "close_kitchen", "disable_quota_guard", "reload_session"} assert UNGATED_TOOLS == expected @@ -130,17 +141,8 @@ def test_gate_error_result_accepts_custom_message(): assert parsed["needs_retry"] is False -def test_helpers_has_no_gate_error_result_duplicate(): - import autoskillit.server.helpers as helpers_mod - - assert not hasattr(helpers_mod, "_gate_error_result"), ( - "_gate_error_result must be removed from server.helpers — " - "use gate_error_result() from pipeline.gate instead" - ) - - def test_gate_imports_only_from_core(): - """gate.py (L1 pipeline) may only import from autoskillit.core (L0).""" + """gate.py (IL-1 pipeline) may only import from autoskillit.core (IL-0).""" import ast from autoskillit.core.paths import pkg_root @@ -153,7 +155,7 @@ def test_gate_imports_only_from_core(): assert node.module == "autoskillit.core" or node.module.startswith( "autoskillit.core." ), ( - f"gate.py (L1) may only import from autoskillit.core (L0): " + f"gate.py (IL-1) may only import from autoskillit.core (IL-0): " f"found 'from {node.module} import ...'" ) if isinstance(node, ast.Import): @@ -163,30 +165,6 @@ def test_gate_imports_only_from_core(): ) -def test_gated_tools_does_not_contain_run_recipe(): - from autoskillit.pipeline.gate import GATED_TOOLS - - assert "run_recipe" not in GATED_TOOLS - - -def test_headless_tools_contains_expected_names(): - from autoskillit.core.types import HEADLESS_TOOLS - - assert HEADLESS_TOOLS == {"test_check"} - - -def test_free_range_tools_contains_expected_names(): - from autoskillit.core.types import FREE_RANGE_TOOLS - - assert FREE_RANGE_TOOLS == {"open_kitchen", "close_kitchen", "disable_quota_guard"} - - -def test_ungated_tools_equals_free_range_tools(): - from autoskillit.core.types import FREE_RANGE_TOOLS, UNGATED_TOOLS - - assert UNGATED_TOOLS == FREE_RANGE_TOOLS - - def test_all_tool_sets_disjoint_and_complete(): from autoskillit.core.types import HEADLESS_TOOLS from autoskillit.pipeline.gate import GATED_TOOLS, UNGATED_TOOLS @@ -243,3 +221,12 @@ def test_headless_error_result_accepts_custom_message(): parsed = json.loads(headless_error_result("Custom headless error")) assert parsed["result"] == "Custom headless error" assert parsed["subtype"] == "headless_error" + + +def test_headless_message_uses_l_numbers(): + from autoskillit.pipeline.gate import _DEFAULT_HEADLESS_MESSAGE + + assert "L1" in _DEFAULT_HEADLESS_MESSAGE + assert "L2" in _DEFAULT_HEADLESS_MESSAGE + assert "Tier 1" not in _DEFAULT_HEADLESS_MESSAGE + assert "Tier 2" not in _DEFAULT_HEADLESS_MESSAGE diff --git a/tests/pipeline/test_github_api_log.py b/tests/pipeline/test_github_api_log.py new file mode 100644 index 0000000000..1100bc7b0e --- /dev/null +++ b/tests/pipeline/test_github_api_log.py @@ -0,0 +1,249 @@ +import asyncio + +import pytest + +from autoskillit.pipeline.github_api_log import DefaultGitHubApiLog + +pytestmark = [pytest.mark.layer("pipeline"), pytest.mark.small, pytest.mark.anyio] + + +async def test_empty_log_returns_none_usage(): + log = DefaultGitHubApiLog() + assert log.to_usage("sess-1") is None + + +async def test_record_httpx_increments_total(): + log = DefaultGitHubApiLog() + await log.record_httpx( + method="GET", + path="/repos/owner/repo/issues/1", + status_code=200, + latency_ms=120.0, + rate_limit_remaining=4900, + rate_limit_used=100, + rate_limit_reset=1714000000, + timestamp="2026-04-27T10:00:00Z", + ) + usage = log.to_usage("sess-1") + assert usage is not None + assert usage["total_requests"] == 1 + assert usage["by_source"]["httpx"] == 1 + + +async def test_record_gh_cli_increments_total(): + log = DefaultGitHubApiLog() + await log.record_gh_cli( + subcommand="gh pr list", + exit_code=0, + latency_ms=85.0, + timestamp="2026-04-27T10:00:01Z", + ) + usage = log.to_usage("sess-1") + assert usage is not None + assert usage["total_requests"] == 1 + assert usage["by_source"]["gh_cli"] == 1 + + +async def test_endpoint_categorization(): + log = DefaultGitHubApiLog() + paths = [ + ("/repos/o/r/issues/1", "issues"), + ("/repos/o/r/pulls/2", "pulls"), + ("/repos/o/r/actions/runs", "actions"), + ("/search/issues", "search"), + ("/graphql", "graphql"), + ("/repos/o/r/labels", "other"), + ] + for path, expected_category in paths: + await log.record_httpx( + method="GET", + path=path, + status_code=200, + latency_ms=10.0, + rate_limit_remaining=4999, + rate_limit_used=1, + rate_limit_reset=0, + timestamp="2026-04-27T10:00:00Z", + ) + usage = log.to_usage("sess-1") + assert usage["by_category"]["issues"] == 1 + assert usage["by_category"]["pulls"] == 1 + assert usage["by_category"]["actions"] == 1 + assert usage["by_category"]["search"] == 1 + assert usage["by_category"]["graphql"] == 1 + assert usage["by_category"]["other"] == 1 + + +async def test_min_rate_limit_remaining_tracks_minimum(): + log = DefaultGitHubApiLog() + for remaining in [4900, 4500, 4800]: + await log.record_httpx( + method="GET", + path="/repos/o/r/issues/1", + status_code=200, + latency_ms=10.0, + rate_limit_remaining=remaining, + rate_limit_used=0, + rate_limit_reset=0, + timestamp="2026-04-27T10:00:00Z", + ) + usage = log.to_usage("sess-1") + assert usage["min_rate_limit_remaining"] == 4500 + + +async def test_error_categorization(): + log = DefaultGitHubApiLog() + await log.record_httpx( + method="GET", + path="/repos/o/r/issues/1", + status_code=404, + latency_ms=50.0, + rate_limit_remaining=4999, + rate_limit_used=1, + rate_limit_reset=0, + timestamp="2026-04-27T10:00:00Z", + ) + await log.record_httpx( + method="POST", + path="/repos/o/r/issues", + status_code=500, + latency_ms=200.0, + rate_limit_remaining=4998, + rate_limit_used=2, + rate_limit_reset=0, + timestamp="2026-04-27T10:00:01Z", + ) + usage = log.to_usage("sess-1") + assert usage["errors"]["4xx"] == 1 + assert usage["errors"]["5xx"] == 1 + + +async def test_latency_aggregation(): + log = DefaultGitHubApiLog() + await log.record_httpx( + method="GET", + path="/repos/o/r/issues/1", + status_code=200, + latency_ms=100.0, + rate_limit_remaining=4999, + rate_limit_used=1, + rate_limit_reset=0, + timestamp="2026-04-27T10:00:00Z", + ) + await log.record_httpx( + method="GET", + path="/repos/o/r/issues/2", + status_code=200, + latency_ms=200.0, + rate_limit_remaining=4998, + rate_limit_used=2, + rate_limit_reset=0, + timestamp="2026-04-27T10:00:01Z", + ) + usage = log.to_usage("sess-1") + assert usage["total_latency_ms"] == pytest.approx(300.0) + assert usage["avg_latency_ms"] == pytest.approx(150.0) + + +async def test_first_last_timestamps(): + log = DefaultGitHubApiLog() + await log.record_httpx( + method="GET", + path="/repos/o/r/issues/1", + status_code=200, + latency_ms=10.0, + rate_limit_remaining=4999, + rate_limit_used=1, + rate_limit_reset=0, + timestamp="2026-04-27T10:00:00Z", + ) + await log.record_httpx( + method="GET", + path="/repos/o/r/issues/2", + status_code=200, + latency_ms=10.0, + rate_limit_remaining=4998, + rate_limit_used=2, + rate_limit_reset=0, + timestamp="2026-04-27T10:04:32Z", + ) + usage = log.to_usage("sess-1") + assert usage["first_request_ts"] == "2026-04-27T10:00:00Z" + assert usage["last_request_ts"] == "2026-04-27T10:04:32Z" + + +async def test_clear_resets_state(): + log = DefaultGitHubApiLog() + await log.record_httpx( + method="GET", + path="/repos/o/r/issues/1", + status_code=200, + latency_ms=10.0, + rate_limit_remaining=4999, + rate_limit_used=1, + rate_limit_reset=0, + timestamp="2026-04-27T10:00:00Z", + ) + log.clear() + assert log.to_usage("sess-1") is None + + +async def test_concurrent_record_is_safe(): + log = DefaultGitHubApiLog() + + async def _record(i: int) -> None: + await log.record_httpx( + method="GET", + path=f"/repos/o/r/issues/{i}", + status_code=200, + latency_ms=10.0, + rate_limit_remaining=4999, + rate_limit_used=1, + rate_limit_reset=0, + timestamp="2026-04-27T10:00:00Z", + ) + + await asyncio.gather(*[_record(i) for i in range(50)]) + usage = log.to_usage("sess-1") + assert usage["total_requests"] == 50 + + +async def test_session_id_in_usage(): + log = DefaultGitHubApiLog() + await log.record_httpx( + method="GET", + path="/repos/o/r/issues/1", + status_code=200, + latency_ms=10.0, + rate_limit_remaining=4999, + rate_limit_used=1, + rate_limit_reset=0, + timestamp="2026-04-27T10:00:00Z", + ) + usage = log.to_usage("my-session-123") + assert usage["session_id"] == "my-session-123" + + +async def test_drain_returns_usage_and_clears(): + log = DefaultGitHubApiLog() + await log.record_httpx( + method="GET", + path="/repos/o/r/issues/1", + status_code=200, + latency_ms=10.0, + rate_limit_remaining=4999, + rate_limit_used=1, + rate_limit_reset=0, + timestamp="2026-04-27T10:00:00Z", + ) + + usage = log.drain("sess-drain") + + assert usage is not None + assert usage["total_requests"] == 1 + assert usage["session_id"] == "sess-drain" + + # After drain the accumulator is empty — to_usage and drain both return None + assert len(log._entries) == 0 + assert log.to_usage("sess-drain") is None + assert log.drain("sess-drain") is None diff --git a/tests/pipeline/test_mcp_response.py b/tests/pipeline/test_mcp_response.py index 500f9a0d52..21505f5fc0 100644 --- a/tests/pipeline/test_mcp_response.py +++ b/tests/pipeline/test_mcp_response.py @@ -2,8 +2,12 @@ from __future__ import annotations +import pytest + from autoskillit.pipeline.mcp_response import DefaultMcpResponseLog, McpResponseEntry +pytestmark = [pytest.mark.layer("pipeline"), pytest.mark.small] + class TestMcpResponseEntry: def test_defaults_are_zero(self): diff --git a/tests/pipeline/test_pr_gates.py b/tests/pipeline/test_pr_gates.py index 0dadcd7934..5af5c59fd7 100644 --- a/tests/pipeline/test_pr_gates.py +++ b/tests/pipeline/test_pr_gates.py @@ -9,12 +9,17 @@ import json +import pytest + from autoskillit.pipeline.pr_gates import ( is_ci_passing, + is_pipeline_sourced, is_review_passing, partition_prs, ) +pytestmark = [pytest.mark.layer("pipeline"), pytest.mark.small] + # --------------------------------------------------------------------------- # CI Gate # --------------------------------------------------------------------------- @@ -131,7 +136,11 @@ def test_manifest_includes_blocked_lists(): def test_eligible_prs_ordered_without_blocked(): """Blocked PRs do not appear in the eligible_prs array.""" + _sig = "" prs = [_make_pr(10), _make_pr(20), _make_pr(30)] + prs[0]["body"] = _sig + prs[1]["body"] = "" + prs[2]["body"] = _sig checks = { 10: [{"conclusion": "success"}], 20: [{"conclusion": "failure"}], # blocked @@ -187,8 +196,11 @@ def test_review_blocked_pr_has_reason_string(): def test_all_prs_eligible_when_no_gates_fire(): - """When all PRs pass both gates, ci_blocked and review_blocked are empty.""" + """When all PRs pass both gates and carry the pipeline signature, they are eligible.""" + _sig = "" prs = [_make_pr(1), _make_pr(2)] + prs[0]["body"] = _sig + prs[1]["body"] = _sig checks = {1: [{"conclusion": "success"}], 2: [{"conclusion": "skipped"}]} reviews = {1: [], 2: [{"state": "APPROVED"}]} @@ -210,3 +222,102 @@ def test_ci_gate_fires_before_review_gate(): assert len(result["ci_blocked_prs"]) == 1 assert result["review_blocked_prs"] == [] + + +# --------------------------------------------------------------------------- +# Vocabulary contract +# --------------------------------------------------------------------------- + + +class TestPRGatesVocabularyContract: + """CHANGES_REQUESTED and _CI_PASSING_CONCLUSIONS must be declared as named + constants, and those constants must be consistent with KNOWN_REVIEW_STATES.""" + + def test_known_review_states_constant_exists(self): + from autoskillit.pipeline import pr_gates + + assert hasattr(pr_gates, "KNOWN_REVIEW_STATES") + assert isinstance(pr_gates.KNOWN_REVIEW_STATES, frozenset) + assert "CHANGES_REQUESTED" in pr_gates.KNOWN_REVIEW_STATES + + def test_changes_requested_in_known_review_states(self): + from autoskillit.pipeline.pr_gates import KNOWN_REVIEW_STATES + + assert "CHANGES_REQUESTED" in KNOWN_REVIEW_STATES + + def test_ci_passing_conclusions_constant_exists(self): + from autoskillit.pipeline import pr_gates + + assert hasattr(pr_gates, "_CI_PASSING_CONCLUSIONS") + assert isinstance(pr_gates._CI_PASSING_CONCLUSIONS, frozenset) + assert "success" in pr_gates._CI_PASSING_CONCLUSIONS + + +# --------------------------------------------------------------------------- +# Pipeline provenance gate (1d) +# --------------------------------------------------------------------------- + + +class TestPipelineProvenance: + """is_pipeline_sourced and partition_prs provenance_blocked_prs bucket.""" + + def test_is_pipeline_sourced_true_when_signature_present(self): + body = "PR description\n" + assert is_pipeline_sourced(body) is True + + def test_is_pipeline_sourced_false_when_absent(self): + assert is_pipeline_sourced("🤖 Generated with Claude Code via AutoSkillit") is False + + def test_is_pipeline_sourced_false_on_empty_body(self): + assert is_pipeline_sourced("") is False + + def test_is_pipeline_sourced_false_on_none_body(self): + assert is_pipeline_sourced(None) is False # type: ignore[arg-type] + + def test_is_pipeline_sourced_tolerates_whitespace_variants(self): + body = "" + assert is_pipeline_sourced(body) is True + + def test_partition_prs_produces_provenance_blocked_bucket(self): + """PRs that pass CI+review gates but lack the pipeline signature are provenance-blocked.""" + prs = [ + _make_pr(1, "Has signature"), + _make_pr(2, "No signature"), + ] + checks: dict = {1: [], 2: []} + reviews: dict = {1: [], 2: []} + # Inject body field with signature for PR 1 only + prs[0]["body"] = "" + prs[1]["body"] = "" + + result = partition_prs(prs, checks, reviews) + + assert "provenance_blocked_prs" in result, ( + "partition_prs must return a 'provenance_blocked_prs' key" + ) + provenance_blocked = result["provenance_blocked_prs"] + assert len(provenance_blocked) == 1 + assert provenance_blocked[0]["number"] == 2 + + def test_partition_prs_eligible_prs_excluded_from_provenance_blocked(self): + """PRs with the pipeline signature are not in provenance_blocked_prs.""" + prs = [_make_pr(3, "Pipeline PR")] + prs[0]["body"] = "" + checks: dict = {3: []} + reviews: dict = {3: []} + + result = partition_prs(prs, checks, reviews) + + assert result["provenance_blocked_prs"] == [] + + def test_partition_prs_ci_blocked_excluded_from_provenance_check(self): + """CI-blocked PRs are not also put in provenance_blocked_prs.""" + prs = [_make_pr(4, "CI failing")] + prs[0]["body"] = "" + checks: dict = {4: [{"conclusion": "failure"}]} + reviews: dict = {4: []} + + result = partition_prs(prs, checks, reviews) + + assert len(result["ci_blocked_prs"]) == 1 + assert result["provenance_blocked_prs"] == [] diff --git a/tests/pipeline/test_telemetry_formatter.py b/tests/pipeline/test_telemetry_formatter.py index 23b14e8973..21660f7c4f 100644 --- a/tests/pipeline/test_telemetry_formatter.py +++ b/tests/pipeline/test_telemetry_formatter.py @@ -2,8 +2,13 @@ from __future__ import annotations +import pytest + +from autoskillit.core import ModelTotalEntry from autoskillit.pipeline.telemetry_fmt import TelemetryFormatter +pytestmark = [pytest.mark.layer("pipeline"), pytest.mark.small] + # --------------------------------------------------------------------------- # Shared test data # --------------------------------------------------------------------------- @@ -59,6 +64,8 @@ def test_produces_markdown_table(self) -> None: assert "|---" in result assert "| uncached |" in result assert "| cache_read |" in result + assert "| peak_ctx |" in result + assert "| turns |" in result assert "| cache_write |" in result assert "- input_tokens:" not in result assert "# Token Summary" not in result @@ -99,6 +106,106 @@ def test_falls_back_to_elapsed_when_no_wall_clock(self) -> None: result = TelemetryFormatter.format_token_table(steps, total) assert "30s" in result + # T1 + def test_non_anthropic_step_marked(self) -> None: + steps = [ + { + "step_name": "implement", + "model": "MiniMax-M2.7-highspeed", + "input_tokens": 307600, + "output_tokens": 3400, + "cache_read_input_tokens": 0, + "cache_creation_input_tokens": 0, + "invocation_count": 1, + "wall_clock_seconds": 60.0, + }, + ] + total = { + "input_tokens": 307600, + "output_tokens": 3400, + "cache_read_input_tokens": 0, + "cache_creation_input_tokens": 0, + "total_elapsed_seconds": 60.0, + } + result = TelemetryFormatter.format_token_table(steps, total) + assert "| implement* |" in result + + # T2 + def test_non_anthropic_footnote_present(self) -> None: + steps = [ + { + "step_name": "implement", + "model": "MiniMax-M2.7-highspeed", + "input_tokens": 1000, + "output_tokens": 500, + "cache_read_input_tokens": 0, + "cache_creation_input_tokens": 0, + "invocation_count": 1, + "wall_clock_seconds": 10.0, + }, + ] + total = { + "input_tokens": 1000, + "output_tokens": 500, + "cache_read_input_tokens": 0, + "cache_creation_input_tokens": 0, + "total_elapsed_seconds": 10.0, + } + result = TelemetryFormatter.format_token_table(steps, total) + assert "non-Anthropic provider" in result + assert "caching behavior may differ" in result + + # T3 + def test_anthropic_step_not_marked(self) -> None: + steps = [ + { + "step_name": "plan", + "model": "claude-sonnet-4-6", + "input_tokens": 1000, + "output_tokens": 500, + "cache_read_input_tokens": 200, + "cache_creation_input_tokens": 100, + "invocation_count": 1, + "wall_clock_seconds": 10.0, + }, + ] + total = { + "input_tokens": 1000, + "output_tokens": 500, + "cache_read_input_tokens": 200, + "cache_creation_input_tokens": 100, + "total_elapsed_seconds": 10.0, + } + result = TelemetryFormatter.format_token_table(steps, total) + assert "| plan |" in result + assert "plan*" not in result + assert "non-Anthropic" not in result + + # T4 + def test_empty_model_not_marked(self) -> None: + steps = [ + { + "step_name": "plan", + "model": "", + "input_tokens": 1000, + "output_tokens": 500, + "cache_read_input_tokens": 200, + "cache_creation_input_tokens": 100, + "invocation_count": 1, + "wall_clock_seconds": 10.0, + }, + ] + total = { + "input_tokens": 1000, + "output_tokens": 500, + "cache_read_input_tokens": 200, + "cache_creation_input_tokens": 100, + "total_elapsed_seconds": 10.0, + } + result = TelemetryFormatter.format_token_table(steps, total) + assert "plan*" not in result + assert "non-Anthropic" not in result + def test_snapshot(self) -> None: """Golden snapshot test for token table format.""" steps = [ @@ -108,6 +215,8 @@ def test_snapshot(self) -> None: "output_tokens": 500, "cache_creation_input_tokens": 100, "cache_read_input_tokens": 200, + "peak_context": 200, + "turn_count": 5, "invocation_count": 1, "wall_clock_seconds": 45.7, }, @@ -117,6 +226,7 @@ def test_snapshot(self) -> None: "output_tokens": 500, "cache_creation_input_tokens": 100, "cache_read_input_tokens": 200, + "peak_context": 200, "total_elapsed_seconds": 45.7, } result = TelemetryFormatter.format_token_table(steps, total) @@ -124,10 +234,10 @@ def test_snapshot(self) -> None: [ "## Token Usage Summary", "", - "| Step | uncached | output | cache_read | cache_write | count | time |", - "|------|----------|--------|------------|-------------|-------|------|", - "| plan | 1.0k | 500 | 200 | 100 | 1 | 46s |", - "| **Total** | 1.0k | 500 | 200 | 100 | | 46s |", + "| Step | Model | count | uncached | output | cache_read | peak_ctx | turns | cache_write | time |", # noqa: E501 + "|------|-------|-------|----------|--------|------------|----------|-------|-------------|------|", # noqa: E501 + "| plan | | 1 | 1.0k | 500 | 200 | 200 | 5 | 100 | 46s |", + "| **Total** | | | 1.0k | 500 | 200 | 200 | | 100 | 46s |", ] ) assert result == expected @@ -227,22 +337,28 @@ def test_format_timing_table_terminal_output_has_leading_indent() -> None: # --------------------------------------------------------------------------- -def test_terminal_table_has_four_token_columns() -> None: - """Terminal table must show UNCACHED, CACHE_RD, CACHE_WR column headers.""" +def test_terminal_table_has_token_columns() -> None: + """Terminal table must show COUNT, UNCACHED, CACHE_RD, PEAK_CTX, TURNS, CACHE_WR headers.""" result = TelemetryFormatter.format_token_table_terminal(_STEPS, _TOTAL) + assert "COUNT" in result assert "UNCACHED" in result assert "CACHE_RD" in result + assert "PEAK_CTX" in result + assert "TURNS" in result assert "CACHE_WR" in result def test_compact_kv_four_token_prefixes() -> None: - """Compact KV format must use uc:, cr:, cw: prefixes and split totals.""" + """Compact KV format must use uc:, cr:, pk:, cw:, turns: prefixes and split totals.""" result = TelemetryFormatter.format_compact_kv(_STEPS, _TOTAL) assert "uc:" in result assert "cr:" in result + assert "pk:" in result assert "cw:" in result + assert "turns:" in result assert "total_uncached:" in result assert "total_cache_read:" in result + assert "total_peak_context:" in result assert "total_cache_write:" in result assert "in:" not in result assert " cached:" not in result @@ -325,3 +441,585 @@ def test_minutes(self) -> None: def test_hours(self) -> None: assert TelemetryFormatter._fmt_duration(3720.0) == "1h 2m" + + +# --------------------------------------------------------------------------- +# T-EFF: Token Efficiency table +# --------------------------------------------------------------------------- + + +# T-EFF-1 +def test_efficiency_table_omitted_when_no_loc() -> None: + """format_efficiency_table returns '' when all steps have loc_changed=0.""" + steps = [ + { + "step_name": "plan", + "cache_read_input_tokens": 1000, + "cache_creation_input_tokens": 200, + "output_tokens": 50, + "loc_insertions": 0, + "loc_deletions": 0, + } + ] + total = { + "loc_insertions": 0, + "loc_deletions": 0, + "cache_read_input_tokens": 1000, + "cache_creation_input_tokens": 200, + "output_tokens": 50, + } + assert TelemetryFormatter.format_efficiency_table(steps, total) == "" + + +# T-EFF-2 +def test_efficiency_table_columns() -> None: + """Markdown efficiency table has correct header columns.""" + steps = [ + { + "step_name": "implement", + "peak_context": 500, + "cache_read_input_tokens": 1000, + "cache_creation_input_tokens": 200, + "output_tokens": 50, + "loc_insertions": 80, + "loc_deletions": 20, + } + ] + total = { + "loc_insertions": 80, + "loc_deletions": 20, + "peak_context": 500, + "cache_read_input_tokens": 1000, + "cache_creation_input_tokens": 200, + "output_tokens": 50, + } + result = TelemetryFormatter.format_efficiency_table(steps, total) + assert "## Token Efficiency" in result + assert "LoC Changed" in result + assert "cache_read/LoC" in result + assert "cache_write/LoC" in result + assert "output/LoC" in result + assert result.split("\n")[2].count("|") == 6 # 5 columns + outer pipes + + +# T-EFF-3 +def test_efficiency_table_ratios() -> None: + """Ratio columns = token_count / loc_changed, rounded to 1 decimal place.""" + steps = [ + { + "step_name": "implement", + "peak_context": 1000, + "cache_creation_input_tokens": 200, + "output_tokens": 50, + "loc_insertions": 80, + "loc_deletions": 20, + } + ] + total = { + "loc_insertions": 80, + "loc_deletions": 20, + "peak_context": 1000, + "cache_creation_input_tokens": 200, + "output_tokens": 50, + } + result = TelemetryFormatter.format_efficiency_table(steps, total) + # loc_changed = 100; cache_write/LoC = 2.0; output/LoC = 0.5 + assert "2.0" in result + assert "0.5" in result + assert "100" in result # LoC Changed column + + +# T-EFF-4 +def test_efficiency_table_zero_loc_step_shows_dash() -> None: + """Steps with loc_changed=0 show — in ratio columns.""" + steps = [ + { + "step_name": "no-change", + "peak_context": 500, + "cache_creation_input_tokens": 100, + "output_tokens": 20, + "loc_insertions": 0, + "loc_deletions": 0, + }, + { + "step_name": "implement", + "peak_context": 1000, + "cache_creation_input_tokens": 200, + "output_tokens": 50, + "loc_insertions": 50, + "loc_deletions": 10, + }, + ] + total = { + "loc_insertions": 50, + "loc_deletions": 10, + "peak_context": 1000, + "cache_creation_input_tokens": 300, + "output_tokens": 70, + } + result = TelemetryFormatter.format_efficiency_table(steps, total) + assert "—" in result # zero-LoC step + + +# T-EFF-5 +def test_efficiency_table_total_row_uses_aggregate_totals() -> None: + """Total row ratios are computed from aggregate totals, not averaged per-step.""" + steps = [ + { + "step_name": "plan", + "peak_context": 200, + "cache_creation_input_tokens": 0, + "output_tokens": 10, + "loc_insertions": 10, + "loc_deletions": 0, + }, + { + "step_name": "implement", + "peak_context": 800, + "cache_creation_input_tokens": 100, + "output_tokens": 40, + "loc_insertions": 90, + "loc_deletions": 10, + }, + ] + total = { + "loc_insertions": 100, + "loc_deletions": 10, + "peak_context": 1000, + "cache_creation_input_tokens": 100, + "output_tokens": 50, + } + result = TelemetryFormatter.format_efficiency_table(steps, total) + # Total loc_changed=110; cache_write/LoC=100/110≈0.9 (aggregate, not per-step average 0.5) + assert "0.9" in result + assert "**Total**" in result + + +# T-EFF-6 +def test_efficiency_table_terminal_no_markdown() -> None: + """Terminal efficiency table uses padded columns, no markdown syntax.""" + steps = [ + { + "step_name": "implement", + "peak_context": 1000, + "cache_creation_input_tokens": 200, + "output_tokens": 50, + "loc_insertions": 80, + "loc_deletions": 20, + } + ] + total = { + "loc_insertions": 80, + "loc_deletions": 20, + "peak_context": 1000, + "cache_creation_input_tokens": 200, + "output_tokens": 50, + } + result = TelemetryFormatter.format_efficiency_table_terminal(steps, total) + assert "|" not in result # no markdown pipes + assert "LOC" in result # column header present + assert "RD/LOC" in result + assert "WR/LOC" in result + assert "OUT/LOC" in result + + +# T-EFF-7 +def test_efficiency_table_has_all_ratio_columns() -> None: + """Markdown efficiency table must contain all four ratio columns and have 6-column header.""" + steps = [ + { + "step_name": "implement", + "peak_context": 500, + "cache_read_input_tokens": 1000, + "cache_creation_input_tokens": 200, + "output_tokens": 50, + "loc_insertions": 80, + "loc_deletions": 20, + } + ] + total = { + "loc_insertions": 80, + "loc_deletions": 20, + "peak_context": 500, + "cache_read_input_tokens": 1000, + "cache_creation_input_tokens": 200, + "output_tokens": 50, + } + result = TelemetryFormatter.format_efficiency_table(steps, total) + assert "cache_read/LoC" in result + assert "cache_write/LoC" in result + assert "output/LoC" in result + header_line = result.split("\n")[2] + assert header_line.count("|") == 6 # 5 columns + outer pipes + + +# T-EFF-8 +def test_efficiency_table_terminal_has_all_columns() -> None: + """Terminal efficiency table must show RD/LOC alongside PK/LOC, WR/LOC, OUT/LOC.""" + steps = [ + { + "step_name": "implement", + "peak_context": 1000, + "cache_read_input_tokens": 500, + "cache_creation_input_tokens": 200, + "output_tokens": 50, + "loc_insertions": 80, + "loc_deletions": 20, + } + ] + total = { + "loc_insertions": 80, + "loc_deletions": 20, + "peak_context": 1000, + "cache_read_input_tokens": 500, + "cache_creation_input_tokens": 200, + "output_tokens": 50, + } + result = TelemetryFormatter.format_efficiency_table_terminal(steps, total) + assert "RD/LOC" in result + assert "WR/LOC" in result + assert "OUT/LOC" in result + + +# T-EFF-9 +def test_efficiency_markdown_terminal_column_parity() -> None: + """Markdown and terminal efficiency tables must have the same number of data columns.""" + from autoskillit.pipeline.telemetry_fmt import _EFFICIENCY_COLUMNS + + steps = [ + { + "step_name": "implement", + "peak_context": 1000, + "cache_read_input_tokens": 500, + "cache_creation_input_tokens": 200, + "output_tokens": 50, + "loc_insertions": 80, + "loc_deletions": 20, + } + ] + total = { + "loc_insertions": 80, + "loc_deletions": 20, + "peak_context": 1000, + "cache_read_input_tokens": 500, + "cache_creation_input_tokens": 200, + "output_tokens": 50, + } + result = TelemetryFormatter.format_efficiency_table(steps, total) + md_header = result.split("\n")[2] + md_cols = [c.strip() for c in md_header.strip("|").split("|")] + assert len(md_cols) == len(_EFFICIENCY_COLUMNS) + + +# --------------------------------------------------------------------------- +# Field coverage contract tests +# --------------------------------------------------------------------------- + + +def test_token_column_field_coverage() -> None: + """Every TokenEntry field must be explicitly assigned to DISPLAY or EXCLUDED.""" + import dataclasses + + from autoskillit.pipeline.telemetry_fmt import _TOKEN_DISPLAY_FIELDS, _TOKEN_EXCLUDED_FIELDS + from autoskillit.pipeline.tokens import TokenEntry + + all_fields = frozenset(f.name for f in dataclasses.fields(TokenEntry)) + covered = _TOKEN_DISPLAY_FIELDS | _TOKEN_EXCLUDED_FIELDS + assert covered == all_fields, ( + f"Uncovered fields: {all_fields - covered}; Stale exclusions: {covered - all_fields}" + ) + + +def test_token_display_fields_have_columns() -> None: + """Every field in _TOKEN_DISPLAY_FIELDS must have a corresponding _TOKEN_COLUMNS entry.""" + from autoskillit.pipeline.telemetry_fmt import ( + _TOKEN_COLUMNS, + _TOKEN_DISPLAY_FIELDS, + _TOKEN_FIELD_TO_COLUMN, + ) + + column_labels = frozenset(c.label for c in _TOKEN_COLUMNS) + for field in _TOKEN_DISPLAY_FIELDS: + target_col = _TOKEN_FIELD_TO_COLUMN[field] + assert target_col in column_labels, ( + f"Field '{field}' maps to column '{target_col}' which is not in _TOKEN_COLUMNS" + ) + + +def test_token_table_renders_invocation_count() -> None: + """Token table must include invocation_count as a visible column.""" + steps = [ + { + "step_name": "fix", + "input_tokens": 100, + "output_tokens": 50, + "cache_read_input_tokens": 200, + "cache_creation_input_tokens": 10, + "peak_context": 500, + "turn_count": 40, + "invocation_count": 3, + "wall_clock_seconds": 120.0, + } + ] + total = { + "input_tokens": 100, + "output_tokens": 50, + "cache_read_input_tokens": 200, + "cache_creation_input_tokens": 10, + "peak_context": 500, + "total_elapsed_seconds": 120.0, + "invocation_count": 3, + "turn_count": 40, + } + result = TelemetryFormatter.format_token_table(steps, total) + assert "| 3 |" in result, "invocation_count=3 not rendered" + + +def test_token_markdown_terminal_column_parity() -> None: + """Markdown and terminal token tables must have the same number of data columns.""" + from autoskillit.pipeline.telemetry_fmt import _TOKEN_COLUMNS + + steps = [ + { + "step_name": "plan", + "input_tokens": 1000, + "output_tokens": 500, + "cache_read_input_tokens": 200, + "cache_creation_input_tokens": 100, + "peak_context": 200, + "turn_count": 5, + "invocation_count": 1, + "wall_clock_seconds": 45.7, + } + ] + total = { + "input_tokens": 1000, + "output_tokens": 500, + "cache_read_input_tokens": 200, + "cache_creation_input_tokens": 100, + "peak_context": 200, + "total_elapsed_seconds": 45.7, + } + result = TelemetryFormatter.format_token_table(steps, total) + md_header = result.split("\n")[2] + md_cols = [c.strip() for c in md_header.strip("|").split("|")] + assert len(md_cols) == len(_TOKEN_COLUMNS) + + +def test_timing_markdown_terminal_column_parity() -> None: + """Timing table markdown header must be derived from _TIMING_COLUMNS.""" + from autoskillit.pipeline.telemetry_fmt import _TIMING_COLUMNS + + steps = [{"step_name": "plan", "total_seconds": 45.0, "invocation_count": 2}] + total = {"total_seconds": 45.0} + result = TelemetryFormatter.format_timing_table(steps, total) + md_header = result.split("\n")[2] + md_cols = [c.strip() for c in md_header.strip("|").split("|")] + assert len(md_cols) == len(_TIMING_COLUMNS) + + +def test_efficiency_table_no_peak_ctx_ratio() -> None: + """Efficiency table must not contain peak_ctx/LoC — it's a meaningless ratio.""" + steps = [ + { + "step_name": "fix", + "input_tokens": 100, + "output_tokens": 500, + "cache_read_input_tokens": 2000, + "cache_creation_input_tokens": 100, + "peak_context": 5000, + "loc_insertions": 10, + "loc_deletions": 5, + } + ] + total = { + "input_tokens": 100, + "output_tokens": 500, + "cache_read_input_tokens": 2000, + "cache_creation_input_tokens": 100, + "peak_context": 5000, + "loc_insertions": 10, + "loc_deletions": 5, + } + result = TelemetryFormatter.format_efficiency_table(steps, total) + assert "peak_ctx/LoC" not in result + assert "PK/LOC" not in TelemetryFormatter.format_efficiency_table_terminal(steps, total) + + +# --------------------------------------------------------------------------- +# Model column in format_token_table +# --------------------------------------------------------------------------- + + +def test_format_token_table_includes_model_column() -> None: + steps = [ + { + "step_name": "plan", + "model": "claude-sonnet-4-6", + "input_tokens": 1000, + "output_tokens": 500, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + "invocation_count": 1, + "wall_clock_seconds": 60.0, + "peak_context": 0, + "turn_count": 5, + } + ] + total = { + "input_tokens": 1000, + "output_tokens": 500, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + "total_elapsed_seconds": 60.0, + "peak_context": 0, + } + table = TelemetryFormatter.format_token_table(steps, total) + assert "| Model |" in table + assert "claude-sonnet-4-6" in table + + +# --------------------------------------------------------------------------- +# format_model_table +# --------------------------------------------------------------------------- + + +def test_format_model_table() -> None: + model_totals: list[ModelTotalEntry] = [ + { + "model": "claude-sonnet-4-6", + "step_count": 2, + "input_tokens": 300, + "output_tokens": 130, + "cache_read_input_tokens": 5000, + "cache_creation_input_tokens": 100, + "elapsed_seconds": 120.0, + }, + { + "model": "MiniMax-M2.7", + "step_count": 1, + "input_tokens": 500, + "output_tokens": 200, + "cache_read_input_tokens": 1200, + "cache_creation_input_tokens": 0, + "elapsed_seconds": 60.0, + }, + ] + table = TelemetryFormatter.format_model_table(model_totals) + assert "## Model Usage Breakdown" in table + assert "| Model | steps | uncached | output | cache_read | cache_write | time |" in table + assert "| claude-sonnet-4-6 | 2 | 300 | 130 | 5.0k | 100 | 2m 0s |" in table + assert "| MiniMax-M2.7 | 1 | 500 | 200 | 1.2k | 0 | 1m 0s |" in table + table_lines = [ln for ln in table.splitlines() if ln.startswith("|")] + assert len(table_lines) == 4 # header + separator + 2 model rows + + +def test_format_model_table_empty_returns_empty() -> None: + assert TelemetryFormatter.format_model_table([]) == "" + + +def test_format_model_table_all_unknown_returns_empty() -> None: + totals: list[ModelTotalEntry] = [ + { + "model": "unknown", + "step_count": 1, + "input_tokens": 100, + "output_tokens": 0, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + "elapsed_seconds": 0.0, + } + ] + assert TelemetryFormatter.format_model_table(totals) == "" + + +# --------------------------------------------------------------------------- +# model tag in compact KV +# --------------------------------------------------------------------------- + + +def test_compact_kv_includes_model() -> None: + steps = [ + { + "step_name": "plan", + "model": "claude-sonnet-4-6", + "input_tokens": 100, + "output_tokens": 50, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + "invocation_count": 1, + "turn_count": 0, + "wall_clock_seconds": 10.0, + } + ] + total = {"input_tokens": 100, "output_tokens": 50} + output = TelemetryFormatter.format_compact_kv(steps, total) + assert "model:claude-sonnet-4-6" in output + + +def test_compact_kv_no_model_tag_when_empty() -> None: + steps = [ + { + "step_name": "plan", + "input_tokens": 100, + "output_tokens": 50, + "invocation_count": 1, + "turn_count": 0, + "wall_clock_seconds": 10.0, + } + ] + total = {} + output = TelemetryFormatter.format_compact_kv(steps, total) + assert "model:" not in output + + +# T5 +def test_non_anthropic_step_marked_terminal() -> None: + steps = [ + { + "step_name": "implement", + "model": "MiniMax-M2.7-highspeed", + "input_tokens": 307600, + "output_tokens": 3400, + "cache_read_input_tokens": 0, + "cache_creation_input_tokens": 0, + "invocation_count": 1, + "wall_clock_seconds": 60.0, + }, + ] + total = { + "input_tokens": 307600, + "output_tokens": 3400, + "cache_read_input_tokens": 0, + "cache_creation_input_tokens": 0, + "total_elapsed_seconds": 60.0, + } + result = TelemetryFormatter.format_token_table_terminal(steps, total) + assert "implement*" in result + assert "non-Anthropic provider" in result + + +# T6 +def test_non_anthropic_step_marked_compact_kv() -> None: + steps = [ + { + "step_name": "implement", + "model": "MiniMax-M2.7-highspeed", + "input_tokens": 307600, + "output_tokens": 3400, + "cache_read_input_tokens": 0, + "cache_creation_input_tokens": 0, + "invocation_count": 1, + "wall_clock_seconds": 60.0, + }, + ] + total = { + "input_tokens": 307600, + "output_tokens": 3400, + "cache_read_input_tokens": 0, + "cache_creation_input_tokens": 0, + "total_elapsed_seconds": 60.0, + } + result = TelemetryFormatter.format_compact_kv(steps, total) + assert "implement*" in result + assert "non-Anthropic provider" in result diff --git a/tests/pipeline/test_timings.py b/tests/pipeline/test_timings.py index 49a47a2e40..53ba75c2b3 100644 --- a/tests/pipeline/test_timings.py +++ b/tests/pipeline/test_timings.py @@ -8,6 +8,8 @@ import pytest +pytestmark = [pytest.mark.layer("pipeline"), pytest.mark.small] + class TestTimingEntry: def test_fields_exist(self): @@ -390,3 +392,91 @@ def test_timing_no_filter_aggregates_all_orders(self): assert all_entries[0]["step_name"] == "plan" assert all_entries[0]["total_seconds"] == 15.0 assert all_entries[0]["invocation_count"] == 2 + + +# --- Group N: campaign_id_filter tests --- + + +def _write_timing_session_cid( + log_root: Path, + dir_name: str, + campaign_id: str = "", + timestamp: str = "2026-04-20T00:00:00+00:00", +) -> None: + """Write a step_timing session with campaign_id in the sessions.jsonl index.""" + session_dir = log_root / "sessions" / dir_name + session_dir.mkdir(parents=True, exist_ok=True) + (session_dir / "step_timing.json").write_text( + json.dumps({"step_name": "implement", "total_seconds": 10.0}) + ) + index_entry = { + "dir_name": dir_name, + "timestamp": timestamp, + "campaign_id": campaign_id, + } + with (log_root / "sessions.jsonl").open("a") as f: + f.write(json.dumps(index_entry) + "\n") + + +def test_timing_load_campaign_id_filter(tmp_path): + """DefaultTimingLog.load_from_log_dir respects campaign_id_filter.""" + from autoskillit.pipeline.timings import DefaultTimingLog + + _write_timing_session_cid(tmp_path, "c1-a", campaign_id="c1") + _write_timing_session_cid(tmp_path, "c1-b", campaign_id="c1") + _write_timing_session_cid(tmp_path, "c2-a", campaign_id="c2") + + log = DefaultTimingLog() + n = log.load_from_log_dir(tmp_path, campaign_id_filter="c1") + assert n == 2 + + log2 = DefaultTimingLog() + n2 = log2.load_from_log_dir(tmp_path, campaign_id_filter="c2") + assert n2 == 1 + + log3 = DefaultTimingLog() + n3 = log3.load_from_log_dir(tmp_path, campaign_id_filter="") + assert n3 == 3 + + +def _write_timing_session_dispatch_id( + log_root: Path, + dir_name: str, + dispatch_id: str, + timestamp: str = "2026-05-01T00:00:00+00:00", +) -> None: + """Write a step_timing session with dispatch_id in the index.""" + session_dir = log_root / "sessions" / dir_name + session_dir.mkdir(parents=True, exist_ok=True) + (session_dir / "step_timing.json").write_text( + json.dumps({"step_name": "implement", "total_seconds": 10.0}) + ) + index_entry = { + "dir_name": dir_name, + "timestamp": timestamp, + "campaign_id": "", + "dispatch_id": dispatch_id, + } + with (log_root / "sessions.jsonl").open("a") as f: + f.write(json.dumps(index_entry) + "\n") + + +def test_timing_load_dispatch_id_filter(tmp_path): + """DefaultTimingLog.load_from_log_dir respects dispatch_id_filter.""" + from autoskillit.pipeline.timings import DefaultTimingLog + + _write_timing_session_dispatch_id(tmp_path, "d1-a", dispatch_id="d1") + _write_timing_session_dispatch_id(tmp_path, "d1-b", dispatch_id="d1") + _write_timing_session_dispatch_id(tmp_path, "d2-a", dispatch_id="d2") + + log = DefaultTimingLog() + n = log.load_from_log_dir(tmp_path, dispatch_id_filter="d1") + assert n == 2 + + log_d2 = DefaultTimingLog() + n_d2 = log_d2.load_from_log_dir(tmp_path, dispatch_id_filter="d2") + assert n_d2 == 1 + + log2 = DefaultTimingLog() + n2 = log2.load_from_log_dir(tmp_path, dispatch_id_filter="") + assert n2 == 3 diff --git a/tests/pipeline/test_tokens.py b/tests/pipeline/test_tokens_core.py similarity index 55% rename from tests/pipeline/test_tokens.py rename to tests/pipeline/test_tokens_core.py index d0577ec5b5..86466795bb 100644 --- a/tests/pipeline/test_tokens.py +++ b/tests/pipeline/test_tokens_core.py @@ -1,4 +1,4 @@ -"""Tests for autoskillit.pipeline.tokens — pipeline token usage tracking.""" +"""Tests for pipeline.tokens — TokenEntry, DefaultTokenLog core, and log-dir loading.""" from __future__ import annotations @@ -10,6 +10,8 @@ from autoskillit.pipeline.tokens import DefaultTokenLog, TokenEntry +pytestmark = [pytest.mark.layer("pipeline"), pytest.mark.small] + def _make_usage(**overrides: int) -> dict[str, int]: defaults = { @@ -27,12 +29,17 @@ def test_fields_exist(self): field_names = {f.name for f in fields(entry)} assert field_names == { "step_name", + "model", "input_tokens", "output_tokens", "cache_creation_input_tokens", "cache_read_input_tokens", "invocation_count", "elapsed_seconds", + "loc_insertions", + "loc_deletions", + "peak_context", + "turn_count", } def test_default_counts_are_zero(self): @@ -53,12 +60,17 @@ def test_to_dict_contains_all_fields(self): d = entry.to_dict() assert set(d.keys()) == { "step_name", + "model", "input_tokens", "output_tokens", "cache_creation_input_tokens", "cache_read_input_tokens", "invocation_count", "elapsed_seconds", + "loc_insertions", + "loc_deletions", + "peak_context", + "turn_count", } assert d["step_name"] == "implement" assert d["input_tokens"] == 42 @@ -144,6 +156,10 @@ def test_compute_total_empty_log(self): "cache_creation_input_tokens": 0, "cache_read_input_tokens": 0, "total_elapsed_seconds": 0.0, + "loc_insertions": 0, + "loc_deletions": 0, + "peak_context": 0, + "turn_count": 0, } def test_token_entry_has_elapsed_seconds_field(self): @@ -416,351 +432,281 @@ def test_load_null_timing_seconds(self, tmp_path): assert report[0]["elapsed_seconds"] == 0.0 -class TestLoadFromLogDirDesignDocumentation: - """P9-F1: tokens.py must document design decisions inline.""" - - def test_load_from_log_dir_documents_timing_seconds_rename(self): - """tokens.py must comment the timing_seconds → elapsed_seconds key rename.""" - from pathlib import Path - - src = ( - Path(__file__).parent.parent.parent / "src/autoskillit/pipeline/tokens.py" - ).read_text() - assert "timing_seconds" in src - assert "elapsed_seconds" in src - - def test_load_from_log_dir_documents_invocation_count_design(self): - """tokens.py must comment why invocation_count += 1 per file is intentional.""" - from pathlib import Path - - src = ( - Path(__file__).parent.parent.parent / "src/autoskillit/pipeline/tokens.py" - ).read_text() - assert "invocation_count" in src - assert ( - "one file" in src.lower() - or "one invocation" in src.lower() - or "per file" in src.lower() - ) - - # --------------------------------------------------------------------------- -# cwd_filter tests +# T-LOC: LoC (lines-of-code-changed) fields # --------------------------------------------------------------------------- -def _write_session_cwd( - log_root: Path, - dir_name: str, - tu_data: dict, - cwd: str, - timestamp: str = "2026-03-07T00:00:00+00:00", -) -> None: - """Write a session with a cwd field in the sessions.jsonl index.""" - session_dir = log_root / "sessions" / dir_name - session_dir.mkdir(parents=True, exist_ok=True) - (session_dir / "token_usage.json").write_text(json.dumps(tu_data)) - index_entry = { - "dir_name": dir_name, - "timestamp": timestamp, - "session_id": dir_name, - "cwd": cwd, - } - with (log_root / "sessions.jsonl").open("a") as f: - f.write(json.dumps(index_entry) + "\n") - - -class TestLoadFromLogDirCwdFilter: - """Tests for cwd_filter parameter on DefaultTokenLog.load_from_log_dir.""" - - _PLAN_DATA = { - "step_name": "plan", - "input_tokens": 100, - "output_tokens": 50, - "cache_creation_input_tokens": 0, - "cache_read_input_tokens": 0, - "timing_seconds": 10.0, - } - _IMPL_DATA = { - "step_name": "implement", - "input_tokens": 200, - "output_tokens": 80, - "cache_creation_input_tokens": 0, - "cache_read_input_tokens": 0, - "timing_seconds": 20.0, - } - - def test_cwd_filter_isolates_to_matching_cwd(self, tmp_path): - """cwd_filter loads only sessions whose cwd matches the given path.""" - _write_session_cwd(tmp_path, "sess-A", self._PLAN_DATA, "/runs/pipeline-A") - _write_session_cwd(tmp_path, "sess-B", self._IMPL_DATA, "/runs/pipeline-B") - - log = DefaultTokenLog() - count = log.load_from_log_dir(tmp_path, cwd_filter="/runs/pipeline-A") - assert count == 1, "Should only load the session whose cwd matches" - steps = log.get_report() - assert len(steps) == 1 - assert steps[0]["step_name"] == "plan" - assert steps[0]["input_tokens"] == 100 - - def test_cwd_filter_empty_loads_all(self, tmp_path): - """Empty cwd_filter loads all sessions (backward compatibility).""" - _write_session_cwd(tmp_path, "sess-A", self._PLAN_DATA, "/runs/pipeline-A") - _write_session_cwd(tmp_path, "sess-B", self._IMPL_DATA, "/runs/pipeline-B") - - log = DefaultTokenLog() - count = log.load_from_log_dir(tmp_path, cwd_filter="") - assert count == 2 - - def test_cwd_filter_no_matches_returns_zero(self, tmp_path): - """cwd_filter that matches nothing returns 0 and leaves the log empty.""" - _write_session_cwd(tmp_path, "sess-A", self._PLAN_DATA, "/runs/pipeline-A") - _write_session_cwd(tmp_path, "sess-B", self._IMPL_DATA, "/runs/pipeline-B") - - log = DefaultTokenLog() - count = log.load_from_log_dir(tmp_path, cwd_filter="/runs/nonexistent") - assert count == 0 - assert log.get_report() == [] - - def test_cwd_filter_default_loads_all(self, tmp_path): - """Calling load_from_log_dir without cwd_filter loads all sessions.""" - _write_session_cwd(tmp_path, "sess-A", self._PLAN_DATA, "/runs/pipeline-A") - _write_session_cwd(tmp_path, "sess-B", self._IMPL_DATA, "/runs/pipeline-B") - - log = DefaultTokenLog() - count = log.load_from_log_dir(tmp_path) - assert count == 2 - - -# --------------------------------------------------------------------------- -# Step name normalization tests -# --------------------------------------------------------------------------- +# T-LOC-1 +def test_token_entry_has_loc_fields(): + """TokenEntry carries loc_insertions and loc_deletions defaulting to 0.""" + e = TokenEntry(step_name="implement") + assert e.loc_insertions == 0 + assert e.loc_deletions == 0 -class TestCanonicalStepName: - """canonical_step_name() must strip trailing -N suffixes.""" - - @pytest.mark.parametrize( - "raw,expected", - [ - ("plan", "plan"), - ("plan-30", "plan"), - ("implement-31", "implement"), - ("open_pr-28", "open_pr"), - ("retry_worktree", "retry_worktree"), # underscore-separated, no hyphen to strip - ("audit_impl", "audit_impl"), # underscore, no hyphen - ("open-pr", "open-pr"), # hyphen + non-numeric — not stripped - ("step-2", "step"), # two-char suffix stripped - ("step-123", "step"), # multi-digit stripped - ("plan-30-retry", "plan-30-retry"), # ends non-numeric — not stripped - ("", ""), # empty string safe - ], - ) - def test_strips_trailing_numeric_suffix(self, raw, expected): - from autoskillit.pipeline.tokens import canonical_step_name - - assert canonical_step_name(raw) == expected - - -class TestTokenLogStepNameNormalization: - def test_suffixed_step_names_aggregate_to_canonical(self): - """ - record("plan-30", ...) and record("plan-31", ...) must produce - a single entry keyed by "plan", with invocation_count=2. - """ - log = DefaultTokenLog() - usage = { +# T-LOC-2 +def test_token_log_record_accumulates_loc(): + """record() accumulates loc_insertions and loc_deletions across invocations.""" + log = DefaultTokenLog() + log.record( + "implement", + { "input_tokens": 100, - "output_tokens": 50, + "output_tokens": 10, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 0, - } - - log.record("plan-30", usage) - log.record("plan-31", usage) - - report = log.get_report() - assert len(report) == 1, ( - f"Expected 1 canonical entry, got: {[e['step_name'] for e in report]}" - ) - entry = report[0] - assert entry["step_name"] == "plan" - assert entry["invocation_count"] == 2 - assert entry["input_tokens"] == 200 - - def test_canonical_name_and_suffixed_name_merge(self): - """ - record("plan", ...) and record("plan-30", ...) must merge - into a single "plan" entry, not two separate entries. - """ - log = DefaultTokenLog() - usage = { - "input_tokens": 100, - "output_tokens": 50, + }, + loc_insertions=50, + loc_deletions=20, + ) + log.record( + "implement", + { + "input_tokens": 200, + "output_tokens": 20, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 0, - } + }, + loc_insertions=30, + loc_deletions=5, + ) + report = log.get_report() + assert report[0]["loc_insertions"] == 80 + assert report[0]["loc_deletions"] == 25 - log.record("plan", usage) - log.record("plan-30", usage) - report = log.get_report() - assert len(report) == 1 - assert report[0]["step_name"] == "plan" - assert report[0]["invocation_count"] == 2 - - def test_non_numeric_suffix_is_not_stripped(self): - """ - "open-pr" must not be stripped to "open" — only trailing -N is normalized. - """ - log = DefaultTokenLog() - usage = { - "input_tokens": 100, - "output_tokens": 50, +# T-LOC-3 +def test_token_log_compute_total_includes_loc(): + """compute_total() aggregates loc_insertions and loc_deletions.""" + log = DefaultTokenLog() + log.record( + "implement", + { + "input_tokens": 0, + "output_tokens": 0, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 0, - } - log.record("open-pr", usage) - - report = log.get_report() - assert report[0]["step_name"] == "open-pr" - - -def test_load_from_log_dir_normalizes_suffixed_step_names(tmp_path): - """ - Sessions written to disk with step_name="plan-30" must be recovered - under the canonical key "plan", aggregating with other plan-N sessions. - """ - import json - - log_dir = tmp_path / "logs" - sessions_dir = log_dir / "sessions" - cwd = str(tmp_path / "pipeline") - timestamp = "2026-03-21T00:00:00+00:00" - - for step_name, dir_name in [("plan-30", "s-30"), ("plan-31", "s-31")]: - session_dir = sessions_dir / dir_name - session_dir.mkdir(parents=True, exist_ok=True) - tu_data = { - "step_name": step_name, - "input_tokens": 100, - "output_tokens": 50, + }, + loc_insertions=100, + loc_deletions=30, + ) + log.record( + "test-fix", + { + "input_tokens": 0, + "output_tokens": 0, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 0, - "timing_seconds": 5.0, - } - (session_dir / "token_usage.json").write_text(json.dumps(tu_data)) - index_entry = {"dir_name": dir_name, "timestamp": timestamp, "cwd": cwd} - with (log_dir / "sessions.jsonl").open("a") as f: - f.write(json.dumps(index_entry) + "\n") - + }, + loc_insertions=20, + loc_deletions=5, + ) + total = log.compute_total() + assert total["loc_insertions"] == 120 + assert total["loc_deletions"] == 35 + + +# T-LOC-4 +def test_token_log_load_from_log_dir_reads_loc(tmp_path): + """load_from_log_dir() reads loc_insertions / loc_deletions from token_usage.json.""" + sessions_dir = tmp_path / "sessions" / "sess-abc" + sessions_dir.mkdir(parents=True) + (tmp_path / "sessions.jsonl").write_text( + json.dumps( + { + "dir_name": "sess-abc", + "kitchen_id": "k1", + "cwd": "/repo", + "order_id": "", + "campaign_id": "", + } + ) + + "\n" + ) + (sessions_dir / "token_usage.json").write_text( + json.dumps( + { + "step_name": "implement", + "input_tokens": 100, + "output_tokens": 10, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + "timing_seconds": 5.0, + "order_id": "", + "loc_insertions": 42, + "loc_deletions": 8, + } + ) + ) log = DefaultTokenLog() - n = log.load_from_log_dir(str(log_dir), cwd_filter=cwd) - - assert n == 2 + log.load_from_log_dir(tmp_path) report = log.get_report() - assert len(report) == 1, f"Expected 1 canonical entry, got: {[e['step_name'] for e in report]}" - assert report[0]["step_name"] == "plan" - assert report[0]["invocation_count"] == 2 - assert report[0]["input_tokens"] == 200 + assert report[0]["loc_insertions"] == 42 + assert report[0]["loc_deletions"] == 8 -class TestOrderIdScoping: - """Group A: order_id scoping for DefaultTokenLog.""" +# T-LOC-5 +def test_token_log_load_from_log_dir_missing_loc_defaults_to_zero(tmp_path): + """Old token_usage.json without loc fields loads with loc_insertions=0, loc_deletions=0.""" + sessions_dir = tmp_path / "sessions" / "sess-old" + sessions_dir.mkdir(parents=True) + (tmp_path / "sessions.jsonl").write_text( + json.dumps( + { + "dir_name": "sess-old", + "kitchen_id": "k1", + "cwd": "/repo", + "order_id": "", + "campaign_id": "", + } + ) + + "\n" + ) + (sessions_dir / "token_usage.json").write_text( + json.dumps( + { + "step_name": "plan", + "input_tokens": 500, + "output_tokens": 50, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + "timing_seconds": 3.0, + "order_id": "", + # NO loc_insertions or loc_deletions + } + ) + ) + log = DefaultTokenLog() + log.load_from_log_dir(tmp_path) + report = log.get_report() + assert report[0]["loc_insertions"] == 0 + assert report[0]["loc_deletions"] == 0 - def _make_usage(self, **overrides: int) -> dict: - base = { - "input_tokens": 100, - "output_tokens": 50, - "cache_creation_input_tokens": 10, - "cache_read_input_tokens": 5, - } - return {**base, **overrides} - def test_record_with_order_id_creates_scoped_entry(self): - """A-1: two different order_ids produce separate entries, no cross-contamination.""" - log = DefaultTokenLog() - log.record("plan", self._make_usage(), order_id="issue-185") - log.record("plan", self._make_usage(), order_id="issue-186") - - report_185 = log.get_report(order_id="issue-185") - report_186 = log.get_report(order_id="issue-186") - - assert len(report_185) == 1 - assert len(report_186) == 1 - assert report_185[0]["step_name"] == "plan" - assert report_186[0]["step_name"] == "plan" - # No cross-contamination: each order has invocation_count=1 - assert report_185[0]["invocation_count"] == 1 - assert report_186[0]["invocation_count"] == 1 - - def test_same_step_different_orders_do_not_contaminate(self): - """A-2: same step name in two orders does not aggregate to one entry.""" - log = DefaultTokenLog() - log.record("plan", self._make_usage(), order_id="issue-185") - log.record("plan", self._make_usage(), order_id="issue-186") +# --------------------------------------------------------------------------- +# T-PEAK: peak_context and turn_count +# --------------------------------------------------------------------------- - # get_report() with no filter returns both entries - all_entries = log.get_report() - assert len(all_entries) == 1 # aggregated by step_name → 1 canonical entry - # But the aggregate invocation_count is 2 (one per order) - assert all_entries[0]["invocation_count"] == 2 - def test_get_report_order_id_filter_isolates_by_order(self): - """A-3: get_report(order_id='A') returns only A's entries; B's are absent.""" - log = DefaultTokenLog() - log.record("plan", self._make_usage(input_tokens=111), order_id="A") - log.record("implement", self._make_usage(input_tokens=222), order_id="B") +def test_token_entry_has_peak_context_and_turn_count_fields(): + e = TokenEntry(step_name="plan") + assert e.peak_context == 0 + assert e.turn_count == 0 - report_a = log.get_report(order_id="A") - report_b = log.get_report(order_id="B") - assert len(report_a) == 1 - assert report_a[0]["step_name"] == "plan" - assert report_a[0]["input_tokens"] == 111 +def test_record_peak_context_uses_max(): + log = DefaultTokenLog() + log.record("impl", {"cache_read_input_tokens": 100, "peak_context": 50000, "turn_count": 10}) + log.record("impl", {"cache_read_input_tokens": 200, "peak_context": 80000, "turn_count": 15}) + report = log.get_report() + assert report[0]["peak_context"] == 80000 - assert len(report_b) == 1 - assert report_b[0]["step_name"] == "implement" - assert report_b[0]["input_tokens"] == 222 - def test_get_report_no_filter_aggregates_all_orders(self): - """A-4: get_report() with no order_id aggregates across all orders (backward compat).""" - log = DefaultTokenLog() - log.record("plan", self._make_usage(input_tokens=100), order_id="A") - log.record("plan", self._make_usage(input_tokens=200), order_id="B") - - all_entries = log.get_report() - assert len(all_entries) == 1 - assert all_entries[0]["step_name"] == "plan" - assert all_entries[0]["input_tokens"] == 300 # 100 + 200 - assert all_entries[0]["invocation_count"] == 2 +def test_record_turn_count_sums(): + log = DefaultTokenLog() + log.record("impl", {"cache_read_input_tokens": 100, "peak_context": 50000, "turn_count": 10}) + log.record("impl", {"cache_read_input_tokens": 200, "peak_context": 80000, "turn_count": 15}) + report = log.get_report() + assert report[0]["turn_count"] == 25 - def test_compute_total_order_id_filter(self): - """A-5: compute_total(order_id='issue-185') sums only that order's tokens.""" - log = DefaultTokenLog() - log.record("plan", self._make_usage(input_tokens=100), order_id="issue-185") - log.record("plan", self._make_usage(input_tokens=200), order_id="issue-186") - total_185 = log.compute_total(order_id="issue-185") - assert total_185["input_tokens"] == 100 - assert total_185["output_tokens"] == 50 +def test_compute_total_peak_context_is_max_across_steps(): + log = DefaultTokenLog() + log.record("plan", {"cache_read_input_tokens": 100, "peak_context": 40000, "turn_count": 5}) + log.record("impl", {"cache_read_input_tokens": 200, "peak_context": 70000, "turn_count": 18}) + total = log.compute_total() + assert total["peak_context"] == 70000 + assert total["turn_count"] == 23 + + +def test_load_from_log_dir_reads_peak_context_and_turn_count(tmp_path): + sessions_dir = tmp_path / "sessions" / "sess-001" + sessions_dir.mkdir(parents=True) + (tmp_path / "sessions.jsonl").write_text( + json.dumps({"dir_name": "sess-001", "timestamp": "2026-03-07T00:00:00+00:00"}) + "\n" + ) + (sessions_dir / "token_usage.json").write_text( + json.dumps( + { + "step_name": "impl", + "input_tokens": 100, + "output_tokens": 50, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 200, + "timing_seconds": 5.0, + "order_id": "", + "peak_context": 60000, + "turn_count": 12, + } + ) + ) + log = DefaultTokenLog() + log.load_from_log_dir(tmp_path) + report = log.get_report() + assert report[0]["peak_context"] == 60000 + assert report[0]["turn_count"] == 12 - def test_compute_total_no_filter_aggregates_all(self): - """A-6: compute_total() with no filter aggregates all orders (backward compat).""" - log = DefaultTokenLog() - log.record("plan", self._make_usage(input_tokens=100), order_id="issue-185") - log.record("plan", self._make_usage(input_tokens=200), order_id="issue-186") - total = log.compute_total() - assert total["input_tokens"] == 300 +def test_load_from_log_dir_missing_peak_context_defaults_to_zero(tmp_path): + sessions_dir = tmp_path / "sessions" / "sess-old" + sessions_dir.mkdir(parents=True) + (tmp_path / "sessions.jsonl").write_text( + json.dumps({"dir_name": "sess-old", "timestamp": "2026-03-07T00:00:00+00:00"}) + "\n" + ) + (sessions_dir / "token_usage.json").write_text( + json.dumps( + { + "step_name": "plan", + "input_tokens": 500, + "output_tokens": 50, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + "timing_seconds": 3.0, + "order_id": "", + } + ) + ) + log = DefaultTokenLog() + log.load_from_log_dir(tmp_path) + report = log.get_report() + assert report[0]["peak_context"] == 0 + assert report[0]["turn_count"] == 0 - def test_unscoped_record_aggregates_with_no_filter(self): - """A-7: unscoped record and scoped record both appear in get_report() with no filter.""" - log = DefaultTokenLog() - log.record("plan", self._make_usage(input_tokens=100)) # no order_id - log.record("plan", self._make_usage(input_tokens=200), order_id="X") - all_entries = log.get_report() - assert len(all_entries) == 1 # same step_name → aggregated - assert all_entries[0]["invocation_count"] == 2 - assert all_entries[0]["input_tokens"] == 300 +def test_token_log_record_accepts_resolved_label(): + """record() stores and returns entries keyed by non-recipe labels (e.g. dispatch:id).""" + log = DefaultTokenLog() + log.record( + "dispatch:abc-123", + {"input_tokens": 100, "output_tokens": 50, "cache_read_input_tokens": 80}, + ) + report = log.get_report() + assert len(report) == 1 + assert report[0]["step_name"] == "dispatch:abc-123" + assert report[0]["input_tokens"] == 100 + assert report[0]["cache_read_input_tokens"] == 80 + + +def test_load_from_log_dir_restores_orchestrator_sessions(tmp_path): + """load_from_log_dir reads token_usage.json written with session_label key (L2 sessions).""" + _write_session( + tmp_path, + "dispatch-s001", + { + "session_label": "(ad-hoc)", + "input_tokens": 200, + "output_tokens": 100, + "cache_creation_input_tokens": 30, + "cache_read_input_tokens": 70, + "timing_seconds": 5.0, + }, + ) + log = DefaultTokenLog() + n = log.load_from_log_dir(tmp_path) + assert n == 1 + report = log.get_report() + assert len(report) == 1 + assert report[0]["input_tokens"] == 200 + assert report[0]["cache_creation_input_tokens"] == 30 + assert report[0]["cache_read_input_tokens"] == 70 diff --git a/tests/pipeline/test_tokens_filters.py b/tests/pipeline/test_tokens_filters.py new file mode 100644 index 0000000000..6e1be81804 --- /dev/null +++ b/tests/pipeline/test_tokens_filters.py @@ -0,0 +1,484 @@ +"""Tests for pipeline.tokens — cwd filter, step name normalization, order/campaign ID scoping.""" + +from __future__ import annotations + +import json +from pathlib import Path + +import pytest + +from autoskillit.pipeline.tokens import DefaultTokenLog + +pytestmark = [pytest.mark.layer("pipeline"), pytest.mark.small] + + +# --------------------------------------------------------------------------- +# cwd_filter tests +# --------------------------------------------------------------------------- + + +def _write_session_cwd( + log_root: Path, + dir_name: str, + tu_data: dict, + cwd: str, + timestamp: str = "2026-03-07T00:00:00+00:00", +) -> None: + """Write a session with a cwd field in the sessions.jsonl index.""" + session_dir = log_root / "sessions" / dir_name + session_dir.mkdir(parents=True, exist_ok=True) + (session_dir / "token_usage.json").write_text(json.dumps(tu_data)) + index_entry = { + "dir_name": dir_name, + "timestamp": timestamp, + "session_id": dir_name, + "cwd": cwd, + } + with (log_root / "sessions.jsonl").open("a") as f: + f.write(json.dumps(index_entry) + "\n") + + +class TestLoadFromLogDirCwdFilter: + """Tests for cwd_filter parameter on DefaultTokenLog.load_from_log_dir.""" + + _PLAN_DATA = { + "step_name": "plan", + "input_tokens": 100, + "output_tokens": 50, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + "timing_seconds": 10.0, + } + _IMPL_DATA = { + "step_name": "implement", + "input_tokens": 200, + "output_tokens": 80, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + "timing_seconds": 20.0, + } + + def test_cwd_filter_isolates_to_matching_cwd(self, tmp_path): + """cwd_filter loads only sessions whose cwd matches the given path.""" + _write_session_cwd(tmp_path, "sess-A", self._PLAN_DATA, "/runs/pipeline-A") + _write_session_cwd(tmp_path, "sess-B", self._IMPL_DATA, "/runs/pipeline-B") + + log = DefaultTokenLog() + count = log.load_from_log_dir(tmp_path, cwd_filter="/runs/pipeline-A") + assert count == 1, "Should only load the session whose cwd matches" + steps = log.get_report() + assert len(steps) == 1 + assert steps[0]["step_name"] == "plan" + assert steps[0]["input_tokens"] == 100 + + def test_cwd_filter_empty_loads_all(self, tmp_path): + """Empty cwd_filter loads all sessions (backward compatibility).""" + _write_session_cwd(tmp_path, "sess-A", self._PLAN_DATA, "/runs/pipeline-A") + _write_session_cwd(tmp_path, "sess-B", self._IMPL_DATA, "/runs/pipeline-B") + + log = DefaultTokenLog() + count = log.load_from_log_dir(tmp_path, cwd_filter="") + assert count == 2 + + def test_cwd_filter_no_matches_returns_zero(self, tmp_path): + """cwd_filter that matches nothing returns 0 and leaves the log empty.""" + _write_session_cwd(tmp_path, "sess-A", self._PLAN_DATA, "/runs/pipeline-A") + _write_session_cwd(tmp_path, "sess-B", self._IMPL_DATA, "/runs/pipeline-B") + + log = DefaultTokenLog() + count = log.load_from_log_dir(tmp_path, cwd_filter="/runs/nonexistent") + assert count == 0 + assert log.get_report() == [] + + def test_cwd_filter_default_loads_all(self, tmp_path): + """Calling load_from_log_dir without cwd_filter loads all sessions.""" + _write_session_cwd(tmp_path, "sess-A", self._PLAN_DATA, "/runs/pipeline-A") + _write_session_cwd(tmp_path, "sess-B", self._IMPL_DATA, "/runs/pipeline-B") + + log = DefaultTokenLog() + count = log.load_from_log_dir(tmp_path) + assert count == 2 + + +# --------------------------------------------------------------------------- +# Step name normalization tests +# --------------------------------------------------------------------------- + + +class TestCanonicalStepName: + """canonical_step_name() must strip trailing -N suffixes.""" + + @pytest.mark.parametrize( + "raw,expected", + [ + ("plan", "plan"), + ("plan-30", "plan"), + ("implement-31", "implement"), + ("open_pr-28", "open_pr"), + ("retry_worktree", "retry_worktree"), # underscore-separated, no hyphen to strip + ("audit_impl", "audit_impl"), # underscore, no hyphen + ("open-pr", "open-pr"), # hyphen + non-numeric — not stripped + ("step-2", "step"), # two-char suffix stripped + ("step-123", "step"), # multi-digit stripped + ("plan-30-retry", "plan-30-retry"), # ends non-numeric — not stripped + ("", ""), # empty string safe + ], + ) + def test_strips_trailing_numeric_suffix(self, raw, expected): + from autoskillit.pipeline.tokens import canonical_step_name + + assert canonical_step_name(raw) == expected + + +class TestTokenLogStepNameNormalization: + def test_suffixed_step_names_aggregate_to_canonical(self): + """ + record("plan-30", ...) and record("plan-31", ...) must produce + a single entry keyed by "plan", with invocation_count=2. + """ + log = DefaultTokenLog() + usage = { + "input_tokens": 100, + "output_tokens": 50, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + } + + log.record("plan-30", usage) + log.record("plan-31", usage) + + report = log.get_report() + assert len(report) == 1, ( + f"Expected 1 canonical entry, got: {[e['step_name'] for e in report]}" + ) + entry = report[0] + assert entry["step_name"] == "plan" + assert entry["invocation_count"] == 2 + assert entry["input_tokens"] == 200 + + def test_canonical_name_and_suffixed_name_merge(self): + """ + record("plan", ...) and record("plan-30", ...) must merge + into a single "plan" entry, not two separate entries. + """ + log = DefaultTokenLog() + usage = { + "input_tokens": 100, + "output_tokens": 50, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + } + + log.record("plan", usage) + log.record("plan-30", usage) + + report = log.get_report() + assert len(report) == 1 + assert report[0]["step_name"] == "plan" + assert report[0]["invocation_count"] == 2 + + def test_non_numeric_suffix_is_not_stripped(self): + """ + "open-pr" must not be stripped to "open" — only trailing -N is normalized. + """ + log = DefaultTokenLog() + usage = { + "input_tokens": 100, + "output_tokens": 50, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + } + log.record("open-pr", usage) + + report = log.get_report() + assert report[0]["step_name"] == "open-pr" + + +def test_load_from_log_dir_normalizes_suffixed_step_names(tmp_path): + """ + Sessions written to disk with step_name="plan-30" must be recovered + under the canonical key "plan", aggregating with other plan-N sessions. + """ + import json + + log_dir = tmp_path / "logs" + sessions_dir = log_dir / "sessions" + cwd = str(tmp_path / "pipeline") + timestamp = "2026-03-21T00:00:00+00:00" + + for step_name, dir_name in [("plan-30", "s-30"), ("plan-31", "s-31")]: + session_dir = sessions_dir / dir_name + session_dir.mkdir(parents=True, exist_ok=True) + tu_data = { + "step_name": step_name, + "input_tokens": 100, + "output_tokens": 50, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + "timing_seconds": 5.0, + } + (session_dir / "token_usage.json").write_text(json.dumps(tu_data)) + index_entry = {"dir_name": dir_name, "timestamp": timestamp, "cwd": cwd} + with (log_dir / "sessions.jsonl").open("a") as f: + f.write(json.dumps(index_entry) + "\n") + + log = DefaultTokenLog() + n = log.load_from_log_dir(str(log_dir), cwd_filter=cwd) + + assert n == 2 + report = log.get_report() + assert len(report) == 1, f"Expected 1 canonical entry, got: {[e['step_name'] for e in report]}" + assert report[0]["step_name"] == "plan" + assert report[0]["invocation_count"] == 2 + assert report[0]["input_tokens"] == 200 + + +class TestOrderIdScoping: + """Group A: order_id scoping for DefaultTokenLog.""" + + def _make_usage(self, **overrides: int) -> dict: + base = { + "input_tokens": 100, + "output_tokens": 50, + "cache_creation_input_tokens": 10, + "cache_read_input_tokens": 5, + } + return {**base, **overrides} + + def test_record_with_order_id_creates_scoped_entry(self): + """A-1: two different order_ids produce separate entries, no cross-contamination.""" + log = DefaultTokenLog() + log.record("plan", self._make_usage(), order_id="issue-185") + log.record("plan", self._make_usage(), order_id="issue-186") + + report_185 = log.get_report(order_id="issue-185") + report_186 = log.get_report(order_id="issue-186") + + assert len(report_185) == 1 + assert len(report_186) == 1 + assert report_185[0]["step_name"] == "plan" + assert report_186[0]["step_name"] == "plan" + # No cross-contamination: each order has invocation_count=1 + assert report_185[0]["invocation_count"] == 1 + assert report_186[0]["invocation_count"] == 1 + + def test_unfiltered_report_aggregates_across_order_ids(self): + """A-2: unfiltered report aggregates same step across orders into one entry.""" + log = DefaultTokenLog() + log.record("plan", self._make_usage(), order_id="issue-185") + log.record("plan", self._make_usage(), order_id="issue-186") + + # get_report() with no filter returns both entries + all_entries = log.get_report() + assert len(all_entries) == 1 # aggregated by step_name → 1 canonical entry + # But the aggregate invocation_count is 2 (one per order) + assert all_entries[0]["invocation_count"] == 2 + + def test_get_report_order_id_filter_isolates_by_order(self): + """A-3: get_report(order_id='A') returns only A's entries; B's are absent.""" + log = DefaultTokenLog() + log.record("plan", self._make_usage(input_tokens=111), order_id="A") + log.record("implement", self._make_usage(input_tokens=222), order_id="B") + + report_a = log.get_report(order_id="A") + report_b = log.get_report(order_id="B") + + assert len(report_a) == 1 + assert report_a[0]["step_name"] == "plan" + assert report_a[0]["input_tokens"] == 111 + + assert len(report_b) == 1 + assert report_b[0]["step_name"] == "implement" + assert report_b[0]["input_tokens"] == 222 + + def test_get_report_no_filter_aggregates_all_orders(self): + """A-4: get_report() with no order_id aggregates across all orders (backward compat).""" + log = DefaultTokenLog() + log.record("plan", self._make_usage(input_tokens=100), order_id="A") + log.record("plan", self._make_usage(input_tokens=200), order_id="B") + + all_entries = log.get_report() + assert len(all_entries) == 1 + assert all_entries[0]["step_name"] == "plan" + assert all_entries[0]["input_tokens"] == 300 # 100 + 200 + assert all_entries[0]["invocation_count"] == 2 + + def test_compute_total_order_id_filter(self): + """A-5: compute_total(order_id='issue-185') sums only that order's tokens.""" + log = DefaultTokenLog() + log.record("plan", self._make_usage(input_tokens=100), order_id="issue-185") + log.record("plan", self._make_usage(input_tokens=200), order_id="issue-186") + + total_185 = log.compute_total(order_id="issue-185") + assert total_185["input_tokens"] == 100 + assert total_185["output_tokens"] == 50 + + def test_compute_total_no_filter_aggregates_all(self): + """A-6: compute_total() with no filter aggregates all orders (backward compat).""" + log = DefaultTokenLog() + log.record("plan", self._make_usage(input_tokens=100), order_id="issue-185") + log.record("plan", self._make_usage(input_tokens=200), order_id="issue-186") + + total = log.compute_total() + assert total["input_tokens"] == 300 + + def test_unscoped_record_aggregates_with_no_filter(self): + """A-7: unscoped record and scoped record both appear in get_report() with no filter.""" + log = DefaultTokenLog() + log.record("plan", self._make_usage(input_tokens=100)) # no order_id + log.record("plan", self._make_usage(input_tokens=200), order_id="X") + + all_entries = log.get_report() + assert len(all_entries) == 1 # same step_name → aggregated + assert all_entries[0]["invocation_count"] == 2 + assert all_entries[0]["input_tokens"] == 300 + + +# --- Group N: campaign_id_filter tests --- + +_TOKEN_DATA = { + "step_name": "plan", + "input_tokens": 10, + "output_tokens": 5, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + "timing_seconds": 1.0, +} + + +def _write_token_session_cid( + log_root: Path, + dir_name: str, + campaign_id: str = "", + timestamp: str = "2026-04-20T00:00:00+00:00", +) -> None: + """Write a token_usage session with campaign_id in the sessions.jsonl index.""" + session_dir = log_root / "sessions" / dir_name + session_dir.mkdir(parents=True, exist_ok=True) + (session_dir / "token_usage.json").write_text(json.dumps(_TOKEN_DATA)) + index_entry = { + "dir_name": dir_name, + "timestamp": timestamp, + "session_id": dir_name, + "campaign_id": campaign_id, + } + with (log_root / "sessions.jsonl").open("a") as f: + f.write(json.dumps(index_entry) + "\n") + + +def test_token_load_campaign_id_filter(tmp_path): + """DefaultTokenLog.load_from_log_dir respects campaign_id_filter.""" + _write_token_session_cid(tmp_path, "c1-a", campaign_id="c1") + _write_token_session_cid(tmp_path, "c1-b", campaign_id="c1") + _write_token_session_cid(tmp_path, "c2-a", campaign_id="c2") + + log = DefaultTokenLog() + n = log.load_from_log_dir(tmp_path, campaign_id_filter="c1") + assert n == 2 + + log2 = DefaultTokenLog() + n2 = log2.load_from_log_dir(tmp_path, campaign_id_filter="c2") + assert n2 == 1 + + log3 = DefaultTokenLog() + n3 = log3.load_from_log_dir(tmp_path, campaign_id_filter="") + assert n3 == 3 + + +# --- Group B: order_id_filter tests --- + + +def _write_session_order_id( + log_root: Path, + dir_name: str, + step_name: str, + order_id: str, + cwd: str = "/clone/test", + timestamp: str = "2026-05-01T00:00:00+00:00", +) -> None: + """Write a token_usage session with order_id in the sessions.jsonl index.""" + session_dir = log_root / "sessions" / dir_name + session_dir.mkdir(parents=True, exist_ok=True) + tu_data = { + "step_name": step_name, + "input_tokens": 100, + "output_tokens": 50, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + "timing_seconds": 5.0, + "order_id": order_id, + } + (session_dir / "token_usage.json").write_text(json.dumps(tu_data)) + index_entry = { + "dir_name": dir_name, + "timestamp": timestamp, + "session_id": dir_name, + "cwd": cwd, + "order_id": order_id, + } + with (log_root / "sessions.jsonl").open("a") as f: + f.write(json.dumps(index_entry) + "\n") + + +def test_load_from_log_dir_order_id_filter_cross_cwd(tmp_path): + """Sessions from two cwd paths with same order_id are ALL loaded with order_id_filter.""" + _write_session_order_id(tmp_path, "s-a", "rectify", order_id="issue-42", cwd="/clone-A") + _write_session_order_id(tmp_path, "s-b", "implement", order_id="issue-42", cwd="/clone-B") + _write_session_order_id(tmp_path, "s-c", "review", order_id="issue-99", cwd="/clone-B") + + log = DefaultTokenLog() + n = log.load_from_log_dir(tmp_path, order_id_filter="issue-42") + + assert n == 2, "order_id_filter should load both clones for the same order" + steps = log.get_report(order_id="issue-42") + step_names = {s["step_name"] for s in steps} + assert "rectify" in step_names + assert "implement" in step_names + assert "review" not in step_names + + +def _write_token_session_dispatch_id( + log_root: Path, + dir_name: str, + dispatch_id: str, + timestamp: str = "2026-05-01T00:00:00+00:00", +) -> None: + """Write a token_usage session with dispatch_id in the index.""" + session_dir = log_root / "sessions" / dir_name + session_dir.mkdir(parents=True, exist_ok=True) + tu_data = { + "step_name": "implement", + "input_tokens": 100, + "output_tokens": 50, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + "timing_seconds": 5.0, + } + (session_dir / "token_usage.json").write_text(json.dumps(tu_data)) + index_entry = { + "dir_name": dir_name, + "timestamp": timestamp, + "session_id": dir_name, + "dispatch_id": dispatch_id, + } + with (log_root / "sessions.jsonl").open("a") as f: + f.write(json.dumps(index_entry) + "\n") + + +def test_token_load_dispatch_id_filter(tmp_path): + """DefaultTokenLog.load_from_log_dir respects dispatch_id_filter.""" + _write_token_session_dispatch_id(tmp_path, "d1-a", dispatch_id="d1") + _write_token_session_dispatch_id(tmp_path, "d1-b", dispatch_id="d1") + _write_token_session_dispatch_id(tmp_path, "d2-a", dispatch_id="d2") + + log = DefaultTokenLog() + n = log.load_from_log_dir(tmp_path, dispatch_id_filter="d1") + assert n == 2 + + log_d2 = DefaultTokenLog() + n_d2 = log_d2.load_from_log_dir(tmp_path, dispatch_id_filter="d2") + assert n_d2 == 1 + + log2 = DefaultTokenLog() + n2 = log2.load_from_log_dir(tmp_path, dispatch_id_filter="") + assert n2 == 3 diff --git a/tests/pipeline/test_tokens_model.py b/tests/pipeline/test_tokens_model.py new file mode 100644 index 0000000000..3cd3998700 --- /dev/null +++ b/tests/pipeline/test_tokens_model.py @@ -0,0 +1,166 @@ +"""Tests for pipeline.tokens — model capture, compute_model_totals, and load_from_log_dir model.""" + +from __future__ import annotations + +import json +from pathlib import Path + +import pytest + +from autoskillit.pipeline.tokens import DefaultTokenLog, TokenEntry + +pytestmark = [pytest.mark.layer("pipeline"), pytest.mark.small] + + +def _make_usage_with_model( + model: str, *, input_tokens: int = 100, output_tokens: int = 50 +) -> dict: + return { + "input_tokens": input_tokens, + "output_tokens": output_tokens, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + "model_breakdown": {model: {"input_tokens": input_tokens, "output_tokens": output_tokens}}, + } + + +def test_token_entry_has_model_field() -> None: + entry = TokenEntry(step_name="plan", model="claude-sonnet-4-6") + assert entry.model == "claude-sonnet-4-6" + d = entry.to_dict() + assert d["model"] == "claude-sonnet-4-6" + + +def test_token_entry_model_defaults_empty() -> None: + entry = TokenEntry(step_name="plan") + assert entry.model == "" + + +def test_record_captures_model_from_breakdown() -> None: + log = DefaultTokenLog() + log.record("plan", _make_usage_with_model("claude-sonnet-4-6")) + report = log.get_report() + assert report[0]["model"] == "claude-sonnet-4-6" + + +def test_record_no_breakdown_model_empty() -> None: + log = DefaultTokenLog() + log.record("plan", {"input_tokens": 100, "output_tokens": 50}) + report = log.get_report() + assert report[0]["model"] == "" + + +def test_record_model_set_on_first_invocation_only() -> None: + log = DefaultTokenLog() + log.record("plan", _make_usage_with_model("claude-sonnet-4-6")) + log.record("plan", _make_usage_with_model("MiniMax-M2.7")) + report = log.get_report() + assert report[0]["model"] == "claude-sonnet-4-6" + + +def test_compute_model_totals_single_model() -> None: + log = DefaultTokenLog() + log.record("plan", _make_usage_with_model("claude-sonnet-4-6", input_tokens=100)) + log.record("verify", _make_usage_with_model("claude-sonnet-4-6", input_tokens=200)) + totals = log.compute_model_totals() + assert len(totals) == 1 + assert totals[0]["model"] == "claude-sonnet-4-6" + assert totals[0]["step_count"] == 2 + assert totals[0]["input_tokens"] == 300 + + +def test_compute_model_totals_mixed_models() -> None: + log = DefaultTokenLog() + log.record("plan", _make_usage_with_model("claude-sonnet-4-6", input_tokens=100)) + log.record( + "implement", + _make_usage_with_model("MiniMax-M2.7", input_tokens=500, output_tokens=200), + ) + totals = log.compute_model_totals() + assert len(totals) == 2 + by_model = {t["model"]: t for t in totals} + assert by_model["claude-sonnet-4-6"]["step_count"] == 1 + assert by_model["MiniMax-M2.7"]["input_tokens"] == 500 + + +def test_compute_model_totals_no_model_returns_unknown() -> None: + log = DefaultTokenLog() + log.record("plan", {"input_tokens": 100, "output_tokens": 50}) + totals = log.compute_model_totals() + assert len(totals) == 1 + assert totals[0]["model"] == "unknown" + + +def test_compute_model_totals_empty_log() -> None: + log = DefaultTokenLog() + totals = log.compute_model_totals() + assert totals == [] + + +@pytest.mark.parametrize("use_session_label", [True, False]) +def test_load_from_log_dir_reads_model_identifier(tmp_path: Path, use_session_label: bool) -> None: + sessions_dir = tmp_path / "sessions" / "s1" + sessions_dir.mkdir(parents=True) + step_key = "session_label" if use_session_label else "step_name" + (sessions_dir / "token_usage.json").write_text( + json.dumps( + { + step_key: "plan", + "input_tokens": 100, + "output_tokens": 50, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + "timing_seconds": 10.0, + "model_identifier": "claude-sonnet-4-6", + } + ) + ) + index_entry = {"session_id": "s1", "dir_name": "s1", "timestamp": "2026-01-01T00:00:00Z"} + (tmp_path / "sessions.jsonl").write_text(json.dumps(index_entry) + "\n") + + log = DefaultTokenLog() + log.load_from_log_dir(tmp_path) + report = log.get_report() + assert len(report) == 1 + assert report[0]["model"] == "claude-sonnet-4-6" + + +def test_load_from_log_dir_no_model_identifier_defaults_empty(tmp_path: Path) -> None: + sessions_dir = tmp_path / "sessions" / "s1" + sessions_dir.mkdir(parents=True) + (sessions_dir / "token_usage.json").write_text( + json.dumps( + { + "session_label": "plan", + "input_tokens": 100, + "output_tokens": 50, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + "timing_seconds": 10.0, + } + ) + ) + index_entry = {"session_id": "s1", "dir_name": "s1", "timestamp": "2026-01-01T00:00:00Z"} + (tmp_path / "sessions.jsonl").write_text(json.dumps(index_entry) + "\n") + + log = DefaultTokenLog() + log.load_from_log_dir(tmp_path) + report = log.get_report() + assert report[0]["model"] == "" + + +def test_model_totals_single_model_one_row() -> None: + log = DefaultTokenLog() + for step in ["plan", "implement", "verify"]: + log.record(step, _make_usage_with_model("claude-sonnet-4-6")) + totals = log.compute_model_totals() + assert len(totals) == 1 + assert totals[0]["step_count"] == 3 + + +def test_model_totals_mixed_providers() -> None: + log = DefaultTokenLog() + log.record("plan", _make_usage_with_model("claude-sonnet-4-6")) + log.record("implement", _make_usage_with_model("MiniMax-M2.7-highspeed", input_tokens=500)) + totals = log.compute_model_totals() + assert len(totals) == 2 diff --git a/tests/planner/CLAUDE.md b/tests/planner/CLAUDE.md new file mode 100644 index 0000000000..2480a574a0 --- /dev/null +++ b/tests/planner/CLAUDE.md @@ -0,0 +1,28 @@ +# planner/ + +Planner manifest, validation, compilation, and merge tests. + +## Files + +| File | Purpose | +|------|---------| +| `__init__.py` | empty | +| `conftest.py` | Planner test helpers | +| `test_compiler.py` | Tests for compile_plan callable | +| `test_consolidation.py` | Tests for autoskillit.planner.consolidation.consolidate_wps | +| `test_elaborate_wps_contract.py` | Contract conformance tests for planner-elaborate-wps skill registration | +| `test_manifests.py` | Planner manifests tests | +| `test_merge.py` | Planner merge tests | +| `test_pipeline_integration.py` | End-to-end pipeline integration tests | +| `test_planner.py` | Tests for the planner L1 subpackage scaffold | +| `test_refine_assignments_contract.py` | Contract conformance tests for planner-refine-assignments skill registration | +| `test_refine_phases_contract.py` | Contract conformance tests for planner-refine-phases skill registration | +| `test_refine_wps_contract.py` | Contract conformance tests for planner-refine-wps skill registration | +| `test_schema_conformance.py` | Schema conformance tests: SKILL.md-compliant data flows correctly through the pipeline | +| `test_sort_utils.py` | Tests for _natural_sort_key: numeric segment ordering, pure text, pure digits, empty string | +| `test_typed_dict_conformance.py` | TypedDict conformance: required-key sets, SKILL.md alignment, factory validation | +| `test_validation.py` | Tests for validate_plan callable | + +## Architecture Notes + +`conftest.py` provides shared planner test helpers. The `fixtures/` subdirectory contains YAML data files used by planner tests. diff --git a/tests/planner/__init__.py b/tests/planner/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/planner/conftest.py b/tests/planner/conftest.py new file mode 100644 index 0000000000..090288e78a --- /dev/null +++ b/tests/planner/conftest.py @@ -0,0 +1,69 @@ +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any + +from autoskillit.planner.schema import ( + validate_assignment_result, + validate_phase_result, + validate_wp_result, +) + + +def make_phase_result( + phase_number: int, *, name: str = "Test Phase", **overrides: Any +) -> dict[str, Any]: + data: dict[str, Any] = { + "id": f"P{phase_number}", + "name": name, + "goal": f"Goal for phase {phase_number}", + "scope": [], + "ordering": phase_number, + "relationship_notes": "", + "assignments_preview": [], + **overrides, + } + return validate_phase_result(data) + + +def make_assignment_result( + phase_number: int, assignment_number: int, **overrides: Any +) -> dict[str, Any]: + data: dict[str, Any] = { + "id": f"P{phase_number}-A{assignment_number}", + "name": f"Assignment {assignment_number}", + "phase_id": f"P{phase_number}", + "goal": "Test goal", + "technical_approach": "Test approach", + "proposed_work_packages": [], + **overrides, + } + return validate_assignment_result(data) + + +def make_wp_result(wp_id: str, *, allow_stub: bool = False, **overrides: Any) -> dict[str, Any]: + data: dict[str, Any] = { + "id": wp_id, + "name": f"WP {wp_id}", + "summary": "summary", + "goal": "goal", + "deliverables": [f"src/mod_{wp_id}.py"], + "technical_steps": ["step 1"], + "acceptance_criteria": ["criterion 1"], + "depends_on": [], + **overrides, + } + return validate_wp_result(data, allow_stub=allow_stub) + + +def write_json(path: Path, data: object) -> None: + """Write ``data`` as JSON to ``path``, creating parent dirs as needed.""" + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(data)) + + +def write_task_file(tmp_path: Path, content: str = "test task") -> str: + f = tmp_path / "task_input.md" + f.write_text(content) + return str(f) diff --git a/tests/planner/test_compiler.py b/tests/planner/test_compiler.py new file mode 100644 index 0000000000..4cb5724176 --- /dev/null +++ b/tests/planner/test_compiler.py @@ -0,0 +1,408 @@ +"""Tests for compile_plan callable.""" + +from __future__ import annotations + +import json +from pathlib import Path + +import pytest +import structlog.testing + +from autoskillit.planner.compiler import _render_issue_body, compile_plan +from tests.planner.conftest import ( + make_assignment_result, + make_phase_result, + make_wp_result, + write_json, + write_task_file, +) + +pytestmark = [pytest.mark.layer("planner"), pytest.mark.small, pytest.mark.feature("planner")] + + +# --------------------------------------------------------------------------- +# Fixture helpers +# --------------------------------------------------------------------------- + + +def _make_valid_output_dir( + tmp_path: Path, + *, + num_phases: int = 1, + with_dep_graph: bool = False, + dependency_chain: bool = False, +) -> Path: + """Build a valid output_dir with validation.json verdict=pass.""" + phases_dir = tmp_path / "phases" + assigns_dir = tmp_path / "assignments" + wps_dir = tmp_path / "work_packages" + + for p in range(1, num_phases + 1): + write_json( + phases_dir / f"P{p}_result.json", + make_phase_result(p, name=f"Phase {p}"), + ) + write_json( + assigns_dir / f"P{p}-A1_result.json", + make_assignment_result( + p, + 1, + name=f"Test Assignment P{p}", + proposed_work_packages=[f"P{p}-A1-WP1"], + ), + ) + deps: list[str] = [] + if dependency_chain and p > 1: + deps = [f"P{p - 1}-A1-WP1"] + write_json( + wps_dir / f"P{p}-A1-WP1_result.json", + make_wp_result( + f"P{p}-A1-WP1", + name=f"WP P{p}-A1-WP1", + summary=f"Summary P{p}", + goal=f"Goal P{p}", + deliverables=[f"src/mod_p{p}.py"], + technical_steps=[f"step for p{p}"], + acceptance_criteria=[f"criterion for p{p}"], + depends_on=deps, + ), + ) + + manifest_items = [{"id": f"P{p}-A1-WP1", "status": "done"} for p in range(1, num_phases + 1)] + write_json( + wps_dir / "wp_manifest.json", {"pass_name": "work_packages", "items": manifest_items} + ) + + write_json( + tmp_path / "validation.json", + {"verdict": "pass", "findings": [], "warnings": [], "schema_version": 2}, + ) + + if with_dep_graph: + write_json( + tmp_path / "dep_graph.json", + { + "added_backward_deps": {}, + "forward_deps": {"P1-A1-WP1": ["P1-A1-WP2"]}, + }, + ) + + return tmp_path + + +def _make_chain_3_wps(tmp_path: Path) -> Path: + """3 WPs: WP1 → WP2 → WP3.""" + phases_dir = tmp_path / "phases" + assigns_dir = tmp_path / "assignments" + wps_dir = tmp_path / "work_packages" + + write_json( + phases_dir / "P1_result.json", + make_phase_result(1, name="Foundation"), + ) + write_json( + assigns_dir / "P1-A1_result.json", + make_assignment_result( + 1, + 1, + name="Test Assignment", + proposed_work_packages=["P1-A1-WP1", "P1-A1-WP2", "P1-A1-WP3"], + ), + ) + for i, deps in [(1, []), (2, ["P1-A1-WP1"]), (3, ["P1-A1-WP2"])]: + write_json( + wps_dir / f"P1-A1-WP{i}_result.json", + make_wp_result( + f"P1-A1-WP{i}", + name=f"WP {i}", + summary=f"Summary {i}", + goal=f"Goal {i}", + deliverables=[f"src/mod{i}.py"], + technical_steps=[f"step {i}"], + acceptance_criteria=[f"criterion {i}"], + depends_on=deps, + ), + ) + manifest_items = [{"id": f"P1-A1-WP{i}", "status": "done"} for i in range(1, 4)] + write_json( + wps_dir / "wp_manifest.json", {"pass_name": "work_packages", "items": manifest_items} + ) + write_json( + tmp_path / "validation.json", + {"verdict": "pass", "findings": [], "warnings": [], "schema_version": 2}, + ) + return tmp_path + + +# --------------------------------------------------------------------------- +# Tests +# --------------------------------------------------------------------------- + + +def test_compile_plan_topological_sort_order(tmp_path: Path) -> None: + _make_chain_3_wps(tmp_path) + compile_plan(str(tmp_path), write_task_file(tmp_path), "/src") + manifest = json.loads((tmp_path / "manifest.json").read_text()) + assert manifest["execution_order"] == ["P1-A1-WP1", "P1-A1-WP2", "P1-A1-WP3"] + + +def test_compile_plan_issue_body_sections(tmp_path: Path) -> None: + _make_valid_output_dir(tmp_path) + compile_plan(str(tmp_path), write_task_file(tmp_path), "/src") + issue = (tmp_path / "issues" / "P1-A1-WP1_issue.md").read_text() + for section in [ + "## Goal", + "## Context", + "## Deliverables", + "## Technical Steps", + "## Acceptance Criteria", + ]: + assert section in issue + + +def test_compile_plan_issue_body_context_fields(tmp_path: Path) -> None: + _make_valid_output_dir(tmp_path) + compile_plan(str(tmp_path), write_task_file(tmp_path), "/src") + issue = (tmp_path / "issues" / "P1-A1-WP1_issue.md").read_text() + assert "Phase 1 (Milestone: 1-phase-1)" in issue + assert "P1-A1 (Test Assignment P1)" in issue + + +def test_compile_plan_depends_on_cross_ref(tmp_path: Path) -> None: + _make_chain_3_wps(tmp_path) + compile_plan(str(tmp_path), write_task_file(tmp_path), "/src") + issue_wp2 = (tmp_path / "issues" / "P1-A1-WP2_issue.md").read_text() + assert "P1-A1-WP1" in issue_wp2 + + +def test_compile_plan_depended_on_by_cross_ref(tmp_path: Path) -> None: + _make_chain_3_wps(tmp_path) + write_json( + tmp_path / "dep_graph.json", + { + "added_backward_deps": {}, + "forward_deps": {"P1-A1-WP1": ["P1-A1-WP2"]}, + }, + ) + compile_plan(str(tmp_path), write_task_file(tmp_path), "/src") + issue_wp1 = (tmp_path / "issues" / "P1-A1-WP1_issue.md").read_text() + assert "P1-A1-WP2" in issue_wp1 + + +def test_compile_plan_milestones_one_per_phase(tmp_path: Path) -> None: + _make_valid_output_dir(tmp_path, num_phases=2, dependency_chain=True) + compile_plan(str(tmp_path), write_task_file(tmp_path), "/src") + milestones_data = json.loads((tmp_path / "milestones.json").read_text()) + assert len(milestones_data["milestones"]) == 2 + for entry in milestones_data["milestones"]: + assert "phase_number" in entry + assert "name" in entry + assert "name_slug" in entry + + +def test_compile_plan_plan_md_is_valid_markdown(tmp_path: Path) -> None: + _make_valid_output_dir(tmp_path) + result = compile_plan(str(tmp_path), write_task_file(tmp_path, "my task"), "/src") + plan_md = Path(result["plan_path"]).read_text() + assert plan_md.startswith("#") + assert "my task" in plan_md + + +def test_compile_plan_manifest_json_schema(tmp_path: Path) -> None: + _make_valid_output_dir(tmp_path) + compile_plan(str(tmp_path), write_task_file(tmp_path), "/src") + manifest = json.loads((tmp_path / "manifest.json").read_text()) + assert "task" in manifest + assert "source_dir" in manifest + assert "execution_order" in manifest + assert "issues" in manifest + assert isinstance(manifest["execution_order"], list) + assert isinstance(manifest["issues"], dict) + + +def test_compile_plan_plan_parts_matches_issue_file_count(tmp_path: Path) -> None: + _make_chain_3_wps(tmp_path) + result = compile_plan(str(tmp_path), write_task_file(tmp_path), "/src") + parts = [p for p in result["plan_parts"].split("\n") if p] + assert len(parts) == 3 + for part_path in parts: + assert Path(part_path).exists() + + +def test_compile_plan_return_values_are_strings(tmp_path: Path) -> None: + _make_valid_output_dir(tmp_path) + result = compile_plan(str(tmp_path), write_task_file(tmp_path), "/src") + assert all(isinstance(v, str) for v in result.values()) + + +def test_compile_plan_creates_issues_directory(tmp_path: Path) -> None: + _make_valid_output_dir(tmp_path) + compile_plan(str(tmp_path), write_task_file(tmp_path, "task"), "/src") + assert (tmp_path / "issues").is_dir() + + +class TestCompilePlanEdgeCases: + def test_nonpassing_verdict_returns_dict(self, tmp_path: Path) -> None: + _make_valid_output_dir(tmp_path) + (tmp_path / "validation.json").write_text( + json.dumps( + { + "verdict": "fail", + "findings": [{"message": "test", "severity": "error", "check": "test"}], + "warnings": [], + "schema_version": 2, + } + ) + ) + + with structlog.testing.capture_logs() as logs: + result = compile_plan(str(tmp_path), write_task_file(tmp_path, "t"), "s") + + assert {"plan_path", "plan_json_path", "plan_parts"} == result.keys() + warning_logs = [e for e in logs if e.get("log_level") == "warning"] + assert any( + "non-passing" in e.get("event", "") and e.get("verdict") == "fail" + for e in warning_logs + ), f"expected non-passing warning log, got: {warning_logs}" + + def test_wp_references_absent_phase_raises(self, tmp_path: Path) -> None: + _make_valid_output_dir(tmp_path) + wps_dir = tmp_path / "work_packages" + write_json(wps_dir / "P99-A1-WP1_result.json", make_wp_result("P99-A1-WP1")) + + with pytest.raises(RuntimeError, match="references phase"): + compile_plan(str(tmp_path), write_task_file(tmp_path, "t"), "s") + + def test_wp_references_absent_assignment_raises(self, tmp_path: Path) -> None: + _make_valid_output_dir(tmp_path) + wps_dir = tmp_path / "work_packages" + write_json(wps_dir / "P1-A99-WP1_result.json", make_wp_result("P1-A99-WP1")) + + with pytest.raises(RuntimeError, match="references assignment"): + compile_plan(str(tmp_path), write_task_file(tmp_path, "t"), "s") + + def test_malformed_wp_id_raises(self, tmp_path: Path) -> None: + _make_valid_output_dir(tmp_path) + wps_dir = tmp_path / "work_packages" + write_json( + wps_dir / "P1-A1-WP99_result.json", + {"id": "BADID", "name": "Bad", "deliverables": ["x.py"]}, + ) + + with pytest.raises(ValueError, match="Invalid WP id format"): + compile_plan(str(tmp_path), write_task_file(tmp_path, "t"), "s") + + +def test_render_issue_body_includes_research_section_when_recommended() -> None: + phase = make_phase_result(1) + assignment = make_assignment_result(1, 1) + wp = make_wp_result( + "P1-A1-WP1", + review_approach_recommended=True, + review_approach_reasoning="Multiple viable architectural approaches exist.", + ) + body = _render_issue_body(wp, phase, assignment) + assert "## Review Approach" in body + assert "review-approach recommended" in body + assert "Multiple viable architectural approaches exist." in body + + +def test_render_issue_body_omits_research_section_when_not_recommended() -> None: + phase = make_phase_result(1) + assignment = make_assignment_result(1, 1) + wp = make_wp_result("P1-A1-WP1") + body = _render_issue_body(wp, phase, assignment) + assert "## Review Approach" not in body + + +def test_compile_plan_merges_assessment_when_file_present(tmp_path: Path) -> None: + output_dir = _make_valid_output_dir( + tmp_path, num_phases=1, with_dep_graph=False, dependency_chain=False + ) + assessment = { + "schema_version": 1, + "assessments": [ + { + "wp_id": "P1-A1-WP1", + "review_approach_recommended": True, + "review_approach_reasoning": "Unfamiliar external API integration.", + } + ], + } + write_json(output_dir / "review_approach_assessment.json", assessment) + compile_plan( + str(output_dir), task_file_path=write_task_file(output_dir), source_dir=str(tmp_path) + ) + issue_body = (output_dir / "issues" / "P1-A1-WP1_issue.md").read_text() + assert "## Review Approach" in issue_body + assert "Unfamiliar external API integration." in issue_body + + +def test_compile_plan_omits_research_when_assessment_file_absent(tmp_path: Path) -> None: + output_dir = _make_valid_output_dir( + tmp_path, num_phases=1, with_dep_graph=False, dependency_chain=False + ) + compile_plan( + str(output_dir), task_file_path=write_task_file(output_dir), source_dir=str(tmp_path) + ) + issue_body = (output_dir / "issues" / "P1-A1-WP1_issue.md").read_text() + assert "## Review Approach" not in issue_body + + +def test_compile_plan_raises_on_malformed_assessment_file(tmp_path: Path) -> None: + output_dir = _make_valid_output_dir( + tmp_path, num_phases=1, with_dep_graph=False, dependency_chain=False + ) + (output_dir / "review_approach_assessment.json").write_text("not valid json") + with pytest.raises(RuntimeError, match="Malformed assessment file"): + compile_plan( + str(output_dir), task_file_path=write_task_file(output_dir), source_dir=str(tmp_path) + ) + + +def test_compile_plan_skips_assessment_entries_missing_wp_id(tmp_path: Path) -> None: + output_dir = _make_valid_output_dir( + tmp_path, num_phases=1, with_dep_graph=False, dependency_chain=False + ) + assessment = { + "schema_version": 1, + "assessments": [ + {"review_approach_recommended": True, "review_approach_reasoning": "no wp_id"}, + { + "wp_id": "P1-A1-WP1", + "review_approach_recommended": True, + "review_approach_reasoning": "valid entry", + }, + ], + } + write_json(output_dir / "review_approach_assessment.json", assessment) + compile_plan( + str(output_dir), task_file_path=write_task_file(output_dir), source_dir=str(tmp_path) + ) + issue_body = (output_dir / "issues" / "P1-A1-WP1_issue.md").read_text() + assert "## Review Approach" in issue_body + assert "valid entry" in issue_body + + +def test_compile_plan_reads_task_from_file(tmp_path: Path) -> None: + output_dir = _make_valid_output_dir(tmp_path) + task_file = tmp_path / "task_desc.txt" + task_file.write_text("Full task description from file") + result = compile_plan( + str(output_dir), task_file_path=str(task_file), source_dir="/src", task_label="label" + ) + plan_json = json.loads((output_dir / "plan.json").read_text()) + assert plan_json["task"] == "Full task description from file" + plan_md = Path(result["plan_path"]).read_text() + assert plan_md.startswith("# Plan: label") + + +def test_compile_plan_derives_label_when_not_provided(tmp_path: Path) -> None: + output_dir = _make_valid_output_dir(tmp_path) + task_file = tmp_path / "task_desc.txt" + task_file.write_text("inline task") + result = compile_plan(str(output_dir), task_file_path=str(task_file), source_dir="/src") + plan_json = json.loads((output_dir / "plan.json").read_text()) + assert plan_json["task"] == "inline task" + plan_md = Path(result["plan_path"]).read_text() + assert plan_md.startswith("# Plan: inline task") diff --git a/tests/planner/test_consolidation.py b/tests/planner/test_consolidation.py new file mode 100644 index 0000000000..38f6d96a0d --- /dev/null +++ b/tests/planner/test_consolidation.py @@ -0,0 +1,809 @@ +"""Tests for autoskillit.planner.consolidation.consolidate_wps.""" + +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any + +import pytest + +from autoskillit.planner.compiler import compile_plan +from autoskillit.planner.consolidation import consolidate_wps +from autoskillit.planner.validation import validate_plan +from tests.planner.conftest import ( + make_assignment_result, + make_phase_result, + make_wp_result, + write_json, + write_task_file, +) + +pytestmark = [pytest.mark.layer("planner"), pytest.mark.small, pytest.mark.feature("planner")] + + +def _make_refined_wps(tmp_path: Path, wps: list[dict[str, Any]]) -> Path: + doc = {"task": "Test task", "source_dir": "/src", "work_packages": wps, "schema_version": 1} + p = tmp_path / "refined_wps.json" + write_json(p, doc) + return p + + +def _make_manifest(consolidation_dir: Path, phase_id: str, groups: list[dict[str, Any]]) -> None: + consolidation_dir.mkdir(parents=True, exist_ok=True) + write_json( + consolidation_dir / f"{phase_id}_consolidation.json", + {"phase_id": phase_id, "groups": groups}, + ) + + +def test_passthrough_unchanged_when_no_manifests(tmp_path: Path) -> None: + wps = [make_wp_result(f"P1-A1-WP{i}") for i in range(1, 4)] + refined_path = _make_refined_wps(tmp_path, wps) + + result = consolidate_wps(refined_wps_path=str(refined_path), planner_dir=str(tmp_path)) + + consolidated = json.loads((tmp_path / "consolidated_wps.json").read_text()) + assert len(consolidated["work_packages"]) == 3 + assert {wp["id"] for wp in consolidated["work_packages"]} == { + "P1-A1-WP1", + "P1-A1-WP2", + "P1-A1-WP3", + } + index = json.loads((tmp_path / "work_packages" / "wp_index.json").read_text()) + assert {e["id"] for e in index} == {"P1-A1-WP1", "P1-A1-WP2", "P1-A1-WP3"} + # consolidate_wps returns str values (compatible with run_python result passing) + assert result["total_count"] == "3" + assert result["merged_count"] == "0" + + +def test_merge_union_fields(tmp_path: Path) -> None: + wp1 = make_wp_result( + "P1-A1-WP1", + deliverables=["src/a.py"], + acceptance_criteria=["criterion A"], + files_touched=["src/a.py"], + apis_defined=["api_a"], + apis_consumed=["ext_a"], + ) + wp2 = make_wp_result( + "P1-A1-WP2", + deliverables=["src/b.py"], + acceptance_criteria=["criterion B"], + files_touched=["src/b.py"], + apis_defined=["api_b"], + apis_consumed=["ext_b"], + ) + refined_path = _make_refined_wps(tmp_path, [wp1, wp2]) + consolidation_dir = tmp_path / "work_packages" / "consolidation" + _make_manifest( + consolidation_dir, + "P1", + [ + { + "merged_id": "P1-A1-WP1", + "source_wp_ids": ["P1-A1-WP1", "P1-A1-WP2"], + "merge_order": ["P1-A1-WP1", "P1-A1-WP2"], + "name": None, + "goal": None, + } + ], + ) + + result = consolidate_wps(refined_wps_path=str(refined_path), planner_dir=str(tmp_path)) + + consolidated = json.loads((tmp_path / "consolidated_wps.json").read_text()) + assert len(consolidated["work_packages"]) == 1 + merged = consolidated["work_packages"][0] + assert set(merged["deliverables"]) == {"src/a.py", "src/b.py"} + assert set(merged["acceptance_criteria"]) == {"criterion A", "criterion B"} + assert set(merged["files_touched"]) == {"src/a.py", "src/b.py"} + assert set(merged["apis_defined"]) == {"api_a", "api_b"} + assert set(merged["apis_consumed"]) == {"ext_a", "ext_b"} + assert result["total_count"] == "1" + assert result["merged_count"] == "1" + + +def test_merge_technical_steps_concatenated_in_merge_order(tmp_path: Path) -> None: + wp1 = make_wp_result("P1-A1-WP1", technical_steps=["step A"]) + wp2 = make_wp_result("P1-A1-WP2", technical_steps=["step B"]) + refined_path = _make_refined_wps(tmp_path, [wp1, wp2]) + consolidation_dir = tmp_path / "work_packages" / "consolidation" + _make_manifest( + consolidation_dir, + "P1", + [ + { + "merged_id": "P1-A1-WP1", + "source_wp_ids": ["P1-A1-WP1", "P1-A1-WP2"], + "merge_order": ["P1-A1-WP2", "P1-A1-WP1"], + "name": None, + "goal": None, + } + ], + ) + + consolidate_wps(refined_wps_path=str(refined_path), planner_dir=str(tmp_path)) + + consolidated = json.loads((tmp_path / "consolidated_wps.json").read_text()) + merged = consolidated["work_packages"][0] + assert merged["technical_steps"] == ["step B", "step A"] + + +def test_merge_name_and_goal_from_primary(tmp_path: Path) -> None: + wp1 = make_wp_result("P1-A1-WP1", name="Primary WP", goal="Primary goal") + wp2 = make_wp_result("P1-A1-WP2", name="Secondary WP", goal="Secondary goal") + refined_path = _make_refined_wps(tmp_path, [wp1, wp2]) + consolidation_dir = tmp_path / "work_packages" / "consolidation" + _make_manifest( + consolidation_dir, + "P1", + [ + { + "merged_id": "P1-A1-WP1", + "source_wp_ids": ["P1-A1-WP1", "P1-A1-WP2"], + "merge_order": ["P1-A1-WP1", "P1-A1-WP2"], + "name": None, + "goal": None, + } + ], + ) + + consolidate_wps(refined_wps_path=str(refined_path), planner_dir=str(tmp_path)) + + consolidated = json.loads((tmp_path / "consolidated_wps.json").read_text()) + merged = consolidated["work_packages"][0] + assert merged["name"] == "Primary WP" + assert merged["goal"] == "Primary goal" + + +def test_merge_name_override_from_manifest(tmp_path: Path) -> None: + wp1 = make_wp_result("P1-A1-WP1", name="Original WP1", goal="Original goal 1") + wp2 = make_wp_result("P1-A1-WP2") + refined_path = _make_refined_wps(tmp_path, [wp1, wp2]) + consolidation_dir = tmp_path / "work_packages" / "consolidation" + _make_manifest( + consolidation_dir, + "P1", + [ + { + "merged_id": "P1-A1-WP1", + "source_wp_ids": ["P1-A1-WP1", "P1-A1-WP2"], + "merge_order": ["P1-A1-WP1", "P1-A1-WP2"], + "name": "Combined file sharding refactor", + "goal": "Refactor all file-sharding paths", + } + ], + ) + + consolidate_wps(refined_wps_path=str(refined_path), planner_dir=str(tmp_path)) + + consolidated = json.loads((tmp_path / "consolidated_wps.json").read_text()) + merged = consolidated["work_packages"][0] + assert merged["name"] == "Combined file sharding refactor" + assert merged["goal"] == "Refactor all file-sharding paths" + + +def test_dep_rewriting_intra_group_removed(tmp_path: Path) -> None: + wp1 = make_wp_result("P1-A1-WP1", depends_on=["P1-A1-WP2"]) + wp2 = make_wp_result("P1-A1-WP2", depends_on=[]) + refined_path = _make_refined_wps(tmp_path, [wp1, wp2]) + consolidation_dir = tmp_path / "work_packages" / "consolidation" + _make_manifest( + consolidation_dir, + "P1", + [ + { + "merged_id": "P1-A1-WP1", + "source_wp_ids": ["P1-A1-WP1", "P1-A1-WP2"], + "merge_order": ["P1-A1-WP1", "P1-A1-WP2"], + "name": None, + "goal": None, + } + ], + ) + + consolidate_wps(refined_wps_path=str(refined_path), planner_dir=str(tmp_path)) + + consolidated = json.loads((tmp_path / "consolidated_wps.json").read_text()) + merged = consolidated["work_packages"][0] + assert merged["depends_on"] == [] + + +def test_dep_rewriting_source_to_merged_id(tmp_path: Path) -> None: + wp1 = make_wp_result("P1-A1-WP1") + wp2 = make_wp_result("P1-A1-WP2") + wp3 = make_wp_result("P1-A1-WP3", depends_on=["P1-A1-WP2"]) + refined_path = _make_refined_wps(tmp_path, [wp1, wp2, wp3]) + consolidation_dir = tmp_path / "work_packages" / "consolidation" + _make_manifest( + consolidation_dir, + "P1", + [ + { + "merged_id": "P1-A1-WP1", + "source_wp_ids": ["P1-A1-WP1", "P1-A1-WP2"], + "merge_order": ["P1-A1-WP1", "P1-A1-WP2"], + "name": None, + "goal": None, + }, + { + "merged_id": "P1-A1-WP3", + "source_wp_ids": ["P1-A1-WP3"], + "merge_order": ["P1-A1-WP3"], + "name": None, + "goal": None, + }, + ], + ) + + consolidate_wps(refined_wps_path=str(refined_path), planner_dir=str(tmp_path)) + + consolidated = json.loads((tmp_path / "consolidated_wps.json").read_text()) + wp3_out = next(wp for wp in consolidated["work_packages"] if wp["id"] == "P1-A1-WP3") + assert wp3_out["depends_on"] == ["P1-A1-WP1"] + + +def test_external_dep_preserved(tmp_path: Path) -> None: + wp1 = make_wp_result("P1-A1-WP1", depends_on=["P2-A1-WP1"]) + wp2 = make_wp_result("P1-A1-WP2") + refined_path = _make_refined_wps(tmp_path, [wp1, wp2]) + consolidation_dir = tmp_path / "work_packages" / "consolidation" + _make_manifest( + consolidation_dir, + "P1", + [ + { + "merged_id": "P1-A1-WP1", + "source_wp_ids": ["P1-A1-WP1", "P1-A1-WP2"], + "merge_order": ["P1-A1-WP1", "P1-A1-WP2"], + "name": None, + "goal": None, + } + ], + ) + + consolidate_wps(refined_wps_path=str(refined_path), planner_dir=str(tmp_path)) + + consolidated = json.loads((tmp_path / "consolidated_wps.json").read_text()) + merged = consolidated["work_packages"][0] + assert "P2-A1-WP1" in merged["depends_on"] + + +def test_wp_index_rebuilt_with_merged_ids(tmp_path: Path) -> None: + wps = [make_wp_result(f"P1-A1-WP{i}") for i in range(1, 4)] + refined_path = _make_refined_wps(tmp_path, wps) + consolidation_dir = tmp_path / "work_packages" / "consolidation" + _make_manifest( + consolidation_dir, + "P1", + [ + { + "merged_id": "P1-A1-WP1", + "source_wp_ids": ["P1-A1-WP1", "P1-A1-WP2"], + "merge_order": ["P1-A1-WP1", "P1-A1-WP2"], + "name": None, + "goal": None, + }, + { + "merged_id": "P1-A1-WP3", + "source_wp_ids": ["P1-A1-WP3"], + "merge_order": ["P1-A1-WP3"], + "name": None, + "goal": None, + }, + ], + ) + + consolidate_wps(refined_wps_path=str(refined_path), planner_dir=str(tmp_path)) + + index = json.loads((tmp_path / "work_packages" / "wp_index.json").read_text()) + assert len(index) == 2 + merged_entry = next(e for e in index if e["id"] == "P1-A1-WP1") + assert merged_entry["id"] == "P1-A1-WP1" + assert "name" in merged_entry + + +def test_consolidated_wps_path_returned(tmp_path: Path) -> None: + wps = [make_wp_result("P1-A1-WP1")] + refined_path = _make_refined_wps(tmp_path, wps) + + result = consolidate_wps(refined_wps_path=str(refined_path), planner_dir=str(tmp_path)) + + assert "consolidated_wps_path" in result + out_path = Path(result["consolidated_wps_path"]) + assert out_path.exists() + data = json.loads(out_path.read_text()) + assert "work_packages" in data + + +def test_multiple_phases_multiple_manifests(tmp_path: Path) -> None: + wps_p1 = [make_wp_result(f"P1-A1-WP{i}") for i in range(1, 3)] + wps_p2 = [make_wp_result(f"P2-A1-WP{i}") for i in range(1, 3)] + wps_p3 = [make_wp_result(f"P3-A1-WP{i}") for i in range(1, 3)] + all_wps = wps_p1 + wps_p2 + wps_p3 + refined_path = _make_refined_wps(tmp_path, all_wps) + consolidation_dir = tmp_path / "work_packages" / "consolidation" + for phase_id, prefix in [("P1", "P1-A1"), ("P2", "P2-A1"), ("P3", "P3-A1")]: + _make_manifest( + consolidation_dir, + phase_id, + [ + { + "merged_id": f"{prefix}-WP1", + "source_wp_ids": [f"{prefix}-WP1", f"{prefix}-WP2"], + "merge_order": [f"{prefix}-WP1", f"{prefix}-WP2"], + "name": None, + "goal": None, + } + ], + ) + + result = consolidate_wps(refined_wps_path=str(refined_path), planner_dir=str(tmp_path)) + + consolidated = json.loads((tmp_path / "consolidated_wps.json").read_text()) + assert len(consolidated["work_packages"]) == 3 + ids = {wp["id"] for wp in consolidated["work_packages"]} + assert ids == {"P1-A1-WP1", "P2-A1-WP1", "P3-A1-WP1"} + # Cross-phase deps must not be affected; absorbed source IDs must not appear + absorbed_ids = {"P1-A1-WP2", "P2-A1-WP2", "P3-A1-WP2"} + for wp in consolidated["work_packages"]: + assert not any(dep in absorbed_ids for dep in wp["depends_on"]) + assert result["merged_count"] == "3" + + +def test_missing_source_wp_in_manifest_raises(tmp_path: Path) -> None: + wp1 = make_wp_result("P1-A1-WP1") + refined_path = _make_refined_wps(tmp_path, [wp1]) + consolidation_dir = tmp_path / "work_packages" / "consolidation" + _make_manifest( + consolidation_dir, + "P1", + [ + { + "merged_id": "P1-A1-WP1", + "source_wp_ids": ["P1-A1-WP1", "P1-A1-WP99"], + "merge_order": ["P1-A1-WP1", "P1-A1-WP99"], + "name": None, + "goal": None, + } + ], + ) + + with pytest.raises(ValueError, match="unknown WP"): + consolidate_wps(refined_wps_path=str(refined_path), planner_dir=str(tmp_path)) + + +def test_consolidate_wps_rejects_merge_with_empty_deliverables(tmp_path: Path) -> None: + """Merging WPs where all sources have empty deliverables must raise ValueError.""" + wps = [ + make_wp_result("P1-A1-WP1", allow_stub=True, deliverables=[]), + make_wp_result("P1-A1-WP2", allow_stub=True, deliverables=[]), + ] + refined_path = _make_refined_wps(tmp_path, wps) + consolidation_dir = tmp_path / "work_packages" / "consolidation" + _make_manifest( + consolidation_dir, + "P1", + [ + { + "merged_id": "P1-A1-WP1", + "source_wp_ids": ["P1-A1-WP1", "P1-A1-WP2"], + "merge_order": ["P1-A1-WP1", "P1-A1-WP2"], + "name": None, + "goal": None, + } + ], + ) + + with pytest.raises(ValueError, match="has 0 deliverables"): + consolidate_wps(refined_wps_path=str(refined_path), planner_dir=str(tmp_path)) + + +def test_fallback_merges_same_assignment_shared_files(tmp_path: Path) -> None: + wps = [ + make_wp_result("P1-A1-WP1", files_touched=["src/config.yaml"]), + make_wp_result("P1-A1-WP2", files_touched=["src/config.yaml"]), + make_wp_result("P1-A1-WP3", files_touched=["src/config.yaml"]), + make_wp_result("P1-A1-WP4", files_touched=["src/other.py"]), + make_wp_result("P1-A1-WP5", files_touched=["src/other.py"]), + ] + refined_path = _make_refined_wps(tmp_path, wps) + + result = consolidate_wps(refined_wps_path=str(refined_path), planner_dir=str(tmp_path)) + + consolidated = json.loads((tmp_path / "consolidated_wps.json").read_text()) + output_ids = {wp["id"] for wp in consolidated["work_packages"]} + assert output_ids == {"P1-A1-WP1", "P1-A1-WP4"} + assert len(consolidated["work_packages"]) == 2 + assert result["merged_count"] == "2" + + +def test_fallback_skipped_when_manifest_has_merges(tmp_path: Path) -> None: + wps = [ + make_wp_result("P1-A1-WP1", files_touched=["src/config.yaml"]), + make_wp_result("P1-A1-WP2", files_touched=["src/config.yaml"]), + make_wp_result("P1-A1-WP3", files_touched=["src/config.yaml"]), + make_wp_result("P1-A1-WP4", files_touched=["src/other.py"]), + make_wp_result("P1-A1-WP5", files_touched=["src/other.py"]), + ] + refined_path = _make_refined_wps(tmp_path, wps) + consolidation_dir = tmp_path / "work_packages" / "consolidation" + _make_manifest( + consolidation_dir, + "P1", + [ + { + "merged_id": "P1-A1-WP1", + "source_wp_ids": ["P1-A1-WP1", "P1-A1-WP2"], + "merge_order": ["P1-A1-WP1", "P1-A1-WP2"], + "name": None, + "goal": None, + }, + { + "merged_id": "P1-A1-WP3", + "source_wp_ids": ["P1-A1-WP3"], + "merge_order": ["P1-A1-WP3"], + "name": None, + "goal": None, + }, + { + "merged_id": "P1-A1-WP4", + "source_wp_ids": ["P1-A1-WP4"], + "merge_order": ["P1-A1-WP4"], + "name": None, + "goal": None, + }, + { + "merged_id": "P1-A1-WP5", + "source_wp_ids": ["P1-A1-WP5"], + "merge_order": ["P1-A1-WP5"], + "name": None, + "goal": None, + }, + ], + ) + + result = consolidate_wps(refined_wps_path=str(refined_path), planner_dir=str(tmp_path)) + + consolidated = json.loads((tmp_path / "consolidated_wps.json").read_text()) + assert len(consolidated["work_packages"]) == 4 + assert result["merged_count"] == "1" + + +def test_fallback_skipped_when_wp_count_below_threshold(tmp_path: Path) -> None: + wps = [ + make_wp_result("P1-A1-WP1", files_touched=["src/config.yaml"]), + make_wp_result("P1-A1-WP2", files_touched=["src/config.yaml"]), + make_wp_result("P1-A1-WP3", files_touched=["src/config.yaml"]), + ] + refined_path = _make_refined_wps(tmp_path, wps) + + result = consolidate_wps(refined_wps_path=str(refined_path), planner_dir=str(tmp_path)) + + consolidated = json.loads((tmp_path / "consolidated_wps.json").read_text()) + assert len(consolidated["work_packages"]) == 3 + assert result["merged_count"] == "0" + + +def test_fallback_respects_dependency_ordering(tmp_path: Path) -> None: + wps = [ + make_wp_result("P1-A1-WP1", files_touched=["src/x.py"], depends_on=[]), + make_wp_result("P1-A1-WP2", files_touched=["src/x.py"], depends_on=["P1-A1-WP1"]), + make_wp_result("P1-A1-WP3", files_touched=["src/x.py"], depends_on=["P1-A1-WP2"]), + make_wp_result("P1-A1-WP4", files_touched=["src/y.py"], depends_on=[]), + make_wp_result("P1-A1-WP5", files_touched=["src/y.py"], depends_on=[]), + ] + refined_path = _make_refined_wps(tmp_path, wps) + + consolidate_wps(refined_wps_path=str(refined_path), planner_dir=str(tmp_path)) + + consolidated = json.loads((tmp_path / "consolidated_wps.json").read_text()) + output_ids = {wp["id"] for wp in consolidated["work_packages"]} + assert "P1-A1-WP2" not in output_ids + assert "P1-A1-WP3" not in output_ids + merged_a = next(wp for wp in consolidated["work_packages"] if wp["id"] == "P1-A1-WP1") + assert merged_a["id"] == "P1-A1-WP1" + assert "P1-A1-WP2" not in merged_a["depends_on"] + assert "P1-A1-WP3" not in merged_a["depends_on"] + + +def test_fallback_does_not_cross_assignment_boundary(tmp_path: Path) -> None: + wps = [ + make_wp_result("P1-A1-WP1", files_touched=["src/shared.py"]), + make_wp_result("P1-A1-WP2", files_touched=["src/shared.py"]), + make_wp_result("P1-A1-WP3", files_touched=["src/shared.py"]), + make_wp_result("P1-A2-WP1", files_touched=["src/shared.py"]), + make_wp_result("P1-A2-WP2", files_touched=["src/shared.py"]), + make_wp_result("P1-A2-WP3", files_touched=["src/shared.py"]), + ] + refined_path = _make_refined_wps(tmp_path, wps) + + consolidate_wps(refined_wps_path=str(refined_path), planner_dir=str(tmp_path)) + + consolidated = json.loads((tmp_path / "consolidated_wps.json").read_text()) + output_ids = {wp["id"] for wp in consolidated["work_packages"]} + assert "P1-A1-WP1" in output_ids + assert "P1-A2-WP1" in output_ids + assert len(consolidated["work_packages"]) == 2 + + +def test_fallback_caps_group_size(tmp_path: Path) -> None: + wps = [make_wp_result(f"P1-A1-WP{i}", files_touched=["src/big.py"]) for i in range(1, 9)] + refined_path = _make_refined_wps(tmp_path, wps) + + result = consolidate_wps(refined_wps_path=str(refined_path), planner_dir=str(tmp_path)) + + consolidated = json.loads((tmp_path / "consolidated_wps.json").read_text()) + assert len(consolidated["work_packages"]) == 2 + assert result["merged_count"] == "2" + + +# --------------------------------------------------------------------------- +# Write-back tests (T1-T7) +# --------------------------------------------------------------------------- + + +def test_writeback_overwrites_primary_result_file(tmp_path: Path) -> None: + wp1 = make_wp_result("P1-A1-WP1", deliverables=["src/a.py"]) + wp2 = make_wp_result("P1-A1-WP2", deliverables=["src/b.py"]) + refined_path = _make_refined_wps(tmp_path, [wp1, wp2]) + wp_dir = tmp_path / "work_packages" + write_json(wp_dir / "P1-A1-WP1_result.json", wp1) + write_json(wp_dir / "P1-A1-WP2_result.json", wp2) + consolidation_dir = wp_dir / "consolidation" + _make_manifest( + consolidation_dir, + "P1", + [ + { + "merged_id": "P1-A1-WP1", + "source_wp_ids": ["P1-A1-WP1", "P1-A1-WP2"], + "merge_order": ["P1-A1-WP1", "P1-A1-WP2"], + "name": None, + "goal": None, + } + ], + ) + + consolidate_wps(refined_wps_path=str(refined_path), planner_dir=str(tmp_path)) + + result_file = wp_dir / "P1-A1-WP1_result.json" + assert result_file.exists() + merged = json.loads(result_file.read_text()) + assert set(merged["deliverables"]) == {"src/a.py", "src/b.py"} + consolidated = json.loads((tmp_path / "consolidated_wps.json").read_text()) + assert merged == consolidated["work_packages"][0] + + +def test_writeback_removes_absorbed_result_files(tmp_path: Path) -> None: + wp1 = make_wp_result("P1-A1-WP1") + wp2 = make_wp_result("P1-A1-WP2") + refined_path = _make_refined_wps(tmp_path, [wp1, wp2]) + wp_dir = tmp_path / "work_packages" + write_json(wp_dir / "P1-A1-WP1_result.json", wp1) + write_json(wp_dir / "P1-A1-WP2_result.json", wp2) + consolidation_dir = wp_dir / "consolidation" + _make_manifest( + consolidation_dir, + "P1", + [ + { + "merged_id": "P1-A1-WP1", + "source_wp_ids": ["P1-A1-WP1", "P1-A1-WP2"], + "merge_order": ["P1-A1-WP1", "P1-A1-WP2"], + "name": None, + "goal": None, + } + ], + ) + + consolidate_wps(refined_wps_path=str(refined_path), planner_dir=str(tmp_path)) + + assert not (wp_dir / "P1-A1-WP2_result.json").exists() + assert (wp_dir / "P1-A1-WP1_result.json").exists() + + +def test_writeback_preserves_uninvolved_result_files(tmp_path: Path) -> None: + wp1 = make_wp_result("P1-A1-WP1") + wp2 = make_wp_result("P1-A1-WP2") + wp3 = make_wp_result("P1-A1-WP3") + refined_path = _make_refined_wps(tmp_path, [wp1, wp2, wp3]) + wp_dir = tmp_path / "work_packages" + write_json(wp_dir / "P1-A1-WP1_result.json", wp1) + write_json(wp_dir / "P1-A1-WP2_result.json", wp2) + write_json(wp_dir / "P1-A1-WP3_result.json", wp3) + consolidation_dir = wp_dir / "consolidation" + _make_manifest( + consolidation_dir, + "P1", + [ + { + "merged_id": "P1-A1-WP1", + "source_wp_ids": ["P1-A1-WP1", "P1-A1-WP2"], + "merge_order": ["P1-A1-WP1", "P1-A1-WP2"], + "name": None, + "goal": None, + } + ], + ) + + consolidate_wps(refined_wps_path=str(refined_path), planner_dir=str(tmp_path)) + + wp3_result = wp_dir / "P1-A1-WP3_result.json" + assert wp3_result.exists() + data = json.loads(wp3_result.read_text()) + assert data["id"] == "P1-A1-WP3" + assert (wp_dir / "P1-A1-WP1_result.json").exists() + assert not (wp_dir / "P1-A1-WP2_result.json").exists() + + +def test_writeback_noop_when_no_merges(tmp_path: Path) -> None: + wps = [make_wp_result(f"P1-A1-WP{i}") for i in range(1, 4)] + refined_path = _make_refined_wps(tmp_path, wps) + wp_dir = tmp_path / "work_packages" + for wp in wps: + write_json(wp_dir / f"{wp['id']}_result.json", wp) + + consolidate_wps(refined_wps_path=str(refined_path), planner_dir=str(tmp_path)) + + for i in range(1, 4): + assert (wp_dir / f"P1-A1-WP{i}_result.json").exists() + + +def test_writeback_fallback_merges_update_result_files(tmp_path: Path) -> None: + wps = [ + make_wp_result("P1-A1-WP1", files_touched=["src/config.yaml"]), + make_wp_result("P1-A1-WP2", files_touched=["src/config.yaml"]), + make_wp_result("P1-A1-WP3", files_touched=["src/config.yaml"]), + make_wp_result("P1-A1-WP4", files_touched=["src/other.py"]), + make_wp_result("P1-A1-WP5", files_touched=["src/other.py"]), + ] + refined_path = _make_refined_wps(tmp_path, wps) + wp_dir = tmp_path / "work_packages" + for wp in wps: + write_json(wp_dir / f"{wp['id']}_result.json", wp) + + consolidate_wps(refined_wps_path=str(refined_path), planner_dir=str(tmp_path)) + + assert (wp_dir / "P1-A1-WP1_result.json").exists() + assert (wp_dir / "P1-A1-WP4_result.json").exists() + assert not (wp_dir / "P1-A1-WP2_result.json").exists() + assert not (wp_dir / "P1-A1-WP3_result.json").exists() + assert not (wp_dir / "P1-A1-WP5_result.json").exists() + merged = json.loads((wp_dir / "P1-A1-WP1_result.json").read_text()) + assert "src/config.yaml" in merged["files_touched"] + + +def test_consolidate_then_validate_sees_merged_wps(tmp_path: Path) -> None: + wp1 = make_wp_result("P1-A1-WP1", depends_on=[]) + wp2 = make_wp_result("P1-A1-WP2", depends_on=[]) + wp3 = make_wp_result("P1-A1-WP3", depends_on=["P1-A1-WP1"]) + refined_path = _make_refined_wps(tmp_path, [wp1, wp2, wp3]) + + phases_dir = tmp_path / "phases" + assigns_dir = tmp_path / "assignments" + wp_dir = tmp_path / "work_packages" + + write_json(phases_dir / "P1_result.json", make_phase_result(1)) + write_json( + assigns_dir / "P1-A1_result.json", + make_assignment_result( + 1, 1, proposed_work_packages=["P1-A1-WP1", "P1-A1-WP2", "P1-A1-WP3"] + ), + ) + write_json(wp_dir / "P1-A1-WP1_result.json", wp1) + write_json(wp_dir / "P1-A1-WP2_result.json", wp2) + write_json(wp_dir / "P1-A1-WP3_result.json", wp3) + write_json( + wp_dir / "wp_manifest.json", + { + "pass_name": "work_packages", + "items": [ + {"id": "P1-A1-WP1", "status": "done"}, + {"id": "P1-A1-WP2", "status": "done"}, + {"id": "P1-A1-WP3", "status": "done"}, + ], + }, + ) + consolidation_dir = wp_dir / "consolidation" + _make_manifest( + consolidation_dir, + "P1", + [ + { + "merged_id": "P1-A1-WP1", + "source_wp_ids": ["P1-A1-WP1", "P1-A1-WP2"], + "merge_order": ["P1-A1-WP1", "P1-A1-WP2"], + "name": None, + "goal": None, + } + ], + ) + + consolidate_wps(refined_wps_path=str(refined_path), planner_dir=str(tmp_path)) + result = validate_plan(str(tmp_path)) + + assert result["verdict"] == "pass" + assert result["issue_count"] == "0" + + +def test_consolidate_then_compile_emits_correct_issue_count(tmp_path: Path) -> None: + wp1 = make_wp_result("P1-A1-WP1", depends_on=[]) + wp2 = make_wp_result("P1-A1-WP2", depends_on=[]) + wp3 = make_wp_result("P1-A1-WP3", depends_on=["P1-A1-WP1"]) + refined_path = _make_refined_wps(tmp_path, [wp1, wp2, wp3]) + + phases_dir = tmp_path / "phases" + assigns_dir = tmp_path / "assignments" + wp_dir = tmp_path / "work_packages" + + write_json(phases_dir / "P1_result.json", make_phase_result(1)) + write_json( + assigns_dir / "P1-A1_result.json", + make_assignment_result( + 1, 1, proposed_work_packages=["P1-A1-WP1", "P1-A1-WP2", "P1-A1-WP3"] + ), + ) + write_json(wp_dir / "P1-A1-WP1_result.json", wp1) + write_json(wp_dir / "P1-A1-WP2_result.json", wp2) + write_json(wp_dir / "P1-A1-WP3_result.json", wp3) + write_json( + wp_dir / "wp_manifest.json", + { + "pass_name": "work_packages", + "items": [ + {"id": "P1-A1-WP1", "status": "done"}, + {"id": "P1-A1-WP2", "status": "done"}, + {"id": "P1-A1-WP3", "status": "done"}, + ], + }, + ) + consolidation_dir = wp_dir / "consolidation" + _make_manifest( + consolidation_dir, + "P1", + [ + { + "merged_id": "P1-A1-WP1", + "source_wp_ids": ["P1-A1-WP1", "P1-A1-WP2"], + "merge_order": ["P1-A1-WP1", "P1-A1-WP2"], + "name": None, + "goal": None, + } + ], + ) + + consolidate_wps(refined_wps_path=str(refined_path), planner_dir=str(tmp_path)) + + task_file_path = write_task_file(tmp_path) + write_json( + tmp_path / "validation.json", {"verdict": "pass", "findings": [], "schema_version": 2} + ) + compile_plan(str(tmp_path), task_file_path, "/src") + + issues_dir = tmp_path / "issues" + issue_files = sorted(issues_dir.glob("*_issue.md")) + assert len(issue_files) == 2 + issue_names = {f.name for f in issue_files} + assert "P1-A1-WP1_issue.md" in issue_names + assert "P1-A1-WP3_issue.md" in issue_names + assert "P1-A1-WP2_issue.md" not in issue_names + + manifest = json.loads((tmp_path / "manifest.json").read_text()) + assert len(manifest["execution_order"]) == 2 + + merged_issue = (issues_dir / "P1-A1-WP1_issue.md").read_text() + assert "src/mod_P1-A1-WP1.py" in merged_issue + assert "src/mod_P1-A1-WP2.py" in merged_issue + + +def test_consolidate_wps_wp_index_in_work_packages(tmp_path: Path) -> None: + """Rebuilt wp_index.json must land in work_packages/, not planner root.""" + wps = [make_wp_result("P1-A1-WP1")] + refined_path = _make_refined_wps(tmp_path, wps) + + consolidate_wps(refined_wps_path=str(refined_path), planner_dir=str(tmp_path)) + + assert (tmp_path / "work_packages" / "wp_index.json").exists() + assert not (tmp_path / "wp_index.json").exists() diff --git a/tests/planner/test_elaborate_wps_contract.py b/tests/planner/test_elaborate_wps_contract.py new file mode 100644 index 0000000000..c3220f32ae --- /dev/null +++ b/tests/planner/test_elaborate_wps_contract.py @@ -0,0 +1,154 @@ +""" +Contract conformance tests for planner-elaborate-wps skill registration. +""" + +from pathlib import Path + +import pytest +import yaml + +pytestmark = [ + pytest.mark.layer("planner"), + pytest.mark.small, + pytest.mark.feature("planner"), +] + +SKILL_NAME = "planner-elaborate-wps" +CONTRACTS_PATH = ( + Path(__file__).parent.parent.parent / "src/autoskillit/recipe/skill_contracts.yaml" +) +SKILL_MD_PATH = ( + Path(__file__).parent.parent.parent / f"src/autoskillit/skills_extended/{SKILL_NAME}/SKILL.md" +) + + +@pytest.fixture(scope="module") +def contracts() -> dict: + return yaml.safe_load(CONTRACTS_PATH.read_text()) + + +@pytest.fixture(scope="module") +def skill_contract(contracts: dict) -> dict: + skills = contracts.get("skills", {}) + assert SKILL_NAME in skills, ( + f"{SKILL_NAME!r} not found in skill_contracts.yaml. " + f"Known planner skills: {[k for k in skills if 'planner' in k]}" + ) + return skills[SKILL_NAME] + + +@pytest.fixture(scope="module") +def skill_md() -> str: + assert SKILL_MD_PATH.exists(), ( + f"SKILL.md not found at {SKILL_MD_PATH}. " + "Create src/autoskillit/skills_extended/planner-elaborate-wps/SKILL.md" + ) + return SKILL_MD_PATH.read_text() + + +class TestContractRegistration: + def test_skill_entry_exists(self, skill_contract: dict) -> None: + assert isinstance(skill_contract, dict), ( + f"{SKILL_NAME!r} contract entry must be a dict, got {type(skill_contract)}" + ) + + def test_write_behavior_always(self, skill_contract: dict) -> None: + assert skill_contract.get("write_behavior") == "always", ( + "write_behavior must be 'always' for planner-elaborate-wps. " + "Without it, a session that never writes is marked successful." + ) + + def test_phase_wps_result_dir_output_present(self, skill_contract: dict) -> None: + outputs = skill_contract.get("outputs", []) + names = [o.get("name") for o in outputs] + assert "phase_wps_result_dir" in names, ( + f"phase_wps_result_dir not in outputs. Found: {names}" + ) + + def test_phase_wps_result_dir_type_is_directory_path(self, skill_contract: dict) -> None: + outputs = skill_contract.get("outputs", []) + token = next((o for o in outputs if o.get("name") == "phase_wps_result_dir"), None) + assert token is not None + assert token.get("type") == "directory_path", ( + f"phase_wps_result_dir type must be 'directory_path', got {token.get('type')!r}." + ) + + def test_expected_output_pattern_present(self, skill_contract: dict) -> None: + patterns = skill_contract.get("expected_output_patterns", []) + assert any("phase_wps_result_dir" in p for p in patterns), ( + f"No expected_output_pattern referencing phase_wps_result_dir. Found: {patterns}" + ) + + def test_two_inputs_declared(self, skill_contract: dict) -> None: + inputs = skill_contract.get("inputs", []) + assert len(inputs) == 2, ( + f"Expected exactly 2 inputs, got {len(inputs)}: {[i.get('name') for i in inputs]}" + ) + input_names = {i.get("name") for i in inputs} + assert input_names == {"context_file", "planner_dir"}, ( + f"Unexpected input names: {input_names}" + ) + + +class TestSkillMdPresence: + def test_has_workflow_section(self, skill_md: str) -> None: + assert "## Workflow" in skill_md, ( + "SKILL.md must contain a '## Workflow' section — " + "file exists but appears empty or structurally incomplete." + ) + + def test_has_categories_planner(self, skill_md: str) -> None: + assert "categories: [planner]" in skill_md, ( + "SKILL.md must include 'categories: [planner]' in frontmatter. " + "Required for feature-gate and skill-category routing." + ) + + def test_has_never_always_block(self, skill_md: str) -> None: + assert "NEVER" in skill_md and "ALWAYS" in skill_md, ( + "SKILL.md must include a NEVER/ALWAYS constraint block following " + "the pattern of all other planner skills." + ) + + def test_has_write_path_restriction(self, skill_md: str) -> None: + assert "AUTOSKILLIT_TEMP" in skill_md and "planner/" in skill_md, ( + "SKILL.md NEVER block must restrict write paths to {{AUTOSKILLIT_TEMP}}/planner/." + ) + + def test_has_output_token(self, skill_md: str) -> None: + assert "phase_wps_result_dir" in skill_md, ( + "SKILL.md must document 'phase_wps_result_dir = ' output token." + ) + + def test_has_l0_response_fields(self, skill_md: str) -> None: + for field in [ + "id", + "name", + "goal", + "summary", + "technical_steps", + "files_touched", + "apis_defined", + "apis_consumed", + "depends_on", + "deliverables", + "acceptance_criteria", + ]: + assert field in skill_md, ( + f"SKILL.md must document L0 structured response field: {field}." + ) + + def test_has_batch_ceiling(self, skill_md: str) -> None: + assert "batches of 6" in skill_md, ( + "SKILL.md must specify the maximum parallel L0 batch size of 6." + ) + + def test_has_partial_failure_handling(self, skill_md: str) -> None: + assert "CRITICAL" in skill_md or "partial" in skill_md.lower(), ( + "SKILL.md must describe partial-failure handling — " + "proceed with N-1 WPs and log CRITICAL if an L0 fails." + ) + + def test_has_wp_sentinels(self, skill_md: str) -> None: + assert "wp_sentinels" in skill_md, ( + "SKILL.md must document the sentinel directory naming convention (wp_sentinels)." + ) diff --git a/tests/planner/test_manifests.py b/tests/planner/test_manifests.py new file mode 100644 index 0000000000..3eb6e3f7d7 --- /dev/null +++ b/tests/planner/test_manifests.py @@ -0,0 +1,759 @@ +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any + +import pytest + +from tests.planner.conftest import ( + make_assignment_result, + make_phase_result, + make_wp_result, + write_json, +) + +pytestmark = [pytest.mark.layer("planner"), pytest.mark.small, pytest.mark.feature("planner")] + + +def test_build_phase_assignment_manifest_creates_one_item_per_phase(tmp_path): + """P1 with 3 assignments and P2 with 2 assignments produce a 2-item manifest.""" + from autoskillit.planner import build_phase_assignment_manifest + + phases_dir = tmp_path / "phases" + phases_dir.mkdir() + output_dir = tmp_path / "assignments" + output_dir.mkdir() + + (phases_dir / "P1_result.json").write_text( + json.dumps( + make_phase_result(1, assignments_preview=["Auth Setup", "DB Schema", "API Layer"]) + ) + ) + (phases_dir / "P2_result.json").write_text( + json.dumps(make_phase_result(2, assignments_preview=["CLI Integration", "Config Loading"])) + ) + + result = build_phase_assignment_manifest(str(phases_dir), str(output_dir)) + + manifest = json.loads(Path(result["manifest_path"]).read_text()) + assert len(manifest["items"]) == 2 + + p1_item = next(i for i in manifest["items"] if i["id"] == "P1") + p2_item = next(i for i in manifest["items"] if i["id"] == "P2") + assert p1_item["metadata"]["assignment_count"] == 3 + assert len(p1_item["metadata"]["assignment_names"]) == 3 + assert p2_item["metadata"]["assignment_count"] == 2 + assert len(p2_item["metadata"]["assignment_names"]) == 2 + + +def test_build_phase_assignment_manifest_item_ids_match_phase_ids(tmp_path): + """Item IDs in the manifest match phase IDs (P1, P2) not assignment IDs.""" + from autoskillit.planner import build_phase_assignment_manifest + + phases_dir = tmp_path / "phases" + phases_dir.mkdir() + output_dir = tmp_path / "assignments" + output_dir.mkdir() + + (phases_dir / "P1_result.json").write_text( + json.dumps(make_phase_result(1, assignments_preview=["Task A"])) + ) + (phases_dir / "P2_result.json").write_text( + json.dumps(make_phase_result(2, assignments_preview=["Task B"])) + ) + + result = build_phase_assignment_manifest(str(phases_dir), str(output_dir)) + + manifest = json.loads(Path(result["manifest_path"]).read_text()) + item_ids = {i["id"] for i in manifest["items"]} + assert item_ids == {"P1", "P2"} + + +def test_build_phase_assignment_manifest_items_start_pending(tmp_path): + """All items in the manifest start with status 'pending'.""" + from autoskillit.planner import build_phase_assignment_manifest + + phases_dir = tmp_path / "phases" + phases_dir.mkdir() + output_dir = tmp_path / "assignments" + output_dir.mkdir() + + (phases_dir / "P1_result.json").write_text( + json.dumps(make_phase_result(1, assignments_preview=["Task X", "Task Y"])) + ) + + result = build_phase_assignment_manifest(str(phases_dir), str(output_dir)) + + manifest = json.loads(Path(result["manifest_path"]).read_text()) + for item in manifest["items"]: + assert item["status"] == "pending", f"item {item['id']} status was {item['status']!r}" + + +# --------------------------------------------------------------------------- +# build_phase_wp_manifest tests (T1–T7) +# --------------------------------------------------------------------------- + + +def test_build_phase_wp_manifest_groups_by_phase(tmp_path): + """T1: Two phases produce a manifest with exactly 2 items.""" + from autoskillit.planner import build_phase_wp_manifest + + assignments_dir = tmp_path / "assignments" + assignments_dir.mkdir() + output_dir = tmp_path / "out" + output_dir.mkdir() + + (assignments_dir / "P1-A1_result.json").write_text( + json.dumps( + make_assignment_result( + 1, + 1, + proposed_work_packages=[ + {"name": "WP1", "scope": "s", "estimated_files": ["a.py"]}, + {"name": "WP2", "scope": "s", "estimated_files": ["b.py"]}, + {"name": "WP3", "scope": "s", "estimated_files": ["c.py"]}, + ], + ) + ) + ) + (assignments_dir / "P1-A2_result.json").write_text( + json.dumps( + make_assignment_result( + 1, + 2, + proposed_work_packages=[ + {"name": "WP1", "scope": "s", "estimated_files": ["d.py"]}, + {"name": "WP2", "scope": "s", "estimated_files": ["e.py"]}, + {"name": "WP3", "scope": "s", "estimated_files": ["f.py"]}, + ], + ) + ) + ) + (assignments_dir / "P2-A1_result.json").write_text( + json.dumps( + make_assignment_result( + 2, + 1, + proposed_work_packages=[ + {"name": "WP1", "scope": "s", "estimated_files": ["g.py"]}, + {"name": "WP2", "scope": "s", "estimated_files": ["h.py"]}, + ], + ) + ) + ) + + result = build_phase_wp_manifest(str(assignments_dir), str(output_dir)) + + manifest = json.loads(Path(result["manifest_path"]).read_text()) + assert len(manifest["items"]) == 2 + item_ids = [i["id"] for i in manifest["items"]] + assert item_ids == ["P1", "P2"] + + +def test_build_phase_wp_manifest_metadata_carries_wp_details(tmp_path): + """T2: Phase metadata contains wp_count, wp_ids, wp_names, wp_scopes, wp_estimated_files.""" + from autoskillit.planner import build_phase_wp_manifest + + assignments_dir = tmp_path / "assignments" + assignments_dir.mkdir() + output_dir = tmp_path / "out" + output_dir.mkdir() + + (assignments_dir / "P1-A1_result.json").write_text( + json.dumps( + make_assignment_result( + 1, + 1, + proposed_work_packages=[ + {"name": "Alpha", "scope": "scope-a", "estimated_files": ["a.py"]}, + {"name": "Beta", "scope": "scope-b", "estimated_files": ["b.py", "c.py"]}, + ], + ) + ) + ) + (assignments_dir / "P1-A2_result.json").write_text( + json.dumps( + make_assignment_result( + 1, + 2, + proposed_work_packages=[ + {"name": "Gamma", "scope": "scope-c", "estimated_files": ["d.py"]}, + ], + ) + ) + ) + + result = build_phase_wp_manifest(str(assignments_dir), str(output_dir)) + + manifest = json.loads(Path(result["manifest_path"]).read_text()) + p1 = manifest["items"][0] + meta = p1["metadata"] + assert meta["wp_count"] == 3 + assert meta["wp_ids"] == ["P1-A1-WP1", "P1-A1-WP2", "P1-A2-WP1"] + assert meta["wp_names"] == ["Alpha", "Beta", "Gamma"] + assert meta["wp_scopes"] == ["scope-a", "scope-b", "scope-c"] + assert meta["wp_estimated_files"] == [["a.py"], ["b.py", "c.py"], ["d.py"]] + + +def test_build_phase_wp_manifest_pass_name_and_result_dir(tmp_path): + """T3: Manifest has pass_name 'phase_work_packages' and result_dir pointing to wp_sentinels.""" + from autoskillit.planner import build_phase_wp_manifest + + assignments_dir = tmp_path / "assignments" + assignments_dir.mkdir() + output_dir = tmp_path / "out" + output_dir.mkdir() + + (assignments_dir / "P1-A1_result.json").write_text( + json.dumps( + make_assignment_result( + 1, + 1, + proposed_work_packages=[{"name": "WP1", "scope": "", "estimated_files": []}], + ) + ) + ) + + result = build_phase_wp_manifest(str(assignments_dir), str(output_dir)) + + manifest = json.loads(Path(result["manifest_path"]).read_text()) + assert manifest["pass_name"] == "phase_work_packages" + assert "wp_sentinels" in manifest["result_dir"] + assert Path(manifest["result_dir"]).exists() + + +def test_build_phase_wp_manifest_initializes_wp_index(tmp_path): + """T4: wp_index.json exists and contains [] after calling build_phase_wp_manifest.""" + from autoskillit.planner import build_phase_wp_manifest + + assignments_dir = tmp_path / "assignments" + assignments_dir.mkdir() + output_dir = tmp_path / "out" + output_dir.mkdir() + + (assignments_dir / "P1-A1_result.json").write_text( + json.dumps( + make_assignment_result( + 1, + 1, + proposed_work_packages=[{"name": "WP1", "scope": "", "estimated_files": []}], + ) + ) + ) + + build_phase_wp_manifest(str(assignments_dir), str(output_dir)) + + wp_index = output_dir / "work_packages" / "wp_index.json" + assert wp_index.exists() + assert json.loads(wp_index.read_text()) == [] + + +def test_build_phase_wp_manifest_sorts_by_phase_number(tmp_path): + """T5: Items are sorted by phase number regardless of input order.""" + from autoskillit.planner import build_phase_wp_manifest + + assignments_dir = tmp_path / "assignments" + assignments_dir.mkdir() + output_dir = tmp_path / "out" + output_dir.mkdir() + + for pn in (3, 1, 2): + (assignments_dir / f"P{pn}-A1_result.json").write_text( + json.dumps( + make_assignment_result( + pn, + 1, + proposed_work_packages=[ + {"name": f"WP-P{pn}", "scope": "", "estimated_files": []} + ], + ) + ) + ) + + result = build_phase_wp_manifest(str(assignments_dir), str(output_dir)) + + manifest = json.loads(Path(result["manifest_path"]).read_text()) + ids = [i["id"] for i in manifest["items"]] + assert ids == ["P1", "P2", "P3"] + + +def test_build_phase_wp_manifest_empty_assignments_dir(tmp_path): + """T6: Empty assignments directory produces manifest with items: [].""" + from autoskillit.planner import build_phase_wp_manifest + + assignments_dir = tmp_path / "assignments" + assignments_dir.mkdir() + output_dir = tmp_path / "out" + output_dir.mkdir() + + result = build_phase_wp_manifest(str(assignments_dir), str(output_dir)) + + assert result["total_count"] == "0" + manifest = json.loads(Path(result["manifest_path"]).read_text()) + assert manifest["items"] == [] + + +def test_build_phase_wp_manifest_creates_sentinel_dir(tmp_path): + """T7: The callable creates wp_sentinels/ directory.""" + from autoskillit.planner import build_phase_wp_manifest + + assignments_dir = tmp_path / "assignments" + assignments_dir.mkdir() + output_dir = tmp_path / "out" + output_dir.mkdir() + + build_phase_wp_manifest(str(assignments_dir), str(output_dir)) + + sentinel_dir = output_dir / "work_packages" / "wp_sentinels" + assert sentinel_dir.is_dir() + + +# --------------------------------------------------------------------------- +# finalize_wp_manifest tests (T8–T11) +# --------------------------------------------------------------------------- + + +def test_finalize_wp_manifest_from_result_files(tmp_path): + """T8: 4 result files produce wp_manifest.json with 4 items, all status done.""" + from autoskillit.planner import finalize_wp_manifest + + wp_dir = tmp_path / "work_packages" + wp_dir.mkdir() + output_dir = tmp_path / "out" + output_dir.mkdir() + + for i in range(1, 5): + wp_id = f"P1-A1-WP{i}" + (wp_dir / f"{wp_id}_result.json").write_text(json.dumps(make_wp_result(wp_id))) + + result = finalize_wp_manifest(str(wp_dir), str(output_dir)) + + assert result["total_count"] == "4" + manifest = json.loads(Path(result["manifest_path"]).read_text()) + assert len(manifest["items"]) == 4 + for item in manifest["items"]: + assert item["status"] == "done" + assert item["result_path"] + assert item["id"] + assert item["name"] + + +def test_finalize_wp_manifest_skips_non_result_files(tmp_path): + """T9: Non-result files (wp_manifest.json, wp_index.json, sentinel subdir) are skipped.""" + from autoskillit.planner import finalize_wp_manifest + + wp_dir = tmp_path / "work_packages" + wp_dir.mkdir() + output_dir = tmp_path / "out" + output_dir.mkdir() + + wp_id = "P1-A1-WP1" + (wp_dir / f"{wp_id}_result.json").write_text(json.dumps(make_wp_result(wp_id))) + (wp_dir / "wp_manifest.json").write_text('{"pass_name": "old"}') + (wp_dir / "wp_index.json").write_text("[]") + sentinel_dir = wp_dir / "wp_sentinels" + sentinel_dir.mkdir() + (sentinel_dir / "P1_result.json").write_text('{"ok": true}') + + result = finalize_wp_manifest(str(wp_dir), str(output_dir)) + + assert result["total_count"] == "1" + manifest = json.loads(Path(result["manifest_path"]).read_text()) + assert len(manifest["items"]) == 1 + assert manifest["items"][0]["id"] == wp_id + + +def test_finalize_wp_manifest_empty_dir(tmp_path): + """T10: Empty work_packages/ produces manifest with items: [].""" + from autoskillit.planner import finalize_wp_manifest + + wp_dir = tmp_path / "work_packages" + wp_dir.mkdir() + output_dir = tmp_path / "out" + output_dir.mkdir() + + result = finalize_wp_manifest(str(wp_dir), str(output_dir)) + + assert result["total_count"] == "0" + manifest = json.loads(Path(result["manifest_path"]).read_text()) + assert manifest["items"] == [] + + +def test_finalize_wp_manifest_regenerates_wp_index(tmp_path): + """T11: wp_index.json is regenerated with compact entries sorted by WP ID.""" + from autoskillit.planner import finalize_wp_manifest + + wp_dir = tmp_path / "work_packages" + wp_dir.mkdir() + output_dir = tmp_path / "out" + output_dir.mkdir() + + for wp_id in ("P2-A1-WP1", "P1-A1-WP1", "P1-A2-WP1"): + (wp_dir / f"{wp_id}_result.json").write_text(json.dumps(make_wp_result(wp_id))) + + finalize_wp_manifest(str(wp_dir), str(output_dir)) + + index = json.loads((wp_dir / "wp_index.json").read_text()) + assert len(index) == 3 + ids = [e["id"] for e in index] + assert ids == ["P1-A1-WP1", "P1-A2-WP1", "P2-A1-WP1"] + for entry in index: + assert "id" in entry + assert "name" in entry + assert "summary" in entry + + +# --------------------------------------------------------------------------- +# build_phase_assignment_manifest error branches +# --------------------------------------------------------------------------- + + +def test_build_phase_assignment_manifest_corrupt_json_raises(tmp_path): + from autoskillit.planner import build_phase_assignment_manifest + + phases_dir = tmp_path / "phases" + phases_dir.mkdir() + output_dir = tmp_path / "out" + output_dir.mkdir() + (phases_dir / "P1_result.json").write_text("{not json") + + with pytest.raises(json.JSONDecodeError, match="Failed to parse"): + build_phase_assignment_manifest(str(phases_dir), str(output_dir)) + + +def test_build_phase_assignment_manifest_missing_required_keys_raises(tmp_path): + from autoskillit.planner import build_phase_assignment_manifest + + phases_dir = tmp_path / "phases" + phases_dir.mkdir() + output_dir = tmp_path / "out" + output_dir.mkdir() + (phases_dir / "P1_result.json").write_text(json.dumps({"foo": "bar"})) + + with pytest.raises(ValueError, match="Invalid phase result in"): + build_phase_assignment_manifest(str(phases_dir), str(output_dir)) + + +def test_build_phase_assignment_manifest_empty_string_dir_raises(tmp_path): + from autoskillit.planner import build_phase_assignment_manifest + + output_dir = tmp_path / "out" + output_dir.mkdir() + + with pytest.raises(ValueError, match="phases_dir and output_dir must not be empty"): + build_phase_assignment_manifest("", str(output_dir)) + + +# --------------------------------------------------------------------------- +# build_phase_wp_manifest error branches +# --------------------------------------------------------------------------- + + +def test_build_phase_wp_manifest_corrupt_json_raises(tmp_path): + from autoskillit.planner import build_phase_wp_manifest + + assignments_dir = tmp_path / "assignments" + assignments_dir.mkdir() + output_dir = tmp_path / "out" + output_dir.mkdir() + (assignments_dir / "P1-A1_result.json").write_text("{not json") + + with pytest.raises(json.JSONDecodeError, match="Failed to parse"): + build_phase_wp_manifest(str(assignments_dir), str(output_dir)) + + +def test_build_phase_wp_manifest_empty_string_raises(tmp_path): + from autoskillit.planner import build_phase_wp_manifest + + output_dir = tmp_path / "out" + output_dir.mkdir() + + with pytest.raises(ValueError, match="assignments_dir and output_dir must not be empty"): + build_phase_wp_manifest("", str(output_dir)) + + +def test_build_phase_wp_manifest_nonexistent_dir_raises(tmp_path): + from autoskillit.planner import build_phase_wp_manifest + + output_dir = tmp_path / "out" + output_dir.mkdir() + + with pytest.raises(FileNotFoundError, match="assignments_dir does not exist"): + build_phase_wp_manifest(str(tmp_path / "nonexistent"), str(output_dir)) + + +# --------------------------------------------------------------------------- +# finalize_wp_manifest error branches +# --------------------------------------------------------------------------- + + +def test_finalize_wp_manifest_corrupt_json_raises(tmp_path): + from autoskillit.planner import finalize_wp_manifest + + wp_dir = tmp_path / "work_packages" + wp_dir.mkdir() + output_dir = tmp_path / "out" + output_dir.mkdir() + (wp_dir / "P1-A1-WP1_result.json").write_text("{not json") + + with pytest.raises(json.JSONDecodeError, match="Failed to parse"): + finalize_wp_manifest(str(wp_dir), str(output_dir)) + + +def test_finalize_wp_manifest_empty_string_raises(tmp_path): + from autoskillit.planner import finalize_wp_manifest + + output_dir = tmp_path / "out" + output_dir.mkdir() + + with pytest.raises(ValueError, match="work_packages_dir and output_dir must not be empty"): + finalize_wp_manifest("", str(output_dir)) + + +def test_finalize_wp_manifest_nonexistent_dir_raises(tmp_path): + from autoskillit.planner import finalize_wp_manifest + + output_dir = tmp_path / "out" + output_dir.mkdir() + + with pytest.raises(FileNotFoundError, match="work_packages_dir does not exist"): + finalize_wp_manifest(str(tmp_path / "nonexistent"), str(output_dir)) + + +def test_resolve_task_input_file_with_heading(tmp_path): + from autoskillit.planner import resolve_task_input + + task_file = tmp_path / "task.md" + task_file.write_text("# Deploy Auth Service\n\nDetailed description...") + planner_dir = tmp_path / "planner" + planner_dir.mkdir() + result = resolve_task_input(str(task_file), str(planner_dir)) + assert list(planner_dir.iterdir()) == [] + assert result["task_file_path"] == str(task_file) + assert result["task_label"] == "Deploy Auth Service" + + +def test_resolve_task_input_file_no_heading(tmp_path): + from autoskillit.planner import resolve_task_input + + task_file = tmp_path / "task.txt" + task_file.write_text("Implement the feature flag system for gradual rollout") + planner_dir = tmp_path / "planner" + planner_dir.mkdir() + result = resolve_task_input(str(task_file), str(planner_dir)) + assert result["task_file_path"] == str(task_file) + assert result["task_label"] == "Implement the feature flag system for gradual rollout" + + +def test_resolve_task_input_inline_text(tmp_path): + from autoskillit.planner import resolve_task_input + + planner_dir = tmp_path / "planner" + planner_dir.mkdir() + result = resolve_task_input("Add dark mode toggle", str(planner_dir)) + assert result["task_file_path"] == str(planner_dir / "task_input.md") + assert Path(result["task_file_path"]).read_text() == "Add dark mode toggle" + assert result["task_label"] == "Add dark mode toggle" + + +def test_resolve_task_input_inline_with_heading(tmp_path): + from autoskillit.planner import resolve_task_input + + planner_dir = tmp_path / "planner" + planner_dir.mkdir() + text = "# Auth Overhaul\n\nRebuild the entire authentication layer..." + result = resolve_task_input(text, str(planner_dir)) + assert result["task_label"] == "Auth Overhaul" + assert Path(result["task_file_path"]).read_text() == text + + +def test_resolve_task_input_long_inline_truncates_label(tmp_path): + from autoskillit.planner import resolve_task_input + + planner_dir = tmp_path / "planner" + planner_dir.mkdir() + text = "A" * 120 + result = resolve_task_input(text, str(planner_dir)) + assert len(result["task_label"]) <= 80 + assert text.startswith(result["task_label"]) + + +def test_build_phase_wp_manifest_ignores_phase_sentinel_in_assignments(tmp_path): + """Phase sentinel files in assignments/ must not crash build_phase_wp_manifest.""" + from autoskillit.planner import build_phase_wp_manifest + + assign_dir = tmp_path / "assignments" + assign_dir.mkdir() + out_dir = tmp_path / "out" + out_dir.mkdir() + + (assign_dir / "P1-A1_result.json").write_text( + json.dumps(make_assignment_result(1, 1, proposed_work_packages=[])) + ) + (assign_dir / "P1_result.json").write_text( + json.dumps({"id": "P1", "status": "complete", "assignment_count": 1, "failed_count": 0}) + ) + + result = build_phase_wp_manifest(str(assign_dir), str(out_dir)) + assert "manifest_path" in result + + +def test_expand_wps_result_dir_points_to_wp_sentinels(tmp_path): + """expand_wps must create wp_sentinels/ and point result_dir there.""" + from autoskillit.planner import expand_wps + + refined = { + "assignments": [ + { + "id": "P1-A1", + "name": "Assignment 1", + "phase_id": "P1", + "phase_name": "Phase 1", + "goal": "test", + "technical_approach": "test", + "proposed_work_packages": [ + { + "id_suffix": "WP1", + "name": "WP 1", + "scope": "core", + "estimated_files": ["f.py"], + } + ], + } + ], + "task": "test task", + } + refined_path = tmp_path / "refined_assignments.json" + refined_path.write_text(json.dumps(refined)) + + result = expand_wps(str(refined_path), str(tmp_path)) + + manifest = json.loads(Path(result["manifest_path"]).read_text()) + assert manifest["result_dir"].endswith("wp_sentinels") + assert (tmp_path / "work_packages" / "wp_sentinels").is_dir() + + +def test_finalize_wp_manifest_writes_to_work_packages_dir(tmp_path): + """wp_manifest.json must be written inside work_packages_dir, not output_dir.""" + from autoskillit.planner import finalize_wp_manifest + + wp_dir = tmp_path / "work_packages" + wp_dir.mkdir() + (wp_dir / "P1-A1-WP1_result.json").write_text(json.dumps(make_wp_result("P1-A1-WP1"))) + + result = finalize_wp_manifest(str(wp_dir), str(tmp_path)) + + assert (wp_dir / "wp_manifest.json").exists() + assert not (tmp_path / "wp_manifest.json").exists() + assert "work_packages" in result["manifest_path"] + + +def test_finalize_wp_manifest_writes_wp_index_to_work_packages_dir(tmp_path): + """wp_index.json must be written inside work_packages_dir, not output_dir.""" + from autoskillit.planner import finalize_wp_manifest + + wp_dir = tmp_path / "work_packages" + wp_dir.mkdir() + (wp_dir / "P1-A1-WP1_result.json").write_text(json.dumps(make_wp_result("P1-A1-WP1"))) + + finalize_wp_manifest(str(wp_dir), str(tmp_path)) + + assert (wp_dir / "wp_index.json").exists() + assert not (tmp_path / "wp_index.json").exists() + + +def test_expand_wps_wp_index_path_in_work_packages(tmp_path): + """wp_index.json and context wp_index_path must point to work_packages/.""" + from autoskillit.planner import expand_wps + + refined = { + "task": "test", + "assignments": [ + { + "id": "P1-A1", + "name": "Assignment 1", + "phase_id": "P1", + "phase_name": "Phase 1", + "goal": "test", + "technical_approach": "test", + "proposed_work_packages": [ + {"id_suffix": "WP1", "name": "WP1", "scope": "s", "estimated_files": ["f.py"]} + ], + } + ], + } + ra_path = tmp_path / "refined_assignments.json" + ra_path.write_text(json.dumps(refined)) + + expand_wps(str(ra_path), str(tmp_path)) + + wp_dir = tmp_path / "work_packages" + assert (wp_dir / "wp_index.json").exists() + assert not (tmp_path / "wp_index.json").exists() + + ctx_files = list(wp_dir.glob("context_*.json")) + assert len(ctx_files) >= 1 + ctx = json.loads(ctx_files[0].read_text()) + assert "work_packages" in ctx["wp_index_path"] + + +def _raw_wp(wp_id: str, **overrides: Any) -> dict[str, Any]: + """Build a raw WP dict WITHOUT running validate_wp_result (allows invalid data).""" + base: dict[str, Any] = { + "id": wp_id, + "name": f"WP {wp_id}", + "summary": "", + "goal": "", + "deliverables": [f"src/mod_{wp_id}.py"], + "technical_steps": [], + "files_touched": [], + "apis_defined": [], + "apis_consumed": [], + "depends_on": [], + "acceptance_criteria": [], + } + base.update(overrides) + return base + + +def test_finalize_wp_manifest_accumulates_all_validation_errors(tmp_path: Path) -> None: + """When multiple WP result files fail validation, all errors appear in the exception.""" + from autoskillit.planner.manifests import finalize_wp_manifest + + wp_dir = tmp_path / "work_packages" + wp_dir.mkdir() + + # WP with 0 deliverables (violates lower bound) — use _raw_wp to bypass validation + write_json(wp_dir / "P1-A1-WP1_result.json", _raw_wp("P1-A1-WP1", deliverables=[])) + # WP with 6 deliverables (violates upper bound) + write_json( + wp_dir / "P1-A1-WP2_result.json", + _raw_wp("P1-A1-WP2", deliverables=[f"f{i}.py" for i in range(6)]), + ) + # Valid WP (should not appear in error) + write_json(wp_dir / "P1-A1-WP3_result.json", make_wp_result("P1-A1-WP3")) + + with pytest.raises(ValueError, match=r"2 WP validation errors") as exc_info: + finalize_wp_manifest(str(wp_dir), str(tmp_path)) + + msg = str(exc_info.value) + assert "P1-A1-WP1_result.json" in msg + assert "P1-A1-WP2_result.json" in msg + + +def test_finalize_wp_manifest_single_validation_error_message(tmp_path: Path) -> None: + """Single validation error still produces a clear message.""" + from autoskillit.planner.manifests import finalize_wp_manifest + + wp_dir = tmp_path / "work_packages" + wp_dir.mkdir() + + # Use _raw_wp to bypass make_wp_result validation for out-of-bounds deliverable count + write_json( + wp_dir / "P1-A1-WP1_result.json", + _raw_wp("P1-A1-WP1", deliverables=[f"f{i}.py" for i in range(6)]), + ) + + with pytest.raises(ValueError, match=r"1 WP validation error"): + finalize_wp_manifest(str(wp_dir), str(tmp_path)) diff --git a/tests/planner/test_merge.py b/tests/planner/test_merge.py new file mode 100644 index 0000000000..ed8b2dc3de --- /dev/null +++ b/tests/planner/test_merge.py @@ -0,0 +1,926 @@ +from __future__ import annotations + +import json +from pathlib import Path + +import pytest + +from autoskillit.planner.merge import ( + build_plan_snapshot, + extract_item, + merge_files, + merge_refined_assignments, + merge_tier_results, + replace_item, +) +from tests.planner.conftest import make_phase_result, write_json, write_task_file + +pytestmark = [pytest.mark.layer("planner"), pytest.mark.small, pytest.mark.feature("planner")] + + +def test_merge_files_creates_combined_document(tmp_path): + """merge_files writes a PlanDocument with the given key populated.""" + items = [{"id": "P1", "name": "Phase 1"}, {"id": "P2", "name": "Phase 2"}] + file_paths = [] + for item in items: + p = tmp_path / f"{item['id']}_result.json" + p.write_text(json.dumps(item)) + file_paths.append(str(p)) + + out = tmp_path / "combined.json" + + result = merge_files( + file_paths=file_paths, + output_path=str(out), + key="phases", + task_file_path=write_task_file(tmp_path, "my task"), + source_dir="/src", + ) + + assert result["merged_path"] == str(out) + assert result["item_count"] == "2" # item_count is always str per MCP tool contract + data = json.loads(out.read_text()) + assert data["task"] == "my task" + assert data["source_dir"] == "/src" + assert {p["id"] for p in data["phases"]} == {"P1", "P2"} + + +def test_merge_files_schema_version_1(tmp_path): + """Output always carries schema_version: 1.""" + p = tmp_path / "p1.json" + p.write_text(json.dumps({"id": "P1", "name": "x"})) + out = tmp_path / "combined.json" + + merge_files(file_paths=[str(p)], output_path=str(out), key="phases") + + assert json.loads(out.read_text())["schema_version"] == 1 + + +def test_merge_files_accumulates_existing(tmp_path): + """merge_files appends to existing key list when output already exists.""" + existing = { + "task": "t", + "source_dir": "/s", + "phases": [{"id": "P1", "name": "Phase 1"}], + "schema_version": 1, + } + out = tmp_path / "combined.json" + out.write_text(json.dumps(existing)) + new_file = tmp_path / "p2.json" + new_file.write_text(json.dumps({"id": "P2", "name": "Phase 2"})) + + result = merge_files(file_paths=[str(new_file)], output_path=str(out), key="phases") + + data = json.loads(out.read_text()) + assert len(data["phases"]) == 2 + assert result["item_count"] == "2" # item_count is always str per MCP tool contract + + +def test_merge_files_deduplicates_by_id(tmp_path): + """Re-merging a file with same id does not create duplicates.""" + item = {"id": "P1", "name": "Phase 1"} + existing = {"task": "", "source_dir": "", "phases": [item], "schema_version": 1} + out = tmp_path / "combined.json" + out.write_text(json.dumps(existing)) + dup_file = tmp_path / "p1_dup.json" + dup_file.write_text(json.dumps(item)) + + merge_files(file_paths=[str(dup_file)], output_path=str(out), key="phases") + + assert len(json.loads(out.read_text())["phases"]) == 1 + + +def test_merge_files_strict_raises_on_missing_file(tmp_path): + """strict=True (default) raises ValueError for nonexistent input file.""" + with pytest.raises(ValueError, match="File not found"): + merge_files( + file_paths=["/nonexistent/path.json"], + output_path=str(tmp_path / "out.json"), + key="phases", + ) + + +def test_merge_files_non_strict_collects_errors(tmp_path): + """strict=False collects errors for missing files and continues.""" + result = merge_files( + file_paths=["/nonexistent/path.json"], + output_path=str(tmp_path / "out.json"), + key="phases", + strict=False, + ) + assert "errors" in result + assert len(result["errors"]) == 1 + + +def test_merge_files_non_strict_invalid_json(tmp_path): + """strict=False collects errors for malformed JSON and continues.""" + bad = tmp_path / "bad.json" + bad.write_text("not json{{{") + + result = merge_files( + file_paths=[str(bad)], + output_path=str(tmp_path / "out.json"), + key="phases", + strict=False, + ) + assert "errors" in result + + +def test_merge_files_invalid_key_raises(tmp_path): + """merge_files raises ValueError for an unrecognised tier key.""" + with pytest.raises(ValueError, match="Invalid key"): + merge_files( + file_paths=[], + output_path=str(tmp_path / "out.json"), + key="unknown_tier", + ) + + +def test_extract_item_writes_extracted_item(tmp_path): + phases = [{"id": "P1", "name": "Phase 1"}, {"id": "P2", "name": "Phase 2"}] + doc = {"task": "", "source_dir": "", "phases": phases, "schema_version": 1} + src = tmp_path / "combined.json" + src.write_text(json.dumps(doc)) + out = tmp_path / "extracted.json" + + result = extract_item(source_path=str(src), item_id="P2", output_path=str(out)) + + assert result["extracted_path"] == str(out) + extracted = json.loads(out.read_text()) + assert extracted["id"] == "P2" + assert extracted["schema_version"] == 1 + + +def test_extract_item_missing_id_raises(tmp_path): + doc = {"task": "", "source_dir": "", "phases": [], "schema_version": 1} + src = tmp_path / "combined.json" + src.write_text(json.dumps(doc)) + + with pytest.raises(ValueError, match="not found"): + extract_item( + source_path=str(src), + item_id="MISSING", + output_path=str(tmp_path / "out.json"), + ) + + +def test_extract_item_missing_file_raises(tmp_path): + with pytest.raises(ValueError, match="Source file not found"): + extract_item( + source_path=str(tmp_path / "nonexistent.json"), + item_id="P1", + output_path=str(tmp_path / "out.json"), + ) + + +def test_extract_item_searches_all_tiers(tmp_path): + """extract_item finds items in phases, assignments, and work_packages.""" + doc = { + "task": "", + "source_dir": "", + "phases": [{"id": "P1"}], + "assignments": [{"id": "P1-A1"}], + "work_packages": [{"id": "P1-A1-WP1"}], + "schema_version": 1, + } + src = tmp_path / "combined.json" + src.write_text(json.dumps(doc)) + out = tmp_path / "extracted.json" + + extract_item(source_path=str(src), item_id="P1-A1-WP1", output_path=str(out)) + assert json.loads(out.read_text())["id"] == "P1-A1-WP1" + + +def test_replace_item_updates_combined_document(tmp_path): + phases = [{"id": "P1", "name": "Old"}, {"id": "P2", "name": "Phase 2"}] + doc = {"task": "", "source_dir": "", "phases": phases, "schema_version": 1} + src = tmp_path / "combined.json" + src.write_text(json.dumps(doc)) + rep_file = tmp_path / "rep.json" + rep_file.write_text(json.dumps({"id": "P1", "name": "New", "goal": "updated"})) + + result = replace_item(source_path=str(src), item_id="P1", replacement_path=str(rep_file)) + + assert result["replaced_id"] == "P1" + assert result["updated_path"] == str(src) + data = json.loads(src.read_text()) + p1 = next(p for p in data["phases"] if p["id"] == "P1") + assert p1["name"] == "New" + assert p1["goal"] == "updated" + assert len(data["phases"]) == 2 + + +def test_replace_item_in_assignments_tier(tmp_path): + """replace_item correctly updates items in the assignments tier.""" + doc = { + "task": "", + "source_dir": "", + "phases": [], + "assignments": [{"id": "P1-A1", "name": "Old assignment"}], + "schema_version": 1, + } + src = tmp_path / "combined.json" + src.write_text(json.dumps(doc)) + rep_file = tmp_path / "rep.json" + rep_file.write_text(json.dumps({"id": "P1-A1", "name": "Updated assignment"})) + + result = replace_item(source_path=str(src), item_id="P1-A1", replacement_path=str(rep_file)) + + assert result["replaced_id"] == "P1-A1" + data = json.loads(src.read_text()) + a1 = next(a for a in data["assignments"] if a["id"] == "P1-A1") + assert a1["name"] == "Updated assignment" + + +def test_replace_item_in_work_packages_tier(tmp_path): + """replace_item correctly updates items in the work_packages tier.""" + doc = { + "task": "", + "source_dir": "", + "phases": [], + "work_packages": [{"id": "P1-A1-WP1", "name": "Old WP"}], + "schema_version": 1, + } + src = tmp_path / "combined.json" + src.write_text(json.dumps(doc)) + rep_file = tmp_path / "rep.json" + rep_file.write_text(json.dumps({"id": "P1-A1-WP1", "name": "Updated WP"})) + + result = replace_item( + source_path=str(src), item_id="P1-A1-WP1", replacement_path=str(rep_file) + ) + + assert result["replaced_id"] == "P1-A1-WP1" + data = json.loads(src.read_text()) + wp1 = next(w for w in data["work_packages"] if w["id"] == "P1-A1-WP1") + assert wp1["name"] == "Updated WP" + + +def test_replace_item_missing_id_raises(tmp_path): + doc = {"task": "", "source_dir": "", "phases": [], "schema_version": 1} + src = tmp_path / "combined.json" + src.write_text(json.dumps(doc)) + rep_file = tmp_path / "rep.json" + rep_file.write_text(json.dumps({"id": "MISSING"})) + + with pytest.raises(ValueError, match="not found"): + replace_item(source_path=str(src), item_id="MISSING", replacement_path=str(rep_file)) + + +def test_replace_item_preserves_schema_version(tmp_path): + doc = { + "task": "", + "source_dir": "", + "phases": [{"id": "P1", "name": "x"}], + "schema_version": 1, + } + src = tmp_path / "combined.json" + src.write_text(json.dumps(doc)) + rep_file = tmp_path / "rep.json" + rep_file.write_text(json.dumps({"id": "P1", "name": "updated"})) + + replace_item(source_path=str(src), item_id="P1", replacement_path=str(rep_file)) + + assert json.loads(src.read_text())["schema_version"] == 1 + + +def test_build_plan_snapshot_produces_phase_ids(tmp_path): + phases_dir = tmp_path / "phases" + phases_dir.mkdir() + for phase_id in ["P1", "P2"]: + r = { + "id": phase_id, + "name": f"Phase {phase_id[1:]}", + "ordering": int(phase_id[1:]), + } + (phases_dir / f"{phase_id}_result.json").write_text(json.dumps(r)) + out = tmp_path / "snapshot.json" + + result = build_plan_snapshot( + phases_dir=str(phases_dir), + output_path=str(out), + task_file_path=write_task_file(tmp_path, "my task"), + source_dir="/src", + ) + + assert result["snapshot_path"] == str(out) + assert "P1" in result["phase_ids"] + assert "P2" in result["phase_ids"] + + +def test_build_plan_snapshot_writes_short_form_only(tmp_path): + """PhaseShort must not include elaborated fields; parallel workers receive only these keys.""" + phases_dir = tmp_path / "phases" + phases_dir.mkdir() + r = { + "id": "P1", + "name": "Phase One", + "goal": "some goal", + "ordering": 1, + "scope": ["s1"], + "relationship_notes": "should not appear", + "assignments_preview": ["A1"], + } + (phases_dir / "P1_result.json").write_text(json.dumps(r)) + out = tmp_path / "snapshot.json" + + build_plan_snapshot( + phases_dir=str(phases_dir), + output_path=str(out), + task_file_path=write_task_file(tmp_path, "t"), + source_dir="/s", + ) + + data = json.loads(out.read_text()) + assert data["task"] == "t" + assert data["source_dir"] == "/s" + assert data["schema_version"] == 1 + phase = data["phases"][0] + assert set(phase.keys()) == {"id", "name", "goal", "scope", "ordering"} + + +def test_build_plan_snapshot_projects_ordering(tmp_path) -> None: + """ordering is the sort key in build_plan_snapshot; validate_phase_result raises if absent.""" + phases_dir = tmp_path / "phases" + phases_dir.mkdir() + result = { + "id": "P1", + "name": "Foundation", + "goal": "Setup base", + "scope": ["core"], + "ordering": 1, + "assignments_preview": [], + "relationship_notes": "", + } + (phases_dir / "P1_result.json").write_text(json.dumps(result)) + out = tmp_path / "snapshot.json" + + build_plan_snapshot( + str(phases_dir), + str(out), + task_file_path=write_task_file(tmp_path, "test"), + source_dir="/src", + ) + + doc = json.loads(out.read_text()) + phase = doc["phases"][0] + assert phase["ordering"] == 1 + + +# --------------------------------------------------------------------------- +# WP1: build_plan_snapshot additional tests +# --------------------------------------------------------------------------- + + +def test_build_plan_snapshot_happy_path_two_phases_sorted(tmp_path) -> None: + phases_dir = tmp_path / "phases" + phases_dir.mkdir() + for phase_num, ordering in [(2, 2), (1, 1)]: + (phases_dir / f"P{phase_num}_result.json").write_text( + json.dumps(make_phase_result(phase_num)) + ) + out = tmp_path / "snapshot.json" + + result = build_plan_snapshot(str(phases_dir), str(out)) + + data = json.loads(out.read_text()) + assert len(data["phases"]) == 2 + assert data["phases"][0]["ordering"] == 1 + assert data["phases"][1]["ordering"] == 2 + assert "P1" in result["phase_ids"] + assert "P2" in result["phase_ids"] + + +def test_build_plan_snapshot_corrupt_json_skipped(tmp_path) -> None: + phases_dir = tmp_path / "phases" + phases_dir.mkdir() + (phases_dir / "P1_result.json").write_text(json.dumps(make_phase_result(1))) + (phases_dir / "bad_result.json").write_text("{not json") + out = tmp_path / "snapshot.json" + + result = build_plan_snapshot(str(phases_dir), str(out)) + + data = json.loads(out.read_text()) + assert len(data["phases"]) == 1 + assert result["phase_ids"] == "P1" + + +def test_build_plan_snapshot_nonexistent_phases_dir(tmp_path) -> None: + out = tmp_path / "snapshot.json" + + result = build_plan_snapshot(str(tmp_path / "nonexistent"), str(out)) + + data = json.loads(out.read_text()) + assert data["phases"] == [] + assert result["phase_ids"] == "" + + +def test_build_plan_snapshot_empty_dir_produces_empty_phases(tmp_path) -> None: + phases_dir = tmp_path / "phases" + phases_dir.mkdir() + out = tmp_path / "snapshot.json" + + result = build_plan_snapshot(str(phases_dir), str(out)) + + data = json.loads(out.read_text()) + assert data["phases"] == [] + assert result["phase_ids"] == "" + + +# --------------------------------------------------------------------------- +# WP3: merge_tier_results tests +# --------------------------------------------------------------------------- + + +def test_merge_tier_results_empty_dir_raises(tmp_path) -> None: + empty_dir = tmp_path / "phases" + empty_dir.mkdir() + out = tmp_path / "combined.json" + + with pytest.raises(ValueError, match="No \\*_result.json files found"): + merge_tier_results(str(empty_dir), str(out), "phases") + + +def test_merge_tier_results_single_file(tmp_path) -> None: + results_dir = tmp_path / "phases" + results_dir.mkdir() + (results_dir / "P1_result.json").write_text( + json.dumps({"id": "P1", "name": "Phase 1", "ordering": 1}) + ) + out = tmp_path / "combined.json" + + result = merge_tier_results(str(results_dir), str(out), "phases") + + assert result["item_count"] == "1" + assert out.exists() + data = json.loads(out.read_text()) + assert len(data["phases"]) == 1 + assert data["phases"][0]["id"] == "P1" + + +# --------------------------------------------------------------------------- +# WP4: replace_item error branches +# --------------------------------------------------------------------------- + + +def test_replace_item_nonexistent_replacement_file_raises(tmp_path) -> None: + src = tmp_path / "combined.json" + src.write_text(json.dumps({"phases": [{"id": "P1", "name": "x"}], "schema_version": 1})) + + with pytest.raises(ValueError, match="Replacement file not found"): + replace_item(str(src), "P1", str(tmp_path / "nonexistent.json")) + + +def test_replace_item_corrupt_replacement_json_raises(tmp_path) -> None: + src = tmp_path / "combined.json" + src.write_text(json.dumps({"phases": [{"id": "P1", "name": "x"}], "schema_version": 1})) + corrupt = tmp_path / "corrupt.json" + corrupt.write_text("{not json") + + with pytest.raises(ValueError, match="Invalid JSON"): + replace_item(str(src), "P1", str(corrupt)) + + +def test_build_plan_snapshot_reads_task_from_task_file_path(tmp_path) -> None: + phases_dir = tmp_path / "phases" + phases_dir.mkdir() + (phases_dir / "P1_result.json").write_text(json.dumps(make_phase_result(1))) + out = tmp_path / "snapshot.json" + task_file = tmp_path / "task_desc.txt" + task_file.write_text("Full task from file") + + build_plan_snapshot(str(phases_dir), str(out), task_file_path=str(task_file)) + + data = json.loads(out.read_text()) + assert data["task"] == "Full task from file" + + +def test_merge_tier_results_reads_task_from_task_file_path(tmp_path) -> None: + results_dir = tmp_path / "phases" + results_dir.mkdir() + (results_dir / "P1_result.json").write_text( + json.dumps({"id": "P1", "name": "Phase 1", "ordering": 1}) + ) + out = tmp_path / "combined.json" + task_file = tmp_path / "task_desc.txt" + task_file.write_text("Full task from file") + + merge_tier_results(str(results_dir), str(out), "phases", task_file_path=str(task_file)) + + data = json.loads(out.read_text()) + assert data["task"] == "Full task from file" + + +def test_merge_files_reads_task_from_task_file_path(tmp_path) -> None: + item = {"id": "P1", "name": "Phase 1"} + p = tmp_path / "P1_result.json" + p.write_text(json.dumps(item)) + out = tmp_path / "combined.json" + task_file = tmp_path / "task_desc.txt" + task_file.write_text("Full task from file") + + merge_files([str(p)], str(out), "phases", task_file_path=str(task_file)) + + data = json.loads(out.read_text()) + assert data["task"] == "Full task from file" + + +# --------------------------------------------------------------------------- +# Per-phase refine context file tests +# --------------------------------------------------------------------------- + + +def test_merge_tier_results_writes_refine_contexts_for_assignments(tmp_path): + results_dir = tmp_path / "assignments" + results_dir.mkdir() + write_json( + results_dir / "P1-A1_result.json", + { + "id": "P1-A1", + "phase_id": "P1", + "name": "Auth", + "goal": "Auth goal", + "technical_approach": "JWT", + "proposed_work_packages": [], + }, + ) + write_json( + results_dir / "P2-A1_result.json", + { + "id": "P2-A1", + "phase_id": "P2", + "name": "Data", + "goal": "Data goal", + "technical_approach": "ORM", + "proposed_work_packages": [], + }, + ) + out = tmp_path / "combined_assignments.json" + result = merge_tier_results(str(results_dir), str(out), "assignments") + assert "refine_context_paths" in result + paths = [p for p in result["refine_context_paths"].split(",") if p.strip()] + assert len(paths) == 2 + p1_ctx = tmp_path / "refine_contexts" / "context_P1.json" + p2_ctx = tmp_path / "refine_contexts" / "context_P2.json" + assert p1_ctx.exists() + assert p2_ctx.exists() + assert str(p1_ctx) in paths + assert str(p2_ctx) in paths + + +def test_refine_context_own_assignments_in_full_detail(tmp_path): + results_dir = tmp_path / "assignments" + results_dir.mkdir() + write_json( + results_dir / "P1-A1_result.json", + { + "id": "P1-A1", + "phase_id": "P1", + "name": "Auth", + "goal": "Auth goal", + "technical_approach": "JWT", + "proposed_work_packages": [{"id": "wp1"}], + }, + ) + write_json( + results_dir / "P2-A1_result.json", + { + "id": "P2-A1", + "phase_id": "P2", + "name": "Data", + "goal": "Data goal", + "technical_approach": "ORM", + "proposed_work_packages": [], + }, + ) + out = tmp_path / "combined_assignments.json" + merge_tier_results(str(results_dir), str(out), "assignments") + ctx = json.loads((tmp_path / "refine_contexts" / "context_P1.json").read_text()) + assert len(ctx["assignments"]) == 1 + own = ctx["assignments"][0] + assert own["id"] == "P1-A1" + assert own["technical_approach"] == "JWT" + assert own["proposed_work_packages"] == [{"id": "wp1"}] + assert all(a["id"] != "P2-A1" for a in ctx["assignments"]) + + +def test_refine_context_peer_summaries_have_filtered_fields(tmp_path): + results_dir = tmp_path / "assignments" + results_dir.mkdir() + write_json( + results_dir / "P1-A1_result.json", + { + "id": "P1-A1", + "phase_id": "P1", + "name": "Auth", + "goal": "Auth goal", + "technical_approach": "JWT", + "proposed_work_packages": [], + }, + ) + write_json( + results_dir / "P2-A1_result.json", + { + "id": "P2-A1", + "phase_id": "P2", + "name": "Data", + "goal": "Data goal", + "technical_approach": "ORM", + "proposed_work_packages": [{"id": "wp2"}], + }, + ) + out = tmp_path / "combined_assignments.json" + merge_tier_results(str(results_dir), str(out), "assignments") + ctx = json.loads((tmp_path / "refine_contexts" / "context_P1.json").read_text()) + assert len(ctx["peer_summaries"]) == 1 + peer = ctx["peer_summaries"][0] + assert peer == {"id": "P2-A1", "name": "Data", "goal": "Data goal"} + assert "technical_approach" not in peer + assert "proposed_work_packages" not in peer + + +def test_refine_context_has_task_file_path_not_inline_task(tmp_path): + results_dir = tmp_path / "assignments" + results_dir.mkdir() + task_path = write_task_file(tmp_path, "Build a system") + write_json( + results_dir / "P1-A1_result.json", + { + "id": "P1-A1", + "phase_id": "P1", + "name": "Auth", + "goal": "Auth goal", + "technical_approach": "", + "proposed_work_packages": [], + }, + ) + out = tmp_path / "combined_assignments.json" + merge_tier_results(str(results_dir), str(out), "assignments", task_file_path=task_path) + ctx = json.loads((tmp_path / "refine_contexts" / "context_P1.json").read_text()) + assert ctx["task_file_path"] == task_path + assert "task" not in ctx + + +def test_refine_context_paths_returned_sorted(tmp_path): + results_dir = tmp_path / "assignments" + results_dir.mkdir() + for pid in ("P3", "P1", "P2"): + write_json( + results_dir / f"{pid}-A1_result.json", + { + "id": f"{pid}-A1", + "phase_id": pid, + "name": pid, + "goal": f"{pid} goal", + "technical_approach": "", + "proposed_work_packages": [], + }, + ) + out = tmp_path / "combined_assignments.json" + result = merge_tier_results(str(results_dir), str(out), "assignments") + paths = [p for p in result["refine_context_paths"].split(",") if p.strip()] + phase_ids = [Path(p).stem.replace("context_", "") for p in paths] + assert phase_ids == sorted(phase_ids) + assert set(phase_ids) == {"P1", "P2", "P3"} + + +def test_merge_tier_results_no_refine_contexts_for_phases_key(tmp_path): + results_dir = tmp_path / "phases" + results_dir.mkdir() + write_json( + results_dir / "P1_result.json", + { + "id": "P1", + "name": "Phase 1", + "ordering": 1, + "goal": "g", + "scope": [], + }, + ) + out = tmp_path / "combined_plan.json" + result = merge_tier_results(str(results_dir), str(out), "phases") + assert "refine_context_paths" not in result + assert not (tmp_path / "refine_contexts").exists() + + +def test_merge_tier_results_no_refine_contexts_for_work_packages_key(tmp_path): + results_dir = tmp_path / "work_packages" + results_dir.mkdir() + write_json( + results_dir / "P1-A1-WP1_result.json", + { + "id": "P1-A1-WP1", + "name": "WP One", + "summary": "s", + "goal": "g", + "technical_steps": [], + "files_touched": [], + "apis_defined": [], + "apis_consumed": [], + "depends_on": [], + "deliverables": ["d1"], + "acceptance_criteria": [], + }, + ) + out = tmp_path / "combined_wps.json" + result = merge_tier_results(str(results_dir), str(out), "work_packages") + assert "refine_context_paths" not in result + assert not (tmp_path / "refine_contexts").exists() + + +def test_write_refine_contexts_rejects_unsafe_phase_id(tmp_path): + results_dir = tmp_path / "assignments" + results_dir.mkdir() + write_json( + results_dir / "P1-A1_result.json", + {"id": "P1-A1", "phase_id": "../../evil", "name": "x", "goal": "g"}, + ) + out = tmp_path / "combined.json" + with pytest.raises(ValueError, match="disallowed characters"): + merge_tier_results(str(results_dir), str(out), "assignments") + + +# --------------------------------------------------------------------------- +# merge_refined_assignments tests (B1–B5) +# --------------------------------------------------------------------------- + + +def _make_assignment(aid: str, phase_id: str, files: list[str]) -> dict: + return { + "id": aid, + "phase_id": phase_id, + "name": f"Assignment {aid}", + "goal": f"Goal for {aid}", + "technical_approach": "", + "proposed_work_packages": [ + { + "id": f"{aid}-WP1", + "name": "WP1", + "summary": "s", + "goal": "g", + "technical_steps": [], + "files_touched": files, + "apis_defined": [], + "apis_consumed": [], + "depends_on": [], + "deliverables": ["d1"], + "acceptance_criteria": [], + } + ], + } + + +def _write_phase_result(dir_: Path, phase_id: str, assignments: list[dict]) -> None: + write_json( + dir_ / f"{phase_id}_result.json", + {"schema_version": 1, "assignments": assignments}, + ) + + +def test_merge_refined_assignments_basic(tmp_path): + ctx_dir = tmp_path / "refine_contexts" + ctx_dir.mkdir() + _write_phase_result( + ctx_dir, + "P1", + [ + _make_assignment("P1-A1", "P1", ["src/a.py"]), + _make_assignment("P1-A2", "P1", ["src/b.py"]), + ], + ) + _write_phase_result( + ctx_dir, + "P2", + [ + _make_assignment("P2-A1", "P2", ["src/c.py"]), + _make_assignment("P2-A2", "P2", ["src/d.py"]), + ], + ) + + result = merge_refined_assignments(planner_dir=str(tmp_path)) + + assert "refined_assignments_path" in result + out_path = Path(result["refined_assignments_path"]) + assert out_path.exists() + data = json.loads(out_path.read_text()) + assert len(data["assignments"]) == 4 + assert result["item_count"] == "4" + + +def test_merge_refined_assignments_wp_conflict_earlier_wins(tmp_path): + ctx_dir = tmp_path / "refine_contexts" + ctx_dir.mkdir() + _write_phase_result( + ctx_dir, + "P1", + [ + _make_assignment("P1-A1", "P1", ["src/foo.py", "src/bar.py"]), + ], + ) + _write_phase_result( + ctx_dir, + "P2", + [ + _make_assignment("P2-A1", "P2", ["src/foo.py", "src/baz.py"]), + ], + ) + + result = merge_refined_assignments(planner_dir=str(tmp_path)) + + data = json.loads(Path(result["refined_assignments_path"]).read_text()) + assignments = {a["id"]: a for a in data["assignments"]} + + p1_files = assignments["P1-A1"]["proposed_work_packages"][0]["files_touched"] + p2_files = assignments["P2-A1"]["proposed_work_packages"][0]["files_touched"] + + assert "src/foo.py" in p1_files + assert "src/foo.py" not in p2_files + assert "src/baz.py" in p2_files + assert result["conflict_count"] == "1" + + +def test_merge_refined_assignments_no_conflict(tmp_path): + ctx_dir = tmp_path / "refine_contexts" + ctx_dir.mkdir() + _write_phase_result( + ctx_dir, + "P1", + [ + _make_assignment("P1-A1", "P1", ["src/a.py"]), + ], + ) + _write_phase_result( + ctx_dir, + "P2", + [ + _make_assignment("P2-A1", "P2", ["src/b.py"]), + ], + ) + + result = merge_refined_assignments(planner_dir=str(tmp_path)) + + data = json.loads(Path(result["refined_assignments_path"]).read_text()) + assignments = {a["id"]: a for a in data["assignments"]} + + assert assignments["P1-A1"]["proposed_work_packages"][0]["files_touched"] == ["src/a.py"] + assert assignments["P2-A1"]["proposed_work_packages"][0]["files_touched"] == ["src/b.py"] + assert result["conflict_count"] == "0" + + +def test_merge_refined_assignments_empty_dir_raises(tmp_path): + ctx_dir = tmp_path / "refine_contexts" + ctx_dir.mkdir() + # Only context files (no *_result.json) + write_json(ctx_dir / "context_P1.json", {"phase_id": "P1", "assignments": []}) + + with pytest.raises(ValueError, match="No.*_result.json"): + merge_refined_assignments(planner_dir=str(tmp_path)) + + +def test_merge_refined_assignments_writes_to_planner_dir(tmp_path): + ctx_dir = tmp_path / "refine_contexts" + ctx_dir.mkdir() + _write_phase_result(ctx_dir, "P1", [_make_assignment("P1-A1", "P1", ["src/x.py"])]) + _write_phase_result(ctx_dir, "P2", [_make_assignment("P2-A1", "P2", ["src/y.py"])]) + + result = merge_refined_assignments(planner_dir=str(tmp_path)) + + expected = tmp_path / "refined_assignments.json" + assert Path(result["refined_assignments_path"]) == expected + assert expected.exists() + + +def test_merge_tier_results_skips_non_assignment_files(tmp_path): + """Sentinel files in assignments/ must not be included in merge output.""" + from tests.planner.conftest import make_assignment_result + + assign_dir = tmp_path / "assignments" + assign_dir.mkdir() + out = tmp_path / "combined.json" + + write_json( + assign_dir / "P1-A1_result.json", + make_assignment_result(1, 1), + ) + write_json( + assign_dir / "P1_result.json", + {"id": "P1", "status": "complete", "assignment_count": 1, "failed_count": 0}, + ) + + task_file = write_task_file(tmp_path) + merge_tier_results(str(assign_dir), str(out), "assignments", task_file_path=task_file) + + merged = json.loads(out.read_text()) + ids = [a["id"] for a in merged["assignments"]] + assert "P1-A1" in ids + assert "P1" not in ids + + +def test_merge_files_produces_indented_output(tmp_path: Path) -> None: + out = tmp_path / "combined.json" + item = tmp_path / "item.json" + item.write_text(json.dumps({"id": "P1", "name": "Phase 1"})) + merge_files(file_paths=[str(item)], output_path=str(out), key="phases") + raw = out.read_text(encoding="utf-8") + lines = raw.strip().splitlines() + assert len(lines) > 1, "merge_files output must be multi-line (indented)" diff --git a/tests/planner/test_pipeline_integration.py b/tests/planner/test_pipeline_integration.py new file mode 100644 index 0000000000..7b97b418e3 --- /dev/null +++ b/tests/planner/test_pipeline_integration.py @@ -0,0 +1,219 @@ +"""End-to-end pipeline integration tests. + +These tests are the architectural backstop: any future change that breaks the schema +alignment between SKILL.md output, the Python validation boundary, and the consumers +will cause at least one test here to fail. +""" + +from __future__ import annotations + +import json +from pathlib import Path + +import pytest + +from tests.planner.conftest import ( + make_assignment_result, + make_phase_result, + make_wp_result, + write_json, + write_task_file, +) + +pytestmark = [pytest.mark.layer("planner"), pytest.mark.small, pytest.mark.feature("planner")] + + +def test_multi_phase_pipeline_end_to_end(tmp_path: Path) -> None: + """Two-phase pipeline: SKILL.md-compliant data flows through every pipeline stage.""" + from autoskillit.planner import ( + compile_plan, + expand_assignments, + expand_wps, + finalize_wp_manifest, + validate_plan, + ) + + # Phase results (needed by validate_plan/compile_plan) + write_json( + tmp_path / "phases" / "P1_result.json", + { + "id": "P1", + "name": "Foundation", + "ordering": 1, + "goal": "Establish core infrastructure", + "scope": ["core"], + "relationship_notes": "No prior dependencies", + "assignments_preview": ["Core setup"], + }, + ) + write_json( + tmp_path / "phases" / "P2_result.json", + { + "id": "P2", + "name": "Application Layer", + "ordering": 2, + "goal": "Implement application logic on top of foundation", + "scope": ["app"], + "relationship_notes": "Depends on Phase 1", + "assignments_preview": ["App module"], + }, + ) + + write_json( + tmp_path / "refined_plan.json", + { + "phases": [ + { + "id": "P1", + "name": "Foundation", + "ordering": 1, + "assignments_preview": [{"id": "P1-A1", "name": "Core setup"}], + }, + { + "id": "P2", + "name": "Application Layer", + "ordering": 2, + "assignments_preview": [{"id": "P2-A1", "name": "App module"}], + }, + ] + }, + ) + + expand_assignments(str(tmp_path / "refined_plan.json"), str(tmp_path)) + + write_json( + tmp_path / "assignments" / "P1-A1_result.json", + { + "id": "P1-A1", + "name": "Core setup", + "phase_id": "P1", + "goal": "Set up core modules", + "technical_approach": "Direct implementation", + "proposed_work_packages": [ + { + "id_suffix": "WP1", + "name": "Core module", + "scope": "core", + "estimated_files": ["core.py"], + } + ], + }, + ) + write_json( + tmp_path / "assignments" / "P2-A1_result.json", + { + "id": "P2-A1", + "name": "App module", + "phase_id": "P2", + "goal": "Build application layer", + "technical_approach": "Build on core", + "proposed_work_packages": [ + { + "id_suffix": "WP1", + "name": "App module", + "scope": "app", + "estimated_files": ["app.py"], + } + ], + }, + ) + + write_json( + tmp_path / "refined_assignments.json", + { + "assignments": [ + { + "id": "P1-A1", + "phase_id": "P1", + "phase_name": "Foundation", + "phase_number": 1, + "assignment_number": 1, + "proposed_work_packages": [ + {"id": "P1-A1-WP1", "name": "Core module", "scope": "core"}, + ], + }, + { + "id": "P2-A1", + "phase_id": "P2", + "phase_name": "Application Layer", + "phase_number": 2, + "assignment_number": 1, + "proposed_work_packages": [ + {"id": "P2-A1-WP1", "name": "App module", "scope": "app"}, + ], + }, + ] + }, + ) + + expand_wps(str(tmp_path / "refined_assignments.json"), str(tmp_path)) + + wp_dir = tmp_path / "work_packages" + write_json( + wp_dir / "P1-A1-WP1_result.json", + make_wp_result("P1-A1-WP1", deliverables=["src/core.py"]), + ) + write_json( + wp_dir / "P2-A1-WP1_result.json", + make_wp_result("P2-A1-WP1", deliverables=["src/app.py"], depends_on=["P1-A1-WP1"]), + ) + + finalize_wp_manifest(str(wp_dir), str(tmp_path)) + + validate_result = validate_plan(str(tmp_path)) + assert validate_result["verdict"] == "pass", validate_result + + compile_result = compile_plan( + str(tmp_path), write_task_file(tmp_path, "integration test task"), "/src" + ) + + plan_path = Path(compile_result["plan_path"]) + assert plan_path.exists() + plan_md = plan_path.read_text() + assert "## Phase 1: Foundation" in plan_md + assert "## Phase 2: Application Layer" in plan_md + + milestones = json.loads((tmp_path / "milestones.json").read_text()) + slugs = [m["name_slug"] for m in milestones["milestones"]] + assert "foundation" in slugs + assert "application-layer" in slugs + + manifest = json.loads((tmp_path / "manifest.json").read_text()) + order = manifest["execution_order"] + assert order.index("P1-A1-WP1") < order.index("P2-A1-WP1") + + +def test_pipeline_factory_fixtures_are_schema_compliant(tmp_path: Path) -> None: + """Factory-built fixtures pass through the full pipeline without errors.""" + from autoskillit.planner import compile_plan, validate_plan + + # Build everything using factory functions (SKILL.md-derived, not raw backend fields) + write_json( + tmp_path / "phases" / "P1_result.json", + make_phase_result(1, name="Schema Alignment"), + ) + write_json( + tmp_path / "assignments" / "P1-A1_result.json", + make_assignment_result(1, 1, name="Implement schema"), + ) + write_json( + tmp_path / "work_packages" / "P1-A1-WP1_result.json", + make_wp_result("P1-A1-WP1"), + ) + write_json( + tmp_path / "work_packages" / "wp_manifest.json", + {"pass_name": "work_packages", "items": [{"id": "P1-A1-WP1", "status": "done"}]}, + ) + write_json( + tmp_path / "validation.json", + {"verdict": "pass", "findings": [], "warnings": [], "schema_version": 2}, + ) + + validate_result = validate_plan(str(tmp_path)) + assert validate_result["verdict"] == "pass" + + compile_result = compile_plan(str(tmp_path), write_task_file(tmp_path, "factory test"), "/src") + assert Path(compile_result["plan_path"]).exists() + + milestones = json.loads((tmp_path / "milestones.json").read_text()) + assert milestones["milestones"][0]["name_slug"] == "schema-alignment" diff --git a/tests/planner/test_planner.py b/tests/planner/test_planner.py new file mode 100644 index 0000000000..4316bb8955 --- /dev/null +++ b/tests/planner/test_planner.py @@ -0,0 +1,572 @@ +"""Tests for the planner L1 subpackage scaffold.""" + +from __future__ import annotations + +import inspect +import json +from pathlib import Path + +import pytest + +pytestmark = [pytest.mark.layer("planner"), pytest.mark.small, pytest.mark.feature("planner")] + + +def test_planner_package_importable() -> None: + import autoskillit.planner # noqa: F401 + + +def test_planner_all_exports() -> None: + from autoskillit.planner import __all__ + + assert set(__all__) == { + "ASSIGN_RESULT_FILE_RE", + "DELIVERABLE_BOUNDS", + "PHASE_RESULT_FILE_RE", + "WP_RESULT_FILE_RE", + "build_phase_assignment_manifest", + "build_phase_wp_manifest", + "compile_plan", + "consolidate_wps", + "create_run_dir", + "expand_assignments", + "expand_wps", + "finalize_wp_manifest", + "validate_plan", + "PlannerManifest", + "PlannerManifestItem", + "merge_files", + "merge_tier_results", + "extract_item", + "replace_item", + "build_plan_snapshot", + "merge_refined_assignments", + "PlanDocument", + "PhaseShort", + "PhaseElaborated", + "AssignmentShort", + "AssignmentElaborated", + "WPShort", + "WPElaborated", + "ASSIGNMENT_REQUIRED_KEYS", + "PHASE_REQUIRED_KEYS", + "WP_REQUIRED_KEYS", + "resolve_wp_id", + "validate_refined_assignments", + "validate_refined_plan", + "ValidationFinding", + "resolve_task_input", + } + + +def test_sequential_state_machine_removed() -> None: + """Guard: sequential state machine functions must not exist in manifests.""" + import autoskillit.planner.manifests as m + + for name in ( + "check_remaining", + "build_assignment_manifest", + "build_wp_manifest", + "_backstop_wp_index", + ): + assert not hasattr(m, name), f"{name} still exists — should have been removed" + + +def test_create_run_dir_requires_temp_dir_argument() -> None: + """create_run_dir must accept temp_dir as explicit parameter, not read os.environ.""" + from autoskillit.planner import create_run_dir + + sig = inspect.signature(create_run_dir) + assert "temp_dir" in sig.parameters, ( + "create_run_dir must accept temp_dir as an explicit keyword argument" + ) + assert sig.parameters["temp_dir"].default is inspect.Parameter.empty, ( + "temp_dir must be a required parameter (no default)" + ) + + +def test_create_run_dir_does_not_read_environ(tmp_path, monkeypatch) -> None: + """create_run_dir must not depend on AUTOSKILLIT_TEMP env var.""" + from autoskillit.planner import create_run_dir + + monkeypatch.delenv("AUTOSKILLIT_TEMP", raising=False) + result = create_run_dir(temp_dir=str(tmp_path)) + assert result["planner_dir"] + assert Path(result["planner_dir"]).exists() + + +def test_create_run_dir_creates_timestamped_directory(tmp_path) -> None: + from autoskillit.planner import create_run_dir + + result = create_run_dir(temp_dir=str(tmp_path)) + + assert "planner_dir" in result + planner_dir = Path(result["planner_dir"]) + assert planner_dir.exists() + assert planner_dir.parent.name == "planner" + assert planner_dir.name.startswith("run-") + assert (planner_dir / "phases").is_dir() + assert (planner_dir / "assignments").is_dir() + assert (planner_dir / "work_packages").is_dir() + + +def test_create_run_dir_unique_across_calls(tmp_path) -> None: + from autoskillit.planner import create_run_dir + + r1 = create_run_dir(temp_dir=str(tmp_path)) + r2 = create_run_dir(temp_dir=str(tmp_path)) + + assert r1["planner_dir"] != r2["planner_dir"] + planner_dir2 = Path(r2["planner_dir"]) + assert planner_dir2.exists() + assert planner_dir2.parent.name == "planner" + assert planner_dir2.name.startswith("run-") + assert (planner_dir2 / "phases").is_dir() + assert (planner_dir2 / "assignments").is_dir() + assert (planner_dir2 / "work_packages").is_dir() + + +def test_create_run_dir_empty_string_raises_value_error() -> None: + from autoskillit.planner import create_run_dir + + with pytest.raises(ValueError, match="non-empty"): + create_run_dir("") + + +def test_planner_feature_skill_categories() -> None: + from autoskillit.core.types._type_constants import FEATURE_REGISTRY + + assert FEATURE_REGISTRY["planner"].skill_categories == frozenset({"planner"}) + + +# --- T6: New callables importable --- + + +def test_expand_assignments_importable() -> None: + from autoskillit.planner.manifests import expand_assignments + + assert callable(expand_assignments) + + +def test_expand_wps_importable() -> None: + from autoskillit.planner.manifests import expand_wps + + assert callable(expand_wps) + + +# --- T7: New callables functional tests --- + + +def test_merge_tier_results_globs_and_merges(tmp_path) -> None: + from autoskillit.planner.merge import merge_tier_results + + results_dir = tmp_path / "phases" + results_dir.mkdir() + for pid in ("P1", "P2"): + (results_dir / f"{pid}_result.json").write_text( + json.dumps({"id": pid, "name": f"Phase {pid}", "ordering": int(pid[1])}) + ) + out = tmp_path / "combined.json" + result = merge_tier_results(str(results_dir), str(out), "phases") + assert result["item_count"] == "2" + assert out.exists() + merged = json.loads(out.read_text()) + merged_ids = {item["id"] for item in merged["phases"]} + assert merged_ids == {"P1", "P2"} + + +def test_expand_assignments_creates_contexts(tmp_path) -> None: + from autoskillit.planner.manifests import expand_assignments + + refined = tmp_path / "refined_plan.json" + refined.write_text( + json.dumps( + { + "phases": [ + { + "id": "P1", + "name": "Phase 1", + "ordering": 1, + "assignments_preview": [{"name": "A1"}, {"name": "A2"}], + }, + ] + } + ) + ) + result = expand_assignments(str(refined), str(tmp_path)) + assert "manifest_path" in result + assert "context_paths" in result + assert "P1" in result.get("item_ids", "") + + +def test_expand_wps_creates_contexts(tmp_path) -> None: + from autoskillit.planner.manifests import expand_wps + + refined = tmp_path / "refined_assignments.json" + refined.write_text( + json.dumps( + { + "assignments": [ + { + "phase_id": "P1", + "phase_name": "Phase 1", + "phase_number": 1, + "assignment_number": 1, + "proposed_work_packages": [ + {"id": "P1-A1-WP1", "name": "WP1", "scope": "scope1"}, + ], + }, + ] + } + ) + ) + result = expand_wps(str(refined), str(tmp_path)) + assert "manifest_path" in result + assert "context_paths" in result + assert "P1" in result.get("item_ids", "") + + +# --- Validation gate tests (ARCH-010) --- + + +def test_validate_refined_assignments_rejects_empty_id() -> None: + from autoskillit.planner.schema import validate_refined_assignments + + data = { + "assignments": [ + { + "id": "P1-A1", + "phase_id": "P1", + "phase_number": 1, + "assignment_number": 1, + "proposed_work_packages": [{"name": "No ID", "scope": "s"}], + } + ] + } + with pytest.raises(ValueError, match="neither 'id' nor 'id_suffix'"): + validate_refined_assignments(data) + + +def test_validate_refined_assignments_resolves_id_suffix() -> None: + from autoskillit.planner.schema import validate_refined_assignments + + data = { + "assignments": [ + { + "id": "P1-A1", + "phase_id": "P1", + "phase_number": 1, + "assignment_number": 1, + "proposed_work_packages": [ + {"id_suffix": "WP1", "name": "First", "scope": "s1"}, + ], + } + ] + } + result = validate_refined_assignments(data) + wp = result["assignments"][0]["proposed_work_packages"][0] + assert wp["id"] == "P1-A1-WP1" + + +def test_validate_refined_assignments_rejects_unresolvable_assign_id() -> None: + from autoskillit.planner.schema import validate_refined_assignments + + data = { + "assignments": [ + { + "phase_name": "Phase 1", + "proposed_work_packages": [{"id_suffix": "WP1", "name": "X"}], + } + ] + } + with pytest.raises(ValueError, match="assignment.*resolvable id"): + validate_refined_assignments(data) + + +def test_validate_refined_plan_rejects_empty_phase_id() -> None: + from autoskillit.planner.schema import validate_refined_plan + + data = { + "phases": [ + { + "id": "", + "name": "Bad Phase", + "assignments_preview": [], + } + ] + } + with pytest.raises(ValueError, match="empty.*id"): + validate_refined_plan(data) + + +def test_expand_assignments_rejects_empty_preview_ids(tmp_path) -> None: + from autoskillit.planner.manifests import expand_assignments + + refined = tmp_path / "refined_plan.json" + refined.write_text( + json.dumps( + { + "phases": [ + { + "id": "P1", + "name": "Phase 1", + "assignments_preview": [{"name": "A1"}], + } + ] + } + ) + ) + result = expand_assignments(str(refined), str(tmp_path)) + manifest = json.loads(Path(result["manifest_path"]).read_text()) + ids = manifest["items"][0]["metadata"]["assignment_ids"] + assert all(id_val != "" for id_val in ids) + + +def test_resolve_wp_id_from_id() -> None: + from autoskillit.planner.schema import resolve_wp_id + + assert resolve_wp_id({"id": "P1-A1-WP1"}, "P1-A1") == "P1-A1-WP1" + + +def test_resolve_wp_id_from_suffix() -> None: + from autoskillit.planner.schema import resolve_wp_id + + assert resolve_wp_id({"id_suffix": "WP2"}, "P1-A1") == "P1-A1-WP2" + + +def test_resolve_wp_id_raises_on_neither() -> None: + from autoskillit.planner.schema import resolve_wp_id + + with pytest.raises(ValueError, match="neither 'id' nor 'id_suffix'"): + resolve_wp_id({"name": "No ID"}, "P1-A1") + + +def test_resolve_wp_id_rejects_empty_string_id() -> None: + from autoskillit.planner.schema import resolve_wp_id + + assert resolve_wp_id({"id": "", "id_suffix": "WP1"}, "P1-A1") == "P1-A1-WP1" + + +def test_expand_wps_resolves_id_suffix(tmp_path) -> None: + from autoskillit.planner.manifests import expand_wps + + refined = tmp_path / "refined_assignments.json" + refined.write_text( + json.dumps( + { + "assignments": [ + { + "id": "P1-A1", + "phase_id": "P1", + "phase_number": 1, + "assignment_number": 1, + "proposed_work_packages": [ + {"id_suffix": "WP1", "name": "First", "scope": "s1"}, + {"id_suffix": "WP2", "name": "Second", "scope": "s2"}, + ], + } + ] + } + ) + ) + result = expand_wps(str(refined), str(tmp_path)) + manifest = json.loads(Path(result["manifest_path"]).read_text()) + wp_ids = manifest["items"][0]["metadata"]["wp_ids"] + assert wp_ids == ["P1-A1-WP1", "P1-A1-WP2"] + + +def test_expand_wps_synthesizes_assign_id_from_numbers(tmp_path) -> None: + from autoskillit.planner.manifests import expand_wps + + refined = tmp_path / "refined_assignments.json" + refined.write_text( + json.dumps( + { + "assignments": [ + { + "phase_number": 2, + "assignment_number": 3, + "proposed_work_packages": [ + {"id_suffix": "WP1", "name": "WP", "scope": "s"}, + ], + } + ] + } + ) + ) + result = expand_wps(str(refined), str(tmp_path)) + manifest = json.loads(Path(result["manifest_path"]).read_text()) + wp_ids = manifest["items"][0]["metadata"]["wp_ids"] + assert wp_ids == ["P2-A3-WP1"] + + +def test_expand_wps_raises_on_missing_id_and_suffix(tmp_path) -> None: + from autoskillit.planner.manifests import expand_wps + + refined = tmp_path / "refined_assignments.json" + refined.write_text( + json.dumps( + { + "assignments": [ + { + "id": "P1-A1", + "phase_id": "P1", + "phase_number": 1, + "assignment_number": 1, + "proposed_work_packages": [ + {"name": "No ID WP", "scope": "s1"}, + ], + } + ] + } + ) + ) + with pytest.raises(ValueError, match="neither 'id' nor 'id_suffix'"): + expand_wps(str(refined), str(tmp_path)) + + +def test_merge_files_does_not_use_atomic_write( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """merge_files must write directly under flock, not via atomic_write/os.replace.""" + from unittest.mock import MagicMock + + from autoskillit.planner.merge import merge_files + + spy = MagicMock(side_effect=AssertionError("atomic_write must not be called")) + monkeypatch.setattr("autoskillit.planner.merge.write_versioned_json", spy) + + results_dir = tmp_path / "phases" + results_dir.mkdir() + (results_dir / "P1_result.json").write_text( + json.dumps({"id": "P1", "name": "Phase 1", "ordering": 1}) + ) + + out = tmp_path / "combined.json" + result = merge_files([str(results_dir / "P1_result.json")], str(out), "phases") + assert result["item_count"] == "1" + merged = json.loads(out.read_text()) + assert merged["schema_version"] == 1 + assert len(merged["phases"]) == 1 + spy.assert_not_called() + + +def test_replace_item_does_not_use_atomic_write( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """replace_item must write directly under flock, not via atomic_write/os.replace.""" + from unittest.mock import MagicMock + + from autoskillit.planner.merge import replace_item + + spy = MagicMock(side_effect=AssertionError("atomic_write must not be called")) + monkeypatch.setattr("autoskillit.planner.merge.write_versioned_json", spy) + + src = tmp_path / "combined.json" + src.write_text(json.dumps({"phases": [{"id": "P1", "name": "Old"}], "schema_version": 1})) + repl = tmp_path / "replacement.json" + repl.write_text(json.dumps({"id": "P1", "name": "New"})) + + result = replace_item(str(src), "P1", str(repl)) + assert result["replaced_id"] == "P1" + data = json.loads(src.read_text()) + assert data["phases"][0]["name"] == "New" + assert data["schema_version"] == 1 + spy.assert_not_called() + + +# --- expand functions propagate task context --- + + +def test_expand_assignments_includes_task_file_path_in_context(tmp_path: Path) -> None: + from autoskillit.planner.manifests import expand_assignments + + task_file = tmp_path / "task_input.md" + task_file.write_text("Split research.yaml into 4 sub-recipes") + refined = { + "task": "Split research.yaml into 4 sub-recipes", + "phases": [ + { + "id": "P1", + "name": "Recipe Split", + "assignments_preview": [{"name": "Sub-recipe A"}, {"name": "Sub-recipe B"}], + } + ], + } + plan_path = tmp_path / "refined_plan.json" + plan_path.write_text(json.dumps(refined)) + result = expand_assignments( + refined_plan_path=str(plan_path), + output_dir=str(tmp_path), + task_file_path=str(task_file), + ) + context_paths = [p for p in result["context_paths"].split(",") if p.strip()] + assert context_paths, "Must produce at least one context path" + for cp in context_paths: + context = json.loads(Path(cp).read_text()) + assert "task_file_path" in context, "Context file must include task_file_path field" + assert context["task_file_path"] == str(task_file) + assert "task" not in context, "Context file must not include inline task text" + + +def test_expand_assignments_without_task_file_path_uses_empty(tmp_path: Path) -> None: + from autoskillit.planner.manifests import expand_assignments + + refined = { + "task": "Some task", + "phases": [{"id": "P1", "name": "Phase", "assignments_preview": [{"name": "A"}]}], + } + plan_path = tmp_path / "refined_plan.json" + plan_path.write_text(json.dumps(refined)) + result = expand_assignments(refined_plan_path=str(plan_path), output_dir=str(tmp_path)) + context_paths = [p for p in result["context_paths"].split(",") if p.strip()] + assert context_paths, "Must produce at least one context path" + for cp in context_paths: + context = json.loads(Path(cp).read_text()) + assert context.get("task_file_path", "") == "" + assert "task" not in context, "Context file must not include inline task text" + + +def test_expand_wps_includes_task_file_path_in_context(tmp_path: Path) -> None: + from autoskillit.planner.manifests import expand_wps + + task_file = tmp_path / "task_input.md" + task_file.write_text("Add telemetry to fleet dispatch") + refined = { + "schema_version": 2, + "task": "Add telemetry to fleet dispatch", + "source_dir": "/fake", + "assignments": [ + { + "id": "P1-A1", + "phase_id": "P1", + "phase_number": 1, + "assignment_number": 1, + "name": "Telemetry", + "goal": "Add metrics", + "scope": ["fleet"], + "deliverables": ["fleet/_api.py"], + "proposed_work_packages": [ + { + "id": "P1-A1-WP1", + "name": "WP: Instrument dispatch", + "scope": "fleet/_api.py", + "estimated_files": ["fleet/_api.py"], + } + ], + } + ], + } + plan_path = tmp_path / "refined_assignments.json" + plan_path.write_text(json.dumps(refined)) + result = expand_wps( + refined_assignments_path=str(plan_path), + output_dir=str(tmp_path), + task_file_path=str(task_file), + ) + context_paths = [p for p in result["context_paths"].split(",") if p.strip()] + assert context_paths, "Must produce at least one context path" + for cp in context_paths: + context = json.loads(Path(cp).read_text()) + assert "task_file_path" in context, "Context file must include task_file_path field" + assert context["task_file_path"] == str(task_file) + assert "task" not in context, "Context file must not include inline task text" diff --git a/tests/planner/test_refine_assignments_contract.py b/tests/planner/test_refine_assignments_contract.py new file mode 100644 index 0000000000..dc48512bd6 --- /dev/null +++ b/tests/planner/test_refine_assignments_contract.py @@ -0,0 +1,172 @@ +""" +Contract conformance tests for planner-refine-assignments skill registration. +""" + +from pathlib import Path + +import pytest +import yaml + +pytestmark = [ + pytest.mark.layer("planner"), + pytest.mark.small, + pytest.mark.feature("planner"), +] + +SKILL_NAME = "planner-refine-assignments" +CONTRACTS_PATH = ( + Path(__file__).parent.parent.parent / "src/autoskillit/recipe/skill_contracts.yaml" +) +SKILL_MD_PATH = ( + Path(__file__).parent.parent.parent / f"src/autoskillit/skills_extended/{SKILL_NAME}/SKILL.md" +) + + +@pytest.fixture(scope="module") +def contracts() -> dict: + return yaml.safe_load(CONTRACTS_PATH.read_text()) + + +@pytest.fixture(scope="module") +def skill_contract(contracts: dict) -> dict: + skills = contracts.get("skills", {}) + assert SKILL_NAME in skills, ( + f"{SKILL_NAME!r} not found in skill_contracts.yaml. " + f"Known planner skills: {[k for k in skills if 'planner' in k]}" + ) + return skills[SKILL_NAME] + + +@pytest.fixture(scope="module") +def skill_md() -> str: + assert SKILL_MD_PATH.exists(), ( + f"SKILL.md not found at {SKILL_MD_PATH}. " + "Create src/autoskillit/skills_extended/planner-refine-assignments/SKILL.md" + ) + return SKILL_MD_PATH.read_text() + + +class TestContractRegistration: + def test_skill_entry_exists(self, skill_contract: dict) -> None: + """skill_contracts.yaml must have an entry for planner-refine-assignments.""" + assert isinstance(skill_contract, dict), ( + f"{SKILL_NAME!r} contract entry must be a dict, got {type(skill_contract)}" + ) + + def test_write_behavior_always(self, skill_contract: dict) -> None: + """write_behavior must be 'always' — prevents silent no-write success.""" + assert skill_contract.get("write_behavior") == "always", ( + "write_behavior must be 'always' for planner-refine-assignments. " + "Without it, a session that never writes is marked successful." + ) + + def test_phase_refined_path_output_present(self, skill_contract: dict) -> None: + """outputs must declare phase_refined_path.""" + outputs = skill_contract.get("outputs", []) + names = [o.get("name") for o in outputs] + assert "phase_refined_path" in names, f"phase_refined_path not in outputs. Found: {names}" + + def test_phase_refined_path_type_is_file_path(self, skill_contract: dict) -> None: + """phase_refined_path output must have type file_path.""" + outputs = skill_contract.get("outputs", []) + token = next((o for o in outputs if o.get("name") == "phase_refined_path"), None) + assert token is not None + assert token.get("type") == "file_path", ( + f"phase_refined_path type must be 'file_path', got {token.get('type')!r}. " + "Path-contamination detection requires file_path type." + ) + + def test_expected_output_pattern_present(self, skill_contract: dict) -> None: + """expected_output_patterns must include a pattern matching phase_refined_path.""" + patterns = skill_contract.get("expected_output_patterns", []) + assert any("phase_refined_path" in p for p in patterns), ( + f"No expected_output_pattern referencing phase_refined_path. Found: {patterns}" + ) + + def test_three_inputs_declared(self, skill_contract: dict) -> None: + """Three inputs: context_file, refined_plan_path, output_dir.""" + inputs = skill_contract.get("inputs", []) + assert len(inputs) == 3, ( + f"Expected exactly 3 inputs, got {len(inputs)}: {[i.get('name') for i in inputs]}" + ) + input_names = {i.get("name") for i in inputs} + assert input_names == {"context_file", "refined_plan_path", "output_dir"}, ( + f"Unexpected input names: {input_names}" + ) + + +class TestSkillMdPresence: + def test_skill_md_exists(self, skill_md: str) -> None: + """SKILL.md must exist under skills_extended/planner-refine-assignments/.""" + assert "## Workflow" in skill_md, ( + "SKILL.md must contain a '## Workflow' section — " + "file exists but appears empty or structurally incomplete." + ) + + def test_skill_md_has_categories_planner(self, skill_md: str) -> None: + """Frontmatter must declare categories: [planner].""" + assert "categories: [planner]" in skill_md, ( + "SKILL.md must include 'categories: [planner]' in frontmatter. " + "Required for feature-gate and skill-category routing." + ) + + def test_skill_md_has_never_always_block(self, skill_md: str) -> None: + """SKILL.md must contain a NEVER/ALWAYS constraint block.""" + assert "NEVER" in skill_md and "ALWAYS" in skill_md, ( + "SKILL.md must include a NEVER/ALWAYS constraint block following " + "the pattern of all other planner skills." + ) + + def test_skill_md_has_write_path_restriction(self, skill_md: str) -> None: + """SKILL.md NEVER block must restrict writes to AUTOSKILLIT_TEMP/planner/.""" + assert "AUTOSKILLIT_TEMP" in skill_md and "planner/" in skill_md, ( + "SKILL.md NEVER block must restrict write paths to {{AUTOSKILLIT_TEMP}}/planner/." + ) + + def test_skill_contains_phase_refined_path(self, skill_md: str) -> None: + """SKILL.md must document the phase_refined_path output token.""" + assert "phase_refined_path" in skill_md, ( + "SKILL.md must document 'phase_refined_path = ' output token." + ) + + def test_skill_contains_peer_summaries(self, skill_md: str) -> None: + """SKILL.md must reference peer_summaries from the context file.""" + assert "peer_summaries" in skill_md, ( + "SKILL.md must reference 'peer_summaries' — the per-phase context file provides " + "id/name/goal stubs for all assignments in other phases." + ) + + def test_skill_md_has_l0_response_fields(self, skill_md: str) -> None: + """SKILL.md must describe L0 response fields for assignments.""" + assert "assignment_id" in skill_md, ( + "SKILL.md must document L0 structured response field: assignment_id." + ) + assert "changes" in skill_md, ( + "SKILL.md must document L0 structured response field: changes." + ) + assert "dependency_corrections" in skill_md, ( + "SKILL.md must document L0 structured response field: dependency_corrections." + ) + assert "wp_proposal_adjustments" in skill_md, ( + "SKILL.md must document L0 structured response field: wp_proposal_adjustments." + ) + + def test_skill_md_has_batch_limit_spec(self, skill_md: str) -> None: + """SKILL.md must specify the L0 batch ceiling of 6.""" + assert "Spawn more than 6" in skill_md, ( + "SKILL.md must specify the maximum parallel L0 batch size of 6 " + "(expected literal phrase 'Spawn more than 6')." + ) + + def test_skill_md_has_wp_conflict_policy(self, skill_md: str) -> None: + """SKILL.md must document the WP conflict resolution policy (earlier assignment wins).""" + assert "earlier" in skill_md.lower(), ( + "SKILL.md must document that the earlier assignment_id wins WP ownership conflicts." + ) + + def test_skill_md_has_partial_failure_handling(self, skill_md: str) -> None: + """SKILL.md must describe partial-failure behavior (N-1 assignments on L0 failure).""" + assert "CRITICAL" in skill_md or "partial" in skill_md.lower(), ( + "SKILL.md must describe partial-failure handling — " + "proceed with N-1 suggestions and log CRITICAL if an L0 fails." + ) diff --git a/tests/planner/test_refine_phases_contract.py b/tests/planner/test_refine_phases_contract.py new file mode 100644 index 0000000000..95ec94a89a --- /dev/null +++ b/tests/planner/test_refine_phases_contract.py @@ -0,0 +1,132 @@ +""" +Contract conformance tests for planner-refine-phases skill registration. +""" + +from pathlib import Path + +import pytest +import yaml + +pytestmark = [ + pytest.mark.layer("planner"), + pytest.mark.small, + pytest.mark.feature("planner"), +] + +SKILL_NAME = "planner-refine-phases" +CONTRACTS_PATH = ( + Path(__file__).parent.parent.parent / "src/autoskillit/recipe/skill_contracts.yaml" +) +SKILL_MD_PATH = ( + Path(__file__).parent.parent.parent / f"src/autoskillit/skills_extended/{SKILL_NAME}/SKILL.md" +) + + +@pytest.fixture(scope="module") +def contracts() -> dict: + return yaml.safe_load(CONTRACTS_PATH.read_text()) + + +@pytest.fixture(scope="module") +def skill_contract(contracts: dict) -> dict: + skills = contracts.get("skills", {}) + assert SKILL_NAME in skills, ( + f"{SKILL_NAME!r} not found in skill_contracts.yaml. " + f"Known planner skills: {[k for k in skills if 'planner' in k]}" + ) + return skills[SKILL_NAME] + + +@pytest.fixture(scope="module") +def skill_md() -> str: + assert SKILL_MD_PATH.exists(), ( + f"SKILL.md not found at {SKILL_MD_PATH}. " + "Create src/autoskillit/skills_extended/planner-refine-phases/SKILL.md" + ) + return SKILL_MD_PATH.read_text() + + +class TestContractRegistration: + def test_skill_entry_exists(self, skill_contract: dict) -> None: + """skill_contracts.yaml must have an entry for planner-refine-phases.""" + assert isinstance(skill_contract, dict), ( + f"{SKILL_NAME!r} contract entry must be a dict, got {type(skill_contract)}" + ) + + def test_write_behavior_always(self, skill_contract: dict) -> None: + """write_behavior must be 'always' — prevents silent no-write success.""" + assert skill_contract.get("write_behavior") == "always", ( + "write_behavior must be 'always' for planner-refine-phases. " + "Without it, a session that never writes is marked successful." + ) + + def test_refined_plan_path_output_present(self, skill_contract: dict) -> None: + """outputs must declare refined_plan_path.""" + outputs = skill_contract.get("outputs", []) + names = [o.get("name") for o in outputs] + assert "refined_plan_path" in names, f"refined_plan_path not in outputs. Found: {names}" + + def test_refined_plan_path_type_is_file_path(self, skill_contract: dict) -> None: + """refined_plan_path output must have type file_path.""" + outputs = skill_contract.get("outputs", []) + token = next((o for o in outputs if o.get("name") == "refined_plan_path"), None) + assert token is not None + assert token.get("type") == "file_path", ( + f"refined_plan_path type must be 'file_path', got {token.get('type')!r}. " + "Path-contamination detection requires file_path type." + ) + + def test_expected_output_pattern_present(self, skill_contract: dict) -> None: + """expected_output_patterns must include a pattern matching refined_plan_path.""" + patterns = skill_contract.get("expected_output_patterns", []) + assert any("refined_plan_path" in p for p in patterns), ( + f"No expected_output_pattern referencing refined_plan_path. Found: {patterns}" + ) + + +class TestSkillMdPresence: + def test_skill_md_exists(self, skill_md: str) -> None: + """SKILL.md must exist under skills_extended/planner-refine-phases/.""" + assert "## Workflow" in skill_md, ( + "SKILL.md must contain a '## Workflow' section — " + "file exists but appears empty or structurally incomplete." + ) + + def test_skill_md_has_categories_planner(self, skill_md: str) -> None: + """Frontmatter must declare categories: [planner].""" + assert "categories: [planner]" in skill_md, ( + "SKILL.md must include 'categories: [planner]' in frontmatter. " + "Required for feature-gate and skill-category routing." + ) + + def test_skill_md_has_never_always_block(self, skill_md: str) -> None: + """SKILL.md must contain a NEVER/ALWAYS constraint block.""" + assert "NEVER" in skill_md and "ALWAYS" in skill_md, ( + "SKILL.md must include a NEVER/ALWAYS constraint block following " + "the pattern of all other planner skills." + ) + + def test_skill_md_has_write_path_restriction(self, skill_md: str) -> None: + """SKILL.md NEVER block must restrict writes to AUTOSKILLIT_TEMP/planner/.""" + assert "AUTOSKILLIT_TEMP" in skill_md and "planner/" in skill_md, ( + "SKILL.md NEVER block must restrict write paths to {{AUTOSKILLIT_TEMP}}/planner/." + ) + + def test_skill_md_has_output_token(self, skill_md: str) -> None: + """SKILL.md must document the refined_plan_path output token.""" + assert "refined_plan_path" in skill_md, ( + "SKILL.md must document 'refined_plan_path = ' output token." + ) + + def test_skill_md_has_l0_validation_instructions(self, skill_md: str) -> None: + """SKILL.md must describe L0 response validation (phase_id, changes, conflicts).""" + assert "phase_id" in skill_md and "changes" in skill_md and "conflicts" in skill_md, ( + "SKILL.md must document L0 structured response fields: phase_id, changes, conflicts." + ) + + def test_skill_md_has_partial_failure_handling(self, skill_md: str) -> None: + """SKILL.md must describe partial-failure behavior (N-1 phases on L0 failure).""" + assert "CRITICAL" in skill_md or "partial" in skill_md.lower(), ( + "SKILL.md must describe partial-failure handling — " + "proceed with N-1 suggestions and log CRITICAL if an L0 fails." + ) diff --git a/tests/planner/test_refine_wps_contract.py b/tests/planner/test_refine_wps_contract.py new file mode 100644 index 0000000000..a8b0c91fa9 --- /dev/null +++ b/tests/planner/test_refine_wps_contract.py @@ -0,0 +1,171 @@ +"""Contract conformance tests for planner-refine-wps skill registration.""" + +from pathlib import Path + +import pytest +import yaml + +pytestmark = [ + pytest.mark.layer("planner"), + pytest.mark.small, + pytest.mark.feature("planner"), +] + +SKILL_NAME = "planner-refine-wps" +CONTRACTS_PATH = ( + Path(__file__).parent.parent.parent / "src/autoskillit/recipe/skill_contracts.yaml" +) +SKILL_MD_PATH = ( + Path(__file__).parent.parent.parent / f"src/autoskillit/skills_extended/{SKILL_NAME}/SKILL.md" +) + + +@pytest.fixture(scope="module") +def contracts() -> dict: + return yaml.safe_load(CONTRACTS_PATH.read_text()) + + +@pytest.fixture(scope="module") +def skill_contract(contracts: dict) -> dict: + skills = contracts.get("skills", {}) + assert SKILL_NAME in skills, ( + f"{SKILL_NAME!r} not found in skill_contracts.yaml. " + f"Known planner skills: {[k for k in skills if 'planner' in k]}" + ) + return skills[SKILL_NAME] + + +@pytest.fixture(scope="module") +def skill_md() -> str: + assert SKILL_MD_PATH.exists(), ( + f"SKILL.md not found at {SKILL_MD_PATH}. " + "Create src/autoskillit/skills_extended/planner-refine-wps/SKILL.md" + ) + return SKILL_MD_PATH.read_text() + + +class TestContractRegistration: + def test_skill_entry_exists(self, skill_contract: dict) -> None: + """skill_contracts.yaml must have an entry for planner-refine-wps.""" + assert isinstance(skill_contract, dict), ( + f"{SKILL_NAME!r} contract entry must be a dict, got {type(skill_contract)}" + ) + + def test_write_behavior_always(self, skill_contract: dict) -> None: + """write_behavior must be 'always' — prevents silent no-write success.""" + assert skill_contract.get("write_behavior") == "always", ( + "write_behavior must be 'always' for planner-refine-wps. " + "Without it, a session that never writes is marked successful." + ) + + def test_refined_wps_path_output_present(self, skill_contract: dict) -> None: + """outputs must declare refined_wps_path.""" + outputs = skill_contract.get("outputs", []) + names = [o.get("name") for o in outputs] + assert "refined_wps_path" in names, f"refined_wps_path not in outputs. Found: {names}" + + def test_refined_wps_path_type_is_file_path(self, skill_contract: dict) -> None: + """refined_wps_path output must have type file_path.""" + outputs = skill_contract.get("outputs", []) + token = next((o for o in outputs if o.get("name") == "refined_wps_path"), None) + assert token is not None + assert token.get("type") == "file_path", ( + f"refined_wps_path type must be 'file_path', got {token.get('type')!r}. " + "Path-contamination detection requires file_path type." + ) + + def test_expected_output_pattern_present(self, skill_contract: dict) -> None: + """expected_output_patterns must include a pattern matching refined_wps_path.""" + patterns = skill_contract.get("expected_output_patterns", []) + assert any("refined_wps_path" in p for p in patterns), ( + f"No expected_output_pattern referencing refined_wps_path. Found: {patterns}" + ) + + def test_four_inputs_declared(self, skill_contract: dict) -> None: + """Four inputs: combined_wps_path, refined_plan_path, etc.""" + inputs = skill_contract.get("inputs", []) + assert len(inputs) == 4, ( + f"Expected exactly 4 inputs, got {len(inputs)}: {[i.get('name') for i in inputs]}" + ) + input_names = {i.get("name") for i in inputs} + assert input_names == { + "combined_wps_path", + "refined_plan_path", + "refined_assignments_path", + "output_dir", + }, f"Unexpected input names: {input_names}" + + +class TestSkillMdPresence: + def test_skill_md_exists(self, skill_md: str) -> None: + """SKILL.md must exist under skills_extended/planner-refine-wps/.""" + assert "## Workflow" in skill_md, ( + "SKILL.md must contain a '## Workflow' section — " + "file exists but appears empty or structurally incomplete." + ) + + def test_skill_md_has_categories_planner(self, skill_md: str) -> None: + """Frontmatter must declare categories: [planner].""" + assert "categories: [planner]" in skill_md, ( + "SKILL.md must include 'categories: [planner]' in frontmatter. " + "Required for feature-gate and skill-category routing." + ) + + def test_skill_md_has_never_always_block(self, skill_md: str) -> None: + """SKILL.md must contain a NEVER/ALWAYS constraint block.""" + assert "NEVER" in skill_md and "ALWAYS" in skill_md, ( + "SKILL.md must include a NEVER/ALWAYS constraint block following " + "the pattern of all other planner skills." + ) + + def test_skill_md_has_write_path_restriction(self, skill_md: str) -> None: + """SKILL.md NEVER block must restrict writes to AUTOSKILLIT_TEMP/planner/.""" + assert "AUTOSKILLIT_TEMP" in skill_md and "planner/" in skill_md, ( + "SKILL.md NEVER block must restrict write paths to {{AUTOSKILLIT_TEMP}}/planner/." + ) + + def test_skill_md_has_output_token(self, skill_md: str) -> None: + """SKILL.md must document the refined_wps_path output token.""" + assert "refined_wps_path" in skill_md, ( + "SKILL.md must document 'refined_wps_path = ' output token." + ) + + def test_skill_md_has_l0_response_fields(self, skill_md: str) -> None: + """SKILL.md must describe L0 response fields for WP refinement.""" + for field in [ + "phase_id", + "wp_changes", + "cross_phase_deps", + "deliverable_conflicts", + "api_mismatches", + ]: + assert field in skill_md, ( + f"SKILL.md must document L0 structured response field: {field}." + ) + + def test_skill_md_has_batch_limit_spec(self, skill_md: str) -> None: + """SKILL.md must specify the L0 batch ceiling of 6.""" + assert "Spawn more than 6" in skill_md, ( + "SKILL.md must specify the maximum parallel L0 batch size of 6 " + "(expected literal phrase 'Spawn more than 6')." + ) + + def test_skill_md_has_per_phase_dispatch(self, skill_md: str) -> None: + """SKILL.md must document per-phase dispatch (not per-WP).""" + assert "per phase" in skill_md.lower(), ( + "SKILL.md must document per-phase L0 dispatch — " + "distinguishes from per-WP dispatch pattern." + ) + + def test_skill_md_has_conflict_resolution_policy(self, skill_md: str) -> None: + """SKILL.md must document the deliverable conflict resolution policy (earlier WP wins).""" + assert "earlier" in skill_md.lower(), ( + "SKILL.md must document that the earlier WP ID wins deliverable ownership conflicts." + ) + + def test_skill_md_has_partial_failure_handling(self, skill_md: str) -> None: + """SKILL.md must describe partial-failure behavior (N-1 phases on L0 failure).""" + assert "CRITICAL" in skill_md or "partial" in skill_md.lower(), ( + "SKILL.md must describe partial-failure handling — " + "proceed with N-1 suggestions and log CRITICAL if an L0 fails." + ) diff --git a/tests/planner/test_schema_conformance.py b/tests/planner/test_schema_conformance.py new file mode 100644 index 0000000000..ee7bc24813 --- /dev/null +++ b/tests/planner/test_schema_conformance.py @@ -0,0 +1,359 @@ +"""Schema conformance tests: SKILL.md-compliant data flows correctly through the pipeline. + +Tests 1a–1d document that SKILL.md field names are accepted and normalized at the +validation boundary. Test 1f is the end-to-end integration backstop. +""" + +from __future__ import annotations + +import json +from pathlib import Path + +import pytest + +from autoskillit.planner.validation import _load_assignment_results, _load_phase_results +from tests.planner.conftest import write_json, write_task_file + +pytestmark = [pytest.mark.layer("planner"), pytest.mark.small, pytest.mark.feature("planner")] + + +# --------------------------------------------------------------------------- +# 1a: SKILL.md-compliant phase data accepted and normalized by _load_phase_results +# --------------------------------------------------------------------------- + + +def test_skill_compliant_phase_data_accepted_and_normalized(tmp_path: Path) -> None: + phases_dir = tmp_path / "phases" + phases_dir.mkdir() + write_json( + phases_dir / "P1_result.json", + { + "id": "P1", + "name": "Phase One", + "ordering": 1, + "goal": "Build the foundation", + "scope": ["core"], + "relationship_notes": "", + "assignments_preview": ["Schema design", "Implementation"], + }, + ) + + results = _load_phase_results(tmp_path) + + assert "P1" in results + assert results["P1"]["phase_number"] == 1 + assert results["P1"]["name_slug"] == "phase-one" + assert results["P1"]["assignments"] == [ + {"name": "Schema design", "metadata": {}}, + {"name": "Implementation", "metadata": {}}, + ] + + +# --------------------------------------------------------------------------- +# 1b: SKILL.md-compliant assignment data accepted and normalized +# --------------------------------------------------------------------------- + + +def test_skill_compliant_assignment_data_accepted_and_normalized(tmp_path: Path) -> None: + assign_dir = tmp_path / "assignments" + assign_dir.mkdir() + write_json( + assign_dir / "P1-A2_result.json", + { + "id": "P1-A2", + "name": "Second assignment", + "phase_id": "P1", + "goal": "Design schema", + "technical_approach": "Use TypedDict", + "proposed_work_packages": [], + }, + ) + + results = _load_assignment_results(tmp_path) + + assert "P1-A2" in results + assert results["P1-A2"]["phase_number"] == 1 + assert results["P1-A2"]["assignment_number"] == 2 + + +# --------------------------------------------------------------------------- +# 1c: expand_assignments with SKILL.md field names produces correct context files +# --------------------------------------------------------------------------- + + +def test_expand_assignments_with_skill_md_field_names(tmp_path: Path) -> None: + from autoskillit.planner import expand_assignments + + write_json( + tmp_path / "refined_plan.json", + { + "phases": [ + { + "id": "P1", + "name": "Phase One", + "ordering": 1, + "goal": "test", + "scope": [], + "relationship_notes": "", + "assignments_preview": [ + {"id": "P1-A1", "name": "First assignment"}, + {"id": "P1-A2", "name": "Second assignment"}, + ], + } + ] + }, + ) + + result = expand_assignments(str(tmp_path / "refined_plan.json"), str(tmp_path)) + + item_ids = result["item_ids"].split(",") + assert item_ids == ["P1"] + context_paths = result["context_paths"].split(",") + assert len(context_paths) == 1 + ctx = json.loads(Path(context_paths[0]).read_text()) + assert ctx["id"] == "P1" + assert ctx["metadata"]["assignment_count"] == 2 + + +# --------------------------------------------------------------------------- +# 1d: compile_plan derives name_slug from name when not explicitly provided +# --------------------------------------------------------------------------- + + +def test_compile_plan_derives_name_slug_from_name(tmp_path: Path) -> None: + from autoskillit.planner.compiler import compile_plan + + write_json( + tmp_path / "phases" / "P1_result.json", + { + "id": "P1", + "name": "Phase One", + "ordering": 1, + "goal": "test", + "scope": [], + "relationship_notes": "", + "assignments_preview": [], + }, + ) + write_json( + tmp_path / "assignments" / "P1-A1_result.json", + { + "id": "P1-A1", + "name": "Assignment 1", + "phase_id": "P1", + "goal": "test", + "technical_approach": "test", + "proposed_work_packages": [], + }, + ) + write_json( + tmp_path / "work_packages" / "P1-A1-WP1_result.json", + { + "id": "P1-A1-WP1", + "name": "WP 1", + "goal": "test", + "deliverables": ["src/mod.py"], + "technical_steps": ["step 1"], + "acceptance_criteria": ["criterion 1"], + "depends_on": [], + }, + ) + write_json( + tmp_path / "work_packages" / "wp_manifest.json", + {"pass_name": "work_packages", "items": [{"id": "P1-A1-WP1", "status": "done"}]}, + ) + write_json( + tmp_path / "validation.json", + {"verdict": "pass", "findings": [], "warnings": [], "schema_version": 2}, + ) + + compile_plan(str(tmp_path), write_task_file(tmp_path), "/src") + + issue = (tmp_path / "issues" / "P1-A1-WP1_issue.md").read_text() + assert "phase-one" in issue + + +# --------------------------------------------------------------------------- +# 1f: Full pipeline integration test with SKILL.md-compliant data +# (this is the architectural backstop — any future schema drift breaks this) +# --------------------------------------------------------------------------- + + +def test_skill_output_through_full_pipeline(tmp_path: Path) -> None: + from autoskillit.planner import ( + compile_plan, + expand_assignments, + expand_wps, + finalize_wp_manifest, + validate_plan, + ) + + write_json( + tmp_path / "phases" / "P1_result.json", + { + "id": "P1", + "name": "Foundation", + "ordering": 1, + "goal": "Build foundation", + "scope": ["core"], + "relationship_notes": "", + "assignments_preview": ["Schema design", "Implementation"], + }, + ) + + write_json( + tmp_path / "refined_plan.json", + { + "phases": [ + { + "id": "P1", + "name": "Foundation", + "ordering": 1, + "assignments_preview": [ + {"id": "P1-A1", "name": "Schema design"}, + {"id": "P1-A2", "name": "Implementation"}, + ], + } + ] + }, + ) + + expand_assignments(str(tmp_path / "refined_plan.json"), str(tmp_path)) + + write_json( + tmp_path / "assignments" / "P1-A1_result.json", + { + "id": "P1-A1", + "name": "Schema design", + "phase_id": "P1", + "goal": "Design the schema", + "technical_approach": "Use TypedDict", + "proposed_work_packages": [ + { + "id_suffix": "WP1", + "name": "Schema module", + "scope": "core", + "estimated_files": ["schema.py"], + } + ], + }, + ) + write_json( + tmp_path / "assignments" / "P1-A2_result.json", + { + "id": "P1-A2", + "name": "Implementation", + "phase_id": "P1", + "goal": "Implement the feature", + "technical_approach": "Code it", + "proposed_work_packages": [ + { + "id_suffix": "WP1", + "name": "Implementation module", + "scope": "core", + "estimated_files": ["impl.py"], + } + ], + }, + ) + + write_json( + tmp_path / "refined_assignments.json", + { + "assignments": [ + { + "id": "P1-A1", + "phase_id": "P1", + "phase_name": "Foundation", + "phase_number": 1, + "assignment_number": 1, + "proposed_work_packages": [ + {"id": "P1-A1-WP1", "name": "Schema module", "scope": "core"}, + ], + }, + { + "id": "P1-A2", + "phase_id": "P1", + "phase_name": "Foundation", + "phase_number": 1, + "assignment_number": 2, + "proposed_work_packages": [ + {"id": "P1-A2-WP1", "name": "Implementation module", "scope": "core"}, + ], + }, + ] + }, + ) + + expand_wps(str(tmp_path / "refined_assignments.json"), str(tmp_path)) + + for wp_id in ["P1-A1-WP1", "P1-A2-WP1"]: + write_json( + tmp_path / "work_packages" / f"{wp_id}_result.json", + { + "id": wp_id, + "name": f"WP {wp_id}", + "goal": "test goal", + "deliverables": [f"src/{wp_id}.py"], + "technical_steps": ["step 1"], + "acceptance_criteria": ["criterion 1"], + "depends_on": [], + }, + ) + + finalize_wp_manifest(str(tmp_path / "work_packages"), str(tmp_path)) + + validate_result = validate_plan(str(tmp_path)) + assert validate_result["verdict"] == "pass" + + compile_result = compile_plan(str(tmp_path), write_task_file(tmp_path), "/src") + assert Path(compile_result["plan_path"]).exists() + milestones = json.loads((tmp_path / "milestones.json").read_text()) + assert milestones["milestones"][0]["name_slug"] == "foundation" + + +# --------------------------------------------------------------------------- +# Tier filename regex contracts +# --------------------------------------------------------------------------- + + +def test_tier_filename_regexes_match_expected_patterns() -> None: + """Filename regexes accept tier-correct names and reject foreign ones.""" + from autoskillit.planner.schema import ( + ASSIGN_RESULT_FILE_RE, + PHASE_RESULT_FILE_RE, + WP_RESULT_FILE_RE, + ) + + assert PHASE_RESULT_FILE_RE.match("P1_result.json") + assert PHASE_RESULT_FILE_RE.match("P99_result.json") + assert not PHASE_RESULT_FILE_RE.match("P1-A1_result.json") + + assert ASSIGN_RESULT_FILE_RE.match("P1-A1_result.json") + assert ASSIGN_RESULT_FILE_RE.match("P12-A3_result.json") + assert not ASSIGN_RESULT_FILE_RE.match("P1_result.json") + + assert WP_RESULT_FILE_RE.match("P1-A1-WP1_result.json") + assert WP_RESULT_FILE_RE.match("P2-A3-WP14_result.json") + assert not WP_RESULT_FILE_RE.match("P1-A1_result.json") + + for name in ("context_P1.json", "phase_assignment_manifest.json", "wp_manifest.json"): + assert not PHASE_RESULT_FILE_RE.match(name) + assert not ASSIGN_RESULT_FILE_RE.match(name) + assert not WP_RESULT_FILE_RE.match(name) + + +def test_refine_wps_skill_md_no_prefix_derivation_instruction() -> None: + """planner-refine-wps SKILL.md must not instruct derivation from id prefix.""" + skill_md = Path("src/autoskillit/skills_extended/planner-refine-wps/SKILL.md") + content = skill_md.read_text() + assert "extracted from `id` prefix" not in content + assert "extracted from id prefix" not in content + + +def test_consolidate_wps_skill_md_no_ambiguous_phase_id() -> None: + """planner-consolidate-wps SKILL.md must not offer dual-source phase_id.""" + skill_md = Path("src/autoskillit/skills_extended/planner-consolidate-wps/SKILL.md") + content = skill_md.read_text() + assert "or from the `id` prefix" not in content + assert "or from the id prefix" not in content diff --git a/tests/planner/test_sort_utils.py b/tests/planner/test_sort_utils.py new file mode 100644 index 0000000000..9c2f6f09b6 --- /dev/null +++ b/tests/planner/test_sort_utils.py @@ -0,0 +1,23 @@ +import pytest + +from autoskillit.planner._sort_utils import _natural_sort_key + +pytestmark = [pytest.mark.layer("planner"), pytest.mark.small, pytest.mark.feature("planner")] + + +def test_natural_sort_key_numeric_segments(): + items = ["P1-A10", "P1-A2", "P1-A1", "P1-A20"] + result = sorted(items, key=_natural_sort_key) + assert result == ["P1-A1", "P1-A2", "P1-A10", "P1-A20"] + + +def test_natural_sort_key_pure_text(): + assert _natural_sort_key("abc") == ["abc"] + + +def test_natural_sort_key_pure_digits(): + assert _natural_sort_key("42") == ["", 42, ""] + + +def test_natural_sort_key_empty_string(): + assert _natural_sort_key("") == [""] diff --git a/tests/planner/test_typed_dict_conformance.py b/tests/planner/test_typed_dict_conformance.py new file mode 100644 index 0000000000..465a53a324 --- /dev/null +++ b/tests/planner/test_typed_dict_conformance.py @@ -0,0 +1,171 @@ +"""TypedDict conformance: required-key sets, SKILL.md alignment, factory validation.""" + +from __future__ import annotations + +import pytest + +from autoskillit.planner.schema import ( + ASSIGNMENT_REQUIRED_KEYS, + DELIVERABLE_BOUNDS, + PHASE_REQUIRED_KEYS, + WP_REQUIRED_KEYS, + AssignmentResult, + PhaseResult, + PhaseShort, + WPResult, + validate_assignment_result, + validate_phase_result, + validate_wp_result, +) + +pytestmark = [pytest.mark.layer("planner"), pytest.mark.small, pytest.mark.feature("planner")] + + +def test_phase_required_keys_subset_of_typed_dict_fields() -> None: + assert PHASE_REQUIRED_KEYS <= PhaseResult.__required_keys__ + + +def test_assignment_required_keys_subset_of_typed_dict_fields() -> None: + assert ASSIGNMENT_REQUIRED_KEYS <= AssignmentResult.__required_keys__ + + +def test_wp_required_keys_subset_of_typed_dict_fields() -> None: + assert WP_REQUIRED_KEYS <= WPResult.__required_keys__ + + +def test_validate_phase_result_rejects_missing_required_keys() -> None: + with pytest.raises(ValueError, match="missing required keys"): + validate_phase_result({"name": "Only Name"}) + + +def test_validate_assignment_result_rejects_missing_required_keys() -> None: + with pytest.raises(ValueError, match="missing required keys"): + validate_assignment_result({"name": "Only Name"}) + + +def test_validate_wp_result_rejects_missing_required_keys() -> None: + with pytest.raises(ValueError, match="missing required keys"): + validate_wp_result({"name": "Only Name"}) + + +def test_factory_phase_result_contains_all_backend_keys() -> None: + from tests.planner.conftest import make_phase_result + + result = make_phase_result(3, name="Third Phase") + + assert result["phase_number"] == 3 + assert result["name_slug"] == "third-phase" + assert isinstance(result["assignments"], list) + + +def test_factory_assignment_result_contains_all_backend_keys() -> None: + from tests.planner.conftest import make_assignment_result + + result = make_assignment_result(2, 4) + + assert result["phase_number"] == 2 + assert result["assignment_number"] == 4 + assert result["id"] == "P2-A4" + + +def test_factory_wp_result_contains_required_keys() -> None: + from tests.planner.conftest import make_wp_result + + result = make_wp_result("P1-A1-WP2") + + assert result["id"] == "P1-A1-WP2" + assert isinstance(result["deliverables"], list) + assert len(result["deliverables"]) >= 1 + + +@pytest.mark.parametrize( + "assignments_preview,expected_count", + [ + ([], 0), + (["Schema design"], 1), + (["Task A", "Task B", "Task C"], 3), + ], +) +def test_validate_phase_result_derives_assignments_from_preview( + assignments_preview: list[str], expected_count: int +) -> None: + result = validate_phase_result( + { + "id": "P1", + "name": "Phase One", + "ordering": 1, + "assignments_preview": assignments_preview, + } + ) + assert len(result["assignments"]) == expected_count + for item in result["assignments"]: + assert "name" in item + assert "metadata" in item + + +@pytest.mark.parametrize( + "assign_id,expected_pn,expected_an", + [ + ("P1-A1", 1, 1), + ("P3-A7", 3, 7), + ("P10-A2", 10, 2), + ], +) +def test_validate_assignment_result_parses_id( + assign_id: str, expected_pn: int, expected_an: int +) -> None: + result = validate_assignment_result( + { + "id": assign_id, + "name": "Test", + "proposed_work_packages": [], + } + ) + assert result["phase_number"] == expected_pn + assert result["assignment_number"] == expected_an + + +@pytest.mark.parametrize( + "name,expected_slug", + [ + ("Phase One", "phase-one"), + ("Foundation", "foundation"), + ("Schema & Validation", "schema-validation"), + ("Phase 1", "phase-1"), + ], +) +def test_validate_phase_result_slugifies_name(name: str, expected_slug: str) -> None: + result = validate_phase_result({"id": "P1", "name": name, "ordering": 1}) + assert result["name_slug"] == expected_slug + + +def test_phase_short_includes_ordering() -> None: + import typing + + hints = typing.get_type_hints(PhaseShort) + assert "ordering" in hints, "PhaseShort must include ordering per Issue 02 spec" + assert hints["ordering"] is int + + +def test_validate_wp_result_rejects_empty_deliverables() -> None: + with pytest.raises(ValueError, match="has 0 deliverables"): + validate_wp_result({"id": "P1-A1-WP1", "name": "WP", "deliverables": []}) + + +def test_validate_wp_result_accepts_empty_deliverables_with_allow_stub() -> None: + result = validate_wp_result( + {"id": "P1-A1-WP1", "name": "WP", "deliverables": []}, allow_stub=True + ) + assert result["deliverables"] == [] + + +def test_validate_wp_result_rejects_too_many_deliverables() -> None: + _, hi = DELIVERABLE_BOUNDS + with pytest.raises(ValueError, match=f"has {hi + 1} deliverables"): + validate_wp_result( + { + "id": "P1-A1-WP1", + "name": "WP", + "deliverables": [f"f{i}.py" for i in range(hi + 1)], + } + ) diff --git a/tests/planner/test_validation.py b/tests/planner/test_validation.py new file mode 100644 index 0000000000..8cd0a56f2a --- /dev/null +++ b/tests/planner/test_validation.py @@ -0,0 +1,642 @@ +"""Tests for validate_plan callable.""" + +from __future__ import annotations + +import json +from pathlib import Path + +import pytest + +from autoskillit.planner.schema import DELIVERABLE_BOUNDS +from autoskillit.planner.validation import ( + _check_duplicate_deliverables, + _check_sizing_bounds, + validate_plan, +) +from tests.planner.conftest import ( + make_assignment_result, + make_phase_result, + make_wp_result, + write_json, +) + +pytestmark = [pytest.mark.layer("planner"), pytest.mark.small, pytest.mark.feature("planner")] + + +# --------------------------------------------------------------------------- +# Fixture helpers +# --------------------------------------------------------------------------- + + +def _make_minimal_output_dir( + tmp_path: Path, + *, + num_phases: int = 1, + wps_per_assignment: int = 1, + deliverables_override: list[str] | None = None, + depends_on_override: dict[str, list[str]] | None = None, + extra_phases: list[int] | None = None, + extra_assignments: list[tuple[int, int]] | None = None, +) -> Path: + """Build a minimal valid output_dir structure.""" + phases_dir = tmp_path / "phases" + assigns_dir = tmp_path / "assignments" + wps_dir = tmp_path / "work_packages" + + for p in range(1, num_phases + 1): + write_json( + phases_dir / f"P{p}_result.json", + make_phase_result(p, name=f"Phase {p}"), + ) + + for p in range(1, num_phases + 1): + for a in range(1, 2): + write_json( + assigns_dir / f"P{p}-A{a}_result.json", + make_assignment_result( + p, + a, + name=f"Assignment P{p}-A{a}", + proposed_work_packages=[ + f"P{p}-A{a}-WP{w}" for w in range(1, wps_per_assignment + 1) + ], + ), + ) + + for p in range(1, num_phases + 1): + for a in range(1, 2): + for w in range(1, wps_per_assignment + 1): + wp_id = f"P{p}-A{a}-WP{w}" + deliverables = ( + deliverables_override + if deliverables_override is not None + else [f"src/mod_{wp_id}.py"] + ) + deps = (depends_on_override or {}).get(wp_id, []) + write_json( + wps_dir / f"{wp_id}_result.json", + make_wp_result(wp_id, deliverables=deliverables, depends_on=deps), + ) + + manifest_items = [] + for p in range(1, num_phases + 1): + for a in range(1, 2): + for w in range(1, wps_per_assignment + 1): + manifest_items.append({"id": f"P{p}-A{a}-WP{w}", "status": "done"}) + write_json( + wps_dir / "wp_manifest.json", + {"pass_name": "work_packages", "items": manifest_items}, + ) + + if extra_phases: + for p in extra_phases: + write_json( + phases_dir / f"P{p}_result.json", + make_phase_result(p, name=f"Phase {p}"), + ) + + if extra_assignments: + for p, a in extra_assignments: + write_json( + assigns_dir / f"P{p}-A{a}_result.json", + make_assignment_result(p, a, name=f"Orphan assignment P{p}-A{a}"), + ) + + return tmp_path + + +# --------------------------------------------------------------------------- +# Tests +# --------------------------------------------------------------------------- + + +def test_validate_plan_valid_returns_pass(tmp_path: Path) -> None: + _make_minimal_output_dir(tmp_path) + result = validate_plan(str(tmp_path)) + assert result["verdict"] == "pass" + assert result["issue_count"] == "0" + + +def test_validate_plan_cyclic_dep_fails(tmp_path: Path) -> None: + _make_minimal_output_dir(tmp_path, wps_per_assignment=2) + wp1 = "P1-A1-WP1" + wp2 = "P1-A1-WP2" + wp_dir = tmp_path / "work_packages" + data1 = json.loads((wp_dir / f"{wp1}_result.json").read_text()) + data1["depends_on"] = [wp2] + (wp_dir / f"{wp1}_result.json").write_text(json.dumps(data1)) + data2 = json.loads((wp_dir / f"{wp2}_result.json").read_text()) + data2["depends_on"] = [wp1] + (wp_dir / f"{wp2}_result.json").write_text(json.dumps(data2)) + + result = validate_plan(str(tmp_path)) + assert result["verdict"] == "fail" + validation = json.loads((tmp_path / "validation.json").read_text()) + assert any(wp1 in f["message"] and wp2 in f["message"] for f in validation["findings"]) + + +def test_validate_plan_missing_dep_ref_fails(tmp_path: Path) -> None: + _make_minimal_output_dir(tmp_path, depends_on_override={"P1-A1-WP1": ["P1-A1-WP99"]}) + result = validate_plan(str(tmp_path)) + assert result["verdict"] == "fail" + + +def test_validate_plan_phase_no_assignments_fails(tmp_path: Path) -> None: + _make_minimal_output_dir(tmp_path, extra_phases=[2]) + result = validate_plan(str(tmp_path)) + assert result["verdict"] == "fail" + + +def test_validate_plan_assignment_no_wps_fails(tmp_path: Path) -> None: + _make_minimal_output_dir(tmp_path, extra_assignments=[(1, 2)]) + result = validate_plan(str(tmp_path)) + assert result["verdict"] == "fail" + + +def test_validate_plan_wp_zero_deliverables_fails(tmp_path: Path) -> None: + _make_minimal_output_dir(tmp_path) + wp_path = tmp_path / "work_packages" / "P1-A1-WP1_result.json" + raw = json.loads(wp_path.read_text()) + raw["deliverables"] = [] + wp_path.write_text(json.dumps(raw)) + result = validate_plan(str(tmp_path)) + assert result["verdict"] == "fail" + + +def test_validate_plan_wp_too_many_deliverables_fails(tmp_path: Path) -> None: + _make_minimal_output_dir(tmp_path) + wp_path = tmp_path / "work_packages" / "P1-A1-WP1_result.json" + raw = json.loads(wp_path.read_text()) + raw["deliverables"] = ["a", "b", "c", "d", "e", "f"] + wp_path.write_text(json.dumps(raw)) + result = validate_plan(str(tmp_path)) + assert result["verdict"] == "fail" + + +def test_validate_plan_duplicate_deliverables_fails(tmp_path: Path) -> None: + _make_minimal_output_dir(tmp_path, wps_per_assignment=2, deliverables_override=["src/foo.py"]) + result = validate_plan(str(tmp_path)) + assert result["verdict"] == "fail" + + +def test_validate_plan_failed_wp_flagged_but_not_sole_fail_cause(tmp_path: Path) -> None: + _make_minimal_output_dir(tmp_path) + manifest_path = tmp_path / "work_packages" / "wp_manifest.json" + manifest = json.loads(manifest_path.read_text()) + manifest["items"][0]["status"] = "failed" + manifest_path.write_text(json.dumps(manifest)) + + result = validate_plan(str(tmp_path)) + assert result["verdict"] == "fail" + validation = json.loads((tmp_path / "validation.json").read_text()) + assert any("P1-A1-WP1" in f["message"] for f in validation["findings"]) + + +def test_validate_plan_dep_graph_backward_dep_injection(tmp_path: Path) -> None: + _make_minimal_output_dir(tmp_path, wps_per_assignment=2) + write_json( + tmp_path / "dep_graph.json", + {"added_backward_deps": {"P1-A1-WP2": ["P1-A1-WP1"]}, "forward_deps": {}}, + ) + result = validate_plan(str(tmp_path)) + assert result["verdict"] == "pass" + + +def test_validate_plan_dep_graph_creates_cycle_fails(tmp_path: Path) -> None: + _make_minimal_output_dir( + tmp_path, wps_per_assignment=2, depends_on_override={"P1-A1-WP1": ["P1-A1-WP2"]} + ) + write_json( + tmp_path / "dep_graph.json", + {"added_backward_deps": {"P1-A1-WP2": ["P1-A1-WP1"]}, "forward_deps": {}}, + ) + result = validate_plan(str(tmp_path)) + assert result["verdict"] == "fail" + + +def test_validate_plan_writes_validation_json(tmp_path: Path) -> None: + _make_minimal_output_dir(tmp_path) + result = validate_plan(str(tmp_path)) + validation_path = tmp_path / "validation.json" + assert validation_path.exists() + data = json.loads(validation_path.read_text()) + assert "verdict" in data + assert result["validation_path"] == str(validation_path) + + +def test_validate_plan_return_values_are_strings(tmp_path: Path) -> None: + _make_minimal_output_dir(tmp_path) + result = validate_plan(str(tmp_path)) + assert all(isinstance(v, str) for v in result.values()) + + +# --------------------------------------------------------------------------- +# _check_duplicate_files_touched tests (T12–T14) +# --------------------------------------------------------------------------- + + +def test_check_duplicate_files_touched_detects_overlap() -> None: + """T12: Two WPs touching the same file produce a finding.""" + from autoskillit.planner.validation import _check_duplicate_files_touched + + wp_results = { + "P1-A1-WP1": {"files_touched": ["src/foo.py", "src/bar.py"]}, + "P2-A1-WP1": {"files_touched": ["src/foo.py", "src/baz.py"]}, + } + findings = _check_duplicate_files_touched(wp_results) + assert len(findings) == 1 + assert "src/foo.py" in findings[0]["message"] + assert "P1-A1-WP1" in findings[0]["message"] + assert "P2-A1-WP1" in findings[0]["message"] + + +def test_check_duplicate_files_touched_no_false_positives() -> None: + """T13: WPs with disjoint files_touched produce no findings.""" + from autoskillit.planner.validation import _check_duplicate_files_touched + + wp_results = { + "P1-A1-WP1": {"files_touched": ["src/foo.py"]}, + "P1-A1-WP2": {"files_touched": ["src/bar.py"]}, + } + findings = _check_duplicate_files_touched(wp_results) + assert findings == [] + + +def test_validate_plan_includes_duplicate_files_touched(tmp_path: Path) -> None: + """T14: validate_plan detects files_touched overlap as warning, not error.""" + _make_minimal_output_dir(tmp_path, wps_per_assignment=2) + wp_dir = tmp_path / "work_packages" + for wp_id in ("P1-A1-WP1", "P1-A1-WP2"): + result_path = wp_dir / f"{wp_id}_result.json" + data = json.loads(result_path.read_text()) + data["files_touched"] = ["src/shared.py"] + result_path.write_text(json.dumps(data)) + + result = validate_plan(str(tmp_path)) + assert result["verdict"] == "pass" + validation = json.loads((tmp_path / "validation.json").read_text()) + assert any("src/shared.py" in w["message"] for w in validation["warnings"]) + + +# --------------------------------------------------------------------------- +# Severity-level tests (T15–T19) +# --------------------------------------------------------------------------- + + +def test_warning_severity_does_not_fail_verdict(tmp_path: Path) -> None: + """T15: files_touched overlap is a warning, not an error — verdict stays pass.""" + _make_minimal_output_dir(tmp_path, wps_per_assignment=2) + wp_dir = tmp_path / "work_packages" + for wp_id in ("P1-A1-WP1", "P1-A1-WP2"): + result_path = wp_dir / f"{wp_id}_result.json" + data = json.loads(result_path.read_text()) + data["files_touched"] = ["src/shared.py"] + result_path.write_text(json.dumps(data)) + + result = validate_plan(str(tmp_path)) + assert result["verdict"] == "pass" + assert result["issue_count"] == "0" + validation = json.loads((tmp_path / "validation.json").read_text()) + assert len(validation["findings"]) == 0 + assert len(validation["warnings"]) == 1 + assert validation["warnings"][0]["severity"] == "warning" + assert validation["warnings"][0]["check"] == "duplicate_files_touched" + assert "src/shared.py" in validation["warnings"][0]["message"] + + +def test_error_findings_have_structured_fields(tmp_path: Path) -> None: + """T16: Error findings contain message, severity, and check fields.""" + _make_minimal_output_dir(tmp_path, wps_per_assignment=2) + wp1 = "P1-A1-WP1" + wp2 = "P1-A1-WP2" + wp_dir = tmp_path / "work_packages" + data1 = json.loads((wp_dir / f"{wp1}_result.json").read_text()) + data1["depends_on"] = [wp2] + (wp_dir / f"{wp1}_result.json").write_text(json.dumps(data1)) + data2 = json.loads((wp_dir / f"{wp2}_result.json").read_text()) + data2["depends_on"] = [wp1] + (wp_dir / f"{wp2}_result.json").write_text(json.dumps(data2)) + + validate_plan(str(tmp_path)) + validation = json.loads((tmp_path / "validation.json").read_text()) + for finding in validation["findings"]: + assert "message" in finding + assert "severity" in finding + assert "check" in finding + assert finding["severity"] == "error" + cycle_findings = [f for f in validation["findings"] if f["check"] == "dag_acyclic"] + assert len(cycle_findings) == 1 + + +def test_mixed_errors_and_warnings(tmp_path: Path) -> None: + """T17: Sizing violation (error) + files_touched overlap (warning) coexist.""" + _make_minimal_output_dir(tmp_path, wps_per_assignment=2) + wp_dir = tmp_path / "work_packages" + for wp_id in ("P1-A1-WP1", "P1-A1-WP2"): + result_path = wp_dir / f"{wp_id}_result.json" + data = json.loads(result_path.read_text()) + data["deliverables"] = [] + data["files_touched"] = ["src/shared.py"] + result_path.write_text(json.dumps(data)) + + result = validate_plan(str(tmp_path)) + assert result["verdict"] == "fail" + validation = json.loads((tmp_path / "validation.json").read_text()) + error_findings = [f for f in validation["findings"] if f["severity"] == "error"] + assert len(error_findings) >= 1 + assert len(validation["warnings"]) == 1 + assert validation["warnings"][0]["severity"] == "warning" + + +def test_validation_json_schema_version_2(tmp_path: Path) -> None: + """T18: validation.json uses schema_version 2 and includes warnings key.""" + _make_minimal_output_dir(tmp_path) + validate_plan(str(tmp_path)) + validation = json.loads((tmp_path / "validation.json").read_text()) + assert validation["schema_version"] == 2 + assert "warnings" in validation + + +def test_check_duplicate_files_touched_returns_structured_findings() -> None: + """T19: _check_duplicate_files_touched returns dicts with message/severity/check.""" + from autoskillit.planner.validation import _check_duplicate_files_touched + + wp_results = { + "P1-A1-WP1": {"files_touched": ["src/foo.py"]}, + "P2-A1-WP1": {"files_touched": ["src/foo.py"]}, + } + findings = _check_duplicate_files_touched(wp_results) + assert len(findings) == 1 + assert findings[0]["severity"] == "warning" + assert findings[0]["check"] == "duplicate_files_touched" + assert "message" in findings[0] + + +# --------------------------------------------------------------------------- +# _check_version_bump_steps tests (T-VB-1 through T-VB-8) +# --------------------------------------------------------------------------- + + +def test_version_bump_step_pyproject(tmp_path: Path) -> None: + """T-VB-1: WP with technical_steps referencing pyproject.toml version edit → warning.""" + _make_minimal_output_dir(tmp_path) + wp_dir = tmp_path / "work_packages" + data = json.loads((wp_dir / "P1-A1-WP1_result.json").read_text()) + data["technical_steps"] = ["Edit pyproject.toml version field to X.Y.Z"] + (wp_dir / "P1-A1-WP1_result.json").write_text(json.dumps(data)) + + result = validate_plan(str(tmp_path)) + assert result["verdict"] == "pass" + validation = json.loads((tmp_path / "validation.json").read_text()) + assert validation["findings"] == [] + assert any(w["check"] == "version_bump_step" for w in validation["warnings"]) + + +def test_version_bump_step_sync_versions_task(tmp_path: Path) -> None: + """T-VB-2: WP with technical_steps containing sync-versions → warning.""" + _make_minimal_output_dir(tmp_path) + wp_dir = tmp_path / "work_packages" + data = json.loads((wp_dir / "P1-A1-WP1_result.json").read_text()) + data["technical_steps"] = ["Run task sync-versions"] + (wp_dir / "P1-A1-WP1_result.json").write_text(json.dumps(data)) + + result = validate_plan(str(tmp_path)) + assert result["verdict"] == "pass" + validation = json.loads((tmp_path / "validation.json").read_text()) + assert any(w["check"] == "version_bump_step" for w in validation["warnings"]) + + +def test_version_bump_step_sync_versions_py(tmp_path: Path) -> None: + """T-VB-3: WP with technical_steps containing sync_versions.py → warning.""" + _make_minimal_output_dir(tmp_path) + wp_dir = tmp_path / "work_packages" + data = json.loads((wp_dir / "P1-A1-WP1_result.json").read_text()) + data["technical_steps"] = ["python3 scripts/sync_versions.py"] + (wp_dir / "P1-A1-WP1_result.json").write_text(json.dumps(data)) + + result = validate_plan(str(tmp_path)) + assert result["verdict"] == "pass" + validation = json.loads((tmp_path / "validation.json").read_text()) + assert any(w["check"] == "version_bump_step" for w in validation["warnings"]) + + +def test_version_bump_step_no_match(tmp_path: Path) -> None: + """T-VB-4: WP with unrelated technical_steps → no version_bump_step warning.""" + _make_minimal_output_dir(tmp_path) + wp_dir = tmp_path / "work_packages" + data = json.loads((wp_dir / "P1-A1-WP1_result.json").read_text()) + data["technical_steps"] = ["Refactor the API handler"] + (wp_dir / "P1-A1-WP1_result.json").write_text(json.dumps(data)) + + result = validate_plan(str(tmp_path)) + assert result["verdict"] == "pass" + validation = json.loads((tmp_path / "validation.json").read_text()) + assert not any(w["check"] == "version_bump_step" for w in validation["warnings"]) + + +def test_version_bump_step_in_name(tmp_path: Path) -> None: + """T-VB-5: WP with version-bump pattern in name → warning.""" + _make_minimal_output_dir(tmp_path) + wp_dir = tmp_path / "work_packages" + data = json.loads((wp_dir / "P1-A1-WP1_result.json").read_text()) + data["name"] = "WP3: Version Bump" + data["technical_steps"] = [] + (wp_dir / "P1-A1-WP1_result.json").write_text(json.dumps(data)) + + result = validate_plan(str(tmp_path)) + assert result["verdict"] == "pass" + validation = json.loads((tmp_path / "validation.json").read_text()) + assert any(w["check"] == "version_bump_step" for w in validation["warnings"]) + + +def test_version_bump_step_no_technical_steps_key(tmp_path: Path) -> None: + """T-VB-6: WP with no technical_steps at all → no crash, no warning.""" + _make_minimal_output_dir(tmp_path) + wp_dir = tmp_path / "work_packages" + data = json.loads((wp_dir / "P1-A1-WP1_result.json").read_text()) + data.pop("technical_steps", None) + (wp_dir / "P1-A1-WP1_result.json").write_text(json.dumps(data)) + + result = validate_plan(str(tmp_path)) + assert result["verdict"] == "pass" + validation = json.loads((tmp_path / "validation.json").read_text()) + assert not any(w["check"] == "version_bump_step" for w in validation["warnings"]) + + +def test_version_bump_step_only_flagged_wp(tmp_path: Path) -> None: + """T-VB-7: Multiple WPs, only one triggers the warning → one entry per flagged WP.""" + _make_minimal_output_dir(tmp_path, wps_per_assignment=2) + wp_dir = tmp_path / "work_packages" + data1 = json.loads((wp_dir / "P1-A1-WP1_result.json").read_text()) + data1["technical_steps"] = ["Edit pyproject.toml version field to 1.2.3"] + (wp_dir / "P1-A1-WP1_result.json").write_text(json.dumps(data1)) + data2 = json.loads((wp_dir / "P1-A1-WP2_result.json").read_text()) + data2["technical_steps"] = ["Implement the feature"] + (wp_dir / "P1-A1-WP2_result.json").write_text(json.dumps(data2)) + + result = validate_plan(str(tmp_path)) + assert result["verdict"] == "pass" + validation = json.loads((tmp_path / "validation.json").read_text()) + vb_warnings = [w for w in validation["warnings"] if w["check"] == "version_bump_step"] + assert len(vb_warnings) == 1 + assert "P1-A1-WP1" in vb_warnings[0]["message"] + + +def test_version_bump_step_coexists_with_duplicate_files_touched(tmp_path: Path) -> None: + """T-VB-8: version_bump_step warning coexists with duplicate_files_touched warning.""" + _make_minimal_output_dir(tmp_path, wps_per_assignment=2) + wp_dir = tmp_path / "work_packages" + data1 = json.loads((wp_dir / "P1-A1-WP1_result.json").read_text()) + data1["technical_steps"] = ["Edit pyproject.toml version field to 1.2.3"] + data1["files_touched"] = ["src/shared.py"] + (wp_dir / "P1-A1-WP1_result.json").write_text(json.dumps(data1)) + data2 = json.loads((wp_dir / "P1-A1-WP2_result.json").read_text()) + data2["files_touched"] = ["src/shared.py"] + (wp_dir / "P1-A1-WP2_result.json").write_text(json.dumps(data2)) + + result = validate_plan(str(tmp_path)) + assert result["verdict"] == "pass" + validation = json.loads((tmp_path / "validation.json").read_text()) + checks = {w["check"] for w in validation["warnings"]} + assert "version_bump_step" in checks + assert "duplicate_files_touched" in checks + assert len(validation["findings"]) == 0 + + +def test_version_bump_step_via_summary(tmp_path: Path) -> None: + """T-VB-9: WP with version-bump pattern in summary only → warning.""" + _make_minimal_output_dir(tmp_path) + wp_dir = tmp_path / "work_packages" + data = json.loads((wp_dir / "P1-A1-WP1_result.json").read_text()) + data["summary"] = "Perform a version-bump before releasing." + data["technical_steps"] = [] + (wp_dir / "P1-A1-WP1_result.json").write_text(json.dumps(data)) + + result = validate_plan(str(tmp_path)) + assert result["verdict"] == "pass" + validation = json.loads((tmp_path / "validation.json").read_text()) + assert any(w["check"] == "version_bump_step" for w in validation["warnings"]) + + +def test_validate_plan_ignores_phase_sentinel_in_assignments_dir(tmp_path: Path) -> None: + """Phase sentinel files in assignments/ must not crash validate_plan.""" + _make_minimal_output_dir(tmp_path) + sentinel = {"id": "P1", "status": "complete", "assignment_count": 1, "failed_count": 0} + write_json(tmp_path / "assignments" / "P1_result.json", sentinel) + + result = validate_plan(str(tmp_path)) + assert result["verdict"] == "pass" + + +def test_validate_plan_ignores_non_wp_file_in_work_packages_dir(tmp_path: Path) -> None: + """Non-WP files at top level of work_packages/ must not crash validate_plan.""" + _make_minimal_output_dir(tmp_path) + sentinel = {"id": "P1", "status": "complete", "assignment_count": 1, "failed_count": 0} + write_json(tmp_path / "work_packages" / "P1_result.json", sentinel) + + result = validate_plan(str(tmp_path)) + assert result["verdict"] == "pass" + + +# --------------------------------------------------------------------------- +# Direct unit tests for _check_sizing_bounds +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize( + ("count", "expected_findings"), + [ + (0, 1), + (1, 0), + (5, 0), + (6, 1), + ], +) +def test_check_sizing_bounds_boundary_values(count: int, expected_findings: int) -> None: + wp_results = { + "P1-A1-WP1": {"deliverables": [f"f{i}.py" for i in range(count)]}, + } + findings = _check_sizing_bounds(wp_results) + assert len(findings) == expected_findings + if expected_findings: + assert findings[0]["check"] == "sizing_bounds" + assert findings[0]["severity"] == "error" + + +def test_check_sizing_bounds_uses_deliverable_bounds_constant() -> None: + lo, hi = DELIVERABLE_BOUNDS + wp_at_lo = {"P1-A1-WP1": {"deliverables": [f"f{i}.py" for i in range(lo)]}} + wp_at_hi = {"P1-A1-WP2": {"deliverables": [f"f{i}.py" for i in range(hi)]}} + assert _check_sizing_bounds(wp_at_lo) == [] + assert _check_sizing_bounds(wp_at_hi) == [] + + +# --------------------------------------------------------------------------- +# Direct unit tests for _check_duplicate_deliverables +# --------------------------------------------------------------------------- + + +def test_check_duplicate_deliverables_detects_shared() -> None: + wp_results = { + "P1-A1-WP1": {"deliverables": ["src/shared.py", "src/a.py"]}, + "P1-A1-WP2": {"deliverables": ["src/shared.py", "src/b.py"]}, + } + findings = _check_duplicate_deliverables(wp_results) + assert len(findings) == 1 + assert findings[0]["check"] == "duplicate_deliverables" + assert "src/shared.py" in findings[0]["message"] + + +def test_check_duplicate_deliverables_no_false_positives() -> None: + wp_results = { + "P1-A1-WP1": {"deliverables": ["src/a.py"]}, + "P1-A1-WP2": {"deliverables": ["src/b.py"]}, + } + findings = _check_duplicate_deliverables(wp_results) + assert findings == [] + + +def test_check_duplicate_deliverables_returns_structured_findings() -> None: + wp_results = { + "P1-A1-WP1": {"deliverables": ["src/shared.py"]}, + "P1-A1-WP2": {"deliverables": ["src/shared.py"]}, + } + findings = _check_duplicate_deliverables(wp_results) + assert len(findings) == 1 + f = findings[0] + assert "message" in f + assert f["severity"] == "error" + assert f["check"] == "duplicate_deliverables" + + +# --------------------------------------------------------------------------- +# Deduplication-to-orphan cascade +# --------------------------------------------------------------------------- + + +def test_deduplication_orphan_cascade(tmp_path: Path) -> None: + """Simulates dedup removing WP-B's only deliverable → sizing_bounds error.""" + _make_minimal_output_dir(tmp_path, wps_per_assignment=2) + wp_dir = tmp_path / "work_packages" + wp_b = json.loads((wp_dir / "P1-A1-WP2_result.json").read_text()) + wp_b["deliverables"] = [] + (wp_dir / "P1-A1-WP2_result.json").write_text(json.dumps(wp_b)) + + result = validate_plan(str(tmp_path)) + assert result["verdict"] == "fail" + validation = json.loads((tmp_path / "validation.json").read_text()) + sizing_findings = [f for f in validation["findings"] if f["check"] == "sizing_bounds"] + assert len(sizing_findings) == 1 + assert "P1-A1-WP2" in sizing_findings[0]["message"] + + +def test_validate_plan_reads_finalized_manifest(tmp_path: Path) -> None: + """validate_plan must find wp_manifest.json written by finalize_wp_manifest.""" + from autoskillit.planner import finalize_wp_manifest + + _make_minimal_output_dir(tmp_path) + wp_dir = tmp_path / "work_packages" + finalize_wp_manifest(str(wp_dir), str(tmp_path)) + + result = validate_plan(str(tmp_path)) + assert result["verdict"] == "pass" diff --git a/tests/recipe/CLAUDE.md b/tests/recipe/CLAUDE.md new file mode 100644 index 0000000000..fe70feae3b --- /dev/null +++ b/tests/recipe/CLAUDE.md @@ -0,0 +1,165 @@ +# recipe/ + +Recipe I/O, validation, semantic rules, schema, and bundled recipe tests. + +## Files + +| File | Purpose | +|------|---------| +| `__init__.py` | empty | +| `conftest.py` | Shared fixtures for tests/recipe/ | +| `test_analysis_public_api.py` | Tests for the recipe analysis public API surface | +| `test_anti_pattern_guards.py` | Guards for anti-patterns in recipe definitions | +| `test_api.py` | Tests for recipe/_api.py orchestration API | +| `test_api_split.py` | Structural guard for recipe API split | +| `test_audit_trail_artifacts.py` | Tests for audit/ directory creation and artifact copy in create_worktree.sh | +| `test_audit_trail_format_doc.py` | Tests for audit-trail-format.md documentation | +| `test_audit_trail_recipe_contracts.py` | Tests for research.yaml audit-trail captures and threading | +| `test_bem_wrapper_structure.py` | Tests for BEM wrapper recipe structure | +| `test_bundled_model_field.py` | Tests that all run_skill steps declare a model: field across bundled recipes | +| `test_bundled_recipe_hidden_policy.py` | Tests for hidden policy in bundled recipes | +| `test_bundled_recipes_general.py` | General structural tests for all bundled recipes | +| `test_bundled_recipes_no_inversions.py` | Guard: no inversion step order violations in bundled recipes | +| `test_bundled_recipes_pipeline_structure.py` | Pipeline structure tests for bundled recipes | +| `test_bundled_recipes_research.py` | Tests for research bundled recipe structure | +| `test_bundled_recipes_research_design.py` | Tests for research-design bundled recipe structure | +| `test_bundled_recipes_review_pr.py` | Tests for review-pr bundled recipe structure | +| `test_callable_contracts.py` | Contract tests for run_python callable inputs and outputs | +| `test_campaign_loader.py` | Tests for campaign recipe loading | +| `test_check_ci_already_passed_routing.py` | Tests for check_ci_already_passed routing logic | +| `test_check_repo_merge_state_routing.py` | Tests for check_repo_merge_state routing logic | +| `test_cmd_rpc.py` | Tests for recipe/_cmd_rpc.py run_python callables | +| `test_cmd_rpc_null_safety.py` | Null safety tests for _cmd_rpc callables | +| `test_contract_verdict_output_required.py` | Contract: verdict output is required in recipe step | +| `test_contracts.py` | Contract tests for recipe schema and recipe step contracts | +| `test_contracts_block_fingerprint.py` | Tests for block fingerprint contract | +| `test_diagnose_ci_subtype_output.py` | Tests for CI subtype diagnosis output | +| `test_registry_types.py` | Tests for `RuleDef` and `BlockRuleDef` frozen dataclasses | +| `test_diagrams.py` | Tests for recipe flow diagram generation and staleness detection | +| `test_experiment_type_registry.py` | Tests for ExperimentTypeSpec and load_all_experiment_types | +| `test_full_audit_recipe.py` | Tests for the full audit recipe structure | +| `test_hidden_ingredients.py` | Tests for hidden ingredient handling in recipes | +| `test_identity.py` | Tests for recipe identity hashing (content and composite fingerprints) | +| `test_implement_findings_recipe.py` | Tests for implement-findings recipe structure | +| `test_implementation.py` | Tests for implementation recipe structure | +| `test_implementation_groups_pr_decomposition.py` | Tests for implementation groups PR decomposition | +| `test_implementation_groups_review_loop.py` | Tests for implementation groups review loop structure | +| `test_implementation_pr_decomposition.py` | Tests for implementation PR decomposition recipe | +| `test_io_discovery.py` | Tests for recipe I/O discovery (list_recipes, recipe iteration) | +| `test_io_parsing.py` | Tests for recipe YAML parsing and load_recipe | +| `test_io_schema_fields.py` | Tests for recipe schema field validation in I/O layer | +| `test_io_json_precompile.py` | Tests for JSON pre-compilation fast path in recipe I/O | +| `test_issue_url_pipeline.py` | Tests for issue URL pipeline in recipe steps | +| `test_loader.py` | Tests for path-based recipe metadata utilities | +| `test_make_campaign_output_schema.py` | Tests for make-campaign recipe output schema | +| `test_methodology_tradition_registry.py` | Tests for `MethodologyTraditionSpec`, `load_all_methodology_traditions`, and `is_out_of_scope_tradition` | +| `test_methodology_disambiguation.py` | Tests for `DisambiguationRuleDef`, `CrossTraditionOverlapDef`, `DisambiguationResult`, `disambiguate`, and `load_disambiguation_rules` | +| `test_methodology_tradition_router.py` | Tests for `classify_methodology` two-stage Tier-C router: per-tradition classification, multi-match, union rules, determinism | +| `test_methodology_venue_appendix.py` | Tests for Stage B venue appendix resolution: folding map, conditional branching, constraint evaluation | +| `test_merge_prs.py` | Tests for merge-prs recipe structure | +| `test_merge_prs_queue_any.py` | Tests for merge-prs-queue (any strategy) recipe | +| `test_merge_prs_queue_common.py` | Shared queue behavior tests across all queue-capable recipes | +| `test_merge_prs_queue_pmp.py` | Tests for merge-prs-queue (PMP strategy) recipe | +| `test_merge_prs_queue_release_timeout.py` | Release timeout and ci-watch-post-queue-fix retry logic tests | +| `test_merge_sub_recipe_hidden.py` | Tests for hidden sub-recipe merging | +| `test_plan_visualization_step.py` | Tests for plan visualization step in recipes | +| `test_planner_recipe.py` | Tests for the planner recipe structure | +| `test_promote_to_main_wrapper.py` | Tests for the promote-to-main wrapper recipe | +| `test_recipe_ci_watch_event.py` | Tests for CI watch event in recipe steps | +| `test_recipe_order.py` | Tests for BUNDLED_RECIPE_ORDER stable display order registry | +| `test_report_frontmatter_schema.py` | Tests for report.md YAML frontmatter audit-trail schema | +| `test_recipe_scripts.py` | Tests for recipe script callables | +| `test_recipe_temp_substitution.py` | Tests for {{AUTOSKILLIT_TEMP}} substitution in recipe steps | +| `test_remediation_depth_ingredient.py` | Tests for remediation depth ingredient configuration | +| `test_remediation_pr_decomposition.py` | Tests for remediation PR decomposition recipe | +| `test_remediation_recipe.py` | Tests for the remediation recipe structure | +| `test_research_campaign.py` | Contract card assertions for the research-campaign recipe | +| `test_repository.py` | Tests for DefaultRecipeRepository | +| `test_research_archive_recipe.py` | Tests for research-archive sub-recipe structure | +| `test_research_bundle_lifecycle.py` | Tests for research bundle lifecycle in recipes | +| `test_research_context_tracking.py` | Tests for research context tracking in recipes | +| `test_research_implement_recipe.py` | Tests for research-implement sub-recipe structure | +| `test_research_output_mode.py` | Tests for research output mode configuration | +| `test_research_recipe_diag.py` | Tests for research recipe diagnostic output | +| `test_research_campaign_rules.py` | Contract tests for research-campaign.yaml semantic rules | +| `test_research_campaign_structure.py` | Structural assertions for the bundled research-campaign.yaml | +| `test_research_yaml_backward_compat.py` | Regression guard: research.yaml loads cleanly, validates with zero errors, and retains RecipeKind.STANDARD | +| `test_research_review_recipe.py` | Tests for research-review sub-recipe structure | +| `test_research_smoke_fixtures.py` | Smoke-test fixture constants and classification/structural validation for the research recipe | +| `test_research_stage_data_step.py` | Tests for stage-data step in research recipes | +| `test_research_sub_recipes.py` | Tests for research sub-recipe YAML structure (design, implement, review, archive) | +| `test_research_sub_recipe_rules.py` | Contract tests for research sub-recipe semantic rules and dataflow analysis | +| `test_resolve_ci_routing_invariant.py` | Tests for CI routing invariant in resolve steps | +| `test_review_loop_routing_integration.py` | Integration tests for review loop routing | +| `test_review_loop_routing_invariant.py` | Routing invariant tests: check_review_loop is mandatory waypoint between review_pr and check_repo_ci_event | +| `test_rule_decomposition.py` | Tests for semantic rule decomposition structure | +| `test_rules_actions.py` | Tests for actions semantic validation rule | +| `test_rules_blocks.py` | Tests for blocks semantic validation rule | +| `test_rules_bypass.py` | Tests for bypass semantic validation rule | +| `test_rules_callable_inputs.py` | Tests for callable_inputs semantic validation rule | +| `test_rules_callable_scope.py` | Tests for callable-requires-scoped-discovery semantic validation rule | +| `test_rules_campaign.py` | Tests for campaign semantic validation rule | +| `test_rules_ci.py` | Tests for CI semantic validation rule | +| `test_rules_ci_loops.py` | Tests for ci-timed-out-self-loop-unguarded and ci-conflict-path-missing-auto-trigger rules | +| `test_rules_clone.py` | Tests for clone semantic validation rule | +| `test_rules_cmd.py` | Tests for cmd semantic validation rule | +| `test_rules_conditional_push.py` | Tests for conditional_push semantic validation rule | +| `test_rules_contracts.py` | Tests for contracts semantic validation rule | +| `test_rules_dataflow_capture.py` | Tests for dataflow capture semantic validation rule | +| `test_rules_dataflow_handoff.py` | Tests for dataflow handoff semantic validation rule | +| `test_rules_dataflow_merge.py` | Tests for dataflow merge semantic validation rule | +| `test_rules_dataflow_nullable.py` | Tests for dataflow nullable semantic validation rule | +| `test_rules_dedup.py` | Tests for dedup semantic validation rule | +| `test_rules_features.py` | Tests for features semantic validation rule | +| `test_rules_food_truck.py` | Tests for food_truck semantic validation rule | +| `test_rules_graph.py` | Tests for graph semantic validation rule | +| `test_rules_inline_script.py` | Tests for inline_script semantic validation rule | +| `test_rules_inputs.py` | Tests for inputs semantic validation rule | +| `test_rules_integration_predicate.py` | Tests for integration_predicate semantic validation rule | +| `test_rules_isolation.py` | Tests for isolation semantic validation rule | +| `test_rules_merge.py` | Tests for merge semantic validation rule | +| `test_rules_merge_base_unpublished.py` | Tests for merge_base_unpublished semantic validation rule | +| `test_rules_merge_queue_push.py` | Tests for merge_queue_push semantic validation rule | +| `test_rules_merge_routing_incomplete.py` | Tests for merge_routing_incomplete semantic validation rule | +| `test_rules_merge_failure_domain.py` | Tests for merge-failure-skill-domain-mismatch semantic validation rule | +| `test_rules_multipart_iteration.py` | Tests for multipart_iteration semantic validation rule | +| `test_rules_on_context_limit.py` | Tests for on_context_limit semantic validation rule | +| `test_rules_on_result_failure_route.py` | Tests for on_result_failure_route semantic validation rule | +| `test_rules_outdated_script_version.py` | Tests for outdated_script_version semantic validation rule | +| `test_rules_packs.py` | Tests for packs semantic validation rule | +| `test_rules_pipeline_internal.py` | Tests for pipeline_internal semantic validation rule | +| `test_rules_predicate_routing.py` | Tests for predicate_routing semantic validation rule | +| `test_rules_project_local_override.py` | Tests for project_local_override semantic validation rule | +| `test_rules_reachability.py` | Tests for reachability semantic validation rule | +| `test_rules_review_loop_waypoint.py` | Tests for review-loop-waypoint-guard semantic rule | +| `test_rules_recipe.py` | Tests for recipe-level semantic validation rule | +| `test_rules_registry.py` | Tests for rule registry and decorator | +| `test_rules_shadowed_input.py` | Tests for shadowed_input semantic validation rule | +| `test_rules_skill_command_prefix.py` | Tests for skill_command_prefix semantic validation rule | +| `test_rules_skill_content.py` | Tests for skill_content semantic validation rule | +| `test_rules_skills.py` | Tests for skills semantic validation rule | +| `test_rules_skill_routing.py` | Tests for skill-result-routing-gap rule and merge-pr contract allowed_values | +| `test_rules_subset_disabled.py` | Tests for subset_disabled semantic validation rule | +| `test_rules_temp_path.py` | Tests for temp_path semantic validation rule | +| `test_rules_tools.py` | Tests for tools semantic validation rule | +| `test_rules_unreachable_model.py` | Tests for unreachable_model semantic validation rule | +| `test_rules_unsatisfied_input.py` | Tests for unsatisfied_input semantic validation rule | +| `test_rules_verdict.py` | Tests for verdict semantic validation rule | +| `test_rules_weak_constraint.py` | Tests for weak_constraint semantic validation rule | +| `test_rules_worktree.py` | Tests for worktree semantic validation rule | +| `test_schema.py` | Tests for Recipe, RecipeStep, and DataFlowWarning schema | +| `test_skill_emit_consistency.py` | Tests for skill emit consistency in recipe steps | +| `test_silent_type_convention.py` | Tests for silent-type-convention.md documentation | +| `test_silent_type_flows.py` | Integration tests for silent-type convention flows (vis-lens out-of-scope path) | +| `test_staleness_cache.py` | Tests for recipe staleness cache | +| `test_sub_recipe_loading.py` | Tests for sub-recipe loading and composition | +| `test_sub_recipe_schema.py` | Tests for sub-recipe schema structure | +| `test_sub_recipe_validation.py` | Tests for sub-recipe validation | +| `test_validator_dataflow.py` | Tests for recipe validator dataflow analysis | +| `test_validator_graph_and_actions.py` | Tests for recipe validator graph and actions analysis | +| `test_validator_skill_hints.py` | Tests for recipe validator skill hints | +| `test_validator_structural.py` | Tests for recipe validator structural analysis | + +## Architecture Notes + +`conftest.py` provides shared fixtures for recipe tests. The `fixtures/` subdirectory contains YAML test data files including sample recipes and expected diagram output. The `test_rules_*.py` files each test a single semantic validation rule from `recipe/rules/`. diff --git a/tests/recipe/conftest.py b/tests/recipe/conftest.py index 78af6daefb..70c385f6f6 100644 --- a/tests/recipe/conftest.py +++ b/tests/recipe/conftest.py @@ -4,17 +4,42 @@ from pathlib import Path +import pytest import yaml -from autoskillit.recipe.io import _parse_step -from autoskillit.recipe.schema import Recipe +from autoskillit.recipe.io import _parse_step, builtin_recipes_dir, load_recipe +from autoskillit.recipe.schema import Recipe, RecipeStep -# Known violations fixed in Parts B and C — excluded from general semantic-error assertions. +# Excluded from general semantic-error assertions — these rules fire on recipes +# that have not yet been updated. NO_AUTOSKILLIT_IMPORT = "no-autoskillit-import-in-skill-python-block" +KNOWN_PART_B_VIOLATIONS: frozenset[str] = frozenset( + { + NO_AUTOSKILLIT_IMPORT, + "skill-result-routing-gap", + "unbounded-cycle", + } +) + + +@pytest.fixture(scope="module") +def pmp_recipe(): + return load_recipe(builtin_recipes_dir() / "merge-prs.yaml") + + +@pytest.fixture(scope="module") +def impl_recipe(): + return load_recipe(builtin_recipes_dir() / "implementation.yaml") -# --------------------------------------------------------------------------- -# Shared helper: recipe workflow factory -# --------------------------------------------------------------------------- + +@pytest.fixture(scope="module") +def remed_recipe(): + return load_recipe(builtin_recipes_dir() / "remediation.yaml") + + +@pytest.fixture(scope="module") +def impl_groups_recipe(): + return load_recipe(builtin_recipes_dir() / "implementation-groups.yaml") def _make_workflow(steps: dict[str, dict]) -> Recipe: @@ -22,6 +47,25 @@ def _make_workflow(steps: dict[str, dict]) -> Recipe: return Recipe(name="test", description="test", steps=parsed_steps, kitchen_rules=["test"]) +def _build_merge_worktree_recipe(capture: dict) -> Recipe: + """Helper: build a minimal Recipe with a merge_worktree step and the given capture dict.""" + return Recipe( + name="test-merge", + description="Test merge recipe", + summary="merge > done", + steps={ + "merge": RecipeStep( + tool="merge_worktree", + with_args={"worktree_path": "${{ context.worktree_path }}", "base_branch": "main"}, + capture=capture, + on_success="done", + on_failure="done", + ), + "done": RecipeStep(action="stop", message="Done"), + }, + ) + + # --------------------------------------------------------------------------- # Shared fixture data: valid recipe dict and YAML writer # --------------------------------------------------------------------------- @@ -50,3 +94,30 @@ def _make_workflow(steps: dict[str, dict]) -> Recipe: def _write_yaml(path: Path, data: dict) -> Path: path.write_text(yaml.dump(data, default_flow_style=False)) return path + + +# --------------------------------------------------------------------------- +# Audit trail test fixtures +# --------------------------------------------------------------------------- + + +@pytest.fixture +def sample_report_frontmatter() -> dict: + return { + "experiment_type": "causal_inference", + "methodology_traditions": ["controlled_intervention"], + "disambiguation_rule_applied": None, + "tier_c_lens": "vis-lens-methodology-norms", + "design_review_verdict": "GO", + "classification_timestamp": "2026-04-13T15:32:00Z", + "audit_trail_path": { + "design_review": "research/test-slug/audit/design-review-dashboard.md", + "visualization_trace": "research/test-slug/audit/visualization-plan-trace.md", + }, + } + + +@pytest.fixture +def sample_report_text(sample_report_frontmatter: dict) -> str: + fm = yaml.dump(sample_report_frontmatter, default_flow_style=False, sort_keys=False) + return f"---\n{fm}---\n\n# Test Report\n\nBody content." diff --git a/tests/recipe/test__api.py b/tests/recipe/test__api.py deleted file mode 100644 index b08b41791b..0000000000 --- a/tests/recipe/test__api.py +++ /dev/null @@ -1,46 +0,0 @@ -"""Tests for recipe/_api.py internals.""" - -from __future__ import annotations - -# --------------------------------------------------------------------------- -# T5 — _drop_sub_recipe_step uses dataclasses.replace -# --------------------------------------------------------------------------- - - -def test_drop_sub_recipe_step_preserves_future_fields() -> None: - """_drop_sub_recipe_step round-trips all Recipe fields (catches future field additions).""" - - from autoskillit.recipe._api import _drop_sub_recipe_step - from autoskillit.recipe.schema import Recipe, RecipeStep - - recipe = Recipe( - name="test", - description="desc", - summary="sum", - steps={"placeholder": RecipeStep(sub_recipe="sub"), "other": RecipeStep(action="stop")}, - kitchen_rules=["rule1"], - version="0.2.0", - experimental=True, - ) - result = _drop_sub_recipe_step(recipe, "placeholder") - assert result.name == "test" - assert result.summary == "sum" - assert result.experimental is True - assert result.kitchen_rules == ["rule1"] - assert "placeholder" not in result.steps - assert "other" in result.steps - - -# --------------------------------------------------------------------------- -# T2 — _path_mtime_ns replaces the two old helpers -# (migrated from recipe/test_io.py — this test introspects recipe._api) -# --------------------------------------------------------------------------- - - -def test_path_mtime_ns_exists_and_old_helpers_removed() -> None: - """recipe/_api.py must expose _path_mtime_ns; _file_mtime_ns/_dir_mtime_ns removed.""" - import autoskillit.recipe._api as api - - assert hasattr(api, "_path_mtime_ns"), "_path_mtime_ns must exist" - assert not hasattr(api, "_file_mtime_ns"), "_file_mtime_ns must be removed" - assert not hasattr(api, "_dir_mtime_ns"), "_dir_mtime_ns must be removed" diff --git a/tests/recipe/test_analysis_public_api.py b/tests/recipe/test_analysis_public_api.py new file mode 100644 index 0000000000..72535f7638 --- /dev/null +++ b/tests/recipe/test_analysis_public_api.py @@ -0,0 +1,75 @@ +"""Tests for public API surface of recipe._analysis (Finding 6).""" + +from __future__ import annotations + +import pytest + +pytestmark = [pytest.mark.layer("recipe"), pytest.mark.small] + + +# --------------------------------------------------------------------------- +# T4: bfs_reachable is a public symbol (Finding 6) +# --------------------------------------------------------------------------- + + +def test_bfs_reachable_is_public() -> None: + """bfs_reachable must be importable from autoskillit.recipe._analysis.""" + from autoskillit.recipe._analysis import bfs_reachable # must not raise ImportError + + assert callable(bfs_reachable) + + +def test_bfs_reachable_private_name_gone() -> None: + """_bfs_reachable private name must be absent after promotion to public.""" + import autoskillit.recipe._analysis as mod + + assert not hasattr(mod, "_bfs_reachable"), ( + "_bfs_reachable private name must be removed after promotion to bfs_reachable" + ) + + +def test_bfs_reachable_traverses_graph() -> None: + """bfs_reachable returns all reachable nodes from start, excluding start itself.""" + from autoskillit.recipe._analysis import bfs_reachable + + graph = { + "a": {"b", "c"}, + "b": {"d"}, + "c": set(), + "d": set(), + } + assert bfs_reachable(graph, "a") == {"b", "c", "d"} + assert bfs_reachable(graph, "b") == {"d"} + assert bfs_reachable(graph, "d") == set() + + +# --------------------------------------------------------------------------- +# ValidationContext.skill_resolver tests +# --------------------------------------------------------------------------- + + +def test_validation_context_skill_resolver_default() -> None: + """ValidationContext.skill_resolver defaults to None.""" + from autoskillit.recipe._analysis import ValidationContext + from autoskillit.recipe.schema import DataFlowReport, Recipe + + recipe = Recipe(name="t", description="t", steps={}, kitchen_rules=["t"]) + ctx = ValidationContext( + recipe=recipe, + step_graph={}, + dataflow=DataFlowReport(warnings=[], summary=""), + ) + assert ctx.skill_resolver is None + + +def test_make_validation_context_passes_skill_resolver() -> None: + """make_validation_context sets skill_resolver on the returned context.""" + from unittest.mock import Mock + + from autoskillit.recipe._analysis import make_validation_context + from autoskillit.recipe.schema import Recipe + + recipe = Recipe(name="t", description="t", steps={}, kitchen_rules=["t"]) + resolver = Mock() + ctx = make_validation_context(recipe, skill_resolver=resolver) + assert ctx.skill_resolver is resolver diff --git a/tests/recipe/test_anti_pattern_guards.py b/tests/recipe/test_anti_pattern_guards.py index 6dc2fbc7a2..1faac3747d 100644 --- a/tests/recipe/test_anti_pattern_guards.py +++ b/tests/recipe/test_anti_pattern_guards.py @@ -6,7 +6,9 @@ import yaml from autoskillit.recipe.io import builtin_recipes_dir, load_recipe -from autoskillit.recipe.rules_blocks import _block_budgets # re-use the cached loader +from autoskillit.recipe.rules.rules_blocks import _block_budgets # re-use the cached loader + +pytestmark = [pytest.mark.layer("recipe"), pytest.mark.small] def _budget_for(block_name: str) -> dict: # type: ignore[type-arg] @@ -83,20 +85,20 @@ def test_implementation_groups_ci_failure_routes_to_diagnose_ci(): def test_implementation_create_branch_uses_create_unique_branch_tool(): - """AP3: create_branch step must use create_unique_branch MCP tool, not run_cmd.""" + """AP3: create_and_publish step must use create_and_publish_branch MCP tool, not run_cmd.""" recipe = load_recipe(builtin_recipes_dir() / "implementation.yaml") - step = recipe.steps["create_branch"] - assert step.tool == "create_unique_branch", ( - "create_branch must use create_unique_branch tool (not run_cmd bash script)" + step = recipe.steps["create_and_publish"] + assert step.tool == "create_and_publish_branch", ( + "create_and_publish must use create_and_publish_branch tool (not run_cmd bash script)" ) def test_merge_prs_no_run_cmd_push(): - """AP2: merge-prs.yaml push_integration_branch must use push_to_remote, not run_cmd.""" + """AP2: merge-prs.yaml push_batch_branch must use push_to_remote, not run_cmd.""" recipe = load_recipe(builtin_recipes_dir() / "merge-prs.yaml") - step = recipe.steps["push_integration_branch"] + step = recipe.steps["push_batch_branch"] assert step.tool == "push_to_remote", ( - "push_integration_branch must use push_to_remote MCP tool, not run_cmd" + "push_batch_branch must use push_to_remote MCP tool, not run_cmd" ) @@ -106,10 +108,11 @@ def test_merge_prs_has_no_loop_push_kitchen_rule(): steps = raw.get("steps", {}) push_steps = {name for name, step in steps.items() if step.get("tool") == "push_to_remote"} unexpected = push_steps - { - "publish_integration_branch", - "push_integration_branch", + "publish_batch_branch", + "push_batch_branch", "re_push_review_integration", # authorized: re-pushes after review fixes "push_ejected_fix", # authorized: pushes conflict-resolved ejected PR branch back + "push_rebased_next_pr", # authorized: proactive rebase push before enqueue } assert not unexpected, ( f"push_to_remote found in unexpected steps (loop pushes are prohibited): {unexpected}" diff --git a/tests/recipe/test_api.py b/tests/recipe/test_api.py index 38fd37e336..cb007d0fa9 100644 --- a/tests/recipe/test_api.py +++ b/tests/recipe/test_api.py @@ -4,6 +4,10 @@ from pathlib import Path +import pytest + +pytestmark = [pytest.mark.layer("recipe"), pytest.mark.small] + # Minimal recipe YAML with kitchen_rules _RECIPE_WITH_RULES = """\ name: test-recipe-with-rules @@ -111,11 +115,12 @@ def test_load_and_validate_includes_requires_packs(tmp_path): # T4d -def test_load_recipe_result_requires_packs_absent_for_standard_recipe(): - """Standard recipes without requires_packs omit the key (matches kitchen_rules pattern).""" +def test_load_recipe_result_requires_packs_absent_for_standard_recipe(tmp_path): + """Recipes without requires_packs omit the key (matches kitchen_rules pattern).""" from autoskillit.recipe._api import load_and_validate - result = load_and_validate(name="implementation", project_dir=None) + _setup_project_recipe(tmp_path, "test-recipe-no-rules", _RECIPE_NO_RULES) + result = load_and_validate(name="test-recipe-no-rules", project_dir=tmp_path) assert "requires_packs" not in result @@ -153,13 +158,13 @@ def test_load_and_validate_returns_cached_result_on_second_call(tmp_path, monkey recipe_yaml.write_text(MINIMAL_RECIPE_YAML) calls = [] - real_validate = api_mod.validate_recipe + real_validate = api_mod.validate_recipe_structure def counting_validate(recipe): calls.append(1) return real_validate(recipe) - monkeypatch.setattr(api_mod, "validate_recipe", counting_validate) + monkeypatch.setattr(api_mod, "validate_recipe_structure", counting_validate) api_mod.load_and_validate("myrecipe", tmp_path) api_mod.load_and_validate("myrecipe", tmp_path) @@ -179,13 +184,13 @@ def test_load_and_validate_cache_invalidated_on_recipe_mtime_change(tmp_path, mo recipe_yaml.write_text(MINIMAL_RECIPE_YAML) calls = [] - real_validate = api_mod.validate_recipe + real_validate = api_mod.validate_recipe_structure def counting_validate(recipe): calls.append(1) return real_validate(recipe) - monkeypatch.setattr(api_mod, "validate_recipe", counting_validate) + monkeypatch.setattr(api_mod, "validate_recipe_structure", counting_validate) api_mod.load_and_validate("myrecipe", tmp_path) recipe_yaml.touch() @@ -205,13 +210,13 @@ def test_load_and_validate_cache_invalidated_on_pkg_version_change(tmp_path, mon (recipes_dir / "myrecipe.yaml").write_text(MINIMAL_RECIPE_YAML) calls = [] - real_validate = api_mod.validate_recipe + real_validate = api_mod.validate_recipe_structure def counting_validate(recipe): calls.append(1) return real_validate(recipe) - monkeypatch.setattr(api_mod, "validate_recipe", counting_validate) + monkeypatch.setattr(api_mod, "validate_recipe_structure", counting_validate) api_mod.load_and_validate("myrecipe", tmp_path) monkeypatch.setattr(api_mod, "_get_pkg_version", lambda: "99.99.99") @@ -231,13 +236,13 @@ def test_load_and_validate_cache_invalidated_on_dir_mtime_change(tmp_path, monke (recipes_dir / "myrecipe.yaml").write_text(MINIMAL_RECIPE_YAML) calls = [] - real_validate = api_mod.validate_recipe + real_validate = api_mod.validate_recipe_structure def counting_validate(recipe): calls.append(1) return real_validate(recipe) - monkeypatch.setattr(api_mod, "validate_recipe", counting_validate) + monkeypatch.setattr(api_mod, "validate_recipe_structure", counting_validate) api_mod.load_and_validate("myrecipe", tmp_path) (recipes_dir / "newrecipe.yaml").write_text( @@ -277,7 +282,7 @@ def capturing_t(label: str, t0: float, name: str) -> float: assert len(stage_calls) >= 4 assert "find_recipe" in stage_calls assert "yaml_parse" in stage_calls - assert "validate_recipe" in stage_calls + assert "validate_recipe_structure" in stage_calls assert "semantic_rules" in stage_calls @@ -325,17 +330,20 @@ def capturing_fn( *, suppressed=None, recipe_info=None, + recipe_list=None, resolved_defaults=None, ingredient_overrides=None, temp_dir=None, temp_dir_relpath=None, ): captured["recipe_info"] = recipe_info + captured["recipe_list"] = recipe_list return real_fn( name, project_dir, suppressed=suppressed, recipe_info=recipe_info, + recipe_list=recipe_list, resolved_defaults=resolved_defaults, ingredient_overrides=ingredient_overrides, temp_dir=temp_dir, @@ -360,27 +368,27 @@ class TestIngredientSortOrder: """Ingredients must sort: required > auto-detect > flags > constants > optional.""" def test_sort_key_required_is_highest_priority(self): - from autoskillit.recipe._api import _ingredient_sort_key + from autoskillit.recipe._recipe_ingredients import _ingredient_sort_key key = _ingredient_sort_key("task", required=True, default=None) assert key[0] == 0 def test_sort_key_auto_detect_above_flags(self): - from autoskillit.recipe._api import _ingredient_sort_key + from autoskillit.recipe._recipe_ingredients import _ingredient_sort_key auto = _ingredient_sort_key("source_dir", required=False, default="") flag = _ingredient_sort_key("audit", required=False, default="true") assert auto[0] < flag[0], "auto-detect must sort above boolean flags" def test_sort_key_flags_above_optional(self): - from autoskillit.recipe._api import _ingredient_sort_key + from autoskillit.recipe._recipe_ingredients import _ingredient_sort_key flag = _ingredient_sort_key("audit", required=False, default="true") opt = _ingredient_sort_key("issue_url", required=False, default=None) assert flag[0] < opt[0], "boolean flags must sort above optional" def test_sort_key_optional_above_constants(self): - from autoskillit.recipe._api import _ingredient_sort_key + from autoskillit.recipe._recipe_ingredients import _ingredient_sort_key opt = _ingredient_sort_key("issue_url", required=False, default=None) const = _ingredient_sort_key("run_name", required=False, default="impl") @@ -388,7 +396,7 @@ def test_sort_key_optional_above_constants(self): def test_sort_key_full_tier_ordering(self): """All five tiers must be strictly ordered.""" - from autoskillit.recipe._api import _ingredient_sort_key + from autoskillit.recipe._recipe_ingredients import _ingredient_sort_key tiers = [ _ingredient_sort_key("task", required=True, default=None)[0], # required @@ -465,7 +473,10 @@ def test_gfm_description_column_capped_at_max_width(self): A 220-char description (as in implementation.yaml run_mode) must not produce a 220-wide GFM column. Each data row's description cell must be <= the cap. """ - from autoskillit.recipe._api import _GFM_DESC_MAX_WIDTH, format_ingredients_table + from autoskillit.recipe._recipe_ingredients import ( + _GFM_DESC_MAX_WIDTH, + format_ingredients_table, + ) recipe = self._recipe_with_long_desc("X" * 220) table = format_ingredients_table(recipe) @@ -539,7 +550,10 @@ def test_gfm_description_cap_with_real_implementation_recipe(self): This is the regression test for GitHub Issue #489 (run_mode 220-char description). """ from autoskillit.core import pkg_root - from autoskillit.recipe._api import _GFM_DESC_MAX_WIDTH, format_ingredients_table + from autoskillit.recipe._recipe_ingredients import ( + _GFM_DESC_MAX_WIDTH, + format_ingredients_table, + ) from autoskillit.recipe.io import find_recipe_by_name, load_recipe recipe_info = find_recipe_by_name("implementation", pkg_root() / "recipes") @@ -559,6 +573,21 @@ def test_gfm_description_cap_with_real_implementation_recipe(self): ) +def test_orchestration_rules_include_stop_step_semantics(): + """orchestration_rules includes stop-step semantics when recipe has stop steps.""" + from autoskillit.recipe._api import _build_orchestration_rules + from autoskillit.recipe.schema import Recipe, RecipeStep + + recipe = Recipe( + name="test", + description="test", + steps={"done": RecipeStep(action="stop", message="Pipeline complete.")}, + ) + rules = _build_orchestration_rules(recipe) + assert "ACTION: STOP" in rules.upper() or "action: stop" in rules + assert "done" in rules + + def test_build_ingredient_rows_resolved_overrides_literal_default(): """T1: config-resolved value wins over a non-empty YAML literal default.""" from autoskillit.recipe._api import build_ingredient_rows @@ -568,8 +597,8 @@ def test_build_ingredient_rows_resolved_overrides_literal_default(): "base_branch", RecipeIngredient(description="Base branch", default="main"), ) - rows = build_ingredient_rows(recipe, resolved_defaults={"base_branch": "integration"}) - assert ("base_branch", "Base branch", "integration") in rows + rows = build_ingredient_rows(recipe, resolved_defaults={"base_branch": "develop"}) + assert ("base_branch", "Base branch", "develop") in rows def test_build_ingredient_rows_literal_default_preserved_when_no_resolved(): @@ -594,8 +623,8 @@ def test_build_ingredient_rows_resolved_overrides_empty_sentinel(): "base_branch", RecipeIngredient(description="Base branch", default=""), ) - rows = build_ingredient_rows(recipe, resolved_defaults={"base_branch": "integration"}) - assert ("base_branch", "Base branch", "integration") in rows + rows = build_ingredient_rows(recipe, resolved_defaults={"base_branch": "develop"}) + assert ("base_branch", "Base branch", "develop") in rows def test_build_ingredient_rows_empty_sentinel_falls_back_to_auto_detect(): @@ -654,3 +683,157 @@ def test_build_ingredient_rows_returns_tuples(): # Full descriptions must be present (not truncated at this layer) all_descs = [r[1] for r in rows] assert any(len(d) > 60 for d in all_descs), "Expected at least one long description" + + +# --------------------------------------------------------------------------- +# T5 — _drop_sub_recipe_step uses dataclasses.replace +# --------------------------------------------------------------------------- + + +def test_drop_sub_recipe_step_preserves_future_fields() -> None: + """_drop_sub_recipe_step round-trips all Recipe fields (catches future field additions).""" + + from autoskillit.recipe._recipe_composition import _drop_sub_recipe_step + from autoskillit.recipe.schema import Recipe, RecipeStep + + recipe = Recipe( + name="test", + description="desc", + summary="sum", + steps={"placeholder": RecipeStep(sub_recipe="sub"), "other": RecipeStep(action="stop")}, + kitchen_rules=["rule1"], + version="0.2.0", + experimental=True, + ) + result = _drop_sub_recipe_step(recipe, "placeholder") + assert result.name == "test" + assert result.summary == "sum" + assert result.experimental is True + assert result.kitchen_rules == ["rule1"] + assert "placeholder" not in result.steps + assert "other" in result.steps + + +# --------------------------------------------------------------------------- +# T2 — _path_mtime_ns replaces the two old helpers +# (migrated from recipe/test_io.py — this test introspects recipe._api) +# --------------------------------------------------------------------------- + + +def test_path_mtime_ns_exists_and_old_helpers_removed() -> None: + """recipe/_api.py must expose _path_mtime_ns; _file_mtime_ns/_dir_mtime_ns removed.""" + import autoskillit.recipe._api as api + + assert hasattr(api, "_path_mtime_ns"), "_path_mtime_ns must exist" + assert not hasattr(api, "_file_mtime_ns"), "_file_mtime_ns must be removed" + assert not hasattr(api, "_dir_mtime_ns"), "_dir_mtime_ns must be removed" + assert callable(api._path_mtime_ns), "_path_mtime_ns must be callable" + assert isinstance(api._path_mtime_ns(Path(".")), int), "_path_mtime_ns must return int" + + +def test_compute_registry_hash_changes_on_mtime(tmp_path: Path) -> None: + """Registry hash changes when a YAML file's mtime changes.""" + import os + + from autoskillit.recipe._api import _compute_registry_hash + + d = tmp_path / "types" + d.mkdir() + f = d / "test.yaml" + f.write_text("name: test\n") + h1 = _compute_registry_hash(d) + + f.write_text("name: test\n") # same content + stat = f.stat() + os.utime(f, ns=(stat.st_atime_ns, stat.st_mtime_ns + 1_000_000_000)) # +1 second + h2 = _compute_registry_hash(d) + + assert h1 != h2 + + +def test_load_and_validate_reuses_content_hash_from_recipe_info(tmp_path, monkeypatch): + """load_and_validate reuses match.content_hash for the result's content_hash field.""" + import autoskillit.recipe._api as api_mod + + monkeypatch.setattr(api_mod, "_LOAD_CACHE", {}) + + recipes_dir = tmp_path / ".autoskillit" / "recipes" + recipes_dir.mkdir(parents=True) + recipe_path = recipes_dir / "myrecipe.yaml" + recipe_path.write_text(MINIMAL_RECIPE_YAML) + + # Create a RecipeInfo with content_hash pre-populated + from autoskillit.core import RecipeSource + from autoskillit.recipe.schema import RecipeInfo + + info = RecipeInfo( + name="myrecipe", + description="test", + source=RecipeSource.PROJECT, + path=recipe_path, + content=None, + content_hash="sha256:abcd1234deadbeef", + ) + + result = api_mod.load_and_validate("myrecipe", tmp_path, recipe_info=info) + + assert result.get("content_hash") == "sha256:abcd1234deadbeef" + + +def test_load_and_validate_calls_list_recipes_once(tmp_path, monkeypatch): + """list_recipes is called exactly once when recipe_info is not provided.""" + import autoskillit.recipe._api as api_mod + + monkeypatch.setattr(api_mod, "_LOAD_CACHE", {}) + + recipes_dir = tmp_path / ".autoskillit" / "recipes" + recipes_dir.mkdir(parents=True) + recipe_path = recipes_dir / "myrecipe.yaml" + recipe_path.write_text(MINIMAL_RECIPE_YAML) + + call_count = {"n": 0} + original_list_recipes = api_mod.list_recipes + + def counting_list_recipes(*args, **kwargs): + call_count["n"] += 1 + return original_list_recipes(*args, **kwargs) + + monkeypatch.setattr(api_mod, "list_recipes", counting_list_recipes) + + api_mod.load_and_validate("myrecipe", tmp_path) + + assert call_count["n"] == 1 + + +def test_load_and_validate_skips_list_recipes_when_recipe_list_provided(tmp_path, monkeypatch): + """list_recipes is not called when recipe_list is provided alongside recipe_info.""" + import autoskillit.recipe._api as api_mod + + monkeypatch.setattr(api_mod, "_LOAD_CACHE", {}) + + recipes_dir = tmp_path / ".autoskillit" / "recipes" + recipes_dir.mkdir(parents=True) + recipe_path = recipes_dir / "myrecipe.yaml" + recipe_path.write_text(MINIMAL_RECIPE_YAML) + + from autoskillit.core import RecipeSource + from autoskillit.recipe.schema import RecipeInfo + + info = RecipeInfo( + name="myrecipe", + description="test", + source=RecipeSource.PROJECT, + path=recipe_path, + content=None, + content_hash=None, + ) + + def raising_list_recipes(*args, **kwargs): + raise AssertionError("list_recipes must not be called when recipe_list is provided") + + monkeypatch.setattr(api_mod, "list_recipes", raising_list_recipes) + + result = api_mod.load_and_validate("myrecipe", tmp_path, recipe_info=info, recipe_list=[info]) + + assert "error" not in result + assert result.get("content_hash") is not None diff --git a/tests/recipe/test_api_split.py b/tests/recipe/test_api_split.py new file mode 100644 index 0000000000..13ae6bc888 --- /dev/null +++ b/tests/recipe/test_api_split.py @@ -0,0 +1,81 @@ +import pytest + +pytestmark = [pytest.mark.layer("recipe"), pytest.mark.small] + + +def test_ingredients_importable_from_submodule(): + from autoskillit.recipe._recipe_ingredients import ( + format_ingredients_table, + ) + + assert callable(format_ingredients_table) + + +def test_composition_importable_from_submodule(): + from autoskillit.recipe._recipe_composition import _build_active_recipe + + assert callable(_build_active_recipe) + + +def test_analysis_graph_importable(): + from autoskillit.recipe._analysis_graph import ( + RouteEdge, + ) + + assert callable(RouteEdge) + + +def test_analysis_bfs_importable(): + from autoskillit.recipe._analysis_bfs import bfs_reachable + + assert callable(bfs_reachable) + + +def test_analysis_detectors_importable(): + from autoskillit.recipe._analysis_detectors import ( + _detect_dead_outputs, + ) + + assert callable(_detect_dead_outputs) + + +def test_analysis_blocks_importable(): + from autoskillit.recipe._analysis_blocks import extract_blocks + + assert callable(extract_blocks) + + +def test_recipe_init_surface_unchanged(): + # ValidationContext, build_recipe_graph, bfs_reachable are NOT in recipe/__init__.py + # — they live in _analysis.py and are consumed directly by rule modules. + from autoskillit.recipe import ( + make_validation_context, + ) + + assert callable(make_validation_context) + + +def test_analysis_graph_no_toplevel_igraph_import(): + import ast + from pathlib import Path + + import autoskillit.recipe._analysis_graph as _analysis_graph_mod + + src_file = Path(_analysis_graph_mod.__file__) + tree = ast.parse(src_file.read_text()) + # iter_child_nodes only visits direct children of the module node — it does not + # recurse into nested try/except or if-blocks. A top-level try/except ImportError + # wrapping igraph would be missed, but that edge case would also defeat the lazy-import + # purpose (module-load overhead at import time), so the coverage is intentionally scoped. + for node in ast.iter_child_nodes(tree): + if isinstance(node, ast.Import): + for alias in node.names: + assert alias.name != "igraph", ( + f"Top-level 'import igraph' found at line {node.lineno}; " + "must be a function-level import inside build_recipe_graph()" + ) + if isinstance(node, ast.ImportFrom) and (node.module or "").split(".")[0] == "igraph": + assert False, ( + f"Top-level 'from {node.module} import ...' found at line {node.lineno}; " + "igraph must only be imported inside build_recipe_graph()" + ) diff --git a/tests/recipe/test_audit_trail_artifacts.py b/tests/recipe/test_audit_trail_artifacts.py new file mode 100644 index 0000000000..fcdaafb6b0 --- /dev/null +++ b/tests/recipe/test_audit_trail_artifacts.py @@ -0,0 +1,63 @@ +"""Verify audit/ directory creation and artifact copy contract.""" + +import shutil +from pathlib import Path + +import pytest + +pytestmark = [pytest.mark.layer("recipe"), pytest.mark.small] + + +def test_create_worktree_creates_audit_directory(tmp_path: Path): + """Audit directory contains both expected artifacts after copy.""" + temp_dir = tmp_path / "temp" / "review-design" + temp_dir.mkdir(parents=True) + dashboard = temp_dir / "evaluation_dashboard_test_2026-05-07_120000.md" + dashboard.write_text("# Evaluation Dashboard\nverdict: GO\n") + + vis_dir = tmp_path / "temp" / "plan-visualization" + vis_dir.mkdir(parents=True) + trace = vis_dir / "visualization-plan-trace.md" + trace.write_text("# Visualization Plan Trace\nprimary_tradition: controlled_intervention\n") + + research_dir = tmp_path / "research" / "2026-05-07-test-slug" + audit_dir = research_dir / "audit" + audit_dir.mkdir(parents=True) + + shutil.copy2(dashboard, audit_dir / "design-review-dashboard.md") + shutil.copy2(trace, audit_dir / "visualization-plan-trace.md") + + assert (audit_dir / "design-review-dashboard.md").exists() + assert (audit_dir / "visualization-plan-trace.md").exists() + + +def test_audit_dashboard_copied_to_audit_dir(tmp_path: Path): + """evaluation_dashboard copied to audit/design-review-dashboard.md.""" + temp_dir = tmp_path / "temp" / "review-design" + temp_dir.mkdir(parents=True) + dashboard = temp_dir / "evaluation_dashboard_test_2026-05-07_120000.md" + dashboard.write_text("# Evaluation Dashboard\nverdict: GO\n") + + audit_dir = tmp_path / "audit" + audit_dir.mkdir() + target = audit_dir / "design-review-dashboard.md" + shutil.copy2(dashboard, target) + + assert target.exists() + assert "verdict: GO" in target.read_text() + + +def test_visualization_trace_copied_to_audit_dir(tmp_path: Path): + """visualization-plan-trace.md is copied to audit/ with content preserved.""" + vis_dir = tmp_path / "temp" / "plan-visualization" + vis_dir.mkdir(parents=True) + trace = vis_dir / "visualization-plan-trace.md" + trace.write_text("# Visualization Plan Trace\nprimary_tradition: controlled_intervention\n") + + audit_dir = tmp_path / "audit" + audit_dir.mkdir() + target = audit_dir / "visualization-plan-trace.md" + shutil.copy2(trace, target) + + assert target.exists() + assert "primary_tradition" in target.read_text() diff --git a/tests/recipe/test_audit_trail_format_doc.py b/tests/recipe/test_audit_trail_format_doc.py new file mode 100644 index 0000000000..00edb3fb7e --- /dev/null +++ b/tests/recipe/test_audit_trail_format_doc.py @@ -0,0 +1,20 @@ +"""Verify audit-trail-format.md exists and documents the audit/ structure.""" + +from pathlib import Path + +import pytest + +pytestmark = [pytest.mark.layer("recipe"), pytest.mark.small] + +AUDIT_FORMAT_PATH = Path("docs/research/audit-trail-format.md") + + +def test_audit_trail_format_doc_exists(): + assert AUDIT_FORMAT_PATH.exists() + + +def test_format_doc_documents_audit_directory(): + content = AUDIT_FORMAT_PATH.read_text() + assert "audit/" in content + assert "design-review-dashboard.md" in content + assert "visualization-plan-trace.md" in content diff --git a/tests/recipe/test_audit_trail_recipe_contracts.py b/tests/recipe/test_audit_trail_recipe_contracts.py new file mode 100644 index 0000000000..fd0930c033 --- /dev/null +++ b/tests/recipe/test_audit_trail_recipe_contracts.py @@ -0,0 +1,41 @@ +"""Verify research.yaml captures and threads audit-trail variables.""" + +from pathlib import Path + +import pytest +import yaml + +pytestmark = [pytest.mark.layer("recipe"), pytest.mark.small] + +RECIPE_PATH = Path("src/autoskillit/recipes/research.yaml") + + +def test_plan_visualization_captures_disambiguation_fields(): + """plan_visualization step captures disambiguation_rule_applied and tier_c_lens.""" + recipe = yaml.safe_load(RECIPE_PATH.read_text()) + pv_step = recipe["steps"]["plan_visualization"] + captures = pv_step.get("capture", {}) + assert "disambiguation_rule_applied" in captures + assert "tier_c_lens" in captures + + +def test_review_design_captures_classification_timestamp(): + """review_design step captures classification_timestamp.""" + recipe = yaml.safe_load(RECIPE_PATH.read_text()) + rd_step = recipe["steps"]["review_design"] + captures = rd_step.get("capture", {}) + assert "classification_timestamp" in captures + + +def test_generate_report_receives_audit_fields(): + """generate_report step receives disambiguation, verdict, and timestamp flags.""" + recipe = yaml.safe_load(RECIPE_PATH.read_text()) + gr_step = recipe["steps"]["generate_report"] + skill_command = gr_step.get("with", {}).get("skill_command", "") + for field in [ + "design-review-verdict", + "disambiguation-rule-applied", + "tier-c-lens", + "classification-timestamp", + ]: + assert field in skill_command, f"generate_report skill_command must contain --{field}" diff --git a/tests/recipe/test_bem_wrapper_structure.py b/tests/recipe/test_bem_wrapper_structure.py new file mode 100644 index 0000000000..33539a4475 --- /dev/null +++ b/tests/recipe/test_bem_wrapper_structure.py @@ -0,0 +1,139 @@ +import pytest + +from autoskillit.recipe.contracts import load_bundled_manifest +from autoskillit.recipe.io import builtin_recipes_dir, load_recipe +from autoskillit.recipe.validator import validate_recipe_structure + +pytestmark = [pytest.mark.layer("recipe"), pytest.mark.small] + +RECIPE_PATH = builtin_recipes_dir() / "bem-wrapper.yaml" + + +def _load(): + return load_recipe(RECIPE_PATH) + + +def test_bem_wrapper_recipe_file_exists(): + assert RECIPE_PATH.exists(), "bem-wrapper.yaml not found in bundled recipes" + + +def test_bem_wrapper_recipe_loads(): + recipe = _load() + assert recipe.name == "bem-wrapper" + assert recipe.kind == "food-truck" + + +def test_bem_wrapper_passes_validation(): + recipe = _load() + errors = validate_recipe_structure(recipe) + assert errors == [], f"Unexpected validation errors: {errors}" + + +def test_bem_wrapper_ingredients_declared(): + recipe = _load() + assert "issue_urls" in recipe.ingredients + assert recipe.ingredients["issue_urls"].required is True + assert "base_branch" in recipe.ingredients + assert recipe.ingredients["base_branch"].default == "develop" + + +def test_bem_wrapper_routing_chain(): + """Verify step-level routing covers all expected transitions.""" + recipe = _load() + steps = recipe.steps + + assert steps["refetch_issues"].on_success == "check_issue_count" + assert steps["refetch_issues"].on_failure == "emit_fallback_map" + + # check_issue_count is a route action with two branches + cic = steps["check_issue_count"] + assert cic.action == "route" + route_targets = {c.route for c in cic.on_result.conditions} + assert "emit_empty_result" in route_targets + assert "run_bem" in route_targets + + assert steps["run_bem"].on_success == "emit_result" + assert steps["run_bem"].on_failure == "emit_fallback_map" + + assert steps["emit_fallback_map"].on_success == "emit_result" + assert steps["emit_fallback_map"].on_failure == "escalate_stop" + + assert steps["emit_result"].on_success == "done" + assert steps["emit_result"].on_failure == "done" + + assert steps["emit_empty_result"].on_success == "done" + assert steps["emit_empty_result"].on_failure == "done" + + +def test_bem_wrapper_fallback_references_autoskillit_temp(): + """Fallback map must reference {{AUTOSKILLIT_TEMP}} placeholder, not bare $AUTOSKILLIT_TEMP.""" + raw = RECIPE_PATH.read_text() + assert "{{AUTOSKILLIT_TEMP}}" in raw + assert "$AUTOSKILLIT_TEMP" not in raw + + +def test_bem_wrapper_fallback_writes_file_not_json_to_stdout(): + """emit_fallback_map must write JSON to a file and return the path. + + The campaign captures the path via result.execution_map. If raw JSON were + returned, implement-findings would get ENOENT when opening it as a file path. + """ + recipe = _load() + step = recipe.steps["emit_fallback_map"] + assert step.tool == "run_python" + assert step.with_args["callable"] == "autoskillit.recipe._cmd_rpc.emit_fallback_map" + assert step.capture.get("execution_map") == "${{ result.execution_map }}" + + +def test_bem_wrapper_emit_result_consumes_execution_map_context(): + """emit_result must reference context.execution_map in its cmd to satisfy dead_output.""" + recipe = _load() + cmd = recipe.steps["emit_result"].with_args["cmd"] + assert "context.execution_map" in cmd + + +def test_bem_wrapper_done_message_mentions_dispatch_plan(): + recipe = _load() + msg = recipe.steps["done"].message + assert "dispatch_plan" in msg + + +def test_bem_wrapper_done_message_mentions_execution_map(): + recipe = _load() + msg = recipe.steps["done"].message + assert "execution_map" in msg + + +def test_bem_wrapper_run_skill_capture_keys_in_contract(): + """Captured keys from run_bem must exist in build-execution-map skill contract.""" + recipe = _load() + manifest = load_bundled_manifest() + bem_outputs = {o["name"] for o in manifest["skills"]["build-execution-map"]["outputs"]} + step = recipe.steps["run_bem"] + for key in step.capture: + capture_val = step.capture[key] + # capture value format: "${{ result. }}" + field = capture_val.replace("${{ result.", "").replace(" }}", "").strip() + assert field in bem_outputs, f"Captured field '{field}' not in build-execution-map outputs" + + +def test_bem_wrapper_emit_empty_result_cmd_echoes_none(): + recipe = _load() + cmd = recipe.steps["emit_empty_result"].with_args["cmd"] + assert "none" in cmd.lower() + + +def test_bem_wrapper_escalate_stop_is_stop_action(): + recipe = _load() + assert recipe.steps["escalate_stop"].action == "stop" + assert recipe.steps["done"].action == "stop" + + +def test_bem_wrapper_no_dead_outputs(): + """No captured context variables should be dead (captured but never consumed).""" + from autoskillit.recipe._analysis import analyze_dataflow + + recipe = _load() + report = analyze_dataflow(recipe) + dead = {w.field for w in report.warnings if w.code == "DEAD_OUTPUT"} + assert dead == set(), f"Dead output variables detected: {dead}" diff --git a/tests/recipe/test_bundled_model_field.py b/tests/recipe/test_bundled_model_field.py new file mode 100644 index 0000000000..3031a91cf8 --- /dev/null +++ b/tests/recipe/test_bundled_model_field.py @@ -0,0 +1,25 @@ +"""Verify all run_skill steps declare a model: field across bundled recipes.""" + +import pytest + +from autoskillit.recipe.io import builtin_recipes_dir, load_recipe + +pytestmark = [pytest.mark.layer("recipe"), pytest.mark.medium] + +_ALL_BUNDLED_RECIPES = [f.name for f in sorted(builtin_recipes_dir().glob("*.yaml"))] +assert _ALL_BUNDLED_RECIPES, "no bundled recipes found" + + +class TestAllRunSkillStepsHaveModel: + """Every run_skill step must declare model: so the orchestrator can propagate it.""" + + @pytest.mark.parametrize("recipe_name", _ALL_BUNDLED_RECIPES) + def test_model_field_is_string(self, recipe_name: str) -> None: + """model: field must be a string (empty or expression), never None.""" + recipe_path = builtin_recipes_dir() / recipe_name + recipe = load_recipe(recipe_path) + for name, step in recipe.steps.items(): + if step.tool == "run_skill": + assert isinstance(step.model, str), ( + f"{recipe_name}.{name}: model field should be str, got {type(step.model)}" + ) diff --git a/tests/recipe/test_bundled_recipe_hidden_policy.py b/tests/recipe/test_bundled_recipe_hidden_policy.py index 9bca70c632..3f125aa224 100644 --- a/tests/recipe/test_bundled_recipe_hidden_policy.py +++ b/tests/recipe/test_bundled_recipe_hidden_policy.py @@ -6,11 +6,14 @@ from autoskillit.recipe.io import load_recipe from autoskillit.recipe.registry import run_semantic_rules +pytestmark = [pytest.mark.layer("recipe"), pytest.mark.small] + BUNDLED_RECIPE_NAMES = [ "implementation", "remediation", "implementation-groups", "merge-prs", + "full-audit", ] diff --git a/tests/recipe/test_bundled_recipes.py b/tests/recipe/test_bundled_recipes.py deleted file mode 100644 index 566f649504..0000000000 --- a/tests/recipe/test_bundled_recipes.py +++ /dev/null @@ -1,2242 +0,0 @@ -"""Tests for structural assertions on individual bundled YAML recipe files.""" - -from __future__ import annotations - -from pathlib import Path - -import pytest -import structlog.testing - -from autoskillit.core import SKILL_TOOLS -from autoskillit.recipe._analysis import build_recipe_graph -from autoskillit.recipe.contracts import load_bundled_manifest -from autoskillit.recipe.io import builtin_recipes_dir, load_recipe -from autoskillit.recipe.rules_merge import _is_commit_guard -from autoskillit.recipe.validator import analyze_dataflow, run_semantic_rules - -PROJECT_ROOT = Path(__file__).resolve().parent.parent.parent -SMOKE_RECIPE = PROJECT_ROOT / ".autoskillit" / "recipes" / "smoke-test.yaml" - - -def _resolve_recipe_path(name: str) -> Path: - """Resolve recipe name to path, handling project-local smoke-test.""" - if name == "smoke-test": - return SMOKE_RECIPE - return builtin_recipes_dir() / f"{name}.yaml" - - -_ALL_CLONE_RECIPE_PATHS: list[Path] = [] -for _dir in (builtin_recipes_dir(), PROJECT_ROOT / ".autoskillit" / "recipes"): - if _dir.is_dir(): - for _p in sorted(_dir.glob("*.yaml")): - _wf = load_recipe(_p) - if any(s.tool == "clone_repo" for s in _wf.steps.values()): - _ALL_CLONE_RECIPE_PATHS.append(_p) - - -def _assert_ci_conflict_fix_on_context_limit(recipe) -> None: - """Shared assertion: ci_conflict_fix must abort via release_issue_failure on context limit.""" - step = recipe.steps["ci_conflict_fix"] - assert step.on_context_limit == "release_issue_failure", ( - "ci_conflict_fix is advisory; an incomplete conflict fix cannot be safely " - "pushed — abort via release_issue_failure" - ) - - -def _assert_ci_steps(recipe) -> None: - """Assert shared CI step structure across all pipeline recipe variants. - - Covers T_CI1–T_CI6: ci_watch, resolve_ci, and re_push steps. - Called from each recipe test class to eliminate triplication. - """ - # ci_watch step (T_CI1, T_CI2, T_CI3) - assert "ci_watch" in recipe.steps - ci = recipe.steps["ci_watch"] - assert ci.tool == "wait_for_ci" - assert ci.skip_when_false == "inputs.open_pr" - assert ci.with_args.get("timeout_seconds") == 300 - assert ci.on_success == "check_repo_merge_state" - assert ci.on_failure == "detect_ci_conflict" - assert "release_issue_success" in recipe.steps - assert "context.merge_target" in ci.with_args["branch"] - assert "cmd" not in ci.with_args - assert "ci_conclusion" in ci.capture - assert "ci_failed_jobs" in ci.capture - - # resolve_ci step (T_CI4, T_CI5) - assert "resolve_ci" in recipe.steps - resolve_ci = recipe.steps["resolve_ci"] - assert resolve_ci.tool == "run_skill" - skill_cmd = resolve_ci.with_args.get("skill_command", "") - assert "resolve-failures" in skill_cmd - assert resolve_ci.retries == 2 - assert resolve_ci.on_exhausted == "release_issue_failure" - assert "context.work_dir" in skill_cmd - - # re_push step (T_CI6) - assert "re_push" in recipe.steps - re_push = recipe.steps["re_push"] - assert re_push.tool == "push_to_remote" - assert re_push.on_success == "ci_watch" - assert re_push.on_failure == "release_issue_failure" - - -# --------------------------------------------------------------------------- -# TestImplementationPipelineStructure -# --------------------------------------------------------------------------- - - -class TestImplementationPipelineStructure: - @pytest.fixture(scope="class") - def recipe(self): - return load_recipe(builtin_recipes_dir() / "implementation.yaml") - - def test_ip2_review_step_captures_review_path(self, recipe) -> None: - """T_IP2: review step has capture containing key review_path.""" - assert "review_path" in recipe.steps["review"].capture - - def test_ip3_audit_impl_has_verdict_and_remediation_capture_and_on_result( - self, - recipe, - ) -> None: - """T_IP3: audit_impl captures remediation_path and routes via on_result using verdict. - - Uses predicate format (v0.3.0): verdict is read directly from result.verdict in predicate - conditions — it is not captured as context.verdict (which would create a dead output). - """ - step = recipe.steps["audit_impl"] - assert "remediation_path" in step.capture - assert step.on_result is not None - # Predicate format: conditions list (not legacy field+routes dict) - conds = step.on_result.conditions - assert len(conds) > 0, "audit_impl on_result must have predicate conditions" - assert any("result.verdict" in (c.when or "") for c in conds), ( - "audit_impl on_result must have a condition checking result.verdict" - ) - - def test_ip4_verify_step_references_context_review_path(self, recipe) -> None: - """T_IP4: verify step with_args contains a reference to context.review_path.""" - verify_with = recipe.steps["verify"].with_args - assert any("context.review_path" in str(v) for v in verify_with.values()) - - def test_ip5_audit_impl_has_on_failure(self, recipe) -> None: - """T_IP5: audit_impl declares on_failure for tool-level failure routing. - - In the two-tier failure model, on_result.conditions fire when run_skill returns - success=true with a result object. on_failure fires when run_skill returns - success=false (tool-level failure, no result object). Both must be declared. - """ - step = recipe.steps["audit_impl"] - assert step.on_success is None # on_result is used; on_success remains absent - assert step.on_failure == "escalate_stop", ( - "audit_impl must declare on_failure: escalate_stop. " - "Tool-level failures produce no result object — on_result conditions cannot fire." - ) - - def test_ip6_plan_step_note_contains_glob_pattern(self, recipe) -> None: - """T_IP6: plan step note must contain *_part_*.md glob pattern for multi-part discovery.""" - note = recipe.steps["plan"].note or "" - assert "*_part_*.md" in note, ( - "plan step note must contain glob pattern for multi-part discovery; " - "if removed, agents will not discover part files" - ) - - def test_ip7_verify_step_note_sequential_constraint(self, recipe) -> None: - """T_IP7: verify step note must contain sequential execution constraint.""" - note = recipe.steps["verify"].note or "" - assert "SEQUENTIAL EXECUTION" in note or "full cycle" in note.lower(), ( - "verify step note must contain sequential constraint; " - "without it agents may batch-verify all parts before implementing any" - ) - - def test_ip8_next_or_done_routes_more_parts_to_verify(self, recipe) -> None: - """T_IP8: next_or_done routes more_parts back to verify for sequential processing. - - Uses predicate format (v0.3.0): when-condition checks result.next == more_parts. - """ - step = recipe.steps["next_or_done"] - assert step.on_result is not None - conds = step.on_result.conditions - assert any( - c.route == "verify" and c.when is not None and "more_parts" in c.when for c in conds - ), "next_or_done must have a predicate routing more_parts → verify" - - def test_ip9_next_or_done_routes_all_done_to_audit_impl(self, recipe) -> None: - """T_IP9: next_or_done must route all_done to audit_impl. - - Uses predicate format (v0.3.0): fallthrough condition (when=None) routes to audit_impl. - """ - step = recipe.steps["next_or_done"] - assert step.on_result is not None - conds = step.on_result.conditions - # The fallthrough condition (when=None) is the default route to audit_impl - assert any(c.route == "audit_impl" for c in conds), ( - "next_or_done must have a condition routing to audit_impl" - ) - - def test_ip_audit_impl_uses_base_sha_as_ref(self, recipe) -> None: - """T_IP_B2: audit_impl must use context.base_sha (not context.branch_name). - - branch_name is deleted by git branch -D inside merge_worktree. A commit SHA - names a git object and survives unconditionally. - """ - step = recipe.steps["audit_impl"] - skill_cmd = step.with_args.get("skill_command", "") - assert "context.base_sha" in skill_cmd, ( - "audit_impl must use context.base_sha as implementation_ref — " - "context.branch_name is deleted by merge_worktree before audit_impl runs" - ) - assert "context.branch_name" not in skill_cmd, ( - "audit_impl must NOT use context.branch_name (deleted by merge_worktree)" - ) - - def test_ip_c1_fix_step_routes_via_on_result_to_test(self, recipe) -> None: - """T_IP_C1: fix step must route via verdict-gated on_result back to test. - - After Part B, fix uses on_result: verdict dispatch instead of unconditional - on_success: test. real_fix and already_green verdicts route to test for - re-validation before entering merge_worktree. - """ - step = recipe.steps["fix"] - assert step.on_success is None, ( - "fix step must use on_result: verdict dispatch, not unconditional on_success" - ) - assert step.on_result is not None, "fix step must have on_result: verdict dispatch" - test_routes = [ - c.route for c in step.on_result.conditions if c.when and "real_fix" in c.when - ] - assert any(r == "test" for r in test_routes), ( - "fix step on_result must route verdict=real_fix to test for re-validation" - ) - - def test_ip_base_sha_captured_before_implement(self, recipe) -> None: - """A1: base_sha must be captured by a step that precedes the implement loop.""" - steps = list(recipe.steps.items()) - step_names = [name for name, _ in steps] - - # Find step that captures base_sha - sha_step = next( - (name for name, step in steps if step.capture and "base_sha" in step.capture), - None, - ) - assert sha_step is not None, "No step captures base_sha" - - implement_idx = next( - ( - i - for i, (name, step) in enumerate(steps) - if step.tool in {"run_skill"} - and "implement-worktree" in step.with_args.get("skill_command", "") - ), - None, - ) - assert implement_idx is not None, "No implement step found" - sha_idx = step_names.index(sha_step) - assert sha_idx < implement_idx, ( - f"base_sha capture step '{sha_step}' (idx {sha_idx}) must come before " - f"implement step (idx {implement_idx})" - ) - - def test_ip_merge_target_unconditionally_set(self, recipe) -> None: - """A3: merge_target must be captured by a non-optional step before merge/push.""" - steps = list(recipe.steps.items()) - step_names = [name for name, _ in steps] - - merge_idx = step_names.index("merge") if "merge" in step_names else None - assert merge_idx is not None, "merge step not found" - - # Find any non-optional step that captures merge_target before merge - unconditional_capture = next( - ( - name - for name, step in steps[:merge_idx] - if step.capture and "merge_target" in step.capture and not step.optional - ), - None, - ) - assert unconditional_capture is not None, ( - "merge_target must be captured by a non-optional step before the merge step. " - "Currently it is only captured by the optional create_branch step, leaving it " - "undefined in direct mode (open_pr=false)." - ) - - def test_ip_base_sha_capture_uses_work_dir(self, recipe) -> None: - """A4: The base_sha capture command must reference context.work_dir.""" - sha_step = next( - ( - step - for name, step in recipe.steps.items() - if step.capture and "base_sha" in step.capture - ), - None, - ) - assert sha_step is not None, "No step captures base_sha" - with_args_str = str(sha_step.with_args) - assert "context.work_dir" in with_args_str, ( - "base_sha capture must run inside context.work_dir (the clone directory)" - ) - - def test_ip_base_sha_used_by_audit_impl(self, recipe) -> None: - """A5: base_sha captured must be consumed — specifically by audit_impl.""" - report = analyze_dataflow(recipe) - dead = {w.field for w in report.warnings if w.code == "DEAD_OUTPUT"} - assert "base_sha" not in dead, ( - "base_sha is captured but never consumed — audit_impl must reference it" - ) - - def test_ip_push_after_audit_warning_fires(self, recipe) -> None: - """T_IP_PBA: after Part B, audit_impl has skip_when_false so push-before-audit - fires as a WARNING. This is correct and expected — the user can opt out of audit - (audit=false), and the rule signals that push is reachable without audit on that path. - """ - from autoskillit.core.types import Severity - from autoskillit.recipe.validator import run_semantic_rules - - findings = run_semantic_rules(recipe) - violations = [f for f in findings if f.rule == "push-before-audit"] - assert len(violations) >= 1, ( - "push-before-audit must fire: audit_impl has skip_when_false so push is " - "reachable via the audit=false bypass path" - ) - assert all(v.severity == Severity.WARNING for v in violations) - - def test_ip_compose_pr_routes_to_extract_pr_number(self, recipe) -> None: - """compose_pr.on_success routes to extract_pr_number before review_pr.""" - compose_pr = recipe.steps["compose_pr"] - assert compose_pr.on_success == "extract_pr_number", ( - "compose_pr must route to extract_pr_number" - ) - - def test_ip_compose_pr_has_skip_when_false(self, recipe) -> None: - """compose_pr must declare skip_when_false: inputs.open_pr.""" - compose_pr = recipe.steps["compose_pr"] - assert compose_pr.skip_when_false == "inputs.open_pr" - - def test_ip_audit_impl_has_skip_when_false(self, recipe) -> None: - """audit_impl must declare skip_when_false: inputs.audit.""" - audit_step = recipe.steps["audit_impl"] - assert audit_step.skip_when_false == "inputs.audit" - - def test_ip_create_branch_has_skip_when_false(self, recipe) -> None: - """create_branch must declare skip_when_false: inputs.open_pr.""" - create_branch = recipe.steps["create_branch"] - assert create_branch.skip_when_false == "inputs.open_pr" - - def test_create_branch_does_not_use_run_name_verbatim(self, recipe) -> None: - """compute_branch must not use inputs.run_name as the full branch name.""" - cmd = recipe.steps["compute_branch"].with_args["cmd"] - assert "git checkout -b ${{ inputs.run_name }} &&" not in cmd - - def test_create_branch_checks_remote_for_collisions(self, recipe) -> None: - """create_branch must use create_unique_branch tool (which always checks ls-remote).""" - assert recipe.steps["create_branch"].tool == "create_unique_branch" - - def test_create_branch_references_issue_number(self, recipe) -> None: - """compute_branch cmd must reference context.issue_number for branch naming.""" - cmd = recipe.steps["compute_branch"].with_args["cmd"] - assert "context.issue_number" in cmd - - def test_create_branch_uses_run_name_as_prefix(self, recipe) -> None: - """compute_branch must use inputs.run_name as a prefix in branch naming.""" - cmd = recipe.steps["compute_branch"].with_args["cmd"] - assert "inputs.run_name" in cmd - - def test_ip_main_push_step_not_reachable_after_compose_pr(self, recipe) -> None: - """The main `push` step must not be reachable after compose_pr — - that would be a double-push. The new `re_push` step IS reachable and is correct.""" - from autoskillit.recipe.validator import _build_step_graph - - graph = _build_step_graph(recipe) - visited: set[str] = set() - queue = [recipe.steps["compose_pr"].on_success] - while queue: - current = queue.pop(0) - if current in visited or current not in recipe.steps: - continue - visited.add(current) - queue.extend(graph.get(current, [])) - assert "push" not in visited, ( - "'push' step is reachable after compose_pr — double-push risk. " - "(re_push is allowed; push is not)" - ) - - def test_ip_open_pr_false_path_reaches_push_then_cleanup(self, recipe) -> None: - """When compose_pr is bypassed (open_pr=false), execution must go: - audit_impl (GO) → push → [compose_pr bypassed] → cleanup_success → done. - After the fix, audit_impl's GO route points directly to push, so push is - always reachable from audit_impl's successors.""" - from autoskillit.recipe.validator import _build_step_graph - - graph = _build_step_graph(recipe) - # After the fix: audit_impl.on_result.GO → push (directly). - # Verify push is reachable from audit_impl's successors. - reachable: set[str] = set() - queue = list(graph.get("audit_impl", [])) - while queue: - node = queue.pop(0) - if node in reachable or node not in recipe.steps: - continue - reachable.add(node) - queue.extend(graph.get(node, [])) - assert "push" in reachable - - def test_ip_plan_step_captures_all_plan_paths(self, recipe) -> None: - """plan step must capture all_plan_paths for multi-group accumulation.""" - assert "all_plan_paths" in recipe.steps["plan"].capture - assert "result.plan_path" in recipe.steps["plan"].capture["all_plan_paths"] - - def test_ip_prepare_pr_references_all_plan_paths(self, recipe) -> None: - """prepare_pr must pass all accumulated plan paths, not just the last.""" - cmd = recipe.steps["prepare_pr"].with_args.get("skill_command", "") - assert "context.all_plan_paths" in cmd - assert "context.plan_path" not in cmd - - def test_ip_plan_step_note_contains_accumulation_instruction(self, recipe) -> None: - """plan step note must instruct agent to accumulate plan paths across groups.""" - note = recipe.steps["plan"].note or "" - assert "ACCUMULATION" in note - assert "all_plan_paths" in note - - def test_ip_no_group_step(self, recipe) -> None: - """implementation.yaml must not contain a group step.""" - assert "group" not in recipe.steps - - def test_ip_task_ingredient_required(self, recipe) -> None: - """task ingredient must be required in the direct recipe.""" - task_ing = recipe.ingredients.get("task") - assert task_ing is not None - assert task_ing.required is True or task_ing.default is None - - def test_ip_no_make_groups_ingredient(self, recipe) -> None: - """make_groups ingredient must not be present.""" - assert "make_groups" not in recipe.ingredients - - def test_ip_no_source_doc_ingredient(self, recipe) -> None: - """source_doc ingredient must not be present.""" - assert "source_doc" not in recipe.ingredients - - def test_ip_next_or_done_no_more_groups_route(self, recipe) -> None: - """next_or_done must not route more_groups — no groups in the direct recipe.""" - step = recipe.steps["next_or_done"] - assert step.on_result is not None - conds = step.on_result.conditions - assert not any("more_groups" in (c.when or "") for c in conds) - - def test_ci_step_structure(self, recipe) -> None: - """T_CI1–T_CI6: shared ci_watch, resolve_ci, and re_push structure.""" - _assert_ci_steps(recipe) - - def test_ip_re_push_has_explicit_remote_url(self, recipe) -> None: - """T_CI7: re_push uses explicit remote_url (satisfies push-missing-explicit-remote-url).""" - with_args = recipe.steps["re_push"].with_args - assert "remote_url" in with_args - assert "context.remote_url" in with_args["remote_url"] - - def test_ip_compose_pr_routes_to_extract_pr_number_ci(self, recipe) -> None: - """T_CI8: compose_pr.on_success routes to extract_pr_number before review_pr.""" - step = recipe.steps["compose_pr"] - assert step.on_success == "extract_pr_number", "compose_pr must route to extract_pr_number" - - def test_ip_detect_ci_conflict_exists(self, recipe) -> None: - assert "detect_ci_conflict" in recipe.steps - step = recipe.steps["detect_ci_conflict"] - assert step.tool == "run_cmd" - - def test_ip_detect_ci_conflict_uses_merge_base(self, recipe) -> None: - step = recipe.steps["detect_ci_conflict"] - cmd = (step.with_args or {}).get("cmd", "") - assert "merge-base" in cmd or "is-ancestor" in cmd - - def test_ip_detect_ci_conflict_routing(self, recipe) -> None: - step = recipe.steps["detect_ci_conflict"] - assert step.on_success == "ci_conflict_fix" - assert step.on_failure == "diagnose_ci" - - def test_ip_ci_conflict_fix_exists(self, recipe) -> None: - assert "ci_conflict_fix" in recipe.steps - step = recipe.steps["ci_conflict_fix"] - assert step.tool == "run_skill" - skill_cmd = (step.with_args or {}).get("skill_command", "") - assert "resolve-merge-conflicts" in skill_cmd - - def test_ip_ci_conflict_fix_routing(self, recipe) -> None: - step = recipe.steps["ci_conflict_fix"] - assert step.on_failure == "release_issue_failure" - assert step.on_exhausted == "release_issue_failure" - conditions = step.on_result.conditions if step.on_result else [] - routes = {c.when: c.route for c in conditions} - assert any( - "escalation_required" in (w or "") and r == "release_issue_failure" - for w, r in routes.items() - ) - default_routes = [r for w, r in routes.items() if w is None] - assert default_routes == ["re_push"] - - def test_ip_detect_ci_conflict_skip_when_false(self, recipe) -> None: - step = recipe.steps["detect_ci_conflict"] - assert step.skip_when_false == "inputs.open_pr" - - def test_ip_ci_conflict_fix_skip_when_false(self, recipe) -> None: - step = recipe.steps["ci_conflict_fix"] - assert step.skip_when_false == "inputs.open_pr" - - def test_ip_review_step_has_skip_when_false(self, recipe) -> None: - """REQ-C7-02: review step must declare skip_when_false: inputs.review_approach.""" - step = recipe.steps["review"] - assert step.skip_when_false == "inputs.review_approach", ( - "review step must declare skip_when_false: inputs.review_approach — " - "the skip intent is already in the note: field but not schema-enforced" - ) - - def test_implementation_review_step_has_on_context_limit(self, recipe) -> None: - step = recipe.steps["review"] - assert step.on_context_limit == "verify", ( - "review is advisory (skip_when_false); on context limit it should skip to " - "verify, not abort via on_failure" - ) - - def test_implementation_review_step_has_retries(self, recipe) -> None: - step = recipe.steps["review"] - assert step.retries >= 1 - - def test_ip_audit_impl_has_on_context_limit(self, recipe) -> None: - step = recipe.steps["audit_impl"] - assert step.on_context_limit == "escalate_stop", ( - "audit_impl is a merge gate; a context-exhausted audit cannot provide " - "a valid verdict — aborting via escalate_stop is correct" - ) - - def test_ip_compose_pr_has_on_context_limit(self, recipe) -> None: - step = recipe.steps["compose_pr"] - assert step.on_context_limit == "release_issue_failure", ( - "compose_pr is advisory (skip_when_false); on context limit the pipeline " - "cannot determine PR state — release the issue via release_issue_failure" - ) - - def test_ip_ci_conflict_fix_has_on_context_limit(self, recipe) -> None: - _assert_ci_conflict_fix_on_context_limit(recipe) - - -# --------------------------------------------------------------------------- -# TestImplementationGroupsStructure -# --------------------------------------------------------------------------- - - -class TestImplementationGroupsStructure: - @pytest.fixture(scope="class") - def recipe(self): - return load_recipe(builtin_recipes_dir() / "implementation-groups.yaml") - - def test_ig1_group_step_captures_group_files(self, recipe) -> None: - """T_IG1: group step captures group_files, not groups_path.""" - assert "group_files" in recipe.steps["group"].capture - assert "groups_path" not in recipe.steps["group"].capture - - def test_ig2_group_step_is_not_optional(self, recipe) -> None: - """T_IG2: group step must always run — no skip_when_false, not conditional.""" - step = recipe.steps["group"] - assert step.skip_when_false is None - assert not step.optional - - def test_ig3_source_doc_required(self, recipe) -> None: - """T_IG3: source_doc must be a required ingredient in the groups recipe.""" - src = recipe.ingredients.get("source_doc") - assert src is not None - assert src.required is True - - def test_ig4_no_make_groups_ingredient(self, recipe) -> None: - """T_IG4: make_groups must not be present — groups are always used in this recipe.""" - assert "make_groups" not in recipe.ingredients - - def test_ig5_next_or_done_routes_more_groups_to_plan(self, recipe) -> None: - """T_IG5: next_or_done must route more_groups back to plan for group iteration.""" - step = recipe.steps["next_or_done"] - assert step.on_result is not None - conds = step.on_result.conditions - assert any( - c.route == "plan" and c.when is not None and "more_groups" in c.when for c in conds - ), "next_or_done must have a predicate routing more_groups → plan" - - def test_ig6_next_or_done_routes_more_parts_to_verify(self, recipe) -> None: - """T_IG6: next_or_done must route more_parts to verify for sequential part processing.""" - step = recipe.steps["next_or_done"] - assert step.on_result is not None - conds = step.on_result.conditions - assert any( - c.route == "verify" and c.when is not None and "more_parts" in c.when for c in conds - ), "next_or_done must have a predicate routing more_parts → verify" - - def test_ig7_next_or_done_fallthrough_to_audit_impl(self, recipe) -> None: - """T_IG7: next_or_done fallthrough (all done) must route to audit_impl.""" - step = recipe.steps["next_or_done"] - assert step.on_result is not None - conds = step.on_result.conditions - assert any(c.route == "audit_impl" for c in conds) - - def test_ig8_plan_note_contains_accumulation_instruction(self, recipe) -> None: - """T_IG8: plan step note must instruct agent to accumulate plan paths across groups.""" - note = recipe.steps["plan"].note or "" - assert "ACCUMULATION" in note - assert "all_plan_paths" in note - - def test_ig_push_merge_target_routes_to_group(self, recipe) -> None: - """push_merge_target must route to group, not plan, in the groups recipe.""" - step = recipe.steps.get("push_merge_target") - assert step is not None - assert step.on_success == "group" - - def test_ig_audit_impl_uses_base_sha_as_ref(self, recipe) -> None: - """audit_impl must use context.base_sha as implementation_ref.""" - step = recipe.steps["audit_impl"] - skill_cmd = step.with_args.get("skill_command", "") - assert "context.base_sha" in skill_cmd - assert "context.branch_name" not in skill_cmd - - def test_ig_fix_step_routes_via_on_result_to_test(self, recipe) -> None: - """fix step must route via verdict-gated on_result to test.""" - step = recipe.steps["fix"] - assert step.on_success is None, "fix step must use on_result: verdict dispatch" - assert step.on_result is not None, "fix step must have on_result: verdict dispatch" - test_routes = [ - c.route for c in step.on_result.conditions if c.when and "real_fix" in c.when - ] - assert any(r == "test" for r in test_routes), ( - "fix step on_result must route verdict=real_fix to test" - ) - - def test_ig_push_after_audit_warning_fires(self, recipe) -> None: - """push-before-audit semantic rule fires as WARNING (audit has skip_when_false).""" - from autoskillit.core.types import Severity - from autoskillit.recipe.validator import run_semantic_rules - - findings = run_semantic_rules(recipe) - violations = [f for f in findings if f.rule == "push-before-audit"] - assert len(violations) >= 1 - assert all(v.severity == Severity.WARNING for v in violations) - - def test_ci_step_structure(self, recipe) -> None: - """T_CI1–T_CI6: shared ci_watch, resolve_ci, and re_push structure.""" - _assert_ci_steps(recipe) - - def test_ig_re_push_has_explicit_remote_url(self, recipe) -> None: - """T_CI7: re_push uses explicit remote_url.""" - with_args = recipe.steps["re_push"].with_args - assert "remote_url" in with_args - assert "context.remote_url" in with_args["remote_url"] - - def test_ig_detect_ci_conflict_exists(self, recipe) -> None: - assert "detect_ci_conflict" in recipe.steps - step = recipe.steps["detect_ci_conflict"] - assert step.tool == "run_cmd" - - def test_ig_detect_ci_conflict_uses_merge_base(self, recipe) -> None: - step = recipe.steps["detect_ci_conflict"] - cmd = (step.with_args or {}).get("cmd", "") - assert "merge-base" in cmd or "is-ancestor" in cmd - - def test_ig_detect_ci_conflict_routing(self, recipe) -> None: - step = recipe.steps["detect_ci_conflict"] - assert step.on_success == "ci_conflict_fix" - assert step.on_failure == "diagnose_ci" - - def test_ig_ci_conflict_fix_exists(self, recipe) -> None: - assert "ci_conflict_fix" in recipe.steps - step = recipe.steps["ci_conflict_fix"] - assert step.tool == "run_skill" - skill_cmd = (step.with_args or {}).get("skill_command", "") - assert "resolve-merge-conflicts" in skill_cmd - - def test_ig_ci_conflict_fix_routing(self, recipe) -> None: - step = recipe.steps["ci_conflict_fix"] - assert step.on_failure == "release_issue_failure" - assert step.on_exhausted == "release_issue_failure" - conditions = step.on_result.conditions if step.on_result else [] - routes = {c.when: c.route for c in conditions} - assert any( - "escalation_required" in (w or "") and r == "release_issue_failure" - for w, r in routes.items() - ) - default_routes = [r for w, r in routes.items() if w is None] - assert default_routes == ["re_push"] - - def test_ig_detect_ci_conflict_skip_when_false(self, recipe) -> None: - step = recipe.steps["detect_ci_conflict"] - assert step.skip_when_false == "inputs.open_pr" - - def test_ig_ci_conflict_fix_skip_when_false(self, recipe) -> None: - step = recipe.steps["ci_conflict_fix"] - assert step.skip_when_false == "inputs.open_pr" - - def test_ig_auto_merge_ingredient_exists(self, recipe) -> None: - """REQ-C7-01: auto_merge ingredient must exist in implementation-groups.yaml.""" - assert "auto_merge" in recipe.ingredients, ( - "auto_merge ingredient is missing — required for merge queue lifecycle" - ) - assert recipe.ingredients["auto_merge"].default == "true" - - def test_ig_extract_pr_number_step_exists(self, recipe) -> None: - """REQ-C7-01: extract_pr_number step must exist to supply pr_number to queue steps.""" - assert "extract_pr_number" in recipe.steps - step = recipe.steps["extract_pr_number"] - assert step.tool == "run_cmd" - assert "pr_number" in step.capture - assert step.on_success == "annotate_pr_diff" - - def test_ig_compose_pr_routes_to_extract_pr_number(self, recipe) -> None: - """REQ-C7-01: compose_pr must route to extract_pr_number (not review_pr directly).""" - step = recipe.steps["compose_pr"] - assert step.on_success == "extract_pr_number", ( - "compose_pr must route to extract_pr_number so pr_number is available " - "for enable_auto_merge and wait_for_queue" - ) - - def test_ig_ci_watch_routes_to_check_repo_merge_state(self, recipe) -> None: - """REQ-C7-01: ci_watch.on_success must route to check_repo_merge_state (not release_issue_success).""" # noqa: E501 - step = recipe.steps["ci_watch"] - assert step.on_success == "check_repo_merge_state", ( - "ci_watch must route to check_repo_merge_state so the PR can enter the merge queue. " - "Routing directly to release_issue_success skips the queue lifecycle entirely." - ) - - def test_ig_check_repo_merge_state_step_exists(self, recipe) -> None: - """REQ-C7-01: check_repo_merge_state step must exist.""" - assert "check_repo_merge_state" in recipe.steps - step = recipe.steps["check_repo_merge_state"] - assert step.tool == "check_repo_merge_state" - assert step.block == "pre_queue_gate" - assert "queue_available" in step.capture - - def test_ig_wait_for_queue_step_exists(self, recipe) -> None: - """REQ-C7-01: wait_for_queue step must exist with correct tool and routing.""" - assert "wait_for_queue" in recipe.steps - step = recipe.steps["wait_for_queue"] - assert step.tool == "wait_for_merge_queue" - assert step.with_args.get("timeout_seconds") == 900 - conds = step.on_result.conditions if step.on_result else [] - merged_cond = next((c for c in conds if c.when and "merged" in c.when), None) - assert merged_cond is not None and merged_cond.route == "release_issue_success" - - def test_ig_release_issue_success_has_target_branch(self, recipe) -> None: - """REQ-C7-01: release_issue_success must pass target_branch to trigger staged label.""" - step = recipe.steps["release_issue_success"] - with_args = step.with_args or {} - assert "target_branch" in with_args, ( - "release_issue_success must pass target_branch: ${{ inputs.base_branch }} — " - "without it release_issue cannot apply the staged label on non-default branches" - ) - assert "inputs.base_branch" in with_args["target_branch"] - - def test_ig_review_step_has_skip_when_false(self, recipe) -> None: - """REQ-C7-02: review step must declare skip_when_false: inputs.review_approach.""" - step = recipe.steps["review"] - assert step.skip_when_false == "inputs.review_approach", ( - "review step must declare skip_when_false: inputs.review_approach to make the " - "skip intent schema-enforced. Currently it is prose-only in the note: field." - ) - - def test_ig_review_step_has_on_context_limit(self, recipe) -> None: - step = recipe.steps["review"] - assert step.on_context_limit == "verify", ( - "review is advisory (skip_when_false); on context limit it should skip to " - "verify, not abort via on_failure" - ) - - def test_ig_review_step_has_retries(self, recipe) -> None: - step = recipe.steps["review"] - assert step.retries >= 1 - - def test_ig_audit_impl_has_on_context_limit(self, recipe) -> None: - step = recipe.steps["audit_impl"] - assert step.on_context_limit == "escalate_stop", ( - "audit_impl is a merge gate; a context-exhausted audit cannot provide " - "a valid verdict — aborting via escalate_stop is correct" - ) - - def test_ig_compose_pr_has_on_context_limit(self, recipe) -> None: - step = recipe.steps["compose_pr"] - assert step.on_context_limit == "release_issue_failure", ( - "compose_pr is advisory (skip_when_false); on context limit the pipeline " - "cannot determine PR state — release the issue via release_issue_failure" - ) - - def test_ig_ci_conflict_fix_has_on_context_limit(self, recipe) -> None: - _assert_ci_conflict_fix_on_context_limit(recipe) - - -# --------------------------------------------------------------------------- -# TestInvestigateFirstStructure -# --------------------------------------------------------------------------- - - -class TestInvestigateFirstStructure: - @pytest.fixture(scope="class") - def recipe(self): - return load_recipe(builtin_recipes_dir() / "remediation.yaml") - - def test_if1_audit_impl_has_verdict_and_remediation_capture_and_on_result( - self, - recipe, - ) -> None: - """T_IF1: audit_impl captures remediation_path and routes via on_result using verdict. - - Uses predicate format (v0.3.0): verdict is read directly from result.verdict in predicate - conditions — it is not captured as context.verdict (which would create a dead output). - """ - step = recipe.steps["audit_impl"] - assert "remediation_path" in step.capture - assert step.on_result is not None - conds = step.on_result.conditions - assert len(conds) > 0, "audit_impl on_result must have predicate conditions" - assert any("result.verdict" in (c.when or "") for c in conds), ( - "audit_impl on_result must have a condition checking result.verdict" - ) - - def test_if2_remediate_step_routes_to_make_plan(self, recipe) -> None: - """T_IF2: remediate step exists and routes to make_plan (not rectify).""" - assert "remediate" in recipe.steps - assert recipe.steps["remediate"].on_success == "make_plan" - - def test_if5_make_plan_step_has_correct_structure(self, recipe) -> None: - """T_IF5: make_plan step calls make-plan with remediation_path and captures outputs.""" - assert "make_plan" in recipe.steps - step = recipe.steps["make_plan"] - assert step.tool == "run_skill" - skill_cmd = step.with_args.get("skill_command", "") - assert "/autoskillit:make-plan" in skill_cmd - assert "context.remediation_path" in skill_cmd - assert "plan_path" in step.capture - assert "plan_parts" in step.capture_list - assert step.on_success == "dry_walkthrough" - assert step.on_failure == "release_issue_failure" - - def test_if3_test_step_uses_implementation_ref(self, recipe) -> None: - """T_IF3: test step worktree_path must reference context.implementation_ref.""" - worktree_arg = recipe.steps["test"].with_args.get("worktree_path", "") - assert "context.implementation_ref" in worktree_arg - assert "context.work_dir" not in worktree_arg - - def test_if4_merge_step_uses_implementation_ref(self, recipe) -> None: - """T_IF4: merge step worktree_path must reference context.implementation_ref.""" - worktree_arg = recipe.steps["merge"].with_args.get("worktree_path", "") - assert "context.implementation_ref" in worktree_arg - assert "context.work_dir" not in worktree_arg - - def test_if_b1_implement_captures_branch_name(self, recipe) -> None: - """T_IF_B1: implement step must capture branch_name from result.""" - step = recipe.steps["implement"] - assert "branch_name" in step.capture, ( - "implement step must capture branch_name so audit_impl can pass a " - "stable git ref to audit-impl after merge_worktree deletes the worktree" - ) - - def test_if_b2_audit_impl_uses_branch_name_as_ref(self, recipe) -> None: - """T_IF_B2: audit_impl with: must reference context.branch_name as implementation_ref.""" - step = recipe.steps["audit_impl"] - skill_cmd = step.with_args.get("skill_command", "") - assert "context.branch_name" in skill_cmd, ( - "audit_impl must pass context.branch_name as implementation_ref — not " - "context.implementation_ref or context.worktree_path (stale after merge)" - ) - - def test_if_b3_retry_worktree_captures_branch_name(self, recipe) -> None: - """T_IF_B3: retry_worktree step must also capture branch_name.""" - step = recipe.steps["retry_worktree"] - assert "branch_name" in step.capture, ( - "retry_worktree also updates the active worktree reference; " - "it must capture branch_name for downstream audit_impl use" - ) - - def test_create_branch_does_not_use_run_name_verbatim(self, recipe) -> None: - """compute_branch must not use inputs.run_name as the full branch name.""" - cmd = recipe.steps["compute_branch"].with_args["cmd"] - assert "git checkout -b ${{ inputs.run_name }} &&" not in cmd - - def test_create_branch_checks_remote_for_collisions(self, recipe) -> None: - """create_branch must use create_unique_branch tool (which always checks ls-remote).""" - assert recipe.steps["create_branch"].tool == "create_unique_branch" - - def test_create_branch_references_issue_number(self, recipe) -> None: - """compute_branch cmd must reference context.issue_number for branch naming.""" - cmd = recipe.steps["compute_branch"].with_args["cmd"] - assert "context.issue_number" in cmd - - def test_create_branch_uses_run_name_as_prefix(self, recipe) -> None: - """compute_branch must use inputs.run_name as a prefix in branch naming.""" - cmd = recipe.steps["compute_branch"].with_args["cmd"] - assert "inputs.run_name" in cmd - - def test_if_c1_implement_uses_no_merge_skill(self, recipe) -> None: - """T_IF_C1: implement step must use implement-worktree-no-merge. - - implement-worktree merges and deletes the worktree internally; subsequent - verify (test_check) and assess (resolve-failures) steps would run against - a non-existent path. implement-worktree-no-merge leaves the worktree intact - for the orchestrator's gate-test-merge cycle. - """ - step = recipe.steps["implement"] - skill_cmd = step.with_args.get("skill_command", "") - assert "implement-worktree-no-merge" in skill_cmd, ( - "implement step must use implement-worktree-no-merge; " - "implement-worktree merges immediately, making verify and assess unreachable" - ) - - def test_remediation_investigate_captures_investigation_path(self, recipe) -> None: - """1c: investigate step must have a capture block containing investigation_path.""" - step = recipe.steps["investigate"] - assert step.capture is not None and "investigation_path" in step.capture, ( - "investigate step must capture investigation_path so rectify receives " - "the explicit path rather than scanning the filesystem" - ) - assert step.capture["investigation_path"] == "${{ result.investigation_path }}" - - def test_remediation_rectify_uses_context_investigation_path(self, recipe) -> None: - """1d: rectify step must pass ${{ context.investigation_path }} in skill_command.""" - step = recipe.steps["rectify"] - skill_cmd = step.with_args.get("skill_command", "") - assert "${{ context.investigation_path }}" in skill_cmd, ( - "rectify step skill_command must include ${{ context.investigation_path }} " - "to pass the explicit path from the capture block" - ) - - def test_ci_step_structure(self, recipe) -> None: - """T_CI1–T_CI6: shared ci_watch, resolve_ci, and re_push structure.""" - _assert_ci_steps(recipe) - - def test_if_re_push_has_explicit_remote_url(self, recipe) -> None: - """T_CI7: re_push uses explicit remote_url.""" - with_args = recipe.steps["re_push"].with_args - assert "remote_url" in with_args - assert "context.remote_url" in with_args["remote_url"] - - def test_if_compose_pr_routes_to_extract_pr_number(self, recipe) -> None: - """T_CI8: compose_pr.on_success routes to extract_pr_number before review_pr.""" - step = recipe.steps["compose_pr"] - assert step.on_success == "extract_pr_number", "compose_pr must route to extract_pr_number" - - def test_if_resolve_review_uses_resolve_review_skill(self, recipe) -> None: - """T_IF_RR1: resolve_review step must invoke resolve-review, not resolve-failures. - - resolve-failures is test-driven and finds no work when tests are green. - resolve-review reads PR review comments and applies the reviewer's requested changes. - """ - assert "resolve_review" in recipe.steps - step = recipe.steps["resolve_review"] - assert step.tool == "run_skill" - skill_cmd = step.with_args.get("skill_command", "") - assert "resolve-review" in skill_cmd, ( - "resolve_review step must call /autoskillit:resolve-review; " - "resolve-failures is test-driven and ignores PR review comments" - ) - assert "resolve-failures" not in skill_cmd, ( - "resolve_review step must not call resolve-failures; " - "that skill does not read review comments" - ) - - def test_if_resolve_review_passes_merge_target(self, recipe) -> None: - """T_IF_RR2: resolve_review skill_command must pass context.merge_target.""" - step = recipe.steps["resolve_review"] - skill_cmd = step.with_args.get("skill_command", "") - assert "context.merge_target" in skill_cmd, ( - "resolve-review requires feature_branch as first arg; " - "context.merge_target holds the feature branch name" - ) - - def test_remediation_no_add_dir_dead_param(self, recipe) -> None: - """Remediation recipe must not have add_dir as a with: key (removed dead param).""" - findings = run_semantic_rules(recipe) - add_dir_dead = [ - f for f in findings if f.rule == "dead-with-param" and "add_dir" in f.message - ] - assert not add_dir_dead, ( - f"Remediation recipe still has dead add_dir param: " - f"{[(f.step_name, f.message) for f in add_dir_dead]}" - ) - - def test_remediation_assess_step_has_on_context_limit(self, recipe) -> None: - """REQ-RCP-002: assess step in remediation.yaml must declare on_context_limit: test. - - assess runs resolve-failures inside an existing worktree. Partial fixes are committed - to disk, so routing to test checks whether partial work was sufficient — same rationale - as the fix step in implementation.yaml. - """ - assess = recipe.steps["assess"] - assert assess.on_context_limit == "test", ( - f"remediation.yaml assess step must declare on_context_limit: test, " - f"got: {assess.on_context_limit!r}" - ) - - def test_if_detect_ci_conflict_exists(self, recipe) -> None: - assert "detect_ci_conflict" in recipe.steps - step = recipe.steps["detect_ci_conflict"] - assert step.tool == "run_cmd" - - def test_if_detect_ci_conflict_uses_merge_base(self, recipe) -> None: - step = recipe.steps["detect_ci_conflict"] - cmd = (step.with_args or {}).get("cmd", "") - assert "merge-base" in cmd or "is-ancestor" in cmd - - def test_if_detect_ci_conflict_routing(self, recipe) -> None: - step = recipe.steps["detect_ci_conflict"] - assert step.on_success == "ci_conflict_fix" - assert step.on_failure == "diagnose_ci" - - def test_if_ci_conflict_fix_exists(self, recipe) -> None: - assert "ci_conflict_fix" in recipe.steps - step = recipe.steps["ci_conflict_fix"] - assert step.tool == "run_skill" - skill_cmd = (step.with_args or {}).get("skill_command", "") - assert "resolve-merge-conflicts" in skill_cmd - - def test_if_ci_conflict_fix_routing(self, recipe) -> None: - step = recipe.steps["ci_conflict_fix"] - assert step.on_failure == "release_issue_failure" - assert step.on_exhausted == "release_issue_failure" - conditions = step.on_result.conditions if step.on_result else [] - routes = {c.when: c.route for c in conditions} - assert any( - "escalation_required" in (w or "") and r == "release_issue_failure" - for w, r in routes.items() - ) - default_routes = [r for w, r in routes.items() if w is None] - assert default_routes == ["re_push"] - - def test_if_detect_ci_conflict_skip_when_false(self, recipe) -> None: - step = recipe.steps["detect_ci_conflict"] - assert step.skip_when_false == "inputs.open_pr" - - def test_if_ci_conflict_fix_skip_when_false(self, recipe) -> None: - step = recipe.steps["ci_conflict_fix"] - assert step.skip_when_false == "inputs.open_pr" - - def test_if_review_step_has_skip_when_false(self, recipe) -> None: - """REQ-C7-02: review step in remediation.yaml must declare skip_when_false.""" - step = recipe.steps["review"] - assert step.skip_when_false == "inputs.review_approach", ( - "review step must declare skip_when_false: inputs.review_approach — " - "the skip intent is already in the note: field but not schema-enforced" - ) - - def test_if_review_step_has_on_context_limit(self, recipe) -> None: - step = recipe.steps["review"] - assert step.on_context_limit == "dry_walkthrough", ( - "review is advisory (skip_when_false); on context limit it should skip to " - "dry_walkthrough, not abort via on_failure" - ) - - def test_if_review_step_has_retries(self, recipe) -> None: - step = recipe.steps["review"] - assert step.retries >= 1, ( - "review step should allow at least one retry before routing to on_context_limit" - ) - - def test_if_audit_impl_has_on_context_limit(self, recipe) -> None: - step = recipe.steps["audit_impl"] - assert step.on_context_limit == "escalate_stop", ( - "audit_impl is a merge gate; a context-exhausted audit cannot provide " - "a valid verdict — aborting via escalate_stop is correct" - ) - - def test_if_compose_pr_has_on_context_limit(self, recipe) -> None: - step = recipe.steps["compose_pr"] - assert step.on_context_limit == "release_issue_failure", ( - "compose_pr is advisory (skip_when_false); on context limit the pipeline " - "cannot determine PR state — release the issue via release_issue_failure" - ) - - def test_if_ci_conflict_fix_has_on_context_limit(self, recipe) -> None: - _assert_ci_conflict_fix_on_context_limit(recipe) - - -# --------------------------------------------------------------------------- -# Contract tests — plan_parts output (D4–D5) -# --------------------------------------------------------------------------- - - -def test_make_plan_contract_declares_plan_parts_output() -> None: - """D4: make-plan contract must declare plan_parts as an output.""" - manifest = load_bundled_manifest() - make_plan = manifest.get("skills", {}).get("make-plan", {}) - output_names = [o["name"] for o in make_plan.get("outputs", [])] - assert "plan_parts" in output_names, ( - "make-plan contract must declare plan_parts as an output " - "so capture_list coverage validation can enforce it" - ) - - -def test_rectify_contract_declares_plan_parts_output() -> None: - """D5: rectify contract must declare plan_parts as an output.""" - manifest = load_bundled_manifest() - rectify = manifest.get("skills", {}).get("rectify", {}) - output_names = [o["name"] for o in rectify.get("outputs", [])] - assert "plan_parts" in output_names - - -# --------------------------------------------------------------------------- -# Bundled diagram tests (DG-21, DG-29) -# --------------------------------------------------------------------------- - - -def test_bundled_recipes_diagrams_dir_exists() -> None: - """Diagrams directory exists for bundled recipes.""" - from autoskillit.core.paths import pkg_root - - assert (pkg_root() / "recipes" / "diagrams").is_dir() - - -# --------------------------------------------------------------------------- -# Two-tier failure model tests -# --------------------------------------------------------------------------- - - -def test_all_predicate_steps_have_on_failure() -> None: - """Every tool/python step with on_result.conditions must declare on_failure.""" - paths = { - "implementation": builtin_recipes_dir() / "implementation.yaml", - "remediation": builtin_recipes_dir() / "remediation.yaml", - "smoke-test": SMOKE_RECIPE, - } - for recipe_name, recipe_path in paths.items(): - recipe = load_recipe(recipe_path) - for step_name, step in recipe.steps.items(): - is_tool = step.tool is not None or step.python is not None - if is_tool and step.on_result and step.on_result.conditions: - assert step.on_failure is not None, ( - f"{recipe_name}.{step_name}: predicate step must declare on_failure" - ) - - -def test_audit_impl_on_failure_routes_to_escalation() -> None: - """audit_impl.on_failure must route to an escalation step in each recipe.""" - impl = load_recipe(builtin_recipes_dir() / "implementation.yaml") - rem = load_recipe(builtin_recipes_dir() / "remediation.yaml") - assert impl.steps["audit_impl"].on_failure == "escalate_stop" - assert rem.steps["audit_impl"].on_failure == "escalate_stop" - - -@pytest.mark.parametrize( - "recipe_name,yaml_name", - [ - ("implementation", "implementation.yaml"), - ("remediation", "remediation.yaml"), - ("merge-prs", "merge-prs.yaml"), - ("implementation-groups", "implementation-groups.yaml"), - ], -) -def test_audit_ingredient_defaults_to_false(recipe_name: str, yaml_name: str) -> None: - """audit must default to 'false' (OFF) in all recipes — opt-in, not opt-out.""" - recipe = load_recipe(builtin_recipes_dir() / yaml_name) - audit_ing = recipe.ingredients.get("audit") - assert audit_ing is not None, f"{recipe_name}: 'audit' ingredient not found" - assert audit_ing.default == "false", ( - f"{recipe_name}: audit.default must be 'false' (OFF by default), got {audit_ing.default!r}" - ) - - -# --------------------------------------------------------------------------- -# SKILL.md emit instruction tests (1b, 1c, 1d) -# --------------------------------------------------------------------------- - - -def test_audit_impl_skill_md_emits_verdict_and_remediation_path() -> None: - """1b: audit-impl SKILL.md must contain verdict and remediation_path emit lines.""" - from autoskillit.core.paths import pkg_root - - content = (pkg_root() / "skills_extended" / "audit-impl" / "SKILL.md").read_text() - assert "verdict = " in content, "audit-impl SKILL.md missing 'verdict = ' emit line" - assert "remediation_path = " in content, ( - "audit-impl SKILL.md missing 'remediation_path = ' emit line" - ) - - -def test_review_approach_skill_md_emits_review_path() -> None: - """1c: review-approach SKILL.md must contain review_path emit line.""" - from autoskillit.core.paths import pkg_root - - content = (pkg_root() / "skills_extended" / "review-approach" / "SKILL.md").read_text() - assert "review_path = " in content, ( - "review-approach SKILL.md missing 'review_path = ' emit line" - ) - - -def test_make_groups_skill_md_emits_group_files() -> None: - """1d: make-groups SKILL.md must contain group_files, groups_path, manifest_path lines.""" - from autoskillit.core.paths import pkg_root - - content = (pkg_root() / "skills_extended" / "make-groups" / "SKILL.md").read_text() - assert "group_files = " in content, "make-groups SKILL.md missing 'group_files = ' emit line" - assert "groups_path = " in content, "make-groups SKILL.md missing 'groups_path = ' emit line" - assert "manifest_path = " in content, ( - "make-groups SKILL.md missing 'manifest_path = ' emit line" - ) - - -# --------------------------------------------------------------------------- -# Bundled recipe uncaptured-handoff-consumer rule (1i) -# --------------------------------------------------------------------------- - - -def test_bundled_recipes_pass_uncaptured_handoff_consumer() -> None: - """1i: all bundled recipes must produce zero uncaptured-handoff-consumer findings.""" - for yaml_file in sorted(builtin_recipes_dir().glob("*.yaml")): - recipe = load_recipe(yaml_file) - findings = run_semantic_rules(recipe) - handoff_findings = [f for f in findings if f.rule == "uncaptured-handoff-consumer"] - assert not handoff_findings, f"{yaml_file.name}: {handoff_findings}" - - -# --------------------------------------------------------------------------- -# PR Review Loop integration tests (T_RP*) -# --------------------------------------------------------------------------- - - -class TestReviewPrRecipeIntegration: - @pytest.fixture( - scope="class", - params=[ - "implementation.yaml", - "implementation-groups.yaml", - "remediation.yaml", - ], - ) - def recipe(self, request: pytest.FixtureRequest) -> object: - return load_recipe(builtin_recipes_dir() / request.param) - - def test_compose_pr_routes_to_extract_pr_number(self, recipe: object) -> None: - """T_RP1: compose_pr.on_success routes per-recipe to extract_pr_number. - - All queue-aware recipes (implementation, remediation, implementation-groups) insert - extract_pr_number between compose_pr and review_pr to capture the PR number for - merge queue support. - """ - recipe_name = recipe.name # type: ignore[attr-defined] - on_success = recipe.steps["compose_pr"].on_success # type: ignore[attr-defined] - assert on_success == "extract_pr_number", ( - f"{recipe_name}: compose_pr.on_success must be 'extract_pr_number', got {on_success!r}" - ) - - def test_review_pr_step_exists_and_is_run_skill(self, recipe: object) -> None: - """T_RP2: review_pr step exists and uses run_skill tool.""" - step = recipe.steps["review_pr"] # type: ignore[attr-defined] - assert step.tool == "run_skill" - - def test_review_pr_skipped_when_open_pr_false(self, recipe: object) -> None: - """T_RP3: review_pr is gated by inputs.open_pr (skip_when_false).""" - step = recipe.steps["review_pr"] # type: ignore[attr-defined] - assert step.skip_when_false == "inputs.open_pr" - - def test_review_pr_routes_to_ci_watch_on_success(self, recipe: object) -> None: - """T_RP4: review_pr has on_result with catch-all route to ci_watch.""" - step = recipe.steps["review_pr"] # type: ignore[attr-defined] - assert step.on_result is not None - default_conditions = [ - c for c in step.on_result.conditions if c.when is None or c.when == "true" - ] - assert any(c.route == "ci_watch" for c in default_conditions) - - def test_review_pr_captures_verdict(self, recipe: object) -> None: - """T_RP4b: review_pr captures the verdict output from the skill result.""" - step = recipe.steps["review_pr"] # type: ignore[attr-defined] - assert "verdict" in step.capture - assert step.capture["verdict"] == "${{ result.verdict }}" - - def test_review_pr_changes_requested_routes_to_resolve_review(self, recipe: object) -> None: - """T_RP4c: on_result routes changes_requested verdict to resolve_review.""" - step = recipe.steps["review_pr"] # type: ignore[attr-defined] - assert step.on_result is not None - changes_conditions = [ - c for c in step.on_result.conditions if c.when and "changes_requested" in c.when - ] - assert any(c.route == "resolve_review" for c in changes_conditions) - - def test_review_pr_routes_to_resolve_review_on_failure(self, recipe: object) -> None: - """T_RP5: review_pr.on_failure routes to resolve_review.""" - assert recipe.steps["review_pr"].on_failure == "resolve_review" # type: ignore[attr-defined] - - def test_resolve_review_has_retries(self, recipe: object) -> None: - """T_RP6: resolve_review has retries=2 matching resolve_ci pattern.""" - assert recipe.steps["resolve_review"].retries == 2 # type: ignore[attr-defined] - - def test_resolve_review_routes_to_re_push_review(self, recipe: object) -> None: - """T_RP7: resolve_review uses on_result: verdict dispatch routing to re_push_review.""" - step = recipe.steps["resolve_review"] # type: ignore[attr-defined] - assert step.on_success is None, ( - "resolve_review must use on_result: verdict dispatch, not unconditional on_success" - ) - assert step.on_result is not None, ( - "resolve_review must have on_result: block for verdict-gated routing" - ) - real_fix_routes = [ - c.route for c in step.on_result.conditions if c.when and "real_fix" in c.when - ] - assert any("re_push_review" in r for r in real_fix_routes), ( - "resolve_review on_result must route verdict=real_fix to re_push_review" - ) - - def test_re_push_review_routes_to_ci_watch(self, recipe: object) -> None: - """T_RP8: re_push_review routes to ci_watch (one-shot review gate).""" - assert recipe.steps["re_push_review"].on_success == "ci_watch" # type: ignore[attr-defined] - - def test_ci_watch_present(self, recipe: object) -> None: - """T_RP9: ci_watch step present in all four recipes.""" - assert "ci_watch" in recipe.steps # type: ignore[attr-defined] - - def test_review_pr_needs_human_has_explicit_route(self, recipe: object) -> None: - """needs_human must have a dedicated on_result route in every recipe.""" - review_pr_step = recipe.steps["review_pr"] # type: ignore[attr-defined] - explicit_conditions = [ - c.when - for c in review_pr_step.on_result.conditions - if c.when and "needs_human" in c.when and c.when.strip() != "true" - ] - assert len(explicit_conditions) >= 1, ( - "review_pr on_result must have an explicit condition for 'needs_human'. " - "It must not silently fall through the catch-all." - ) - - def test_resolve_review_step_uses_correct_skill(self, recipe: object) -> None: - """resolve_review step must invoke /autoskillit:resolve-review in all recipes.""" - resolve_step = recipe.steps["resolve_review"] # type: ignore[attr-defined] - skill_cmd = resolve_step.with_args.get("skill_command", "") - assert "resolve-review" in skill_cmd and "resolve-failures" not in skill_cmd, ( - "resolve_review step must call /autoskillit:resolve-review, " - f"not resolve-failures. Got: {skill_cmd}" - ) - - -def test_telemetry_before_open_pr_rule_not_in_registry() -> None: - """The telemetry-before-open-pr rule must not be in the rule registry. - - This rule was removed because open-pr now self-retrieves token telemetry - from disk using cwd_filter (Step 0b). If this test fails, the rule was - re-added to the registry and would silently fire on bundled production recipes. - """ - import autoskillit.recipe # noqa: F401 — triggers rule registration - from autoskillit.recipe.registry import _RULE_REGISTRY - - rule_names = {spec.name for spec in _RULE_REGISTRY} - assert "telemetry-before-open-pr" not in rule_names, ( - "telemetry-before-open-pr was re-added to the registry; " - "open-pr self-retrieves token telemetry via cwd_filter — " - "this rule is no longer needed and must not be registered" - ) - - -def test_bundled_recipes_pass_unrouted_verdict_value_rule() -> None: - """All bundled recipes must pass the unrouted-verdict-value semantic rule.""" - for yaml_path in sorted(builtin_recipes_dir().glob("*.yaml")): - recipe = load_recipe(yaml_path) - findings = run_semantic_rules(recipe) - verdict_errors = [f for f in findings if f.rule == "unrouted-verdict-value"] - assert len(verdict_errors) == 0, ( - f"Recipe '{yaml_path.stem}' has unrouted verdict values: " - + ", ".join(f.message for f in verdict_errors) - ) - - -def test_implementation_groups_has_ci_watch() -> None: - """T_RP10: implementation-groups now has ci_watch (parity with other recipes).""" - recipe = load_recipe(builtin_recipes_dir() / "implementation-groups.yaml") - assert "ci_watch" in recipe.steps - assert "resolve_ci" in recipe.steps - assert "re_push" in recipe.steps - - -# --------------------------------------------------------------------------- -# TestBaseBranchDefaults -# --------------------------------------------------------------------------- - - -class TestBaseBranchDefaults: - @pytest.mark.parametrize( - "recipe_name", - [ - "implementation", - "remediation", - "implementation-groups", - "merge-prs", - ], - ) - def test_recipe_base_branch_auto_detects(self, recipe_name: str) -> None: - """Non-exempt bundled recipes must use auto-detect for base_branch.""" - recipe = load_recipe(builtin_recipes_dir() / f"{recipe_name}.yaml") - assert recipe.ingredients["base_branch"].default == "", ( - f"{recipe_name}.yaml: base_branch must use auto-detect (default: '')" - ) - - def test_smoke_test_base_branch_uses_auto_detect(self) -> None: - """smoke-test.yaml must use auto-detect for base_branch so config-resolved value wins.""" - recipe = load_recipe(SMOKE_RECIPE) - assert recipe.ingredients["base_branch"].default == "", ( - "smoke-test.yaml base_branch must use auto-detect (default: '') so that " - "branching.default_base_branch from config is honoured at runtime (#703)" - ) - - -# --------------------------------------------------------------------------- -# TestImplementationRecipeMergeQueueRule -# --------------------------------------------------------------------------- - - -class TestImplementationRecipeMergeQueueRule: - """implementation.yaml kitchen_rules must reference merge queue detection.""" - - @pytest.fixture(scope="class") - def recipe(self): - return load_recipe(builtin_recipes_dir() / "implementation.yaml") - - def test_kitchen_rules_mention_check_repo_merge_state(self, recipe) -> None: - all_rules = " ".join(recipe.kitchen_rules) - assert "check_repo_merge_state" in all_rules, ( - "implementation.yaml kitchen_rules must reference check_repo_merge_state" - ) - assert "MERGE ROUTING" in all_rules, ( - "implementation.yaml kitchen_rules must contain a MERGE ROUTING rule" - ) - - def test_kitchen_rules_prohibit_direct_gh_pr_merge(self, recipe) -> None: - # Find the specific rule that mentions "gh pr merge" and check for - # prohibition language within that rule, not across all rules. - merge_rules = [r for r in recipe.kitchen_rules if "gh pr merge" in r] - assert merge_rules, ( - "implementation.yaml kitchen_rules must contain a rule mentioning 'gh pr merge'" - ) - has_prohibition = any( - any(phrase in rule.lower() for phrase in ("never", "prohibited", "do not")) - for rule in merge_rules - ) - assert has_prohibition, ( - "implementation.yaml kitchen_rules must explicitly prohibit calling " - "gh pr merge directly outside of recipe steps" - ) - - -# --------------------------------------------------------------------------- -# WF7: build_recipe_graph emits zero warnings for all bundled recipes -# --------------------------------------------------------------------------- - -_BUNDLED_RECIPE_PATHS = sorted(builtin_recipes_dir().glob("*.yaml")) - - -@pytest.mark.parametrize("recipe_path", _BUNDLED_RECIPE_PATHS, ids=lambda p: p.stem) -def test_bundled_recipes_emit_no_graph_warnings(recipe_path): - """WF7: build_recipe_graph emits zero warnings for all bundled recipes.""" - recipe = load_recipe(recipe_path) - with structlog.testing.capture_logs() as cap_logs: - build_recipe_graph(recipe) - warning_events = [entry for entry in cap_logs if entry.get("log_level") == "warning"] - assert warning_events == [], ( - f"build_recipe_graph emitted {len(warning_events)} warnings for " - f"{recipe_path.name}: {warning_events}" - ) - - -@pytest.mark.parametrize("recipe_path", _BUNDLED_RECIPE_PATHS, ids=lambda p: p.stem) -def test_all_advisory_run_skill_steps_have_on_context_limit(recipe_path): - """ - Every run_skill step with skip_when_false must declare on_context_limit. - A step that can be skipped by configuration must also be skippable on context limit. - """ - recipe = load_recipe(recipe_path) - violations = [ - name - for name, step in recipe.steps.items() - if step.tool in SKILL_TOOLS - and step.skip_when_false is not None - and step.on_context_limit is None - ] - assert violations == [], ( - f"Advisory run_skill steps in {recipe_path.name} missing on_context_limit: " - f"{violations}. Set on_context_limit to the appropriate skip/recovery step." - ) - - -# --------------------------------------------------------------------------- -# Cross-recipe review-pr pre-computation tests -# --------------------------------------------------------------------------- - - -@pytest.mark.parametrize( - "recipe_name", - ["implementation", "remediation", "implementation-groups", "merge-prs"], -) -def test_review_pr_step_passes_annotated_diff_inputs(recipe_name: str) -> None: - """Every review-pr invocation must pass annotated_diff_path= and hunk_ranges_path= - in skill_command, or have a reachable annotate_pr_diff predecessor step.""" - recipe = load_recipe(builtin_recipes_dir() / f"{recipe_name}.yaml") - # Find review-pr steps - review_steps = [ - (name, step) - for name, step in recipe.steps.items() - if step.tool in SKILL_TOOLS and "review-pr" in step.with_args.get("skill_command", "") - ] - assert review_steps, f"No review-pr step found in {recipe_name}.yaml" - for step_name, step in review_steps: - cmd = step.with_args.get("skill_command", "") - has_inline = "annotated_diff_path=" in cmd and "hunk_ranges_path=" in cmd - has_predecessor = any( - s.with_args.get("callable", "") == "autoskillit.smoke_utils.annotate_pr_diff" - for s in recipe.steps.values() - if s.tool == "run_python" - ) - assert has_inline or has_predecessor, ( - f"{recipe_name}.yaml: step '{step_name}' invokes review-pr but neither " - f"passes annotated_diff_path=/hunk_ranges_path= inline nor has an " - f"annotate_pr_diff predecessor step" - ) - - -@pytest.mark.parametrize( - "recipe_name", - ["implementation", "remediation", "implementation-groups", "merge-prs"], -) -def test_annotate_pr_diff_captures_both_paths(recipe_name: str) -> None: - """The annotate_pr_diff step must capture annotated_diff_path and hunk_ranges_path.""" - recipe = load_recipe(builtin_recipes_dir() / f"{recipe_name}.yaml") - annotate_steps = [ - (name, step) - for name, step in recipe.steps.items() - if step.tool == "run_python" - and step.with_args.get("callable", "") == "autoskillit.smoke_utils.annotate_pr_diff" - ] - assert annotate_steps, f"No annotate_pr_diff step found in {recipe_name}.yaml" - for step_name, step in annotate_steps: - assert step.capture is not None, ( - f"{recipe_name}.yaml: annotate_pr_diff step '{step_name}' has no capture block" - ) - assert "annotated_diff_path" in step.capture, ( - f"{recipe_name}.yaml: annotate_pr_diff step '{step_name}' must capture " - f"annotated_diff_path" - ) - assert "hunk_ranges_path" in step.capture, ( - f"{recipe_name}.yaml: annotate_pr_diff step '{step_name}' must capture " - f"hunk_ranges_path" - ) - - -# --------------------------------------------------------------------------- -# TestRunModeIngredient -# --------------------------------------------------------------------------- - - -class TestRunModeIngredient: - """REQ-INGREDIENT-001 through REQ-INGREDIENT-005: run_mode ingredient in multi-issue recipes.""" # noqa: E501 - - @pytest.fixture(scope="class") - def impl_recipe(self): - return load_recipe(builtin_recipes_dir() / "implementation.yaml") - - @pytest.fixture(scope="class") - def remed_recipe(self): - return load_recipe(builtin_recipes_dir() / "remediation.yaml") - - def test_implementation_has_run_mode_ingredient(self, impl_recipe) -> None: - """REQ-INGREDIENT-001: implementation.yaml declares run_mode ingredient.""" - assert "run_mode" in impl_recipe.ingredients, ( - "implementation.yaml must declare run_mode ingredient" - ) - - def test_implementation_run_mode_default_is_sequential(self, impl_recipe) -> None: - """REQ-INGREDIENT-002: run_mode defaults to 'sequential'.""" - ing = impl_recipe.ingredients["run_mode"] - assert ing.default == "sequential", ( - "implementation.yaml run_mode must default to 'sequential'" - ) - - def test_implementation_run_mode_description_mentions_parallel(self, impl_recipe) -> None: - """REQ-INGREDIENT-001: description must document 'parallel' as a valid option.""" - ing = impl_recipe.ingredients["run_mode"] - assert "parallel" in ing.description.lower(), ( - "run_mode description must mention 'parallel' as an option" - ) - - def test_remediation_has_run_mode_ingredient(self, remed_recipe) -> None: - """REQ-INGREDIENT-001: remediation.yaml declares run_mode ingredient.""" - assert "run_mode" in remed_recipe.ingredients, ( - "remediation.yaml must declare run_mode ingredient" - ) - - def test_remediation_run_mode_default_is_sequential(self, remed_recipe) -> None: - """REQ-INGREDIENT-002: run_mode defaults to 'sequential'.""" - ing = remed_recipe.ingredients["run_mode"] - assert ing.default == "sequential", ( - "remediation.yaml run_mode must default to 'sequential'" - ) - - def test_remediation_run_mode_description_mentions_parallel(self, remed_recipe) -> None: - """REQ-INGREDIENT-001: description must document 'parallel' as a valid option.""" - ing = remed_recipe.ingredients["run_mode"] - assert "parallel" in ing.description.lower(), ( - "run_mode description must mention 'parallel' as an option" - ) - - -def test_no_bare_temp_paths_in_bundled_recipe_notes() -> None: - """No bundled recipe YAML should reference temp/ without .autoskillit/ prefix. - - Bare temp/ references are incorrect; all project-local temp output must be - rooted under .autoskillit/temp/ per CLAUDE.md §3.2. - """ - import re - - recipes_dir = builtin_recipes_dir() - bare_temp = re.compile(r"(? None: - """Recipes must not declare 'defer_cleanup' — that design is removed.""" - recipe = load_recipe(_resolve_recipe_path(recipe_name)) - assert "defer_cleanup" not in recipe.ingredients, ( - f"{recipe_name}.yaml must not declare 'defer_cleanup'" - ) - - -@pytest.mark.parametrize( - "recipe_name", - ["implementation", "remediation", "implementation-groups", "merge-prs", "smoke-test"], -) -def test_recipe_has_no_registry_path_ingredient(recipe_name: str) -> None: - """Recipes must not declare 'registry_path' — replaced by a well-known default.""" - recipe = load_recipe(_resolve_recipe_path(recipe_name)) - assert "registry_path" not in recipe.ingredients, ( - f"{recipe_name}.yaml must not declare 'registry_path'" - ) - - -@pytest.mark.parametrize( - "recipe_name", - ["implementation", "remediation", "implementation-groups", "merge-prs", "smoke-test"], -) -def test_recipe_has_no_interactive_cleanup_steps(recipe_name: str) -> None: - """Recipes must not have confirm_cleanup or delete_clone — these blocked unattended runs.""" - recipe = load_recipe(_resolve_recipe_path(recipe_name)) - assert "confirm_cleanup" not in recipe.steps, ( - f"{recipe_name}.yaml must not have 'confirm_cleanup' step" - ) - assert "delete_clone" not in recipe.steps, ( - f"{recipe_name}.yaml must not have 'delete_clone' step" - ) - - -@pytest.mark.parametrize( - "recipe_name", ["implementation", "remediation", "implementation-groups", "merge-prs"] -) -def test_recipe_has_unconditional_register_steps(recipe_name: str) -> None: - """register_clone_success routes to done; register_clone_failure routes to escalate_stop.""" - recipe = load_recipe(builtin_recipes_dir() / f"{recipe_name}.yaml") - assert "register_clone_success" in recipe.steps - assert "register_clone_failure" in recipe.steps - s = recipe.steps["register_clone_success"] - f = recipe.steps["register_clone_failure"] - assert s.on_success == "done" - assert f.on_success == "escalate_stop" - assert f.on_failure == "escalate_stop" - assert "check_defer_cleanup" not in recipe.steps - assert "check_defer_on_failure" not in recipe.steps - - -@pytest.mark.parametrize( - "recipe_name", - ["implementation.yaml", "implementation-groups.yaml", "remediation.yaml"], -) -def test_re_push_steps_have_force_true(recipe_name: str) -> None: - """All re_push/* steps must have force='true'. - - Post-rebase push requires --force-with-lease. - """ - recipe = load_recipe(builtin_recipes_dir() / recipe_name) - for step_name in ( - "re_push", - "re_push_queue_fix", - "re_push_direct_fix", - "re_push_immediate_fix", - ): - assert step_name in recipe.steps, f"Expected step {step_name!r} in {recipe_name}" - step = recipe.steps[step_name] - assert step.tool == "push_to_remote" - assert step.with_args.get("force") == "true", ( - f"{step_name} in {recipe_name} must include force='true' — " - "post-rebase push requires --force-with-lease" - ) - - -class TestResearchRecipeStructure: - @pytest.fixture - def recipe(self): - return load_recipe(builtin_recipes_dir() / "research.yaml") - - def test_research_has_review_pr_ingredient(self, recipe) -> None: - """research.yaml must declare a review_pr ingredient with default 'false'.""" - assert "review_pr" in recipe.ingredients, ( - "research.yaml must declare a 'review_pr' ingredient to gate the optional " - "review-research-pr step" - ) - assert recipe.ingredients["review_pr"].default == "false" - - def test_research_has_review_research_pr_step(self, recipe) -> None: - """research.yaml must include a review_research_pr step.""" - assert "review_research_pr" in recipe.steps - - def test_research_review_step_skip_when_false(self, recipe) -> None: - """review_research_pr step must use skip_when_false: inputs.review_pr.""" - step = recipe.steps["review_research_pr"] - assert step.skip_when_false == "inputs.review_pr" - - def test_research_no_issue_number_ingredient(self, recipe) -> None: - """Removed: issue_number is the phase-2 gate.""" - assert "issue_number" not in recipe.ingredients - - def test_research_no_setup_phases_ingredient(self, recipe) -> None: - """Removed: setup_phases toggle replaced by always-decompose.""" - assert "setup_phases" not in recipe.ingredients - - def test_research_no_check_phase_step(self, recipe) -> None: - assert "check_phase" not in recipe.steps - - def test_research_no_phase1_done_step(self, recipe) -> None: - assert "phase1_done" not in recipe.steps - - def test_research_no_open_plan_issue_step(self, recipe) -> None: - assert "open_plan_issue" not in recipe.steps - - def test_research_no_save_experiment_plan_step(self, recipe) -> None: - assert "save_experiment_plan" not in recipe.steps - - def test_research_no_check_setup_needed_step(self, recipe) -> None: - assert "check_setup_needed" not in recipe.steps - - def test_research_no_implement_experiment_step(self, recipe) -> None: - assert "implement_experiment" not in recipe.steps - - def test_research_has_issue_url_ingredient(self, recipe) -> None: - assert "issue_url" in recipe.ingredients - assert not recipe.ingredients["issue_url"].required - - def test_research_has_review_design_ingredient(self, recipe) -> None: - assert "review_design" in recipe.ingredients - assert recipe.ingredients["review_design"].default == "true" - - def test_research_has_review_design_step(self, recipe) -> None: - assert "review_design" in recipe.steps - - def test_review_design_step_skip_when_false(self, recipe) -> None: - step = recipe.steps["review_design"] - assert step.skip_when_false == "inputs.review_design" - - def test_review_design_step_retries_and_exhausted(self, recipe) -> None: - step = recipe.steps["review_design"] - assert step.retries == 2 - assert step.on_exhausted == "create_worktree" - - def test_plan_experiment_routes_to_review_design(self, recipe) -> None: - step = recipe.steps["plan_experiment"] - assert step.on_success == "review_design" - - def test_review_design_on_result_routing(self, recipe) -> None: - """review_design STOP verdict routes to resolve_design_review (not design_rejected).""" - step = recipe.steps["review_design"] - assert step.on_result is not None - routes = {c.when: c.route for c in step.on_result.conditions if c.when} - assert any("GO" in (w or "") for w in routes), "Missing GO route" - go_route = next(c.route for c in step.on_result.conditions if c.when and "GO" in c.when) - assert go_route == "plan_visualization" - revise_route = next( - c.route for c in step.on_result.conditions if c.when and "REVISE" in c.when - ) - assert revise_route == "revise_design" - stop_route = next( - c.route for c in step.on_result.conditions if c.when and "STOP" in c.when - ) - assert stop_route == "resolve_design_review" - - def test_has_revise_design_step(self, recipe) -> None: - assert "revise_design" in recipe.steps - - def test_has_design_rejected_step(self, recipe) -> None: - assert "design_rejected" in recipe.steps - - def test_design_rejected_step_is_action_stop(self, recipe) -> None: - """design_rejected must halt the pipeline (action=stop) with an explanatory message. - - Regression guard: changing action to 'route' would silently break - the hard-halt guarantee for fundamentally flawed designs. - """ - step = recipe.steps["design_rejected"] - assert step.action == "stop" - assert step.message, "design_rejected must have a non-empty message" - assert "STOP" in step.message, ( - "design_rejected message must reference the STOP verdict for clarity." - ) - - def test_kitchen_rule_6_acknowledges_stop_verdict_exception(self, recipe) -> None: - """Kitchen rule #6 must scope 'do not stop' to exhaustion/failure, not STOP verdict. - - The current blanket 'do not stop' language contradicts the design_rejected - hard halt. The rule must explicitly note that verdict=STOP is the sole exception. - """ - rule6 = recipe.kitchen_rules[5] # 0-indexed; rule #6 is index 5 - rule6_lower = rule6.lower() - assert "stop" in rule6_lower, ( - "Kitchen rule #6 must reference 'stop' in the verdict exception context." - ) - assert "verdict" in rule6_lower or "exception" in rule6_lower, ( - "Kitchen rule #6 must acknowledge that verdict=STOP is an exception to " - "'do not stop on exhaustion/failure'. Current language is overbroad and " - "contradicts the design_rejected routing." - ) - - def test_has_resolve_design_review_step(self, recipe) -> None: - """resolve_design_review step must be present in research.yaml.""" - assert "resolve_design_review" in recipe.steps - - def test_resolve_design_review_routes_revised_to_revise_design(self, recipe) -> None: - """resolve_design_review routes resolution=revised back to revise_design.""" - step = recipe.steps["resolve_design_review"] - assert step.on_result is not None - revised_route = next( - c.route for c in step.on_result.conditions if c.when and "revised" in c.when - ) - assert revised_route == "revise_design" - - def test_resolve_design_review_routes_failed_to_design_rejected(self, recipe) -> None: - """resolve_design_review routes resolution=failed to design_rejected.""" - step = recipe.steps["resolve_design_review"] - assert step.on_result is not None - failed_route = next( - c.route for c in step.on_result.conditions if c.when and "failed" in c.when - ) - assert failed_route == "design_rejected" - - def test_resolve_design_review_fallbacks_to_design_rejected(self, recipe) -> None: - """resolve_design_review routes on_failure and on_context_limit to design_rejected.""" - step = recipe.steps["resolve_design_review"] - assert step.on_failure == "design_rejected" - assert step.on_context_limit == "design_rejected" - - def test_resolve_design_review_captures_revision_guidance(self, recipe) -> None: - """resolve_design_review must capture revision_guidance for the revise_design loop.""" - step = recipe.steps["resolve_design_review"] - assert "revision_guidance" in step.capture, ( - "resolve_design_review must capture revision_guidance so revise_design → " - "plan_experiment picks up the triage-generated guidance" - ) - - def test_has_resolve_research_review_step(self, recipe) -> None: - step = recipe.steps["resolve_research_review"] - assert step.retries == 2 - assert step.on_exhausted == "route_claims_resolve" - assert step.on_success == "route_claims_resolve" - assert step.on_failure == "route_claims_resolve" - - def test_has_re_push_research_step(self, recipe) -> None: - assert "re_push_research" in recipe.steps - - def test_research_validates_cleanly(self, recipe) -> None: - """validate_recipe returns no errors on the simplified recipe.""" - from autoskillit.recipe.validator import validate_recipe - - errors = validate_recipe(recipe) - assert errors == [], f"Validation errors: {errors}" - - def test_requires_packs_includes_exp_lens(self, recipe) -> None: - assert "exp-lens" in recipe.requires_packs - assert "research" in recipe.requires_packs - - def test_re_run_experiment_step(self, recipe) -> None: - assert "re_run_experiment" in recipe.steps - step = recipe.steps["re_run_experiment"] - assert step.tool == "run_skill" - assert "--adjust" in step.with_args.get("skill_command", "") - assert step.on_success == "re_generate_report" - - def test_re_generate_report_step(self, recipe) -> None: - assert "re_generate_report" in recipe.steps - step = recipe.steps["re_generate_report"] - assert step.tool == "run_skill" - assert step.on_success == "re_stage_bundle" - - def test_re_test_step(self, recipe) -> None: - assert "re_test" in recipe.steps - step = recipe.steps["re_test"] - assert step.tool == "test_check" - assert step.on_success == "re_push_research" - - def test_revalidation_loop_all_paths_reach_begin_archival(self, recipe) -> None: - """Every path from merge_escalations reaches begin_archival.""" - for step_name in ("re_run_experiment", "re_generate_report", "re_test"): - step = recipe.steps[step_name] - assert step.on_failure in ("begin_archival", "re_push_research") - assert recipe.steps["re_push_research"].on_success == "finalize_bundle" - - def test_audit_claims_ingredient_default_false(self, recipe) -> None: - assert "audit_claims" in recipe.ingredients - assert recipe.ingredients["audit_claims"].required is False - assert recipe.ingredients["audit_claims"].default == "false" - - def test_review_research_pr_captures_review_verdict(self, recipe) -> None: - step = recipe.steps["review_research_pr"] - assert "review_verdict" in step.capture - assert "verdict" not in step.capture # old key must be gone - - def test_audit_claims_step_routes_to_route_review_resolve(self, recipe) -> None: - step = recipe.steps["audit_claims"] - assert step.tool == "run_skill" - assert "audit_claims" in step.skip_when_false - # all on_result routes point to route_review_resolve - routes = {c.route for c in step.on_result.conditions} - assert routes == {"route_review_resolve"} - - def test_new_routing_steps_exist(self, recipe) -> None: - assert "route_review_resolve" in recipe.steps - assert "route_claims_resolve" in recipe.steps - assert "merge_escalations" in recipe.steps - assert "check_escalations" not in recipe.steps # replaced - - def test_route_review_resolve_routing_logic(self, recipe) -> None: - step = recipe.steps["route_review_resolve"] - assert step.action == "route" - conditions = step.on_result.conditions - # changes_requested → resolve_research_review; else → route_claims_resolve - guarded = [c for c in conditions if c.when is not None] - default = [c for c in conditions if c.when is None] - assert len(guarded) == 1 - assert "changes_requested" in guarded[0].when - assert guarded[0].route == "resolve_research_review" - assert len(default) == 1 - assert default[0].route == "route_claims_resolve" - - def test_route_claims_resolve_routing_logic(self, recipe) -> None: - step = recipe.steps["route_claims_resolve"] - assert step.action == "route" - conditions = step.on_result.conditions - # changes_requested → resolve_claims_review; else → merge_escalations - guarded = [c for c in conditions if c.when is not None] - default = [c for c in conditions if c.when is None] - assert len(guarded) == 1 - assert "changes_requested" in guarded[0].when - assert guarded[0].route == "resolve_claims_review" - assert len(default) == 1 - assert default[0].route == "merge_escalations" - - def test_merge_escalations_routing_logic(self, recipe) -> None: - step = recipe.steps["merge_escalations"] - assert step.action == "route" - conditions = step.on_result.conditions - # review_needs_rerun == true → re_run_experiment - # claims_needs_rerun == true → re_run_experiment - # else → re_push_research - guarded = [c for c in conditions if c.when is not None] - default = [c for c in conditions if c.when is None] - assert len(guarded) == 2 - rerun_routes = {c.route for c in guarded} - assert rerun_routes == {"re_run_experiment"} - assert any("review_needs_rerun" in c.when for c in guarded) - assert any("claims_needs_rerun" in c.when for c in guarded) - assert len(default) == 1 - assert default[0].route == "re_push_research" - - def test_resolve_claims_review_step_exists(self, recipe) -> None: - step = recipe.steps["resolve_claims_review"] - assert "claims_needs_rerun" in step.capture - - def test_resolve_research_review_captures_review_needs_rerun(self, recipe) -> None: - step = recipe.steps["resolve_research_review"] - assert "review_needs_rerun" in step.capture - assert "needs_rerun" not in step.capture # old key must be gone - - def test_research_recipe_validates_cleanly_with_new_steps(self, recipe) -> None: - from autoskillit.recipe.validator import validate_recipe - - errors = validate_recipe(recipe) - assert errors == [], f"Validation errors: {errors}" - - # --- Archival phase tests --- - - def test_archival_begin_archival_step_exists(self, recipe) -> None: - """begin_archival must be a route step gating archival on pr_url.""" - assert "begin_archival" in recipe.steps - step = recipe.steps["begin_archival"] - assert step.action == "route" - - def test_archival_begin_archival_routes_to_capture(self, recipe) -> None: - """begin_archival routes to capture_experiment_branch when pr_url is truthy.""" - step = recipe.steps["begin_archival"] - assert step.on_result is not None - conditions = step.on_result.conditions - truthy_route = next((c for c in conditions if c.when and "pr_url" in c.when), None) - assert truthy_route is not None - assert truthy_route.route == "capture_experiment_branch" - - def test_archival_begin_archival_default_to_complete(self, recipe) -> None: - """begin_archival default route goes to research_complete.""" - step = recipe.steps["begin_archival"] - conditions = step.on_result.conditions - default = next((c for c in conditions if not c.when), None) - assert default is not None - assert default.route == "research_complete" - - def test_archival_capture_experiment_branch_step(self, recipe) -> None: - """capture_experiment_branch captures experiment_branch from git rev-parse.""" - assert "capture_experiment_branch" in recipe.steps - step = recipe.steps["capture_experiment_branch"] - assert step.tool == "run_cmd" - assert "experiment_branch" in step.capture - assert step.on_success == "create_artifact_branch" - assert step.on_failure == "research_complete" - - def test_archival_create_artifact_branch_step(self, recipe) -> None: - """create_artifact_branch creates temp worktree and captures artifact_branch.""" - assert "create_artifact_branch" in recipe.steps - step = recipe.steps["create_artifact_branch"] - assert step.tool == "run_cmd" - assert "artifact_branch" in step.capture - assert step.on_success == "open_artifact_pr" - assert step.on_failure == "research_complete" - - def test_archival_create_artifact_branch_uses_research_checkout(self, recipe) -> None: - """create_artifact_branch must use git checkout -- research/ pattern.""" - step = recipe.steps["create_artifact_branch"] - cmd = step.with_args["cmd"] - assert "research/" in cmd, "Must checkout research/ directory from experiment branch" - assert "worktree add" in cmd, "Must create a temporary worktree" - - def test_archival_open_artifact_pr_step(self, recipe) -> None: - """open_artifact_pr creates the artifact-only PR and captures artifact_pr_url.""" - assert "open_artifact_pr" in recipe.steps - step = recipe.steps["open_artifact_pr"] - assert step.tool == "run_cmd" - assert "artifact_pr_url" in step.capture - assert step.on_success == "tag_experiment_branch" - assert step.on_failure == "research_complete" - - def test_archival_tag_experiment_branch_step(self, recipe) -> None: - """tag_experiment_branch creates annotated archive tag and captures archive_tag.""" - assert "tag_experiment_branch" in recipe.steps - step = recipe.steps["tag_experiment_branch"] - assert step.tool == "run_cmd" - assert "archive_tag" in step.capture - assert step.on_success == "close_experiment_pr" - assert step.on_failure == "research_complete" - - def test_archival_tag_uses_archive_prefix(self, recipe) -> None: - """Tag name must use archive/research/ prefix convention.""" - step = recipe.steps["tag_experiment_branch"] - cmd = step.with_args["cmd"] - assert "archive/research/" in cmd - - def test_archival_close_experiment_pr_step(self, recipe) -> None: - """close_experiment_pr closes the original PR and routes to research_complete.""" - assert "close_experiment_pr" in recipe.steps - step = recipe.steps["close_experiment_pr"] - assert step.tool == "run_cmd" - assert step.on_success == "research_complete" - assert step.on_failure == "research_complete" - - def test_archival_close_pr_references_artifact_and_tag(self, recipe) -> None: - """close_experiment_pr comment must reference both artifact PR and archive tag.""" - step = recipe.steps["close_experiment_pr"] - cmd = step.with_args["cmd"] - assert "artifact_pr_url" in cmd, "Must reference artifact PR URL" - assert "archive_tag" in cmd, "Must reference archive tag" - - def test_archival_graceful_degradation(self, recipe) -> None: - """Every archival step must route on_failure to research_complete.""" - archival_steps = [ - "capture_experiment_branch", - "create_artifact_branch", - "open_artifact_pr", - "tag_experiment_branch", - "close_experiment_pr", - ] - for name in archival_steps: - step = recipe.steps[name] - assert step.on_failure == "research_complete", ( - f"{name}.on_failure must be research_complete for graceful degradation" - ) - - def test_re_push_research_routes_to_finalize_bundle(self, recipe) -> None: - """re_push_research routes to finalize_bundle on success, begin_archival on failure.""" - step = recipe.steps["re_push_research"] - assert step.on_success == "finalize_bundle" - assert step.on_failure == "begin_archival" - - def test_finalize_bundle_routes_to_finalize_bundle_render(self, recipe) -> None: - """finalize_bundle on_success routes to finalize_bundle_render (not begin_archival).""" - step = recipe.steps["finalize_bundle"] - assert step.on_success == "finalize_bundle_render" - assert step.on_failure == "begin_archival" - - def test_finalize_bundle_render_step_exists_and_routes(self, recipe) -> None: - """finalize_bundle_render routes to route_archive_or_export on both outcomes.""" - assert "finalize_bundle_render" in recipe.steps - step = recipe.steps["finalize_bundle_render"] - assert step.on_success == "route_archive_or_export" - assert step.on_failure == "route_archive_or_export" - - def test_create_worktree_copies_review_cycle_artifacts(self, recipe) -> None: - """create_worktree must copy review-design dashboards and revision guidance.""" - step = recipe.steps["create_worktree"] - cmd = step.with_args["cmd"] - assert "review-cycles" in cmd, ( - "create_worktree must create artifacts/review-cycles/ subdirectory" - ) - assert "evaluation_dashboard" in cmd and "review-cycles" in cmd, ( - "create_worktree must copy evaluation dashboards to review-cycles/" - ) - assert "revision_guidance" in cmd and "review-cycles" in cmd, ( - "create_worktree must copy revision guidance to review-cycles/" - ) - - def test_create_worktree_copies_plan_version_artifacts(self, recipe) -> None: - """create_worktree must copy intermediate plan versions.""" - step = recipe.steps["create_worktree"] - cmd = step.with_args["cmd"] - assert "plan-versions" in cmd, ( - "create_worktree must create artifacts/plan-versions/ subdirectory" - ) - assert "experiment_plan" in cmd and "plan-versions" in cmd, ( - "create_worktree must copy plan versions to plan-versions/" - ) - - def test_stage_bundle_step_exists(self, recipe) -> None: - """A stage_bundle step must exist to organize phase artifacts.""" - assert "stage_bundle" in recipe.steps, ( - "research.yaml must have a stage_bundle step for phase artifact staging" - ) - step = recipe.steps["stage_bundle"] - cmd = step.with_args["cmd"] - assert "phase-groups" in cmd, "Must copy make-groups output" - assert "phase-plans" in cmd, "Must copy make-plan output" - - def test_test_routes_to_push_branch(self, recipe) -> None: - """test step must route to push_branch now that commit_research_artifacts is removed.""" - step = recipe.steps["test"] - assert step.on_success == "push_branch", ( - "test.on_success must be push_branch after commit_research_artifacts is removed" - ) - - def test_retest_routes_to_push_branch(self, recipe) -> None: - """retest step must route to push_branch now that commit_research_artifacts is removed.""" - step = recipe.steps["retest"] - assert step.on_success == "push_branch", ( - "retest.on_success must be push_branch after commit_research_artifacts is removed" - ) - - def test_stage_bundle_routes_to_route_pr_or_local(self, recipe) -> None: - """stage_bundle must route to route_pr_or_local on success and failure.""" - step = recipe.steps["stage_bundle"] - assert step.on_success == "route_pr_or_local" - assert step.on_failure == "route_pr_or_local" - - -# --------------------------------------------------------------------------- -# Cross-recipe CI parameterization guard -# --------------------------------------------------------------------------- - - -def test_bundled_recipes_have_no_ci_hardcoded_workflow() -> None: - """No bundled recipe should hardcode workflow in wait_for_ci steps.""" - for yaml_path in sorted(builtin_recipes_dir().glob("*.yaml")): - recipe = load_recipe(yaml_path) - findings = run_semantic_rules(recipe) - wf_findings = [f for f in findings if f.rule == "ci-hardcoded-workflow"] - assert wf_findings == [], f"{recipe.name} has hardcoded workflow: {wf_findings}" - - -# --------------------------------------------------------------------------- -# Commit guard and context-limit completeness guards -# --------------------------------------------------------------------------- - - -def test_merge_worktree_has_commit_guard_predecessor() -> None: - """Every merge_worktree step in bundled recipes must have a commit_guard predecessor. - - A commit_guard step auto-commits any dirty files before the merge_worktree - dirty-tree gate runs, providing structural immunity to context-exhausted skills - that leave edits on disk without committing them. - """ - from autoskillit.recipe.validator import make_validation_context - - for yaml_path in sorted(builtin_recipes_dir().glob("*.yaml")): - recipe = load_recipe(yaml_path) - merge_steps = [n for n, s in recipe.steps.items() if s.tool == "merge_worktree"] - if not merge_steps: - continue - ctx = make_validation_context(recipe) - for step_name in merge_steps: - preds = ctx.predecessors.get(step_name, set()) - has_guard = any(_is_commit_guard(p, ctx) for p in preds) - assert has_guard, ( - f"{yaml_path.name}: merge_worktree step '{step_name}' has no commit_guard " - f"predecessor. Add a commit_guard run_cmd step immediately before merge. " - f"Predecessors: {sorted(preds)}" - ) - - -def test_resolve_failures_steps_have_on_context_limit() -> None: - """Every step invoking resolve-failures must declare on_context_limit. - - Without on_context_limit, context exhaustion mid-fix falls through to - on_failure, discarding all uncommitted edits and losing partial progress. - """ - from autoskillit.recipe.contracts import resolve_skill_name - - for yaml_path in sorted(builtin_recipes_dir().glob("*.yaml")): - recipe = load_recipe(yaml_path) - for step_name, step in recipe.steps.items(): - if step.tool not in SKILL_TOOLS: - continue - skill_cmd = step.with_args.get("skill_command", "") - skill = resolve_skill_name(skill_cmd) - if skill != "resolve-failures": - continue - assert step.on_context_limit is not None, ( - f"{yaml_path.name}: step '{step_name}' invokes resolve-failures " - f"but has no on_context_limit. Context exhaustion will strand uncommitted " - f"edits on disk and fall through to on_failure, losing progress." - ) - - -# --------------------------------------------------------------------------- -# Cross-recipe cwd-after-clone guard -# --------------------------------------------------------------------------- - - -@pytest.mark.parametrize("recipe_path", _ALL_CLONE_RECIPE_PATHS, ids=lambda p: p.stem) -def test_all_clone_recipes_use_context_cwd_after_clone(recipe_path: Path) -> None: - """After clone_repo, no step should use inputs.* as cwd.""" - import re - - input_re = re.compile(r"\$\{\{\s*inputs\.\w+\s*\}\}") - recipe = load_recipe(recipe_path) - - seen_clone = False - for name, step in recipe.steps.items(): - if step.tool == "clone_repo": - seen_clone = True - continue - if seen_clone and step.with_args: - cwd = step.with_args.get("cwd", "") - assert not input_re.search(cwd), ( - f"{recipe_path.stem}: step '{name}' uses inputs.* as cwd after clone: {cwd}" - ) diff --git a/tests/recipe/test_bundled_recipes_general.py b/tests/recipe/test_bundled_recipes_general.py new file mode 100644 index 0000000000..5d7eccd04b --- /dev/null +++ b/tests/recipe/test_bundled_recipes_general.py @@ -0,0 +1,714 @@ +"""General structural invariants for all bundled recipe YAML files.""" + +from __future__ import annotations + +from pathlib import Path + +import pytest +import structlog.testing + +from autoskillit.core import PACK_REGISTRY, SKILL_TOOLS +from autoskillit.recipe._analysis import build_recipe_graph +from autoskillit.recipe._skill_helpers import _get_skill_category_map +from autoskillit.recipe.contracts import load_bundled_manifest, resolve_skill_name +from autoskillit.recipe.io import builtin_recipes_dir, load_recipe +from autoskillit.recipe.rules.rules_merge import _is_commit_guard +from autoskillit.recipe.validator import run_semantic_rules + +pytestmark = [pytest.mark.layer("recipe"), pytest.mark.small] + +PROJECT_ROOT = Path(__file__).resolve().parent.parent.parent +SMOKE_RECIPE = PROJECT_ROOT / ".autoskillit" / "recipes" / "smoke-test.yaml" + + +def _resolve_recipe_path(name: str) -> Path: + """Resolve recipe name to path, handling project-local smoke-test.""" + if name == "smoke-test": + return SMOKE_RECIPE + return builtin_recipes_dir() / f"{name}.yaml" + + +_ALL_CLONE_RECIPE_PATHS: list[Path] = [] +for _dir in (builtin_recipes_dir(), PROJECT_ROOT / ".autoskillit" / "recipes"): + if _dir.is_dir(): + for _p in sorted(_dir.glob("*.yaml")): + _wf = load_recipe(_p) + if any(s.tool == "clone_repo" for s in _wf.steps.values()): + _ALL_CLONE_RECIPE_PATHS.append(_p) + +_CI_WATCH_CYCLE_STEPS = {"ci_watch", "handle_no_ci_runs", "check_ci_loop"} + + +def test_every_bundled_recipe_declares_requires_packs() -> None: + """All top-level bundled recipes must declare a non-empty requires_packs.""" + for path in sorted(builtin_recipes_dir().glob("*.yaml")): + recipe = load_recipe(path) + assert recipe.requires_packs, f"{path.name} does not declare requires_packs" + + +@pytest.mark.parametrize("recipe_name", ["remediation", "implementation", "implementation-groups"]) +def test_bundled_recipe_no_unbounded_cycle_findings(recipe_name: str) -> None: + """Pipeline recipes must have zero unbounded-cycle findings for the ci_watch cycle.""" + recipe = load_recipe(builtin_recipes_dir() / f"{recipe_name}.yaml") + findings = run_semantic_rules(recipe) + ci_watch_cycle_findings = [ + f for f in findings if f.rule == "unbounded-cycle" and f.step_name in _CI_WATCH_CYCLE_STEPS + ] + assert ci_watch_cycle_findings == [], ( + f"{recipe_name} has unbounded-cycle findings for ci_watch cycle: " + + "; ".join(f.message for f in ci_watch_cycle_findings) + ) + + +def test_make_plan_contract_declares_plan_parts_output() -> None: + """D4: make-plan contract must declare plan_parts as an output.""" + manifest = load_bundled_manifest() + make_plan = manifest.get("skills", {}).get("make-plan", {}) + output_names = [o["name"] for o in make_plan.get("outputs", [])] + assert "plan_parts" in output_names, ( + "make-plan contract must declare plan_parts as an output " + "so capture_list coverage validation can enforce it" + ) + + +def test_rectify_contract_declares_plan_parts_output() -> None: + """D5: rectify contract must declare plan_parts as an output.""" + manifest = load_bundled_manifest() + rectify = manifest.get("skills", {}).get("rectify", {}) + output_names = [o["name"] for o in rectify.get("outputs", [])] + assert "plan_parts" in output_names + + +def test_bundled_recipes_diagrams_dir_exists() -> None: + """Diagrams directory exists for bundled recipes.""" + from autoskillit.core.paths import pkg_root + + assert (pkg_root() / "recipes" / "diagrams").is_dir() + + +def test_all_predicate_steps_have_on_failure() -> None: + """Every tool/python step with on_result.conditions must declare on_failure.""" + paths = { + "implementation": builtin_recipes_dir() / "implementation.yaml", + "remediation": builtin_recipes_dir() / "remediation.yaml", + "smoke-test": SMOKE_RECIPE, + } + for recipe_name, recipe_path in paths.items(): + recipe = load_recipe(recipe_path) + for step_name, step in recipe.steps.items(): + is_tool = step.tool is not None or step.python is not None + if is_tool and step.on_result and step.on_result.conditions: + assert step.on_failure is not None, ( + f"{recipe_name}.{step_name}: predicate step must declare on_failure" + ) + + +def test_audit_impl_on_failure_routes_to_escalation() -> None: + """audit_impl.on_failure must route to an escalation step in each recipe.""" + impl = load_recipe(builtin_recipes_dir() / "implementation.yaml") + rem = load_recipe(builtin_recipes_dir() / "remediation.yaml") + assert impl.steps["audit_impl"].on_failure == "escalate_stop" + assert rem.steps["audit_impl"].on_failure == "escalate_stop" + + +@pytest.mark.parametrize( + "recipe_name,yaml_name", + [ + ("implementation", "implementation.yaml"), + ("remediation", "remediation.yaml"), + ("merge-prs", "merge-prs.yaml"), + ("implementation-groups", "implementation-groups.yaml"), + ], +) +def test_audit_ingredient_defaults_to_false(recipe_name: str, yaml_name: str) -> None: + """audit must default to 'false' (OFF) in all recipes — opt-in, not opt-out.""" + recipe = load_recipe(builtin_recipes_dir() / yaml_name) + audit_ing = recipe.ingredients.get("audit") + assert audit_ing is not None, f"{recipe_name}: 'audit' ingredient not found" + assert audit_ing.default == "false", ( + f"{recipe_name}: audit.default must be 'false' (OFF by default), got {audit_ing.default!r}" + ) + + +def test_audit_impl_skill_md_emits_verdict_and_remediation_path() -> None: + """1b: audit-impl SKILL.md must contain verdict and remediation_path emit lines.""" + from autoskillit.core.paths import pkg_root + + content = (pkg_root() / "skills_extended" / "audit-impl" / "SKILL.md").read_text() + assert "verdict = " in content, "audit-impl SKILL.md missing 'verdict = ' emit line" + assert "remediation_path = " in content, ( + "audit-impl SKILL.md missing 'remediation_path = ' emit line" + ) + + +def test_review_approach_skill_md_emits_review_path() -> None: + """1c: review-approach SKILL.md must contain review_path emit line.""" + from autoskillit.core.paths import pkg_root + + content = (pkg_root() / "skills_extended" / "review-approach" / "SKILL.md").read_text() + assert "review_path = " in content, ( + "review-approach SKILL.md missing 'review_path = ' emit line" + ) + + +def test_make_groups_skill_md_emits_group_files() -> None: + """1d: make-groups SKILL.md must contain group_files, groups_path, manifest_path lines.""" + from autoskillit.core.paths import pkg_root + + content = (pkg_root() / "skills_extended" / "make-groups" / "SKILL.md").read_text() + assert "group_files = " in content, "make-groups SKILL.md missing 'group_files = ' emit line" + assert "groups_path = " in content, "make-groups SKILL.md missing 'groups_path = ' emit line" + assert "manifest_path = " in content, ( + "make-groups SKILL.md missing 'manifest_path = ' emit line" + ) + + +def test_bundled_recipes_pass_uncaptured_handoff_consumer() -> None: + """1i: all bundled recipes must produce zero uncaptured-handoff-consumer findings.""" + for yaml_file in sorted(builtin_recipes_dir().glob("*.yaml")): + recipe = load_recipe(yaml_file) + findings = run_semantic_rules(recipe) + handoff_findings = [f for f in findings if f.rule == "uncaptured-handoff-consumer"] + assert not handoff_findings, f"{yaml_file.name}: {handoff_findings}" + + +def test_telemetry_before_open_pr_rule_not_in_registry() -> None: + """The telemetry-before-open-pr rule must not be in the rule registry. + + This rule was removed because open-pr now self-retrieves token telemetry + from disk using cwd_filter (Step 0b). If this test fails, the rule was + re-added to the registry and would silently fire on bundled production recipes. + """ + import autoskillit.recipe # noqa: F401 — triggers rule registration + from autoskillit.recipe.registry import _RULE_REGISTRY + + rule_names = {spec.name for spec in _RULE_REGISTRY} + assert "telemetry-before-open-pr" not in rule_names, ( + "telemetry-before-open-pr was re-added to the registry; " + "open-pr self-retrieves token telemetry via cwd_filter — " + "this rule is no longer needed and must not be registered" + ) + + +def test_bundled_recipes_pass_unrouted_verdict_value_rule() -> None: + """All bundled recipes must pass the unrouted-verdict-value semantic rule.""" + for yaml_path in sorted(builtin_recipes_dir().glob("*.yaml")): + recipe = load_recipe(yaml_path) + findings = run_semantic_rules(recipe) + verdict_errors = [f for f in findings if f.rule == "unrouted-verdict-value"] + assert len(verdict_errors) == 0, ( + f"Recipe '{yaml_path.stem}' has unrouted verdict values: " + + ", ".join(f.message for f in verdict_errors) + ) + + +class TestBaseBranchDefaults: + @pytest.mark.parametrize( + "recipe_name", + [ + "implementation", + "remediation", + "implementation-groups", + "merge-prs", + ], + ) + def test_recipe_base_branch_auto_detects(self, recipe_name: str) -> None: + """Non-exempt bundled recipes must use auto-detect for base_branch.""" + recipe = load_recipe(builtin_recipes_dir() / f"{recipe_name}.yaml") + assert recipe.ingredients["base_branch"].default == "", ( + f"{recipe_name}.yaml: base_branch must use auto-detect (default: '')" + ) + + def test_smoke_test_base_branch_uses_auto_detect(self) -> None: + """smoke-test.yaml must use auto-detect for base_branch so config-resolved value wins.""" + recipe = load_recipe(SMOKE_RECIPE) + assert recipe.ingredients["base_branch"].default == "", ( + "smoke-test.yaml base_branch must use auto-detect (default: '') so that " + "branching.default_base_branch from config is honoured at runtime (#703)" + ) + + +class TestImplementationRecipeMergeQueueRule: + """implementation.yaml kitchen_rules must reference merge queue detection.""" + + @pytest.fixture(scope="class") + def recipe(self): + return load_recipe(builtin_recipes_dir() / "implementation.yaml") + + def test_kitchen_rules_mention_check_repo_merge_state(self, recipe) -> None: + all_rules = " ".join(recipe.kitchen_rules) + assert "check_repo_merge_state" in all_rules, ( + "implementation.yaml kitchen_rules must reference check_repo_merge_state" + ) + assert "MERGE ROUTING" in all_rules, ( + "implementation.yaml kitchen_rules must contain a MERGE ROUTING rule" + ) + + def test_kitchen_rules_prohibit_direct_gh_pr_merge(self, recipe) -> None: + # Find the specific rule that mentions "gh pr merge" and check for + # prohibition language within that rule, not across all rules. + merge_rules = [r for r in recipe.kitchen_rules if "gh pr merge" in r] + assert merge_rules, ( + "implementation.yaml kitchen_rules must contain a rule mentioning 'gh pr merge'" + ) + has_prohibition = any( + any(phrase in rule.lower() for phrase in ("never", "prohibited", "do not")) + for rule in merge_rules + ) + assert has_prohibition, ( + "implementation.yaml kitchen_rules must explicitly prohibit calling " + "gh pr merge directly outside of recipe steps" + ) + + +_BUNDLED_RECIPE_PATHS = sorted(builtin_recipes_dir().glob("*.yaml")) + + +@pytest.mark.parametrize("recipe_path", _BUNDLED_RECIPE_PATHS, ids=lambda p: p.stem) +def test_bundled_recipes_emit_no_graph_warnings(recipe_path): + """WF7: build_recipe_graph emits zero warnings for all bundled recipes.""" + recipe = load_recipe(recipe_path) + with structlog.testing.capture_logs() as cap_logs: + build_recipe_graph(recipe) + warning_events = [entry for entry in cap_logs if entry.get("log_level") == "warning"] + assert warning_events == [], ( + f"build_recipe_graph emitted {len(warning_events)} warnings for " + f"{recipe_path.name}: {warning_events}" + ) + + +@pytest.mark.parametrize("recipe_path", _BUNDLED_RECIPE_PATHS, ids=lambda p: p.stem) +def test_all_advisory_run_skill_steps_have_on_context_limit(recipe_path): + """ + Every run_skill step with skip_when_false must declare on_context_limit. + A step that can be skipped by configuration must also be skippable on context limit. + """ + recipe = load_recipe(recipe_path) + violations = [ + name + for name, step in recipe.steps.items() + if step.tool in SKILL_TOOLS + and step.skip_when_false is not None + and step.on_context_limit is None + ] + assert violations == [], ( + f"Advisory run_skill steps in {recipe_path.name} missing on_context_limit: " + f"{violations}. Set on_context_limit to the appropriate skip/recovery step." + ) + + +@pytest.mark.parametrize( + "recipe_name", + ["implementation", "remediation", "implementation-groups", "merge-prs"], +) +def test_review_pr_step_passes_annotated_diff_inputs(recipe_name: str) -> None: + """Every review-pr invocation must pass annotated_diff_path= and hunk_ranges_path= + in skill_command, or have a reachable annotate_pr_diff predecessor step.""" + recipe = load_recipe(builtin_recipes_dir() / f"{recipe_name}.yaml") + # Find review-pr steps + review_steps = [ + (name, step) + for name, step in recipe.steps.items() + if step.tool in SKILL_TOOLS and "review-pr" in step.with_args.get("skill_command", "") + ] + assert review_steps, f"No review-pr step found in {recipe_name}.yaml" + for step_name, step in review_steps: + cmd = step.with_args.get("skill_command", "") + has_inline = "annotated_diff_path=" in cmd and "hunk_ranges_path=" in cmd + has_predecessor = any( + s.with_args.get("callable", "") == "autoskillit.smoke_utils.annotate_pr_diff" + for s in recipe.steps.values() + if s.tool == "run_python" + ) + assert has_inline or has_predecessor, ( + f"{recipe_name}.yaml: step '{step_name}' invokes review-pr but neither " + f"passes annotated_diff_path=/hunk_ranges_path= inline nor has an " + f"annotate_pr_diff predecessor step" + ) + + +@pytest.mark.parametrize( + "recipe_name", + ["implementation", "remediation", "implementation-groups", "merge-prs"], +) +def test_annotate_pr_diff_captures_both_paths(recipe_name: str) -> None: + """The annotate_pr_diff step must capture annotated_diff_path and hunk_ranges_path.""" + recipe = load_recipe(builtin_recipes_dir() / f"{recipe_name}.yaml") + annotate_steps = [ + (name, step) + for name, step in recipe.steps.items() + if step.tool == "run_python" + and step.with_args.get("callable", "") == "autoskillit.smoke_utils.annotate_pr_diff" + ] + assert annotate_steps, f"No annotate_pr_diff step found in {recipe_name}.yaml" + for step_name, step in annotate_steps: + assert step.capture is not None, ( + f"{recipe_name}.yaml: annotate_pr_diff step '{step_name}' has no capture block" + ) + assert "annotated_diff_path" in step.capture, ( + f"{recipe_name}.yaml: annotate_pr_diff step '{step_name}' must capture " + f"annotated_diff_path" + ) + assert "hunk_ranges_path" in step.capture, ( + f"{recipe_name}.yaml: annotate_pr_diff step '{step_name}' must capture " + f"hunk_ranges_path" + ) + + +class TestRunModeIngredient: + """REQ-INGREDIENT-001 through REQ-INGREDIENT-005: run_mode ingredient in multi-issue recipes.""" # noqa: E501 + + @pytest.fixture(scope="class") + def impl_recipe(self): + return load_recipe(builtin_recipes_dir() / "implementation.yaml") + + @pytest.fixture(scope="class") + def remed_recipe(self): + return load_recipe(builtin_recipes_dir() / "remediation.yaml") + + def test_implementation_has_run_mode_ingredient(self, impl_recipe) -> None: + """REQ-INGREDIENT-001: implementation.yaml declares run_mode ingredient.""" + assert "run_mode" in impl_recipe.ingredients, ( + "implementation.yaml must declare run_mode ingredient" + ) + + def test_implementation_run_mode_default_is_sequential(self, impl_recipe) -> None: + """REQ-INGREDIENT-002: run_mode defaults to 'sequential'.""" + ing = impl_recipe.ingredients["run_mode"] + assert ing.default == "sequential", ( + "implementation.yaml run_mode must default to 'sequential'" + ) + + def test_implementation_run_mode_description_mentions_parallel(self, impl_recipe) -> None: + """REQ-INGREDIENT-001: description must document 'parallel' as a valid option.""" + ing = impl_recipe.ingredients["run_mode"] + assert "parallel" in ing.description.lower(), ( + "run_mode description must mention 'parallel' as an option" + ) + + def test_remediation_has_run_mode_ingredient(self, remed_recipe) -> None: + """REQ-INGREDIENT-001: remediation.yaml declares run_mode ingredient.""" + assert "run_mode" in remed_recipe.ingredients, ( + "remediation.yaml must declare run_mode ingredient" + ) + + def test_remediation_run_mode_default_is_sequential(self, remed_recipe) -> None: + """REQ-INGREDIENT-002: run_mode defaults to 'sequential'.""" + ing = remed_recipe.ingredients["run_mode"] + assert ing.default == "sequential", ( + "remediation.yaml run_mode must default to 'sequential'" + ) + + def test_remediation_run_mode_description_mentions_parallel(self, remed_recipe) -> None: + """REQ-INGREDIENT-001: description must document 'parallel' as a valid option.""" + ing = remed_recipe.ingredients["run_mode"] + assert "parallel" in ing.description.lower(), ( + "run_mode description must mention 'parallel' as an option" + ) + + +class TestMaxParallelIngredient: + """REQ-ING-001, REQ-ING-002, REQ-ING-003: max_parallel presence, default, and hidden flag.""" + + @pytest.mark.parametrize("recipe_name", ["implementation", "remediation"]) + def test_recipe_has_max_parallel_ingredient(self, recipe_name: str) -> None: + recipe = load_recipe(builtin_recipes_dir() / f"{recipe_name}.yaml") + assert "max_parallel" in recipe.ingredients + + @pytest.mark.parametrize("recipe_name", ["implementation", "remediation"]) + def test_max_parallel_defaults_to_six(self, recipe_name: str) -> None: + recipe = load_recipe(builtin_recipes_dir() / f"{recipe_name}.yaml") + ing = recipe.ingredients["max_parallel"] + assert ing.default == "6" + + @pytest.mark.parametrize("recipe_name", ["implementation", "remediation"]) + def test_max_parallel_is_hidden(self, recipe_name: str) -> None: + recipe = load_recipe(builtin_recipes_dir() / f"{recipe_name}.yaml") + ing = recipe.ingredients["max_parallel"] + assert ing.hidden is True + + @pytest.mark.parametrize("recipe_name", ["implementation", "remediation"]) + def test_max_parallel_description_mentions_parallel_groups(self, recipe_name: str) -> None: + recipe = load_recipe(builtin_recipes_dir() / f"{recipe_name}.yaml") + ing = recipe.ingredients["max_parallel"] + assert "parallel" in ing.description.lower() + assert "group" in ing.description.lower() + + +def test_no_bare_temp_paths_in_bundled_recipe_notes() -> None: + """No bundled recipe YAML should reference temp/ without .autoskillit/ prefix. + + Bare temp/ references are incorrect; all project-local temp output must be + rooted under .autoskillit/temp/ per CLAUDE.md §3.2. + """ + import re + + recipes_dir = builtin_recipes_dir() + bare_temp = re.compile(r"(? None: + """Recipes must not declare 'defer_cleanup' — that design is removed.""" + recipe = load_recipe(_resolve_recipe_path(recipe_name)) + assert "defer_cleanup" not in recipe.ingredients, ( + f"{recipe_name}.yaml must not declare 'defer_cleanup'" + ) + + +@pytest.mark.parametrize( + "recipe_name", + ["implementation", "remediation", "implementation-groups", "merge-prs", "smoke-test"], +) +def test_recipe_has_no_registry_path_ingredient(recipe_name: str) -> None: + """Recipes must not declare 'registry_path' — replaced by a well-known default.""" + recipe = load_recipe(_resolve_recipe_path(recipe_name)) + assert "registry_path" not in recipe.ingredients, ( + f"{recipe_name}.yaml must not declare 'registry_path'" + ) + + +@pytest.mark.parametrize( + "recipe_name", + ["implementation", "remediation", "implementation-groups", "merge-prs", "smoke-test"], +) +def test_recipe_has_no_interactive_cleanup_steps(recipe_name: str) -> None: + """Recipes must not have confirm_cleanup or delete_clone — these blocked unattended runs.""" + recipe = load_recipe(_resolve_recipe_path(recipe_name)) + assert "confirm_cleanup" not in recipe.steps, ( + f"{recipe_name}.yaml must not have 'confirm_cleanup' step" + ) + assert "delete_clone" not in recipe.steps, ( + f"{recipe_name}.yaml must not have 'delete_clone' step" + ) + + +@pytest.mark.parametrize( + "recipe_name", ["implementation", "remediation", "implementation-groups", "merge-prs"] +) +def test_recipe_has_unconditional_register_steps(recipe_name: str) -> None: + """register_clone_success routes to done; register_clone_failure routes to escalate_stop.""" + recipe = load_recipe(builtin_recipes_dir() / f"{recipe_name}.yaml") + assert "register_clone_success" in recipe.steps + assert "register_clone_failure" in recipe.steps + s = recipe.steps["register_clone_success"] + f = recipe.steps["register_clone_failure"] + assert s.on_success == "done" + assert f.on_success == "escalate_stop" + assert f.on_failure == "escalate_stop" + assert "check_defer_cleanup" not in recipe.steps + assert "check_defer_on_failure" not in recipe.steps + + +@pytest.mark.parametrize( + "recipe_name", + ["implementation.yaml", "implementation-groups.yaml", "remediation.yaml"], +) +def test_re_push_steps_have_force_true(recipe_name: str) -> None: + """All re_push/* steps must have force='true'. + + Post-rebase push requires --force-with-lease. + """ + recipe = load_recipe(builtin_recipes_dir() / recipe_name) + for step_name in ( + "re_push", + "re_push_queue_fix", + "re_push_direct_fix", + "re_push_immediate_fix", + ): + assert step_name in recipe.steps, f"Expected step {step_name!r} in {recipe_name}" + step = recipe.steps[step_name] + assert step.tool == "push_to_remote" + assert step.with_args.get("force") == "true", ( + f"{step_name} in {recipe_name} must include force='true' — " + "post-rebase push requires --force-with-lease" + ) + + +def test_bundled_recipes_have_no_ci_hardcoded_workflow() -> None: + """No bundled recipe should hardcode workflow in wait_for_ci steps.""" + for yaml_path in sorted(builtin_recipes_dir().glob("*.yaml")): + recipe = load_recipe(yaml_path) + findings = run_semantic_rules(recipe) + wf_findings = [f for f in findings if f.rule == "ci-hardcoded-workflow"] + assert wf_findings == [], f"{recipe.name} has hardcoded workflow: {wf_findings}" + + +def test_merge_worktree_has_commit_guard_predecessor() -> None: + """Every merge_worktree step in bundled recipes must have a commit_guard predecessor. + + A commit_guard step auto-commits any dirty files before the merge_worktree + dirty-tree gate runs, providing structural immunity to context-exhausted skills + that leave edits on disk without committing them. + """ + from autoskillit.recipe.validator import make_validation_context + + for yaml_path in sorted(builtin_recipes_dir().glob("*.yaml")): + recipe = load_recipe(yaml_path) + merge_steps = [n for n, s in recipe.steps.items() if s.tool == "merge_worktree"] + if not merge_steps: + continue + ctx = make_validation_context(recipe) + for step_name in merge_steps: + preds = ctx.predecessors.get(step_name, set()) + has_guard = any(_is_commit_guard(p, ctx) for p in preds) + assert has_guard, ( + f"{yaml_path.name}: merge_worktree step '{step_name}' has no commit_guard " + f"predecessor. Add a commit_guard run_cmd step immediately before merge. " + f"Predecessors: {sorted(preds)}" + ) + + +def test_resolve_failures_steps_have_on_context_limit() -> None: + """Every step invoking resolve-failures must declare on_context_limit. + + Without on_context_limit, context exhaustion mid-fix falls through to + on_failure, discarding all uncommitted edits and losing partial progress. + """ + + for yaml_path in sorted(builtin_recipes_dir().glob("*.yaml")): + recipe = load_recipe(yaml_path) + for step_name, step in recipe.steps.items(): + if step.tool not in SKILL_TOOLS: + continue + skill_cmd = step.with_args.get("skill_command", "") + skill = resolve_skill_name(skill_cmd) + if skill != "resolve-failures": + continue + assert step.on_context_limit is not None, ( + f"{yaml_path.name}: step '{step_name}' invokes resolve-failures " + f"but has no on_context_limit. Context exhaustion will strand uncommitted " + f"edits on disk and fall through to on_failure, losing progress." + ) + + +@pytest.mark.parametrize("recipe_path", _ALL_CLONE_RECIPE_PATHS, ids=lambda p: p.stem) +def test_all_clone_recipes_use_context_cwd_after_clone(recipe_path: Path) -> None: + """After clone_repo, no step should use inputs.* as cwd.""" + import re + + input_re = re.compile(r"\$\{\{\s*inputs\.\w+\s*\}\}") + recipe = load_recipe(recipe_path) + + seen_clone = False + for name, step in recipe.steps.items(): + if step.tool == "clone_repo": + seen_clone = True + continue + if seen_clone and step.with_args: + cwd = step.with_args.get("cwd", "") + assert not input_re.search(cwd), ( + f"{recipe_path.stem}: step '{name}' uses inputs.* as cwd after clone: {cwd}" + ) + + +@pytest.mark.parametrize("recipe_name", ["implementation", "implementation-groups", "remediation"]) +def test_bundled_recipes_have_no_release_issue_on_unconfirmed_merge(recipe_name: str) -> None: + """No bundled recipe may route a merge-wait timeout exit to release_issue. + + When wait_for_merge_queue (or equivalent) times out, the PR may still be + in the queue. Calling release_issue on that path removes the in-progress + label while the merge is still pending, leaving the issue visually unclaimed. + """ + recipe = load_recipe(builtin_recipes_dir() / f"{recipe_name}.yaml") + findings = run_semantic_rules(recipe) + violations = [f for f in findings if f.rule == "release-issue-on-unconfirmed-merge"] + assert violations == [], ( + f"{recipe_name} has release-issue-on-unconfirmed-merge violations: {violations}" + ) + + +@pytest.mark.parametrize("recipe_name", ["remediation", "implementation", "implementation-groups"]) +def test_pre_ci_watch_mergeable_check_exists(recipe_name: str) -> None: + """A check_pr_state step must exist between check_repo_ci_event and ci_watch.""" + recipe = load_recipe(builtin_recipes_dir() / f"{recipe_name}.yaml") + assert "check_pr_state" in recipe.steps + pr_state = recipe.steps["check_pr_state"] + assert pr_state.tool == "check_pr_mergeable" + assert pr_state.on_result is not None + routes = {c.route for c in pr_state.on_result.conditions} + assert "ci_watch" in routes, "MERGEABLE must route to ci_watch" + conflicting_routes = { + c.route for c in pr_state.on_result.conditions if c.when and "CONFLICTING" in c.when + } + assert conflicting_routes, "CONFLICTING condition must be handled" + assert "ci_watch" not in conflicting_routes, "CONFLICTING must not route to ci_watch" + ci_event_step = recipe.steps["check_repo_ci_event"] + assert ci_event_step.on_success == "check_pr_state" + + +@pytest.mark.parametrize("recipe_name", ["remediation", "implementation", "implementation-groups"]) +def test_active_ci_trigger_step_exists(recipe_name: str) -> None: + """escalate_stop_no_ci predecessor must attempt active CI trigger before stopping.""" + recipe = load_recipe(builtin_recipes_dir() / f"{recipe_name}.yaml") + assert "trigger_ci_actively" in recipe.steps + trigger = recipe.steps["trigger_ci_actively"] + assert trigger.tool == "run_cmd" + assert "check_ci_loop" in recipe.steps, "check_ci_loop step missing from recipe" + guard = recipe.steps["check_ci_loop"] + guard_routes = {c.route for c in guard.on_result.conditions} + assert "check_active_trigger_loop" in guard_routes + assert "check_active_trigger_loop" in recipe.steps, ( + "check_active_trigger_loop guard step missing" + ) + trigger_guard = recipe.steps["check_active_trigger_loop"] + trigger_guard_routes = {c.route for c in trigger_guard.on_result.conditions} + assert "trigger_ci_actively" in trigger_guard_routes + + +@pytest.mark.parametrize("recipe_name", ["remediation", "implementation", "implementation-groups"]) +def test_reenroll_stalled_pr_has_loop_guard(recipe_name: str) -> None: + """reenroll_stalled_pr cycle must have an iteration guard.""" + recipe = load_recipe(builtin_recipes_dir() / f"{recipe_name}.yaml") + reenroll = recipe.steps["reenroll_stalled_pr"] + assert reenroll.on_success != "wait_for_queue", ( + "reenroll_stalled_pr must route through a loop guard, not directly to wait_for_queue" + ) + assert reenroll.on_success == "check_stall_loop", ( + f"reenroll_stalled_pr must route to check_stall_loop, got: {reenroll.on_success!r}" + ) + + +def test_requires_packs_covers_all_default_disabled_skill_categories() -> None: + """Every recipe's requires_packs must cover all default-disabled pack categories + needed by its run_skill steps. This is a structural defense-in-depth guard + complementary to the undeclared-pack-requirement semantic rule.""" + category_map = _get_skill_category_map() + default_disabled = frozenset( + name for name, pdef in PACK_REGISTRY.items() if not pdef.default_enabled + ) + for path in sorted(builtin_recipes_dir().glob("*.yaml")): + try: + recipe = load_recipe(path) + except Exception as exc: + pytest.fail(f"{path.name}: {exc}") + needed_packs: set[str] = set() + for step_name, step in recipe.steps.items(): + if step.tool not in SKILL_TOOLS: + continue + skill_cmd = (step.with_args or {}).get("skill_command") or "" + skill_name = resolve_skill_name(skill_cmd) + if skill_name is None: + continue + categories = category_map.get(skill_name, frozenset()) + needed_packs.update(categories & default_disabled) + missing = needed_packs - set(recipe.requires_packs) + assert not missing, ( + f"{path.name}: run_skill steps reference default-disabled packs {sorted(missing)} " + f"but requires_packs={recipe.requires_packs!r} is missing them" + ) diff --git a/tests/recipe/test_bundled_recipes_no_inversions.py b/tests/recipe/test_bundled_recipes_no_inversions.py index 25f68393f3..6c0bd6bf80 100644 --- a/tests/recipe/test_bundled_recipes_no_inversions.py +++ b/tests/recipe/test_bundled_recipes_no_inversions.py @@ -10,6 +10,8 @@ from autoskillit.recipe.io import builtin_recipes_dir, load_recipe from autoskillit.recipe.registry import run_semantic_rules +pytestmark = [pytest.mark.layer("recipe"), pytest.mark.small] + _BUNDLED_RECIPE_PATHS: list[Path] = sorted(builtin_recipes_dir().glob("*.yaml")) diff --git a/tests/recipe/test_bundled_recipes_pipeline_structure.py b/tests/recipe/test_bundled_recipes_pipeline_structure.py new file mode 100644 index 0000000000..7f715f4920 --- /dev/null +++ b/tests/recipe/test_bundled_recipes_pipeline_structure.py @@ -0,0 +1,899 @@ +"""Tests for structural assertions on pipeline recipe variants +(implementation, implementation-groups, remediation). + +TestPipelineVariantInvariants covers the 16 tests that were copy-paste identical +across the three per-variant classes. +""" + +from __future__ import annotations + +import pytest + +from autoskillit.core.types import Severity +from autoskillit.recipe.io import builtin_recipes_dir, load_recipe +from autoskillit.recipe.validator import analyze_dataflow, run_semantic_rules + +pytestmark = [pytest.mark.layer("recipe"), pytest.mark.small] + + +def _assert_ci_conflict_fix_on_context_limit(recipe) -> None: + """Shared assertion: ci_conflict_fix must abort via release_issue_failure on context limit.""" + step = recipe.steps["ci_conflict_fix"] + assert step.on_context_limit == "release_issue_failure", ( + "ci_conflict_fix is advisory; an incomplete conflict fix cannot be safely " + "pushed — abort via release_issue_failure" + ) + + +def _assert_ci_steps(recipe) -> None: + """Assert shared CI step structure across all pipeline recipe variants. + + Covers T_CI1–T_CI6: ci_watch, resolve_ci, and re_push steps. + Called from each recipe test class to eliminate triplication. + """ + # ci_watch step (T_CI1, T_CI2, T_CI3) + assert "ci_watch" in recipe.steps + ci = recipe.steps["ci_watch"] + assert ci.tool == "wait_for_ci" + assert ci.skip_when_false == "inputs.open_pr" + assert ci.with_args.get("timeout_seconds") == 600 + assert ci.on_result is not None + result_routes = {c.route for c in ci.on_result.conditions} + assert "check_repo_merge_state" in result_routes + assert "handle_no_ci_runs" in result_routes + assert ci.on_failure == "detect_ci_conflict" + assert "release_issue_success" in recipe.steps + assert "context.merge_target" in ci.with_args["branch"] + assert "cmd" not in ci.with_args + assert "ci_conclusion" in ci.capture + assert "ci_failed_jobs" in ci.capture + + # resolve_ci step (T_CI4, T_CI5) + assert "resolve_ci" in recipe.steps + resolve_ci = recipe.steps["resolve_ci"] + assert resolve_ci.tool == "run_skill" + skill_cmd = resolve_ci.with_args.get("skill_command", "") + assert "resolve-failures" in skill_cmd + assert resolve_ci.retries == 2 + assert resolve_ci.on_exhausted == "release_issue_failure" + assert "context.work_dir" in skill_cmd + + # re_push step (T_CI6) + assert "re_push" in recipe.steps + re_push = recipe.steps["re_push"] + assert re_push.tool == "push_to_remote" + assert re_push.on_success == "check_repo_ci_event" + assert re_push.on_failure == "release_issue_failure" + + # check_ci_loop guard (loop bounding for ci_watch / handle_no_ci_runs cycle) + assert "check_ci_loop" in recipe.steps, "ci_watch cycle must have check_ci_loop guard" + guard = recipe.steps["check_ci_loop"] + assert guard.tool == "run_python" + assert "check_loop_iteration" in guard.with_args.get("callable", "") + assert guard.on_result is not None + guard_routes = {c.route for c in guard.on_result.conditions} + assert "ci_watch" in guard_routes, "check_ci_loop must route back to ci_watch" + assert "check_active_trigger_loop" in guard_routes, ( + "check_ci_loop must route to check_active_trigger_loop on budget exhaustion" + ) + assert "check_active_trigger_loop" in recipe.steps, ( + "check_active_trigger_loop guard step missing" + ) + trigger_guard = recipe.steps["check_active_trigger_loop"] + trigger_guard_routes = {c.route for c in trigger_guard.on_result.conditions} + assert "trigger_ci_actively" in trigger_guard_routes, ( + "check_active_trigger_loop must route to trigger_ci_actively" + ) + assert "ci_loop_count" in guard.capture + handle = recipe.steps["handle_no_ci_runs"] + assert handle.on_success == "check_ci_loop", ( + "handle_no_ci_runs must route to check_ci_loop, not ci_watch" + ) + + +# Maps recipe name → expected on_context_limit value for the review step. +# review step routes to verify in the implementation variants (can skip to next step) +# but to dry_walkthrough in remediation (review is optional before dry run). +_REVIEW_STEP_OCL = { + "implementation": "verify", + "implementation-groups": "verify", + "remediation": "dry_walkthrough", +} + + +class TestPipelineVariantInvariants: + """Assertions that hold for all three pipeline variants. + + Each test runs once per variant (implementation / implementation-groups / remediation). + Before this refactor, each assertion was copy-paste triplicated across + TestImplementationPipelineStructure, TestImplementationGroupsStructure, and + TestInvestigateFirstStructure. + """ + + @pytest.fixture( + scope="class", + params=["implementation", "implementation-groups", "remediation"], + ids=lambda x: x, + ) + def recipe(self, request: pytest.FixtureRequest): + return load_recipe(builtin_recipes_dir() / f"{request.param}.yaml") + + def test_ci_step_structure(self, recipe) -> None: + """T_CI1–T_CI6: shared ci_watch, resolve_ci, and re_push structure.""" + _assert_ci_steps(recipe) + + def test_re_push_has_explicit_remote_url(self, recipe) -> None: + """T_CI7: re_push uses explicit remote_url.""" + with_args = recipe.steps["re_push"].with_args + assert "remote_url" in with_args + assert "context.remote_url" in with_args["remote_url"] + + def test_compose_pr_routes_to_extract_pr_number(self, recipe) -> None: + """T_CI8: compose_pr.on_success routes to extract_pr_number before review_pr.""" + step = recipe.steps["compose_pr"] + assert step.on_success == "extract_pr_number", "compose_pr must route to extract_pr_number" + + def test_detect_ci_conflict_exists(self, recipe) -> None: + assert "detect_ci_conflict" in recipe.steps + step = recipe.steps["detect_ci_conflict"] + assert step.tool == "run_cmd" + + def test_detect_ci_conflict_uses_merge_base(self, recipe) -> None: + step = recipe.steps["detect_ci_conflict"] + cmd = (step.with_args or {}).get("cmd", "") + assert "merge-base" in cmd or "is-ancestor" in cmd + + def test_detect_ci_conflict_routing(self, recipe) -> None: + step = recipe.steps["detect_ci_conflict"] + assert step.on_success == "ci_conflict_fix" + assert step.on_failure == "diagnose_ci" + + def test_ci_conflict_fix_exists(self, recipe) -> None: + assert "ci_conflict_fix" in recipe.steps + step = recipe.steps["ci_conflict_fix"] + assert step.tool == "run_skill" + skill_cmd = (step.with_args or {}).get("skill_command", "") + assert "resolve-merge-conflicts" in skill_cmd + + def test_ci_conflict_fix_routing(self, recipe) -> None: + step = recipe.steps["ci_conflict_fix"] + assert step.on_failure == "release_issue_failure" + assert step.on_exhausted == "release_issue_failure" + conditions = step.on_result.conditions if step.on_result else [] + routes = {c.when: c.route for c in conditions} + assert any( + "escalation_required" in (w or "") and r == "release_issue_failure" + for w, r in routes.items() + ) + default_routes = [r for w, r in routes.items() if w is None] + assert default_routes == ["re_push"] + + def test_detect_ci_conflict_skip_when_false(self, recipe) -> None: + step = recipe.steps["detect_ci_conflict"] + assert step.skip_when_false == "inputs.open_pr" + + def test_ci_conflict_fix_skip_when_false(self, recipe) -> None: + step = recipe.steps["ci_conflict_fix"] + assert step.skip_when_false == "inputs.open_pr" + + def test_review_step_has_skip_when_false(self, recipe) -> None: + step = recipe.steps["review"] + assert step.skip_when_false == "inputs.review_approach" + + def test_review_step_has_retries(self, recipe) -> None: + step = recipe.steps["review"] + assert step.retries >= 1 + + def test_review_step_has_on_context_limit(self, recipe) -> None: + """review step on_context_limit differs per variant (verify vs dry_walkthrough).""" + step = recipe.steps["review"] + expected = _REVIEW_STEP_OCL[recipe.name] + assert step.on_context_limit == expected + + def test_audit_impl_has_on_context_limit(self, recipe) -> None: + step = recipe.steps["audit_impl"] + assert step.on_context_limit == "escalate_stop", ( + "audit_impl is a merge gate; a context-exhausted audit cannot provide " + "a valid verdict — aborting via escalate_stop is correct" + ) + + def test_compose_pr_has_on_context_limit(self, recipe) -> None: + step = recipe.steps["compose_pr"] + assert step.on_context_limit == "release_issue_failure", ( + "compose_pr is advisory (skip_when_false); on context limit the pipeline " + "cannot determine PR state — release the issue via release_issue_failure" + ) + + def test_ci_conflict_fix_has_on_context_limit(self, recipe) -> None: + _assert_ci_conflict_fix_on_context_limit(recipe) + + +# --------------------------------------------------------------------------- +# TestImplementationPipelineStructure +# --------------------------------------------------------------------------- + + +# Negative-parity mirror of TestImplementationGroupsStructure: asserts that +# implementation.yaml uniquely owns features absent from implementation-groups.yaml, +# and vice versa. Changes to one class should be reviewed alongside the other. +class TestImplementationPipelineStructure: + @pytest.fixture(scope="class") + def recipe(self): + return load_recipe(builtin_recipes_dir() / "implementation.yaml") + + def test_ip2_review_step_captures_review_path(self, recipe) -> None: + """T_IP2: review step has capture containing key review_path.""" + assert "review_path" in recipe.steps["review"].capture + + def test_ip3_audit_impl_has_verdict_and_remediation_capture_and_on_result( + self, + recipe, + ) -> None: + """T_IP3: audit_impl captures remediation_path and routes via on_result using verdict. + + Uses predicate format (v0.3.0): verdict is read directly from result.verdict in predicate + conditions — it is not captured as context.verdict (which would create a dead output). + """ + step = recipe.steps["audit_impl"] + assert "remediation_path" in step.capture + assert step.on_result is not None + # Predicate format: conditions list (not legacy field+routes dict) + conds = step.on_result.conditions + assert len(conds) > 0, "audit_impl on_result must have predicate conditions" + assert any("result.verdict" in (c.when or "") for c in conds), ( + "audit_impl on_result must have a condition checking result.verdict" + ) + + def test_ip4_verify_step_references_context_review_path(self, recipe) -> None: + """T_IP4: verify step with_args contains a reference to context.review_path.""" + verify_with = recipe.steps["verify"].with_args + assert any("context.review_path" in str(v) for v in verify_with.values()) + + def test_ip5_audit_impl_has_on_failure(self, recipe) -> None: + """T_IP5: audit_impl declares on_failure for tool-level failure routing. + + In the two-tier failure model, on_result.conditions fire when run_skill returns + success=true with a result object. on_failure fires when run_skill returns + success=false (tool-level failure, no result object). Both must be declared. + """ + step = recipe.steps["audit_impl"] + assert step.on_success is None # on_result is used; on_success remains absent + assert step.on_failure == "escalate_stop", ( + "audit_impl must declare on_failure: escalate_stop. " + "Tool-level failures produce no result object — on_result conditions cannot fire." + ) + + def test_ip6_plan_step_note_contains_glob_pattern(self, recipe) -> None: + """T_IP6: plan step note must contain *_part_*.md glob pattern for multi-part discovery.""" + note = recipe.steps["plan"].note or "" + assert "*_part_*.md" in note, ( + "plan step note must contain glob pattern for multi-part discovery; " + "if removed, agents will not discover part files" + ) + + def test_ip7_verify_step_note_sequential_constraint(self, recipe) -> None: + """T_IP7: verify step note must contain sequential execution constraint.""" + note = recipe.steps["verify"].note or "" + assert "SEQUENTIAL EXECUTION" in note or "full cycle" in note.lower(), ( + "verify step note must contain sequential constraint; " + "without it agents may batch-verify all parts before implementing any" + ) + + def test_ip8_next_or_done_routes_more_parts_to_verify(self, recipe) -> None: + """T_IP8: next_or_done routes more_parts back to verify for sequential processing. + + Uses predicate format (v0.3.0): when-condition checks result.next == more_parts. + """ + step = recipe.steps["next_or_done"] + assert step.on_result is not None + conds = step.on_result.conditions + assert any( + c.route == "verify" and c.when is not None and "more_parts" in c.when for c in conds + ), "next_or_done must have a predicate routing more_parts → verify" + + def test_ip9_next_or_done_routes_all_done_to_audit_impl(self, recipe) -> None: + """T_IP9: next_or_done must route all_done to audit_impl. + + Uses predicate format (v0.3.0): fallthrough condition (when=None) routes to audit_impl. + """ + step = recipe.steps["next_or_done"] + assert step.on_result is not None + conds = step.on_result.conditions + # The fallthrough condition (when=None) is the default route to audit_impl + assert any(c.route == "audit_impl" for c in conds), ( + "next_or_done must have a condition routing to audit_impl" + ) + + def test_ip_audit_impl_uses_base_sha_as_ref(self, recipe) -> None: + """T_IP_B2: audit_impl must use context.base_sha (not context.branch_name). + + branch_name is deleted by git branch -D inside merge_worktree. A commit SHA + names a git object and survives unconditionally. + """ + step = recipe.steps["audit_impl"] + skill_cmd = step.with_args.get("skill_command", "") + assert "context.base_sha" in skill_cmd, ( + "audit_impl must use context.base_sha as implementation_ref — " + "context.branch_name is deleted by merge_worktree before audit_impl runs" + ) + assert "context.branch_name" not in skill_cmd, ( + "audit_impl must NOT use context.branch_name (deleted by merge_worktree)" + ) + + def test_ip_c1_fix_step_routes_via_on_result_to_test(self, recipe) -> None: + """T_IP_C1: fix step must route via verdict-gated on_result back to test. + + After Part B, fix uses on_result: verdict dispatch instead of unconditional + on_success: test. real_fix and already_green verdicts route to test for + re-validation before entering merge_worktree. + """ + step = recipe.steps["fix"] + assert step.on_success is None, ( + "fix step must use on_result: verdict dispatch, not unconditional on_success" + ) + assert step.on_result is not None, "fix step must have on_result: verdict dispatch" + test_routes = [ + c.route for c in step.on_result.conditions if c.when and "real_fix" in c.when + ] + assert any(r == "test" for r in test_routes), ( + "fix step on_result must route verdict=real_fix to test for re-validation" + ) + + def test_ip_base_sha_captured_before_implement(self, recipe) -> None: + """A1: base_sha must be captured by a step that precedes the implement loop.""" + steps = list(recipe.steps.items()) + step_names = [name for name, _ in steps] + + # Find step that captures base_sha + sha_step = next( + (name for name, step in steps if step.capture and "base_sha" in step.capture), + None, + ) + assert sha_step is not None, "No step captures base_sha" + + implement_idx = next( + ( + i + for i, (name, step) in enumerate(steps) + if step.tool in {"run_skill"} + and "implement-worktree" in step.with_args.get("skill_command", "") + ), + None, + ) + assert implement_idx is not None, "No implement step found" + sha_idx = step_names.index(sha_step) + assert sha_idx < implement_idx, ( + f"base_sha capture step '{sha_step}' (idx {sha_idx}) must come before " + f"implement step (idx {implement_idx})" + ) + + def test_ip_merge_target_unconditionally_set(self, recipe) -> None: + """A3: merge_target must be captured by a non-optional step before merge/push.""" + steps = list(recipe.steps.items()) + step_names = [name for name, _ in steps] + + merge_idx = step_names.index("merge") if "merge" in step_names else None + assert merge_idx is not None, "merge step not found" + + # Find any non-optional step that captures merge_target before merge + unconditional_capture = next( + ( + name + for name, step in steps[:merge_idx] + if step.capture and "merge_target" in step.capture and not step.optional + ), + None, + ) + assert unconditional_capture is not None, ( + "merge_target must be captured by a non-optional step before the merge step. " + "Currently it is only captured by the optional create_branch step, leaving it " + "undefined in direct mode (open_pr=false)." + ) + + def test_ip_base_sha_capture_uses_work_dir(self, recipe) -> None: + """A4: base_sha must be captured by the bootstrap_clone step.""" + sha_step = next( + ( + step + for name, step in recipe.steps.items() + if step.capture and "base_sha" in step.capture + ), + None, + ) + assert sha_step is not None, "No step captures base_sha" + assert sha_step.tool == "bootstrap_clone", ( + "base_sha must be captured by bootstrap_clone (which runs git rev-parse inside the " + "clone directory internally)" + ) + + def test_ip_base_sha_used_by_audit_impl(self, recipe) -> None: + """A5: base_sha captured must be consumed — specifically by audit_impl.""" + report = analyze_dataflow(recipe) + dead = {w.field for w in report.warnings if w.code == "DEAD_OUTPUT"} + assert "base_sha" not in dead, ( + "base_sha is captured but never consumed — audit_impl must reference it" + ) + + def test_ip_push_after_audit_warning_fires(self, recipe) -> None: + """T_IP_PBA: after Part B, audit_impl has skip_when_false so push-before-audit + fires as a WARNING. This is correct and expected — the user can opt out of audit + (audit=false), and the rule signals that push is reachable without audit on that path. + """ + findings = run_semantic_rules(recipe) + violations = [f for f in findings if f.rule == "push-before-audit"] + assert len(violations) >= 1, ( + "push-before-audit must fire: audit_impl has skip_when_false so push is " + "reachable via the audit=false bypass path" + ) + assert all(v.severity == Severity.WARNING for v in violations) + + def test_ip_compose_pr_has_skip_when_false(self, recipe) -> None: + """compose_pr must declare skip_when_false: inputs.open_pr.""" + compose_pr = recipe.steps["compose_pr"] + assert compose_pr.skip_when_false == "inputs.open_pr" + + def test_ip_audit_impl_has_skip_when_false(self, recipe) -> None: + """audit_impl must declare skip_when_false: inputs.audit.""" + audit_step = recipe.steps["audit_impl"] + assert audit_step.skip_when_false == "inputs.audit" + + def test_ip_create_branch_has_skip_when_false(self, recipe) -> None: + """create_and_publish must declare skip_when_false: inputs.open_pr.""" + create_and_publish = recipe.steps["create_and_publish"] + assert create_and_publish.skip_when_false == "inputs.open_pr" + + def test_create_branch_uses_callable(self, recipe) -> None: + """create_and_publish must use create_and_publish_branch MCP tool.""" + step = recipe.steps["create_and_publish"] + assert step.tool == "create_and_publish_branch" + + def test_create_branch_checks_remote_for_collisions(self, recipe) -> None: + """create_and_publish uses create_and_publish_branch (which checks ls-remote).""" + assert recipe.steps["create_and_publish"].tool == "create_and_publish_branch" + + def test_create_branch_references_issue_number(self, recipe) -> None: + """create_and_publish with_args must include issue_number for branch naming.""" + assert "issue_number" in recipe.steps["create_and_publish"].with_args + + def test_create_branch_uses_run_name_as_prefix(self, recipe) -> None: + """create_and_publish with_args must include run_name for branch naming.""" + assert "run_name" in recipe.steps["create_and_publish"].with_args + + def test_ip_main_push_step_not_reachable_after_compose_pr(self, recipe) -> None: + """The main `push` step must not be reachable after compose_pr — + that would be a double-push. The new `re_push` step IS reachable and is correct.""" + from autoskillit.recipe.validator import _build_step_graph + + graph = _build_step_graph(recipe) + visited: set[str] = set() + queue = [recipe.steps["compose_pr"].on_success] + while queue: + current = queue.pop(0) + if current in visited or current not in recipe.steps: + continue + visited.add(current) + queue.extend(graph.get(current, [])) + assert "push" not in visited, ( + "'push' step is reachable after compose_pr — double-push risk. " + "(re_push is allowed; push is not)" + ) + + def test_ip_open_pr_false_path_reaches_push_then_cleanup(self, recipe) -> None: + """When compose_pr is bypassed (open_pr=false), execution must go: + audit_impl (GO) → push → [compose_pr bypassed] → cleanup_success → done. + After the fix, audit_impl's GO route points directly to push, so push is + always reachable from audit_impl's successors.""" + from autoskillit.recipe.validator import _build_step_graph + + graph = _build_step_graph(recipe) + # After the fix: audit_impl.on_result.GO → push (directly). + # Verify push is reachable from audit_impl's successors. + reachable: set[str] = set() + queue = list(graph.get("audit_impl", [])) + while queue: + node = queue.pop(0) + if node in reachable or node not in recipe.steps: + continue + reachable.add(node) + queue.extend(graph.get(node, [])) + assert "push" in reachable + + def test_ip_plan_step_captures_all_plan_paths(self, recipe) -> None: + """plan step must capture all_plan_paths for multi-group accumulation.""" + assert "all_plan_paths" in recipe.steps["plan"].capture + assert "result.plan_path" in recipe.steps["plan"].capture["all_plan_paths"] + + def test_ip_prepare_pr_references_all_plan_paths(self, recipe) -> None: + """prepare_pr must pass all accumulated plan paths, not just the last.""" + cmd = recipe.steps["prepare_pr"].with_args.get("skill_command", "") + assert "context.all_plan_paths" in cmd + assert "context.plan_path" not in cmd + + def test_ip_plan_step_note_contains_accumulation_instruction(self, recipe) -> None: + """plan step note must instruct agent to accumulate plan paths across groups.""" + note = recipe.steps["plan"].note or "" + assert "ACCUMULATION" in note + assert "all_plan_paths" in note + + def test_ip_no_group_step(self, recipe) -> None: + """implementation.yaml must not contain a group step.""" + assert "group" not in recipe.steps + + def test_ip_task_ingredient_required(self, recipe) -> None: + """task ingredient must be required in the direct recipe.""" + task_ing = recipe.ingredients.get("task") + assert task_ing is not None + assert task_ing.required is True or task_ing.default is None + + def test_ip_no_make_groups_ingredient(self, recipe) -> None: + """make_groups ingredient must not be present.""" + assert "make_groups" not in recipe.ingredients + + def test_ip_no_source_doc_ingredient(self, recipe) -> None: + """source_doc ingredient must not be present.""" + assert "source_doc" not in recipe.ingredients + + def test_ip_next_or_done_no_more_groups_route(self, recipe) -> None: + """next_or_done must not route more_groups — no groups in the direct recipe.""" + step = recipe.steps["next_or_done"] + assert step.on_result is not None + conds = step.on_result.conditions + assert not any("more_groups" in (c.when or "") for c in conds) + + +# --------------------------------------------------------------------------- +# TestImplementationGroupsStructure +# --------------------------------------------------------------------------- + + +class TestImplementationGroupsStructure: + @pytest.fixture(scope="class") + def recipe(self): + return load_recipe(builtin_recipes_dir() / "implementation-groups.yaml") + + def test_ig1_group_step_captures_group_files(self, recipe) -> None: + """T_IG1: group step captures group_files, not groups_path.""" + assert "group_files" in recipe.steps["group"].capture + assert "groups_path" not in recipe.steps["group"].capture + + def test_ig2_group_step_is_not_optional(self, recipe) -> None: + """T_IG2: group step must always run — no skip_when_false, not conditional.""" + step = recipe.steps["group"] + assert step.skip_when_false is None + assert not step.optional + + def test_ig3_source_doc_required(self, recipe) -> None: + """T_IG3: source_doc must be a required ingredient in the groups recipe.""" + src = recipe.ingredients.get("source_doc") + assert src is not None + assert src.required is True + + def test_ig4_no_make_groups_ingredient(self, recipe) -> None: + """T_IG4: make_groups must not be present — groups are always used in this recipe.""" + assert "make_groups" not in recipe.ingredients + + def test_ig5_next_or_done_routes_more_groups_to_plan(self, recipe) -> None: + """T_IG5: next_or_done must route more_groups back to plan for group iteration.""" + step = recipe.steps["next_or_done"] + assert step.on_result is not None + conds = step.on_result.conditions + assert any( + c.route == "plan" and c.when is not None and "more_groups" in c.when for c in conds + ), "next_or_done must have a predicate routing more_groups → plan" + + def test_ig6_next_or_done_routes_more_parts_to_verify(self, recipe) -> None: + """T_IG6: next_or_done must route more_parts to verify for sequential part processing.""" + step = recipe.steps["next_or_done"] + assert step.on_result is not None + conds = step.on_result.conditions + assert any( + c.route == "verify" and c.when is not None and "more_parts" in c.when for c in conds + ), "next_or_done must have a predicate routing more_parts → verify" + + def test_ig7_next_or_done_fallthrough_to_audit_impl(self, recipe) -> None: + """T_IG7: next_or_done fallthrough (all done) must route to audit_impl.""" + step = recipe.steps["next_or_done"] + assert step.on_result is not None + conds = step.on_result.conditions + assert any(c.route == "audit_impl" for c in conds) + + def test_ig8_plan_note_contains_accumulation_instruction(self, recipe) -> None: + """T_IG8: plan step note must instruct agent to accumulate plan paths across groups.""" + note = recipe.steps["plan"].note or "" + assert "ACCUMULATION" in note + assert "all_plan_paths" in note + + def test_ig_push_merge_target_routes_to_group(self, recipe) -> None: + """create_and_publish must route to group, not plan, in the groups recipe.""" + step = recipe.steps.get("create_and_publish") + assert step is not None + assert step.on_success == "group" + + def test_ig_audit_impl_uses_base_sha_as_ref(self, recipe) -> None: + """audit_impl must use context.base_sha as implementation_ref.""" + step = recipe.steps["audit_impl"] + skill_cmd = step.with_args.get("skill_command", "") + assert "context.base_sha" in skill_cmd + assert "context.branch_name" not in skill_cmd + + def test_ig_fix_step_routes_via_on_result_to_test(self, recipe) -> None: + """fix step must route via verdict-gated on_result to test.""" + step = recipe.steps["fix"] + assert step.on_success is None, "fix step must use on_result: verdict dispatch" + assert step.on_result is not None, "fix step must have on_result: verdict dispatch" + test_routes = [ + c.route for c in step.on_result.conditions if c.when and "real_fix" in c.when + ] + assert any(r == "test" for r in test_routes), ( + "fix step on_result must route verdict=real_fix to test" + ) + + def test_ig_push_after_audit_warning_fires(self, recipe) -> None: + """push-before-audit semantic rule fires as WARNING (audit has skip_when_false).""" + findings = run_semantic_rules(recipe) + violations = [f for f in findings if f.rule == "push-before-audit"] + assert len(violations) >= 1 + assert all(v.severity == Severity.WARNING for v in violations) + + def test_ig_auto_merge_ingredient_exists(self, recipe) -> None: + """REQ-C7-01: auto_merge ingredient must exist in implementation-groups.yaml.""" + assert "auto_merge" in recipe.ingredients, ( + "auto_merge ingredient is missing — required for merge queue lifecycle" + ) + assert recipe.ingredients["auto_merge"].default == "true" + + def test_ig_extract_pr_number_step_exists(self, recipe) -> None: + """REQ-C7-01: extract_pr_number step must exist to supply pr_number to queue steps.""" + assert "extract_pr_number" in recipe.steps + step = recipe.steps["extract_pr_number"] + assert step.tool == "run_cmd" + assert "pr_number" in step.capture + assert step.on_success == "annotate_pr_diff" + + def test_ig_ci_watch_routes_to_check_repo_merge_state(self, recipe) -> None: + """REQ-C7-01: ci_watch on_result success must route to check_repo_merge_state.""" + step = recipe.steps["ci_watch"] + assert step.on_result is not None, "ci_watch must use on_result predicate routing" + result_routes = {c.route for c in step.on_result.conditions} + assert "check_repo_merge_state" in result_routes, ( + "ci_watch must route to check_repo_merge_state so the PR can enter the merge queue. " + "Routing directly to release_issue_success skips the queue lifecycle entirely." + ) + + def test_ig_check_repo_merge_state_step_exists(self, recipe) -> None: + """REQ-C7-01: check_repo_merge_state step must exist.""" + assert "check_repo_merge_state" in recipe.steps + step = recipe.steps["check_repo_merge_state"] + assert step.tool == "check_repo_merge_state" + assert step.block == "pre_queue_gate" + assert "queue_available" in step.capture + + def test_ig_wait_for_queue_step_exists(self, recipe) -> None: + """REQ-C7-01: wait_for_queue step must exist with correct tool and routing.""" + assert "wait_for_queue" in recipe.steps + step = recipe.steps["wait_for_queue"] + assert step.tool == "wait_for_merge_queue" + assert step.with_args.get("timeout_seconds") == 900 + conds = step.on_result.conditions if step.on_result else [] + merged_cond = next((c for c in conds if c.when and "merged" in c.when), None) + assert merged_cond is not None and merged_cond.route == "release_issue_success" + + def test_ig_release_issue_success_has_target_branch(self, recipe) -> None: + """REQ-C7-01: release_issue_success must pass target_branch to trigger staged label.""" + step = recipe.steps["release_issue_success"] + with_args = step.with_args or {} + assert "target_branch" in with_args, ( + "release_issue_success must pass target_branch: ${{ inputs.base_branch }} — " + "without it release_issue cannot apply the staged label on non-default branches" + ) + assert "inputs.base_branch" in with_args["target_branch"] + + def test_create_branch_uses_callable(self, recipe) -> None: + """create_and_publish must use create_and_publish_branch MCP tool.""" + step = recipe.steps["create_and_publish"] + assert step.tool == "create_and_publish_branch" + + def test_create_branch_checks_remote_for_collisions(self, recipe) -> None: + """create_and_publish uses create_and_publish_branch (which checks ls-remote).""" + assert recipe.steps["create_and_publish"].tool == "create_and_publish_branch" + + def test_create_branch_references_issue_number(self, recipe) -> None: + """create_and_publish with_args must include issue_number for branch naming.""" + assert "issue_number" in recipe.steps["create_and_publish"].with_args + + def test_create_branch_uses_run_name_as_prefix(self, recipe) -> None: + """create_and_publish with_args must include run_name for branch naming.""" + assert "run_name" in recipe.steps["create_and_publish"].with_args + + +# --------------------------------------------------------------------------- +# TestInvestigateFirstStructure +# --------------------------------------------------------------------------- + + +class TestInvestigateFirstStructure: + @pytest.fixture(scope="class") + def recipe(self): + return load_recipe(builtin_recipes_dir() / "remediation.yaml") + + def test_if1_audit_impl_has_verdict_and_remediation_capture_and_on_result( + self, + recipe, + ) -> None: + """T_IF1: audit_impl captures remediation_path and routes via on_result using verdict. + + Uses predicate format (v0.3.0): verdict is read directly from result.verdict in predicate + conditions — it is not captured as context.verdict (which would create a dead output). + """ + step = recipe.steps["audit_impl"] + assert "remediation_path" in step.capture + assert step.on_result is not None + conds = step.on_result.conditions + assert len(conds) > 0, "audit_impl on_result must have predicate conditions" + assert any("result.verdict" in (c.when or "") for c in conds), ( + "audit_impl on_result must have a condition checking result.verdict" + ) + + def test_if2_remediate_step_routes_to_make_plan(self, recipe) -> None: + """T_IF2: remediate step exists and routes to make_plan (not rectify).""" + assert "remediate" in recipe.steps + assert recipe.steps["remediate"].on_success == "make_plan" + + def test_if5_make_plan_step_has_correct_structure(self, recipe) -> None: + """T_IF5: make_plan step calls make-plan with remediation_path and captures outputs.""" + assert "make_plan" in recipe.steps + step = recipe.steps["make_plan"] + assert step.tool == "run_skill" + skill_cmd = step.with_args.get("skill_command", "") + assert "/autoskillit:make-plan" in skill_cmd + assert "context.remediation_path" in skill_cmd + assert "plan_path" in step.capture + assert "plan_parts" in step.capture_list + assert step.on_success == "dry_walkthrough" + assert step.on_failure == "release_issue_failure" + + def test_if3_test_step_uses_implementation_ref(self, recipe) -> None: + """T_IF3: test step worktree_path must reference context.implementation_ref.""" + worktree_arg = recipe.steps["test"].with_args.get("worktree_path", "") + assert "context.implementation_ref" in worktree_arg + assert "context.work_dir" not in worktree_arg + + def test_if4_merge_step_uses_implementation_ref(self, recipe) -> None: + """T_IF4: merge step worktree_path must reference context.implementation_ref.""" + worktree_arg = recipe.steps["merge"].with_args.get("worktree_path", "") + assert "context.implementation_ref" in worktree_arg + assert "context.work_dir" not in worktree_arg + + def test_if_b1_implement_captures_branch_name(self, recipe) -> None: + """T_IF_B1: implement step must capture branch_name from result.""" + step = recipe.steps["implement"] + assert "branch_name" in step.capture, ( + "implement step must capture branch_name so audit_impl can pass a " + "stable git ref to audit-impl after merge_worktree deletes the worktree" + ) + + def test_if_b2_audit_impl_uses_branch_name_as_ref(self, recipe) -> None: + """T_IF_B2: audit_impl with: must reference context.branch_name as implementation_ref.""" + step = recipe.steps["audit_impl"] + skill_cmd = step.with_args.get("skill_command", "") + assert "context.branch_name" in skill_cmd, ( + "audit_impl must pass context.branch_name as implementation_ref — not " + "context.implementation_ref or context.worktree_path (stale after merge)" + ) + + def test_if_b3_retry_worktree_captures_branch_name(self, recipe) -> None: + """T_IF_B3: retry_worktree step must also capture branch_name.""" + step = recipe.steps["retry_worktree"] + assert "branch_name" in step.capture, ( + "retry_worktree also updates the active worktree reference; " + "it must capture branch_name for downstream audit_impl use" + ) + + def test_create_branch_uses_callable(self, recipe) -> None: + """create_and_publish must use create_and_publish_branch MCP tool.""" + step = recipe.steps["create_and_publish"] + assert step.tool == "create_and_publish_branch" + + def test_create_branch_checks_remote_for_collisions(self, recipe) -> None: + """create_and_publish uses create_and_publish_branch (which checks ls-remote).""" + assert recipe.steps["create_and_publish"].tool == "create_and_publish_branch" + + def test_create_branch_references_issue_number(self, recipe) -> None: + """create_and_publish with_args must include issue_number for branch naming.""" + assert "issue_number" in recipe.steps["create_and_publish"].with_args + + def test_create_branch_uses_run_name_as_prefix(self, recipe) -> None: + """create_and_publish with_args must include run_name for branch naming.""" + assert "run_name" in recipe.steps["create_and_publish"].with_args + + def test_if_push_after_audit_warning_fires(self, recipe) -> None: + """push-before-audit semantic rule fires as WARNING (audit has skip_when_false).""" + findings = run_semantic_rules(recipe) + violations = [f for f in findings if f.rule == "push-before-audit"] + assert len(violations) >= 1 + assert all(v.severity == Severity.WARNING for v in violations) + + def test_if_c1_implement_uses_no_merge_skill(self, recipe) -> None: + """T_IF_C1: implement step must use implement-worktree-no-merge. + + implement-worktree merges and deletes the worktree internally; subsequent + verify (test_check) and assess (resolve-failures) steps would run against + a non-existent path. implement-worktree-no-merge leaves the worktree intact + for the orchestrator's gate-test-merge cycle. + """ + step = recipe.steps["implement"] + skill_cmd = step.with_args.get("skill_command", "") + assert "implement-worktree-no-merge" in skill_cmd, ( + "implement step must use implement-worktree-no-merge; " + "implement-worktree merges immediately, making verify and assess unreachable" + ) + + def test_remediation_investigate_captures_investigation_path(self, recipe) -> None: + """1c: investigate step must have a capture block containing investigation_path.""" + step = recipe.steps["investigate"] + assert step.capture is not None and "investigation_path" in step.capture, ( + "investigate step must capture investigation_path so rectify receives " + "the explicit path rather than scanning the filesystem" + ) + assert step.capture["investigation_path"] == "${{ result.investigation_path }}" + + def test_remediation_rectify_uses_context_investigation_path(self, recipe) -> None: + """1d: rectify step must pass ${{ context.investigation_path }} in skill_command.""" + step = recipe.steps["rectify"] + skill_cmd = step.with_args.get("skill_command", "") + assert "${{ context.investigation_path }}" in skill_cmd, ( + "rectify step skill_command must include ${{ context.investigation_path }} " + "to pass the explicit path from the capture block" + ) + + def test_if_resolve_review_uses_resolve_review_skill(self, recipe) -> None: + """T_IF_RR1: resolve_review step must invoke resolve-review, not resolve-failures. + + resolve-failures is test-driven and finds no work when tests are green. + resolve-review reads PR review comments and applies the reviewer's requested changes. + """ + assert "resolve_review" in recipe.steps + step = recipe.steps["resolve_review"] + assert step.tool == "run_skill" + skill_cmd = step.with_args.get("skill_command", "") + assert "resolve-review" in skill_cmd, ( + "resolve_review step must call /autoskillit:resolve-review; " + "resolve-failures is test-driven and ignores PR review comments" + ) + assert "resolve-failures" not in skill_cmd, ( + "resolve_review step must not call resolve-failures; " + "that skill does not read review comments" + ) + + def test_if_resolve_review_passes_merge_target(self, recipe) -> None: + """T_IF_RR2: resolve_review skill_command must pass context.merge_target.""" + step = recipe.steps["resolve_review"] + skill_cmd = step.with_args.get("skill_command", "") + assert "context.merge_target" in skill_cmd, ( + "resolve-review requires feature_branch as first arg; " + "context.merge_target holds the feature branch name" + ) + + def test_remediation_no_add_dir_dead_param(self, recipe) -> None: + """Remediation recipe must not have add_dir as a with: key (removed dead param).""" + findings = run_semantic_rules(recipe) + add_dir_dead = [ + f for f in findings if f.rule == "dead-with-param" and "add_dir" in f.message + ] + assert not add_dir_dead, ( + f"Remediation recipe still has dead add_dir param: " + f"{[(f.step_name, f.message) for f in add_dir_dead]}" + ) + + def test_remediation_assess_step_has_on_context_limit(self, recipe) -> None: + """REQ-RCP-002: assess step in remediation.yaml must declare on_context_limit: test. + + assess runs resolve-failures inside an existing worktree. Partial fixes are committed + to disk, so routing to test checks whether partial work was sufficient — same rationale + as the fix step in implementation.yaml. + """ + assess = recipe.steps["assess"] + assert assess.on_context_limit == "test", ( + f"remediation.yaml assess step must declare on_context_limit: test, " + f"got: {assess.on_context_limit!r}" + ) diff --git a/tests/recipe/test_bundled_recipes_research.py b/tests/recipe/test_bundled_recipes_research.py new file mode 100644 index 0000000000..239e5195d1 --- /dev/null +++ b/tests/recipe/test_bundled_recipes_research.py @@ -0,0 +1,475 @@ +"""Tests for research.yaml bundled recipe structure and archival steps.""" + +from __future__ import annotations + +import pytest + +from autoskillit.recipe.io import builtin_recipes_dir, load_recipe +from autoskillit.recipe.validator import validate_recipe_structure + +pytestmark = [pytest.mark.layer("recipe"), pytest.mark.small] + + +class TestResearchRecipeStructure: + @pytest.fixture(scope="class") + def recipe(self): + return load_recipe(builtin_recipes_dir() / "research.yaml") + + def test_research_has_review_pr_ingredient(self, recipe) -> None: + """research.yaml must declare a review_pr ingredient with default 'false'.""" + assert "review_pr" in recipe.ingredients, ( + "research.yaml must declare a 'review_pr' ingredient to gate the optional " + "review-research-pr step" + ) + assert recipe.ingredients["review_pr"].default == "false" + + def test_research_has_review_research_pr_step(self, recipe) -> None: + """research.yaml must include a review_research_pr step.""" + assert "review_research_pr" in recipe.steps + + def test_research_review_step_skip_when_false(self, recipe) -> None: + """review_research_pr step must use skip_when_false: inputs.review_pr.""" + step = recipe.steps["review_research_pr"] + assert step.skip_when_false == "inputs.review_pr" + + def test_research_no_issue_number_ingredient(self, recipe) -> None: + """Removed: issue_number is the phase-2 gate.""" + assert "issue_number" not in recipe.ingredients + + def test_research_no_setup_phases_ingredient(self, recipe) -> None: + """Removed: setup_phases toggle replaced by always-decompose.""" + assert "setup_phases" not in recipe.ingredients + + def test_research_no_check_phase_step(self, recipe) -> None: + assert "check_phase" not in recipe.steps + + def test_research_no_phase1_done_step(self, recipe) -> None: + assert "phase1_done" not in recipe.steps + + def test_research_no_open_plan_issue_step(self, recipe) -> None: + assert "open_plan_issue" not in recipe.steps + + def test_research_no_save_experiment_plan_step(self, recipe) -> None: + assert "save_experiment_plan" not in recipe.steps + + def test_research_no_check_setup_needed_step(self, recipe) -> None: + assert "check_setup_needed" not in recipe.steps + + def test_research_no_implement_experiment_step(self, recipe) -> None: + assert "implement_experiment" not in recipe.steps + + def test_research_has_issue_url_ingredient(self, recipe) -> None: + assert "issue_url" in recipe.ingredients + assert not recipe.ingredients["issue_url"].required + + def test_research_has_review_design_ingredient(self, recipe) -> None: + assert "review_design" in recipe.ingredients + assert recipe.ingredients["review_design"].default == "true" + + def test_research_has_review_design_step(self, recipe) -> None: + assert "review_design" in recipe.steps + + def test_review_design_step_skip_when_false(self, recipe) -> None: + step = recipe.steps["review_design"] + assert step.skip_when_false == "inputs.review_design" + + def test_review_design_step_retries_and_exhausted(self, recipe) -> None: + step = recipe.steps["review_design"] + assert step.retries == 2 + assert step.on_exhausted == "create_worktree" + + def test_plan_experiment_routes_to_review_design(self, recipe) -> None: + step = recipe.steps["plan_experiment"] + assert step.on_success == "review_design" + + def test_review_design_on_result_routing(self, recipe) -> None: + """review_design STOP verdict routes to resolve_design_review (not design_rejected).""" + step = recipe.steps["review_design"] + assert step.on_result is not None + go_cond = next((c for c in step.on_result.conditions if c.when and "GO" in c.when), None) + assert go_cond is not None, "Missing GO route" + assert go_cond.route == "plan_visualization" + revise_cond = next( + (c for c in step.on_result.conditions if c.when and "REVISE" in c.when), None + ) + assert revise_cond is not None, "Missing REVISE route" + assert revise_cond.route == "revise_design" + stop_cond = next( + (c for c in step.on_result.conditions if c.when and "STOP" in c.when), None + ) + assert stop_cond is not None, "Missing STOP route" + assert stop_cond.route == "resolve_design_review" + + def test_has_revise_design_step(self, recipe) -> None: + assert "revise_design" in recipe.steps + + def test_has_design_rejected_step(self, recipe) -> None: + assert "design_rejected" in recipe.steps + + def test_design_rejected_step_is_action_stop(self, recipe) -> None: + """design_rejected must halt the pipeline (action=stop) with an explanatory message. + + Regression guard: changing action to 'route' would silently break + the hard-halt guarantee for fundamentally flawed designs. + """ + step = recipe.steps["design_rejected"] + assert step.action == "stop" + assert step.message, "design_rejected must have a non-empty message" + assert "STOP" in step.message, ( + "design_rejected message must reference the STOP verdict for clarity." + ) + + def test_kitchen_rule_6_acknowledges_stop_verdict_exception(self, recipe) -> None: + """Kitchen rule #6 must scope 'do not stop' to exhaustion/failure, not STOP verdict. + + The current blanket 'do not stop' language contradicts the design_rejected + hard halt. The rule must explicitly note that verdict=STOP is the sole exception. + """ + rule6 = recipe.kitchen_rules[5] # 0-indexed; rule #6 is index 5 + rule6_lower = rule6.lower() + assert "stop" in rule6_lower, ( + "Kitchen rule #6 must reference 'stop' in the verdict exception context." + ) + assert "verdict" in rule6_lower or "exception" in rule6_lower, ( + "Kitchen rule #6 must acknowledge that verdict=STOP is an exception to " + "'do not stop on exhaustion/failure'. Current language is overbroad and " + "contradicts the design_rejected routing." + ) + + def test_has_resolve_design_review_step(self, recipe) -> None: + """resolve_design_review step must be present in research.yaml.""" + assert "resolve_design_review" in recipe.steps + + def test_resolve_design_review_routes_revised_to_revise_design(self, recipe) -> None: + """resolve_design_review routes resolution=revised back to revise_design.""" + step = recipe.steps["resolve_design_review"] + assert step.on_result is not None + revised_cond = next( + (c for c in step.on_result.conditions if c.when and "revised" in c.when), None + ) + assert revised_cond is not None, "Missing revised route" + assert revised_cond.route == "revise_design" + + def test_resolve_design_review_routes_failed_to_design_rejected(self, recipe) -> None: + """resolve_design_review routes resolution=failed to design_rejected.""" + step = recipe.steps["resolve_design_review"] + assert step.on_result is not None + failed_cond = next( + (c for c in step.on_result.conditions if c.when and "failed" in c.when), None + ) + assert failed_cond is not None, "Missing failed route" + assert failed_cond.route == "design_rejected" + + def test_resolve_design_review_fallbacks_to_design_rejected(self, recipe) -> None: + """resolve_design_review routes on_failure and on_context_limit to design_rejected.""" + step = recipe.steps["resolve_design_review"] + assert step.on_failure == "design_rejected" + assert step.on_context_limit == "design_rejected" + + def test_resolve_design_review_captures_revision_guidance(self, recipe) -> None: + """resolve_design_review must capture revision_guidance for the revise_design loop.""" + step = recipe.steps["resolve_design_review"] + assert "revision_guidance" in step.capture, ( + "resolve_design_review must capture revision_guidance so revise_design → " + "plan_experiment picks up the triage-generated guidance" + ) + + def test_has_resolve_research_review_step(self, recipe) -> None: + step = recipe.steps["resolve_research_review"] + assert step.retries == 2 + assert step.on_exhausted == "route_claims_resolve" + assert step.on_success == "route_claims_resolve" + assert step.on_failure == "route_claims_resolve" + + def test_has_re_push_research_step(self, recipe) -> None: + assert "re_push_research" in recipe.steps + + def test_requires_packs_includes_exp_lens(self, recipe) -> None: + assert "exp-lens" in recipe.requires_packs + assert "research" in recipe.requires_packs + + def test_re_run_experiment_step(self, recipe) -> None: + assert "re_run_experiment" in recipe.steps + step = recipe.steps["re_run_experiment"] + assert step.tool == "run_skill" + assert "--adjust" in step.with_args.get("skill_command", "") + assert step.on_success == "re_generate_report" + + def test_re_generate_report_step(self, recipe) -> None: + assert "re_generate_report" in recipe.steps + step = recipe.steps["re_generate_report"] + assert step.tool == "run_skill" + assert step.on_success == "re_stage_bundle" + + def test_re_test_step(self, recipe) -> None: + assert "re_test" in recipe.steps + step = recipe.steps["re_test"] + assert step.tool == "test_check" + assert step.on_success == "re_push_research" + + def test_revalidation_loop_all_paths_reach_begin_archival(self, recipe) -> None: + """Every path from merge_escalations reaches begin_archival.""" + for step_name in ("re_run_experiment", "re_generate_report", "re_test"): + step = recipe.steps[step_name] + assert step.on_failure in ("begin_archival", "re_push_research") + assert recipe.steps["re_push_research"].on_success == "finalize_bundle_render" + + def test_audit_claims_ingredient_default_false(self, recipe) -> None: + assert "audit_claims" in recipe.ingredients + assert recipe.ingredients["audit_claims"].required is False + assert recipe.ingredients["audit_claims"].default == "false" + + def test_review_research_pr_captures_review_verdict(self, recipe) -> None: + step = recipe.steps["review_research_pr"] + assert "review_verdict" in step.capture + assert "verdict" not in step.capture # old key must be gone + + def test_audit_claims_step_routes_to_route_review_resolve(self, recipe) -> None: + step = recipe.steps["audit_claims"] + assert step.tool == "run_skill" + assert "audit_claims" in step.skip_when_false + # all on_result routes point to route_review_resolve + routes = {c.route for c in step.on_result.conditions} + assert routes == {"route_review_resolve"} + + def test_new_routing_steps_exist(self, recipe) -> None: + assert "route_review_resolve" in recipe.steps + assert "route_claims_resolve" in recipe.steps + assert "merge_escalations" in recipe.steps + assert "check_escalations" not in recipe.steps # replaced + + def test_route_review_resolve_routing_logic(self, recipe) -> None: + step = recipe.steps["route_review_resolve"] + assert step.action == "route" + conditions = step.on_result.conditions + # changes_requested → resolve_research_review; else → route_claims_resolve + guarded = [c for c in conditions if c.when is not None] + default = [c for c in conditions if c.when is None] + assert len(guarded) == 1 + assert "changes_requested" in guarded[0].when + assert guarded[0].route == "resolve_research_review" + assert len(default) == 1 + assert default[0].route == "route_claims_resolve" + + def test_route_claims_resolve_routing_logic(self, recipe) -> None: + step = recipe.steps["route_claims_resolve"] + assert step.action == "route" + conditions = step.on_result.conditions + # changes_requested → resolve_claims_review; else → merge_escalations + guarded = [c for c in conditions if c.when is not None] + default = [c for c in conditions if c.when is None] + assert len(guarded) == 1 + assert "changes_requested" in guarded[0].when + assert guarded[0].route == "resolve_claims_review" + assert len(default) == 1 + assert default[0].route == "merge_escalations" + + def test_merge_escalations_routing_logic(self, recipe) -> None: + step = recipe.steps["merge_escalations"] + assert step.action == "route" + conditions = step.on_result.conditions + # review_needs_rerun == true → re_run_experiment + # claims_needs_rerun == true → re_run_experiment + # else → finalize_bundle + guarded = [c for c in conditions if c.when is not None] + default = [c for c in conditions if c.when is None] + assert len(guarded) == 2 + rerun_routes = {c.route for c in guarded} + assert rerun_routes == {"re_run_experiment"} + assert any("review_needs_rerun" in c.when for c in guarded) + assert any("claims_needs_rerun" in c.when for c in guarded) + assert len(default) == 1 + assert default[0].route == "finalize_bundle" + + def test_resolve_claims_review_step_exists(self, recipe) -> None: + step = recipe.steps["resolve_claims_review"] + assert "claims_needs_rerun" in step.capture + + def test_resolve_research_review_captures_review_needs_rerun(self, recipe) -> None: + step = recipe.steps["resolve_research_review"] + assert "review_needs_rerun" in step.capture + assert "needs_rerun" not in step.capture # old key must be gone + + def test_research_recipe_validates_cleanly_with_new_steps(self, recipe) -> None: + errors = validate_recipe_structure(recipe) + assert errors == [], f"Validation errors: {errors}" + + # --- Archival phase tests --- + + def test_archival_begin_archival_step_exists(self, recipe) -> None: + """begin_archival must be a route step gating archival on pr_url.""" + assert "begin_archival" in recipe.steps + step = recipe.steps["begin_archival"] + assert step.action == "route" + + def test_archival_begin_archival_routes_to_capture(self, recipe) -> None: + """begin_archival routes to capture_experiment_branch when pr_url is truthy.""" + step = recipe.steps["begin_archival"] + assert step.on_result is not None + conditions = step.on_result.conditions + truthy_route = next((c for c in conditions if c.when and "pr_url" in c.when), None) + assert truthy_route is not None + assert truthy_route.route == "capture_experiment_branch" + + def test_archival_begin_archival_default_to_complete(self, recipe) -> None: + """begin_archival default route goes to research_complete.""" + step = recipe.steps["begin_archival"] + conditions = step.on_result.conditions + default = next((c for c in conditions if not c.when), None) + assert default is not None + assert default.route == "patch_token_summary" + + def test_archival_capture_experiment_branch_step(self, recipe) -> None: + """capture_experiment_branch captures experiment_branch from git rev-parse.""" + assert "capture_experiment_branch" in recipe.steps + step = recipe.steps["capture_experiment_branch"] + assert step.tool == "run_cmd" + assert "experiment_branch" in step.capture + assert step.on_success == "create_artifact_branch" + assert step.on_failure == "patch_token_summary" + + def test_archival_create_artifact_branch_step(self, recipe) -> None: + """create_artifact_branch creates temp worktree and captures artifact_branch.""" + assert "create_artifact_branch" in recipe.steps + step = recipe.steps["create_artifact_branch"] + assert step.tool == "run_cmd" + assert "artifact_branch" in step.capture + assert step.on_success == "open_artifact_pr" + assert step.on_failure == "patch_token_summary" + + def test_archival_create_artifact_branch_uses_external_script(self, recipe) -> None: + """create_artifact_branch must invoke the externalized shell script.""" + step = recipe.steps["create_artifact_branch"] + cmd = step.with_args["cmd"] + assert "create_artifact_branch.sh" in cmd + assert "context.research_dir" in cmd + + def test_archival_open_artifact_pr_step(self, recipe) -> None: + """open_artifact_pr creates the artifact-only PR and captures artifact_pr_url.""" + assert "open_artifact_pr" in recipe.steps + step = recipe.steps["open_artifact_pr"] + assert step.tool == "run_cmd" + assert "artifact_pr_url" in step.capture + assert step.on_success == "tag_experiment_branch" + assert step.on_failure == "patch_token_summary" + + def test_archival_tag_experiment_branch_step(self, recipe) -> None: + """tag_experiment_branch creates annotated archive tag and captures archive_tag.""" + assert "tag_experiment_branch" in recipe.steps + step = recipe.steps["tag_experiment_branch"] + assert step.tool == "run_cmd" + assert "archive_tag" in step.capture + assert step.on_success == "close_experiment_pr" + assert step.on_failure == "patch_token_summary" + + def test_archival_tag_uses_external_script(self, recipe) -> None: + """tag_experiment_branch must invoke the externalized shell script.""" + step = recipe.steps["tag_experiment_branch"] + cmd = step.with_args["cmd"] + assert "tag_experiment_branch.sh" in cmd + + def test_archival_close_experiment_pr_step(self, recipe) -> None: + """close_experiment_pr closes the original PR and routes to research_complete.""" + assert "close_experiment_pr" in recipe.steps + step = recipe.steps["close_experiment_pr"] + assert step.tool == "run_cmd" + assert step.on_success == "patch_token_summary" + assert step.on_failure == "patch_token_summary" + + def test_archival_close_pr_references_artifact_and_tag(self, recipe) -> None: + """close_experiment_pr comment must reference both artifact PR and archive tag.""" + step = recipe.steps["close_experiment_pr"] + cmd = step.with_args["cmd"] + assert "artifact_pr_url" in cmd, "Must reference artifact PR URL" + assert "archive_tag" in cmd, "Must reference archive tag" + + def test_archival_graceful_degradation(self, recipe) -> None: + """Every archival step must route on_failure to research_complete.""" + archival_steps = [ + "capture_experiment_branch", + "create_artifact_branch", + "open_artifact_pr", + "tag_experiment_branch", + "close_experiment_pr", + ] + for name in archival_steps: + step = recipe.steps[name] + assert step.on_failure == "patch_token_summary", ( + f"{name}.on_failure must be patch_token_summary for graceful degradation" + ) + + def test_re_push_research_routes_to_finalize_bundle_render(self, recipe) -> None: + """re_push_research routes to finalize_bundle_render on success.""" + step = recipe.steps["re_push_research"] + assert step.on_success == "finalize_bundle_render" + assert step.on_failure == "begin_archival" + + def test_finalize_bundle_routes_to_re_push_research(self, recipe) -> None: + """finalize_bundle on_success routes to re_push_research (push includes the commit).""" + step = recipe.steps["finalize_bundle"] + assert step.on_success == "re_push_research" + assert step.on_failure == "begin_archival" + + def test_finalize_bundle_render_step_exists_and_routes(self, recipe) -> None: + """finalize_bundle_render routes to route_archive_or_export on both outcomes.""" + assert "finalize_bundle_render" in recipe.steps + step = recipe.steps["finalize_bundle_render"] + assert step.on_success == "route_archive_or_export" + assert step.on_failure == "route_archive_or_export" + + def test_create_worktree_uses_external_script(self, recipe) -> None: + """create_worktree must invoke the externalized shell script.""" + step = recipe.steps["create_worktree"] + cmd = step.with_args["cmd"] + assert "create_worktree.sh" in cmd + assert "context.experiment_plan" in cmd + + def test_create_worktree_passes_source_dir(self, recipe) -> None: + """create_worktree must pass source_dir to the external script.""" + step = recipe.steps["create_worktree"] + cmd = step.with_args["cmd"] + assert "inputs.source_dir" in cmd + + def test_stage_bundle_step_exists(self, recipe) -> None: + """A stage_bundle step must exist to organize phase artifacts.""" + assert "stage_bundle" in recipe.steps, ( + "research.yaml must have a stage_bundle step for phase artifact staging" + ) + step = recipe.steps["stage_bundle"] + cmd = step.with_args["cmd"] + assert "stage_bundle.sh" in cmd, "Must invoke externalized stage_bundle script" + + def test_test_routes_to_push_branch(self, recipe) -> None: + """test step must route to push_branch now that commit_research_artifacts is removed.""" + step = recipe.steps["test"] + assert step.on_success == "push_branch", ( + "test.on_success must be push_branch after commit_research_artifacts is removed" + ) + + def test_retest_routes_to_push_branch(self, recipe) -> None: + """retest step must route to push_branch now that commit_research_artifacts is removed.""" + step = recipe.steps["retest"] + assert step.on_success == "push_branch", ( + "retest.on_success must be push_branch after commit_research_artifacts is removed" + ) + + def test_push_branch_cmd_externalizes_guard(self, recipe) -> None: + """push_branch cmd must invoke push_branch.sh and forward output_mode. + + Regression guard: ADR-0002 bans inline shell control flow in cmd fields. + The local-mode guard lives in scripts/recipe/push_branch.sh. Issue #831. + """ + step = recipe.steps["push_branch"] + cmd = step.with_args["cmd"] + assert "push_branch.sh" in cmd, ( + "push_branch cmd must invoke scripts/recipe/push_branch.sh (ADR-0002)" + ) + assert "${{ inputs.output_mode }}" in cmd, ( + "push_branch cmd must forward inputs.output_mode to the script" + ) + + def test_stage_bundle_routes_to_route_pr_or_local(self, recipe) -> None: + """stage_bundle must route to route_pr_or_local on success and failure.""" + step = recipe.steps["stage_bundle"] + assert step.on_success == "route_pr_or_local" + assert step.on_failure == "route_pr_or_local" diff --git a/tests/recipe/test_bundled_recipes_research_design.py b/tests/recipe/test_bundled_recipes_research_design.py new file mode 100644 index 0000000000..bded2d643a --- /dev/null +++ b/tests/recipe/test_bundled_recipes_research_design.py @@ -0,0 +1,266 @@ +from __future__ import annotations + +import pytest + +from autoskillit.recipe.io import builtin_recipes_dir, load_recipe +from autoskillit.recipe.validator import validate_recipe_structure + +pytestmark = [pytest.mark.layer("recipe"), pytest.mark.small] + + +class TestResearchDesignRecipeStructure: + @pytest.fixture(scope="class") + def recipe(self): + return load_recipe(builtin_recipes_dir() / "research-design.yaml") + + def test_loads_without_exception(self, recipe) -> None: + assert recipe is not None + + def test_validates_with_zero_errors(self, recipe) -> None: + errors = validate_recipe_structure(recipe) + assert errors == [], f"Validation errors: {errors}" + + def test_recipe_name(self, recipe) -> None: + assert recipe.name == "research-design" + + def test_recipe_version(self, recipe) -> None: + assert recipe.recipe_version == "1.0.0" + + def test_categories(self, recipe) -> None: + assert recipe.categories == ["research-family"] + + def test_requires_packs(self, recipe) -> None: + assert recipe.requires_packs == ["research", "vis-lens"] + + def test_ingredient_count(self, recipe) -> None: + assert len(recipe.ingredients) == 5 + + def test_task_ingredient_required(self, recipe) -> None: + assert "task" in recipe.ingredients + assert recipe.ingredients["task"].required is True + + def test_source_dir_ingredient_required(self, recipe) -> None: + assert "source_dir" in recipe.ingredients + assert recipe.ingredients["source_dir"].required is True + + def test_base_branch_ingredient_default(self, recipe) -> None: + assert "base_branch" in recipe.ingredients + assert recipe.ingredients["base_branch"].default == "main" + + def test_review_design_ingredient_default(self, recipe) -> None: + assert "review_design" in recipe.ingredients + assert recipe.ingredients["review_design"].default == "true" + + def test_issue_url_ingredient_optional(self, recipe) -> None: + assert "issue_url" in recipe.ingredients + assert recipe.ingredients["issue_url"].required is False + + def test_step_count(self, recipe) -> None: + assert len(recipe.steps) == 10 + + def test_step_names(self, recipe) -> None: + expected = { + "scope", + "plan_experiment", + "review_design", + "plan_visualization", + "revise_design", + "check_design_review_loop", + "resolve_design_review", + "design_rejected", + "design_complete", + "escalate_stop", + } + assert set(recipe.steps.keys()) == expected + + def test_scope_routing(self, recipe) -> None: + step = recipe.steps["scope"] + assert step.on_success == "plan_experiment" + assert step.on_failure == "escalate_stop" + + def test_scope_captures_scope_report(self, recipe) -> None: + assert "scope_report" in recipe.steps["scope"].capture + + def test_plan_experiment_routing(self, recipe) -> None: + step = recipe.steps["plan_experiment"] + assert step.on_success == "review_design" + assert step.on_failure == "escalate_stop" + + def test_plan_experiment_captures_experiment_plan(self, recipe) -> None: + assert "experiment_plan" in recipe.steps["plan_experiment"].capture + + def test_plan_experiment_optional_context_refs(self, recipe) -> None: + assert "revision_guidance" in recipe.steps["plan_experiment"].optional_context_refs + + def test_review_design_skip_when_false(self, recipe) -> None: + assert recipe.steps["review_design"].skip_when_false == "inputs.review_design" + + def test_review_design_retries(self, recipe) -> None: + assert recipe.steps["review_design"].retries == 2 + + def test_review_design_on_exhausted(self, recipe) -> None: + assert recipe.steps["review_design"].on_exhausted == "plan_visualization" + + def test_review_design_on_context_limit(self, recipe) -> None: + assert recipe.steps["review_design"].on_context_limit == "plan_visualization" + + def test_review_design_on_failure(self, recipe) -> None: + assert recipe.steps["review_design"].on_failure == "plan_visualization" + + def test_review_design_on_result_go(self, recipe) -> None: + step = recipe.steps["review_design"] + assert step.on_result is not None + go_cond = next((c for c in step.on_result.conditions if c.when and "GO" in c.when), None) + assert go_cond is not None, "Missing GO route" + assert go_cond.route == "plan_visualization" + + def test_review_design_on_result_revise(self, recipe) -> None: + step = recipe.steps["review_design"] + assert step.on_result is not None + revise_cond = next( + (c for c in step.on_result.conditions if c.when and "REVISE" in c.when), None + ) + assert revise_cond is not None, "Missing REVISE route" + assert revise_cond.route == "revise_design" + + def test_review_design_on_result_stop(self, recipe) -> None: + step = recipe.steps["review_design"] + assert step.on_result is not None + stop_cond = next( + (c for c in step.on_result.conditions if c.when and "STOP" in c.when), None + ) + assert stop_cond is not None, "Missing STOP route" + assert stop_cond.route == "resolve_design_review" + + def test_review_design_on_result_fallback(self, recipe) -> None: + step = recipe.steps["review_design"] + assert step.on_result is not None + fallback = next((c for c in step.on_result.conditions if c.when is None), None) + assert fallback is not None, "Missing fallback route" + assert fallback.route == "plan_visualization" + + def test_review_design_no_on_success(self, recipe) -> None: + assert recipe.steps["review_design"].on_success is None + + def test_review_design_captures(self, recipe) -> None: + capture = recipe.steps["review_design"].capture + for key in ("verdict", "experiment_type", "evaluation_dashboard", "revision_guidance"): + assert key in capture, f"Missing capture key: {key}" + + def test_plan_visualization_on_success(self, recipe) -> None: + assert recipe.steps["plan_visualization"].on_success == "design_complete" + + def test_plan_visualization_on_failure(self, recipe) -> None: + assert recipe.steps["plan_visualization"].on_failure == "escalate_stop" + + def test_plan_visualization_captures(self, recipe) -> None: + capture = recipe.steps["plan_visualization"].capture + assert "visualization_plan_path" in capture + assert "report_plan_path" in capture + + def test_revise_design_is_route_action(self, recipe) -> None: + assert recipe.steps["revise_design"].action == "route" + + def test_revise_design_routes_to_check_design_review_loop(self, recipe) -> None: + step = recipe.steps["revise_design"] + assert step.on_result is not None + default = next((c for c in step.on_result.conditions if c.when is None), None) + assert default is not None + assert default.route == "check_design_review_loop" + + def test_resolve_design_review_retries(self, recipe) -> None: + assert recipe.steps["resolve_design_review"].retries == 1 + + def test_resolve_design_review_on_failure(self, recipe) -> None: + assert recipe.steps["resolve_design_review"].on_failure == "design_rejected" + + def test_resolve_design_review_on_context_limit(self, recipe) -> None: + assert recipe.steps["resolve_design_review"].on_context_limit == "design_rejected" + + def test_resolve_design_review_on_result_revised(self, recipe) -> None: + step = recipe.steps["resolve_design_review"] + assert step.on_result is not None + revised = next( + (c for c in step.on_result.conditions if c.when and "revised" in c.when), None + ) + assert revised is not None, "Missing revised route" + assert revised.route == "revise_design" + + def test_resolve_design_review_on_result_failed(self, recipe) -> None: + step = recipe.steps["resolve_design_review"] + assert step.on_result is not None + failed = next( + (c for c in step.on_result.conditions if c.when and "failed" in c.when), None + ) + assert failed is not None, "Missing failed route" + assert failed.route == "design_rejected" + + def test_resolve_design_review_fallback(self, recipe) -> None: + step = recipe.steps["resolve_design_review"] + assert step.on_result is not None + fallback = next((c for c in step.on_result.conditions if c.when is None), None) + assert fallback is not None, "Missing fallback route" + assert fallback.route == "design_rejected" + + def test_resolve_design_review_captures_revision_guidance(self, recipe) -> None: + assert "revision_guidance" in recipe.steps["resolve_design_review"].capture + + def test_design_rejected_is_stop(self, recipe) -> None: + step = recipe.steps["design_rejected"] + assert step.action == "stop" + assert step.message, "design_rejected must have a non-empty message" + + def test_design_complete_is_stop(self, recipe) -> None: + step = recipe.steps["design_complete"] + assert step.action == "stop" + assert step.message, "design_complete must have a non-empty message" + + def test_design_complete_sentinel_fields(self, recipe) -> None: + message = recipe.steps["design_complete"].message + for field in ( + "scope_report", + "experiment_plan", + "visualization_plan_path", + "report_plan_path", + "experiment_type", + ): + assert field in message, ( + f"design_complete message must mention sentinel field: {field}" + ) + + def test_escalate_stop_is_stop(self, recipe) -> None: + step = recipe.steps["escalate_stop"] + assert step.action == "stop" + assert step.message, "escalate_stop must have a non-empty message" + + def test_no_dangling_create_worktree(self, recipe) -> None: + for name, step in recipe.steps.items(): + assert "create_worktree" not in name, f"Step name '{name}' references create_worktree" + for attr in ( + step.on_success, + step.on_failure, + step.on_exhausted, + step.on_context_limit, + ): + if attr: + assert "create_worktree" not in attr, ( + f"Step '{name}' routes to create_worktree via {attr!r}" + ) + if step.on_result: + for cond in step.on_result.conditions: + assert "create_worktree" not in cond.route, ( + f"Step '{name}' on_result routes to create_worktree" + ) + + def test_kitchen_rules_count(self, recipe) -> None: + assert len(recipe.kitchen_rules) == 2 + + def test_kitchen_rule_no_native_tools(self, recipe) -> None: + rule = recipe.kitchen_rules[0] + for tool in ("Read", "Grep", "Glob", "Edit", "Write", "Bash"): + assert tool in rule, f"Kitchen rule 1 must forbid native tool: {tool}" + + def test_kitchen_rule_food_truck_sentinel(self, recipe) -> None: + rule = recipe.kitchen_rules[1].lower() + assert "food truck" in rule, "Kitchen rule 2 must mention food truck" + assert "sentinel" in rule, "Kitchen rule 2 must mention sentinel emission" diff --git a/tests/recipe/test_bundled_recipes_review_pr.py b/tests/recipe/test_bundled_recipes_review_pr.py new file mode 100644 index 0000000000..d6c969cbf7 --- /dev/null +++ b/tests/recipe/test_bundled_recipes_review_pr.py @@ -0,0 +1,311 @@ +"""Tests for review-PR integration across pipeline recipe variants.""" + +from __future__ import annotations + +import pytest + +from autoskillit.recipe.io import builtin_recipes_dir, load_recipe + +pytestmark = [pytest.mark.layer("recipe"), pytest.mark.small] + + +class TestReviewPrRecipeIntegration: + @pytest.fixture( + scope="class", + params=[ + "implementation.yaml", + "implementation-groups.yaml", + "remediation.yaml", + ], + ) + def recipe(self, request: pytest.FixtureRequest) -> object: + return load_recipe(builtin_recipes_dir() / request.param) + + def test_compose_pr_routes_to_extract_pr_number(self, recipe: object) -> None: + """T_RP1: compose_pr.on_success routes per-recipe to extract_pr_number. + + All queue-aware recipes (implementation, remediation, implementation-groups) insert + extract_pr_number between compose_pr and review_pr to capture the PR number for + merge queue support. + """ + recipe_name = recipe.name # type: ignore[attr-defined] + on_success = recipe.steps["compose_pr"].on_success # type: ignore[attr-defined] + assert on_success == "extract_pr_number", ( + f"{recipe_name}: compose_pr.on_success must be 'extract_pr_number', got {on_success!r}" + ) + + def test_review_pr_step_exists_and_is_run_skill(self, recipe: object) -> None: + """T_RP2: review_pr step exists and uses run_skill tool.""" + step = recipe.steps["review_pr"] # type: ignore[attr-defined] + assert step.tool == "run_skill" + + def test_review_pr_skipped_when_open_pr_false(self, recipe: object) -> None: + """T_RP3: review_pr is gated by inputs.open_pr (skip_when_false).""" + step = recipe.steps["review_pr"] # type: ignore[attr-defined] + assert step.skip_when_false == "inputs.open_pr" + + def test_review_pr_catch_all_routes_to_check_review_loop(self, recipe: object) -> None: + """T_RP4: catch-all routes through check_review_loop (mandatory waypoint). + + All verdicts (approved, needs_human, any future verdict) must pass through + check_review_loop to ensure review_loop_count is always incremented. + """ + step = recipe.steps["review_pr"] # type: ignore[attr-defined] + assert step.on_result is not None + default_conditions = [ + c for c in step.on_result.conditions if c.when is None or c.when == "true" + ] + assert any(c.route == "check_review_loop" for c in default_conditions), ( + "catch-all must route to check_review_loop, not directly to check_repo_ci_event" + ) + + def test_review_pr_captures_verdict(self, recipe: object) -> None: + """T_RP4b: review_pr captures the verdict output as review_verdict to avoid clobber.""" + step = recipe.steps["review_pr"] # type: ignore[attr-defined] + assert "review_verdict" in step.capture + assert step.capture["review_verdict"] == "${{ result.verdict }}" + + def test_review_pr_changes_requested_routes_to_resolve_review(self, recipe: object) -> None: + """T_RP4c: changes_requested reaches resolve_review.""" + step = recipe.steps["review_pr"] # type: ignore[attr-defined] + assert step.on_result is not None + changes_conditions = [ + c for c in step.on_result.conditions if c.when and "changes_requested" in c.when + ] + routes = {c.route for c in changes_conditions} + if "enrich_diff_context" in routes: + enrich = recipe.steps["enrich_diff_context"] # type: ignore[attr-defined] + assert enrich.on_success == "resolve_review" + else: + assert "resolve_review" in routes + + def test_review_pr_routes_to_check_repo_ci_event_on_failure(self, recipe: object) -> None: + """T_RP5: review_pr.on_failure routes to check_repo_ci_event (no review to resolve).""" + assert recipe.steps["review_pr"].on_failure == "check_repo_ci_event" # type: ignore[attr-defined] + + def test_resolve_review_only_reachable_via_verdict(self, recipe: object) -> None: + """T_RP5b: resolve_review reachable via verdict, not on_failure.""" + step = recipe.steps["review_pr"] # type: ignore[attr-defined] + assert step.on_failure != "resolve_review" + assert step.on_context_limit != "resolve_review" + verdict_routes = [ + c.route + for c in step.on_result.conditions + if c.route in ("resolve_review", "enrich_diff_context") + ] + assert len(verdict_routes) >= 1, "resolve_review must be reachable via on_result" + + def test_review_pr_failure_and_context_limit_converge(self, recipe: object) -> None: + """T_RP5c: on_failure and on_context_limit both route to check_repo_ci_event.""" + step = recipe.steps["review_pr"] # type: ignore[attr-defined] + assert step.on_failure == step.on_context_limit == "check_repo_ci_event" + + def test_resolve_review_has_retries(self, recipe: object) -> None: + """T_RP6: resolve_review has retries=2 matching resolve_ci pattern.""" + assert recipe.steps["resolve_review"].retries == 2 # type: ignore[attr-defined] + + def test_resolve_review_routes_to_re_push_review(self, recipe: object) -> None: + """T_RP7: resolve_review uses on_result: verdict dispatch routing to re_push_review.""" + step = recipe.steps["resolve_review"] # type: ignore[attr-defined] + assert step.on_success is None, ( + "resolve_review must use on_result: verdict dispatch, not unconditional on_success" + ) + assert step.on_result is not None, ( + "resolve_review must have on_result: block for verdict-gated routing" + ) + real_fix_routes = [ + c.route for c in step.on_result.conditions if c.when and "real_fix" in c.when + ] + assert any("re_push_review" in r for r in real_fix_routes), ( + "resolve_review on_result must route verdict=real_fix to re_push_review" + ) + + def test_re_push_review_routes_to_check_review_loop(self, recipe: object) -> None: + """T_RP8: re_push_review routes to check_review_loop (bounded retry gate).""" + assert recipe.steps["re_push_review"].on_success == "check_review_loop" # type: ignore[attr-defined] + + def test_ci_watch_present(self, recipe: object) -> None: + """T_RP9: ci_watch step present in all four recipes.""" + assert "ci_watch" in recipe.steps # type: ignore[attr-defined] + + def test_review_pr_needs_human_has_explicit_route(self, recipe: object) -> None: + """needs_human must have a dedicated on_result route in every recipe.""" + review_pr_step = recipe.steps["review_pr"] # type: ignore[attr-defined] + explicit_conditions = [ + c.when + for c in review_pr_step.on_result.conditions + if c.when and "needs_human" in c.when and c.when.strip() != "true" + ] + assert len(explicit_conditions) >= 1, ( + "review_pr on_result must have an explicit condition for 'needs_human'. " + "It must not silently fall through the catch-all." + ) + + def test_annotate_step_captures_diff_metrics_path(self, recipe: object) -> None: + step = recipe.steps["annotate_pr_diff"] # type: ignore[attr-defined] + assert "diff_metrics_path" in step.capture + assert step.capture["diff_metrics_path"] == "${{ result.diff_metrics_path }}" + + def test_review_pr_command_includes_diff_metrics_path(self, recipe: object) -> None: + step = recipe.steps["review_pr"] # type: ignore[attr-defined] + cmd = step.with_args.get("skill_command", "") + assert "diff_metrics_path=" in cmd + + def test_resolve_review_step_uses_correct_skill(self, recipe: object) -> None: + """resolve_review step must invoke /autoskillit:resolve-review in all recipes.""" + resolve_step = recipe.steps["resolve_review"] # type: ignore[attr-defined] + skill_cmd = resolve_step.with_args.get("skill_command", "") + assert "resolve-review" in skill_cmd and "resolve-failures" not in skill_cmd, ( + "resolve_review step must call /autoskillit:resolve-review, " + f"not resolve-failures. Got: {skill_cmd}" + ) + + +def test_implementation_groups_has_ci_watch() -> None: + """T_RP10: implementation-groups now has ci_watch (parity with other recipes).""" + recipe = load_recipe(builtin_recipes_dir() / "implementation-groups.yaml") + assert "ci_watch" in recipe.steps + assert "resolve_ci" in recipe.steps + assert "re_push" in recipe.steps + + +def test_merge_prs_review_pr_integration_includes_diff_metrics_path() -> None: + recipe = load_recipe(builtin_recipes_dir() / "merge-prs.yaml") + step = recipe.steps["review_pr_integration"] + cmd = step.with_args.get("skill_command", "") + assert "diff_metrics_path=" in cmd + + +def test_merge_prs_annotate_step_captures_diff_metrics_path() -> None: + recipe = load_recipe(builtin_recipes_dir() / "merge-prs.yaml") + step = recipe.steps["annotate_pr_diff"] + assert "diff_metrics_path" in step.capture + + +# --------------------------------------------------------------------------- +# T4.1–T4.7: local_review_rounds wiring tests +# --------------------------------------------------------------------------- + + +class TestAnnotatePrDiffLocalReviewRounds: + @pytest.fixture( + scope="class", + params=[ + "implementation.yaml", + "implementation-groups.yaml", + "remediation.yaml", + # merge-prs.yaml is excluded: it uses annotate_pr_diff with a fixed base and does + # not expose local_review_rounds as a recipe ingredient. Its annotate step wiring is + # covered by test_merge_prs_annotate_step_captures_diff_metrics_path. + ], + ) + def recipe(self, request: pytest.FixtureRequest) -> object: + return load_recipe(builtin_recipes_dir() / request.param) + + def test_annotate_step_captures_review_mode(self, recipe: object) -> None: + """T4.1: annotate_pr_diff step captures review_mode.""" + step = recipe.steps["annotate_pr_diff"] + assert "review_mode" in step.capture + assert step.capture["review_mode"] == "${{ result.review_mode }}" + + def test_annotate_step_passes_local_review_rounds(self, recipe: object) -> None: + """T4.2: annotate_pr_diff step passes local_review_rounds via with_args.""" + step = recipe.steps["annotate_pr_diff"] + assert "local_review_rounds" in step.with_args + + def test_annotate_step_passes_current_iteration(self, recipe: object) -> None: + """T4.3: annotate_pr_diff step passes current_iteration via with_args.""" + step = recipe.steps["annotate_pr_diff"] + assert "current_iteration" in step.with_args + + def test_annotate_step_passes_base_branch(self, recipe: object) -> None: + """T4.4: annotate_pr_diff step passes base_branch via with_args.""" + step = recipe.steps["annotate_pr_diff"] + assert "base_branch" in step.with_args + + def test_review_pr_command_includes_mode(self, recipe: object) -> None: + """T4.5: review_pr skill_command includes mode=${{ context.review_mode }}.""" + step = recipe.steps["review_pr"] + cmd = step.with_args.get("skill_command", "") + assert "mode=${{ context.review_mode }}" in cmd + + def test_resolve_review_command_includes_mode(self, recipe: object) -> None: + """T4.6: resolve_review skill_command includes mode=${{ context.review_mode }}.""" + step = recipe.steps["resolve_review"] + cmd = step.with_args.get("skill_command", "") + assert "mode=${{ context.review_mode }}" in cmd + + def test_local_review_rounds_ingredient_exists(self, recipe: object) -> None: + """T4.7: local_review_rounds is in recipe.ingredients.""" + assert "local_review_rounds" in recipe.ingredients + + +# --------------------------------------------------------------------------- +# T1: local_review_rounds default is not empty string (explicit numeric) +# T5: display and runtime defaults are consistent +# T7: review_max_retries default tested for all three recipes +# --------------------------------------------------------------------------- + + +class TestLocalReviewRoundsDefault: + """T1 + T5: local_review_rounds must have explicit non-empty default.""" + + @pytest.fixture( + scope="class", + params=[ + "implementation.yaml", + "implementation-groups.yaml", + "remediation.yaml", + ], + ) + def recipe(self, request: pytest.FixtureRequest) -> object: + return load_recipe(builtin_recipes_dir() / request.param) + + def test_local_review_rounds_default_is_not_empty_string(self, recipe: object) -> None: + """T1: local_review_rounds.default must NOT be '' in bundled recipes.""" + ing = recipe.ingredients["local_review_rounds"] + assert ing.default != "", ( + f"{recipe.name}: local_review_rounds.default is '', " + "which creates a shadow-default that bypasses type semantics" + ) + + def test_local_review_rounds_default_is_parseable_as_int(self, recipe: object) -> None: + """T1: local_review_rounds.default must be parseable as int.""" + ing = recipe.ingredients["local_review_rounds"] + assert ing.default is not None + try: + int(ing.default) + except ValueError: + pytest.fail( + f"{recipe.name}: local_review_rounds.default={ing.default!r} " + "is not parseable as int" + ) + + def test_local_review_rounds_type_is_integer(self, recipe: object) -> None: + """T5: local_review_rounds must declare type=integer for semantic rule enforcement.""" + ing = recipe.ingredients["local_review_rounds"] + assert ing.type == "integer", ( + f"{recipe.name}: local_review_rounds.type must be 'integer', got {ing.type!r}" + ) + + +class TestReviewMaxRetriesDefault: + """T7: review_max_retries.default must be '3' for all three main recipes.""" + + @pytest.fixture( + scope="class", + params=[ + "implementation.yaml", + "implementation-groups.yaml", + "remediation.yaml", + ], + ) + def recipe(self, request: pytest.FixtureRequest) -> object: + return load_recipe(builtin_recipes_dir() / request.param) + + def test_review_max_retries_default_is_3(self, recipe: object) -> None: + """T7: review_max_retries.default must be '3' in all bundled recipes.""" + ing = recipe.ingredients["review_max_retries"] + assert ing.default == "3", ( + f"{recipe.name}: review_max_retries.default is {ing.default!r}, expected '3'" + ) diff --git a/tests/recipe/test_callable_contracts.py b/tests/recipe/test_callable_contracts.py new file mode 100644 index 0000000000..8ce431d5f5 --- /dev/null +++ b/tests/recipe/test_callable_contracts.py @@ -0,0 +1,35 @@ +from __future__ import annotations + +import importlib +import inspect + +import pytest + +from autoskillit.recipe.contracts import load_bundled_manifest + +pytestmark = [pytest.mark.layer("recipe"), pytest.mark.small] + + +def test_all_callable_contracts_declare_inputs(): + """Every callable_contract entry must declare inputs matching the function signature.""" + manifest = load_bundled_manifest() + callables = manifest.get("callable_contracts", {}) + for dotted_path, entry in callables.items(): + if "." not in dotted_path: + pytest.fail(f"{dotted_path}: not a dotted module path") + module_path, attr_name = dotted_path.rsplit(".", 1) + mod = importlib.import_module(module_path) + func = getattr(mod, attr_name) + sig = inspect.signature(func) + required_params = [ + name + for name, p in sig.parameters.items() + if p.default is inspect.Parameter.empty + and p.kind not in (inspect.Parameter.VAR_KEYWORD, inspect.Parameter.VAR_POSITIONAL) + ] + declared_inputs = [inp["name"] for inp in entry.get("inputs", [])] + for param in required_params: + assert param in declared_inputs, ( + f"{dotted_path}: required parameter '{param}' not declared in " + f"callable_contracts inputs" + ) diff --git a/tests/recipe/test_campaign_loader.py b/tests/recipe/test_campaign_loader.py new file mode 100644 index 0000000000..bbdaa2ae4f --- /dev/null +++ b/tests/recipe/test_campaign_loader.py @@ -0,0 +1,613 @@ +"""Tests for campaign recipe loader — _parse_recipe, list_campaign_recipes, +find_campaign_by_name, load_campaign_recipes_in_packs, and validate_recipe campaign branch.""" + +from __future__ import annotations + +import re +from pathlib import Path + +import pytest +import yaml + +from autoskillit.core import DispatchGateType, Severity, pkg_root +from autoskillit.recipe._analysis import make_validation_context +from autoskillit.recipe.io import ( + _parse_recipe, + find_campaign_by_name, + list_campaign_recipes, + load_campaign_recipes_in_packs, + load_recipe, +) +from autoskillit.recipe.registry import run_semantic_rules +from autoskillit.recipe.schema import CampaignDispatch, Recipe, RecipeKind +from autoskillit.recipe.validator import validate_recipe_structure + +pytestmark = [pytest.mark.layer("recipe"), pytest.mark.small] + + +def _write_yaml(path: Path, data: dict) -> Path: + path.write_text(yaml.dump(data, default_flow_style=False), encoding="utf-8") + return path + + +def _campaign_data(**overrides: object) -> dict: + base: dict = { + "name": "my-campaign", + "description": "A test campaign", + "kind": "campaign", + "kitchen_rules": ["NEVER do bad things"], + "dispatches": [ + { + "name": "phase-one", + "recipe": "implementation", + "task": "Do the thing", + "ingredients": {"task": "Do the thing"}, + "depends_on": [], + } + ], + } + base.update(overrides) + return base + + +# --------------------------------------------------------------------------- +# _parse_recipe campaign field parsing +# --------------------------------------------------------------------------- + + +def test_parse_recipe_reads_campaign_kind(): + data = _campaign_data() + recipe = _parse_recipe(data) + assert recipe.kind == RecipeKind.CAMPAIGN + + +def test_parse_recipe_reads_dispatches(): + data = _campaign_data( + dispatches=[ + { + "name": "phase-a", + "recipe": "implementation", + "task": "First task", + "ingredients": {"task": "First task"}, + "depends_on": [], + }, + { + "name": "phase-b", + "recipe": "research", + "task": "Second task", + "ingredients": {}, + "depends_on": ["phase-a"], + }, + ] + ) + recipe = _parse_recipe(data) + assert len(recipe.dispatches) == 2 + assert recipe.dispatches[0].name == "phase-a" + assert recipe.dispatches[0].recipe == "implementation" + assert recipe.dispatches[0].task == "First task" + assert recipe.dispatches[0].ingredients == {"task": "First task"} + assert recipe.dispatches[0].depends_on == [] + assert recipe.dispatches[1].name == "phase-b" + assert recipe.dispatches[1].depends_on == ["phase-a"] + + +def test_parse_recipe_reads_campaign_metadata_fields(): + data = _campaign_data( + categories=["implementation-family"], + requires_recipe_packs=["implementation-family"], + allowed_recipes=["special-recipe"], + continue_on_failure=True, + ) + recipe = _parse_recipe(data) + assert recipe.categories == ["implementation-family"] + assert recipe.requires_recipe_packs == ["implementation-family"] + assert recipe.allowed_recipes == ["special-recipe"] + assert recipe.continue_on_failure is True + + +def test_parse_recipe_defaults_campaign_fields_when_absent(): + data = { + "name": "standard-recipe", + "description": "No campaign fields", + "kitchen_rules": ["NEVER"], + "steps": {"stop": {"action": "stop", "message": "done"}}, + } + recipe = _parse_recipe(data) + assert recipe.kind == RecipeKind.STANDARD + assert recipe.dispatches == [] + assert recipe.categories == [] + assert recipe.requires_recipe_packs == [] + assert recipe.allowed_recipes == [] + assert recipe.continue_on_failure is False + + +# --------------------------------------------------------------------------- +# list_campaign_recipes +# --------------------------------------------------------------------------- + + +def test_list_campaign_recipes_scans_campaigns_dir( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +): + import autoskillit.recipe.io as _rio + + monkeypatch.setattr(_rio, "pkg_root", lambda: tmp_path) + campaigns_dir = tmp_path / ".autoskillit" / "recipes" / "campaigns" + campaigns_dir.mkdir(parents=True) + _write_yaml( + campaigns_dir / "my-campaign.yaml", + _campaign_data(name="my-campaign"), + ) + result = list_campaign_recipes(tmp_path) + assert len(result.items) == 1 + assert result.items[0].name == "my-campaign" + + +def test_list_campaign_recipes_returns_empty_when_no_dir( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +): + import autoskillit.recipe.io as _rio + + monkeypatch.setattr(_rio, "pkg_root", lambda: tmp_path) + result = list_campaign_recipes(tmp_path) + assert result.items == [] + + +# --------------------------------------------------------------------------- +# find_campaign_by_name +# --------------------------------------------------------------------------- + + +def test_find_campaign_by_name_returns_match(tmp_path: Path): + campaigns_dir = tmp_path / ".autoskillit" / "recipes" / "campaigns" + campaigns_dir.mkdir(parents=True) + _write_yaml( + campaigns_dir / "my-campaign.yaml", + _campaign_data(name="my-campaign"), + ) + result = find_campaign_by_name("my-campaign", tmp_path) + assert result is not None + assert result.name == "my-campaign" + + +def test_find_campaign_by_name_returns_none_when_missing(tmp_path: Path): + result = find_campaign_by_name("nonexistent", tmp_path) + assert result is None + + +# --------------------------------------------------------------------------- +# load_campaign_recipes_in_packs +# --------------------------------------------------------------------------- + + +def test_load_campaign_recipes_in_packs_filters_by_categories(tmp_path: Path): + campaigns_dir = tmp_path / ".autoskillit" / "recipes" / "campaigns" + campaigns_dir.mkdir(parents=True) + _write_yaml( + campaigns_dir / "impl-campaign.yaml", + _campaign_data(name="impl-campaign", categories=["implementation-family"]), + ) + _write_yaml( + campaigns_dir / "research-campaign.yaml", + _campaign_data(name="research-campaign", categories=["research-family"]), + ) + results = load_campaign_recipes_in_packs(frozenset({"implementation-family"}), tmp_path) + assert len(results) == 1 + assert results[0].name == "impl-campaign" + + +def test_load_campaign_recipes_in_packs_includes_allowed_recipes(tmp_path: Path): + campaigns_dir = tmp_path / ".autoskillit" / "recipes" / "campaigns" + campaigns_dir.mkdir(parents=True) + _write_yaml( + campaigns_dir / "special-campaign.yaml", + _campaign_data(name="special-campaign", categories=["research-family"]), + ) + results = load_campaign_recipes_in_packs( + frozenset({"implementation-family"}), + tmp_path, + allowed_recipe_names=frozenset({"special-campaign"}), + ) + assert len(results) == 1 + assert results[0].name == "special-campaign" + + +# --------------------------------------------------------------------------- +# validate_recipe campaign branch +# --------------------------------------------------------------------------- + + +def test_validate_recipe_skips_step_check_for_campaign(): + recipe = Recipe( + name="my-campaign", + description="test", + kind=RecipeKind.CAMPAIGN, + dispatches=[CampaignDispatch(name="phase-one", recipe="impl", task="Do it")], + steps={}, + ) + errors = validate_recipe_structure(recipe) + assert "Recipe must have at least one step." not in errors + + +def test_validate_recipe_requires_dispatches_for_campaign(): + recipe = Recipe( + name="my-campaign", + description="test", + kind=RecipeKind.CAMPAIGN, + dispatches=[], + steps={}, + ) + errors = validate_recipe_structure(recipe) + assert any("dispatch" in e.lower() for e in errors) + + +def test_validate_recipe_standard_recipe_still_requires_steps(): + recipe = Recipe( + name="standard", + description="test", + kind=RecipeKind.STANDARD, + steps={}, + ) + errors = validate_recipe_structure(recipe) + assert any("step" in e.lower() for e in errors) + + +def test_bundled_example_campaign_parseable(): + example_path = pkg_root() / "recipes" / "examples" / "example-campaign.yaml" + recipe = load_recipe(example_path) + assert recipe.kind == RecipeKind.CAMPAIGN + assert len(recipe.dispatches) == 2 + + +# --------------------------------------------------------------------------- +# promote-to-main campaign +# --------------------------------------------------------------------------- + + +def test_promote_to_main_campaign_parseable(): + path = pkg_root() / "recipes" / "campaigns" / "promote-to-main.yaml" + recipe = load_recipe(path) + assert recipe.name == "promote-to-main" + assert recipe.kind == RecipeKind.CAMPAIGN + assert recipe.recipe_version == "1.0.0" + + +def test_promote_to_main_campaign_passes_validation(): + path = pkg_root() / "recipes" / "campaigns" / "promote-to-main.yaml" + recipe = load_recipe(path) + findings = validate_recipe_structure(recipe) + assert findings == [], f"Unexpected findings: {findings}" + + +def test_promote_to_main_campaign_dispatch_chain(): + path = pkg_root() / "recipes" / "campaigns" / "promote-to-main.yaml" + recipe = load_recipe(path) + names = [d.name for d in recipe.dispatches] + assert names == ["full-audit", "review-gate", "build-map", "implement-findings", "promote"] + gate_dispatches = [d for d in recipe.dispatches if d.gate] + assert len(gate_dispatches) == 1 + gd = gate_dispatches[0] + assert gd.name == "review-gate" + assert gd.gate == DispatchGateType.CONFIRM + assert gd.message + assert not gd.recipe + assert not gd.capture + + +def test_promote_to_main_campaign_in_list_campaign_recipes(tmp_path: Path): + result = list_campaign_recipes(tmp_path) + assert result.errors == [] + names = [r.name for r in result.items] + assert "promote-to-main" in names + + +# --------------------------------------------------------------------------- +# research-campaign skeleton +# --------------------------------------------------------------------------- + + +def test_research_campaign_parseable(): + path = pkg_root() / "recipes" / "campaigns" / "research-campaign.yaml" + recipe = load_recipe(path) + assert recipe.name == "research-campaign" + assert recipe.kind == RecipeKind.CAMPAIGN + assert recipe.recipe_version == "1.0.0" + + +def test_research_campaign_passes_structural_validation(): + path = pkg_root() / "recipes" / "campaigns" / "research-campaign.yaml" + recipe = load_recipe(path) + findings = validate_recipe_structure(recipe) + assert findings == [], f"Unexpected findings: {findings}" + + +def test_research_campaign_header_fields(): + path = pkg_root() / "recipes" / "campaigns" / "research-campaign.yaml" + recipe = load_recipe(path) + assert recipe.kind == RecipeKind.CAMPAIGN + assert recipe.categories == ["research-family"] + assert recipe.requires_recipe_packs == ["research-family"] + assert recipe.allowed_recipes == [ + "research-design", + "research-implement", + "research-review", + "research-archive", + ] + assert recipe.continue_on_failure is False + assert recipe.recipe_version == "1.0.0" + assert recipe.version is None + + +def test_research_campaign_ingredients_match_research_yaml(): + # research.yaml is the canonical ingredient source for the research family; + # research-campaign.yaml must expose a subset of those keys so the campaign + # orchestrator can forward ingredients to each sub-recipe unchanged. + # If research.yaml is ever renamed, update this test's path accordingly. + campaign_path = pkg_root() / "recipes" / "campaigns" / "research-campaign.yaml" + research_path = pkg_root() / "recipes" / "research.yaml" + campaign_recipe = load_recipe(campaign_path) + research_recipe = load_recipe(research_path) + assert set(campaign_recipe.ingredients.keys()).issubset( + set(research_recipe.ingredients.keys()) + ) + assert campaign_recipe.ingredients["task"].required is True + assert campaign_recipe.ingredients["source_dir"].required is True + assert campaign_recipe.ingredients["issue_url"].required is False + assert campaign_recipe.ingredients["issue_url"].default is None + assert campaign_recipe.ingredients["base_branch"].default == "main" + assert campaign_recipe.ingredients["review_design"].default == "true" + assert campaign_recipe.ingredients["review_pr"].default == "false" + assert campaign_recipe.ingredients["audit_claims"].default == "false" + assert campaign_recipe.ingredients["output_mode"].default == "local" + + +def test_research_campaign_has_four_dispatches(): + path = pkg_root() / "recipes" / "campaigns" / "research-campaign.yaml" + recipe = load_recipe(path) + assert len(recipe.dispatches) == 4 + assert recipe.steps == {} + + +def test_research_campaign_allowed_recipes_kebab_case(): + path = pkg_root() / "recipes" / "campaigns" / "research-campaign.yaml" + recipe = load_recipe(path) + + for name in recipe.allowed_recipes: + assert re.match(r"^[a-z0-9]+(-[a-z0-9]+)*$", name) + + +def test_research_campaign_dispatch_chain(): + path = pkg_root() / "recipes" / "campaigns" / "research-campaign.yaml" + recipe = load_recipe(path) + names = [d.name for d in recipe.dispatches] + assert names == ["run-design", "run-implement", "run-review", "run-archive"] + assert recipe.dispatches[0].recipe == "research-design" + assert recipe.dispatches[0].depends_on == [] + assert recipe.dispatches[1].recipe == "research-implement" + assert recipe.dispatches[1].depends_on == ["run-design"] + assert recipe.dispatches[2].recipe == "research-review" + assert recipe.dispatches[2].depends_on == ["run-implement"] + assert recipe.dispatches[3].recipe == "research-archive" + assert recipe.dispatches[3].depends_on == ["run-review"] + for d in recipe.dispatches: + assert d.task, f"Dispatch {d.name!r} has empty task" + assert d.gate is None + + +def test_research_campaign_dispatch_ingredients_are_strings(): + path = pkg_root() / "recipes" / "campaigns" / "research-campaign.yaml" + recipe = load_recipe(path) + for d in recipe.dispatches: + for key, val in d.ingredients.items(): + assert isinstance(val, str), ( + f"Dispatch {d.name!r} ingredient {key!r} is {type(val).__name__}, not str" + ) + + +def test_research_campaign_run_design_capture(): + path = pkg_root() / "recipes" / "campaigns" / "research-campaign.yaml" + recipe = load_recipe(path) + d = recipe.dispatches[0] + assert d.name == "run-design" + assert set(d.capture.keys()) == { + "worktree_path", + "research_dir", + "experiment_plan", + "visualization_plan_path", + } + for key, val in d.capture.items(): + assert val == f"${{{{ result.{key} }}}}" + + +def test_research_campaign_run_implement_ingredients(): + path = pkg_root() / "recipes" / "campaigns" / "research-campaign.yaml" + recipe = load_recipe(path) + d = recipe.dispatches[1] + assert d.name == "run-implement" + assert set(d.ingredients.keys()) == { + "worktree_path", + "research_dir", + "experiment_plan", + "visualization_plan_path", + "source_dir", + "base_branch", + "issue_url", + "output_mode", + } + assert d.ingredients["worktree_path"] == "${{ campaign.worktree_path }}" + assert d.ingredients["research_dir"] == "${{ campaign.research_dir }}" + assert d.ingredients["experiment_plan"] == "${{ campaign.experiment_plan }}" + assert d.ingredients["visualization_plan_path"] == "${{ campaign.visualization_plan_path }}" + assert d.ingredients["source_dir"] == "${{ inputs.source_dir }}" + assert d.ingredients["base_branch"] == "${{ inputs.base_branch }}" + assert d.ingredients["issue_url"] == "${{ inputs.issue_url }}" + assert d.ingredients["output_mode"] == "${{ inputs.output_mode }}" + + +def test_research_campaign_run_implement_capture(): + path = pkg_root() / "recipes" / "campaigns" / "research-campaign.yaml" + recipe = load_recipe(path) + d = recipe.dispatches[1] + assert d.name == "run-implement" + assert set(d.capture.keys()) == {"report_path"} + assert d.capture["report_path"] == "${{ result.report_path }}" + + +def test_research_campaign_run_review_ingredients(): + path = pkg_root() / "recipes" / "campaigns" / "research-campaign.yaml" + recipe = load_recipe(path) + d = recipe.dispatches[2] + assert d.name == "run-review" + assert set(d.ingredients.keys()) == { + "worktree_path", + "research_dir", + "experiment_plan", + "visualization_plan_path", + "report_path", + "source_dir", + "base_branch", + "issue_url", + "output_mode", + "review_pr", + "audit_claims", + } + assert d.ingredients["source_dir"] == "${{ inputs.source_dir }}" + assert d.ingredients["base_branch"] == "${{ inputs.base_branch }}" + assert d.ingredients["issue_url"] == "${{ inputs.issue_url }}" + assert d.ingredients["output_mode"] == "${{ inputs.output_mode }}" + assert d.ingredients["review_pr"] == "${{ inputs.review_pr }}" + assert d.ingredients["audit_claims"] == "${{ inputs.audit_claims }}" + for key in [ + "worktree_path", + "research_dir", + "experiment_plan", + "visualization_plan_path", + "report_path", + ]: + assert d.ingredients[key] == f"${{{{ campaign.{key} }}}}" + + +def test_research_campaign_run_review_capture(): + path = pkg_root() / "recipes" / "campaigns" / "research-campaign.yaml" + recipe = load_recipe(path) + d = recipe.dispatches[2] + assert d.name == "run-review" + assert set(d.capture.keys()) == {"pr_url", "all_diagram_paths", "report_path_after_finalize"} + assert d.capture["pr_url"] == "${{ result.pr_url }}" + assert d.capture["all_diagram_paths"] == "${{ result.all_diagram_paths }}" + assert d.capture["report_path_after_finalize"] == "${{ result.report_path_after_finalize }}" + + +def test_research_campaign_run_archive_ingredients(): + path = pkg_root() / "recipes" / "campaigns" / "research-campaign.yaml" + recipe = load_recipe(path) + d = recipe.dispatches[3] + assert d.name == "run-archive" + assert set(d.ingredients.keys()) == { + "base_branch", + "worktree_path", + "research_dir", + "pr_url", + "all_diagram_paths", + "report_path_after_finalize", + } + assert d.ingredients["base_branch"] == "${{ inputs.base_branch }}" + for key in d.ingredients: + if key == "base_branch": + continue + assert d.ingredients[key] == f"${{{{ campaign.{key} }}}}" + + +def test_research_campaign_no_campaign_refs_in_run_design(): + path = pkg_root() / "recipes" / "campaigns" / "research-campaign.yaml" + recipe = load_recipe(path) + d = recipe.dispatches[0] + for key, val in d.ingredients.items(): + assert "${{ campaign." not in val + + +def test_research_campaign_run_archive_no_capture(): + path = pkg_root() / "recipes" / "campaigns" / "research-campaign.yaml" + recipe = load_recipe(path) + d = recipe.dispatches[3] + assert d.name == "run-archive" + assert d.capture == {} + + +def test_research_campaign_capture_keys_are_identifiers(): + from autoskillit.recipe.rules.rules_campaign import _IDENT_RE + + path = pkg_root() / "recipes" / "campaigns" / "research-campaign.yaml" + recipe = load_recipe(path) + for d in recipe.dispatches: + for key in d.capture.keys(): + assert _IDENT_RE.match(key), ( + f"Dispatch {d.name!r} capture key {key!r} is not a valid identifier" + ) + + +def test_research_campaign_capture_values_reference_result(): + from autoskillit.recipe.rules.rules_campaign import _RESULT_TEMPLATE_RE + + path = pkg_root() / "recipes" / "campaigns" / "research-campaign.yaml" + recipe = load_recipe(path) + for d in recipe.dispatches: + if d.name == "run-archive": + assert d.capture == {}, f"run-archive must have empty capture, got {d.capture!r}" + continue + for val in d.capture.values(): + assert _RESULT_TEMPLATE_RE.match(val.strip()), ( + f"Dispatch {d.name!r} capture value {val!r} does not reference result" + ) + + +_CAMPAIGN_ERROR_RULES = [ + "dispatch-ingredients-keys-in-target-schema", + "dispatch-ingredient-values-are-strings", + "depends-on-refers-to-valid-dispatches", + "depends-on-acyclic", + "campaign-ingredient-refs-have-prior-capture", +] + + +def _research_campaign_ctx(): + path = pkg_root() / "recipes" / "campaigns" / "research-campaign.yaml" + recipe = load_recipe(path) + project_dir = pkg_root() / "recipes" + available_recipes = frozenset(p.stem for p in project_dir.glob("*.yaml")) + return make_validation_context( + recipe, + available_recipes=available_recipes, + project_dir=project_dir, + ) + + +@pytest.mark.parametrize("rule", _CAMPAIGN_ERROR_RULES) +def test_research_campaign_zero_error_findings(rule: str) -> None: + ctx = _research_campaign_ctx() + errors = [ + f for f in run_semantic_rules(ctx) if f.rule == rule and f.severity == Severity.ERROR + ] + assert errors == [], f"research-campaign.yaml has ERROR findings for {rule!r}:\n" + "\n".join( + f" {f.step_name}: {f.message}" for f in errors + ) + + +def test_implement_findings_has_model_context_window_ingredient(): + path = pkg_root() / "recipes" / "implement-findings.yaml" + recipe = load_recipe(path) + assert "model_context_window" in recipe.ingredients + ing = recipe.ingredients["model_context_window"] + assert ing.hidden is True + assert ing.default == "200000" + + +def test_full_audit_done_step_emits_csv_format(): + path = pkg_root() / "recipes" / "full-audit.yaml" + recipe = load_recipe(path) + done_step = recipe.steps["done"] + assert ( + "comma-separated" in done_step.message.lower() + or "comma,separated" in done_step.message.lower() + ) + assert '["url' not in done_step.message diff --git a/tests/recipe/test_check_ci_already_passed_routing.py b/tests/recipe/test_check_ci_already_passed_routing.py new file mode 100644 index 0000000000..cb650ab82c --- /dev/null +++ b/tests/recipe/test_check_ci_already_passed_routing.py @@ -0,0 +1,194 @@ +"""Tests verifying the check_ci_already_passed safety net step in CI watch recipes.""" + +import json +import shutil +import subprocess + +import pytest + +from autoskillit.recipe.io import builtin_recipes_dir, load_recipe + +pytestmark = [pytest.mark.layer("recipe"), pytest.mark.medium] + +_requires_jq = pytest.mark.skipif( + shutil.which("jq") is None, reason="jq binary not available in PATH" +) + +RECIPE_NAMES = ["implementation", "remediation", "implementation-groups"] + + +@pytest.fixture(params=RECIPE_NAMES) +def recipe(request): + return load_recipe(builtin_recipes_dir() / f"{request.param}.yaml") + + +def _run_jq(rollup: list[dict]) -> str: + """Extract the jq expression from the recipe and run it against test data.""" + recipe = load_recipe(builtin_recipes_dir() / "implementation.yaml") + step = recipe.steps["check_ci_already_passed"] + cmd_str = step.with_args["cmd"] + jq_start = cmd_str.index("--jq '") + len("--jq '") + jq_end = cmd_str.rindex("'") + jq_expr = cmd_str[jq_start:jq_end] + input_json = json.dumps({"statusCheckRollup": rollup}) + result = subprocess.run( + ["jq", "-r", jq_expr], + input=input_json, + capture_output=True, + text=True, + check=True, + ) + return result.stdout.strip() + + +@pytest.mark.medium +@_requires_jq +def test_jq_produces_failed_for_failure_state(): + """When statusCheckRollup contains FAILURE, jq must output 'failed' not 'false'.""" + assert _run_jq([{"state": "SUCCESS"}, {"state": "FAILURE"}]) == "failed" + + +@pytest.mark.medium +@_requires_jq +def test_jq_produces_no_checks_for_empty_rollup(): + """When statusCheckRollup is empty, jq must output 'no_checks' not 'false'.""" + assert _run_jq([]) == "no_checks" + + +@pytest.mark.medium +@_requires_jq +def test_jq_produces_passed_for_all_success(): + assert _run_jq([{"state": "SUCCESS"}, {"state": "NEUTRAL"}]) == "passed" + + +@pytest.mark.medium +@_requires_jq +def test_jq_produces_pending_for_in_progress(): + assert _run_jq([{"state": "SUCCESS"}, {"state": "IN_PROGRESS"}]) == "pending" + + +def test_check_ci_already_passed_step_exists(recipe): + """New safety net step is present in each recipe.""" + assert "check_ci_already_passed" in recipe.steps + + +def test_check_ci_already_passed_routes_to_merge_state_on_success(recipe): + """stdout == 'passed' routes to check_repo_merge_state.""" + step = recipe.steps["check_ci_already_passed"] + assert step.on_result is not None + success_routes = [ + c.route for c in (step.on_result.conditions or []) if c.when and "passed" in c.when + ] + assert "check_repo_merge_state" in success_routes + + +def test_failed_routes_to_detect_ci_conflict(recipe): + """stdout == 'failed' must route to detect_ci_conflict for remediation.""" + step = recipe.steps["check_ci_already_passed"] + assert step.on_result is not None + failed_routes = [ + c.route for c in (step.on_result.conditions or []) if c.when and "failed" in c.when + ] + assert "detect_ci_conflict" in failed_routes + + +def test_pending_routes_back_to_ci_watch(recipe): + """stdout == 'pending' must route back to ci_watch for another iteration.""" + step = recipe.steps["check_ci_already_passed"] + assert step.on_result is not None + pending_routes = [ + c.route for c in (step.on_result.conditions or []) if c.when and "pending" in c.when + ] + assert "ci_watch" in pending_routes + + +def test_check_ci_already_passed_fallthrough_routes_to_escalate(recipe): + """Default (non-matching) on_result condition routes to mark_issue_failed_no_ci.""" + step = recipe.steps["check_ci_already_passed"] + assert step.on_result is not None + conditions = step.on_result.conditions or [] + fallthrough_routes = [c.route for c in conditions if not c.when] + assert "mark_issue_failed_no_ci" in fallthrough_routes + + +def test_check_ci_already_passed_on_failure_routes_to_escalate(recipe): + """on_failure routes to mark_issue_failed_no_ci (gh CLI failure).""" + step = recipe.steps["check_ci_already_passed"] + assert step.on_failure == "mark_issue_failed_no_ci" + + +def test_check_ci_already_passed_uses_run_cmd(recipe): + """Step uses run_cmd tool (shell invocation of gh CLI).""" + step = recipe.steps["check_ci_already_passed"] + assert step.tool == "run_cmd" + + +def test_check_ci_already_passed_references_pr_number(recipe): + """Step command references context.pr_number.""" + step = recipe.steps["check_ci_already_passed"] + assert "context.pr_number" in step.with_args.get("cmd", "") + + +def test_check_ci_already_passed_has_skip_when_false(recipe): + """Step only runs when inputs.open_pr is set (no PR context -> no-op).""" + step = recipe.steps["check_ci_already_passed"] + assert step.skip_when_false == "inputs.open_pr" + + +def test_no_direct_escalate_stop_no_ci_callers_except_safety_net(recipe): + """Only check_ci_already_passed and escalate_stop_no_ci itself reference + escalate_stop_no_ci.""" + EXCLUDED = { + "check_ci_already_passed", + "escalate_stop_no_ci", + "mark_issue_failed_no_ci", + "register_clone_no_ci", + } + for name, step in recipe.steps.items(): + if name in EXCLUDED: + continue + direct_routes = {step.on_success, step.on_failure, step.on_exhausted} + for cond in step.on_result.conditions if step.on_result else []: + direct_routes.add(cond.route) + assert "escalate_stop_no_ci" not in direct_routes, ( + f"Step '{name}' still routes directly to escalate_stop_no_ci" + ) + + +def test_mark_issue_failed_no_ci_routes_to_register_clone_no_ci(recipe): + """mark_issue_failed_no_ci calls release_issue with fail_label and routes to register_clone.""" + step = recipe.steps["mark_issue_failed_no_ci"] + assert step.tool == "release_issue" + assert step.on_success == "register_clone_no_ci" + assert step.on_failure == "register_clone_no_ci" + assert step.with_args.get("fail_label") == "fail" + + +def test_register_clone_no_ci_routes_to_escalate_stop_no_ci(recipe): + """register_clone_no_ci registers clone as error and routes to escalate_stop.""" + step = recipe.steps["register_clone_no_ci"] + assert step.tool == "register_clone_status" + assert step.on_success == "escalate_stop_no_ci" + assert step.on_failure == "escalate_stop_no_ci" + assert step.with_args.get("status") == "error" + + +def test_no_ci_failure_chain_complete(recipe): + """Full chain: check_ci_already_passed -> mark_issue_failed -> register_clone -> escalate.""" + ci_step = recipe.steps["check_ci_already_passed"] + conditions = ci_step.on_result.conditions or [] + fallthrough_routes = [c.route for c in conditions if not c.when] + assert "mark_issue_failed_no_ci" in fallthrough_routes + + mark_step = recipe.steps["mark_issue_failed_no_ci"] + assert mark_step.on_success == "register_clone_no_ci" + + reg_step = recipe.steps["register_clone_no_ci"] + assert reg_step.on_success == "escalate_stop_no_ci" + + +def test_escalate_stop_message_accurate_for_no_checks_case(recipe): + """The escalation stop message must not claim 'no CI runs found' unconditionally.""" + step = recipe.steps["escalate_stop_no_ci"] + msg = (step.message or "").lower() + assert "no ci runs found" not in msg diff --git a/tests/recipe/test_check_repo_merge_state_routing.py b/tests/recipe/test_check_repo_merge_state_routing.py index 2560f008aa..d68f0a98cb 100644 --- a/tests/recipe/test_check_repo_merge_state_routing.py +++ b/tests/recipe/test_check_repo_merge_state_routing.py @@ -10,6 +10,8 @@ from autoskillit.recipe.io import builtin_recipes_dir, load_recipe +pytestmark = [pytest.mark.layer("recipe"), pytest.mark.small] + @pytest.mark.parametrize("recipe_name", ["implementation", "remediation", "implementation-groups"]) def test_check_repo_merge_state_on_failure_routes_to_immediate_merge_not_success(recipe_name): diff --git a/tests/recipe/test_cmd_rpc.py b/tests/recipe/test_cmd_rpc.py new file mode 100644 index 0000000000..22bddf53fe --- /dev/null +++ b/tests/recipe/test_cmd_rpc.py @@ -0,0 +1,701 @@ +"""Tests for recipe._cmd_rpc — externalized run_python callables.""" + +from __future__ import annotations + +import json +import subprocess +from pathlib import Path +from unittest.mock import patch + +import pytest + +from autoskillit.recipe._cmd_rpc import ( + batch_create_issues, + check_dropped_healthy_loop, + check_eject_limit, + commit_guard, + compute_branch, + create_audit_run_dir, + emit_fallback_map, + ensure_results, + export_local_bundle, + force_push_and_wait_mergeability, + refetch_issues, + wait_for_direct_merge, +) + +pytestmark = [pytest.mark.layer("recipe"), pytest.mark.medium] + + +def test_compute_branch_with_issue(): + result = compute_branch(issue_slug="feat-widget", run_name="run1", issue_number="42") + assert result == {"branch_name": "feat-widget/42"} + + +def test_compute_branch_without_issue(): + result = compute_branch(issue_slug="feat-widget", run_name="run1", issue_number="") + assert "feat-widget/" in result["branch_name"] + + +def test_compute_branch_uses_run_name_when_no_slug(): + result = compute_branch(issue_slug="", run_name="my-run", issue_number="7") + assert result == {"branch_name": "my-run/7"} + + +def test_check_eject_limit_under_threshold(tmp_path): + counter = tmp_path / "count" + result = check_eject_limit(counter_file=str(counter), max_ejects="3") + assert result["status"] == "EJECT_OK" + assert result["count"] == "1" + + +def test_check_eject_limit_exceeded(tmp_path): + counter = tmp_path / "count" + counter.write_text("3") + result = check_eject_limit(counter_file=str(counter), max_ejects="3") + assert result["status"] == "EJECT_LIMIT_EXCEEDED" + assert result["count"] == "4" + + +def test_check_eject_limit_creates_parent_dirs(tmp_path): + counter = tmp_path / "nested" / "dir" / "count" + result = check_eject_limit(counter_file=str(counter), max_ejects="5") + assert result["status"] == "EJECT_OK" + assert counter.exists() + + +def test_check_dropped_healthy_loop_under(tmp_path): + counter = tmp_path / "dropped" + result = check_dropped_healthy_loop(counter_file=str(counter), max_drops="2") + assert result["status"] == "DROPPED_OK" + assert result["count"] == "1" + + +def test_check_dropped_healthy_loop_exceeded(tmp_path): + counter = tmp_path / "dropped" + counter.write_text("2") + result = check_dropped_healthy_loop(counter_file=str(counter), max_drops="2") + assert result["status"] == "DROPPED_LIMIT_EXCEEDED" + assert result["count"] == "3" + + +def _init_git_repo(tmp_path): + subprocess.run(["git", "init"], cwd=tmp_path, check=True, capture_output=True) + subprocess.run( + ["git", "config", "user.email", "test@test.local"], + cwd=tmp_path, + check=True, + capture_output=True, + ) + subprocess.run( + ["git", "config", "user.name", "Test"], + cwd=tmp_path, + check=True, + capture_output=True, + ) + subprocess.run( + ["git", "commit", "--allow-empty", "-m", "init"], + cwd=tmp_path, + check=True, + capture_output=True, + ) + + +def test_commit_guard_clean_tree(tmp_path): + _init_git_repo(tmp_path) + result = commit_guard(worktree_path=str(tmp_path)) + assert result["committed"] == "false" + + +def test_commit_guard_dirty_tree(tmp_path): + _init_git_repo(tmp_path) + (tmp_path / "newfile.txt").write_text("content") + result = commit_guard(worktree_path=str(tmp_path)) + assert result["committed"] == "true" + + +def test_ensure_results_with_existing_path(): + result = ensure_results(experiment_results="/some/path.md", worktree_path="/tmp") + assert result == {"experiment_results": "/some/path.md"} + + +def test_ensure_results_creates_placeholder(tmp_path): + result = ensure_results( + experiment_results="", + worktree_path=str(tmp_path), + temp_subdir=".autoskillit/temp", + ) + path = Path(result["experiment_results"]) + assert path.exists() + assert "INCONCLUSIVE" in path.read_text() + + +def test_emit_fallback_map(tmp_path): + result = emit_fallback_map( + issue_urls="https://github.com/org/repo/issues/1,https://github.com/org/repo/issues/2", + temp_dir=str(tmp_path), + ) + assert "execution_map" in result + data = json.loads(Path(result["execution_map"]).read_text()) + assert data["merge_order"] == [1, 2] + assert len(data["groups"]) == 1 + + +def test_emit_fallback_map_no_urls(tmp_path): + with pytest.raises(RuntimeError, match="no issue numbers"): + emit_fallback_map(issue_urls="", temp_dir=str(tmp_path)) + + +def test_export_local_bundle(tmp_path): + source_dir = tmp_path / "source" + source_dir.mkdir() + research_dir = tmp_path / "2024-01-01-test" + research_dir.mkdir() + (research_dir / "report.md").write_text("# Report") + result = export_local_bundle(source_dir=str(source_dir), research_dir=str(research_dir)) + assert Path(result["local_bundle_path"]).exists() + assert (Path(result["local_bundle_path"]) / "report.md").read_text() == "# Report" + + +def test_refetch_issues_builds_query(): + with patch("autoskillit.recipe._cmd_rpc.subprocess.run") as mock_run: + mock_run.return_value = subprocess.CompletedProcess( + args=[], returncode=0, stdout="1 2", stderr="" + ) + result = refetch_issues( + issue_urls="https://github.com/org/repo/issues/1,https://github.com/org/repo/issues/2" + ) + assert result["issue_numbers"] == "1 2" + call_args = mock_run.call_args[0][0] + assert "gh" in call_args + assert "graphql" in call_args + query_arg = next(a for a in call_args if a.startswith("query=")) + assert "org" in query_arg + assert "repo" in query_arg + assert "issue(number: 1)" in query_arg + assert "issue(number: 2)" in query_arg + + +# ─── batch_create_issues tests ───────────────────────────────────────────── + + +def _make_side_effect(repo_id="R_123", label_ids=None, issue_data=None): + if label_ids is None: + label_ids = ["L_1", "L_2"] + if issue_data is None: + issue_data = [{"number": 1, "url": "https://github.com/org/repo/issues/1"}] + alias_data = {} + for idx, issue in enumerate(issue_data): + alias_data[f"issue{idx}"] = {"issue": issue} + return [ + subprocess.CompletedProcess(args=[], returncode=0, stdout="org repo\n", stderr=""), + subprocess.CompletedProcess( + args=[], + returncode=0, + stdout=json.dumps({"data": {"repository": {"id": repo_id}}}), + stderr="", + ), + subprocess.CompletedProcess(args=[], returncode=0, stdout="", stderr=""), + subprocess.CompletedProcess(args=[], returncode=0, stdout="", stderr=""), + subprocess.CompletedProcess( + args=[], + returncode=0, + stdout=json.dumps( + { + "data": { + "repository": { + "impl": {"id": label_ids[0]}, + "enh": {"id": label_ids[1]}, + } + } + } + ), + stderr="", + ), + subprocess.CompletedProcess( + args=[], + returncode=0, + stdout=json.dumps({"data": alias_data}), + stderr="", + ), + ] + + +def test_batch_create_issues_discovers_ticket_bodies(tmp_path): + va_dir = tmp_path / ".autoskillit" / "temp" / "validate-audit" + va_dir.mkdir(parents=True) + for n in range(1, 4): + (va_dir / f"ticket_body_tests_{n}_2026-01-01_120000.md").write_text( + f"validated: true\n\n# Title {n}\n\n| col1 | col2 |\n" + ) + with ( + patch("autoskillit.recipe._cmd_rpc.subprocess.run") as mock_run, + patch("autoskillit.recipe._cmd_rpc.time.sleep"), + ): + mock_run.side_effect = _make_side_effect( + issue_data=[ + {"number": 1, "url": "https://github.com/org/repo/issues/1"}, + {"number": 2, "url": "https://github.com/org/repo/issues/2"}, + {"number": 3, "url": "https://github.com/org/repo/issues/3"}, + ] + ) + result = batch_create_issues(workspace=str(tmp_path)) + assert result["issue_count"] == "3" + + +def test_batch_create_issues_strips_body_content(tmp_path): + va_dir = tmp_path / ".autoskillit" / "temp" / "validate-audit" + va_dir.mkdir(parents=True) + (va_dir / "ticket_body_tests_1_2026-01-01_120000.md").write_text( + "validated: true\n\n# Audit: Missing test coverage\n\n" + "\n\n" + "| CONTESTED | finding |\n\n" + "| VALID BUT EXCEPTION WARRANTED | also contested |\n\n" + "| Item | **Contested:** 2 | **Exception warranted:** 1 |\n\n" + "## Findings with Exceptions\n\n" + "Some finding.\n\n---\n\n" + "**Exception note:** this is an exception.\n" + ) + with ( + patch("autoskillit.recipe._cmd_rpc.subprocess.run") as mock_run, + patch("autoskillit.recipe._cmd_rpc.time.sleep"), + ): + mock_run.side_effect = _make_side_effect() + batch_create_issues(workspace=str(tmp_path)) + # Find the createIssue mutation call + mutation_call: dict[str, object] = {} + for call in mock_run.call_args_list: + args = call[0][0] + if "--input" in args: + mutation_call = json.loads(call[1].get("input", "{}")) + break + assert mutation_call, "no createIssue mutation call found in mock_run calls" + body = mutation_call["variables"]["i0"]["body"] + assert ".autoskillit/" not in body + assert "| CONTESTED |" not in body + assert "| VALID BUT EXCEPTION WARRANTED |" not in body + assert "**Exception note:**" not in body + assert "Findings with Exceptions" not in body + assert "**Contested:**" not in body + + +def test_batch_create_issues_extracts_h1_title(tmp_path): + va_dir = tmp_path / ".autoskillit" / "temp" / "validate-audit" + va_dir.mkdir(parents=True) + (va_dir / "ticket_body_tests_1_2026-01-01_120000.md").write_text( + "validated: true\n\n# Audit: Missing test coverage\n\nBody content." + ) + with ( + patch("autoskillit.recipe._cmd_rpc.subprocess.run") as mock_run, + patch("autoskillit.recipe._cmd_rpc.time.sleep"), + ): + mock_run.side_effect = _make_side_effect() + batch_create_issues(workspace=str(tmp_path)) + found = False + for call in mock_run.call_args_list: + kwargs = call[1] + if kwargs.get("input"): + mutation_call = json.loads(kwargs["input"]) + assert mutation_call["variables"]["i0"]["title"] == "Audit: Missing test coverage" + found = True + break + assert found, "no createIssue mutation call found in mock_run calls" + + +def test_batch_create_issues_constructs_graphql_mutation(tmp_path): + va_dir = tmp_path / ".autoskillit" / "temp" / "validate-audit" + va_dir.mkdir(parents=True) + (va_dir / "ticket_body_tests_1_2026-01-01_120000.md").write_text("# Issue One\n\nBody one.") + (va_dir / "ticket_body_tests_2_2026-01-01_120000.md").write_text("# Issue Two\n\nBody two.") + with ( + patch("autoskillit.recipe._cmd_rpc.subprocess.run") as mock_run, + patch("autoskillit.recipe._cmd_rpc.time.sleep"), + ): + mock_run.side_effect = _make_side_effect( + issue_data=[ + {"number": 1, "url": "https://github.com/org/repo/issues/1"}, + {"number": 2, "url": "https://github.com/org/repo/issues/2"}, + ] + ) + batch_create_issues(workspace=str(tmp_path)) + found = False + for call in mock_run.call_args_list: + kwargs = call[1] + if kwargs.get("input"): + mutation_call = json.loads(kwargs["input"]) + query = mutation_call["query"] + variables = mutation_call["variables"] + assert "issue0: createIssue" in query + assert "issue1: createIssue" in query + assert variables["i0"]["repositoryId"] == "R_123" + assert variables["i1"]["repositoryId"] == "R_123" + assert variables["i0"]["labelIds"] == ["L_1", "L_2"] + assert variables["i1"]["labelIds"] == ["L_1", "L_2"] + found = True + break + assert found, "no createIssue mutation call found in mock_run calls" + + +def test_batch_create_issues_chunks_large_batches(tmp_path): + va_dir = tmp_path / ".autoskillit" / "temp" / "validate-audit" + va_dir.mkdir(parents=True) + for n in range(25): + (va_dir / f"ticket_body_tests_{n + 1}_2026-01-01_120000.md").write_text( + f"# Issue {n + 1}\n\nBody {n + 1}." + ) + with ( + patch("autoskillit.recipe._cmd_rpc.subprocess.run") as mock_run, + patch("autoskillit.recipe._cmd_rpc.time.sleep"), + ): + + def side_effect_factory(): + call_count = [0] + + def side_effect(_args, **_kwargs): + c = call_count[0] + call_count[0] += 1 + if c == 0: + return subprocess.CompletedProcess( + args=[], returncode=0, stdout="org repo\n", stderr="" + ) + if c == 1: + return subprocess.CompletedProcess( + args=[], + returncode=0, + stdout=json.dumps({"data": {"repository": {"id": "R_123"}}}), + stderr="", + ) + if c in (2, 3): + return subprocess.CompletedProcess(args=[], returncode=0, stdout="", stderr="") + if c == 4: + return subprocess.CompletedProcess( + args=[], + returncode=0, + stdout=json.dumps( + {"data": {"repository": {"impl": {"id": "L_1"}, "enh": {"id": "L_2"}}}} + ), + stderr="", + ) + if c == 5: + data5 = { + f"issue{i}": { + "issue": { + "number": i + 1, + "url": f"https://github.com/org/repo/issues/{i + 1}", + } + } + for i in range(10) + } + return subprocess.CompletedProcess( + args=[], returncode=0, stdout=json.dumps({"data": data5}), stderr="" + ) + if c == 6: + data6 = { + f"issue{i}": { + "issue": { + "number": i + 11, + "url": f"https://github.com/org/repo/issues/{i + 11}", + } + } + for i in range(10) + } + return subprocess.CompletedProcess( + args=[], returncode=0, stdout=json.dumps({"data": data6}), stderr="" + ) + if c == 7: + data7 = { + f"issue{i}": { + "issue": { + "number": i + 21, + "url": f"https://github.com/org/repo/issues/{i + 21}", + } + } + for i in range(5) + } + return subprocess.CompletedProcess( + args=[], returncode=0, stdout=json.dumps({"data": data7}), stderr="" + ) + return subprocess.CompletedProcess(args=[], returncode=0, stdout="", stderr="") + + return side_effect + + mock_run.side_effect = side_effect_factory() + result = batch_create_issues(workspace=str(tmp_path), chunk_size="10") + mutation_calls = sum( + 1 + for call in mock_run.call_args_list + if call[1].get("input") and "createIssue" in call[1]["input"] + ) + assert mutation_calls == 3 + assert result["issue_count"] == "25" + + +def test_batch_create_issues_ignores_validation_summary_file(tmp_path): + """batch_create_issues must not append validation_summary content to issue bodies. + + The validation_summary file is a bulk audit artifact covering all findings + for a source. SKILL.md § Issue Body Construction states it is NOT part of the issue body. + """ + va_dir = tmp_path / ".autoskillit" / "temp" / "validate-audit" + va_dir.mkdir(parents=True) + (va_dir / "ticket_body_tests_1_2026-01-01_120000.md").write_text( + "validated: true\n\n# Audit Finding\n\nSome finding." + ) + (va_dir / "validation_summary_tests_2026-01-01_120000.md").write_text( + "## Validation Summary\nAll clear." + ) + with ( + patch("autoskillit.recipe._cmd_rpc.subprocess.run") as mock_run, + patch("autoskillit.recipe._cmd_rpc.time.sleep"), + ): + mock_run.side_effect = _make_side_effect() + batch_create_issues(workspace=str(tmp_path)) + for call in mock_run.call_args_list: + kwargs = call[1] + if kwargs.get("input"): + mutation_call = json.loads(kwargs["input"]) + body = mutation_call["variables"]["i0"]["body"] + assert "## Validation Summary" not in body, ( + "validation_summary content must not be appended to issue bodies" + ) + assert "All clear." not in body, ( + "validation_summary content must not be appended to issue bodies" + ) + return + pytest.fail("no createIssue mutation call found in mock_run calls") + + +def test_batch_create_issues_handles_no_tickets(tmp_path): + va_dir = tmp_path / ".autoskillit" / "temp" / "validate-audit" + va_dir.mkdir(parents=True) + # Leave directory empty + result = batch_create_issues(workspace=str(tmp_path)) + assert result == {"issue_urls": "", "issue_count": "0"} + + +def test_batch_create_issues_handles_graphql_error(tmp_path): + va_dir = tmp_path / ".autoskillit" / "temp" / "validate-audit" + va_dir.mkdir(parents=True) + (va_dir / "ticket_body_tests_1_2026-01-01_120000.md").write_text("# One Issue\n\nBody.") + error_side_effect = [ + subprocess.CompletedProcess(args=[], returncode=0, stdout="org repo\n", stderr=""), + subprocess.CompletedProcess( + args=[], + returncode=0, + stdout=json.dumps({"data": {"repository": {"id": "R_123"}}}), + stderr="", + ), + subprocess.CompletedProcess(args=[], returncode=0, stdout="", stderr=""), + subprocess.CompletedProcess(args=[], returncode=0, stdout="", stderr=""), + subprocess.CompletedProcess( + args=[], + returncode=0, + stdout=json.dumps( + {"data": {"repository": {"impl": {"id": "L_1"}, "enh": {"id": "L_2"}}}} + ), + stderr="", + ), + subprocess.CompletedProcess(args=[], returncode=1, stdout="", stderr="rate limited"), + ] + with ( + patch("autoskillit.recipe._cmd_rpc.subprocess.run") as mock_run, + patch("autoskillit.recipe._cmd_rpc.time.sleep"), + ): + mock_run.side_effect = error_side_effect + with pytest.raises(RuntimeError, match="rate limited"): + batch_create_issues(workspace=str(tmp_path)) + + +# ─── Type coercion: int pr_number for _cmd_rpc callables (Step 1c) ─────────── + + +@patch("autoskillit.recipe._cmd_rpc.subprocess.run") +@patch("autoskillit.recipe._cmd_rpc.time.sleep") +def test_wait_for_direct_merge_int_pr_number(mock_sleep, mock_run): + """wait_for_direct_merge handles int pr_number from LLM JSON boundary.""" + mock_run.return_value = subprocess.CompletedProcess( + args=[], returncode=0, stdout="MERGED\n", stderr="" + ) + result = wait_for_direct_merge(pr_number=42, max_polls="1", poll_interval="1") # type: ignore[arg-type] + assert result["state"] == "merged" + # Verify the gh call was made with the PR number (coerced to str by str() guard) + assert mock_run.call_count >= 1 + + +# ─── Multi-run accumulation tests for batch_create_issues ──────────────────── + + +def test_batch_create_issues_ignores_prior_run_files(tmp_path): + """batch_create_issues must only process files from the current run, not prior runs. + + Reproduces the bug: without audit_run_dir scoping, the callable globs ALL + ticket_body_*.md files in the flat validate-audit/ directory, including those + written by previous pipeline runs. The issue_count should reflect only the + files present when the callable is invoked. + """ + va_dir = tmp_path / ".autoskillit" / "temp" / "validate-audit" + va_dir.mkdir(parents=True) + + # Simulate Run 1: 3 ticket bodies + for n in range(1, 4): + (va_dir / f"ticket_body_tests_{n}_2026-05-05_120000.md").write_text( + f"# Issue {n}\n\nBody from Run 1." + ) + + # Call batch_create_issues — should return 3 issues + with ( + patch("autoskillit.recipe._cmd_rpc.subprocess.run") as mock_run, + patch("autoskillit.recipe._cmd_rpc.time.sleep"), + ): + mock_run.side_effect = _make_side_effect( + issue_data=[ + {"number": 1, "url": "https://github.com/org/repo/issues/1"}, + {"number": 2, "url": "https://github.com/org/repo/issues/2"}, + {"number": 3, "url": "https://github.com/org/repo/issues/3"}, + ] + ) + result = batch_create_issues(workspace=str(tmp_path)) + + assert result["issue_count"] == "3" + + # Simulate Run 2: 2 MORE ticket bodies added (total 5 in directory) + for n in range(4, 6): + (va_dir / f"ticket_body_tests_{n}_2026-05-06_130000.md").write_text( + f"# Issue {n}\n\nBody from Run 2." + ) + + # Call batch_create_issues WITHOUT audit_run_dir — globs all 5 files + with ( + patch("autoskillit.recipe._cmd_rpc.subprocess.run") as mock_run, + patch("autoskillit.recipe._cmd_rpc.time.sleep"), + ): + mock_run.side_effect = _make_side_effect( + issue_data=[ + {"number": i, "url": f"https://github.com/org/repo/issues/{i}"} + for i in range(1, 6) + ] + ) + result = batch_create_issues(workspace=str(tmp_path)) + + # Without audit_run_dir, batch_create_issues counts ALL files in the directory + assert result["issue_count"] == "5", ( + "Without audit_run_dir, batch_create_issues counts ALL files in the directory" + ) + + +def test_batch_create_issues_scoped_to_audit_run_dir(tmp_path): + """batch_create_issues with audit_run_dir must only process files in that directory. + + This is the key test for the fix: when audit_run_dir is provided, the callable + must glob within that specific run directory, not the flat validate-audit/ dir. + """ + va_dir = tmp_path / ".autoskillit" / "temp" / "validate-audit" + va_dir.mkdir(parents=True) + + # Create two per-run subdirectories (simulating two pipeline runs) + run1_dir = va_dir / "run-20260505-120000-aabb1122" + run2_dir = va_dir / "run-20260506-130000-ccdd3344" + run1_dir.mkdir() + run2_dir.mkdir() + + # Run 1 files + for n in range(1, 4): + (run1_dir / f"ticket_body_tests_{n}_2026-05-05_120000.md").write_text( + f"# Issue {n}\n\nBody from Run 1." + ) + + # Run 2 files + for n in range(4, 6): + (run2_dir / f"ticket_body_tests_{n}_2026-05-06_130000.md").write_text( + f"# Issue {n}\n\nBody from Run 2." + ) + + # Call batch_create_issues scoped to run2_dir only + with ( + patch("autoskillit.recipe._cmd_rpc.subprocess.run") as mock_run, + patch("autoskillit.recipe._cmd_rpc.time.sleep"), + ): + mock_run.side_effect = _make_side_effect( + issue_data=[ + {"number": 4, "url": "https://github.com/org/repo/issues/4"}, + {"number": 5, "url": "https://github.com/org/repo/issues/5"}, + ] + ) + result = batch_create_issues(workspace=str(tmp_path), audit_run_dir=str(run2_dir)) + + assert result["issue_count"] == "2", ( + "batch_create_issues should only process files in audit_run_dir, not the parent directory" + ) + + +def test_batch_create_issues_audit_run_dir_only(tmp_path): + """batch_create_issues with only audit_run_dir (no workspace-derived fallback). + + When audit_run_dir is provided, it must be used as the sole discovery path, + completely ignoring the workspace-derived path. + """ + va_dir = tmp_path / ".autoskillit" / "temp" / "validate-audit" + va_dir.mkdir(parents=True) + + # Files in the workspace-derived path (should be IGNORED) + for n in range(1, 4): + (va_dir / f"ticket_body_tests_{n}_2026-05-05_120000.md").write_text( + "# Stale Issue\n\nShould be ignored." + ) + + # Files in the audit_run_dir (should be processed) + scoped_dir = va_dir / "run-20260506-130000-ccdd3344" + scoped_dir.mkdir() + (scoped_dir / "ticket_body_tests_1_2026-05-06_130000.md").write_text("# Active Issue\n\nBody.") + + with ( + patch("autoskillit.recipe._cmd_rpc.subprocess.run") as mock_run, + patch("autoskillit.recipe._cmd_rpc.time.sleep"), + ): + mock_run.side_effect = _make_side_effect( + issue_data=[{"number": 99, "url": "https://github.com/org/repo/issues/99"}] + ) + result = batch_create_issues(workspace=str(tmp_path), audit_run_dir=str(scoped_dir)) + + assert result["issue_count"] == "1", ( + "batch_create_issues should only process files in audit_run_dir" + ) + + +def test_batch_create_issues_audit_run_dir_not_a_directory(tmp_path): + """batch_create_issues raises ValueError when audit_run_dir does not exist.""" + bogus = str(tmp_path / "nonexistent") + with pytest.raises(ValueError, match="audit_run_dir must be an existing directory"): + batch_create_issues(workspace=str(tmp_path), audit_run_dir=bogus) + + +def test_create_audit_run_dir_wraps_os_error(tmp_path, monkeypatch): + """create_audit_run_dir wraps OSError with a descriptive ValueError.""" + monkeypatch.setattr(Path, "mkdir", _raise_os_error) + with pytest.raises(ValueError, match="cannot create audit run directory"): + create_audit_run_dir(temp_dir=str(tmp_path)) + + +def _raise_os_error(*_args, **_kwargs): + raise OSError("Permission denied") + + +@patch("autoskillit.recipe._cmd_rpc.subprocess.run") +@patch("autoskillit.recipe._cmd_rpc.time.sleep") +def test_force_push_int_review_pr_number(mock_sleep, mock_run, tmp_path): + """force_push_and_wait_mergeability handles int review_pr_number.""" + with patch("autoskillit.recipe._cmd_rpc._detect_remote", return_value="origin"): + mock_run.side_effect = [ + subprocess.CompletedProcess(args=[], returncode=0, stdout="", stderr=""), + subprocess.CompletedProcess(args=[], returncode=0, stdout="TRUE\n", stderr=""), + ] + result = force_push_and_wait_mergeability( + work_dir=str(tmp_path), + batch_branch="feat-x/42", + review_pr_number=1958, # type: ignore[arg-type] + max_polls="1", + poll_interval="1", + ) + assert result["ok"] == "true" + # Verify gh pr view was called (coerced to str by str() guard) + assert mock_run.call_count >= 2 diff --git a/tests/recipe/test_cmd_rpc_null_safety.py b/tests/recipe/test_cmd_rpc_null_safety.py new file mode 100644 index 0000000000..d8e8027e0a --- /dev/null +++ b/tests/recipe/test_cmd_rpc_null_safety.py @@ -0,0 +1,21 @@ +"""Null-safety tests for run_python callables in recipe/_cmd_rpc.py.""" + +from __future__ import annotations + +import pytest + +from autoskillit.recipe._cmd_rpc import check_dropped_healthy_loop, check_eject_limit + +pytestmark = [pytest.mark.layer("recipe"), pytest.mark.small] + + +def test_check_eject_limit_none_max(tmp_path) -> None: + counter = tmp_path / "counter" + result = check_eject_limit(counter_file=str(counter), max_ejects=None) # type: ignore[arg-type] + assert "status" in result + + +def test_check_dropped_healthy_loop_none_max(tmp_path) -> None: + counter = tmp_path / "counter" + result = check_dropped_healthy_loop(counter_file=str(counter), max_drops=None) # type: ignore[arg-type] + assert "status" in result diff --git a/tests/recipe/test_contract_verdict_output_required.py b/tests/recipe/test_contract_verdict_output_required.py index 3b0a32ca31..de797e72dd 100644 --- a/tests/recipe/test_contract_verdict_output_required.py +++ b/tests/recipe/test_contract_verdict_output_required.py @@ -15,6 +15,8 @@ from autoskillit.recipe.contracts import load_bundled_manifest +pytestmark = [pytest.mark.layer("recipe"), pytest.mark.small] + _AUTO_FIX_SKILLS = [ "resolve-failures", "resolve-review", @@ -139,7 +141,12 @@ def test_conditional_write_skills_have_verdict_or_fixes_applied_declared() -> No Excludes resolve-merge-conflicts (different oracle: pre-commit + manifest, out of scope) and retry-worktree (phases_implemented oracle, deferred to a separate Part). """ - _EXCLUDED = {"resolve-merge-conflicts", "retry-worktree"} + _EXCLUDED = { + "resolve-merge-conflicts", + "retry-worktree", + "prepare-pr", + "bundle-local-report", + } manifest = load_bundled_manifest() violations: list[str] = [] for skill_name, skill in manifest.get("skills", {}).items(): diff --git a/tests/recipe/test_contracts.py b/tests/recipe/test_contracts.py index 57e4012f1c..5c3470d159 100644 --- a/tests/recipe/test_contracts.py +++ b/tests/recipe/test_contracts.py @@ -18,6 +18,8 @@ ) from autoskillit.workspace import bundled_skills_extended_dir +pytestmark = [pytest.mark.layer("recipe"), pytest.mark.small] + # --------------------------------------------------------------------------- # Bundled manifest tests # --------------------------------------------------------------------------- @@ -47,6 +49,33 @@ def test_load_bundled_manifest_skill_inputs_typed() -> None: ) +def test_load_bundled_manifest_callable_inputs_have_explicit_required() -> None: + manifest = load_bundled_manifest() + for dotted_path, entry in manifest.get("callable_contracts", {}).items(): + for inp in entry.get("inputs", []): + assert "required" in inp, f"{dotted_path}: input {inp['name']} missing 'required'" + + +def test_get_skill_contract_defaults_required_false() -> None: + from autoskillit.recipe.contracts import get_skill_contract + + manifest = {"skills": {"test-skill": {"inputs": [{"name": "x", "type": "string"}]}}} + contract = get_skill_contract("test-skill", manifest) + assert contract is not None + assert len(contract.inputs) == 1 + assert contract.inputs[0].required is False + + +def test_get_callable_contract_defaults_required_true() -> None: + from autoskillit.recipe.contracts import get_callable_contract + + manifest = {"callable_contracts": {"mod.func": {"inputs": [{"name": "x", "type": "string"}]}}} + contract = get_callable_contract("mod.func", manifest) + assert contract is not None + assert len(contract.inputs) == 1 + assert contract.inputs[0].required is True + + # --------------------------------------------------------------------------- # resolve_skill_name tests # --------------------------------------------------------------------------- @@ -70,9 +99,9 @@ def test_resolve_skill_name_with_use_prefix() -> None: def test_resolve_skill_name_no_prefix() -> None: - from autoskillit.recipe.contracts import resolve_skill_name + from autoskillit.core import resolve_skill_name - assert resolve_skill_name("/do-stuff") is None + assert resolve_skill_name("/do-stuff") == "do-stuff" def test_resolve_skill_name_dynamic() -> None: @@ -536,6 +565,7 @@ def test_pipeline_summary_contract_declared() -> None: "resolve-merge-conflicts", "retry-worktree", "review-pr", + "planner-generate-phases", "arch-lens-c4-container", "arch-lens-concurrency", "arch-lens-data-lineage", @@ -627,15 +657,44 @@ def test_write_behavior_defaults_to_none() -> None: ALWAYS_WRITE_SKILLS = { + "audit-tests", + "build-execution-map", + "compose-pr", + "compose-research-pr", + "design-guards", + "diagnose-ci", "dry-walkthrough", + "generate-report", + "implement-experiment", "implement-worktree", "implement-worktree-no-merge", - "rectify", + "make-campaign", + "make-groups", "make-plan", + "plan-experiment", + "plan-visualization", + "planner-consolidate-wps", + "planner-elaborate-assignments", + "planner-elaborate-phase", + "planner-elaborate-wps", + "planner-generate-phases", + "planner-refine-assignments", + "planner-refine-phases", + "planner-refine-wps", + "planner-assess-review-approach", + "planner-validate-task-alignment", + "prepare-research-pr", + "promote-to-main", + "rectify", "report-bug", - "design-guards", + "resolve-design-review", + "review-design", + "run-experiment", + "scope", + "setup-environment", + "stage-data", + "troubleshoot-experiment", "write-recipe", - "diagnose-ci", } @@ -650,6 +709,21 @@ def test_every_always_write_skill_has_contract(skill_name: str) -> None: ) +def test_always_write_skills_matches_yaml() -> None: + """ALWAYS_WRITE_SKILLS test set must equal the set from skill_contracts.yaml.""" + manifest = load_bundled_manifest() + yaml_always = { + name + for name, data in manifest.get("skills", {}).items() + if data.get("write_behavior") == "always" + } + assert ALWAYS_WRITE_SKILLS == yaml_always, ( + f"ALWAYS_WRITE_SKILLS is out of sync with skill_contracts.yaml.\n" + f"In test but not YAML: {ALWAYS_WRITE_SKILLS - yaml_always}\n" + f"In YAML but not test: {yaml_always - ALWAYS_WRITE_SKILLS}" + ) + + # Skills that write conditionally — write expected only when the completion # token indicates actual work was performed. CONDITIONAL_WRITE_SKILLS: dict[str, str] = { @@ -693,6 +767,24 @@ def test_every_conditional_write_skill_has_correct_contract( # --------------------------------------------------------------------------- +def test_prepare_pr_contract_is_conditional() -> None: + """prepare-pr must be conditional — it has a documented no-write exit path.""" + manifest = load_bundled_manifest() + contract = get_skill_contract("prepare-pr", manifest) + assert contract is not None + assert contract.write_behavior == "conditional" + assert contract.write_expected_when + + +def test_bundle_local_report_contract_is_conditional() -> None: + """bundle-local-report must be conditional — it has a documented no-write exit path.""" + manifest = load_bundled_manifest() + contract = get_skill_contract("bundle-local-report", manifest) + assert contract is not None + assert contract.write_behavior == "conditional" + assert contract.write_expected_when + + def test_dataflow_entry_uppercase_f() -> None: """DataFlowEntry (uppercase F) must be importable; old DataflowEntry must be gone.""" import autoskillit.recipe.contracts as m @@ -701,3 +793,189 @@ def test_dataflow_entry_uppercase_f() -> None: assert not hasattr(m, "DataflowEntry"), "DataflowEntry (lowercase f) must be removed" entry = DataFlowEntry(step="s", available=[], required=[], produced=[]) assert entry.step == "s" + + +def test_all_bundled_recipe_skills_in_master_manifest() -> None: + """T1d: every skill name used in a run_skill step across all bundled recipes must have + an entry in skill_contracts.yaml (dynamic/template skill names are excluded).""" + from autoskillit.recipe.contracts import resolve_skill_name + from autoskillit.recipe.io import builtin_recipes_dir, load_recipe + + manifest = load_bundled_manifest() + registered_skills = set(manifest.get("skills", {}).keys()) + + recipes_dir = builtin_recipes_dir() + unregistered: list[str] = [] + + for recipe_path in sorted(recipes_dir.glob("*.yaml")): + recipe = load_recipe(recipe_path) + for step in recipe.steps.values(): + if step.tool != "run_skill": + continue + skill_cmd = step.with_args.get("skill_command", "") + name = resolve_skill_name(skill_cmd) + if name is None: + continue # dynamic skill name — can't validate statically + if name not in registered_skills: + unregistered.append(f"{recipe_path.name}:{name}") + + assert not unregistered, ( + f"Skills used in bundled recipes but missing from skill_contracts.yaml: " + f"{sorted(set(unregistered))}" + ) + + +def test_result_field_spec_roundtrip() -> None: + """ResultFieldSpec must be parseable from skill_contracts.yaml for planner-generate-phases.""" + from autoskillit.recipe.contracts import ResultFieldSpec, get_skill_contract + + manifest = load_bundled_manifest() + contract = get_skill_contract("planner-generate-phases", manifest) + assert contract is not None + required_names = {rf.name for rf in contract.result_fields if rf.required} + assert required_names == {"id", "name", "ordering"}, ( + f"planner-generate-phases result_fields required names mismatch: {required_names}" + ) + for rf in contract.result_fields: + assert isinstance(rf, ResultFieldSpec) + assert rf.required is True + + +def test_callable_contract_nullable_field() -> None: + from autoskillit.recipe.contracts import get_callable_contract + + manifest = load_bundled_manifest() + contracts = manifest.get("callable_contracts", {}) + assert "autoskillit.recipe._cmd_rpc.advance_queue_pr" in contracts, ( + "advance_queue_pr must be declared in callable_contracts" + ) + aq_contract = contracts["autoskillit.recipe._cmd_rpc.advance_queue_pr"] + pr_input = next((i for i in aq_contract["inputs"] if i["name"] == "current_pr_number"), None) + assert pr_input is not None, "current_pr_number input must be declared" + assert pr_input.get("nullable") is False, ( + "current_pr_number must be explicitly non-nullable (nullable: false)" + ) + parsed = get_callable_contract("autoskillit.recipe._cmd_rpc.advance_queue_pr", manifest) + assert parsed is not None + pr_parsed = next((i for i in parsed.inputs if i.name == "current_pr_number"), None) + assert pr_parsed is not None + assert pr_parsed.nullable is False + + +# --------------------------------------------------------------------------- +# tool_output_contracts +# --------------------------------------------------------------------------- + + +def test_tool_output_contracts_section_exists() -> None: + """skill_contracts.yaml must have a tool_output_contracts section.""" + manifest = load_bundled_manifest() + tool_contracts = manifest.get("tool_output_contracts", {}) + assert "wait_for_ci" in tool_contracts + assert "allowed_values" in tool_contracts["wait_for_ci"]["fields"]["conclusion"] + + +def test_wait_for_ci_conclusion_allowed_values() -> None: + manifest = load_bundled_manifest() + conclusion = manifest["tool_output_contracts"]["wait_for_ci"]["fields"]["conclusion"] + values = set(conclusion["allowed_values"]) + assert {"success", "failure", "cancelled", "timed_out", "no_runs", "error"}.issubset(values) + + +# --------------------------------------------------------------------------- +# Research recipe contract card tests +# --------------------------------------------------------------------------- + + +def test_research_recipe_card_generated_at_is_iso8601() -> None: + """research.yaml card's generated_at is a parseable ISO-8601 timestamp.""" + import datetime + + from autoskillit.recipe.io import builtin_recipes_dir + + card = load_recipe_card("research", builtin_recipes_dir()) + assert card is not None + generated_at = card.get("generated_at", "") + assert isinstance(generated_at, str) and generated_at, ( + "generated_at must be a non-empty string" + ) + try: + datetime.datetime.fromisoformat(generated_at) + except ValueError as exc: + pytest.fail(f"generated_at is not a valid ISO-8601 datetime: {generated_at!r}: {exc}") + + +def test_research_recipe_card_structure() -> None: + """Research contract card contains all required top-level keys.""" + from autoskillit.recipe.io import builtin_recipes_dir + + card = load_recipe_card("research", builtin_recipes_dir()) + assert card is not None + for key in ("generated_at", "bundled_manifest_version", "skill_hashes", "skills", "dataflow"): + assert key in card, f"Missing key: {key}" + assert card["bundled_manifest_version"] == "0.1.0" + assert card["skill_hashes"] == {} + + +def test_research_recipe_card_contains_research_skills() -> None: + """Research card captures contracts for statically-resolvable skills.""" + from autoskillit.recipe.io import builtin_recipes_dir + + card = load_recipe_card("research", builtin_recipes_dir()) + assert card is not None + skills = card["skills"] + expected_subset = { + "scope", + "plan-experiment", + "review-design", + "run-experiment", + "generate-report", + "resolve-failures", + "prepare-research-pr", + "compose-research-pr", + "implement-experiment", + "make-groups", + "make-plan", + "bundle-local-report", + } + required_fields = {"inputs", "outputs", "expected_output_patterns", "write_behavior"} + for skill_name in expected_subset: + assert skill_name in skills, f"Missing skill: {skill_name}" + entry = skills[skill_name] + for field in required_fields: + assert field in entry, f"{skill_name} missing '{field}'" + assert isinstance(entry["inputs"], list), f"{skill_name} 'inputs' must be a list" + assert isinstance(entry["outputs"], list), f"{skill_name} 'outputs' must be a list" + assert isinstance(entry["expected_output_patterns"], list), ( + f"{skill_name} 'expected_output_patterns' must be a list" + ) + assert entry["write_behavior"] in {"always", "conditional", "never"}, ( + f"{skill_name} 'write_behavior' has unexpected value: {entry['write_behavior']!r}" + ) + + +def test_research_recipe_card_dataflow() -> None: + """Research card dataflow section is populated and has expected steps.""" + from autoskillit.recipe.io import builtin_recipes_dir + + card = load_recipe_card("research", builtin_recipes_dir()) + assert card is not None + dataflow = card["dataflow"] + step_names = [entry["step"] for entry in dataflow] + assert "scope" in step_names + assert "plan_experiment" in step_names + for entry in dataflow: + assert "step" in entry, f"Dataflow entry missing 'step': {entry}" + assert "available" in entry, f"Step '{entry.get('step')}' missing 'available'" + assert "required" in entry, f"Step '{entry.get('step')}' missing 'required'" + assert "produced" in entry, f"Step '{entry.get('step')}' missing 'produced'" + + +def test_research_contract_card_not_stale() -> None: + """Freshly generated card has no staleness warnings.""" + from autoskillit.recipe.io import builtin_recipes_dir + + contract = load_recipe_card("research", builtin_recipes_dir()) + assert contract is not None + stale = check_contract_staleness(contract) + assert stale == [] diff --git a/tests/recipe/test_contracts_block_fingerprint.py b/tests/recipe/test_contracts_block_fingerprint.py index 002b37860e..e8df6a7daf 100644 --- a/tests/recipe/test_contracts_block_fingerprint.py +++ b/tests/recipe/test_contracts_block_fingerprint.py @@ -2,6 +2,8 @@ from __future__ import annotations +import pytest + from autoskillit.recipe.contracts import ( check_contract_staleness, generate_recipe_card, @@ -9,6 +11,8 @@ from autoskillit.recipe.io import builtin_recipes_dir, load_recipe from autoskillit.recipe.schema import Recipe, RecipeStep +pytestmark = [pytest.mark.layer("recipe"), pytest.mark.small] + def test_recipe_card_contains_block_fingerprint_for_every_declared_block(): """generate_recipe_card(recipe) must produce a RecipeCard with block_fingerprints diff --git a/tests/recipe/test_diagnose_ci_subtype_output.py b/tests/recipe/test_diagnose_ci_subtype_output.py index 12c9dc90b9..e63d25b9ac 100644 --- a/tests/recipe/test_diagnose_ci_subtype_output.py +++ b/tests/recipe/test_diagnose_ci_subtype_output.py @@ -10,9 +10,13 @@ import re +import pytest + from autoskillit.core import pkg_root from autoskillit.recipe.contracts import load_bundled_manifest +pytestmark = [pytest.mark.layer("recipe"), pytest.mark.small] + _SKILL_MD = pkg_root() / "skills_extended" / "diagnose-ci" / "SKILL.md" diff --git a/tests/recipe/test_diagrams.py b/tests/recipe/test_diagrams.py index 575301cbf2..eac5ad82c5 100644 --- a/tests/recipe/test_diagrams.py +++ b/tests/recipe/test_diagrams.py @@ -14,6 +14,8 @@ ) from autoskillit.recipe.io import load_recipe +pytestmark = [pytest.mark.layer("recipe"), pytest.mark.medium] + def _extract_graph_section(content: str) -> str: """Extract the ### Graph section content.""" @@ -337,3 +339,39 @@ def test_spec_fixture_version_matches_diagram_format_constant() -> None: f"spec embeds {m.group(1)!r} but DIAGRAM_FORMAT_VERSION={DIAGRAM_FORMAT_VERSION!r}. " "Either bump DIAGRAM_FORMAT_VERSION or regenerate the spec fixture." ) + + +# --------------------------------------------------------------------------- +# T-FORMAT-GUARD: no bundled diagram contains Mermaid stubs +# --------------------------------------------------------------------------- + + +def test_bundled_diagrams_contain_no_mermaid_stubs() -> None: + """T-FORMAT-GUARD: all bundled recipes/diagrams/*.md must be ASCII art, not Mermaid.""" + import autoskillit + + diagrams_dir = Path(autoskillit.__file__).parent / "recipes" / "diagrams" + assert diagrams_dir.is_dir(), f"diagrams dir missing: {diagrams_dir}" + md_files = sorted(diagrams_dir.glob("*.md")) + assert md_files, "No .md files found in diagrams/" + for md_file in md_files: + content = md_file.read_text(encoding="utf-8") + assert "flowchart TD" not in content, ( + f"{md_file.name} contains 'flowchart TD' (Mermaid stub). " + "Bundled diagrams must be ASCII art produced by /render-recipe." + ) + + +# --------------------------------------------------------------------------- +# T-SPEC-1: generate_recipe_diagram removed +# --------------------------------------------------------------------------- + + +def test_generate_recipe_diagram_removed() -> None: + """T-SPEC-1: generate_recipe_diagram must not exist — rendering is skill-only.""" + import autoskillit.recipe.diagrams as diagrams_mod + + assert not hasattr(diagrams_mod, "generate_recipe_diagram"), ( + "generate_recipe_diagram still exists in diagrams module. " + "Diagram rendering must be handled by /render-recipe skill only." + ) diff --git a/tests/recipe/test_experiment_type_registry.py b/tests/recipe/test_experiment_type_registry.py index 192828acc8..9562659019 100644 --- a/tests/recipe/test_experiment_type_registry.py +++ b/tests/recipe/test_experiment_type_registry.py @@ -2,21 +2,35 @@ from __future__ import annotations +import os +from dataclasses import replace from pathlib import Path +import pytest import yaml from autoskillit.recipe.experiment_type_registry import ( ExperimentTypeSpec, + get_experiment_type_by_name, + is_silent_type, load_all_experiment_types, ) +pytestmark = [pytest.mark.layer("recipe"), pytest.mark.small] + EXPECTED_TYPES = { "benchmark", "configuration_study", "causal_inference", "robustness_audit", "exploratory", + "evidence_synthesis", + "factorial_design", + "simulation_modeling", + "instrument_validation", + "single_subject", + "observational_correlational", + "qualitative_interpretive", } VALID_WEIGHT_VALUES = {"H", "M", "L", "S"} @@ -32,67 +46,79 @@ "agent_implementability", } +NEW_TYPES = { + "evidence_synthesis", + "factorial_design", + "simulation_modeling", + "instrument_validation", + "single_subject", + "observational_correlational", + "qualitative_interpretive", +} + def test_all_bundled_types_present() -> None: - """All 5 bundled types load without error.""" + """All 12 bundled types load without error.""" types = load_all_experiment_types() - assert set(types.keys()) == EXPECTED_TYPES + assert {s.name for s in types} == EXPECTED_TYPES def test_each_type_has_required_fields() -> None: """Each type spec has all required fields with correct structure.""" types = load_all_experiment_types() - for name, spec in types.items(): - assert spec.name == name, f"{name}: spec.name mismatch" - assert isinstance(spec.classification_triggers, list), f"{name}: triggers not list" - assert len(spec.classification_triggers) >= 1, f"{name}: no triggers" - assert isinstance(spec.dimension_weights, dict), f"{name}: weights not dict" - assert isinstance(spec.applicable_lenses, dict), f"{name}: lenses not dict" - assert isinstance(spec.red_team_focus, dict), f"{name}: red_team_focus not dict" - assert isinstance(spec.l1_severity, dict), f"{name}: l1_severity not dict" + for spec in types: + assert isinstance(spec.classification_triggers, list), f"{spec.name}: triggers not list" + assert len(spec.classification_triggers) >= 1, f"{spec.name}: no triggers" + assert isinstance(spec.dimension_weights, dict), f"{spec.name}: weights not dict" + assert isinstance(spec.applicable_lenses, dict), f"{spec.name}: lenses not dict" + assert isinstance(spec.red_team_focus, dict), f"{spec.name}: red_team_focus not dict" + assert isinstance(spec.l1_severity, dict), f"{spec.name}: l1_severity not dict" def test_all_weight_values_are_valid() -> None: """All dimension_weights values are one of H, M, L, S.""" types = load_all_experiment_types() - for name, spec in types.items(): + for spec in types: for dim, weight in spec.dimension_weights.items(): assert weight in VALID_WEIGHT_VALUES, ( - f"{name}.dimension_weights[{dim!r}] = {weight!r} — not in {VALID_WEIGHT_VALUES}" + f"{spec.name}.dimension_weights[{dim!r}] = {weight!r} " + f"— not in {VALID_WEIGHT_VALUES}" ) def test_all_eight_dimensions_present() -> None: """All 8 dimensions from the SKILL.md matrix are present in each bundled type.""" types = load_all_experiment_types() - for name, spec in types.items(): + for spec in types: missing = EXPECTED_DIMENSIONS - set(spec.dimension_weights.keys()) - assert not missing, f"{name} missing dimensions: {missing}" + assert not missing, f"{spec.name} missing dimensions: {missing}" def test_dimension_weights_match_skill_matrix() -> None: """Spot-check dimension weights against the values in SKILL.md.""" types = load_all_experiment_types() - bench = types["benchmark"] + by_name = {s.name: s for s in types} + + bench = by_name["benchmark"] assert bench.dimension_weights["causal_structure"] == "S" assert bench.dimension_weights["variance_protocol"] == "H" assert bench.dimension_weights["agent_implementability"] == "H" assert bench.dimension_weights["statistical_corrections"] == "M" - causal = types["causal_inference"] + causal = by_name["causal_inference"] assert causal.dimension_weights["causal_structure"] == "H" assert causal.dimension_weights["statistical_corrections"] == "H" assert causal.dimension_weights["variance_protocol"] == "L" - config = types["configuration_study"] + config = by_name["configuration_study"] assert config.dimension_weights["causal_structure"] == "S" assert config.dimension_weights["statistical_corrections"] == "H" - robust = types["robustness_audit"] + robust = by_name["robustness_audit"] assert robust.dimension_weights["ecological_validity"] == "H" assert robust.dimension_weights["data_acquisition"] == "H" - exploratory = types["exploratory"] + exploratory = by_name["exploratory"] assert exploratory.dimension_weights["statistical_corrections"] == "S" assert exploratory.dimension_weights["agent_implementability"] == "L" @@ -100,36 +126,39 @@ def test_dimension_weights_match_skill_matrix() -> None: def test_red_team_severity_caps_match_skill() -> None: """Red-team severity caps match values in SKILL.md Step 7 RT_MAX_SEVERITY.""" types = load_all_experiment_types() - assert types["causal_inference"].red_team_focus["severity_cap"] == "critical" - assert types["benchmark"].red_team_focus["severity_cap"] == "warning" - assert types["configuration_study"].red_team_focus["severity_cap"] == "warning" - assert types["robustness_audit"].red_team_focus["severity_cap"] == "warning" - assert types["exploratory"].red_team_focus["severity_cap"] == "info" + by_name = {s.name: s for s in types} + assert by_name["causal_inference"].red_team_focus["severity_cap"] == "critical" + assert by_name["benchmark"].red_team_focus["severity_cap"] == "warning" + assert by_name["configuration_study"].red_team_focus["severity_cap"] == "warning" + assert by_name["robustness_audit"].red_team_focus["severity_cap"] == "warning" + assert by_name["exploratory"].red_team_focus["severity_cap"] == "info" def test_red_team_type_specific_focus_present() -> None: """Each type has a type-specific red_team_focus.specific value.""" types = load_all_experiment_types() - assert "asymmetric effort" in types["benchmark"].red_team_focus["specific"] - assert "overfitting" in types["configuration_study"].red_team_focus["specific"] - assert "backdoor" in types["causal_inference"].red_team_focus["specific"] - assert "threat distribution" in types["robustness_audit"].red_team_focus["specific"] - assert "HARKing" in types["exploratory"].red_team_focus["specific"] + by_name = {s.name: s for s in types} + assert "asymmetric effort" in by_name["benchmark"].red_team_focus["specific"] + assert "overfitting" in by_name["configuration_study"].red_team_focus["specific"] + assert "backdoor" in by_name["causal_inference"].red_team_focus["specific"] + assert "threat distribution" in by_name["robustness_audit"].red_team_focus["specific"] + assert "HARKing" in by_name["exploratory"].red_team_focus["specific"] def test_l1_severity_values() -> None: """l1_severity values are one of: critical, warning, info.""" valid_severities = {"critical", "warning", "info"} types = load_all_experiment_types() - for name, spec in types.items(): + for spec in types: for dim, sev in spec.l1_severity.items(): - assert sev in valid_severities, f"{name}.l1_severity[{dim!r}] = {sev!r}" + assert sev in valid_severities, f"{spec.name}.l1_severity[{dim!r}] = {sev!r}" def test_l1_severity_causal_inference_is_critical() -> None: """causal_inference has critical l1_severity for both L1 dimensions.""" types = load_all_experiment_types() - causal = types["causal_inference"] + by_name = {s.name: s for s in types} + causal = by_name["causal_inference"] assert causal.l1_severity["estimand_clarity"] == "critical" assert causal.l1_severity["hypothesis_falsifiability"] == "critical" @@ -137,7 +166,8 @@ def test_l1_severity_causal_inference_is_critical() -> None: def test_l1_severity_exploratory_is_info() -> None: """exploratory has info l1_severity for both L1 dimensions.""" types = load_all_experiment_types() - exp = types["exploratory"] + by_name = {s.name: s for s in types} + exp = by_name["exploratory"] assert exp.l1_severity["estimand_clarity"] == "info" assert exp.l1_severity["hypothesis_falsifiability"] == "info" @@ -145,7 +175,7 @@ def test_l1_severity_exploratory_is_info() -> None: def test_no_project_dir_returns_bundled_only() -> None: """With project_dir=None, only bundled types are returned.""" types = load_all_experiment_types(project_dir=None) - assert set(types.keys()) == EXPECTED_TYPES + assert {s.name for s in types} == EXPECTED_TYPES def test_user_override_replaces_bundled_type(tmp_path: Path) -> None: @@ -168,15 +198,16 @@ def test_user_override_replaces_bundled_type(tmp_path: Path) -> None: ) ) types = load_all_experiment_types(project_dir=tmp_path) - bench = types["benchmark"] + by_name = {s.name: s for s in types} + bench = by_name["benchmark"] # Custom values take effect assert bench.classification_triggers == ["custom trigger only"] assert bench.dimension_weights == {"causal_structure": "H"} # Bundled fields (variance_protocol, etc.) are NOT present — full replacement assert "variance_protocol" not in bench.dimension_weights # Other bundled types remain intact - assert "causal_inference" in types - assert len(types) == 5 + assert "causal_inference" in by_name + assert len(types) == len(EXPECTED_TYPES) def test_user_new_type_is_added(tmp_path: Path) -> None: @@ -208,18 +239,477 @@ def test_user_new_type_is_added(tmp_path: Path) -> None: ) ) types = load_all_experiment_types(project_dir=tmp_path) - assert "network_analysis" in types - assert len(types) == 6 # 5 bundled + 1 user + by_name = {s.name: s for s in types} + assert "network_analysis" in by_name + assert len(types) == 13 # 12 bundled + 1 user def test_missing_user_override_dir_is_silent(tmp_path: Path) -> None: """A project_dir with no .autoskillit/experiment-types/ is fine — bundled only returned.""" types = load_all_experiment_types(project_dir=tmp_path) - assert set(types.keys()) == EXPECTED_TYPES + assert {s.name for s in types} == EXPECTED_TYPES -def test_returns_dict_of_experiment_type_spec() -> None: - """load_all_experiment_types returns dict[str, ExperimentTypeSpec].""" - types = load_all_experiment_types() - for _name, spec in types.items(): +def test_returns_list_of_experiment_type_spec() -> None: + """load_all_experiment_types returns list[ExperimentTypeSpec].""" + result = load_all_experiment_types() + assert isinstance(result, list) + for spec in result: assert isinstance(spec, ExperimentTypeSpec) + + +def test_causal_inference_classification_triggers_require_manipulation() -> None: + """causal_inference triggers require explicit manipulation/randomization. + + Not just causal language alone. + """ + types = load_all_experiment_types() + by_name = {s.name: s for s in types} + causal = by_name["causal_inference"] + assert causal.classification_triggers == [ + ( + "Explicit randomization, manipulation, or intervention assignment: " + "'randomly assigned', 'intervention applied', 'treatment group', " + "'control group', 'manipulated X', 'assigned to condition'" + ), + ( + "Causal claim ('test whether X causes Y') AND explicit confounder " + "adjustment or treatment assignment described" + ), + ] + + +def test_causal_inference_trigger_covers_rct_language() -> None: + """causal_inference triggers cover RCT keywords: randomized assignment, treatment/control.""" + types = load_all_experiment_types() + by_name = {s.name: s for s in types} + triggers_text = " ".join(by_name["causal_inference"].classification_triggers) + assert "randomly assigned" in triggers_text + assert "treatment group" in triggers_text + assert "control group" in triggers_text + + +def test_no_citation_markers_in_yaml_files() -> None: + """Experiment-type YAML files must not contain synthetic citation markers.""" + from autoskillit.recipe.experiment_type_registry import BUNDLED_EXPERIMENT_TYPES_DIR + + for yaml_path in sorted(BUNDLED_EXPERIMENT_TYPES_DIR.glob("*.yaml")): + content = yaml_path.read_text() + assert chr(0x3010) not in content, f"{yaml_path.name} contains synthetic citation marker" + assert "†L" not in content, f"{yaml_path.name} contains synthetic citation marker" + + +def test_all_types_have_schema_version() -> None: + """Every bundled type has schema_version == '1.0'.""" + types = load_all_experiment_types() + for spec in types: + assert spec.schema_version == "1.0", ( + f"{spec.name}: schema_version = {spec.schema_version!r}" + ) + + +def test_priority_values_are_unique_except_fallback() -> None: + """No duplicate priority values among non-fallback types; exploratory has priority 999.""" + types = load_all_experiment_types() + by_name = {s.name: s for s in types} + non_fallback_priorities = [s.priority for s in types if not s.is_fallback] + assert len(non_fallback_priorities) == len(set(non_fallback_priorities)), ( + "Duplicate priority values among non-fallback types" + ) + assert by_name["exploratory"].priority == 999 + + +def test_only_exploratory_is_fallback() -> None: + """exploratory is the sole fallback type; all others have is_fallback=False.""" + types = load_all_experiment_types() + by_name = {s.name: s for s in types} + assert by_name["exploratory"].is_fallback is True + for spec in types: + if spec.name != "exploratory": + assert spec.is_fallback is False, f"{spec.name}: is_fallback should be False" + + +def test_priority_assignments_match_contract() -> None: + """Each type's priority matches the contract from the implementation plan.""" + EXPECTED_PRIORITIES = { + "causal_inference": 1, + "evidence_synthesis": 2, + "benchmark": 3, + "factorial_design": 4, + "configuration_study": 5, + "simulation_modeling": 6, + "instrument_validation": 7, + "robustness_audit": 8, + "single_subject": 9, + "observational_correlational": 10, + "qualitative_interpretive": 11, + "exploratory": 999, + } + assert set(EXPECTED_PRIORITIES.keys()) == EXPECTED_TYPES + types = load_all_experiment_types() + by_name = {s.name: s for s in types} + for name, expected_priority in EXPECTED_PRIORITIES.items(): + assert by_name[name].priority == expected_priority, ( + f"{name}: expected priority {expected_priority}, got {by_name[name].priority}" + ) + + +def test_new_types_dimension_weight_rationale_coverage() -> None: + """For each new type, every H or M dimension weight has a rationale entry.""" + types = load_all_experiment_types() + by_name = {s.name: s for s in types} + for name in NEW_TYPES: + spec = by_name[name] + for dim, weight in spec.dimension_weights.items(): + if weight in ("H", "M"): + assert dim in spec.dimension_weight_rationale, ( + f"{name}: dimension '{dim}' has weight '{weight}' but no rationale entry" + ) + assert spec.dimension_weight_rationale[dim], ( + f"{name}: dimension '{dim}' rationale is empty" + ) + + +def test_new_types_red_team_severity_caps() -> None: + """evidence_synthesis has severity_cap=critical; all other new types have warning.""" + types = load_all_experiment_types() + by_name = {s.name: s for s in types} + assert by_name["evidence_synthesis"].red_team_focus["severity_cap"] == "critical" + for name in ( + "factorial_design", + "simulation_modeling", + "instrument_validation", + "single_subject", + "observational_correlational", + "qualitative_interpretive", + ): + assert by_name[name].red_team_focus["severity_cap"] == "warning", ( + f"{name}: expected severity_cap=warning" + ) + + +def test_new_types_dimension_weights_spot_check() -> None: + """Spot-check key distinguishing weights for each new type.""" + types = load_all_experiment_types() + by_name = {s.name: s for s in types} + assert by_name["evidence_synthesis"].dimension_weights["measurement_alignment"] == "H" + assert by_name["evidence_synthesis"].dimension_weights["causal_structure"] == "L" + + assert by_name["factorial_design"].dimension_weights["causal_structure"] == "H" + assert by_name["factorial_design"].dimension_weights["ecological_validity"] == "L" + + assert by_name["qualitative_interpretive"].dimension_weights["causal_structure"] == "S" + assert by_name["qualitative_interpretive"].dimension_weights["statistical_corrections"] == "S" + assert by_name["qualitative_interpretive"].dimension_weights["ecological_validity"] == "S" + assert by_name["qualitative_interpretive"].dimension_weights["data_acquisition"] == "L" + + assert by_name["single_subject"].dimension_weights["statistical_corrections"] == "S" + assert by_name["single_subject"].dimension_weights["variance_protocol"] == "H" + + assert by_name["simulation_modeling"].dimension_weights["data_acquisition"] == "L" + assert by_name["simulation_modeling"].dimension_weights["agent_implementability"] == "H" + + assert by_name["instrument_validation"].dimension_weights["causal_structure"] == "S" + assert by_name["instrument_validation"].dimension_weights["measurement_alignment"] == "H" + + assert by_name["observational_correlational"].dimension_weights["ecological_validity"] == "H" + assert by_name["observational_correlational"].dimension_weights["variance_protocol"] == "L" + + +@pytest.mark.parametrize("type_name", sorted(NEW_TYPES)) +def test_new_type_full_schema_valid(type_name: str) -> None: + """Each new type has a complete, well-formed schema.""" + types = load_all_experiment_types() + by_name = {s.name: s for s in types} + spec = by_name[type_name] + assert spec.name == type_name + assert set(spec.dimension_weights.keys()) == EXPECTED_DIMENSIONS + for dim, weight in spec.dimension_weights.items(): + assert weight in VALID_WEIGHT_VALUES, f"{type_name}.{dim} = {weight!r}" + assert isinstance(spec.classification_triggers, list) + assert len(spec.classification_triggers) >= 1 + assert spec.schema_version == "1.0" + assert spec.is_fallback is False + non_fallback_count = sum(1 for s in types if not s.is_fallback) + assert 1 <= spec.priority <= non_fallback_count + assert "estimand_clarity" in spec.l1_severity + assert "hypothesis_falsifiability" in spec.l1_severity + assert "severity_cap" in spec.red_team_focus + assert "specific" in spec.red_team_focus + + +def test_qualitative_interpretive_falsifiability_is_info() -> None: + """qualitative_interpretive has hypothesis_falsifiability=info (unique among all types).""" + types = load_all_experiment_types() + by_name = {s.name: s for s in types} + assert by_name["qualitative_interpretive"].l1_severity["hypothesis_falsifiability"] == "info" + + +def test_evidence_synthesis_estimand_clarity_is_critical() -> None: + """evidence_synthesis has estimand_clarity=critical.""" + types = load_all_experiment_types() + by_name = {s.name: s for s in types} + assert by_name["evidence_synthesis"].l1_severity["estimand_clarity"] == "critical" + + +# --------------------------------------------------------------------------- +# New tests: T1-T7 +# --------------------------------------------------------------------------- + + +def test_exploratory_always_last() -> None: + """exploratory (is_fallback=True) is always the last element in the returned list.""" + types = load_all_experiment_types() + assert types[-1].is_fallback is True + assert types[-1].name == "exploratory" + for spec in types[:-1]: + assert spec.is_fallback is False + + +def test_evaluation_order_matches_canonical() -> None: + """Returned list matches the exact 12-element canonical ordering by priority.""" + types = load_all_experiment_types() + names = [s.name for s in types] + assert names == [ + "causal_inference", + "evidence_synthesis", + "benchmark", + "factorial_design", + "configuration_study", + "simulation_modeling", + "instrument_validation", + "robustness_audit", + "single_subject", + "observational_correlational", + "qualitative_interpretive", + "exploratory", + ] + + +def test_user_types_interleave_by_priority(tmp_path: Path) -> None: + """A user type with priority=4 interleaves after benchmark, before configuration_study.""" + user_dir = tmp_path / ".autoskillit" / "experiment-types" + user_dir.mkdir(parents=True) + (user_dir / "my_custom.yaml").write_text( + yaml.dump( + { + "name": "my_custom", + "priority": 4, + "classification_triggers": ["custom trigger"], + "dimension_weights": { + "causal_structure": "M", + "variance_protocol": "M", + "statistical_corrections": "M", + "ecological_validity": "M", + "measurement_alignment": "M", + "resource_proportionality": "M", + "data_acquisition": "M", + "agent_implementability": "M", + }, + "applicable_lenses": {}, + "red_team_focus": {"specific": "none", "severity_cap": "warning"}, + "l1_severity": { + "estimand_clarity": "warning", + "hypothesis_falsifiability": "warning", + }, + } + ) + ) + types = load_all_experiment_types(project_dir=tmp_path) + names = [s.name for s in types] + # factorial_design has priority=4, my_custom also priority=4 + # sorted by (priority, name): "factorial_design" < "my_custom" alphabetically + # Both at priority 4, so factorial_design comes before my_custom + bench_idx = names.index("benchmark") + custom_idx = names.index("my_custom") + config_idx = names.index("configuration_study") + assert bench_idx < custom_idx < config_idx + + +def test_schema_mismatch_warns(tmp_path: Path) -> None: + """A user type with schema_version != '1.0' triggers a WARNING but still loads.""" + import structlog.testing + + user_dir = tmp_path / ".autoskillit" / "experiment-types" + user_dir.mkdir(parents=True) + (user_dir / "future_type.yaml").write_text( + yaml.dump( + { + "name": "future_type", + "schema_version": "2.0", + "classification_triggers": ["future trigger"], + "dimension_weights": { + "causal_structure": "M", + "variance_protocol": "M", + "statistical_corrections": "M", + "ecological_validity": "M", + "measurement_alignment": "M", + "resource_proportionality": "M", + "data_acquisition": "M", + "agent_implementability": "M", + }, + "applicable_lenses": {}, + "red_team_focus": {"specific": "none", "severity_cap": "warning"}, + "l1_severity": { + "estimand_clarity": "warning", + "hypothesis_falsifiability": "warning", + }, + } + ) + ) + with structlog.testing.capture_logs() as cap_logs: + types = load_all_experiment_types(project_dir=tmp_path) + + assert any(entry.get("schema_version") == "2.0" for entry in cap_logs), ( + "Expected WARNING about schema_version mismatch" + ) + by_name = {s.name: s for s in types} + assert "future_type" in by_name + + +def test_get_experiment_type_by_name_found() -> None: + """get_experiment_type_by_name returns the matching spec for a known type.""" + spec = get_experiment_type_by_name("benchmark") + assert spec is not None + assert spec.name == "benchmark" + assert isinstance(spec, ExperimentTypeSpec) + + +def test_get_experiment_type_by_name_not_found() -> None: + """get_experiment_type_by_name returns None for an unknown type name.""" + spec = get_experiment_type_by_name("nonexistent") + assert spec is None + + +# --------------------------------------------------------------------------- +# T1: Caching tests +# --------------------------------------------------------------------------- + +_MINIMAL_EXP_TYPE_YAML = { + "name": "test_exp", + "classification_triggers": ["test trigger"], + "dimension_weights": { + "causal_structure": "H", + "variance_protocol": "M", + "statistical_corrections": "M", + "ecological_validity": "M", + "measurement_alignment": "M", + "resource_proportionality": "M", + "data_acquisition": "M", + "agent_implementability": "M", + }, + "applicable_lenses": {}, + "red_team_focus": {"specific": "test", "severity_cap": "warning"}, + "l1_severity": { + "estimand_clarity": "warning", + "hypothesis_falsifiability": "warning", + }, +} + +_MINIMAL_EXP_TYPE_YAML_2 = { + "name": "test_exp_2", + "classification_triggers": ["test trigger 2"], + "dimension_weights": { + "causal_structure": "H", + "variance_protocol": "M", + "statistical_corrections": "M", + "ecological_validity": "M", + "measurement_alignment": "M", + "resource_proportionality": "M", + "data_acquisition": "M", + "agent_implementability": "M", + }, + "applicable_lenses": {}, + "red_team_focus": {"specific": "test", "severity_cap": "warning"}, + "l1_severity": { + "estimand_clarity": "warning", + "hypothesis_falsifiability": "warning", + }, +} + + +class TestExperimentTypeCaching: + def test_bundled_result_is_cached_on_repeat_call(self) -> None: + """Two calls with project_dir=None return equal defensive copies.""" + r1 = load_all_experiment_types() + r2 = load_all_experiment_types() + assert r1 == r2 + assert r1 is not r2 + + def test_mtime_change_invalidates_cache(self, tmp_path: Path) -> None: + """Adding a file to the user dir changes dir mtime, causing cache miss.""" + user_dir = tmp_path / ".autoskillit" / "experiment-types" + user_dir.mkdir(parents=True) + (user_dir / "custom_exp.yaml").write_text(yaml.dump(_MINIMAL_EXP_TYPE_YAML)) + r1 = load_all_experiment_types(project_dir=tmp_path) + (user_dir / "custom_exp_2.yaml").write_text(yaml.dump(_MINIMAL_EXP_TYPE_YAML_2)) + old_mt = user_dir.stat().st_mtime + os.utime(user_dir, (old_mt + 2, old_mt + 2)) + r2 = load_all_experiment_types(project_dir=tmp_path) + assert r1 is not r2 + assert len(r2) > len(r1) + + +# --------------------------------------------------------------------------- +# T1-T4: is_silent_type tests +# --------------------------------------------------------------------------- + + +def test_is_silent_type_qualitative_interpretive() -> None: + """qualitative_interpretive is a silent type (>=6 of 8 S dimensions).""" + types = load_all_experiment_types() + by_name = {s.name: s for s in types} + assert is_silent_type(by_name["qualitative_interpretive"]) is True + + +@pytest.mark.parametrize( + "name", + sorted(EXPECTED_TYPES - {"qualitative_interpretive"}), +) +def test_is_silent_type_false_for_standard_types(name: str) -> None: + """Standard types are not silent types.""" + types = load_all_experiment_types() + by_name = {s.name: s for s in types} + assert is_silent_type(by_name[name]) is False + + +def test_is_silent_type_threshold_boundary() -> None: + """5 of 8 S dims is not silent; 6 of 8 is silent.""" + base = ExperimentTypeSpec( + name="test_type", + classification_triggers=["test"], + dimension_weights={ + "causal_structure": "S", + "variance_protocol": "S", + "statistical_corrections": "S", + "ecological_validity": "S", + "measurement_alignment": "S", + "resource_proportionality": "H", + "data_acquisition": "H", + "agent_implementability": "H", + }, + applicable_lenses={}, + red_team_focus={}, + l1_severity={}, + ) + assert is_silent_type(base) is False # 5 of 8 + + six_s = replace( + base, + dimension_weights={ + **base.dimension_weights, + "resource_proportionality": "S", + }, + ) + assert is_silent_type(six_s) is True # 6 of 8 + + +def test_qualitative_interpretive_silent_type_weights() -> None: + """qualitative_interpretive has >=6 S dims and correct non-S dims.""" + types = load_all_experiment_types() + by_name = {s.name: s for s in types} + qi = by_name["qualitative_interpretive"] + silent_count = sum(1 for v in qi.dimension_weights.values() if v == "S") + assert silent_count >= 6 + assert qi.dimension_weights["data_acquisition"] == "L" + assert qi.dimension_weights["agent_implementability"] == "L" diff --git a/tests/recipe/test_full_audit_recipe.py b/tests/recipe/test_full_audit_recipe.py index 363de5eadd..4815fdf3db 100644 --- a/tests/recipe/test_full_audit_recipe.py +++ b/tests/recipe/test_full_audit_recipe.py @@ -1,20 +1,27 @@ from pathlib import Path -RECIPE_PATH = Path(__file__).resolve().parents[2] / ".autoskillit" / "recipes" / "full-audit.yaml" +import pytest +import yaml + +pytestmark = [pytest.mark.layer("recipe"), pytest.mark.small] + +RECIPE_PATH = ( + Path(__file__).resolve().parents[2] / "src" / "autoskillit" / "recipes" / "full-audit.yaml" +) def test_full_audit_recipe_file_exists() -> None: - """The recipe YAML exists at the project-scoped location.""" + """The recipe YAML exists at the bundled location.""" assert RECIPE_PATH.exists(), f"Expected recipe at {RECIPE_PATH}" def test_full_audit_recipe_validates() -> None: """full-audit.yaml loads without errors and passes validate_recipe.""" from autoskillit.recipe.io import load_recipe - from autoskillit.recipe.validator import validate_recipe + from autoskillit.recipe.validator import validate_recipe_structure recipe = load_recipe(RECIPE_PATH) - errors = validate_recipe(recipe) + errors = validate_recipe_structure(recipe) assert errors == [], f"Validation errors: {errors}" @@ -41,6 +48,7 @@ def test_full_audit_recipe_step_names() -> None: recipe = load_recipe(RECIPE_PATH) expected = { "checkout", + "init_audit_run", "run_audits", "validate_audits", "create_issues", @@ -74,25 +82,38 @@ def test_full_audit_kitchen_rules() -> None: from autoskillit.recipe.io import load_recipe recipe = load_recipe(RECIPE_PATH) - assert len(recipe.kitchen_rules) == 3 + assert len(recipe.kitchen_rules) == 4 + + +def test_full_audit_done_step_has_message() -> None: + """done step must have a non-empty message field.""" + from autoskillit.recipe.io import load_recipe + + recipe = load_recipe(RECIPE_PATH) + assert recipe.steps["done"].message, "done step must have a non-empty message" + +def test_full_audit_done_step_message_embeds_issue_urls_in_sentinel() -> None: + """done step message must instruct the model to include issue_urls in the sentinel JSON.""" + + data = yaml.safe_load(RECIPE_PATH.read_text()) + message = data["steps"]["done"]["message"] + assert "issue_urls" in message, "done message must reference issue_urls" + msg_lower = message.lower() + assert "sentinel" in msg_lower or "json" in msg_lower or '{"success"' in message, ( + "done message must instruct model to embed issue_urls in the sentinel JSON block" + ) -def test_full_audit_discovered_as_project_recipe(tmp_path: Path) -> None: - """list_recipes discovers the recipe as RecipeSource.PROJECT.""" - import shutil +def test_full_audit_discovered_as_builtin_recipe(tmp_path: Path) -> None: + """list_recipes discovers the recipe as RecipeSource.BUILTIN.""" from autoskillit.core.types import RecipeSource from autoskillit.recipe.io import list_recipes - project_dir = tmp_path / "project" - recipe_dir = project_dir / ".autoskillit" / "recipes" - recipe_dir.mkdir(parents=True) - shutil.copy2(RECIPE_PATH, recipe_dir / "full-audit.yaml") - - result = list_recipes(project_dir) + result = list_recipes(tmp_path) # tmp_path has no project-scoped recipes match = [r for r in result.items if r.name == "full-audit"] assert len(match) == 1 - assert match[0].source == RecipeSource.PROJECT + assert match[0].source == RecipeSource.BUILTIN def test_full_audit_semantic_rules_no_errors() -> None: @@ -108,11 +129,131 @@ def test_full_audit_semantic_rules_no_errors() -> None: "audit-tests", "audit-cohesion", "audit-arch", + "audit-feature-gates", + "audit-docs", + "audit-review-decisions", "validate-audit", - "prepare-issue", + "validate-test-audit", + "validate-review-decisions", } ) ctx = make_validation_context(recipe, available_skills=known_skills) findings = run_semantic_rules(ctx) errors = [f for f in findings if f.severity == Severity.ERROR] assert not errors, f"Semantic rule errors: {[f.rule + ': ' + f.message for f in errors]}" + + +def test_full_audit_description_mentions_review_decisions() -> None: + """full-audit description must mention audit-review-decisions.""" + + data = yaml.safe_load(RECIPE_PATH.read_text()) + assert "audit-review-decisions" in data["description"] + + +def test_full_audit_summary_mentions_six() -> None: + """full-audit summary must reference 6 parallel chains.""" + + data = yaml.safe_load(RECIPE_PATH.read_text()) + assert "6" in data["summary"] + + +def test_full_audit_run_audits_note_mentions_review_decisions() -> None: + """run_audits note must include audit-review-decisions as the 6th skill.""" + + data = yaml.safe_load(RECIPE_PATH.read_text()) + note = data["steps"]["run_audits"]["note"] + assert "audit-review-decisions" in note + + +def test_full_audit_validate_audits_note_mentions_review_decisions() -> None: + """validate_audits note must include review_decisions validation.""" + + data = yaml.safe_load(RECIPE_PATH.read_text()) + note = data["steps"]["validate_audits"]["note"] + assert "review_decisions" in note + + +def test_full_audit_validate_audits_routes_tests_to_specialized_skill() -> None: + """validate_audits step must route test audits to validate-test-audit.""" + + data = yaml.safe_load(RECIPE_PATH.read_text()) + note = data["steps"]["validate_audits"]["note"] + assert "validate-test-audit" in note + + +def test_full_audit_validate_audits_routes_review_decisions_to_specialized_skill() -> None: + """validate_audits step must route review-decisions to validate-review-decisions.""" + + data = yaml.safe_load(RECIPE_PATH.read_text()) + note = data["steps"]["validate_audits"]["note"] + assert "validate-review-decisions" in note + + +def test_full_audit_has_max_parallel_ingredient() -> None: + from autoskillit.recipe.io import load_recipe + + recipe = load_recipe(RECIPE_PATH) + assert "max_parallel" in recipe.ingredients + + +def test_full_audit_max_parallel_defaults_to_three() -> None: + from autoskillit.recipe.io import load_recipe + + recipe = load_recipe(RECIPE_PATH) + ing = recipe.ingredients["max_parallel"] + assert ing.default == "3" + + +def test_full_audit_max_parallel_is_hidden() -> None: + from autoskillit.recipe.io import load_recipe + + recipe = load_recipe(RECIPE_PATH) + ing = recipe.ingredients["max_parallel"] + assert ing.hidden is True + + +def test_full_audit_kitchen_rule_mentions_prefer_completion() -> None: + from autoskillit.recipe.io import load_recipe + + recipe = load_recipe(RECIPE_PATH) + rules_text = " ".join(recipe.kitchen_rules).lower() + assert "prefer-completion" in rules_text or "prefer completion" in rules_text + + +def test_full_audit_run_audits_note_mentions_max_parallel() -> None: + + data = yaml.safe_load(RECIPE_PATH.read_text()) + note = data["steps"]["run_audits"]["note"].lower() + assert "max_parallel" in note or "max parallel" in note + + +def test_full_audit_validate_audits_no_wave_barrier() -> None: + + data = yaml.safe_load(RECIPE_PATH.read_text()) + note = data["steps"]["validate_audits"]["note"].lower() + assert "as each" in note or "as soon as" in note or "slot" in note + + +def test_full_audit_create_issues_uses_batched_graphql() -> None: + + data = yaml.safe_load(RECIPE_PATH.read_text()) + step = data["steps"]["create_issues"] + assert step["tool"] == "run_python" + assert "batch_create_issues" in step["with"]["callable"] + note = step["note"].lower() + assert "batched" in note or "graphql" in note + + +def test_full_audit_create_issues_captures_issue_urls() -> None: + import yaml + + data = yaml.safe_load(RECIPE_PATH.read_text()) + capture = data["steps"]["create_issues"]["capture"] + assert "issue_urls" in capture + + +def test_full_audit_recipe_version_bumped() -> None: + from packaging.version import Version + + data = yaml.safe_load(RECIPE_PATH.read_text()) + assert Version(data["recipe_version"]) > Version("1.0.0") diff --git a/tests/recipe/test_hidden_ingredients.py b/tests/recipe/test_hidden_ingredients.py index 686d446481..42efede3b6 100644 --- a/tests/recipe/test_hidden_ingredients.py +++ b/tests/recipe/test_hidden_ingredients.py @@ -2,9 +2,13 @@ from __future__ import annotations +import pytest + from autoskillit.recipe._api import format_ingredients_table from autoskillit.recipe.schema import Recipe, RecipeIngredient, RecipeStep +pytestmark = [pytest.mark.layer("recipe"), pytest.mark.small] + def _make_recipe(**ingredients: RecipeIngredient) -> Recipe: steps = { @@ -33,8 +37,8 @@ def test_hidden_ingredient_parsed() -> None: "description": "test", "kitchen_rules": ["no native tools"], "ingredients": { - "sprint_mode": { - "description": "Enable sprint mode", + "secret_flag": { + "description": "Enable secret flag", "default": "false", "hidden": True, } @@ -42,7 +46,7 @@ def test_hidden_ingredient_parsed() -> None: "steps": {"do_it": {"tool": "run_cmd", "with": {"cmd": "echo hi"}, "on_success": "done"}}, } recipe = _parse_recipe(data) - assert recipe.ingredients["sprint_mode"].hidden is True + assert recipe.ingredients["secret_flag"].hidden is True def test_hidden_ingredient_default_false() -> None: @@ -68,8 +72,8 @@ def test_hidden_ingredient_default_false() -> None: def test_hidden_ingredient_excluded_from_table() -> None: """format_ingredients_table omits hidden ingredients.""" recipe = _make_recipe( - sprint_mode=RecipeIngredient( - description="Enable sprint mode", + secret_flag=RecipeIngredient( + description="Enable secret flag", default="false", hidden=True, ), @@ -80,14 +84,14 @@ def test_hidden_ingredient_excluded_from_table() -> None: ) table = format_ingredients_table(recipe) assert table is not None - assert "sprint_mode" not in table + assert "secret_flag" not in table def test_non_hidden_ingredient_included_in_table() -> None: """Non-hidden ingredients still appear in the table.""" recipe = _make_recipe( - sprint_mode=RecipeIngredient( - description="Enable sprint mode", + secret_flag=RecipeIngredient( + description="Enable secret flag", default="false", hidden=True, ), @@ -104,7 +108,7 @@ def test_non_hidden_ingredient_included_in_table() -> None: def test_all_hidden_ingredients_returns_none() -> None: """format_ingredients_table returns None when all ingredients are hidden.""" recipe = _make_recipe( - sprint_mode=RecipeIngredient( + secret_flag=RecipeIngredient( description="Hidden flag", default="false", hidden=True, diff --git a/tests/recipe/test_identity.py b/tests/recipe/test_identity.py new file mode 100644 index 0000000000..4476b419bd --- /dev/null +++ b/tests/recipe/test_identity.py @@ -0,0 +1,299 @@ +"""Tests for recipe identity hashing — composite hash computation, query, and re-run detection.""" + +from __future__ import annotations + +import hashlib +import json +from pathlib import Path + +import pytest + +pytestmark = [pytest.mark.layer("recipe"), pytest.mark.medium] + + +def _write_recipe(tmp_path: Path, name: str = "test", *, skill_name: str = "") -> Path: + """Write a minimal valid recipe YAML and return its path.""" + msg = f"/autoskillit:{skill_name}" if skill_name else "hi" + content = ( + f"name: {name}\ndescription: d\nkitchen_rules:\n - rule\n" + f"steps:\n s1:\n tool: run_skill\n message: '{msg}'\n" + f" on_success: done\n done:\n action: stop\n message: Done\n" + ) + p = tmp_path / f"{name}.yaml" + p.write_text(content) + return p + + +def _make_skill(skills_dir: Path, name: str, content: str = "# skill") -> None: + """Create a skill directory with SKILL.md.""" + d = skills_dir / name + d.mkdir(parents=True, exist_ok=True) + (d / "SKILL.md").write_text(content) + + +def test_composite_hash_changes_with_recipe_content(tmp_path): + from autoskillit.recipe.identity import compute_composite_hash + from autoskillit.recipe.io import load_recipe + + skills_dir = tmp_path / "skills" + skills_dir.mkdir() + + p1 = _write_recipe(tmp_path, "test") + r1 = load_recipe(p1) + h1 = compute_composite_hash(p1, r1, skills_dir=skills_dir, project_dir=tmp_path) + + # Overwrite the same file with different step content (same name) + p1.write_text( + "name: test\ndescription: d\nkitchen_rules:\n - rule\n" + "steps:\n s1:\n tool: run_skill\n message: 'different'\n" + " on_success: done\n done:\n action: stop\n message: Done\n" + ) + r2 = load_recipe(p1) + h2 = compute_composite_hash(p1, r2, skills_dir=skills_dir, project_dir=tmp_path) + + assert h1 != h2 + + +def test_composite_hash_changes_with_skill_content(tmp_path): + from autoskillit.recipe.identity import compute_composite_hash + from autoskillit.recipe.io import load_recipe + + skills_dir = tmp_path / "skills" + _make_skill(skills_dir, "investigate", "# version 1") + + p = _write_recipe(tmp_path, skill_name="investigate") + recipe = load_recipe(p) + h1 = compute_composite_hash(p, recipe, skills_dir=skills_dir, project_dir=tmp_path) + + _make_skill(skills_dir, "investigate", "# version 2") + h2 = compute_composite_hash(p, recipe, skills_dir=skills_dir, project_dir=tmp_path) + + assert h1 != h2 + + +def test_composite_hash_changes_with_sub_recipe(tmp_path): + from autoskillit.recipe.identity import compute_composite_hash + from autoskillit.recipe.io import load_recipe + + skills_dir = tmp_path / "skills" + skills_dir.mkdir() + + sub_dir = tmp_path / ".autoskillit" / "recipes" / "sub-recipes" + sub_dir.mkdir(parents=True) + (sub_dir / "child.yaml").write_text( + "name: child\ndescription: d\nsteps:\n" + " s1:\n tool: run_cmd\n message: echo hi\n on_success: done\n" + " done:\n action: stop\n message: Done\n" + ) + + recipe_content = ( + "name: parent\ndescription: d\nkitchen_rules:\n - rule\n" + "steps:\n s1:\n sub_recipe: child\n on_success: done\n" + " done:\n action: stop\n message: Done\n" + ) + p = tmp_path / "parent.yaml" + p.write_text(recipe_content) + recipe = load_recipe(p) + h1 = compute_composite_hash(p, recipe, skills_dir=skills_dir, project_dir=tmp_path) + + (sub_dir / "child.yaml").write_text( + "name: child\ndescription: d modified\nsteps:\n" + " s1:\n tool: run_cmd\n message: echo bye\n on_success: done\n" + " done:\n action: stop\n message: Done\n" + ) + h2 = compute_composite_hash(p, recipe, skills_dir=skills_dir, project_dir=tmp_path) + + assert h1 != h2 + + +def test_composite_hash_is_deterministic(tmp_path): + from autoskillit.recipe.identity import compute_composite_hash + from autoskillit.recipe.io import load_recipe + + skills_dir = tmp_path / "skills" + _make_skill(skills_dir, "investigate") + + p = _write_recipe(tmp_path, skill_name="investigate") + recipe = load_recipe(p) + h1 = compute_composite_hash(p, recipe, skills_dir=skills_dir, project_dir=tmp_path) + h2 = compute_composite_hash(p, recipe, skills_dir=skills_dir, project_dir=tmp_path) + + assert h1 == h2 + + +def test_composite_hash_no_dependencies_differs_from_content_hash(tmp_path): + from autoskillit.recipe.identity import compute_composite_hash + from autoskillit.recipe.io import load_recipe + + skills_dir = tmp_path / "skills" + skills_dir.mkdir() + + p = _write_recipe(tmp_path) + recipe = load_recipe(p) + composite = compute_composite_hash(p, recipe, skills_dir=skills_dir, project_dir=tmp_path) + content = "sha256:" + hashlib.sha256(p.read_bytes()).hexdigest() + + assert composite != content + + +# --------------------------------------------------------------------------- +# Part B: find_prior_runs tests +# --------------------------------------------------------------------------- + + +def test_find_prior_runs_empty(tmp_path): + from autoskillit.recipe.identity import find_prior_runs + + result = find_prior_runs(tmp_path / "sessions.jsonl", composite_hash="sha256:abc") + assert result == [] + + +def test_find_prior_runs_by_composite_hash(tmp_path): + from autoskillit.recipe.identity import find_prior_runs + + jsonl = tmp_path / "sessions.jsonl" + entry = { + "recipe_name": "impl", + "recipe_composite_hash": "sha256:abc", + "timestamp": "2026-04-01T00:00:00", + "session_id": "s1", + "success": True, + } + jsonl.write_text(json.dumps(entry) + "\n") + result = find_prior_runs(jsonl, composite_hash="sha256:abc") + assert len(result) == 1 + assert result[0]["session_id"] == "s1" + + +def test_find_prior_runs_filters_mismatch(tmp_path): + from autoskillit.recipe.identity import find_prior_runs + + jsonl = tmp_path / "sessions.jsonl" + e1 = { + "recipe_name": "impl", + "recipe_composite_hash": "sha256:abc", + "timestamp": "T1", + "session_id": "s1", + } + e2 = { + "recipe_name": "impl", + "recipe_composite_hash": "sha256:xyz", + "timestamp": "T2", + "session_id": "s2", + } + jsonl.write_text(json.dumps(e1) + "\n" + json.dumps(e2) + "\n") + result = find_prior_runs(jsonl, composite_hash="sha256:abc") + assert len(result) == 1 + + +def test_find_prior_runs_by_name(tmp_path): + from autoskillit.recipe.identity import find_prior_runs + + jsonl = tmp_path / "sessions.jsonl" + e1 = { + "recipe_name": "impl", + "recipe_composite_hash": "sha256:abc", + "timestamp": "T1", + } + e2 = { + "recipe_name": "research", + "recipe_composite_hash": "sha256:def", + "timestamp": "T2", + } + jsonl.write_text(json.dumps(e1) + "\n" + json.dumps(e2) + "\n") + result = find_prior_runs(jsonl, recipe_name="impl") + assert len(result) == 1 + + +def test_find_prior_runs_sorted_descending(tmp_path): + from autoskillit.recipe.identity import find_prior_runs + + jsonl = tmp_path / "sessions.jsonl" + e1 = { + "recipe_name": "impl", + "recipe_composite_hash": "sha256:abc", + "timestamp": "2026-04-01", + } + e2 = { + "recipe_name": "impl", + "recipe_composite_hash": "sha256:abc", + "timestamp": "2026-04-10", + } + jsonl.write_text(json.dumps(e1) + "\n" + json.dumps(e2) + "\n") + result = find_prior_runs(jsonl, composite_hash="sha256:abc") + assert result[0]["timestamp"] == "2026-04-10" + + +def test_find_prior_runs_skips_malformed(tmp_path): + from autoskillit.recipe.identity import find_prior_runs + + jsonl = tmp_path / "sessions.jsonl" + jsonl.write_text( + "not json\n" + + json.dumps( + { + "recipe_name": "impl", + "recipe_composite_hash": "sha256:abc", + "timestamp": "T1", + } + ) + + "\n" + ) + result = find_prior_runs(jsonl, composite_hash="sha256:abc") + assert len(result) == 1 + + +# --------------------------------------------------------------------------- +# Part B: Re-run detection tests +# --------------------------------------------------------------------------- + + +def test_check_rerun_no_prior(tmp_path): + from autoskillit.recipe.identity import check_rerun_detection + + result = check_rerun_detection(tmp_path / "sessions.jsonl", composite_hash="sha256:new") + assert result is None + + +def test_check_rerun_found(tmp_path): + from autoskillit.recipe.identity import check_rerun_detection + + jsonl = tmp_path / "sessions.jsonl" + entry = { + "recipe_composite_hash": "sha256:abc", + "timestamp": "2026-04-10", + "session_id": "s1", + } + jsonl.write_text(json.dumps(entry) + "\n") + result = check_rerun_detection(jsonl, composite_hash="sha256:abc") + assert result is not None + assert result["rule"] == "duplicate-run-detected" + assert "2026-04-10" in result["message"] + + +def test_composite_hash_uses_content_bytes_when_provided(tmp_path): + """compute_composite_hash uses content_bytes instead of reading from disk.""" + from autoskillit.recipe.identity import compute_composite_hash + from autoskillit.recipe.io import load_recipe + + skills_dir = tmp_path / "skills" + skills_dir.mkdir() + + p = _write_recipe(tmp_path, "test") + recipe = load_recipe(p) + bytes_data = p.read_bytes() + + # Delete the file so disk read would fail + p.unlink() + + h = compute_composite_hash( + p, + recipe, + skills_dir=skills_dir, + project_dir=tmp_path, + content_bytes=bytes_data, + ) + + # Should succeed using provided bytes; hash should be valid sha256: format + assert h.startswith("sha256:") + assert len(h) == 71 # sha256: prefix + 64 hex chars diff --git a/tests/recipe/test_implement_findings_recipe.py b/tests/recipe/test_implement_findings_recipe.py new file mode 100644 index 0000000000..ed9cdc9b70 --- /dev/null +++ b/tests/recipe/test_implement_findings_recipe.py @@ -0,0 +1,117 @@ +"""Tests for implement-findings.yaml recipe (T1–T14).""" + +from __future__ import annotations + +import pytest + +from autoskillit.core.types import Severity +from autoskillit.recipe.io import builtin_recipes_dir, load_recipe +from autoskillit.recipe.schema import RecipeKind +from autoskillit.recipe.validator import run_semantic_rules +from tests.recipe.conftest import NO_AUTOSKILLIT_IMPORT + +pytestmark = [ + pytest.mark.layer("recipe"), + pytest.mark.small, + pytest.mark.xdist_group(name="implement_findings_recipe"), +] + +RECIPE_PATH = builtin_recipes_dir() / "implement-findings.yaml" + + +@pytest.fixture(scope="module") +def recipe(): + return load_recipe(RECIPE_PATH) + + +# T1 +def test_implement_findings_recipe_file_exists() -> None: + assert RECIPE_PATH.exists() + + +# T2 +def test_implement_findings_recipe_parses(recipe) -> None: + assert recipe.name == "implement-findings" + assert recipe.kind == RecipeKind.STANDARD + + +# T3 +def test_implement_findings_recipe_validates_cleanly(recipe) -> None: + findings = run_semantic_rules(recipe) + errors = [ + f for f in findings if f.severity == Severity.ERROR and f.rule != NO_AUTOSKILLIT_IMPORT + ] + assert errors == [] + + +# T4 +def test_implement_findings_issue_urls_required(recipe) -> None: + assert recipe.ingredients["issue_urls"].required is True + assert recipe.ingredients["issue_urls"].default is None + + +# T5 +def test_implement_findings_base_branch_default(recipe) -> None: + assert recipe.ingredients["base_branch"].default == "develop" + + +# T6 +def test_implement_findings_execution_map_hidden(recipe) -> None: + ing = recipe.ingredients["execution_map"] + assert ing.hidden is True + assert ing.default == "" + + +# T7 +def test_implement_findings_max_parallel_hidden(recipe) -> None: + ing = recipe.ingredients["max_parallel"] + assert ing.hidden is True + assert ing.default == "6" + + +# T8 +def test_implement_findings_check_resume_callable(recipe) -> None: + step = recipe.steps["check_resume"] + assert step.with_args["callable"] == "autoskillit.fleet._findings_rpc.parse_and_resume" + + +# T9 +def test_implement_findings_load_bem_callable(recipe) -> None: + step = recipe.steps["load_bem_from_file"] + assert step.with_args["callable"] == "autoskillit.fleet._findings_rpc.load_execution_map" + + +# T10 +@pytest.mark.parametrize("step_name", ["run_bem_internally"]) +def test_implement_findings_run_skill_fault_guards(recipe, step_name: str) -> None: + step = recipe.steps[step_name] + assert step.on_failure is not None + assert step.on_context_limit is not None + + +# T11 +def test_implement_findings_requires_packs(recipe) -> None: + packs = set(recipe.requires_packs) + assert {"github", "clone"}.issubset(packs) + + +# T12 +def test_implement_findings_route_bem_mode_is_route(recipe) -> None: + step = recipe.steps["route_bem_mode"] + assert step.action == "route" + assert step.on_result is not None + + +# T13 +@pytest.mark.parametrize("step_name", ["done", "escalate_stop"]) +def test_implement_findings_terminal_steps(recipe, step_name: str) -> None: + step = recipe.steps[step_name] + assert step.action == "stop" + assert step.message + + +# T14 +def test_implement_findings_no_undeclared_capture_keys(recipe) -> None: + findings = run_semantic_rules(recipe) + undeclared = [f for f in findings if f.rule == "undeclared-capture-key"] + assert undeclared == [] diff --git a/tests/recipe/test_implementation.py b/tests/recipe/test_implementation.py new file mode 100644 index 0000000000..c75d310105 --- /dev/null +++ b/tests/recipe/test_implementation.py @@ -0,0 +1,197 @@ +"""Tests for review-resolve retry loop steps in implementation.yaml (T_IP_LOOP1–T_IP_LOOP10).""" + +from __future__ import annotations + +import pytest + +from autoskillit.recipe.io import builtin_recipes_dir, load_recipe +from autoskillit.recipe.validator import validate_recipe_structure + +pytestmark = [pytest.mark.layer("recipe"), pytest.mark.small] + +RECIPE_PATH = builtin_recipes_dir() / "implementation.yaml" + + +@pytest.fixture(scope="module") +def recipe(): + return load_recipe(RECIPE_PATH) + + +# T_IP_LOOP1 +def test_check_review_loop_step_exists(recipe) -> None: + """check_review_loop step must exist in implementation recipe.""" + assert "check_review_loop" in recipe.steps + + +# T_IP_LOOP2 +def test_check_review_loop_uses_run_python_with_callable(recipe) -> None: + """check_review_loop must use run_python tool with the smoke_utils callable.""" + step = recipe.steps["check_review_loop"] + assert step.tool == "run_python" + assert step.with_args.get("callable") == "autoskillit.smoke_utils.check_review_loop" + + +def test_ci_watch_timed_out_routes_to_guard(recipe) -> None: + """ci_watch timed_out route goes to check_ci_timed_out_loop, not self.""" + step = recipe.steps["ci_watch"] + assert step.on_result is not None + timed_out_conds = [c for c in step.on_result.conditions if c.when and "timed_out" in c.when] + assert timed_out_conds, "ci_watch must have a timed_out condition" + assert timed_out_conds[0].route == "check_ci_timed_out_loop", ( + "ci_watch timed_out must route to check_ci_timed_out_loop, not self" + ) + + +def test_check_ci_timed_out_loop_exists_with_correct_pattern(recipe) -> None: + """check_ci_timed_out_loop uses check_loop_iteration with max_iterations: 2.""" + assert "check_ci_timed_out_loop" in recipe.steps + step = recipe.steps["check_ci_timed_out_loop"] + assert step.tool == "run_python" + assert "check_loop_iteration" in step.with_args.get("callable", "") + assert step.with_args.get("max_iterations") == "2" + assert step.on_result is not None + max_exceeded_conds = [ + c + for c in step.on_result.conditions + if c.when and "max_exceeded" in c.when and "true" in c.when + ] + assert max_exceeded_conds, "check_ci_timed_out_loop must route max_exceeded==true" + assert max_exceeded_conds[0].route == "detect_ci_conflict" + assert step.with_args.get("callable") == "autoskillit.smoke_utils.check_loop_iteration" + + +# T_IP_LOOP3 +def test_check_review_loop_has_skip_when_false_open_pr(recipe) -> None: + """check_review_loop must be skipped when inputs.open_pr is false.""" + step = recipe.steps["check_review_loop"] + assert step.skip_when_false == "inputs.open_pr" + + +# T_IP_LOOP4 +def test_check_review_loop_on_result_routes_to_review_pr_when_had_blocking_and_not_max_exceeded( + recipe, +) -> None: + """check_review_loop on_result routes to review_pr only when + had_blocking=true AND max_exceeded=false.""" + step = recipe.steps["check_review_loop"] + assert step.on_result is not None + review_conditions = [ + c for c in step.on_result.conditions if c.when is not None and c.route == "review_pr" + ] + assert review_conditions, "No conditional route to review_pr found" + cond = review_conditions[0].when + assert "had_blocking" in cond + assert "max_exceeded" in cond + + +# T_IP_LOOP5 +def test_check_review_loop_on_result_default_routes_to_ci_watch(recipe) -> None: + """check_review_loop on_result falls through to check_repo_ci_event when no blocking.""" + step = recipe.steps["check_review_loop"] + assert step.on_result is not None + default_conditions = [c for c in step.on_result.conditions if c.when is None] + assert default_conditions, "No default (when=None) condition found" + assert default_conditions[0].route == "check_repo_ci_event" + + +# T_IP_LOOP6 +def test_check_review_loop_has_on_failure(recipe) -> None: + """check_review_loop must declare on_failure because it uses on_result + (on-result-missing-failure-route semantic rule requires it).""" + step = recipe.steps["check_review_loop"] + assert step.on_failure == "check_repo_ci_event" + + +# T_IP_LOOP7 +def test_re_push_review_routes_to_check_review_loop(recipe) -> None: + """re_push_review on_success must route to check_review_loop, not ci_watch directly.""" + step = recipe.steps["re_push_review"] + assert step.on_success == "check_review_loop" + + +# T_IP_LOOP8 +def test_review_max_retries_ingredient_exists_with_default_3(recipe) -> None: + """review_max_retries ingredient must exist with default='3'.""" + assert "review_max_retries" in recipe.ingredients + ingredient = recipe.ingredients["review_max_retries"] + assert ingredient.default == "3" + + +# T_IP_LOOP9 +def test_check_review_loop_has_optional_context_refs_with_review_loop_count(recipe) -> None: + """check_review_loop must declare review_loop_count in optional_context_refs.""" + step = recipe.steps["check_review_loop"] + assert "review_loop_count" in (step.optional_context_refs or []) + + +# T_IP_LOOP10 +def test_check_review_loop_captures_review_loop_count(recipe) -> None: + """check_review_loop must capture review_loop_count from result.next_iteration.""" + step = recipe.steps["check_review_loop"] + capture = step.capture or {} + assert "review_loop_count" in capture + assert "next_iteration" in capture["review_loop_count"] + + +# T_IP_LOOP11 +def test_review_pr_routes_approved_with_comments_to_resolve_review(recipe) -> None: + """approved_with_comments routes through enrich_diff_context to resolve_review.""" + step = recipe.steps["review_pr"] + assert step.on_result is not None + routes = {c.when: c.route for c in step.on_result.conditions if c.when} + matching = [ + when + for when, route in routes.items() + if "approved_with_comments" in when and route == "enrich_diff_context" + ] + assert matching, "No approved_with_comments → enrich_diff_context route found" + enrich_step = recipe.steps["enrich_diff_context"] + assert enrich_step.on_success == "resolve_review" + + +# T_IP_LOOP12 +def test_review_pr_captures_review_verdict(recipe) -> None: + """review_pr must capture verdict as review_verdict (not verdict) to avoid clobber.""" + step = recipe.steps["review_pr"] + capture = step.capture or {} + assert "review_verdict" in capture + assert "result.verdict" in capture["review_verdict"] + + +# T_IP_LOOP13 +def test_check_review_loop_with_args_has_previous_verdict(recipe) -> None: + """check_review_loop with: must pass previous_verdict from context.review_verdict.""" + step = recipe.steps["check_review_loop"] + assert "previous_verdict" in step.with_args + assert "review_verdict" in step.with_args["previous_verdict"] + + +def test_capture_base_sha_captures_both_base_sha_and_merge_target(recipe) -> None: + """bootstrap_clone must capture both base_sha and merge_target.""" + step = recipe.steps["clone"] + capture = step.capture or {} + assert "base_sha" in capture, ( + "clone (bootstrap_clone) must capture base_sha — the SHA of base_branch before any merge" + ) + assert "merge_target" in capture, ( + "clone (bootstrap_clone) must capture merge_target — the fallback target branch name" + ) + + +def test_implementation_has_no_sprint_mode_ingredient() -> None: + """implementation.yaml must not declare sprint_mode after sprint-prefix removal.""" + recipe = load_recipe(RECIPE_PATH) + assert "sprint_mode" not in recipe.ingredients + + +def test_implementation_has_no_sprint_entry_step() -> None: + """implementation.yaml must not have a sprint_entry step after sprint-prefix removal.""" + recipe = load_recipe(RECIPE_PATH) + assert "sprint_entry" not in recipe.steps + + +def test_implementation_validates_clean_after_sprint_removal() -> None: + """implementation.yaml must pass schema validation after sprint references removed.""" + recipe = load_recipe(RECIPE_PATH) + errors = validate_recipe_structure(recipe) + assert not errors, f"implementation.yaml failed validation after sprint removal: {errors}" diff --git a/tests/recipe/test_implementation_groups_pr_decomposition.py b/tests/recipe/test_implementation_groups_pr_decomposition.py new file mode 100644 index 0000000000..9ce682ba88 --- /dev/null +++ b/tests/recipe/test_implementation_groups_pr_decomposition.py @@ -0,0 +1,58 @@ +"""Verify implementation-groups.yaml has arch_lenses ingredient and routing.""" + +import pytest + +from autoskillit.recipe.io import builtin_recipes_dir, load_recipe +from autoskillit.recipe.validator import validate_recipe_structure + +pytestmark = [pytest.mark.layer("recipe"), pytest.mark.small] + +RECIPE_PATH = builtin_recipes_dir() / "implementation-groups.yaml" + + +@pytest.fixture(scope="module") +def recipe(): + return load_recipe(RECIPE_PATH) + + +def test_arch_lenses_ingredient_declared(recipe): + assert "arch_lenses" in recipe.ingredients + + +def test_arch_lenses_is_not_hidden(recipe): + assert recipe.ingredients["arch_lenses"].hidden is False + + +def test_arch_lenses_defaults_to_false(recipe): + assert recipe.ingredients["arch_lenses"].default == "false" + + +def test_prepare_pr_routes_to_compose_pr_when_arch_lenses_false(recipe): + step = recipe.steps["prepare_pr"] + compose_cond = next( + (c for c in step.on_result.conditions if c.route == "compose_pr" and c.when), + None, + ) + assert compose_cond is not None + assert "prep_path" in compose_cond.when + assert "arch_lenses" not in compose_cond.when + + +def test_prepare_pr_arch_lenses_route_checks_ingredient(recipe): + step = recipe.steps["prepare_pr"] + arch_lens_condition = next( + (c for c in step.on_result.conditions if c.route == "run_arch_lenses" and c.when), + None, + ) + assert arch_lens_condition is not None + assert "arch_lenses" in arch_lens_condition.when + + +def test_run_arch_lenses_still_gated_on_open_pr(recipe): + step = recipe.steps["run_arch_lenses"] + assert step.skip_when_false == "inputs.open_pr" + + +def test_recipe_validates_clean(recipe): + errors = validate_recipe_structure(recipe) + assert not errors, f"Validation errors: {errors}" diff --git a/tests/recipe/test_implementation_groups_review_loop.py b/tests/recipe/test_implementation_groups_review_loop.py new file mode 100644 index 0000000000..4213b79816 --- /dev/null +++ b/tests/recipe/test_implementation_groups_review_loop.py @@ -0,0 +1,114 @@ +"""Tests for review-resolve retry loop steps in implementation-groups.yaml +(T_IG_LOOP1–T_IG_LOOP3).""" + +from __future__ import annotations + +import pytest + +from autoskillit.recipe.io import builtin_recipes_dir, load_recipe + +pytestmark = [pytest.mark.layer("recipe"), pytest.mark.small] + +RECIPE_PATH = builtin_recipes_dir() / "implementation-groups.yaml" + + +@pytest.fixture(scope="module") +def recipe(): + return load_recipe(RECIPE_PATH) + + +# T_IG_LOOP1 +def test_check_review_loop_step_exists(recipe) -> None: + """check_review_loop step must exist in implementation-groups recipe.""" + assert "check_review_loop" in recipe.steps + + +# T_IG_LOOP2 +def test_re_push_review_routes_to_check_review_loop(recipe) -> None: + """re_push_review on_success must route to check_review_loop in implementation-groups + recipe.""" + step = recipe.steps["re_push_review"] + assert step.on_success == "check_review_loop" + + +# T_IG_LOOP3 +def test_check_review_loop_on_result_routes_to_review_pr_when_had_blocking_and_not_max_exceeded( + recipe, +) -> None: + """check_review_loop on_result condition must gate on had_blocking AND max_exceeded.""" + step = recipe.steps["check_review_loop"] + assert step.on_result is not None + review_conditions = [ + c for c in step.on_result.conditions if c.when is not None and c.route == "review_pr" + ] + assert review_conditions, "No conditional route to review_pr found" + cond = review_conditions[0].when + assert "had_blocking" in cond + assert "max_exceeded" in cond + + +# T_IG_LOOP4 +def test_check_review_loop_has_on_failure(recipe) -> None: + """check_review_loop must declare on_failure because it uses on_result + (on-result-missing-failure-route semantic rule requires it).""" + step = recipe.steps["check_review_loop"] + assert step.on_failure == "check_repo_ci_event" + + +# T_IG_LOOP5 +def test_review_pr_routes_approved_with_comments_to_resolve_review(recipe) -> None: + """review_pr on_result must route approved_with_comments to resolve_review.""" + step = recipe.steps["review_pr"] + assert step.on_result is not None + routes = {c.when: c.route for c in step.on_result.conditions if c.when} + matching = [ + when + for when, route in routes.items() + if "approved_with_comments" in when and route == "resolve_review" + ] + assert matching, "No approved_with_comments → resolve_review route found" + + +# T_IG_LOOP6 +def test_review_pr_captures_review_verdict(recipe) -> None: + """review_pr must capture verdict as review_verdict (not verdict) to avoid clobber.""" + step = recipe.steps["review_pr"] + capture = step.capture or {} + assert "review_verdict" in capture + assert "result.verdict" in capture["review_verdict"] + + +# T_IG_LOOP7 +def test_check_review_loop_with_args_has_previous_verdict(recipe) -> None: + """check_review_loop with: must pass previous_verdict from context.review_verdict.""" + step = recipe.steps["check_review_loop"] + assert "previous_verdict" in step.with_args + assert "review_verdict" in step.with_args["previous_verdict"] + + +def test_ci_watch_timed_out_routes_to_guard(recipe) -> None: + """ci_watch timed_out route goes to check_ci_timed_out_loop, not self.""" + step = recipe.steps["ci_watch"] + assert step.on_result is not None + timed_out_conds = [c for c in step.on_result.conditions if c.when and "timed_out" in c.when] + assert timed_out_conds, "ci_watch must have a timed_out condition" + assert timed_out_conds[0].route == "check_ci_timed_out_loop", ( + "ci_watch timed_out must route to check_ci_timed_out_loop, not self" + ) + + +def test_check_ci_timed_out_loop_exists_with_correct_pattern(recipe) -> None: + """check_ci_timed_out_loop uses check_loop_iteration with max_iterations: 2.""" + assert "check_ci_timed_out_loop" in recipe.steps + step = recipe.steps["check_ci_timed_out_loop"] + assert step.tool == "run_python" + assert "check_loop_iteration" in step.with_args.get("callable", "") + assert step.with_args.get("max_iterations") == "2" + assert step.on_result is not None + max_exceeded_conds = [ + c + for c in step.on_result.conditions + if c.when and "max_exceeded" in c.when and "true" in c.when + ] + assert max_exceeded_conds, "check_ci_timed_out_loop must route max_exceeded==true" + assert max_exceeded_conds[0].route == "detect_ci_conflict" diff --git a/tests/recipe/test_implementation_pr_decomposition.py b/tests/recipe/test_implementation_pr_decomposition.py index d6e9720c23..3ae80df079 100644 --- a/tests/recipe/test_implementation_pr_decomposition.py +++ b/tests/recipe/test_implementation_pr_decomposition.py @@ -3,7 +3,9 @@ import pytest from autoskillit.recipe.io import builtin_recipes_dir, load_recipe -from autoskillit.recipe.validator import validate_recipe +from autoskillit.recipe.validator import validate_recipe_structure + +pytestmark = [pytest.mark.layer("recipe"), pytest.mark.small] RECIPE_PATH = builtin_recipes_dir() / "implementation.yaml" @@ -24,7 +26,9 @@ def test_implementation_recipe_has_three_pr_steps(recipe): def test_prepare_pr_routes_to_run_arch_lenses(recipe): step = recipe.steps["prepare_pr"] - assert step.on_success == "run_arch_lenses" + assert step.on_result is not None + routes = [c.route for c in step.on_result.conditions if c.when and "prep_path" in c.when] + assert "run_arch_lenses" in routes def test_run_arch_lenses_routes_to_compose_on_success(recipe): @@ -62,6 +66,40 @@ def test_compose_pr_captures_pr_url(recipe): assert "pr_url" in (step.capture or {}) +def test_arch_lenses_ingredient_declared(recipe): + assert "arch_lenses" in recipe.ingredients + + +def test_arch_lenses_defaults_to_false(recipe): + assert recipe.ingredients["arch_lenses"].default == "false" + + +def test_arch_lenses_is_not_hidden(recipe): + assert recipe.ingredients["arch_lenses"].hidden is False + + +def test_prepare_pr_routes_to_compose_pr_when_arch_lenses_false(recipe): + step = recipe.steps["prepare_pr"] + assert step.on_result is not None + routes = [c.route for c in step.on_result.conditions if c.when and "prep_path" in c.when] + assert "compose_pr" in routes + + +def test_prepare_pr_arch_lenses_route_checks_ingredient(recipe): + step = recipe.steps["prepare_pr"] + arch_lens_condition = next( + (c for c in step.on_result.conditions if c.route == "run_arch_lenses" and c.when), + None, + ) + assert arch_lens_condition is not None + assert "arch_lenses" in arch_lens_condition.when + + +def test_run_arch_lenses_still_gated_on_open_pr(recipe): + step = recipe.steps["run_arch_lenses"] + assert step.skip_when_false == "inputs.open_pr" + + def test_implementation_recipe_validates_after_decomposition(recipe): - errors = validate_recipe(recipe) + errors = validate_recipe_structure(recipe) assert not errors, f"Validation errors: {errors}" diff --git a/tests/recipe/test_implementation_sprint_mode.py b/tests/recipe/test_implementation_sprint_mode.py deleted file mode 100644 index ef493895f5..0000000000 --- a/tests/recipe/test_implementation_sprint_mode.py +++ /dev/null @@ -1,138 +0,0 @@ -"""Tests for sprint_mode ingredient and sprint_entry step in implementation.yaml.""" - -from __future__ import annotations - -import pytest - -from autoskillit.core.paths import pkg_root -from autoskillit.recipe._analysis import make_validation_context -from autoskillit.recipe._api import _build_active_recipe, format_ingredients_table -from autoskillit.recipe.io import builtin_recipes_dir, load_recipe -from autoskillit.recipe.registry import run_semantic_rules -from autoskillit.recipe.validator import validate_recipe -from tests.recipe.conftest import NO_AUTOSKILLIT_IMPORT as _NO_AUTOSKILLIT_IMPORT - - -@pytest.fixture(scope="module") -def impl_recipe(): - return load_recipe(builtin_recipes_dir() / "implementation.yaml") - - -def test_implementation_has_sprint_mode_ingredient(impl_recipe) -> None: - """implementation.yaml declares 'sprint_mode' ingredient with default 'false'.""" - assert "sprint_mode" in impl_recipe.ingredients - assert impl_recipe.ingredients["sprint_mode"].default == "false" - - -def test_implementation_sprint_mode_is_hidden(impl_recipe) -> None: - """implementation.yaml sprint_mode ingredient has hidden=True.""" - assert impl_recipe.ingredients["sprint_mode"].hidden is True - - -def test_implementation_has_sprint_entry_step(impl_recipe) -> None: - """implementation.yaml has 'sprint_entry' step with sub_recipe='sprint-prefix'.""" - assert "sprint_entry" in impl_recipe.steps - step = impl_recipe.steps["sprint_entry"] - assert step.sub_recipe == "sprint-prefix" - - -def test_implementation_sprint_entry_gates_on_sprint_mode(impl_recipe) -> None: - """implementation.yaml sprint_entry step has gate='sprint_mode'.""" - step = impl_recipe.steps["sprint_entry"] - assert step.gate == "sprint_mode" - - -def test_implementation_sprint_entry_on_success_is_done(impl_recipe) -> None: - """implementation.yaml sprint_entry.on_success == 'done'.""" - step = impl_recipe.steps["sprint_entry"] - assert step.on_success == "done" - - -def test_implementation_sprint_mode_false_loads_clean() -> None: - """load_and_validate('implementation') with sprint_mode=false drops sprint_entry step.""" - active, combined = _build_active_recipe( - load_recipe(builtin_recipes_dir() / "implementation.yaml"), - None, - pkg_root().parent, - ) - assert "sprint_entry" not in active.steps - assert combined is None - - -def test_implementation_sprint_mode_false_first_step_is_clone() -> None: - """When sprint_mode=false, served recipe's first step is 'clone'.""" - active, _ = _build_active_recipe( - load_recipe(builtin_recipes_dir() / "implementation.yaml"), - None, - pkg_root().parent, - ) - first_step_name = next(iter(active.steps)) - assert first_step_name == "clone" - - -def test_implementation_sprint_mode_false_no_sprint_steps_in_content() -> None: - """When sprint_mode=false, served recipe content contains no sprint_ steps.""" - active, _ = _build_active_recipe( - load_recipe(builtin_recipes_dir() / "implementation.yaml"), - None, - pkg_root().parent, - ) - sprint_steps = [n for n in active.steps if n.startswith("sprint_")] - assert not sprint_steps, f"Sprint steps should be absent: {sprint_steps}" - - -def test_implementation_sprint_mode_true_has_sprint_steps() -> None: - """load_and_validate('implementation', overrides={sprint_mode: true}) has sprint_ steps.""" - active, combined = _build_active_recipe( - load_recipe(builtin_recipes_dir() / "implementation.yaml"), - {"sprint_mode": "true"}, - pkg_root().parent, - ) - assert combined is not None - sprint_steps = [n for n in active.steps if "sprint" in n] - assert sprint_steps, "Sprint steps should appear when sprint_mode=true" - - -def test_implementation_sprint_mode_true_sprint_triage_is_first() -> None: - """When sprint_mode=true, sprint_triage is the first step in served recipe.""" - active, _ = _build_active_recipe( - load_recipe(builtin_recipes_dir() / "implementation.yaml"), - {"sprint_mode": "true"}, - pkg_root().parent, - ) - first_step_name = next(iter(active.steps)) - # The first merged step should contain 'triage' (prefixed from sprint-prefix) - assert "triage" in first_step_name, ( - f"First step should be a sprint triage step, got: {first_step_name}" - ) - - -def test_implementation_sprint_mode_false_hidden_ingredient_not_in_table() -> None: - """format_ingredients_table for implementation (sprint_mode=false) excludes sprint_mode.""" - active, _ = _build_active_recipe( - load_recipe(builtin_recipes_dir() / "implementation.yaml"), - None, - pkg_root().parent, - ) - table = format_ingredients_table(active) - assert table is None or "sprint_mode" not in (table or "") - - -def test_implementation_structural_validation_clean(impl_recipe) -> None: - """validate_recipe(implementation) returns no errors (standalone path).""" - errors = validate_recipe(impl_recipe) - assert not errors, f"Structural validation errors: {errors}" - - -def test_implementation_no_semantic_errors(impl_recipe) -> None: - """Semantic rules on implementation.yaml (sprint_mode=false) return no ERRORs.""" - from autoskillit.core.types import Severity - - active, _ = _build_active_recipe(impl_recipe, None, pkg_root().parent) - ctx = make_validation_context(active, available_sub_recipes=frozenset({"sprint-prefix"})) - errors = [ - f - for f in run_semantic_rules(ctx) - if f.severity == Severity.ERROR and f.rule != _NO_AUTOSKILLIT_IMPORT - ] - assert not errors, f"Semantic errors: {errors}" diff --git a/tests/recipe/test_io_discovery.py b/tests/recipe/test_io_discovery.py new file mode 100644 index 0000000000..0250c901fd --- /dev/null +++ b/tests/recipe/test_io_discovery.py @@ -0,0 +1,384 @@ +"""Tests for recipe I/O — list_recipes discovery, builtin_recipes_dir, and pack fields.""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from autoskillit.core.types import RecipeSource +from autoskillit.recipe.io import ( + builtin_recipes_dir, + list_recipes, + load_recipe, +) +from autoskillit.recipe.schema import RecipeKind + +pytestmark = [pytest.mark.layer("recipe"), pytest.mark.medium] + + +class TestListRecipes: + """TestListRecipes: discovery from project and builtin sources.""" + + def test_finds_builtins(self, tmp_path: Path) -> None: + result = list_recipes(tmp_path) + recipes = result.items + names = {w.name for w in recipes} + assert "implementation" in names + assert len(recipes) > 0 + assert all(r.source.value in ("project", "builtin") for r in recipes) + + def test_list_recipes_bundled_appear_before_project(self, tmp_path: Path) -> None: + """Non-experimental BUILTIN recipes must appear before PROJECT recipes.""" + recipes_dir = tmp_path / ".autoskillit" / "recipes" + recipes_dir.mkdir(parents=True) + (recipes_dir / "aardvark.yaml").write_text( + "name: aardvark\ndescription: test\nsteps: {}\n" + ) + result = list_recipes(tmp_path) + non_exp = [r for r in result.items if not r.experimental] + seen_project = False + for r in non_exp: + if r.source == RecipeSource.PROJECT: + seen_project = True + elif r.source == RecipeSource.BUILTIN: + assert not seen_project, ( + "A BUILTIN recipe appeared after a PROJECT recipe — ordering is broken" + ) + + def test_list_recipes_alphabetical_within_bundled_tiers(self, tmp_path: Path) -> None: + """Unregistered core bundled recipes are alphabetical after registered ones. + Add-on bundled recipes remain alphabetical. + """ + from autoskillit.core.types._type_constants import CORE_PACKS + from autoskillit.recipe.order import BUNDLED_RECIPE_ORDER + + result = list_recipes(tmp_path) + core_names = [ + r.name + for r in result.items + if r.source == RecipeSource.BUILTIN + and not r.experimental + and all(p in CORE_PACKS for p in r.requires_packs) + ] + addon_names = [ + r.name + for r in result.items + if r.source == RecipeSource.BUILTIN + and not r.experimental + and not all(p in CORE_PACKS for p in r.requires_packs) + ] + # Registered entries appear first; the unregistered tail must be alphabetical + unregistered_core = [n for n in core_names if n not in BUNDLED_RECIPE_ORDER] + registered_core = [n for n in core_names if n in BUNDLED_RECIPE_ORDER] + assert registered_core, ( + "BUNDLED_RECIPE_ORDER is empty at test time — registered_core must be non-empty " + "for the ordering contract to be verifiable" + ) + last_registered_idx = core_names.index(registered_core[-1]) + first_unregistered_idx = ( + core_names.index(unregistered_core[0]) if unregistered_core else len(core_names) + ) + assert last_registered_idx < first_unregistered_idx, ( + "Registered core recipes must appear before unregistered ones" + ) + assert unregistered_core == sorted(unregistered_core), ( + f"Unregistered core recipes not alphabetical: {unregistered_core}" + ) + assert addon_names == sorted(addon_names), ( + f"Add-on bundled recipes not in alphabetical order: {addon_names}" + ) + + def test_list_recipes_alphabetical_within_project_tier(self, tmp_path: Path) -> None: + """Project recipes must be sorted alphabetically by name within their tier.""" + recipes_dir = tmp_path / ".autoskillit" / "recipes" + recipes_dir.mkdir(parents=True) + for name in ("zebra", "apple", "mango"): + (recipes_dir / f"{name}.yaml").write_text( + f"name: {name}\ndescription: test\nsteps: {{}}\n" + ) + result = list_recipes(tmp_path) + project_names = [r.name for r in result.items if r.source == RecipeSource.PROJECT] + assert project_names == sorted(project_names), ( + f"Project recipes not in alphabetical order: {project_names}" + ) + + def test_list_recipes_excludes_campaign_when_fleet_disabled(self, tmp_path: Path) -> None: + """list_recipes with exclude_kinds={CAMPAIGN} must omit campaign-kind recipes.""" + recipe_dir = tmp_path / ".autoskillit" / "recipes" + recipe_dir.mkdir(parents=True) + (recipe_dir / "my-campaign.yaml").write_text( + "name: my-campaign\ndescription: test\nkind: campaign\nsteps: {}\n" + ) + result = list_recipes(tmp_path, exclude_kinds=frozenset({RecipeKind.CAMPAIGN})) + assert all(r.kind != RecipeKind.CAMPAIGN for r in result.items) + + def test_list_recipes_includes_campaign_when_fleet_enabled(self, tmp_path: Path) -> None: + """list_recipes with no exclusions must include campaign-kind recipes.""" + recipe_dir = tmp_path / ".autoskillit" / "recipes" + recipe_dir.mkdir(parents=True) + (recipe_dir / "my-campaign.yaml").write_text( + "name: my-campaign\ndescription: test\nkind: campaign\nsteps: {}\n" + ) + result = list_recipes(tmp_path) + names = [r.name for r in result.items] + assert "my-campaign" in names + + def test_recipe_info_kind_field_populated(self, tmp_path: Path) -> None: + """RecipeInfo.kind must be populated from the YAML kind field.""" + recipe_dir = tmp_path / ".autoskillit" / "recipes" + recipe_dir.mkdir(parents=True) + (recipe_dir / "std.yaml").write_text("name: std\ndescription: standard\nsteps: {}\n") + (recipe_dir / "camp.yaml").write_text( + "name: camp\ndescription: campaign\nkind: campaign\nsteps: {}\n" + ) + result = list_recipes(tmp_path) + kinds = {r.name: r.kind for r in result.items} + assert kinds["std"] == RecipeKind.STANDARD + assert kinds["camp"] == RecipeKind.CAMPAIGN + + def test_recipe_info_experimental_field_false_by_default(self, tmp_path: Path) -> None: + """RecipeInfo.experimental must default to False for standard recipes.""" + recipe_dir = tmp_path / ".autoskillit" / "recipes" + recipe_dir.mkdir(parents=True) + (recipe_dir / "plain.yaml").write_text("name: plain\ndescription: plain\nsteps: {}\n") + result = list_recipes(tmp_path) + r = next(r for r in result.items if r.name == "plain") + assert r.experimental is False + + def test_recipe_info_experimental_field_true_when_set(self, tmp_path: Path) -> None: + """RecipeInfo.experimental must be True when YAML sets experimental: true.""" + recipe_dir = tmp_path / ".autoskillit" / "recipes" + recipe_dir.mkdir(parents=True) + (recipe_dir / "research.yaml").write_text( + "name: research\ndescription: exp\nexperimental: true\nsteps: {}\n" + ) + result = list_recipes(tmp_path) + r = next(r for r in result.items if r.name == "research") + assert r.experimental is True + + def test_list_recipes_bundled_before_family_before_experimental(self, tmp_path: Path) -> None: + """list_recipes must order: BUILTIN-non-experimental → PROJECT → experimental.""" + from autoskillit.core.types._type_constants import CORE_PACKS + + recipe_dir = tmp_path / ".autoskillit" / "recipes" + recipe_dir.mkdir(parents=True) + (recipe_dir / "proj.yaml").write_text("name: proj\ndescription: p\nsteps: {}\n") + (recipe_dir / "exp-proj.yaml").write_text( + "name: exp-proj\ndescription: ep\nexperimental: true\nsteps: {}\n" + ) + result = list_recipes(tmp_path) + ranks = [] + for r in result.items: + if r.experimental: + rank = 3 + elif r.source == RecipeSource.PROJECT: + rank = 2 + elif all(p in CORE_PACKS for p in r.requires_packs): + rank = 0 + else: + rank = 1 + if not ranks or ranks[-1] != rank: + ranks.append(rank) + assert ranks == sorted(ranks), f"Groups interleaved: {ranks}" + # experimental must be last + assert 3 in ranks + assert ranks[-1] == 3 + + def test_list_recipes_alphabetical_within_experimental_group(self, tmp_path: Path) -> None: + """Experimental recipes must be sorted alphabetically by name within their group.""" + recipe_dir = tmp_path / ".autoskillit" / "recipes" + recipe_dir.mkdir(parents=True) + for name in ("zebra-exp", "apple-exp", "mango-exp"): + (recipe_dir / f"{name}.yaml").write_text( + f"name: {name}\ndescription: test\nexperimental: true\nsteps: {{}}\n" + ) + result = list_recipes(tmp_path) + exp_names = [r.name for r in result.items if r.experimental] + assert exp_names == sorted(exp_names) + + def test_list_recipes_bundled_experimental_sorted_last(self, tmp_path: Path) -> None: + """A BUILTIN recipe with experimental: true must appear after non-experimental builtins.""" + result = list_recipes(tmp_path) + non_exp_builtin_indices = [ + i + for i, r in enumerate(result.items) + if r.source == RecipeSource.BUILTIN and not r.experimental + ] + exp_builtin_indices = [ + i + for i, r in enumerate(result.items) + if r.source == RecipeSource.BUILTIN and r.experimental + ] + if non_exp_builtin_indices and exp_builtin_indices: + assert max(non_exp_builtin_indices) < min(exp_builtin_indices) + + def test_recipe_info_has_requires_packs_field(self, tmp_path: Path) -> None: + """RecipeInfo must have a requires_packs field defaulting to empty list.""" + from autoskillit.recipe.schema import RecipeInfo + + r = RecipeInfo( + name="x", description="d", source=RecipeSource.BUILTIN, path=tmp_path / "x.yaml" + ) + assert r.requires_packs == [] + + def test_requires_packs_forwarded_to_recipe_info(self, tmp_path: Path) -> None: + """_collect_recipes must populate RecipeInfo.requires_packs from YAML.""" + recipe_dir = tmp_path / ".autoskillit" / "recipes" + recipe_dir.mkdir(parents=True) + (recipe_dir / "custom.yaml").write_text( + "name: custom\ndescription: d\nrequires_packs: [github, ci]\nsteps: {}\n" + ) + result = list_recipes(tmp_path) + r = next(r for r in result.items if r.name == "custom") + assert r.requires_packs == ["github", "ci"] + + def test_core_bundled_before_addon_bundled(self, tmp_path: Path) -> None: + """Core bundled recipes (CORE_PACKS only) must sort before add-on bundled recipes.""" + from autoskillit.core.types._type_constants import CORE_PACKS + + result = list_recipes(tmp_path) + core_indices = [ + i + for i, r in enumerate(result.items) + if r.source == RecipeSource.BUILTIN + and not r.experimental + and r.requires_packs + and all(p in CORE_PACKS for p in r.requires_packs) + ] + addon_indices = [ + i + for i, r in enumerate(result.items) + if r.source == RecipeSource.BUILTIN + and not r.experimental + and r.requires_packs + and not all(p in CORE_PACKS for p in r.requires_packs) + ] + if core_indices and addon_indices: + assert max(core_indices) < min(addon_indices), ( + "Core bundled recipes must appear before add-on bundled recipes" + ) + + +class TestBuiltinRecipesDir: + """Tests for builtin_recipes_dir() function.""" + + def test_returns_existing_directory(self) -> None: + d = builtin_recipes_dir() + assert d.is_dir(), f"builtin_recipes_dir() {d} is not a directory" + + def test_points_to_recipes(self) -> None: + d = builtin_recipes_dir() + assert d.name == "recipes", ( + f"builtin_recipes_dir() should point to 'recipes', got '{d.name}'" + ) + + def test_contains_yaml_files(self) -> None: + d = builtin_recipes_dir() + yaml_files = list(d.glob("*.yaml")) + assert len(yaml_files) > 0, "builtin_recipes_dir() contains no YAML files" + + +def test_list_recipes_stable_with_project_recipe_added(tmp_path: Path) -> None: + """Adding a project recipe must not shift the positions of bundled recipes.""" + # Collect bundled positions without any project recipes + before = [r.name for r in list_recipes(tmp_path).items] + + # Add a project recipe whose name sorts before all bundled recipes + recipes_dir = tmp_path / ".autoskillit" / "recipes" + recipes_dir.mkdir(parents=True) + (recipes_dir / "aaa-custom.yaml").write_text( + "name: aaa-custom\ndescription: test\nsteps: {}\n" + ) + after = [r.name for r in list_recipes(tmp_path).items] + + # Bundled names must occupy the same leading positions + bundled_before = list(before) + bundled_after = [n for n in after if n in set(bundled_before)] + assert bundled_after == bundled_before, ( + "Adding a project recipe must not shift bundled recipe positions" + ) + + +def test_parse_recipe_reads_requires_packs(): + from autoskillit.recipe.io import _parse_recipe + + data = { + "name": "test", + "description": "d", + "requires_packs": ["research", "github"], + } + recipe = _parse_recipe(data) + assert recipe.requires_packs == ["research", "github"] + + +def test_parse_recipe_requires_packs_defaults_to_empty(): + from autoskillit.recipe.io import _parse_recipe + + recipe = _parse_recipe({"name": "test", "description": "d"}) + assert recipe.requires_packs == [] + + +# --------------------------------------------------------------------------- +# Food-truck kind tests +# --------------------------------------------------------------------------- + + +def test_food_truck_excluded_from_order_menu(tmp_path: Path) -> None: + """kind: food-truck recipe is excluded when exclude_kinds includes FOOD_TRUCK.""" + recipes_dir = tmp_path / ".autoskillit" / "recipes" + recipes_dir.mkdir(parents=True) + (recipes_dir / "food-truck-test.yaml").write_text( + "name: food-truck-test\ndescription: test\nkind: food-truck\n" + "steps:\n done:\n action: stop\n message: sentinel done\n" + ) + result = list_recipes( + tmp_path, + exclude_kinds=frozenset({RecipeKind.CAMPAIGN, RecipeKind.FOOD_TRUCK}), + ) + names = {r.name for r in result.items} + assert "food-truck-test" not in names + + +def test_food_truck_included_when_not_excluded(tmp_path: Path) -> None: + """kind: food-truck recipe is included when exclude_kinds does not include FOOD_TRUCK.""" + recipes_dir = tmp_path / ".autoskillit" / "recipes" + recipes_dir.mkdir(parents=True) + (recipes_dir / "food-truck-test.yaml").write_text( + "name: food-truck-test\ndescription: test\nkind: food-truck\n" + "steps:\n done:\n action: stop\n message: sentinel done\n" + ) + result = list_recipes(tmp_path) + names = {r.name for r in result.items} + assert "food-truck-test" in names + + +def test_list_all_excludes_food_truck_when_fleet_off(tmp_path: Path) -> None: + """list_all with fleet disabled excludes FOOD_TRUCK kind recipes.""" + from autoskillit.recipe._api import list_all + + recipes_dir = tmp_path / ".autoskillit" / "recipes" + recipes_dir.mkdir(parents=True) + (recipes_dir / "my-food-truck.yaml").write_text( + "name: my-food-truck\ndescription: test\nkind: food-truck\n" + "steps:\n done:\n action: stop\n message: sentinel done\n" + ) + result = list_all(tmp_path, features={"fleet": False}) + names = {r["name"] for r in result["recipes"]} + assert "my-food-truck" not in names + + +def test_research_recipe_loads_without_error(): + from autoskillit.core.paths import pkg_root + + path = pkg_root() / "recipes" / "research.yaml" + recipe = load_recipe(path) + assert recipe.name == "research" + + +def test_research_recipe_declares_requires_packs(): + from autoskillit.core.paths import pkg_root + + path = pkg_root() / "recipes" / "research.yaml" + recipe = load_recipe(path) + assert recipe.requires_packs == ["research", "exp-lens", "vis-lens"] diff --git a/tests/recipe/test_io_json_precompile.py b/tests/recipe/test_io_json_precompile.py new file mode 100644 index 0000000000..5bb062dbad --- /dev/null +++ b/tests/recipe/test_io_json_precompile.py @@ -0,0 +1,221 @@ +"""Tests for JSON pre-compilation fast path in recipe I/O.""" + +from __future__ import annotations + +import json +import os +from pathlib import Path + +import pytest +import yaml + +from autoskillit.core import RecipeSource +from autoskillit.recipe.io import ( + _collect_recipes, + _load_recipe_dict, + builtin_recipes_dir, +) + +pytestmark = [pytest.mark.layer("recipe"), pytest.mark.medium] + +_MINIMAL_RECIPE = { + "name": "test-recipe", + "description": "A test recipe", + "steps": { + "done": {"action": "stop", "message": "Done."}, + }, +} + + +def _write_yaml(path: Path, data: dict) -> None: + path.write_text(yaml.dump(data, default_flow_style=False), encoding="utf-8") + + +def _compile_json(yaml_path: Path) -> None: + """Inline compilation: YAML -> JSON sibling.""" + data = yaml.safe_load(yaml_path.read_bytes()) + json_path = yaml_path.with_suffix(".json") + json_path.write_text(json.dumps(data, indent=2, ensure_ascii=False) + "\n", encoding="utf-8") + + +def test_load_recipe_dict_prefers_json_when_fresh(tmp_path, monkeypatch): + yaml_path = tmp_path / "recipe.yaml" + _write_yaml(yaml_path, _MINIMAL_RECIPE) + _compile_json(yaml_path) + # Ensure JSON mtime is strictly greater than YAML mtime (required by > freshness gate) + json_path = yaml_path.with_suffix(".json") + future_mtime_ns = yaml_path.stat().st_mtime_ns + 10_000_000_000 + os.utime(json_path, ns=(future_mtime_ns, future_mtime_ns)) + + load_yaml_calls = [] + monkeypatch.setattr( + "autoskillit.recipe.io.load_yaml", + lambda *a, **kw: load_yaml_calls.append(1) or _MINIMAL_RECIPE, + ) + + result = _load_recipe_dict(yaml_path) + + assert result == _MINIMAL_RECIPE + assert load_yaml_calls == [], "load_yaml should not be called when JSON is fresh" + + +def test_load_recipe_dict_falls_back_when_json_missing(tmp_path, monkeypatch): + yaml_path = tmp_path / "recipe.yaml" + _write_yaml(yaml_path, _MINIMAL_RECIPE) + + load_yaml_calls = [] + monkeypatch.setattr( + "autoskillit.recipe.io.load_yaml", + lambda *a, **kw: load_yaml_calls.append(1) or _MINIMAL_RECIPE, + ) + + result = _load_recipe_dict(yaml_path) + + assert result == _MINIMAL_RECIPE + assert load_yaml_calls == [1], "load_yaml should be called when JSON sibling is absent" + + +def test_load_recipe_dict_falls_back_when_json_stale(tmp_path, monkeypatch): + yaml_path = tmp_path / "recipe.yaml" + _write_yaml(yaml_path, _MINIMAL_RECIPE) + json_path = yaml_path.with_suffix(".json") + + _compile_json(yaml_path) + # Manually set JSON mtime to the past so YAML is newer + js_mtime_ns = json_path.stat().st_mtime_ns + os.utime(json_path, ns=(js_mtime_ns - 10_000_000_000, js_mtime_ns - 10_000_000_000)) + + load_yaml_calls = [] + monkeypatch.setattr( + "autoskillit.recipe.io.load_yaml", + lambda *a, **kw: load_yaml_calls.append(1) or _MINIMAL_RECIPE, + ) + + result = _load_recipe_dict(yaml_path) + + assert result == _MINIMAL_RECIPE + assert load_yaml_calls == [1], "load_yaml should be called when JSON is stale" + + +def test_load_recipe_dict_applies_substitution_on_json(tmp_path): + recipe_with_placeholder = { + "name": "test-recipe", + "description": "A test", + "steps": { + "run": { + "tool": "test_check", + "with": {"worktree_path": "{{AUTOSKILLIT_TEMP}}"}, + }, + }, + } + yaml_path = tmp_path / "recipe.yaml" + _write_yaml(yaml_path, recipe_with_placeholder) + _compile_json(yaml_path) + json_path = yaml_path.with_suffix(".json") + future_mtime_ns = yaml_path.stat().st_mtime_ns + 10_000_000_000 + os.utime(json_path, ns=(future_mtime_ns, future_mtime_ns)) + + result = _load_recipe_dict(yaml_path, temp_dir_relpath="custom/temp") + + assert result["steps"]["run"]["with"]["worktree_path"] == "custom/temp" + + +def test_load_recipe_dict_handles_json_decode_error(tmp_path, monkeypatch): + yaml_path = tmp_path / "recipe.yaml" + _write_yaml(yaml_path, _MINIMAL_RECIPE) + json_path = yaml_path.with_suffix(".json") + # Write corrupt content with an explicitly future mtime so the freshness gate passes + json_path.write_text("{ invalid json }", encoding="utf-8") + future_mtime_ns = yaml_path.stat().st_mtime_ns + 10_000_000_000 + os.utime(json_path, ns=(future_mtime_ns, future_mtime_ns)) + + load_yaml_calls = [] + monkeypatch.setattr( + "autoskillit.recipe.io.load_yaml", + lambda *a, **kw: load_yaml_calls.append(1) or _MINIMAL_RECIPE, + ) + + result = _load_recipe_dict(yaml_path) + + assert result == _MINIMAL_RECIPE + assert load_yaml_calls == [1], "load_yaml should be called when JSON is corrupt" + + +def test_load_recipe_dict_falls_back_when_json_is_not_mapping(tmp_path, monkeypatch): + yaml_path = tmp_path / "recipe.yaml" + _write_yaml(yaml_path, _MINIMAL_RECIPE) + json_path = yaml_path.with_suffix(".json") + # Write valid JSON that is a list (not a mapping) with a future mtime + json_path.write_text("[1, 2, 3]\n", encoding="utf-8") + future_mtime_ns = yaml_path.stat().st_mtime_ns + 10_000_000_000 + os.utime(json_path, ns=(future_mtime_ns, future_mtime_ns)) + + load_yaml_calls = [] + monkeypatch.setattr( + "autoskillit.recipe.io.load_yaml", + lambda *a, **kw: load_yaml_calls.append(1) or _MINIMAL_RECIPE, + ) + + result = _load_recipe_dict(yaml_path) + + assert result == _MINIMAL_RECIPE + assert load_yaml_calls == [1], "load_yaml should be called when JSON is not a mapping" + + +def test_collect_recipes_identical_with_json(tmp_path): + recipes = [ + {"name": "recipe-a", "description": "A", "steps": {"done": {"action": "stop"}}}, + {"name": "recipe-b", "description": "B", "steps": {"done": {"action": "stop"}}}, + {"name": "recipe-c", "description": "C", "steps": {"done": {"action": "stop"}}}, + ] + for r in recipes: + _write_yaml(tmp_path / f"{r['name']}.yaml", r) + + # Collect without JSON siblings + seen1: set[str] = set() + items1: list = [] + errors1: list = [] + _collect_recipes(RecipeSource.PROJECT, tmp_path, seen1, items1, errors1) + + # Compile JSON siblings and set future mtime so the fast path is exercised + for r in recipes: + yaml_path = tmp_path / f"{r['name']}.yaml" + _compile_json(yaml_path) + json_path = yaml_path.with_suffix(".json") + future_mtime_ns = yaml_path.stat().st_mtime_ns + 10_000_000_000 + os.utime(json_path, ns=(future_mtime_ns, future_mtime_ns)) + + # Collect with JSON siblings + seen2: set[str] = set() + items2: list = [] + errors2: list = [] + _collect_recipes(RecipeSource.PROJECT, tmp_path, seen2, items2, errors2) + + # Assert identical name/description/version/kind + for a, b in zip(sorted(items1, key=lambda x: x.name), sorted(items2, key=lambda x: x.name)): + assert a.name == b.name + assert a.description == b.description + assert a.version == b.version + assert a.kind == b.kind + + +def test_compile_recipes_roundtrip(): + for yaml_path in sorted(builtin_recipes_dir().rglob("*.yaml")): + data = yaml.safe_load(yaml_path.read_bytes()) + json_text = json.dumps(data, indent=2, ensure_ascii=False) + "\n" + roundtripped = json.loads(json_text) + assert roundtripped == data, f"Roundtrip failed for {yaml_path.name}" + + +def test_bundled_json_files_are_fresh(): + for yaml_path in sorted(builtin_recipes_dir().rglob("*.yaml")): + json_path = yaml_path.with_suffix(".json") + assert json_path.exists(), f"Missing JSON sibling for {yaml_path.name}" + + yaml_data = yaml.safe_load(yaml_path.read_bytes()) + json_data = json.loads(json_path.read_text(encoding="utf-8")) + assert json_data == yaml_data, f"JSON is stale for {yaml_path.name}" + assert json_path.stat().st_mtime_ns > yaml_path.stat().st_mtime_ns, ( + f"JSON mtime is older than YAML mtime for {yaml_path.name}" + " — fast-path would be bypassed" + ) diff --git a/tests/recipe/test_io.py b/tests/recipe/test_io_parsing.py similarity index 62% rename from tests/recipe/test_io.py rename to tests/recipe/test_io_parsing.py index 7b852f5b48..c14f135603 100644 --- a/tests/recipe/test_io.py +++ b/tests/recipe/test_io_parsing.py @@ -1,13 +1,13 @@ -"""Tests for recipe I/O and parsing (recipe_io module).""" +"""Tests for recipe I/O — parse functions, TestRecipeParser, kitchen_rules, schema drift.""" from __future__ import annotations import re +import textwrap from pathlib import Path import pytest -from autoskillit.core.types import RecipeSource from autoskillit.recipe.io import ( _parse_recipe, _parse_step, @@ -22,6 +22,8 @@ ) from tests.recipe.conftest import VALID_RECIPE, _write_yaml +pytestmark = [pytest.mark.layer("recipe"), pytest.mark.medium] + def test_load_recipe_smoke() -> None: """load_recipe(path) returns a Recipe with correct name.""" @@ -114,6 +116,8 @@ def test_ingredient_defaults_applied(self, tmp_path: Path) -> None: # WF8 def test_project_recipe_overrides_builtin(self, tmp_path: Path) -> None: + from autoskillit.core.types import RecipeSource + wf_dir = tmp_path / ".autoskillit" / "recipes" wf_dir.mkdir(parents=True) override = {**VALID_RECIPE, "name": "implementation", "description": "Custom override"} @@ -433,6 +437,16 @@ def test_parse_step_model_absent(self) -> None: step = _parse_step({"tool": "run_skill"}) assert step.model is None + # PROV1 + def test_parse_step_extracts_provider(self) -> None: + step = _parse_step({"tool": "run_skill", "provider": "anthropic"}) + assert step.provider == "anthropic" + + # PROV2 + def test_parse_step_provider_absent(self) -> None: + step = _parse_step({"tool": "run_skill"}) + assert step.provider is None + def test_parse_step_reads_stale_threshold(self) -> None: data = {"tool": "run_skill", "stale_threshold": 2400, "on_success": "done"} step = _parse_step(data) @@ -468,259 +482,14 @@ def test_bundled_resolve_failures_steps_use_config_default(self) -> None: step.with_args.get("skill_command") and "resolve-failures" in step.with_args["skill_command"] ): - assert step.model is None, ( + # model: "" is the "use config default" sentinel — `not step.model` + # accepts both None and "" so resolve-failures steps pass either way. + assert not step.model, ( f"{f.name} step '{step_name}' should not have explicit model " - "(sonnet is the config default)" + f"(sonnet is the config default); got {step.model!r}" ) -class TestListRecipes: - """TestListRecipes: discovery from project and builtin sources.""" - - def test_finds_builtins(self, tmp_path: Path) -> None: - result = list_recipes(tmp_path) - recipes = result.items - names = {w.name for w in recipes} - assert "implementation" in names - assert len(recipes) > 0 - assert all(r.source.value in ("project", "builtin") for r in recipes) - - def test_list_recipes_bundled_appear_before_project(self, tmp_path: Path) -> None: - """Bundled recipes must appear before project recipes in list_recipes() output.""" - recipes_dir = tmp_path / ".autoskillit" / "recipes" - recipes_dir.mkdir(parents=True) - # Write a project recipe whose name would sort before bundled "implementation" - (recipes_dir / "aardvark.yaml").write_text( - "name: aardvark\ndescription: test\nsteps: {}\n" - ) - result = list_recipes(tmp_path) - sources = [r.source for r in result.items] - # All BUILTIN items must precede all PROJECT items - seen_project = False - for source in sources: - if source == RecipeSource.PROJECT: - seen_project = True - elif source == RecipeSource.BUILTIN: - assert not seen_project, ( - "A BUILTIN recipe appeared after a PROJECT recipe — ordering is broken" - ) - - def test_list_recipes_alphabetical_within_bundled_tier(self, tmp_path: Path) -> None: - """Bundled recipes must be sorted alphabetically by name within their tier.""" - result = list_recipes(tmp_path) - builtin_names = [r.name for r in result.items if r.source == RecipeSource.BUILTIN] - assert builtin_names == sorted(builtin_names), ( - f"Bundled recipes not in alphabetical order: {builtin_names}" - ) - - def test_list_recipes_alphabetical_within_project_tier(self, tmp_path: Path) -> None: - """Project recipes must be sorted alphabetically by name within their tier.""" - recipes_dir = tmp_path / ".autoskillit" / "recipes" - recipes_dir.mkdir(parents=True) - for name in ("zebra", "apple", "mango"): - (recipes_dir / f"{name}.yaml").write_text( - f"name: {name}\ndescription: test\nsteps: {{}}\n" - ) - result = list_recipes(tmp_path) - project_names = [r.name for r in result.items if r.source == RecipeSource.PROJECT] - assert project_names == sorted(project_names), ( - f"Project recipes not in alphabetical order: {project_names}" - ) - - -class TestBuiltinRecipesDir: - """Tests for builtin_recipes_dir() function.""" - - def test_returns_existing_directory(self) -> None: - d = builtin_recipes_dir() - assert d.is_dir(), f"builtin_recipes_dir() {d} is not a directory" - - def test_points_to_recipes(self) -> None: - d = builtin_recipes_dir() - assert d.name == "recipes", ( - f"builtin_recipes_dir() should point to 'recipes', got '{d.name}'" - ) - - def test_contains_yaml_files(self) -> None: - d = builtin_recipes_dir() - yaml_files = list(d.glob("*.yaml")) - assert len(yaml_files) > 0, "builtin_recipes_dir() contains no YAML files" - - -class TestVersionField: - """autoskillit_version field on Recipe dataclass.""" - - # VER1+VER2 merged - @pytest.mark.parametrize("version_val,expected", [(None, None), ("0.2.0", "0.2.0")]) - def test_version_field(self, version_val, expected) -> None: - data = { - "name": "v-test", - "description": "d", - "steps": { - "do_it": {"tool": "run_cmd", "on_success": "done"}, - "done": {"action": "stop", "message": "Done."}, - }, - } - if version_val is not None: - data["autoskillit_version"] = version_val - wf = _parse_recipe(data) - assert wf.version == expected - - # VER4 - def test_version_preserved_in_round_trip(self, tmp_path: Path) -> None: - data = { - "name": "version-test-recipe", - "description": "A recipe for testing the version field", - "kitchen_rules": ["Only use AutoSkillit MCP tools during pipeline execution"], - "steps": { - "do_it": {"tool": "run_cmd", "on_success": "done"}, - "done": {"action": "stop", "message": "Done."}, - }, - "autoskillit_version": "1.3.0", - } - path = _write_yaml(tmp_path / "recipe.yaml", data) - wf = load_recipe(path) - assert wf.version == "1.3.0" - - -# --------------------------------------------------------------------------- -# capture_list field tests (D1–D3, D8–D9) -# --------------------------------------------------------------------------- - - -# D1 -def test_recipe_step_accepts_capture_list_field() -> None: - """RecipeStep accepts capture_list field and stores it.""" - step = RecipeStep( - tool="run_skill", - with_args={"skill_command": "/autoskillit:make-plan inputs.task"}, - capture={"plan_path": "${{ result.plan_path }}"}, - capture_list={"plan_parts": "${{ result.plan_parts }}"}, - on_success="verify", - ) - assert step.capture_list == {"plan_parts": "${{ result.plan_parts }}"} - - -# D2 -def test_recipe_step_capture_list_defaults_empty() -> None: - """RecipeStep.capture_list defaults to an empty dict.""" - step = RecipeStep(tool="run_skill", with_args={}, on_success="done") - assert step.capture_list == {} - - -# D3 -def test_recipe_yaml_with_capture_list_parses(tmp_path: Path) -> None: - """YAML recipe with capture_list key is parsed into RecipeStep.capture_list.""" - data = { - "name": "test-recipe", - "description": "test", - "ingredients": {}, - "steps": { - "plan": { - "tool": "run_skill", - "with": {"skill_command": "/autoskillit:make-plan inputs.task"}, - "capture": {"plan_path": "${{ result.plan_path }}"}, - "capture_list": {"plan_parts": "${{ result.plan_parts }}"}, - "on_success": "done", - }, - "done": {"action": "stop", "message": "Done"}, - }, - } - path = _write_yaml(tmp_path / "recipe.yaml", data) - recipe = load_recipe(path) - assert recipe.steps["plan"].capture_list == {"plan_parts": "${{ result.plan_parts }}"} - - -# D8 -def test_iter_steps_with_context_includes_capture_list_keys() -> None: - """iter_steps_with_context must include capture_list keys in available_context.""" - from autoskillit.recipe.io import iter_steps_with_context - - recipe = Recipe( - name="test", - description="test", - ingredients={}, - steps={ - "plan": RecipeStep( - tool="run_skill", - with_args={"skill_command": "/autoskillit:make-plan t"}, - capture={"plan_path": "${{ result.plan_path }}"}, - capture_list={"plan_parts": "${{ result.plan_parts }}"}, - on_success="verify", - ), - "verify": RecipeStep( - tool="run_skill", - with_args={"skill_command": "/autoskillit:dry-walkthrough c"}, - on_success="done", - ), - "done": RecipeStep(action="stop", message="Done"), - }, - kitchen_rules=[], - ) - steps = list(iter_steps_with_context(recipe)) - verify_ctx = next(ctx for name, _, ctx in steps if name == "verify") - assert "plan_parts" in verify_ctx, ( - "capture_list keys must appear in available_context for downstream steps" - ) - - -# D9 -def test_implementation_pipeline_captures_plan_parts_as_list() -> None: - """implementation.yaml plan step must capture plan_parts via capture_list.""" - recipe = load_recipe(builtin_recipes_dir() / "implementation.yaml") - step = recipe.steps["plan"] - assert hasattr(step, "capture_list"), "RecipeStep must have capture_list field" - assert "plan_parts" in step.capture_list, ( - "implementation plan step must capture plan_parts via capture_list" - ) - - -# IO-1: RecipeInfo dataclass accepts content kwarg; defaults to None -def test_recipe_info_has_content_field_defaulting_to_none() -> None: - """RecipeInfo.content defaults to None when not provided.""" - from autoskillit.recipe.schema import RecipeInfo - - info = RecipeInfo( - name="x", - description="y", - source=RecipeSource.BUILTIN, - path=Path("/x.yaml"), - ) - assert info.content is None - - -# IO-2: list_recipes populates content field with raw YAML text -def test_list_recipes_populates_content(tmp_path: Path) -> None: - """list_recipes() populates the content field with raw YAML text.""" - recipes_dir = tmp_path / ".autoskillit" / "recipes" - recipes_dir.mkdir(parents=True) - raw = "name: my-recipe\ndescription: test\nsteps: {}\n" - (recipes_dir / "my-recipe.yaml").write_text(raw) - result = list_recipes(tmp_path) - assert result.items, "expected at least one recipe" - item = next(r for r in result.items if r.name == "my-recipe") - assert item.content == raw - - -# IO-3: content field preserved through find_recipe_by_name -def test_find_recipe_by_name_returns_info_with_content(tmp_path: Path) -> None: - """find_recipe_by_name() returns a RecipeInfo with content populated.""" - from autoskillit.recipe.io import find_recipe_by_name - - recipes_dir = tmp_path / ".autoskillit" / "recipes" - recipes_dir.mkdir(parents=True) - raw = "name: my-recipe\ndescription: test\nsteps: {}\n" - (recipes_dir / "my-recipe.yaml").write_text(raw) - info = find_recipe_by_name("my-recipe", tmp_path) - assert info is not None - assert info.content == raw - - -# --------------------------------------------------------------------------- -# Bundled recipe skill_command prefix contract -# --------------------------------------------------------------------------- - - def test_parse_step_with_key_maps_to_with_args() -> None: """_parse_step maps YAML 'with' key to RecipeStep.with_args.""" step = _parse_step({"tool": "claim_issue", "with": {"issue_url": "https://example.com"}}) @@ -733,11 +502,6 @@ def test_parse_step_with_args_key_is_not_read() -> None: assert step.with_args == {}, "with_args YAML key must not be read — use 'with:' instead" -# --------------------------------------------------------------------------- -# P9-F1: step description field mapping -# --------------------------------------------------------------------------- - - def test_parse_step_maps_description_field() -> None: """_parse_step maps 'description' YAML key to RecipeStep.description.""" step = _parse_step({"tool": "run_cmd", "description": "Run the build"}) @@ -752,8 +516,6 @@ def test_parse_step_description_defaults_to_empty_string() -> None: def test_load_recipe_preserves_step_description(tmp_path: Path) -> None: """End-to-end: load_recipe preserves description: on a step.""" - import textwrap - yaml_content = textwrap.dedent("""\ name: desc-test kitchen_rules: [rule1] @@ -770,42 +532,18 @@ def test_load_recipe_preserves_step_description(tmp_path: Path) -> None: assert recipe.steps["build"].description == "Run the full build suite" -# --------------------------------------------------------------------------- -# CC-F4: kitchen_rules non-list raises ValueError -# --------------------------------------------------------------------------- - - -def test_parse_recipe_kitchen_rules_string_raises() -> None: - """_parse_recipe raises ValueError when kitchen_rules is a string.""" - with pytest.raises(ValueError, match="kitchen_rules"): - _parse_recipe( - { - "name": "bad", - "kitchen_rules": "not-a-list", - "steps": {"s": {"tool": "run_cmd"}}, - } - ) - - -def test_parse_recipe_kitchen_rules_dict_raises() -> None: - """_parse_recipe raises ValueError when kitchen_rules is a dict.""" - with pytest.raises(ValueError, match="kitchen_rules"): - _parse_recipe( - { - "name": "bad", - "kitchen_rules": {"rule": "val"}, - "steps": {"s": {"tool": "run_cmd"}}, - } - ) - - -def test_parse_recipe_kitchen_rules_int_raises() -> None: - """_parse_recipe raises ValueError when kitchen_rules is an int.""" +@pytest.mark.parametrize( + "bad_val", + ["not-a-list", {"rule": "val"}, 42], + ids=["string", "dict", "int"], +) +def test_parse_recipe_kitchen_rules_rejects_non_list(bad_val: object) -> None: + """_parse_recipe raises ValueError when kitchen_rules is not a list.""" with pytest.raises(ValueError, match="kitchen_rules"): _parse_recipe( { "name": "bad", - "kitchen_rules": 42, + "kitchen_rules": bad_val, "steps": {"s": {"tool": "run_cmd"}}, } ) @@ -850,15 +588,9 @@ def test_bundled_recipes_all_skill_commands_start_with_slash() -> None: assert not violations, f"Bundled recipe steps with prose skill_command: {violations}" -# --------------------------------------------------------------------------- -# T1 — schema-drift guard -# --------------------------------------------------------------------------- - - def test_parse_step_handles_all_recipe_step_fields() -> None: """_PARSE_STEP_HANDLED_FIELDS must equal RecipeStep.__dataclass_fields__ keys.""" from autoskillit.recipe.io import _PARSE_STEP_HANDLED_FIELDS - from autoskillit.recipe.schema import RecipeStep schema_fields = frozenset(RecipeStep.__dataclass_fields__) assert _PARSE_STEP_HANDLED_FIELDS == schema_fields, ( @@ -868,64 +600,91 @@ def test_parse_step_handles_all_recipe_step_fields() -> None: ) -# --------------------------------------------------------------------------- -# REQ-ORD-003: Stable positions when project recipe added -# --------------------------------------------------------------------------- +def test_parse_recipe_handles_all_recipe_fields() -> None: + """Combined handled+computed fields must equal Recipe.__dataclass_fields__.""" + from autoskillit.recipe.io import _PARSE_RECIPE_HANDLED_FIELDS, _RECIPE_COMPUTED_FIELDS + schema_fields = frozenset(Recipe.__dataclass_fields__) + combined = _PARSE_RECIPE_HANDLED_FIELDS | _RECIPE_COMPUTED_FIELDS + assert combined == schema_fields, ( + f"_parse_recipe field list is out of sync.\n" + f" Missing from handled+computed: {schema_fields - combined}\n" + f" Extra in handled+computed: {combined - schema_fields}" + ) -def test_list_recipes_stable_with_project_recipe_added(tmp_path: Path) -> None: - """Adding a project recipe must not shift the positions of bundled recipes.""" - # Collect bundled positions without any project recipes - before = [r.name for r in list_recipes(tmp_path).items] - # Add a project recipe whose name sorts before all bundled recipes - recipes_dir = tmp_path / ".autoskillit" / "recipes" - recipes_dir.mkdir(parents=True) - (recipes_dir / "aaa-custom.yaml").write_text( - "name: aaa-custom\ndescription: test\nsteps: {}\n" - ) - after = [r.name for r in list_recipes(tmp_path).items] +def test_parse_recipe_handled_and_computed_are_disjoint() -> None: + """Parsed and computed field sets must not overlap.""" + from autoskillit.recipe.io import _PARSE_RECIPE_HANDLED_FIELDS, _RECIPE_COMPUTED_FIELDS - # Bundled names must occupy the same leading positions - bundled_before = list(before) - bundled_after = [n for n in after if n in set(bundled_before)] - assert bundled_after == bundled_before, ( - "Adding a project recipe must not shift bundled recipe positions" - ) + overlap = _PARSE_RECIPE_HANDLED_FIELDS & _RECIPE_COMPUTED_FIELDS + assert not overlap, f"Fields in both handled and computed: {overlap}" -def test_parse_recipe_reads_requires_packs(): - from autoskillit.recipe.io import _parse_recipe +def test_step_provider_field_parses_correctly() -> None: + step = _parse_step({"tool": "run_skill", "provider": "minimax"}) + assert step.provider == "minimax" - data = { - "name": "test", - "description": "d", - "requires_packs": ["research", "github"], - } - recipe = _parse_recipe(data) - assert recipe.requires_packs == ["research", "github"] +def test_step_provider_field_default_is_none() -> None: + step = _parse_step({"tool": "run_skill"}) + assert step.provider is None -def test_parse_recipe_requires_packs_defaults_to_empty(): - from autoskillit.recipe.io import _parse_recipe - recipe = _parse_recipe({"name": "test", "description": "d"}) - assert recipe.requires_packs == [] +def test_load_recipe_step_with_provider_field(tmp_path: Path) -> None: + yaml_content = textwrap.dedent("""\ + name: provider-test + kitchen_rules: [rule1] + steps: + run_step: + tool: run_skill + provider: minimax + on_success: done + done: + action: stop + message: Done. + """) + recipe_file = tmp_path / "provider-test.yaml" + recipe_file.write_text(yaml_content) + recipe = load_recipe(recipe_file) + assert recipe.steps["run_step"].provider == "minimax" -def test_research_recipe_loads_without_error(): - from autoskillit.core.paths import pkg_root - from autoskillit.recipe.io import load_recipe +# --------------------------------------------------------------------------- +# T4: _parse_recipe reads type: from YAML ingredient block +# --------------------------------------------------------------------------- - path = pkg_root() / "recipes" / "research.yaml" - recipe = load_recipe(path) - assert recipe.name == "research" +def test_parse_recipe_reads_ingredient_type_field() -> None: + """_parse_recipe must parse the 'type' key from a YAML ingredient block.""" + data = { + "name": "test", + "description": "Test recipe", + "ingredients": { + "local_review_rounds": { + "description": "Number of local review rounds", + "type": "integer", + "default": "3", + } + }, + "steps": {"done": {"action": "stop", "message": "Done."}}, + } + recipe = _parse_recipe(data) + assert recipe.ingredients["local_review_rounds"].type == "integer" -def test_research_recipe_declares_requires_packs(): - from autoskillit.core.paths import pkg_root - from autoskillit.recipe.io import load_recipe - path = pkg_root() / "recipes" / "research.yaml" - recipe = load_recipe(path) - assert recipe.requires_packs == ["research", "exp-lens", "vis-lens"] +def test_parse_recipe_defaults_type_to_none_when_absent() -> None: + """When 'type' is absent from a YAML ingredient block, .type must be None.""" + data = { + "name": "test", + "description": "Test recipe", + "ingredients": { + "base_branch": { + "description": "Base branch to target", + "default": "main", + } + }, + "steps": {"done": {"action": "stop", "message": "Done."}}, + } + recipe = _parse_recipe(data) + assert recipe.ingredients["base_branch"].type is None diff --git a/tests/recipe/test_io_schema_fields.py b/tests/recipe/test_io_schema_fields.py new file mode 100644 index 0000000000..f1e8296cab --- /dev/null +++ b/tests/recipe/test_io_schema_fields.py @@ -0,0 +1,287 @@ +"""Tests for recipe I/O — capture_list, RecipeInfo.content, version fields, and content hash.""" + +from __future__ import annotations + +import hashlib +from pathlib import Path + +import pytest + +from autoskillit.core.types import RecipeSource +from autoskillit.recipe.io import ( + builtin_recipes_dir, + list_recipes, + load_recipe, +) +from autoskillit.recipe.schema import ( + Recipe, + RecipeStep, +) +from tests.recipe.conftest import _write_yaml + +pytestmark = [pytest.mark.layer("recipe"), pytest.mark.medium] + + +class TestVersionField: + """autoskillit_version field on Recipe dataclass.""" + + # VER1+VER2 merged + @pytest.mark.parametrize("version_val,expected", [(None, None), ("0.2.0", "0.2.0")]) + def test_version_field(self, version_val, expected) -> None: + from autoskillit.recipe.io import _parse_recipe + + data = { + "name": "v-test", + "description": "d", + "steps": { + "do_it": {"tool": "run_cmd", "on_success": "done"}, + "done": {"action": "stop", "message": "Done."}, + }, + } + if version_val is not None: + data["autoskillit_version"] = version_val + wf = _parse_recipe(data) + assert wf.version == expected + + # VER4 + def test_version_preserved_in_round_trip(self, tmp_path: Path) -> None: + data = { + "name": "version-test-recipe", + "description": "A recipe for testing the version field", + "kitchen_rules": ["Only use AutoSkillit MCP tools during pipeline execution"], + "steps": { + "do_it": {"tool": "run_cmd", "on_success": "done"}, + "done": {"action": "stop", "message": "Done."}, + }, + "autoskillit_version": "1.3.0", + } + path = _write_yaml(tmp_path / "recipe.yaml", data) + wf = load_recipe(path) + assert wf.version == "1.3.0" + + +# --------------------------------------------------------------------------- +# capture_list field tests (D1–D3, D8–D9) +# --------------------------------------------------------------------------- + + +# D1 +def test_recipe_step_accepts_capture_list_field() -> None: + """RecipeStep accepts capture_list field and stores it.""" + step = RecipeStep( + tool="run_skill", + with_args={"skill_command": "/autoskillit:make-plan inputs.task"}, + capture={"plan_path": "${{ result.plan_path }}"}, + capture_list={"plan_parts": "${{ result.plan_parts }}"}, + on_success="verify", + ) + assert step.capture_list == {"plan_parts": "${{ result.plan_parts }}"} + + +# D2 +def test_recipe_step_capture_list_defaults_empty() -> None: + """RecipeStep.capture_list defaults to an empty dict.""" + step = RecipeStep(tool="run_skill", with_args={}, on_success="done") + assert step.capture_list == {} + + +# D3 +def test_recipe_yaml_with_capture_list_parses(tmp_path: Path) -> None: + """YAML recipe with capture_list key is parsed into RecipeStep.capture_list.""" + data = { + "name": "test-recipe", + "description": "test", + "ingredients": {}, + "steps": { + "plan": { + "tool": "run_skill", + "with": {"skill_command": "/autoskillit:make-plan inputs.task"}, + "capture": {"plan_path": "${{ result.plan_path }}"}, + "capture_list": {"plan_parts": "${{ result.plan_parts }}"}, + "on_success": "done", + }, + "done": {"action": "stop", "message": "Done"}, + }, + } + path = _write_yaml(tmp_path / "recipe.yaml", data) + recipe = load_recipe(path) + assert recipe.steps["plan"].capture_list == {"plan_parts": "${{ result.plan_parts }}"} + + +# D8 +def test_iter_steps_with_context_includes_capture_list_keys() -> None: + """iter_steps_with_context must include capture_list keys in available_context.""" + from autoskillit.recipe.io import iter_steps_with_context + + recipe = Recipe( + name="test", + description="test", + ingredients={}, + steps={ + "plan": RecipeStep( + tool="run_skill", + with_args={"skill_command": "/autoskillit:make-plan t"}, + capture={"plan_path": "${{ result.plan_path }}"}, + capture_list={"plan_parts": "${{ result.plan_parts }}"}, + on_success="verify", + ), + "verify": RecipeStep( + tool="run_skill", + with_args={"skill_command": "/autoskillit:dry-walkthrough c"}, + on_success="done", + ), + "done": RecipeStep(action="stop", message="Done"), + }, + kitchen_rules=[], + ) + steps = list(iter_steps_with_context(recipe)) + verify_ctx = next(ctx for name, _, ctx in steps if name == "verify") + assert "plan_parts" in verify_ctx, ( + "capture_list keys must appear in available_context for downstream steps" + ) + + +# D9 +def test_implementation_pipeline_captures_plan_parts_as_list() -> None: + """implementation.yaml plan step must capture plan_parts via capture_list.""" + recipe = load_recipe(builtin_recipes_dir() / "implementation.yaml") + step = recipe.steps["plan"] + assert hasattr(step, "capture_list"), "RecipeStep must have capture_list field" + assert "plan_parts" in step.capture_list, ( + "implementation plan step must capture plan_parts via capture_list" + ) + + +# IO-1: RecipeInfo dataclass accepts content kwarg; defaults to None +def test_recipe_info_has_content_field_defaulting_to_none() -> None: + """RecipeInfo.content defaults to None when not provided.""" + from autoskillit.recipe.schema import RecipeInfo + + info = RecipeInfo( + name="x", + description="y", + source=RecipeSource.BUILTIN, + path=Path("/x.yaml"), + ) + assert info.content is None + + +# IO-2: list_recipes populates content field with raw YAML text +def test_list_recipes_populates_content(tmp_path: Path) -> None: + """list_recipes() populates the content field with raw YAML text.""" + recipes_dir = tmp_path / ".autoskillit" / "recipes" + recipes_dir.mkdir(parents=True) + raw = "name: my-recipe\ndescription: test\nsteps: {}\n" + (recipes_dir / "my-recipe.yaml").write_text(raw) + result = list_recipes(tmp_path) + assert result.items, "expected at least one recipe" + item = next(r for r in result.items if r.name == "my-recipe") + assert item.content == raw + + +# IO-3: content field preserved through find_recipe_by_name +def test_find_recipe_by_name_returns_info_with_content(tmp_path: Path) -> None: + """find_recipe_by_name() returns a RecipeInfo with content populated.""" + from autoskillit.recipe.io import find_recipe_by_name + + recipes_dir = tmp_path / ".autoskillit" / "recipes" + recipes_dir.mkdir(parents=True) + raw = "name: my-recipe\ndescription: test\nsteps: {}\n" + (recipes_dir / "my-recipe.yaml").write_text(raw) + info = find_recipe_by_name("my-recipe", tmp_path) + assert info is not None + assert info.content == raw + + +class TestRecipeVersionField: + def test_parse_recipe_reads_recipe_version(self, tmp_path): + yaml_content = ( + "name: test\ndescription: d\nrecipe_version: '1.2.0'\n" + "kitchen_rules:\n - rule\nsteps:\n s1:\n tool: run_skill\n" + " message: hi\n on_success: done\n" + " done:\n action: stop\n message: Done\n" + ) + p = tmp_path / "r.yaml" + p.write_text(yaml_content) + recipe = load_recipe(p) + assert recipe.recipe_version == "1.2.0" + + def test_parse_recipe_no_recipe_version(self, tmp_path): + yaml_content = ( + "name: test\ndescription: d\nkitchen_rules:\n - rule\n" + "steps:\n s1:\n tool: run_skill\n message: hi\n" + " on_success: done\n done:\n action: stop\n message: Done\n" + ) + p = tmp_path / "r.yaml" + p.write_text(yaml_content) + recipe = load_recipe(p) + assert recipe.recipe_version is None + + def test_parse_recipe_rejects_float_recipe_version(self, tmp_path): + yaml_content = ( + "name: test\ndescription: d\nrecipe_version: 1.0\n" + "kitchen_rules:\n - rule\nsteps:\n s1:\n tool: run_skill\n" + " message: hi\n on_success: done\n" + " done:\n action: stop\n message: Done\n" + ) + p = tmp_path / "r.yaml" + p.write_text(yaml_content) + with pytest.raises(ValueError, match="recipe_version must be a quoted string"): + load_recipe(p) + + +class TestContentHash: + def test_load_recipe_sets_content_hash(self, tmp_path): + yaml_content = ( + "name: test\ndescription: d\nkitchen_rules:\n - rule\n" + "steps:\n s1:\n tool: run_skill\n message: hi\n" + " on_success: done\n done:\n action: stop\n message: Done\n" + ) + p = tmp_path / "r.yaml" + p.write_text(yaml_content) + recipe = load_recipe(p) + expected = "sha256:" + hashlib.sha256(p.read_bytes()).hexdigest() + assert recipe.content_hash == expected + + def test_list_recipes_populates_content_hash(self, tmp_path): + recipe_dir = tmp_path / ".autoskillit" / "recipes" + recipe_dir.mkdir(parents=True) + (recipe_dir / "test.yaml").write_text( + "name: test\ndescription: d\nkitchen_rules:\n - rule\n" + "steps:\n s1:\n tool: run_skill\n message: hi\n" + " on_success: done\n done:\n action: stop\n message: Done\n" + ) + result = list_recipes(tmp_path) + assert result.items[0].content_hash.startswith("sha256:") + + +# --------------------------------------------------------------------------- +# provider field tests (PROV-S1–PROV-S3) +# --------------------------------------------------------------------------- + + +# PROV-S1 +def test_recipe_step_provider_field_accepts_value() -> None: + """RecipeStep accepts provider field and stores it.""" + step = RecipeStep( + tool="run_skill", + with_args={"skill_command": "/autoskillit:make-plan inputs.task"}, + on_success="done", + provider="anthropic", + ) + assert step.provider == "anthropic" + + +# PROV-S2 +def test_recipe_step_provider_defaults_to_none() -> None: + """RecipeStep.provider defaults to None when not provided.""" + step = RecipeStep(tool="run_skill", with_args={}, on_success="done") + assert step.provider is None + + +# PROV-S3 +def test_recipe_step_provider_in_handled_fields() -> None: + """provider key is registered in _PARSE_STEP_HANDLED_FIELDS.""" + from autoskillit.recipe.io import _PARSE_STEP_HANDLED_FIELDS + + assert "provider" in _PARSE_STEP_HANDLED_FIELDS diff --git a/tests/recipe/test_issue_url_pipeline.py b/tests/recipe/test_issue_url_pipeline.py index bbbc333589..18a80efd43 100644 --- a/tests/recipe/test_issue_url_pipeline.py +++ b/tests/recipe/test_issue_url_pipeline.py @@ -2,11 +2,15 @@ from pathlib import Path +import pytest import yaml from autoskillit.recipe._api import validate_from_path from autoskillit.recipe.io import load_recipe -from tests.recipe.conftest import NO_AUTOSKILLIT_IMPORT as _NO_AUTOSKILLIT_IMPORT +from autoskillit.recipe.registry import run_semantic_rules +from tests.recipe.conftest import KNOWN_PART_B_VIOLATIONS as _KNOWN_PART_B_VIOLATIONS + +pytestmark = [pytest.mark.layer("recipe"), pytest.mark.small] RECIPES_DIR = Path(__file__).parent.parent.parent / "src" / "autoskillit" / "recipes" @@ -22,7 +26,7 @@ def test_recipe_validates_clean(self): errors = [ f for f in result.get("findings", []) - if f.get("severity") == "error" and f.get("rule") != _NO_AUTOSKILLIT_IMPORT + if f.get("severity") == "error" and f.get("rule") not in _KNOWN_PART_B_VIOLATIONS ] assert errors == [], f"Unexpected errors: {errors}" @@ -40,30 +44,31 @@ def test_no_fetch_issue_step(self): assert "fetch_issue" not in data["steps"] def test_get_issue_title_step_present(self): - """get_issue_title step must exist with correct structure.""" + """claim_and_resolve step must exist with correct structure.""" data = yaml.safe_load(_recipe_path("implementation").read_text()) - assert "get_issue_title" in data["steps"] + assert "claim_and_resolve" in data["steps"] assert "parse_issue_number" not in data["steps"] - step = data["steps"]["get_issue_title"] - assert step["tool"] == "get_issue_title" + step = data["steps"]["claim_and_resolve"] + assert step["tool"] == "claim_and_resolve_issue" assert step.get("optional") is True assert step.get("skip_when_false") == "inputs.issue_url" assert "issue_number" in step.get("capture", {}) assert "issue_title" in step.get("capture", {}) assert "issue_slug" in step.get("capture", {}) - def test_get_issue_title_between_set_merge_target_and_create_branch(self): - """get_issue_title must be positioned after set_merge_target, before create_branch.""" + def test_get_issue_title_between_capture_base_sha_and_create_branch(self): + """clone step must route to claim_and_resolve, which routes to create_and_publish.""" data = yaml.safe_load(_recipe_path("implementation").read_text()) - assert data["steps"]["set_merge_target"]["on_success"] == "get_issue_title" - assert data["steps"]["get_issue_title"]["on_success"] == "claim_issue" + assert data["steps"]["clone"]["on_success"] == "claim_and_resolve" + on_result = data["steps"]["claim_and_resolve"].get("on_result", []) + true_routes = [r["route"] for r in on_result if r.get("when", "").endswith("== true")] + assert "create_and_publish" in true_routes - def test_create_branch_uses_slug_fallback(self): - """create_branch shell uses ${SLUG:-$RUN} pattern.""" + def test_create_branch_uses_callable(self): + """create_and_publish step must use create_and_publish_branch MCP tool.""" data = yaml.safe_load(_recipe_path("implementation").read_text()) - cmd = data["steps"]["compute_branch"]["with"]["cmd"] - assert "SLUG" in cmd - assert "${SLUG:-" in cmd + step = data["steps"]["create_and_publish"] + assert step["tool"] == "create_and_publish_branch" def test_issue_url_referenced_in_downstream_skill_step(self): """plan step must reference inputs.issue_url, not issue_content.""" @@ -113,7 +118,7 @@ def test_recipe_validates_clean(self): errors = [ f for f in result.get("findings", []) - if f.get("severity") == "error" and f.get("rule") != _NO_AUTOSKILLIT_IMPORT + if f.get("severity") == "error" and f.get("rule") not in _KNOWN_PART_B_VIOLATIONS ] assert errors == [], f"Unexpected errors: {errors}" @@ -130,12 +135,12 @@ def test_no_fetch_issue_step(self): assert "fetch_issue" not in data["steps"] def test_get_issue_title_step_present(self): - """get_issue_title step must exist with correct structure.""" + """claim_and_resolve step must exist with correct structure.""" data = yaml.safe_load(_recipe_path("remediation").read_text()) - assert "get_issue_title" in data["steps"] + assert "claim_and_resolve" in data["steps"] assert "parse_issue_number" not in data["steps"] - step = data["steps"]["get_issue_title"] - assert step["tool"] == "get_issue_title" + step = data["steps"]["claim_and_resolve"] + assert step["tool"] == "claim_and_resolve_issue" assert step.get("optional") is True assert step.get("skip_when_false") == "inputs.issue_url" assert "issue_number" in step.get("capture", {}) @@ -143,17 +148,18 @@ def test_get_issue_title_step_present(self): assert "issue_slug" in step.get("capture", {}) def test_get_issue_title_between_set_merge_target_and_create_branch(self): - """get_issue_title must be positioned after set_merge_target, before create_branch.""" + """clone step must route to claim_and_resolve (set_merge_target is now part of clone).""" data = yaml.safe_load(_recipe_path("remediation").read_text()) - assert data["steps"]["set_merge_target"]["on_success"] == "get_issue_title" - assert data["steps"]["get_issue_title"]["on_success"] == "claim_issue" + assert data["steps"]["clone"]["on_success"] == "claim_and_resolve" + on_result = data["steps"]["claim_and_resolve"].get("on_result", []) + true_routes = [r["route"] for r in on_result if r.get("when", "").endswith("== true")] + assert "create_and_publish" in true_routes - def test_create_branch_uses_slug_fallback(self): - """create_branch shell uses ${SLUG:-$RUN} pattern.""" + def test_create_branch_uses_callable(self): + """create_and_publish step must use create_and_publish_branch MCP tool.""" data = yaml.safe_load(_recipe_path("remediation").read_text()) - cmd = data["steps"]["compute_branch"]["with"]["cmd"] - assert "SLUG" in cmd - assert "${SLUG:-" in cmd + step = data["steps"]["create_and_publish"] + assert step["tool"] == "create_and_publish_branch" def test_issue_url_referenced_in_downstream_skill_step(self): """investigate step must reference inputs.issue_url, not issue_content.""" @@ -202,33 +208,32 @@ def test_recipe_validates_clean(self): errors = [ f for f in result.get("findings", []) - if f.get("severity") == "error" and f.get("rule") != _NO_AUTOSKILLIT_IMPORT + if f.get("severity") == "error" and f.get("rule") not in _KNOWN_PART_B_VIOLATIONS ] assert errors == [], f"Unexpected errors: {errors}" def test_fetch_issue_step_replaced(self): data = yaml.safe_load(_recipe_path("implementation-groups").read_text()) assert "fetch_issue" not in data["steps"] - assert "get_issue_title" in data["steps"] + assert "claim_and_resolve" in data["steps"] def test_get_issue_title_captures_three_fields(self): data = yaml.safe_load(_recipe_path("implementation-groups").read_text()) - step = data["steps"]["get_issue_title"] + step = data["steps"]["claim_and_resolve"] assert "issue_number" in step["capture"] assert "issue_title" in step["capture"] assert "issue_slug" in step["capture"] def test_get_issue_title_skips_when_no_url(self): data = yaml.safe_load(_recipe_path("implementation-groups").read_text()) - step = data["steps"]["get_issue_title"] + step = data["steps"]["claim_and_resolve"] assert step.get("skip_when_false") == "inputs.issue_url" assert step.get("optional") is True - def test_create_branch_uses_slug_fallback(self): + def test_create_branch_uses_callable(self): data = yaml.safe_load(_recipe_path("implementation-groups").read_text()) - cmd = data["steps"]["compute_branch"]["with"]["cmd"] - assert "SLUG" in cmd - assert "${SLUG:-" in cmd + step = data["steps"]["create_and_publish"] + assert step["tool"] == "create_and_publish_branch" def test_no_issue_content_capture(self): """issue_content must not be captured anywhere in the recipe.""" @@ -281,23 +286,25 @@ def test_split_lists_are_exhaustive(self): def test_claim_issue_step_present(self): for name in self.RECIPES: data = yaml.safe_load(_recipe_path(name).read_text()) - assert "claim_issue" in data["steps"], f"{name}: missing claim_issue step" + assert "claim_and_resolve" in data["steps"], ( + f"{name}: missing claim_and_resolve step (claim_and_resolve_issue tool)" + ) def test_get_issue_title_routes_to_claim_issue(self): for name in self.RECIPES: data = yaml.safe_load(_recipe_path(name).read_text()) - assert data["steps"]["get_issue_title"]["on_success"] == "claim_issue", ( - f"{name}: get_issue_title.on_success should be claim_issue" + assert data["steps"]["clone"]["on_success"] == "claim_and_resolve", ( + f"{name}: clone.on_success should be claim_and_resolve" ) def test_claim_issue_routes_to_create_branch_on_true(self): for name in self.RECIPES: data = yaml.safe_load(_recipe_path(name).read_text()) - step = data["steps"]["claim_issue"] + step = data["steps"]["claim_and_resolve"] routes = step.get("on_result", []) true_routes = [r["route"] for r in routes if r.get("when", "").endswith("== true")] - assert "compute_branch" in true_routes, ( - f"{name}: claim_issue should route to compute_branch when claimed==true" + assert "create_and_publish" in true_routes, ( + f"{name}: claim_and_resolve should route to create_and_publish when claimed==true" ) def test_release_issue_steps_present(self): @@ -319,8 +326,8 @@ def test_release_issue_success_routes_to_register_clone_success(self): for name in self.RECIPES_WITH_RELEASE_SUCCESS_STEP: data = yaml.safe_load(_recipe_path(name).read_text()) step = data["steps"]["release_issue_success"] - assert step["on_success"] == "register_clone_success", ( - f"{name}: release_issue_success.on_success should be register_clone_success" + assert step["on_success"] == "patch_token_summary", ( + f"{name}: release_issue_success.on_success should be patch_token_summary" ) def test_release_issue_failure_routes_to_register_clone_failure(self): @@ -334,7 +341,7 @@ def test_release_issue_failure_routes_to_register_clone_failure(self): f"{name}: release_issue_failure.on_failure should be register_clone_failure" ) - def test_ci_watch_on_success_routing(self): + def test_ci_watch_on_result_routing(self): expected = { **{name: "check_repo_merge_state" for name in self.RECIPES_WITHOUT_RELEASE_SUCCESS}, **{name: "release_issue_success" for name in self.RECIPES_WITH_RELEASE_SUCCESS}, @@ -344,12 +351,22 @@ def test_ci_watch_on_success_routing(self): ) for name, expected_route in expected.items(): data = yaml.safe_load(_recipe_path(name).read_text()) - assert data["steps"]["ci_watch"]["on_success"] == expected_route, ( - f"{name}: ci_watch.on_success should be {expected_route!r}" - ) + ci_step = data["steps"]["ci_watch"] + on_result = ci_step.get("on_result") + if on_result: + success_routes = [ + r["route"] for r in on_result if r.get("when", "").endswith("== 'success'") + ] + assert expected_route in success_routes, ( + f"{name}: ci_watch on_result success should route to {expected_route!r}" + ) + else: + assert ci_step.get("on_success") == expected_route, ( + f"{name}: ci_watch.on_success should be {expected_route!r}" + ) def test_claim_issue_with_args_contains_issue_url(self): - """CC-F1: claim_issue.with_args must contain issue_url after parsing. + """CC-F1: claim_and_resolve.with_args must contain issue_url after parsing. Fails when the YAML uses `with_args:` key (bug) because _parse_step reads data.get("with", {}) and returns {} for that key. @@ -357,9 +374,9 @@ def test_claim_issue_with_args_contains_issue_url(self): """ for name in self.RECIPES: recipe = load_recipe(_recipe_path(name)) - step = recipe.steps["claim_issue"] + step = recipe.steps["claim_and_resolve"] assert "issue_url" in step.with_args, ( - f"{name}: claim_issue.with_args missing issue_url — " + f"{name}: claim_and_resolve.with_args missing issue_url — " f"YAML likely uses 'with_args:' instead of 'with:'" ) @@ -382,11 +399,13 @@ def test_release_issue_failure_with_args_contains_issue_url(self): ) def test_claim_issue_step_passes_allow_reentry_from_upfront_claimed(self): - """claim_issue with: block includes allow_reentry mapped from inputs.upfront_claimed.""" + """claim_and_resolve with: block includes allow_reentry from inputs.upfront_claimed.""" for name in self.RECIPES: data = yaml.safe_load(_recipe_path(name).read_text()) - claim_with = data["steps"]["claim_issue"].get("with", {}) - assert "allow_reentry" in claim_with, f"{name}: claim_issue.with missing allow_reentry" + claim_with = data["steps"]["claim_and_resolve"].get("with", {}) + assert "allow_reentry" in claim_with, ( + f"{name}: claim_and_resolve.with missing allow_reentry" + ) assert claim_with["allow_reentry"] == "${{ inputs.upfront_claimed }}", ( f"{name}: allow_reentry must be exactly '${{{{ inputs.upfront_claimed }}}}'" ) @@ -402,12 +421,31 @@ def test_upfront_claimed_ingredient_present_with_default_false(self): ) def test_claim_issue_routes_escalate_stop_when_not_claimed(self): - """claim_issue fallthrough routes to escalate_stop (preserves single-issue defense).""" + """claim_and_resolve fallthrough routes to escalate_stop.""" for name in self.RECIPES: data = yaml.safe_load(_recipe_path(name).read_text()) - on_result = data["steps"]["claim_issue"].get("on_result", []) + on_result = data["steps"]["claim_and_resolve"].get("on_result", []) # The fallthrough route (no 'when' key) must route to escalate_stop fallthrough_routes = [r["route"] for r in on_result if "when" not in r] assert "escalate_stop" in fallthrough_routes, ( - f"{name}: claim_issue must have escalate_stop as fallthrough on_result route" + f"{name}: claim_and_resolve must have escalate_stop as fallthrough on_result route" ) + + +def test_release_issue_failure_steps_include_fail_label(): + """All release_issue_failure steps must pass fail_label for atomic failure marking.""" + for recipe_name in ("remediation", "implementation", "implementation-groups"): + recipe = load_recipe(_recipe_path(recipe_name)) + step = recipe.steps["release_issue_failure"] + assert "fail_label" in step.with_args, ( + f"{recipe_name}: release_issue_failure missing fail_label" + ) + + +def test_bundled_recipes_pass_release_issue_disposition_rule(): + """All bundled recipes pass the release-issue-requires-disposition rule.""" + for recipe_name in ("remediation", "implementation", "implementation-groups"): + recipe = load_recipe(_recipe_path(recipe_name)) + findings = run_semantic_rules(recipe) + hits = [f for f in findings if f.rule == "release-issue-requires-disposition"] + assert not hits, f"{recipe_name}: {hits}" diff --git a/tests/recipe/test_loader.py b/tests/recipe/test_loader.py index 94c85c223f..3fa314d0af 100644 --- a/tests/recipe/test_loader.py +++ b/tests/recipe/test_loader.py @@ -11,6 +11,8 @@ parse_recipe_metadata, ) +pytestmark = [pytest.mark.layer("recipe"), pytest.mark.medium] + class TestParseRecipeMetadata: def test_single_document(self, tmp_path: Path) -> None: @@ -148,24 +150,3 @@ def test_version_key_not_hardcoded_in_recipe_loader(): assert literals == [], ( "recipe_loader must import AUTOSKILLIT_VERSION_KEY, not hard-code the string" ) - - -# --------------------------------------------------------------------------- -# RL-VER1: all bundled recipes must declare autoskillit_version -# --------------------------------------------------------------------------- -def test_all_bundled_recipes_have_autoskillit_version() -> None: - """All bundled recipes must declare autoskillit_version to prevent spurious auto-migration.""" - import yaml - - from autoskillit.recipe.io import builtin_recipes_dir - - missing = [] - for recipe_file in builtin_recipes_dir().glob("*.yaml"): - data = yaml.safe_load(recipe_file.read_text()) - if not data or "autoskillit_version" not in data: - missing.append(recipe_file.name) - assert not missing, ( - "Bundled recipes missing 'autoskillit_version' field " - "(will trigger spurious auto-migration on every load_recipe call):\n" - + "\n".join(sorted(missing)) - ) diff --git a/tests/recipe/test_make_campaign_output_schema.py b/tests/recipe/test_make_campaign_output_schema.py new file mode 100644 index 0000000000..e19732e7fc --- /dev/null +++ b/tests/recipe/test_make_campaign_output_schema.py @@ -0,0 +1,161 @@ +"""Tests that example output from the make-campaign skill passes campaign validation. + +Validates the validation paths that the skill exercises: structural, semantic +(depends-on-acyclic, dispatch-ingredients-keys-in-target-schema), and that a +minimal valid campaign YAML passes all rules. +""" + +from __future__ import annotations + +from typing import Any + +import pytest + +import autoskillit.recipe # noqa: F401 -- triggers rule registration +from autoskillit.core import Severity +from autoskillit.recipe._analysis import make_validation_context +from autoskillit.recipe.registry import RuleFinding, run_semantic_rules +from autoskillit.recipe.schema import CampaignDispatch, Recipe, RecipeKind + +pytestmark = [pytest.mark.layer("recipe"), pytest.mark.small] + + +def _dispatch(name: str, recipe: str, task: str, **kwargs: Any) -> CampaignDispatch: + return CampaignDispatch(name=name, recipe=recipe, task=task, **kwargs) + + +def _minimal_campaign(**overrides: Any) -> Recipe: + defaults: dict[str, Any] = { + "name": "my-feature-campaign", + "description": "Campaign to implement and verify a feature", + "kind": RecipeKind.CAMPAIGN, + "dispatches": [ + _dispatch( + name="phase-1-implement", + recipe="implementation", + task="Implement the core feature", + ingredients={"task": "Implement the core feature"}, + depends_on=[], + ), + _dispatch( + name="phase-2-verify", + recipe="implementation", + task="Add tests for the core feature", + ingredients={"task": "Add tests for the core feature"}, + depends_on=["phase-1-implement"], + ), + ], + "requires_recipe_packs": ["implementation-family"], + "continue_on_failure": False, + "kitchen_rules": ["NEVER modify files outside the project directory"], + } + defaults.update(overrides) + return Recipe(**defaults) + + +def _findings_for_rule(recipe: Recipe, rule: str, **ctx_kwargs: Any) -> list[RuleFinding]: + ctx = make_validation_context(recipe, **ctx_kwargs) + return [f for f in run_semantic_rules(ctx) if f.rule == rule] + + +def _error_findings(recipe: Recipe, **ctx_kwargs: Any) -> list[RuleFinding]: + """Return only ERROR-severity semantic findings.""" + ctx = make_validation_context(recipe, **ctx_kwargs) + return [f for f in run_semantic_rules(ctx) if f.severity == Severity.ERROR] + + +def test_minimal_campaign_yaml_validates() -> None: + """Minimal campaign YAML matching skill output format passes structural validation.""" + campaign = _minimal_campaign() + # Provide the recipe name as available so dispatch-recipe-exists doesn't fire + errors = _error_findings(campaign, available_recipes=frozenset({"implementation"})) + # Only remaining potential error would be dispatch-ingredients-keys-in-target-schema, + # which is skipped when project_dir is None (no recipe file on disk to inspect). + assert not errors, ( + f"Minimal campaign YAML must pass all semantic rules with no ERROR findings.\n" + f"Got errors: {[(f.rule, f.message) for f in errors]}" + ) + + +def test_campaign_with_cycle_fails_semantic_validation() -> None: + """Cyclic depends_on detected by semantic rules (depends-on-acyclic).""" + campaign = _minimal_campaign( + dispatches=[ + _dispatch( + name="phase-1", + recipe="implementation", + task="Step one", + depends_on=["phase-2"], + ), + _dispatch( + name="phase-2", + recipe="implementation", + task="Step two", + depends_on=["phase-1"], + ), + ] + ) + findings = _findings_for_rule(campaign, "depends-on-acyclic") + assert findings, "Cyclic depends_on must trigger depends-on-acyclic rule" + assert findings[0].severity == Severity.ERROR + + +def test_campaign_invalid_ingredient_key_detected(tmp_path) -> None: + """Invalid ingredient key caught by semantic rule when available_recipes provided.""" + import yaml + + # Write a minimal target recipe to disk so _load_dispatch_target can inspect it + # find_recipe_by_name looks in project_dir / ".autoskillit" / "recipes" + recipe_dir = tmp_path / ".autoskillit" / "recipes" + recipe_dir.mkdir(parents=True) + target_recipe = { + "name": "target-recipe", + "description": "A simple recipe", + "autoskillit_version": "0.9.0", + "ingredients": { + "task": { + "description": "The task to perform", + "required": True, + }, + }, + "steps": { + "run": { + "tool": "run_cmd", + "with": {"command": "echo done"}, + "on_success": "stop", + "on_failure": "stop", + }, + "stop": {"action": "stop", "message": "done"}, + }, + "kitchen_rules": ["NEVER"], + } + (recipe_dir / "target-recipe.yaml").write_text( + yaml.dump(target_recipe, default_flow_style=False), encoding="utf-8" + ) + + campaign = _minimal_campaign( + dispatches=[ + _dispatch( + name="phase-1", + recipe="target-recipe", + task="Do the thing", + ingredients={ + "task": "Do it", + "nonexistent_key": "some-value", # invalid key + }, + ), + ], + requires_recipe_packs=["implementation-family"], + ) + + findings = _findings_for_rule( + campaign, + "dispatch-ingredients-keys-in-target-schema", + available_recipes=frozenset({"target-recipe"}), + project_dir=tmp_path, + ) + assert findings, ( + "Invalid ingredient key must trigger dispatch-ingredients-keys-in-target-schema rule" + ) + assert findings[0].severity == Severity.ERROR + assert "nonexistent_key" in findings[0].message diff --git a/tests/recipe/test_merge_prs.py b/tests/recipe/test_merge_prs.py index 8e21a02901..799a004e4e 100644 --- a/tests/recipe/test_merge_prs.py +++ b/tests/recipe/test_merge_prs.py @@ -6,69 +6,61 @@ from autoskillit.recipe.io import builtin_recipes_dir, iter_steps_with_context, load_recipe +pytestmark = [pytest.mark.layer("recipe"), pytest.mark.small] + @pytest.fixture(scope="module") def recipe(): return load_recipe(builtin_recipes_dir() / "merge-prs.yaml") -def test_pmp_check_impl_plans_step_exists(recipe) -> None: - """check_impl_plans step must exist in the recipe.""" - assert "check_impl_plans" in recipe.steps, ( - "check_impl_plans step is missing — it gates audit_impl when no " - "implementation plans were generated" - ) - - -def test_pmp_collect_artifacts_routes_to_check_impl_plans(recipe) -> None: - """collect_artifacts.on_success must route to check_impl_plans, not audit_impl.""" - step = recipe.steps["collect_artifacts"] - assert step.on_success == "check_impl_plans", ( - "collect_artifacts.on_success must route to check_impl_plans, not audit_impl — " - "the check step decides whether audit is meaningful" +def test_pmp_collect_and_check_impl_plans_step_exists(recipe) -> None: + """collect_and_check_impl_plans step must exist in the recipe.""" + assert "collect_and_check_impl_plans" in recipe.steps, ( + "collect_and_check_impl_plans step is missing — it copies plan artifacts and gates " + "audit_impl when no implementation plans were generated" ) -def test_pmp_collect_artifacts_failure_routes_to_check_impl_plans(recipe) -> None: - """collect_artifacts.on_failure must also route to check_impl_plans.""" - step = recipe.steps["collect_artifacts"] - assert step.on_failure == "check_impl_plans", ( - "collect_artifacts.on_failure must route to check_impl_plans " - "so the gate runs even when artifact copying fails" - ) +def test_pmp_collect_and_check_impl_plans_is_run_cmd(recipe) -> None: + """collect_and_check_impl_plans step must use the run_cmd tool.""" + step = recipe.steps["collect_and_check_impl_plans"] + assert step.tool == "run_cmd" -def test_pmp_check_impl_plans_is_run_cmd(recipe) -> None: - """check_impl_plans step must use the run_cmd tool.""" - step = recipe.steps["check_impl_plans"] - assert step.tool == "run_cmd" +def test_pmp_collect_and_check_impl_plans_cmd_contains_copy_and_count(recipe) -> None: + """collect_and_check_impl_plans cmd must contain both cp and wc -l operations.""" + step = recipe.steps["collect_and_check_impl_plans"] + cmd = step.with_args.get("cmd", "") + assert "cp" in cmd, "cmd must contain a cp operation to copy plan files" + assert "wc -l" in cmd, "cmd must contain wc -l to count implementation plans" def test_pmp_check_impl_plans_excludes_pr_analysis_plan(recipe) -> None: - """check_impl_plans cmd must exclude pr_analysis_plan_*.md from its count. + """collect_and_check_impl_plans cmd must exclude pr_analysis_plan_*.md from its count. pr_analysis_plan_*.md is always written by analyze-prs and is not an implementation plan — including it would cause audit_impl to always run. """ - step = recipe.steps["check_impl_plans"] + step = recipe.steps["collect_and_check_impl_plans"] cmd = step.with_args.get("cmd", "") assert "pr_analysis_plan" in cmd, ( - "check_impl_plans must exclude pr_analysis_plan_*.md from its count — " + "collect_and_check_impl_plans must exclude pr_analysis_plan_*.md from its count — " "that file is always present and is not an implementation plan" ) def test_pmp_check_impl_plans_routes_to_open_integration_pr_on_empty(recipe) -> None: - """check_impl_plans routes to compute_domain_partitions when no impl plans exist. + """collect_and_check_impl_plans routes to compute_domain_partitions when no impl plans exist. compute_domain_partitions is the pre-staging step before open_integration_pr. """ - step = recipe.steps["check_impl_plans"] - assert step.on_result is not None, "check_impl_plans must use on_result routing" + step = recipe.steps["collect_and_check_impl_plans"] + assert step.on_result is not None, "collect_and_check_impl_plans must use on_result routing" conds = step.on_result.conditions routes = {c.route for c in conds} assert "compute_domain_partitions" in routes, ( - "check_impl_plans must route to compute_domain_partitions when count is 0" + "collect_and_check_impl_plans must route to compute_domain_partitions when count is 0" ) zero_conds = [c for c in conds if c.when is not None and "0" in (c.when or "")] assert any(c.route == "compute_domain_partitions" for c in zero_conds), ( @@ -77,14 +69,17 @@ def test_pmp_check_impl_plans_routes_to_open_integration_pr_on_empty(recipe) -> def test_pmp_check_impl_plans_has_fallthrough_to_audit_impl(recipe) -> None: - """check_impl_plans fallthrough (when=None) must go to audit_impl.""" - step = recipe.steps["check_impl_plans"] + """collect_and_check_impl_plans fallthrough (when=None) must go to audit_impl.""" + step = recipe.steps["collect_and_check_impl_plans"] assert step.on_result is not None conds = step.on_result.conditions fallthrough = [c for c in conds if c.when is None] - assert len(fallthrough) == 1, "check_impl_plans must have exactly one fallthrough condition" + assert len(fallthrough) == 1, ( + "collect_and_check_impl_plans must have exactly one fallthrough condition" + ) assert fallthrough[0].route == "audit_impl", ( - "check_impl_plans fallthrough must route to audit_impl when implementation plans exist" + "collect_and_check_impl_plans fallthrough must route to audit_impl when " + "implementation plans exist" ) @@ -172,11 +167,11 @@ def test_pmp_open_integration_pr_captures_pr_url(recipe) -> None: def test_pmp_open_integration_pr_passes_four_args(recipe) -> None: - """skill_command must supply integration_branch, base_branch, pr_order_file, verdict.""" + """skill_command must supply batch_branch, base_branch, pr_order_file, verdict.""" step = recipe.steps["open_integration_pr"] cmd = step.with_args.get("skill_command", "") for arg in [ - "context.integration_branch", + "context.batch_branch", "inputs.base_branch", "context.pr_order_file", "context.verdict", @@ -255,16 +250,20 @@ def test_pmp_confirm_create_integration_routes_to_escalate_on_failure(recipe) -> def test_pmp_has_create_persistent_integration_step(recipe) -> None: - """create_persistent_integration must exist and use run_cmd.""" + """create_persistent_integration must exist and use run_python callable.""" assert "create_persistent_integration" in recipe.steps - assert recipe.steps["create_persistent_integration"].tool == "run_cmd" + step = recipe.steps["create_persistent_integration"] + assert step.tool == "run_python" + assert ( + step.with_args["callable"] == "autoskillit.recipe._cmd_rpc.create_persistent_integration" + ) -def test_pmp_create_persistent_integration_auto_detects_default_branch(recipe) -> None: - """create_persistent_integration cmd must auto-detect the repo's default branch.""" - cmd = recipe.steps["create_persistent_integration"].with_args["cmd"] - assert "symbolic-ref" in cmd - assert "upstream_branch" not in cmd +def test_pmp_create_persistent_integration_passes_required_args(recipe) -> None: + """create_persistent_integration must pass work_dir and base_branch.""" + args = recipe.steps["create_persistent_integration"].with_args + assert "work_dir" in args + assert "base_branch" in args def test_pmp_create_persistent_integration_routes_to_analyze_prs(recipe) -> None: @@ -352,16 +351,18 @@ def test_ci_watch_pr_exists_with_correct_tool(recipe) -> None: assert step.tool == "wait_for_ci" -def test_ci_watch_pr_uses_integration_branch(recipe) -> None: - """ci_watch_pr must use context.integration_branch as the branch parameter.""" +def test_ci_watch_pr_uses_batch_branch(recipe) -> None: + """ci_watch_pr must use context.batch_branch as the branch parameter.""" step = recipe.steps["ci_watch_pr"] - assert "context.integration_branch" in step.with_args["branch"] + assert "context.batch_branch" in step.with_args["branch"] def test_ci_watch_pr_routing(recipe) -> None: - """ci_watch_pr on_success -> register_clone_success; on_failure -> diagnose_ci.""" + """ci_watch_pr on_result success -> register_clone_success; on_failure -> diagnose_ci.""" step = recipe.steps["ci_watch_pr"] - assert step.on_success == "register_clone_success" + assert step.on_result is not None, "ci_watch_pr must use on_result predicate routing" + result_routes = {c.route for c in step.on_result.conditions} + assert "patch_token_summary" in result_routes assert step.on_failure == "diagnose_ci" @@ -387,10 +388,13 @@ def test_pmp_open_integration_pr_routes_to_wait_for_mergeability(recipe) -> None def test_pmp_has_wait_for_review_pr_mergeability_step(recipe) -> None: - """B2: wait_for_review_pr_mergeability step must exist and use run_cmd tool.""" + """B2: wait_for_review_pr_mergeability step must exist and use run_python callable.""" assert "wait_for_review_pr_mergeability" in recipe.steps step = recipe.steps["wait_for_review_pr_mergeability"] - assert step.tool == "run_cmd" + assert step.tool == "run_python" + assert ( + step.with_args["callable"] == "autoskillit.recipe._cmd_rpc.wait_for_review_pr_mergeability" + ) def test_pmp_wait_for_mergeability_captures_review_pr_number(recipe) -> None: @@ -441,44 +445,40 @@ def test_pmp_has_resolve_integration_conflicts_step(recipe) -> None: def test_pmp_resolve_integration_conflicts_routes_to_force_push(recipe) -> None: - """B9: resolve_integration_conflicts must route to force_push_after_rebase.""" + """B9: resolve_integration_conflicts must route to force_push_and_wait_mergeability.""" step = recipe.steps["resolve_integration_conflicts"] # Step uses on_result conditions; the default (no-when) bare route must route to force_push assert step.on_result is not None conditions = step.on_result.conditions default_routes = [c for c in conditions if c.when is None] - assert any(c.route == "force_push_after_rebase" for c in default_routes) - + assert any(c.route == "force_push_and_wait_mergeability" for c in default_routes) -def test_pmp_has_force_push_after_rebase_step(recipe) -> None: - """B10: force_push_after_rebase step must exist with run_cmd tool and --force-with-lease.""" - assert "force_push_after_rebase" in recipe.steps - step = recipe.steps["force_push_after_rebase"] - assert step.tool == "run_cmd" - assert "--force-with-lease" in step.with_args.get("cmd", "") - -def test_pmp_force_push_after_rebase_routes_to_wait_for_post_rebase_mergeability(recipe) -> None: - """B23: force_push_after_rebase.on_success must route to wait_for_post_rebase_mergeability.""" - step = recipe.steps["force_push_after_rebase"] - assert step.on_success == "wait_for_post_rebase_mergeability" - - -def test_pmp_has_wait_for_post_rebase_mergeability_step(recipe) -> None: - """B24: wait_for_post_rebase_mergeability step must exist and use run_cmd tool.""" - assert "wait_for_post_rebase_mergeability" in recipe.steps - step = recipe.steps["wait_for_post_rebase_mergeability"] - assert step.tool == "run_cmd" +def test_pmp_has_force_push_and_wait_mergeability_step(recipe) -> None: + """B10: force_push_and_wait_mergeability must exist with run_python callable.""" + assert "force_push_and_wait_mergeability" in recipe.steps + step = recipe.steps["force_push_and_wait_mergeability"] + assert step.tool == "run_python" + assert ( + step.with_args["callable"] + == "autoskillit.recipe._cmd_rpc.force_push_and_wait_mergeability" + ) -def test_pmp_wait_for_post_rebase_mergeability_routes_to_check_post_rebase( +def test_pmp_force_push_and_wait_mergeability_routes_to_check_post_rebase( recipe, ) -> None: - """B25: wait_for_post_rebase_mergeability.on_success must route to check_mergeability_post_rebase.""" # noqa: E501 - step = recipe.steps["wait_for_post_rebase_mergeability"] + """B25: force_push_and_wait_mergeability.on_success must route to check_mergeability_post_rebase.""" # noqa: E501 + step = recipe.steps["force_push_and_wait_mergeability"] assert step.on_success == "check_mergeability_post_rebase" +def test_pmp_force_push_and_wait_mergeability_on_failure(recipe) -> None: + """force_push_and_wait_mergeability.on_failure must route to register_clone_failure.""" + step = recipe.steps["force_push_and_wait_mergeability"] + assert step.on_failure == "register_clone_failure" + + def test_pmp_has_check_mergeability_post_rebase_step(recipe) -> None: """B11: check_mergeability_post_rebase step must exist with check_pr_mergeable tool.""" assert "check_mergeability_post_rebase" in recipe.steps @@ -503,10 +503,10 @@ def test_pmp_has_review_pr_integration_step(recipe) -> None: assert "review-pr" in step.with_args.get("skill_command", "") -def test_pmp_review_pr_integration_uses_integration_branch(recipe) -> None: - """B14: review_pr_integration skill_command must reference context.integration_branch.""" +def test_pmp_review_pr_integration_uses_batch_branch(recipe) -> None: + """B14: review_pr_integration skill_command must reference context.batch_branch.""" step = recipe.steps["review_pr_integration"] - assert "context.integration_branch" in step.with_args.get("skill_command", "") + assert "context.batch_branch" in step.with_args.get("skill_command", "") def test_pmp_review_pr_integration_routes_changes_requested_to_resolve_review(recipe) -> None: @@ -564,10 +564,10 @@ def test_pmp_has_re_push_review_integration_step(recipe) -> None: assert step.tool == "push_to_remote" -def test_pmp_re_push_review_integration_uses_integration_branch(recipe) -> None: - """B21: re_push_review_integration must pass context.integration_branch as branch arg.""" +def test_pmp_re_push_review_integration_uses_batch_branch(recipe) -> None: + """B21: re_push_review_integration must pass context.batch_branch as branch arg.""" step = recipe.steps["re_push_review_integration"] - assert "context.integration_branch" in step.with_args.get("branch", "") + assert "context.batch_branch" in step.with_args.get("branch", "") def test_pmp_re_push_review_integration_routes_to_ci_watch(recipe) -> None: @@ -612,3 +612,108 @@ def test_pmp_push_ejected_fix_has_force_true(recipe) -> None: "push_ejected_fix must include force='true' — it follows a resolve-merge-conflicts " "step that rewrites commit SHAs, so a non-fast-forward force push is required" ) + + +def test_annotate_pr_diff_passes_local_review_rounds_explicitly(recipe) -> None: + """annotate_pr_diff step must include local_review_rounds in its with_args. + + When local_review_rounds is omitted, the Python default '' is passed to + annotate_pr_diff, forcing review_mode='github' regardless of user config. + Making it explicit documents the intent and respects the recipe's own default. + """ + step = recipe.steps["annotate_pr_diff"] + assert "local_review_rounds" in step.with_args, ( + "annotate_pr_diff step must explicitly pass local_review_rounds to remove " + "the dependency on the Python default '' which forces github review mode" + ) + + +def test_merge_pr_captures_merged(recipe) -> None: + """merge_pr step captures result.merged.""" + step = recipe.steps["merge_pr"] + assert step.capture is not None + assert "merged" in step.capture + assert "${{ result.merged }}" in step.capture["merged"] + + +def test_merge_pr_routes_merged_false_to_failure(recipe) -> None: + """merge_pr on_result routes merged=false to register_clone_failure.""" + step = recipe.steps["merge_pr"] + assert step.on_result is not None + conditions = step.on_result.conditions + merged_false_conds = [ + c for c in conditions if c.when and "merged" in c.when and "false" in c.when + ] + assert merged_false_conds, "merge_pr must have a when clause for merged==false" + assert merged_false_conds[0].route == "register_clone_failure", ( + "merged=false route must go to register_clone_failure, not " + f"{merged_false_conds[0].route!r}" + ) + + +def test_wait_for_conflict_ci_has_auto_trigger(recipe) -> None: + """wait_for_conflict_ci has auto_trigger: true.""" + step = recipe.steps["wait_for_conflict_ci"] + assert step.with_args.get("auto_trigger") == "true" + + +def test_wait_for_conflict_ci_timed_out_routes_to_guard(recipe) -> None: + """wait_for_conflict_ci timed_out route goes to check_conflict_ci_loop, not self.""" + step = recipe.steps["wait_for_conflict_ci"] + assert step.on_result is not None + timed_out_conds = [c for c in step.on_result.conditions if c.when and "timed_out" in c.when] + assert timed_out_conds, "wait_for_conflict_ci must have a timed_out condition" + assert timed_out_conds[0].route == "check_conflict_ci_loop", ( + "wait_for_conflict_ci timed_out must route to check_conflict_ci_loop, not self" + ) + + +def test_check_conflict_ci_loop_exists_with_correct_pattern(recipe) -> None: + """check_conflict_ci_loop step uses check_loop_iteration with max_iterations: 2.""" + assert "check_conflict_ci_loop" in recipe.steps + step = recipe.steps["check_conflict_ci_loop"] + assert step.tool == "run_python" + assert "check_loop_iteration" in step.with_args.get("callable", "") + assert step.with_args.get("max_iterations") == "2" + assert step.on_result is not None + max_exceeded_conds = [ + c + for c in step.on_result.conditions + if c.when and "max_exceeded" in c.when and "true" in c.when + ] + assert max_exceeded_conds, "check_conflict_ci_loop must route max_exceeded==true" + assert max_exceeded_conds[0].route == "register_clone_failure" + + +def test_ci_watch_pr_has_auto_trigger(recipe) -> None: + """ci_watch_pr has auto_trigger: true.""" + step = recipe.steps["ci_watch_pr"] + assert step.with_args.get("auto_trigger") == "true" + + +def test_ci_watch_pr_timed_out_routes_to_guard(recipe) -> None: + """ci_watch_pr timed_out route goes to check_ci_watch_pr_loop, not self.""" + step = recipe.steps["ci_watch_pr"] + assert step.on_result is not None + timed_out_conds = [c for c in step.on_result.conditions if c.when and "timed_out" in c.when] + assert timed_out_conds, "ci_watch_pr must have a timed_out condition" + assert timed_out_conds[0].route == "check_ci_watch_pr_loop", ( + "ci_watch_pr timed_out must route to check_ci_watch_pr_loop, not self" + ) + + +def test_check_ci_watch_pr_loop_exists_with_correct_pattern(recipe) -> None: + """check_ci_watch_pr_loop step uses check_loop_iteration with max_iterations: 2.""" + assert "check_ci_watch_pr_loop" in recipe.steps + step = recipe.steps["check_ci_watch_pr_loop"] + assert step.tool == "run_python" + assert "check_loop_iteration" in step.with_args.get("callable", "") + assert step.with_args.get("max_iterations") == "2" + assert step.on_result is not None + max_exceeded_conds = [ + c + for c in step.on_result.conditions + if c.when and "max_exceeded" in c.when and "true" in c.when + ] + assert max_exceeded_conds, "check_ci_watch_pr_loop must route max_exceeded==true" + assert max_exceeded_conds[0].route == "register_clone_failure" diff --git a/tests/recipe/test_merge_prs_queue.py b/tests/recipe/test_merge_prs_queue.py deleted file mode 100644 index f19fe11ea7..0000000000 --- a/tests/recipe/test_merge_prs_queue.py +++ /dev/null @@ -1,1120 +0,0 @@ -"""Queue mode structural assertions for the bundled recipes. - -Tests that the queue-aware steps added in Part B are present and correctly -wired in merge-prs.yaml, implementation.yaml, and remediation.yaml. -""" - -from __future__ import annotations - -import pytest - -from autoskillit.core import PRState -from autoskillit.recipe.io import builtin_recipes_dir, load_recipe -from autoskillit.recipe.validator import validate_recipe - -# --------------------------------------------------------------------------- -# Fixtures -# --------------------------------------------------------------------------- - - -@pytest.fixture(scope="module") -def pmp_recipe(): - return load_recipe(builtin_recipes_dir() / "merge-prs.yaml") - - -@pytest.fixture(scope="module") -def impl_recipe(): - return load_recipe(builtin_recipes_dir() / "implementation.yaml") - - -@pytest.fixture(scope="module") -def remed_recipe(): - return load_recipe(builtin_recipes_dir() / "remediation.yaml") - - -@pytest.fixture(scope="module") -def impl_groups_recipe(): - return load_recipe(builtin_recipes_dir() / "implementation-groups.yaml") - - -# --------------------------------------------------------------------------- -# merge-prs.yaml — validate passes -# --------------------------------------------------------------------------- - - -def test_merge_prs_queue_recipe_is_valid(pmp_recipe) -> None: - """validate_recipe must pass with no errors after queue steps are added.""" - errors = validate_recipe(pmp_recipe) - assert errors == [], f"validate_recipe errors: {errors}" - - -# --------------------------------------------------------------------------- -# merge-prs.yaml — new queue mode steps present -# --------------------------------------------------------------------------- - - -def test_merge_prs_route_by_queue_mode_exists(pmp_recipe) -> None: - """route_by_queue_mode step must exist in merge-prs.""" - assert "route_by_queue_mode" in pmp_recipe.steps - - -def test_merge_prs_route_by_queue_mode_is_route_action(pmp_recipe) -> None: - """route_by_queue_mode must be an action: route step.""" - step = pmp_recipe.steps["route_by_queue_mode"] - assert step.action == "route" - - -def test_merge_prs_enqueue_all_prs_exists(pmp_recipe) -> None: - """enqueue_all_prs step must exist with tool=run_cmd.""" - assert "enqueue_all_prs" in pmp_recipe.steps - step = pmp_recipe.steps["enqueue_all_prs"] - assert step.tool == "run_cmd" - - -def test_merge_prs_wait_queue_pr_exists(pmp_recipe) -> None: - """wait_queue_pr step must exist with tool=wait_for_merge_queue.""" - assert "wait_queue_pr" in pmp_recipe.steps - step = pmp_recipe.steps["wait_queue_pr"] - assert step.tool == "wait_for_merge_queue" - - -def test_merge_prs_reenter_queue_exists(pmp_recipe) -> None: - """reenter_queue step must exist.""" - assert "reenter_queue" in pmp_recipe.steps - - -def test_merge_prs_next_queue_pr_or_done_is_route_action(pmp_recipe) -> None: - """next_queue_pr_or_done must be an action: route step.""" - assert "next_queue_pr_or_done" in pmp_recipe.steps - step = pmp_recipe.steps["next_queue_pr_or_done"] - assert step.action == "route" - - -# --------------------------------------------------------------------------- -# merge-prs.yaml — analyze_prs captures queue_mode -# --------------------------------------------------------------------------- - - -def test_merge_prs_analyze_prs_captures_queue_mode(pmp_recipe) -> None: - """analyze_prs must capture queue_mode from the skill result.""" - step = pmp_recipe.steps["analyze_prs"] - assert "queue_mode" in (step.capture or {}), ( - "analyze_prs must capture queue_mode to enable route_by_queue_mode" - ) - - -def test_merge_prs_analyze_prs_routes_to_route_by_queue_mode(pmp_recipe) -> None: - """analyze_prs.on_success must route to route_by_queue_mode.""" - step = pmp_recipe.steps["analyze_prs"] - assert step.on_success == "route_by_queue_mode" - - -# --------------------------------------------------------------------------- -# merge-prs.yaml — classic path still intact -# --------------------------------------------------------------------------- - - -def test_merge_prs_classic_path_create_integration_branch_present(pmp_recipe) -> None: - """create_integration_branch step must still be present (classic path).""" - assert "create_integration_branch" in pmp_recipe.steps - - -def test_merge_prs_classic_path_merge_pr_present(pmp_recipe) -> None: - """merge_pr step must still be present (classic path).""" - assert "merge_pr" in pmp_recipe.steps - - -def test_merge_prs_classic_path_push_integration_branch_present(pmp_recipe) -> None: - """push_integration_branch step must still be present (classic path).""" - assert "push_integration_branch" in pmp_recipe.steps - - -# --------------------------------------------------------------------------- -# implementation.yaml — validate passes -# --------------------------------------------------------------------------- - - -def test_implementation_recipe_is_valid(impl_recipe) -> None: - """validate_recipe must pass with no errors after queue finale steps are added.""" - errors = validate_recipe(impl_recipe) - assert errors == [], f"validate_recipe errors: {errors}" - - -# --------------------------------------------------------------------------- -# implementation.yaml — new queue finale steps present -# --------------------------------------------------------------------------- - - -def test_implementation_has_check_repo_merge_state(impl_recipe) -> None: - """check_repo_merge_state step must exist with correct structure.""" - assert "check_repo_merge_state" in impl_recipe.steps - step = impl_recipe.steps["check_repo_merge_state"] - assert step.tool == "check_repo_merge_state" - assert step.block == "pre_queue_gate" - assert set(step.capture or {}) >= { - "queue_available", - "merge_group_trigger", - "auto_merge_available", - } - - -def test_implementation_pre_queue_gate_routes_to_route_queue_mode(impl_recipe) -> None: - """check_repo_merge_state.on_success must route to route_queue_mode.""" - step = impl_recipe.steps["check_repo_merge_state"] - assert step.on_success == "route_queue_mode" - - -def test_implementation_route_queue_mode_is_route_action(impl_recipe) -> None: - """route_queue_mode must be an action: route step.""" - assert "route_queue_mode" in impl_recipe.steps - step = impl_recipe.steps["route_queue_mode"] - assert step.action == "route" - - -def test_implementation_enable_auto_merge_is_run_cmd(impl_recipe) -> None: - """enable_auto_merge must use tool=run_cmd.""" - assert "enable_auto_merge" in impl_recipe.steps - step = impl_recipe.steps["enable_auto_merge"] - assert step.tool == "run_cmd" - - -def test_implementation_wait_for_queue_is_wait_for_merge_queue(impl_recipe) -> None: - """wait_for_queue must use tool=wait_for_merge_queue.""" - assert "wait_for_queue" in impl_recipe.steps - step = impl_recipe.steps["wait_for_queue"] - assert step.tool == "wait_for_merge_queue" - - -def test_implementation_queue_ejected_fix_exists(impl_recipe) -> None: - """queue_ejected_fix step must exist.""" - assert "queue_ejected_fix" in impl_recipe.steps - - -def test_queue_ejected_fix_tool_is_run_cmd(impl_recipe) -> None: - step = impl_recipe.steps["queue_ejected_fix"] - assert step.tool == "run_cmd", ( - "queue_ejected_fix must be a run_cmd cheap-rebase step, not a run_skill" - ) - - -def test_queue_ejected_fix_clean_routes_to_re_push(impl_recipe) -> None: - step = impl_recipe.steps["queue_ejected_fix"] - clean_route = next( - (c.route for c in step.on_result.conditions if c.when and "clean" in c.when), - None, - ) - assert clean_route == "re_push_queue_fix" - - -def test_queue_ejected_fix_conflicts_routes_to_resolve_skill(impl_recipe) -> None: - step = impl_recipe.steps["queue_ejected_fix"] - # The catch-all (no 'when') must route to the renamed skill step - fallback_route = next( - (c.route for c in step.on_result.conditions if not c.when), - None, - ) - assert fallback_route == "resolve_queue_merge_conflicts" - - -def test_queue_ejected_fix_on_failure_routes_to_resolve_skill(impl_recipe) -> None: - step = impl_recipe.steps["queue_ejected_fix"] - assert step.on_failure == "resolve_queue_merge_conflicts" - - -def test_resolve_queue_merge_conflicts_exists_with_run_skill(impl_recipe) -> None: - assert "resolve_queue_merge_conflicts" in impl_recipe.steps - step = impl_recipe.steps["resolve_queue_merge_conflicts"] - assert step.tool == "run_skill" - - -def test_resolve_queue_merge_conflicts_captures_escalation(impl_recipe) -> None: - step = impl_recipe.steps["resolve_queue_merge_conflicts"] - assert "conflict_escalation_required" in step.capture - - -def test_resolve_queue_merge_conflicts_routes_to_re_push(impl_recipe) -> None: - step = impl_recipe.steps["resolve_queue_merge_conflicts"] - fallback_route = next( - (c.route for c in step.on_result.conditions if not c.when), - None, - ) - assert fallback_route == "re_push_queue_fix" - - -def test_implementation_queue_finale_steps_all_have_skip_when_false(impl_recipe) -> None: - """All queue finale steps must have skip_when_false: inputs.open_pr.""" - finale_steps = [ - "check_repo_merge_state", - "route_queue_mode", - "enable_auto_merge", - "wait_for_queue", - "queue_ejected_fix", - "resolve_queue_merge_conflicts", - "re_push_queue_fix", - "reenter_merge_queue", - ] - for step_name in finale_steps: - assert step_name in impl_recipe.steps, f"Missing step: {step_name}" - step = impl_recipe.steps[step_name] - assert step.skip_when_false == "inputs.open_pr", ( - f"{step_name}.skip_when_false must be 'inputs.open_pr' " - f"so the open_pr=false path is unchanged" - ) - - -def test_implementation_ci_watch_routes_to_check_repo_merge_state_on_success( - impl_recipe, -) -> None: - """ci_watch.on_success must route to check_repo_merge_state.""" - step = impl_recipe.steps["ci_watch"] - assert step.on_success == "check_repo_merge_state" - - -def test_implementation_extract_pr_number_exists(impl_recipe) -> None: - """extract_pr_number step must exist with tool=run_cmd.""" - assert "extract_pr_number" in impl_recipe.steps - step = impl_recipe.steps["extract_pr_number"] - assert step.tool == "run_cmd" - - -def test_implementation_compose_pr_routes_to_extract_pr_number(impl_recipe) -> None: - """compose_pr.on_success must route to extract_pr_number.""" - step = impl_recipe.steps["compose_pr"] - assert step.on_success == "extract_pr_number" - - -def test_implementation_route_queue_mode_requires_merge_group_trigger(impl_recipe) -> None: - """route_queue_mode must NOT route to enable_auto_merge without checking merge_group_trigger. - - Specifically, the conditions list must not contain a bare 'queue_available == true' - → enable_auto_merge without also requiring merge_group_trigger == true. - """ - step = impl_recipe.steps["route_queue_mode"] - assert step.action == "route" - conditions = step.on_result.conditions if step.on_result else [] - # Find any condition that routes to enable_auto_merge - queue_conditions = [c for c in conditions if c.route == "enable_auto_merge"] - assert len(queue_conditions) == 1, "Exactly one condition must route to enable_auto_merge" - cond_when = queue_conditions[0].when or "" - assert "merge_group_trigger" in cond_when, ( - "The enable_auto_merge route condition must reference merge_group_trigger " - "to prevent queue enrollment when the CI workflow lacks the trigger" - ) - - -# --------------------------------------------------------------------------- -# remediation.yaml — validate passes -# --------------------------------------------------------------------------- - - -def test_remediation_recipe_is_valid(remed_recipe) -> None: - """validate_recipe must pass with no errors after queue finale steps are added.""" - errors = validate_recipe(remed_recipe) - assert errors == [], f"validate_recipe errors: {errors}" - - -# --------------------------------------------------------------------------- -# remediation.yaml — new queue finale steps present (identical to implementation) -# --------------------------------------------------------------------------- - - -def test_remediation_has_check_repo_merge_state(remed_recipe) -> None: - """check_repo_merge_state step must exist with correct structure in remediation.""" - assert "check_repo_merge_state" in remed_recipe.steps - step = remed_recipe.steps["check_repo_merge_state"] - assert step.tool == "check_repo_merge_state" - assert step.block == "pre_queue_gate" - assert set(step.capture or {}) >= { - "queue_available", - "merge_group_trigger", - "auto_merge_available", - } - - -def test_remediation_route_queue_mode_is_route_action(remed_recipe) -> None: - """route_queue_mode must be an action: route step.""" - assert "route_queue_mode" in remed_recipe.steps - step = remed_recipe.steps["route_queue_mode"] - assert step.action == "route" - - -def test_remediation_enable_auto_merge_is_run_cmd(remed_recipe) -> None: - """enable_auto_merge must use tool=run_cmd.""" - assert "enable_auto_merge" in remed_recipe.steps - step = remed_recipe.steps["enable_auto_merge"] - assert step.tool == "run_cmd" - - -def test_remediation_wait_for_queue_is_wait_for_merge_queue(remed_recipe) -> None: - """wait_for_queue must use tool=wait_for_merge_queue.""" - assert "wait_for_queue" in remed_recipe.steps - step = remed_recipe.steps["wait_for_queue"] - assert step.tool == "wait_for_merge_queue" - - -def test_remediation_queue_ejected_fix_exists(remed_recipe) -> None: - """queue_ejected_fix step must exist.""" - assert "queue_ejected_fix" in remed_recipe.steps - - -def test_remediation_queue_finale_steps_all_have_skip_when_false(remed_recipe) -> None: - """All queue finale steps must have skip_when_false: inputs.open_pr.""" - finale_steps = [ - "check_repo_merge_state", - "route_queue_mode", - "enable_auto_merge", - "wait_for_queue", - "queue_ejected_fix", - "resolve_queue_merge_conflicts", - "re_push_queue_fix", - "reenter_merge_queue", - ] - for step_name in finale_steps: - assert step_name in remed_recipe.steps, f"Missing step: {step_name}" - step = remed_recipe.steps[step_name] - assert step.skip_when_false == "inputs.open_pr", ( - f"{step_name}.skip_when_false must be 'inputs.open_pr' " - f"so the open_pr=false path is unchanged" - ) - - -def test_remediation_ci_watch_routes_to_check_repo_merge_state_on_success( - remed_recipe, -) -> None: - """ci_watch.on_success must route to check_repo_merge_state.""" - step = remed_recipe.steps["ci_watch"] - assert step.on_success == "check_repo_merge_state" - - -def test_remediation_extract_pr_number_exists(remed_recipe) -> None: - """extract_pr_number step must exist with tool=run_cmd.""" - assert "extract_pr_number" in remed_recipe.steps - step = remed_recipe.steps["extract_pr_number"] - assert step.tool == "run_cmd" - - -def test_remediation_compose_pr_routes_to_extract_pr_number(remed_recipe) -> None: - """compose_pr.on_success must route to extract_pr_number.""" - step = remed_recipe.steps["compose_pr"] - assert step.on_success == "extract_pr_number" - - -def test_remediation_route_queue_mode_requires_merge_group_trigger(remed_recipe) -> None: - """route_queue_mode must NOT route to enable_auto_merge without checking merge_group_trigger. - - Specifically, the conditions list must not contain a bare 'queue_available == true' - → enable_auto_merge without also requiring merge_group_trigger == true. - """ - step = remed_recipe.steps["route_queue_mode"] - conditions = step.on_result.conditions if step.on_result else [] - queue_conditions = [c for c in conditions if c.route == "enable_auto_merge"] - assert len(queue_conditions) == 1 - cond_when = queue_conditions[0].when or "" - assert "merge_group_trigger" in cond_when - - -def test_impl_groups_has_check_repo_merge_state(impl_groups_recipe) -> None: - """check_repo_merge_state step must exist in implementation-groups recipe.""" - assert "check_repo_merge_state" in impl_groups_recipe.steps - step = impl_groups_recipe.steps["check_repo_merge_state"] - assert step.block == "pre_queue_gate" - - -def test_impl_groups_route_queue_mode_requires_merge_group_trigger(impl_groups_recipe) -> None: - """route_queue_mode must NOT route to enable_auto_merge without checking merge_group_trigger. - - Specifically, the conditions list must not contain a bare 'queue_available == true' - → enable_auto_merge without also requiring merge_group_trigger == true. - """ - step = impl_groups_recipe.steps["route_queue_mode"] - conditions = step.on_result.conditions if step.on_result else [] - queue_conditions = [c for c in conditions if c.route == "enable_auto_merge"] - assert len(queue_conditions) == 1 - cond_when = queue_conditions[0].when or "" - assert "merge_group_trigger" in cond_when - - -@pytest.fixture(scope="module", params=["impl", "remed", "impl_groups"]) -def any_recipe(request, impl_recipe, remed_recipe, impl_groups_recipe): - return {"impl": impl_recipe, "remed": remed_recipe, "impl_groups": impl_groups_recipe}[ - request.param - ] - - -def test_auto_merge_ingredient(any_recipe) -> None: - assert "auto_merge" in any_recipe.ingredients - ing = any_recipe.ingredients["auto_merge"] - assert ing.default == "true" - assert ing.required is False - - -def test_route_queue_mode_auto_merge_condition_first(any_recipe) -> None: - step = any_recipe.steps["route_queue_mode"] - conds = step.on_result.conditions - auto_merge_idx = next(i for i, c in enumerate(conds) if c.when and "auto_merge" in c.when) - queue_available_idx = next( - i for i, c in enumerate(conds) if c.when and "queue_available" in c.when - ) - assert auto_merge_idx < queue_available_idx - - -def test_auto_merge_false_routes_to_register_clone_success(any_recipe) -> None: - step = any_recipe.steps["route_queue_mode"] - auto_merge_cond = next( - c for c in step.on_result.conditions if c.when and "auto_merge" in c.when - ) - assert auto_merge_cond.when == "${{ inputs.auto_merge }} != 'true'" - assert auto_merge_cond.route == "register_clone_success" - - -# --------------------------------------------------------------------------- -# Direct merge fallback — all three affected recipes -# --------------------------------------------------------------------------- - - -def test_route_queue_mode_default_routes_to_immediate_merge(any_recipe) -> None: - """Default (fallthrough) condition must route to immediate_merge, not direct_merge.""" - step = any_recipe.steps["route_queue_mode"] - default_cond = next((c for c in step.on_result.conditions if c.when is None), None) - assert default_cond is not None, "Expected a default (when=None) condition in route_queue_mode" - assert default_cond.route == "immediate_merge" - - -def test_direct_merge_step_exists(any_recipe) -> None: - assert "direct_merge" in any_recipe.steps - step = any_recipe.steps["direct_merge"] - assert step.tool == "run_cmd" - - -def test_direct_merge_routes_to_wait_for_direct_merge(any_recipe) -> None: - step = any_recipe.steps["direct_merge"] - assert step.on_success == "wait_for_direct_merge" - - -def test_direct_merge_failure_routes_to_register_clone_success(any_recipe) -> None: - step = any_recipe.steps["direct_merge"] - assert step.on_failure == "register_clone_success" - - -def test_wait_for_direct_merge_step_exists(any_recipe) -> None: - assert "wait_for_direct_merge" in any_recipe.steps - step = any_recipe.steps["wait_for_direct_merge"] - assert step.tool == "run_cmd" - - -def test_wait_for_direct_merge_merged_routes_to_success(any_recipe) -> None: - step = any_recipe.steps["wait_for_direct_merge"] - merged_cond = next( - (c for c in step.on_result.conditions if c.when and "merged" in c.when), None - ) - assert merged_cond is not None, "Expected a 'merged' condition in wait_for_direct_merge" - assert merged_cond.route == "release_issue_success" - - -def test_wait_for_direct_merge_closed_routes_to_conflict_fix(any_recipe) -> None: - step = any_recipe.steps["wait_for_direct_merge"] - closed_cond = next( - (c for c in step.on_result.conditions if c.when and "closed" in c.when), None - ) - assert closed_cond is not None, "Expected a 'closed' condition in wait_for_direct_merge" - assert closed_cond.route == "direct_merge_conflict_fix" - - -def test_direct_merge_conflict_fix_exists(any_recipe) -> None: - assert "direct_merge_conflict_fix" in any_recipe.steps - step = any_recipe.steps["direct_merge_conflict_fix"] - assert step.tool == "run_cmd" - - -def test_direct_merge_conflict_fix_clean_routes_to_re_push(any_recipe) -> None: - step = any_recipe.steps["direct_merge_conflict_fix"] - clean_route = next( - (c.route for c in step.on_result.conditions if c.when and "clean" in c.when), - None, - ) - assert clean_route == "re_push_direct_fix" - - -def test_resolve_direct_merge_conflicts_exists_with_run_skill(any_recipe) -> None: - assert "resolve_direct_merge_conflicts" in any_recipe.steps - assert any_recipe.steps["resolve_direct_merge_conflicts"].tool == "run_skill" - - -def test_re_push_direct_fix_exists(any_recipe) -> None: - assert "re_push_direct_fix" in any_recipe.steps - step = any_recipe.steps["re_push_direct_fix"] - assert step.tool == "push_to_remote" - assert step.on_success == "redirect_merge" - - -def test_redirect_merge_step_exists(any_recipe) -> None: - assert "redirect_merge" in any_recipe.steps - step = any_recipe.steps["redirect_merge"] - assert step.tool == "run_cmd" - assert step.on_success == "wait_for_direct_merge" - - -def test_direct_merge_steps_have_skip_when_false(any_recipe) -> None: - new_steps = [ - "direct_merge", - "wait_for_direct_merge", - "direct_merge_conflict_fix", - "resolve_direct_merge_conflicts", - "re_push_direct_fix", - "redirect_merge", - ] - for step_name in new_steps: - assert step_name in any_recipe.steps, f"Missing step: {step_name}" - step = any_recipe.steps[step_name] - assert step.skip_when_false == "inputs.open_pr", ( - f"{step_name}.skip_when_false must be 'inputs.open_pr'" - ) - - -# --------------------------------------------------------------------------- -# check_repo_merge_state — consolidated pre-queue gate step (any_recipe) -# --------------------------------------------------------------------------- - - -def test_check_repo_merge_state_step_exists(any_recipe) -> None: - """check_repo_merge_state step must exist in all three queue-capable recipes.""" - assert "check_repo_merge_state" in any_recipe.steps - - -def test_check_repo_merge_state_captures_all_three_fields(any_recipe) -> None: - step = any_recipe.steps["check_repo_merge_state"] - assert set(step.capture or {}) >= { - "queue_available", - "merge_group_trigger", - "auto_merge_available", - } - - -def test_check_repo_merge_state_routes_to_route_queue_mode_on_success(any_recipe) -> None: - step = any_recipe.steps["check_repo_merge_state"] - assert step.on_success == "route_queue_mode" - - -def test_check_repo_merge_state_has_skip_when_false(any_recipe) -> None: - step = any_recipe.steps["check_repo_merge_state"] - assert step.skip_when_false == "inputs.open_pr" - - -def test_check_repo_merge_state_is_in_pre_queue_gate_block(any_recipe) -> None: - step = any_recipe.steps["check_repo_merge_state"] - assert step.block == "pre_queue_gate" - - -def test_route_queue_mode_has_auto_merge_available_condition(any_recipe) -> None: - """route_queue_mode must have an explicit condition for auto_merge_available == true.""" - step = any_recipe.steps["route_queue_mode"] - conds = step.on_result.conditions - assert any(c.when and "auto_merge_available" in c.when for c in conds) - - -def test_route_queue_mode_auto_merge_available_routes_to_direct_merge(any_recipe) -> None: - step = any_recipe.steps["route_queue_mode"] - cond = next( - c - for c in step.on_result.conditions - if c.when and "auto_merge_available" in c.when and "queue_available" not in c.when - ) - assert cond.when == "${{ context.auto_merge_available }} == true" - assert cond.route == "direct_merge" - - -# --------------------------------------------------------------------------- -# Immediate merge path — new for autoMergeAllowed=false repos -# --------------------------------------------------------------------------- - - -def test_immediate_merge_step_exists(any_recipe) -> None: - assert "immediate_merge" in any_recipe.steps - step = any_recipe.steps["immediate_merge"] - assert step.tool == "run_cmd" - - -def test_immediate_merge_uses_squash_without_auto(any_recipe) -> None: - """immediate_merge must use --squash without --auto.""" - step = any_recipe.steps["immediate_merge"] - cmd = step.with_args.get("cmd", "") - assert "--squash" in cmd - assert "--auto" not in cmd - - -def test_immediate_merge_routes_to_wait_for_immediate_merge(any_recipe) -> None: - step = any_recipe.steps["immediate_merge"] - assert step.on_success == "wait_for_immediate_merge" - - -def test_immediate_merge_failure_routes_to_register_clone_success(any_recipe) -> None: - step = any_recipe.steps["immediate_merge"] - assert step.on_failure == "register_clone_success" - - -def test_wait_for_immediate_merge_step_exists(any_recipe) -> None: - assert "wait_for_immediate_merge" in any_recipe.steps - step = any_recipe.steps["wait_for_immediate_merge"] - assert step.tool == "run_cmd" - - -def test_wait_for_immediate_merge_merged_routes_to_success(any_recipe) -> None: - step = any_recipe.steps["wait_for_immediate_merge"] - merged_cond = next( - ( - c - for c in step.on_result.conditions - if c.when == "${{ result.stdout | trim }} == merged" - ), - None, - ) - assert merged_cond is not None - assert merged_cond.route == "release_issue_success" - - -def test_wait_for_immediate_merge_closed_routes_to_conflict_fix(any_recipe) -> None: - step = any_recipe.steps["wait_for_immediate_merge"] - closed_cond = next( - ( - c - for c in step.on_result.conditions - if c.when == "${{ result.stdout | trim }} == closed" - ), - None, - ) - assert closed_cond is not None - assert closed_cond.route == "immediate_merge_conflict_fix" - - -def test_immediate_merge_conflict_fix_exists(any_recipe) -> None: - assert "immediate_merge_conflict_fix" in any_recipe.steps - step = any_recipe.steps["immediate_merge_conflict_fix"] - assert step.tool == "run_cmd" - - -def test_immediate_merge_conflict_fix_clean_routes_to_re_push(any_recipe) -> None: - step = any_recipe.steps["immediate_merge_conflict_fix"] - clean_route = next( - (c.route for c in step.on_result.conditions if c.when and "clean" in c.when), - None, - ) - assert clean_route == "re_push_immediate_fix" - - -def test_resolve_immediate_merge_conflicts_exists_with_run_skill(any_recipe) -> None: - assert "resolve_immediate_merge_conflicts" in any_recipe.steps - assert any_recipe.steps["resolve_immediate_merge_conflicts"].tool == "run_skill" - - -def test_re_push_immediate_fix_exists(any_recipe) -> None: - assert "re_push_immediate_fix" in any_recipe.steps - step = any_recipe.steps["re_push_immediate_fix"] - assert step.tool == "push_to_remote" - assert step.on_success == "remerge_immediate" - - -def test_remerge_immediate_exists(any_recipe) -> None: - assert "remerge_immediate" in any_recipe.steps - step = any_recipe.steps["remerge_immediate"] - assert step.tool == "run_cmd" - assert step.on_success == "wait_for_immediate_merge" - - -def test_remerge_immediate_uses_squash_without_auto(any_recipe) -> None: - """remerge_immediate must also use --squash without --auto.""" - step = any_recipe.steps["remerge_immediate"] - cmd = step.with_args.get("cmd", "") - assert "--squash" in cmd - assert "--auto" not in cmd - - -def test_all_immediate_merge_steps_have_skip_when_false(any_recipe) -> None: - immediate_steps = [ - "immediate_merge", - "wait_for_immediate_merge", - "immediate_merge_conflict_fix", - "resolve_immediate_merge_conflicts", - "re_push_immediate_fix", - "remerge_immediate", - ] - for step_name in immediate_steps: - assert step_name in any_recipe.steps, f"Missing step: {step_name}" - step = any_recipe.steps[step_name] - assert step.skip_when_false == "inputs.open_pr", ( - f"{step_name}.skip_when_false must be 'inputs.open_pr'" - ) - - -def test_auto_merge_ingredient_description_updated(any_recipe) -> None: - ing = any_recipe.ingredients["auto_merge"] - assert "direct merge" in ing.description.lower() or "direct" in ing.description.lower(), ( - "auto_merge description must mention direct merge as an alternative to queue" - ) - - -def test_implementation_recipe_still_valid(impl_recipe) -> None: - errors = validate_recipe(impl_recipe) - assert errors == [], f"validate_recipe errors: {errors}" - - -def test_remediation_recipe_still_valid(remed_recipe) -> None: - errors = validate_recipe(remed_recipe) - assert errors == [], f"validate_recipe errors: {errors}" - - -def test_impl_groups_recipe_still_valid(impl_groups_recipe) -> None: - errors = validate_recipe(impl_groups_recipe) - assert errors == [], f"validate_recipe errors: {errors}" - - -# --------------------------------------------------------------------------- -# Gap 1 + Gap 6: ci_watch_post_queue_fix step + ejected_ci_failure routing -# (applies to all three recipes) -# --------------------------------------------------------------------------- - -QUEUE_RECIPES = ["impl_recipe", "remed_recipe", "impl_groups_recipe"] - - -@pytest.mark.parametrize("recipe_fixture", QUEUE_RECIPES) -def test_ci_watch_post_queue_fix_exists(recipe_fixture, request): - """ci_watch_post_queue_fix step must exist in all three queue-capable recipes.""" - recipe = request.getfixturevalue(recipe_fixture) - assert "ci_watch_post_queue_fix" in recipe.steps, ( - f"ci_watch_post_queue_fix must be a step in {recipe_fixture}" - ) - - -@pytest.mark.parametrize("recipe_fixture", QUEUE_RECIPES) -def test_re_push_queue_fix_routes_to_ci_watch_post_queue_fix(recipe_fixture, request): - """re_push_queue_fix.on_success must route to ci_watch_post_queue_fix.""" - recipe = request.getfixturevalue(recipe_fixture) - step = recipe.steps["re_push_queue_fix"] - assert step.on_success == "ci_watch_post_queue_fix", ( - f"re_push_queue_fix.on_success must be 'ci_watch_post_queue_fix' in {recipe_fixture}" - ) - - -@pytest.mark.parametrize("recipe_fixture", QUEUE_RECIPES) -def test_ci_watch_post_queue_fix_routes_reenter_on_success(recipe_fixture, request): - """ci_watch_post_queue_fix.on_success must route to reenter_merge_queue.""" - recipe = request.getfixturevalue(recipe_fixture) - assert "ci_watch_post_queue_fix" in recipe.steps, ( - f"ci_watch_post_queue_fix must be a step in {recipe_fixture}" - ) - step = recipe.steps["ci_watch_post_queue_fix"] - assert step.on_success == "reenter_merge_queue", ( - f"ci_watch_post_queue_fix.on_success must be 'reenter_merge_queue' in {recipe_fixture}" - ) - - -@pytest.mark.parametrize("recipe_fixture", QUEUE_RECIPES) -def test_ci_watch_post_queue_fix_routes_detect_ci_conflict_on_failure(recipe_fixture, request): - """ci_watch_post_queue_fix.on_failure must route to detect_ci_conflict.""" - recipe = request.getfixturevalue(recipe_fixture) - assert "ci_watch_post_queue_fix" in recipe.steps, ( - f"ci_watch_post_queue_fix must be a step in {recipe_fixture}" - ) - step = recipe.steps["ci_watch_post_queue_fix"] - assert step.on_failure == "detect_ci_conflict", ( - f"ci_watch_post_queue_fix.on_failure must be 'detect_ci_conflict' in {recipe_fixture}" - ) - - -@pytest.mark.parametrize("recipe_fixture", QUEUE_RECIPES) -def test_ci_watch_post_queue_fix_uses_wait_for_ci_tool(recipe_fixture, request): - """ci_watch_post_queue_fix must use the wait_for_ci tool.""" - recipe = request.getfixturevalue(recipe_fixture) - assert "ci_watch_post_queue_fix" in recipe.steps, ( - f"ci_watch_post_queue_fix must be a step in {recipe_fixture}" - ) - step = recipe.steps["ci_watch_post_queue_fix"] - assert step.tool == "wait_for_ci" - - -@pytest.mark.parametrize("recipe_fixture", QUEUE_RECIPES) -def test_ci_watch_post_queue_fix_has_skip_when_false(recipe_fixture, request): - """ci_watch_post_queue_fix must have skip_when_false: inputs.open_pr.""" - recipe = request.getfixturevalue(recipe_fixture) - assert "ci_watch_post_queue_fix" in recipe.steps, ( - f"ci_watch_post_queue_fix must be a step in {recipe_fixture}" - ) - step = recipe.steps["ci_watch_post_queue_fix"] - assert step.skip_when_false == "inputs.open_pr" - - -@pytest.mark.parametrize("recipe_fixture", QUEUE_RECIPES) -def test_wait_for_queue_routes_ejected_ci_failure_to_diagnose_ci(recipe_fixture, request): - """wait_for_queue.on_result must route ejected_ci_failure to diagnose_ci.""" - recipe = request.getfixturevalue(recipe_fixture) - step = recipe.steps["wait_for_queue"] - assert step.on_result is not None, "wait_for_queue must have on_result" - conditions = step.on_result.conditions - ci_failure_routes = [ - c - for c in conditions - if c.when is not None and "ejected_ci_failure" in c.when and c.route == "diagnose_ci" - ] - assert ci_failure_routes, ( - f"wait_for_queue.on_result must route ejected_ci_failure to diagnose_ci" - f" in {recipe_fixture}" - ) - - -@pytest.mark.parametrize("recipe_fixture", QUEUE_RECIPES) -def test_wait_for_queue_ejected_ci_failure_precedes_ejected(recipe_fixture, request): - """ejected_ci_failure route must precede generic ejected in wait_for_queue.on_result.""" - recipe = request.getfixturevalue(recipe_fixture) - step = recipe.steps["wait_for_queue"] - assert step.on_result is not None, "wait_for_queue must have on_result" - conditions = step.on_result.conditions - whens = [c.when or "" for c in conditions] - ci_fail_idx = next((i for i, w in enumerate(whens) if "ejected_ci_failure" in w), None) - ejected_idx = next( - (i for i, w in enumerate(whens) if w.strip() == "${{ result.pr_state }} == ejected"), None - ) - assert ci_fail_idx is not None, "ejected_ci_failure route must exist" - assert ejected_idx is not None, "ejected route must still exist" - assert ci_fail_idx < ejected_idx, ( - "ejected_ci_failure route must appear before generic ejected route " - "to prevent CI failure ejections from being handled as conflict ejections" - ) - - -# --------------------------------------------------------------------------- -# Routing matrix exhaustiveness — queue_available × auto_merge_available -# --------------------------------------------------------------------------- - - -def test_route_queue_mode_queue_with_auto_routes_to_enable_auto_merge(any_recipe) -> None: - """queue+auto cell must route to enable_auto_merge.""" - step = any_recipe.steps["route_queue_mode"] - cond = next( - c - for c in step.on_result.conditions - if c.when - and "queue_available" in c.when - and "merge_group_trigger" in c.when - and "auto_merge_available" in c.when - and "== true" in c.when.split("auto_merge_available")[1] - ) - assert cond.route == "enable_auto_merge" - - -def test_route_queue_mode_queue_without_auto_routes_to_queue_enqueue_no_auto( - any_recipe, -) -> None: - """queue+no-auto cell must route to queue_enqueue_no_auto.""" - step = any_recipe.steps["route_queue_mode"] - cond = next( - c - for c in step.on_result.conditions - if c.when - and "queue_available" in c.when - and "merge_group_trigger" in c.when - and "auto_merge_available" in c.when - and "== false" in c.when.split("auto_merge_available")[1] - ) - assert cond.route == "queue_enqueue_no_auto" - - -def test_route_queue_mode_no_queue_with_auto_routes_to_direct_merge(any_recipe) -> None: - """no-queue+auto cell must route to direct_merge.""" - step = any_recipe.steps["route_queue_mode"] - cond = next( - c - for c in step.on_result.conditions - if c.when and "auto_merge_available" in c.when and "queue_available" not in c.when - ) - assert cond.route == "direct_merge" - - -def test_route_queue_mode_no_queue_no_auto_falls_through_to_immediate_merge( - any_recipe, -) -> None: - """Default (when is None) condition must route to immediate_merge.""" - step = any_recipe.steps["route_queue_mode"] - cond = next(c for c in step.on_result.conditions if c.when is None) - assert cond.route == "immediate_merge" - - -def test_route_queue_mode_never_routes_to_enable_auto_merge_when_auto_unavailable( - any_recipe, -) -> None: - """Every condition routing to enable_auto_merge must require auto_merge_available == true.""" - step = any_recipe.steps["route_queue_mode"] - for cond in step.on_result.conditions: - if cond.route == "enable_auto_merge": - assert cond.when is not None - assert "auto_merge_available" in cond.when - assert "}} == true" in cond.when.split("auto_merge_available")[1], ( - "enable_auto_merge route must require auto_merge_available == true; " - f"got: {cond.when}" - ) - - -def test_enable_auto_merge_route_count(any_recipe) -> None: - """Exactly one condition must route to enable_auto_merge.""" - step = any_recipe.steps["route_queue_mode"] - count = sum(1 for c in step.on_result.conditions if c.route == "enable_auto_merge") - assert count == 1, f"Expected exactly 1 enable_auto_merge route, got {count}" - - -# --------------------------------------------------------------------------- -# New step: queue_enqueue_no_auto -# --------------------------------------------------------------------------- - - -def test_queue_enqueue_no_auto_step_exists(any_recipe) -> None: - assert "queue_enqueue_no_auto" in any_recipe.steps - - -def test_queue_enqueue_no_auto_is_run_cmd(any_recipe) -> None: - step = any_recipe.steps["queue_enqueue_no_auto"] - assert step.tool == "run_cmd" - - -def test_queue_enqueue_no_auto_uses_plain_squash(any_recipe) -> None: - """queue_enqueue_no_auto must use --squash without --auto.""" - step = any_recipe.steps["queue_enqueue_no_auto"] - cmd = step.with_args.get("cmd", "") - assert "--squash" in cmd - assert "--auto" not in cmd - - -def test_queue_enqueue_no_auto_routes_to_wait_for_queue(any_recipe) -> None: - step = any_recipe.steps["queue_enqueue_no_auto"] - assert step.on_success == "wait_for_queue" - - -def test_queue_enqueue_no_auto_failure_routes_to_register_clone_success(any_recipe) -> None: - step = any_recipe.steps["queue_enqueue_no_auto"] - assert step.on_failure == "register_clone_success" - - -def test_queue_enqueue_no_auto_skip_when_false(any_recipe) -> None: - step = any_recipe.steps["queue_enqueue_no_auto"] - assert step.skip_when_false == "inputs.open_pr" - - -def test_queue_enqueue_no_auto_step_name(any_recipe) -> None: - step = any_recipe.steps["queue_enqueue_no_auto"] - assert step.with_args["step_name"] == "queue_enqueue_no_auto" - - -# --------------------------------------------------------------------------- -# --------------------------------------------------------------------------- -# T12: Regression — remediation.yaml timeout arm and correct fallback -# --------------------------------------------------------------------------- - - -def test_remediation_wait_for_queue_has_timeout_arm_and_release_timeout_fallback( - remed_recipe, -) -> None: - """remediation.yaml wait_for_queue must have explicit timeout arm and correct fallback.""" - step = remed_recipe.steps["wait_for_queue"] - assert step.on_result is not None, "wait_for_queue must have on_result" - conditions = step.on_result.conditions - - # Must have an explicit timeout arm routing to release_issue_timeout - timeout_conditions = [ - c - for c in conditions - if c.when is not None and "timeout" in c.when and c.route == "release_issue_timeout" - ] - assert timeout_conditions, ( - "remediation.yaml wait_for_queue must have explicit " - "'${{ result.pr_state }} == timeout -> release_issue_timeout' arm" - ) - - # Fallback (when=None) must route to release_issue_timeout, not register_clone_success - fallback_conditions = [c for c in conditions if c.when is None] - assert fallback_conditions, "wait_for_queue must have a fallback condition (when=None)" - assert fallback_conditions[0].route == "release_issue_timeout", ( - f"remediation.yaml wait_for_queue fallback must be release_issue_timeout, " - f"got: {fallback_conditions[0].route!r}" - ) - - # on_failure must route to release_issue_timeout, not register_clone_success - assert step.on_failure == "release_issue_timeout", ( - f"remediation.yaml wait_for_queue on_failure must be release_issue_timeout, " - f"got: {step.on_failure!r}" - ) - - -# --------------------------------------------------------------------------- -# T9: Full routing parity — every PRState covered, fallback and on_failure identical -# --------------------------------------------------------------------------- - -_REQUIRED_PR_STATE_VALUES = frozenset(s.value for s in PRState if s != PRState.ERROR) - - -@pytest.mark.parametrize("recipe_fixture", QUEUE_RECIPES) -def test_wait_for_queue_routing_covers_every_pr_state(recipe_fixture, request) -> None: - """wait_for_queue.on_result must cover every non-error PRState with explicit when arms.""" - recipe = request.getfixturevalue(recipe_fixture) - step = recipe.steps["wait_for_queue"] - assert step.on_result is not None, "wait_for_queue must have on_result" - conditions = step.on_result.conditions - - # Collect the explicit when values by extracting the value after '==' - import re - - _PR_STATE_WHEN_RE = re.compile(r"\$\{\{\s*result\.pr_state\s*\}\}\s*==\s*(\w+)") - covered: set[str] = set() - for c in conditions: - if c.when is None: - continue - m = _PR_STATE_WHEN_RE.search(c.when) - if m: - covered.add(m.group(1)) - - missing = _REQUIRED_PR_STATE_VALUES - covered - assert not missing, ( - f"{recipe_fixture}: wait_for_queue.on_result is missing explicit routing arms " - f"for PRState values: {sorted(missing)}. Every non-error PRState must have a " - f"when condition." - ) - - # Fallback target must be release_issue_timeout - fallback_conditions = [c for c in conditions if c.when is None] - assert fallback_conditions, ( - f"{recipe_fixture}: wait_for_queue.on_result must have a fallback condition" - ) - assert fallback_conditions[0].route == "release_issue_timeout", ( - f"{recipe_fixture}: wait_for_queue fallback must route to release_issue_timeout, " - f"got: {fallback_conditions[0].route!r}" - ) - - # on_failure target must be release_issue_timeout - assert step.on_failure == "release_issue_timeout", ( - f"{recipe_fixture}: wait_for_queue on_failure must be release_issue_timeout, " - f"got: {step.on_failure!r}" - ) - - # ejected_ci_failure must precede generic ejected (existing invariant preserved) - whens = [c.when or "" for c in conditions] - ci_fail_idx = next((i for i, w in enumerate(whens) if "ejected_ci_failure" in w), None) - ejected_idx = next( - (i for i, w in enumerate(whens) if w.strip() == "${{ result.pr_state }} == ejected"), - None, - ) - assert ci_fail_idx is not None, f"{recipe_fixture}: ejected_ci_failure route must exist" - assert ejected_idx is not None, f"{recipe_fixture}: ejected route must exist" - assert ci_fail_idx < ejected_idx, ( - f"{recipe_fixture}: ejected_ci_failure route must appear before generic ejected route" - ) - - # reenter_merge_queue_cheap must exist and be reachable from dropped_healthy - assert "reenter_merge_queue_cheap" in recipe.steps, ( - f"{recipe_fixture}: reenter_merge_queue_cheap step must exist" - ) - dropped_routes = [ - c - for c in conditions - if c.when is not None - and "dropped_healthy" in c.when - and c.route == "reenter_merge_queue_cheap" - ] - assert dropped_routes, ( - f"{recipe_fixture}: dropped_healthy must route to reenter_merge_queue_cheap" - ) diff --git a/tests/recipe/test_merge_prs_queue_any.py b/tests/recipe/test_merge_prs_queue_any.py new file mode 100644 index 0000000000..a4c00fe236 --- /dev/null +++ b/tests/recipe/test_merge_prs_queue_any.py @@ -0,0 +1,315 @@ +"""Queue mode structural assertions for the implementation, remediation, and +implementation-groups recipes (strategy-specific per-recipe checks).""" + +from __future__ import annotations + +import pytest + +from autoskillit.recipe.validator import validate_recipe_structure + +pytestmark = [pytest.mark.layer("recipe"), pytest.mark.small] + + +# --------------------------------------------------------------------------- +# implementation.yaml — validate passes +# --------------------------------------------------------------------------- + + +def test_implementation_recipe_is_valid(impl_recipe) -> None: + """validate_recipe_structure must pass with no errors after queue finale steps are added.""" + errors = validate_recipe_structure(impl_recipe) + assert errors == [], f"validate_recipe_structure errors: {errors}" + + +# --------------------------------------------------------------------------- +# implementation.yaml — new queue finale steps present +# --------------------------------------------------------------------------- + + +def test_implementation_has_check_repo_merge_state(impl_recipe) -> None: + """check_repo_merge_state step must exist with correct structure.""" + assert "check_repo_merge_state" in impl_recipe.steps + step = impl_recipe.steps["check_repo_merge_state"] + assert step.tool == "check_repo_merge_state" + assert step.block == "pre_queue_gate" + assert set(step.capture or {}) >= { + "queue_available", + "merge_group_trigger", + "auto_merge_available", + } + + +def test_implementation_pre_queue_gate_routes_to_route_queue_mode(impl_recipe) -> None: + """check_repo_merge_state.on_success must route to route_queue_mode.""" + step = impl_recipe.steps["check_repo_merge_state"] + assert step.on_success == "route_queue_mode" + + +def test_implementation_route_queue_mode_is_route_action(impl_recipe) -> None: + """route_queue_mode must be an action: route step.""" + assert "route_queue_mode" in impl_recipe.steps + step = impl_recipe.steps["route_queue_mode"] + assert step.action == "route" + + +def test_implementation_enqueue_to_queue_is_enqueue_pr(impl_recipe) -> None: + """enqueue_to_queue must use tool=enqueue_pr.""" + assert "enqueue_to_queue" in impl_recipe.steps + step = impl_recipe.steps["enqueue_to_queue"] + assert step.tool == "enqueue_pr" + + +def test_implementation_wait_for_queue_is_wait_for_merge_queue(impl_recipe) -> None: + """wait_for_queue must use tool=wait_for_merge_queue.""" + assert "wait_for_queue" in impl_recipe.steps + step = impl_recipe.steps["wait_for_queue"] + assert step.tool == "wait_for_merge_queue" + + +def test_implementation_queue_ejected_fix_exists(impl_recipe) -> None: + """queue_ejected_fix step must exist.""" + assert "queue_ejected_fix" in impl_recipe.steps + + +def test_queue_ejected_fix_tool_is_run_python(impl_recipe) -> None: + step = impl_recipe.steps["queue_ejected_fix"] + assert step.tool == "run_python", ( + "queue_ejected_fix must be a run_python callable step, not a run_skill or run_cmd" + ) + assert step.with_args.get("callable") == "autoskillit.recipe._cmd_rpc.queue_ejected_fix" + + +def test_queue_ejected_fix_clean_routes_to_re_push(impl_recipe) -> None: + step = impl_recipe.steps["queue_ejected_fix"] + clean_route = next( + (c.route for c in step.on_result.conditions if c.when and "clean" in c.when), + None, + ) + assert clean_route == "re_push_queue_fix" + + +def test_queue_ejected_fix_conflicts_routes_to_resolve_skill(impl_recipe) -> None: + step = impl_recipe.steps["queue_ejected_fix"] + # The catch-all (no 'when') must route to the renamed skill step + fallback_route = next( + (c.route for c in step.on_result.conditions if not c.when), + None, + ) + assert fallback_route == "resolve_queue_merge_conflicts" + + +def test_queue_ejected_fix_on_failure_routes_to_resolve_skill(impl_recipe) -> None: + step = impl_recipe.steps["queue_ejected_fix"] + assert step.on_failure == "resolve_queue_merge_conflicts" + + +def test_resolve_queue_merge_conflicts_exists_with_run_skill(impl_recipe) -> None: + assert "resolve_queue_merge_conflicts" in impl_recipe.steps + step = impl_recipe.steps["resolve_queue_merge_conflicts"] + assert step.tool == "run_skill" + + +def test_resolve_queue_merge_conflicts_captures_escalation(impl_recipe) -> None: + step = impl_recipe.steps["resolve_queue_merge_conflicts"] + assert "conflict_escalation_required" in step.capture + + +def test_resolve_queue_merge_conflicts_routes_to_re_push(impl_recipe) -> None: + step = impl_recipe.steps["resolve_queue_merge_conflicts"] + fallback_route = next( + (c.route for c in step.on_result.conditions if not c.when), + None, + ) + assert fallback_route == "re_push_queue_fix" + + +def test_implementation_queue_finale_steps_all_have_skip_when_false(impl_recipe) -> None: + """All queue finale steps must have skip_when_false: inputs.open_pr.""" + finale_steps = [ + "check_repo_merge_state", + "route_queue_mode", + "enqueue_to_queue", + "wait_for_queue", + "queue_ejected_fix", + "resolve_queue_merge_conflicts", + "re_push_queue_fix", + "reenter_merge_queue", + ] + for step_name in finale_steps: + assert step_name in impl_recipe.steps, f"Missing step: {step_name}" + step = impl_recipe.steps[step_name] + assert step.skip_when_false == "inputs.open_pr", ( + f"{step_name}.skip_when_false must be 'inputs.open_pr' " + f"so the open_pr=false path is unchanged" + ) + + +def test_implementation_ci_watch_routes_to_check_repo_merge_state_on_success( + impl_recipe, +) -> None: + """ci_watch on_result success must route to check_repo_merge_state.""" + step = impl_recipe.steps["ci_watch"] + assert step.on_result is not None, "ci_watch must use on_result predicate routing" + success_routes = [ + c.route for c in step.on_result.conditions if c.when and "'success'" in c.when + ] + assert "check_repo_merge_state" in success_routes + + +def test_implementation_extract_pr_number_exists(impl_recipe) -> None: + """extract_pr_number step must exist with tool=run_cmd.""" + assert "extract_pr_number" in impl_recipe.steps + step = impl_recipe.steps["extract_pr_number"] + assert step.tool == "run_cmd" + + +def test_implementation_compose_pr_routes_to_extract_pr_number(impl_recipe) -> None: + """compose_pr.on_success must route to extract_pr_number.""" + step = impl_recipe.steps["compose_pr"] + assert step.on_success == "extract_pr_number" + + +def test_implementation_route_queue_mode_requires_merge_group_trigger(impl_recipe) -> None: + """route_queue_mode must NOT route to enqueue_to_queue without checking merge_group_trigger. + + Specifically, the conditions list must not contain a bare 'queue_available == true' + → enqueue_to_queue without also requiring merge_group_trigger == true. + """ + step = impl_recipe.steps["route_queue_mode"] + assert step.action == "route" + conditions = step.on_result.conditions if step.on_result else [] + queue_conditions = [c for c in conditions if c.route == "enqueue_to_queue"] + assert len(queue_conditions) == 1, "Exactly one condition must route to enqueue_to_queue" + cond_when = queue_conditions[0].when or "" + assert "merge_group_trigger" in cond_when, ( + "The enqueue_to_queue route condition must reference merge_group_trigger " + "to prevent queue enrollment when the CI workflow lacks the trigger" + ) + + +# --------------------------------------------------------------------------- +# remediation.yaml — validate passes +# --------------------------------------------------------------------------- + + +def test_remediation_recipe_is_valid(remed_recipe) -> None: + """validate_recipe_structure must pass with no errors after queue finale steps are added.""" + errors = validate_recipe_structure(remed_recipe) + assert errors == [], f"validate_recipe_structure errors: {errors}" + + +# --------------------------------------------------------------------------- +# remediation.yaml — new queue finale steps present (identical to implementation) +# --------------------------------------------------------------------------- + + +def test_remediation_has_check_repo_merge_state(remed_recipe) -> None: + """check_repo_merge_state step must exist with correct structure in remediation.""" + assert "check_repo_merge_state" in remed_recipe.steps + step = remed_recipe.steps["check_repo_merge_state"] + assert step.tool == "check_repo_merge_state" + assert step.block == "pre_queue_gate" + assert set(step.capture or {}) >= { + "queue_available", + "merge_group_trigger", + "auto_merge_available", + } + + +def test_remediation_route_queue_mode_is_route_action(remed_recipe) -> None: + """route_queue_mode must be an action: route step.""" + assert "route_queue_mode" in remed_recipe.steps + step = remed_recipe.steps["route_queue_mode"] + assert step.action == "route" + + +def test_remediation_enqueue_to_queue_is_enqueue_pr(remed_recipe) -> None: + """enqueue_to_queue must use tool=enqueue_pr.""" + assert "enqueue_to_queue" in remed_recipe.steps + step = remed_recipe.steps["enqueue_to_queue"] + assert step.tool == "enqueue_pr" + + +def test_remediation_wait_for_queue_is_wait_for_merge_queue(remed_recipe) -> None: + """wait_for_queue must use tool=wait_for_merge_queue.""" + assert "wait_for_queue" in remed_recipe.steps + step = remed_recipe.steps["wait_for_queue"] + assert step.tool == "wait_for_merge_queue" + + +def test_remediation_queue_ejected_fix_exists(remed_recipe) -> None: + """queue_ejected_fix step must exist.""" + assert "queue_ejected_fix" in remed_recipe.steps + + +def test_remediation_queue_finale_steps_all_have_skip_when_false(remed_recipe) -> None: + """All queue finale steps must have skip_when_false: inputs.open_pr.""" + finale_steps = [ + "check_repo_merge_state", + "route_queue_mode", + "enqueue_to_queue", + "wait_for_queue", + "queue_ejected_fix", + "resolve_queue_merge_conflicts", + "re_push_queue_fix", + "reenter_merge_queue", + ] + for step_name in finale_steps: + assert step_name in remed_recipe.steps, f"Missing step: {step_name}" + step = remed_recipe.steps[step_name] + assert step.skip_when_false == "inputs.open_pr", ( + f"{step_name}.skip_when_false must be 'inputs.open_pr' " + f"so the open_pr=false path is unchanged" + ) + + +def test_remediation_ci_watch_routes_to_check_repo_merge_state_on_success( + remed_recipe, +) -> None: + """ci_watch on_result success must route to check_repo_merge_state.""" + step = remed_recipe.steps["ci_watch"] + assert step.on_result is not None, "ci_watch must use on_result predicate routing" + success_routes = [ + c.route for c in step.on_result.conditions if c.when and "'success'" in c.when + ] + assert "check_repo_merge_state" in success_routes + + +def test_remediation_extract_pr_number_exists(remed_recipe) -> None: + """extract_pr_number step must exist with tool=run_cmd.""" + assert "extract_pr_number" in remed_recipe.steps + step = remed_recipe.steps["extract_pr_number"] + assert step.tool == "run_cmd" + + +def test_remediation_compose_pr_routes_to_extract_pr_number(remed_recipe) -> None: + """compose_pr.on_success must route to extract_pr_number.""" + step = remed_recipe.steps["compose_pr"] + assert step.on_success == "extract_pr_number" + + +def test_remediation_route_queue_mode_requires_merge_group_trigger(remed_recipe) -> None: + """route_queue_mode must NOT route to enqueue_to_queue without checking merge_group_trigger.""" + step = remed_recipe.steps["route_queue_mode"] + conditions = step.on_result.conditions if step.on_result else [] + queue_conditions = [c for c in conditions if c.route == "enqueue_to_queue"] + assert len(queue_conditions) == 1 + cond_when = queue_conditions[0].when or "" + assert "merge_group_trigger" in cond_when + + +def test_impl_groups_has_check_repo_merge_state(impl_groups_recipe) -> None: + """check_repo_merge_state step must exist in implementation-groups recipe.""" + assert "check_repo_merge_state" in impl_groups_recipe.steps + step = impl_groups_recipe.steps["check_repo_merge_state"] + assert step.block == "pre_queue_gate" + + +def test_impl_groups_route_queue_mode_requires_merge_group_trigger(impl_groups_recipe) -> None: + """route_queue_mode must NOT route to enqueue_to_queue without checking merge_group_trigger.""" + step = impl_groups_recipe.steps["route_queue_mode"] + conditions = step.on_result.conditions if step.on_result else [] + queue_conditions = [c for c in conditions if c.route == "enqueue_to_queue"] + assert len(queue_conditions) == 1 + cond_when = queue_conditions[0].when or "" + assert "merge_group_trigger" in cond_when diff --git a/tests/recipe/test_merge_prs_queue_common.py b/tests/recipe/test_merge_prs_queue_common.py new file mode 100644 index 0000000000..db328c9286 --- /dev/null +++ b/tests/recipe/test_merge_prs_queue_common.py @@ -0,0 +1,772 @@ +"""Queue mode structural assertions shared across all queue-capable recipes +(parametrized over any_recipe or QUEUE_RECIPES).""" + +from __future__ import annotations + +import re + +import pytest + +from autoskillit.core import PRState +from autoskillit.recipe.io import builtin_recipes_dir, load_recipe + +pytestmark = [pytest.mark.layer("recipe"), pytest.mark.small] + + +# pmp is excluded: pmp-specific assertions live in test_merge_prs_queue_pmp.py +@pytest.fixture(scope="module", params=["impl", "remed", "impl_groups"]) +def any_recipe(request, impl_recipe, remed_recipe, impl_groups_recipe): + return {"impl": impl_recipe, "remed": remed_recipe, "impl_groups": impl_groups_recipe}[ + request.param + ] + + +# --------------------------------------------------------------------------- +# Queue recipe auto-discovery — any recipe with wait_for_merge_queue routing +# --------------------------------------------------------------------------- + + +def _discover_queue_recipe_fixtures() -> list[str]: + """Return fixture names for all bundled recipes with wait_for_merge_queue routing.""" + fixture_map = { + "implementation": "impl_recipe", + "remediation": "remed_recipe", + "implementation-groups": "impl_groups_recipe", + "merge-prs": "pmp_recipe", + } + result = [] + for yaml_path in sorted(builtin_recipes_dir().glob("*.yaml")): + name = yaml_path.stem + recipe = load_recipe(yaml_path) + for step in recipe.steps.values(): + if ( + step.tool == "wait_for_merge_queue" + and step.on_result is not None + and getattr(step.on_result, "conditions", None) + ): + fixture_name = fixture_map.get(name) + if fixture_name: + result.append(fixture_name) + break + return sorted(result) + + +# Intentional: fail-fast at collection time if a bundled recipe YAML is malformed. +QUEUE_RECIPES = _discover_queue_recipe_fixtures() + +# Recipes that use register_clone_unconfirmed as the queue error escalation step. +# Imported by test_merge_prs_queue_release_timeout.py to avoid duplicate definition. +RELEASE_TIMEOUT_RECIPES = ["impl_recipe", "remed_recipe", "impl_groups_recipe"] + + +# --------------------------------------------------------------------------- +# any_recipe parametrized — auto_merge ingredient +# --------------------------------------------------------------------------- + + +def test_auto_merge_ingredient(any_recipe) -> None: + assert "auto_merge" in any_recipe.ingredients + ing = any_recipe.ingredients["auto_merge"] + assert ing.default == "true" + assert ing.required is False + + +def test_route_queue_mode_auto_merge_condition_first(any_recipe) -> None: + step = any_recipe.steps["route_queue_mode"] + assert step.on_result is not None, "route_queue_mode must have on_result routing" + conds = step.on_result.conditions + auto_merge_idx = next( + (i for i, c in enumerate(conds) if c.when and "auto_merge" in c.when), None + ) + assert auto_merge_idx is not None, "Expected a condition with 'auto_merge' in route_queue_mode" + queue_available_idx = next( + (i for i, c in enumerate(conds) if c.when and "queue_available" in c.when), None + ) + assert queue_available_idx is not None, ( + "Expected a condition with 'queue_available' in route_queue_mode" + ) + assert auto_merge_idx < queue_available_idx + + +def test_auto_merge_false_routes_to_register_clone_success(any_recipe) -> None: + step = any_recipe.steps["route_queue_mode"] + assert step.on_result is not None, "route_queue_mode must have on_result routing" + auto_merge_cond = next( + (c for c in step.on_result.conditions if c.when and "auto_merge" in c.when), None + ) + assert auto_merge_cond is not None, ( + "Expected a condition with 'auto_merge' in route_queue_mode" + ) + assert auto_merge_cond.when == "${{ inputs.auto_merge }} != 'true'" + assert auto_merge_cond.route == "patch_token_summary" + + +# --------------------------------------------------------------------------- +# Direct merge fallback — all three affected recipes +# --------------------------------------------------------------------------- + + +def test_route_queue_mode_default_routes_to_immediate_merge(any_recipe) -> None: + """Default (fallthrough) condition must route to immediate_merge, not direct_merge.""" + step = any_recipe.steps["route_queue_mode"] + default_cond = next((c for c in step.on_result.conditions if c.when is None), None) + assert default_cond is not None, "Expected a default (when=None) condition in route_queue_mode" + assert default_cond.route == "immediate_merge" + + +def test_direct_merge_step_exists(any_recipe) -> None: + assert "direct_merge" in any_recipe.steps + step = any_recipe.steps["direct_merge"] + assert step.tool == "run_cmd" + + +def test_direct_merge_routes_to_wait_for_direct_merge(any_recipe) -> None: + step = any_recipe.steps["direct_merge"] + assert step.on_success == "wait_for_direct_merge" + + +def test_direct_merge_failure_routes_to_release_issue_failure(any_recipe) -> None: + step = any_recipe.steps["direct_merge"] + assert step.on_failure == "release_issue_failure" + + +def test_wait_for_direct_merge_step_exists(any_recipe) -> None: + assert "wait_for_direct_merge" in any_recipe.steps + step = any_recipe.steps["wait_for_direct_merge"] + assert step.tool == "run_python" + + +def test_wait_for_direct_merge_merged_routes_to_success(any_recipe) -> None: + step = any_recipe.steps["wait_for_direct_merge"] + merged_cond = next( + (c for c in step.on_result.conditions if c.when and "merged" in c.when), None + ) + assert merged_cond is not None, "Expected a 'merged' condition in wait_for_direct_merge" + assert merged_cond.route == "release_issue_success" + + +def test_wait_for_direct_merge_closed_routes_to_conflict_fix(any_recipe) -> None: + step = any_recipe.steps["wait_for_direct_merge"] + closed_cond = next( + (c for c in step.on_result.conditions if c.when and "closed" in c.when), None + ) + assert closed_cond is not None, "Expected a 'closed' condition in wait_for_direct_merge" + assert closed_cond.route == "direct_merge_conflict_fix" + + +def test_direct_merge_conflict_fix_exists(any_recipe) -> None: + assert "direct_merge_conflict_fix" in any_recipe.steps + step = any_recipe.steps["direct_merge_conflict_fix"] + assert step.tool == "run_python" + + +def test_direct_merge_conflict_fix_clean_routes_to_re_push(any_recipe) -> None: + step = any_recipe.steps["direct_merge_conflict_fix"] + clean_route = next( + (c.route for c in step.on_result.conditions if c.when and "clean" in c.when), + None, + ) + assert clean_route == "re_push_direct_fix" + + +def test_resolve_direct_merge_conflicts_exists_with_run_skill(any_recipe) -> None: + assert "resolve_direct_merge_conflicts" in any_recipe.steps + assert any_recipe.steps["resolve_direct_merge_conflicts"].tool == "run_skill" + + +def test_re_push_direct_fix_exists(any_recipe) -> None: + assert "re_push_direct_fix" in any_recipe.steps + step = any_recipe.steps["re_push_direct_fix"] + assert step.tool == "push_to_remote" + assert step.on_success == "redirect_merge" + + +def test_redirect_merge_step_exists(any_recipe) -> None: + assert "redirect_merge" in any_recipe.steps + step = any_recipe.steps["redirect_merge"] + assert step.tool == "run_cmd" + assert step.on_success == "wait_for_direct_merge" + + +def test_direct_merge_steps_have_skip_when_false(any_recipe) -> None: + new_steps = [ + "direct_merge", + "wait_for_direct_merge", + "direct_merge_conflict_fix", + "resolve_direct_merge_conflicts", + "re_push_direct_fix", + "redirect_merge", + ] + for step_name in new_steps: + assert step_name in any_recipe.steps, f"Missing step: {step_name}" + step = any_recipe.steps[step_name] + assert step.skip_when_false == "inputs.open_pr", ( + f"{step_name}.skip_when_false must be 'inputs.open_pr'" + ) + + +# --------------------------------------------------------------------------- +# check_repo_merge_state — consolidated pre-queue gate step (any_recipe) +# --------------------------------------------------------------------------- + + +def test_check_repo_merge_state_step_exists(any_recipe) -> None: + """check_repo_merge_state step must exist in all three queue-capable recipes.""" + assert "check_repo_merge_state" in any_recipe.steps + + +def test_check_repo_merge_state_captures_all_three_fields(any_recipe) -> None: + step = any_recipe.steps["check_repo_merge_state"] + assert set(step.capture or {}) >= { + "queue_available", + "merge_group_trigger", + "auto_merge_available", + } + + +def test_check_repo_merge_state_routes_to_route_queue_mode_on_success(any_recipe) -> None: + step = any_recipe.steps["check_repo_merge_state"] + assert step.on_success == "route_queue_mode" + + +def test_check_repo_merge_state_has_skip_when_false(any_recipe) -> None: + step = any_recipe.steps["check_repo_merge_state"] + assert step.skip_when_false == "inputs.open_pr" + + +def test_check_repo_merge_state_is_in_pre_queue_gate_block(any_recipe) -> None: + step = any_recipe.steps["check_repo_merge_state"] + assert step.block == "pre_queue_gate" + + +def test_route_queue_mode_has_auto_merge_available_condition(any_recipe) -> None: + """route_queue_mode must have an explicit condition for auto_merge_available == true.""" + step = any_recipe.steps["route_queue_mode"] + assert step.on_result is not None, "route_queue_mode must have on_result routing" + conds = step.on_result.conditions + assert any(c.when and "auto_merge_available" in c.when for c in conds) + + +def test_route_queue_mode_auto_merge_available_routes_to_direct_merge(any_recipe) -> None: + step = any_recipe.steps["route_queue_mode"] + cond = next( + c + for c in step.on_result.conditions + if c.when and "auto_merge_available" in c.when and "queue_available" not in c.when + ) + assert cond.when == "${{ context.auto_merge_available }} == true" + assert cond.route == "direct_merge" + + +# --------------------------------------------------------------------------- +# Immediate merge path — new for autoMergeAllowed=false repos +# --------------------------------------------------------------------------- + + +def test_immediate_merge_step_exists(any_recipe) -> None: + assert "immediate_merge" in any_recipe.steps + step = any_recipe.steps["immediate_merge"] + assert step.tool == "run_cmd" + + +def test_immediate_merge_uses_squash_without_auto(any_recipe) -> None: + """immediate_merge must use --squash without --auto.""" + step = any_recipe.steps["immediate_merge"] + assert step.with_args is not None, "immediate_merge must have with_args" + cmd = step.with_args.get("cmd", "") + assert "--squash" in cmd + assert "--auto" not in cmd + + +def test_immediate_merge_routes_to_wait_for_immediate_merge(any_recipe) -> None: + step = any_recipe.steps["immediate_merge"] + assert step.on_success == "wait_for_immediate_merge" + + +def test_immediate_merge_failure_routes_to_release_issue_failure(any_recipe) -> None: + step = any_recipe.steps["immediate_merge"] + assert step.on_failure == "release_issue_failure" + + +def test_wait_for_immediate_merge_step_exists(any_recipe) -> None: + assert "wait_for_immediate_merge" in any_recipe.steps + step = any_recipe.steps["wait_for_immediate_merge"] + assert step.tool == "run_python" + + +def test_wait_for_immediate_merge_merged_routes_to_success(any_recipe) -> None: + step = any_recipe.steps["wait_for_immediate_merge"] + merged_cond = next( + (c for c in step.on_result.conditions if c.when and "merged" in c.when), + None, + ) + assert merged_cond is not None + assert merged_cond.route == "release_issue_success" + + +def test_wait_for_immediate_merge_closed_routes_to_conflict_fix(any_recipe) -> None: + step = any_recipe.steps["wait_for_immediate_merge"] + closed_cond = next( + (c for c in step.on_result.conditions if c.when and "closed" in c.when), + None, + ) + assert closed_cond is not None + assert closed_cond.route == "immediate_merge_conflict_fix" + + +def test_immediate_merge_conflict_fix_exists(any_recipe) -> None: + assert "immediate_merge_conflict_fix" in any_recipe.steps + step = any_recipe.steps["immediate_merge_conflict_fix"] + assert step.tool == "run_python" + + +def test_immediate_merge_conflict_fix_clean_routes_to_re_push(any_recipe) -> None: + step = any_recipe.steps["immediate_merge_conflict_fix"] + clean_route = next( + (c.route for c in step.on_result.conditions if c.when and "clean" in c.when), + None, + ) + assert clean_route == "re_push_immediate_fix" + + +def test_resolve_immediate_merge_conflicts_exists_with_run_skill(any_recipe) -> None: + assert "resolve_immediate_merge_conflicts" in any_recipe.steps + assert any_recipe.steps["resolve_immediate_merge_conflicts"].tool == "run_skill" + + +def test_re_push_immediate_fix_exists(any_recipe) -> None: + assert "re_push_immediate_fix" in any_recipe.steps + step = any_recipe.steps["re_push_immediate_fix"] + assert step.tool == "push_to_remote" + assert step.on_success == "remerge_immediate" + + +def test_remerge_immediate_exists(any_recipe) -> None: + assert "remerge_immediate" in any_recipe.steps + step = any_recipe.steps["remerge_immediate"] + assert step.tool == "run_cmd" + assert step.on_success == "wait_for_immediate_merge" + + +def test_remerge_immediate_uses_squash_without_auto(any_recipe) -> None: + """remerge_immediate must also use --squash without --auto.""" + step = any_recipe.steps["remerge_immediate"] + cmd = step.with_args.get("cmd", "") + assert "--squash" in cmd + assert "--auto" not in cmd + + +def test_all_immediate_merge_steps_have_skip_when_false(any_recipe) -> None: + immediate_steps = [ + "immediate_merge", + "wait_for_immediate_merge", + "immediate_merge_conflict_fix", + "resolve_immediate_merge_conflicts", + "re_push_immediate_fix", + "remerge_immediate", + ] + for step_name in immediate_steps: + assert step_name in any_recipe.steps, f"Missing step: {step_name}" + step = any_recipe.steps[step_name] + assert step.skip_when_false == "inputs.open_pr", ( + f"{step_name}.skip_when_false must be 'inputs.open_pr'" + ) + + +def test_auto_merge_ingredient_description_updated(any_recipe) -> None: + ing = any_recipe.ingredients["auto_merge"] + assert "direct merge" in ing.description.lower() or "direct" in ing.description.lower(), ( + "auto_merge description must mention direct merge as an alternative to queue" + ) + + +# --------------------------------------------------------------------------- +# Routing matrix exhaustiveness — queue_available × auto_merge_available +# --------------------------------------------------------------------------- + + +def test_route_queue_mode_queue_routes_to_enqueue_to_queue(any_recipe) -> None: + """queue+merge_group_trigger cell must route to enqueue_to_queue (unified).""" + step = any_recipe.steps["route_queue_mode"] + cond = next( + c + for c in step.on_result.conditions + if c.when and "queue_available" in c.when and "merge_group_trigger" in c.when + ) + assert cond.route == "enqueue_to_queue" + # The unified route must NOT split on auto_merge_available + assert "auto_merge_available" not in (cond.when or "") + + +def test_route_queue_mode_no_queue_with_auto_routes_to_direct_merge(any_recipe) -> None: + """no-queue+auto cell must route to direct_merge.""" + step = any_recipe.steps["route_queue_mode"] + assert step.on_result is not None, "route_queue_mode must have on_result routing" + cond = next( + ( + c + for c in step.on_result.conditions + if c.when and "auto_merge_available" in c.when and "queue_available" not in c.when + ), + None, + ) + assert cond is not None, "Expected a no-queue+auto condition in route_queue_mode" + assert cond.route == "direct_merge" + + +def test_route_queue_mode_no_queue_no_auto_falls_through_to_immediate_merge( + any_recipe, +) -> None: + """Default (when is None) condition must route to immediate_merge.""" + step = any_recipe.steps["route_queue_mode"] + assert step.on_result is not None, "route_queue_mode must have on_result routing" + cond = next((c for c in step.on_result.conditions if c.when is None), None) + assert cond is not None, "Expected a default (when=None) condition in route_queue_mode" + assert cond.route == "immediate_merge" + + +def test_enqueue_to_queue_route_count(any_recipe) -> None: + """Exactly one condition must route to enqueue_to_queue.""" + step = any_recipe.steps["route_queue_mode"] + assert step.on_result is not None, "route_queue_mode must have on_result routing" + count = sum(1 for c in step.on_result.conditions if c.route == "enqueue_to_queue") + assert count == 1, f"Expected exactly 1 enqueue_to_queue route, got {count}" + + +# --------------------------------------------------------------------------- +# Unified enrollment step: enqueue_to_queue (replaces enable_auto_merge + queue_enqueue_no_auto) +# --------------------------------------------------------------------------- + + +def test_enqueue_to_queue_uses_enqueue_pr_tool(any_recipe) -> None: + """The unified enrollment step must use the enqueue_pr tool, not run_cmd.""" + step = any_recipe.steps["enqueue_to_queue"] + assert step.tool == "enqueue_pr" + + +def test_enqueue_to_queue_passes_auto_merge_available(any_recipe) -> None: + """enqueue_to_queue must pass context.auto_merge_available to the tool.""" + step = any_recipe.steps["enqueue_to_queue"] + assert "auto_merge_available" in (step.with_args or {}) + assert "context.auto_merge_available" in (step.with_args or {}).get("auto_merge_available", "") + + +def test_reenter_merge_queue_uses_enqueue_pr_tool(any_recipe) -> None: + """Re-entry after ejection must use enqueue_pr tool.""" + step = any_recipe.steps["reenter_merge_queue"] + assert step.tool == "enqueue_pr" + + +def test_reenter_merge_queue_cheap_uses_enqueue_pr_tool(any_recipe) -> None: + """Re-entry after drop must use enqueue_pr tool.""" + step = any_recipe.steps["reenter_merge_queue_cheap"] + assert step.tool == "enqueue_pr" + + +def test_reenroll_stalled_pr_uses_enqueue_pr_tool(any_recipe) -> None: + """Stall recovery must use enqueue_pr tool, not toggle_auto_merge.""" + step = any_recipe.steps["reenroll_stalled_pr"] + assert step.tool == "enqueue_pr" + + +def test_no_gh_pr_merge_in_queue_enrollment_steps(any_recipe) -> None: + """Every enrollment step must use enqueue_pr (not run_cmd with gh pr merge).""" + enrollment_steps = [ + "enqueue_to_queue", + "reenter_merge_queue", + "reenter_merge_queue_cheap", + "reenroll_stalled_pr", + ] + for step_name in enrollment_steps: + step = any_recipe.steps.get(step_name) + if step is None: + continue + assert step.tool == "enqueue_pr", ( + f"{step_name} must use enqueue_pr tool, got {step.tool!r}" + ) + + +def test_no_enable_auto_merge_step_exists(any_recipe) -> None: + """enable_auto_merge step must be removed (replaced by enqueue_to_queue).""" + assert "enable_auto_merge" not in any_recipe.steps + + +def test_no_queue_enqueue_no_auto_step_exists(any_recipe) -> None: + """queue_enqueue_no_auto step must be removed (merged into enqueue_to_queue).""" + assert "queue_enqueue_no_auto" not in any_recipe.steps + + +# --------------------------------------------------------------------------- +# Merge step failure routing — silent success degradation guards +# --------------------------------------------------------------------------- + + +def test_enqueue_to_queue_failure_routes_to_verify_queue_enrollment(any_recipe) -> None: + """enqueue_to_queue on_failure must route to verify_queue_enrollment.""" + step = any_recipe.steps["enqueue_to_queue"] + assert step.on_failure == "verify_queue_enrollment" + + +def test_verify_queue_enrollment_exists(any_recipe) -> None: + assert "verify_queue_enrollment" in any_recipe.steps + + +def test_verify_queue_enrollment_uses_wait_for_merge_queue(any_recipe) -> None: + step = any_recipe.steps["verify_queue_enrollment"] + assert step.tool == "wait_for_merge_queue" + + +def test_verify_queue_enrollment_on_failure_escalates(any_recipe) -> None: + step = any_recipe.steps["verify_queue_enrollment"] + assert step.on_failure == "register_clone_unconfirmed" + + +def test_verify_queue_enrollment_merged_routes_to_release_issue_success(any_recipe) -> None: + step = any_recipe.steps["verify_queue_enrollment"] + assert step.on_result is not None + merged_routes = [c.route for c in step.on_result.conditions if c.when and "merged" in c.when] + assert merged_routes == ["release_issue_success"] + + +def test_verify_queue_enrollment_fallback_routes_to_register_clone_unconfirmed(any_recipe) -> None: + step = any_recipe.steps["verify_queue_enrollment"] + assert step.on_result is not None + fallback = [c.route for c in step.on_result.conditions if c.when is None] + assert fallback == ["register_clone_unconfirmed"] + + +def test_verify_queue_enrollment_ejected_ci_failure_routes_directly_to_diagnose_ci( + any_recipe, +) -> None: + """verify_queue_enrollment must route ejected_ci_failure directly to diagnose_ci. + + A 60s probe that already confirmed CI failure should not feed into a 900s + wait_for_queue watch that would only route to diagnose_ci anyway. + """ + step = any_recipe.steps["verify_queue_enrollment"] + assert step.on_result is not None + ejected_ci_routes = [ + c.route + for c in step.on_result.conditions + if c.when is not None and "ejected_ci_failure" in c.when + ] + assert ejected_ci_routes == ["diagnose_ci"], ( + f"verify_queue_enrollment must route ejected_ci_failure directly to diagnose_ci, " + f"got: {ejected_ci_routes}" + ) + + +@pytest.mark.parametrize("recipe_fixture", RELEASE_TIMEOUT_RECIPES) +def test_wait_for_direct_merge_on_failure_routes_to_register_clone_unconfirmed( + recipe_fixture: str, request +) -> None: + """wait_for_direct_merge.on_failure must be register_clone_unconfirmed in all three recipes.""" + recipe = request.getfixturevalue(recipe_fixture) + step = recipe.steps["wait_for_direct_merge"] + assert step.on_failure == "register_clone_unconfirmed", ( + f"{recipe_fixture}: wait_for_direct_merge.on_failure must be " + f"'register_clone_unconfirmed', got: {step.on_failure!r}" + ) + + +# --------------------------------------------------------------------------- +# Full routing parity — every PRState covered (universal + family-specific) +# --------------------------------------------------------------------------- + +_REQUIRED_PR_STATE_VALUES = frozenset( + s.value for s in PRState if s not in {PRState.ERROR, PRState.NOT_ENROLLED} +) +_PR_STATE_WHEN_RE = re.compile(r"\$\{\{\s*result\.pr_state\s*\}\}\s*==\s*(\w+)") + + +@pytest.mark.parametrize("recipe_fixture", QUEUE_RECIPES) +def test_wait_for_queue_routing_covers_every_pr_state(recipe_fixture, request) -> None: + """Every wait_for_merge_queue step must cover every non-error PRState.""" + recipe = request.getfixturevalue(recipe_fixture) + + # Find wait_for_merge_queue steps dynamically + mq_steps = { + name: step + for name, step in recipe.steps.items() + if step.tool == "wait_for_merge_queue" + and step.on_result is not None + and step.on_result.conditions + } + assert mq_steps, f"{recipe_fixture}: no wait_for_merge_queue step with on_result found" + + for step_name, step in mq_steps.items(): + conditions = step.on_result.conditions + + covered: set[str] = set() + for c in conditions: + if c.when is None: + continue + m = _PR_STATE_WHEN_RE.search(c.when) + if m: + covered.add(m.group(1)) + + missing = _REQUIRED_PR_STATE_VALUES - covered + assert not missing, ( + f"{recipe_fixture}: {step_name}.on_result is missing explicit routing arms " + f"for PRState values: {sorted(missing)}. Every non-error PRState must have a " + f"when condition." + ) + + # ejected_ci_failure must precede generic ejected + whens = [c.when or "" for c in conditions] + ci_fail_idx = next((i for i, w in enumerate(whens) if "ejected_ci_failure" in w), None) + ejected_idx = next( + (i for i, w in enumerate(whens) if w.strip() == "${{ result.pr_state }} == ejected"), + None, + ) + assert ci_fail_idx is not None, ( + f"{recipe_fixture}: {step_name} ejected_ci_failure route must exist" + ) + assert ejected_idx is not None, f"{recipe_fixture}: {step_name} ejected route must exist" + assert ci_fail_idx < ejected_idx, ( + f"{recipe_fixture}: {step_name} ejected_ci_failure route must appear " + f"before generic ejected route" + ) + + +# --------------------------------------------------------------------------- +# Auto-discovery structural guards +# --------------------------------------------------------------------------- + + +def test_auto_discovery_includes_merge_prs() -> None: + """merge-prs.yaml must appear in the auto-discovered queue recipe list.""" + assert "pmp_recipe" in QUEUE_RECIPES, ( + "merge-prs.yaml (pmp_recipe) must be in QUEUE_RECIPES — if this fails, " + "someone removed wait_for_merge_queue from merge-prs.yaml" + ) + + +def test_auto_discovery_matches_known_queue_recipes() -> None: + """Auto-discovered queue recipes must match the expected set exactly.""" + expected = {"impl_recipe", "remed_recipe", "impl_groups_recipe", "pmp_recipe"} + actual = set(QUEUE_RECIPES) + assert actual == expected, ( + f"QUEUE_RECIPES mismatch — expected {sorted(expected)}, got {sorted(actual)}. " + f"Missing: {sorted(expected - actual)}, Extra: {sorted(actual - expected)}" + ) + + +# --------------------------------------------------------------------------- +# NOT_ENROLLED routing — all queue-capable recipes +# --------------------------------------------------------------------------- + + +def test_wait_for_queue_routes_not_enrolled(any_recipe) -> None: + """wait_for_queue must have an explicit routing arm for not_enrolled.""" + step = any_recipe.steps["wait_for_queue"] + conditions = [c.when for c in step.on_result.conditions] + assert any("not_enrolled" in c for c in conditions if c) + + +def test_verify_queue_enrollment_routes_not_enrolled(any_recipe) -> None: + """verify_queue_enrollment must route not_enrolled state.""" + step = any_recipe.steps["verify_queue_enrollment"] + conditions = [c.when for c in step.on_result.conditions] + assert any("not_enrolled" in c for c in conditions if c) + + +# --------------------------------------------------------------------------- +# any_recipe — hardcoded-origin semantic rule +# --------------------------------------------------------------------------- + + +def test_no_hardcoded_origin_in_run_cmd_queue_capable(any_recipe) -> None: + """After REMOTE probe fix, run_semantic_rules must report zero hardcoded-origin-in-run-cmd.""" + from autoskillit.recipe.validator import run_semantic_rules + + findings = run_semantic_rules(any_recipe) + violations = [f for f in findings if f.rule == "hardcoded-origin-in-run-cmd"] + assert violations == [], ( + f"hardcoded-origin-in-run-cmd fired on {any_recipe.name}: " + f"{[v.step_name for v in violations]}" + ) + + +# --------------------------------------------------------------------------- +# any_recipe — check_eject_limit +# --------------------------------------------------------------------------- + + +def test_check_eject_limit_step_exists_in_queue_capable(any_recipe) -> None: + """check_eject_limit step must exist in each queue-capable recipe.""" + assert "check_eject_limit" in any_recipe.steps, ( + f"check_eject_limit step missing from {any_recipe.name}" + ) + + +def test_check_eject_limit_routes_to_queue_ejected_fix(any_recipe) -> None: + """check_eject_limit default route must point to queue_ejected_fix.""" + step = any_recipe.steps["check_eject_limit"] + assert step.on_result is not None + conds = step.on_result.conditions + default_conds = [c for c in conds if c.when is None] + assert len(default_conds) == 1, "check_eject_limit must have exactly one default route" + assert default_conds[0].route == "queue_ejected_fix" + + +def test_check_eject_limit_routes_to_failure_when_exceeded(any_recipe) -> None: + """check_eject_limit must route to release_issue_failure when EJECT_LIMIT_EXCEEDED.""" + step = any_recipe.steps["check_eject_limit"] + assert step.on_result is not None + conds = step.on_result.conditions + limit_conds = [c for c in conds if c.when and "EJECT_LIMIT_EXCEEDED" in c.when] + assert len(limit_conds) == 1 + assert limit_conds[0].route == "release_issue_failure" + + +def test_check_eject_limit_callable_references_counter_file(any_recipe) -> None: + """check_eject_limit callable must receive counter_file arg under .autoskillit/temp/.""" + step = any_recipe.steps["check_eject_limit"] + counter_file = step.with_args.get("counter_file", "") + assert "eject_count" in counter_file, "counter_file must reference eject_count" + assert ".autoskillit/temp" in counter_file, "counter_file must be under .autoskillit/temp" + + +def test_check_eject_limit_callable_uses_limit_3(any_recipe) -> None: + """check_eject_limit callable must cap at 3 ejections.""" + step = any_recipe.steps["check_eject_limit"] + assert step.with_args.get("max_ejects") == "3", "max_ejects must be '3'" + + +def test_check_eject_limit_on_result_uses_exact_eq_match(any_recipe) -> None: + """check_eject_limit on_result must use exact == match (consistent with recipe patterns).""" + step = any_recipe.steps["check_eject_limit"] + assert step.on_result is not None + limit_conds = [ + c for c in step.on_result.conditions if c.when and "EJECT_LIMIT_EXCEEDED" in c.when + ] + assert len(limit_conds) == 1 + assert "==" in limit_conds[0].when, "predicate must use exact == match" + + +def test_unbounded_cycle_severity_downgraded_by_eject_limit(any_recipe) -> None: + """After check_eject_limit, unbounded-cycle must not fire ERROR for queue ejection cycle. + + check_eject_limit is a run_python step in _STRUCTURAL_ON_RESULT_TOOLS with an on_result + condition routing to release_issue_failure (outside the cycle). The has_on_result_exit + path in rules_graph.py recognises this as a structural bound and suppresses the finding + entirely — so zero unbounded-cycle findings are expected for the queue ejection cycle. + """ + from autoskillit.core.types import Severity + from autoskillit.recipe.validator import run_semantic_rules + + findings = run_semantic_rules(any_recipe) + cycle_findings = [f for f in findings if f.rule == "unbounded-cycle"] + queue_cycle_error_findings = [ + f + for f in cycle_findings + if f.severity == Severity.ERROR + and any( + kw in f.message for kw in ("wait_for_queue", "queue_ejected_fix", "check_eject_limit") + ) + ] + assert queue_cycle_error_findings == [], ( + f"unbounded-cycle must not be ERROR for queue ejection cycle after check_eject_limit; " + f"got ERROR on: {[f.step_name for f in queue_cycle_error_findings]}" + ) diff --git a/tests/recipe/test_merge_prs_queue_pmp.py b/tests/recipe/test_merge_prs_queue_pmp.py new file mode 100644 index 0000000000..5933376515 --- /dev/null +++ b/tests/recipe/test_merge_prs_queue_pmp.py @@ -0,0 +1,535 @@ +"""Queue mode structural assertions for merge-prs.yaml.""" + +from __future__ import annotations + +import pytest + +from autoskillit.recipe.io import builtin_recipes_dir, load_recipe +from autoskillit.recipe.validator import validate_recipe_structure + +pytestmark = [pytest.mark.layer("recipe"), pytest.mark.small] + +# --------------------------------------------------------------------------- +# Fixtures +# --------------------------------------------------------------------------- + + +@pytest.fixture(scope="module") +def pmp_recipe(): + return load_recipe(builtin_recipes_dir() / "merge-prs.yaml") + + +# --------------------------------------------------------------------------- +# merge-prs.yaml — validate passes +# --------------------------------------------------------------------------- + + +def test_merge_prs_queue_recipe_is_valid(pmp_recipe) -> None: + """validate_recipe_structure must pass with no errors after queue steps are added.""" + errors = validate_recipe_structure(pmp_recipe) + assert errors == [], f"validate_recipe_structure errors: {errors}" + + +# --------------------------------------------------------------------------- +# merge-prs.yaml — new queue mode steps present +# --------------------------------------------------------------------------- + + +def test_merge_prs_route_by_queue_mode_exists(pmp_recipe) -> None: + """route_by_queue_mode step must exist in merge-prs.""" + assert "route_by_queue_mode" in pmp_recipe.steps + + +def test_merge_prs_route_by_queue_mode_is_route_action(pmp_recipe) -> None: + """route_by_queue_mode must be an action: route step.""" + step = pmp_recipe.steps["route_by_queue_mode"] + assert step.action == "route" + + +def test_merge_prs_enqueue_current_pr_exists(pmp_recipe) -> None: + """enqueue_current_pr step must exist with tool=enqueue_pr.""" + assert "enqueue_current_pr" in pmp_recipe.steps + step = pmp_recipe.steps["enqueue_current_pr"] + assert step.tool == "enqueue_pr" + + +def test_merge_prs_wait_queue_pr_exists(pmp_recipe) -> None: + """wait_queue_pr step must exist with tool=wait_for_merge_queue.""" + assert "wait_queue_pr" in pmp_recipe.steps + step = pmp_recipe.steps["wait_queue_pr"] + assert step.tool == "wait_for_merge_queue" + + +def test_merge_prs_reenter_queue_exists(pmp_recipe) -> None: + """reenter_queue step must exist.""" + assert "reenter_queue" in pmp_recipe.steps + + +def test_merge_prs_advance_queue_pr_exists(pmp_recipe) -> None: + """advance_queue_pr step must exist with tool=run_python.""" + assert "advance_queue_pr" in pmp_recipe.steps + step = pmp_recipe.steps["advance_queue_pr"] + assert step.tool == "run_python" + + +# --------------------------------------------------------------------------- +# merge-prs.yaml — analyze_prs captures queue_mode +# --------------------------------------------------------------------------- + + +def test_merge_prs_analyze_prs_captures_queue_mode(pmp_recipe) -> None: + """analyze_prs must capture queue_mode from the skill result.""" + step = pmp_recipe.steps["analyze_prs"] + assert "queue_mode" in (step.capture or {}), ( + "analyze_prs must capture queue_mode to enable route_by_queue_mode" + ) + + +def test_merge_prs_analyze_prs_routes_to_route_by_queue_mode(pmp_recipe) -> None: + """analyze_prs.on_success must route to route_by_queue_mode.""" + step = pmp_recipe.steps["analyze_prs"] + assert step.on_success == "route_by_queue_mode" + + +# --------------------------------------------------------------------------- +# merge-prs.yaml — classic path still intact +# --------------------------------------------------------------------------- + + +def test_merge_prs_classic_path_create_batch_branch_present(pmp_recipe) -> None: + """create_batch_branch step must still be present (classic path).""" + assert "create_batch_branch" in pmp_recipe.steps + + +def test_merge_prs_classic_path_merge_pr_present(pmp_recipe) -> None: + """merge_pr step must still be present (classic path).""" + assert "merge_pr" in pmp_recipe.steps + + +def test_merge_prs_classic_path_push_batch_branch_present(pmp_recipe) -> None: + """push_batch_branch step must still be present (classic path).""" + assert "push_batch_branch" in pmp_recipe.steps + + +# --------------------------------------------------------------------------- +# merge-prs.yaml — sequential enqueue behavioral (Test 1A) +# --------------------------------------------------------------------------- + + +def test_merge_prs_enqueue_all_prs_removed(pmp_recipe) -> None: + """enqueue_all_prs batch step must be removed.""" + assert "enqueue_all_prs" not in pmp_recipe.steps + + +def test_merge_prs_enqueue_current_pr_routes_to_wait(pmp_recipe) -> None: + """enqueue_current_pr.on_success must route to wait_queue_pr.""" + step = pmp_recipe.steps["enqueue_current_pr"] + assert step.on_success == "wait_queue_pr" + + +def test_merge_prs_enqueue_current_pr_references_single_pr(pmp_recipe) -> None: + """enqueue_current_pr must reference context.current_pr_number (no batch loop).""" + step = pmp_recipe.steps["enqueue_current_pr"] + pr_number = step.with_args.get("pr_number", "") + assert "context.current_pr_number" in pr_number + + +def test_merge_prs_get_first_pr_number_captures_pr_number(pmp_recipe) -> None: + """get_first_pr_number must capture current_pr_number.""" + step = pmp_recipe.steps["get_first_pr_number"] + assert "current_pr_number" in (step.capture or {}) + + +def test_merge_prs_get_first_pr_number_routes_to_enqueue(pmp_recipe) -> None: + """get_first_pr_number.on_success must route to enqueue_current_pr.""" + step = pmp_recipe.steps["get_first_pr_number"] + assert step.on_success == "enqueue_current_pr" + + +# --------------------------------------------------------------------------- +# merge-prs.yaml — cheap rebase pre-check (Test 1C) +# --------------------------------------------------------------------------- + + +def test_merge_prs_attempt_cheap_rebase_exists(pmp_recipe) -> None: + """attempt_cheap_rebase step must exist with tool=run_python.""" + assert "attempt_cheap_rebase" in pmp_recipe.steps + step = pmp_recipe.steps["attempt_cheap_rebase"] + assert step.tool == "run_python" + assert step.with_args.get("callable") == "autoskillit.recipe._cmd_rpc.attempt_cheap_rebase" + + +def test_merge_prs_attempt_cheap_rebase_routing(pmp_recipe) -> None: + """attempt_cheap_rebase clean routes to push_ejected_fix, conflicts to resolve.""" + step = pmp_recipe.steps["attempt_cheap_rebase"] + assert step.on_result is not None + conditions = step.on_result.conditions + clean_routes = [c for c in conditions if c.when and "clean" in c.when] + assert clean_routes, "must have a 'clean' condition" + assert clean_routes[0].route == "push_ejected_fix" + fallback = [c for c in conditions if c.when is None] + assert fallback, "must have a fallback condition" + assert fallback[0].route == "resolve_ejected_conflicts" + + +def test_merge_prs_get_ejected_routes_to_cheap_rebase(pmp_recipe) -> None: + """get_ejected_pr_branch.on_success must route to attempt_cheap_rebase.""" + step = pmp_recipe.steps["get_ejected_pr_branch"] + assert step.on_success == "attempt_cheap_rebase" + + +def test_merge_prs_checkout_ejected_pr_removed(pmp_recipe) -> None: + """checkout_ejected_pr step must be removed (consolidated into attempt_cheap_rebase).""" + assert "checkout_ejected_pr" not in pmp_recipe.steps + + +# --------------------------------------------------------------------------- +# merge-prs.yaml — Proactive rebase block (Test 1E) +# --------------------------------------------------------------------------- + + +def test_merge_prs_advance_queue_pr_fallthrough_routes_to_get_next_pr_branch( + pmp_recipe, +) -> None: + step = pmp_recipe.steps["advance_queue_pr"] + assert step.on_result is not None + fallback_route = next( + (c.route for c in step.on_result.conditions if c.when is None), + None, + ) + assert fallback_route == "get_next_pr_branch" + + +def test_merge_prs_get_next_pr_branch_exists(pmp_recipe) -> None: + assert "get_next_pr_branch" in pmp_recipe.steps + assert pmp_recipe.steps["get_next_pr_branch"].tool == "run_cmd" + + +def test_merge_prs_get_next_pr_branch_cmd_fetches_headref(pmp_recipe) -> None: + cmd = pmp_recipe.steps["get_next_pr_branch"].with_args.get("cmd", "") + assert "gh pr view" in cmd + assert "headRefName" in cmd + + +def test_merge_prs_get_next_pr_branch_captures_next_pr_branch(pmp_recipe) -> None: + step = pmp_recipe.steps["get_next_pr_branch"] + assert "next_pr_branch" in (step.capture or {}) + + +def test_merge_prs_get_next_pr_branch_on_success_routes_to_proactive_rebase( + pmp_recipe, +) -> None: + assert pmp_recipe.steps["get_next_pr_branch"].on_success == "proactive_rebase_next_pr" + + +def test_merge_prs_get_next_pr_branch_on_failure_routes_to_enqueue(pmp_recipe) -> None: + # Safety net: if gh pr view fails, fall through to enqueue as if no rebase happened + assert pmp_recipe.steps["get_next_pr_branch"].on_failure == "enqueue_current_pr" + + +def test_merge_prs_proactive_rebase_next_pr_exists(pmp_recipe) -> None: + assert "proactive_rebase_next_pr" in pmp_recipe.steps + step = pmp_recipe.steps["proactive_rebase_next_pr"] + assert step.tool == "run_python" + assert step.with_args.get("callable") == "autoskillit.recipe._cmd_rpc.proactive_rebase_next_pr" + + +def test_merge_prs_proactive_rebase_next_pr_routing(pmp_recipe) -> None: + step = pmp_recipe.steps["proactive_rebase_next_pr"] + assert step.on_result is not None + clean_route = next( + (c.route for c in step.on_result.conditions if c.when and "clean" in c.when), + None, + ) + assert clean_route == "push_rebased_next_pr" + fallback_route = next( + (c.route for c in step.on_result.conditions if c.when is None), + None, + ) + assert fallback_route == "resolve_proactive_rebase_conflicts" + + +def test_merge_prs_proactive_rebase_next_pr_on_failure_routes_to_enqueue(pmp_recipe) -> None: + # Safety net: git command errors fall through to enqueue (don't block the pipeline) + assert pmp_recipe.steps["proactive_rebase_next_pr"].on_failure == "enqueue_current_pr" + + +def test_merge_prs_push_rebased_next_pr_exists(pmp_recipe) -> None: + step = pmp_recipe.steps["push_rebased_next_pr"] + assert step.tool == "push_to_remote" + assert step.with_args.get("force") == "true" + + +def test_merge_prs_push_rebased_next_pr_routes_to_enqueue(pmp_recipe) -> None: + assert pmp_recipe.steps["push_rebased_next_pr"].on_success == "enqueue_current_pr" + + +def test_merge_prs_push_rebased_next_pr_on_failure_routes_to_register_failure( + pmp_recipe, +) -> None: + assert pmp_recipe.steps["push_rebased_next_pr"].on_failure == "register_clone_failure" + + +def test_merge_prs_resolve_proactive_conflicts_exists(pmp_recipe) -> None: + step = pmp_recipe.steps["resolve_proactive_rebase_conflicts"] + assert step.tool == "run_skill" + assert "resolve-merge-conflicts" in step.with_args.get("skill_command", "") + + +def test_merge_prs_resolve_proactive_conflicts_routing(pmp_recipe) -> None: + step = pmp_recipe.steps["resolve_proactive_rebase_conflicts"] + assert step.on_result is not None + escalation_route = next( + (c.route for c in step.on_result.conditions if c.when and "escalation_required" in c.when), + None, + ) + assert escalation_route == "register_clone_failure" + success_route = next( + (c.route for c in step.on_result.conditions if c.when is None), + None, + ) + assert success_route == "push_rebased_next_pr" + + +def test_merge_prs_resolve_proactive_conflicts_has_retries(pmp_recipe) -> None: + step = pmp_recipe.steps["resolve_proactive_rebase_conflicts"] + assert step.retries == 1 + assert step.on_exhausted == "enqueue_current_pr" + + +# --------------------------------------------------------------------------- +# merge-prs.yaml — CI watch before reenter_queue (Test 1D) +# --------------------------------------------------------------------------- + + +def test_merge_prs_ci_watch_post_queue_fix_exists(pmp_recipe) -> None: + """ci_watch_post_queue_fix step must exist in merge-prs.yaml.""" + assert "ci_watch_post_queue_fix" in pmp_recipe.steps + + +def test_merge_prs_push_ejected_fix_routes_to_ci_watch(pmp_recipe) -> None: + """push_ejected_fix.on_success must route to ci_watch_post_queue_fix.""" + step = pmp_recipe.steps["push_ejected_fix"] + assert step.on_success == "ci_watch_post_queue_fix" + + +def test_merge_prs_ci_watch_routes_to_reenter(pmp_recipe) -> None: + """ci_watch_post_queue_fix on_result success must route to reenter_queue.""" + step = pmp_recipe.steps["ci_watch_post_queue_fix"] + assert step.on_result is not None, "ci_watch_post_queue_fix must use on_result routing" + success_routes = [ + c.route for c in step.on_result.conditions if c.when and "'success'" in c.when + ] + assert "reenter_queue" in success_routes + + +# --------------------------------------------------------------------------- +# merge-prs.yaml — recipe-level capture for advancement (Test 1E) +# --------------------------------------------------------------------------- + + +def test_merge_prs_advance_queue_pr_is_run_python(pmp_recipe) -> None: + """advance_queue_pr step must exist with tool=run_python.""" + assert "advance_queue_pr" in pmp_recipe.steps + step = pmp_recipe.steps["advance_queue_pr"] + assert step.tool == "run_python" + + +def test_merge_prs_next_queue_pr_or_done_removed(pmp_recipe) -> None: + """next_queue_pr_or_done step must be removed (replaced by advance_queue_pr).""" + assert "next_queue_pr_or_done" not in pmp_recipe.steps + + +def test_merge_prs_advance_queue_pr_callable_references_pr_order(pmp_recipe) -> None: + """advance_queue_pr callable args must reference pr_order_file and current_pr_number.""" + step = pmp_recipe.steps["advance_queue_pr"] + args = step.with_args + assert "pr_order_file" in args + assert "current_pr_number" in args + + +def test_merge_prs_advance_queue_pr_captures_pr_number(pmp_recipe) -> None: + """advance_queue_pr must have a capture block for current_pr_number.""" + step = pmp_recipe.steps["advance_queue_pr"] + capture = step.capture or {} + assert "current_pr_number" in capture + + +def test_merge_prs_advance_queue_pr_routing(pmp_recipe) -> None: + """advance_queue_pr routes to get_next_pr_branch (default) or collect_and_check_impl_plans.""" + step = pmp_recipe.steps["advance_queue_pr"] + assert step.on_result is not None + conditions = step.on_result.conditions + done_routes = [c for c in conditions if c.when and "done" in c.when] + assert done_routes, "must have a 'done' condition" + assert done_routes[0].route == "collect_and_check_impl_plans" + default_routes = [c for c in conditions if c.when is None] + assert default_routes, "must have a default route" + assert default_routes[0].route == "get_next_pr_branch" + + +# --------------------------------------------------------------------------- +# merge-prs.yaml — new PRState route steps (Test 1F) +# --------------------------------------------------------------------------- + + +def test_merge_prs_diagnose_queue_ci_exists(pmp_recipe) -> None: + """diagnose_queue_ci step must exist with tool=run_skill.""" + assert "diagnose_queue_ci" in pmp_recipe.steps + step = pmp_recipe.steps["diagnose_queue_ci"] + assert step.tool == "run_skill" + + +def test_merge_prs_reenroll_stalled_queue_pr_exists(pmp_recipe) -> None: + """reenroll_stalled_queue_pr step must exist with tool=enqueue_pr.""" + assert "reenroll_stalled_queue_pr" in pmp_recipe.steps + step = pmp_recipe.steps["reenroll_stalled_queue_pr"] + assert step.tool == "enqueue_pr" + + +def test_merge_prs_reenroll_stalled_routes_to_wait(pmp_recipe) -> None: + """reenroll_stalled_queue_pr must route through the stall loop guard.""" + step = pmp_recipe.steps["reenroll_stalled_queue_pr"] + assert step.on_success == "check_queue_stall_loop" + + +def test_merge_prs_dropped_healthy_routes_through_circuit_breaker(pmp_recipe) -> None: + """dropped_healthy in wait_queue_pr must route to check_dropped_healthy_loop, + which in turn routes to reenter_queue on DROPPED_OK.""" + assert "check_dropped_healthy_loop" in pmp_recipe.steps, ( + "check_dropped_healthy_loop step must exist in merge-prs recipe" + ) + step = pmp_recipe.steps["wait_queue_pr"] + assert step.on_result is not None + dropped_routes = [ + c + for c in step.on_result.conditions + if c.when is not None + and "dropped_healthy" in c.when + and c.route == "check_dropped_healthy_loop" + ] + assert dropped_routes, "dropped_healthy must route to check_dropped_healthy_loop" + cb_step = pmp_recipe.steps["check_dropped_healthy_loop"] + assert cb_step.on_result is not None + ok_routes = [c for c in cb_step.on_result.conditions if c.when is None] + assert ok_routes and ok_routes[0].route == "reenter_queue", ( + "check_dropped_healthy_loop fallthrough must route to reenter_queue" + ) + + +# --------------------------------------------------------------------------- +# merge-prs.yaml — Part B: expanded captures + unified enrollment +# --------------------------------------------------------------------------- + + +def test_merge_prs_check_repo_ci_event_captures_auto_merge_available(pmp_recipe) -> None: + """check_repo_ci_event must capture auto_merge_available (expanded captures).""" + step = pmp_recipe.steps["check_repo_ci_event"] + assert step.tool == "check_repo_merge_state" + assert "auto_merge_available" in (step.capture or {}) + + +def test_merge_prs_enqueue_current_pr_uses_enqueue_pr_tool(pmp_recipe) -> None: + """enqueue_current_pr must use enqueue_pr tool.""" + step = pmp_recipe.steps["enqueue_current_pr"] + assert step.tool == "enqueue_pr" + + +def test_merge_prs_reenter_queue_uses_enqueue_pr_tool(pmp_recipe) -> None: + """reenter_queue must use enqueue_pr tool.""" + step = pmp_recipe.steps["reenter_queue"] + assert step.tool == "enqueue_pr" + + +def test_merge_prs_reenroll_stalled_queue_pr_uses_enqueue_pr_tool(pmp_recipe) -> None: + """reenroll_stalled_queue_pr must use enqueue_pr tool.""" + step = pmp_recipe.steps["reenroll_stalled_queue_pr"] + assert step.tool == "enqueue_pr" + + +# --------------------------------------------------------------------------- +# NOT_ENROLLED routing — merge-prs.yaml only +# --------------------------------------------------------------------------- + + +def test_merge_prs_wait_queue_pr_routes_not_enrolled(pmp_recipe) -> None: + """wait_queue_pr must have an explicit routing arm for not_enrolled.""" + step = pmp_recipe.steps["wait_queue_pr"] + conditions = [c.when for c in step.on_result.conditions] + assert any("not_enrolled" in c for c in conditions if c) + + +# --------------------------------------------------------------------------- +# merge-prs.yaml — hardcoded-origin semantic rule +# --------------------------------------------------------------------------- + + +def test_no_hardcoded_origin_in_run_cmd_merge_prs(pmp_recipe) -> None: + """merge-prs.yaml setup_remote suppresses hardcoded-origin-in-run-cmd recipe-wide.""" + from autoskillit.recipe.validator import run_semantic_rules + + findings = run_semantic_rules(pmp_recipe) + violations = [f for f in findings if f.rule == "hardcoded-origin-in-run-cmd"] + assert violations == [], ( + f"hardcoded-origin-in-run-cmd fired on merge-prs.yaml: {[v.step_name for v in violations]}" + ) + + +# --------------------------------------------------------------------------- +# merge-prs.yaml — check_eject_limit +# --------------------------------------------------------------------------- + + +def test_check_eject_limit_step_exists_in_merge_prs(pmp_recipe) -> None: + """check_eject_limit step must exist in merge-prs.yaml.""" + assert "check_eject_limit" in pmp_recipe.steps + + +def test_check_eject_limit_routes_to_get_ejected_pr_branch_in_merge_prs(pmp_recipe) -> None: + """check_eject_limit default route in merge-prs.yaml must point to get_ejected_pr_branch.""" + step = pmp_recipe.steps["check_eject_limit"] + assert step.on_result is not None + conds = step.on_result.conditions + default_conds = [c for c in conds if c.when is None] + assert len(default_conds) == 1 + assert default_conds[0].route == "get_ejected_pr_branch" + + +def test_check_eject_limit_routes_to_register_clone_failure_in_merge_prs(pmp_recipe) -> None: + """check_eject_limit in merge-prs.yaml must route to register_clone_failure on limit.""" + step = pmp_recipe.steps["check_eject_limit"] + assert step.on_result is not None + conds = step.on_result.conditions + limit_conds = [c for c in conds if c.when and "EJECT_LIMIT_EXCEEDED" in c.when] + assert len(limit_conds) == 1 + assert limit_conds[0].route == "register_clone_failure" + + +def test_merge_prs_ci_watch_post_queue_fix_no_runs_routes_to_failure(pmp_recipe) -> None: + """ci_watch_post_queue_fix no_runs must route to register_clone_failure in merge-prs.""" + step = pmp_recipe.steps["ci_watch_post_queue_fix"] + no_runs_routes = [ + c.route for c in step.on_result.conditions if c.when and "'no_runs'" in c.when + ] + assert "register_clone_failure" in no_runs_routes + + +def test_merge_prs_ci_watch_post_queue_fix_uses_branch_param(pmp_recipe) -> None: + """ci_watch_post_queue_fix must use 'branch' parameter, not 'pr_number'.""" + step = pmp_recipe.steps["ci_watch_post_queue_fix"] + assert "branch" in step.with_args, ( + "ci_watch_post_queue_fix must use 'branch' parameter, not 'pr_number'" + ) + assert "pr_number" not in step.with_args, ( + "ci_watch_post_queue_fix must not pass pr_number to wait_for_ci" + ) + + +def test_merge_prs_ci_watch_post_queue_fix_timed_out_bounded(pmp_recipe) -> None: + """ci_watch_post_queue_fix timed_out must route through check_ci_post_queue_loop.""" + step = pmp_recipe.steps["ci_watch_post_queue_fix"] + timed_out_routes = [ + c.route for c in step.on_result.conditions if c.when and "'timed_out'" in c.when + ] + assert timed_out_routes, "ci_watch_post_queue_fix: missing timed_out routing" + assert timed_out_routes[0] == "check_ci_post_queue_loop" diff --git a/tests/recipe/test_merge_prs_queue_release_timeout.py b/tests/recipe/test_merge_prs_queue_release_timeout.py new file mode 100644 index 0000000000..2da739cda4 --- /dev/null +++ b/tests/recipe/test_merge_prs_queue_release_timeout.py @@ -0,0 +1,288 @@ +"""Release timeout and retry logic tests for queue-capable recipes that use +register_clone_unconfirmed as the queue error escalation step +(implementation, remediation, implementation-groups).""" + +from __future__ import annotations + +import pytest + +from tests.recipe.test_merge_prs_queue_common import RELEASE_TIMEOUT_RECIPES + +pytestmark = [pytest.mark.layer("recipe"), pytest.mark.small] + + +# --------------------------------------------------------------------------- +# ci_watch_post_queue_fix step + ejected_ci_failure routing +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize("recipe_fixture", RELEASE_TIMEOUT_RECIPES) +def test_ci_watch_post_queue_fix_exists(recipe_fixture, request): + """ci_watch_post_queue_fix step must exist in register_clone_unconfirmed family recipes.""" + recipe = request.getfixturevalue(recipe_fixture) + assert "ci_watch_post_queue_fix" in recipe.steps, ( + f"ci_watch_post_queue_fix must be a step in {recipe_fixture}" + ) + + +@pytest.mark.parametrize("recipe_fixture", RELEASE_TIMEOUT_RECIPES) +def test_re_push_queue_fix_routes_to_ci_watch_post_queue_fix(recipe_fixture, request): + """re_push_queue_fix.on_success must route to ci_watch_post_queue_fix.""" + recipe = request.getfixturevalue(recipe_fixture) + step = recipe.steps["re_push_queue_fix"] + assert step.on_success == "ci_watch_post_queue_fix", ( + f"re_push_queue_fix.on_success must be 'ci_watch_post_queue_fix' in {recipe_fixture}" + ) + + +@pytest.mark.parametrize("recipe_fixture", RELEASE_TIMEOUT_RECIPES) +def test_ci_watch_post_queue_fix_routes_reenter_on_success(recipe_fixture, request): + """ci_watch_post_queue_fix on_result success must route to reenter_merge_queue.""" + recipe = request.getfixturevalue(recipe_fixture) + step = recipe.steps["ci_watch_post_queue_fix"] + assert step.on_result is not None, ( + f"ci_watch_post_queue_fix must use on_result routing in {recipe_fixture}" + ) + success_routes = [ + c.route for c in step.on_result.conditions if c.when and "'success'" in c.when + ] + assert "reenter_merge_queue" in success_routes, ( + f"ci_watch_post_queue_fix on_result success must route to 'reenter_merge_queue'" + f" in {recipe_fixture}" + ) + + +@pytest.mark.parametrize("recipe_fixture", RELEASE_TIMEOUT_RECIPES) +def test_ci_watch_post_queue_fix_routes_detect_ci_conflict_on_failure(recipe_fixture, request): + """ci_watch_post_queue_fix.on_failure must route to detect_ci_conflict.""" + recipe = request.getfixturevalue(recipe_fixture) + assert "ci_watch_post_queue_fix" in recipe.steps, ( + f"ci_watch_post_queue_fix step must exist in {recipe_fixture}" + ) + step = recipe.steps["ci_watch_post_queue_fix"] + assert step.on_failure == "detect_ci_conflict", ( + f"ci_watch_post_queue_fix.on_failure must be 'detect_ci_conflict' in {recipe_fixture}" + ) + + +@pytest.mark.parametrize("recipe_fixture", RELEASE_TIMEOUT_RECIPES) +def test_ci_watch_post_queue_fix_uses_wait_for_ci_tool(recipe_fixture, request): + """ci_watch_post_queue_fix must use the wait_for_ci tool.""" + recipe = request.getfixturevalue(recipe_fixture) + assert "ci_watch_post_queue_fix" in recipe.steps, ( + f"ci_watch_post_queue_fix step must exist in {recipe_fixture}" + ) + step = recipe.steps["ci_watch_post_queue_fix"] + assert step.tool == "wait_for_ci" + + +@pytest.mark.parametrize("recipe_fixture", RELEASE_TIMEOUT_RECIPES) +def test_ci_watch_post_queue_fix_has_skip_when_false(recipe_fixture, request): + """ci_watch_post_queue_fix must have skip_when_false: inputs.open_pr.""" + recipe = request.getfixturevalue(recipe_fixture) + assert "ci_watch_post_queue_fix" in recipe.steps, ( + f"ci_watch_post_queue_fix step must exist in {recipe_fixture}" + ) + step = recipe.steps["ci_watch_post_queue_fix"] + assert step.skip_when_false == "inputs.open_pr" + + +@pytest.mark.parametrize("recipe_fixture", RELEASE_TIMEOUT_RECIPES) +def test_wait_for_queue_routes_ejected_ci_failure_to_diagnose_ci(recipe_fixture, request): + """wait_for_queue.on_result must route ejected_ci_failure to diagnose_ci.""" + recipe = request.getfixturevalue(recipe_fixture) + step = recipe.steps["wait_for_queue"] + assert step.on_result is not None, "wait_for_queue must have on_result" + conditions = step.on_result.conditions + ci_failure_routes = [ + c + for c in conditions + if c.when is not None and "ejected_ci_failure" in c.when and c.route == "diagnose_ci" + ] + assert ci_failure_routes, ( + f"wait_for_queue.on_result must route ejected_ci_failure to diagnose_ci" + f" in {recipe_fixture}" + ) + + +@pytest.mark.parametrize("recipe_fixture", RELEASE_TIMEOUT_RECIPES) +def test_wait_for_queue_ejected_ci_failure_precedes_ejected(recipe_fixture, request): + """ejected_ci_failure route must precede generic ejected in wait_for_queue.on_result.""" + recipe = request.getfixturevalue(recipe_fixture) + step = recipe.steps["wait_for_queue"] + assert step.on_result is not None, "wait_for_queue must have on_result" + conditions = step.on_result.conditions + whens = [c.when or "" for c in conditions] + ci_fail_idx = next((i for i, w in enumerate(whens) if "ejected_ci_failure" in w), None) + ejected_idx = next( + (i for i, w in enumerate(whens) if w.strip() == "${{ result.pr_state }} == ejected"), None + ) + assert ci_fail_idx is not None, "ejected_ci_failure route must exist" + assert ejected_idx is not None, "ejected route must still exist" + assert ci_fail_idx < ejected_idx, ( + "ejected_ci_failure route must appear before generic ejected route " + "to prevent CI failure ejections from being handled as conflict ejections" + ) + + +# --------------------------------------------------------------------------- +# Regression — remediation.yaml timeout arm and correct fallback +# --------------------------------------------------------------------------- + + +def test_remediation_wait_for_queue_has_timeout_arm_and_release_timeout_fallback( + remed_recipe, +) -> None: + """remediation.yaml wait_for_queue must have explicit timeout arm and correct fallback.""" + step = remed_recipe.steps["wait_for_queue"] + assert step.on_result is not None, "wait_for_queue must have on_result" + conditions = step.on_result.conditions + + # Must have an explicit timeout arm routing to register_clone_unconfirmed + timeout_conditions = [ + c + for c in conditions + if c.when is not None and "timeout" in c.when and c.route == "register_clone_unconfirmed" + ] + assert timeout_conditions, ( + "remediation.yaml wait_for_queue must have explicit " + "'${{ result.pr_state }} == timeout -> register_clone_unconfirmed' arm" + ) + + # Fallback (when=None) must route to register_clone_unconfirmed, not register_clone_success + fallback_conditions = [c for c in conditions if c.when is None] + assert fallback_conditions, "wait_for_queue must have a fallback condition (when=None)" + assert fallback_conditions[0].route == "register_clone_unconfirmed", ( + f"remediation.yaml wait_for_queue fallback must be register_clone_unconfirmed, " + f"got: {fallback_conditions[0].route!r}" + ) + + # on_failure must route to register_clone_unconfirmed, not register_clone_success + assert step.on_failure == "register_clone_unconfirmed", ( + f"remediation.yaml wait_for_queue on_failure must be register_clone_unconfirmed, " + f"got: {step.on_failure!r}" + ) + + +# --------------------------------------------------------------------------- +# T9 family-specific: fallback, on_failure, reenter_merge_queue_cheap +# (register_clone_unconfirmed family only) +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize("recipe_fixture", RELEASE_TIMEOUT_RECIPES) +def test_wait_for_queue_fallback_routes_to_register_clone_unconfirmed( + recipe_fixture, request +) -> None: + """wait_for_queue fallback (when=None) must route to register_clone_unconfirmed.""" + recipe = request.getfixturevalue(recipe_fixture) + step = recipe.steps["wait_for_queue"] + assert step.on_result is not None + fallback_conditions = [c for c in step.on_result.conditions if c.when is None] + assert fallback_conditions, ( + f"{recipe_fixture}: wait_for_queue.on_result must have a fallback condition" + ) + assert fallback_conditions[0].route == "register_clone_unconfirmed", ( + f"{recipe_fixture}: wait_for_queue fallback must route to register_clone_unconfirmed, " + f"got: {fallback_conditions[0].route!r}" + ) + + +@pytest.mark.parametrize("recipe_fixture", RELEASE_TIMEOUT_RECIPES) +def test_wait_for_queue_on_failure_routes_to_register_clone_unconfirmed( + recipe_fixture, request +) -> None: + """wait_for_queue.on_failure must route to register_clone_unconfirmed.""" + recipe = request.getfixturevalue(recipe_fixture) + step = recipe.steps["wait_for_queue"] + assert step.on_failure == "register_clone_unconfirmed", ( + f"{recipe_fixture}: wait_for_queue on_failure must be register_clone_unconfirmed, " + f"got: {step.on_failure!r}" + ) + + +@pytest.mark.parametrize("recipe_fixture", RELEASE_TIMEOUT_RECIPES) +def test_wait_for_queue_dropped_healthy_routes_through_circuit_breaker( + recipe_fixture, request +) -> None: + """dropped_healthy must route to check_dropped_healthy_loop circuit breaker, + which in turn routes to reenter_merge_queue_cheap on DROPPED_OK.""" + recipe = request.getfixturevalue(recipe_fixture) + assert "reenter_merge_queue_cheap" in recipe.steps, ( + f"{recipe_fixture}: reenter_merge_queue_cheap step must exist" + ) + assert "check_dropped_healthy_loop" in recipe.steps, ( + f"{recipe_fixture}: check_dropped_healthy_loop step must exist" + ) + step = recipe.steps["wait_for_queue"] + assert step.on_result is not None + dropped_routes = [ + c + for c in step.on_result.conditions + if c.when is not None + and "dropped_healthy" in c.when + and c.route == "check_dropped_healthy_loop" + ] + assert dropped_routes, ( + f"{recipe_fixture}: dropped_healthy must route to check_dropped_healthy_loop" + ) + cb_step = recipe.steps["check_dropped_healthy_loop"] + assert cb_step.on_result is not None + ok_routes = [c for c in cb_step.on_result.conditions if c.when is None] + assert ok_routes and ok_routes[0].route == "reenter_merge_queue_cheap", ( + f"{recipe_fixture}: check_dropped_healthy_loop fallthrough must route to" + " reenter_merge_queue_cheap" + ) + + +# --------------------------------------------------------------------------- +# ci_watch_post_queue_fix bounded loops +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize("recipe_fixture", RELEASE_TIMEOUT_RECIPES) +def test_ci_watch_post_queue_fix_no_runs_routes_to_failure(recipe_fixture, request): + """ci_watch_post_queue_fix no_runs must route to release_issue_failure.""" + recipe = request.getfixturevalue(recipe_fixture) + step = recipe.steps["ci_watch_post_queue_fix"] + no_runs_routes = [ + c.route for c in step.on_result.conditions if c.when and "'no_runs'" in c.when + ] + assert "release_issue_failure" in no_runs_routes, ( + f"{recipe_fixture}: ci_watch_post_queue_fix no_runs must route to release_issue_failure" + ) + + +@pytest.mark.parametrize("recipe_fixture", RELEASE_TIMEOUT_RECIPES) +def test_ci_watch_post_queue_fix_timed_out_is_bounded(recipe_fixture, request): + """ci_watch_post_queue_fix timed_out must route through check_ci_post_queue_loop.""" + recipe = request.getfixturevalue(recipe_fixture) + step = recipe.steps["ci_watch_post_queue_fix"] + timed_out_routes = [ + c.route for c in step.on_result.conditions if c.when and "'timed_out'" in c.when + ] + assert timed_out_routes, f"{recipe_fixture}: missing timed_out routing" + assert timed_out_routes[0] == "check_ci_post_queue_loop", ( + f"{recipe_fixture}: timed_out must route through check_ci_post_queue_loop, " + f"not {timed_out_routes[0]}" + ) + + +@pytest.mark.parametrize("recipe_fixture", RELEASE_TIMEOUT_RECIPES) +def test_check_ci_post_queue_loop_exists_and_bounded(recipe_fixture, request): + """check_ci_post_queue_loop step must exist and be bounded.""" + recipe = request.getfixturevalue(recipe_fixture) + assert "check_ci_post_queue_loop" in recipe.steps, ( + f"{recipe_fixture}: check_ci_post_queue_loop step must exist" + ) + step = recipe.steps["check_ci_post_queue_loop"] + assert step.with_args is not None, ( + f"{recipe_fixture}: check_ci_post_queue_loop must have with_args" + ) + assert step.with_args.get("callable") == "autoskillit.smoke_utils.check_loop_iteration" + assert "max_iterations" in step.with_args, ( + f"{recipe_fixture}: check_ci_post_queue_loop must specify max_iterations" + ) + assert int(step.with_args["max_iterations"]) >= 2, ( + f"{recipe_fixture}: check_ci_post_queue_loop max_iterations must be >= 2" + ) diff --git a/tests/recipe/test_merge_sub_recipe_hidden.py b/tests/recipe/test_merge_sub_recipe_hidden.py index ccb3a730a5..ecfe2f687b 100644 --- a/tests/recipe/test_merge_sub_recipe_hidden.py +++ b/tests/recipe/test_merge_sub_recipe_hidden.py @@ -1,8 +1,12 @@ """Test that _merge_sub_recipe skips hidden sub-recipe ingredients.""" -from autoskillit.recipe._api import _merge_sub_recipe +import pytest + +from autoskillit.recipe._recipe_composition import _merge_sub_recipe from autoskillit.recipe.schema import Recipe, RecipeIngredient, RecipeStep +pytestmark = [pytest.mark.layer("recipe"), pytest.mark.small] + def _recipe( ingredients: dict, steps: dict | None = None, requires_packs: list[str] | None = None diff --git a/tests/recipe/test_methodology_disambiguation.py b/tests/recipe/test_methodology_disambiguation.py new file mode 100644 index 0000000000..9ce3c3c4e0 --- /dev/null +++ b/tests/recipe/test_methodology_disambiguation.py @@ -0,0 +1,136 @@ +"""Tests for methodology disambiguation.""" + +from __future__ import annotations + +import dataclasses + +import pytest + +from autoskillit.recipe.methodology_disambiguation import ( + disambiguate, + load_disambiguation_rules, +) + +pytestmark = [pytest.mark.layer("recipe"), pytest.mark.small] + + +class TestRuleTests: + def test_rule_prisma_dominance(self) -> None: + result = disambiguate({"systematic_synthesis", "controlled_intervention"}) + assert result.primary_tradition == "systematic_synthesis" + assert "rule_prisma_dominance" in result.precedence_trace + + def test_rule_prisma_dominance_tripod_srma_exception(self) -> None: + result = disambiguate({"systematic_synthesis", "prediction_model_validation"}) + assert result.primary_tradition == "systematic_synthesis" + assert "TRIPOD_SRMA" in result.applied_union_rules + assert "exception_prediction_model_validation" in result.precedence_trace + + def test_rule_rct_economic_union(self) -> None: + result = disambiguate({"controlled_intervention", "economic_evaluation"}) + assert result.primary_tradition == "controlled_intervention" + assert "CHEERS_union" in result.applied_union_rules + assert "rule_rct_economic_union" in result.precedence_trace + + def test_rule_arrive_supersedes_consort(self) -> None: + result = disambiguate({"animal_preclinical", "controlled_intervention"}) + assert result.primary_tradition == "animal_preclinical" + assert result.applied_union_rules == () + assert "rule_arrive_supersedes_consort" in result.precedence_trace + + def test_rule_benchmarking_prediction_nested(self) -> None: + result = disambiguate({"method_comparison_benchmarking", "prediction_model_validation"}) + assert result.primary_tradition == "method_comparison_benchmarking" + assert "TRIPOD_nested" in result.applied_union_rules + assert "rule_benchmarking_prediction_nested" in result.precedence_trace + + +class TestOverlapTests: + def test_overlap_tripod_consort_union(self) -> None: + result = disambiguate({"prediction_model_validation", "controlled_intervention"}) + assert result.primary_tradition == "controlled_intervention" + assert "TRIPOD_union" in result.applied_union_rules + assert "overlap_tripod_consort_union" in result.precedence_trace + + def test_overlap_strobe_prisma_moose(self) -> None: + result = disambiguate({"observational_correlational", "systematic_synthesis"}) + assert result.primary_tradition == "systematic_synthesis" + assert "MOOSE_override" in result.applied_union_rules + assert "rule_prisma_dominance" in result.precedence_trace + assert "overlap_strobe_prisma_moose" in result.precedence_trace + + def test_overlap_odd_controlled_nesting(self) -> None: + result = disambiguate({"simulation_modeling_tradition", "controlled_intervention"}) + assert result.primary_tradition == "simulation_modeling_tradition" + assert "controlled_intervention_secondary" in result.applied_union_rules + assert "overlap_odd_controlled_nesting" in result.precedence_trace + + def test_overlap_benchmarking_prisma_separation(self) -> None: + result = disambiguate({"method_comparison_benchmarking", "systematic_synthesis"}) + assert result.primary_tradition == "method_comparison_benchmarking" + assert "PRISMA_curation_phase" in result.applied_union_rules + assert "rule_prisma_dominance" in result.precedence_trace + assert "overlap_benchmarking_prisma_separation" in result.precedence_trace + + def test_overlap_srqr_consort_parallel(self) -> None: + result = disambiguate({"qualitative_interpretive_tradition", "controlled_intervention"}) + assert result.primary_tradition == "controlled_intervention" + assert "SRQR_parallel" in result.applied_union_rules + assert "overlap_srqr_consort_parallel" in result.precedence_trace + + +class TestStructuralBehavioralTests: + def test_fallthrough_uses_highest_priority(self) -> None: + result = disambiguate({"diagnostic_accuracy", "quality_improvement"}) + assert result.primary_tradition == "diagnostic_accuracy" + assert "fallthrough_priority" in result.precedence_trace + + def test_result_is_frozen(self) -> None: + result = disambiguate({"controlled_intervention", "economic_evaluation"}) + with pytest.raises(dataclasses.FrozenInstanceError): + result.primary_tradition = "something_else" + + def test_single_candidate_returns_that_candidate(self) -> None: + result = disambiguate({"controlled_intervention"}) + assert result.primary_tradition == "controlled_intervention" + assert result.precedence_trace == "single_candidate" + assert result.applied_union_rules == () + + def test_empty_candidate_set(self) -> None: + result = disambiguate(set()) + assert result.primary_tradition is None + assert result.precedence_trace == "no_candidates" + assert result.candidate_set == () + + def test_load_disambiguation_rules_bundled(self) -> None: + rules, overlaps = load_disambiguation_rules() + rule_names = {r.name for r in rules} + assert rule_names == { + "prisma_dominance", + "rct_economic_union", + "arrive_supersedes_consort", + "benchmarking_prediction_nested", + } + overlap_names = {o.name for o in overlaps} + assert overlap_names == { + "tripod_consort_union", + "strobe_prisma_moose", + "odd_controlled_nesting", + "benchmarking_prisma_separation", + "srqr_consort_parallel", + } + + def test_disambiguation_deterministic(self) -> None: + candidates = {"controlled_intervention", "economic_evaluation"} + results = [disambiguate(candidates) for _ in range(10)] + for r in results[1:]: + assert r.primary_tradition == results[0].primary_tradition + assert r.applied_union_rules == results[0].applied_union_rules + assert r.precedence_trace == results[0].precedence_trace + + def test_rule_precedence_order(self) -> None: + result = disambiguate( + {"systematic_synthesis", "animal_preclinical", "controlled_intervention"} + ) + assert result.primary_tradition == "systematic_synthesis" + assert result.precedence_trace.startswith("rule_prisma_dominance") diff --git a/tests/recipe/test_methodology_tradition_registry.py b/tests/recipe/test_methodology_tradition_registry.py new file mode 100644 index 0000000000..1d714dea44 --- /dev/null +++ b/tests/recipe/test_methodology_tradition_registry.py @@ -0,0 +1,317 @@ +"""Tests for methodology tradition registry loader.""" + +from __future__ import annotations + +import os +from pathlib import Path + +import pytest +import yaml +from structlog.testing import capture_logs + +from autoskillit.recipe.methodology_tradition_registry import ( + BUNDLED_METHODOLOGY_TRADITIONS_DIR, + MethodologyTraditionSpec, + get_methodology_tradition_by_name, + is_out_of_scope_tradition, + load_all_methodology_traditions, +) + +pytestmark = [pytest.mark.layer("recipe"), pytest.mark.small] + +EXPECTED_TRADITIONS = { + "controlled_intervention", + "systematic_synthesis", + "observational_correlational", + "diagnostic_accuracy", + "prediction_model_validation", + "simulation_modeling_tradition", + "measurement_instrument_validation_tradition", + "quality_improvement", + "economic_evaluation", + "animal_preclinical", + "qualitative_interpretive_tradition", + "method_comparison_benchmarking", +} + + +def test_all_bundled_traditions_present() -> None: + traditions = load_all_methodology_traditions() + assert {s.name for s in traditions} == EXPECTED_TRADITIONS + + +def test_each_tradition_has_required_fields() -> None: + traditions = load_all_methodology_traditions() + for spec in traditions: + assert isinstance(spec.name, str), f"{spec.name}: name not str" + assert spec.name, f"{spec.name}: name is empty" + assert isinstance(spec.display_name, str), f"{spec.name}: display_name not str" + assert isinstance(spec.schema_version, str), f"{spec.name}: schema_version not str" + assert isinstance(spec.priority, int), f"{spec.name}: priority not int" + assert isinstance(spec.canonical_guideline, dict), ( + f"{spec.name}: canonical_guideline not dict" + ) + assert isinstance(spec.fields_spanned, list), f"{spec.name}: fields_spanned not list" + assert len(spec.fields_spanned) >= 1, f"{spec.name}: fields_spanned is empty" + assert isinstance(spec.detection_keywords, list), ( + f"{spec.name}: detection_keywords not list" + ) + assert len(spec.detection_keywords) >= 1, f"{spec.name}: detection_keywords is empty" + assert isinstance(spec.mandatory_figures, list), f"{spec.name}: mandatory_figures not list" + assert isinstance(spec.strongly_expected_figures, list), ( + f"{spec.name}: strongly_expected_figures not list" + ) + assert isinstance(spec.anti_patterns, list), f"{spec.name}: anti_patterns not list" + assert len(spec.anti_patterns) >= 1, f"{spec.name}: anti_patterns is empty" + + +def test_all_traditions_have_schema_version_1_0() -> None: + traditions = load_all_methodology_traditions() + for spec in traditions: + assert spec.schema_version == "1.0", ( + f"{spec.name}: schema_version = {spec.schema_version!r}" + ) + + +def test_priority_values_are_unique() -> None: + traditions = load_all_methodology_traditions() + priorities = [s.priority for s in traditions] + assert len(priorities) == len(set(priorities)), "Duplicate priority values found" + + +def test_evaluation_order_matches_priority() -> None: + traditions = load_all_methodology_traditions() + names = [s.name for s in traditions] + sorted_by_priority = sorted(traditions, key=lambda s: (s.priority, s.name)) + assert names == [s.name for s in sorted_by_priority] + + +def test_qualitative_interpretive_has_empty_mandatory_figures() -> None: + traditions = load_all_methodology_traditions() + by_name = {s.name: s for s in traditions} + qual = by_name["qualitative_interpretive_tradition"] + assert qual.mandatory_figures == [] + + +def test_is_out_of_scope_tradition_qualitative() -> None: + traditions = load_all_methodology_traditions() + by_name = {s.name: s for s in traditions} + qual = by_name["qualitative_interpretive_tradition"] + assert is_out_of_scope_tradition(qual) is True + + +def test_is_out_of_scope_tradition_normal() -> None: + traditions = load_all_methodology_traditions() + by_name = {s.name: s for s in traditions} + for name, spec in by_name.items(): + if name != "qualitative_interpretive_tradition": + assert is_out_of_scope_tradition(spec) is False, f"{name}: should not be out of scope" + + +def test_user_override_replaces_bundled(tmp_path: Path) -> None: + user_dir = tmp_path / ".autoskillit" / "methodology-traditions" + user_dir.mkdir(parents=True) + (user_dir / "controlled_intervention.yaml").write_text( + yaml.dump( + { + "name": "controlled_intervention", + "schema_version": "1.0", + "priority": 1, + "display_name": "My Custom RCT", + "canonical_guideline": { + "name": "CUSTOM", + "governing_body": "custom", + "stable_for_decade": False, + "canonical": False, + }, + "fields_spanned": ["custom_field"], + "detection_keywords": ["custom keyword"], + "mandatory_figures": [], + "strongly_expected_figures": [], + "anti_patterns": [], + } + ) + ) + traditions = load_all_methodology_traditions(project_dir=tmp_path) + by_name = {s.name: s for s in traditions} + assert by_name["controlled_intervention"].display_name == "My Custom RCT" + assert len(traditions) == 12 + + +def test_user_new_tradition_is_added(tmp_path: Path) -> None: + user_dir = tmp_path / ".autoskillit" / "methodology-traditions" + user_dir.mkdir(parents=True) + (user_dir / "my_new_tradition.yaml").write_text( + yaml.dump( + { + "name": "my_new_tradition", + "schema_version": "1.0", + "priority": 99, + "display_name": "My New Tradition", + "canonical_guideline": { + "name": "CUSTOM", + "governing_body": "custom", + "stable_for_decade": True, + "canonical": True, + }, + "fields_spanned": ["custom_field"], + "detection_keywords": ["custom keyword"], + "mandatory_figures": [], + "strongly_expected_figures": [], + "anti_patterns": [], + } + ) + ) + traditions = load_all_methodology_traditions(project_dir=tmp_path) + by_name = {s.name: s for s in traditions} + assert "my_new_tradition" in by_name + assert len(traditions) == 13 + + +def test_missing_user_override_dir_is_silent(tmp_path: Path) -> None: + traditions = load_all_methodology_traditions(project_dir=tmp_path) + assert {s.name for s in traditions} == EXPECTED_TRADITIONS + + +def test_no_project_dir_returns_bundled_only() -> None: + traditions = load_all_methodology_traditions(project_dir=None) + assert {s.name for s in traditions} == EXPECTED_TRADITIONS + + +def test_schema_mismatch_warns(tmp_path: Path) -> None: + + user_dir = tmp_path / ".autoskillit" / "methodology-traditions" + user_dir.mkdir(parents=True) + (user_dir / "future_tradition.yaml").write_text( + yaml.dump( + { + "name": "future_tradition", + "schema_version": "2.0", + "priority": 99, + "display_name": "Future Tradition", + "canonical_guideline": { + "name": "FUTURE", + "governing_body": "future", + "stable_for_decade": True, + "canonical": True, + }, + "fields_spanned": ["future_field"], + "detection_keywords": ["future"], + "mandatory_figures": [], + "strongly_expected_figures": [], + "anti_patterns": [], + } + ) + ) + with capture_logs() as cap_logs: + traditions = load_all_methodology_traditions(project_dir=tmp_path) + + assert any( + entry.get("schema_version") == "2.0" and entry.get("log_level") == "warning" + for entry in cap_logs + ), "Expected WARNING about schema_version mismatch" + by_name = {s.name: s for s in traditions} + assert "future_tradition" in by_name + + +def test_get_methodology_tradition_by_name_found() -> None: + spec = get_methodology_tradition_by_name("controlled_intervention") + assert spec is not None + assert spec.name == "controlled_intervention" + assert isinstance(spec, MethodologyTraditionSpec) + + +def test_get_methodology_tradition_by_name_not_found() -> None: + spec = get_methodology_tradition_by_name("nonexistent_tradition") + assert spec is None + + +def test_no_citation_markers_in_yaml_files() -> None: + for yaml_path in sorted(BUNDLED_METHODOLOGY_TRADITIONS_DIR.glob("*.yaml")): + content = yaml_path.read_text() + assert chr(0x3010) not in content, f"{yaml_path.name} contains synthetic citation marker" + assert "daggerL" not in content, f"{yaml_path.name} contains synthetic citation marker" + + +def test_canonical_guideline_fields_complete() -> None: + traditions = load_all_methodology_traditions() + required_subkeys = {"name", "governing_body", "stable_for_decade", "canonical"} + for spec in traditions: + assert set(spec.canonical_guideline.keys()) == required_subkeys, ( + f"{spec.name}: canonical_guideline missing fields" + ) + + +def test_controlled_intervention_is_priority_1() -> None: + traditions = load_all_methodology_traditions() + by_name = {s.name: s for s in traditions} + assert by_name["controlled_intervention"].priority == 1 + + +def test_detection_keywords_are_distinct() -> None: + traditions = load_all_methodology_traditions() + seen: dict[frozenset, str] = {} + for spec in traditions: + kws = frozenset(spec.detection_keywords) + assert kws not in seen, f"{spec.name} and {seen[kws]} have identical detection_keywords" + seen[kws] = spec.name + + +# --------------------------------------------------------------------------- +# T2: Caching tests +# --------------------------------------------------------------------------- + +_MINIMAL_TRADITION_YAML = { + "name": "test_tradition", + "schema_version": "1.0", + "priority": 99, + "display_name": "Test Tradition", + "canonical_guideline": { + "name": "TEST", + "governing_body": "test", + "stable_for_decade": True, + "canonical": True, + }, + "fields_spanned": ["test_field"], + "detection_keywords": ["test keyword"], + "mandatory_figures": [], + "strongly_expected_figures": [], + "anti_patterns": [], +} + +_MINIMAL_TRADITION_YAML_2 = { + "name": "test_tradition_2", + "schema_version": "1.0", + "priority": 99, + "display_name": "Test Tradition 2", + "canonical_guideline": { + "name": "TEST2", + "governing_body": "test", + "stable_for_decade": True, + "canonical": True, + }, + "fields_spanned": ["test_field"], + "detection_keywords": ["test keyword 2"], + "mandatory_figures": [], + "strongly_expected_figures": [], + "anti_patterns": [], +} + + +class TestMethodologyTraditionCaching: + def test_bundled_result_is_stable_on_repeat_call(self) -> None: + r1 = load_all_methodology_traditions() + r2 = load_all_methodology_traditions() + assert r1 == r2 + + def test_mtime_change_invalidates_cache(self, tmp_path: Path) -> None: + user_dir = tmp_path / ".autoskillit" / "methodology-traditions" + user_dir.mkdir(parents=True) + (user_dir / "custom_trad.yaml").write_text(yaml.dump(_MINIMAL_TRADITION_YAML)) + r1 = load_all_methodology_traditions(project_dir=tmp_path) + (user_dir / "custom_trad_2.yaml").write_text(yaml.dump(_MINIMAL_TRADITION_YAML_2)) + old_mt = user_dir.stat().st_mtime + os.utime(user_dir, (old_mt + 2, old_mt + 2)) + r2 = load_all_methodology_traditions(project_dir=tmp_path) + assert r1 is not r2 + assert len(r2) > len(r1) diff --git a/tests/recipe/test_methodology_tradition_router.py b/tests/recipe/test_methodology_tradition_router.py new file mode 100644 index 0000000000..74cf67e17b --- /dev/null +++ b/tests/recipe/test_methodology_tradition_router.py @@ -0,0 +1,202 @@ +"""Tests for methodology_tradition_router two-stage Tier-C router.""" + +from __future__ import annotations + +import pytest + +from autoskillit.recipe.methodology_tradition_registry import ( + load_all_methodology_traditions, +) +from autoskillit.recipe.methodology_tradition_router import ( + UnionRuleDef, + classify_methodology, +) + +pytestmark = [pytest.mark.layer("recipe"), pytest.mark.small] + + +TRADITION_FIXTURES: list[tuple[str, str]] = [ + ( + "controlled_intervention", + "This randomized controlled trial follows CONSORT guidelines with intent-to-treat " + "analysis across parallel group arms.", + ), + ( + "systematic_synthesis", + "The systematic review employed PRISMA standards and performed meta-analysis with " + "forest plots to report pooled estimates across studies.", + ), + ( + "observational_correlational", + "An observational cohort study was conducted using STROBE reporting guidelines to " + "examine the association between exposure and outcome.", + ), + ( + "diagnostic_accuracy", + "The diagnostic accuracy study evaluated sensitivity and specificity using ROC curve " + "analysis following STARD recommendations.", + ), + ( + "prediction_model_validation", + "External validation of the clinical prediction rule was performed " + "using TRIPOD guidelines, assessing calibration and discrimination via c-statistic.", + ), + ( + "simulation_modeling_tradition", + "An agent-based model was developed following the ODD protocol, implemented in NetLogo " + "to simulate emergent behavior across parameter sweeps.", + ), + ( + "measurement_instrument_validation_tradition", + "Measurement properties were assessed using COSMIN criteria, " + "evaluating internal consistency via Cronbach alpha and test-retest reliability.", + ), + ( + "quality_improvement", + "The quality improvement initiative followed SQUIRE guidelines, employing PDSA cycles " + "to drive process improvement in clinical pathways.", + ), + ( + "economic_evaluation", + "Cost-effectiveness analysis was conducted following CHEERS guidelines, calculating " + "ICER and QALY outcomes using Markov modeling.", + ), + ( + "animal_preclinical", + "The animal study was designed per ARRIVE guidelines, with randomization of animals " + "and blinding of assessors to evaluate the intervention.", + ), + ( + "qualitative_interpretive_tradition", + "Qualitative research was conducted following COREQ reporting standards, using " + "thematic analysis of semi-structured interview transcripts.", + ), + ( + "method_comparison_benchmarking", + "The method comparison study established a SOTA baseline using leaderboard evaluation " + "and ablation study to benchmark performance against prior work.", + ), +] + + +@pytest.mark.parametrize("tradition_slug,plan_snippet", TRADITION_FIXTURES) +def test_single_tradition_detected(tradition_slug, plan_snippet): + result = classify_methodology(plan_snippet) + assert result.primary_tradition == tradition_slug + assert result.precedence_trace == "stage1_single_match" + assert tradition_slug in result.candidate_set + assert len(result.candidate_set) == 1 + assert result.applied_union_rules == () + + +def test_no_tradition_detected(): + result = classify_methodology("This paper describes a novel deep learning architecture.") + assert result.primary_tradition is None + assert result.precedence_trace == "stage1_no_match_fallback" + assert result.candidate_set == () + assert result.applied_union_rules == () + + +def test_multi_match_returns_candidates(): + plan_text = ( + "This systematic review and meta-analysis includes randomized controlled trial " + "following PRISMA and CONSORT guidelines with forest plots." + ) + result = classify_methodology(plan_text) + assert result.primary_tradition is None + assert result.precedence_trace == "stage1_multi_match" + assert len(result.candidate_set) >= 2 + assert "controlled_intervention" in result.candidate_set + assert "systematic_synthesis" in result.candidate_set + + +def test_multi_match_resolved_by_priority(): + plan_text = ( + "This systematic review and meta-analysis includes randomized controlled trial " + "following PRISMA and CONSORT guidelines with forest plots." + ) + result = classify_methodology(plan_text, resolve_by_priority=True) + traditions = load_all_methodology_traditions() + priority_map = {s.name: s.priority for s in traditions} + # Winner is the tradition with the lowest priority number (highest precedence) + expected_winner = min(result.candidate_set, key=lambda n: (priority_map[n], n)) + assert result.primary_tradition == expected_winner + assert result.precedence_trace == "stage1_multi_match_resolved_by_priority" + assert len(result.candidate_set) >= 2 + + +def test_union_rule_applied(): + rule = UnionRuleDef( + name="rct_with_synthesis", + member_traditions=frozenset({"controlled_intervention", "systematic_synthesis"}), + resolved_tradition="systematic_synthesis", + ) + plan_text = ( + "This systematic review and meta-analysis includes randomized controlled trial " + "following PRISMA and CONSORT guidelines." + ) + result = classify_methodology(plan_text, union_rules=[rule]) + assert result.primary_tradition == "systematic_synthesis" + assert "rct_with_synthesis" in result.applied_union_rules + assert "stage2_tiebreak_by_rule_rct_with_synthesis" == result.precedence_trace + + +def test_union_rule_not_applied_when_no_member_match(): + rule = UnionRuleDef( + name="unrelated_rule", + member_traditions=frozenset({"economic_evaluation", "quality_improvement"}), + resolved_tradition="economic_evaluation", + ) + plan_text = ( + "This randomized controlled trial follows CONSORT guidelines " + "with intent-to-treat analysis." + ) + result = classify_methodology(plan_text, union_rules=[rule]) + assert result.primary_tradition == "controlled_intervention" + assert result.precedence_trace == "stage1_single_match" + assert result.applied_union_rules == () + + +def test_deterministic_classification(): + plan_text = ( + "This randomized controlled trial follows CONSORT guidelines " + "with intent-to-treat analysis across parallel group arms." + ) + results = [classify_methodology(plan_text) for _ in range(10)] + first = results[0] + for r in results[1:]: + assert r.primary_tradition == first.primary_tradition + assert r.precedence_trace == first.precedence_trace + assert r.candidate_set == first.candidate_set + + +def test_candidate_set_sorted_by_priority(): + plan_text = ( + "This systematic review and meta-analysis includes randomized controlled trials " + "following PRISMA and CONSORT guidelines." + ) + result = classify_methodology(plan_text) + traditions = load_all_methodology_traditions() + priority_map = {s.name: s.priority for s in traditions} + priorities = [priority_map[c] for c in result.candidate_set] + assert priorities == sorted(priorities) # ascending: lower number = higher precedence + + +def test_high_threshold_eliminates_weak_matches(): + plan_text = "The study used an RCT design with novel computational methods." + result_threshold_1 = classify_methodology(plan_text, min_keyword_matches=1) + result_threshold_3 = classify_methodology(plan_text, min_keyword_matches=3) + assert "controlled_intervention" in result_threshold_1.candidate_set + assert "controlled_intervention" not in result_threshold_3.candidate_set + + +def test_case_insensitive_matching(): + plan_text = "This RANDOMIZED CONTROLLED TRIAL follows consort guidelines with INTENT-TO-TREAT." + result = classify_methodology(plan_text) + assert result.primary_tradition == "controlled_intervention" + + +def test_result_is_frozen(): + result = classify_methodology("This randomized controlled trial uses CONSORT and ITT.") + with pytest.raises(AttributeError): + result.primary_tradition = "something_else" diff --git a/tests/recipe/test_methodology_venue_appendix.py b/tests/recipe/test_methodology_venue_appendix.py new file mode 100644 index 0000000000..ed98c0618c --- /dev/null +++ b/tests/recipe/test_methodology_venue_appendix.py @@ -0,0 +1,465 @@ +"""Tests for methodology_venue_appendix — Stage B venue appendix resolution.""" + +from __future__ import annotations + +import dataclasses + +import pytest + +from autoskillit.recipe.methodology_tradition_registry import ( + VenueAppendixDef, + load_all_methodology_traditions, +) +from autoskillit.recipe.methodology_venue_appendix import ( + MLSubAreaFoldingDef, + VenueAppendixMatch, + load_ml_sub_area_folding, + resolve_venue_appendices, +) + +pytestmark = [pytest.mark.layer("recipe"), pytest.mark.small] + + +# --------------------------------------------------------------------------- +# T1. Folding map structural tests +# --------------------------------------------------------------------------- + + +class TestFoldingMapStructure: + def test_all_eight_sub_areas_present(self) -> None: + entries = load_ml_sub_area_folding() + slugs = {e.sub_area for e in entries} + expected = { + "foundation_models", + "reinforcement_learning", + "supervised_classification", + "nlp", + "computer_vision", + "generative_models", + "agentic_systems", + "time_series", + } + assert slugs == expected + + def test_each_entry_has_primary_parent(self) -> None: + for entry in load_ml_sub_area_folding(): + assert entry.primary_parent, f"{entry.sub_area} has no primary_parent" + + def test_each_entry_has_at_least_one_alternate(self) -> None: + for entry in load_ml_sub_area_folding(): + assert len(entry.alternate_parents) >= 1, f"{entry.sub_area} has no alternate_parents" + + def test_all_parent_slugs_reference_valid_traditions(self) -> None: + traditions = {s.name for s in load_all_methodology_traditions()} + for entry in load_ml_sub_area_folding(): + assert entry.primary_parent in traditions, ( + f"{entry.sub_area} primary_parent={entry.primary_parent} not in traditions" + ) + for alt in entry.alternate_parents: + assert alt.parent in traditions, ( + f"{entry.sub_area} alternate parent={alt.parent} not in traditions" + ) + + def test_foundation_models_cosmin_has_constraint(self) -> None: + fm = next(e for e in load_ml_sub_area_folding() if e.sub_area == "foundation_models") + cosmin = next( + ( + a + for a in fm.alternate_parents + if a.parent == "measurement_instrument_validation_tradition" + ), + None, + ) + assert cosmin is not None + assert cosmin.constraint == "only_if_explicit_construct_measurement" + + def test_folding_entries_are_frozen(self) -> None: + for entry in load_ml_sub_area_folding(): + assert isinstance(entry, MLSubAreaFoldingDef) + with pytest.raises(dataclasses.FrozenInstanceError): + entry.sub_area = "mutated" # type: ignore[misc] + + def test_folding_map_deterministic(self) -> None: + results = [load_ml_sub_area_folding() for _ in range(10)] + assert all(r == results[0] for r in results) + + +# --------------------------------------------------------------------------- +# T2. Venue appendix schema tests +# --------------------------------------------------------------------------- + + +class TestVenueAppendixSchema: + def test_all_traditions_with_appendices_load(self) -> None: + traditions = list(load_all_methodology_traditions()) + assert traditions + + def test_appendix_entries_have_required_fields(self) -> None: + traditions = load_all_methodology_traditions() + for spec in traditions: + for app in spec.venue_specific_appendices: + assert app.sub_area + assert len(app.trigger_keywords) >= 1 + assert len(app.expectations) >= 1 + + def test_appendix_expectations_have_figure_and_source(self) -> None: + traditions = load_all_methodology_traditions() + for spec in traditions: + for app in spec.venue_specific_appendices: + for exp in app.expectations: + assert "figure" in exp + assert "source" in exp + assert exp["figure"] + assert exp["source"] + + def test_appendix_sub_areas_match_folding_map(self) -> None: + folding = load_ml_sub_area_folding() + valid_sub_areas = {e.sub_area for e in folding} + traditions = load_all_methodology_traditions() + for spec in traditions: + for app in spec.venue_specific_appendices: + assert app.sub_area in valid_sub_areas, ( + f"{spec.name} appendix sub_area={app.sub_area} not in folding map" + ) + + def test_primary_parent_traditions_have_appendices(self) -> None: + mcb = next( + ( + s + for s in load_all_methodology_traditions() + if s.name == "method_comparison_benchmarking" + ), + None, + ) + assert mcb is not None + assert len(mcb.venue_specific_appendices) >= len(load_ml_sub_area_folding()) + + def test_alternate_parent_traditions_have_appendices(self) -> None: + folding = load_ml_sub_area_folding() + alt_parents = set() + for entry in folding: + for alt in entry.alternate_parents: + alt_parents.add(alt.parent) + traditions = {s.name: s for s in load_all_methodology_traditions()} + for parent in alt_parents: + spec = traditions.get(parent) + assert spec is not None + assert len(spec.venue_specific_appendices) >= 1 + + def test_venue_appendix_def_is_frozen(self) -> None: + traditions = load_all_methodology_traditions() + for spec in traditions: + for app in spec.venue_specific_appendices: + assert isinstance(app, VenueAppendixDef) + with pytest.raises(dataclasses.FrozenInstanceError): + app.sub_area = "mutated" # type: ignore[misc] + + +# --------------------------------------------------------------------------- +# T3. Stage B routing tests — 24 branching fixtures +# --------------------------------------------------------------------------- + +VENUE_BRANCHING_FIXTURES: list[tuple[str, str, str, bool]] = [ + # --- Foundation Models --- + ( + "foundation_models", + "We evaluate GPT-4 on the BIG-bench leaderboard comparing few-shot performance " + "against SOTA baselines using ablation study and benchmark evaluation protocol.", + "method_comparison_benchmarking", + False, + ), + ( + "foundation_models", + "We assess GPT-4 calibration on held-out clinical tasks, measuring few-shot scaling " + "behavior and zero-shot validation across risk strata.", + "prediction_model_validation", + True, + ), + ( + "foundation_models", + "We apply psychometric analysis with item response theory to evaluate the latent " + "trait structure of GPT-4 outputs using Cronbach alpha.", + "measurement_instrument_validation_tradition", + True, + ), + # --- Reinforcement Learning --- + ( + "reinforcement_learning", + "We benchmark our RL agent against SOTA baselines on the Atari leaderboard " + "using ablation study and performance comparison across evaluation protocol.", + "method_comparison_benchmarking", + False, + ), + ( + "reinforcement_learning", + "We study multi-agent emergent behavior in a population dynamics simulation " + "with swarm intelligence and agent-based reward structures.", + "simulation_modeling_tradition", + True, + ), + ( + "reinforcement_learning", + "We perform policy evaluation on held-out task environments measuring " + "sample efficiency across training budgets.", + "prediction_model_validation", + True, + ), + # --- Supervised Classification --- + ( + "supervised_classification", + "We compare classification methods on the benchmark leaderboard using " + "ablation study and SOTA baseline performance comparison.", + "method_comparison_benchmarking", + False, + ), + ( + "supervised_classification", + "We evaluate medical imaging classification for radiology pathology detection " + "reporting sensitivity and specificity at operating points.", + "diagnostic_accuracy", + True, + ), + ( + "supervised_classification", + "We build a clinical decision support model for patient outcome prognosis " + "and risk prediction in the supervised classification pipeline.", + "prediction_model_validation", + True, + ), + # --- NLP --- + ( + "nlp", + "We benchmark our NLP model against SOTA on the GLUE leaderboard with " + "ablation study and performance comparison protocol.", + "method_comparison_benchmarking", + False, + ), + ( + "nlp", + "We develop a clinical NLP system for patient outcome risk scoring " + "from clinical text with calibrated predictions.", + "prediction_model_validation", + True, + ), + ( + "nlp", + "We conduct discourse analysis of clinical conversations using thematic " + "coding and narrative structure with qualitative assessment.", + "qualitative_interpretive_tradition", + True, + ), + # --- Computer Vision --- + ( + "computer_vision", + "We benchmark our detection model against SOTA methods on COCO using " + "ablation study and mAP performance comparison.", + "method_comparison_benchmarking", + False, + ), + ( + "computer_vision", + "We evaluate medical imaging for radiology pathology detection measuring " + "sensitivity and specificity per diagnostic threshold.", + "diagnostic_accuracy", + True, + ), + ( + "computer_vision", + "We build a patient outcome prognosis model from imaging biomarker features " + "with clinical decision support and risk prediction.", + "prediction_model_validation", + True, + ), + # --- Generative Models --- + ( + "generative_models", + "We benchmark our GAN against SOTA on FID and IS metrics using " + "ablation study and leaderboard performance comparison.", + "method_comparison_benchmarking", + False, + ), + ( + "generative_models", + "We conduct human evaluation with qualitative assessment of generated " + "content using expert rating and content analysis protocols.", + "qualitative_interpretive_tradition", + True, + ), + ( + "generative_models", + "We validate the human evaluation instrument measuring inter-rater " + "reliability and construct validity with scale development.", + "measurement_instrument_validation_tradition", + True, + ), + # --- Agentic Systems --- + ( + "agentic_systems", + "We benchmark our agentic system against SOTA baselines on task success " + "using ablation study and leaderboard evaluation protocol.", + "method_comparison_benchmarking", + False, + ), + ( + "agentic_systems", + "We build a multi-agent simulation with emergent behavior using " + "agent-based model design and swarm intelligence patterns.", + "simulation_modeling_tradition", + True, + ), + ( + "agentic_systems", + "We evaluate process improvement and workflow optimization via " + "agent-driven task efficiency and error reduction metrics.", + "quality_improvement", + True, + ), + # --- Time-Series --- + ( + "time_series", + "We benchmark our forecasting model against SOTA on the Monash archive " + "using ablation study and performance comparison protocol.", + "method_comparison_benchmarking", + False, + ), + ( + "time_series", + "We develop clinical forecasting for patient trajectory and mortality " + "prediction with temporal risk scoring.", + "prediction_model_validation", + True, + ), + ( + "time_series", + "We build a dynamical system simulation with differential equation " + "and compartmental model for system dynamics forecasting.", + "simulation_modeling_tradition", + True, + ), +] + + +class TestVenueBranchingFixtures: + @pytest.mark.parametrize( + "sub_area,plan_text,expected_parent,expected_rerouted", + VENUE_BRANCHING_FIXTURES, + ids=[f"{sa}-{ep}" for sa, _, ep, _ in VENUE_BRANCHING_FIXTURES], + ) + def test_venue_branching( + self, sub_area: str, plan_text: str, expected_parent: str, expected_rerouted: bool + ) -> None: + matches = resolve_venue_appendices(plan_text) + assert matches, f"resolve_venue_appendices returned empty list for plan_text={plan_text!r}" + sub_area_matches = [m for m in matches if m.sub_area == sub_area] + assert len(sub_area_matches) == 1, ( + f"Expected 1 match for sub_area={sub_area!r}, " + f"got {len(sub_area_matches)}: {sub_area_matches}" + ) + match = sub_area_matches[0] + assert match.resolved_parent == expected_parent + assert match.re_routed == expected_rerouted + + +# --------------------------------------------------------------------------- +# T4. Constraint tests +# --------------------------------------------------------------------------- + + +class TestConstraintLogic: + def test_cosmin_constraint_blocks_without_explicit_construct(self) -> None: + plan_text = ( + "We study GPT-4 using Cronbach alpha to measure internal consistency " + "of the benchmark scores across evaluation protocol." + ) + matches = resolve_venue_appendices(plan_text) + fm = [m for m in matches if m.sub_area == "foundation_models"] + assert len(fm) == 1 + assert fm[0].resolved_parent == "method_comparison_benchmarking" + + def test_cosmin_constraint_allows_with_explicit_construct(self) -> None: + plan_text = ( + "We apply psychometric analysis with item response theory to evaluate the latent " + "trait structure of GPT-4 outputs using Cronbach alpha." + ) + matches = resolve_venue_appendices(plan_text) + fm = [m for m in matches if m.sub_area == "foundation_models"] + assert len(fm) == 1 + assert fm[0].resolved_parent == "measurement_instrument_validation_tradition" + assert fm[0].re_routed is True + + +# --------------------------------------------------------------------------- +# T5. Edge case and behavioral tests +# --------------------------------------------------------------------------- + + +class TestEdgeCases: + def test_no_sub_area_detected_returns_empty(self) -> None: + plan_text = "We study the effect of temperature on enzyme activity." + matches = resolve_venue_appendices(plan_text) + assert matches == [] + + def test_multiple_sub_areas_detected(self) -> None: + plan_text = ( + "We compare classification methods on medical imaging for radiology detection " + "using ablation study and benchmark evaluation protocol." + ) + matches = resolve_venue_appendices(plan_text) + sub_areas = {m.sub_area for m in matches} + assert {"supervised_classification", "computer_vision"}.issubset(sub_areas), ( + f"Expected supervised_classification and computer_vision in detected sub_areas, " + f"got {sub_areas}" + ) + + def test_deterministic_resolution(self) -> None: + plan_text = ( + "We benchmark our NLP model against SOTA on the GLUE leaderboard with " + "ablation study and performance comparison protocol." + ) + results = [resolve_venue_appendices(plan_text) for _ in range(10)] + assert all(r == results[0] for r in results) + + def test_result_is_frozen(self) -> None: + plan_text = ( + "We benchmark our NLP model against SOTA on the GLUE leaderboard with " + "ablation study and performance comparison protocol." + ) + matches = resolve_venue_appendices(plan_text) + for m in matches: + assert isinstance(m, VenueAppendixMatch) + with pytest.raises(dataclasses.FrozenInstanceError): + m.sub_area = "mutated" # type: ignore[misc] + + def test_case_insensitive_keyword_matching(self) -> None: + plan_text_lower = ( + "we evaluate medical imaging classification for radiology pathology detection " + "reporting sensitivity and specificity at operating points." + ) + plan_text_mixed = ( + "We EVALUATE Medical Imaging Classification for RADIOLOGY Pathology Detection " + "reporting Sensitivity and Specificity at operating points." + ) + matches_lower = resolve_venue_appendices(plan_text_lower) + matches_mixed = resolve_venue_appendices(plan_text_mixed) + assert {m.sub_area for m in matches_lower} == {m.sub_area for m in matches_mixed} + + +# --------------------------------------------------------------------------- +# T4: Caching tests +# --------------------------------------------------------------------------- + + +class TestFoldingCaching: + @pytest.fixture(autouse=True) + def _clear_cache(self) -> None: # type: ignore[misc] + load_ml_sub_area_folding.cache_clear() + yield # type: ignore[misc] + load_ml_sub_area_folding.cache_clear() + + def test_result_is_cached_via_lru_cache(self) -> None: + r1 = load_ml_sub_area_folding() + r2 = load_ml_sub_area_folding() + assert r1 is r2 + info = load_ml_sub_area_folding.cache_info() + assert info.hits >= 1 + assert info.misses == 1 diff --git a/tests/recipe/test_plan_visualization_step.py b/tests/recipe/test_plan_visualization_step.py index 5dce46b396..09d7a14cb2 100644 --- a/tests/recipe/test_plan_visualization_step.py +++ b/tests/recipe/test_plan_visualization_step.py @@ -7,6 +7,8 @@ from autoskillit.core.paths import pkg_root from autoskillit.recipe.io import builtin_recipes_dir, load_recipe +pytestmark = [pytest.mark.layer("recipe"), pytest.mark.small] + RESEARCH_RECIPE_PATH = builtin_recipes_dir() / "research.yaml" @@ -45,18 +47,14 @@ def test_plan_visualization_step_captures_paths(recipe) -> None: def test_create_worktree_copies_viz_plan(recipe) -> None: - """create_worktree cmd must copy visualization-plan.md and report-plan.md.""" + """create_worktree cmd must pass visualization_plan_path and report_plan_path.""" step = recipe.steps["create_worktree"] cmd = step.with_args.get("cmd", "") - assert "VISUALIZATION_PLAN" in cmd, ( - "create_worktree must reference VISUALIZATION_PLAN context variable" - ) - assert "REPORT_PLAN" in cmd, "create_worktree must reference REPORT_PLAN context variable" - assert "visualization-plan.md" in cmd, ( - "create_worktree must copy visualization-plan.md into the research dir" + assert "visualization_plan_path" in cmd, ( + "create_worktree must pass context.visualization_plan_path to the script" ) - assert "report-plan.md" in cmd, ( - "create_worktree must copy report-plan.md into the research dir" + assert "report_plan_path" in cmd, ( + "create_worktree must pass context.report_plan_path to the script" ) @@ -64,3 +62,25 @@ def test_plan_visualization_skill_dir_exists() -> None: """src/autoskillit/skills_extended/plan-visualization/SKILL.md must exist.""" skill_path = pkg_root() / "skills_extended" / "plan-visualization" / "SKILL.md" assert skill_path.exists(), f"plan-visualization skill directory not found at {skill_path}" + + +def test_tier_c_table_removed_from_skill_md() -> None: + """Old Tier-C target_domain routing table must be removed from SKILL.md.""" + path = pkg_root() / "skills_extended" / "plan-visualization" / "SKILL.md" + content = path.read_text() + assert "| nlp | vis-lens-methodology-norms |" not in content + assert "| cv | vis-lens-methodology-norms |" not in content + assert "| rl | vis-lens-temporal |" not in content + assert "| target_domain |" not in content + + +def test_tier_c_methodology_router_present() -> None: + """New two-stage methodology router keywords must be present in SKILL.md.""" + path = pkg_root() / "skills_extended" / "plan-visualization" / "SKILL.md" + content = path.read_text() + assert "detection_keywords" in content + assert "candidate_set" in content + assert "precedence_trace" in content + assert "stage1_single_match" in content + assert "stage1_no_match_fallback" in content + assert "stage2_tiebreak" in content diff --git a/tests/recipe/test_planner_recipe.py b/tests/recipe/test_planner_recipe.py new file mode 100644 index 0000000000..0e97744011 --- /dev/null +++ b/tests/recipe/test_planner_recipe.py @@ -0,0 +1,335 @@ +"""Structural assertions for the planner recipe.""" + +import importlib + +import pytest + +from autoskillit.recipe.io import builtin_recipes_dir, load_recipe +from autoskillit.recipe.validator import run_semantic_rules + +pytestmark = [pytest.mark.layer("recipe"), pytest.mark.small] + + +@pytest.fixture(scope="module") +def planner_recipe(): + return load_recipe(builtin_recipes_dir() / "planner.yaml") + + +def test_planner_recipe_loads(planner_recipe): + assert planner_recipe.name == "planner" + + +def test_planner_recipe_has_required_steps(planner_recipe): + required_steps = { + "init", + "resolve_task", + "analyze", + "extract_domain", + "generate_phases", + "build_plan_snapshot", + "elaborate_phases", + "merge_phases", + "refine_phases", + "expand_assignments", + "elaborate_assignments", + "merge_assignments", + "refine_assignments", + "expand_wps", + "elaborate_wps", + "finalize_wp_manifest", + "merge_wps", + "refine_wps", + "validate_task_alignment", + "reconcile_deps", + "validate", + "check_verdict", + "refine", + "compile", + "done", + "escalate_stop", + } + assert required_steps <= planner_recipe.steps.keys(), ( + f"Missing steps: {required_steps - planner_recipe.steps.keys()}" + ) + + +def test_planner_recipe_declares_requires_packs(planner_recipe): + assert planner_recipe.requires_packs + assert "kitchen-core" in planner_recipe.requires_packs + + +def test_planner_recipe_python_callables_importable(planner_recipe): + for name, step in planner_recipe.steps.items(): + if step.tool == "run_python" and step.with_args: + callable_path = step.with_args.get("callable", "") + if not callable_path: + continue + module_path, func_name = callable_path.rsplit(".", 1) + mod = importlib.import_module(module_path) + assert hasattr(mod, func_name), ( + f"Step {name!r} callable {callable_path!r} not importable" + ) + + +def test_planner_recipe_has_kitchen_rules(planner_recipe): + assert planner_recipe.kitchen_rules + assert len(planner_recipe.kitchen_rules) >= 3 + + +def test_kitchen_rules_include_sequential_dispatch(planner_recipe): + assert any("SEQUENTIAL DISPATCH" in rule for rule in planner_recipe.kitchen_rules), ( + "kitchen_rules must include a SEQUENTIAL DISPATCH rule" + ) + + +def test_planner_recipe_validate_routes_to_refine_on_fail(planner_recipe): + assert "validate" in planner_recipe.steps, "validate step must exist" + assert "check_verdict" in planner_recipe.steps, "check_verdict step must exist" + refine_reachable = False + for step_name in ["validate", "check_verdict"]: + step = planner_recipe.steps[step_name] + routes = [] + if step.on_result: + routes += [c.route for c in step.on_result.conditions] + if step.on_success: + routes.append(step.on_success) + if "refine" in routes: + refine_reachable = True + assert refine_reachable, "refine step must be reachable from validate or check_verdict" + + +def test_planner_recipe_validation_has_no_errors(planner_recipe): + findings = run_semantic_rules(planner_recipe) + errors = [f for f in findings if f.severity == "ERROR"] + assert errors == [], f"Unexpected ERROR findings: {[f.rule for f in errors]}" + + +def test_planner_recipe_contract_exists(): + contracts_dir = builtin_recipes_dir() / "contracts" + assert (contracts_dir / "planner.yaml").exists(), "Run: autoskillit recipes validate planner" + + +def test_planner_recipe_extract_domain_uses_positional_args(planner_recipe): + step = planner_recipe.steps["extract_domain"] + skill_cmd = step.with_args.get("skill_command", "") + assert "analysis.json" in skill_cmd, "Must pass analysis.json as positional arg" + assert "context.task_file_path" in skill_cmd, "Must pass task_file_path as positional arg" + assert "env" not in step.with_args, "Must not use env: block (ADR-0003)" + + +def test_planner_init_captures_planner_dir(planner_recipe): + init_step = planner_recipe.steps["init"] + assert init_step.tool == "run_python" + assert init_step.with_args.get("callable") == "autoskillit.planner.create_run_dir" + assert "planner_dir" in (init_step.capture or {}) + + +def test_planner_steps_use_context_planner_dir(): + import yaml + + from autoskillit.recipe.io import builtin_recipes_dir + + raw = yaml.safe_load((builtin_recipes_dir() / "planner.yaml").read_text()) + raw_steps = raw.get("steps", {}) + for step_name, step_dict in raw_steps.items(): + step_str = str(step_dict) + assert "{{AUTOSKILLIT_TEMP}}/planner" not in step_str, ( + f"Step '{step_name}' still references bare AUTOSKILLIT_TEMP/planner path" + ) + + +def test_elaborate_assignments_uses_planner_elaborate_assignments_skill(planner_recipe): + step = planner_recipe.steps["elaborate_assignments"] + skill_cmd = step.with_args.get("skill_command", "") + assert "planner-elaborate-assignments" in skill_cmd, ( + "elaborate_assignments must invoke planner-elaborate-assignments skill" + ) + + +def test_planner_recipe_has_phase_wp_steps(planner_recipe): + expected = { + "expand_wps", + "elaborate_wps", + "merge_wps", + "refine_wps", + "finalize_wp_manifest", + } + assert expected <= planner_recipe.steps.keys(), ( + f"Missing phase-WP steps: {expected - planner_recipe.steps.keys()}" + ) + + +def test_planner_recipe_elaborate_wps_uses_correct_skill(planner_recipe): + step = planner_recipe.steps["elaborate_wps"] + skill_cmd = step.with_args.get("skill_command", "") + assert "planner-elaborate-wps" in skill_cmd, ( + "elaborate_wps must invoke planner-elaborate-wps skill" + ) + + +def test_planner_recipe_finalize_step_routes_to_merge_wps(planner_recipe): + step = planner_recipe.steps["finalize_wp_manifest"] + assert step.on_success == "merge_wps", ( + f"finalize_wp_manifest must route to merge_wps on success, got {step.on_success!r}" + ) + + +# --- T2: Parallel dispatch pattern tests --- + + +def test_elaborate_phases_uses_capture_list(planner_recipe): + step = planner_recipe.steps["elaborate_phases"] + assert step.capture_list, "elaborate_phases must use capture_list for parallel accumulation" + assert "elab_result_path" in step.capture_list + + +def test_elaborate_assignments_uses_capture_list(planner_recipe): + step = planner_recipe.steps["elaborate_assignments"] + assert step.capture_list + assert "phase_assignments_result_dir" in step.capture_list + + +def test_elaborate_wps_uses_capture_list(planner_recipe): + step = planner_recipe.steps["elaborate_wps"] + assert step.capture_list + assert "phase_wps_result_dir" in step.capture_list + + +def test_parallel_elaborate_steps_have_dispatch_note(planner_recipe): + for step_name in ("elaborate_phases", "elaborate_assignments"): + step = planner_recipe.steps[step_name] + assert step.note, f"{step_name} must have a note for parallel dispatch instructions" + assert "parallel" in step.note.lower(), f"{step_name} note must mention parallel dispatch" + + +def test_elaborate_wps_has_sequential_dispatch_note(planner_recipe): + step = planner_recipe.steps["elaborate_wps"] + assert step.note, "elaborate_wps must have a note for sequential dispatch instructions" + assert "sequential" in step.note.lower(), "elaborate_wps note must mention sequential dispatch" + assert "parallel" not in step.note.lower(), ( + "elaborate_wps note must not mention parallel dispatch" + ) + + +# --- T3: No sequential loops --- + + +def test_no_check_remaining_loops(planner_recipe): + for name, step in planner_recipe.steps.items(): + if step.tool == "run_python" and step.with_args.get("callable", ""): + assert "check_remaining" not in step.with_args["callable"], ( + f"Step {name} still uses check_remaining — recipe should use parallel dispatch" + ) + + +# --- T4: Refine tier steps wired --- + + +@pytest.mark.parametrize( + "step_name,skill_name", + [ + ("refine_phases", "planner-refine-phases"), + ("refine_assignments", "planner-refine-assignments"), + ("refine_wps", "planner-refine-wps"), + ], +) +def test_refine_tier_steps_use_correct_skills(planner_recipe, step_name, skill_name): + step = planner_recipe.steps[step_name] + assert step.tool == "run_skill" + assert skill_name in step.with_args.get("skill_command", "") + + +# --- T5: Merge steps use merge_tier_results --- + + +@pytest.mark.parametrize("step_name", ["merge_phases", "merge_assignments", "merge_wps"]) +def test_merge_steps_use_merge_tier_results(planner_recipe, step_name): + step = planner_recipe.steps[step_name] + assert step.tool == "run_python" + assert step.with_args.get("callable") == "autoskillit.planner.merge.merge_tier_results" + + +# --- validate_task_alignment step integration --- + + +def test_validate_task_alignment_step_exists(planner_recipe): + assert "validate_task_alignment" in planner_recipe.steps + step = planner_recipe.steps["validate_task_alignment"] + assert step.tool == "run_skill" + assert "planner-validate-task-alignment" in step.with_args.get("skill_command", "") + + +def test_planner_recipe_has_assess_review_approach_step(planner_recipe): + assert "assess_review_approach_step" in planner_recipe.steps + step = planner_recipe.steps["assess_review_approach_step"] + assert step.tool == "run_skill" + assert "planner-assess-review-approach" in (step.with_args.get("skill_command") or "") + assert step.optional is True + + +def test_planner_recipe_assess_review_approach_routes_to_validate_on_failure(planner_recipe): + step = planner_recipe.steps["assess_review_approach_step"] + assert step.on_success == "validate" + assert step.on_failure == "validate" + + +def test_planner_recipe_no_task_file_ingredient(planner_recipe): + assert "task_file" not in planner_recipe.ingredients + + +def test_planner_recipe_has_resolve_task_step(planner_recipe): + assert "resolve_task" in planner_recipe.steps + step = planner_recipe.steps["resolve_task"] + assert step.tool == "run_python" + assert step.with_args.get("callable") == "autoskillit.planner.resolve_task_input" + assert "task_file_path" in (step.capture or {}) + assert "task_label" in (step.capture or {}) + + +def test_planner_init_routes_to_resolve_task(planner_recipe): + init_step = planner_recipe.steps["init"] + assert init_step.on_success == "resolve_task" + + +def test_planner_resolve_task_routes_to_analyze(planner_recipe): + step = planner_recipe.steps["resolve_task"] + assert step.on_success == "analyze" + + +def test_extract_domain_passes_task_file_path_positionally(planner_recipe): + step = planner_recipe.steps["extract_domain"] + skill_cmd = step.with_args.get("skill_command", "") + assert "context.task_file_path" in skill_cmd + assert "env" not in step.with_args, "Must not use env: block (ADR-0003)" + + +def test_generate_phases_passes_task_file_path_positionally(planner_recipe): + step = planner_recipe.steps["generate_phases"] + skill_cmd = step.with_args.get("skill_command", "") + assert "context.task_file_path" in skill_cmd + assert "env" not in step.with_args, "Must not use env: block (ADR-0003)" + + +@pytest.mark.parametrize( + "step_name", ["build_plan_snapshot", "merge_phases", "merge_assignments", "merge_wps"] +) +def test_merge_steps_receive_task_file_path(planner_recipe, step_name): + step = planner_recipe.steps[step_name] + assert "task_file_path" in step.with_args, f"{step_name} must receive task_file_path" + assert "context.task_file_path" in step.with_args["task_file_path"] + + +def test_compile_step_receives_task_file_path_and_label(planner_recipe): + step = planner_recipe.steps["compile"] + assert "task_file_path" in step.with_args + assert "task_label" in step.with_args + assert "context.task_file_path" in step.with_args["task_file_path"] + assert "context.task_label" in step.with_args["task_label"] + + +def test_no_step_references_inputs_task_file(planner_recipe): + for name, step in planner_recipe.steps.items(): + step_str = str(step.with_args) + assert "inputs.task_file" not in step_str, ( + f"Step {name!r} still references inputs.task_file" + ) diff --git a/tests/recipe/test_promote_to_main_wrapper.py b/tests/recipe/test_promote_to_main_wrapper.py new file mode 100644 index 0000000000..6cc906b690 --- /dev/null +++ b/tests/recipe/test_promote_to_main_wrapper.py @@ -0,0 +1,81 @@ +import functools + +import pytest + +from autoskillit.core.paths import pkg_root +from autoskillit.recipe.io import load_recipe +from autoskillit.recipe.validator import run_semantic_rules + +pytestmark = [pytest.mark.layer("recipe"), pytest.mark.small] + +_RECIPE_PATH = pkg_root() / "recipes" / "promote-to-main-wrapper.yaml" + + +@functools.lru_cache(maxsize=1) +def _load(): + return load_recipe(_RECIPE_PATH) + + +def test_recipe_parses() -> None: + recipe = _load() + assert recipe.name == "promote-to-main-wrapper" + + +def test_recipe_validates_cleanly() -> None: + from autoskillit.core.types import Severity + from tests.recipe.conftest import KNOWN_PART_B_VIOLATIONS + + recipe = _load() + findings = run_semantic_rules(recipe) + errors = [ + f + for f in findings + if f.severity == Severity.ERROR and f.rule not in KNOWN_PART_B_VIOLATIONS + ] + undeclared = [f for f in findings if f.rule == "undeclared-capture-key"] + assert errors == [], errors + assert undeclared == [], undeclared + + +def test_source_dir_is_required_no_default() -> None: + recipe = _load() + ing = recipe.ingredients["source_dir"] + assert ing.required is True + assert ing.default is None + + +def test_promote_step_has_fault_guards() -> None: + step = _load().steps["promote"] + assert step.on_failure == "escalate_stop" + assert step.on_context_limit == "escalate_stop" + + +def test_verdict_routing() -> None: + step = _load().steps["promote"] + conditions = step.on_result.conditions + preflight = next((c for c in conditions if c.when and "preflight_failed" in c.when), None) + assert preflight is not None, "no preflight_failed condition found" + assert preflight.route == "escalate_stop" + fallback = next((c for c in conditions if c.when is None), None) + assert fallback is not None, "no fallback condition (when=None) found" + assert fallback.route == "emit_result" + + +def test_category_summary_captured() -> None: + step = _load().steps["promote"] + assert "category_summary" in step.capture + + +def test_emit_result_echoes_all_tokens() -> None: + cmd = _load().steps["emit_result"].with_args["cmd"] + for token in ("pr_url", "verdict", "category_summary"): + assert f"context.{token}" in cmd, f"missing context substitution for: {token}" + + +def test_requires_packs_declared() -> None: + assert _load().requires_packs == ["github"] + + +def test_kind_is_food_truck() -> None: + recipe = _load() + assert recipe.kind == "food-truck" diff --git a/tests/recipe/test_recipe_ci_watch_event.py b/tests/recipe/test_recipe_ci_watch_event.py new file mode 100644 index 0000000000..379465e43d --- /dev/null +++ b/tests/recipe/test_recipe_ci_watch_event.py @@ -0,0 +1,53 @@ +"""Structural tests: primary ci_watch steps must have event: and be preceded by +ci_event derivation.""" + +from __future__ import annotations + +import pytest +import yaml + +pytestmark = [pytest.mark.layer("recipe"), pytest.mark.small] + +RECIPE_FILES = [ + "src/autoskillit/recipes/implementation.yaml", + "src/autoskillit/recipes/remediation.yaml", + "src/autoskillit/recipes/implementation-groups.yaml", +] + + +@pytest.mark.parametrize("recipe_path", RECIPE_FILES) +def test_ci_watch_step_has_event_parameter(recipe_path: str) -> None: + """Primary ci_watch steps must pass event: to prevent branch-event mismatch.""" + with open(recipe_path) as f: + recipe = yaml.safe_load(f) + steps = recipe["steps"] + assert "ci_watch" in steps, f"{recipe_path}: no ci_watch step found" + ci_watch = steps["ci_watch"] + with_args = ci_watch.get("with") or {} + assert "event" in with_args, ( + f"{recipe_path}: ci_watch missing event: parameter. " + f"Feature branches excluded from push triggers will see no_runs timeout." + ) + + +@pytest.mark.parametrize("recipe_path", RECIPE_FILES) +def test_check_repo_ci_event_step_exists_before_ci_watch(recipe_path: str) -> None: + """Each recipe must have a check_repo_ci_event step that routes to ci_watch.""" + with open(recipe_path) as f: + recipe = yaml.safe_load(f) + steps = recipe["steps"] + assert "check_repo_ci_event" in steps, ( + f"{recipe_path}: missing check_repo_ci_event step. " + f"ci_event must be derived before ci_watch runs." + ) + early_step = steps["check_repo_ci_event"] + capture = early_step.get("capture") or {} + assert "ci_event" in capture, ( + f"{recipe_path}: check_repo_ci_event must capture ci_event into context" + ) + assert early_step.get("on_success") in ("ci_watch", "check_pr_state"), ( + f"{recipe_path}: check_repo_ci_event.on_success must route to ci_watch or check_pr_state" + ) + assert early_step.get("on_failure") == "ci_watch", ( + f"{recipe_path}: check_repo_ci_event.on_failure must be ci_watch (non-blocking)" + ) diff --git a/tests/recipe/test_recipe_order.py b/tests/recipe/test_recipe_order.py new file mode 100644 index 0000000000..783cc75509 --- /dev/null +++ b/tests/recipe/test_recipe_order.py @@ -0,0 +1,122 @@ +"""Tests for stable display order registry — recipe/order.py and its effect on list_recipes.""" + +from __future__ import annotations + +import warnings +from pathlib import Path + +import pytest + +from autoskillit.recipe.io import group_rank, list_recipes +from autoskillit.recipe.order import BUNDLED_RECIPE_ORDER + +pytestmark = [pytest.mark.layer("recipe"), pytest.mark.medium] + + +def test_implementation_appears_before_remediation(tmp_path: Path) -> None: + """'implementation' must appear before 'remediation' in list_recipes output.""" + result = list_recipes(tmp_path) + names = [r.name for r in result.items] + assert "implementation" in names and "remediation" in names + assert names.index("implementation") < names.index("remediation") + + +def test_registered_recipes_precede_unregistered_in_group0( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """Registered Group-0 recipes must appear before any unregistered Group-0 recipe.""" + import autoskillit.recipe.io as recipe_io + + monkeypatch.setattr(recipe_io, "BUNDLED_RECIPE_ORDER", ["implementation"]) + result = list_recipes(tmp_path) + group0 = [r.name for r in result.items if group_rank(r) == 0] + assert group0[0] == "implementation" + unregistered = [n for n in group0 if n != "implementation"] + assert unregistered, ( + "Expected at least one unregistered Group-0 recipe to verify the ordering contract" + ) + assert all(group0.index(n) > 0 for n in unregistered), ( + f"Some unregistered Group-0 recipes appear before 'implementation': {group0}" + ) + + +def test_unregistered_group0_recipes_alphabetical_after_registered( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """Unregistered Group-0 recipes must sort alphabetically after all registered ones.""" + import autoskillit.recipe.io as recipe_io + + monkeypatch.setattr(recipe_io, "BUNDLED_RECIPE_ORDER", ["implementation"]) + result = list_recipes(tmp_path) + group0 = [r.name for r in result.items if group_rank(r) == 0] + unregistered = group0[1:] + if not unregistered: + pytest.skip( + "No unregistered Group-0 recipes — all Group-0 recipes are registered; " + "alphabetical-tail contract cannot be verified" + ) + assert unregistered == sorted(unregistered), ( + f"Unregistered Group-0 recipes not alphabetical: {unregistered}" + ) + + +def test_adding_unregistered_recipe_does_not_shift_registered( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """Inserting a name before 'implementation' alphabetically must not shift its position.""" + import autoskillit.recipe.io as recipe_io + + monkeypatch.setattr(recipe_io, "BUNDLED_RECIPE_ORDER", ["implementation", "remediation"]) + result = list_recipes(tmp_path) + group0 = [r.name for r in result.items if group_rank(r) == 0] + impl_idx = group0.index("implementation") + remed_idx = group0.index("remediation") + + # Confirm registry order is honoured + assert impl_idx < remed_idx, "'implementation' must come before 'remediation'" + + # Verify unregistered recipes that alphabetically precede 'implementation' + # (e.g. 'bem-wrapper') do not displace it from its registered position + unregistered = [n for n in group0 if n not in {"implementation", "remediation"}] + preceding = [n for n in unregistered if n < "implementation"] + assert preceding, ( + "Expected at least one unregistered Group-0 recipe alphabetically before " + "'implementation' (e.g. 'bem-wrapper') to verify the shift-invariant" + ) + for name in preceding: + assert group0.index(name) > impl_idx, ( + f"Unregistered recipe {name!r} (alphabetically before 'implementation') " + f"displaced it — registry position-stability contract violated: {group0}" + ) + + +def test_registry_does_not_affect_addon_group( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """BUNDLED_RECIPE_ORDER must not alter relative order of Group-1 Add-on recipes.""" + import autoskillit.recipe.io as recipe_io + + result_before = list_recipes(tmp_path) + addon_before = [r.name for r in result_before.items if group_rank(r) == 1] + + monkeypatch.setattr(recipe_io, "BUNDLED_RECIPE_ORDER", ["implementation"] + addon_before) + result_after = list_recipes(tmp_path) + addon_after = [r.name for r in result_after.items if group_rank(r) == 1] + + assert addon_after == sorted(addon_after), ( + "Group-1 Add-on recipes must remain alphabetical regardless of registry" + ) + + +def test_bundled_recipe_order_covers_all_group0_recipes(tmp_path: Path) -> None: + """Warn (do not fail) if a Group-0 bundled recipe is missing from BUNDLED_RECIPE_ORDER.""" + result = list_recipes(tmp_path) + group0_names = [r.name for r in result.items if group_rank(r) == 0] + missing = [n for n in group0_names if n not in BUNDLED_RECIPE_ORDER] + if missing: + warnings.warn( + f"Group-0 bundled recipes missing from BUNDLED_RECIPE_ORDER: {missing}. " + "Add them to src/autoskillit/recipe/order.py to pin their display position.", + UserWarning, + stacklevel=1, + ) diff --git a/tests/recipe/test_recipe_scripts.py b/tests/recipe/test_recipe_scripts.py new file mode 100644 index 0000000000..4205b73a76 --- /dev/null +++ b/tests/recipe/test_recipe_scripts.py @@ -0,0 +1,53 @@ +"""Tests for scripts/recipe/ — externalized shell scripts.""" + +from __future__ import annotations + +import os +import subprocess + +import pytest + +from autoskillit.core.paths import pkg_root +from autoskillit.recipe.io import builtin_recipes_dir, load_recipe +from autoskillit.recipe.validator import run_semantic_rules + +pytestmark = [pytest.mark.layer("recipe"), pytest.mark.medium] + +SCRIPTS_DIR = pkg_root().parent.parent / "scripts" / "recipe" + + +def test_recipe_scripts_are_executable(): + scripts = list(SCRIPTS_DIR.glob("*.sh")) + assert scripts, f"No .sh scripts found in {SCRIPTS_DIR}" + for sh in scripts: + assert os.access(sh, os.X_OK), f"{sh} is not executable" + + +def test_recipe_scripts_pass_syntax_check(): + scripts = list(SCRIPTS_DIR.glob("*.sh")) + assert scripts, f"No .sh scripts found in {SCRIPTS_DIR}" + for sh in scripts: + result = subprocess.run(["bash", "-n", str(sh)], capture_output=True) + assert result.returncode == 0, f"{sh} has syntax errors: {result.stderr.decode()}" + + +@pytest.mark.parametrize( + "recipe_name", + [ + "implementation", + "remediation", + "implementation-groups", + "merge-prs", + "research", + "bem-wrapper", + ], +) +def test_recipes_pass_inline_script_rule(recipe_name): + recipe = load_recipe(builtin_recipes_dir() / f"{recipe_name}.yaml") + findings = run_semantic_rules(recipe) + inline_findings = [ + f for f in findings if f.rule in ("inline-script-in-cmd", "inline-python-in-cmd") + ] + assert inline_findings == [], ( + f"Recipe {recipe_name} has inline script findings: {inline_findings}" + ) diff --git a/tests/recipe/test_recipe_temp_substitution.py b/tests/recipe/test_recipe_temp_substitution.py index 4050349ce0..4f2d675254 100644 --- a/tests/recipe/test_recipe_temp_substitution.py +++ b/tests/recipe/test_recipe_temp_substitution.py @@ -8,6 +8,8 @@ from autoskillit.recipe.io import load_recipe +pytestmark = [pytest.mark.layer("recipe"), pytest.mark.small] + _RECIPE_TEMPLATE = """\ name: temp_subst_demo description: demo recipe for temp dir substitution diff --git a/tests/recipe/test_registry_types.py b/tests/recipe/test_registry_types.py new file mode 100644 index 0000000000..6d9d17aa79 --- /dev/null +++ b/tests/recipe/test_registry_types.py @@ -0,0 +1,21 @@ +"""Tests for RuleDef and BlockRuleDef registry types.""" + +from __future__ import annotations + +import dataclasses + +import pytest + +from autoskillit.recipe.registry import BlockRuleDef, RuleDef + +pytestmark = [pytest.mark.layer("recipe"), pytest.mark.small] + + +def test_rule_def_is_frozen() -> None: + assert dataclasses.is_dataclass(RuleDef) + assert RuleDef.__dataclass_params__.frozen is True + + +def test_block_rule_def_is_frozen() -> None: + assert dataclasses.is_dataclass(BlockRuleDef) + assert BlockRuleDef.__dataclass_params__.frozen is True diff --git a/tests/recipe/test_remediation_depth_ingredient.py b/tests/recipe/test_remediation_depth_ingredient.py new file mode 100644 index 0000000000..24dc5ccf1c --- /dev/null +++ b/tests/recipe/test_remediation_depth_ingredient.py @@ -0,0 +1,94 @@ +"""Tests for the remediation recipe depth ingredient and investigate step updates.""" + +import pytest + +from autoskillit.core.paths import pkg_root +from autoskillit.recipe.io import load_recipe +from autoskillit.recipe.schema import Recipe + +pytestmark = [pytest.mark.layer("recipe"), pytest.mark.small] + + +@pytest.fixture(scope="module") +def remediation_recipe() -> Recipe: + return load_recipe(pkg_root() / "recipes" / "remediation.yaml") + + +def test_remediation_has_depth_ingredient(remediation_recipe: Recipe) -> None: + """remediation.yaml must declare a 'depth' ingredient.""" + assert "depth" in remediation_recipe.ingredients, ( + "remediation.yaml must contain a 'depth' ingredient for investigation depth mode" + ) + + +def test_remediation_depth_ingredient_is_hidden(remediation_recipe: Recipe) -> None: + """The depth ingredient must have hidden: true.""" + ing = remediation_recipe.ingredients["depth"] + assert ing.hidden is True, ( + "remediation.yaml depth ingredient must have 'hidden: true' — " + "it is a pipeline-internal ingredient not exposed to end users" + ) + + +def test_remediation_depth_ingredient_defaults_to_standard(remediation_recipe: Recipe) -> None: + """The depth ingredient must default to 'standard'.""" + ing = remediation_recipe.ingredients["depth"] + assert ing.default == "standard", ( + "remediation.yaml depth ingredient must have default='standard' " + "so the recipe uses standard investigation mode unless explicitly overridden" + ) + + +def test_remediation_depth_ingredient_is_string_type(remediation_recipe: Recipe) -> None: + """The depth ingredient description must reference depth or mode.""" + ing = remediation_recipe.ingredients["depth"] + assert ing.description is not None, "remediation.yaml depth ingredient must have a description" + desc_lower = ing.description.lower() + assert "depth" in desc_lower or "mode" in desc_lower, ( + "remediation.yaml depth ingredient description must contain 'depth' or 'mode'" + ) + + +def test_remediation_investigate_step_has_depth_conditional(remediation_recipe: Recipe) -> None: + """The investigate step skill_command must include the --depth deep conditional.""" + step = remediation_recipe.steps.get("investigate") + assert step is not None, "remediation.yaml must contain an 'investigate' step" + skill_command = step.with_args.get("skill_command", "") + assert "--depth deep" in skill_command, ( + "remediation.yaml investigate step skill_command must contain '--depth deep' " + "conditional expression for when inputs.depth == 'deep'" + ) + + +def test_remediation_investigate_step_has_model_field(remediation_recipe: Recipe) -> None: + """The investigate step must have a 'model' field.""" + step = remediation_recipe.steps.get("investigate") + assert step is not None, "remediation.yaml must contain an 'investigate' step" + assert step.model is not None, ( + "remediation.yaml investigate step must include a 'model' field to set the " + "main session model based on depth" + ) + + +def test_remediation_investigate_step_model_uses_opus_for_deep(remediation_recipe: Recipe) -> None: + """The investigate step model expression must reference opus[1m] for deep mode.""" + step = remediation_recipe.steps.get("investigate") + assert step is not None, "remediation.yaml must contain an 'investigate' step" + model_expr = step.model or "" + assert "opus[1m]" in model_expr, ( + "remediation.yaml investigate step model expression must reference 'opus[1m]' " + "for deep mode — the main session uses Opus for deep investigation" + ) + + +def test_remediation_investigate_step_has_on_context_limit(remediation_recipe: Recipe) -> None: + """The investigate step must have on_context_limit routing.""" + step = remediation_recipe.steps.get("investigate") + assert step is not None, "remediation.yaml must contain an 'investigate' step" + assert step.on_context_limit is not None, ( + "remediation.yaml investigate step must have 'on_context_limit' to route to " + "release_issue_failure when the deep mode session exhausts context" + ) + assert step.on_context_limit == "release_issue_failure", ( + "remediation.yaml investigate step on_context_limit must route to 'release_issue_failure'" + ) diff --git a/tests/recipe/test_remediation_pr_decomposition.py b/tests/recipe/test_remediation_pr_decomposition.py index 5f1f5d86da..52a4c035b6 100644 --- a/tests/recipe/test_remediation_pr_decomposition.py +++ b/tests/recipe/test_remediation_pr_decomposition.py @@ -3,7 +3,9 @@ import pytest from autoskillit.recipe.io import builtin_recipes_dir, load_recipe -from autoskillit.recipe.validator import validate_recipe +from autoskillit.recipe.validator import validate_recipe_structure + +pytestmark = [pytest.mark.layer("recipe"), pytest.mark.small] RECIPE_PATH = builtin_recipes_dir() / "remediation.yaml" @@ -23,7 +25,9 @@ def test_remediation_recipe_has_three_pr_steps(recipe): def test_prepare_pr_routes_to_run_arch_lenses(recipe): step = recipe.steps["prepare_pr"] - assert step.on_success == "run_arch_lenses" + assert step.on_result is not None + routes = [c.route for c in step.on_result.conditions if c.when and "prep_path" in c.when] + assert "run_arch_lenses" in routes def test_run_arch_lenses_routes_to_compose_on_success(recipe): @@ -60,6 +64,40 @@ def test_compose_pr_captures_pr_url(recipe): assert "pr_url" in (step.capture or {}) +def test_arch_lenses_ingredient_declared(recipe): + assert "arch_lenses" in recipe.ingredients + + +def test_arch_lenses_defaults_to_false(recipe): + assert recipe.ingredients["arch_lenses"].default == "false" + + +def test_arch_lenses_is_not_hidden(recipe): + assert recipe.ingredients["arch_lenses"].hidden is False + + +def test_prepare_pr_routes_to_compose_pr_when_arch_lenses_false(recipe): + step = recipe.steps["prepare_pr"] + assert step.on_result is not None + routes = [c.route for c in step.on_result.conditions if c.when and "prep_path" in c.when] + assert "compose_pr" in routes + + +def test_prepare_pr_arch_lenses_route_checks_ingredient(recipe): + step = recipe.steps["prepare_pr"] + arch_lens_condition = next( + (c for c in step.on_result.conditions if c.route == "run_arch_lenses" and c.when), + None, + ) + assert arch_lens_condition is not None + assert "arch_lenses" in arch_lens_condition.when + + +def test_run_arch_lenses_still_gated_on_open_pr(recipe): + step = recipe.steps["run_arch_lenses"] + assert step.skip_when_false == "inputs.open_pr" + + def test_remediation_recipe_validates_after_decomposition(recipe): - errors = validate_recipe(recipe) + errors = validate_recipe_structure(recipe) assert not errors, f"Validation errors: {errors}" diff --git a/tests/recipe/test_remediation_recipe.py b/tests/recipe/test_remediation_recipe.py index d4d0ddeee1..6b8970b72e 100644 --- a/tests/recipe/test_remediation_recipe.py +++ b/tests/recipe/test_remediation_recipe.py @@ -5,7 +5,9 @@ import pytest from autoskillit.recipe.io import load_recipe -from autoskillit.recipe.validator import validate_recipe +from autoskillit.recipe.validator import validate_recipe_structure + +pytestmark = [pytest.mark.layer("recipe"), pytest.mark.small] RECIPE_PATH = ( Path(__file__).parent.parent.parent / "src" / "autoskillit" / "recipes" / "remediation.yaml" @@ -23,10 +25,163 @@ def test_remediation_recipe_has_release_issue_success_step(recipe): Absence of this step means issues resolved via remediation never get the staged label applied. """ - errors = validate_recipe(recipe) + errors = validate_recipe_structure(recipe) assert not errors, f"remediation.yaml failed schema validation: {errors}" step_names = list(recipe.steps.keys()) assert any("release_issue" in name and "success" in name for name in step_names), ( "remediation.yaml is missing a release_issue step on the success path. " "Without it, issues are never promoted to staged state after a successful remediation." ) + + +# T_REM_LOOP1 +def test_check_review_loop_step_exists(recipe) -> None: + """check_review_loop step must exist in remediation recipe.""" + assert "check_review_loop" in recipe.steps + + +# T_REM_LOOP2 +def test_re_push_review_routes_to_check_review_loop(recipe) -> None: + """re_push_review on_success must route to check_review_loop in remediation recipe.""" + step = recipe.steps["re_push_review"] + assert step.on_success == "check_review_loop" + + +# T_REM_LOOP3 +def test_check_review_loop_on_result_routes_to_review_pr_when_had_blocking_and_not_max_exceeded( + recipe, +) -> None: + """check_review_loop on_result condition must gate on had_blocking AND max_exceeded.""" + step = recipe.steps["check_review_loop"] + assert step.on_result is not None + review_conditions = [ + c for c in step.on_result.conditions if c.when is not None and c.route == "review_pr" + ] + assert review_conditions, "No conditional route to review_pr found" + cond = review_conditions[0].when + assert "had_blocking" in cond + assert "max_exceeded" in cond + + +# T_REM_LOOP4 +def test_check_review_loop_has_on_failure(recipe) -> None: + """check_review_loop must declare on_failure because it uses on_result + (on-result-missing-failure-route semantic rule requires it).""" + step = recipe.steps["check_review_loop"] + assert step.on_failure == "check_repo_ci_event" + + +# T_REM_LOOP5 +def test_review_pr_routes_approved_with_comments_to_resolve_review(recipe) -> None: + """review_pr on_result must route approved_with_comments to resolve_review.""" + step = recipe.steps["review_pr"] + assert step.on_result is not None + routes = {c.when: c.route for c in step.on_result.conditions if c.when} + matching = [ + when + for when, route in routes.items() + if "approved_with_comments" in when and route == "resolve_review" + ] + assert matching, "No approved_with_comments → resolve_review route found" + + +# T_REM_LOOP6 +def test_review_pr_captures_review_verdict(recipe) -> None: + """review_pr must capture verdict as review_verdict (not verdict) to avoid clobber.""" + step = recipe.steps["review_pr"] + capture = step.capture or {} + assert "review_verdict" in capture + assert "result.verdict" in capture["review_verdict"] + + +# T_REM_LOOP7 +def test_check_review_loop_with_args_has_previous_verdict(recipe) -> None: + """check_review_loop with: must pass previous_verdict from context.review_verdict.""" + step = recipe.steps["check_review_loop"] + assert "previous_verdict" in step.with_args + assert "review_verdict" in step.with_args["previous_verdict"] + + +def test_ci_watch_timed_out_routes_to_guard(recipe) -> None: + """ci_watch timed_out route goes to check_ci_timed_out_loop, not self.""" + step = recipe.steps["ci_watch"] + assert step.on_result is not None + timed_out_conds = [c for c in step.on_result.conditions if c.when and "timed_out" in c.when] + assert timed_out_conds, "ci_watch must have a timed_out condition" + assert timed_out_conds[0].route == "check_ci_timed_out_loop", ( + "ci_watch timed_out must route to check_ci_timed_out_loop, not self" + ) + + +def test_check_ci_timed_out_loop_exists_with_correct_pattern(recipe) -> None: + """check_ci_timed_out_loop uses check_loop_iteration with max_iterations: 2.""" + assert "check_ci_timed_out_loop" in recipe.steps + step = recipe.steps["check_ci_timed_out_loop"] + assert step.tool == "run_python" + assert "check_loop_iteration" in step.with_args.get("callable", "") + assert step.with_args.get("max_iterations") == "2" + assert step.on_result is not None + max_exceeded_conds = [ + c + for c in step.on_result.conditions + if c.when and "max_exceeded" in c.when and "true" in c.when + ] + assert max_exceeded_conds, "check_ci_timed_out_loop must route max_exceeded==true" + assert max_exceeded_conds[0].route == "detect_ci_conflict" + + +def test_remediation_next_or_done_step_exists(recipe) -> None: + """T_REM_MP1: remediation.yaml must have a next_or_done routing step.""" + assert "next_or_done" in recipe.steps + step = recipe.steps["next_or_done"] + assert step.action == "route" + + +def test_remediation_next_or_done_routes_more_parts_to_dry_walkthrough(recipe) -> None: + """T_REM_MP2: next_or_done must route more_parts back to dry_walkthrough.""" + step = recipe.steps["next_or_done"] + assert step.on_result is not None + conds = step.on_result.conditions + assert any( + c.route == "dry_walkthrough" and c.when is not None and "more_parts" in c.when + for c in conds + ), "next_or_done must have a predicate routing more_parts → dry_walkthrough" + + +def test_remediation_next_or_done_routes_done_to_push(recipe) -> None: + """T_REM_MP3: next_or_done fallthrough must route to push (all parts complete).""" + step = recipe.steps["next_or_done"] + assert step.on_result is not None + conds = step.on_result.conditions + assert any(c.route == "push" for c in conds), ( + "next_or_done must have a fallthrough condition routing to push" + ) + + +def test_remediation_merge_routes_to_next_or_done(recipe) -> None: + """T_REM_MP4: merge step default route must be next_or_done, not push.""" + step = recipe.steps["merge"] + assert step.on_result is not None + default_routes = [c.route for c in step.on_result.conditions if c.when is None] + assert default_routes == ["next_or_done"], ( + f"merge default route must be next_or_done, got {default_routes}" + ) + + +def test_remediation_has_no_sprint_mode_ingredient() -> None: + """remediation.yaml must not declare sprint_mode after sprint-prefix removal.""" + recipe = load_recipe(RECIPE_PATH) + assert "sprint_mode" not in recipe.ingredients + + +def test_remediation_has_no_sprint_entry_step() -> None: + """remediation.yaml must not have a sprint_entry step after sprint-prefix removal.""" + recipe = load_recipe(RECIPE_PATH) + assert "sprint_entry" not in recipe.steps + + +def test_remediation_validates_clean_after_sprint_removal() -> None: + """remediation.yaml must pass schema validation after sprint references removed.""" + recipe = load_recipe(RECIPE_PATH) + errors = validate_recipe_structure(recipe) + assert not errors, f"remediation.yaml failed validation after sprint removal: {errors}" diff --git a/tests/recipe/test_remediation_sprint_mode.py b/tests/recipe/test_remediation_sprint_mode.py deleted file mode 100644 index 4f43930930..0000000000 --- a/tests/recipe/test_remediation_sprint_mode.py +++ /dev/null @@ -1,67 +0,0 @@ -"""Tests for sprint_mode ingredient and sprint_entry step in remediation.yaml.""" - -from __future__ import annotations - -import pytest - -from autoskillit.core.paths import pkg_root -from autoskillit.recipe._analysis import make_validation_context -from autoskillit.recipe._api import _build_active_recipe -from autoskillit.recipe.io import builtin_recipes_dir, load_recipe -from autoskillit.recipe.registry import run_semantic_rules -from autoskillit.recipe.validator import validate_recipe -from tests.recipe.conftest import NO_AUTOSKILLIT_IMPORT as _NO_AUTOSKILLIT_IMPORT - - -@pytest.fixture(scope="module") -def rem_recipe(): - return load_recipe(builtin_recipes_dir() / "remediation.yaml") - - -def test_remediation_has_sprint_mode_ingredient(rem_recipe) -> None: - """remediation.yaml declares 'sprint_mode' ingredient with default 'false'.""" - assert "sprint_mode" in rem_recipe.ingredients - assert rem_recipe.ingredients["sprint_mode"].default == "false" - - -def test_remediation_sprint_mode_is_hidden(rem_recipe) -> None: - """remediation.yaml sprint_mode ingredient has hidden=True.""" - assert rem_recipe.ingredients["sprint_mode"].hidden is True - - -def test_remediation_has_sprint_entry_step(rem_recipe) -> None: - """remediation.yaml has 'sprint_entry' step with sub_recipe='sprint-prefix'.""" - assert "sprint_entry" in rem_recipe.steps - step = rem_recipe.steps["sprint_entry"] - assert step.sub_recipe == "sprint-prefix" - - -def test_remediation_sprint_mode_false_loads_clean() -> None: - """load_and_validate('remediation') with sprint_mode=false drops sprint_entry step.""" - active, combined = _build_active_recipe( - load_recipe(builtin_recipes_dir() / "remediation.yaml"), - None, - pkg_root().parent, - ) - assert "sprint_entry" not in active.steps - assert combined is None - - -def test_remediation_structural_validation_clean(rem_recipe) -> None: - """validate_recipe(remediation) returns no errors (standalone path).""" - errors = validate_recipe(rem_recipe) - assert not errors, f"Structural validation errors: {errors}" - - -def test_remediation_no_semantic_errors(rem_recipe) -> None: - """Semantic rules on remediation.yaml (sprint_mode=false) return no ERRORs.""" - from autoskillit.core.types import Severity - - active, _ = _build_active_recipe(rem_recipe, None, pkg_root().parent) - ctx = make_validation_context(active, available_sub_recipes=frozenset({"sprint-prefix"})) - errors = [ - f - for f in run_semantic_rules(ctx) - if f.severity == Severity.ERROR and f.rule != _NO_AUTOSKILLIT_IMPORT - ] - assert not errors, f"Semantic errors: {errors}" diff --git a/tests/recipe/test_report_frontmatter_schema.py b/tests/recipe/test_report_frontmatter_schema.py new file mode 100644 index 0000000000..75b1ceb976 --- /dev/null +++ b/tests/recipe/test_report_frontmatter_schema.py @@ -0,0 +1,64 @@ +"""Verify report.md YAML frontmatter matches the audit-trail schema.""" + +import pytest +import yaml + +pytestmark = [pytest.mark.layer("recipe"), pytest.mark.small] + +REQUIRED_FIELDS = { + "experiment_type": str, + "methodology_traditions": list, + "disambiguation_rule_applied": (str, type(None)), + "tier_c_lens": (str, type(None)), + "design_review_verdict": str, + "classification_timestamp": (str, type(None)), + "audit_trail_path": dict, +} + +VALID_VERDICTS = {"GO", "REVISE", "STOP"} + +REQUIRED_TRAIL_KEYS = {"design_review", "visualization_trace"} + + +def test_frontmatter_has_all_required_fields(sample_report_frontmatter: dict): + """All schema fields present and correctly typed.""" + for field, expected_type in REQUIRED_FIELDS.items(): + assert field in sample_report_frontmatter, f"Missing field: {field}" + value = sample_report_frontmatter[field] + if value is not None: + assert isinstance(value, expected_type), ( + f"{field}: expected {expected_type}, got {type(value)}" + ) + + +def test_frontmatter_verdict_is_valid(sample_report_frontmatter: dict): + """design_review_verdict is one of GO/REVISE/STOP.""" + verdict = sample_report_frontmatter.get("design_review_verdict") + if verdict is not None: + assert verdict in VALID_VERDICTS + + +def test_frontmatter_audit_trail_paths(sample_report_frontmatter: dict): + """audit_trail_path contains both required sub-keys.""" + trail = sample_report_frontmatter.get("audit_trail_path", {}) + assert REQUIRED_TRAIL_KEYS.issubset(trail.keys()) + for key in REQUIRED_TRAIL_KEYS: + assert isinstance(trail[key], str) + assert "audit/" in trail[key] + + +def test_frontmatter_methodology_traditions_is_list(sample_report_frontmatter: dict): + """methodology_traditions is always a list, even with one entry.""" + traditions = sample_report_frontmatter["methodology_traditions"] + assert isinstance(traditions, list) + + +def test_frontmatter_round_trips_through_yaml(sample_report_text: str): + """Frontmatter can be extracted and parsed by a standard YAML loader.""" + lines = sample_report_text.strip().splitlines() + assert lines[0] == "---", "Report must start with YAML frontmatter delimiter" + end_idx = lines.index("---", 1) + yaml_block = "\n".join(lines[1:end_idx]) + parsed = yaml.safe_load(yaml_block) + assert isinstance(parsed, dict) + assert "experiment_type" in parsed diff --git a/tests/recipe/test_repository.py b/tests/recipe/test_repository.py index c42edc6570..a079960257 100644 --- a/tests/recipe/test_repository.py +++ b/tests/recipe/test_repository.py @@ -8,10 +8,12 @@ import pytest from autoskillit.core import RecipeSource -from autoskillit.core._type_results import LoadResult +from autoskillit.core.types._type_results import LoadResult from autoskillit.recipe.repository import DefaultRecipeRepository from autoskillit.recipe.schema import RecipeInfo +pytestmark = [pytest.mark.layer("recipe"), pytest.mark.small] + def _make_recipe_info(name: str, path: Path) -> RecipeInfo: return RecipeInfo( @@ -181,4 +183,90 @@ def test_list_all_delegates_to_api() -> None: result = repo.list_all() assert result == expected - mock_api.assert_called_once_with(project_dir=None) + mock_api.assert_called_once_with(project_dir=None, features=None) + + +# --------------------------------------------------------------------------- +# Protocol boundary enforcement +# --------------------------------------------------------------------------- + + +def test_recipe_repository_protocol_find_return_type_is_recipe_info() -> None: + """RecipeRepository.find() must declare RecipeInfo | None. + + If it returns Any, mypy cannot catch callers accessing Recipe-only attributes + on the result — exactly the bug class this guard is designed to prevent. + + Uses string inspection because from __future__ import annotations stores + annotations as strings at runtime. + """ + import inspect + + from autoskillit.core.types._type_protocols_recipe import RecipeRepository + + sig = inspect.signature(RecipeRepository.find) + ann = sig.return_annotation + assert "RecipeInfo" in str(ann), ( + f"RecipeRepository.find() must return RecipeInfo | None. " + f"Got: {ann!r}. " + "Returning Any silences mypy on all callers and hides type boundary violations." + ) + + +def test_in_memory_recipe_repo_rejects_recipe_objects() -> None: + """InMemoryRecipeRepository.add_recipe must only accept RecipeInfo objects. + + Accepting Recipe objects masks the production type mismatch in all dispatch tests. + """ + from autoskillit.recipe.schema import Recipe + from tests.fakes import InMemoryRecipeRepository + + repo = InMemoryRecipeRepository() + with pytest.raises(TypeError, match="RecipeInfo"): + repo.add_recipe("x", Recipe(name="x", description="x")) + + +def test_load_and_validate_coerces_str_project_dir_to_path(tmp_path: Path) -> None: + mock_api = MagicMock(return_value={}) + foo = _make_recipe_info("test-recipe", tmp_path / "test-recipe.yaml") + str_dir = str(tmp_path) + + with patch("autoskillit.recipe._api.load_and_validate", mock_api): + with patch("autoskillit.recipe.repository.list_recipes", return_value=_load_result(foo)): + with patch("autoskillit.recipe.repository._dir_mtime", return_value=1.0): + repo = DefaultRecipeRepository() + repo.load_and_validate("test-recipe", str_dir) + + call_kwargs = mock_api.call_args + assert isinstance(call_kwargs.kwargs["project_dir"], Path), ( + f"Expected Path, got {type(call_kwargs.kwargs['project_dir'])}" + ) + assert call_kwargs.kwargs["project_dir"] == tmp_path + + +def test_repository_load_and_validate_passes_recipe_list_to_api(tmp_path: Path) -> None: + """DefaultRecipeRepository passes its cached recipe list to _api.load_and_validate.""" + captured_kwargs = {} + + def capturing_load_and_validate(*args, **kwargs): + captured_kwargs.update(kwargs) + return {} + + foo = _make_recipe_info("foo", tmp_path / "foo.yaml") + + with patch.object( + DefaultRecipeRepository, + "_get_list", + return_value=_load_result(foo), + ): + with patch( + "autoskillit.recipe._api.load_and_validate", + side_effect=capturing_load_and_validate, + ): + repo = DefaultRecipeRepository() + repo.load_and_validate("foo", tmp_path) + + assert "recipe_list" in captured_kwargs + assert captured_kwargs["recipe_list"] is not None + assert isinstance(captured_kwargs["recipe_list"], list) + assert captured_kwargs["recipe_list"][0].name == "foo" diff --git a/tests/recipe/test_research_archive_recipe.py b/tests/recipe/test_research_archive_recipe.py new file mode 100644 index 0000000000..811a70a69d --- /dev/null +++ b/tests/recipe/test_research_archive_recipe.py @@ -0,0 +1,213 @@ +from __future__ import annotations + +import pytest + +from autoskillit.recipe.io import builtin_recipes_dir, load_recipe +from autoskillit.recipe.validator import validate_recipe_structure + +pytestmark = [pytest.mark.layer("recipe"), pytest.mark.small] + + +class TestResearchArchiveRecipe: + @pytest.fixture(scope="class") + def recipe(self): + return load_recipe(builtin_recipes_dir() / "research-archive.yaml") + + # ── Header ────────────────────────────────────────────────────── + + def test_validates_with_zero_errors(self, recipe) -> None: + errors = validate_recipe_structure(recipe) + assert errors == [], f"Validation errors: {errors}" + + def test_recipe_name(self, recipe) -> None: + assert recipe.name == "research-archive" + + def test_recipe_version(self, recipe) -> None: + assert recipe.recipe_version == "1.0.0" + + def test_categories(self, recipe) -> None: + assert recipe.categories == ["research-family"] + + def test_requires_packs(self, recipe) -> None: + assert set(recipe.requires_packs) == {"github", "ci"} + + def test_no_autoskillit_version(self) -> None: + # Intentionally does NOT use self.recipe fixture — validates raw file content + path = builtin_recipes_dir() / "research-archive.yaml" + content = path.read_text() + assert "autoskillit_version" not in content + + # ── Ingredients ───────────────────────────────────────────────── + + def test_ingredient_names(self, recipe) -> None: + assert set(recipe.ingredients.keys()) == { + "worktree_path", + "research_dir", + "pr_url", + "all_diagram_paths", + "report_path_after_finalize", + "base_branch", + } + + def test_campaign_sourced_ingredients_hidden_and_required(self, recipe) -> None: + for name in ("worktree_path", "research_dir", "pr_url"): + ing = recipe.ingredients[name] + assert ing.required is True, f"{name} should be required" + assert ing.hidden is True, f"{name} should be hidden" + + def test_base_branch_default(self, recipe) -> None: + assert recipe.ingredients["base_branch"].default == "main" + + def test_base_branch_not_hidden(self, recipe) -> None: + assert recipe.ingredients["base_branch"].hidden is False + + # ── Steps ─────────────────────────────────────────────────────── + + def test_step_names(self, recipe) -> None: + expected = { + "begin_archival", + "capture_experiment_branch", + "create_artifact_branch", + "open_artifact_pr", + "tag_experiment_branch", + "close_experiment_pr", + "patch_token_summary", + "research_complete", + "escalate_stop", + } + assert set(recipe.steps.keys()) == expected + + # ── Critical routing fix: inputs.pr_url ───────────────────────── + + def test_begin_archival_uses_inputs_pr_url(self, recipe) -> None: + step = recipe.steps["begin_archival"] + assert step.action == "route" + conditions = step.on_result.conditions + pr_cond = next( + (c for c in conditions if c.when and "inputs.pr_url" in c.when), + None, + ) + assert pr_cond is not None, "begin_archival must route on inputs.pr_url" + assert pr_cond.route == "capture_experiment_branch" + + def test_begin_archival_fallback_routes_to_patch(self, recipe) -> None: + step = recipe.steps["begin_archival"] + conditions = step.on_result.conditions + fallback = [c for c in conditions if c.when is None] + assert len(fallback) == 1 + assert fallback[0].route == "patch_token_summary" + + def test_no_context_pr_url_in_recipe(self) -> None: + # Intentionally does NOT use self.recipe fixture — validates raw file content + content = (builtin_recipes_dir() / "research-archive.yaml").read_text() + assert "context.pr_url" not in content + + # ── Capture fields ────────────────────────────────────────────── + + def test_capture_experiment_branch(self, recipe) -> None: + step = recipe.steps["capture_experiment_branch"] + assert "experiment_branch" in step.capture + assert step.on_success == "create_artifact_branch" + assert step.on_failure == "patch_token_summary" + + def test_create_artifact_branch_capture(self, recipe) -> None: + step = recipe.steps["create_artifact_branch"] + assert "artifact_branch" in step.capture + assert step.on_success == "open_artifact_pr" + + def test_open_artifact_pr_capture(self, recipe) -> None: + step = recipe.steps["open_artifact_pr"] + assert "artifact_pr_url" in step.capture + assert step.on_success == "tag_experiment_branch" + + def test_open_artifact_pr_uses_inputs_pr_url(self, recipe) -> None: + step = recipe.steps["open_artifact_pr"] + assert "inputs.pr_url" in step.with_args.get("cmd", "") + + def test_tag_experiment_branch_capture(self, recipe) -> None: + step = recipe.steps["tag_experiment_branch"] + assert "archive_tag" in step.capture + assert step.on_success == "close_experiment_pr" + + def test_close_experiment_pr_uses_inputs_pr_url(self, recipe) -> None: + step = recipe.steps["close_experiment_pr"] + assert "inputs.pr_url" in step.with_args.get("cmd", "") + + # ── Graceful degradation ──────────────────────────────────────── + + def test_all_failures_route_to_patch_token_summary(self, recipe) -> None: + for name in ( + "capture_experiment_branch", + "create_artifact_branch", + "open_artifact_pr", + "tag_experiment_branch", + "close_experiment_pr", + ): + step = recipe.steps[name] + assert step.on_failure == "patch_token_summary", ( + f"{name} on_failure should be patch_token_summary" + ) + + # ── patch_token_summary ───────────────────────────────────────── + + def test_patch_token_summary_callable(self, recipe) -> None: + step = recipe.steps["patch_token_summary"] + assert step.tool == "run_python" + assert step.with_args["callable"] == "autoskillit.smoke_utils.patch_pr_token_summary" + + def test_patch_token_summary_uses_inputs_pr_url(self, recipe) -> None: + step = recipe.steps["patch_token_summary"] + assert step.with_args["pr_url"] == "${{ inputs.pr_url }}" + + def test_patch_token_summary_routes_to_complete(self, recipe) -> None: + step = recipe.steps["patch_token_summary"] + assert step.on_success == "research_complete" + assert step.on_failure == "escalate_stop" + + # ── Terminal stops ────────────────────────────────────────────── + + def test_research_complete_is_stop(self, recipe) -> None: + step = recipe.steps["research_complete"] + assert step.action == "stop" + assert len(step.message) >= 10 + + def test_escalate_stop_is_stop(self, recipe) -> None: + step = recipe.steps["escalate_stop"] + assert step.action == "stop" + assert len(step.message) >= 10 + + def test_escalate_stop_is_reachable(self, recipe) -> None: + routers = [ + name + for name, step in recipe.steps.items() + if step.on_failure == "escalate_stop" or step.on_success == "escalate_stop" + ] + assert routers, "escalate_stop must be reachable from at least one step" + + # ── Kitchen rules ─────────────────────────────────────────────── + + def test_kitchen_rules_forbid_native_tools(self, recipe) -> None: + combined = " ".join(recipe.kitchen_rules) + for tool in ( + "Read", + "Grep", + "Glob", + "Edit", + "Write", + "Bash", + "Agent", + "WebFetch", + "WebSearch", + "NotebookEdit", + ): + assert tool in combined, f"kitchen_rules must mention {tool}" + + # ── cwd uses inputs.worktree_path ─────────────────────────────── + + def test_capture_experiment_branch_cwd(self, recipe) -> None: + step = recipe.steps["capture_experiment_branch"] + assert step.with_args.get("cwd") == "${{ inputs.worktree_path }}" + + def test_patch_token_summary_cwd(self, recipe) -> None: + step = recipe.steps["patch_token_summary"] + assert step.with_args.get("cwd") == "${{ inputs.worktree_path }}" diff --git a/tests/recipe/test_research_bundle_lifecycle.py b/tests/recipe/test_research_bundle_lifecycle.py index 5ded9c33ff..9091dececd 100644 --- a/tests/recipe/test_research_bundle_lifecycle.py +++ b/tests/recipe/test_research_bundle_lifecycle.py @@ -2,6 +2,8 @@ from autoskillit.recipe.io import builtin_recipes_dir, load_recipe +pytestmark = [pytest.mark.layer("recipe"), pytest.mark.small] + RESEARCH_RECIPE_PATH = builtin_recipes_dir() / "research.yaml" @@ -11,10 +13,10 @@ def recipe(): def test_stage_bundle_is_idempotent(recipe): - """stage_bundle cmd uses cp (idempotent); no mv/rename, no compression.""" + """stage_bundle invokes an external script; no inline compression or commit.""" step = recipe.steps["stage_bundle"] cmd = step.with_args.get("cmd", "") - assert "cp " in cmd, "stage_bundle must use cp for idempotent file copying" + assert "stage_bundle.sh" in cmd, "stage_bundle must delegate to scripts/recipe/stage_bundle.sh" assert "tar czf" not in cmd and "tar -czf" not in cmd, ( "stage_bundle must NOT compress — second run must be a no-op" ) @@ -31,48 +33,30 @@ def test_stage_bundle_does_not_compress(recipe): def test_finalize_bundle_pr_mode(recipe): - """finalize_bundle must rename report.md→README.md, compress, append manifest, commit.""" + """finalize_bundle must delegate to external script with output_mode, research_dir.""" step = recipe.steps["finalize_bundle"] cmd = step.with_args.get("cmd", "") - # rename - assert "README.md" in cmd and "report.md" in cmd, ( - "finalize_bundle must rename report.md to README.md" - ) - # compression - assert "tar czf" in cmd or "tar -czf" in cmd, "finalize_bundle must create tarball" - assert "-C" in cmd or "--directory" in cmd, "finalize_bundle must use -C for relative paths" - # manifest - assert "tar tzf" in cmd, "finalize_bundle must generate archive manifest" - assert "Archive Manifest" in cmd, "finalize_bundle manifest section header required" - assert ">> " in cmd and "README.md" in cmd, "manifest must be appended (>>) to README.md" - # post-compression guard - assert "artifacts.tar.gz" in cmd and "exit 1" in cmd, ( - "finalize_bundle must guard that artifacts.tar.gz exists or exit 1" + assert "finalize_bundle.sh" in cmd, ( + "finalize_bundle must delegate to scripts/recipe/finalize_bundle.sh" ) - # commit - assert "git commit" in cmd, "finalize_bundle must commit the result" - # dynamic tar inputs - assert "ls -1" in cmd and "grep -vE" in cmd, ( - "finalize_bundle must use ls -1 | grep -vE for dynamic TAR_ITEMS" + assert "inputs.output_mode" in cmd, ( + "finalize_bundle script must receive output_mode as first argument" ) - # cleanup loop - assert "for item in" in cmd and "rm -rf" in cmd, ( - "finalize_bundle must rm -rf each archived item" + assert "context.research_dir" in cmd, ( + "finalize_bundle script must receive research_dir as argument" ) - # rename before archive (rename_pos < tar_pos) - rename_pos = cmd.find("report.md") - tar_pos = cmd.find("tar czf") - assert rename_pos < tar_pos, ( - "report.md rename must appear before tar czf so README.md is excluded" + assert "context.worktree_path" in cmd, ( + "finalize_bundle script must receive worktree_path as argument" ) def test_finalize_bundle_runs_exactly_once_after_rerun(recipe): - """finalize_bundle is only reachable via re_push_research.on_success.""" - # only entry point is re_push_research.on_success - re_push = recipe.steps["re_push_research"] - assert re_push.on_success == "finalize_bundle", ( - "re_push_research.on_success must be finalize_bundle" + """finalize_bundle entry point is merge_escalations (not re_push_research).""" + # merge_escalations is the step that routes to finalize_bundle + merge = recipe.steps["merge_escalations"] + fallthrough_routes = [cond.route for cond in merge.on_result.conditions if cond.when is None] + assert "finalize_bundle" in fallthrough_routes, ( + "merge_escalations fallthrough must reach finalize_bundle exactly once" ) # test and retest do NOT route to finalize_bundle (they route to push_branch) assert recipe.steps["test"].on_success != "finalize_bundle" @@ -82,3 +66,32 @@ def test_finalize_bundle_runs_exactly_once_after_rerun(recipe): assert "tar czf" not in stage_cmd and "tar -czf" not in stage_cmd, ( "stage_bundle must not compress — only finalize_bundle may produce artifacts.tar.gz" ) + + +def test_compression_commit_precedes_push(recipe): + """merge_escalations must route to finalize_bundle, not re_push_research.""" + merge = recipe.steps["merge_escalations"] + # The fallthrough route (last on_result entry without a when-condition) must + # be finalize_bundle, not re_push_research. + fallthrough_routes = [cond.route for cond in merge.on_result.conditions if cond.when is None] + assert fallthrough_routes == ["finalize_bundle"], ( + "merge_escalations fallthrough must route to finalize_bundle " + "so the compression commit is created before the push" + ) + + +def test_finalize_bundle_on_success_routes_to_re_push_research(recipe): + """finalize_bundle must push after committing — on_success must be re_push_research.""" + step = recipe.steps["finalize_bundle"] + assert step.on_success == "re_push_research", ( + "finalize_bundle.on_success must be re_push_research so the compression " + "commit is included in the push" + ) + + +def test_re_push_research_on_success_routes_to_finalize_bundle_render(recipe): + """re_push_research must advance to HTML rendering, not loop back to finalize_bundle.""" + step = recipe.steps["re_push_research"] + assert step.on_success == "finalize_bundle_render", ( + "re_push_research.on_success must be finalize_bundle_render" + ) diff --git a/tests/recipe/test_research_campaign.py b/tests/recipe/test_research_campaign.py new file mode 100644 index 0000000000..6adb1a78aa --- /dev/null +++ b/tests/recipe/test_research_campaign.py @@ -0,0 +1,37 @@ +"""Contract card assertions for the research-campaign recipe.""" + +import pytest +import yaml + +from autoskillit.recipe.contracts import check_contract_staleness, load_bundled_manifest +from autoskillit.recipe.io import builtin_recipes_dir + +pytestmark = [pytest.mark.layer("recipe"), pytest.mark.medium] + + +def test_research_campaign_contract_exists(): + contracts_dir = builtin_recipes_dir() / "contracts" + assert (contracts_dir / "research-campaign.yaml").exists(), ( + 'Regenerate: python -c "from autoskillit.recipe.contracts import generate_recipe_card; ' + "from autoskillit.recipe.io import builtin_recipes_dir; " + "generate_recipe_card(" + "builtin_recipes_dir()/'campaigns'/'research-campaign.yaml', builtin_recipes_dir())\"" + ) + + +def test_research_campaign_contract_is_fresh(): + contract_path = builtin_recipes_dir() / "contracts" / "research-campaign.yaml" + contract = yaml.safe_load(contract_path.read_text()) + assert isinstance(contract, dict), f"Malformed contract: expected dict, got {type(contract)}" + stale = check_contract_staleness(contract) + assert stale == [], f"Contract is stale: {stale}" + + +def test_research_campaign_contract_version_matches(): + contract_path = builtin_recipes_dir() / "contracts" / "research-campaign.yaml" + contract = yaml.safe_load(contract_path.read_text()) + assert isinstance(contract, dict), f"Malformed contract: expected dict, got {type(contract)}" + assert "bundled_manifest_version" in contract, ( + f"Contract missing 'bundled_manifest_version' key: {list(contract.keys())}" + ) + assert contract["bundled_manifest_version"] == load_bundled_manifest()["version"] diff --git a/tests/recipe/test_research_campaign_rules.py b/tests/recipe/test_research_campaign_rules.py new file mode 100644 index 0000000000..7fb1bd3acf --- /dev/null +++ b/tests/recipe/test_research_campaign_rules.py @@ -0,0 +1,81 @@ +"""Contract tests: research-campaign.yaml semantic rules.""" + +from __future__ import annotations + +import pytest + +import autoskillit.recipe # noqa: F401 -- pyright: ignore[reportUnusedImport] -- triggers rule registration +from autoskillit.core import Severity +from autoskillit.recipe._analysis import make_validation_context +from autoskillit.recipe.io import builtin_recipes_dir, load_recipe +from autoskillit.recipe.registry import _RULE_REGISTRY, run_semantic_rules + +pytestmark = [pytest.mark.layer("recipe"), pytest.mark.small] + +RECIPE_PATH = builtin_recipes_dir() / "campaigns" / "research-campaign.yaml" + + +@pytest.fixture(scope="module") +def validation_ctx(): + if not RECIPE_PATH.exists(): + pytest.skip("research-campaign.yaml not yet created") + recipe = load_recipe(RECIPE_PATH) + return make_validation_context( + recipe, + available_recipes=frozenset( + {"research-design", "research-implement", "research-review", "research-archive"} + ), + project_dir=RECIPE_PATH.parent.parent.parent, + ) + + +def test_research_campaign_file_exists() -> None: + assert RECIPE_PATH.exists() + + +def test_research_campaign_no_error_findings(validation_ctx) -> None: + findings = run_semantic_rules(validation_ctx) + errors = [f for f in findings if f.severity == Severity.ERROR] + assert not errors, f"{len(errors)} ERROR finding(s): " + "; ".join( + f"{f.rule}: {f.message}" for f in errors + ) + + +def test_dispatch_recipe_exists_passes(validation_ctx) -> None: + findings = run_semantic_rules(validation_ctx) + matched = [f for f in findings if f.rule == "dispatch-recipe-exists"] + assert not matched, "; ".join(f"{f.rule}: {f.message}" for f in matched) + + +def test_campaign_ingredient_refs_have_prior_capture_passes(validation_ctx) -> None: + findings = run_semantic_rules(validation_ctx) + matched = [f for f in findings if f.rule == "campaign-ingredient-refs-have-prior-capture"] + assert not matched, "; ".join(f"{f.rule}: {f.message}" for f in matched) + + +def test_dispatch_capture_value_references_result_passes(validation_ctx) -> None: + findings = run_semantic_rules(validation_ctx) + matched = [f for f in findings if f.rule == "dispatch-capture-value-references-result"] + assert not matched, "; ".join(f"{f.rule}: {f.message}" for f in matched) + + +def test_depends_on_acyclic_passes(validation_ctx) -> None: + findings = run_semantic_rules(validation_ctx) + matched = [f for f in findings if f.rule == "depends-on-acyclic"] + assert not matched, "; ".join(f"{f.rule}: {f.message}" for f in matched) + + +def test_dispatch_capture_fields_in_sentinel_contract_passes(validation_ctx) -> None: + rule_name = "dispatch-capture-fields-in-sentinel-contract" + registered_names = {r.name for r in _RULE_REGISTRY} + if rule_name not in registered_names: + pytest.skip(f"Rule {rule_name!r} not yet registered") + findings = run_semantic_rules(validation_ctx) + matched = [f for f in findings if f.rule == rule_name] + assert not matched, "; ".join(f"{f.rule}: {f.message}" for f in matched) + + +def test_campaign_dangling_ingredient_passes(validation_ctx) -> None: + findings = run_semantic_rules(validation_ctx) + matched = [f for f in findings if f.rule == "campaign-dangling-ingredient"] + assert not matched, "; ".join(f"{f.rule}: {f.message}" for f in matched) diff --git a/tests/recipe/test_research_campaign_structure.py b/tests/recipe/test_research_campaign_structure.py new file mode 100644 index 0000000000..e8bd30391c --- /dev/null +++ b/tests/recipe/test_research_campaign_structure.py @@ -0,0 +1,270 @@ +"""Structural assertions for the bundled research-campaign.yaml.""" + +from __future__ import annotations + +import re + +import pytest + +from autoskillit.recipe.io import builtin_recipes_dir, load_recipe +from autoskillit.recipe.schema import CAMPAIGN_REF_RE, CampaignDispatch, Recipe, RecipeKind + +pytestmark = [pytest.mark.layer("recipe"), pytest.mark.small] + +RECIPE_PATH = builtin_recipes_dir() / "campaigns" / "research-campaign.yaml" + +_IDENT_RE = re.compile(r"^[a-zA-Z_][a-zA-Z0-9_]*$") +_RESULT_TMPL_RE = re.compile(r"^\$\{\{\s*result\.\w+\s*\}\}$") + + +@pytest.fixture(scope="module") +def recipe(): + if not RECIPE_PATH.exists(): + pytest.skip("research-campaign.yaml not yet created") + return load_recipe(RECIPE_PATH) + + +def _dispatch_by_name(recipe: Recipe, name: str) -> CampaignDispatch | None: + return next((d for d in recipe.dispatches if d.name == name), None) + + +# --------------------------------------------------------------------------- +# Group 2: Kind/Count/Order +# --------------------------------------------------------------------------- + + +def test_research_campaign_kind(recipe: Recipe) -> None: + assert recipe.kind == RecipeKind.CAMPAIGN + + +def test_research_campaign_has_four_dispatches(recipe: Recipe) -> None: + assert len(recipe.dispatches) == 4 + + +def test_research_campaign_dispatch_names(recipe: Recipe) -> None: + assert [d.name for d in recipe.dispatches] == [ + "run-design", + "run-implement", + "run-review", + "run-archive", + ] + + +# --------------------------------------------------------------------------- +# Group 3: depends_on Chain +# --------------------------------------------------------------------------- + + +def test_run_design_depends_on_empty(recipe: Recipe) -> None: + d = _dispatch_by_name(recipe, "run-design") + assert d is not None + assert d.depends_on == [] + + +def test_run_implement_depends_on_design(recipe: Recipe) -> None: + d = _dispatch_by_name(recipe, "run-implement") + assert d is not None + assert d.depends_on == ["run-design"] + + +def test_run_review_depends_on_implement(recipe: Recipe) -> None: + d = _dispatch_by_name(recipe, "run-review") + assert d is not None + assert d.depends_on == ["run-implement"] + + +def test_run_archive_depends_on_review(recipe: Recipe) -> None: + d = _dispatch_by_name(recipe, "run-archive") + assert d is not None + assert d.depends_on == ["run-review"] + + +def test_depends_on_chain_is_strictly_linear(recipe: Recipe) -> None: + for d in recipe.dispatches: + assert len(d.depends_on) <= 1, f"{d.name} has {len(d.depends_on)} predecessors" + seen: dict[str, str] = {} + for d in recipe.dispatches: + if d.depends_on: + pred = d.depends_on[0] + assert pred not in seen or seen[pred] == d.name, ( + f"Predecessor {pred!r} claimed by both {seen[pred]!r} and {d.name!r}" + ) + seen[pred] = d.name + + +# --------------------------------------------------------------------------- +# Group 4: Capture Dict Validation +# --------------------------------------------------------------------------- + + +def test_run_design_has_non_empty_capture(recipe: Recipe) -> None: + d = _dispatch_by_name(recipe, "run-design") + assert d is not None + assert d.capture, "run-design capture should not be empty" + for key in d.capture: + assert _IDENT_RE.match(key), f"Capture key {key!r} is not a valid identifier" + for val in d.capture.values(): + assert _RESULT_TMPL_RE.match(val.strip()), ( + f"Capture value {val!r} does not match result template" + ) + + +def test_run_design_capture_keys(recipe: Recipe) -> None: + d = _dispatch_by_name(recipe, "run-design") + assert d is not None + assert set(d.capture.keys()) == { + "worktree_path", + "research_dir", + "experiment_plan", + "visualization_plan_path", + } + + +def test_run_implement_has_non_empty_capture(recipe: Recipe) -> None: + d = _dispatch_by_name(recipe, "run-implement") + assert d is not None + assert d.capture, "run-implement capture should not be empty" + for key in d.capture: + assert _IDENT_RE.match(key), f"Capture key {key!r} is not a valid identifier" + for val in d.capture.values(): + assert _RESULT_TMPL_RE.match(val.strip()), ( + f"Capture value {val!r} does not match result template" + ) + + +def test_run_implement_capture_keys(recipe: Recipe) -> None: + d = _dispatch_by_name(recipe, "run-implement") + assert d is not None + assert set(d.capture.keys()) == {"report_path"} + + +def test_run_review_has_non_empty_capture(recipe: Recipe) -> None: + d = _dispatch_by_name(recipe, "run-review") + assert d is not None + assert d.capture, "run-review capture should not be empty" + for key in d.capture: + assert _IDENT_RE.match(key), f"Capture key {key!r} is not a valid identifier" + for val in d.capture.values(): + assert _RESULT_TMPL_RE.match(val.strip()), ( + f"Capture value {val!r} does not match result template" + ) + + +def test_run_review_capture_keys(recipe: Recipe) -> None: + d = _dispatch_by_name(recipe, "run-review") + assert d is not None + assert set(d.capture.keys()) == { + "pr_url", + "all_diagram_paths", + "report_path_after_finalize", + } + + +def test_run_archive_has_empty_capture(recipe: Recipe) -> None: + d = _dispatch_by_name(recipe, "run-archive") + assert d is not None + assert d.capture == {} + + +# --------------------------------------------------------------------------- +# Group 5: Campaign Ref Resolution +# --------------------------------------------------------------------------- + + +def test_all_campaign_refs_resolve_to_ancestor_captures(recipe: Recipe) -> None: + accumulated_captures: set[str] = set() + for d in recipe.dispatches: + for val in d.ingredients.values(): + match = CAMPAIGN_REF_RE.match(val) + if match: + ref_name = match.group(1) + assert ref_name in accumulated_captures, ( + f"Dispatch {d.name!r} references campaign.{ref_name} " + f"but it was not captured by any ancestor dispatch" + ) + accumulated_captures.update(d.capture.keys()) + + +def test_run_implement_references_run_design_capture(recipe: Recipe) -> None: + design = _dispatch_by_name(recipe, "run-design") + implement = _dispatch_by_name(recipe, "run-implement") + assert design is not None + assert implement is not None + design_capture_keys = set(design.capture.keys()) + implement_ref_keys: set[str] = set() + for val in implement.ingredients.values(): + match = CAMPAIGN_REF_RE.match(val) + if match: + implement_ref_keys.add(match.group(1)) + assert implement_ref_keys & design_capture_keys, ( + "run-implement should reference at least one key captured by run-design" + ) + + +def test_campaign_ref_tracing_on_inline_fixture() -> None: + recipe = Recipe( + name="inline-campaign", + description="test", + kind=RecipeKind.CAMPAIGN, + dispatches=[ + CampaignDispatch( + name="phase-a", + recipe="impl", + task="Do A", + ingredients={}, + depends_on=[], + capture={"output_path": "${{ result.output_path }}"}, + ), + CampaignDispatch( + name="phase-b", + recipe="impl", + task="Do B", + ingredients={"input_path": "${{ campaign.output_path }}"}, + depends_on=["phase-a"], + capture={}, + ), + ], + ) + accumulated: set[str] = set() + for d in recipe.dispatches: + for val in d.ingredients.values(): + match = CAMPAIGN_REF_RE.match(val) + if match: + assert match.group(1) in accumulated + accumulated.update(d.capture.keys()) + + +def test_campaign_ref_missing_capture_is_detectable() -> None: + recipe = Recipe( + name="broken-campaign", + description="test", + kind=RecipeKind.CAMPAIGN, + dispatches=[ + CampaignDispatch( + name="phase-a", + recipe="impl", + task="Do A", + ingredients={}, + depends_on=[], + capture={"known_key": "${{ result.known_key }}"}, + ), + CampaignDispatch( + name="phase-b", + recipe="impl", + task="Do B", + ingredients={"bad_ref": "${{ campaign.missing_key }}"}, + depends_on=["phase-a"], + capture={}, + ), + ], + ) + accumulated: set[str] = set() + unresolved: list[tuple[str, str]] = [] + for d in recipe.dispatches: + for val in d.ingredients.values(): + match = CAMPAIGN_REF_RE.match(val) + if match and match.group(1) not in accumulated: + unresolved.append((d.name, match.group(1))) + accumulated.update(d.capture.keys()) + assert len(unresolved) == 1 + assert unresolved[0] == ("phase-b", "missing_key") diff --git a/tests/recipe/test_research_context_tracking.py b/tests/recipe/test_research_context_tracking.py new file mode 100644 index 0000000000..c55345c63d --- /dev/null +++ b/tests/recipe/test_research_context_tracking.py @@ -0,0 +1,123 @@ +from __future__ import annotations + +import re + +import pytest + +from autoskillit.core.paths import pkg_root +from autoskillit.recipe.io import builtin_recipes_dir, load_recipe +from autoskillit.recipe.validator import run_semantic_rules, validate_recipe_structure + +RESEARCH_RECIPE_PATH = builtin_recipes_dir() / "research.yaml" +SCRIPTS_DIR = pkg_root().parent.parent / "scripts" / "recipe" + +pytestmark = [pytest.mark.layer("recipe"), pytest.mark.small] + +_FIND_HEURISTIC_RE = re.compile(r"\bfind\b.+\|\s*sort\b.+\|\s*(tail|head)\b") + + +def _has_find_heuristic(text: str) -> bool: + return bool(_FIND_HEURISTIC_RE.search(text)) + + +@pytest.fixture(scope="module") +def recipe(): + return load_recipe(RESEARCH_RECIPE_PATH) + + +def test_create_worktree_captures_research_dir(recipe): + step = recipe.steps["create_worktree"] + assert "research_dir" in step.capture, ( + "create_worktree must capture research_dir for downstream steps" + ) + + +def test_create_worktree_script_emits_research_dir(): + script_path = SCRIPTS_DIR / "create_worktree.sh" + assert script_path.exists(), f"create_worktree.sh not found at {script_path}" + script = script_path.read_text() + assert 'echo "research_dir=' in script, ( + "create_worktree.sh must emit research_dir= to stdout for capture" + ) + + +def test_create_worktree_script_has_git_init_guard(): + """create_worktree.sh must auto-init a git repo when .git is absent.""" + script_path = SCRIPTS_DIR / "create_worktree.sh" + script = script_path.read_text() + assert '[ ! -d "$SOURCE_DIR/.git" ]' in script or "! -d" in script, ( + "create_worktree.sh must check for .git directory absence" + ) + assert "git init" in script, "create_worktree.sh must run git init when .git is missing" + assert "--allow-empty" in script, ( + "create_worktree.sh must create an empty seed commit for git worktree add" + ) + + +def test_create_worktree_git_init_guard_precedes_worktree_add(): + """The git-init guard must appear BEFORE git worktree add.""" + script_path = SCRIPTS_DIR / "create_worktree.sh" + lines = script_path.read_text().splitlines() + init_line = next(i for i, ln in enumerate(lines) if "git init" in ln) + worktree_line = next(i for i, ln in enumerate(lines) if "worktree add" in ln) + assert init_line < worktree_line, "git init guard must be PREPENDED before git worktree add" + + +def test_finalize_bundle_uses_context_research_dir(recipe): + step = recipe.steps["finalize_bundle"] + cmd = step.with_args.get("cmd", "") + assert "context.research_dir" in cmd, ( + "finalize_bundle must source research_dir from context, not re-discover it" + ) + + +def test_finalize_bundle_no_find_heuristic(recipe): + step = recipe.steps["finalize_bundle"] + cmd = step.with_args.get("cmd", "") + assert not _has_find_heuristic(cmd), ( + "finalize_bundle cmd uses find|sort|tail heuristic — must use context.research_dir instead" + ) + script_path = SCRIPTS_DIR / "finalize_bundle.sh" + assert script_path.exists(), f"finalize_bundle.sh not found at {script_path}" + script = script_path.read_text() + assert not _has_find_heuristic(script), ( + "finalize_bundle.sh uses find|sort|tail heuristic — " + "must use positional arg from context.research_dir instead" + ) + + +def test_create_artifact_branch_scopes_via_research_dir(recipe): + step = recipe.steps["create_artifact_branch"] + cmd = step.with_args.get("cmd", "") + assert "context.research_dir" in cmd, ( + "create_artifact_branch must pass context.research_dir to script" + ) + script_path = SCRIPTS_DIR / "create_artifact_branch.sh" + assert script_path.exists(), f"create_artifact_branch.sh not found at {script_path}" + script = script_path.read_text() + assert "basename" in script, "create_artifact_branch.sh must use basename to scope checkout" + + +def test_finalize_bundle_has_post_compression_guard(): + script_path = SCRIPTS_DIR / "finalize_bundle.sh" + assert script_path.exists(), f"finalize_bundle.sh not found at {script_path}" + script = script_path.read_text() + assert re.search( + r"artifacts\.tar\.gz[^\n]*\|\|[^\n]*exit\s+1|artifacts\.tar\.gz.*\\\n.*exit\s+1", script + ), "finalize_bundle.sh must conditionally exit 1 when artifacts.tar.gz is absent" + + +def test_research_recipe_passes_semantic_rules(recipe): + errors = validate_recipe_structure(recipe) + assert not errors, f"Structural validation errors: {errors}" + findings = run_semantic_rules(recipe) + from tests.recipe.conftest import KNOWN_PART_B_VIOLATIONS + + error_findings = [ + f + for f in findings + if f.severity.value == "error" and f.rule not in KNOWN_PART_B_VIOLATIONS + ] + assert not error_findings, ( + f"Semantic rule errors: {[f'{f.rule}: {f.message}' for f in error_findings]}" + ) diff --git a/tests/recipe/test_research_implement_recipe.py b/tests/recipe/test_research_implement_recipe.py new file mode 100644 index 0000000000..3edafb9a78 --- /dev/null +++ b/tests/recipe/test_research_implement_recipe.py @@ -0,0 +1,57 @@ +from __future__ import annotations + +import pytest + +from autoskillit.recipe.io import builtin_recipes_dir, load_recipe +from autoskillit.recipe.validator import validate_recipe_structure + +pytestmark = [pytest.mark.layer("recipe"), pytest.mark.small] + + +class TestResearchImplementRecipe: + @pytest.fixture(scope="module") + def recipe(self): + return load_recipe(builtin_recipes_dir() / "research-implement.yaml") + + def test_research_implement_validates_clean(self, recipe) -> None: + errors = validate_recipe_structure(recipe) + assert errors == [], f"Validation errors: {errors}" + + def test_research_implement_step_count(self, recipe) -> None: + assert len(recipe.steps) == 22 + + def test_research_implement_header(self, recipe) -> None: + assert recipe.name == "research-implement" + assert recipe.recipe_version == "1.0.0" + assert "research-family" in recipe.categories + assert "research" in recipe.requires_packs + + def test_research_implement_ingredients(self, recipe) -> None: + names = set(recipe.ingredients.keys()) + assert {"task", "source_dir", "base_branch", "output_mode", "issue_url"} <= names + assert {"worktree_path", "research_dir", "experiment_plan"} <= names + + def test_excluded_ingredients_absent(self, recipe) -> None: + names = set(recipe.ingredients.keys()) + assert "scope_report" not in names + assert "report_plan_path" not in names + assert "experiment_type" not in names + + def test_no_dangling_upstream_refs(self) -> None: + path = builtin_recipes_dir() / "research-implement.yaml" + content = path.read_text() + assert "context.scope_report" not in content + assert "context.visualization_plan_path" not in content + assert "context.report_plan_path" not in content + assert "context.experiment_type" not in content + + def test_push_branch_routes_to_implement_complete(self, recipe) -> None: + push_step = recipe.steps["push_branch"] + assert push_step.on_success == "implement_complete" + + def test_terminal_stops(self, recipe) -> None: + assert recipe.steps["escalate_stop"].action == "stop" + assert recipe.steps["implement_complete"].action == "stop" + assert "${{ context.worktree_path }}" in recipe.steps["implement_complete"].message + assert "${{ context.report_path }}" in recipe.steps["implement_complete"].message + assert "${{ context.experiment_results }}" in recipe.steps["implement_complete"].message diff --git a/tests/recipe/test_research_output_mode.py b/tests/recipe/test_research_output_mode.py index 95530cad30..6971666e73 100644 --- a/tests/recipe/test_research_output_mode.py +++ b/tests/recipe/test_research_output_mode.py @@ -3,6 +3,8 @@ from autoskillit.recipe.io import builtin_recipes_dir, load_recipe +pytestmark = [pytest.mark.layer("recipe"), pytest.mark.small] + RESEARCH_RECIPE_PATH = builtin_recipes_dir() / "research.yaml" @@ -110,28 +112,35 @@ def test_export_local_bundle_exists(recipe): def test_export_local_bundle_emits_local_bundle_path(recipe): step = recipe.steps["export_local_bundle"] - cmd = step.with_args.get("cmd", "") - assert "local_bundle_path=" in cmd, ( - "export_local_bundle cmd must emit local_bundle_path= for capture" + # export_local_bundle is a run_python callable step — no inline cmd. + # Verify the callable is registered and the captured output key is declared. + assert step.tool == "run_python", "export_local_bundle must use run_python" + assert step.with_args.get("callable") == "autoskillit.recipe._cmd_rpc.export_local_bundle", ( + "export_local_bundle must reference the export_local_bundle callable" + ) + assert "local_bundle_path" in (step.capture or {}), ( + "export_local_bundle must capture local_bundle_path" ) def test_export_local_bundle_uses_source_dir_research_bundles(recipe): step = recipe.steps["export_local_bundle"] - cmd = step.with_args.get("cmd", "") - assert "research-bundles" in cmd, ( - "export_local_bundle must export to {source_dir}/research-bundles/" + # export_local_bundle is a run_python callable step — verify with_args keys. + assert "source_dir" in step.with_args, ( + "export_local_bundle must pass source_dir to the callable" + ) + assert "research_dir" in step.with_args, ( + "export_local_bundle must pass research_dir to the callable" ) - assert "source_dir" in cmd, "export_local_bundle cmd must reference inputs.source_dir" -def test_export_local_bundle_routes_to_research_complete(recipe): +def test_export_local_bundle_routes_to_patch_token_summary(recipe): step = recipe.steps["export_local_bundle"] - assert step.on_success == "research_complete", ( - "export_local_bundle.on_success must be research_complete" + assert step.on_success == "patch_token_summary", ( + "export_local_bundle.on_success must be patch_token_summary" ) - assert step.on_failure == "research_complete", ( - "export_local_bundle.on_failure must be research_complete" + assert step.on_failure == "patch_token_summary", ( + "export_local_bundle.on_failure must be patch_token_summary" ) @@ -139,42 +148,54 @@ def test_export_local_bundle_routes_to_research_complete(recipe): def test_finalize_bundle_reads_output_mode(recipe): - """finalize_bundle cmd must read OUTPUT_MODE from inputs.output_mode.""" + """finalize_bundle cmd must pass output_mode to the external script.""" step = recipe.steps["finalize_bundle"] cmd = step.with_args.get("cmd", "") - assert "OUTPUT_MODE" in cmd, "finalize_bundle must set OUTPUT_MODE from inputs.output_mode" - assert "inputs.output_mode" in cmd or "inputs.output_mode" in cmd, ( - "finalize_bundle cmd must reference ${{ inputs.output_mode }}" + assert "finalize_bundle.sh" in cmd, ( + "finalize_bundle must delegate to scripts/recipe/finalize_bundle.sh" + ) + assert "inputs.output_mode" in cmd, ( + "finalize_bundle cmd must pass ${{ inputs.output_mode }} as first script argument" ) def test_finalize_bundle_skips_commit_in_local_mode(recipe): - """finalize_bundle cmd must gate git commit behind OUTPUT_MODE != local.""" + """finalize_bundle delegates local/pr branching to external script.""" step = recipe.steps["finalize_bundle"] cmd = step.with_args.get("cmd", "") - # git commit must exist (for pr mode) but be conditional - assert "git commit" in cmd, "finalize_bundle must still have git commit (for pr mode)" - commit_pos = cmd.find("git commit") - output_mode_check_pos = cmd.rfind("OUTPUT_MODE", 0, commit_pos) - assert output_mode_check_pos != -1, ( - "git commit block in finalize_bundle must be guarded by OUTPUT_MODE check" + # The script receives output_mode as its first positional arg and handles mode branching. + assert "finalize_bundle.sh" in cmd, ( + "finalize_bundle must delegate to scripts/recipe/finalize_bundle.sh" + ) + assert "inputs.output_mode" in cmd, ( + "finalize_bundle must pass output_mode so the script can gate the git commit" ) def test_finalize_bundle_preserves_html_in_local_mode(recipe): - """finalize_bundle tar exclusion must include report.html for local mode browsing.""" + """finalize_bundle delegates local-mode HTML preservation to the external script.""" step = recipe.steps["finalize_bundle"] cmd = step.with_args.get("cmd", "") - assert r"report\.html" in cmd, ( - r"finalize_bundle must reference report\.html (escaped) in tar exclusion for local mode" + # The script (finalize_bundle.sh) handles report.html exclusion for local mode. + # The recipe cmd passes output_mode so the script can apply the correct exclusion. + assert "finalize_bundle.sh" in cmd, ( + "finalize_bundle must delegate to scripts/recipe/finalize_bundle.sh " + "(which excludes report.html from tar in local mode)" + ) + assert "inputs.output_mode" in cmd, ( + "finalize_bundle must pass output_mode so the script preserves report.html in local mode" ) def test_finalize_bundle_render_always_runs(recipe): - """finalize_bundle.on_success must always be finalize_bundle_render (no skip).""" + """finalize_bundle_render is always reached via re_push_research.""" fb = recipe.steps["finalize_bundle"] - assert fb.on_success == "finalize_bundle_render", ( - "finalize_bundle.on_success must be finalize_bundle_render (always runs)" + assert fb.on_success == "re_push_research", ( + "finalize_bundle.on_success must be re_push_research (push includes the commit)" + ) + rpr = recipe.steps["re_push_research"] + assert rpr.on_success == "finalize_bundle_render", ( + "re_push_research.on_success must be finalize_bundle_render (always runs)" ) fbr = recipe.steps["finalize_bundle_render"] assert not getattr(fbr, "skip_when_false", None), ( diff --git a/tests/recipe/test_research_recipe_diag.py b/tests/recipe/test_research_recipe_diag.py index 2b26cb335e..0402c34c6a 100644 --- a/tests/recipe/test_research_recipe_diag.py +++ b/tests/recipe/test_research_recipe_diag.py @@ -1,7 +1,9 @@ import pytest from autoskillit.recipe.io import builtin_recipes_dir, load_recipe -from autoskillit.recipe.validator import validate_recipe +from autoskillit.recipe.validator import validate_recipe_structure + +pytestmark = [pytest.mark.layer("recipe"), pytest.mark.small] RESEARCH_RECIPE_PATH = builtin_recipes_dir() / "research.yaml" @@ -13,7 +15,7 @@ def recipe(): def test_research_recipe_validates_after_diag_changes(recipe): """research.yaml must pass structural validation with new troubleshoot steps.""" - errors = validate_recipe(recipe) + errors = validate_recipe_structure(recipe) assert not errors, f"Validation errors: {errors}" @@ -113,3 +115,62 @@ def test_run_experiment_lenses_has_capture_list_for_diagram_paths(recipe): def test_stage_bundle_exists(recipe): """stage_bundle step must exist in the recipe.""" assert "stage_bundle" in recipe.steps, "research.yaml must have a stage_bundle step" + + +def test_run_experiment_failure_routes_to_troubleshoot(recipe): + """run_experiment on_failure must route to troubleshoot_run_failure.""" + step = recipe.steps["run_experiment"] + assert step.on_failure == "troubleshoot_run_failure" + + +def test_research_recipe_has_troubleshoot_run_steps(recipe): + """research.yaml must contain the troubleshoot_run_failure and route_run_failure steps.""" + step_names = list(recipe.steps.keys()) + assert "troubleshoot_run_failure" in step_names + assert "route_run_failure" in step_names + + +def test_troubleshoot_run_captures_required_tokens(recipe): + """troubleshoot_run_failure must capture run_is_fixable for downstream routing.""" + step = recipe.steps["troubleshoot_run_failure"] + capture = step.capture or {} + assert "run_is_fixable" in capture + + +def test_troubleshoot_run_uses_run_experiment_step_name(recipe): + """troubleshoot_run_failure skill_command must pass run_experiment, not implement_phase.""" + step = recipe.steps["troubleshoot_run_failure"] + skill_cmd = step.with_args.get("skill_command", "") + assert "troubleshoot-experiment" in skill_cmd + assert "run_experiment" in skill_cmd + assert "implement_phase" not in skill_cmd + + +def test_route_run_failure_routes_fixable_to_adjust(recipe): + """route_run_failure must route fixable failures to adjust_experiment.""" + step = recipe.steps["route_run_failure"] + assert step.action == "route" + conditions = step.on_result.conditions + fixable_cond = next(c for c in conditions if c.when and "run_is_fixable" in c.when) + assert fixable_cond.route == "adjust_experiment" + + +def test_route_run_failure_default_escalates(recipe): + """route_run_failure catch-all must route to escalate_stop.""" + step = recipe.steps["route_run_failure"] + conditions = step.on_result.conditions + default_cond = next(c for c in conditions if c.when is None) + assert default_cond.route == "escalate_stop" + + +def test_adjust_experiment_routing_unchanged(recipe): + """adjust_experiment routes through check_run_fix_loop guard before run_experiment.""" + step = recipe.steps["adjust_experiment"] + assert step.on_success == "check_run_fix_loop" + assert step.on_failure == "ensure_results" + + +def test_run_experiment_exhausted_unchanged(recipe): + """run_experiment on_exhausted must still route to ensure_results.""" + step = recipe.steps["run_experiment"] + assert step.on_exhausted == "ensure_results" diff --git a/tests/recipe/test_research_review_recipe.py b/tests/recipe/test_research_review_recipe.py new file mode 100644 index 0000000000..cf78a545f3 --- /dev/null +++ b/tests/recipe/test_research_review_recipe.py @@ -0,0 +1,194 @@ +from __future__ import annotations + +import pytest + +from autoskillit.recipe.io import builtin_recipes_dir, load_recipe +from autoskillit.recipe.validator import validate_recipe_structure + +pytestmark = [pytest.mark.layer("recipe"), pytest.mark.small] + + +class TestResearchReviewRecipe: + @pytest.fixture(scope="class") + def recipe(self): + return load_recipe(builtin_recipes_dir() / "research-review.yaml") + + # --- Header --- + def test_loads_without_exception(self, recipe) -> None: + assert recipe is not None + + def test_validates_with_zero_errors(self, recipe) -> None: + errors = validate_recipe_structure(recipe) + assert errors == [], f"Validation errors: {errors}" + + def test_recipe_name(self, recipe) -> None: + assert recipe.name == "research-review" + + def test_recipe_version(self, recipe) -> None: + assert recipe.recipe_version == "1.0.0" + + def test_categories(self, recipe) -> None: + assert recipe.categories == ["research-family"] + + def test_requires_packs(self, recipe) -> None: + assert set(recipe.requires_packs) == {"research", "exp-lens", "vis-lens"} + + def test_no_autoskillit_version(self) -> None: + path = builtin_recipes_dir() / "research-review.yaml" + content = path.read_text() + assert "autoskillit_version" not in content + + # --- Ingredients --- + def test_user_ingredient_count(self, recipe) -> None: + user_ingredients = {k: v for k, v in recipe.ingredients.items() if not v.hidden} + assert len(user_ingredients) == 15 + + def test_hidden_ingredient_count(self, recipe) -> None: + hidden = {k: v for k, v in recipe.ingredients.items() if v.hidden} + assert len(hidden) == 0 + + def test_user_ingredients_present(self, recipe) -> None: + names = set(recipe.ingredients.keys()) + assert { + "task", + "source_dir", + "base_branch", + "output_mode", + "review_pr", + "audit_claims", + "issue_url", + } <= names + + def test_task_required(self, recipe) -> None: + assert recipe.ingredients["task"].required is True + + def test_source_dir_required(self, recipe) -> None: + assert recipe.ingredients["source_dir"].required is True + + def test_base_branch_default(self, recipe) -> None: + assert recipe.ingredients["base_branch"].default == "main" + + def test_output_mode_default(self, recipe) -> None: + assert recipe.ingredients["output_mode"].default == "local" + + def test_review_pr_default(self, recipe) -> None: + assert recipe.ingredients["review_pr"].default == "false" + + def test_audit_claims_default(self, recipe) -> None: + assert recipe.ingredients["audit_claims"].default == "false" + + # --- Steps --- + def test_step_count(self, recipe) -> None: + # 22 active steps + 3 terminal stops = 25 + assert len(recipe.steps) == 25 + + def test_active_step_names(self, recipe) -> None: + expected = { + "prepare_research_pr", + "run_experiment_lenses", + "stage_bundle", + "route_pr_or_local", + "compose_research_pr", + "guard_pr_url", + "review_research_pr", + "audit_claims", + "route_review_resolve", + "resolve_research_review", + "route_claims_resolve", + "resolve_claims_review", + "merge_escalations", + "re_run_experiment", + "re_generate_report", + "re_stage_bundle", + "re_test", + "re_push_research", + "finalize_bundle", + "finalize_bundle_render", + "route_archive_or_export", + "export_local_bundle", + } + assert expected <= set(recipe.steps.keys()) + + def test_terminal_stops_present(self, recipe) -> None: + assert recipe.steps["review_pr_complete"].action == "stop" + assert recipe.steps["review_local_complete"].action == "stop" + assert recipe.steps["escalate_stop"].action == "stop" + + # --- Key routing adaptations --- + def test_prepare_research_pr_uses_context_worktree_path(self, recipe) -> None: + step = recipe.steps["prepare_research_pr"] + assert "context.worktree_path" in step.with_args.get("skill_command", "") + assert step.with_args.get("cwd") == "${{ context.worktree_path }}" + + def test_finalize_bundle_routes_to_finalize_bundle_render(self, recipe) -> None: + step = recipe.steps["finalize_bundle"] + assert step.on_success == "finalize_bundle_render" + + def test_finalize_bundle_failure_routes_to_escalate_stop(self, recipe) -> None: + step = recipe.steps["finalize_bundle"] + assert step.on_failure == "escalate_stop" + + def test_route_archive_or_export_pr_fallthrough_to_review_pr_complete(self, recipe) -> None: + step = recipe.steps["route_archive_or_export"] + # The fallback (non-local) route should be review_pr_complete + conditions = step.on_result.conditions if step.on_result else [] + fallback = [c for c in conditions if c.when is None] + assert len(fallback) == 1 + assert fallback[0].route == "review_pr_complete" + + def test_export_local_bundle_routes_to_review_local_complete(self, recipe) -> None: + step = recipe.steps["export_local_bundle"] + assert step.on_success == "review_local_complete" + assert step.on_failure == "review_local_complete" + + def test_guard_pr_url_fallback_to_review_pr_complete(self, recipe) -> None: + step = recipe.steps["guard_pr_url"] + conditions = step.on_result.conditions if step.on_result else [] + fallback = [c for c in conditions if c.when is None] + assert len(fallback) == 1 + assert fallback[0].route == "review_pr_complete" + + def test_re_push_research_routes_to_finalize_bundle(self, recipe) -> None: + step = recipe.steps["re_push_research"] + assert step.on_success == "finalize_bundle" + + # --- Negative guards --- + def test_no_archival_steps(self, recipe) -> None: + step_names = set(recipe.steps.keys()) + archival_steps = { + "begin_archival", + "capture_experiment_branch", + "create_artifact_branch", + "open_artifact_pr", + "tag_experiment_branch", + "close_experiment_pr", + "patch_token_summary", + "research_complete", + } + assert archival_steps.isdisjoint(step_names) + + def test_no_patch_token_summary_references(self) -> None: + path = builtin_recipes_dir() / "research-review.yaml" + content = path.read_text() + assert "patch_token_summary" not in content + + def test_no_begin_archival_references(self) -> None: + path = builtin_recipes_dir() / "research-review.yaml" + content = path.read_text() + assert "begin_archival" not in content + + # --- Terminal stop sentinels --- + def test_review_pr_complete_sentinel_fields(self, recipe) -> None: + msg = recipe.steps["review_pr_complete"].message + assert "pr_url" in msg + assert "report_path_after_finalize" in msg + assert "worktree_path" in msg + assert "research_dir" in msg + + def test_review_local_complete_sentinel_fields(self, recipe) -> None: + msg = recipe.steps["review_local_complete"].message + assert "local_bundle_path" in msg + + # --- kitchen_rules --- + def test_kitchen_rules_not_empty(self, recipe) -> None: + assert len(recipe.kitchen_rules) >= 4 diff --git a/tests/recipe/test_research_smoke_fixtures.py b/tests/recipe/test_research_smoke_fixtures.py new file mode 100644 index 0000000000..64c59bb104 --- /dev/null +++ b/tests/recipe/test_research_smoke_fixtures.py @@ -0,0 +1,128 @@ +"""Smoke-test fixtures for the research recipe: classification and structural validation. + +Validates classification pipeline behavior using two canonical fixture plans: +- FIXTURE_TRIVIAL_PLAN: corpus-search question with no methodology tradition keywords +- FIXTURE_RCT_PLAN: RCT-shaped plan with CONSORT keywords triggering controlled_intervention +""" + +from __future__ import annotations + +import re + +import pytest + +from autoskillit.recipe.io import builtin_recipes_dir, load_recipe +from autoskillit.recipe.methodology_tradition_registry import load_all_methodology_traditions +from autoskillit.recipe.methodology_tradition_router import classify_methodology + +pytestmark = [pytest.mark.layer("recipe"), pytest.mark.small] + + +FIXTURE_TRIVIAL_PLAN = ( + "Does the phrase 'machine learning' appear more frequently than " + "'deep learning' in recent conference proceedings from 2020 to 2024? " + "Count occurrences in title and abstract fields from NeurIPS, ICML, " + "and ICLR proceedings. Report frequency trends by year." +) + +FIXTURE_RCT_PLAN = ( + "In this hypothetical randomized controlled trial, 100 participants " + "were randomly assigned to treatment or placebo groups. The primary " + "endpoint is symptom score change at 6 weeks. Analysis follows " + "intent-to-treat principles with CONSORT reporting guidelines. " + "Confounders are addressed via baseline stratification. Random " + "allocation uses computer-generated sequences with allocation " + "concealment via sealed envelopes." +) + + +@pytest.fixture(scope="module") +def research_recipe(): + return load_recipe(builtin_recipes_dir() / "research.yaml") + + +# --------------------------------------------------------------------------- +# Test Group A: Classification Validation (T_RSF_1 – T_RSF_10) +# --------------------------------------------------------------------------- + + +def test_trivial_fixture_no_methodology_tradition(): + result = classify_methodology(FIXTURE_TRIVIAL_PLAN) + assert result.primary_tradition is None + + +def test_trivial_fixture_precedence_fallback(): + result = classify_methodology(FIXTURE_TRIVIAL_PLAN) + assert result.precedence_trace == "stage1_no_match_fallback" + + +def test_trivial_fixture_empty_candidate_set(): + result = classify_methodology(FIXTURE_TRIVIAL_PLAN) + assert result.candidate_set == () + + +def test_rct_fixture_controlled_intervention_tradition(): + result = classify_methodology(FIXTURE_RCT_PLAN) + assert result.primary_tradition == "controlled_intervention" + + +def test_rct_fixture_single_match_precedence(): + result = classify_methodology(FIXTURE_RCT_PLAN) + assert result.precedence_trace == "stage1_single_match" + + +def test_rct_fixture_single_candidate(): + result = classify_methodology(FIXTURE_RCT_PLAN) + assert len(result.candidate_set) == 1 + + +def test_rct_fixture_consort_keywords_present(): + traditions = load_all_methodology_traditions() + ci_spec = next(s for s in traditions if s.name == "controlled_intervention") + + def _kw_pattern(keyword: str) -> re.Pattern[str]: + return re.compile(r"(?= 3 + + +def test_both_fixtures_nonempty_and_distinct(): + assert FIXTURE_TRIVIAL_PLAN + assert FIXTURE_RCT_PLAN + assert FIXTURE_TRIVIAL_PLAN != FIXTURE_RCT_PLAN + + +# --------------------------------------------------------------------------- +# Test Group B: Recipe Structural Gaps (T_RSF_11 – T_RSF_15) +# --------------------------------------------------------------------------- + + +def test_plan_visualization_captures_methodology_tradition(research_recipe): + assert "methodology_tradition" in research_recipe.steps["plan_visualization"].capture + + +def test_generate_report_receives_experiment_type_arg(research_recipe): + skill_cmd = research_recipe.steps["generate_report"].with_args.get("skill_command", "") + assert "--experiment-type" in skill_cmd + + +def test_generate_report_receives_methodology_traditions_arg(research_recipe): + skill_cmd = research_recipe.steps["generate_report"].with_args.get("skill_command", "") + assert "--methodology-traditions" in skill_cmd + + +def test_generate_report_inconclusive_receives_same_args(research_recipe): + skill_cmd = research_recipe.steps["generate_report_inconclusive"].with_args.get( + "skill_command", "" + ) + assert "--experiment-type" in skill_cmd + assert "--methodology-traditions" in skill_cmd + + +def test_re_generate_report_receives_same_args(research_recipe): + skill_cmd = research_recipe.steps["re_generate_report"].with_args.get("skill_command", "") + assert "--experiment-type" in skill_cmd + assert "--methodology-traditions" in skill_cmd diff --git a/tests/recipe/test_research_stage_data_step.py b/tests/recipe/test_research_stage_data_step.py index d6adedb7b6..b6a4df5654 100644 --- a/tests/recipe/test_research_stage_data_step.py +++ b/tests/recipe/test_research_stage_data_step.py @@ -6,6 +6,8 @@ from autoskillit.recipe.io import builtin_recipes_dir, load_recipe +pytestmark = [pytest.mark.layer("recipe"), pytest.mark.small] + RESEARCH_RECIPE_PATH = builtin_recipes_dir() / "research.yaml" @@ -53,18 +55,18 @@ def test_stage_data_captures_resource_report(recipe) -> None: assert "resource_report" in step.capture -def test_stage_data_pass_routes_to_decompose_phases(recipe) -> None: - """stage_data PASS verdict must route to decompose_phases.""" +def test_stage_data_pass_routes_to_setup_environment(recipe) -> None: + """stage_data PASS verdict must route to setup_environment.""" step = recipe.steps["stage_data"] assert step.on_result is not None - assert step.on_result.routes["PASS"] == "decompose_phases" + assert step.on_result.routes["PASS"] == "setup_environment" -def test_stage_data_warn_routes_to_decompose_phases(recipe) -> None: - """stage_data WARN verdict must route to decompose_phases.""" +def test_stage_data_warn_routes_to_setup_environment(recipe) -> None: + """stage_data WARN verdict must route to setup_environment.""" step = recipe.steps["stage_data"] assert step.on_result is not None - assert step.on_result.routes["WARN"] == "decompose_phases" + assert step.on_result.routes["WARN"] == "setup_environment" def test_stage_data_fail_does_not_route_to_decompose_phases(recipe) -> None: @@ -82,7 +84,8 @@ def test_stage_data_on_failure_escalates(recipe) -> None: def test_research_recipe_still_validates(recipe) -> None: """research.yaml must pass structural validation after stage_data is added.""" - from autoskillit.recipe.validator import validate_recipe - errors = validate_recipe(recipe) + from autoskillit.recipe.validator import validate_recipe_structure + + errors = validate_recipe_structure(recipe) assert errors == [], f"Validation errors: {errors}" diff --git a/tests/recipe/test_research_sub_recipe_rules.py b/tests/recipe/test_research_sub_recipe_rules.py new file mode 100644 index 0000000000..69e69951ba --- /dev/null +++ b/tests/recipe/test_research_sub_recipe_rules.py @@ -0,0 +1,51 @@ +"""Contract tests: research sub-recipe semantic rules and dataflow analysis.""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +import autoskillit.recipe # noqa: F401 -- pyright: ignore[reportUnusedImport] -- triggers rule registration +from autoskillit.core import Severity +from autoskillit.recipe.io import builtin_sub_recipes_dir, load_recipe +from autoskillit.recipe.schema import DataFlowReport +from autoskillit.recipe.validator import analyze_dataflow, run_semantic_rules + +pytestmark = [pytest.mark.layer("recipe"), pytest.mark.small] + +_RESEARCH_SUB_RECIPE_PATHS: list[Path] = sorted( + builtin_sub_recipes_dir().glob("research-*.yaml"), +) + +if not _RESEARCH_SUB_RECIPE_PATHS: + pytest.skip("No research sub-recipe YAMLs found", allow_module_level=True) + + +@pytest.mark.parametrize( + "recipe_path", + _RESEARCH_SUB_RECIPE_PATHS, + ids=lambda p: p.stem, +) +def test_research_sub_recipe_has_no_error_severity_findings(recipe_path: Path) -> None: + recipe = load_recipe(recipe_path) + findings = run_semantic_rules(recipe) + errors = [f for f in findings if f.severity == Severity.ERROR] + assert not errors, f"{recipe_path.stem}: {len(errors)} ERROR finding(s): " + "; ".join( + f"{f.rule}: {f.message}" for f in errors + ) + + +@pytest.mark.parametrize( + "path", + _RESEARCH_SUB_RECIPE_PATHS, + ids=lambda p: p.stem, +) +def test_research_sub_recipe_has_clean_dataflow(path: Path) -> None: + recipe = load_recipe(path) + report = analyze_dataflow(recipe) + assert isinstance(report, DataFlowReport) + assert report.warnings == [], ( + f"{path.stem}: {len(report.warnings)} dataflow warning(s): " + + "; ".join(f"{w.code}@{w.step_name}: {w.message}" for w in report.warnings) + ) diff --git a/tests/recipe/test_research_sub_recipes.py b/tests/recipe/test_research_sub_recipes.py new file mode 100644 index 0000000000..01aa0d81ea --- /dev/null +++ b/tests/recipe/test_research_sub_recipes.py @@ -0,0 +1,111 @@ +"""Tests for research sub-recipe YAML structure (design, implement, review, archive). + +All tests skip when the sub-recipe YAML files do not yet exist in recipes/sub-recipes/. +""" + +from __future__ import annotations + +import pytest + +from autoskillit.recipe.io import builtin_sub_recipes_dir, load_recipe +from autoskillit.recipe.validator import validate_recipe_structure + +pytestmark = [pytest.mark.layer("recipe"), pytest.mark.small] + + +class TestResearchDesignSubRecipe: + @pytest.fixture(scope="class") + def recipe(self): + path = builtin_sub_recipes_dir() / "research-design.yaml" + if not path.exists(): + pytest.skip(f"{path.name} not yet created") + return load_recipe(path) + + def test_has_required_ingredients(self, recipe) -> None: + assert "task" in recipe.ingredients + assert recipe.ingredients["task"].required is True + assert "source_dir" in recipe.ingredients + assert recipe.ingredients["source_dir"].required is True + + def test_has_terminal_step(self, recipe) -> None: + assert any(s.action == "stop" for s in recipe.steps.values()) + + def test_key_steps_capture_required_outputs(self, recipe) -> None: + assert any("experiment_plan" in s.capture for s in recipe.steps.values()) + assert any("visualization_plan_path" in s.capture for s in recipe.steps.values()) + + def test_validates_cleanly(self, recipe) -> None: + assert validate_recipe_structure(recipe) == [] + + +class TestResearchImplementSubRecipe: + @pytest.fixture(scope="class") + def recipe(self): + path = builtin_sub_recipes_dir() / "research-implement.yaml" + if not path.exists(): + pytest.skip(f"{path.name} not yet created") + return load_recipe(path) + + def test_has_required_ingredients(self, recipe) -> None: + assert "task" in recipe.ingredients + assert recipe.ingredients["task"].required is True + assert "source_dir" in recipe.ingredients + assert recipe.ingredients["source_dir"].required is True + + def test_has_terminal_step(self, recipe) -> None: + assert any(s.action == "stop" for s in recipe.steps.values()) + + def test_key_steps_capture_required_outputs(self, recipe) -> None: + assert any("worktree_path" in s.capture for s in recipe.steps.values()) + assert any("research_dir" in s.capture for s in recipe.steps.values()) + + def test_validates_cleanly(self, recipe) -> None: + assert validate_recipe_structure(recipe) == [] + + +class TestResearchReviewSubRecipe: + @pytest.fixture(scope="class") + def recipe(self): + path = builtin_sub_recipes_dir() / "research-review.yaml" + if not path.exists(): + pytest.skip(f"{path.name} not yet created") + return load_recipe(path) + + def test_has_required_ingredients(self, recipe) -> None: + assert "task" in recipe.ingredients + assert recipe.ingredients["task"].required is True + assert "source_dir" in recipe.ingredients + assert recipe.ingredients["source_dir"].required is True + + def test_has_terminal_step(self, recipe) -> None: + assert any(s.action == "stop" for s in recipe.steps.values()) + + def test_key_steps_capture_required_outputs(self, recipe) -> None: + assert any("review_verdict" in s.capture for s in recipe.steps.values()) + + def test_validates_cleanly(self, recipe) -> None: + assert validate_recipe_structure(recipe) == [] + + +class TestResearchArchiveSubRecipe: + @pytest.fixture(scope="class") + def recipe(self): + path = builtin_sub_recipes_dir() / "research-archive.yaml" + if not path.exists(): + pytest.skip(f"{path.name} not yet created") + return load_recipe(path) + + def test_has_required_ingredients(self, recipe) -> None: + assert "worktree_path" in recipe.ingredients + assert recipe.ingredients["worktree_path"].required is True + assert "research_dir" in recipe.ingredients + assert recipe.ingredients["research_dir"].required is True + + def test_has_terminal_step(self, recipe) -> None: + assert any(s.action == "stop" for s in recipe.steps.values()) + + def test_key_steps_capture_required_outputs(self, recipe) -> None: + assert any("archive_tag" in s.capture for s in recipe.steps.values()) + + def test_validates_cleanly(self, recipe) -> None: + assert validate_recipe_structure(recipe) == [] diff --git a/tests/recipe/test_research_yaml_backward_compat.py b/tests/recipe/test_research_yaml_backward_compat.py new file mode 100644 index 0000000000..4b5a07ae36 --- /dev/null +++ b/tests/recipe/test_research_yaml_backward_compat.py @@ -0,0 +1,25 @@ +from __future__ import annotations + +import pytest + +from autoskillit.recipe.io import builtin_recipes_dir, load_recipe +from autoskillit.recipe.schema import RecipeKind +from autoskillit.recipe.validator import validate_recipe_structure + +pytestmark = [pytest.mark.layer("recipe"), pytest.mark.small] + + +class TestResearchYamlBackwardCompat: + @pytest.fixture(scope="class") + def recipe(self): + return load_recipe(builtin_recipes_dir() / "research.yaml") + + def test_research_yaml_loads_without_error(self, recipe): + assert recipe is not None + + def test_research_yaml_validates_with_no_errors(self, recipe): + errors = validate_recipe_structure(recipe) + assert errors == [] + + def test_research_yaml_kind_is_standard(self, recipe): + assert recipe.kind == RecipeKind.STANDARD diff --git a/tests/recipe/test_resolve_ci_routing_invariant.py b/tests/recipe/test_resolve_ci_routing_invariant.py index 6c386e0a34..7a4acc6405 100644 --- a/tests/recipe/test_resolve_ci_routing_invariant.py +++ b/tests/recipe/test_resolve_ci_routing_invariant.py @@ -12,6 +12,9 @@ from autoskillit.core import pkg_root from autoskillit.recipe.io import load_recipe +from autoskillit.recipe.rules.rules_verdict import _classify_route_target + +pytestmark = [pytest.mark.layer("recipe"), pytest.mark.small] _RECIPES_DIR = pkg_root() / "recipes" _PIPELINE_RECIPES = [ @@ -156,14 +159,14 @@ def test_resolve_ci_on_result_routes_real_fix_to_re_push(recipe_name: str) -> No def test_resolve_ci_on_result_routes_escalation_verdicts_to_failure( recipe_name: str, ) -> None: - """flake_suspected and ci_only_failure must route to release_issue_failure.""" + """ci_only_failure must route to release_issue_failure.""" recipe_path = _RECIPES_DIR / recipe_name recipe = load_recipe(recipe_path) resolve_steps = _find_resolve_steps_reaching_push(recipe) for step_name, step in resolve_steps: if step.on_result is None: continue - for verdict in ("flake_suspected", "ci_only_failure"): + for verdict in ("ci_only_failure",): routes = [ cond.route for cond in (step.on_result.conditions or []) @@ -177,6 +180,57 @@ def test_resolve_ci_on_result_routes_escalation_verdicts_to_failure( ) +@pytest.mark.parametrize("recipe_name", _PIPELINE_RECIPES) +def test_resolve_ci_routes_flake_suspected_to_re_push(recipe_name: str) -> None: + """flake_suspected in resolve_ci must route to re_push, not release_issue_failure.""" + recipe_path = _RECIPES_DIR / recipe_name + recipe = load_recipe(recipe_path) + resolve_steps = _find_resolve_steps_reaching_push(recipe) + for step_name, step in resolve_steps: + if step.on_result is None: + continue + routes = [ + cond.route + for cond in (step.on_result.conditions or []) + if cond.when and "flake_suspected" in cond.when + ] + if not routes: + continue + assert any("re_push" in r for r in routes), ( + f"{recipe_name}/{step_name}: verdict=flake_suspected must route to " + f"a re_push step (retry), got routes: {routes}" + ) + + +@pytest.mark.parametrize("recipe_name", _PIPELINE_RECIPES) +def test_flake_suspected_routes_consistently_across_fix_and_resolve_ci( + recipe_name: str, +) -> None: + """flake_suspected must route to continuation in both fix/assess and resolve_ci steps.""" + recipe_path = _RECIPES_DIR / recipe_name + recipe = load_recipe(recipe_path) + flake_routes: list[tuple[str, str, bool]] = [] # (step_name, route, is_escalation) + for step_name, step in recipe.steps.items(): + if step.tool != "run_skill": + continue + cmd = (step.with_args or {}).get("skill_command", "") + if "resolve-failures" not in cmd: + continue + if step.on_result is None: + continue + for cond in step.on_result.conditions or []: + if cond.when and "flake_suspected" in cond.when: + is_esc = _classify_route_target(cond.route) == "escalation" + flake_routes.append((step_name, cond.route, is_esc)) + + assert flake_routes, f"{recipe_name}: no flake_suspected routes found" + escalation_steps = [f"{name} → {route}" for name, route, is_esc in flake_routes if is_esc] + assert not escalation_steps, ( + f"{recipe_name}: flake_suspected routes to escalation in: {escalation_steps}. " + "All steps invoking resolve-failures should route flake_suspected to continuation (retry)." + ) + + @pytest.mark.parametrize("recipe_name", _PIPELINE_RECIPES) def test_pre_resolve_rebase_step_exists(recipe_name: str) -> None: """pre_resolve_rebase step must exist for already_green re-entry path.""" diff --git a/tests/recipe/test_review_loop_routing_integration.py b/tests/recipe/test_review_loop_routing_integration.py new file mode 100644 index 0000000000..b0bc368e0d --- /dev/null +++ b/tests/recipe/test_review_loop_routing_integration.py @@ -0,0 +1,182 @@ +"""Compound routing integration tests for the review-resolve retry loop.""" + +from __future__ import annotations + +import pytest + +from autoskillit.recipe.io import builtin_recipes_dir, load_recipe +from autoskillit.smoke_utils import check_review_loop + +pytestmark = [pytest.mark.layer("recipe"), pytest.mark.small] + +RECIPE_NAMES = ["implementation.yaml", "remediation.yaml", "implementation-groups.yaml"] + + +@pytest.mark.parametrize("recipe_name", RECIPE_NAMES) +def test_review_loop_routes_to_review_pr_after_resolve_cycle(recipe_name: str) -> None: + """End-to-end routing test: after resolve_review -> re_push_review, + check_review_loop must route to review_pr (not ci_watch) when + previous verdict was changes_requested and iterations remain. + + This test calls the actual check_review_loop callable with + previous_verdict=changes_requested, current_iteration=0 and max_iterations=3, + then evaluates the recipe's on_result conditions against the callable's return value. + """ + result = check_review_loop( + pr_number="42", + current_iteration="0", + max_iterations="3", + previous_verdict="changes_requested", + ) + + recipe = load_recipe(builtin_recipes_dir() / recipe_name) + step = recipe.steps["check_review_loop"] + assert step.on_result is not None + conditions = step.on_result.conditions + + # Simulate template interpolation + when_expr = conditions[0].when + for key, value in result.items(): + when_expr = when_expr.replace(f"${{{{ result.{key} }}}}", value) + + # The first condition (route to review_pr) must be satisfiable + # when_expr becomes "true == true and false == false" — evaluate compound + assert _eval_compound_condition(when_expr) + assert conditions[0].route == "review_pr" + + +@pytest.mark.parametrize("recipe_name", RECIPE_NAMES) +def test_review_loop_routes_to_ci_watch_at_max_iterations(recipe_name: str) -> None: + """When max_iterations is exhausted, routing must fall through to check_repo_ci_event.""" + result = check_review_loop( + pr_number="42", + current_iteration="2", + max_iterations="3", + previous_verdict="changes_requested", + ) + + recipe = load_recipe(builtin_recipes_dir() / recipe_name) + step = recipe.steps["check_review_loop"] + assert step.on_result is not None + conditions = step.on_result.conditions + + # Simulate template interpolation on the first (review_pr) condition + when_expr = conditions[0].when + for key, value in result.items(): + when_expr = when_expr.replace(f"${{{{ result.{key} }}}}", value) + + # The first condition must NOT be satisfied — falls through to default check_repo_ci_event + # when_expr becomes "true == true and true == false" — second clause is False + assert not _eval_compound_condition(when_expr) + assert conditions[1].route == "check_repo_ci_event" + + +@pytest.mark.parametrize("recipe_name", RECIPE_NAMES) +def test_check_review_loop_review_pr_route_uses_had_blocking_and_max_exceeded( + recipe_name: str, +) -> None: + """The check_review_loop on_result condition for review_pr must gate on + had_blocking AND max_exceeded.""" + recipe = load_recipe(builtin_recipes_dir() / recipe_name) + step = recipe.steps["check_review_loop"] + assert step.on_result is not None + review_conditions = [c for c in step.on_result.conditions if c.route == "review_pr"] + assert review_conditions, f"No conditional route to review_pr found in {recipe_name}" + review_condition = review_conditions[0] + assert "had_blocking" in review_condition.when + assert "max_exceeded" in review_condition.when + + +def _eval_compound_condition(when_expr: str) -> bool: + """Evaluate a compound condition like 'true == true and false == false'. + + Splits on ' and ', evaluates each 'lhs == rhs' pair, ANDs results. + """ + parts = when_expr.split(" and ") + results = [] + for part in parts: + lhs, rhs = [s.strip() for s in part.split("==", 1)] + results.append(lhs == rhs) + return all(results) + + +@pytest.mark.parametrize("recipe_name", RECIPE_NAMES) +def test_check_review_loop_approved_with_comments_is_non_blocking(recipe_name: str) -> None: + """When previous_verdict=approved_with_comments and local_rounds=0, had_blocking=false. + + The approved_with_comments verdict does not trigger re-review — it is non-blocking + and routes directly to CI after the one-shot resolve_review cycle. + """ + result = check_review_loop( + pr_number="42", + current_iteration="0", + max_iterations="3", + previous_verdict="approved_with_comments", + ) + assert result["had_blocking"] == "false" + assert result["max_exceeded"] == "false" + recipe = load_recipe(builtin_recipes_dir() / recipe_name) + step = recipe.steps["check_review_loop"] + conditions = step.on_result.conditions + review_condition = next(c for c in conditions if c.route == "review_pr") + when_expr = review_condition.when + for key, value in result.items(): + when_expr = when_expr.replace(f"${{{{ result.{key} }}}}", value) + # Compound condition "false == true and false == false" must evaluate to False + assert not _eval_compound_condition(when_expr) + + +@pytest.mark.parametrize("recipe_name", RECIPE_NAMES) +def test_check_review_loop_approved_with_local_rounds_forces_continuation( + recipe_name: str, +) -> None: + """When previous_verdict=approved but local_review_rounds > 0 and not exhausted, + had_blocking=true. + + The approved verdict is blocking when local review rounds are still being + exhausted, forcing continuation through the review loop until local_review_rounds + are exhausted. + """ + result = check_review_loop( + pr_number="42", + current_iteration="0", + max_iterations="6", + previous_verdict="approved", + local_review_rounds="3", + ) + assert result["had_blocking"] == "true" + assert result["max_exceeded"] == "false" + assert result["next_iteration"] == "1" + recipe = load_recipe(builtin_recipes_dir() / recipe_name) + step = recipe.steps["check_review_loop"] + conditions = step.on_result.conditions + review_condition = next(c for c in conditions if c.route == "review_pr") + when_expr = review_condition.when + for key, value in result.items(): + when_expr = when_expr.replace(f"${{{{ result.{key} }}}}", value) + # Compound condition "true == true and false == false" must evaluate to True + assert _eval_compound_condition(when_expr) + + +@pytest.mark.parametrize("recipe_name", RECIPE_NAMES) +def test_review_loop_routes_to_review_pr_when_had_blocking_and_not_max_exceeded( + recipe_name: str, +) -> None: + """When had_blocking=true and max_exceeded=false, routing goes back to review_pr.""" + result = check_review_loop( + pr_number="42", + current_iteration="0", + max_iterations="3", + previous_verdict="changes_requested", + ) + assert result["had_blocking"] == "true" + assert result["max_exceeded"] == "false" + recipe = load_recipe(builtin_recipes_dir() / recipe_name) + step = recipe.steps["check_review_loop"] + conditions = step.on_result.conditions + review_condition = next(c for c in conditions if c.route == "review_pr") + when_expr = review_condition.when + for key, value in result.items(): + when_expr = when_expr.replace(f"${{{{ result.{key} }}}}", value) + # "true == true and false == false" must evaluate to True + assert _eval_compound_condition(when_expr) diff --git a/tests/recipe/test_review_loop_routing_invariant.py b/tests/recipe/test_review_loop_routing_invariant.py new file mode 100644 index 0000000000..283ef8e042 --- /dev/null +++ b/tests/recipe/test_review_loop_routing_invariant.py @@ -0,0 +1,69 @@ +"""Routing invariant tests for the review loop waypoint. + +Invariant: check_review_loop is a mandatory waypoint between review_pr and +check_repo_ci_event. No verdict may route directly from review_pr to +check_repo_ci_event without first crossing check_review_loop. + +These tests verify the structural routing graph of the three recipes that +contain both review_pr and check_review_loop: implementation, remediation, +and implementation-groups. +""" + +from __future__ import annotations + +import pytest + +from autoskillit.recipe.io import builtin_recipes_dir, load_recipe + +pytestmark = [pytest.mark.layer("recipe"), pytest.mark.small] + +REVIEW_LOOP_RECIPES = ["implementation", "remediation", "implementation-groups"] + + +@pytest.mark.parametrize("recipe_name", REVIEW_LOOP_RECIPES) +def test_approved_verdict_must_not_route_directly_to_ci(recipe_name: str) -> None: + """The catch-all on_result may not route directly to check_repo_ci_event. + + All verdicts must pass through check_review_loop so that review_loop_count + is always incremented before exiting to CI. + """ + recipe = load_recipe(builtin_recipes_dir() / f"{recipe_name}.yaml") + step = recipe.steps["review_pr"] + direct_routes = {c.route for c in (step.on_result.conditions if step.on_result else [])} + assert "check_repo_ci_event" not in direct_routes, ( + f"[{recipe_name}] review_pr routes directly to check_repo_ci_event, " + f"bypassing check_review_loop. All verdicts must pass through check_review_loop." + ) + + +@pytest.mark.parametrize("recipe_name", REVIEW_LOOP_RECIPES) +def test_check_repo_ci_event_not_bfs_reachable_without_barrier(recipe_name: str) -> None: + """BFS from review_pr must not reach check_repo_ci_event without check_review_loop. + + The barrier pattern ensures that every path to check_repo_ci_event first + crosses check_review_loop, guaranteeing counter graduation. + """ + from autoskillit.recipe._analysis_bfs import bfs_reachable_without_barrier + + recipe = load_recipe(builtin_recipes_dir() / f"{recipe_name}.yaml") + reachable = bfs_reachable_without_barrier( + recipe=recipe, start="review_pr", barrier="check_review_loop" + ) + assert "check_repo_ci_event" not in reachable, ( + f"[{recipe_name}] check_repo_ci_event BFS-reachable from review_pr " + f"without crossing check_review_loop. Counter graduation is bypassed." + ) + + +@pytest.mark.parametrize("recipe_name", REVIEW_LOOP_RECIPES) +def test_check_review_loop_step_receives_local_review_rounds(recipe_name: str) -> None: + """The check_review_loop step must receive local_review_rounds in with_args. + + Without this ingredient, the callable cannot compute had_blocking correctly + for the local_review_rounds graduation policy. + """ + recipe = load_recipe(builtin_recipes_dir() / f"{recipe_name}.yaml") + step = recipe.steps["check_review_loop"] + assert "local_review_rounds" in step.with_args, ( + f"[{recipe_name}] check_review_loop step missing local_review_rounds in with_args." + ) diff --git a/tests/recipe/test_rule_decomposition.py b/tests/recipe/test_rule_decomposition.py index 5b70b33d6e..337d6efd47 100644 --- a/tests/recipe/test_rule_decomposition.py +++ b/tests/recipe/test_rule_decomposition.py @@ -5,11 +5,15 @@ import ast import pathlib +import pytest + +pytestmark = [pytest.mark.layer("recipe"), pytest.mark.small] + def test_no_deferred_validator_imports_in_rule_modules() -> None: """T1: No rule sub-module should defer-import from validator.py inside a function body.""" recipe_dir = pathlib.Path(__file__).resolve().parents[2] / "src/autoskillit/recipe" - rule_files = list(recipe_dir.glob("rules_*.py")) + rule_files = list((recipe_dir / "rules").glob("rules_*.py")) assert len(rule_files) >= 5, "Expected at least 5 rule sub-modules" for path in rule_files: tree = ast.parse(path.read_text()) @@ -95,6 +99,6 @@ def test_validator_does_not_import_rules() -> None: assert not module.endswith(".rules"), ( "validator.py must not import from rules.py (cycle eliminated)" ) - assert "rules_" not in module or module.startswith("autoskillit.recipe.rules_"), ( - f"validator.py must not import rule sub-modules directly: {module}" - ) + assert "rules_" not in module or module.startswith( + "autoskillit.recipe.rules.rules_" + ), f"validator.py must not import rule sub-modules directly: {module}" diff --git a/tests/recipe/test_rules_actions.py b/tests/recipe/test_rules_actions.py new file mode 100644 index 0000000000..2ed377fe01 --- /dev/null +++ b/tests/recipe/test_rules_actions.py @@ -0,0 +1,116 @@ +"""Tests for recipe/rules_actions.py semantic validation rules.""" + +from __future__ import annotations + +import pytest + +from autoskillit.core import Severity +from autoskillit.recipe.registry import run_semantic_rules +from autoskillit.recipe.schema import Recipe, RecipeStep + +pytestmark = [pytest.mark.layer("recipe"), pytest.mark.small] + + +def _make_recipe(steps_dict: dict) -> Recipe: + steps = {k: RecipeStep(**v) for k, v in steps_dict.items()} + return Recipe(name="test", description="test", steps=steps) + + +def test_stop_step_with_routing_fields_fires_error(): + """A stop step with on_success/on_failure should fire stop-step-has-no-routing.""" + recipe = _make_recipe( + { + "done": {"action": "stop", "message": "Done.", "on_success": "other"}, + "other": {"action": "stop", "message": "Other."}, + } + ) + findings = [f for f in run_semantic_rules(recipe) if f.rule == "stop-step-has-no-routing"] + assert len(findings) == 1 + assert findings[0].severity == Severity.ERROR + + +def test_stop_step_with_default_on_exhausted_passes(): + """A stop step with only the default on_exhausted='escalate' should NOT fire.""" + recipe = _make_recipe({"done": {"action": "stop", "message": "Done."}}) + findings = [f for f in run_semantic_rules(recipe) if f.rule == "stop-step-has-no-routing"] + assert len(findings) == 0 + + +def test_stop_step_without_routing_passes(): + """A clean stop step should not fire stop-step-has-no-routing.""" + recipe = _make_recipe({"done": {"action": "stop", "message": "Pipeline complete."}}) + findings = [f for f in run_semantic_rules(recipe) if f.rule == "stop-step-has-no-routing"] + assert len(findings) == 0 + + +def test_recipe_without_terminal_step_fires_error(): + """A recipe with no reachable stop step should fire recipe-has-terminal-step.""" + recipe = _make_recipe( + {"step_a": {"tool": "run_cmd", "on_success": "step_a", "on_failure": "step_a"}} + ) + findings = [f for f in run_semantic_rules(recipe) if f.rule == "recipe-has-terminal-step"] + assert len(findings) == 1 + assert findings[0].severity == Severity.ERROR + + +def test_recipe_with_terminal_step_passes(): + """A recipe with at least one stop step should NOT fire recipe-has-terminal-step.""" + recipe = _make_recipe({"done": {"action": "stop", "message": "Done."}}) + findings = [f for f in run_semantic_rules(recipe) if f.rule == "recipe-has-terminal-step"] + assert len(findings) == 0 + + +def test_route_step_without_on_result_fires_warning(): + """A route step without on_result conditions should fire route-step-requires-on-result.""" + recipe = _make_recipe( + { + "gate": {"action": "route", "on_success": "next"}, + "next": {"action": "stop", "message": "Done."}, + } + ) + findings = [f for f in run_semantic_rules(recipe) if f.rule == "route-step-requires-on-result"] + assert len(findings) == 1 + assert findings[0].severity == Severity.WARNING + + +def test_route_step_with_on_result_passes(): + """A route step with on_result should NOT fire route-step-requires-on-result.""" + from autoskillit.recipe.schema import StepResultCondition, StepResultRoute + + # Pydantic accepts a pre-constructed StepResultRoute here even though _make_recipe + # calls RecipeStep(**v) — Pydantic's model validation accepts existing model instances. + recipe = _make_recipe( + { + "gate": { + "action": "route", + "on_result": StepResultRoute( + conditions=[StepResultCondition(when="ctx.x == 'y'", route="next")] + ), + "on_failure": "next", + }, + "next": {"action": "stop", "message": "Done."}, + } + ) + findings = [f for f in run_semantic_rules(recipe) if f.rule == "route-step-requires-on-result"] + assert len(findings) == 0 + + +class TestStopStepMessageQuality: + def test_short_message_fires_warning(self): + recipe = _make_recipe({"done": {"action": "stop", "message": "bye"}}) + findings = [f for f in run_semantic_rules(recipe) if f.rule == "stop-step-message-quality"] + assert len(findings) == 1 + assert findings[0].severity == Severity.WARNING + + def test_placeholder_message_fires_warning(self): + recipe = _make_recipe({"done": {"action": "stop", "message": "done"}}) + findings = [f for f in run_semantic_rules(recipe) if f.rule == "stop-step-message-quality"] + assert len(findings) == 1 + assert findings[0].severity == Severity.WARNING + + def test_meaningful_message_is_clean(self): + recipe = _make_recipe( + {"done": {"action": "stop", "message": "Pipeline completed successfully"}} + ) + findings = [f for f in run_semantic_rules(recipe) if f.rule == "stop-step-message-quality"] + assert len(findings) == 0 diff --git a/tests/recipe/test_rules_blocks.py b/tests/recipe/test_rules_blocks.py index d94499929f..f0410262c0 100644 --- a/tests/recipe/test_rules_blocks.py +++ b/tests/recipe/test_rules_blocks.py @@ -4,8 +4,12 @@ import pytest +from autoskillit.core import Severity from autoskillit.recipe.io import builtin_recipes_dir, load_recipe from autoskillit.recipe.registry import run_semantic_rules +from autoskillit.recipe.schema import Recipe, RecipeStep + +pytestmark = [pytest.mark.layer("recipe"), pytest.mark.small] _QUEUE_CAPABLE = ("implementation.yaml", "remediation.yaml", "implementation-groups.yaml") @@ -59,8 +63,6 @@ def _make_synthetic_recipe_with_two_gh_api_run_cmds_in_block(block_name: str): Used by the synthetic block budget test below. """ - from autoskillit.recipe.schema import Recipe, RecipeStep - step_a = RecipeStep( tool="run_cmd", with_args={"cmd": "gh api /repos/o/r/pulls"}, @@ -93,3 +95,168 @@ def test_synthetic_block_with_two_run_cmds_emits_block_run_cmd_budget_finding(): matching = [f for f in findings if f.rule == "block-run-cmd-budget"] assert len(matching) == 1 assert "test_block" in matching[0].message + + +def _make_block_recipe(steps: dict[str, RecipeStep]) -> Recipe: + """Minimal Recipe factory for block rule tests.""" + return Recipe( + name="synthetic", + description="synthetic test recipe", + ingredients={}, + steps=steps, + kitchen_rules=[], + version=None, + experimental=False, + requires_packs=[], + ) + + +class TestBlockMcpToolBudget: + def test_two_mcp_tools_in_pre_queue_gate_fires_error(self): + step_a = RecipeStep( + tool="check_repo_merge_state", block="pre_queue_gate", on_success="step_b" + ) + step_b = RecipeStep( + tool="check_repo_merge_state", block="pre_queue_gate", on_success="done" + ) + stop = RecipeStep(action="stop", message="Pre-queue gate complete.") + recipe = _make_block_recipe({"step_a": step_a, "step_b": step_b, "done": stop}) + findings = run_semantic_rules(recipe) + matching = [f for f in findings if f.rule == "block-mcp-tool-budget"] + assert len(matching) == 1 + assert matching[0].severity == Severity.ERROR + assert "pre_queue_gate" in matching[0].message + + def test_one_mcp_tool_in_pre_queue_gate_is_clean(self): + step_a = RecipeStep( + tool="check_repo_merge_state", block="pre_queue_gate", on_success="done" + ) + stop = RecipeStep(action="stop", message="Pre-queue gate complete.") + recipe = _make_block_recipe({"step_a": step_a, "done": stop}) + findings = run_semantic_rules(recipe) + matching = [f for f in findings if f.rule == "block-mcp-tool-budget"] + assert len(matching) == 0 + + +class TestBlockGhApiForbidden: + def test_gh_api_in_pre_queue_gate_fires_error(self): + step_a = RecipeStep( + tool="run_cmd", + with_args={"cmd": "gh api /repos/o/r"}, + block="pre_queue_gate", + on_success="done", + ) + stop = RecipeStep(action="stop", message="Pre-queue gate complete.") + recipe = _make_block_recipe({"step_a": step_a, "done": stop}) + findings = run_semantic_rules(recipe) + matching = [f for f in findings if f.rule == "block-gh-api-forbidden"] + assert len(matching) == 1 + assert matching[0].severity == Severity.ERROR + + def test_run_cmd_without_gh_api_is_clean(self): + step_a = RecipeStep( + tool="run_cmd", + with_args={"cmd": "echo ok"}, + block="pre_queue_gate", + on_success="done", + ) + stop = RecipeStep(action="stop", message="Pre-queue gate complete.") + recipe = _make_block_recipe({"step_a": step_a, "done": stop}) + findings = run_semantic_rules(recipe) + matching = [f for f in findings if f.rule == "block-gh-api-forbidden"] + assert len(matching) == 0 + + +class TestBlockSingleProducer: + def test_duplicate_capture_key_fires_warning(self): + step_a = RecipeStep( + tool="run_cmd", + with_args={"cmd": "echo a"}, + block="test_block", + capture={"shared_key": "a"}, + on_success="step_b", + ) + step_a.name = "step_a" + step_b = RecipeStep( + tool="run_cmd", + with_args={"cmd": "echo b"}, + block="test_block", + capture={"shared_key": "b"}, + on_success="done", + ) + step_b.name = "step_b" + stop = RecipeStep(action="stop", message="Block single producer test done.") + recipe = _make_block_recipe({"step_a": step_a, "step_b": step_b, "done": stop}) + findings = run_semantic_rules(recipe) + matching = [f for f in findings if f.rule == "block-single-producer"] + assert len(matching) == 1 + assert matching[0].severity == Severity.WARNING + assert "shared_key" in matching[0].message + + def test_distinct_capture_keys_is_clean(self): + step_a = RecipeStep( + tool="run_cmd", + with_args={"cmd": "echo a"}, + block="test_block", + capture={"key_a": "a"}, + on_success="step_b", + ) + step_a.name = "step_a" + step_b = RecipeStep( + tool="run_cmd", + with_args={"cmd": "echo b"}, + block="test_block", + capture={"key_b": "b"}, + on_success="done", + ) + step_b.name = "step_b" + stop = RecipeStep(action="stop", message="Block single producer test done.") + recipe = _make_block_recipe({"step_a": step_a, "step_b": step_b, "done": stop}) + findings = run_semantic_rules(recipe) + matching = [f for f in findings if f.rule == "block-single-producer"] + assert len(matching) == 0 + + +class TestBlockEntryExitReachable: + def test_disconnected_block_steps_fire_warning(self): + step_a = RecipeStep( + tool="run_cmd", + with_args={"cmd": "echo a"}, + block="test_block", + on_success="done", + ) + step_a.name = "step_a" + step_b = RecipeStep( + tool="run_cmd", + with_args={"cmd": "echo b"}, + block="test_block", + on_success="done", + ) + step_b.name = "step_b" + stop = RecipeStep(action="stop", message="Block entry exit test done.") + recipe = _make_block_recipe({"step_a": step_a, "step_b": step_b, "done": stop}) + findings = run_semantic_rules(recipe) + matching = [f for f in findings if f.rule == "block-entry-exit-reachable"] + assert len(matching) == 2 + assert all(f.severity == Severity.WARNING for f in matching) + + def test_linear_block_chain_is_clean(self): + step_a = RecipeStep( + tool="run_cmd", + with_args={"cmd": "echo a"}, + block="test_block", + on_success="step_b", + ) + step_a.name = "step_a" + step_b = RecipeStep( + tool="run_cmd", + with_args={"cmd": "echo b"}, + block="test_block", + on_success="done", + ) + step_b.name = "step_b" + stop = RecipeStep(action="stop", message="Block entry exit test done.") + recipe = _make_block_recipe({"step_a": step_a, "step_b": step_b, "done": stop}) + findings = run_semantic_rules(recipe) + matching = [f for f in findings if f.rule == "block-entry-exit-reachable"] + assert len(matching) == 0 diff --git a/tests/recipe/test_rules_bypass.py b/tests/recipe/test_rules_bypass.py index 5cf640cf10..9b58618027 100644 --- a/tests/recipe/test_rules_bypass.py +++ b/tests/recipe/test_rules_bypass.py @@ -2,6 +2,8 @@ from __future__ import annotations +import pytest + from autoskillit.core.types import Severity from autoskillit.recipe.schema import ( Recipe, @@ -10,6 +12,8 @@ ) from autoskillit.recipe.validator import run_semantic_rules +pytestmark = [pytest.mark.layer("recipe"), pytest.mark.small] + # --------------------------------------------------------------------------- # skip_when_false bypass routing tests # --------------------------------------------------------------------------- diff --git a/tests/recipe/test_rules_callable_inputs.py b/tests/recipe/test_rules_callable_inputs.py new file mode 100644 index 0000000000..234fdf1f25 --- /dev/null +++ b/tests/recipe/test_rules_callable_inputs.py @@ -0,0 +1,48 @@ +from __future__ import annotations + +import pytest + +from autoskillit.core.types import Severity +from autoskillit.recipe.validator import run_semantic_rules +from tests.recipe.conftest import _make_workflow + +pytestmark = [pytest.mark.layer("recipe"), pytest.mark.small] + + +def test_missing_callable_input_rule_fires(): + """Semantic rule must ERROR when run_python step lacks required args.""" + recipe = _make_workflow( + { + "init": { + "tool": "run_python", + "with": {"callable": "autoskillit.planner.create_run_dir"}, + "on_success": "done", + }, + "done": {"action": "stop", "message": "Done"}, + } + ) + findings = run_semantic_rules(recipe) + missing = [f for f in findings if f.rule == "missing-callable-input"] + assert len(missing) >= 1 + assert all(f.severity == Severity.ERROR for f in missing) + + +def test_callable_signature_mismatch_rule_fires(): + """Semantic rule must ERROR when recipe args keys don't match callable signature.""" + recipe = _make_workflow( + { + "init": { + "tool": "run_python", + "with": { + "callable": "autoskillit.planner.create_run_dir", + "args": {"wrong_key": "x"}, + }, + "on_success": "done", + }, + "done": {"action": "stop", "message": "Done"}, + } + ) + findings = run_semantic_rules(recipe) + mismatch = [f for f in findings if f.rule == "callable-signature-mismatch"] + assert len(mismatch) >= 1 + assert all(f.severity == Severity.ERROR for f in mismatch) diff --git a/tests/recipe/test_rules_callable_scope.py b/tests/recipe/test_rules_callable_scope.py new file mode 100644 index 0000000000..4ff844d8a9 --- /dev/null +++ b/tests/recipe/test_rules_callable_scope.py @@ -0,0 +1,92 @@ +"""Tests for callable-requires-scoped-discovery semantic rule.""" + +from __future__ import annotations + +import pytest + +from autoskillit.core.types import Severity +from autoskillit.recipe.validator import run_semantic_rules +from tests.recipe.conftest import _make_workflow + +pytestmark = [pytest.mark.layer("recipe"), pytest.mark.small] + + +def test_callable_requires_scoped_discovery_flags_batch_create_issues(): + """Rule must ERROR when batch_create_issues step lacks audit_run_dir. + + Reproduces the bug: batch_create_issues uses an unscoped glob on the flat + validate-audit/ directory. Without audit_run_dir, it cannot distinguish + current-run files from prior-run files. + """ + recipe = _make_workflow( + { + "init": { + "tool": "run_python", + "with": { + "callable": "autoskillit.recipe._cmd_rpc.batch_create_issues", + "workspace": "${{ inputs.workspace }}", + }, + "on_success": "done", + }, + "done": {"action": "stop", "message": "Done"}, + } + ) + findings = run_semantic_rules(recipe) + scoped_rule_findings = [f for f in findings if f.rule == "callable-requires-scoped-discovery"] + assert len(scoped_rule_findings) == 1, ( + "callable-requires-scoped-discovery must fire exactly once when batch_create_issues " + "is called without audit_run_dir" + ) + assert all(f.severity == Severity.ERROR for f in scoped_rule_findings) + + +def test_callable_requires_scoped_discovery_passes_with_audit_run_dir(): + """Rule must NOT fire when batch_create_issues receives audit_run_dir. + + audit_run_dir scopes the glob to a per-run directory, preventing cross-run + file accumulation. + """ + recipe = _make_workflow( + { + "init": { + "tool": "run_python", + "with": { + "callable": "autoskillit.recipe._cmd_rpc.batch_create_issues", + "workspace": "${{ inputs.workspace }}", + "audit_run_dir": "${{ context.audit_run_dir }}", + }, + "on_success": "done", + }, + "done": {"action": "stop", "message": "Done"}, + } + ) + findings = run_semantic_rules(recipe) + scoped_rule_findings = [f for f in findings if f.rule == "callable-requires-scoped-discovery"] + assert len(scoped_rule_findings) == 0, ( + "callable-requires-scoped-discovery must not fire when audit_run_dir is provided" + ) + + +def test_callable_requires_scoped_discovery_not_fired_for_other_callables(): + """Rule must only fire for known file-discovering callables. + + Unknown callables that are not in SCOPED_CALLABLES should not trigger the rule. + """ + recipe = _make_workflow( + { + "init": { + "tool": "run_python", + "with": { + "callable": "autoskillit.planner.create_run_dir", + "temp_dir": "{{AUTOSKILLIT_TEMP}}", + }, + "on_success": "done", + }, + "done": {"action": "stop", "message": "Done"}, + } + ) + findings = run_semantic_rules(recipe) + scoped_rule_findings = [f for f in findings if f.rule == "callable-requires-scoped-discovery"] + assert len(scoped_rule_findings) == 0, ( + "callable-requires-scoped-discovery must not fire for create_run_dir" + ) diff --git a/tests/recipe/test_rules_campaign.py b/tests/recipe/test_rules_campaign.py new file mode 100644 index 0000000000..470e4bc323 --- /dev/null +++ b/tests/recipe/test_rules_campaign.py @@ -0,0 +1,969 @@ +"""Tests for campaign semantic validation rules (rules_campaign.py).""" + +from __future__ import annotations + +from pathlib import Path + +import pytest +import yaml + +import autoskillit.recipe # noqa: F401 -- triggers rule registration +from autoskillit.core import Severity +from autoskillit.recipe._analysis import make_validation_context +from autoskillit.recipe.registry import run_semantic_rules +from autoskillit.recipe.schema import CampaignDispatch, Recipe, RecipeKind, RecipeStep + +pytestmark = [pytest.mark.layer("recipe"), pytest.mark.small] + + +def _standard_recipe(**kwargs: object) -> Recipe: + return Recipe( + name="standard", + description="standard recipe", + steps={"stop": RecipeStep(action="stop", message="done")}, + kitchen_rules=["NEVER"], + **kwargs, + ) + + +def _campaign(**kwargs: object) -> Recipe: + defaults: dict = { + "name": "my-campaign", + "description": "test campaign", + "kind": RecipeKind.CAMPAIGN, + "dispatches": [ + CampaignDispatch( + name="phase-one", + recipe="implementation", + task="Do the thing", + ingredients={"task": "Do it"}, + ) + ], + "requires_recipe_packs": ["implementation-family"], + "kitchen_rules": ["NEVER"], + } + defaults.update(kwargs) + return Recipe(**defaults) + + +def _findings(recipe: Recipe, rule: str, **ctx_kwargs: object) -> list: + ctx = make_validation_context(recipe, **ctx_kwargs) + return [f for f in run_semantic_rules(ctx) if f.rule == rule] + + +def _write_recipe_yaml(path: Path, data: dict) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(yaml.dump(data, default_flow_style=False), encoding="utf-8") + + +# --------------------------------------------------------------------------- +# T15: campaign-kind-is-campaign +# --------------------------------------------------------------------------- + + +def test_campaign_kind_is_campaign_fires_on_wrong_kind(): + recipe = _standard_recipe( + dispatches=[CampaignDispatch(name="p1", recipe="impl", task="do it")], + ) + found = _findings(recipe, "campaign-kind-is-campaign") + assert found + assert found[0].severity == Severity.ERROR + + +# --------------------------------------------------------------------------- +# T16: campaign-has-dispatches +# --------------------------------------------------------------------------- + + +def test_campaign_has_dispatches_fires_on_empty(): + recipe = _campaign(dispatches=[]) + found = _findings(recipe, "campaign-has-dispatches") + assert found + assert found[0].severity == Severity.ERROR + + +# --------------------------------------------------------------------------- +# T17: dispatch-names-unique +# --------------------------------------------------------------------------- + + +def test_dispatch_names_unique_detects_duplicates(): + recipe = _campaign( + dispatches=[ + CampaignDispatch(name="phase-one", recipe="impl", task="a"), + CampaignDispatch(name="phase-one", recipe="impl", task="b"), + ] + ) + found = _findings(recipe, "dispatch-names-unique") + assert found + assert found[0].severity == Severity.ERROR + assert "phase-one" in found[0].message + + +# --------------------------------------------------------------------------- +# T18: dispatch-names-kebab-case +# --------------------------------------------------------------------------- + + +def test_dispatch_names_kebab_case_warns_on_underscore(): + recipe = _campaign( + dispatches=[ + CampaignDispatch(name="phase_one", recipe="impl", task="do it"), + ] + ) + found = _findings(recipe, "dispatch-names-kebab-case") + assert found + assert found[0].severity == Severity.WARNING + assert "phase_one" in found[0].message + + +# --------------------------------------------------------------------------- +# T19: dispatch-recipe-exists +# --------------------------------------------------------------------------- + + +def test_dispatch_recipe_exists_fires_on_unknown(): + recipe = _campaign( + dispatches=[ + CampaignDispatch(name="phase-one", recipe="unknown-recipe", task="do it"), + ] + ) + found = _findings( + recipe, + "dispatch-recipe-exists", + available_recipes=frozenset({"implementation", "research"}), + ) + assert found + assert "unknown-recipe" in found[0].message + + +# --------------------------------------------------------------------------- +# T20: dispatch-recipe-is-standard +# --------------------------------------------------------------------------- + + +def test_campaign_rejects_dispatch_of_campaign_recipe(tmp_path: Path): + recipes_dir = tmp_path / ".autoskillit" / "recipes" + recipes_dir.mkdir(parents=True) + _write_recipe_yaml( + recipes_dir / "target-campaign.yaml", + { + "name": "target-campaign", + "description": "another campaign", + "kind": "campaign", + "kitchen_rules": ["NEVER"], + "dispatches": [ + { + "name": "sub-phase", + "recipe": "implementation", + "task": "work", + } + ], + }, + ) + recipe = _campaign( + dispatches=[ + CampaignDispatch(name="phase-one", recipe="target-campaign", task="do it"), + ] + ) + found = _findings(recipe, "dispatch-recipe-is-standard", project_dir=tmp_path) + assert found + assert found[0].severity == Severity.ERROR + + +def test_dispatch_recipe_is_standard_allows_food_truck_target(tmp_path: Path): + """Campaign dispatching to a food-truck-kind recipe produces no findings.""" + recipes_dir = tmp_path / ".autoskillit" / "recipes" + recipes_dir.mkdir(parents=True) + _write_recipe_yaml( + recipes_dir / "target-food-truck.yaml", + { + "name": "target-food-truck", + "description": "a food truck", + "kind": "food-truck", + "kitchen_rules": ["NEVER"], + "steps": {"done": {"action": "stop", "message": "sentinel done"}}, + }, + ) + recipe = _campaign( + dispatches=[ + CampaignDispatch(name="phase-one", recipe="target-food-truck", task="do it"), + ] + ) + found = _findings(recipe, "dispatch-recipe-is-standard", project_dir=tmp_path) + assert found == [] + + +# --------------------------------------------------------------------------- +# T21: dispatch-recipe-in-declared-packs +# --------------------------------------------------------------------------- + + +def test_dispatch_recipe_in_declared_packs_warns(tmp_path: Path): + recipes_dir = tmp_path / ".autoskillit" / "recipes" + recipes_dir.mkdir(parents=True) + _write_recipe_yaml( + recipes_dir / "research-recipe.yaml", + { + "name": "research-recipe", + "description": "research", + "kind": "standard", + "kitchen_rules": ["NEVER"], + "categories": ["research-family"], + "steps": {"stop": {"action": "stop", "message": "done"}}, + }, + ) + recipe = _campaign( + requires_recipe_packs=["implementation-family"], + dispatches=[ + CampaignDispatch(name="phase-one", recipe="research-recipe", task="do it"), + ], + ) + found = _findings(recipe, "dispatch-recipe-in-declared-packs", project_dir=tmp_path) + assert found + assert found[0].severity == Severity.WARNING + + +# --------------------------------------------------------------------------- +# T22: campaign-requires-recipe-packs-exist +# --------------------------------------------------------------------------- + + +def test_campaign_requires_recipe_packs_exist_warns_on_unknown(): + recipe = _campaign(requires_recipe_packs=["nonexistent-family"]) + found = _findings(recipe, "campaign-requires-recipe-packs-exist") + assert found + assert found[0].severity == Severity.WARNING + assert "nonexistent-family" in found[0].message + + +def test_campaign_requires_recipe_packs_exist_no_warning_for_known_pack(): + recipe = _campaign(requires_recipe_packs=["implementation-family"]) + found = _findings(recipe, "campaign-requires-recipe-packs-exist") + assert not found + + +# --------------------------------------------------------------------------- +# T23: dispatch-ingredients-keys-in-target-schema +# --------------------------------------------------------------------------- + + +def test_dispatch_ingredients_keys_in_target_schema_fires(tmp_path: Path): + recipes_dir = tmp_path / ".autoskillit" / "recipes" + recipes_dir.mkdir(parents=True) + _write_recipe_yaml( + recipes_dir / "target-recipe.yaml", + { + "name": "target-recipe", + "description": "target", + "kind": "standard", + "kitchen_rules": ["NEVER"], + "ingredients": { + "task": {"description": "The task", "required": True}, + }, + "steps": {"stop": {"action": "stop", "message": "done"}}, + }, + ) + recipe = _campaign( + dispatches=[ + CampaignDispatch( + name="phase-one", + recipe="target-recipe", + task="do it", + ingredients={"task": "do it", "nonexistent_key": "value"}, + ), + ] + ) + found = _findings(recipe, "dispatch-ingredients-keys-in-target-schema", project_dir=tmp_path) + assert found + assert "nonexistent_key" in found[0].message + + +# --------------------------------------------------------------------------- +# T24: dispatch-ingredient-values-are-strings +# --------------------------------------------------------------------------- + + +def test_dispatch_ingredient_values_are_strings_fires_on_non_string(): + dispatch = CampaignDispatch( + name="phase-one", + recipe="impl", + task="do it", + ingredients={"key": 123}, # type: ignore[arg-type] + ) + recipe = _campaign(dispatches=[dispatch]) + found = _findings(recipe, "dispatch-ingredient-values-are-strings") + assert found + assert found[0].severity == Severity.ERROR + assert "key" in found[0].message + + +# --------------------------------------------------------------------------- +# T25: depends-on-refers-to-valid-dispatches +# --------------------------------------------------------------------------- + + +def test_depends_on_refers_to_valid_dispatches_fires(): + recipe = _campaign( + dispatches=[ + CampaignDispatch( + name="phase-one", + recipe="impl", + task="do it", + depends_on=["nonexistent"], + ), + ] + ) + found = _findings(recipe, "depends-on-refers-to-valid-dispatches") + assert found + assert "nonexistent" in found[0].message + + +# --------------------------------------------------------------------------- +# T26: depends-on-acyclic (cycle) +# --------------------------------------------------------------------------- + + +def test_depends_on_acyclic_detects_cycle(): + recipe = _campaign( + dispatches=[ + CampaignDispatch(name="a", recipe="impl", task="a", depends_on=["c"]), + CampaignDispatch(name="b", recipe="impl", task="b", depends_on=["a"]), + CampaignDispatch(name="c", recipe="impl", task="c", depends_on=["b"]), + ] + ) + found = _findings(recipe, "depends-on-acyclic") + assert found + assert found[0].severity == Severity.ERROR + + +# --------------------------------------------------------------------------- +# T27: depends-on-acyclic (DAG — no cycle) +# --------------------------------------------------------------------------- + + +def test_depends_on_acyclic_passes_on_dag(): + recipe = _campaign( + dispatches=[ + CampaignDispatch(name="a", recipe="impl", task="a", depends_on=[]), + CampaignDispatch(name="b", recipe="impl", task="b", depends_on=["a"]), + CampaignDispatch(name="c", recipe="impl", task="c", depends_on=["b"]), + ] + ) + found = _findings(recipe, "depends-on-acyclic") + assert not found + + +# --------------------------------------------------------------------------- +# T-S1: campaign-dispatch-depends-on-is-sequential +# --------------------------------------------------------------------------- + + +def test_campaign_dispatch_depends_on_is_sequential_fires_on_fan_in(): + recipe = _campaign( + dispatches=[ + CampaignDispatch(name="phase-one", recipe="impl", task="a", depends_on=[]), + CampaignDispatch(name="phase-two", recipe="impl", task="b", depends_on=[]), + CampaignDispatch( + name="phase-merge", + recipe="impl", + task="c", + depends_on=["phase-one", "phase-two"], + ), + ] + ) + found = _findings(recipe, "campaign-dispatch-depends-on-is-sequential") + assert len(found) == 1 + assert found[0].severity == Severity.ERROR + assert "phase-merge" in found[0].message + assert "phase-one" in found[0].message or "phase-two" in found[0].message + + +def test_campaign_dispatch_depends_on_is_sequential_passes_on_linear_chain(): + recipe = _campaign( + dispatches=[ + CampaignDispatch(name="a", recipe="impl", task="a", depends_on=[]), + CampaignDispatch(name="b", recipe="impl", task="b", depends_on=["a"]), + CampaignDispatch(name="c", recipe="impl", task="c", depends_on=["b"]), + CampaignDispatch(name="d", recipe="impl", task="d", depends_on=["c"]), + ] + ) + found = _findings(recipe, "campaign-dispatch-depends-on-is-sequential") + assert found == [] + + +# --------------------------------------------------------------------------- +# T28: campaign-task-non-empty +# --------------------------------------------------------------------------- + + +def test_campaign_task_non_empty_fires(): + recipe = _campaign( + dispatches=[ + CampaignDispatch(name="phase-one", recipe="impl", task=""), + ] + ) + found = _findings(recipe, "campaign-task-non-empty") + assert found + assert "phase-one" in found[0].message + + +# --------------------------------------------------------------------------- +# T29: autoskillit-version-compatible +# --------------------------------------------------------------------------- + + +def test_autoskillit_version_compatible_warns(): + recipe = _campaign(version="999.0.0") + found = _findings(recipe, "autoskillit-version-compatible") + assert found + assert found[0].severity == Severity.WARNING + assert "999.0.0" in found[0].message + + +# --------------------------------------------------------------------------- +# T30: standard recipe skips all campaign rules +# --------------------------------------------------------------------------- + + +def test_campaign_rules_skip_for_standard_recipe(): + recipe = _standard_recipe() + all_findings = run_semantic_rules(recipe) + campaign_findings = [ + f + for f in all_findings + if f.rule.startswith("campaign-") + or f.rule.startswith("dispatch-") + or f.rule.startswith("depends-on-") + ] + assert not campaign_findings, ( + f"Campaign rules must not fire on standard recipe: {campaign_findings}" + ) + + +# --------------------------------------------------------------------------- +# T31: valid campaign passes all rules +# --------------------------------------------------------------------------- + + +def test_campaign_valid_passes_all_rules(): + recipe = _campaign( + dispatches=[ + CampaignDispatch( + name="phase-one", + recipe="implementation", + task="Implement the feature", + ingredients={"task": "Implement the feature"}, + ), + CampaignDispatch( + name="phase-two", + recipe="implementation", + task="Write tests", + depends_on=["phase-one"], + ), + ], + requires_recipe_packs=["implementation-family"], + version="0.1.0", + ) + ctx = make_validation_context( + recipe, + available_recipes=frozenset({"implementation"}), + ) + all_findings = run_semantic_rules(ctx) + _is_campaign_rule = lambda f: ( # noqa: E731 + f.rule.startswith("campaign-") + or f.rule.startswith("dispatch-") + or f.rule.startswith("depends-on-") + ) + error_findings = [ + f for f in all_findings if f.severity == Severity.ERROR and _is_campaign_rule(f) + ] + warning_findings = [ + f for f in all_findings if f.severity == Severity.WARNING and _is_campaign_rule(f) + ] + assert not error_findings, f"Valid campaign must not have ERROR findings: {error_findings}" + assert not warning_findings, ( + f"Valid campaign must not have WARNING findings: {warning_findings}" + ) + + +# --------------------------------------------------------------------------- +# T32: dispatch-capture-keys-are-identifiers +# --------------------------------------------------------------------------- + + +def test_capture_key_must_be_identifier(): + recipe = _campaign( + dispatches=[ + CampaignDispatch( + name="phase-one", + recipe="implementation", + task="t", + capture={"bad-key": "${{ result.v }}"}, + ) + ] + ) + findings = _findings(recipe, "dispatch-capture-keys-are-identifiers") + assert len(findings) == 1 + assert findings[0].severity == Severity.ERROR + + +# --------------------------------------------------------------------------- +# T33: dispatch-capture-value-references-result +# --------------------------------------------------------------------------- + + +def test_capture_value_must_reference_result(): + recipe = _campaign( + dispatches=[ + CampaignDispatch( + name="phase-one", + recipe="implementation", + task="t", + capture={"k": "not_a_template"}, + ) + ] + ) + findings = _findings(recipe, "dispatch-capture-value-references-result") + assert len(findings) == 1 + assert findings[0].severity == Severity.ERROR + + +# --------------------------------------------------------------------------- +# T34: campaign-ingredient-refs-have-prior-capture (unresolvable ref) +# --------------------------------------------------------------------------- + + +def test_campaign_ingredient_ref_requires_prior_capture(): + recipe = _campaign( + dispatches=[ + CampaignDispatch( + name="phase-one", + recipe="implementation", + task="t", + ), + CampaignDispatch( + name="phase-two", + recipe="implementation", + task="t", + ingredients={"x": "${{ campaign.x }}"}, + depends_on=["phase-one"], + ), + ] + ) + findings = _findings(recipe, "campaign-ingredient-refs-have-prior-capture") + assert len(findings) == 1 + assert findings[0].severity == Severity.ERROR + + +# --------------------------------------------------------------------------- +# T35: campaign-ingredient-refs-have-prior-capture (satisfied by ancestor) +# --------------------------------------------------------------------------- + + +def test_campaign_ingredient_ref_satisfied_by_ancestor(): + recipe = _campaign( + dispatches=[ + CampaignDispatch( + name="phase-one", + recipe="implementation", + task="t", + capture={"x": "${{ result.x }}"}, + ), + CampaignDispatch( + name="phase-two", + recipe="implementation", + task="t", + ingredients={"x": "${{ campaign.x }}"}, + depends_on=["phase-one"], + ), + ] + ) + findings = _findings(recipe, "campaign-ingredient-refs-have-prior-capture") + assert findings == [] + + +# --------------------------------------------------------------------------- +# T36: valid capture spec passes +# --------------------------------------------------------------------------- + + +def test_valid_capture_spec_passes(): + recipe = _campaign( + dispatches=[ + CampaignDispatch( + name="phase-one", + recipe="implementation", + task="t", + capture={"out_file": "${{ result.out_file }}"}, + ) + ] + ) + capture_key_findings = _findings(recipe, "dispatch-capture-keys-are-identifiers") + capture_val_findings = _findings(recipe, "dispatch-capture-value-references-result") + assert capture_key_findings == [] + assert capture_val_findings == [] + + +# --------------------------------------------------------------------------- +# T-G1: gate-dispatch-valid-type +# --------------------------------------------------------------------------- + + +def test_gate_dispatch_valid_type_fires_on_unknown_value(): + recipe = _campaign( + dispatches=[CampaignDispatch(name="gate-check", gate="approve", message="Approve?")] + ) + found = _findings(recipe, "gate-dispatch-valid-type") + assert found + assert found[0].severity == Severity.ERROR + + +def test_gate_dispatch_valid_type_passes_for_confirm(): + recipe = _campaign( + dispatches=[CampaignDispatch(name="gate-check", gate="confirm", message="Proceed?")] + ) + found = _findings(recipe, "gate-dispatch-valid-type") + assert found == [] + + +# --------------------------------------------------------------------------- +# T-G3: gate-dispatch-has-message +# --------------------------------------------------------------------------- + + +def test_gate_dispatch_has_message_fires_on_empty_message(): + recipe = _campaign( + dispatches=[CampaignDispatch(name="gate-check", gate="confirm", message="")] + ) + found = _findings(recipe, "gate-dispatch-has-message") + assert found + assert found[0].severity == Severity.ERROR + + +# --------------------------------------------------------------------------- +# T-G4: gate-dispatch-no-recipe +# --------------------------------------------------------------------------- + + +def test_gate_dispatch_no_recipe_fires_when_recipe_is_set(): + recipe = _campaign( + dispatches=[ + CampaignDispatch( + name="gate-check", + gate="confirm", + recipe="some-recipe", + task="do it", + message="Proceed?", + ) + ] + ) + found = _findings(recipe, "gate-dispatch-no-recipe") + assert found + assert found[0].severity == Severity.ERROR + + +# --------------------------------------------------------------------------- +# T-G5: gate-dispatch-no-capture +# --------------------------------------------------------------------------- + + +def test_gate_dispatch_no_capture_fires_when_capture_is_set(): + recipe = _campaign( + dispatches=[ + CampaignDispatch( + name="gate-check", + gate="confirm", + message="Proceed?", + capture={"key": "${{ result.val }}"}, + ) + ] + ) + found = _findings(recipe, "gate-dispatch-no-capture") + assert found + assert found[0].severity == Severity.ERROR + + +# --------------------------------------------------------------------------- +# T-G6: campaign-task-non-empty exempts gate dispatches +# --------------------------------------------------------------------------- + + +def test_campaign_task_non_empty_exempts_gate_dispatches(): + recipe = _campaign( + dispatches=[CampaignDispatch(name="gate-check", gate="confirm", message="Proceed?")] + ) + found = _findings(recipe, "campaign-task-non-empty") + assert found == [] + + +# --------------------------------------------------------------------------- +# T-G7: dispatch-recipe-exists exempts gate dispatches +# --------------------------------------------------------------------------- + + +def test_dispatch_recipe_exists_exempts_gate_dispatches(): + recipe = _campaign( + dispatches=[CampaignDispatch(name="gate-check", gate="confirm", message="Proceed?")] + ) + found = _findings( + recipe, "dispatch-recipe-exists", available_recipes=frozenset({"some-other-recipe"}) + ) + assert found == [] + + +# --------------------------------------------------------------------------- +# T-G8: dispatch-recipe-is-standard exempts gate dispatches +# --------------------------------------------------------------------------- + + +def test_dispatch_recipe_is_standard_exempts_gate_dispatches(): + recipe = _campaign( + dispatches=[CampaignDispatch(name="gate-check", gate="confirm", message="Proceed?")] + ) + found = _findings(recipe, "dispatch-recipe-is-standard", project_dir=None) + assert found == [] + + +# --------------------------------------------------------------------------- +# T-G9: dispatch-ingredients-keys-in-target-schema exempts gate dispatches +# --------------------------------------------------------------------------- + + +def test_dispatch_ingredients_keys_exempts_gate_dispatches(): + recipe = _campaign( + dispatches=[ + CampaignDispatch( + name="gate-check", + gate="confirm", + message="Proceed?", + ingredients={"foo": "bar"}, + ) + ] + ) + found = _findings(recipe, "dispatch-ingredients-keys-in-target-schema") + assert found == [] + + +# --------------------------------------------------------------------------- +# T-D1: campaign-dangling-ingredient +# --------------------------------------------------------------------------- + + +def test_campaign_dangling_ingredient_fires_on_missing_forwarding(tmp_path: Path): + """Rule fires when campaign ingredient is declared by target but dispatch omits it.""" + recipes_dir = tmp_path / ".autoskillit" / "recipes" + recipes_dir.mkdir(parents=True) + _write_recipe_yaml( + recipes_dir / "target-recipe.yaml", + { + "name": "target-recipe", + "description": "target", + "kind": "standard", + "kitchen_rules": ["NEVER"], + "ingredients": { + "review_pr": {"description": "Review PR", "default": "false"}, + }, + "steps": {"stop": {"action": "stop", "message": "done"}}, + }, + ) + recipe = _campaign( + ingredients={"review_pr": {"description": "Review PR", "default": "false"}}, + dispatches=[ + CampaignDispatch( + name="phase-one", + recipe="target-recipe", + task="do it", + ingredients={}, + ), + ], + ) + found = _findings(recipe, "campaign-dangling-ingredient", project_dir=tmp_path) + assert len(found) == 1 + assert found[0].severity == Severity.WARNING + assert "review_pr" in found[0].message + + +def test_campaign_dangling_ingredient_passes_when_forwarded(tmp_path: Path): + """Rule passes when ingredient is forwarded.""" + recipes_dir = tmp_path / ".autoskillit" / "recipes" + recipes_dir.mkdir(parents=True) + _write_recipe_yaml( + recipes_dir / "target-recipe.yaml", + { + "name": "target-recipe", + "description": "target", + "kind": "standard", + "kitchen_rules": ["NEVER"], + "ingredients": { + "review_pr": {"description": "Review PR", "default": "false"}, + }, + "steps": {"stop": {"action": "stop", "message": "done"}}, + }, + ) + recipe = _campaign( + ingredients={"review_pr": {"description": "Review PR", "default": "false"}}, + dispatches=[ + CampaignDispatch( + name="phase-one", + recipe="target-recipe", + task="do it", + ingredients={"review_pr": "${{ inputs.review_pr }}"}, + ), + ], + ) + found = _findings(recipe, "campaign-dangling-ingredient", project_dir=tmp_path) + assert found == [] + + +def test_campaign_dangling_ingredient_passes_when_target_does_not_declare(tmp_path: Path): + """Rule passes when target sub-recipe does not declare the ingredient.""" + recipes_dir = tmp_path / ".autoskillit" / "recipes" + recipes_dir.mkdir(parents=True) + _write_recipe_yaml( + recipes_dir / "target-recipe.yaml", + { + "name": "target-recipe", + "description": "target", + "kind": "standard", + "kitchen_rules": ["NEVER"], + "ingredients": { + "other_ing": {"description": "Other", "default": "x"}, + }, + "steps": {"stop": {"action": "stop", "message": "done"}}, + }, + ) + recipe = _campaign( + ingredients={"output_mode": {"description": "Output mode", "default": "local"}}, + dispatches=[ + CampaignDispatch( + name="phase-one", + recipe="target-recipe", + task="do it", + ingredients={}, + ), + ], + ) + found = _findings(recipe, "campaign-dangling-ingredient", project_dir=tmp_path) + assert found == [] + + +def test_campaign_dangling_ingredient_skips_unloadable_target(tmp_path: Path): + """Rule skips when target recipe is not loadable.""" + recipe = _campaign( + ingredients={"review_pr": {"description": "Review PR", "default": "false"}}, + dispatches=[ + CampaignDispatch( + name="phase-one", + recipe="nonexistent-recipe", + task="do it", + ingredients={}, + ), + ], + ) + found = _findings(recipe, "campaign-dangling-ingredient", project_dir=tmp_path) + assert found == [] + + +def test_campaign_dangling_ingredient_fires_per_dispatch(tmp_path: Path): + """Rule fires per-dispatch for multi-dispatch campaigns.""" + recipes_dir = tmp_path / ".autoskillit" / "recipes" + recipes_dir.mkdir(parents=True) + _write_recipe_yaml( + recipes_dir / "target-forwarding.yaml", + { + "name": "target-forwarding", + "description": "target", + "kind": "standard", + "kitchen_rules": ["NEVER"], + "ingredients": { + "output_mode": {"description": "Output mode", "default": "local"}, + }, + "steps": {"stop": {"action": "stop", "message": "done"}}, + }, + ) + _write_recipe_yaml( + recipes_dir / "target-omitting.yaml", + { + "name": "target-omitting", + "description": "target", + "kind": "standard", + "kitchen_rules": ["NEVER"], + "ingredients": { + "output_mode": {"description": "Output mode", "default": "local"}, + }, + "steps": {"stop": {"action": "stop", "message": "done"}}, + }, + ) + recipe = _campaign( + ingredients={"output_mode": {"description": "Output mode", "default": "local"}}, + dispatches=[ + CampaignDispatch( + name="phase-forward", + recipe="target-forwarding", + task="do it", + ingredients={"output_mode": "${{ inputs.output_mode }}"}, + ), + CampaignDispatch( + name="phase-omit", + recipe="target-omitting", + task="do it", + ingredients={}, + ), + ], + ) + found = _findings(recipe, "campaign-dangling-ingredient", project_dir=tmp_path) + assert len(found) == 1 + assert "phase-omit" in found[0].message + assert "phase-forward" not in found[0].message + + +def test_campaign_dangling_ingredient_ignores_gate_dispatches(tmp_path: Path): + """Rule ignores gate dispatches.""" + recipe = _campaign( + ingredients={"review_pr": {"description": "Review PR", "default": "false"}}, + dispatches=[ + CampaignDispatch(name="gate-check", gate="confirm", message="Proceed?"), + ], + ) + found = _findings(recipe, "campaign-dangling-ingredient", project_dir=tmp_path) + assert found == [] + + +def test_campaign_dangling_ingredient_ignores_non_campaign_recipes(tmp_path: Path): + """Rule ignores non-campaign recipes.""" + recipe = _standard_recipe() + found = _findings(recipe, "campaign-dangling-ingredient", project_dir=tmp_path) + assert found == [] + + +def test_campaign_dangling_ingredient_exempts_task(tmp_path: Path): + """Rule accounts for task auto-injection and should not fire for it.""" + recipes_dir = tmp_path / ".autoskillit" / "recipes" + recipes_dir.mkdir(parents=True) + _write_recipe_yaml( + recipes_dir / "target-recipe.yaml", + { + "name": "target-recipe", + "description": "target", + "kind": "standard", + "kitchen_rules": ["NEVER"], + "ingredients": { + "task": {"description": "The task", "required": True}, + }, + "steps": {"stop": {"action": "stop", "message": "done"}}, + }, + ) + recipe = _campaign( + ingredients={"task": {"description": "The task", "required": True}}, + dispatches=[ + CampaignDispatch( + name="phase-one", + recipe="target-recipe", + task="Do the thing", # task field set, but not forwarded in ingredients + ingredients={}, + ), + ], + ) + found = _findings(recipe, "campaign-dangling-ingredient", project_dir=tmp_path) + assert found == [] diff --git a/tests/recipe/test_rules_ci.py b/tests/recipe/test_rules_ci.py index ca5b257f6c..71b5ac8d1c 100644 --- a/tests/recipe/test_rules_ci.py +++ b/tests/recipe/test_rules_ci.py @@ -2,11 +2,15 @@ from __future__ import annotations +import pytest + from autoskillit.core import PRState, Severity from autoskillit.recipe.io import _parse_step, builtin_recipes_dir, load_recipe from autoskillit.recipe.registry import run_semantic_rules from autoskillit.recipe.schema import Recipe, RecipeStep, StepResultCondition, StepResultRoute +pytestmark = [pytest.mark.layer("recipe"), pytest.mark.small] + def _make_recipe(steps: dict[str, RecipeStep]) -> Recipe: """Minimal recipe factory for CI rule tests.""" @@ -243,8 +247,45 @@ def test_ci_failure_no_on_failure_skips_rule() -> None: # --------------------------------------------------------------------------- -def test_wait_for_ci_without_event_is_warning() -> None: - """wait_for_ci step with no event param should trigger ci-missing-event-scope.""" +def test_ci_missing_event_scope_is_error_severity() -> None: + """ci-missing-event-scope must be ERROR to prevent unscoped CI watch steps.""" + recipe = _make_workflow( + steps={ + "my_ci_watch": { + "tool": "wait_for_ci", + "with": {"branch": "main", "timeout_seconds": 300}, + } + } + ) + findings = run_semantic_rules(recipe) + scope_findings = [f for f in findings if f.rule == "ci-missing-event-scope"] + assert scope_findings, "expected ci-missing-event-scope finding" + assert all(f.severity == Severity.ERROR for f in scope_findings), ( + "ci-missing-event-scope must be ERROR, not WARNING" + ) + + +def test_ci_watch_with_event_produces_no_scope_finding() -> None: + """wait_for_ci with event: present must not trigger ci-missing-event-scope.""" + recipe = _make_workflow( + steps={ + "my_ci_watch": { + "tool": "wait_for_ci", + "with": { + "branch": "main", + "event": "${{ context.ci_event }}", + "timeout_seconds": 300, + }, + } + } + ) + findings = run_semantic_rules(recipe) + scope_findings = [f for f in findings if f.rule == "ci-missing-event-scope"] + assert not scope_findings + + +def test_wait_for_ci_without_event_is_error() -> None: + """wait_for_ci step with no event param should trigger ci-missing-event-scope at ERROR.""" recipe = _make_recipe( { "ci": RecipeStep( @@ -259,7 +300,7 @@ def test_wait_for_ci_without_event_is_warning() -> None: findings = run_semantic_rules(recipe) event_findings = [f for f in findings if f.rule == "ci-missing-event-scope"] assert len(event_findings) == 1 - assert event_findings[0].severity == Severity.WARNING + assert event_findings[0].severity == Severity.ERROR def test_wait_for_ci_with_event_is_clean() -> None: @@ -280,8 +321,8 @@ def test_wait_for_ci_with_event_is_clean() -> None: assert event_findings == [] -def test_get_ci_status_without_event_is_warning() -> None: - """get_ci_status step with no event param should trigger ci-missing-event-scope.""" +def test_get_ci_status_without_event_is_error() -> None: + """get_ci_status step with no event param should trigger ci-missing-event-scope at ERROR.""" recipe = _make_recipe( { "ci": RecipeStep( @@ -296,7 +337,7 @@ def test_get_ci_status_without_event_is_warning() -> None: findings = run_semantic_rules(recipe) event_findings = [f for f in findings if f.rule == "ci-missing-event-scope"] assert len(event_findings) == 1 - assert event_findings[0].severity == Severity.WARNING + assert event_findings[0].severity == Severity.ERROR assert "get_ci_status" in event_findings[0].message @@ -371,7 +412,7 @@ def test_wait_for_ci_no_workflow_is_clean() -> None: def _make_mq_conditions( *, exclude: set[str] | None = None, - fallback_route: str = "release_issue_timeout", + fallback_route: str = "register_clone_unconfirmed", ) -> list[StepResultCondition]: """Build a complete wait_for_merge_queue on_result conditions list. @@ -399,8 +440,8 @@ def _make_mq_conditions( def _make_mq_step( *, exclude: set[str] | None = None, - fallback_route: str = "release_issue_timeout", - on_failure: str = "release_issue_timeout", + fallback_route: str = "register_clone_unconfirmed", + on_failure: str = "register_clone_unconfirmed", ) -> RecipeStep: return RecipeStep( tool="wait_for_merge_queue", @@ -414,16 +455,34 @@ def _make_mq_step( def _make_mq_recipe( *, exclude: set[str] | None = None, - fallback_route: str = "release_issue_timeout", - on_failure: str = "release_issue_timeout", + fallback_route: str = "register_clone_unconfirmed", + on_failure: str = "register_clone_unconfirmed", +) -> Recipe: + """Build a recipe with register_clone_unconfirmed (impl/remed family pattern).""" + return _make_recipe( + { + "wait_for_queue": _make_mq_step( + exclude=exclude, fallback_route=fallback_route, on_failure=on_failure + ), + "register_clone_unconfirmed": RecipeStep(action="stop", message="timeout"), + "done": RecipeStep(action="stop", message="done"), + } + ) + + +def _make_mq_recipe_no_sentinel( + *, + exclude: set[str] | None = None, + fallback_route: str = "register_clone_failure", + on_failure: str = "register_clone_failure", ) -> Recipe: - """Build a recipe that includes release_issue_timeout so the new rules activate.""" + """Build a recipe WITHOUT register_clone_unconfirmed (e.g. merge-prs family pattern).""" return _make_recipe( { "wait_for_queue": _make_mq_step( exclude=exclude, fallback_route=fallback_route, on_failure=on_failure ), - "release_issue_timeout": RecipeStep(action="stop", message="timeout"), + "register_clone_failure": RecipeStep(action="stop", message="failed"), "done": RecipeStep(action="stop", message="done"), } ) @@ -479,3 +538,431 @@ def test_conformance_rule_clean_when_targets_correct() -> None: assert conformance_findings == [], ( f"Expected no conformance findings for correct targets, got: {conformance_findings}" ) + + +# --------------------------------------------------------------------------- +# Tool-presence scope gate tests (I7 fires without register_clone_unconfirmed sentinel) +# --------------------------------------------------------------------------- + + +def test_coverage_rule_fires_without_register_clone_unconfirmed_step() -> None: + """I7 must fire on recipes without register_clone_unconfirmed if they have mq routing.""" + recipe = _make_mq_recipe_no_sentinel(exclude={"dropped_healthy", "stalled"}) + findings = run_semantic_rules(recipe) + coverage_findings = [f for f in findings if f.rule == _COVERAGE_RULE] + assert len(coverage_findings) >= 1, ( + f"Expected coverage finding (no sentinel step), got: {findings}" + ) + assert coverage_findings[0].severity == Severity.ERROR + assert "dropped_healthy" in coverage_findings[0].message + assert "stalled" in coverage_findings[0].message + + +def test_coverage_rule_clean_without_sentinel_when_all_states_present() -> None: + """I7 is silent for a non-sentinel recipe when all PRState values are present.""" + recipe = _make_mq_recipe_no_sentinel() + findings = run_semantic_rules(recipe) + coverage_findings = [f for f in findings if f.rule == _COVERAGE_RULE] + assert coverage_findings == [], ( + f"Expected no coverage findings for complete routing without sentinel, " + f"got: {coverage_findings}" + ) + + +def test_coverage_rule_silent_for_non_queue_recipe() -> None: + """I7 must not fire on recipes that have no wait_for_merge_queue step.""" + recipe = _make_recipe( + { + "do_work": RecipeStep(tool="run_skill", on_success="done"), + "done": RecipeStep(action="stop", message="done"), + } + ) + findings = run_semantic_rules(recipe) + coverage_findings = [f for f in findings if f.rule == _COVERAGE_RULE] + assert coverage_findings == [], ( + f"Non-queue recipe should produce no I7 findings, got: {coverage_findings}" + ) + + +def test_conformance_rule_silent_without_sentinel() -> None: + """I8 must NOT fire on recipes without register_clone_unconfirmed (family-specific rule).""" + recipe = _make_mq_recipe_no_sentinel( + fallback_route="register_clone_failure", + on_failure="register_clone_failure", + ) + findings = run_semantic_rules(recipe) + conformance_findings = [f for f in findings if f.rule == _CONFORMANCE_RULE] + assert conformance_findings == [], ( + f"I8 should be silent for non-sentinel recipes, got: {conformance_findings}" + ) + + +# --------------------------------------------------------------------------- +# ci-no-runs-unguarded rule tests +# --------------------------------------------------------------------------- + +_NO_RUNS_RULE = "ci-no-runs-unguarded" + + +def test_ci_no_runs_unguarded_detects_bare_on_success() -> None: + """ci_watch with on_success -> merge step must be flagged as unguarded.""" + wf = _make_workflow( + { + "ci_watch": { + "tool": "wait_for_ci", + "on_success": "merge_step", + "on_failure": "handle_failure", + "with": {"event": "${{ context.ci_event }}", "branch": "main"}, + }, + "merge_step": {"tool": "enqueue_pr"}, + "handle_failure": { + "tool": "run_skill", + "with": {"skill_command": "/autoskillit:diagnose-ci b - -"}, + }, + } + ) + findings = run_semantic_rules(wf) + no_runs_findings = [f for f in findings if f.rule == _NO_RUNS_RULE] + assert len(no_runs_findings) == 1 + assert no_runs_findings[0].severity == Severity.ERROR + assert no_runs_findings[0].step_name == "ci_watch" + + +def test_ci_no_runs_unguarded_passes_with_on_result() -> None: + """ci_watch with on_result routing on conclusion is not flagged.""" + wf = _make_workflow( + { + "ci_watch": { + "tool": "wait_for_ci", + "on_result": [ + {"when": "${{ result.conclusion }} == 'no_runs'", "route": "handle_no_runs"}, + {"when": "${{ result.conclusion }} == 'success'", "route": "merge_step"}, + {"route": "handle_failure"}, + ], + "with": {"event": "${{ context.ci_event }}", "branch": "main"}, + }, + "merge_step": {"tool": "enqueue_pr"}, + "handle_no_runs": { + "tool": "run_skill", + "with": {"skill_command": "/autoskillit:diagnose-ci b - -"}, + }, + "handle_failure": { + "tool": "run_skill", + "with": {"skill_command": "/autoskillit:diagnose-ci b - -"}, + }, + } + ) + findings = run_semantic_rules(wf) + no_runs_findings = [f for f in findings if f.rule == _NO_RUNS_RULE] + assert len(no_runs_findings) == 0 + + +def test_ci_no_runs_unguarded_flags_on_result_without_no_runs_guard() -> None: + """on_result with only explicit when-conditions (no catch-all) still flags.""" + wf = _make_workflow( + { + "ci_watch": { + "tool": "wait_for_ci", + "on_success": "merge_step", + "on_result": [ + {"when": "${{ result.conclusion }} == 'success'", "route": "merge_step"}, + {"when": "${{ result.conclusion }} == 'failure'", "route": "handle_failure"}, + ], + "with": {"event": "${{ context.ci_event }}", "branch": "main"}, + }, + "merge_step": {"tool": "enqueue_pr"}, + "handle_failure": { + "tool": "run_skill", + "with": {"skill_command": "/autoskillit:diagnose-ci b - -"}, + }, + } + ) + findings = run_semantic_rules(wf) + no_runs_findings = [f for f in findings if f.rule == _NO_RUNS_RULE] + assert len(no_runs_findings) == 1 + + +def test_ci_no_runs_unguarded_silent_for_non_ci_tools() -> None: + """Non-wait_for_ci steps with on_success are not flagged.""" + wf = _make_workflow( + { + "run_tests": { + "tool": "run_cmd", + "on_success": "merge_step", + "with": {"cmd": "echo ok"}, + }, + "merge_step": {"tool": "enqueue_pr"}, + } + ) + findings = run_semantic_rules(wf) + no_runs_findings = [f for f in findings if f.rule == _NO_RUNS_RULE] + assert len(no_runs_findings) == 0 + + +def test_bundled_recipes_no_runs_guarded() -> None: + """All bundled recipes with wait_for_ci must guard against no_runs.""" + for yaml_path in sorted(builtin_recipes_dir().glob("*.yaml")): + recipe = load_recipe(yaml_path) + findings = run_semantic_rules(recipe) + no_runs_findings = [f for f in findings if f.rule == _NO_RUNS_RULE] + assert len(no_runs_findings) == 0, ( + f"Recipe '{yaml_path.stem}' has unguarded no_runs paths: " + + ", ".join(f.message for f in no_runs_findings) + ) + + +# --------------------------------------------------------------------------- +# ci-timed-out-unguarded rule tests +# --------------------------------------------------------------------------- + +_TIMED_OUT_RULE = "ci-timed-out-unguarded" + + +def test_ci_timed_out_unguarded_detects_bare_on_success() -> None: + """wait_for_ci with on_success but no on_result must be flagged.""" + recipe = _make_workflow( + { + "ci_watch": { + "tool": "wait_for_ci", + "on_success": "merge_step", + "on_failure": "handle_failure", + "with": {"event": "${{ context.ci_event }}", "branch": "main"}, + }, + "merge_step": {"tool": "enqueue_pr"}, + "handle_failure": { + "tool": "run_skill", + "with": {"skill_command": "/autoskillit:diagnose-ci b - -"}, + }, + } + ) + findings = [f for f in run_semantic_rules(recipe) if f.rule == _TIMED_OUT_RULE] + assert len(findings) == 1 + assert findings[0].severity == Severity.ERROR + assert findings[0].step_name == "ci_watch" + + +def test_ci_timed_out_unguarded_fires_with_only_catch_all() -> None: + recipe = _make_workflow( + { + "ci_watch": { + "tool": "wait_for_ci", + "with": {"branch": "main", "event": "push", "timeout_seconds": 300}, + "on_result": [ + {"when": "${{ result.conclusion }} == 'success'", "route": "merge"}, + {"when": "${{ result.conclusion }} == 'no_runs'", "route": "handle_no_runs"}, + {"route": "detect_ci_conflict"}, + ], + }, + "merge": {"tool": "run_cmd", "with": {"cmd": "echo merge"}}, + "handle_no_runs": {"tool": "run_cmd", "with": {"cmd": "echo no_runs"}}, + "detect_ci_conflict": {"tool": "run_cmd", "with": {"cmd": "echo conflict"}}, + } + ) + findings = [f for f in run_semantic_rules(recipe) if f.rule == _TIMED_OUT_RULE] + assert len(findings) == 1 + assert findings[0].severity == Severity.ERROR + assert findings[0].step_name == "ci_watch" + + +def test_ci_timed_out_unguarded_passes_with_explicit_arm() -> None: + recipe = _make_workflow( + { + "ci_watch": { + "tool": "wait_for_ci", + "with": {"branch": "main", "event": "push", "timeout_seconds": 300}, + "on_result": [ + {"when": "${{ result.conclusion }} == 'success'", "route": "merge"}, + {"when": "${{ result.conclusion }} == 'timed_out'", "route": "ci_watch"}, + {"route": "detect_ci_conflict"}, + ], + }, + "merge": {"tool": "run_cmd", "with": {"cmd": "echo merge"}}, + "detect_ci_conflict": {"tool": "run_cmd", "with": {"cmd": "echo conflict"}}, + } + ) + findings = [f for f in run_semantic_rules(recipe) if f.rule == _TIMED_OUT_RULE] + assert len(findings) == 0 + + +def test_ci_timed_out_unguarded_silent_for_non_ci_tools() -> None: + recipe = _make_workflow( + { + "run_tests": { + "tool": "run_cmd", + "with": {"cmd": "pytest"}, + "on_result": [{"route": "done"}], + }, + "done": {"tool": "run_cmd", "with": {"cmd": "echo done"}}, + } + ) + findings = [f for f in run_semantic_rules(recipe) if f.rule == _TIMED_OUT_RULE] + assert len(findings) == 0 + + +def test_bundled_recipes_timed_out_guarded() -> None: + for yaml_path in sorted(builtin_recipes_dir().glob("*.yaml")): + recipe = load_recipe(yaml_path) + findings = [f for f in run_semantic_rules(recipe) if f.rule == _TIMED_OUT_RULE] + assert not findings, f"{yaml_path.name}: {[f.message for f in findings]}" + + +def test_ci_watch_post_queue_fix_timeout_600s() -> None: + target_recipes = ["implementation.yaml", "remediation.yaml", "implementation-groups.yaml"] + for name in target_recipes: + recipe = load_recipe(builtin_recipes_dir() / name) + step = recipe.steps["ci_watch_post_queue_fix"] + assert step.with_args["timeout_seconds"] == 600, ( + f"{name}: ci_watch_post_queue_fix timeout_seconds is " + f"{step.with_args['timeout_seconds']}, expected 600" + ) + + +# --------------------------------------------------------------------------- +# ci-event-literal-merge-group rule tests +# --------------------------------------------------------------------------- + +_CI_EVENT_MG_RULE = "ci-event-literal-merge-group" + + +def test_wait_for_ci_with_literal_merge_group_event_is_flagged() -> None: + """wait_for_ci with event='merge_group' must trigger ci-event-literal-merge-group ERROR.""" + steps = { + "ci_watch": RecipeStep( + tool="wait_for_ci", + with_args={"branch": "main", "event": "merge_group"}, + ), + } + recipe = _make_recipe(steps) + findings = run_semantic_rules(recipe) + mg_findings = [f for f in findings if f.rule == _CI_EVENT_MG_RULE] + assert len(mg_findings) == 1 + assert mg_findings[0].severity == Severity.ERROR + assert mg_findings[0].step_name == "ci_watch" + assert "merge_group" in mg_findings[0].message + + +def test_wait_for_ci_with_push_event_not_flagged() -> None: + """wait_for_ci with event='push' must not trigger ci-event-literal-merge-group.""" + steps = { + "ci_watch": RecipeStep( + tool="wait_for_ci", + with_args={"branch": "main", "event": "push"}, + ), + } + recipe = _make_recipe(steps) + findings = run_semantic_rules(recipe) + mg_findings = [f for f in findings if f.rule == _CI_EVENT_MG_RULE] + assert len(mg_findings) == 0 + + +def test_wait_for_ci_without_event_not_flagged() -> None: + """wait_for_ci with no event arg must not trigger ci-event-literal-merge-group.""" + steps = { + "ci_watch": RecipeStep( + tool="wait_for_ci", + with_args={"branch": "main"}, + ), + } + recipe = _make_recipe(steps) + findings = run_semantic_rules(recipe) + mg_findings = [f for f in findings if f.rule == _CI_EVENT_MG_RULE] + assert len(mg_findings) == 0 + + +def test_bundled_recipes_no_literal_merge_group_event() -> None: + """No bundled recipe must hardcode event='merge_group' in a wait_for_ci step.""" + for yaml_path in sorted(builtin_recipes_dir().glob("*.yaml")): + recipe = load_recipe(yaml_path) + findings = run_semantic_rules(recipe) + mg_findings = [f for f in findings if f.rule == _CI_EVENT_MG_RULE] + assert len(mg_findings) == 0, ( + f"Recipe '{yaml_path.stem}' hardcodes event='merge_group': " + + ", ".join(f.message for f in mg_findings) + ) + + +class TestCiTimeoutMinimum: + def test_timeout_below_minimum_fires_warning(self) -> None: + steps = { + "ci": RecipeStep( + tool="wait_for_ci", + with_args={"branch": "main", "event": "push", "timeout_seconds": "300"}, + on_success="done", + on_failure="done", + ), + "done": RecipeStep(action="stop", message="CI check complete."), + } + recipe = _make_recipe(steps) + findings = [f for f in run_semantic_rules(recipe) if f.rule == "ci-timeout-minimum"] + assert len(findings) == 1 + assert findings[0].severity == Severity.WARNING + assert "300" in findings[0].message + + def test_timeout_at_boundary_is_clean(self) -> None: + steps = { + "ci": RecipeStep( + tool="wait_for_ci", + with_args={"branch": "main", "event": "push", "timeout_seconds": "600"}, + on_success="done", + on_failure="done", + ), + "done": RecipeStep(action="stop", message="CI check complete."), + } + recipe = _make_recipe(steps) + findings = [f for f in run_semantic_rules(recipe) if f.rule == "ci-timeout-minimum"] + assert len(findings) == 0 + + def test_missing_timeout_key_is_clean(self) -> None: + steps = { + "ci": RecipeStep( + tool="wait_for_ci", + with_args={"branch": "main", "event": "push"}, + on_success="done", + on_failure="done", + ), + "done": RecipeStep(action="stop", message="CI check complete."), + } + recipe = _make_recipe(steps) + findings = [f for f in run_semantic_rules(recipe) if f.rule == "ci-timeout-minimum"] + assert len(findings) == 0 + + +def test_ci_watch_post_queue_fix_has_auto_trigger() -> None: + """ci_watch_post_queue_fix steps using wait_for_ci must have auto_trigger.""" + target_recipes = [ + "implementation.yaml", + "remediation.yaml", + "implementation-groups.yaml", + "merge-prs.yaml", + ] + for name in target_recipes: + recipe = load_recipe(builtin_recipes_dir() / name) + for step_name in ("ci_watch", "ci_watch_post_queue_fix"): + if step_name not in recipe.steps: + continue + step = recipe.steps[step_name] + if step.tool != "wait_for_ci": + continue + assert step.with_args.get("auto_trigger") in ("true", True), ( + f"{name}: {step_name} missing auto_trigger: true" + ) + + +def test_no_runs_re_is_module_level_constant(): + import regex + + import autoskillit.recipe.rules.rules_ci as rules_ci + + assert hasattr(rules_ci, "_NO_RUNS_RE") + assert isinstance(rules_ci._NO_RUNS_RE, regex.Pattern) + assert rules_ci._NO_RUNS_RE.pattern == r"""==\s*['"]?no_runs['"]?""" + + +def test_timed_out_re_is_module_level_constant(): + import regex + + import autoskillit.recipe.rules.rules_ci as rules_ci + + assert hasattr(rules_ci, "_TIMED_OUT_RE") + assert isinstance(rules_ci._TIMED_OUT_RE, regex.Pattern) + assert rules_ci._TIMED_OUT_RE.pattern == r"""==\s*['"]?timed_out['"]?""" diff --git a/tests/recipe/test_rules_ci_loops.py b/tests/recipe/test_rules_ci_loops.py new file mode 100644 index 0000000000..d323a5c286 --- /dev/null +++ b/tests/recipe/test_rules_ci_loops.py @@ -0,0 +1,303 @@ +"""Tests for ci-timed-out-self-loop-unguarded and ci-conflict-path-missing-auto-trigger rules.""" + +from __future__ import annotations + +import pytest + +from autoskillit.core import Severity +from autoskillit.recipe.io import builtin_recipes_dir, load_recipe +from autoskillit.recipe.registry import run_semantic_rules +from autoskillit.recipe.schema import Recipe, RecipeStep, StepResultCondition, StepResultRoute + +pytestmark = [pytest.mark.layer("recipe"), pytest.mark.small] + + +def _make_recipe(steps: dict[str, RecipeStep]) -> Recipe: + return Recipe( + name="test-ci-loops", + description="Test recipe for CI loop rules.", + version="0.2.0", + kitchen_rules=["test"], + steps=steps, + ) + + +# --------------------------------------------------------------------------- +# ci-timed-out-self-loop-unguarded +# --------------------------------------------------------------------------- + + +def test_timed_out_self_loop_without_guard_is_error() -> None: + """wait_for_ci with timed_out→self and no guard step → ERROR.""" + steps = { + "ci_watch": RecipeStep( + tool="wait_for_ci", + with_args={"branch": "main", "timeout_seconds": 300}, + on_result=StepResultRoute( + conditions=[ + StepResultCondition( + route="ci_watch", + when="${{ result.timed_out }} == true", + ), + StepResultCondition( + route="done", + when="${{ result.success }} == true", + ), + ] + ), + ), + "done": RecipeStep(action="stop", message="done"), + } + recipe = _make_recipe(steps) + findings = run_semantic_rules(recipe) + loop_findings = [f for f in findings if f.rule == "ci-timed-out-self-loop-unguarded"] + assert len(loop_findings) >= 1, ( + "ci-timed-out-self-loop-unguarded must fire for unguarded timed_out self-loop" + ) + assert loop_findings[0].severity == Severity.ERROR + + +def test_timed_out_self_loop_with_guard_is_clean() -> None: + """wait_for_ci with timed_out→self and a check_loop_iteration guard on path → no finding.""" + steps = { + "ci_watch": RecipeStep( + tool="wait_for_ci", + with_args={"branch": "main", "timeout_seconds": 300}, + on_result=StepResultRoute( + conditions=[ + StepResultCondition( + route="check_loop", + when="${{ result.timed_out }} == true", + ), + StepResultCondition( + route="done", + when="${{ result.success }} == true", + ), + ] + ), + ), + "check_loop": RecipeStep( + tool="run_python", + with_args={"callable": "autoskillit.smoke_utils.check_loop_iteration"}, + on_result=StepResultRoute( + conditions=[ + StepResultCondition( + route="ci_watch", + when="${{ result.max_exceeded }} == false", + ), + StepResultCondition( + route="done", + when="${{ result.max_exceeded }} == true", + ), + ] + ), + ), + "done": RecipeStep(action="stop", message="done"), + } + recipe = _make_recipe(steps) + findings = run_semantic_rules(recipe) + loop_findings = [f for f in findings if f.rule == "ci-timed-out-self-loop-unguarded"] + assert len(loop_findings) == 0, ( + "ci-timed-out-self-loop-unguarded must not fire when guard step is on the loop path" + ) + + +def test_merge_prs_yaml_wait_for_conflict_ci_flags_timed_out_loop() -> None: + """merge-prs.yaml: wait_for_conflict_ci has guarded timed_out loop — no finding expected.""" + recipe = load_recipe(builtin_recipes_dir() / "merge-prs.yaml") + findings = run_semantic_rules(recipe) + loop_findings = [ + f + for f in findings + if f.rule == "ci-timed-out-self-loop-unguarded" and f.step_name == "wait_for_conflict_ci" + ] + assert len(loop_findings) == 0, ( + "wait_for_conflict_ci in merge-prs.yaml must not trigger ci-timed-out-self-loop-unguarded" + ) + + +def test_merge_prs_yaml_ci_watch_pr_flags_timed_out_loop() -> None: + """merge-prs.yaml: ci_watch_pr has guarded timed_out loop — no finding expected.""" + recipe = load_recipe(builtin_recipes_dir() / "merge-prs.yaml") + findings = run_semantic_rules(recipe) + loop_findings = [ + f + for f in findings + if f.rule == "ci-timed-out-self-loop-unguarded" and f.step_name == "ci_watch_pr" + ] + assert len(loop_findings) == 0, ( + "ci_watch_pr in merge-prs.yaml must not trigger ci-timed-out-self-loop-unguarded" + ) + + +@pytest.mark.parametrize( + "recipe_name", + ["implementation.yaml", "remediation.yaml", "implementation-groups.yaml"], +) +def test_ci_watch_in_recipes_flags_timed_out_loop(recipe_name: str) -> None: + """Recipes with ci_watch step that has guarded timed_out loop — no finding expected.""" + recipe = load_recipe(builtin_recipes_dir() / recipe_name) + findings = run_semantic_rules(recipe) + loop_findings = [ + f + for f in findings + if f.rule == "ci-timed-out-self-loop-unguarded" and f.step_name == "ci_watch" + ] + assert len(loop_findings) == 0, ( + f"{recipe_name}: ci_watch must not trigger ci-timed-out-self-loop-unguarded" + ) + + +# --------------------------------------------------------------------------- +# ci-conflict-path-missing-auto-trigger +# --------------------------------------------------------------------------- + + +def test_conflict_path_without_auto_trigger_is_error() -> None: + """wait_for_ci on conflict path without auto_trigger → ERROR.""" + steps = { + "check_mergeability": RecipeStep( + tool="run_skill", + with_args={"skill_command": "/autoskillit:check-pr-mergeable pr main"}, + on_result=StepResultRoute( + conditions=[ + StepResultCondition( + route="wait_for_conflict_ci", + when="${{ result.mergeable_status }} == CONFLICTING", + ), + StepResultCondition( + route="ci_watch", + when="${{ result.mergeable_status }} == MERGEABLE", + ), + ] + ), + ), + "wait_for_conflict_ci": RecipeStep( + tool="wait_for_ci", + with_args={"branch": "main", "timeout_seconds": 300}, + on_result=StepResultRoute( + conditions=[ + StepResultCondition( + route="done", + when="${{ result.success }} == true", + ), + StepResultCondition( + route="done", + when="${{ result.timed_out }} == true", + ), + ] + ), + ), + "ci_watch": RecipeStep( + tool="wait_for_ci", + with_args={"branch": "main", "timeout_seconds": 300, "auto_trigger": "true"}, + on_result=StepResultRoute( + conditions=[ + StepResultCondition( + route="done", + when="${{ result.success }} == true", + ), + ] + ), + ), + "done": RecipeStep(action="stop", message="done"), + } + recipe = _make_recipe(steps) + findings = run_semantic_rules(recipe) + trigger_findings = [f for f in findings if f.rule == "ci-conflict-path-missing-auto-trigger"] + assert len(trigger_findings) >= 1, ( + "ci-conflict-path-missing-auto-trigger must fire for conflict path without auto_trigger" + ) + assert trigger_findings[0].severity == Severity.ERROR + + +def test_conflict_path_with_auto_trigger_is_clean() -> None: + """wait_for_ci on conflict path with auto_trigger: true → no finding.""" + steps = { + "check_mergeability": RecipeStep( + tool="run_skill", + with_args={"skill_command": "/autoskillit:check-pr-mergeable pr main"}, + on_result=StepResultRoute( + conditions=[ + StepResultCondition( + route="wait_for_conflict_ci", + when="${{ result.mergeable_status }} == CONFLICTING", + ), + ] + ), + ), + "wait_for_conflict_ci": RecipeStep( + tool="wait_for_ci", + with_args={"branch": "main", "timeout_seconds": 300, "auto_trigger": "true"}, + on_result=StepResultRoute( + conditions=[ + StepResultCondition( + route="done", + when="${{ result.success }} == true", + ), + ] + ), + ), + "done": RecipeStep(action="stop", message="done"), + } + recipe = _make_recipe(steps) + findings = run_semantic_rules(recipe) + trigger_findings = [f for f in findings if f.rule == "ci-conflict-path-missing-auto-trigger"] + assert len(trigger_findings) == 0, ( + "ci-conflict-path-missing-auto-trigger must not fire when auto_trigger is set" + ) + + +def test_merge_prs_yaml_wait_for_conflict_ci_flags_missing_auto_trigger() -> None: + """merge-prs.yaml: wait_for_conflict_ci has auto_trigger: true — no finding expected.""" + recipe = load_recipe(builtin_recipes_dir() / "merge-prs.yaml") + findings = run_semantic_rules(recipe) + trigger_findings = [ + f + for f in findings + if f.rule == "ci-conflict-path-missing-auto-trigger" + and f.step_name == "wait_for_conflict_ci" + ] + assert len(trigger_findings) == 0, ( + "wait_for_conflict_ci in merge-prs.yaml must not trigger" + " ci-conflict-path-missing-auto-trigger" + ) + + +def test_merge_prs_yaml_ci_watch_pr_flags_missing_auto_trigger() -> None: + """merge-prs.yaml: ci_watch_pr has auto_trigger: true — no finding expected.""" + recipe = load_recipe(builtin_recipes_dir() / "merge-prs.yaml") + findings = run_semantic_rules(recipe) + trigger_findings = [ + f + for f in findings + if f.rule == "ci-conflict-path-missing-auto-trigger" and f.step_name == "ci_watch_pr" + ] + assert len(trigger_findings) == 0, ( + "ci_watch_pr in merge-prs.yaml must not trigger ci-conflict-path-missing-auto-trigger" + ) + + +def test_non_conflict_path_ci_watch_is_not_flagged() -> None: + """wait_for_ci not on a conflict path must not trigger the rule.""" + steps = { + "ci_watch": RecipeStep( + tool="wait_for_ci", + with_args={"branch": "main", "timeout_seconds": 300}, + on_result=StepResultRoute( + conditions=[ + StepResultCondition( + route="done", + when="${{ result.success }} == true", + ), + ] + ), + ), + "done": RecipeStep(action="stop", message="done"), + } + recipe = _make_recipe(steps) + findings = run_semantic_rules(recipe) + trigger_findings = [f for f in findings if f.rule == "ci-conflict-path-missing-auto-trigger"] + assert len(trigger_findings) == 0, ( + "Non-conflict-path wait_for_ci must not trigger ci-conflict-path-missing-auto-trigger" + ) diff --git a/tests/recipe/test_rules_clone.py b/tests/recipe/test_rules_clone.py index 1f5f1c581c..3e7697f0fd 100644 --- a/tests/recipe/test_rules_clone.py +++ b/tests/recipe/test_rules_clone.py @@ -11,6 +11,8 @@ from autoskillit.recipe.registry import run_semantic_rules from autoskillit.recipe.schema import Recipe, RecipeStep +pytestmark = [pytest.mark.layer("recipe"), pytest.mark.small] + _BUNDLED_RECIPE_YAMLS = list(builtin_recipes_dir().rglob("*.yaml")) assert _BUNDLED_RECIPE_YAMLS, ( "builtin_recipes_dir() returned no YAMLs — parametrize would silently vacuous-pass" diff --git a/tests/recipe/test_rules_cmd.py b/tests/recipe/test_rules_cmd.py index 9e94ff249b..d571331e93 100644 --- a/tests/recipe/test_rules_cmd.py +++ b/tests/recipe/test_rules_cmd.py @@ -2,9 +2,13 @@ from __future__ import annotations +import pytest + from autoskillit.recipe.validator import run_semantic_rules from tests.recipe.conftest import _make_workflow +pytestmark = [pytest.mark.layer("recipe"), pytest.mark.small] + def _make_recipe(steps: dict) -> object: """Build a minimal Recipe from a dict of step dicts, with a terminal END step.""" @@ -132,3 +136,100 @@ def test_find_rediscovery_no_warning_without_heuristic(): ) findings = run_semantic_rules(recipe) assert all(f.rule != "run-cmd-find-rediscovery" for f in findings) + + +def test_hardcoded_origin_in_run_cmd_fires_on_fetch_origin(): + """run_cmd with 'git fetch origin main' triggers the warning.""" + recipe = _make_recipe( + { + "step_a": { + "tool": "run_cmd", + "with": {"cmd": "git fetch origin main && git rebase origin/main"}, + "on_success": "END", + } + } + ) + findings = run_semantic_rules(recipe) + codes = [f.rule for f in findings] + assert "hardcoded-origin-in-run-cmd" in codes + finding = next(f for f in findings if f.rule == "hardcoded-origin-in-run-cmd") + assert "step_a" in finding.step_name + assert "origin" in finding.message + + +def test_hardcoded_origin_in_run_cmd_clean_with_remote_var(): + """run_cmd using $REMOTE instead of literal 'origin' does not trigger.""" + recipe = _make_recipe( + { + "step_a": { + "tool": "run_cmd", + "with": { + "cmd": ( + "REMOTE=$(git remote get-url upstream >/dev/null 2>&1 " + "&& echo upstream || echo origin)\n" + "git fetch $REMOTE main && git rebase $REMOTE/main" + ) + }, + "on_success": "END", + } + } + ) + findings = run_semantic_rules(recipe) + assert all(f.rule != "hardcoded-origin-in-run-cmd" for f in findings) + + +def test_hardcoded_origin_in_run_cmd_suppressed_by_set_url_step(): + """Step containing 'git remote set-url origin' is not flagged; other steps still are.""" + recipe = _make_recipe( + { + "step_setup": { + "tool": "run_cmd", + "with": {"cmd": "git remote set-url origin https://github.com/org/repo.git"}, + "on_success": "step_fetch", + }, + "step_fetch": { + "tool": "run_cmd", + "with": {"cmd": "git fetch origin main"}, + "on_success": "END", + }, + } + ) + findings = run_semantic_rules(recipe) + violations = [f for f in findings if f.rule == "hardcoded-origin-in-run-cmd"] + step_names = [v.step_name for v in violations] + assert "step_setup" not in step_names, "step with set-url origin must not be flagged" + assert "step_fetch" in step_names, ( + "step without set-url using hardcoded origin must be flagged" + ) + + +def test_hardcoded_origin_in_run_cmd_fires_on_push_origin(): + """run_cmd with 'git push origin main' triggers the warning (push is in the verb set).""" + recipe = _make_recipe( + { + "step_a": { + "tool": "run_cmd", + "with": {"cmd": "git push origin main"}, + "on_success": "END", + } + } + ) + findings = run_semantic_rules(recipe) + codes = [f.rule for f in findings] + assert "hardcoded-origin-in-run-cmd" in codes + + +def test_hardcoded_origin_in_run_cmd_fires_on_merge_base_origin(): + """run_cmd with 'git merge-base origin/main HEAD' triggers the warning.""" + recipe = _make_recipe( + { + "step_a": { + "tool": "run_cmd", + "with": {"cmd": "git merge-base origin/main HEAD"}, + "on_success": "END", + } + } + ) + findings = run_semantic_rules(recipe) + codes = [f.rule for f in findings] + assert "hardcoded-origin-in-run-cmd" in codes diff --git a/tests/recipe/test_rules_conditional_push.py b/tests/recipe/test_rules_conditional_push.py index deb2f35e41..06a91dfa1a 100644 --- a/tests/recipe/test_rules_conditional_push.py +++ b/tests/recipe/test_rules_conditional_push.py @@ -7,7 +7,9 @@ from __future__ import annotations -import autoskillit.recipe.rules_fixing as _rf +import pytest + +import autoskillit.recipe.rules.rules_fixing as _rf from autoskillit.core.types import Severity from autoskillit.recipe.schema import ( Recipe, @@ -17,6 +19,8 @@ ) from autoskillit.recipe.validator import run_semantic_rules +pytestmark = [pytest.mark.layer("recipe"), pytest.mark.small] + _RULE = "conditional-skill-ungated-push" # Minimal manifest with a fake conditional-write skill and a declared verdict output. diff --git a/tests/recipe/test_rules_contracts.py b/tests/recipe/test_rules_contracts.py index bab4b6d553..bc9b72dd7f 100644 --- a/tests/recipe/test_rules_contracts.py +++ b/tests/recipe/test_rules_contracts.py @@ -2,16 +2,21 @@ from __future__ import annotations +from pathlib import Path from unittest.mock import patch -import autoskillit.recipe.rules_contracts as _rc +import pytest + +import autoskillit.recipe.rules.rules_contracts as _rc from autoskillit.core.paths import pkg_root from autoskillit.core.types import Severity -from autoskillit.recipe.contracts import SkillContract +from autoskillit.recipe.contracts import ResultFieldSpec, SkillContract from autoskillit.recipe.io import load_recipe from autoskillit.recipe.registry import run_semantic_rules from autoskillit.recipe.schema import Recipe, RecipeStep +pytestmark = [pytest.mark.layer("recipe"), pytest.mark.small] + def test_rule_flags_skills_with_empty_output_patterns() -> None: """The missing-output-patterns rule exists and emits no warnings on bundled recipes.""" @@ -133,7 +138,7 @@ def test_write_behavior_invalid_value_flagged() -> None: recipe = _make_recipe_with_skill("/autoskillit:make-plan task") contract = _make_contract(write_behavior="invalid") with patch( - "autoskillit.recipe.rules_contracts.get_skill_contract", + "autoskillit.recipe.rules.rules_contracts.get_skill_contract", return_value=contract, ): findings = run_semantic_rules(recipe) @@ -146,7 +151,7 @@ def test_conditional_without_patterns_flagged() -> None: recipe = _make_recipe_with_skill("/autoskillit:make-plan task") contract = _make_contract(write_behavior="conditional", write_expected_when=[]) with patch( - "autoskillit.recipe.rules_contracts.get_skill_contract", + "autoskillit.recipe.rules.rules_contracts.get_skill_contract", return_value=contract, ): findings = run_semantic_rules(recipe) @@ -159,7 +164,7 @@ def test_always_with_patterns_flagged() -> None: recipe = _make_recipe_with_skill("/autoskillit:make-plan task") contract = _make_contract(write_behavior="always", write_expected_when=["pattern"]) with patch( - "autoskillit.recipe.rules_contracts.get_skill_contract", + "autoskillit.recipe.rules.rules_contracts.get_skill_contract", return_value=contract, ): findings = run_semantic_rules(recipe) @@ -172,7 +177,7 @@ def test_invalid_regex_in_patterns_flagged() -> None: recipe = _make_recipe_with_skill("/autoskillit:make-plan task") contract = _make_contract(write_behavior="conditional", write_expected_when=["[invalid"]) with patch( - "autoskillit.recipe.rules_contracts.get_skill_contract", + "autoskillit.recipe.rules.rules_contracts.get_skill_contract", return_value=contract, ): findings = run_semantic_rules(recipe) @@ -208,7 +213,7 @@ def test_always_write_skill_with_documented_no_write_exit_flagged() -> None: patterns (e.g. 'may be 0' from resolve-failures, graceful degradation from resolve-review). An empty or incomplete set would silently miss all bugs. """ - from autoskillit.recipe.rules_contracts import _ALWAYS_WITH_NO_WRITE_EXIT_PHRASES + from autoskillit.recipe.rules.rules_contracts import _ALWAYS_WITH_NO_WRITE_EXIT_PHRASES # Verify the phrase set is populated (not empty — would miss all bugs) assert len(_ALWAYS_WITH_NO_WRITE_EXIT_PHRASES) > 0 @@ -219,3 +224,380 @@ def test_always_write_skill_with_documented_no_write_exit_flagged() -> None: assert any( "graceful" in p.lower() or "skip" in p.lower() for p in _ALWAYS_WITH_NO_WRITE_EXIT_PHRASES ), "Must detect graceful degradation phrases — the exact pattern in resolve-review Step 1" + + +def test_template_args_do_not_skip_rule(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + """Skills with template ARGUMENTS (not template names) must be checked. + + '${{' in skill_command arguments must NOT bypass contract rules. + Only '${{' in the skill NAME should bypass (handled by resolve_skill_name). + """ + manifest = { + "version": "0.1.0", + "skills": { + "prepare-pr": { + "inputs": [], + "outputs": [{"name": "prep_path", "type": "file_path"}], + "expected_output_patterns": ["prep_path\\s*=\\s*/.+"], + "pattern_examples": ["prep_path = /tmp/prep.md"], + "write_behavior": "always", + "write_expected_when": [], + } + }, + } + monkeypatch.setattr(_rc, "load_bundled_manifest", lambda: manifest) + + recipe = _make_recipe_with_skill( + "/autoskillit:prepare-pr ${{ context.plan_path }} ${{ inputs.branch }}" + ) + + skill_dir = tmp_path / "skills_extended" / "prepare-pr" + skill_dir.mkdir(parents=True) + (skill_dir / "SKILL.md").write_text( + "If context exhausted, emit prep_path = (empty) — graceful degradation." + ) + monkeypatch.setattr(_rc, "pkg_root", lambda: tmp_path) + + findings = run_semantic_rules(recipe) + exit_findings = [f for f in findings if f.rule == "always-has-no-write-exit"] + assert len(exit_findings) >= 1, ( + "always-has-no-write-exit must fire on prepare-pr even when " + "skill_command contains template arguments" + ) + + +def test_unreadable_skill_md_emits_warning_finding( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """OSError reading SKILL.md must emit a WARNING finding, not a silent skip.""" + skill_dir = tmp_path / "skills" / "fake-skill" + skill_dir.mkdir(parents=True) + (skill_dir / "SKILL.md").write_text("placeholder") + + monkeypatch.setattr(_rc, "pkg_root", lambda: tmp_path) + + recipe = _make_recipe_with_skill("/autoskillit:fake-skill") + contract = _make_contract(write_behavior="always") + original_read_text = Path.read_text + + def fail_read_text(self: Path, *args: object, **kwargs: object) -> str: + if self.name == "SKILL.md": + raise OSError("permission denied") + return original_read_text(self, *args, **kwargs) # type: ignore[arg-type] + + with ( + patch( + "autoskillit.recipe.rules.rules_contracts.get_skill_contract", return_value=contract + ), + patch.object(Path, "read_text", fail_read_text), + ): + findings = run_semantic_rules(recipe) + + relevant = [f for f in findings if f.rule == "always-has-no-write-exit"] + assert len(relevant) == 1 + assert relevant[0].severity == Severity.WARNING + assert relevant[0].rule == "always-has-no-write-exit" + assert relevant[0].step_name == "test_step" + assert "fake-skill" in relevant[0].message + + +# --------------------------------------------------------------------------- +# result-field-drift tests +# --------------------------------------------------------------------------- + + +class TestResultFieldDriftRule: + def test_matching_result_fields_no_finding(self) -> None: + """result_fields that exactly match PHASE_REQUIRED_KEYS must not fire.""" + recipe = _make_recipe_with_skill( + "/autoskillit:planner-generate-phases {{AUTOSKILLIT_TEMP}}/planner/analysis.json" + ) + contract = SkillContract( + inputs=[], + outputs=[], + result_fields=[ + ResultFieldSpec(name="id", type="str", required=True), + ResultFieldSpec(name="name", type="str", required=True), + ResultFieldSpec(name="ordering", type="int", required=True), + ], + ) + with patch( + "autoskillit.recipe.rules.rules_contracts.get_skill_contract", + return_value=contract, + ): + findings = run_semantic_rules(recipe) + drift = [f for f in findings if f.rule == "result-field-drift"] + assert drift == [] + + def test_extra_required_field_fires_error(self) -> None: + """A required field in result_fields that is NOT in the TypedDict fires ERROR.""" + recipe = _make_recipe_with_skill( + "/autoskillit:planner-generate-phases {{AUTOSKILLIT_TEMP}}/planner/analysis.json" + ) + contract = SkillContract( + inputs=[], + outputs=[], + result_fields=[ + ResultFieldSpec(name="id", type="str", required=True), + ResultFieldSpec(name="name", type="str", required=True), + ResultFieldSpec(name="ordering", type="int", required=True), + ResultFieldSpec(name="phantom_field", type="str", required=True), + ], + ) + with patch( + "autoskillit.recipe.rules.rules_contracts.get_skill_contract", + return_value=contract, + ): + findings = run_semantic_rules(recipe) + drift = [f for f in findings if f.rule == "result-field-drift"] + assert len(drift) == 1 + assert drift[0].severity == Severity.ERROR + assert "phantom_field" in drift[0].message + assert "extra in contract" in drift[0].message + + def test_missing_required_field_fires_error(self) -> None: + """A TypedDict required key absent from result_fields fires ERROR.""" + recipe = _make_recipe_with_skill( + "/autoskillit:planner-generate-phases {{AUTOSKILLIT_TEMP}}/planner/analysis.json" + ) + contract = SkillContract( + inputs=[], + outputs=[], + result_fields=[ + ResultFieldSpec(name="id", type="str", required=True), + # 'name' and 'ordering' are missing + ], + ) + with patch( + "autoskillit.recipe.rules.rules_contracts.get_skill_contract", + return_value=contract, + ): + findings = run_semantic_rules(recipe) + drift = [f for f in findings if f.rule == "result-field-drift"] + assert len(drift) == 1 + assert drift[0].severity == Severity.ERROR + assert "missing from contract" in drift[0].message + + def test_optional_extra_field_ignored(self) -> None: + """A field with required=False in result_fields that is not in the TypedDict is ignored.""" + recipe = _make_recipe_with_skill( + "/autoskillit:planner-generate-phases {{AUTOSKILLIT_TEMP}}/planner/analysis.json" + ) + contract = SkillContract( + inputs=[], + outputs=[], + result_fields=[ + ResultFieldSpec(name="id", type="str", required=True), + ResultFieldSpec(name="name", type="str", required=True), + ResultFieldSpec(name="ordering", type="int", required=True), + ResultFieldSpec(name="optional_extra", type="str", required=False), + ], + ) + with patch( + "autoskillit.recipe.rules.rules_contracts.get_skill_contract", + return_value=contract, + ): + findings = run_semantic_rules(recipe) + drift = [f for f in findings if f.rule == "result-field-drift"] + assert drift == [] + + def test_skill_without_result_fields_in_contract_skipped(self) -> None: + """Skills not in _RESULT_FIELD_DRIFT_SKILLS are ignored by the rule.""" + recipe = _make_recipe_with_skill("/autoskillit:implement-worktree-no-merge") + findings = run_semantic_rules(recipe) + drift = [f for f in findings if f.rule == "result-field-drift"] + assert drift == [] + + def test_planner_recipe_has_no_result_field_drift(self) -> None: + """The bundled planner recipe must not trigger result-field-drift.""" + recipe = load_recipe(pkg_root() / "recipes" / "planner.yaml") + findings = run_semantic_rules(recipe) + drift = [f for f in findings if f.rule == "result-field-drift"] + assert drift == [], f"result-field-drift fired on planner recipe: {drift}" + + +# --------------------------------------------------------------------------- +# example-covers-all-allowed-values rule tests +# --------------------------------------------------------------------------- + + +def test_example_covers_all_allowed_values_fires_on_missing_example( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """example-covers-all-allowed-values fires ERROR when an allowed_value has no example.""" + manifest = { + "version": "0.1.0", + "skills": { + "test-skill": { + "inputs": [], + "outputs": [ + {"name": "verdict", "type": "string", "allowed_values": ["go", "stop"]} + ], + "expected_output_patterns": ["verdict\\s*=\\s*(go|stop)"], + "pattern_examples": ["verdict = go\n%%ORDER_UP%%"], # only "go", missing "stop" + } + }, + } + monkeypatch.setattr(_rc, "load_bundled_manifest", lambda: manifest) + recipe = _make_recipe_with_skill("/autoskillit:test-skill") + findings = run_semantic_rules(recipe) + rule_findings = [f for f in findings if f.rule == "example-covers-all-allowed-values"] + assert len(rule_findings) >= 1, ( + "example-covers-all-allowed-values must fire when allowed_value 'stop' " + "has no corresponding pattern_examples entry" + ) + assert rule_findings[0].severity == Severity.ERROR + assert "stop" in rule_findings[0].message + + +def test_example_covers_all_allowed_values_passes_when_complete( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """example-covers-all-allowed-values must not fire when all allowed_values have examples.""" + manifest = { + "version": "0.1.0", + "skills": { + "test-skill": { + "inputs": [], + "outputs": [ + {"name": "verdict", "type": "string", "allowed_values": ["go", "stop"]} + ], + "expected_output_patterns": ["verdict\\s*=\\s*(go|stop)"], + "pattern_examples": [ + "verdict = go\n%%ORDER_UP%%", + "verdict = stop\n%%ORDER_UP%%", + ], + } + }, + } + monkeypatch.setattr(_rc, "load_bundled_manifest", lambda: manifest) + recipe = _make_recipe_with_skill("/autoskillit:test-skill") + findings = run_semantic_rules(recipe) + rule_findings = [f for f in findings if f.rule == "example-covers-all-allowed-values"] + assert rule_findings == [], ( + "example-covers-all-allowed-values must not fire when both 'go' and 'stop' " + "appear in pattern_examples" + ) + + +# --------------------------------------------------------------------------- +# all-examples-match-all-patterns rule tests +# --------------------------------------------------------------------------- + + +def test_all_examples_match_all_patterns_fires_on_conditional_pattern( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """all-examples-match-all-patterns fires ERROR when an example fails a pattern.""" + manifest = { + "version": "0.1.0", + "skills": { + "test-skill": { + "inputs": [], + "outputs": [{"name": "verdict", "type": "string"}], + "expected_output_patterns": [ + "verdict\\s*=\\s*(go|stop)", + "%%TAG%%", + ], + "pattern_examples": [ + "verdict = go\n%%TAG%%", # matches both patterns + "verdict = stop", # missing %%TAG%% — fails second pattern + ], + } + }, + } + monkeypatch.setattr(_rc, "load_bundled_manifest", lambda: manifest) + recipe = _make_recipe_with_skill("/autoskillit:test-skill") + findings = run_semantic_rules(recipe) + rule_findings = [f for f in findings if f.rule == "all-examples-match-all-patterns"] + assert len(rule_findings) >= 1, ( + "all-examples-match-all-patterns must fire when 'verdict = stop' example " + "does not match the '%%TAG%%' pattern" + ) + assert rule_findings[0].severity == Severity.ERROR + assert "%%TAG%%" in rule_findings[0].message or "verdict = stop" in rule_findings[0].message + + +def test_all_examples_match_all_patterns_passes_when_all_match( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """all-examples-match-all-patterns must not fire when all examples match all patterns.""" + manifest = { + "version": "0.1.0", + "skills": { + "test-skill": { + "inputs": [], + "outputs": [{"name": "verdict", "type": "string"}], + "expected_output_patterns": [ + "verdict\\s*=\\s*(go|stop)", + "%%TAG%%", + ], + "pattern_examples": [ + "verdict = go\n%%TAG%%", + "verdict = stop\n%%TAG%%", + ], + } + }, + } + monkeypatch.setattr(_rc, "load_bundled_manifest", lambda: manifest) + recipe = _make_recipe_with_skill("/autoskillit:test-skill") + findings = run_semantic_rules(recipe) + rule_findings = [f for f in findings if f.rule == "all-examples-match-all-patterns"] + assert rule_findings == [], ( + "all-examples-match-all-patterns must not fire when all examples match all patterns" + ) + + +def test_all_examples_match_all_patterns_skips_without_examples( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """all-examples-match-all-patterns must not fire when there are no pattern_examples.""" + manifest = { + "version": "0.1.0", + "skills": { + "test-skill": { + "inputs": [], + "outputs": [{"name": "verdict", "type": "string"}], + "expected_output_patterns": ["verdict\\s*=\\s*(go|stop)"], + # No pattern_examples + } + }, + } + monkeypatch.setattr(_rc, "load_bundled_manifest", lambda: manifest) + recipe = _make_recipe_with_skill("/autoskillit:test-skill") + findings = run_semantic_rules(recipe) + rule_findings = [f for f in findings if f.rule == "all-examples-match-all-patterns"] + assert rule_findings == [], ( + "all-examples-match-all-patterns must not fire without pattern_examples " + "(defer to missing-pattern-examples rule)" + ) + + +# --------------------------------------------------------------------------- +# Integration test: all bundled recipes pass the new contract immunity rules +# --------------------------------------------------------------------------- + + +def test_bundled_recipes_pass_all_new_contract_rules() -> None: + """All bundled recipes must pass the three new contract immunity rules with zero ERRORs.""" + new_rules = { + "example-covers-all-allowed-values", + "all-examples-match-all-patterns", + "on-result-values-in-allowed-values", + } + recipes_dir = pkg_root() / "recipes" + recipe_files = sorted(recipes_dir.glob("*.yaml")) + assert recipe_files, "No bundled recipes found" + + violations: list[str] = [] + for recipe_path in recipe_files: + recipe = load_recipe(recipe_path) + findings = run_semantic_rules(recipe) + for f in findings: + if f.rule in new_rules and f.severity == Severity.ERROR: + violations.append(f"{recipe_path.name}:{f.step_name}: [{f.rule}] {f.message}") + + assert not violations, ( + "New contract immunity rules fired on bundled recipes " + "(ensure Part A contract fixes are applied):\n" + "\n".join(violations) + ) diff --git a/tests/recipe/test_rules_dataflow_capture.py b/tests/recipe/test_rules_dataflow_capture.py new file mode 100644 index 0000000000..c2ca9f75c4 --- /dev/null +++ b/tests/recipe/test_rules_dataflow_capture.py @@ -0,0 +1,355 @@ +"""Tests for data-flow semantic rules — capture output coverage, dead output, implicit handoff.""" + +from __future__ import annotations + +import textwrap + +import pytest +import yaml + +from autoskillit.core.types import Severity +from autoskillit.recipe.io import _parse_recipe +from autoskillit.recipe.validator import run_semantic_rules +from tests.recipe.conftest import _build_merge_worktree_recipe, _make_workflow + +pytestmark = [pytest.mark.layer("recipe"), pytest.mark.small] + + +# --------------------------------------------------------------------------- +# TestCaptureOutputCoverageRule +# --------------------------------------------------------------------------- + + +class TestCaptureOutputCoverageRule: + def test_capture_declared_output_key_no_warning(self) -> None: + """A capture that references a key declared in the skill's outputs contract + must not produce an undeclared-capture-key warning.""" + recipe_yaml = textwrap.dedent("""\ + name: capture-valid + description: test + steps: + implement: + tool: run_skill + with: + skill_command: /autoskillit:implement-worktree-no-merge ${{ inputs.plan }} + capture: + worktree_path: "${{ result.worktree_path }}" + on_success: done + on_failure: done + done: + action: stop + message: Done + """) + recipe = _parse_recipe(yaml.safe_load(recipe_yaml)) + findings = run_semantic_rules(recipe) + undeclared = [f for f in findings if f.rule == "undeclared-capture-key"] + assert undeclared == [] + + def test_capture_undeclared_key_emits_error(self) -> None: + """A capture that references a key NOT listed in the skill's outputs contract + must produce a Severity.ERROR finding with rule 'undeclared-capture-key'.""" + recipe_yaml = textwrap.dedent("""\ + name: capture-invalid-key + description: test + steps: + implement: + tool: run_skill + with: + skill_command: /autoskillit:implement-worktree-no-merge ${{ inputs.plan }} + capture: + nonexistent_output: "${{ result.nonexistent_output }}" + on_success: done + on_failure: done + done: + action: stop + message: Done + """) + recipe = _parse_recipe(yaml.safe_load(recipe_yaml)) + findings = run_semantic_rules(recipe) + undeclared = [f for f in findings if f.rule == "undeclared-capture-key"] + assert len(undeclared) == 1 + assert undeclared[0].severity == Severity.ERROR + assert "nonexistent_output" in undeclared[0].message + assert "implement-worktree-no-merge" in undeclared[0].message + + def test_capture_from_skill_with_no_contract_emits_error(self) -> None: + """A capture step whose skill has no entry in skill_contracts.yaml at all + must produce a Severity.ERROR finding with rule 'undeclared-capture-key'.""" + recipe_yaml = textwrap.dedent("""\ + name: capture-unknown-skill + description: test + steps: + run_custom: + tool: run_skill + with: + skill_command: /autoskillit:not-a-real-skill some_arg + capture: + result_key: "${{ result.some_key }}" + on_success: done + on_failure: done + done: + action: stop + message: Done + """) + recipe = _parse_recipe(yaml.safe_load(recipe_yaml)) + findings = run_semantic_rules(recipe) + undeclared = [f for f in findings if f.rule == "undeclared-capture-key"] + assert len(undeclared) == 1 + assert undeclared[0].severity == Severity.ERROR + assert "not-a-real-skill" in undeclared[0].message + assert "no outputs contract entry" in undeclared[0].message + + def test_capture_key_from_empty_outputs_skill_emits_error(self) -> None: + """audit-friction has outputs: [] — any capture key from it is undeclared.""" + recipe_yaml = textwrap.dedent("""\ + name: capture-empty-outputs + description: test + steps: + audit: + tool: run_skill + with: + skill_command: /autoskillit:audit-friction + capture: + report_path: "${{ result.report_path }}" + on_success: done + on_failure: done + done: + action: stop + message: Done + """) + recipe = _parse_recipe(yaml.safe_load(recipe_yaml)) + findings = run_semantic_rules(recipe) + undeclared = [f for f in findings if f.rule == "undeclared-capture-key"] + assert len(undeclared) == 1 + assert undeclared[0].severity == Severity.ERROR + assert "report_path" in undeclared[0].message + assert "audit-friction" in undeclared[0].message + + def test_undeclared_capture_key_blocks_validity(self) -> None: + """undeclared-capture-key at ERROR severity must cause compute_recipe_validity=False.""" + from autoskillit.recipe.registry import compute_recipe_validity + + recipe_yaml = textwrap.dedent("""\ + name: capture-invalid-blocks-validity + description: test + steps: + implement: + tool: run_skill + with: + skill_command: /autoskillit:implement-worktree-no-merge ${{ inputs.plan }} + capture: + ghost_key: "${{ result.ghost_key }}" + on_success: done + on_failure: done + done: + action: stop + message: Done + """) + recipe = _parse_recipe(yaml.safe_load(recipe_yaml)) + findings = run_semantic_rules(recipe) + assert not compute_recipe_validity( + errors=[], + semantic_findings=findings, + contract_findings=[], + ) + + +# --------------------------------------------------------------------------- +# TestDeadOutputRule +# --------------------------------------------------------------------------- + + +class TestDeadOutputRule: + def test_do1_dead_output_rule_in_registry(self) -> None: + """T_DO1: dead-output is in _RULE_REGISTRY.""" + from autoskillit.recipe.validator import _RULE_REGISTRY + + rule_names = [r.name for r in _RULE_REGISTRY] + assert "dead-output" in rule_names + + def test_do2_fires_error_for_unconsumed_capture(self) -> None: + """T_DO2: dead-output fires ERROR when a captured key is never consumed downstream.""" + steps = { + "plan": { + "tool": "run_skill", + "with": {"skill_command": "/autoskillit:make-plan do the task"}, + "capture": {"plan_path": "${{ result.plan_path }}"}, + "on_success": "done", + }, + "done": {"action": "stop", "message": "Done."}, + } + recipe = _make_workflow(steps) + findings = run_semantic_rules(recipe) + dead = [f for f in findings if f.rule == "dead-output"] + assert len(dead) >= 1 + assert any(f.severity == Severity.ERROR and f.step_name == "plan" for f in dead) + + def test_do3_does_not_fire_for_on_result_self_consumption(self) -> None: + """T_DO3: dead-output does NOT fire when on_result.field equals the captured key.""" + steps = { + "audit_impl": { + "tool": "run_skill", + "with": { + "skill_command": "/autoskillit:audit-impl plan.md myref main", + }, + "capture": {"verdict": "${{ result.verdict }}"}, + "on_result": { + "field": "verdict", + "routes": {"GO": "done", "NO GO": "done"}, + }, + }, + "done": {"action": "stop", "message": "Done."}, + } + recipe = _make_workflow(steps) + findings = run_semantic_rules(recipe) + dead = [f for f in findings if f.rule == "dead-output" and f.step_name == "audit_impl"] + assert dead == [] + + def test_do4_cleanup_succeeded_from_merge_worktree_not_dead_output(self) -> None: + """T_DO4: dead-output does NOT fire for cleanup_succeeded captured from merge_worktree. + + cleanup_succeeded is an observability capture required by merge-cleanup-uncaptured. + It has no downstream consumer by design — the exemption prevents the two rules + from conflicting. + """ + recipe = _build_merge_worktree_recipe( + capture={"cleanup_succeeded": "${{ result.cleanup_succeeded }}"} + ) + findings = run_semantic_rules(recipe) + dead = [f for f in findings if f.rule == "dead-output" and f.step_name == "merge"] + assert dead == [] + + def test_do5_predicate_condition_consumes_captured_var(self) -> None: + """T_DO5: dead-output does NOT fire when a predicate on_result condition + references the captured variable via context.X in its when clause.""" + steps = { + "review_pr": { + "tool": "run_skill", + "with": {"skill_command": "/autoskillit:review-pr feature main"}, + "capture": {"verdict": "${{ result.verdict }}"}, + "on_result": [ + { + "when": "${{ result.verdict }} == changes_requested", + "route": "resolve_review", + }, + {"when": "true", "route": "done"}, + ], + "on_failure": "resolve_review", + }, + "resolve_review": { + "tool": "run_skill", + "with": { + "skill_command": "/autoskillit:resolve-failures worktree plan main", + }, + "on_success": "done", + "on_failure": "done", + }, + "done": {"action": "stop", "message": "Done."}, + } + recipe = _make_workflow(steps) + findings = run_semantic_rules(recipe) + dead = [f for f in findings if f.rule == "dead-output" and f.step_name == "review_pr"] + assert dead == [] + + +# --------------------------------------------------------------------------- +# TestImplicitHandoffRule +# --------------------------------------------------------------------------- + + +class TestImplicitHandoffRule: + def test_ih1_implicit_handoff_rule_in_registry(self) -> None: + """T_IH1: implicit-handoff is in _RULE_REGISTRY.""" + from autoskillit.recipe.validator import _RULE_REGISTRY + + rule_names = [r.name for r in _RULE_REGISTRY] + assert "implicit-handoff" in rule_names + + def test_ih2_fires_error_for_skill_with_outputs_and_no_capture(self) -> None: + """T_IH2: implicit-handoff fires ERROR when skill has outputs but step has no capture.""" + steps = { + "plan": { + "tool": "run_skill", + "with": {"skill_command": "/autoskillit:make-plan do the task"}, + "on_success": "done", + }, + "done": {"action": "stop", "message": "Done."}, + } + recipe = _make_workflow(steps) + findings = run_semantic_rules(recipe) + ih = [f for f in findings if f.rule == "implicit-handoff"] + assert len(ih) >= 1 + assert any(f.severity == Severity.ERROR and f.step_name == "plan" for f in ih) + + def test_ih3_does_not_fire_when_capture_block_present(self) -> None: + """T_IH3: implicit-handoff does NOT fire when the step has a capture: block.""" + steps = { + "plan": { + "tool": "run_skill", + "with": {"skill_command": "/autoskillit:make-plan do the task"}, + "capture": {"plan_path": "${{ result.plan_path }}"}, + "on_success": "done", + }, + "done": {"action": "stop", "message": "Done."}, + } + recipe = _make_workflow(steps) + findings = run_semantic_rules(recipe) + ih = [f for f in findings if f.rule == "implicit-handoff" and f.step_name == "plan"] + assert ih == [] + + def test_ih4_does_not_fire_for_skill_with_empty_outputs(self) -> None: + """T_IH4: implicit-handoff does NOT fire for a skill with outputs: []. + + dry-walkthrough has outputs: [] — no fields to capture, so no finding. + """ + steps = { + "walkthrough": { + "tool": "run_skill", + "with": {"skill_command": "/autoskillit:dry-walkthrough plan"}, + "on_success": "done", + }, + "done": {"action": "stop", "message": "Done."}, + } + recipe = _make_workflow(steps) + findings = run_semantic_rules(recipe) + ih = [f for f in findings if f.rule == "implicit-handoff" and f.step_name == "walkthrough"] + assert ih == [] + + def test_ih5_does_not_fire_for_unknown_skill(self) -> None: + """T_IH5: implicit-handoff does NOT fire for a skill with no contract entry.""" + steps = { + "step": { + "tool": "run_skill", + "with": {"skill_command": "/autoskillit:not-a-real-skill something"}, + "on_success": "done", + }, + "done": {"action": "stop", "message": "Done."}, + } + recipe = _make_workflow(steps) + findings = run_semantic_rules(recipe) + ih = [f for f in findings if f.rule == "implicit-handoff" and f.step_name == "step"] + assert ih == [] + + def test_implicit_handoff_fires_for_investigate_without_capture(self) -> None: + """T_IH_1g: implicit-handoff fires for investigate step without capture block. + + After investigate.outputs gains investigation_path in skill_contracts.yaml, + a recipe step invoking /autoskillit:investigate without a capture block + triggers the implicit-handoff rule. Uses the real bundled manifest. + """ + steps = { + "investigate": { + "tool": "run_skill", + "with": {"skill_command": "/autoskillit:investigate the bug"}, + "on_success": "done", + }, + "done": {"action": "stop", "message": "Done."}, + } + recipe = _make_workflow(steps) + findings = run_semantic_rules(recipe) + ih = [f for f in findings if f.rule == "implicit-handoff" and f.step_name == "investigate"] + assert len(ih) >= 1, ( + "implicit-handoff must fire for an investigate step with no capture block " + "once investigate declares investigation_path in skill_contracts.yaml" + ) + assert any(f.severity == Severity.ERROR for f in ih) diff --git a/tests/recipe/test_rules_dataflow_handoff.py b/tests/recipe/test_rules_dataflow_handoff.py new file mode 100644 index 0000000000..9621185900 --- /dev/null +++ b/tests/recipe/test_rules_dataflow_handoff.py @@ -0,0 +1,272 @@ +"""Tests for data-flow semantic rules — handoff consumer, callable contracts, python capture.""" + +from __future__ import annotations + +import textwrap + +import pytest +import yaml + +from autoskillit.core.types import Severity +from autoskillit.recipe.io import _parse_recipe +from autoskillit.recipe.validator import run_semantic_rules +from tests.recipe.conftest import _make_workflow + +pytestmark = [pytest.mark.layer("recipe"), pytest.mark.small] + + +# --------------------------------------------------------------------------- +# TestUncapturedHandoffConsumerRule (1e–1h) +# --------------------------------------------------------------------------- + + +class TestUncapturedHandoffConsumerRule: + def test_uncaptured_handoff_consumer_rule_is_registered(self) -> None: + """1e: uncaptured-handoff-consumer rule is in _RULE_REGISTRY.""" + from autoskillit.recipe.registry import _RULE_REGISTRY + + assert any(spec.name == "uncaptured-handoff-consumer" for spec in _RULE_REGISTRY) + + def test_uncaptured_handoff_consumer_fires_for_empty_output_skill(self) -> None: + """1f: rule fires WARNING when outputs:[] producer precedes file_path consumer. + + Uses audit-friction (real contract: outputs: []) as producer and + review-approach (real contract: optional plan_path: file_path) as consumer. + The consumer's skill_command does NOT include context.plan_path. + """ + steps = { + "friction": { + "tool": "run_skill", + "with": {"skill_command": "/autoskillit:audit-friction"}, + "on_success": "review", + }, + "review": { + "tool": "run_skill", + "with": {"skill_command": "/autoskillit:review-approach some topic"}, + "on_success": "done", + }, + "done": {"action": "stop", "message": "Done."}, + } + recipe = _make_workflow(steps) + findings = run_semantic_rules(recipe) + handoff = [f for f in findings if f.rule == "uncaptured-handoff-consumer"] + assert len(handoff) >= 1 + assert any(f.severity == Severity.WARNING for f in handoff) + assert any("plan_path" in f.message for f in handoff) + + def test_uncaptured_handoff_consumer_silent_when_context_ref_present(self) -> None: + """1g: rule is silent when consumer's file-path input is wired via context ref.""" + steps = { + "friction": { + "tool": "run_skill", + "with": {"skill_command": "/autoskillit:audit-friction"}, + "on_success": "review", + }, + "review": { + "tool": "run_skill", + "with": {"skill_command": "/autoskillit:review-approach ${{ context.plan_path }}"}, + "on_success": "done", + }, + "done": {"action": "stop", "message": "Done."}, + } + recipe = _make_workflow(steps) + findings = run_semantic_rules(recipe) + handoff = [f for f in findings if f.rule == "uncaptured-handoff-consumer"] + assert handoff == [] + + def test_uncaptured_handoff_consumer_silent_when_no_file_path_inputs(self) -> None: + """1h: rule is silent when consumer has no file-path or directory-path inputs. + + Uses audit-friction (outputs: []) as producer and make-plan (single task: string + input — no file-path inputs) as consumer. The rule must be silent. + """ + steps = { + "friction": { + "tool": "run_skill", + "with": {"skill_command": "/autoskillit:audit-friction"}, + "on_success": "plan", + }, + "plan": { + "tool": "run_skill", + "with": {"skill_command": "/autoskillit:make-plan some task"}, + "on_success": "done", + }, + "done": {"action": "stop", "message": "Done."}, + } + recipe = _make_workflow(steps) + findings = run_semantic_rules(recipe) + handoff = [f for f in findings if f.rule == "uncaptured-handoff-consumer"] + assert handoff == [] + + +# --------------------------------------------------------------------------- +# TestCallableContracts +# --------------------------------------------------------------------------- + + +class TestCallableContracts: + """Tests for callable output contracts (run_python steps).""" + + def test_callable_contract_check_review_loop(self) -> None: + """The callable contract for check_review_loop must declare its output fields.""" + from autoskillit.recipe.contracts import get_callable_contract + + contract = get_callable_contract("autoskillit.smoke_utils.check_review_loop") + assert contract is not None + declared = {out.name for out in contract.outputs} + assert "next_iteration" in declared + assert "max_exceeded" in declared + + def test_callable_contract_missing_returns_none(self) -> None: + """get_callable_contract returns None for unknown callable paths.""" + from autoskillit.recipe.contracts import get_callable_contract + + contract = get_callable_contract("nonexistent.module.function") + assert contract is None + + +# --------------------------------------------------------------------------- +# TestPythonCaptureOutputCoverageRule +# --------------------------------------------------------------------------- + + +class TestPythonCaptureOutputCoverageRule: + def test_python_capture_declared_key_no_warning(self) -> None: + """A run_python capture referencing a declared callable output key + must not produce an undeclared-python-capture-key warning.""" + recipe_yaml = textwrap.dedent("""\ + name: python-capture-valid + description: test + steps: + check: + tool: run_python + with: + callable: "autoskillit.smoke_utils.check_review_loop" + pr_number: "42" + cwd: "/tmp" + capture: + loop_count: "${{ result.next_iteration }}" + on_result: + - when: "${{ result.max_exceeded }} == false" + route: review + - route: done + review: + tool: run_skill + with: + skill_command: /autoskillit:review-pr + on_success: done + on_failure: done + done: + action: stop + message: Done + """) + recipe = _parse_recipe(yaml.safe_load(recipe_yaml)) + findings = run_semantic_rules(recipe) + undeclared = [f for f in findings if f.rule == "undeclared-python-capture-key"] + assert undeclared == [] + + def test_python_capture_undeclared_key_fires_warning(self) -> None: + """A run_python step referencing result.nonexistent must fire a warning.""" + recipe_yaml = textwrap.dedent("""\ + name: python-capture-invalid + description: test + steps: + check: + tool: run_python + with: + callable: "autoskillit.smoke_utils.check_review_loop" + pr_number: "42" + cwd: "/tmp" + capture: + bad: "${{ result.nonexistent }}" + on_success: done + on_failure: done + done: + action: stop + message: Done + """) + recipe = _parse_recipe(yaml.safe_load(recipe_yaml)) + findings = run_semantic_rules(recipe) + undeclared = [f for f in findings if f.rule == "undeclared-python-capture-key"] + assert len(undeclared) == 1 + assert undeclared[0].severity == Severity.WARNING + assert "nonexistent" in undeclared[0].message + + def test_python_capture_no_contract_fires_warning(self) -> None: + """A run_python step with an unknown callable must warn about missing contract.""" + recipe_yaml = textwrap.dedent("""\ + name: python-capture-no-contract + description: test + steps: + check: + tool: run_python + with: + callable: "some.unknown.callable" + capture: + val: "${{ result.output }}" + on_success: done + on_failure: done + done: + action: stop + message: Done + """) + recipe = _parse_recipe(yaml.safe_load(recipe_yaml)) + findings = run_semantic_rules(recipe) + undeclared = [f for f in findings if f.rule == "undeclared-python-capture-key"] + assert len(undeclared) == 1 + assert "no callable contract entry" in undeclared[0].message + + def test_python_on_result_undeclared_ref_fires_warning(self) -> None: + """A run_python on_result condition referencing an undeclared field must fire.""" + recipe_yaml = textwrap.dedent("""\ + name: python-on-result-invalid + description: test + steps: + check: + tool: run_python + with: + callable: "autoskillit.smoke_utils.check_review_loop" + pr_number: "42" + cwd: "/tmp" + on_result: + - when: "${{ result.has_blocking }} == true" + route: review + - route: done + review: + tool: run_skill + with: + skill_command: /autoskillit:review-pr + on_success: done + on_failure: done + done: + action: stop + message: Done + """) + recipe = _parse_recipe(yaml.safe_load(recipe_yaml)) + findings = run_semantic_rules(recipe) + undeclared = [f for f in findings if f.rule == "undeclared-python-capture-key"] + assert len(undeclared) == 1 + assert "has_blocking" in undeclared[0].message + + def test_non_python_step_not_flagged(self) -> None: + """run_skill steps must not be flagged by the python capture rule.""" + recipe_yaml = textwrap.dedent("""\ + name: skill-step-ignored + description: test + steps: + run: + tool: run_skill + with: + skill_command: /autoskillit:implement-worktree-no-merge ${{ inputs.plan }} + capture: + wp: "${{ result.worktree_path }}" + on_success: done + on_failure: done + done: + action: stop + message: Done + """) + recipe = _parse_recipe(yaml.safe_load(recipe_yaml)) + findings = run_semantic_rules(recipe) + undeclared = [f for f in findings if f.rule == "undeclared-python-capture-key"] + assert undeclared == [] diff --git a/tests/recipe/test_rules_dataflow.py b/tests/recipe/test_rules_dataflow_merge.py similarity index 51% rename from tests/recipe/test_rules_dataflow.py rename to tests/recipe/test_rules_dataflow_merge.py index ab4e7ba4d2..e9e5c21771 100644 --- a/tests/recipe/test_rules_dataflow.py +++ b/tests/recipe/test_rules_dataflow_merge.py @@ -1,30 +1,16 @@ -"""Tests for data-flow semantic rules.""" +"""Tests for data-flow semantic rules — merge cleanup, stale ref after merge, push rules.""" from __future__ import annotations -import textwrap - import pytest -import yaml from autoskillit.core.types import Severity -from autoskillit.recipe.io import ( - _parse_recipe, - builtin_recipes_dir, - load_recipe, -) -from autoskillit.recipe.schema import ( - Recipe, - RecipeStep, -) -from autoskillit.recipe.validator import ( - run_semantic_rules, -) -from tests.recipe.conftest import _make_workflow +from autoskillit.recipe.io import builtin_recipes_dir, load_recipe +from autoskillit.recipe.schema import Recipe, RecipeStep +from autoskillit.recipe.validator import run_semantic_rules +from tests.recipe.conftest import _build_merge_worktree_recipe, _make_workflow -# --------------------------------------------------------------------------- -# Fixtures -# --------------------------------------------------------------------------- +pytestmark = [pytest.mark.layer("recipe"), pytest.mark.small] @pytest.fixture @@ -36,342 +22,48 @@ def all_bundled_recipes() -> list[tuple[str, Recipe]]: return result -# --------------------------------------------------------------------------- -# Helper for merge_worktree recipe tests -# --------------------------------------------------------------------------- - - -def _build_merge_worktree_recipe(capture: dict) -> Recipe: - """Helper: build a minimal Recipe with a merge_worktree step and the given capture dict.""" +def _make_stale_worktree_path_recipe() -> Recipe: + """Return the stale-worktree-path recipe used by tests B1 and B6.""" return Recipe( - name="test-merge", - description="Test merge recipe", - summary="merge > done", + name="test-stale-path", + description="test", steps={ + "implement": RecipeStep( + tool="run_skill", + with_args={ + "skill_command": "/autoskillit:implement-worktree-no-merge plan.md", + }, + capture={"worktree_path": "${{ result.worktree_path }}"}, + on_success="test", + ), + "test": RecipeStep( + tool="test_check", + with_args={"worktree_path": "${{ context.worktree_path }}"}, + on_success="merge", + ), "merge": RecipeStep( tool="merge_worktree", - with_args={"worktree_path": "${{ context.worktree_path }}", "base_branch": "main"}, - capture=capture, + with_args={ + "worktree_path": "${{ context.worktree_path }}", + "base_branch": "main", + }, + capture={"cleanup_succeeded": "${{ result.cleanup_succeeded }}"}, + on_success="audit", + ), + "audit": RecipeStep( + tool="run_skill", + with_args={ + "skill_command": ( + "/autoskillit:audit-impl plan.md ${{ context.worktree_path }} main" + ), + }, on_success="done", - on_failure="done", ), - "done": RecipeStep(action="stop", message="Done"), + "done": RecipeStep(action="stop", message="done"), }, ) -# --------------------------------------------------------------------------- -# TestCaptureOutputCoverageRule -# --------------------------------------------------------------------------- - - -class TestCaptureOutputCoverageRule: - def test_capture_declared_output_key_no_warning(self) -> None: - """A capture that references a key declared in the skill's outputs contract - must not produce an undeclared-capture-key warning.""" - recipe_yaml = textwrap.dedent("""\ - name: capture-valid - description: test - steps: - implement: - tool: run_skill - with: - skill_command: /autoskillit:implement-worktree-no-merge ${{ inputs.plan }} - capture: - worktree_path: "${{ result.worktree_path }}" - on_success: done - on_failure: done - done: - action: stop - message: Done - """) - recipe = _parse_recipe(yaml.safe_load(recipe_yaml)) - findings = run_semantic_rules(recipe) - undeclared = [f for f in findings if f.rule == "undeclared-capture-key"] - assert undeclared == [] - - def test_capture_undeclared_key_emits_warning(self) -> None: - """A capture that references a key NOT listed in the skill's outputs contract - must produce a Severity.WARNING finding with rule 'undeclared-capture-key'.""" - recipe_yaml = textwrap.dedent("""\ - name: capture-invalid-key - description: test - steps: - implement: - tool: run_skill - with: - skill_command: /autoskillit:implement-worktree-no-merge ${{ inputs.plan }} - capture: - nonexistent_output: "${{ result.nonexistent_output }}" - on_success: done - on_failure: done - done: - action: stop - message: Done - """) - recipe = _parse_recipe(yaml.safe_load(recipe_yaml)) - findings = run_semantic_rules(recipe) - undeclared = [f for f in findings if f.rule == "undeclared-capture-key"] - assert len(undeclared) == 1 - assert undeclared[0].severity == Severity.WARNING - assert "nonexistent_output" in undeclared[0].message - assert "implement-worktree-no-merge" in undeclared[0].message - - def test_capture_from_skill_with_no_contract_emits_warning(self) -> None: - """A capture step whose skill has no entry in skill_contracts.yaml at all - must produce a Severity.WARNING finding with rule 'undeclared-capture-key'.""" - recipe_yaml = textwrap.dedent("""\ - name: capture-unknown-skill - description: test - steps: - run_custom: - tool: run_skill - with: - skill_command: /autoskillit:not-a-real-skill some_arg - capture: - result_key: "${{ result.some_key }}" - on_success: done - on_failure: done - done: - action: stop - message: Done - """) - recipe = _parse_recipe(yaml.safe_load(recipe_yaml)) - findings = run_semantic_rules(recipe) - undeclared = [f for f in findings if f.rule == "undeclared-capture-key"] - assert len(undeclared) == 1 - assert undeclared[0].severity == Severity.WARNING - assert "not-a-real-skill" in undeclared[0].message - assert "no outputs contract entry" in undeclared[0].message - - def test_capture_key_from_empty_outputs_skill_emits_warning(self) -> None: - """audit-friction has outputs: [] — any capture key from it is undeclared.""" - recipe_yaml = textwrap.dedent("""\ - name: capture-empty-outputs - description: test - steps: - audit: - tool: run_skill - with: - skill_command: /autoskillit:audit-friction - capture: - report_path: "${{ result.report_path }}" - on_success: done - on_failure: done - done: - action: stop - message: Done - """) - recipe = _parse_recipe(yaml.safe_load(recipe_yaml)) - findings = run_semantic_rules(recipe) - undeclared = [f for f in findings if f.rule == "undeclared-capture-key"] - assert len(undeclared) == 1 - assert undeclared[0].severity == Severity.WARNING - assert "report_path" in undeclared[0].message - assert "audit-friction" in undeclared[0].message - - -# --------------------------------------------------------------------------- -# TestDeadOutputRule -# --------------------------------------------------------------------------- - - -class TestDeadOutputRule: - def test_do1_dead_output_rule_in_registry(self) -> None: - """T_DO1: dead-output is in _RULE_REGISTRY.""" - from autoskillit.recipe.validator import _RULE_REGISTRY - - rule_names = [r.name for r in _RULE_REGISTRY] - assert "dead-output" in rule_names - - def test_do2_fires_error_for_unconsumed_capture(self) -> None: - """T_DO2: dead-output fires ERROR when a captured key is never consumed downstream.""" - steps = { - "plan": { - "tool": "run_skill", - "with": {"skill_command": "/autoskillit:make-plan do the task"}, - "capture": {"plan_path": "${{ result.plan_path }}"}, - "on_success": "done", - }, - "done": {"action": "stop", "message": "Done."}, - } - recipe = _make_workflow(steps) - findings = run_semantic_rules(recipe) - dead = [f for f in findings if f.rule == "dead-output"] - assert len(dead) >= 1 - assert any(f.severity == Severity.ERROR and f.step_name == "plan" for f in dead) - - def test_do3_does_not_fire_for_on_result_self_consumption(self) -> None: - """T_DO3: dead-output does NOT fire when on_result.field equals the captured key.""" - steps = { - "audit_impl": { - "tool": "run_skill", - "with": { - "skill_command": "/autoskillit:audit-impl plan.md myref main", - }, - "capture": {"verdict": "${{ result.verdict }}"}, - "on_result": { - "field": "verdict", - "routes": {"GO": "done", "NO GO": "done"}, - }, - }, - "done": {"action": "stop", "message": "Done."}, - } - recipe = _make_workflow(steps) - findings = run_semantic_rules(recipe) - dead = [f for f in findings if f.rule == "dead-output" and f.step_name == "audit_impl"] - assert dead == [] - - def test_do4_cleanup_succeeded_from_merge_worktree_not_dead_output(self) -> None: - """T_DO4: dead-output does NOT fire for cleanup_succeeded captured from merge_worktree. - - cleanup_succeeded is an observability capture required by merge-cleanup-uncaptured. - It has no downstream consumer by design — the exemption prevents the two rules - from conflicting. - """ - recipe = _build_merge_worktree_recipe( - capture={"cleanup_succeeded": "${{ result.cleanup_succeeded }}"} - ) - findings = run_semantic_rules(recipe) - dead = [f for f in findings if f.rule == "dead-output" and f.step_name == "merge"] - assert dead == [] - - def test_do5_predicate_condition_consumes_captured_var(self) -> None: - """T_DO5: dead-output does NOT fire when a predicate on_result condition - references the captured variable via context.X in its when clause.""" - steps = { - "review_pr": { - "tool": "run_skill", - "with": {"skill_command": "/autoskillit:review-pr feature main"}, - "capture": {"verdict": "${{ result.verdict }}"}, - "on_result": [ - { - "when": "${{ result.verdict }} == changes_requested", - "route": "resolve_review", - }, - {"when": "true", "route": "done"}, - ], - "on_failure": "resolve_review", - }, - "resolve_review": { - "tool": "run_skill", - "with": { - "skill_command": "/autoskillit:resolve-failures worktree plan main", - }, - "on_success": "done", - "on_failure": "done", - }, - "done": {"action": "stop", "message": "Done."}, - } - recipe = _make_workflow(steps) - findings = run_semantic_rules(recipe) - dead = [f for f in findings if f.rule == "dead-output" and f.step_name == "review_pr"] - assert dead == [] - - -# --------------------------------------------------------------------------- -# TestImplicitHandoffRule -# --------------------------------------------------------------------------- - - -class TestImplicitHandoffRule: - def test_ih1_implicit_handoff_rule_in_registry(self) -> None: - """T_IH1: implicit-handoff is in _RULE_REGISTRY.""" - from autoskillit.recipe.validator import _RULE_REGISTRY - - rule_names = [r.name for r in _RULE_REGISTRY] - assert "implicit-handoff" in rule_names - - def test_ih2_fires_error_for_skill_with_outputs_and_no_capture(self) -> None: - """T_IH2: implicit-handoff fires ERROR when skill has outputs but step has no capture.""" - steps = { - "plan": { - "tool": "run_skill", - "with": {"skill_command": "/autoskillit:make-plan do the task"}, - "on_success": "done", - }, - "done": {"action": "stop", "message": "Done."}, - } - recipe = _make_workflow(steps) - findings = run_semantic_rules(recipe) - ih = [f for f in findings if f.rule == "implicit-handoff"] - assert len(ih) >= 1 - assert any(f.severity == Severity.ERROR and f.step_name == "plan" for f in ih) - - def test_ih3_does_not_fire_when_capture_block_present(self) -> None: - """T_IH3: implicit-handoff does NOT fire when the step has a capture: block.""" - steps = { - "plan": { - "tool": "run_skill", - "with": {"skill_command": "/autoskillit:make-plan do the task"}, - "capture": {"plan_path": "${{ result.plan_path }}"}, - "on_success": "done", - }, - "done": {"action": "stop", "message": "Done."}, - } - recipe = _make_workflow(steps) - findings = run_semantic_rules(recipe) - ih = [f for f in findings if f.rule == "implicit-handoff" and f.step_name == "plan"] - assert ih == [] - - def test_ih4_does_not_fire_for_skill_with_empty_outputs(self) -> None: - """T_IH4: implicit-handoff does NOT fire for a skill with outputs: []. - - dry-walkthrough has outputs: [] — no fields to capture, so no finding. - """ - steps = { - "walkthrough": { - "tool": "run_skill", - "with": {"skill_command": "/autoskillit:dry-walkthrough plan"}, - "on_success": "done", - }, - "done": {"action": "stop", "message": "Done."}, - } - recipe = _make_workflow(steps) - findings = run_semantic_rules(recipe) - ih = [f for f in findings if f.rule == "implicit-handoff" and f.step_name == "walkthrough"] - assert ih == [] - - def test_ih5_does_not_fire_for_unknown_skill(self) -> None: - """T_IH5: implicit-handoff does NOT fire for a skill with no contract entry.""" - steps = { - "step": { - "tool": "run_skill", - "with": {"skill_command": "/autoskillit:not-a-real-skill something"}, - "on_success": "done", - }, - "done": {"action": "stop", "message": "Done."}, - } - recipe = _make_workflow(steps) - findings = run_semantic_rules(recipe) - ih = [f for f in findings if f.rule == "implicit-handoff" and f.step_name == "step"] - assert ih == [] - - def test_implicit_handoff_fires_for_investigate_without_capture(self) -> None: - """T_IH_1g: implicit-handoff fires for investigate step without capture block. - - After investigate.outputs gains investigation_path in skill_contracts.yaml, - a recipe step invoking /autoskillit:investigate without a capture block - triggers the implicit-handoff rule. Uses the real bundled manifest. - """ - steps = { - "investigate": { - "tool": "run_skill", - "with": {"skill_command": "/autoskillit:investigate the bug"}, - "on_success": "done", - }, - "done": {"action": "stop", "message": "Done."}, - } - recipe = _make_workflow(steps) - findings = run_semantic_rules(recipe) - ih = [f for f in findings if f.rule == "implicit-handoff" and f.step_name == "investigate"] - assert len(ih) >= 1, ( - "implicit-handoff must fire for an investigate step with no capture block " - "once investigate declares investigation_path in skill_contracts.yaml" - ) - assert any(f.severity == Severity.ERROR for f in ih) - - # --------------------------------------------------------------------------- # merge-cleanup-uncaptured tests # --------------------------------------------------------------------------- @@ -436,53 +128,6 @@ def test_bundled_recipes_capture_cleanup_succeeded() -> None: ) -# --------------------------------------------------------------------------- -# Helper for stale-ref tests -# --------------------------------------------------------------------------- - - -def _make_stale_worktree_path_recipe() -> Recipe: - """Return the stale-worktree-path recipe used by tests B1 and B6.""" - return Recipe( - name="test-stale-path", - description="test", - steps={ - "implement": RecipeStep( - tool="run_skill", - with_args={ - "skill_command": "/autoskillit:implement-worktree-no-merge plan.md", - }, - capture={"worktree_path": "${{ result.worktree_path }}"}, - on_success="test", - ), - "test": RecipeStep( - tool="test_check", - with_args={"worktree_path": "${{ context.worktree_path }}"}, - on_success="merge", - ), - "merge": RecipeStep( - tool="merge_worktree", - with_args={ - "worktree_path": "${{ context.worktree_path }}", - "base_branch": "main", - }, - capture={"cleanup_succeeded": "${{ result.cleanup_succeeded }}"}, - on_success="audit", - ), - "audit": RecipeStep( - tool="run_skill", - with_args={ - "skill_command": ( - "/autoskillit:audit-impl plan.md ${{ context.worktree_path }} main" - ), - }, - on_success="done", - ), - "done": RecipeStep(action="stop", message="done"), - }, - ) - - # --------------------------------------------------------------------------- # TestStaleRefAfterMerge # --------------------------------------------------------------------------- @@ -842,87 +487,3 @@ def test_no_finding_when_no_push_to_remote_step(self) -> None: f for f in run_semantic_rules(recipe) if f.rule == "push-missing-explicit-remote-url" ] assert findings == [] - - -# --------------------------------------------------------------------------- -# TestUncapturedHandoffConsumerRule (1e–1h) -# --------------------------------------------------------------------------- - - -class TestUncapturedHandoffConsumerRule: - def test_uncaptured_handoff_consumer_rule_is_registered(self) -> None: - """1e: uncaptured-handoff-consumer rule is in _RULE_REGISTRY.""" - from autoskillit.recipe.registry import _RULE_REGISTRY - - assert any(spec.name == "uncaptured-handoff-consumer" for spec in _RULE_REGISTRY) - - def test_uncaptured_handoff_consumer_fires_for_empty_output_skill(self) -> None: - """1f: rule fires WARNING when outputs:[] producer precedes file_path consumer. - - Uses audit-friction (real contract: outputs: []) as producer and - review-approach (real contract: optional plan_path: file_path) as consumer. - The consumer's skill_command does NOT include context.plan_path. - """ - steps = { - "friction": { - "tool": "run_skill", - "with": {"skill_command": "/autoskillit:audit-friction"}, - "on_success": "review", - }, - "review": { - "tool": "run_skill", - "with": {"skill_command": "/autoskillit:review-approach some topic"}, - "on_success": "done", - }, - "done": {"action": "stop", "message": "Done."}, - } - recipe = _make_workflow(steps) - findings = run_semantic_rules(recipe) - handoff = [f for f in findings if f.rule == "uncaptured-handoff-consumer"] - assert len(handoff) >= 1 - assert any(f.severity == Severity.WARNING for f in handoff) - assert any("plan_path" in f.message for f in handoff) - - def test_uncaptured_handoff_consumer_silent_when_context_ref_present(self) -> None: - """1g: rule is silent when consumer's file-path input is wired via context ref.""" - steps = { - "friction": { - "tool": "run_skill", - "with": {"skill_command": "/autoskillit:audit-friction"}, - "on_success": "review", - }, - "review": { - "tool": "run_skill", - "with": {"skill_command": "/autoskillit:review-approach ${{ context.plan_path }}"}, - "on_success": "done", - }, - "done": {"action": "stop", "message": "Done."}, - } - recipe = _make_workflow(steps) - findings = run_semantic_rules(recipe) - handoff = [f for f in findings if f.rule == "uncaptured-handoff-consumer"] - assert handoff == [] - - def test_uncaptured_handoff_consumer_silent_when_no_file_path_inputs(self) -> None: - """1h: rule is silent when consumer has no file-path or directory-path inputs. - - Uses audit-friction (outputs: []) as producer and make-plan (single task: string - input — no file-path inputs) as consumer. The rule must be silent. - """ - steps = { - "friction": { - "tool": "run_skill", - "with": {"skill_command": "/autoskillit:audit-friction"}, - "on_success": "plan", - }, - "plan": { - "tool": "run_skill", - "with": {"skill_command": "/autoskillit:make-plan some task"}, - "on_success": "done", - }, - "done": {"action": "stop", "message": "Done."}, - } - recipe = _make_workflow(steps) - findings = run_semantic_rules(recipe) - handoff = [f for f in findings if f.rule == "uncaptured-handoff-consumer"] - assert handoff == [] diff --git a/tests/recipe/test_rules_dataflow_nullable.py b/tests/recipe/test_rules_dataflow_nullable.py new file mode 100644 index 0000000000..92fcd05251 --- /dev/null +++ b/tests/recipe/test_rules_dataflow_nullable.py @@ -0,0 +1,60 @@ +"""Tests for the nullable-optional-context-ref semantic rule.""" + +from __future__ import annotations + +import pytest + +from autoskillit.core.types import Severity +from autoskillit.recipe.validator import run_semantic_rules +from tests.recipe.conftest import _make_workflow + +pytestmark = [pytest.mark.layer("recipe"), pytest.mark.small] + + +class TestNullableOptionalContextRefRule: + def test_optional_context_ref_to_non_nullable_input_is_error(self) -> None: + recipe = _make_workflow( + { + "advance": { + "tool": "run_python", + "with": { + "callable": "autoskillit.recipe._cmd_rpc.advance_queue_pr", + "args": { + "current_pr_number": "${{ context.current_pr }}", + "pr_order_file": "/some/path", + }, + }, + "optional_context_refs": ["current_pr"], + "on_result": {"current_pr_number": {"'0'": "done", "*": "advance"}}, + }, + "done": {"action": "stop", "message": "done"}, + } + ) + findings = run_semantic_rules(recipe) + nullable_findings = [f for f in findings if f.rule == "nullable-optional-context-ref"] + assert any(f.severity == Severity.ERROR for f in nullable_findings), ( + f"Expected ERROR finding for nullable-optional-context-ref, got: {nullable_findings}" + ) + + def test_non_optional_context_ref_does_not_trigger_rule(self) -> None: + recipe = _make_workflow( + { + "advance": { + "tool": "run_python", + "with": { + "callable": "autoskillit.recipe._cmd_rpc.advance_queue_pr", + "args": { + "current_pr_number": "${{ context.current_pr }}", + "pr_order_file": "/some/path", + }, + }, + "on_result": {"current_pr_number": {"'0'": "done", "*": "advance"}}, + }, + "done": {"action": "stop", "message": "done"}, + } + ) + findings = run_semantic_rules(recipe) + nullable_findings = [f for f in findings if f.rule == "nullable-optional-context-ref"] + assert not any(f.severity == Severity.ERROR for f in nullable_findings), ( + f"Should not flag when optional_context_refs is absent, got: {nullable_findings}" + ) diff --git a/tests/recipe/test_rules_dedup.py b/tests/recipe/test_rules_dedup.py new file mode 100644 index 0000000000..22dc5ae51d --- /dev/null +++ b/tests/recipe/test_rules_dedup.py @@ -0,0 +1,21 @@ +"""Guard against re-introduction of _get_skill_category_map duplication. + +Asserts that both rule modules share the exact same function object, meaning +the function is defined once in a shared helper and imported in both places. +""" + +from __future__ import annotations + +import pytest + +pytestmark = [pytest.mark.layer("recipe"), pytest.mark.small] + + +def test_skill_category_map_not_duplicated() -> None: + from autoskillit.recipe.rules import rules_features, rules_skills + + assert rules_skills._get_skill_category_map is rules_features._get_skill_category_map, ( + "_get_skill_category_map must be the same object in rules_skills and " + "rules_features — define it once in recipe/_skill_helpers.py and import it " + "in both modules." + ) diff --git a/tests/recipe/test_rules_features.py b/tests/recipe/test_rules_features.py new file mode 100644 index 0000000000..5d17431e35 --- /dev/null +++ b/tests/recipe/test_rules_features.py @@ -0,0 +1,418 @@ +"""Tests for feature-gate-tool-reference validation rule (T-FEAT-001..005).""" + +from __future__ import annotations + +import pytest + +pytestmark = [pytest.mark.layer("recipe"), pytest.mark.medium] + + +def test_feature_gate_rule_fires_on_disabled_feature_tool() -> None: + """Severity.ERROR when recipe uses dispatch_food_truck with fleet disabled.""" + from autoskillit.core import Severity + from autoskillit.recipe._analysis import make_validation_context + from autoskillit.recipe.registry import run_semantic_rules + from autoskillit.recipe.schema import Recipe, RecipeStep + + recipe = Recipe( + name="r", + description="d", + version="0.2.0", + kitchen_rules="k", + steps={"s": RecipeStep(tool="dispatch_food_truck", with_args={})}, + ) + ctx = make_validation_context(recipe, disabled_features=frozenset({"fleet"})) + findings = [f for f in run_semantic_rules(ctx) if f.rule == "feature-gate-tool-reference"] + assert findings, "expected feature-gate-tool-reference finding for disabled fleet tool" + assert all(f.severity == Severity.ERROR for f in findings) + assert any("fleet" in f.message for f in findings) + assert any("dispatch_food_truck" in f.message for f in findings) + + +def test_feature_gate_rule_passes_when_feature_enabled() -> None: + """No feature-gate-tool-reference finding when disabled_features is empty.""" + from autoskillit.recipe._analysis import make_validation_context + from autoskillit.recipe.registry import run_semantic_rules + from autoskillit.recipe.schema import Recipe, RecipeStep + + recipe = Recipe( + name="r", + description="d", + version="0.2.0", + kitchen_rules="k", + steps={"s": RecipeStep(tool="dispatch_food_truck", with_args={})}, + ) + ctx = make_validation_context(recipe, disabled_features=frozenset()) + findings = [f for f in run_semantic_rules(ctx) if f.rule == "feature-gate-tool-reference"] + assert not findings + + +def test_feature_gate_rule_ignores_non_feature_tools() -> None: + """run_cmd has no feature tags — no finding even when fleet is disabled.""" + from autoskillit.recipe._analysis import make_validation_context + from autoskillit.recipe.registry import run_semantic_rules + from autoskillit.recipe.schema import Recipe, RecipeStep + + recipe = Recipe( + name="r", + description="d", + version="0.2.0", + kitchen_rules="k", + steps={"s": RecipeStep(tool="run_cmd", with_args={"cmd": "echo hi"})}, + ) + ctx = make_validation_context(recipe, disabled_features=frozenset({"fleet"})) + findings = [f for f in run_semantic_rules(ctx) if f.rule == "feature-gate-tool-reference"] + assert not findings + + +def test_feature_gate_rule_fires_on_disabled_feature_skill(monkeypatch) -> None: + """ERROR when recipe uses a skill from a feature's skill_categories.""" + import autoskillit.core.types._type_constants as _consts + from autoskillit.core import FeatureDef, FeatureLifecycle, Severity + from autoskillit.recipe._analysis import ( + ValidationContext, + _build_step_graph, + analyze_dataflow, + ) + from autoskillit.recipe.registry import run_semantic_rules + from autoskillit.recipe.schema import Recipe, RecipeStep + + # Inject a test feature with skill_categories into the registry + test_fdef = FeatureDef( + lifecycle=FeatureLifecycle.EXPERIMENTAL, + description="Test feature gating a skill category", + tool_tags=frozenset(), + skill_categories=frozenset({"arch-lens"}), + import_package=None, + ) + monkeypatch.setitem(_consts.FEATURE_REGISTRY, "test-skill-gate", test_fdef) + + recipe = Recipe( + name="r", + description="d", + version="0.2.0", + kitchen_rules="k", + steps={ + "s": RecipeStep( + tool="run_skill", + with_args={"skill_command": "/autoskillit:arch-lens-c4-container"}, + ) + }, + ) + step_graph = _build_step_graph(recipe) + ctx = ValidationContext( + recipe=recipe, + step_graph=step_graph, + dataflow=analyze_dataflow(recipe, step_graph=step_graph), + disabled_features=frozenset({"test-skill-gate"}), + skill_category_map={"arch-lens-c4-container": frozenset({"arch-lens"})}, + ) + findings = [f for f in run_semantic_rules(ctx) if f.rule == "feature-gate-tool-reference"] + assert findings, "expected feature-gate-tool-reference finding for disabled skill category" + assert all(f.severity == Severity.ERROR for f in findings) + assert any("test-skill-gate" in f.message for f in findings) + + +def test_feature_gate_rule_with_multiple_features(monkeypatch) -> None: + """Each disabled feature independently flags its own tools.""" + import autoskillit.core.types._type_constants as _consts + from autoskillit.core import FeatureDef, FeatureLifecycle, Severity + from autoskillit.recipe._analysis import make_validation_context + from autoskillit.recipe.registry import run_semantic_rules + from autoskillit.recipe.schema import Recipe, RecipeStep + + # A second test feature that controls 'ci' tools + test_ci_fdef = FeatureDef( + lifecycle=FeatureLifecycle.EXPERIMENTAL, + description="Test feature gating ci tools", + tool_tags=frozenset({"ci"}), + skill_categories=frozenset(), + import_package=None, + ) + monkeypatch.setitem(_consts.FEATURE_REGISTRY, "test-ci-gate", test_ci_fdef) + + recipe = Recipe( + name="r", + description="d", + version="0.2.0", + kitchen_rules="k", + steps={ + "fleet_step": RecipeStep(tool="dispatch_food_truck", with_args={}), + "ci_step": RecipeStep(tool="wait_for_ci", with_args={}), + }, + ) + ctx = make_validation_context(recipe, disabled_features=frozenset({"fleet", "test-ci-gate"})) + findings = [f for f in run_semantic_rules(ctx) if f.rule == "feature-gate-tool-reference"] + + step_names = {f.step_name for f in findings} + assert len(step_names) == 2, f"expected exactly 2 flagged steps, got {step_names!r}" + assert "fleet_step" in step_names, "fleet tool not flagged" + assert "ci_step" in step_names, "ci tool not flagged" + assert all(f.severity == Severity.ERROR for f in findings) + + +def test_feature_gate_rule_fires_on_run_python_planner_callable() -> None: + """ERROR when run_python callable starts with planner import_package and planner disabled.""" + from autoskillit.core import Severity + from autoskillit.recipe._analysis import make_validation_context + from autoskillit.recipe.registry import run_semantic_rules + from autoskillit.recipe.schema import Recipe, RecipeStep + + recipe = Recipe( + name="r", + description="d", + version="0.2.0", + kitchen_rules="k", + steps={ + "plan": RecipeStep( + tool="run_python", + with_args={"callable": "autoskillit.planner.validate_plan"}, + ) + }, + ) + ctx = make_validation_context(recipe, disabled_features=frozenset({"planner"})) + findings = [f for f in run_semantic_rules(ctx) if f.rule == "feature-gate-tool-reference"] + assert findings, "expected feature-gate-tool-reference finding for disabled planner callable" + assert all(f.severity == Severity.ERROR for f in findings) + assert any("planner" in f.message for f in findings) + assert any("plan" in f.step_name for f in findings) + + +def test_feature_gate_rule_no_false_positive_run_python_non_feature_callable() -> None: + """No finding when run_python callable does not belong to any disabled feature.""" + from autoskillit.recipe._analysis import make_validation_context + from autoskillit.recipe.registry import run_semantic_rules + from autoskillit.recipe.schema import Recipe, RecipeStep + + recipe = Recipe( + name="r", + description="d", + version="0.2.0", + kitchen_rules="k", + steps={ + "smoke": RecipeStep( + tool="run_python", + with_args={"callable": "autoskillit.smoke_utils.check_something"}, + ) + }, + ) + ctx = make_validation_context(recipe, disabled_features=frozenset({"planner"})) + findings = [f for f in run_semantic_rules(ctx) if f.rule == "feature-gate-tool-reference"] + assert not findings, f"unexpected finding for non-feature callable: {findings}" + + +def test_feature_gate_run_python_no_finding_when_fdef_has_no_import_package(monkeypatch) -> None: + """No false positive when feature has no import_package (import_package=None).""" + import autoskillit.core.types._type_constants as _consts + from autoskillit.core import FeatureDef, FeatureLifecycle + from autoskillit.recipe._analysis import make_validation_context + from autoskillit.recipe.registry import run_semantic_rules + from autoskillit.recipe.schema import Recipe, RecipeStep + + no_pkg_fdef = FeatureDef( + lifecycle=FeatureLifecycle.EXPERIMENTAL, + description="Feature with no import_package", + tool_tags=frozenset(), + skill_categories=frozenset(), + import_package=None, + ) + monkeypatch.setitem(_consts.FEATURE_REGISTRY, "no-pkg-feature", no_pkg_fdef) + + recipe = Recipe( + name="r", + description="d", + version="0.2.0", + kitchen_rules="k", + steps={ + "s": RecipeStep( + tool="run_python", + with_args={"callable": "autoskillit.planner.validate_plan"}, + ) + }, + ) + ctx = make_validation_context(recipe, disabled_features=frozenset({"no-pkg-feature"})) + findings = [f for f in run_semantic_rules(ctx) if f.rule == "feature-gate-tool-reference"] + assert not findings, f"unexpected finding when import_package is None: {findings}" + + +def test_provider_requires_profile_fires_on_missing_provider() -> None: + from autoskillit.core import Severity + from autoskillit.recipe._analysis import make_validation_context + from autoskillit.recipe.registry import run_semantic_rules + from autoskillit.recipe.schema import Recipe, RecipeStep + + recipe = Recipe( + name="r", + description="d", + version="0.2.0", + kitchen_rules="k", + steps={ + "run_step": RecipeStep( + tool="run_skill", provider="custom-provider", on_success="done" + ), + "done": RecipeStep(action="stop", message="Done."), + }, + ) + ctx = make_validation_context(recipe, provider_profiles=frozenset({"other-provider"})) + findings = [f for f in run_semantic_rules(ctx) if f.rule == "provider-requires-profile"] + assert len(findings) == 1 + assert findings[0].severity == Severity.ERROR + assert "custom-provider" in findings[0].message + + +def test_provider_requires_profile_passes_when_profile_matches() -> None: + from autoskillit.recipe._analysis import make_validation_context + from autoskillit.recipe.registry import run_semantic_rules + from autoskillit.recipe.schema import Recipe, RecipeStep + + recipe = Recipe( + name="r", + description="d", + version="0.2.0", + kitchen_rules="k", + steps={ + "run_step": RecipeStep(tool="run_skill", provider="my-provider", on_success="done"), + "done": RecipeStep(action="stop", message="Done."), + }, + ) + ctx = make_validation_context(recipe, provider_profiles=frozenset({"my-provider"})) + findings = [f for f in run_semantic_rules(ctx) if f.rule == "provider-requires-profile"] + assert findings == [] + + +def test_provider_requires_profile_skips_when_no_profiles_configured() -> None: + from autoskillit.recipe._analysis import make_validation_context + from autoskillit.recipe.registry import run_semantic_rules + from autoskillit.recipe.schema import Recipe, RecipeStep + + recipe = Recipe( + name="r", + description="d", + version="0.2.0", + kitchen_rules="k", + steps={ + "run_step": RecipeStep(tool="run_skill", provider="any-provider", on_success="done"), + "done": RecipeStep(action="stop", message="Done."), + }, + ) + ctx = make_validation_context(recipe, provider_profiles=frozenset()) + findings = [f for f in run_semantic_rules(ctx) if f.rule == "provider-requires-profile"] + assert findings == [] + + +def test_provider_requires_profile_skips_none_provider_steps() -> None: + from autoskillit.recipe._analysis import make_validation_context + from autoskillit.recipe.registry import run_semantic_rules + from autoskillit.recipe.schema import Recipe, RecipeStep + + recipe = Recipe( + name="r", + description="d", + version="0.2.0", + kitchen_rules="k", + steps={ + "run_step": RecipeStep(tool="run_skill", provider=None, on_success="done"), + "done": RecipeStep(action="stop", message="Done."), + }, + ) + ctx = make_validation_context(recipe, provider_profiles=frozenset({"some-profile"})) + findings = [f for f in run_semantic_rules(ctx) if f.rule == "provider-requires-profile"] + assert findings == [] + + +def test_provider_requires_profile_multiple_steps_mixed() -> None: + from autoskillit.recipe._analysis import make_validation_context + from autoskillit.recipe.registry import run_semantic_rules + from autoskillit.recipe.schema import Recipe, RecipeStep + + recipe = Recipe( + name="r", + description="d", + version="0.2.0", + kitchen_rules="k", + steps={ + "no_provider_step": RecipeStep( + tool="run_skill", provider=None, on_success="valid_step" + ), + "valid_step": RecipeStep( + tool="run_skill", provider="valid", on_success="invalid_step" + ), + "invalid_step": RecipeStep(tool="run_skill", provider="invalid", on_success="done"), + "done": RecipeStep(action="stop", message="Done."), + }, + ) + ctx = make_validation_context(recipe, provider_profiles=frozenset({"valid"})) + findings = [f for f in run_semantic_rules(ctx) if f.rule == "provider-requires-profile"] + assert len(findings) == 1 + assert findings[0].step_name == "invalid_step" + + +def test_validation_context_provider_profiles_default() -> None: + from autoskillit.recipe._analysis import make_validation_context + from autoskillit.recipe.schema import Recipe, RecipeStep + + recipe = Recipe( + name="r", + description="d", + version="0.2.0", + kitchen_rules="k", + steps={"done": RecipeStep(action="stop", message="Done.")}, + ) + ctx = make_validation_context(recipe) + assert ctx.provider_profiles == frozenset() + + +def test_feature_gate_rule_fires_on_undeclared_requires_features() -> None: + """Recipe using planner skill_command without requires_features: [planner] gets ERROR.""" + from autoskillit.core import SKILL_TOOLS, Severity + from autoskillit.recipe._analysis import make_validation_context + from autoskillit.recipe.registry import run_semantic_rules + from autoskillit.recipe.schema import Recipe, RecipeStep + + skill_tool = next(iter(SKILL_TOOLS)) + recipe = Recipe( + name="r", + description="d", + version="0.2.0", + kitchen_rules="k", + requires_features=[], + steps={ + "s": RecipeStep( + tool=skill_tool, + with_args={"skill_command": "/autoskillit:planner-analyze"}, + ) + }, + ) + ctx = make_validation_context(recipe, disabled_features=frozenset()) + findings = [f for f in run_semantic_rules(ctx) if f.rule == "undeclared-feature-requirement"] + assert findings, ( + "expected undeclared-feature-requirement finding for planner skill " + "without requires_features" + ) + assert all(f.severity == Severity.ERROR for f in findings) + + +def test_feature_gate_rule_passes_when_requires_features_declared() -> None: + """No undeclared-feature-requirement finding when requires_features includes the feature.""" + from autoskillit.core import SKILL_TOOLS + from autoskillit.recipe._analysis import make_validation_context + from autoskillit.recipe.registry import run_semantic_rules + from autoskillit.recipe.schema import Recipe, RecipeStep + + skill_tool = next(iter(SKILL_TOOLS)) + recipe = Recipe( + name="r", + description="d", + version="0.2.0", + kitchen_rules="k", + requires_features=["planner"], + steps={ + "s": RecipeStep( + tool=skill_tool, + with_args={"skill_command": "/autoskillit:planner-analyze"}, + ) + }, + ) + ctx = make_validation_context(recipe, disabled_features=frozenset()) + findings = [f for f in run_semantic_rules(ctx) if f.rule == "undeclared-feature-requirement"] + assert not findings, f"unexpected findings: {findings}" diff --git a/tests/recipe/test_rules_food_truck.py b/tests/recipe/test_rules_food_truck.py new file mode 100644 index 0000000000..c71f02a47c --- /dev/null +++ b/tests/recipe/test_rules_food_truck.py @@ -0,0 +1,83 @@ +"""Tests for food-truck semantic validation rules (rules_food_truck.py).""" + +from __future__ import annotations + +import pytest + +import autoskillit.recipe # noqa: F401 -- triggers rule registration +from autoskillit.core import Severity +from autoskillit.recipe._analysis import make_validation_context +from autoskillit.recipe.registry import run_semantic_rules +from autoskillit.recipe.schema import Recipe, RecipeKind, RecipeStep + +pytestmark = [pytest.mark.layer("recipe"), pytest.mark.small] + + +def _food_truck_recipe(**kwargs: object) -> Recipe: + defaults: dict = { + "name": "my-food-truck", + "description": "test food truck", + "kind": RecipeKind.FOOD_TRUCK, + "steps": {"done": RecipeStep(action="stop", message="sentinel done")}, + "kitchen_rules": ["NEVER"], + } + defaults.update(kwargs) + return Recipe(**defaults) + + +def _standard_recipe(**kwargs: object) -> Recipe: + return Recipe( + name="standard", + description="standard recipe", + steps={"done": RecipeStep(action="stop", message="done")}, + kitchen_rules=["NEVER"], + **kwargs, + ) + + +def _findings(recipe: Recipe, rule: str, **ctx_kwargs: object) -> list: + ctx = make_validation_context(recipe, **ctx_kwargs) + return [f for f in run_semantic_rules(ctx) if f.rule == rule] + + +# --------------------------------------------------------------------------- +# T8: food-truck-has-sentinel-stop (fires when missing) +# --------------------------------------------------------------------------- + + +def test_food_truck_has_sentinel_stop_rule_fires_on_missing_sentinel(): + """food-truck recipe with no sentinel in stop message triggers a finding.""" + recipe = _food_truck_recipe( + steps={"done": RecipeStep(action="stop", message="Promotion complete.")}, + ) + found = _findings(recipe, "food-truck-has-sentinel-stop") + assert found + assert found[0].severity == Severity.WARNING + + +# --------------------------------------------------------------------------- +# T9: food-truck-has-sentinel-stop (passes when present) +# --------------------------------------------------------------------------- + + +def test_food_truck_has_sentinel_stop_rule_passes_when_present(): + """food-truck recipe with sentinel in stop message produces no findings.""" + recipe = _food_truck_recipe( + steps={ + "done": RecipeStep(action="stop", message="Done. Emit your L3 sentinel JSON block.") + }, + ) + found = _findings(recipe, "food-truck-has-sentinel-stop") + assert found == [] + + +# --------------------------------------------------------------------------- +# T10: food-truck-has-sentinel-stop (skips standard) +# --------------------------------------------------------------------------- + + +def test_food_truck_has_sentinel_stop_rule_skips_standard(): + """Standard recipe produces no findings for the food-truck sentinel rule.""" + recipe = _standard_recipe() + found = _findings(recipe, "food-truck-has-sentinel-stop") + assert found == [] diff --git a/tests/recipe/test_rules_graph.py b/tests/recipe/test_rules_graph.py index e807537856..4bafb769fa 100644 --- a/tests/recipe/test_rules_graph.py +++ b/tests/recipe/test_rules_graph.py @@ -2,9 +2,13 @@ from __future__ import annotations +import pytest + from autoskillit.core import Severity from autoskillit.recipe.registry import run_semantic_rules -from autoskillit.recipe.schema import Recipe, RecipeStep, StepResultRoute +from autoskillit.recipe.schema import Recipe, RecipeStep, StepResultCondition, StepResultRoute + +pytestmark = [pytest.mark.layer("recipe"), pytest.mark.small] def _make_recipe(steps: dict[str, RecipeStep]) -> Recipe: @@ -57,8 +61,8 @@ def test_cycle_with_only_on_failure_exit_is_warning() -> None: assert cycle_findings[0].severity == Severity.WARNING -def test_cycle_with_retry_exit_is_clean() -> None: - """Cycle where retrying step's success path stays in cycle → WARNING (outer loop unbounded).""" +def test_cycle_with_retry_exit_is_error() -> None: + """Cycle where retrying step's success path stays in cycle → ERROR (outer loop unbounded).""" recipe = _make_recipe( { "A": RecipeStep( @@ -75,12 +79,12 @@ def test_cycle_with_retry_exit_is_clean() -> None: findings = run_semantic_rules(recipe) cycle_findings = [f for f in findings if f.rule == "unbounded-cycle"] assert len(cycle_findings) == 1 - assert cycle_findings[0].severity == Severity.WARNING + assert cycle_findings[0].severity == Severity.ERROR -def test_cycle_with_retry_exit_but_success_reenters_is_warning() -> None: +def test_cycle_with_retry_exit_but_success_reenters_is_error() -> None: """A→B(retries=2, on_exhausted=done)→C→A: B exits on exhaustion but - success path C→A re-enters the cycle. Must produce WARNING.""" + success path C→A re-enters the cycle. Must produce ERROR.""" recipe = _make_recipe( { "A": RecipeStep( @@ -113,8 +117,8 @@ def test_cycle_with_retry_exit_but_success_reenters_is_warning() -> None: findings = run_semantic_rules(recipe) cycle_findings = [f for f in findings if f.rule == "unbounded-cycle"] assert len(cycle_findings) >= 1 - # Must be WARNING (has conditional exit but no outer bound) - assert any(f.severity == Severity.WARNING for f in cycle_findings) + # Must be ERROR (has retry exit but no outer bound) + assert any(f.severity == Severity.ERROR for f in cycle_findings) def test_no_cycle_is_clean() -> None: @@ -131,6 +135,111 @@ def test_no_cycle_is_clean() -> None: assert cycle_findings == [] +def test_cycle_with_on_result_conditional_exit_is_clean() -> None: + """A→B→C(on_result: max_exceeded→done, else→A) → no unbounded-cycle finding. + + The on_result conditional exit provides a structural bound because one + route exits the cycle and one re-enters, meaning the exit condition is + evaluated on every iteration.""" + recipe = _make_recipe( + { + "A": RecipeStep( + tool="wait_for_ci", + with_args={"branch": "main", "timeout_seconds": 300}, + on_success="B", + on_failure="B", + ), + "B": RecipeStep( + tool="check_repo_merge_state", + with_args={"branch": "main", "cwd": "/tmp", "remote_url": ""}, + on_success="C", + on_failure="done", + ), + "C": RecipeStep( + tool="run_python", + with_args={"callable": "autoskillit.smoke_utils.check_loop_iteration"}, + on_result=StepResultRoute( + conditions=[ + StepResultCondition( + when="${{ result.max_exceeded }} == true", route="done" + ), + StepResultCondition(route="A"), + ] + ), + on_failure="done", + ), + "done": RecipeStep(action="stop", message="done"), + } + ) + findings = run_semantic_rules(recipe) + cycle_findings = [f for f in findings if f.rule == "unbounded-cycle"] + assert cycle_findings == [] + + +def test_cycle_with_on_result_all_routes_in_cycle_is_still_flagged() -> None: + """A→B(on_result: both routes inside cycle) → still flagged.""" + recipe = _make_recipe( + { + "A": RecipeStep( + tool="run_cmd", + with_args={"cmd": "echo a"}, + on_success="B", + on_failure="done", + ), + "B": RecipeStep( + tool="run_python", + with_args={"callable": "some.func"}, + on_result=StepResultRoute( + conditions=[ + StepResultCondition(when="${{ result.x }} == true", route="A"), + StepResultCondition(route="A"), + ] + ), + on_failure="done", + ), + "done": RecipeStep(action="stop", message="done"), + } + ) + findings = run_semantic_rules(recipe) + cycle_findings = [f for f in findings if f.rule == "unbounded-cycle"] + assert len(cycle_findings) == 1 + + +def test_cycle_with_loop_guard_step_is_clean() -> None: + """A cycle containing a check_loop_iteration guard with an exit route is bounded.""" + recipe = _make_recipe( + { + "step_a": RecipeStep( + tool="run_skill", + with_args={"skill_command": "/autoskillit:foo"}, + on_success="check_loop", + on_failure="done", + ), + "check_loop": RecipeStep( + tool="run_python", + with_args={ + "callable": "autoskillit.smoke_utils.check_loop_iteration", + "current_iteration": "${{ context.loop_count }}", + "max_iterations": "3", + }, + on_result=StepResultRoute( + conditions=[ + StepResultCondition( + when="${{ result.max_exceeded }} == true", + route="done", + ), + StepResultCondition(route="step_a"), + ] + ), + on_failure="done", + ), + "done": RecipeStep(action="stop", message="done"), + } + ) + findings = [f for f in run_semantic_rules(recipe) if f.rule == "unbounded-cycle"] + assert len(findings) == 0 + + # --------------------------------------------------------------------------- # on-result-missing-failure-route # --------------------------------------------------------------------------- @@ -429,3 +538,35 @@ def test_merge_base_literal_is_clean() -> None: findings = run_semantic_rules(recipe) flagged = [f for f in findings if f.rule == "merge-base-unpublished"] assert flagged == [] + + +# --------------------------------------------------------------------------- +# on-result-missing-tool-output-value +# --------------------------------------------------------------------------- + + +def test_on_result_missing_tool_output_value_catches_terminal_catchall() -> None: + """Recoverable tool output values falling through to a terminal step trigger WARNING.""" + recipe = _make_recipe( + { + "watch": RecipeStep( + tool="wait_for_ci", + on_result=StepResultRoute( + conditions=[ + StepResultCondition( + route="merge", when="${{ result.conclusion }} == success" + ), + StepResultCondition(route="fail"), + ], + ), + on_failure="fail", + ), + "merge": RecipeStep(on_success="done"), + "fail": RecipeStep(action="stop"), + "done": RecipeStep(action="stop"), + } + ) + findings = run_semantic_rules(recipe) + flagged = [f for f in findings if f.rule == "on-result-missing-tool-output-value"] + assert len(flagged) >= 1 + assert all(f.severity == Severity.WARNING for f in flagged) diff --git a/tests/recipe/test_rules_inline_script.py b/tests/recipe/test_rules_inline_script.py new file mode 100644 index 0000000000..d353cb5f9d --- /dev/null +++ b/tests/recipe/test_rules_inline_script.py @@ -0,0 +1,243 @@ +"""Tests for inline-script-in-cmd and inline-python-in-cmd lint rules.""" + +from __future__ import annotations + +import pytest + +from autoskillit.core import Severity +from autoskillit.recipe.validator import run_semantic_rules +from tests.recipe.conftest import _make_workflow + +pytestmark = [pytest.mark.layer("recipe"), pytest.mark.small] + + +def _make_recipe(steps: dict) -> object: + """Build a minimal Recipe from a dict of step dicts, with a terminal END step.""" + all_steps = {**steps, "END": {"action": "stop", "message": "Done"}} + return _make_workflow(all_steps) + + +# --------------------------------------------------------------------------- +# inline-script-in-cmd: ERROR cases +# --------------------------------------------------------------------------- + + +def test_inline_script_fires_on_control_flow(): + recipe = _make_recipe( + { + "step_a": { + "tool": "run_cmd", + "with": { + "cmd": 'VAR=1 && if [ "$VAR" -gt 0 ]; then echo "yes"; else echo "no"; fi' + }, + "on_success": "END", + } + } + ) + findings = run_semantic_rules(recipe) + codes = [f.rule for f in findings] + assert "inline-script-in-cmd" in codes + + +def test_inline_script_fires_on_for_loop(): + recipe = _make_recipe( + { + "step_a": { + "tool": "run_cmd", + "with": {"cmd": 'for i in $(seq 1 10); do echo "$i"; done'}, + "on_success": "END", + } + } + ) + findings = run_semantic_rules(recipe) + codes = [f.rule for f in findings] + assert "inline-script-in-cmd" in codes + + +def test_inline_script_fires_on_while_loop(): + recipe = _make_recipe( + { + "step_a": { + "tool": "run_cmd", + "with": {"cmd": "while true; do sleep 1; done"}, + "on_success": "END", + } + } + ) + findings = run_semantic_rules(recipe) + codes = [f.rule for f in findings] + assert "inline-script-in-cmd" in codes + + +def test_inline_script_fires_on_excessive_vars_and_chains(): + recipe = _make_recipe( + { + "step_a": { + "tool": "run_cmd", + "with": {"cmd": 'A=$(foo) && B=$(bar) && C=$(baz) && echo "$A $B $C" && cleanup'}, + "on_success": "END", + } + } + ) + findings = run_semantic_rules(recipe) + codes = [f.rule for f in findings] + assert "inline-script-in-cmd" in codes + + +def test_inline_script_fires_on_bash_builtins(): + recipe = _make_recipe( + { + "step_a": { + "tool": "run_cmd", + "with": {"cmd": 'declare -A map && map[key]=value && echo "${map[key]}"'}, + "on_success": "END", + } + } + ) + findings = run_semantic_rules(recipe) + codes = [f.rule for f in findings] + assert "inline-script-in-cmd" in codes + + +def test_inline_script_error_on_control_flow(): + recipe = _make_recipe( + { + "step_a": { + "tool": "run_cmd", + "with": {"cmd": 'if [ -f foo ]; then echo "yes"; else echo "no"; fi'}, + "on_success": "END", + } + } + ) + findings = run_semantic_rules(recipe) + inline_findings = [f for f in findings if f.rule == "inline-script-in-cmd"] + assert len(inline_findings) > 0 + assert inline_findings[0].severity == Severity.ERROR + + +# --------------------------------------------------------------------------- +# inline-script-in-cmd: WARNING cases +# --------------------------------------------------------------------------- + + +def test_inline_script_warning_on_multiline_with_vars(): + cmd = ( + "REMOTE=$(git remote) &&\n" + "SHA=$(git rev-parse HEAD) &&\n" + "BRANCH=$(git branch --show-current) &&\n" + 'echo "$REMOTE $SHA $BRANCH"' + ) + recipe = _make_recipe( + { + "step_a": { + "tool": "run_cmd", + "with": {"cmd": cmd}, + "on_success": "END", + } + } + ) + findings = run_semantic_rules(recipe) + inline_findings = [f for f in findings if f.rule == "inline-script-in-cmd"] + assert len(inline_findings) > 0 + assert inline_findings[0].severity == Severity.WARNING + + +# --------------------------------------------------------------------------- +# inline-python-in-cmd +# --------------------------------------------------------------------------- + + +def test_inline_python_fires_on_python3_c(): + recipe = _make_recipe( + { + "step_a": { + "tool": "run_cmd", + "with": {"cmd": 'python3 -c "import json; print(json.dumps({}))"'}, + "on_success": "END", + } + } + ) + findings = run_semantic_rules(recipe) + codes = [f.rule for f in findings] + assert "inline-python-in-cmd" in codes + + +# --------------------------------------------------------------------------- +# Clean cases: rule must NOT fire +# --------------------------------------------------------------------------- + + +def test_inline_script_clean_for_simple_commands(): + recipe = _make_recipe( + { + "step_a": { + "tool": "run_cmd", + "with": {"cmd": "git push -u origin HEAD"}, + "on_success": "END", + } + } + ) + findings = run_semantic_rules(recipe) + assert all(f.rule != "inline-script-in-cmd" for f in findings) + + +def test_inline_script_clean_for_flag_continuation(): + recipe = _make_recipe( + { + "step_a": { + "tool": "run_cmd", + "with": {"cmd": "gh pr create --base main --head feat --title 'foo' --body 'bar'"}, + "on_success": "END", + } + } + ) + findings = run_semantic_rules(recipe) + assert all(f.rule != "inline-script-in-cmd" for f in findings) + + +def test_inline_script_clean_for_jq_expressions(): + recipe = _make_recipe( + { + "step_a": { + "tool": "run_cmd", + "with": { + "cmd": ( + "gh pr view 42 --json statusCheckRollup " + "--jq '.statusCheckRollup | if length == 0 " + 'then "none" else "exists" end\'' + ) + }, + "on_success": "END", + } + } + ) + findings = run_semantic_rules(recipe) + assert all(f.rule != "inline-script-in-cmd" for f in findings) + + +def test_inline_script_fires_for_formerly_allowlisted_steps(): + recipe = _make_recipe( + { + "compute_branch": { + "tool": "run_cmd", + "with": {"cmd": 'SLUG="foo" && if [ -n "$SLUG" ]; then printf "%s" "$SLUG"; fi'}, + "on_success": "END", + } + } + ) + findings = run_semantic_rules(recipe) + assert any(f.rule == "inline-script-in-cmd" for f in findings) + + +def test_inline_script_ignores_non_run_cmd_steps(): + recipe = _make_recipe( + { + "step_a": { + "tool": "run_python", + "with": {"callable": "autoskillit.smoke_utils.check_loop_iteration"}, + "on_success": "END", + } + } + ) + findings = run_semantic_rules(recipe) + assert all(f.rule != "inline-script-in-cmd" for f in findings) diff --git a/tests/recipe/test_rules_inputs.py b/tests/recipe/test_rules_inputs.py index 1e9bf91c77..f626cf4d3c 100644 --- a/tests/recipe/test_rules_inputs.py +++ b/tests/recipe/test_rules_inputs.py @@ -5,9 +5,13 @@ import ast import pathlib +import pytest + from autoskillit.core import Severity from autoskillit.recipe.registry import run_semantic_rules -from autoskillit.recipe.schema import Recipe, RecipeStep +from autoskillit.recipe.schema import Recipe, RecipeIngredient, RecipeStep + +pytestmark = [pytest.mark.layer("recipe"), pytest.mark.small] def _make_recipe_with_skill_step(skill_command: str) -> Recipe: @@ -64,7 +68,8 @@ def test_rules_inputs_terminal_targets_match_schema(): # Structural check: rules_inputs.py must NOT hardcode sentinel strings via # .discard("done") or .discard("escalate"). It must use _TERMINAL_TARGETS instead. src_path = ( - pathlib.Path(__file__).parent.parent.parent / "src/autoskillit/recipe/rules_inputs.py" + pathlib.Path(__file__).parent.parent.parent + / "src/autoskillit/recipe/rules/rules_inputs.py" ) src = src_path.read_text() tree = ast.parse(src) @@ -167,3 +172,182 @@ def test_required_hidden_ingredient_without_default_no_warning(): findings = run_semantic_rules(recipe) warning_findings = [f for f in findings if f.rule == "required-ingredient-no-default"] assert len(warning_findings) == 0 + + +class TestResearchOutputModeEnum: + def test_bogus_default_fires_error(self): + recipe = Recipe( + name="research", + description="Test research recipe", + ingredients={ + "output_mode": RecipeIngredient( + description="Output mode for research results", + default="bogus", + ) + }, + steps={"done": RecipeStep(action="stop", message="Research complete.")}, + ) + findings = [f for f in run_semantic_rules(recipe) if f.rule == "research-output-mode-enum"] + assert len(findings) == 1 + assert findings[0].severity == Severity.ERROR + + def test_valid_pr_default_is_clean(self): + recipe = Recipe( + name="research", + description="Test research recipe", + ingredients={ + "output_mode": RecipeIngredient( + description="Output mode for research results", + default="pr", + ) + }, + steps={"done": RecipeStep(action="stop", message="Research complete.")}, + ) + findings = [f for f in run_semantic_rules(recipe) if f.rule == "research-output-mode-enum"] + assert len(findings) == 0 + + def test_no_output_mode_ingredient_is_clean(self): + recipe = Recipe( + name="research", + description="Test research recipe", + ingredients={}, + steps={"done": RecipeStep(action="stop", message="Research complete.")}, + ) + findings = [f for f in run_semantic_rules(recipe) if f.rule == "research-output-mode-enum"] + assert len(findings) == 0 + + def test_non_research_recipe_is_clean(self): + recipe = Recipe( + name="other", + description="Test other recipe", + ingredients={ + "output_mode": RecipeIngredient( + description="Output mode", + default="bogus", + ) + }, + steps={"done": RecipeStep(action="stop", message="Other recipe complete.")}, + ) + findings = [f for f in run_semantic_rules(recipe) if f.rule == "research-output-mode-enum"] + assert len(findings) == 0 + + +class TestIngredientTypeDefaultInvalid: + """T3: ingredient-type-default-invalid semantic rule tests.""" + + def test_type_integer_with_empty_default_fires_error(self): + """An integer-typed ingredient with default='' must produce an ERROR.""" + recipe = Recipe( + name="test", + description="Test recipe", + ingredients={ + "local_review_rounds": RecipeIngredient( + description="Number of local review rounds", + type="integer", + default="", + ) + }, + steps={"done": RecipeStep(action="stop", message="done")}, + ) + findings = run_semantic_rules(recipe) + rule_findings = [f for f in findings if f.rule == "ingredient-type-default-invalid"] + assert len(rule_findings) == 1, ( + "Expected ingredient-type-default-invalid ERROR for integer type with empty default" + ) + assert rule_findings[0].severity == Severity.ERROR + assert "local_review_rounds" in rule_findings[0].message + assert "integer" in rule_findings[0].message + + def test_type_integer_with_non_numeric_default_fires_error(self): + """An integer-typed ingredient with a non-parseable default must produce an ERROR.""" + recipe = Recipe( + name="test", + description="Test recipe", + ingredients={ + "count": RecipeIngredient( + description="A count", + type="integer", + default="not-a-number", + ) + }, + steps={"done": RecipeStep(action="stop", message="done")}, + ) + findings = run_semantic_rules(recipe) + rule_findings = [f for f in findings if f.rule == "ingredient-type-default-invalid"] + assert len(rule_findings) == 1 + assert rule_findings[0].severity == Severity.ERROR + + def test_type_integer_with_valid_int_default_is_clean(self): + """An integer-typed ingredient with a valid int default must NOT produce an error.""" + recipe = Recipe( + name="test", + description="Test recipe", + ingredients={ + "local_review_rounds": RecipeIngredient( + description="Number of local review rounds", + type="integer", + default="3", + ) + }, + steps={"done": RecipeStep(action="stop", message="done")}, + ) + findings = run_semantic_rules(recipe) + rule_findings = [f for f in findings if f.rule == "ingredient-type-default-invalid"] + assert len(rule_findings) == 0 + + def test_type_integer_required_no_default_is_clean(self): + """An integer-typed ingredient with required=True and default=None is clean + (caller must supply it).""" + recipe = Recipe( + name="test", + description="Test recipe", + ingredients={ + "local_review_rounds": RecipeIngredient( + description="Number of local review rounds", + type="integer", + required=True, + default=None, + ) + }, + steps={"done": RecipeStep(action="stop", message="done")}, + ) + findings = run_semantic_rules(recipe) + rule_findings = [f for f in findings if f.rule == "ingredient-type-default-invalid"] + assert len(rule_findings) == 0 + + def test_no_type_with_empty_default_is_clean(self): + """An ingredient with no type annotation and default='' is not flagged + by this rule (string auto-detect is allowed).""" + recipe = Recipe( + name="test", + description="Test recipe", + ingredients={ + "base_branch": RecipeIngredient( + description="Base branch", + default="", + ) + }, + steps={"done": RecipeStep(action="stop", message="done")}, + ) + findings = run_semantic_rules(recipe) + rule_findings = [f for f in findings if f.rule == "ingredient-type-default-invalid"] + assert len(rule_findings) == 0 + + def test_type_string_with_empty_default_is_clean(self): + """A string-typed ingredient with default='' is not flagged + (string auto-detect is allowed).""" + recipe = Recipe( + name="test", + description="Test recipe", + ingredients={ + "base_branch": RecipeIngredient( + description="Base branch", + type="string", + default="", + ) + }, + steps={"done": RecipeStep(action="stop", message="done")}, + ) + findings = run_semantic_rules(recipe) + rule_findings = [f for f in findings if f.rule == "ingredient-type-default-invalid"] + assert len(rule_findings) == 0 diff --git a/tests/recipe/test_rules_integration_predicate.py b/tests/recipe/test_rules_integration_predicate.py new file mode 100644 index 0000000000..c98a3e321b --- /dev/null +++ b/tests/recipe/test_rules_integration_predicate.py @@ -0,0 +1,118 @@ +from __future__ import annotations + +import pytest + +from autoskillit.core.types import Severity +from autoskillit.recipe.io import builtin_recipes_dir, load_recipe +from autoskillit.recipe.validator import run_semantic_rules +from tests.recipe.conftest import KNOWN_PART_B_VIOLATIONS as _KNOWN_PART_B_VIOLATIONS + +pytestmark = [pytest.mark.layer("recipe"), pytest.mark.small] + + +class TestRecipeIntegrationPredicateRouting: + """Integration tests: bundled recipes with predicate on_result validate correctly.""" + + @pytest.fixture(scope="class", autouse=True) + def _load_recipes(self, request) -> None: + request.cls.if_recipe = load_recipe(builtin_recipes_dir() / "remediation.yaml") + request.cls.ip_recipe = load_recipe(builtin_recipes_dir() / "implementation.yaml") + + def test_investigate_first_merge_step_has_predicate_on_result(self) -> None: + """The merge step in remediation.yaml has predicate on_result.""" + step = self.if_recipe.steps["merge"] + assert step.on_result is not None + assert len(step.on_result.conditions) == 6 + + cond0 = step.on_result.conditions[0] + assert cond0.when == "result.failed_step == 'dirty_tree'" + assert cond0.route == "assess" + + cond1 = step.on_result.conditions[1] + assert cond1.when == "result.failed_step == 'test_gate'" + assert cond1.route == "assess" + + cond2 = step.on_result.conditions[2] + assert cond2.when == "result.failed_step == 'post_rebase_test_gate'" + assert cond2.route == "assess" + + cond3 = step.on_result.conditions[3] + assert cond3.when == "result.failed_step == 'rebase'" + assert cond3.route == "rebase_conflict_fix" + + cond4 = step.on_result.conditions[4] + assert cond4.when == "result.error" + assert cond4.route == "release_issue_failure" + + cond5 = step.on_result.conditions[5] + assert cond5.when is None + assert cond5.route == "next_or_done" + + def test_investigate_first_merge_step_captures_worktree_path(self) -> None: + """The merge step captures worktree_path from result.worktree_path.""" + step = self.if_recipe.steps["merge"] + assert "worktree_path" in step.capture + assert "result.worktree_path" in step.capture["worktree_path"] + + def test_implementation_pipeline_merge_step_has_predicate_on_result(self) -> None: + """The merge step in implementation.yaml has predicate on_result.""" + step = self.ip_recipe.steps["merge"] + assert step.on_result is not None + assert len(step.on_result.conditions) == 6 + + cond0 = step.on_result.conditions[0] + assert cond0.when == "result.failed_step == 'dirty_tree'" + assert cond0.route == "fix" + + cond1 = step.on_result.conditions[1] + assert cond1.when == "result.failed_step == 'test_gate'" + assert cond1.route == "fix" + + cond2 = step.on_result.conditions[2] + assert cond2.when == "result.failed_step == 'post_rebase_test_gate'" + assert cond2.route == "fix" + + cond3 = step.on_result.conditions[3] + assert cond3.when == "result.failed_step == 'rebase'" + assert cond3.route == "rebase_conflict_fix" + + cond4 = step.on_result.conditions[4] + assert cond4.when == "result.error" + assert cond4.route == "release_issue_failure" + + cond5 = step.on_result.conditions[5] + assert cond5.when is None + assert cond5.route == "next_or_done" + + def test_implementation_pipeline_merge_step_captures_worktree_path(self) -> None: + """The merge step in implementation.yaml captures worktree_path.""" + step = self.ip_recipe.steps["merge"] + assert "worktree_path" in step.capture + assert "result.worktree_path" in step.capture["worktree_path"] + + def test_both_recipes_validate_cleanly(self) -> None: + """Both recipes have no structural errors after predicate routing changes.""" + + from autoskillit.recipe.validator import validate_recipe_structure + + if_errors = validate_recipe_structure(self.if_recipe) + assert if_errors == [], f"remediation.yaml has validation errors: {if_errors}" + + ip_errors = validate_recipe_structure(self.ip_recipe) + assert ip_errors == [], f"implementation.yaml has validation errors: {ip_errors}" + + def test_both_recipes_no_error_semantic_findings(self) -> None: + """Both recipes pass semantic rules with no ERROR-severity findings.""" + for recipe, name in [ + (self.if_recipe, "remediation"), + (self.ip_recipe, "implementation"), + ]: + findings = run_semantic_rules(recipe) + errors = [ + f + for f in findings + if f.severity == Severity.ERROR and f.rule not in _KNOWN_PART_B_VIOLATIONS + ] + assert errors == [], f"{name} has ERROR-severity semantic findings: " + str( + [(f.rule, f.step_name, f.message) for f in errors] + ) diff --git a/tests/recipe/test_rules_isolation.py b/tests/recipe/test_rules_isolation.py index 9ca21970ff..2d20a8ce38 100644 --- a/tests/recipe/test_rules_isolation.py +++ b/tests/recipe/test_rules_isolation.py @@ -5,11 +5,15 @@ from __future__ import annotations +import pytest + from autoskillit.core.types import Severity from autoskillit.recipe.io import builtin_recipes_dir, load_recipe from autoskillit.recipe.registry import _RULE_REGISTRY, run_semantic_rules from tests.recipe.conftest import _make_workflow +pytestmark = [pytest.mark.layer("recipe"), pytest.mark.small] + # --------------------------------------------------------------------------- # Test 1a: Rule registration # --------------------------------------------------------------------------- diff --git a/tests/recipe/test_rules_merge.py b/tests/recipe/test_rules_merge.py index 5409dbd8d4..cf6325034f 100644 --- a/tests/recipe/test_rules_merge.py +++ b/tests/recipe/test_rules_merge.py @@ -2,11 +2,16 @@ from __future__ import annotations +import pytest + from autoskillit.core import Severity +from autoskillit.recipe.io import builtin_recipes_dir, load_recipe from autoskillit.recipe.registry import run_semantic_rules -from autoskillit.recipe.rules_merge import _RECOVERABLE_FAILED_STEPS +from autoskillit.recipe.rules.rules_merge import _RECOVERABLE_FAILED_STEPS from autoskillit.recipe.schema import Recipe, RecipeStep, StepResultCondition, StepResultRoute +pytestmark = [pytest.mark.layer("recipe"), pytest.mark.small] + def _make_recipe(steps: dict[str, RecipeStep]) -> Recipe: """Minimal recipe factory for rules_merge tests.""" @@ -260,3 +265,400 @@ def test_multiple_merge_steps_each_checked_independently() -> None: flagged = [f for f in findings if f.rule == "merge-routing-incomplete"] assert len(flagged) == 1 assert flagged[0].step_name == "merge_bad" + + +# --------------------------------------------------------------------------- +# merge-fix-cycle-without-iteration-guard rule tests +# --------------------------------------------------------------------------- + + +def test_merge_fix_cycle_without_guard_fires() -> None: + """merge→fix→test→merge cycle without check_loop_iteration → ERROR.""" + recipe = _make_recipe( + { + "commit_guard": RecipeStep( + tool="run_cmd", + with_args={"cmd": "git commit -m 'x' || true"}, + on_success="merge", + on_failure="merge", + ), + "merge": RecipeStep( + tool="merge_worktree", + with_args={"worktree_path": "/tmp/wt", "base_branch": "main"}, + on_result=StepResultRoute( + conditions=[ + StepResultCondition(route="fix", when="result.failed_step == 'rebase'"), + ] + ), + on_success="done", + on_failure="done", + ), + "fix": RecipeStep( + tool="run_skill", + with_args={ + "skill_command": "/autoskillit:resolve-failures /tmp/wt", + "step_name": "fix", + }, + on_result=StepResultRoute( + conditions=[ + StepResultCondition(route="test", when="result.verdict == 'real_fix'"), + ] + ), + on_failure="done", + ), + "test": RecipeStep( + tool="test_check", + with_args={"worktree_path": "/tmp/wt"}, + on_success="commit_guard", + on_failure="fix", + ), + "done": RecipeStep(action="stop", message="done"), + } + ) + findings = run_semantic_rules(recipe) + flagged = [f for f in findings if f.rule == "merge-fix-cycle-without-iteration-guard"] + assert len(flagged) == 1, f"Expected 1 ERROR, got: {flagged}" + assert flagged[0].severity == Severity.ERROR + + +def test_merge_fix_cycle_with_guard_is_clean() -> None: + """merge→fix→test→check_merge_fix_loop→commit_guard→merge → no finding.""" + recipe = _make_recipe( + { + "commit_guard": RecipeStep( + tool="run_cmd", + with_args={"cmd": "git commit -m 'x' || true"}, + on_success="merge", + on_failure="merge", + ), + "merge": RecipeStep( + tool="merge_worktree", + with_args={"worktree_path": "/tmp/wt", "base_branch": "main"}, + on_result=StepResultRoute( + conditions=[ + StepResultCondition(route="fix", when="result.failed_step == 'rebase'"), + ] + ), + on_success="done", + on_failure="done", + ), + "fix": RecipeStep( + tool="run_skill", + with_args={ + "skill_command": "/autoskillit:resolve-failures /tmp/wt", + "step_name": "fix", + }, + on_result=StepResultRoute( + conditions=[ + StepResultCondition(route="test", when="result.verdict == 'real_fix'"), + ] + ), + on_failure="done", + ), + "test": RecipeStep( + tool="test_check", + with_args={"worktree_path": "/tmp/wt"}, + on_success="check_merge_fix_loop", + on_failure="fix", + ), + "check_merge_fix_loop": RecipeStep( + tool="run_python", + with_args={ + "callable": "autoskillit.smoke_utils.check_loop_iteration", + "current_iteration": "${{ context.merge_fix_count }}", + "max_iterations": "3", + }, + on_result=StepResultRoute( + conditions=[ + StepResultCondition( + route="release_issue_failure", + when="${{ result.max_exceeded }} == true", + ), + ] + ), + on_success="commit_guard", + on_failure="release_issue_failure", + ), + "release_issue_failure": RecipeStep(action="stop", message="failure"), + "done": RecipeStep(action="stop", message="done"), + } + ) + findings = run_semantic_rules(recipe) + flagged = [f for f in findings if f.rule == "merge-fix-cycle-without-iteration-guard"] + assert flagged == [], f"Unexpected finding: {flagged}" + + +@pytest.mark.parametrize("recipe_name", ["implementation", "remediation", "implementation-groups"]) +def test_bundled_recipes_merge_fix_cycle_guarded(recipe_name: str) -> None: + """All bundled recipes with merge→fix cycle must have check_merge_fix_loop guard.""" + recipe = load_recipe(builtin_recipes_dir() / f"{recipe_name}.yaml") + findings = run_semantic_rules(recipe) + flagged = [f for f in findings if f.rule == "merge-fix-cycle-without-iteration-guard"] + assert flagged == [], f"{recipe_name}: {flagged}" + + +# --------------------------------------------------------------------------- +# gh-pr-merge-silent-success-routing rule tests +# --------------------------------------------------------------------------- + + +def _make_gh_pr_merge_recipe(*, on_failure: str) -> Recipe: + """Minimal recipe with a critical gh-pr-merge run_cmd step.""" + return _make_recipe( + { + "merge_pr": RecipeStep( + tool="run_cmd", + with_args={"cmd": "gh pr merge '123' --squash --auto", "cwd": "/tmp/work"}, + on_success="done", + on_failure=on_failure, + ), + "register_clone_success": RecipeStep(action="stop", message="success"), + "release_issue_failure": RecipeStep(action="stop", message="failure"), + "verify_queue_enrollment": RecipeStep( + tool="wait_for_merge_queue", + with_args={}, + on_failure="release_issue_failure", + ), + "done": RecipeStep(action="stop", message="done"), + } + ) + + +def _make_cleanup_gh_pr_merge_recipe() -> Recipe: + """Recipe with an optional cleanup step that uses gh pr merge — exempt from the rule.""" + return _make_recipe( + { + "cleanup_merge": RecipeStep( + tool="run_cmd", + with_args={"cmd": "gh pr merge '123' --squash", "cwd": "/tmp/work"}, + on_success="register_clone_success", + on_failure="register_clone_success", + optional=True, + ), + "register_clone_success": RecipeStep(action="stop", message="success"), + "done": RecipeStep(action="stop", message="done"), + } + ) + + +def test_rule_fires_on_merge_cmd_failure_to_success_terminal() -> None: + """A gh-pr-merge step with on_failure=register_clone_success must produce an ERROR finding.""" + recipe = _make_gh_pr_merge_recipe(on_failure="register_clone_success") + findings = run_semantic_rules(recipe) + assert any( + f.rule == "gh-pr-merge-silent-success-routing" and f.severity == Severity.ERROR + for f in findings + ) + + +def test_rule_does_not_fire_on_optional_cleanup_step() -> None: + """Cleanup steps (optional=True) are exempt from the silent-success-degradation rule.""" + recipe = _make_cleanup_gh_pr_merge_recipe() + findings = run_semantic_rules(recipe) + assert not any(f.rule == "gh-pr-merge-silent-success-routing" for f in findings) + + +def test_rule_does_not_fire_on_correct_escalation_routing() -> None: + """A merge step routing on_failure to an escalation target produces no finding.""" + recipe = _make_gh_pr_merge_recipe(on_failure="release_issue_failure") + findings = run_semantic_rules(recipe) + assert not any(f.rule == "gh-pr-merge-silent-success-routing" for f in findings) + + +def test_rule_does_not_fire_on_verify_queue_enrollment_routing() -> None: + """A merge step routing on_failure to verify_queue_enrollment produces no finding.""" + recipe = _make_gh_pr_merge_recipe(on_failure="verify_queue_enrollment") + findings = run_semantic_rules(recipe) + assert not any(f.rule == "gh-pr-merge-silent-success-routing" for f in findings) + + +@pytest.mark.parametrize("recipe_name", ["implementation", "remediation", "implementation-groups"]) +def test_bundled_recipes_have_no_silent_success_degradation(recipe_name: str) -> None: + """No bundled recipe may have a merge step whose on_failure reaches a success terminal.""" + recipe = load_recipe(builtin_recipes_dir() / f"{recipe_name}.yaml") + findings = run_semantic_rules(recipe) + silent_success = [f for f in findings if f.rule == "gh-pr-merge-silent-success-routing"] + assert silent_success == [], ( + f"Silent success degradation found in {recipe_name}: {silent_success}" + ) + + +# --------------------------------------------------------------------------- +# release-issue-on-unconfirmed-merge rule tests +# --------------------------------------------------------------------------- + + +def test_rule_fires_when_release_issue_reachable_from_timeout() -> None: + """release_issue reachable from a merge-wait timeout exit → ERROR finding.""" + recipe = _make_recipe( + { + "wait_queue": RecipeStep( + tool="wait_for_merge_queue", + with_args={}, + on_result=StepResultRoute( + conditions=[ + StepResultCondition( + route="release_timeout", + when="result.pr_state == timeout", + ) + ] + ), + on_success="done", + on_failure="release_timeout", + ), + "release_timeout": RecipeStep( + tool="release_issue", + with_args={"issue_url": "owner/repo#1"}, + on_success="done", + on_failure="done", + ), + "done": RecipeStep(action="stop", message="done"), + } + ) + findings = run_semantic_rules(recipe) + assert any(f.rule == "release-issue-on-unconfirmed-merge" for f in findings), ( + f"Expected release-issue-on-unconfirmed-merge finding, got: {findings}" + ) + + +def test_rule_does_not_fire_when_register_clone_unconfirmed_used() -> None: + """Timeout routes to register_clone_status(status=unconfirmed) → no finding.""" + recipe = _make_recipe( + { + "wait_queue": RecipeStep( + tool="wait_for_merge_queue", + with_args={}, + on_result=StepResultRoute( + conditions=[ + StepResultCondition( + route="register_unconf", + when="result.pr_state == timeout", + ) + ] + ), + on_success="done", + on_failure="register_unconf", + ), + "register_unconf": RecipeStep( + tool="register_clone_status", + with_args={"clone_path": "/some/path", "status": "unconfirmed"}, + on_success="done", + on_failure="done", + ), + "done": RecipeStep(action="stop", message="done"), + } + ) + findings = run_semantic_rules(recipe) + assert not any(f.rule == "release-issue-on-unconfirmed-merge" for f in findings), ( + f"Unexpected release-issue-on-unconfirmed-merge finding: " + f"{[f for f in findings if f.rule == 'release-issue-on-unconfirmed-merge']}" + ) + + +# --------------------------------------------------------------------------- +# merge-enrollment-auto-consistency rule tests +# --------------------------------------------------------------------------- + + +def test_rule_flags_auto_step_reachable_from_no_auto_route() -> None: + """A gh pr merge --auto step reachable from an auto_merge_available=false + routing condition must emit a finding.""" + recipe = _make_recipe( + { + "route_queue_mode": RecipeStep( + action="route", + on_result=StepResultRoute( + conditions=[ + StepResultCondition( + route="enroll_with_auto", + when="context.auto_merge_available == 'true'", + ), + StepResultCondition( + route="enroll_no_auto", + when="context.auto_merge_available == 'false'", + ), + ], + ), + ), + "enroll_with_auto": RecipeStep( + tool="run_cmd", + with_args={"cmd": "gh pr merge '42' --squash --auto", "cwd": "/tmp"}, + on_success="wait_queue", + ), + "enroll_no_auto": RecipeStep( + tool="run_cmd", + with_args={"cmd": "gh pr merge '42' --squash", "cwd": "/tmp"}, + on_success="wait_queue", + on_failure="reenter", + ), + "wait_queue": RecipeStep( + tool="wait_for_merge_queue", + with_args={}, + on_success="done", + on_failure="reenter", + ), + "reenter": RecipeStep( + tool="run_cmd", + with_args={"cmd": "gh pr merge '42' --squash --auto", "cwd": "/tmp"}, + on_success="wait_queue", + ), + "done": RecipeStep(action="stop", message="done"), + } + ) + findings = run_semantic_rules(recipe) + flagged = [f for f in findings if f.rule == "merge-enrollment-auto-consistency"] + assert len(flagged) >= 1 + flagged_steps = {f.step_name for f in flagged} + assert "reenter" in flagged_steps + + +@pytest.mark.parametrize("recipe_name", ["implementation", "remediation", "implementation-groups"]) +def test_merge_enrollment_auto_consistency_passes_after_migration(recipe_name: str) -> None: + """After migration, no recipe should have --auto steps reachable from + auto_merge_available=false routing arms.""" + recipe = load_recipe(builtin_recipes_dir() / f"{recipe_name}.yaml") + findings = run_semantic_rules(recipe) + auto_findings = [f for f in findings if f.rule == "merge-enrollment-auto-consistency"] + assert auto_findings == [], f"{recipe_name}: {auto_findings}" + + +def test_rule_passes_when_auto_steps_only_reachable_from_auto_route() -> None: + """Steps using --auto that are only reachable from auto_merge_available=true + routing conditions should not emit findings.""" + recipe = _make_recipe( + { + "route_queue_mode": RecipeStep( + action="route", + on_result=StepResultRoute( + conditions=[ + StepResultCondition( + route="enroll_with_auto", + when="context.auto_merge_available == 'true'", + ), + StepResultCondition( + route="enroll_no_auto", + when="context.auto_merge_available == 'false'", + ), + ], + ), + ), + "enroll_with_auto": RecipeStep( + tool="run_cmd", + with_args={"cmd": "gh pr merge '42' --squash --auto", "cwd": "/tmp"}, + on_success="done", + ), + "enroll_no_auto": RecipeStep( + tool="enqueue_pr", + with_args={ + "pr_number": "42", + "target_branch": "main", + "auto_merge_available": "false", + }, + on_success="done", + ), + "done": RecipeStep(action="stop", message="done"), + } + ) + findings = run_semantic_rules(recipe) + flagged = [f for f in findings if f.rule == "merge-enrollment-auto-consistency"] + assert flagged == [] diff --git a/tests/recipe/test_rules_merge_base_unpublished.py b/tests/recipe/test_rules_merge_base_unpublished.py new file mode 100644 index 0000000000..d28e6bfe81 --- /dev/null +++ b/tests/recipe/test_rules_merge_base_unpublished.py @@ -0,0 +1,116 @@ +from __future__ import annotations + +import pytest + +from autoskillit.core.types import Severity +from autoskillit.recipe.io import builtin_recipes_dir, load_recipe +from autoskillit.recipe.validator import run_semantic_rules +from tests.recipe.conftest import _make_workflow + +pytestmark = [pytest.mark.layer("recipe"), pytest.mark.small] + + +class TestMergeBaseUnpublishedRule: + """Tests for the merge-base-unpublished semantic rule.""" + + def test_merge_base_unpublished_rule_fires_when_push_absent(self) -> None: + """merge-base-unpublished ERROR fires when merge_worktree.base_branch + is a context variable without a preceding push_to_remote step.""" + recipe = _make_workflow( + { + "clone": { + "tool": "clone_repo", + "with": {"source_dir": ".", "run_name": "test"}, + "on_success": "create_branch", + }, + "create_branch": { + "tool": "run_cmd", + "with": {"cmd": "echo test", "cwd": "/tmp"}, + "capture": {"merge_target": "${{ result.stdout }}"}, + "on_success": "merge_step", + }, + "merge_step": { + "tool": "merge_worktree", + "with": { + "worktree_path": "${{ context.worktree_path }}", + "base_branch": "${{ context.merge_target }}", + }, + "on_success": "done", + }, + "done": {"action": "stop", "message": "done"}, + } + ) + findings = run_semantic_rules(recipe) + rule_findings = [f for f in findings if f.rule == "merge-base-unpublished"] + assert len(rule_findings) >= 1 + assert any(f.severity == Severity.ERROR for f in rule_findings) + + def test_merge_base_unpublished_rule_passes_when_push_precedes_merge(self) -> None: + """merge-base-unpublished does NOT fire when push_to_remote appears + on the path to merge_worktree for the same context variable.""" + recipe = _make_workflow( + { + "clone": { + "tool": "clone_repo", + "with": {"source_dir": ".", "run_name": "test"}, + "on_success": "create_branch", + }, + "create_branch": { + "tool": "run_cmd", + "with": {"cmd": "echo test", "cwd": "/tmp"}, + "capture": {"merge_target": "${{ result.stdout }}"}, + "on_success": "push_target", + }, + "push_target": { + "tool": "push_to_remote", + "with": { + "clone_path": "/tmp", + "branch": "${{ context.merge_target }}", + "remote_url": "https://example.com/repo.git", + }, + "on_success": "merge_step", + }, + "merge_step": { + "tool": "merge_worktree", + "with": { + "worktree_path": "${{ context.worktree_path }}", + "base_branch": "${{ context.merge_target }}", + }, + "on_success": "done", + }, + "done": {"action": "stop", "message": "done"}, + } + ) + findings = run_semantic_rules(recipe) + assert not any(f.rule == "merge-base-unpublished" for f in findings) + + def test_merge_base_unpublished_rule_does_not_fire_for_literal_branch(self) -> None: + """merge-base-unpublished does NOT fire when base_branch is a + literal string — literals like 'main' are always published.""" + recipe = _make_workflow( + { + "start": { + "tool": "run_cmd", + "with": {"cmd": "echo ok", "cwd": "/tmp"}, + "on_success": "merge_step", + }, + "merge_step": { + "tool": "merge_worktree", + "with": { + "worktree_path": "${{ context.worktree_path }}", + "base_branch": "main", + }, + "on_success": "done", + }, + "done": {"action": "stop", "message": "done"}, + } + ) + findings = run_semantic_rules(recipe) + assert not any(f.rule == "merge-base-unpublished" for f in findings) + + def test_implementation_pipeline_satisfies_push_before_merge_contract(self) -> None: + """implementation.yaml must pass the merge-base-unpublished + rule after the push_merge_target step is added.""" + recipe = load_recipe(builtin_recipes_dir() / "implementation.yaml") + findings = run_semantic_rules(recipe) + assert not any(f.rule == "merge-base-unpublished" for f in findings) diff --git a/tests/recipe/test_rules_merge_failure_domain.py b/tests/recipe/test_rules_merge_failure_domain.py new file mode 100644 index 0000000000..9eff19fa86 --- /dev/null +++ b/tests/recipe/test_rules_merge_failure_domain.py @@ -0,0 +1,190 @@ +import pytest + +from autoskillit.recipe.validator import run_semantic_rules +from tests.recipe.conftest import _make_workflow + +pytestmark = [pytest.mark.layer("recipe"), pytest.mark.small] + + +class TestMergeFailureSkillDomainMismatch: + """Tests for the merge-failure-skill-domain-mismatch semantic rule.""" + + def test_rebase_routed_to_resolve_failures_fires(self): + """ERROR when failed_step == 'rebase' routes to resolve-failures.""" + recipe = _make_workflow( + { + "merge": { + "tool": "merge_worktree", + "with": {"worktree_path": "/tmp/wt", "base_branch": "main"}, + "on_result": [ + {"when": "result.failed_step == 'dirty_tree'", "route": "fix"}, + {"when": "result.failed_step == 'test_gate'", "route": "fix"}, + {"when": "result.failed_step == 'post_rebase_test_gate'", "route": "fix"}, + {"when": "result.failed_step == 'rebase'", "route": "fix"}, + {"when": "result.error", "route": "escalate"}, + {"route": "done"}, + ], + }, + "fix": { + "tool": "run_skill", + "with": { + "skill_command": "/autoskillit:resolve-failures /tmp/wt /tmp/plan main" + }, + "on_success": "done", + }, + "done": {"action": "stop", "message": "Done."}, + "escalate": {"action": "stop", "message": "Escalate."}, + } + ) + findings = run_semantic_rules(recipe) + errors = [f for f in findings if f.rule == "merge-failure-skill-domain-mismatch"] + assert len(errors) == 1 + assert "rebase" in errors[0].message + assert "resolve-merge-conflicts" in errors[0].message + + def test_rebase_routed_to_resolve_merge_conflicts_is_clean(self): + """No finding when rebase routes to resolve-merge-conflicts.""" + recipe = _make_workflow( + { + "merge": { + "tool": "merge_worktree", + "with": {"worktree_path": "/tmp/wt", "base_branch": "main"}, + "on_result": [ + {"when": "result.failed_step == 'dirty_tree'", "route": "fix"}, + {"when": "result.failed_step == 'test_gate'", "route": "fix"}, + {"when": "result.failed_step == 'post_rebase_test_gate'", "route": "fix"}, + {"when": "result.failed_step == 'rebase'", "route": "rebase_fix"}, + {"when": "result.error", "route": "escalate"}, + {"route": "done"}, + ], + }, + "fix": { + "tool": "run_skill", + "with": { + "skill_command": "/autoskillit:resolve-failures /tmp/wt /tmp/plan main" + }, + "on_success": "done", + }, + "rebase_fix": { + "tool": "run_skill", + "with": { + "skill_command": ( + "/autoskillit:resolve-merge-conflicts /tmp/wt /tmp/plan main" + ) + }, + "on_success": "done", + }, + "done": {"action": "stop", "message": "Done."}, + "escalate": {"action": "stop", "message": "Escalate."}, + } + ) + findings = run_semantic_rules(recipe) + errors = [f for f in findings if f.rule == "merge-failure-skill-domain-mismatch"] + assert errors == [] + + def test_code_failure_routed_to_resolve_merge_conflicts_fires(self): + """ERROR when dirty_tree routes to resolve-merge-conflicts.""" + recipe = _make_workflow( + { + "merge": { + "tool": "merge_worktree", + "with": {"worktree_path": "/tmp/wt", "base_branch": "main"}, + "on_result": [ + {"when": "result.failed_step == 'dirty_tree'", "route": "conflict_fix"}, + {"when": "result.failed_step == 'test_gate'", "route": "fix"}, + {"when": "result.failed_step == 'post_rebase_test_gate'", "route": "fix"}, + {"when": "result.failed_step == 'rebase'", "route": "conflict_fix"}, + {"when": "result.error", "route": "escalate"}, + {"route": "done"}, + ], + }, + "fix": { + "tool": "run_skill", + "with": { + "skill_command": "/autoskillit:resolve-failures /tmp/wt /tmp/plan main" + }, + "on_success": "done", + }, + "conflict_fix": { + "tool": "run_skill", + "with": { + "skill_command": ( + "/autoskillit:resolve-merge-conflicts /tmp/wt /tmp/plan main" + ) + }, + "on_success": "done", + }, + "done": {"action": "stop", "message": "Done."}, + "escalate": {"action": "stop", "message": "Escalate."}, + } + ) + findings = run_semantic_rules(recipe) + errors = [f for f in findings if f.rule == "merge-failure-skill-domain-mismatch"] + assert len(errors) == 1 + assert "dirty_tree" in errors[0].message + assert "resolve-failures" in errors[0].message + + def test_rule_does_not_fire_for_non_merge_worktree(self): + """Rule is scoped to merge_worktree steps only.""" + recipe = _make_workflow( + { + "run": { + "tool": "run_skill", + "with": {"skill_command": "/autoskillit:implement-worktree /tmp/wt"}, + "on_result": [ + {"when": "result.error", "route": "done"}, + {"route": "done"}, + ], + }, + "done": {"action": "stop", "message": "Done."}, + } + ) + findings = run_semantic_rules(recipe) + errors = [f for f in findings if f.rule == "merge-failure-skill-domain-mismatch"] + assert errors == [] + + +@pytest.mark.medium +class TestBundledRecipesPassFailureDomainCheck: + """Integration tests: bundled recipes pass the new rule.""" + + @pytest.mark.parametrize( + "recipe_name", + ["implementation", "remediation", "implementation-groups"], + ) + def test_bundled_recipes_pass_failure_domain_skill_check(self, recipe_name): + """All three merge_worktree recipes route rebase to resolve-merge-conflicts.""" + from autoskillit.recipe.io import builtin_recipes_dir, load_recipe + + recipe = load_recipe(builtin_recipes_dir() / f"{recipe_name}.yaml") + findings = run_semantic_rules(recipe) + errors = [f for f in findings if f.rule == "merge-failure-skill-domain-mismatch"] + assert errors == [], f"{recipe_name}: {[e.message for e in errors]}" + + @pytest.mark.parametrize( + "recipe_name,merge_step_name", + [ + ("implementation", "merge"), + ("remediation", "merge"), + ("implementation-groups", "merge"), + ], + ) + def test_rebase_routes_to_merge_conflict_skill(self, recipe_name, merge_step_name): + """rebase condition in each recipe routes to a step invoking resolve-merge-conflicts.""" + from autoskillit.recipe.io import builtin_recipes_dir, load_recipe + + recipe = load_recipe(builtin_recipes_dir() / f"{recipe_name}.yaml") + merge_step = recipe.steps[merge_step_name] + if merge_step.on_result is None or not merge_step.on_result.conditions: + pytest.skip("merge step has no on_result conditions") + found_rebase = False + for cond in merge_step.on_result.conditions: + if cond.when and "rebase" in cond.when and "post_rebase" not in cond.when: + found_rebase = True + target_step = recipe.steps[cond.route] + skill_cmd = (target_step.with_args or {}).get("skill_command", "") + assert "resolve-merge-conflicts" in skill_cmd, ( + f"{recipe_name}: rebase routes to '{cond.route}' which invokes " + f"'{skill_cmd}', expected resolve-merge-conflicts" + ) + assert found_rebase, f"{recipe_name}: no rebase condition found in merge step" diff --git a/tests/recipe/test_rules_merge_queue_push.py b/tests/recipe/test_rules_merge_queue_push.py new file mode 100644 index 0000000000..8cc542ed3a --- /dev/null +++ b/tests/recipe/test_rules_merge_queue_push.py @@ -0,0 +1,139 @@ +"""Tests for push-after-queue-requires-queued-branch-route semantic rule.""" + +from __future__ import annotations + +import pytest + +from autoskillit.recipe.registry import run_semantic_rules +from autoskillit.recipe.schema import ( + Recipe, + RecipeStep, + StepResultCondition, + StepResultRoute, +) + +pytestmark = [pytest.mark.layer("recipe"), pytest.mark.small] + +_RULE_NAME = "push-after-queue-requires-queued-branch-route" + + +def _make_recipe_missing_queued_branch_route() -> Recipe: + """Minimal recipe: wait_for_queue → eject → push with no queued_branch route.""" + steps = { + "wait_for_queue": RecipeStep( + name="wait_for_queue", + tool="wait_for_merge_queue", + on_result=StepResultRoute( + conditions=[ + StepResultCondition( + when="${{ result.pr_state }} == 'ejected'", + route="queue_ejected_fix", + ), + ] + ), + ), + "queue_ejected_fix": RecipeStep( + name="queue_ejected_fix", + tool="run_python", + on_success="re_push_queue_fix", + ), + "re_push_queue_fix": RecipeStep( + name="re_push_queue_fix", + tool="push_to_remote", + on_success="done", + on_failure="terminal_failure", + ), + "terminal_failure": RecipeStep( + name="terminal_failure", + action="stop", + ), + "done": RecipeStep( + name="done", + action="stop", + ), + } + return Recipe( + name="test-missing-route", + description="test recipe missing queued_branch route", + steps=steps, + ) + + +def _make_recipe_with_queued_branch_route() -> Recipe: + """Minimal recipe: push has on_failure → classify_push_failure with queued_branch route.""" + steps = { + "wait_for_queue": RecipeStep( + name="wait_for_queue", + tool="wait_for_merge_queue", + on_result=StepResultRoute( + conditions=[ + StepResultCondition( + when="${{ result.pr_state }} == 'ejected'", + route="queue_ejected_fix", + ), + ] + ), + ), + "queue_ejected_fix": RecipeStep( + name="queue_ejected_fix", + tool="run_python", + on_success="re_push_queue_fix", + ), + "re_push_queue_fix": RecipeStep( + name="re_push_queue_fix", + tool="push_to_remote", + on_success="done", + on_failure="classify_push_failure", + ), + "classify_push_failure": RecipeStep( + name="classify_push_failure", + action="route", + on_result=StepResultRoute( + conditions=[ + StepResultCondition( + when="${{ context.push_error_type }} == queued_branch", + route="wait_dequeue_retry", + ), + StepResultCondition( + when=None, + route="terminal_failure", + ), + ] + ), + ), + "wait_dequeue_retry": RecipeStep( + name="wait_dequeue_retry", + tool="wait_for_merge_queue", + on_success="re_push_queue_fix", + ), + "terminal_failure": RecipeStep( + name="terminal_failure", + action="stop", + ), + "done": RecipeStep( + name="done", + action="stop", + ), + } + return Recipe( + name="test-with-route", + description="test recipe with queued_branch route", + steps=steps, + ) + + +def test_rule_fires_when_queued_branch_route_missing(): + """push_to_remote reachable from ejection route without queued_branch triggers finding.""" + recipe = _make_recipe_missing_queued_branch_route() + findings = run_semantic_rules(recipe) + matching = [f for f in findings if f.rule == _RULE_NAME] + assert len(matching) == 1 + assert "re_push_queue_fix" in matching[0].step_name + + +def test_rule_passes_when_queued_branch_route_present(): + """push_to_remote with queued_branch route in failure chain produces no finding.""" + recipe = _make_recipe_with_queued_branch_route() + findings = run_semantic_rules(recipe) + matching = [f for f in findings if f.rule == _RULE_NAME] + assert matching == [] diff --git a/tests/recipe/test_rules_merge_routing_incomplete.py b/tests/recipe/test_rules_merge_routing_incomplete.py new file mode 100644 index 0000000000..ae6eb34042 --- /dev/null +++ b/tests/recipe/test_rules_merge_routing_incomplete.py @@ -0,0 +1,146 @@ +from __future__ import annotations + +import pytest + +from autoskillit.recipe.schema import Recipe +from autoskillit.recipe.validator import run_semantic_rules +from tests.recipe.conftest import _make_workflow + +pytestmark = [pytest.mark.layer("recipe"), pytest.mark.small] + + +class TestMergeRoutingIncompleteRule: + """Tests for the merge-routing-incomplete semantic rule (RMR*).""" + + def _make_merge_step(self, conditions: list[dict]) -> Recipe: + """Build a minimal recipe with a merge_worktree step using predicate on_result.""" + return _make_workflow( + { + "merge": { + "tool": "merge_worktree", + "with": {"worktree_path": "/tmp/wt", "base_branch": "main"}, + "on_result": conditions, + "capture": {"cleanup_succeeded": "${{ result.cleanup_succeeded }}"}, + }, + "recover": {"action": "stop", "message": "Recover."}, + "done": {"action": "stop", "message": "Done."}, + "escalate": {"action": "stop", "message": "Escalate."}, + } + ) + + def test_rmr1_fires_when_test_gate_missing(self): + """RMR1: ERROR when test_gate is not explicitly routed.""" + recipe = self._make_merge_step( + [ + {"when": "result.failed_step == 'dirty_tree'", "route": "recover"}, + {"when": "result.failed_step == 'post_rebase_test_gate'", "route": "recover"}, + {"when": "result.failed_step == 'rebase'", "route": "recover"}, + {"when": "result.error", "route": "escalate"}, + {"route": "done"}, + ] + ) + findings = run_semantic_rules(recipe) + errors = [f for f in findings if f.rule == "merge-routing-incomplete"] + assert len(errors) == 1 + assert "test_gate" in errors[0].message + + def test_rmr2_fires_when_post_rebase_test_gate_missing(self): + """RMR2: ERROR when post_rebase_test_gate is not explicitly routed.""" + recipe = self._make_merge_step( + [ + {"when": "result.failed_step == 'dirty_tree'", "route": "recover"}, + {"when": "result.failed_step == 'test_gate'", "route": "recover"}, + {"when": "result.failed_step == 'rebase'", "route": "recover"}, + {"when": "result.error", "route": "escalate"}, + {"route": "done"}, + ] + ) + findings = run_semantic_rules(recipe) + errors = [f for f in findings if f.rule == "merge-routing-incomplete"] + assert len(errors) == 1 + assert "post_rebase_test_gate" in errors[0].message + + def test_rmr3_fires_when_rebase_missing(self): + """RMR3: ERROR when rebase is not explicitly routed.""" + recipe = self._make_merge_step( + [ + {"when": "result.failed_step == 'dirty_tree'", "route": "recover"}, + {"when": "result.failed_step == 'test_gate'", "route": "recover"}, + {"when": "result.failed_step == 'post_rebase_test_gate'", "route": "recover"}, + {"when": "result.error", "route": "escalate"}, + {"route": "done"}, + ] + ) + findings = run_semantic_rules(recipe) + errors = [f for f in findings if f.rule == "merge-routing-incomplete"] + assert len(errors) == 1 + assert "rebase" in errors[0].message + + def test_rmr4_clears_when_all_four_covered(self): + """RMR4: No finding when all recoverable values are explicitly routed.""" + recipe = self._make_merge_step( + [ + {"when": "result.failed_step == 'dirty_tree'", "route": "recover"}, + {"when": "result.failed_step == 'test_gate'", "route": "recover"}, + {"when": "result.failed_step == 'post_rebase_test_gate'", "route": "recover"}, + {"when": "result.failed_step == 'rebase'", "route": "recover"}, + {"when": "result.error", "route": "escalate"}, + {"route": "done"}, + ] + ) + findings = run_semantic_rules(recipe) + errors = [f for f in findings if f.rule == "merge-routing-incomplete"] + assert errors == [] + + def test_rmr7_fires_when_dirty_tree_missing(self): + """RMR7: ERROR when dirty_tree is not explicitly routed.""" + recipe = self._make_merge_step( + [ + {"when": "result.failed_step == 'test_gate'", "route": "recover"}, + {"when": "result.failed_step == 'post_rebase_test_gate'", "route": "recover"}, + {"when": "result.failed_step == 'rebase'", "route": "recover"}, + {"when": "result.error", "route": "escalate"}, + {"route": "done"}, + ] + ) + findings = run_semantic_rules(recipe) + errors = [f for f in findings if f.rule == "merge-routing-incomplete"] + assert len(errors) == 1 + assert "dirty_tree" in errors[0].message + + def test_rmr5_does_not_fire_for_non_merge_worktree_step(self): + """RMR5: Rule is scoped to merge_worktree steps only.""" + recipe = _make_workflow( + { + "run": { + "tool": "run_skill", + "with": {"skill_command": "/autoskillit:implement-worktree", "cwd": "/tmp"}, + "on_result": [ + {"when": "result.error", "route": "done"}, + {"route": "done"}, + ], + }, + "done": {"action": "stop", "message": "Done."}, + } + ) + findings = run_semantic_rules(recipe) + errors = [f for f in findings if f.rule == "merge-routing-incomplete"] + assert errors == [] + + def test_rmr6_does_not_fire_when_no_on_result(self): + """RMR6: Rule is silent for merge_worktree steps without on_result.""" + recipe = _make_workflow( + { + "merge": { + "tool": "merge_worktree", + "with": {"worktree_path": "/tmp/wt", "base_branch": "main"}, + "on_success": "done", + "on_failure": "escalate", + }, + "done": {"action": "stop", "message": "Done."}, + "escalate": {"action": "stop", "message": "Escalate."}, + } + ) + findings = run_semantic_rules(recipe) + errors = [f for f in findings if f.rule == "merge-routing-incomplete"] + assert errors == [] diff --git a/tests/recipe/test_rules_multipart_iteration.py b/tests/recipe/test_rules_multipart_iteration.py new file mode 100644 index 0000000000..1a5ce9bf72 --- /dev/null +++ b/tests/recipe/test_rules_multipart_iteration.py @@ -0,0 +1,189 @@ +from __future__ import annotations + +import pytest + +from autoskillit.recipe.schema import Recipe, RecipeStep, StepResultRoute +from autoskillit.recipe.validator import run_semantic_rules + +pytestmark = [pytest.mark.layer("recipe"), pytest.mark.small] + + +class TestMultipartIterationRule: + def test_mi1_multipart_rule_warns_on_missing_glob_note(self) -> None: + """T_MI1: multipart-glob-note fires when make-plan step has no *_part_*.md in note.""" + recipe = Recipe( + name="test-recipe", + description="test", + ingredients={}, + steps={ + "plan": RecipeStep( + tool="run_skill", + with_args={"skill_command": "/autoskillit:make-plan inputs.task"}, + on_success="verify", + note="Produces a plan file.", + ), + "verify": RecipeStep( + tool="run_skill", + with_args={"skill_command": "/autoskillit:dry-walkthrough context.plan_path"}, + on_success="done", + ), + "done": RecipeStep(action="stop", message="Done"), + }, + kitchen_rules=[], + ) + warnings = run_semantic_rules(recipe) + rule_names = [w.rule for w in warnings] + assert "multipart-glob-note" in rule_names + + def test_mi2_multipart_rule_passes_compliant_recipe(self) -> None: + """T_MI2: Validator emits no multipart warnings when all conventions are present.""" + recipe = Recipe( + name="test-recipe", + description="test", + ingredients={}, + steps={ + "plan": RecipeStep( + tool="run_skill", + with_args={"skill_command": "/autoskillit:make-plan inputs.task"}, + on_success="verify", + note="Glob plan_dir for *_part_*.md or single plan file.", + ), + "verify": RecipeStep( + tool="run_skill", + with_args={"skill_command": "/autoskillit:dry-walkthrough context.plan_path"}, + on_success="next_or_done", + ), + "next_or_done": RecipeStep( + action="route", + on_result=StepResultRoute( + field="next", routes={"more_parts": "verify", "all_done": "done"} + ), + ), + "done": RecipeStep(action="stop", message="Done"), + }, + kitchen_rules=["SEQUENTIAL EXECUTION: complete full cycle per part before advancing."], + ) + warnings = run_semantic_rules(recipe) + rule_names = [w.rule for w in warnings] + assert "multipart-glob-note" not in rule_names + assert "multipart-sequential-kitchen-rule" not in rule_names + assert "multipart-route-back" not in rule_names + + def test_mi3_multipart_rule_passes_non_verify_backroute(self) -> None: + """T_MI3: multipart-route-back must NOT fire when more_parts routes to dry_walkthrough.""" + recipe = Recipe( + name="test-recipe", + description="test", + ingredients={}, + steps={ + "rectify": RecipeStep( + tool="run_skill", + with_args={"skill_command": "/autoskillit:rectify context.investigation_path"}, + on_success="dry_walkthrough", + note="Glob plan_dir for *_part_*.md or single plan file.", + ), + "dry_walkthrough": RecipeStep( + tool="run_skill", + with_args={"skill_command": "/autoskillit:dry-walkthrough context.plan_path"}, + on_success="next_or_done", + ), + "next_or_done": RecipeStep( + action="route", + on_result=StepResultRoute( + field="next", + routes={"more_parts": "dry_walkthrough", "all_done": "done"}, + ), + ), + "done": RecipeStep(action="stop", message="Done"), + }, + kitchen_rules=["SEQUENTIAL EXECUTION: complete full cycle per part before advancing."], + ) + warnings = run_semantic_rules(recipe) + rule_names = [w.rule for w in warnings] + assert "multipart-route-back" not in rule_names + + def test_mi4_multipart_rule_fires_when_no_more_parts_route(self) -> None: + """T_MI4: multipart-route-back fires when next_or_done has no more_parts condition.""" + recipe = Recipe( + name="test-recipe", + description="test", + ingredients={}, + steps={ + "plan": RecipeStep( + tool="run_skill", + with_args={"skill_command": "/autoskillit:make-plan inputs.task"}, + on_success="next_or_done", + note="Glob plan_dir for *_part_*.md or single plan file.", + ), + "next_or_done": RecipeStep( + action="route", + on_result=StepResultRoute(field="next", routes={"all_done": "done"}), + ), + "done": RecipeStep(action="stop", message="Done"), + }, + kitchen_rules=["SEQUENTIAL EXECUTION: complete full cycle per part before advancing."], + ) + warnings = run_semantic_rules(recipe) + rule_names = [w.rule for w in warnings] + assert "multipart-route-back" in rule_names + + +@pytest.fixture +def compliant_multipart_recipe_no_list() -> Recipe: + """Recipe with make-plan step but no capture_list for plan_parts.""" + return Recipe( + name="test", + description="test", + ingredients={}, + steps={ + "plan": RecipeStep( + tool="run_skill", + with_args={"skill_command": "/autoskillit:make-plan inputs.task"}, + capture={"plan_path": "${{ result.plan_path }}"}, + note="Glob plan_dir for *_part_*.md or single plan file. Sort into plan_parts[].", + on_success="done", + ), + "done": RecipeStep(action="stop", message="Done"), + }, + kitchen_rules=["SEQUENTIAL EXECUTION: complete full cycle per part."], + ) + + +@pytest.fixture +def compliant_multipart_recipe_with_list() -> Recipe: + """Recipe with make-plan step and correct capture_list for plan_parts.""" + return Recipe( + name="test", + description="test", + ingredients={}, + steps={ + "plan": RecipeStep( + tool="run_skill", + with_args={"skill_command": "/autoskillit:make-plan inputs.task"}, + capture={"plan_path": "${{ result.plan_path }}"}, + capture_list={"plan_parts": "${{ result.plan_parts }}"}, + note="Glob plan_dir for *_part_*.md or single plan file. Sort into plan_parts[].", + on_success="done", + ), + "done": RecipeStep(action="stop", message="Done"), + }, + kitchen_rules=["SEQUENTIAL EXECUTION: complete full cycle per part."], + ) + + +def test_validator_warns_when_plan_parts_not_captured( + compliant_multipart_recipe_no_list: Recipe, +) -> None: + """D6: Validator warns when make-plan step lacks capture_list for plan_parts.""" + warnings = run_semantic_rules(compliant_multipart_recipe_no_list) + rule_names = [w.rule for w in warnings] + assert "multipart-plan-parts-not-captured" in rule_names + + +def test_validator_passes_when_plan_parts_captured( + compliant_multipart_recipe_with_list: Recipe, +) -> None: + """D7: Validator passes when make-plan step has capture_list for plan_parts.""" + warnings = run_semantic_rules(compliant_multipart_recipe_with_list) + rule_names = [w.rule for w in warnings] + assert "multipart-plan-parts-not-captured" not in rule_names diff --git a/tests/recipe/test_rules_on_context_limit.py b/tests/recipe/test_rules_on_context_limit.py new file mode 100644 index 0000000000..9a789c7108 --- /dev/null +++ b/tests/recipe/test_rules_on_context_limit.py @@ -0,0 +1,214 @@ +from __future__ import annotations + +import pytest + +from autoskillit.core.types import Severity +from autoskillit.recipe.schema import Recipe, RecipeStep +from autoskillit.recipe.validator import run_semantic_rules, validate_recipe_structure + +pytestmark = [pytest.mark.layer("recipe"), pytest.mark.small] + + +class TestOnContextLimitField: + """Tests for on_context_limit as a routing field and cycle detection.""" + + def test_on_context_limit_invalid_target_raises_validation_error(self) -> None: + """on_context_limit must reference a declared step name.""" + + recipe = Recipe( + name="test", + description="test", + summary="test", + ingredients={}, + kitchen_rules=["test"], + steps={ + "fix": RecipeStep( + tool="run_skill", + on_success="done", + on_failure="cleanup", + on_context_limit="nonexistent_step", + with_args={"skill_command": "x", "cwd": "/tmp"}, + ), + "cleanup": RecipeStep(action="stop", message="done"), + "done": RecipeStep(action="stop", message="done"), + }, + ) + errors = validate_recipe_structure(recipe) + assert errors, "Expected validation errors for unknown on_context_limit target" + assert any("on_context_limit" in e for e in errors) + + def test_on_context_limit_valid_target_passes_validation(self) -> None: + """on_context_limit referencing a valid step passes validation.""" + + recipe = Recipe( + name="test", + description="test", + summary="test", + ingredients={}, + kitchen_rules=["test"], + steps={ + "implement": RecipeStep( + tool="run_skill", + on_success="done", + on_failure="cleanup", + on_context_limit="retry_worktree", + retries=0, + with_args={"skill_command": "/autoskillit:implement-worktree-no-merge x"}, + ), + "retry_worktree": RecipeStep( + tool="run_skill", + on_success="done", + on_failure="cleanup", + with_args={"skill_command": "/autoskillit:retry-worktree x y"}, + ), + "cleanup": RecipeStep(action="stop", message="done"), + "done": RecipeStep(action="stop", message="done"), + }, + ) + errors = validate_recipe_structure(recipe) + assert not errors, f"Expected no errors but got: {errors}" + + def test_on_exhausted_invalid_target_raises_validation_error(self) -> None: + """on_exhausted must reference a declared step name or be a reserved terminal.""" + + recipe = Recipe( + name="test", + description="test", + summary="test", + ingredients={}, + kitchen_rules=["test"], + steps={ + "fix": RecipeStep( + tool="run_skill", + on_success="done", + on_failure="cleanup", + on_exhausted="nonexistent_step", + with_args={"skill_command": "x", "cwd": "/tmp"}, + ), + "cleanup": RecipeStep(action="stop", message="done"), + "done": RecipeStep(action="stop", message="done"), + }, + ) + errors = validate_recipe_structure(recipe) + assert errors, "Expected validation errors for unknown on_exhausted target" + + def test_on_exhausted_escalate_reserved_passes_validation(self) -> None: + """on_exhausted: 'escalate' is reserved — passes validation without an escalate step.""" + + recipe = Recipe( + name="test", + description="test", + summary="test", + ingredients={}, + kitchen_rules=["test"], + steps={ + "fix": RecipeStep( + tool="run_skill", + on_success="done", + on_failure="done", + on_exhausted="escalate", + with_args={"skill_command": "/autoskillit:investigate x"}, + ), + "done": RecipeStep(action="stop", message="done"), + }, + ) + errors = validate_recipe_structure(recipe) + assert not errors, f"Expected no errors but got: {errors}" + + def test_unbounded_cycle_without_retries_produces_warning(self) -> None: + """verify → assess → verify cycle with retries=0 must produce a warning.""" + recipe = Recipe( + name="test", + description="test", + summary="test", + ingredients={}, + kitchen_rules=["test"], + steps={ + "assess": RecipeStep( + tool="run_skill", + on_success="verify", + on_failure="cleanup", + retries=0, + with_args={"skill_command": "x", "cwd": "/tmp"}, + ), + "verify": RecipeStep( + tool="test_check", + on_success="done", + on_failure="assess", + with_args={"worktree_path": "/tmp"}, + ), + "cleanup": RecipeStep(action="stop", message="done"), + "done": RecipeStep(action="stop", message="done"), + }, + ) + findings = run_semantic_rules(recipe) + warnings = [f for f in findings if f.severity == Severity.WARNING] + assert any( + "unbounded" in f.message.lower() or "cycle" in f.message.lower() for f in warnings + ) + + def test_cycle_with_retries_warns_when_success_stays_in_cycle(self) -> None: + """A cycle where the retrying step's success path stays inside the cycle + must emit a WARNING. fix → test → fix: fix.on_success='test' re-enters + the cycle, so the outer loop is unbounded despite the retry exit.""" + recipe = Recipe( + name="test", + description="test", + summary="test", + ingredients={}, + kitchen_rules=["test"], + steps={ + "fix": RecipeStep( + tool="run_skill", + on_success="test", + on_failure="cleanup", + retries=3, + on_exhausted="cleanup", + with_args={"skill_command": "x", "cwd": "/tmp"}, + ), + "test": RecipeStep( + tool="test_check", + on_success="done", + on_failure="fix", + with_args={"worktree_path": "/tmp"}, + ), + "cleanup": RecipeStep(action="stop", message="done"), + "done": RecipeStep(action="stop", message="done"), + }, + ) + findings = run_semantic_rules(recipe) + cycle_findings = [ + f for f in findings if "cycle" in f.message.lower() or "unbounded" in f.message.lower() + ] + assert len(cycle_findings) >= 1, "Expected a cycle ERROR but got none" + assert any(f.severity == Severity.ERROR for f in cycle_findings) + + def test_truly_trapped_cycle_without_exit_produces_error(self) -> None: + """A cycle where every step's edges stay inside the cycle must produce an ERROR.""" + recipe = Recipe( + name="test", + description="test", + summary="test", + ingredients={}, + kitchen_rules=["test"], + steps={ + "assess": RecipeStep( + tool="run_skill", + on_success="verify", + on_failure="verify", + retries=0, + with_args={"skill_command": "x", "cwd": "/tmp"}, + ), + "verify": RecipeStep( + tool="test_check", + on_success="assess", + on_failure="assess", + with_args={"worktree_path": "/tmp"}, + ), + }, + ) + findings = run_semantic_rules(recipe) + errors = [f for f in findings if f.severity == Severity.ERROR] + assert any( + "cycle" in f.message.lower() or "unbounded" in f.message.lower() for f in errors + ) diff --git a/tests/recipe/test_rules_on_result_failure_route.py b/tests/recipe/test_rules_on_result_failure_route.py new file mode 100644 index 0000000000..03db8b2ebb --- /dev/null +++ b/tests/recipe/test_rules_on_result_failure_route.py @@ -0,0 +1,112 @@ +from __future__ import annotations + +import pytest + +from autoskillit.core.types import Severity +from autoskillit.recipe.validator import run_semantic_rules +from tests.recipe.conftest import _make_workflow + +pytestmark = [pytest.mark.layer("recipe"), pytest.mark.small] + + +class TestOnResultMissingFailureRoute: + """RCA: on-result-missing-failure-route semantic rule.""" + + def test_RCA1_tool_step_on_result_no_on_failure_fires_error(self) -> None: + """RCA1: run_skill step with on_result but no on_failure → Severity.ERROR finding.""" + wf = _make_workflow( + { + "audit": { + "tool": "run_skill", + "with": {"skill_command": "/autoskillit:audit-impl plan.md impl main"}, + "capture": {"verdict": "${{ result.verdict }}"}, + "on_result": {"field": "verdict", "routes": {"GO": "done", "NO GO": "fix"}}, + # no on_failure — the gap + }, + "fix": {"action": "stop", "message": "Fix needed."}, + "done": {"action": "stop", "message": "Done."}, + } + ) + findings = run_semantic_rules(wf) + errors = [f for f in findings if f.severity == Severity.ERROR] + rule_names = [f.rule for f in errors] + assert any( + f.rule == "on-result-missing-failure-route" and f.step_name == "audit" for f in errors + ), f"Expected on-result-missing-failure-route ERROR on 'audit'. Got: {rule_names}" + + def test_RCA2_python_step_on_result_no_on_failure_fires_error(self) -> None: + """RCA2: python step with on_result but no on_failure → Severity.ERROR.""" + wf = _make_workflow( + { + "check": { + "python": "mymod.check_result", + "on_result": {"field": "status", "routes": {"ok": "done", "fail": "fix"}}, + # no on_failure + }, + "fix": {"action": "stop", "message": "Fix."}, + "done": {"action": "stop", "message": "Done."}, + } + ) + findings = run_semantic_rules(wf) + errors = [f for f in findings if f.severity == Severity.ERROR] + assert any(f.rule == "on-result-missing-failure-route" for f in errors) + + def test_RCA3_on_result_with_on_failure_no_finding(self) -> None: + """RCA3: on_result + on_failure present → rule does not fire.""" + wf = _make_workflow( + { + "audit": { + "tool": "run_skill", + "with": {"skill_command": "/autoskillit:audit-impl plan.md impl main"}, + "capture": {"verdict": "${{ result.verdict }}"}, + "on_result": {"field": "verdict", "routes": {"GO": "done", "NO GO": "fix"}}, + "on_failure": "done", + }, + "fix": {"action": "stop", "message": "Fix needed."}, + "done": {"action": "stop", "message": "Done."}, + } + ) + findings = run_semantic_rules(wf) + assert not any(f.rule == "on-result-missing-failure-route" for f in findings) + + def test_RCA4_action_route_on_result_no_on_failure_not_an_error(self) -> None: + """RCA4: action:route with on_result but no on_failure → NOT flagged. + Agent routing decisions are not MCP tool invocations; they cannot fail + the same way and are exempt from this rule. + """ + wf = _make_workflow( + { + "decide": { + "action": "route", + "on_result": { + "field": "parts", + "routes": {"more": "implement", "done": "finish"}, + }, + # no on_failure — intentional for action:route + }, + "implement": {"action": "stop", "message": "Implement."}, + "finish": {"action": "stop", "message": "Finish."}, + } + ) + findings = run_semantic_rules(wf) + assert not any(f.rule == "on-result-missing-failure-route" for f in findings) + + def test_RCA5_optional_true_plus_on_result_no_on_failure_fires(self) -> None: + """RCA5: optional:true does not exempt a step from needing on_failure.""" + wf = _make_workflow( + { + "audit": { + "tool": "run_skill", + "with": {"skill_command": "/autoskillit:audit-impl plan.md impl main"}, + "capture": {"verdict": "${{ result.verdict }}"}, + "on_result": {"field": "verdict", "routes": {"GO": "done", "NO GO": "fix"}}, + "optional": True, + # no on_failure + }, + "fix": {"action": "stop", "message": "Fix needed."}, + "done": {"action": "stop", "message": "Done."}, + } + ) + findings = run_semantic_rules(wf) + errors = [f for f in findings if f.severity == Severity.ERROR] + assert any(f.rule == "on-result-missing-failure-route" for f in errors) diff --git a/tests/recipe/test_rules_outdated_script_version.py b/tests/recipe/test_rules_outdated_script_version.py new file mode 100644 index 0000000000..f4f964e075 --- /dev/null +++ b/tests/recipe/test_rules_outdated_script_version.py @@ -0,0 +1,78 @@ +from __future__ import annotations + +import pytest + +from autoskillit.core.types import Severity +from autoskillit.recipe.validator import run_semantic_rules +from tests.recipe.conftest import _make_workflow + +pytestmark = [pytest.mark.layer("recipe"), pytest.mark.small] + + +class TestOutdatedScriptVersionRule: + @pytest.mark.parametrize( + "script_ver,installed_ver,expected_count", + [ + ("0.1.0", "0.2.0", 1), # MSR1: below installed → fires + ("0.2.0", "0.2.0", 0), # MSR2: matches installed → does not fire + (None, "0.2.0", 0), # MSR3: None → does not fire + ], + ) + def test_outdated_recipe_version_rule( + self, monkeypatch: pytest.MonkeyPatch, script_ver, installed_ver, expected_count + ) -> None: + import autoskillit + import autoskillit.core.types as _core_types + import autoskillit.recipe.rules.rules_inputs as _rules_mod + + monkeypatch.setattr(autoskillit, "__version__", installed_ver) + monkeypatch.setattr(_core_types, "AUTOSKILLIT_INSTALLED_VERSION", installed_ver) + monkeypatch.setattr(_rules_mod, "AUTOSKILLIT_INSTALLED_VERSION", installed_ver) + wf = _make_workflow( + { + "do_thing": {"tool": "run_cmd", "on_success": "done"}, + "done": {"action": "stop", "message": "Done."}, + } + ) + wf.version = script_ver + findings = [f for f in run_semantic_rules(wf) if f.rule == "outdated-recipe-version"] + assert len(findings) == expected_count + + def test_outdated_recipe_version_rule_severity_is_warning( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + import autoskillit + import autoskillit.core.types as _core_types + import autoskillit.recipe.rules.rules_inputs as _rules_mod + + monkeypatch.setattr(autoskillit, "__version__", "0.2.0") + monkeypatch.setattr(_core_types, "AUTOSKILLIT_INSTALLED_VERSION", "0.2.0") + monkeypatch.setattr(_rules_mod, "AUTOSKILLIT_INSTALLED_VERSION", "0.2.0") + wf = _make_workflow( + { + "do_thing": {"tool": "run_cmd", "on_success": "done"}, + "done": {"action": "stop", "message": "Done."}, + } + ) + wf.version = "0.1.0" + findings = [f for f in run_semantic_rules(wf) if f.rule == "outdated-recipe-version"] + assert findings[0].severity == Severity.WARNING + + def test_none_version_does_not_fire(self, monkeypatch: pytest.MonkeyPatch) -> None: + """Bundled recipes have version=None — rule must not fire on them.""" + import autoskillit + import autoskillit.core.types as _core_types + import autoskillit.recipe.rules.rules_inputs as _rules_mod + + monkeypatch.setattr(autoskillit, "__version__", "99.0.0") + monkeypatch.setattr(_core_types, "AUTOSKILLIT_INSTALLED_VERSION", "99.0.0") + monkeypatch.setattr(_rules_mod, "AUTOSKILLIT_INSTALLED_VERSION", "99.0.0") + wf = _make_workflow( + { + "do_thing": {"tool": "run_cmd", "on_success": "done"}, + "done": {"action": "stop", "message": "Done."}, + } + ) + wf.version = None + findings = [f for f in run_semantic_rules(wf) if f.rule == "outdated-recipe-version"] + assert findings == [] diff --git a/tests/recipe/test_rules_packs.py b/tests/recipe/test_rules_packs.py index e40fbf3a6c..3b617c2136 100644 --- a/tests/recipe/test_rules_packs.py +++ b/tests/recipe/test_rules_packs.py @@ -1,9 +1,18 @@ -"""Tests for the unknown-required-pack semantic rule.""" +"""Tests for the unknown-required-pack and undeclared-pack-requirement semantic rules.""" + +from __future__ import annotations + +from dataclasses import replace + +import pytest from autoskillit.core import Severity +from autoskillit.recipe.io import builtin_recipes_dir, load_recipe from autoskillit.recipe.registry import run_semantic_rules from autoskillit.recipe.schema import Recipe, RecipeStep +pytestmark = [pytest.mark.layer("recipe"), pytest.mark.small] + def _make_recipe(requires_packs: list[str]) -> Recipe: return Recipe( @@ -15,14 +24,38 @@ def _make_recipe(requires_packs: list[str]) -> Recipe: ) -def test_unknown_pack_produces_warning(): - """Pack name not in PACK_REGISTRY produces a WARNING finding.""" +def _make_recipe_with_run_skill( + requires_packs: list[str], + skill_command: str, + step_name: str = "run_it", +) -> Recipe: + return Recipe( + name="test", + description="test recipe", + version="0.7.2", + requires_packs=requires_packs, + steps={ + step_name: RecipeStep( + tool="run_skill", + with_args={"skill_command": skill_command}, + ) + }, + ) + + +# ---------------------------------------------------------------------- +# unknown-required-pack tests +# ---------------------------------------------------------------------- + + +def test_unknown_pack_produces_error(): + """Pack name not in PACK_REGISTRY produces an ERROR finding.""" import autoskillit.recipe # noqa: F401 -- triggers rule registration recipe = _make_recipe(["nonexistent-pack"]) findings = [f for f in run_semantic_rules(recipe) if f.rule == "unknown-required-pack"] assert findings - assert findings[0].severity == Severity.WARNING + assert findings[0].severity == Severity.ERROR assert "nonexistent-pack" in findings[0].message @@ -62,3 +95,116 @@ def test_all_builtin_packs_pass(): recipe = _make_recipe(list(PACK_REGISTRY.keys())) findings = [f for f in run_semantic_rules(recipe) if f.rule == "unknown-required-pack"] assert not findings, f"Built-in packs must not trigger unknown-required-pack: {findings}" + + +# ---------------------------------------------------------------------- +# undeclared-pack-requirement tests +# ---------------------------------------------------------------------- + + +class TestUndeclaredPackRequirement: + """Tests for the undeclared-pack-requirement semantic rule.""" + + def _run_rule( + self, + recipe: Recipe, + skill_category_map: dict[str, frozenset[str]] | None = None, + ) -> list: + import autoskillit.recipe # noqa: F401 -- triggers rule registration + + if skill_category_map is not None: + from autoskillit.recipe._analysis import make_validation_context + + ctx = make_validation_context(recipe) + ctx.skill_category_map = skill_category_map + return [f for f in run_semantic_rules(ctx) if f.rule == "undeclared-pack-requirement"] + return [f for f in run_semantic_rules(recipe) if f.rule == "undeclared-pack-requirement"] + + def test_skill_needing_disabled_pack_without_declaration_is_error(self): + """A recipe that uses a vis-lens skill without declaring vis-lens emits ERROR.""" + recipe = _make_recipe_with_run_skill( + requires_packs=["research"], # missing vis-lens + skill_command="/autoskillit:plan-visualization", + ) + mock_map = {"plan-visualization": frozenset({"vis-lens"})} + findings = self._run_rule(recipe, skill_category_map=mock_map) + assert len(findings) == 1 + assert findings[0].severity == Severity.ERROR + assert "vis-lens" in findings[0].message + + def test_skill_needing_disabled_pack_with_declaration_passes(self): + """A recipe that uses a vis-lens skill AND declares vis-lens emits no finding.""" + recipe = _make_recipe_with_run_skill( + requires_packs=["research", "vis-lens"], + skill_command="/autoskillit:plan-visualization", + ) + mock_map = {"plan-visualization": frozenset({"vis-lens"})} + findings = self._run_rule(recipe, skill_category_map=mock_map) + assert not findings + + def test_skill_needing_only_enabled_packs_passes(self): + """A recipe that uses only default-enabled pack skills emits no finding.""" + from autoskillit.core import PACK_REGISTRY + + assert PACK_REGISTRY["github"].default_enabled, ( + "precondition: github must be default-enabled" + ) + recipe = _make_recipe_with_run_skill( + requires_packs=["github"], + skill_command="/autoskillit:github-issue", + ) + mock_map = {"github-issue": frozenset({"github"})} + findings = self._run_rule(recipe, skill_category_map=mock_map) + assert not findings + + def test_multiple_missing_packs_emits_multiple_errors(self): + """A recipe missing both exp-lens and vis-lens emits two findings.""" + recipe = _make_recipe_with_run_skill( + requires_packs=["research"], # missing both exp-lens and vis-lens + skill_command="/autoskillit:experiment-compare", + ) + mock_map = {"experiment-compare": frozenset({"exp-lens", "vis-lens"})} + findings = self._run_rule(recipe, skill_category_map=mock_map) + assert len(findings) == 2 + assert all(f.severity == Severity.ERROR for f in findings) + pack_names = {p for f in findings for p in ["exp-lens", "vis-lens"] if p in f.message} + assert pack_names == {"exp-lens", "vis-lens"} + + def test_dynamic_skill_name_is_skipped(self): + """A skill_command using a template expression emits no finding (graceful skip).""" + recipe = Recipe( + name="test", + description="test recipe", + requires_packs=["research"], + steps={ + "run_it": RecipeStep( + tool="run_skill", + with_args={"skill_command": "${{ inputs.target_skill }}"}, + ) + }, + ) + findings = self._run_rule(recipe) + assert not findings + + def test_unknown_skill_name_is_skipped(self): + """Non-existent skill reference emits no finding (handled by unknown-skill-command).""" + recipe = _make_recipe_with_run_skill( + requires_packs=["research"], + skill_command="/autoskillit:nonexistent-skill-xyz", + ) + findings = self._run_rule(recipe) + assert not findings + + def test_research_design_yaml_triggers_error_without_vis_lens(self): + """research-design.yaml with only [research] triggers vis-lens ERROR.""" + import autoskillit.recipe # noqa: F401 -- triggers rule registration + + base_recipe = load_recipe(builtin_recipes_dir() / "research-design.yaml") + recipe = replace(base_recipe, requires_packs=["research"]) + findings = [ + f for f in run_semantic_rules(recipe) if f.rule == "undeclared-pack-requirement" + ] + assert len(findings) >= 1, "Expected vis-lens to be flagged as missing" + vis_lens_findings = [f for f in findings if "vis-lens" in f.message] + assert len(vis_lens_findings) == 1 + assert vis_lens_findings[0].severity == Severity.ERROR diff --git a/tests/recipe/test_rules_pipeline_internal.py b/tests/recipe/test_rules_pipeline_internal.py index d270b82b5b..70f770830b 100644 --- a/tests/recipe/test_rules_pipeline_internal.py +++ b/tests/recipe/test_rules_pipeline_internal.py @@ -1,9 +1,13 @@ """Tests for the pipeline-internal-not-hidden semantic rule.""" +import pytest + from autoskillit.core import Severity from autoskillit.recipe.registry import _RULE_REGISTRY, run_semantic_rules from autoskillit.recipe.schema import Recipe, RecipeIngredient +pytestmark = [pytest.mark.layer("recipe"), pytest.mark.small] + def _make_recipe_with_ingredients(ingredients: dict) -> Recipe: """Build a minimal Recipe with the given ingredients dict for rule testing.""" @@ -53,9 +57,9 @@ def test_pipeline_internal_not_hidden_no_fire_when_hidden_true(): """Pipeline-internal ingredient that already has hidden=True must not warn.""" recipe = _make_recipe_with_ingredients( { - "sprint_mode": RecipeIngredient( - description="Set by process-issues to enable sprint mode.", - default="false", + "run_mode": RecipeIngredient( + description="Set by process-issues to control run mode.", + default="sequential", hidden=True, ) } diff --git a/tests/recipe/test_rules_predicate_routing.py b/tests/recipe/test_rules_predicate_routing.py new file mode 100644 index 0000000000..314e5e2ebd --- /dev/null +++ b/tests/recipe/test_rules_predicate_routing.py @@ -0,0 +1,228 @@ +from __future__ import annotations + +import pytest + +from autoskillit.recipe.schema import Recipe, RecipeStep, StepResultRoute +from autoskillit.recipe.validator import run_semantic_rules, validate_recipe_structure +from tests.recipe.conftest import _make_workflow + +pytestmark = [pytest.mark.layer("recipe"), pytest.mark.small] + + +class TestPredicateOnResultValidation: + """Structural validation for predicate-format on_result (conditions list).""" + + def _make_merge_recipe(self, merge_step: dict, extra_steps: dict | None = None) -> Recipe: + steps: dict = { + "merge": merge_step, + "assess": {"action": "stop", "message": "Assess."}, + "cleanup_failure": {"action": "stop", "message": "Cleanup."}, + "push": {"action": "stop", "message": "Push."}, + } + if extra_steps: + steps.update(extra_steps) + return _make_workflow(steps) + + def test_predicate_on_result_on_success_mutually_exclusive(self) -> None: + """Step with predicate on_result (list) + on_success → validation error.""" + + wf = self._make_merge_recipe( + { + "tool": "merge_worktree", + "with": {"worktree_path": "/tmp/wt", "base_branch": "main"}, + "on_result": [ + {"when": "result.error", "route": "cleanup_failure"}, + {"route": "push"}, + ], + "on_success": "push", # mutually exclusive + } + ) + errors = validate_recipe_structure(wf) + assert any("on_result" in e and "on_success" in e for e in errors) + + def test_predicate_condition_invalid_route_target_rejected(self) -> None: + """A condition referencing an unknown step name is a validation error.""" + + wf = self._make_merge_recipe( + { + "tool": "merge_worktree", + "with": {"worktree_path": "/tmp/wt", "base_branch": "main"}, + "on_result": [ + {"when": "result.error", "route": "nonexistent_step"}, + {"route": "push"}, + ], + } + ) + errors = validate_recipe_structure(wf) + assert any("nonexistent_step" in e for e in errors) + + def test_predicate_condition_route_valid_step_accepted(self) -> None: + """All condition routes pointing to valid step names pass validation.""" + + wf = self._make_merge_recipe( + { + "tool": "merge_worktree", + "with": {"worktree_path": "/tmp/wt", "base_branch": "main"}, + "on_result": [ + {"when": "result.error", "route": "cleanup_failure"}, + {"route": "push"}, + ], + "capture": {"cleanup_succeeded": "${{ result.cleanup_succeeded }}"}, + } + ) + errors = validate_recipe_structure(wf) + assert errors == [] + + def test_predicate_on_result_empty_conditions_rejected(self) -> None: + """on_result with conditions=[] bypasses predicate path; emits field error. + + When StepResultRoute(conditions=[]) is constructed directly (bypassing _parse_step, + which collapses empty conditions to on_result=None), the validator falls through to + legacy format validation and emits an explicit error for the missing field. + """ + + recipe = Recipe( + name="test-predicate-empty", + description="test", + steps={ + "start": RecipeStep( + tool="run_skill", + with_args={"skill_command": "x", "cwd": "y"}, + on_result=StepResultRoute(conditions=[]), + ), + "done": RecipeStep(action="stop", message="done"), + }, + ) + errors = validate_recipe_structure(recipe) + assert any("on_result.field must be non-empty" in e for e in errors) + + def test_predicate_format_with_on_failure_allowed(self) -> None: + """validator.py must not reject on_failure alongside on_result.conditions.""" + + wf = self._make_merge_recipe( + { + "tool": "merge_worktree", + "with": {"worktree_path": "/tmp/wt", "base_branch": "main"}, + "on_result": [ + {"when": "result.error", "route": "cleanup_failure"}, + {"route": "push"}, + ], + "on_failure": "cleanup_failure", + } + ) + errors = validate_recipe_structure(wf) + assert not any("mutually exclusive" in e for e in errors), errors + + def test_on_result_missing_failure_route_fires_for_predicate_format(self) -> None: + """Predicate-format steps with no on_failure must trigger ERROR finding.""" + wf = self._make_merge_recipe( + { + "tool": "merge_worktree", + "with": {"worktree_path": "/tmp/wt", "base_branch": "main"}, + "on_result": [ + {"when": "result.error", "route": "cleanup_failure"}, + {"route": "push"}, + ], + # no on_failure — should trigger finding + } + ) + findings = run_semantic_rules(wf) + names = [f.rule for f in findings] + assert "on-result-missing-failure-route" in names + + def test_on_result_missing_failure_route_clear_when_predicate_has_on_failure(self) -> None: + """Predicate-format step with on_failure must not trigger the rule.""" + wf = self._make_merge_recipe( + { + "tool": "merge_worktree", + "with": {"worktree_path": "/tmp/wt", "base_branch": "main"}, + "on_result": [ + {"when": "result.error", "route": "cleanup_failure"}, + {"route": "push"}, + ], + "on_failure": "cleanup_failure", + } + ) + findings = run_semantic_rules(wf) + names = [f.rule for f in findings] + assert "on-result-missing-failure-route" not in names + + +class TestPredicateBuildStepGraph: + """_build_step_graph includes condition.route edges.""" + + def test_build_step_graph_includes_condition_routes(self) -> None: + """_build_step_graph produces edges for condition.route targets.""" + from autoskillit.recipe.validator import _build_step_graph + + wf = _make_workflow( + { + "merge": { + "tool": "merge_worktree", + "with": {"worktree_path": "/tmp/wt", "base_branch": "main"}, + "on_result": [ + {"when": "result.failed_step == 'test_gate'", "route": "assess"}, + {"when": "result.error", "route": "cleanup"}, + {"route": "push"}, + ], + "capture": {"cleanup_succeeded": "${{ result.cleanup_succeeded }}"}, + }, + "assess": {"action": "stop", "message": "Assess."}, + "cleanup": {"action": "stop", "message": "Cleanup."}, + "push": {"action": "stop", "message": "Push."}, + } + ) + graph = _build_step_graph(wf) + assert "assess" in graph["merge"] + assert "cleanup" in graph["merge"] + assert "push" in graph["merge"] + + +class TestPredicateSemanticRules: + """Semantic rules behave correctly for predicate-format on_result.""" + + def test_unreachable_step_includes_condition_routes(self) -> None: + """A step reachable only via condition route is NOT flagged as unreachable.""" + wf = _make_workflow( + { + "merge": { + "tool": "merge_worktree", + "with": {"worktree_path": "/tmp/wt", "base_branch": "main"}, + "on_result": [ + {"when": "result.failed_step == 'test_gate'", "route": "assess"}, + {"when": "result.error", "route": "cleanup"}, + {"route": "push"}, + ], + "capture": {"cleanup_succeeded": "${{ result.cleanup_succeeded }}"}, + }, + "assess": {"action": "stop", "message": "Assess."}, + "cleanup": {"action": "stop", "message": "Cleanup."}, + "push": {"action": "stop", "message": "Push."}, + } + ) + findings = run_semantic_rules(wf) + unreachable = [f for f in findings if f.rule == "unreachable-step"] + step_names = {f.step_name for f in unreachable} + assert "assess" not in step_names + assert "cleanup" not in step_names + assert "push" not in step_names + + def test_on_result_missing_failure_route_still_fires_for_legacy_format( + self, + ) -> None: + """RCA1 rule continues to fire for legacy format with no on_failure (no regression).""" + wf = _make_workflow( + { + "audit": { + "tool": "run_skill", + "with": {"skill_command": "/autoskillit:audit-impl plan.md impl main"}, + "capture": {"verdict": "${{ result.verdict }}"}, + "on_result": {"field": "verdict", "routes": {"GO": "done", "NO GO": "fix"}}, + # no on_failure — the gap + }, + "fix": {"action": "stop", "message": "Fix."}, + "done": {"action": "stop", "message": "Done."}, + } + ) + findings = run_semantic_rules(wf) + assert any(f.rule == "on-result-missing-failure-route" for f in findings) diff --git a/tests/recipe/test_rules_project_local_override.py b/tests/recipe/test_rules_project_local_override.py index 1e9e1ee902..9f1c9fad4e 100644 --- a/tests/recipe/test_rules_project_local_override.py +++ b/tests/recipe/test_rules_project_local_override.py @@ -2,6 +2,10 @@ from __future__ import annotations +import pytest + +pytestmark = [pytest.mark.layer("recipe"), pytest.mark.medium] + def test_project_local_override_rule_emits_warning(tmp_path): """T-OVR-015: /autoskillit:review-pr with project-local override → WARNING finding.""" diff --git a/tests/recipe/test_rules_reachability.py b/tests/recipe/test_rules_reachability.py index 71b50d1337..ae6ddb68f2 100644 --- a/tests/recipe/test_rules_reachability.py +++ b/tests/recipe/test_rules_reachability.py @@ -20,6 +20,8 @@ StepResultRoute, ) +pytestmark = [pytest.mark.layer("recipe"), pytest.mark.small] + _QUEUE_CAPABLE = ("implementation.yaml", "remediation.yaml", "implementation-groups.yaml") diff --git a/tests/recipe/test_rules_recipe.py b/tests/recipe/test_rules_recipe.py index a79bae35a6..bc595c8f92 100644 --- a/tests/recipe/test_rules_recipe.py +++ b/tests/recipe/test_rules_recipe.py @@ -1,4 +1,4 @@ -"""Tests for rules_recipe semantic rules (unknown-sub-recipe, circular-sub-recipe).""" +"""Tests for rules_recipe semantic rules.""" from __future__ import annotations @@ -6,10 +6,13 @@ import pytest +from autoskillit.core import Severity from autoskillit.recipe._analysis import make_validation_context from autoskillit.recipe.registry import _RULE_REGISTRY, run_semantic_rules from autoskillit.recipe.schema import Recipe, RecipeIngredient, RecipeStep +pytestmark = [pytest.mark.layer("recipe"), pytest.mark.small] + def _make_recipe_with_sub_recipe(sub_recipe_name: str) -> Recipe: return Recipe( @@ -33,9 +36,7 @@ def _make_recipe_with_sub_recipe(sub_recipe_name: str) -> Recipe: def test_unknown_sub_recipe_rule_fires() -> None: """unknown-sub-recipe finding when sub_recipe name not in available_sub_recipes.""" recipe = _make_recipe_with_sub_recipe("nonexistent") - ctx = make_validation_context( - recipe, available_sub_recipes=frozenset({"sprint-prefix", "other"}) - ) + ctx = make_validation_context(recipe, available_sub_recipes=frozenset({"other"})) findings = run_semantic_rules(ctx) unknown = [f for f in findings if f.rule == "unknown-sub-recipe"] assert unknown @@ -44,8 +45,8 @@ def test_unknown_sub_recipe_rule_fires() -> None: def test_unknown_sub_recipe_rule_passes_when_name_known() -> None: """No finding when sub_recipe name is in available_sub_recipes.""" - recipe = _make_recipe_with_sub_recipe("sprint-prefix") - ctx = make_validation_context(recipe, available_sub_recipes=frozenset({"sprint-prefix"})) + recipe = _make_recipe_with_sub_recipe("other") + ctx = make_validation_context(recipe, available_sub_recipes=frozenset({"other"})) findings = run_semantic_rules(ctx) unknown = [f for f in findings if f.rule == "unknown-sub-recipe"] assert not unknown @@ -67,7 +68,7 @@ def test_unknown_sub_recipe_rule_skips_non_sub_recipe_steps() -> None: }, kitchen_rules=["no native tools"], ) - ctx = make_validation_context(recipe, available_sub_recipes=frozenset({"sprint-prefix"})) + ctx = make_validation_context(recipe, available_sub_recipes=frozenset()) findings = run_semantic_rules(ctx) unknown = [f for f in findings if f.rule == "unknown-sub-recipe"] assert not unknown @@ -123,10 +124,11 @@ def test_circular_sub_recipe_rule_fires(tmp_path: Path) -> None: def test_rules_recipe_registered() -> None: - """Both rules_recipe rules are present in the global registry.""" + """All rules_recipe rules are present in the global registry.""" rule_names = {spec.name for spec in _RULE_REGISTRY} assert "unknown-sub-recipe" in rule_names assert "circular-sub-recipe" in rule_names + assert "env-key-in-with-args" in rule_names def test_all_bundled_recipes_pass_rules_recipe() -> None: @@ -164,9 +166,57 @@ def test_all_bundled_recipes_pass_rules_recipe() -> None: def test_rules_recipe_uses_check_prefix() -> None: """Rule functions in rules_recipe.py must use the _check_* naming convention.""" - import autoskillit.recipe.rules_recipe as m + import autoskillit.recipe.rules.rules_recipe as m assert hasattr(m, "_check_unknown_sub_recipe"), "_check_unknown_sub_recipe not found" assert hasattr(m, "_check_circular_sub_recipe"), "_check_circular_sub_recipe not found" + assert hasattr(m, "_check_env_key_in_with_args"), "_check_env_key_in_with_args not found" assert not hasattr(m, "_unknown_sub_recipe"), "_unknown_sub_recipe should be renamed" assert not hasattr(m, "_circular_sub_recipe"), "_circular_sub_recipe should be renamed" + + +# --------------------------------------------------------------------------- +# env-key-in-with-args rule tests (ADR-0003) +# --------------------------------------------------------------------------- + + +def test_env_key_in_with_args_rejected() -> None: + """A step with env: in with_args must trigger env-key-in-with-args ERROR.""" + recipe = Recipe( + name="test-env-rule", + description="test", + steps={ + "bad_step": RecipeStep( + tool="run_skill", + with_args={"skill_command": "/autoskillit:some-skill", "env": {"FOO": "bar"}}, # type: ignore[dict-item] + on_success="done", + ), + "done": RecipeStep(action="stop"), + }, + ) + ctx = make_validation_context(recipe, available_sub_recipes=frozenset()) + findings = run_semantic_rules(ctx) + env_findings = [f for f in findings if f.rule == "env-key-in-with-args"] + assert len(env_findings) == 1 + assert env_findings[0].severity == Severity.ERROR + assert env_findings[0].step_name == "bad_step" + + +def test_env_key_in_with_args_clean_step_passes() -> None: + """A step without env: in with_args must not trigger the rule.""" + recipe = Recipe( + name="test-env-rule-clean", + description="test", + steps={ + "good_step": RecipeStep( + tool="run_skill", + with_args={"skill_command": "/autoskillit:some-skill"}, + on_success="done", + ), + "done": RecipeStep(action="stop"), + }, + ) + ctx = make_validation_context(recipe, available_sub_recipes=frozenset()) + findings = run_semantic_rules(ctx) + env_findings = [f for f in findings if f.rule == "env-key-in-with-args"] + assert len(env_findings) == 0 diff --git a/tests/recipe/test_rules_registry.py b/tests/recipe/test_rules_registry.py new file mode 100644 index 0000000000..8a85a825f6 --- /dev/null +++ b/tests/recipe/test_rules_registry.py @@ -0,0 +1,71 @@ +from __future__ import annotations + +import pytest + +from autoskillit.core.types import Severity +from autoskillit.recipe.io import builtin_recipes_dir, load_recipe +from autoskillit.recipe.validator import RuleFinding, run_semantic_rules +from tests.recipe.conftest import ( + KNOWN_PART_B_VIOLATIONS as _KNOWN_PART_B_VIOLATIONS, +) +from tests.recipe.conftest import ( + _make_workflow, +) + +pytestmark = [pytest.mark.layer("recipe"), pytest.mark.small] + + +def test_registry_collects_rules() -> None: + wf = _make_workflow( + { + "do_thing": {"tool": "run_cmd", "on_success": "done"}, + "orphan": {"tool": "run_cmd", "on_success": "done"}, + "done": {"action": "stop", "message": "Done."}, + } + ) + rule_ids = [f.rule for f in run_semantic_rules(wf)] + assert "unreachable-step" in rule_ids + + +def test_rule_finding_to_dict() -> None: + finding = RuleFinding( + rule="test-rule", + severity=Severity.WARNING, + step_name="some_step", + message="Something is wrong.", + ) + d = finding.to_dict() + assert d == { + "rule": "test-rule", + "severity": "warning", + "step": "some_step", + "message": "Something is wrong.", + } + + +def test_old_rule_removed() -> None: + from autoskillit.recipe.validator import _RULE_REGISTRY + + assert not any(r.name == "retry-without-worktree-path" for r in _RULE_REGISTRY) + + +def test_bundled_workflows_pass_semantic_rules() -> None: + wf_dir = builtin_recipes_dir() + yaml_files = list(wf_dir.glob("*.yaml")) + assert yaml_files + + for path in yaml_files: + wf = load_recipe(path) + findings = run_semantic_rules(wf) + errors = [ + f + for f in findings + if f.severity == Severity.ERROR and f.rule not in _KNOWN_PART_B_VIOLATIONS + ] + assert not errors, ( + f"Bundled workflow {path.name} has error-severity semantic findings: {errors}" + ) + undeclared_findings = [f for f in findings if f.rule == "undeclared-capture-key"] + assert undeclared_findings == [], ( + f"Recipe '{wf.name}' has undeclared-capture-key findings: " + repr(undeclared_findings) + ) diff --git a/tests/recipe/test_rules_review_loop_waypoint.py b/tests/recipe/test_rules_review_loop_waypoint.py new file mode 100644 index 0000000000..8ef2612b3a --- /dev/null +++ b/tests/recipe/test_rules_review_loop_waypoint.py @@ -0,0 +1,107 @@ +""" +Tests for the review-loop-waypoint-guard semantic rule. + +The rule fires when check_repo_ci_event is BFS-reachable from review_pr +without crossing check_review_loop. +""" + +import pytest + +from autoskillit.recipe.io import builtin_recipes_dir, load_recipe +from autoskillit.recipe.registry import _RULE_REGISTRY, run_semantic_rules +from autoskillit.recipe.schema import Recipe, RecipeStep, StepResultCondition, StepResultRoute + +pytestmark = [pytest.mark.layer("recipe"), pytest.mark.small] + +REVIEW_LOOP_RECIPES = ["implementation", "remediation", "implementation-groups"] +RULE_ID = "review-loop-waypoint-guard" + + +def _make_recipe(steps: dict[str, RecipeStep]) -> Recipe: + return Recipe(name="test", description="test", steps=steps, kitchen_rules=["test"]) + + +def test_waypoint_rule_is_registered(): + """The rule must be registered in the semantic rule registry.""" + rule_ids = {r.name for r in _RULE_REGISTRY} + assert RULE_ID in rule_ids, ( + f"{RULE_ID} not found in rule registry. Registered rules: {sorted(rule_ids)}" + ) + + +@pytest.mark.parametrize("recipe_name", REVIEW_LOOP_RECIPES) +def test_waypoint_rule_silent_on_fixed_recipes(recipe_name): + """ + After Part A fix (all verdicts route through check_review_loop), the rule + must not fire on any of the three review-loop recipes. + This is the primary regression guard. + """ + recipe = load_recipe(builtin_recipes_dir() / f"{recipe_name}.yaml") + violations = run_semantic_rules(recipe) + rule_violations = [v for v in violations if v.rule == RULE_ID] + assert len(rule_violations) == 0, ( + f"[{recipe_name}] {RULE_ID} fired after Part A fix: {rule_violations}" + ) + + +def test_waypoint_rule_fires_on_recipe_with_bypass(): + """ + When a recipe has a direct review_pr → check_repo_ci_event route (the old + bug pattern), the rule must fire with a descriptive violation. + """ + recipe = _make_recipe( + { + "review_pr": RecipeStep( + on_result=StepResultRoute( + conditions=[StepResultCondition(when="true", route="check_repo_ci_event")] + ) + ), + "check_review_loop": RecipeStep(on_success="check_repo_ci_event"), + "check_repo_ci_event": RecipeStep(), + } + ) + violations = run_semantic_rules(recipe) + rule_violations = [v for v in violations if v.rule == RULE_ID] + assert len(rule_violations) == 1 + assert "check_review_loop" in rule_violations[0].message + assert "check_repo_ci_event" in rule_violations[0].message + + +def test_waypoint_rule_silent_when_all_verdicts_go_through_barrier(): + """ + When review_pr routes all verdicts through check_review_loop, rule must not fire. + """ + recipe = _make_recipe( + { + "review_pr": RecipeStep( + on_result=StepResultRoute( + conditions=[StepResultCondition(when="true", route="check_review_loop")] + ) + ), + "check_review_loop": RecipeStep( + on_result=StepResultRoute( + conditions=[StepResultCondition(when="true", route="check_repo_ci_event")] + ) + ), + "check_repo_ci_event": RecipeStep(), + } + ) + violations = run_semantic_rules(recipe) + rule_violations = [v for v in violations if v.rule == RULE_ID] + assert len(rule_violations) == 0 + + +def test_waypoint_rule_skips_recipes_without_review_loop_steps(): + """ + Recipes that don't have the review_pr / check_review_loop / check_repo_ci_event + trio are not subject to this rule. + """ + recipe = _make_recipe( + { + "merge_worktree": RecipeStep(tool="merge_worktree"), + "push_to_remote": RecipeStep(tool="push_to_remote"), + } + ) + violations = run_semantic_rules(recipe) + rule_violations = [v for v in violations if v.rule == RULE_ID] + assert len(rule_violations) == 0 diff --git a/tests/recipe/test_rules_shadowed_input.py b/tests/recipe/test_rules_shadowed_input.py new file mode 100644 index 0000000000..65e01f2d06 --- /dev/null +++ b/tests/recipe/test_rules_shadowed_input.py @@ -0,0 +1,101 @@ +from __future__ import annotations + +import pytest + +from autoskillit.recipe.io import _parse_step +from autoskillit.recipe.schema import Recipe, RecipeIngredient +from autoskillit.recipe.validator import run_semantic_rules + +pytestmark = [pytest.mark.layer("recipe"), pytest.mark.small] + + +class TestShadowedRequiredInput: + """Tests for the shadowed-required-input semantic rule.""" + + def test_fires_when_required_input_in_context_but_passed_as_prose(self) -> None: + """Rule fires when plan_path is an ingredient but skill_command passes prose text.""" + recipe = Recipe( + name="test", + description="test", + ingredients={ + "plan_path": RecipeIngredient(description="Plan file path", required=True), + }, + steps={ + "implement": _parse_step( + { + "tool": "run_skill", + "with": { + "skill_command": ( + "/autoskillit:implement-worktree-no-merge temp/my-plan.md" + ), + "cwd": "/tmp", + }, + "on_success": "done", + } + ), + "done": _parse_step({"action": "stop", "message": "Done."}), + }, + kitchen_rules=["test"], + ) + findings = run_semantic_rules(recipe) + assert any( + f.rule == "shadowed-required-input" and f.step_name == "implement" for f in findings + ), "Expected shadowed-required-input finding when plan_path is ingredient but prose passed" + + def test_clean_when_template_ref_used(self) -> None: + """Rule is silent when skill_command uses ${{ context.plan_path }} template reference.""" + recipe = Recipe( + name="test", + description="test", + ingredients={ + "plan_path": RecipeIngredient(description="Plan file path", required=True), + }, + steps={ + "implement": _parse_step( + { + "tool": "run_skill", + "with": { + "skill_command": ( + "/autoskillit:implement-worktree-no-merge ${{ context.plan_path }}" + ), + "cwd": "/tmp", + }, + "on_success": "done", + } + ), + "done": _parse_step({"action": "stop", "message": "Done."}), + }, + kitchen_rules=["test"], + ) + findings = run_semantic_rules(recipe) + assert not any(f.rule == "shadowed-required-input" for f in findings), ( + "Expected no shadowed-required-input finding when template ref is used" + ) + + def test_clean_when_input_not_yet_in_context(self) -> None: + """Rule is silent when plan_path is not an ingredient and not in available context.""" + recipe = Recipe( + name="test", + description="test", + ingredients={}, # plan_path not declared — not yet available + steps={ + "implement": _parse_step( + { + "tool": "run_skill", + "with": { + "skill_command": ( + "/autoskillit:implement-worktree-no-merge temp/my-plan.md" + ), + "cwd": "/tmp", + }, + "on_success": "done", + } + ), + "done": _parse_step({"action": "stop", "message": "Done."}), + }, + kitchen_rules=["test"], + ) + findings = run_semantic_rules(recipe) + assert not any(f.rule == "shadowed-required-input" for f in findings), ( + "Expected no shadowed-required-input finding when input is not available in context" + ) diff --git a/tests/recipe/test_rules_skill_command_prefix.py b/tests/recipe/test_rules_skill_command_prefix.py new file mode 100644 index 0000000000..d601c5812d --- /dev/null +++ b/tests/recipe/test_rules_skill_command_prefix.py @@ -0,0 +1,114 @@ +from __future__ import annotations + +import pytest + +from autoskillit.core.types import Severity +from autoskillit.recipe.validator import run_semantic_rules +from tests.recipe.conftest import _make_workflow + +pytestmark = [pytest.mark.layer("recipe"), pytest.mark.small] + + +class TestSkillCommandMissingPrefixRule: + """Tests for the skill-command-missing-prefix semantic rule.""" + + def test_scp1_prose_run_skill_warns(self) -> None: + """SCP1: run_skill with prose skill_command → WARNING finding.""" + wf = _make_workflow( + { + "step": { + "tool": "run_skill", + "with": {"skill_command": "Fix the auth bug in main.py", "cwd": "/tmp"}, + "on_success": "done", + }, + "done": {"action": "stop", "message": "Done."}, + } + ) + findings = run_semantic_rules(wf) + assert any( + f.rule == "skill-command-missing-prefix" and f.severity == Severity.WARNING + for f in findings + ), "Expected skill-command-missing-prefix WARNING for prose skill_command" + + def test_scp2_prose_run_skill_warns(self) -> None: + """SCP2: prose skill_command with on_failure step → WARNING finding. + + Distinguishing input: step has both on_success and on_failure transitions, + and skill_command is a plain prose string without /autoskillit: prefix. + """ + wf = _make_workflow( + { + "step": { + "tool": "run_skill", + "with": {"skill_command": "Investigate the bug", "cwd": "/tmp"}, + "on_success": "done", + "on_failure": "done", + }, + "done": {"action": "stop", "message": "Done."}, + } + ) + findings = run_semantic_rules(wf) + assert any(f.rule == "skill-command-missing-prefix" for f in findings) + + def test_scp3_autoskillit_prefix_no_warning(self) -> None: + """SCP3: /autoskillit:investigate → no skill-command-missing-prefix warning.""" + wf = _make_workflow( + { + "step": { + "tool": "run_skill", + "with": {"skill_command": "/autoskillit:investigate error", "cwd": "/tmp"}, + "on_success": "done", + }, + "done": {"action": "stop", "message": "Done."}, + } + ) + findings = run_semantic_rules(wf) + assert not any(f.rule == "skill-command-missing-prefix" for f in findings) + + def test_scp4_bare_slash_local_skill_no_warning(self) -> None: + """SCP4: /audit-arch (local skill, starts with /) → no warning.""" + wf = _make_workflow( + { + "step": { + "tool": "run_skill", + "with": {"skill_command": "/audit-arch", "cwd": "/tmp"}, + "on_success": "done", + }, + "done": {"action": "stop", "message": "Done."}, + } + ) + findings = run_semantic_rules(wf) + assert not any(f.rule == "skill-command-missing-prefix" for f in findings) + + def test_scp5_dynamic_prefix_no_warning(self) -> None: + """SCP5: /audit-${{ inputs.audit_type }} → no warning (starts with /).""" + wf = _make_workflow( + { + "step": { + "tool": "run_skill", + "with": { + "skill_command": "/audit-${{ inputs.audit_type }}", + "cwd": "/tmp", + }, + "on_success": "done", + }, + "done": {"action": "stop", "message": "Done."}, + } + ) + findings = run_semantic_rules(wf) + assert not any(f.rule == "skill-command-missing-prefix" for f in findings) + + def test_scp6_non_skill_tool_no_warning(self) -> None: + """SCP6: run_cmd step (not run_skill) → rule does not fire.""" + wf = _make_workflow( + { + "step": { + "tool": "run_cmd", + "with": {"cmd": "ls -la", "cwd": "/tmp"}, + "on_success": "done", + }, + "done": {"action": "stop", "message": "Done."}, + } + ) + findings = run_semantic_rules(wf) + assert not any(f.rule == "skill-command-missing-prefix" for f in findings) diff --git a/tests/recipe/test_rules_skill_content.py b/tests/recipe/test_rules_skill_content.py index e2e68c6058..f27180f9af 100644 --- a/tests/recipe/test_rules_skill_content.py +++ b/tests/recipe/test_rules_skill_content.py @@ -9,10 +9,13 @@ import pytest -import autoskillit.recipe.rules_skill_content as _rsc +import autoskillit.recipe.rules.rules_skill_content as _rsc +from autoskillit.core import Severity from autoskillit.recipe.io import load_recipe from autoskillit.recipe.registry import run_semantic_rules +pytestmark = [pytest.mark.layer("recipe"), pytest.mark.small] + # Minimal recipe YAML that calls a synthetic bad skill with an undefined placeholder. # The YAML key for skill arguments is `with:` (maps to `with_args` via _parse_recipe). _SYNTHETIC_BAD_SKILL_MD = textwrap.dedent( @@ -177,7 +180,7 @@ def test_output_section_no_markdown_rule_fires_when_directive_missing(tmp_path: with ( patch.object(_rsc, "SKILL_SEARCH_DIRS", [tmp_path]), patch( - "autoskillit.recipe.rules_skill_content.load_bundled_manifest", + "autoskillit.recipe.rules.rules_skill_content.load_bundled_manifest", return_value=_MOCK_MANIFEST_WITH_PATTERNS, ), ): @@ -218,7 +221,7 @@ def test_output_section_no_markdown_rule_passes_when_directive_present(tmp_path: with ( patch.object(_rsc, "SKILL_SEARCH_DIRS", [tmp_path]), patch( - "autoskillit.recipe.rules_skill_content.load_bundled_manifest", + "autoskillit.recipe.rules.rules_skill_content.load_bundled_manifest", return_value=_MOCK_MANIFEST_WITH_PATTERNS, ), ): @@ -630,3 +633,253 @@ def test_no_autoskillit_import_zero_findings_on_bundled_recipes() -> None: f"Expected zero findings for {_PKG_RULE_ID!r}, got {len(pkg_findings)}: " + "; ".join(f.message for f in pkg_findings) ) + + +# --------------------------------------------------------------------------- +# grep-bre-alternation-in-skill tests +# --------------------------------------------------------------------------- + +_BRE_RULE_ID = "grep-bre-alternation-in-skill" + + +def test_grep_bre_alternation_is_flagged(tmp_path: Path) -> None: + """grep BRE \\| in a bash block must produce a rule finding.""" + skill_dir = tmp_path / "test-skill" + skill_dir.mkdir() + (skill_dir / "SKILL.md").write_text( + textwrap.dedent( + """\ + # test-skill + + ### Step 1 + ```bash + grep -in 'foo\\|bar' some_file.txt + ``` + """ + ) + ) + recipe_path = tmp_path / "recipe.yaml" + recipe_path.write_text(_make_recipe_for_skill("test-skill", {})) + recipe = load_recipe(recipe_path) + with patch.object(_rsc, "SKILL_SEARCH_DIRS", [tmp_path]): + findings = run_semantic_rules(recipe) + assert _BRE_RULE_ID in [f.rule for f in findings], ( + "Expected rule to fire for grep BRE \\| pattern in bash block" + ) + + +def test_git_grep_bre_is_excluded(tmp_path: Path) -> None: + """git log --grep='foo\\|bar' must NOT produce a rule finding (BRE correct for git).""" + skill_dir = tmp_path / "test-skill" + skill_dir.mkdir() + (skill_dir / "SKILL.md").write_text( + textwrap.dedent( + """\ + # test-skill + + ### Step 1 + ```bash + git log --oneline --grep="fix\\|revert" -- src/ + ``` + """ + ) + ) + recipe_path = tmp_path / "recipe.yaml" + recipe_path.write_text(_make_recipe_for_skill("test-skill", {})) + recipe = load_recipe(recipe_path) + with patch.object(_rsc, "SKILL_SEARCH_DIRS", [tmp_path]): + findings = run_semantic_rules(recipe) + assert _BRE_RULE_ID not in [f.rule for f in findings], ( + "Rule must not fire for --grep= BRE context (git uses BRE for --grep=)" + ) + + +def test_git_remote_command_re_imported_from_git_helpers() -> None: + """_GIT_REMOTE_COMMAND_RE must be imported from _git_helpers, not defined locally.""" + import autoskillit.recipe._git_helpers as _gh + import autoskillit.recipe.rules.rules_skill_content as _rsc # noqa: F401 + + # The regex object in rules_skill_content must be the same object as in _git_helpers + # (identity check confirms it's an import, not a re-definition). + assert _rsc._GIT_REMOTE_COMMAND_RE is _gh._GIT_REMOTE_COMMAND_RE + assert _rsc._LITERAL_ORIGIN_RE is _gh._LITERAL_ORIGIN_RE + + +@pytest.mark.parametrize( + "skill_name,ingredients", + [ + ("implement-worktree-no-merge", {"plan_path": "plan"}), + ("implement-experiment", {"plan_path": "plan"}), + ("merge-pr", {}), + ("review-pr", {}), + ("pipeline-summary", {}), + ], +) +def test_hardcoded_origin_does_not_fire_on_part_b_fixed_skills( + tmp_path: Path, skill_name: str, ingredients: dict +) -> None: + """ + Regression anchor: bundled skills fixed in Part B must NOT trigger hardcoded-origin-remote. + + Uses SKILL_SEARCH_DIRS isolation so the test fails with a clear assertion error + (not an opaque ENOENT) if the skill is renamed. + """ + from autoskillit.workspace import DefaultSkillResolver # noqa: PLC0415 + + skill_info = DefaultSkillResolver().resolve(skill_name) + assert skill_info is not None, f"bundled {skill_name!r} skill not found" + skill_dir = tmp_path / skill_name + skill_dir.mkdir() + (skill_dir / "SKILL.md").write_bytes(skill_info.path.read_bytes()) + recipe_path = tmp_path / "recipe.yaml" + recipe_path.write_text(_make_recipe_for_skill(skill_name, ingredients)) + recipe = load_recipe(recipe_path) + with patch.object(_rsc, "SKILL_SEARCH_DIRS", [tmp_path]): + findings = run_semantic_rules(recipe) + assert "hardcoded-origin-remote" not in [f.rule for f in findings], ( + f"hardcoded-origin-remote fired on {skill_name!r} after Part B fix — " + "check that all literal 'origin' references in bash blocks have been replaced with $REMOTE" + ) + + +def test_skill_no_issue_comments_rule_registered() -> None: + import autoskillit.recipe.rules.rules_skill_content # noqa: F401 — triggers decorator registration + from autoskillit.recipe.registry import _RULE_REGISTRY + + rule_names = [r.name for r in _RULE_REGISTRY] + assert "skill-no-issue-comments" in rule_names + + +# --------------------------------------------------------------------------- +# transition-boundary-anti-confirmation tests +# --------------------------------------------------------------------------- + +_BOUNDARY_RULE_ID = "transition-boundary-anti-confirmation" + +_SYNTHETIC_BOUNDARY_SKILL_FIRES = textwrap.dedent( + """\ + # test-skill + + ## EXECUTION MAP + + Group iteration is outer loop. The group number (1, 2, 3, ...) is the + primary ordering within the dispatch plan. + Merge-wait is mandatory between groups. After all Group N pipelines + complete, verify all Group N PRs have merged before starting Group N+1. + """ +) + +_SYNTHETIC_BOUNDARY_SKILL_PASSES = textwrap.dedent( + """\ + # test-skill + + ## EXECUTION MAP + + Group iteration is outer loop. The group number (1, 2, 3, ...) is the + primary ordering within the dispatch plan. + Merge-wait is mandatory between groups. After all Group N pipelines + complete, verify all Group N PRs have merged before starting Group N+1. + NEVER use AskUserQuestion to ask whether to proceed to the next group. + """ +) + + +def test_transition_boundary_anti_confirmation_rule_fires(tmp_path: Path) -> None: + """Rule must emit a finding when a SKILL.md section describes a transition boundary + but contains no anti-confirmation instruction.""" + skill_dir = tmp_path / "test-skill" + skill_dir.mkdir() + (skill_dir / "SKILL.md").write_text(_SYNTHETIC_BOUNDARY_SKILL_FIRES) + + recipe_path = tmp_path / "recipe.yaml" + recipe_path.write_text(_make_recipe_for_skill("test-skill", {})) + recipe = load_recipe(recipe_path) + + with patch.object(_rsc, "SKILL_SEARCH_DIRS", [tmp_path]): + findings = run_semantic_rules(recipe) + + rule_ids = [f.rule for f in findings] + assert _BOUNDARY_RULE_ID in rule_ids, ( + f"Expected '{_BOUNDARY_RULE_ID}' finding for skill with unprotected group boundary, " + f"got: {rule_ids}" + ) + matching = [f for f in findings if f.rule == _BOUNDARY_RULE_ID] + assert matching[0].severity == Severity.WARNING + + +def test_transition_boundary_anti_confirmation_rule_passes(tmp_path: Path) -> None: + """Rule must NOT emit a finding when the transition boundary has an anti-confirmation + instruction in the same section.""" + skill_dir = tmp_path / "test-skill" + skill_dir.mkdir() + (skill_dir / "SKILL.md").write_text(_SYNTHETIC_BOUNDARY_SKILL_PASSES) + + recipe_path = tmp_path / "recipe.yaml" + recipe_path.write_text(_make_recipe_for_skill("test-skill", {})) + recipe = load_recipe(recipe_path) + + with patch.object(_rsc, "SKILL_SEARCH_DIRS", [tmp_path]): + findings = run_semantic_rules(recipe) + + rule_ids = [f.rule for f in findings] + assert _BOUNDARY_RULE_ID not in rule_ids, ( + f"Rule must not fire when anti-confirmation instruction is present, got: {rule_ids}" + ) + + +def test_rules_pass_ctx_skill_resolver_to_resolve_skill_md(tmp_path: Path) -> None: + """Rule functions thread ctx.skill_resolver through to _resolve_skill_md.""" + from autoskillit.recipe._analysis import make_validation_context + from autoskillit.workspace.skills import DefaultSkillResolver + + # Build a minimal recipe with a run_skill step that triggers skill content rules + skill_dir = tmp_path / "test-skill" + skill_dir.mkdir() + (skill_dir / "SKILL.md").write_text("# test-skill\n## Arguments\nNone.\n") + + recipe_yaml = tmp_path / "recipe.yaml" + recipe_yaml.write_text( + textwrap.dedent( + """\ + name: test-recipe + kitchen_rules: + - "Use run_skill only." + steps: + run_impl: + tool: run_skill + with: + skill_command: "/autoskillit:test-skill" + on_success: done + done: + action: stop + message: "Done." + """ + ) + ) + recipe = load_recipe(recipe_yaml) + + # Track whether _resolve_skill_md received a non-None resolver + received_resolvers: list[object] = [] + original_fn = _rsc._resolve_skill_md + + def tracking_fn(skill_name: str, *, resolver: object = None) -> object: + received_resolvers.append(resolver) + return original_fn(skill_name, resolver=resolver) + + resolver = DefaultSkillResolver() + ctx = make_validation_context(recipe, skill_resolver=resolver) + + with ( + patch.object(_rsc, "_resolve_skill_md", tracking_fn), + patch.object(_rsc, "SKILL_SEARCH_DIRS", [tmp_path]), + ): + run_semantic_rules(ctx) + + # At least one call should have received the resolver from ctx + non_none = [r for r in received_resolvers if r is not None] + assert len(non_none) > 0, ( + "Expected rule functions to pass ctx.skill_resolver to _resolve_skill_md" + ) + assert all(r is resolver for r in non_none), ( + "Expected every non-None resolver to be the exact instance from ctx" + ) diff --git a/tests/recipe/test_rules_skill_routing.py b/tests/recipe/test_rules_skill_routing.py new file mode 100644 index 0000000000..63c28f5038 --- /dev/null +++ b/tests/recipe/test_rules_skill_routing.py @@ -0,0 +1,148 @@ +"""Tests for skill-result-routing-gap and contract allowed_values rules.""" + +from __future__ import annotations + +import pytest + +from autoskillit.recipe.contracts import load_bundled_manifest +from autoskillit.recipe.io import builtin_recipes_dir, load_recipe +from autoskillit.recipe.registry import run_semantic_rules +from autoskillit.recipe.schema import Recipe, RecipeStep, StepResultCondition, StepResultRoute + +pytestmark = [pytest.mark.layer("recipe"), pytest.mark.small] + + +def _make_recipe(steps: dict[str, RecipeStep]) -> Recipe: + return Recipe( + name="test-skill-routing", + description="Test recipe for skill-result-routing-gap rule.", + version="0.2.0", + kitchen_rules=["test"], + steps=steps, + ) + + +# --------------------------------------------------------------------------- +# skill-result-routing-gap +# --------------------------------------------------------------------------- + + +def test_merge_pr_step_flags_merged_false_not_routed() -> None: + """merge-prs.yaml: merge_pr step has merged=false guard — no finding expected.""" + recipe = load_recipe(builtin_recipes_dir() / "merge-prs.yaml") + findings = run_semantic_rules(recipe) + skill_routing_findings = [ + f for f in findings if f.rule == "skill-result-routing-gap" and f.step_name == "merge_pr" + ] + assert len(skill_routing_findings) == 0, ( + "merge_pr step in merge-prs.yaml must not trigger skill-result-routing-gap" + ) + + +def test_skill_result_routing_gap_passes_when_all_values_routed() -> None: + """Recipe where run_skill step routes all allowed_values explicitly must not fire the rule.""" + steps = { + "merge": RecipeStep( + tool="run_skill", + with_args={"skill_command": "/autoskillit:merge-pr pr123 simple"}, + capture={"merged": "${{ result.merged }}"}, + on_result=StepResultRoute( + conditions=[ + StepResultCondition( + route="done", + when="${{ result.merged }} == true", + ), + StepResultCondition( + route="escalate", + when="${{ result.merged }} == false", + ), + ] + ), + ), + "done": RecipeStep(action="stop", message="merged"), + "escalate": RecipeStep(action="stop", message="not merged"), + } + recipe = _make_recipe(steps) + findings = run_semantic_rules(recipe) + skill_routing_findings = [f for f in findings if f.rule == "skill-result-routing-gap"] + assert len(skill_routing_findings) == 0, ( + "skill-result-routing-gap must not fire when all allowed values are explicitly routed" + ) + + +def test_skill_result_routing_gap_fires_when_catchall_goes_to_success() -> None: + """Rule must fire when unrouted value falls through to a non-terminal catch-all.""" + steps = { + "merge": RecipeStep( + tool="run_skill", + with_args={"skill_command": "/autoskillit:merge-pr pr123 simple"}, + capture={"merged": "${{ result.merged }}"}, + on_result=StepResultRoute( + conditions=[ + StepResultCondition( + route="done", + when="${{ result.merged }} == true", + ), + StepResultCondition( + route="next_pr", + ), + ] + ), + ), + "done": RecipeStep(action="stop", message="merged"), + "next_pr": RecipeStep(tool="run_cmd", with_args={"cmd": "echo next"}), + } + recipe = _make_recipe(steps) + findings = run_semantic_rules(recipe) + skill_routing_findings = [f for f in findings if f.rule == "skill-result-routing-gap"] + assert len(skill_routing_findings) >= 1, ( + "skill-result-routing-gap must fire when catch-all routes to non-terminal" + ) + + +def test_skill_result_routing_gap_does_not_fire_for_terminal_catchall() -> None: + """Rule must not fire when the catch-all routes to a terminal (action=stop).""" + steps = { + "merge": RecipeStep( + tool="run_skill", + with_args={"skill_command": "/autoskillit:merge-pr pr123 simple"}, + capture={"merged": "${{ result.merged }}"}, + on_result=StepResultRoute( + conditions=[ + StepResultCondition( + route="done", + when="${{ result.merged }} == true", + ), + StepResultCondition( + route="stopped", # catch-all routes to terminal — no silent fallthrough + ), + ] + ), + ), + "done": RecipeStep(action="stop", message="merged"), + "stopped": RecipeStep(action="stop", message="not merged"), + } + recipe = _make_recipe(steps) + findings = run_semantic_rules(recipe) + skill_routing_findings = [f for f in findings if f.rule == "skill-result-routing-gap"] + assert len(skill_routing_findings) == 0, ( + "skill-result-routing-gap must not fire when catch-all routes to terminal" + ) + + +# --------------------------------------------------------------------------- +# Contract: allowed_values on merged output +# --------------------------------------------------------------------------- + + +def test_merge_pr_contract_declares_allowed_values_for_merged() -> None: + """The merge-pr skill contract must declare allowed_values for the merged output.""" + manifest = load_bundled_manifest() + merge_pr_skill = manifest.get("skills", {}).get("merge-pr", {}) + outputs = merge_pr_skill.get("outputs", []) + merged_output = next((o for o in outputs if o.get("name") == "merged"), None) + assert merged_output is not None, "merge-pr skill must have a merged output" + assert "allowed_values" in merged_output, "merged output must declare allowed_values" + assert merged_output["allowed_values"] == ["true", "false"], ( + f"merged allowed_values must be ['true', 'false'], got {merged_output['allowed_values']}" + ) diff --git a/tests/recipe/test_rules_skills.py b/tests/recipe/test_rules_skills.py index e6339613b1..3f09c58a01 100644 --- a/tests/recipe/test_rules_skills.py +++ b/tests/recipe/test_rules_skills.py @@ -2,11 +2,15 @@ from __future__ import annotations +import pytest + from autoskillit.core import Severity from autoskillit.recipe.io import builtin_recipes_dir, load_recipe from autoskillit.recipe.registry import run_semantic_rules from autoskillit.recipe.schema import Recipe, RecipeStep +pytestmark = [pytest.mark.layer("recipe"), pytest.mark.small] + def _make_recipe(skill_command: str) -> Recipe: """Minimal recipe factory for unknown-skill-command rule tests.""" @@ -79,7 +83,7 @@ def test_non_skill_step_not_checked() -> None: def test_get_bundled_skill_names_is_not_lru_cached() -> None: """_get_bundled_skill_names must not use @lru_cache (Composition Root fix).""" - import autoskillit.recipe.rules_skills as mod + import autoskillit.recipe.rules.rules_skills as mod assert not hasattr(mod._get_bundled_skill_names, "cache_clear"), ( "_get_bundled_skill_names must not be lru_cache-decorated" @@ -96,7 +100,7 @@ def test_bundled_skill_names_not_computed_at_import_v2() -> None: from unittest.mock import patch # Remove cached module so reload is a fresh import - mod_name = "autoskillit.recipe.rules_skills" + mod_name = "autoskillit.recipe.rules.rules_skills" sys.modules.pop(mod_name, None) with patch("autoskillit.workspace.skills.DefaultSkillResolver.list_all") as mock_list: diff --git a/tests/recipe/test_rules_structure.py b/tests/recipe/test_rules_structure.py deleted file mode 100644 index 382076f667..0000000000 --- a/tests/recipe/test_rules_structure.py +++ /dev/null @@ -1,1618 +0,0 @@ -"""Tests for structural semantic rules (step reachability, field validation, recipe version).""" - -from __future__ import annotations - -import pytest - -from autoskillit.core.types import Severity -from autoskillit.recipe.io import ( - _parse_step, - builtin_recipes_dir, - load_recipe, -) -from autoskillit.recipe.schema import ( - Recipe, - RecipeIngredient, - RecipeStep, - StepResultRoute, -) -from autoskillit.recipe.validator import ( - RuleFinding, - run_semantic_rules, -) -from tests.recipe.conftest import NO_AUTOSKILLIT_IMPORT as _NO_AUTOSKILLIT_IMPORT -from tests.recipe.conftest import _make_workflow - -# --------------------------------------------------------------------------- -# Module-level semantic rule tests -# --------------------------------------------------------------------------- - - -def test_registry_collects_rules() -> None: - wf = _make_workflow( - { - "do_thing": {"tool": "run_cmd", "on_success": "done"}, - "done": {"action": "stop", "message": "Done."}, - } - ) - findings = run_semantic_rules(wf) - assert isinstance(findings, list) - assert all(isinstance(f, RuleFinding) for f in findings) - - -def test_unsatisfied_input_replaces_worktree_path_check() -> None: - wf = _make_workflow( - { - "implement": { - "tool": "run_skill", - "with": { - "skill_command": ( - "/autoskillit:implement-worktree-no-merge ${{ context.plan_path }}" - ), - }, - "capture": {"worktree_path": "${{ result.worktree_path }}"}, - "retries": 0, - "on_context_limit": "retry_step", - "on_success": "retry_step", - }, - "retry_step": { - "tool": "run_skill", - "with": { - "skill_command": "/autoskillit:retry-worktree ${{ context.plan_path }}", - }, - "on_success": "done", - }, - "done": {"action": "stop", "message": "Done."}, - } - ) - findings = run_semantic_rules(wf) - errors = [f for f in findings if f.severity == Severity.ERROR] - assert any(f.rule == "missing-ingredient" and "worktree_path" in f.message for f in errors) - - -def test_unsatisfied_input_clean_when_provided() -> None: - wf = _make_workflow( - { - "implement": { - "tool": "run_skill", - "with": { - "skill_command": ( - "/autoskillit:implement-worktree-no-merge ${{ context.plan_path }}" - ), - }, - "capture": {"worktree_path": "${{ result.worktree_path }}"}, - "retries": 0, - "on_context_limit": "retry_step", - "on_success": "retry_step", - }, - "retry_step": { - "tool": "run_skill", - "with": { - "skill_command": ( - "/autoskillit:retry-worktree " - "${{ context.plan_path }} ${{ context.worktree_path }}" - ), - }, - "on_success": "done", - }, - "done": {"action": "stop", "message": "Done."}, - } - ) - findings = run_semantic_rules(wf) - assert not any(f.rule == "missing-ingredient" for f in findings) - - -def test_unsatisfied_input_not_available() -> None: - wf = _make_workflow( - { - "retry_step": { - "tool": "run_skill", - "with": {"skill_command": "/autoskillit:retry-worktree ${{ context.plan_path }}"}, - "on_success": "done", - }, - "done": {"action": "stop", "message": "Done."}, - } - ) - findings = run_semantic_rules(wf) - errors = [ - f for f in findings if f.rule == "missing-ingredient" and f.severity == Severity.ERROR - ] - assert any("worktree_path" in f.message for f in errors) - - -def test_unsatisfied_input_unknown_skill_ignored() -> None: - wf = _make_workflow( - { - "step": { - "tool": "run_skill", - "with": {"skill_command": "/some-unknown-skill"}, - "on_success": "done", - }, - "done": {"action": "stop", "message": "Done."}, - } - ) - findings = run_semantic_rules(wf) - assert not any(f.rule == "missing-ingredient" for f in findings) - - -def test_unsatisfied_input_from_pipeline_inputs() -> None: - wf = Recipe( - name="test", - description="test", - ingredients={ - "plan_path": RecipeIngredient(description="Plan file", required=True), - "worktree_path": RecipeIngredient(description="Worktree", required=True), - }, - steps={ - "retry_step": _parse_step( - { - "tool": "run_skill", - "with": { - "skill_command": ( - "/autoskillit:retry-worktree " - "${{ inputs.plan_path }} ${{ inputs.worktree_path }}" - ), - }, - "on_success": "done", - } - ), - "done": _parse_step({"action": "stop", "message": "Done."}), - }, - kitchen_rules=["test"], - ) - findings = run_semantic_rules(wf) - assert not any(f.rule == "missing-ingredient" for f in findings) - - -def test_unsatisfied_input_inline_positional_args_skipped() -> None: - wf = _make_workflow( - { - "investigate": { - "tool": "run_skill", - "with": {"skill_command": "/autoskillit:investigate the test failures"}, - "on_success": "done", - }, - "done": {"action": "stop", "message": "Done."}, - } - ) - findings = run_semantic_rules(wf) - assert not any(f.rule == "missing-ingredient" for f in findings) - - -def test_unreachable_steps_detects_orphan() -> None: - wf = _make_workflow( - { - "start": {"tool": "run_cmd", "on_success": "done"}, - "orphan": {"tool": "run_cmd", "on_success": "done"}, - "done": {"action": "stop", "message": "Done."}, - } - ) - findings = run_semantic_rules(wf) - assert any(f.rule == "unreachable-step" and "orphan" in f.message for f in findings) - - -def test_unreachable_steps_first_step_clean() -> None: - wf = _make_workflow( - { - "start": {"tool": "run_cmd", "on_success": "done"}, - "done": {"action": "stop", "message": "Done."}, - } - ) - findings = run_semantic_rules(wf) - assert not any(f.rule == "unreachable-step" and "start" in f.step_name for f in findings) - - -def test_model_on_non_skill_triggers() -> None: - wf = _make_workflow( - { - "check": {"tool": "test_check", "model": "sonnet", "on_success": "done"}, - "done": {"action": "stop", "message": "Done."}, - } - ) - findings = run_semantic_rules(wf) - assert any(f.rule == "model-on-non-skill-step" for f in findings) - - -def test_model_on_non_skill_clean() -> None: - wf = _make_workflow( - { - "do": {"tool": "run_skill", "model": "sonnet", "on_success": "done"}, - "done": {"action": "stop", "message": "Done."}, - } - ) - findings = run_semantic_rules(wf) - assert not any(f.rule == "model-on-non-skill-step" for f in findings) - - -def test_rule_finding_to_dict() -> None: - finding = RuleFinding( - rule="test-rule", - severity=Severity.WARNING, - step_name="some_step", - message="Something is wrong.", - ) - d = finding.to_dict() - assert d == { - "rule": "test-rule", - "severity": "warning", - "step": "some_step", - "message": "Something is wrong.", - } - - -def test_old_rule_removed() -> None: - from autoskillit.recipe.validator import _RULE_REGISTRY - - assert not any(r.name == "retry-without-worktree-path" for r in _RULE_REGISTRY) - - -def test_bundled_workflows_pass_semantic_rules() -> None: - wf_dir = builtin_recipes_dir() - yaml_files = list(wf_dir.glob("*.yaml")) - assert yaml_files - - for path in yaml_files: - wf = load_recipe(path) - findings = run_semantic_rules(wf) - errors = [ - f - for f in findings - if f.severity == Severity.ERROR and f.rule != _NO_AUTOSKILLIT_IMPORT - ] - assert not errors, ( - f"Bundled workflow {path.name} has error-severity semantic findings: {errors}" - ) - undeclared_findings = [f for f in findings if f.rule == "undeclared-capture-key"] - assert undeclared_findings == [], ( - f"Recipe '{wf.name}' has undeclared-capture-key findings: " + repr(undeclared_findings) - ) - - -# --------------------------------------------------------------------------- -# TestOutdatedScriptVersionRule -# --------------------------------------------------------------------------- - - -class TestOutdatedScriptVersionRule: - @pytest.mark.parametrize( - "script_ver,installed_ver,expected_count", - [ - ("0.1.0", "0.2.0", 1), # MSR1: below installed → fires - ("0.2.0", "0.2.0", 0), # MSR2: matches installed → does not fire - (None, "0.2.0", 1), # MSR3: None → fires - ], - ) - def test_outdated_recipe_version_rule( - self, monkeypatch: pytest.MonkeyPatch, script_ver, installed_ver, expected_count - ) -> None: - import autoskillit - import autoskillit.core.types as _core_types - import autoskillit.recipe.rules_inputs as _rules_mod - - monkeypatch.setattr(autoskillit, "__version__", installed_ver) - monkeypatch.setattr(_core_types, "AUTOSKILLIT_INSTALLED_VERSION", installed_ver) - monkeypatch.setattr(_rules_mod, "AUTOSKILLIT_INSTALLED_VERSION", installed_ver) - wf = _make_workflow( - { - "do_thing": {"tool": "run_cmd", "on_success": "done"}, - "done": {"action": "stop", "message": "Done."}, - } - ) - wf.version = script_ver - findings = [f for f in run_semantic_rules(wf) if f.rule == "outdated-recipe-version"] - assert len(findings) == expected_count - - def test_outdated_recipe_version_rule_severity_is_warning( - self, monkeypatch: pytest.MonkeyPatch - ) -> None: - import autoskillit - import autoskillit.core.types as _core_types - import autoskillit.recipe.rules_inputs as _rules_mod - - monkeypatch.setattr(autoskillit, "__version__", "0.2.0") - monkeypatch.setattr(_core_types, "AUTOSKILLIT_INSTALLED_VERSION", "0.2.0") - monkeypatch.setattr(_rules_mod, "AUTOSKILLIT_INSTALLED_VERSION", "0.2.0") - wf = _make_workflow( - { - "do_thing": {"tool": "run_cmd", "on_success": "done"}, - "done": {"action": "stop", "message": "Done."}, - } - ) - wf.version = "0.1.0" - findings = [f for f in run_semantic_rules(wf) if f.rule == "outdated-recipe-version"] - assert findings[0].severity == Severity.WARNING - - -# --------------------------------------------------------------------------- -# TestWeakConstraintRule -# --------------------------------------------------------------------------- - - -class TestWeakConstraintRule: - def _make_recipe_with_kitchen_rules(self, kitchen_rules: list[str]) -> Recipe: - steps = { - "run": _parse_step({"tool": "test_check", "on_success": "done"}), - "done": _parse_step({"action": "stop", "message": "Done"}), - } - return Recipe(name="test", description="test", steps=steps, kitchen_rules=kitchen_rules) - - def test_weak_constraint_text_detected(self) -> None: - wf = self._make_recipe_with_kitchen_rules(["Only use AutoSkillit MCP tools."]) - findings = run_semantic_rules(wf) - weak = [f for f in findings if f.rule == "weak-constraint-text"] - assert weak - - def test_detailed_constraints_pass(self) -> None: - from autoskillit.core.types import PIPELINE_FORBIDDEN_TOOLS - - tool_list = ", ".join(PIPELINE_FORBIDDEN_TOOLS) - constraint = f"NEVER use native tools ({tool_list}) from the orchestrator." - wf = self._make_recipe_with_kitchen_rules([constraint]) - findings = run_semantic_rules(wf) - weak = [f for f in findings if f.rule == "weak-constraint-text"] - assert not weak - - -# --------------------------------------------------------------------------- -# TestMultipartIterationRule -# --------------------------------------------------------------------------- - - -class TestMultipartIterationRule: - def test_mi1_multipart_rule_warns_on_missing_glob_note(self) -> None: - """T_MI1: multipart-glob-note fires when make-plan step has no *_part_*.md in note.""" - recipe = Recipe( - name="test-recipe", - description="test", - ingredients={}, - steps={ - "plan": RecipeStep( - tool="run_skill", - with_args={"skill_command": "/autoskillit:make-plan inputs.task"}, - on_success="verify", - note="Produces a plan file.", - ), - "verify": RecipeStep( - tool="run_skill", - with_args={"skill_command": "/autoskillit:dry-walkthrough context.plan_path"}, - on_success="done", - ), - "done": RecipeStep(action="stop", message="Done"), - }, - kitchen_rules=[], - ) - warnings = run_semantic_rules(recipe) - rule_names = [w.rule for w in warnings] - assert "multipart-glob-note" in rule_names - - def test_mi2_multipart_rule_passes_compliant_recipe(self) -> None: - """T_MI2: Validator emits no multipart warnings when all conventions are present.""" - recipe = Recipe( - name="test-recipe", - description="test", - ingredients={}, - steps={ - "plan": RecipeStep( - tool="run_skill", - with_args={"skill_command": "/autoskillit:make-plan inputs.task"}, - on_success="verify", - note="Glob plan_dir for *_part_*.md or single plan file.", - ), - "verify": RecipeStep( - tool="run_skill", - with_args={"skill_command": "/autoskillit:dry-walkthrough context.plan_path"}, - on_success="next_or_done", - ), - "next_or_done": RecipeStep( - action="route", - on_result=StepResultRoute( - field="next", routes={"more_parts": "verify", "all_done": "done"} - ), - ), - "done": RecipeStep(action="stop", message="Done"), - }, - kitchen_rules=["SEQUENTIAL EXECUTION: complete full cycle per part before advancing."], - ) - warnings = run_semantic_rules(recipe) - rule_names = [w.rule for w in warnings] - assert "multipart-glob-note" not in rule_names - assert "multipart-sequential-kitchen-rule" not in rule_names - assert "multipart-route-back" not in rule_names - - -# --------------------------------------------------------------------------- -# Multipart plan_parts capture tests (D6–D7) -# --------------------------------------------------------------------------- - - -@pytest.fixture -def compliant_multipart_recipe_no_list() -> Recipe: - """Recipe with make-plan step but no capture_list for plan_parts.""" - return Recipe( - name="test", - description="test", - ingredients={}, - steps={ - "plan": RecipeStep( - tool="run_skill", - with_args={"skill_command": "/autoskillit:make-plan inputs.task"}, - capture={"plan_path": "${{ result.plan_path }}"}, - note="Glob plan_dir for *_part_*.md or single plan file. Sort into plan_parts[].", - on_success="done", - ), - "done": RecipeStep(action="stop", message="Done"), - }, - kitchen_rules=["SEQUENTIAL EXECUTION: complete full cycle per part."], - ) - - -@pytest.fixture -def compliant_multipart_recipe_with_list() -> Recipe: - """Recipe with make-plan step and correct capture_list for plan_parts.""" - return Recipe( - name="test", - description="test", - ingredients={}, - steps={ - "plan": RecipeStep( - tool="run_skill", - with_args={"skill_command": "/autoskillit:make-plan inputs.task"}, - capture={"plan_path": "${{ result.plan_path }}"}, - capture_list={"plan_parts": "${{ result.plan_parts }}"}, - note="Glob plan_dir for *_part_*.md or single plan file. Sort into plan_parts[].", - on_success="done", - ), - "done": RecipeStep(action="stop", message="Done"), - }, - kitchen_rules=["SEQUENTIAL EXECUTION: complete full cycle per part."], - ) - - -def test_validator_warns_when_plan_parts_not_captured( - compliant_multipart_recipe_no_list: Recipe, -) -> None: - """D6: Validator warns when make-plan step lacks capture_list for plan_parts.""" - warnings = run_semantic_rules(compliant_multipart_recipe_no_list) - rule_names = [w.rule for w in warnings] - assert "multipart-plan-parts-not-captured" in rule_names - - -def test_validator_passes_when_plan_parts_captured( - compliant_multipart_recipe_with_list: Recipe, -) -> None: - """D7: Validator passes when make-plan step has capture_list for plan_parts.""" - warnings = run_semantic_rules(compliant_multipart_recipe_with_list) - rule_names = [w.rule for w in warnings] - assert "multipart-plan-parts-not-captured" not in rule_names - - -# --------------------------------------------------------------------------- -# TestOnResultMissingFailureRoute -# --------------------------------------------------------------------------- - - -class TestOnResultMissingFailureRoute: - """RCA: on-result-missing-failure-route semantic rule.""" - - def test_RCA1_tool_step_on_result_no_on_failure_fires_error(self) -> None: - """RCA1: run_skill step with on_result but no on_failure → Severity.ERROR finding.""" - wf = _make_workflow( - { - "audit": { - "tool": "run_skill", - "with": {"skill_command": "/autoskillit:audit-impl plan.md impl main"}, - "capture": {"verdict": "${{ result.verdict }}"}, - "on_result": {"field": "verdict", "routes": {"GO": "done", "NO GO": "fix"}}, - # no on_failure — the gap - }, - "fix": {"action": "stop", "message": "Fix needed."}, - "done": {"action": "stop", "message": "Done."}, - } - ) - findings = run_semantic_rules(wf) - errors = [f for f in findings if f.severity == Severity.ERROR] - rule_names = [f.rule for f in errors] - assert any( - f.rule == "on-result-missing-failure-route" and f.step_name == "audit" for f in errors - ), f"Expected on-result-missing-failure-route ERROR on 'audit'. Got: {rule_names}" - - def test_RCA2_python_step_on_result_no_on_failure_fires_error(self) -> None: - """RCA2: python step with on_result but no on_failure → Severity.ERROR.""" - wf = _make_workflow( - { - "check": { - "python": "mymod.check_result", - "on_result": {"field": "status", "routes": {"ok": "done", "fail": "fix"}}, - # no on_failure - }, - "fix": {"action": "stop", "message": "Fix."}, - "done": {"action": "stop", "message": "Done."}, - } - ) - findings = run_semantic_rules(wf) - errors = [f for f in findings if f.severity == Severity.ERROR] - assert any(f.rule == "on-result-missing-failure-route" for f in errors) - - def test_RCA3_on_result_with_on_failure_no_finding(self) -> None: - """RCA3: on_result + on_failure present → rule does not fire.""" - wf = _make_workflow( - { - "audit": { - "tool": "run_skill", - "with": {"skill_command": "/autoskillit:audit-impl plan.md impl main"}, - "capture": {"verdict": "${{ result.verdict }}"}, - "on_result": {"field": "verdict", "routes": {"GO": "done", "NO GO": "fix"}}, - "on_failure": "done", - }, - "fix": {"action": "stop", "message": "Fix needed."}, - "done": {"action": "stop", "message": "Done."}, - } - ) - findings = run_semantic_rules(wf) - assert not any(f.rule == "on-result-missing-failure-route" for f in findings) - - def test_RCA4_action_route_on_result_no_on_failure_not_an_error(self) -> None: - """RCA4: action:route with on_result but no on_failure → NOT flagged. - Agent routing decisions are not MCP tool invocations; they cannot fail - the same way and are exempt from this rule. - """ - wf = _make_workflow( - { - "decide": { - "action": "route", - "on_result": { - "field": "parts", - "routes": {"more": "implement", "done": "finish"}, - }, - # no on_failure — intentional for action:route - }, - "implement": {"action": "stop", "message": "Implement."}, - "finish": {"action": "stop", "message": "Finish."}, - } - ) - findings = run_semantic_rules(wf) - assert not any(f.rule == "on-result-missing-failure-route" for f in findings) - - def test_RCA5_optional_true_plus_on_result_no_on_failure_fires(self) -> None: - """RCA5: optional:true does not exempt a step from needing on_failure.""" - wf = _make_workflow( - { - "audit": { - "tool": "run_skill", - "with": {"skill_command": "/autoskillit:audit-impl plan.md impl main"}, - "capture": {"verdict": "${{ result.verdict }}"}, - "on_result": {"field": "verdict", "routes": {"GO": "done", "NO GO": "fix"}}, - "optional": True, - # no on_failure - }, - "fix": {"action": "stop", "message": "Fix needed."}, - "done": {"action": "stop", "message": "Done."}, - } - ) - findings = run_semantic_rules(wf) - errors = [f for f in findings if f.severity == Severity.ERROR] - assert any(f.rule == "on-result-missing-failure-route" for f in errors) - - -# --------------------------------------------------------------------------- -# TestOnContextLimitField -# --------------------------------------------------------------------------- - - -class TestOnContextLimitField: - """Tests for on_context_limit as a routing field and cycle detection.""" - - def test_on_context_limit_invalid_target_raises_validation_error(self) -> None: - """on_context_limit must reference a declared step name.""" - from autoskillit.recipe.validator import validate_recipe - - recipe = Recipe( - name="test", - description="test", - summary="test", - ingredients={}, - kitchen_rules=["test"], - steps={ - "fix": RecipeStep( - tool="run_skill", - on_success="done", - on_failure="cleanup", - on_context_limit="nonexistent_step", - with_args={"skill_command": "x", "cwd": "/tmp"}, - ), - "cleanup": RecipeStep(action="stop", message="done"), - "done": RecipeStep(action="stop", message="done"), - }, - ) - errors = validate_recipe(recipe) - assert errors, "Expected validation errors for unknown on_context_limit target" - assert any("on_context_limit" in e for e in errors) - - def test_on_context_limit_valid_target_passes_validation(self) -> None: - """on_context_limit referencing a valid step passes validation.""" - from autoskillit.recipe.validator import validate_recipe - - recipe = Recipe( - name="test", - description="test", - summary="test", - ingredients={}, - kitchen_rules=["test"], - steps={ - "implement": RecipeStep( - tool="run_skill", - on_success="done", - on_failure="cleanup", - on_context_limit="retry_worktree", - retries=0, - with_args={"skill_command": "/autoskillit:implement-worktree-no-merge x"}, - ), - "retry_worktree": RecipeStep( - tool="run_skill", - on_success="done", - on_failure="cleanup", - with_args={"skill_command": "/autoskillit:retry-worktree x y"}, - ), - "cleanup": RecipeStep(action="stop", message="done"), - "done": RecipeStep(action="stop", message="done"), - }, - ) - errors = validate_recipe(recipe) - assert not errors, f"Expected no errors but got: {errors}" - - def test_on_exhausted_invalid_target_raises_validation_error(self) -> None: - """on_exhausted must reference a declared step name or be a reserved terminal.""" - from autoskillit.recipe.validator import validate_recipe - - recipe = Recipe( - name="test", - description="test", - summary="test", - ingredients={}, - kitchen_rules=["test"], - steps={ - "fix": RecipeStep( - tool="run_skill", - on_success="done", - on_failure="cleanup", - on_exhausted="nonexistent_step", - with_args={"skill_command": "x", "cwd": "/tmp"}, - ), - "cleanup": RecipeStep(action="stop", message="done"), - "done": RecipeStep(action="stop", message="done"), - }, - ) - errors = validate_recipe(recipe) - assert errors, "Expected validation errors for unknown on_exhausted target" - - def test_on_exhausted_escalate_reserved_passes_validation(self) -> None: - """on_exhausted: 'escalate' is reserved — passes validation without an escalate step.""" - from autoskillit.recipe.validator import validate_recipe - - recipe = Recipe( - name="test", - description="test", - summary="test", - ingredients={}, - kitchen_rules=["test"], - steps={ - "fix": RecipeStep( - tool="run_skill", - on_success="done", - on_failure="done", - on_exhausted="escalate", - with_args={"skill_command": "/autoskillit:investigate x"}, - ), - "done": RecipeStep(action="stop", message="done"), - }, - ) - errors = validate_recipe(recipe) - assert not errors, f"Expected no errors but got: {errors}" - - def test_unbounded_cycle_without_retries_produces_warning(self) -> None: - """verify → assess → verify cycle with retries=0 must produce a warning.""" - recipe = Recipe( - name="test", - description="test", - summary="test", - ingredients={}, - kitchen_rules=["test"], - steps={ - "assess": RecipeStep( - tool="run_skill", - on_success="verify", - on_failure="cleanup", - retries=0, - with_args={"skill_command": "x", "cwd": "/tmp"}, - ), - "verify": RecipeStep( - tool="test_check", - on_success="done", - on_failure="assess", - with_args={"worktree_path": "/tmp"}, - ), - "cleanup": RecipeStep(action="stop", message="done"), - "done": RecipeStep(action="stop", message="done"), - }, - ) - findings = run_semantic_rules(recipe) - warnings = [f for f in findings if f.severity == Severity.WARNING] - assert any( - "unbounded" in f.message.lower() or "cycle" in f.message.lower() for f in warnings - ) - - def test_cycle_with_retries_warns_when_success_stays_in_cycle(self) -> None: - """A cycle where the retrying step's success path stays inside the cycle - must emit a WARNING. fix → test → fix: fix.on_success='test' re-enters - the cycle, so the outer loop is unbounded despite the retry exit.""" - recipe = Recipe( - name="test", - description="test", - summary="test", - ingredients={}, - kitchen_rules=["test"], - steps={ - "fix": RecipeStep( - tool="run_skill", - on_success="test", - on_failure="cleanup", - retries=3, - on_exhausted="cleanup", - with_args={"skill_command": "x", "cwd": "/tmp"}, - ), - "test": RecipeStep( - tool="test_check", - on_success="done", - on_failure="fix", - with_args={"worktree_path": "/tmp"}, - ), - "cleanup": RecipeStep(action="stop", message="done"), - "done": RecipeStep(action="stop", message="done"), - }, - ) - findings = run_semantic_rules(recipe) - cycle_warnings = [ - f for f in findings if "cycle" in f.message.lower() or "unbounded" in f.message.lower() - ] - assert len(cycle_warnings) >= 1, "Expected a cycle WARNING but got none" - assert any(f.severity == Severity.WARNING for f in cycle_warnings) - - def test_truly_trapped_cycle_without_exit_produces_error(self) -> None: - """A cycle where every step's edges stay inside the cycle must produce an ERROR.""" - recipe = Recipe( - name="test", - description="test", - summary="test", - ingredients={}, - kitchen_rules=["test"], - steps={ - "assess": RecipeStep( - tool="run_skill", - on_success="verify", - on_failure="verify", - retries=0, - with_args={"skill_command": "x", "cwd": "/tmp"}, - ), - "verify": RecipeStep( - tool="test_check", - on_success="assess", - on_failure="assess", - with_args={"worktree_path": "/tmp"}, - ), - }, - ) - findings = run_semantic_rules(recipe) - errors = [f for f in findings if f.severity == Severity.ERROR] - assert any( - "cycle" in f.message.lower() or "unbounded" in f.message.lower() for f in errors - ) - - -# --------------------------------------------------------------------------- -# TestSkillCommandMissingPrefixRule -# --------------------------------------------------------------------------- - - -class TestSkillCommandMissingPrefixRule: - """Tests for the skill-command-missing-prefix semantic rule.""" - - def test_scp1_prose_run_skill_warns(self) -> None: - """SCP1: run_skill with prose skill_command → WARNING finding.""" - wf = _make_workflow( - { - "step": { - "tool": "run_skill", - "with": {"skill_command": "Fix the auth bug in main.py", "cwd": "/tmp"}, - "on_success": "done", - }, - "done": {"action": "stop", "message": "Done."}, - } - ) - findings = run_semantic_rules(wf) - assert any( - f.rule == "skill-command-missing-prefix" and f.severity == Severity.WARNING - for f in findings - ), "Expected skill-command-missing-prefix WARNING for prose skill_command" - - def test_scp2_prose_run_skill_warns(self) -> None: - """SCP2: prose skill_command with on_failure step → WARNING finding. - - Distinguishing input: step has both on_success and on_failure transitions, - and skill_command is a plain prose string without /autoskillit: prefix. - """ - wf = _make_workflow( - { - "step": { - "tool": "run_skill", - "with": {"skill_command": "Investigate the bug", "cwd": "/tmp"}, - "on_success": "done", - "on_failure": "done", - }, - "done": {"action": "stop", "message": "Done."}, - } - ) - findings = run_semantic_rules(wf) - assert any(f.rule == "skill-command-missing-prefix" for f in findings) - - def test_scp3_autoskillit_prefix_no_warning(self) -> None: - """SCP3: /autoskillit:investigate → no skill-command-missing-prefix warning.""" - wf = _make_workflow( - { - "step": { - "tool": "run_skill", - "with": {"skill_command": "/autoskillit:investigate error", "cwd": "/tmp"}, - "on_success": "done", - }, - "done": {"action": "stop", "message": "Done."}, - } - ) - findings = run_semantic_rules(wf) - assert not any(f.rule == "skill-command-missing-prefix" for f in findings) - - def test_scp4_bare_slash_local_skill_no_warning(self) -> None: - """SCP4: /audit-arch (local skill, starts with /) → no warning.""" - wf = _make_workflow( - { - "step": { - "tool": "run_skill", - "with": {"skill_command": "/audit-arch", "cwd": "/tmp"}, - "on_success": "done", - }, - "done": {"action": "stop", "message": "Done."}, - } - ) - findings = run_semantic_rules(wf) - assert not any(f.rule == "skill-command-missing-prefix" for f in findings) - - def test_scp5_dynamic_prefix_no_warning(self) -> None: - """SCP5: /audit-${{ inputs.audit_type }} → no warning (starts with /).""" - wf = _make_workflow( - { - "step": { - "tool": "run_skill", - "with": { - "skill_command": "/audit-${{ inputs.audit_type }}", - "cwd": "/tmp", - }, - "on_success": "done", - }, - "done": {"action": "stop", "message": "Done."}, - } - ) - findings = run_semantic_rules(wf) - assert not any(f.rule == "skill-command-missing-prefix" for f in findings) - - def test_scp6_non_skill_tool_no_warning(self) -> None: - """SCP6: run_cmd step (not run_skill) → rule does not fire.""" - wf = _make_workflow( - { - "step": { - "tool": "run_cmd", - "with": {"cmd": "ls -la", "cwd": "/tmp"}, - "on_success": "done", - }, - "done": {"action": "stop", "message": "Done."}, - } - ) - findings = run_semantic_rules(wf) - assert not any(f.rule == "skill-command-missing-prefix" for f in findings) - - -# --------------------------------------------------------------------------- -# TestShadowedRequiredInput -# --------------------------------------------------------------------------- - - -class TestShadowedRequiredInput: - """Tests for the shadowed-required-input semantic rule.""" - - def test_fires_when_required_input_in_context_but_passed_as_prose(self) -> None: - """Rule fires when plan_path is an ingredient but skill_command passes prose text.""" - recipe = Recipe( - name="test", - description="test", - ingredients={ - "plan_path": RecipeIngredient(description="Plan file path", required=True), - }, - steps={ - "implement": _parse_step( - { - "tool": "run_skill", - "with": { - "skill_command": ( - "/autoskillit:implement-worktree-no-merge temp/my-plan.md" - ), - "cwd": "/tmp", - }, - "on_success": "done", - } - ), - "done": _parse_step({"action": "stop", "message": "Done."}), - }, - kitchen_rules=["test"], - ) - findings = run_semantic_rules(recipe) - assert any( - f.rule == "shadowed-required-input" and f.step_name == "implement" for f in findings - ), "Expected shadowed-required-input finding when plan_path is ingredient but prose passed" - - def test_clean_when_template_ref_used(self) -> None: - """Rule is silent when skill_command uses ${{ context.plan_path }} template reference.""" - recipe = Recipe( - name="test", - description="test", - ingredients={ - "plan_path": RecipeIngredient(description="Plan file path", required=True), - }, - steps={ - "implement": _parse_step( - { - "tool": "run_skill", - "with": { - "skill_command": ( - "/autoskillit:implement-worktree-no-merge ${{ context.plan_path }}" - ), - "cwd": "/tmp", - }, - "on_success": "done", - } - ), - "done": _parse_step({"action": "stop", "message": "Done."}), - }, - kitchen_rules=["test"], - ) - findings = run_semantic_rules(recipe) - assert not any(f.rule == "shadowed-required-input" for f in findings), ( - "Expected no shadowed-required-input finding when template ref is used" - ) - - def test_clean_when_input_not_yet_in_context(self) -> None: - """Rule is silent when plan_path is not an ingredient and not in available context.""" - recipe = Recipe( - name="test", - description="test", - ingredients={}, # plan_path not declared — not yet available - steps={ - "implement": _parse_step( - { - "tool": "run_skill", - "with": { - "skill_command": ( - "/autoskillit:implement-worktree-no-merge temp/my-plan.md" - ), - "cwd": "/tmp", - }, - "on_success": "done", - } - ), - "done": _parse_step({"action": "stop", "message": "Done."}), - }, - kitchen_rules=["test"], - ) - findings = run_semantic_rules(recipe) - assert not any(f.rule == "shadowed-required-input" for f in findings), ( - "Expected no shadowed-required-input finding when input is not available in context" - ) - - -# --------------------------------------------------------------------------- -# TestMergeBaseUnpublishedRule -# --------------------------------------------------------------------------- - - -class TestMergeBaseUnpublishedRule: - """Tests for the merge-base-unpublished semantic rule.""" - - def test_merge_base_unpublished_rule_fires_when_push_absent(self) -> None: - """merge-base-unpublished ERROR fires when merge_worktree.base_branch - is a context variable without a preceding push_to_remote step.""" - recipe = _make_workflow( - { - "clone": { - "tool": "clone_repo", - "with": {"source_dir": ".", "run_name": "test"}, - "on_success": "create_branch", - }, - "create_branch": { - "tool": "run_cmd", - "with": {"cmd": "echo test", "cwd": "/tmp"}, - "capture": {"merge_target": "${{ result.stdout }}"}, - "on_success": "merge_step", - }, - "merge_step": { - "tool": "merge_worktree", - "with": { - "worktree_path": "${{ context.worktree_path }}", - "base_branch": "${{ context.merge_target }}", - }, - "on_success": "done", - }, - "done": {"action": "stop", "message": "done"}, - } - ) - findings = run_semantic_rules(recipe) - rule_findings = [f for f in findings if f.rule == "merge-base-unpublished"] - assert len(rule_findings) >= 1 - assert any(f.severity == Severity.ERROR for f in rule_findings) - - def test_merge_base_unpublished_rule_passes_when_push_precedes_merge(self) -> None: - """merge-base-unpublished does NOT fire when push_to_remote appears - on the path to merge_worktree for the same context variable.""" - recipe = _make_workflow( - { - "clone": { - "tool": "clone_repo", - "with": {"source_dir": ".", "run_name": "test"}, - "on_success": "create_branch", - }, - "create_branch": { - "tool": "run_cmd", - "with": {"cmd": "echo test", "cwd": "/tmp"}, - "capture": {"merge_target": "${{ result.stdout }}"}, - "on_success": "push_target", - }, - "push_target": { - "tool": "push_to_remote", - "with": { - "clone_path": "/tmp", - "branch": "${{ context.merge_target }}", - "remote_url": "https://example.com/repo.git", - }, - "on_success": "merge_step", - }, - "merge_step": { - "tool": "merge_worktree", - "with": { - "worktree_path": "${{ context.worktree_path }}", - "base_branch": "${{ context.merge_target }}", - }, - "on_success": "done", - }, - "done": {"action": "stop", "message": "done"}, - } - ) - findings = run_semantic_rules(recipe) - assert not any(f.rule == "merge-base-unpublished" for f in findings) - - def test_merge_base_unpublished_rule_does_not_fire_for_literal_branch(self) -> None: - """merge-base-unpublished does NOT fire when base_branch is a - literal string — literals like 'main' are always published.""" - recipe = _make_workflow( - { - "start": { - "tool": "run_cmd", - "with": {"cmd": "echo ok", "cwd": "/tmp"}, - "on_success": "merge_step", - }, - "merge_step": { - "tool": "merge_worktree", - "with": { - "worktree_path": "${{ context.worktree_path }}", - "base_branch": "main", - }, - "on_success": "done", - }, - "done": {"action": "stop", "message": "done"}, - } - ) - findings = run_semantic_rules(recipe) - assert not any(f.rule == "merge-base-unpublished" for f in findings) - - def test_implementation_pipeline_satisfies_push_before_merge_contract(self) -> None: - """implementation.yaml must pass the merge-base-unpublished - rule after the push_merge_target step is added.""" - recipe = load_recipe(builtin_recipes_dir() / "implementation.yaml") - findings = run_semantic_rules(recipe) - assert not any(f.rule == "merge-base-unpublished" for f in findings) - - -# --------------------------------------------------------------------------- -# TestPredicateOnResultValidation -# --------------------------------------------------------------------------- - - -class TestPredicateOnResultValidation: - """Structural validation for predicate-format on_result (conditions list).""" - - def _make_merge_recipe(self, merge_step: dict, extra_steps: dict | None = None) -> Recipe: - steps: dict = { - "merge": merge_step, - "assess": {"action": "stop", "message": "Assess."}, - "cleanup_failure": {"action": "stop", "message": "Cleanup."}, - "push": {"action": "stop", "message": "Push."}, - } - if extra_steps: - steps.update(extra_steps) - return _make_workflow(steps) - - def test_predicate_on_result_on_success_mutually_exclusive(self) -> None: - """Step with predicate on_result (list) + on_success → validation error.""" - from autoskillit.recipe.validator import validate_recipe - - wf = self._make_merge_recipe( - { - "tool": "merge_worktree", - "with": {"worktree_path": "/tmp/wt", "base_branch": "main"}, - "on_result": [ - {"when": "result.error", "route": "cleanup_failure"}, - {"route": "push"}, - ], - "on_success": "push", # mutually exclusive - } - ) - errors = validate_recipe(wf) - assert any("on_result" in e and "on_success" in e for e in errors) - - def test_predicate_condition_invalid_route_target_rejected(self) -> None: - """A condition referencing an unknown step name is a validation error.""" - from autoskillit.recipe.validator import validate_recipe - - wf = self._make_merge_recipe( - { - "tool": "merge_worktree", - "with": {"worktree_path": "/tmp/wt", "base_branch": "main"}, - "on_result": [ - {"when": "result.error", "route": "nonexistent_step"}, - {"route": "push"}, - ], - } - ) - errors = validate_recipe(wf) - assert any("nonexistent_step" in e for e in errors) - - def test_predicate_condition_route_valid_step_accepted(self) -> None: - """All condition routes pointing to valid step names pass validation.""" - from autoskillit.recipe.validator import validate_recipe - - wf = self._make_merge_recipe( - { - "tool": "merge_worktree", - "with": {"worktree_path": "/tmp/wt", "base_branch": "main"}, - "on_result": [ - {"when": "result.error", "route": "cleanup_failure"}, - {"route": "push"}, - ], - "capture": {"cleanup_succeeded": "${{ result.cleanup_succeeded }}"}, - } - ) - errors = validate_recipe(wf) - assert errors == [] - - def test_predicate_on_result_empty_conditions_rejected(self) -> None: - """on_result with conditions=[] bypasses predicate path; emits field error. - - When StepResultRoute(conditions=[]) is constructed directly (bypassing _parse_step, - which collapses empty conditions to on_result=None), the validator falls through to - legacy format validation and emits an explicit error for the missing field. - """ - from autoskillit.recipe.validator import validate_recipe - - recipe = Recipe( - name="test-predicate-empty", - description="test", - steps={ - "start": RecipeStep( - tool="run_skill", - with_args={"skill_command": "x", "cwd": "y"}, - on_result=StepResultRoute(conditions=[]), - ), - "done": RecipeStep(action="stop", message="done"), - }, - ) - errors = validate_recipe(recipe) - assert any("on_result.field must be non-empty" in e for e in errors) - - def test_predicate_format_with_on_failure_allowed(self) -> None: - """validator.py must not reject on_failure alongside on_result.conditions.""" - from autoskillit.recipe.validator import validate_recipe - - wf = self._make_merge_recipe( - { - "tool": "merge_worktree", - "with": {"worktree_path": "/tmp/wt", "base_branch": "main"}, - "on_result": [ - {"when": "result.error", "route": "cleanup_failure"}, - {"route": "push"}, - ], - "on_failure": "cleanup_failure", - } - ) - errors = validate_recipe(wf) - assert not any("mutually exclusive" in e for e in errors), errors - - def test_on_result_missing_failure_route_fires_for_predicate_format(self) -> None: - """Predicate-format steps with no on_failure must trigger ERROR finding.""" - wf = self._make_merge_recipe( - { - "tool": "merge_worktree", - "with": {"worktree_path": "/tmp/wt", "base_branch": "main"}, - "on_result": [ - {"when": "result.error", "route": "cleanup_failure"}, - {"route": "push"}, - ], - # no on_failure — should trigger finding - } - ) - findings = run_semantic_rules(wf) - names = [f.rule for f in findings] - assert "on-result-missing-failure-route" in names - - def test_on_result_missing_failure_route_clear_when_predicate_has_on_failure(self) -> None: - """Predicate-format step with on_failure must not trigger the rule.""" - wf = self._make_merge_recipe( - { - "tool": "merge_worktree", - "with": {"worktree_path": "/tmp/wt", "base_branch": "main"}, - "on_result": [ - {"when": "result.error", "route": "cleanup_failure"}, - {"route": "push"}, - ], - "on_failure": "cleanup_failure", - } - ) - findings = run_semantic_rules(wf) - names = [f.rule for f in findings] - assert "on-result-missing-failure-route" not in names - - -# --------------------------------------------------------------------------- -# TestPredicateBuildStepGraph -# --------------------------------------------------------------------------- - - -class TestPredicateBuildStepGraph: - """_build_step_graph includes condition.route edges.""" - - def test_build_step_graph_includes_condition_routes(self) -> None: - """_build_step_graph produces edges for condition.route targets.""" - from autoskillit.recipe.validator import _build_step_graph - - wf = _make_workflow( - { - "merge": { - "tool": "merge_worktree", - "with": {"worktree_path": "/tmp/wt", "base_branch": "main"}, - "on_result": [ - {"when": "result.failed_step == 'test_gate'", "route": "assess"}, - {"when": "result.error", "route": "cleanup"}, - {"route": "push"}, - ], - "capture": {"cleanup_succeeded": "${{ result.cleanup_succeeded }}"}, - }, - "assess": {"action": "stop", "message": "Assess."}, - "cleanup": {"action": "stop", "message": "Cleanup."}, - "push": {"action": "stop", "message": "Push."}, - } - ) - graph = _build_step_graph(wf) - assert "assess" in graph["merge"] - assert "cleanup" in graph["merge"] - assert "push" in graph["merge"] - - -# --------------------------------------------------------------------------- -# TestPredicateSemanticRules -# --------------------------------------------------------------------------- - - -class TestPredicateSemanticRules: - """Semantic rules behave correctly for predicate-format on_result.""" - - def test_unreachable_step_includes_condition_routes(self) -> None: - """A step reachable only via condition route is NOT flagged as unreachable.""" - wf = _make_workflow( - { - "merge": { - "tool": "merge_worktree", - "with": {"worktree_path": "/tmp/wt", "base_branch": "main"}, - "on_result": [ - {"when": "result.failed_step == 'test_gate'", "route": "assess"}, - {"when": "result.error", "route": "cleanup"}, - {"route": "push"}, - ], - "capture": {"cleanup_succeeded": "${{ result.cleanup_succeeded }}"}, - }, - "assess": {"action": "stop", "message": "Assess."}, - "cleanup": {"action": "stop", "message": "Cleanup."}, - "push": {"action": "stop", "message": "Push."}, - } - ) - findings = run_semantic_rules(wf) - unreachable = [f for f in findings if f.rule == "unreachable-step"] - step_names = {f.step_name for f in unreachable} - assert "assess" not in step_names - assert "cleanup" not in step_names - assert "push" not in step_names - - def test_on_result_missing_failure_route_still_fires_for_legacy_format( - self, - ) -> None: - """RCA1 rule continues to fire for legacy format with no on_failure (no regression).""" - wf = _make_workflow( - { - "audit": { - "tool": "run_skill", - "with": {"skill_command": "/autoskillit:audit-impl plan.md impl main"}, - "capture": {"verdict": "${{ result.verdict }}"}, - "on_result": {"field": "verdict", "routes": {"GO": "done", "NO GO": "fix"}}, - # no on_failure — the gap - }, - "fix": {"action": "stop", "message": "Fix."}, - "done": {"action": "stop", "message": "Done."}, - } - ) - findings = run_semantic_rules(wf) - assert any(f.rule == "on-result-missing-failure-route" for f in findings) - - -# --------------------------------------------------------------------------- -# TestMergeRoutingIncompleteRule -# --------------------------------------------------------------------------- - - -class TestMergeRoutingIncompleteRule: - """Tests for the merge-routing-incomplete semantic rule (RMR*).""" - - def _make_merge_step(self, conditions: list[dict]) -> Recipe: - """Build a minimal recipe with a merge_worktree step using predicate on_result.""" - return _make_workflow( - { - "merge": { - "tool": "merge_worktree", - "with": {"worktree_path": "/tmp/wt", "base_branch": "main"}, - "on_result": conditions, - "capture": {"cleanup_succeeded": "${{ result.cleanup_succeeded }}"}, - }, - "recover": {"action": "stop", "message": "Recover."}, - "done": {"action": "stop", "message": "Done."}, - "escalate": {"action": "stop", "message": "Escalate."}, - } - ) - - def test_rmr1_fires_when_test_gate_missing(self): - """RMR1: ERROR when test_gate is not explicitly routed.""" - recipe = self._make_merge_step( - [ - {"when": "result.failed_step == 'dirty_tree'", "route": "recover"}, - {"when": "result.failed_step == 'post_rebase_test_gate'", "route": "recover"}, - {"when": "result.failed_step == 'rebase'", "route": "recover"}, - {"when": "result.error", "route": "escalate"}, - {"route": "done"}, - ] - ) - findings = run_semantic_rules(recipe) - errors = [f for f in findings if f.rule == "merge-routing-incomplete"] - assert len(errors) == 1 - assert "test_gate" in errors[0].message - - def test_rmr2_fires_when_post_rebase_test_gate_missing(self): - """RMR2: ERROR when post_rebase_test_gate is not explicitly routed.""" - recipe = self._make_merge_step( - [ - {"when": "result.failed_step == 'dirty_tree'", "route": "recover"}, - {"when": "result.failed_step == 'test_gate'", "route": "recover"}, - {"when": "result.failed_step == 'rebase'", "route": "recover"}, - {"when": "result.error", "route": "escalate"}, - {"route": "done"}, - ] - ) - findings = run_semantic_rules(recipe) - errors = [f for f in findings if f.rule == "merge-routing-incomplete"] - assert len(errors) == 1 - assert "post_rebase_test_gate" in errors[0].message - - def test_rmr3_fires_when_rebase_missing(self): - """RMR3: ERROR when rebase is not explicitly routed.""" - recipe = self._make_merge_step( - [ - {"when": "result.failed_step == 'dirty_tree'", "route": "recover"}, - {"when": "result.failed_step == 'test_gate'", "route": "recover"}, - {"when": "result.failed_step == 'post_rebase_test_gate'", "route": "recover"}, - {"when": "result.error", "route": "escalate"}, - {"route": "done"}, - ] - ) - findings = run_semantic_rules(recipe) - errors = [f for f in findings if f.rule == "merge-routing-incomplete"] - assert len(errors) == 1 - assert "rebase" in errors[0].message - - def test_rmr4_clears_when_all_four_covered(self): - """RMR4: No finding when all recoverable values are explicitly routed.""" - recipe = self._make_merge_step( - [ - {"when": "result.failed_step == 'dirty_tree'", "route": "recover"}, - {"when": "result.failed_step == 'test_gate'", "route": "recover"}, - {"when": "result.failed_step == 'post_rebase_test_gate'", "route": "recover"}, - {"when": "result.failed_step == 'rebase'", "route": "recover"}, - {"when": "result.error", "route": "escalate"}, - {"route": "done"}, - ] - ) - findings = run_semantic_rules(recipe) - errors = [f for f in findings if f.rule == "merge-routing-incomplete"] - assert errors == [] - - def test_rmr7_fires_when_dirty_tree_missing(self): - """RMR7: ERROR when dirty_tree is not explicitly routed.""" - recipe = self._make_merge_step( - [ - {"when": "result.failed_step == 'test_gate'", "route": "recover"}, - {"when": "result.failed_step == 'post_rebase_test_gate'", "route": "recover"}, - {"when": "result.failed_step == 'rebase'", "route": "recover"}, - {"when": "result.error", "route": "escalate"}, - {"route": "done"}, - ] - ) - findings = run_semantic_rules(recipe) - errors = [f for f in findings if f.rule == "merge-routing-incomplete"] - assert len(errors) == 1 - assert "dirty_tree" in errors[0].message - - def test_rmr5_does_not_fire_for_non_merge_worktree_step(self): - """RMR5: Rule is scoped to merge_worktree steps only.""" - recipe = _make_workflow( - { - "run": { - "tool": "run_skill", - "with": {"skill_command": "/autoskillit:implement-worktree", "cwd": "/tmp"}, - "on_result": [ - {"when": "result.error", "route": "done"}, - {"route": "done"}, - ], - }, - "done": {"action": "stop", "message": "Done."}, - } - ) - findings = run_semantic_rules(recipe) - errors = [f for f in findings if f.rule == "merge-routing-incomplete"] - assert errors == [] - - def test_rmr6_does_not_fire_when_no_on_result(self): - """RMR6: Rule is silent for merge_worktree steps without on_result.""" - recipe = _make_workflow( - { - "merge": { - "tool": "merge_worktree", - "with": {"worktree_path": "/tmp/wt", "base_branch": "main"}, - "on_success": "done", - "on_failure": "escalate", - }, - "done": {"action": "stop", "message": "Done."}, - "escalate": {"action": "stop", "message": "Escalate."}, - } - ) - findings = run_semantic_rules(recipe) - errors = [f for f in findings if f.rule == "merge-routing-incomplete"] - assert errors == [] - - -# --------------------------------------------------------------------------- -# TestRecipeIntegrationPredicateRouting -# --------------------------------------------------------------------------- - - -class TestRecipeIntegrationPredicateRouting: - """Integration tests: bundled recipes with predicate on_result validate correctly.""" - - def setup_method(self) -> None: - self.if_recipe = load_recipe(builtin_recipes_dir() / "remediation.yaml") - self.ip_recipe = load_recipe(builtin_recipes_dir() / "implementation.yaml") - - def test_investigate_first_merge_step_has_predicate_on_result(self) -> None: - """The merge step in remediation.yaml has predicate on_result.""" - step = self.if_recipe.steps["merge"] - assert step.on_result is not None - assert len(step.on_result.conditions) == 6 - - cond0 = step.on_result.conditions[0] - assert cond0.when == "result.failed_step == 'dirty_tree'" - assert cond0.route == "assess" - - cond1 = step.on_result.conditions[1] - assert cond1.when == "result.failed_step == 'test_gate'" - assert cond1.route == "assess" - - cond2 = step.on_result.conditions[2] - assert cond2.when == "result.failed_step == 'post_rebase_test_gate'" - assert cond2.route == "assess" - - cond3 = step.on_result.conditions[3] - assert cond3.when == "result.failed_step == 'rebase'" - assert cond3.route == "assess" - - cond4 = step.on_result.conditions[4] - assert cond4.when == "result.error" - assert cond4.route == "release_issue_failure" - - cond5 = step.on_result.conditions[5] - assert cond5.when is None - assert cond5.route == "push" - - def test_investigate_first_merge_step_captures_worktree_path(self) -> None: - """The merge step captures worktree_path from result.worktree_path.""" - step = self.if_recipe.steps["merge"] - assert "worktree_path" in step.capture - assert "result.worktree_path" in step.capture["worktree_path"] - - def test_implementation_pipeline_merge_step_has_predicate_on_result(self) -> None: - """The merge step in implementation.yaml has predicate on_result.""" - step = self.ip_recipe.steps["merge"] - assert step.on_result is not None - assert len(step.on_result.conditions) == 6 - - cond0 = step.on_result.conditions[0] - assert cond0.when == "result.failed_step == 'dirty_tree'" - assert cond0.route == "fix" - - cond1 = step.on_result.conditions[1] - assert cond1.when == "result.failed_step == 'test_gate'" - assert cond1.route == "fix" - - cond2 = step.on_result.conditions[2] - assert cond2.when == "result.failed_step == 'post_rebase_test_gate'" - assert cond2.route == "fix" - - cond3 = step.on_result.conditions[3] - assert cond3.when == "result.failed_step == 'rebase'" - assert cond3.route == "fix" - - cond4 = step.on_result.conditions[4] - assert cond4.when == "result.error" - assert cond4.route == "release_issue_failure" - - cond5 = step.on_result.conditions[5] - assert cond5.when is None - assert cond5.route == "next_or_done" - - def test_implementation_pipeline_merge_step_captures_worktree_path(self) -> None: - """The merge step in implementation.yaml captures worktree_path.""" - step = self.ip_recipe.steps["merge"] - assert "worktree_path" in step.capture - assert "result.worktree_path" in step.capture["worktree_path"] - - def test_both_recipes_validate_cleanly(self) -> None: - """Both recipes have no structural errors after predicate routing changes.""" - from autoskillit.recipe.validator import validate_recipe - - if_errors = validate_recipe(self.if_recipe) - assert if_errors == [], f"remediation.yaml has validation errors: {if_errors}" - - ip_errors = validate_recipe(self.ip_recipe) - assert ip_errors == [], f"implementation.yaml has validation errors: {ip_errors}" - - def test_both_recipes_no_error_semantic_findings(self) -> None: - """Both recipes pass semantic rules with no ERROR-severity findings.""" - for recipe, name in [ - (self.if_recipe, "remediation"), - (self.ip_recipe, "implementation"), - ]: - findings = run_semantic_rules(recipe) - errors = [ - f - for f in findings - if f.severity == Severity.ERROR and f.rule != _NO_AUTOSKILLIT_IMPORT - ] - assert errors == [], f"{name} has ERROR-severity semantic findings: " + str( - [(f.rule, f.step_name, f.message) for f in errors] - ) diff --git a/tests/recipe/test_rules_subset_disabled.py b/tests/recipe/test_rules_subset_disabled.py index 8f867bf0e6..8d1941cc7a 100644 --- a/tests/recipe/test_rules_subset_disabled.py +++ b/tests/recipe/test_rules_subset_disabled.py @@ -2,6 +2,10 @@ from __future__ import annotations +import pytest + +pytestmark = [pytest.mark.layer("recipe"), pytest.mark.medium] + def _make_skill_recipe(skill_command: str): from autoskillit.recipe.schema import Recipe, RecipeStep diff --git a/tests/recipe/test_rules_temp_path.py b/tests/recipe/test_rules_temp_path.py new file mode 100644 index 0000000000..e1093b4db1 --- /dev/null +++ b/tests/recipe/test_rules_temp_path.py @@ -0,0 +1,113 @@ +"""Tests for the non-unique-output-path lint rule.""" + +from __future__ import annotations + +import pytest + +from autoskillit.core import Severity +from autoskillit.recipe.validator import run_semantic_rules +from tests.recipe.conftest import _make_workflow + +pytestmark = [pytest.mark.layer("recipe"), pytest.mark.small] + + +class TestNonUniqueOutputPath: + def test_bare_temp_path_in_output_dir_errors(self) -> None: + wf = _make_workflow( + { + "init_step": { + "tool": "run_python", + "with": { + "callable": "some.callable", + "output_dir": "{{AUTOSKILLIT_TEMP}}/planner", + }, + "on_success": "done", + }, + "done": {"action": "stop", "message": "Done."}, + } + ) + findings = run_semantic_rules(wf) + errors = [f for f in findings if f.rule == "non-unique-output-path"] + assert any(f.severity == Severity.ERROR and f.step_name == "init_step" for f in errors) + + def test_bare_temp_path_in_cmd_errors(self) -> None: + wf = _make_workflow( + { + "mk_step": { + "tool": "run_cmd", + "with": { + "command": "mkdir -p {{AUTOSKILLIT_TEMP}}/myrecipe/data", + }, + "on_success": "done", + }, + "done": {"action": "stop", "message": "Done."}, + } + ) + findings = run_semantic_rules(wf) + errors = [f for f in findings if f.rule == "non-unique-output-path"] + assert any(f.severity == Severity.ERROR and f.step_name == "mk_step" for f in errors) + + def test_bare_temp_path_in_env_var_errors(self) -> None: + wf = _make_workflow( + { + "env_step": { + "tool": "run_cmd", + "with": { + "command": "/some-skill", + "env": {"MY_FILE": "{{AUTOSKILLIT_TEMP}}/recipe/data.json"}, + }, + "on_success": "done", + }, + "done": {"action": "stop", "message": "Done."}, + } + ) + findings = run_semantic_rules(wf) + errors = [f for f in findings if f.rule == "non-unique-output-path"] + assert any(f.severity == Severity.ERROR and f.step_name == "env_step" for f in errors) + + def test_context_scoped_path_passes(self) -> None: + wf = _make_workflow( + { + "scoped_step": { + "tool": "run_python", + "with": { + "callable": "some.callable", + "output_dir": "${{ context.planner_dir }}", + }, + "on_success": "done", + }, + "done": {"action": "stop", "message": "Done."}, + } + ) + findings = run_semantic_rules(wf) + assert not any(f.rule == "non-unique-output-path" for f in findings) + + def test_bare_temp_path_in_list_arg_errors(self) -> None: + wf = _make_workflow( + { + "list_step": { + "tool": "run_python", + "with": { + "callable": "some.callable", + "paths": ["{{AUTOSKILLIT_TEMP}}/planner/out.json"], + }, + "on_success": "done", + }, + "done": {"action": "stop", "message": "Done."}, + } + ) + findings = run_semantic_rules(wf) + errors = [f for f in findings if f.rule == "non-unique-output-path"] + assert any(f.severity == Severity.ERROR and f.step_name == "list_step" for f in errors) + + def test_message_field_not_flagged(self) -> None: + wf = _make_workflow( + { + "done": { + "action": "stop", + "message": "Output at {{AUTOSKILLIT_TEMP}}/planner/", + }, + } + ) + findings = run_semantic_rules(wf) + assert not any(f.rule == "non-unique-output-path" for f in findings) diff --git a/tests/recipe/test_rules_tools.py b/tests/recipe/test_rules_tools.py index 2d90741dae..ca978a6989 100644 --- a/tests/recipe/test_rules_tools.py +++ b/tests/recipe/test_rules_tools.py @@ -5,11 +5,12 @@ import pytest -from autoskillit.core import Severity -from autoskillit.pipeline.gate import GATED_TOOLS, UNGATED_TOOLS +from autoskillit.core import GATED_TOOLS, UNGATED_TOOLS, Severity from autoskillit.recipe.registry import run_semantic_rules from autoskillit.recipe.schema import Recipe, RecipeStep +pytestmark = [pytest.mark.layer("recipe"), pytest.mark.small] + def _make_recipe(tool: str | None = None, action: str | None = None) -> Recipe: """Minimal recipe factory for unknown-tool rule tests.""" @@ -264,7 +265,7 @@ def test_wait_for_ci_rejects_poll_interval() -> None: def test_rules_tools_batch_cleanup_clones_accepts_all_owners_param() -> None: """T20 — batch_cleanup_clones with all_owners param must not trigger dead-with-param.""" - from autoskillit.recipe.rules_tools import _TOOL_PARAMS + from autoskillit.recipe.rules.rules_tools import _TOOL_PARAMS assert "all_owners" in _TOOL_PARAMS["batch_cleanup_clones"] @@ -334,16 +335,18 @@ def test_rebase_then_push_with_force_true_passes_validation() -> None: # --------------------------------------------------------------------------- _SERVER_TOOL_MODULES = [ - "autoskillit.server.tools_ci", - "autoskillit.server.tools_clone", - "autoskillit.server.tools_execution", - "autoskillit.server.tools_git", - "autoskillit.server.tools_recipe", - "autoskillit.server.tools_status", - "autoskillit.server.tools_github", - "autoskillit.server.tools_issue_lifecycle", - "autoskillit.server.tools_pr_ops", - "autoskillit.server.tools_workspace", + "autoskillit.server.tools.tools_ci", + "autoskillit.server.tools.tools_ci_watch", + "autoskillit.server.tools.tools_ci_merge_queue", + "autoskillit.server.tools.tools_clone", + "autoskillit.server.tools.tools_execution", + "autoskillit.server.tools.tools_git", + "autoskillit.server.tools.tools_recipe", + "autoskillit.server.tools.tools_status", + "autoskillit.server.tools.tools_github", + "autoskillit.server.tools.tools_issue_lifecycle", + "autoskillit.server.tools.tools_pr_ops", + "autoskillit.server.tools.tools_workspace", ] _FRAMEWORK_PARAMS = frozenset({"ctx"}) @@ -361,7 +364,7 @@ def _build_handler_map() -> dict[str, object]: def test_tool_params_matches_mcp_handler_signatures() -> None: """T8: _TOOL_PARAMS keys match actual MCP handler signatures — drift fails CI.""" - from autoskillit.recipe.rules_tools import _TOOL_PARAMS + from autoskillit.recipe.rules.rules_tools import _TOOL_PARAMS handler_map = _build_handler_map() mismatches: list[str] = [] @@ -385,3 +388,85 @@ def test_tool_params_matches_mcp_handler_signatures() -> None: assert not mismatches, ( "_TOOL_PARAMS is out of sync with MCP handler signatures:\n" + "\n".join(mismatches) ) + + +class TestConstantStepWithArgs: + def test_constant_with_args_fires_error(self): + recipe = Recipe( + name="test-recipe", + description="Test constant rule", + steps={ + "s": RecipeStep(constant="val", with_args={"x": "1"}, on_success="done"), + "done": RecipeStep(action="stop", message="Done checking constant."), + }, + ) + findings = [f for f in run_semantic_rules(recipe) if f.rule == "constant-step-with-args"] + assert len(findings) == 1 + assert findings[0].severity == Severity.ERROR + + def test_constant_only_is_clean(self): + recipe = Recipe( + name="test-recipe", + description="Test constant rule", + steps={ + "s": RecipeStep(constant="val", on_success="done"), + "done": RecipeStep(action="stop", message="Done checking constant."), + }, + ) + findings = [f for f in run_semantic_rules(recipe) if f.rule == "constant-step-with-args"] + assert len(findings) == 0 + + def test_with_args_only_is_clean(self): + recipe = Recipe( + name="test-recipe", + description="Test constant rule", + steps={ + "s": RecipeStep(tool="run_cmd", with_args={"cmd": "echo hi"}, on_success="done"), + "done": RecipeStep(action="stop", message="Done checking constant."), + }, + ) + findings = [f for f in run_semantic_rules(recipe) if f.rule == "constant-step-with-args"] + assert len(findings) == 0 + + +# --------------------------------------------------------------------------- +# release-issue-requires-disposition rule tests +# --------------------------------------------------------------------------- + + +def test_release_issue_requires_disposition_fires_on_bare_call() -> None: + """Rule fires when release_issue step has neither fail_label nor target_branch.""" + recipe = _make_recipe_with_args( + "release_issue", + {"issue_url": "https://github.com/o/r/issues/1"}, + ) + findings = run_semantic_rules(recipe) + hits = [f for f in findings if f.rule == "release-issue-requires-disposition"] + assert hits + assert hits[0].severity == Severity.ERROR + assert "run" in hits[0].step_name + + +def test_release_issue_requires_disposition_passes_with_fail_label() -> None: + """Rule does NOT fire when fail_label is present.""" + recipe = _make_recipe_with_args( + "release_issue", + {"issue_url": "https://github.com/o/r/issues/1", "fail_label": "fail"}, + ) + findings = run_semantic_rules(recipe) + hits = [f for f in findings if f.rule == "release-issue-requires-disposition"] + assert not hits + + +def test_release_issue_requires_disposition_passes_with_target_branch() -> None: + """Rule does NOT fire when target_branch is present.""" + recipe = _make_recipe_with_args( + "release_issue", + { + "issue_url": "https://github.com/o/r/issues/1", + "target_branch": "${{ inputs.base_branch }}", + }, + ) + findings = run_semantic_rules(recipe) + hits = [f for f in findings if f.rule == "release-issue-requires-disposition"] + assert not hits diff --git a/tests/recipe/test_rules_unreachable_model.py b/tests/recipe/test_rules_unreachable_model.py new file mode 100644 index 0000000000..d29936a3f3 --- /dev/null +++ b/tests/recipe/test_rules_unreachable_model.py @@ -0,0 +1,53 @@ +from __future__ import annotations + +import pytest + +from autoskillit.recipe.validator import run_semantic_rules +from tests.recipe.conftest import _make_workflow + +pytestmark = [pytest.mark.layer("recipe"), pytest.mark.small] + + +def test_unreachable_steps_detects_orphan() -> None: + wf = _make_workflow( + { + "start": {"tool": "run_cmd", "on_success": "done"}, + "orphan": {"tool": "run_cmd", "on_success": "done"}, + "done": {"action": "stop", "message": "Done."}, + } + ) + findings = run_semantic_rules(wf) + assert any(f.rule == "unreachable-step" and "orphan" in f.message for f in findings) + + +def test_unreachable_steps_first_step_clean() -> None: + wf = _make_workflow( + { + "start": {"tool": "run_cmd", "on_success": "done"}, + "done": {"action": "stop", "message": "Done."}, + } + ) + findings = run_semantic_rules(wf) + assert not any(f.rule == "unreachable-step" and "start" in f.step_name for f in findings) + + +def test_model_on_non_skill_triggers() -> None: + wf = _make_workflow( + { + "check": {"tool": "test_check", "model": "sonnet", "on_success": "done"}, + "done": {"action": "stop", "message": "Done."}, + } + ) + findings = run_semantic_rules(wf) + assert any(f.rule == "model-on-non-skill-step" for f in findings) + + +def test_model_on_non_skill_clean() -> None: + wf = _make_workflow( + { + "do": {"tool": "run_skill", "model": "sonnet", "on_success": "done"}, + "done": {"action": "stop", "message": "Done."}, + } + ) + findings = run_semantic_rules(wf) + assert not any(f.rule == "model-on-non-skill-step" for f in findings) diff --git a/tests/recipe/test_rules_unsatisfied_input.py b/tests/recipe/test_rules_unsatisfied_input.py new file mode 100644 index 0000000000..4e30be6102 --- /dev/null +++ b/tests/recipe/test_rules_unsatisfied_input.py @@ -0,0 +1,150 @@ +from __future__ import annotations + +import pytest + +from autoskillit.core.types import Severity +from autoskillit.recipe.io import _parse_step +from autoskillit.recipe.schema import Recipe, RecipeIngredient +from autoskillit.recipe.validator import run_semantic_rules +from tests.recipe.conftest import _make_workflow + +pytestmark = [pytest.mark.layer("recipe"), pytest.mark.small] + + +def test_unsatisfied_input_replaces_worktree_path_check() -> None: + wf = _make_workflow( + { + "implement": { + "tool": "run_skill", + "with": { + "skill_command": ( + "/autoskillit:implement-worktree-no-merge ${{ context.plan_path }}" + ), + }, + "capture": {"worktree_path": "${{ result.worktree_path }}"}, + "retries": 0, + "on_context_limit": "retry_step", + "on_success": "retry_step", + }, + "retry_step": { + "tool": "run_skill", + "with": { + "skill_command": "/autoskillit:retry-worktree ${{ context.plan_path }}", + }, + "on_success": "done", + }, + "done": {"action": "stop", "message": "Done."}, + } + ) + findings = run_semantic_rules(wf) + errors = [f for f in findings if f.severity == Severity.ERROR] + assert any(f.rule == "missing-ingredient" and "worktree_path" in f.message for f in errors) + + +def test_unsatisfied_input_clean_when_provided() -> None: + wf = _make_workflow( + { + "implement": { + "tool": "run_skill", + "with": { + "skill_command": ( + "/autoskillit:implement-worktree-no-merge ${{ context.plan_path }}" + ), + }, + "capture": {"worktree_path": "${{ result.worktree_path }}"}, + "retries": 0, + "on_context_limit": "retry_step", + "on_success": "retry_step", + }, + "retry_step": { + "tool": "run_skill", + "with": { + "skill_command": ( + "/autoskillit:retry-worktree " + "${{ context.plan_path }} ${{ context.worktree_path }}" + ), + }, + "on_success": "done", + }, + "done": {"action": "stop", "message": "Done."}, + } + ) + findings = run_semantic_rules(wf) + assert not any(f.rule == "missing-ingredient" for f in findings) + + +def test_unsatisfied_input_not_available() -> None: + wf = _make_workflow( + { + "retry_step": { + "tool": "run_skill", + "with": {"skill_command": "/autoskillit:retry-worktree ${{ context.plan_path }}"}, + "on_success": "done", + }, + "done": {"action": "stop", "message": "Done."}, + } + ) + findings = run_semantic_rules(wf) + errors = [ + f for f in findings if f.rule == "missing-ingredient" and f.severity == Severity.ERROR + ] + assert any("worktree_path" in f.message for f in errors) + + +def test_unsatisfied_input_unknown_skill_ignored() -> None: + wf = _make_workflow( + { + "step": { + "tool": "run_skill", + "with": {"skill_command": "/some-unknown-skill"}, + "on_success": "done", + }, + "done": {"action": "stop", "message": "Done."}, + } + ) + findings = run_semantic_rules(wf) + assert not any(f.rule == "missing-ingredient" for f in findings) + + +def test_unsatisfied_input_from_pipeline_inputs() -> None: + wf = Recipe( + name="test", + description="test", + ingredients={ + "plan_path": RecipeIngredient(description="Plan file", required=True), + "worktree_path": RecipeIngredient(description="Worktree", required=True), + }, + steps={ + "retry_step": _parse_step( + { + "tool": "run_skill", + "with": { + "skill_command": ( + "/autoskillit:retry-worktree " + "${{ inputs.plan_path }} ${{ inputs.worktree_path }}" + ), + }, + "on_success": "done", + } + ), + "done": _parse_step({"action": "stop", "message": "Done."}), + }, + kitchen_rules=["test"], + ) + findings = run_semantic_rules(wf) + assert not any(f.rule == "missing-ingredient" for f in findings) + + +def test_unsatisfied_input_inline_positional_args_skipped() -> None: + wf = _make_workflow( + { + "investigate": { + "tool": "run_skill", + "with": {"skill_command": "/autoskillit:investigate the test failures"}, + "on_success": "done", + }, + "done": {"action": "stop", "message": "Done."}, + } + ) + findings = run_semantic_rules(wf) + assert not any(f.rule == "missing-ingredient" for f in findings) diff --git a/tests/recipe/test_rules_verdict.py b/tests/recipe/test_rules_verdict.py index 1a5684dc73..bd246cc55d 100644 --- a/tests/recipe/test_rules_verdict.py +++ b/tests/recipe/test_rules_verdict.py @@ -1,12 +1,18 @@ -"""Tests for the unrouted-verdict-value semantic rule. +"""Tests for verdict semantic rules: unrouted-verdict-value and verdict-routing-asymmetry. Verifies that recipe steps using skills with declared allowed_values must explicitly handle every allowed value in their on_result block — no value may silently fall through a catch-all condition. + +Also verifies that steps invoking the same skill must route the same verdict +value to the same outcome category (continuation vs escalation). """ from __future__ import annotations +import pytest + +import autoskillit.recipe.rules.rules_verdict as _rv from autoskillit.core.types import Severity from autoskillit.recipe.io import builtin_recipes_dir, load_recipe from autoskillit.recipe.schema import ( @@ -17,6 +23,8 @@ ) from autoskillit.recipe.validator import run_semantic_rules +pytestmark = [pytest.mark.layer("recipe"), pytest.mark.small] + def _make_recipe(steps: dict[str, RecipeStep]) -> Recipe: return Recipe(name="test", description="test", steps=steps, kitchen_rules=["test"]) @@ -61,12 +69,20 @@ def _explicit_all_steps() -> dict[str, RecipeStep]: review_step = steps["review_pr"] existing = review_step.on_result.conditions review_step.on_result.conditions = [ - existing[0], + existing[0], # changes_requested → fix + StepResultCondition( + route="fix", + when="${{ result.verdict }} == approved_with_comments", + ), + StepResultCondition( + route="done", + when="${{ result.verdict }} == approved", + ), StepResultCondition( route="escalate", when="${{ result.verdict }} == needs_human", ), - existing[1], + existing[1], # true → done (catch-all) ] steps["escalate"] = RecipeStep( tool="run_skill", @@ -143,3 +159,358 @@ def test_unrouted_verdict_passes_for_review_design_in_research_recipe() -> None: if f.rule == "unrouted-verdict-value" and f.step_name == "review_design" ] assert not unrouted, f"unrouted-verdict-value fired for review_design step: {unrouted}" + + +# --------------------------------------------------------------------------- +# verdict-routing-asymmetry rule tests +# --------------------------------------------------------------------------- + + +def _asymmetric_resolve_steps() -> dict[str, RecipeStep]: + """Two steps invoking the same skill, routing flake_suspected differently.""" + return { + "fix": RecipeStep( + tool="run_skill", + with_args={"skill_command": "/autoskillit:resolve-failures wp pp bb"}, + capture={"verdict": "${{ result.verdict }}"}, + on_result=StepResultRoute( + conditions=[ + StepResultCondition( + route="test_step", + when="${{ result.verdict }} == 'flake_suspected'", + ), + StepResultCondition( + route="failure_step", + when="${{ result.verdict }} == 'ci_only_failure'", + ), + StepResultCondition( + route="push_step", + when="${{ result.verdict }} == 'real_fix'", + ), + StepResultCondition( + route="rebase_step", + when="${{ result.verdict }} == 'already_green'", + ), + ] + ), + on_failure="failure_step", + ), + "resolve_ci": RecipeStep( + tool="run_skill", + with_args={"skill_command": "/autoskillit:resolve-failures wp pp bb"}, + capture={"verdict": "${{ result.verdict }}"}, + on_result=StepResultRoute( + conditions=[ + StepResultCondition( + route="failure_step", + when="${{ result.verdict }} == 'flake_suspected'", + ), + StepResultCondition( + route="failure_step", + when="${{ result.verdict }} == 'ci_only_failure'", + ), + StepResultCondition( + route="push_step", + when="${{ result.verdict }} == 'real_fix'", + ), + StepResultCondition( + route="rebase_step", + when="${{ result.verdict }} == 'already_green'", + ), + ] + ), + on_failure="failure_step", + ), + "test_step": RecipeStep(tool="test_check"), + "push_step": RecipeStep(tool="push_to_remote"), + "rebase_step": RecipeStep(tool="run_cmd"), + "failure_step": RecipeStep(tool="run_cmd"), + } + + +def _consistent_resolve_steps() -> dict[str, RecipeStep]: + """Two steps invoking the same skill, routing flake_suspected consistently.""" + steps = _asymmetric_resolve_steps() + # Make resolve_ci route flake_suspected to test_step (continuation), matching fix + steps["resolve_ci"].on_result.conditions[0] = StepResultCondition( + route="test_step", + when="${{ result.verdict }} == 'flake_suspected'", + ) + return steps + + +def _different_skills_steps() -> dict[str, RecipeStep]: + """Two steps invoking different skills — asymmetry should be ignored.""" + steps = _asymmetric_resolve_steps() + # Change resolve_ci to invoke resolve-review instead + steps["resolve_ci"] = RecipeStep( + tool="run_skill", + with_args={"skill_command": "/autoskillit:resolve-review fb bb"}, + capture={"verdict": "${{ result.verdict }}"}, + on_result=StepResultRoute( + conditions=[ + StepResultCondition( + route="failure_step", + when="${{ result.verdict }} == 'flake_suspected'", + ), + StepResultCondition( + route="failure_step", + when="${{ result.verdict }} == 'ci_only_failure'", + ), + StepResultCondition( + route="push_step", + when="${{ result.verdict }} == 'real_fix'", + ), + StepResultCondition( + route="rebase_step", + when="${{ result.verdict }} == 'already_green'", + ), + ] + ), + on_failure="failure_step", + ) + return steps + + +def test_verdict_routing_asymmetry_fires_on_inconsistent_routes() -> None: + """Rule must error when same skill routes same verdict to different outcome categories.""" + findings = run_semantic_rules(_make_recipe(_asymmetric_resolve_steps())) + rule_names = [f.rule for f in findings] + assert "verdict-routing-asymmetry" in rule_names, ( + "run_semantic_rules must emit 'verdict-routing-asymmetry' finding when " + "'flake_suspected' routes to continuation in fix but escalation in resolve_ci." + ) + + +def test_verdict_routing_asymmetry_passes_when_consistent() -> None: + """Rule must not fire when all steps route same verdict to same outcome category.""" + findings = run_semantic_rules(_make_recipe(_consistent_resolve_steps())) + rule_names = [f.rule for f in findings] + assert "verdict-routing-asymmetry" not in rule_names, ( + "verdict-routing-asymmetry must not fire when both steps route " + "'flake_suspected' to continuation steps." + ) + + +def test_verdict_routing_asymmetry_ignores_different_skills() -> None: + """Rule must not fire when different skills route same verdict differently.""" + findings = run_semantic_rules(_make_recipe(_different_skills_steps())) + rule_names = [f.rule for f in findings] + assert "verdict-routing-asymmetry" not in rule_names, ( + "verdict-routing-asymmetry must not compare routing across different skills." + ) + + +def test_verdict_routing_asymmetry_reports_correct_step_names() -> None: + """Rule finding message must include both step names involved in the asymmetry.""" + findings = run_semantic_rules(_make_recipe(_asymmetric_resolve_steps())) + verdict_findings = [f for f in findings if f.rule == "verdict-routing-asymmetry"] + assert len(verdict_findings) >= 1 + message = verdict_findings[0].message + assert "fix" in message and "resolve_ci" in message, ( + f"Finding message must reference both step names, got: {message}" + ) + + +def test_verdict_routing_asymmetry_severity_is_error() -> None: + """Rule must emit ERROR severity.""" + findings = run_semantic_rules(_make_recipe(_asymmetric_resolve_steps())) + verdict_findings = [f for f in findings if f.rule == "verdict-routing-asymmetry"] + assert len(verdict_findings) >= 1 + for finding in verdict_findings: + assert finding.severity == Severity.ERROR, ( + f"verdict-routing-asymmetry must be ERROR severity, got {finding.severity}" + ) + + +# --------------------------------------------------------------------------- +# on-result-values-in-allowed-values rule tests +# --------------------------------------------------------------------------- + + +def _make_review_pr_recipe( + verdict_route: str, + allowed_values: list[str], +) -> tuple[Recipe, dict]: + """Helper: recipe routing verdict_route with a manifest restricting allowed_values.""" + manifest = { + "version": "0.1.0", + "skills": { + "review-pr": { + "inputs": [], + "outputs": [ + { + "name": "verdict", + "type": "string", + "allowed_values": allowed_values, + } + ], + } + }, + } + recipe = _make_recipe( + { + "review_pr": RecipeStep( + tool="run_skill", + with_args={"skill_command": "/autoskillit:review-pr main main"}, + on_result=StepResultRoute( + conditions=[ + StepResultCondition( + route="fix", + when=f"${{{{ result.verdict }}}} == {verdict_route}", + ), + ] + ), + on_failure="fix", + ), + "fix": RecipeStep( + tool="run_skill", + with_args={"skill_command": "/autoskillit:smoke-task"}, + ), + } + ) + return recipe, manifest + + +def test_on_result_values_in_allowed_values_fires_on_unregistered_route( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """on-result-values-in-allowed-values fires ERROR when recipe routes an unlisted value.""" + recipe, manifest = _make_review_pr_recipe( + verdict_route="approved_with_comments", + allowed_values=["approved", "changes_requested"], + ) + monkeypatch.setattr(_rv, "load_bundled_manifest", lambda: manifest) + findings = run_semantic_rules(recipe) + rule_findings = [f for f in findings if f.rule == "on-result-values-in-allowed-values"] + assert len(rule_findings) >= 1, ( + "on-result-values-in-allowed-values must fire when recipe routes " + "'approved_with_comments' but it is not in allowed_values" + ) + assert rule_findings[0].severity == Severity.ERROR + assert "approved_with_comments" in rule_findings[0].message + + +def test_on_result_values_in_allowed_values_passes_when_consistent( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """on-result-values-in-allowed-values does not fire when all routed values are allowed.""" + recipe, manifest = _make_review_pr_recipe( + verdict_route="approved", + allowed_values=["approved", "changes_requested", "approved_with_comments", "needs_human"], + ) + monkeypatch.setattr(_rv, "load_bundled_manifest", lambda: manifest) + findings = run_semantic_rules(recipe) + rule_findings = [f for f in findings if f.rule == "on-result-values-in-allowed-values"] + assert rule_findings == [], ( + "on-result-values-in-allowed-values must not fire when all routed values " + "are present in allowed_values" + ) + + +def test_on_result_values_in_allowed_values_ignores_non_verdict_conditions( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """on-result-values-in-allowed-values must not fire for catch-all or non-output conditions.""" + manifest = { + "version": "0.1.0", + "skills": { + "review-pr": { + "inputs": [], + "outputs": [ + { + "name": "verdict", + "type": "string", + "allowed_values": ["approved", "changes_requested"], + } + ], + } + }, + } + recipe = _make_recipe( + { + "review_pr": RecipeStep( + tool="run_skill", + with_args={"skill_command": "/autoskillit:review-pr main main"}, + on_result=StepResultRoute( + conditions=[ + StepResultCondition( + route="fix", + when="${{ result.verdict }} == changes_requested", # in allowed_values + ), + StepResultCondition( + route="done", + when="true", # catch-all, no verdict value extracted + ), + ] + ), + on_failure="fix", + ), + "fix": RecipeStep( + tool="run_skill", + with_args={"skill_command": "/autoskillit:smoke-task"}, + ), + "done": RecipeStep( + tool="run_skill", + with_args={"skill_command": "/autoskillit:smoke-task"}, + ), + } + ) + monkeypatch.setattr(_rv, "load_bundled_manifest", lambda: manifest) + findings = run_semantic_rules(recipe) + rule_findings = [f for f in findings if f.rule == "on-result-values-in-allowed-values"] + assert rule_findings == [], ( + "on-result-values-in-allowed-values must not fire for catch-all 'when: true' " + "conditions that don't reference a specific verdict value" + ) + + +# --------------------------------------------------------------------------- +# unrouted-verdict-value tightening: single-value to non-terminal catch-all +# --------------------------------------------------------------------------- + + +def test_unrouted_verdict_catches_bypass_to_non_terminal_even_when_single_value(): + """ + unrouted-verdict-value must fire when the single unrouted value falls to a + catch-all that routes to a non-terminal step — not just any catch-all. + Previously the rule silently allowed any single unrouted value. + + review-pr has allowed_values [approved, changes_requested, needs_human]. + Routing changes_requested and needs_human explicitly leaves approved as the + sole unrouted value falling to a non-terminal catch-all. + """ + recipe = _make_recipe( + { + "review_pr": RecipeStep( + tool="run_skill", + with_args={"skill_command": "/autoskillit:review-pr main main"}, + capture={"verdict": "${{ result.verdict }}"}, + on_result=StepResultRoute( + conditions=[ + StepResultCondition( + route="resolve_review", + when="${{ result.verdict }} == changes_requested", + ), + StepResultCondition( + route="resolve_review", + when="${{ result.verdict }} == needs_human", + ), + StepResultCondition( + route="check_repo_ci_event", + when="true", # approved falls through to non-terminal + ), + ] + ), + on_failure="resolve_review", + ), + "resolve_review": RecipeStep(on_success="check_repo_ci_event"), + "check_repo_ci_event": RecipeStep(), + } + ) + violations = run_semantic_rules(recipe) + rule_ids = [v.rule for v in violations] + assert "unrouted-verdict-value" in rule_ids, ( + "unrouted-verdict-value must fire when the single unrouted value falls " + "through to a non-terminal catch-all step." + ) diff --git a/tests/recipe/test_rules_weak_constraint.py b/tests/recipe/test_rules_weak_constraint.py new file mode 100644 index 0000000000..ed24d627ac --- /dev/null +++ b/tests/recipe/test_rules_weak_constraint.py @@ -0,0 +1,34 @@ +from __future__ import annotations + +import pytest + +from autoskillit.recipe.io import _parse_step +from autoskillit.recipe.schema import Recipe +from autoskillit.recipe.validator import run_semantic_rules + +pytestmark = [pytest.mark.layer("recipe"), pytest.mark.small] + + +class TestWeakConstraintRule: + def _make_recipe_with_kitchen_rules(self, kitchen_rules: list[str]) -> Recipe: + steps = { + "run": _parse_step({"tool": "test_check", "on_success": "done"}), + "done": _parse_step({"action": "stop", "message": "Done"}), + } + return Recipe(name="test", description="test", steps=steps, kitchen_rules=kitchen_rules) + + def test_weak_constraint_text_detected(self) -> None: + wf = self._make_recipe_with_kitchen_rules(["Only use AutoSkillit MCP tools."]) + findings = run_semantic_rules(wf) + weak = [f for f in findings if f.rule == "weak-constraint-text"] + assert weak + + def test_detailed_constraints_pass(self) -> None: + from autoskillit.core.types import PIPELINE_FORBIDDEN_TOOLS + + tool_list = ", ".join(PIPELINE_FORBIDDEN_TOOLS) + constraint = f"NEVER use native tools ({tool_list}) from the orchestrator." + wf = self._make_recipe_with_kitchen_rules([constraint]) + findings = run_semantic_rules(wf) + weak = [f for f in findings if f.rule == "weak-constraint-text"] + assert not weak diff --git a/tests/recipe/test_rules_worktree.py b/tests/recipe/test_rules_worktree.py index 87c2653855..bff7e3d60c 100644 --- a/tests/recipe/test_rules_worktree.py +++ b/tests/recipe/test_rules_worktree.py @@ -2,6 +2,8 @@ from __future__ import annotations +import pytest + from autoskillit.core.types import Severity from autoskillit.recipe.io import ( _parse_recipe, @@ -17,6 +19,8 @@ ) from tests.recipe.conftest import _make_workflow +pytestmark = [pytest.mark.layer("recipe"), pytest.mark.small] + # --------------------------------------------------------------------------- # retry-worktree-cwd tests # --------------------------------------------------------------------------- @@ -545,7 +549,7 @@ def test_worktree_modifying_skills_includes_experiment() -> None: means recipes using it without on_context_limit silently bypass the missing-context-limit-on-worktree rule. """ - from autoskillit.recipe.rules_worktree import _WORKTREE_MODIFYING_SKILLS + from autoskillit.recipe.rules.rules_worktree import _WORKTREE_MODIFYING_SKILLS assert "implement-experiment" in _WORKTREE_MODIFYING_SKILLS, ( "_WORKTREE_MODIFYING_SKILLS must include implement-experiment so that the " diff --git a/tests/recipe/test_schema.py b/tests/recipe/test_schema.py index 505de2efae..5fbeb4c3c2 100644 --- a/tests/recipe/test_schema.py +++ b/tests/recipe/test_schema.py @@ -8,6 +8,8 @@ import pytest +pytestmark = [pytest.mark.layer("recipe"), pytest.mark.small] + def test_all_dataclasses_importable() -> None: """All dataclasses are importable from recipe.schema.""" @@ -200,6 +202,18 @@ def test_recipe_step_has_constant_field() -> None: assert step.constant is None +# --------------------------------------------------------------------------- +# RecipeKind enum +# --------------------------------------------------------------------------- + + +def test_food_truck_kind_exists() -> None: + """RecipeKind.FOOD_TRUCK exists and equals 'food-truck'.""" + from autoskillit.recipe.schema import RecipeKind + + assert RecipeKind.FOOD_TRUCK == "food-truck" + + def test_constant_step_parse_from_yaml() -> None: """A constant step is parsed from YAML into RecipeStep.constant.""" from autoskillit.recipe.io import _parse_step @@ -297,3 +311,288 @@ def test_recipe_dataclass_has_requires_packs_field(): assert "requires_packs" in fields r = Recipe(name="x", description="y") assert r.requires_packs == [] + + +def test_recipe_has_content_hash_field(): + from autoskillit.recipe.schema import Recipe + + r = Recipe(name="test", description="d") + assert r.content_hash == "" + + +def test_recipe_has_composite_hash_field(): + from autoskillit.recipe.schema import Recipe + + r = Recipe(name="test", description="d") + assert r.composite_hash == "" + + +def test_recipe_has_recipe_version_field(): + from autoskillit.recipe.schema import Recipe + + r = Recipe(name="test", description="d") + assert r.recipe_version is None + + +def test_recipe_info_has_content_hash_field(): + from autoskillit.core.types import RecipeSource + from autoskillit.recipe.schema import RecipeInfo + + ri = RecipeInfo( + name="t", description="d", source=RecipeSource.BUILTIN, path=pathlib.Path("/x") + ) + assert ri.content_hash == "" + + +def test_recipe_info_has_recipe_version_field(): + from autoskillit.core.types import RecipeSource + from autoskillit.recipe.schema import RecipeInfo + + ri = RecipeInfo( + name="t", description="d", source=RecipeSource.BUILTIN, path=pathlib.Path("/x") + ) + assert ri.recipe_version is None + + +# --------------------------------------------------------------------------- +# RecipeKind, CampaignDispatch, Recipe new fields (franchise schema extension) +# --------------------------------------------------------------------------- + + +def test_recipe_kind_enum_defined() -> None: + from enum import StrEnum + + from autoskillit.recipe.schema import RecipeKind + + assert issubclass(RecipeKind, StrEnum) + assert RecipeKind.STANDARD == "standard" + assert RecipeKind.CAMPAIGN == "campaign" + assert RecipeKind.FOOD_TRUCK == "food-truck" + assert len(RecipeKind) == 3 + + +def test_campaign_dispatch_dataclass() -> None: + from autoskillit.recipe.schema import CampaignDispatch + + assert dataclasses.is_dataclass(CampaignDispatch) + d = CampaignDispatch(name="impl", recipe="implementation", task="build feature") + assert d.name == "impl" + assert d.recipe == "implementation" + assert d.task == "build feature" + assert d.ingredients == {} + assert d.depends_on == [] + + +def test_recipe_has_kind_field_defaulting_to_standard() -> None: + from autoskillit.recipe.schema import Recipe, RecipeKind + + r = Recipe(name="x", description="y") + assert r.kind == RecipeKind.STANDARD + + +def test_recipe_has_categories_field_defaulting_to_empty() -> None: + from autoskillit.recipe.schema import Recipe + + r = Recipe(name="x", description="y") + assert r.categories == [] + + +def test_recipe_has_dispatches_field_defaulting_to_empty() -> None: + from autoskillit.recipe.schema import Recipe + + r = Recipe(name="x", description="y") + assert r.dispatches == [] + + +def test_recipe_has_requires_recipe_packs_field_defaulting_to_empty() -> None: + from autoskillit.recipe.schema import Recipe + + r = Recipe(name="x", description="y") + assert r.requires_recipe_packs == [] + + +def test_recipe_has_allowed_recipes_field_defaulting_to_empty() -> None: + from autoskillit.recipe.schema import Recipe + + r = Recipe(name="x", description="y") + assert r.allowed_recipes == [] + + +def test_recipe_has_continue_on_failure_field_defaulting_to_false() -> None: + from autoskillit.recipe.schema import Recipe + + r = Recipe(name="x", description="y") + assert r.continue_on_failure is False + + +def test_existing_recipe_construction_unchanged() -> None: + """All pre-existing Recipe construction patterns still work.""" + from autoskillit.recipe.schema import Recipe, RecipeKind, RecipeStep + + r = Recipe( + name="test", + description="test recipe", + steps={"stop": RecipeStep(action="stop")}, + requires_packs=["github"], + ) + assert r.kind == RecipeKind.STANDARD + assert r.categories == [] + assert r.dispatches == [] + assert r.requires_recipe_packs == [] + assert r.allowed_recipes == [] + assert r.continue_on_failure is False + + +def test_campaign_recipe_construction() -> None: + from autoskillit.recipe.schema import CampaignDispatch, Recipe, RecipeKind + + r = Recipe( + name="multi-impl", + description="campaign", + kind=RecipeKind.CAMPAIGN, + dispatches=[ + CampaignDispatch( + name="phase-1", + recipe="implementation", + task="Build feature A", + ingredients={"branch": "feature-a"}, + depends_on=[], + ), + CampaignDispatch( + name="phase-2", + recipe="implementation", + task="Build feature B", + depends_on=["phase-1"], + ), + ], + requires_recipe_packs=["implementation-family"], + continue_on_failure=True, + ) + assert r.kind == RecipeKind.CAMPAIGN + assert len(r.dispatches) == 2 + assert r.dispatches[1].depends_on == ["phase-1"] + assert r.requires_recipe_packs == ["implementation-family"] + assert r.continue_on_failure is True + assert r.steps == {} + assert r.requires_packs == [] + assert r.allowed_recipes == [] + assert r.categories == [] + + +def test_recipe_info_has_experimental_field() -> None: + """RecipeInfo must have an experimental field defaulting to False.""" + from pathlib import Path + + from autoskillit.core import RecipeSource + from autoskillit.recipe.schema import RecipeInfo + + r = RecipeInfo(name="x", description="d", source=RecipeSource.BUILTIN, path=Path("/x.yaml")) + assert hasattr(r, "experimental") + assert r.experimental is False + + +def test_recipe_info_experimental_can_be_set_true() -> None: + """RecipeInfo.experimental must be settable to True.""" + from pathlib import Path + + from autoskillit.core import RecipeSource + from autoskillit.recipe.schema import RecipeInfo + + r = RecipeInfo( + name="x", + description="d", + source=RecipeSource.BUILTIN, + path=Path("/x.yaml"), + experimental=True, + ) + assert r.experimental is True + + +# --------------------------------------------------------------------------- +# T3: categories validation (Finding 5) +# --------------------------------------------------------------------------- + + +def test_recipe_rejects_unknown_category() -> None: + """Recipe.__post_init__ raises ValueError for unknown category values.""" + from autoskillit.recipe.schema import Recipe + + with pytest.raises(ValueError, match="Unknown categories"): + Recipe(name="x", description="d", categories=["no_such_category"]) + + +def test_recipe_accepts_valid_category() -> None: + """Recipe accepts a valid RECIPE_PACK_TAGS member in categories.""" + from autoskillit.recipe.schema import Recipe + + r = Recipe(name="x", description="d", categories=["implementation-family"]) + assert r.categories == ["implementation-family"] + + +def test_recipe_accepts_empty_categories() -> None: + """Recipe accepts an empty categories list.""" + from autoskillit.recipe.schema import Recipe + + r = Recipe(name="x", description="d", categories=[]) + assert r.categories == [] + + +def test_recipe_requires_features_field() -> None: + """Recipe dataclass accepts requires_features list.""" + from autoskillit.recipe.schema import Recipe + + r = Recipe(name="x", description="d", requires_features=["planner"]) + assert r.requires_features == ["planner"] + + +def test_recipe_requires_features_defaults_to_empty() -> None: + """Recipe.requires_features defaults to empty list.""" + from autoskillit.recipe.schema import Recipe + + r = Recipe(name="x", description="d") + assert r.requires_features == [] + + +def test_recipe_requires_features_rejects_unknown_feature() -> None: + """Recipe raises ValueError for unknown feature names in requires_features.""" + from autoskillit.recipe.schema import Recipe + + with pytest.raises(ValueError, match="Unknown features"): + Recipe(name="x", description="d", requires_features=["nonexistent_feature_xyz"]) + + +# --------------------------------------------------------------------------- +# T2: RecipeIngredient.type field (ingredient type enforcement) +# --------------------------------------------------------------------------- + + +def test_recipe_ingredient_type_field_exists() -> None: + """RecipeIngredient must have a type field.""" + from autoskillit.recipe.schema import RecipeIngredient + + ing = RecipeIngredient(description="d", type="integer", default="3") + assert hasattr(ing, "type") + + +def test_recipe_ingredient_type_stores_value() -> None: + """RecipeIngredient.type must store the provided type string.""" + from autoskillit.recipe.schema import RecipeIngredient + + ing = RecipeIngredient(description="d", type="integer", default="3") + assert ing.type == "integer" + + +def test_recipe_ingredient_type_defaults_to_none() -> None: + """RecipeIngredient.type must default to None when not provided.""" + from autoskillit.recipe.schema import RecipeIngredient + + ing = RecipeIngredient(description="d") + assert ing.type is None + + +def test_recipe_ingredient_type_can_be_explicitly_none() -> None: + """RecipeIngredient.type may be explicitly set to None.""" + from autoskillit.recipe.schema import RecipeIngredient + + ing = RecipeIngredient(description="d", type=None) + assert ing.type is None diff --git a/tests/recipe/test_silent_type_convention.py b/tests/recipe/test_silent_type_convention.py new file mode 100644 index 0000000000..0561add4b2 --- /dev/null +++ b/tests/recipe/test_silent_type_convention.py @@ -0,0 +1,33 @@ +"""Verify silent-type-convention.md exists and contains required sections.""" + +from pathlib import Path + +import pytest + +pytestmark = [pytest.mark.layer("recipe"), pytest.mark.small] + +_REPO_ROOT = Path(__file__).resolve().parents[2] +CONVENTION_PATH = _REPO_ROOT / "docs" / "research" / "silent-type-convention.md" + + +def test_silent_type_convention_exists(): + assert CONVENTION_PATH.exists() + + +def test_convention_has_detection_criteria(): + content = CONVENTION_PATH.read_text() + assert "dimension_weights" in content + assert "mandatory_figures" in content + + +def test_convention_has_advisory_schema(): + content = CONVENTION_PATH.read_text() + assert "advisory_context" in content + assert "subject_kind" in content + assert "requires_decision" in content + + +def test_convention_has_write_targets(): + content = CONVENTION_PATH.read_text() + assert "design-review-dashboard.md" in content + assert "visualization-plan-trace.md" in content diff --git a/tests/recipe/test_silent_type_flows.py b/tests/recipe/test_silent_type_flows.py new file mode 100644 index 0000000000..61b50cd6f1 --- /dev/null +++ b/tests/recipe/test_silent_type_flows.py @@ -0,0 +1,86 @@ +"""Integration tests for silent-type convention flows. + +Tests verify that registry, convention doc, and SKILL.md files are aligned +on advisory schema and behavior. +""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from autoskillit.recipe.methodology_tradition_registry import ( + get_methodology_tradition_by_name, + is_out_of_scope_tradition, + load_all_methodology_traditions, +) + +pytestmark = [pytest.mark.layer("recipe"), pytest.mark.small] + +_REPO_ROOT = Path(__file__).resolve().parents[2] +_SKILLS_DIR = _REPO_ROOT / "src" / "autoskillit" / "skills_extended" +_CONVENTION_PATH = _REPO_ROOT / "docs" / "research" / "silent-type-convention.md" + + +class TestVisLensOutOfScopeFlow: + """vis-lens-methodology-norms out-of-scope tradition path.""" + + def test_qualitative_tradition_is_out_of_scope(self) -> None: + spec = get_methodology_tradition_by_name("qualitative_interpretive_tradition") + assert spec is not None + assert is_out_of_scope_tradition(spec) is True + + def test_qualitative_tradition_has_strongly_expected_figures(self) -> None: + spec = get_methodology_tradition_by_name("qualitative_interpretive_tradition") + assert spec is not None + assert len(spec.strongly_expected_figures) >= 1 + for entry in spec.strongly_expected_figures: + assert isinstance(entry, dict), f"Expected dict, got {type(entry)}" + assert "figure" in entry + assert "source" in entry + + def test_qualitative_tradition_has_reference_framework(self) -> None: + spec = get_methodology_tradition_by_name("qualitative_interpretive_tradition") + assert spec is not None + assert "name" in spec.canonical_guideline, "canonical_guideline missing 'name' key" + assert spec.canonical_guideline["name"] == "COREQ/SRQR" + + def test_vis_lens_skill_documents_out_of_scope_advisory(self) -> None: + skill_md = (_SKILLS_DIR / "vis-lens-methodology-norms" / "SKILL.md").read_text() + for term in [ + "Out-of-Scope Tradition Handling", + "advisory_context", + "subject_kind", + "methodology_tradition", + "strongly_expected_figures", + "is_out_of_scope_tradition", + "verdict: GO", + "requires_decision: false", + "visualization-plan-trace.md", + ]: + assert term in skill_md, f"SKILL.md missing '{term}'" + + def test_convention_doc_and_skill_agree_on_write_target(self) -> None: + convention = _CONVENTION_PATH.read_text() + skill_md = (_SKILLS_DIR / "vis-lens-methodology-norms" / "SKILL.md").read_text() + target = "visualization-plan-trace.md" + assert target in convention, f"Convention doc missing '{target}'" + assert target in skill_md, f"SKILL.md missing '{target}'" + + def test_all_out_of_scope_traditions_have_strongly_expected_figures(self) -> None: + traditions = load_all_methodology_traditions() + oos = [s for s in traditions if is_out_of_scope_tradition(s)] + assert len(oos) >= 1, "Expected at least one out-of-scope tradition" + for spec in oos: + assert len(spec.strongly_expected_figures) >= 1, ( + f"{spec.name}: out-of-scope tradition must have strongly_expected_figures" + ) + + def test_no_in_scope_tradition_has_empty_mandatory_figures(self) -> None: + traditions = load_all_methodology_traditions() + for spec in traditions: + if not is_out_of_scope_tradition(spec): + assert len(spec.mandatory_figures) >= 1, ( + f"{spec.name}: in-scope tradition must have mandatory_figures" + ) diff --git a/tests/recipe/test_skill_emit_consistency.py b/tests/recipe/test_skill_emit_consistency.py index 7d004c21db..9cf00502b5 100644 --- a/tests/recipe/test_skill_emit_consistency.py +++ b/tests/recipe/test_skill_emit_consistency.py @@ -4,9 +4,13 @@ import re +import pytest + from autoskillit.core.paths import pkg_root from autoskillit.recipe.contracts import load_bundled_manifest +pytestmark = [pytest.mark.layer("recipe"), pytest.mark.small] + # Key pattern: if a contract output pattern requires an absolute path (contains # \s*=\s*/.+), verify that the SKILL.md does not assign the variable to a # relative path (i.e., assignments starting with ../ or a non-/ non-$ character). diff --git a/tests/recipe/test_sprint_sub_recipe.py b/tests/recipe/test_sprint_sub_recipe.py deleted file mode 100644 index 7e67e903b1..0000000000 --- a/tests/recipe/test_sprint_sub_recipe.py +++ /dev/null @@ -1,115 +0,0 @@ -"""Tests for the sprint-prefix sub-recipe content and structure.""" - -from __future__ import annotations - -import pytest - -from autoskillit.recipe._analysis import make_validation_context -from autoskillit.recipe.io import builtin_sub_recipes_dir, load_recipe -from autoskillit.recipe.registry import run_semantic_rules -from autoskillit.recipe.validator import validate_recipe - - -@pytest.fixture(scope="module") -def sprint_prefix_recipe(): - path = builtin_sub_recipes_dir() / "sprint-prefix.yaml" - return load_recipe(path) - - -def test_sprint_sub_recipe_loads(sprint_prefix_recipe) -> None: - """sprint-prefix.yaml parses without errors.""" - assert sprint_prefix_recipe.name == "sprint-prefix" - - -def test_sprint_sub_recipe_structural_validation_clean(sprint_prefix_recipe) -> None: - """validate_recipe(sprint_prefix) returns no errors.""" - errors = validate_recipe(sprint_prefix_recipe) - assert not errors, f"Structural validation errors: {errors}" - - -def test_sprint_sub_recipe_no_semantic_errors(sprint_prefix_recipe) -> None: - """run_semantic_rules on sprint-prefix has no ERROR-severity findings.""" - from autoskillit.core.types import Severity - - ctx = make_validation_context( - sprint_prefix_recipe, - available_skills=frozenset( - { - "triage-issues", - "sprint-planner", - "process-issues", - } - ), - ) - errors = [f for f in run_semantic_rules(ctx) if f.severity == Severity.ERROR] - assert not errors, f"Semantic errors: {errors}" - - -def test_sprint_sub_recipe_has_triage_step(sprint_prefix_recipe) -> None: - """sprint-prefix.yaml has a 'triage' step calling triage-issues skill.""" - assert "triage" in sprint_prefix_recipe.steps - triage = sprint_prefix_recipe.steps["triage"] - assert triage.tool == "run_skill" - skill_cmd = triage.with_args.get("skill_command", "") - assert "triage-issues" in skill_cmd - - -def test_sprint_sub_recipe_has_plan_step(sprint_prefix_recipe) -> None: - """sprint-prefix.yaml has a 'plan' step for sprint planning.""" - assert "plan" in sprint_prefix_recipe.steps - plan = sprint_prefix_recipe.steps["plan"] - assert plan.tool == "run_skill" - skill_cmd = plan.with_args.get("skill_command", "") - assert "sprint-planner" in skill_cmd - - -def test_sprint_sub_recipe_has_confirm_step(sprint_prefix_recipe) -> None: - """sprint-prefix.yaml has a confirm action step for user approval.""" - assert "confirm" in sprint_prefix_recipe.steps - confirm = sprint_prefix_recipe.steps["confirm"] - assert confirm.action == "confirm" - - -def test_sprint_sub_recipe_has_dispatch_step(sprint_prefix_recipe) -> None: - """sprint-prefix.yaml has a dispatch step calling process-issues.""" - assert "dispatch" in sprint_prefix_recipe.steps - dispatch = sprint_prefix_recipe.steps["dispatch"] - assert dispatch.tool == "run_skill" - skill_cmd = dispatch.with_args.get("skill_command", "") - assert "process-issues" in skill_cmd - - -def test_sprint_sub_recipe_has_report_step(sprint_prefix_recipe) -> None: - """sprint-prefix.yaml has a done-terminal report step.""" - assert "report" in sprint_prefix_recipe.steps - report = sprint_prefix_recipe.steps["report"] - assert report.action == "stop" - - -def test_sprint_sub_recipe_has_sprint_size_ingredient(sprint_prefix_recipe) -> None: - """sprint-prefix.yaml declares 'sprint_size' ingredient with default '4'.""" - assert "sprint_size" in sprint_prefix_recipe.ingredients - ingredient = sprint_prefix_recipe.ingredients["sprint_size"] - assert ingredient.default == "4" - - -def test_sprint_sub_recipe_terminal_routes_to_done(sprint_prefix_recipe) -> None: - """All terminal paths in sprint-prefix.yaml end at 'done' or 'escalate'.""" - terminal_names = {"done", "escalate", "report"} - for name, step in sprint_prefix_recipe.steps.items(): - if name in terminal_names: - assert step.action == "stop", f"Step '{name}' should be action: stop" - - -def test_sprint_sub_recipe_kitchen_rules_present(sprint_prefix_recipe) -> None: - """sprint-prefix.yaml has non-empty kitchen_rules.""" - assert sprint_prefix_recipe.kitchen_rules, "sprint-prefix.yaml must have kitchen_rules" - assert len(sprint_prefix_recipe.kitchen_rules) >= 1 - - -def test_sprint_sub_recipe_diagram_exists() -> None: - """Diagram file exists at recipes/sub-recipes/diagrams/sprint-prefix.md.""" - diagram_path = builtin_sub_recipes_dir() / "diagrams" / "sprint-prefix.md" - assert diagram_path.exists(), f"Diagram not found: {diagram_path}" - content = diagram_path.read_text() - assert "sprint-prefix" in content diff --git a/tests/recipe/test_staleness_cache.py b/tests/recipe/test_staleness_cache.py index e0e5cd96dd..f7c821ff97 100644 --- a/tests/recipe/test_staleness_cache.py +++ b/tests/recipe/test_staleness_cache.py @@ -2,6 +2,8 @@ from __future__ import annotations +import pytest + from autoskillit.recipe.staleness_cache import ( StalenessEntry, compute_recipe_hash, @@ -9,6 +11,8 @@ write_staleness_cache, ) +pytestmark = [pytest.mark.layer("recipe"), pytest.mark.medium] + def _make_entry(**kwargs) -> StalenessEntry: defaults = dict( diff --git a/tests/recipe/test_sub_recipe_loading.py b/tests/recipe/test_sub_recipe_loading.py index 6367d76a70..203ad530fa 100644 --- a/tests/recipe/test_sub_recipe_loading.py +++ b/tests/recipe/test_sub_recipe_loading.py @@ -5,29 +5,33 @@ import textwrap from pathlib import Path +import pytest + from autoskillit.recipe._api import _build_active_recipe from autoskillit.recipe.schema import Recipe, RecipeIngredient, RecipeStep +pytestmark = [pytest.mark.layer("recipe"), pytest.mark.small] + def _make_parent_recipe( gate_default: str = "false", on_success: str = "clone", on_failure: str = "escalate", ) -> Recipe: - """Parent recipe with a sprint_entry sub_recipe placeholder step.""" + """Parent recipe with a test_entry sub_recipe placeholder step.""" return Recipe( name="implementation", description="Main recipe", ingredients={ - "sprint_mode": RecipeIngredient( - description="Enable sprint mode", default=gate_default, hidden=True + "flag_mode": RecipeIngredient( + description="Enable flag mode", default=gate_default, hidden=True ), "task": RecipeIngredient(description="Task description", required=True), }, steps={ - "sprint_entry": RecipeStep( - sub_recipe="sprint-prefix", - gate="sprint_mode", + "test_entry": RecipeStep( + sub_recipe="test-sub", + gate="flag_mode", on_success=on_success, on_failure=on_failure, on_exhausted="escalate", @@ -43,39 +47,11 @@ def _make_parent_recipe( ) -def _make_sub_recipe() -> Recipe: - """A simple sub-recipe with two steps.""" - return Recipe( - name="sprint-prefix", - description="Sprint setup prefix", - ingredients={ - "sprint_branch": RecipeIngredient(description="Branch to use", default="main"), - }, - steps={ - "check_sprint": RecipeStep( - tool="run_cmd", - with_args={"cmd": "git status"}, - on_success="setup_sprint", - on_failure="escalate", - on_exhausted="escalate", - ), - "setup_sprint": RecipeStep( - tool="run_cmd", - with_args={"cmd": "echo ready"}, - on_success="done", - on_failure="escalate", - on_exhausted="escalate", - ), - }, - kitchen_rules=["no native tools in sub-recipe"], - ) - - def test_load_with_gate_false_drops_sub_recipe_step(tmp_path: Path) -> None: """When gate ingredient is false, sub_recipe step is absent from served Recipe.""" parent = _make_parent_recipe(gate_default="false") active, combined = _build_active_recipe(parent, None, tmp_path) - assert "sprint_entry" not in active.steps + assert "test_entry" not in active.steps assert combined is None @@ -94,7 +70,7 @@ def test_load_with_gate_true_merges_steps(tmp_path: Path) -> None: sub_dir = tmp_path / ".autoskillit" / "recipes" / "sub-recipes" sub_dir.mkdir(parents=True) sub_recipe_content = textwrap.dedent(""" - name: sprint-prefix + name: test-sub description: Sprint setup kitchen_rules: - no native tools @@ -106,13 +82,13 @@ def test_load_with_gate_true_merges_steps(tmp_path: Path) -> None: on_success: done on_failure: escalate """) - (sub_dir / "sprint-prefix.yaml").write_text(sub_recipe_content) + (sub_dir / "test-sub.yaml").write_text(sub_recipe_content) parent = _make_parent_recipe(gate_default="false") - active, combined = _build_active_recipe(parent, {"sprint_mode": "true"}, tmp_path) + active, combined = _build_active_recipe(parent, {"flag_mode": "true"}, tmp_path) assert combined is not None # Placeholder step gone; sub-recipe steps present with prefix - assert "sprint_entry" not in active.steps + assert "test_entry" not in active.steps assert any("check_sprint" in name for name in active.steps) @@ -121,7 +97,7 @@ def test_merged_steps_have_prefixed_names(tmp_path: Path) -> None: sub_dir = tmp_path / ".autoskillit" / "recipes" / "sub-recipes" sub_dir.mkdir(parents=True) sub_content = textwrap.dedent(""" - name: sprint-prefix + name: test-sub description: Sprint setup kitchen_rules: - no native tools @@ -133,13 +109,13 @@ def test_merged_steps_have_prefixed_names(tmp_path: Path) -> None: on_success: done on_failure: escalate """) - (sub_dir / "sprint-prefix.yaml").write_text(sub_content) + (sub_dir / "test-sub.yaml").write_text(sub_content) parent = _make_parent_recipe() - active, _ = _build_active_recipe(parent, {"sprint_mode": "true"}, tmp_path) + active, _ = _build_active_recipe(parent, {"flag_mode": "true"}, tmp_path) # All merged step names should have the sub-recipe prefix sub_step_names = [n for n in active.steps if n != "clone"] - assert all(name.startswith("sprint_prefix_") for name in sub_step_names) + assert all(name.startswith("test_sub_") for name in sub_step_names) def test_merged_step_done_routes_to_on_success(tmp_path: Path) -> None: @@ -147,7 +123,7 @@ def test_merged_step_done_routes_to_on_success(tmp_path: Path) -> None: sub_dir = tmp_path / ".autoskillit" / "recipes" / "sub-recipes" sub_dir.mkdir(parents=True) sub_content = textwrap.dedent(""" - name: sprint-prefix + name: test-sub description: Sprint setup kitchen_rules: - no native tools @@ -159,14 +135,14 @@ def test_merged_step_done_routes_to_on_success(tmp_path: Path) -> None: on_success: done on_failure: escalate """) - (sub_dir / "sprint-prefix.yaml").write_text(sub_content) + (sub_dir / "test-sub.yaml").write_text(sub_content) parent = _make_parent_recipe(on_success="clone") - active, _ = _build_active_recipe(parent, {"sprint_mode": "true"}, tmp_path) + active, _ = _build_active_recipe(parent, {"flag_mode": "true"}, tmp_path) # The sub-recipe step's on_success (was "done") should now route to parent's on_success - sprint_step = next((s for name, s in active.steps.items() if "check_sprint" in name), None) - assert sprint_step is not None - assert sprint_step.on_success == "clone" + merged_step = next((s for name, s in active.steps.items() if "check_sprint" in name), None) + assert merged_step is not None + assert merged_step.on_success == "clone" def test_merged_step_escalate_routes_to_on_failure(tmp_path: Path) -> None: @@ -174,7 +150,7 @@ def test_merged_step_escalate_routes_to_on_failure(tmp_path: Path) -> None: sub_dir = tmp_path / ".autoskillit" / "recipes" / "sub-recipes" sub_dir.mkdir(parents=True) sub_content = textwrap.dedent(""" - name: sprint-prefix + name: test-sub description: Sprint setup kitchen_rules: - no native tools @@ -186,13 +162,13 @@ def test_merged_step_escalate_routes_to_on_failure(tmp_path: Path) -> None: on_success: done on_failure: escalate """) - (sub_dir / "sprint-prefix.yaml").write_text(sub_content) + (sub_dir / "test-sub.yaml").write_text(sub_content) parent = _make_parent_recipe(on_failure="escalate") - active, _ = _build_active_recipe(parent, {"sprint_mode": "true"}, tmp_path) - sprint_step = next((s for name, s in active.steps.items() if "check_sprint" in name), None) - assert sprint_step is not None - assert sprint_step.on_failure == "escalate" + active, _ = _build_active_recipe(parent, {"flag_mode": "true"}, tmp_path) + merged_step = next((s for name, s in active.steps.items() if "check_sprint" in name), None) + assert merged_step is not None + assert merged_step.on_failure == "escalate" def test_merged_ingredients_include_sub_recipe_ingredients(tmp_path: Path) -> None: @@ -200,7 +176,7 @@ def test_merged_ingredients_include_sub_recipe_ingredients(tmp_path: Path) -> No sub_dir = tmp_path / ".autoskillit" / "recipes" / "sub-recipes" sub_dir.mkdir(parents=True) sub_content = textwrap.dedent(""" - name: sprint-prefix + name: test-sub description: Sprint setup kitchen_rules: - no native tools @@ -216,10 +192,10 @@ def test_merged_ingredients_include_sub_recipe_ingredients(tmp_path: Path) -> No on_success: done on_failure: escalate """) - (sub_dir / "sprint-prefix.yaml").write_text(sub_content) + (sub_dir / "test-sub.yaml").write_text(sub_content) parent = _make_parent_recipe() - active, _ = _build_active_recipe(parent, {"sprint_mode": "true"}, tmp_path) + active, _ = _build_active_recipe(parent, {"flag_mode": "true"}, tmp_path) assert "sprint_branch" in active.ingredients @@ -228,7 +204,7 @@ def test_merged_kitchen_rules_union(tmp_path: Path) -> None: sub_dir = tmp_path / ".autoskillit" / "recipes" / "sub-recipes" sub_dir.mkdir(parents=True) sub_content = textwrap.dedent(""" - name: sprint-prefix + name: test-sub description: Sprint setup kitchen_rules: - sub recipe rule @@ -240,10 +216,10 @@ def test_merged_kitchen_rules_union(tmp_path: Path) -> None: on_success: done on_failure: escalate """) - (sub_dir / "sprint-prefix.yaml").write_text(sub_content) + (sub_dir / "test-sub.yaml").write_text(sub_content) parent = _make_parent_recipe() - active, _ = _build_active_recipe(parent, {"sprint_mode": "true"}, tmp_path) + active, _ = _build_active_recipe(parent, {"flag_mode": "true"}, tmp_path) assert "no native tools" in active.kitchen_rules assert "sub recipe rule" in active.kitchen_rules @@ -255,7 +231,7 @@ def test_sub_recipe_hidden_ingredients_remain_hidden(tmp_path: Path) -> None: sub_dir = tmp_path / ".autoskillit" / "recipes" / "sub-recipes" sub_dir.mkdir(parents=True) sub_content = textwrap.dedent(""" - name: sprint-prefix + name: test-sub description: Sprint setup kitchen_rules: - no native tools @@ -272,11 +248,19 @@ def test_sub_recipe_hidden_ingredients_remain_hidden(tmp_path: Path) -> None: on_success: done on_failure: escalate """) - (sub_dir / "sprint-prefix.yaml").write_text(sub_content) + (sub_dir / "test-sub.yaml").write_text(sub_content) parent = _make_parent_recipe() - active, _ = _build_active_recipe(parent, {"sprint_mode": "true"}, tmp_path) + active, _ = _build_active_recipe(parent, {"flag_mode": "true"}, tmp_path) # hidden sub-recipe ingredients must not be merged into the parent at all assert "hidden_flag" not in active.ingredients table = format_ingredients_table(active) assert table is None or "hidden_flag" not in (table or "") + + +def test_sprint_prefix_sub_recipe_does_not_exist() -> None: + """sprint-prefix sub-recipe must not exist in the bundled sub-recipe directory.""" + from autoskillit.recipe.io import builtin_sub_recipes_dir + + sub_recipe_path = builtin_sub_recipes_dir() / "sprint-prefix.yaml" + assert not sub_recipe_path.exists() diff --git a/tests/recipe/test_sub_recipe_schema.py b/tests/recipe/test_sub_recipe_schema.py index 2526f2d3d3..905ce34be3 100644 --- a/tests/recipe/test_sub_recipe_schema.py +++ b/tests/recipe/test_sub_recipe_schema.py @@ -2,9 +2,13 @@ from __future__ import annotations +import pytest + from autoskillit.recipe.io import _parse_step from autoskillit.recipe.schema import Recipe, RecipeIngredient, RecipeStep -from autoskillit.recipe.validator import validate_recipe +from autoskillit.recipe.validator import validate_recipe_structure + +pytestmark = [pytest.mark.layer("recipe"), pytest.mark.small] def _make_minimal_recipe(steps: dict, ingredients: dict | None = None) -> Recipe: @@ -21,13 +25,13 @@ def test_sub_recipe_step_parsed() -> None: """RecipeStep with sub_recipe and gate fields is parsed correctly.""" step = _parse_step( { - "sub_recipe": "sprint-prefix", - "gate": "sprint_mode", + "sub_recipe": "test-sub", + "gate": "flag_mode", "on_success": "clone", } ) - assert step.sub_recipe == "sprint-prefix" - assert step.gate == "sprint_mode" + assert step.sub_recipe == "test-sub" + assert step.gate == "flag_mode" assert step.on_success == "clone" @@ -36,18 +40,18 @@ def test_sub_recipe_is_valid_discriminator() -> None: recipe = Recipe( name="test-recipe", description="Test", - ingredients={"sprint_mode": RecipeIngredient(description="Gate", default="false")}, + ingredients={"flag_mode": RecipeIngredient(description="Gate", default="false")}, steps={ - "sprint_entry": RecipeStep( - sub_recipe="sprint-prefix", - gate="sprint_mode", + "test_entry": RecipeStep( + sub_recipe="test-sub", + gate="flag_mode", on_success="done", on_exhausted="escalate", ), }, kitchen_rules=["no native tools"], ) - errors = validate_recipe(recipe) + errors = validate_recipe_structure(recipe) # Should not error about missing discriminator discriminator_errors = [e for e in errors if "must have 'tool'" in e] assert not discriminator_errors @@ -58,10 +62,10 @@ def test_sub_recipe_requires_gate() -> None: recipe = Recipe( name="test-recipe", description="Test", - ingredients={"sprint_mode": RecipeIngredient(description="Gate", default="false")}, + ingredients={"flag_mode": RecipeIngredient(description="Gate", default="false")}, steps={ - "sprint_entry": RecipeStep( - sub_recipe="sprint-prefix", + "test_entry": RecipeStep( + sub_recipe="test-sub", on_success="done", on_exhausted="escalate", # gate is missing @@ -69,7 +73,7 @@ def test_sub_recipe_requires_gate() -> None: }, kitchen_rules=["no native tools"], ) - errors = validate_recipe(recipe) + errors = validate_recipe_structure(recipe) assert any("must have a 'gate' field" in e for e in errors) @@ -78,19 +82,19 @@ def test_sub_recipe_gate_must_be_known_ingredient() -> None: recipe = Recipe( name="test-recipe", description="Test", - ingredients={}, # sprint_mode NOT declared + ingredients={}, # flag_mode NOT declared steps={ - "sprint_entry": RecipeStep( - sub_recipe="sprint-prefix", - gate="sprint_mode", + "test_entry": RecipeStep( + sub_recipe="test-sub", + gate="flag_mode", on_success="done", on_exhausted="escalate", ), }, kitchen_rules=["no native tools"], ) - errors = validate_recipe(recipe) - assert any("undeclared ingredient 'sprint_mode'" in e for e in errors) + errors = validate_recipe_structure(recipe) + assert any("undeclared ingredient 'flag_mode'" in e for e in errors) def test_sub_recipe_discriminator_exclusion() -> None: @@ -98,11 +102,11 @@ def test_sub_recipe_discriminator_exclusion() -> None: recipe = Recipe( name="test-recipe", description="Test", - ingredients={"sprint_mode": RecipeIngredient(description="Gate", default="false")}, + ingredients={"flag_mode": RecipeIngredient(description="Gate", default="false")}, steps={ - "sprint_entry": RecipeStep( - sub_recipe="sprint-prefix", - gate="sprint_mode", + "test_entry": RecipeStep( + sub_recipe="test-sub", + gate="flag_mode", tool="run_skill", # conflict! on_success="done", on_exhausted="escalate", @@ -110,7 +114,7 @@ def test_sub_recipe_discriminator_exclusion() -> None: }, kitchen_rules=["no native tools"], ) - errors = validate_recipe(recipe) + errors = validate_recipe_structure(recipe) assert any("sub_recipe" in e and "mutually exclusive" in e for e in errors) @@ -119,16 +123,16 @@ def test_sub_recipe_step_requires_on_success() -> None: recipe = Recipe( name="test-recipe", description="Test", - ingredients={"sprint_mode": RecipeIngredient(description="Gate", default="false")}, + ingredients={"flag_mode": RecipeIngredient(description="Gate", default="false")}, steps={ - "sprint_entry": RecipeStep( - sub_recipe="sprint-prefix", - gate="sprint_mode", + "test_entry": RecipeStep( + sub_recipe="test-sub", + gate="flag_mode", # on_success missing on_exhausted="escalate", ), }, kitchen_rules=["no native tools"], ) - errors = validate_recipe(recipe) + errors = validate_recipe_structure(recipe) assert any("must have 'on_success'" in e for e in errors) diff --git a/tests/recipe/test_sub_recipe_validation.py b/tests/recipe/test_sub_recipe_validation.py index 8903a9c293..80662e1557 100644 --- a/tests/recipe/test_sub_recipe_validation.py +++ b/tests/recipe/test_sub_recipe_validation.py @@ -5,9 +5,13 @@ import textwrap from pathlib import Path +import pytest + from autoskillit.recipe._analysis import make_validation_context from autoskillit.recipe.schema import Recipe, RecipeIngredient, RecipeStep +pytestmark = [pytest.mark.layer("recipe"), pytest.mark.small] + def _make_parent_recipe( gate_default: str = "false", @@ -18,14 +22,14 @@ def _make_parent_recipe( name="test-recipe", description="Test", ingredients={ - "sprint_mode": RecipeIngredient( - description="Enable sprint", default=gate_default, hidden=True + "flag_mode": RecipeIngredient( + description="Enable flag mode", default=gate_default, hidden=True ), }, steps={ - "sprint_entry": RecipeStep( - sub_recipe="sprint-prefix", - gate="sprint_mode", + "test_entry": RecipeStep( + sub_recipe="test-sub", + gate="flag_mode", on_success=on_success, on_failure=on_failure, on_exhausted="escalate", @@ -35,7 +39,7 @@ def _make_parent_recipe( ) -def _write_sub_recipe(sub_dir: Path, name: str = "sprint-prefix") -> None: +def _write_sub_recipe(sub_dir: Path, name: str = "test-sub") -> None: sub_dir.mkdir(parents=True, exist_ok=True) content = textwrap.dedent(f""" name: {name} @@ -84,12 +88,12 @@ def test_circular_sub_recipe_detected(tmp_path: Path) -> None: name="test-recipe", description="Test", ingredients={ - "sprint_mode": RecipeIngredient(description="Gate", default="false"), + "flag_mode": RecipeIngredient(description="Gate", default="false"), }, steps={ - "sprint_entry": RecipeStep( + "test_entry": RecipeStep( sub_recipe="self-ref", - gate="sprint_mode", + gate="flag_mode", on_success="done", on_exhausted="escalate", ), @@ -110,12 +114,12 @@ def test_unknown_sub_recipe_name_detected() -> None: name="test-recipe", description="Test", ingredients={ - "sprint_mode": RecipeIngredient(description="Gate", default="false"), + "flag_mode": RecipeIngredient(description="Gate", default="false"), }, steps={ - "sprint_entry": RecipeStep( + "test_entry": RecipeStep( sub_recipe="nonexistent-sub", - gate="sprint_mode", + gate="flag_mode", on_success="done", on_exhausted="escalate", ), @@ -137,19 +141,19 @@ def test_known_sub_recipe_name_passes() -> None: name="test-recipe", description="Test", ingredients={ - "sprint_mode": RecipeIngredient(description="Gate", default="false"), + "flag_mode": RecipeIngredient(description="Gate", default="false"), }, steps={ - "sprint_entry": RecipeStep( - sub_recipe="sprint-prefix", - gate="sprint_mode", + "test_entry": RecipeStep( + sub_recipe="test-sub", + gate="flag_mode", on_success="done", on_exhausted="escalate", ), }, kitchen_rules=["no native tools"], ) - ctx = make_validation_context(recipe, available_sub_recipes=frozenset({"sprint-prefix"})) + ctx = make_validation_context(recipe, available_sub_recipes=frozenset({"test-sub"})) findings = run_semantic_rules(ctx) unknown_findings = [f for f in findings if f.rule == "unknown-sub-recipe"] assert not unknown_findings @@ -174,14 +178,14 @@ def test_dual_validation_runs_standalone_and_combined(tmp_path: Path) -> None: - "NEVER use native Claude Code tools (Read, Grep, Glob, Edit, Write, Bash, Agent, WebFetch, WebSearch, NotebookEdit) from the orchestrator." ingredients: - sprint_mode: + flag_mode: description: Enable sprint default: "false" hidden: true steps: - sprint_entry: - sub_recipe: sprint-prefix - gate: sprint_mode + test_entry: + sub_recipe: test-sub + gate: flag_mode on_success: finish on_failure: escalate finish: @@ -196,7 +200,7 @@ def test_dual_validation_runs_standalone_and_combined(tmp_path: Path) -> None: result = load_and_validate( "test-recipe", project_dir=tmp_path, - ingredient_overrides={"sprint_mode": "true"}, + ingredient_overrides={"flag_mode": "true"}, ) assert "error" not in result # Both standalone and combined graphs were validated @@ -219,12 +223,12 @@ def test_dual_validation_standalone_errors_surfaced(tmp_path: Path) -> None: kitchen_rules: - no native tools ingredients: - sprint_mode: + flag_mode: description: Enable sprint default: "false" steps: - sprint_entry: - sub_recipe: sprint-prefix + test_entry: + sub_recipe: test-sub on_success: done """) (recipes_dir / "test-recipe.yaml").write_text(recipe_content) @@ -249,14 +253,14 @@ def test_combined_graph_dataflow_validated(tmp_path: Path) -> None: kitchen_rules: - no native tools ingredients: - sprint_mode: + flag_mode: description: Enable sprint default: "false" hidden: true steps: - sprint_entry: - sub_recipe: sprint-prefix - gate: sprint_mode + test_entry: + sub_recipe: test-sub + gate: flag_mode on_success: done """) (recipes_dir / "test-recipe.yaml").write_text(recipe_content) @@ -264,7 +268,7 @@ def test_combined_graph_dataflow_validated(tmp_path: Path) -> None: result = load_and_validate( "test-recipe", project_dir=tmp_path, - ingredient_overrides={"sprint_mode": "true"}, + ingredient_overrides={"flag_mode": "true"}, ) # When gate=true, semantic rules should have run (result contains suggestions list) assert "suggestions" in result diff --git a/tests/recipe/test_validator.py b/tests/recipe/test_validator.py deleted file mode 100644 index aa321648ab..0000000000 --- a/tests/recipe/test_validator.py +++ /dev/null @@ -1,1073 +0,0 @@ -"""Tests for recipe structural validation and dataflow analysis.""" - -from __future__ import annotations - -import dataclasses -from pathlib import Path - -import yaml - -from autoskillit.recipe.io import ( - _parse_recipe, - _parse_step, - builtin_recipes_dir, - iter_steps_with_context, - load_recipe, -) -from autoskillit.recipe.schema import ( - Recipe, - RecipeStep, -) -from autoskillit.recipe.validator import ( - analyze_dataflow, - run_semantic_rules, - validate_recipe, -) -from tests.recipe.conftest import VALID_RECIPE, _make_workflow, _write_yaml - -# --------------------------------------------------------------------------- -# TestValidateRecipe — migrated from test_recipe_parser.py -# --------------------------------------------------------------------------- - - -class TestValidateRecipe: - def test_valid_recipe_no_errors(self, tmp_path: Path) -> None: - wf = load_recipe(_write_yaml(tmp_path / "recipe.yaml", VALID_RECIPE)) - errors = validate_recipe(wf) - assert errors == [] - - # WF2 - def test_recipe_requires_name(self, tmp_path: Path) -> None: - data = {**VALID_RECIPE, "name": ""} - f = _write_yaml(tmp_path / "recipe.yaml", data) - wf = load_recipe(f) - errors = validate_recipe(wf) - assert any("name" in e.lower() for e in errors) - - # WF3 - def test_recipe_requires_steps(self, tmp_path: Path) -> None: - data = {"name": "no-steps", "description": "Missing steps", "kitchen_rules": ["test"]} - f = _write_yaml(tmp_path / "recipe.yaml", data) - wf = load_recipe(f) - errors = validate_recipe(wf) - assert any("step" in e.lower() for e in errors) - - # WF5 - def test_goto_targets_validated(self, tmp_path: Path) -> None: - data = { - "name": "bad-goto", - "description": "Invalid goto", - "kitchen_rules": ["test"], - "steps": { - "start": {"tool": "run_cmd", "on_success": "nonexistent"}, - "end": {"action": "stop", "message": "Done."}, - }, - } - f = _write_yaml(tmp_path / "recipe.yaml", data) - wf = load_recipe(f) - errors = validate_recipe(wf) - assert any("nonexistent" in e for e in errors) - - # WF6 - def test_builtin_recipes_valid(self) -> None: - bd = builtin_recipes_dir() - yamls = list(bd.glob("*.yaml")) - assert len(yamls) >= 4 - for f in yamls: - wf = load_recipe(f) - errors = validate_recipe(wf) - assert errors == [], f"Validation errors in {f.name}: {errors}" - - # WF10 - def test_terminal_step_has_message(self, tmp_path: Path) -> None: - data = { - "name": "no-msg", - "description": "Terminal without message", - "kitchen_rules": ["test"], - "steps": {"end": {"action": "stop"}}, - } - f = _write_yaml(tmp_path / "recipe.yaml", data) - wf = load_recipe(f) - errors = validate_recipe(wf) - assert any("message" in e.lower() for e in errors) - - def test_step_needs_tool_or_action(self, tmp_path: Path) -> None: - data = { - "name": "bad-step", - "description": "Neither tool nor action", - "kitchen_rules": ["test"], - "steps": {"empty": {"note": "just a note"}}, - } - f = _write_yaml(tmp_path / "recipe.yaml", data) - wf = load_recipe(f) - errors = validate_recipe(wf) - assert any("tool" in e and "action" in e for e in errors) - - def test_input_reference_validation(self, tmp_path: Path) -> None: - data = { - "name": "bad-ref", - "description": "References undeclared input", - "kitchen_rules": ["test"], - "steps": { - "run": {"tool": "run_cmd", "with": {"cmd": "${{ inputs.missing_input }}"}}, - }, - } - f = _write_yaml(tmp_path / "recipe.yaml", data) - wf = load_recipe(f) - errors = validate_recipe(wf) - assert any("missing_input" in e for e in errors) - - def test_retry_block_raises_on_load(self, tmp_path: Path) -> None: - """The old retry: block is no longer supported — loading raises ValueError.""" - import pytest - - data = { - "name": "bad-retry-block", - "description": "Old retry block is unsupported", - "kitchen_rules": ["test"], - "steps": { - "impl": { - "tool": "run_skill", - "retry": { - "max_attempts": 3, - "on": "needs_retry", - "on_exhausted": "fail", - }, - }, - "fail": {"action": "stop", "message": "Failed."}, - }, - } - f = _write_yaml(tmp_path / "recipe.yaml", data) - with pytest.raises(ValueError, match="retry.*no longer supported"): - load_recipe(f) - - def test_step_rejects_both_python_and_tool(self, tmp_path: Path) -> None: - data = { - "name": "bad", - "description": "Both python and tool", - "kitchen_rules": ["test"], - "steps": {"run": {"python": "mod.fn", "tool": "run_cmd"}}, - } - wf = load_recipe(_write_yaml(tmp_path / "recipe.yaml", data)) - errors = validate_recipe(wf) - assert any("python" in e and "tool" in e for e in errors) - - def test_python_step_requires_dotted_path(self, tmp_path: Path) -> None: - data = { - "name": "bad-path", - "description": "No dot", - "kitchen_rules": ["test"], - "steps": {"check": {"python": "bare_name"}}, - } - wf = load_recipe(_write_yaml(tmp_path / "recipe.yaml", data)) - errors = validate_recipe(wf) - assert any("dotted" in e.lower() or "module" in e.lower() for e in errors) - - # CAP3 - def test_capture_result_refs_valid(self, tmp_path: Path) -> None: - data = { - "name": "cap-valid", - "description": "Valid captures", - "kitchen_rules": ["test"], - "steps": { - "run": { - "tool": "run_skill", - "capture": { - "wp": "${{ result.worktree_path }}", - "ctx": "${{ result.failure_context }}", - }, - }, - "done": {"action": "stop", "message": "ok"}, - }, - } - wf = load_recipe(_write_yaml(tmp_path / "recipe.yaml", data)) - errors = validate_recipe(wf) - assert not any("capture" in e for e in errors) - - # CAP4 - def test_capture_non_result_namespace_rejected(self, tmp_path: Path) -> None: - data = { - "name": "cap-bad-ns", - "description": "Bad namespace", - "kitchen_rules": ["test"], - "steps": { - "run": {"tool": "run_cmd", "capture": {"foo": "${{ inputs.bar }}"}}, - "done": {"action": "stop", "message": "ok"}, - }, - } - wf = load_recipe(_write_yaml(tmp_path / "recipe.yaml", data)) - errors = validate_recipe(wf) - assert any("result" in e and "capture" in e for e in errors) - - # CAP5 - def test_capture_literal_value_rejected(self, tmp_path: Path) -> None: - data = { - "name": "cap-literal", - "description": "Literal capture", - "kitchen_rules": ["test"], - "steps": { - "run": {"tool": "run_cmd", "capture": {"foo": "literal string"}}, - "done": {"action": "stop", "message": "ok"}, - }, - } - wf = load_recipe(_write_yaml(tmp_path / "recipe.yaml", data)) - errors = validate_recipe(wf) - assert any("capture" in e and "result" in e for e in errors) - - # CAP6 - def test_context_ref_to_captured_var_valid(self, tmp_path: Path) -> None: - data = { - "name": "ctx-valid", - "description": "Valid context ref", - "kitchen_rules": ["test"], - "steps": { - "impl": { - "tool": "run_skill", - "capture": {"worktree_path": "${{ result.worktree_path }}"}, - "on_success": "test", - }, - "test": { - "tool": "test_check", - "with": {"worktree_path": "${{ context.worktree_path }}"}, - "on_success": "done", - }, - "done": {"action": "stop", "message": "ok"}, - }, - } - wf = load_recipe(_write_yaml(tmp_path / "recipe.yaml", data)) - errors = validate_recipe(wf) - assert not any("context" in e for e in errors) - - # CAP7 - def test_context_ref_to_uncaptured_var_rejected(self, tmp_path: Path) -> None: - data = { - "name": "ctx-bad", - "description": "Uncaptured ref", - "kitchen_rules": ["test"], - "steps": { - "test": { - "tool": "test_check", - "with": {"worktree_path": "${{ context.nonexistent }}"}, - }, - "done": {"action": "stop", "message": "ok"}, - }, - } - wf = load_recipe(_write_yaml(tmp_path / "recipe.yaml", data)) - errors = validate_recipe(wf) - assert any("nonexistent" in e and "context" in e for e in errors) - - # CAP8 - def test_context_forward_reference_rejected(self, tmp_path: Path) -> None: - data = { - "name": "ctx-fwd", - "description": "Forward ref", - "kitchen_rules": ["test"], - "steps": { - "check": { - "tool": "test_check", - "with": {"worktree_path": "${{ context.wp }}"}, - "on_success": "done", - }, - "produce": { - "tool": "run_skill", - "capture": {"wp": "${{ result.worktree_path }}"}, - }, - "done": {"action": "stop", "message": "ok"}, - }, - } - wf = load_recipe(_write_yaml(tmp_path / "recipe.yaml", data)) - errors = validate_recipe(wf) - assert any("wp" in e and "context" in e for e in errors) - - # CON1 - def test_recipe_schema_supports_kitchen_rules(self) -> None: - field_names = {f.name for f in dataclasses.fields(Recipe)} - assert "kitchen_rules" in field_names - - # CON3 - def test_validate_recipe_warns_missing_kitchen_rules(self, tmp_path: Path) -> None: - data = {**VALID_RECIPE} - data.pop("kitchen_rules", None) - wf = load_recipe(_write_yaml(tmp_path / "recipe.yaml", data)) - errors = validate_recipe(wf) - warnings = [e for e in errors if "kitchen_rules" in e.lower()] - assert warnings - - # CON4 - def test_bundled_recipes_have_kitchen_rules(self) -> None: - wf_dir = builtin_recipes_dir() - failures = [] - for path in sorted(wf_dir.glob("*.yaml")): - wf = load_recipe(path) - if not wf.kitchen_rules: - failures.append(f"{path.name}: missing kitchen_rules") - assert not failures - - # T_OR2 - def test_on_result_and_on_success_mutually_exclusive(self, tmp_path: Path) -> None: - data = { - "name": "conflict-recipe", - "description": "Both on_result and on_success", - "kitchen_rules": ["test"], - "steps": { - "classify": { - "tool": "classify_fix", - "on_result": { - "field": "restart_scope", - "routes": {"full_restart": "done"}, - }, - "on_success": "done", - "on_failure": "escalate", - }, - "done": {"action": "stop", "message": "Done."}, - "escalate": {"action": "stop", "message": "Escalating."}, - }, - } - f = _write_yaml(tmp_path / "recipe.yaml", data) - wf = load_recipe(f) - errors = validate_recipe(wf) - assert any("on_result" in e and "on_success" in e for e in errors) - - # T_OR6 - def test_on_result_route_done_is_valid(self, tmp_path: Path) -> None: - data = { - "name": "done-route-recipe", - "description": "Route to done", - "kitchen_rules": ["test"], - "steps": { - "classify": { - "tool": "classify_fix", - "on_result": { - "field": "restart_scope", - "routes": {"full_restart": "done", "partial_restart": "done"}, - }, - "on_failure": "escalate", - }, - "escalate": {"action": "stop", "message": "Escalating."}, - }, - } - f = _write_yaml(tmp_path / "recipe.yaml", data) - wf = load_recipe(f) - errors = validate_recipe(wf) - assert errors == [] - - # VER3 - def test_version_does_not_cause_validation_errors(self) -> None: - data = { - "name": "version-test-recipe", - "description": "A recipe for testing the version field", - "kitchen_rules": ["Only use AutoSkillit MCP tools during pipeline execution"], - "steps": { - "do_it": {"tool": "run_cmd", "on_success": "done"}, - "done": {"action": "stop", "message": "Done."}, - }, - "autoskillit_version": "0.2.0", - } - wf = _parse_recipe(data) - errors = validate_recipe(wf) - assert errors == [] - - def test_validator_rejects_zero_stale_threshold(self) -> None: - recipe = Recipe( - name="test", - description="test", - steps={"s": RecipeStep(tool="run_skill", on_success="done", stale_threshold=0)}, - kitchen_rules=["test"], - ) - errors = validate_recipe(recipe) - assert any("stale_threshold" in e for e in errors) - - def test_validator_rejects_negative_stale_threshold(self) -> None: - recipe = Recipe( - name="test", - description="test", - steps={"s": RecipeStep(tool="run_skill", on_success="done", stale_threshold=-1)}, - kitchen_rules=["test"], - ) - errors = validate_recipe(recipe) - assert any("stale_threshold" in e for e in errors) - - def test_validator_accepts_positive_stale_threshold(self) -> None: - recipe = Recipe( - name="test", - description="test", - steps={"s": RecipeStep(tool="run_skill", on_success="done", stale_threshold=2400)}, - kitchen_rules=["test"], - ) - errors = validate_recipe(recipe) - assert not any("stale_threshold" in e for e in errors) - - def test_validator_rejects_negative_idle_output_timeout(self) -> None: - recipe = Recipe( - name="test", - description="test", - steps={"s": RecipeStep(tool="run_skill", on_success="done", idle_output_timeout=-1)}, - kitchen_rules=["test"], - ) - errors = validate_recipe(recipe) - assert any("idle_output_timeout" in e for e in errors) - - def test_validator_accepts_zero_idle_output_timeout(self) -> None: - # 0 = disabled, must NOT be rejected - recipe = Recipe( - name="test", - description="test", - steps={"s": RecipeStep(tool="run_skill", on_success="done", idle_output_timeout=0)}, - kitchen_rules=["test"], - ) - errors = validate_recipe(recipe) - assert not any("idle_output_timeout" in e for e in errors) - - def test_validator_accepts_positive_idle_output_timeout(self) -> None: - recipe = Recipe( - name="test", - description="test", - steps={"s": RecipeStep(tool="run_skill", on_success="done", idle_output_timeout=120)}, - kitchen_rules=["test"], - ) - errors = validate_recipe(recipe) - assert not any("idle_output_timeout" in e for e in errors) - - -# --------------------------------------------------------------------------- -# TestDataFlowQuality — migrated from test_recipe_parser.py -# --------------------------------------------------------------------------- - - -class TestDataFlowQuality: - """Tests for data-flow quality analysis (DFQ prefix).""" - - def _make_recipe(self, steps: dict[str, dict]) -> Recipe: - parsed_steps = {name: _parse_step(data) for name, data in steps.items()} - return Recipe( - name="test", - description="test", - steps=parsed_steps, - kitchen_rules=["test"], - ) - - # DFQ1 - def test_analyze_dataflow_returns_report(self) -> None: - from autoskillit.recipe.schema import DataFlowReport - - wf = self._make_recipe( - { - "run": {"tool": "test_check", "on_success": "done"}, - "done": {"action": "stop", "message": "Done"}, - } - ) - report = analyze_dataflow(wf) - assert isinstance(report, DataFlowReport) - assert isinstance(report.warnings, list) - assert isinstance(report.summary, str) - assert report.warnings == [] - assert report.summary is not None - assert len(report.summary) > 0 - - # DFQ2 - def test_dead_output_detected(self) -> None: - wf = self._make_recipe( - { - "impl": { - "tool": "run_skill", - "capture": {"worktree_path": "${{ result.worktree_path }}"}, - "on_success": "finish", - }, - "finish": {"action": "stop", "message": "Done"}, - } - ) - report = analyze_dataflow(wf) - dead = [w for w in report.warnings if w.code == "DEAD_OUTPUT"] - assert len(dead) == 1 - assert dead[0].step_name == "impl" - assert dead[0].field == "worktree_path" - - # DFQ3 - def test_consumed_output_not_flagged(self) -> None: - wf = self._make_recipe( - { - "impl": { - "tool": "run_skill", - "capture": {"worktree_path": "${{ result.worktree_path }}"}, - "on_success": "test", - }, - "test": { - "tool": "test_check", - "with": {"worktree_path": "${{ context.worktree_path }}"}, - "on_success": "done", - }, - "done": {"action": "stop", "message": "Done"}, - } - ) - report = analyze_dataflow(wf) - dead = [w for w in report.warnings if w.code == "DEAD_OUTPUT"] - assert len(dead) == 0 - - # DFQ5 - def test_implicit_handoff_detected(self) -> None: - wf = self._make_recipe( - { - "impl": {"tool": "run_skill", "on_success": "done"}, - "done": {"action": "stop", "message": "Done"}, - } - ) - report = analyze_dataflow(wf) - implicit = [w for w in report.warnings if w.code == "IMPLICIT_HANDOFF"] - assert len(implicit) == 1 - assert implicit[0].step_name == "impl" - - # DFQ6 - def test_non_skill_step_no_implicit_handoff(self) -> None: - wf = self._make_recipe( - { - "test": {"tool": "test_check", "on_success": "done"}, - "done": {"action": "stop", "message": "Done"}, - } - ) - report = analyze_dataflow(wf) - implicit = [w for w in report.warnings if w.code == "IMPLICIT_HANDOFF"] - assert len(implicit) == 0 - - # DFQ11 - def test_summary_reports_counts(self) -> None: - wf = self._make_recipe( - { - "impl": { - "tool": "run_skill", - "capture": {"worktree_path": "${{ result.worktree_path }}"}, - "on_success": "run", - }, - "run": {"tool": "run_skill", "on_success": "done"}, - "done": {"action": "stop", "message": "Done"}, - } - ) - report = analyze_dataflow(wf) - assert "2 data-flow warnings" in report.summary - - # DFQ13 - def test_bundled_recipes_produce_reports(self) -> None: - from autoskillit.recipe.schema import DataFlowReport - - wf_dir = builtin_recipes_dir() - yaml_files = list(wf_dir.glob("*.yaml")) + list(wf_dir.glob("*.yml")) - assert len(yaml_files) > 0 - for yaml_file in yaml_files: - wf = load_recipe(yaml_file) - report = analyze_dataflow(wf) - assert isinstance(report, DataFlowReport) - assert isinstance(report.warnings, list) - - -# --------------------------------------------------------------------------- -# iter_steps_with_context integration -# --------------------------------------------------------------------------- - - -def test_validate_recipe_uses_iter_steps_with_context_for_capture_refs(tmp_path: Path) -> None: - """validate_recipe catches context refs not captured by preceding steps.""" - data = { - "name": "ctx-test", - "description": "Context validation test", - "kitchen_rules": ["test"], - "steps": { - "step1": { - "tool": "run_cmd", - "with": {"cmd": "echo hello"}, - "on_success": "step2", - }, - "step2": { - "tool": "test_check", - "with": {"worktree_path": "${{ context.worktree_path }}"}, - "on_success": "done", - }, - "done": {"action": "stop", "message": "ok"}, - }, - } - wf = load_recipe(_write_yaml(tmp_path / "recipe.yaml", data)) - # step1 has no captures, so step2 should see empty context - steps = list(iter_steps_with_context(wf)) - assert steps[1][2] == frozenset() - # validate_recipe should catch the unsatisfied context reference - errors = validate_recipe(wf) - assert any("worktree_path" in e for e in errors) - - -# --------------------------------------------------------------------------- -# isinstance guard tests (T_GD1, T_GV1) -# --------------------------------------------------------------------------- - - -class TestIsInstanceGuards: - def test_gd1_analyze_dataflow_no_raise_on_bool_with_arg(self) -> None: - """T_GD1: _detect_dead_outputs does not raise TypeError for boolean with_args.""" - data = { - "name": "bool-guard", - "description": "test", - "kitchen_rules": ["test"], - "steps": { - "plan": { - "tool": "run_skill", - "with": {"skill_command": "/autoskillit:make-plan do the task"}, - "capture": {"plan_path": "${{ result.plan_path }}"}, - "on_success": "downstream", - }, - "downstream": { - "tool": "run_cmd", - "with": {"worktree_path": True}, - "on_success": "done", - }, - "done": {"action": "stop", "message": "Done."}, - }, - } - recipe = _parse_recipe(data) - report = analyze_dataflow(recipe) - assert report is not None - - def test_gv1_validate_recipe_no_raise_on_bool_with_arg(self) -> None: - """T_GV1: validate_recipe does not raise TypeError for boolean with_args.""" - data = { - "name": "bool-guard-validate", - "description": "test", - "kitchen_rules": ["test"], - "steps": { - "step1": { - "tool": "run_cmd", - "with": {"flag": True}, - "on_success": "done", - }, - "done": {"action": "stop", "message": "Done."}, - }, - } - recipe = _parse_recipe(data) - result = validate_recipe(recipe) - assert isinstance(result, list) - assert not any("dead" in str(f).lower() for f in result), ( - "DEAD_OUTPUT rule fired on a valid recipe — false positive" - ) - - -# --------------------------------------------------------------------------- -# on_result self-consumption tests (T_OR1, T_OR2) -# --------------------------------------------------------------------------- - - -class TestOnResultConsumption: - def test_or1_on_result_field_is_not_dead_output(self) -> None: - """T_OR1: verdict captured and used as on_result.field is NOT flagged DEAD_OUTPUT.""" - steps = { - "audit_impl": { - "tool": "run_skill", - "with": { - "skill_command": "/autoskillit:audit-impl plan.md myref main", - }, - "capture": {"verdict": "${{ result.verdict }}"}, - "on_result": { - "field": "verdict", - "routes": {"GO": "done", "NO GO": "done"}, - }, - }, - "done": {"action": "stop", "message": "Done."}, - } - recipe = _make_workflow(steps) - report = analyze_dataflow(recipe) - dead = [w for w in report.warnings if w.code == "DEAD_OUTPUT" and w.field == "verdict"] - assert dead == [] - - def test_or2_different_on_result_field_flags_dead_output(self) -> None: - """T_OR2: verdict is flagged DEAD_OUTPUT when on_result.field is a different key.""" - steps = { - "audit_impl": { - "tool": "run_skill", - "with": { - "skill_command": "/autoskillit:audit-impl plan.md myref main", - }, - "capture": {"verdict": "${{ result.verdict }}"}, - "on_result": { - "field": "restart_scope", - "routes": {"full_restart": "done", "partial_restart": "done"}, - }, - }, - "done": {"action": "stop", "message": "Done."}, - } - recipe = _make_workflow(steps) - report = analyze_dataflow(recipe) - dead = [w for w in report.warnings if w.code == "DEAD_OUTPUT" and w.field == "verdict"] - assert len(dead) == 1 - - -# --------------------------------------------------------------------------- -# skip_when_false bypass edge tests -# --------------------------------------------------------------------------- - - -def test_build_step_graph_adds_bypass_edges_for_skip_when_false() -> None: - """When a step has skip_when_false, predecessors get a direct edge to its on_success.""" - from autoskillit.recipe.schema import RecipeIngredient - from autoskillit.recipe.validator import _build_step_graph - - # Recipe: entry → optional_step (skip_when_false) → final_step - # Expected bypass edge: entry → final_step - recipe = Recipe( - name="test", - description="test", - steps={ - "entry": RecipeStep(tool="run_cmd", on_success="optional_step"), - "optional_step": RecipeStep( - tool="run_skill", - on_success="final_step", - skip_when_false="inputs.flag", - ), - "final_step": RecipeStep(action="stop", message="done"), - }, - ingredients={"flag": RecipeIngredient(description="", required=False, default="true")}, - kitchen_rules=["test"], - ) - graph = _build_step_graph(recipe) - # Bypass edge: entry can reach final_step directly (skip of optional_step) - assert "final_step" in graph["entry"] - - -def test_build_step_graph_bypass_does_not_remove_normal_edge() -> None: - """The bypass edge is additional, not a replacement for the normal edge.""" - from autoskillit.recipe.schema import RecipeIngredient - from autoskillit.recipe.validator import _build_step_graph - - recipe = Recipe( - name="test", - description="test", - steps={ - "entry": RecipeStep(tool="run_cmd", on_success="optional_step"), - "optional_step": RecipeStep( - tool="run_skill", - on_success="final_step", - skip_when_false="inputs.flag", - ), - "final_step": RecipeStep(action="stop", message="done"), - }, - ingredients={"flag": RecipeIngredient(description="", required=False, default="true")}, - kitchen_rules=["test"], - ) - graph = _build_step_graph(recipe) - # Normal edge still present - assert "optional_step" in graph["entry"] - # Bypass edge also present - assert "final_step" in graph["entry"] - - -# --------------------------------------------------------------------------- -# _build_step_graph predicate on_result tests -# --------------------------------------------------------------------------- - - -class TestPredicateBuildStepGraph: - """_build_step_graph includes condition.route edges.""" - - def test_build_step_graph_includes_condition_routes(self) -> None: - """_build_step_graph produces edges for condition.route targets.""" - from autoskillit.recipe.validator import _build_step_graph - - wf = _make_workflow( - { - "merge": { - "tool": "merge_worktree", - "with": {"worktree_path": "/tmp/wt", "base_branch": "main"}, - "on_result": [ - {"when": "result.failed_step == 'test_gate'", "route": "assess"}, - {"when": "result.error", "route": "cleanup"}, - {"route": "push"}, - ], - "capture": {"cleanup_succeeded": "${{ result.cleanup_succeeded }}"}, - }, - "assess": {"action": "stop", "message": "Assess."}, - "cleanup": {"action": "stop", "message": "Cleanup."}, - "push": {"action": "stop", "message": "Push."}, - } - ) - graph = _build_step_graph(wf) - assert "assess" in graph["merge"] - assert "cleanup" in graph["merge"] - assert "push" in graph["merge"] - - -# --------------------------------------------------------------------------- -# ValidationContext tests -# --------------------------------------------------------------------------- - - -def test_run_semantic_rules_builds_step_graph_exactly_once(monkeypatch): - """_build_step_graph is called only once regardless of how many rules need it.""" - from autoskillit.recipe import _analysis - - call_count = [] - real_fn = _analysis._build_step_graph - - def counting_fn(recipe): - call_count.append(1) - return real_fn(recipe) - - monkeypatch.setattr(_analysis, "_build_step_graph", counting_fn) - - recipe = _parse_recipe( - { - "name": "test", - "description": "test", - "autoskillit_version": "0.2.0", - "kitchen_rules": ["Never use native tools"], - "steps": {"stop": {"action": "stop", "message": "done"}}, - } - ) - run_semantic_rules(recipe) - - assert len(call_count) == 1 - - -def test_run_semantic_rules_calls_analyze_dataflow_exactly_once(monkeypatch): - """analyze_dataflow is called only once regardless of how many rules consume it.""" - from autoskillit.recipe import _analysis - - call_count = [] - real_fn = _analysis.analyze_dataflow - - def counting_fn(recipe, **kwargs): - call_count.append(1) - return real_fn(recipe, **kwargs) - - monkeypatch.setattr(_analysis, "analyze_dataflow", counting_fn) - - recipe = _parse_recipe( - { - "name": "test", - "description": "test", - "autoskillit_version": "0.2.0", - "kitchen_rules": ["Never use native tools"], - "steps": {"stop": {"action": "stop", "message": "done"}}, - } - ) - run_semantic_rules(recipe) - - assert len(call_count) == 1 - - -def test_validation_context_exposes_recipe_step_graph_and_dataflow(): - """ValidationContext contains recipe, step_graph, and dataflow attributes.""" - from autoskillit.recipe._analysis import ValidationContext, make_validation_context - - recipe = _parse_recipe( - { - "name": "test", - "description": "test", - "autoskillit_version": "0.2.0", - "kitchen_rules": ["Never use native tools"], - "steps": {"stop": {"action": "stop", "message": "done"}}, - } - ) - ctx = make_validation_context(recipe) - - assert isinstance(ctx, ValidationContext) - assert ctx.recipe is recipe - assert isinstance(ctx.step_graph, dict) - assert hasattr(ctx.dataflow, "warnings") - - -def test_analyze_dataflow_accepts_prebuilt_step_graph(): - """analyze_dataflow can reuse a pre-built step_graph to avoid duplicate computation.""" - from autoskillit.recipe._analysis import _build_step_graph, analyze_dataflow - - recipe = _parse_recipe( - { - "name": "test", - "description": "test", - "autoskillit_version": "0.2.0", - "kitchen_rules": ["Never use native tools"], - "steps": { - "run": { - "tool": "run_skill", - "capture": {"wp": "${{ result.worktree_path }}"}, - "on_success": "use_it", - }, - "use_it": { - "tool": "test_check", - "with": {"worktree_path": "${{ context.wp }}"}, - "on_success": "done", - }, - "done": {"action": "stop", "message": "done"}, - }, - } - ) - graph = _build_step_graph(recipe) - report1 = analyze_dataflow(recipe, step_graph=graph) - report2 = analyze_dataflow(recipe) - assert report1.warnings == report2.warnings - - -def test_validate_recipe_catches_rectify_with_missing_investigation_path_capture() -> None: - """1h: validate_recipe catches missing investigation_path capture before rectify step. - - When rectify is invoked with ${{ context.investigation_path }} but no preceding step - captures investigation_path, the existing undeclared-context-reference validator must - return an error — confirming the enforcement layer works for the new context variable. - """ - recipe = _parse_recipe( - yaml.safe_load( - """\ -name: test-missing-capture -description: Recipe with investigate→rectify but no capture block -kitchen_rules: - - test -steps: - investigate: - tool: run_skill - with: - skill_command: "/autoskillit:investigate the bug" - on_success: rectify - on_failure: done - rectify: - tool: run_skill - with: - skill_command: "/autoskillit:rectify ${{ context.investigation_path }}" - on_success: done - on_failure: done - done: - action: stop - message: Done. -""" - ) - ) - errors = validate_recipe(recipe) - assert any("investigation_path" in e for e in errors), ( - "validate_recipe must report an error about undeclared context reference " - "'investigation_path' when the investigate step has no capture block" - ) - - -# --------------------------------------------------------------------------- -# TestConfirmAction — action: "confirm" validation -# --------------------------------------------------------------------------- - - -class TestConfirmAction: - def test_confirm_action_requires_message(self) -> None: - """action: confirm without message is a validation error.""" - recipe = _make_workflow( - { - "step1": {"action": "confirm", "on_success": "done", "on_failure": "done"}, - "done": {"action": "stop", "message": "Done."}, - } - ) - errors = validate_recipe(recipe) - assert any("message" in e.lower() for e in errors), ( - "Expected a validation error mentioning 'message'" - ) - assert any("confirm" in e.lower() for e in errors), ( - "Expected a validation error mentioning 'confirm'" - ) - - def test_confirm_action_requires_on_success(self) -> None: - """action: confirm without on_success is a validation error.""" - recipe = _make_workflow( - { - "step1": {"action": "confirm", "message": "Delete?", "on_failure": "done"}, - "done": {"action": "stop", "message": "Done."}, - } - ) - errors = validate_recipe(recipe) - assert any("on_success" in e.lower() for e in errors) - - def test_confirm_action_requires_on_failure(self) -> None: - """action: confirm without on_failure is a validation error.""" - recipe = _make_workflow( - { - "step1": {"action": "confirm", "message": "Delete?", "on_success": "done"}, - "done": {"action": "stop", "message": "Done."}, - } - ) - errors = validate_recipe(recipe) - assert any("on_failure" in e.lower() for e in errors) - - def test_confirm_action_valid_step(self) -> None: - """action: confirm with all required fields passes validation.""" - recipe = _make_workflow( - { - "confirm_step": { - "action": "confirm", - "message": "Delete the clone?", - "on_success": "delete_step", - "on_failure": "done", - }, - "delete_step": { - "tool": "remove_clone", - "with": {"clone_path": "/tmp/clone", "keep": "false"}, - "on_success": "done", - "on_failure": "done", - }, - "done": {"action": "stop", "message": "Done."}, - } - ) - errors = validate_recipe(recipe) - assert not errors - - def test_confirm_action_routing_targets_validated(self) -> None: - """on_success and on_failure of a confirm step must name defined steps.""" - recipe = _make_workflow( - { - "confirm_step": { - "action": "confirm", - "message": "Delete?", - "on_success": "nonexistent_step", - "on_failure": "done", - }, - "done": {"action": "stop", "message": "Done."}, - } - ) - errors = validate_recipe(recipe) - assert any("nonexistent_step" in e for e in errors) - - -# --------------------------------------------------------------------------- -# RECIPE-8: constant step type — structural validation -# --------------------------------------------------------------------------- - - -class TestConstantStepValidation: - def test_constant_step_is_valid_discriminator(self) -> None: - """A step with only constant set passes structural validation.""" - recipe = _make_workflow({"step1": {"constant": "main", "on_success": "done"}}) - errors = validate_recipe(recipe) - assert errors == [] - - def test_constant_step_rejected_with_tool(self) -> None: - """constant + tool on same step is rejected (multiple discriminators).""" - recipe = _make_workflow( - {"step1": {"constant": "main", "tool": "run_cmd", "on_success": "done"}} - ) - errors = validate_recipe(recipe) - assert any("multiple discriminators" in e for e in errors) - - def test_constant_step_capture_allows_literal_values(self) -> None: - """constant step capture values may be plain strings (no result.* required).""" - step_data = { - "constant": "main", - "capture": {"merge_target": "main"}, - "on_success": "done", - } - recipe = _make_workflow({"step1": step_data}) - errors = validate_recipe(recipe) - assert errors == [] - - def test_non_constant_step_capture_still_requires_template(self) -> None: - """Non-constant step capture still requires ${{ result.* }} expression.""" - step_data = { - "tool": "run_cmd", - "with": {"command": "echo hi"}, - "capture": {"out": "literal_no_template"}, - "on_success": "done", - } - recipe = _make_workflow({"step1": step_data}) - errors = validate_recipe(recipe) - assert any("result." in e for e in errors) - - def test_step_without_any_discriminator_still_rejected(self) -> None: - """A step with no discriminator (no tool/action/python/constant) is invalid.""" - recipe = _make_workflow({"step1": {"on_success": "done"}}) - errors = validate_recipe(recipe) - assert any("must have" in e for e in errors) diff --git a/tests/recipe/test_validator_dataflow.py b/tests/recipe/test_validator_dataflow.py new file mode 100644 index 0000000000..7a607b3cdb --- /dev/null +++ b/tests/recipe/test_validator_dataflow.py @@ -0,0 +1,286 @@ +"""Tests for recipe dataflow quality analysis and context-ref validation.""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from autoskillit.recipe.io import ( + _parse_recipe, + _parse_step, + builtin_recipes_dir, + iter_steps_with_context, + load_recipe, +) +from autoskillit.recipe.schema import Recipe +from autoskillit.recipe.validator import analyze_dataflow, validate_recipe_structure +from tests.recipe.conftest import _make_workflow, _write_yaml + +pytestmark = [pytest.mark.layer("recipe"), pytest.mark.medium] + + +# --------------------------------------------------------------------------- +# TestDataFlowQuality — migrated from test_recipe_parser.py +# --------------------------------------------------------------------------- + + +class TestDataFlowQuality: + """Tests for data-flow quality analysis (DFQ prefix).""" + + def _make_recipe(self, steps: dict[str, dict]) -> Recipe: + parsed_steps = {name: _parse_step(data) for name, data in steps.items()} + return Recipe( + name="test", + description="test", + steps=parsed_steps, + kitchen_rules=["test"], + ) + + # DFQ1 + def test_analyze_dataflow_returns_report(self) -> None: + from autoskillit.recipe.schema import DataFlowReport + + wf = self._make_recipe( + { + "run": {"tool": "test_check", "on_success": "done"}, + "done": {"action": "stop", "message": "Done"}, + } + ) + report = analyze_dataflow(wf) + assert isinstance(report, DataFlowReport) + assert isinstance(report.warnings, list) + assert isinstance(report.summary, str) + assert report.warnings == [] + assert report.summary is not None + assert len(report.summary) > 0 + + # DFQ2 + def test_dead_output_detected(self) -> None: + wf = self._make_recipe( + { + "impl": { + "tool": "run_skill", + "capture": {"worktree_path": "${{ result.worktree_path }}"}, + "on_success": "finish", + }, + "finish": {"action": "stop", "message": "Done"}, + } + ) + report = analyze_dataflow(wf) + dead = [w for w in report.warnings if w.code == "DEAD_OUTPUT"] + assert len(dead) == 1 + assert dead[0].step_name == "impl" + assert dead[0].field == "worktree_path" + + # DFQ3 + def test_consumed_output_not_flagged(self) -> None: + wf = self._make_recipe( + { + "impl": { + "tool": "run_skill", + "capture": {"worktree_path": "${{ result.worktree_path }}"}, + "on_success": "test", + }, + "test": { + "tool": "test_check", + "with": {"worktree_path": "${{ context.worktree_path }}"}, + "on_success": "done", + }, + "done": {"action": "stop", "message": "Done"}, + } + ) + report = analyze_dataflow(wf) + dead = [w for w in report.warnings if w.code == "DEAD_OUTPUT"] + assert len(dead) == 0 + + # DFQ5 + def test_implicit_handoff_detected(self) -> None: + wf = self._make_recipe( + { + "impl": {"tool": "run_skill", "on_success": "done"}, + "done": {"action": "stop", "message": "Done"}, + } + ) + report = analyze_dataflow(wf) + implicit = [w for w in report.warnings if w.code == "IMPLICIT_HANDOFF"] + assert len(implicit) == 1 + assert implicit[0].step_name == "impl" + + # DFQ6 + def test_non_skill_step_no_implicit_handoff(self) -> None: + wf = self._make_recipe( + { + "test": {"tool": "test_check", "on_success": "done"}, + "done": {"action": "stop", "message": "Done"}, + } + ) + report = analyze_dataflow(wf) + implicit = [w for w in report.warnings if w.code == "IMPLICIT_HANDOFF"] + assert len(implicit) == 0 + + # DFQ11 + def test_summary_reports_counts(self) -> None: + wf = self._make_recipe( + { + "impl": { + "tool": "run_skill", + "capture": {"worktree_path": "${{ result.worktree_path }}"}, + "on_success": "run", + }, + "run": {"tool": "run_skill", "on_success": "done"}, + "done": {"action": "stop", "message": "Done"}, + } + ) + report = analyze_dataflow(wf) + assert "2 data-flow warnings" in report.summary + + # DFQ13 + def test_bundled_recipes_produce_reports(self) -> None: + from autoskillit.recipe.schema import DataFlowReport + + wf_dir = builtin_recipes_dir() + yaml_files = list(wf_dir.glob("*.yaml")) + list(wf_dir.glob("*.yml")) + assert len(yaml_files) > 0 + for yaml_file in yaml_files: + wf = load_recipe(yaml_file) + report = analyze_dataflow(wf) + assert isinstance(report, DataFlowReport) + assert isinstance(report.warnings, list) + + +# --------------------------------------------------------------------------- +# iter_steps_with_context integration +# --------------------------------------------------------------------------- + + +def test_validate_recipe_uses_iter_steps_with_context_for_capture_refs(tmp_path: Path) -> None: + """validate_recipe catches context refs not captured by preceding steps.""" + data = { + "name": "ctx-test", + "description": "Context validation test", + "kitchen_rules": ["test"], + "steps": { + "step1": { + "tool": "run_cmd", + "with": {"cmd": "echo hello"}, + "on_success": "step2", + }, + "step2": { + "tool": "test_check", + "with": {"worktree_path": "${{ context.worktree_path }}"}, + "on_success": "done", + }, + "done": {"action": "stop", "message": "ok"}, + }, + } + wf = load_recipe(_write_yaml(tmp_path / "recipe.yaml", data)) + # step1 has no captures, so step2 should see empty context + steps = list(iter_steps_with_context(wf)) + assert steps[1][2] == frozenset() + # validate_recipe should catch the unsatisfied context reference + errors = validate_recipe_structure(wf) + assert any("worktree_path" in e for e in errors) + + +# --------------------------------------------------------------------------- +# isinstance guard tests (T_GD1, T_GV1) +# --------------------------------------------------------------------------- + + +class TestIsInstanceGuards: + def test_gd1_analyze_dataflow_no_raise_on_bool_with_arg(self) -> None: + """T_GD1: _detect_dead_outputs does not raise TypeError for boolean with_args.""" + data = { + "name": "bool-guard", + "description": "test", + "kitchen_rules": ["test"], + "steps": { + "plan": { + "tool": "run_skill", + "with": {"skill_command": "/autoskillit:make-plan do the task"}, + "capture": {"plan_path": "${{ result.plan_path }}"}, + "on_success": "downstream", + }, + "downstream": { + "tool": "run_cmd", + "with": {"worktree_path": True}, + "on_success": "done", + }, + "done": {"action": "stop", "message": "Done."}, + }, + } + recipe = _parse_recipe(data) + report = analyze_dataflow(recipe) + assert report is not None + + def test_gv1_validate_recipe_no_raise_on_bool_with_arg(self) -> None: + """T_GV1: validate_recipe does not raise TypeError for boolean with_args.""" + data = { + "name": "bool-guard-validate", + "description": "test", + "kitchen_rules": ["test"], + "steps": { + "step1": { + "tool": "run_cmd", + "with": {"flag": True}, + "on_success": "done", + }, + "done": {"action": "stop", "message": "Done."}, + }, + } + recipe = _parse_recipe(data) + result = validate_recipe_structure(recipe) + assert isinstance(result, list) + assert not any("dead" in str(f).lower() for f in result), ( + "DEAD_OUTPUT rule fired on a valid recipe — false positive" + ) + + +# --------------------------------------------------------------------------- +# on_result self-consumption tests (T_OR1, T_OR2) +# --------------------------------------------------------------------------- + + +class TestOnResultConsumption: + def test_or1_on_result_field_is_not_dead_output(self) -> None: + """T_OR1: verdict captured and used as on_result.field is NOT flagged DEAD_OUTPUT.""" + steps = { + "audit_impl": { + "tool": "run_skill", + "with": { + "skill_command": "/autoskillit:audit-impl plan.md myref main", + }, + "capture": {"verdict": "${{ result.verdict }}"}, + "on_result": { + "field": "verdict", + "routes": {"GO": "done", "NO GO": "done"}, + }, + }, + "done": {"action": "stop", "message": "Done."}, + } + recipe = _make_workflow(steps) + report = analyze_dataflow(recipe) + dead = [w for w in report.warnings if w.code == "DEAD_OUTPUT" and w.field == "verdict"] + assert dead == [] + + def test_or2_different_on_result_field_flags_dead_output(self) -> None: + """T_OR2: verdict is flagged DEAD_OUTPUT when on_result.field is a different key.""" + steps = { + "audit_impl": { + "tool": "run_skill", + "with": { + "skill_command": "/autoskillit:audit-impl plan.md myref main", + }, + "capture": {"verdict": "${{ result.verdict }}"}, + "on_result": { + "field": "restart_scope", + "routes": {"full_restart": "done", "partial_restart": "done"}, + }, + }, + "done": {"action": "stop", "message": "Done."}, + } + recipe = _make_workflow(steps) + report = analyze_dataflow(recipe) + dead = [w for w in report.warnings if w.code == "DEAD_OUTPUT" and w.field == "verdict"] + assert len(dead) == 1 diff --git a/tests/recipe/test_validator_graph_and_actions.py b/tests/recipe/test_validator_graph_and_actions.py new file mode 100644 index 0000000000..274d064c23 --- /dev/null +++ b/tests/recipe/test_validator_graph_and_actions.py @@ -0,0 +1,389 @@ +"""Tests for step graph construction, ValidationContext, and action-type validation.""" + +from __future__ import annotations + +import pytest +import yaml + +from autoskillit.recipe.io import _parse_recipe +from autoskillit.recipe.schema import Recipe, RecipeIngredient, RecipeStep +from autoskillit.recipe.validator import run_semantic_rules, validate_recipe_structure +from tests.recipe.conftest import _make_workflow + +pytestmark = [pytest.mark.layer("recipe"), pytest.mark.medium] + + +# --------------------------------------------------------------------------- +# skip_when_false bypass edge tests +# --------------------------------------------------------------------------- + + +def test_build_step_graph_adds_bypass_edges_for_skip_when_false() -> None: + """When a step has skip_when_false, predecessors get a direct edge to its on_success.""" + from autoskillit.recipe.validator import _build_step_graph + + # Recipe: entry → optional_step (skip_when_false) → final_step + # Expected bypass edge: entry → final_step + recipe = Recipe( + name="test", + description="test", + steps={ + "entry": RecipeStep(tool="run_cmd", on_success="optional_step"), + "optional_step": RecipeStep( + tool="run_skill", + on_success="final_step", + skip_when_false="inputs.flag", + ), + "final_step": RecipeStep(action="stop", message="done"), + }, + ingredients={"flag": RecipeIngredient(description="", required=False, default="true")}, + kitchen_rules=["test"], + ) + graph = _build_step_graph(recipe) + # Bypass edge: entry can reach final_step directly (skip of optional_step) + assert "final_step" in graph["entry"] + + +def test_build_step_graph_bypass_does_not_remove_normal_edge() -> None: + """The bypass edge is additional, not a replacement for the normal edge.""" + from autoskillit.recipe.validator import _build_step_graph + + recipe = Recipe( + name="test", + description="test", + steps={ + "entry": RecipeStep(tool="run_cmd", on_success="optional_step"), + "optional_step": RecipeStep( + tool="run_skill", + on_success="final_step", + skip_when_false="inputs.flag", + ), + "final_step": RecipeStep(action="stop", message="done"), + }, + ingredients={"flag": RecipeIngredient(description="", required=False, default="true")}, + kitchen_rules=["test"], + ) + graph = _build_step_graph(recipe) + # Normal edge still present + assert "optional_step" in graph["entry"] + # Bypass edge also present + assert "final_step" in graph["entry"] + + +# --------------------------------------------------------------------------- +# _build_step_graph predicate on_result tests +# --------------------------------------------------------------------------- + + +class TestPredicateBuildStepGraph: + """_build_step_graph includes condition.route edges.""" + + def test_build_step_graph_includes_condition_routes(self) -> None: + """_build_step_graph produces edges for condition.route targets.""" + from autoskillit.recipe.validator import _build_step_graph + + wf = _make_workflow( + { + "merge": { + "tool": "merge_worktree", + "with": {"worktree_path": "/tmp/wt", "base_branch": "main"}, + "on_result": [ + {"when": "result.failed_step == 'test_gate'", "route": "assess"}, + {"when": "result.error", "route": "cleanup"}, + {"route": "push"}, + ], + "capture": {"cleanup_succeeded": "${{ result.cleanup_succeeded }}"}, + }, + "assess": {"action": "stop", "message": "Assess."}, + "cleanup": {"action": "stop", "message": "Cleanup."}, + "push": {"action": "stop", "message": "Push."}, + } + ) + graph = _build_step_graph(wf) + assert "assess" in graph["merge"] + assert "cleanup" in graph["merge"] + assert "push" in graph["merge"] + + +# --------------------------------------------------------------------------- +# ValidationContext tests +# --------------------------------------------------------------------------- + + +def test_run_semantic_rules_builds_step_graph_exactly_once(monkeypatch): + """_build_step_graph is called only once regardless of how many rules need it.""" + from autoskillit.recipe import _analysis + + call_count = [] + real_fn = _analysis._build_step_graph + + def counting_fn(recipe): + call_count.append(1) + return real_fn(recipe) + + monkeypatch.setattr(_analysis, "_build_step_graph", counting_fn) + + recipe = _parse_recipe( + { + "name": "test", + "description": "test", + "autoskillit_version": "0.2.0", + "kitchen_rules": ["Never use native tools"], + "steps": {"stop": {"action": "stop", "message": "done"}}, + } + ) + run_semantic_rules(recipe) + + assert len(call_count) == 1 + + +def test_run_semantic_rules_calls_analyze_dataflow_exactly_once(monkeypatch): + """analyze_dataflow is called only once regardless of how many rules consume it.""" + from autoskillit.recipe import _analysis + + call_count = [] + real_fn = _analysis.analyze_dataflow + + def counting_fn(recipe, **kwargs): + call_count.append(1) + return real_fn(recipe, **kwargs) + + monkeypatch.setattr(_analysis, "analyze_dataflow", counting_fn) + + recipe = _parse_recipe( + { + "name": "test", + "description": "test", + "autoskillit_version": "0.2.0", + "kitchen_rules": ["Never use native tools"], + "steps": {"stop": {"action": "stop", "message": "done"}}, + } + ) + run_semantic_rules(recipe) + + assert len(call_count) == 1 + + +def test_validation_context_exposes_recipe_step_graph_and_dataflow(): + """ValidationContext contains recipe, step_graph, and dataflow attributes.""" + from autoskillit.recipe._analysis import ValidationContext, make_validation_context + + recipe = _parse_recipe( + { + "name": "test", + "description": "test", + "autoskillit_version": "0.2.0", + "kitchen_rules": ["Never use native tools"], + "steps": {"stop": {"action": "stop", "message": "done"}}, + } + ) + ctx = make_validation_context(recipe) + + assert isinstance(ctx, ValidationContext) + assert ctx.recipe is recipe + assert isinstance(ctx.step_graph, dict) + assert hasattr(ctx.dataflow, "warnings") + + +def test_analyze_dataflow_accepts_prebuilt_step_graph(): + """analyze_dataflow can reuse a pre-built step_graph to avoid duplicate computation.""" + from autoskillit.recipe._analysis import _build_step_graph, analyze_dataflow + + recipe = _parse_recipe( + { + "name": "test", + "description": "test", + "autoskillit_version": "0.2.0", + "kitchen_rules": ["Never use native tools"], + "steps": { + "run": { + "tool": "run_skill", + "capture": {"wp": "${{ result.worktree_path }}"}, + "on_success": "use_it", + }, + "use_it": { + "tool": "test_check", + "with": {"worktree_path": "${{ context.wp }}"}, + "on_success": "done", + }, + "done": {"action": "stop", "message": "done"}, + }, + } + ) + graph = _build_step_graph(recipe) + report1 = analyze_dataflow(recipe, step_graph=graph) + report2 = analyze_dataflow(recipe) + assert report1.warnings == report2.warnings + + +def test_validate_recipe_catches_rectify_with_missing_investigation_path_capture() -> None: + """1h: validate_recipe catches missing investigation_path capture before rectify step. + + When rectify is invoked with ${{ context.investigation_path }} but no preceding step + captures investigation_path, the existing undeclared-context-reference validator must + return an error — confirming the enforcement layer works for the new context variable. + """ + recipe = _parse_recipe( + yaml.safe_load( + """\ +name: test-missing-capture +description: Recipe with investigate→rectify but no capture block +kitchen_rules: + - test +steps: + investigate: + tool: run_skill + with: + skill_command: "/autoskillit:investigate the bug" + on_success: rectify + on_failure: done + rectify: + tool: run_skill + with: + skill_command: "/autoskillit:rectify ${{ context.investigation_path }}" + on_success: done + on_failure: done + done: + action: stop + message: Done. +""" + ) + ) + errors = validate_recipe_structure(recipe) + assert any("investigation_path" in e for e in errors), ( + "validate_recipe_structure must report an error about undeclared context reference " + "'investigation_path' when the investigate step has no capture block" + ) + + +# --------------------------------------------------------------------------- +# TestConfirmAction — action: "confirm" validation +# --------------------------------------------------------------------------- + + +class TestConfirmAction: + def test_confirm_action_requires_message(self) -> None: + """action: confirm without message is a validation error.""" + recipe = _make_workflow( + { + "step1": {"action": "confirm", "on_success": "done", "on_failure": "done"}, + "done": {"action": "stop", "message": "Done."}, + } + ) + errors = validate_recipe_structure(recipe) + assert any("message" in e.lower() for e in errors), ( + "Expected a validation error mentioning 'message'" + ) + assert any("confirm" in e.lower() for e in errors), ( + "Expected a validation error mentioning 'confirm'" + ) + + def test_confirm_action_requires_on_success(self) -> None: + """action: confirm without on_success is a validation error.""" + recipe = _make_workflow( + { + "step1": {"action": "confirm", "message": "Delete?", "on_failure": "done"}, + "done": {"action": "stop", "message": "Done."}, + } + ) + errors = validate_recipe_structure(recipe) + assert any("on_success" in e.lower() for e in errors) + + def test_confirm_action_requires_on_failure(self) -> None: + """action: confirm without on_failure is a validation error.""" + recipe = _make_workflow( + { + "step1": {"action": "confirm", "message": "Delete?", "on_success": "done"}, + "done": {"action": "stop", "message": "Done."}, + } + ) + errors = validate_recipe_structure(recipe) + assert any("on_failure" in e.lower() for e in errors) + + def test_confirm_action_valid_step(self) -> None: + """action: confirm with all required fields passes validation.""" + recipe = _make_workflow( + { + "confirm_step": { + "action": "confirm", + "message": "Delete the clone?", + "on_success": "delete_step", + "on_failure": "done", + }, + "delete_step": { + "tool": "remove_clone", + "with": {"clone_path": "/tmp/clone", "keep": "false"}, + "on_success": "done", + "on_failure": "done", + }, + "done": {"action": "stop", "message": "Done."}, + } + ) + errors = validate_recipe_structure(recipe) + assert not errors + + def test_confirm_action_routing_targets_validated(self) -> None: + """on_success and on_failure of a confirm step must name defined steps.""" + recipe = _make_workflow( + { + "confirm_step": { + "action": "confirm", + "message": "Delete?", + "on_success": "nonexistent_step", + "on_failure": "done", + }, + "done": {"action": "stop", "message": "Done."}, + } + ) + errors = validate_recipe_structure(recipe) + assert any("nonexistent_step" in e for e in errors) + + +# --------------------------------------------------------------------------- +# RECIPE-8: constant step type — structural validation +# --------------------------------------------------------------------------- + + +class TestConstantStepValidation: + def test_constant_step_is_valid_discriminator(self) -> None: + """A step with only constant set passes structural validation.""" + recipe = _make_workflow({"step1": {"constant": "main", "on_success": "done"}}) + errors = validate_recipe_structure(recipe) + assert errors == [] + + def test_constant_step_rejected_with_tool(self) -> None: + """constant + tool on same step is rejected (multiple discriminators).""" + recipe = _make_workflow( + {"step1": {"constant": "main", "tool": "run_cmd", "on_success": "done"}} + ) + errors = validate_recipe_structure(recipe) + assert any("multiple discriminators" in e for e in errors) + + def test_constant_step_capture_allows_literal_values(self) -> None: + """constant step capture values may be plain strings (no result.* required).""" + step_data = { + "constant": "main", + "capture": {"merge_target": "main"}, + "on_success": "done", + } + recipe = _make_workflow({"step1": step_data}) + errors = validate_recipe_structure(recipe) + assert errors == [] + + def test_non_constant_step_capture_still_requires_template(self) -> None: + """Non-constant step capture still requires ${{ result.* }} expression.""" + step_data = { + "tool": "run_cmd", + "with": {"command": "echo hi"}, + "capture": {"out": "literal_no_template"}, + "on_success": "done", + } + recipe = _make_workflow({"step1": step_data}) + errors = validate_recipe_structure(recipe) + assert any("result." in e for e in errors) + + def test_step_without_any_discriminator_still_rejected(self) -> None: + """A step with no discriminator (no tool/action/python/constant) is invalid.""" + recipe = _make_workflow({"step1": {"on_success": "done"}}) + errors = validate_recipe_structure(recipe) + assert any("must have" in e for e in errors) diff --git a/tests/recipe/test_validator_skill_hints.py b/tests/recipe/test_validator_skill_hints.py new file mode 100644 index 0000000000..21cc9750bc --- /dev/null +++ b/tests/recipe/test_validator_skill_hints.py @@ -0,0 +1,101 @@ +"""Tests: validate_recipe errors include /autoskillit:write-recipe skill hints.""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from autoskillit.recipe.io import load_recipe +from autoskillit.recipe.validator import validate_recipe_structure +from tests.recipe.conftest import VALID_RECIPE, _write_yaml + +pytestmark = [pytest.mark.layer("recipe"), pytest.mark.medium] + +_HINT = "write-recipe" + + +class TestValidatorSkillHints: + def test_missing_name_error_includes_hint(self, tmp_path: Path) -> None: + """'Recipe must have a name' error includes write-recipe hint.""" + data = {**VALID_RECIPE, "name": ""} + wf = load_recipe(_write_yaml(tmp_path / "r.yaml", data)) + errors = validate_recipe_structure(wf) + name_errors = [e for e in errors if "name" in e.lower() and "must have" in e.lower()] + assert name_errors, f"Expected a name-required error, got: {errors}" + assert any(_HINT in e for e in name_errors), ( + f"Expected write-recipe hint in name error, got: {name_errors}" + ) + + def test_missing_steps_error_includes_hint(self, tmp_path: Path) -> None: + """'Recipe must have at least one step' error includes write-recipe hint.""" + data = {"name": "no-steps", "description": "test", "kitchen_rules": ["rule"]} + wf = load_recipe(_write_yaml(tmp_path / "r.yaml", data)) + errors = validate_recipe_structure(wf) + step_errors = [e for e in errors if "step" in e.lower() and "at least" in e.lower()] + assert step_errors, f"Expected a steps-required error, got: {errors}" + assert any(_HINT in e for e in step_errors), ( + f"Expected write-recipe hint in steps error, got: {step_errors}" + ) + + def test_missing_kitchen_rules_error_includes_hint(self, tmp_path: Path) -> None: + """'Recipe has no kitchen_rules' error includes write-recipe hint.""" + data = {k: v for k, v in VALID_RECIPE.items() if k != "kitchen_rules"} + wf = load_recipe(_write_yaml(tmp_path / "r.yaml", data)) + errors = validate_recipe_structure(wf) + kr_errors = [e for e in errors if "kitchen_rules" in e] + assert kr_errors, f"Expected a kitchen_rules error, got: {errors}" + assert any(_HINT in e for e in kr_errors), ( + f"Expected write-recipe hint in kitchen_rules error, got: {kr_errors}" + ) + + def test_capture_no_template_error_includes_hint(self, tmp_path: Path) -> None: + """Capture value with no result template expression includes write-recipe hint.""" + data = { + **VALID_RECIPE, + "steps": { + "step1": { + "tool": "run_cmd", + "with": {"cmd": "echo hi"}, + "capture": {"my_output": "no_template_here"}, + "on_success": "done", + }, + "done": {"action": "stop", "message": "Done."}, + }, + } + wf = load_recipe(_write_yaml(tmp_path / "r.yaml", data)) + errors = validate_recipe_structure(wf) + cap_errors = [e for e in errors if "capture" in e and "result" in e] + assert cap_errors, f"Expected a capture expression error, got: {errors}" + assert any(_HINT in e for e in cap_errors), ( + f"Expected write-recipe hint in capture error, got: {cap_errors}" + ) + + def test_undeclared_input_error_includes_hint(self, tmp_path: Path) -> None: + """Undeclared input reference error includes write-recipe hint.""" + data = { + "name": "test", + "description": "test", + "kitchen_rules": ["rule"], + "steps": { + "step1": { + "tool": "run_cmd", + "with": {"cmd": "${{ inputs.nonexistent_input }}"}, + "on_success": "done", + }, + "done": {"action": "stop", "message": "Done."}, + }, + } + wf = load_recipe(_write_yaml(tmp_path / "r.yaml", data)) + errors = validate_recipe_structure(wf) + input_errors = [e for e in errors if "undeclared input" in e] + assert input_errors, f"Expected an undeclared-input error, got: {errors}" + assert any(_HINT in e for e in input_errors), ( + f"Expected write-recipe hint in undeclared-input error, got: {input_errors}" + ) + + def test_valid_recipe_has_no_hint_errors(self, tmp_path: Path) -> None: + """A valid recipe produces zero validation errors (sanity check).""" + wf = load_recipe(_write_yaml(tmp_path / "r.yaml", VALID_RECIPE)) + errors = validate_recipe_structure(wf) + assert errors == [], f"Valid recipe should have no errors, got: {errors}" diff --git a/tests/recipe/test_validator_structural.py b/tests/recipe/test_validator_structural.py new file mode 100644 index 0000000000..6663dfc68a --- /dev/null +++ b/tests/recipe/test_validator_structural.py @@ -0,0 +1,429 @@ +"""Tests for recipe structural validation — TestValidateRecipe.""" + +from __future__ import annotations + +import dataclasses +from pathlib import Path + +import pytest + +from autoskillit.recipe.io import ( + _parse_recipe, + builtin_recipes_dir, + load_recipe, +) +from autoskillit.recipe.schema import Recipe, RecipeStep +from autoskillit.recipe.validator import validate_recipe_structure +from tests.recipe.conftest import VALID_RECIPE, _write_yaml + +pytestmark = [pytest.mark.layer("recipe"), pytest.mark.medium] + + +# --------------------------------------------------------------------------- +# TestValidateRecipe — migrated from test_recipe_parser.py +# --------------------------------------------------------------------------- + + +class TestValidateRecipe: + def test_valid_recipe_no_errors(self, tmp_path: Path) -> None: + wf = load_recipe(_write_yaml(tmp_path / "recipe.yaml", VALID_RECIPE)) + errors = validate_recipe_structure(wf) + assert errors == [] + + # WF2 + def test_recipe_requires_name(self, tmp_path: Path) -> None: + data = {**VALID_RECIPE, "name": ""} + f = _write_yaml(tmp_path / "recipe.yaml", data) + wf = load_recipe(f) + errors = validate_recipe_structure(wf) + assert any("name" in e.lower() for e in errors) + + # WF3 + def test_recipe_requires_steps(self, tmp_path: Path) -> None: + data = {"name": "no-steps", "description": "Missing steps", "kitchen_rules": ["test"]} + f = _write_yaml(tmp_path / "recipe.yaml", data) + wf = load_recipe(f) + errors = validate_recipe_structure(wf) + assert any("step" in e.lower() for e in errors) + + # WF5 + def test_goto_targets_validated(self, tmp_path: Path) -> None: + data = { + "name": "bad-goto", + "description": "Invalid goto", + "kitchen_rules": ["test"], + "steps": { + "start": {"tool": "run_cmd", "on_success": "nonexistent"}, + "end": {"action": "stop", "message": "Done."}, + }, + } + f = _write_yaml(tmp_path / "recipe.yaml", data) + wf = load_recipe(f) + errors = validate_recipe_structure(wf) + assert any("nonexistent" in e for e in errors) + + # WF6 + def test_builtin_recipes_valid(self) -> None: + bd = builtin_recipes_dir() + yamls = list(bd.glob("*.yaml")) + assert len(yamls) >= 4 + for f in yamls: + wf = load_recipe(f) + errors = validate_recipe_structure(wf) + assert errors == [], f"Validation errors in {f.name}: {errors}" + + # WF10 + def test_terminal_step_has_message(self, tmp_path: Path) -> None: + data = { + "name": "no-msg", + "description": "Terminal without message", + "kitchen_rules": ["test"], + "steps": {"end": {"action": "stop"}}, + } + f = _write_yaml(tmp_path / "recipe.yaml", data) + wf = load_recipe(f) + errors = validate_recipe_structure(wf) + assert any("message" in e.lower() for e in errors) + + def test_step_needs_tool_or_action(self, tmp_path: Path) -> None: + data = { + "name": "bad-step", + "description": "Neither tool nor action", + "kitchen_rules": ["test"], + "steps": {"empty": {"note": "just a note"}}, + } + f = _write_yaml(tmp_path / "recipe.yaml", data) + wf = load_recipe(f) + errors = validate_recipe_structure(wf) + assert any("tool" in e and "action" in e for e in errors) + + def test_input_reference_validation(self, tmp_path: Path) -> None: + data = { + "name": "bad-ref", + "description": "References undeclared input", + "kitchen_rules": ["test"], + "steps": { + "run": {"tool": "run_cmd", "with": {"cmd": "${{ inputs.missing_input }}"}}, + }, + } + f = _write_yaml(tmp_path / "recipe.yaml", data) + wf = load_recipe(f) + errors = validate_recipe_structure(wf) + assert any("missing_input" in e for e in errors) + + def test_retry_block_raises_on_load(self, tmp_path: Path) -> None: + """The old retry: block is no longer supported — loading raises ValueError.""" + import pytest + + data = { + "name": "bad-retry-block", + "description": "Old retry block is unsupported", + "kitchen_rules": ["test"], + "steps": { + "impl": { + "tool": "run_skill", + "retry": { + "max_attempts": 3, + "on": "needs_retry", + "on_exhausted": "fail", + }, + }, + "fail": {"action": "stop", "message": "Failed."}, + }, + } + f = _write_yaml(tmp_path / "recipe.yaml", data) + with pytest.raises(ValueError, match="retry.*no longer supported"): + load_recipe(f) + + def test_step_rejects_both_python_and_tool(self, tmp_path: Path) -> None: + data = { + "name": "bad", + "description": "Both python and tool", + "kitchen_rules": ["test"], + "steps": {"run": {"python": "mod.fn", "tool": "run_cmd"}}, + } + wf = load_recipe(_write_yaml(tmp_path / "recipe.yaml", data)) + errors = validate_recipe_structure(wf) + assert any("python" in e and "tool" in e for e in errors) + + def test_python_step_requires_dotted_path(self, tmp_path: Path) -> None: + data = { + "name": "bad-path", + "description": "No dot", + "kitchen_rules": ["test"], + "steps": {"check": {"python": "bare_name"}}, + } + wf = load_recipe(_write_yaml(tmp_path / "recipe.yaml", data)) + errors = validate_recipe_structure(wf) + assert any("dotted" in e.lower() or "module" in e.lower() for e in errors) + + # CAP3 + def test_capture_result_refs_valid(self, tmp_path: Path) -> None: + data = { + "name": "cap-valid", + "description": "Valid captures", + "kitchen_rules": ["test"], + "steps": { + "run": { + "tool": "run_skill", + "capture": { + "wp": "${{ result.worktree_path }}", + "ctx": "${{ result.failure_context }}", + }, + }, + "done": {"action": "stop", "message": "ok"}, + }, + } + wf = load_recipe(_write_yaml(tmp_path / "recipe.yaml", data)) + errors = validate_recipe_structure(wf) + assert not any("capture" in e for e in errors) + + # CAP4 + def test_capture_non_result_namespace_rejected(self, tmp_path: Path) -> None: + data = { + "name": "cap-bad-ns", + "description": "Bad namespace", + "kitchen_rules": ["test"], + "steps": { + "run": {"tool": "run_cmd", "capture": {"foo": "${{ inputs.bar }}"}}, + "done": {"action": "stop", "message": "ok"}, + }, + } + wf = load_recipe(_write_yaml(tmp_path / "recipe.yaml", data)) + errors = validate_recipe_structure(wf) + assert any("result" in e and "capture" in e for e in errors) + + # CAP5 + def test_capture_literal_value_rejected(self, tmp_path: Path) -> None: + data = { + "name": "cap-literal", + "description": "Literal capture", + "kitchen_rules": ["test"], + "steps": { + "run": {"tool": "run_cmd", "capture": {"foo": "literal string"}}, + "done": {"action": "stop", "message": "ok"}, + }, + } + wf = load_recipe(_write_yaml(tmp_path / "recipe.yaml", data)) + errors = validate_recipe_structure(wf) + assert any("capture" in e and "result" in e for e in errors) + + # CAP6 + def test_context_ref_to_captured_var_valid(self, tmp_path: Path) -> None: + data = { + "name": "ctx-valid", + "description": "Valid context ref", + "kitchen_rules": ["test"], + "steps": { + "impl": { + "tool": "run_skill", + "capture": {"worktree_path": "${{ result.worktree_path }}"}, + "on_success": "test", + }, + "test": { + "tool": "test_check", + "with": {"worktree_path": "${{ context.worktree_path }}"}, + "on_success": "done", + }, + "done": {"action": "stop", "message": "ok"}, + }, + } + wf = load_recipe(_write_yaml(tmp_path / "recipe.yaml", data)) + errors = validate_recipe_structure(wf) + assert not any("context" in e for e in errors) + + # CAP7 + def test_context_ref_to_uncaptured_var_rejected(self, tmp_path: Path) -> None: + data = { + "name": "ctx-bad", + "description": "Uncaptured ref", + "kitchen_rules": ["test"], + "steps": { + "test": { + "tool": "test_check", + "with": {"worktree_path": "${{ context.nonexistent }}"}, + }, + "done": {"action": "stop", "message": "ok"}, + }, + } + wf = load_recipe(_write_yaml(tmp_path / "recipe.yaml", data)) + errors = validate_recipe_structure(wf) + assert any("nonexistent" in e and "context" in e for e in errors) + + # CAP8 + def test_context_forward_reference_rejected(self, tmp_path: Path) -> None: + data = { + "name": "ctx-fwd", + "description": "Forward ref", + "kitchen_rules": ["test"], + "steps": { + "check": { + "tool": "test_check", + "with": {"worktree_path": "${{ context.wp }}"}, + "on_success": "done", + }, + "produce": { + "tool": "run_skill", + "capture": {"wp": "${{ result.worktree_path }}"}, + }, + "done": {"action": "stop", "message": "ok"}, + }, + } + wf = load_recipe(_write_yaml(tmp_path / "recipe.yaml", data)) + errors = validate_recipe_structure(wf) + assert any("wp" in e and "context" in e for e in errors) + + # CON1 + def test_recipe_schema_supports_kitchen_rules(self) -> None: + field_names = {f.name for f in dataclasses.fields(Recipe)} + assert "kitchen_rules" in field_names + + # CON3 + def test_validate_recipe_warns_missing_kitchen_rules(self, tmp_path: Path) -> None: + data = {**VALID_RECIPE} + data.pop("kitchen_rules", None) + wf = load_recipe(_write_yaml(tmp_path / "recipe.yaml", data)) + errors = validate_recipe_structure(wf) + warnings = [e for e in errors if "kitchen_rules" in e.lower()] + assert warnings + + # CON4 + def test_bundled_recipes_have_kitchen_rules(self) -> None: + wf_dir = builtin_recipes_dir() + failures = [] + for path in sorted(wf_dir.glob("*.yaml")): + wf = load_recipe(path) + if not wf.kitchen_rules: + failures.append(f"{path.name}: missing kitchen_rules") + assert not failures + + # T_OR2 + def test_on_result_and_on_success_mutually_exclusive(self, tmp_path: Path) -> None: + data = { + "name": "conflict-recipe", + "description": "Both on_result and on_success", + "kitchen_rules": ["test"], + "steps": { + "classify": { + "tool": "classify_fix", + "on_result": { + "field": "restart_scope", + "routes": {"full_restart": "done"}, + }, + "on_success": "done", + "on_failure": "escalate", + }, + "done": {"action": "stop", "message": "Done."}, + "escalate": {"action": "stop", "message": "Escalating."}, + }, + } + f = _write_yaml(tmp_path / "recipe.yaml", data) + wf = load_recipe(f) + errors = validate_recipe_structure(wf) + assert any("on_result" in e and "on_success" in e for e in errors) + + # T_OR6 + def test_on_result_route_done_is_valid(self, tmp_path: Path) -> None: + data = { + "name": "done-route-recipe", + "description": "Route to done", + "kitchen_rules": ["test"], + "steps": { + "classify": { + "tool": "classify_fix", + "on_result": { + "field": "restart_scope", + "routes": {"full_restart": "done", "partial_restart": "done"}, + }, + "on_failure": "escalate", + }, + "escalate": {"action": "stop", "message": "Escalating."}, + }, + } + f = _write_yaml(tmp_path / "recipe.yaml", data) + wf = load_recipe(f) + errors = validate_recipe_structure(wf) + assert errors == [] + + # VER3 + def test_version_does_not_cause_validation_errors(self) -> None: + data = { + "name": "version-test-recipe", + "description": "A recipe for testing the version field", + "kitchen_rules": ["Only use AutoSkillit MCP tools during pipeline execution"], + "steps": { + "do_it": {"tool": "run_cmd", "on_success": "done"}, + "done": {"action": "stop", "message": "Done."}, + }, + "autoskillit_version": "0.2.0", + } + wf = _parse_recipe(data) + errors = validate_recipe_structure(wf) + assert errors == [] + + def test_validator_rejects_zero_stale_threshold(self) -> None: + recipe = Recipe( + name="test", + description="test", + steps={"s": RecipeStep(tool="run_skill", on_success="done", stale_threshold=0)}, + kitchen_rules=["test"], + ) + errors = validate_recipe_structure(recipe) + assert any("stale_threshold" in e for e in errors) + + def test_validator_rejects_negative_stale_threshold(self) -> None: + recipe = Recipe( + name="test", + description="test", + steps={"s": RecipeStep(tool="run_skill", on_success="done", stale_threshold=-1)}, + kitchen_rules=["test"], + ) + errors = validate_recipe_structure(recipe) + assert any("stale_threshold" in e for e in errors) + + def test_validator_accepts_positive_stale_threshold(self) -> None: + recipe = Recipe( + name="test", + description="test", + steps={"s": RecipeStep(tool="run_skill", on_success="done", stale_threshold=2400)}, + kitchen_rules=["test"], + ) + errors = validate_recipe_structure(recipe) + assert not any("stale_threshold" in e for e in errors) + + def test_validator_rejects_negative_idle_output_timeout(self) -> None: + recipe = Recipe( + name="test", + description="test", + steps={"s": RecipeStep(tool="run_skill", on_success="done", idle_output_timeout=-1)}, + kitchen_rules=["test"], + ) + errors = validate_recipe_structure(recipe) + assert any("idle_output_timeout" in e for e in errors) + + def test_validator_accepts_zero_idle_output_timeout(self) -> None: + # 0 = disabled, must NOT be rejected + recipe = Recipe( + name="test", + description="test", + steps={"s": RecipeStep(tool="run_skill", on_success="done", idle_output_timeout=0)}, + kitchen_rules=["test"], + ) + errors = validate_recipe_structure(recipe) + assert not any("idle_output_timeout" in e for e in errors) + + def test_validator_accepts_positive_idle_output_timeout(self) -> None: + recipe = Recipe( + name="test", + description="test", + steps={"s": RecipeStep(tool="run_skill", on_success="done", idle_output_timeout=120)}, + kitchen_rules=["test"], + ) + errors = validate_recipe_structure(recipe) + assert not any("idle_output_timeout" in e for e in errors) + + +def test_validate_recipe_structure_importable() -> None: + from autoskillit.recipe import validate_recipe_structure + + assert callable(validate_recipe_structure) diff --git a/tests/server/CLAUDE.md b/tests/server/CLAUDE.md new file mode 100644 index 0000000000..f7852a7eae --- /dev/null +++ b/tests/server/CLAUDE.md @@ -0,0 +1,96 @@ +# server/ + +Server tool handler unit tests — kitchen, execution, CI, clone, workspace tools. + +## Files + +| File | Purpose | +|------|---------| +| `__init__.py` | empty | +| `_helpers.py` | Shared test builder utilities for tests/server/ | +| `_type_coercion_fixtures.py` | Test fixtures for _import_and_call annotation-aware type coercion | +| `conftest.py` | Shared fixtures for tests/server/ | +| `test_editable_guard.py` | Unit tests for server/_editable_guard.py — scan_editable_installs_for_worktree | +| `test_factory.py` | Tests for server/_factory.py make_context() composition root | +| `test_factory_recording.py` | Tests for make_context recording/replay runner wiring | +| `test_git.py` | Tests for server/git.py perform_merge() | +| `test_git_merge_dirty_check.py` | Tests for the pre-merge dirty check in perform_merge (Layer 3) | +| `test_guards_module.py` | Smoke test: all 6 guards are importable from _guards | +| `test_headless_session.py` | Headless session kitchen visibility via AUTOSKILLIT_HEADLESS=1 | +| `test_helpers_gate.py` | Contract tests: server helpers gate response schema | +| `test_helpers_tier_guards.py` | Tests for tier-aware guard helpers in server._guards | +| `test_kitchen_lifecycle.py` | Kitchen lifecycle tests | +| `test_lifespan.py` | Tests that the FastMCP lifespan calls recorder.finalize() on server shutdown | +| `test_lifespan_readiness_structural.py` | AST structural guard for _autoskillit_lifespan readiness invariants | +| `test_mcp_overrides.py` | Tests for MCP tool ingredient_overrides parameter propagation | +| `test_misc_module.py` | Contract tests: server._misc module | +| `test_no_raw_signal_handler.py` | AST guard: no raw signal.signal(SIGTERM, ...) in cli/app.py | +| `test_perform_merge_editable_guard.py` | Integration tests verifying perform_merge() aborts before cleanup on poisoned installs | +| `test_quota_refresh_loop.py` | Tests for _quota_refresh_loop in server/_misc.py | +| `test_release_issue_fail_label.py` | Tests for release_issue fail_label path and fail label cleanup | +| `test_reload_session.py` | Tests for the reload_session MCP tool and supporting helpers | +| `test_research_smoke_pipeline.py` | Research recipe smoke pipeline: gated E2E tests (RESEARCH_SMOKE_TEST=1) | +| `test_resolve_model_as_profile.py` | Tests for _resolve_model_as_profile — model-value-as-provider-profile resolution in _guards.py | +| `test_resolve_provider_profile.py` | Tests for _resolve_provider_profile four-tier provider resolution in _guards.py | +| `test_run_skill_add_dirs.py` | Contract tests: run_skill passes correct add_dirs to executor (T-OVR-014) | +| `test_run_skill_resume.py` | Tests for resume_session_id threading from run_skill through executor | +| `test_server_init_gate.py` | Tests for server init: gate access, visibility, subset management, wire format compliance | +| `test_server_init_session_visibility.py` | Tests for server init: session type visibility, fleet gate boot, feature gate visibility | +| `test_server_tool_registration.py` | Tests for MCP tool registration, config-driven behavior, and schema contracts | +| `test_server_version_telemetry.py` | Tests for server version info, plugin metadata, lazy init, and telemetry | +| `test_service_wrappers.py` | Behavior tests for DefaultRecipeRepository and DefaultMigrationService (REQ-ARCH-006, 007) | +| `test_session_type_tags.py` | Tests for _collect_fleet_tool_tags in server._session_type (Finding 1) | +| `test_set_commit_status.py` | Tests for the set_commit_status MCP tool handler | +| `test_smoke_pipeline.py` | Smoke-test pipeline: structural validation and end-to-end execution tests | +| `test_smoke_recipe_scope_guard.py` | Structural guard: smoke_recipe fixture must not use scope='module' | +| `test_state.py` | Tests for server/_state.py: server initialization | +| `test_tool_annotation_completeness.py` | Runtime annotation test shield for MCP tool readOnlyHint semantics (layers 2, 3, 4) | +| `test_tool_exception_boundary.py` | Tests for the exception boundary in track_response_size | +| `test_tools_ci.py` | Tests for wait_for_ci, get_ci_status, and wait_for_merge_queue MCP tool handlers | +| `test_tools_ci_enqueue.py` | Tests for enqueue_pr MCP tool handler | +| `test_tools_ci_merge_state.py` | Tests for check_repo_merge_state MCP tool handler | +| `test_tools_ci_split.py` | CI split structural guard | +| `test_tools_ci_watch.py` | Tests for wait_for_ci event validation, null coercion, and auto_trigger recovery | +| `test_tools_clone.py` | Tests for autoskillit server clone tools | +| `test_tools_dispatch.py` | Tests for dispatch_food_truck tool handler: gates, validation, execution, and execute_dispatch domain function | +| `test_tools_dispatch_halt.py` | Tests for dispatch_food_truck campaign halt enforcement gate | +| `test_tools_execution_command.py` | Tests for run_skill command building, timeouts, env, model, and per-invocation markers | +| `test_tools_execution_input_gates.py` | Tests for run_skill input validation gates and CWD checking | +| `test_tools_execution_provider.py` | Tests for provider_extras/profile_name forwarding through run_skill() | +| `test_tools_execution_response.py` | Contract tests: MCP tool response fields use correct enum types | +| `test_tools_execution_results.py` | Tests for run_skill result shapes, failure paths, timing, flush telemetry, and gate checks | +| `test_tools_execution_routing.py` | Tests for run_skill routing, executor delegation, and session skill management | +| `test_tools_git.py` | Tests for classify_fix and merge_worktree MCP tools | +| `test_tools_github.py` | Tests for server/tools_github.py — fetch_github_issue and get_issue_title | +| `test_tools_github_api_tracking.py` | GitHub API tracking tests | +| `test_tools_integrations.py` | Integration tests for issue lifecycle, headless tool diagnostics, and PR ops | +| `test_tools_integrations_release.py` | Tests for release_issue staged lifecycle behaviour | +| `test_tools_issue_lifecycle.py` | Tests for server/tools_issue_lifecycle.py | +| `test_tools_kitchen_envelope.py` | Tests for tools_kitchen.py: hook drift warnings and failure envelopes | +| `test_tools_kitchen_gate.py` | Tests for tools_kitchen.py: gate mechanics, hook config, recipe packs, refresh, misc | +| `test_tools_kitchen_visibility.py` | Tests for tools_kitchen.py: visibility, component management, sous-chef, redisable_subsets | +| `test_tools_label_validation.py` | Tests for label whitelist validation in server tool handlers | +| `test_tools_list_recipes.py` | Tests for autoskillit server list_recipes tool | +| `test_tools_load_recipe.py` | Tests for autoskillit server load_recipe tool | +| `test_tools_migrate_recipe.py` | Tests for autoskillit server migrate_recipe tool | +| `test_tools_pr_ops.py` | Tests for server/tools_pr_ops.py | +| `test_tools_recipe.py` | Tests for autoskillit server validate_recipe tool and recipe docstring contracts | +| `test_tools_report_bug.py` | Tests for report_bug MCP tool handler and supporting helpers (_parse_fingerprint, _extract_block, _parse_prepare_result, _parse_enrich_result) | +| `test_tools_run_cmd.py` | Tests for run_cmd and run_python MCP tool handlers | +| `test_tools_run_cmd_unit.py` | Unit tests for run_cmd: observability, timing, and headless gate enforcement | +| `test_tools_run_python.py` | Unit tests for run_python: observability and headless gate enforcement | +| `test_tools_bootstrap.py` | Tests for bootstrap composite MCP tools (bootstrap_clone, claim_and_resolve_issue, create_and_publish_branch) | +| `test_tools_run_skill_retry.py` | Tests verifying run_skill_retry was removed and run_skill handles all sessions | +| `test_tools_session_diagnostics.py` | Tests for session diagnostics helpers in tools_github | +| `test_tools_status_kitchen.py` | Tests for server status tools: kitchen status, pipeline report, and telemetry recovery | +| `test_tools_status_mcp_response.py` | Tests for MCP response tracking integration in tools_status handlers | +| `test_tools_status_quota_and_db.py` | Tests for server status tools: quota events, telemetry writing, and DB access | +| `test_tools_status_summaries.py` | Tests for server status tools: token and timing summaries | +| `test_tools_types_module.py` | Tests for server/tools/_types.py TypedDict imports | +| `test_tools_workspace.py` | Tests for autoskillit server workspace tools | +| `test_track_response_size.py` | Tests for the track_response_size decorator in autoskillit.server._notify | +| `test_wire_compat.py` | Wire compatibility tests | + +## Architecture Notes + +`conftest.py` provides shared fixtures including `tool_ctx` (full-stack L3 context) used across server tests. `_helpers.py` provides shared test builder utilities. The `test_tools_execution_*.py` files test run_skill in focused slices (command, input gates, response, results, routing). diff --git a/tests/server/_helpers.py b/tests/server/_helpers.py index 665b636c13..27ed300336 100644 --- a/tests/server/_helpers.py +++ b/tests/server/_helpers.py @@ -4,6 +4,19 @@ from autoskillit.core import SkillResult from autoskillit.core.types import RetryReason +from tests.fleet._helpers import _make_recipe_info as _fleet_make_recipe_info + + +def _make_recipe_info(name: str = "test-recipe"): + return _fleet_make_recipe_info(name, path_prefix="/fake/recipes/") + + +def _make_standard_recipe(name: str = "test-recipe", ingredient_keys: list[str] | None = None): + """Return a minimal Recipe with kind=STANDARD for use as load_recipe mock return value.""" + from autoskillit.recipe.schema import Recipe, RecipeIngredient, RecipeKind + + ingredients = {k: RecipeIngredient(description=k) for k in (ingredient_keys or [])} + return Recipe(name=name, description="test", ingredients=ingredients, kind=RecipeKind.STANDARD) def _skill_ok(report_text: str = "## Bug Report\ndetails") -> SkillResult: @@ -32,3 +45,30 @@ def _skill_fail() -> SkillResult: retry_reason=RetryReason.NONE, stderr="something went wrong", ) + + +_MINIMAL_SCRIPT_YAML = """\ +name: test-script +description: Test +summary: test +ingredients: + task: + description: What to do + required: true +steps: + do-thing: + tool: run_skill + with: + skill_command: "/autoskillit:investigate ${{ inputs.task }}" + cwd: "." + on_success: done + on_failure: escalate + done: + action: stop + message: "Done." + escalate: + action: stop + message: "Failed." +kitchen_rules: + - "Follow routing rules" +""" diff --git a/tests/server/_type_coercion_fixtures.py b/tests/server/_type_coercion_fixtures.py new file mode 100644 index 0000000000..f987374bc0 --- /dev/null +++ b/tests/server/_type_coercion_fixtures.py @@ -0,0 +1,34 @@ +"""Typed callables for _import_and_call type-coercion tests (Step 1a).""" + +from __future__ import annotations + +from pathlib import Path + + +def _typed_callable(name: str, count: int, ratio: float = 1.0) -> dict: + assert isinstance(name, str), f"name should be str, got {type(name)}" + assert isinstance(count, int), f"count should be int, got {type(count)}" + assert isinstance(ratio, float), f"ratio should be float, got {type(ratio)}" + return {"name": name, "count": count, "ratio": ratio} + + +def _str_only_param(value: str) -> dict: + assert isinstance(value, str), f"value should be str, got {type(value)}" + return {"value": value} + + +def _int_param(value: int) -> dict: + assert isinstance(value, int), f"value should be int, got {type(value)}" + return {"value": value} + + +def _str_optional_param(value: str = "default") -> dict: + return {"value": value} + + +def _str_path_param(path: str) -> Path: + return Path(path) + + +def _kwargs_callable(name: str, **kwargs: object) -> dict: + return {"name": name, "extras": kwargs} diff --git a/tests/server/conftest.py b/tests/server/conftest.py index 6338ccb00a..631a5414cc 100644 --- a/tests/server/conftest.py +++ b/tests/server/conftest.py @@ -2,24 +2,167 @@ from __future__ import annotations +from typing import TYPE_CHECKING +from unittest.mock import MagicMock + import pytest +if TYPE_CHECKING: + from autoskillit.pipeline.timings import DefaultTimingLog + + +@pytest.fixture(autouse=True) +def _reset_server_state(monkeypatch): + """Reset module-level _ctx in server._state after each test. + + Tests that call _initialize() directly set _state._ctx to a mock without + cleanup. Subsequent tests in the same xdist worker then find a stale mock + _ctx, causing _apply_triage_gate to await a regular MagicMock and fail. + + monkeypatch records the current value before yield and restores it after, + giving each test a clean slate regardless of what _initialize() sets. + """ + from autoskillit.server import _state + + monkeypatch.setattr(_state, "_ctx", _state._ctx) + + +@pytest.fixture(autouse=True) +def _reset_mcp_tags(): + """Reset MCP tag visibility to default (kitchen disabled) before each test. + + The mcp singleton is process-global. Each mcp.enable()/disable() call appends + a Visibility transform to an internal list — the list never shrinks. Over a + full test suite (11k+ tests), thousands of accumulated transforms can cause + version-dependent ordering issues in FastMCP's "last match wins" evaluation. + + Fix: truncate the transforms list back to its fresh state, then explicitly + disable all gated tags — matching the server/__init__.py import-time baseline + and preventing orchestrator-path tests from leaking fleet-dispatch or + kitchen-core enables into subsequent fleet visibility tests. + """ + from autoskillit.core import ALL_VISIBILITY_TAGS + from autoskillit.server import mcp + + mcp._transforms.clear() + for tag in sorted(ALL_VISIBILITY_TAGS): + mcp.disable(tags={tag}) + yield + mcp._transforms.clear() + for tag in sorted(ALL_VISIBILITY_TAGS): + mcp.disable(tags={tag}) + @pytest.fixture() def kitchen_enabled(): """Enable the kitchen tag on the MCP server for the duration of the test.""" + from autoskillit.core import ALL_VISIBILITY_TAGS from autoskillit.server import mcp + mcp._transforms.clear() + for tag in sorted(ALL_VISIBILITY_TAGS): + mcp.disable(tags={tag}) mcp.enable(tags={"kitchen"}) yield - mcp.disable(tags={"kitchen"}) + mcp._transforms.clear() + for tag in sorted(ALL_VISIBILITY_TAGS): + mcp.disable(tags={tag}) @pytest.fixture() def headless_enabled(): """Enable the headless tag on the MCP server for the duration of the test.""" + from autoskillit.core import ALL_VISIBILITY_TAGS from autoskillit.server import mcp + mcp._transforms.clear() + for tag in sorted(ALL_VISIBILITY_TAGS): + mcp.disable(tags={tag}) mcp.enable(tags={"headless"}) yield - mcp.disable(tags={"headless"}) + mcp._transforms.clear() + for tag in sorted(ALL_VISIBILITY_TAGS): + mcp.disable(tags={tag}) + + +def assert_step_timed(timing_log: DefaultTimingLog, step_name: str) -> None: + assert any(e["step_name"] == step_name for e in timing_log.get_report()) + + +def assert_no_timing(timing_log: DefaultTimingLog) -> None: + assert timing_log.get_report() == [] + + +def _make_mock_ctx() -> MagicMock: + """Return a minimal mock ToolContext with a gate.""" + gate = MagicMock() + gate.enabled = False + ctx = MagicMock() + ctx.gate = gate + ctx.config.subsets.disabled = [] # REQ-VIS-008: no subsets disabled by default + return ctx + + +_SUCCESS_JSON = ( + '{"type": "result", "subtype": "success", "is_error": false,' + ' "result": "done", "session_id": "s1"}' +) + + +@pytest.fixture(autouse=True) +def _suppress_nudge(monkeypatch): + """Prevent the contract-nudge gate from firing on mock EARLY_STOP results. + + Server command-building tests use mock results that lack the per-invocation + completion marker, causing _retry_fsm to classify them as EARLY_STOP. With + the widened nudge gate (no provider_extras guard), the nudge fires and + appends an extra runner call that breaks call_args_list[-1] assertions. + """ + + async def _noop(*_args, **_kwargs): + return None + + monkeypatch.setattr("autoskillit.execution.headless._attempt_contract_nudge", _noop) + + +@pytest.fixture +def build_ctx(tmp_path): + """Factory: build_ctx(**overrides) → minimal ToolContext with overrides applied.""" + from autoskillit.config.settings import AutomationConfig + from autoskillit.core.types import DirectInstall + from autoskillit.pipeline.audit import DefaultAuditLog + from autoskillit.pipeline.context import ToolContext + from autoskillit.pipeline.gate import DefaultGateState + from autoskillit.pipeline.timings import DefaultTimingLog + from autoskillit.pipeline.tokens import DefaultTokenLog + + def _factory(**overrides): + ctx = ToolContext( + config=AutomationConfig(features={"fleet": True}), + audit=DefaultAuditLog(), + token_log=DefaultTokenLog(), + timing_log=DefaultTimingLog(), + gate=DefaultGateState(enabled=False), + plugin_source=DirectInstall(plugin_dir=tmp_path), + runner=None, + temp_dir=tmp_path / ".autoskillit" / "temp", + project_dir=tmp_path, + ) + for field_name, value in overrides.items(): + setattr(ctx, field_name, value) + return ctx + + return _factory + + +@pytest.fixture +def build_ctx_open(build_ctx): + """build_ctx variant with gate open — returns a factory callable like build_ctx.""" + from autoskillit.pipeline.gate import DefaultGateState + + def _factory(**overrides): + ctx = build_ctx(**overrides) + ctx.gate = DefaultGateState(enabled=True) + return ctx + + return _factory diff --git a/tests/server/test_editable_guard.py b/tests/server/test_editable_guard.py index 8eedf174e1..04542bb987 100644 --- a/tests/server/test_editable_guard.py +++ b/tests/server/test_editable_guard.py @@ -3,8 +3,12 @@ import json from pathlib import Path +import pytest + from autoskillit.server._editable_guard import scan_editable_installs_for_worktree +pytestmark = [pytest.mark.layer("server"), pytest.mark.small] + def _make_dist_info(site_packages: Path, pkg: str, version: str, direct_url: dict) -> None: dist_info = site_packages / f"{pkg}-{version}.dist-info" diff --git a/tests/server/test_factory.py b/tests/server/test_factory.py index a5301dbf4f..f759472340 100644 --- a/tests/server/test_factory.py +++ b/tests/server/test_factory.py @@ -20,10 +20,12 @@ resolve_skill_name, ) from autoskillit.recipe.repository import DefaultRecipeRepository -from autoskillit.server._factory import TokenFactory, _gh_cli_token, make_context +from autoskillit.server._factory import _gh_cli_token, _LazyTokenFactory, make_context from autoskillit.workspace import DefaultCloneManager, SkillResolver from autoskillit.workspace.cleanup import DefaultWorkspaceManager -from tests.conftest import MockSubprocessRunner +from tests.fakes import MockSubprocessRunner + +pytestmark = [pytest.mark.layer("server"), pytest.mark.small] def _runner() -> MockSubprocessRunner: @@ -172,6 +174,30 @@ async def run( token_usage=None, ) + async def dispatch_food_truck( + self, + orchestrator_prompt: str, + cwd: str, + *, + completion_marker: str = "", + model: str = "", + step_name: str = "", + on_spawn=None, + **kwargs, + ) -> SkillResult: + return SkillResult( + success=True, + result="", + session_id="", + subtype="", + is_error=False, + exit_code=0, + needs_retry=False, + retry_reason="none", + stderr="", + token_usage=None, + ) + ctx = make_context(AutomationConfig(), runner=_runner()) ctx.executor = FakeExecutor() assert isinstance(ctx.executor, HeadlessExecutor) @@ -205,36 +231,6 @@ def test_output_patterns_nonempty_for_investigate() -> None: ) -# --------------------------------------------------------------------------- -# Write-expected resolver integration tests -# --------------------------------------------------------------------------- - - -def test_write_expected_resolver_wired_on_context() -> None: - """make_context() must wire a write_expected_resolver onto ToolContext.""" - ctx = make_context(AutomationConfig(), runner=_runner()) - assert ctx.write_expected_resolver is not None - spec = ctx.write_expected_resolver("/autoskillit:make-plan some task") - assert spec.mode == "always" - - -def test_write_expected_resolver_unknown_skill() -> None: - """Unknown skills produce a WriteBehaviorSpec with mode=None.""" - ctx = make_context(AutomationConfig(), runner=_runner()) - assert ctx.write_expected_resolver is not None - spec = ctx.write_expected_resolver("/autoskillit:nonexistent-skill foo") - assert spec.mode is None - - -def test_write_expected_resolver_conditional_skill() -> None: - """resolve-merge-conflicts produces mode='conditional' with patterns.""" - ctx = make_context(AutomationConfig(), runner=_runner()) - assert ctx.write_expected_resolver is not None - spec = ctx.write_expected_resolver("/autoskillit:resolve-merge-conflicts") - assert spec.mode == "conditional" - assert len(spec.expected_when) > 0 - - @pytest.mark.parametrize( "invocation,expected_mode,required_tokens", [ @@ -273,6 +269,9 @@ def test_write_expected_resolver_conditional_skill() -> None: "conditional", ["verdict"], ), + ("/autoskillit:make-plan some task", "always", []), + ("/autoskillit:nonexistent-skill foo", None, []), + ("/autoskillit:resolve-merge-conflicts", "conditional", ["conflict_report_path"]), ], ) def test_write_expected_resolver_mode( @@ -286,7 +285,6 @@ def test_write_expected_resolver_mode( if expected_mode is None: assert spec.expected_when == () else: - assert len(spec.expected_when) > 0 for token in required_tokens: assert any(token in p for p in spec.expected_when) @@ -316,7 +314,7 @@ def _count_ctor_positional_args(src_path: Path) -> int: return -1 root = pkg_root() - cook_path = root / "cli" / "_cook.py" + cook_path = root / "cli" / "session" / "_cook.py" factory_path = root / "server" / "_factory.py" cook_count = _count_ctor_positional_args(cook_path) @@ -383,7 +381,7 @@ def _resolver(): call_count += 1 return "ghp_test_token" - factory = TokenFactory(_resolver) + factory = _LazyTokenFactory(_resolver) assert call_count == 0, "TokenFactory resolved eagerly at construction" assert not factory.is_resolved @@ -407,7 +405,7 @@ def _resolver(): call_count += 1 return None - factory = TokenFactory(_resolver) + factory = _LazyTokenFactory(_resolver) assert factory() is None assert call_count == 1 assert factory() is None @@ -434,23 +432,37 @@ def tracking_run(*args, **kwargs): assert gh_calls == [], f"_gh_cli_token() called during make_context: {gh_calls}" -def test_make_context_sets_plugin_dir_none_when_plugin_installed( - monkeypatch: pytest.MonkeyPatch, +def test_make_context_marketplace_install_yields_marketplace_plugin_source( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path ) -> None: - """make_context() sets plugin_dir=None when marketplace plugin is installed.""" + """make_context() with a marketplace-detected install produces MarketplaceInstall.""" + from autoskillit.core.types._type_plugin_source import MarketplaceInstall + + fake_cache = tmp_path / "cache" / "autoskillit-local" / "autoskillit" / "1.0.0" + fake_cache.mkdir(parents=True) + monkeypatch.setattr("autoskillit.server._factory._check_plugin_installed", lambda: True) + monkeypatch.setattr( + "autoskillit.server._factory._resolve_marketplace_cache_path", + lambda: fake_cache, + ) + ctx = make_context(AutomationConfig(), runner=None) - assert ctx.plugin_dir is None + assert isinstance(ctx.plugin_source, MarketplaceInstall) + assert ctx.plugin_source.cache_path == fake_cache -def test_make_context_sets_plugin_dir_when_plugin_not_installed( - monkeypatch: pytest.MonkeyPatch, +def test_make_context_direct_install_yields_direct_plugin_source( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path ) -> None: - """make_context() sets plugin_dir to package root when plugin is not installed.""" + """make_context() with a direct install produces DirectInstall with pkg_root.""" + from autoskillit.core.types._type_plugin_source import DirectInstall + monkeypatch.setattr("autoskillit.server._factory._check_plugin_installed", lambda: False) - ctx = make_context(AutomationConfig(), runner=None) - assert ctx.plugin_dir is not None - assert ctx.plugin_dir != "" + + ctx = make_context(AutomationConfig(), runner=None, plugin_dir=str(tmp_path)) + assert isinstance(ctx.plugin_source, DirectInstall) + assert ctx.plugin_source.plugin_dir == tmp_path def test_make_context_sets_token_factory(tmp_path): @@ -458,3 +470,130 @@ def test_make_context_sets_token_factory(tmp_path): cfg = AutomationConfig() ctx = make_context(cfg, runner=None, plugin_dir=str(tmp_path)) assert callable(ctx.token_factory) + + +# --- Group P-2: project_dir env inheritance --- + + +def test_make_context_reads_project_dir_env(tmp_path, monkeypatch): + """make_context reads AUTOSKILLIT_PROJECT_DIR and stores it on ctx.project_dir.""" + monkeypatch.setenv("AUTOSKILLIT_PROJECT_DIR", str(tmp_path)) + ctx = make_context(AutomationConfig(), runner=_runner()) + assert ctx.project_dir == tmp_path + + +def test_make_context_project_dir_git_root_fallback(monkeypatch): + """make_context resolves project_dir via git toplevel when env var is not set.""" + monkeypatch.delenv("AUTOSKILLIT_PROJECT_DIR", raising=False) + ctx = make_context(AutomationConfig(), runner=_runner()) + import subprocess as _sp + + expected = Path( + _sp.run( + ["git", "rev-parse", "--show-toplevel"], + capture_output=True, + text=True, + ).stdout.strip() + ) + assert ctx.project_dir == expected + + +# --- _resolve_project_dir unit tests --- + + +def test_resolve_project_dir_env_var(tmp_path, monkeypatch): + from autoskillit.server._factory import _resolve_project_dir + + monkeypatch.setenv("AUTOSKILLIT_PROJECT_DIR", str(tmp_path)) + assert _resolve_project_dir() == tmp_path + + +def test_resolve_project_dir_git_root(monkeypatch): + import subprocess as _subprocess + + from autoskillit.server._factory import _resolve_project_dir + + monkeypatch.delenv("AUTOSKILLIT_PROJECT_DIR", raising=False) + + def fake_run(cmd, *, capture_output, text, timeout): + return _subprocess.CompletedProcess(cmd, 0, stdout="/fake/git/root\n", stderr="") + + monkeypatch.setattr("autoskillit.server._factory.subprocess.run", fake_run) + assert _resolve_project_dir() == Path("/fake/git/root") + + +def test_resolve_project_dir_cwd_fallback(monkeypatch): + import subprocess as _subprocess + + from autoskillit.server._factory import _resolve_project_dir + + monkeypatch.delenv("AUTOSKILLIT_PROJECT_DIR", raising=False) + + def fake_run(cmd, *, capture_output, text, timeout): + return _subprocess.CompletedProcess(cmd, 1, stdout="", stderr="not a git repo") + + monkeypatch.setattr("autoskillit.server._factory.subprocess.run", fake_run) + assert _resolve_project_dir() == Path.cwd() + + +def test_make_context_env_profile_overrides_default_provider(monkeypatch): + """AUTOSKILLIT_PROVIDER_PROFILE in env must set config.providers.default_provider.""" + monkeypatch.setenv("AUTOSKILLIT_PROVIDER_PROFILE", "minimax") + config = AutomationConfig() + config.providers.default_provider = None + config.providers.profiles = {"minimax": {}} + ctx = make_context(config, runner=_runner()) + assert ctx.config.providers.default_provider == "minimax" + + +def test_make_context_env_profile_overrides_existing_default(monkeypatch): + """AUTOSKILLIT_PROVIDER_PROFILE overrides even a config-set default_provider.""" + monkeypatch.setenv("AUTOSKILLIT_PROVIDER_PROFILE", "minimax") + config = AutomationConfig() + config.providers.default_provider = "openai" + config.providers.profiles = {"minimax": {}} + ctx = make_context(config, runner=_runner()) + assert ctx.config.providers.default_provider == "minimax" + + +def test_make_context_unknown_env_profile_is_ignored(monkeypatch): + """AUTOSKILLIT_PROVIDER_PROFILE not in profiles is ignored — no mutation.""" + monkeypatch.setenv("AUTOSKILLIT_PROVIDER_PROFILE", "stale-unknown") + config = AutomationConfig() + config.providers.default_provider = "anthropic" + config.providers.profiles = {} + ctx = make_context(config, runner=_runner()) + assert ctx.config.providers.default_provider == "anthropic" + + +def test_make_context_no_env_profile_preserves_config_default(monkeypatch): + """Without AUTOSKILLIT_PROVIDER_PROFILE in env, default_provider is unchanged.""" + monkeypatch.delenv("AUTOSKILLIT_PROVIDER_PROFILE", raising=False) + config = AutomationConfig() + config.providers.default_provider = "openai" + config.providers.profiles = {} + ctx = make_context(config, runner=_runner()) + assert ctx.config.providers.default_provider == "openai" + + +# --------------------------------------------------------------------------- +# Fixture gate-default verification tests +# --------------------------------------------------------------------------- + + +def test_tool_ctx_fixture_gate_starts_closed(tool_ctx) -> None: + """tool_ctx must start with gate closed to match production.""" + assert tool_ctx.gate.enabled is False, ( + "tool_ctx must start with gate closed to match production. " + "Use tool_ctx_kitchen_open for tests that need an open gate." + ) + + +def test_minimal_ctx_fixture_gate_starts_closed(minimal_ctx) -> None: + """minimal_ctx must start with gate closed to match production.""" + assert minimal_ctx.gate.enabled is False + + +def test_tool_ctx_kitchen_open_fixture_gate_starts_open(tool_ctx_kitchen_open) -> None: + """tool_ctx_kitchen_open must start with gate open.""" + assert tool_ctx_kitchen_open.gate.enabled is True diff --git a/tests/server/test_factory_recording.py b/tests/server/test_factory_recording.py new file mode 100644 index 0000000000..28fd01f3d5 --- /dev/null +++ b/tests/server/test_factory_recording.py @@ -0,0 +1,300 @@ +"""Tests for make_context recording/replay runner wiring and related run_headless_core behavior.""" + +from __future__ import annotations + +from dataclasses import dataclass +from unittest.mock import Mock + +import pytest + +from autoskillit.execution.recording import RecordingSubprocessRunner, ReplayingSubprocessRunner +from tests.conftest import _make_result +from tests.fakes import MockSubprocessRunner + +pytestmark = [pytest.mark.layer("server"), pytest.mark.small] + + +@dataclass +class FakeStepResult: + cassette_exit_code: int + cassette_path: str + cassette_duration_ms: int + + +# --- T9: run_headless_core passes scenario_step_name through --- + + +@pytest.mark.anyio +async def test_run_headless_core_injects_scenario_step_name(tool_ctx, tmp_path): + """run_headless_core passes step_name as scenario_step_name and routes it via env kwarg.""" + from autoskillit.execution.headless import run_headless_core + + await run_headless_core("/investigate foo", str(tmp_path), tool_ctx, step_name="investigate") + + assert tool_ctx.runner.call_args_list, "runner was never called" + cmd, _cwd, _timeout, kwargs = tool_ctx.runner.call_args_list[0] + assert cmd[0] != "env" + env = kwargs.get("env") + assert env is not None + assert env["SCENARIO_STEP_NAME"] == "investigate" + + +# --- T-AUTO-DERIVE: run_headless_core auto-derives step name --- + + +@pytest.mark.anyio +async def test_run_headless_core_auto_derives_step_name_when_recording(tmp_path): + """When runner is RecordingSubprocessRunner and step_name is empty, + run_headless_core auto-derives step_name from the skill command.""" + from autoskillit.config import AutomationConfig + from autoskillit.execution.headless import run_headless_core + from autoskillit.pipeline import DefaultGateState + from autoskillit.server._factory import make_context + + mock_recorder = Mock() + mock_recorder.record_step.return_value = FakeStepResult( + cassette_exit_code=0, + cassette_path="", + cassette_duration_ms=100, + ) + inner = MockSubprocessRunner() + inner.set_default(_make_result()) + recording_runner = RecordingSubprocessRunner(recorder=mock_recorder, inner=inner) + + ctx = make_context(AutomationConfig(), runner=recording_runner, plugin_dir=str(tmp_path)) + ctx.gate = DefaultGateState(enabled=True) + ctx.config.linux_tracing.log_dir = str(tmp_path / "logs") + + # Call WITHOUT step_name — auto-derivation should kick in + await run_headless_core("/autoskillit:smoke-task", str(tmp_path), ctx) + + # record_step must be called with the derived step name + mock_recorder.record_step.assert_called_once() + call_kwargs = mock_recorder.record_step.call_args.kwargs + assert call_kwargs["step_name"] == "smoke-task", ( + f"Expected derived step_name 'smoke-task', got {call_kwargs['step_name']!r}" + ) + + +# --- T10: make_context wraps runner when RECORD_SCENARIO set --- + + +def test_make_context_wraps_runner_when_record_scenario(monkeypatch, tmp_path): + scenario_dir = tmp_path / "scenario" + scenario_dir.mkdir() + monkeypatch.setenv("RECORD_SCENARIO", "1") + monkeypatch.setenv("RECORD_SCENARIO_DIR", str(scenario_dir)) + monkeypatch.setenv("RECORD_SCENARIO_RECIPE", "smoke-test") + mock_recorder = Mock() + import api_simulator.claude as _api_sim_claude + + monkeypatch.setattr( + _api_sim_claude, "make_scenario_recorder", Mock(return_value=mock_recorder), raising=False + ) + mock_atexit = Mock() + monkeypatch.setattr("atexit.register", mock_atexit) + + from autoskillit.config import AutomationConfig + from autoskillit.server._factory import make_context + + ctx = make_context(AutomationConfig(), plugin_dir=str(tmp_path)) + assert isinstance(ctx.runner, RecordingSubprocessRunner) + mock_atexit.assert_not_called() + + +# --- T11: make_context default runner unchanged without env var --- + + +def test_make_context_default_runner_without_record_scenario(monkeypatch, tmp_path): + monkeypatch.delenv("RECORD_SCENARIO", raising=False) + monkeypatch.delenv("REPLAY_SCENARIO", raising=False) + + from autoskillit.config import AutomationConfig + from autoskillit.execution.process import DefaultSubprocessRunner + from autoskillit.server._factory import make_context + + ctx = make_context(AutomationConfig(), plugin_dir=str(tmp_path)) + assert isinstance(ctx.runner, DefaultSubprocessRunner) + + +# --- T20: make_context wires ReplayingSubprocessRunner when REPLAY_SCENARIO set --- + + +def test_make_context_wires_sequencing_runner_when_replay_scenario(monkeypatch, tmp_path): + """REPLAY_SCENARIO=1 + valid dir → ctx.runner is ReplayingSubprocessRunner.""" + replay_dir = tmp_path / "replay" + replay_dir.mkdir() + monkeypatch.setenv("REPLAY_SCENARIO", "1") + monkeypatch.setenv("REPLAY_SCENARIO_DIR", str(replay_dir)) + monkeypatch.delenv("RECORD_SCENARIO", raising=False) + + mock_scenario = Mock() + mock_scenario.step_sequence = [] + mock_player = Mock() + mock_player.scenario.return_value = mock_scenario + mock_player.build_session_map.return_value = {} + mock_make_player = Mock(return_value=mock_player) + + import api_simulator.claude as _api_sim_claude + + monkeypatch.setattr(_api_sim_claude, "make_scenario_player", mock_make_player, raising=False) + + from autoskillit.config import AutomationConfig + from autoskillit.server._factory import make_context + + ctx = make_context(AutomationConfig(), plugin_dir=str(tmp_path)) + assert isinstance(ctx.runner, ReplayingSubprocessRunner) + mock_make_player.assert_called_once() + call_kwargs = mock_make_player.call_args.kwargs + assert call_kwargs.get("scenario_dir") == str(replay_dir) + + +# --- T21: REPLAY_SCENARIO takes precedence over RECORD_SCENARIO --- + + +def test_replay_takes_precedence_over_record(monkeypatch, tmp_path): + """When both REPLAY and RECORD env vars set, REPLAY wins.""" + replay_dir = tmp_path / "replay" + replay_dir.mkdir() + monkeypatch.setenv("REPLAY_SCENARIO", "1") + monkeypatch.setenv("REPLAY_SCENARIO_DIR", str(replay_dir)) + monkeypatch.setenv("RECORD_SCENARIO", "1") + monkeypatch.setenv("RECORD_SCENARIO_DIR", str(replay_dir)) + + mock_scenario = Mock() + mock_scenario.step_sequence = [] + mock_player = Mock() + mock_player.scenario.return_value = mock_scenario + mock_player.build_session_map.return_value = {} + mock_recorder = Mock() + mock_make_recorder = Mock(return_value=mock_recorder) + + import api_simulator.claude as _api_sim_claude + + monkeypatch.setattr( + _api_sim_claude, "make_scenario_player", Mock(return_value=mock_player), raising=False + ) + monkeypatch.setattr( + _api_sim_claude, "make_scenario_recorder", mock_make_recorder, raising=False + ) + # Stabilize against xdist ordering: weakref.finalize auto-registers its _exitfunc + # with atexit.register exactly once per process (guarded by _registered_with_atexit). + # If this test runs first in an xdist worker, that registration slips through + # mock_atexit. Force the flag True before mocking so no stray call occurs. + import weakref as _wrf + + _wrf.finalize._registered_with_atexit = True + + mock_atexit = Mock() + monkeypatch.setattr("atexit.register", mock_atexit) + + from autoskillit.config import AutomationConfig + from autoskillit.server._factory import make_context + + ctx = make_context(AutomationConfig(), plugin_dir=str(tmp_path)) + assert isinstance(ctx.runner, ReplayingSubprocessRunner) + mock_make_recorder.assert_not_called() # REPLAY takes precedence over RECORD + mock_atexit.assert_not_called() + + +# --- T-BUILD-REPLAY-SNAP: build_replay_runner scans and surfaces skill_snapshots --- + + +def test_build_replay_runner_scans_skill_snapshots(tmp_path, monkeypatch): + """build_replay_runner() populates runner.skill_snapshots from scenario dir.""" + + from autoskillit.execution.recording import build_replay_runner + + replay_dir = tmp_path / "replay" + replay_dir.mkdir() + (replay_dir / "skill-snapshots" / "investigate").mkdir(parents=True) + + mock_scenario = Mock() + mock_scenario.step_sequence = [] + mock_player = Mock() + mock_player.scenario.return_value = mock_scenario + mock_player.build_session_map.return_value = {} + + import api_simulator.claude as _api_sim_claude + + monkeypatch.setattr( + _api_sim_claude, "make_scenario_player", Mock(return_value=mock_player), raising=False + ) + + runner = build_replay_runner(str(replay_dir)) + + assert "investigate" in runner.skill_snapshots + assert runner.skill_snapshots["investigate"] == replay_dir / "skill-snapshots" / "investigate" + + +# --- T-RUN-SKILL-REPLAY: snapshot present → init_session NOT called --- + + +@pytest.mark.anyio +async def test_run_skill_replay_uses_snapshot_over_init_session( + tool_ctx_kitchen_open, tmp_path, monkeypatch +): + """With a skill snapshot for the step, run_skill skips init_session.""" + from unittest.mock import MagicMock + + from autoskillit.execution.recording import ReplayingSubprocessRunner + from autoskillit.server.tools.tools_execution import run_skill + from tests.fakes import InMemoryHeadlessExecutor + + snap_dir = tmp_path / "snap" / "investigate" + skill_md = snap_dir / ".claude" / "skills" / "investigate" / "SKILL.md" + skill_md.parent.mkdir(parents=True) + skill_md.write_text("# investigate\n", encoding="utf-8") + + replay_runner = ReplayingSubprocessRunner({}, {}, skill_snapshots={"investigate": snap_dir}) + tool_ctx_kitchen_open.runner = replay_runner + + mock_ssm = MagicMock() + tool_ctx_kitchen_open.session_skill_manager = mock_ssm + + executor = InMemoryHeadlessExecutor() + tool_ctx_kitchen_open.executor = executor + + ephemeral_root = tmp_path / "sessions" + tool_ctx_kitchen_open.ephemeral_root = ephemeral_root + monkeypatch.setattr("autoskillit.server._ctx", tool_ctx_kitchen_open) + + await run_skill("/investigate foo", str(tmp_path), step_name="investigate") + + mock_ssm.init_session.assert_not_called() + assert len(executor.calls) == 1 + add_dir_paths = [d.path for d in executor.calls[0].add_dirs] + assert any(p.startswith(str(ephemeral_root)) for p in add_dir_paths) + + +# --- T-RUN-SKILL-REPLAY-FALLBACK: no snapshot → init_session IS called --- + + +@pytest.mark.anyio +async def test_run_skill_replay_fallback_to_init_session( + tool_ctx_kitchen_open, tmp_path, monkeypatch +): + """With no snapshot for the step, run_skill falls back to init_session.""" + from unittest.mock import MagicMock + + from autoskillit.core import ValidatedAddDir + from autoskillit.execution.recording import ReplayingSubprocessRunner + from autoskillit.server.tools.tools_execution import run_skill + from tests.fakes import InMemoryHeadlessExecutor + + replay_runner = ReplayingSubprocessRunner({}, {}, skill_snapshots={}) + tool_ctx_kitchen_open.runner = replay_runner + + fake_validated = ValidatedAddDir(path="/fake/session") + mock_ssm = MagicMock() + mock_ssm.init_session.return_value = fake_validated + tool_ctx_kitchen_open.session_skill_manager = mock_ssm + + executor = InMemoryHeadlessExecutor() + tool_ctx_kitchen_open.executor = executor + + monkeypatch.setattr("autoskillit.server._ctx", tool_ctx_kitchen_open) + + await run_skill("/investigate foo", str(tmp_path), step_name="investigate") + + mock_ssm.init_session.assert_called_once() diff --git a/tests/server/test_git.py b/tests/server/test_git.py index 79292e24e1..5a45b4c7a3 100644 --- a/tests/server/test_git.py +++ b/tests/server/test_git.py @@ -14,7 +14,9 @@ TerminationReason, TestResult, ) -from tests.conftest import MockSubprocessRunner, StatefulMockTester +from tests.fakes import InMemoryTestRunner, MockSubprocessRunner + +pytestmark = [pytest.mark.layer("server"), pytest.mark.small] def _make_result( @@ -39,7 +41,7 @@ def default_config(): @pytest.fixture def conftest_mock_runner(): - from tests.conftest import MockSubprocessRunner + from tests.fakes import MockSubprocessRunner return MockSubprocessRunner() @@ -153,7 +155,7 @@ async def test_perform_merge_returns_success_on_green_tests( from autoskillit.server.git import perform_merge fake_wt = str(tmp_path) - tester = StatefulMockTester( + tester = InMemoryTestRunner( results=[TestResult(True, "= 50 passed =", ""), TestResult(True, "= 50 passed =", "")] ) # Queue 9 steps: rev-parse, branch, dirty check, fetch, rebase, @@ -169,6 +171,7 @@ async def test_perform_merge_returns_success_on_green_tests( conftest_mock_runner.push(_make_result(0, "", "")) # git ls-files (generated file check) conftest_mock_runner.push(_make_result(0, f"worktree {fake_wt}\n", "")) # wt list conftest_mock_runner.push(_make_result(0, "dev\n", "")) # step 7.5: branch --show-current + conftest_mock_runner.push(_make_result(0, "", "")) # step 7.6: git status --porcelain (clean) conftest_mock_runner.push(_make_result(0, "", "")) # merge conftest_mock_runner.push(_make_result(0, "", "")) # wt remove conftest_mock_runner.push(_make_result(0, "", "")) # branch -D @@ -202,7 +205,7 @@ async def test_perform_merge_blocks_on_post_rebase_test_failure( from autoskillit.server.git import perform_merge # Pre-rebase: pass; post-rebase: fail - tester = StatefulMockTester( + tester = InMemoryTestRunner( results=[TestResult(True, "= 10 passed =", ""), TestResult(False, "= 1 failed =", "")] ) # Queue: rev-parse (valid worktree), branch, dirty check, fetch ok, @@ -236,7 +239,7 @@ async def test_perform_merge_uses_no_edit_flag(default_config, conftest_mock_run from autoskillit.server.git import perform_merge fake_wt = str(tmp_path) - tester = StatefulMockTester(results=[TestResult(True, "ok", ""), TestResult(True, "ok", "")]) + tester = InMemoryTestRunner(results=[TestResult(True, "ok", ""), TestResult(True, "ok", "")]) # Queue all 10 steps for success path conftest_mock_runner.push(_make_result(0, f"{fake_wt}/.git/worktrees/wt", "")) conftest_mock_runner.push(_make_result(0, "feature-branch\n", "")) @@ -248,6 +251,7 @@ async def test_perform_merge_uses_no_edit_flag(default_config, conftest_mock_run conftest_mock_runner.push(_make_result(0, "", "")) # git ls-files (generated file check) conftest_mock_runner.push(_make_result(0, f"worktree {fake_wt}\n", "")) # wt list conftest_mock_runner.push(_make_result(0, "dev\n", "")) # step 7.5: branch --show-current + conftest_mock_runner.push(_make_result(0, "", "")) # step 7.6: git status --porcelain (clean) conftest_mock_runner.push(_make_result(0, "", "")) # merge conftest_mock_runner.push(_make_result(0, "", "")) # wt remove conftest_mock_runner.push(_make_result(0, "", "")) # branch -D @@ -284,7 +288,7 @@ async def test_perform_merge_blocks_on_missing_remote_tracking_ref( worktree_dir = tmp_path / "wt" worktree_dir.mkdir() - tester = StatefulMockTester(results=[TestResult(True, "= 10 passed =", "")]) + tester = InMemoryTestRunner(results=[TestResult(True, "= 10 passed =", "")]) # Step 2: worktree verified (needs /worktrees/ in git-dir path) conftest_mock_runner.push(_make_result(0, str(tmp_path / ".git/worktrees/wt"), "")) # Step 3: branch name found @@ -321,7 +325,7 @@ async def test_perform_merge_strips_tracked_generated_files( from autoskillit.server.git import perform_merge fake_wt = str(tmp_path) - tester = StatefulMockTester(results=[TestResult(True, "ok", ""), TestResult(True, "ok", "")]) + tester = InMemoryTestRunner(results=[TestResult(True, "ok", ""), TestResult(True, "ok", "")]) conftest_mock_runner.push(_make_result(0, f"{fake_wt}/.git/worktrees/wt", "")) # rev-parse conftest_mock_runner.push(_make_result(0, "feature-branch\n", "")) # branch conftest_mock_runner.push( @@ -336,6 +340,7 @@ async def test_perform_merge_strips_tracked_generated_files( conftest_mock_runner.push(_make_result(0, "", "")) # rebase conftest_mock_runner.push(_make_result(0, f"worktree {fake_wt}\n", "")) # wt list conftest_mock_runner.push(_make_result(0, "dev\n", "")) # step 7.5: branch --show-current + conftest_mock_runner.push(_make_result(0, "", "")) # step 7.6: git status --porcelain (clean) conftest_mock_runner.push(_make_result(0, "", "")) # merge conftest_mock_runner.push(_make_result(0, "", "")) # wt remove conftest_mock_runner.push(_make_result(0, "", "")) # branch -D @@ -375,7 +380,7 @@ async def test_perform_merge_noop_when_no_generated_files_tracked( from autoskillit.server.git import perform_merge fake_wt = str(tmp_path) - tester = StatefulMockTester(results=[TestResult(True, "ok", ""), TestResult(True, "ok", "")]) + tester = InMemoryTestRunner(results=[TestResult(True, "ok", ""), TestResult(True, "ok", "")]) conftest_mock_runner.push(_make_result(0, f"{fake_wt}/.git/worktrees/wt", "")) # rev-parse conftest_mock_runner.push(_make_result(0, "feature-branch\n", "")) # branch conftest_mock_runner.push(_make_result(0, "", "")) # git ls-files (empty — no generated files) @@ -386,6 +391,7 @@ async def test_perform_merge_noop_when_no_generated_files_tracked( conftest_mock_runner.push(_make_result(0, "", "")) # rebase conftest_mock_runner.push(_make_result(0, f"worktree {fake_wt}\n", "")) # wt list conftest_mock_runner.push(_make_result(0, "dev\n", "")) # step 7.5: branch --show-current + conftest_mock_runner.push(_make_result(0, "", "")) # step 7.6: git status --porcelain (clean) conftest_mock_runner.push(_make_result(0, "", "")) # merge conftest_mock_runner.push(_make_result(0, "", "")) # wt remove conftest_mock_runner.push(_make_result(0, "", "")) # branch -D @@ -413,7 +419,7 @@ async def test_perform_merge_fails_on_generated_file_cleanup_error( from autoskillit.server.git import perform_merge fake_wt = str(tmp_path) - tester = StatefulMockTester(results=[TestResult(True, "ok", "")]) + tester = InMemoryTestRunner(results=[TestResult(True, "ok", "")]) conftest_mock_runner.push(_make_result(0, f"{fake_wt}/.git/worktrees/wt", "")) # rev-parse conftest_mock_runner.push(_make_result(0, "feature-branch\n", "")) # branch conftest_mock_runner.push( @@ -441,7 +447,7 @@ async def test_perform_merge_dirty_check_ignores_generated_files( from autoskillit.server.git import perform_merge fake_wt = str(tmp_path) - tester = StatefulMockTester(results=[TestResult(True, "ok", ""), TestResult(True, "ok", "")]) + tester = InMemoryTestRunner(results=[TestResult(True, "ok", ""), TestResult(True, "ok", "")]) conftest_mock_runner.push(_make_result(0, f"{fake_wt}/.git/worktrees/wt", "")) # rev-parse conftest_mock_runner.push(_make_result(0, "feature-branch\n", "")) # branch conftest_mock_runner.push(_make_result(0, "", "")) # git ls-files (no tracked generated files) @@ -454,6 +460,7 @@ async def test_perform_merge_dirty_check_ignores_generated_files( conftest_mock_runner.push(_make_result(0, "", "")) # rebase conftest_mock_runner.push(_make_result(0, f"worktree {fake_wt}\n", "")) # wt list conftest_mock_runner.push(_make_result(0, "dev\n", "")) # step 7.5: branch --show-current + conftest_mock_runner.push(_make_result(0, "", "")) # step 7.6: git status --porcelain (clean) conftest_mock_runner.push(_make_result(0, "", "")) # merge conftest_mock_runner.push(_make_result(0, "", "")) # wt remove conftest_mock_runner.push(_make_result(0, "", "")) # branch -D @@ -477,7 +484,7 @@ async def test_perform_merge_strips_generated_files_before_dirty_check( from autoskillit.server.git import perform_merge fake_wt = str(tmp_path) - tester = StatefulMockTester(results=[TestResult(True, "ok", ""), TestResult(True, "ok", "")]) + tester = InMemoryTestRunner(results=[TestResult(True, "ok", ""), TestResult(True, "ok", "")]) conftest_mock_runner.push(_make_result(0, f"{fake_wt}/.git/worktrees/wt", "")) # rev-parse conftest_mock_runner.push(_make_result(0, "feature-branch\n", "")) # branch conftest_mock_runner.push( @@ -492,6 +499,7 @@ async def test_perform_merge_strips_generated_files_before_dirty_check( conftest_mock_runner.push(_make_result(0, "", "")) # rebase conftest_mock_runner.push(_make_result(0, f"worktree {fake_wt}\n", "")) # wt list conftest_mock_runner.push(_make_result(0, "dev\n", "")) # step 7.5: branch --show-current + conftest_mock_runner.push(_make_result(0, "", "")) # step 7.6: git status --porcelain (clean) conftest_mock_runner.push(_make_result(0, "", "")) # merge conftest_mock_runner.push(_make_result(0, "", "")) # wt remove conftest_mock_runner.push(_make_result(0, "", "")) # branch -D @@ -508,9 +516,9 @@ async def test_perform_merge_strips_generated_files_before_dirty_check( assert ls_files_idx < status_idx < rebase_idx -def _make_tester() -> StatefulMockTester: - """Return a StatefulMockTester with two passing test results (pre- and post-rebase).""" - return StatefulMockTester( +def _make_tester() -> InMemoryTestRunner: + """Return a InMemoryTestRunner with two passing test results (pre- and post-rebase).""" + return InMemoryTestRunner( results=[ TestResult(True, "PASS\n= 10 passed =", ""), TestResult(True, "PASS\n= 10 passed =", ""), @@ -527,7 +535,7 @@ def _push_full_success_sequence( """Push the git subprocess sequence for a successful merge onto runner. Covers all git calls in perform_merge (steps 2-9, 11, 12). The test gate - (step 4) is handled by StatefulMockTester, not via the runner. + (step 4) is handled by InMemoryTestRunner, not via the runner. Cleanup steps (remove_git_worktree, branch -D) use the runner default (rc=0). """ runner.push(_make_result(0, "/repo/.git/worktrees/impl-test\n")) # rev-parse --git-dir @@ -542,6 +550,7 @@ def _push_full_success_sequence( _make_result(0, f"worktree /repo\nHEAD abc123\nbranch refs/heads/{base_branch}\n\n") ) runner.push(_make_result(0, f"{base_branch}\n")) # step 7.5: branch --show-current (main_repo) + runner.push(_make_result(0, "")) # step 7.6: git status --porcelain (main_repo clean) runner.push(_make_result(0, "")) # git merge --no-edit @@ -710,6 +719,7 @@ async def test_succeeds_when_main_repo_branch_matches(self, tmp_path): runner.push(_make_result(0, "worktree /repo\nHEAD abc123\nbranch refs/heads/dev\n\n")) # Step 7.5: git branch --show-current on main_repo returns 'dev' runner.push(_make_result(0, "dev\n")) + runner.push(_make_result(0, "")) # step 7.6: git status --porcelain (clean) runner.push(_make_result(0, "")) # git merge --no-edit # cleanup: remove_git_worktree + branch -D use runner defaults (rc=0) @@ -747,6 +757,7 @@ async def test_merge_command_cwd_is_main_repo(self, tmp_path): runner.push(_make_result(0, "worktree /repo\nHEAD abc123\nbranch refs/heads/dev\n\n")) # Step 7.5: branch verification runner.push(_make_result(0, "dev\n")) + runner.push(_make_result(0, "")) # step 7.6: git status --porcelain (clean) runner.push(_make_result(0, "")) # git merge --no-edit result = await perform_merge( @@ -789,6 +800,7 @@ async def test_into_branch_reflects_verified_state(self, tmp_path): runner.push(_make_result(0, "worktree /repo\nHEAD abc123\nbranch refs/heads/dev\n\n")) # Step 7.5: verified branch runner.push(_make_result(0, "dev\n")) + runner.push(_make_result(0, "")) # step 7.6: git status --porcelain (clean) runner.push(_make_result(0, "")) # merge result = await perform_merge( @@ -802,3 +814,48 @@ async def test_into_branch_reflects_verified_state(self, tmp_path): assert result["merge_succeeded"] is True # into_branch must match the verified branch from step 7.5 assert result["into_branch"] == "dev" + + +def _make_runner_for_fetch(fake_wt: str) -> MockSubprocessRunner: + """Prepare a MockSubprocessRunner that succeeds through the dirty check then fails at fetch.""" + runner = MockSubprocessRunner() + runner.push(_make_result(0, f"{fake_wt}/.git/worktrees/wt", "")) # rev-parse (worktree ok) + runner.push(_make_result(0, "feature-branch\n", "")) # branch --show-current + runner.push(_make_result(0, "", "")) # ls-files (no tracked generated files) + runner.push(_make_result(0, "", "")) # status --porcelain (clean) + runner.push(_make_result(1, "", "fetch failed")) # fetch → fail to stop early + return runner + + +@pytest.mark.anyio +async def test_perform_merge_uses_custom_remote(tmp_path): + """perform_merge(..., remote='upstream') must use 'upstream' in all git remote operations.""" + from autoskillit.config import SafetyConfig + from autoskillit.server.git import perform_merge + + fake_wt = str(tmp_path) + runner = _make_runner_for_fetch(fake_wt) + config = AutomationConfig(safety=SafetyConfig(test_gate_on_merge=False)) + + result = await perform_merge(fake_wt, "dev", remote="upstream", config=config, runner=runner) + + assert result["failed_step"] == MergeFailedStep.FETCH + fetch_cmds = [cmd for cmd, *_ in runner.call_args_list if cmd[:2] == ["git", "fetch"]] + assert fetch_cmds == [["git", "fetch", "upstream"]] + + +@pytest.mark.anyio +async def test_perform_merge_defaults_to_origin(tmp_path): + """perform_merge() without remote kwarg uses 'origin' for backward compatibility.""" + from autoskillit.config import SafetyConfig + from autoskillit.server.git import perform_merge + + fake_wt = str(tmp_path) + runner = _make_runner_for_fetch(fake_wt) + config = AutomationConfig(safety=SafetyConfig(test_gate_on_merge=False)) + + result = await perform_merge(fake_wt, "dev", config=config, runner=runner) + + assert result["failed_step"] == MergeFailedStep.FETCH + fetch_cmds = [cmd for cmd, *_ in runner.call_args_list if cmd[:2] == ["git", "fetch"]] + assert fetch_cmds == [["git", "fetch", "origin"]] diff --git a/tests/server/test_git_merge_dirty_check.py b/tests/server/test_git_merge_dirty_check.py new file mode 100644 index 0000000000..a9ff24c16e --- /dev/null +++ b/tests/server/test_git_merge_dirty_check.py @@ -0,0 +1,146 @@ +"""Tests for the pre-merge dirty check in perform_merge (Layer 3).""" + +from __future__ import annotations + +from pathlib import Path +from unittest.mock import patch + +import pytest + +from autoskillit.config import AutomationConfig +from autoskillit.core import ( + MergeFailedStep, + MergeState, + SubprocessResult, + TerminationReason, + TestResult, +) +from tests.fakes import InMemoryTestRunner, MockSubprocessRunner + +pytestmark = [pytest.mark.layer("server"), pytest.mark.small] + + +def _make_result( + returncode: int = 0, + stdout: str = "", + stderr: str = "", +) -> SubprocessResult: + return SubprocessResult( + returncode=returncode, + stdout=stdout, + stderr=stderr, + termination=TerminationReason.NATURAL_EXIT, + pid=12345, + ) + + +def _make_tester() -> InMemoryTestRunner: + return InMemoryTestRunner( + results=[ + TestResult(True, "PASS\n= 10 passed =", ""), + TestResult(True, "PASS\n= 10 passed =", ""), + ] + ) + + +def _push_through_verify_merge( + runner: MockSubprocessRunner, + fake_wt: str, + *, + base_branch: str = "dev", +) -> None: + """Push responses through step 7.5 (_verify_merge_target).""" + runner.push(_make_result(0, f"{fake_wt}/.git/worktrees/wt")) # rev-parse --git-dir + runner.push(_make_result(0, "feature-branch\n")) # branch --show-current (worktree) + runner.push(_make_result(0, "")) # git ls-files (generated) + runner.push(_make_result(0, "")) # git status --porcelain (worktree clean) + runner.push(_make_result(0, "")) # fetch + runner.push(_make_result(0, "abc123\n")) # rev-parse --verify + runner.push(_make_result(0, "")) # git log --merges + runner.push(_make_result(0, "")) # rebase + runner.push( + _make_result(0, f"worktree /repo\nHEAD abc123\nbranch refs/heads/{base_branch}\n\n") + ) # wt list + runner.push(_make_result(0, f"{base_branch}\n")) # step 7.5: branch --show-current (main_repo) + + +@pytest.mark.anyio +async def test_dirty_main_repo_returns_error(tmp_path): + from autoskillit.server.git import perform_merge + + fake_wt = str(tmp_path / "worktrees" / "impl-test") + Path(fake_wt).mkdir(parents=True) + (Path(fake_wt) / ".git").write_text("gitdir: /repo/.git/worktrees/impl-test") + + runner = MockSubprocessRunner() + _push_through_verify_merge(runner, fake_wt) + runner.push(_make_result(0, " M src/foo.py\n M tests/bar.py\n")) # step 7.6: dirty! + + with patch("autoskillit.server.git.scan_editable_installs_for_worktree", return_value=[]): + result = await perform_merge( + fake_wt, + "dev", + config=AutomationConfig(), + runner=runner, + tester=_make_tester(), + ) + + assert "error" in result + assert result["failed_step"] == MergeFailedStep.DIRTY_MAIN_REPO + assert result["state"] == MergeState.WORKTREE_INTACT + assert result["worktree_path"] == fake_wt + assert len(result["dirty_files"]) == 2 + + +@pytest.mark.anyio +async def test_clean_main_repo_proceeds_to_merge(tmp_path): + from autoskillit.server.git import perform_merge + + fake_wt = str(tmp_path / "worktrees" / "impl-test") + Path(fake_wt).mkdir(parents=True) + (Path(fake_wt) / ".git").write_text("gitdir: /repo/.git/worktrees/impl-test") + + runner = MockSubprocessRunner() + _push_through_verify_merge(runner, fake_wt) + runner.push(_make_result(0, "")) # step 7.6: clean + runner.push(_make_result(0, "")) # merge + runner.push(_make_result(0, "")) # wt remove + runner.push(_make_result(0, "")) # branch -D + + with patch("autoskillit.server.git.scan_editable_installs_for_worktree", return_value=[]): + result = await perform_merge( + fake_wt, + "dev", + config=AutomationConfig(), + runner=runner, + tester=_make_tester(), + ) + + assert result.get("merge_succeeded") is True + + +@pytest.mark.anyio +async def test_dirty_check_error_format(tmp_path): + from autoskillit.server.git import perform_merge + + fake_wt = str(tmp_path / "worktrees" / "impl-test") + Path(fake_wt).mkdir(parents=True) + (Path(fake_wt) / ".git").write_text("gitdir: /repo/.git/worktrees/impl-test") + + runner = MockSubprocessRunner() + _push_through_verify_merge(runner, fake_wt) + dirty_output = " M file1.py\n M file2.py\n?? newfile.txt\n" + runner.push(_make_result(0, dirty_output)) # step 7.6: dirty + + with patch("autoskillit.server.git.scan_editable_installs_for_worktree", return_value=[]): + result = await perform_merge( + fake_wt, + "dev", + config=AutomationConfig(), + runner=runner, + tester=_make_tester(), + ) + + assert "3 dirty file(s)" in result["error"] + assert "Clean the working tree" in result["error"] + assert len(result["dirty_files"]) == 3 diff --git a/tests/server/test_guards_module.py b/tests/server/test_guards_module.py new file mode 100644 index 0000000000..459147caa6 --- /dev/null +++ b/tests/server/test_guards_module.py @@ -0,0 +1,32 @@ +"""Tests that _guards functions document their orchestration level requirements.""" + +import pytest + +pytestmark = [pytest.mark.layer("server"), pytest.mark.small] + + +def test_require_orchestrator_or_higher_doc_mentions_l1(): + from autoskillit.server._guards import _require_orchestrator_or_higher + + doc = _require_orchestrator_or_higher.__doc__ or "" + assert "L1" in doc + assert "L2" in doc + assert "L3" in doc + + +def test_require_orchestrator_exact_doc_mentions_l2(): + from autoskillit.server._guards import _require_orchestrator_exact + + doc = _require_orchestrator_exact.__doc__ or "" + assert "L2" in doc + assert "L1" in doc + assert "L3" in doc + + +def test_require_fleet_doc_mentions_l3(): + from autoskillit.server._guards import _require_fleet + + doc = _require_fleet.__doc__ or "" + assert "L3" in doc + assert "L1" in doc + assert "L2" in doc diff --git a/tests/server/test_headless_session.py b/tests/server/test_headless_session.py index f98b50b1b1..ba916a6bfd 100644 --- a/tests/server/test_headless_session.py +++ b/tests/server/test_headless_session.py @@ -4,6 +4,8 @@ import pytest +pytestmark = [pytest.mark.layer("server"), pytest.mark.medium] + @pytest.mark.anyio async def test_mcp_enable_kitchen_reveals_gated_tools(kitchen_enabled) -> None: @@ -20,4 +22,27 @@ async def test_mcp_enable_kitchen_reveals_gated_tools(kitchen_enabled) -> None: async with Client(mcp) as client: tool_names = {t.name for t in await client.list_tools()} - assert GATED_TOOLS.issubset(tool_names), f"Missing gated tools: {GATED_TOOLS - tool_names}" + from autoskillit.core import FLEET_DISPATCH_TOOLS, FLEET_TOOLS + + non_fleet_gated = GATED_TOOLS - FLEET_TOOLS - FLEET_DISPATCH_TOOLS + assert non_fleet_gated.issubset(tool_names), ( + f"Missing gated tools: {non_fleet_gated - tool_names}" + ) + + +@pytest.mark.anyio +async def test_mcp_enable_kitchen_does_not_reveal_fleet_tools(kitchen_enabled) -> None: + """mcp.enable(tags={'kitchen'}) must NOT reveal fleet-only tools to the client. + + Fleet tools carry kitchen-core (visible to L2/L3 via tag enable) but are NOT + revealed when only the 'kitchen' umbrella tag is enabled. + """ + from fastmcp.client import Client + + from autoskillit.core import FLEET_DISPATCH_TOOLS, FLEET_TOOLS + from autoskillit.server import mcp + + async with Client(mcp) as client: + tool_names = {t.name for t in await client.list_tools()} + fleet_visible = (FLEET_TOOLS | FLEET_DISPATCH_TOOLS) & tool_names + assert fleet_visible == set(), f"Fleet tools visible after kitchen enable: {fleet_visible}" diff --git a/tests/server/test_helpers_gate.py b/tests/server/test_helpers_gate.py new file mode 100644 index 0000000000..4833a4089a --- /dev/null +++ b/tests/server/test_helpers_gate.py @@ -0,0 +1,39 @@ +"""Contract tests: server helpers gate response schema.""" + +from __future__ import annotations + +import json + +import pytest + +pytestmark = [pytest.mark.layer("server"), pytest.mark.small] + + +class TestGateDisabledSchema: + """Gate-disabled response schema matches the expected skill result keys.""" + + EXPECTED_SKILL_KEYS = { + "success", + "result", + "session_id", + "subtype", + "cli_subtype", + "is_error", + "exit_code", + "kill_reason", + "needs_retry", + "retry_reason", + "stderr", + "token_usage", + "write_path_warnings", + "write_call_count", + } + + def test_gate_disabled_schema(self, tool_ctx): + """Gate-disabled response has standard keys.""" + from autoskillit.pipeline.gate import DefaultGateState + from autoskillit.server._guards import _require_enabled + + tool_ctx.gate = DefaultGateState(enabled=False) + response = json.loads(_require_enabled()) + assert set(response.keys()) == self.EXPECTED_SKILL_KEYS diff --git a/tests/server/test_helpers_tier_guards.py b/tests/server/test_helpers_tier_guards.py new file mode 100644 index 0000000000..987030e075 --- /dev/null +++ b/tests/server/test_helpers_tier_guards.py @@ -0,0 +1,161 @@ +"""Tests for tier-aware guard helpers in server._guards.""" + +from __future__ import annotations + +import json +import warnings + +import pytest + +pytestmark = [pytest.mark.layer("server"), pytest.mark.small] + + +# --------------------------------------------------------------------------- +# _require_orchestrator_or_higher +# --------------------------------------------------------------------------- + + +def test_A1_require_orchestrator_or_higher_permits_interactive(monkeypatch) -> None: + monkeypatch.delenv("AUTOSKILLIT_HEADLESS", raising=False) + from autoskillit.server._guards import _require_orchestrator_or_higher + + assert _require_orchestrator_or_higher("run_cmd") is None + + +def test_A2_require_orchestrator_or_higher_permits_headless_orchestrator(monkeypatch) -> None: + monkeypatch.setenv("AUTOSKILLIT_HEADLESS", "1") + monkeypatch.setenv("AUTOSKILLIT_SESSION_TYPE", "orchestrator") + from autoskillit.server._guards import _require_orchestrator_or_higher + + assert _require_orchestrator_or_higher("run_cmd") is None + + +def test_A3_require_orchestrator_or_higher_permits_headless_fleet(monkeypatch) -> None: + monkeypatch.setenv("AUTOSKILLIT_HEADLESS", "1") + monkeypatch.setenv("AUTOSKILLIT_SESSION_TYPE", "fleet") + from autoskillit.server._guards import _require_orchestrator_or_higher + + assert _require_orchestrator_or_higher("run_cmd") is None + + +def test_A4_require_orchestrator_or_higher_denies_headless_skill(monkeypatch) -> None: + monkeypatch.setenv("AUTOSKILLIT_HEADLESS", "1") + monkeypatch.setenv("AUTOSKILLIT_SESSION_TYPE", "skill") + from autoskillit.server._guards import _require_orchestrator_or_higher + + result = _require_orchestrator_or_higher("run_cmd") + assert result is not None + data = json.loads(result) + assert data["subtype"] == "headless_error" + + +def test_A5_require_orchestrator_or_higher_denies_headless_unset_session_type( + monkeypatch, +) -> None: + monkeypatch.setenv("AUTOSKILLIT_HEADLESS", "1") + monkeypatch.delenv("AUTOSKILLIT_SESSION_TYPE", raising=False) + from autoskillit.server._guards import _require_orchestrator_or_higher + + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + result = _require_orchestrator_or_higher("run_cmd") + assert result is not None + data = json.loads(result) + assert data["subtype"] == "headless_error" + + +def test_A6_require_orchestrator_or_higher_denies_headless_invalid_session_type( + monkeypatch, +) -> None: + monkeypatch.setenv("AUTOSKILLIT_HEADLESS", "1") + monkeypatch.setenv("AUTOSKILLIT_SESSION_TYPE", "bogus") + from autoskillit.server._guards import _require_orchestrator_or_higher + + with warnings.catch_warnings(record=True): + warnings.simplefilter("always") + result = _require_orchestrator_or_higher("run_cmd") + assert result is not None + data = json.loads(result) + assert data["subtype"] == "headless_error" + + +# --------------------------------------------------------------------------- +# _require_orchestrator_exact +# --------------------------------------------------------------------------- + + +def test_A7_require_orchestrator_exact_permits_interactive(monkeypatch) -> None: + monkeypatch.delenv("AUTOSKILLIT_HEADLESS", raising=False) + from autoskillit.server._guards import _require_orchestrator_exact + + assert _require_orchestrator_exact("open_kitchen") is None + + +def test_A8_require_orchestrator_exact_permits_headless_orchestrator(monkeypatch) -> None: + monkeypatch.setenv("AUTOSKILLIT_HEADLESS", "1") + monkeypatch.setenv("AUTOSKILLIT_SESSION_TYPE", "orchestrator") + from autoskillit.server._guards import _require_orchestrator_exact + + assert _require_orchestrator_exact("open_kitchen") is None + + +def test_A9_require_orchestrator_exact_denies_headless_fleet(monkeypatch) -> None: + monkeypatch.setenv("AUTOSKILLIT_HEADLESS", "1") + monkeypatch.setenv("AUTOSKILLIT_SESSION_TYPE", "fleet") + from autoskillit.server._guards import _require_orchestrator_exact + + result = _require_orchestrator_exact("open_kitchen") + assert result is not None + data = json.loads(result) + assert data["subtype"] == "headless_error" + msg = data.get("result", "").lower() + assert "fleet" in msg + + +def test_A10_require_orchestrator_exact_denies_headless_skill(monkeypatch) -> None: + monkeypatch.setenv("AUTOSKILLIT_HEADLESS", "1") + monkeypatch.setenv("AUTOSKILLIT_SESSION_TYPE", "skill") + from autoskillit.server._guards import _require_orchestrator_exact + + result = _require_orchestrator_exact("open_kitchen") + assert result is not None + data = json.loads(result) + assert data["subtype"] == "headless_error" + + +# --------------------------------------------------------------------------- +# _require_fleet +# --------------------------------------------------------------------------- + + +@pytest.mark.feature("fleet") +def test_A11_require_fleet_permits_fleet_session(monkeypatch) -> None: + monkeypatch.setenv("AUTOSKILLIT_SESSION_TYPE", "fleet") + from autoskillit.server._guards import _require_fleet + + assert _require_fleet() is None + + +@pytest.mark.feature("fleet") +def test_A12_require_fleet_denies_orchestrator(monkeypatch) -> None: + monkeypatch.setenv("AUTOSKILLIT_SESSION_TYPE", "orchestrator") + from autoskillit.server._guards import _require_fleet + + result = _require_fleet() + assert result is not None + data = json.loads(result) + assert data["subtype"] == "headless_error" + + +@pytest.mark.feature("fleet") +def test_A13_require_fleet_denies_interactive_no_session_type(monkeypatch) -> None: + monkeypatch.delenv("AUTOSKILLIT_SESSION_TYPE", raising=False) + monkeypatch.delenv("AUTOSKILLIT_HEADLESS", raising=False) + from autoskillit.server._guards import _require_fleet + + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + result = _require_fleet() + assert result is not None + data = json.loads(result) + assert data["subtype"] == "headless_error" diff --git a/tests/server/test_kitchen_lifecycle.py b/tests/server/test_kitchen_lifecycle.py new file mode 100644 index 0000000000..fb7142a71e --- /dev/null +++ b/tests/server/test_kitchen_lifecycle.py @@ -0,0 +1,60 @@ +import asyncio +import json +from unittest.mock import AsyncMock, patch + +import pytest + +from autoskillit.config import AutomationConfig +from autoskillit.core.types._type_plugin_source import DirectInstall +from autoskillit.hooks import _HOOK_CONFIG_PATH_COMPONENTS +from autoskillit.server import _state +from autoskillit.server._factory import make_context +from autoskillit.server.tools.tools_kitchen import _close_kitchen_handler, _open_kitchen_handler + +pytestmark = [pytest.mark.layer("server"), pytest.mark.medium] + + +async def test_kitchen_open_close_lifecycle(monkeypatch, tmp_path): + """Gate: disabled→enabled→disabled; hook_config written then removed; task cancelled.""" + monkeypatch.chdir(tmp_path) + + ctx = make_context( + AutomationConfig(), + runner=None, + plugin_source=DirectInstall(plugin_dir=tmp_path), + ) + monkeypatch.setattr(_state, "_ctx", ctx) + monkeypatch.setattr(_state, "_startup_ready", None) + + hook_config_path = tmp_path.joinpath(*_HOOK_CONFIG_PATH_COMPONENTS) + + with ( + patch("autoskillit.server.tools.tools_kitchen._prime_quota_cache", new_callable=AsyncMock), + patch("autoskillit.core.register_active_kitchen"), + patch("autoskillit.core.unregister_active_kitchen"), + ): + # initial state + assert ctx.gate.enabled is False + + # open kitchen + result = await _open_kitchen_handler() + assert result is None # no failure envelope + + assert ctx.gate.enabled is True + assert hook_config_path.exists() + data = json.loads(hook_config_path.read_text()) + assert "quota_guard" in data + assert "kitchen_id" in data + task = ctx.quota_refresh_task + assert task is not None + + # close kitchen + _close_kitchen_handler() + + assert ctx.gate.enabled is False + assert not hook_config_path.exists() + assert ctx.quota_refresh_task is None + + # drain the cancelled task from the event loop + await asyncio.sleep(0) + assert task.cancelled() or task.done() diff --git a/tests/server/test_lifespan.py b/tests/server/test_lifespan.py index 5fd28d8d08..d3939d6e83 100644 --- a/tests/server/test_lifespan.py +++ b/tests/server/test_lifespan.py @@ -8,6 +8,8 @@ from autoskillit.execution.recording import RecordingSubprocessRunner +pytestmark = [pytest.mark.layer("server"), pytest.mark.small] + @pytest.mark.asyncio async def test_lifespan_calls_finalize_on_recording_runner(): @@ -107,6 +109,69 @@ async def test_lifespan_sets_startup_ready_event(): _state._startup_ready = original +# T-WT-4: MCP lifespan startup detects broken hooks +def test_startup_broken_hook_detection(tmp_path: Path, monkeypatch) -> None: + """run_startup_hook_health_check must detect broken hook scripts across all scopes.""" + import json as _json + + from autoskillit.server._lifespan import run_startup_hook_health_check + + user_settings = tmp_path / ".claude" / "settings.json" + user_settings.parent.mkdir(parents=True) + user_settings.write_text( + _json.dumps( + { + "hooks": { + "PreToolUse": [ + { + "matcher": ".*", + "hooks": [ + { + "type": "command", + "command": "python3 /deleted/worktree/hooks/quota_guard.py", + } + ], + } + ] + } + } + ) + ) + + monkeypatch.setattr( + "autoskillit.hook_registry.iter_all_scope_paths", + lambda project_root=None: iter([("user", user_settings)]), + ) + + broken = run_startup_hook_health_check() + assert broken # must return non-empty list of broken hook paths + + +def test_startup_hook_health_checks_plugin_cache(tmp_path: Path, monkeypatch) -> None: + """run_startup_hook_health_check must include broken plugin cache paths in the result.""" + from autoskillit.server._lifespan import run_startup_hook_health_check + + stale_commands = [ + "python3 /stale/cache/path/hooks/quota_guard.py", + "python3 /stale/cache/path/hooks/write_guard.py", + ] + + monkeypatch.setattr( + "autoskillit.hook_registry.iter_all_scope_paths", + lambda project_root=None: iter([]), + ) + monkeypatch.setattr( + "autoskillit.hook_registry.validate_plugin_cache_hooks", + lambda cache_dir=None: stale_commands, + ) + + broken = run_startup_hook_health_check() + + assert set(stale_commands).issubset(set(broken)), ( + "run_startup_hook_health_check must include broken plugin cache paths in result" + ) + + def test_serve_startup_regenerates_on_hash_mismatch(tmp_path: Path, monkeypatch) -> None: """run_startup_drift_check() regenerates hooks.json when hash is mismatched.""" import json as _json @@ -127,3 +192,15 @@ def test_serve_startup_regenerates_on_hash_mismatch(tmp_path: Path, monkeypatch) updated = _json.loads((hooks_dir / "hooks.json").read_text()) assert updated.get("_autoskillit_registry_hash") == HOOK_REGISTRY_HASH + + +def test_lifespan_boot_registry_covers_all_session_types() -> None: + """_LIFESPAN_BOOT_REGISTRY must have an entry for every SessionType value.""" + from autoskillit.core import SessionType + from autoskillit.server._lifespan import _LIFESPAN_BOOT_REGISTRY + + missing = set(SessionType) - set(_LIFESPAN_BOOT_REGISTRY) + assert not missing, ( + f"SessionType values not in _LIFESPAN_BOOT_REGISTRY: {missing}. " + "Every SessionType must declare a boot function or None." + ) diff --git a/tests/server/test_lifespan_readiness_structural.py b/tests/server/test_lifespan_readiness_structural.py index a552072fb1..80026a134f 100644 --- a/tests/server/test_lifespan_readiness_structural.py +++ b/tests/server/test_lifespan_readiness_structural.py @@ -17,7 +17,9 @@ import pytest -from autoskillit.core._type_constants import RETIRED_READINESS_TOKENS +from autoskillit.core.types._type_constants import RETIRED_READINESS_TOKENS + +pytestmark = [pytest.mark.layer("server"), pytest.mark.small] _LIFESPAN_PATH = ( Path(__file__).parent.parent.parent / "src" / "autoskillit" / "server" / "_lifespan.py" diff --git a/tests/server/test_mcp_overrides.py b/tests/server/test_mcp_overrides.py index 761d683bfd..4e9509a7dc 100644 --- a/tests/server/test_mcp_overrides.py +++ b/tests/server/test_mcp_overrides.py @@ -6,6 +6,10 @@ from pathlib import Path from unittest.mock import AsyncMock, MagicMock, patch +import pytest + +pytestmark = [pytest.mark.layer("server"), pytest.mark.small] + def _make_mock_recipes(load_result: dict) -> MagicMock: """Create a mock recipe repository that returns the given load result.""" @@ -36,21 +40,25 @@ async def test_load_recipe_tool_accepts_overrides_param(tmp_path: Path) -> None: mock_tool_ctx = _make_mock_ctx(mock_recipes) with ( - patch("autoskillit.server.tools_recipe._require_enabled", return_value=None), - patch("autoskillit.server.tools_recipe._get_ctx_or_none", return_value=mock_tool_ctx), + patch("autoskillit.server.tools.tools_recipe._require_enabled", return_value=None), + patch( + "autoskillit.server.tools.tools_recipe._get_ctx_or_none", return_value=mock_tool_ctx + ), patch( "autoskillit.config.resolve_ingredient_defaults", return_value={}, ), patch( - "autoskillit.server.helpers._apply_triage_gate", + "autoskillit.server._misc._apply_triage_gate", new_callable=AsyncMock, return_value={"content": "test", "valid": True, "suggestions": []}, ), ): - from autoskillit.server.tools_recipe import load_recipe as _load_recipe_tool + from autoskillit.server.tools.tools_recipe import load_recipe as _load_recipe_tool - result_str = await _load_recipe_tool(name="test-recipe", overrides={"sprint_mode": "true"}) + result_str = await _load_recipe_tool( + name="test-recipe", overrides={"run_mode": "sequential"} + ) result = json.loads(result_str) assert "error" not in result assert result.get("valid") is True @@ -58,7 +66,7 @@ async def test_load_recipe_tool_accepts_overrides_param(tmp_path: Path) -> None: # Verify overrides were passed through to load_and_validate mock_recipes.load_and_validate.assert_called_once() call_kwargs = mock_recipes.load_and_validate.call_args - assert call_kwargs.kwargs.get("ingredient_overrides") == {"sprint_mode": "true"} + assert call_kwargs.kwargs.get("ingredient_overrides") == {"run_mode": "sequential"} async def test_open_kitchen_accepts_overrides_param(tmp_path: Path) -> None: @@ -76,9 +84,11 @@ async def test_open_kitchen_accepts_overrides_param(tmp_path: Path) -> None: mock_mcp_ctx.enable_components = AsyncMock() with ( - patch("autoskillit.server.tools_kitchen._require_not_headless", return_value=None), patch( - "autoskillit.server.tools_kitchen._open_kitchen_handler", + "autoskillit.server.tools.tools_kitchen._require_orchestrator_exact", return_value=None + ), + patch( + "autoskillit.server.tools.tools_kitchen._open_kitchen_handler", new_callable=AsyncMock, return_value=None, ), @@ -88,17 +98,17 @@ async def test_open_kitchen_accepts_overrides_param(tmp_path: Path) -> None: return_value={}, ), patch( - "autoskillit.server.helpers._apply_triage_gate", + "autoskillit.server._misc._apply_triage_gate", new_callable=AsyncMock, return_value={"content": "test", "valid": True, "suggestions": []}, ), - patch("autoskillit.server.tools_kitchen.__version__", "0.0.0"), + patch("autoskillit.server.tools.tools_kitchen.__version__", "0.0.0"), ): - from autoskillit.server.tools_kitchen import open_kitchen as _open_kitchen_tool + from autoskillit.server.tools.tools_kitchen import open_kitchen as _open_kitchen_tool result_str = await _open_kitchen_tool( name="test-recipe", - overrides={"sprint_mode": "true"}, + overrides={"run_mode": "sequential"}, ctx=mock_mcp_ctx, ) result = json.loads(result_str) @@ -108,7 +118,7 @@ async def test_open_kitchen_accepts_overrides_param(tmp_path: Path) -> None: # Verify overrides were passed through to load_and_validate mock_recipes.load_and_validate.assert_called_once() call_kwargs = mock_recipes.load_and_validate.call_args - assert call_kwargs.kwargs.get("ingredient_overrides") == {"sprint_mode": "true"} + assert call_kwargs.kwargs.get("ingredient_overrides") == {"run_mode": "sequential"} async def test_unknown_override_key_ignored(tmp_path: Path) -> None: diff --git a/tests/server/test_misc_module.py b/tests/server/test_misc_module.py new file mode 100644 index 0000000000..a354e74cff --- /dev/null +++ b/tests/server/test_misc_module.py @@ -0,0 +1,72 @@ +"""Contract tests: server._misc module.""" + +from __future__ import annotations + +import subprocess +from pathlib import Path + +import pytest + +pytestmark = [pytest.mark.layer("server"), pytest.mark.medium] + + +@pytest.mark.anyio +async def test_resolve_repo_from_remote_returns_empty_for_file_url(tmp_path: Path) -> None: + """Regression guard: file:// origin, no upstream → resolve_repo_from_remote returns ''. + + Creates a minimal git repo with only a file:// origin remote (no upstream). + """ + from autoskillit.server._misc import resolve_repo_from_remote + + repo = tmp_path / "clone" + repo.mkdir() + subprocess.run(["git", "init", str(repo)], check=True, capture_output=True) + subprocess.run( + ["git", "remote", "add", "origin", f"file://{tmp_path}/bare"], + cwd=str(repo), + check=True, + capture_output=True, + ) + # No upstream remote — simulates file:// origin-only scenario + + result = await resolve_repo_from_remote(str(repo)) + assert result == "" + + +def test_resolve_repo_from_remote_exists() -> None: + """Function renamed from infer_repo_from_remote must exist at new name.""" + import inspect + + from autoskillit.server._misc import resolve_repo_from_remote + + assert inspect.iscoroutinefunction(resolve_repo_from_remote) + + +def test_notify_module_exports(): + from autoskillit.server._notify import _get_ctx_or_none, _notify, track_response_size + + assert callable(_notify) + assert callable(track_response_size) + assert callable(_get_ctx_or_none) + + +def test_misc_module_exports(): + from autoskillit.server._misc import ( + _HOOK_CONFIG_FILENAME, + _HOOK_DIR_COMPONENTS, + _apply_triage_gate, + _extract_block, + _hook_config_path, + _prime_quota_cache, + _quota_refresh_loop, + resolve_repo_from_remote, + ) + + assert callable(_prime_quota_cache) + assert callable(_quota_refresh_loop) + assert callable(_apply_triage_gate) + assert callable(_hook_config_path) + assert callable(_extract_block) + assert callable(resolve_repo_from_remote) + assert isinstance(_HOOK_CONFIG_FILENAME, str) + assert isinstance(_HOOK_DIR_COMPONENTS, tuple) diff --git a/tests/server/test_no_raw_signal_handler.py b/tests/server/test_no_raw_signal_handler.py index cd0df12fce..ab6fb62f36 100644 --- a/tests/server/test_no_raw_signal_handler.py +++ b/tests/server/test_no_raw_signal_handler.py @@ -15,6 +15,10 @@ import re from pathlib import Path +import pytest + +pytestmark = [pytest.mark.layer("server"), pytest.mark.small] + _APP_PATH = Path(__file__).parent.parent.parent / "src" / "autoskillit" / "cli" / "app.py" _SRC_ROOT = Path(__file__).parent.parent.parent / "src" / "autoskillit" diff --git a/tests/server/test_perform_merge_editable_guard.py b/tests/server/test_perform_merge_editable_guard.py index 2cc351ad40..429cae3e43 100644 --- a/tests/server/test_perform_merge_editable_guard.py +++ b/tests/server/test_perform_merge_editable_guard.py @@ -12,7 +12,9 @@ TerminationReason, TestResult, ) -from tests.conftest import MockSubprocessRunner, StatefulMockTester +from tests.fakes import InMemoryTestRunner, MockSubprocessRunner + +pytestmark = [pytest.mark.layer("server"), pytest.mark.small] def _make_result(returncode: int = 0, stdout: str = "", stderr: str = "") -> SubprocessResult: @@ -46,7 +48,7 @@ async def test_perform_merge_aborts_before_cleanup_on_poisoned_install( ) runner = MockSubprocessRunner() - tester = StatefulMockTester( + tester = InMemoryTestRunner( results=[TestResult(True, "= 10 passed =", ""), TestResult(True, "= 10 passed =", "")] ) # Queue git calls in step order through merge (step 8); cleanup is never reached. @@ -60,6 +62,7 @@ async def test_perform_merge_aborts_before_cleanup_on_poisoned_install( runner.push(_make_result(0, "")) # git rebase (step 6) runner.push(_make_result(0, f"worktree {fake_wt}\n")) # git worktree list (step 7) runner.push(_make_result(0, "dev\n")) # git branch --show-current (step 7.5) + runner.push(_make_result(0, "")) # git status --porcelain (step 7.6) runner.push(_make_result(0, "")) # git merge (step 8) # Step 8.5: editable guard fires (mocked above) — cleanup steps never reached @@ -104,7 +107,7 @@ async def test_perform_merge_proceeds_normally_when_guard_returns_empty( ) runner = MockSubprocessRunner() - tester = StatefulMockTester( + tester = InMemoryTestRunner( results=[TestResult(True, "= 10 passed =", ""), TestResult(True, "= 10 passed =", "")] ) runner.push(_make_result(0, f"{fake_wt}/.git/worktrees/wt")) # rev-parse (step 2) @@ -117,6 +120,7 @@ async def test_perform_merge_proceeds_normally_when_guard_returns_empty( runner.push(_make_result(0, "")) # git rebase (step 6) runner.push(_make_result(0, f"worktree {fake_wt}\n")) # git worktree list (step 7) runner.push(_make_result(0, "dev\n")) # git branch --show-current (step 7.5) + runner.push(_make_result(0, "")) # git status --porcelain (step 7.6) runner.push(_make_result(0, "")) # git merge (step 8) # Step 8.5: guard returns [] — cleanup proceeds # Steps 9-10 (wt remove, branch -D) use MockSubprocessRunner default (rc=0, stdout="") diff --git a/tests/server/test_quota_refresh_loop.py b/tests/server/test_quota_refresh_loop.py index b40f1d876c..42228f1aa5 100644 --- a/tests/server/test_quota_refresh_loop.py +++ b/tests/server/test_quota_refresh_loop.py @@ -1,4 +1,4 @@ -"""Tests for _quota_refresh_loop in server/helpers.py.""" +"""Tests for _quota_refresh_loop in server/_misc.py.""" from __future__ import annotations @@ -9,11 +9,13 @@ from autoskillit.config.settings import QuotaGuardConfig +pytestmark = [pytest.mark.layer("server"), pytest.mark.small] + @pytest.mark.anyio async def test_quota_refresh_loop_calls_refresh_at_each_interval(monkeypatch): """Loop calls _refresh_quota_cache once per cache_refresh_interval sleep.""" - from autoskillit.server.helpers import _quota_refresh_loop + from autoskillit.server._misc import _quota_refresh_loop call_count = 0 sleep_count = 0 @@ -28,8 +30,8 @@ async def fake_refresh(config): nonlocal call_count call_count += 1 - monkeypatch.setattr("autoskillit.server.helpers.asyncio.sleep", fake_sleep) - monkeypatch.setattr("autoskillit.server.helpers._refresh_quota_cache", fake_refresh) + monkeypatch.setattr("autoskillit.server._misc.asyncio.sleep", fake_sleep) + monkeypatch.setattr("autoskillit.server._misc._refresh_quota_cache", fake_refresh) config = QuotaGuardConfig(cache_refresh_interval=240) with pytest.raises(asyncio.CancelledError): @@ -41,13 +43,13 @@ async def fake_refresh(config): @pytest.mark.anyio async def test_quota_refresh_loop_exits_cleanly_on_cancel(monkeypatch): """CancelledError from asyncio.sleep propagates; loop does not swallow it.""" - from autoskillit.server.helpers import _quota_refresh_loop + from autoskillit.server._misc import _quota_refresh_loop async def immediate_cancel(n): raise asyncio.CancelledError - monkeypatch.setattr("autoskillit.server.helpers.asyncio.sleep", immediate_cancel) - monkeypatch.setattr("autoskillit.server.helpers._refresh_quota_cache", AsyncMock()) + monkeypatch.setattr("autoskillit.server._misc.asyncio.sleep", immediate_cancel) + monkeypatch.setattr("autoskillit.server._misc._refresh_quota_cache", AsyncMock()) task = asyncio.create_task(_quota_refresh_loop(QuotaGuardConfig())) task.cancel() with pytest.raises(asyncio.CancelledError): @@ -57,7 +59,7 @@ async def immediate_cancel(n): @pytest.mark.anyio async def test_quota_refresh_loop_continues_after_refresh_exception(monkeypatch): """A transient error in _refresh_quota_cache does not kill the loop.""" - from autoskillit.server.helpers import _quota_refresh_loop + from autoskillit.server._misc import _quota_refresh_loop call_count = 0 sleep_count = 0 @@ -74,8 +76,8 @@ async def flaky_refresh(config): if call_count == 1: raise OSError("network blip") - monkeypatch.setattr("autoskillit.server.helpers.asyncio.sleep", fake_sleep) - monkeypatch.setattr("autoskillit.server.helpers._refresh_quota_cache", flaky_refresh) + monkeypatch.setattr("autoskillit.server._misc.asyncio.sleep", fake_sleep) + monkeypatch.setattr("autoskillit.server._misc._refresh_quota_cache", flaky_refresh) with pytest.raises(asyncio.CancelledError): await _quota_refresh_loop(QuotaGuardConfig()) diff --git a/tests/server/test_release_issue_fail_label.py b/tests/server/test_release_issue_fail_label.py new file mode 100644 index 0000000000..0de1837920 --- /dev/null +++ b/tests/server/test_release_issue_fail_label.py @@ -0,0 +1,117 @@ +"""Tests for release_issue fail_label path and fail label cleanup on claim/release.""" + +from __future__ import annotations + +import json +from unittest.mock import AsyncMock + +import pytest + +from autoskillit.server.tools.tools_issue_lifecycle import claim_issue, release_issue + +pytestmark = [pytest.mark.layer("server"), pytest.mark.small] + + +class TestReleaseIssueFailLabel: + @pytest.mark.anyio + async def test_release_issue_fail_label_swap(self, tool_ctx_kitchen_open, monkeypatch): + """fail_label swaps in-progress for fail label.""" + mock_client = AsyncMock() + mock_client.ensure_label.return_value = {"success": True, "created": True} + mock_client.swap_labels.return_value = {"success": True, "labels": ["fail"]} + monkeypatch.setattr(tool_ctx_kitchen_open, "github_client", mock_client) + + result = json.loads( + await release_issue( + issue_url="https://github.com/owner/repo/issues/42", + fail_label="fail", + ) + ) + + assert result["success"] is True + assert result["failed"] is True + assert result["fail_label"] == "fail" + mock_client.ensure_label.assert_called_once_with( + "owner", "repo", "fail", color="d73a4a", description="Recipe execution failed" + ) + swap_call = mock_client.swap_labels.call_args + assert set(swap_call.kwargs["remove_labels"]) == {"in-progress", "queued"} + assert swap_call.kwargs["add_labels"] == ["fail"] + + @pytest.mark.anyio + async def test_release_issue_success_removes_fail_label( + self, tool_ctx_kitchen_open, monkeypatch + ): + """Staging path includes fail label in remove_labels for cleanup.""" + mock_client = AsyncMock() + mock_client.ensure_label.return_value = {"success": True, "created": True} + mock_client.swap_labels.return_value = {"success": True, "labels": ["staged"]} + monkeypatch.setattr(tool_ctx_kitchen_open, "github_client", mock_client) + + result = json.loads( + await release_issue( + issue_url="https://github.com/owner/repo/issues/42", + target_branch="develop", + ) + ) + + assert result["success"] is True + assert result["staged"] is True + swap_call = mock_client.swap_labels.call_args + assert "in-progress" in swap_call.kwargs["remove_labels"] + assert "fail" in swap_call.kwargs["remove_labels"] + assert "queued" in swap_call.kwargs["remove_labels"] + assert "staged" in swap_call.kwargs["add_labels"] + + @pytest.mark.anyio + async def test_release_issue_simple_remove_cleans_fail_label( + self, tool_ctx_kitchen_open, monkeypatch + ): + """Simple release (no staging, no fail_label) also removes fail label.""" + mock_client = AsyncMock() + mock_client.swap_labels.return_value = {"success": True, "labels": []} + monkeypatch.setattr(tool_ctx_kitchen_open, "github_client", mock_client) + + result = json.loads( + await release_issue( + issue_url="https://github.com/owner/repo/issues/42", + ) + ) + + assert result["success"] is True + swap_call = mock_client.swap_labels.call_args + assert "in-progress" in swap_call.kwargs["remove_labels"] + assert "fail" in swap_call.kwargs["remove_labels"] + assert "queued" in swap_call.kwargs["remove_labels"] + + +class TestClaimIssueFailLabelCleanup: + @pytest.mark.anyio + async def test_claim_issue_removes_fail_label_on_claim( + self, tool_ctx_kitchen_open, monkeypatch + ): + """claim_issue uses swap_labels to remove fail label while adding in-progress.""" + mock_client = AsyncMock() + mock_client.fetch_issue.return_value = { + "success": True, + "labels": [{"name": "bug"}], + } + mock_client.ensure_label.return_value = {"success": True, "created": False} + mock_client.swap_labels.return_value = { + "success": True, + "labels": ["in-progress"], + } + monkeypatch.setattr(tool_ctx_kitchen_open, "github_client", mock_client) + + result = json.loads( + await claim_issue( + issue_url="https://github.com/owner/repo/issues/42", + ) + ) + + assert result["success"] is True + assert result["claimed"] is True + call_kwargs = mock_client.swap_labels.call_args.kwargs + assert set(call_kwargs["remove_labels"]) == {"queued", "fail"} + assert call_kwargs["add_labels"] == ["in-progress"] + mock_client.add_labels.assert_not_called() diff --git a/tests/server/test_reload_session.py b/tests/server/test_reload_session.py new file mode 100644 index 0000000000..48057ceacc --- /dev/null +++ b/tests/server/test_reload_session.py @@ -0,0 +1,165 @@ +"""Tests for the reload_session MCP tool and supporting helpers.""" + +from __future__ import annotations + +import json +from pathlib import Path +from unittest.mock import patch + +import pytest + +pytestmark = [pytest.mark.layer("server"), pytest.mark.small] + + +# --------------------------------------------------------------------------- +# RS-1 — reload_session is in FREE_RANGE_TOOLS +# --------------------------------------------------------------------------- + + +def test_reload_session_in_free_range_tools() -> None: + from autoskillit.core.types._type_constants import FREE_RANGE_TOOLS + + assert "reload_session" in FREE_RANGE_TOOLS + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + + +def _write_kitchen_marker(project_dir: Path, session_id: str) -> None: + """Write a kitchen marker under AUTOSKILLIT_STATE_DIR = project_dir.""" + state_dir = project_dir / "kitchen_state" + state_dir.mkdir(parents=True, exist_ok=True) + marker_path = state_dir / f"{session_id}.json" + from datetime import UTC, datetime + + payload = { + "session_id": session_id, + "opened_at": datetime.now(UTC).isoformat(), + "recipe_name": None, + "marker_version": 1, + "content_hash": "", + "composite_hash": "", + } + marker_path.write_text(json.dumps(payload), encoding="utf-8") + + +# --------------------------------------------------------------------------- +# RS-2 — Tool writes sentinel with session_id from kitchen marker +# --------------------------------------------------------------------------- + + +def test_reload_session_writes_sentinel_from_kitchen_marker( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.chdir(tmp_path) + monkeypatch.setenv("AUTOSKILLIT_STATE_DIR", str(tmp_path)) + _write_kitchen_marker(tmp_path, "abc123") + + from autoskillit.server.tools.tools_kitchen import _reload_session_handler + + result = _reload_session_handler() + + sentinel_path = tmp_path / ".autoskillit" / "temp" / "reload_sentinel" / "abc123.json" + assert sentinel_path.exists(), f"Sentinel not found at {sentinel_path}" + data = json.loads(sentinel_path.read_text(encoding="utf-8")) + assert data["session_id"] == "abc123" + assert result["session_id"] == "abc123" + + +# --------------------------------------------------------------------------- +# RS-3 — Tool falls back to find_latest_session_id when no kitchen marker +# --------------------------------------------------------------------------- + + +def test_reload_session_falls_back_to_find_latest( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.chdir(tmp_path) + # No kitchen marker — state dir is empty + monkeypatch.setenv("AUTOSKILLIT_STATE_DIR", str(tmp_path / "empty")) + + with patch( + "autoskillit.server.tools.tools_kitchen.find_latest_session_id", return_value="fallback-id" + ): + from autoskillit.server.tools.tools_kitchen import ( + _reload_session_handler as reload_session, + ) + + result = reload_session() + + sentinel_path = tmp_path / ".autoskillit" / "temp" / "reload_sentinel" / "fallback-id.json" + assert sentinel_path.exists() + data = json.loads(sentinel_path.read_text(encoding="utf-8")) + assert data["session_id"] == "fallback-id" + assert result["session_id"] == "fallback-id" + + +# --------------------------------------------------------------------------- +# RS-4 — Tool returns action instruction telling Claude to run /exit +# --------------------------------------------------------------------------- + + +def test_reload_session_returns_exit_instruction( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.chdir(tmp_path) + monkeypatch.setenv("AUTOSKILLIT_STATE_DIR", str(tmp_path / "empty")) + + with patch( + "autoskillit.server.tools.tools_kitchen.find_latest_session_id", return_value="sess-exit" + ): + from autoskillit.server.tools.tools_kitchen import ( + _reload_session_handler as reload_session, + ) + + result = reload_session() + + assert "/exit" in result["next_action"] + assert result["status"] == "reload_requested" + + +# --------------------------------------------------------------------------- +# RS-5 — Tool raises informative error when session_id cannot be determined +# --------------------------------------------------------------------------- + + +def test_reload_session_raises_when_no_session_id( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.chdir(tmp_path) + monkeypatch.setenv("AUTOSKILLIT_STATE_DIR", str(tmp_path / "empty")) + + with patch("autoskillit.server.tools.tools_kitchen.find_latest_session_id", return_value=None): + from autoskillit.server.tools.tools_kitchen import ( + _reload_session_handler as reload_session, + ) + + with pytest.raises(ValueError, match="session ID"): + reload_session() + + +# --------------------------------------------------------------------------- +# RS-6 — The async wrapper serializes to str (not raw dict) +# --------------------------------------------------------------------------- + + +@pytest.mark.anyio +async def test_reload_session_tool_wrapper_returns_str( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.chdir(tmp_path) + monkeypatch.setenv("AUTOSKILLIT_STATE_DIR", str(tmp_path / "empty")) + + with patch( + "autoskillit.server.tools.tools_kitchen.find_latest_session_id", return_value="sess-wrap" + ): + from autoskillit.server.tools.tools_kitchen import reload_session + + result = await reload_session() + + assert isinstance(result, str), f"Expected str, got {type(result).__name__}" + parsed = json.loads(result) + assert parsed["status"] == "reload_requested" + assert parsed["session_id"] == "sess-wrap" diff --git a/tests/server/test_research_smoke_pipeline.py b/tests/server/test_research_smoke_pipeline.py new file mode 100644 index 0000000000..0875f3f5b5 --- /dev/null +++ b/tests/server/test_research_smoke_pipeline.py @@ -0,0 +1,47 @@ +"""Research recipe smoke pipeline: structural and gated E2E tests. + +Gated E2E tests run only when RESEARCH_SMOKE_TEST=1 is set (via ``task test-smoke-research``). +The class is a documentation anchor for future pipeline execution validation. +""" + +from __future__ import annotations + +import os + +import pytest + +pytestmark = [pytest.mark.layer("server"), pytest.mark.medium] + + +# --------------------------------------------------------------------------- +# Test Group C: Gated E2E Smoke (T_RSE_1 – T_RSE_5) +# --------------------------------------------------------------------------- + + +@pytest.mark.skipif( + not os.environ.get("RESEARCH_SMOKE_TEST"), + reason="Set RESEARCH_SMOKE_TEST=1 to run research smoke tests", +) +@pytest.mark.smoke +class TestResearchSmokePipelineExecution: + """Full end-to-end research smoke execution. + + Run via ``task test-smoke-research`` which sets RESEARCH_SMOKE_TEST=1 and + invokes the research recipe with output_mode=local. This class is a + documentation anchor — the execution itself uses the research recipe pipeline. + """ + + def test_fixture_trivial_pipeline_completes(self) -> None: + pytest.skip("E2E execution requires RESEARCH_SMOKE_TEST=1 and API access") + + def test_fixture_rct_pipeline_completes(self) -> None: + pytest.skip("E2E execution requires RESEARCH_SMOKE_TEST=1 and API access") + + def test_fixture_rct_frontmatter_experiment_type(self) -> None: + pytest.skip("E2E execution requires RESEARCH_SMOKE_TEST=1 and API access") + + def test_fixture_rct_frontmatter_methodology_tradition(self) -> None: + pytest.skip("E2E execution requires RESEARCH_SMOKE_TEST=1 and API access") + + def test_fixture_uses_temp_dir_no_git(self) -> None: + pytest.skip("E2E execution requires RESEARCH_SMOKE_TEST=1 and API access") diff --git a/tests/server/test_resolve_model_as_profile.py b/tests/server/test_resolve_model_as_profile.py new file mode 100644 index 0000000000..c551079552 --- /dev/null +++ b/tests/server/test_resolve_model_as_profile.py @@ -0,0 +1,101 @@ +"""Tests for _resolve_model_as_profile model-as-profile resolution in _guards.py.""" + +from __future__ import annotations + +import pytest +import structlog + +pytestmark = [pytest.mark.layer("server"), pytest.mark.small] + + +def _make_config(**kwargs): + from autoskillit.config._config_dataclasses import ProvidersConfig + + return ProvidersConfig(**kwargs) + + +def test_model_matches_profile_with_anthropic_model(): + from autoskillit.server._guards import _resolve_model_as_profile + + cfg = _make_config( + profiles={ + "minimax": { + "ANTHROPIC_MODEL": "MiniMax-M2.7", + "ANTHROPIC_BASE_URL": "https://api.minimax.chat/v1", + } + } + ) + result = _resolve_model_as_profile("minimax", cfg) + assert result == ( + "MiniMax-M2.7", + "minimax", + {"ANTHROPIC_BASE_URL": "https://api.minimax.chat/v1"}, + ) + + +def test_model_matches_profile_without_anthropic_model(): + from autoskillit.server._guards import _resolve_model_as_profile + + cfg = _make_config(profiles={"broken": {"ANTHROPIC_BASE_URL": "https://example.com"}}) + with structlog.testing.capture_logs() as logs: + result = _resolve_model_as_profile("broken", cfg) + assert result == ("", "", None) + assert any( + log.get("event") == "provider_profile_no_model" and log.get("profile") == "broken" + for log in logs + ) + + +def test_model_no_match_passthrough(): + from autoskillit.server._guards import _resolve_model_as_profile + + cfg = _make_config(profiles={"minimax": {"ANTHROPIC_MODEL": "MiniMax-M2.7"}}) + result = _resolve_model_as_profile("sonnet", cfg) + assert result == ("sonnet", "", None) + + +def test_empty_model_passthrough(): + from autoskillit.server._guards import _resolve_model_as_profile + + cfg = _make_config() + with structlog.testing.capture_logs() as logs: + result = _resolve_model_as_profile("", cfg) + assert result == ("", "", None) + assert not any(log.get("event") == "provider_profile_no_model" for log in logs) + + +def test_anthropic_model_key_excluded_from_extras(): + from autoskillit.server._guards import _resolve_model_as_profile + + cfg = _make_config( + profiles={ + "minimax": { + "ANTHROPIC_MODEL": "M2.7", + "ANTHROPIC_BASE_URL": "https://api.minimax.chat/v1", + "ANTHROPIC_API_KEY": "test-key", + } + } + ) + effective_model, profile_name, extras = _resolve_model_as_profile("minimax", cfg) + assert effective_model == "M2.7" + assert profile_name == "minimax" + assert extras is not None + assert "ANTHROPIC_MODEL" not in extras + assert extras["ANTHROPIC_BASE_URL"] == "https://api.minimax.chat/v1" + assert extras["ANTHROPIC_API_KEY"] == "test-key" + + +def test_profile_with_only_anthropic_model_returns_empty_extras(): + from autoskillit.server._guards import _resolve_model_as_profile + + cfg = _make_config(profiles={"simple": {"ANTHROPIC_MODEL": "some-model"}}) + result = _resolve_model_as_profile("simple", cfg) + assert result == ("some-model", "simple", {}) + + +def test_anthropic_as_model_value_no_profile(): + from autoskillit.server._guards import _resolve_model_as_profile + + cfg = _make_config(profiles={"minimax": {"ANTHROPIC_MODEL": "M2.7"}}) + result = _resolve_model_as_profile("anthropic", cfg) + assert result == ("anthropic", "", None) diff --git a/tests/server/test_resolve_provider_profile.py b/tests/server/test_resolve_provider_profile.py new file mode 100644 index 0000000000..6d29d027fe --- /dev/null +++ b/tests/server/test_resolve_provider_profile.py @@ -0,0 +1,102 @@ +"""Tests for _resolve_provider_profile four-tier provider resolution.""" + +from __future__ import annotations + +import pytest + +pytestmark = [pytest.mark.layer("server"), pytest.mark.small] + + +def _make_config(**kwargs): + from autoskillit.config._config_dataclasses import ProvidersConfig + + return ProvidersConfig(**kwargs) + + +def test_step_override_wins(): + from autoskillit.server._guards import _resolve_provider_profile + + cfg = _make_config( + step_overrides={"my_step": "bedrock"}, + profiles={"bedrock": {"AWS_REGION": "us-east-1"}}, + ) + result = _resolve_provider_profile("my_step", "my_recipe", cfg) + assert result == ("bedrock", {"AWS_REGION": "us-east-1"}) + + +def test_recipe_wildcard_wins_when_no_step_override(): + from autoskillit.server._guards import _resolve_provider_profile + + cfg = _make_config( + step_overrides={"*": "vertex"}, + profiles={"vertex": {"GOOGLE_CLOUD_PROJECT": "proj"}}, + ) + result = _resolve_provider_profile("other_step", "my_recipe", cfg) + assert result == ("vertex", {"GOOGLE_CLOUD_PROJECT": "proj"}) + + +def test_step_yaml_provider_wins_when_no_config_overrides(): + from autoskillit.server._guards import _resolve_provider_profile + + cfg = _make_config( + profiles={"bedrock": {"AWS_REGION": "eu-west-1"}}, + ) + result = _resolve_provider_profile("bedrock", "my_recipe", cfg) + assert result == ("bedrock", {"AWS_REGION": "eu-west-1"}) + + +def test_default_anthropic_when_all_tiers_absent(): + from autoskillit.server._guards import _resolve_provider_profile + + cfg = _make_config() + result = _resolve_provider_profile("", "", cfg) + assert result == ("anthropic", {}) + + +def test_step_override_beats_wildcard_when_both_match(): + from autoskillit.server._guards import _resolve_provider_profile + + cfg = _make_config( + step_overrides={"my_step": "bedrock", "*": "vertex"}, + profiles={ + "bedrock": {"AWS_REGION": "us-east-1"}, + "vertex": {"GOOGLE_CLOUD_PROJECT": "proj"}, + }, + ) + result = _resolve_provider_profile("my_step", "my_recipe", cfg) + assert result == ("bedrock", {"AWS_REGION": "us-east-1"}) + + +def test_anthropic_profile_returns_empty_env_regardless(): + from autoskillit.server._guards import _resolve_provider_profile + + cfg = _make_config( + default_provider="anthropic", + profiles={"anthropic": {"SHOULD_IGNORE": "this"}}, + ) + result = _resolve_provider_profile("", "", cfg) + assert result == ("anthropic", {}) + + +def test_non_anthropic_profile_returns_correct_env_dict(): + from autoskillit.server._guards import _resolve_provider_profile + + cfg = _make_config( + step_overrides={"my_step": "bedrock"}, + profiles={"bedrock": {"AWS_ACCESS_KEY_ID": "test"}}, + ) + result = _resolve_provider_profile("my_step", "my_recipe", cfg) + assert result == ("bedrock", {"AWS_ACCESS_KEY_ID": "test"}) + + +def test_no_recipe_name_skips_step_override(): + # Without recipe context, Tier 1 is bypassed; Tier 3 uses step_name as the + # provider name. If the override had applied we'd get ("vertex", {"K": "V"}). + from autoskillit.server._guards import _resolve_provider_profile + + cfg = _make_config( + step_overrides={"bedrock": "vertex"}, + profiles={"vertex": {"K": "V"}, "bedrock": {"X": "Y"}}, + ) + result = _resolve_provider_profile("bedrock", "", cfg) + assert result == ("bedrock", {"X": "Y"}) diff --git a/tests/server/test_run_skill_add_dirs.py b/tests/server/test_run_skill_add_dirs.py new file mode 100644 index 0000000000..24b693a740 --- /dev/null +++ b/tests/server/test_run_skill_add_dirs.py @@ -0,0 +1,47 @@ +"""Contract tests: run_skill passes correct add_dirs to executor (T-OVR-014).""" + +from __future__ import annotations + +import pytest + +from autoskillit.core import ValidatedAddDir + +pytestmark = [pytest.mark.layer("server"), pytest.mark.small] + + +@pytest.mark.anyio +async def test_raw_skills_extended_excluded_from_run_skill_add_dirs(tool_ctx_kitchen_open): + """T-OVR-014: run_skill passes ephemeral session dir (not raw skills_extended/) as add_dirs.""" + from autoskillit.server.tools.tools_execution import run_skill + from tests.fakes import InMemoryHeadlessExecutor + + executor = InMemoryHeadlessExecutor() + tool_ctx_kitchen_open.executor = executor + + await run_skill("/autoskillit:investigate foo", "/some/cwd") + + add_dirs = executor.calls[0].add_dirs + + # All add_dirs entries must be ValidatedAddDir instances + assert all(isinstance(d, ValidatedAddDir) for d in add_dirs) + + # At least one add_dir must have .claude/skills/*/SKILL.md layout + from pathlib import Path + + has_skills = False + for d in add_dirs: + p = Path(d.path) + if list(p.glob(".claude/skills/*/SKILL.md")): + has_skills = True + break + assert has_skills, "run_skill must pass an --add-dir with discoverable skills" + + # Must NOT include raw skills_extended/ path + from autoskillit.workspace.skills import bundled_skills_extended_dir + + skills_ext = str(bundled_skills_extended_dir()) + add_dir_paths = [d.path for d in add_dirs] + assert skills_ext not in add_dir_paths, ( + "run_skill must not pass skills_extended/ directly — " + "it should route through DefaultSessionSkillManager" + ) diff --git a/tests/server/test_run_skill_resume.py b/tests/server/test_run_skill_resume.py new file mode 100644 index 0000000000..bbb1eb4007 --- /dev/null +++ b/tests/server/test_run_skill_resume.py @@ -0,0 +1,58 @@ +"""Tests for run_skill resume_session_id parameter threading (T4).""" + +from __future__ import annotations + +import json + +import pytest + +from autoskillit.server.tools.tools_execution import run_skill + +pytestmark = [pytest.mark.layer("server"), pytest.mark.small] + + +@pytest.mark.anyio +async def test_resume_session_id_threaded_to_executor(tool_ctx_kitchen_open, monkeypatch) -> None: + """resume_session_id flows from run_skill → executor.run().""" + from tests.fakes import InMemoryHeadlessExecutor + + executor = InMemoryHeadlessExecutor() + tool_ctx_kitchen_open.executor = executor + monkeypatch.setattr("autoskillit.server._ctx", tool_ctx_kitchen_open) + + await run_skill("/implement foo", "/tmp", resume_session_id="sess-123") + + assert len(executor.calls) == 1 + assert executor.calls[0].resume_session_id == "sess-123" + + +@pytest.mark.anyio +async def test_resume_skips_skill_command_validation(tool_ctx_kitchen_open, monkeypatch) -> None: + """When resume_session_id is set, non-slash skill_command is allowed.""" + from tests.fakes import InMemoryHeadlessExecutor + + executor = InMemoryHeadlessExecutor() + tool_ctx_kitchen_open.executor = executor + monkeypatch.setattr("autoskillit.server._ctx", tool_ctx_kitchen_open) + + result = await run_skill( + "Continue from where you left off", + "/tmp", + resume_session_id="sess-123", + ) + data = json.loads(result) + assert data["success"] is True # not rejected by _validate_skill_command + + +@pytest.mark.anyio +async def test_no_resume_still_validates_skill_command(tool_ctx_kitchen_open, monkeypatch) -> None: + """Without resume_session_id, non-slash skill_command is still rejected.""" + monkeypatch.setattr("autoskillit.server._ctx", tool_ctx_kitchen_open) + + result = await run_skill("Continue from where you left off", "/tmp") + data = json.loads(result) + assert data["success"] is False + assert ( + "slash" in data.get("error", "").lower() + or "skill_command" in data.get("result", "").lower() + ) diff --git a/tests/server/test_server_init.py b/tests/server/test_server_init_gate.py similarity index 83% rename from tests/server/test_server_init.py rename to tests/server/test_server_init_gate.py index 7d467e2f5e..8abe3d201a 100644 --- a/tests/server/test_server_init.py +++ b/tests/server/test_server_init_gate.py @@ -1,4 +1,4 @@ -"""Tests for kitchen gate access, visibility, and subset management.""" +"""Tests for server init: gate access, visibility, subset management, wire format compliance.""" from __future__ import annotations @@ -10,8 +10,10 @@ import structlog.testing from autoskillit.pipeline.gate import DefaultGateState -from autoskillit.server.helpers import _require_enabled -from autoskillit.server.tools_kitchen import _close_kitchen_handler, _open_kitchen_handler +from autoskillit.server._guards import _require_enabled +from autoskillit.server.tools.tools_kitchen import _close_kitchen_handler, _open_kitchen_handler + +pytestmark = [pytest.mark.layer("server"), pytest.mark.medium] class TestKitchenVisibility: @@ -22,7 +24,7 @@ async def test_kitchen_tools_hidden_at_startup(self): """No kitchen tool (gated or headless-tagged) appears in tools/list for a fresh session.""" from fastmcp.client import Client - from autoskillit.core.types import HEADLESS_TOOLS + from autoskillit.core import HEADLESS_TOOLS from autoskillit.pipeline.gate import GATED_TOOLS from autoskillit.server import mcp @@ -51,6 +53,19 @@ async def test_ungated_tools_visible_at_startup(self): # test_check must NOT be visible at startup (has kitchen tag) assert "test_check" not in tool_names, "test_check should not be visible at startup" + @pytest.mark.anyio + async def test_redisable_subsets_hides_kitchen_core(self) -> None: + """When kitchen-core is in disabled subsets, those tools stay hidden after open_kitchen.""" + from unittest.mock import AsyncMock + + from autoskillit.server.tools.tools_kitchen import _redisable_subsets + + mock_ctx = AsyncMock() + + await _redisable_subsets(mock_ctx, ["kitchen-core"], features={"fleet": True}) + + mock_ctx.disable_components.assert_called_once_with(tags={"kitchen-core"}) + class TestGatedToolAccess: """Prompt-gated tool access: tools disabled by default, user-only activation.""" @@ -63,7 +78,7 @@ def _close_kitchen(self, tool_ctx): @pytest.mark.anyio async def test_tools_return_error_when_disabled(self, tool_ctx): """All tools return standard gate error when gate is disabled.""" - from autoskillit.server.tools_execution import run_cmd + from autoskillit.server.tools.tools_execution import run_cmd result = json.loads(await run_cmd(cmd="echo hi", cwd="/tmp")) assert result["success"] is False @@ -75,8 +90,8 @@ async def test_tools_work_after_enable(self, tool_ctx): """After open_kitchen prompt handler sets the flag, tools execute normally.""" from unittest.mock import AsyncMock - from autoskillit.server import tools_kitchen as tools_kitchen_mod - from autoskillit.server.tools_execution import run_cmd + from autoskillit.server.tools import tools_kitchen as tools_kitchen_mod + from autoskillit.server.tools.tools_execution import run_cmd from tests.conftest import _make_result with patch.object(tools_kitchen_mod, "_prime_quota_cache", new=AsyncMock()): @@ -91,8 +106,8 @@ async def test_disable_reverses_enable(self, tool_ctx): """After close_kitchen prompt handler, tools return error again.""" from unittest.mock import AsyncMock - from autoskillit.server import tools_kitchen as tools_kitchen_mod - from autoskillit.server.tools_execution import run_cmd + from autoskillit.server.tools import tools_kitchen as tools_kitchen_mod + from autoskillit.server.tools.tools_execution import run_cmd with patch.object(tools_kitchen_mod, "_prime_quota_cache", new=AsyncMock()): with patch.object(tools_kitchen_mod, "_write_hook_config"): @@ -129,7 +144,7 @@ async def test_kitchen_tools_not_registered_as_prompts(self): @pytest.mark.anyio async def test_run_python_gated(self): """run_python requires tools to be enabled.""" - from autoskillit.server.tools_execution import run_python + from autoskillit.server.tools.tools_execution import run_python result = json.loads(await run_python(callable="json.dumps", args={"obj": 1})) assert result["success"] is False @@ -154,7 +169,7 @@ class TestGateTransitionLogs: async def test_open_kitchen_logs_gate_open(self, tool_ctx): from unittest.mock import AsyncMock - from autoskillit.server import tools_kitchen as tools_kitchen_mod + from autoskillit.server.tools import tools_kitchen as tools_kitchen_mod with patch.object(tools_kitchen_mod, "_prime_quota_cache", new=AsyncMock()): with patch.object(tools_kitchen_mod, "_write_hook_config"): @@ -202,7 +217,7 @@ async def test_close_kitchen_returns_cooking_confirmation(self, tool_ctx): """close_kitchen must return a cooking-themed closing message.""" from unittest.mock import AsyncMock, MagicMock - from autoskillit.server.tools_kitchen import close_kitchen + from autoskillit.server.tools.tools_kitchen import close_kitchen mock_ctx = MagicMock() mock_ctx.reset_visibility = AsyncMock() @@ -235,8 +250,8 @@ def _prompt_text(result) -> str: async def test_open_kitchen_instructs_status_call(self): from unittest.mock import AsyncMock, MagicMock - from autoskillit.server import tools_kitchen as tools_kitchen_mod - from autoskillit.server.tools_kitchen import open_kitchen + from autoskillit.server.tools import tools_kitchen as tools_kitchen_mod + from autoskillit.server.tools.tools_kitchen import open_kitchen mock_ctx = MagicMock() mock_ctx.enable_components = AsyncMock() @@ -252,8 +267,8 @@ async def test_open_kitchen_carries_orchestrator_contract(self): from unittest.mock import AsyncMock, MagicMock from autoskillit.server import PIPELINE_FORBIDDEN_TOOLS - from autoskillit.server import tools_kitchen as tools_kitchen_mod - from autoskillit.server.tools_kitchen import open_kitchen + from autoskillit.server.tools import tools_kitchen as tools_kitchen_mod + from autoskillit.server.tools.tools_kitchen import open_kitchen mock_ctx = MagicMock() mock_ctx.enable_components = AsyncMock() @@ -283,8 +298,8 @@ async def test_open_kitchen_carries_orchestrator_contract(self): async def test_open_kitchen_still_enables_on_mismatch(self, tmp_path, tool_ctx): from unittest.mock import AsyncMock, MagicMock - from autoskillit.server import tools_kitchen as tools_kitchen_mod - from autoskillit.server.tools_kitchen import open_kitchen + from autoskillit.server.tools import tools_kitchen as tools_kitchen_mod + from autoskillit.server.tools.tools_kitchen import open_kitchen mock_ctx = MagicMock() mock_ctx.enable_components = AsyncMock() @@ -293,7 +308,9 @@ async def test_open_kitchen_still_enables_on_mismatch(self, tmp_path, tool_ctx): (plugin_dir / "plugin.json").write_text( json.dumps({"name": "autoskillit", "version": "0.0.0"}) ) - tool_ctx.plugin_dir = str(tmp_path) + from autoskillit.core.types._type_plugin_source import DirectInstall + + tool_ctx.plugin_source = DirectInstall(plugin_dir=tmp_path) with patch.object(tools_kitchen_mod, "_prime_quota_cache", new=AsyncMock()): with patch.object(tools_kitchen_mod, "_write_hook_config"): await open_kitchen(ctx=mock_ctx) @@ -322,8 +339,8 @@ async def test_sous_chef_rules_injected_at_open_kitchen(self): """open_kitchen must include sous-chef global orchestration rules.""" from unittest.mock import AsyncMock, MagicMock - import autoskillit.server.tools_kitchen as tools_kitchen_mod - from autoskillit.server.tools_kitchen import open_kitchen + import autoskillit.server.tools.tools_kitchen as tools_kitchen_mod + from autoskillit.server.tools.tools_kitchen import open_kitchen mock_ctx = MagicMock() mock_ctx.enable_components = AsyncMock() @@ -339,8 +356,8 @@ async def test_open_kitchen_degrades_gracefully_without_sous_chef(self, monkeypa """open_kitchen must not raise when sous-chef/SKILL.md is absent.""" from unittest.mock import AsyncMock, MagicMock - import autoskillit.server.tools_kitchen as tools_kitchen_mod - from autoskillit.server.tools_kitchen import open_kitchen + import autoskillit.server.tools.tools_kitchen as tools_kitchen_mod + from autoskillit.server.tools.tools_kitchen import open_kitchen mock_ctx = MagicMock() mock_ctx.enable_components = AsyncMock() @@ -359,8 +376,8 @@ async def test_open_kitchen_has_no_update_advisory(tool_ctx): from unittest.mock import AsyncMock, MagicMock from autoskillit.pipeline.gate import DefaultGateState - from autoskillit.server import tools_kitchen as tools_kitchen_mod - from autoskillit.server.tools_kitchen import open_kitchen + from autoskillit.server.tools import tools_kitchen as tools_kitchen_mod + from autoskillit.server.tools.tools_kitchen import open_kitchen # Ensure kitchen is closed before calling open_kitchen tool_ctx.gate = DefaultGateState(enabled=False) @@ -440,19 +457,26 @@ def test_server_init_no_shebang() -> None: # Wire-format compliance (Claude Code #25081) # --------------------------------------------------------------------------- -# Fields that Claude Code #25081 silently rejects on tools/list responses. -# If any tool carries a non-None value for these fields, Claude Code drops -# the ENTIRE tool list from that server. -_REJECTED_FIELDS = {"outputSchema", "annotations"} +# Fields the middleware strips to avoid Claude Code #25081 tool-list rejection. +# Maps wire field name (camelCase) to the FastMCP Tool snake_case attribute name. +# annotations is intentionally absent: it carries readOnlyHint and must be preserved. +_STRIPPED_WIRE_FIELDS: dict[str, str] = { + "outputSchema": "output_schema", + "title": "title", +} class TestWireFormatCompliance: """tools/list wire response must be compatible with Claude Code's MCP parser.""" @pytest.mark.anyio - async def test_tools_list_contains_no_rejected_fields(self): + async def test_tools_list_contains_no_stripped_fields(self): """tools/list wire response must not contain fields that trigger - Claude Code #25081 (silent full-tool-list rejection).""" + Claude Code #25081 (silent full-tool-list rejection). + + Only output_schema and title are stripped. annotations is preserved + because it carries readOnlyHint for parallel execution semantics. + """ from fastmcp.client import Client from autoskillit.server import mcp @@ -460,12 +484,10 @@ async def test_tools_list_contains_no_rejected_fields(self): async with Client(mcp) as client: tools = await client.list_tools() for tool in tools: - for field in _REJECTED_FIELDS: - # FastMCP Tool objects use snake_case attrs for camelCase wire fields - attr = "output_schema" if field == "outputSchema" else field + for wire_field, attr in _STRIPPED_WIRE_FIELDS.items(): value = getattr(tool, attr, None) assert value is None, ( - f"Tool '{tool.name}' has non-None {field}={value!r}. " + f"Tool '{tool.name}' has non-None {wire_field}={value!r}. " f"Claude Code #25081 silently drops ALL tools when this field is present." ) @@ -484,11 +506,11 @@ async def test_middleware_strips_output_schema(self): tool = MagicMock() tool.name = "test_tool" tool.output_schema = {"type": "string"} - tool.annotations = None + tool.annotations = MagicMock(readOnlyHint=True) tool.model_copy.return_value = MagicMock( name="test_tool", output_schema=None, - annotations=None, + annotations=MagicMock(readOnlyHint=True), title=None, ) @@ -497,13 +519,13 @@ async def test_middleware_strips_output_schema(self): result = await mw.on_list_tools(ctx, call_next) tool.model_copy.assert_called_once_with( - update={"output_schema": None, "annotations": None, "title": None}, + update={"output_schema": None, "title": None}, ) assert result[0].output_schema is None @pytest.mark.anyio - async def test_middleware_strips_annotations(self): - """Middleware must strip annotations from every tool.""" + async def test_middleware_preserves_annotations(self): + """Middleware must preserve annotations (including readOnlyHint) on every tool.""" from unittest.mock import AsyncMock, MagicMock from autoskillit.server._wire_compat import ClaudeCodeCompatMiddleware @@ -516,7 +538,7 @@ async def test_middleware_strips_annotations(self): tool.model_copy.return_value = MagicMock( name="test_tool", output_schema=None, - annotations=None, + annotations=MagicMock(readOnlyHint=True), title=None, ) @@ -524,11 +546,12 @@ async def test_middleware_strips_annotations(self): call_next = AsyncMock(return_value=[tool]) result = await mw.on_list_tools(ctx, call_next) - assert result[0].annotations is None + assert result[0].annotations is not None + assert result[0].annotations.readOnlyHint is True @pytest.mark.anyio async def test_middleware_preserves_tool_identity(self): - """Middleware must not alter tool name, description, or inputSchema.""" + """Middleware must not alter tool name, description, inputSchema, or annotations.""" from unittest.mock import AsyncMock, MagicMock from autoskillit.server._wire_compat import ClaudeCodeCompatMiddleware @@ -545,7 +568,7 @@ async def test_middleware_preserves_tool_identity(self): copy_mock.description = "Does things" copy_mock.parameters = {"type": "object", "properties": {}} copy_mock.output_schema = None - copy_mock.annotations = None + copy_mock.annotations = MagicMock(readOnlyHint=True) copy_mock.title = None tool.model_copy.return_value = copy_mock @@ -556,3 +579,4 @@ async def test_middleware_preserves_tool_identity(self): assert result[0].name == "my_tool" assert result[0].description == "Does things" assert result[0].parameters == {"type": "object", "properties": {}} + assert result[0].annotations is not None diff --git a/tests/server/test_server_init_session_visibility.py b/tests/server/test_server_init_session_visibility.py new file mode 100644 index 0000000000..5b0af5368e --- /dev/null +++ b/tests/server/test_server_init_session_visibility.py @@ -0,0 +1,729 @@ +"""Tests for server init: session type visibility, fleet gate boot, feature gate visibility.""" + +from __future__ import annotations + +from pathlib import Path + +import pytest +import structlog.testing + +pytestmark = [pytest.mark.layer("server"), pytest.mark.medium] + + +@pytest.mark.feature("fleet") +class TestSessionTypeVisibility: + """3-branch session-type tag visibility dispatch.""" + + @pytest.mark.anyio + async def test_fleet_dispatch_mode_enables_fleet_dispatch_tools(self, monkeypatch): + """fleet + FLEET_MODE=dispatch reveals fleet tools + fleet-dispatch tools.""" + from autoskillit.core import ( + FLEET_DISPATCH_MODE, + FLEET_DISPATCH_TOOLS, + FLEET_MODE_ENV_VAR, + FLEET_TOOLS, + FREE_RANGE_TOOLS, + ) + from autoskillit.server import _apply_session_type_visibility, mcp + + monkeypatch.setenv("AUTOSKILLIT_SESSION_TYPE", "fleet") + monkeypatch.setenv(FLEET_MODE_ENV_VAR, FLEET_DISPATCH_MODE) + _apply_session_type_visibility() + + tools = list(await mcp.list_tools()) + visible = {t.name for t in tools} + + expected = FLEET_TOOLS | FLEET_DISPATCH_TOOLS | FREE_RANGE_TOOLS + assert visible == expected + + @pytest.mark.parametrize("mode_value", ["campaign", None]) + @pytest.mark.anyio + async def test_fleet_campaign_mode_hides_fleet_dispatch_tools(self, monkeypatch, mode_value): + """fleet + FLEET_MODE=campaign (or absent) hides fleet-dispatch tools.""" + from autoskillit.core import FLEET_DISPATCH_TOOLS, FLEET_MODE_ENV_VAR + from autoskillit.server import _apply_session_type_visibility, mcp + + monkeypatch.setenv("AUTOSKILLIT_SESSION_TYPE", "fleet") + if mode_value is not None: + monkeypatch.setenv(FLEET_MODE_ENV_VAR, mode_value) + else: + monkeypatch.delenv(FLEET_MODE_ENV_VAR, raising=False) + _apply_session_type_visibility() + + tools = list(await mcp.list_tools()) + visible = {t.name for t in tools} + assert visible.isdisjoint(FLEET_DISPATCH_TOOLS), ( + f"fleet-dispatch tools unexpectedly visible with FLEET_MODE={mode_value!r}" + ) + + @pytest.mark.anyio + async def test_fleet_dispatch_constant_matches_tagged_tools(self, monkeypatch): + """FLEET_DISPATCH_TOOLS constant must exactly match tools tagged fleet-dispatch.""" + from autoskillit.core import FLEET_DISPATCH_MODE, FLEET_DISPATCH_TOOLS, FLEET_MODE_ENV_VAR + from autoskillit.server import _apply_session_type_visibility, mcp + + monkeypatch.setenv("AUTOSKILLIT_SESSION_TYPE", "fleet") + monkeypatch.setenv(FLEET_MODE_ENV_VAR, FLEET_DISPATCH_MODE) + _apply_session_type_visibility() + + all_tools = {t.name: t for t in await mcp.list_tools()} + tagged = {name for name, t in all_tools.items() if "fleet-dispatch" in t.tags} + assert tagged == FLEET_DISPATCH_TOOLS, ( + f"FLEET_DISPATCH_TOOLS constant out of sync. " + f"Extra in constant: {FLEET_DISPATCH_TOOLS - tagged}. " + f"Extra on server: {tagged - FLEET_DISPATCH_TOOLS}." + ) + + @pytest.mark.anyio + async def test_fleet_enables_fleet_tag(self, monkeypatch): + from autoskillit.core import FLEET_TOOLS, GATED_TOOLS, HEADLESS_TOOLS + from autoskillit.server import _apply_session_type_visibility, mcp + + monkeypatch.setenv("AUTOSKILLIT_SESSION_TYPE", "fleet") + _apply_session_type_visibility() + + tool_names = {t.name for t in await mcp.list_tools()} + + # Positive: fleet-tagged tools are visible + for name in FLEET_TOOLS: + assert name in tool_names, f"{name} should be visible for fleet session" + # Negative: non-fleet kitchen/headless tools remain hidden + for name in GATED_TOOLS - FLEET_TOOLS: + assert name not in tool_names, f"{name} should be hidden for fleet session" + for name in HEADLESS_TOOLS: + assert name not in tool_names, f"{name} should be hidden for fleet session" + + @pytest.mark.anyio + async def test_fleet_tools_do_not_carry_kitchen_tag(self, monkeypatch): + """Fleet-tagged tools must NOT carry the kitchen tag (tag partition).""" + from autoskillit.core import FLEET_TOOLS + from autoskillit.server import _apply_session_type_visibility, mcp + + monkeypatch.setenv("AUTOSKILLIT_SESSION_TYPE", "fleet") + _apply_session_type_visibility() + + all_tools = {t.name: t for t in await mcp.list_tools()} + for name in FLEET_TOOLS: + tool = all_tools.get(name) + assert tool is not None, f"{name} not registered" + assert "kitchen" not in tool.tags, f"{name} must not carry kitchen tag" + assert "fleet" in tool.tags, f"{name} must have fleet tag" + assert "autoskillit" in tool.tags, f"{name} must retain autoskillit tag" + + @pytest.mark.anyio + async def test_fleet_tools_constant_matches_tagged_tools(self, monkeypatch): + """FLEET_TOOLS constant matches exactly the tools with fleet tag.""" + from autoskillit.core import FLEET_TOOLS + from autoskillit.server import _apply_session_type_visibility, mcp + + monkeypatch.setenv("AUTOSKILLIT_SESSION_TYPE", "fleet") + _apply_session_type_visibility() + + all_tools = {t.name: t for t in await mcp.list_tools()} + tagged = {name for name, t in all_tools.items() if "fleet" in t.tags} + assert tagged == FLEET_TOOLS, ( + f"FLEET_TOOLS constant out of sync. " + f"Extra in constant: {FLEET_TOOLS - tagged}. " + f"Extra on server: {tagged - FLEET_TOOLS}." + ) + + @pytest.mark.anyio + async def test_orchestrator_headless_enables_kitchen_tag(self, monkeypatch): + from autoskillit.core import FLEET_DISPATCH_TOOLS, FLEET_TOOLS, GATED_TOOLS + from autoskillit.server import _apply_session_type_visibility, mcp + + monkeypatch.setenv("AUTOSKILLIT_SESSION_TYPE", "orchestrator") + monkeypatch.setenv("AUTOSKILLIT_HEADLESS", "1") + _apply_session_type_visibility() + + tools = list(await mcp.list_tools()) + tool_names = {t.name for t in tools} + for name in GATED_TOOLS - FLEET_TOOLS - FLEET_DISPATCH_TOOLS: + assert name in tool_names, f"{name} should be visible for orchestrator+headless" + + @pytest.mark.anyio + async def test_orchestrator_interactive_no_pre_reveal(self, monkeypatch): + from autoskillit.core import GATED_TOOLS, HEADLESS_TOOLS + from autoskillit.server import _apply_session_type_visibility, mcp + + monkeypatch.setenv("AUTOSKILLIT_SESSION_TYPE", "orchestrator") + monkeypatch.delenv("AUTOSKILLIT_HEADLESS", raising=False) + _apply_session_type_visibility() + + tools = list(await mcp.list_tools()) + tool_names = {t.name for t in tools} + for name in GATED_TOOLS: + assert name not in tool_names, f"{name} should be hidden for orchestrator+interactive" + for name in HEADLESS_TOOLS: + assert name not in tool_names, f"{name} should be hidden for orchestrator+interactive" + + @pytest.mark.anyio + async def test_skill_headless_enables_headless_tag(self, monkeypatch): + from autoskillit.core import GATED_TOOLS + from autoskillit.server import _apply_session_type_visibility, mcp + + monkeypatch.setenv("AUTOSKILLIT_SESSION_TYPE", "skill") + monkeypatch.setenv("AUTOSKILLIT_HEADLESS", "1") + _apply_session_type_visibility() + + tools = list(await mcp.list_tools()) + tool_names = {t.name for t in tools} + assert "test_check" in tool_names, "test_check should be visible for skill+headless" + for name in GATED_TOOLS: + assert name not in tool_names, f"{name} (kitchen) should be hidden for skill+headless" + + @pytest.mark.anyio + async def test_food_truck_with_tool_tags_sees_kitchen_core_plus_declared(self, monkeypatch): + """ORCHESTRATOR+HEADLESS with FOOD_TRUCK_TOOL_TAGS sees kitchen-core + github only.""" + from autoskillit.server import _apply_session_type_visibility, mcp + + monkeypatch.setenv("AUTOSKILLIT_SESSION_TYPE", "orchestrator") + monkeypatch.setenv("AUTOSKILLIT_HEADLESS", "1") + monkeypatch.setenv("AUTOSKILLIT_FOOD_TRUCK_TOOL_TAGS", "github") + _apply_session_type_visibility() + + tools = list(await mcp.list_tools()) + tool_names = {t.name for t in tools} + + assert "run_cmd" in tool_names + assert "run_skill" in tool_names + assert "merge_worktree" in tool_names + assert "fetch_github_issue" in tool_names + assert "wait_for_ci" not in tool_names + assert "clone_repo" not in tool_names + + @pytest.mark.anyio + async def test_food_truck_with_multiple_packs(self, monkeypatch): + """ORCHESTRATOR+HEADLESS with FOOD_TRUCK_TOOL_TAGS=github,ci sees both packs.""" + from autoskillit.server import _apply_session_type_visibility, mcp + + monkeypatch.setenv("AUTOSKILLIT_SESSION_TYPE", "orchestrator") + monkeypatch.setenv("AUTOSKILLIT_HEADLESS", "1") + monkeypatch.setenv("AUTOSKILLIT_FOOD_TRUCK_TOOL_TAGS", "github,ci") + _apply_session_type_visibility() + + tools = list(await mcp.list_tools()) + tool_names = {t.name for t in tools} + + assert "fetch_github_issue" in tool_names + assert "wait_for_ci" in tool_names + assert "clone_repo" not in tool_names + + @pytest.mark.anyio + async def test_food_truck_without_tool_tags_sees_full_kitchen(self, monkeypatch): + """ORCHESTRATOR+HEADLESS without FOOD_TRUCK_TOOL_TAGS falls back to full kitchen.""" + from autoskillit.core import FLEET_DISPATCH_TOOLS, FLEET_TOOLS, GATED_TOOLS + from autoskillit.server import _apply_session_type_visibility, mcp + + monkeypatch.setenv("AUTOSKILLIT_SESSION_TYPE", "orchestrator") + monkeypatch.setenv("AUTOSKILLIT_HEADLESS", "1") + monkeypatch.delenv("AUTOSKILLIT_FOOD_TRUCK_TOOL_TAGS", raising=False) + _apply_session_type_visibility() + + tools = list(await mcp.list_tools()) + tool_names = {t.name for t in tools} + + for name in GATED_TOOLS - FLEET_TOOLS - FLEET_DISPATCH_TOOLS: + assert name in tool_names + + @pytest.mark.anyio + async def test_cook_interactive_unaffected_by_tool_tags(self, monkeypatch): + """Interactive ORCHESTRATOR (cook) ignores FOOD_TRUCK_TOOL_TAGS.""" + from autoskillit.core import GATED_TOOLS + from autoskillit.server import _apply_session_type_visibility, mcp + + monkeypatch.setenv("AUTOSKILLIT_SESSION_TYPE", "orchestrator") + monkeypatch.delenv("AUTOSKILLIT_HEADLESS", raising=False) + monkeypatch.setenv("AUTOSKILLIT_FOOD_TRUCK_TOOL_TAGS", "github") + _apply_session_type_visibility() + + tools = list(await mcp.list_tools()) + tool_names = {t.name for t in tools} + + for name in GATED_TOOLS: + assert name not in tool_names + + @pytest.mark.anyio + async def test_skill_interactive_no_pre_reveal(self, monkeypatch): + from autoskillit.core import GATED_TOOLS, HEADLESS_TOOLS + from autoskillit.server import _apply_session_type_visibility, mcp + + monkeypatch.setenv("AUTOSKILLIT_SESSION_TYPE", "skill") + monkeypatch.delenv("AUTOSKILLIT_HEADLESS", raising=False) + _apply_session_type_visibility() + + tools = list(await mcp.list_tools()) + tool_names = {t.name for t in tools} + for name in GATED_TOOLS: + assert name not in tool_names, f"{name} should be hidden for skill+interactive" + for name in HEADLESS_TOOLS: + assert name not in tool_names, f"{name} should be hidden for skill+interactive" + + @pytest.mark.anyio + async def test_transitional_bridge_enables_headless(self, monkeypatch): + import warnings + + from autoskillit.core import GATED_TOOLS + from autoskillit.server import _apply_session_type_visibility, mcp + + monkeypatch.delenv("AUTOSKILLIT_SESSION_TYPE", raising=False) + monkeypatch.setenv("AUTOSKILLIT_HEADLESS", "1") + with warnings.catch_warnings(): + warnings.simplefilter("ignore", DeprecationWarning) + _apply_session_type_visibility() + + tools = list(await mcp.list_tools()) + tool_names = {t.name for t in tools} + assert "test_check" in tool_names, "test_check should be visible for bridge HEADLESS=1" + for name in GATED_TOOLS: + assert name not in tool_names, f"{name} (kitchen) should be hidden for bridge" + + @pytest.mark.anyio + async def test_fleet_tag_reset_by_conftest(self, monkeypatch): + from autoskillit.server import mcp + + # The conftest _reset_mcp_tags fixture has already disabled the fleet tag. + # Verify: no fleet-enabled state leaked from a previous test. + tools = list(await mcp.list_tools()) + tool_names = {t.name for t in tools} + # No kitchen tools should be visible — fleet tag was reset + from autoskillit.core import GATED_TOOLS + + for name in GATED_TOOLS: + assert name not in tool_names, f"{name} should be hidden after conftest reset" + + +@pytest.mark.feature("fleet") +class TestFleetAutoGateBoot: + """Fleet lifespan auto-gate: _fleet_auto_gate_boot opens gate before first tool call.""" + + @pytest.mark.anyio + async def test_fleet_lifespan_auto_opens_gate(self, tool_ctx): + """Fleet session: gate is open after _fleet_auto_gate_boot() runs.""" + import os + from unittest.mock import AsyncMock, MagicMock, patch + + from autoskillit.pipeline.gate import DefaultGateState + from autoskillit.server._lifespan import _fleet_auto_gate_boot + + tool_ctx.gate = DefaultGateState(enabled=False) + tool_ctx.quota_refresh_task = None + + with patch( + "autoskillit.server.tools.tools_kitchen._write_hook_config" + ) as mock_write_hook_config: + with patch( + "autoskillit.server._misc._prime_quota_cache", new=AsyncMock() + ) as mock_prime_quota_cache: + with patch( + "autoskillit.pipeline.create_background_task", + return_value=MagicMock(), + ) as mock_create_bg_task: + with patch( + "autoskillit.core.register_active_kitchen" + ) as mock_register_kitchen: + await _fleet_auto_gate_boot(tool_ctx) + + assert tool_ctx.gate.enabled is True + assert tool_ctx.kitchen_id is not None + assert tool_ctx.active_recipe_packs == frozenset() + mock_write_hook_config.assert_called_once_with() + mock_prime_quota_cache.assert_awaited_once_with() + mock_create_bg_task.assert_called_once() + mock_register_kitchen.assert_called_once_with( + tool_ctx.kitchen_id, os.getpid(), str(Path.cwd()) + ) + + @pytest.mark.anyio + async def test_fleet_lifespan_auto_gate_fails_open_on_hook_config_error(self, tool_ctx): + """Fleet auto-gate keeps gate open even when _write_hook_config() raises.""" + from unittest.mock import AsyncMock, MagicMock, patch + + from autoskillit.pipeline.gate import DefaultGateState + from autoskillit.server._lifespan import _fleet_auto_gate_boot + + tool_ctx.gate = DefaultGateState(enabled=False) + tool_ctx.quota_refresh_task = None + + with patch( + "autoskillit.server.tools.tools_kitchen._write_hook_config", + side_effect=OSError("disk full"), + ): + with patch("autoskillit.server._misc._prime_quota_cache", new=AsyncMock()): + with patch( + "autoskillit.pipeline.create_background_task", + return_value=MagicMock(), + ): + with patch("autoskillit.core.register_active_kitchen"): + await _fleet_auto_gate_boot(tool_ctx) + + assert tool_ctx.gate.enabled is True # gate stays open despite hook_config failure + + @pytest.mark.anyio + async def test_fleet_lifespan_auto_gate_fails_open_on_quota_cache_error(self, tool_ctx): + """Fleet auto-gate keeps gate open even when _prime_quota_cache() raises.""" + from unittest.mock import AsyncMock, MagicMock, patch + + from autoskillit.pipeline.gate import DefaultGateState + from autoskillit.server._lifespan import _fleet_auto_gate_boot + + tool_ctx.gate = DefaultGateState(enabled=False) + tool_ctx.quota_refresh_task = None + + with patch("autoskillit.server.tools.tools_kitchen._write_hook_config"): + with patch( + "autoskillit.server._misc._prime_quota_cache", + new=AsyncMock(side_effect=RuntimeError("quota cache error")), + ): + with patch( + "autoskillit.pipeline.create_background_task", + return_value=MagicMock(), + ): + with patch("autoskillit.core.register_active_kitchen"): + await _fleet_auto_gate_boot(tool_ctx) + + assert tool_ctx.gate.enabled is True # gate stays open despite quota cache failure + + @pytest.mark.anyio + async def test_fleet_lifespan_auto_gate_fails_open_on_background_task_error(self, tool_ctx): + """Fleet auto-gate keeps gate open even when create_background_task() raises.""" + from unittest.mock import AsyncMock, patch + + from autoskillit.pipeline.gate import DefaultGateState + from autoskillit.server._lifespan import _fleet_auto_gate_boot + + tool_ctx.gate = DefaultGateState(enabled=False) + tool_ctx.quota_refresh_task = None + + with patch("autoskillit.server.tools.tools_kitchen._write_hook_config"): + with patch("autoskillit.server._misc._prime_quota_cache", new=AsyncMock()): + with patch( + "autoskillit.pipeline.create_background_task", + side_effect=RuntimeError("task creation error"), + ): + with patch("autoskillit.core.register_active_kitchen"): + await _fleet_auto_gate_boot(tool_ctx) + + assert tool_ctx.gate.enabled is True # gate stays open despite background task failure + + @pytest.mark.anyio + async def test_fleet_lifespan_auto_gate_fails_open_on_registry_error(self, tool_ctx): + """Fleet auto-gate keeps gate open even when register_active_kitchen() raises.""" + from unittest.mock import AsyncMock, MagicMock, patch + + from autoskillit.pipeline.gate import DefaultGateState + from autoskillit.server._lifespan import _fleet_auto_gate_boot + + tool_ctx.gate = DefaultGateState(enabled=False) + tool_ctx.quota_refresh_task = None + + with patch("autoskillit.server.tools.tools_kitchen._write_hook_config"): + with patch("autoskillit.server._misc._prime_quota_cache", new=AsyncMock()): + with patch( + "autoskillit.pipeline.create_background_task", + return_value=MagicMock(), + ): + with patch( + "autoskillit.core.register_active_kitchen", + side_effect=OSError("registry write error"), + ): + await _fleet_auto_gate_boot(tool_ctx) + + assert tool_ctx.gate.enabled is True # gate stays open despite registry failure + + @pytest.mark.anyio + async def test_fleet_lifespan_auto_gate_logs_boot_event(self, tool_ctx): + """fleet_auto_gate_boot emits structured log event.""" + from unittest.mock import AsyncMock, MagicMock, patch + + from autoskillit.pipeline.gate import DefaultGateState + from autoskillit.server._lifespan import _fleet_auto_gate_boot + + tool_ctx.gate = DefaultGateState(enabled=False) + tool_ctx.quota_refresh_task = None + + with patch("autoskillit.server.tools.tools_kitchen._write_hook_config"): + with patch("autoskillit.server._misc._prime_quota_cache", new=AsyncMock()): + with patch( + "autoskillit.pipeline.create_background_task", + return_value=MagicMock(), + ): + with patch("autoskillit.core.register_active_kitchen"): + with structlog.testing.capture_logs() as logs: + await _fleet_auto_gate_boot(tool_ctx) + + assert any( + entry.get("event") == "fleet_auto_gate_boot" and entry.get("gate_state") == "open" + for entry in logs + ) + + @pytest.mark.anyio + async def test_fleet_auto_gate_boot_suppresses_fleet_tools_when_feature_disabled( + self, tool_ctx, monkeypatch + ): + """_fleet_auto_gate_boot with features.fleet: false → fleet MCP tags disabled.""" + import dataclasses + from unittest.mock import AsyncMock, MagicMock, patch + + from fastmcp.client import Client + + from autoskillit.core import FLEET_TOOLS + from autoskillit.pipeline.gate import DefaultGateState + from autoskillit.server import mcp + from autoskillit.server._lifespan import _fleet_auto_gate_boot + + # First enable fleet tag (as import-time phase 1 would) + mcp.enable(tags={"fleet"}) + + tool_ctx.gate = DefaultGateState(enabled=False) + tool_ctx.quota_refresh_task = None + tool_ctx.config = dataclasses.replace(tool_ctx.config, features={"fleet": False}) + + monkeypatch.setattr("autoskillit.server._lifespan._get_ctx_or_none", lambda: tool_ctx) + + with patch("autoskillit.server.tools.tools_kitchen._write_hook_config"): + with patch("autoskillit.server._misc._prime_quota_cache", new=AsyncMock()): + with patch( + "autoskillit.pipeline.create_background_task", + return_value=MagicMock(), + ): + with patch("autoskillit.core.register_active_kitchen"): + await _fleet_auto_gate_boot(tool_ctx) + + async with Client(mcp) as client: + tools = await client.list_tools() + tool_names = {t.name for t in tools} + assert FLEET_TOOLS, "FLEET_TOOLS must not be empty — loop would pass vacuously" + for name in FLEET_TOOLS: + assert name not in tool_names, f"{name} should be hidden when fleet feature disabled" + + @pytest.mark.anyio + async def test_fleet_auto_gate_boot_calls_shared_helper(self, tool_ctx, monkeypatch): + """_fleet_auto_gate_boot delegates to _collect_disabled_feature_tags, not inline logic.""" + import dataclasses + from unittest.mock import AsyncMock, MagicMock, patch + + from autoskillit.pipeline.gate import DefaultGateState + from autoskillit.server._lifespan import _fleet_auto_gate_boot + + tool_ctx.gate = DefaultGateState(enabled=False) + tool_ctx.quota_refresh_task = None + tool_ctx.config = dataclasses.replace(tool_ctx.config, features={"fleet": False}) + + monkeypatch.setattr("autoskillit.server._lifespan._get_ctx_or_none", lambda: tool_ctx) + + with patch("autoskillit.core._collect_disabled_feature_tags") as mock_helper: + mock_helper.return_value = frozenset({"fleet"}) + with patch("autoskillit.server.tools.tools_kitchen._write_hook_config"): + with patch("autoskillit.server._misc._prime_quota_cache", new=AsyncMock()): + with patch( + "autoskillit.pipeline.create_background_task", + return_value=MagicMock(), + ): + with patch("autoskillit.core.register_active_kitchen"): + await _fleet_auto_gate_boot(tool_ctx) + + mock_helper.assert_called_once_with( + tool_ctx.config.features, experimental_enabled=tool_ctx.config.experimental_enabled + ) + + +@pytest.mark.feature("orchestrator") +class TestFoodTruckAutoGateBoot: + """Food truck lifespan auto-gate: _food_truck_auto_gate_boot opens gate.""" + + @pytest.mark.anyio + async def test_food_truck_headless_lifespan_auto_opens_gate( + self, tool_ctx, monkeypatch + ) -> None: + from unittest.mock import AsyncMock, MagicMock, patch + + from autoskillit.core import FOOD_TRUCK_TOOL_TAGS_ENV_VAR + from autoskillit.pipeline.gate import DefaultGateState + from autoskillit.server._lifespan import _food_truck_auto_gate_boot + + tool_ctx.gate = DefaultGateState(enabled=False) + tool_ctx.quota_refresh_task = None + monkeypatch.setenv("AUTOSKILLIT_HEADLESS", "1") + monkeypatch.setenv(FOOD_TRUCK_TOOL_TAGS_ENV_VAR, "kitchen-core,rectify") + + with patch("autoskillit.server.tools.tools_kitchen._write_hook_config"): + with patch("autoskillit.server._misc._prime_quota_cache", new=AsyncMock()): + with patch( + "autoskillit.pipeline.create_background_task", return_value=MagicMock() + ): + with patch("autoskillit.core.register_active_kitchen"): + await _food_truck_auto_gate_boot(tool_ctx) + + assert tool_ctx.gate.enabled is True + + @pytest.mark.anyio + async def test_food_truck_auto_gate_boot_sets_active_recipe_packs( + self, tool_ctx, monkeypatch + ) -> None: + from unittest.mock import AsyncMock, MagicMock, patch + + from autoskillit.core import FOOD_TRUCK_TOOL_TAGS_ENV_VAR + from autoskillit.pipeline.gate import DefaultGateState + from autoskillit.server._lifespan import _food_truck_auto_gate_boot + + tool_ctx.gate = DefaultGateState(enabled=False) + tool_ctx.quota_refresh_task = None + monkeypatch.setenv("AUTOSKILLIT_HEADLESS", "1") + monkeypatch.setenv(FOOD_TRUCK_TOOL_TAGS_ENV_VAR, "kitchen-core,rectify") + + with patch("autoskillit.server.tools.tools_kitchen._write_hook_config"): + with patch("autoskillit.server._misc._prime_quota_cache", new=AsyncMock()): + with patch( + "autoskillit.pipeline.create_background_task", return_value=MagicMock() + ): + with patch("autoskillit.core.register_active_kitchen"): + await _food_truck_auto_gate_boot(tool_ctx) + + assert tool_ctx.active_recipe_packs == frozenset({"kitchen-core", "rectify"}) + assert tool_ctx.active_recipe_features == frozenset() + + @pytest.mark.anyio + async def test_food_truck_auto_gate_boot_skips_non_headless_orchestrator( + self, tool_ctx, monkeypatch + ) -> None: + from autoskillit.core import FOOD_TRUCK_TOOL_TAGS_ENV_VAR + from autoskillit.pipeline.gate import DefaultGateState + from autoskillit.server._lifespan import _food_truck_auto_gate_boot + + tool_ctx.gate = DefaultGateState(enabled=False) + monkeypatch.delenv("AUTOSKILLIT_HEADLESS", raising=False) + monkeypatch.setenv(FOOD_TRUCK_TOOL_TAGS_ENV_VAR, "kitchen-core") + + await _food_truck_auto_gate_boot(tool_ctx) + + assert tool_ctx.gate.enabled is False + + @pytest.mark.anyio + async def test_food_truck_auto_gate_boot_skips_when_no_tool_tags( + self, tool_ctx, monkeypatch + ) -> None: + from autoskillit.core import FOOD_TRUCK_TOOL_TAGS_ENV_VAR + from autoskillit.pipeline.gate import DefaultGateState + from autoskillit.server._lifespan import _food_truck_auto_gate_boot + + tool_ctx.gate = DefaultGateState(enabled=False) + monkeypatch.setenv("AUTOSKILLIT_HEADLESS", "1") + monkeypatch.delenv(FOOD_TRUCK_TOOL_TAGS_ENV_VAR, raising=False) + + await _food_truck_auto_gate_boot(tool_ctx) + + assert tool_ctx.gate.enabled is False + + def test_food_truck_tool_tags_env_var_constant_value(self) -> None: + from autoskillit.core import FOOD_TRUCK_TOOL_TAGS_ENV_VAR + + assert FOOD_TRUCK_TOOL_TAGS_ENV_VAR == "AUTOSKILLIT_FOOD_TRUCK_TOOL_TAGS" + + @pytest.mark.anyio + async def test_run_skill_gate_error_when_food_truck_gate_closed( + self, tool_ctx, monkeypatch + ) -> None: + import json + + from autoskillit.pipeline.gate import DefaultGateState + from autoskillit.server.tools.tools_execution import run_skill + + tool_ctx.gate = DefaultGateState(enabled=False) + + result = json.loads(await run_skill("/some-skill", "/tmp")) + assert result["subtype"] == "gate_error" + assert result["success"] is False + + @pytest.mark.anyio + async def test_run_skill_succeeds_after_food_truck_auto_gate_boot( + self, tool_ctx, monkeypatch + ) -> None: + import json + from unittest.mock import AsyncMock, MagicMock, patch + + from autoskillit.core import FOOD_TRUCK_TOOL_TAGS_ENV_VAR + from autoskillit.pipeline.gate import DefaultGateState + from autoskillit.server._lifespan import _food_truck_auto_gate_boot + from autoskillit.server.tools.tools_execution import run_skill + from tests.fakes import InMemoryHeadlessExecutor + + tool_ctx.gate = DefaultGateState(enabled=False) + tool_ctx.executor = InMemoryHeadlessExecutor() + tool_ctx.quota_refresh_task = None + monkeypatch.setenv("AUTOSKILLIT_HEADLESS", "1") + monkeypatch.setenv("AUTOSKILLIT_SESSION_TYPE", "orchestrator") + monkeypatch.setenv(FOOD_TRUCK_TOOL_TAGS_ENV_VAR, "kitchen-core") + + with patch("autoskillit.server.tools.tools_kitchen._write_hook_config"): + with patch("autoskillit.server._misc._prime_quota_cache", new=AsyncMock()): + with patch( + "autoskillit.pipeline.create_background_task", return_value=MagicMock() + ): + with patch("autoskillit.core.register_active_kitchen"): + await _food_truck_auto_gate_boot(tool_ctx) + + result = json.loads(await run_skill("/some-skill", "/tmp")) + assert result.get("success") is True + + +@pytest.mark.feature("fleet") +class TestFeatureGateVisibility: + """Session-type dispatch in _apply_session_type_visibility (phase 1 only).""" + + @pytest.fixture(autouse=True) + def _reset_mcp_visibility(self): + """Reset gated tag visibility on the shared mcp singleton before each test.""" + from autoskillit.core import ALL_VISIBILITY_TAGS + from autoskillit.server import mcp + + mcp._transforms.clear() + for tag in sorted(ALL_VISIBILITY_TAGS): + mcp.disable(tags={tag}) + yield + mcp._transforms.clear() + for tag in sorted(ALL_VISIBILITY_TAGS): + mcp.disable(tags={tag}) + + @pytest.mark.anyio + async def test_fleet_tools_visible_when_feature_enabled(self, monkeypatch): + """SESSION_TYPE=fleet → fleet tools visible (session-type dispatch only).""" + from autoskillit.core import FLEET_TOOLS + from autoskillit.server import mcp + from autoskillit.server._session_type import _apply_session_type_visibility + + monkeypatch.setenv("AUTOSKILLIT_SESSION_TYPE", "fleet") + _apply_session_type_visibility() + + tools = list(await mcp.list_tools()) + tool_names = {t.name for t in tools} + for name in FLEET_TOOLS: + assert name in tool_names, ( + f"{name} should be visible for fleet session (phase-1 reveal)" + ) + + def test_apply_session_type_visibility_sole_calling_convention(self): + """No feature_gates parameter exists — session-type dispatch only.""" + import inspect + + from autoskillit.server._session_type import _apply_session_type_visibility + + sig = inspect.signature(_apply_session_type_visibility) + assert "feature_gates" not in sig.parameters + + @pytest.mark.anyio + async def test_session_type_fleet_enables_fleet_tags(self, monkeypatch): + """FLEET session activates fleet tool visibility (no feature gate needed).""" + from autoskillit.core import FLEET_TOOLS + from autoskillit.server import mcp + from autoskillit.server._session_type import _apply_session_type_visibility + + monkeypatch.setenv("AUTOSKILLIT_SESSION_TYPE", "fleet") + monkeypatch.setenv("AUTOSKILLIT_HEADLESS", "1") + _apply_session_type_visibility() + + tools = list(await mcp.list_tools()) + tool_names = {t.name for t in tools} + assert FLEET_TOOLS + for tool in FLEET_TOOLS: + assert tool in tool_names diff --git a/tests/server/test_server_tool_registration.py b/tests/server/test_server_tool_registration.py index 4241f50f39..8e99b315ba 100644 --- a/tests/server/test_server_tool_registration.py +++ b/tests/server/test_server_tool_registration.py @@ -12,11 +12,13 @@ SafetyConfig, ) from autoskillit.pipeline.gate import DefaultGateState -from autoskillit.server.tools_status import ( +from autoskillit.server.tools.tools_status import ( get_quota_events, get_token_summary, ) +pytestmark = [pytest.mark.layer("server"), pytest.mark.small] + class TestNoSkillsDirectoryProvider: """T3: SkillsDirectoryProvider is not used in the new plugin architecture.""" @@ -30,7 +32,7 @@ def test_no_skills_directory_provider(self): class TestToolRegistration: - """All 44 tools are registered on the MCP server.""" + """All 52 tools are registered on the MCP server.""" @pytest.mark.anyio async def test_all_tools_exist(self, kitchen_enabled): @@ -38,6 +40,9 @@ async def test_all_tools_exist(self, kitchen_enabled): from autoskillit.server import mcp + mcp.enable(tags={"fleet"}) + mcp.enable(tags={"fleet-dispatch"}) + async with Client(mcp) as client: all_tools = await client.list_tools() tool_names = {t.name for t in all_tools} @@ -75,6 +80,7 @@ async def test_all_tools_exist(self, kitchen_enabled): "wait_for_merge_queue", "check_repo_merge_state", "toggle_auto_merge", + "enqueue_pr", "get_ci_status", "open_kitchen", "close_kitchen", @@ -87,6 +93,13 @@ async def test_all_tools_exist(self, kitchen_enabled): "set_commit_status", "register_clone_status", "batch_cleanup_clones", + "dispatch_food_truck", + "record_gate_dispatch", + "reload_session", + "analyze_tool_sequences", + "bootstrap_clone", + "claim_and_resolve_issue", + "create_and_publish_branch", } assert expected == tool_names @@ -120,12 +133,16 @@ async def test_ungated_tools_lack_kitchen_tag(self, kitchen_enabled): @pytest.mark.anyio async def test_kitchen_tools_have_autoskillit_and_kitchen_tags(self, kitchen_enabled): - """Every tool in GATED_TOOLS carries both 'autoskillit' and 'kitchen' tags.""" + """Every non-fleet tool in GATED_TOOLS carries both 'autoskillit' and 'kitchen' tags.""" + from autoskillit.core import FLEET_DISPATCH_TOOLS, FLEET_TOOLS from autoskillit.pipeline.gate import GATED_TOOLS from autoskillit.server import mcp + mcp.enable(tags={"fleet"}) + mcp.enable(tags={"fleet-dispatch"}) + all_tools = {t.name: t for t in await mcp.list_tools()} - for name in GATED_TOOLS: + for name in GATED_TOOLS - FLEET_TOOLS - FLEET_DISPATCH_TOOLS: tool = all_tools.get(name) assert tool is not None, f"Gated tool '{name}' not registered on server" assert "autoskillit" in tool.tags, f"Gated tool '{name}' missing 'autoskillit' tag" @@ -179,6 +196,51 @@ async def test_headless_enable_reveals_only_headless_tagged_tools(self, headless for name in kitchen_only: assert name not in visible, f"{name} should not be revealed by headless-only enable" + @pytest.mark.anyio + async def test_no_tool_has_bare_kitchen_tag_only(self, kitchen_enabled) -> None: + """Every kitchen-tagged tool must also carry kitchen-core or a pack tag.""" + from autoskillit.core.types import PACK_REGISTRY + from autoskillit.server import mcp + + pack_tags = frozenset(PACK_REGISTRY.keys()) | {"headless"} + all_tools = {t.name: t for t in await mcp.list_tools()} + for tool in all_tools.values(): + if "kitchen" not in tool.tags: + continue + has_subtag = bool(tool.tags & pack_tags) + assert has_subtag, ( + f"{tool.name} has 'kitchen' tag but no pack/kitchen-core/headless subtag: " + f"{sorted(tool.tags)}" + ) + + @pytest.mark.anyio + async def test_kitchen_core_and_packs_partition_kitchen_gated_tools( + self, kitchen_enabled + ) -> None: + """Every gated/headless tool has kitchen-core and/or a pack tag — full coverage.""" + from autoskillit.core.types import HEADLESS_TOOLS, PACK_REGISTRY + from autoskillit.pipeline.gate import GATED_TOOLS + from autoskillit.server import mcp + + mcp.enable(tags={"fleet"}) + mcp.enable(tags={"fleet-dispatch"}) + + all_gated = GATED_TOOLS | HEADLESS_TOOLS + pack_tags = frozenset(PACK_REGISTRY.keys()) + + all_tools = {t.name: t for t in await mcp.list_tools()} + missing: list[str] = [] + for name in sorted(all_gated): + tool = all_tools[name] + has_classification = bool(tool.tags & pack_tags) + if not has_classification: + missing.append(f"{name}: tags={sorted(tool.tags)}") + + assert not missing, ( + "Tools in GATED_TOOLS|HEADLESS_TOOLS without kitchen-core or pack tag:\n" + + "\n".join(f" {m}" for m in missing) + ) + class TestConfigDrivenBehavior: """S1-S10: Verify tools use config instead of hardcoded values.""" @@ -188,7 +250,7 @@ async def test_test_check_uses_config_command(self, tool_ctx): """S1: test_check runs config.test_check.command.""" from autoskillit.config import TestCheckConfig from autoskillit.execution import DefaultTestRunner - from autoskillit.server.tools_workspace import test_check + from autoskillit.server.tools.tools_workspace import test_check from tests.conftest import _make_result tool_ctx.config = AutomationConfig( @@ -201,71 +263,79 @@ async def test_test_check_uses_config_command(self, tool_ctx): await test_check(worktree_path="/tmp/wt") assert tool_ctx.runner.call_args_list[0][0] == ["pytest", "-x"] - assert tool_ctx.runner.call_args_list[0][2] == 300.0 + assert tool_ctx.runner.call_args_list[0][2] == pytest.approx(300.0, abs=1.0) @pytest.mark.anyio - async def test_classify_fix_uses_config_prefixes(self, tool_ctx, tmp_path): + async def test_classify_fix_uses_config_prefixes(self, tool_ctx_kitchen_open, tmp_path): """S2: classify_fix uses config.classify_fix.path_prefixes.""" from autoskillit.config import ClassifyFixConfig from autoskillit.core.types import RestartScope - from autoskillit.server.tools_git import classify_fix + from autoskillit.server.tools.tools_git import classify_fix from tests.conftest import _make_result - tool_ctx.config = AutomationConfig( + tool_ctx_kitchen_open.config = AutomationConfig( classify_fix=ClassifyFixConfig(path_prefixes=["src/custom/"]) ) changed = "src/custom/handler.py\nsrc/other/util.py\n" - tool_ctx.runner.push(_make_result(0, "", "")) # git fetch succeeds - tool_ctx.runner.push(_make_result(0, changed, "")) + tool_ctx_kitchen_open.runner.push(_make_result(0, "", "")) # git fetch succeeds + tool_ctx_kitchen_open.runner.push(_make_result(0, changed, "")) result = json.loads(await classify_fix(worktree_path=str(tmp_path), base_branch="main")) assert result["restart_scope"] == RestartScope.FULL_RESTART assert "src/custom/handler.py" in result["critical_files"] @pytest.mark.anyio - async def test_classify_fix_empty_prefixes_always_partial(self, tool_ctx, tmp_path): + async def test_classify_fix_empty_prefixes_always_partial( + self, tool_ctx_kitchen_open, tmp_path + ): """S3: Empty prefix list -> always returns partial_restart.""" from autoskillit.config import ClassifyFixConfig from autoskillit.core.types import RestartScope - from autoskillit.server.tools_git import classify_fix + from autoskillit.server.tools.tools_git import classify_fix from tests.conftest import _make_result - tool_ctx.config = AutomationConfig(classify_fix=ClassifyFixConfig(path_prefixes=[])) + tool_ctx_kitchen_open.config = AutomationConfig( + classify_fix=ClassifyFixConfig(path_prefixes=[]) + ) changed = "src/core/handler.py\n" - tool_ctx.runner.push(_make_result(0, "", "")) # git fetch succeeds - tool_ctx.runner.push(_make_result(0, changed, "")) + tool_ctx_kitchen_open.runner.push(_make_result(0, "", "")) # git fetch succeeds + tool_ctx_kitchen_open.runner.push(_make_result(0, changed, "")) result = json.loads(await classify_fix(worktree_path=str(tmp_path), base_branch="main")) assert result["restart_scope"] == RestartScope.PARTIAL_RESTART @pytest.mark.anyio - async def test_reset_workspace_uses_config_command(self, tool_ctx, tmp_path): + async def test_reset_workspace_uses_config_command(self, tool_ctx_kitchen_open, tmp_path): """S4: reset_workspace runs config.reset_workspace.command.""" from autoskillit.config import ResetWorkspaceConfig - from autoskillit.server.tools_workspace import reset_workspace + from autoskillit.server.tools.tools_workspace import reset_workspace from tests.conftest import _make_result - tool_ctx.config = AutomationConfig( + tool_ctx_kitchen_open.config = AutomationConfig( reset_workspace=ResetWorkspaceConfig(command=["make", "reset"]) ) workspace = tmp_path / "workspace" workspace.mkdir(parents=True) (workspace / ".autoskillit-workspace").write_text("# marker\n") - tool_ctx.runner.push(_make_result(0, "", "")) + tool_ctx_kitchen_open.runner.push(_make_result(0, "", "")) await reset_workspace(test_dir=str(workspace)) - assert tool_ctx.runner.call_args_list[0][0] == ["make", "reset"] + assert tool_ctx_kitchen_open.runner.call_args_list[0][0] == ["make", "reset"] @pytest.mark.anyio - async def test_reset_workspace_not_configured_returns_error(self, tool_ctx, tmp_path): + async def test_reset_workspace_not_configured_returns_error( + self, tool_ctx_kitchen_open, tmp_path + ): """S5: command=None -> returns not-configured error.""" from autoskillit.config import ResetWorkspaceConfig - from autoskillit.server.tools_workspace import reset_workspace + from autoskillit.server.tools.tools_workspace import reset_workspace - tool_ctx.config = AutomationConfig(reset_workspace=ResetWorkspaceConfig(command=None)) + tool_ctx_kitchen_open.config = AutomationConfig( + reset_workspace=ResetWorkspaceConfig(command=None) + ) workspace = tmp_path / "workspace" workspace.mkdir(parents=True) @@ -275,13 +345,15 @@ async def test_reset_workspace_not_configured_returns_error(self, tool_ctx, tmp_ assert result["error"] == "reset_workspace not configured for this project" @pytest.mark.anyio - async def test_reset_workspace_uses_config_preserve_dirs(self, tool_ctx, tmp_path): + async def test_reset_workspace_uses_config_preserve_dirs( + self, tool_ctx_kitchen_open, tmp_path + ): """S6: Preserves config.reset_workspace.preserve_dirs.""" from autoskillit.config import ResetWorkspaceConfig - from autoskillit.server.tools_workspace import reset_workspace + from autoskillit.server.tools.tools_workspace import reset_workspace from tests.conftest import _make_result - tool_ctx.config = AutomationConfig( + tool_ctx_kitchen_open.config = AutomationConfig( reset_workspace=ResetWorkspaceConfig( command=["true"], preserve_dirs={"keep_me"}, @@ -293,7 +365,7 @@ async def test_reset_workspace_uses_config_preserve_dirs(self, tool_ctx, tmp_pat (workspace / ".autoskillit-workspace").write_text("# marker\n") (workspace / "keep_me").mkdir() (workspace / "delete_me").touch() - tool_ctx.runner.push(_make_result(0, "", "")) + tool_ctx_kitchen_open.runner.push(_make_result(0, "", "")) result = json.loads(await reset_workspace(test_dir=str(workspace))) @@ -305,7 +377,7 @@ async def test_reset_workspace_uses_config_preserve_dirs(self, tool_ctx, tmp_pat def test_dry_walkthrough_uses_config_marker(self, tool_ctx, tmp_path): """S7: Gate checks config.implement_gate.marker.""" from autoskillit.config import ImplementGateConfig - from autoskillit.server.helpers import _check_dry_walkthrough + from autoskillit.server._guards import _check_dry_walkthrough tool_ctx.config = AutomationConfig( implement_gate=ImplementGateConfig(marker="CUSTOM MARKER") @@ -313,17 +385,17 @@ def test_dry_walkthrough_uses_config_marker(self, tool_ctx, tmp_path): plan = tmp_path / "plan.md" plan.write_text("CUSTOM MARKER\n# Plan content") - result = _check_dry_walkthrough(f"/autoskillit:implement-worktree {plan}", str(tmp_path)) + result = _check_dry_walkthrough(f"/implement-worktree {plan}", str(tmp_path)) assert result is None # passes with custom marker plan.write_text("Dry-walkthrough verified = TRUE\n# Plan content") - result = _check_dry_walkthrough(f"/autoskillit:implement-worktree {plan}", str(tmp_path)) + result = _check_dry_walkthrough(f"/implement-worktree {plan}", str(tmp_path)) assert result is not None # fails — marker doesn't match def test_dry_walkthrough_uses_config_skill_names(self, tool_ctx, tmp_path): """S8: Gate checks config.implement_gate.skill_names.""" from autoskillit.config import ImplementGateConfig - from autoskillit.server.helpers import _check_dry_walkthrough + from autoskillit.server._guards import _check_dry_walkthrough tool_ctx.config = AutomationConfig( implement_gate=ImplementGateConfig(skill_names={"/custom-impl"}) @@ -339,40 +411,48 @@ def test_dry_walkthrough_uses_config_skill_names(self, tool_ctx, tmp_path): assert result is None # /autoskillit:implement-worktree is NOT gated (not in skill_names) @pytest.mark.anyio - async def test_merge_worktree_uses_config_test_command(self, tool_ctx, tmp_path): + async def test_merge_worktree_uses_config_test_command(self, tool_ctx_kitchen_open, tmp_path): """S9: Merge's test gate runs config.test_check.command.""" from autoskillit.config import TestCheckConfig from autoskillit.core.types import MergeFailedStep from autoskillit.execution import DefaultTestRunner - from autoskillit.server.tools_git import merge_worktree + from autoskillit.server.tools.tools_git import merge_worktree from tests.conftest import _make_result - tool_ctx.config = AutomationConfig( + tool_ctx_kitchen_open.config = AutomationConfig( test_check=TestCheckConfig(command=["make", "test"], timeout=120) ) # Re-create tester with updated config so it reads the new command - tool_ctx.tester = DefaultTestRunner(config=tool_ctx.config, runner=tool_ctx.runner) + tool_ctx_kitchen_open.tester = DefaultTestRunner( + config=tool_ctx_kitchen_open.config, runner=tool_ctx_kitchen_open.runner + ) wt = tmp_path / "worktree" wt.mkdir() (wt / ".git").write_text("gitdir: /repo/.git/worktrees/wt") - tool_ctx.runner.push(_make_result(0, "/repo/.git/worktrees/wt\n", "")) # rev-parse - tool_ctx.runner.push(_make_result(0, "impl-branch\n", "")) # branch - tool_ctx.runner.push(_make_result(0, "", "")) # git ls-files (pre-dirty-tree check) - tool_ctx.runner.push(_make_result(0, "", "")) # git status --porcelain (clean) - tool_ctx.runner.push(_make_result(1, "FAIL", "")) # test gate fails + tool_ctx_kitchen_open.runner.push( + _make_result(0, "/repo/.git/worktrees/wt\n", "") + ) # rev-parse + tool_ctx_kitchen_open.runner.push(_make_result(0, "impl-branch\n", "")) # branch + tool_ctx_kitchen_open.runner.push( + _make_result(0, "", "") + ) # git ls-files (pre-dirty-tree check) + tool_ctx_kitchen_open.runner.push( + _make_result(0, "", "") + ) # git status --porcelain (clean) + tool_ctx_kitchen_open.runner.push(_make_result(1, "FAIL", "")) # test gate fails result = json.loads(await merge_worktree(str(wt), "dev")) assert result["failed_step"] == MergeFailedStep.TEST_GATE # Verify the test command was ["make", "test"] (5th call, after ls-files + porcelain) - test_call = tool_ctx.runner.call_args_list[4] + test_call = tool_ctx_kitchen_open.runner.call_args_list[4] assert test_call[0] == ["make", "test"] @pytest.mark.anyio async def test_require_enabled_still_gates_execution(self, tool_ctx): """S10: _require_enabled() defense-in-depth still works with config.""" - from autoskillit.server.tools_execution import run_cmd + from autoskillit.server.tools.tools_execution import run_cmd tool_ctx.gate = DefaultGateState(enabled=False) result = json.loads(await run_cmd(cmd="echo hi", cwd="/tmp")) @@ -385,9 +465,9 @@ class TestSafetyConfigWiring: """Safety config fields are read at the point of enforcement.""" @pytest.mark.anyio - async def test_reset_test_dir_allows_with_marker(self, tool_ctx, tmp_path): + async def test_reset_test_dir_allows_with_marker(self, tool_ctx_kitchen_open, tmp_path): """2a: Directory with marker passes the reset guard.""" - from autoskillit.server.tools_workspace import reset_test_dir + from autoskillit.server.tools.tools_workspace import reset_test_dir target = tmp_path / "my_project" target.mkdir() @@ -398,9 +478,11 @@ async def test_reset_test_dir_allows_with_marker(self, tool_ctx, tmp_path): assert result["success"] is True @pytest.mark.anyio - async def test_reset_test_dir_enforces_marker_when_missing(self, tool_ctx, tmp_path): + async def test_reset_test_dir_enforces_marker_when_missing( + self, tool_ctx_kitchen_open, tmp_path + ): """2b: Missing marker blocks reset_test_dir.""" - from autoskillit.server.tools_workspace import reset_test_dir + from autoskillit.server.tools.tools_workspace import reset_test_dir target = tmp_path / "unmarked" target.mkdir() @@ -409,12 +491,14 @@ async def test_reset_test_dir_enforces_marker_when_missing(self, tool_ctx, tmp_p assert "marker" in result["error"].lower() @pytest.mark.anyio - async def test_reset_workspace_enforces_marker(self, tool_ctx, tmp_path): + async def test_reset_workspace_enforces_marker(self, tool_ctx_kitchen_open, tmp_path): """2c: reset_workspace requires marker, then checks command config.""" from autoskillit.config import ResetWorkspaceConfig - from autoskillit.server.tools_workspace import reset_workspace + from autoskillit.server.tools.tools_workspace import reset_workspace - tool_ctx.config = AutomationConfig(reset_workspace=ResetWorkspaceConfig(command=None)) + tool_ctx_kitchen_open.config = AutomationConfig( + reset_workspace=ResetWorkspaceConfig(command=None) + ) target = tmp_path / "my_project" target.mkdir() @@ -425,58 +509,76 @@ async def test_reset_workspace_enforces_marker(self, tool_ctx, tmp_path): assert result["error"] == "reset_workspace not configured for this project" @pytest.mark.anyio - async def test_merge_worktree_skips_test_gate_when_disabled(self, tool_ctx, tmp_path): + async def test_merge_worktree_skips_test_gate_when_disabled( + self, tool_ctx_kitchen_open, tmp_path + ): """2d: test_gate_on_merge=False skips test execution.""" - from autoskillit.server.tools_git import merge_worktree + from autoskillit.server.tools.tools_git import merge_worktree from tests.conftest import _make_result - tool_ctx.config = AutomationConfig(safety=SafetyConfig(test_gate_on_merge=False)) + tool_ctx_kitchen_open.config = AutomationConfig( + safety=SafetyConfig(test_gate_on_merge=False) + ) wt = tmp_path / "worktree" wt.mkdir() (wt / ".git").write_text("gitdir: /repo/.git/worktrees/wt") - tool_ctx.runner.push(_make_result(0, "/repo/.git/worktrees/wt\n", "")) # rev-parse - tool_ctx.runner.push(_make_result(0, "impl-branch\n", "")) # branch - tool_ctx.runner.push(_make_result(0, "", "")) # git ls-files (pre-dirty-tree check) - tool_ctx.runner.push(_make_result(0, "", "")) # git status --porcelain (clean) + tool_ctx_kitchen_open.runner.push( + _make_result(0, "/repo/.git/worktrees/wt\n", "") + ) # rev-parse + tool_ctx_kitchen_open.runner.push(_make_result(0, "impl-branch\n", "")) # branch + tool_ctx_kitchen_open.runner.push( + _make_result(0, "", "") + ) # git ls-files (pre-dirty-tree check) + tool_ctx_kitchen_open.runner.push( + _make_result(0, "", "") + ) # git status --porcelain (clean) # NO test-check call — skipped - tool_ctx.runner.push(_make_result(0, "", "")) # git fetch - tool_ctx.runner.push(_make_result(0, "abc123\n", "")) # rev-parse --verify (step 5.5) - tool_ctx.runner.push( + tool_ctx_kitchen_open.runner.push(_make_result(0, "", "")) # git fetch + tool_ctx_kitchen_open.runner.push( + _make_result(0, "abc123\n", "") + ) # rev-parse --verify (step 5.5) + tool_ctx_kitchen_open.runner.push( _make_result(0, "", "") ) # git log --merges (step 5.6 — no merge commits) - tool_ctx.runner.push(_make_result(0, "", "")) # git rebase - tool_ctx.runner.push( + tool_ctx_kitchen_open.runner.push(_make_result(0, "", "")) # git rebase + tool_ctx_kitchen_open.runner.push( _make_result( 0, "worktree /repo\nHEAD abc\nbranch refs/heads/dev\n\n", "", ) ) # worktree list - tool_ctx.runner.push(_make_result(0, "dev\n", "")) # git branch --show-current (step 7.5) - tool_ctx.runner.push(_make_result(0, "", "")) # git merge - tool_ctx.runner.push(_make_result(0, "", "")) # worktree remove - tool_ctx.runner.push(_make_result(0, "", "")) # branch -D + tool_ctx_kitchen_open.runner.push( + _make_result(0, "dev\n", "") + ) # git branch --show-current (step 7.5) + tool_ctx_kitchen_open.runner.push(_make_result(0, "", "")) # git merge + tool_ctx_kitchen_open.runner.push(_make_result(0, "", "")) # worktree remove + tool_ctx_kitchen_open.runner.push(_make_result(0, "", "")) # branch -D result = json.loads(await merge_worktree(str(wt), "dev")) assert result["merge_succeeded"] is True # Verify no test command was called — the 5th call should be git fetch, not test - fifth_call_cmd = tool_ctx.runner.call_args_list[4][0] + fifth_call_cmd = tool_ctx_kitchen_open.runner.call_args_list[4][0] assert fifth_call_cmd == ["git", "fetch", "origin"] @pytest.mark.anyio - async def test_run_skill_2e_skips_dry_walkthrough_when_disabled(self, tool_ctx, tmp_path): + async def test_run_skill_2e_skips_dry_walkthrough_when_disabled( + self, tool_ctx_kitchen_open, tmp_path + ): """2e: require_dry_walkthrough=False bypasses dry-walkthrough gate (using run_skill).""" - from autoskillit.server.tools_execution import run_skill + from autoskillit.server.tools.tools_execution import run_skill from tests.conftest import _make_result - tool_ctx.config = AutomationConfig(safety=SafetyConfig(require_dry_walkthrough=False)) + tool_ctx_kitchen_open.config = AutomationConfig( + safety=SafetyConfig(require_dry_walkthrough=False) + ) plan = tmp_path / "plan.md" plan.write_text("# No marker plan") - tool_ctx.runner.push(_make_result(0, '{"result": "done"}', "")) + tool_ctx_kitchen_open.runner.push(_make_result(0, '{"result": "done"}', "")) result = json.loads( await run_skill(f"/autoskillit:implement-worktree {plan}", str(tmp_path)) ) @@ -484,37 +586,20 @@ async def test_run_skill_2e_skips_dry_walkthrough_when_disabled(self, tool_ctx, assert result["exit_code"] == 0 @pytest.mark.anyio - async def test_run_skill_enforces_dry_walkthrough_when_enabled(self, tool_ctx, tmp_path): + async def test_run_skill_enforces_dry_walkthrough_when_enabled( + self, tool_ctx_kitchen_open, tmp_path + ): """2f: run_skill enforces dry-walkthrough gate when enabled (default).""" - from autoskillit.server.tools_execution import run_skill + from autoskillit.server.tools.tools_execution import run_skill plan = tmp_path / "plan.md" plan.write_text("# No marker plan") - result = json.loads( - await run_skill(f"/autoskillit:implement-worktree {plan}", str(tmp_path)) - ) + result = json.loads(await run_skill(f"/implement-worktree {plan}", str(tmp_path))) assert result["success"] is False assert result["is_error"] is True assert "dry-walked" in result["result"].lower() - @pytest.mark.anyio - async def test_run_skill_skips_dry_walkthrough_when_disabled(self, tool_ctx, tmp_path): - """2g: run_skill skips dry-walkthrough gate when disabled.""" - from autoskillit.server.tools_execution import run_skill - from tests.conftest import _make_result - - tool_ctx.config = AutomationConfig(safety=SafetyConfig(require_dry_walkthrough=False)) - - plan = tmp_path / "plan.md" - plan.write_text("# No marker plan") - - tool_ctx.runner.push(_make_result(0, '{"result": "done"}', "")) - result = json.loads( - await run_skill(f"/autoskillit:implement-worktree {plan}", str(tmp_path)) - ) - assert result["subtype"] != "gate_error" - class TestToolSchemas: """Regression guard: tool descriptions must not contain legacy terminology.""" @@ -557,6 +642,9 @@ async def _get_all_tools(self, kitchen_enabled) -> dict: from autoskillit.server import mcp + mcp.enable(tags={"fleet"}) + mcp.enable(tags={"fleet-dispatch"}) + async with Client(mcp) as client: tools = await client.list_tools() return {t.name: t for t in tools} diff --git a/tests/server/test_server_version_telemetry.py b/tests/server/test_server_version_telemetry.py index f81fad3ba0..d9cb33cdbd 100644 --- a/tests/server/test_server_version_telemetry.py +++ b/tests/server/test_server_version_telemetry.py @@ -8,6 +8,8 @@ import pytest +pytestmark = [pytest.mark.layer("server"), pytest.mark.small] + class TestPluginMetadataExists: """T1: Plugin metadata files are shipped in the package.""" @@ -36,24 +38,24 @@ def test_mcp_json_exists(self): class TestPluginDirConstant: - """T6: tool_ctx.plugin_dir defaults to the package root directory.""" + """T6: tool_ctx.plugin_source defaults to a DirectInstall for the package root.""" - def test_plugin_dir_assignment_is_visible_via_get_ctx(self, tool_ctx): - """By default tool_ctx.plugin_dir is set to tmp_path by the fixture. + def test_plugin_source_assignment_is_visible_via_get_plugin_dir(self, tool_ctx): + """By default tool_ctx.plugin_source is set to DirectInstall(tmp_path) by the fixture. The real package dir is what the server uses at runtime (set by cli.py serve()). - This test verifies that the fixture wires plugin_dir through _ctx correctly. + This test verifies that the fixture wires plugin_source through _ctx correctly. """ import autoskillit + from autoskillit.core.types._type_plugin_source import DirectInstall + from autoskillit.server._state import _get_plugin_dir # The real package dir is what the server sets at startup. - # We verify the attribute path works (tool_ctx.plugin_dir is accessible). - real_pkg_dir = str(Path(autoskillit.__file__).parent) + real_pkg_dir = Path(autoskillit.__file__).parent # tool_ctx uses tmp_path; set it to verify end-to-end wiring - tool_ctx.plugin_dir = real_pkg_dir - from autoskillit.server import _get_ctx + tool_ctx.plugin_source = DirectInstall(plugin_dir=real_pkg_dir) - assert _get_ctx().plugin_dir == real_pkg_dir + assert _get_plugin_dir() == str(real_pkg_dir) class TestVersionInfo: @@ -71,6 +73,7 @@ def test_version_info_returns_package_and_plugin_versions(self, monkeypatch): assert info["match"] is True def test_version_info_detects_mismatch(self, tmp_path, tool_ctx): + from autoskillit.core.types._type_plugin_source import DirectInstall from autoskillit.server import version_info plugin_dir = tmp_path / ".claude-plugin" @@ -78,16 +81,17 @@ def test_version_info_detects_mismatch(self, tmp_path, tool_ctx): (plugin_dir / "plugin.json").write_text( json.dumps({"name": "autoskillit", "version": "0.0.0"}) ) - tool_ctx.plugin_dir = str(tmp_path) + tool_ctx.plugin_source = DirectInstall(plugin_dir=tmp_path) info = version_info() assert info["match"] is False assert info["package_version"] != info["plugin_json_version"] assert info["plugin_json_version"] == "0.0.0" def test_version_info_handles_missing_plugin_json(self, tmp_path, tool_ctx): + from autoskillit.core.types._type_plugin_source import DirectInstall from autoskillit.server import version_info - tool_ctx.plugin_dir = str(tmp_path) + tool_ctx.plugin_source = DirectInstall(plugin_dir=tmp_path) info = version_info() assert info["plugin_json_version"] is None assert info["match"] is False @@ -145,6 +149,11 @@ class TestInitializeClearMarker: def test_initialize_uses_clear_marker_as_since_bound(self, tool_ctx, tmp_path, monkeypatch): from datetime import UTC, datetime, timedelta + from autoskillit.core.types._type_results import ProviderOutcome + from autoskillit.core.types._type_results_execution import ( + RecipeIdentity, + SessionTelemetry, + ) from autoskillit.execution.session_log import ( flush_session_log, ) @@ -167,13 +176,22 @@ def test_initialize_uses_clear_marker_as_since_bound(self, tool_ctx, tmp_path, m start_ts=five_hours_ago.isoformat(), proc_snapshots=None, step_name="old-step", - token_usage={ - "input_tokens": 1000, - "output_tokens": 500, - "cache_creation_input_tokens": 0, - "cache_read_input_tokens": 0, - }, - timing_seconds=10.0, + telemetry=SessionTelemetry( + token_usage={ + "input_tokens": 1000, + "output_tokens": 500, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + }, + timing_seconds=10.0, + audit_record=None, + github_api_usage=None, + github_api_requests=0, + loc_insertions=0, + loc_deletions=0, + ), + provider_outcome=ProviderOutcome.none_used(), + recipe_identity=RecipeIdentity.empty(), ) # Write a clear marker 3 hours ago (after the session completed) diff --git a/tests/server/test_service_wrappers.py b/tests/server/test_service_wrappers.py index 67bfd2878a..4b4d234349 100644 --- a/tests/server/test_service_wrappers.py +++ b/tests/server/test_service_wrappers.py @@ -14,6 +14,8 @@ import autoskillit +pytestmark = [pytest.mark.layer("server"), pytest.mark.small] + class TestDefaultRecipeRepository: def setup_method(self) -> None: diff --git a/tests/server/test_session_type_tags.py b/tests/server/test_session_type_tags.py new file mode 100644 index 0000000000..17d530f78c --- /dev/null +++ b/tests/server/test_session_type_tags.py @@ -0,0 +1,28 @@ +"""Tests for _collect_fleet_tool_tags in server._session_type (Finding 1).""" + +from __future__ import annotations + +import pytest + +pytestmark = [pytest.mark.layer("server"), pytest.mark.small] + + +# --------------------------------------------------------------------------- +# T5: _collect_fleet_tool_tags auto-discovers tags from FEATURE_REGISTRY +# --------------------------------------------------------------------------- + + +def test_collect_fleet_tool_tags_is_union_of_registry() -> None: + """_collect_fleet_tool_tags() equals the union of all FeatureDef.tool_tags.""" + from autoskillit.core.types._type_constants import FEATURE_REGISTRY + from autoskillit.server._session_type import _collect_fleet_tool_tags + + expected = frozenset().union(*(fdef.tool_tags for fdef in FEATURE_REGISTRY.values())) + assert _collect_fleet_tool_tags() == expected + + +def test_collect_fleet_tool_tags_includes_fleet() -> None: + """The 'fleet' tag from the fleet FeatureDef must appear in the collected tags.""" + from autoskillit.server._session_type import _collect_fleet_tool_tags + + assert "fleet" in _collect_fleet_tool_tags() diff --git a/tests/server/test_set_commit_status.py b/tests/server/test_set_commit_status.py index 3e84f084a7..58b4f89614 100644 --- a/tests/server/test_set_commit_status.py +++ b/tests/server/test_set_commit_status.py @@ -7,9 +7,11 @@ import pytest from autoskillit.pipeline.gate import GATED_TOOLS, DefaultGateState -from autoskillit.server.tools_ci import set_commit_status +from autoskillit.server.tools.tools_ci import set_commit_status from tests.conftest import _make_result +pytestmark = [pytest.mark.layer("server"), pytest.mark.small] + async def _async_return(value: object) -> object: """Minimal async helper returning a fixed value — for monkeypatching async callables.""" @@ -55,13 +57,13 @@ async def test_set_commit_status_gate_check(tool_ctx): @pytest.mark.anyio -async def test_set_commit_status_posts_pending(tool_ctx, monkeypatch): +async def test_set_commit_status_posts_pending(tool_ctx_kitchen_open, monkeypatch): """Tool posts gh api with state=pending to the correct endpoint.""" monkeypatch.setattr( - "autoskillit.server.tools_ci.infer_repo_from_remote", + "autoskillit.server.tools.tools_ci.resolve_repo_from_remote", lambda cwd, hint=None: _async_return("owner/repo"), ) - tool_ctx.runner.push(_make_result(0, "", "")) + tool_ctx_kitchen_open.runner.push(_make_result(0, "", "")) raw = await set_commit_status( sha="deadbeef", @@ -78,7 +80,7 @@ async def test_set_commit_status_posts_pending(tool_ctx, monkeypatch): assert result["context"] == "autoskillit/ai-review" # Verify the POST call went to the right endpoint - cmd, *_ = tool_ctx.runner.call_args_list[0] + cmd, *_ = tool_ctx_kitchen_open.runner.call_args_list[0] cmd_str = " ".join(cmd) assert "/repos/owner/repo/statuses/deadbeef" in cmd_str assert "pending" in cmd_str @@ -90,13 +92,13 @@ async def test_set_commit_status_posts_pending(tool_ctx, monkeypatch): @pytest.mark.anyio -async def test_set_commit_status_posts_success(tool_ctx, monkeypatch): +async def test_set_commit_status_posts_success(tool_ctx_kitchen_open, monkeypatch): """Tool posts gh api with state=success and context preserved.""" monkeypatch.setattr( - "autoskillit.server.tools_ci.infer_repo_from_remote", + "autoskillit.server.tools.tools_ci.resolve_repo_from_remote", lambda cwd, hint=None: _async_return("myorg/myrepo"), ) - tool_ctx.runner.push(_make_result(0, "", "")) + tool_ctx_kitchen_open.runner.push(_make_result(0, "", "")) raw = await set_commit_status( sha="cafebabe", @@ -110,7 +112,7 @@ async def test_set_commit_status_posts_success(tool_ctx, monkeypatch): assert result["state"] == "success" assert result["context"] == "autoskillit/ai-review" - post_cmd = " ".join(tool_ctx.runner.call_args_list[0][0]) + post_cmd = " ".join(tool_ctx_kitchen_open.runner.call_args_list[0][0]) assert "success" in post_cmd @@ -120,13 +122,13 @@ async def test_set_commit_status_posts_success(tool_ctx, monkeypatch): @pytest.mark.anyio -async def test_set_commit_status_posts_failure(tool_ctx, monkeypatch): +async def test_set_commit_status_posts_failure(tool_ctx_kitchen_open, monkeypatch): """Tool posts gh api with state=failure.""" monkeypatch.setattr( - "autoskillit.server.tools_ci.infer_repo_from_remote", + "autoskillit.server.tools.tools_ci.resolve_repo_from_remote", lambda cwd, hint=None: _async_return("owner/repo"), ) - tool_ctx.runner.push(_make_result(0, "", "")) + tool_ctx_kitchen_open.runner.push(_make_result(0, "", "")) raw = await set_commit_status( sha="feedface", @@ -139,7 +141,7 @@ async def test_set_commit_status_posts_failure(tool_ctx, monkeypatch): assert result["success"] is True assert result["state"] == "failure" - post_cmd = " ".join(tool_ctx.runner.call_args_list[0][0]) + post_cmd = " ".join(tool_ctx_kitchen_open.runner.call_args_list[0][0]) assert "failure" in post_cmd @@ -149,16 +151,16 @@ async def test_set_commit_status_posts_failure(tool_ctx, monkeypatch): @pytest.mark.anyio -async def test_set_commit_status_infers_repo_from_cwd(tool_ctx, monkeypatch): - """Tool infers owner/repo via infer_repo_from_remote when repo param is absent.""" +async def test_set_commit_status_infers_repo_from_cwd(tool_ctx_kitchen_open, monkeypatch): + """Tool infers owner/repo via resolve_repo_from_remote when repo param is absent.""" infer_calls: list[str] = [] async def fake_infer(cwd: str, hint: object = None) -> str: infer_calls.append(cwd) return "inferred/repo" - monkeypatch.setattr("autoskillit.server.tools_ci.infer_repo_from_remote", fake_infer) - tool_ctx.runner.push(_make_result(0, "", "")) + monkeypatch.setattr("autoskillit.server.tools.tools_ci.resolve_repo_from_remote", fake_infer) + tool_ctx_kitchen_open.runner.push(_make_result(0, "", "")) raw = await set_commit_status( sha="abc001", @@ -170,51 +172,55 @@ async def fake_infer(cwd: str, hint: object = None) -> str: result = json.loads(raw) assert result["success"] is True - assert infer_calls, "infer_repo_from_remote was not called" + assert infer_calls, "resolve_repo_from_remote was not called" # POST must use the inferred repo - post_cmd = tool_ctx.runner.call_args_list[0][0] + post_cmd = tool_ctx_kitchen_open.runner.call_args_list[0][0] assert "inferred/repo" in " ".join(post_cmd) @pytest.mark.anyio -async def test_set_commit_status_falls_back_to_plugin_dir_when_no_cwd(tool_ctx, monkeypatch): - """When neither repo nor cwd is provided, tool falls back to plugin_dir for inference.""" +async def test_set_commit_status_falls_back_to_plugin_dir_when_no_cwd( + tool_ctx_kitchen_open, monkeypatch +): + """When neither repo nor cwd is provided, tool falls back to plugin_source for inference.""" infer_calls: list[str] = [] async def fake_infer(cwd: str, hint: object = None) -> str: infer_calls.append(cwd) return "fallback/repo" - monkeypatch.setattr("autoskillit.server.tools_ci.infer_repo_from_remote", fake_infer) - tool_ctx.runner.push(_make_result(0, "", "")) + monkeypatch.setattr("autoskillit.server.tools.tools_ci.resolve_repo_from_remote", fake_infer) + tool_ctx_kitchen_open.runner.push(_make_result(0, "", "")) raw = await set_commit_status( sha="abc003", state="pending", context="autoskillit/ai-review", - # neither repo nor cwd — falls back to tool_ctx.plugin_dir + # neither repo nor cwd — falls back to tool_ctx_kitchen_open.plugin_source ) result = json.loads(raw) assert result["success"] is True - assert infer_calls, "infer_repo_from_remote was not called for fallback" + assert infer_calls, "resolve_repo_from_remote was not called for fallback" # POST must reference the fallback repo - post_cmd, *_ = tool_ctx.runner.call_args_list[0] + post_cmd, *_ = tool_ctx_kitchen_open.runner.call_args_list[0] assert "fallback/repo" in " ".join(post_cmd) @pytest.mark.anyio -async def test_set_commit_status_uses_explicit_repo_without_inference(tool_ctx, monkeypatch): - """When repo is provided explicitly, infer_repo_from_remote is not called.""" +async def test_set_commit_status_uses_explicit_repo_without_inference( + tool_ctx_kitchen_open, monkeypatch +): + """When repo is provided explicitly, resolve_repo_from_remote is not called.""" infer_calls: list[str] = [] async def fake_infer(cwd: str, hint: object = None) -> str: infer_calls.append(cwd) return "should-not-be-used/repo" - monkeypatch.setattr("autoskillit.server.tools_ci.infer_repo_from_remote", fake_infer) - tool_ctx.runner.push(_make_result(0, "", "")) + monkeypatch.setattr("autoskillit.server.tools.tools_ci.resolve_repo_from_remote", fake_infer) + tool_ctx_kitchen_open.runner.push(_make_result(0, "", "")) raw = await set_commit_status( sha="abc002", @@ -226,8 +232,8 @@ async def fake_infer(cwd: str, hint: object = None) -> str: result = json.loads(raw) assert result["success"] is True - assert not infer_calls, "infer_repo_from_remote should not be called when repo is explicit" - post_cmd = tool_ctx.runner.call_args_list[0][0] + assert not infer_calls, "resolve_repo_from_remote should not be called when repo is explicit" + post_cmd = tool_ctx_kitchen_open.runner.call_args_list[0][0] assert "explicit/repo" in " ".join(post_cmd) @@ -237,13 +243,15 @@ async def fake_infer(cwd: str, hint: object = None) -> str: @pytest.mark.anyio -async def test_set_commit_status_on_gh_failure_returns_error_dict(tool_ctx, monkeypatch): +async def test_set_commit_status_on_gh_failure_returns_error_dict( + tool_ctx_kitchen_open, monkeypatch +): """When gh api returns non-zero, tool returns success=false, never raises.""" monkeypatch.setattr( - "autoskillit.server.tools_ci.infer_repo_from_remote", + "autoskillit.server.tools.tools_ci.resolve_repo_from_remote", lambda cwd, hint=None: _async_return("owner/repo"), ) - tool_ctx.runner.push(_make_result(1, "", "API rate limit exceeded")) + tool_ctx_kitchen_open.runner.push(_make_result(1, "", "API rate limit exceeded")) raw = await set_commit_status( sha="baadf00d", @@ -258,10 +266,12 @@ async def test_set_commit_status_on_gh_failure_returns_error_dict(tool_ctx, monk @pytest.mark.anyio -async def test_set_commit_status_repo_inference_failure_returns_error(tool_ctx, monkeypatch): +async def test_set_commit_status_repo_inference_failure_returns_error( + tool_ctx_kitchen_open, monkeypatch +): """When repo inference returns empty string, tool returns success=false.""" monkeypatch.setattr( - "autoskillit.server.tools_ci.infer_repo_from_remote", + "autoskillit.server.tools.tools_ci.resolve_repo_from_remote", lambda cwd, hint=None: _async_return(""), ) @@ -306,7 +316,7 @@ async def test_set_commit_status_validation_errors_return_str(tool_ctx): @pytest.mark.anyio -async def test_set_commit_status_success_returns_str(tool_ctx, monkeypatch): +async def test_set_commit_status_success_returns_str(tool_ctx_kitchen_open, monkeypatch): """Success path returns str.""" import json @@ -315,22 +325,19 @@ async def test_set_commit_status_success_returns_str(tool_ctx, monkeypatch): async def fake_infer(cwd: str, hint: object = None) -> str: return "owner/repo" - monkeypatch.setattr("autoskillit.server.tools_ci.infer_repo_from_remote", fake_infer) - tool_ctx.runner.push(_make_result(0, "", "")) + monkeypatch.setattr("autoskillit.server.tools.tools_ci.resolve_repo_from_remote", fake_infer) + tool_ctx_kitchen_open.runner.push(_make_result(0, "", "")) result = await set_commit_status(sha="abc123", state="success", context="ci", cwd="/tmp") assert isinstance(result, str) parsed = json.loads(result) assert parsed["success"] is True -# --------------------------------------------------------------------------- -# T2 — Uses infer_repo_from_remote, not gh repo view subprocess -# --------------------------------------------------------------------------- - - @pytest.mark.anyio -async def test_set_commit_status_uses_infer_repo_not_gh_subprocess(tool_ctx, monkeypatch): - """Must call infer_repo_from_remote, not launch gh repo view subprocess.""" +async def test_set_commit_status_uses_infer_repo_not_gh_subprocess( + tool_ctx_kitchen_open, monkeypatch +): + """Must call resolve_repo_from_remote, not launch gh repo view subprocess.""" from tests.conftest import _make_result calls: list[str] = [] @@ -339,13 +346,13 @@ async def fake_infer(cwd: str, hint: object = None) -> str: calls.append(cwd) return "owner/repo" - monkeypatch.setattr("autoskillit.server.tools_ci.infer_repo_from_remote", fake_infer) + monkeypatch.setattr("autoskillit.server.tools.tools_ci.resolve_repo_from_remote", fake_infer) # Only one runner push needed (the POST call, repo already resolved) - tool_ctx.runner.push(_make_result(0, "", "")) + tool_ctx_kitchen_open.runner.push(_make_result(0, "", "")) await set_commit_status(sha="abc", state="success", context="ci", cwd="/tmp") - assert calls, "infer_repo_from_remote was never called" + assert calls, "resolve_repo_from_remote was never called" # gh repo view must NOT have been invoked as a subprocess - for cmd_args, *_ in tool_ctx.runner.call_args_list: + for cmd_args, *_ in tool_ctx_kitchen_open.runner.call_args_list: assert not ("repo" in cmd_args and "view" in cmd_args), ( - "gh repo view subprocess was invoked; infer_repo_from_remote should be used instead" + "gh repo view subprocess was invoked; resolve_repo_from_remote should be used instead" ) diff --git a/tests/recipe/test_smoke_pipeline.py b/tests/server/test_smoke_pipeline.py similarity index 97% rename from tests/recipe/test_smoke_pipeline.py rename to tests/server/test_smoke_pipeline.py index 9820b38057..279bec77d3 100644 --- a/tests/recipe/test_smoke_pipeline.py +++ b/tests/server/test_smoke_pipeline.py @@ -24,7 +24,9 @@ import pytest from autoskillit.recipe.io import builtin_recipes_dir -from autoskillit.server.tools_recipe import list_recipes, validate_recipe +from autoskillit.server.tools.tools_recipe import list_recipes, validate_recipe + +pytestmark = [pytest.mark.layer("server"), pytest.mark.medium] PROJECT_ROOT = Path(__file__).resolve().parent.parent.parent SMOKE_SCRIPT = PROJECT_ROOT / ".autoskillit" / "recipes" / "smoke-test.yaml" @@ -35,7 +37,7 @@ # --------------------------------------------------------------------------- -@pytest.fixture(scope="module") +@pytest.fixture() def smoke_recipe(): from autoskillit.recipe.io import load_recipe as _load_recipe @@ -72,8 +74,8 @@ class TestSmokeScriptValidation: """Validate the smoke-test pipeline YAML structure.""" @pytest.fixture(autouse=True) - def _setup_ctx(self, tool_ctx): - """Pull tool_ctx into scope to trigger its monkeypatch side-effect. + def _setup_ctx(self, tool_ctx_kitchen_open): + """Pull tool_ctx_kitchen_open into scope to trigger its monkeypatch side-effect. The server module's _ctx is set by the tool_ctx fixture. Tests in this class call server tools (e.g. validate_recipe) that read _ctx. Without diff --git a/tests/server/test_smoke_recipe_scope_guard.py b/tests/server/test_smoke_recipe_scope_guard.py new file mode 100644 index 0000000000..d4a3f4bcde --- /dev/null +++ b/tests/server/test_smoke_recipe_scope_guard.py @@ -0,0 +1,34 @@ +"""Structural guard: smoke_recipe fixture must not use scope='module'.""" + +from __future__ import annotations + +import ast +from pathlib import Path + +import pytest + +pytestmark = [pytest.mark.layer("server"), pytest.mark.small] + + +def test_smoke_recipe_fixture_is_not_module_scoped(): + """smoke_recipe must not use scope='module'; Recipe is a mutable dataclass.""" + src = (Path(__file__).parent / "test_smoke_pipeline.py").read_text() + tree = ast.parse(src) + found_smoke_recipe = False + for node in ast.walk(tree): + if not isinstance(node, ast.FunctionDef) or node.name != "smoke_recipe": + continue + found_smoke_recipe = True + for decorator in node.decorator_list: + if not ( + isinstance(decorator, ast.Call) + and isinstance(decorator.func, ast.Attribute) + and decorator.func.attr == "fixture" + ): + continue + for kw in decorator.keywords: + if kw.arg == "scope": + assert not ( + isinstance(kw.value, ast.Constant) and kw.value.value == "module" + ), "smoke_recipe must not use scope='module' — Recipe is mutable" + assert found_smoke_recipe, "smoke_recipe fixture not found in test_smoke_pipeline.py" diff --git a/tests/server/test_state.py b/tests/server/test_state.py index bd5e9776a4..de82ed9de9 100644 --- a/tests/server/test_state.py +++ b/tests/server/test_state.py @@ -11,11 +11,15 @@ import pytest +pytestmark = [pytest.mark.layer("server"), pytest.mark.small] + def _make_mock_ctx(tmp_path: Path) -> MagicMock: """Return a minimal mock ToolContext for _initialize tests.""" + from autoskillit.core.types._type_plugin_source import MarketplaceInstall + ctx = MagicMock() - ctx.plugin_dir = None + ctx.plugin_source = MarketplaceInstall(cache_path=tmp_path) # Provide a minimal linux_tracing config stub tracing_cfg = MagicMock() tracing_cfg.tmpfs_path = str(tmp_path / "tmpfs") diff --git a/tests/server/test_tool_annotation_completeness.py b/tests/server/test_tool_annotation_completeness.py new file mode 100644 index 0000000000..7334d858f1 --- /dev/null +++ b/tests/server/test_tool_annotation_completeness.py @@ -0,0 +1,104 @@ +"""Runtime annotation test shield for MCP tool readOnlyHint semantics. + +Layer 2 — Pre-middleware: internal registry has non-None annotations on every tool. +Layer 3 — Post-middleware: wire output preserves annotations (readOnlyHint survives). +Layer 4 — Universal assertion: every tool has readOnlyHint=True on the wire. + +No registry cross-check — the invariant is universal: all tools are read-only. +Layers 1a/1b (AST) live in tests/arch/test_tool_annotation_completeness.py. +""" + +from __future__ import annotations + +import pytest + +pytestmark = [pytest.mark.layer("server"), pytest.mark.small] + + +class TestPreMiddlewareAnnotations: + """Layer 2: mcp.list_tools() (bypasses middleware) must return non-None annotations.""" + + @pytest.mark.anyio + async def test_all_tools_have_annotations_in_registry(self, kitchen_enabled): + """Every tool in the internal FastMCP registry must have a non-None annotations object + with a bool readOnlyHint. + + This catches tools that omit annotations= from their @mcp.tool() decorator, + which leaves them with no readOnlyHint even after the middleware fix. + """ + from autoskillit.server import mcp + + all_tools = await mcp.list_tools() + violations: list[str] = [] + for tool in all_tools: + if tool.annotations is None: + violations.append(f" {tool.name!r}: annotations is None") + elif tool.annotations.readOnlyHint is None: + violations.append(f" {tool.name!r}: annotations.readOnlyHint is None") + + assert not violations, ( + "The following tools are missing readOnlyHint in their internal registry entry.\n" + "Add annotations={'readOnlyHint': True/False} to the @mcp.tool() decorator:\n\n" + + "\n".join(violations) + ) + + +class TestPostMiddlewareAnnotations: + """Layer 3: Client(mcp).list_tools() (through middleware) must preserve annotations.""" + + @pytest.mark.anyio + async def test_annotations_survive_middleware(self, kitchen_enabled): + """Wire output must preserve annotations.readOnlyHint for every tool. + + If the middleware strips annotations, this test fails — catching any future + regression that re-introduces unconditional annotation stripping. + """ + from fastmcp.client import Client + + from autoskillit.server import mcp + + async with Client(mcp) as client: + tools = await client.list_tools() + + violations: list[str] = [] + for tool in tools: + if tool.annotations is None: + violations.append(f" {tool.name!r}: annotations stripped by middleware") + + assert not violations, ( + "The following tools have annotations=None after passing through the middleware.\n" + "The middleware must preserve annotations (only strip output_schema and title):\n\n" + + "\n".join(violations) + ) + + @pytest.mark.anyio + async def test_all_tools_have_readonly_hint_true(self, kitchen_enabled): + """Layer 4 — Universal invariant: every tool must have readOnlyHint=True. + + All pipelines operate on independent branches and worktrees with zero + cross-pipeline interference. readOnlyHint=False serializes parallel tool + calls and causes catastrophic pipeline slowdowns (40+ minutes instead of + 5 minutes for concurrent CI watches). Zero exceptions. + """ + from fastmcp.client import Client + + from autoskillit.server import mcp + + async with Client(mcp) as client: + tools = await client.list_tools() + + violations: list[str] = [] + for tool in tools: + if tool.annotations is None: + continue + if tool.annotations.readOnlyHint is not True: + violations.append( + f" {tool.name!r}: readOnlyHint={tool.annotations.readOnlyHint!r} " + f"(must be True)" + ) + + assert not violations, ( + "Every MCP tool must have readOnlyHint=True. All pipelines operate on " + "independent branches/worktrees — there is no valid reason for False.\n" + "Fix the @mcp.tool(annotations=...) decorator:\n\n" + "\n".join(violations) + ) diff --git a/tests/server/test_tool_exception_boundary.py b/tests/server/test_tool_exception_boundary.py index e022b97272..8444e7b14f 100644 --- a/tests/server/test_tool_exception_boundary.py +++ b/tests/server/test_tool_exception_boundary.py @@ -14,17 +14,19 @@ from autoskillit.pipeline.mcp_response import DefaultMcpResponseLog +pytestmark = [pytest.mark.layer("server"), pytest.mark.small] + class TestToolExceptionBoundary: @pytest.mark.anyio async def test_exception_produces_structured_error_json(self): - from autoskillit.server.helpers import track_response_size + from autoskillit.server._notify import track_response_size @track_response_size("test_tool") async def boom(): raise RuntimeError("boom") - with patch("autoskillit.server.helpers._get_ctx_or_none") as mock_ctx: + with patch("autoskillit.server._notify._get_ctx_or_none") as mock_ctx: log = DefaultMcpResponseLog() mock_ctx.return_value = MagicMock( response_log=log, @@ -40,13 +42,13 @@ async def boom(): @pytest.mark.anyio async def test_assertion_error_produces_structured_error(self): - from autoskillit.server.helpers import track_response_size + from autoskillit.server._notify import track_response_size @track_response_size("test_tool") async def bad(): raise AssertionError("No subprocess runner configured") - with patch("autoskillit.server.helpers._get_ctx_or_none", return_value=None): + with patch("autoskillit.server._notify._get_ctx_or_none", return_value=None): result = await bad() data = json.loads(result) @@ -55,13 +57,13 @@ async def bad(): @pytest.mark.anyio async def test_os_error_produces_structured_error(self): - from autoskillit.server.helpers import track_response_size + from autoskillit.server._notify import track_response_size @track_response_size("test_tool") async def bad(): raise OSError("[Errno 2] No such file or directory: '/bad/path'") - with patch("autoskillit.server.helpers._get_ctx_or_none", return_value=None): + with patch("autoskillit.server._notify._get_ctx_or_none", return_value=None): result = await bad() data = json.loads(result) @@ -70,7 +72,7 @@ async def bad(): @pytest.mark.anyio async def test_normal_return_passes_through_unchanged(self): - from autoskillit.server.helpers import track_response_size + from autoskillit.server._notify import track_response_size expected = '{"success": true, "result": "ok"}' @@ -78,21 +80,21 @@ async def test_normal_return_passes_through_unchanged(self): async def ok(): return expected - with patch("autoskillit.server.helpers._get_ctx_or_none", return_value=None): + with patch("autoskillit.server._notify._get_ctx_or_none", return_value=None): result = await ok() assert result == expected @pytest.mark.anyio async def test_response_size_still_recorded_on_exception(self): - from autoskillit.server.helpers import track_response_size + from autoskillit.server._notify import track_response_size @track_response_size("test_tool") async def bad(): raise ValueError("kaboom") log = DefaultMcpResponseLog() - with patch("autoskillit.server.helpers._get_ctx_or_none") as mock_ctx: + with patch("autoskillit.server._notify._get_ctx_or_none") as mock_ctx: mock_ctx.return_value = MagicMock( response_log=log, config=MagicMock(mcp_response=MagicMock(alert_threshold_tokens=0)), diff --git a/tests/server/test_tools_bootstrap.py b/tests/server/test_tools_bootstrap.py new file mode 100644 index 0000000000..c9bd51a1a1 --- /dev/null +++ b/tests/server/test_tools_bootstrap.py @@ -0,0 +1,390 @@ +"""Tests for bootstrap composite MCP tools: bootstrap_clone, claim_and_resolve_issue, +create_and_publish_branch.""" + +from __future__ import annotations + +import json +from unittest.mock import AsyncMock, MagicMock + +import pytest + +from autoskillit.server.tools.tools_clone import bootstrap_clone +from autoskillit.server.tools.tools_git import create_and_publish_branch +from autoskillit.server.tools.tools_issue_composite import claim_and_resolve_issue +from tests.conftest import _make_result +from tests.server.conftest import assert_step_timed + +pytestmark = [pytest.mark.layer("server"), pytest.mark.small] + + +class TestBootstrapClone: + @pytest.mark.anyio + async def test_bootstrap_clone_success(self, tool_ctx_kitchen_open, tmp_path): + """bootstrap_clone returns work_dir, remote_url, base_sha, merge_target.""" + mock_mgr = MagicMock() + mock_mgr.clone_repo.return_value = { + "clone_path": str(tmp_path), + "source_dir": "/src", + "remote_url": "https://github.com/o/r.git", + } + tool_ctx_kitchen_open.clone_mgr = mock_mgr + tool_ctx_kitchen_open.runner.push(_make_result(0, "abc123def\n", "")) + result = json.loads( + await bootstrap_clone(source_dir="/src", run_name="impl", base_branch="main") + ) + assert result["work_dir"] == str(tmp_path) + assert result["remote_url"] == "https://github.com/o/r.git" + assert result["base_sha"] == "abc123def" + assert result["merge_target"] == "main" + + @pytest.mark.anyio + async def test_bootstrap_clone_gate_closed(self, tool_ctx): + tool_ctx.gate.enabled = False + result = json.loads( + await bootstrap_clone(source_dir="/src", run_name="impl", base_branch="main") + ) + assert result["subtype"] == "gate_error" + + @pytest.mark.anyio + async def test_bootstrap_clone_clone_failure(self, tool_ctx_kitchen_open): + mock_mgr = MagicMock() + mock_mgr.clone_repo.side_effect = RuntimeError("disk full") + tool_ctx_kitchen_open.clone_mgr = mock_mgr + result = json.loads( + await bootstrap_clone(source_dir="/src", run_name="impl", base_branch="main") + ) + assert "error" in result + assert "disk full" in result["error"] + + @pytest.mark.anyio + async def test_bootstrap_clone_revparse_failure(self, tool_ctx_kitchen_open, tmp_path): + mock_mgr = MagicMock() + mock_mgr.clone_repo.return_value = { + "clone_path": str(tmp_path), + "source_dir": "/src", + "remote_url": "url", + } + tool_ctx_kitchen_open.clone_mgr = mock_mgr + tool_ctx_kitchen_open.runner.push(_make_result(128, "", "fatal: not a git repo\n")) + result = json.loads( + await bootstrap_clone(source_dir="/src", run_name="impl", base_branch="main") + ) + assert "error" in result + assert "rev-parse" in result["error"] + + @pytest.mark.anyio + async def test_bootstrap_clone_timing(self, tool_ctx_kitchen_open, tmp_path): + mock_mgr = MagicMock() + mock_mgr.clone_repo.return_value = { + "clone_path": str(tmp_path), + "source_dir": "/src", + "remote_url": "url", + } + tool_ctx_kitchen_open.clone_mgr = mock_mgr + tool_ctx_kitchen_open.runner.push(_make_result(0, "sha\n", "")) + await bootstrap_clone( + source_dir="/src", run_name="impl", base_branch="main", step_name="bootstrap" + ) + assert_step_timed(tool_ctx_kitchen_open.timing_log, "bootstrap") + + @pytest.mark.anyio + async def test_bootstrap_clone_sub_timings(self, tool_ctx_kitchen_open, tmp_path): + mock_mgr = MagicMock() + mock_mgr.clone_repo.return_value = { + "clone_path": str(tmp_path), + "source_dir": "/src", + "remote_url": "url", + } + tool_ctx_kitchen_open.clone_mgr = mock_mgr + tool_ctx_kitchen_open.runner.push(_make_result(0, "sha\n", "")) + result = json.loads( + await bootstrap_clone(source_dir="/src", run_name="impl", base_branch="main") + ) + assert "timings" in result + assert "clone_ms" in result["timings"] + assert "rev_parse_ms" in result["timings"] + + +class TestClaimAndResolveIssue: + @pytest.mark.anyio + async def test_claim_and_resolve_issue_claimed(self, tool_ctx_kitchen_open): + tool_ctx_kitchen_open.github_client = AsyncMock() + tool_ctx_kitchen_open.github_client.fetch_title = AsyncMock( + return_value={"success": True, "number": 42, "title": "Fix bug", "slug": "fix-bug"} + ) + tool_ctx_kitchen_open.github_client.fetch_issue = AsyncMock( + return_value={"success": True, "labels": []} + ) + tool_ctx_kitchen_open.github_client.ensure_label = AsyncMock( + return_value={"success": True} + ) + tool_ctx_kitchen_open.github_client.swap_labels = AsyncMock(return_value={"success": True}) + result = json.loads(await claim_and_resolve_issue(issue_url="owner/repo#42")) + assert result["claimed"] is True + assert result["issue_number"] == 42 + assert result["issue_title"] == "Fix bug" + assert result["issue_slug"] == "fix-bug" + + @pytest.mark.anyio + async def test_claim_and_resolve_issue_already_claimed(self, tool_ctx_kitchen_open): + tool_ctx_kitchen_open.github_client = AsyncMock() + tool_ctx_kitchen_open.github_client.fetch_title = AsyncMock( + return_value={"success": True, "number": 42, "title": "Fix bug", "slug": "fix-bug"} + ) + tool_ctx_kitchen_open.github_client.fetch_issue = AsyncMock( + return_value={ + "success": True, + "labels": [{"name": "in-progress"}], + } + ) + result = json.loads(await claim_and_resolve_issue(issue_url="owner/repo#42")) + assert result["claimed"] is False + assert result["issue_number"] == 42 + assert result["issue_title"] == "Fix bug" + assert result["issue_slug"] == "fix-bug" + + @pytest.mark.anyio + async def test_claim_and_resolve_issue_reentry(self, tool_ctx_kitchen_open): + tool_ctx_kitchen_open.github_client = AsyncMock() + tool_ctx_kitchen_open.github_client.fetch_title = AsyncMock( + return_value={"success": True, "number": 42, "title": "Fix bug", "slug": "fix-bug"} + ) + tool_ctx_kitchen_open.github_client.fetch_issue = AsyncMock( + return_value={ + "success": True, + "labels": [{"name": "in-progress"}], + } + ) + result = json.loads( + await claim_and_resolve_issue(issue_url="owner/repo#42", allow_reentry=True) + ) + assert result["claimed"] is True + assert result["reentry"] is True + + @pytest.mark.anyio + async def test_claim_and_resolve_issue_title_failure(self, tool_ctx_kitchen_open): + tool_ctx_kitchen_open.github_client = AsyncMock() + tool_ctx_kitchen_open.github_client.fetch_title = AsyncMock( + return_value={"success": False, "error": "not found"} + ) + result = json.loads(await claim_and_resolve_issue(issue_url="owner/repo#42")) + assert result["success"] is False + assert "not found" in result["error"] + + @pytest.mark.anyio + async def test_claim_and_resolve_issue_gate_closed(self, tool_ctx): + tool_ctx.gate.enabled = False + result = json.loads(await claim_and_resolve_issue(issue_url="owner/repo#42")) + assert result["subtype"] == "gate_error" + + @pytest.mark.anyio + async def test_claim_and_resolve_issue_timings(self, tool_ctx_kitchen_open): + tool_ctx_kitchen_open.github_client = AsyncMock() + tool_ctx_kitchen_open.github_client.fetch_title = AsyncMock( + return_value={"success": True, "number": 42, "title": "X", "slug": "x"} + ) + tool_ctx_kitchen_open.github_client.fetch_issue = AsyncMock( + return_value={"success": True, "labels": []} + ) + tool_ctx_kitchen_open.github_client.ensure_label = AsyncMock( + return_value={"success": True} + ) + tool_ctx_kitchen_open.github_client.swap_labels = AsyncMock(return_value={"success": True}) + result = json.loads(await claim_and_resolve_issue(issue_url="owner/repo#42")) + assert "timings" in result + assert "fetch_title_ms" in result["timings"] + assert "claim_ms" in result["timings"] + + @pytest.mark.anyio + async def test_claim_and_resolve_with_queued_label(self, tool_ctx_kitchen_open): + """claim_and_resolve_issue with label=queued uses registry color and removes fail.""" + tool_ctx_kitchen_open.github_client = AsyncMock() + tool_ctx_kitchen_open.github_client.fetch_title = AsyncMock( + return_value={"success": True, "number": 42, "title": "Fix bug", "slug": "fix-bug"} + ) + tool_ctx_kitchen_open.github_client.fetch_issue = AsyncMock( + return_value={"success": True, "labels": []} + ) + tool_ctx_kitchen_open.github_client.ensure_label = AsyncMock( + return_value={"success": True} + ) + tool_ctx_kitchen_open.github_client.swap_labels = AsyncMock(return_value={"success": True}) + result = json.loads( + await claim_and_resolve_issue(issue_url="owner/repo#42", label="queued") + ) + assert result["claimed"] is True + tool_ctx_kitchen_open.github_client.ensure_label.assert_called_once_with( + "owner", + "repo", + "queued", + color="c2e0c6", + description="Issue claimed by orchestrator, waiting for recipe pickup", + ) + call_kwargs = tool_ctx_kitchen_open.github_client.swap_labels.call_args.kwargs + assert set(call_kwargs["remove_labels"]) == {"fail"} + assert call_kwargs["add_labels"] == ["queued"] + + @pytest.mark.anyio + async def test_claim_and_resolve_default_removes_queued(self, tool_ctx_kitchen_open): + """claim_and_resolve_issue with default label removes both queued and fail.""" + tool_ctx_kitchen_open.github_client = AsyncMock() + tool_ctx_kitchen_open.github_client.fetch_title = AsyncMock( + return_value={"success": True, "number": 42, "title": "Fix bug", "slug": "fix-bug"} + ) + tool_ctx_kitchen_open.github_client.fetch_issue = AsyncMock( + return_value={"success": True, "labels": []} + ) + tool_ctx_kitchen_open.github_client.ensure_label = AsyncMock( + return_value={"success": True} + ) + tool_ctx_kitchen_open.github_client.swap_labels = AsyncMock(return_value={"success": True}) + result = json.loads(await claim_and_resolve_issue(issue_url="owner/repo#42")) + assert result["claimed"] is True + call_kwargs = tool_ctx_kitchen_open.github_client.swap_labels.call_args.kwargs + assert set(call_kwargs["remove_labels"]) >= {"queued", "fail"} + assert call_kwargs["add_labels"] == ["in-progress"] + + +class TestCreateAndPublishBranch: + @pytest.mark.anyio + async def test_create_and_publish_branch_success(self, tool_ctx_kitchen_open, tmp_path): + # ls-remote: empty (branch available) + tool_ctx_kitchen_open.runner.push(_make_result(0, "", "")) + # branch --show-current + tool_ctx_kitchen_open.runner.push(_make_result(0, "main\n", "")) + # git checkout -b + tool_ctx_kitchen_open.runner.push(_make_result(0, "", "")) + mock_mgr = MagicMock() + mock_mgr.push_to_remote.return_value = {"success": True, "stderr": ""} + tool_ctx_kitchen_open.clone_mgr = mock_mgr + result = json.loads( + await create_and_publish_branch( + issue_slug="fix-bug", + run_name="impl", + issue_number="42", + work_dir=str(tmp_path), + remote_url="https://github.com/o/r.git", + ) + ) + assert result["merge_target"] == "fix-bug/42" + + @pytest.mark.anyio + async def test_create_and_publish_branch_collision(self, tool_ctx_kitchen_open, tmp_path): + # ls-remote: branch exists + tool_ctx_kitchen_open.runner.push(_make_result(0, "abc123\trefs/heads/fix-bug/42\n", "")) + # ls-remote: suffix -2 is free + tool_ctx_kitchen_open.runner.push(_make_result(0, "", "")) + # branch --show-current + tool_ctx_kitchen_open.runner.push(_make_result(0, "main\n", "")) + # git checkout -b + tool_ctx_kitchen_open.runner.push(_make_result(0, "", "")) + mock_mgr = MagicMock() + mock_mgr.push_to_remote.return_value = {"success": True, "stderr": ""} + tool_ctx_kitchen_open.clone_mgr = mock_mgr + result = json.loads( + await create_and_publish_branch( + issue_slug="fix-bug", + run_name="impl", + issue_number="42", + work_dir=str(tmp_path), + remote_url="https://github.com/o/r.git", + ) + ) + assert result["merge_target"] == "fix-bug/42-2" + + @pytest.mark.anyio + async def test_create_and_publish_branch_push_failure(self, tool_ctx_kitchen_open, tmp_path): + tool_ctx_kitchen_open.runner.push(_make_result(0, "", "")) + tool_ctx_kitchen_open.runner.push(_make_result(0, "main\n", "")) + tool_ctx_kitchen_open.runner.push(_make_result(0, "", "")) + mock_mgr = MagicMock() + mock_mgr.push_to_remote.return_value = { + "success": False, + "stderr": "rejected", + "error_type": "", + } + tool_ctx_kitchen_open.clone_mgr = mock_mgr + result = json.loads( + await create_and_publish_branch( + issue_slug="fix-bug", + run_name="impl", + issue_number="42", + work_dir=str(tmp_path), + remote_url="https://github.com/o/r.git", + ) + ) + assert "error" in result + + @pytest.mark.anyio + async def test_create_and_publish_branch_gate_closed(self, tool_ctx): + tool_ctx.gate.enabled = False + result = json.loads( + await create_and_publish_branch( + issue_slug="x", + run_name="impl", + issue_number="1", + work_dir="/tmp", + remote_url="url", + ) + ) + assert result["subtype"] == "gate_error" + + @pytest.mark.anyio + async def test_create_and_publish_branch_no_issue(self, tool_ctx_kitchen_open, tmp_path): + tool_ctx_kitchen_open.runner.push(_make_result(0, "", "")) + tool_ctx_kitchen_open.runner.push(_make_result(0, "main\n", "")) + tool_ctx_kitchen_open.runner.push(_make_result(0, "", "")) + mock_mgr = MagicMock() + mock_mgr.push_to_remote.return_value = {"success": True, "stderr": ""} + tool_ctx_kitchen_open.clone_mgr = mock_mgr + result = json.loads( + await create_and_publish_branch( + issue_slug="", + run_name="impl", + issue_number="", + work_dir=str(tmp_path), + remote_url="url", + ) + ) + # branch name is "impl/YYYYMMDD" + assert result["merge_target"].startswith("impl/") + + @pytest.mark.anyio + async def test_create_and_publish_branch_timings(self, tool_ctx_kitchen_open, tmp_path): + tool_ctx_kitchen_open.runner.push(_make_result(0, "", "")) + tool_ctx_kitchen_open.runner.push(_make_result(0, "main\n", "")) + tool_ctx_kitchen_open.runner.push(_make_result(0, "", "")) + mock_mgr = MagicMock() + mock_mgr.push_to_remote.return_value = {"success": True, "stderr": ""} + tool_ctx_kitchen_open.clone_mgr = mock_mgr + result = json.loads( + await create_and_publish_branch( + issue_slug="x", + run_name="impl", + issue_number="1", + work_dir=str(tmp_path), + remote_url="url", + ) + ) + assert "timings" in result + assert "branch_create_ms" in result["timings"] + assert "push_ms" in result["timings"] + + +def test_implementation_recipe_valid(): + """Consolidated bootstrap steps pass recipe validation.""" + from pathlib import Path + + from autoskillit.recipe.io import load_recipe + from autoskillit.recipe.validator import validate_recipe_structure + + recipe_path = ( + Path(__file__).parent.parent.parent + / "src" + / "autoskillit" + / "recipes" + / "implementation.yaml" + ) + recipe = load_recipe(recipe_path) + errors = validate_recipe_structure(recipe) + assert not errors, f"implementation.yaml failed validation: {errors}" diff --git a/tests/server/test_tools_ci.py b/tests/server/test_tools_ci.py index 57364c4348..db8c344803 100644 --- a/tests/server/test_tools_ci.py +++ b/tests/server/test_tools_ci.py @@ -5,17 +5,16 @@ import json from unittest.mock import AsyncMock, patch -import httpx import pytest from autoskillit.core import PRState, SubprocessResult, TerminationReason from autoskillit.pipeline.gate import GATED_TOOLS, UNGATED_TOOLS, DefaultGateState -from autoskillit.server.tools_ci import ( - check_repo_merge_state, - get_ci_status, - wait_for_ci, - wait_for_merge_queue, -) +from autoskillit.server.tools.tools_ci_merge_queue import wait_for_merge_queue +from autoskillit.server.tools.tools_ci_watch import get_ci_status, wait_for_ci +from tests.fakes import InMemoryCIWatcher, InMemoryMergeQueueWatcher +from tests.server.conftest import assert_no_timing, assert_step_timed + +pytestmark = [pytest.mark.layer("server"), pytest.mark.small] # --------------------------------------------------------------------------- # Gate membership @@ -51,13 +50,12 @@ async def test_wait_for_ci_gate_check(tool_ctx): @pytest.mark.anyio -async def test_wait_for_ci_success_response(tool_ctx): - mock_watcher = AsyncMock() - mock_watcher.wait = AsyncMock( - return_value={"run_id": 12345, "conclusion": "success", "failed_jobs": []} +async def test_wait_for_ci_success_response(tool_ctx_kitchen_open): + watcher = InMemoryCIWatcher( + wait_result={"run_id": 12345, "conclusion": "success", "failed_jobs": []} ) - tool_ctx.ci_watcher = mock_watcher - tool_ctx.runner.push( + tool_ctx_kitchen_open.ci_watcher = watcher + tool_ctx_kitchen_open.runner.push( SubprocessResult( returncode=0, stdout="abc123\n", @@ -75,17 +73,16 @@ async def test_wait_for_ci_success_response(tool_ctx): @pytest.mark.anyio -async def test_wait_for_ci_failure_response(tool_ctx): - mock_watcher = AsyncMock() - mock_watcher.wait = AsyncMock( - return_value={ +async def test_wait_for_ci_failure_response(tool_ctx_kitchen_open): + watcher = InMemoryCIWatcher( + wait_result={ "run_id": 12345, "conclusion": "failure", "failed_jobs": ["test", "lint"], } ) - tool_ctx.ci_watcher = mock_watcher - tool_ctx.runner.push( + tool_ctx_kitchen_open.ci_watcher = watcher + tool_ctx_kitchen_open.runner.push( SubprocessResult( returncode=0, stdout="abc123\n", @@ -107,14 +104,13 @@ async def test_wait_for_ci_failure_response(tool_ctx): @pytest.mark.anyio -async def test_wait_for_ci_infers_head_sha(tool_ctx): +async def test_wait_for_ci_infers_head_sha(tool_ctx_kitchen_open): """When head_sha is not provided, it's inferred via git rev-parse HEAD.""" - mock_watcher = AsyncMock() - mock_watcher.wait = AsyncMock( - return_value={"run_id": 1, "conclusion": "success", "failed_jobs": []} + watcher = InMemoryCIWatcher( + wait_result={"run_id": 1, "conclusion": "success", "failed_jobs": []} ) - tool_ctx.ci_watcher = mock_watcher - tool_ctx.runner.push( + tool_ctx_kitchen_open.ci_watcher = watcher + tool_ctx_kitchen_open.runner.push( SubprocessResult( returncode=0, stdout="abc123\n", @@ -127,21 +123,19 @@ async def test_wait_for_ci_infers_head_sha(tool_ctx): await wait_for_ci("main", cwd="/some/repo") # Verify that wait was called with the inferred head_sha inside scope - call_kwargs = mock_watcher.wait.call_args - assert call_kwargs.kwargs["scope"].head_sha == "abc123" + assert watcher.wait_calls[-1]["scope"].head_sha == "abc123" @pytest.mark.anyio -async def test_wait_for_ci_head_sha_uses_runner(tool_ctx): +async def test_wait_for_ci_head_sha_uses_runner(tool_ctx_kitchen_open): """git rev-parse HEAD must flow through MockSubprocessRunner, not raw asyncio.""" - mock_watcher = AsyncMock() - mock_watcher.wait = AsyncMock( - return_value={"run_id": 1, "conclusion": "success", "failed_jobs": []} + watcher = InMemoryCIWatcher( + wait_result={"run_id": 1, "conclusion": "success", "failed_jobs": []} ) - tool_ctx.ci_watcher = mock_watcher + tool_ctx_kitchen_open.ci_watcher = watcher # Pre-configure runner to return a valid SHA when git rev-parse is called - tool_ctx.runner.push( + tool_ctx_kitchen_open.runner.push( SubprocessResult( returncode=0, stdout="deadbeef\n", @@ -154,13 +148,12 @@ async def test_wait_for_ci_head_sha_uses_runner(tool_ctx): await wait_for_ci("main", cwd="/some/repo") # Runner must have been called with the git command - assert tool_ctx.runner.call_args_list, "runner was never called" - cmd = tool_ctx.runner.call_args_list[0][0] + assert tool_ctx_kitchen_open.runner.call_args_list, "runner was never called" + cmd = tool_ctx_kitchen_open.runner.call_args_list[0][0] assert cmd == ["git", "rev-parse", "HEAD"], f"Unexpected runner call: {cmd}" # SHA extracted from runner output must have been passed to the CI watcher - scope = mock_watcher.wait.call_args.kwargs["scope"] - assert scope.head_sha == "deadbeef" + assert watcher.wait_calls[-1]["scope"].head_sha == "deadbeef" # --------------------------------------------------------------------------- @@ -169,8 +162,8 @@ async def test_wait_for_ci_head_sha_uses_runner(tool_ctx): @pytest.mark.anyio -async def test_wait_for_ci_no_watcher(tool_ctx): - tool_ctx.ci_watcher = None +async def test_wait_for_ci_no_watcher(tool_ctx_kitchen_open): + tool_ctx_kitchen_open.ci_watcher = None result = json.loads(await wait_for_ci("main")) assert result["conclusion"] == "error" assert "not configured" in result["error"] @@ -190,9 +183,9 @@ async def test_get_ci_status_gate_check(tool_ctx): @pytest.mark.anyio -async def test_get_ci_status_missing_branch_and_run_id(tool_ctx): - mock_watcher = AsyncMock() - tool_ctx.ci_watcher = mock_watcher +async def test_get_ci_status_missing_branch_and_run_id(tool_ctx_kitchen_open): + watcher = InMemoryCIWatcher() + tool_ctx_kitchen_open.ci_watcher = watcher result = json.loads(await get_ci_status()) assert result["runs"] == [] @@ -200,8 +193,8 @@ async def test_get_ci_status_missing_branch_and_run_id(tool_ctx): @pytest.mark.anyio -async def test_get_ci_status_no_watcher(tool_ctx): - tool_ctx.ci_watcher = None +async def test_get_ci_status_no_watcher(tool_ctx_kitchen_open): + tool_ctx_kitchen_open.ci_watcher = None result = json.loads(await get_ci_status(branch="main")) assert result["runs"] == [] assert "not configured" in result["error"] @@ -213,21 +206,42 @@ async def test_get_ci_status_no_watcher(tool_ctx): @pytest.mark.anyio -async def test_wait_for_ci_passes_event_to_scope(tool_ctx): +async def test_wait_for_ci_passes_event_to_scope(tool_ctx_kitchen_open): """wait_for_ci must propagate event param into CIRunScope.""" - captured_scope = None + watcher = InMemoryCIWatcher( + wait_result={"run_id": 1, "conclusion": "success", "failed_jobs": []} + ) + tool_ctx_kitchen_open.ci_watcher = watcher + await wait_for_ci(branch="main", event="push", cwd="/tmp") + assert len(watcher.wait_calls) == 1 + assert watcher.wait_calls[-1]["scope"].event == "push" - async def _side_effect(branch, *, repo, scope, **kw): - nonlocal captured_scope - captured_scope = scope - return {"run_id": 1, "conclusion": "success", "failed_jobs": []} - mock_watcher = AsyncMock() - mock_watcher.wait = AsyncMock(side_effect=_side_effect) - tool_ctx.ci_watcher = mock_watcher - await wait_for_ci(branch="main", event="push", cwd="/tmp") - assert captured_scope is not None - assert captured_scope.event == "push" +# --------------------------------------------------------------------------- +# wait_for_ci lookback_seconds forwarding +# --------------------------------------------------------------------------- + + +@pytest.mark.anyio +async def test_wait_for_ci_forwards_lookback_seconds(tool_ctx_kitchen_open): + """wait_for_ci must propagate lookback_seconds to ci_watcher.wait().""" + watcher = InMemoryCIWatcher( + wait_result={"run_id": 1, "conclusion": "success", "failed_jobs": []} + ) + tool_ctx_kitchen_open.ci_watcher = watcher + await wait_for_ci(branch="main", lookback_seconds=7200, cwd="/tmp") + assert watcher.wait_calls[-1]["lookback_seconds"] == 7200 + + +@pytest.mark.anyio +async def test_wait_for_ci_lookback_defaults_to_3600(tool_ctx_kitchen_open): + """wait_for_ci default lookback_seconds is 3600 (1 hour).""" + watcher = InMemoryCIWatcher( + wait_result={"run_id": 1, "conclusion": "success", "failed_jobs": []} + ) + tool_ctx_kitchen_open.ci_watcher = watcher + await wait_for_ci(branch="main", cwd="/tmp") + assert watcher.wait_calls[-1]["lookback_seconds"] == 3600 # --------------------------------------------------------------------------- @@ -249,12 +263,11 @@ async def test_gate_closed_returns_gate_error(tool_ctx): @pytest.mark.anyio -async def test_delegates_to_merge_queue_watcher(tool_ctx): - mock_watcher = AsyncMock() - mock_watcher.wait = AsyncMock( - return_value={"success": True, "pr_state": "merged", "reason": "PR merged"} +async def test_delegates_to_merge_queue_watcher(tool_ctx_kitchen_open): + watcher = InMemoryMergeQueueWatcher( + wait_result={"success": True, "pr_state": "merged", "reason": "PR merged"} ) - tool_ctx.merge_queue_watcher = mock_watcher + tool_ctx_kitchen_open.merge_queue_watcher = watcher with patch( "autoskillit.execution.remote_resolver.asyncio.create_subprocess_exec", @@ -268,23 +281,21 @@ async def test_delegates_to_merge_queue_watcher(tool_ctx): mock_proc.return_value = proc_inst result = json.loads( - await wait_for_merge_queue(pr_number=42, target_branch="integration", cwd=".") + await wait_for_merge_queue(pr_number=42, target_branch="develop", cwd=".") ) assert result["pr_state"] == "merged" - mock_watcher.wait.assert_called_once() - call_kwargs = mock_watcher.wait.call_args - assert call_kwargs.kwargs["pr_number"] == 42 - assert call_kwargs.kwargs["target_branch"] == "integration" + assert len(watcher.wait_calls) == 1 + assert watcher.wait_calls[-1]["pr_number"] == 42 + assert watcher.wait_calls[-1]["target_branch"] == "develop" @pytest.mark.anyio -async def test_infers_repo_from_git_remote_when_repo_empty(tool_ctx): - mock_watcher = AsyncMock() - mock_watcher.wait = AsyncMock( - return_value={"success": True, "pr_state": "merged", "reason": "PR merged"} +async def test_infers_repo_from_git_remote_when_repo_empty(tool_ctx_kitchen_open): + watcher = InMemoryMergeQueueWatcher( + wait_result={"success": True, "pr_state": "merged", "reason": "PR merged"} ) - tool_ctx.merge_queue_watcher = mock_watcher + tool_ctx_kitchen_open.merge_queue_watcher = watcher with patch( "autoskillit.execution.remote_resolver.asyncio.create_subprocess_exec", @@ -299,17 +310,15 @@ async def test_infers_repo_from_git_remote_when_repo_empty(tool_ctx): await wait_for_merge_queue(pr_number=42, target_branch="main", cwd=".", repo="") - call_kwargs = mock_watcher.wait.call_args - assert call_kwargs.kwargs["repo"] == "owner/repo" + assert watcher.wait_calls[-1]["repo"] == "owner/repo" @pytest.mark.anyio -async def test_explicit_repo_skips_subprocess(tool_ctx): - mock_watcher = AsyncMock() - mock_watcher.wait = AsyncMock( - return_value={"success": True, "pr_state": "merged", "reason": "PR merged"} +async def test_explicit_repo_skips_subprocess(tool_ctx_kitchen_open): + watcher = InMemoryMergeQueueWatcher( + wait_result={"success": True, "pr_state": "merged", "reason": "PR merged"} ) - tool_ctx.merge_queue_watcher = mock_watcher + tool_ctx_kitchen_open.merge_queue_watcher = watcher with patch( "autoskillit.execution.remote_resolver.asyncio.create_subprocess_exec", @@ -323,13 +332,12 @@ async def test_explicit_repo_skips_subprocess(tool_ctx): ) mock_proc.assert_not_called() - call_kwargs = mock_watcher.wait.call_args - assert call_kwargs.kwargs["repo"] == "owner/explicit-repo" + assert watcher.wait_calls[-1]["repo"] == "owner/explicit-repo" @pytest.mark.anyio -async def test_watcher_none_returns_error(tool_ctx): - tool_ctx.merge_queue_watcher = None +async def test_watcher_none_returns_error(tool_ctx_kitchen_open): + tool_ctx_kitchen_open.merge_queue_watcher = None result = json.loads(await wait_for_merge_queue(pr_number=42, target_branch="main", cwd=".")) assert result["success"] is False assert "pr_state" in result @@ -342,17 +350,18 @@ async def test_watcher_none_returns_error(tool_ctx): @pytest.mark.anyio -async def test_wait_for_ci_parses_remote_url_to_resolve_repo(tool_ctx): +async def test_wait_for_ci_parses_remote_url_to_resolve_repo(tool_ctx_kitchen_open): """When remote_url is provided, wait_for_ci must parse it to owner/repo and pass that to the watcher without calling any subprocess.""" - mock_watcher = AsyncMock() - mock_watcher.wait.return_value = { - "conclusion": "success", - "run_id": 1, - "failed_jobs": [], - "head_sha": "abc123", - } - tool_ctx.ci_watcher = mock_watcher + watcher = InMemoryCIWatcher( + wait_result={ + "conclusion": "success", + "run_id": 1, + "failed_jobs": [], + "head_sha": "abc123", + } + ) + tool_ctx_kitchen_open.ci_watcher = watcher result = json.loads( await wait_for_ci( @@ -362,28 +371,28 @@ async def test_wait_for_ci_parses_remote_url_to_resolve_repo(tool_ctx): ) ) assert result["conclusion"] == "success" - call_kwargs = mock_watcher.wait.call_args - assert call_kwargs.kwargs.get("repo") == "owner/repo" + assert watcher.wait_calls[-1].get("repo") == "owner/repo" @pytest.mark.anyio -async def test_wait_for_ci_remote_url_wins_over_empty_repo(tool_ctx): +async def test_wait_for_ci_remote_url_wins_over_empty_repo(tool_ctx_kitchen_open): """remote_url= supersedes repo='' — hint priority in resolve_remote_repo.""" - mock_watcher = AsyncMock() - mock_watcher.wait.return_value = { - "conclusion": "success", - "run_id": 1, - "failed_jobs": [], - "head_sha": "abc", - } - tool_ctx.ci_watcher = mock_watcher + watcher = InMemoryCIWatcher( + wait_result={ + "conclusion": "success", + "run_id": 1, + "failed_jobs": [], + "head_sha": "abc", + } + ) + tool_ctx_kitchen_open.ci_watcher = watcher await wait_for_ci( branch="main", remote_url="https://github.com/owner/repo.git", repo="", # empty — remote_url must win cwd="/any/cwd", ) - assert mock_watcher.wait.call_args.kwargs.get("repo") == "owner/repo" + assert watcher.wait_calls[-1].get("repo") == "owner/repo" # --------------------------------------------------------------------------- @@ -392,12 +401,13 @@ async def test_wait_for_ci_remote_url_wins_over_empty_repo(tool_ctx): @pytest.mark.anyio -async def test_wait_for_merge_queue_parses_remote_url_to_resolve_repo(tool_ctx): +async def test_wait_for_merge_queue_parses_remote_url_to_resolve_repo(tool_ctx_kitchen_open): """When remote_url is provided, wait_for_merge_queue parses it to owner/repo without calling any subprocess.""" - mock_watcher = AsyncMock() - mock_watcher.wait.return_value = {"success": True, "pr_state": "merged", "pr_number": 42} - tool_ctx.merge_queue_watcher = mock_watcher + watcher = InMemoryMergeQueueWatcher( + wait_result={"success": True, "pr_state": "merged", "pr_number": 42} + ) + tool_ctx_kitchen_open.merge_queue_watcher = watcher result = json.loads( await wait_for_merge_queue( @@ -408,44 +418,40 @@ async def test_wait_for_merge_queue_parses_remote_url_to_resolve_repo(tool_ctx): ) ) assert result["pr_state"] == "merged" - call_kwargs = mock_watcher.wait.call_args - assert call_kwargs.kwargs.get("repo") == "owner/repo" + assert watcher.wait_calls[-1].get("repo") == "owner/repo" class TestWaitForCiTiming: """wait_for_ci records wall-clock timing when step_name is provided.""" @pytest.mark.anyio - async def test_wait_for_ci_step_name_records_timing(self, tool_ctx): - mock_watcher = AsyncMock() - mock_watcher.wait = AsyncMock( - return_value={"run_id": 1, "conclusion": "success", "failed_jobs": []} + async def test_wait_for_ci_step_name_records_timing(self, tool_ctx_kitchen_open): + watcher = InMemoryCIWatcher( + wait_result={"run_id": 1, "conclusion": "success", "failed_jobs": []} ) - tool_ctx.ci_watcher = mock_watcher + tool_ctx_kitchen_open.ci_watcher = watcher await wait_for_ci("main", step_name="ci_wait") - assert any(e["step_name"] == "ci_wait" for e in tool_ctx.timing_log.get_report()) + assert_step_timed(tool_ctx_kitchen_open.timing_log, "ci_wait") @pytest.mark.anyio async def test_wait_for_ci_empty_step_name_skips_timing(self, tool_ctx): - mock_watcher = AsyncMock() - mock_watcher.wait = AsyncMock( - return_value={"run_id": 1, "conclusion": "success", "failed_jobs": []} + watcher = InMemoryCIWatcher( + wait_result={"run_id": 1, "conclusion": "success", "failed_jobs": []} ) - tool_ctx.ci_watcher = mock_watcher + tool_ctx.ci_watcher = watcher await wait_for_ci("main") - assert tool_ctx.timing_log.get_report() == [] + assert_no_timing(tool_ctx.timing_log) class TestWaitForMergeQueueTiming: """wait_for_merge_queue records wall-clock timing when step_name is provided.""" @pytest.mark.anyio - async def test_wait_for_merge_queue_step_name_records_timing(self, tool_ctx): - mock_watcher = AsyncMock() - mock_watcher.wait = AsyncMock( - return_value={"success": True, "pr_state": "merged", "reason": "PR merged"} + async def test_wait_for_merge_queue_step_name_records_timing(self, tool_ctx_kitchen_open): + watcher = InMemoryMergeQueueWatcher( + wait_result={"success": True, "pr_state": "merged", "reason": "PR merged"} ) - tool_ctx.merge_queue_watcher = mock_watcher + tool_ctx_kitchen_open.merge_queue_watcher = watcher with patch( "autoskillit.execution.remote_resolver.asyncio.create_subprocess_exec", new_callable=AsyncMock, @@ -459,15 +465,14 @@ async def test_wait_for_merge_queue_step_name_records_timing(self, tool_ctx): await wait_for_merge_queue( pr_number=1, target_branch="main", cwd=".", step_name="mq_wait" ) - assert any(e["step_name"] == "mq_wait" for e in tool_ctx.timing_log.get_report()) + assert_step_timed(tool_ctx_kitchen_open.timing_log, "mq_wait") @pytest.mark.anyio async def test_wait_for_merge_queue_empty_step_name_skips_timing(self, tool_ctx): - mock_watcher = AsyncMock() - mock_watcher.wait = AsyncMock( - return_value={"success": True, "pr_state": "merged", "reason": "PR merged"} + watcher = InMemoryMergeQueueWatcher( + wait_result={"success": True, "pr_state": "merged", "reason": "PR merged"} ) - tool_ctx.merge_queue_watcher = mock_watcher + tool_ctx.merge_queue_watcher = watcher with patch( "autoskillit.execution.remote_resolver.asyncio.create_subprocess_exec", new_callable=AsyncMock, @@ -479,24 +484,25 @@ async def test_wait_for_merge_queue_empty_step_name_skips_timing(self, tool_ctx) proc_inst.returncode = 0 mock_proc.return_value = proc_inst await wait_for_merge_queue(pr_number=1, target_branch="main", cwd=".") - assert tool_ctx.timing_log.get_report() == [] + assert_no_timing(tool_ctx.timing_log) @pytest.mark.anyio async def test_wait_for_merge_queue_invalid_remote_url_falls_through_to_inference( - tool_ctx, tmp_path + tool_ctx_kitchen_open, tmp_path ): """ remote_url that parses to None (e.g. file://) does NOT short-circuit; inference continues via resolve_remote_repo(cwd). """ - mock_watcher = AsyncMock() - mock_watcher.wait.return_value = { - "success": False, - "pr_state": "error", - "reason": "Invalid repo format: None", - } - tool_ctx.merge_queue_watcher = mock_watcher + watcher = InMemoryMergeQueueWatcher( + wait_result={ + "success": False, + "pr_state": "error", + "reason": "Invalid repo format: None", + } + ) + tool_ctx_kitchen_open.merge_queue_watcher = watcher # provide a file:// remote_url — should fall through, eventually fail gracefully result = json.loads( @@ -509,7 +515,7 @@ async def test_wait_for_merge_queue_invalid_remote_url_falls_through_to_inferenc ) assert result["pr_state"] == "error" # The file:// URL must not resolve to a GitHub repo, so watcher receives repo=None - assert mock_watcher.wait.call_args.kwargs.get("repo") is None + assert watcher.wait_calls[-1].get("repo") is None # --------------------------------------------------------------------------- @@ -518,33 +524,29 @@ async def test_wait_for_merge_queue_invalid_remote_url_falls_through_to_inferenc @pytest.mark.anyio -async def test_wait_for_ci_handler_passes_workflow(tool_ctx): +async def test_wait_for_ci_handler_passes_workflow(tool_ctx_kitchen_open): """wait_for_ci MCP handler must forward workflow to watcher via scope.""" - mock_watcher = AsyncMock() - mock_watcher.wait = AsyncMock( - return_value={"conclusion": "success", "failed_jobs": [], "run_id": 1} + watcher = InMemoryCIWatcher( + wait_result={"conclusion": "success", "failed_jobs": [], "run_id": 1} ) - tool_ctx.ci_watcher = mock_watcher + tool_ctx_kitchen_open.ci_watcher = watcher # cwd="" → head_sha inference skipped (empty string is falsy) json.loads(await wait_for_ci(branch="main", workflow="tests.yml", cwd="")) - mock_watcher.wait.assert_called_once() - call_kwargs = mock_watcher.wait.call_args - assert call_kwargs.kwargs["scope"].workflow == "tests.yml" + assert len(watcher.wait_calls) == 1 + assert watcher.wait_calls[-1]["scope"].workflow == "tests.yml" @pytest.mark.anyio -async def test_get_ci_status_handler_passes_workflow(tool_ctx): +async def test_get_ci_status_handler_passes_workflow(tool_ctx_kitchen_open): """get_ci_status MCP handler must forward workflow to watcher via scope.""" - mock_watcher = AsyncMock() - mock_watcher.status = AsyncMock(return_value={"runs": []}) - tool_ctx.ci_watcher = mock_watcher + watcher = InMemoryCIWatcher(status_result={"runs": []}) + tool_ctx_kitchen_open.ci_watcher = watcher await get_ci_status(branch="main", workflow="tests.yml") - call_kwargs = mock_watcher.status.call_args - assert call_kwargs.kwargs["scope"].workflow == "tests.yml" + assert watcher.status_calls[-1]["scope"].workflow == "tests.yml" # --------------------------------------------------------------------------- @@ -553,15 +555,15 @@ async def test_get_ci_status_handler_passes_workflow(tool_ctx): @pytest.mark.anyio -async def test_wait_for_ci_watcher_exception_returns_structured_json(tool_ctx): - """wait_for_ci returns {success: false, error: ...} when watcher.wait() raises. +async def test_wait_for_ci_watcher_exception_returns_structured_json(tool_ctx_kitchen_open): + """wait_for_ci returns structured JSON with conclusion='error' when watcher.wait() raises. BEFORE fix: bare raise propagates to track_response_size which adds subtype='tool_exception'. AFTER fix: explicit return gives clean JSON. """ - mock_watcher = AsyncMock() - mock_watcher.wait = AsyncMock(side_effect=RuntimeError("network timeout")) - tool_ctx.ci_watcher = mock_watcher + watcher = InMemoryCIWatcher() + watcher.wait_side_effect = RuntimeError("network timeout") + tool_ctx_kitchen_open.ci_watcher = watcher with patch( "autoskillit.execution.remote_resolver.asyncio.create_subprocess_exec", @@ -574,22 +576,24 @@ async def test_wait_for_ci_watcher_exception_returns_structured_json(tool_ctx): result = json.loads(await wait_for_ci("main", cwd="/some/repo")) - assert result["success"] is False + assert result["conclusion"] == "error" assert "network timeout" in result["error"] assert "subtype" not in result # no decorator fallback marker assert "exit_code" not in result # no decorator fallback marker @pytest.mark.anyio -async def test_wait_for_merge_queue_watcher_exception_returns_structured_json(tool_ctx): +async def test_wait_for_merge_queue_watcher_exception_returns_structured_json( + tool_ctx_kitchen_open, +): """wait_for_merge_queue returns {success: false, error: ...} when watcher.wait() raises. BEFORE fix: bare raise propagates to track_response_size decorator. AFTER fix: explicit return gives clean JSON. """ - mock_watcher = AsyncMock() - mock_watcher.wait = AsyncMock(side_effect=RuntimeError("connection refused")) - tool_ctx.merge_queue_watcher = mock_watcher + watcher = InMemoryMergeQueueWatcher() + watcher.wait_side_effect = RuntimeError("connection refused") + tool_ctx_kitchen_open.merge_queue_watcher = watcher with patch( "autoskillit.execution.remote_resolver.asyncio.create_subprocess_exec", @@ -615,14 +619,13 @@ async def test_wait_for_merge_queue_watcher_exception_returns_structured_json(to @pytest.mark.anyio -async def test_wait_for_ci_includes_head_sha_in_result(tool_ctx): +async def test_wait_for_ci_includes_head_sha_in_result(tool_ctx_kitchen_open): """wait_for_ci result includes head_sha when git rev-parse HEAD succeeds.""" - mock_watcher = AsyncMock() - mock_watcher.wait = AsyncMock( - return_value={"run_id": 1, "conclusion": "success", "failed_jobs": []} + watcher = InMemoryCIWatcher( + wait_result={"run_id": 1, "conclusion": "success", "failed_jobs": []} ) - tool_ctx.ci_watcher = mock_watcher - tool_ctx.runner.push( + tool_ctx_kitchen_open.ci_watcher = watcher + tool_ctx_kitchen_open.runner.push( SubprocessResult( returncode=0, stdout="deadbeef1234\n", @@ -638,14 +641,13 @@ async def test_wait_for_ci_includes_head_sha_in_result(tool_ctx): @pytest.mark.anyio -async def test_wait_for_ci_omits_head_sha_when_git_fails(tool_ctx): +async def test_wait_for_ci_omits_head_sha_when_git_fails(tool_ctx_kitchen_open): """wait_for_ci result omits head_sha when git rev-parse fails.""" - mock_watcher = AsyncMock() - mock_watcher.wait = AsyncMock( - return_value={"run_id": 1, "conclusion": "success", "failed_jobs": []} + watcher = InMemoryCIWatcher( + wait_result={"run_id": 1, "conclusion": "success", "failed_jobs": []} ) - tool_ctx.ci_watcher = mock_watcher - tool_ctx.runner.push( + tool_ctx_kitchen_open.ci_watcher = watcher + tool_ctx_kitchen_open.runner.push( SubprocessResult( returncode=128, stdout="", @@ -668,20 +670,19 @@ async def test_wait_for_ci_omits_head_sha_when_git_fails(tool_ctx): @pytest.mark.anyio @pytest.mark.parametrize("pr_state", list(PRState)) -async def test_wait_for_merge_queue_serializes_every_pr_state(pr_state, tool_ctx): +async def test_wait_for_merge_queue_serializes_every_pr_state(pr_state, tool_ctx_kitchen_open): """Every PRState value round-trips faithfully through the MCP handler. Adding a new PRState member without a handler test fails this parametrized suite. """ - mock_watcher = AsyncMock() - mock_watcher.wait = AsyncMock( - return_value={ + watcher = InMemoryMergeQueueWatcher( + wait_result={ "success": pr_state == PRState.MERGED, "pr_state": pr_state.value, "reason": f"test reason for {pr_state.value}", } ) - tool_ctx.merge_queue_watcher = mock_watcher + tool_ctx_kitchen_open.merge_queue_watcher = watcher with patch( "autoskillit.execution.remote_resolver.asyncio.create_subprocess_exec", @@ -720,90 +721,20 @@ def test_pr_state_docstring_documents_all_members(): # --------------------------------------------------------------------------- -# check_repo_merge_state: token_factory and http_status +# C9-2: exception path shape — conclusion key required for recipe on_result routing # --------------------------------------------------------------------------- @pytest.mark.anyio -async def test_check_repo_merge_state_uses_token_factory(tool_ctx, monkeypatch): - """check_repo_merge_state calls token_factory() when set, not config.github.token.""" - resolved_calls = [] - - def factory(): - resolved_calls.append(1) - return "factory-token" - - tool_ctx.token_factory = factory - tool_ctx.config.github.token = "config-token" - - captured_tokens = [] - - async def fake_fetch(owner, repo, branch, token): - captured_tokens.append(token) - return { - "queue_available": False, - "merge_group_trigger": False, - "auto_merge_available": False, - "ci_event": None, - } - - monkeypatch.setattr("autoskillit.server.tools_ci.fetch_repo_merge_state", fake_fetch) - monkeypatch.setattr( - "autoskillit.server.tools_ci.infer_repo_from_remote", - AsyncMock(return_value="owner/repo"), - ) - - await check_repo_merge_state(branch="main") - assert captured_tokens == ["factory-token"] - assert resolved_calls == [1] - - -@pytest.mark.anyio -async def test_check_repo_merge_state_falls_back_to_config_token_when_no_factory( - tool_ctx, monkeypatch -): - """When token_factory is None, config.github.token is used.""" - tool_ctx.token_factory = None - tool_ctx.config.github.token = "config-token" - - captured_tokens = [] - - async def fake_fetch(owner, repo, branch, token): - captured_tokens.append(token) - return { - "queue_available": False, - "merge_group_trigger": False, - "auto_merge_available": False, - "ci_event": None, - } - - monkeypatch.setattr("autoskillit.server.tools_ci.fetch_repo_merge_state", fake_fetch) - monkeypatch.setattr( - "autoskillit.server.tools_ci.infer_repo_from_remote", - AsyncMock(return_value="owner/repo"), - ) - - await check_repo_merge_state(branch="main") - assert captured_tokens == ["config-token"] - - -@pytest.mark.anyio -async def test_check_repo_merge_state_error_includes_http_status(tool_ctx, monkeypatch): - """HTTP error response envelope contains http_status field.""" +async def test_wait_for_ci_exception_returns_conclusion_key(tool_ctx_kitchen_open, monkeypatch): + """Inner exception path must return conclusion='error' for recipe on_result routing.""" - async def fake_fetch(owner, repo, branch, token): - response = httpx.Response( - 403, - request=httpx.Request("POST", "https://api.github.com/graphql"), - ) - raise httpx.HTTPStatusError("403 Forbidden", request=response.request, response=response) - - monkeypatch.setattr("autoskillit.server.tools_ci.fetch_repo_merge_state", fake_fetch) - monkeypatch.setattr( - "autoskillit.server.tools_ci.infer_repo_from_remote", - AsyncMock(return_value="owner/repo"), - ) + async def _exploding_wait(*a, **kw): + raise RuntimeError("boom") - result = json.loads(await check_repo_merge_state(branch="main")) - assert "http_status" in result - assert result["http_status"] == 403 + monkeypatch.setattr(tool_ctx_kitchen_open.ci_watcher, "wait", _exploding_wait) + raw = await wait_for_ci(branch="main", cwd="/tmp") + result = json.loads(raw) + assert "conclusion" in result, "Exception path must include conclusion key" + assert result["conclusion"] == "error" + assert "error" in result diff --git a/tests/server/test_tools_ci_enqueue.py b/tests/server/test_tools_ci_enqueue.py new file mode 100644 index 0000000000..f4bf148630 --- /dev/null +++ b/tests/server/test_tools_ci_enqueue.py @@ -0,0 +1,63 @@ +"""Tests for enqueue_pr MCP tool handler.""" + +from __future__ import annotations + +import json +from unittest.mock import AsyncMock, patch + +import pytest + +from tests.fakes import InMemoryMergeQueueWatcher + +pytestmark = [pytest.mark.layer("server"), pytest.mark.small] + + +@pytest.mark.anyio +async def test_enqueue_pr_delegates_to_watcher_enqueue(tool_ctx_kitchen_open): + """enqueue_pr tool passes auto_merge_available to watcher.enqueue().""" + watcher = InMemoryMergeQueueWatcher( + enqueue_result={ + "success": True, + "pr_number": 42, + "enrollment_method": "direct_enqueue", + }, + ) + tool_ctx_kitchen_open.merge_queue_watcher = watcher + + with patch( + "autoskillit.server.tools.tools_ci_merge_queue.resolve_repo_from_remote", + new_callable=AsyncMock, + return_value="owner/repo", + ): + from autoskillit.server.tools.tools_ci_merge_queue import enqueue_pr + + raw = await enqueue_pr( + pr_number=42, + target_branch="main", + cwd="/tmp/work", + auto_merge_available=False, + ) + result = json.loads(raw) + assert result["success"] is True + assert result["enrollment_method"] == "direct_enqueue" + assert len(watcher.enqueue_calls) == 1 + call = watcher.enqueue_calls[0] + assert call["auto_merge_available"] is False + + +@pytest.mark.anyio +async def test_enqueue_pr_returns_structured_error_when_watcher_none(tool_ctx_kitchen_open): + """enqueue_pr returns {"success": false} when merge_queue_watcher is None.""" + tool_ctx_kitchen_open.merge_queue_watcher = None + + from autoskillit.server.tools.tools_ci_merge_queue import enqueue_pr + + raw = await enqueue_pr( + pr_number=42, + target_branch="main", + cwd="/tmp/work", + auto_merge_available=True, + ) + result = json.loads(raw) + assert result["success"] is False + assert "error" in result diff --git a/tests/server/test_tools_ci_merge_state.py b/tests/server/test_tools_ci_merge_state.py new file mode 100644 index 0000000000..a8e1c9fb7e --- /dev/null +++ b/tests/server/test_tools_ci_merge_state.py @@ -0,0 +1,104 @@ +"""Tests for check_repo_merge_state MCP tool handler.""" + +from __future__ import annotations + +import json +from unittest.mock import AsyncMock + +import httpx +import pytest + +from autoskillit.server.tools.tools_ci import check_repo_merge_state + +pytestmark = [pytest.mark.layer("server"), pytest.mark.small] + + +# --------------------------------------------------------------------------- +# check_repo_merge_state: token_factory and http_status +# --------------------------------------------------------------------------- + + +@pytest.mark.anyio +async def test_check_repo_merge_state_uses_token_factory(tool_ctx_kitchen_open, monkeypatch): + """check_repo_merge_state calls token_factory() when set, not config.github.token.""" + resolved_calls = [] + + def factory(): + resolved_calls.append(1) + return "factory-token" + + tool_ctx_kitchen_open.token_factory = factory + tool_ctx_kitchen_open.config.github.token = "config-token" + + captured_tokens = [] + + async def fake_fetch(owner, repo, branch, token): + captured_tokens.append(token) + return { + "queue_available": False, + "merge_group_trigger": False, + "auto_merge_available": False, + "ci_event": None, + } + + monkeypatch.setattr("autoskillit.server.tools.tools_ci.fetch_repo_merge_state", fake_fetch) + monkeypatch.setattr( + "autoskillit.server.tools.tools_ci.resolve_repo_from_remote", + AsyncMock(return_value="owner/repo"), + ) + + await check_repo_merge_state(branch="main") + assert captured_tokens == ["factory-token"] + assert resolved_calls == [1] + + +@pytest.mark.anyio +async def test_check_repo_merge_state_falls_back_to_config_token_when_no_factory( + tool_ctx_kitchen_open, monkeypatch +): + """When token_factory is None, config.github.token is used.""" + tool_ctx_kitchen_open.token_factory = None + tool_ctx_kitchen_open.config.github.token = "config-token" + + captured_tokens = [] + + async def fake_fetch(owner, repo, branch, token): + captured_tokens.append(token) + return { + "queue_available": False, + "merge_group_trigger": False, + "auto_merge_available": False, + "ci_event": None, + } + + monkeypatch.setattr("autoskillit.server.tools.tools_ci.fetch_repo_merge_state", fake_fetch) + monkeypatch.setattr( + "autoskillit.server.tools.tools_ci.resolve_repo_from_remote", + AsyncMock(return_value="owner/repo"), + ) + + await check_repo_merge_state(branch="main") + assert captured_tokens == ["config-token"] + + +@pytest.mark.anyio +async def test_check_repo_merge_state_error_includes_http_status( + tool_ctx_kitchen_open, monkeypatch +): + """HTTP error response envelope contains http_status field.""" + + async def fake_fetch(owner, repo, branch, token): + response = httpx.Response( + 403, + request=httpx.Request("POST", "https://api.github.com/graphql"), + ) + raise httpx.HTTPStatusError("403 Forbidden", request=response.request, response=response) + + monkeypatch.setattr("autoskillit.server.tools.tools_ci.fetch_repo_merge_state", fake_fetch) + monkeypatch.setattr( + "autoskillit.server.tools.tools_ci.resolve_repo_from_remote", + AsyncMock(return_value="owner/repo"), + ) + + result = json.loads(await check_repo_merge_state(branch="main")) + assert result["http_status"] == 403 diff --git a/tests/server/test_tools_ci_split.py b/tests/server/test_tools_ci_split.py new file mode 100644 index 0000000000..144f2f07dd --- /dev/null +++ b/tests/server/test_tools_ci_split.py @@ -0,0 +1,23 @@ +import pytest + +pytestmark = [pytest.mark.layer("server"), pytest.mark.small] + + +def test_ci_watch_tools_importable(): + from autoskillit.server.tools.tools_ci_watch import wait_for_ci + + assert callable(wait_for_ci) + + +def test_ci_merge_queue_tools_importable(): + from autoskillit.server.tools.tools_ci_merge_queue import ( + toggle_auto_merge, + ) + + assert callable(toggle_auto_merge) + + +def test_ci_status_tools_in_reduced_file(): + from autoskillit.server.tools.tools_ci import set_commit_status + + assert callable(set_commit_status) diff --git a/tests/server/test_tools_ci_watch.py b/tests/server/test_tools_ci_watch.py new file mode 100644 index 0000000000..1aa680fd18 --- /dev/null +++ b/tests/server/test_tools_ci_watch.py @@ -0,0 +1,218 @@ +"""Tests for wait_for_ci event validation, null coercion, and auto_trigger recovery.""" + +from __future__ import annotations + +import json + +import pytest + +from autoskillit.core import SubprocessResult, TerminationReason +from autoskillit.server.tools.tools_ci_watch import wait_for_ci +from tests.fakes import InMemoryCIWatcher + +pytestmark = [pytest.mark.layer("server"), pytest.mark.small] + + +@pytest.mark.anyio +async def test_wait_for_ci_rejects_invalid_event(tool_ctx_kitchen_open) -> None: + tool_ctx_kitchen_open.ci_watcher = InMemoryCIWatcher() + result = json.loads(await wait_for_ci("main", event="made_up_event")) + assert result["conclusion"] == "error" + assert "event" in result.get("error", "").lower() + + +@pytest.mark.anyio +async def test_wait_for_ci_coerces_string_none_to_null(tool_ctx_kitchen_open) -> None: + tool_ctx_kitchen_open.ci_watcher = InMemoryCIWatcher( + wait_result={"run_id": 42, "conclusion": "success", "failed_jobs": []} + ) + result = json.loads(await wait_for_ci("main", event="None")) + assert result["conclusion"] == "success", ( + f"String 'None' must be coerced to null before event validation — " + f"KNOWN_CI_EVENTS would reject 'None' as invalid if coercion did not occur. " + f"Got: {result}" + ) + + +# --------------------------------------------------------------------------- +# wait_for_ci auto_trigger +# --------------------------------------------------------------------------- + +_NO_RUNS = {"conclusion": "no_runs", "run_id": None, "failed_jobs": []} +_SUCCESS = {"conclusion": "success", "run_id": 42, "failed_jobs": []} + + +def _sub(returncode: int, stdout: str = "", stderr: str = "") -> SubprocessResult: + return SubprocessResult( + returncode=returncode, + stdout=stdout, + stderr=stderr, + termination=TerminationReason.NATURAL_EXIT, + pid=0, + ) + + +class TestWaitForCiAutoTrigger: + """wait_for_ci auto_trigger=True: active webhook recovery.""" + + @pytest.mark.anyio + async def test_auto_trigger_default_false_does_not_fire(self, tool_ctx_kitchen_open): + watcher = InMemoryCIWatcher(wait_result=_NO_RUNS) + tool_ctx_kitchen_open.ci_watcher = watcher + tool_ctx_kitchen_open.runner.push(_sub(0, "abc123\n")) + + result = json.loads(await wait_for_ci("branch", cwd="/repo")) + + assert len(watcher.wait_calls) == 1 + assert len(tool_ctx_kitchen_open.runner.call_args_list) == 1 + assert result["conclusion"] == "no_runs" + assert "triggered" not in result + + @pytest.mark.anyio + async def test_auto_trigger_fires_on_no_runs(self, tool_ctx_kitchen_open): + watcher = InMemoryCIWatcher(wait_results=[_NO_RUNS, _SUCCESS]) + tool_ctx_kitchen_open.ci_watcher = watcher + tool_ctx_kitchen_open.runner.push( + _sub(0, "abc123\n") + ) # git rev-parse HEAD — wait_for_ci initial HEAD inference + tool_ctx_kitchen_open.runner.push(_sub(0, '{"mergeable":"MERGEABLE"}\n')) # gh pr view + tool_ctx_kitchen_open.runner.push(_sub(0)) # git commit --allow-empty + tool_ctx_kitchen_open.runner.push( + _sub(0, "def456\n") + ) # git rev-parse HEAD — new HEAD after empty commit + tool_ctx_kitchen_open.runner.push( + _sub(0, "https://github.com/org/repo\n") + ) # git remote get-url upstream + tool_ctx_kitchen_open.runner.push(_sub(0)) # git push --force-with-lease + + result = json.loads(await wait_for_ci("feature-branch", cwd="/repo", auto_trigger=True)) + + assert len(watcher.wait_calls) == 2 + assert watcher.wait_calls[1]["scope"].head_sha == "def456" + assert result["conclusion"] == "success" + assert result["triggered"] is True + assert result["head_sha"] == "def456" + + @pytest.mark.anyio + async def test_auto_trigger_skips_on_conflicting_pr(self, tool_ctx_kitchen_open): + watcher = InMemoryCIWatcher(wait_result=_NO_RUNS) + tool_ctx_kitchen_open.ci_watcher = watcher + tool_ctx_kitchen_open.runner.push(_sub(0, "abc123\n")) # git rev-parse HEAD + tool_ctx_kitchen_open.runner.push(_sub(0, '{"mergeable":"CONFLICTING"}\n')) # gh pr view + + result = json.loads(await wait_for_ci("branch", cwd="/repo", auto_trigger=True)) + + assert len(watcher.wait_calls) == 1 + assert result["conclusion"] == "merge_conflict" + assert result["triggered"] is False + assert result["run_id"] is None + assert result["failed_jobs"] == [] + + @pytest.mark.anyio + async def test_auto_trigger_returns_gh_view_failed_on_gh_failure(self, tool_ctx_kitchen_open): + watcher = InMemoryCIWatcher(wait_result=_NO_RUNS) + tool_ctx_kitchen_open.ci_watcher = watcher + tool_ctx_kitchen_open.runner.push(_sub(0, "abc123\n")) # git rev-parse HEAD (initial) + tool_ctx_kitchen_open.runner.push(_sub(1)) # gh pr view — CLI failure (no PR) + + result = json.loads(await wait_for_ci("branch", cwd="/repo", auto_trigger=True)) + + assert len(watcher.wait_calls) == 1 + assert result["conclusion"] == "gh_view_failed" + assert result["triggered"] is False + + @pytest.mark.anyio + async def test_auto_trigger_commit_failure_returns_no_runs(self, tool_ctx_kitchen_open): + watcher = InMemoryCIWatcher(wait_result=_NO_RUNS) + tool_ctx_kitchen_open.ci_watcher = watcher + tool_ctx_kitchen_open.runner.push(_sub(0, "abc123\n")) # git rev-parse HEAD + tool_ctx_kitchen_open.runner.push(_sub(0, '{"mergeable":"MERGEABLE"}\n')) # gh pr view + tool_ctx_kitchen_open.runner.push( + _sub(128, stderr="error: pre-commit hook rejected commit") + ) # git commit fails + + result = json.loads(await wait_for_ci("branch", cwd="/repo", auto_trigger=True)) + + assert len(watcher.wait_calls) == 1 + assert result["conclusion"] == "no_runs" + assert "triggered" not in result + + @pytest.mark.anyio + async def test_auto_trigger_push_failure_returns_no_runs(self, tool_ctx_kitchen_open): + watcher = InMemoryCIWatcher(wait_result=_NO_RUNS) + tool_ctx_kitchen_open.ci_watcher = watcher + tool_ctx_kitchen_open.runner.push(_sub(0, "abc123\n")) # git rev-parse HEAD + tool_ctx_kitchen_open.runner.push(_sub(0, '{"mergeable":"MERGEABLE"}\n')) # gh pr view + tool_ctx_kitchen_open.runner.push(_sub(0)) # git commit --allow-empty + tool_ctx_kitchen_open.runner.push(_sub(0, "def456\n")) # git rev-parse HEAD (new) + tool_ctx_kitchen_open.runner.push( + _sub(0, "https://github.com/org/repo\n") + ) # git remote get-url upstream + tool_ctx_kitchen_open.runner.push( + _sub(1, stderr="error: remote rejected") + ) # git push fails + tool_ctx_kitchen_open.runner.push(_sub(0)) # git reset --soft HEAD~1 (cleanup) + + result = json.loads(await wait_for_ci("branch", cwd="/repo", auto_trigger=True)) + + reset_cmd = tool_ctx_kitchen_open.runner.call_args_list[-1][0] + assert reset_cmd == ["git", "reset", "--soft", "HEAD~1"], ( + f"Expected reset, got: {reset_cmd}" + ) + assert len(watcher.wait_calls) == 1 + assert result["conclusion"] == "no_runs" + assert "triggered" not in result + + @pytest.mark.anyio + async def test_auto_trigger_ci_poll_exception_returns_auto_trigger_failed( + self, tool_ctx_kitchen_open + ): + call_count_holder = [0] + + def poll_raises_on_second_call() -> dict: + call_count_holder[0] += 1 + if call_count_holder[0] >= 2: + raise RuntimeError("auto_trigger poll failed") + return _NO_RUNS + + watcher = InMemoryCIWatcher(wait_result=_NO_RUNS) + watcher.wait_side_effect = poll_raises_on_second_call + tool_ctx_kitchen_open.ci_watcher = watcher + tool_ctx_kitchen_open.runner.push( + _sub(0, "abc123\n") + ) # git rev-parse HEAD — wait_for_ci initial HEAD inference + tool_ctx_kitchen_open.runner.push(_sub(0, '{"mergeable":"MERGEABLE"}\n')) # gh pr view + tool_ctx_kitchen_open.runner.push(_sub(0)) # git commit --allow-empty + tool_ctx_kitchen_open.runner.push( + _sub(0, "def456\n") + ) # git rev-parse HEAD — new HEAD after empty commit + tool_ctx_kitchen_open.runner.push( + _sub(0, "https://github.com/org/repo\n") + ) # git remote get-url upstream + tool_ctx_kitchen_open.runner.push(_sub(0)) # git push --force-with-lease + + result = json.loads(await wait_for_ci("branch", cwd="/repo", auto_trigger=True)) + + assert len(watcher.wait_calls) == 2 + assert result["conclusion"] == "auto_trigger_failed" + assert result["triggered"] is False + + @pytest.mark.anyio + async def test_auto_trigger_ci_forwards_lookback_seconds(self, tool_ctx_kitchen_open): + """The second wait() call in _auto_trigger_ci must forward lookback_seconds.""" + watcher = InMemoryCIWatcher(wait_results=[_NO_RUNS, _SUCCESS]) + tool_ctx_kitchen_open.ci_watcher = watcher + tool_ctx_kitchen_open.runner.push(_sub(0, "abc123\n")) # git rev-parse HEAD + tool_ctx_kitchen_open.runner.push(_sub(0, '{"mergeable":"MERGEABLE"}\n')) # gh pr view + tool_ctx_kitchen_open.runner.push(_sub(0)) # git commit --allow-empty + tool_ctx_kitchen_open.runner.push(_sub(0, "def456\n")) # git rev-parse HEAD — new HEAD + tool_ctx_kitchen_open.runner.push( + _sub(0, "https://github.com/org/repo\n") + ) # git remote get-url upstream + tool_ctx_kitchen_open.runner.push(_sub(0)) # git push --force-with-lease + + await wait_for_ci("branch", cwd="/repo", auto_trigger=True, lookback_seconds=7200) + + assert len(watcher.wait_calls) == 2 + assert watcher.wait_calls[0]["lookback_seconds"] == 7200 + assert watcher.wait_calls[1]["lookback_seconds"] == 7200 diff --git a/tests/server/test_tools_clone.py b/tests/server/test_tools_clone.py index 7ae9f30984..da3723d065 100644 --- a/tests/server/test_tools_clone.py +++ b/tests/server/test_tools_clone.py @@ -8,7 +8,7 @@ import pytest -from autoskillit.server.tools_clone import ( +from autoskillit.server.tools.tools_clone import ( batch_cleanup_clones, clone_repo, push_to_remote, @@ -17,6 +17,8 @@ ) from autoskillit.workspace import clone_registry +pytestmark = [pytest.mark.layer("server"), pytest.mark.small] + class TestCloneRepoTool: @pytest.mark.anyio @@ -26,51 +28,51 @@ async def test_returns_gate_error_when_disabled(self, tool_ctx): assert result["subtype"] == "gate_error" @pytest.mark.anyio - async def test_delegates_to_workspace_clone(self, tool_ctx): + async def test_delegates_to_workspace_clone(self, tool_ctx_kitchen_open): mock_mgr = MagicMock() mock_mgr.clone_repo.return_value = {"clone_path": "/clone/path", "source_dir": "/src"} - tool_ctx.clone_mgr = mock_mgr + tool_ctx_kitchen_open.clone_mgr = mock_mgr result = json.loads(await clone_repo(source_dir="/src", run_name="myrun")) assert result["clone_path"] == "/clone/path" assert result["source_dir"] == "/src" @pytest.mark.anyio - async def test_returns_error_on_value_error(self, tool_ctx): + async def test_returns_error_on_value_error(self, tool_ctx_kitchen_open): mock_mgr = MagicMock() mock_mgr.clone_repo.side_effect = ValueError("resolved to nonexistent") - tool_ctx.clone_mgr = mock_mgr + tool_ctx_kitchen_open.clone_mgr = mock_mgr result = json.loads(await clone_repo(source_dir="/bad/path", run_name="run")) assert "error" in result assert "resolved to" in result["error"] @pytest.mark.anyio - async def test_returns_error_on_runtime_error(self, tool_ctx): + async def test_returns_error_on_runtime_error(self, tool_ctx_kitchen_open): mock_mgr = MagicMock() mock_mgr.clone_repo.side_effect = RuntimeError("git clone failed") - tool_ctx.clone_mgr = mock_mgr + tool_ctx_kitchen_open.clone_mgr = mock_mgr result = json.loads(await clone_repo(source_dir="/src", run_name="run")) assert "error" in result @pytest.mark.anyio - async def test_cb17_forwards_branch_to_clone_manager(self, tool_ctx): + async def test_cb17_forwards_branch_to_clone_manager(self, tool_ctx_kitchen_open): """T_CB17: branch param is forwarded to the underlying clone_repo call.""" mock_mgr = MagicMock() mock_mgr.clone_repo.return_value = {"clone_path": "/clone/path", "source_dir": "/src"} - tool_ctx.clone_mgr = mock_mgr + tool_ctx_kitchen_open.clone_mgr = mock_mgr await clone_repo(source_dir="/src", run_name="r", branch="dev") mock_mgr.clone_repo.assert_called_once_with("/src", "r", "dev", "", "") @pytest.mark.anyio - async def test_cb18_forwards_strategy_to_clone_manager(self, tool_ctx): + async def test_cb18_forwards_strategy_to_clone_manager(self, tool_ctx_kitchen_open): """T_CB18: strategy param is forwarded to the underlying clone_repo call.""" mock_mgr = MagicMock() mock_mgr.clone_repo.return_value = {"clone_path": "/clone/path", "source_dir": "/src"} - tool_ctx.clone_mgr = mock_mgr + tool_ctx_kitchen_open.clone_mgr = mock_mgr await clone_repo(source_dir="/src", run_name="r", strategy="proceed") mock_mgr.clone_repo.assert_called_once_with("/src", "r", "", "proceed", "") @pytest.mark.anyio - async def test_ru3_forwards_remote_url_to_clone_manager(self, tool_ctx): + async def test_ru3_forwards_remote_url_to_clone_manager(self, tool_ctx_kitchen_open): """T_RU3: remote_url param is forwarded to the underlying clone_repo call.""" mock_mgr = MagicMock() mock_mgr.clone_repo.return_value = { @@ -78,7 +80,7 @@ async def test_ru3_forwards_remote_url_to_clone_manager(self, tool_ctx): "source_dir": "/src", "remote_url": "https://github.com/example/repo.git", } - tool_ctx.clone_mgr = mock_mgr + tool_ctx_kitchen_open.clone_mgr = mock_mgr result = json.loads( await clone_repo( source_dir="/src", @@ -92,7 +94,7 @@ async def test_ru3_forwards_remote_url_to_clone_manager(self, tool_ctx): assert result["remote_url"] == "https://github.com/example/repo.git" @pytest.mark.anyio - async def test_cb19_returns_uncommitted_changes_result_as_json(self, tool_ctx): + async def test_cb19_returns_uncommitted_changes_result_as_json(self, tool_ctx_kitchen_open): """T_CB19: uncommitted_changes warning dict passes through without 'error' key.""" uncommitted_result = { "uncommitted_changes": "true", @@ -103,7 +105,7 @@ async def test_cb19_returns_uncommitted_changes_result_as_json(self, tool_ctx): } mock_mgr = MagicMock() mock_mgr.clone_repo.return_value = uncommitted_result - tool_ctx.clone_mgr = mock_mgr + tool_ctx_kitchen_open.clone_mgr = mock_mgr result = json.loads(await clone_repo(source_dir="/src", run_name="r")) assert result["uncommitted_changes"] == "true" assert "error" not in result @@ -117,18 +119,18 @@ async def test_returns_gate_error_when_disabled(self, tool_ctx): assert result["subtype"] == "gate_error" @pytest.mark.anyio - async def test_delegates_to_workspace_clone(self, tool_ctx): + async def test_delegates_to_workspace_clone(self, tool_ctx_kitchen_open): mock_mgr = MagicMock() mock_mgr.remove_clone.return_value = {"removed": "true"} - tool_ctx.clone_mgr = mock_mgr + tool_ctx_kitchen_open.clone_mgr = mock_mgr result = json.loads(await remove_clone(clone_path="/clone/path", keep="false")) assert result["removed"] == "true" @pytest.mark.anyio - async def test_keep_true_passes_through(self, tool_ctx): + async def test_keep_true_passes_through(self, tool_ctx_kitchen_open): mock_mgr = MagicMock() mock_mgr.remove_clone.return_value = {"removed": "false", "reason": "keep=true"} - tool_ctx.clone_mgr = mock_mgr + tool_ctx_kitchen_open.clone_mgr = mock_mgr result = json.loads(await remove_clone(clone_path="/clone/path", keep="true")) assert result["removed"] == "false" @@ -149,10 +151,10 @@ async def test_returns_gate_error_when_disabled(self, tool_ctx): assert result["subtype"] == "gate_error" @pytest.mark.anyio - async def test_delegates_to_workspace_clone_on_success(self, tool_ctx): + async def test_delegates_to_workspace_clone_on_success(self, tool_ctx_kitchen_open): mock_mgr = MagicMock() mock_mgr.push_to_remote.return_value = {"success": "true", "stderr": ""} - tool_ctx.clone_mgr = mock_mgr + tool_ctx_kitchen_open.clone_mgr = mock_mgr result = json.loads( await push_to_remote(clone_path="/clone", source_dir="/src", branch="main") ) @@ -160,10 +162,10 @@ async def test_delegates_to_workspace_clone_on_success(self, tool_ctx): assert "error" not in result @pytest.mark.anyio - async def test_returns_error_key_when_push_fails(self, tool_ctx): + async def test_returns_error_key_when_push_fails(self, tool_ctx_kitchen_open): mock_mgr = MagicMock() mock_mgr.push_to_remote.return_value = {"success": False, "stderr": "remote rejected"} - tool_ctx.clone_mgr = mock_mgr + tool_ctx_kitchen_open.clone_mgr = mock_mgr result = json.loads( await push_to_remote(clone_path="/clone", source_dir="/src", branch="main") ) @@ -171,7 +173,9 @@ async def test_returns_error_key_when_push_fails(self, tool_ctx): assert "remote rejected" in result["stderr"] @pytest.mark.anyio - async def test_push_to_remote_failure_response_includes_success_false(self, tool_ctx): + async def test_push_to_remote_failure_response_includes_success_false( + self, tool_ctx_kitchen_open + ): """REQ-C9-01: failure payload must include success=False for on_failure routing.""" mock_mgr = MagicMock() mock_mgr.push_to_remote.return_value = { @@ -179,7 +183,7 @@ async def test_push_to_remote_failure_response_includes_success_false(self, tool "stderr": "remote rejected", "error_type": "push_rejected", } - tool_ctx.clone_mgr = mock_mgr + tool_ctx_kitchen_open.clone_mgr = mock_mgr result = json.loads( await push_to_remote(clone_path="/clone", source_dir="/src", branch="main") ) @@ -187,21 +191,23 @@ async def test_push_to_remote_failure_response_includes_success_false(self, tool assert "error" in result @pytest.mark.anyio - async def test_push_to_remote_mcp_handler_passes_force_true_to_clone_mgr(self, tool_ctx): + async def test_push_to_remote_mcp_handler_passes_force_true_to_clone_mgr( + self, tool_ctx_kitchen_open + ): """T3: push_to_remote MCP handler converts force='true' to bool True for clone_mgr.""" mock_mgr = MagicMock() mock_mgr.push_to_remote.return_value = {"success": True, "stderr": ""} - tool_ctx.clone_mgr = mock_mgr + tool_ctx_kitchen_open.clone_mgr = mock_mgr await push_to_remote(clone_path="/clone", source_dir="/src", branch="main", force="true") _args, kwargs = mock_mgr.push_to_remote.call_args assert kwargs.get("force") is True @pytest.mark.anyio - async def test_push_to_remote_mcp_handler_default_force_false(self, tool_ctx): + async def test_push_to_remote_mcp_handler_default_force_false(self, tool_ctx_kitchen_open): """T4: push_to_remote MCP handler defaults to force=False when not supplied.""" mock_mgr = MagicMock() mock_mgr.push_to_remote.return_value = {"success": True, "stderr": ""} - tool_ctx.clone_mgr = mock_mgr + tool_ctx_kitchen_open.clone_mgr = mock_mgr await push_to_remote(clone_path="/clone", source_dir="/src", branch="main") _args, kwargs = mock_mgr.push_to_remote.call_args assert kwargs.get("force") is False @@ -211,12 +217,12 @@ class TestCloneRepoTiming: """clone_repo records wall-clock timing when step_name is provided.""" @pytest.mark.anyio - async def test_clone_repo_step_name_records_timing(self, tool_ctx): + async def test_clone_repo_step_name_records_timing(self, tool_ctx_kitchen_open): mock_mgr = MagicMock() mock_mgr.clone_repo.return_value = {"clone_path": "/clone", "source_dir": "/src"} - tool_ctx.clone_mgr = mock_mgr + tool_ctx_kitchen_open.clone_mgr = mock_mgr await clone_repo(source_dir="/src", run_name="test", step_name="clone") - report = tool_ctx.timing_log.get_report() + report = tool_ctx_kitchen_open.timing_log.get_report() assert any(e["step_name"] == "clone" for e in report) @pytest.mark.anyio @@ -232,12 +238,12 @@ class TestRemoveCloneTiming: """remove_clone records wall-clock timing when step_name is provided.""" @pytest.mark.anyio - async def test_remove_clone_step_name_records_timing(self, tool_ctx): + async def test_remove_clone_step_name_records_timing(self, tool_ctx_kitchen_open): mock_mgr = MagicMock() mock_mgr.remove_clone.return_value = {"removed": "true"} - tool_ctx.clone_mgr = mock_mgr + tool_ctx_kitchen_open.clone_mgr = mock_mgr await remove_clone(clone_path="/clone", step_name="cleanup") - report = tool_ctx.timing_log.get_report() + report = tool_ctx_kitchen_open.timing_log.get_report() assert any(e["step_name"] == "cleanup" for e in report) @pytest.mark.anyio @@ -253,12 +259,12 @@ class TestPushToRemoteTiming: """push_to_remote records wall-clock timing when step_name is provided.""" @pytest.mark.anyio - async def test_push_to_remote_step_name_records_timing(self, tool_ctx): + async def test_push_to_remote_step_name_records_timing(self, tool_ctx_kitchen_open): mock_mgr = MagicMock() mock_mgr.push_to_remote.return_value = {"success": "true", "stderr": ""} - tool_ctx.clone_mgr = mock_mgr + tool_ctx_kitchen_open.clone_mgr = mock_mgr await push_to_remote(clone_path="/clone", branch="main", step_name="push") - report = tool_ctx.timing_log.get_report() + report = tool_ctx_kitchen_open.timing_log.get_report() assert any(e["step_name"] == "push" for e in report) @pytest.mark.anyio @@ -272,9 +278,9 @@ async def test_push_to_remote_empty_step_name_skips_timing(self, tool_ctx): class TestRegisterCloneStatusTool: @pytest.mark.anyio - async def test_register_clone_status_success(self, tool_ctx, tmp_path): + async def test_register_clone_status_success(self, tool_ctx_kitchen_open, tmp_path): """register_clone_status status='success' writes registry and returns registered=true.""" - tool_ctx.kitchen_id = "kit-test" + tool_ctx_kitchen_open.kitchen_id = "kit-test" registry_path = str(tmp_path / "registry.json") result = json.loads( await register_clone_status( @@ -287,9 +293,9 @@ async def test_register_clone_status_success(self, tool_ctx, tmp_path): assert "registry_path" in result @pytest.mark.anyio - async def test_register_clone_status_error(self, tool_ctx, tmp_path): + async def test_register_clone_status_error(self, tool_ctx_kitchen_open, tmp_path): """register_clone_status status='error' writes registry and returns registered=true.""" - tool_ctx.kitchen_id = "kit-test" + tool_ctx_kitchen_open.kitchen_id = "kit-test" registry_path = str(tmp_path / "registry.json") result = json.loads( await register_clone_status( @@ -302,7 +308,7 @@ async def test_register_clone_status_error(self, tool_ctx, tmp_path): assert "registry_path" in result @pytest.mark.anyio - async def test_register_clone_status_invalid_status(self, tool_ctx, tmp_path): + async def test_register_clone_status_invalid_status(self, tool_ctx_kitchen_open, tmp_path): """register_clone_status with status='invalid' returns error without writing.""" registry_path = str(tmp_path / "registry.json") result = json.loads( @@ -317,12 +323,34 @@ async def test_register_clone_status_invalid_status(self, tool_ctx, tmp_path): # Registry file must not have been created assert not (tmp_path / "registry.json").exists() + @pytest.mark.anyio + async def test_register_clone_status_unconfirmed_accepted( + self, tool_ctx_kitchen_open, tmp_path + ): + """status='unconfirmed' is accepted: writes registry entry, returns registered=true.""" + tool_ctx_kitchen_open.kitchen_id = "kit-test" + registry_path = str(tmp_path / "registry.json") + result = json.loads( + await register_clone_status( + clone_path=str(tmp_path / "repo"), + status="unconfirmed", + registry_path=registry_path, + ) + ) + assert result["registered"] == "true" + # Verify the registry entry has status == "unconfirmed" + data = json.loads(Path(registry_path).read_text()) + assert len(data["clones"]) == 1 + assert data["clones"][0]["status"] == "unconfirmed" + class TestBatchCleanupClonesTool: @pytest.mark.anyio - async def test_batch_cleanup_clones_deletes_success_preserves_error(self, tool_ctx, tmp_path): + async def test_batch_cleanup_clones_deletes_success_preserves_error( + self, tool_ctx_kitchen_open, tmp_path + ): """batch_cleanup_clones removes success clones, skips error clones.""" - tool_ctx.kitchen_id = "kit-test" + tool_ctx_kitchen_open.kitchen_id = "kit-test" registry_path = str(tmp_path / "registry.json") success_path = str(tmp_path / "success_clone") error_path = str(tmp_path / "error_clone") @@ -334,7 +362,7 @@ async def test_batch_cleanup_clones_deletes_success_preserves_error(self, tool_c # Mock clone_mgr so remove_clone reports success for the success clone mock_mgr = MagicMock() mock_mgr.remove_clone.return_value = {"removed": "true"} - tool_ctx.clone_mgr = mock_mgr + tool_ctx_kitchen_open.clone_mgr = mock_mgr result = json.loads(await batch_cleanup_clones(registry_path=registry_path)) @@ -344,17 +372,19 @@ async def test_batch_cleanup_clones_deletes_success_preserves_error(self, tool_c mock_mgr.remove_clone.assert_called_once_with(success_path, "false") @pytest.mark.anyio - async def test_batch_cleanup_clones_empty_registry(self, tool_ctx, tmp_path): + async def test_batch_cleanup_clones_empty_registry(self, tool_ctx_kitchen_open, tmp_path): """batch_cleanup_clones with missing registry returns deleted=[], preserved=[].""" - tool_ctx.kitchen_id = "kit-test" + tool_ctx_kitchen_open.kitchen_id = "kit-test" registry_path = str(tmp_path / "nonexistent.json") result = json.loads(await batch_cleanup_clones(registry_path=registry_path)) assert result == {"deleted": [], "delete_failures": [], "preserved": []} @pytest.mark.anyio - async def test_batch_cleanup_clones_nonexistent_path_does_not_raise(self, tool_ctx, tmp_path): + async def test_batch_cleanup_clones_nonexistent_path_does_not_raise( + self, tool_ctx_kitchen_open, tmp_path + ): """batch_cleanup_clones reports failure gracefully when a success clone path is gone.""" - tool_ctx.kitchen_id = "kit-test" + tool_ctx_kitchen_open.kitchen_id = "kit-test" registry_path = str(tmp_path / "registry.json") missing_path = str(tmp_path / "gone_clone") @@ -364,7 +394,7 @@ async def test_batch_cleanup_clones_nonexistent_path_does_not_raise(self, tool_c # Mock clone_mgr to report removal failure (path not found) mock_mgr = MagicMock() mock_mgr.remove_clone.return_value = {"removed": "false", "reason": "not found"} - tool_ctx.clone_mgr = mock_mgr + tool_ctx_kitchen_open.clone_mgr = mock_mgr result = json.loads(await batch_cleanup_clones(registry_path=registry_path)) @@ -383,9 +413,11 @@ class TestRegisterCloneStatusOwner: """T12–T13: register_clone_status propagates kitchen_id as owner.""" @pytest.mark.anyio - async def test_register_clone_status_propagates_kitchen_id_as_owner(self, tool_ctx, tmp_path): + async def test_register_clone_status_propagates_kitchen_id_as_owner( + self, tool_ctx_kitchen_open, tmp_path + ): """T12 — register_clone_status writes entry with owner == kitchen_id.""" - tool_ctx.kitchen_id = "kit-xyz" + tool_ctx_kitchen_open.kitchen_id = "kit-xyz" reg = str(tmp_path / "registry.json") result = json.loads( await register_clone_status(clone_path="/c", status="success", registry_path=reg) @@ -397,9 +429,11 @@ async def test_register_clone_status_propagates_kitchen_id_as_owner(self, tool_c assert data["clones"][0]["owner"] == "kit-xyz" @pytest.mark.anyio - async def test_register_clone_status_rejects_when_kitchen_id_empty(self, tool_ctx, tmp_path): + async def test_register_clone_status_rejects_when_kitchen_id_empty( + self, tool_ctx_kitchen_open, tmp_path + ): """T13 — register_clone_status returns registered=false when kitchen_id is empty.""" - tool_ctx.kitchen_id = "" + tool_ctx_kitchen_open.kitchen_id = "" reg = str(tmp_path / "registry.json") result = json.loads( await register_clone_status(clone_path="/c", status="success", registry_path=reg) @@ -414,17 +448,17 @@ class TestBatchCleanupClonesOwner: @pytest.mark.anyio async def test_batch_cleanup_clones_default_scopes_to_current_kitchen_id( - self, tool_ctx, tmp_path + self, tool_ctx_kitchen_open, tmp_path ): """T14 — default call only removes current kitchen's clones.""" reg = str(tmp_path / "registry.json") clone_registry.register_clone("/clone-A", "success", "kit-A", reg) clone_registry.register_clone("/clone-B", "success", "kit-B", reg) - tool_ctx.kitchen_id = "kit-A" + tool_ctx_kitchen_open.kitchen_id = "kit-A" mock_mgr = MagicMock() mock_mgr.remove_clone.return_value = {"removed": "true"} - tool_ctx.clone_mgr = mock_mgr + tool_ctx_kitchen_open.clone_mgr = mock_mgr result = json.loads(await batch_cleanup_clones(registry_path=reg)) @@ -434,17 +468,17 @@ async def test_batch_cleanup_clones_default_scopes_to_current_kitchen_id( @pytest.mark.anyio async def test_batch_cleanup_clones_all_owners_true_removes_every_success( - self, tool_ctx, tmp_path + self, tool_ctx_kitchen_open, tmp_path ): """T15 — all_owners='true' escape hatch removes all success entries.""" reg = str(tmp_path / "registry.json") clone_registry.register_clone("/clone-A", "success", "kit-A", reg) clone_registry.register_clone("/clone-B", "success", "kit-B", reg) - tool_ctx.kitchen_id = "kit-A" + tool_ctx_kitchen_open.kitchen_id = "kit-A" mock_mgr = MagicMock() mock_mgr.remove_clone.return_value = {"removed": "true"} - tool_ctx.clone_mgr = mock_mgr + tool_ctx_kitchen_open.clone_mgr = mock_mgr result = json.loads(await batch_cleanup_clones(registry_path=reg, all_owners="true")) @@ -454,13 +488,13 @@ async def test_batch_cleanup_clones_all_owners_true_removes_every_success( @pytest.mark.anyio async def test_batch_cleanup_clones_empty_kitchen_id_and_all_owners_false_returns_error( - self, tool_ctx, tmp_path + self, tool_ctx_kitchen_open, tmp_path ): """T16 — empty kitchen_id with default all_owners='false' returns error.""" - tool_ctx.kitchen_id = "" + tool_ctx_kitchen_open.kitchen_id = "" reg = str(tmp_path / "registry.json") mock_mgr = MagicMock() - tool_ctx.clone_mgr = mock_mgr + tool_ctx_kitchen_open.clone_mgr = mock_mgr result = json.loads(await batch_cleanup_clones(registry_path=reg)) @@ -470,17 +504,17 @@ async def test_batch_cleanup_clones_empty_kitchen_id_and_all_owners_false_return @pytest.mark.anyio async def test_batch_cleanup_clones_empty_kitchen_id_with_all_owners_true_succeeds( - self, tool_ctx, tmp_path + self, tool_ctx_kitchen_open, tmp_path ): """T17 — escape hatch works even when kitchen_id is empty (legacy recovery).""" reg_path = tmp_path / "registry.json" reg_path.write_text(json.dumps({"clones": [{"path": "/legacy", "status": "success"}]})) - tool_ctx.kitchen_id = "" + tool_ctx_kitchen_open.kitchen_id = "" mock_mgr = MagicMock() mock_mgr.remove_clone.return_value = {"removed": "true"} - tool_ctx.clone_mgr = mock_mgr + tool_ctx_kitchen_open.clone_mgr = mock_mgr result = json.loads( await batch_cleanup_clones(registry_path=str(reg_path), all_owners="true") @@ -491,17 +525,17 @@ async def test_batch_cleanup_clones_empty_kitchen_id_with_all_owners_true_succee @pytest.mark.anyio async def test_batch_cleanup_clones_invalid_all_owners_literal_treated_as_false( - self, tool_ctx, tmp_path + self, tool_ctx_kitchen_open, tmp_path ): """T18 — all_owners='yes' is not the escape hatch; scoped behaviour applies.""" reg = str(tmp_path / "registry.json") clone_registry.register_clone("/clone-A", "success", "kit-A", reg) clone_registry.register_clone("/clone-B", "success", "kit-B", reg) - tool_ctx.kitchen_id = "kit-A" + tool_ctx_kitchen_open.kitchen_id = "kit-A" mock_mgr = MagicMock() mock_mgr.remove_clone.return_value = {"removed": "true"} - tool_ctx.clone_mgr = mock_mgr + tool_ctx_kitchen_open.clone_mgr = mock_mgr result = json.loads(await batch_cleanup_clones(registry_path=reg, all_owners="yes")) @@ -509,7 +543,9 @@ async def test_batch_cleanup_clones_invalid_all_owners_literal_treated_as_false( assert "/clone-B" not in result["deleted"] @pytest.mark.anyio - async def test_two_kitchens_register_and_cleanup_isolated(self, tool_ctx, tmp_path): + async def test_two_kitchens_register_and_cleanup_isolated( + self, tool_ctx_kitchen_open, tmp_path + ): """T19 — kitchen A's cleanup does not touch kitchen B's registry entry.""" reg = str(tmp_path / "registry.json") @@ -519,10 +555,10 @@ async def test_two_kitchens_register_and_cleanup_isolated(self, tool_ctx, tmp_pa clone_registry.register_clone("/clone-2", "success", "kit-2", reg) # Session 1 cleans up via the MCP tool - tool_ctx.kitchen_id = "kit-1" + tool_ctx_kitchen_open.kitchen_id = "kit-1" mock_mgr = MagicMock() mock_mgr.remove_clone.return_value = {"removed": "true"} - tool_ctx.clone_mgr = mock_mgr + tool_ctx_kitchen_open.clone_mgr = mock_mgr result = json.loads(await batch_cleanup_clones(registry_path=reg)) @@ -536,3 +572,106 @@ async def test_two_kitchens_register_and_cleanup_isolated(self, tool_ctx, tmp_pa remaining = {e["path"]: e.get("owner") for e in data["clones"]} assert "/clone-2" in remaining assert remaining["/clone-2"] == "kit-2" + + +class TestRegisterCloneStatusCampaignPreference: + """T26–T27: register_clone_status prefers AUTOSKILLIT_CAMPAIGN_ID over kitchen_id.""" + + @pytest.mark.anyio + async def test_register_clone_status_prefers_campaign_id_over_kitchen_id( + self, tool_ctx_kitchen_open, tmp_path, monkeypatch: pytest.MonkeyPatch + ): + """T26 — CAMPAIGN_ID env var takes precedence over tool_ctx_kitchen_open.kitchen_id.""" + monkeypatch.setenv("AUTOSKILLIT_CAMPAIGN_ID", "camp-x") + tool_ctx_kitchen_open.kitchen_id = "kit-y" + reg = str(tmp_path / "registry.json") + + result = json.loads( + await register_clone_status(clone_path="/c", status="success", registry_path=reg) + ) + assert result["registered"] == "true" + + data = json.loads(Path(reg).read_text()) + assert data["clones"][0]["owner"] == "camp-x" + + @pytest.mark.anyio + async def test_register_clone_status_falls_back_to_kitchen_id_without_campaign_id( + self, tool_ctx_kitchen_open, tmp_path, monkeypatch: pytest.MonkeyPatch + ): + """T27 — Falls back to tool_ctx_kitchen_open.kitchen_id when CAMPAIGN_ID is not set.""" + monkeypatch.delenv("AUTOSKILLIT_CAMPAIGN_ID", raising=False) + tool_ctx_kitchen_open.kitchen_id = "kit-fallback" + reg = str(tmp_path / "registry.json") + + result = json.loads( + await register_clone_status(clone_path="/c", status="success", registry_path=reg) + ) + assert result["registered"] == "true" + + data = json.loads(Path(reg).read_text()) + assert data["clones"][0]["owner"] == "kit-fallback" + + +class TestBatchCleanupClonesOwnerFilter: + """T28–T30: batch_cleanup_clones explicit owner_filter parameter.""" + + @pytest.mark.anyio + async def test_batch_cleanup_clones_owner_filter_scopes_to_specified_owner( + self, tool_ctx_kitchen_open, tmp_path + ): + """T28 — owner_filter='camp-A' removes only camp-A's clones regardless of kitchen_id.""" + reg = str(tmp_path / "registry.json") + clone_registry.register_clone("/clone-a", "success", "camp-A", reg) + clone_registry.register_clone("/clone-b", "success", "camp-B", reg) + + tool_ctx_kitchen_open.kitchen_id = "kit-x" + mock_mgr = MagicMock() + mock_mgr.remove_clone.return_value = {"removed": "true"} + tool_ctx_kitchen_open.clone_mgr = mock_mgr + + result = json.loads(await batch_cleanup_clones(registry_path=reg, owner_filter="camp-A")) + + assert "/clone-a" in result["deleted"] + assert "/clone-b" not in result["deleted"] + mock_mgr.remove_clone.assert_called_once_with("/clone-a", "false") + + @pytest.mark.anyio + async def test_batch_cleanup_clones_owner_filter_empty_falls_back_to_kitchen_id( + self, tool_ctx_kitchen_open, tmp_path + ): + """T29 — owner_filter='' falls back to kitchen_id (backward compatible).""" + reg = str(tmp_path / "registry.json") + clone_registry.register_clone("/clone-a", "success", "kit-x", reg) + clone_registry.register_clone("/clone-b", "success", "kit-y", reg) + + tool_ctx_kitchen_open.kitchen_id = "kit-x" + mock_mgr = MagicMock() + mock_mgr.remove_clone.return_value = {"removed": "true"} + tool_ctx_kitchen_open.clone_mgr = mock_mgr + + result = json.loads(await batch_cleanup_clones(registry_path=reg, owner_filter="")) + + assert "/clone-a" in result["deleted"] + assert "/clone-b" not in result["deleted"] + + @pytest.mark.anyio + async def test_batch_cleanup_clones_owner_filter_with_all_owners_true_ignores_filter( + self, tool_ctx_kitchen_open, tmp_path + ): + """T30 — owner_filter='camp-A' + all_owners='true' → all_owners takes precedence.""" + reg = str(tmp_path / "registry.json") + clone_registry.register_clone("/clone-a", "success", "camp-A", reg) + clone_registry.register_clone("/clone-b", "success", "camp-B", reg) + + tool_ctx_kitchen_open.kitchen_id = "kit-x" + mock_mgr = MagicMock() + mock_mgr.remove_clone.return_value = {"removed": "true"} + tool_ctx_kitchen_open.clone_mgr = mock_mgr + + result = json.loads( + await batch_cleanup_clones(registry_path=reg, owner_filter="camp-A", all_owners="true") + ) + + deleted = result["deleted"] + assert "/clone-a" in deleted + assert "/clone-b" in deleted diff --git a/tests/server/test_tools_dispatch.py b/tests/server/test_tools_dispatch.py new file mode 100644 index 0000000000..6310f9ee80 --- /dev/null +++ b/tests/server/test_tools_dispatch.py @@ -0,0 +1,902 @@ +"""Tests for dispatch_food_truck tool handler and execute_dispatch domain function.""" + +from __future__ import annotations + +import asyncio +import json +from pathlib import Path +from unittest.mock import AsyncMock + +import pytest + +from autoskillit.fleet import FleetSemaphore +from tests.fakes import InMemoryHeadlessExecutor, InMemoryRecipeRepository +from tests.server._helpers import _make_recipe_info, _make_standard_recipe + +pytestmark = [pytest.mark.layer("server"), pytest.mark.medium, pytest.mark.feature("fleet")] + +# --------------------------------------------------------------------------- +# Shared test helpers +# --------------------------------------------------------------------------- + + +def _simple_prompt_builder(**kwargs) -> str: + """Minimal prompt builder for tests — avoids CLI imports.""" + return f"prompt-for-{kwargs.get('recipe', 'unknown')}" + + +async def _no_sleep_quota_checker(config, **kwargs) -> dict: + """Quota checker stub: always returns no-sleep result.""" + return { + "should_sleep": False, + "sleep_seconds": 0, + "utilization": None, + "resets_at": None, + "window_name": None, + } + + +async def _noop_quota_refresher(config, **kwargs) -> None: + """Quota refresher stub: no-op.""" + + +# --------------------------------------------------------------------------- +# Class TestDispatchFoodTruckGates — headless refusal, kitchen gate, lock contention +# --------------------------------------------------------------------------- + + +class TestDispatchFoodTruckGates: + @pytest.mark.anyio + async def test_dispatch_food_truck_hard_refusal_headless( + self, tool_ctx_kitchen_open, monkeypatch + ): + """AUTOSKILLIT_HEADLESS=1 → fleet_hard_refusal_headless, regardless of SESSION_TYPE.""" + monkeypatch.setenv("AUTOSKILLIT_HEADLESS", "1") + from autoskillit.server.tools.tools_fleet_dispatch import dispatch_food_truck + + result = json.loads(await dispatch_food_truck(recipe="r", task="t")) + assert result["success"] is False + assert result["subtype"] == "headless_error" + + @pytest.mark.anyio + async def test_dispatch_food_truck_requires_fleet_session_type( + self, tool_ctx_kitchen_open, monkeypatch + ): + """Non-fleet session type → headless_error, even for interactive callers.""" + monkeypatch.setenv("AUTOSKILLIT_SESSION_TYPE", "orchestrator") + monkeypatch.delenv("AUTOSKILLIT_HEADLESS", raising=False) + from autoskillit.server.tools.tools_fleet_dispatch import dispatch_food_truck + + result = json.loads(await dispatch_food_truck(recipe="r", task="t")) + assert result["success"] is False + assert result["subtype"] == "headless_error" + + @pytest.mark.anyio + async def test_dispatch_food_truck_requires_kitchen_open(self, tool_ctx, monkeypatch): + """Kitchen closed → gate_error_result JSON.""" + from autoskillit.pipeline.gate import DefaultGateState + from autoskillit.server.tools.tools_fleet_dispatch import dispatch_food_truck + + tool_ctx.gate = DefaultGateState(enabled=False) + result = json.loads(await dispatch_food_truck(recipe="r", task="t")) + assert result["success"] is False + assert result["subtype"] == "gate_error" + + @pytest.mark.anyio + async def test_dispatch_food_truck_parallel_refused_when_locked( + self, tool_ctx, monkeypatch, tmp_path + ): + """fleet_lock.at_capacity() == True → fleet_parallel_refused error.""" + from autoskillit.fleet._api import execute_dispatch + + lock = FleetSemaphore(max_concurrent=1) + await lock.acquire() # lock it + tool_ctx.fleet_lock = lock + + result = json.loads( + await execute_dispatch( + tool_ctx=tool_ctx, + recipe="r", + task="t", + ingredients=None, + dispatch_name=None, + timeout_sec=None, + prompt_builder=_simple_prompt_builder, + quota_checker=_no_sleep_quota_checker, + quota_refresher=_noop_quota_refresher, + ) + ) + assert result["success"] is False + assert result["error"] == "fleet_parallel_refused" + + @pytest.mark.anyio + async def test_dispatch_food_truck_refuses_when_fleet_feature_disabled( + self, tool_ctx_kitchen_open, monkeypatch + ): + """features.fleet: false in config → fleet_feature_disabled, regardless of gate state.""" + import dataclasses + + from autoskillit.server.tools.tools_fleet_dispatch import dispatch_food_truck + + monkeypatch.setenv("AUTOSKILLIT_SESSION_TYPE", "fleet") + # Gate is open (fleet session already booted), env var absent + # Only config file has fleet disabled + monkeypatch.delenv("AUTOSKILLIT_FEATURES__FLEET", raising=False) + tool_ctx_kitchen_open.config = dataclasses.replace( + tool_ctx_kitchen_open.config, features={"fleet": False} + ) + + result = json.loads(await dispatch_food_truck(recipe="r", task="t")) + assert result["success"] is False + assert result["error"] == "fleet_feature_disabled" + + +# --------------------------------------------------------------------------- +# Class TestDispatchFoodTruckValidation — recipe kind, ingredient keys, non-string values +# --------------------------------------------------------------------------- + + +class TestDispatchFoodTruckValidation: + @pytest.mark.anyio + async def test_dispatch_food_truck_rejects_non_standard_recipe(self, tool_ctx, monkeypatch): + """Campaign recipe → fleet_invalid_recipe_kind error.""" + from autoskillit.fleet._api import execute_dispatch + from autoskillit.recipe.schema import Recipe, RecipeKind + + tool_ctx.fleet_lock = FleetSemaphore(max_concurrent=1) + repo = InMemoryRecipeRepository() + recipe_info = _make_recipe_info("campaign-recipe") + repo.add_recipe("campaign-recipe", recipe_info) + repo.add_full_recipe( + recipe_info.path, + Recipe(name="campaign-recipe", description="test", kind=RecipeKind.CAMPAIGN), + ) + tool_ctx.recipes = repo + + result = json.loads( + await execute_dispatch( + tool_ctx=tool_ctx, + recipe="campaign-recipe", + task="t", + ingredients=None, + dispatch_name=None, + timeout_sec=None, + prompt_builder=_simple_prompt_builder, + quota_checker=_no_sleep_quota_checker, + quota_refresher=_noop_quota_refresher, + ) + ) + assert result["success"] is False + assert result["error"] == "fleet_invalid_recipe_kind" + + @pytest.mark.anyio + async def test_dispatch_food_truck_rejects_unknown_ingredients(self, tool_ctx, monkeypatch): + """Keys not in recipe.ingredients → fleet_unknown_ingredient error.""" + from autoskillit.fleet._api import execute_dispatch + + tool_ctx.fleet_lock = FleetSemaphore(max_concurrent=1) + repo = InMemoryRecipeRepository() + recipe_info = _make_recipe_info("test-recipe") + repo.add_recipe("test-recipe", recipe_info) + repo.add_full_recipe(recipe_info.path, _make_standard_recipe("test-recipe", ["task"])) + tool_ctx.recipes = repo + tool_ctx.executor = InMemoryHeadlessExecutor() + + result = json.loads( + await execute_dispatch( + tool_ctx=tool_ctx, + recipe="test-recipe", + task="t", + ingredients={"task": "v", "unknown_key": "bad"}, + dispatch_name=None, + timeout_sec=None, + prompt_builder=_simple_prompt_builder, + quota_checker=_no_sleep_quota_checker, + quota_refresher=_noop_quota_refresher, + ) + ) + assert result["success"] is False + assert result["error"] == "fleet_unknown_ingredient" + assert "unknown_key" in result["user_visible_message"] + + @pytest.mark.anyio + async def test_dispatch_food_truck_rejects_non_string_values(self, tool_ctx, monkeypatch): + """Non-string ingredient values rejected before lock acquisition.""" + from autoskillit.fleet._api import execute_dispatch + + lock = FleetSemaphore(max_concurrent=1) + tool_ctx.fleet_lock = lock + + result = json.loads( + await execute_dispatch( + tool_ctx=tool_ctx, + recipe="r", + task="t", + ingredients={"key": 123}, # type: ignore[dict-item] + dispatch_name=None, + timeout_sec=None, + prompt_builder=_simple_prompt_builder, + quota_checker=_no_sleep_quota_checker, + quota_refresher=_noop_quota_refresher, + ) + ) + assert result["success"] is False + assert result["error"] == "fleet_unknown_ingredient" + # Lock must not have been acquired + assert not lock.at_capacity() + + @pytest.mark.anyio + async def test_dispatch_food_truck_no_recipes_configured(self, tool_ctx, monkeypatch): + """recipes=None → fleet_manifest_missing error.""" + from autoskillit.fleet._api import execute_dispatch + + tool_ctx.fleet_lock = FleetSemaphore(max_concurrent=1) + tool_ctx.recipes = None + + result = json.loads( + await execute_dispatch( + tool_ctx=tool_ctx, + recipe="r", + task="t", + ingredients=None, + dispatch_name=None, + timeout_sec=None, + prompt_builder=_simple_prompt_builder, + quota_checker=_no_sleep_quota_checker, + quota_refresher=_noop_quota_refresher, + ) + ) + assert result["success"] is False + assert result["error"] == "fleet_manifest_missing" + + @pytest.mark.anyio + async def test_dispatch_food_truck_no_executor_configured(self, tool_ctx, monkeypatch): + """executor=None → fleet_manifest_missing error.""" + from autoskillit.fleet._api import execute_dispatch + + tool_ctx.fleet_lock = FleetSemaphore(max_concurrent=1) + repo = InMemoryRecipeRepository() + recipe_info = _make_recipe_info("test-recipe") + repo.add_recipe("test-recipe", recipe_info) + repo.add_full_recipe(recipe_info.path, _make_standard_recipe("test-recipe")) + tool_ctx.recipes = repo + tool_ctx.executor = None + + result = json.loads( + await execute_dispatch( + tool_ctx=tool_ctx, + recipe="test-recipe", + task="t", + ingredients=None, + dispatch_name=None, + timeout_sec=None, + prompt_builder=_simple_prompt_builder, + quota_checker=_no_sleep_quota_checker, + quota_refresher=_noop_quota_refresher, + ) + ) + assert result["success"] is False + assert result["error"] == "fleet_manifest_missing" + + @pytest.mark.anyio + async def test_dispatch_recipe_info_kind_attribute_error_is_fixed(self, tool_ctx): + """find() returns RecipeInfo in production; dispatch must not crash on .kind. + + Previously all dispatch tests stored Recipe objects in the fake, masking the + AttributeError. This test uses RecipeInfo (the actual production return type). + Before fix: recipe_obj.kind raises AttributeError → L3_STARTUP_OR_CRASH. + After fix: load_recipe upgrades RecipeInfo → Recipe before kind check. + """ + from autoskillit.fleet._api import execute_dispatch + from autoskillit.recipe.schema import Recipe, RecipeInfo, RecipeKind, RecipeSource + + tool_ctx.fleet_lock = FleetSemaphore(max_concurrent=1) + tool_ctx.executor = None + repo = InMemoryRecipeRepository() + recipe_info = RecipeInfo( + name="test-recipe", + description="test", + source=RecipeSource.PROJECT, + path=Path("/fake/recipes/test-recipe.yaml"), + ) + repo.add_recipe("test-recipe", recipe_info) + repo.add_full_recipe( + recipe_info.path, + Recipe(name="test-recipe", description="test", kind=RecipeKind.STANDARD), + ) + tool_ctx.recipes = repo + + result = json.loads( + await execute_dispatch( + tool_ctx=tool_ctx, + recipe="test-recipe", + task="run task", + ingredients=None, + dispatch_name=None, + timeout_sec=None, + prompt_builder=_simple_prompt_builder, + quota_checker=_no_sleep_quota_checker, + quota_refresher=_noop_quota_refresher, + ) + ) + + assert result.get("error") != "fleet_l3_startup_or_crash", ( + "Expected structured validation error, not FLEET_L3_STARTUP_OR_CRASH. " + "RecipeInfo.kind AttributeError is not fixed." + ) + + @pytest.mark.anyio + async def test_dispatch_recipe_info_ingredients_attribute_error_is_fixed(self, tool_ctx): + """find() returns RecipeInfo; ingredients validation must not crash on + recipe_obj.ingredients when non-empty ingredients are passed. + + Before fix: recipe_obj.ingredients raises AttributeError → L3_STARTUP_OR_CRASH. + After fix: load_recipe upgrades RecipeInfo → Recipe; unknown ingredient detected. + """ + from autoskillit.fleet._api import execute_dispatch + from autoskillit.recipe.schema import ( + Recipe, + RecipeInfo, + RecipeIngredient, + RecipeKind, + RecipeSource, + ) + + tool_ctx.fleet_lock = FleetSemaphore(max_concurrent=1) + tool_ctx.executor = None + repo = InMemoryRecipeRepository() + recipe_info = RecipeInfo( + name="test-recipe", + description="test", + source=RecipeSource.PROJECT, + path=Path("/fake/recipes/test-recipe.yaml"), + ) + repo.add_recipe("test-recipe", recipe_info) + repo.add_full_recipe( + recipe_info.path, + Recipe( + name="test-recipe", + description="test", + kind=RecipeKind.STANDARD, + ingredients={"env": RecipeIngredient(description="env var")}, + ), + ) + tool_ctx.recipes = repo + + result = json.loads( + await execute_dispatch( + tool_ctx=tool_ctx, + recipe="test-recipe", + task="run task", + ingredients={"unknown_key": "val"}, + dispatch_name=None, + timeout_sec=None, + prompt_builder=_simple_prompt_builder, + quota_checker=_no_sleep_quota_checker, + quota_refresher=_noop_quota_refresher, + ) + ) + + assert result.get("error") == "fleet_unknown_ingredient", ( + f"Expected fleet_unknown_ingredient error. Got: {result}" + ) + + +# --------------------------------------------------------------------------- +# Class TestDispatchFoodTruckExecution — lock lifecycle, success, pid, quota, cleanup +# --------------------------------------------------------------------------- + + +class TestDispatchFoodTruckExecution: + def _setup_standard_dispatch(self, tool_ctx, monkeypatch): + """Wire tool_ctx for a successful standard dispatch.""" + tool_ctx.fleet_lock = FleetSemaphore(max_concurrent=1) + repo = InMemoryRecipeRepository() + recipe_info = _make_recipe_info("test-recipe") + repo.add_recipe("test-recipe", recipe_info) + repo.add_full_recipe(recipe_info.path, _make_standard_recipe("test-recipe", ["task"])) + tool_ctx.recipes = repo + tool_ctx.executor = InMemoryHeadlessExecutor() + + @pytest.mark.anyio + async def test_dispatch_food_truck_releases_lock_on_success(self, tool_ctx, monkeypatch): + """Lock released after successful dispatch.""" + from autoskillit.fleet._api import execute_dispatch + + self._setup_standard_dispatch(tool_ctx, monkeypatch) + + await execute_dispatch( + tool_ctx=tool_ctx, + recipe="test-recipe", + task="t", + ingredients=None, + dispatch_name=None, + timeout_sec=None, + prompt_builder=_simple_prompt_builder, + quota_checker=_no_sleep_quota_checker, + quota_refresher=_noop_quota_refresher, + ) + assert not tool_ctx.fleet_lock.at_capacity() + + @pytest.mark.anyio + async def test_dispatch_food_truck_releases_lock_on_exception(self, tool_ctx, monkeypatch): + """Lock released when executor raises.""" + from autoskillit.fleet._api import execute_dispatch + + self._setup_standard_dispatch(tool_ctx, monkeypatch) + tool_ctx.executor.dispatch_food_truck = AsyncMock( + side_effect=RuntimeError("executor crashed") + ) + + result = json.loads( + await execute_dispatch( + tool_ctx=tool_ctx, + recipe="test-recipe", + task="t", + ingredients=None, + dispatch_name=None, + timeout_sec=None, + prompt_builder=_simple_prompt_builder, + quota_checker=_no_sleep_quota_checker, + quota_refresher=_noop_quota_refresher, + ) + ) + assert result["success"] is False + assert not tool_ctx.fleet_lock.at_capacity() + + @pytest.mark.anyio + async def test_dispatch_food_truck_releases_lock_on_cancellation(self, tool_ctx, monkeypatch): + """Lock released on asyncio.CancelledError.""" + from autoskillit.fleet._api import execute_dispatch + + self._setup_standard_dispatch(tool_ctx, monkeypatch) + tool_ctx.executor.dispatch_food_truck = AsyncMock(side_effect=asyncio.CancelledError()) + + with pytest.raises(asyncio.CancelledError): + await execute_dispatch( + tool_ctx=tool_ctx, + recipe="test-recipe", + task="t", + ingredients=None, + dispatch_name=None, + timeout_sec=None, + prompt_builder=_simple_prompt_builder, + quota_checker=_no_sleep_quota_checker, + quota_refresher=_noop_quota_refresher, + ) + assert not tool_ctx.fleet_lock.at_capacity() + + @pytest.mark.anyio + async def test_dispatch_food_truck_success_envelope(self, tool_ctx, monkeypatch): + """Returns envelope with success, dispatch_id, l3_payload, token_usage, l3_parse_source.""" + import dataclasses + + from autoskillit.fleet._api import execute_dispatch + from autoskillit.fleet.result_parser import L3ParseResult + from tests.fakes import _DEFAULT_SKILL_RESULT + + self._setup_standard_dispatch(tool_ctx, monkeypatch) + tool_ctx.executor = InMemoryHeadlessExecutor( + default_result=dataclasses.replace( + _DEFAULT_SKILL_RESULT, + success=True, + result="dispatch done", + session_id="sess-abc", + token_usage={"input_tokens": 100}, + ) + ) + + canned_payload = {"success": True, "data": "dispatch done"} + canned_result = L3ParseResult( + outcome="completed_clean", + payload=canned_payload, + raw_body=None, + parse_error=None, + source="stdout", + ) + monkeypatch.setattr( + "autoskillit.fleet._api.parse_l3_result_block", + lambda **_kwargs: canned_result, + ) + + raw = await execute_dispatch( + tool_ctx=tool_ctx, + recipe="test-recipe", + task="complete the task", + ingredients={"task": "override-task"}, + dispatch_name=None, + timeout_sec=None, + prompt_builder=_simple_prompt_builder, + quota_checker=_no_sleep_quota_checker, + quota_refresher=_noop_quota_refresher, + ) + result = json.loads(raw) + assert result["success"] is True + assert "dispatch_id" in result + assert result["dispatched_session_id"] == "sess-abc" + assert result["l3_payload"] == canned_payload + assert result["token_usage"] == {"input_tokens": 100} + assert result["l3_parse_source"] == "stdout" + + @pytest.mark.anyio + async def test_dispatch_food_truck_on_spawn_writes_pid(self, tool_ctx, monkeypatch): + """on_spawn callback writes dispatched_pid into state.json via mark_dispatch_running.""" + from autoskillit.fleet._api import _write_pid + from autoskillit.fleet.state import DispatchRecord, write_initial_state + + state_path = tool_ctx.temp_dir / "dispatches" / "test-dispatch.json" + state_path.parent.mkdir(parents=True, exist_ok=True) + write_initial_state( + state_path, + campaign_id="kitchen-id", + campaign_name="test-dispatch-name", + manifest_path="", + dispatches=[DispatchRecord(name="test-dispatch-name")], + ) + + _write_pid(state_path, "test-dispatch-name", "dispatch-id-abc", 54321, 0) + + state_data = json.loads(state_path.read_text()) + dispatch_record = state_data["dispatches"][0] + assert dispatch_record["dispatched_pid"] == 54321 + assert dispatch_record["status"] == "running" + assert dispatch_record["dispatch_id"] == "dispatch-id-abc" + + @pytest.mark.anyio + async def test_dispatch_food_truck_passes_on_spawn_to_executor(self, tool_ctx, monkeypatch): + """execute_dispatch passes an on_spawn that writes the PID to the state file.""" + from autoskillit.fleet._api import execute_dispatch + from autoskillit.fleet.state import read_state + + self._setup_standard_dispatch(tool_ctx, monkeypatch) + + # Wrap dispatch_food_truck to invoke on_spawn before returning, + # simulating the real headless executor calling the callback on process start. + original_dispatch = tool_ctx.executor.dispatch_food_truck + + async def _dispatch_invoking_spawn(*args, on_spawn=None, **kwargs): + result = await original_dispatch(*args, on_spawn=on_spawn, **kwargs) + if on_spawn is not None: + on_spawn(99999, 0) + return result + + monkeypatch.setattr(tool_ctx.executor, "dispatch_food_truck", _dispatch_invoking_spawn) + + await execute_dispatch( + tool_ctx=tool_ctx, + recipe="test-recipe", + task="t", + ingredients=None, + dispatch_name=None, + timeout_sec=None, + prompt_builder=_simple_prompt_builder, + quota_checker=_no_sleep_quota_checker, + quota_refresher=_noop_quota_refresher, + ) + dispatch_id = tool_ctx.executor.dispatch_calls[0].order_id + state_path = tool_ctx.temp_dir / "dispatches" / f"{dispatch_id}.json" + state = read_state(state_path) + assert state is not None + assert any(d.dispatched_pid == 99999 for d in state.dispatches) + + @pytest.mark.anyio + async def test_dispatch_food_truck_invalidates_quota_cache(self, tool_ctx, monkeypatch): + """After dispatch completes, quota cache is refreshed via background supervisor.""" + from autoskillit.fleet._api import execute_dispatch + + self._setup_standard_dispatch(tool_ctx, monkeypatch) + + submitted_labels: list[str] = [] + + def _capture_submit(coro, label: str = "") -> None: + submitted_labels.append(label) + coro.close() + + monkeypatch.setattr(tool_ctx.background, "submit", _capture_submit) + + await execute_dispatch( + tool_ctx=tool_ctx, + recipe="test-recipe", + task="t", + ingredients=None, + dispatch_name=None, + timeout_sec=None, + prompt_builder=_simple_prompt_builder, + quota_checker=_no_sleep_quota_checker, + quota_refresher=_noop_quota_refresher, + ) + assert "quota_post_dispatch_refresh" in submitted_labels + + @pytest.mark.anyio + async def test_dispatch_food_truck_cleans_session_skills( + self, tool_ctx, monkeypatch, tmp_path + ): + """Completed L3 session skill dir is cleaned up.""" + from autoskillit.fleet._api import execute_dispatch + + self._setup_standard_dispatch(tool_ctx, monkeypatch) + import dataclasses + + from tests.fakes import _DEFAULT_SKILL_RESULT + + tool_ctx.executor = InMemoryHeadlessExecutor( + default_result=dataclasses.replace( + _DEFAULT_SKILL_RESULT, + success=True, + session_id="l2-session-xyz", + ) + ) + + cleanup_calls: list[str] = [] + + def _capture_cleanup(session_id: str) -> bool: + cleanup_calls.append(session_id) + return False + + monkeypatch.setattr(tool_ctx.session_skill_manager, "cleanup_session", _capture_cleanup) + + await execute_dispatch( + tool_ctx=tool_ctx, + recipe="test-recipe", + task="t", + ingredients=None, + dispatch_name=None, + timeout_sec=None, + prompt_builder=_simple_prompt_builder, + quota_checker=_no_sleep_quota_checker, + quota_refresher=_noop_quota_refresher, + ) + assert "l2-session-xyz" in cleanup_calls + + @pytest.mark.anyio + async def test_dispatch_food_truck_invalidates_quota_cache_file(self, tool_ctx, monkeypatch): + """After dispatch, cache_invalidator is called with the configured cache path.""" + from autoskillit.fleet._api import execute_dispatch + + self._setup_standard_dispatch(tool_ctx, monkeypatch) + + invalidate_calls: list[str] = [] + + def _capture_invalidate(cache_path: str) -> None: + invalidate_calls.append(cache_path) + + await execute_dispatch( + tool_ctx=tool_ctx, + recipe="test-recipe", + task="t", + ingredients=None, + dispatch_name=None, + timeout_sec=None, + prompt_builder=_simple_prompt_builder, + quota_checker=_no_sleep_quota_checker, + quota_refresher=_noop_quota_refresher, + cache_invalidator=_capture_invalidate, + ) + + assert tool_ctx.config.quota_guard.cache_path in invalidate_calls + + @pytest.mark.anyio + @pytest.mark.parametrize("exc_cls", [RuntimeError, OSError]) + async def test_dispatch_food_truck_succeeds_when_cleanup_session_raises( + self, tool_ctx, monkeypatch, exc_cls + ): + """Dispatch result is success even when cleanup_session raises an exception.""" + import dataclasses + import json + + from autoskillit.fleet._api import execute_dispatch + from autoskillit.fleet.result_parser import L3ParseResult + from tests.fakes import _DEFAULT_SKILL_RESULT + + self._setup_standard_dispatch(tool_ctx, monkeypatch) + tool_ctx.executor = InMemoryHeadlessExecutor( + default_result=dataclasses.replace( + _DEFAULT_SKILL_RESULT, + success=True, + session_id="l2-session-err", + ) + ) + monkeypatch.setattr( + "autoskillit.fleet._api.parse_l3_result_block", + lambda **_kwargs: L3ParseResult( + outcome="completed_clean", + payload={"success": True}, + raw_body=None, + parse_error=None, + source="stdout", + ), + ) + + def _raise_error(session_id: str) -> bool: + raise exc_cls("simulated cleanup failure") + + monkeypatch.setattr(tool_ctx.session_skill_manager, "cleanup_session", _raise_error) + + result_json = await execute_dispatch( + tool_ctx=tool_ctx, + recipe="test-recipe", + task="t", + ingredients=None, + dispatch_name=None, + timeout_sec=None, + prompt_builder=_simple_prompt_builder, + quota_checker=_no_sleep_quota_checker, + quota_refresher=_noop_quota_refresher, + ) + + result = json.loads(result_json) + assert result["success"] is True + + +@pytest.mark.anyio +async def test_dispatch_food_truck_tool_passes_resume_session_id_to_executor( + tool_ctx_kitchen_open, monkeypatch +): + """dispatch_food_truck MCP tool forwards resume_session_id all the way to the executor.""" + from autoskillit.server.tools.tools_fleet_dispatch import dispatch_food_truck + + monkeypatch.setenv("AUTOSKILLIT_SESSION_TYPE", "fleet") + tool_ctx_kitchen_open.fleet_lock = FleetSemaphore(max_concurrent=1) + repo = InMemoryRecipeRepository() + recipe_info = _make_recipe_info("test-recipe") + repo.add_recipe("test-recipe", recipe_info) + repo.add_full_recipe(recipe_info.path, _make_standard_recipe("test-recipe")) + tool_ctx_kitchen_open.recipes = repo + executor = InMemoryHeadlessExecutor() + tool_ctx_kitchen_open.executor = executor + + await dispatch_food_truck( + recipe="test-recipe", + task="do-work", + resume_session_id="sess-resume-123", + ) + + assert executor.dispatch_calls, "dispatch_food_truck executor was never called" + assert executor.dispatch_calls[0].resume_session_id == "sess-resume-123" + + +@pytest.mark.anyio +async def test_dispatch_food_truck_marketplace_install_succeeds(tool_ctx_marketplace, monkeypatch): + """dispatch_food_truck does not raise when plugin_source is MarketplaceInstall. + + This test was impossible before the fix — it would raise ValueError. + """ + from autoskillit.fleet._api import execute_dispatch + from autoskillit.fleet.result_parser import L3ParseResult + + monkeypatch.setattr( + "autoskillit.fleet._api.parse_l3_result_block", + lambda **_kwargs: L3ParseResult( + outcome="completed_clean", + payload={"success": True}, + raw_body=None, + parse_error=None, + source="stdout", + ), + ) + + tool_ctx_marketplace.fleet_lock = FleetSemaphore(max_concurrent=1) + repo = InMemoryRecipeRepository() + recipe_info = _make_recipe_info("test-recipe") + repo.add_recipe("test-recipe", recipe_info) + repo.add_full_recipe(recipe_info.path, _make_standard_recipe("test-recipe", ["task"])) + tool_ctx_marketplace.recipes = repo + tool_ctx_marketplace.executor = InMemoryHeadlessExecutor() + + result = json.loads( + await execute_dispatch( + tool_ctx=tool_ctx_marketplace, + recipe="test-recipe", + task="t", + ingredients=None, + dispatch_name=None, + timeout_sec=None, + prompt_builder=_simple_prompt_builder, + quota_checker=_no_sleep_quota_checker, + quota_refresher=_noop_quota_refresher, + ) + ) + assert result["success"] is True + + +class TestDispatchFoodTruckIdleTimeout: + """Tests for idle_output_timeout passthrough through the dispatch chain.""" + + @pytest.fixture(autouse=True) + def _set_fleet_session(self, monkeypatch): + monkeypatch.setenv("AUTOSKILLIT_SESSION_TYPE", "fleet") + + def _setup_dispatch(self, tool_ctx): + """Wire tool_ctx for a standard dispatch with InMemoryHeadlessExecutor.""" + tool_ctx.fleet_lock = FleetSemaphore(max_concurrent=1) + repo = InMemoryRecipeRepository() + recipe_info = _make_recipe_info("test-recipe") + repo.add_recipe("test-recipe", recipe_info) + repo.add_full_recipe(recipe_info.path, _make_standard_recipe("test-recipe")) + tool_ctx.recipes = repo + tool_ctx.executor = InMemoryHeadlessExecutor() + + @pytest.mark.anyio + async def test_dispatch_food_truck_passes_idle_output_timeout_to_executor( + self, tool_ctx_kitchen_open, monkeypatch + ): + """dispatch_food_truck MCP tool forwards idle_output_timeout to executor.""" + from autoskillit.server.tools.tools_fleet_dispatch import dispatch_food_truck + + self._setup_dispatch(tool_ctx_kitchen_open) + + await dispatch_food_truck( + recipe="test-recipe", + task="do-work", + idle_output_timeout=0, + ) + + executor = tool_ctx_kitchen_open.executor + assert executor.dispatch_calls, "dispatch_food_truck executor was never called" + assert executor.dispatch_calls[0].idle_output_timeout == 0.0 + + @pytest.mark.anyio + async def test_dispatch_food_truck_idle_timeout_none_when_not_specified( + self, tool_ctx_kitchen_open, monkeypatch + ): + """When idle_output_timeout is not specified, executor receives None.""" + from autoskillit.server.tools.tools_fleet_dispatch import dispatch_food_truck + + self._setup_dispatch(tool_ctx_kitchen_open) + + await dispatch_food_truck( + recipe="test-recipe", + task="do-work", + ) + + executor = tool_ctx_kitchen_open.executor + assert executor.dispatch_calls, "dispatch_food_truck executor was never called" + assert executor.dispatch_calls[0].idle_output_timeout is None + + @pytest.mark.anyio + async def test_dispatch_food_truck_idle_timeout_overrides_config_default( + self, tool_ctx_kitchen_open, monkeypatch + ): + """Explicit idle_output_timeout=0 overrides the config default of 1000.""" + from autoskillit.server.tools.tools_fleet_dispatch import dispatch_food_truck + + self._setup_dispatch(tool_ctx_kitchen_open) + # Config idle_output_timeout is 1000 (default from RunSkillConfig) + assert tool_ctx_kitchen_open.config.run_skill.idle_output_timeout == 1000 + + await dispatch_food_truck( + recipe="test-recipe", + task="do-work", + idle_output_timeout=0, + ) + + executor = tool_ctx_kitchen_open.executor + assert executor.dispatch_calls, "dispatch_food_truck executor was never called" + # Executor receives 0.0, overriding the config default + assert executor.dispatch_calls[0].idle_output_timeout == 0.0 + + @pytest.mark.anyio + async def test_execute_dispatch_passes_idle_output_timeout_to_executor( + self, tool_ctx, monkeypatch + ): + """execute_dispatch forwards idle_output_timeout to executor.dispatch_food_truck.""" + from autoskillit.fleet._api import execute_dispatch + + self._setup_dispatch(tool_ctx) + + await execute_dispatch( + tool_ctx=tool_ctx, + recipe="test-recipe", + task="do-work", + ingredients=None, + dispatch_name=None, + timeout_sec=None, + prompt_builder=_simple_prompt_builder, + quota_checker=_no_sleep_quota_checker, + quota_refresher=_noop_quota_refresher, + idle_output_timeout=0, + ) + + executor = tool_ctx.executor + assert executor.dispatch_calls, "dispatch_food_truck was never called" + assert executor.dispatch_calls[0].idle_output_timeout == 0.0 diff --git a/tests/server/test_tools_dispatch_halt.py b/tests/server/test_tools_dispatch_halt.py new file mode 100644 index 0000000000..8a18e114e4 --- /dev/null +++ b/tests/server/test_tools_dispatch_halt.py @@ -0,0 +1,273 @@ +"""Tests for dispatch_food_truck campaign halt enforcement gate.""" + +from __future__ import annotations + +import json +from pathlib import Path + +import pytest + +from autoskillit.fleet import FleetSemaphore +from tests.fakes import InMemoryHeadlessExecutor, InMemoryRecipeRepository +from tests.server._helpers import _make_recipe_info, _make_standard_recipe + +pytestmark = [pytest.mark.layer("server"), pytest.mark.medium, pytest.mark.feature("fleet")] + + +def _write_campaign_state(state_path: Path, dispatches: list[dict]) -> None: + """Write a minimal campaign state file with the given dispatch records.""" + import time + + state_path.parent.mkdir(parents=True, exist_ok=True) + state_path.write_text( + json.dumps( + { + "schema_version": 1, + "campaign_id": "test-campaign", + "campaign_name": "test", + "manifest_path": "/fake/manifest.yaml", + "started_at": time.time(), + "dispatches": dispatches, + "captured_values": {}, + } + ) + ) + + +class TestDispatchFoodTruckHaltEnforcement: + @pytest.fixture(autouse=True) + def _set_fleet_session(self, monkeypatch): + monkeypatch.setenv("AUTOSKILLIT_SESSION_TYPE", "fleet") + + @pytest.mark.anyio + async def test_dispatch_refuses_after_failure_when_halt_enabled( + self, tool_ctx_kitchen_open, monkeypatch, tmp_path + ): + """Prior FAILURE + continue_on_failure=false → FLEET_CAMPAIGN_HALTED.""" + state_path = tmp_path / "state.json" + _write_campaign_state(state_path, [{"name": "d1", "status": "failure"}]) + monkeypatch.setenv("AUTOSKILLIT_CAMPAIGN_STATE_PATH", str(state_path)) + monkeypatch.setenv("AUTOSKILLIT_CONTINUE_ON_FAILURE", "false") + + from autoskillit.server.tools.tools_fleet_dispatch import dispatch_food_truck + + result = json.loads(await dispatch_food_truck(recipe="r", task="t")) + assert result["success"] is False + assert result["error"] == "fleet_campaign_halted" + + @pytest.mark.anyio + async def test_dispatch_proceeds_after_failure_when_continue_enabled( + self, tool_ctx_kitchen_open, monkeypatch, tmp_path + ): + """dispatch_food_truck proceeds when continue_on_failure=true even with prior failure.""" + state_path = tmp_path / "state.json" + _write_campaign_state(state_path, [{"name": "d1", "status": "failure"}]) + monkeypatch.setenv("AUTOSKILLIT_CAMPAIGN_STATE_PATH", str(state_path)) + monkeypatch.setenv("AUTOSKILLIT_CONTINUE_ON_FAILURE", "true") + + self._setup_standard_dispatch(tool_ctx_kitchen_open, monkeypatch) + from autoskillit.server.tools.tools_fleet_dispatch import dispatch_food_truck + + result = json.loads(await dispatch_food_truck(recipe="test-recipe", task="t")) + assert "dispatch_id" in result + + @pytest.mark.anyio + async def test_dispatch_proceeds_when_no_campaign_state_path( + self, tool_ctx_kitchen_open, monkeypatch + ): + """Without AUTOSKILLIT_CAMPAIGN_STATE_PATH, halt check is skipped.""" + monkeypatch.delenv("AUTOSKILLIT_CAMPAIGN_STATE_PATH", raising=False) + + self._setup_standard_dispatch(tool_ctx_kitchen_open, monkeypatch) + from autoskillit.server.tools.tools_fleet_dispatch import dispatch_food_truck + + result = json.loads(await dispatch_food_truck(recipe="test-recipe", task="t")) + assert "dispatch_id" in result + + @pytest.mark.anyio + async def test_dispatch_proceeds_when_no_failures_in_state( + self, tool_ctx_kitchen_open, monkeypatch, tmp_path + ): + """dispatch_food_truck proceeds when all prior dispatches are SUCCESS.""" + state_path = tmp_path / "state.json" + _write_campaign_state(state_path, [{"name": "d1", "status": "success"}]) + monkeypatch.setenv("AUTOSKILLIT_CAMPAIGN_STATE_PATH", str(state_path)) + monkeypatch.setenv("AUTOSKILLIT_CONTINUE_ON_FAILURE", "false") + + self._setup_standard_dispatch(tool_ctx_kitchen_open, monkeypatch) + from autoskillit.server.tools.tools_fleet_dispatch import dispatch_food_truck + + result = json.loads(await dispatch_food_truck(recipe="test-recipe", task="t")) + assert "dispatch_id" in result + + @pytest.mark.anyio + async def test_dispatch_proceeds_when_state_file_missing( + self, tool_ctx_kitchen_open, monkeypatch, tmp_path + ): + """If state file doesn't exist, halt check is skipped (fail-open).""" + monkeypatch.setenv("AUTOSKILLIT_CAMPAIGN_STATE_PATH", str(tmp_path / "nonexistent.json")) + monkeypatch.setenv("AUTOSKILLIT_CONTINUE_ON_FAILURE", "false") + + self._setup_standard_dispatch(tool_ctx_kitchen_open, monkeypatch) + from autoskillit.server.tools.tools_fleet_dispatch import dispatch_food_truck + + result = json.loads(await dispatch_food_truck(recipe="test-recipe", task="t")) + assert "dispatch_id" in result + + @pytest.mark.anyio + async def test_campaign_state_write_uses_envelope_dispatch_status( + self, tool_ctx_kitchen_open, monkeypatch, tmp_path + ): + """Campaign-state write reads dispatch_status from envelope, not success flag. + + Mocks execute_dispatch to return an envelope with dispatch_status='resumable' + and success=False. Verifies the campaign-state write uses RESUMABLE, not FAILURE. + """ + from unittest.mock import AsyncMock + + from autoskillit.fleet import DispatchStatus, read_state, write_initial_state + + state_path = tmp_path / "campaign_state.json" + monkeypatch.setenv("AUTOSKILLIT_CAMPAIGN_STATE_PATH", str(state_path)) + monkeypatch.setenv("AUTOSKILLIT_CONTINUE_ON_FAILURE", "false") + + write_initial_state(state_path, "cid", "camp", "/m.yaml", []) + + resumable_envelope = json.dumps( + { + "success": False, + "dispatch_status": "resumable", + "dispatch_id": "test-dispatch-id", + "dispatched_session_id": "sess-abc", + "reason": "fleet_l3_no_result_block", + "token_usage": None, + "l3_parse_source": "stdout", + "lifespan_started": True, + "l3_payload": None, + } + ) + monkeypatch.setattr( + "autoskillit.fleet.execute_dispatch", + AsyncMock(return_value=resumable_envelope), + ) + + from autoskillit.server.tools.tools_fleet_dispatch import dispatch_food_truck + + self._setup_standard_dispatch(tool_ctx_kitchen_open, monkeypatch) + await dispatch_food_truck(recipe="test-recipe", task="t", dispatch_name="d1") + + state = read_state(state_path) + assert state is not None + written = next( + (d for d in state.dispatches if d.name == "d1" and d.status != DispatchStatus.PENDING), + None, + ) + assert written is not None, "No campaign dispatch record was written for d1" + assert written.status == DispatchStatus.RESUMABLE, ( + f"Expected RESUMABLE but got {written.status}" + ) + + @pytest.mark.anyio + async def test_no_result_block_failure_does_not_halt_campaign( + self, tool_ctx_kitchen_open, monkeypatch, tmp_path + ): + """dispatch_food_truck proceeds when prior FAILURE has fleet_l3_no_result_block reason.""" + state_path = tmp_path / "state.json" + _write_campaign_state( + state_path, + [{"name": "d1", "status": "failure", "reason": "fleet_l3_no_result_block"}], + ) + monkeypatch.setenv("AUTOSKILLIT_CAMPAIGN_STATE_PATH", str(state_path)) + monkeypatch.setenv("AUTOSKILLIT_CONTINUE_ON_FAILURE", "false") + + self._setup_standard_dispatch(tool_ctx_kitchen_open, monkeypatch) + from autoskillit.server.tools.tools_fleet_dispatch import dispatch_food_truck + + result = json.loads(await dispatch_food_truck(recipe="test-recipe", task="t")) + assert "dispatch_id" in result + + def _setup_standard_dispatch(self, tool_ctx, monkeypatch): + """Wire tool_ctx for a successful standard dispatch.""" + tool_ctx.fleet_lock = FleetSemaphore(max_concurrent=1) + repo = InMemoryRecipeRepository() + recipe_info = _make_recipe_info("test-recipe") + repo.add_recipe("test-recipe", recipe_info) + repo.add_full_recipe(recipe_info.path, _make_standard_recipe("test-recipe")) + tool_ctx.recipes = repo + tool_ctx.executor = InMemoryHeadlessExecutor() + + +class TestDispatchFoodTruckRetryOnFailure: + @pytest.fixture(autouse=True) + def _set_fleet_session(self, monkeypatch): + monkeypatch.setenv("AUTOSKILLIT_SESSION_TYPE", "fleet") + + @pytest.mark.anyio + async def test_dispatch_resets_and_proceeds_when_retrying_failed_dispatch( + self, tool_ctx_kitchen_open, monkeypatch, tmp_path + ): + """dispatch_name matches failed dispatch → reset to PENDING, proceed.""" + state_path = tmp_path / "state.json" + _write_campaign_state( + state_path, [{"name": "d1", "status": "failure", "reason": "task_failed"}] + ) + monkeypatch.setenv("AUTOSKILLIT_CAMPAIGN_STATE_PATH", str(state_path)) + monkeypatch.setenv("AUTOSKILLIT_CONTINUE_ON_FAILURE", "false") + + self._setup_standard_dispatch(tool_ctx_kitchen_open, monkeypatch) + from autoskillit.server.tools.tools_fleet_dispatch import dispatch_food_truck + + result = json.loads( + await dispatch_food_truck(recipe="test-recipe", task="t", dispatch_name="d1") + ) + assert "dispatch_id" in result, "Expected dispatch to run, not be blocked by fleet halt" + + @pytest.mark.anyio + async def test_dispatch_halts_when_different_dispatch_has_failure( + self, tool_ctx_kitchen_open, monkeypatch, tmp_path + ): + """dispatch_name does NOT match failed dispatch → still halts.""" + state_path = tmp_path / "state.json" + _write_campaign_state( + state_path, [{"name": "d1", "status": "failure", "reason": "task_failed"}] + ) + monkeypatch.setenv("AUTOSKILLIT_CAMPAIGN_STATE_PATH", str(state_path)) + monkeypatch.setenv("AUTOSKILLIT_CONTINUE_ON_FAILURE", "false") + + self._setup_standard_dispatch(tool_ctx_kitchen_open, monkeypatch) + from autoskillit.server.tools.tools_fleet_dispatch import dispatch_food_truck + + result = json.loads( + await dispatch_food_truck(recipe="test-recipe", task="t", dispatch_name="d2") + ) + assert result["success"] is False + assert result["error"] == "fleet_campaign_halted" + + @pytest.mark.anyio + async def test_dispatch_halts_when_no_dispatch_name_provided( + self, tool_ctx_kitchen_open, monkeypatch, tmp_path + ): + """No dispatch_name + prior failure → halts (current behavior).""" + state_path = tmp_path / "state.json" + _write_campaign_state( + state_path, [{"name": "d1", "status": "failure", "reason": "task_failed"}] + ) + monkeypatch.setenv("AUTOSKILLIT_CAMPAIGN_STATE_PATH", str(state_path)) + monkeypatch.setenv("AUTOSKILLIT_CONTINUE_ON_FAILURE", "false") + + self._setup_standard_dispatch(tool_ctx_kitchen_open, monkeypatch) + from autoskillit.server.tools.tools_fleet_dispatch import dispatch_food_truck + + result = json.loads(await dispatch_food_truck(recipe="test-recipe", task="t")) + assert result["success"] is False + assert result["error"] == "fleet_campaign_halted" + + def _setup_standard_dispatch(self, tool_ctx, monkeypatch): + """Wire tool_ctx for a successful standard dispatch.""" + tool_ctx.fleet_lock = FleetSemaphore(max_concurrent=1) + repo = InMemoryRecipeRepository() + recipe_info = _make_recipe_info("test-recipe") + repo.add_recipe("test-recipe", recipe_info) + repo.add_full_recipe(recipe_info.path, _make_standard_recipe("test-recipe")) + tool_ctx.recipes = repo + tool_ctx.executor = InMemoryHeadlessExecutor() diff --git a/tests/server/test_tools_execution.py b/tests/server/test_tools_execution.py deleted file mode 100644 index ae765e54be..0000000000 --- a/tests/server/test_tools_execution.py +++ /dev/null @@ -1,1524 +0,0 @@ -"""Tests for autoskillit server execution tools.""" - -from __future__ import annotations - -import json -from unittest.mock import AsyncMock - -import pytest -import structlog.contextvars -import structlog.testing - -from autoskillit.config import ( - AutomationConfig, - RunSkillConfig, -) -from autoskillit.core.types import ( - ChannelConfirmation, - RetryReason, -) -from autoskillit.execution.commands import _inject_completion_directive -from autoskillit.execution.headless import _session_log_dir -from autoskillit.server.helpers import ( - _check_dry_walkthrough, -) -from autoskillit.server.tools_execution import run_skill -from tests.conftest import _make_result - -_SUCCESS_JSON = ( - '{"type": "result", "subtype": "success", "is_error": false,' - ' "result": "done", "session_id": "s1"}' -) - -# Deterministic UUID for tests that need to predict the per-invocation marker. -_DETERMINISTIC_HEX = "a1b2c3d4e5f6a7b890123456" -_DETERMINISTIC_MARKER = f"%%ORDER_UP::{_DETERMINISTIC_HEX[:8]}%%" - - -class _FixedUUID: - hex = _DETERMINISTIC_HEX - - -def _patch_uuid4(monkeypatch): - """Monkeypatch uuid4 to return a deterministic value for marker prediction.""" - monkeypatch.setattr("uuid.uuid4", lambda: _FixedUUID()) - - -class TestRunSkillPluginDir: - """T2: run_skill passes --plugin-dir to the claude command.""" - - @pytest.mark.anyio - async def test_run_skill_passes_plugin_dir(self, tool_ctx): - """run_skill includes --plugin-dir and the plugin_dir from tool_ctx in the command.""" - tool_ctx.runner.push( - _make_result( - 0, - '{"type": "result", "subtype": "success", "is_error": false,' - ' "result": "done", "session_id": "s1"}', - "", - ) - ) - await run_skill("/investigate some-error", "/tmp") - - cmd = tool_ctx.runner.call_args_list[0][0] - assert "--plugin-dir" in cmd - plugin_dir_idx = cmd.index("--plugin-dir") - assert cmd[plugin_dir_idx + 1] == tool_ctx.plugin_dir - # --output-format and stream-json must be present - assert "--output-format" in cmd - assert cmd[cmd.index("--output-format") + 1] == "stream-json" - # cwd must propagate to the subprocess runner - from pathlib import Path - - actual_cwd = tool_ctx.runner.call_args_list[0][1] - assert actual_cwd == Path("/tmp"), f"Subprocess cwd mismatch: {actual_cwd} != /tmp" - - -class TestCheckDryWalkthrough: - """Dry-walkthrough gate blocks both /autoskillit:implement-worktree variants.""" - - def test_dry_walkthrough_gate_blocks_implement_no_merge(self, tool_ctx, tmp_path): - """Gate blocks /autoskillit:implement-worktree-no-merge when plan lacks marker.""" - plan = tmp_path / "plan.md" - plan.write_text("# My Plan\n\nSome content") - result = _check_dry_walkthrough( - f"/autoskillit:implement-worktree-no-merge {plan}", str(tmp_path) - ) - assert result is not None - parsed = json.loads(result) - assert parsed["success"] is False - assert parsed["is_error"] is True - assert "dry-walked" in parsed["result"].lower() - - def test_dry_walkthrough_gate_passes_implement_no_merge(self, tool_ctx, tmp_path): - """Gate allows /autoskillit:implement-worktree-no-merge when plan has marker.""" - plan = tmp_path / "plan.md" - plan.write_text("Dry-walkthrough verified = TRUE\n# My Plan") - result = _check_dry_walkthrough( - f"/autoskillit:implement-worktree-no-merge {plan}", str(tmp_path) - ) - assert result is None - - def test_dry_walkthrough_gate_still_works_for_implement_worktree(self, tool_ctx, tmp_path): - """Original /autoskillit:implement-worktree gating is not broken.""" - plan = tmp_path / "plan.md" - plan.write_text("# No marker plan") - result = _check_dry_walkthrough(f"/autoskillit:implement-worktree {plan}", str(tmp_path)) - assert result is not None - parsed = json.loads(result) - assert parsed["success"] is False - assert parsed["is_error"] is True - - def test_dry_walkthrough_gate_ignores_unrelated_skills(self, tool_ctx): - """Gate ignores skills that are not implement-worktree variants.""" - result = _check_dry_walkthrough("/autoskillit:investigate some-error", "/tmp") - assert result is None - - def test_dry_walkthrough_gate_with_part_a_named_file_marked(self, tmp_path, tool_ctx): - """Gate accepts _part_a.md file when marker is present.""" - plan = tmp_path / "task_plan_2026-01-01_part_a.md" - plan.write_text("Dry-walkthrough verified = TRUE\n\nContent here") - result = _check_dry_walkthrough( - f"/autoskillit:implement-worktree-no-merge {plan}", str(tmp_path) - ) - assert result is None - - def test_dry_walkthrough_gate_with_part_b_named_file_unmarked(self, tmp_path, tool_ctx): - """Gate blocks _part_b.md file when marker is absent.""" - plan = tmp_path / "task_plan_2026-01-01_part_b.md" - plan.write_text("> **PART B ONLY.**\n\nNo walkthrough marker here") - result = _check_dry_walkthrough( - f"/autoskillit:implement-worktree-no-merge {plan}", str(tmp_path) - ) - assert result is not None - parsed = json.loads(result) - assert parsed["subtype"] == "gate_error" - - def test_dry_walkthrough_gate_distinguishes_parts_independently(self, tmp_path, tool_ctx): - """Gate correctly distinguishes marked part_a from unmarked part_b.""" - part_a = tmp_path / "task_plan_part_a.md" - part_b = tmp_path / "task_plan_part_b.md" - part_a.write_text("Dry-walkthrough verified = TRUE\n\nPart A content") - part_b.write_text("> **PART B ONLY.**\n\nPart B content — no marker") - - result_a = _check_dry_walkthrough( - f"/autoskillit:implement-worktree-no-merge {part_a}", str(tmp_path) - ) - result_b = _check_dry_walkthrough( - f"/autoskillit:implement-worktree-no-merge {part_b}", str(tmp_path) - ) - assert result_a is None - assert result_b is not None - assert json.loads(result_b)["subtype"] == "gate_error" - - -class TestRunSkillPrefix: - """run_skill passes prefixed command to subprocess.""" - - @pytest.mark.anyio - async def test_run_skill_prefixes_skill_command(self, tool_ctx): - tool_ctx.runner.push( - _make_result( - 0, - '{"type": "result", "subtype": "success", "is_error": false, ' - '"result": "done", "session_id": "s1"}', - "", - ) - ) - await run_skill("/investigate error", "/tmp") - cmd = tool_ctx.runner.call_args_list[0][0] - prompt_idx = cmd.index("--print") + 1 if "--print" in cmd else cmd.index("-p") + 1 - assert cmd[prompt_idx].startswith("Use /investigate error") - # cwd must propagate to the subprocess runner - from pathlib import Path - - actual_cwd = tool_ctx.runner.call_args_list[0][1] - assert actual_cwd == Path("/tmp"), f"Subprocess cwd mismatch: {actual_cwd} != /tmp" - - @pytest.mark.anyio - async def test_run_skill_rejects_prose_without_slash(self, tool_ctx): - """FRICT-6-1: prose command without slash returns gate_error before reaching executor.""" - result = json.loads(await run_skill("Fix the authentication bug in main.py", "/tmp")) - assert result["success"] is False - assert result["is_error"] is True - assert result["subtype"] == "gate_error" - # executor must NOT have been called - assert tool_ctx.runner.call_args_list == [] - - @pytest.mark.anyio - async def test_run_skill_rejects_empty_skill_command(self, tool_ctx): - """FRICT-6-1: empty string returns gate_error without hitting executor.""" - result = json.loads(await run_skill("", "/tmp")) - assert result["success"] is False - assert result["is_error"] is True - assert result["subtype"] == "gate_error" - assert tool_ctx.runner.call_args_list == [] - - @pytest.mark.anyio - async def test_run_skill_rejects_whitespace_only(self, tool_ctx): - """FRICT-6-1: whitespace-only command returns gate_error (strip before check).""" - result = json.loads(await run_skill(" ", "/tmp")) - assert result["success"] is False - assert result["is_error"] is True - assert result["subtype"] == "gate_error" - assert tool_ctx.runner.call_args_list == [] - - @pytest.mark.anyio - async def test_run_skill_format_error_includes_slash_examples(self, tool_ctx): - """FRICT-6-1: error message for invalid format includes concrete slash-command examples.""" - result = json.loads(await run_skill("investigate this bug", "/tmp")) - assert result["success"] is False - assert "/autoskillit:" in result["result"] - assert "/" in result["result"] - - @pytest.mark.anyio - async def test_run_skill_includes_completion_directive(self, tool_ctx): - tool_ctx.runner.push( - _make_result( - 0, - '{"type": "result", "subtype": "success", "is_error": false, ' - '"result": "done", "session_id": "s1"}', - "", - ) - ) - await run_skill("/investigate error", "/tmp") - cmd = tool_ctx.runner.call_args_list[0][0] - prompt_idx = cmd.index("--print") + 1 if "--print" in cmd else cmd.index("-p") + 1 - assert "%%ORDER_UP::" in cmd[prompt_idx] - # cwd must propagate to the subprocess runner - from pathlib import Path - - actual_cwd = tool_ctx.runner.call_args_list[0][1] - assert actual_cwd == Path("/tmp"), f"Subprocess cwd mismatch: {actual_cwd} != /tmp" - - -class TestValidateSkillCommand: - """Unit tests for _validate_skill_command helper.""" - - def test_returns_none_for_slash_command(self, tool_ctx): - from autoskillit.server.helpers import _validate_skill_command - - assert _validate_skill_command("/autoskillit:investigate") is None - - def test_returns_none_for_bare_slash_command(self, tool_ctx): - from autoskillit.server.helpers import _validate_skill_command - - assert _validate_skill_command("/audit-arch") is None - - def test_returns_error_json_for_prose(self, tool_ctx): - from autoskillit.server.helpers import _validate_skill_command - - result = _validate_skill_command("Fix the bug") - assert result is not None - parsed = json.loads(result) - assert parsed["success"] is False - assert parsed["subtype"] == "gate_error" - - def test_returns_error_json_for_empty_string(self, tool_ctx): - from autoskillit.server.helpers import _validate_skill_command - - result = _validate_skill_command("") - assert result is not None - - def test_strips_whitespace_before_check(self, tool_ctx): - from autoskillit.server.helpers import _validate_skill_command - - # Leading whitespace before slash → valid - assert _validate_skill_command(" /autoskillit:investigate") is None - # Leading whitespace before prose → invalid - result = _validate_skill_command(" investigate bug") - assert result is not None - - -class TestDryWalkthroughGateWithPrefix: - """Dry-walkthrough gate still receives raw command before prefix is applied.""" - - @pytest.mark.anyio - async def test_gate_still_fires_for_implement_skill(self, tool_ctx, tmp_path): - plan = tmp_path / "plan.md" - plan.write_text("# No marker plan") - result = json.loads( - await run_skill(f"/autoskillit:implement-worktree {plan}", str(tmp_path)) - ) - assert result["success"] is False - assert result["is_error"] is True - assert "dry-walked" in result["result"].lower() - - -class TestRunSkillTimeoutFromConfig: - """run_skill uses configurable timeouts.""" - - @pytest.mark.anyio - async def test_run_skill_timeout_from_config(self, tool_ctx): - """run_skill uses _config.run_skill.timeout instead of hardcoded value.""" - cfg = AutomationConfig() - cfg.run_skill = RunSkillConfig(timeout=120) - cfg.safety.require_dry_walkthrough = False - tool_ctx.config = cfg - - tool_ctx.runner.push( - _make_result( - 0, - '{"type": "result", "subtype": "success", "is_error": false,' - ' "result": "done", "session_id": "s1"}', - "", - ) - ) - await run_skill("/investigate foo", "/tmp") - - assert tool_ctx.runner.call_args_list[-1][2] == 120.0 - - -class TestRunSkillInjectsCompletionDirective: - """run_skill injects completion directive into the skill command.""" - - @pytest.mark.anyio - async def test_run_skill_injects_completion_directive(self, tool_ctx): - """Skill command passed to claude -p contains the completion marker instruction.""" - cfg = AutomationConfig() - cfg.safety.require_dry_walkthrough = False - tool_ctx.config = cfg - - tool_ctx.runner.push( - _make_result( - 0, - '{"type": "result", "subtype": "success", "is_error": false,' - ' "result": "done", "session_id": "s1"}', - "", - ) - ) - await run_skill("/investigate foo", "/tmp") - - cmd = tool_ctx.runner.call_args_list[-1][0] - prompt_idx = cmd.index("--print") + 1 if "--print" in cmd else cmd.index("-p") + 1 - skill_arg = cmd[prompt_idx] - assert "%%ORDER_UP::" in skill_arg - assert "ORCHESTRATION DIRECTIVE" in skill_arg - - def test_inject_completion_directive_prohibits_standalone_marker(self): - """ - The directive wording must explicitly instruct the model to emit the marker - in the SAME message as its substantive output, not as a standalone message. - This prevents the model from interpreting the directive as a post-task acknowledgment. - """ - result = _inject_completion_directive("/audit-impl", "%%ORDER_UP%%") - lowered = result.lower() - assert ( - "same message" in lowered - or "not as a separate" in lowered - or ("standalone" in lowered and "not" in lowered) - ), f"Directive must prohibit standalone marker emission. Got: {result!r}" - - -class TestRunSkillEnvPrefix: - """run_skill always injects AUTOSKILLIT_HEADLESS=1 and optionally CLAUDE_CODE_EXIT_AFTER_STOP_DELAY via the env kwarg.""" # noqa: E501 - - @pytest.mark.anyio - async def test_default_delay_populates_env(self, tool_ctx): - tool_ctx.runner.push(_make_result(0, _SUCCESS_JSON, "")) - await run_skill("/investigate something", "/tmp") - cmd, _cwd, _timeout, kwargs = tool_ctx.runner.call_args_list[0] - assert cmd[0] == "claude" - env = kwargs["env"] - assert env["AUTOSKILLIT_HEADLESS"] == "1" - assert env["CLAUDE_CODE_EXIT_AFTER_STOP_DELAY"] == "2000" - - @pytest.mark.anyio - async def test_zero_delay_omits_delay_env_var(self, tool_ctx): - cfg = AutomationConfig() - cfg.run_skill = RunSkillConfig(exit_after_stop_delay_ms=0) - cfg.safety.require_dry_walkthrough = False - tool_ctx.config = cfg - tool_ctx.runner.push(_make_result(0, _SUCCESS_JSON, "")) - await run_skill("/investigate something", "/tmp") - cmd, _cwd, _timeout, kwargs = tool_ctx.runner.call_args_list[0] - assert cmd[0] == "claude" - env = kwargs["env"] - assert env["AUTOSKILLIT_HEADLESS"] == "1" - assert "CLAUDE_CODE_EXIT_AFTER_STOP_DELAY" not in env - - @pytest.mark.anyio - async def test_custom_delay_value_in_env(self, tool_ctx): - cfg = AutomationConfig() - cfg.run_skill = RunSkillConfig( - exit_after_stop_delay_ms=60000, natural_exit_grace_seconds=61.0 - ) - cfg.safety.require_dry_walkthrough = False - tool_ctx.config = cfg - tool_ctx.runner.push(_make_result(0, _SUCCESS_JSON, "")) - await run_skill("/investigate something", "/tmp") - cmd, _cwd, _timeout, kwargs = tool_ctx.runner.call_args_list[0] - assert cmd[0] == "claude" - env = kwargs["env"] - assert env["AUTOSKILLIT_HEADLESS"] == "1" - assert env["CLAUDE_CODE_EXIT_AFTER_STOP_DELAY"] == "60000" - - -class TestRunSkillPassesSessionLogDir: - """run_skill passes session_log_dir derived from cwd.""" - - @pytest.mark.anyio - async def test_run_skill_passes_session_log_dir(self, tool_ctx): - """runner receives session_log_dir derived from cwd.""" - cfg = AutomationConfig() - cfg.safety.require_dry_walkthrough = False - tool_ctx.config = cfg - - tool_ctx.runner.push( - _make_result( - 0, - '{"type": "result", "subtype": "success", "is_error": false,' - ' "result": "done", "session_id": "s1"}', - "", - ) - ) - await run_skill("/investigate foo", "/some/project") - - call_kwargs = tool_ctx.runner.call_args_list[-1][3] - expected_dir = _session_log_dir("/some/project") - assert call_kwargs["session_log_dir"] == expected_dir - assert "-some-project" in str(expected_dir) - - -class TestGateErrorSchemaNormalization: - """Gate errors use the standard 9-field response schema.""" - - def test_require_enabled_gate_returns_standard_schema(self, tool_ctx): - """Gate errors must use the same schema as normal responses.""" - from autoskillit.pipeline.gate import DefaultGateState - from autoskillit.server.helpers import _require_enabled - - tool_ctx.gate = DefaultGateState(enabled=False) - gate_result = _require_enabled() - assert gate_result is not None - response = json.loads(gate_result) - assert response["success"] is False - assert response["is_error"] is True - assert response["needs_retry"] is False - assert "result" in response - - def test_dry_walkthrough_gate_returns_standard_schema(self, tool_ctx, tmp_path): - """Dry-walkthrough gate errors must use the standard response schema.""" - plan = tmp_path / "plan.md" - plan.write_text("No marker here") - skill_cmd = f"/autoskillit:implement-worktree {plan}" - result = _check_dry_walkthrough(skill_cmd, str(tmp_path)) - assert result is not None - response = json.loads(result) - assert response["success"] is False - assert response["is_error"] is True - assert response["subtype"] == "gate_error" - - -class TestRunSkillFailurePaths: - """run_skill surfaces session outcome on failure.""" - - @pytest.mark.anyio - async def test_returns_subtype_on_incomplete_session(self, tool_ctx): - """run_skill includes subtype when session didn't finish.""" - stdout = json.dumps( - { - "type": "result", - "subtype": "error_max_turns", - "is_error": False, - "session_id": "s1", - } - ) - tool_ctx.runner.push(_make_result(1, stdout, "")) - result = json.loads(await run_skill("/investigate error", "/tmp")) - assert result["session_id"] == "s1" - assert result["subtype"] == "error_max_turns" - - @pytest.mark.anyio - async def test_returns_is_error_on_context_limit(self, tool_ctx): - """run_skill includes is_error when context limit is hit.""" - stdout = json.dumps( - { - "type": "result", - "subtype": "success", - "is_error": True, - "result": "Prompt is too long", - "session_id": "s1", - } - ) - tool_ctx.runner.push(_make_result(1, stdout, "")) - result = json.loads(await run_skill("/investigate error", "/tmp")) - assert result["is_error"] is True - assert result["subtype"] == "missing_completion_marker" - assert result["cli_subtype"] == "success" - - @pytest.mark.anyio - async def test_handles_empty_stdout(self, tool_ctx): - """run_skill returns error result when stdout is empty.""" - tool_ctx.runner.push( - _make_result(1, "", "segfault", channel_confirmation=ChannelConfirmation.UNMONITORED) - ) - result = json.loads(await run_skill("/investigate error", "/tmp")) - assert result["exit_code"] == 1 - assert result["is_error"] is True - assert result["subtype"] == "empty_output" - assert result["success"] is False - - @pytest.mark.anyio - async def test_empty_stdout_exit_zero_is_retriable(self, tool_ctx): - """Infrastructure failure (empty stdout, exit 0) is retriable with stderr.""" - tool_ctx.runner.push( - _make_result( - 0, "", "session dropped", channel_confirmation=ChannelConfirmation.UNMONITORED - ) - ) - result = json.loads(await run_skill("/investigate error", "/tmp")) - assert result["subtype"] == "empty_output" - assert result["success"] is False - assert result["needs_retry"] is True - assert result["retry_reason"] == RetryReason.EMPTY_OUTPUT - assert result["stderr"] == "session dropped" - - -class TestRunSkillModel: - """Tests for model parameter in run_skill.""" - - _MOCK_STDOUT = ( - '{"type": "result", "subtype": "success", "is_error": false, ' - '"result": "done", "session_id": "s1"}' - ) - - # MOD_S1 - @pytest.mark.anyio - async def test_run_skill_passes_model_flag(self, tool_ctx): - tool_ctx.runner.push(_make_result(0, self._MOCK_STDOUT, "")) - await run_skill("/investigate error", "/tmp", model="sonnet") - cmd = tool_ctx.runner.call_args_list[0][0] - assert "--model" in cmd - assert cmd[cmd.index("--model") + 1] == "sonnet" - - # MOD_S3 - @pytest.mark.anyio - async def test_run_skill_no_model_flag_when_empty(self, tool_ctx): - tool_ctx.config.model.default = "" # ← add this line - tool_ctx.runner.push(_make_result(0, self._MOCK_STDOUT, "")) - await run_skill("/investigate error", "/tmp", model="") - cmd = tool_ctx.runner.call_args_list[0][0] - assert "--model" not in cmd - - -class TestRunSkillStepName: - """step_name param drives token_log accumulation.""" - - def _make_ndjson(self) -> str: - result_rec = json.dumps( - { - "type": "result", - "subtype": "success", - "is_error": False, - "result": "Task complete.", - "session_id": "sess-abc", - "usage": { - "input_tokens": 200, - "output_tokens": 80, - "cache_creation_input_tokens": 8, - "cache_read_input_tokens": 3, - }, - } - ) - return result_rec - - @pytest.mark.anyio - async def test_step_name_records_token_usage(self, tool_ctx): - tool_ctx.runner.push(_make_result(returncode=0, stdout=self._make_ndjson())) - await run_skill( - skill_command="/autoskillit:investigate topic", cwd="/tmp", step_name="plan" - ) - report = tool_ctx.token_log.get_report() - assert len(report) == 1 - assert report[0]["step_name"] == "plan" - assert report[0]["input_tokens"] == 200 - - @pytest.mark.anyio - async def test_no_step_name_does_not_record(self, tool_ctx): - tool_ctx.runner.push(_make_result(returncode=0, stdout=self._make_ndjson())) - await run_skill(skill_command="/autoskillit:investigate topic", cwd="/tmp", step_name="") - assert tool_ctx.token_log.get_report() == [] - - @pytest.mark.anyio - async def test_null_token_usage_does_not_record(self, tool_ctx): - # Return NDJSON with no usage field → token_usage will be null - no_usage_ndjson = json.dumps( - { - "type": "result", - "subtype": "success", - "is_error": False, - "result": "done", - "session_id": "s1", - } - ) - tool_ctx.runner.push(_make_result(returncode=0, stdout=no_usage_ndjson)) - await run_skill( - skill_command="/autoskillit:investigate topic", cwd="/tmp", step_name="plan" - ) - assert tool_ctx.token_log.get_report() == [] - - @pytest.mark.anyio - async def test_step_name_run_skill_long_running(self, tool_ctx): - """run_skill accumulates token usage by step_name (run_skill_retry test replacement).""" - tool_ctx.runner.push(_make_result(returncode=0, stdout=self._make_ndjson())) - await run_skill( - skill_command="/autoskillit:investigate the test failures", - cwd="/tmp", - step_name="implement", - ) - report = tool_ctx.token_log.get_report() - assert len(report) == 1 - assert report[0]["step_name"] == "implement" - assert report[0]["input_tokens"] == 200 - - -class TestGatedToolObservability: - """Each gated tool binds structlog contextvars and calls ctx.info/ctx.error.""" - - @pytest.fixture - def mock_ctx(self): - """AsyncMock ctx for verifying ctx.info/ctx.error calls.""" - ctx = AsyncMock() - ctx.info = AsyncMock() - ctx.error = AsyncMock() - return ctx - - @pytest.mark.anyio - async def test_run_skill_binds_tool_contextvar_and_calls_ctx_info(self, tool_ctx, mock_ctx): - """run_skill binds tool='run_skill' contextvar and calls ctx.info on success.""" - tool_ctx.runner.push( - _make_result( - 0, - '{"type": "result", "subtype": "success", "is_error": false,' - ' "result": "done", "session_id": "s1"}', - "", - ) - ) - with structlog.testing.capture_logs( - processors=[structlog.contextvars.merge_contextvars] - ) as logs: - await run_skill("/autoskillit:investigate task", "/tmp", ctx=mock_ctx) - assert any(entry.get("tool") == "run_skill" for entry in logs) - - @pytest.mark.anyio - async def test_run_skill_returns_failure_result_on_error_output(self, tool_ctx, mock_ctx): - """run_skill reports failure (success=false) when headless session fails.""" - tool_ctx.runner.push( - _make_result( - 1, - '{"type": "result", "subtype": "error", "is_error": true,' - ' "result": "failed", "session_id": "s1"}', - "", - channel_confirmation=ChannelConfirmation.UNMONITORED, - ) - ) - result = json.loads(await run_skill("/autoskillit:investigate task", "/tmp", ctx=mock_ctx)) - assert result["success"] is False - - -class TestNotifyHelper: - """Unit tests for the centralized _notify() notification helper.""" - - @pytest.mark.anyio - async def test_notify_raises_value_error_for_reserved_key_name(self): - """The 'name' key that caused the original bug must be rejected.""" - from autoskillit.server.helpers import _notify - - ctx = AsyncMock() - ctx.info = AsyncMock() - with pytest.raises(ValueError, match="reserved LogRecord"): - await _notify( - ctx, - "info", - "migrate_recipe: foo", - "autoskillit.migrate_recipe", - extra={"name": "foo"}, - ) - ctx.info.assert_not_awaited() - - @pytest.mark.anyio - async def test_notify_raises_for_all_reserved_keys(self): - """Every key in RESERVED_LOG_RECORD_KEYS must be rejected.""" - from autoskillit.core.types import RESERVED_LOG_RECORD_KEYS - from autoskillit.server.helpers import _notify - - ctx = AsyncMock() - ctx.info = AsyncMock() - for reserved_key in RESERVED_LOG_RECORD_KEYS: - with pytest.raises(ValueError, match="reserved LogRecord"): - await _notify(ctx, "info", "msg", "logger", extra={reserved_key: "value"}) - - @pytest.mark.anyio - async def test_notify_accepts_safe_key_recipe_name(self): - """'recipe_name' (the corrected key for migrate_recipe) must be accepted.""" - from autoskillit.server.helpers import _notify - - ctx = AsyncMock() - ctx.info = AsyncMock() - await _notify( - ctx, - "info", - "migrate_recipe: foo", - "autoskillit.migrate_recipe", - extra={"recipe_name": "foo"}, - ) - ctx.info.assert_awaited_once_with( - "migrate_recipe: foo", - logger_name="autoskillit.migrate_recipe", - extra={"recipe_name": "foo"}, - ) - - @pytest.mark.anyio - async def test_notify_accepts_none_extra(self): - from autoskillit.server.helpers import _notify - - ctx = AsyncMock() - ctx.info = AsyncMock() - await _notify(ctx, "info", "msg", "logger") # no extra - ctx.info.assert_awaited_once() - - @pytest.mark.anyio - async def test_notify_accepts_empty_extra(self): - from autoskillit.server.helpers import _notify - - ctx = AsyncMock() - ctx.info = AsyncMock() - await _notify(ctx, "info", "msg", "logger", extra={}) - ctx.info.assert_awaited_once() - - @pytest.mark.anyio - async def test_notify_swallows_attribute_error_from_ctx(self): - """Contract: must not raise even when ctx.info raises AttributeError - (e.g. _CurrentContext sentinel). Test completion is the assertion.""" - from autoskillit.server.helpers import _notify - - ctx = AsyncMock() - ctx.info = AsyncMock(side_effect=AttributeError("no info")) - # Must not raise - await _notify(ctx, "info", "msg", "logger", extra={"cwd": "/tmp"}) - - @pytest.mark.anyio - async def test_notify_swallows_runtime_error_from_ctx(self): - """Contract: must not raise even when ctx.info raises RuntimeError - (no active MCP session). Test completion is the assertion.""" - from autoskillit.server.helpers import _notify - - ctx = AsyncMock() - ctx.info = AsyncMock(side_effect=RuntimeError("session not available")) - await _notify(ctx, "info", "msg", "logger", extra={"cwd": "/tmp"}) - - @pytest.mark.anyio - async def test_notify_swallows_key_error_from_ctx(self): - """Contract: must not raise even when ctx.info raises KeyError - (FastMCP stdlib logging path). Test completion is the assertion.""" - from autoskillit.server.helpers import _notify - - ctx = AsyncMock() - ctx.info = AsyncMock(side_effect=KeyError("Attempt to overwrite 'name' in LogRecord")) - await _notify(ctx, "info", "msg", "logger", extra={"cwd": "/tmp"}) - - @pytest.mark.anyio - async def test_notify_dispatches_error_level(self): - from autoskillit.server.helpers import _notify - - ctx = AsyncMock() - ctx.error = AsyncMock() - await _notify( - ctx, - "error", - "run_cmd failed", - "autoskillit.run_cmd", - extra={"exit_code": 1}, - ) - ctx.error.assert_awaited_once_with( - "run_cmd failed", - logger_name="autoskillit.run_cmd", - extra={"exit_code": 1}, - ) - - -@pytest.mark.anyio -async def test_tools_execution_routes_through_executor(tool_ctx, monkeypatch) -> None: - """run_skill routes through ctx.executor.run(), not run_headless_core directly.""" - from autoskillit.core import SkillResult - - calls = [] - - class MockExecutor: - async def run( - self, - skill_command: str, - cwd: str, - *, - model: str = "", - step_name: str = "", - add_dirs=(), - kitchen_id: str = "", - order_id: str = "", - timeout: float | None = None, - stale_threshold: float | None = None, - idle_output_timeout: float | None = None, - expected_output_patterns: tuple[str, ...] | list[str] = (), - write_behavior=None, - completion_marker: str = "", - ) -> SkillResult: - calls.append((skill_command, cwd)) - return SkillResult( - success=True, - result="ok", - session_id="", - subtype="success", - is_error=False, - exit_code=0, - needs_retry=False, - retry_reason="none", - stderr="", - token_usage=None, - ) - - tool_ctx.executor = MockExecutor() - monkeypatch.setattr("autoskillit.server._ctx", tool_ctx) - - from autoskillit.server.tools_execution import run_skill - - await run_skill("/test skill", "/tmp") - assert calls == [("/test skill", "/tmp")] - - -@pytest.mark.anyio -async def test_run_skill_passes_validated_add_dirs(tool_ctx, monkeypatch) -> None: - """run_skill passes ValidatedAddDir instances (not raw strings) as add_dirs.""" - from autoskillit.core import SkillResult, ValidatedAddDir - - captured: dict = {} - - class MockExecutor: - async def run( - self, - skill_command: str, - cwd: str, - *, - model: str = "", - step_name: str = "", - add_dirs=(), - kitchen_id: str = "", - order_id: str = "", - timeout: float | None = None, - stale_threshold: float | None = None, - idle_output_timeout: float | None = None, - expected_output_patterns: tuple[str, ...] | list[str] = (), - write_behavior=None, - completion_marker: str = "", - ) -> SkillResult: - captured["add_dirs"] = add_dirs - captured["cwd"] = cwd - return SkillResult( - success=True, - result="ok", - session_id="", - subtype="success", - is_error=False, - exit_code=0, - needs_retry=False, - retry_reason="none", - stderr="", - token_usage=None, - ) - - tool_ctx.executor = MockExecutor() - monkeypatch.setattr("autoskillit.server._ctx", tool_ctx) - - from autoskillit.server.tools_execution import run_skill - - await run_skill("/test skill", "/tmp") - # All add_dirs must be ValidatedAddDir instances - assert len(captured["add_dirs"]) >= 1 - assert all(isinstance(d, ValidatedAddDir) for d in captured["add_dirs"]) - # Must not include raw skills_extended/ path - from autoskillit.workspace.skills import bundled_skills_extended_dir - - skills_ext = str(bundled_skills_extended_dir()) - add_dir_paths = [d.path for d in captured["add_dirs"]] - assert skills_ext not in add_dir_paths - - -@pytest.mark.anyio -async def test_run_skill_calls_session_skill_manager_init_session(tool_ctx, monkeypatch) -> None: - """run_skill routes through session_skill_manager.init_session() for add_dirs.""" - from unittest.mock import MagicMock - - from autoskillit.core import SkillResult, ValidatedAddDir - - # Create a spy on init_session - fake_validated = ValidatedAddDir(path="/fake/session/dir") - mock_ssm = MagicMock() - mock_ssm.init_session.return_value = fake_validated - tool_ctx.session_skill_manager = mock_ssm - - captured: dict = {} - - class MockExecutor: - async def run(self, skill_command, cwd, *, add_dirs=(), **kwargs) -> SkillResult: - captured["add_dirs"] = add_dirs - return SkillResult( - success=True, - result="ok", - session_id="", - subtype="success", - is_error=False, - exit_code=0, - needs_retry=False, - retry_reason="none", - stderr="", - token_usage=None, - ) - - tool_ctx.executor = MockExecutor() - monkeypatch.setattr("autoskillit.server._ctx", tool_ctx) - - from autoskillit.server.tools_execution import run_skill - - await run_skill("/test skill", "/tmp") - - # init_session was called with cook_session=False (headless, not cook) - mock_ssm.init_session.assert_called_once() - call_kwargs = mock_ssm.init_session.call_args - assert call_kwargs.kwargs.get("cook_session") is False - - # The returned ValidatedAddDir is in add_dirs - assert fake_validated in captured["add_dirs"] - - -@pytest.mark.anyio -async def test_run_skill_activates_deps_for_tier3_target(tool_ctx, monkeypatch) -> None: - """run_skill calls activate_skill_deps even when target is tier3 (not in tier2 list).""" - from unittest.mock import MagicMock - - from autoskillit.core import SkillResult, ValidatedAddDir - - fake_validated = ValidatedAddDir(path="/fake/session/dir") - mock_ssm = MagicMock() - mock_ssm.init_session.return_value = fake_validated - tool_ctx.session_skill_manager = mock_ssm - - # Set up skill_resolver to produce a resolved name - mock_resolver = MagicMock() - mock_resolver.resolve.return_value = MagicMock(source=MagicMock(value="bundled_extended")) - tool_ctx.skill_resolver = mock_resolver - - class MockExecutor: - async def run(self, skill_command, cwd, *, add_dirs=(), **kwargs) -> SkillResult: - return SkillResult( - success=True, - result="ok", - session_id="", - subtype="success", - is_error=False, - exit_code=0, - needs_retry=False, - retry_reason="none", - stderr="", - token_usage=None, - ) - - tool_ctx.executor = MockExecutor() - monkeypatch.setattr("autoskillit.server._ctx", tool_ctx) - - from autoskillit.server.tools_execution import run_skill - - # Use a tier3 skill name - await run_skill("/open-pr", "/tmp") - - # activate_skill_deps must have been called regardless of tier - mock_ssm.activate_skill_deps.assert_called_once() - - -@pytest.mark.anyio -async def test_run_skill_result_includes_order_id_when_passed(tool_ctx, monkeypatch) -> None: - """run_skill injects order_id into the result JSON when order_id is non-empty.""" - import json as _json - - from autoskillit.core import SkillResult - - class MockExecutor: - async def run(self, skill_command, cwd, *, add_dirs=(), **kwargs) -> SkillResult: - return SkillResult( - success=True, - result="done", - session_id="", - subtype="success", - is_error=False, - exit_code=0, - needs_retry=False, - retry_reason="none", - stderr="", - token_usage=None, - ) - - tool_ctx.executor = MockExecutor() - monkeypatch.setattr("autoskillit.server._ctx", tool_ctx) - - from autoskillit.server.tools_execution import run_skill - - result_json = await run_skill("/test skill", "/tmp", order_id="issue-185") - data = _json.loads(result_json) - assert data.get("order_id") == "issue-185" - - -@pytest.mark.anyio -async def test_run_skill_result_no_order_id_field_when_empty(tool_ctx, monkeypatch) -> None: - """run_skill does NOT inject order_id into result JSON when order_id is empty.""" - import json as _json - - from autoskillit.core import SkillResult - - class MockExecutor: - async def run(self, skill_command, cwd, *, add_dirs=(), **kwargs) -> SkillResult: - return SkillResult( - success=True, - result="done", - session_id="", - subtype="success", - is_error=False, - exit_code=0, - needs_retry=False, - retry_reason="none", - stderr="", - token_usage=None, - ) - - tool_ctx.executor = MockExecutor() - monkeypatch.setattr("autoskillit.server._ctx", tool_ctx) - - from autoskillit.server.tools_execution import run_skill - - result_json = await run_skill("/test skill", "/tmp") # no order_id - data = _json.loads(result_json) - assert "order_id" not in data - - -class TestHeadlessGateEnforcement: - """T_HGE: run_skill, run_cmd, run_python each return headless_error - when the session is running with AUTOSKILLIT_HEADLESS=1. - - The gate is open (tool_ctx default), so _require_enabled() passes. - _require_not_headless() fires first and returns subtype='headless_error'. - """ - - @pytest.fixture(autouse=True) - def _set_headless_env(self, monkeypatch): - monkeypatch.setenv("AUTOSKILLIT_HEADLESS", "1") - - @pytest.mark.anyio - async def test_run_skill_blocked_in_headless_session(self, tool_ctx): - """run_skill returns headless_error when AUTOSKILLIT_HEADLESS=1.""" - result = json.loads(await run_skill("/autoskillit:investigate some-error", "/tmp")) - assert result["subtype"] == "headless_error" - - -class TestResponseFieldsAreTypeSafe: - """Every discriminator field in MCP tool responses uses enum values.""" - - @pytest.mark.anyio - async def test_retry_reason_is_enum_value(self, tool_ctx): - stdout = json.dumps( - { - "type": "result", - "subtype": "error_max_turns", - "is_error": False, - "session_id": "s1", - "num_turns": 200, - "errors": [], - } - ) - tool_ctx.runner.push(_make_result(1, stdout, "")) - result = json.loads(await run_skill("/retry-worktree plan.md", "/tmp")) - assert result["retry_reason"] in {e.value for e in RetryReason} - - @pytest.mark.anyio - async def test_retry_reason_none_is_enum_value(self, tool_ctx): - stdout = json.dumps( - { - "type": "result", - "subtype": "success", - "is_error": False, - "result": "Done.", - "session_id": "s1", - "num_turns": 50, - } - ) - tool_ctx.runner.push(_make_result(0, stdout, "")) - result = json.loads(await run_skill("/retry-worktree plan.md", "/tmp")) - assert result["retry_reason"] in {e.value for e in RetryReason} - - -class TestRunSkillTiming: - """run_skill accumulates wall-clock timing when step_name is provided.""" - - @pytest.mark.anyio - async def test_run_skill_records_timing_via_step_name(self, tool_ctx): - tool_ctx.runner.push(_make_result(0, _SUCCESS_JSON, "")) - await run_skill("/investigate foo", "/tmp", step_name="implement") - report = tool_ctx.timing_log.get_report() - assert len(report) == 1 - assert report[0]["step_name"] == "implement" - assert report[0]["invocation_count"] == 1 - - @pytest.mark.anyio - async def test_run_skill_empty_step_name_skips_timing(self, tool_ctx): - tool_ctx.runner.push(_make_result(0, _SUCCESS_JSON, "")) - await run_skill("/investigate foo", "/tmp") - assert tool_ctx.timing_log.get_report() == [] - - -class TestRunHeadlessCoreFlushTelemetry: - """flush_session_log receives telemetry kwargs when step_name is provided.""" - - def _make_ndjson_with_usage(self) -> str: - asst = json.dumps( - { - "type": "assistant", - "message": { - "usage": { - "input_tokens": 200, - "output_tokens": 100, - "cache_creation_input_tokens": 0, - "cache_read_input_tokens": 0, - } - }, - "model": "claude-opus-4-6", - } - ) - result = json.dumps( - { - "type": "result", - "subtype": "success", - "is_error": False, - "result": "done", - "session_id": "s1", - "usage": { - "input_tokens": 200, - "output_tokens": 100, - "cache_creation_input_tokens": 0, - "cache_read_input_tokens": 0, - }, - } - ) - return asst + "\n" + result - - @pytest.mark.anyio - async def test_passes_step_telemetry_to_flush(self, tool_ctx, monkeypatch): - """flush_session_log is called with step_name, token_usage, and timing_seconds.""" - import autoskillit.execution.session_log as sl_mod - - calls = [] - - def mock_flush(**kwargs): - calls.append(kwargs) - - monkeypatch.setattr(sl_mod, "flush_session_log", mock_flush) - tool_ctx.runner.push(_make_result(returncode=0, stdout=self._make_ndjson_with_usage())) - await run_skill("/investigate foo", "/tmp", step_name="implement") - assert len(calls) == 1 - assert calls[0]["step_name"] == "implement" - assert calls[0]["token_usage"] is not None - assert calls[0]["timing_seconds"] is not None - - @pytest.mark.anyio - async def test_flush_session_log_session_id_matches_returned_skill_result( - self, tool_ctx, monkeypatch - ): - """flush_session_log receives the same session_id as the returned SkillResult.""" - import autoskillit.execution.session_log as sl_mod - from autoskillit.core.types import SubprocessResult, TerminationReason - - calls = [] - - def mock_flush(**kwargs): - calls.append(kwargs) - - monkeypatch.setattr(sl_mod, "flush_session_log", mock_flush) - # Stale result with session_id resolved from Channel B - stale_result = SubprocessResult( - returncode=-1, - stdout="", - stderr="", - termination=TerminationReason.STALE, - pid=12345, - session_id="test-uuid-coherence-check", - ) - tool_ctx.runner.push(stale_result) - result_json = json.loads( - await run_skill("/investigate foo", "/tmp", step_name="implement") - ) - assert len(calls) == 1 - # flush and returned SkillResult must carry the same session_id - assert calls[0]["session_id"] == result_json["session_id"] - assert result_json["session_id"] != "" - - @pytest.mark.anyio - async def test_flushes_on_success_when_step_name_set(self, tool_ctx, monkeypatch): - """Successful sessions without proc_snapshots still flush when step_name is provided.""" - import autoskillit.execution.session_log as sl_mod - - calls = [] - - def mock_flush(**kwargs): - calls.append(kwargs) - - monkeypatch.setattr(sl_mod, "flush_session_log", mock_flush) - tool_ctx.runner.push(_make_result(returncode=0, stdout=_SUCCESS_JSON)) - await run_skill("/investigate foo", "/tmp", step_name="plan") - assert len(calls) == 1 - - @pytest.mark.anyio - async def test_records_timing_in_timing_log(self, tool_ctx): - """ctx.timing_log.record() is called with step_name and computed timing_seconds.""" - tool_ctx.runner.push(_make_result(returncode=0, stdout=_SUCCESS_JSON)) - await run_skill("/investigate foo", "/tmp", step_name="plan") - report = tool_ctx.timing_log.get_report() - assert len(report) == 1 - assert report[0]["step_name"] == "plan" - assert report[0]["total_seconds"] >= 0.0 - - -class TestRunSkillCwdValidation: - """run_skill rejects non-empty relative cwd at the boundary.""" - - @pytest.mark.anyio - async def test_run_skill_rejects_relative_cwd(self, tool_ctx): - """Non-empty relative cwd is rejected immediately with a clear diagnostic.""" - result = json.loads( - await run_skill( - "/autoskillit:retry-worktree plan.md ../worktrees/impl-fix", - cwd="../worktrees/impl-fix-20260316", - ) - ) - assert result["success"] is False - assert "cwd must be an absolute path" in result["error"] - assert "../worktrees/impl-fix-20260316" in result["error"] - assert tool_ctx.runner.call_args_list == [] - - @pytest.mark.anyio - async def test_run_skill_accepts_empty_cwd(self, tool_ctx, monkeypatch): - """Empty cwd is accepted (some skills have no specific cwd requirement).""" - _patch_uuid4(monkeypatch) - marker = _DETERMINISTIC_MARKER - success_json = ( - '{"type": "result", "subtype": "success", "is_error": false,' - f' "result": "done {marker}", "session_id": "s1"}}' - ) - tool_ctx.runner.push(_make_result(returncode=0, stdout=success_json)) - result = json.loads(await run_skill("/investigate foo", cwd="")) - assert result["success"] is True - - @pytest.mark.anyio - async def test_run_skill_accepts_absolute_cwd(self, tool_ctx, monkeypatch): - """Absolute cwd passes the boundary check and proceeds normally.""" - _patch_uuid4(monkeypatch) - marker = _DETERMINISTIC_MARKER - success_json = ( - '{"type": "result", "subtype": "success", "is_error": false,' - f' "result": "done {marker}", "session_id": "s1"}}' - ) - tool_ctx.runner.push(_make_result(returncode=0, stdout=success_json)) - result = json.loads(await run_skill("/investigate foo", cwd="/tmp")) - assert result["success"] is True - - -class TestRunSkillPerInvocationMarker: - """Per-invocation completion markers are unique across run_skill calls.""" - - @pytest.mark.anyio - async def test_run_skill_markers_are_unique_per_invocation(self, tool_ctx): - """Two run_skill calls must generate different completion_marker values.""" - success_json = ( - '{"type": "result", "subtype": "success", "is_error": false,' - ' "result": "done", "session_id": "s1"}' - ) - tool_ctx.runner.push(_make_result(returncode=0, stdout=success_json)) - tool_ctx.runner.push(_make_result(returncode=0, stdout=success_json)) - - await run_skill("/investigate a", cwd="/tmp") - await run_skill("/investigate b", cwd="/tmp") - - calls = tool_ctx.runner.call_args_list - assert len(calls) >= 2 - marker1 = calls[0][3]["completion_marker"] - marker2 = calls[1][3]["completion_marker"] - assert marker1 != marker2 - assert "%%ORDER_UP::" in marker1 - assert "%%ORDER_UP::" in marker2 - - -@pytest.mark.anyio -async def test_run_skill_passes_allow_only_to_init_session(tool_ctx, monkeypatch) -> None: - """run_skill computes the closure for the resolved target and forwards it as allow_only.""" - from unittest.mock import MagicMock - - from autoskillit.core import SkillResult, ValidatedAddDir - - fake_validated = ValidatedAddDir(path="/fake/session/dir") - expected_closure = frozenset({"investigate", "mermaid"}) - - mock_ssm = MagicMock() - mock_ssm.init_session.return_value = fake_validated - mock_ssm.compute_skill_closure.return_value = expected_closure - tool_ctx.session_skill_manager = mock_ssm - - mock_resolver = MagicMock() - mock_resolver.resolve.return_value = MagicMock(source=MagicMock(value="bundled_extended")) - tool_ctx.skill_resolver = mock_resolver - - class MockExecutor: - async def run(self, skill_command, cwd, *, add_dirs=(), **kwargs) -> SkillResult: - return SkillResult( - success=True, - result="ok", - session_id="", - subtype="success", - is_error=False, - exit_code=0, - needs_retry=False, - retry_reason="none", - stderr="", - token_usage=None, - ) - - tool_ctx.executor = MockExecutor() - monkeypatch.setattr("autoskillit.server._ctx", tool_ctx) - - from autoskillit.server.tools_execution import run_skill - - await run_skill("/autoskillit:investigate the bug", "/tmp") - - mock_ssm.compute_skill_closure.assert_called_once_with("investigate") - mock_ssm.init_session.assert_called_once() - assert mock_ssm.init_session.call_args.kwargs.get("allow_only") == expected_closure - - -@pytest.mark.anyio -async def test_run_skill_no_target_skill_passes_none_allow_only(tool_ctx, monkeypatch) -> None: - """When skill_resolver is unset, target_name is None and allow_only stays None.""" - from unittest.mock import MagicMock - - from autoskillit.core import SkillResult, ValidatedAddDir - - fake_validated = ValidatedAddDir(path="/fake/session/dir") - mock_ssm = MagicMock() - mock_ssm.init_session.return_value = fake_validated - tool_ctx.session_skill_manager = mock_ssm - tool_ctx.skill_resolver = None # disables resolve_target_skill - - class MockExecutor: - async def run(self, skill_command, cwd, *, add_dirs=(), **kwargs) -> SkillResult: - return SkillResult( - success=True, - result="ok", - session_id="", - subtype="success", - is_error=False, - exit_code=0, - needs_retry=False, - retry_reason="none", - stderr="", - token_usage=None, - ) - - tool_ctx.executor = MockExecutor() - monkeypatch.setattr("autoskillit.server._ctx", tool_ctx) - - from autoskillit.server.tools_execution import run_skill - - await run_skill("/test skill", "/tmp") - - mock_ssm.init_session.assert_called_once() - assert mock_ssm.init_session.call_args.kwargs.get("allow_only") is None - mock_ssm.compute_skill_closure.assert_not_called() - - -@pytest.mark.anyio -async def test_run_skill_make_plan_closure_includes_arch_lens_pack(tool_ctx, monkeypatch) -> None: - """End-to-end: /make-plan resolves a closure containing the entire arch-lens pack.""" - from unittest.mock import MagicMock - - from autoskillit.core import SkillResult, ValidatedAddDir - from autoskillit.workspace.session_skills import ( - DefaultSessionSkillManager, - SkillsDirectoryProvider, - ) - - real_provider = SkillsDirectoryProvider() - real_mgr = DefaultSessionSkillManager(provider=real_provider, ephemeral_root=tool_ctx.temp_dir) - - captured: dict = {} - - class _RecordingManager: - def __init__(self, real: DefaultSessionSkillManager) -> None: - self._real = real - - def init_session(self, session_id, **kwargs): - captured["allow_only"] = kwargs.get("allow_only") - return ValidatedAddDir(path="/fake/session/dir") - - def compute_skill_closure(self, target_name): - return self._real.compute_skill_closure(target_name) - - def activate_skill_deps(self, session_id, skill_name): - return True - - def cleanup_stale(self, max_age_seconds=86400): - return 0 - - tool_ctx.session_skill_manager = _RecordingManager(real_mgr) - - mock_resolver = MagicMock() - mock_resolver.resolve.return_value = MagicMock(source=MagicMock(value="bundled_extended")) - tool_ctx.skill_resolver = mock_resolver - - class MockExecutor: - async def run(self, skill_command, cwd, *, add_dirs=(), **kwargs) -> SkillResult: - return SkillResult( - success=True, - result="ok", - session_id="", - subtype="success", - is_error=False, - exit_code=0, - needs_retry=False, - retry_reason="none", - stderr="", - token_usage=None, - ) - - tool_ctx.executor = MockExecutor() - monkeypatch.setattr("autoskillit.server._ctx", tool_ctx) - - from autoskillit.server.tools_execution import run_skill - - await run_skill("/autoskillit:make-plan refactor", "/tmp") - - closure = captured["allow_only"] - assert closure is not None - assert "make-plan" in closure - assert "mermaid" in closure - arch_members = {n for n in closure if n.startswith("arch-lens-")} - assert len(arch_members) >= 1 - - -def _make_capturing_executor(): - """Return (executor, captured_dict) for testing idle_output_timeout propagation.""" - from autoskillit.core import SkillResult - - captured: dict = {} - - class MockExecutor: - async def run( - self, skill_command, cwd, *, idle_output_timeout=None, **kwargs - ) -> SkillResult: - captured["idle_output_timeout"] = idle_output_timeout - return SkillResult( - success=True, - result="ok", - session_id="", - subtype="success", - is_error=False, - exit_code=0, - needs_retry=False, - retry_reason="none", - stderr="", - token_usage=None, - ) - - return MockExecutor(), captured - - -@pytest.mark.anyio -async def test_run_skill_passes_idle_output_timeout(tool_ctx, monkeypatch) -> None: - """run_skill passes idle_output_timeout (as float) to executor.run().""" - executor, captured = _make_capturing_executor() - tool_ctx.executor = executor - monkeypatch.setattr("autoskillit.server._ctx", tool_ctx) - - from autoskillit.server.tools_execution import run_skill - - await run_skill("/test skill", "/tmp", idle_output_timeout=120) - assert captured["idle_output_timeout"] == 120.0 # int→float conversion - - -@pytest.mark.anyio -async def test_run_skill_idle_output_timeout_defaults_to_none(tool_ctx, monkeypatch) -> None: - """run_skill passes None to executor.run() when idle_output_timeout is not set.""" - executor, captured = _make_capturing_executor() - tool_ctx.executor = executor - monkeypatch.setattr("autoskillit.server._ctx", tool_ctx) - - from autoskillit.server.tools_execution import run_skill - - await run_skill("/test skill", "/tmp") - assert captured["idle_output_timeout"] is None - - -@pytest.mark.anyio -async def test_run_skill_returns_structured_error_when_executor_raises( - tool_ctx, monkeypatch, tmp_path -) -> None: - """run_skill returns SkillResult-shaped JSON even if executor.run() raises unexpectedly.""" - from autoskillit.core import SkillResult - - class ExplodingExecutor: - async def run(self, *args, **kwargs) -> SkillResult: - raise RuntimeError("unexpected executor failure") - - tool_ctx.executor = ExplodingExecutor() - monkeypatch.setattr("autoskillit.server._ctx", tool_ctx) - - from autoskillit.server.tools_execution import run_skill - - result_json = await run_skill("/test cmd", str(tmp_path)) - data = json.loads(result_json) - assert data["success"] is False - assert data["subtype"] == "crashed" - assert data["needs_retry"] is False - assert "unexpected executor failure" in data["result"] diff --git a/tests/server/test_tools_execution_command.py b/tests/server/test_tools_execution_command.py new file mode 100644 index 0000000000..0d04e8bec5 --- /dev/null +++ b/tests/server/test_tools_execution_command.py @@ -0,0 +1,251 @@ +"""Tests for run_skill command building, timeouts, env, model, and per-invocation markers.""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from autoskillit.config import ( + AutomationConfig, + RunSkillConfig, +) +from autoskillit.execution.commands import _inject_completion_directive +from autoskillit.execution.headless import _session_log_dir +from autoskillit.server.tools.tools_execution import run_skill +from tests.conftest import _make_result +from tests.server.conftest import _SUCCESS_JSON + +pytestmark = [pytest.mark.layer("server"), pytest.mark.small] + + +class TestRunSkillPluginDir: + """T2: run_skill passes --plugin-dir to the claude command.""" + + @pytest.mark.anyio + async def test_run_skill_passes_plugin_dir(self, tool_ctx_kitchen_open): + """T2: run_skill passes --plugin-dir to the claude command.""" + tool_ctx_kitchen_open.runner.push(_make_result(returncode=1)) # clone guard snapshot + tool_ctx_kitchen_open.runner.push( + _make_result( + 0, + '{"type": "result", "subtype": "success", "is_error": false,' + ' "result": "done", "session_id": "s1"}', + "", + ) + ) + await run_skill("/investigate some-error", "/tmp") + + cmd = tool_ctx_kitchen_open.runner.call_args_list[-1][0] + assert "--plugin-dir" in cmd + plugin_dir_idx = cmd.index("--plugin-dir") + from autoskillit.core.types._type_plugin_source import DirectInstall + + assert isinstance(tool_ctx_kitchen_open.plugin_source, DirectInstall) + assert cmd[plugin_dir_idx + 1] == str(tool_ctx_kitchen_open.plugin_source.plugin_dir) + assert "--output-format" in cmd + assert cmd[cmd.index("--output-format") + 1] == "stream-json" + actual_cwd = tool_ctx_kitchen_open.runner.call_args_list[-1][1] + assert actual_cwd == Path("/tmp"), f"Subprocess cwd mismatch: {actual_cwd} != /tmp" + + +class TestRunSkillTimeoutFromConfig: + """run_skill uses configurable timeouts.""" + + @pytest.mark.anyio + async def test_run_skill_timeout_from_config(self, tool_ctx_kitchen_open): + """run_skill uses _config.run_skill.timeout instead of hardcoded value.""" + cfg = AutomationConfig() + cfg.run_skill = RunSkillConfig(timeout=120) + cfg.safety.require_dry_walkthrough = False + tool_ctx_kitchen_open.config = cfg + + tool_ctx_kitchen_open.runner.push(_make_result(returncode=1)) # clone guard snapshot + tool_ctx_kitchen_open.runner.push( + _make_result( + 0, + '{"type": "result", "subtype": "success", "is_error": false,' + ' "result": "done", "session_id": "s1"}', + "", + ) + ) + await run_skill("/investigate foo", "/tmp") + + assert tool_ctx_kitchen_open.runner.call_args_list[-1][2] == 120.0 + + +class TestRunSkillInjectsCompletionDirective: + """run_skill injects completion directive into the skill command.""" + + @pytest.mark.anyio + async def test_run_skill_injects_completion_directive(self, tool_ctx_kitchen_open): + """Skill command passed to claude -p contains the completion marker instruction.""" + cfg = AutomationConfig() + cfg.safety.require_dry_walkthrough = False + tool_ctx_kitchen_open.config = cfg + + tool_ctx_kitchen_open.runner.push(_make_result(returncode=1)) # clone guard snapshot + tool_ctx_kitchen_open.runner.push( + _make_result( + 0, + '{"type": "result", "subtype": "success", "is_error": false,' + ' "result": "done", "session_id": "s1"}', + "", + ) + ) + await run_skill("/investigate foo", "/tmp") + + cmd = tool_ctx_kitchen_open.runner.call_args_list[-1][0] + prompt_idx = cmd.index("--print") + 1 if "--print" in cmd else cmd.index("-p") + 1 + skill_arg = cmd[prompt_idx] + assert "%%ORDER_UP::" in skill_arg + assert "ORCHESTRATION DIRECTIVE" in skill_arg + + def test_inject_completion_directive_prohibits_standalone_marker(self): + """ + The directive wording must explicitly instruct the model to emit the marker + in the SAME message as its substantive output, not as a standalone message. + This prevents the model from interpreting the directive as a post-task acknowledgment. + """ + result = _inject_completion_directive("/audit-impl", "%%ORDER_UP%%") + lowered = result.lower() + assert ( + "same message" in lowered + or "not as a separate" in lowered + or ("standalone" in lowered and "not" in lowered) + ), f"Directive must prohibit standalone marker emission. Got: {result!r}" + + +class TestRunSkillEnvPrefix: + """run_skill always injects AUTOSKILLIT_HEADLESS=1 and optionally CLAUDE_CODE_EXIT_AFTER_STOP_DELAY via the env kwarg.""" # noqa: E501 + + @pytest.mark.anyio + async def test_default_delay_populates_env(self, tool_ctx_kitchen_open): + tool_ctx_kitchen_open.runner.push(_make_result(returncode=1)) # clone guard snapshot + tool_ctx_kitchen_open.runner.push(_make_result(0, _SUCCESS_JSON, "")) + await run_skill("/investigate something", "/tmp") + cmd, _cwd, _timeout, kwargs = tool_ctx_kitchen_open.runner.call_args_list[-1] + assert cmd[0] == "claude" + env = kwargs["env"] + assert env["AUTOSKILLIT_HEADLESS"] == "1" + assert env["CLAUDE_CODE_EXIT_AFTER_STOP_DELAY"] == "2000" + + @pytest.mark.anyio + async def test_zero_delay_omits_delay_env_var(self, tool_ctx_kitchen_open): + cfg = AutomationConfig() + cfg.run_skill = RunSkillConfig(exit_after_stop_delay_ms=0) + cfg.safety.require_dry_walkthrough = False + tool_ctx_kitchen_open.config = cfg + tool_ctx_kitchen_open.runner.push(_make_result(returncode=1)) # clone guard snapshot + tool_ctx_kitchen_open.runner.push(_make_result(0, _SUCCESS_JSON, "")) + await run_skill("/investigate something", "/tmp") + cmd, _cwd, _timeout, kwargs = tool_ctx_kitchen_open.runner.call_args_list[-1] + assert cmd[0] == "claude" + env = kwargs["env"] + assert env["AUTOSKILLIT_HEADLESS"] == "1" + assert "CLAUDE_CODE_EXIT_AFTER_STOP_DELAY" not in env + + @pytest.mark.anyio + async def test_custom_delay_value_in_env(self, tool_ctx_kitchen_open): + cfg = AutomationConfig() + cfg.run_skill = RunSkillConfig( + exit_after_stop_delay_ms=60000, natural_exit_grace_seconds=61.0 + ) + cfg.safety.require_dry_walkthrough = False + tool_ctx_kitchen_open.config = cfg + tool_ctx_kitchen_open.runner.push(_make_result(returncode=1)) # clone guard snapshot + tool_ctx_kitchen_open.runner.push(_make_result(0, _SUCCESS_JSON, "")) + await run_skill("/investigate something", "/tmp") + cmd, _cwd, _timeout, kwargs = tool_ctx_kitchen_open.runner.call_args_list[-1] + assert cmd[0] == "claude" + env = kwargs["env"] + assert env["AUTOSKILLIT_HEADLESS"] == "1" + assert env["CLAUDE_CODE_EXIT_AFTER_STOP_DELAY"] == "60000" + + +class TestRunSkillPassesSessionLogDir: + """run_skill passes session_log_dir derived from cwd.""" + + @pytest.mark.anyio + async def test_run_skill_passes_session_log_dir(self, tool_ctx_kitchen_open): + """runner receives session_log_dir derived from cwd.""" + cfg = AutomationConfig() + cfg.safety.require_dry_walkthrough = False + tool_ctx_kitchen_open.config = cfg + + tool_ctx_kitchen_open.runner.push(_make_result(returncode=1)) # clone guard snapshot + tool_ctx_kitchen_open.runner.push( + _make_result( + 0, + '{"type": "result", "subtype": "success", "is_error": false,' + ' "result": "done", "session_id": "s1"}', + "", + ) + ) + await run_skill("/investigate foo", "/some/project") + + call_kwargs = tool_ctx_kitchen_open.runner.call_args_list[-1][3] + expected_dir = _session_log_dir("/some/project") + assert call_kwargs["session_log_dir"] == expected_dir + assert "-some-project" in str(expected_dir) + + +class TestRunSkillModel: + """Tests for model parameter in run_skill.""" + + _MOCK_STDOUT = ( + '{"type": "result", "subtype": "success", "is_error": false, ' + '"result": "done", "session_id": "s1"}' + ) + + # MOD_S1 + @pytest.mark.anyio + async def test_run_skill_passes_model_flag(self, tool_ctx_kitchen_open): + tool_ctx_kitchen_open.runner.push(_make_result(returncode=1)) # clone guard snapshot + tool_ctx_kitchen_open.runner.push(_make_result(0, self._MOCK_STDOUT, "")) + await run_skill("/investigate error", "/tmp", model="sonnet") + cmd = tool_ctx_kitchen_open.runner.call_args_list[-1][0] + assert "--model" in cmd + assert cmd[cmd.index("--model") + 1] == "sonnet" + + # MOD_S3 + @pytest.mark.anyio + async def test_run_skill_no_model_flag_when_empty(self, tool_ctx_kitchen_open): + tool_ctx_kitchen_open.config.model.default = "" # ← add this line + tool_ctx_kitchen_open.runner.push(_make_result(returncode=1)) # clone guard snapshot + tool_ctx_kitchen_open.runner.push(_make_result(0, self._MOCK_STDOUT, "")) + await run_skill("/investigate error", "/tmp", model="") + cmd = tool_ctx_kitchen_open.runner.call_args_list[-1][0] + assert "--model" not in cmd + + +class TestRunSkillPerInvocationMarker: + """Per-invocation completion markers are unique across run_skill calls.""" + + @pytest.mark.anyio + async def test_run_skill_markers_are_unique_per_invocation(self, tool_ctx_kitchen_open): + """Two run_skill calls must generate different completion_marker values.""" + success_json = ( + '{"type": "result", "subtype": "success", "is_error": false,' + ' "result": "done", "session_id": "s1"}' + ) + tool_ctx_kitchen_open.runner.push( + _make_result(returncode=1) + ) # clone guard snapshot call 1 + tool_ctx_kitchen_open.runner.push(_make_result(returncode=0, stdout=success_json)) + tool_ctx_kitchen_open.runner.push( + _make_result(returncode=1) + ) # clone guard snapshot call 2 + tool_ctx_kitchen_open.runner.push(_make_result(returncode=0, stdout=success_json)) + + await run_skill("/investigate a", cwd="/tmp") + await run_skill("/investigate b", cwd="/tmp") + + calls = tool_ctx_kitchen_open.runner.call_args_list + claude_calls = [c for c in calls if c[0][0] == "claude"] + assert len(claude_calls) >= 2 + marker1 = claude_calls[0][3]["completion_marker"] + marker2 = claude_calls[1][3]["completion_marker"] + assert marker1 != marker2 + assert "%%ORDER_UP::" in marker1 + assert "%%ORDER_UP::" in marker2 diff --git a/tests/server/test_tools_execution_input_gates.py b/tests/server/test_tools_execution_input_gates.py new file mode 100644 index 0000000000..181f2a7db6 --- /dev/null +++ b/tests/server/test_tools_execution_input_gates.py @@ -0,0 +1,303 @@ +"""Tests for run_skill input validation gates and CWD checking.""" + +from __future__ import annotations + +import json +from pathlib import Path + +import pytest + +from autoskillit.server._guards import _check_dry_walkthrough +from autoskillit.server._state import _get_config +from autoskillit.server.tools.tools_execution import run_skill + +pytestmark = [pytest.mark.layer("server"), pytest.mark.small] + +# Deterministic UUID for tests that need to predict the per-invocation marker. +_DETERMINISTIC_HEX = "a1b2c3d4e5f6a7b890123456" +_DETERMINISTIC_MARKER = f"%%ORDER_UP::{_DETERMINISTIC_HEX[:8]}%%" + + +class _FixedUUID: + hex = _DETERMINISTIC_HEX + + +def _patch_uuid4(monkeypatch): + """Monkeypatch uuid4 to return a deterministic value for marker prediction.""" + monkeypatch.setattr("uuid.uuid4", lambda: _FixedUUID()) + + +class TestCheckDryWalkthrough: + """Dry-walkthrough gate blocks both /implement-worktree variants.""" + + def test_dry_walkthrough_gate_blocks_implement_no_merge(self, tool_ctx, tmp_path): + """Gate blocks /implement-worktree-no-merge when plan lacks marker.""" + plan = tmp_path / "plan.md" + plan.write_text("# My Plan\n\nSome content") + result = _check_dry_walkthrough(f"/implement-worktree-no-merge {plan}", str(tmp_path)) + assert result is not None + parsed = json.loads(result) + assert parsed["success"] is False + assert parsed["is_error"] is True + assert "dry-walked" in parsed["result"].lower() + + def test_dry_walkthrough_gate_passes_implement_no_merge(self, tool_ctx, tmp_path): + """Gate allows /implement-worktree-no-merge when plan has marker.""" + plan = tmp_path / "plan.md" + plan.write_text("Dry-walkthrough verified = TRUE\n# My Plan") + result = _check_dry_walkthrough(f"/implement-worktree-no-merge {plan}", str(tmp_path)) + assert result is None + + def test_dry_walkthrough_gate_still_works_for_implement_worktree(self, tool_ctx, tmp_path): + """Original /implement-worktree gating is not broken.""" + plan = tmp_path / "plan.md" + plan.write_text("# No marker plan") + result = _check_dry_walkthrough(f"/implement-worktree {plan}", str(tmp_path)) + assert result is not None + parsed = json.loads(result) + assert parsed["success"] is False + assert parsed["is_error"] is True + + def test_dry_walkthrough_gate_ignores_unrelated_skills(self, tool_ctx): + """Gate ignores skills that are not implement-worktree variants.""" + result = _check_dry_walkthrough("/autoskillit:investigate some-error", "/tmp") + assert result is None + + def test_dry_walkthrough_gate_with_part_a_named_file_marked(self, tmp_path, tool_ctx): + """Gate accepts _part_a.md file when marker is present.""" + plan = tmp_path / "task_plan_2026-01-01_part_a.md" + plan.write_text("Dry-walkthrough verified = TRUE\n\nContent here") + result = _check_dry_walkthrough(f"/implement-worktree-no-merge {plan}", str(tmp_path)) + assert result is None + + def test_dry_walkthrough_gate_with_part_b_named_file_unmarked(self, tmp_path, tool_ctx): + """Gate blocks _part_b.md file when marker is absent.""" + plan = tmp_path / "task_plan_2026-01-01_part_b.md" + plan.write_text("> **PART B ONLY.**\n\nNo walkthrough marker here") + result = _check_dry_walkthrough(f"/implement-worktree-no-merge {plan}", str(tmp_path)) + assert result is not None + parsed = json.loads(result) + assert parsed["subtype"] == "gate_error" + + def test_dry_walkthrough_gate_distinguishes_parts_independently(self, tmp_path, tool_ctx): + """Gate correctly distinguishes marked part_a from unmarked part_b.""" + part_a = tmp_path / "task_plan_part_a.md" + part_b = tmp_path / "task_plan_part_b.md" + part_a.write_text("Dry-walkthrough verified = TRUE\n\nPart A content") + part_b.write_text("> **PART B ONLY.**\n\nPart B content — no marker") + + result_a = _check_dry_walkthrough(f"/implement-worktree-no-merge {part_a}", str(tmp_path)) + result_b = _check_dry_walkthrough(f"/implement-worktree-no-merge {part_b}", str(tmp_path)) + assert result_a is None + assert result_b is not None + assert json.loads(result_b)["subtype"] == "gate_error" + + def test_gate_with_trailing_markdown_header_finds_plan(self, tmp_path, tool_ctx): + """Trailing markdown headers must not corrupt the plan path.""" + plan = tmp_path / "plan.md" + plan.write_text(_get_config().implement_gate.marker + "\n\nrest") + cmd = f"/implement-worktree-no-merge {plan}\n\n## Base Branch\nimpl-926" + assert _check_dry_walkthrough(cmd, str(tmp_path)) is None + + def test_gate_with_extra_token_after_path(self, tmp_path, tool_ctx): + """Space-separated token after path must not corrupt the plan path.""" + plan = tmp_path / "plan.md" + plan.write_text(_get_config().implement_gate.marker + "\n\nrest") + cmd = f"/implement-worktree-no-merge {plan} impl-926" + assert _check_dry_walkthrough(cmd, str(tmp_path)) is None + + def test_gate_multiline_no_marker_reports_dry_walk_error(self, tmp_path, tool_ctx): + """With trailing headers and plan missing marker: dry-walk error, not file-not-found.""" + plan = tmp_path / "plan.md" + plan.write_text("# Plan title\n\nNo marker here") + cmd = f"/implement-worktree-no-merge {plan}\n\n## Base Branch\nimpl-926" + result = _check_dry_walkthrough(cmd, str(tmp_path)) + assert result is not None + data = json.loads(result) + message = data.get("result", "").lower() + assert "not found" not in message, "Should fail on marker absence, not path lookup" + assert "dry-walk" in message or "dry-walked" in message + + +class TestRunSkillPrefix: + """run_skill passes prefixed command to subprocess.""" + + @pytest.mark.anyio + async def test_run_skill_prefixes_skill_command(self, tool_ctx_kitchen_open): + from tests.conftest import _make_result + + tool_ctx_kitchen_open.runner.push(_make_result(returncode=1)) # clone guard snapshot + tool_ctx_kitchen_open.runner.push( + _make_result( + 0, + '{"type": "result", "subtype": "success", "is_error": false, ' + '"result": "done", "session_id": "s1"}', + "", + ) + ) + await run_skill("/investigate error", "/tmp") + cmd = tool_ctx_kitchen_open.runner.call_args_list[-1][0] + prompt_idx = cmd.index("--print") + 1 if "--print" in cmd else cmd.index("-p") + 1 + assert cmd[prompt_idx].startswith("Use the /investigate skill error") + actual_cwd = tool_ctx_kitchen_open.runner.call_args_list[-1][1] + assert actual_cwd == Path("/tmp"), f"Subprocess cwd mismatch: {actual_cwd} != /tmp" + + @pytest.mark.anyio + async def test_run_skill_rejects_prose_without_slash(self, tool_ctx): + """FRICT-6-1: prose command without slash returns gate_error before reaching executor.""" + result = json.loads(await run_skill("Fix the authentication bug in main.py", "/tmp")) + assert result["success"] is False + assert result["is_error"] is True + assert result["subtype"] == "gate_error" + # executor must NOT have been called + assert tool_ctx.runner.call_args_list == [] + + @pytest.mark.anyio + async def test_run_skill_rejects_empty_skill_command(self, tool_ctx): + """FRICT-6-1: empty string returns gate_error without hitting executor.""" + result = json.loads(await run_skill("", "/tmp")) + assert result["success"] is False + assert result["is_error"] is True + assert result["subtype"] == "gate_error" + assert tool_ctx.runner.call_args_list == [] + + @pytest.mark.anyio + async def test_run_skill_rejects_whitespace_only(self, tool_ctx): + """FRICT-6-1: whitespace-only command returns gate_error (strip before check).""" + result = json.loads(await run_skill(" ", "/tmp")) + assert result["success"] is False + assert result["is_error"] is True + assert result["subtype"] == "gate_error" + assert tool_ctx.runner.call_args_list == [] + + @pytest.mark.anyio + async def test_run_skill_format_error_includes_slash_examples(self, tool_ctx_kitchen_open): + """FRICT-6-1: error message for invalid format includes concrete slash-command examples.""" + result = json.loads(await run_skill("investigate this bug", "/tmp")) + assert result["success"] is False + assert "/autoskillit:" in result["result"] + assert "/" in result["result"] + + @pytest.mark.anyio + async def test_run_skill_includes_completion_directive(self, tool_ctx_kitchen_open): + from tests.conftest import _make_result + + tool_ctx_kitchen_open.runner.push(_make_result(returncode=1)) # clone guard snapshot + tool_ctx_kitchen_open.runner.push( + _make_result( + 0, + '{"type": "result", "subtype": "success", "is_error": false, ' + '"result": "done", "session_id": "s1"}', + "", + ) + ) + await run_skill("/investigate error", "/tmp") + cmd = tool_ctx_kitchen_open.runner.call_args_list[-1][0] + prompt_idx = cmd.index("--print") + 1 if "--print" in cmd else cmd.index("-p") + 1 + assert "%%ORDER_UP::" in cmd[prompt_idx] + from pathlib import Path + + actual_cwd = tool_ctx_kitchen_open.runner.call_args_list[-1][1] + assert actual_cwd == Path("/tmp"), f"Subprocess cwd mismatch: {actual_cwd} != /tmp" + + +class TestValidateSkillCommand: + """Unit tests for _validate_skill_command helper.""" + + def test_returns_none_for_slash_command(self, tool_ctx): + from autoskillit.server._guards import _validate_skill_command + + assert _validate_skill_command("/autoskillit:investigate") is None + + def test_returns_none_for_bare_slash_command(self, tool_ctx): + from autoskillit.server._guards import _validate_skill_command + + assert _validate_skill_command("/audit-arch") is None + + def test_returns_error_json_for_prose(self, tool_ctx): + from autoskillit.server._guards import _validate_skill_command + + result = _validate_skill_command("Fix the bug") + assert result is not None + parsed = json.loads(result) + assert parsed["success"] is False + assert parsed["subtype"] == "gate_error" + + def test_returns_error_json_for_empty_string(self, tool_ctx): + from autoskillit.server._guards import _validate_skill_command + + result = _validate_skill_command("") + assert result is not None + + def test_strips_whitespace_before_check(self, tool_ctx): + from autoskillit.server._guards import _validate_skill_command + + # Leading whitespace before slash → valid + assert _validate_skill_command(" /autoskillit:investigate") is None + # Leading whitespace before prose → invalid + result = _validate_skill_command(" investigate bug") + assert result is not None + + +class TestDryWalkthroughGateWithPrefix: + """Dry-walkthrough gate still receives raw command before prefix is applied.""" + + @pytest.mark.anyio + async def test_gate_still_fires_for_implement_skill(self, tool_ctx_kitchen_open, tmp_path): + plan = tmp_path / "plan.md" + plan.write_text("# No marker plan") + result = json.loads(await run_skill(f"/implement-worktree {plan}", str(tmp_path))) + assert result["success"] is False + assert result["is_error"] is True + assert "dry-walked" in result["result"].lower() + + +class TestRunSkillCwdValidation: + """run_skill rejects non-empty relative cwd at the boundary.""" + + @pytest.mark.anyio + async def test_run_skill_rejects_relative_cwd(self, tool_ctx_kitchen_open): + """Non-empty relative cwd is rejected immediately with a clear diagnostic.""" + result = json.loads( + await run_skill( + "/autoskillit:retry-worktree plan.md ../worktrees/impl-fix", + cwd="../worktrees/impl-fix-20260316", + ) + ) + assert result["success"] is False + assert "cwd must be an absolute path" in result["error"] + assert "../worktrees/impl-fix-20260316" in result["error"] + assert tool_ctx_kitchen_open.runner.call_args_list == [] + + @pytest.mark.anyio + async def test_run_skill_accepts_empty_cwd(self, tool_ctx_kitchen_open, monkeypatch): + """Empty cwd is accepted (some skills have no specific cwd requirement).""" + from tests.conftest import _make_result + + _patch_uuid4(monkeypatch) + marker = _DETERMINISTIC_MARKER + success_json = ( + '{"type": "result", "subtype": "success", "is_error": false,' + f' "result": "done {marker}", "session_id": "s1"}}' + ) + tool_ctx_kitchen_open.runner.push(_make_result(returncode=0, stdout=success_json)) + result = json.loads(await run_skill("/investigate foo", cwd="")) + assert "cwd must be an absolute path" not in result.get("error", "") + assert result.get("subtype") != "gate_error" + + @pytest.mark.anyio + async def test_run_skill_accepts_absolute_cwd(self, tool_ctx_kitchen_open, monkeypatch): + """Absolute cwd passes the boundary check and proceeds normally.""" + from tests.conftest import _make_result + + _patch_uuid4(monkeypatch) + marker = _DETERMINISTIC_MARKER + success_json = ( + '{"type": "result", "subtype": "success", "is_error": false,' + f' "result": "done {marker}", "session_id": "s1"}}' + ) + tool_ctx_kitchen_open.runner.push(_make_result(returncode=1)) # clone guard snapshot + tool_ctx_kitchen_open.runner.push(_make_result(returncode=0, stdout=success_json)) + result = json.loads(await run_skill("/investigate foo", cwd="/tmp")) + assert "cwd must be an absolute path" not in result.get("error", "") + assert result.get("subtype") != "gate_error" diff --git a/tests/server/test_tools_execution_provider.py b/tests/server/test_tools_execution_provider.py new file mode 100644 index 0000000000..895ee6a8fe --- /dev/null +++ b/tests/server/test_tools_execution_provider.py @@ -0,0 +1,226 @@ +"""Tests for provider_extras/profile_name forwarding through run_skill().""" + +from __future__ import annotations + +import pytest + +from autoskillit.server.tools.tools_execution import run_skill + +pytestmark = [pytest.mark.layer("server"), pytest.mark.small] + + +@pytest.mark.anyio +async def test_run_skill_provider_extras_none_when_feature_disabled( + tool_ctx_kitchen_open, tmp_path, monkeypatch +) -> None: + from tests.fakes import InMemoryHeadlessExecutor + + executor = InMemoryHeadlessExecutor() + tool_ctx_kitchen_open.executor = executor + monkeypatch.setattr("autoskillit.server._ctx", tool_ctx_kitchen_open) + monkeypatch.setattr("autoskillit.core.is_feature_enabled", lambda *a, **kw: False) + + captured: dict = {} + original_run = executor.run + + async def spy_run(*args, **kwargs): + captured.update(kwargs) + return await original_run(*args, **kwargs) + + monkeypatch.setattr(executor, "run", spy_run) + + await run_skill("/autoskillit:probe", str(tmp_path)) + + assert captured.get("provider_extras") is None + assert captured.get("profile_name") == "" + + +@pytest.mark.anyio +async def test_run_skill_provider_extras_none_for_anthropic_sentinel( + tool_ctx_kitchen_open, tmp_path, monkeypatch +) -> None: + from tests.fakes import InMemoryHeadlessExecutor + + executor = InMemoryHeadlessExecutor() + tool_ctx_kitchen_open.executor = executor + monkeypatch.setattr("autoskillit.server._ctx", tool_ctx_kitchen_open) + monkeypatch.setattr("autoskillit.core.is_feature_enabled", lambda *a, **kw: True) + monkeypatch.setattr( + "autoskillit.server._guards._resolve_provider_profile", + lambda *a: ("anthropic", {"SOME_KEY": "val"}), + ) + + captured: dict = {} + original_run = executor.run + + async def spy_run(*args, **kwargs): + captured.update(kwargs) + return await original_run(*args, **kwargs) + + monkeypatch.setattr(executor, "run", spy_run) + + await run_skill("/autoskillit:probe", str(tmp_path)) + + assert captured.get("provider_extras") is None + assert captured.get("profile_name") == "" + + +@pytest.mark.anyio +async def test_run_skill_provider_extras_forwarded_for_non_anthropic( + tool_ctx_kitchen_open, tmp_path, monkeypatch +) -> None: + from tests.fakes import InMemoryHeadlessExecutor + + executor = InMemoryHeadlessExecutor() + tool_ctx_kitchen_open.executor = executor + monkeypatch.setattr("autoskillit.server._ctx", tool_ctx_kitchen_open) + monkeypatch.setattr("autoskillit.core.is_feature_enabled", lambda *a, **kw: True) + monkeypatch.setattr( + "autoskillit.server._guards._resolve_provider_profile", + lambda *a: ("bedrock", {"AWS_REGION": "us-east-1"}), + ) + + captured: dict = {} + original_run = executor.run + + async def spy_run(*args, **kwargs): + captured.update(kwargs) + return await original_run(*args, **kwargs) + + monkeypatch.setattr(executor, "run", spy_run) + + await run_skill("/autoskillit:probe", str(tmp_path)) + + assert captured.get("provider_extras") == {"AWS_REGION": "us-east-1"} + assert captured.get("profile_name") == "bedrock" + + +@pytest.mark.anyio +async def test_run_skill_model_as_profile_resolves_provider( + tool_ctx_kitchen_open, tmp_path, monkeypatch +) -> None: + from tests.fakes import InMemoryHeadlessExecutor + + executor = InMemoryHeadlessExecutor() + tool_ctx_kitchen_open.executor = executor + monkeypatch.setattr("autoskillit.server._ctx", tool_ctx_kitchen_open) + monkeypatch.setattr("autoskillit.core.is_feature_enabled", lambda *a, **kw: True) + monkeypatch.setattr( + "autoskillit.server._guards._resolve_provider_profile", + lambda *a: ("anthropic", {}), + ) + monkeypatch.setattr( + "autoskillit.server._guards._resolve_model_as_profile", + lambda *a: ("M2.7", "minimax", {"BASE_URL": "https://api.minimax.chat/v1"}), + ) + + captured: dict = {} + original_run = executor.run + + async def spy_run(*args, **kwargs): + captured.update(kwargs) + return await original_run(*args, **kwargs) + + monkeypatch.setattr(executor, "run", spy_run) + + await run_skill("/autoskillit:probe", str(tmp_path)) + + assert captured.get("model") == "M2.7" + assert captured.get("provider_extras") == {"BASE_URL": "https://api.minimax.chat/v1"} + assert captured.get("profile_name") == "minimax" + + +@pytest.mark.anyio +async def test_run_skill_step_overrides_win_over_model_as_profile( + tool_ctx_kitchen_open, tmp_path, monkeypatch +) -> None: + from tests.fakes import InMemoryHeadlessExecutor + + executor = InMemoryHeadlessExecutor() + tool_ctx_kitchen_open.executor = executor + monkeypatch.setattr("autoskillit.server._ctx", tool_ctx_kitchen_open) + monkeypatch.setattr("autoskillit.core.is_feature_enabled", lambda *a, **kw: True) + monkeypatch.setattr( + "autoskillit.server._guards._resolve_provider_profile", + lambda *a: ("bedrock", {"AWS_REGION": "us-east-1"}), + ) + map_called = [] + monkeypatch.setattr( + "autoskillit.server._guards._resolve_model_as_profile", + lambda *a: map_called.append(True) or ("", "", None), + ) + + captured: dict = {} + original_run = executor.run + + async def spy_run(*args, **kwargs): + captured.update(kwargs) + return await original_run(*args, **kwargs) + + monkeypatch.setattr(executor, "run", spy_run) + + await run_skill("/autoskillit:probe", str(tmp_path), model="minimax") + + assert captured.get("provider_extras") == {"AWS_REGION": "us-east-1"} + assert captured.get("profile_name") == "bedrock" + assert not map_called + + +@pytest.mark.anyio +async def test_run_skill_model_as_profile_disabled_when_feature_off( + tool_ctx_kitchen_open, tmp_path, monkeypatch +) -> None: + from tests.fakes import InMemoryHeadlessExecutor + + executor = InMemoryHeadlessExecutor() + tool_ctx_kitchen_open.executor = executor + monkeypatch.setattr("autoskillit.server._ctx", tool_ctx_kitchen_open) + monkeypatch.setattr("autoskillit.core.is_feature_enabled", lambda *a, **kw: False) + + captured: dict = {} + original_run = executor.run + + async def spy_run(*args, **kwargs): + captured.update(kwargs) + return await original_run(*args, **kwargs) + + monkeypatch.setattr(executor, "run", spy_run) + + await run_skill("/autoskillit:probe", str(tmp_path), model="minimax") + + assert captured.get("model") == "minimax" + assert captured.get("provider_extras") is None + + +@pytest.mark.anyio +async def test_run_skill_model_as_profile_no_anthropic_model_falls_through( + tool_ctx_kitchen_open, tmp_path, monkeypatch +) -> None: + from tests.fakes import InMemoryHeadlessExecutor + + executor = InMemoryHeadlessExecutor() + tool_ctx_kitchen_open.executor = executor + monkeypatch.setattr("autoskillit.server._ctx", tool_ctx_kitchen_open) + monkeypatch.setattr("autoskillit.core.is_feature_enabled", lambda *a, **kw: True) + monkeypatch.setattr( + "autoskillit.server._guards._resolve_provider_profile", + lambda *a: ("anthropic", {}), + ) + monkeypatch.setattr( + "autoskillit.server._guards._resolve_model_as_profile", + lambda *a: ("", "", None), + ) + + captured: dict = {} + original_run = executor.run + + async def spy_run(*args, **kwargs): + captured.update(kwargs) + return await original_run(*args, **kwargs) + + monkeypatch.setattr(executor, "run", spy_run) + + await run_skill("/autoskillit:probe", str(tmp_path)) + + assert captured.get("model") == "" + assert captured.get("provider_extras") is None diff --git a/tests/server/test_tools_execution_response.py b/tests/server/test_tools_execution_response.py new file mode 100644 index 0000000000..5d3aa90d70 --- /dev/null +++ b/tests/server/test_tools_execution_response.py @@ -0,0 +1,71 @@ +"""Contract tests: MCP tool response fields use correct enum types.""" + +from __future__ import annotations + +import json + +import pytest + +from autoskillit.core.types import ( + ChannelConfirmation, + RetryReason, + SubprocessResult, + TerminationReason, +) +from autoskillit.server.tools.tools_execution import run_skill + +pytestmark = [pytest.mark.layer("server"), pytest.mark.small] + + +def _make_session_result( + returncode: int = 0, + stdout: str = "", + stderr: str = "", + termination_reason: TerminationReason = TerminationReason.NATURAL_EXIT, + channel_confirmation: ChannelConfirmation = ChannelConfirmation.UNMONITORED, +) -> SubprocessResult: + """Create a SubprocessResult for mocking run_managed_async.""" + return SubprocessResult( + returncode=returncode, + stdout=stdout, + stderr=stderr, + termination=termination_reason, + pid=12345, + channel_confirmation=channel_confirmation, + ) + + +class TestResponseFieldsAreTypeSafe: + """Every discriminator field in MCP tool responses uses enum values.""" + + @pytest.mark.anyio + async def test_retry_reason_is_enum_value(self, tool_ctx): + stdout = json.dumps( + { + "type": "result", + "subtype": "error_max_turns", + "is_error": False, + "session_id": "s1", + "num_turns": 200, + "errors": [], + } + ) + tool_ctx.runner.push(_make_session_result(1, stdout, "")) + result = json.loads(await run_skill("/retry-worktree plan.md", "/tmp")) + assert result["retry_reason"] in {e.value for e in RetryReason} + + @pytest.mark.anyio + async def test_retry_reason_none_is_enum_value(self, tool_ctx): + stdout = json.dumps( + { + "type": "result", + "subtype": "success", + "is_error": False, + "result": "Done.", + "session_id": "s1", + "num_turns": 50, + } + ) + tool_ctx.runner.push(_make_session_result(0, stdout, "")) + result = json.loads(await run_skill("/retry-worktree plan.md", "/tmp")) + assert result["retry_reason"] in {e.value for e in RetryReason} diff --git a/tests/server/test_tools_execution_results.py b/tests/server/test_tools_execution_results.py new file mode 100644 index 0000000000..61ebe42f3a --- /dev/null +++ b/tests/server/test_tools_execution_results.py @@ -0,0 +1,704 @@ +"""Tests for run_skill result shapes, failure paths, timing, flush telemetry, and gate checks.""" + +from __future__ import annotations + +import json +from unittest.mock import AsyncMock + +import pytest +import structlog.contextvars +import structlog.testing + +from autoskillit.core.types import ( + ChannelConfirmation, + RetryReason, +) +from autoskillit.server.tools.tools_execution import run_skill +from tests.conftest import _make_result +from tests.server.conftest import _SUCCESS_JSON, assert_no_timing, assert_step_timed + +pytestmark = [pytest.mark.layer("server"), pytest.mark.small] + + +class TestGateErrorSchemaNormalization: + """Gate errors use the standard 9-field response schema.""" + + def test_require_enabled_gate_returns_standard_schema(self, tool_ctx): + """Gate errors must use the same schema as normal responses.""" + from autoskillit.pipeline.gate import DefaultGateState + from autoskillit.server._guards import _require_enabled + + tool_ctx.gate = DefaultGateState(enabled=False) + gate_result = _require_enabled() + assert gate_result is not None + response = json.loads(gate_result) + assert response["success"] is False + assert response["is_error"] is True + assert response["needs_retry"] is False + assert "result" in response + + def test_dry_walkthrough_gate_returns_standard_schema(self, tool_ctx, tmp_path): + """Dry-walkthrough gate errors must use the standard response schema.""" + from autoskillit.server._guards import _check_dry_walkthrough + + plan = tmp_path / "plan.md" + plan.write_text("No marker here") + skill_cmd = f"/implement-worktree {plan}" + result = _check_dry_walkthrough(skill_cmd, str(tmp_path)) + assert result is not None + response = json.loads(result) + assert response["success"] is False + assert response["is_error"] is True + assert response["subtype"] == "gate_error" + + +class TestRunSkillFailurePaths: + """run_skill surfaces session outcome on failure.""" + + @pytest.mark.anyio + async def test_returns_subtype_on_incomplete_session(self, tool_ctx_kitchen_open): + """run_skill includes subtype when session didn't finish.""" + stdout = json.dumps( + { + "type": "result", + "subtype": "error_max_turns", + "is_error": False, + "session_id": "s1", + } + ) + tool_ctx_kitchen_open.runner.push(_make_result(returncode=1)) # clone guard snapshot + tool_ctx_kitchen_open.runner.push(_make_result(1, stdout, "")) + result = json.loads(await run_skill("/investigate error", "/tmp")) + assert result["session_id"] == "s1" + assert result["subtype"] == "error_max_turns" + + @pytest.mark.anyio + async def test_returns_is_error_on_context_limit(self, tool_ctx_kitchen_open): + """run_skill includes is_error when context limit is hit.""" + stdout = json.dumps( + { + "type": "result", + "subtype": "success", + "is_error": True, + "result": "Prompt is too long", + "session_id": "s1", + } + ) + tool_ctx_kitchen_open.runner.push(_make_result(returncode=1)) # clone guard snapshot + tool_ctx_kitchen_open.runner.push(_make_result(1, stdout, "")) + result = json.loads(await run_skill("/investigate error", "/tmp")) + assert result["is_error"] is True + assert result["subtype"] == "context_exhausted" + assert result["cli_subtype"] == "success" + + @pytest.mark.anyio + async def test_handles_empty_stdout(self, tool_ctx_kitchen_open): + """run_skill returns error result when stdout is empty.""" + tool_ctx_kitchen_open.runner.push(_make_result(returncode=1)) # clone guard snapshot + tool_ctx_kitchen_open.runner.push( + _make_result(1, "", "segfault", channel_confirmation=ChannelConfirmation.UNMONITORED) + ) + result = json.loads(await run_skill("/investigate error", "/tmp")) + assert result["exit_code"] == 1 + assert result["is_error"] is True + assert result["subtype"] == "empty_output" + assert result["success"] is False + + @pytest.mark.anyio + async def test_empty_stdout_exit_zero_is_retriable(self, tool_ctx_kitchen_open): + """Infrastructure failure (empty stdout, exit 0) is retriable with stderr.""" + tool_ctx_kitchen_open.runner.push(_make_result(returncode=1)) # clone guard snapshot + tool_ctx_kitchen_open.runner.push( + _make_result( + 0, "", "session dropped", channel_confirmation=ChannelConfirmation.UNMONITORED + ) + ) + result = json.loads(await run_skill("/investigate error", "/tmp")) + assert result["subtype"] == "empty_output" + assert result["success"] is False + assert result["needs_retry"] is True + assert result["retry_reason"] == RetryReason.EMPTY_OUTPUT + assert result["stderr"] == "session dropped" + + +class TestRunSkillStepName: + """step_name param drives token_log accumulation.""" + + def _make_ndjson(self) -> str: + result_rec = json.dumps( + { + "type": "result", + "subtype": "success", + "is_error": False, + "result": "Task complete.", + "session_id": "sess-abc", + "usage": { + "input_tokens": 200, + "output_tokens": 80, + "cache_creation_input_tokens": 8, + "cache_read_input_tokens": 3, + }, + } + ) + return result_rec + + @pytest.mark.anyio + async def test_step_name_records_token_usage(self, tool_ctx_kitchen_open): + tool_ctx_kitchen_open.runner.push( + _make_result(returncode=1) + ) # clone guard snapshot (not a git repo) + tool_ctx_kitchen_open.runner.push(_make_result(returncode=0, stdout=self._make_ndjson())) + await run_skill( + skill_command="/autoskillit:investigate topic", cwd="/tmp", step_name="plan" + ) + report = tool_ctx_kitchen_open.token_log.get_report() + assert len(report) == 1 + assert report[0]["step_name"] == "plan" + assert report[0]["input_tokens"] == 200 + + @pytest.mark.anyio + async def test_no_step_name_records_with_ad_hoc_label(self, tool_ctx_kitchen_open): + tool_ctx_kitchen_open.runner.push( + _make_result(returncode=1) + ) # clone guard snapshot (not a git repo) + tool_ctx_kitchen_open.runner.push(_make_result(returncode=0, stdout=self._make_ndjson())) + await run_skill(skill_command="/autoskillit:investigate topic", cwd="/tmp", step_name="") + report = tool_ctx_kitchen_open.token_log.get_report() + assert len(report) == 1 + assert report[0]["step_name"] == "(ad-hoc)" + assert report[0]["input_tokens"] == 200 + + @pytest.mark.anyio + async def test_dispatch_id_env_records_with_dispatch_label( + self, tool_ctx_kitchen_open, monkeypatch + ): + """step_name='' with AUTOSKILLIT_DISPATCH_ID set records under dispatch:{id} label.""" + monkeypatch.setenv("AUTOSKILLIT_DISPATCH_ID", "abc-123") + tool_ctx_kitchen_open.runner.push( + _make_result(returncode=1) + ) # clone guard snapshot (not a git repo) + tool_ctx_kitchen_open.runner.push(_make_result(returncode=0, stdout=self._make_ndjson())) + await run_skill(skill_command="/autoskillit:investigate topic", cwd="/tmp", step_name="") + report = tool_ctx_kitchen_open.token_log.get_report() + assert len(report) == 1 + assert report[0]["step_name"] == "dispatch:abc-123" + assert report[0]["input_tokens"] == 200 + + @pytest.mark.anyio + async def test_null_token_usage_does_not_record(self, tool_ctx): + # Return NDJSON with no usage field → token_usage will be null + no_usage_ndjson = json.dumps( + { + "type": "result", + "subtype": "success", + "is_error": False, + "result": "done", + "session_id": "s1", + } + ) + tool_ctx.runner.push(_make_result(returncode=1)) # clone guard snapshot (not a git repo) + tool_ctx.runner.push(_make_result(returncode=0, stdout=no_usage_ndjson)) + await run_skill( + skill_command="/autoskillit:investigate topic", cwd="/tmp", step_name="plan" + ) + assert tool_ctx.token_log.get_report() == [] + + @pytest.mark.anyio + async def test_step_name_run_skill_long_running(self, tool_ctx_kitchen_open): + """run_skill accumulates token usage by step_name (run_skill_retry test replacement).""" + tool_ctx_kitchen_open.runner.push( + _make_result(returncode=1) + ) # clone guard snapshot (not a git repo) + tool_ctx_kitchen_open.runner.push(_make_result(returncode=0, stdout=self._make_ndjson())) + await run_skill( + skill_command="/autoskillit:investigate the test failures", + cwd="/tmp", + step_name="implement", + ) + report = tool_ctx_kitchen_open.token_log.get_report() + assert len(report) == 1 + assert report[0]["step_name"] == "implement" + assert report[0]["input_tokens"] == 200 + + +class TestGatedToolObservability: + """Each gated tool binds structlog contextvars and calls ctx.info/ctx.error.""" + + @pytest.fixture + def mock_ctx(self): + """AsyncMock ctx for verifying ctx.info/ctx.error calls.""" + ctx = AsyncMock() + ctx.info = AsyncMock() + ctx.error = AsyncMock() + return ctx + + @pytest.mark.anyio + async def test_run_skill_binds_tool_contextvar_and_calls_ctx_info( + self, tool_ctx_kitchen_open, mock_ctx + ): + """run_skill binds tool='run_skill' contextvar and calls ctx.info on success.""" + tool_ctx_kitchen_open.runner.push( + _make_result( + 0, + '{"type": "result", "subtype": "success", "is_error": false,' + ' "result": "done", "session_id": "s1"}', + "", + ) + ) + with structlog.testing.capture_logs( + processors=[structlog.contextvars.merge_contextvars] + ) as logs: + await run_skill("/autoskillit:investigate task", "/tmp", ctx=mock_ctx) + assert any(entry.get("tool") == "run_skill" for entry in logs) + + @pytest.mark.anyio + async def test_run_skill_returns_failure_result_on_error_output(self, tool_ctx, mock_ctx): + """run_skill reports failure (success=false) when headless session fails.""" + tool_ctx.runner.push( + _make_result( + 1, + '{"type": "result", "subtype": "error", "is_error": true,' + ' "result": "failed", "session_id": "s1"}', + "", + channel_confirmation=ChannelConfirmation.UNMONITORED, + ) + ) + result = json.loads(await run_skill("/autoskillit:investigate task", "/tmp", ctx=mock_ctx)) + assert result["success"] is False + + +class TestNotifyHelper: + """Unit tests for the centralized _notify() notification helper.""" + + @pytest.mark.anyio + async def test_notify_raises_value_error_for_reserved_key_name(self): + """The 'name' key that caused the original bug must be rejected.""" + from autoskillit.server._notify import _notify + + ctx = AsyncMock() + ctx.info = AsyncMock() + with pytest.raises(ValueError, match="reserved LogRecord"): + await _notify( + ctx, + "info", + "migrate_recipe: foo", + "autoskillit.migrate_recipe", + extra={"name": "foo"}, + ) + ctx.info.assert_not_awaited() + + @pytest.mark.anyio + async def test_notify_raises_for_all_reserved_keys(self): + """Every key in RESERVED_LOG_RECORD_KEYS must be rejected.""" + from autoskillit.core.types import RESERVED_LOG_RECORD_KEYS + from autoskillit.server._notify import _notify + + ctx = AsyncMock() + ctx.info = AsyncMock() + for reserved_key in RESERVED_LOG_RECORD_KEYS: + with pytest.raises(ValueError, match="reserved LogRecord"): + await _notify(ctx, "info", "msg", "logger", extra={reserved_key: "value"}) + + @pytest.mark.anyio + async def test_notify_accepts_safe_key_recipe_name(self): + """'recipe_name' (the corrected key for migrate_recipe) must be accepted.""" + from autoskillit.server._notify import _notify + + ctx = AsyncMock() + ctx.info = AsyncMock() + await _notify( + ctx, + "info", + "migrate_recipe: foo", + "autoskillit.migrate_recipe", + extra={"recipe_name": "foo"}, + ) + ctx.info.assert_awaited_once_with( + "migrate_recipe: foo", + logger_name="autoskillit.migrate_recipe", + extra={"recipe_name": "foo"}, + ) + + @pytest.mark.anyio + async def test_notify_accepts_none_extra(self): + from autoskillit.server._notify import _notify + + ctx = AsyncMock() + ctx.info = AsyncMock() + await _notify(ctx, "info", "msg", "logger") # no extra + ctx.info.assert_awaited_once() + + @pytest.mark.anyio + async def test_notify_accepts_empty_extra(self): + from autoskillit.server._notify import _notify + + ctx = AsyncMock() + ctx.info = AsyncMock() + await _notify(ctx, "info", "msg", "logger", extra={}) + ctx.info.assert_awaited_once() + + @pytest.mark.anyio + async def test_notify_swallows_attribute_error_from_ctx(self): + """Contract: must not raise even when ctx.info raises AttributeError + (e.g. _CurrentContext sentinel). Test completion is the assertion.""" + from autoskillit.server._notify import _notify + + ctx = AsyncMock() + ctx.info = AsyncMock(side_effect=AttributeError("no info")) + # Must not raise + await _notify(ctx, "info", "msg", "logger", extra={"cwd": "/tmp"}) + + @pytest.mark.anyio + async def test_notify_swallows_runtime_error_from_ctx(self): + """Contract: must not raise even when ctx.info raises RuntimeError + (no active MCP session). Test completion is the assertion.""" + from autoskillit.server._notify import _notify + + ctx = AsyncMock() + ctx.info = AsyncMock(side_effect=RuntimeError("session not available")) + await _notify(ctx, "info", "msg", "logger", extra={"cwd": "/tmp"}) + + @pytest.mark.anyio + async def test_notify_swallows_key_error_from_ctx(self): + """Contract: must not raise even when ctx.info raises KeyError + (FastMCP stdlib logging path). Test completion is the assertion.""" + from autoskillit.server._notify import _notify + + ctx = AsyncMock() + ctx.info = AsyncMock(side_effect=KeyError("Attempt to overwrite 'name' in LogRecord")) + await _notify(ctx, "info", "msg", "logger", extra={"cwd": "/tmp"}) + + @pytest.mark.anyio + async def test_notify_dispatches_error_level(self): + from autoskillit.server._notify import _notify + + ctx = AsyncMock() + ctx.error = AsyncMock() + await _notify( + ctx, + "error", + "run_cmd failed", + "autoskillit.run_cmd", + extra={"exit_code": 1}, + ) + ctx.error.assert_awaited_once_with( + "run_cmd failed", + logger_name="autoskillit.run_cmd", + extra={"exit_code": 1}, + ) + + +class TestHeadlessGateEnforcement: + """T_HGE: run_skill, run_cmd, run_python each return headless_error + when the session is running with AUTOSKILLIT_HEADLESS=1 and SESSION_TYPE=skill. + + The gate is open (tool_ctx default), so _require_enabled() passes. + _require_orchestrator_or_higher() fires first and returns subtype='headless_error'. + """ + + @pytest.fixture(autouse=True) + def _set_headless_env(self, monkeypatch): + monkeypatch.setenv("AUTOSKILLIT_HEADLESS", "1") + monkeypatch.setenv("AUTOSKILLIT_SESSION_TYPE", "skill") + + @pytest.mark.anyio + async def test_run_skill_blocked_in_headless_session(self, tool_ctx): + """run_skill returns headless_error when AUTOSKILLIT_HEADLESS=1 and SESSION_TYPE=skill.""" + result = json.loads(await run_skill("/autoskillit:investigate some-error", "/tmp")) + assert result["subtype"] == "headless_error" + + +@pytest.mark.feature("fleet") +class TestTierAwareGateEnforcement: + """T_TAGE: tier-aware guard permits orchestrator, denies skill and fleet as appropriate.""" + + @pytest.mark.anyio + async def test_run_skill_permitted_for_orchestrator_tier( + self, tool_ctx_kitchen_open, monkeypatch + ): + """run_skill does NOT return headless_error for orchestrator-tier headless sessions.""" + monkeypatch.setenv("AUTOSKILLIT_HEADLESS", "1") + monkeypatch.setenv("AUTOSKILLIT_SESSION_TYPE", "orchestrator") + tool_ctx_kitchen_open.runner.push( + _make_result(returncode=1) + ) # clone guard snapshot (not a git repo) + tool_ctx_kitchen_open.runner.push( + _make_result( + returncode=0, + stdout=json.dumps({"type": "result", "subtype": "success", "is_error": False}), + ) + ) + result = json.loads(await run_skill("/autoskillit:investigate some-error", "/tmp")) + assert result.get("cli_subtype") == "success" + + @pytest.mark.anyio + async def test_run_skill_denied_for_skill_tier(self, tool_ctx, monkeypatch): + """run_skill returns headless_error for skill-tier headless sessions.""" + monkeypatch.setenv("AUTOSKILLIT_HEADLESS", "1") + monkeypatch.setenv("AUTOSKILLIT_SESSION_TYPE", "skill") + result = json.loads(await run_skill("/autoskillit:investigate some-error", "/tmp")) + assert result["subtype"] == "headless_error" + + @pytest.mark.anyio + async def test_open_kitchen_denied_for_fleet_tier(self, tool_ctx, monkeypatch): + """open_kitchen returns HeadlessDenied for fleet-tier sessions.""" + from autoskillit.server.tools.tools_kitchen import open_kitchen + + monkeypatch.setenv("AUTOSKILLIT_HEADLESS", "1") + monkeypatch.setenv("AUTOSKILLIT_SESSION_TYPE", "fleet") + result = json.loads(await open_kitchen()) + assert result.get("error") == "HeadlessDenied" + msg = result.get("user_visible_message", "").lower() + assert "fleet" in msg + + @pytest.mark.anyio + async def test_close_kitchen_denied_for_fleet_tier(self, tool_ctx, monkeypatch): + """close_kitchen returns headless_error for fleet-tier sessions.""" + from autoskillit.server.tools.tools_kitchen import close_kitchen + + monkeypatch.setenv("AUTOSKILLIT_HEADLESS", "1") + monkeypatch.setenv("AUTOSKILLIT_SESSION_TYPE", "fleet") + result = json.loads(await close_kitchen()) + assert result["subtype"] == "headless_error" + + +class TestResponseFieldsAreTypeSafe: + """Every discriminator field in MCP tool responses uses enum values.""" + + @pytest.mark.anyio + async def test_retry_reason_is_enum_value(self, tool_ctx): + stdout = json.dumps( + { + "type": "result", + "subtype": "error_max_turns", + "is_error": False, + "session_id": "s1", + "num_turns": 200, + "errors": [], + } + ) + tool_ctx.runner.push(_make_result(1, stdout, "")) + result = json.loads(await run_skill("/retry-worktree plan.md", "/tmp")) + assert result["retry_reason"] in {e.value for e in RetryReason} + + @pytest.mark.anyio + async def test_retry_reason_none_is_enum_value(self, tool_ctx): + stdout = json.dumps( + { + "type": "result", + "subtype": "success", + "is_error": False, + "result": "Done.", + "session_id": "s1", + "num_turns": 50, + } + ) + tool_ctx.runner.push(_make_result(0, stdout, "")) + result = json.loads(await run_skill("/retry-worktree plan.md", "/tmp")) + assert result["retry_reason"] in {e.value for e in RetryReason} + + +class TestRunSkillTiming: + """run_skill accumulates wall-clock timing when step_name is provided.""" + + @pytest.mark.anyio + async def test_run_skill_records_timing_via_step_name(self, tool_ctx_kitchen_open): + tool_ctx_kitchen_open.runner.push(_make_result(0, _SUCCESS_JSON, "")) + await run_skill("/investigate foo", "/tmp", step_name="implement") + assert_step_timed(tool_ctx_kitchen_open.timing_log, "implement") + + @pytest.mark.anyio + async def test_run_skill_empty_step_name_skips_timing(self, tool_ctx): + tool_ctx.runner.push(_make_result(0, _SUCCESS_JSON, "")) + await run_skill("/investigate foo", "/tmp") + assert_no_timing(tool_ctx.timing_log) + + +class TestRunHeadlessCoreFlushTelemetry: + """flush_session_log receives telemetry kwargs when step_name is provided.""" + + def _make_ndjson_with_usage(self) -> str: + asst = json.dumps( + { + "type": "assistant", + "message": { + "usage": { + "input_tokens": 200, + "output_tokens": 100, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + } + }, + "model": "claude-opus-4-6", + } + ) + result = json.dumps( + { + "type": "result", + "subtype": "success", + "is_error": False, + "result": "done", + "session_id": "s1", + "usage": { + "input_tokens": 200, + "output_tokens": 100, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + }, + } + ) + return asst + "\n" + result + + @pytest.mark.anyio + async def test_passes_step_telemetry_to_flush(self, tool_ctx_kitchen_open, monkeypatch): + """flush_session_log is called with step_name, token_usage, and timing_seconds.""" + import autoskillit.execution.session_log as sl_mod + + calls = [] + + def mock_flush(**kwargs): + calls.append(kwargs) + + monkeypatch.setattr(sl_mod, "flush_session_log", mock_flush) + tool_ctx_kitchen_open.runner.push(_make_result(returncode=1)) # clone guard snapshot + tool_ctx_kitchen_open.runner.push( + _make_result(returncode=0, stdout=self._make_ndjson_with_usage()) + ) + await run_skill("/investigate foo", "/tmp", step_name="implement") + assert len(calls) == 1 + assert calls[0]["step_name"] == "implement" + assert calls[0]["telemetry"].token_usage is not None + assert calls[0]["telemetry"].timing_seconds is not None + + @pytest.mark.anyio + async def test_flush_session_log_session_id_matches_returned_skill_result( + self, tool_ctx_kitchen_open, monkeypatch + ): + """flush_session_log receives the same session_id as the returned SkillResult.""" + import autoskillit.execution.session_log as sl_mod + from autoskillit.core.types import SubprocessResult, TerminationReason + + calls = [] + + def mock_flush(**kwargs): + calls.append(kwargs) + + monkeypatch.setattr(sl_mod, "flush_session_log", mock_flush) + tool_ctx_kitchen_open.runner.push(_make_result(returncode=1)) # clone guard snapshot + # Stale result with session_id resolved from Channel B + stale_result = SubprocessResult( + returncode=-1, + stdout="", + stderr="", + termination=TerminationReason.STALE, + pid=12345, + session_id="test-uuid-coherence-check", + ) + tool_ctx_kitchen_open.runner.push(stale_result) + result_json = json.loads( + await run_skill("/investigate foo", "/tmp", step_name="implement") + ) + assert len(calls) == 1 + # flush and returned SkillResult must carry the same session_id + assert calls[0]["session_id"] == result_json["session_id"] + assert result_json["session_id"] != "" + + @pytest.mark.anyio + async def test_flushes_on_success_when_step_name_set(self, tool_ctx_kitchen_open, monkeypatch): + """Successful sessions without proc_snapshots still flush when step_name is provided.""" + import autoskillit.execution.session_log as sl_mod + + calls = [] + + def mock_flush(**kwargs): + calls.append(kwargs) + + monkeypatch.setattr(sl_mod, "flush_session_log", mock_flush) + tool_ctx_kitchen_open.runner.push(_make_result(returncode=0, stdout=_SUCCESS_JSON)) + await run_skill("/investigate foo", "/tmp", step_name="plan") + assert len(calls) == 1 + + @pytest.mark.anyio + async def test_records_timing_in_timing_log(self, tool_ctx_kitchen_open): + """ctx.timing_log.record() is called with step_name and computed timing_seconds.""" + tool_ctx_kitchen_open.runner.push(_make_result(returncode=0, stdout=_SUCCESS_JSON)) + await run_skill("/investigate foo", "/tmp", step_name="plan") + report = tool_ctx_kitchen_open.timing_log.get_report() + assert len(report) == 1 + assert report[0]["step_name"] == "plan" + assert report[0]["total_seconds"] >= 0.0 + + @pytest.mark.anyio + async def test_passes_github_api_log_to_flush(self, tool_ctx_kitchen_open, monkeypatch): + """headless.py drains github_api_log into telemetry.github_api_usage.""" + import autoskillit.execution.session_log as sl_mod + from autoskillit.pipeline.github_api_log import DefaultGitHubApiLog + + log = DefaultGitHubApiLog() + await log.record_gh_cli( + subcommand="gh issue list", + exit_code=0, + latency_ms=50.0, + timestamp="2026-05-02T10:00:00Z", + ) + tool_ctx_kitchen_open.github_api_log = log + + calls = [] + + def mock_flush(**kwargs): + calls.append(kwargs) + + monkeypatch.setattr(sl_mod, "flush_session_log", mock_flush) + tool_ctx_kitchen_open.runner.push( + _make_result(returncode=0, stdout=self._make_ndjson_with_usage()) + ) + await run_skill("/investigate foo", "/tmp", step_name="implement") + assert len(calls) == 1 + assert calls[0]["telemetry"].github_api_usage is not None + assert calls[0]["telemetry"].github_api_requests > 0 + + @pytest.mark.anyio + async def test_flush_telemetry_kwargs_exhaustive(self, tool_ctx_kitchen_open, monkeypatch): + """headless.py passes a SessionTelemetry bundle covering all telemetry fields.""" + import autoskillit.execution.session_log as sl_mod + from autoskillit.core.types._type_results_execution import SessionTelemetry + + calls = [] + + def mock_flush(**kwargs): + calls.append(kwargs) + + monkeypatch.setattr(sl_mod, "flush_session_log", mock_flush) + tool_ctx_kitchen_open.runner.push( + _make_result(returncode=0, stdout=self._make_ndjson_with_usage()) + ) + await run_skill("/investigate foo", "/tmp", step_name="implement") + assert len(calls) == 1 + assert "telemetry" in calls[0], "flush_session_log must receive telemetry= kwarg" + assert isinstance(calls[0]["telemetry"], SessionTelemetry), ( + "telemetry must be a SessionTelemetry instance" + ) + + +@pytest.mark.anyio +async def test_run_skill_returns_structured_error_when_executor_raises( + tool_ctx_kitchen_open, monkeypatch, tmp_path +) -> None: + """run_skill returns SkillResult-shaped JSON even if executor.run() raises unexpectedly.""" + from autoskillit.core import SkillResult + + class ExplodingExecutor: + async def run(self, *args, **kwargs) -> SkillResult: + raise RuntimeError("unexpected executor failure") + + tool_ctx_kitchen_open.executor = ExplodingExecutor() + monkeypatch.setattr("autoskillit.server._ctx", tool_ctx_kitchen_open) + + from autoskillit.server.tools.tools_execution import run_skill + + result_json = await run_skill("/test cmd", str(tmp_path)) + data = json.loads(result_json) + assert data["success"] is False + assert data["subtype"] == "crashed" + assert data["needs_retry"] is False + assert "unexpected executor failure" in data["result"] diff --git a/tests/server/test_tools_execution_routing.py b/tests/server/test_tools_execution_routing.py new file mode 100644 index 0000000000..13137bfeee --- /dev/null +++ b/tests/server/test_tools_execution_routing.py @@ -0,0 +1,380 @@ +"""Tests for run_skill routing, executor delegation, and session skill management.""" + +from __future__ import annotations + +import pytest + +from autoskillit.server.tools.tools_execution import run_skill + +pytestmark = [pytest.mark.layer("server"), pytest.mark.small] + + +@pytest.mark.anyio +async def test_tools_execution_routes_through_executor(tool_ctx_kitchen_open, monkeypatch) -> None: + """run_skill routes through ctx.executor.run(), not run_headless_core directly.""" + from tests.fakes import InMemoryHeadlessExecutor + + executor = InMemoryHeadlessExecutor() + tool_ctx_kitchen_open.executor = executor + monkeypatch.setattr("autoskillit.server._ctx", tool_ctx_kitchen_open) + + await run_skill("/test skill", "/tmp") + assert len(executor.calls) == 1 + assert executor.calls[0].skill_command == "/test skill" + assert executor.calls[0].cwd == "/tmp" + + +@pytest.mark.anyio +async def test_run_skill_passes_validated_add_dirs(tool_ctx_kitchen_open, monkeypatch) -> None: + """run_skill passes ValidatedAddDir instances (not raw strings) as add_dirs.""" + from autoskillit.core import ValidatedAddDir + from tests.fakes import InMemoryHeadlessExecutor + + executor = InMemoryHeadlessExecutor() + tool_ctx_kitchen_open.executor = executor + monkeypatch.setattr("autoskillit.server._ctx", tool_ctx_kitchen_open) + + await run_skill("/test skill", "/tmp") + # All add_dirs must be ValidatedAddDir instances + assert len(executor.calls[0].add_dirs) >= 1 + assert all(isinstance(d, ValidatedAddDir) for d in executor.calls[0].add_dirs) + # Must not include raw skills_extended/ path + from autoskillit.workspace.skills import bundled_skills_extended_dir + + skills_ext = str(bundled_skills_extended_dir()) + add_dir_paths = [d.path for d in executor.calls[0].add_dirs] + assert skills_ext not in add_dir_paths + + +@pytest.mark.anyio +async def test_run_skill_calls_session_skill_manager_init_session( + tool_ctx_kitchen_open, monkeypatch +) -> None: + """run_skill routes through session_skill_manager.init_session() for add_dirs.""" + from unittest.mock import MagicMock + + from autoskillit.core import ValidatedAddDir + + # Create a spy on init_session + fake_validated = ValidatedAddDir(path="/fake/session/dir") + mock_ssm = MagicMock() + mock_ssm.init_session.return_value = fake_validated + tool_ctx_kitchen_open.session_skill_manager = mock_ssm + + from tests.fakes import InMemoryHeadlessExecutor + + executor = InMemoryHeadlessExecutor() + tool_ctx_kitchen_open.executor = executor + monkeypatch.setattr("autoskillit.server._ctx", tool_ctx_kitchen_open) + + await run_skill("/test skill", "/tmp") + + # init_session was called with cook_session=False (headless, not cook) + mock_ssm.init_session.assert_called_once() + call_kwargs = mock_ssm.init_session.call_args + assert call_kwargs.kwargs.get("cook_session") is False + + # The returned ValidatedAddDir is in add_dirs + assert fake_validated in executor.calls[0].add_dirs + + +@pytest.mark.anyio +async def test_run_skill_activates_deps_for_tier3_target( + tool_ctx_kitchen_open, monkeypatch +) -> None: + """run_skill calls activate_skill_deps even when target is tier3 (not in tier2 list).""" + from unittest.mock import MagicMock + + from autoskillit.core import ValidatedAddDir + + fake_validated = ValidatedAddDir(path="/fake/session/dir") + mock_ssm = MagicMock() + mock_ssm.init_session.return_value = fake_validated + tool_ctx_kitchen_open.session_skill_manager = mock_ssm + + # Set up skill_resolver to produce a resolved name + mock_resolver = MagicMock() + mock_resolver.resolve.return_value = MagicMock(source=MagicMock(value="bundled_extended")) + tool_ctx_kitchen_open.skill_resolver = mock_resolver + + from tests.fakes import InMemoryHeadlessExecutor + + tool_ctx_kitchen_open.executor = InMemoryHeadlessExecutor() + monkeypatch.setattr("autoskillit.server._ctx", tool_ctx_kitchen_open) + + # Use a tier3 skill name + await run_skill("/open-pr", "/tmp") + + # activate_skill_deps must have been called regardless of tier + mock_ssm.activate_skill_deps.assert_called_once() + + +@pytest.mark.anyio +async def test_run_skill_result_includes_order_id_when_passed( + tool_ctx_kitchen_open, monkeypatch +) -> None: + """run_skill injects order_id into the result JSON when order_id is non-empty.""" + import json as _json + + from tests.fakes import InMemoryHeadlessExecutor + + tool_ctx_kitchen_open.executor = InMemoryHeadlessExecutor() + monkeypatch.setattr("autoskillit.server._ctx", tool_ctx_kitchen_open) + + result_json = await run_skill("/test skill", "/tmp", order_id="issue-185") + data = _json.loads(result_json) + assert data.get("order_id") == "issue-185" + + +@pytest.mark.anyio +async def test_run_skill_result_order_id_empty_string_when_not_passed( + tool_ctx_kitchen_open, monkeypatch +) -> None: + """run_skill emits order_id as empty string in result JSON when none provided.""" + import json as _json + + from tests.fakes import InMemoryHeadlessExecutor + + tool_ctx_kitchen_open.executor = InMemoryHeadlessExecutor() + monkeypatch.setattr("autoskillit.server._ctx", tool_ctx_kitchen_open) + + result_json = await run_skill("/test skill", "/tmp") # no order_id + data = _json.loads(result_json) + assert data.get("order_id") == "" + + +@pytest.mark.anyio +async def test_run_skill_passes_allow_only_to_init_session( + tool_ctx_kitchen_open, monkeypatch +) -> None: + """run_skill computes the closure for the resolved target and forwards it as allow_only.""" + from unittest.mock import MagicMock + + from autoskillit.core import ValidatedAddDir + from tests.fakes import InMemoryHeadlessExecutor + + fake_validated = ValidatedAddDir(path="/fake/session/dir") + expected_closure = frozenset({"investigate", "mermaid"}) + + mock_ssm = MagicMock() + mock_ssm.init_session.return_value = fake_validated + mock_ssm.compute_skill_closure.return_value = expected_closure + tool_ctx_kitchen_open.session_skill_manager = mock_ssm + + mock_resolver = MagicMock() + mock_resolver.resolve.return_value = MagicMock(source=MagicMock(value="bundled_extended")) + tool_ctx_kitchen_open.skill_resolver = mock_resolver + + tool_ctx_kitchen_open.executor = InMemoryHeadlessExecutor() + monkeypatch.setattr("autoskillit.server._ctx", tool_ctx_kitchen_open) + + await run_skill("/autoskillit:investigate the bug", "/tmp") + + mock_ssm.compute_skill_closure.assert_called_once_with("investigate") + mock_ssm.init_session.assert_called_once() + assert mock_ssm.init_session.call_args.kwargs.get("allow_only") == expected_closure + + +@pytest.mark.anyio +async def test_run_skill_no_target_skill_passes_none_allow_only( + tool_ctx_kitchen_open, monkeypatch +) -> None: + """When skill_resolver is unset, target_name is None and allow_only stays None.""" + from unittest.mock import MagicMock + + from autoskillit.core import ValidatedAddDir + from tests.fakes import InMemoryHeadlessExecutor + + fake_validated = ValidatedAddDir(path="/fake/session/dir") + mock_ssm = MagicMock() + mock_ssm.init_session.return_value = fake_validated + tool_ctx_kitchen_open.session_skill_manager = mock_ssm + tool_ctx_kitchen_open.skill_resolver = None # disables resolve_target_skill + + tool_ctx_kitchen_open.executor = InMemoryHeadlessExecutor() + monkeypatch.setattr("autoskillit.server._ctx", tool_ctx_kitchen_open) + + await run_skill("/test skill", "/tmp") + + mock_ssm.init_session.assert_called_once() + assert mock_ssm.init_session.call_args.kwargs.get("allow_only") is None + mock_ssm.compute_skill_closure.assert_not_called() + + +@pytest.mark.anyio +async def test_run_skill_make_plan_closure_includes_arch_lens_pack( + tool_ctx_kitchen_open, monkeypatch +) -> None: + """End-to-end: /make-plan resolves a closure containing the entire arch-lens pack.""" + from unittest.mock import MagicMock + + from autoskillit.core import ValidatedAddDir + from autoskillit.workspace.session_skills import ( + DefaultSessionSkillManager, + SkillsDirectoryProvider, + ) + from tests.fakes import InMemoryHeadlessExecutor + + real_provider = SkillsDirectoryProvider() + real_mgr = DefaultSessionSkillManager( + provider=real_provider, ephemeral_root=tool_ctx_kitchen_open.temp_dir + ) + + captured: dict = {} + + class _RecordingManager: + def __init__(self, real: DefaultSessionSkillManager) -> None: + self._real = real + + def init_session(self, session_id, **kwargs): + captured["allow_only"] = kwargs.get("allow_only") + return ValidatedAddDir(path="/fake/session/dir") + + def compute_skill_closure(self, target_name): + return self._real.compute_skill_closure(target_name) + + def activate_skill_deps(self, session_id, skill_name): + return True + + def cleanup_stale(self, max_age_seconds=86400): + return 0 + + tool_ctx_kitchen_open.session_skill_manager = _RecordingManager(real_mgr) + + mock_resolver = MagicMock() + mock_resolver.resolve.return_value = MagicMock(source=MagicMock(value="bundled_extended")) + tool_ctx_kitchen_open.skill_resolver = mock_resolver + + tool_ctx_kitchen_open.executor = InMemoryHeadlessExecutor() + monkeypatch.setattr("autoskillit.server._ctx", tool_ctx_kitchen_open) + + await run_skill("/autoskillit:make-plan refactor", "/tmp") + + closure = captured["allow_only"] + assert closure is not None + assert "make-plan" in closure + assert "mermaid" in closure + arch_members = {n for n in closure if n.startswith("arch-lens-")} + assert len(arch_members) >= 1 + + +@pytest.mark.anyio +async def test_run_skill_passes_idle_output_timeout(tool_ctx_kitchen_open, monkeypatch) -> None: + """run_skill passes idle_output_timeout (as float) to executor.run().""" + from tests.fakes import InMemoryHeadlessExecutor + + executor = InMemoryHeadlessExecutor() + tool_ctx_kitchen_open.executor = executor + monkeypatch.setattr("autoskillit.server._ctx", tool_ctx_kitchen_open) + + await run_skill("/test skill", "/tmp", idle_output_timeout=120) + assert executor.calls[0].idle_output_timeout == 120.0 # int→float conversion + + +@pytest.mark.anyio +async def test_run_skill_idle_output_timeout_defaults_to_none( + tool_ctx_kitchen_open, monkeypatch +) -> None: + """run_skill passes None to executor.run() when idle_output_timeout is not set.""" + from tests.fakes import InMemoryHeadlessExecutor + + executor = InMemoryHeadlessExecutor() + tool_ctx_kitchen_open.executor = executor + monkeypatch.setattr("autoskillit.server._ctx", tool_ctx_kitchen_open) + + await run_skill("/test skill", "/tmp") + assert executor.calls[0].idle_output_timeout is None + + +class TestOutputDirParameter: + """output_dir parameter plumbing from run_skill to executor.""" + + def test_run_skill_has_output_dir_parameter(self) -> None: + """run_skill() accepts output_dir parameter.""" + import inspect + + sig = inspect.signature(run_skill) + assert "output_dir" in sig.parameters + param = sig.parameters["output_dir"] + assert param.default == "" + + @pytest.mark.anyio + async def test_run_skill_forwards_output_dir_to_write_watch_dirs( + self, tool_ctx_kitchen_open, monkeypatch, tmp_path + ) -> None: + """output_dir is resolved and forwarded to executor.run() as write_watch_dirs.""" + from pathlib import Path + + from tests.fakes import InMemoryHeadlessExecutor + + executor = InMemoryHeadlessExecutor() + tool_ctx_kitchen_open.executor = executor + monkeypatch.setattr("autoskillit.server._ctx", tool_ctx_kitchen_open) + + output_dir = str(tmp_path / "output") + await run_skill("/test skill", str(tmp_path), output_dir=output_dir) + + assert len(executor.calls) == 1 + assert Path(output_dir) in executor.calls[0].write_watch_dirs + + +@pytest.mark.anyio +async def test_run_skill_injects_provider_extras_when_feature_enabled( + tool_ctx_kitchen_open, monkeypatch, tmp_path +) -> None: + """run_skill records provider_extras and profile_name when feature is enabled.""" + from tests.fakes import InMemoryHeadlessExecutor + + executor = InMemoryHeadlessExecutor() + tool_ctx_kitchen_open.executor = executor + monkeypatch.setattr("autoskillit.server._ctx", tool_ctx_kitchen_open) + monkeypatch.setattr("autoskillit.core.is_feature_enabled", lambda *a, **kw: True) + monkeypatch.setattr( + "autoskillit.server._guards._resolve_provider_profile", + lambda *a: ("vertex", {"ANTHROPIC_API_KEY": "test-key-xyz"}), + ) + + await run_skill("/autoskillit:probe", str(tmp_path)) + + assert executor.calls[0].provider_extras == {"ANTHROPIC_API_KEY": "test-key-xyz"} + assert executor.calls[0].profile_name == "vertex" + + +@pytest.mark.anyio +async def test_run_skill_provider_extras_none_when_feature_disabled( + tool_ctx_kitchen_open, monkeypatch, tmp_path +) -> None: + """run_skill records None provider_extras and empty profile_name when feature is disabled.""" + from tests.fakes import InMemoryHeadlessExecutor + + executor = InMemoryHeadlessExecutor() + tool_ctx_kitchen_open.executor = executor + monkeypatch.setattr("autoskillit.server._ctx", tool_ctx_kitchen_open) + monkeypatch.setattr("autoskillit.core.is_feature_enabled", lambda *a, **kw: False) + + await run_skill("/autoskillit:probe", str(tmp_path)) + + assert executor.calls[0].provider_extras is None + assert executor.calls[0].profile_name == "" + + +@pytest.mark.anyio +async def test_run_skill_provider_extras_none_when_default_profile( + tool_ctx_kitchen_open, monkeypatch, tmp_path +) -> None: + """run_skill records None provider_extras when profile resolves to default anthropic.""" + from tests.fakes import InMemoryHeadlessExecutor + + executor = InMemoryHeadlessExecutor() + tool_ctx_kitchen_open.executor = executor + monkeypatch.setattr("autoskillit.server._ctx", tool_ctx_kitchen_open) + monkeypatch.setattr("autoskillit.core.is_feature_enabled", lambda *a, **kw: True) + monkeypatch.setattr( + "autoskillit.server._guards._resolve_provider_profile", + lambda *a: ("anthropic", {}), + ) + + await run_skill("/autoskillit:probe", str(tmp_path)) + + assert executor.calls[0].provider_extras is None + assert executor.calls[0].profile_name == "" diff --git a/tests/server/test_tools_git.py b/tests/server/test_tools_git.py index 797fa33386..3b5c60dd6b 100644 --- a/tests/server/test_tools_git.py +++ b/tests/server/test_tools_git.py @@ -12,13 +12,16 @@ from autoskillit.config import AutomationConfig, ClassifyFixConfig from autoskillit.core import CleanupResult from autoskillit.core.types import MergeFailedStep, MergeState, RestartScope -from autoskillit.server.tools_git import ( +from autoskillit.server.tools.tools_git import ( check_pr_mergeable, classify_fix, create_unique_branch, merge_worktree, ) from tests.conftest import _make_result +from tests.server.conftest import assert_no_timing, assert_step_timed + +pytestmark = [pytest.mark.layer("server"), pytest.mark.small] class TestClassifyFix: @@ -38,10 +41,10 @@ def _set_prefixes(self, tool_ctx): ) @pytest.mark.anyio - async def test_critical_files_return_full_restart(self, tool_ctx, tmp_path): + async def test_critical_files_return_full_restart(self, tool_ctx_kitchen_open, tmp_path): changed = "src/core/handler.py\nlib/utils/helpers.py\n" - tool_ctx.runner.push(_make_result(0, "", "")) # git fetch succeeds - tool_ctx.runner.push(_make_result(0, changed, "")) + tool_ctx_kitchen_open.runner.push(_make_result(0, "", "")) # git fetch succeeds + tool_ctx_kitchen_open.runner.push(_make_result(0, changed, "")) result = json.loads(await classify_fix(worktree_path=str(tmp_path), base_branch="main")) @@ -51,10 +54,10 @@ async def test_critical_files_return_full_restart(self, tool_ctx, tmp_path): assert len(result["all_changed_files"]) == 2 @pytest.mark.anyio - async def test_non_critical_returns_partial_restart(self, tool_ctx, tmp_path): + async def test_non_critical_returns_partial_restart(self, tool_ctx_kitchen_open, tmp_path): changed = "src/workers/runner.py\nlib/utils/helpers.py\n" - tool_ctx.runner.push(_make_result(0, "", "")) # git fetch succeeds - tool_ctx.runner.push(_make_result(0, changed, "")) + tool_ctx_kitchen_open.runner.push(_make_result(0, "", "")) # git fetch succeeds + tool_ctx_kitchen_open.runner.push(_make_result(0, changed, "")) result = json.loads(await classify_fix(worktree_path=str(tmp_path), base_branch="main")) @@ -63,9 +66,9 @@ async def test_non_critical_returns_partial_restart(self, tool_ctx, tmp_path): assert len(result["all_changed_files"]) == 2 @pytest.mark.anyio - async def test_git_diff_failure(self, tool_ctx, tmp_path): - tool_ctx.runner.push(_make_result(0, "", "")) # git fetch succeeds - tool_ctx.runner.push(_make_result(128, "", "fatal: bad revision")) + async def test_git_diff_failure(self, tool_ctx_kitchen_open, tmp_path): + tool_ctx_kitchen_open.runner.push(_make_result(0, "", "")) # git fetch succeeds + tool_ctx_kitchen_open.runner.push(_make_result(128, "", "fatal: bad revision")) result = json.loads(await classify_fix(worktree_path=str(tmp_path), base_branch="main")) @@ -73,30 +76,49 @@ async def test_git_diff_failure(self, tool_ctx, tmp_path): assert "Cannot diff" in result["reason"] @pytest.mark.anyio - async def test_critical_path_in_diff_triggers_full_restart(self, tool_ctx, tmp_path): + async def test_critical_path_in_diff_triggers_full_restart( + self, tool_ctx_kitchen_open, tmp_path + ): changed = "src/api/routes.py\n" - tool_ctx.runner.push(_make_result(0, "", "")) # git fetch succeeds - tool_ctx.runner.push(_make_result(0, changed, "")) + tool_ctx_kitchen_open.runner.push(_make_result(0, "", "")) # git fetch succeeds + tool_ctx_kitchen_open.runner.push(_make_result(0, changed, "")) result = json.loads(await classify_fix(worktree_path=str(tmp_path), base_branch="main")) assert result["restart_scope"] == RestartScope.FULL_RESTART @pytest.mark.anyio - async def test_classify_fix_nonexistent_worktree_path_returns_clear_error(self, tool_ctx): + async def test_classify_fix_nonexistent_worktree_path_returns_clear_error( + self, tool_ctx_kitchen_open + ): """[FAILS NOW] nonexistent path returns a distinct path-not-found error.""" result = json.loads(await classify_fix("/no/such/path", "main")) assert result["restart_scope"] == RestartScope.FULL_RESTART assert "does not exist" in result["reason"].lower() @pytest.mark.anyio - async def test_classify_fix_git_fetch_called_before_diff(self, tool_ctx, tmp_path): - """[FAILS NOW] git fetch must be issued before git diff.""" - tool_ctx.runner.push(_make_result(0, "", "")) # fetch succeeds - tool_ctx.runner.push(_make_result(0, "src/foo.py\n", "")) # diff succeeds - await classify_fix(str(tmp_path), "main") - assert tool_ctx.runner.call_args_list[0][0] == ["git", "fetch", "origin", "main"] - assert tool_ctx.runner.call_args_list[1][0][0:3] == ["git", "diff", "--name-only"] + async def test_classify_fix_git_fetch_called_before_diff( + self, tool_ctx_kitchen_open, tmp_path + ): + """git fetch must be issued before git diff.""" + tool_ctx_kitchen_open.runner.push(_make_result(0, "", "")) # fetch succeeds + tool_ctx_kitchen_open.runner.push(_make_result(0, "src/foo.py\n", "")) # diff succeeds + with patch( + "autoskillit.server._misc.resolve_remote_name", + new=AsyncMock(return_value="origin"), + ): + await classify_fix(str(tmp_path), "main") + assert tool_ctx_kitchen_open.runner.call_args_list[0][0] == [ + "git", + "fetch", + "origin", + "main", + ] + assert tool_ctx_kitchen_open.runner.call_args_list[1][0][0:3] == [ + "git", + "diff", + "--name-only", + ] @pytest.mark.anyio async def test_classify_fix_gate_closed_returns_gate_error( @@ -111,11 +133,11 @@ async def test_classify_fix_gate_closed_returns_gate_error( @pytest.mark.anyio async def test_classify_fix_empty_diff_returns_partial_restart_with_no_files( - self, tool_ctx, tmp_path + self, tool_ctx_kitchen_open, tmp_path ): """[NEW COVERAGE] empty diff is a valid state returning partial_restart with no files.""" - tool_ctx.runner.push(_make_result(0, "", "")) # fetch - tool_ctx.runner.push(_make_result(0, "", "")) # diff (empty) + tool_ctx_kitchen_open.runner.push(_make_result(0, "", "")) # fetch + tool_ctx_kitchen_open.runner.push(_make_result(0, "", "")) # diff (empty) result = json.loads(await classify_fix(str(tmp_path), "main")) assert result["restart_scope"] == RestartScope.PARTIAL_RESTART assert result["all_changed_files"] == [] @@ -126,19 +148,27 @@ class TestMergeWorktree: """merge_worktree enforces test gate, rebases, and merges.""" @pytest.mark.anyio - async def test_merge_worktree_blocks_on_failing_tests(self, tool_ctx, tmp_path): + async def test_merge_worktree_blocks_on_failing_tests(self, tool_ctx_kitchen_open, tmp_path): """merge_worktree returns error with failed_step when test-check fails.""" wt = tmp_path / "worktree" wt.mkdir() (wt / ".git").write_text("gitdir: /repo/.git/worktrees/wt") - tool_ctx.runner.push( + tool_ctx_kitchen_open.runner.push( _make_result(0, "/repo/.git/worktrees/wt\n", "") ) # rev-parse --git-dir - tool_ctx.runner.push(_make_result(0, "impl-branch\n", "")) # branch --show-current - tool_ctx.runner.push(_make_result(0, "", "")) # git ls-files (pre-dirty-tree check) - tool_ctx.runner.push(_make_result(0, "", "")) # git status --porcelain (clean) - tool_ctx.runner.push(_make_result(1, "FAIL\n= 3 failed, 97 passed =", "")) # test-check + tool_ctx_kitchen_open.runner.push( + _make_result(0, "impl-branch\n", "") + ) # branch --show-current + tool_ctx_kitchen_open.runner.push( + _make_result(0, "", "") + ) # git ls-files (pre-dirty-tree check) + tool_ctx_kitchen_open.runner.push( + _make_result(0, "", "") + ) # git status --porcelain (clean) + tool_ctx_kitchen_open.runner.push( + _make_result(1, "FAIL\n= 3 failed, 97 passed =", "") + ) # test-check result = json.loads(await merge_worktree(str(wt), "dev")) assert "error" in result assert result["failed_step"] == MergeFailedStep.TEST_GATE @@ -146,25 +176,37 @@ async def test_merge_worktree_blocks_on_failing_tests(self, tool_ctx, tmp_path): assert "test_summary" not in result @pytest.mark.anyio - async def test_merge_worktree_merges_on_green(self, tool_ctx, tmp_path): + async def test_merge_worktree_merges_on_green(self, tool_ctx_kitchen_open, tmp_path): """merge_worktree performs rebase+merge when tests pass.""" wt = tmp_path / "worktree" wt.mkdir() (wt / ".git").write_text("gitdir: /repo/.git/worktrees/wt") - tool_ctx.runner.push(_make_result(0, "/repo/.git/worktrees/wt\n", "")) # rev-parse - tool_ctx.runner.push(_make_result(0, "impl-branch\n", "")) # branch - tool_ctx.runner.push(_make_result(0, "", "")) # git ls-files (pre-dirty-tree check) - tool_ctx.runner.push(_make_result(0, "", "")) # git status --porcelain (clean) - tool_ctx.runner.push(_make_result(0, "PASS\n= 100 passed =", "")) # pre-rebase test-check - tool_ctx.runner.push(_make_result(0, "", "")) # git fetch - tool_ctx.runner.push(_make_result(0, "abc123\n", "")) # rev-parse --verify (step 5.5) - tool_ctx.runner.push( + tool_ctx_kitchen_open.runner.push( + _make_result(0, "/repo/.git/worktrees/wt\n", "") + ) # rev-parse + tool_ctx_kitchen_open.runner.push(_make_result(0, "impl-branch\n", "")) # branch + tool_ctx_kitchen_open.runner.push( + _make_result(0, "", "") + ) # git ls-files (pre-dirty-tree check) + tool_ctx_kitchen_open.runner.push( + _make_result(0, "", "") + ) # git status --porcelain (clean) + tool_ctx_kitchen_open.runner.push( + _make_result(0, "PASS\n= 100 passed =", "") + ) # pre-rebase test-check + tool_ctx_kitchen_open.runner.push(_make_result(0, "", "")) # git fetch + tool_ctx_kitchen_open.runner.push( + _make_result(0, "abc123\n", "") + ) # rev-parse --verify (step 5.5) + tool_ctx_kitchen_open.runner.push( _make_result(0, "", "") ) # git log --merges (no merge commits — step 5.6) - tool_ctx.runner.push(_make_result(0, "", "")) # git rebase - tool_ctx.runner.push(_make_result(0, "PASS\n= 100 passed =", "")) # post-rebase test-check - tool_ctx.runner.push( + tool_ctx_kitchen_open.runner.push(_make_result(0, "", "")) # git rebase + tool_ctx_kitchen_open.runner.push( + _make_result(0, "PASS\n= 100 passed =", "") + ) # post-rebase test-check + tool_ctx_kitchen_open.runner.push( _make_result( 0, "worktree /repo\nHEAD abc123\nbranch refs/heads/dev\n\n" @@ -172,10 +214,15 @@ async def test_merge_worktree_merges_on_green(self, tool_ctx, tmp_path): "", ) ) # worktree list --porcelain - tool_ctx.runner.push(_make_result(0, "dev\n", "")) # step 7.5: branch --show-current - tool_ctx.runner.push(_make_result(0, "", "")) # git merge - tool_ctx.runner.push(_make_result(0, "", "")) # worktree remove - tool_ctx.runner.push(_make_result(0, "", "")) # branch -D + tool_ctx_kitchen_open.runner.push( + _make_result(0, "dev\n", "") + ) # step 7.5: branch --show-current + tool_ctx_kitchen_open.runner.push( + _make_result(0, "", "") + ) # step 7.6: git status --porcelain (clean) + tool_ctx_kitchen_open.runner.push(_make_result(0, "", "")) # git merge + tool_ctx_kitchen_open.runner.push(_make_result(0, "", "")) # worktree remove + tool_ctx_kitchen_open.runner.push(_make_result(0, "", "")) # branch -D result = json.loads(await merge_worktree(str(wt), "dev")) assert result["merge_succeeded"] is True assert result["into_branch"] == "dev" @@ -185,30 +232,42 @@ async def test_merge_worktree_merges_on_green(self, tool_ctx, tmp_path): # Verify merge command cwd is the main_repo (/repo) merge_call = next( args - for args in tool_ctx.runner.call_args_list + for args in tool_ctx_kitchen_open.runner.call_args_list if len(args[0]) > 1 and args[0][1] == "merge" ) assert merge_call[1] == Path("/repo") @pytest.mark.anyio - async def test_merge_worktree_aborts_on_rebase_failure(self, tool_ctx, tmp_path): + async def test_merge_worktree_aborts_on_rebase_failure(self, tool_ctx_kitchen_open, tmp_path): """merge_worktree runs rebase --abort and returns step-specific error.""" wt = tmp_path / "worktree" wt.mkdir() (wt / ".git").write_text("gitdir: /repo/.git/worktrees/wt") - tool_ctx.runner.push(_make_result(0, "/repo/.git/worktrees/wt\n", "")) # rev-parse - tool_ctx.runner.push(_make_result(0, "impl-branch\n", "")) # branch - tool_ctx.runner.push(_make_result(0, "", "")) # git ls-files (pre-dirty-tree check) - tool_ctx.runner.push(_make_result(0, "", "")) # git status --porcelain (clean) - tool_ctx.runner.push(_make_result(0, "PASS\n= 100 passed =", "")) # test-check - tool_ctx.runner.push(_make_result(0, "", "")) # git fetch - tool_ctx.runner.push(_make_result(0, "abc123\n", "")) # rev-parse --verify (step 5.5) - tool_ctx.runner.push( + tool_ctx_kitchen_open.runner.push( + _make_result(0, "/repo/.git/worktrees/wt\n", "") + ) # rev-parse + tool_ctx_kitchen_open.runner.push(_make_result(0, "impl-branch\n", "")) # branch + tool_ctx_kitchen_open.runner.push( + _make_result(0, "", "") + ) # git ls-files (pre-dirty-tree check) + tool_ctx_kitchen_open.runner.push( + _make_result(0, "", "") + ) # git status --porcelain (clean) + tool_ctx_kitchen_open.runner.push( + _make_result(0, "PASS\n= 100 passed =", "") + ) # test-check + tool_ctx_kitchen_open.runner.push(_make_result(0, "", "")) # git fetch + tool_ctx_kitchen_open.runner.push( + _make_result(0, "abc123\n", "") + ) # rev-parse --verify (step 5.5) + tool_ctx_kitchen_open.runner.push( _make_result(0, "", "") ) # git log --merges (no merge commits — step 5.6) - tool_ctx.runner.push(_make_result(1, "", "CONFLICT (content): ...")) # git rebase FAILS - tool_ctx.runner.push(_make_result(0, "", "")) # git rebase --abort + tool_ctx_kitchen_open.runner.push( + _make_result(1, "", "CONFLICT (content): ...") + ) # git rebase FAILS + tool_ctx_kitchen_open.runner.push(_make_result(0, "", "")) # git rebase --abort result = json.loads(await merge_worktree(str(wt), "dev")) assert "error" in result assert result["failed_step"] == MergeFailedStep.REBASE @@ -220,13 +279,13 @@ async def test_merge_worktree_aborts_on_rebase_failure(self, tool_ctx, tmp_path) assert "CONFLICT" in result["stderr"] @pytest.mark.anyio - async def test_merge_worktree_rejects_nonexistent_path(self, tool_ctx): + async def test_merge_worktree_rejects_nonexistent_path(self, tool_ctx_kitchen_open): """merge_worktree rejects non-existent paths.""" result = json.loads(await merge_worktree("/nonexistent/path", "dev")) assert "error" in result @pytest.mark.anyio - async def test_merge_worktree_rejects_non_worktree(self, tool_ctx, tmp_path): + async def test_merge_worktree_rejects_non_worktree(self, tool_ctx_kitchen_open, tmp_path): """merge_worktree rejects paths that aren't git worktrees.""" result = json.loads(await merge_worktree(str(tmp_path), "dev")) assert "error" in result @@ -244,17 +303,23 @@ async def test_skip_test_gate_parameter_rejected(self): assert "skip_test_gate" in result["error"] @pytest.mark.anyio - async def test_internal_gate_cross_validates_output(self, tool_ctx, tmp_path): + async def test_internal_gate_cross_validates_output(self, tool_ctx_kitchen_open, tmp_path): """merge_worktree's internal gate catches rc=0 with failure text.""" wt = tmp_path / "worktree" wt.mkdir() (wt / ".git").write_text("gitdir: /repo/.git/worktrees/wt") - tool_ctx.runner.push(_make_result(0, "/repo/.git/worktrees/wt\n", "")) # rev-parse - tool_ctx.runner.push(_make_result(0, "impl-branch\n", "")) # branch - tool_ctx.runner.push(_make_result(0, "", "")) # git ls-files (pre-dirty-tree check) - tool_ctx.runner.push(_make_result(0, "", "")) # git status --porcelain (clean) - tool_ctx.runner.push( + tool_ctx_kitchen_open.runner.push( + _make_result(0, "/repo/.git/worktrees/wt\n", "") + ) # rev-parse + tool_ctx_kitchen_open.runner.push(_make_result(0, "impl-branch\n", "")) # branch + tool_ctx_kitchen_open.runner.push( + _make_result(0, "", "") + ) # git ls-files (pre-dirty-tree check) + tool_ctx_kitchen_open.runner.push( + _make_result(0, "", "") + ) # git status --porcelain (clean) + tool_ctx_kitchen_open.runner.push( _make_result(0, "= 3 failed, 97 passed =", "") ) # test-check: rc=0 but failed text result = json.loads(await merge_worktree(str(wt), "dev")) @@ -262,16 +327,22 @@ async def test_internal_gate_cross_validates_output(self, tool_ctx, tmp_path): assert result["failed_step"] == MergeFailedStep.TEST_GATE @pytest.mark.anyio - async def test_gate_failure_does_not_expose_summary(self, tool_ctx, tmp_path): + async def test_gate_failure_does_not_expose_summary(self, tool_ctx_kitchen_open, tmp_path): """When gate blocks, response contains no test output details.""" wt = tmp_path / "worktree" wt.mkdir() (wt / ".git").write_text("gitdir: /repo/.git/worktrees/wt") - tool_ctx.runner.push(_make_result(0, "/repo/.git/worktrees/wt\n", "")) # rev-parse - tool_ctx.runner.push(_make_result(0, "impl-branch\n", "")) # branch - tool_ctx.runner.push(_make_result(0, "", "")) # git status --porcelain (clean) - tool_ctx.runner.push(_make_result(1, "= 3 failed, 97 passed =", "")) # test-check + tool_ctx_kitchen_open.runner.push( + _make_result(0, "/repo/.git/worktrees/wt\n", "") + ) # rev-parse + tool_ctx_kitchen_open.runner.push(_make_result(0, "impl-branch\n", "")) # branch + tool_ctx_kitchen_open.runner.push( + _make_result(0, "", "") + ) # git status --porcelain (clean) + tool_ctx_kitchen_open.runner.push( + _make_result(1, "= 3 failed, 97 passed =", "") + ) # test-check result = json.loads(await merge_worktree(str(wt), "dev")) assert "error" in result assert "test_summary" not in result @@ -281,34 +352,51 @@ class TestMergeWorktreeCleanupReporting: """merge_worktree reports accurate cleanup results.""" @pytest.mark.anyio - async def test_reports_worktree_remove_failure(self, tool_ctx, tmp_path): + async def test_reports_worktree_remove_failure(self, tool_ctx_kitchen_open, tmp_path): """3a: worktree_removed reflects actual worktree removal result.""" wt = tmp_path / "worktree" wt.mkdir() (wt / ".git").write_text("gitdir: /repo/.git/worktrees/wt") - tool_ctx.runner.push(_make_result(0, "/repo/.git/worktrees/wt\n", "")) # rev-parse - tool_ctx.runner.push(_make_result(0, "impl-branch\n", "")) # branch - tool_ctx.runner.push(_make_result(0, "", "")) # git ls-files (pre-dirty-tree check) - tool_ctx.runner.push(_make_result(0, "", "")) # git status --porcelain (clean) - tool_ctx.runner.push(_make_result(0, "PASS\n= 100 passed =", "")) # pre-rebase test-check - tool_ctx.runner.push(_make_result(0, "", "")) # git fetch - tool_ctx.runner.push(_make_result(0, "abc123\n", "")) # rev-parse --verify (step 5.5) - tool_ctx.runner.push( + tool_ctx_kitchen_open.runner.push( + _make_result(0, "/repo/.git/worktrees/wt\n", "") + ) # rev-parse + tool_ctx_kitchen_open.runner.push(_make_result(0, "impl-branch\n", "")) # branch + tool_ctx_kitchen_open.runner.push( + _make_result(0, "", "") + ) # git ls-files (pre-dirty-tree check) + tool_ctx_kitchen_open.runner.push( + _make_result(0, "", "") + ) # git status --porcelain (clean) + tool_ctx_kitchen_open.runner.push( + _make_result(0, "PASS\n= 100 passed =", "") + ) # pre-rebase test-check + tool_ctx_kitchen_open.runner.push(_make_result(0, "", "")) # git fetch + tool_ctx_kitchen_open.runner.push( + _make_result(0, "abc123\n", "") + ) # rev-parse --verify (step 5.5) + tool_ctx_kitchen_open.runner.push( _make_result(0, "", "") ) # git log --merges (no merge commits — step 5.6) - tool_ctx.runner.push(_make_result(0, "", "")) # git rebase - tool_ctx.runner.push(_make_result(0, "PASS\n= 100 passed =", "")) # post-rebase test-check - tool_ctx.runner.push( + tool_ctx_kitchen_open.runner.push(_make_result(0, "", "")) # git rebase + tool_ctx_kitchen_open.runner.push( + _make_result(0, "PASS\n= 100 passed =", "") + ) # post-rebase test-check + tool_ctx_kitchen_open.runner.push( _make_result( 0, "worktree /repo\nHEAD abc\nbranch refs/heads/dev\n\n", "", ) ) # worktree list - tool_ctx.runner.push(_make_result(0, "dev\n", "")) # step 7.5: branch --show-current - tool_ctx.runner.push(_make_result(0, "", "")) # git merge - tool_ctx.runner.push(_make_result(0, "", "")) # branch -D + tool_ctx_kitchen_open.runner.push( + _make_result(0, "dev\n", "") + ) # step 7.5: branch --show-current + tool_ctx_kitchen_open.runner.push( + _make_result(0, "", "") + ) # step 7.6: git status --porcelain (clean) + tool_ctx_kitchen_open.runner.push(_make_result(0, "", "")) # git merge + tool_ctx_kitchen_open.runner.push(_make_result(0, "", "")) # branch -D with patch( "autoskillit.server.git.remove_git_worktree", new=AsyncMock( @@ -321,35 +409,54 @@ async def test_reports_worktree_remove_failure(self, tool_ctx, tmp_path): assert result["worktree_removed"] is False @pytest.mark.anyio - async def test_reports_branch_delete_failure(self, tool_ctx, tmp_path): + async def test_reports_branch_delete_failure(self, tool_ctx_kitchen_open, tmp_path): """3b: branch_deleted reflects actual git branch -D result.""" wt = tmp_path / "worktree" wt.mkdir() (wt / ".git").write_text("gitdir: /repo/.git/worktrees/wt") - tool_ctx.runner.push(_make_result(0, "/repo/.git/worktrees/wt\n", "")) # rev-parse - tool_ctx.runner.push(_make_result(0, "impl-branch\n", "")) # branch - tool_ctx.runner.push(_make_result(0, "", "")) # git ls-files (pre-dirty-tree check) - tool_ctx.runner.push(_make_result(0, "", "")) # git status --porcelain (clean) - tool_ctx.runner.push(_make_result(0, "PASS\n= 100 passed =", "")) # pre-rebase test-check - tool_ctx.runner.push(_make_result(0, "", "")) # git fetch - tool_ctx.runner.push(_make_result(0, "abc123\n", "")) # rev-parse --verify (step 5.5) - tool_ctx.runner.push( + tool_ctx_kitchen_open.runner.push( + _make_result(0, "/repo/.git/worktrees/wt\n", "") + ) # rev-parse + tool_ctx_kitchen_open.runner.push(_make_result(0, "impl-branch\n", "")) # branch + tool_ctx_kitchen_open.runner.push( + _make_result(0, "", "") + ) # git ls-files (pre-dirty-tree check) + tool_ctx_kitchen_open.runner.push( + _make_result(0, "", "") + ) # git status --porcelain (clean) + tool_ctx_kitchen_open.runner.push( + _make_result(0, "PASS\n= 100 passed =", "") + ) # pre-rebase test-check + tool_ctx_kitchen_open.runner.push(_make_result(0, "", "")) # git fetch + tool_ctx_kitchen_open.runner.push( + _make_result(0, "abc123\n", "") + ) # rev-parse --verify (step 5.5) + tool_ctx_kitchen_open.runner.push( _make_result(0, "", "") ) # git log --merges (no merge commits — step 5.6) - tool_ctx.runner.push(_make_result(0, "", "")) # git rebase - tool_ctx.runner.push(_make_result(0, "PASS\n= 100 passed =", "")) # post-rebase test-check - tool_ctx.runner.push( + tool_ctx_kitchen_open.runner.push(_make_result(0, "", "")) # git rebase + tool_ctx_kitchen_open.runner.push( + _make_result(0, "PASS\n= 100 passed =", "") + ) # post-rebase test-check + tool_ctx_kitchen_open.runner.push( _make_result( 0, "worktree /repo\nHEAD abc\nbranch refs/heads/dev\n\n", "", ) ) # worktree list - tool_ctx.runner.push(_make_result(0, "dev\n", "")) # step 7.5: branch --show-current - tool_ctx.runner.push(_make_result(0, "", "")) # git merge - tool_ctx.runner.push(_make_result(0, "", "")) # worktree remove - tool_ctx.runner.push(_make_result(1, "", "error: branch not found")) # branch -D FAILS + tool_ctx_kitchen_open.runner.push( + _make_result(0, "dev\n", "") + ) # step 7.5: branch --show-current + tool_ctx_kitchen_open.runner.push( + _make_result(0, "", "") + ) # step 7.6: git status --porcelain (clean) + tool_ctx_kitchen_open.runner.push(_make_result(0, "", "")) # git merge + tool_ctx_kitchen_open.runner.push(_make_result(0, "", "")) # worktree remove + tool_ctx_kitchen_open.runner.push( + _make_result(1, "", "error: branch not found") + ) # branch -D FAILS result = json.loads(await merge_worktree(str(wt), "dev")) assert result["merge_succeeded"] is True assert result["cleanup_succeeded"] is False @@ -357,18 +464,26 @@ async def test_reports_branch_delete_failure(self, tool_ctx, tmp_path): assert result["branch_deleted"] is False @pytest.mark.anyio - async def test_checks_fetch_result(self, tool_ctx, tmp_path): + async def test_checks_fetch_result(self, tool_ctx_kitchen_open, tmp_path): """3c: git fetch failure returns error before rebase attempt.""" wt = tmp_path / "worktree" wt.mkdir() (wt / ".git").write_text("gitdir: /repo/.git/worktrees/wt") - tool_ctx.runner.push(_make_result(0, "/repo/.git/worktrees/wt\n", "")) # rev-parse - tool_ctx.runner.push(_make_result(0, "impl-branch\n", "")) # branch - tool_ctx.runner.push(_make_result(0, "", "")) # git ls-files (pre-dirty-tree check) - tool_ctx.runner.push(_make_result(0, "", "")) # git status --porcelain (clean) - tool_ctx.runner.push(_make_result(0, "PASS\n= 100 passed =", "")) # test-check - tool_ctx.runner.push( + tool_ctx_kitchen_open.runner.push( + _make_result(0, "/repo/.git/worktrees/wt\n", "") + ) # rev-parse + tool_ctx_kitchen_open.runner.push(_make_result(0, "impl-branch\n", "")) # branch + tool_ctx_kitchen_open.runner.push( + _make_result(0, "", "") + ) # git ls-files (pre-dirty-tree check) + tool_ctx_kitchen_open.runner.push( + _make_result(0, "", "") + ) # git status --porcelain (clean) + tool_ctx_kitchen_open.runner.push( + _make_result(0, "PASS\n= 100 passed =", "") + ) # test-check + tool_ctx_kitchen_open.runner.push( _make_result(1, "", "fatal: could not connect to remote") ) # git fetch FAILS result = json.loads(await merge_worktree(str(wt), "dev")) @@ -380,29 +495,44 @@ class TestMergeWorktreeCleanupWarnings: """merge_worktree emits logger.warning when cleanup steps fail post-merge.""" @pytest.mark.anyio - async def test_warns_on_worktree_remove_failure(self, tool_ctx, tmp_path): + async def test_warns_on_worktree_remove_failure(self, tool_ctx_kitchen_open, tmp_path): wt = tmp_path / "worktree" wt.mkdir() (wt / ".git").write_text("gitdir: /repo/.git/worktrees/wt") - tool_ctx.runner.push(_make_result(0, "/repo/.git/worktrees/wt\n", "")) - tool_ctx.runner.push(_make_result(0, "impl-branch\n", "")) - tool_ctx.runner.push(_make_result(0, "", "")) # git ls-files (pre-dirty-tree check) - tool_ctx.runner.push(_make_result(0, "", "")) # git status --porcelain (clean) - tool_ctx.runner.push(_make_result(0, "PASS\n= 100 passed =", "")) # pre-rebase test-check - tool_ctx.runner.push(_make_result(0, "", "")) # fetch - tool_ctx.runner.push(_make_result(0, "abc123\n", "")) # rev-parse --verify (step 5.5) - tool_ctx.runner.push( + tool_ctx_kitchen_open.runner.push(_make_result(0, "/repo/.git/worktrees/wt\n", "")) + tool_ctx_kitchen_open.runner.push(_make_result(0, "impl-branch\n", "")) + tool_ctx_kitchen_open.runner.push( + _make_result(0, "", "") + ) # git ls-files (pre-dirty-tree check) + tool_ctx_kitchen_open.runner.push( + _make_result(0, "", "") + ) # git status --porcelain (clean) + tool_ctx_kitchen_open.runner.push( + _make_result(0, "PASS\n= 100 passed =", "") + ) # pre-rebase test-check + tool_ctx_kitchen_open.runner.push(_make_result(0, "", "")) # fetch + tool_ctx_kitchen_open.runner.push( + _make_result(0, "abc123\n", "") + ) # rev-parse --verify (step 5.5) + tool_ctx_kitchen_open.runner.push( _make_result(0, "", "") ) # git log --merges (no merge commits — step 5.6) - tool_ctx.runner.push(_make_result(0, "", "")) # rebase - tool_ctx.runner.push(_make_result(0, "PASS\n= 100 passed =", "")) # post-rebase test-check - tool_ctx.runner.push( + tool_ctx_kitchen_open.runner.push(_make_result(0, "", "")) # rebase + tool_ctx_kitchen_open.runner.push( + _make_result(0, "PASS\n= 100 passed =", "") + ) # post-rebase test-check + tool_ctx_kitchen_open.runner.push( _make_result(0, "worktree /repo\nHEAD abc\nbranch refs/heads/dev\n\n", "") ) - tool_ctx.runner.push(_make_result(0, "dev\n", "")) # step 7.5: branch --show-current - tool_ctx.runner.push(_make_result(0, "", "")) # merge - tool_ctx.runner.push(_make_result(0, "", "")) # branch -D + tool_ctx_kitchen_open.runner.push( + _make_result(0, "dev\n", "") + ) # step 7.5: branch --show-current + tool_ctx_kitchen_open.runner.push( + _make_result(0, "", "") + ) # step 7.6: git status --porcelain (clean) + tool_ctx_kitchen_open.runner.push(_make_result(0, "", "")) # merge + tool_ctx_kitchen_open.runner.push(_make_result(0, "", "")) # branch -D with ( patch( @@ -422,30 +552,47 @@ async def test_warns_on_worktree_remove_failure(self, tool_ctx, tmp_path): assert any(entry.get("operation") == "worktree_remove" for entry in warning_entries) @pytest.mark.anyio - async def test_warns_on_branch_delete_failure(self, tool_ctx, tmp_path): + async def test_warns_on_branch_delete_failure(self, tool_ctx_kitchen_open, tmp_path): wt = tmp_path / "worktree" wt.mkdir() (wt / ".git").write_text("gitdir: /repo/.git/worktrees/wt") - tool_ctx.runner.push(_make_result(0, "/repo/.git/worktrees/wt\n", "")) - tool_ctx.runner.push(_make_result(0, "impl-branch\n", "")) - tool_ctx.runner.push(_make_result(0, "", "")) # git ls-files (pre-dirty-tree check) - tool_ctx.runner.push(_make_result(0, "", "")) # git status --porcelain (clean) - tool_ctx.runner.push(_make_result(0, "PASS\n= 100 passed =", "")) # pre-rebase test-check - tool_ctx.runner.push(_make_result(0, "", "")) # fetch - tool_ctx.runner.push(_make_result(0, "abc123\n", "")) # rev-parse --verify (step 5.5) - tool_ctx.runner.push( + tool_ctx_kitchen_open.runner.push(_make_result(0, "/repo/.git/worktrees/wt\n", "")) + tool_ctx_kitchen_open.runner.push(_make_result(0, "impl-branch\n", "")) + tool_ctx_kitchen_open.runner.push( + _make_result(0, "", "") + ) # git ls-files (pre-dirty-tree check) + tool_ctx_kitchen_open.runner.push( + _make_result(0, "", "") + ) # git status --porcelain (clean) + tool_ctx_kitchen_open.runner.push( + _make_result(0, "PASS\n= 100 passed =", "") + ) # pre-rebase test-check + tool_ctx_kitchen_open.runner.push(_make_result(0, "", "")) # fetch + tool_ctx_kitchen_open.runner.push( + _make_result(0, "abc123\n", "") + ) # rev-parse --verify (step 5.5) + tool_ctx_kitchen_open.runner.push( _make_result(0, "", "") ) # git log --merges (no merge commits — step 5.6) - tool_ctx.runner.push(_make_result(0, "", "")) # rebase - tool_ctx.runner.push(_make_result(0, "PASS\n= 100 passed =", "")) # post-rebase test-check - tool_ctx.runner.push( + tool_ctx_kitchen_open.runner.push(_make_result(0, "", "")) # rebase + tool_ctx_kitchen_open.runner.push( + _make_result(0, "PASS\n= 100 passed =", "") + ) # post-rebase test-check + tool_ctx_kitchen_open.runner.push( _make_result(0, "worktree /repo\nHEAD abc\nbranch refs/heads/dev\n\n", "") ) - tool_ctx.runner.push(_make_result(0, "dev\n", "")) # step 7.5: branch --show-current - tool_ctx.runner.push(_make_result(0, "", "")) # merge - tool_ctx.runner.push(_make_result(0, "", "")) # worktree remove - tool_ctx.runner.push(_make_result(1, "", "error: branch not found")) # branch -D FAILS + tool_ctx_kitchen_open.runner.push( + _make_result(0, "dev\n", "") + ) # step 7.5: branch --show-current + tool_ctx_kitchen_open.runner.push( + _make_result(0, "", "") + ) # step 7.6: git status --porcelain (clean) + tool_ctx_kitchen_open.runner.push(_make_result(0, "", "")) # merge + tool_ctx_kitchen_open.runner.push(_make_result(0, "", "")) # worktree remove + tool_ctx_kitchen_open.runner.push( + _make_result(1, "", "error: branch not found") + ) # branch -D FAILS with structlog.testing.capture_logs() as logs: result = json.loads(await merge_worktree(str(wt), "dev")) @@ -457,30 +604,45 @@ async def test_warns_on_branch_delete_failure(self, tool_ctx, tmp_path): assert any(entry.get("operation") == "branch_delete" for entry in warning_entries) @pytest.mark.anyio - async def test_no_warning_on_clean_cleanup(self, tool_ctx, tmp_path): + async def test_no_warning_on_clean_cleanup(self, tool_ctx_kitchen_open, tmp_path): wt = tmp_path / "worktree" wt.mkdir() (wt / ".git").write_text("gitdir: /repo/.git/worktrees/wt") - tool_ctx.runner.push(_make_result(0, "/repo/.git/worktrees/wt\n", "")) - tool_ctx.runner.push(_make_result(0, "impl-branch\n", "")) - tool_ctx.runner.push(_make_result(0, "", "")) # git ls-files (pre-dirty-tree check) - tool_ctx.runner.push(_make_result(0, "", "")) # git status --porcelain (clean) - tool_ctx.runner.push(_make_result(0, "PASS\n= 100 passed =", "")) # pre-rebase test-check - tool_ctx.runner.push(_make_result(0, "", "")) # fetch - tool_ctx.runner.push(_make_result(0, "abc123\n", "")) # rev-parse --verify (step 5.5) - tool_ctx.runner.push( + tool_ctx_kitchen_open.runner.push(_make_result(0, "/repo/.git/worktrees/wt\n", "")) + tool_ctx_kitchen_open.runner.push(_make_result(0, "impl-branch\n", "")) + tool_ctx_kitchen_open.runner.push( + _make_result(0, "", "") + ) # git ls-files (pre-dirty-tree check) + tool_ctx_kitchen_open.runner.push( + _make_result(0, "", "") + ) # git status --porcelain (clean) + tool_ctx_kitchen_open.runner.push( + _make_result(0, "PASS\n= 100 passed =", "") + ) # pre-rebase test-check + tool_ctx_kitchen_open.runner.push(_make_result(0, "", "")) # fetch + tool_ctx_kitchen_open.runner.push( + _make_result(0, "abc123\n", "") + ) # rev-parse --verify (step 5.5) + tool_ctx_kitchen_open.runner.push( _make_result(0, "", "") ) # git log --merges (no merge commits — step 5.6) - tool_ctx.runner.push(_make_result(0, "", "")) # rebase - tool_ctx.runner.push(_make_result(0, "PASS\n= 100 passed =", "")) # post-rebase test-check - tool_ctx.runner.push( + tool_ctx_kitchen_open.runner.push(_make_result(0, "", "")) # rebase + tool_ctx_kitchen_open.runner.push( + _make_result(0, "PASS\n= 100 passed =", "") + ) # post-rebase test-check + tool_ctx_kitchen_open.runner.push( _make_result(0, "worktree /repo\nHEAD abc\nbranch refs/heads/dev\n\n", "") ) - tool_ctx.runner.push(_make_result(0, "dev\n", "")) # step 7.5: branch --show-current - tool_ctx.runner.push(_make_result(0, "", "")) # merge - tool_ctx.runner.push(_make_result(0, "", "")) # worktree remove — success - tool_ctx.runner.push(_make_result(0, "", "")) # branch -D — success + tool_ctx_kitchen_open.runner.push( + _make_result(0, "dev\n", "") + ) # step 7.5: branch --show-current + tool_ctx_kitchen_open.runner.push( + _make_result(0, "", "") + ) # step 7.6: git status --porcelain (clean) + tool_ctx_kitchen_open.runner.push(_make_result(0, "", "")) # merge + tool_ctx_kitchen_open.runner.push(_make_result(0, "", "")) # worktree remove — success + tool_ctx_kitchen_open.runner.push(_make_result(0, "", "")) # branch -D — success with structlog.testing.capture_logs() as logs: result = json.loads(await merge_worktree(str(wt), "dev")) @@ -500,21 +662,27 @@ class TestMergeWorktreeRemoteTrackingGuard: @pytest.mark.anyio async def test_merge_worktree_diagnoses_unpublished_base_branch( - self, tool_ctx: object, tmp_path: Path + self, tool_ctx_kitchen_open: object, tmp_path: Path ) -> None: """merge_worktree returns BASE_NOT_PUBLISHED error when ref is absent after fetch.""" wt = tmp_path / "wt" wt.mkdir() (wt / ".git").write_text("gitdir: /repo/.git/worktrees/wt") - tool_ctx.runner.push(_make_result(stdout="/repo/.git/worktrees/wt")) # rev-parse - tool_ctx.runner.push(_make_result(stdout="impl/task-01")) # branch --show-current - tool_ctx.runner.push(_make_result()) # git ls-files (pre-dirty-tree check) - tool_ctx.runner.push(_make_result()) # git status --porcelain (clean) - tool_ctx.runner.push(_make_result(stdout="PASS\n= 100 passed =")) # test check - tool_ctx.runner.push(_make_result()) # git fetch origin + tool_ctx_kitchen_open.runner.push( + _make_result(stdout="/repo/.git/worktrees/wt") + ) # rev-parse + tool_ctx_kitchen_open.runner.push( + _make_result(stdout="impl/task-01") + ) # branch --show-current + tool_ctx_kitchen_open.runner.push(_make_result()) # git ls-files (pre-dirty-tree check) + tool_ctx_kitchen_open.runner.push(_make_result()) # git status --porcelain (clean) + tool_ctx_kitchen_open.runner.push( + _make_result(stdout="PASS\n= 100 passed =") + ) # test check + tool_ctx_kitchen_open.runner.push(_make_result()) # git fetch origin # Step 5.5: ref check fails — branch not on remote - tool_ctx.runner.push( + tool_ctx_kitchen_open.runner.push( _make_result(returncode=128, stderr="fatal: Needed a single revision") ) @@ -528,20 +696,22 @@ async def test_merge_worktree_diagnoses_unpublished_base_branch( @pytest.mark.anyio async def test_merge_worktree_unpublished_base_reports_pre_rebase_check_step( - self, tool_ctx: object, tmp_path: Path + self, tool_ctx_kitchen_open: object, tmp_path: Path ) -> None: """Step 5.5 failure must report failed_step=PRE_REBASE_CHECK, not REBASE.""" wt = tmp_path / "wt" wt.mkdir() (wt / ".git").write_text("gitdir: /repo/.git/worktrees/wt") - tool_ctx.runner.push(_make_result(stdout="/repo/.git/worktrees/wt")) # rev-parse - tool_ctx.runner.push(_make_result(stdout="feat/x")) # branch - tool_ctx.runner.push(_make_result()) # git ls-files (pre-dirty-tree check) - tool_ctx.runner.push(_make_result()) # git status --porcelain (clean) - tool_ctx.runner.push(_make_result(stdout="PASS\n= 5 passed =")) # test-check - tool_ctx.runner.push(_make_result()) # fetch - tool_ctx.runner.push( + tool_ctx_kitchen_open.runner.push( + _make_result(stdout="/repo/.git/worktrees/wt") + ) # rev-parse + tool_ctx_kitchen_open.runner.push(_make_result(stdout="feat/x")) # branch + tool_ctx_kitchen_open.runner.push(_make_result()) # git ls-files (pre-dirty-tree check) + tool_ctx_kitchen_open.runner.push(_make_result()) # git status --porcelain (clean) + tool_ctx_kitchen_open.runner.push(_make_result(stdout="PASS\n= 5 passed =")) # test-check + tool_ctx_kitchen_open.runner.push(_make_result()) # fetch + tool_ctx_kitchen_open.runner.push( _make_result(returncode=128, stderr="fatal: Needed a single revision") ) # step 5.5 @@ -552,28 +722,30 @@ async def test_merge_worktree_unpublished_base_reports_pre_rebase_check_step( @pytest.mark.anyio async def test_merge_worktree_fatal_invalid_upstream_produces_rebase_aborted( - self, tool_ctx: object, tmp_path: Path + self, tool_ctx_kitchen_open: object, tmp_path: Path ) -> None: """Regression: git rebase fatal: invalid upstream is caught as rebase failure.""" wt = tmp_path / "wt" wt.mkdir() (wt / ".git").write_text("gitdir: /repo/.git/worktrees/wt") - tool_ctx.runner.push(_make_result(stdout="/repo/.git/worktrees/wt")) - tool_ctx.runner.push(_make_result(stdout="impl/task-01")) - tool_ctx.runner.push(_make_result()) # git ls-files (pre-dirty-tree check) - tool_ctx.runner.push(_make_result()) # git status --porcelain (clean) - tool_ctx.runner.push(_make_result(stdout="PASS\n= 100 passed =")) # test gate - tool_ctx.runner.push(_make_result()) # fetch - tool_ctx.runner.push(_make_result()) # ref check passes - tool_ctx.runner.push(_make_result()) # git log --merges (no merge commits — step 5.6) + tool_ctx_kitchen_open.runner.push(_make_result(stdout="/repo/.git/worktrees/wt")) + tool_ctx_kitchen_open.runner.push(_make_result(stdout="impl/task-01")) + tool_ctx_kitchen_open.runner.push(_make_result()) # git ls-files (pre-dirty-tree check) + tool_ctx_kitchen_open.runner.push(_make_result()) # git status --porcelain (clean) + tool_ctx_kitchen_open.runner.push(_make_result(stdout="PASS\n= 100 passed =")) # test gate + tool_ctx_kitchen_open.runner.push(_make_result()) # fetch + tool_ctx_kitchen_open.runner.push(_make_result()) # ref check passes + tool_ctx_kitchen_open.runner.push( + _make_result() + ) # git log --merges (no merge commits — step 5.6) # Rebase fails with fatal: invalid upstream (bypassed guard scenario) - tool_ctx.runner.push( + tool_ctx_kitchen_open.runner.push( _make_result( returncode=128, stderr="fatal: invalid upstream 'origin/feature/local-only'" ) ) - tool_ctx.runner.push(_make_result()) # rebase --abort + tool_ctx_kitchen_open.runner.push(_make_result()) # rebase --abort result = json.loads(await merge_worktree(str(wt), "feature/local-only")) @@ -586,23 +758,22 @@ class TestMergeWorktreeTiming: """merge_worktree records wall-clock timing when step_name is provided.""" @pytest.mark.anyio - async def test_merge_worktree_step_name_records_timing(self, tool_ctx, tmp_path): + async def test_merge_worktree_step_name_records_timing(self, tool_ctx_kitchen_open, tmp_path): wt = tmp_path / "wt" wt.mkdir() (wt / ".git").write_text("gitdir: /repo/.git/worktrees/wt") - tool_ctx.runner.push(_make_result(stdout="/repo/.git/worktrees/wt")) - tool_ctx.runner.push(_make_result(stdout="impl/task-01")) - tool_ctx.runner.push(_make_result()) - tool_ctx.runner.push(_make_result(stdout="PASS\n= 100 passed =")) - tool_ctx.runner.push(_make_result()) - tool_ctx.runner.push(_make_result()) - tool_ctx.runner.push(_make_result()) # git log --merges (step 5.6) - tool_ctx.runner.push(_make_result()) - tool_ctx.runner.push(_make_result()) + tool_ctx_kitchen_open.runner.push(_make_result(stdout="/repo/.git/worktrees/wt")) + tool_ctx_kitchen_open.runner.push(_make_result(stdout="impl/task-01")) + tool_ctx_kitchen_open.runner.push(_make_result()) + tool_ctx_kitchen_open.runner.push(_make_result(stdout="PASS\n= 100 passed =")) + tool_ctx_kitchen_open.runner.push(_make_result()) + tool_ctx_kitchen_open.runner.push(_make_result()) + tool_ctx_kitchen_open.runner.push(_make_result()) # git log --merges (step 5.6) + tool_ctx_kitchen_open.runner.push(_make_result()) + tool_ctx_kitchen_open.runner.push(_make_result()) await merge_worktree(str(wt), "dev", step_name="merge") - report = tool_ctx.timing_log.get_report() - assert any(e["step_name"] == "merge" for e in report) + assert_step_timed(tool_ctx_kitchen_open.timing_log, "merge") @pytest.mark.anyio async def test_merge_worktree_empty_step_name_skips_timing(self, tool_ctx, tmp_path): @@ -620,45 +791,54 @@ async def test_merge_worktree_empty_step_name_skips_timing(self, tool_ctx, tmp_p tool_ctx.runner.push(_make_result()) await merge_worktree(str(wt), "dev") - assert tool_ctx.timing_log.get_report() == [] + assert_no_timing(tool_ctx.timing_log) class TestClassifyFixTiming: """classify_fix records wall-clock timing when step_name is provided.""" @pytest.mark.anyio - async def test_classify_fix_step_name_records_timing(self, tool_ctx, tmp_path): - tool_ctx.runner.push(_make_result(stdout="src/other/file.py\n")) + async def test_classify_fix_step_name_records_timing(self, tool_ctx_kitchen_open, tmp_path): + tool_ctx_kitchen_open.runner.push(_make_result(stdout="src/other/file.py\n")) await classify_fix(str(tmp_path), "main", step_name="classify") - report = tool_ctx.timing_log.get_report() - assert any(e["step_name"] == "classify" for e in report) + assert_step_timed(tool_ctx_kitchen_open.timing_log, "classify") @pytest.mark.anyio async def test_classify_fix_empty_step_name_skips_timing(self, tool_ctx, tmp_path): tool_ctx.runner.push(_make_result(stdout="src/other/file.py\n")) await classify_fix(str(tmp_path), "main") - assert tool_ctx.timing_log.get_report() == [] + assert_no_timing(tool_ctx.timing_log) class TestMergeWorktreeMergeCommitDetection: """merge_worktree detects merge commits before rebase and returns actionable error.""" @pytest.mark.anyio - async def test_detects_merge_commits_before_rebase(self, tool_ctx, tmp_path): + async def test_detects_merge_commits_before_rebase(self, tool_ctx_kitchen_open, tmp_path): """Step 5.6: merge commits in worktree history abort before rebase with specific error.""" wt = tmp_path / "wt" wt.mkdir() (wt / ".git").write_text("gitdir: /repo/.git/worktrees/wt") - tool_ctx.runner.push(_make_result(0, "/repo/.git/worktrees/wt\n", "")) # rev-parse - tool_ctx.runner.push(_make_result(0, "impl-branch\n", "")) # branch - tool_ctx.runner.push(_make_result(0, "", "")) # git ls-files (pre-dirty-tree check) - tool_ctx.runner.push(_make_result(0, "", "")) # git status --porcelain (clean) - tool_ctx.runner.push(_make_result(0, "PASS\n= 100 passed =", "")) # test-check - tool_ctx.runner.push(_make_result(0, "", "")) # git fetch - tool_ctx.runner.push(_make_result(0, "abc123\n", "")) # rev-parse --verify (step 5.5) + tool_ctx_kitchen_open.runner.push( + _make_result(0, "/repo/.git/worktrees/wt\n", "") + ) # rev-parse + tool_ctx_kitchen_open.runner.push(_make_result(0, "impl-branch\n", "")) # branch + tool_ctx_kitchen_open.runner.push( + _make_result(0, "", "") + ) # git ls-files (pre-dirty-tree check) + tool_ctx_kitchen_open.runner.push( + _make_result(0, "", "") + ) # git status --porcelain (clean) + tool_ctx_kitchen_open.runner.push( + _make_result(0, "PASS\n= 100 passed =", "") + ) # test-check + tool_ctx_kitchen_open.runner.push(_make_result(0, "", "")) # git fetch + tool_ctx_kitchen_open.runner.push( + _make_result(0, "abc123\n", "") + ) # rev-parse --verify (step 5.5) # Step 5.6: git log --merges finds merge commits - tool_ctx.runner.push(_make_result(0, "bb481aa Merge PR branch\n", "")) + tool_ctx_kitchen_open.runner.push(_make_result(0, "bb481aa Merge PR branch\n", "")) result = json.loads(await merge_worktree(str(wt), "dev")) @@ -668,20 +848,24 @@ async def test_detects_merge_commits_before_rebase(self, tool_ctx, tmp_path): assert result["merge_commits"] == ["bb481aa Merge PR branch"] @pytest.mark.anyio - async def test_merge_commit_error_message_is_actionable(self, tool_ctx, tmp_path): + async def test_merge_commit_error_message_is_actionable(self, tool_ctx_kitchen_open, tmp_path): """Step 5.6: error message names cherry-pick, checkout, and forbids run_cmd bypass.""" wt = tmp_path / "wt" wt.mkdir() (wt / ".git").write_text("gitdir: /repo/.git/worktrees/wt") - tool_ctx.runner.push(_make_result(0, "/repo/.git/worktrees/wt\n", "")) - tool_ctx.runner.push(_make_result(0, "impl-branch\n", "")) - tool_ctx.runner.push(_make_result(0, "", "")) # git ls-files (pre-dirty-tree check) - tool_ctx.runner.push(_make_result(0, "", "")) # git status --porcelain (clean) - tool_ctx.runner.push(_make_result(0, "PASS\n= 100 passed =", "")) - tool_ctx.runner.push(_make_result(0, "", "")) - tool_ctx.runner.push(_make_result(0, "abc123\n", "")) - tool_ctx.runner.push(_make_result(0, "bb481aa Merge PR branch\n", "")) + tool_ctx_kitchen_open.runner.push(_make_result(0, "/repo/.git/worktrees/wt\n", "")) + tool_ctx_kitchen_open.runner.push(_make_result(0, "impl-branch\n", "")) + tool_ctx_kitchen_open.runner.push( + _make_result(0, "", "") + ) # git ls-files (pre-dirty-tree check) + tool_ctx_kitchen_open.runner.push( + _make_result(0, "", "") + ) # git status --porcelain (clean) + tool_ctx_kitchen_open.runner.push(_make_result(0, "PASS\n= 100 passed =", "")) + tool_ctx_kitchen_open.runner.push(_make_result(0, "", "")) + tool_ctx_kitchen_open.runner.push(_make_result(0, "abc123\n", "")) + tool_ctx_kitchen_open.runner.push(_make_result(0, "bb481aa Merge PR branch\n", "")) result = json.loads(await merge_worktree(str(wt), "dev")) @@ -691,22 +875,30 @@ async def test_merge_commit_error_message_is_actionable(self, tool_ctx, tmp_path assert "run_cmd" in result["error"] @pytest.mark.anyio - async def test_linear_history_passes_merge_commit_check(self, tool_ctx, tmp_path): + async def test_linear_history_passes_merge_commit_check(self, tool_ctx_kitchen_open, tmp_path): """Step 5.6: empty git log --merges output allows pipeline to continue to rebase.""" wt = tmp_path / "wt" wt.mkdir() (wt / ".git").write_text("gitdir: /repo/.git/worktrees/wt") - tool_ctx.runner.push(_make_result(0, "/repo/.git/worktrees/wt\n", "")) - tool_ctx.runner.push(_make_result(0, "impl-branch\n", "")) - tool_ctx.runner.push(_make_result(0, "", "")) # git ls-files (pre-dirty-tree check) - tool_ctx.runner.push(_make_result(0, "", "")) # git status --porcelain (clean) - tool_ctx.runner.push(_make_result(0, "PASS\n= 100 passed =", "")) - tool_ctx.runner.push(_make_result(0, "", "")) - tool_ctx.runner.push(_make_result(0, "abc123\n", "")) - tool_ctx.runner.push(_make_result(0, "", "")) # git log --merges returns empty (step 5.6) - tool_ctx.runner.push(_make_result(1, "", "CONFLICT (content): ...")) # rebase fails - tool_ctx.runner.push(_make_result(0, "", "")) # rebase --abort + tool_ctx_kitchen_open.runner.push(_make_result(0, "/repo/.git/worktrees/wt\n", "")) + tool_ctx_kitchen_open.runner.push(_make_result(0, "impl-branch\n", "")) + tool_ctx_kitchen_open.runner.push( + _make_result(0, "", "") + ) # git ls-files (pre-dirty-tree check) + tool_ctx_kitchen_open.runner.push( + _make_result(0, "", "") + ) # git status --porcelain (clean) + tool_ctx_kitchen_open.runner.push(_make_result(0, "PASS\n= 100 passed =", "")) + tool_ctx_kitchen_open.runner.push(_make_result(0, "", "")) + tool_ctx_kitchen_open.runner.push(_make_result(0, "abc123\n", "")) + tool_ctx_kitchen_open.runner.push( + _make_result(0, "", "") + ) # git log --merges returns empty (step 5.6) + tool_ctx_kitchen_open.runner.push( + _make_result(1, "", "CONFLICT (content): ...") + ) # rebase fails + tool_ctx_kitchen_open.runner.push(_make_result(0, "", "")) # rebase --abort result = json.loads(await merge_worktree(str(wt), "dev")) @@ -717,38 +909,48 @@ async def test_linear_history_passes_merge_commit_check(self, tool_ctx, tmp_path class TestCreateUniqueBranch: @pytest.mark.anyio - async def test_creates_branch_when_unique(self, tool_ctx): - tool_ctx.runner.push(_make_result(0, "", "")) # ls-remote: empty = absent - tool_ctx.runner.push(_make_result(0, "main\n", "")) # branch --show-current (HEAD state) - tool_ctx.runner.push(_make_result(0, "", "")) # git checkout -b + async def test_creates_branch_when_unique(self, tool_ctx_kitchen_open): + tool_ctx_kitchen_open.runner.push(_make_result(0, "", "")) # ls-remote: empty = absent + tool_ctx_kitchen_open.runner.push( + _make_result(0, "main\n", "") + ) # branch --show-current (HEAD state) + tool_ctx_kitchen_open.runner.push(_make_result(0, "", "")) # git checkout -b result = json.loads(await create_unique_branch("feat-foo", 42, "origin", ".")) assert result["branch_name"] == "feat-foo-42" assert result["was_unique"] is True @pytest.mark.anyio - async def test_appends_suffix_when_branch_exists_on_remote(self, tool_ctx): - tool_ctx.runner.push(_make_result(0, "abc123\trefs/heads/feat-foo-42\n", "")) # exists - tool_ctx.runner.push(_make_result(0, "", "")) # -2 not found - tool_ctx.runner.push(_make_result(0, "main\n", "")) # branch --show-current (HEAD state) - tool_ctx.runner.push(_make_result(0, "", "")) # checkout -b feat-foo-42-2 + async def test_appends_suffix_when_branch_exists_on_remote(self, tool_ctx_kitchen_open): + tool_ctx_kitchen_open.runner.push( + _make_result(0, "abc123\trefs/heads/feat-foo-42\n", "") + ) # exists + tool_ctx_kitchen_open.runner.push(_make_result(0, "", "")) # -2 not found + tool_ctx_kitchen_open.runner.push( + _make_result(0, "main\n", "") + ) # branch --show-current (HEAD state) + tool_ctx_kitchen_open.runner.push(_make_result(0, "", "")) # checkout -b feat-foo-42-2 result = json.loads(await create_unique_branch("feat-foo", 42, "origin", ".")) assert result["branch_name"] == "feat-foo-42-2" assert result["was_unique"] is False @pytest.mark.anyio - async def test_ls_remote_auth_failure_falls_back_gracefully(self, tool_ctx): - tool_ctx.runner.push(_make_result(128, "", "fatal: Authentication failed")) - tool_ctx.runner.push(_make_result(0, "main\n", "")) # branch --show-current (HEAD state) - tool_ctx.runner.push(_make_result(0, "", "")) # checkout proceeds with base name + async def test_ls_remote_auth_failure_falls_back_gracefully(self, tool_ctx_kitchen_open): + tool_ctx_kitchen_open.runner.push(_make_result(128, "", "fatal: Authentication failed")) + tool_ctx_kitchen_open.runner.push( + _make_result(0, "main\n", "") + ) # branch --show-current (HEAD state) + tool_ctx_kitchen_open.runner.push( + _make_result(0, "", "") + ) # checkout proceeds with base name result = json.loads(await create_unique_branch("feat-foo", 42, "origin", ".")) assert result["branch_name"] == "feat-foo-42" assert result["was_unique"] is True @pytest.mark.anyio - async def test_no_issue_uses_slug_only(self, tool_ctx): - tool_ctx.runner.push(_make_result(0, "", "")) # ls-remote - tool_ctx.runner.push(_make_result(0, "main\n", "")) # branch --show-current - tool_ctx.runner.push(_make_result(0, "", "")) # checkout -b + async def test_no_issue_uses_slug_only(self, tool_ctx_kitchen_open): + tool_ctx_kitchen_open.runner.push(_make_result(0, "", "")) # ls-remote + tool_ctx_kitchen_open.runner.push(_make_result(0, "main\n", "")) # branch --show-current + tool_ctx_kitchen_open.runner.push(_make_result(0, "", "")) # checkout -b result = json.loads(await create_unique_branch("feat-bar", None, "origin", ".")) assert result["branch_name"] == "feat-bar" @@ -760,46 +962,58 @@ async def test_gate_closed_returns_gate_error(self, tool_ctx): assert result["subtype"] == "gate_error" @pytest.mark.anyio - async def test_timing_recorded_when_step_name_provided(self, tool_ctx): - tool_ctx.runner.push(_make_result(0, "", "")) # ls-remote - tool_ctx.runner.push(_make_result(0, "main\n", "")) # branch --show-current - tool_ctx.runner.push(_make_result(0, "", "")) # checkout -b + async def test_timing_recorded_when_step_name_provided(self, tool_ctx_kitchen_open): + tool_ctx_kitchen_open.runner.push(_make_result(0, "", "")) # ls-remote + tool_ctx_kitchen_open.runner.push(_make_result(0, "main\n", "")) # branch --show-current + tool_ctx_kitchen_open.runner.push(_make_result(0, "", "")) # checkout -b await create_unique_branch("feat-x", 1, "origin", ".", step_name="branch_step") - assert any(e["step_name"] == "branch_step" for e in tool_ctx.timing_log.get_report()) + assert any( + e["step_name"] == "branch_step" for e in tool_ctx_kitchen_open.timing_log.get_report() + ) @pytest.mark.anyio - async def test_create_unique_branch_uses_base_branch_name_when_provided(self, tool_ctx): + async def test_create_unique_branch_uses_base_branch_name_when_provided( + self, tool_ctx_kitchen_open + ): """AP3: when base_branch_name is provided, use it directly as the base.""" - tool_ctx.runner.push(_make_result(0, "", "")) # ls-remote: empty = absent - tool_ctx.runner.push(_make_result(0, "main\n", "")) # branch --show-current - tool_ctx.runner.push(_make_result(0, "", "")) # git checkout -b + tool_ctx_kitchen_open.runner.push(_make_result(0, "", "")) # ls-remote: empty = absent + tool_ctx_kitchen_open.runner.push(_make_result(0, "main\n", "")) # branch --show-current + tool_ctx_kitchen_open.runner.push(_make_result(0, "", "")) # git checkout -b result = json.loads(await create_unique_branch(base_branch_name="impl/238", cwd=".")) assert result["branch_name"] == "impl/238" # ls-remote must check the exact base_branch_name, not slug-issue composition ls_remote_cmd = next( - (args[0] for args in tool_ctx.runner.call_args_list if "ls-remote" in args[0]), + ( + args[0] + for args in tool_ctx_kitchen_open.runner.call_args_list + if "ls-remote" in args[0] + ), None, ) assert ls_remote_cmd is not None, "No ls-remote subprocess call found" assert any("impl/238" in arg for arg in ls_remote_cmd) @pytest.mark.anyio - async def test_reports_base_ref_in_return_value(self, tool_ctx): + async def test_reports_base_ref_in_return_value(self, tool_ctx_kitchen_open): """Test 1.6: create_unique_branch reports base_ref (the branch HEAD was on).""" - tool_ctx.runner.push(_make_result(0, "", "")) # ls-remote: empty = absent - tool_ctx.runner.push(_make_result(0, "feature-branch\n", "")) # branch --show-current - tool_ctx.runner.push(_make_result(0, "", "")) # git checkout -b + tool_ctx_kitchen_open.runner.push(_make_result(0, "", "")) # ls-remote: empty = absent + tool_ctx_kitchen_open.runner.push( + _make_result(0, "feature-branch\n", "") + ) # branch --show-current + tool_ctx_kitchen_open.runner.push(_make_result(0, "", "")) # git checkout -b result = json.loads(await create_unique_branch("feat-foo", 42, "origin", ".")) assert result["branch_name"] == "feat-foo-42" assert result["was_unique"] is True assert result["base_ref"] == "feature-branch" @pytest.mark.anyio - async def test_reports_detached_head_as_base_ref(self, tool_ctx): + async def test_reports_detached_head_as_base_ref(self, tool_ctx_kitchen_open): """create_unique_branch reports DETACHED_HEAD when HEAD is detached.""" - tool_ctx.runner.push(_make_result(0, "", "")) # ls-remote: empty = absent - tool_ctx.runner.push(_make_result(0, "", "")) # branch --show-current returns empty - tool_ctx.runner.push(_make_result(0, "", "")) # git checkout -b + tool_ctx_kitchen_open.runner.push(_make_result(0, "", "")) # ls-remote: empty = absent + tool_ctx_kitchen_open.runner.push( + _make_result(0, "", "") + ) # branch --show-current returns empty + tool_ctx_kitchen_open.runner.push(_make_result(0, "", "")) # git checkout -b result = json.loads(await create_unique_branch("feat-foo", 42, "origin", ".")) assert result["branch_name"] == "feat-foo-42" assert result["base_ref"] == "DETACHED_HEAD" @@ -807,8 +1021,8 @@ async def test_reports_detached_head_as_base_ref(self, tool_ctx): class TestCheckPrMergeable: @pytest.mark.anyio - async def test_mergeable_pr(self, tool_ctx): - tool_ctx.runner.push( + async def test_mergeable_pr(self, tool_ctx_kitchen_open): + tool_ctx_kitchen_open.runner.push( _make_result( 0, json.dumps({"mergeable": "MERGEABLE", "mergeStateStatus": "CLEAN"}), "" ) @@ -819,8 +1033,8 @@ async def test_mergeable_pr(self, tool_ctx): assert result["mergeable_status"] == "MERGEABLE" @pytest.mark.anyio - async def test_conflicting_pr_is_not_mergeable(self, tool_ctx): - tool_ctx.runner.push( + async def test_conflicting_pr_is_not_mergeable(self, tool_ctx_kitchen_open): + tool_ctx_kitchen_open.runner.push( _make_result( 0, json.dumps({"mergeable": "CONFLICTING", "mergeStateStatus": "DIRTY"}), "" ) @@ -831,8 +1045,8 @@ async def test_conflicting_pr_is_not_mergeable(self, tool_ctx): assert result["mergeable_status"] == "CONFLICTING" @pytest.mark.anyio - async def test_unknown_mergeable_status_returned_raw(self, tool_ctx): - tool_ctx.runner.push( + async def test_unknown_mergeable_status_returned_raw(self, tool_ctx_kitchen_open): + tool_ctx_kitchen_open.runner.push( _make_result( 0, json.dumps({"mergeable": "UNKNOWN", "mergeStateStatus": "UNKNOWN"}), "" ) @@ -855,13 +1069,53 @@ async def test_gate_closed_returns_gate_error(self, tool_ctx): assert result["subtype"] == "gate_error" @pytest.mark.anyio - async def test_repo_flag_passed_to_gh(self, tool_ctx): - tool_ctx.runner.push( + async def test_repo_flag_passed_to_gh(self, tool_ctx_kitchen_open): + tool_ctx_kitchen_open.runner.push( _make_result( 0, json.dumps({"mergeable": "MERGEABLE", "mergeStateStatus": "CLEAN"}), "" ) ) result = json.loads(await check_pr_mergeable(42, ".", repo="owner/myrepo")) - call_cmd = tool_ctx.runner.call_args_list[-1][0] + call_cmd = tool_ctx_kitchen_open.runner.call_args_list[-1][0] assert "-R" in call_cmd assert result["mergeable"] is True + + +class TestClassifyFixRemoteResolution: + """T3: classify_fix resolves the remote via resolve_remote_name internally.""" + + @pytest.mark.anyio + async def test_classify_fix_uses_upstream_when_available( + self, tool_ctx_kitchen_open, tmp_path + ): + """When resolve_remote_name returns 'upstream', fetch and diff use 'upstream'.""" + tool_ctx_kitchen_open.runner.push(_make_result(0, "", "")) # fetch succeeds + tool_ctx_kitchen_open.runner.push(_make_result(0, "src/foo.py\n", "")) # diff succeeds + + with patch( + "autoskillit.server._misc.resolve_remote_name", + new=AsyncMock(return_value="upstream"), + ): + await classify_fix(str(tmp_path), "main") + + fetch_cmd = tool_ctx_kitchen_open.runner.call_args_list[0][0] + assert fetch_cmd == ["git", "fetch", "upstream", "main"] + diff_cmd = tool_ctx_kitchen_open.runner.call_args_list[1][0] + assert "upstream/main...HEAD" in diff_cmd[-1] + + @pytest.mark.anyio + async def test_classify_fix_falls_back_to_origin(self, tool_ctx_kitchen_open, tmp_path): + """When resolve_remote_name returns 'origin', fetch and diff use 'origin'.""" + tool_ctx_kitchen_open.runner.push(_make_result(0, "", "")) # fetch succeeds + tool_ctx_kitchen_open.runner.push(_make_result(0, "src/bar.py\n", "")) # diff succeeds + + with patch( + "autoskillit.server._misc.resolve_remote_name", + new=AsyncMock(return_value="origin"), + ): + await classify_fix(str(tmp_path), "main") + + fetch_cmd = tool_ctx_kitchen_open.runner.call_args_list[0][0] + assert fetch_cmd == ["git", "fetch", "origin", "main"] + diff_cmd = tool_ctx_kitchen_open.runner.call_args_list[1][0] + assert "origin/main...HEAD" in diff_cmd[-1] diff --git a/tests/server/test_tools_github.py b/tests/server/test_tools_github.py index 5b4a437473..e7e5ed2e70 100644 --- a/tests/server/test_tools_github.py +++ b/tests/server/test_tools_github.py @@ -3,71 +3,17 @@ from __future__ import annotations import json -from pathlib import Path -from unittest.mock import AsyncMock, MagicMock +from unittest.mock import AsyncMock -import anyio import pytest -from autoskillit.config import AutomationConfig -from autoskillit.core import SkillResult -from autoskillit.core.types import RetryReason from autoskillit.pipeline.gate import GATED_TOOLS, UNGATED_TOOLS, DefaultGateState -from autoskillit.server.helpers import _extract_block -from autoskillit.server.tools_github import ( - _FINGERPRINT_END, - _FINGERPRINT_START, - _parse_fingerprint, +from autoskillit.server.tools.tools_github import ( fetch_github_issue, get_issue_title, - report_bug, ) -from autoskillit.server.tools_issue_lifecycle import ( - _ENRICH_RESULT_END, - _ENRICH_RESULT_START, - _PREPARE_RESULT_END, - _PREPARE_RESULT_START, - _parse_enrich_result, - _parse_prepare_result, -) -from tests.server._helpers import _skill_fail, _skill_ok - -# --------------------------------------------------------------------------- -# _parse_fingerprint unit tests -# --------------------------------------------------------------------------- - - -def test_parse_fingerprint_present(): - report = ( - "Some preamble\n" - f"{_FINGERPRINT_START}\n" - "KeyError in recipe/validator.py: missing ingredient ref\n" - f"{_FINGERPRINT_END}\n" - "Report written to /tmp/report.md" - ) - assert _parse_fingerprint(report) == "KeyError in recipe/validator.py: missing ingredient ref" - - -def test_parse_fingerprint_missing_returns_none(): - assert _parse_fingerprint("No fingerprint block here") is None - - -def test_parse_fingerprint_empty_block_returns_none(): - report = f"{_FINGERPRINT_START}\n{_FINGERPRINT_END}\n" - assert _parse_fingerprint(report) is None - - -def test_parse_fingerprint_first_nonempty_line(): - """Only the first non-empty line inside the block is returned.""" - report = ( - f"{_FINGERPRINT_START}\n" - "\n" - " TypeError in execution/headless.py: runner=None \n" - "extra line\n" - f"{_FINGERPRINT_END}\n" - ) - assert _parse_fingerprint(report) == "TypeError in execution/headless.py: runner=None" +pytestmark = [pytest.mark.layer("server"), pytest.mark.small] # --------------------------------------------------------------------------- # fetch_github_issue @@ -84,16 +30,16 @@ async def test_fetch_github_issue_gate_closed(tool_ctx) -> None: @pytest.mark.anyio -async def test_fetch_github_issue_no_client(tool_ctx) -> None: +async def test_fetch_github_issue_no_client(tool_ctx_kitchen_open) -> None: """github_client=None → {"success": False, "error": "GitHub client not configured"}.""" - tool_ctx.github_client = None + tool_ctx_kitchen_open.github_client = None result = json.loads(await fetch_github_issue("owner/repo#42")) assert result["success"] is False assert "GitHub client not configured" in result["error"] @pytest.mark.anyio -async def test_fetch_github_issue_success(tool_ctx) -> None: +async def test_fetch_github_issue_success(tool_ctx_kitchen_open) -> None: """client.fetch_issue returns data → JSON with that data.""" issue_data = { "success": True, @@ -104,8 +50,8 @@ async def test_fetch_github_issue_success(tool_ctx) -> None: "labels": [], "content": "## Body\nSome content.", } - tool_ctx.github_client = AsyncMock() - tool_ctx.github_client.fetch_issue = AsyncMock(return_value=issue_data) + tool_ctx_kitchen_open.github_client = AsyncMock() + tool_ctx_kitchen_open.github_client.fetch_issue = AsyncMock(return_value=issue_data) result = json.loads(await fetch_github_issue("https://github.com/owner/repo/issues/42")) assert result["success"] is True @@ -113,10 +59,10 @@ async def test_fetch_github_issue_success(tool_ctx) -> None: @pytest.mark.anyio -async def test_fetch_github_issue_bare_number_no_default_repo(tool_ctx) -> None: +async def test_fetch_github_issue_bare_number_no_default_repo(tool_ctx_kitchen_open) -> None: """issue_url='42', no default_repo → error response.""" - tool_ctx.github_client = AsyncMock() - tool_ctx.config.github.default_repo = "" + tool_ctx_kitchen_open.github_client = AsyncMock() + tool_ctx_kitchen_open.config.github.default_repo = "" result = json.loads(await fetch_github_issue("42")) assert result["success"] is False @@ -125,10 +71,10 @@ async def test_fetch_github_issue_bare_number_no_default_repo(tool_ctx) -> None: @pytest.mark.anyio async def test_fetch_github_issue_bare_number_with_default_repo( - tool_ctx, + tool_ctx_kitchen_open, ) -> None: """issue_url='42', default_repo='owner/repo' → resolves to 'owner/repo#42'.""" - tool_ctx.config.github.default_repo = "owner/repo" + tool_ctx_kitchen_open.config.github.default_repo = "owner/repo" issue_data = { "success": True, "issue_number": 42, @@ -138,18 +84,18 @@ async def test_fetch_github_issue_bare_number_with_default_repo( "labels": [], "content": "content", } - tool_ctx.github_client = AsyncMock() - tool_ctx.github_client.fetch_issue = AsyncMock(return_value=issue_data) + tool_ctx_kitchen_open.github_client = AsyncMock() + tool_ctx_kitchen_open.github_client.fetch_issue = AsyncMock(return_value=issue_data) result = json.loads(await fetch_github_issue("42")) assert result["success"] is True # Verify fetch_issue was called with the resolved ref - call_args = tool_ctx.github_client.fetch_issue.call_args + call_args = tool_ctx_kitchen_open.github_client.fetch_issue.call_args assert "owner/repo#42" in call_args.args[0] @pytest.mark.anyio -async def test_fetch_github_issue_delegates_to_client(tool_ctx): +async def test_fetch_github_issue_delegates_to_client(tool_ctx_kitchen_open): mock_client = AsyncMock() mock_client.fetch_issue.return_value = { "success": True, @@ -160,7 +106,7 @@ async def test_fetch_github_issue_delegates_to_client(tool_ctx): "labels": [], "content": "# T", } - tool_ctx.github_client = mock_client + tool_ctx_kitchen_open.github_client = mock_client result = json.loads(await fetch_github_issue("owner/repo#1")) assert result["success"] is True mock_client.fetch_issue.assert_called_once_with("owner/repo#1", include_comments=True) @@ -180,6 +126,27 @@ def test_fetch_github_issue_in_gated_tools(): assert "fetch_github_issue" not in UNGATED_TOOLS +def test_fetch_github_issue_docstring_is_role_scoped() -> None: + """Docstring must not contain a blanket 'use automatically whenever' directive. + + The old directive caused orchestrator sessions to fetch the issue before + dispatch_food_truck even when only routing the URL downstream as an ingredient, + wasting ~2K+ tokens per dispatch call. + """ + doc = fetch_github_issue.__doc__ or "" + doc_lower = doc.lower() + assert "automatically whenever" not in doc_lower, ( + "fetch_github_issue docstring must not contain blanket 'automatically whenever' " + "directive — it causes dispatcher sessions to fetch issues they don't need. " + "See issue #1170." + ) + assert "dispatch_food_truck" in doc_lower or "ingredient" in doc_lower, ( + "fetch_github_issue docstring must include role-aware guidance anchored on " + "'dispatch_food_truck' or 'ingredient' — generic routing words are too loose " + "to catch accidental docstring regression. See issue #1170." + ) + + # --------------------------------------------------------------------------- # get_issue_title # --------------------------------------------------------------------------- @@ -203,7 +170,7 @@ async def test_get_issue_title_no_client(tool_ctx) -> None: @pytest.mark.anyio -async def test_get_issue_title_success(tool_ctx) -> None: +async def test_get_issue_title_success(tool_ctx_kitchen_open) -> None: """client.fetch_title returns data → JSON with title and slug.""" title_data = { "success": True, @@ -211,8 +178,8 @@ async def test_get_issue_title_success(tool_ctx) -> None: "title": "Fix the bug", "slug": "fix-the-bug", } - tool_ctx.github_client = AsyncMock() - tool_ctx.github_client.fetch_title = AsyncMock(return_value=title_data) + tool_ctx_kitchen_open.github_client = AsyncMock() + tool_ctx_kitchen_open.github_client.fetch_title = AsyncMock(return_value=title_data) result = json.loads(await get_issue_title("owner/repo#42")) assert result["success"] is True @@ -221,7 +188,7 @@ async def test_get_issue_title_success(tool_ctx) -> None: class TestGetIssueTitleTool: @pytest.mark.anyio - async def test_get_issue_title_success(self, tool_ctx): + async def test_get_issue_title_success(self, tool_ctx_kitchen_open): """Delegates to github_client.fetch_title; returns JSON result.""" mock_client = AsyncMock() mock_client.fetch_title.return_value = { @@ -230,7 +197,7 @@ async def test_get_issue_title_success(self, tool_ctx): "title": "Fix merge conflict triage", "slug": "fix-merge-conflict-triage", } - tool_ctx.github_client = mock_client + tool_ctx_kitchen_open.github_client = mock_client result = json.loads(await get_issue_title("https://github.com/owner/repo/issues/42")) assert result["success"] is True assert result["number"] == 42 @@ -251,606 +218,3 @@ def test_get_issue_title_is_gated(self): """'get_issue_title' in GATED_TOOLS.""" assert "get_issue_title" in GATED_TOOLS assert "get_issue_title" not in UNGATED_TOOLS - - -# --------------------------------------------------------------------------- -# report_bug config defaults -# --------------------------------------------------------------------------- - - -def test_report_bug_config_defaults(): - cfg = AutomationConfig() - assert cfg.report_bug.timeout == 600 - assert cfg.report_bug.model is None - assert cfg.report_bug.report_dir is None - assert cfg.report_bug.github_filing is True - assert "autoreported" in cfg.report_bug.github_labels - assert "bug" in cfg.report_bug.github_labels - - -def test_github_config_defaults(): - config = AutomationConfig() - assert config.github.token is None - assert config.github.default_repo is None - - -# --------------------------------------------------------------------------- -# TestReportBugTool -# --------------------------------------------------------------------------- - - -class TestReportBugTool: - @pytest.mark.anyio - async def test_report_bug_failure_includes_session_id_and_stderr(self, tool_ctx, tmp_path): - """Blocking failure response must include session_id and stderr for diagnosis.""" - tool_ctx.config.report_bug.report_dir = str(tmp_path / "bug-reports") - tool_ctx.config.report_bug.github_filing = False - - mock_executor = AsyncMock() - mock_executor.run.return_value = SkillResult( - success=False, - result="", - session_id="fail-session-id", - subtype="missing_completion_marker", - is_error=True, - exit_code=1, - needs_retry=False, - retry_reason=RetryReason.NONE, - stderr="Claude crashed", - ) - tool_ctx.executor = mock_executor - - result = json.loads(await report_bug("some error", str(tmp_path), severity="blocking")) - - assert result["success"] is False - assert result["session_id"] == "fail-session-id" - assert result["stderr"] == "Claude crashed" - - @pytest.mark.anyio - async def test_report_bug_passes_expected_output_patterns_to_executor( - self, tool_ctx, tmp_path - ): - """output_pattern_resolver is consulted and patterns are passed to executor.run().""" - tool_ctx.config.report_bug.report_dir = str(tmp_path / "bug-reports") - tool_ctx.config.report_bug.github_filing = False - - mock_executor = AsyncMock() - mock_executor.run.return_value = SkillResult( - success=True, - result="## Report\nfindings", - session_id="sid", - subtype="success", - is_error=False, - exit_code=0, - needs_retry=False, - retry_reason=RetryReason.NONE, - stderr="", - ) - tool_ctx.executor = mock_executor - tool_ctx.output_pattern_resolver = lambda cmd: ["---bug-fingerprint---"] - - await report_bug("error ctx", str(tmp_path), severity="blocking") - - call_kwargs = mock_executor.run.call_args.kwargs - assert call_kwargs.get("expected_output_patterns") == ["---bug-fingerprint---"] - - -# --------------------------------------------------------------------------- -# _extract_block unit tests -# --------------------------------------------------------------------------- - - -def test_extract_block_returns_lines_within_delimiters(): - text = "preamble\n---start---\nline1\nline2\n---end---\npostamble" - assert _extract_block(text, "---start---", "---end---") == ["line1", "line2"] - - -def test_extract_block_no_start_returns_empty(): - assert _extract_block("no delimiters here", "---start---", "---end---") == [] - - -def test_extract_block_no_end_returns_empty(): - # end delimiter absent — no complete block - text = "---start---\nline1\nline2" - assert _extract_block(text, "---start---", "---end---") == [] - - -def test_extract_block_empty_block_returns_empty_list(): - text = "---start---\n---end---" - assert _extract_block(text, "---start---", "---end---") == [] - - -def test_extract_block_preserves_whitespace_in_lines(): - text = "---start---\n indented\n---end---" - assert _extract_block(text, "---start---", "---end---") == [" indented"] - - -# --------------------------------------------------------------------------- -# _parse_prepare_result unit tests -# --------------------------------------------------------------------------- - - -def test_parse_prepare_result_valid_json(): - payload = '{"success": true, "issue_url": "https://github.com/x/y/issues/1"}' - text = f"{_PREPARE_RESULT_START}\n{payload}\n{_PREPARE_RESULT_END}" - result = _parse_prepare_result(text) - assert result == {"success": True, "issue_url": "https://github.com/x/y/issues/1"} - - -def test_parse_prepare_result_no_block(): - result = _parse_prepare_result("no block here") - assert result == {"success": False, "error": "no result block found"} - - -def test_parse_prepare_result_invalid_json(): - text = f"{_PREPARE_RESULT_START}\nnot-json\n{_PREPARE_RESULT_END}" - result = _parse_prepare_result(text) - assert result == {"success": False, "error": "result block contained invalid JSON"} - - -# --------------------------------------------------------------------------- -# _parse_enrich_result unit tests -# --------------------------------------------------------------------------- - - -def test_parse_enrich_result_valid_json(): - payload = '{"enriched": [42], "skipped_already_enriched": []}' - text = f"{_ENRICH_RESULT_START}\n{payload}\n{_ENRICH_RESULT_END}" - result = _parse_enrich_result(text) - assert result == {"enriched": [42], "skipped_already_enriched": []} - - -def test_parse_enrich_result_no_block(): - result = _parse_enrich_result("no block here") - assert result == {"success": False, "error": "no result block found"} - - -def test_parse_enrich_result_invalid_json(): - text = f"{_ENRICH_RESULT_START}\nnot-json\n{_ENRICH_RESULT_END}" - result = _parse_enrich_result(text) - assert result == {"success": False, "error": "result block contained invalid JSON"} - - -# --------------------------------------------------------------------------- -# report_bug gate tests -# --------------------------------------------------------------------------- - - -@pytest.mark.anyio -async def test_report_bug_gate_closed(tool_ctx): - tool_ctx.gate.disable() - result = json.loads(await report_bug("some error", "/tmp")) - assert result["success"] is False - assert "not enabled" in result["result"].lower() or "gate" in result["result"].lower() - - -@pytest.mark.anyio -async def test_report_bug_no_executor(tool_ctx): - tool_ctx.executor = None - result = json.loads(await report_bug("error ctx", "/tmp")) - assert result["success"] is False - assert "executor" in result["error"].lower() - - -# --------------------------------------------------------------------------- -# Blocking mode -# --------------------------------------------------------------------------- - - -@pytest.mark.anyio -async def test_report_bug_blocking_success(tool_ctx, tmp_path): - """Blocking mode awaits the session and returns status=complete.""" - tool_ctx.config.report_bug.report_dir = str(tmp_path / "bug-reports") - tool_ctx.config.report_bug.github_filing = False - - mock_executor = AsyncMock() - mock_executor.run.return_value = _skill_ok("## Report\nroot cause found") - tool_ctx.executor = mock_executor - - result = json.loads(await report_bug("KeyError in foo", str(tmp_path), severity="blocking")) - - assert result["success"] is True - assert result["status"] == "complete" - assert "report" in result - assert "report_path" in result - mock_executor.run.assert_awaited_once() - - -@pytest.mark.anyio -async def test_report_bug_blocking_failure_propagated(tool_ctx, tmp_path): - """If the headless session fails, status=failed is returned.""" - tool_ctx.config.report_bug.report_dir = str(tmp_path / "bug-reports") - tool_ctx.config.report_bug.github_filing = False - - mock_executor = AsyncMock() - mock_executor.run.return_value = _skill_fail() - tool_ctx.executor = mock_executor - - result = json.loads(await report_bug("crash here", str(tmp_path), severity="blocking")) - - assert result["success"] is False - assert result["status"] == "failed" - - -@pytest.mark.anyio -async def test_report_bug_blocking_writes_report_file(tool_ctx, tmp_path): - """The report text must be written to the resolved report_path.""" - report_dir = tmp_path / "rpts" - tool_ctx.config.report_bug.report_dir = str(report_dir) - tool_ctx.config.report_bug.github_filing = False - - mock_executor = AsyncMock() - mock_executor.run.return_value = _skill_ok("# Bug Report\nfoo bar") - tool_ctx.executor = mock_executor - - result = json.loads(await report_bug("err", str(tmp_path), severity="blocking")) - - report_path = Path(result["report_path"]) - assert report_path.exists() - assert "Bug Report" in report_path.read_text() - - -# --------------------------------------------------------------------------- -# Non-blocking mode -# --------------------------------------------------------------------------- - - -@pytest.mark.anyio -async def test_report_bug_non_blocking_returns_immediately(tool_ctx, tmp_path): - """Non-blocking mode must return dispatched before the session completes.""" - tool_ctx.config.report_bug.report_dir = str(tmp_path / "bug-reports") - tool_ctx.config.report_bug.github_filing = False - - ready = anyio.Event() - - async def slow_run(*args, **kwargs): - await ready.wait() # blocks until test signals - return _skill_ok() - - mock_executor = MagicMock() - mock_executor.run = slow_run - tool_ctx.executor = mock_executor - - result = json.loads(await report_bug("error ctx", str(tmp_path), severity="non_blocking")) - - assert result["success"] is True - assert result["status"] == "dispatched" - assert "report_path" in result - - # Let the background task finish cleanly. - ready.set() - await anyio.sleep(0) - - -@pytest.mark.anyio -async def test_report_bug_non_blocking_default_severity(tool_ctx, tmp_path): - """The default severity is non_blocking.""" - tool_ctx.config.report_bug.report_dir = str(tmp_path / "bug-reports") - tool_ctx.config.report_bug.github_filing = False - - mock_executor = AsyncMock() - mock_executor.run.return_value = _skill_ok() - tool_ctx.executor = mock_executor - - result = json.loads(await report_bug("err", str(tmp_path))) - assert result["status"] == "dispatched" - - -# --------------------------------------------------------------------------- -# Non-blocking outcome tests (supervised background task) -# --------------------------------------------------------------------------- - - -@pytest.mark.anyio -async def test_report_bug_non_blocking_outcome_writes_report_file(tool_ctx, tmp_path): - """After the background task completes, report_path must exist with content.""" - tool_ctx.config.report_bug.report_dir = str(tmp_path / "bug-reports") - tool_ctx.config.report_bug.github_filing = False - - mock_executor = AsyncMock() - mock_executor.run.return_value = _skill_ok("# Bug Report\nroot cause: missing guard") - tool_ctx.executor = mock_executor - - result = json.loads( - await report_bug("KeyError in foo", str(tmp_path), severity="non_blocking") - ) - assert result["status"] == "dispatched" - - report_path = Path(result["report_path"]) - await tool_ctx.background.drain() - - assert report_path.exists(), "report_path must exist after background task completes" - assert "Bug Report" in report_path.read_text() - - -@pytest.mark.anyio -async def test_report_bug_non_blocking_writes_status_file_on_success(tool_ctx, tmp_path): - """A status.json file must be written with status='complete' after successful dispatch.""" - tool_ctx.config.report_bug.report_dir = str(tmp_path / "bug-reports") - tool_ctx.config.report_bug.github_filing = False - - mock_executor = AsyncMock() - mock_executor.run.return_value = _skill_ok("# Report\nfoo") - tool_ctx.executor = mock_executor - - result = json.loads(await report_bug("err", str(tmp_path), severity="non_blocking")) - report_path = Path(result["report_path"]) - - # Before task runs: pending status file should exist - status_path = report_path.with_suffix(".status.json") - assert status_path.exists(), "status file must be written synchronously on dispatch" - pending = json.loads(status_path.read_text()) - assert pending["status"] == "pending" - - # After task completes: status should update to complete - await tool_ctx.background.drain() - - data = json.loads(status_path.read_text()) - assert data["status"] == "complete" - assert "completed_at" in data - - -@pytest.mark.anyio -async def test_report_bug_non_blocking_writes_status_file_on_failure(tool_ctx, tmp_path): - """When the headless session fails, status.json must reflect the failure.""" - tool_ctx.config.report_bug.report_dir = str(tmp_path / "bug-reports") - tool_ctx.config.report_bug.github_filing = False - - mock_executor = AsyncMock() - mock_executor.run.return_value = _skill_fail() - tool_ctx.executor = mock_executor - - result = json.loads(await report_bug("crash here", str(tmp_path), severity="non_blocking")) - status_path = Path(result["report_path"]).with_suffix(".status.json") - - await tool_ctx.background.drain() - - data = json.loads(status_path.read_text()) - assert data["status"] == "failed" - assert "completed_at" in data - - -@pytest.mark.anyio -async def test_report_bug_non_blocking_executor_raises_is_observed(tool_ctx, tmp_path): - """If executor.run() raises, the exception must be logged — not silently dropped.""" - - tool_ctx.config.report_bug.report_dir = str(tmp_path / "bug-reports") - tool_ctx.config.report_bug.github_filing = False - - mock_executor = AsyncMock() - mock_executor.run.side_effect = RuntimeError("executor exploded") - tool_ctx.executor = mock_executor - - result = json.loads(await report_bug("error ctx", str(tmp_path), severity="non_blocking")) - assert result["status"] == "dispatched" - - await tool_ctx.background.drain() - - # The exception must be captured and logged — not silently dropped - status_path = Path(result["report_path"]).with_suffix(".status.json") - assert status_path.exists(), "status file must exist even when executor raises" - data = json.loads(status_path.read_text()) - assert data["status"] == "failed" - assert "executor exploded" in data.get("error", "") - - -@pytest.mark.anyio -async def test_report_bug_no_pending_tasks_after_completion(tool_ctx, tmp_path): - """After background task completes, no tasks remain in the supervisor's pending set.""" - tool_ctx.config.report_bug.report_dir = str(tmp_path / "bug-reports") - tool_ctx.config.report_bug.github_filing = False - - mock_executor = AsyncMock() - mock_executor.run.return_value = _skill_ok() - tool_ctx.executor = mock_executor - - await report_bug("err", str(tmp_path), severity="non_blocking") - - await tool_ctx.background.drain() - - assert tool_ctx.background.pending_count == 0 - - -# --------------------------------------------------------------------------- -# GitHub filing — blocking mode (easier to assert synchronously) -# --------------------------------------------------------------------------- - - -@pytest.mark.anyio -async def test_report_bug_creates_github_issue_on_no_duplicate(tool_ctx, tmp_path): - """When no matching issue is found, create_issue is called.""" - tool_ctx.config.report_bug.report_dir = str(tmp_path / "bug-reports") - tool_ctx.config.report_bug.github_filing = True - tool_ctx.config.github.default_repo = "owner/repo" - - report_with_fp = ( - f"{_FINGERPRINT_START}\n" - "KeyError in recipe/validator.py: missing ref\n" - f"{_FINGERPRINT_END}\n" - "## Bug Report\ndetails" - ) - mock_executor = AsyncMock() - mock_executor.run.return_value = _skill_ok(report_with_fp) - tool_ctx.executor = mock_executor - - mock_gh = AsyncMock() - mock_gh.has_token = True - mock_gh.search_issues.return_value = {"success": True, "total_count": 0, "items": []} - mock_gh.create_issue.return_value = { - "success": True, - "issue_number": 99, - "url": "https://github.com/owner/repo/issues/99", - } - tool_ctx.github_client = mock_gh - - result = json.loads(await report_bug("KeyError crash", str(tmp_path), severity="blocking")) - - assert result["success"] is True - mock_gh.search_issues.assert_awaited_once() - mock_gh.create_issue.assert_awaited_once() - assert result["github"]["duplicate"] is False - assert result["github"]["issue_url"] == "https://github.com/owner/repo/issues/99" - - -@pytest.mark.anyio -async def test_report_bug_comments_on_duplicate_issue(tool_ctx, tmp_path): - """When a matching issue exists and error_context is new, add_comment is called.""" - tool_ctx.config.report_bug.report_dir = str(tmp_path / "bug-reports") - tool_ctx.config.report_bug.github_filing = True - tool_ctx.config.github.default_repo = "owner/repo" - - mock_executor = AsyncMock() - mock_executor.run.return_value = _skill_ok( - "## Report\n" + _FINGERPRINT_START + "\nfp\n" + _FINGERPRINT_END - ) - tool_ctx.executor = mock_executor - - mock_gh = AsyncMock() - mock_gh.has_token = True - mock_gh.search_issues.return_value = { - "success": True, - "total_count": 1, - "items": [ - { - "number": 7, - "title": "fp", - "html_url": "https://github.com/owner/repo/issues/7", - "body": "Original body — no error_context here", - "state": "open", - } - ], - } - mock_gh.add_comment.return_value = {"success": True, "comment_id": 55, "url": "u"} - tool_ctx.github_client = mock_gh - - result = json.loads( - await report_bug("brand new error text", str(tmp_path), severity="blocking") - ) - - assert result["success"] is True - mock_gh.add_comment.assert_awaited_once() - assert result["github"]["duplicate"] is True - assert result["github"]["comment_added"] is True - - -@pytest.mark.anyio -async def test_report_bug_skips_comment_if_already_present(tool_ctx, tmp_path): - """If error_context is already in the issue body, no comment is posted.""" - tool_ctx.config.report_bug.report_dir = str(tmp_path / "bug-reports") - tool_ctx.config.report_bug.github_filing = True - tool_ctx.config.github.default_repo = "owner/repo" - - error_ctx = "exact error text already filed" - mock_executor = AsyncMock() - mock_executor.run.return_value = _skill_ok() - tool_ctx.executor = mock_executor - - mock_gh = AsyncMock() - mock_gh.has_token = True - mock_gh.search_issues.return_value = { - "success": True, - "total_count": 1, - "items": [ - { - "number": 3, - "title": "fp", - "html_url": "https://github.com/owner/repo/issues/3", - "body": f"body contains {error_ctx} already", - "state": "open", - } - ], - } - tool_ctx.github_client = mock_gh - - result = json.loads(await report_bug(error_ctx, str(tmp_path), severity="blocking")) - - mock_gh.add_comment.assert_not_awaited() - assert result["github"]["comment_added"] is False - - -@pytest.mark.anyio -async def test_report_bug_skips_github_if_no_token(tool_ctx, tmp_path): - tool_ctx.config.report_bug.report_dir = str(tmp_path / "bug-reports") - tool_ctx.config.report_bug.github_filing = True - - mock_executor = AsyncMock() - mock_executor.run.return_value = _skill_ok() - tool_ctx.executor = mock_executor - - mock_gh = AsyncMock() - mock_gh.has_token = False - tool_ctx.github_client = mock_gh - - result = json.loads(await report_bug("err", str(tmp_path), severity="blocking")) - - assert result["success"] is True - mock_gh.search_issues.assert_not_awaited() - assert result["github"]["skipped"] is True - assert result["github"]["reason"] == "no_token" - - -@pytest.mark.anyio -async def test_report_bug_skips_github_if_no_default_repo(tool_ctx, tmp_path): - tool_ctx.config.report_bug.report_dir = str(tmp_path / "bug-reports") - tool_ctx.config.report_bug.github_filing = True - tool_ctx.config.github.default_repo = None - - mock_executor = AsyncMock() - mock_executor.run.return_value = _skill_ok() - tool_ctx.executor = mock_executor - - mock_gh = AsyncMock() - mock_gh.has_token = True - tool_ctx.github_client = mock_gh - - result = json.loads(await report_bug("err", str(tmp_path), severity="blocking")) - - mock_gh.search_issues.assert_not_awaited() - assert result["github"]["skipped"] is True - - -@pytest.mark.anyio -async def test_report_bug_github_filing_disabled(tool_ctx, tmp_path): - """github_filing=false must skip all GitHub calls.""" - tool_ctx.config.report_bug.report_dir = str(tmp_path / "bug-reports") - tool_ctx.config.report_bug.github_filing = False - tool_ctx.config.github.default_repo = "owner/repo" - - mock_executor = AsyncMock() - mock_executor.run.return_value = _skill_ok() - tool_ctx.executor = mock_executor - - mock_gh = AsyncMock() - mock_gh.has_token = True - tool_ctx.github_client = mock_gh - - result = json.loads(await report_bug("err", str(tmp_path), severity="blocking")) - - mock_gh.search_issues.assert_not_awaited() - assert result["github"] == {} - - -@pytest.mark.anyio -async def test_report_bug_blocking_github_client_raises_does_not_propagate(tool_ctx, tmp_path): - """If the GitHub client raises unexpectedly, the error must be captured in the github dict.""" - tool_ctx.config.report_bug.report_dir = str(tmp_path / "bug-reports") - tool_ctx.config.report_bug.github_filing = True - tool_ctx.config.github.default_repo = "owner/repo" - - mock_executor = AsyncMock() - mock_executor.run.return_value = _skill_ok( - "# Report\n" + _FINGERPRINT_START + "\nfp1\n" + _FINGERPRINT_END - ) - tool_ctx.executor = mock_executor - - mock_gh = MagicMock() - mock_gh.has_token = True - mock_gh.search_issues = AsyncMock(side_effect=RuntimeError("network failure")) - tool_ctx.github_client = mock_gh - - # Must not raise — exception is captured in github dict - result = json.loads(await report_bug("err", str(tmp_path), severity="blocking")) - assert result["success"] is True # session succeeded - assert result["github"].get("skipped") is True - assert "unexpected error" in result["github"].get("reason", "") - assert "network failure" in result["github"].get("reason", "") diff --git a/tests/server/test_tools_github_api_tracking.py b/tests/server/test_tools_github_api_tracking.py new file mode 100644 index 0000000000..649d4db622 --- /dev/null +++ b/tests/server/test_tools_github_api_tracking.py @@ -0,0 +1,162 @@ +from unittest.mock import patch + +import pytest + +from autoskillit.core.types._type_results import ProviderOutcome +from autoskillit.core.types._type_results_execution import ( + RecipeIdentity, + SessionTelemetry, +) +from autoskillit.core.types._type_subprocess import SubprocessResult, TerminationReason +from tests.fakes import MockSubprocessRunner + +pytestmark = [pytest.mark.layer("server"), pytest.mark.small, pytest.mark.anyio] + + +async def test_gh_cli_call_is_recorded(build_ctx): + from autoskillit.pipeline.github_api_log import DefaultGitHubApiLog + + log = DefaultGitHubApiLog() + ctx = build_ctx(github_api_log=log) + runner = MockSubprocessRunner() + runner.set_default( + SubprocessResult( + returncode=0, + stdout="[]", + stderr="", + termination=TerminationReason.NATURAL_EXIT, + pid=1, + ) + ) + ctx.runner = runner + + from autoskillit.server._subprocess import _run_subprocess + + with patch("autoskillit.server._subprocess._get_ctx", return_value=ctx): + await _run_subprocess(["gh", "pr", "list", "--json", "number"], cwd="/tmp", timeout=30) + + usage = log.to_usage("sess-1") + assert usage is not None + assert usage["total_requests"] == 1 + assert usage["by_source"]["gh_cli"] == 1 + + +async def test_non_gh_command_is_not_recorded(build_ctx): + from autoskillit.pipeline.github_api_log import DefaultGitHubApiLog + + log = DefaultGitHubApiLog() + ctx = build_ctx(github_api_log=log) + runner = MockSubprocessRunner() + runner.set_default( + SubprocessResult( + returncode=0, + stdout="ok", + stderr="", + termination=TerminationReason.NATURAL_EXIT, + pid=1, + ) + ) + ctx.runner = runner + + from autoskillit.server._subprocess import _run_subprocess + + with patch("autoskillit.server._subprocess._get_ctx", return_value=ctx): + await _run_subprocess(["git", "status"], cwd="/tmp", timeout=30) + + assert log.to_usage("sess-1") is None + + +async def test_gh_cli_records_exit_code_and_latency(build_ctx): + from autoskillit.pipeline.github_api_log import DefaultGitHubApiLog + + log = DefaultGitHubApiLog() + ctx = build_ctx(github_api_log=log) + runner = MockSubprocessRunner() + runner.set_default( + SubprocessResult( + returncode=1, + stdout="", + stderr="error", + termination=TerminationReason.NATURAL_EXIT, + pid=1, + ) + ) + ctx.runner = runner + + from autoskillit.server._subprocess import _run_subprocess + + _tick = 0.0 + + def _fake_monotonic() -> float: + nonlocal _tick + _tick += 0.001 + return _tick + + with ( + patch("autoskillit.server._subprocess._get_ctx", return_value=ctx), + patch("autoskillit.server._subprocess.time.monotonic", _fake_monotonic), + ): + await _run_subprocess(["gh", "api", "repos/o/r/issues"], cwd="/tmp", timeout=30) + + usage = log.to_usage("sess-1") + assert usage["total_requests"] == 1 + assert usage["total_latency_ms"] > 0 + assert log._entries[0].status_code == 1 + + +async def test_flush_session_log_writes_github_api_usage(tmp_path): + import json + + from autoskillit.execution.session_log import flush_session_log + from autoskillit.pipeline.github_api_log import DefaultGitHubApiLog + + log = DefaultGitHubApiLog() + await log.record_httpx( + method="GET", + path="/repos/o/r/issues/1", + status_code=200, + latency_ms=100.0, + rate_limit_remaining=4900, + rate_limit_used=100, + rate_limit_reset=0, + timestamp="2026-04-27T10:00:00Z", + ) + + _usage = log.drain("test-session") + flush_session_log( + log_dir=str(tmp_path), + cwd="/tmp", + session_id="test-session", + pid=1234, + skill_command="test", + success=True, + subtype="headless", + exit_code=0, + start_ts="2026-04-27T10:00:00Z", + proc_snapshots=None, + telemetry=SessionTelemetry( + token_usage=None, + timing_seconds=None, + audit_record=None, + github_api_usage=_usage, + github_api_requests=_usage.get("total_requests", 0) if _usage else 0, + loc_insertions=0, + loc_deletions=0, + ), + provider_outcome=ProviderOutcome.none_used(), + recipe_identity=RecipeIdentity.empty(), + ) + + usage_file = tmp_path / "sessions" / "test-session" / "github_api_usage.json" + assert usage_file.exists() + data = json.loads(usage_file.read_text()) + assert data["session_id"] == "test-session" + assert data["total_requests"] == 1 + + summary_file = tmp_path / "sessions" / "test-session" / "summary.json" + summary = json.loads(summary_file.read_text()) + assert summary["github_api_requests"] == 1 + + index_file = tmp_path / "sessions.jsonl" + lines = [json.loads(line) for line in index_file.read_text().splitlines()] + assert lines[0]["github_api_requests"] == 1 diff --git a/tests/server/test_tools_integrations.py b/tests/server/test_tools_integrations.py index 8cd893ab51..01360c3f61 100644 --- a/tests/server/test_tools_integrations.py +++ b/tests/server/test_tools_integrations.py @@ -9,7 +9,7 @@ from autoskillit.core import SkillResult from autoskillit.core.types import RetryReason -from autoskillit.server.tools_issue_lifecycle import ( +from autoskillit.server.tools.tools_issue_lifecycle import ( _PREPARE_RESULT_END, _PREPARE_RESULT_START, claim_issue, @@ -17,9 +17,11 @@ prepare_issue, release_issue, ) -from autoskillit.server.tools_pr_ops import bulk_close_issues, get_pr_reviews +from autoskillit.server.tools.tools_pr_ops import bulk_close_issues, get_pr_reviews from tests.conftest import _make_result +pytestmark = [pytest.mark.layer("server"), pytest.mark.small] + # --------------------------------------------------------------------------- # claim_issue / release_issue / prepare_issue / enrich_issues — gated tools # --------------------------------------------------------------------------- @@ -32,48 +34,39 @@ def test_claim_issue_is_gated(self): assert "claim_issue" in GATED_TOOLS @pytest.mark.anyio - async def test_claim_issue_returns_gate_error_when_kitchen_closed(self, tool_ctx): - from autoskillit.pipeline.gate import DefaultGateState - - tool_ctx.gate = DefaultGateState(enabled=False) - result = json.loads(await claim_issue("https://github.com/owner/repo/issues/42")) - assert result["success"] is False - assert result["subtype"] == "gate_error" - - @pytest.mark.anyio - async def test_claim_issue_returns_error_without_github_client(self, tool_ctx): - tool_ctx.github_client = None + async def test_claim_issue_returns_error_without_github_client(self, tool_ctx_kitchen_open): + tool_ctx_kitchen_open.github_client = None result = json.loads(await claim_issue("https://github.com/owner/repo/issues/42")) assert result["success"] is False assert "error" in result @pytest.mark.anyio - async def test_claim_issue_success(self, tool_ctx): + async def test_claim_issue_success(self, tool_ctx_kitchen_open): mock_client = AsyncMock() mock_client.fetch_issue.return_value = {"success": True, "labels": []} mock_client.ensure_label.return_value = {"success": True, "created": True} mock_client.add_labels.return_value = {"success": True, "labels": ["in-progress"]} - tool_ctx.github_client = mock_client + tool_ctx_kitchen_open.github_client = mock_client result = json.loads(await claim_issue("https://github.com/owner/repo/issues/42")) assert result["success"] is True assert result["claimed"] is True assert result["issue_number"] == 42 @pytest.mark.anyio - async def test_claim_issue_already_claimed(self, tool_ctx): + async def test_claim_issue_already_claimed(self, tool_ctx_kitchen_open): mock_client = AsyncMock() mock_client.fetch_issue.return_value = { "success": True, "labels": [{"name": "in-progress"}], } - tool_ctx.github_client = mock_client + tool_ctx_kitchen_open.github_client = mock_client result = json.loads(await claim_issue("https://github.com/owner/repo/issues/42")) assert result["success"] is True assert result["claimed"] is False # P5F4-T1 @pytest.mark.anyio - async def test_claim_issue_binds_structlog_context(self, tool_ctx, monkeypatch): + async def test_claim_issue_binds_structlog_context(self, tool_ctx_kitchen_open, monkeypatch): """claim_issue must bind structlog context vars via bind_contextvars.""" import structlog @@ -85,7 +78,7 @@ def fake_bind_contextvars(**kwargs): monkeypatch.setattr(structlog.contextvars, "bind_contextvars", fake_bind_contextvars) monkeypatch.setattr(structlog.contextvars, "clear_contextvars", lambda: None) - tool_ctx.github_client = None # triggers early return after bind + tool_ctx_kitchen_open.github_client = None # triggers early return after bind await claim_issue(issue_url="https://github.com/owner/repo/issues/1") assert captured == { @@ -95,7 +88,7 @@ def fake_bind_contextvars(**kwargs): @pytest.mark.anyio async def test_claim_issue_allow_reentry_true_returns_claimed_true_when_already_labeled( - self, tool_ctx + self, tool_ctx_kitchen_open ): """allow_reentry=True with label present: returns claimed=True and reentry=True.""" mock_client = AsyncMock() @@ -103,7 +96,7 @@ async def test_claim_issue_allow_reentry_true_returns_claimed_true_when_already_ "success": True, "labels": [{"name": "in-progress"}], } - tool_ctx.github_client = mock_client + tool_ctx_kitchen_open.github_client = mock_client result = json.loads( await claim_issue("https://github.com/owner/repo/issues/42", allow_reentry=True) ) @@ -114,7 +107,7 @@ async def test_claim_issue_allow_reentry_true_returns_claimed_true_when_already_ @pytest.mark.anyio async def test_claim_issue_allow_reentry_false_returns_claimed_false_when_already_labeled( - self, tool_ctx + self, tool_ctx_kitchen_open ): """Default allow_reentry=False: claimed=False when label already present.""" mock_client = AsyncMock() @@ -122,27 +115,77 @@ async def test_claim_issue_allow_reentry_false_returns_claimed_false_when_alread "success": True, "labels": [{"name": "in-progress"}], } - tool_ctx.github_client = mock_client + tool_ctx_kitchen_open.github_client = mock_client result = json.loads(await claim_issue("https://github.com/owner/repo/issues/42")) assert result["success"] is True assert result["claimed"] is False assert "reentry" not in result @pytest.mark.anyio - async def test_claim_issue_allow_reentry_true_still_claims_when_label_absent(self, tool_ctx): + async def test_claim_issue_allow_reentry_true_still_claims_when_label_absent( + self, tool_ctx_kitchen_open + ): """allow_reentry=True with no pre-existing label performs normal claim.""" mock_client = AsyncMock() mock_client.fetch_issue.return_value = {"success": True, "labels": []} mock_client.ensure_label.return_value = {"success": True, "created": True} - mock_client.add_labels.return_value = {"success": True, "labels": ["in-progress"]} - tool_ctx.github_client = mock_client + mock_client.swap_labels.return_value = {"success": True, "labels": ["in-progress"]} + tool_ctx_kitchen_open.github_client = mock_client result = json.loads( await claim_issue("https://github.com/owner/repo/issues/42", allow_reentry=True) ) assert result["success"] is True assert result["claimed"] is True assert result.get("reentry", False) is False - mock_client.add_labels.assert_called_once_with("owner", "repo", 42, ["in-progress"]) + call_kwargs = mock_client.swap_labels.call_args.kwargs + assert set(call_kwargs["remove_labels"]) == {"queued", "fail"} + assert call_kwargs["add_labels"] == ["in-progress"] + + @pytest.mark.anyio + async def test_claim_issue_with_queued_label(self, tool_ctx_kitchen_open): + """claim_issue with label=queued uses registry color/description and removes fail.""" + mock_client = AsyncMock() + mock_client.fetch_issue.return_value = {"success": True, "labels": []} + mock_client.ensure_label.return_value = {"success": True, "created": True} + mock_client.swap_labels.return_value = {"success": True, "labels": ["queued"]} + tool_ctx_kitchen_open.github_client = mock_client + result = json.loads( + await claim_issue(issue_url="https://github.com/owner/repo/issues/1", label="queued") + ) + assert result["success"] is True + assert result["claimed"] is True + mock_client.ensure_label.assert_called_once_with( + "owner", + "repo", + "queued", + color="c2e0c6", + description="Issue claimed by orchestrator, waiting for recipe pickup", + ) + call_kwargs = mock_client.swap_labels.call_args.kwargs + assert set(call_kwargs["remove_labels"]) == {"fail"} + assert call_kwargs["add_labels"] == ["queued"] + + @pytest.mark.anyio + async def test_claim_issue_default_removes_queued_and_fail(self, tool_ctx_kitchen_open): + """claim_issue with default label removes both queued and fail labels.""" + mock_client = AsyncMock() + mock_client.fetch_issue.return_value = {"success": True, "labels": []} + mock_client.ensure_label.return_value = {"success": True, "created": True} + mock_client.swap_labels.return_value = {"success": True, "labels": ["in-progress"]} + tool_ctx_kitchen_open.github_client = mock_client + result = json.loads(await claim_issue(issue_url="https://github.com/owner/repo/issues/1")) + assert result["success"] is True + assert result["claimed"] is True + mock_client.ensure_label.assert_called_once_with( + "owner", + "repo", + "in-progress", + color="fbca04", + description="Issue is actively being processed by a pipeline session", + ) + call_kwargs = mock_client.swap_labels.call_args.kwargs + assert set(call_kwargs["remove_labels"]) >= {"queued", "fail"} + assert call_kwargs["add_labels"] == ["in-progress"] class TestReleaseIssueTool: @@ -152,33 +195,24 @@ def test_release_issue_is_gated(self): assert "release_issue" in GATED_TOOLS @pytest.mark.anyio - async def test_release_issue_returns_gate_error_when_kitchen_closed(self, tool_ctx): - from autoskillit.pipeline.gate import DefaultGateState - - tool_ctx.gate = DefaultGateState(enabled=False) - result = json.loads(await release_issue("https://github.com/owner/repo/issues/42")) - assert result["success"] is False - assert result["subtype"] == "gate_error" - - @pytest.mark.anyio - async def test_release_issue_returns_error_without_github_client(self, tool_ctx): - tool_ctx.github_client = None + async def test_release_issue_returns_error_without_github_client(self, tool_ctx_kitchen_open): + tool_ctx_kitchen_open.github_client = None result = json.loads(await release_issue("https://github.com/owner/repo/issues/42")) assert result["success"] is False assert "error" in result @pytest.mark.anyio - async def test_release_issue_success(self, tool_ctx): + async def test_release_issue_success(self, tool_ctx_kitchen_open): mock_client = AsyncMock() mock_client.remove_label.return_value = {"success": True} - tool_ctx.github_client = mock_client + tool_ctx_kitchen_open.github_client = mock_client result = json.loads(await release_issue("https://github.com/owner/repo/issues/42")) assert result["success"] is True assert result["issue_number"] == 42 # P5F4-T2 @pytest.mark.anyio - async def test_release_issue_binds_structlog_context(self, tool_ctx, monkeypatch): + async def test_release_issue_binds_structlog_context(self, tool_ctx_kitchen_open, monkeypatch): """release_issue must bind structlog context vars via bind_contextvars.""" import structlog @@ -190,7 +224,7 @@ def fake_bind_contextvars(**kwargs): monkeypatch.setattr(structlog.contextvars, "bind_contextvars", fake_bind_contextvars) monkeypatch.setattr(structlog.contextvars, "clear_contextvars", lambda: None) - tool_ctx.github_client = None # triggers early return after bind + tool_ctx_kitchen_open.github_client = None # triggers early return after bind await release_issue(issue_url="https://github.com/owner/repo/issues/1") assert captured == { @@ -206,16 +240,7 @@ def test_prepare_issue_is_gated(self): assert "prepare_issue" in GATED_TOOLS @pytest.mark.anyio - async def test_prepare_issue_returns_gate_error_when_kitchen_closed(self, tool_ctx): - from autoskillit.pipeline.gate import DefaultGateState - - tool_ctx.gate = DefaultGateState(enabled=False) - result = json.loads(await prepare_issue("Test title", "Test body")) - assert result["success"] is False - assert result["subtype"] == "gate_error" - - @pytest.mark.anyio - async def test_prepare_issue_success_with_result_block(self, tool_ctx): + async def test_prepare_issue_success_with_result_block(self, tool_ctx_kitchen_open): """Happy path: executor returns success=True with a valid result block.""" result_text = ( f"{_PREPARE_RESULT_START}\n" @@ -237,7 +262,7 @@ async def test_prepare_issue_success_with_result_block(self, tool_ctx): retry_reason=RetryReason.NONE, stderr="", ) - tool_ctx.executor = mock_executor + tool_ctx_kitchen_open.executor = mock_executor result = json.loads(await prepare_issue("Test title", "Test body")) @@ -247,7 +272,9 @@ async def test_prepare_issue_success_with_result_block(self, tool_ctx): assert "error" not in result @pytest.mark.anyio - async def test_prepare_issue_success_empty_result_channel_b_drain_race(self, tool_ctx): + async def test_prepare_issue_success_empty_result_channel_b_drain_race( + self, tool_ctx_kitchen_open + ): """Channel B drain race: executor returns success=True but result is empty. Response must be success=False with diagnostics — THE KEY CONTRADICTION TEST. """ @@ -263,7 +290,7 @@ async def test_prepare_issue_success_empty_result_channel_b_drain_race(self, too retry_reason=RetryReason.NONE, stderr="", ) - tool_ctx.executor = mock_executor + tool_ctx_kitchen_open.executor = mock_executor result = json.loads(await prepare_issue("Test title", "Test body")) @@ -274,7 +301,7 @@ async def test_prepare_issue_success_empty_result_channel_b_drain_race(self, too assert result["status"] != "complete" # contradiction must be impossible @pytest.mark.anyio - async def test_prepare_issue_failure_with_diagnostics(self, tool_ctx): + async def test_prepare_issue_failure_with_diagnostics(self, tool_ctx_kitchen_open): """Executor failure: response must surface session_id, stderr, subtype, exit_code.""" mock_executor = AsyncMock() mock_executor.run.return_value = SkillResult( @@ -288,7 +315,7 @@ async def test_prepare_issue_failure_with_diagnostics(self, tool_ctx): retry_reason=RetryReason.NONE, stderr="Claude exited unexpectedly", ) - tool_ctx.executor = mock_executor + tool_ctx_kitchen_open.executor = mock_executor result = json.loads(await prepare_issue("Test title", "Test body")) @@ -299,7 +326,9 @@ async def test_prepare_issue_failure_with_diagnostics(self, tool_ctx): assert result["exit_code"] == 1 @pytest.mark.anyio - async def test_prepare_issue_passes_expected_output_patterns_to_executor(self, tool_ctx): + async def test_prepare_issue_passes_expected_output_patterns_to_executor( + self, tool_ctx_kitchen_open + ): """output_pattern_resolver is consulted and patterns are passed to executor.run().""" mock_executor = AsyncMock() mock_executor.run.return_value = SkillResult( @@ -313,8 +342,8 @@ async def test_prepare_issue_passes_expected_output_patterns_to_executor(self, t retry_reason=RetryReason.NONE, stderr="", ) - tool_ctx.executor = mock_executor - tool_ctx.output_pattern_resolver = lambda cmd: ["---prepare-issue-result---"] + tool_ctx_kitchen_open.executor = mock_executor + tool_ctx_kitchen_open.output_pattern_resolver = lambda cmd: ["---prepare-issue-result---"] await prepare_issue("Title", "Body") @@ -323,7 +352,7 @@ async def test_prepare_issue_passes_expected_output_patterns_to_executor(self, t @pytest.mark.anyio async def test_prepare_issue_response_success_field_never_overwritten_by_parsed_spread( - self, tool_ctx + self, tool_ctx_kitchen_open ): """When parsed block contains 'success': false, the outer success=True is preserved.""" result_text = ( @@ -343,7 +372,7 @@ async def test_prepare_issue_response_success_field_never_overwritten_by_parsed_ retry_reason=RetryReason.NONE, stderr="", ) - tool_ctx.executor = mock_executor + tool_ctx_kitchen_open.executor = mock_executor result = json.loads(await prepare_issue("Title", "Body")) @@ -359,7 +388,7 @@ async def test_prepare_issue_response_success_field_never_overwritten_by_parsed_ ], ) async def test_prepare_issue_contradictory_state_is_impossible( - self, tool_ctx, skill_success, skill_result_text + self, tool_ctx_kitchen_open, skill_success, skill_result_text ): """status=complete and success=False must never co-exist in any response.""" mock_executor = AsyncMock() @@ -374,7 +403,7 @@ async def test_prepare_issue_contradictory_state_is_impossible( retry_reason=RetryReason.NONE, stderr="", ) - tool_ctx.executor = mock_executor + tool_ctx_kitchen_open.executor = mock_executor result = json.loads(await prepare_issue("Title", "Body")) @@ -382,7 +411,7 @@ async def test_prepare_issue_contradictory_state_is_impossible( assert result["status"] == "failed" @pytest.mark.anyio - async def test_prepare_issue_no_result_block_includes_stderr(self, tool_ctx): + async def test_prepare_issue_no_result_block_includes_stderr(self, tool_ctx_kitchen_open): """success=True + non-empty result + no delimiters → stderr surfaced.""" mock_executor = AsyncMock() mock_executor.run.return_value = SkillResult( @@ -396,7 +425,7 @@ async def test_prepare_issue_no_result_block_includes_stderr(self, tool_ctx): needs_retry=False, retry_reason=RetryReason.NONE, ) - tool_ctx.executor = mock_executor + tool_ctx_kitchen_open.executor = mock_executor response = json.loads(await prepare_issue("Test Issue", "")) assert response["success"] is False assert response["error"] == "no result block found" @@ -405,7 +434,7 @@ async def test_prepare_issue_no_result_block_includes_stderr(self, tool_ctx): assert response["session_id"] == "abc-123" @pytest.mark.anyio - async def test_prepare_issue_empty_output_includes_stderr(self, tool_ctx): + async def test_prepare_issue_empty_output_includes_stderr(self, tool_ctx_kitchen_open): """success=True + empty result (drain race) → stderr surfaced.""" mock_executor = AsyncMock() mock_executor.run.return_value = SkillResult( @@ -419,7 +448,7 @@ async def test_prepare_issue_empty_output_includes_stderr(self, tool_ctx): needs_retry=False, retry_reason=RetryReason.NONE, ) - tool_ctx.executor = mock_executor + tool_ctx_kitchen_open.executor = mock_executor response = json.loads(await prepare_issue("Test Issue", "")) assert response["success"] is False assert "drain race" in response["error"] @@ -428,7 +457,9 @@ async def test_prepare_issue_empty_output_includes_stderr(self, tool_ctx): assert response["session_id"] == "abc-456" @pytest.mark.anyio - async def test_prepare_issue_session_failure_uses_subtype_not_block_sentinel(self, tool_ctx): + async def test_prepare_issue_session_failure_uses_subtype_not_block_sentinel( + self, tool_ctx_kitchen_open + ): """success=False must NOT call _parse_prepare_result. The error must reflect actual failure reason, not 'no result block found'. """ @@ -444,7 +475,7 @@ async def test_prepare_issue_session_failure_uses_subtype_not_block_sentinel(sel needs_retry=True, retry_reason=RetryReason.RESUME, ) - tool_ctx.executor = mock_executor + tool_ctx_kitchen_open.executor = mock_executor response = json.loads(await prepare_issue("Test Issue", "")) assert response["success"] is False assert response["error"] != "no result block found", ( @@ -460,16 +491,9 @@ def test_enrich_issues_is_gated(self): assert "enrich_issues" in GATED_TOOLS @pytest.mark.anyio - async def test_enrich_issues_returns_gate_error_when_kitchen_closed(self, tool_ctx): - from autoskillit.pipeline.gate import DefaultGateState - - tool_ctx.gate = DefaultGateState(enabled=False) - result = json.loads(await enrich_issues()) - assert result["success"] is False - assert result["subtype"] == "gate_error" - - @pytest.mark.anyio - async def test_enrich_issues_success_empty_result_includes_diagnostics(self, tool_ctx): + async def test_enrich_issues_success_empty_result_includes_diagnostics( + self, tool_ctx_kitchen_open + ): """Drain race for enrich_issues: success=True with empty result must yield failure.""" mock_executor = AsyncMock() mock_executor.run.return_value = SkillResult( @@ -483,7 +507,7 @@ async def test_enrich_issues_success_empty_result_includes_diagnostics(self, too retry_reason=RetryReason.NONE, stderr="", ) - tool_ctx.executor = mock_executor + tool_ctx_kitchen_open.executor = mock_executor result = json.loads(await enrich_issues()) @@ -491,7 +515,9 @@ async def test_enrich_issues_success_empty_result_includes_diagnostics(self, too assert result["session_id"] == "sid789" @pytest.mark.anyio - async def test_enrich_issues_failure_includes_session_id_and_stderr(self, tool_ctx): + async def test_enrich_issues_failure_includes_session_id_and_stderr( + self, tool_ctx_kitchen_open + ): """Executor failure: response includes session_id and stderr for diagnosis.""" mock_executor = AsyncMock() mock_executor.run.return_value = SkillResult( @@ -505,7 +531,7 @@ async def test_enrich_issues_failure_includes_session_id_and_stderr(self, tool_c retry_reason=RetryReason.NONE, stderr="Session timed out", ) - tool_ctx.executor = mock_executor + tool_ctx_kitchen_open.executor = mock_executor result = json.loads(await enrich_issues()) @@ -514,7 +540,9 @@ async def test_enrich_issues_failure_includes_session_id_and_stderr(self, tool_c assert result["stderr"] == "Session timed out" @pytest.mark.anyio - async def test_enrich_issues_passes_expected_output_patterns_to_executor(self, tool_ctx): + async def test_enrich_issues_passes_expected_output_patterns_to_executor( + self, tool_ctx_kitchen_open + ): """output_pattern_resolver is consulted and patterns are passed to executor.run().""" mock_executor = AsyncMock() mock_executor.run.return_value = SkillResult( @@ -528,8 +556,8 @@ async def test_enrich_issues_passes_expected_output_patterns_to_executor(self, t retry_reason=RetryReason.NONE, stderr="", ) - tool_ctx.executor = mock_executor - tool_ctx.output_pattern_resolver = lambda cmd: ["---enrich-issues-result---"] + tool_ctx_kitchen_open.executor = mock_executor + tool_ctx_kitchen_open.output_pattern_resolver = lambda cmd: ["---enrich-issues-result---"] await enrich_issues() @@ -537,7 +565,7 @@ async def test_enrich_issues_passes_expected_output_patterns_to_executor(self, t assert call_kwargs.get("expected_output_patterns") == ["---enrich-issues-result---"] @pytest.mark.anyio - async def test_enrich_issues_no_result_block_includes_stderr(self, tool_ctx): + async def test_enrich_issues_no_result_block_includes_stderr(self, tool_ctx_kitchen_open): """success=True + non-empty result + no delimiters → stderr surfaced.""" mock_executor = AsyncMock() mock_executor.run.return_value = SkillResult( @@ -551,7 +579,7 @@ async def test_enrich_issues_no_result_block_includes_stderr(self, tool_ctx): needs_retry=False, retry_reason=RetryReason.NONE, ) - tool_ctx.executor = mock_executor + tool_ctx_kitchen_open.executor = mock_executor response = json.loads(await enrich_issues()) assert response["success"] is False assert response["error"] == "no result block found" @@ -664,7 +692,7 @@ async def test_enrich_issues_no_result_block_includes_stderr(self, tool_ctx): @pytest.mark.anyio @pytest.mark.parametrize("tool_name,skill_result_kwargs", _HEADLESS_FAILURE_SCENARIOS) async def test_headless_tool_failure_paths_include_all_diagnostic_fields( - tool_name, skill_result_kwargs, tool_ctx + tool_name, skill_result_kwargs, tool_ctx_kitchen_open ): """Contract test: every failure path of every headless session tool must surface the full diagnostic set: success, error, session_id, @@ -673,7 +701,7 @@ async def test_headless_tool_failure_paths_include_all_diagnostic_fields( tool_fn = {"prepare_issue": prepare_issue, "enrich_issues": enrich_issues}[tool_name] mock_executor = AsyncMock() mock_executor.run.return_value = SkillResult(**skill_result_kwargs) - tool_ctx.executor = mock_executor + tool_ctx_kitchen_open.executor = mock_executor kwargs: dict = {} if tool_name == "prepare_issue": @@ -689,8 +717,8 @@ async def test_headless_tool_failure_paths_include_all_diagnostic_fields( class TestGetPrReviews: @pytest.mark.anyio - async def test_returns_structured_reviews(self, tool_ctx): - tool_ctx.runner.push( + async def test_returns_structured_reviews(self, tool_ctx_kitchen_open): + tool_ctx_kitchen_open.runner.push( _make_result( 0, json.dumps( @@ -711,8 +739,8 @@ async def test_returns_structured_reviews(self, tool_ctx): assert result["reviews"][0] == {"author": "reviewer1", "state": "APPROVED", "body": "LGTM"} @pytest.mark.anyio - async def test_empty_reviews(self, tool_ctx): - tool_ctx.runner.push(_make_result(0, json.dumps([]), "")) + async def test_empty_reviews(self, tool_ctx_kitchen_open): + tool_ctx_kitchen_open.runner.push(_make_result(0, json.dumps([]), "")) result = json.loads(await get_pr_reviews(42, ".", repo="owner/repo")) assert result["reviews"] == [] @@ -723,8 +751,8 @@ async def test_gh_command_failure_returns_error(self, tool_ctx): assert result["success"] is False @pytest.mark.anyio - async def test_without_repo_uses_pr_view(self, tool_ctx): - tool_ctx.runner.push( + async def test_without_repo_uses_pr_view(self, tool_ctx_kitchen_open): + tool_ctx_kitchen_open.runner.push( _make_result( 0, json.dumps( @@ -749,34 +777,47 @@ async def test_gate_closed_returns_gate_error(self, tool_ctx): class TestBulkCloseIssues: + @pytest.fixture(autouse=True) + def _mock_rate_limit_sleep(self, monkeypatch): + monkeypatch.setattr( + "autoskillit.server.tools.tools_pr_ops.asyncio.sleep", + AsyncMock(), + ) + @pytest.mark.anyio - async def test_closes_all_issues_successfully(self, tool_ctx): + async def test_closes_all_issues_successfully(self, tool_ctx_kitchen_open): for _ in range(3): - tool_ctx.runner.push(_make_result(0, "", "")) + tool_ctx_kitchen_open.runner.push(_make_result(0, "", "")) result = json.loads(await bulk_close_issues([1, 2, 3], "", ".")) assert result["closed"] == [1, 2, 3] assert result["failed"] == [] @pytest.mark.anyio - async def test_partial_failure_tracked_per_issue(self, tool_ctx): - tool_ctx.runner.push(_make_result(0, "", "")) - tool_ctx.runner.push(_make_result(1, "", "not found")) - tool_ctx.runner.push(_make_result(0, "", "")) + async def test_partial_failure_tracked_per_issue(self, tool_ctx_kitchen_open): + tool_ctx_kitchen_open.runner.push(_make_result(0, "", "")) + tool_ctx_kitchen_open.runner.push(_make_result(1, "", "not found")) + tool_ctx_kitchen_open.runner.push(_make_result(0, "", "")) result = json.loads(await bulk_close_issues([1, 2, 3], "", ".")) assert result["closed"] == [1, 3] assert result["failed"] == [2] @pytest.mark.anyio - async def test_empty_numbers_list(self, tool_ctx): + async def test_empty_numbers_list(self, tool_ctx_kitchen_open): result = json.loads(await bulk_close_issues([], "", ".")) assert result == {"closed": [], "failed": []} @pytest.mark.anyio - async def test_comment_flag_included_when_provided(self, tool_ctx): - tool_ctx.runner.push(_make_result(0, "", "")) - await bulk_close_issues([7], "Closed by pipeline.", ".") - call_cmd = tool_ctx.runner.call_args_list[-1][0] - assert "--comment" in call_cmd + async def test_comment_appended_to_body_when_provided(self, tool_ctx_kitchen_open): + tool_ctx_kitchen_open.runner.push(_make_result(0, "existing body", "")) + tool_ctx_kitchen_open.runner.push(_make_result(0, "", "")) + tool_ctx_kitchen_open.runner.push(_make_result(0, "", "")) + result = json.loads(await bulk_close_issues([7], "Closed by pipeline.", ".")) + all_cmds = [call[0] for call in tool_ctx_kitchen_open.runner.call_args_list] + edit_calls = [cmd for cmd in all_cmds if "edit" in cmd] + assert any("--body-file" in cmd for cmd in edit_calls), ( + "Expected gh issue edit --body-file call" + ) + assert result["closed"] == [7] @pytest.mark.anyio async def test_gate_closed_returns_gate_error(self, tool_ctx): diff --git a/tests/server/test_tools_integrations_release.py b/tests/server/test_tools_integrations_release.py index a5b885c1fd..ef27f3b20c 100644 --- a/tests/server/test_tools_integrations_release.py +++ b/tests/server/test_tools_integrations_release.py @@ -7,43 +7,49 @@ import pytest -from autoskillit.server.tools_issue_lifecycle import release_issue +from autoskillit.server.tools.tools_issue_lifecycle import release_issue + +pytestmark = [pytest.mark.layer("server"), pytest.mark.small] class TestReleaseIssueStagedLifecycle: @pytest.mark.anyio - async def test_release_issue_non_default_branch_applies_staged(self, tool_ctx, monkeypatch): - """Non-default target_branch removes in-progress and applies staged label.""" + async def test_release_issue_non_default_branch_applies_staged( + self, tool_ctx_kitchen_open, monkeypatch + ): + """Non-default target_branch: swap_labels atomically removes in-progress, adds staged.""" mock_client = AsyncMock() - mock_client.remove_label.return_value = {"success": True} mock_client.ensure_label.return_value = {"success": True, "created": True} - mock_client.add_labels.return_value = {"success": True, "labels": ["staged"]} - monkeypatch.setattr(tool_ctx, "github_client", mock_client) + mock_client.swap_labels.return_value = {"success": True, "labels": ["bug", "staged"]} + monkeypatch.setattr(tool_ctx_kitchen_open, "github_client", mock_client) result = json.loads( await release_issue( issue_url="https://github.com/owner/repo/issues/42", - target_branch="integration", + target_branch="develop", ) ) assert result["success"] is True assert result["staged"] is True assert result["staged_label"] == "staged" - mock_client.remove_label.assert_called_once_with("owner", "repo", 42, "in-progress") + mock_client.remove_label.assert_not_called() + mock_client.add_labels.assert_not_called() mock_client.ensure_label.assert_called_once_with( "owner", "repo", "staged", color="0075ca", - description="Implementation staged and waiting for promotion to main", + description="Issue resolved, PR staged for promotion", ) - mock_client.add_labels.assert_called_once_with("owner", "repo", 42, ["staged"]) + mock_client.swap_labels.assert_called_once() @pytest.mark.anyio - async def test_release_issue_default_branch_no_staged(self, tool_ctx, monkeypatch): - """release_issue with target_branch='main' only removes in-progress.""" + async def test_release_issue_default_branch_no_staged( + self, tool_ctx_kitchen_open, monkeypatch + ): + """release_issue with target_branch='main' uses swap_labels to remove in-progress.""" mock_client = AsyncMock() - mock_client.remove_label.return_value = {"success": True} - monkeypatch.setattr(tool_ctx, "github_client", mock_client) + mock_client.swap_labels.return_value = {"success": True, "labels": []} + monkeypatch.setattr(tool_ctx_kitchen_open, "github_client", mock_client) result = json.loads( await release_issue( issue_url="https://github.com/owner/repo/issues/42", @@ -52,16 +58,19 @@ async def test_release_issue_default_branch_no_staged(self, tool_ctx, monkeypatc ) assert result["success"] is True assert result.get("staged") is False - mock_client.remove_label.assert_called_once_with("owner", "repo", 42, "in-progress") + mock_client.remove_label.assert_not_called() mock_client.ensure_label.assert_not_called() mock_client.add_labels.assert_not_called() + mock_client.swap_labels.assert_called_once() @pytest.mark.anyio - async def test_release_issue_no_target_branch_no_staged(self, tool_ctx, monkeypatch): - """release_issue without target_branch: current behavior, no staged label.""" + async def test_release_issue_no_target_branch_no_staged( + self, tool_ctx_kitchen_open, monkeypatch + ): + """release_issue without target_branch: uses swap_labels, no staged label.""" mock_client = AsyncMock() - mock_client.remove_label.return_value = {"success": True} - monkeypatch.setattr(tool_ctx, "github_client", mock_client) + mock_client.swap_labels.return_value = {"success": True, "labels": []} + monkeypatch.setattr(tool_ctx_kitchen_open, "github_client", mock_client) result = json.loads( await release_issue( issue_url="https://github.com/owner/repo/issues/42", @@ -71,37 +80,35 @@ async def test_release_issue_no_target_branch_no_staged(self, tool_ctx, monkeypa assert result.get("staged") is False @pytest.mark.anyio - async def test_release_issue_staged_label_idempotent(self, tool_ctx, monkeypatch): + async def test_release_issue_staged_label_idempotent(self, tool_ctx_kitchen_open, monkeypatch): """ensure_label treats 422 (already exists) as success — applies it without error.""" mock_client = AsyncMock() - mock_client.remove_label.return_value = {"success": True} mock_client.ensure_label.return_value = {"success": True, "created": False} # 422 path - mock_client.add_labels.return_value = {"success": True, "labels": ["staged"]} - monkeypatch.setattr(tool_ctx, "github_client", mock_client) + mock_client.swap_labels.return_value = {"success": True, "labels": ["staged"]} + monkeypatch.setattr(tool_ctx_kitchen_open, "github_client", mock_client) result = json.loads( await release_issue( issue_url="https://github.com/owner/repo/issues/42", - target_branch="integration", + target_branch="develop", ) ) assert result["success"] is True assert result["staged"] is True @pytest.mark.anyio - async def test_release_issue_custom_staged_label(self, tool_ctx, monkeypatch): + async def test_release_issue_custom_staged_label(self, tool_ctx_kitchen_open, monkeypatch): """staged_label parameter overrides the default 'staged' label name.""" mock_client = AsyncMock() - mock_client.remove_label.return_value = {"success": True} mock_client.ensure_label.return_value = {"success": True, "created": True} - mock_client.add_labels.return_value = { + mock_client.swap_labels.return_value = { "success": True, "labels": ["awaiting-promotion"], } - monkeypatch.setattr(tool_ctx, "github_client", mock_client) + monkeypatch.setattr(tool_ctx_kitchen_open, "github_client", mock_client) result = json.loads( await release_issue( issue_url="https://github.com/owner/repo/issues/42", - target_branch="integration", + target_branch="develop", staged_label="awaiting-promotion", ) ) @@ -117,45 +124,49 @@ async def test_release_issue_custom_staged_label(self, tool_ctx, monkeypatch): ) @pytest.mark.anyio - async def test_release_issue_ensure_label_failure_returns_error(self, tool_ctx, monkeypatch): + async def test_release_issue_ensure_label_failure_returns_error( + self, tool_ctx_kitchen_open, monkeypatch + ): """When ensure_label fails, release_issue returns an error without applying the label.""" mock_client = AsyncMock() - mock_client.remove_label.return_value = {"success": True} mock_client.ensure_label.return_value = {"success": False, "error": "API error"} - monkeypatch.setattr(tool_ctx, "github_client", mock_client) + monkeypatch.setattr(tool_ctx_kitchen_open, "github_client", mock_client) result = json.loads( await release_issue( issue_url="https://github.com/owner/repo/issues/42", - target_branch="integration", + target_branch="develop", ) ) assert result["success"] is False assert "staged label" in result["error"] - mock_client.add_labels.assert_not_called() + mock_client.swap_labels.assert_not_called() @pytest.mark.anyio - async def test_release_issue_add_labels_failure_returns_error(self, tool_ctx, monkeypatch): - """When add_labels fails after ensure_label, release_issue returns an error.""" + async def test_release_issue_add_labels_failure_returns_error( + self, tool_ctx_kitchen_open, monkeypatch + ): + """When swap_labels fails after ensure_label, release_issue returns an error.""" mock_client = AsyncMock() - mock_client.remove_label.return_value = {"success": True} mock_client.ensure_label.return_value = {"success": True, "created": True} - mock_client.add_labels.return_value = {"success": False, "error": "Labels limit"} - monkeypatch.setattr(tool_ctx, "github_client", mock_client) + mock_client.swap_labels.return_value = {"success": False, "error": "Labels limit"} + monkeypatch.setattr(tool_ctx_kitchen_open, "github_client", mock_client) result = json.loads( await release_issue( issue_url="https://github.com/owner/repo/issues/42", - target_branch="integration", + target_branch="develop", ) ) assert result["success"] is False assert "staged label" in result["error"] @pytest.mark.anyio - async def test_release_issue_staged_null_when_not_staged(self, tool_ctx, monkeypatch): + async def test_release_issue_staged_null_when_not_staged( + self, tool_ctx_kitchen_open, monkeypatch + ): """staged_label field is None in response when staging was not applied.""" mock_client = AsyncMock() - mock_client.remove_label.return_value = {"success": True} - monkeypatch.setattr(tool_ctx, "github_client", mock_client) + mock_client.swap_labels.return_value = {"success": True, "labels": []} + monkeypatch.setattr(tool_ctx_kitchen_open, "github_client", mock_client) result = json.loads( await release_issue( issue_url="https://github.com/owner/repo/issues/42", @@ -168,20 +179,20 @@ async def test_release_issue_staged_null_when_not_staged(self, tool_ctx, monkeyp @pytest.mark.parametrize( "default_base_branch,promotion_target,target_branch,expected_staged", [ - # production scenario: default_base_branch overridden to integration for routing - ("integration", "main", "integration", True), - # integration explicitly set as promotion_target: landing there = done - ("integration", "integration", "integration", False), + # production scenario: default_base_branch overridden to develop for routing + ("develop", "main", "develop", True), + # develop explicitly set as promotion_target: landing there = done + ("develop", "develop", "develop", False), # non-default target against main promotion target - ("main", "main", "integration", True), + ("main", "main", "develop", True), # promotion_target overridden to something other than main ("main", "stable", "stable", False), - ("main", "stable", "integration", True), + ("main", "stable", "develop", True), ], ) async def test_release_issue_staging_uses_promotion_target( self, - tool_ctx, + tool_ctx_kitchen_open, monkeypatch, default_base_branch, promotion_target, @@ -190,21 +201,20 @@ async def test_release_issue_staging_uses_promotion_target( ): """Regression: staging comparison uses promotion_target, not default_base_branch. - When default_base_branch == target_branch (e.g. both "integration"), + When default_base_branch == target_branch (e.g. both "develop"), staged label must still be applied if promotion_target != target_branch. Conversely, no staged label when target_branch == promotion_target, regardless of default_base_branch. """ - tool_ctx.config.branching.default_base_branch = default_base_branch - tool_ctx.config.branching.promotion_target = promotion_target + tool_ctx_kitchen_open.config.branching.default_base_branch = default_base_branch + tool_ctx_kitchen_open.config.branching.promotion_target = promotion_target mock_client = AsyncMock() - mock_client.remove_label.return_value = {"success": True} mock_client.ensure_label.return_value = {"success": True, "created": False} - mock_client.add_labels.return_value = { + mock_client.swap_labels.return_value = { "success": True, - "labels": [tool_ctx.config.github.staged_label], + "labels": [tool_ctx_kitchen_open.config.github.staged_label], } - monkeypatch.setattr(tool_ctx, "github_client", mock_client) + monkeypatch.setattr(tool_ctx_kitchen_open, "github_client", mock_client) result = json.loads( await release_issue( issue_url="https://github.com/owner/repo/issues/42", @@ -226,3 +236,40 @@ async def test_release_issue_staging_uses_promotion_target( f"staged_label must be None when staged=False " f"(target_branch={target_branch!r}, promotion_target={promotion_target!r})" ) + + @pytest.mark.anyio + async def test_release_issue_staged_uses_swap_labels(self, tool_ctx_kitchen_open, monkeypatch): + """release_issue staged path: uses swap_labels; remove_label/add_labels not called.""" + mock_client = AsyncMock() + mock_client.ensure_label.return_value = {"success": True, "created": True} + mock_client.swap_labels.return_value = {"success": True, "labels": ["bug", "staged"]} + monkeypatch.setattr(tool_ctx_kitchen_open, "github_client", mock_client) + result = json.loads( + await release_issue( + issue_url="https://github.com/owner/repo/issues/42", + target_branch="develop", + ) + ) + assert result["success"] is True + assert result["staged"] is True + mock_client.swap_labels.assert_called_once() + mock_client.remove_label.assert_not_called() + mock_client.add_labels.assert_not_called() + + @pytest.mark.anyio + async def test_release_issue_no_stage_uses_swap_labels( + self, tool_ctx_kitchen_open, monkeypatch + ): + """release_issue without target_branch uses swap_labels; remove_label not called.""" + mock_client = AsyncMock() + mock_client.swap_labels.return_value = {"success": True, "labels": ["bug"]} + monkeypatch.setattr(tool_ctx_kitchen_open, "github_client", mock_client) + result = json.loads( + await release_issue( + issue_url="https://github.com/owner/repo/issues/42", + ) + ) + assert result["success"] is True + assert result["staged"] is False + mock_client.swap_labels.assert_called_once() + mock_client.remove_label.assert_not_called() diff --git a/tests/server/test_tools_issue_lifecycle.py b/tests/server/test_tools_issue_lifecycle.py index 4849c6e908..b9ac93271d 100644 --- a/tests/server/test_tools_issue_lifecycle.py +++ b/tests/server/test_tools_issue_lifecycle.py @@ -9,7 +9,7 @@ from autoskillit.core import RetryReason, SkillResult from autoskillit.pipeline.gate import DefaultGateState -from autoskillit.server.tools_issue_lifecycle import ( +from autoskillit.server.tools.tools_issue_lifecycle import ( _build_enrich_skill_command, _build_headless_error_response, _build_prepare_skill_command, @@ -24,6 +24,8 @@ release_issue, ) +pytestmark = [pytest.mark.layer("server"), pytest.mark.small] + def _make_skill_result( success: bool = True, @@ -96,9 +98,9 @@ def test_without_success_key_no_success_is_noop() -> None: def test_build_prepare_skill_command_basic() -> None: - """No labels, no dry_run → '/autoskillit:prepare-issue\\n\\nTitle: T\\n\\nBody:\\nB'.""" + """No labels, no dry_run → '/prepare-issue\\n\\nTitle: T\\n\\nBody:\\nB'.""" cmd = _build_prepare_skill_command("T", "B", "", None, False, False) - assert cmd == "/autoskillit:prepare-issue\n\nTitle: T\n\nBody:\nB" + assert cmd == "/prepare-issue\n\nTitle: T\n\nBody:\nB" def test_build_prepare_skill_command_with_flags() -> None: @@ -135,7 +137,7 @@ def test_parse_prepare_result_invalid_json() -> None: def test_build_enrich_skill_command_with_all_args() -> None: """All args provided → assembled command includes all flags.""" cmd = _build_enrich_skill_command(42, 3, True, "owner/repo") - assert "/autoskillit:enrich-issues" in cmd + assert "/enrich-issues" in cmd assert "--issue 42" in cmd assert "--batch 3" in cmd assert "--dry-run" in cmd @@ -171,9 +173,9 @@ async def test_prepare_issue_gate_closed(tool_ctx) -> None: @pytest.mark.anyio -async def test_prepare_issue_no_executor(tool_ctx) -> None: +async def test_prepare_issue_no_executor(tool_ctx_kitchen_open) -> None: """executor=None → {"success": False, "error": "Executor not configured"}.""" - tool_ctx.executor = None + tool_ctx_kitchen_open.executor = None result = json.loads(await prepare_issue("Title", "Body")) assert result["success"] is False assert "Executor not configured" in result["error"] @@ -195,11 +197,11 @@ async def test_prepare_issue_session_failure(tool_ctx) -> None: @pytest.mark.anyio -async def test_prepare_issue_empty_output(tool_ctx) -> None: +async def test_prepare_issue_empty_output(tool_ctx_kitchen_open) -> None: """success=True but result="" → drain-race error.""" skill_result = _make_skill_result(success=True, result="") - tool_ctx.executor = AsyncMock() - tool_ctx.executor.run = AsyncMock(return_value=skill_result) + tool_ctx_kitchen_open.executor = AsyncMock() + tool_ctx_kitchen_open.executor.run = AsyncMock(return_value=skill_result) result = json.loads(await prepare_issue("Title", "Body")) assert result["success"] is False @@ -207,11 +209,11 @@ async def test_prepare_issue_empty_output(tool_ctx) -> None: @pytest.mark.anyio -async def test_prepare_issue_block_parse_error(tool_ctx) -> None: +async def test_prepare_issue_block_parse_error(tool_ctx_kitchen_open) -> None: """success=True, output present, but no delimiters → 'no result block found' error.""" skill_result = _make_skill_result(success=True, result="some output without delimiters") - tool_ctx.executor = AsyncMock() - tool_ctx.executor.run = AsyncMock(return_value=skill_result) + tool_ctx_kitchen_open.executor = AsyncMock() + tool_ctx_kitchen_open.executor.run = AsyncMock(return_value=skill_result) result = json.loads(await prepare_issue("Title", "Body")) assert result["success"] is False @@ -219,14 +221,14 @@ async def test_prepare_issue_block_parse_error(tool_ctx) -> None: @pytest.mark.anyio -async def test_prepare_issue_success(tool_ctx) -> None: +async def test_prepare_issue_success(tool_ctx_kitchen_open) -> None: """Complete success path → success=True, block fields merged without 'success' key conflict.""" block_data = {"issue_url": "https://github.com/o/r/issues/1", "issue_number": 1} payload = json.dumps(block_data) output = f"---prepare-issue-result---\n{payload}\n---/prepare-issue-result---\n" skill_result = _make_skill_result(success=True, result=output) - tool_ctx.executor = AsyncMock() - tool_ctx.executor.run = AsyncMock(return_value=skill_result) + tool_ctx_kitchen_open.executor = AsyncMock() + tool_ctx_kitchen_open.executor.run = AsyncMock(return_value=skill_result) result = json.loads(await prepare_issue("Title", "Body")) assert result["success"] is True @@ -244,14 +246,14 @@ async def test_enrich_issues_gate_closed(tool_ctx) -> None: @pytest.mark.anyio -async def test_enrich_issues_success(tool_ctx) -> None: +async def test_enrich_issues_success(tool_ctx_kitchen_open) -> None: """Successful enrich → success=True, enriched list in response.""" block_data = {"enriched": [42], "skipped_already_enriched": []} payload = json.dumps(block_data) output = f"---enrich-issues-result---\n{payload}\n---/enrich-issues-result---\n" skill_result = _make_skill_result(success=True, result=output) - tool_ctx.executor = AsyncMock() - tool_ctx.executor.run = AsyncMock(return_value=skill_result) + tool_ctx_kitchen_open.executor = AsyncMock() + tool_ctx_kitchen_open.executor.run = AsyncMock(return_value=skill_result) result = json.loads(await enrich_issues()) assert result["success"] is True @@ -268,9 +270,9 @@ async def test_claim_issue_gate_closed(tool_ctx) -> None: @pytest.mark.anyio -async def test_claim_issue_no_client(tool_ctx) -> None: +async def test_claim_issue_no_client(tool_ctx_kitchen_open) -> None: """github_client=None → error response.""" - tool_ctx.github_client = None + tool_ctx_kitchen_open.github_client = None result = json.loads(await claim_issue("https://github.com/owner/repo/issues/42")) assert result["success"] is False assert "token" in result["error"].lower() or "github" in result["error"].lower() @@ -278,15 +280,15 @@ async def test_claim_issue_no_client(tool_ctx) -> None: @pytest.mark.anyio async def test_claim_issue_already_claimed_returns_not_claimed( - tool_ctx, + tool_ctx_kitchen_open, ) -> None: """Label already present, allow_reentry=False → claimed=False.""" issue_data = { "success": True, "labels": [{"name": "autoskillit:in-progress"}], } - tool_ctx.github_client = AsyncMock() - tool_ctx.github_client.fetch_issue = AsyncMock(return_value=issue_data) + tool_ctx_kitchen_open.github_client = AsyncMock() + tool_ctx_kitchen_open.github_client.fetch_issue = AsyncMock(return_value=issue_data) result = json.loads( await claim_issue( @@ -300,14 +302,14 @@ async def test_claim_issue_already_claimed_returns_not_claimed( @pytest.mark.anyio -async def test_claim_issue_reentry_allowed(tool_ctx) -> None: +async def test_claim_issue_reentry_allowed(tool_ctx_kitchen_open) -> None: """Label already present, allow_reentry=True → claimed=True, reentry=True.""" issue_data = { "success": True, "labels": [{"name": "autoskillit:in-progress"}], } - tool_ctx.github_client = AsyncMock() - tool_ctx.github_client.fetch_issue = AsyncMock(return_value=issue_data) + tool_ctx_kitchen_open.github_client = AsyncMock() + tool_ctx_kitchen_open.github_client.fetch_issue = AsyncMock(return_value=issue_data) result = json.loads( await claim_issue( @@ -322,13 +324,13 @@ async def test_claim_issue_reentry_allowed(tool_ctx) -> None: @pytest.mark.anyio -async def test_claim_issue_success(tool_ctx) -> None: +async def test_claim_issue_success(tool_ctx_kitchen_open) -> None: """Label not present → applies label, claimed=True.""" issue_data = {"success": True, "labels": []} - tool_ctx.github_client = AsyncMock() - tool_ctx.github_client.fetch_issue = AsyncMock(return_value=issue_data) - tool_ctx.github_client.ensure_label = AsyncMock(return_value={"success": True}) - tool_ctx.github_client.add_labels = AsyncMock(return_value={"success": True}) + tool_ctx_kitchen_open.github_client = AsyncMock() + tool_ctx_kitchen_open.github_client.fetch_issue = AsyncMock(return_value=issue_data) + tool_ctx_kitchen_open.github_client.ensure_label = AsyncMock(return_value={"success": True}) + tool_ctx_kitchen_open.github_client.swap_labels = AsyncMock(return_value={"success": True}) result = json.loads( await claim_issue( @@ -352,12 +354,12 @@ async def test_release_issue_gate_closed(tool_ctx) -> None: @pytest.mark.anyio async def test_release_issue_no_staging_when_same_branch( - tool_ctx, + tool_ctx_kitchen_open, ) -> None: """target_branch == promotion_target → staged=False.""" - tool_ctx.github_client = AsyncMock() - tool_ctx.github_client.remove_label = AsyncMock(return_value={"success": True}) - promotion_target = tool_ctx.config.branching.promotion_target + tool_ctx_kitchen_open.github_client = AsyncMock() + tool_ctx_kitchen_open.github_client.remove_label = AsyncMock(return_value={"success": True}) + promotion_target = tool_ctx_kitchen_open.config.branching.promotion_target result = json.loads( await release_issue( @@ -372,13 +374,13 @@ async def test_release_issue_no_staging_when_same_branch( @pytest.mark.anyio async def test_release_issue_stages_when_different_branch( - tool_ctx, + tool_ctx_kitchen_open, ) -> None: """target_branch != promotion_target → staged=True, staged_label applied.""" - tool_ctx.github_client = AsyncMock() - tool_ctx.github_client.remove_label = AsyncMock(return_value={"success": True}) - tool_ctx.github_client.ensure_label = AsyncMock(return_value={"success": True}) - tool_ctx.github_client.add_labels = AsyncMock(return_value={"success": True}) + tool_ctx_kitchen_open.github_client = AsyncMock() + tool_ctx_kitchen_open.github_client.remove_label = AsyncMock(return_value={"success": True}) + tool_ctx_kitchen_open.github_client.ensure_label = AsyncMock(return_value={"success": True}) + tool_ctx_kitchen_open.github_client.swap_labels = AsyncMock(return_value={"success": True}) result = json.loads( await release_issue( diff --git a/tests/server/test_tools_kitchen.py b/tests/server/test_tools_kitchen.py deleted file mode 100644 index 7e83ce32ea..0000000000 --- a/tests/server/test_tools_kitchen.py +++ /dev/null @@ -1,1156 +0,0 @@ -"""Tests for server/tools_kitchen.py: open_kitchen and close_kitchen gate management.""" - -from __future__ import annotations - -import asyncio -import dataclasses -import json -from unittest.mock import AsyncMock, MagicMock, patch - -import pytest - -from autoskillit.hooks._fmt_primitives import _HOOK_CONFIG_PATH_COMPONENTS - - -def _make_mock_ctx(): - """Return a minimal mock ToolContext with a gate.""" - gate = MagicMock() - gate.enabled = False - ctx = MagicMock() - ctx.gate = gate - ctx.config.subsets.disabled = [] # REQ-VIS-008: no subsets disabled by default - return ctx - - -# T2a -@pytest.mark.anyio -async def test_open_kitchen_enables_gate(tmp_path, monkeypatch): - """After _open_kitchen_handler(), gate is enabled.""" - monkeypatch.chdir(tmp_path) - mock_ctx = _make_mock_ctx() - - with patch("autoskillit.server._get_ctx", return_value=mock_ctx): - with patch("autoskillit.server.logger"): - with patch("autoskillit.server.tools_kitchen._prime_quota_cache", new=AsyncMock()): - with patch("autoskillit.server.tools_kitchen._write_hook_config"): - from autoskillit.server.tools_kitchen import _open_kitchen_handler - - await _open_kitchen_handler() - - mock_ctx.gate.enable.assert_called_once() - - -# T2b -def test_close_kitchen_disables_gate(tmp_path, monkeypatch): - """After _close_kitchen_handler(), gate is disabled.""" - monkeypatch.chdir(tmp_path) - mock_ctx = _make_mock_ctx() - - with patch("autoskillit.server._get_ctx", return_value=mock_ctx): - with patch("autoskillit.server.logger"): - from autoskillit.server.tools_kitchen import _close_kitchen_handler - - _close_kitchen_handler() - - mock_ctx.gate.disable.assert_called_once() - - -def test_hook_config_path_is_inside_temp_dir(tmp_path): - """_hook_config_path() must resolve to a path inside the project temp directory. - - This invariant ensures .hook_config.json receives automatic gitignore coverage - from temp/.gitignore ('*') and can never produce a gitignore gap regardless of - whatever entries are present in .gitignore or _AUTOSKILLIT_GITIGNORE_ENTRIES. - """ - from autoskillit.core.io import resolve_temp_dir - from autoskillit.server.helpers import _hook_config_path - - hook_path = _hook_config_path(tmp_path) - temp_dir = resolve_temp_dir(tmp_path, None) - - assert hook_path.is_relative_to(temp_dir), ( - f"_hook_config_path() returned {hook_path!r} which is NOT inside " - f"temp dir {temp_dir!r}. Session-bridge files must live in temp/ " - f"to receive automatic gitignore coverage via temp/.gitignore." - ) - - -# T2c -def test_close_kitchen_no_file_no_error(tmp_path, monkeypatch): - """_close_kitchen_handler() doesn't raise when no gate file exists.""" - monkeypatch.chdir(tmp_path) - mock_ctx = _make_mock_ctx() - - with patch("autoskillit.server._get_ctx", return_value=mock_ctx): - with patch("autoskillit.server.logger"): - from autoskillit.server.tools_kitchen import _close_kitchen_handler - - _close_kitchen_handler() # Should not raise - - # Gate file was never created — confirm it still does not exist - assert not tmp_path.joinpath(*_HOOK_CONFIG_PATH_COMPONENTS).exists() - - -# T-CACHE-1 -@pytest.mark.anyio -async def test_open_kitchen_primes_quota_cache(tmp_path, monkeypatch): - """open_kitchen must call _prime_quota_cache before any run_skill hook fires.""" - monkeypatch.chdir(tmp_path) - mock_ctx = _make_mock_ctx() - prime_mock = AsyncMock() - - with patch("autoskillit.server._get_ctx", return_value=mock_ctx): - with patch("autoskillit.server.logger"): - with patch("autoskillit.server.tools_kitchen._prime_quota_cache", prime_mock): - with patch("autoskillit.server.tools_kitchen._write_hook_config"): - from autoskillit.server.tools_kitchen import _open_kitchen_handler - - await _open_kitchen_handler() - - prime_mock.assert_called_once() - - -# T-CACHE-2 -@pytest.mark.anyio -async def test_open_kitchen_writes_hook_config_json(tmp_path, monkeypatch): - """open_kitchen must write .autoskillit/.hook_config.json with user quota_guard values.""" - monkeypatch.chdir(tmp_path) - mock_ctx = _make_mock_ctx() - mock_ctx.config.quota_guard.short_window_threshold = 85.0 - mock_ctx.config.quota_guard.long_window_threshold = 98.0 - mock_ctx.config.quota_guard.long_window_patterns = ["weekly", "sonnet", "opus"] - mock_ctx.config.quota_guard.cache_max_age = 300 - mock_ctx.config.quota_guard.cache_path = "/custom/path.json" - mock_ctx.config.quota_guard.buffer_seconds = 60 - - # _write_hook_config uses 'from autoskillit.server import _get_ctx' at call time. - # Patching autoskillit.server._get_ctx correctly intercepts that deferred import; - # assert call_count >= 2 confirms the patch covered both _open_kitchen_handler and - # _write_hook_config (not just one of them). - with patch("autoskillit.server._get_ctx", return_value=mock_ctx) as mock_get_ctx: - with patch("autoskillit.server.logger"): - with patch("autoskillit.server.tools_kitchen._prime_quota_cache", new=AsyncMock()): - from autoskillit.server.tools_kitchen import _open_kitchen_handler - - await _open_kitchen_handler() - - assert mock_get_ctx.call_count >= 2, ( - "_get_ctx must be called in both _open_kitchen_handler and _write_hook_config; " - "if call_count < 2 the patch did not cover _write_hook_config's deferred import" - ) - hook_cfg = tmp_path.joinpath(*_HOOK_CONFIG_PATH_COMPONENTS) - assert hook_cfg.exists(), "Hook config file must be written by open_kitchen" - data = json.loads(hook_cfg.read_text()) - assert data["quota_guard"]["cache_max_age"] == 300 - assert data["quota_guard"]["cache_path"] == "/custom/path.json" - assert data["quota_guard"]["buffer_seconds"] == 60 - # threshold fields are pre-computed into should_block in the cache — not written to hook_config - assert "threshold" not in data["quota_guard"] - assert "short_window_threshold" not in data["quota_guard"] - assert "long_window_threshold" not in data["quota_guard"] - assert "long_window_patterns" not in data["quota_guard"] - # Confirm kitchen_id rename: hook config must contain 'kitchen_id' (not 'pipeline_id') - assert "kitchen_id" in data, ( - "hook config must contain 'kitchen_id' after rename from 'pipeline_id'" - ) - assert isinstance(data["kitchen_id"], str) and data["kitchen_id"], ( - "kitchen_id must be a non-empty string (UUID set by _open_kitchen_handler)" - ) - - -# T-CACHE-3 -@pytest.mark.anyio -async def test_close_kitchen_removes_hook_config_json(tmp_path, monkeypatch): - """close_kitchen must remove .autoskillit/.hook_config.json to prevent stale config.""" - monkeypatch.chdir(tmp_path) - mock_ctx = _make_mock_ctx() - mock_ctx.config.quota_guard.short_window_threshold = 85.0 - mock_ctx.config.quota_guard.long_window_threshold = 98.0 - mock_ctx.config.quota_guard.long_window_patterns = ["weekly", "sonnet", "opus"] - mock_ctx.config.quota_guard.cache_max_age = 300 - mock_ctx.config.quota_guard.cache_path = "~/.claude/quota_cache.json" - mock_ctx.config.quota_guard.buffer_seconds = 60 - - with patch("autoskillit.server._get_ctx", return_value=mock_ctx): - with patch("autoskillit.server.logger"): - with patch("autoskillit.server.tools_kitchen._prime_quota_cache", new=AsyncMock()): - from autoskillit.server.tools_kitchen import ( - _close_kitchen_handler, - _open_kitchen_handler, - ) - - await _open_kitchen_handler() - _close_kitchen_handler() - - hook_cfg = tmp_path.joinpath(*_HOOK_CONFIG_PATH_COMPONENTS) - assert not hook_cfg.exists(), "Hook config must be removed by close_kitchen" - - -# T-CACHE-4 -def test_open_kitchen_handler_is_async(): - """_open_kitchen_handler must be an async def so it can await _prime_quota_cache.""" - import inspect - - from autoskillit.server.tools_kitchen import _open_kitchen_handler - - assert inspect.iscoroutinefunction(_open_kitchen_handler), ( - "_open_kitchen_handler must be async" - ) - - -# T-VISIBILITY-1: open_kitchen tool calls ctx.enable_components -@pytest.mark.anyio -async def test_open_kitchen_tool_calls_enable_components(tmp_path, monkeypatch): - """open_kitchen tool must call ctx.enable_components(tags={'kitchen'}).""" - monkeypatch.chdir(tmp_path) - mock_ctx = _make_mock_ctx() - mock_ctx.enable_components = AsyncMock() - - with patch("autoskillit.server._get_ctx", return_value=mock_ctx): - with patch("autoskillit.server.logger"): - with patch("autoskillit.server.tools_kitchen._prime_quota_cache", new=AsyncMock()): - with patch("autoskillit.server.tools_kitchen._write_hook_config"): - from autoskillit.server.tools_kitchen import open_kitchen - - await open_kitchen(ctx=mock_ctx) - - mock_ctx.enable_components.assert_called_once_with(tags={"kitchen"}) - - -# T-VISIBILITY-2: close_kitchen tool calls ctx.reset_visibility -@pytest.mark.anyio -async def test_close_kitchen_tool_calls_reset_visibility(tmp_path, monkeypatch): - """close_kitchen tool must call ctx.reset_visibility().""" - monkeypatch.chdir(tmp_path) - mock_ctx = _make_mock_ctx() - mock_ctx.reset_visibility = AsyncMock() - - with patch("autoskillit.server._get_ctx", return_value=mock_ctx): - with patch("autoskillit.server.logger"): - from autoskillit.server.tools_kitchen import close_kitchen - - await close_kitchen(ctx=mock_ctx) - - mock_ctx.reset_visibility.assert_called_once() - - -@pytest.mark.anyio -async def test_open_kitchen_does_not_write_gate_file(tmp_path, monkeypatch): - """_open_kitchen_handler must never write a gate file.""" - monkeypatch.chdir(tmp_path) - mock_ctx = _make_mock_ctx() - mock_ctx.config.quota_guard.short_window_threshold = 85.0 - mock_ctx.config.quota_guard.long_window_threshold = 98.0 - mock_ctx.config.quota_guard.long_window_patterns = ["weekly", "sonnet", "opus"] - mock_ctx.config.quota_guard.cache_max_age = 300 - mock_ctx.config.quota_guard.cache_path = "~/.claude/quota_cache.json" - mock_ctx.config.quota_guard.buffer_seconds = 60 - with patch("autoskillit.server._get_ctx", return_value=mock_ctx): - with patch("autoskillit.server.logger"): - with patch("autoskillit.server.tools_kitchen._prime_quota_cache", new=AsyncMock()): - from autoskillit.server.tools_kitchen import _open_kitchen_handler - - await _open_kitchen_handler() - gate_file = tmp_path / ".autoskillit" / "temp" / ".kitchen_gate" - assert not gate_file.exists() - - -def test_close_kitchen_does_not_produce_gate_file(tmp_path, monkeypatch): - """_close_kitchen_handler must not interact with any gate file path.""" - monkeypatch.chdir(tmp_path) - mock_ctx = _make_mock_ctx() - with patch("autoskillit.server._get_ctx", return_value=mock_ctx): - with patch("autoskillit.server.logger"): - from autoskillit.server.tools_kitchen import _close_kitchen_handler - - _close_kitchen_handler() - gate_file = tmp_path / ".autoskillit" / "temp" / ".kitchen_gate" - assert not gate_file.exists() - - -@pytest.mark.anyio -async def test_open_kitchen_includes_categorized_tool_listing(tmp_path, monkeypatch): - """open_kitchen response contains static categorized tool groups from _DISPLAY_CATEGORIES.""" - from autoskillit.server.tools_kitchen import _DISPLAY_CATEGORIES, open_kitchen - - monkeypatch.chdir(tmp_path) - mock_ctx = _make_mock_ctx() - mock_ctx.enable_components = AsyncMock() - - with patch("autoskillit.server._get_ctx", return_value=mock_ctx): - with patch("autoskillit.server.logger"): - with patch("autoskillit.server.tools_kitchen._prime_quota_cache", new=AsyncMock()): - with patch("autoskillit.server.tools_kitchen._write_hook_config"): - result_str = await open_kitchen(ctx=mock_ctx) - - parsed = json.loads(result_str) - content = parsed["content"] - seen: set[str] = set() - for category_name, tools in _DISPLAY_CATEGORIES: - assert category_name in content, ( - f"Category '{category_name}' missing from open_kitchen response" - ) - for tool_name in tools: - if tool_name not in seen: - assert tool_name in content, ( - f"Tool '{tool_name}' missing from open_kitchen response" - ) - seen.add(tool_name) - - -# --------------------------------------------------------------------------- -# open_kitchen with recipe name -# --------------------------------------------------------------------------- - - -@pytest.mark.anyio -async def test_open_kitchen_with_recipe_returns_combined_response(tmp_path, monkeypatch): - """open_kitchen(name='x') opens kitchen AND loads the recipe in one call.""" - monkeypatch.chdir(tmp_path) - recipes_dir = tmp_path / ".autoskillit" / "recipes" - recipes_dir.mkdir(parents=True) - yaml_content = ( - "name: test-recipe\ndescription: test\nsteps:\n do:\n tool: run_cmd\n" - " with:\n cmd: echo hi\n on_success: done\n on_failure: done\n" - " done:\n action: stop\n message: Done\n" - ) - (recipes_dir / "test-recipe.yaml").write_text(yaml_content) - mock_ctx = _make_mock_ctx() - mock_ctx.enable_components = AsyncMock() - mock_ctx.recipes = MagicMock() - mock_ctx.recipes.load_and_validate.return_value = { - "content": yaml_content, - "valid": True, - "suggestions": [], - "diagram": None, - } - mock_ctx.recipes.find.return_value = None - mock_ctx.config.migration.suppressed = [] - - with patch("autoskillit.server._get_ctx", return_value=mock_ctx): - with patch("autoskillit.server.logger"): - with patch("autoskillit.server.tools_kitchen._prime_quota_cache", new=AsyncMock()): - with patch("autoskillit.server.tools_kitchen._write_hook_config"): - from autoskillit.server.tools_kitchen import open_kitchen - - result_str = await open_kitchen(name="test-recipe", ctx=mock_ctx) - - result = json.loads(result_str) - assert result["success"] is True - assert result["kitchen"] == "open" - assert "version" in result - assert "content" in result - assert "test-recipe" in result["content"] - mock_ctx.gate.enable.assert_called_once() - mock_ctx.enable_components.assert_called_once_with(tags={"kitchen"}) - - -@pytest.mark.anyio -async def test_open_kitchen_with_recipe_not_found(tmp_path, monkeypatch): - """open_kitchen(name='nonexistent') returns error with kitchen still open.""" - monkeypatch.chdir(tmp_path) - mock_ctx = _make_mock_ctx() - mock_ctx.enable_components = AsyncMock() - mock_ctx.recipes = MagicMock() - mock_ctx.recipes.load_and_validate.return_value = { - "error": "No recipe named 'nonexistent' found", - } - mock_ctx.recipes.find.return_value = None - mock_ctx.config.migration.suppressed = [] - - with patch("autoskillit.server._get_ctx", return_value=mock_ctx): - with patch("autoskillit.server.logger"): - with patch("autoskillit.server.tools_kitchen._prime_quota_cache", new=AsyncMock()): - with patch("autoskillit.server.tools_kitchen._write_hook_config"): - from autoskillit.server.tools_kitchen import open_kitchen - - result_str = await open_kitchen(name="nonexistent", ctx=mock_ctx) - - result = json.loads(result_str) - assert "error" in result - assert "nonexistent" in result["error"] - assert result["kitchen"] == "open" - # Kitchen should still be opened even if recipe fails - mock_ctx.gate.enable.assert_called_once() - - -@pytest.mark.anyio -async def test_open_kitchen_without_recipe_returns_json_envelope(tmp_path, monkeypatch): - """open_kitchen() without name returns JSON envelope with success=True.""" - monkeypatch.chdir(tmp_path) - mock_ctx = _make_mock_ctx() - mock_ctx.enable_components = AsyncMock() - - with patch("autoskillit.server._get_ctx", return_value=mock_ctx): - with patch("autoskillit.server.logger"): - with patch("autoskillit.server.tools_kitchen._prime_quota_cache", new=AsyncMock()): - with patch("autoskillit.server.tools_kitchen._write_hook_config"): - from autoskillit.server.tools_kitchen import open_kitchen - - result = await open_kitchen(ctx=mock_ctx) - - assert isinstance(result, str) - parsed = json.loads(result) - assert parsed["success"] is True - assert parsed["kitchen"] == "open" - assert "Kitchen is open" in parsed["content"] - assert parsed["ingredients_table"] is None - - -# --------------------------------------------------------------------------- -# Headless gate enforcement for kitchen tools -# --------------------------------------------------------------------------- - - -@pytest.mark.anyio -async def test_open_kitchen_denied_by_gate_when_headless(tmp_path, monkeypatch): - monkeypatch.setenv("AUTOSKILLIT_HEADLESS", "1") - monkeypatch.chdir(tmp_path) - from autoskillit.server.tools_kitchen import open_kitchen - - result = json.loads(await open_kitchen()) - assert result["success"] is False - assert result["kitchen"] == "failed" - assert "user_visible_message" in result - assert len(result["user_visible_message"]) > 0 - assert result["stage"] == "headless_guard" - - -@pytest.mark.anyio -async def test_close_kitchen_denied_when_headless(tmp_path, monkeypatch): - monkeypatch.setenv("AUTOSKILLIT_HEADLESS", "1") - monkeypatch.chdir(tmp_path) - from autoskillit.server.tools_kitchen import close_kitchen - - result = json.loads(await close_kitchen()) - assert result["success"] is False - assert result["subtype"] == "headless_error" - - -# T-VIS-003 -@pytest.mark.anyio -async def test_open_kitchen_redisables_subsets(tmp_path, monkeypatch): - """open_kitchen must call ctx.disable_components for each disabled subset.""" - from autoskillit.config.settings import AutomationConfig, SubsetsConfig - - monkeypatch.chdir(tmp_path) - mock_ctx = _make_mock_ctx() - mock_ctx.enable_components = AsyncMock() - mock_ctx.disable_components = AsyncMock() - mock_ctx.config = AutomationConfig(subsets=SubsetsConfig(disabled=["github", "ci"])) - - with patch("autoskillit.server._get_ctx", return_value=mock_ctx): - with patch("autoskillit.server.logger"): - with patch("autoskillit.server.tools_kitchen._prime_quota_cache", new=AsyncMock()): - with patch("autoskillit.server.tools_kitchen._write_hook_config"): - from autoskillit.server.tools_kitchen import open_kitchen - - await open_kitchen(ctx=mock_ctx) - - disable_calls = mock_ctx.disable_components.call_args_list - disabled_tags = [ - c.kwargs.get("tags") or (c.args[0] if c.args else None) for c in disable_calls - ] - assert {"github"} in disabled_tags - assert {"ci"} in disabled_tags - - -# T-VIS-004 -@pytest.mark.anyio -async def test_open_kitchen_redisable_order(tmp_path, monkeypatch): - """ctx.disable_components must be called after ctx.enable_components (order matters).""" - from autoskillit.config.settings import AutomationConfig, SubsetsConfig - - monkeypatch.chdir(tmp_path) - call_order = [] - mock_ctx = _make_mock_ctx() - - async def record_enable(**kwargs): - call_order.append(("enable", kwargs)) - - async def record_disable(**kwargs): - call_order.append(("disable", kwargs)) - - mock_ctx.enable_components = record_enable - mock_ctx.disable_components = record_disable - mock_ctx.config = AutomationConfig(subsets=SubsetsConfig(disabled=["github"])) - - with patch("autoskillit.server._get_ctx", return_value=mock_ctx): - with patch("autoskillit.server.logger"): - with patch("autoskillit.server.tools_kitchen._prime_quota_cache", new=AsyncMock()): - with patch("autoskillit.server.tools_kitchen._write_hook_config"): - from autoskillit.server.tools_kitchen import open_kitchen - - await open_kitchen(ctx=mock_ctx) - - enable_idx = next(i for i, (op, _) in enumerate(call_order) if op == "enable") - disable_idx = next(i for i, (op, _) in enumerate(call_order) if op == "disable") - assert enable_idx < disable_idx, "disable_components must be called after enable_components" - - -# T-VIS-005 -@pytest.mark.anyio -async def test_open_kitchen_no_redisable_when_empty(tmp_path, monkeypatch): - """open_kitchen must NOT call disable_components when no subsets are disabled.""" - from autoskillit.config.settings import AutomationConfig, SubsetsConfig - - monkeypatch.chdir(tmp_path) - mock_ctx = _make_mock_ctx() - mock_ctx.enable_components = AsyncMock() - mock_ctx.disable_components = AsyncMock() - mock_ctx.config = AutomationConfig(subsets=SubsetsConfig(disabled=[])) - - with patch("autoskillit.server._get_ctx", return_value=mock_ctx): - with patch("autoskillit.server.logger"): - with patch("autoskillit.server.tools_kitchen._prime_quota_cache", new=AsyncMock()): - with patch("autoskillit.server.tools_kitchen._write_hook_config"): - from autoskillit.server.tools_kitchen import open_kitchen - - await open_kitchen(ctx=mock_ctx) - - mock_ctx.disable_components.assert_not_called() - - -# REQ-PACK-008: open_kitchen stores active_recipe_packs -@pytest.mark.anyio -async def test_open_kitchen_sets_active_recipe_packs(tmp_path, monkeypatch): - """After _open_kitchen_handler(), ctx.active_recipe_packs is frozenset().""" - monkeypatch.chdir(tmp_path) - mock_ctx = _make_mock_ctx() - - with patch("autoskillit.server._get_ctx", return_value=mock_ctx): - with patch("autoskillit.server.logger"): - with patch("autoskillit.server.tools_kitchen._prime_quota_cache", new=AsyncMock()): - with patch("autoskillit.server.tools_kitchen._write_hook_config"): - from autoskillit.server.tools_kitchen import _open_kitchen_handler - - await _open_kitchen_handler() - - assert mock_ctx.active_recipe_packs == frozenset() - - -# REQ-PACK-008: close_kitchen clears active_recipe_packs -def test_close_kitchen_clears_active_recipe_packs(tmp_path, monkeypatch): - """After _close_kitchen_handler(), ctx.active_recipe_packs is None.""" - monkeypatch.chdir(tmp_path) - mock_ctx = _make_mock_ctx() - mock_ctx.active_recipe_packs = frozenset(["research"]) - - with patch("autoskillit.server._get_ctx", return_value=mock_ctx): - with patch("autoskillit.server.logger"): - from autoskillit.server.tools_kitchen import _close_kitchen_handler - - _close_kitchen_handler() - - assert mock_ctx.active_recipe_packs is None - - -# T-REFRESH-1 -@pytest.mark.anyio -async def test_open_kitchen_starts_quota_refresh_task(tmp_path, monkeypatch): - """After _open_kitchen_handler(), ctx.quota_refresh_task is a running asyncio.Task.""" - monkeypatch.chdir(tmp_path) - mock_ctx = _make_mock_ctx() - - async def instant_loop(config): - await asyncio.sleep(0) - - with patch("autoskillit.server._get_ctx", return_value=mock_ctx): - with patch("autoskillit.server.logger"): - with patch("autoskillit.server.tools_kitchen._prime_quota_cache", AsyncMock()): - with patch("autoskillit.server.tools_kitchen._write_hook_config"): - with patch( - "autoskillit.server.tools_kitchen._quota_refresh_loop", instant_loop - ): - from autoskillit.server.tools_kitchen import _open_kitchen_handler - - await _open_kitchen_handler() - - assert mock_ctx.quota_refresh_task is not None - assert isinstance(mock_ctx.quota_refresh_task, asyncio.Task) - mock_ctx.quota_refresh_task.cancel() - - -# T-REFRESH-2 -def test_close_kitchen_cancels_quota_refresh_task(tmp_path, monkeypatch): - """_close_kitchen_handler cancels ctx.quota_refresh_task and sets it to None.""" - monkeypatch.chdir(tmp_path) - mock_ctx = _make_mock_ctx() - mock_task = MagicMock(spec=asyncio.Task) - mock_ctx.quota_refresh_task = mock_task - - with patch("autoskillit.server._get_ctx", return_value=mock_ctx): - with patch("autoskillit.server.logger"): - from autoskillit.server.tools_kitchen import _close_kitchen_handler - - _close_kitchen_handler() - - mock_task.cancel.assert_called_once() - assert mock_ctx.quota_refresh_task is None - - -# T-REFRESH-3 -def test_tool_context_has_quota_refresh_task_field(): - """ToolContext must have a quota_refresh_task field defaulting to None.""" - from autoskillit.pipeline.context import ToolContext - - fields = {f.name: f for f in dataclasses.fields(ToolContext)} - assert "quota_refresh_task" in fields - assert fields["quota_refresh_task"].default is None - - -# T-KITCHEN-1 -@pytest.mark.anyio -async def test_open_kitchen_warns_on_orphaned_hooks(tmp_path, monkeypatch): - """When settings.json contains a hook not in HOOK_REGISTRY, open_kitchen() - must include a drift warning in its response.""" - from autoskillit.hook_registry import HookDriftResult - - settings_dir = tmp_path / ".claude" - settings_dir.mkdir() - (settings_dir / "settings.json").write_text("{}") - - monkeypatch.setattr( - "autoskillit.hook_registry._claude_settings_path", - lambda scope: settings_dir / "settings.json", - ) - monkeypatch.setattr( - "autoskillit.hook_registry._count_hook_registry_drift", - lambda _: HookDriftResult(missing=0, orphaned=1), - ) - monkeypatch.setattr( - "autoskillit.hook_registry.find_broken_hook_scripts", - lambda _: [], - ) - - mock_ctx = _make_mock_ctx() - mock_ctx.enable_components = AsyncMock() - - with patch("autoskillit.server._get_ctx", return_value=mock_ctx): - with patch("autoskillit.server.logger"): - with patch("autoskillit.server.tools_kitchen._prime_quota_cache", new=AsyncMock()): - with patch("autoskillit.server.tools_kitchen._write_hook_config"): - from autoskillit.server.tools_kitchen import open_kitchen - - result = await open_kitchen(ctx=mock_ctx) - - parsed = json.loads(result) - content = parsed["content"] - assert ( - "orphan" in content.lower() or "drift" in content.lower() or "install" in content.lower() - ), "open_kitchen() must include a hook drift warning when orphaned > 0" - - -# T-KITCHEN-2 -@pytest.mark.anyio -async def test_open_kitchen_warns_on_missing_hook_scripts(tmp_path, monkeypatch): - """When hook scripts are absent from disk, open_kitchen() must warn.""" - from autoskillit.hook_registry import HookDriftResult - - settings_dir = tmp_path / ".claude" - settings_dir.mkdir() - (settings_dir / "settings.json").write_text("{}") - - monkeypatch.setattr( - "autoskillit.hook_registry._claude_settings_path", - lambda scope: settings_dir / "settings.json", - ) - monkeypatch.setattr( - "autoskillit.hook_registry.find_broken_hook_scripts", - lambda _: ["python3 /missing/status_health_guard.py"], - ) - monkeypatch.setattr( - "autoskillit.hook_registry._count_hook_registry_drift", - lambda _: HookDriftResult(missing=0, orphaned=0), - ) - - mock_ctx = _make_mock_ctx() - mock_ctx.enable_components = AsyncMock() - - with patch("autoskillit.server._get_ctx", return_value=mock_ctx): - with patch("autoskillit.server.logger"): - with patch("autoskillit.server.tools_kitchen._prime_quota_cache", new=AsyncMock()): - with patch("autoskillit.server.tools_kitchen._write_hook_config"): - from autoskillit.server.tools_kitchen import open_kitchen - - result = await open_kitchen(ctx=mock_ctx) - - parsed = json.loads(result) - content = parsed["content"] - assert "Hook scripts not found" in content, ( - "open_kitchen() must include the exact _build_hook_diagnostic_warning phrase" - ) - - -@pytest.mark.anyio -async def test_prime_quota_cache_catches_typeerror(monkeypatch): - """_prime_quota_cache must catch TypeError and not propagate — 'fails open' contract.""" - import autoskillit.execution as _exec_mod - from autoskillit.server.helpers import _prime_quota_cache - - async def raise_type_error(*a, **kw): - raise TypeError("float() argument must be a string or a real number, not 'NoneType'") - - monkeypatch.setattr(_exec_mod, "check_and_sleep_if_needed", raise_type_error) - - mock_ctx = MagicMock() - mock_ctx.config.quota_guard = MagicMock() - - with patch("autoskillit.server._get_ctx", return_value=mock_ctx): - with patch("autoskillit.server.helpers.logger") as mock_logger: - # Must not raise — fails open - await _prime_quota_cache() - mock_logger.warning.assert_called_once_with("quota_prime_failed", exc_info=True) - - -# --------------------------------------------------------------------------- -# Envelope contract tests — Phase 3 (#711 Part B) -# --------------------------------------------------------------------------- - - -@pytest.mark.anyio -async def test_open_kitchen_no_name_returns_json_envelope_with_success_true(tmp_path, monkeypatch): - """No-recipe open_kitchen returns JSON envelope with success=True.""" - monkeypatch.chdir(tmp_path) - mock_ctx = _make_mock_ctx() - mock_ctx.enable_components = AsyncMock() - - with patch("autoskillit.server._get_ctx", return_value=mock_ctx): - with patch("autoskillit.server.logger"): - with patch("autoskillit.server.tools_kitchen._prime_quota_cache", new=AsyncMock()): - with patch("autoskillit.server.tools_kitchen._write_hook_config"): - from autoskillit.server.tools_kitchen import open_kitchen - - result = await open_kitchen(ctx=mock_ctx) - - parsed = json.loads(result) - assert parsed["success"] is True - assert parsed["kitchen"] == "open" - assert "Kitchen is open" in parsed["content"] - assert parsed["ingredients_table"] is None - - -@pytest.mark.anyio -async def test_open_kitchen_recipe_found_returns_envelope_with_content_and_ingredients_table( - tmp_path, monkeypatch -): - """Recipe loads successfully: success=True, kitchen=open, version present.""" - monkeypatch.chdir(tmp_path) - mock_ctx = _make_mock_ctx() - mock_ctx.enable_components = AsyncMock() - mock_ctx.recipes = MagicMock() - mock_ctx.recipes.load_and_validate.return_value = { - "content": "name: demo\nsteps:\n do:\n tool: run_cmd\n", - "valid": True, - "suggestions": [], - "diagram": None, - "ingredients_table": "--- INGREDIENTS TABLE ---\n task required\n--- END TABLE ---", - } - mock_ctx.recipes.find.return_value = None - mock_ctx.config.migration.suppressed = [] - - with patch("autoskillit.server._get_ctx", return_value=mock_ctx): - with patch("autoskillit.server.logger"): - with patch("autoskillit.server.tools_kitchen._prime_quota_cache", new=AsyncMock()): - with patch("autoskillit.server.tools_kitchen._write_hook_config"): - from autoskillit.server.tools_kitchen import open_kitchen - - result_str = await open_kitchen(name="demo", ctx=mock_ctx) - - parsed = json.loads(result_str) - assert parsed["success"] is True - assert parsed["kitchen"] == "open" - assert "version" in parsed - assert "--- INGREDIENTS TABLE ---" in result_str - - -@pytest.mark.anyio -async def test_open_kitchen_smoke_test_renders_resolved_base_branch(monkeypatch): - """T7: open_kitchen smoke-test renders the config-resolved base_branch value.""" - import autoskillit.recipe._api as api_mod - from autoskillit.core import pkg_root - from autoskillit.recipe.repository import DefaultRecipeRepository - - project_dir = pkg_root().parent.parent - monkeypatch.chdir(project_dir) - monkeypatch.setattr(api_mod, "_LOAD_CACHE", {}) - monkeypatch.setattr( - "autoskillit.server.tools_kitchen.resolve_ingredient_defaults", - lambda _: {"base_branch": "integration"}, - ) - - mock_ctx = _make_mock_ctx() - mock_ctx.enable_components = AsyncMock() - mock_ctx.quota_refresh_task = None - mock_ctx.recipes = DefaultRecipeRepository() - mock_ctx.config.migration.suppressed = [] - - with patch("autoskillit.server._get_ctx", return_value=mock_ctx): - with patch("autoskillit.server.logger"): - with patch("autoskillit.server.tools_kitchen._prime_quota_cache", new=AsyncMock()): - with patch("autoskillit.server.tools_kitchen._write_hook_config"): - from autoskillit.server.tools_kitchen import open_kitchen - - result_str = await open_kitchen(name="smoke-test", ctx=mock_ctx) - - parsed = json.loads(result_str) - assert parsed["success"] is True - ing_table = parsed.get("ingredients_table") or "" - assert ing_table, "ingredients_table must be present and non-empty" - assert "integration" in ing_table - # base_branch row must NOT show the YAML literal "main" - base_branch_rows = [line for line in ing_table.splitlines() if "base_branch" in line] - assert base_branch_rows, "base_branch row must appear in ingredients_table" - assert all("main" not in row for row in base_branch_rows) - - -@pytest.mark.anyio -async def test_open_kitchen_recipe_not_found_returns_failure_envelope(tmp_path, monkeypatch): - """Invalid recipe name returns failure envelope (via load_and_validate raising).""" - monkeypatch.chdir(tmp_path) - mock_ctx = _make_mock_ctx() - mock_ctx.enable_components = AsyncMock() - mock_ctx.recipes = MagicMock() - mock_ctx.recipes.load_and_validate.side_effect = ValueError("No recipe 'bad' found") - mock_ctx.config.migration.suppressed = [] - - with patch("autoskillit.server._get_ctx", return_value=mock_ctx): - with patch("autoskillit.server.logger"): - with patch("autoskillit.server.tools_kitchen._prime_quota_cache", new=AsyncMock()): - with patch("autoskillit.server.tools_kitchen._write_hook_config"): - from autoskillit.server.tools_kitchen import open_kitchen - - result_str = await open_kitchen(name="bad", ctx=mock_ctx) - - parsed = json.loads(result_str) - assert parsed["success"] is False - assert parsed["kitchen"] == "failed" - assert len(parsed["user_visible_message"]) > 0 - assert "ValueError" in parsed["error"] - - -@pytest.mark.anyio -async def test_open_kitchen_server_not_initialized_returns_failure_envelope(tmp_path, monkeypatch): - """tool_ctx.recipes is None → failure envelope with user_visible_message.""" - monkeypatch.chdir(tmp_path) - mock_ctx = _make_mock_ctx() - mock_ctx.enable_components = AsyncMock() - mock_ctx.recipes = None - mock_ctx.config.migration.suppressed = [] - - with patch("autoskillit.server._get_ctx", return_value=mock_ctx): - with patch("autoskillit.server.logger"): - with patch("autoskillit.server.tools_kitchen._prime_quota_cache", new=AsyncMock()): - with patch("autoskillit.server.tools_kitchen._write_hook_config"): - from autoskillit.server.tools_kitchen import open_kitchen - - result_str = await open_kitchen(name="demo", ctx=mock_ctx) - - parsed = json.loads(result_str) - assert parsed["success"] is False - assert parsed["kitchen"] == "failed" - assert "user_visible_message" in parsed - assert "not initialized" in parsed["user_visible_message"] - - -@pytest.mark.anyio -async def test_open_kitchen_headless_denied_returns_failure_envelope(tmp_path, monkeypatch): - """AUTOSKILLIT_HEADLESS=1: failure envelope with user_visible_message present.""" - monkeypatch.setenv("AUTOSKILLIT_HEADLESS", "1") - monkeypatch.chdir(tmp_path) - from autoskillit.server.tools_kitchen import open_kitchen - - result = json.loads(await open_kitchen()) - assert result["success"] is False - assert result["kitchen"] == "failed" - assert "user_visible_message" in result - assert len(result["user_visible_message"]) > 0 - assert result["stage"] == "headless_guard" - - -@pytest.mark.anyio -async def test_open_kitchen_prime_quota_cache_typeerror_returns_failure_envelope( - tmp_path, monkeypatch -): - """_prime_quota_cache raising TypeError → failure envelope with stage=prime_quota_cache.""" - monkeypatch.chdir(tmp_path) - mock_ctx = _make_mock_ctx() - mock_ctx.enable_components = AsyncMock() - - async def raise_type_error(): - raise TypeError("float() argument must be a string or a real number, not 'NoneType'") - - with patch("autoskillit.server._get_ctx", return_value=mock_ctx): - with patch("autoskillit.server.logger"): - with patch( - "autoskillit.server.tools_kitchen._prime_quota_cache", - new=raise_type_error, - ): - with patch("autoskillit.server.tools_kitchen._write_hook_config"): - from autoskillit.server.tools_kitchen import open_kitchen - - result_str = await open_kitchen(ctx=mock_ctx) - - parsed = json.loads(result_str) - assert parsed["success"] is False - assert parsed["kitchen"] == "failed" - assert parsed["stage"] == "prime_quota_cache" - assert "TypeError" in parsed["error"] - assert len(parsed["user_visible_message"]) > 0 - - -@pytest.mark.anyio -async def test_open_kitchen_prime_quota_cache_runtimeerror_returns_failure_envelope( - tmp_path, monkeypatch -): - """_prime_quota_cache raising RuntimeError → failure envelope.""" - monkeypatch.chdir(tmp_path) - mock_ctx = _make_mock_ctx() - mock_ctx.enable_components = AsyncMock() - - async def raise_runtime(): - raise RuntimeError("unexpected failure") - - with patch("autoskillit.server._get_ctx", return_value=mock_ctx): - with patch("autoskillit.server.logger"): - with patch( - "autoskillit.server.tools_kitchen._prime_quota_cache", - new=raise_runtime, - ): - with patch("autoskillit.server.tools_kitchen._write_hook_config"): - from autoskillit.server.tools_kitchen import open_kitchen - - result_str = await open_kitchen(ctx=mock_ctx) - - parsed = json.loads(result_str) - assert parsed["success"] is False - assert parsed["stage"] == "prime_quota_cache" - - -@pytest.mark.anyio -async def test_open_kitchen_create_background_task_raises_returns_failure_envelope( - tmp_path, monkeypatch -): - """create_background_task raising → failure envelope with stage=start_quota_refresh.""" - monkeypatch.chdir(tmp_path) - mock_ctx = _make_mock_ctx() - mock_ctx.enable_components = AsyncMock() - - with patch("autoskillit.server._get_ctx", return_value=mock_ctx): - with patch("autoskillit.server.logger"): - with patch("autoskillit.server.tools_kitchen._prime_quota_cache", new=AsyncMock()): - with patch("autoskillit.server.tools_kitchen._write_hook_config"): - with patch( - "autoskillit.server.tools_kitchen.create_background_task", - side_effect=RuntimeError("task creation failed"), - ): - from autoskillit.server.tools_kitchen import open_kitchen - - result_str = await open_kitchen(ctx=mock_ctx) - - parsed = json.loads(result_str) - assert parsed["success"] is False - assert parsed["stage"] == "start_quota_refresh" - - -@pytest.mark.anyio -async def test_open_kitchen_load_and_validate_raises_returns_failure_envelope( - tmp_path, monkeypatch -): - """load_and_validate raising → failure envelope with stage=load_and_validate.""" - monkeypatch.chdir(tmp_path) - mock_ctx = _make_mock_ctx() - mock_ctx.enable_components = AsyncMock() - mock_ctx.recipes = MagicMock() - mock_ctx.recipes.load_and_validate.side_effect = OSError("disk full") - mock_ctx.config.migration.suppressed = [] - - with patch("autoskillit.server._get_ctx", return_value=mock_ctx): - with patch("autoskillit.server.logger"): - with patch("autoskillit.server.tools_kitchen._prime_quota_cache", new=AsyncMock()): - with patch("autoskillit.server.tools_kitchen._write_hook_config"): - from autoskillit.server.tools_kitchen import open_kitchen - - result_str = await open_kitchen(name="demo", ctx=mock_ctx) - - parsed = json.loads(result_str) - assert parsed["success"] is False - assert parsed["stage"] == "load_and_validate" - - -@pytest.mark.anyio -async def test_open_kitchen_apply_triage_gate_raises_returns_failure_envelope( - tmp_path, monkeypatch -): - """_apply_triage_gate raising → failure envelope with stage=apply_triage_gate.""" - monkeypatch.chdir(tmp_path) - mock_ctx = _make_mock_ctx() - mock_ctx.enable_components = AsyncMock() - mock_ctx.recipes = MagicMock() - mock_ctx.recipes.load_and_validate.return_value = { - "content": "test", - "valid": True, - "suggestions": [], - } - mock_ctx.recipes.find.return_value = None - mock_ctx.config.migration.suppressed = [] - - async def raise_apply(*a, **kw): - raise RuntimeError("triage failed") - - with patch("autoskillit.server._get_ctx", return_value=mock_ctx): - with patch("autoskillit.server.logger"): - with patch("autoskillit.server.tools_kitchen._prime_quota_cache", new=AsyncMock()): - with patch("autoskillit.server.tools_kitchen._write_hook_config"): - with patch( - "autoskillit.server.tools_kitchen._apply_triage_gate", - new=raise_apply, - ): - from autoskillit.server.tools_kitchen import open_kitchen - - result_str = await open_kitchen(name="demo", ctx=mock_ctx) - - parsed = json.loads(result_str) - assert parsed["success"] is False - assert parsed["stage"] == "apply_triage_gate" - - -@pytest.mark.anyio -async def test_open_kitchen_enable_components_raises_returns_failure_envelope( - tmp_path, monkeypatch -): - """ctx.enable_components raising → failure envelope with stage=enable_components.""" - monkeypatch.chdir(tmp_path) - mock_ctx = _make_mock_ctx() - mock_ctx.enable_components = AsyncMock(side_effect=RuntimeError("enable failed")) - - with patch("autoskillit.server._get_ctx", return_value=mock_ctx): - with patch("autoskillit.server.logger"): - with patch("autoskillit.server.tools_kitchen._prime_quota_cache", new=AsyncMock()): - with patch("autoskillit.server.tools_kitchen._write_hook_config"): - from autoskillit.server.tools_kitchen import open_kitchen - - result_str = await open_kitchen(ctx=mock_ctx) - - parsed = json.loads(result_str) - assert parsed["success"] is False - assert parsed["stage"] == "enable_components" - - -@pytest.mark.anyio -async def test_open_kitchen_sous_chef_read_raises_returns_failure_envelope(tmp_path, monkeypatch): - """Path.read_text raising OSError → failure envelope with stage=read_sous_chef.""" - monkeypatch.chdir(tmp_path) - mock_ctx = _make_mock_ctx() - mock_ctx.enable_components = AsyncMock() - - import autoskillit.server.tools_kitchen as tk_mod - - _original_pkg_root = tk_mod.pkg_root - - def fake_pkg_root(): - root = tmp_path / "fake_pkg" - sc_path = root / "skills" / "sous-chef" - sc_path.mkdir(parents=True, exist_ok=True) - skill_md = sc_path / "SKILL.md" - skill_md.write_text("dummy") - skill_md.chmod(0o000) - return root - - with patch("autoskillit.server._get_ctx", return_value=mock_ctx): - with patch("autoskillit.server.logger"): - with patch("autoskillit.server.tools_kitchen._prime_quota_cache", new=AsyncMock()): - with patch("autoskillit.server.tools_kitchen._write_hook_config"): - with patch.object(tk_mod, "pkg_root", fake_pkg_root): - from autoskillit.server.tools_kitchen import open_kitchen - - result_str = await open_kitchen(ctx=mock_ctx) - - # Restore permissions for cleanup - for p in tmp_path.rglob("SKILL.md"): - p.chmod(0o644) - - parsed = json.loads(result_str) - assert parsed["success"] is False - assert parsed["stage"] == "read_sous_chef" - - -# Parametrized: every failure envelope has user_visible_message -_FAILURE_STAGES = [ - "headless_guard", - "prime_quota_cache", - "start_quota_refresh", - "enable_components", - "load_and_validate", - "apply_triage_gate", - "recipe_context", -] - - -@pytest.mark.parametrize("stage", _FAILURE_STAGES) -def test_every_failure_envelope_has_user_visible_message(stage): - """All failure envelopes have a non-empty user_visible_message string.""" - from autoskillit.server.tools_kitchen import _kitchen_failure_envelope - - envelope = json.loads(_kitchen_failure_envelope(RuntimeError("test"), stage=stage)) - assert isinstance(envelope["user_visible_message"], str) - assert len(envelope["user_visible_message"]) > 0 - assert envelope["success"] is False - assert envelope["kitchen"] == "failed" - - -@pytest.mark.parametrize( - "stage", - _FAILURE_STAGES + ["hook_diagnostic", "read_sous_chef", "redisable_subsets"], -) -def test_every_return_path_parses_as_json_and_has_boolean_success(stage): - """Every failure envelope parses as JSON with boolean success.""" - from autoskillit.server.tools_kitchen import _kitchen_failure_envelope - - envelope = json.loads(_kitchen_failure_envelope(RuntimeError("test"), stage=stage)) - assert isinstance(envelope["success"], bool) - - -@pytest.mark.anyio -async def test_disable_quota_guard_writes_disabled_flag(tmp_path, monkeypatch): - """disable_quota_guard() sets quota_guard.disabled=True in the hook config file.""" - monkeypatch.chdir(tmp_path) - hook_cfg_path = tmp_path.joinpath(*_HOOK_CONFIG_PATH_COMPONENTS) - hook_cfg_path.parent.mkdir(parents=True, exist_ok=True) - hook_cfg_path.write_text( - json.dumps({"quota_guard": {"cache_path": "/some/path.json", "cache_max_age": 300}}) - ) - - from autoskillit.server.tools_kitchen import disable_quota_guard - - result_str = await disable_quota_guard() - parsed = json.loads(result_str) - assert parsed["success"] is True - - payload = json.loads(hook_cfg_path.read_text()) - assert payload["quota_guard"]["disabled"] is True - - -@pytest.mark.anyio -async def test_disable_quota_guard_denies_headless(tmp_path, monkeypatch): - """disable_quota_guard() returns an error when called from a headless session.""" - monkeypatch.chdir(tmp_path) - monkeypatch.setenv("AUTOSKILLIT_HEADLESS", "1") - - from autoskillit.server.tools_kitchen import disable_quota_guard - - result_str = await disable_quota_guard() - parsed = json.loads(result_str) - assert parsed["success"] is False - - -@pytest.mark.anyio -async def test_disable_quota_guard_returns_error_when_kitchen_not_open(tmp_path, monkeypatch): - """disable_quota_guard() returns an error when the kitchen is not open.""" - monkeypatch.chdir(tmp_path) - - from autoskillit.server.tools_kitchen import disable_quota_guard - - result_str = await disable_quota_guard() - parsed = json.loads(result_str) - assert parsed["success"] is False diff --git a/tests/server/test_tools_kitchen_envelope.py b/tests/server/test_tools_kitchen_envelope.py new file mode 100644 index 0000000000..7ac42d3aa9 --- /dev/null +++ b/tests/server/test_tools_kitchen_envelope.py @@ -0,0 +1,551 @@ +"""Tests for tools_kitchen.py: hook drift warnings and failure envelopes.""" + +from __future__ import annotations + +import json +from unittest.mock import AsyncMock, MagicMock, patch + +import pytest + +from tests.server.conftest import _make_mock_ctx + +pytestmark = [pytest.mark.layer("server"), pytest.mark.small] + + +# --------------------------------------------------------------------------- +# Group E — hook drift / diagnostic warnings +# --------------------------------------------------------------------------- + + +# T-KITCHEN-1 +@pytest.mark.anyio +async def test_open_kitchen_warns_on_orphaned_hooks(tmp_path, monkeypatch): + """When settings.json contains a hook not in HOOK_REGISTRY, open_kitchen() + must include a drift warning in its response.""" + from autoskillit.hook_registry import HookDriftResult + + settings_dir = tmp_path / ".claude" + settings_dir.mkdir() + (settings_dir / "settings.json").write_text("{}") + + monkeypatch.setattr( + "autoskillit.hook_registry._claude_settings_path", + lambda scope: settings_dir / "settings.json", + ) + monkeypatch.setattr( + "autoskillit.hook_registry._count_hook_registry_drift", + lambda _: HookDriftResult(missing=0, orphaned=1), + ) + monkeypatch.setattr( + "autoskillit.hook_registry.find_broken_hook_scripts", + lambda _: [], + ) + + mock_ctx = _make_mock_ctx() + mock_ctx.enable_components = AsyncMock() + + with patch("autoskillit.server._get_ctx", return_value=mock_ctx): + with patch("autoskillit.server.logger"): + with patch( + "autoskillit.server.tools.tools_kitchen._prime_quota_cache", new=AsyncMock() + ): + with patch("autoskillit.server.tools.tools_kitchen._write_hook_config"): + from autoskillit.server.tools.tools_kitchen import open_kitchen + + result = await open_kitchen(ctx=mock_ctx) + + parsed = json.loads(result) + content = parsed["content"] + assert ( + "orphan" in content.lower() or "drift" in content.lower() or "install" in content.lower() + ), "open_kitchen() must include a hook drift warning when orphaned > 0" + + +# T-KITCHEN-2 +@pytest.mark.anyio +async def test_open_kitchen_warns_on_missing_hook_scripts(tmp_path, monkeypatch): + """When hook scripts are absent from disk, open_kitchen() must warn.""" + from autoskillit.hook_registry import HookDriftResult + + settings_dir = tmp_path / ".claude" + settings_dir.mkdir() + (settings_dir / "settings.json").write_text("{}") + + monkeypatch.setattr( + "autoskillit.hook_registry._claude_settings_path", + lambda scope: settings_dir / "settings.json", + ) + monkeypatch.setattr( + "autoskillit.hook_registry.find_broken_hook_scripts", + lambda _: ["python3 /missing/status_health_guard.py"], + ) + monkeypatch.setattr( + "autoskillit.hook_registry._count_hook_registry_drift", + lambda _: HookDriftResult(missing=0, orphaned=0), + ) + + mock_ctx = _make_mock_ctx() + mock_ctx.enable_components = AsyncMock() + + with patch("autoskillit.server._get_ctx", return_value=mock_ctx): + with patch("autoskillit.server.logger"): + with patch( + "autoskillit.server.tools.tools_kitchen._prime_quota_cache", new=AsyncMock() + ): + with patch("autoskillit.server.tools.tools_kitchen._write_hook_config"): + from autoskillit.server.tools.tools_kitchen import open_kitchen + + result = await open_kitchen(ctx=mock_ctx) + + parsed = json.loads(result) + content = parsed["content"] + assert "Hook scripts not found" in content, ( + "open_kitchen() must include the exact _build_hook_diagnostic_warning phrase" + ) + + +@pytest.mark.anyio +async def test_prime_quota_cache_catches_typeerror(monkeypatch): + """_prime_quota_cache must catch TypeError and not propagate — 'fails open' contract.""" + import autoskillit.server._misc as _misc_mod + from autoskillit.server._misc import _prime_quota_cache + + async def raise_type_error(*a, **kw): + raise TypeError("float() argument must be a string or a real number, not 'NoneType'") + + monkeypatch.setattr(_misc_mod, "check_and_sleep_if_needed", raise_type_error) + + mock_ctx = MagicMock() + mock_ctx.config.quota_guard = MagicMock() + + with patch("autoskillit.server._state._get_ctx", return_value=mock_ctx): + with patch("autoskillit.server._misc.logger") as mock_logger: + # Must not raise — fails open + await _prime_quota_cache() + mock_logger.warning.assert_called_once_with("quota_prime_failed", exc_info=True) + + +# --------------------------------------------------------------------------- +# Group F — failure envelopes — Phase 3 (#711 Part B) +# --------------------------------------------------------------------------- + + +@pytest.mark.anyio +async def test_open_kitchen_no_name_returns_json_envelope_with_success_true(tmp_path, monkeypatch): + """No-recipe open_kitchen returns JSON envelope with success=True.""" + monkeypatch.chdir(tmp_path) + mock_ctx = _make_mock_ctx() + mock_ctx.enable_components = AsyncMock() + + with patch("autoskillit.server._get_ctx", return_value=mock_ctx): + with patch("autoskillit.server.logger"): + with patch( + "autoskillit.server.tools.tools_kitchen._prime_quota_cache", new=AsyncMock() + ): + with patch("autoskillit.server.tools.tools_kitchen._write_hook_config"): + from autoskillit.server.tools.tools_kitchen import open_kitchen + + result = await open_kitchen(ctx=mock_ctx) + + parsed = json.loads(result) + assert parsed["success"] is True + assert parsed["kitchen"] == "open" + assert "Kitchen is open" in parsed["content"] + assert parsed["ingredients_table"] is None + + +@pytest.mark.anyio +async def test_open_kitchen_recipe_found_returns_envelope_with_content_and_ingredients_table( + tmp_path, monkeypatch +): + """Recipe loads successfully: success=True, kitchen=open, version present.""" + monkeypatch.chdir(tmp_path) + mock_ctx = _make_mock_ctx() + mock_ctx.enable_components = AsyncMock() + mock_ctx.recipes = MagicMock() + mock_ctx.recipes.load_and_validate.return_value = { + "content": "name: demo\nsteps:\n do:\n tool: run_cmd\n", + "valid": True, + "suggestions": [], + "diagram": None, + "ingredients_table": "--- INGREDIENTS TABLE ---\n task required\n--- END TABLE ---", + } + mock_ctx.recipes.find.return_value = None + mock_ctx.config.migration.suppressed = [] + + with patch("autoskillit.server._get_ctx", return_value=mock_ctx): + with patch("autoskillit.server.logger"): + with patch( + "autoskillit.server.tools.tools_kitchen._prime_quota_cache", new=AsyncMock() + ): + with patch("autoskillit.server.tools.tools_kitchen._write_hook_config"): + from autoskillit.server.tools.tools_kitchen import open_kitchen + + result_str = await open_kitchen(name="demo", ctx=mock_ctx) + + parsed = json.loads(result_str) + assert parsed["success"] is True + assert parsed["kitchen"] == "open" + assert "version" in parsed + assert "--- INGREDIENTS TABLE ---" in result_str + + +@pytest.mark.anyio +async def test_open_kitchen_smoke_test_renders_resolved_base_branch(monkeypatch): + """T7: open_kitchen smoke-test renders the config-resolved base_branch value.""" + import autoskillit.recipe._api as api_mod + from autoskillit.core import pkg_root + from autoskillit.recipe.repository import DefaultRecipeRepository + + project_dir = pkg_root().parent.parent + monkeypatch.chdir(project_dir) + monkeypatch.setattr(api_mod, "_LOAD_CACHE", {}) + monkeypatch.setattr( + "autoskillit.server.tools.tools_kitchen.resolve_ingredient_defaults", + lambda _: {"base_branch": "develop"}, + ) + + mock_ctx = _make_mock_ctx() + mock_ctx.enable_components = AsyncMock() + mock_ctx.quota_refresh_task = None + mock_ctx.recipes = DefaultRecipeRepository() + mock_ctx.config.migration.suppressed = [] + + with patch("autoskillit.server._get_ctx", return_value=mock_ctx): + with patch("autoskillit.server.logger"): + with patch( + "autoskillit.server.tools.tools_kitchen._prime_quota_cache", new=AsyncMock() + ): + with patch("autoskillit.server.tools.tools_kitchen._write_hook_config"): + from autoskillit.server.tools.tools_kitchen import open_kitchen + + result_str = await open_kitchen(name="smoke-test", ctx=mock_ctx) + + parsed = json.loads(result_str) + assert parsed["success"] is True + ing_table = parsed.get("ingredients_table") or "" + assert ing_table, "ingredients_table must be present and non-empty" + assert "develop" in ing_table + # base_branch row must NOT show the YAML literal "main" + base_branch_rows = [line for line in ing_table.splitlines() if "base_branch" in line] + assert base_branch_rows, "base_branch row must appear in ingredients_table" + assert all("main" not in row for row in base_branch_rows) + + +@pytest.mark.anyio +async def test_open_kitchen_recipe_not_found_returns_failure_envelope(tmp_path, monkeypatch): + """Invalid recipe name returns failure envelope (via load_and_validate raising).""" + monkeypatch.chdir(tmp_path) + mock_ctx = _make_mock_ctx() + mock_ctx.enable_components = AsyncMock() + mock_ctx.recipes = MagicMock() + mock_ctx.recipes.load_and_validate.side_effect = ValueError("No recipe 'bad' found") + mock_ctx.config.migration.suppressed = [] + + with patch("autoskillit.server._get_ctx", return_value=mock_ctx): + with patch("autoskillit.server.logger"): + with patch( + "autoskillit.server.tools.tools_kitchen._prime_quota_cache", new=AsyncMock() + ): + with patch("autoskillit.server.tools.tools_kitchen._write_hook_config"): + from autoskillit.server.tools.tools_kitchen import open_kitchen + + result_str = await open_kitchen(name="bad", ctx=mock_ctx) + + parsed = json.loads(result_str) + assert parsed["success"] is False + assert parsed["kitchen"] == "failed" + assert len(parsed["user_visible_message"]) > 0 + assert "ValueError" in parsed["error"] + + +@pytest.mark.anyio +async def test_open_kitchen_server_not_initialized_returns_failure_envelope(tmp_path, monkeypatch): + """tool_ctx.recipes is None → failure envelope with user_visible_message.""" + monkeypatch.chdir(tmp_path) + mock_ctx = _make_mock_ctx() + mock_ctx.enable_components = AsyncMock() + mock_ctx.recipes = None + mock_ctx.config.migration.suppressed = [] + + with patch("autoskillit.server._get_ctx", return_value=mock_ctx): + with patch("autoskillit.server.logger"): + with patch( + "autoskillit.server.tools.tools_kitchen._prime_quota_cache", new=AsyncMock() + ): + with patch("autoskillit.server.tools.tools_kitchen._write_hook_config"): + from autoskillit.server.tools.tools_kitchen import open_kitchen + + result_str = await open_kitchen(name="demo", ctx=mock_ctx) + + parsed = json.loads(result_str) + assert parsed["success"] is False + assert parsed["kitchen"] == "failed" + assert "user_visible_message" in parsed + assert "not initialized" in parsed["user_visible_message"] + + +@pytest.mark.anyio +async def test_open_kitchen_headless_denied_returns_failure_envelope(tmp_path, monkeypatch): + """AUTOSKILLIT_HEADLESS=1: failure envelope with user_visible_message present.""" + monkeypatch.setenv("AUTOSKILLIT_HEADLESS", "1") + monkeypatch.chdir(tmp_path) + from autoskillit.server.tools.tools_kitchen import open_kitchen + + result = json.loads(await open_kitchen()) + assert result["success"] is False + assert result["kitchen"] == "failed" + assert "user_visible_message" in result + assert len(result["user_visible_message"]) > 0 + assert result["stage"] == "headless_guard" + + +@pytest.mark.anyio +async def test_open_kitchen_prime_quota_cache_typeerror_returns_failure_envelope( + tmp_path, monkeypatch +): + """_prime_quota_cache raising TypeError → failure envelope with stage=prime_quota_cache.""" + monkeypatch.chdir(tmp_path) + mock_ctx = _make_mock_ctx() + mock_ctx.enable_components = AsyncMock() + + async def raise_type_error(): + raise TypeError("float() argument must be a string or a real number, not 'NoneType'") + + with patch("autoskillit.server._get_ctx", return_value=mock_ctx): + with patch("autoskillit.server.logger"): + with patch( + "autoskillit.server.tools.tools_kitchen._prime_quota_cache", + new=raise_type_error, + ): + with patch("autoskillit.server.tools.tools_kitchen._write_hook_config"): + from autoskillit.server.tools.tools_kitchen import open_kitchen + + result_str = await open_kitchen(ctx=mock_ctx) + + parsed = json.loads(result_str) + assert parsed["success"] is False + assert parsed["kitchen"] == "failed" + assert parsed["stage"] == "prime_quota_cache" + assert "TypeError" in parsed["error"] + assert len(parsed["user_visible_message"]) > 0 + + +@pytest.mark.anyio +async def test_open_kitchen_prime_quota_cache_runtimeerror_returns_failure_envelope( + tmp_path, monkeypatch +): + """_prime_quota_cache raising RuntimeError → failure envelope.""" + monkeypatch.chdir(tmp_path) + mock_ctx = _make_mock_ctx() + mock_ctx.enable_components = AsyncMock() + + async def raise_runtime(): + raise RuntimeError("unexpected failure") + + with patch("autoskillit.server._get_ctx", return_value=mock_ctx): + with patch("autoskillit.server.logger"): + with patch( + "autoskillit.server.tools.tools_kitchen._prime_quota_cache", + new=raise_runtime, + ): + with patch("autoskillit.server.tools.tools_kitchen._write_hook_config"): + from autoskillit.server.tools.tools_kitchen import open_kitchen + + result_str = await open_kitchen(ctx=mock_ctx) + + parsed = json.loads(result_str) + assert parsed["success"] is False + assert parsed["stage"] == "prime_quota_cache" + + +@pytest.mark.anyio +async def test_open_kitchen_create_background_task_raises_returns_failure_envelope( + tmp_path, monkeypatch +): + """create_background_task raising → failure envelope with stage=start_quota_refresh.""" + monkeypatch.chdir(tmp_path) + mock_ctx = _make_mock_ctx() + mock_ctx.enable_components = AsyncMock() + + with patch("autoskillit.server._get_ctx", return_value=mock_ctx): + with patch("autoskillit.server.logger"): + with patch( + "autoskillit.server.tools.tools_kitchen._prime_quota_cache", new=AsyncMock() + ): + with patch("autoskillit.server.tools.tools_kitchen._write_hook_config"): + with patch( + "autoskillit.server.tools.tools_kitchen.create_background_task", + side_effect=RuntimeError("task creation failed"), + ): + from autoskillit.server.tools.tools_kitchen import open_kitchen + + result_str = await open_kitchen(ctx=mock_ctx) + + parsed = json.loads(result_str) + assert parsed["success"] is False + assert parsed["stage"] == "start_quota_refresh" + + +@pytest.mark.anyio +async def test_open_kitchen_load_and_validate_raises_returns_failure_envelope( + tmp_path, monkeypatch +): + """load_and_validate raising → failure envelope with stage=load_and_validate.""" + monkeypatch.chdir(tmp_path) + mock_ctx = _make_mock_ctx() + mock_ctx.enable_components = AsyncMock() + mock_ctx.recipes = MagicMock() + mock_ctx.recipes.load_and_validate.side_effect = OSError("disk full") + mock_ctx.config.migration.suppressed = [] + + with patch("autoskillit.server._get_ctx", return_value=mock_ctx): + with patch("autoskillit.server.logger"): + with patch( + "autoskillit.server.tools.tools_kitchen._prime_quota_cache", new=AsyncMock() + ): + with patch("autoskillit.server.tools.tools_kitchen._write_hook_config"): + from autoskillit.server.tools.tools_kitchen import open_kitchen + + result_str = await open_kitchen(name="demo", ctx=mock_ctx) + + parsed = json.loads(result_str) + assert parsed["success"] is False + assert parsed["stage"] == "load_and_validate" + + +@pytest.mark.anyio +async def test_open_kitchen_apply_triage_gate_raises_returns_failure_envelope( + tmp_path, monkeypatch +): + """_apply_triage_gate raising → failure envelope with stage=apply_triage_gate.""" + monkeypatch.chdir(tmp_path) + mock_ctx = _make_mock_ctx() + mock_ctx.enable_components = AsyncMock() + mock_ctx.recipes = MagicMock() + mock_ctx.recipes.load_and_validate.return_value = { + "content": "test", + "valid": True, + "suggestions": [], + } + mock_ctx.recipes.find.return_value = None + mock_ctx.config.migration.suppressed = [] + + async def raise_apply(*a, **kw): + raise RuntimeError("triage failed") + + with patch("autoskillit.server._get_ctx", return_value=mock_ctx): + with patch("autoskillit.server.logger"): + with patch( + "autoskillit.server.tools.tools_kitchen._prime_quota_cache", new=AsyncMock() + ): + with patch("autoskillit.server.tools.tools_kitchen._write_hook_config"): + with patch( + "autoskillit.server.tools.tools_kitchen._apply_triage_gate", + new=raise_apply, + ): + from autoskillit.server.tools.tools_kitchen import open_kitchen + + result_str = await open_kitchen(name="demo", ctx=mock_ctx) + + parsed = json.loads(result_str) + assert parsed["success"] is False + assert parsed["stage"] == "apply_triage_gate" + + +@pytest.mark.anyio +async def test_open_kitchen_enable_components_raises_returns_failure_envelope( + tmp_path, monkeypatch +): + """ctx.enable_components raising → failure envelope with stage=enable_components.""" + monkeypatch.chdir(tmp_path) + mock_ctx = _make_mock_ctx() + mock_ctx.enable_components = AsyncMock(side_effect=RuntimeError("enable failed")) + + with patch("autoskillit.server._get_ctx", return_value=mock_ctx): + with patch("autoskillit.server.logger"): + with patch( + "autoskillit.server.tools.tools_kitchen._prime_quota_cache", new=AsyncMock() + ): + with patch("autoskillit.server.tools.tools_kitchen._write_hook_config"): + from autoskillit.server.tools.tools_kitchen import open_kitchen + + result_str = await open_kitchen(ctx=mock_ctx) + + parsed = json.loads(result_str) + assert parsed["success"] is False + assert parsed["stage"] == "enable_components" + + +@pytest.mark.anyio +async def test_open_kitchen_sous_chef_read_raises_returns_failure_envelope(tmp_path, monkeypatch): + """Path.read_text raising OSError → failure envelope with stage=read_sous_chef.""" + monkeypatch.chdir(tmp_path) + mock_ctx = _make_mock_ctx() + mock_ctx.enable_components = AsyncMock() + + import autoskillit.server.tools.tools_kitchen as tk_mod + + def fake_pkg_root(): + root = tmp_path / "fake_pkg" + sc_path = root / "skills" / "sous-chef" + sc_path.mkdir(parents=True, exist_ok=True) + skill_md = sc_path / "SKILL.md" + skill_md.write_text("dummy") + skill_md.chmod(0o000) + return root + + with patch("autoskillit.server._get_ctx", return_value=mock_ctx): + with patch("autoskillit.server.logger"): + with patch( + "autoskillit.server.tools.tools_kitchen._prime_quota_cache", new=AsyncMock() + ): + with patch("autoskillit.server.tools.tools_kitchen._write_hook_config"): + with patch.object(tk_mod, "pkg_root", fake_pkg_root): + from autoskillit.server.tools.tools_kitchen import open_kitchen + + result_str = await open_kitchen(ctx=mock_ctx) + + # Restore permissions for cleanup + for p in tmp_path.rglob("SKILL.md"): + p.chmod(0o644) + + parsed = json.loads(result_str) + assert parsed["success"] is False + assert parsed["stage"] == "read_sous_chef" + + +# Parametrized: every failure envelope has user_visible_message +_FAILURE_STAGES = [ + "headless_guard", + "prime_quota_cache", + "start_quota_refresh", + "enable_components", + "load_and_validate", + "apply_triage_gate", + "recipe_context", +] + + +@pytest.mark.parametrize("stage", _FAILURE_STAGES) +def test_every_failure_envelope_has_user_visible_message(stage): + """All failure envelopes have a non-empty user_visible_message string.""" + from autoskillit.server.tools.tools_kitchen import _kitchen_failure_envelope + + envelope = json.loads(_kitchen_failure_envelope(RuntimeError("test"), stage=stage)) + assert isinstance(envelope["user_visible_message"], str) + assert len(envelope["user_visible_message"]) > 0 + assert envelope["success"] is False + assert envelope["kitchen"] == "failed" + + +@pytest.mark.parametrize( + "stage", + _FAILURE_STAGES + ["hook_diagnostic", "read_sous_chef", "redisable_subsets"], +) +def test_every_return_path_parses_as_json_and_has_boolean_success(stage): + """Every failure envelope parses as JSON with boolean success.""" + from autoskillit.server.tools.tools_kitchen import _kitchen_failure_envelope + + envelope = json.loads(_kitchen_failure_envelope(RuntimeError("test"), stage=stage)) + assert isinstance(envelope["success"], bool) diff --git a/tests/server/test_tools_kitchen_gate.py b/tests/server/test_tools_kitchen_gate.py new file mode 100644 index 0000000000..c2d703c786 --- /dev/null +++ b/tests/server/test_tools_kitchen_gate.py @@ -0,0 +1,712 @@ +"""Tests for tools_kitchen.py: gate mechanics, hook config, recipe packs, refresh, misc.""" + +from __future__ import annotations + +import asyncio +import dataclasses +import json +from unittest.mock import AsyncMock, MagicMock, patch + +import pytest + +from autoskillit.config.settings import QuotaGuardConfig +from autoskillit.hooks.formatters._fmt_primitives import _HOOK_CONFIG_PATH_COMPONENTS +from tests.server.conftest import _make_mock_ctx + +pytestmark = [pytest.mark.layer("server"), pytest.mark.small] + + +# T2a +@pytest.mark.anyio +async def test_open_kitchen_enables_gate(tmp_path, monkeypatch): + """After _open_kitchen_handler(), gate is enabled.""" + monkeypatch.chdir(tmp_path) + mock_ctx = _make_mock_ctx() + + with patch("autoskillit.server._get_ctx", return_value=mock_ctx): + with patch("autoskillit.server.logger"): + with patch( + "autoskillit.server.tools.tools_kitchen._prime_quota_cache", new=AsyncMock() + ): + with patch("autoskillit.server.tools.tools_kitchen._write_hook_config"): + from autoskillit.server.tools.tools_kitchen import _open_kitchen_handler + + await _open_kitchen_handler() + + mock_ctx.gate.enable.assert_called_once() + + +# T2b +def test_close_kitchen_disables_gate(tmp_path, monkeypatch): + """After _close_kitchen_handler(), gate is disabled.""" + monkeypatch.chdir(tmp_path) + mock_ctx = _make_mock_ctx() + + with patch("autoskillit.server._get_ctx", return_value=mock_ctx): + with patch("autoskillit.server.logger"): + from autoskillit.server.tools.tools_kitchen import _close_kitchen_handler + + _close_kitchen_handler() + + mock_ctx.gate.disable.assert_called_once() + + +def test_hook_config_path_is_inside_temp_dir(tmp_path): + """_hook_config_path() must resolve to a path inside the project temp directory. + + This invariant ensures .hook_config.json receives automatic gitignore coverage + from temp/.gitignore ('*') and can never produce a gitignore gap regardless of + whatever entries are present in .gitignore or _AUTOSKILLIT_GITIGNORE_ENTRIES. + """ + from autoskillit.core.io import resolve_temp_dir + from autoskillit.server._misc import _hook_config_path + + hook_path = _hook_config_path(tmp_path) + temp_dir = resolve_temp_dir(tmp_path, None) + + assert hook_path.is_relative_to(temp_dir), ( + f"_hook_config_path() returned {hook_path!r} which is NOT inside " + f"temp dir {temp_dir!r}. Session-bridge files must live in temp/ " + f"to receive automatic gitignore coverage via temp/.gitignore." + ) + + +# T2c +def test_close_kitchen_no_file_no_error(tmp_path, monkeypatch): + """_close_kitchen_handler() doesn't raise when no gate file exists.""" + monkeypatch.chdir(tmp_path) + mock_ctx = _make_mock_ctx() + + with patch("autoskillit.server._get_ctx", return_value=mock_ctx): + with patch("autoskillit.server.logger"): + from autoskillit.server.tools.tools_kitchen import _close_kitchen_handler + + _close_kitchen_handler() # Should not raise + + # Gate file was never created — confirm it still does not exist + assert not tmp_path.joinpath(*_HOOK_CONFIG_PATH_COMPONENTS).exists() + + +# T-CACHE-1 +@pytest.mark.anyio +async def test_open_kitchen_primes_quota_cache(tmp_path, monkeypatch): + """open_kitchen must call _prime_quota_cache before any run_skill hook fires.""" + monkeypatch.chdir(tmp_path) + mock_ctx = _make_mock_ctx() + prime_mock = AsyncMock() + + with patch("autoskillit.server._get_ctx", return_value=mock_ctx): + with patch("autoskillit.server.logger"): + with patch("autoskillit.server.tools.tools_kitchen._prime_quota_cache", prime_mock): + with patch("autoskillit.server.tools.tools_kitchen._write_hook_config"): + from autoskillit.server.tools.tools_kitchen import _open_kitchen_handler + + await _open_kitchen_handler() + + prime_mock.assert_called_once() + + +# T-CACHE-2 +@pytest.mark.anyio +async def test_open_kitchen_writes_hook_config_json(tmp_path, monkeypatch): + """open_kitchen must write .autoskillit/.hook_config.json with user quota_guard values.""" + monkeypatch.chdir(tmp_path) + mock_ctx = _make_mock_ctx() + mock_ctx.config.quota_guard.short_window_threshold = 85.0 + mock_ctx.config.quota_guard.long_window_threshold = 98.0 + mock_ctx.config.quota_guard.long_window_patterns = ["weekly", "sonnet", "opus"] + mock_ctx.config.quota_guard.cache_max_age = 300 + mock_ctx.config.quota_guard.cache_path = "/custom/path.json" + mock_ctx.config.quota_guard.buffer_seconds = 60 + + with patch("autoskillit.server._get_ctx", return_value=mock_ctx) as mock_get_ctx: + with patch("autoskillit.server.logger"): + with patch( + "autoskillit.server.tools.tools_kitchen._prime_quota_cache", new=AsyncMock() + ): + from autoskillit.server.tools.tools_kitchen import _open_kitchen_handler + + await _open_kitchen_handler() + + assert mock_get_ctx.call_count >= 2, ( + "_get_ctx must be called in both _open_kitchen_handler and _write_hook_config; " + "if call_count < 2 the patch did not cover _write_hook_config's deferred import" + ) + hook_cfg = tmp_path.joinpath(*_HOOK_CONFIG_PATH_COMPONENTS) + assert hook_cfg.exists(), "Hook config file must be written by open_kitchen" + data = json.loads(hook_cfg.read_text()) + assert data["quota_guard"]["cache_max_age"] == 300 + assert data["quota_guard"]["cache_path"] == "/custom/path.json" + assert data["quota_guard"]["buffer_seconds"] == 60 + # threshold fields are pre-computed into should_block in the cache — not written to hook_config + assert "threshold" not in data["quota_guard"] + assert "short_window_threshold" not in data["quota_guard"] + assert "long_window_threshold" not in data["quota_guard"] + assert "long_window_patterns" not in data["quota_guard"] + # disabled is always written by _quota_guard_hook_payload + # MagicMock.enabled is truthy by default, so disabled must be False + assert data["quota_guard"]["disabled"] is False + # Confirm kitchen_id rename: hook config must contain 'kitchen_id' (not 'pipeline_id') + assert "kitchen_id" in data, ( + "hook config must contain 'kitchen_id' after rename from 'pipeline_id'" + ) + assert isinstance(data["kitchen_id"], str) and data["kitchen_id"], ( + "kitchen_id must be a non-empty string (UUID set by _open_kitchen_handler)" + ) + + +@pytest.mark.parametrize("enabled,expected_disabled", [(True, False), (False, True)]) +@pytest.mark.anyio +async def test_open_kitchen_bridges_enabled_flag_as_disabled( + tmp_path, monkeypatch, enabled, expected_disabled +): + """_write_hook_config() must serialize cfg.enabled as disabled: not cfg.enabled. + + Regression test for bridge completeness: QuotaGuardConfig.enabled must + translate to quota_guard.disabled in .hook_config.json so that the hook + subprocess can respect the profile-wide opt-out. + """ + monkeypatch.chdir(tmp_path) + mock_ctx = _make_mock_ctx() + mock_ctx.config.quota_guard = QuotaGuardConfig( + enabled=enabled, cache_max_age=300, cache_path="/p/q.json", buffer_seconds=60 + ) + + with patch("autoskillit.server._get_ctx", return_value=mock_ctx): + with patch("autoskillit.server.logger"): + with patch( + "autoskillit.server.tools.tools_kitchen._prime_quota_cache", new=AsyncMock() + ): + from autoskillit.server.tools.tools_kitchen import _open_kitchen_handler + + await _open_kitchen_handler() + + data = json.loads(tmp_path.joinpath(*_HOOK_CONFIG_PATH_COMPONENTS).read_text()) + assert data["quota_guard"]["disabled"] is expected_disabled, ( + f"enabled={enabled} must produce disabled={expected_disabled} in hook config" + ) + + +# T-CACHE-3 +@pytest.mark.anyio +async def test_close_kitchen_removes_hook_config_json(tmp_path, monkeypatch): + """close_kitchen must remove .autoskillit/.hook_config.json to prevent stale config.""" + monkeypatch.chdir(tmp_path) + mock_ctx = _make_mock_ctx() + mock_ctx.config.quota_guard.short_window_threshold = 85.0 + mock_ctx.config.quota_guard.long_window_threshold = 98.0 + mock_ctx.config.quota_guard.long_window_patterns = ["weekly", "sonnet", "opus"] + mock_ctx.config.quota_guard.cache_max_age = 300 + mock_ctx.config.quota_guard.cache_path = "~/.claude/quota_cache.json" + mock_ctx.config.quota_guard.buffer_seconds = 60 + + with patch("autoskillit.server._get_ctx", return_value=mock_ctx): + with patch("autoskillit.server.logger"): + with patch( + "autoskillit.server.tools.tools_kitchen._prime_quota_cache", new=AsyncMock() + ): + from autoskillit.server.tools.tools_kitchen import ( + _close_kitchen_handler, + _open_kitchen_handler, + ) + + await _open_kitchen_handler() + _close_kitchen_handler() + + hook_cfg = tmp_path.joinpath(*_HOOK_CONFIG_PATH_COMPONENTS) + assert not hook_cfg.exists(), "Hook config must be removed by close_kitchen" + + +# T-CACHE-4 +def test_open_kitchen_handler_is_async(): + """_open_kitchen_handler must be an async def so it can await _prime_quota_cache.""" + import inspect + + from autoskillit.server.tools.tools_kitchen import _open_kitchen_handler + + assert inspect.iscoroutinefunction(_open_kitchen_handler), ( + "_open_kitchen_handler must be async" + ) + + +# REQ-PACK-008: open_kitchen stores active_recipe_packs +@pytest.mark.anyio +async def test_open_kitchen_sets_active_recipe_packs(tmp_path, monkeypatch): + """After _open_kitchen_handler(), ctx.active_recipe_packs is frozenset().""" + monkeypatch.chdir(tmp_path) + mock_ctx = _make_mock_ctx() + + with patch("autoskillit.server._get_ctx", return_value=mock_ctx): + with patch("autoskillit.server.logger"): + with patch( + "autoskillit.server.tools.tools_kitchen._prime_quota_cache", new=AsyncMock() + ): + with patch("autoskillit.server.tools.tools_kitchen._write_hook_config"): + from autoskillit.server.tools.tools_kitchen import _open_kitchen_handler + + await _open_kitchen_handler() + + assert mock_ctx.active_recipe_packs == frozenset() + + +# REQ-PACK-008: close_kitchen clears active_recipe_packs +def test_close_kitchen_clears_active_recipe_packs(tmp_path, monkeypatch): + """After _close_kitchen_handler(), ctx.active_recipe_packs is None.""" + monkeypatch.chdir(tmp_path) + mock_ctx = _make_mock_ctx() + mock_ctx.active_recipe_packs = frozenset(["research"]) + + with patch("autoskillit.server._get_ctx", return_value=mock_ctx): + with patch("autoskillit.server.logger"): + from autoskillit.server.tools.tools_kitchen import _close_kitchen_handler + + _close_kitchen_handler() + + assert mock_ctx.active_recipe_packs is None + + +# T-REFRESH-1 +@pytest.mark.anyio +async def test_open_kitchen_starts_quota_refresh_task(tmp_path, monkeypatch): + """After _open_kitchen_handler(), ctx.quota_refresh_task is a running asyncio.Task.""" + monkeypatch.chdir(tmp_path) + mock_ctx = _make_mock_ctx() + + async def instant_loop(config): + await asyncio.sleep(0) + + with patch("autoskillit.server._get_ctx", return_value=mock_ctx): + with patch("autoskillit.server.logger"): + with patch("autoskillit.server.tools.tools_kitchen._prime_quota_cache", AsyncMock()): + with patch("autoskillit.server.tools.tools_kitchen._write_hook_config"): + with patch( + "autoskillit.server.tools.tools_kitchen._quota_refresh_loop", instant_loop + ): + from autoskillit.server.tools.tools_kitchen import _open_kitchen_handler + + await _open_kitchen_handler() + + assert mock_ctx.quota_refresh_task is not None + assert isinstance(mock_ctx.quota_refresh_task, asyncio.Task) + mock_ctx.quota_refresh_task.cancel() + + +# T-REFRESH-2 +def test_close_kitchen_cancels_quota_refresh_task(tmp_path, monkeypatch): + """_close_kitchen_handler cancels ctx.quota_refresh_task and sets it to None.""" + monkeypatch.chdir(tmp_path) + mock_ctx = _make_mock_ctx() + mock_task = MagicMock(spec=asyncio.Task) + mock_ctx.quota_refresh_task = mock_task + + with patch("autoskillit.server._get_ctx", return_value=mock_ctx): + with patch("autoskillit.server.logger"): + from autoskillit.server.tools.tools_kitchen import _close_kitchen_handler + + _close_kitchen_handler() + + mock_task.cancel.assert_called_once() + assert mock_ctx.quota_refresh_task is None + + +# T-REFRESH-3 +def test_tool_context_has_quota_refresh_task_field(): + """ToolContext must have a quota_refresh_task field defaulting to None.""" + from autoskillit.pipeline.context import ToolContext + + fields = {f.name: f for f in dataclasses.fields(ToolContext)} + assert "quota_refresh_task" in fields + assert fields["quota_refresh_task"].default is None + + +# --------------------------------------------------------------------------- +# Group G — disable_quota_guard tool +# --------------------------------------------------------------------------- + + +@pytest.mark.anyio +async def test_disable_quota_guard_writes_disabled_flag(tmp_path, monkeypatch): + """disable_quota_guard() sets quota_guard.disabled=True in the hook config file.""" + monkeypatch.chdir(tmp_path) + hook_cfg_path = tmp_path.joinpath(*_HOOK_CONFIG_PATH_COMPONENTS) + hook_cfg_path.parent.mkdir(parents=True, exist_ok=True) + hook_cfg_path.write_text( + json.dumps({"quota_guard": {"cache_path": "/some/path.json", "cache_max_age": 300}}) + ) + + from autoskillit.server.tools.tools_kitchen import disable_quota_guard + + result_str = await disable_quota_guard() + parsed = json.loads(result_str) + assert parsed["success"] is True + + payload = json.loads(hook_cfg_path.read_text()) + assert payload["quota_guard"]["disabled"] is True + + +@pytest.mark.anyio +async def test_disable_quota_guard_denies_headless(tmp_path, monkeypatch): + """disable_quota_guard() returns an error when called from a headless session.""" + monkeypatch.chdir(tmp_path) + monkeypatch.setenv("AUTOSKILLIT_HEADLESS", "1") + + from autoskillit.server.tools.tools_kitchen import disable_quota_guard + + result_str = await disable_quota_guard() + parsed = json.loads(result_str) + assert parsed["success"] is False + + +@pytest.mark.anyio +async def test_disable_quota_guard_returns_error_when_kitchen_not_open(tmp_path, monkeypatch): + """disable_quota_guard() returns an error when the kitchen is not open.""" + monkeypatch.chdir(tmp_path) + + from autoskillit.server.tools.tools_kitchen import disable_quota_guard + + result_str = await disable_quota_guard() + parsed = json.loads(result_str) + assert parsed["success"] is False + + +# --------------------------------------------------------------------------- +# Group J — T7, T8, T5, alwaysLoad MCP meta +# --------------------------------------------------------------------------- + + +def test_kitchen_failure_envelope_hint_says_install_not_reinstall() -> None: + from autoskillit.server.tools.tools_kitchen import _kitchen_failure_envelope + + result = json.loads(_kitchen_failure_envelope(exc=RuntimeError("x"), stage="test")) + msg = result["user_visible_message"] + assert "autoskillit install" in msg + assert "reinstall" not in msg + + +def test_display_categories_omits_fleet_when_disabled() -> None: + """Fleet category must not appear in iter_display_categories output when fleet is disabled.""" + from autoskillit.config import iter_display_categories + + cfg_features: dict[str, bool] = {"fleet": False} + categories = [name for name, _ in iter_display_categories(cfg_features)] + assert "Fleet" not in categories + + +def test_display_categories_includes_fleet_when_enabled() -> None: + """Fleet category must appear in iter_display_categories output when fleet is enabled.""" + from autoskillit.config import iter_display_categories + + cfg_features: dict[str, bool] = {"fleet": True} + categories = [name for name, _ in iter_display_categories(cfg_features)] + assert "Fleet" in categories + + +# --------------------------------------------------------------------------- +# T5: close_kitchen cleans up review_gate_state.json +# --------------------------------------------------------------------------- + +_REVIEW_GATE_STATE_RELPATH = (".autoskillit", "temp", "review_gate_state.json") + + +# T5-1 +def test_close_kitchen_preserves_review_gate_when_loop_active(tmp_path, monkeypatch): + """Preserve review_gate_state.json when an active review loop is in progress.""" + monkeypatch.chdir(tmp_path) + mock_ctx = _make_mock_ctx() + + state_path = tmp_path.joinpath(*_REVIEW_GATE_STATE_RELPATH) + state_path.parent.mkdir(parents=True, exist_ok=True) + state_path.write_text( + json.dumps( + { + "gate": "LOOP_REQUIRED", + "review_verdict": "changes_requested", + "check_review_loop_called": False, + "pr_number": "1290", + "set_at": "2026-04-26T04:30:00+00:00", + } + ) + ) + assert state_path.exists(), "State file must exist before close_kitchen" + + with patch("autoskillit.server._get_ctx", return_value=mock_ctx): + with patch("autoskillit.server.logger"): + from autoskillit.server.tools.tools_kitchen import _close_kitchen_handler + + _close_kitchen_handler() + + assert state_path.exists(), "Active review loop state must survive close_kitchen" + + +# T5-2 +def test_close_kitchen_no_review_gate_state_no_error(tmp_path, monkeypatch): + """_close_kitchen_handler() must not raise when review_gate_state.json is absent.""" + monkeypatch.chdir(tmp_path) + mock_ctx = _make_mock_ctx() + + with patch("autoskillit.server._get_ctx", return_value=mock_ctx): + with patch("autoskillit.server.logger"): + from autoskillit.server.tools.tools_kitchen import _close_kitchen_handler + + _close_kitchen_handler() # Must not raise + + assert not tmp_path.joinpath(*_REVIEW_GATE_STATE_RELPATH).exists() + + +# T5-3 +def test_close_kitchen_removes_review_gate_when_loop_complete(tmp_path, monkeypatch): + """Remove review_gate_state.json when check_review_loop_called is True.""" + monkeypatch.chdir(tmp_path) + mock_ctx = _make_mock_ctx() + + state_path = tmp_path.joinpath(*_REVIEW_GATE_STATE_RELPATH) + state_path.parent.mkdir(parents=True, exist_ok=True) + state_path.write_text( + json.dumps( + { + "gate": "LOOP_REQUIRED", + "review_verdict": "changes_requested", + "check_review_loop_called": True, + "pr_number": "1290", + "set_at": "2026-04-26T04:30:00+00:00", + } + ) + ) + + with patch("autoskillit.server._get_ctx", return_value=mock_ctx): + with patch("autoskillit.server.logger"): + from autoskillit.server.tools.tools_kitchen import _close_kitchen_handler + + _close_kitchen_handler() + + assert not state_path.exists(), "Completed loop state must be cleaned up on close" + + +# T5-4 +def test_close_kitchen_removes_review_gate_when_gate_not_loop_required(tmp_path, monkeypatch): + """_close_kitchen_handler() must remove review_gate_state.json when gate != LOOP_REQUIRED.""" + monkeypatch.chdir(tmp_path) + mock_ctx = _make_mock_ctx() + + state_path = tmp_path.joinpath(*_REVIEW_GATE_STATE_RELPATH) + state_path.parent.mkdir(parents=True, exist_ok=True) + state_path.write_text( + json.dumps( + { + "gate": "CLEAR", + "check_review_loop_called": False, + "pr_number": "1290", + "set_at": "2026-04-26T04:30:00+00:00", + } + ) + ) + + with patch("autoskillit.server._get_ctx", return_value=mock_ctx): + with patch("autoskillit.server.logger"): + from autoskillit.server.tools.tools_kitchen import _close_kitchen_handler + + _close_kitchen_handler() + + assert not state_path.exists(), "Non-LOOP_REQUIRED gate state must be cleaned up on close" + + +# T5-5 +def test_close_kitchen_removes_review_gate_on_corrupt_json(tmp_path, monkeypatch): + """Delete review_gate_state.json when JSON is malformed (fail-safe).""" + monkeypatch.chdir(tmp_path) + mock_ctx = _make_mock_ctx() + + state_path = tmp_path.joinpath(*_REVIEW_GATE_STATE_RELPATH) + state_path.parent.mkdir(parents=True, exist_ok=True) + state_path.write_text("{not valid json") + + with patch("autoskillit.server._get_ctx", return_value=mock_ctx): + with patch("autoskillit.server.logger"): + from autoskillit.server.tools.tools_kitchen import _close_kitchen_handler + + _close_kitchen_handler() + + assert not state_path.exists(), "Corrupt gate state must be deleted (fail-safe)" + + +@pytest.mark.anyio +async def test_open_kitchen_has_always_load_meta() -> None: + """open_kitchen must carry anthropic/alwaysLoad: true in its MCP meta. + + alwaysLoad ensures open_kitchen is always in the initial tool context for + direct 'claude' sessions (where ToolSearch is enabled). Defense-in-depth + against the MCP first-call race for non-order/cook sessions. + """ + from autoskillit.server import mcp + + tool = await mcp.get_tool("open_kitchen") + assert tool is not None + assert tool.meta is not None and tool.meta.get("anthropic/alwaysLoad") is True, ( + "open_kitchen missing anthropic/alwaysLoad:true — add to @mcp.tool(meta={...})" + ) + + +# --------------------------------------------------------------------------- +# Triple-ID unification: kitchen_id inherits AUTOSKILLIT_CAMPAIGN_ID +# --------------------------------------------------------------------------- + + +@pytest.mark.anyio +async def test_open_kitchen_inherits_campaign_id_from_env(tmp_path, monkeypatch): + """When AUTOSKILLIT_CAMPAIGN_ID is in env, open_kitchen uses it as kitchen_id.""" + monkeypatch.chdir(tmp_path) + monkeypatch.setenv("AUTOSKILLIT_CAMPAIGN_ID", "abc123def456") + mock_ctx = _make_mock_ctx() + + with patch("autoskillit.server._get_ctx", return_value=mock_ctx): + with patch("autoskillit.server.logger"): + with patch( + "autoskillit.server.tools.tools_kitchen._prime_quota_cache", new=AsyncMock() + ): + with patch("autoskillit.server.tools.tools_kitchen._write_hook_config"): + from autoskillit.server.tools.tools_kitchen import _open_kitchen_handler + + await _open_kitchen_handler() + + assert mock_ctx.kitchen_id == "abc123def456" + + +@pytest.mark.anyio +async def test_open_kitchen_generates_uuid_without_campaign_env(tmp_path, monkeypatch): + """Without AUTOSKILLIT_CAMPAIGN_ID, open_kitchen generates a fresh UUID.""" + monkeypatch.chdir(tmp_path) + monkeypatch.delenv("AUTOSKILLIT_CAMPAIGN_ID", raising=False) + mock_ctx = _make_mock_ctx() + + with patch("autoskillit.server._get_ctx", return_value=mock_ctx): + with patch("autoskillit.server.logger"): + with patch( + "autoskillit.server.tools.tools_kitchen._prime_quota_cache", new=AsyncMock() + ): + with patch("autoskillit.server.tools.tools_kitchen._write_hook_config"): + from autoskillit.server.tools.tools_kitchen import _open_kitchen_handler + + await _open_kitchen_handler() + + kitchen_id = mock_ctx.kitchen_id + assert len(kitchen_id) == 36 and kitchen_id.count("-") == 4 + + +# --------------------------------------------------------------------------- +# Group K — ingredients_only parameter +# --------------------------------------------------------------------------- + + +@pytest.mark.anyio +async def test_open_kitchen_ingredients_only_strips_content(tmp_path, monkeypatch): + """open_kitchen(name=X, ingredients_only=True) must omit content from result.""" + monkeypatch.chdir(tmp_path) + mock_ctx = _make_mock_ctx() + mock_ctx.enable_components = AsyncMock() + mock_ctx.recipes = MagicMock() + mock_ctx.recipes.load_and_validate.return_value = { + "content": "name: test\nsteps:\n s1:\n tool: run_skill\n", + "ingredients_table": ( + "| Name | Description | Default |\n| task | What to do | (required) |" + ), + "suggestions": [], + "valid": True, + "orchestration_rules": "MANDATORY: execute every step", + "stop_step_semantics": "stop semantics text", + "content_hash": "abc123", + "composite_hash": "def456", + "recipe_version": "1.0", + } + mock_ctx.recipes.find.return_value = MagicMock() + mock_ctx.config.migration.suppressed = [] + + with patch("autoskillit.server._get_ctx", return_value=mock_ctx): + with patch("autoskillit.server.logger"): + with patch( + "autoskillit.server.tools.tools_kitchen._prime_quota_cache", new=AsyncMock() + ): + with patch("autoskillit.server.tools.tools_kitchen._write_hook_config"): + with patch( + "autoskillit.server.tools.tools_kitchen._apply_triage_gate", + new=AsyncMock(side_effect=lambda r, *a, **kw: r), + ): + from autoskillit.server.tools.tools_kitchen import open_kitchen + + result_json = await open_kitchen( + name="test", ingredients_only=True, ctx=mock_ctx + ) + + result = json.loads(result_json) + assert result["success"] is True + assert result["ingredients_table"] is not None + assert "content" not in result + assert "orchestration_rules" not in result + assert "sous_chef_discipline" not in result + assert "stop_step_semantics" not in result + + +@pytest.mark.anyio +async def test_open_kitchen_ingredients_only_preserves_metadata(tmp_path, monkeypatch): + """ingredients_only=True must preserve success, kitchen, version, valid, suggestions.""" + monkeypatch.chdir(tmp_path) + mock_ctx = _make_mock_ctx() + mock_ctx.enable_components = AsyncMock() + mock_ctx.recipes = MagicMock() + mock_ctx.recipes.load_and_validate.return_value = { + "content": "name: test\nsteps:\n s1:\n tool: run_skill\n", + "ingredients_table": "| Name | Description | Default |", + "suggestions": [], + "valid": True, + "content_hash": "abc123", + "composite_hash": "def456", + "recipe_version": "1.0", + } + mock_ctx.recipes.find.return_value = MagicMock() + mock_ctx.config.migration.suppressed = [] + + with patch("autoskillit.server._get_ctx", return_value=mock_ctx): + with patch("autoskillit.server.logger"): + with patch( + "autoskillit.server.tools.tools_kitchen._prime_quota_cache", new=AsyncMock() + ): + with patch("autoskillit.server.tools.tools_kitchen._write_hook_config"): + with patch( + "autoskillit.server.tools.tools_kitchen._apply_triage_gate", + new=AsyncMock(side_effect=lambda r, *a, **kw: r), + ): + from autoskillit.server.tools.tools_kitchen import open_kitchen + + result_json = await open_kitchen( + name="test", ingredients_only=True, ctx=mock_ctx + ) + + result = json.loads(result_json) + assert result["success"] is True + assert result["kitchen"] == "open" + assert "version" in result + assert result["valid"] is True + assert result["suggestions"] == [] + + +@pytest.mark.anyio +async def test_open_kitchen_ingredients_only_no_name_ignored(tmp_path, monkeypatch): + """ingredients_only=True with name=None should behave like regular no-name open.""" + monkeypatch.chdir(tmp_path) + mock_ctx = _make_mock_ctx() + mock_ctx.enable_components = AsyncMock() + + with patch("autoskillit.server._get_ctx", return_value=mock_ctx): + with patch("autoskillit.server.logger"): + with patch( + "autoskillit.server.tools.tools_kitchen._prime_quota_cache", new=AsyncMock() + ): + with patch("autoskillit.server.tools.tools_kitchen._write_hook_config"): + from autoskillit.server.tools.tools_kitchen import open_kitchen + + result_json = await open_kitchen(ingredients_only=True, ctx=mock_ctx) + + result = json.loads(result_json) + assert result["success"] is True + assert "content" in result + assert result["kitchen"] == "open" + assert "version" in result diff --git a/tests/server/test_tools_kitchen_visibility.py b/tests/server/test_tools_kitchen_visibility.py new file mode 100644 index 0000000000..c5d46c7d18 --- /dev/null +++ b/tests/server/test_tools_kitchen_visibility.py @@ -0,0 +1,548 @@ +"""Tests for tools_kitchen.py: visibility, component management, sous-chef, redisable_subsets.""" + +from __future__ import annotations + +import json +from unittest.mock import AsyncMock, MagicMock, patch + +import pytest + +from autoskillit.core.types._type_constants import SOUS_CHEF_MANDATORY_SECTIONS +from tests.server.conftest import _make_mock_ctx + +pytestmark = [pytest.mark.layer("server"), pytest.mark.small] + + +# --------------------------------------------------------------------------- +# Group C — visibility + component management +# --------------------------------------------------------------------------- + + +# T-VISIBILITY-1: open_kitchen tool calls ctx.enable_components +@pytest.mark.anyio +async def test_open_kitchen_tool_calls_enable_components(tmp_path, monkeypatch): + """open_kitchen tool must call ctx.enable_components(tags={'kitchen'}).""" + monkeypatch.chdir(tmp_path) + mock_ctx = _make_mock_ctx() + mock_ctx.enable_components = AsyncMock() + + with patch("autoskillit.server._get_ctx", return_value=mock_ctx): + with patch("autoskillit.server.logger"): + with patch( + "autoskillit.server.tools.tools_kitchen._prime_quota_cache", new=AsyncMock() + ): + with patch("autoskillit.server.tools.tools_kitchen._write_hook_config"): + from autoskillit.server.tools.tools_kitchen import open_kitchen + + await open_kitchen(ctx=mock_ctx) + + mock_ctx.enable_components.assert_called_once_with(tags={"kitchen"}) + + +# T-VISIBILITY-2: close_kitchen tool calls ctx.reset_visibility +@pytest.mark.anyio +async def test_close_kitchen_tool_calls_reset_visibility(tmp_path, monkeypatch): + """close_kitchen tool must call ctx.reset_visibility().""" + monkeypatch.chdir(tmp_path) + mock_ctx = _make_mock_ctx() + mock_ctx.reset_visibility = AsyncMock() + + with patch("autoskillit.server._get_ctx", return_value=mock_ctx): + with patch("autoskillit.server.logger"): + from autoskillit.server.tools.tools_kitchen import close_kitchen + + await close_kitchen(ctx=mock_ctx) + + mock_ctx.reset_visibility.assert_called_once() + + +@pytest.mark.anyio +async def test_open_kitchen_does_not_write_gate_file(tmp_path, monkeypatch): + """_open_kitchen_handler must never write a gate file.""" + monkeypatch.chdir(tmp_path) + mock_ctx = _make_mock_ctx() + mock_ctx.config.quota_guard.short_window_threshold = 85.0 + mock_ctx.config.quota_guard.long_window_threshold = 98.0 + mock_ctx.config.quota_guard.long_window_patterns = ["weekly", "sonnet", "opus"] + mock_ctx.config.quota_guard.cache_max_age = 300 + mock_ctx.config.quota_guard.cache_path = "~/.claude/quota_cache.json" + mock_ctx.config.quota_guard.buffer_seconds = 60 + with patch("autoskillit.server._get_ctx", return_value=mock_ctx): + with patch("autoskillit.server.logger"): + with patch( + "autoskillit.server.tools.tools_kitchen._prime_quota_cache", new=AsyncMock() + ): + from autoskillit.server.tools.tools_kitchen import _open_kitchen_handler + + await _open_kitchen_handler() + gate_file = tmp_path / ".autoskillit" / "temp" / ".kitchen_gate" + assert not gate_file.exists() + + +def test_close_kitchen_does_not_produce_gate_file(tmp_path, monkeypatch): + """_close_kitchen_handler must not interact with any gate file path.""" + monkeypatch.chdir(tmp_path) + mock_ctx = _make_mock_ctx() + with patch("autoskillit.server._get_ctx", return_value=mock_ctx): + with patch("autoskillit.server.logger"): + from autoskillit.server.tools.tools_kitchen import _close_kitchen_handler + + _close_kitchen_handler() + gate_file = tmp_path / ".autoskillit" / "temp" / ".kitchen_gate" + assert not gate_file.exists() + + +@pytest.mark.anyio +async def test_open_kitchen_includes_categorized_tool_listing(tmp_path, monkeypatch): + """open_kitchen response contains static categorized tool groups from _DISPLAY_CATEGORIES.""" + from autoskillit.config.ingredient_defaults import _DISPLAY_CATEGORIES + from autoskillit.server.tools.tools_kitchen import open_kitchen + + monkeypatch.chdir(tmp_path) + mock_ctx = _make_mock_ctx() + mock_ctx.enable_components = AsyncMock() + + with patch("autoskillit.server._get_ctx", return_value=mock_ctx): + with patch("autoskillit.server.logger"): + with patch( + "autoskillit.server.tools.tools_kitchen._prime_quota_cache", new=AsyncMock() + ): + with patch("autoskillit.server.tools.tools_kitchen._write_hook_config"): + result_str = await open_kitchen(ctx=mock_ctx) + + parsed = json.loads(result_str) + content = parsed["content"] + seen: set[str] = set() + for category_name, tools in _DISPLAY_CATEGORIES: + assert category_name in content, ( + f"Category '{category_name}' missing from open_kitchen response" + ) + for tool_name in tools: + if tool_name not in seen: + assert tool_name in content, ( + f"Tool '{tool_name}' missing from open_kitchen response" + ) + seen.add(tool_name) + + +@pytest.mark.anyio +async def test_open_kitchen_with_recipe_returns_combined_response(tmp_path, monkeypatch): + """open_kitchen(name='x') opens kitchen AND loads the recipe in one call.""" + monkeypatch.chdir(tmp_path) + recipes_dir = tmp_path / ".autoskillit" / "recipes" + recipes_dir.mkdir(parents=True) + yaml_content = ( + "name: test-recipe\ndescription: test\nsteps:\n do:\n tool: run_cmd\n" + " with:\n cmd: echo hi\n on_success: done\n on_failure: done\n" + " done:\n action: stop\n message: Done\n" + ) + (recipes_dir / "test-recipe.yaml").write_text(yaml_content) + mock_ctx = _make_mock_ctx() + mock_ctx.enable_components = AsyncMock() + mock_ctx.recipes = MagicMock() + mock_ctx.recipes.load_and_validate.return_value = { + "content": yaml_content, + "valid": True, + "suggestions": [], + "diagram": None, + } + mock_ctx.recipes.find.return_value = None + mock_ctx.config.migration.suppressed = [] + + with patch("autoskillit.server._get_ctx", return_value=mock_ctx): + with patch("autoskillit.server.logger"): + with patch( + "autoskillit.server.tools.tools_kitchen._prime_quota_cache", new=AsyncMock() + ): + with patch("autoskillit.server.tools.tools_kitchen._write_hook_config"): + from autoskillit.server.tools.tools_kitchen import open_kitchen + + result_str = await open_kitchen(name="test-recipe", ctx=mock_ctx) + + result = json.loads(result_str) + assert result["success"] is True + assert result["kitchen"] == "open" + assert "version" in result + assert "content" in result + assert "test-recipe" in result["content"] + mock_ctx.gate.enable.assert_called_once() + mock_ctx.enable_components.assert_called_once_with(tags={"kitchen"}) + + +@pytest.mark.anyio +async def test_open_kitchen_with_recipe_not_found(tmp_path, monkeypatch): + """open_kitchen(name='nonexistent') returns error with kitchen still open.""" + monkeypatch.chdir(tmp_path) + mock_ctx = _make_mock_ctx() + mock_ctx.enable_components = AsyncMock() + mock_ctx.recipes = MagicMock() + mock_ctx.recipes.load_and_validate.return_value = { + "error": "No recipe named 'nonexistent' found", + } + mock_ctx.recipes.find.return_value = None + mock_ctx.config.migration.suppressed = [] + + with patch("autoskillit.server._get_ctx", return_value=mock_ctx): + with patch("autoskillit.server.logger"): + with patch( + "autoskillit.server.tools.tools_kitchen._prime_quota_cache", new=AsyncMock() + ): + with patch("autoskillit.server.tools.tools_kitchen._write_hook_config"): + from autoskillit.server.tools.tools_kitchen import open_kitchen + + result_str = await open_kitchen(name="nonexistent", ctx=mock_ctx) + + result = json.loads(result_str) + assert "error" in result + assert "nonexistent" in result["error"] + assert result["kitchen"] == "open" + # Kitchen should still be opened even if recipe fails + mock_ctx.gate.enable.assert_called_once() + + +@pytest.mark.anyio +async def test_open_kitchen_without_recipe_returns_json_envelope(tmp_path, monkeypatch): + """open_kitchen() without name returns JSON envelope with success=True.""" + monkeypatch.chdir(tmp_path) + mock_ctx = _make_mock_ctx() + mock_ctx.enable_components = AsyncMock() + + with patch("autoskillit.server._get_ctx", return_value=mock_ctx): + with patch("autoskillit.server.logger"): + with patch( + "autoskillit.server.tools.tools_kitchen._prime_quota_cache", new=AsyncMock() + ): + with patch("autoskillit.server.tools.tools_kitchen._write_hook_config"): + from autoskillit.server.tools.tools_kitchen import open_kitchen + + result = await open_kitchen(ctx=mock_ctx) + + assert isinstance(result, str) + parsed = json.loads(result) + assert parsed["success"] is True + assert parsed["kitchen"] == "open" + assert "Kitchen is open" in parsed["content"] + assert parsed["ingredients_table"] is None + + +@pytest.mark.anyio +async def test_open_kitchen_denied_by_gate_when_headless(tmp_path, monkeypatch): + monkeypatch.setenv("AUTOSKILLIT_HEADLESS", "1") + monkeypatch.chdir(tmp_path) + from autoskillit.server.tools.tools_kitchen import open_kitchen + + result = json.loads(await open_kitchen()) + assert result["success"] is False + assert result["kitchen"] == "failed" + assert "user_visible_message" in result + assert len(result["user_visible_message"]) > 0 + assert result["stage"] == "headless_guard" + + +@pytest.mark.anyio +async def test_close_kitchen_denied_when_headless(tmp_path, monkeypatch): + monkeypatch.setenv("AUTOSKILLIT_HEADLESS", "1") + monkeypatch.chdir(tmp_path) + from autoskillit.server.tools.tools_kitchen import close_kitchen + + result = json.loads(await close_kitchen()) + assert result["success"] is False + assert result["subtype"] == "headless_error" + + +# T-VIS-003 +@pytest.mark.anyio +async def test_open_kitchen_redisables_subsets(tmp_path, monkeypatch): + """open_kitchen must call ctx.disable_components for each disabled subset.""" + from autoskillit.config.settings import AutomationConfig, SubsetsConfig + + monkeypatch.chdir(tmp_path) + mock_ctx = _make_mock_ctx() + mock_ctx.enable_components = AsyncMock() + mock_ctx.disable_components = AsyncMock() + mock_ctx.config = AutomationConfig(subsets=SubsetsConfig(disabled=["github", "ci"])) + + with patch("autoskillit.server._get_ctx", return_value=mock_ctx): + with patch("autoskillit.server.logger"): + with patch( + "autoskillit.server.tools.tools_kitchen._prime_quota_cache", new=AsyncMock() + ): + with patch("autoskillit.server.tools.tools_kitchen._write_hook_config"): + from autoskillit.server.tools.tools_kitchen import open_kitchen + + await open_kitchen(ctx=mock_ctx) + + disable_calls = mock_ctx.disable_components.call_args_list + disabled_tags = [ + c.kwargs.get("tags") or (c.args[0] if c.args else None) for c in disable_calls + ] + assert {"github"} in disabled_tags + assert {"ci"} in disabled_tags + + +# T-VIS-004 +@pytest.mark.anyio +async def test_open_kitchen_redisable_order(tmp_path, monkeypatch): + """ctx.disable_components must be called after ctx.enable_components (order matters).""" + from autoskillit.config.settings import AutomationConfig, SubsetsConfig + + monkeypatch.chdir(tmp_path) + call_order = [] + mock_ctx = _make_mock_ctx() + + async def record_enable(**kwargs): + call_order.append(("enable", kwargs)) + + async def record_disable(**kwargs): + call_order.append(("disable", kwargs)) + + mock_ctx.enable_components = record_enable + mock_ctx.disable_components = record_disable + mock_ctx.config = AutomationConfig(subsets=SubsetsConfig(disabled=["github"])) + + with patch("autoskillit.server._get_ctx", return_value=mock_ctx): + with patch("autoskillit.server.logger"): + with patch( + "autoskillit.server.tools.tools_kitchen._prime_quota_cache", new=AsyncMock() + ): + with patch("autoskillit.server.tools.tools_kitchen._write_hook_config"): + from autoskillit.server.tools.tools_kitchen import open_kitchen + + await open_kitchen(ctx=mock_ctx) + + enable_idx = next(i for i, (op, _) in enumerate(call_order) if op == "enable") + disable_idx = next(i for i, (op, _) in enumerate(call_order) if op == "disable") + assert enable_idx < disable_idx, "disable_components must be called after enable_components" + + +# T-VIS-005 +@pytest.mark.anyio +async def test_open_kitchen_no_redisable_when_empty(tmp_path, monkeypatch): + """open_kitchen must NOT call disable_components when no subsets are disabled.""" + from autoskillit.config.settings import AutomationConfig, SubsetsConfig + + monkeypatch.chdir(tmp_path) + mock_ctx = _make_mock_ctx() + mock_ctx.enable_components = AsyncMock() + mock_ctx.disable_components = AsyncMock() + mock_ctx.config = AutomationConfig( + subsets=SubsetsConfig(disabled=[]), features={"fleet": True} + ) + + with patch("autoskillit.server._get_ctx", return_value=mock_ctx): + with patch("autoskillit.server.logger"): + with patch( + "autoskillit.server.tools.tools_kitchen._prime_quota_cache", new=AsyncMock() + ): + with patch("autoskillit.server.tools.tools_kitchen._write_hook_config"): + from autoskillit.server.tools.tools_kitchen import open_kitchen + + await open_kitchen(ctx=mock_ctx) + + mock_ctx.disable_components.assert_not_called() + + +# --------------------------------------------------------------------------- +# Group H — sous-chef discipline injection +# --------------------------------------------------------------------------- + + +@pytest.mark.anyio +async def test_sous_chef_discipline_not_in_open_kitchen_result(tmp_path, monkeypatch): + """open_kitchen does not inject sous_chef_discipline — delivery is via system prompt.""" + monkeypatch.chdir(tmp_path) + mock_ctx = _make_mock_ctx() + mock_ctx.enable_components = AsyncMock() + mock_ctx.recipes = MagicMock() + mock_ctx.recipes.load_and_validate.return_value = { + "content": "name: implementation\nsteps:\n do:\n tool: run_cmd\n", + "valid": True, + "suggestions": [], + "diagram": None, + } + mock_ctx.recipes.find.return_value = None + mock_ctx.config.migration.suppressed = [] + + with patch("autoskillit.server._get_ctx", return_value=mock_ctx): + with patch("autoskillit.server.logger"): + with patch( + "autoskillit.server.tools.tools_kitchen._prime_quota_cache", new=AsyncMock() + ): + with patch("autoskillit.server.tools.tools_kitchen._write_hook_config"): + from autoskillit.server.tools.tools_kitchen import open_kitchen + + result_str = await open_kitchen(name="implementation", ctx=mock_ctx) + + result = json.loads(result_str) + assert result["success"] is True + assert "sous_chef_discipline" not in result, ( + "sous_chef_discipline must not be injected by open_kitchen; " + "food truck delivery is handled by _build_admiral_dispatch_block() " + "in fleet/_prompts.py" + ) + + +@pytest.mark.anyio +async def test_open_kitchen_result_keys_match_typed_dict(tmp_path, monkeypatch): + """All keys in open_kitchen result are declared in OpenKitchenResult.""" + from autoskillit.recipe._recipe_ingredients import OpenKitchenResult + + monkeypatch.chdir(tmp_path) + mock_ctx = _make_mock_ctx() + mock_ctx.enable_components = AsyncMock() + mock_ctx.recipes = MagicMock() + mock_ctx.recipes.load_and_validate.return_value = { + "content": "name: implementation\nsteps:\n do:\n tool: run_cmd\n", + "valid": True, + "suggestions": [], + "diagram": None, + } + mock_ctx.recipes.find.return_value = None + mock_ctx.config.migration.suppressed = [] + + with patch("autoskillit.server._get_ctx", return_value=mock_ctx): + with patch("autoskillit.server.logger"): + with patch( + "autoskillit.server.tools.tools_kitchen._prime_quota_cache", new=AsyncMock() + ): + with patch("autoskillit.server.tools.tools_kitchen._write_hook_config"): + from autoskillit.server.tools.tools_kitchen import open_kitchen + + result_str = await open_kitchen(name="implementation", ctx=mock_ctx) + + result = json.loads(result_str) + declared = set(OpenKitchenResult.__annotations__) + undeclared = set(result.keys()) - declared + assert undeclared == set(), ( + f"open_kitchen returned keys not declared in OpenKitchenResult: {sorted(undeclared)}. " + "Add each to OpenKitchenResult in recipe/_recipe_ingredients.py." + ) + for key in ("success", "kitchen", "version"): + assert key in result, ( + f"open_kitchen result missing always-present key {key!r}. " + "OpenKitchenResult declares it but the handler does not populate it." + ) + + +@pytest.mark.anyio +async def test_sous_chef_rules_injected_at_open_kitchen(tmp_path, monkeypatch): + """Path B (no-name) must inject full sous-chef SKILL.md into response text.""" + monkeypatch.chdir(tmp_path) + mock_ctx = _make_mock_ctx() + mock_ctx.enable_components = AsyncMock() + + with patch("autoskillit.server._get_ctx", return_value=mock_ctx): + with patch("autoskillit.server.logger"): + with patch( + "autoskillit.server.tools.tools_kitchen._prime_quota_cache", new=AsyncMock() + ): + with patch("autoskillit.server.tools.tools_kitchen._write_hook_config"): + from autoskillit.server.tools.tools_kitchen import open_kitchen + + result = await open_kitchen(ctx=mock_ctx) + + parsed = json.loads(result) + content = parsed["content"] + for header in SOUS_CHEF_MANDATORY_SECTIONS: + assert header in content, ( + f"open_kitchen no-name response missing sous-chef section: {header!r}" + ) + + +@pytest.mark.anyio +async def test_open_kitchen_degrades_gracefully_without_sous_chef(tmp_path, monkeypatch): + """When sous-chef SKILL.md is absent, Path B must return a valid response without raising.""" + monkeypatch.chdir(tmp_path) + mock_ctx = _make_mock_ctx() + mock_ctx.enable_components = AsyncMock() + + import autoskillit.server.tools.tools_kitchen as tk_mod + + fake_pkg_root_dir = tmp_path / "fake_pkg" + (fake_pkg_root_dir / "skills" / "sous-chef").mkdir(parents=True) + # sous-chef SKILL.md is deliberately absent — directory exists, file does not + + def fake_pkg_root() -> object: + return fake_pkg_root_dir + + with patch("autoskillit.server._get_ctx", return_value=mock_ctx): + with patch("autoskillit.server.logger"): + with patch( + "autoskillit.server.tools.tools_kitchen._prime_quota_cache", new=AsyncMock() + ): + with patch("autoskillit.server.tools.tools_kitchen._write_hook_config"): + with patch.object(tk_mod, "pkg_root", fake_pkg_root): + from autoskillit.server.tools.tools_kitchen import open_kitchen + + result = await open_kitchen(ctx=mock_ctx) + + parsed = json.loads(result) + assert parsed["success"] is True + + +# --------------------------------------------------------------------------- +# Group I — _redisable_subsets unit tests +# --------------------------------------------------------------------------- + + +@pytest.mark.anyio +async def test_redisable_subsets_includes_feature_tags() -> None: + """When fleet feature is disabled, _redisable_subsets disables fleet tag.""" + from autoskillit.server.tools.tools_kitchen import _redisable_subsets + + mock_ctx = AsyncMock() + + await _redisable_subsets(mock_ctx, [], features={"fleet": False}) + + calls = mock_ctx.disable_components.call_args_list + disabled_tag_sets = [c.kwargs.get("tags", set()) for c in calls] + assert any("fleet" in tags for tags in disabled_tag_sets), ( + "fleet tag must be disabled when fleet feature is off" + ) + + +@pytest.mark.anyio +async def test_redisable_subsets_does_not_disable_kitchen_core_tag() -> None: + """_redisable_subsets must not pass kitchen-core to disable_components. + + FastMCP union model: any enabled tag keeps the tool visible. Verifies that + kitchen-core is not included in the suppressed tag sets so that tools with + the kitchen-core tag retain visibility after the feature gate pass. + """ + from autoskillit.server.tools.tools_kitchen import _redisable_subsets + + disabled_tags: list[set] = [] + mock_ctx = AsyncMock() + + async def capture_disable(*, tags): + disabled_tags.append(tags) + + mock_ctx.disable_components.side_effect = capture_disable + + # No subsets disabled, but fleet feature is explicitly disabled + await _redisable_subsets(mock_ctx, [], features={"fleet": False}) + + # fleet tag should be disabled + assert any("fleet" in t for t in disabled_tags), ( + "fleet tag must be suppressed when feature is off" + ) + # kitchen-core must NOT be in the disabled set (union model: still visible) + assert not any("kitchen-core" in t for t in disabled_tags), ( + "kitchen-core tag must never be disabled by the feature gate pass" + ) + + +@pytest.mark.anyio +async def test_redisable_subsets_uses_shared_helper() -> None: + """_redisable_subsets delegates Pass 2 to _collect_disabled_feature_tags.""" + from unittest.mock import AsyncMock, patch + + from autoskillit.server.tools.tools_kitchen import _redisable_subsets + + mock_ctx = AsyncMock() + + with patch("autoskillit.server.tools.tools_kitchen._collect_disabled_feature_tags") as mock_h: + mock_h.return_value = frozenset({"fleet"}) + await _redisable_subsets(mock_ctx, [], features={"fleet": False}) + + mock_h.assert_called_once_with({"fleet": False}, experimental_enabled=False) diff --git a/tests/server/test_tools_label_validation.py b/tests/server/test_tools_label_validation.py index c17dfe8328..48aa71eb89 100644 --- a/tests/server/test_tools_label_validation.py +++ b/tests/server/test_tools_label_validation.py @@ -7,16 +7,22 @@ import pytest -from autoskillit.server.tools_issue_lifecycle import claim_issue, prepare_issue, release_issue +from autoskillit.server.tools.tools_issue_lifecycle import ( + claim_issue, + prepare_issue, + release_issue, +) + +pytestmark = [pytest.mark.layer("server"), pytest.mark.small] class TestClaimIssueWhitelist: @pytest.mark.anyio - async def test_claim_issue_rejects_disallowed_label(self, tool_ctx, monkeypatch): + async def test_claim_issue_rejects_disallowed_label(self, tool_ctx_kitchen_open, monkeypatch): """claim_issue returns error when effective_label is not in whitelist.""" - tool_ctx.config.github.allowed_labels = ["bug", "enhancement"] + tool_ctx_kitchen_open.config.github.allowed_labels = ["bug", "enhancement"] mock_client = AsyncMock() - monkeypatch.setattr(tool_ctx, "github_client", mock_client) + monkeypatch.setattr(tool_ctx_kitchen_open, "github_client", mock_client) result = json.loads( await claim_issue( @@ -27,20 +33,20 @@ async def test_claim_issue_rejects_disallowed_label(self, tool_ctx, monkeypatch) assert result["success"] is False assert "unlisted-label" in result["error"] mock_client.ensure_label.assert_not_called() - mock_client.add_labels.assert_not_called() + mock_client.swap_labels.assert_not_called() @pytest.mark.anyio - async def test_claim_issue_allows_whitelisted_label(self, tool_ctx, monkeypatch): + async def test_claim_issue_allows_whitelisted_label(self, tool_ctx_kitchen_open, monkeypatch): """claim_issue proceeds when effective_label is in whitelist.""" - tool_ctx.config.github.allowed_labels = ["in-progress", "bug"] + tool_ctx_kitchen_open.config.github.allowed_labels = ["in-progress", "bug"] mock_client = AsyncMock() mock_client.fetch_issue.return_value = { "success": True, "labels": [], } mock_client.ensure_label.return_value = {"success": True, "created": True} - mock_client.add_labels.return_value = {"success": True, "labels": ["in-progress"]} - monkeypatch.setattr(tool_ctx, "github_client", mock_client) + mock_client.swap_labels.return_value = {"success": True, "labels": ["in-progress"]} + monkeypatch.setattr(tool_ctx_kitchen_open, "github_client", mock_client) result = json.loads( await claim_issue( @@ -51,14 +57,16 @@ async def test_claim_issue_allows_whitelisted_label(self, tool_ctx, monkeypatch) assert result["success"] is True @pytest.mark.anyio - async def test_claim_issue_empty_whitelist_permits_any_label(self, tool_ctx, monkeypatch): + async def test_claim_issue_empty_whitelist_permits_any_label( + self, tool_ctx_kitchen_open, monkeypatch + ): """claim_issue with empty allowed_labels proceeds without restriction.""" - tool_ctx.config.github.allowed_labels = [] + tool_ctx_kitchen_open.config.github.allowed_labels = [] mock_client = AsyncMock() mock_client.fetch_issue.return_value = {"success": True, "labels": []} mock_client.ensure_label.return_value = {"success": True, "created": True} - mock_client.add_labels.return_value = {"success": True, "labels": ["any-label"]} - monkeypatch.setattr(tool_ctx, "github_client", mock_client) + mock_client.swap_labels.return_value = {"success": True, "labels": ["any-label"]} + monkeypatch.setattr(tool_ctx_kitchen_open, "github_client", mock_client) result = json.loads( await claim_issue( @@ -68,20 +76,40 @@ async def test_claim_issue_empty_whitelist_permits_any_label(self, tool_ctx, mon ) assert result["success"] is True + @pytest.mark.anyio + async def test_queued_label_is_allowed_by_default(self, tool_ctx_kitchen_open, monkeypatch): + """claim_issue with label=queued succeeds when allowed_labels defaults to empty.""" + tool_ctx_kitchen_open.config.github.allowed_labels = [] + mock_client = AsyncMock() + mock_client.fetch_issue.return_value = {"success": True, "labels": []} + mock_client.ensure_label.return_value = {"success": True, "created": True} + mock_client.swap_labels.return_value = {"success": True, "labels": ["queued"]} + monkeypatch.setattr(tool_ctx_kitchen_open, "github_client", mock_client) + + result = json.loads( + await claim_issue( + issue_url="https://github.com/owner/repo/issues/1", + label="queued", + ) + ) + assert result["success"] is True + class TestReleaseIssueWhitelist: @pytest.mark.anyio - async def test_release_issue_rejects_disallowed_staged_label(self, tool_ctx, monkeypatch): + async def test_release_issue_rejects_disallowed_staged_label( + self, tool_ctx_kitchen_open, monkeypatch + ): """release_issue returns error when staged label is not in whitelist.""" - tool_ctx.config.github.allowed_labels = ["in-progress", "bug"] + tool_ctx_kitchen_open.config.github.allowed_labels = ["in-progress", "bug"] mock_client = AsyncMock() mock_client.remove_label.return_value = {"success": True} - monkeypatch.setattr(tool_ctx, "github_client", mock_client) + monkeypatch.setattr(tool_ctx_kitchen_open, "github_client", mock_client) result = json.loads( await release_issue( issue_url="https://github.com/owner/repo/issues/1", - target_branch="integration", + target_branch="develop", staged_label="unlisted-staged", ) ) @@ -90,12 +118,14 @@ async def test_release_issue_rejects_disallowed_staged_label(self, tool_ctx, mon mock_client.ensure_label.assert_not_called() @pytest.mark.anyio - async def test_release_issue_no_staging_skips_validation(self, tool_ctx, monkeypatch): + async def test_release_issue_no_staging_skips_validation( + self, tool_ctx_kitchen_open, monkeypatch + ): """release_issue without staging doesn't validate staged_label.""" - tool_ctx.config.github.allowed_labels = ["in-progress"] + tool_ctx_kitchen_open.config.github.allowed_labels = ["in-progress"] mock_client = AsyncMock() mock_client.remove_label.return_value = {"success": True} - monkeypatch.setattr(tool_ctx, "github_client", mock_client) + monkeypatch.setattr(tool_ctx_kitchen_open, "github_client", mock_client) # target_branch=main means no staging — staged label not validated result = json.loads( @@ -109,11 +139,13 @@ async def test_release_issue_no_staging_skips_validation(self, tool_ctx, monkeyp class TestPrepareIssueWhitelist: @pytest.mark.anyio - async def test_prepare_issue_rejects_disallowed_explicit_label(self, tool_ctx, monkeypatch): + async def test_prepare_issue_rejects_disallowed_explicit_label( + self, tool_ctx_kitchen_open, monkeypatch + ): """prepare_issue returns error when explicit labels contain a disallowed entry.""" - tool_ctx.config.github.allowed_labels = ["bug", "enhancement"] + tool_ctx_kitchen_open.config.github.allowed_labels = ["bug", "enhancement"] mock_executor = AsyncMock() - tool_ctx.executor = mock_executor + tool_ctx_kitchen_open.executor = mock_executor result = json.loads( await prepare_issue( @@ -127,18 +159,20 @@ async def test_prepare_issue_rejects_disallowed_explicit_label(self, tool_ctx, m mock_executor.run.assert_not_called() @pytest.mark.anyio - async def test_prepare_issue_allows_whitelisted_explicit_labels(self, tool_ctx, monkeypatch): + async def test_prepare_issue_allows_whitelisted_explicit_labels( + self, tool_ctx_kitchen_open, monkeypatch + ): """prepare_issue proceeds when all explicit labels are whitelisted.""" import json as _json from autoskillit.core import SkillResult from autoskillit.core.types import RetryReason - from autoskillit.server.tools_issue_lifecycle import ( + from autoskillit.server.tools.tools_issue_lifecycle import ( _PREPARE_RESULT_END, _PREPARE_RESULT_START, ) - tool_ctx.config.github.allowed_labels = ["bug", "enhancement"] + tool_ctx_kitchen_open.config.github.allowed_labels = ["bug", "enhancement"] mock_executor = AsyncMock() payload = _json.dumps({"issue_url": "https://github.com/x/y/issues/1", "route": "impl"}) mock_executor.run.return_value = SkillResult( @@ -152,7 +186,7 @@ async def test_prepare_issue_allows_whitelisted_explicit_labels(self, tool_ctx, retry_reason=RetryReason.NONE, stderr="", ) - tool_ctx.executor = mock_executor + tool_ctx_kitchen_open.executor = mock_executor result = json.loads( await prepare_issue( @@ -165,18 +199,20 @@ async def test_prepare_issue_allows_whitelisted_explicit_labels(self, tool_ctx, mock_executor.run.assert_awaited_once() @pytest.mark.anyio - async def test_prepare_issue_no_labels_skips_validation(self, tool_ctx, monkeypatch): + async def test_prepare_issue_no_labels_skips_validation( + self, tool_ctx_kitchen_open, monkeypatch + ): """prepare_issue with labels=None skips whitelist validation entirely.""" import json as _json from autoskillit.core import SkillResult from autoskillit.core.types import RetryReason - from autoskillit.server.tools_issue_lifecycle import ( + from autoskillit.server.tools.tools_issue_lifecycle import ( _PREPARE_RESULT_END, _PREPARE_RESULT_START, ) - tool_ctx.config.github.allowed_labels = ["bug"] # restrictive whitelist + tool_ctx_kitchen_open.config.github.allowed_labels = ["bug"] # restrictive whitelist mock_executor = AsyncMock() payload = _json.dumps({"issue_url": "https://github.com/x/y/issues/1", "route": "impl"}) mock_executor.run.return_value = SkillResult( @@ -190,7 +226,7 @@ async def test_prepare_issue_no_labels_skips_validation(self, tool_ctx, monkeypa retry_reason=RetryReason.NONE, stderr="", ) - tool_ctx.executor = mock_executor + tool_ctx_kitchen_open.executor = mock_executor result = json.loads( await prepare_issue( diff --git a/tests/server/test_tools_list_recipes.py b/tests/server/test_tools_list_recipes.py index 7b356b9e78..617212d23c 100644 --- a/tests/server/test_tools_list_recipes.py +++ b/tests/server/test_tools_list_recipes.py @@ -9,15 +9,17 @@ import pytest from autoskillit.pipeline.gate import DefaultGateState -from autoskillit.server.tools_recipe import list_recipes +from autoskillit.server.tools.tools_recipe import list_recipes + +pytestmark = [pytest.mark.layer("server"), pytest.mark.small] class TestListRecipeTools: """Tests for kitchen-gated list_recipes tool.""" @pytest.fixture(autouse=True) - def _ensure_ctx(self, tool_ctx): - """Ensure server context is initialized (gate open by default).""" + def _ensure_ctx(self, tool_ctx_kitchen_open): + """Ensure server context is initialized with gate open.""" # SS1 @pytest.mark.anyio @@ -145,11 +147,11 @@ async def test_list_recipes_response_includes_source_field(self, mock_list): # P5F2-T1 @pytest.mark.anyio -async def test_list_recipes_no_recipes_returns_empty(tool_ctx): - """list_recipes returns empty-list JSON when recipes is not configured.""" - tool_ctx.recipes = None +async def test_list_recipes_no_recipes_returns_empty(tool_ctx_kitchen_open): + """list_recipes returns error JSON when recipes is not configured.""" + tool_ctx_kitchen_open.recipes = None result = json.loads(await list_recipes()) - assert result == [] + assert isinstance(result, dict) and "error" in result # P5F2-T1b diff --git a/tests/server/test_tools_load_recipe.py b/tests/server/test_tools_load_recipe.py index 95de6935fd..39cb42dfd5 100644 --- a/tests/server/test_tools_load_recipe.py +++ b/tests/server/test_tools_load_recipe.py @@ -1,4 +1,4 @@ -"""Tests for autoskillit server load_recipe and migrate_recipe tools.""" +"""Tests for autoskillit server load_recipe tool.""" from __future__ import annotations @@ -11,41 +11,11 @@ from autoskillit.config import AutomationConfig from autoskillit.core import SkillResult from autoskillit.core.types import RetryReason -from autoskillit.pipeline.gate import GATED_TOOLS, UNGATED_TOOLS, DefaultGateState -from autoskillit.server.tools_recipe import ( - load_recipe, - migrate_recipe, -) +from autoskillit.pipeline.gate import DefaultGateState +from autoskillit.server.tools.tools_recipe import load_recipe +from tests.server._helpers import _MINIMAL_SCRIPT_YAML -# --------------------------------------------------------------------------- -# Minimal valid script YAML used across migration suggestion tests -# --------------------------------------------------------------------------- - -_MINIMAL_SCRIPT_YAML = """\ -name: test-script -description: Test -summary: test -ingredients: - task: - description: What to do - required: true -steps: - do-thing: - tool: run_skill - with: - skill_command: "/autoskillit:investigate ${{ inputs.task }}" - cwd: "." - on_success: done - on_failure: escalate - done: - action: stop - message: "Done." - escalate: - action: stop - message: "Failed." -kitchen_rules: - - "Follow routing rules" -""" +pytestmark = [pytest.mark.layer("server"), pytest.mark.small] def _write_minimal_script(scripts_dir: Path, name: str = "test-script") -> Path: @@ -91,8 +61,8 @@ class TestLoadRecipeTools: """Tests for kitchen-gated load_recipe tool.""" @pytest.fixture(autouse=True) - def _ensure_ctx(self, tool_ctx): - """Ensure server context is initialized (gate open by default).""" + def _ensure_ctx(self, tool_ctx_kitchen_open): + """Ensure server context is initialized with gate open.""" # SS2 @pytest.mark.anyio @@ -165,7 +135,7 @@ async def test_load_recipe_parse_failure_is_logged_and_surfaced( "autoskillit.recipe._api.run_semantic_rules", side_effect=ValueError("injected crash"), ), - patch("autoskillit.recipe._api._logger") as mock_logger, + patch("autoskillit.recipe._api.logger") as mock_logger, ): result = json.loads(await load_recipe(name="test")) @@ -225,8 +195,8 @@ class TestLoadRecipeExceptionHandling: """CC-1: Outer except in load_recipe must catch anticipated exceptions only.""" @pytest.fixture(autouse=True) - def _setup_ctx(self, tool_ctx): - """Initialize ToolContext so load_recipe can call _get_config().""" + def _setup_ctx(self, tool_ctx_kitchen_open): + """Initialize ToolContext with gate open so load_recipe can call _get_config().""" @pytest.mark.anyio async def test_yaml_error_surfaces_as_suggestion( @@ -239,7 +209,7 @@ async def test_yaml_error_surfaces_as_suggestion( recipes_dir = tmp_path / ".autoskillit" / "recipes" recipes_dir.mkdir(parents=True) (recipes_dir / "test.yaml").write_text("name: test\n") - with patch("autoskillit.recipe._api.load_yaml", side_effect=YAMLError("bad yaml")): + with patch("autoskillit.recipe._api._load_recipe_dict", side_effect=YAMLError("bad yaml")): result = json.loads(await load_recipe(name="test")) assert "error" not in result assert any( @@ -330,7 +300,7 @@ class TestMigrationSuppression: # SUP1 @pytest.mark.anyio async def test_outdated_version_not_in_suggestions_when_suppressed( - self, tmp_path, monkeypatch, tool_ctx + self, tmp_path, monkeypatch, tool_ctx_kitchen_open ): """SUP1: outdated-recipe-version absent when recipe is suppressed; headless not called.""" from autoskillit.config import MigrationConfig @@ -339,7 +309,9 @@ async def test_outdated_version_not_in_suggestions_when_suppressed( scripts_dir = tmp_path / ".autoskillit" / "recipes" _write_minimal_script(scripts_dir, "test-script") - tool_ctx.config = AutomationConfig(migration=MigrationConfig(suppressed=["test-script"])) + tool_ctx_kitchen_open.config = AutomationConfig( + migration=MigrationConfig(suppressed=["test-script"]) + ) mock_headless = AsyncMock( return_value=SkillResult( @@ -365,17 +337,19 @@ async def test_outdated_version_not_in_suggestions_when_suppressed( # SUP4 @pytest.mark.anyio async def test_validate_always_includes_outdated_version_regardless_of_suppression( - self, tmp_path, tool_ctx + self, tmp_path, tool_ctx_kitchen_open ): """SUP4: validate_recipe includes outdated-script-version even when suppressed.""" from autoskillit.config import MigrationConfig - from autoskillit.server.tools_recipe import validate_recipe + from autoskillit.server.tools.tools_recipe import validate_recipe script = tmp_path / "test-script.yaml" - script.write_text(_MINIMAL_SCRIPT_YAML) + script.write_text(_MINIMAL_SCRIPT_YAML + 'autoskillit_version: "0.0.1"\n') # Even with script suppressed in config, validate_recipe does not filter - tool_ctx.config = AutomationConfig(migration=MigrationConfig(suppressed=["test-script"])) + tool_ctx_kitchen_open.config = AutomationConfig( + migration=MigrationConfig(suppressed=["test-script"]) + ) result = json.loads(await validate_recipe(script_path=str(script))) assert "findings" in result @@ -398,7 +372,7 @@ async def test_apply_triage_gate_second_call_skips_triage( import copy from autoskillit.recipe.staleness_cache import read_staleness_cache - from autoskillit.server.helpers import _apply_triage_gate + from autoskillit.server._misc import _apply_triage_gate monkeypatch.chdir(tmp_path) recipes_dir = tmp_path / ".autoskillit" / "recipes" @@ -463,8 +437,8 @@ class TestLoadRecipeReadOnly: """P4: load_recipe is strictly read-only — no migration, no contract card generation.""" @pytest.fixture(autouse=True) - def _ensure_ctx(self, tool_ctx): - """Ensure server context is initialized (gate open by default).""" + def _ensure_ctx(self, tool_ctx_kitchen_open): + """Ensure server context is initialized with gate open.""" @pytest.mark.anyio async def test_load_recipe_does_not_call_migration_engine(self, tmp_path, monkeypatch): @@ -494,300 +468,6 @@ async def test_load_recipe_does_not_auto_generate_contract_card(self, tmp_path, mock_gen.assert_not_called() -class TestMigrateRecipe: - """P4: migrate_recipe is a gated tool that runs migration engine and regenerates cards.""" - - @pytest.fixture(autouse=True) - def _open_kitchen(self, tool_ctx): - """migrate_recipe requires tool activation.""" - tool_ctx.gate = DefaultGateState(enabled=True) - - def _setup_migration_env( - self, - tmp_path, - monkeypatch, - tool_ctx, - *, - suppressed: list[str] | None = None, - ): - """Create directory structure, fake migration YAML, and config.""" - import autoskillit - import autoskillit.migration.loader as ml - from autoskillit.config import MigrationConfig - - monkeypatch.chdir(tmp_path) - recipes_dir = tmp_path / ".autoskillit" / "recipes" - recipes_dir.mkdir(parents=True) - recipe_path = recipes_dir / "test-script.yaml" - recipe_path.write_text(_MINIMAL_SCRIPT_YAML) - - installed_ver = autoskillit.__version__ - fake_mig_dir = tmp_path / "migrations" - fake_mig_dir.mkdir() - migration_yaml = ( - f"from_version: '0.0.0'\n" - f"to_version: '{installed_ver}'\n" - "description: Upgrade scripts\n" - "changes:\n" - " - id: add-summary-field\n" - " description: Scripts now require a summary field\n" - " instruction: Add summary field to your script\n" - ) - (fake_mig_dir / "0.0.0-migration.yaml").write_text(migration_yaml) - monkeypatch.setattr(ml, "_migrations_dir", lambda: fake_mig_dir) - - tool_ctx.config = AutomationConfig(migration=MigrationConfig(suppressed=suppressed or [])) - - temp_mig_dir = tmp_path / ".autoskillit" / "temp" / "migrations" - temp_mig_dir.mkdir(parents=True) - - migrated_content = _MINIMAL_SCRIPT_YAML + f"autoskillit_version: '{installed_ver}'\n" - return { - "recipe_path": recipe_path, - "temp_mig_dir": temp_mig_dir, - "migrated_content": migrated_content, - "installed_ver": installed_ver, - } - - def test_migrate_recipe_is_in_gated_tools(self): - """migrate_recipe is a gated tool.""" - assert "migrate_recipe" in GATED_TOOLS - - def test_migrate_recipe_not_in_ungated_tools(self): - """migrate_recipe is not an ungated tool.""" - assert "migrate_recipe" not in UNGATED_TOOLS - - @pytest.mark.anyio - async def test_migrate_recipe_requires_gate(self, tool_ctx): - """migrate_recipe returns gate_error when kitchen is closed.""" - tool_ctx.gate = DefaultGateState(enabled=False) - result = json.loads(await migrate_recipe(name="test")) - assert result["success"] is False - assert result["subtype"] == "gate_error" - - @pytest.mark.anyio - async def test_migrate_recipe_not_found(self, tmp_path, monkeypatch): - """migrate_recipe returns error for unknown recipe name.""" - monkeypatch.chdir(tmp_path) - result = json.loads(await migrate_recipe(name="nonexistent")) - assert "error" in result - assert "nonexistent" in result["error"] - - @pytest.mark.anyio - async def test_migrate_recipe_up_to_date(self, tmp_path, monkeypatch): # SRV-UPD-1 - """migrate_recipe returns up_to_date when no migrations applicable and contract fresh.""" - monkeypatch.chdir(tmp_path) - with ( - patch("autoskillit.migration.loader.applicable_migrations", return_value=[]), - patch("autoskillit.recipe.load_recipe_card", return_value={"skill_hashes": {}}), - patch("autoskillit.recipe.check_contract_staleness", return_value=[]), - ): - result = json.loads(await migrate_recipe(name="implementation")) - assert result.get("status") == "up_to_date" - - # LR1 - @pytest.mark.anyio - async def test_auto_migrates_outdated_recipe(self, tmp_path, monkeypatch, tool_ctx): - """LR1: When recipe version < installed, _run_headless_core is called once.""" - ctx = self._setup_migration_env(tmp_path, monkeypatch, tool_ctx) - (ctx["temp_mig_dir"] / "test-script.yaml").write_text(ctx["migrated_content"]) - - mock_headless = AsyncMock( - return_value=SkillResult( - success=True, - result="ok", - session_id="", - subtype="success", - is_error=False, - exit_code=0, - needs_retry=False, - retry_reason=RetryReason.NONE, - stderr="", - ) - ) - with ( - patch("autoskillit.execution.headless.run_headless_core", mock_headless), - patch("autoskillit.recipe.generate_recipe_card", return_value=None), - ): - result = json.loads(await migrate_recipe(name="test-script")) - - mock_headless.assert_awaited_once() - assert result.get("status") == "migrated" - assert "contracts_regenerated" in result - - # LR4 - @pytest.mark.anyio - async def test_clears_failure_record_after_successful_migration( - self, tmp_path, monkeypatch, tool_ctx - ): - """LR4: FailureStore.clear(name) is called when migration succeeds.""" - from autoskillit.migration.store import FailureStore, default_store_path - - ctx = self._setup_migration_env(tmp_path, monkeypatch, tool_ctx) - (ctx["temp_mig_dir"] / "test-script.yaml").write_text(ctx["migrated_content"]) - - store = FailureStore(default_store_path(tmp_path)) - store.record( - name="test-script", - file_path=ctx["recipe_path"], - file_type="recipe", - error="prior failure", - retries_attempted=1, - ) - assert store.has_failure("test-script") - - mock_headless = AsyncMock( - return_value=SkillResult( - success=True, - result="ok", - session_id="", - subtype="success", - is_error=False, - exit_code=0, - needs_retry=False, - retry_reason=RetryReason.NONE, - stderr="", - ) - ) - with ( - patch("autoskillit.execution.headless.run_headless_core", mock_headless), - patch("autoskillit.recipe.contracts.generate_recipe_card", return_value=None), - ): - await migrate_recipe(name="test-script") - - fresh_store = FailureStore(default_store_path(tmp_path)) - assert not fresh_store.has_failure("test-script") - - # LR5 - @pytest.mark.anyio - async def test_records_failure_when_migration_fails(self, tmp_path, monkeypatch, tool_ctx): - """LR5: When headless returns success=False, failure is recorded to failures.json.""" - from autoskillit.migration.store import FailureStore, default_store_path - - self._setup_migration_env(tmp_path, monkeypatch, tool_ctx) - - mock_headless = AsyncMock( - return_value=SkillResult( - success=False, - result="headless failed", - session_id="", - subtype="error", - is_error=True, - exit_code=1, - needs_retry=False, - retry_reason=RetryReason.NONE, - stderr="", - ) - ) - with patch("autoskillit.execution.headless.run_headless_core", mock_headless): - result = json.loads(await migrate_recipe(name="test-script")) - - assert "error" in result - store = FailureStore(default_store_path(tmp_path)) - assert store.has_failure("test-script") - - # LR7 - @pytest.mark.anyio - async def test_suppressed_recipe_not_migrated(self, tmp_path, monkeypatch, tool_ctx): - """LR7: When name in migration.suppressed, headless is never called.""" - self._setup_migration_env(tmp_path, monkeypatch, tool_ctx, suppressed=["test-script"]) - - mock_headless = AsyncMock( - return_value=SkillResult( - success=True, - result="ok", - session_id="", - subtype="success", - is_error=False, - exit_code=0, - needs_retry=False, - retry_reason=RetryReason.NONE, - stderr="", - ) - ) - with patch("autoskillit.execution.headless.run_headless_core", mock_headless): - result = json.loads(await migrate_recipe(name="test-script")) - - mock_headless.assert_not_called() - assert result.get("status") == "up_to_date" - - # LR8 - @pytest.mark.anyio - async def test_up_to_date_recipe_not_migrated(self, tmp_path, monkeypatch, tool_ctx): - """LR8: When applicable_migrations returns [], headless is never called.""" - import autoskillit - import autoskillit.migration.loader as ml - from autoskillit.config import MigrationConfig - - monkeypatch.chdir(tmp_path) - recipes_dir = tmp_path / ".autoskillit" / "recipes" - recipes_dir.mkdir(parents=True) - current_ver = autoskillit.__version__ - (recipes_dir / "test-script.yaml").write_text( - _MINIMAL_SCRIPT_YAML + f"autoskillit_version: '{current_ver}'\n" - ) - - empty_mig_dir = tmp_path / "migrations" - empty_mig_dir.mkdir() - monkeypatch.setattr(ml, "_migrations_dir", lambda: empty_mig_dir) - tool_ctx.config = AutomationConfig(migration=MigrationConfig(suppressed=[])) - - mock_headless = AsyncMock( - return_value=SkillResult( - success=True, - result="ok", - session_id="", - subtype="success", - is_error=False, - exit_code=0, - needs_retry=False, - retry_reason=RetryReason.NONE, - stderr="", - ) - ) - with ( - patch("autoskillit.execution.headless.run_headless_core", mock_headless), - patch("autoskillit.recipe.load_recipe_card", return_value={"skill_hashes": {}}), - patch("autoskillit.recipe.check_contract_staleness", return_value=[]), - ): - result = json.loads(await migrate_recipe(name="test-script")) - - mock_headless.assert_not_called() - assert result.get("status") == "up_to_date" - - # SRV-NEW-1 - @pytest.mark.anyio - async def test_migrate_recipe_regenerates_stale_contract( - self, tmp_path, monkeypatch, tool_ctx - ): - """migrate_recipe with version migration also regenerates stale contracts.""" - ctx = self._setup_migration_env(tmp_path, monkeypatch, tool_ctx) - (ctx["temp_mig_dir"] / "test-script.yaml").write_text(ctx["migrated_content"]) - - mock_headless = AsyncMock( - return_value=SkillResult( - success=True, - result="ok", - session_id="", - subtype="success", - is_error=False, - exit_code=0, - needs_retry=False, - retry_reason=RetryReason.NONE, - stderr="", - ) - ) - with ( - patch("autoskillit.execution.headless.run_headless_core", mock_headless), - patch("autoskillit.recipe.load_recipe_card", return_value=None), - patch("autoskillit.recipe.generate_recipe_card", return_value={}), - ): - result = json.loads(await migrate_recipe(name="test-script")) - - assert result.get("status") == "migrated" - assert result.get("contracts_regenerated") == ["test-script"] - - class TestLoadRecipeDiagram: """Tests for diagram field in load_recipe responses (DG-12 through DG-15).""" @@ -801,7 +481,9 @@ def _setup_project_recipe(self, tmp_path: Path, monkeypatch) -> Path: # DG-12 @pytest.mark.anyio - async def test_load_recipe_response_has_diagram_key(self, tmp_path, monkeypatch, tool_ctx): + async def test_load_recipe_response_has_diagram_key( + self, tmp_path, monkeypatch, tool_ctx_kitchen_open + ): """DG-12: load_recipe response always contains a 'diagram' key.""" self._setup_project_recipe(tmp_path, monkeypatch) result = json.loads(await load_recipe(name="my-recipe")) @@ -810,7 +492,7 @@ async def test_load_recipe_response_has_diagram_key(self, tmp_path, monkeypatch, # DG-13 @pytest.mark.anyio async def test_load_recipe_diagram_none_when_not_generated( - self, tmp_path, monkeypatch, tool_ctx + self, tmp_path, monkeypatch, tool_ctx_kitchen_open ): """DG-13: diagram is None when no diagram file exists.""" self._setup_project_recipe(tmp_path, monkeypatch) @@ -832,3 +514,32 @@ async def test_load_recipe_no_ctx_returns_error(monkeypatch): monkeypatch.setattr(_state_mod, "_ctx", None) result = json.loads(await load_recipe(name="anything")) assert "error" in result + + +# --------------------------------------------------------------------------- +# ingredients_only parameter +# --------------------------------------------------------------------------- + + +class TestLoadRecipeIngredientsOnly: + """load_recipe(ingredients_only=True) strips content, preserves metadata.""" + + @pytest.fixture(autouse=True) + def _ensure_ctx(self, tool_ctx_kitchen_open): + """Ensure server context is initialized with gate open.""" + + @pytest.mark.anyio + async def test_load_recipe_ingredients_only_strips_content(self, tmp_path, monkeypatch): + """load_recipe(name=X, ingredients_only=True) must omit content from result.""" + monkeypatch.chdir(tmp_path) + recipes_dir = tmp_path / ".autoskillit" / "recipes" + recipes_dir.mkdir(parents=True) + (recipes_dir / "test.yaml").write_text( + "name: test\ndescription: Test recipe\nsteps:\n" + " done:\n action: stop\n message: Done\n" + ) + result = json.loads(await load_recipe(name="test", ingredients_only=True)) + assert "content" not in result + assert "orchestration_rules" not in result + assert "stop_step_semantics" not in result + assert "suggestions" in result diff --git a/tests/server/test_tools_migrate_recipe.py b/tests/server/test_tools_migrate_recipe.py new file mode 100644 index 0000000000..6f487f1f6f --- /dev/null +++ b/tests/server/test_tools_migrate_recipe.py @@ -0,0 +1,311 @@ +"""Tests for autoskillit server migrate_recipe tool.""" + +from __future__ import annotations + +import json +from unittest.mock import AsyncMock, patch + +import pytest + +from autoskillit.config import AutomationConfig +from autoskillit.core import SkillResult +from autoskillit.core.types import RetryReason +from autoskillit.pipeline.gate import GATED_TOOLS, UNGATED_TOOLS, DefaultGateState +from autoskillit.server.tools.tools_recipe import migrate_recipe +from tests.server._helpers import _MINIMAL_SCRIPT_YAML + +pytestmark = [pytest.mark.layer("server"), pytest.mark.small] + + +class TestMigrateRecipe: + """P4: migrate_recipe is a gated tool that runs migration engine and regenerates cards.""" + + @pytest.fixture(autouse=True) + def _open_kitchen(self, tool_ctx): + """migrate_recipe requires tool activation.""" + tool_ctx.gate = DefaultGateState(enabled=True) + + def _setup_migration_env( + self, + tmp_path, + monkeypatch, + tool_ctx, + *, + suppressed: list[str] | None = None, + ): + """Create directory structure, fake migration YAML, and config.""" + import autoskillit + import autoskillit.migration.loader as ml + from autoskillit.config import MigrationConfig + + monkeypatch.chdir(tmp_path) + recipes_dir = tmp_path / ".autoskillit" / "recipes" + recipes_dir.mkdir(parents=True) + recipe_path = recipes_dir / "test-script.yaml" + recipe_path.write_text(_MINIMAL_SCRIPT_YAML + 'autoskillit_version: "0.0.0"\n') + + installed_ver = autoskillit.__version__ + fake_mig_dir = tmp_path / "migrations" + fake_mig_dir.mkdir() + migration_yaml = ( + f"from_version: '0.0.0'\n" + f"to_version: '{installed_ver}'\n" + "description: Upgrade scripts\n" + "changes:\n" + " - id: add-summary-field\n" + " description: Scripts now require a summary field\n" + " instruction: Add summary field to your script\n" + ) + (fake_mig_dir / "0.0.0-migration.yaml").write_text(migration_yaml) + monkeypatch.setattr(ml, "_migrations_dir", lambda: fake_mig_dir) + + tool_ctx.config = AutomationConfig(migration=MigrationConfig(suppressed=suppressed or [])) + + temp_mig_dir = tmp_path / ".autoskillit" / "temp" / "migrations" + temp_mig_dir.mkdir(parents=True) + + migrated_content = _MINIMAL_SCRIPT_YAML + f"autoskillit_version: '{installed_ver}'\n" + return { + "recipe_path": recipe_path, + "temp_mig_dir": temp_mig_dir, + "migrated_content": migrated_content, + "installed_ver": installed_ver, + } + + def test_migrate_recipe_is_in_gated_tools(self): + """migrate_recipe is a gated tool.""" + assert "migrate_recipe" in GATED_TOOLS + + def test_migrate_recipe_not_in_ungated_tools(self): + """migrate_recipe is not an ungated tool.""" + assert "migrate_recipe" not in UNGATED_TOOLS + + @pytest.mark.anyio + async def test_migrate_recipe_requires_gate(self, tool_ctx): + """migrate_recipe returns gate_error when kitchen is closed.""" + tool_ctx.gate = DefaultGateState(enabled=False) + result = json.loads(await migrate_recipe(name="test")) + assert result["success"] is False + assert result["subtype"] == "gate_error" + + @pytest.mark.anyio + async def test_migrate_recipe_not_found(self, tmp_path, monkeypatch): + """migrate_recipe returns error for unknown recipe name.""" + monkeypatch.chdir(tmp_path) + result = json.loads(await migrate_recipe(name="nonexistent")) + assert "error" in result + assert "nonexistent" in result["error"] + + @pytest.mark.anyio + async def test_migrate_recipe_up_to_date(self, tmp_path, monkeypatch): # SRV-UPD-1 + """migrate_recipe returns up_to_date when no migrations applicable and contract fresh.""" + monkeypatch.chdir(tmp_path) + with ( + patch("autoskillit.migration.loader.applicable_migrations", return_value=[]), + patch("autoskillit.recipe.load_recipe_card", return_value={"skill_hashes": {}}), + patch("autoskillit.recipe.check_contract_staleness", return_value=[]), + ): + result = json.loads(await migrate_recipe(name="implementation")) + assert result.get("status") == "up_to_date" + + # LR1 + @pytest.mark.anyio + async def test_auto_migrates_outdated_recipe(self, tmp_path, monkeypatch, tool_ctx): + """LR1: When recipe version < installed, _run_headless_core is called once.""" + ctx = self._setup_migration_env(tmp_path, monkeypatch, tool_ctx) + (ctx["temp_mig_dir"] / "test-script.yaml").write_text(ctx["migrated_content"]) + + mock_headless = AsyncMock( + return_value=SkillResult( + success=True, + result="ok", + session_id="", + subtype="success", + is_error=False, + exit_code=0, + needs_retry=False, + retry_reason=RetryReason.NONE, + stderr="", + ) + ) + with ( + patch("autoskillit.execution.headless.run_headless_core", mock_headless), + patch("autoskillit.recipe.generate_recipe_card", return_value=None), + ): + result = json.loads(await migrate_recipe(name="test-script")) + + mock_headless.assert_awaited_once() + assert result.get("status") == "migrated" + assert "contracts_regenerated" in result + + # LR4 + @pytest.mark.anyio + async def test_clears_failure_record_after_successful_migration( + self, tmp_path, monkeypatch, tool_ctx + ): + """LR4: FailureStore.clear(name) is called when migration succeeds.""" + from autoskillit.migration.store import FailureStore, default_store_path + + ctx = self._setup_migration_env(tmp_path, monkeypatch, tool_ctx) + (ctx["temp_mig_dir"] / "test-script.yaml").write_text(ctx["migrated_content"]) + + store = FailureStore(default_store_path(tmp_path)) + store.record( + name="test-script", + file_path=ctx["recipe_path"], + file_type="recipe", + error="prior failure", + retries_attempted=1, + ) + assert store.has_failure("test-script") + + mock_headless = AsyncMock( + return_value=SkillResult( + success=True, + result="ok", + session_id="", + subtype="success", + is_error=False, + exit_code=0, + needs_retry=False, + retry_reason=RetryReason.NONE, + stderr="", + ) + ) + with ( + patch("autoskillit.execution.headless.run_headless_core", mock_headless), + patch("autoskillit.recipe.contracts.generate_recipe_card", return_value=None), + ): + await migrate_recipe(name="test-script") + + fresh_store = FailureStore(default_store_path(tmp_path)) + assert not fresh_store.has_failure("test-script") + + # LR5 + @pytest.mark.anyio + async def test_records_failure_when_migration_fails(self, tmp_path, monkeypatch, tool_ctx): + """LR5: When headless returns success=False, failure is recorded to failures.json.""" + from autoskillit.migration.store import FailureStore, default_store_path + + self._setup_migration_env(tmp_path, monkeypatch, tool_ctx) + + mock_headless = AsyncMock( + return_value=SkillResult( + success=False, + result="headless failed", + session_id="", + subtype="error", + is_error=True, + exit_code=1, + needs_retry=False, + retry_reason=RetryReason.NONE, + stderr="", + ) + ) + with patch("autoskillit.execution.headless.run_headless_core", mock_headless): + result = json.loads(await migrate_recipe(name="test-script")) + + assert "error" in result + store = FailureStore(default_store_path(tmp_path)) + assert store.has_failure("test-script") + + # LR7 + @pytest.mark.anyio + async def test_suppressed_recipe_not_migrated(self, tmp_path, monkeypatch, tool_ctx): + """LR7: When name in migration.suppressed, headless is never called.""" + self._setup_migration_env(tmp_path, monkeypatch, tool_ctx, suppressed=["test-script"]) + + mock_headless = AsyncMock( + return_value=SkillResult( + success=True, + result="ok", + session_id="", + subtype="success", + is_error=False, + exit_code=0, + needs_retry=False, + retry_reason=RetryReason.NONE, + stderr="", + ) + ) + with patch("autoskillit.execution.headless.run_headless_core", mock_headless): + result = json.loads(await migrate_recipe(name="test-script")) + + mock_headless.assert_not_called() + assert result.get("status") == "up_to_date" + + # LR8 + @pytest.mark.anyio + async def test_up_to_date_recipe_not_migrated(self, tmp_path, monkeypatch, tool_ctx): + """LR8: When applicable_migrations returns [], headless is never called.""" + import autoskillit + import autoskillit.migration.loader as ml + from autoskillit.config import MigrationConfig + + monkeypatch.chdir(tmp_path) + recipes_dir = tmp_path / ".autoskillit" / "recipes" + recipes_dir.mkdir(parents=True) + current_ver = autoskillit.__version__ + (recipes_dir / "test-script.yaml").write_text( + _MINIMAL_SCRIPT_YAML + f"autoskillit_version: '{current_ver}'\n" + ) + + empty_mig_dir = tmp_path / "migrations" + empty_mig_dir.mkdir() + monkeypatch.setattr(ml, "_migrations_dir", lambda: empty_mig_dir) + tool_ctx.config = AutomationConfig(migration=MigrationConfig(suppressed=[])) + + mock_headless = AsyncMock( + return_value=SkillResult( + success=True, + result="ok", + session_id="", + subtype="success", + is_error=False, + exit_code=0, + needs_retry=False, + retry_reason=RetryReason.NONE, + stderr="", + ) + ) + with ( + patch("autoskillit.execution.headless.run_headless_core", mock_headless), + patch("autoskillit.recipe.load_recipe_card", return_value={"skill_hashes": {}}), + patch("autoskillit.recipe.check_contract_staleness", return_value=[]), + ): + result = json.loads(await migrate_recipe(name="test-script")) + + mock_headless.assert_not_called() + assert result.get("status") == "up_to_date" + + # SRV-NEW-1 + @pytest.mark.anyio + async def test_migrate_recipe_regenerates_stale_contract( + self, tmp_path, monkeypatch, tool_ctx + ): + """migrate_recipe with version migration also regenerates stale contracts.""" + ctx = self._setup_migration_env(tmp_path, monkeypatch, tool_ctx) + (ctx["temp_mig_dir"] / "test-script.yaml").write_text(ctx["migrated_content"]) + + mock_headless = AsyncMock( + return_value=SkillResult( + success=True, + result="ok", + session_id="", + subtype="success", + is_error=False, + exit_code=0, + needs_retry=False, + retry_reason=RetryReason.NONE, + stderr="", + ) + ) + with ( + patch("autoskillit.execution.headless.run_headless_core", mock_headless), + patch("autoskillit.recipe.load_recipe_card", return_value=None), + patch("autoskillit.recipe.generate_recipe_card", return_value={}), + ): + result = json.loads(await migrate_recipe(name="test-script")) + + assert result.get("status") == "migrated" + assert result.get("contracts_regenerated") == ["test-script"] diff --git a/tests/server/test_tools_pr_ops.py b/tests/server/test_tools_pr_ops.py index 3687cd53c4..8500dd326b 100644 --- a/tests/server/test_tools_pr_ops.py +++ b/tests/server/test_tools_pr_ops.py @@ -8,7 +8,7 @@ import pytest from autoskillit.pipeline.gate import DefaultGateState -from autoskillit.server.tools_pr_ops import ( +from autoskillit.server.tools.tools_pr_ops import ( _close_issues_sequentially, _map_api_reviews, _map_pr_view_reviews, @@ -16,6 +16,8 @@ get_pr_reviews, ) +pytestmark = [pytest.mark.layer("server"), pytest.mark.small] + # --------------------------------------------------------------------------- # Pure helper functions # --------------------------------------------------------------------------- @@ -54,10 +56,14 @@ def test_map_pr_view_reviews_empty() -> None: async def test_close_issues_sequentially_all_succeed() -> None: """All gh calls return rc=0 → closed=[1,2], failed=[].""" with patch( - "autoskillit.server.tools_pr_ops._run_subprocess", + "autoskillit.server.tools.tools_pr_ops._run_subprocess", new=AsyncMock(return_value=(0, "", "")), ): - closed, failed = await _close_issues_sequentially([1, 2], "closing", "/tmp") + with patch( + "autoskillit.server.tools.tools_pr_ops.asyncio.sleep", + new_callable=AsyncMock, + ): + closed, failed = await _close_issues_sequentially([1, 2], "closing", "/tmp") assert closed == [1, 2] assert failed == [] @@ -74,15 +80,44 @@ async def _mock_subprocess(cmd, *, cwd, timeout): return (0, "", "") if n == 0 else (1, "", "not found") with patch( - "autoskillit.server.tools_pr_ops._run_subprocess", + "autoskillit.server.tools.tools_pr_ops._run_subprocess", new=_mock_subprocess, ): - closed, failed = await _close_issues_sequentially([1, 2], "", "/tmp") + with patch( + "autoskillit.server.tools.tools_pr_ops.asyncio.sleep", + new_callable=AsyncMock, + ): + closed, failed = await _close_issues_sequentially([1, 2], "", "/tmp") assert closed == [1] assert failed == [2] +# --------------------------------------------------------------------------- +# T5 — _close_issues_sequentially inserts inter-call delay +# --------------------------------------------------------------------------- + + +@pytest.mark.anyio +async def test_close_issues_sequentially_delays_between_calls() -> None: + """Each gh issue close call is preceded by asyncio.sleep(1) after the first.""" + with patch( + "autoskillit.server.tools.tools_pr_ops._run_subprocess", + new=AsyncMock(return_value=(0, "", "")), + ): + with patch( + "autoskillit.server.tools.tools_pr_ops.asyncio.sleep", + new_callable=AsyncMock, + ) as mock_sleep: + closed, failed = await _close_issues_sequentially([1, 2, 3], "", "/tmp") + + assert closed == [1, 2, 3] + assert failed == [] + assert mock_sleep.call_count == 2 + for call in mock_sleep.call_args_list: + assert call[0][0] == 1 + + # --------------------------------------------------------------------------- # MCP tool handlers # --------------------------------------------------------------------------- @@ -98,11 +133,13 @@ async def test_get_pr_reviews_gate_closed(tool_ctx) -> None: @pytest.mark.anyio -async def test_get_pr_reviews_with_repo_success(tool_ctx, monkeypatch: pytest.MonkeyPatch) -> None: +async def test_get_pr_reviews_with_repo_success( + tool_ctx_kitchen_open, monkeypatch: pytest.MonkeyPatch +) -> None: """repo provided → gh api repos/{repo}/pulls/123/reviews path used.""" api_response = json.dumps([{"user": {"login": "alice"}, "state": "APPROVED", "body": ""}]) with patch( - "autoskillit.server.tools_pr_ops._run_subprocess", + "autoskillit.server.tools.tools_pr_ops._run_subprocess", new=AsyncMock(return_value=(0, api_response, "")), ): result = json.loads(await get_pr_reviews(123, "/tmp", repo="owner/repo")) @@ -113,14 +150,14 @@ async def test_get_pr_reviews_with_repo_success(tool_ctx, monkeypatch: pytest.Mo @pytest.mark.anyio async def test_get_pr_reviews_without_repo_success( - tool_ctx, monkeypatch: pytest.MonkeyPatch + tool_ctx_kitchen_open, monkeypatch: pytest.MonkeyPatch ) -> None: """repo='' → gh pr view 123 --json reviews path used.""" pr_view_response = json.dumps( {"reviews": [{"author": {"login": "bob"}, "state": "CHANGES_REQUESTED", "body": ""}]} ) with patch( - "autoskillit.server.tools_pr_ops._run_subprocess", + "autoskillit.server.tools.tools_pr_ops._run_subprocess", new=AsyncMock(return_value=(0, pr_view_response, "")), ): result = json.loads(await get_pr_reviews(123, "/tmp", repo="")) @@ -129,10 +166,12 @@ async def test_get_pr_reviews_without_repo_success( @pytest.mark.anyio -async def test_get_pr_reviews_gh_failure(tool_ctx, monkeypatch: pytest.MonkeyPatch) -> None: +async def test_get_pr_reviews_gh_failure( + tool_ctx_kitchen_open, monkeypatch: pytest.MonkeyPatch +) -> None: """gh returns rc=1 → {"success": False, "error": ...}.""" with patch( - "autoskillit.server.tools_pr_ops._run_subprocess", + "autoskillit.server.tools.tools_pr_ops._run_subprocess", new=AsyncMock(return_value=(1, "", "repository not found")), ): result = json.loads(await get_pr_reviews(123, "/tmp", repo="owner/repo")) @@ -151,13 +190,19 @@ async def test_bulk_close_issues_gate_closed(tool_ctx) -> None: @pytest.mark.anyio -async def test_bulk_close_issues_all_closed(tool_ctx, monkeypatch: pytest.MonkeyPatch) -> None: +async def test_bulk_close_issues_all_closed( + tool_ctx_kitchen_open, monkeypatch: pytest.MonkeyPatch +) -> None: """All succeed → {"closed": [1, 2, 3], "failed": []}.""" with patch( - "autoskillit.server.tools_pr_ops._run_subprocess", + "autoskillit.server.tools.tools_pr_ops._run_subprocess", new=AsyncMock(return_value=(0, "", "")), ): - result = json.loads(await bulk_close_issues([1, 2, 3], "", "/tmp")) + with patch( + "autoskillit.server.tools.tools_pr_ops.asyncio.sleep", + new_callable=AsyncMock, + ): + result = json.loads(await bulk_close_issues([1, 2, 3], "", "/tmp")) assert result["closed"] == [1, 2, 3] assert result["failed"] == [] @@ -165,7 +210,7 @@ async def test_bulk_close_issues_all_closed(tool_ctx, monkeypatch: pytest.Monkey @pytest.mark.anyio async def test_bulk_close_issues_partial_failure( - tool_ctx, monkeypatch: pytest.MonkeyPatch + tool_ctx_kitchen_open, monkeypatch: pytest.MonkeyPatch ) -> None: """Mixed outcomes → correct closed/failed split.""" call_count = {"n": 0} @@ -176,10 +221,14 @@ async def _mock_subprocess(cmd, *, cwd, timeout): return (0, "", "") if n % 2 == 0 else (1, "", "error") with patch( - "autoskillit.server.tools_pr_ops._run_subprocess", + "autoskillit.server.tools.tools_pr_ops._run_subprocess", new=_mock_subprocess, ): - result = json.loads(await bulk_close_issues([1, 2, 3], "", "/tmp")) + with patch( + "autoskillit.server.tools.tools_pr_ops.asyncio.sleep", + new_callable=AsyncMock, + ): + result = json.loads(await bulk_close_issues([1, 2, 3], "", "/tmp")) assert 1 in result["closed"] assert 2 in result["failed"] diff --git a/tests/server/test_tools_recipe.py b/tests/server/test_tools_recipe.py index 4c9df342cb..9f711a4d2d 100644 --- a/tests/server/test_tools_recipe.py +++ b/tests/server/test_tools_recipe.py @@ -7,7 +7,10 @@ import pytest -from autoskillit.server.tools_recipe import validate_recipe +from autoskillit.server.tools.tools_recipe import list_recipes as list_recipes_tool +from autoskillit.server.tools.tools_recipe import validate_recipe + +pytestmark = [pytest.mark.layer("server"), pytest.mark.small] def _extract_docstring_sections(desc: str) -> dict[str, str]: @@ -96,8 +99,8 @@ class TestValidateRecipeTool: """Tests for kitchen-gated validate_recipe tool.""" @pytest.fixture(autouse=True) - def _ensure_ctx(self, tool_ctx): - """Ensure server context is initialized (gate open by default).""" + def _ensure_ctx(self, tool_ctx_kitchen_open): + """Ensure server context is initialized with gate open.""" # VS1 @pytest.mark.anyio @@ -249,15 +252,15 @@ class TestMigrationSuggestions: """MSUG2: validate_recipe surfaces migration warnings.""" @pytest.fixture(autouse=True) - def _ensure_ctx(self, tool_ctx): - """Ensure server context is initialized (gate open by default).""" + def _ensure_ctx(self, tool_ctx_kitchen_open): + """Ensure server context is initialized with gate open.""" # MSUG2 @pytest.mark.anyio async def test_validate_always_includes_outdated_version(self, tmp_path): """MSUG2: validate_recipe always includes outdated-script-version in semantic results.""" script = tmp_path / "test-script.yaml" - script.write_text(_MINIMAL_SCRIPT_YAML) + script.write_text(_MINIMAL_SCRIPT_YAML + 'autoskillit_version: "0.0.1"\n') result = json.loads(await validate_recipe(script_path=str(script))) assert "findings" in result @@ -279,6 +282,9 @@ async def _get_tools(self) -> dict: from autoskillit.server import mcp + mcp.enable(tags={"fleet"}) + mcp.enable(tags={"fleet-dispatch"}) + async with Client(mcp) as client: tools = await client.list_tools() return {t.name: t for t in tools} @@ -424,6 +430,9 @@ async def _get_tools(self) -> dict: from autoskillit.server import mcp + mcp.enable(tags={"fleet"}) + mcp.enable(tags={"fleet-dispatch"}) + async with Client(mcp) as client: tools = await client.list_tools() return {t.name: t for t in tools} @@ -444,13 +453,48 @@ async def test_description_documents_run_skill_failure(self, kitchen_enabled): # P5F2-T3 @pytest.mark.anyio -async def test_validate_recipe_no_recipes_returns_error(tool_ctx, tmp_path): +async def test_validate_recipe_no_recipes_returns_error(tool_ctx_kitchen_open, tmp_path): """validate_recipe returns invalid JSON when recipes is not configured.""" - tool_ctx.recipes = None + tool_ctx_kitchen_open.recipes = None result = json.loads(await validate_recipe(script_path=str(tmp_path / "x.yaml"))) assert result.get("valid") is False +# T7: list_recipes MCP tool hides campaign when fleet disabled +@pytest.mark.anyio +@pytest.mark.feature("fleet") +async def test_list_recipes_mcp_tool_hides_campaign_when_fleet_disabled( + tool_ctx, tmp_path, monkeypatch +): + """list_recipes MCP tool must exclude campaign recipes when fleet feature is disabled.""" + from pathlib import Path + + recipe_dir = tmp_path / ".autoskillit" / "recipes" + recipe_dir.mkdir(parents=True) + (recipe_dir / "my-campaign.yaml").write_text( + "name: my-campaign\ndescription: test\nkind: campaign\nsteps: {}\n" + ) + tool_ctx.config.features["fleet"] = False + monkeypatch.setattr(Path, "cwd", lambda: tmp_path) + raw = await list_recipes_tool() + result = json.loads(raw) + recipe_names = [r["name"] for r in result.get("recipes", [])] + assert "my-campaign" not in recipe_names, ( + "Campaign recipe must not appear when fleet feature is disabled" + ) + + +@pytest.mark.anyio +async def test_list_recipes_returns_error_string_when_context_missing() -> None: + """list_recipes must return an error message string (not []) when tool_ctx is None.""" + from autoskillit.server.tools.tools_recipe import list_recipes + + result = await list_recipes() + assert isinstance(result, str) + parsed = json.loads(result) + assert "error" in parsed or parsed.get("is_error") is True + + # P5F2-T4 (import hygiene check) def test_tools_recipe_does_not_import_raw_ctx(): """tools_recipe.py must not import _ctx directly from server._state.""" @@ -458,7 +502,7 @@ def test_tools_recipe_does_not_import_raw_ctx(): import pathlib source = ( - pathlib.Path(__file__).parents[2] / "src/autoskillit/server/tools_recipe.py" + pathlib.Path(__file__).parents[2] / "src/autoskillit/server/tools/tools_recipe.py" ).read_text() tree = ast.parse(source) for node in ast.walk(tree): diff --git a/tests/server/test_tools_report_bug.py b/tests/server/test_tools_report_bug.py new file mode 100644 index 0000000000..4250ba27aa --- /dev/null +++ b/tests/server/test_tools_report_bug.py @@ -0,0 +1,777 @@ +"""Tests for report_bug MCP tool handler and supporting helpers.""" + +from __future__ import annotations + +import json +from pathlib import Path +from unittest.mock import AsyncMock, MagicMock + +import anyio +import pytest + +from autoskillit.config import AutomationConfig +from autoskillit.core import SkillResult +from autoskillit.core.types import RetryReason +from autoskillit.server._misc import _extract_block +from autoskillit.server.tools.tools_github import ( + _FINGERPRINT_END, + _FINGERPRINT_START, + _parse_fingerprint, + report_bug, +) +from autoskillit.server.tools.tools_issue_lifecycle import ( + _ENRICH_RESULT_END, + _ENRICH_RESULT_START, + _PREPARE_RESULT_END, + _PREPARE_RESULT_START, + _parse_enrich_result, + _parse_prepare_result, +) +from tests.server._helpers import _skill_fail, _skill_ok + +pytestmark = [pytest.mark.layer("server"), pytest.mark.small] + +# --------------------------------------------------------------------------- +# _parse_fingerprint unit tests +# --------------------------------------------------------------------------- + + +def test_parse_fingerprint_present(): + report = ( + "Some preamble\n" + f"{_FINGERPRINT_START}\n" + "KeyError in recipe/validator.py: missing ingredient ref\n" + f"{_FINGERPRINT_END}\n" + "Report written to /tmp/report.md" + ) + assert _parse_fingerprint(report) == "KeyError in recipe/validator.py: missing ingredient ref" + + +def test_parse_fingerprint_missing_returns_none(): + assert _parse_fingerprint("No fingerprint block here") is None + + +def test_parse_fingerprint_empty_block_returns_none(): + report = f"{_FINGERPRINT_START}\n{_FINGERPRINT_END}\n" + assert _parse_fingerprint(report) is None + + +def test_parse_fingerprint_first_nonempty_line(): + """Only the first non-empty line inside the block is returned.""" + report = ( + f"{_FINGERPRINT_START}\n" + "\n" + " TypeError in execution/headless.py: runner=None \n" + "extra line\n" + f"{_FINGERPRINT_END}\n" + ) + assert _parse_fingerprint(report) == "TypeError in execution/headless.py: runner=None" + + +# --------------------------------------------------------------------------- +# report_bug config defaults +# --------------------------------------------------------------------------- + + +def test_report_bug_config_defaults(): + cfg = AutomationConfig() + assert cfg.report_bug.timeout == 600 + assert cfg.report_bug.model is None + assert cfg.report_bug.report_dir is None + assert cfg.report_bug.github_filing is True + assert "autoreported" in cfg.report_bug.github_labels + assert "bug" in cfg.report_bug.github_labels + + +def test_github_config_defaults(): + config = AutomationConfig() + assert config.github.token is None + assert config.github.default_repo is None + + +# --------------------------------------------------------------------------- +# TestReportBugTool +# --------------------------------------------------------------------------- + + +class TestReportBugTool: + @pytest.mark.anyio + async def test_report_bug_failure_includes_session_id_and_stderr( + self, tool_ctx_kitchen_open, tmp_path + ): + """Blocking failure response must include session_id and stderr for diagnosis.""" + tool_ctx_kitchen_open.config.report_bug.report_dir = str(tmp_path / "bug-reports") + tool_ctx_kitchen_open.config.report_bug.github_filing = False + + mock_executor = AsyncMock() + mock_executor.run.return_value = SkillResult( + success=False, + result="", + session_id="fail-session-id", + subtype="missing_completion_marker", + is_error=True, + exit_code=1, + needs_retry=False, + retry_reason=RetryReason.NONE, + stderr="Claude crashed", + ) + tool_ctx_kitchen_open.executor = mock_executor + + result = json.loads(await report_bug("some error", str(tmp_path), severity="blocking")) + + assert result["success"] is False + assert result["session_id"] == "fail-session-id" + assert result["stderr"] == "Claude crashed" + + @pytest.mark.anyio + async def test_report_bug_passes_expected_output_patterns_to_executor( + self, tool_ctx_kitchen_open, tmp_path + ): + """output_pattern_resolver is consulted and patterns are passed to executor.run().""" + tool_ctx_kitchen_open.config.report_bug.report_dir = str(tmp_path / "bug-reports") + tool_ctx_kitchen_open.config.report_bug.github_filing = False + + mock_executor = AsyncMock() + mock_executor.run.return_value = SkillResult( + success=True, + result="## Report\nfindings", + session_id="sid", + subtype="success", + is_error=False, + exit_code=0, + needs_retry=False, + retry_reason=RetryReason.NONE, + stderr="", + ) + tool_ctx_kitchen_open.executor = mock_executor + tool_ctx_kitchen_open.output_pattern_resolver = lambda cmd: ["---bug-fingerprint---"] + + await report_bug("error ctx", str(tmp_path), severity="blocking") + + call_kwargs = mock_executor.run.call_args.kwargs + assert call_kwargs.get("expected_output_patterns") == ["---bug-fingerprint---"] + + +# --------------------------------------------------------------------------- +# _extract_block unit tests +# --------------------------------------------------------------------------- + + +def test_extract_block_returns_lines_within_delimiters(): + text = "preamble\n---start---\nline1\nline2\n---end---\npostamble" + assert _extract_block(text, "---start---", "---end---") == ["line1", "line2"] + + +def test_extract_block_no_start_returns_empty(): + assert _extract_block("no delimiters here", "---start---", "---end---") == [] + + +def test_extract_block_no_end_returns_empty(): + # end delimiter absent — no complete block + text = "---start---\nline1\nline2" + assert _extract_block(text, "---start---", "---end---") == [] + + +def test_extract_block_empty_block_returns_empty_list(): + text = "---start---\n---end---" + assert _extract_block(text, "---start---", "---end---") == [] + + +def test_extract_block_preserves_whitespace_in_lines(): + text = "---start---\n indented\n---end---" + assert _extract_block(text, "---start---", "---end---") == [" indented"] + + +# --------------------------------------------------------------------------- +# _parse_prepare_result unit tests +# --------------------------------------------------------------------------- + + +def test_parse_prepare_result_valid_json(): + payload = '{"success": true, "issue_url": "https://github.com/x/y/issues/1"}' + text = f"{_PREPARE_RESULT_START}\n{payload}\n{_PREPARE_RESULT_END}" + result = _parse_prepare_result(text) + assert result == {"success": True, "issue_url": "https://github.com/x/y/issues/1"} + + +def test_parse_prepare_result_no_block(): + result = _parse_prepare_result("no block here") + assert result == {"success": False, "error": "no result block found"} + + +def test_parse_prepare_result_invalid_json(): + text = f"{_PREPARE_RESULT_START}\nnot-json\n{_PREPARE_RESULT_END}" + result = _parse_prepare_result(text) + assert result == {"success": False, "error": "result block contained invalid JSON"} + + +# --------------------------------------------------------------------------- +# _parse_enrich_result unit tests +# --------------------------------------------------------------------------- + + +def test_parse_enrich_result_valid_json(): + payload = '{"enriched": [42], "skipped_already_enriched": []}' + text = f"{_ENRICH_RESULT_START}\n{payload}\n{_ENRICH_RESULT_END}" + result = _parse_enrich_result(text) + assert result == {"enriched": [42], "skipped_already_enriched": []} + + +def test_parse_enrich_result_no_block(): + result = _parse_enrich_result("no block here") + assert result == {"success": False, "error": "no result block found"} + + +def test_parse_enrich_result_invalid_json(): + text = f"{_ENRICH_RESULT_START}\nnot-json\n{_ENRICH_RESULT_END}" + result = _parse_enrich_result(text) + assert result == {"success": False, "error": "result block contained invalid JSON"} + + +# --------------------------------------------------------------------------- +# report_bug gate tests +# --------------------------------------------------------------------------- + + +@pytest.mark.anyio +async def test_report_bug_gate_closed(tool_ctx, tmp_path): + tool_ctx.gate.disable() + result = json.loads(await report_bug("some error", str(tmp_path))) + assert result["success"] is False + assert "not enabled" in result["result"].lower() or "gate" in result["result"].lower() + + +@pytest.mark.anyio +async def test_report_bug_no_executor(tool_ctx_kitchen_open, tmp_path): + tool_ctx_kitchen_open.executor = None + result = json.loads(await report_bug("error ctx", str(tmp_path))) + assert result["success"] is False + assert "executor" in result["error"].lower() + + +# --------------------------------------------------------------------------- +# Blocking mode +# --------------------------------------------------------------------------- + + +@pytest.mark.anyio +async def test_report_bug_blocking_success(tool_ctx_kitchen_open, tmp_path): + """Blocking mode awaits the session and returns status=complete.""" + tool_ctx_kitchen_open.config.report_bug.report_dir = str(tmp_path / "bug-reports") + tool_ctx_kitchen_open.config.report_bug.github_filing = False + + mock_executor = AsyncMock() + mock_executor.run.return_value = _skill_ok("## Report\nroot cause found") + tool_ctx_kitchen_open.executor = mock_executor + + result = json.loads(await report_bug("KeyError in foo", str(tmp_path), severity="blocking")) + + assert result["success"] is True + assert result["status"] == "complete" + assert "report" in result + assert "report_path" in result + mock_executor.run.assert_awaited_once() + + +@pytest.mark.anyio +async def test_report_bug_blocking_failure_propagated(tool_ctx_kitchen_open, tmp_path): + """If the headless session fails, status=failed is returned.""" + tool_ctx_kitchen_open.config.report_bug.report_dir = str(tmp_path / "bug-reports") + tool_ctx_kitchen_open.config.report_bug.github_filing = False + + mock_executor = AsyncMock() + mock_executor.run.return_value = _skill_fail() + tool_ctx_kitchen_open.executor = mock_executor + + result = json.loads(await report_bug("crash here", str(tmp_path), severity="blocking")) + + assert result["success"] is False + assert result["status"] == "failed" + + +@pytest.mark.anyio +async def test_report_bug_blocking_writes_report_file(tool_ctx_kitchen_open, tmp_path): + """The report text must be written to the resolved report_path.""" + report_dir = tmp_path / "rpts" + tool_ctx_kitchen_open.config.report_bug.report_dir = str(report_dir) + tool_ctx_kitchen_open.config.report_bug.github_filing = False + + mock_executor = AsyncMock() + mock_executor.run.return_value = _skill_ok("# Bug Report\nfoo bar") + tool_ctx_kitchen_open.executor = mock_executor + + result = json.loads(await report_bug("err", str(tmp_path), severity="blocking")) + + report_path = Path(result["report_path"]) + assert report_path.exists() + assert "Bug Report" in report_path.read_text() + + +# --------------------------------------------------------------------------- +# Non-blocking mode +# --------------------------------------------------------------------------- + + +@pytest.mark.anyio +async def test_report_bug_non_blocking_returns_immediately(tool_ctx_kitchen_open, tmp_path): + """Non-blocking mode must return dispatched before the session completes.""" + tool_ctx_kitchen_open.config.report_bug.report_dir = str(tmp_path / "bug-reports") + tool_ctx_kitchen_open.config.report_bug.github_filing = False + + ready = anyio.Event() + + async def slow_run(*args, **kwargs): + await ready.wait() # blocks until test signals + return _skill_ok() + + mock_executor = MagicMock() + mock_executor.run = slow_run + tool_ctx_kitchen_open.executor = mock_executor + + result = json.loads(await report_bug("error ctx", str(tmp_path), severity="non_blocking")) + + assert result["success"] is True + assert result["status"] == "dispatched" + assert "report_path" in result + + # Let the background task finish cleanly. + ready.set() + await anyio.sleep(0) + + +@pytest.mark.anyio +async def test_report_bug_non_blocking_default_severity(tool_ctx_kitchen_open, tmp_path): + """The default severity is non_blocking.""" + tool_ctx_kitchen_open.config.report_bug.report_dir = str(tmp_path / "bug-reports") + tool_ctx_kitchen_open.config.report_bug.github_filing = False + + mock_executor = AsyncMock() + mock_executor.run.return_value = _skill_ok() + tool_ctx_kitchen_open.executor = mock_executor + + result = json.loads(await report_bug("err", str(tmp_path))) + assert result["status"] == "dispatched" + + +# --------------------------------------------------------------------------- +# Non-blocking outcome tests (supervised background task) +# --------------------------------------------------------------------------- + + +@pytest.mark.anyio +async def test_report_bug_non_blocking_outcome_writes_report_file(tool_ctx_kitchen_open, tmp_path): + """After the background task completes, report_path must exist with content.""" + tool_ctx_kitchen_open.config.report_bug.report_dir = str(tmp_path / "bug-reports") + tool_ctx_kitchen_open.config.report_bug.github_filing = False + + mock_executor = AsyncMock() + mock_executor.run.return_value = _skill_ok("# Bug Report\nroot cause: missing guard") + tool_ctx_kitchen_open.executor = mock_executor + + result = json.loads( + await report_bug("KeyError in foo", str(tmp_path), severity="non_blocking") + ) + assert result["status"] == "dispatched" + + report_path = Path(result["report_path"]) + await tool_ctx_kitchen_open.background.drain() + + assert report_path.exists(), "report_path must exist after background task completes" + assert "Bug Report" in report_path.read_text() + + +@pytest.mark.anyio +async def test_report_bug_non_blocking_writes_status_file_on_success( + tool_ctx_kitchen_open, tmp_path +): + """A status.json file must be written with status='complete' after successful dispatch.""" + tool_ctx_kitchen_open.config.report_bug.report_dir = str(tmp_path / "bug-reports") + tool_ctx_kitchen_open.config.report_bug.github_filing = False + + mock_executor = AsyncMock() + mock_executor.run.return_value = _skill_ok("# Report\nfoo") + tool_ctx_kitchen_open.executor = mock_executor + + result = json.loads(await report_bug("err", str(tmp_path), severity="non_blocking")) + report_path = Path(result["report_path"]) + + # Before task runs: pending status file should exist + status_path = report_path.with_suffix(".status.json") + assert status_path.exists(), "status file must be written synchronously on dispatch" + pending = json.loads(status_path.read_text()) + assert pending["status"] == "pending" + + # After task completes: status should update to complete + await tool_ctx_kitchen_open.background.drain() + + data = json.loads(status_path.read_text()) + assert data["status"] == "complete" + assert "completed_at" in data + + +@pytest.mark.anyio +async def test_report_bug_non_blocking_writes_status_file_on_failure( + tool_ctx_kitchen_open, tmp_path +): + """When the headless session fails, status.json must reflect the failure.""" + tool_ctx_kitchen_open.config.report_bug.report_dir = str(tmp_path / "bug-reports") + tool_ctx_kitchen_open.config.report_bug.github_filing = False + + mock_executor = AsyncMock() + mock_executor.run.return_value = _skill_fail() + tool_ctx_kitchen_open.executor = mock_executor + + result = json.loads(await report_bug("crash here", str(tmp_path), severity="non_blocking")) + status_path = Path(result["report_path"]).with_suffix(".status.json") + + await tool_ctx_kitchen_open.background.drain() + + data = json.loads(status_path.read_text()) + assert data["status"] == "failed" + assert "completed_at" in data + + +@pytest.mark.anyio +async def test_report_bug_non_blocking_executor_raises_is_observed( + tool_ctx_kitchen_open, tmp_path +): + """If executor.run() raises, the exception must be logged — not silently dropped.""" + + tool_ctx_kitchen_open.config.report_bug.report_dir = str(tmp_path / "bug-reports") + tool_ctx_kitchen_open.config.report_bug.github_filing = False + + mock_executor = AsyncMock() + mock_executor.run.side_effect = RuntimeError("executor exploded") + tool_ctx_kitchen_open.executor = mock_executor + + result = json.loads(await report_bug("error ctx", str(tmp_path), severity="non_blocking")) + assert result["status"] == "dispatched" + + await tool_ctx_kitchen_open.background.drain() + + # The exception must be captured and logged — not silently dropped + status_path = Path(result["report_path"]).with_suffix(".status.json") + assert status_path.exists(), "status file must exist even when executor raises" + data = json.loads(status_path.read_text()) + assert data["status"] == "failed" + assert "executor exploded" in data.get("error", "") + + +@pytest.mark.anyio +async def test_report_bug_no_pending_tasks_after_completion(tool_ctx, tmp_path): + """After background task completes, no tasks remain in the supervisor's pending set.""" + tool_ctx.config.report_bug.report_dir = str(tmp_path / "bug-reports") + tool_ctx.config.report_bug.github_filing = False + + mock_executor = AsyncMock() + mock_executor.run.return_value = _skill_ok() + tool_ctx.executor = mock_executor + + await report_bug("err", str(tmp_path), severity="non_blocking") + + await tool_ctx.background.drain() + + assert tool_ctx.background.pending_count == 0 + + +# --------------------------------------------------------------------------- +# GitHub filing — blocking mode (easier to assert synchronously) +# --------------------------------------------------------------------------- + + +@pytest.mark.anyio +async def test_report_bug_creates_github_issue_on_no_duplicate(tool_ctx_kitchen_open, tmp_path): + """When no matching issue is found, create_issue is called.""" + tool_ctx_kitchen_open.config.report_bug.report_dir = str(tmp_path / "bug-reports") + tool_ctx_kitchen_open.config.report_bug.github_filing = True + tool_ctx_kitchen_open.config.github.default_repo = "owner/repo" + + report_with_fp = ( + f"{_FINGERPRINT_START}\n" + "KeyError in recipe/validator.py: missing ref\n" + f"{_FINGERPRINT_END}\n" + "## Bug Report\ndetails" + ) + mock_executor = AsyncMock() + mock_executor.run.return_value = _skill_ok(report_with_fp) + tool_ctx_kitchen_open.executor = mock_executor + + mock_gh = AsyncMock() + mock_gh.has_token = True + mock_gh.search_issues.return_value = {"success": True, "total_count": 0, "items": []} + mock_gh.create_issue.return_value = { + "success": True, + "issue_number": 99, + "url": "https://github.com/owner/repo/issues/99", + } + tool_ctx_kitchen_open.github_client = mock_gh + + result = json.loads(await report_bug("KeyError crash", str(tmp_path), severity="blocking")) + + assert result["success"] is True + mock_gh.search_issues.assert_awaited_once() + mock_gh.create_issue.assert_awaited_once() + assert result["github"]["duplicate"] is False + assert result["github"]["issue_url"] == "https://github.com/owner/repo/issues/99" + + +@pytest.mark.anyio +async def test_report_bug_updates_duplicate_issue_body(tool_ctx_kitchen_open, tmp_path): + """When a matching issue exists and error_context is new, update_issue_body is called.""" + tool_ctx_kitchen_open.config.report_bug.report_dir = str(tmp_path / "bug-reports") + tool_ctx_kitchen_open.config.report_bug.github_filing = True + tool_ctx_kitchen_open.config.github.default_repo = "owner/repo" + + mock_executor = AsyncMock() + mock_executor.run.return_value = _skill_ok( + "## Report\n" + _FINGERPRINT_START + "\nfp\n" + _FINGERPRINT_END + ) + tool_ctx_kitchen_open.executor = mock_executor + + mock_gh = AsyncMock() + mock_gh.has_token = True + mock_gh.search_issues.return_value = { + "success": True, + "total_count": 1, + "items": [ + { + "number": 7, + "title": "fp", + "html_url": "https://github.com/owner/repo/issues/7", + "body": "Original body — no error_context here", + "state": "open", + } + ], + } + mock_gh.update_issue_body.return_value = { + "success": True, + "issue_url": "https://github.com/owner/repo/issues/7", + } + tool_ctx_kitchen_open.github_client = mock_gh + + result = json.loads( + await report_bug("brand new error text", str(tmp_path), severity="blocking") + ) + + assert result["success"] is True + mock_gh.update_issue_body.assert_awaited_once() + assert result["github"]["duplicate"] is True + assert result["github"]["body_updated"] is True + + +@pytest.mark.anyio +async def test_report_bug_skips_comment_if_already_present(tool_ctx_kitchen_open, tmp_path): + """If error_context is already in the issue body, no comment is posted.""" + tool_ctx_kitchen_open.config.report_bug.report_dir = str(tmp_path / "bug-reports") + tool_ctx_kitchen_open.config.report_bug.github_filing = True + tool_ctx_kitchen_open.config.github.default_repo = "owner/repo" + + error_ctx = "exact error text already filed" + mock_executor = AsyncMock() + mock_executor.run.return_value = _skill_ok() + tool_ctx_kitchen_open.executor = mock_executor + + mock_gh = AsyncMock() + mock_gh.has_token = True + mock_gh.search_issues.return_value = { + "success": True, + "total_count": 1, + "items": [ + { + "number": 3, + "title": "fp", + "html_url": "https://github.com/owner/repo/issues/3", + "body": f"body contains {error_ctx} already", + "state": "open", + } + ], + } + tool_ctx_kitchen_open.github_client = mock_gh + + result = json.loads(await report_bug(error_ctx, str(tmp_path), severity="blocking")) + + mock_gh.update_issue_body.assert_not_awaited() + assert result["github"]["body_updated"] is False + + +@pytest.mark.anyio +async def test_report_bug_skips_github_if_no_token(tool_ctx_kitchen_open, tmp_path): + tool_ctx_kitchen_open.config.report_bug.report_dir = str(tmp_path / "bug-reports") + tool_ctx_kitchen_open.config.report_bug.github_filing = True + + mock_executor = AsyncMock() + mock_executor.run.return_value = _skill_ok() + tool_ctx_kitchen_open.executor = mock_executor + + mock_gh = AsyncMock() + mock_gh.has_token = False + tool_ctx_kitchen_open.github_client = mock_gh + + result = json.loads(await report_bug("err", str(tmp_path), severity="blocking")) + + assert result["success"] is True + mock_gh.search_issues.assert_not_awaited() + assert result["github"]["skipped"] is True + assert result["github"]["reason"] == "no_token" + + +@pytest.mark.anyio +async def test_report_bug_skips_github_if_no_default_repo(tool_ctx_kitchen_open, tmp_path): + tool_ctx_kitchen_open.config.report_bug.report_dir = str(tmp_path / "bug-reports") + tool_ctx_kitchen_open.config.report_bug.github_filing = True + tool_ctx_kitchen_open.config.github.default_repo = None + + mock_executor = AsyncMock() + mock_executor.run.return_value = _skill_ok() + tool_ctx_kitchen_open.executor = mock_executor + + mock_gh = AsyncMock() + mock_gh.has_token = True + tool_ctx_kitchen_open.github_client = mock_gh + + result = json.loads(await report_bug("err", str(tmp_path), severity="blocking")) + + mock_gh.search_issues.assert_not_awaited() + assert result["github"]["skipped"] is True + + +@pytest.mark.anyio +async def test_report_bug_github_filing_disabled(tool_ctx_kitchen_open, tmp_path): + """github_filing=false must skip all GitHub calls.""" + tool_ctx_kitchen_open.config.report_bug.report_dir = str(tmp_path / "bug-reports") + tool_ctx_kitchen_open.config.report_bug.github_filing = False + tool_ctx_kitchen_open.config.github.default_repo = "owner/repo" + + mock_executor = AsyncMock() + mock_executor.run.return_value = _skill_ok() + tool_ctx_kitchen_open.executor = mock_executor + + mock_gh = AsyncMock() + mock_gh.has_token = True + tool_ctx_kitchen_open.github_client = mock_gh + + result = json.loads(await report_bug("err", str(tmp_path), severity="blocking")) + + mock_gh.search_issues.assert_not_awaited() + assert result["github"] == {} + + +@pytest.mark.anyio +async def test_report_bug_blocking_github_client_raises_does_not_propagate( + tool_ctx_kitchen_open, tmp_path +): + """If the GitHub client raises unexpectedly, the error must be captured in the github dict.""" + tool_ctx_kitchen_open.config.report_bug.report_dir = str(tmp_path / "bug-reports") + tool_ctx_kitchen_open.config.report_bug.github_filing = True + tool_ctx_kitchen_open.config.github.default_repo = "owner/repo" + + mock_executor = AsyncMock() + mock_executor.run.return_value = _skill_ok( + "# Report\n" + _FINGERPRINT_START + "\nfp1\n" + _FINGERPRINT_END + ) + tool_ctx_kitchen_open.executor = mock_executor + + mock_gh = MagicMock() + mock_gh.has_token = True + mock_gh.search_issues = AsyncMock(side_effect=RuntimeError("network failure")) + tool_ctx_kitchen_open.github_client = mock_gh + + # Must not raise — exception is captured in github dict + result = json.loads(await report_bug("err", str(tmp_path), severity="blocking")) + assert result["success"] is True # session succeeded + assert result["github"].get("skipped") is True + assert "unexpected error" in result["github"].get("reason", "") + assert "network failure" in result["github"].get("reason", "") + + +# --------------------------------------------------------------------------- +# model-as-profile resolution in report_bug +# --------------------------------------------------------------------------- + + +@pytest.mark.anyio +async def test_report_bug_model_as_profile_resolves_provider( + tool_ctx_kitchen_open, tmp_path, monkeypatch +): + tool_ctx_kitchen_open.config.report_bug.report_dir = str(tmp_path / "bug-reports") + tool_ctx_kitchen_open.config.report_bug.github_filing = False + + monkeypatch.setattr("autoskillit.core.is_feature_enabled", lambda *a, **kw: True) + monkeypatch.setattr( + "autoskillit.server._guards._resolve_model_as_profile", + lambda *a: ( + "MiniMax-M2.7", + "minimax", + {"ANTHROPIC_BASE_URL": "https://api.minimax.chat/v1"}, + ), + ) + + mock_executor = AsyncMock() + mock_executor.run.return_value = _skill_ok("report text") + tool_ctx_kitchen_open.executor = mock_executor + + result = json.loads( + await report_bug("error ctx", str(tmp_path), model="minimax", severity="blocking") + ) + + assert result["success"] is True + call_kwargs = mock_executor.run.call_args.kwargs + assert call_kwargs.get("model") == "MiniMax-M2.7" + assert call_kwargs.get("provider_extras") == { + "ANTHROPIC_BASE_URL": "https://api.minimax.chat/v1" + } + assert call_kwargs.get("profile_name") == "minimax" + + +@pytest.mark.anyio +async def test_report_bug_model_as_profile_disabled_when_feature_off( + tool_ctx_kitchen_open, tmp_path, monkeypatch +): + tool_ctx_kitchen_open.config.report_bug.report_dir = str(tmp_path / "bug-reports") + tool_ctx_kitchen_open.config.report_bug.github_filing = False + + monkeypatch.setattr("autoskillit.core.is_feature_enabled", lambda *a, **kw: False) + + mock_executor = AsyncMock() + mock_executor.run.return_value = _skill_ok("report text") + tool_ctx_kitchen_open.executor = mock_executor + + result = json.loads( + await report_bug("error ctx", str(tmp_path), model="minimax", severity="blocking") + ) + + assert result["success"] is True + call_kwargs = mock_executor.run.call_args.kwargs + assert call_kwargs.get("model") == "minimax" + assert call_kwargs.get("provider_extras") is None + + +@pytest.mark.anyio +async def test_report_bug_config_model_as_profile(tool_ctx_kitchen_open, tmp_path, monkeypatch): + tool_ctx_kitchen_open.config.report_bug.report_dir = str(tmp_path / "bug-reports") + tool_ctx_kitchen_open.config.report_bug.github_filing = False + tool_ctx_kitchen_open.config.report_bug.model = "minimax" + + map_calls: list[str] = [] + + def fake_resolve(model_value, *a, **kw): + map_calls.append(model_value) + return ("MiniMax-M2.7", "minimax", {"ANTHROPIC_BASE_URL": "https://api.minimax.chat/v1"}) + + monkeypatch.setattr("autoskillit.core.is_feature_enabled", lambda *a, **kw: True) + monkeypatch.setattr("autoskillit.server._guards._resolve_model_as_profile", fake_resolve) + + mock_executor = AsyncMock() + mock_executor.run.return_value = _skill_ok("report text") + tool_ctx_kitchen_open.executor = mock_executor + + result = json.loads(await report_bug("error ctx", str(tmp_path), severity="blocking")) + + assert result["success"] is True + assert map_calls == ["minimax"] + call_kwargs = mock_executor.run.call_args.kwargs + assert call_kwargs.get("model") == "MiniMax-M2.7" + assert call_kwargs.get("provider_extras") == { + "ANTHROPIC_BASE_URL": "https://api.minimax.chat/v1" + } + assert call_kwargs.get("profile_name") == "minimax" diff --git a/tests/server/test_tools_run_cmd.py b/tests/server/test_tools_run_cmd.py index b12dfe17b1..e040a4490d 100644 --- a/tests/server/test_tools_run_cmd.py +++ b/tests/server/test_tools_run_cmd.py @@ -8,39 +8,41 @@ import pytest import structlog.testing -from autoskillit.server.helpers import _run_subprocess -from autoskillit.server.tools_execution import run_cmd, run_python +from autoskillit.server._subprocess import _run_subprocess +from autoskillit.server.tools.tools_execution import run_cmd, run_python from tests.conftest import _make_result, _make_timeout_result +pytestmark = [pytest.mark.layer("server"), pytest.mark.small] + class TestRunCmd: """T1, T2: run_cmd executes commands and returns exit code semantics.""" @pytest.mark.anyio - async def test_successful_command(self, tool_ctx): - tool_ctx.runner.push(_make_result(0, "hello\n", "")) + async def test_successful_command(self, tool_ctx_kitchen_open): + tool_ctx_kitchen_open.runner.push(_make_result(0, "hello\n", "")) result = json.loads(await run_cmd(cmd="echo hello", cwd="/tmp")) assert result["success"] is True assert result["exit_code"] == 0 assert "hello" in result["stdout"] - assert len(tool_ctx.runner.call_args_list) == 1 - assert tool_ctx.runner.call_args_list[0][0] == ["bash", "-c", "echo hello"] + assert len(tool_ctx_kitchen_open.runner.call_args_list) == 1 + assert tool_ctx_kitchen_open.runner.call_args_list[0][0] == ["bash", "-c", "echo hello"] @pytest.mark.anyio - async def test_failing_command(self, tool_ctx): - tool_ctx.runner.push(_make_result(1, "", "error")) + async def test_failing_command(self, tool_ctx_kitchen_open): + tool_ctx_kitchen_open.runner.push(_make_result(1, "", "error")) result = json.loads(await run_cmd(cmd="false", cwd="/tmp")) assert result["success"] is False assert result["exit_code"] == 1 @pytest.mark.anyio - async def test_custom_timeout(self, tool_ctx): - tool_ctx.runner.push(_make_result(0, "", "")) - await run_cmd(cmd="sleep 1", cwd="/tmp", timeout=30) + async def test_custom_timeout(self, tool_ctx_kitchen_open): + tool_ctx_kitchen_open.runner.push(_make_result(0, "", "")) + await run_cmd(cmd="echo timeout_test", cwd="/tmp", timeout=30) - assert tool_ctx.runner.call_args_list[-1][2] == 30.0 + assert tool_ctx_kitchen_open.runner.call_args_list[-1][2] == 30.0 class TestRunSubprocessDelegatesToManaged: @@ -63,12 +65,12 @@ async def test_timeout_returns_minus_one(self, tool_ctx): class TestProcessRunnerResult: - """_process_runner_result shared helper lives in server.helpers.""" + """_process_runner_result shared helper lives in server._subprocess.""" def test_normal_exit_preserves_fields(self): from autoskillit.core import TerminationReason from autoskillit.execution.process import SubprocessResult - from autoskillit.server.helpers import _process_runner_result + from autoskillit.server._subprocess import _process_runner_result result = SubprocessResult( returncode=0, @@ -85,7 +87,7 @@ def test_normal_exit_preserves_fields(self): def test_timed_out_returns_minus_one_with_message(self): from autoskillit.core import TerminationReason from autoskillit.execution.process import SubprocessResult - from autoskillit.server.helpers import _process_runner_result + from autoskillit.server._subprocess import _process_runner_result result = SubprocessResult( returncode=-1, @@ -101,7 +103,7 @@ def test_timed_out_returns_minus_one_with_message(self): assert "5" in stderr -@pytest.mark.usefixtures("tool_ctx") +@pytest.mark.usefixtures("tool_ctx_kitchen_open") class TestRunPython: """run_python tool: import, call, timeout, async support.""" @@ -199,3 +201,218 @@ async def _hang(**_kw: object) -> None: assert any("timed out" in log.get("event", "").lower() for log in logs), ( f"Expected 'timed out' in warning event, got: {logs}" ) + + +class TestRunCmdSleepInterception: + """Sleep commands are intercepted and converted to asyncio.sleep.""" + + @pytest.mark.anyio + async def test_python_sleep_intercepted(self, tool_ctx_kitchen_open): + result = json.loads( + await run_cmd(cmd='python3 -c "import time; time.sleep(0)"', cwd="/tmp") + ) + assert result == {"success": True, "exit_code": 0, "stdout": "", "stderr": ""} + assert len(tool_ctx_kitchen_open.runner.call_args_list) == 0 + + @pytest.mark.anyio + async def test_bare_sleep_intercepted(self, tool_ctx_kitchen_open): + result = json.loads(await run_cmd(cmd="sleep 0", cwd="/tmp")) + assert result["success"] is True + assert len(tool_ctx_kitchen_open.runner.call_args_list) == 0 + + @pytest.mark.anyio + async def test_python3_single_quotes_intercepted(self, tool_ctx_kitchen_open): + result = json.loads( + await run_cmd(cmd="python3 -c 'import time; time.sleep(0)'", cwd="/tmp") + ) + assert result["success"] is True + assert len(tool_ctx_kitchen_open.runner.call_args_list) == 0 + + @pytest.mark.anyio + async def test_non_sleep_uses_subprocess(self, tool_ctx_kitchen_open): + tool_ctx_kitchen_open.runner.push(_make_result(0, "hello", "")) + await run_cmd(cmd="echo hello", cwd="/tmp") + assert len(tool_ctx_kitchen_open.runner.call_args_list) == 1 + assert tool_ctx_kitchen_open.runner.call_args_list[0][0] == ["bash", "-c", "echo hello"] + + @pytest.mark.anyio + async def test_decimal_seconds_intercepted(self, tool_ctx_kitchen_open): + result = json.loads( + await run_cmd(cmd='python3 -c "import time; time.sleep(0.0)"', cwd="/tmp") + ) + assert result["success"] is True + assert len(tool_ctx_kitchen_open.runner.call_args_list) == 0 + + @pytest.mark.anyio + async def test_compound_sleep_not_intercepted(self, tool_ctx_kitchen_open): + tool_ctx_kitchen_open.runner.push(_make_result(0, "", "")) + await run_cmd(cmd="echo before && sleep 10 && echo after", cwd="/tmp") + assert len(tool_ctx_kitchen_open.runner.call_args_list) == 1 + + @pytest.mark.anyio + async def test_step_name_timing_recorded(self, tool_ctx_kitchen_open): + await run_cmd(cmd="sleep 0", cwd="/tmp", step_name="quota_wait") + assert len(tool_ctx_kitchen_open.runner.call_args_list) == 0 + assert any( + e["step_name"] == "quota_wait" for e in tool_ctx_kitchen_open.timing_log.get_report() + ) + + +# ─── Type coercion tests (Step 1a) ─────────────────────────────────────────── + + +@pytest.mark.usefixtures("tool_ctx_kitchen_open") +class TestImportAndCallTypeCoercion: + """Test _import_and_call annotation-aware type coercion.""" + + @pytest.mark.anyio + async def test_int_coerced_to_str(self): + """int value for str-annotated param is coerced to str.""" + result = json.loads( + await run_python( + callable="tests.server._type_coercion_fixtures._str_only_param", + args={"value": 42}, + timeout=10, + ) + ) + assert result["success"] is True + assert result["result"]["value"] == "42" + + @pytest.mark.anyio + async def test_str_coerced_to_int(self): + """str value for int-annotated param is coerced to int.""" + result = json.loads( + await run_python( + callable="tests.server._type_coercion_fixtures._int_param", + args={"value": "7"}, + timeout=10, + ) + ) + assert result["success"] is True + assert result["result"]["value"] == 7 + + @pytest.mark.anyio + async def test_str_coerced_to_float(self): + """str value for float-annotated param is coerced to float.""" + result = json.loads( + await run_python( + callable="tests.server._type_coercion_fixtures._typed_callable", + args={"name": "test", "count": 1, "ratio": "3.14"}, + timeout=10, + ) + ) + assert result["success"] is True + assert result["result"]["ratio"] == 3.14 + + @pytest.mark.anyio + async def test_int_coerced_to_float(self): + """int value for float-annotated param is coerced to float.""" + result = json.loads( + await run_python( + callable="tests.server._type_coercion_fixtures._typed_callable", + args={"name": "test", "count": 1, "ratio": 5}, + timeout=10, + ) + ) + assert result["success"] is True + assert result["result"]["ratio"] == 5.0 + + @pytest.mark.anyio + async def test_correct_types_unchanged(self): + """Correct types pass through without coercion.""" + result = json.loads( + await run_python( + callable="tests.server._type_coercion_fixtures._typed_callable", + args={"name": "hello", "count": 7, "ratio": 2.5}, + timeout=10, + ) + ) + assert result["success"] is True + assert result["result"] == {"name": "hello", "count": 7, "ratio": 2.5} + + @pytest.mark.anyio + async def test_none_still_uses_default(self): + """None still triggers default substitution (existing behavior).""" + result = json.loads( + await run_python( + callable="tests.server._type_coercion_fixtures._str_optional_param", + args={"value": None}, + timeout=10, + ) + ) + assert result["success"] is True + assert result["result"]["value"] == "default" + + @pytest.mark.anyio + async def test_coercion_logs_warning(self): + """Coercion emits a structlog warning with type info.""" + with structlog.testing.capture_logs() as logs: + result = json.loads( + await run_python( + callable="tests.server._type_coercion_fixtures._str_only_param", + args={"value": 99}, + timeout=10, + ) + ) + assert result["success"] is True + coercion_logs = [log for log in logs if log.get("event") == "run_python type coerced"] + assert len(coercion_logs) == 1 + assert coercion_logs[0]["arg"] == "value" + assert coercion_logs[0]["from_type"] == "int" + assert coercion_logs[0]["to_type"] == "str" + + @pytest.mark.anyio + async def test_unconvertible_value_not_coerced(self): + """Non-numeric str for int param is not coerced, callable fails.""" + result = json.loads( + await run_python( + callable="tests.server._type_coercion_fixtures._int_param", + args={"value": "not_a_number"}, + timeout=10, + ) + ) + assert result["success"] is False + assert "AssertionError" in result["error"] + + @pytest.mark.anyio + async def test_sentinel_keys_stripped_from_args(self): + """run_python sentinel keys in args dict must be stripped before callable dispatch.""" + result = json.loads( + await run_python( + callable="tests.server._type_coercion_fixtures._str_only_param", + args={"value": "x", "timeout": 60, "callable": "bogus.path"}, + timeout=10, + ) + ) + assert result["success"] is True + assert result["result"]["value"] == "x" + + @pytest.mark.anyio + async def test_unrecognized_args_dropped_with_warning(self): + """_import_and_call should log a warning and drop args not in the callable's signature.""" + with structlog.testing.capture_logs() as logs: + result = json.loads( + await run_python( + callable="tests.server._type_coercion_fixtures._str_only_param", + args={"value": "x", "bogus": 42}, + timeout=10, + ) + ) + assert result["success"] is True + assert result["result"]["value"] == "x" + warning_logs = [log for log in logs if "bogus" in str(log.get("extra_args", []))] + assert len(warning_logs) >= 1 + + @pytest.mark.anyio + async def test_extra_args_forwarded_when_callable_accepts_kwargs(self): + """When the callable accepts **kwargs, extra args should be forwarded.""" + result = json.loads( + await run_python( + callable="tests.server._type_coercion_fixtures._kwargs_callable", + args={"name": "test", "extra": "y"}, + timeout=10, + ) + ) + assert result["success"] is True + assert result["result"]["name"] == "test" + assert result["result"]["extras"]["extra"] == "y" diff --git a/tests/server/test_tools_run_cmd_unit.py b/tests/server/test_tools_run_cmd_unit.py index 66fe01126a..3f07798203 100644 --- a/tests/server/test_tools_run_cmd_unit.py +++ b/tests/server/test_tools_run_cmd_unit.py @@ -9,9 +9,11 @@ import structlog.contextvars import structlog.testing -from autoskillit.server.tools_execution import run_cmd +from autoskillit.server.tools.tools_execution import run_cmd from tests.conftest import _make_result +pytestmark = [pytest.mark.layer("server"), pytest.mark.small] + class TestRunCmdObservability: """run_cmd binds structlog contextvars and calls ctx.info/ctx.error.""" @@ -24,9 +26,11 @@ def mock_ctx(self): return ctx @pytest.mark.anyio - async def test_run_cmd_binds_tool_contextvar_and_calls_ctx_info(self, tool_ctx, mock_ctx): + async def test_run_cmd_binds_tool_contextvar_and_calls_ctx_info( + self, tool_ctx_kitchen_open, mock_ctx + ): """run_cmd binds tool='run_cmd' contextvar and calls ctx.info on success.""" - tool_ctx.runner.push(_make_result(0, "ok\n", "")) + tool_ctx_kitchen_open.runner.push(_make_result(0, "ok\n", "")) with structlog.testing.capture_logs( processors=[structlog.contextvars.merge_contextvars] ) as logs: @@ -34,9 +38,11 @@ async def test_run_cmd_binds_tool_contextvar_and_calls_ctx_info(self, tool_ctx, assert any(entry.get("tool") == "run_cmd" for entry in logs) @pytest.mark.anyio - async def test_run_cmd_returns_failure_result_on_nonzero_exit(self, tool_ctx, mock_ctx): + async def test_run_cmd_returns_failure_result_on_nonzero_exit( + self, tool_ctx_kitchen_open, mock_ctx + ): """run_cmd reports failure (success=false) when subprocess exits non-zero.""" - tool_ctx.runner.push(_make_result(1, "", "err")) + tool_ctx_kitchen_open.runner.push(_make_result(1, "", "err")) result = json.loads(await run_cmd(cmd="false", cwd="/tmp", ctx=mock_ctx)) assert result["success"] is False assert result["exit_code"] == 1 @@ -46,9 +52,9 @@ class TestRunCmdTiming: """run_cmd accumulates wall-clock timing when step_name is provided.""" @pytest.mark.anyio - async def test_run_cmd_step_name_records_timing(self, tool_ctx): + async def test_run_cmd_step_name_records_timing(self, tool_ctx_kitchen_open): await run_cmd(cmd="echo hi", cwd="/tmp", step_name="clone") - report = tool_ctx.timing_log.get_report() + report = tool_ctx_kitchen_open.timing_log.get_report() assert len(report) == 1 assert report[0]["step_name"] == "clone" assert report[0]["total_seconds"] >= 0.0 @@ -64,25 +70,29 @@ class TestRunCmdRecording: """run_cmd threads SCENARIO_STEP_NAME into env kwarg for RecordingSubprocessRunner.""" @pytest.mark.anyio - async def test_run_cmd_with_step_name_passes_scenario_step_name_to_runner(self, tool_ctx): + async def test_run_cmd_with_step_name_passes_scenario_step_name_to_runner( + self, tool_ctx_kitchen_open + ): """run_cmd with step_name passes SCENARIO_STEP_NAME in env kwarg to ctx.runner.""" from autoskillit.execution.recording import SCENARIO_STEP_NAME_ENV - tool_ctx.runner.push(_make_result(0, "ok", "")) + tool_ctx_kitchen_open.runner.push(_make_result(0, "ok", "")) await run_cmd(cmd="echo hi", cwd="/tmp", step_name="setup") - call_kwargs = tool_ctx.runner.call_args_list[-1][3] + call_kwargs = tool_ctx_kitchen_open.runner.call_args_list[-1][3] assert call_kwargs.get("env", {}).get(SCENARIO_STEP_NAME_ENV) == "setup" @pytest.mark.anyio - async def test_run_cmd_with_step_name_preserves_parent_env(self, tool_ctx, monkeypatch): + async def test_run_cmd_with_step_name_preserves_parent_env( + self, tool_ctx_kitchen_open, monkeypatch + ): """run_cmd with step_name must not strip PATH/HOME from the child env.""" from autoskillit.execution.recording import SCENARIO_STEP_NAME_ENV monkeypatch.setenv("PATH", "/usr/bin:/usr/local/bin") monkeypatch.setenv("HOME", "/home/testuser") - tool_ctx.runner.push(_make_result(0, "ok", "")) + tool_ctx_kitchen_open.runner.push(_make_result(0, "ok", "")) await run_cmd(cmd="echo hi", cwd="/tmp", step_name="setup") - call_kwargs = tool_ctx.runner.call_args_list[-1][3] + call_kwargs = tool_ctx_kitchen_open.runner.call_args_list[-1][3] env = call_kwargs["env"] assert env is not None assert "PATH" in env, "run_cmd stripped PATH from child environment" @@ -90,30 +100,32 @@ async def test_run_cmd_with_step_name_preserves_parent_env(self, tool_ctx, monke assert env[SCENARIO_STEP_NAME_ENV] == "setup" @pytest.mark.anyio - async def test_run_cmd_without_step_name_passes_no_env(self, tool_ctx): + async def test_run_cmd_without_step_name_passes_no_env(self, tool_ctx_kitchen_open): """run_cmd without step_name passes env=None (no SCENARIO_STEP_NAME in env).""" from autoskillit.execution.recording import SCENARIO_STEP_NAME_ENV - tool_ctx.runner.push(_make_result(0, "", "")) + tool_ctx_kitchen_open.runner.push(_make_result(0, "", "")) await run_cmd(cmd="echo hi", cwd="/tmp") - call_kwargs = tool_ctx.runner.call_args_list[-1][3] + call_kwargs = tool_ctx_kitchen_open.runner.call_args_list[-1][3] env = call_kwargs.get("env") assert env is None or SCENARIO_STEP_NAME_ENV not in env @pytest.mark.anyio - async def test_run_cmd_with_step_name_records_non_session_step(self, tool_ctx, monkeypatch): + async def test_run_cmd_with_step_name_records_non_session_step( + self, tool_ctx_kitchen_open, monkeypatch + ): """End-to-end: run_cmd step_name → RecordingSubprocessRunner.record_non_session_step().""" from unittest.mock import Mock from autoskillit.execution.recording import RecordingSubprocessRunner - from tests.conftest import MockSubprocessRunner from tests.conftest import _make_result as _mr + from tests.fakes import MockSubprocessRunner mock_recorder = Mock() inner = MockSubprocessRunner() inner.push(_mr(0, "task output", "")) recording_runner = RecordingSubprocessRunner(recorder=mock_recorder, inner=inner) - tool_ctx.runner = recording_runner + tool_ctx_kitchen_open.runner = recording_runner await run_cmd(cmd="task install-worktree", cwd="/tmp", step_name="setup") @@ -129,8 +141,8 @@ async def test_run_cmd_without_step_name_skips_recording(self, tool_ctx, monkeyp from unittest.mock import Mock from autoskillit.execution.recording import RecordingSubprocessRunner - from tests.conftest import MockSubprocessRunner from tests.conftest import _make_result as _mr + from tests.fakes import MockSubprocessRunner mock_recorder = Mock() inner = MockSubprocessRunner() diff --git a/tests/server/test_tools_run_python.py b/tests/server/test_tools_run_python.py index a3bace81c1..b672b74cc7 100644 --- a/tests/server/test_tools_run_python.py +++ b/tests/server/test_tools_run_python.py @@ -9,7 +9,9 @@ import structlog.contextvars import structlog.testing -from autoskillit.server.tools_execution import run_python +from autoskillit.server.tools.tools_execution import run_python + +pytestmark = [pytest.mark.layer("server"), pytest.mark.small] class TestRunPythonObservability: @@ -23,7 +25,9 @@ def mock_ctx(self): return ctx @pytest.mark.anyio - async def test_run_python_binds_tool_contextvar_and_calls_ctx_info(self, tool_ctx, mock_ctx): + async def test_run_python_binds_tool_contextvar_and_calls_ctx_info( + self, tool_ctx_kitchen_open, mock_ctx + ): """run_python binds tool='run_python' contextvar and calls ctx.info on success.""" with structlog.testing.capture_logs( processors=[structlog.contextvars.merge_contextvars] diff --git a/tests/server/test_tools_run_skill_retry.py b/tests/server/test_tools_run_skill_retry.py index bf1c9babac..a68e84dc10 100644 --- a/tests/server/test_tools_run_skill_retry.py +++ b/tests/server/test_tools_run_skill_retry.py @@ -8,9 +8,11 @@ from autoskillit.core import SkillResult from autoskillit.core.types import RetryReason, TerminationReason -from autoskillit.server.tools_execution import run_skill +from autoskillit.server.tools.tools_execution import run_skill from tests.conftest import _make_result +pytestmark = [pytest.mark.layer("server"), pytest.mark.small] + # Deterministic UUID for tests that need to predict the per-invocation marker. _DETERMINISTIC_HEX = "a1b2c3d4e5f6a7b890123456" _DETERMINISTIC_MARKER = f"%%ORDER_UP::{_DETERMINISTIC_HEX[:8]}%%" @@ -30,27 +32,18 @@ class TestRunSkillRetryRemoved: def test_run_skill_retry_not_in_tools_execution(self): """run_skill_retry is not importable from tools_execution.""" - import autoskillit.server.tools_execution as module + import autoskillit.server.tools.tools_execution as module assert not hasattr(module, "run_skill_retry"), ( "run_skill_retry still exists in tools_execution — it should be removed" ) - def test_run_skill_retry_not_in_all(self): - """run_skill_retry is not in tools_execution.__all__ (if defined).""" - import autoskillit.server.tools_execution as module - - # tools_execution does not define __all__; verify via direct attribute check - assert not hasattr(module, "run_skill_retry"), ( - "run_skill_retry should not be a public attribute of tools_execution" - ) - class TestRunSkillSessionOutcome: """run_skill correctly classifies all Claude Code session outcomes.""" @pytest.mark.anyio - async def test_detects_max_turns_via_subtype(self, tool_ctx): + async def test_detects_max_turns_via_subtype(self, tool_ctx_kitchen_open): """error_max_turns in JSON output -> needs_retry=True, retry_reason=RESUME.""" stdout = json.dumps( { @@ -61,13 +54,14 @@ async def test_detects_max_turns_via_subtype(self, tool_ctx): "errors": ["Max turns reached"], } ) - tool_ctx.runner.push(_make_result(1, stdout, "")) + tool_ctx_kitchen_open.runner.push(_make_result(returncode=1)) # clone guard snapshot + tool_ctx_kitchen_open.runner.push(_make_result(1, stdout, "")) result = json.loads(await run_skill("/investigate plan.md", "/tmp")) assert result["needs_retry"] is True assert result["retry_reason"] == RetryReason.RESUME @pytest.mark.anyio - async def test_detects_context_limit(self, tool_ctx): + async def test_detects_context_limit(self, tool_ctx_kitchen_open): """'Prompt is too long' -> needs_retry=True, retry_reason='resume'.""" stdout = json.dumps( { @@ -78,7 +72,8 @@ async def test_detects_context_limit(self, tool_ctx): "session_id": "s1", } ) - tool_ctx.runner.push(_make_result(1, stdout, "")) + tool_ctx_kitchen_open.runner.push(_make_result(returncode=1)) # clone guard snapshot + tool_ctx_kitchen_open.runner.push(_make_result(1, stdout, "")) result = json.loads(await run_skill("/investigate plan.md", "/tmp")) assert result["needs_retry"] is True assert result["retry_reason"] == RetryReason.RESUME @@ -91,7 +86,7 @@ async def test_success_not_retriable(self, tool_ctx, monkeypatch): assistant = json.dumps( { "type": "assistant", - "message": {"content": [{"type": "tool_use", "name": "Edit", "id": "tu1"}]}, + "message": {"content": [{"type": "tool_use", "name": "Read", "id": "tu1"}]}, } ) result_record = json.dumps( @@ -104,6 +99,7 @@ async def test_success_not_retriable(self, tool_ctx, monkeypatch): } ) stdout = assistant + "\n" + result_record + tool_ctx.runner.push(_make_result(returncode=1)) # clone guard snapshot tool_ctx.runner.push(_make_result(0, stdout, "")) result = json.loads(await run_skill("/investigate plan.md", "/tmp")) assert result["needs_retry"] is False @@ -121,6 +117,7 @@ async def test_execution_error_not_retriable(self, tool_ctx): "errors": ["crashed"], } ) + tool_ctx.runner.push(_make_result(returncode=1)) # clone guard snapshot tool_ctx.runner.push(_make_result(1, stdout, "")) result = json.loads(await run_skill("/investigate plan.md", "/tmp")) assert result["needs_retry"] is False @@ -128,6 +125,7 @@ async def test_execution_error_not_retriable(self, tool_ctx): @pytest.mark.anyio async def test_unparseable_stdout_not_retriable(self, tool_ctx): """Non-JSON stdout -> needs_retry=False.""" + tool_ctx.runner.push(_make_result(returncode=1)) # clone guard snapshot tool_ctx.runner.push(_make_result(1, "crash dump", "segfault")) result = json.loads(await run_skill("/investigate plan.md", "/tmp")) assert result["needs_retry"] is False @@ -137,7 +135,7 @@ class TestRunSkillAgentResult: """run_skill result field contains actionable text.""" @pytest.mark.anyio - async def test_context_limit_result_is_actionable(self, tool_ctx): + async def test_context_limit_result_is_actionable(self, tool_ctx_kitchen_open): """When context is exhausted, result text must NOT say 'Prompt is too long'.""" stdout = json.dumps( { @@ -148,13 +146,14 @@ async def test_context_limit_result_is_actionable(self, tool_ctx): "session_id": "s1", } ) - tool_ctx.runner.push(_make_result(1, stdout, "")) + tool_ctx_kitchen_open.runner.push(_make_result(returncode=1)) # clone guard snapshot + tool_ctx_kitchen_open.runner.push(_make_result(1, stdout, "")) result = json.loads(await run_skill("/retry-worktree plan.md", "/tmp")) assert "prompt is too long" not in result["result"].lower() assert result["needs_retry"] is True @pytest.mark.anyio - async def test_normal_success_result_passes_through(self, tool_ctx): + async def test_normal_success_result_passes_through(self, tool_ctx_kitchen_open): """Normal success result text is preserved.""" stdout = json.dumps( { @@ -165,7 +164,8 @@ async def test_normal_success_result_passes_through(self, tool_ctx): "session_id": "s1", } ) - tool_ctx.runner.push(_make_result(0, stdout, "")) + tool_ctx_kitchen_open.runner.push(_make_result(returncode=1)) # clone guard snapshot + tool_ctx_kitchen_open.runner.push(_make_result(0, stdout, "")) result = json.loads(await run_skill("/investigate plan.md", "/tmp")) assert result["result"] == "Done." @@ -174,7 +174,7 @@ class TestRunSkillPassesAddDir: """run_skill passes ValidatedAddDir instances to executor.""" @pytest.mark.anyio - async def test_run_skill_passes_validated_add_dirs_to_executor(self, tool_ctx): + async def test_run_skill_passes_validated_add_dirs_to_executor(self, tool_ctx_kitchen_open): """add_dirs forwarded to ctx.executor.run() are ValidatedAddDir instances.""" from unittest.mock import AsyncMock @@ -192,7 +192,7 @@ async def test_run_skill_passes_validated_add_dirs_to_executor(self, tool_ctx): stderr="", ) mock_run = AsyncMock(return_value=mock_result) - tool_ctx.executor = type("MockExec", (), {"run": mock_run})() + tool_ctx_kitchen_open.executor = type("MockExec", (), {"run": mock_run})() await run_skill("/investigate something", "/tmp") add_dirs = mock_run.call_args.kwargs.get("add_dirs", ()) @@ -217,13 +217,14 @@ async def test_includes_needs_retry_false(self, tool_ctx, monkeypatch): "session_id": "s1", } ) + tool_ctx.runner.push(_make_result(returncode=1)) # clone guard snapshot tool_ctx.runner.push(_make_result(0, stdout, "")) result = json.loads(await run_skill("/investigate error", "/tmp")) assert result["needs_retry"] is False assert result["retry_reason"] == RetryReason.NONE @pytest.mark.anyio - async def test_includes_needs_retry_true_on_context_limit(self, tool_ctx): + async def test_includes_needs_retry_true_on_context_limit(self, tool_ctx_kitchen_open): """run_skill response includes needs_retry=True when context is exhausted.""" stdout = json.dumps( { @@ -234,14 +235,17 @@ async def test_includes_needs_retry_true_on_context_limit(self, tool_ctx): "session_id": "s1", } ) - tool_ctx.runner.push(_make_result(1, stdout, "")) + tool_ctx_kitchen_open.runner.push(_make_result(returncode=1)) # clone guard snapshot + tool_ctx_kitchen_open.runner.push(_make_result(1, stdout, "")) result = json.loads(await run_skill("/investigate error", "/tmp")) assert result["needs_retry"] is True assert result["retry_reason"] == RetryReason.RESUME assert "prompt is too long" not in result["result"].lower() @pytest.mark.anyio - async def test_includes_empty_output_reason_on_empty_natural_exit_session(self, tool_ctx): + async def test_includes_empty_output_reason_on_empty_natural_exit_session( + self, tool_ctx_kitchen_open + ): """run_skill returns retry_reason=empty_output for empty natural-exit sessions. NATURAL_EXIT + rc=0 + empty_output subtype: the session exited cleanly but @@ -258,7 +262,7 @@ async def test_includes_empty_output_reason_on_empty_natural_exit_session(self, "session_id": "", } ) - tool_ctx.runner.push( + tool_ctx_kitchen_open.runner.push( _make_result(0, stdout, "", termination_reason=TerminationReason.NATURAL_EXIT) ) result = json.loads(await run_skill("/investigate error", "/tmp")) diff --git a/tests/server/test_tools_session_diagnostics.py b/tests/server/test_tools_session_diagnostics.py index 821be2c29b..8da92bae86 100644 --- a/tests/server/test_tools_session_diagnostics.py +++ b/tests/server/test_tools_session_diagnostics.py @@ -10,12 +10,14 @@ from autoskillit.core import SkillResult from autoskillit.core.types import RetryReason -from autoskillit.server.tools_github import ( +from autoskillit.server.tools.tools_github import ( _format_diagnostics_section, _read_session_diagnostics, report_bug, ) +pytestmark = [pytest.mark.layer("server"), pytest.mark.small] + # --------------------------------------------------------------------------- # _read_session_diagnostics unit tests # --------------------------------------------------------------------------- @@ -271,7 +273,9 @@ def _make_mock_github(search_total: int, existing_body: str = "") -> MagicMock: client.create_issue = AsyncMock( return_value={"success": True, "url": "https://github.com/o/r/issues/99"} ) - client.add_comment = AsyncMock(return_value={"success": True}) + client.update_issue_body = AsyncMock( + return_value={"success": True, "issue_url": "https://github.com/o/r/issues/7"} + ) return client @@ -310,23 +314,23 @@ def _make_session_dir( @pytest.mark.anyio -async def test_report_bug_includes_diagnostics_in_new_issue_body(tool_ctx, tmp_path): +async def test_report_bug_includes_diagnostics_in_new_issue_body(tool_ctx_kitchen_open, tmp_path): """New issue body includes the Session Diagnostics section when diagnostics are available.""" session_id = "diag-session-001" _make_session_dir(tmp_path, session_id) - tool_ctx.config.report_bug.report_dir = str(tmp_path / "bug-reports") - tool_ctx.config.report_bug.github_filing = True - tool_ctx.config.github.default_repo = "owner/repo" - tool_ctx.config.linux_tracing.log_dir = str(tmp_path / "session_logs") + tool_ctx_kitchen_open.config.report_bug.report_dir = str(tmp_path / "bug-reports") + tool_ctx_kitchen_open.config.report_bug.github_filing = True + tool_ctx_kitchen_open.config.github.default_repo = "owner/repo" + tool_ctx_kitchen_open.config.linux_tracing.log_dir = str(tmp_path / "session_logs") - tool_ctx.executor = _make_mock_executor( + tool_ctx_kitchen_open.executor = _make_mock_executor( success=True, result="---bug-fingerprint---\nfp-001\n---/bug-fingerprint---\nReport text.", session_id=session_id, ) github_mock = _make_mock_github(search_total=0) - tool_ctx.github_client = github_mock + tool_ctx_kitchen_open.github_client = github_mock result = json.loads( await report_bug(error_context="Test error", cwd=str(tmp_path), severity="blocking") @@ -340,7 +344,9 @@ async def test_report_bug_includes_diagnostics_in_new_issue_body(tool_ctx, tmp_p @pytest.mark.anyio -async def test_report_bug_includes_condensed_diagnostics_in_duplicate_comment(tool_ctx, tmp_path): +async def test_report_bug_includes_condensed_diagnostics_in_duplicate_comment( + tool_ctx_kitchen_open, tmp_path +): """Duplicate comment includes condensed metrics but no
blocks.""" session_id = "diag-session-002" _make_session_dir( @@ -349,44 +355,44 @@ async def test_report_bug_includes_condensed_diagnostics_in_duplicate_comment(to anomalies=[{"kind": "oom_spike", "severity": "warning", "detail": {}}], ) - tool_ctx.config.report_bug.report_dir = str(tmp_path / "bug-reports") - tool_ctx.config.report_bug.github_filing = True - tool_ctx.config.github.default_repo = "owner/repo" - tool_ctx.config.linux_tracing.log_dir = str(tmp_path / "session_logs") + tool_ctx_kitchen_open.config.report_bug.report_dir = str(tmp_path / "bug-reports") + tool_ctx_kitchen_open.config.report_bug.github_filing = True + tool_ctx_kitchen_open.config.github.default_repo = "owner/repo" + tool_ctx_kitchen_open.config.linux_tracing.log_dir = str(tmp_path / "session_logs") - tool_ctx.executor = _make_mock_executor( + tool_ctx_kitchen_open.executor = _make_mock_executor( success=True, result="---bug-fingerprint---\nfp-002\n---/bug-fingerprint---\nReport text.", session_id=session_id, ) github_mock = _make_mock_github(search_total=1, existing_body="different error") - tool_ctx.github_client = github_mock + tool_ctx_kitchen_open.github_client = github_mock await report_bug(error_context="Test error", cwd=str(tmp_path), severity="blocking") - _args = github_mock.add_comment.call_args - comment_body = _args.kwargs.get("body", _args.args[3]) - assert "Session Diagnostics" in comment_body - assert "
" not in comment_body # condensed — no details blocks + _args = github_mock.update_issue_body.call_args + new_body = _args.kwargs.get("new_body", _args.args[3]) + assert "Session Diagnostics" in new_body + assert "
" not in new_body # condensed — no details blocks @pytest.mark.anyio async def test_report_bug_proceeds_without_diagnostics_when_session_dir_missing( - tool_ctx, tmp_path + tool_ctx_kitchen_open, tmp_path ): """GitHub issue is still filed even when no session diagnostics directory exists.""" - tool_ctx.config.report_bug.report_dir = str(tmp_path / "bug-reports") - tool_ctx.config.report_bug.github_filing = True - tool_ctx.config.github.default_repo = "owner/repo" - tool_ctx.config.linux_tracing.log_dir = str(tmp_path / "session_logs") + tool_ctx_kitchen_open.config.report_bug.report_dir = str(tmp_path / "bug-reports") + tool_ctx_kitchen_open.config.report_bug.github_filing = True + tool_ctx_kitchen_open.config.github.default_repo = "owner/repo" + tool_ctx_kitchen_open.config.linux_tracing.log_dir = str(tmp_path / "session_logs") - tool_ctx.executor = _make_mock_executor( + tool_ctx_kitchen_open.executor = _make_mock_executor( success=True, result="---bug-fingerprint---\nfp-003\n---/bug-fingerprint---\nReport text.", session_id="nonexistent-session-id", # no directory on disk ) github_mock = _make_mock_github(search_total=0) - tool_ctx.github_client = github_mock + tool_ctx_kitchen_open.github_client = github_mock result = json.loads( await report_bug(error_context="Test error", cwd=str(tmp_path), severity="blocking") @@ -400,20 +406,22 @@ async def test_report_bug_proceeds_without_diagnostics_when_session_dir_missing( @pytest.mark.anyio -async def test_report_bug_skips_diagnostics_for_fallback_session_id(tool_ctx, tmp_path): +async def test_report_bug_skips_diagnostics_for_fallback_session_id( + tool_ctx_kitchen_open, tmp_path +): """Fallback session IDs (no_session_*) never trigger diagnostics read.""" - tool_ctx.config.report_bug.report_dir = str(tmp_path / "bug-reports") - tool_ctx.config.report_bug.github_filing = True - tool_ctx.config.github.default_repo = "owner/repo" - tool_ctx.config.linux_tracing.log_dir = str(tmp_path / "session_logs") + tool_ctx_kitchen_open.config.report_bug.report_dir = str(tmp_path / "bug-reports") + tool_ctx_kitchen_open.config.report_bug.github_filing = True + tool_ctx_kitchen_open.config.github.default_repo = "owner/repo" + tool_ctx_kitchen_open.config.linux_tracing.log_dir = str(tmp_path / "session_logs") - tool_ctx.executor = _make_mock_executor( + tool_ctx_kitchen_open.executor = _make_mock_executor( success=True, result="---bug-fingerprint---\nfp-004\n---/bug-fingerprint---\nReport.", session_id="no_session_2026-01-01T00-00-00", ) github_mock = _make_mock_github(search_total=0) - tool_ctx.github_client = github_mock + tool_ctx_kitchen_open.github_client = github_mock result = json.loads( await report_bug(error_context="Test error", cwd=str(tmp_path), severity="blocking") diff --git a/tests/server/test_tools_status.py b/tests/server/test_tools_status.py deleted file mode 100644 index beb6d95e25..0000000000 --- a/tests/server/test_tools_status.py +++ /dev/null @@ -1,1242 +0,0 @@ -"""Tests for autoskillit server status tools.""" - -from __future__ import annotations - -import asyncio -import json -from datetime import UTC, datetime -from pathlib import Path -from unittest.mock import MagicMock - -import anyio -import pytest - -from autoskillit.config import AutomationConfig, ReadDbConfig, TokenUsageConfig -from autoskillit.core.types import ChannelConfirmation -from autoskillit.execution.github import DefaultGitHubFetcher -from autoskillit.pipeline.audit import FailureRecord -from autoskillit.pipeline.gate import DefaultGateState -from autoskillit.server.tools_execution import run_skill -from autoskillit.server.tools_status import ( - get_pipeline_report, - get_quota_events, - get_timing_summary, - get_token_summary, - kitchen_status, - read_db, - write_telemetry_files, -) -from tests.conftest import _make_result - - -def _make_failure_record(**overrides: object) -> FailureRecord: - defaults = dict( - timestamp="2026-02-24T16:00:00Z", - skill_command="/autoskillit:implement-worktree", - exit_code=1, - subtype="error", - needs_retry=False, - retry_reason="none", - stderr="something went wrong", - ) - return FailureRecord(**{**defaults, **overrides}) # type: ignore[arg-type] - - -def _failed_session_json() -> str: - return json.dumps( - { - "type": "result", - "subtype": "error", - "result": "Task failed with an error", - "session_id": "test-session", - "is_error": True, - } - ) - - -class TestKitchenStatus: - """kitchen_status tool returns version health info.""" - - @pytest.fixture(autouse=True) - def _setup(self, tool_ctx, monkeypatch, tmp_path): - monkeypatch.chdir(tmp_path) - - @pytest.mark.anyio - async def test_status_returns_version_info(self, tool_ctx): - import autoskillit - - tool_ctx.plugin_dir = str(Path(autoskillit.__file__).parent) - from autoskillit import __version__ - - result = json.loads(await kitchen_status()) - assert result["package_version"] == __version__ - assert result["plugin_json_version"] == __version__ - assert result["versions_match"] is True - assert "warning" not in result - - @pytest.mark.anyio - async def test_status_reports_mismatch(self, tmp_path, tool_ctx): - plugin_dir = tmp_path / ".claude-plugin" - plugin_dir.mkdir() - (plugin_dir / "plugin.json").write_text( - json.dumps({"name": "autoskillit", "version": "0.0.0"}) - ) - tool_ctx.plugin_dir = str(tmp_path) - result = json.loads(await kitchen_status()) - assert result["versions_match"] is False - assert "warning" in result - assert "mismatch" in result["warning"].lower() - - @pytest.mark.anyio - async def test_status_includes_token_usage_verbosity_default(self): - """TU_S1: kitchen_status includes token_usage_verbosity key with default 'summary'.""" - result = json.loads(await kitchen_status()) - assert "token_usage_verbosity" in result - assert result["token_usage_verbosity"] == "summary" - - @pytest.mark.anyio - async def test_status_reflects_none_verbosity(self, tool_ctx): - """TU_S2: kitchen_status reflects 'none' verbosity from config.""" - cfg = AutomationConfig() - cfg.token_usage = TokenUsageConfig(verbosity="none") - tool_ctx.config = cfg - result = json.loads(await kitchen_status()) - assert result["token_usage_verbosity"] == "none" - - @pytest.mark.anyio - async def test_kitchen_status_includes_github_config(self, tool_ctx): - tool_ctx.config.github.default_repo = "owner/repo" - status = json.loads(await kitchen_status()) - assert "github_default_repo" in status - assert status["github_default_repo"] == "owner/repo" - assert "github_token_configured" in status - - @pytest.mark.anyio - async def test_kitchen_status_github_token_configured_true_from_client(self, tool_ctx): - """kitchen_status must read github_token_configured from ctx.github_client.has_token.""" - tool_ctx.github_client = DefaultGitHubFetcher(token="my-token") - status = json.loads(await kitchen_status()) - assert status["github_token_configured"] is True - - @pytest.mark.anyio - async def test_kitchen_status_github_token_not_configured_from_client( - self, tool_ctx, monkeypatch - ): - monkeypatch.delenv("GITHUB_TOKEN", raising=False) - tool_ctx.github_client = DefaultGitHubFetcher(token=None) - status = json.loads(await kitchen_status()) - assert status["github_token_configured"] is False - - @pytest.mark.anyio - async def test_kitchen_status_github_token_does_not_reflect_post_construction_env( - self, tool_ctx, monkeypatch - ): - """kitchen_status must NOT re-read os.environ — reflects ctx.github_client.has_token.""" - tool_ctx.github_client = DefaultGitHubFetcher(token=None) - monkeypatch.setenv("GITHUB_TOKEN", "set-after-construction") - status = json.loads(await kitchen_status()) - assert status["github_token_configured"] is False - - @pytest.mark.anyio - async def test_kitchen_status_has_no_gate_file_exists_field(self, tool_ctx): - """kitchen_status must not report gate_file_exists.""" - result = json.loads(await kitchen_status()) - assert "gate_file_exists" not in result - - @pytest.mark.anyio - async def test_kitchen_status_has_no_split_brain_warning(self, tool_ctx, tmp_path): - """A file at the old gate path must not trigger any warning in kitchen_status.""" - gate_dir = tmp_path / ".autoskillit" / "temp" - gate_dir.mkdir(parents=True) - (gate_dir / ".kitchen_gate").write_text("{}") - result_str = json.dumps(json.loads(await kitchen_status())) - assert "stale" not in result_str.lower() - assert "gate_file" not in result_str.lower() - - -class TestGetPipelineReport: - """get_pipeline_report is a gated tool that returns accumulated failures.""" - - @pytest.mark.anyio - async def test_gate_closed_returns_gate_error(self, tool_ctx): - """get_pipeline_report returns gate_error when kitchen gate is closed.""" - tool_ctx.gate = DefaultGateState(enabled=False) - result = json.loads(await get_pipeline_report()) - assert result.get("success") is False - assert result.get("subtype") == "gate_error" - - @pytest.mark.anyio - async def test_returns_empty_initially(self, tool_ctx): - result = json.loads(await get_pipeline_report()) - assert result["total_failures"] == 0 - assert result["failures"] == [] - - @pytest.mark.anyio - async def test_accumulates_failures_from_run_skill(self, tool_ctx): - - tool_ctx.gate = DefaultGateState(enabled=True) - tool_ctx.runner.push( - _make_result( - returncode=1, - stdout=_failed_session_json(), - channel_confirmation=ChannelConfirmation.UNMONITORED, - ) - ) - await run_skill(skill_command="/autoskillit:test", cwd="/tmp") - result = json.loads(await get_pipeline_report()) - assert result["total_failures"] == 1 - assert result["failures"][0]["skill_command"].startswith("/autoskillit:test") - - @pytest.mark.anyio - async def test_clear_true_resets_after_returning(self, tool_ctx): - tool_ctx.audit.record_failure(_make_failure_record()) - result = json.loads(await get_pipeline_report(clear=True)) - assert result["total_failures"] == 1 - result2 = json.loads(await get_pipeline_report()) - assert result2["total_failures"] == 0 - - @pytest.mark.anyio - async def test_get_pipeline_report_awaits_startup_ready(self, tool_ctx, monkeypatch): - """get_pipeline_report must await _startup_ready before accessing audit data.""" - import autoskillit.server._state as _state_mod - - ready = asyncio.Event() - monkeypatch.setattr(_state_mod, "_startup_ready", ready) - - task = asyncio.create_task(get_pipeline_report()) - await asyncio.sleep(0.05) - assert not task.done(), "get_pipeline_report returned before _startup_ready was set" - - ready.set() - await asyncio.sleep(0.05) - assert task.done(), "get_pipeline_report did not unblock after _startup_ready was set" - - -class TestGetTokenSummary: - """get_token_summary is a gated tool that returns accumulated token usage.""" - - @pytest.mark.anyio - async def test_gate_closed_returns_gate_error(self, tool_ctx): - """get_token_summary returns gate_error when kitchen gate is closed.""" - tool_ctx.gate = DefaultGateState(enabled=False) - result = json.loads(await get_token_summary()) - assert result.get("success") is False - assert result.get("subtype") == "gate_error" - - @pytest.mark.anyio - async def test_returns_empty_steps_initially(self, tool_ctx): - result = json.loads(await get_token_summary()) - assert result["steps"] == [] - assert result["total"]["input_tokens"] == 0 - assert result["total"]["output_tokens"] == 0 - assert result["total"]["cache_creation_input_tokens"] == 0 - assert result["total"]["cache_read_input_tokens"] == 0 - - @pytest.mark.anyio - async def test_returns_entry_per_step_name(self, tool_ctx): - tool_ctx.token_log.record( - "investigate", - { - "input_tokens": 100, - "output_tokens": 50, - "cache_creation_input_tokens": 10, - "cache_read_input_tokens": 5, - }, - ) - tool_ctx.token_log.record( - "implement", - { - "input_tokens": 200, - "output_tokens": 80, - "cache_creation_input_tokens": 20, - "cache_read_input_tokens": 10, - }, - ) - result = json.loads(await get_token_summary()) - assert len(result["steps"]) == 2 - assert result["steps"][0]["step_name"] == "investigate" - assert result["steps"][1]["step_name"] == "implement" - - @pytest.mark.anyio - async def test_multiple_invocations_same_step_are_summed(self, tool_ctx): - usage = { - "input_tokens": 100, - "output_tokens": 50, - "cache_creation_input_tokens": 0, - "cache_read_input_tokens": 0, - } - tool_ctx.token_log.record("implement", usage) - tool_ctx.token_log.record("implement", usage) - tool_ctx.token_log.record("implement", usage) - result = json.loads(await get_token_summary()) - assert len(result["steps"]) == 1 - assert result["steps"][0]["input_tokens"] == 300 - assert result["steps"][0]["invocation_count"] == 3 - - @pytest.mark.anyio - async def test_total_field_sums_all_steps(self, tool_ctx): - tool_ctx.token_log.record( - "plan", - { - "input_tokens": 100, - "output_tokens": 50, - "cache_creation_input_tokens": 10, - "cache_read_input_tokens": 5, - }, - ) - tool_ctx.token_log.record( - "implement", - { - "input_tokens": 200, - "output_tokens": 80, - "cache_creation_input_tokens": 20, - "cache_read_input_tokens": 10, - }, - ) - result = json.loads(await get_token_summary()) - assert result["total"]["input_tokens"] == 300 - assert result["total"]["output_tokens"] == 130 - assert result["total"]["cache_creation_input_tokens"] == 30 - assert result["total"]["cache_read_input_tokens"] == 15 - - @pytest.mark.anyio - async def test_clear_true_resets_after_returning(self, tool_ctx): - tool_ctx.token_log.record( - "plan", - { - "input_tokens": 100, - "output_tokens": 50, - "cache_creation_input_tokens": 0, - "cache_read_input_tokens": 0, - }, - ) - result = json.loads(await get_token_summary(clear=True)) - assert len(result["steps"]) == 1 - result2 = json.loads(await get_token_summary()) - assert result2["steps"] == [] - - @pytest.mark.anyio - async def test_response_shape(self, tool_ctx): - tool_ctx.token_log.record( - "plan", - { - "input_tokens": 10, - "output_tokens": 5, - "cache_creation_input_tokens": 1, - "cache_read_input_tokens": 2, - }, - ) - result = json.loads(await get_token_summary()) - assert "steps" in result - assert "total" in result - assert isinstance(result["steps"], list) - total_keys = set(result["total"].keys()) - assert { - "input_tokens", - "output_tokens", - "cache_creation_input_tokens", - "cache_read_input_tokens", - } <= total_keys - - @pytest.mark.anyio - async def test_step_includes_elapsed_seconds(self, tool_ctx): - """Each step dict in the response includes elapsed_seconds.""" - start = "2026-01-01T00:00:00+00:00" - end = "2026-01-01T00:00:30+00:00" - tool_ctx.token_log.record("deploy", {"input_tokens": 200}, start_ts=start, end_ts=end) - result = json.loads(await get_token_summary()) - assert result["steps"][0]["elapsed_seconds"] == pytest.approx(30.0) - - @pytest.mark.anyio - async def test_total_includes_total_elapsed_seconds(self, tool_ctx): - """Total dict includes total_elapsed_seconds summed across all steps.""" - tool_ctx.token_log.record( - "a", - {"input_tokens": 10}, - start_ts="2026-01-01T00:00:00+00:00", - end_ts="2026-01-01T00:00:05+00:00", - ) - tool_ctx.token_log.record( - "b", - {"input_tokens": 20}, - start_ts="2026-01-01T00:01:00+00:00", - end_ts="2026-01-01T00:01:08+00:00", - ) - result = json.loads(await get_token_summary()) - assert "total_elapsed_seconds" in result["total"] - assert result["total"]["total_elapsed_seconds"] == pytest.approx(13.0) - - -class TestGetTimingSummary: - """get_timing_summary is a gated tool that returns accumulated wall-clock timing.""" - - @pytest.mark.anyio - async def test_gate_closed_returns_gate_error(self, tool_ctx): - """get_timing_summary returns gate_error when kitchen gate is closed.""" - tool_ctx.gate = DefaultGateState(enabled=False) - result = json.loads(await get_timing_summary()) - assert result.get("success") is False - assert result.get("subtype") == "gate_error" - - @pytest.mark.anyio - async def test_returns_empty_steps_initially(self, tool_ctx): - result = json.loads(await get_timing_summary()) - assert result["steps"] == [] - assert result["total"]["total_seconds"] == 0.0 - - @pytest.mark.anyio - async def test_returns_entry_per_step_name(self, tool_ctx): - tool_ctx.timing_log.record("clone", 4.0) - tool_ctx.timing_log.record("test_check", 12.0) - result = json.loads(await get_timing_summary()) - assert len(result["steps"]) == 2 - step_names = {s["step_name"] for s in result["steps"]} - assert step_names == {"clone", "test_check"} - - @pytest.mark.anyio - async def test_multiple_invocations_same_step_are_summed(self, tool_ctx): - tool_ctx.timing_log.record("impl", 10.0) - tool_ctx.timing_log.record("impl", 10.0) - tool_ctx.timing_log.record("impl", 10.0) - result = json.loads(await get_timing_summary()) - assert len(result["steps"]) == 1 - assert result["steps"][0]["total_seconds"] == 30.0 - assert result["steps"][0]["invocation_count"] == 3 - - @pytest.mark.anyio - async def test_total_field_sums_all_steps(self, tool_ctx): - tool_ctx.timing_log.record("a", 5.0) - tool_ctx.timing_log.record("b", 8.0) - result = json.loads(await get_timing_summary()) - assert result["total"]["total_seconds"] == 13.0 - - @pytest.mark.anyio - async def test_clear_true_resets_after_returning(self, tool_ctx): - tool_ctx.timing_log.record("clone", 4.0) - result = json.loads(await get_timing_summary(clear=True)) - assert len(result["steps"]) == 1 - result2 = json.loads(await get_timing_summary()) - assert result2["steps"] == [] - - @pytest.mark.anyio - async def test_response_shape(self, tool_ctx): - tool_ctx.timing_log.record("plan", 3.0) - result = json.loads(await get_timing_summary()) - assert "steps" in result - assert "total" in result - assert isinstance(result["steps"], list) - assert "total_seconds" in result["total"] - - -def test_check_quota_absent_from_mcp_registry(tool_ctx): - """check_quota must not appear in the agent-visible tool list. - Quota enforcement is the PreToolUse hook's responsibility, not an MCP tool.""" - from autoskillit.server import mcp - - tools = anyio.run(mcp.list_tools) - assert "check_quota" not in {t.name for t in tools}, ( - "check_quota must be removed from the MCP registry. " - "Agents should not call it — the PreToolUse hook enforces quota automatically." - ) - - -class TestTelemetryRecoveryData: - """MCP status tools return data populated via load_from_log_dir recovery.""" - - def _write_token_session( - self, log_root: Path, dir_name: str, step_name: str, input_tokens: int - ) -> None: - session_dir = log_root / "sessions" / dir_name - session_dir.mkdir(parents=True, exist_ok=True) - tu = { - "step_name": step_name, - "input_tokens": input_tokens, - "output_tokens": 50, - "cache_creation_input_tokens": 0, - "cache_read_input_tokens": 0, - "timing_seconds": 10.0, - } - (session_dir / "token_usage.json").write_text(json.dumps(tu)) - idx = { - "dir_name": dir_name, - "timestamp": "2026-03-07T00:00:00+00:00", - "session_id": dir_name, - } - with (log_root / "sessions.jsonl").open("a") as f: - f.write(json.dumps(idx) + "\n") - - def _write_timing_session( - self, log_root: Path, dir_name: str, step_name: str, total_seconds: float - ) -> None: - session_dir = log_root / "sessions" / dir_name - session_dir.mkdir(parents=True, exist_ok=True) - st = {"step_name": step_name, "total_seconds": total_seconds} - (session_dir / "step_timing.json").write_text(json.dumps(st)) - idx = { - "dir_name": dir_name, - "timestamp": "2026-03-07T00:00:00+00:00", - "session_id": dir_name, - } - with (log_root / "sessions.jsonl").open("a") as f: - f.write(json.dumps(idx) + "\n") - - def _write_audit_session(self, log_root: Path, dir_name: str) -> None: - session_dir = log_root / "sessions" / dir_name - session_dir.mkdir(parents=True, exist_ok=True) - record = { - "timestamp": "2026-03-07T00:00:00Z", - "skill_command": "/autoskillit:implement-worktree", - "exit_code": 1, - "subtype": "error", - "needs_retry": False, - "retry_reason": "none", - "stderr": "oops", - } - (session_dir / "audit_log.json").write_text(json.dumps([record])) - idx = { - "dir_name": dir_name, - "timestamp": "2026-03-07T00:00:00+00:00", - "session_id": dir_name, - } - with (log_root / "sessions.jsonl").open("a") as f: - f.write(json.dumps(idx) + "\n") - - @pytest.mark.anyio - async def test_token_summary_reflects_recovered_data(self, tool_ctx, tmp_path): - """get_token_summary returns data loaded via load_from_log_dir.""" - log_root = tmp_path / "logs" - self._write_token_session(log_root, "s001", "implement", 500) - tool_ctx.token_log.load_from_log_dir(log_root) - result = json.loads(await get_token_summary()) - steps = {s["step_name"]: s for s in result["steps"]} - assert "implement" in steps - assert steps["implement"]["input_tokens"] == 500 - - @pytest.mark.anyio - async def test_timing_summary_reflects_recovered_data(self, tool_ctx, tmp_path): - """get_timing_summary returns data loaded via load_from_log_dir.""" - log_root = tmp_path / "logs" - self._write_timing_session(log_root, "s001", "plan", 99.0) - tool_ctx.timing_log.load_from_log_dir(log_root) - result = json.loads(await get_timing_summary()) - steps = {s["step_name"]: s for s in result["steps"]} - assert "plan" in steps - assert steps["plan"]["total_seconds"] == pytest.approx(99.0) - - @pytest.mark.anyio - async def test_pipeline_report_reflects_recovered_audit(self, tool_ctx, tmp_path): - """get_pipeline_report returns failures loaded via load_from_log_dir.""" - log_root = tmp_path / "logs" - self._write_audit_session(log_root, "s001") - tool_ctx.audit.load_from_log_dir(log_root) - result = json.loads(await get_pipeline_report()) - assert result["total_failures"] == 1 - assert result["failures"][0]["skill_command"] == "/autoskillit:implement-worktree" - - -class TestGetQuotaEvents: - @pytest.mark.anyio - async def test_returns_events_from_jsonl(self, tool_ctx, tmp_path, monkeypatch): - events = [ - { - "ts": "2026-03-10T10:00:00+00:00", - "event": "approved", - "threshold": 85.0, - "utilization": 50.0, - }, - { - "ts": "2026-03-10T11:00:00+00:00", - "event": "blocked", - "threshold": 85.0, - "utilization": 92.5, - }, - ] - log_dir = tmp_path / "logs" - log_dir.mkdir() - (log_dir / "quota_events.jsonl").write_text( - "\n".join(json.dumps(e) for e in events) + "\n" - ) - monkeypatch.setattr(tool_ctx.config.linux_tracing, "log_dir", str(log_dir)) - result = json.loads(await get_quota_events()) - assert result["total_count"] == 2 - assert result["events"][0]["event"] == "blocked" # most recent first - - @pytest.mark.anyio - async def test_limits_to_n_events(self, tool_ctx, tmp_path, monkeypatch): - log_dir = tmp_path / "logs" - log_dir.mkdir() - lines = [ - json.dumps({"ts": f"2026-03-10T{h:02d}:00:00+00:00", "event": "approved"}) - for h in range(10) - ] - (log_dir / "quota_events.jsonl").write_text("\n".join(lines) + "\n") - monkeypatch.setattr(tool_ctx.config.linux_tracing, "log_dir", str(log_dir)) - result = json.loads(await get_quota_events(n=3)) - assert len(result["events"]) == 3 - assert result["total_count"] == 10 - # most-recent-first: hours 09, 08, 07 - assert result["events"][0]["ts"] == "2026-03-10T09:00:00+00:00" - assert result["events"][1]["ts"] == "2026-03-10T08:00:00+00:00" - assert result["events"][2]["ts"] == "2026-03-10T07:00:00+00:00" - - @pytest.mark.anyio - async def test_returns_empty_when_file_missing(self, tool_ctx, tmp_path, monkeypatch): - log_dir = tmp_path / "logs" - log_dir.mkdir() - monkeypatch.setattr(tool_ctx.config.linux_tracing, "log_dir", str(log_dir)) - result = json.loads(await get_quota_events()) - assert result["events"] == [] - assert result["total_count"] == 0 - - -class TestWriteTelemetryFiles: - @pytest.mark.anyio - async def test_writes_token_summary_markdown(self, tool_ctx, tmp_path): - tool_ctx.token_log.record( - "step1", - { - "input_tokens": 100, - "output_tokens": 50, - "cache_creation_input_tokens": 0, - "cache_read_input_tokens": 0, - }, - ) - result = json.loads(await write_telemetry_files(str(tmp_path))) - path = Path(result["token_summary_path"]) - assert path.exists() - content = path.read_text() - assert "step1" in content - # Format-structural assertions (table, not bullet list) - assert "| Step |" in content - assert "|---" in content - assert "- input_tokens:" not in content - assert "# Token Summary" not in content - - @pytest.mark.anyio - async def test_writes_timing_summary_markdown(self, tool_ctx, tmp_path): - tool_ctx.timing_log.record("step1", 12.5) - result = json.loads(await write_telemetry_files(str(tmp_path))) - path = Path(result["timing_summary_path"]) - assert path.exists() - content = path.read_text() - assert "step1" in content - # Format-structural assertions (table, not bullet list) - assert "| Step |" in content - assert "|---" in content - assert "- total_seconds:" not in content - assert "# Timing Summary" not in content - - @pytest.mark.anyio - async def test_token_file_uses_wall_clock_seconds(self, tool_ctx, tmp_path): - """write_telemetry_files merges wall_clock_seconds from timing log.""" - tool_ctx.token_log.record( - "deploy", - { - "input_tokens": 100, - "output_tokens": 50, - "cache_creation_input_tokens": 0, - "cache_read_input_tokens": 0, - }, - elapsed_seconds=5.0, - ) - tool_ctx.timing_log.record("deploy", 120.0) - result = json.loads(await write_telemetry_files(str(tmp_path))) - content = Path(result["token_summary_path"]).read_text() - # Should show 2m 0s (wall_clock=120), not 5s (elapsed) - assert "2m 0s" in content - - @pytest.mark.anyio - async def test_creates_output_dir_if_missing(self, tool_ctx, tmp_path): - out = str(tmp_path / "nested" / "telemetry") - result = json.loads(await write_telemetry_files(out)) - assert Path(result["token_summary_path"]).exists() - assert Path(result["timing_summary_path"]).exists() - - @pytest.mark.anyio - async def test_gate_closed_returns_gate_error(self, tool_ctx): - tool_ctx.gate.disable() - result = json.loads(await write_telemetry_files("/tmp")) - assert result["success"] is False - assert result["subtype"] == "gate_error" - - -class TestGetTokenSummaryFormat: - """Tests for get_token_summary format parameter.""" - - @pytest.mark.anyio - async def test_format_json_default(self, tool_ctx): - """format='json' (default) returns JSON dict.""" - tool_ctx.token_log.record( - "plan", - {"input_tokens": 100, "output_tokens": 50}, - ) - result = json.loads(await get_token_summary()) - assert "steps" in result - assert "total" in result - - @pytest.mark.anyio - async def test_format_table_returns_markdown(self, tool_ctx): - """format='table' returns a markdown table string.""" - tool_ctx.token_log.record( - "plan", - {"input_tokens": 100, "output_tokens": 50}, - ) - result = await get_token_summary(format="table") - assert "| Step |" in result - assert "|---" in result - assert "plan" in result - assert "**Total**" in result - # Not JSON - with pytest.raises(json.JSONDecodeError): - json.loads(result) - - @pytest.mark.anyio - async def test_format_table_with_clear(self, tool_ctx): - """format='table' + clear=True clears the log after returning.""" - tool_ctx.token_log.record( - "plan", - {"input_tokens": 100, "output_tokens": 50}, - ) - result = await get_token_summary(clear=True, format="table") - assert "plan" in result - result2 = json.loads(await get_token_summary()) - assert result2["steps"] == [] - - -class TestGetTimingSummaryFormat: - """Tests for get_timing_summary format parameter.""" - - @pytest.mark.anyio - async def test_format_json_default(self, tool_ctx): - """format='json' (default) returns JSON dict.""" - tool_ctx.timing_log.record("plan", 3.0) - result = json.loads(await get_timing_summary()) - assert "steps" in result - assert "total" in result - - @pytest.mark.anyio - async def test_format_table_returns_markdown(self, tool_ctx): - """format='table' returns a markdown table string.""" - tool_ctx.timing_log.record("plan", 3.0) - result = await get_timing_summary(format="table") - assert "| Step |" in result - assert "|---" in result - assert "plan" in result - assert "**Total**" in result - with pytest.raises(json.JSONDecodeError): - json.loads(result) - - -class TestTokenSummaryWallClock: - @pytest.mark.anyio - async def test_wall_clock_seconds_merged_from_timing_log(self, tool_ctx): - from autoskillit.pipeline.timings import DefaultTimingLog - from autoskillit.pipeline.tokens import DefaultTokenLog - - tool_ctx.token_log = DefaultTokenLog() - tool_ctx.timing_log = DefaultTimingLog() - tool_ctx.token_log.record( - "step-a", - { - "input_tokens": 100, - "output_tokens": 50, - "cache_creation_input_tokens": 0, - "cache_read_input_tokens": 0, - }, - elapsed_seconds=8.0, - ) - tool_ctx.timing_log.record("step-a", 12.5) - - result = json.loads(await get_token_summary()) - step = next(s for s in result["steps"] if s["step_name"] == "step-a") - assert step["wall_clock_seconds"] == pytest.approx(12.5) - assert step["elapsed_seconds"] == pytest.approx(8.0) - - @pytest.mark.anyio - async def test_wall_clock_falls_back_to_elapsed_when_no_timing(self, tool_ctx): - from autoskillit.pipeline.timings import DefaultTimingLog - from autoskillit.pipeline.tokens import DefaultTokenLog - - tool_ctx.token_log = DefaultTokenLog() - tool_ctx.timing_log = DefaultTimingLog() - tool_ctx.token_log.record( - "step-b", - { - "input_tokens": 200, - "output_tokens": 100, - "cache_creation_input_tokens": 0, - "cache_read_input_tokens": 0, - }, - elapsed_seconds=5.0, - ) - - # Verify no step-b entry was injected into timing_log (guards against parallel pollution) - assert not any(e["step_name"] == "step-b" for e in tool_ctx.timing_log.get_report()) - result = json.loads(await get_token_summary()) - step = next(s for s in result["steps"] if s["step_name"] == "step-b") - # No timing_log entry → falls back to elapsed_seconds - assert step["wall_clock_seconds"] == pytest.approx(5.0) - - @pytest.mark.anyio - async def test_merge_wall_clock_after_normalization(self, tool_ctx): - """ - When token entries and timing entries are both normalized to canonical names, - _merge_wall_clock_seconds must match them correctly. - """ - from autoskillit.pipeline.timings import DefaultTimingLog - from autoskillit.pipeline.tokens import DefaultTokenLog - - tool_ctx.token_log = DefaultTokenLog() - tool_ctx.timing_log = DefaultTimingLog() - - # Record via suffixed names — both should normalize to "implement" - usage = { - "input_tokens": 100, - "output_tokens": 50, - "cache_creation_input_tokens": 0, - "cache_read_input_tokens": 0, - } - tool_ctx.token_log.record("implement-30", usage) - tool_ctx.token_log.record("implement-31", usage) - tool_ctx.timing_log.record("implement-30", 60.0) - tool_ctx.timing_log.record("implement-31", 55.0) - - result = json.loads(await get_token_summary(clear=False, format="json")) - - assert len(result["steps"]) == 1 - step = result["steps"][0] - assert step["step_name"] == "implement" - assert step["input_tokens"] == 200 - # wall_clock_seconds from timing log should be present - assert "wall_clock_seconds" in step - assert step["wall_clock_seconds"] == pytest.approx(115.0) - - -class TestClearMarkerWritten: - @pytest.mark.anyio - async def test_token_summary_clear_writes_marker(self, tool_ctx, tmp_path, monkeypatch): - from autoskillit.execution.session_log import read_telemetry_clear_marker - - log_dir = tmp_path / "logs" - log_dir.mkdir() - monkeypatch.setattr(tool_ctx.config.linux_tracing, "log_dir", str(log_dir)) - await get_token_summary(clear=True) - marker = read_telemetry_clear_marker(log_dir) - assert marker is not None - - @pytest.mark.anyio - async def test_timing_summary_clear_writes_marker(self, tool_ctx, tmp_path, monkeypatch): - from autoskillit.execution.session_log import read_telemetry_clear_marker - - log_dir = tmp_path / "logs" - log_dir.mkdir() - monkeypatch.setattr(tool_ctx.config.linux_tracing, "log_dir", str(log_dir)) - await get_timing_summary(clear=True) - assert read_telemetry_clear_marker(log_dir) is not None - - @pytest.mark.anyio - async def test_pipeline_report_clear_writes_marker(self, tool_ctx, tmp_path, monkeypatch): - from autoskillit.execution.session_log import read_telemetry_clear_marker - - log_dir = tmp_path / "logs" - log_dir.mkdir() - monkeypatch.setattr(tool_ctx.config.linux_tracing, "log_dir", str(log_dir)) - await get_pipeline_report(clear=True) - assert read_telemetry_clear_marker(log_dir) is not None - - -class TestGetLogRoot: - def test_returns_resolved_log_dir(self, tool_ctx, tmp_path, monkeypatch): - """_get_log_root() returns the resolved path for the configured log_dir.""" - from autoskillit.server.helpers import resolve_log_dir - from autoskillit.server.tools_status import _get_log_root - - log_dir = tmp_path / "custom_logs" - monkeypatch.setattr(tool_ctx.config.linux_tracing, "log_dir", str(log_dir)) - result = _get_log_root() - assert result == resolve_log_dir(str(log_dir)) - - def test_returns_path_type(self, tool_ctx, tmp_path, monkeypatch): - """_get_log_root() returns a Path, not a string.""" - from autoskillit.server.tools_status import _get_log_root - - monkeypatch.setattr(tool_ctx.config.linux_tracing, "log_dir", str(tmp_path)) - assert isinstance(_get_log_root(), Path) - - -@pytest.mark.anyio -async def test_get_token_summary_not_contaminated_by_prior_pipeline( - tmp_path, tool_ctx, monkeypatch -): - """ - get_token_summary() must return only the current pipeline's data. - Entries written by a prior pipeline run must not appear in the summary, - even if the server was restarted with those sessions present in the log dir. - """ - from unittest.mock import patch - - from autoskillit.server import _state - from autoskillit.server._state import _initialize - - # tool_ctx fixture sets log_dir to tmp_path/"session_logs" — write sessions there - # so _initialize reads from the same directory it is configured to use. - log_dir = Path(tool_ctx.config.linux_tracing.log_dir) - log_dir.mkdir(parents=True, exist_ok=True) - - prior_cwd = str(tmp_path / "prior-pipeline-clone") - # Use a current timestamp so it falls within _initialize's 24-hour recovery window - now_ts = datetime.now(UTC).isoformat() - - # Write sessions from a PRIOR pipeline (different cwd) using direct file writes - session_dir = log_dir / "sessions" / "sess-prior" - session_dir.mkdir(parents=True) - (session_dir / "token_usage.json").write_text( - json.dumps( - { - "step_name": "implement", - "input_tokens": 9999, - "output_tokens": 4444, - "cache_creation_input_tokens": 0, - "cache_read_input_tokens": 0, - "timing_seconds": 120.0, - } - ) - ) - with (log_dir / "sessions.jsonl").open("a") as f: - f.write( - json.dumps({"dir_name": "sess-prior", "timestamp": now_ts, "cwd": prior_cwd}) + "\n" - ) - - # Simulate server startup with cross-pipeline sessions in the log dir - with patch("autoskillit.execution.recover_crashed_sessions", return_value=0): - _initialize(tool_ctx) - # Register the _ctx mutation via monkeypatch so teardown is deterministic - # rather than relying on coincidental identity with the tool_ctx fixture's patch. - monkeypatch.setattr(_state, "_ctx", tool_ctx) - - # Now simulate the CURRENT pipeline recording its own step - tool_ctx.token_log.record( - "rectify", - { - "input_tokens": 100, - "output_tokens": 50, - "cache_creation_input_tokens": 0, - "cache_read_input_tokens": 0, - }, - ) - - # get_token_summary uses _get_ctx() internally — tool_ctx fixture wires _ctx correctly - result_json = await get_token_summary(clear=False, format="json") - result = json.loads(result_json) - step_names = [s["step_name"] for s in result["steps"]] - - assert "implement" not in step_names, ( - "Prior pipeline step 'implement' must not appear in current pipeline summary; " - "cross-pipeline contamination detected" - ) - assert "rectify" in step_names, "Current pipeline step must appear" - assert len(step_names) == 1, f"Expected only current pipeline steps, got: {step_names}" - - -class TestOrderIdFilterOnSummaryTools: - """Group D: order_id filter params on get_token_summary and get_timing_summary.""" - - @pytest.mark.anyio - async def test_get_token_summary_order_id_filter_isolates_issue( - self, tool_ctx, monkeypatch - ) -> None: - """D-1: get_token_summary(order_id='issue-185') returns only that order's steps.""" - from autoskillit.server import _state - - monkeypatch.setattr(_state, "_ctx", tool_ctx) - - usage = { - "input_tokens": 100, - "output_tokens": 50, - "cache_creation_input_tokens": 0, - "cache_read_input_tokens": 0, - } - tool_ctx.token_log.record("plan", usage, order_id="issue-185") - tool_ctx.token_log.record("implement", usage, order_id="issue-186") - - result = json.loads(await get_token_summary(order_id="issue-185")) - step_names = [s["step_name"] for s in result["steps"]] - assert "plan" in step_names - assert "implement" not in step_names - - @pytest.mark.anyio - async def test_get_token_summary_no_order_id_returns_all(self, tool_ctx, monkeypatch) -> None: - """D-2: get_token_summary() without order_id returns aggregated data for all orders.""" - from autoskillit.server import _state - - monkeypatch.setattr(_state, "_ctx", tool_ctx) - - usage = { - "input_tokens": 100, - "output_tokens": 50, - "cache_creation_input_tokens": 0, - "cache_read_input_tokens": 0, - } - tool_ctx.token_log.record("plan", usage, order_id="issue-185") - tool_ctx.token_log.record("implement", usage, order_id="issue-186") - - result = json.loads(await get_token_summary()) - step_names = [s["step_name"] for s in result["steps"]] - assert "plan" in step_names - assert "implement" in step_names - - @pytest.mark.anyio - async def test_get_timing_summary_order_id_filter_isolates_issue( - self, tool_ctx, monkeypatch - ) -> None: - """D-3: get_timing_summary(order_id='issue-185') returns only that order's steps.""" - from autoskillit.server import _state - from autoskillit.server.tools_status import get_timing_summary - - monkeypatch.setattr(_state, "_ctx", tool_ctx) - - tool_ctx.timing_log.record("plan", 10.0, order_id="issue-185") - tool_ctx.timing_log.record("implement", 20.0, order_id="issue-186") - - result = json.loads(await get_timing_summary(order_id="issue-185")) - step_names = [s["step_name"] for s in result["steps"]] - assert "plan" in step_names - assert "implement" not in step_names - - -# --------------------------------------------------------------------------- -# TestReadDb — moved from test_tools_workspace.py (tools_status owns read_db) -# --------------------------------------------------------------------------- - - -@pytest.mark.usefixtures("tool_ctx") -class TestReadDb: - """Integration tests for read_db tool with real SQLite databases.""" - - @pytest.fixture - def sample_db(self, tmp_path): - """Create a sample SQLite database for testing.""" - import sqlite3 - - db = tmp_path / "test.db" - conn = sqlite3.connect(str(db)) - conn.execute("CREATE TABLE users (id INTEGER, name TEXT, age INTEGER)") - conn.execute("INSERT INTO users VALUES (1, 'Alice', 30)") - conn.execute("INSERT INTO users VALUES (2, 'Bob', 25)") - conn.execute("INSERT INTO users VALUES (3, 'Charlie', 35)") - conn.commit() - conn.close() - return db - - @pytest.mark.anyio - async def test_simple_select(self, sample_db): - result = json.loads(await read_db(db_path=str(sample_db), query="SELECT * FROM users")) - assert result["row_count"] == 3 - assert result["column_names"] == ["id", "name", "age"] - assert len(result["rows"]) == 3 - assert result["truncated"] is False - - @pytest.mark.anyio - async def test_parameterized_query(self, sample_db): - result = json.loads( - await read_db( - db_path=str(sample_db), - query="SELECT name FROM users WHERE age > ?", - params="[28]", - ) - ) - assert result["row_count"] == 2 - names = [r["name"] for r in result["rows"]] - assert "Alice" in names - assert "Charlie" in names - - @pytest.mark.anyio - async def test_named_params(self, sample_db): - result = json.loads( - await read_db( - db_path=str(sample_db), - query="SELECT name FROM users WHERE age = :age", - params='{"age": 25}', - ) - ) - assert result["row_count"] == 1 - assert result["rows"][0]["name"] == "Bob" - - @pytest.mark.anyio - async def test_empty_result(self, sample_db): - result = json.loads( - await read_db( - db_path=str(sample_db), - query="SELECT * FROM users WHERE age > 100", - ) - ) - assert result["row_count"] == 0 - assert result["rows"] == [] - assert result["column_names"] == ["id", "name", "age"] - - @pytest.mark.anyio - async def test_rejects_insert(self, sample_db): - result = json.loads( - await read_db( - db_path=str(sample_db), - query="INSERT INTO users VALUES (4, 'Dave', 40)", - ) - ) - assert "error" in result - err_lower = result["error"].lower() - assert "forbidden" in err_lower or "select" in err_lower or "not authorized" in err_lower - - @pytest.mark.anyio - async def test_rejects_drop(self, sample_db): - result = json.loads( - await read_db( - db_path=str(sample_db), - query="DROP TABLE users", - ) - ) - assert "error" in result - - @pytest.mark.anyio - async def test_rejects_attach(self, sample_db): - result = json.loads( - await read_db( - db_path=str(sample_db), - query="ATTACH DATABASE ':memory:' AS other", - ) - ) - assert "error" in result - - @pytest.mark.anyio - async def test_nonexistent_db(self, tmp_path): - result = json.loads( - await read_db( - db_path=str(tmp_path / "nonexistent.db"), - query="SELECT 1", - ) - ) - assert "error" in result - assert "does not exist" in result["error"] or "not found" in result["error"].lower() - - @pytest.mark.anyio - async def test_not_a_file(self, tmp_path): - result = json.loads( - await read_db( - db_path=str(tmp_path), - query="SELECT 1", - ) - ) - assert "error" in result - - @pytest.mark.anyio - async def test_invalid_params_json(self, sample_db): - result = json.loads( - await read_db( - db_path=str(sample_db), - query="SELECT * FROM users", - params="not json", - ) - ) - assert "error" in result - assert "params" in result["error"].lower() - - @pytest.mark.anyio - async def test_gated_when_disabled(self, sample_db, tool_ctx): - tool_ctx.gate = DefaultGateState(enabled=False) - result = json.loads( - await read_db( - db_path=str(sample_db), - query="SELECT 1", - ) - ) - assert result["success"] is False - assert result["is_error"] is True - assert "not enabled" in result["result"].lower() - - @pytest.mark.anyio - async def test_max_rows_truncation(self, sample_db, tool_ctx): - tool_ctx.config = AutomationConfig(read_db=ReadDbConfig(max_rows=2)) - result = json.loads( - await read_db( - db_path=str(sample_db), - query="SELECT * FROM users", - ) - ) - assert result["row_count"] == 2 - assert result["truncated"] is True - - @pytest.mark.anyio - async def test_blob_base64_encoding(self, tmp_path): - import base64 - import sqlite3 - - db = tmp_path / "blob.db" - conn = sqlite3.connect(str(db)) - conn.execute("CREATE TABLE data (id INTEGER, content BLOB)") - conn.execute("INSERT INTO data VALUES (1, ?)", (b"\x00\x01\x02\xff",)) - conn.commit() - conn.close() - result = json.loads( - await read_db( - db_path=str(db), - query="SELECT * FROM data", - ) - ) - assert base64.b64decode(result["rows"][0]["content"]) == b"\x00\x01\x02\xff" - - @pytest.mark.anyio - async def test_query_timeout(self, sample_db, tool_ctx): - tool_ctx.config = AutomationConfig(read_db=ReadDbConfig(timeout=1)) - # Cross join 3 rows^18 = ~387 million rows — guaranteed to exceed 1s timeout - slow_query = "SELECT count(*) FROM " + ", ".join(f"users t{i}" for i in range(18)) - result = json.loads( - await read_db( - db_path=str(sample_db), - query=slow_query, - ) - ) - assert "error" in result - assert "timeout" in result["error"].lower() - - @pytest.mark.anyio - async def test_sql_error_returns_error(self, sample_db): - result = json.loads( - await read_db( - db_path=str(sample_db), - query="SELECT nonexistent_column FROM users", - ) - ) - assert "error" in result - - -@pytest.mark.anyio -async def test_tools_status_routes_through_db_reader(tool_ctx, tmp_path) -> None: - """read_db routes through ctx.db_reader.query().""" - import sqlite3 as _sqlite3 - - tool_ctx.db_reader = MagicMock() - tool_ctx.db_reader.query.return_value = {"rows": [], "count": 0} - - db_path = str(tmp_path / "test.db") - # Create an empty sqlite db so path-exists check passes - _sqlite3.connect(db_path).close() - await read_db(db_path, "SELECT 1") - tool_ctx.db_reader.query.assert_called_once() - call_kwargs = tool_ctx.db_reader.query.call_args - assert "SELECT 1" in str(call_kwargs) - - -# T3 — read_db error paths include success: False -# --------------------------------------------------------------------------- - - -@pytest.mark.anyio -async def test_read_db_all_error_paths_include_success_false(tool_ctx, monkeypatch): - """Every read_db error path must return success=False.""" - from autoskillit.server import _state - - monkeypatch.setattr(_state, "_ctx", tool_ctx) - - # Nonexistent db path — exercises the "does not exist" error branch - result = json.loads(await read_db(db_path="/nonexistent/path/db.sqlite", query="SELECT 1")) - assert result.get("success") is False - assert "error" in result diff --git a/tests/server/test_tools_status_kitchen.py b/tests/server/test_tools_status_kitchen.py new file mode 100644 index 0000000000..f201b543b6 --- /dev/null +++ b/tests/server/test_tools_status_kitchen.py @@ -0,0 +1,324 @@ +"""Tests for server status tools: kitchen status, pipeline report, and telemetry recovery.""" + +from __future__ import annotations + +import asyncio +import json +from pathlib import Path + +import anyio +import pytest + +from autoskillit.config import AutomationConfig, TokenUsageConfig +from autoskillit.core.types import ChannelConfirmation +from autoskillit.execution.github import DefaultGitHubFetcher +from autoskillit.pipeline.audit import FailureRecord +from autoskillit.pipeline.gate import DefaultGateState +from autoskillit.server.tools.tools_execution import run_skill +from autoskillit.server.tools.tools_status import ( + get_pipeline_report, + get_timing_summary, + get_token_summary, + kitchen_status, +) +from tests.conftest import _make_result + +pytestmark = [pytest.mark.layer("server"), pytest.mark.small] + + +def _make_failure_record(**overrides: object) -> FailureRecord: + defaults = dict( + timestamp="2026-02-24T16:00:00Z", + skill_command="/autoskillit:implement-worktree", + exit_code=1, + subtype="error", + needs_retry=False, + retry_reason="none", + stderr="something went wrong", + ) + return FailureRecord(**{**defaults, **overrides}) # type: ignore[arg-type] + + +def _failed_session_json() -> str: + return json.dumps( + { + "type": "result", + "subtype": "error", + "result": "Task failed with an error", + "session_id": "test-session", + "is_error": True, + } + ) + + +class TestKitchenStatus: + """kitchen_status tool returns version health info.""" + + @pytest.fixture(autouse=True) + def _setup(self, tool_ctx_kitchen_open, monkeypatch, tmp_path): + monkeypatch.chdir(tmp_path) + + @pytest.mark.anyio + async def test_status_returns_version_info(self, tool_ctx_kitchen_open): + import autoskillit + from autoskillit.core.types._type_plugin_source import DirectInstall + + tool_ctx_kitchen_open.plugin_source = DirectInstall( + plugin_dir=Path(autoskillit.__file__).parent + ) + from autoskillit import __version__ + + result = json.loads(await kitchen_status()) + assert result["package_version"] == __version__ + assert result["plugin_json_version"] == __version__ + assert result["versions_match"] is True + assert "warning" not in result + + @pytest.mark.anyio + async def test_status_reports_mismatch(self, tmp_path, tool_ctx_kitchen_open): + plugin_dir = tmp_path / ".claude-plugin" + plugin_dir.mkdir() + (plugin_dir / "plugin.json").write_text( + json.dumps({"name": "autoskillit", "version": "0.0.0"}) + ) + from autoskillit.core.types._type_plugin_source import DirectInstall + + tool_ctx_kitchen_open.plugin_source = DirectInstall(plugin_dir=tmp_path) + result = json.loads(await kitchen_status()) + assert result["versions_match"] is False + assert "warning" in result + assert "mismatch" in result["warning"].lower() + + @pytest.mark.anyio + async def test_status_includes_token_usage_verbosity_default(self): + """TU_S1: kitchen_status includes token_usage_verbosity key with default 'summary'.""" + result = json.loads(await kitchen_status()) + assert "token_usage_verbosity" in result + assert result["token_usage_verbosity"] == "summary" + + @pytest.mark.anyio + async def test_status_reflects_none_verbosity(self, tool_ctx_kitchen_open): + """TU_S2: kitchen_status reflects 'none' verbosity from config.""" + cfg = AutomationConfig() + cfg.token_usage = TokenUsageConfig(verbosity="none") + tool_ctx_kitchen_open.config = cfg + result = json.loads(await kitchen_status()) + assert result["token_usage_verbosity"] == "none" + + @pytest.mark.anyio + async def test_kitchen_status_includes_github_config(self, tool_ctx_kitchen_open): + tool_ctx_kitchen_open.config.github.default_repo = "owner/repo" + status = json.loads(await kitchen_status()) + assert "github_default_repo" in status + assert status["github_default_repo"] == "owner/repo" + assert "github_token_configured" in status + + @pytest.mark.anyio + async def test_kitchen_status_github_token_configured_true_from_client( + self, tool_ctx_kitchen_open + ): + """kitchen_status must read github_token_configured from ctx.github_client.has_token.""" + tool_ctx_kitchen_open.github_client = DefaultGitHubFetcher(token="my-token") + status = json.loads(await kitchen_status()) + assert status["github_token_configured"] is True + + @pytest.mark.anyio + async def test_kitchen_status_github_token_not_configured_from_client( + self, tool_ctx_kitchen_open, monkeypatch + ): + monkeypatch.delenv("GITHUB_TOKEN", raising=False) + tool_ctx_kitchen_open.github_client = DefaultGitHubFetcher(token=None) + status = json.loads(await kitchen_status()) + assert status["github_token_configured"] is False + + @pytest.mark.anyio + async def test_kitchen_status_github_token_does_not_reflect_post_construction_env( + self, tool_ctx_kitchen_open, monkeypatch + ): + """kitchen_status must NOT re-read os.environ — reflects ctx.github_client.has_token.""" + tool_ctx_kitchen_open.github_client = DefaultGitHubFetcher(token=None) + monkeypatch.setenv("GITHUB_TOKEN", "set-after-construction") + status = json.loads(await kitchen_status()) + assert status["github_token_configured"] is False + + @pytest.mark.anyio + async def test_kitchen_status_has_no_gate_file_exists_field(self, tool_ctx): + """kitchen_status must not report gate_file_exists.""" + result = json.loads(await kitchen_status()) + assert "gate_file_exists" not in result + + @pytest.mark.anyio + async def test_kitchen_status_has_no_split_brain_warning(self, tool_ctx, tmp_path): + """A file at the old gate path must not trigger any warning in kitchen_status.""" + gate_dir = tmp_path / ".autoskillit" / "temp" + gate_dir.mkdir(parents=True) + (gate_dir / ".kitchen_gate").write_text("{}") + result_str = json.dumps(json.loads(await kitchen_status())) + assert "stale" not in result_str.lower() + assert "gate_file" not in result_str.lower() + + +class TestGetPipelineReport: + """get_pipeline_report is a gated tool that returns accumulated failures.""" + + @pytest.mark.anyio + async def test_gate_closed_returns_gate_error(self, tool_ctx): + """get_pipeline_report returns gate_error when kitchen gate is closed.""" + tool_ctx.gate = DefaultGateState(enabled=False) + result = json.loads(await get_pipeline_report()) + assert result.get("success") is False + assert result.get("subtype") == "gate_error" + + @pytest.mark.anyio + async def test_returns_empty_initially(self, tool_ctx_kitchen_open): + result = json.loads(await get_pipeline_report()) + assert result["total_failures"] == 0 + assert result["failures"] == [] + + @pytest.mark.anyio + async def test_accumulates_failures_from_run_skill(self, tool_ctx): + + tool_ctx.gate = DefaultGateState(enabled=True) + tool_ctx.runner.push( + _make_result( + returncode=1, + stdout=_failed_session_json(), + channel_confirmation=ChannelConfirmation.UNMONITORED, + ) + ) + await run_skill(skill_command="/autoskillit:test", cwd="/tmp") + result = json.loads(await get_pipeline_report()) + assert result["total_failures"] == 1 + assert result["failures"][0]["skill_command"].startswith("/autoskillit:test") + + @pytest.mark.anyio + async def test_clear_true_resets_after_returning(self, tool_ctx_kitchen_open): + tool_ctx_kitchen_open.audit.record_failure(_make_failure_record()) + result = json.loads(await get_pipeline_report(clear=True)) + assert result["total_failures"] == 1 + result2 = json.loads(await get_pipeline_report()) + assert result2["total_failures"] == 0 + + @pytest.mark.anyio + async def test_get_pipeline_report_awaits_startup_ready( + self, tool_ctx_kitchen_open, monkeypatch + ): + """get_pipeline_report must await _startup_ready before accessing audit data.""" + import autoskillit.server._state as _state_mod + + ready = asyncio.Event() + monkeypatch.setattr(_state_mod, "_startup_ready", ready) + + task = asyncio.create_task(get_pipeline_report()) + await asyncio.sleep(0.05) + assert not task.done(), "get_pipeline_report returned before _startup_ready was set" + + ready.set() + await asyncio.sleep(0.05) + assert task.done(), "get_pipeline_report did not unblock after _startup_ready was set" + + +def test_check_quota_absent_from_mcp_registry(tool_ctx): + """check_quota must not appear in the agent-visible tool list. + Quota enforcement is the PreToolUse hook's responsibility, not an MCP tool.""" + from autoskillit.server import mcp + + tools = anyio.run(mcp.list_tools) + assert "check_quota" not in {t.name for t in tools}, ( + "check_quota must be removed from the MCP registry. " + "Agents should not call it — the PreToolUse hook enforces quota automatically." + ) + + +class TestTelemetryRecoveryData: + """MCP status tools return data populated via load_from_log_dir recovery.""" + + def _write_token_session( + self, log_root: Path, dir_name: str, step_name: str, input_tokens: int + ) -> None: + session_dir = log_root / "sessions" / dir_name + session_dir.mkdir(parents=True, exist_ok=True) + tu = { + "session_label": step_name, + "input_tokens": input_tokens, + "output_tokens": 50, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + "timing_seconds": 10.0, + } + (session_dir / "token_usage.json").write_text(json.dumps(tu)) + idx = { + "dir_name": dir_name, + "timestamp": "2026-03-07T00:00:00+00:00", + "session_id": dir_name, + } + with (log_root / "sessions.jsonl").open("a") as f: + f.write(json.dumps(idx) + "\n") + + def _write_timing_session( + self, log_root: Path, dir_name: str, step_name: str, total_seconds: float + ) -> None: + session_dir = log_root / "sessions" / dir_name + session_dir.mkdir(parents=True, exist_ok=True) + st = {"step_name": step_name, "total_seconds": total_seconds} + (session_dir / "step_timing.json").write_text(json.dumps(st)) + idx = { + "dir_name": dir_name, + "timestamp": "2026-03-07T00:00:00+00:00", + "session_id": dir_name, + } + with (log_root / "sessions.jsonl").open("a") as f: + f.write(json.dumps(idx) + "\n") + + def _write_audit_session(self, log_root: Path, dir_name: str) -> None: + session_dir = log_root / "sessions" / dir_name + session_dir.mkdir(parents=True, exist_ok=True) + record = { + "timestamp": "2026-03-07T00:00:00Z", + "skill_command": "/autoskillit:implement-worktree", + "exit_code": 1, + "subtype": "error", + "needs_retry": False, + "retry_reason": "none", + "stderr": "oops", + } + (session_dir / "audit_log.json").write_text(json.dumps([record])) + idx = { + "dir_name": dir_name, + "timestamp": "2026-03-07T00:00:00+00:00", + "session_id": dir_name, + } + with (log_root / "sessions.jsonl").open("a") as f: + f.write(json.dumps(idx) + "\n") + + @pytest.mark.anyio + async def test_token_summary_reflects_recovered_data(self, tool_ctx_kitchen_open, tmp_path): + """get_token_summary returns data loaded via load_from_log_dir.""" + log_root = tmp_path / "logs" + self._write_token_session(log_root, "s001", "implement", 500) + tool_ctx_kitchen_open.token_log.load_from_log_dir(log_root) + result = json.loads(await get_token_summary()) + steps = {s["step_name"]: s for s in result["steps"]} + assert "implement" in steps + assert steps["implement"]["input_tokens"] == 500 + + @pytest.mark.anyio + async def test_timing_summary_reflects_recovered_data(self, tool_ctx_kitchen_open, tmp_path): + """get_timing_summary returns data loaded via load_from_log_dir.""" + log_root = tmp_path / "logs" + self._write_timing_session(log_root, "s001", "plan", 99.0) + tool_ctx_kitchen_open.timing_log.load_from_log_dir(log_root) + result = json.loads(await get_timing_summary()) + steps = {s["step_name"]: s for s in result["steps"]} + assert "plan" in steps + assert steps["plan"]["total_seconds"] == pytest.approx(99.0) + + @pytest.mark.anyio + async def test_pipeline_report_reflects_recovered_audit(self, tool_ctx_kitchen_open, tmp_path): + """get_pipeline_report returns failures loaded via load_from_log_dir.""" + log_root = tmp_path / "logs" + self._write_audit_session(log_root, "s001") + tool_ctx_kitchen_open.audit.load_from_log_dir(log_root) + result = json.loads(await get_pipeline_report()) + assert result["total_failures"] == 1 + assert result["failures"][0]["skill_command"] == "/autoskillit:implement-worktree" diff --git a/tests/server/test_tools_status_mcp_response.py b/tests/server/test_tools_status_mcp_response.py index 586e2284cb..a1637987f8 100644 --- a/tests/server/test_tools_status_mcp_response.py +++ b/tests/server/test_tools_status_mcp_response.py @@ -7,14 +7,16 @@ import pytest -from autoskillit.server.tools_status import get_token_summary, write_telemetry_files +from autoskillit.server.tools.tools_status import get_token_summary, write_telemetry_files + +pytestmark = [pytest.mark.layer("server"), pytest.mark.small] class TestGetTokenSummaryMcpResponses: @pytest.mark.anyio - async def test_includes_mcp_responses_section(self, tool_ctx): + async def test_includes_mcp_responses_section(self, tool_ctx_kitchen_open): """get_token_summary returns an mcp_responses key with per-tool data.""" - tool_ctx.response_log.record("run_skill", "a" * 400) + tool_ctx_kitchen_open.response_log.record("run_skill", "a" * 400) result = json.loads(await get_token_summary()) assert "mcp_responses" in result @@ -23,9 +25,9 @@ async def test_includes_mcp_responses_section(self, tool_ctx): assert result["mcp_responses"]["steps"][0]["tool_name"] == "run_skill" @pytest.mark.anyio - async def test_mcp_responses_steps_contain_expected_fields(self, tool_ctx): + async def test_mcp_responses_steps_contain_expected_fields(self, tool_ctx_kitchen_open): """Each step in mcp_responses has the expected fields.""" - tool_ctx.response_log.record("load_recipe", "x" * 800) + tool_ctx_kitchen_open.response_log.record("load_recipe", "x" * 800) result = json.loads(await get_token_summary()) step = result["mcp_responses"]["steps"][0] @@ -35,10 +37,10 @@ async def test_mcp_responses_steps_contain_expected_fields(self, tool_ctx): assert step["invocation_count"] == 1 @pytest.mark.anyio - async def test_mcp_responses_total_fields(self, tool_ctx): + async def test_mcp_responses_total_fields(self, tool_ctx_kitchen_open): """mcp_responses.total contains aggregated byte and token counts.""" - tool_ctx.response_log.record("run_skill", "a" * 400) - tool_ctx.response_log.record("load_recipe", "b" * 800) + tool_ctx_kitchen_open.response_log.record("run_skill", "a" * 400) + tool_ctx_kitchen_open.response_log.record("load_recipe", "b" * 800) result = json.loads(await get_token_summary()) total = result["mcp_responses"]["total"] @@ -47,14 +49,14 @@ async def test_mcp_responses_total_fields(self, tool_ctx): assert total["total_invocations"] == 2 @pytest.mark.anyio - async def test_mcp_responses_empty_when_no_responses_recorded(self, tool_ctx): + async def test_mcp_responses_empty_when_no_responses_recorded(self, tool_ctx_kitchen_open): """mcp_responses section returns empty steps when no responses recorded.""" result = json.loads(await get_token_summary()) assert result["mcp_responses"]["steps"] == [] assert result["mcp_responses"]["total"]["total_invocations"] == 0 @pytest.mark.anyio - async def test_clear_true_also_clears_response_log(self, tool_ctx): + async def test_clear_true_also_clears_response_log(self, tool_ctx_kitchen_open): """When clear=True, response_log is cleared alongside token_log. Note: @track_response_size records the get_token_summary response itself @@ -62,7 +64,7 @@ async def test_clear_true_also_clears_response_log(self, tool_ctx): have a 'get_token_summary' entry from the decorator, but 'run_skill' (recorded before clear) must be absent. """ - tool_ctx.response_log.record("run_skill", "data" * 100) + tool_ctx_kitchen_open.response_log.record("run_skill", "data" * 100) result = json.loads(await get_token_summary(clear=True)) names_before = {s["tool_name"] for s in result["mcp_responses"]["steps"]} @@ -75,9 +77,9 @@ async def test_clear_true_also_clears_response_log(self, tool_ctx): class TestWriteTelemetryFilesMcpResponse: @pytest.mark.anyio - async def test_writes_mcp_response_metrics_json(self, tool_ctx, tmp_path): + async def test_writes_mcp_response_metrics_json(self, tool_ctx_kitchen_open, tmp_path): """write_telemetry_files writes mcp_response_metrics.json alongside token_summary.md.""" - tool_ctx.response_log.record("run_skill", "x" * 800) + tool_ctx_kitchen_open.response_log.record("run_skill", "x" * 800) result = json.loads(await write_telemetry_files(str(tmp_path))) @@ -89,7 +91,7 @@ async def test_writes_mcp_response_metrics_json(self, tool_ctx, tmp_path): assert data["steps"][0]["tool_name"] == "run_skill" @pytest.mark.anyio - async def test_mcp_response_metrics_path_key_in_result(self, tool_ctx, tmp_path): + async def test_mcp_response_metrics_path_key_in_result(self, tool_ctx_kitchen_open, tmp_path): """write_telemetry_files returns mcp_response_metrics_path alongside existing keys.""" result = json.loads(await write_telemetry_files(str(tmp_path))) assert "token_summary_path" in result @@ -97,10 +99,10 @@ async def test_mcp_response_metrics_path_key_in_result(self, tool_ctx, tmp_path) assert "mcp_response_metrics_path" in result @pytest.mark.anyio - async def test_mcp_response_metrics_json_structure(self, tool_ctx, tmp_path): + async def test_mcp_response_metrics_json_structure(self, tool_ctx_kitchen_open, tmp_path): """mcp_response_metrics.json has steps list and total dict.""" - tool_ctx.response_log.record("run_skill", "a" * 400) - tool_ctx.response_log.record("load_recipe", "b" * 800) + tool_ctx_kitchen_open.response_log.record("run_skill", "a" * 400) + tool_ctx_kitchen_open.response_log.record("load_recipe", "b" * 800) result = json.loads(await write_telemetry_files(str(tmp_path))) metrics_path = Path(result["mcp_response_metrics_path"]) @@ -111,7 +113,7 @@ async def test_mcp_response_metrics_json_structure(self, tool_ctx, tmp_path): assert data["total"]["total_response_bytes"] == 1200 @pytest.mark.anyio - async def test_existing_keys_unchanged(self, tool_ctx, tmp_path): + async def test_existing_keys_unchanged(self, tool_ctx_kitchen_open, tmp_path): """token_summary_path and timing_summary_path keys remain correct in result.""" result = json.loads(await write_telemetry_files(str(tmp_path))) assert Path(result["token_summary_path"]).name == "token_summary.md" diff --git a/tests/server/test_tools_status_quota_and_db.py b/tests/server/test_tools_status_quota_and_db.py new file mode 100644 index 0000000000..a848b79e2d --- /dev/null +++ b/tests/server/test_tools_status_quota_and_db.py @@ -0,0 +1,385 @@ +"""Tests for autoskillit server status tools: quota events, telemetry writing, and DB access.""" + +from __future__ import annotations + +import json +from pathlib import Path + +import pytest + +from autoskillit.config import AutomationConfig, ReadDbConfig +from autoskillit.pipeline.gate import DefaultGateState +from autoskillit.server.tools.tools_status import ( + get_quota_events, + read_db, + write_telemetry_files, +) + +pytestmark = [pytest.mark.layer("server"), pytest.mark.small] + + +class TestGetQuotaEvents: + @pytest.mark.anyio + async def test_returns_events_from_jsonl(self, tool_ctx_kitchen_open, tmp_path, monkeypatch): + events = [ + { + "ts": "2026-03-10T10:00:00+00:00", + "event": "approved", + "threshold": 85.0, + "utilization": 50.0, + }, + { + "ts": "2026-03-10T11:00:00+00:00", + "event": "blocked", + "threshold": 85.0, + "utilization": 92.5, + }, + ] + log_dir = tmp_path / "logs" + log_dir.mkdir() + (log_dir / "quota_events.jsonl").write_text( + "\n".join(json.dumps(e) for e in events) + "\n" + ) + monkeypatch.setattr(tool_ctx_kitchen_open.config.linux_tracing, "log_dir", str(log_dir)) + result = json.loads(await get_quota_events()) + assert result["total_count"] == 2 + assert result["events"][0]["event"] == "blocked" # most recent first + + @pytest.mark.anyio + async def test_limits_to_n_events(self, tool_ctx_kitchen_open, tmp_path, monkeypatch): + log_dir = tmp_path / "logs" + log_dir.mkdir() + lines = [ + json.dumps({"ts": f"2026-03-10T{h:02d}:00:00+00:00", "event": "approved"}) + for h in range(10) + ] + (log_dir / "quota_events.jsonl").write_text("\n".join(lines) + "\n") + monkeypatch.setattr(tool_ctx_kitchen_open.config.linux_tracing, "log_dir", str(log_dir)) + result = json.loads(await get_quota_events(n=3)) + assert len(result["events"]) == 3 + assert result["total_count"] == 10 + # most-recent-first: hours 09, 08, 07 + assert result["events"][0]["ts"] == "2026-03-10T09:00:00+00:00" + assert result["events"][1]["ts"] == "2026-03-10T08:00:00+00:00" + assert result["events"][2]["ts"] == "2026-03-10T07:00:00+00:00" + + @pytest.mark.anyio + async def test_returns_empty_when_file_missing( + self, tool_ctx_kitchen_open, tmp_path, monkeypatch + ): + log_dir = tmp_path / "logs" + log_dir.mkdir() + monkeypatch.setattr(tool_ctx_kitchen_open.config.linux_tracing, "log_dir", str(log_dir)) + result = json.loads(await get_quota_events()) + assert result["events"] == [] + assert result["total_count"] == 0 + + +class TestWriteTelemetryFiles: + @pytest.mark.anyio + async def test_writes_token_summary_markdown(self, tool_ctx_kitchen_open, tmp_path): + tool_ctx_kitchen_open.token_log.record( + "step1", + { + "input_tokens": 100, + "output_tokens": 50, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + }, + ) + result = json.loads(await write_telemetry_files(str(tmp_path))) + path = Path(result["token_summary_path"]) + assert path.exists() + content = path.read_text() + assert "step1" in content + # Format-structural assertions (table, not bullet list) + assert "| Step |" in content + assert "|---" in content + assert "- input_tokens:" not in content + assert "# Token Summary" not in content + + @pytest.mark.anyio + async def test_writes_timing_summary_markdown(self, tool_ctx_kitchen_open, tmp_path): + tool_ctx_kitchen_open.timing_log.record("step1", 12.5) + result = json.loads(await write_telemetry_files(str(tmp_path))) + path = Path(result["timing_summary_path"]) + assert path.exists() + content = path.read_text() + assert "step1" in content + # Format-structural assertions (table, not bullet list) + assert "| Step |" in content + assert "|---" in content + assert "- total_seconds:" not in content + assert "# Timing Summary" not in content + + @pytest.mark.anyio + async def test_token_file_uses_wall_clock_seconds(self, tool_ctx_kitchen_open, tmp_path): + """write_telemetry_files merges wall_clock_seconds from timing log.""" + tool_ctx_kitchen_open.token_log.record( + "deploy", + { + "input_tokens": 100, + "output_tokens": 50, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + }, + elapsed_seconds=5.0, + ) + tool_ctx_kitchen_open.timing_log.record("deploy", 120.0) + result = json.loads(await write_telemetry_files(str(tmp_path))) + content = Path(result["token_summary_path"]).read_text() + # Should show 2m 0s (wall_clock=120), not 5s (elapsed) + assert "2m 0s" in content + + @pytest.mark.anyio + async def test_creates_output_dir_if_missing(self, tool_ctx_kitchen_open, tmp_path): + out = str(tmp_path / "nested" / "telemetry") + result = json.loads(await write_telemetry_files(out)) + assert Path(result["token_summary_path"]).exists() + assert Path(result["timing_summary_path"]).exists() + + @pytest.mark.anyio + async def test_gate_closed_returns_gate_error(self, tool_ctx, tmp_path): + tool_ctx.gate.disable() + result = json.loads(await write_telemetry_files(str(tmp_path))) + assert result["success"] is False + assert result["subtype"] == "gate_error" + + +# --------------------------------------------------------------------------- +# TestReadDb — moved from test_tools_workspace.py (tools_status owns read_db) +# --------------------------------------------------------------------------- + + +@pytest.mark.usefixtures("tool_ctx_kitchen_open") +class TestReadDb: + """Integration tests for read_db tool with real SQLite databases.""" + + @pytest.fixture + def sample_db(self, tmp_path): + """Create a sample SQLite database for testing.""" + import sqlite3 + + db = tmp_path / "test.db" + conn = sqlite3.connect(str(db)) + conn.execute("CREATE TABLE users (id INTEGER, name TEXT, age INTEGER)") + conn.execute("INSERT INTO users VALUES (1, 'Alice', 30)") + conn.execute("INSERT INTO users VALUES (2, 'Bob', 25)") + conn.execute("INSERT INTO users VALUES (3, 'Charlie', 35)") + conn.commit() + conn.close() + return db + + @pytest.mark.anyio + async def test_simple_select(self, sample_db): + result = json.loads(await read_db(db_path=str(sample_db), query="SELECT * FROM users")) + assert result["row_count"] == 3 + assert result["column_names"] == ["id", "name", "age"] + assert len(result["rows"]) == 3 + assert result["truncated"] is False + + @pytest.mark.anyio + async def test_parameterized_query(self, sample_db): + result = json.loads( + await read_db( + db_path=str(sample_db), + query="SELECT name FROM users WHERE age > ?", + params="[28]", + ) + ) + assert result["row_count"] == 2 + names = [r["name"] for r in result["rows"]] + assert "Alice" in names + assert "Charlie" in names + + @pytest.mark.anyio + async def test_named_params(self, sample_db): + result = json.loads( + await read_db( + db_path=str(sample_db), + query="SELECT name FROM users WHERE age = :age", + params='{"age": 25}', + ) + ) + assert result["row_count"] == 1 + assert result["rows"][0]["name"] == "Bob" + + @pytest.mark.anyio + async def test_empty_result(self, sample_db): + result = json.loads( + await read_db( + db_path=str(sample_db), + query="SELECT * FROM users WHERE age > 100", + ) + ) + assert result["row_count"] == 0 + assert result["rows"] == [] + assert result["column_names"] == ["id", "name", "age"] + + @pytest.mark.anyio + async def test_rejects_insert(self, sample_db): + result = json.loads( + await read_db( + db_path=str(sample_db), + query="INSERT INTO users VALUES (4, 'Dave', 40)", + ) + ) + assert "error" in result + err_lower = result["error"].lower() + assert "forbidden" in err_lower or "select" in err_lower or "not authorized" in err_lower + + @pytest.mark.anyio + async def test_rejects_drop(self, sample_db): + result = json.loads( + await read_db( + db_path=str(sample_db), + query="DROP TABLE users", + ) + ) + assert "error" in result + + @pytest.mark.anyio + async def test_rejects_attach(self, sample_db): + result = json.loads( + await read_db( + db_path=str(sample_db), + query="ATTACH DATABASE ':memory:' AS other", + ) + ) + assert "error" in result + + @pytest.mark.anyio + async def test_nonexistent_db(self, tmp_path): + result = json.loads( + await read_db( + db_path=str(tmp_path / "nonexistent.db"), + query="SELECT 1", + ) + ) + assert "error" in result + assert "does not exist" in result["error"] or "not found" in result["error"].lower() + + @pytest.mark.anyio + async def test_not_a_file(self, tmp_path): + result = json.loads( + await read_db( + db_path=str(tmp_path), + query="SELECT 1", + ) + ) + assert "error" in result + + @pytest.mark.anyio + async def test_invalid_params_json(self, sample_db): + result = json.loads( + await read_db( + db_path=str(sample_db), + query="SELECT * FROM users", + params="not json", + ) + ) + assert "error" in result + assert "params" in result["error"].lower() + + @pytest.mark.anyio + async def test_gated_when_disabled(self, sample_db, tool_ctx): + tool_ctx.gate = DefaultGateState(enabled=False) + result = json.loads( + await read_db( + db_path=str(sample_db), + query="SELECT 1", + ) + ) + assert result["success"] is False + assert result["is_error"] is True + assert "not enabled" in result["result"].lower() + + @pytest.mark.anyio + async def test_max_rows_truncation(self, sample_db, tool_ctx_kitchen_open): + tool_ctx_kitchen_open.config = AutomationConfig(read_db=ReadDbConfig(max_rows=2)) + result = json.loads( + await read_db( + db_path=str(sample_db), + query="SELECT * FROM users", + ) + ) + assert result["row_count"] == 2 + assert result["truncated"] is True + + @pytest.mark.anyio + async def test_blob_base64_encoding(self, tmp_path): + import base64 + import sqlite3 + + db = tmp_path / "blob.db" + conn = sqlite3.connect(str(db)) + conn.execute("CREATE TABLE data (id INTEGER, content BLOB)") + conn.execute("INSERT INTO data VALUES (1, ?)", (b"\x00\x01\x02\xff",)) + conn.commit() + conn.close() + result = json.loads( + await read_db( + db_path=str(db), + query="SELECT * FROM data", + ) + ) + assert base64.b64decode(result["rows"][0]["content"]) == b"\x00\x01\x02\xff" + + @pytest.mark.anyio + async def test_query_timeout(self, sample_db, tool_ctx_kitchen_open): + tool_ctx_kitchen_open.config = AutomationConfig(read_db=ReadDbConfig(timeout=1)) + # Cross join 3 rows^18 = ~387 million rows — guaranteed to exceed 1s timeout + slow_query = "SELECT count(*) FROM " + ", ".join(f"users t{i}" for i in range(18)) + result = json.loads( + await read_db( + db_path=str(sample_db), + query=slow_query, + ) + ) + assert "error" in result + assert "timeout" in result["error"].lower() + + @pytest.mark.anyio + async def test_sql_error_returns_error(self, sample_db): + result = json.loads( + await read_db( + db_path=str(sample_db), + query="SELECT nonexistent_column FROM users", + ) + ) + assert "error" in result + + +@pytest.mark.anyio +async def test_tools_status_routes_through_db_reader(tool_ctx_kitchen_open, tmp_path) -> None: + """read_db routes through ctx.db_reader.query().""" + import sqlite3 as _sqlite3 + + from tests.fakes import InMemoryDatabaseReader + + reader = InMemoryDatabaseReader(query_result={"rows": [], "count": 0}) + tool_ctx_kitchen_open.db_reader = reader + + db_path = str(tmp_path / "test.db") + # Create an empty sqlite db so path-exists check passes + _sqlite3.connect(db_path).close() + await read_db(db_path, "SELECT 1") + assert len(reader.calls) == 1 + assert "SELECT 1" in reader.calls[0]["sql"] + + +# T3 — read_db error paths include success: False +# --------------------------------------------------------------------------- + + +@pytest.mark.anyio +async def test_read_db_all_error_paths_include_success_false(tool_ctx_kitchen_open, monkeypatch): + """Every read_db error path must return success=False.""" + from autoskillit.server import _state + + monkeypatch.setattr(_state, "_ctx", tool_ctx_kitchen_open) + + # Nonexistent db path — exercises the "does not exist" error branch + result = json.loads(await read_db(db_path="/nonexistent/path/db.sqlite", query="SELECT 1")) + assert result.get("success") is False + assert "error" in result diff --git a/tests/server/test_tools_status_summaries.py b/tests/server/test_tools_status_summaries.py new file mode 100644 index 0000000000..51aa3b6189 --- /dev/null +++ b/tests/server/test_tools_status_summaries.py @@ -0,0 +1,579 @@ +"""Tests for autoskillit server status tools: token and timing summaries.""" + +from __future__ import annotations + +import json +from datetime import UTC, datetime +from pathlib import Path + +import pytest + +from autoskillit.pipeline.gate import DefaultGateState +from autoskillit.server.tools.tools_status import ( + get_pipeline_report, + get_timing_summary, + get_token_summary, +) + +pytestmark = [pytest.mark.layer("server"), pytest.mark.small] + + +class TestGetTokenSummary: + """get_token_summary is a gated tool that returns accumulated token usage.""" + + @pytest.mark.anyio + async def test_gate_closed_returns_gate_error(self, tool_ctx): + """get_token_summary returns gate_error when kitchen gate is closed.""" + tool_ctx.gate = DefaultGateState(enabled=False) + result = json.loads(await get_token_summary()) + assert result.get("success") is False + assert result.get("subtype") == "gate_error" + + @pytest.mark.anyio + async def test_returns_empty_steps_initially(self, tool_ctx_kitchen_open): + result = json.loads(await get_token_summary()) + assert result["steps"] == [] + assert result["total"]["input_tokens"] == 0 + assert result["total"]["output_tokens"] == 0 + assert result["total"]["cache_creation_input_tokens"] == 0 + assert result["total"]["cache_read_input_tokens"] == 0 + + @pytest.mark.anyio + async def test_returns_entry_per_step_name(self, tool_ctx_kitchen_open): + tool_ctx_kitchen_open.token_log.record( + "investigate", + { + "input_tokens": 100, + "output_tokens": 50, + "cache_creation_input_tokens": 10, + "cache_read_input_tokens": 5, + }, + ) + tool_ctx_kitchen_open.token_log.record( + "implement", + { + "input_tokens": 200, + "output_tokens": 80, + "cache_creation_input_tokens": 20, + "cache_read_input_tokens": 10, + }, + ) + result = json.loads(await get_token_summary()) + assert len(result["steps"]) == 2 + assert result["steps"][0]["step_name"] == "investigate" + assert result["steps"][1]["step_name"] == "implement" + + @pytest.mark.anyio + async def test_multiple_invocations_same_step_are_summed(self, tool_ctx_kitchen_open): + usage = { + "input_tokens": 100, + "output_tokens": 50, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + } + tool_ctx_kitchen_open.token_log.record("implement", usage) + tool_ctx_kitchen_open.token_log.record("implement", usage) + tool_ctx_kitchen_open.token_log.record("implement", usage) + result = json.loads(await get_token_summary()) + assert len(result["steps"]) == 1 + assert result["steps"][0]["input_tokens"] == 300 + assert result["steps"][0]["invocation_count"] == 3 + + @pytest.mark.anyio + async def test_total_field_sums_all_steps(self, tool_ctx_kitchen_open): + tool_ctx_kitchen_open.token_log.record( + "plan", + { + "input_tokens": 100, + "output_tokens": 50, + "cache_creation_input_tokens": 10, + "cache_read_input_tokens": 5, + }, + ) + tool_ctx_kitchen_open.token_log.record( + "implement", + { + "input_tokens": 200, + "output_tokens": 80, + "cache_creation_input_tokens": 20, + "cache_read_input_tokens": 10, + }, + ) + result = json.loads(await get_token_summary()) + assert result["total"]["input_tokens"] == 300 + assert result["total"]["output_tokens"] == 130 + assert result["total"]["cache_creation_input_tokens"] == 30 + assert result["total"]["cache_read_input_tokens"] == 15 + + @pytest.mark.anyio + async def test_clear_true_resets_after_returning(self, tool_ctx_kitchen_open): + tool_ctx_kitchen_open.token_log.record( + "plan", + { + "input_tokens": 100, + "output_tokens": 50, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + }, + ) + result = json.loads(await get_token_summary(clear=True)) + assert len(result["steps"]) == 1 + result2 = json.loads(await get_token_summary()) + assert result2["steps"] == [] + + @pytest.mark.anyio + async def test_response_shape(self, tool_ctx_kitchen_open): + tool_ctx_kitchen_open.token_log.record( + "plan", + { + "input_tokens": 10, + "output_tokens": 5, + "cache_creation_input_tokens": 1, + "cache_read_input_tokens": 2, + }, + ) + result = json.loads(await get_token_summary()) + assert "steps" in result + assert "total" in result + assert "model_totals" in result + assert isinstance(result["steps"], list) + assert isinstance(result["model_totals"], list) + assert all("model" in e for e in result["model_totals"]), ( + "each model_totals entry must have a 'model' key" + ) + total_keys = set(result["total"].keys()) + assert { + "input_tokens", + "output_tokens", + "cache_creation_input_tokens", + "cache_read_input_tokens", + } <= total_keys + + @pytest.mark.anyio + async def test_step_includes_elapsed_seconds(self, tool_ctx_kitchen_open): + """Each step dict in the response includes elapsed_seconds.""" + start = "2026-01-01T00:00:00+00:00" + end = "2026-01-01T00:00:30+00:00" + tool_ctx_kitchen_open.token_log.record( + "deploy", {"input_tokens": 200}, start_ts=start, end_ts=end + ) + result = json.loads(await get_token_summary()) + assert result["steps"][0]["elapsed_seconds"] == pytest.approx(30.0) + + @pytest.mark.anyio + async def test_total_includes_total_elapsed_seconds(self, tool_ctx_kitchen_open): + """Total dict includes total_elapsed_seconds summed across all steps.""" + tool_ctx_kitchen_open.token_log.record( + "a", + {"input_tokens": 10}, + start_ts="2026-01-01T00:00:00+00:00", + end_ts="2026-01-01T00:00:05+00:00", + ) + tool_ctx_kitchen_open.token_log.record( + "b", + {"input_tokens": 20}, + start_ts="2026-01-01T00:01:00+00:00", + end_ts="2026-01-01T00:01:08+00:00", + ) + result = json.loads(await get_token_summary()) + assert "total_elapsed_seconds" in result["total"] + assert result["total"]["total_elapsed_seconds"] == pytest.approx(13.0) + + +class TestGetTimingSummary: + """get_timing_summary is a gated tool that returns accumulated wall-clock timing.""" + + @pytest.mark.anyio + async def test_gate_closed_returns_gate_error(self, tool_ctx): + """get_timing_summary returns gate_error when kitchen gate is closed.""" + tool_ctx.gate = DefaultGateState(enabled=False) + result = json.loads(await get_timing_summary()) + assert result.get("success") is False + assert result.get("subtype") == "gate_error" + + @pytest.mark.anyio + async def test_returns_empty_steps_initially(self, tool_ctx_kitchen_open): + result = json.loads(await get_timing_summary()) + assert result["steps"] == [] + assert result["total"]["total_seconds"] == 0.0 + + @pytest.mark.anyio + async def test_returns_entry_per_step_name(self, tool_ctx_kitchen_open): + tool_ctx_kitchen_open.timing_log.record("clone", 4.0) + tool_ctx_kitchen_open.timing_log.record("test_check", 12.0) + result = json.loads(await get_timing_summary()) + assert len(result["steps"]) == 2 + step_names = {s["step_name"] for s in result["steps"]} + assert step_names == {"clone", "test_check"} + + @pytest.mark.anyio + async def test_multiple_invocations_same_step_are_summed(self, tool_ctx_kitchen_open): + tool_ctx_kitchen_open.timing_log.record("impl", 10.0) + tool_ctx_kitchen_open.timing_log.record("impl", 10.0) + tool_ctx_kitchen_open.timing_log.record("impl", 10.0) + result = json.loads(await get_timing_summary()) + assert len(result["steps"]) == 1 + assert result["steps"][0]["total_seconds"] == 30.0 + assert result["steps"][0]["invocation_count"] == 3 + + @pytest.mark.anyio + async def test_total_field_sums_all_steps(self, tool_ctx_kitchen_open): + tool_ctx_kitchen_open.timing_log.record("a", 5.0) + tool_ctx_kitchen_open.timing_log.record("b", 8.0) + result = json.loads(await get_timing_summary()) + assert result["total"]["total_seconds"] == 13.0 + + @pytest.mark.anyio + async def test_clear_true_resets_after_returning(self, tool_ctx_kitchen_open): + tool_ctx_kitchen_open.timing_log.record("clone", 4.0) + result = json.loads(await get_timing_summary(clear=True)) + assert len(result["steps"]) == 1 + result2 = json.loads(await get_timing_summary()) + assert result2["steps"] == [] + + @pytest.mark.anyio + async def test_response_shape(self, tool_ctx_kitchen_open): + tool_ctx_kitchen_open.timing_log.record("plan", 3.0) + result = json.loads(await get_timing_summary()) + assert "steps" in result + assert "total" in result + assert isinstance(result["steps"], list) + assert "total_seconds" in result["total"] + + +class TestGetTokenSummaryFormat: + """Tests for get_token_summary format parameter.""" + + @pytest.mark.anyio + async def test_format_json_default(self, tool_ctx_kitchen_open): + """format='json' (default) returns JSON dict.""" + tool_ctx_kitchen_open.token_log.record( + "plan", + {"input_tokens": 100, "output_tokens": 50}, + ) + result = json.loads(await get_token_summary()) + assert "steps" in result + assert "total" in result + + @pytest.mark.anyio + async def test_format_table_returns_markdown(self, tool_ctx_kitchen_open): + """format='table' returns a markdown table string.""" + tool_ctx_kitchen_open.token_log.record( + "plan", + {"input_tokens": 100, "output_tokens": 50}, + ) + result = await get_token_summary(format="table") + assert "| Step |" in result + assert "|---" in result + assert "plan" in result + assert "**Total**" in result + # Not JSON + with pytest.raises(json.JSONDecodeError): + json.loads(result) + + @pytest.mark.anyio + async def test_format_table_with_clear(self, tool_ctx_kitchen_open): + """format='table' + clear=True clears the log after returning.""" + tool_ctx_kitchen_open.token_log.record( + "plan", + {"input_tokens": 100, "output_tokens": 50}, + ) + result = await get_token_summary(clear=True, format="table") + assert "plan" in result + result2 = json.loads(await get_token_summary()) + assert result2["steps"] == [] + + +class TestGetTimingSummaryFormat: + """Tests for get_timing_summary format parameter.""" + + @pytest.mark.anyio + async def test_format_json_default(self, tool_ctx_kitchen_open): + """format='json' (default) returns JSON dict.""" + tool_ctx_kitchen_open.timing_log.record("plan", 3.0) + result = json.loads(await get_timing_summary()) + assert "steps" in result + assert "total" in result + + @pytest.mark.anyio + async def test_format_table_returns_markdown(self, tool_ctx_kitchen_open): + """format='table' returns a markdown table string.""" + tool_ctx_kitchen_open.timing_log.record("plan", 3.0) + result = await get_timing_summary(format="table") + assert "| Step |" in result + assert "|---" in result + assert "plan" in result + assert "**Total**" in result + with pytest.raises(json.JSONDecodeError): + json.loads(result) + + +class TestTokenSummaryWallClock: + @pytest.mark.anyio + async def test_wall_clock_seconds_merged_from_timing_log(self, tool_ctx_kitchen_open): + from autoskillit.pipeline.timings import DefaultTimingLog + from autoskillit.pipeline.tokens import DefaultTokenLog + + tool_ctx_kitchen_open.token_log = DefaultTokenLog() + tool_ctx_kitchen_open.timing_log = DefaultTimingLog() + tool_ctx_kitchen_open.token_log.record( + "step-a", + { + "input_tokens": 100, + "output_tokens": 50, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + }, + elapsed_seconds=8.0, + ) + tool_ctx_kitchen_open.timing_log.record("step-a", 12.5) + + result = json.loads(await get_token_summary()) + step = next(s for s in result["steps"] if s["step_name"] == "step-a") + assert step["wall_clock_seconds"] == pytest.approx(12.5) + assert step["elapsed_seconds"] == pytest.approx(8.0) + + @pytest.mark.anyio + async def test_wall_clock_falls_back_to_elapsed_when_no_timing(self, tool_ctx_kitchen_open): + from autoskillit.pipeline.timings import DefaultTimingLog + from autoskillit.pipeline.tokens import DefaultTokenLog + + tool_ctx_kitchen_open.token_log = DefaultTokenLog() + tool_ctx_kitchen_open.timing_log = DefaultTimingLog() + tool_ctx_kitchen_open.token_log.record( + "step-b", + { + "input_tokens": 200, + "output_tokens": 100, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + }, + elapsed_seconds=5.0, + ) + + # Verify no step-b entry was injected into timing_log (guards against parallel pollution) + assert not any( + e["step_name"] == "step-b" for e in tool_ctx_kitchen_open.timing_log.get_report() + ) + result = json.loads(await get_token_summary()) + step = next(s for s in result["steps"] if s["step_name"] == "step-b") + # No timing_log entry → falls back to elapsed_seconds + assert step["wall_clock_seconds"] == pytest.approx(5.0) + + @pytest.mark.anyio + async def test_merge_wall_clock_after_normalization(self, tool_ctx_kitchen_open): + """ + When token entries and timing entries are both normalized to canonical names, + _merge_wall_clock_seconds must match them correctly. + """ + from autoskillit.pipeline.timings import DefaultTimingLog + from autoskillit.pipeline.tokens import DefaultTokenLog + + tool_ctx_kitchen_open.token_log = DefaultTokenLog() + tool_ctx_kitchen_open.timing_log = DefaultTimingLog() + + # Record via suffixed names — both should normalize to "implement" + usage = { + "input_tokens": 100, + "output_tokens": 50, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + } + tool_ctx_kitchen_open.token_log.record("implement-30", usage) + tool_ctx_kitchen_open.token_log.record("implement-31", usage) + tool_ctx_kitchen_open.timing_log.record("implement-30", 60.0) + tool_ctx_kitchen_open.timing_log.record("implement-31", 55.0) + + result = json.loads(await get_token_summary(clear=False, format="json")) + + assert len(result["steps"]) == 1 + step = result["steps"][0] + assert step["step_name"] == "implement" + assert step["input_tokens"] == 200 + # wall_clock_seconds from timing log should be present + assert "wall_clock_seconds" in step + assert step["wall_clock_seconds"] == pytest.approx(115.0) + + +class TestClearMarkerWritten: + @pytest.mark.parametrize( + "fn,kwargs", + [ + (get_token_summary, {"clear": True}), + (get_timing_summary, {"clear": True}), + (get_pipeline_report, {"clear": True}), + ], + ) + @pytest.mark.anyio + async def test_clear_writes_marker( + self, tool_ctx_kitchen_open, tmp_path, monkeypatch, fn, kwargs + ): + from autoskillit.execution.session_log import read_telemetry_clear_marker + + log_dir = tmp_path / "logs" + log_dir.mkdir() + monkeypatch.setattr(tool_ctx_kitchen_open.config.linux_tracing, "log_dir", str(log_dir)) + await fn(**kwargs) + assert read_telemetry_clear_marker(log_dir) is not None + + +class TestGetLogRoot: + def test_returns_resolved_log_dir(self, tool_ctx, tmp_path, monkeypatch): + """_get_log_root() returns the resolved path for the configured log_dir.""" + from autoskillit.execution import resolve_log_dir + from autoskillit.server.tools.tools_status import _get_log_root + + log_dir = tmp_path / "custom_logs" + monkeypatch.setattr(tool_ctx.config.linux_tracing, "log_dir", str(log_dir)) + result = _get_log_root() + assert result == resolve_log_dir(str(log_dir)) + + def test_returns_path_type(self, tool_ctx, tmp_path, monkeypatch): + """_get_log_root() returns a Path, not a string.""" + from autoskillit.server.tools.tools_status import _get_log_root + + monkeypatch.setattr(tool_ctx.config.linux_tracing, "log_dir", str(tmp_path)) + assert isinstance(_get_log_root(), Path) + + +@pytest.mark.anyio +async def test_get_token_summary_not_contaminated_by_prior_pipeline( + tmp_path, tool_ctx_kitchen_open, monkeypatch +): + """ + get_token_summary() must return only the current pipeline's data. + Entries written by a prior pipeline run must not appear in the summary, + even if the server was restarted with those sessions present in the log dir. + """ + from unittest.mock import patch + + from autoskillit.server import _state + from autoskillit.server._state import _initialize + + # tool_ctx_kitchen_open fixture sets log_dir to tmp_path/"session_logs" — write sessions there + # so _initialize reads from the same directory it is configured to use. + log_dir = Path(tool_ctx_kitchen_open.config.linux_tracing.log_dir) + log_dir.mkdir(parents=True, exist_ok=True) + + prior_cwd = str(tmp_path / "prior-pipeline-clone") + # Use a current timestamp so it falls within _initialize's 24-hour recovery window + now_ts = datetime.now(UTC).isoformat() + + # Write sessions from a PRIOR pipeline (different cwd) using direct file writes + session_dir = log_dir / "sessions" / "sess-prior" + session_dir.mkdir(parents=True) + (session_dir / "token_usage.json").write_text( + json.dumps( + { + "step_name": "implement", + "input_tokens": 9999, + "output_tokens": 4444, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + "timing_seconds": 120.0, + } + ) + ) + with (log_dir / "sessions.jsonl").open("a") as f: + f.write( + json.dumps({"dir_name": "sess-prior", "timestamp": now_ts, "cwd": prior_cwd}) + "\n" + ) + + # Simulate server startup with cross-pipeline sessions in the log dir + with patch("autoskillit.execution.recover_crashed_sessions", return_value=0): + _initialize(tool_ctx_kitchen_open) + # Register the _ctx mutation via monkeypatch so teardown is deterministic + # rather than relying on coincidental identity with the tool_ctx_kitchen_open fixture's patch. + monkeypatch.setattr(_state, "_ctx", tool_ctx_kitchen_open) + + # Now simulate the CURRENT pipeline recording its own step + tool_ctx_kitchen_open.token_log.record( + "rectify", + { + "input_tokens": 100, + "output_tokens": 50, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + }, + ) + + # get_token_summary uses _get_ctx() internally — + # tool_ctx_kitchen_open fixture wires _ctx correctly + result_json = await get_token_summary(clear=False, format="json") + result = json.loads(result_json) + step_names = [s["step_name"] for s in result["steps"]] + + assert "implement" not in step_names, ( + "Prior pipeline step 'implement' must not appear in current pipeline summary; " + "cross-pipeline contamination detected" + ) + assert "rectify" in step_names, "Current pipeline step must appear" + assert len(step_names) == 1, f"Expected only current pipeline steps, got: {step_names}" + + +class TestOrderIdFilterOnSummaryTools: + """Group D: order_id filter params on get_token_summary and get_timing_summary.""" + + @pytest.mark.anyio + async def test_get_token_summary_order_id_filter_isolates_issue( + self, tool_ctx_kitchen_open, monkeypatch + ) -> None: + """D-1: get_token_summary(order_id='issue-185') returns only that order's steps.""" + from autoskillit.server import _state + + monkeypatch.setattr(_state, "_ctx", tool_ctx_kitchen_open) + + usage = { + "input_tokens": 100, + "output_tokens": 50, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + } + tool_ctx_kitchen_open.token_log.record("plan", usage, order_id="issue-185") + tool_ctx_kitchen_open.token_log.record("implement", usage, order_id="issue-186") + + result = json.loads(await get_token_summary(order_id="issue-185")) + step_names = [s["step_name"] for s in result["steps"]] + assert "plan" in step_names + assert "implement" not in step_names + + @pytest.mark.anyio + async def test_get_token_summary_no_order_id_returns_all( + self, tool_ctx_kitchen_open, monkeypatch + ) -> None: + """D-2: get_token_summary() without order_id returns aggregated data for all orders.""" + from autoskillit.server import _state + + monkeypatch.setattr(_state, "_ctx", tool_ctx_kitchen_open) + + usage = { + "input_tokens": 100, + "output_tokens": 50, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + } + tool_ctx_kitchen_open.token_log.record("plan", usage, order_id="issue-185") + tool_ctx_kitchen_open.token_log.record("implement", usage, order_id="issue-186") + + result = json.loads(await get_token_summary()) + step_names = [s["step_name"] for s in result["steps"]] + assert "plan" in step_names + assert "implement" in step_names + + @pytest.mark.anyio + async def test_get_timing_summary_order_id_filter_isolates_issue( + self, tool_ctx_kitchen_open, monkeypatch + ) -> None: + """D-3: get_timing_summary(order_id='issue-185') returns only that order's steps.""" + from autoskillit.server import _state + from autoskillit.server.tools.tools_status import get_timing_summary + + monkeypatch.setattr(_state, "_ctx", tool_ctx_kitchen_open) + + tool_ctx_kitchen_open.timing_log.record("plan", 10.0, order_id="issue-185") + tool_ctx_kitchen_open.timing_log.record("implement", 20.0, order_id="issue-186") + + result = json.loads(await get_timing_summary(order_id="issue-185")) + step_names = [s["step_name"] for s in result["steps"]] + assert "plan" in step_names + assert "implement" not in step_names diff --git a/tests/server/test_tools_types_module.py b/tests/server/test_tools_types_module.py new file mode 100644 index 0000000000..0d3be1165e --- /dev/null +++ b/tests/server/test_tools_types_module.py @@ -0,0 +1,67 @@ +"""Verify server-layer TypedDicts are importable from server/tools/_types.""" + +import pytest + +pytestmark = [pytest.mark.layer("server"), pytest.mark.small] + + +class TestServerToolTypesImport: + """REQ-RELOC-001: Types exist in server/tools/_types.py.""" + + def test_kitchen_status_result_importable(self): + from autoskillit.server.tools._types import KitchenStatusResult + + assert "package_version" in KitchenStatusResult.__optional_keys__ + + def test_token_summary_result_importable(self): + from autoskillit.server.tools._types import TokenSummaryResult + + assert "model_totals" in TokenSummaryResult.__optional_keys__ + + def test_timing_summary_result_importable(self): + from autoskillit.server.tools._types import TimingSummaryResult + + assert "steps" in TimingSummaryResult.__optional_keys__ + + def test_run_skill_result_importable(self): + from autoskillit.server.tools._types import RunSkillResult + + assert "success" in RunSkillResult.__required_keys__ + + def test_run_cmd_result_importable(self): + from autoskillit.server.tools._types import RunCmdResult + + assert "success" in RunCmdResult.__required_keys__ + + def test_test_check_result_importable(self): + from autoskillit.server.tools._types import TestCheckResult + + assert "passed" in TestCheckResult.__required_keys__ + + def test_merge_worktree_result_importable(self): + from autoskillit.server.tools._types import MergeWorktreeResult + + assert "merge_succeeded" in MergeWorktreeResult.__optional_keys__ + + +class TestServerToolTypesNotInCore: + """REQ-RELOC-002/003: Types removed from core re-export surface.""" + + def test_types_not_in_core_types_all(self): + from autoskillit.core.types._type_results import __all__ as results_all + + relocated = { + "KitchenStatusResult", + "TokenSummaryResult", + "TimingSummaryResult", + "RunSkillResult", + "RunCmdResult", + "TestCheckResult", + "MergeWorktreeResult", + } + assert relocated.isdisjoint(set(results_all)) + + def test_model_total_entry_still_in_core(self): + from autoskillit.core.types._type_results import ModelTotalEntry + + assert "model" in ModelTotalEntry.__required_keys__ diff --git a/tests/server/test_tools_workspace.py b/tests/server/test_tools_workspace.py index 510528e262..7b7462a3a3 100644 --- a/tests/server/test_tools_workspace.py +++ b/tests/server/test_tools_workspace.py @@ -13,10 +13,12 @@ SafetyConfig, ) from autoskillit.core.types import AUTOSKILLIT_PRIVATE_ENV_VARS -from autoskillit.server.tools_workspace import reset_test_dir, reset_workspace, test_check +from autoskillit.server.tools.tools_workspace import reset_test_dir, reset_workspace, test_check from autoskillit.workspace import CleanupResult from tests.conftest import _make_result +pytestmark = [pytest.mark.layer("server"), pytest.mark.small] + test_check.__test__ = False # type: ignore[attr-defined] @@ -24,9 +26,9 @@ class TestResetWorkspace: """T6, T7: reset_workspace preserves configured dirs, requires marker.""" @pytest.fixture(autouse=True) - def _set_reset_command(self, tool_ctx): + def _set_reset_command(self, tool_ctx_kitchen_open): """Configure reset_workspace with a command for these tests.""" - tool_ctx.config = AutomationConfig( + tool_ctx_kitchen_open.config = AutomationConfig( reset_workspace=ResetWorkspaceConfig( command=["make", "clean"], preserve_dirs={".cache", "reports"}, @@ -49,7 +51,7 @@ async def test_rejects_nonexistent_directory(self, tmp_path): assert "does not exist" in result["error"] @pytest.mark.anyio - async def test_preserves_configured_dirs(self, tool_ctx, tmp_path): + async def test_preserves_configured_dirs(self, tool_ctx_kitchen_open, tmp_path): workspace = tmp_path / "workspace" workspace.mkdir(parents=True) (workspace / ".autoskillit-workspace").write_text("# marker\n") @@ -62,7 +64,7 @@ async def test_preserves_configured_dirs(self, tool_ctx, tmp_path): (workspace / "temp_dir").mkdir() (workspace / "temp_dir" / "file.txt").touch() - tool_ctx.runner.push(_make_result(0, "", "")) + tool_ctx_kitchen_open.runner.push(_make_result(0, "", "")) result = json.loads(await reset_workspace(test_dir=str(workspace))) @@ -78,12 +80,12 @@ async def test_preserves_configured_dirs(self, tool_ctx, tmp_path): assert not (workspace / "temp_dir").exists() @pytest.mark.anyio - async def test_reset_command_failure(self, tool_ctx, tmp_path): + async def test_reset_command_failure(self, tool_ctx_kitchen_open, tmp_path): workspace = tmp_path / "workspace" workspace.mkdir(parents=True) (workspace / ".autoskillit-workspace").write_text("# marker\n") - tool_ctx.runner.push(_make_result(1, "", "command not found")) + tool_ctx_kitchen_open.runner.push(_make_result(1, "", "command not found")) result = json.loads(await reset_workspace(test_dir=str(workspace))) @@ -92,16 +94,16 @@ async def test_reset_command_failure(self, tool_ctx, tmp_path): assert result["exit_code"] == 1 @pytest.mark.anyio - async def test_runs_correct_reset_command(self, tool_ctx, tmp_path): + async def test_runs_correct_reset_command(self, tool_ctx_kitchen_open, tmp_path): workspace = tmp_path / "workspace" workspace.mkdir(parents=True) (workspace / ".autoskillit-workspace").write_text("# marker\n") - tool_ctx.runner.push(_make_result(0, "", "")) + tool_ctx_kitchen_open.runner.push(_make_result(0, "", "")) await reset_workspace(test_dir=str(workspace)) - call_args = tool_ctx.runner.call_args_list[0][0] + call_args = tool_ctx_kitchen_open.runner.call_args_list[0][0] assert call_args == [ "make", "clean", @@ -156,7 +158,10 @@ async def test_does_not_expose_summary(self, tool_ctx): result = json.loads(await test_check(worktree_path="/tmp/wt")) assert "summary" not in result assert "output_file" not in result - assert set(result.keys()) == {"passed", "stdout", "stderr"} + assert "passed" in result + assert "stdout" in result + assert "stderr" in result + assert "duration_seconds" in result @pytest.mark.anyio async def test_cross_validates_error_in_output(self, tool_ctx): @@ -271,7 +276,7 @@ async def test_response_schema_includes_stderr(self, tool_ctx): """test_check response contains passed, stdout, and stderr keys.""" tool_ctx.runner.push(_make_result(0, "= 10 passed =\n", "")) result = json.loads(await test_check(worktree_path="/tmp/wt")) - assert set(result.keys()) == {"passed", "stdout", "stderr"} + assert set(result.keys()) == {"passed", "stdout", "stderr", "duration_seconds"} @pytest.mark.anyio async def test_pytest_failure_still_detected(self, tool_ctx): @@ -280,12 +285,63 @@ async def test_pytest_failure_still_detected(self, tool_ctx): result = json.loads(await test_check(worktree_path="/tmp/wt")) assert result["passed"] is False + @pytest.mark.anyio + async def test_test_check_response_includes_duration(self, tool_ctx): + """test_check JSON includes duration_seconds.""" + tool_ctx.runner.push(_make_result(0, "= 10 passed =\n", "")) + raw = await test_check("/tmp/wt") + data = json.loads(raw) + assert "duration_seconds" in data + assert isinstance(data["duration_seconds"], float) + assert data["duration_seconds"] >= 0.0 + + @pytest.mark.anyio + async def test_test_check_response_includes_filter_stats(self, tool_ctx, monkeypatch): + """test_check JSON includes filter fields when sidecar is written.""" + from pathlib import Path as _Path + + from autoskillit.core.types import SubprocessResult, TerminationReason + + sidecar_data = { + "filter_mode": "aggressive", + "tests_selected": 73, + "tests_deselected": 275, + } + + async def fake_runner(command, *, cwd, timeout, env, **kwargs): + sidecar_path = env.get("AUTOSKILLIT_FILTER_STATS_FILE") + assert sidecar_path, "AUTOSKILLIT_FILTER_STATS_FILE must be injected into env" + _Path(sidecar_path).write_text(json.dumps(sidecar_data)) + return SubprocessResult( + returncode=0, + stdout="= 73 passed =\n", + stderr="", + termination=TerminationReason.NATURAL_EXIT, + pid=12345, + ) + + monkeypatch.setattr(tool_ctx.tester, "_runner", fake_runner) + raw = await test_check("/tmp/wt") + data = json.loads(raw) + assert data["filter_mode"] == "aggressive" + assert data["tests_selected"] == 73 + assert data["tests_deselected"] == 275 + + @pytest.mark.anyio + async def test_test_check_response_omits_filter_stats_when_absent(self, tool_ctx): + """Filter fields are absent (not null) from response when no filter active.""" + tool_ctx.runner.push(_make_result(0, "= 10 passed =\n", "")) + raw = await test_check("/tmp/wt") + data = json.loads(raw) + assert "filter_mode" not in data + assert "tests_selected" not in data + class TestResetGuard: """Marker-file-based reset guard for destructive operations.""" @pytest.mark.anyio - async def test_reset_test_dir_refuses_without_marker(self, tool_ctx, tmp_path): + async def test_reset_test_dir_refuses_without_marker(self, tool_ctx_kitchen_open, tmp_path): """Directory without marker file is refused.""" target = tmp_path / "workspace" target.mkdir() @@ -295,7 +351,7 @@ async def test_reset_test_dir_refuses_without_marker(self, tool_ctx, tmp_path): assert "marker" in result["error"].lower() or "reset guard" in result["error"].lower() @pytest.mark.anyio - async def test_reset_test_dir_allows_with_marker(self, tool_ctx, tmp_path): + async def test_reset_test_dir_allows_with_marker(self, tool_ctx_kitchen_open, tmp_path): """Directory with marker file is cleared.""" target = tmp_path / "workspace" target.mkdir() @@ -306,7 +362,7 @@ async def test_reset_test_dir_allows_with_marker(self, tool_ctx, tmp_path): assert not (target / "some_file.txt").exists() @pytest.mark.anyio - async def test_reset_test_dir_preserves_marker(self, tool_ctx, tmp_path): + async def test_reset_test_dir_preserves_marker(self, tool_ctx_kitchen_open, tmp_path): """Reset preserves the marker file so the workspace is reusable.""" target = tmp_path / "workspace" target.mkdir() @@ -317,30 +373,36 @@ async def test_reset_test_dir_preserves_marker(self, tool_ctx, tmp_path): assert (target / ".autoskillit-workspace").is_file() @pytest.mark.anyio - async def test_reset_workspace_refuses_without_marker(self, tool_ctx, tmp_path): + async def test_reset_workspace_refuses_without_marker(self, tool_ctx_kitchen_open, tmp_path): """reset_workspace also checks for marker.""" - tool_ctx.config = AutomationConfig(reset_workspace=ResetWorkspaceConfig(command=["true"])) + tool_ctx_kitchen_open.config = AutomationConfig( + reset_workspace=ResetWorkspaceConfig(command=["true"]) + ) target = tmp_path / "workspace" target.mkdir() result = json.loads(await reset_workspace(test_dir=str(target))) assert "error" in result @pytest.mark.anyio - async def test_reset_workspace_allows_with_marker(self, tool_ctx, tmp_path): + async def test_reset_workspace_allows_with_marker(self, tool_ctx_kitchen_open, tmp_path): """reset_workspace clears when marker is present.""" - tool_ctx.config = AutomationConfig(reset_workspace=ResetWorkspaceConfig(command=["true"])) + tool_ctx_kitchen_open.config = AutomationConfig( + reset_workspace=ResetWorkspaceConfig(command=["true"]) + ) target = tmp_path / "workspace" target.mkdir() (target / ".autoskillit-workspace").write_text("# autoskillit workspace\n") (target / "file.txt").touch() - tool_ctx.runner.push(_make_result(0, "", "")) + tool_ctx_kitchen_open.runner.push(_make_result(0, "", "")) result = json.loads(await reset_workspace(test_dir=str(target))) assert result["success"] is True @pytest.mark.anyio - async def test_custom_marker_name(self, tool_ctx, tmp_path): + async def test_custom_marker_name(self, tool_ctx_kitchen_open, tmp_path): """Config can override marker file name.""" - tool_ctx.config = AutomationConfig(safety=SafetyConfig(reset_guard_marker=".my-workspace")) + tool_ctx_kitchen_open.config = AutomationConfig( + safety=SafetyConfig(reset_guard_marker=".my-workspace") + ) target = tmp_path / "workspace" target.mkdir() (target / ".my-workspace").touch() @@ -349,7 +411,7 @@ async def test_custom_marker_name(self, tool_ctx, tmp_path): assert result["success"] is True @pytest.mark.anyio - async def test_force_overrides_marker_check(self, tool_ctx, tmp_path): + async def test_force_overrides_marker_check(self, tool_ctx_kitchen_open, tmp_path): """force=True on reset_test_dir bypasses marker requirement.""" target = tmp_path / "workspace" target.mkdir() @@ -359,7 +421,7 @@ async def test_force_overrides_marker_check(self, tool_ctx, tmp_path): assert result["success"] is True @pytest.mark.anyio - async def test_rejects_nonexistent(self, tool_ctx, tmp_path): + async def test_rejects_nonexistent(self, tool_ctx_kitchen_open, tmp_path): result = json.loads(await reset_test_dir(test_dir=str(tmp_path / "nope"))) assert "does not exist" in result["error"] @@ -386,7 +448,7 @@ def test_default_classify_fix_empty_prefixes(self): @pytest.mark.anyio -async def test_reset_test_dir_returns_partial_failure_json(tool_ctx, tmp_path): +async def test_reset_test_dir_returns_partial_failure_json(tool_ctx_kitchen_open, tmp_path): """reset_test_dir returns structured JSON on partial failure.""" workspace = tmp_path / "workspace" workspace.mkdir() @@ -398,7 +460,7 @@ async def test_reset_test_dir_returns_partial_failure_json(tool_ctx, tmp_path): failed=[("bad_dir", "PermissionError: denied")], skipped=[], ) - tool_ctx.workspace_mgr = type( + tool_ctx_kitchen_open.workspace_mgr = type( "MockWM", (), {"delete_contents": lambda self, d, preserve=None: mock_result} )() result = json.loads(await reset_test_dir(test_dir=str(workspace), force=False)) @@ -409,22 +471,24 @@ async def test_reset_test_dir_returns_partial_failure_json(tool_ctx, tmp_path): @pytest.mark.anyio -async def test_reset_workspace_returns_partial_failure_json(tool_ctx, tmp_path): +async def test_reset_workspace_returns_partial_failure_json(tool_ctx_kitchen_open, tmp_path): """reset_workspace returns structured JSON on partial failure.""" - tool_ctx.config = AutomationConfig(reset_workspace=ResetWorkspaceConfig(command=["true"])) + tool_ctx_kitchen_open.config = AutomationConfig( + reset_workspace=ResetWorkspaceConfig(command=["true"]) + ) workspace = tmp_path / "workspace" workspace.mkdir(parents=True) (workspace / ".autoskillit-workspace").write_text("# marker\n") - tool_ctx.runner.push(_make_result(0, "", "")) + tool_ctx_kitchen_open.runner.push(_make_result(0, "", "")) mock_result = CleanupResult( deleted=["ok_file"], failed=[("bad_dir", "PermissionError: denied")], skipped=[".cache"], ) - tool_ctx.workspace_mgr = type( + tool_ctx_kitchen_open.workspace_mgr = type( "MockWM", (), {"delete_contents": lambda self, d, preserve=None: mock_result} )() result = json.loads(await reset_workspace(test_dir=str(workspace))) @@ -454,15 +518,15 @@ class TestResetTestDirTiming: """reset_test_dir records wall-clock timing when step_name is provided.""" @pytest.mark.anyio - async def test_reset_test_dir_step_name_records_timing(self, tool_ctx, tmp_path): + async def test_reset_test_dir_step_name_records_timing(self, tool_ctx_kitchen_open, tmp_path): marker = tmp_path / ".autoskillit-workspace" marker.write_text("") mock_result = CleanupResult(deleted=[], failed=[], skipped=[]) - tool_ctx.workspace_mgr = type( + tool_ctx_kitchen_open.workspace_mgr = type( "MockWM", (), {"delete_contents": lambda self, d, preserve=None: mock_result} )() await reset_test_dir(str(tmp_path), step_name="reset") - report = tool_ctx.timing_log.get_report() + report = tool_ctx_kitchen_open.timing_log.get_report() assert any(e["step_name"] == "reset" for e in report) @pytest.mark.anyio diff --git a/tests/server/test_track_response_size.py b/tests/server/test_track_response_size.py index 3e670e711f..cf186d8338 100644 --- a/tests/server/test_track_response_size.py +++ b/tests/server/test_track_response_size.py @@ -1,4 +1,4 @@ -"""Tests for the track_response_size decorator in autoskillit.server.helpers.""" +"""Tests for the track_response_size decorator in autoskillit.server._notify.""" from __future__ import annotations @@ -9,6 +9,8 @@ from autoskillit.pipeline.mcp_response import DefaultMcpResponseLog +pytestmark = [pytest.mark.layer("server"), pytest.mark.small] + class TestTrackResponseSize: @pytest.mark.anyio @@ -17,13 +19,13 @@ async def test_decorator_records_str_response(self): log = DefaultMcpResponseLog() response_str = json.dumps({"steps": [], "total": {}}) - from autoskillit.server.helpers import track_response_size + from autoskillit.server._notify import track_response_size @track_response_size("get_token_summary") async def fake_handler(): return response_str - with patch("autoskillit.server.helpers._get_ctx_or_none") as mock_ctx: + with patch("autoskillit.server._notify._get_ctx_or_none") as mock_ctx: mock_ctx.return_value = MagicMock( response_log=log, config=MagicMock(mcp_response=MagicMock(alert_threshold_tokens=0)), @@ -42,13 +44,13 @@ async def test_decorator_serializes_dict_for_measurement(self): log = DefaultMcpResponseLog() response_dict = {"key": "value"} - from autoskillit.server.helpers import track_response_size + from autoskillit.server._notify import track_response_size @track_response_size("kitchen_status") async def fake_handler(): return response_dict - with patch("autoskillit.server.helpers._get_ctx_or_none") as mock_ctx: + with patch("autoskillit.server._notify._get_ctx_or_none") as mock_ctx: mock_ctx.return_value = MagicMock( response_log=log, config=MagicMock(mcp_response=MagicMock(alert_threshold_tokens=0)), @@ -63,13 +65,13 @@ async def fake_handler(): @pytest.mark.anyio async def test_decorator_noop_when_ctx_unavailable(self): """When _get_ctx_or_none() returns None, decorator is silent.""" - from autoskillit.server.helpers import track_response_size + from autoskillit.server._notify import track_response_size @track_response_size("run_skill") async def fake_handler(): return "response" - with patch("autoskillit.server.helpers._get_ctx_or_none", return_value=None): + with patch("autoskillit.server._notify._get_ctx_or_none", return_value=None): result = await fake_handler() assert result == "response" # no error raised @@ -79,13 +81,13 @@ async def test_decorator_catches_handler_exception_as_structured_json(self): """If the wrapped handler raises, the exception is caught and converted.""" import json - from autoskillit.server.helpers import track_response_size + from autoskillit.server._notify import track_response_size @track_response_size("run_skill") async def bad_handler(): raise ValueError("something went wrong") - with patch("autoskillit.server.helpers._get_ctx_or_none", return_value=None): + with patch("autoskillit.server._notify._get_ctx_or_none", return_value=None): result = await bad_handler() data = json.loads(result) @@ -96,13 +98,13 @@ async def bad_handler(): @pytest.mark.anyio async def test_track_response_size_exception_envelope_includes_user_visible_message(self): """Exception envelope includes non-empty user_visible_message with tool name.""" - from autoskillit.server.helpers import track_response_size + from autoskillit.server._notify import track_response_size @track_response_size("open_kitchen") async def bad_handler(): raise RuntimeError("boom") - with patch("autoskillit.server.helpers._get_ctx_or_none", return_value=None): + with patch("autoskillit.server._notify._get_ctx_or_none", return_value=None): result = await bad_handler() data = json.loads(result) @@ -114,13 +116,13 @@ async def bad_handler(): @pytest.mark.anyio async def test_track_response_size_exception_envelope_preserves_existing_fields(self): """Regression guard: success, error, exit_code, subtype keys still present.""" - from autoskillit.server.helpers import track_response_size + from autoskillit.server._notify import track_response_size @track_response_size("test_tool") async def bad_handler(): raise ValueError("fail") - with patch("autoskillit.server.helpers._get_ctx_or_none", return_value=None): + with patch("autoskillit.server._notify._get_ctx_or_none", return_value=None): result = await bad_handler() data = json.loads(result) diff --git a/tests/server/test_wire_compat.py b/tests/server/test_wire_compat.py new file mode 100644 index 0000000000..0fb43007a8 --- /dev/null +++ b/tests/server/test_wire_compat.py @@ -0,0 +1,115 @@ +from __future__ import annotations + +import pytest + +pytestmark = [pytest.mark.layer("server"), pytest.mark.small] + + +class TestClaudeCodeCompatMiddlewareEdgeCases: + """Edge-case coverage for ClaudeCodeCompatMiddleware.on_list_tools.""" + + @pytest.mark.anyio + async def test_empty_tool_list(self): + from unittest.mock import AsyncMock, MagicMock + + from autoskillit.server._wire_compat import ClaudeCodeCompatMiddleware + + mw = ClaudeCodeCompatMiddleware() + ctx = MagicMock() + call_next = AsyncMock(return_value=[]) + + result = await mw.on_list_tools(ctx, call_next) + + assert result == [] + + @pytest.mark.anyio + async def test_title_stripped_when_output_schema_already_none(self): + from unittest.mock import AsyncMock, MagicMock + + from autoskillit.server._wire_compat import ClaudeCodeCompatMiddleware + + mw = ClaudeCodeCompatMiddleware() + tool = MagicMock() + tool.name = "titled_tool" + tool.title = "My Tool" + tool.output_schema = None + tool.model_copy.return_value = MagicMock( + name="titled_tool", + output_schema=None, + title=None, + ) + + ctx = MagicMock() + call_next = AsyncMock(return_value=[tool]) + + result = await mw.on_list_tools(ctx, call_next) + + assert result[0].title is None + tool.model_copy.assert_called_once_with( + update={"output_schema": None, "title": None}, + ) + + @pytest.mark.anyio + async def test_model_copy_called_unconditionally_when_fields_already_none(self): + from unittest.mock import AsyncMock, MagicMock + + from autoskillit.server._wire_compat import ClaudeCodeCompatMiddleware + + mw = ClaudeCodeCompatMiddleware() + tool = MagicMock() + tool.name = "clean_tool" + tool.output_schema = None + tool.title = None + tool.model_copy.return_value = MagicMock( + name="clean_tool", + output_schema=None, + title=None, + ) + + ctx = MagicMock() + call_next = AsyncMock(return_value=[tool]) + + result = await mw.on_list_tools(ctx, call_next) + + tool.model_copy.assert_called_once_with( + update={"output_schema": None, "title": None}, + ) + assert result[0] is tool.model_copy.return_value + + @pytest.mark.anyio + async def test_mixed_tool_list_all_cleaned(self): + from unittest.mock import AsyncMock, MagicMock + + from autoskillit.server._wire_compat import ClaudeCodeCompatMiddleware + + mw = ClaudeCodeCompatMiddleware() + tool_a = MagicMock() + tool_a.name = "tool_a" + tool_a.output_schema = {"type": "string"} + tool_a.title = "Tool A" + tool_a.model_copy.return_value = MagicMock( + name="tool_a", + output_schema=None, + title=None, + ) + + tool_b = MagicMock() + tool_b.name = "tool_b" + tool_b.output_schema = None + tool_b.title = None + tool_b.model_copy.return_value = MagicMock( + name="tool_b", + output_schema=None, + title=None, + ) + + ctx = MagicMock() + call_next = AsyncMock(return_value=[tool_a, tool_b]) + + result = await mw.on_list_tools(ctx, call_next) + + assert len(result) == 2 + assert result[0].output_schema is None + assert result[0].title is None + assert result[1].output_schema is None + assert result[1].title is None diff --git a/tests/skills/CLAUDE.md b/tests/skills/CLAUDE.md new file mode 100644 index 0000000000..8bb46b13ee --- /dev/null +++ b/tests/skills/CLAUDE.md @@ -0,0 +1,76 @@ +# skills/ + +Skill SKILL.md content compliance, placeholder contracts, and verdict guard tests. + +## Files + +| File | Purpose | +|------|---------| +| `__init__.py` | empty | +| `conftest.py` | Skill test fixtures | +| `test_analyze_prs_contracts.py` | Contract tests for analyze-prs SKILL.md batch branch naming convention | +| `test_arch_lens_context_path.py` | Tests that all arch-lens skills have ## Arguments section and context_path handling | +| `test_audit_arch_preflight_contracts.py` | Audit-arch skill preflight contract tests | +| `test_audit_arch_selfvalidation_contracts.py` | Audit-arch skill self-validation contract tests | +| `test_audit_review_decisions_contracts.py` | Contract tests for the audit-review-decisions skill SKILL.md | +| `test_conflict_resolution_guards.py` | Structural guards for conflict resolution safeguards in SKILL.md files | +| `test_deletion_regression_guards.py` | Structural guards for deletion regression detection in merge-pr and review-pr skills | +| `test_dry_walkthrough_contracts.py` | Structural contracts for the dry-walkthrough historical regression check step | +| `test_investigate_contracts.py` | Structural contracts for the investigate historical recurrence check step | +| `test_investigate_deep_mode_contracts.py` | Structural contracts for the investigate deep analysis mode | +| `test_investigate_design_intent_contracts.py` | Contract tests for Design Intent Analysis requirements in the investigate skill | +| `test_isolation_guidance_contracts.py` | Contract tests verifying shared mutable state isolation guidance exists in pipeline skills | +| `test_make_campaign_compliance.py` | Compliance tests for the make-campaign skill: classification, placeholder hygiene, contract registration | +| `test_merge_pr_ci_gate.py` | Guards for the gh-pr-merge CI gate introduced in Part B of issue #289 | +| `test_open_integration_pr_domain_analysis.py` | Open integration PR domain analysis tests | +| `test_open_research_pr_decomposition.py` | Structural guards: decomposed research-PR skills must not invoke sub-skills via the Skill tool | +| `test_phase2_skills.py` | Phase 2 tests: open-kitchen and close-kitchen SKILL.md files | +| `test_plan_experiment_schema_contracts.py` | Contract tests: plan-experiment YAML frontmatter schema and revision_guidance argument | +| `test_planner_extract_domain.py` | Planner extract domain tests | +| `test_planner_skill_contracts.py` | Planner skill contract tests | +| `test_project_local_audit_skill_content.py` | Tests for project-local audit skill content | +| `test_promote_split_contracts.py` | Contract tests verifying the promote-to-main / review-promotion split | +| `test_resolve_claims_review_guards.py` | Behavioral guards for resolve-claims-review/SKILL.md | +| `test_resolve_design_review_contracts.py` | Structural guards for resolve-design-review SKILL.md | +| `test_resolve_failures_ci_aware.py` | Contract guards for resolve-failures CI-awareness: verdict decision tree | +| `test_resolve_failures_guards.py` | Guards for resolve-failures SKILL.md: polling-cascade and output-bloat fixes | +| `test_resolve_research_review_guards.py` | Behavioral guards for resolve-research-review/SKILL.md | +| `test_resolve_review_diff_context_consumption.py` | Guards: resolve-review loads and uses diff_context handoff file from review-pr | +| `test_resolve_review_diff_hunk_preference.py` | Guards: resolve-review Step 3.5 prefers diff_hunk over source file reads | +| `test_resolve_review_duplicate_comments.py` | Resolve review duplicate comments guard | +| `test_resolve_review_intent_validation.py` | Structural guards for resolve-review intent-validation phase | +| `test_resolve_review_local_mode.py` | Contract tests for resolve-review/SKILL.md local mode (mode=local) behavior | +| `test_resolve_review_severity_and_reject_resolution.py` | Resolve review severity and reject resolution tests | +| `test_resolve_review_thread_resolution.py` | Resolve review thread resolution tests | +| `test_resolve_review_token_optimizations.py` | Structural guards for resolve-review SKILL.md token-optimization edits | +| `test_retired_skill_names.py` | Convention guard: RETIRED_SKILL_NAMES entries must not have live directories, must be lowercase, and runtime raises on retired skill | +| `test_review_design_contracts.py` | Contract tests for review-design SKILL.md behavioral encoding | +| `test_review_design_guards.py` | Guard tests for review-design SKILL.md — data_acquisition dimension | +| `test_review_flag_marker.py` | Tests for the REVIEW-FLAG HTML comment marker used in resolve-review replies | +| `test_review_pr_adaptive_dispatch_guards.py` | Behavioral guard tests for review-pr adaptive subagent dispatch | +| `test_review_pr_diff_context_handoff.py` | Guards: review-pr writes diff_context handoff file in Step 8 before verdict emission | +| `test_review_pr_inline_comment_guards.py` | Structural guards for review-pr/SKILL.md posting mechanics | +| `test_review_pr_local_mode.py` | Contract tests for review-pr/SKILL.md local mode (mode=local) behavior | +| `test_review_pr_prior_thread_awareness.py` | Behavioral guard tests for review-pr/SKILL.md prior-thread awareness (T_RPA1–T_RPA7) | +| `test_review_pr_verdict_guards.py` | Behavioral guard tests for review-pr/SKILL.md verdict logic | +| `test_review_research_pr_guards.py` | Behavioral guards for review-research-pr/SKILL.md | +| `test_skill_body_cleanliness.py` | Assert that no SKILL.md body references %%ORDER_UP%% | +| `test_skill_compliance.py` | SKILL.md compliance tests: structural invariants for skill composition safety | +| `test_skill_genericization.py` | Verify skill SKILL.md files contain no project-specific AutoSkillit internals | +| `test_skill_output_compliance.py` | Tests that all SKILL.md output path instructions use HHMMSS-precision timestamps | +| `test_skill_placeholder_contracts.py` | Validate that no SKILL.md bash code block uses an undefined {placeholder} token | +| `test_skill_preambles.py` | Tests that critical SKILL.md preamble patterns are present (FRICT-1B-1, FRICT-1C-2) | +| `test_skill_tool_syntax_contracts.py` | Validates that SKILL.md bash blocks do not contain grep BRE \\| alternation patterns | +| `test_sous_chef_deferred_escalation.py` | Contract tests for sous-chef deferred issue escalation (T6/T7) | +| `test_tier1_no_temp_reference.py` | Tier 1 SKILL.md files must not reference temp at all | +| `test_tier2_3_no_literal_temp.py` | Tier 2/3 SKILL.md files must use {{AUTOSKILLIT_TEMP}}, never the literal | +| `test_troubleshoot_experiment_contracts.py` | Troubleshoot experiment skill discoverability test | +| `test_validate_audit_contracts.py` | Contract tests for the validate-audit skill SKILL.md | +| `test_validate_test_audit_contracts.py` | Contract tests for the validate-test-audit skill SKILL.md | +| `test_validate_review_decisions_contracts.py` | Contract tests for the validate-review-decisions skill SKILL.md | +| `test_vis_lens_methodology_norms.py` | Skill-level integration tests for vis-lens-methodology-norms tradition expansion | +| `test_vis_lens_structural.py` | Structural assertions for P0 vis-lens skills | + +## Architecture Notes + +`conftest.py` provides skill test fixtures. Tests in this directory target SKILL.md content for correctness and compliance — not the skill loader infrastructure (which is in `tests/workspace/`). diff --git a/tests/skills/conftest.py b/tests/skills/conftest.py new file mode 100644 index 0000000000..275c80bdda --- /dev/null +++ b/tests/skills/conftest.py @@ -0,0 +1,64 @@ +import pytest + +from autoskillit.core.paths import pkg_root + + +@pytest.fixture(scope="module") +def skill_text() -> str: + skill_path = pkg_root() / "skills_extended" / "investigate" / "SKILL.md" + assert skill_path.exists(), f"SKILL.md not found at {skill_path}" + return skill_path.read_text() + + +@pytest.fixture(scope="module") +def deep_workflow_section(skill_text: str) -> str: + """Extract ## Deep Analysis Mode Workflow section to end-of-file (or next ## at depth ≤ 2).""" + start_idx = skill_text.find("## Deep Analysis Mode Workflow") + if start_idx == -1: + pytest.fail("'## Deep Analysis Mode Workflow' not found in investigate SKILL.md") + search_from = start_idx + len("## Deep Analysis Mode Workflow") + next_h2 = -1 + for i in range(search_from, len(skill_text)): + if skill_text[i : i + 3] == "## " and skill_text[i - 1] == "\n": + next_h2 = i + break + if next_h2 != -1: + return skill_text[start_idx:next_h2] + return skill_text[start_idx:] + + +def extract_step_section(deep_workflow_section: str, step_name: str) -> str: + """Helper to extract the text of a specific D-step subsection.""" + start = deep_workflow_section.find(f"### {step_name}") + if start == -1: + return "" + next_step = deep_workflow_section.find("### Step D", start + 1) + if next_step != -1: + return deep_workflow_section[start:next_step] + return deep_workflow_section[start:] + + +@pytest.fixture(scope="module") +def report_section(skill_text: str) -> str: + """Extract the Step 4: Write Report section text. + + Uses the next top-level ``## `` heading that is NOT inside a fenced + code block as the boundary. The report template contains ``## `` + headings inside a markdown code fence (```markdown ... ```) which must + be included — so we skip ``## `` occurrences between fence markers. + """ + step_4_idx = skill_text.find("### Step 4:") + if step_4_idx == -1: + pytest.fail("Step 4 not found in investigate SKILL.md") + in_fence = False + lines = skill_text[step_4_idx:].split("\n") + for i, line in enumerate(lines): + if i == 0: + continue + if line.startswith("```"): + in_fence = not in_fence + continue + if not in_fence and line.startswith("## ") and not line.startswith("### "): + end_idx = step_4_idx + sum(len(prev) + 1 for prev in lines[:i]) + return skill_text[step_4_idx:end_idx] + return skill_text[step_4_idx:] diff --git a/tests/skills/test_analyze_prs_contracts.py b/tests/skills/test_analyze_prs_contracts.py index d71dbaf059..1121e07caf 100644 --- a/tests/skills/test_analyze_prs_contracts.py +++ b/tests/skills/test_analyze_prs_contracts.py @@ -15,34 +15,34 @@ def skill_text() -> str: def test_analyze_prs_batch_branch_uses_pr_batch_prefix(skill_text: str) -> None: - """Batch branch name must use pr-batch/pr-merge- prefix, not integration/ prefix. + """Batch branch name must use pr-batch/pr-merge- prefix, not develop/ prefix. - The permanent integration branch is named 'integration'. Git cannot have - both a branch named 'integration' and branches named 'integration/*' — - the slash-prefix requires 'integration' to be a directory, not a file. + The permanent develop branch is named 'develop'. Git cannot have + both a branch named 'develop' and branches named 'develop/*' — + the slash-prefix requires 'develop' to be a directory, not a file. Batch branches must use a disjoint prefix (pr-batch/). We require 'pr-batch/pr-merge-' (not just 'pr-batch/') to anchor the check to the branch-computation context rather than any incidental occurrence. """ assert "pr-batch/pr-merge-" in skill_text, ( - "analyze-prs must use 'pr-batch/pr-merge-{ts}' naming for integration_branch — " - "using 'integration/' prefix conflicts with the permanent 'integration' branch" + "analyze-prs must use 'pr-batch/pr-merge-{ts}' naming for batch_branch — " + "using 'develop/' prefix conflicts with the permanent 'develop' branch" ) -def test_analyze_prs_does_not_use_integration_slash_prefix(skill_text: str) -> None: - """SKILL.md must not compute batch branch names with integration/ prefix. +def test_analyze_prs_does_not_use_develop_slash_prefix(skill_text: str) -> None: + """SKILL.md must not compute batch branch names with develop/ prefix. The original check only caught 'integration/pr-merge'; it would miss other - integration/-prefixed patterns such as 'integration/batch-...' or - 'integration/run-...'. Checking for the bare 'integration/' substring - prohibits all such patterns while still allowing mentions of 'integration' + develop/-prefixed patterns such as 'develop/batch-...' or + 'develop/run-...'. Checking for the bare 'develop/' substring + prohibits all such patterns while still allowing mentions of 'develop' (the permanent branch name) without a trailing slash. """ - assert "integration/" not in skill_text, ( - "analyze-prs must not use integration/ as a batch branch prefix in any context — " - "this conflicts with the permanent 'integration' branch in git ref storage" + assert "develop/" not in skill_text, ( + "analyze-prs must not use develop/ as a batch branch prefix in any context — " + "this conflicts with the permanent 'develop' branch in git ref storage" ) @@ -61,3 +61,8 @@ def test_analyze_prs_contract_has_merge_queue_data_path() -> None: assert "merge_queue_data_path" in names, ( "analyze-prs contract must have a merge_queue_data_path input entry" ) + + +def test_analyze_prs_step_1_5_graphql_batch(skill_text: str) -> None: + """Step 1.5 must use GraphQL aliases to batch PR queries, not sequential gh CLI calls.""" + assert "graphql" in skill_text.lower() and "alias" in skill_text.lower() diff --git a/tests/skills/test_audit_review_decisions_contracts.py b/tests/skills/test_audit_review_decisions_contracts.py new file mode 100644 index 0000000000..06389b7d15 --- /dev/null +++ b/tests/skills/test_audit_review_decisions_contracts.py @@ -0,0 +1,70 @@ +"""Contract tests for the audit-review-decisions skill SKILL.md.""" + +from __future__ import annotations + +import functools + +from autoskillit.core.types import SkillSource +from autoskillit.workspace.skills import DefaultSkillResolver + + +@functools.cache +def _skill_text() -> str: + info = DefaultSkillResolver().resolve("audit-review-decisions") + assert info is not None, "audit-review-decisions skill not found" + return info.path.read_text() + + +class TestAuditReviewDecisionsExists: + def test_skill_exists(self) -> None: + info = DefaultSkillResolver().resolve("audit-review-decisions") + assert info is not None + assert info.source == SkillSource.BUNDLED_EXTENDED + assert info.path.exists() + + def test_has_audit_category(self) -> None: + info = DefaultSkillResolver().resolve("audit-review-decisions") + assert info is not None + assert "audit" in info.categories + + +class TestAuditReviewDecisionsContent: + def test_graphql_alias_batching(self) -> None: + text = _skill_text().lower() + assert "alias" in text + + def test_ratelimit_in_queries(self) -> None: + assert "rateLimit" in _skill_text() + + def test_haiku_triage_phase(self) -> None: + text = _skill_text().lower() + assert "haiku" in text + assert "triage" in text or "broad pass" in text + + def test_sonnet_validation_phase(self) -> None: + text = _skill_text().lower() + assert "sonnet" in text + assert "valid" in text + + def test_audit_watermark_marker(self) -> None: + assert "[AUDIT]" in _skill_text() + + def test_review_flag_detection(self) -> None: + assert "REVIEW-FLAG" in _skill_text() + + def test_output_dir(self) -> None: + assert "audit-review-decisions/" in _skill_text() + + def test_report_title_matches_validate_audit_detection(self) -> None: + """Report title must contain 'Review Decisions Audit' for validate-audit detection.""" + assert "Review Decisions Audit" in _skill_text() + + def test_structured_output_token(self) -> None: + assert "review_decisions_audit_" in _skill_text() + + def test_sleep_between_mutating_calls(self) -> None: + assert "sleep 1" in _skill_text() + + def test_idempotent_watermark(self) -> None: + text = _skill_text().lower() + assert "idempotent" in text or "already" in text diff --git a/tests/skills/test_conflict_resolution_guards.py b/tests/skills/test_conflict_resolution_guards.py index 8a8ab9ce51..98f3980c64 100644 --- a/tests/skills/test_conflict_resolution_guards.py +++ b/tests/skills/test_conflict_resolution_guards.py @@ -319,6 +319,16 @@ def test_merge_prs_captures_conflict_report_from_resolve_ejected(merge_prs_recip ) +def test_merge_prs_captures_conflict_report_from_resolve_proactive( + merge_prs_recipe: dict, +) -> None: + step = merge_prs_recipe["steps"]["resolve_proactive_rebase_conflicts"] + capture_list = step.get("capture_list", {}) + assert any("conflict_report_path" in v for v in capture_list.values()), ( + "resolve_proactive_rebase_conflicts must accumulate conflict_report_path in capture_list" + ) + + def test_open_integration_pr_embeds_conflict_resolution_decisions(): """open-integration-pr SKILL.md must embed Conflict Resolution Decisions section.""" skill_md_path = SKILLS_ROOT / "open-integration-pr" / "SKILL.md" @@ -446,3 +456,39 @@ def test_resolve_merge_conflicts_no_hardcoded_origin(skill_md: str) -> None: "in git commands. Use $REMOTE (resolved via upstream || origin fallback) instead.\n" "Violations:\n" + "\n".join(f" {v}" for v in violations) ) + + +@pytest.fixture(scope="module") +def step5_section(skill_md: str) -> str: + step5_idx = skill_md.find("### Step 5 —") + step5a_idx = skill_md.find("### Step 5a —", step5_idx) + assert step5_idx != -1, "Step 5 section must be present in SKILL.md" + assert step5a_idx != -1, "Step 5a section must follow Step 5" + return skill_md[step5_idx:step5a_idx] + + +def test_resolve_merge_conflicts_step5_handles_version_consistency(step5_section: str) -> None: + """Step 5 must instruct running sync_versions.py when check-version-consistency fails.""" + assert "sync_versions.py" in step5_section, ( + "Step 5 must instruct running 'python3 scripts/sync_versions.py' " + "when check-version-consistency fails" + ) + + +def test_resolve_merge_conflicts_step5_handles_uv_lock(step5_section: str) -> None: + """Step 5 must instruct running 'uv lock' when uv-lock-check fails.""" + # "uv lock --check" appears in Step 5a (manifest validation); require bare "uv lock" + # to appear in Step 5 as the fix command (not merely the check) + assert re.search(r"\buv lock\b(?!\s+--check)", step5_section), ( + "Step 5 must instruct running 'uv lock' (without --check) " + "when uv-lock-check fails to regenerate the lock file" + ) + + +def test_resolve_merge_conflicts_step5_escalates_on_nonfixable_hooks(step5_section: str) -> None: + """Step 5 must escalate when pre-commit still fails after all auto-fixes are applied.""" + assert "escalation_required" in step5_section, ( + "Step 5 must escalate with escalation_required=true when pre-commit still fails " + "after all auto-fixes (ruff, sync_versions, uv lock) are applied — " + "remaining failures from non-fixable hooks (mypy, gitleaks) require manual remediation" + ) diff --git a/tests/skills/test_investigate_contracts.py b/tests/skills/test_investigate_contracts.py index d66393a7fb..fdf8f1afe5 100644 --- a/tests/skills/test_investigate_contracts.py +++ b/tests/skills/test_investigate_contracts.py @@ -8,15 +8,6 @@ import pytest -from autoskillit.core.paths import pkg_root - - -@pytest.fixture(scope="module") -def skill_text() -> str: - skill_path = pkg_root() / "skills_extended" / "investigate" / "SKILL.md" - assert skill_path.exists(), f"SKILL.md not found at {skill_path}" - return skill_path.read_text() - @pytest.fixture(scope="module") def step_35_section(skill_text: str) -> str: @@ -30,32 +21,6 @@ def step_35_section(skill_text: str) -> str: return skill_text[step_35_idx:] -@pytest.fixture(scope="module") -def report_section(skill_text: str) -> str: - """Extract the Step 4: Write Report section text. - - Uses the next top-level ``## `` heading that is NOT inside a fenced - code block as the boundary. The report template contains ``## `` - headings inside a markdown code fence (```markdown ... ```) which must - be included — so we skip ``## `` occurrences between fence markers. - """ - step_4_idx = skill_text.find("### Step 4:") - if step_4_idx == -1: - pytest.fail("Step 4 not found in investigate SKILL.md") - in_fence = False - lines = skill_text[step_4_idx:].split("\n") - for i, line in enumerate(lines): - if i == 0: - continue - if line.startswith("```"): - in_fence = not in_fence - continue - if not in_fence and line.startswith("## ") and not line.startswith("### "): - end_idx = step_4_idx + sum(len(prev) + 1 for prev in lines[:i]) - return skill_text[step_4_idx:end_idx] - return skill_text[step_4_idx:] - - def test_investigate_has_historical_recurrence_step(skill_text: str) -> None: """investigate must contain a Step 3.5: Historical Recurrence Check section.""" assert "Step 3.5" in skill_text, ( diff --git a/tests/skills/test_investigate_deep_mode_contracts.py b/tests/skills/test_investigate_deep_mode_contracts.py new file mode 100644 index 0000000000..948564f2cc --- /dev/null +++ b/tests/skills/test_investigate_deep_mode_contracts.py @@ -0,0 +1,369 @@ +"""Structural contracts for the investigate deep analysis mode. + +Validates that investigate/SKILL.md carries the required instructions +for Deep Analysis Mode: Steps D1–D6, activation conditions, subagent +templates, and enhanced report template fields. +""" + +import pytest + +from tests.skills.conftest import extract_step_section + + +@pytest.fixture(scope="module") +def deep_mode_section(skill_text: str) -> str: + """Extract the ## Deep Analysis Mode section text (up to the next ## heading).""" + start_idx = skill_text.find("## Deep Analysis Mode") + if start_idx == -1: + pytest.fail("'## Deep Analysis Mode' section not found in investigate SKILL.md") + # Find the next ## heading (not ###) after the section start + search_from = start_idx + len("## Deep Analysis Mode") + next_h2 = -1 + for i in range(search_from, len(skill_text) - 2): + if skill_text[i : i + 3] == "## " and skill_text[i - 1] == "\n": + next_h2 = i + break + if next_h2 != -1: + return skill_text[start_idx:next_h2] + return skill_text[start_idx:] + + +@pytest.fixture(scope="module") +def standard_workflow_section(skill_text: str) -> str: + """Extract ## Standard Mode Workflow section.""" + start_idx = skill_text.find("## Standard Mode Workflow") + if start_idx == -1: + pytest.fail("'## Standard Mode Workflow' not found in investigate SKILL.md") + deep_workflow_idx = skill_text.find("## Deep Analysis Mode Workflow", start_idx) + if deep_workflow_idx != -1: + return skill_text[start_idx:deep_workflow_idx] + return skill_text[start_idx:] + + +# ── Deep Mode Activation & Configuration ────────────────────────────────────── + + +def test_deep_mode_activation_via_depth_flag(deep_mode_section: str) -> None: + """Deep mode section must document the --depth deep activation flag.""" + assert "--depth deep" in deep_mode_section, ( + "Deep Analysis Mode section must document '--depth deep' as an activation flag" + ) + + +def test_deep_mode_activation_via_trigger_phrases(deep_mode_section: str) -> None: + """Deep mode section must document trigger phrases for activation.""" + has_investigate_deeply = "investigate deeply" in deep_mode_section.lower() + has_deep_analysis = "deep analysis" in deep_mode_section.lower() + assert has_investigate_deeply or has_deep_analysis, ( + "Deep Analysis Mode section must document activation trigger phrases such as " + "'investigate deeply' or 'deep analysis'" + ) + + +def test_deep_mode_not_default(deep_mode_section: str) -> None: + """Deep mode must not be enabled by default.""" + assert "never enabled by default" in deep_mode_section.lower(), ( + "Deep Analysis Mode section must state that it is 'never enabled by default'" + ) + + +def test_deep_mode_subagents_use_sonnet(deep_mode_section: str) -> None: + """Deep mode section must specify that subagents use model: sonnet.""" + assert 'model: "sonnet"' in deep_mode_section, ( + "Deep Analysis Mode section must specify 'model: \"sonnet\"' for all subagents" + ) + + +# ── Standard Mode Preserved ─────────────────────────────────────────────────── + + +def test_standard_mode_workflow_heading_exists(skill_text: str) -> None: + """'## Standard Mode Workflow' heading must be present.""" + assert "## Standard Mode Workflow" in skill_text, ( + "investigate SKILL.md must contain a '## Standard Mode Workflow' heading " + "to clearly separate standard and deep modes" + ) + + +def test_standard_mode_has_step_1_through_4(standard_workflow_section: str) -> None: + """Standard mode workflow must contain Steps 1, 2, 3, 3.5, and 4.""" + for step in ("### Step 1:", "### Step 2:", "### Step 3:", "Step 3.5", "### Step 4:"): + assert step in standard_workflow_section, ( + f"Standard Mode Workflow section must contain '{step}'" + ) + + +def test_standard_mode_scope_boundary_in_synthesis(standard_workflow_section: str) -> None: + """Step 3 synthesis list in standard mode must include Scope Boundary.""" + step_3_idx = standard_workflow_section.find("### Step 3:") + step_35_idx = standard_workflow_section.find("Step 3.5", step_3_idx) + step_3_text = standard_workflow_section[step_3_idx:step_35_idx] + assert "Scope Boundary" in step_3_text, ( + "Standard mode Step 3 synthesis list must include 'Scope Boundary' as a finding item" + ) + + +def test_standard_mode_confidence_levels_in_synthesis(standard_workflow_section: str) -> None: + """Step 3 synthesis list in standard mode must include Confidence Levels.""" + step_3_idx = standard_workflow_section.find("### Step 3:") + step_35_idx = standard_workflow_section.find("Step 3.5", step_3_idx) + step_3_text = standard_workflow_section[step_3_idx:step_35_idx] + assert "Confidence Levels" in step_3_text, ( + "Standard mode Step 3 synthesis list must include 'Confidence Levels' as a finding item" + ) + + +# ── Deep Workflow Steps D1–D6 ────────────────────────────────────────────────── + + +@pytest.mark.parametrize("step", ["D1", "D2", "D3", "D4", "D5", "D6"]) +def test_deep_workflow_has_step(deep_workflow_section: str, step: str) -> None: + """Deep workflow must contain each D-step heading.""" + assert f"### Step {step}" in deep_workflow_section, ( + f"Deep Analysis Mode Workflow must contain a '### Step {step}' heading" + ) + + +def test_d1_batch_plan_proposal(deep_workflow_section: str) -> None: + """Step D1 must describe proposing a batch plan.""" + d1 = extract_step_section(deep_workflow_section, "Step D1") + has_batch_plan = "batch plan" in d1.lower() + has_proposed = "Proposed investigation plan" in d1 + assert has_batch_plan or has_proposed, ( + "Step D1 must describe proposing a 'batch plan' or 'Proposed investigation plan'" + ) + + +def test_d1_headless_auto_approve(deep_workflow_section: str) -> None: + """Step D1 must reference AUTOSKILLIT_HEADLESS for auto-approve.""" + d1 = extract_step_section(deep_workflow_section, "Step D1") + assert "AUTOSKILLIT_HEADLESS" in d1, ( + "Step D1 must reference 'AUTOSKILLIT_HEADLESS' environment variable for " + "headless auto-approval of the batch plan" + ) + + +def test_d2_broad_exploration_minimum_subagents(deep_workflow_section: str) -> None: + """Step D2 must specify a minimum of 5 parallel subagents.""" + d2 = extract_step_section(deep_workflow_section, "Step D2") + has_minimum = "minimum" in d2.lower() + has_five = "5" in d2 + assert has_minimum and has_five, ( + "Step D2 must specify a 'minimum' of '5' parallel subagents for broad exploration" + ) + + +def test_d2_inter_batch_synthesis(deep_workflow_section: str) -> None: + """Step D2 must describe inter-batch synthesis.""" + d2 = extract_step_section(deep_workflow_section, "Step D2") + assert "inter-batch synthesis" in d2.lower(), ( + "Step D2 must describe 'inter-batch synthesis' after Batch 1 completes" + ) + + +def test_d2_historical_recurrence_parallel(deep_workflow_section: str) -> None: + """Step D2 must reference Step 3.5 or historical check running in parallel.""" + d2 = extract_step_section(deep_workflow_section, "Step D2") + has_35 = "3.5" in d2 + has_historical = "historical" in d2.lower() + assert has_35 or has_historical, ( + "Step D2 must reference '3.5' or 'historical' check running in parallel with Batch 1" + ) + + +def test_d3_mandatory_web_research(deep_workflow_section: str) -> None: + """Step D3 must require web research in each batch.""" + d3 = extract_step_section(deep_workflow_section, "Step D3") + assert "web research" in d3.lower(), ( + "Step D3 must describe mandatory 'web research' in each deepening batch" + ) + + +def test_d3_mandatory_code_exploration(deep_workflow_section: str) -> None: + """Step D3 must require code exploration in each batch.""" + d3 = extract_step_section(deep_workflow_section, "Step D3") + has_code_exploration = "code exploration" in d3.lower() + has_local_code = "local code" in d3.lower() + assert has_code_exploration or has_local_code, ( + "Step D3 must describe mandatory 'code exploration' or 'local code' search " + "in each deepening batch" + ) + + +def test_d3_early_termination(deep_workflow_section: str) -> None: + """Step D3 must describe early termination conditions.""" + d3 = extract_step_section(deep_workflow_section, "Step D3") + assert "early termination" in d3.lower(), ( + "Step D3 must describe 'early termination' conditions for when all findings " + "are SUPPORTED and no new leads emerge" + ) + + +def test_d4_challenge_round_adversarial(deep_workflow_section: str) -> None: + """Step D4 must describe an adversarial subagent.""" + d4 = extract_step_section(deep_workflow_section, "Step D4") + assert "adversarial" in d4.lower(), ( + "Step D4 must describe an 'adversarial' subagent for the challenge round" + ) + + +def test_d4_needs_evidence_trigger(deep_workflow_section: str) -> None: + """Step D4 must be triggered by NEEDS-EVIDENCE findings.""" + d4 = extract_step_section(deep_workflow_section, "Step D4") + assert "NEEDS-EVIDENCE" in d4, ( + "Step D4 must specify that it fires when a finding is marked 'NEEDS-EVIDENCE'" + ) + + +def test_d4_prior_fix_falsifiability(deep_workflow_section: str) -> None: + """Step D4 must reference prior fix falsifiability check.""" + d4 = extract_step_section(deep_workflow_section, "Step D4") + has_falsifiability = "falsifiability" in d4.lower() + has_prior_fix = "prior fix" in d4.lower() + assert has_falsifiability or has_prior_fix, ( + "Step D4 must describe 'falsifiability' check or 'prior fix' assessment" + ) + + +def test_d5_blast_radius(deep_workflow_section: str) -> None: + """Step D5 must describe blast radius analysis.""" + d5 = extract_step_section(deep_workflow_section, "Step D5") + assert "blast radius" in d5.lower(), ( + "Step D5 must describe 'blast radius' analysis for candidate solutions" + ) + + +def test_d5_single_recommendation(deep_workflow_section: str) -> None: + """Step D5 must converge to a single recommendation.""" + d5 = extract_step_section(deep_workflow_section, "Step D5") + assert "single recommendation" in d5.lower(), ( + "Step D5 must converge to a 'single recommendation' after blast radius analysis" + ) + + +def test_d6_post_report_validation(deep_workflow_section: str) -> None: + """Step D6 must describe post-report validation.""" + d6 = extract_step_section(deep_workflow_section, "Step D6") + assert "validation" in d6.lower(), "Step D6 must describe post-report 'validation'" + + +def test_d6_independent_validators(deep_workflow_section: str) -> None: + """Step D6 must specify 2 or 3 independent validators.""" + d6 = extract_step_section(deep_workflow_section, "Step D6") + has_count = "2" in d6 or "3" in d6 + has_validator = "validator" in d6.lower() + assert has_count and has_validator, ( + "Step D6 must specify '2' or '3' independent 'validator' subagents" + ) + + +def test_d6_factual_accuracy_check(deep_workflow_section: str) -> None: + """Step D6 must include a factual accuracy check.""" + d6 = extract_step_section(deep_workflow_section, "Step D6") + assert "factual accuracy" in d6.lower(), ( + "Step D6 must include a 'factual accuracy' validator role" + ) + + +# ── Enhanced Report Template ─────────────────────────────────────────────────── + + +def test_report_mode_field(report_section: str) -> None: + """Step 4 report template must include a Mode field.""" + assert "Mode" in report_section, ( + "Step 4 report template must include a '**Mode:**' field in the report header" + ) + + +def test_report_scope_boundary_section(report_section: str) -> None: + """Step 4 report template must include a Scope Boundary section.""" + assert "Scope Boundary" in report_section, ( + "Step 4 report template must include a '## Scope Boundary' section" + ) + + +def test_report_confidence_levels(report_section: str) -> None: + """Step 4 report template must show SUPPORTED, UNSUPPORTED, and NEEDS-EVIDENCE.""" + assert "SUPPORTED" in report_section, ( + "Step 4 report template must reference 'SUPPORTED' confidence label" + ) + assert "UNSUPPORTED" in report_section or "NEEDS-EVIDENCE" in report_section, ( + "Step 4 report template must reference 'UNSUPPORTED' or 'NEEDS-EVIDENCE' confidence labels" + ) + + +def test_report_no_auto_file_issues(skill_text: str) -> None: + """NEVER constraints must prohibit automatic GitHub issue filing.""" + never_idx = skill_text.find("**NEVER:**") + assert never_idx != -1, "NEVER constraints block not found in investigate SKILL.md" + # Find the ALWAYS block to bound the search + always_idx = skill_text.find("**ALWAYS:**", never_idx) + never_block = skill_text[never_idx:always_idx] if always_idx != -1 else skill_text[never_idx:] + has_issue = "issue" in never_block.lower() + has_prohibition = "automatically" in never_block.lower() + assert has_issue and has_prohibition, ( + "NEVER constraints must prohibit filing GitHub issues automatically" + ) + + +# ── Subagent Templates ───────────────────────────────────────────────────────── + + +def test_deep_mode_subagent_template_exists(skill_text: str) -> None: + """SKILL.md must contain a Deep Analysis Mode Template section.""" + assert "Deep Analysis Mode Template" in skill_text, ( + "investigate SKILL.md must contain a 'Deep Analysis Mode Template' section " + "under the Subagent Prompt Template heading" + ) + + +def test_adversarial_subagent_template_exists(skill_text: str) -> None: + """SKILL.md must contain an Adversarial Subagent Template section.""" + assert "Adversarial Subagent Template" in skill_text, ( + "investigate SKILL.md must contain an 'Adversarial Subagent Template' section " + "for the D4 challenge round" + ) + + +def test_validation_subagent_template_exists(skill_text: str) -> None: + """SKILL.md must contain a Validation Subagent Template section.""" + assert "Validation Subagent Template" in skill_text, ( + "investigate SKILL.md must contain a 'Validation Subagent Template' section " + "for the D6 post-report validators" + ) + + +def test_deep_template_has_evidence_standards(skill_text: str) -> None: + """Deep Analysis Mode Template must include Evidence standards block.""" + deep_template_idx = skill_text.find("Deep Analysis Mode Template") + if deep_template_idx == -1: + pytest.fail("'Deep Analysis Mode Template' not found in investigate SKILL.md") + adversarial_idx = skill_text.find("Adversarial Subagent Template", deep_template_idx) + template_section = ( + skill_text[deep_template_idx:adversarial_idx] + if adversarial_idx != -1 + else skill_text[deep_template_idx:] + ) + assert "Evidence standards" in template_section, ( + "Deep Analysis Mode Template must include an 'Evidence standards' block " + "instructing subagents to cite files, lines, and mark confidence levels" + ) + + +def test_deep_template_has_inter_batch_context(skill_text: str) -> None: + """Deep Analysis Mode Template must reference prior batch context.""" + deep_template_idx = skill_text.find("Deep Analysis Mode Template") + if deep_template_idx == -1: + pytest.fail("'Deep Analysis Mode Template' not found in investigate SKILL.md") + adversarial_idx = skill_text.find("Adversarial Subagent Template", deep_template_idx) + template_section = ( + skill_text[deep_template_idx:adversarial_idx] + if adversarial_idx != -1 + else skill_text[deep_template_idx:] + ) + has_prior_batches = "prior batches" in template_section.lower() + has_previous_batch = "previous batch" in template_section.lower() + assert has_prior_batches or has_previous_batch, ( + "Deep Analysis Mode Template must include a placeholder for context from " + "'prior batches' or 'previous batch' inter-batch synthesis" + ) diff --git a/tests/skills/test_investigate_design_intent_contracts.py b/tests/skills/test_investigate_design_intent_contracts.py new file mode 100644 index 0000000000..abf2a47b90 --- /dev/null +++ b/tests/skills/test_investigate_design_intent_contracts.py @@ -0,0 +1,313 @@ +"""Contract tests for Design Intent Analysis requirements in the investigate skill. + +REQ-INTENT-001: Design Intent subagent — proactively discovers why mechanisms exist + and what depends on them (standard and deep modes). +REQ-INTENT-002: Adversarial Breakage subagent — challenges removal/change recommendations + by tracing what would break (deep mode only). +""" + +import pytest + +from tests.skills.conftest import extract_step_section + + +@pytest.fixture(scope="module") +def standard_workflow_section(skill_text: str) -> str: + start = skill_text.find("## Standard Mode Workflow") + if start == -1: + return "" + deep_start = skill_text.find("## Deep Analysis Mode Workflow", start + 1) + if deep_start != -1: + return skill_text[start:deep_start] + return skill_text[start:] + + +@pytest.fixture(scope="module") +def step2_section(skill_text: str) -> str: + start = skill_text.find("### Step 2:") + if start == -1: + return "" + end = skill_text.find("### Step 3:", start + 1) + if end != -1: + return skill_text[start:end] + return skill_text[start:] + + +@pytest.fixture(scope="module") +def design_intent_content(step2_section: str) -> str: + start = step2_section.find("**Design Intent**") + if start == -1: + return "" + # Find next bold header or end of step2 + next_header = step2_section.find("\n**", start + 1) + if next_header != -1: + return step2_section[start:next_header] + return step2_section[start:] + + +def _strip_code_fences(text: str) -> str: + """Remove lines inside triple-backtick code fences (template content vs. instructions).""" + result = [] + in_fence = False + for line in text.split("\n"): + if line.strip().startswith("```"): + in_fence = not in_fence + elif not in_fence: + result.append(line) + return "\n".join(result) + + +@pytest.fixture(scope="module") +def d3_section(deep_workflow_section: str) -> str: + return extract_step_section(deep_workflow_section, "Step D3") + + +@pytest.fixture(scope="module") +def d5_section(deep_workflow_section: str) -> str: + return extract_step_section(deep_workflow_section, "Step D5") + + +def test_step2_has_design_intent_vector(step2_section: str) -> None: + """Step 2 section must contain Design Intent as a parallel investigation vector.""" + assert "Design Intent" in step2_section, ( + "Step 2 must include 'Design Intent' as a parallel investigation vector" + ) + + +def test_design_intent_uses_git_log_follow(design_intent_content: str) -> None: + """Design Intent content must contain 'git log --follow'.""" + assert "git log --follow" in design_intent_content, ( + "Design Intent vector must use 'git log --follow' to trace the introducing commit" + ) + + +def test_design_intent_traces_callers_dependents(design_intent_content: str) -> None: + """Design Intent content must mention callers/dependents tracing.""" + has_callers = "callers" in design_intent_content.lower() + has_dependents = "dependents" in design_intent_content.lower() + assert has_callers or has_dependents, ( + "Design Intent vector must mention tracing callers or dependents" + ) + + +def test_design_intent_checks_architecture_docs(design_intent_content: str) -> None: + """Design Intent content must mention architecture docs.""" + has_arch = "architecture" in design_intent_content.lower() + has_claude_md = "CLAUDE.md" in design_intent_content + has_adr = "ADR" in design_intent_content + assert has_arch or has_claude_md or has_adr, ( + "Design Intent vector must mention architecture docs (architecture.md, CLAUDE.md, or ADRs)" + ) + + +def test_design_intent_produces_finding_per_mechanism(design_intent_content: str) -> None: + """Design Intent content must mention 'design intent finding'.""" + assert "design intent finding" in design_intent_content.lower(), ( + "Design Intent vector must produce a 'design intent finding' per mechanism" + ) + + +def test_step3_synthesis_includes_design_intent_findings(skill_text: str) -> None: + """Step 3 numbered list must include 'Design Intent Findings'.""" + start = skill_text.find("### Step 3:") + end = skill_text.find("### Step 3.5", start + 1) + step3 = skill_text[start:end] if end != -1 else skill_text[start:] + assert "Design Intent Findings" in step3, ( + "Step 3 synthesis list must include 'Design Intent Findings'" + ) + + +def test_report_template_has_design_intent_section(skill_text: str) -> None: + """Step 4 report template must include '## Design Intent Findings'.""" + assert "## Design Intent Findings" in skill_text, ( + "Step 4 report template must include '## Design Intent Findings' section" + ) + + +def test_design_intent_section_positioned_after_similar_patterns(skill_text: str) -> None: + """Design Intent Findings must appear after Similar Patterns in the report template.""" + similar_pos = skill_text.find("## Similar Patterns") + design_intent_pos = skill_text.find("## Design Intent Findings") + historical_pos = skill_text.find("## Historical Context") + assert similar_pos != -1 and design_intent_pos != -1, ( + "Both '## Similar Patterns' and '## Design Intent Findings' must exist in report template" + ) + assert similar_pos < design_intent_pos, ( + "'## Design Intent Findings' must appear after '## Similar Patterns' in report template" + ) + assert historical_pos != -1, "'## Historical Context' must exist in report template" + assert design_intent_pos < historical_pos, ( + "'## Design Intent Findings' must appear before '## Historical Context' in report template" + ) + + +def test_d2_includes_design_intent_subagent(deep_workflow_section: str) -> None: + """Step D2 must mention Design Intent as a parallel subagent.""" + d2 = extract_step_section(deep_workflow_section, "Step D2") + assert "Design Intent" in d2, "Step D2 must include 'Design Intent' as a parallel subagent" + + +def test_d2_minimum_subagents_increased(deep_workflow_section: str) -> None: + """Step D2 must specify a minimum of 5 parallel subagents.""" + d2 = extract_step_section(deep_workflow_section, "Step D2") + has_minimum = "minimum" in d2.lower() + has_five = "5" in d2 + assert has_minimum and has_five, ( + "Step D2 must specify a 'minimum' of '5' parallel subagents" + " (was 4, increased for Design Intent)" + ) + + +def test_d3_design_intent_redispatch(d3_section: str) -> None: + """Step D3 must describe re-dispatching Design Intent when new mechanisms surface.""" + has_design_intent = "Design Intent" in d3_section + has_redispatch = "re-dispatch" in d3_section.lower() or "redispatch" in d3_section.lower() + assert has_design_intent and has_redispatch, ( + "Step D3 must mention re-dispatching the Design Intent subagent" + " for newly surfaced mechanisms" + ) + + +def test_d5_has_adversarial_breakage_subagent(d5_section: str) -> None: + """Step D5 must contain adversarial breakage or breakage analysis.""" + has_adversarial = "adversarial breakage" in d5_section.lower() + has_breakage = "breakage analysis" in d5_section.lower() + assert has_adversarial or has_breakage, ( + "Step D5 must include 'adversarial breakage' or 'breakage analysis'" + ) + + +def test_d5_breakage_triggers_on_removal_change(d5_section: str) -> None: + """Step D5 breakage must fire on removal/replacement/change recommendations.""" + has_removal = "removal" in d5_section.lower() + has_replacement = "replacement" in d5_section.lower() + has_change = "change" in d5_section.lower() + assert has_removal or has_replacement or has_change, ( + "Step D5 breakage analysis must trigger on removal, replacement, or change recommendations" + ) + + +def test_d5_breakage_traces_dependency_chain(d5_section: str) -> None: + """Step D5 breakage must mention dependency chain tracing.""" + has_dependency = "dependency chain" in d5_section.lower() + has_callers = "callers" in d5_section.lower() + has_importers = "importers" in d5_section.lower() + assert has_dependency or has_callers or has_importers, ( + "Step D5 breakage analysis must mention dependency chain tracing" + " (callers, importers, flag consumers)" + ) + + +def test_d5_breakage_checks_revert_patterns(d5_section: str) -> None: + """Step D5 breakage must mention checking git history for revert patterns.""" + has_revert = "revert" in d5_section.lower() + has_git = "git" in d5_section.lower() + assert has_revert and has_git, ( + "Step D5 breakage analysis must mention checking git history for revert patterns" + ) + + +def test_d5_breakage_distinct_from_d4(deep_workflow_section: str) -> None: + """D4 is epistemological (hypothesis); D5 breakage is consequentialist (recommendation).""" + d4 = extract_step_section(deep_workflow_section, "Step D4") + d5 = extract_step_section(deep_workflow_section, "Step D5") + # D4: epistemological — about hypothesis/root cause + d4_epistemic = "hypothesis" in d4.lower() or "root cause" in d4.lower() + # D5: consequentialist — about recommendation and what breaks + d5_consequentialist = "recommendation" in d5.lower() and "break" in d5.lower() + assert d4_epistemic, "D4 must mention 'hypothesis' or 'root cause' (epistemological trigger)" + assert d5_consequentialist, ( + "D5 must mention 'recommendation' and 'break' (consequentialist trigger, distinct from D4)" + ) + + +def test_report_template_has_breakage_analysis_deep_mode(skill_text: str) -> None: + """Step 4 report template must include breakage analysis for deep mode.""" + has_breakage = "## Breakage Analysis" in skill_text + assert has_breakage, ( + "Step 4 report template must include '## Breakage Analysis' section for deep mode" + ) + + +def test_standard_mode_no_adversarial_breakage(standard_workflow_section: str) -> None: + """Standard mode workflow instructions must NOT contain adversarial breakage content.""" + # Strip code fences: the Step 4 report template documents what deep mode output looks + # like, which legitimately contains '## Breakage Analysis'. The intent of this guard + # is that the *execution instructions* don't spawn adversarial breakage in standard mode. + instructions = _strip_code_fences(standard_workflow_section) + has_adversarial = "adversarial breakage" in instructions.lower() + has_breakage_analysis = "breakage analysis" in instructions.lower() + assert not has_adversarial and not has_breakage_analysis, ( + "Standard mode workflow instructions must not contain" + " 'adversarial breakage' or 'breakage analysis'" + ) + + +def test_investigation_path_output_contract_unchanged(skill_text: str) -> None: + """SKILL.md must still contain the investigation_path output token pattern.""" + assert "investigation_path = " in skill_text, ( + "SKILL.md must still contain 'investigation_path = ' structured output token" + ) + + +def test_design_intent_subagent_template_exists(skill_text: str) -> None: + """'Design Intent Subagent Template' heading must exist in SKILL.md.""" + assert "Design Intent Subagent Template" in skill_text, ( + "SKILL.md must contain a 'Design Intent Subagent Template' heading" + ) + + +def test_design_intent_template_has_git_log_follow(skill_text: str) -> None: + """Design Intent template must contain 'git log --follow'.""" + template_start = skill_text.find("Design Intent Subagent Template") + if template_start == -1: + pytest.fail("Design Intent Subagent Template section not found") + # Find the next template heading after this one + next_template = skill_text.find("###", template_start + 1) + template_section = ( + skill_text[template_start:next_template] + if next_template != -1 + else skill_text[template_start:] + ) + assert "git log --follow" in template_section, ( + "Design Intent Subagent Template must contain 'git log --follow'" + ) + + +def test_adversarial_breakage_subagent_template_exists(skill_text: str) -> None: + """'Adversarial Breakage Subagent Template' heading must exist in SKILL.md.""" + assert "Adversarial Breakage Subagent Template" in skill_text, ( + "SKILL.md must contain an 'Adversarial Breakage Subagent Template' heading" + ) + + +def test_adversarial_breakage_template_has_dependency_chain(skill_text: str) -> None: + """Adversarial Breakage template must mention dependency chain.""" + template_start = skill_text.find("Adversarial Breakage Subagent Template") + if template_start == -1: + pytest.fail("Adversarial Breakage Subagent Template section not found") + next_template = skill_text.find("###", template_start + 1) + template_section = ( + skill_text[template_start:next_template] + if next_template != -1 + else skill_text[template_start:] + ) + assert "dependency chain" in template_section.lower(), ( + "Adversarial Breakage Subagent Template must mention dependency chain" + ) + + +def test_adversarial_breakage_template_has_revert_check(skill_text: str) -> None: + """Adversarial Breakage template must mention revert pattern check.""" + template_start = skill_text.find("Adversarial Breakage Subagent Template") + if template_start == -1: + pytest.fail("Adversarial Breakage Subagent Template section not found") + next_template = skill_text.find("###", template_start + 1) + template_section = ( + skill_text[template_start:next_template] + if next_template != -1 + else skill_text[template_start:] + ) + assert "revert" in template_section.lower(), ( + "Adversarial Breakage Subagent Template must mention revert pattern check" + ) diff --git a/tests/skills/test_isolation_guidance_contracts.py b/tests/skills/test_isolation_guidance_contracts.py new file mode 100644 index 0000000000..8fa4301d4e --- /dev/null +++ b/tests/skills/test_isolation_guidance_contracts.py @@ -0,0 +1,161 @@ +"""Contract tests verifying shared mutable state isolation guidance exists in pipeline skills.""" + +import pytest + +from autoskillit.core.paths import pkg_root + + +@pytest.fixture(scope="module") +def make_plan_text() -> str: + p = pkg_root() / "skills_extended" / "make-plan" / "SKILL.md" + return p.read_text() + + +@pytest.fixture(scope="module") +def dry_walkthrough_text() -> str: + p = pkg_root() / "skills_extended" / "dry-walkthrough" / "SKILL.md" + return p.read_text() + + +@pytest.fixture(scope="module") +def resolve_failures_text() -> str: + p = pkg_root() / "skills_extended" / "resolve-failures" / "SKILL.md" + return p.read_text() + + +@pytest.fixture(scope="module") +def tests_claude_md_text() -> str: + p = pkg_root().parent.parent / "tests" / "CLAUDE.md" + return p.read_text() + + +def test_make_plan_step1_reads_isolation_patterns(make_plan_text: str) -> None: + planning_steps_idx = make_plan_text.find("## Planning Steps") + assert planning_steps_idx != -1, "make-plan SKILL.md must have a '## Planning Steps' section" + step1_idx = make_plan_text.find("**Understand related systems", planning_steps_idx) + step2_idx = make_plan_text.find("**Explore and design approaches", planning_steps_idx) + assert step1_idx != -1 and step2_idx != -1 + step1_section = make_plan_text[step1_idx:step2_idx] + has_isolation_read = "isolation" in step1_section.lower() and ( + "singleton" in step1_section.lower() + or "module-level" in step1_section.lower() + or "mutating" in step1_section.lower() + ) + assert has_isolation_read, ( + "make-plan Step 1 must instruct reading existing test isolation patterns " + "when the plan involves tests that call mutating methods on singletons or " + "module-level objects" + ) + + +def test_make_plan_step3_isolation_contract(make_plan_text: str) -> None: + planning_steps_idx = make_plan_text.find("## Planning Steps") + assert planning_steps_idx != -1, "make-plan SKILL.md must have a '## Planning Steps' section" + step3_idx = make_plan_text.find("**Design tests first", planning_steps_idx) + step4_idx = make_plan_text.find("**Evaluate approaches", planning_steps_idx) + assert step3_idx != -1 and step4_idx != -1 + step3_section = make_plan_text[step3_idx:step4_idx] + has_isolation_contract = "isolation" in step3_section.lower() and ( + "cleanup" in step3_section.lower() or "reset" in step3_section.lower() + ) + assert has_isolation_contract, ( + "make-plan Step 3 must include a test isolation contract requiring plans " + "to specify how shared state is reset between tests when mutating shared objects" + ) + + +def test_make_plan_step3_incomplete_without_cleanup(make_plan_text: str) -> None: + planning_steps_idx = make_plan_text.find("## Planning Steps") + assert planning_steps_idx != -1, "make-plan SKILL.md must have a '## Planning Steps' section" + step3_idx = make_plan_text.find("**Design tests first", planning_steps_idx) + step4_idx = make_plan_text.find("**Evaluate approaches", planning_steps_idx) + assert step3_idx != -1 and step4_idx != -1 + step3_section = make_plan_text[step3_idx:step4_idx] + assert "incomplete" in step3_section.lower(), ( + "make-plan Step 3 must explicitly label plans that prescribe mutating shared " + "state without specifying cleanup as 'incomplete'" + ) + + +def test_dry_walkthrough_step2_has_shared_state_check(dry_walkthrough_text: str) -> None: + step2_idx = dry_walkthrough_text.find("### Step 2:") + step3_idx = dry_walkthrough_text.find("### Step 3:") + assert step2_idx != -1 and step3_idx != -1 + step2_section = dry_walkthrough_text[step2_idx:step3_idx] + has_shared_state = ( + "shared" in step2_section.lower() + and ( + "cleanup" in step2_section.lower() + or "restore" in step2_section.lower() + or "reset" in step2_section.lower() + ) + and ( + "singleton" in step2_section.lower() + or "module-scope" in step2_section.lower() + or "mutating" in step2_section.lower() + ) + ) + assert has_shared_state, ( + "dry-walkthrough Step 2 checklist must include a check that verifies " + "plans specify cleanup for mutating methods on shared/module-scope objects" + ) + + +def test_dry_walkthrough_step2_mechanical_scan_instruction(dry_walkthrough_text: str) -> None: + step2_idx = dry_walkthrough_text.find("### Step 2:") + step3_idx = dry_walkthrough_text.find("### Step 3:") + assert step2_idx != -1 and step3_idx != -1 + step2_section = dry_walkthrough_text[step2_idx:step3_idx] + has_mechanical = "scan" in step2_section.lower() or "search" in step2_section.lower() + assert has_mechanical, ( + "dry-walkthrough Step 2 shared-state check must include a mechanical scan " + "instruction (e.g., 'scan the plan for method calls on module-scope objects') " + "so the LLM has a concrete action, not just an open-ended question" + ) + + +def test_resolve_failures_accumulation_pattern_guidance(resolve_failures_text: str) -> None: + constraints_idx = resolve_failures_text.find("## Critical Constraints") + assert constraints_idx != -1, ( + "resolve-failures SKILL.md must have a '## Critical Constraints' section" + ) + assert "accumulation-based" in resolve_failures_text[constraints_idx:].lower(), ( + "resolve-failures must describe the 'accumulation-based' pattern — shared state " + "growing unboundedly rather than toggling — as a distinct non-determinism cause" + ) + + +def test_resolve_failures_inverse_method_warning(resolve_failures_text: str) -> None: + has_inverse_warning = ( + "inverse" in resolve_failures_text.lower() or "append" in resolve_failures_text.lower() + ) and "does not reset" in resolve_failures_text.lower() + assert has_inverse_warning, ( + "resolve-failures must warn that calling an inverse method (e.g., disable() " + "to undo enable()) does not reset accumulation-based state if the framework " + "appends rather than toggles" + ) + + +def test_resolve_failures_full_reset_prescription(resolve_failures_text: str) -> None: + assert "clear the collection" in resolve_failures_text.lower(), ( + "resolve-failures must prescribe clearing the collection (full reset) " + "as the fix for accumulation-based state leakage, not inverse operations" + ) + + +def test_tests_claude_md_fastmcp_singleton_rule(tests_claude_md_text: str) -> None: + assert "_transforms" in tests_claude_md_text, ( + "tests/CLAUDE.md must document the mcp._transforms accumulation behavior " + "under the xdist compatibility section" + ) + + +def test_tests_claude_md_clear_restore_pattern(tests_claude_md_text: str) -> None: + has_clear_restore = "_transforms.clear()" in tests_claude_md_text and ( + "disable" in tests_claude_md_text or "baseline" in tests_claude_md_text + ) + assert has_clear_restore, ( + "tests/CLAUDE.md must prescribe the clear+restore pattern: " + "call mcp._transforms.clear() then re-apply baseline state, " + "not mcp.disable() as a teardown" + ) diff --git a/tests/skills/test_make_campaign_compliance.py b/tests/skills/test_make_campaign_compliance.py new file mode 100644 index 0000000000..b35e25a191 --- /dev/null +++ b/tests/skills/test_make_campaign_compliance.py @@ -0,0 +1,74 @@ +"""Compliance tests for the make-campaign skill. + +Verifies skill classification, placeholder hygiene, and contract registration. +No pytestmark layer marker — tests/skills/ is out of scope for layer markers. +""" + +from __future__ import annotations + +import re + +import yaml + +from autoskillit.core import pkg_root +from autoskillit.recipe._skill_placeholder_parser import ( + extract_bash_blocks, + extract_bash_placeholders, + extract_declared_ingredients, + shell_vars_assigned, +) +from tests.skills.test_skill_output_compliance import FIXED_NAME_SKILLS + +_CONTRACTS_PATH = pkg_root() / "recipe" / "skill_contracts.yaml" +_SKILL_MD_PATH = pkg_root() / "skills_extended" / "make-campaign" / "SKILL.md" + + +def test_make_campaign_in_fixed_name_skills() -> None: + """make-campaign IS in FIXED_NAME_SKILLS (name-based output, no timestamp needed).""" + assert "make-campaign" in FIXED_NAME_SKILLS, ( + "make-campaign must be in FIXED_NAME_SKILLS — output is identity-based " + "(.autoskillit/recipes/campaigns/.yaml), no timestamp needed." + ) + + +def test_make_campaign_no_undefined_placeholders() -> None: + """No bash code block in make-campaign SKILL.md uses an undefined placeholder.""" + content = _SKILL_MD_PATH.read_text() + bash_blocks = extract_bash_blocks(content) + placeholders = extract_bash_placeholders(bash_blocks) + declared = extract_declared_ingredients(content) + assigned = shell_vars_assigned(bash_blocks) + allowed = declared | assigned + + undefined = placeholders - allowed + assert not undefined, ( + f"make-campaign SKILL.md has undefined placeholders in bash blocks: {undefined!r}. " + "Either declare them as ingredients or add them to the pseudocode allowlist." + ) + + +def test_make_campaign_skill_contract_registered() -> None: + """make-campaign has a skill_contracts.yaml entry with campaign_path file_path output.""" + raw = yaml.safe_load(_CONTRACTS_PATH.read_text()) + skills = raw.get("skills", {}) if isinstance(raw, dict) else {} + + assert "make-campaign" in skills, "make-campaign must be registered in skill_contracts.yaml" + contract = skills["make-campaign"] + outputs = contract.get("outputs", []) + output_names = {o["name"]: o for o in outputs if isinstance(o, dict)} + + assert "campaign_path" in output_names, ( + "make-campaign contract must declare a campaign_path output. " + f"Found outputs: {list(output_names)!r}" + ) + assert output_names["campaign_path"].get("type") == "file_path", ( + "campaign_path output must have type='file_path'. " + f"Got: {output_names['campaign_path'].get('type')!r}" + ) + + patterns = contract.get("expected_output_patterns", []) + has_path_pattern = any(re.search(r"campaign_path", p) for p in patterns) + assert has_path_pattern, ( + "make-campaign contract must have an expected_output_pattern referencing campaign_path. " + f"Found patterns: {patterns!r}" + ) diff --git a/tests/skills/test_merge_pr_ci_gate.py b/tests/skills/test_merge_pr_ci_gate.py index 8b0b4cfc46..63b81aaa49 100644 --- a/tests/skills/test_merge_pr_ci_gate.py +++ b/tests/skills/test_merge_pr_ci_gate.py @@ -2,6 +2,8 @@ from __future__ import annotations +import re + from autoskillit.core.paths import pkg_root SKILL_PATH = pkg_root() / "skills_extended" / "merge-pr" / "SKILL.md" @@ -43,3 +45,40 @@ def test_merge_pr_skill_references_plain_squash_fallback() -> None: "merge-pr SKILL.md must reference plain '--squash' (without --auto) as a " "fallback for repos where autoMergeAllowed=false" ) + + +def test_merge_pr_skill_has_pre_flight_mergeability_check() -> None: + """Test 1e: merge-pr SKILL.md must contain a pre-flight mergeability check.""" + content = SKILL_PATH.read_text() + step19_pos = content.find("### Step 1.9") + assert step19_pos != -1, "Step 1.9 heading must exist in SKILL.md" + step2_pos = content.find("### Step 2") + assert step2_pos != -1, "Step 2 heading must exist in SKILL.md" + step19_section = content[step19_pos:step2_pos] + assert "mergeable" in step19_section, ( + "merge-pr SKILL.md Step 1.9 must contain a pre-flight mergeability check" + ) + merge_cmd_pos = content.find("gh pr merge", step2_pos) + assert merge_cmd_pos != -1, "gh pr merge command must exist in Step 2 of SKILL.md" + assert step19_pos < merge_cmd_pos, ( + "Pre-flight mergeability check (Step 1.9) must appear before" + " the gh pr merge command in Step 2" + ) + + +def test_merge_pr_skill_has_timeout_output_template() -> None: + """Test 1f: merge-pr SKILL.md Step 5 output templates must include a timeout case.""" + content = SKILL_PATH.read_text() + assert "timeout_error" in content, ( + "merge-pr SKILL.md Step 5 output templates must include a timeout case" + ) + # Verify timeout template has merged=false and timeout_error=true + timeout_section_match = re.search( + r'On timeout.*?\{[^}]*"merged":\s*false[^}]*"timeout_error":\s*true[^}]*\}', + content, + re.DOTALL, + ) + assert timeout_section_match is not None, ( + "merge-pr SKILL.md must have a timeout output case with " + "merged=false and timeout_error=true" + ) diff --git a/tests/skills/test_open_research_pr_decomposition.py b/tests/skills/test_open_research_pr_decomposition.py index f8a4aeca0b..2e6971c754 100644 --- a/tests/skills/test_open_research_pr_decomposition.py +++ b/tests/skills/test_open_research_pr_decomposition.py @@ -48,14 +48,6 @@ def test_compose_research_pr_skill_exists(): assert path.exists() -def test_open_research_pr_skill_retired(): - """open-research-pr must be removed — it has been retired.""" - path = _SKILLS_ROOT / "open-research-pr" / "SKILL.md" - assert not path.exists(), ( - "open-research-pr/SKILL.md still exists — remove it once new skills are in place" - ) - - @pytest.mark.parametrize( "slug", [ diff --git a/tests/test_phase2_skills.py b/tests/skills/test_phase2_skills.py similarity index 100% rename from tests/test_phase2_skills.py rename to tests/skills/test_phase2_skills.py diff --git a/tests/skills/test_planner_extract_domain.py b/tests/skills/test_planner_extract_domain.py new file mode 100644 index 0000000000..0532fe2252 --- /dev/null +++ b/tests/skills/test_planner_extract_domain.py @@ -0,0 +1,22 @@ +import pytest + +from autoskillit.core.paths import pkg_root + +pytestmark = [pytest.mark.layer("skills"), pytest.mark.small] + + +def test_planner_extract_domain_skill_uses_positional_args(): + """SKILL.md must reference $1 and $2, not env vars.""" + skill_dir = pkg_root() / "skills_extended" / "planner-extract-domain" + content = (skill_dir / "SKILL.md").read_text() + assert "$1" in content, "Must document $1 (analysis.json path)" + assert "$2" in content, "Must document $2 (task_file path)" + assert "PLANNER_ANALYSIS_FILE" not in content, ( + "Must not reference PLANNER_ANALYSIS_FILE env var" + ) + + +def test_extract_domain_skill_no_env_var_delivery(): + """SKILL.md must not reference PLANNER_TASK or PLANNER_TASK_FILE as env vars.""" + content = (pkg_root() / "skills_extended" / "planner-extract-domain" / "SKILL.md").read_text() + assert "PLANNER_TASK" not in content, "Must not reference PLANNER_TASK env var" diff --git a/tests/skills/test_planner_skill_contracts.py b/tests/skills/test_planner_skill_contracts.py new file mode 100644 index 0000000000..01e7f4f4c4 --- /dev/null +++ b/tests/skills/test_planner_skill_contracts.py @@ -0,0 +1,463 @@ +import pytest +import yaml + +from autoskillit.core.paths import pkg_root +from autoskillit.recipe.io import load_recipe + +SKILLS_ROOT = pkg_root() / "skills_extended" +RECIPES_ROOT = pkg_root() / "recipes" + +PLANNER_FINALIZATION_SKILLS = [ + "planner-reconcile-deps", + "planner-refine", +] + +ALL_PLANNER_SKILLS = [ + "planner-analyze", + "planner-consolidate-wps", + "planner-extract-domain", + "planner-generate-phases", + "planner-elaborate-phase", + "planner-elaborate-assignments", + "planner-elaborate-wps", + "planner-reconcile-deps", + "planner-refine", + "planner-refine-phases", + "planner-refine-assignments", + "planner-refine-wps", + "planner-validate-task-alignment", + "planner-assess-review-approach", +] + + +@pytest.mark.parametrize("skill_name", PLANNER_FINALIZATION_SKILLS) +def test_skill_directory_exists(skill_name: str) -> None: + assert (SKILLS_ROOT / skill_name).is_dir() + + +@pytest.mark.parametrize("skill_name", PLANNER_FINALIZATION_SKILLS) +def test_skill_md_exists(skill_name: str) -> None: + assert (SKILLS_ROOT / skill_name / "SKILL.md").is_file() + + +@pytest.mark.parametrize("skill_name", PLANNER_FINALIZATION_SKILLS) +def test_skill_has_planner_category(skill_name: str) -> None: + content = (SKILLS_ROOT / skill_name / "SKILL.md").read_text() + assert content.startswith("---"), f"{skill_name}: must start with YAML frontmatter" + parts = content.split("---", 2) + assert len(parts) >= 3 + data = yaml.safe_load(parts[1]) or {} + assert "planner" in (data.get("categories") or []), ( + f"{skill_name}: must declare 'categories: [planner]'" + ) + + +def test_reconcile_deps_output_token() -> None: + content = (SKILLS_ROOT / "planner-reconcile-deps" / "SKILL.md").read_text() + assert "dep_graph_path" in content, ( + "planner-reconcile-deps must document dep_graph_path output token" + ) + + +def test_refine_output_tokens() -> None: + content = (SKILLS_ROOT / "planner-refine" / "SKILL.md").read_text() + assert "refinement_complete" in content, ( + "planner-refine must document refinement_complete output token" + ) + assert "issues_fixed" in content, "planner-refine must document issues_fixed output token" + + +def test_reconcile_deps_reads_wp_index_only() -> None: + content = (SKILLS_ROOT / "planner-reconcile-deps" / "SKILL.md").read_text() + assert "wp_index.json" in content + assert "sub-agent" not in content.lower(), ( + "planner-reconcile-deps must be a single session — no sub-agents" + ) + assert "subagent" not in content.lower(), ( + "planner-reconcile-deps must be a single session — no sub-agents" + ) + + +def test_refine_handles_all_finding_types() -> None: + content = (SKILLS_ROOT / "planner-refine" / "SKILL.md").read_text() + for finding_type in ["failed", "sizing", "duplicate", "dep", "missing"]: + assert finding_type in content.lower(), ( + f"planner-refine must document handling of '{finding_type}' finding type" + ) + + +def test_refine_sizing_violations_escalated() -> None: + """Sizing violations must be escalated as CRITICAL, not auto-split/merged.""" + content = (SKILLS_ROOT / "planner-refine" / "SKILL.md").read_text().lower() + assert "critical" in content and "sizing" in content, ( + "planner-refine must escalate sizing violations as CRITICAL" + ) + + +def test_refine_no_wp_splitting() -> None: + """Alpha-suffix WP IDs break the backend; splitting instructions must not exist.""" + content = (SKILLS_ROOT / "planner-refine" / "SKILL.md").read_text() + for forbidden in ["WP3a", "WP3b", "split into two", "merge into the nearest"]: + assert forbidden not in content, ( + f"planner-refine must not contain WP split/merge instruction: '{forbidden}'" + ) + + +def test_refine_sizing_excluded_from_issues_fixed() -> None: + """Sizing violations are escalated, so must not be counted in issues_fixed.""" + content = (SKILLS_ROOT / "planner-refine" / "SKILL.md").read_text() + marker = "`N` = count of findings" + idx = content.find(marker) + assert idx != -1, "Step 5 N-count explanation not found" + explanation = content[idx : idx + 400] + assert "sizing" in explanation.lower(), ( + "Step 5 must mention sizing in the excluded-from-count explanation" + ) + formula_end = explanation.find(")") + formula = explanation[:formula_end] if formula_end != -1 else "" + assert "sizing" not in formula.lower(), "sizing must not appear in the issues_fixed formula" + + +@pytest.mark.parametrize("skill_name", PLANNER_FINALIZATION_SKILLS) +def test_skill_in_defaults_yaml_tier2(skill_name: str) -> None: + defaults = yaml.safe_load((pkg_root() / "config" / "defaults.yaml").read_text()) + tier2 = defaults["skills"]["tier2"] + assert skill_name in tier2, f"{skill_name} must appear in defaults.yaml skills.tier2" + + +def test_all_planner_recipe_skills_registered_in_contract_card() -> None: + """T1a: every run_skill step in planner.yaml must appear in the planner contract card.""" + recipe = load_recipe(RECIPES_ROOT / "planner.yaml") + recipe_skills = { + step.with_args.get("skill_command", "").split(":")[1].split()[0] + for step in recipe.steps.values() + if step.tool == "run_skill" and "/autoskillit:" in step.with_args.get("skill_command", "") + } + + card_path = RECIPES_ROOT / "contracts" / "planner.yaml" + assert card_path.is_file(), "planner contract card not found — run generate_recipe_card" + card = yaml.safe_load(card_path.read_text()) + registered = set(card.get("skills", {}).keys()) + + missing = recipe_skills - registered + assert not missing, ( + f"Planner skills used in recipe but missing from contract card: {sorted(missing)}" + ) + + +def test_planner_contract_card_records_positional_args_for_generate_phases() -> None: + """T1b: the generate_phases dataflow entry must record positional_args > 0.""" + card_path = RECIPES_ROOT / "contracts" / "planner.yaml" + assert card_path.is_file(), "planner contract card not found — run generate_recipe_card" + card = yaml.safe_load(card_path.read_text()) + + generate_phases_entry = next( + (e for e in card.get("dataflow", []) if e.get("step") == "generate_phases"), + None, + ) + assert generate_phases_entry is not None, ( + "generate_phases step not found in planner contract card dataflow" + ) + assert generate_phases_entry.get("positional_args", 0) > 0, ( + "generate_phases step should record positional_args > 0 in the contract card" + ) + + +@pytest.mark.parametrize("skill_name", ALL_PLANNER_SKILLS) +def test_all_planner_skills_in_master_manifest(skill_name: str) -> None: + """All 8 planner skills must be registered in skill_contracts.yaml.""" + from autoskillit.recipe.contracts import load_bundled_manifest + + manifest = load_bundled_manifest() + assert skill_name in manifest.get("skills", {}), ( + f"Planner skill '{skill_name}' is not registered in skill_contracts.yaml" + ) + + +def test_elaborate_phase_contract_declares_elab_result_path() -> None: + """planner-elaborate-phase must declare elab_result_path for capture_list (Issue 08).""" + from autoskillit.recipe.contracts import load_bundled_manifest + + contracts = load_bundled_manifest() + contract = contracts["skills"]["planner-elaborate-phase"] + output_names = [o["name"] for o in contract.get("outputs", [])] + assert "elab_result_path" in output_names, ( + "planner-elaborate-phase must declare elab_result_path output for " + "capture_list: elab_result_path in the parallel recipe step (Issue 08)" + ) + + +def test_elaborate_phase_contract_has_output_pattern() -> None: + """elab_result_path must appear in expected_output_patterns for run_skill validation.""" + from autoskillit.recipe.contracts import load_bundled_manifest + + contracts = load_bundled_manifest() + contract = contracts["skills"]["planner-elaborate-phase"] + patterns = contract.get("expected_output_patterns", []) + assert any("elab_result_path" in p for p in patterns), ( + "planner-elaborate-phase must have an expected_output_pattern matching elab_result_path" + ) + + +def test_elaborate_phase_contract_write_behavior_always() -> None: + """planner-elaborate-phase must declare write_behavior: always.""" + from autoskillit.recipe.contracts import load_bundled_manifest + + contracts = load_bundled_manifest() + contract = contracts["skills"]["planner-elaborate-phase"] + assert contract.get("write_behavior") == "always" + + +def test_elaborate_assignments_contract_declares_phase_result_dir() -> None: + """planner-elaborate-assignments must declare phase_assignments_result_dir output.""" + from autoskillit.recipe.contracts import load_bundled_manifest + + contracts = load_bundled_manifest() + contract = contracts["skills"]["planner-elaborate-assignments"] + output_names = [o["name"] for o in contract.get("outputs", [])] + assert "phase_assignments_result_dir" in output_names, ( + "planner-elaborate-assignments must declare phase_assignments_result_dir output" + ) + + +def test_elaborate_assignments_contract_has_output_pattern() -> None: + """phase_assignments_result_dir must appear in expected_output_patterns.""" + from autoskillit.recipe.contracts import load_bundled_manifest + + contracts = load_bundled_manifest() + contract = contracts["skills"]["planner-elaborate-assignments"] + patterns = contract.get("expected_output_patterns", []) + assert any("phase_assignments_result_dir" in p for p in patterns), ( + "planner-elaborate-assignments must have an expected_output_pattern" + " matching phase_assignments_result_dir" + ) + + +def test_elaborate_assignments_contract_write_behavior_always() -> None: + """planner-elaborate-assignments must declare write_behavior: always.""" + from autoskillit.recipe.contracts import load_bundled_manifest + + contracts = load_bundled_manifest() + contract = contracts["skills"]["planner-elaborate-assignments"] + assert contract.get("write_behavior") == "always" + + +def test_elaborate_wps_contract_declares_phase_wps_result_dir() -> None: + """planner-elaborate-wps must declare phase_wps_result_dir output.""" + from autoskillit.recipe.contracts import load_bundled_manifest + + contracts = load_bundled_manifest() + contract = contracts["skills"]["planner-elaborate-wps"] + output_names = [o["name"] for o in contract.get("outputs", [])] + assert "phase_wps_result_dir" in output_names, ( + "planner-elaborate-wps must declare phase_wps_result_dir output" + ) + + +def test_elaborate_wps_contract_has_output_pattern() -> None: + """phase_wps_result_dir must appear in expected_output_patterns.""" + from autoskillit.recipe.contracts import load_bundled_manifest + + contracts = load_bundled_manifest() + contract = contracts["skills"]["planner-elaborate-wps"] + patterns = contract.get("expected_output_patterns", []) + assert any("phase_wps_result_dir" in p for p in patterns), ( + "planner-elaborate-wps must have an expected_output_pattern matching phase_wps_result_dir" + ) + + +def test_elaborate_wps_contract_write_behavior_always() -> None: + """planner-elaborate-wps must declare write_behavior: always.""" + from autoskillit.recipe.contracts import load_bundled_manifest + + contracts = load_bundled_manifest() + contract = contracts["skills"]["planner-elaborate-wps"] + assert contract.get("write_behavior") == "always" + + +def test_elaborate_wps_contract_two_inputs() -> None: + """planner-elaborate-wps must declare two inputs: context_file and planner_dir.""" + from autoskillit.recipe.contracts import load_bundled_manifest + + contracts = load_bundled_manifest() + contract = contracts["skills"]["planner-elaborate-wps"] + inputs = contract.get("inputs", []) + assert len(inputs) == 2, ( + f"Expected 2 inputs, got {len(inputs)}: {[i.get('name') for i in inputs]}" + ) + input_names = {i.get("name") for i in inputs} + assert input_names == {"context_file", "planner_dir"} + + +# --- validate_task_alignment skill contract tests --- + + +def test_generate_phases_skill_references_task_file_arg(): + content = (SKILLS_ROOT / "planner-generate-phases" / "SKILL.md").read_text() + assert "$3" in content, "SKILL.md must document $3 positional arg for task file" + assert "task" in content.lower(), "SKILL.md must instruct task-constrained phase generation" + + +TASK_AWARE_SKILLS = [ + "planner-elaborate-phase", + "planner-refine-phases", + "planner-elaborate-assignments", + "planner-refine-assignments", + "planner-elaborate-wps", + "planner-refine-wps", +] + + +@pytest.mark.parametrize("skill_name", TASK_AWARE_SKILLS) +def test_downstream_skill_has_task_alignment_instruction(skill_name): + content = (SKILLS_ROOT / skill_name / "SKILL.md").read_text() + has_alignment = ( + "scope creep" in content.lower() + or "task alignment" in content.lower() + or "serves the stated task" in content.lower() + or "requested by the task" in content.lower() + ) + assert has_alignment, f"{skill_name} SKILL.md must contain task-alignment instruction" + + +def test_validate_task_alignment_skill_exists(): + skill_dir = SKILLS_ROOT / "planner-validate-task-alignment" + assert skill_dir.is_dir(), "planner-validate-task-alignment skill must exist" + skill_md = skill_dir / "SKILL.md" + assert skill_md.is_file(), "SKILL.md must exist" + content = skill_md.read_text() + parts = content.split("---", 2) + data = yaml.safe_load(parts[1]) + assert "planner" in (data.get("categories") or []) + assert "warning" in content.lower(), "Must emit warning-severity findings" + + +def test_generate_phases_has_read_guardrails(): + content = (SKILLS_ROOT / "planner-generate-phases" / "SKILL.md").read_text() + has_guardrail = ( + "do not explore parent" in content.lower() + or "do not read files outside" in content.lower() + or "only read files passed" in content.lower() + or "explore parent directories" in content.lower() + ) + assert has_guardrail, "SKILL.md must contain filesystem read guardrails" + + +# --- run-scoped output directory regression guards --- + + +@pytest.mark.parametrize("skill_name", ALL_PLANNER_SKILLS) +def test_planner_skill_no_hardcoded_temp_in_never_constraint(skill_name: str) -> None: + """No planner SKILL.md should hardcode {{AUTOSKILLIT_TEMP}}/planner in NEVER constraints.""" + skill_md = SKILLS_ROOT / skill_name / "SKILL.md" + if not skill_md.exists(): + pytest.skip(f"{skill_name} has no SKILL.md") + content = skill_md.read_text() + in_never_block = False + for lineno, line in enumerate(content.splitlines(), 1): + if line.strip().startswith("**NEVER"): + in_never_block = True + if line.strip().startswith("**ALWAYS") or ( + in_never_block and line.strip().startswith("##") + ): + in_never_block = False + if in_never_block and "{{AUTOSKILLIT_TEMP}}/planner" in line: + pytest.fail( + f"{skill_name}/SKILL.md line {lineno} has hardcoded " + f"'{{{{AUTOSKILLIT_TEMP}}}}/planner' in NEVER constraint. " + f"Use argument-relative path (e.g., '$2/', '$3/') instead." + ) + + +@pytest.mark.parametrize("skill_name", ALL_PLANNER_SKILLS) +def test_planner_skill_example_paths_are_run_scoped(skill_name: str) -> None: + """Example paths in arg docs should show run-scoped form, not bare shared path.""" + skill_md = SKILLS_ROOT / skill_name / "SKILL.md" + if not skill_md.exists(): + pytest.skip(f"{skill_name} has no SKILL.md") + content = skill_md.read_text() + for lineno, line in enumerate(content.splitlines(), 1): + if "e.g.," in line and "{{AUTOSKILLIT_TEMP}}/planner" in line: + if "planner/run-" not in line: + pytest.fail( + f"{skill_name}/SKILL.md line {lineno} shows bare " + f"'{{{{AUTOSKILLIT_TEMP}}}}/planner' as example. " + f"Use run-scoped form: '{{{{AUTOSKILLIT_TEMP}}}}/planner/run-YYYYMMDD-HHMMSS'" + ) + + +def test_generate_phases_skill_no_env_var_delivery(): + content = (SKILLS_ROOT / "planner-generate-phases" / "SKILL.md").read_text() + assert "PLANNER_TASK_FILE" not in content, "Must not reference PLANNER_TASK_FILE env var" + assert "PLANNER_TASK" not in content, "Must not reference PLANNER_TASK env var" + + +# --- intent-driven decomposition tests (T1–T7) --- + + +def test_generate_phases_no_fixed_range_in_body(): + """SKILL.md body must not contain a fixed '3-6' or '3–6' phase range.""" + content = (SKILLS_ROOT / "planner-generate-phases" / "SKILL.md").read_text() + assert "3-6" not in content, "SKILL.md must not contain '3-6'" + assert "3–6" not in content, "SKILL.md must not contain '3–6'" + + +def test_generate_phases_no_phase_count_never_constraint(): + """The NEVER block must not impose a min/max phase count.""" + content = (SKILLS_ROOT / "planner-generate-phases" / "SKILL.md").read_text() + assert "fewer than 3" not in content.lower() + assert "more than 6" not in content.lower() + + +def test_generate_phases_has_intent_driven_guidance(): + """SKILL.md must contain intent-driven decomposition guidance.""" + content = (SKILLS_ROOT / "planner-generate-phases" / "SKILL.md").read_text().lower() + assert "independently verifiable" in content or "verifiable checkpoint" in content, ( + "Must define phases as independently verifiable checkpoints" + ) + + +ANCHORING_TERMS = [ + "Database Layer", + "Auth Data Model", + "Auth API Endpoints", + "Auth UI Integration", + "user authentication", + "Schema design", + "Migration framework", + "Repository pattern", +] + + +@pytest.mark.parametrize("term", ANCHORING_TERMS) +def test_generate_phases_no_domain_anchoring(term: str): + """SKILL.md must not contain domain-specific example names that cause LLM anchoring.""" + content = (SKILLS_ROOT / "planner-generate-phases" / "SKILL.md").read_text() + assert term not in content, ( + f"Domain-specific example '{term}' causes LLM anchoring bias — " + f"use generic placeholders instead" + ) + + +ELABORATE_ANCHORING_TERMS = [ + "Session Management", + "user session persistence", + "SQLite-backed session table", +] + + +@pytest.mark.parametrize("term", ELABORATE_ANCHORING_TERMS) +def test_elaborate_assignments_no_domain_anchoring(term: str): + """elaborate-assignments SKILL.md must not use domain-specific examples.""" + content = (SKILLS_ROOT / "planner-elaborate-assignments" / "SKILL.md").read_text() + assert term not in content, f"Domain-specific example '{term}' causes LLM anchoring bias" + + +def test_refine_phases_has_batch_ceiling(): + """refine-phases must handle >6 phases via sequential batching (no unbounded spawn).""" + content = (SKILLS_ROOT / "planner-refine-phases" / "SKILL.md").read_text() + has_batch_guidance = "batch of 6" in content.lower() or "batches of 6" in content.lower() + assert has_batch_guidance, ( + "planner-refine-phases must include batch ceiling guidance for >6 phases " + "(consistent with refine-wps and refine-assignments)" + ) diff --git a/tests/skills/test_project_local_audit_skill_content.py b/tests/skills/test_project_local_audit_skill_content.py index bab9052074..de80f7f481 100644 --- a/tests/skills/test_project_local_audit_skill_content.py +++ b/tests/skills/test_project_local_audit_skill_content.py @@ -246,3 +246,206 @@ def test_validate_audit_uses_autoskillit_temp_placeholder() -> None: assert "{{AUTOSKILLIT_TEMP}}" in content, ( "validate-audit/SKILL.md does not contain '{{AUTOSKILLIT_TEMP}}' placeholder" ) + + +# --------------------------------------------------------------------------- +# Test 11a: Per-run directory pattern in project-local validate-audit +# --------------------------------------------------------------------------- + + +def test_validate_audit_uses_per_run_subdirectory_pattern() -> None: + """Project-local validate-audit must use per-run subdirectory pattern.""" + path = SKILLS_DIR / "validate-audit" / "SKILL.md" + content = path.read_text(encoding="utf-8") + assert "validate-audit-{YYYY-MM-DD_HHMMSS}/" in content, ( + "validate-audit/SKILL.md missing per-run subdirectory pattern " + "'validate-audit-{YYYY-MM-DD_HHMMSS}/'" + ) + + +# --------------------------------------------------------------------------- +# Test 11b: Timestamp-free filenames in project-local validate-audit +# --------------------------------------------------------------------------- + + +def test_validate_audit_filenames_have_no_timestamp_suffix() -> None: + """Individual filenames must not carry timestamp suffixes (the run dir has the timestamp).""" + path = SKILLS_DIR / "validate-audit" / "SKILL.md" + content = path.read_text(encoding="utf-8") + assert "validated_report_{source}.md" in content, ( + "validate-audit/SKILL.md should use 'validated_report_{source}.md' (no timestamp suffix)" + ) + + +# --------------------------------------------------------------------------- +# Test 12: C11 heading exists in project-local SKILL.md +# --------------------------------------------------------------------------- + + +def test_audit_tests_has_category_11_filter_integrity() -> None: + content = _read_skill("audit-tests") + assert "Category 11" in content, "audit-tests/SKILL.md missing 'Category 11' heading" + assert "Test Path Filter Integrity" in content, ( + "audit-tests/SKILL.md Category 11 missing 'Test Path Filter Integrity' name" + ) + + +# --------------------------------------------------------------------------- +# Test 13: C11 has all six required subsections +# --------------------------------------------------------------------------- + + +def test_audit_tests_c11_subsections_present() -> None: + content = _read_skill("audit-tests") + for subsection in [ + "Cascade alignment", + "Manifest coverage", + "Size marker correctness", + "Bucket A discipline", + "Test file naming", + "Filter infrastructure staleness", + ]: + assert subsection in content, ( + f"audit-tests/SKILL.md Category 11 missing subsection '{subsection}'" + ) + + +# --------------------------------------------------------------------------- +# Test 14: C11 mentions specific filter system artifacts +# --------------------------------------------------------------------------- + + +def test_audit_tests_c11_references_filter_artifacts() -> None: + content = _read_skill("audit-tests") + for artifact in [ + "LAYER_CASCADE_CONSERVATIVE", + "LAYER_CASCADE_AGGRESSIVE", + "test-filter-manifest.yaml", + "BUCKET_A_PATTERNS", + "_SIZE_DIRS", + "ALWAYS_RUN", + ]: + assert artifact in content, ( + f"audit-tests/SKILL.md Category 11 missing reference to '{artifact}'" + ) + + +# --------------------------------------------------------------------------- +# Test 15: C8 enhanced with cross-layer import rule +# --------------------------------------------------------------------------- + + +def test_audit_tests_c8_has_cross_layer_import_rule() -> None: + content = _read_skill("audit-tests") + c8_start = content.find("### Category 8:") + c8_end = content.find("### Category 9:", c8_start) + c8_section = content[c8_start:c8_end] + assert ( + "cross-layer" in c8_section.lower() or "outside their directory's cascade" in c8_section + ), "audit-tests/SKILL.md Category 8 missing cross-layer import rule" + + +# --------------------------------------------------------------------------- +# Test 16: C6 enhanced with filter staleness checks +# --------------------------------------------------------------------------- + + +def test_audit_tests_c6_has_filter_staleness_checks() -> None: + content = _read_skill("audit-tests") + assert "that don't match the current set of subpackages" in content, ( + "audit-tests/SKILL.md C6 missing cascade key staleness check" + ) + + +# --------------------------------------------------------------------------- +# Test 17: Exclusions mention filter infrastructure test files +# --------------------------------------------------------------------------- + + +def test_audit_tests_exclusions_mention_filter_infra_files() -> None: + content = _read_skill("audit-tests") + excl_start = content.find("## Exclusions") + excl_section = content[excl_start:] + assert "test_test_filter" in excl_section, ( + "audit-tests/SKILL.md Exclusions missing filter infrastructure test file exemption" + ) + + +# --------------------------------------------------------------------------- +# Test 18: Group 6 mentions filter-specific audit steps +# --------------------------------------------------------------------------- + + +def test_audit_tests_group6_has_filter_audit_steps() -> None: + content = _read_skill("audit-tests") + step1_start = content.find("### Step 1:") + step1_end = content.find("### Step 2:", step1_start) + step1_section = content[step1_start:step1_end] + assert "cascade maps" in step1_section.lower(), ( + "audit-tests/SKILL.md Group 6 missing filter-specific audit steps" + ) + assert "manifest completeness" in step1_section.lower(), ( + "audit-tests/SKILL.md Group 6 missing manifest completeness check" + ) + + +# --------------------------------------------------------------------------- +# Test 19: Bundled SKILL.md has C11 (sync guard) +# --------------------------------------------------------------------------- + + +def test_bundled_audit_tests_has_category_11() -> None: + path = REPO_ROOT / "src" / "autoskillit" / "skills_extended" / "audit-tests" / "SKILL.md" + assert path.exists(), f"Bundled audit-tests SKILL.md not found at {path}" + content = path.read_text(encoding="utf-8") + assert "Category 11" in content, ( + "Bundled audit-tests/SKILL.md missing 'Category 11' — out of sync with project-local" + ) + assert "Test Path Filter Integrity" in content, ( + "Bundled audit-tests/SKILL.md Category 11 missing 'Test Path Filter Integrity'" + ) + + +# --------------------------------------------------------------------------- +# Test 20: validate-audit project-local has cross-validation step +# --------------------------------------------------------------------------- + + +def test_validate_audit_project_local_has_cross_validation_step() -> None: + path = SKILLS_DIR / "validate-audit" / "SKILL.md" + content = path.read_text(encoding="utf-8").lower() + assert "cross-valid" in content, ( + "validate-audit/.claude/SKILL.md missing cross-validation step" + ) + assert "read-only" in content, ( + "validate-audit/.claude/SKILL.md cross-validation subagent must be read-only" + ) + + +# --------------------------------------------------------------------------- +# Test 21: validate-audit project-local has ticket grouping step +# --------------------------------------------------------------------------- + + +def test_validate_audit_project_local_has_ticket_grouping_step() -> None: + path = SKILLS_DIR / "validate-audit" / "SKILL.md" + content = path.read_text(encoding="utf-8").lower() + assert "grouping manifest" in content, ( + "validate-audit/.claude/SKILL.md missing ticket grouping manifest" + ) + assert "finding id" in content, ( + "validate-audit/.claude/SKILL.md grouping manifest must reference finding IDs" + ) + + +# --------------------------------------------------------------------------- +# Test 22: validate-audit project-local has validation summary +# --------------------------------------------------------------------------- + + +def test_validate_audit_project_local_has_validation_summary() -> None: + path = SKILLS_DIR / "validate-audit" / "SKILL.md" + content = path.read_text(encoding="utf-8") + assert "validation_summary_" in content, ( + "validate-audit/.claude/SKILL.md missing validation_summary_ output file" + ) diff --git a/tests/skills/test_promote_split_contracts.py b/tests/skills/test_promote_split_contracts.py index 8fe817e1b7..d7cf40d171 100644 --- a/tests/skills/test_promote_split_contracts.py +++ b/tests/skills/test_promote_split_contracts.py @@ -47,6 +47,25 @@ def test_promote_to_main_arch_lens_loop_is_guarded(self): "promote-to-main arch-lens loop must have anti-prose guard" ) + def test_promote_to_main_has_no_subagent_autonomy_grant(self): + """promote-to-main must NOT carry the Subagent Autonomy Grant (impossible at depth-1).""" + content = self._content() + assert "Subagent Autonomy Grant" not in content + assert "spawn additional subagents" not in content.lower() + assert "spawn their own sub-subagents" not in content.lower() + + def test_promote_to_main_outputs_pr_body_path_token(self) -> None: + """promote-to-main must emit pr_body_path (not report_path) structured output token.""" + content = self._content() + assert "pr_body_path = " in content, ( + "promote-to-main output section must declare 'pr_body_path' token " + "(pointing to pr_body_{timestamp}.md)" + ) + assert "report_path = " not in content, ( + "promote-to-main must not emit 'report_path'; that token is reserved " + "for review-promotion's analysis report" + ) + class TestReviewPromotionProjectLocal: def _content(self) -> str: @@ -94,9 +113,23 @@ def test_review_promotion_writes_to_skill_scoped_temp(self): content = self._content() assert ".autoskillit/temp/review-promotion/" in content - def test_review_promotion_has_subagent_autonomy_grant(self): - """review-promotion must carry the Subagent Autonomy Grant.""" + def test_review_promotion_has_no_subagent_autonomy_grant(self): + """review-promotion must NOT carry the Subagent Autonomy Grant (impossible at depth-1).""" content = self._content() - assert ( - "Subagent Autonomy Grant" in content or "spawn additional subagents" in content.lower() - ) + assert "Subagent Autonomy Grant" not in content + assert "spawn additional subagents" not in content.lower() + assert "spawn their own sub-subagents" not in content.lower() + + +class TestPromoteToMainCanonical: + def _content(self) -> str: + return ( + _REPO_ROOT / "src" / "autoskillit" / "skills_extended" / "promote-to-main" / "SKILL.md" + ).read_text() + + def test_canonical_has_no_subagent_autonomy_grant(self): + """Canonical promote-to-main must NOT carry the Subagent Autonomy Grant.""" + content = self._content() + assert "Subagent Autonomy Grant" not in content + assert "spawn additional subagents" not in content.lower() + assert "spawn their own sub-subagents" not in content.lower() diff --git a/tests/skills/test_resolve_failures_ci_aware.py b/tests/skills/test_resolve_failures_ci_aware.py index ce17e7aa91..6aff2279a7 100644 --- a/tests/skills/test_resolve_failures_ci_aware.py +++ b/tests/skills/test_resolve_failures_ci_aware.py @@ -80,7 +80,7 @@ def test_skill_maps_flaky_to_flake_suspected(skill_text: str) -> None: def test_skill_maps_deterministic_green_to_ci_only_failure(skill_text: str) -> None: - """Scenario B: deterministic + local tests green → ci_only_failure.""" + """Scenario B: deterministic + local tests green + NO FIX APPLIED → ci_only_failure.""" assert "ci_only_failure" in skill_text, ( "resolve-failures SKILL.md must include 'ci_only_failure' verdict value " "for the deterministic subtype + local-green scenario" @@ -133,3 +133,133 @@ def test_skill_verdict_covers_all_required_values(skill_text: str) -> None: required = {"real_fix", "already_green", "flake_suspected", "ci_only_failure"} missing = {v for v in required if v not in skill_text} assert not missing, f"resolve-failures SKILL.md is missing these verdict values: {missing}" + + +# --------------------------------------------------------------------------- +# Verdict decision table row-level mapping tests +# --------------------------------------------------------------------------- + + +def _find_table_row_verdict(skill_text: str, subtype: str) -> str | None: + """Find the verdict assigned to a given failure_subtype in the decision table. + + Scans the markdown table in the Verdict Decision Tree section for a row + containing the subtype, and extracts the verdict value from that row. + """ + in_table = False + for line in skill_text.splitlines(): + if "Local result" in line and "failure_subtype" in line and "Verdict" in line: + in_table = True + continue + if in_table and line.strip().startswith("|---"): + continue + if in_table and "|" in line: + cells = [c.strip() for c in line.split("|")] + # cells[0] is empty (before first |), cells[-1] is empty (after last |) + if len(cells) < 4: + continue + subtype_cell = cells[2] # failure_subtype column + verdict_cell = cells[3] # Verdict column + if subtype in subtype_cell: + # Extract verdict token (backtick-wrapped) + match = re.search(r"`(\w+)`", verdict_cell) + if match: + return match.group(1) + elif in_table and line.strip() == "": + break + return None + + +def test_unknown_subtype_maps_to_flake_suspected_not_ci_only(skill_text: str) -> None: + """The 'unknown' failure_subtype must map to flake_suspected, not ci_only_failure.""" + verdict = _find_table_row_verdict(skill_text, "unknown") + assert verdict is not None, ( + "resolve-failures SKILL.md verdict decision table must contain a row for 'unknown'" + ) + assert verdict == "flake_suspected", ( + f"'unknown' subtype must map to 'flake_suspected', got '{verdict}'. " + "Ambiguous subtypes should not be routed to abort." + ) + + +def test_env_subtype_maps_to_flake_suspected_not_ci_only(skill_text: str) -> None: + """The 'env' failure_subtype must map to flake_suspected, not ci_only_failure.""" + verdict = _find_table_row_verdict(skill_text, "env") + assert verdict is not None, ( + "resolve-failures SKILL.md verdict decision table must contain a row for 'env'" + ) + assert verdict == "flake_suspected", ( + f"'env' subtype must map to 'flake_suspected', got '{verdict}'. " + "Ambiguous subtypes should not be routed to abort." + ) + + +# --------------------------------------------------------------------------- +# Post-fix override guards (REQ-RF-001, REQ-RF-002) +# --------------------------------------------------------------------------- + + +def test_skill_fix_applied_overrides_to_real_fix(skill_text: str) -> None: + """When a fix is committed and tests pass, verdict MUST be real_fix regardless of subtype.""" + assert re.search( + r"fix.*(commit|applied).*verdict.*real_fix", skill_text, re.IGNORECASE + ) or re.search(r"real_fix.*regardless.*failure_subtype", skill_text, re.IGNORECASE), ( + "resolve-failures SKILL.md must contain an explicit override rule: " + "when a fix is committed and tests pass, verdict is always real_fix " + "regardless of failure_subtype" + ) + + +def test_step2d_table_scoped_to_no_fix_path(skill_text: str) -> None: + """Step 2d verdict table must only apply when no fix was applied.""" + # Capture the full Step 2d section (from header to next ### heading or EOF) + table_section_match = re.search( + r"Step 2d.*?(?=\n### |\Z)", + skill_text, + re.DOTALL, + ) + assert table_section_match is not None, "Step 2d section must exist" + table_section = table_section_match.group(0) + assert any( + phrase in table_section.lower() + for phrase in ( + "no fix applied", + "no fix was applied", + "fixes_applied == 0", + "without entering step 3", + ) + ), ( + "Step 2d verdict decision table must explicitly state it applies only " + "when no fix was applied — prevents LLM from re-evaluating after Step 3" + ) + + +def test_ci_only_failure_requires_no_fix_applied(skill_text: str) -> None: + """ci_only_failure must only be emittable when no fix was applied.""" + assert re.search( + r"ci_only_failure.*(no fix|never.*fix.*applied|fixes_applied.*0|without.*commit)", + skill_text, + re.IGNORECASE | re.DOTALL, + ) or re.search( + r"(no fix|never.*fix|fixes_applied.*0).*ci_only_failure", + skill_text, + re.IGNORECASE | re.DOTALL, + ), ( + "resolve-failures SKILL.md must explicitly state that ci_only_failure " + "is only emitted when no fix was applied" + ) + + +def test_step3_green_always_yields_real_fix(skill_text: str) -> None: + """Step 3 fix loop: green after fix MUST yield real_fix, never re-evaluates Step 2d.""" + step3_match = re.search( + r"### Step 3.*?(?=\n### Step [45]|\Z)", + skill_text, + re.DOTALL, + ) + assert step3_match is not None, "### Step 3 section must exist in SKILL.md" + step3_text = step3_match.group(0) + assert "real_fix" in step3_text, "Step 3 must directly assign verdict = real_fix on green exit" + assert "step 2d" not in step3_text.lower() or "do not" in step3_text.lower(), ( + "Step 3 must not redirect back to Step 2d for verdict evaluation" + ) diff --git a/tests/skills/test_resolve_review_diff_context_consumption.py b/tests/skills/test_resolve_review_diff_context_consumption.py new file mode 100644 index 0000000000..24f526e1eb --- /dev/null +++ b/tests/skills/test_resolve_review_diff_context_consumption.py @@ -0,0 +1,113 @@ +"""Guards: resolve-review loads and uses diff_context handoff file from review-pr.""" + +from pathlib import Path + +SKILL_PATH = ( + Path(__file__).parent.parent.parent + / "src" + / "autoskillit" + / "skills_extended" + / "resolve-review" + / "SKILL.md" +) + + +def _skill_text() -> str: + return SKILL_PATH.read_text() + + +def _step2_section() -> str: + text = _skill_text() + start = text.find("### Step 2") + end = text.find("### Step 3", start) + return text[start:end] + + +def _step35_section() -> str: + text = _skill_text() + start = text.find("### Step 3.5") + assert start != -1, "### Step 3.5 not found in SKILL.md" + end = text.find("### Step 4", start) + return text[start:end] + + +def _step4_section() -> str: + text = _skill_text() + start = text.find("### Step 4") + assert start != -1, "### Step 4 not found in SKILL.md" + end = text.find("### Step 5", start) + end = end if end != -1 else None + return text[start:end] + + +def test_step2_checks_for_diff_context_file(): + """Step 2 must check for the review-pr diff_context handoff file.""" + section = _step2_section() + assert "diff_context" in section + + +def test_step2_loads_diff_context_map(): + """Step 2 must build a diff_context_map lookup structure.""" + section = _step2_section() + assert "diff_context_map" in section + + +def test_step2_fallback_when_file_absent(): + """Step 2 must fall back to empty map when diff_context file is absent.""" + section = _step2_section() + # Must mention fallback, absence, or empty-map behavior + lower = section.lower() + assert "absent" in lower or "not found" in lower or "fallback" in lower or "{}" in section + + +def test_step35_uses_prebuilt_code_region(): + """Step 3.5 sub-agent prompt must use pre-loaded code_region when available.""" + section = _step35_section() + assert "diff_context_map" in section + + +def test_step35_skips_file_read_when_context_available(): + """Step 3.5 must skip 'read file' instruction when pre-built context is present.""" + section = _step35_section() + lower = section.lower() + # Must indicate the file-read instruction is conditional or skipped + assert ( + "instead of" in lower + or "skip" in lower + or "do not read" in lower + or "use the pre" in lower + ) + + +def test_step4_skips_understanding_read_when_context_present(): + """Step 4 must skip the ±20 line understanding read when diff_context_map has entry.""" + section = _step4_section() + assert "diff_context_map" in section + lower = section.lower() + assert "skip" in lower or "omit" in lower or "already available" in lower + + +def test_step4_still_reads_file_for_editing(): + """Step 4 must still read the file for applying actual edits even with pre-built context.""" + section = _step4_section() + lower = section.lower() + # Must mention that file read for editing is still needed + assert "still read" in lower or "read the file" in lower + + +def test_diff_context_path_matches_review_pr_output_path(): + """resolve-review's diff_context path must match what review-pr writes.""" + full = _skill_text() + assert "review-pr/diff_context" in full + + +def test_step2_map_type_is_dict_of_dicts(): + """Step 2 must declare dict[tuple[str, int], dict] not dict[tuple[str, int], str].""" + section = _step2_section() + assert "dict[tuple[str, int], dict]" in section + + +def test_step35_accesses_code_region_from_dict(): + """Step 3.5 must access code_region via .get('code_region') on the dict value.""" + section = _step35_section() + assert '.get("code_region"' in section or ".get('code_region'" in section diff --git a/tests/skills/test_resolve_review_diff_hunk_preference.py b/tests/skills/test_resolve_review_diff_hunk_preference.py new file mode 100644 index 0000000000..203d35562f --- /dev/null +++ b/tests/skills/test_resolve_review_diff_hunk_preference.py @@ -0,0 +1,92 @@ +"""Guards: resolve-review Step 3.5 prefers diff_hunk over source file reads. + +Enforces the 3-tier context resolution hierarchy: + Tier 1: diff_context_map pre-built code_region (richest, from review-pr handoff) + Tier 2: diff_hunk from the GitHub API review comment (always available) + Tier 3: source file read (last resort — only when hunk is insufficient) +""" + +from pathlib import Path + +SKILL_PATH = ( + Path(__file__).parent.parent.parent + / "src" + / "autoskillit" + / "skills_extended" + / "resolve-review" + / "SKILL.md" +) + + +def _step35_section() -> str: + assert SKILL_PATH.exists(), f"SKILL.md not found at {SKILL_PATH}" + text = SKILL_PATH.read_text() + start = text.find("### Step 3.5") + assert start != -1, "### Step 3.5 not found in SKILL.md" + end = text.find("### Step 4", start) + assert end != -1, "### Step 4 not found after Step 3.5 in SKILL.md" + return text[start:end] + + +def test_diff_hunk_documented_as_preferred_context_in_step35() -> None: + section = _step35_section().lower() + assert "use" in section and "diff_hunk" in section and "context" in section, ( + "Step 3.5 must document diff_hunk as a preferred context source" + ) + + +def test_file_read_conditional_on_hunk_insufficiency() -> None: + section = _step35_section().lower() + assert ( + "only" in section + and "read" in section + and ("source file" in section or "file" in section) + and ("if" in section) + ), "Source file reads must be conditional on hunk insufficiency" + + +def test_external_reference_documented_as_file_read_trigger() -> None: + section = _step35_section().lower() + assert "outside" in section and "hunk" in section, ( + "Step 3.5 must document external references as a file-read trigger" + ) + + +def test_truncated_hunk_documented_as_file_read_trigger() -> None: + section = _step35_section().lower() + assert "truncated" in section or "missing" in section, ( + "Step 3.5 must document truncated/missing hunk as a file-read trigger" + ) + + +def test_inline_shortcut_prefers_diff_hunk() -> None: + section = _step35_section().lower() + shortcut_start = section.find("inline classification shortcut") + assert shortcut_start != -1, "Inline classification shortcut not found" + shortcut_text = section[shortcut_start : shortcut_start + 600] + assert "diff_hunk" in shortcut_text, "Inline classification shortcut must mention diff_hunk" + + +def test_three_tier_hierarchy_documented() -> None: + section = _step35_section() + hierarchy_start = section.find("Context resolution hierarchy") + assert hierarchy_start != -1, "Context resolution hierarchy not found in Step 3.5" + hierarchy = section[hierarchy_start : hierarchy_start + 600] + pos_dcm = hierarchy.find("diff_context_map") + pos_hunk = hierarchy.find("diff_hunk") + pos_file_read = hierarchy.lower().find("source file read") + if pos_file_read == -1: + pos_file_read = hierarchy.lower().find("file read") + assert pos_dcm != -1, "diff_context_map not found in hierarchy" + assert pos_hunk != -1, "diff_hunk not found in hierarchy" + assert pos_file_read != -1, "file read fallback not found in hierarchy" + assert pos_dcm < pos_hunk < pos_file_read, ( + "Three-tier hierarchy must be in order: diff_context_map > diff_hunk > file read" + ) + + +def test_diff_hunk_not_just_passthrough_field() -> None: + section = _step35_section().lower() + assert ("use the" in section and "diff_hunk" in section) or ( + "use" in section and "diff_hunk" in section and "context" in section + ), "Step 3.5 must have a directive to USE diff_hunk, not just list it" diff --git a/tests/skills/test_resolve_review_duplicate_comments.py b/tests/skills/test_resolve_review_duplicate_comments.py new file mode 100644 index 0000000000..7361c62228 --- /dev/null +++ b/tests/skills/test_resolve_review_duplicate_comments.py @@ -0,0 +1,142 @@ +import re +from pathlib import Path + +RESOLVE_SKILL_MD = ( + Path(__file__).parent.parent.parent + / "src" + / "autoskillit" + / "skills_extended" + / "resolve-review" + / "SKILL.md" +).read_text() + +REVIEW_PR_SKILL_MD = ( + Path(__file__).parent.parent.parent + / "src" + / "autoskillit" + / "skills_extended" + / "review-pr" + / "SKILL.md" +).read_text() + +MARKER_RE = re.compile(r" markers.""" + text = _skill_text() + step15_idx = text.find("### Step 1.5") + assert step15_idx >= 0 + step15_section = text[step15_idx : step15_idx + 2500] + assert "REVIEW-FLAG" in step15_section, ( + "resolve-review/SKILL.md Step 1.5 must include REVIEW-FLAG markers " + "in posted deferred observations" + ) + assert "severity" in step15_section.lower() and "dimension" in step15_section.lower(), ( + "resolve-review/SKILL.md Step 1.5 deferred observation comments must " + "include severity and dimension in the REVIEW-FLAG marker" + ) + + +def test_resolve_review_deferred_observations_include_round_number(): + """Assert SKILL.md states that posted deferred observations note which local round + flagged them (round field in JSON, round number in comment body).""" + text = _skill_text() + step15_idx = text.find("### Step 1.5") + assert step15_idx >= 0 + step15_section = text[step15_idx : step15_idx + 2500] + # Should mention round number in the posted comment + assert "round" in step15_section.lower(), ( + "resolve-review/SKILL.md Step 1.5 must include the round number in " + "posted deferred observation comments" + ) + + +def test_resolve_review_local_mode_deduplication(): + """Assert SKILL.md deduplicates DISCUSS entries before appending to prevent + duplicate entries on retry within the same round.""" + text = _skill_text() + step36_idx = text.find("### Step 3.6") + assert step36_idx >= 0, "SKILL.md must have Step 3.6 for DISCUSS accumulation" + step36_section = text[step36_idx : step36_idx + 2000] + assert "deduplicate" in step36_section.lower() or "duplicate" in step36_section.lower(), ( + "resolve-review/SKILL.md Step 3.6 must deduplicate before appending " + "to prevent duplicates if resolve-review is retried within the same round" + ) + # Should check path, line, body + assert "path" in step36_section.lower() and "line" in step36_section.lower(), ( + "resolve-review/SKILL.md Step 3.6 deduplication must check (path, line, body) tuple" + ) + + +def test_resolve_review_local_mode_skip_github_api_fetches(): + """Assert SKILL.md Step 2 (mode=local) skips all GitHub API fetching calls.""" + text = _skill_text() + step2_idx = text.find("### Step 2") + assert step2_idx >= 0 + step2_section = text[step2_idx : step2_idx + 2000] + local_mode_idx = step2_section.lower().find("mode=local") + assert local_mode_idx >= 0, "Step 2 must have a mode=local section" + after_local = step2_section[local_mode_idx : local_mode_idx + 500] + assert any( + phrase in after_local.lower() + for phrase in ["skip", "do not fetch", "no github api", "read from"] + ), ( + "resolve-review/SKILL.md Step 2 mode=local section must skip GitHub API fetching " + "and read from local_findings JSON instead" + ) + + +def test_resolve_review_local_mode_transforms_local_findings(): + """Assert SKILL.md transforms local findings into internal structure with path, line, + body, severity, dimension fields.""" + text = _skill_text() + step2_idx = text.find("### Step 2") + assert step2_idx >= 0 + step2_section = text[step2_idx : step2_idx + 2000] + local_mode_idx = step2_section.lower().find("mode=local") + assert local_mode_idx >= 0 + after_local = step2_section[local_mode_idx : local_mode_idx + 500] + # Should mention transforming to internal structure + assert any( + phrase in after_local.lower() + for phrase in ["transform", "map to", "internal structure", "path", "line"] + ), ( + "resolve-review/SKILL.md Step 2 mode=local must transform local_findings JSON " + "into the internal finding structure" + ) + + +def test_resolve_review_local_mode_reject_patterns_no_timestamp(): + """Assert SKILL.md local mode uses stable reject_patterns filename without timestamp + (accumulating across rounds).""" + text = _skill_text() + step66_idx = text.find("### Step 6.6") + assert step66_idx >= 0 + step66_section = text[step66_idx : step66_idx + 2500] + local_mode_idx = step66_section.lower().find("mode=local") + assert local_mode_idx >= 0, "Step 6.6 must have a mode=local section" + after_local = step66_section[local_mode_idx : local_mode_idx + 500] + # Should mention stable filename without timestamp + assert "reject_patterns" in after_local, ( + "resolve-review/SKILL.md Step 6.6 local mode must use stable " + "reject_patterns_{pr_number}.json filename (no timestamp)" + ) diff --git a/tests/skills/test_resolve_review_severity_and_reject_resolution.py b/tests/skills/test_resolve_review_severity_and_reject_resolution.py new file mode 100644 index 0000000000..9dfe553258 --- /dev/null +++ b/tests/skills/test_resolve_review_severity_and_reject_resolution.py @@ -0,0 +1,157 @@ +from pathlib import Path + +SKILL_PATH = ( + Path(__file__).parent.parent.parent + / "src" + / "autoskillit" + / "skills_extended" + / "resolve-review" + / "SKILL.md" +) +assert SKILL_PATH.exists(), f"SKILL.md not found at {SKILL_PATH}" +SKILL_TEXT = SKILL_PATH.read_text() + + +def test_info_findings_auto_classified_as_discuss() -> None: + """Step 3 must state info findings are auto-classified as DISCUSS, bypassing Step 3.5.""" + step3_idx = SKILL_TEXT.find("### Step 3:") + if step3_idx == -1: + step3_idx = SKILL_TEXT.find("### Step 3") + assert step3_idx != -1, "SKILL.md must have a Step 3 section" + step35_idx = SKILL_TEXT.find("### Step 3.5") + assert step35_idx != -1, "SKILL.md must have a Step 3.5 section" + step3_section = SKILL_TEXT[step3_idx:step35_idx] + assert ( + "auto-classified" in step3_section.lower() + or "auto-classify" in step3_section.lower() + or ("discuss" in step3_section and "info" in step3_section.lower()) + ), "Step 3 must state that info findings are auto-classified as DISCUSS" + assert ( + "do not enter" in step3_section.lower() + or "bypass" in step3_section.lower() + or ( + "not enter step 3.5" in step3_section.lower() + or "do not enter step" in step3_section.lower() + ) + ), "Step 3 must state that info findings do not enter Step 3.5" + + +def test_intent_validation_restricted_to_critical_and_warning() -> None: + """Step 3.5 domain grouping must cover only critical and warning, not all findings.""" + step35_idx = SKILL_TEXT.find("### Step 3.5") + assert step35_idx != -1, "SKILL.md must have a Step 3.5 section" + step4_idx = SKILL_TEXT.find("### Step 4") + step35_section = SKILL_TEXT[step35_idx:step4_idx] + assert "Group all findings" not in step35_section, ( + "Step 3.5 must not say 'Group all findings' — info findings no longer enter Step 3.5" + ) + assert "critical and warning" in step35_section.lower(), ( + "Step 3.5 domain grouping must restrict to critical and warning findings" + ) + + +def test_info_findings_do_not_enter_step_35() -> None: + """Step 3.5 intro must say 'validate every critical and warning finding'.""" + step35_idx = SKILL_TEXT.find("### Step 3.5") + assert step35_idx != -1, "SKILL.md must have a Step 3.5 section" + step4_idx = SKILL_TEXT.find("### Step 4") + step35_section = SKILL_TEXT[step35_idx:step4_idx] + assert "validate every finding" not in step35_section, ( + "Step 3.5 must not say 'validate every finding' — info findings are now excluded" + ) + assert "validate every critical and warning finding" in step35_section, ( + "Step 3.5 must say 'validate every critical and warning finding'" + ) + + +def test_info_reply_template_uses_acknowledged() -> None: + """Step 6.5 must include a templated reply for info (auto-classified) findings.""" + step65_idx = SKILL_TEXT.find("### Step 6.5") + assert step65_idx != -1, "SKILL.md must have a Step 6.5 section" + step66_idx = SKILL_TEXT.find("### Step 6.6", step65_idx) + step65_section = ( + SKILL_TEXT[step65_idx:step66_idx] + if step66_idx != -1 + else SKILL_TEXT[step65_idx : step65_idx + 1200] + ) + assert "acknowledged" in step65_section.lower(), ( + "Step 6.5 must include 'Acknowledged' template for info findings" + ) + assert "minor suggestion" in step65_section.lower(), ( + "Step 6.5 must include 'minor suggestion noted' in the info reply template" + ) + + +def test_reject_threads_added_to_addressed_thread_ids() -> None: + """REJECT findings must have their thread_node_id appended to addressed_thread_ids.""" + gate_idx = SKILL_TEXT.lower().find("classification gate") + assert gate_idx != -1, "SKILL.md must define a classification gate section" + # Find the end of the gate section (next heading or skip section) + skip_idx = SKILL_TEXT.lower().find("skip a finding", gate_idx) + assert skip_idx != -1, "SKILL.md must have a 'skip a finding' section after the gate" + gate_section = SKILL_TEXT[gate_idx:skip_idx] + # REJECT must now add to addressed_thread_ids + assert "addressed_thread_ids" in gate_section, ( + "The REJECT path in the classification gate must append to addressed_thread_ids" + ) + # The old blanket "Do NOT add these findings'" must not appear (it covered both REJECT+DISCUSS) + assert ( + "Do NOT add these findings' `thread_node_id` to `addressed_thread_ids`" not in SKILL_TEXT + ), "The blanket 'Do NOT add' instruction covering both REJECT and DISCUSS must be removed" + + +def test_discuss_threads_still_excluded_from_addressed_thread_ids() -> None: + """DISCUSS findings must still NOT add to addressed_thread_ids (regression guard).""" + gate_idx = SKILL_TEXT.lower().find("classification gate") + assert gate_idx != -1, "SKILL.md must define a classification gate section" + skip_idx = SKILL_TEXT.lower().find("skip a finding", gate_idx) + assert skip_idx != -1 + gate_section = SKILL_TEXT[gate_idx:skip_idx] + # DISCUSS must still be excluded + assert "Do NOT add DISCUSS findings" in gate_section or ( + "discuss" in gate_section.lower() and "do not add" in gate_section.lower() + ), "DISCUSS findings must still be excluded from addressed_thread_ids" + + +def test_step6_5_scope_includes_all_analyzed_comments() -> None: + """Step 6.5 must not restrict scope to 'critical+warning filter' — all analyzed findings.""" + assert "critical+warning\nfilter in Step 3" not in SKILL_TEXT + assert "critical+warning filter in Step 3" not in SKILL_TEXT, ( + "Step 6.5 scope must not reference the removed critical+warning filter" + ) + step65_idx = SKILL_TEXT.find("### Step 6.5") + assert step65_idx != -1, "SKILL.md must have a Step 6.5 section" + step66_idx = SKILL_TEXT.find("### Step 6.6", step65_idx) + step65_section = ( + SKILL_TEXT[step65_idx:step66_idx] + if step66_idx != -1 + else SKILL_TEXT[step65_idx : step65_idx + 800] + ) + assert "intent validation" in step65_section.lower() or "step 3.5" in step65_section, ( + "Step 6.5 scope should reference intent validation / Step 3.5 instead of the old severity filter" # noqa: E501 + ) + + +def test_report_does_not_mark_info_as_skipped() -> None: + """Step 7 report template must not annotate info count with '(skipped — below threshold)'.""" + assert "(skipped — below threshold)" not in SKILL_TEXT, ( + "Step 7 report must not mark info findings as skipped — info is now fully assessed" + ) + # The info count line must still be present (just without the skip annotation) + step7_idx = SKILL_TEXT.find("### Step 7") + assert step7_idx != -1, "SKILL.md must have a Step 7 (Report)" + report_section = SKILL_TEXT[step7_idx:] + assert "info: {n}" in report_section, "Step 7 report must still include the info count line" + + +def test_reject_no_code_changes_still_enforced() -> None: + """REJECT path must still say 'no code changes are applied' (regression guard).""" + gate_idx = SKILL_TEXT.lower().find("classification gate") + assert gate_idx != -1 + skip_idx = SKILL_TEXT.lower().find("skip a finding", gate_idx) + assert skip_idx != -1 + gate_section = SKILL_TEXT[gate_idx:skip_idx].lower() + assert "no code changes are applied" in gate_section, ( + "REJECT path must still state 'no code changes are applied' — resolving the thread" + " does not mean applying code changes" + ) diff --git a/tests/skills/test_resolve_review_thread_resolution.py b/tests/skills/test_resolve_review_thread_resolution.py index 08221b2502..b5a8df1621 100644 --- a/tests/skills/test_resolve_review_thread_resolution.py +++ b/tests/skills/test_resolve_review_thread_resolution.py @@ -82,3 +82,23 @@ def test_skill_reports_thread_resolution_count(resolve_review_skill_md: str) -> "Threads resolved" in resolve_review_skill_md or "threads resolved" in resolve_review_skill_md.lower() ) + + +def test_resolve_review_uses_graphql_aliases(resolve_review_skill_md: str) -> None: + """Thread resolution must use aliased GraphQL mutations, not individual calls.""" + assert "resolveReviewThread" in resolve_review_skill_md, ( + "SKILL.md must include resolveReviewThread mutation for thread resolution" + ) + assert ( + "alias" in resolve_review_skill_md.lower() + or "resolve1:" in resolve_review_skill_md + or "resolve${" in resolve_review_skill_md + ), ( + "SKILL.md must use GraphQL aliases (alias pattern or resolve${i}:) " + "to batch thread resolutions, not individual mutations per call" + ) + + +def test_resolve_review_reply_loop_has_delay(resolve_review_skill_md: str) -> None: + """Reply POST loop must include sleep 1 between calls.""" + assert "sleep 1" in resolve_review_skill_md or "sleep(1)" in resolve_review_skill_md diff --git a/tests/skills/test_resolve_review_token_optimizations.py b/tests/skills/test_resolve_review_token_optimizations.py new file mode 100644 index 0000000000..0eac037f41 --- /dev/null +++ b/tests/skills/test_resolve_review_token_optimizations.py @@ -0,0 +1,212 @@ +"""Structural guards for resolve-review SKILL.md token-optimization edits. + +Enforces that redundant addressed_thread_ids rules are collapsed to a single decision +table, best-effort language is unified across Steps 6 and 6.5, the Output section is +pruned of redundant graceful-degradation detail, the Step 3.5 JSON example is removed, +per-finding file/git-log reads are deduplicated, and the inline classification shortcut +for simple PRs is documented. +""" + +from pathlib import Path + +SKILL_PATH = ( + Path(__file__).parent.parent.parent + / "src" + / "autoskillit" + / "skills_extended" + / "resolve-review" + / "SKILL.md" +) +assert SKILL_PATH.exists(), f"SKILL.md not found at {SKILL_PATH}" +SKILL_TEXT = SKILL_PATH.read_text() + + +# ── 1. addressed_thread_ids decision table ──────────────────────────────────── + + +def test_addressed_thread_ids_consolidated_decision_table() -> None: + """All thread_node_id tracking rules must live in a single decision table.""" + # The plan collapses 5 scattered rules into one table. + # Require the table header row that summarises all four cases. + assert "Append to `addressed_thread_ids`" in SKILL_TEXT, ( + "SKILL.md must contain a consolidated decision table for thread_node_id tracking" + ) + + +def test_addressed_thread_ids_not_repeated_in_accept_flow() -> None: + """After ACCEPT fix commit, inline 'append addressed_thread_ids' must be removed.""" + step4_idx = SKILL_TEXT.find("### Step 4") + assert step4_idx != -1, "### Step 4 must exist in SKILL.md" + step5_idx = SKILL_TEXT.find("### Step 5", step4_idx) + step4_text = SKILL_TEXT[step4_idx:step5_idx] + # Verify absence of the old inline rule inside Step 4's commit block + accept_block_end = step4_text.find("**Classification gate") + accept_block = step4_text[:accept_block_end] if accept_block_end != -1 else step4_text + assert "Append the finding's `thread_node_id` to `addressed_thread_ids`" not in accept_block, ( + "The inline 'append thread_node_id' after ACCEPT commit must be removed; " + "use the consolidated decision table instead" + ) + + +def test_addressed_thread_ids_not_in_skip_flow() -> None: + """The 'skip a finding' flow must not repeat addressed_thread_ids guidance.""" + skip_idx = SKILL_TEXT.lower().find("skip a finding flow") + assert skip_idx != -1, "SKILL.md must have a 'skip a finding flow' section" + next_section_idx = SKILL_TEXT.find("\n###", skip_idx + 1) + skip_section = ( + SKILL_TEXT[skip_idx:next_section_idx] if next_section_idx != -1 else SKILL_TEXT[skip_idx:] + ) + assert "`addressed_thread_ids`" not in skip_section, ( + "The 'skip a finding flow' section must not repeat thread_node_id guidance; " + "all cases are covered by the consolidated decision table" + ) + + +def test_addressed_thread_ids_not_in_file_level_guard() -> None: + """File-level comment guard must not repeat addressed_thread_ids guidance.""" + guard_idx = SKILL_TEXT.find("File-level comment guard") + assert guard_idx != -1, "SKILL.md must have a file-level comment guard" + guard_section = SKILL_TEXT[guard_idx : guard_idx + 300] + assert "`addressed_thread_ids`" not in guard_section, ( + "File-level comment guard must not repeat thread_node_id guidance; " + "the consolidated decision table covers this case" + ) + + +# ── 2. Best-effort language unified ────────────────────────────────────────── + + +def test_best_effort_unified_across_steps_6_and_65() -> None: + """Step 6 best-effort statement must cover both Steps 6 and 6.5.""" + step6_idx = SKILL_TEXT.find("### Step 6:") + if step6_idx == -1: + step6_idx = SKILL_TEXT.find("### Step 6\n") + step65_idx = SKILL_TEXT.find("### Step 6.5", step6_idx) + step6_section = SKILL_TEXT[step6_idx:step65_idx] + assert "6.5" in step6_section, ( + "Step 6's best-effort statement must reference Step 6.5 so it covers both steps" + ) + + +def test_step65_no_standalone_best_effort_statement() -> None: + """Step 6.5 must not repeat a standalone 'best-effort' / exit-code statement.""" + step65_idx = SKILL_TEXT.find("### Step 6.5") + assert step65_idx != -1, "SKILL.md must have a Step 6.5 section" + step66_idx = SKILL_TEXT.find("### Step 6.6", step65_idx) + step65_section = ( + SKILL_TEXT[step65_idx:step66_idx] + if step66_idx != -1 + else SKILL_TEXT[step65_idx : step65_idx + 1200] + ) + assert ( + "best-effort: failure to post any reply must not affect the exit code" + not in step65_section + ), ( + "Step 6.5 must not contain a standalone best-effort/exit-code statement; " + "that is now unified in Step 6" + ) + + +# ── 3. Output section pruned ───────────────────────────────────────────────── + + +def test_output_section_no_graceful_degradation_detail() -> None: + """The ## Output section must not repeat 'graceful degradation' detail.""" + output_idx = SKILL_TEXT.rfind("## Output") + assert output_idx != -1, "SKILL.md must have an ## Output section" + output_section = SKILL_TEXT[output_idx:] + assert "graceful degradation" not in output_section.lower(), ( + "## Output section must not repeat graceful-degradation language; " + "Step 1 and Step 7 already state this" + ) + + +# ── 4. Step 3.5 JSON example removed ───────────────────────────────────────── + + +def test_step35_json_example_removed() -> None: + """The illustrative JSON example block (comment_id: 123) must be removed from Step 3.5.""" + assert '"comment_id": 123' not in SKILL_TEXT, ( + "The Step 3.5 JSON example block must be removed; " + "the schema is fully described by the preceding field list" + ) + + +# ── 5. Per-unique-file read in sub-agent instructions ──────────────────────── + + +def test_per_unique_file_read_instruction() -> None: + """Sub-agent instructions must say to read each unique file once, not once per finding.""" + step35_idx = SKILL_TEXT.find("### Step 3.5") + assert step35_idx != -1, "SKILL.md must have a Step 3.5 section" + step4_idx = SKILL_TEXT.find("### Step 4", step35_idx) + step35_section = SKILL_TEXT[step35_idx:step4_idx] + assert "unique file" in step35_section.lower(), ( + "Step 3.5 sub-agent instructions must say 'unique file' (read once per file, " + "not once per finding)" + ) + + +def test_per_finding_read_instruction_removed() -> None: + """The old 'read the actual code at each flagged line' per-finding phrasing must be gone.""" + step35_idx = SKILL_TEXT.find("### Step 3.5") + assert step35_idx != -1 + step4_idx = SKILL_TEXT.find("### Step 4", step35_idx) + step35_section = SKILL_TEXT[step35_idx:step4_idx] + assert "read the actual code at the flagged line" not in step35_section.lower(), ( + "The old per-finding file-read instruction must be replaced with the per-unique-file " + "read instruction" + ) + + +# ── 6. Per-unique-path git log ──────────────────────────────────────────────── + + +def test_per_unique_path_git_log() -> None: + """Git log instruction must say 'once per unique path, not once per finding'.""" + step35_idx = SKILL_TEXT.find("### Step 3.5") + assert step35_idx != -1 + step4_idx = SKILL_TEXT.find("### Step 4", step35_idx) + step35_section = SKILL_TEXT[step35_idx:step4_idx] + assert ( + "once per unique path" in step35_section.lower() + or "per unique path" in step35_section.lower() + ), "Step 3.5 git log instruction must say 'once per unique path' (not once per finding)" + + +# ── 7. Inline classification shortcut for simple PRs ───────────────────────── + + +def test_inline_classification_shortcut_documented() -> None: + """Step 3.5 must document an inline classification path for ≤3 findings.""" + step35_idx = SKILL_TEXT.find("### Step 3.5") + assert step35_idx != -1 + step4_idx = SKILL_TEXT.find("### Step 4", step35_idx) + step35_section = SKILL_TEXT[step35_idx:step4_idx] + assert ( + "3 or fewer" in step35_section.lower() + or "≤3" in step35_section + or "inline classification" in step35_section.lower() + ), "Step 3.5 must document an inline classification shortcut for PRs with 3 or fewer findings" + + +def test_inline_shortcut_requires_single_domain_group() -> None: + """The inline shortcut condition must require both ≤3 findings AND a single domain group.""" + step35_idx = SKILL_TEXT.find("### Step 3.5") + assert step35_idx != -1 + step4_idx = SKILL_TEXT.find("### Step 4", step35_idx) + step35_section = SKILL_TEXT[step35_idx:step4_idx] + inline_idx = step35_section.lower().find("inline classification") + if inline_idx == -1: + inline_idx = step35_section.lower().find("inline") + assert inline_idx != -1, "Step 3.5 must mention 'inline' classification" + inline_context = step35_section[max(0, inline_idx - 200) : inline_idx + 400].lower() + assert ( + "single domain" in inline_context + or "one domain" in inline_context + or ("single" in inline_context and "group" in inline_context) + ), "The inline classification shortcut must require a single domain group condition" + assert "3 or fewer" in inline_context or "≤3" in inline_context, ( + "The inline classification shortcut must co-locate the ≤3 findings count condition " + "with the domain condition" + ) diff --git a/tests/skills/test_retired_skill_names.py b/tests/skills/test_retired_skill_names.py new file mode 100644 index 0000000000..2e6988ad23 --- /dev/null +++ b/tests/skills/test_retired_skill_names.py @@ -0,0 +1,43 @@ +"""Tests enforcing the RETIRED_SKILL_NAMES convention. + +Analogous to test_no_retired_name_has_a_live_file in tests/hooks/. +Each entry in RETIRED_SKILL_NAMES must NOT have a live skill directory +under skills/ or skills_extended/. +""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + + +def test_no_retired_skill_name_has_a_live_directory() -> None: + """No retired skill name may have a live directory under skills/ or skills_extended/.""" + from autoskillit.core import RETIRED_SKILL_NAMES + from autoskillit.workspace.skills import bundled_skills_dir, bundled_skills_extended_dir + + for name in RETIRED_SKILL_NAMES: + for skill_dir in (bundled_skills_dir(), bundled_skills_extended_dir()): + candidate = skill_dir / name + assert not candidate.is_dir(), ( + f"Retired skill name '{name}' has a live directory at {candidate}. " + "Remove the directory or remove it from RETIRED_SKILL_NAMES." + ) + + +def test_scan_directory_raises_on_retired_skill(tmp_path: Path) -> None: + """DefaultSkillResolver raises RuntimeError if a retired skill directory is discovered.""" + from autoskillit.core import RETIRED_SKILL_NAMES + from autoskillit.workspace.skills import SkillSource, _scan_directory + + if not RETIRED_SKILL_NAMES: + pytest.skip("no retired names defined") + + retired_name = next(iter(sorted(RETIRED_SKILL_NAMES))) + skill_dir = tmp_path / retired_name + skill_dir.mkdir() + (skill_dir / "SKILL.md").write_text("---\ncategories: []\n---\n# Fake\n") + + with pytest.raises(RuntimeError, match=retired_name): + list(_scan_directory(SkillSource.BUNDLED_EXTENDED, tmp_path)) diff --git a/tests/skills/test_review_design_contracts.py b/tests/skills/test_review_design_contracts.py index 406ef5173e..3b57e81984 100644 --- a/tests/skills/test_review_design_contracts.py +++ b/tests/skills/test_review_design_contracts.py @@ -206,6 +206,44 @@ def test_dashboard_yaml_summary_block(skill_text): ) +def test_evaluation_dashboard_includes_rationale_section(skill_text: str) -> None: + """evaluation_dashboard must include a Dimension Rationale subsection.""" + step7_text = skill_text_between("### Step 7", "### Step 8", skill_text) + assert "Dimension Rationale" in step7_text, ( + "Step 7 evaluation_dashboard must include a 'Dimension Rationale' subsection " + "that surfaces dimension_weight_rationale entries from ExperimentTypeSpec" + ) + + +def test_rationale_table_structure_in_dashboard(skill_text: str) -> None: + """Dimension Rationale subsection must specify Dimension, Weight, and Rationale columns.""" + step7_text = skill_text_between("### Step 7", "### Step 8", skill_text) + for column in ("Dimension", "Weight", "Rationale"): + assert column in step7_text, f"Dimension Rationale table must include a '{column}' column" + + +def test_rationale_subsection_references_experiment_type_spec(skill_text: str) -> None: + """Dimension Rationale subsection must reference dimension_weight_rationale field.""" + step7_text = skill_text_between("### Step 7", "### Step 8", skill_text) + assert "dimension_weight_rationale" in step7_text, ( + "Step 7 must reference the dimension_weight_rationale field from ExperimentTypeSpec " + "so the LLM knows where to source the rationale data" + ) + + +def test_rationale_handles_missing_rationale_gracefully(skill_text: str) -> None: + """Dimension Rationale subsection must document behavior for types with no rationale.""" + step7_text = skill_text_between("### Step 7", "### Step 8", skill_text) + rationale_idx = step7_text.find("Dimension Rationale") + assert rationale_idx != -1, "Dimension Rationale subsection not found" + rationale_section = step7_text[rationale_idx:] + text_lower = rationale_section.lower() + assert "omit" in text_lower or "skip" in text_lower or "no rationale" in text_lower, ( + "Dimension Rationale subsection must document behavior when " + "dimension_weight_rationale is empty (legacy types)" + ) + + # ── Output token format ────────────────────────────────────────────────────── @@ -552,3 +590,14 @@ def test_agent_implementability_l4_step5_placement(skill_text: str) -> None: assert "agent_implementability" in step5_text, ( "agent_implementability must be placed in Step 5 (Level 4 dimensions)" ) + + +# ── Silent type handling (T5) ─────────────────────────────────────────────── + + +def test_silent_type_handling_section_in_skill(skill_text: str) -> None: + """SKILL.md must contain a Silent Type Handling section with required fields.""" + assert "Silent Type Handling" in skill_text + assert "is_silent_type" in skill_text + assert "advisory_context" in skill_text + assert "subject_kind" in skill_text diff --git a/tests/skills/test_review_design_guards.py b/tests/skills/test_review_design_guards.py index 5a1bffde11..37a505acbd 100644 --- a/tests/skills/test_review_design_guards.py +++ b/tests/skills/test_review_design_guards.py @@ -22,7 +22,7 @@ def test_data_acquisition_dimension_exists() -> None: def test_data_acquisition_not_l_weight() -> None: """data_acquisition must be M-weight minimum to influence verdict in at least one type.""" types = load_all_experiment_types() - for name, spec in types.items(): + for spec in types: weight = spec.dimension_weights.get("data_acquisition") if weight in ("M", "H"): return @@ -37,6 +37,7 @@ def test_agent_implementability_dimension_exists() -> None: def test_agent_implementability_weight_row() -> None: """agent_implementability must have H/H/M/M/L weights for the 5 bundled types.""" types = load_all_experiment_types() + by_name = {s.name: s for s in types} expected = { "benchmark": "H", "configuration_study": "H", @@ -45,7 +46,7 @@ def test_agent_implementability_weight_row() -> None: "exploratory": "L", } for type_name, exp_weight in expected.items(): - spec = types.get(type_name) + spec = by_name.get(type_name) assert spec is not None, f"Bundled type {type_name!r} not found" actual = spec.dimension_weights.get("agent_implementability") assert actual == exp_weight, ( diff --git a/tests/skills/test_review_flag_marker.py b/tests/skills/test_review_flag_marker.py new file mode 100644 index 0000000000..f7b2401c8d --- /dev/null +++ b/tests/skills/test_review_flag_marker.py @@ -0,0 +1,39 @@ +"""Tests for the REVIEW-FLAG HTML comment marker used in resolve-review replies.""" + +from __future__ import annotations + +import re + +REVIEW_FLAG_RE = re.compile(r"") + + +def test_review_flag_re_matches_discuss_comment(): + body = ( + "Valid observation — flagged for design decision. Evidence.\n" + "" + "\n" + ) + m = REVIEW_FLAG_RE.search(body) + assert m is not None + assert m.group(1) == "warning" + assert m.group(2) == "arch" + + +def test_review_flag_re_matches_info_comment(): + body = ( + "Acknowledged — minor suggestion noted.\n" + "" + "\n" + ) + m = REVIEW_FLAG_RE.search(body) + assert m is not None + assert m.group(1) == "info" + assert m.group(2) == "slop" + + +def test_review_flag_re_no_match_on_legacy_comment(): + legacy = ( + "Valid observation — flagged for design decision. Some evidence.\n" + "" + ) + assert REVIEW_FLAG_RE.search(legacy) is None diff --git a/tests/skills/test_review_pr_adaptive_dispatch_guards.py b/tests/skills/test_review_pr_adaptive_dispatch_guards.py new file mode 100644 index 0000000000..cf2c477dcc --- /dev/null +++ b/tests/skills/test_review_pr_adaptive_dispatch_guards.py @@ -0,0 +1,45 @@ +"""Behavioral guard tests for review-pr adaptive subagent dispatch.""" + +from __future__ import annotations + +from pathlib import Path + +SKILL_PATH = ( + Path(__file__).parent.parent.parent + / "src" + / "autoskillit" + / "skills_extended" + / "review-pr" + / "SKILL.md" +) + + +def _skill_text() -> str: + return SKILL_PATH.read_text() + + +def test_skill_accepts_diff_metrics_path_argument(): + text = _skill_text() + assert "diff_metrics_path" in text + + +def test_skill_defines_diff_size_gate_step(): + text = _skill_text() + assert "dispatch_agents" in text + + +def test_small_diff_skips_defense_bugs_slop(): + text = _skill_text().lower() + assert "small" in text + + +def test_small_diff_always_includes_tests_cohesion(): + text = _skill_text().lower() + assert "tests" in text + assert "cohesion" in text + + +def test_full_fanout_for_medium_and_large(): + text = _skill_text() + for agent in ["arch", "tests", "defense", "bugs", "cohesion", "slop"]: + assert agent in text diff --git a/tests/skills/test_review_pr_diff_context_handoff.py b/tests/skills/test_review_pr_diff_context_handoff.py new file mode 100644 index 0000000000..f2ea4754c8 --- /dev/null +++ b/tests/skills/test_review_pr_diff_context_handoff.py @@ -0,0 +1,84 @@ +"""Guards: review-pr writes diff_context handoff file in Step 8 before verdict emission.""" + +from pathlib import Path + +SKILL_PATH = ( + Path(__file__).parent.parent.parent + / "src" + / "autoskillit" + / "skills_extended" + / "review-pr" + / "SKILL.md" +) + + +def _skill_text() -> str: + return SKILL_PATH.read_text() + + +def _step8_section() -> str: + text = _skill_text() + start = text.find("### Step 8") + # Step 8 is the last step; take to end of file + return text[start:] + + +def test_step8_writes_diff_context_file(): + """Step 8 must declare writing diff_context_{pr_number}.json.""" + section = _step8_section() + assert "diff_context_{pr_number}.json" in section + + +def test_diff_context_path_uses_review_pr_temp_dir(): + """Handoff file must live in AUTOSKILLIT_TEMP/review-pr/, not resolve-review/.""" + section = _step8_section() + assert "review-pr/diff_context" in section + + +def test_diff_context_schema_has_context_entries(): + """Handoff JSON schema must include context_entries field.""" + section = _step8_section() + assert "context_entries" in section + + +def test_diff_context_entries_have_code_region(): + """Each context entry must include a code_region field.""" + section = _step8_section() + assert "code_region" in section + + +def test_code_region_extracted_from_annotated_diff(): + """code_region must be sourced from ANNOTATED_DIFF — no additional file reads.""" + section = _step8_section() + assert "ANNOTATED_DIFF" in section + # Must state zero additional reads + lower = section.lower() + assert "zero" in lower or "no additional" in lower or "no file read" in lower + + +def test_handoff_write_precedes_verdict_emission(): + """diff_context write must appear before 'verdict = ' in Step 8 text.""" + section = _step8_section() + write_pos = section.find("diff_context") + verdict_pos = section.find("verdict = ") + assert write_pos != -1, "diff_context not found in Step 8" + assert verdict_pos != -1, "verdict = not found in Step 8" + assert write_pos < verdict_pos, "diff_context write must precede verdict = emission" + + +def test_diff_context_covers_critical_and_warning(): + """All critical and warning findings must be captured — not just critical.""" + section = _step8_section() + assert "critical" in section and "warning" in section + + +def test_diff_context_schema_version_field(): + """JSON schema must include schema_version for future-proofing.""" + section = _step8_section() + assert "schema_version" in section + + +def test_step8_writes_raw_findings_json(): + """Step 8 must declare writing raw_findings_{pr_number}.json.""" + section = _step8_section() + assert "raw_findings_{pr_number}.json" in section diff --git a/tests/skills/test_review_pr_inline_comment_guards.py b/tests/skills/test_review_pr_inline_comment_guards.py index 6bde0a4061..d520676f4e 100644 --- a/tests/skills/test_review_pr_inline_comment_guards.py +++ b/tests/skills/test_review_pr_inline_comment_guards.py @@ -18,9 +18,38 @@ / "SKILL.md" ) - SKILL_TEXT = SKILL_PATH.read_text() +RESEARCH_SKILL_PATH = ( + Path(__file__).parent.parent.parent + / "src" + / "autoskillit" + / "skills_extended" + / "review-research-pr" + / "SKILL.md" +) +RESEARCH_SKILL_TEXT = RESEARCH_SKILL_PATH.read_text() + +RESOLVE_SKILL_PATH = ( + Path(__file__).parent.parent.parent + / "src" + / "autoskillit" + / "skills_extended" + / "resolve-review" + / "SKILL.md" +) +RESOLVE_SKILL_TEXT = RESOLVE_SKILL_PATH.read_text() + +RESOLVE_RESEARCH_SKILL_PATH = ( + Path(__file__).parent.parent.parent + / "src" + / "autoskillit" + / "skills_extended" + / "resolve-research-review" + / "SKILL.md" +) +RESOLVE_RESEARCH_SKILL_TEXT = RESOLVE_RESEARCH_SKILL_PATH.read_text() + # --- Orchestrator behavioral guardrails --- @@ -289,3 +318,246 @@ def test_mandatory_echo_positioned_between_step4_and_step5(): assert "do not proceed to step 5" in between.lower(), ( "The 'Do not proceed to Step 5' gate must appear between Step 4 and Step 5." ) + + +def test_review_pr_http200_success_signal(): + """HTTP 200 must be treated as review-post success; response body must not be inspected.""" + skill_md = SKILL_TEXT + lower = skill_md.lower() + assert "http 200" in lower, "Skill must reference HTTP 200 as the success signal" + idx = lower.find("http 200") + # Require the body-inspection prohibition appears in proximity to the HTTP 200 + # success signal instruction (not just anywhere in the document). + window = lower[max(0, idx - 100) : idx + 500] + assert "regardless of response body" in window, ( + "The 'regardless of response body' prohibition must appear in proximity to " + "the HTTP 200 success signal instruction" + ) + + +def test_review_pr_tier1_fallback_has_delay(): + """Tier 1 fallback loop must include sleep 1 between individual POSTs.""" + skill_md = SKILL_TEXT + tier1_idx = skill_md.find("Fallback Tier 1") + assert tier1_idx >= 0, "Tier 1 fallback section not found in skill" + tier2_idx = skill_md.find("Tier 2", tier1_idx) + tier1_section = skill_md[tier1_idx:tier2_idx] if tier2_idx >= 0 else skill_md[tier1_idx:] + assert "sleep 1" in tier1_section or "sleep(1)" in tier1_section, ( + "Tier 1 fallback loop must include sleep 1 between individual POST calls" + ) + + +# --- Unpostable findings surfacing guards --- + + +def test_step7_body_includes_outside_diff_range_section(): + """Step 7 review body must include an 'Outside Diff Range' section for UNPOSTABLE_FINDINGS.""" + text = SKILL_TEXT + step7_start = text.find("### Step 7") + step8_start = text.find("### Step 8") + assert step7_start != -1 + assert step8_start != -1 + step7_section = text[step7_start:step8_start] + assert "UNPOSTABLE_FINDINGS" in step7_section, ( + "Step 7 must reference UNPOSTABLE_FINDINGS to surface out-of-hunk findings " + "in the review body." + ) + assert "Outside Diff Range" in step7_section, ( + "Step 7 must include an 'Outside Diff Range' section in the review body " + "for findings that could not be posted as inline comments." + ) + + +def test_step6_posts_file_level_comments_for_critical_unpostable(): + """Step 6 must post file-level comments for critical-severity unpostable findings.""" + text = SKILL_TEXT + step6_start = text.find("### Step 6") + step65_start = text.find("### Step 6.5") + assert step6_start != -1 + assert step65_start != -1 + step6_section = text[step6_start:step65_start] + assert "subject_type" in step6_section, ( + "Step 6 must contain 'subject_type' for file-level comment posting." + ) + assert 'subject_type: "file"' in step6_section or 'subject_type="file"' in step6_section, ( + "Step 6 must use subject_type: 'file' for file-level comments." + ) + + +def test_step6_file_level_uses_individual_endpoint_not_batch(): + """File-level comments must use the individual comments endpoint, not the batch reviews API.""" + text = SKILL_TEXT + step6_start = text.find("### Step 6") + step65_start = text.find("### Step 6.5") + assert step6_start != -1 + assert step65_start != -1 + step6_section = text[step6_start:step65_start] + file_level_start = step6_section.find("File-Level Comments") + assert file_level_start != -1, "Step 6 must contain a 'File-Level Comments' section" + file_level_section = step6_section[file_level_start:] + assert "pulls/{pr_number}/comments" in file_level_section, ( + "File-level comments must use the individual /pulls/{N}/comments endpoint. " + "subject_type: 'file' is NOT valid on the batch Reviews API." + ) + + +def test_step6_file_level_has_rate_limit_delay(): + """File-level comment posting must include sleep 1 for API rate-limit discipline.""" + text = SKILL_TEXT + step6_start = text.find("### Step 6") + step65_start = text.find("### Step 6.5") + assert step6_start != -1 + assert step65_start != -1 + step6_section = text[step6_start:step65_start] + subject_idx = step6_section.find("subject_type") + assert subject_idx != -1 + file_level_section = step6_section[subject_idx:] + assert "sleep 1" in file_level_section, ( + "File-level comment posting must include 'sleep 1' between mutating API calls." + ) + + +def test_step1_5_skips_null_line_threads(): + """Step 1.5 must skip threads with null line and originalLine.""" + text = SKILL_TEXT + step15_start = text.find("### Step 1.5") + step2_start = text.find("### Step 2") + assert step15_start != -1 + assert step2_start != -1 + step15_section = text[step15_start:step2_start] + lower = step15_section.lower() + assert "null" in lower and "skip" in lower, ( + "Step 1.5 must contain guidance to skip threads with null line " + "(file-level comment threads from prior reviews)." + ) + + +def test_step4_documents_unpostable_feeds_step6_and_step7(): + """Step 4 must document that UNPOSTABLE_FINDINGS feeds into both Step 6 and Step 7.""" + text = SKILL_TEXT + step4_start = text.find("### Step 4") + step45_start = text.find("### Step 4.5") + assert step4_start != -1 + assert step45_start != -1 + step4_section = text[step4_start:step45_start] + assert "UNPOSTABLE_FINDINGS" in step4_section, "Step 4 must reference UNPOSTABLE_FINDINGS." + assert "Step 6" in step4_section and "Step 7" in step4_section, ( + "Step 4 must document that UNPOSTABLE_FINDINGS feeds into both " + "Step 6 (file-level comments) and Step 7 (review body)." + ) + + +def test_review_research_pr_step6_posts_file_level_comments(): + """review-research-pr Step 6 must post file-level comments for critical unpostable findings.""" + text = RESEARCH_SKILL_TEXT + step6_start = text.find("### Step 6") + step65_start = text.find("### Step 6.5") + assert step6_start != -1 + assert step65_start != -1 + step6_section = text[step6_start:step65_start] + assert "subject_type" in step6_section, ( + "review-research-pr Step 6 must contain 'subject_type' for file-level comment posting." + ) + assert 'subject_type: "file"' in step6_section or 'subject_type="file"' in step6_section, ( + "review-research-pr Step 6 must use subject_type: 'file' for file-level comments." + ) + + +def test_review_research_pr_step6_uses_individual_endpoint(): + """review-research-pr file-level comments must use the individual comments endpoint.""" + text = RESEARCH_SKILL_TEXT + step6_start = text.find("### Step 6") + step65_start = text.find("### Step 6.5") + assert step6_start != -1 + assert step65_start != -1 + step6_section = text[step6_start:step65_start] + file_level_start = step6_section.find("File-Level Comments") + assert file_level_start != -1, ( + "review-research-pr Step 6 must contain a 'File-Level Comments' section" + ) + file_level_section = step6_section[file_level_start:] + assert "pulls/{pr_number}/comments" in file_level_section, ( + "review-research-pr file-level comments must use the individual " + "/pulls/{N}/comments endpoint." + ) + + +def test_review_research_pr_step6_has_rate_limit_delay(): + """review-research-pr file-level comment posting must include sleep 1.""" + text = RESEARCH_SKILL_TEXT + step6_start = text.find("### Step 6") + step65_start = text.find("### Step 6.5") + assert step6_start != -1 + assert step65_start != -1 + step6_section = text[step6_start:step65_start] + file_level_start = step6_section.find("File-Level Comments") + assert file_level_start != -1 + file_level_section = step6_section[file_level_start:] + assert "sleep 1" in file_level_section, ( + "review-research-pr file-level comment posting must include 'sleep 1' " + "between mutating API calls." + ) + + +def test_review_research_pr_step6_critical_only(): + """review-research-pr file-level comments must be limited to critical-severity findings.""" + text = RESEARCH_SKILL_TEXT + step6_start = text.find("### Step 6") + step65_start = text.find("### Step 6.5") + assert step6_start != -1 + assert step65_start != -1 + step6_section = text[step6_start:step65_start] + file_level_start = step6_section.find("File-Level Comments") + assert file_level_start != -1 + file_level_section = step6_section[file_level_start:] + lower = file_level_section.lower() + assert "critical" in lower, ( + "review-research-pr file-level comments must be limited to " + "critical-severity findings only." + ) + + +def test_review_research_pr_step7_includes_outside_diff_range(): + """review-research-pr Step 7 must include 'Outside Diff Range' section.""" + text = RESEARCH_SKILL_TEXT + step7_start = text.find("### Step 7") + step8_start = text.find("### Step 8") + assert step7_start != -1 + assert step8_start != -1 + step7_section = text[step7_start:step8_start] + assert "UNPOSTABLE_FINDINGS" in step7_section, ( + "review-research-pr Step 7 must reference UNPOSTABLE_FINDINGS." + ) + assert "Outside Diff Range" in step7_section, ( + "review-research-pr Step 7 must include an 'Outside Diff Range' section." + ) + + +def test_resolve_review_handles_null_line_file_level_threads(): + """resolve-review must handle null-line file-level comment threads gracefully.""" + text = RESOLVE_SKILL_TEXT + step3_start = text.find("### Step 3") + step4_start = text.find("### Step 4") + assert step3_start != -1 + assert step4_start != -1 + step3_section = text[step3_start:step4_start] + lower = step3_section.lower() + assert "null" in lower and ("file-level" in lower or "file level" in lower), ( + "resolve-review Step 3 must handle null-line file-level comment threads " + "by skipping them — they have no code anchor." + ) + + +def test_resolve_research_review_handles_null_line_file_level_threads(): + """resolve-research-review must handle null-line file-level comment threads gracefully.""" + text = RESOLVE_RESEARCH_SKILL_TEXT + step3_start = text.find("### Step 3") + step4_start = text.find("### Step 4") + assert step3_start != -1 + assert step4_start != -1 + step3_section = text[step3_start:step4_start] + lower = step3_section.lower() + assert "null" in lower and ("file-level" in lower or "file level" in lower), ( + "resolve-research-review Step 3 must handle null-line file-level comment " + "threads by skipping them — they have no code anchor." + ) diff --git a/tests/skills/test_review_pr_local_mode.py b/tests/skills/test_review_pr_local_mode.py new file mode 100644 index 0000000000..f05fc262da --- /dev/null +++ b/tests/skills/test_review_pr_local_mode.py @@ -0,0 +1,178 @@ +"""Tests for review-pr/SKILL.md local mode (mode=local) behavior. + +Tests assert on SKILL.md content patterns for the local review round feature +(reducing GitHub API calls during iterative local review). +""" + +from pathlib import Path + +SKILL_PATH = ( + Path(__file__).parent.parent.parent + / "src" + / "autoskillit" + / "skills_extended" + / "review-pr" + / "SKILL.md" +) + + +def _skill_text() -> str: + return SKILL_PATH.read_text() + + +def test_review_pr_skill_documents_mode_parameter(): + """Assert SKILL.md contains mode= parameter documentation in Arguments section.""" + text = _skill_text() + assert "mode=" in text, ( + "review-pr/SKILL.md Arguments section must document the mode= keyword argument " + "with format: mode=" + ) + + +def test_review_pr_local_mode_writes_local_findings(): + """Assert SKILL.md contains instructions to write findings to local_findings_{pr_number}.json + when mode=local.""" + text = _skill_text() + assert "local_findings" in text, ( + "review-pr/SKILL.md must contain 'local_findings' when mode=local, " + "specifying the output file path pattern: " + "{{AUTOSKILLIT_TEMP}}/review-pr/local_findings_{pr_number}.json" + ) + assert "mode=local" in text, "review-pr/SKILL.md must reference 'mode=local' behavior" + + +def test_review_pr_local_mode_no_github_posts(): + """Assert SKILL.md contains explicit instruction to skip GitHub API calls when mode=local.""" + text = _skill_text() + # Find the mode=local section + local_mode_idx = text.lower().find("mode=local") + assert local_mode_idx >= 0, "SKILL.md must contain 'mode=local'" + # Check that within the local mode section, GitHub API posting is skipped + after_local = text[local_mode_idx:] + # The mode=local section should say to skip GitHub API calls + assert any( + phrase in after_local.lower() + for phrase in [ + "skip all github api", + "skip github api", + "no github api", + "do not post to github", + "do not call github", + ] + ), ( + "review-pr/SKILL.md mode=local section must explicitly instruct to skip " + "GitHub API calls for posting comments" + ) + + +def test_review_pr_local_mode_emits_gate_tokens(): + """Assert SKILL.md states that gate tokens (%%REVIEW_GATE::*) are emitted in both modes.""" + text = _skill_text() + # Gate tokens must be emitted in both modes + assert "%%REVIEW_GATE::LOOP_REQUIRED%%" in text, ( + "review-pr/SKILL.md must emit %%REVIEW_GATE::LOOP_REQUIRED%% on changes_requested" + ) + assert "%%REVIEW_GATE::CLEAR%%" in text, ( + "review-pr/SKILL.md must emit %%REVIEW_GATE::CLEAR%% on approved/needs_human" + ) + # Verify gate tokens are documented as mode-independent + gate_doc_idx = text.lower().find("gate token") + assert gate_doc_idx >= 0, "SKILL.md must document gate tokens" + # Check near gate token documentation that both modes are mentioned + gate_context = text[max(0, gate_doc_idx - 200) : gate_doc_idx + 400].lower() + assert "mode" in gate_context, ( + "Gate token documentation must mention that emission is mode-independent" + ) + + +def test_review_pr_local_mode_default_is_github(): + """Assert SKILL.md states that absent/unrecognized mode defaults to github.""" + text = _skill_text() + # Find the mode documentation — use a phrase specific to mode default, not generic "default:" + assert "absent or unrecognized" in text.lower() or 'default to "github"' in text.lower(), ( + "review-pr/SKILL.md must document the default value for mode parameter using a " + "specific phrase like 'absent or unrecognized' or 'default to \"github\"'" + ) + # Verify github is the default + local_mode_idx = text.lower().find("mode=github") + assert local_mode_idx >= 0 + # The github mode should be described as the default (absent/unrecognized) + mode_context = text[max(0, local_mode_idx - 300) : local_mode_idx + 200] + assert "default" in mode_context.lower() or "absent" in mode_context.lower(), ( + "review-pr/SKILL.md must state that mode=github is the default when " + "mode is absent or unrecognized" + ) + + +def test_review_pr_local_mode_iteration_tracking(): + """Assert SKILL.md tracks iteration number when writing local_findings for round counting.""" + text = _skill_text() + assert "iteration" in text.lower(), ( + "review-pr/SKILL.md must track iteration number when writing local_findings.json " + "to support round counting across local review cycles" + ) + + +def test_review_pr_local_mode_still_writes_diff_context(): + """Assert SKILL.md still writes diff_context_{pr_number}.json in local mode.""" + text = _skill_text() + # Find mode=local section + local_mode_idx = text.lower().find("mode=local") + assert local_mode_idx >= 0 + after_local = text[local_mode_idx : local_mode_idx + 2000] + assert "diff_context" in after_local, ( + "review-pr/SKILL.md mode=local section must still write diff_context_{pr_number}.json " + "(mode-independent handoff file for resolve-review)" + ) + + +def test_review_pr_local_mode_still_writes_raw_findings(): + """Assert SKILL.md still writes raw_findings_{pr_number}.json in local mode.""" + text = _skill_text() + local_mode_idx = text.lower().find("mode=local") + assert local_mode_idx >= 0 + after_local = text[local_mode_idx : local_mode_idx + 2000] + assert "raw_findings" in after_local, ( + "review-pr/SKILL.md mode=local section must still write raw_findings_{pr_number}.json " + "(mode-independent)" + ) + + +def test_review_pr_local_mode_skips_step6_and_step7(): + """Assert SKILL.md mode=local skips Step 6 (GitHub posting) and Step 7 (review submission).""" + text = _skill_text() + local_mode_idx = text.lower().find("mode=local") + assert local_mode_idx >= 0 + after_local = text[local_mode_idx : local_mode_idx + 2000] + # Should skip to Step 8 after writing local file + assert "step 8" in after_local.lower() or "skip" in after_local.lower(), ( + "review-pr/SKILL.md mode=local must skip to Step 8 (verdict emission) " + "after writing local_findings.json, bypassing Step 6 GitHub posting and Step 7 submission" + ) + + +def test_review_pr_local_mode_json_format(): + """Assert SKILL.md specifies the JSON schema for local_findings output.""" + text = _skill_text() + # Find the local_findings JSON format description + local_findings_idx = text.find("local_findings") + assert local_findings_idx >= 0 + after_local_findings = text[local_findings_idx : local_findings_idx + 1500] + # Should have fields like path, line, body, severity, dimension, verdict, iteration + assert '"findings"' in after_local_findings or "findings" in after_local_findings, ( + "review-pr/SKILL.md must specify the findings array in local_findings JSON schema" + ) + assert "iteration" in after_local_findings.lower(), ( + "review-pr/SKILL.md must include iteration field in local_findings JSON schema" + ) + + +def test_review_pr_step6_mode_branching_header(): + """Assert Step 6 begins with MODE BRANCHING header that separates local vs github paths.""" + text = _skill_text() + step6_idx = text.find("### Step 6") + assert step6_idx >= 0, "SKILL.md must contain Step 6" + step6_section = text[step6_idx : step6_idx + 200] + assert "MODE" in step6_section or "mode" in step6_section, ( + "Step 6 must begin with mode branching to separate local vs github behavior" + ) diff --git a/tests/skills/test_review_pr_prior_thread_awareness.py b/tests/skills/test_review_pr_prior_thread_awareness.py new file mode 100644 index 0000000000..52ac2e15c1 --- /dev/null +++ b/tests/skills/test_review_pr_prior_thread_awareness.py @@ -0,0 +1,112 @@ +"""Behavioral guard tests for review-pr/SKILL.md prior-thread awareness (T_RPA1–T_RPA7). + +These tests verify that the review-pr skill correctly documents and implements +prior review thread suppression and awareness introduced by the check_review_loop +feature. +""" + +from __future__ import annotations + +from pathlib import Path + +SKILL_PATH = ( + Path(__file__).parent.parent.parent + / "src" + / "autoskillit" + / "skills_extended" + / "review-pr" + / "SKILL.md" +) + + +def _skill_text() -> str: + return SKILL_PATH.read_text() + + +# T_RPA1 +def test_skill_contains_step_1_5_section() -> None: + """SKILL.md must contain a 'Step 1.5' section for prior thread fetch.""" + text = _skill_text() + assert "Step 1.5" in text, ( + "review-pr/SKILL.md must contain a 'Step 1.5' section that fetches prior " + "review thread context before running parallel audit subagents." + ) + + +# T_RPA2 +def test_step_1_5_references_graphql_review_threads() -> None: + """Step 1.5 must reference 'gh api graphql' and 'reviewThreads' for fetching threads.""" + text = _skill_text() + assert "reviewThreads" in text, ( + "review-pr/SKILL.md Step 1.5 must reference 'reviewThreads' in the GraphQL " + "query used to fetch prior review thread context." + ) + # Step 1.5 should reference graphql (the query is embedded inline) + assert "graphql" in text.lower(), ( + "review-pr/SKILL.md Step 1.5 must use a GraphQL query to fetch review threads." + ) + + +# T_RPA3 +def test_step_1_5_distinguishes_resolved_from_unresolved() -> None: + """Step 1.5 must distinguish isResolved=true (prior_resolved) from isResolved=false + (prior_unresolved).""" + text = _skill_text() + assert "prior_resolved_findings" in text, ( + "review-pr/SKILL.md Step 1.5 must build a 'prior_resolved_findings' list for " + "threads where isResolved=true and body contains [critical] or [warning]." + ) + assert "prior_unresolved_findings" in text, ( + "review-pr/SKILL.md Step 1.5 must build a 'prior_unresolved_findings' list for " + "threads where isResolved=false and body contains [critical] or [warning]." + ) + + +# T_RPA4 +def test_step_3_includes_do_not_reraise_instruction_for_resolved() -> None: + """Step 3 subagent prompt must include 'DO NOT RE-RAISE' for prior resolved findings.""" + text = _skill_text() + assert "DO NOT RE-RAISE" in text, ( + "review-pr/SKILL.md Step 3 subagent prompt must instruct subagents to not " + "re-raise prior resolved findings (DO NOT RE-RAISE instruction)." + ) + + +# T_RPA5 +def test_step_3_includes_focus_on_instruction_for_unresolved() -> None: + """Step 3 subagent prompt must include 'FOCUS ON' for prior unresolved findings.""" + text = _skill_text() + assert "FOCUS ON" in text, ( + "review-pr/SKILL.md Step 3 subagent prompt must instruct subagents to focus " + "on prior unresolved findings (FOCUS ON instruction)." + ) + + +# T_RPA6 +def test_step_4_mentions_filtering_suppressing_resolved_findings() -> None: + """Step 4 aggregation must mention filtering or suppressing prior_resolved_findings.""" + text = _skill_text() + lower = text.lower() + has_suppress = "suppress" in lower or "filter" in lower or "suppression" in lower + assert has_suppress, ( + "review-pr/SKILL.md Step 4 aggregation must mention filtering or suppressing " + "findings that match prior_resolved_findings to avoid re-raising fixed issues." + ) + # Must also reference the prior_resolved_findings in the Step 4 context + assert "prior_resolved" in text, ( + "review-pr/SKILL.md Step 4 must reference 'prior_resolved' entries when " + "describing the suppression pass." + ) + + +# T_RPA7 +def test_prior_resolved_matched_by_file_and_line_proximity() -> None: + """prior_resolved_findings must be matched by (file, line) proximity (±N lines), + not exact equality.""" + text = _skill_text() + has_proximity = "±" in text or "+/-" in text or "within" in text.lower() + assert has_proximity, ( + "review-pr/SKILL.md must specify that prior_resolved_findings matches use " + "line proximity (±N lines) to handle line drift from fix commits — not exact " + "line equality." + ) diff --git a/tests/skills/test_review_pr_verdict_guards.py b/tests/skills/test_review_pr_verdict_guards.py index d7e61d63a2..2e44a183b6 100644 --- a/tests/skills/test_review_pr_verdict_guards.py +++ b/tests/skills/test_review_pr_verdict_guards.py @@ -91,3 +91,42 @@ def test_needs_human_prose_describes_genuine_ambiguity(): "needs_human verdict prose must describe it as triggered by genuine ambiguity " "or decisions requiring human judgment — not by a count of findings." ) + + +def test_review_pr_own_pr_comment_retry(): + """SKILL.md must handle own-PR REQUEST_CHANGES 422 by retrying with COMMENT event.""" + skill_md = _skill_text() + assert re.search(r"422.{0,300}COMMENT|COMMENT.{0,300}422", skill_md, re.DOTALL), ( + "SKILL.md must describe retrying with COMMENT event when batch POST returns " + "HTTP 422 due to own-PR restriction" + ) + + +def test_contract_yamls_include_approved_with_comments() -> None: + """All 3 contract YAML files must include approved_with_comments in + expected_output_patterns and pattern_examples for the review-pr contract.""" + import yaml + + contracts_dir = ( + Path(__file__).parent.parent.parent / "src" / "autoskillit" / "recipes" / "contracts" + ) + contract_files = [ + contracts_dir / "implementation.yaml", + contracts_dir / "remediation.yaml", + contracts_dir / "implementation-groups.yaml", + ] + for contract_path in contract_files: + data = yaml.safe_load(contract_path.read_text()) + review_pr = data.get("skills", {}).get("review-pr", {}) + patterns = review_pr.get("expected_output_patterns", []) + examples = review_pr.get("pattern_examples", []) + pattern_str = " ".join(patterns) + assert "approved_with_comments" in pattern_str, ( + f"{contract_path.name}: expected_output_patterns for review-pr must include " + "'approved_with_comments'" + ) + examples_str = " ".join(examples) + assert "approved_with_comments" in examples_str, ( + f"{contract_path.name}: pattern_examples for review-pr must include " + "'approved_with_comments'" + ) diff --git a/tests/skills/test_skill_body_cleanliness.py b/tests/skills/test_skill_body_cleanliness.py index 3f178c093b..c0122e9da9 100644 --- a/tests/skills/test_skill_body_cleanliness.py +++ b/tests/skills/test_skill_body_cleanliness.py @@ -1,5 +1,8 @@ """Assert that no SKILL.md body references %%ORDER_UP%%.""" +from __future__ import annotations + +import re from pathlib import Path import pytest @@ -25,3 +28,68 @@ def test_no_order_up_in_skill_body(skill_md: Path) -> None: "The completion directive is injected by _inject_completion_directive() " "in commands.py. Remove it from the skill body." ) + + +_OUTPUT_SECTION_RE = re.compile(r"^## Output\s*$", re.MULTILINE) +_ORCHESTRATION_REF_RE = re.compile( + r"(?i)(ORCHESTRATION DIRECTIVE|completion marker|end your (final )?response with)", + re.MULTILINE, +) +_TOKEN_NAME_RE = re.compile(r"^(\w+)\s*=", re.MULTILINE) + +_KNOWN_VIOLATORS: frozenset[str] = frozenset( + { + "compose-pr", + "compose-research-pr", + "make-campaign", + "make-plan", + "prepare-issue", + "prepare-pr", + "prepare-research-pr", + "rectify", + "resolve-claims-review", + "resolve-research-review", + "resolve-review", + "review-pr", + "setup-environment", + "setup-project", + "stage-data", + } +) + + +@pytest.mark.parametrize("skill_md", _all_skill_mds(), ids=lambda p: p.parent.name) +def test_output_section_compatible_with_orchestration_directive(skill_md: Path) -> None: + """SKILL.md Output sections with contract tokens should reference the orchestration directive. + + If an Output section declares structured output tokens (e.g., plan_path=), it should + reference the ORCHESTRATION DIRECTIVE. This prevents the Output section from overriding + the prompt-level completion marker for non-Claude providers. + """ + skill_name = skill_md.parent.name + text = skill_md.read_text() + output_match = _OUTPUT_SECTION_RE.search(text) + if not output_match: + return + + output_start = output_match.end() + next_section = re.search(r"^##\s+\w", text[output_start:], re.MULTILINE) + output_end = len(text) if next_section is None else output_start + next_section.start() + output_text = text[output_start:output_end] + + token_names = set(_TOKEN_NAME_RE.findall(output_text)) + if not token_names: + return + + has_ref = bool(_ORCHESTRATION_REF_RE.search(output_text)) + if has_ref: + return + + msg = ( + f"{skill_name}/SKILL.md ## Output section specifies contract tokens " + f"({sorted(token_names)}) but does not reference the ORCHESTRATION DIRECTIVE. " + "Add: 'Include the completion marker from the ORCHESTRATION DIRECTIVE'." + ) + if skill_name in _KNOWN_VIOLATORS: + pytest.xfail(msg) + pytest.fail(msg) diff --git a/tests/skills/test_skill_compliance.py b/tests/skills/test_skill_compliance.py index b71cdcf3aa..a646d9de03 100644 --- a/tests/skills/test_skill_compliance.py +++ b/tests/skills/test_skill_compliance.py @@ -66,7 +66,7 @@ ] # Skills whose narration suppression is handled globally by _inject_narration_suppression() -# in build_full_headless_cmd() (headless path) and sous-chef/SKILL.md (cook path). +# in build_skill_session_cmd() (headless path) and sous-chef/SKILL.md (cook path). # Per-loop inline anti-prose guards are intentionally absent — they are redundant. _GLOBALLY_GUARDED_SKILLS: frozenset[str] = frozenset( { @@ -75,6 +75,8 @@ "setup-project", "collapse-issues", "validate-audit", + "validate-test-audit", + "validate-review-decisions", } ) @@ -237,7 +239,7 @@ def test_no_text_then_tool_in_any_step(skill_dir: Path) -> None: Skills in _GLOBALLY_GUARDED_SKILLS are exempt from the loop-boundary check — their narration suppression is injected at the prompt level - by build_full_headless_cmd() and sous-chef/SKILL.md. + by build_skill_session_cmd() and sous-chef/SKILL.md. This is a project-wide structural invariant, not specific to open-pr or arch-lens. @@ -351,3 +353,42 @@ def test_detector_catches_unguarded_mcp_loop() -> None: """ violations = _check_loop_boundary(vulnerable_pattern) assert len(violations) >= 1, "Detector failed to catch unguarded MCP loop" + + +# Detects skills that instruct Agent/Task subagent spawning. +# Any such skill MUST contain the run_in_background prohibition. +_SPAWN_INDICATOR_RE = re.compile( + r"Task tool|Explore subagent" + r"|spawn.*subagent|subagent.*spawn|launch.*subagent" + r"|parallel.*subagent|subagent.*parallel", + re.IGNORECASE, +) +_BACKGROUND_PROHIBITION_RE = re.compile(r"run_in_background.*prohibited", re.IGNORECASE) + +# Skills whose SKILL.md mentions subagents only in a negative/prohibitive context +# (e.g., "rather than spawning subagents", "do not spawn subagents"). The spawn +# indicator regex matches these descriptively — they are not spawning skills. +_NON_SPAWNING_SKILL_DIRS: frozenset[str] = frozenset( + { + "report-bug", # "rather than spawning parallel subagents" — describes non-spawning + "issue-splitter", # "do not spawn subagents" — prohibits spawning inline + } +) + + +@pytest.mark.parametrize("skill_dir", _all_skill_dirs(), ids=lambda p: p.name) +def test_no_background_subagent_in_spawning_skills(skill_dir: Path) -> None: + if skill_dir.name in _NON_SPAWNING_SKILL_DIRS: + return # Skill mentions subagents only descriptively/negatively — rule does not apply. + skill_md = skill_dir / "SKILL.md" + if not skill_md.exists(): + return + content = skill_md.read_text(encoding="utf-8") + if not _SPAWN_INDICATOR_RE.search(content): + return # Skill does not spawn subagents — rule does not apply. + assert _BACKGROUND_PROHIBITION_RE.search(content), ( + f"{skill_dir.name}/SKILL.md contains subagent-spawning instructions " + "but lacks the background-execution prohibition. " + "Add to its NEVER block: " + "'- Run subagents in the background (`run_in_background: true` is prohibited)'" + ) diff --git a/tests/skills/test_skill_genericization.py b/tests/skills/test_skill_genericization.py index 5e8e6c8d32..a7382c0aff 100644 --- a/tests/skills/test_skill_genericization.py +++ b/tests/skills/test_skill_genericization.py @@ -42,6 +42,17 @@ def test_implement_worktree_uses_config_driven_test_command() -> None: ) +def test_implement_worktree_sets_filter_env() -> None: + """implement-worktree/SKILL.md must set filter env vars in Step 5.""" + content = (SKILLS_EXTENDED_DIR / "implement-worktree" / "SKILL.md").read_text() + assert "AUTOSKILLIT_TEST_FILTER" in content, ( + "implement-worktree/SKILL.md must set AUTOSKILLIT_TEST_FILTER before test command" + ) + assert "AUTOSKILLIT_TEST_BASE_REF" in content, ( + "implement-worktree/SKILL.md must set AUTOSKILLIT_TEST_BASE_REF before test command" + ) + + def test_merge_pr_uses_generic_ci_check_name() -> None: """merge-pr/SKILL.md must not name project-specific CI checks.""" content = _skill_content("merge-pr") @@ -51,25 +62,6 @@ def test_merge_pr_uses_generic_ci_check_name() -> None: ) -def test_code_index_examples_are_generic() -> None: - """No bundled SKILL.md may use src/autoskillit/ as a code-index path example.""" - skills_with_violations: list[str] = [] - for skill_dir in SKILLS_DIR.iterdir(): - if not skill_dir.is_dir(): - continue - skill_md = skill_dir / "SKILL.md" - if not skill_md.exists(): - continue - content = skill_md.read_text() - # Check for the specific AutoSkillit path example in code-index instructions - if "src/autoskillit/execution/headless.py" in content: - skills_with_violations.append(skill_dir.name) - assert not skills_with_violations, ( - f"These skills have AutoSkillit-specific code-index path examples: " - f"{skills_with_violations}. Replace with generic placeholders (REQ-GEN-004)." - ) - - def test_scope_has_no_hardcoded_metrics_rs() -> None: """scope/SKILL.md must not reference the hardcoded src/metrics.rs path.""" content = (SKILLS_EXTENDED_DIR / "scope" / "SKILL.md").read_text() @@ -90,3 +82,12 @@ def test_plan_experiment_has_no_hardcoded_metrics_rs() -> None: "plan-experiment/SKILL.md hardcodes 'src/metrics.rs'. " "Use generic evaluation framework language (REQ-GEN-005)." ) + + +def test_make_plan_uses_generic_branch_terminology() -> None: + """make-plan/SKILL.md must not use 'integration branch' — project-specific terminology.""" + content = (SKILLS_EXTENDED_DIR / "make-plan" / "SKILL.md").read_text() + assert "integration branch" not in content, ( + "make-plan/SKILL.md uses 'integration branch' — project-specific terminology " + "from the merge-prs workflow. Use generic language instead (REQ-GEN-006)." + ) diff --git a/tests/skills/test_skill_output_compliance.py b/tests/skills/test_skill_output_compliance.py index 4b07e1f56f..80a7217f96 100644 --- a/tests/skills/test_skill_output_compliance.py +++ b/tests/skills/test_skill_output_compliance.py @@ -17,6 +17,7 @@ FIXED_NAME_SKILLS: frozenset[str] = frozenset( { "write-recipe", # .autoskillit/recipes/{name}.yaml — idempotent + "make-campaign", # .autoskillit/recipes/campaigns/{name}.yaml — idempotent "migrate-recipes", # .autoskillit/temp/migrations/{name}.yaml — idempotent "mermaid", # edits existing files, no temp output "open-kitchen", # singleton config file @@ -65,9 +66,10 @@ r"remediation_path|plan_parts|diagram_path|verdict|group_files|" r"pr_url|decision|needs_plan|deletion_regression|pr_number|" r"pr_branch|pr_title|total_issues|batch_count|recipe_distribution|" - r"integration_branch|pr_count|simple_count|needs_check_count|" + r"batch_branch|pr_count|simple_count|needs_check_count|" r"ci_blocked_count|review_blocked_count|queue_mode|" r"failure_type|is_fixable|escalation_required|escalation_reason|" + r"execution_map|execution_map_report|group_count|review_approach_candidates|" r"merged)=[^\s]", re.MULTILINE, ) @@ -98,9 +100,10 @@ def _get_skills_with_output_tokens() -> list[str]: r"remediation_path|plan_parts|diagram_path|verdict|group_files|" r"pr_url|decision|needs_plan|deletion_regression|pr_number|" r"pr_branch|pr_title|total_issues|batch_count|recipe_distribution|" - r"integration_branch|pr_count|simple_count|needs_check_count|" + r"batch_branch|pr_count|simple_count|needs_check_count|" r"ci_blocked_count|review_blocked_count|queue_mode|" r"failure_type|is_fixable|escalation_required|escalation_reason|" + r"execution_map|execution_map_report|group_count|review_approach_candidates|" r"merged)\s*=\s*", re.MULTILINE, ) @@ -236,11 +239,39 @@ def test_output_path_tokens_synchronized() -> None: "prep_path", # plan-visualization outputs (groupF Part A) "visualization_plan_path", + "visualization_plan_trace_path", "report_plan_path", # bundle-local-report output (groupG) "html_path", # stage-data skill output (resource feasibility report path) "resource_report", + # make-campaign skill output (campaign recipe manifest path) + "campaign_path", + # build-execution-map outputs (dependency-aware dispatch map) + "execution_map", + "execution_map_report", + # planner-generate-phases output (planner recipe) + "phase_manifest_path", + # planner-elaborate-phase output (parallel worker) + "elab_result_path", + # planner-refine-phases output + "refined_plan_path", + # planner-refine-assignments output + "phase_refined_path", + # planner-refine-wps output + "refined_wps_path", + # audit-tests output (bundled full-audit recipe) + "audit_report_path", + # validate-audit output (bundled full-audit recipe) + "validated_report_path", + # promote-to-main skill output (promote-to-main-wrapper recipe) + "pr_body_path", + # planner-validate-task-alignment output + "alignment_findings_path", + # planner-assess-review-approach output + "review_approach_assessment_path", + # setup-environment skill output (research recipe pre-flight gate) + "env_report", } ) @@ -251,23 +282,6 @@ def test_output_path_tokens_synchronized() -> None: ) -def test_resolve_failures_skill_switches_code_index_to_worktree(): - """resolve-failures must set_project_path to worktree_path after env setup.""" - skill_md = (pkg_root() / "skills_extended" / "resolve-failures" / "SKILL.md").read_text() - # Must contain a set_project_path call with worktree_path as the path argument. - # Use a regex so minor whitespace or quoting variations don't cause false failures. - worktree_switch = re.search(r"set_project_path\([^)]*worktree_path[^)]*\)", skill_md) - assert worktree_switch is not None, ( - "resolve-failures SKILL.md must switch code-index to {worktree_path} after env setup" - ) - # The worktree switch must come after the initial PROJECT_ROOT init. - project_root_idx = skill_md.find("PROJECT_ROOT") - assert project_root_idx != -1, "resolve-failures SKILL.md must reference PROJECT_ROOT" - assert worktree_switch.start() > project_root_idx, ( - "worktree_path code-index switch must appear after initial PROJECT_ROOT init" - ) - - # --------------------------------------------------------------------------- # Path-capture structured output compliance # --------------------------------------------------------------------------- @@ -277,6 +291,7 @@ def test_resolve_failures_skill_switches_code_index_to_worktree(): # Skills with path-capture contracts that must have their token instruction # in ## Critical Constraints (not only in ## Output or a late workflow step). PATH_CAPTURE_SKILLS: dict[str, list[str]] = { + "build-execution-map": ["execution_map", "execution_map_report"], "make-plan": ["plan_path"], "rectify": ["plan_path"], "investigate": ["investigation_path"], @@ -301,7 +316,7 @@ def test_resolve_failures_skill_switches_code_index_to_worktree(): "vis-lens-caption-annot": ["diagram_path"], "vis-lens-chart-select": ["diagram_path"], "vis-lens-color-access": ["diagram_path"], - "vis-lens-domain-norms": ["diagram_path"], + "vis-lens-methodology-norms": ["diagram_path"], "vis-lens-figure-table": ["diagram_path"], "vis-lens-multi-compare": ["diagram_path"], "vis-lens-reproducibility": ["diagram_path"], @@ -309,6 +324,19 @@ def test_resolve_failures_skill_switches_code_index_to_worktree(): "vis-lens-temporal": ["diagram_path"], "vis-lens-uncertainty": ["diagram_path"], "review-design": ["evaluation_dashboard", "revision_guidance"], + "planner-assess-review-approach": ["review_approach_assessment_path"], + "planner-elaborate-assignments": ["phase_assignments_result_dir"], + "planner-elaborate-phase": ["elab_result_path"], + "planner-elaborate-wps": ["phase_wps_result_dir"], + "planner-generate-phases": ["phase_manifest_path"], + "planner-refine-assignments": ["phase_refined_path"], + "planner-refine-phases": ["refined_plan_path"], + "planner-refine-wps": ["refined_wps_path"], + "planner-validate-task-alignment": ["alignment_findings_path"], + "audit-tests": ["audit_report_path"], + "validate-audit": ["validated_report_path"], + "validate-review-decisions": ["validated_report_path"], + "bundle-local-report": ["html_path"], } ABSOLUTE_PATH_KEYWORDS = ("absolute", "/abs", "$(pwd)", "$(cd") @@ -337,12 +365,19 @@ def _get_contracted_path_capture_skills() -> dict[str, list[str]]: for skill_name, contract in skills_data.items(): if not isinstance(contract, dict): continue + path_token_names = { + out["name"] + for out in contract.get("outputs", []) + if isinstance(out, dict) + and ( + out.get("type", "").startswith("file_path") or out.get("type") == "directory_path" + ) + } patterns = contract.get("expected_output_patterns", []) tokens = [] for pattern in patterns: - # Path-capture: token_name\s*=\s*/.+ (requires leading /) - m = re.match(r"^(\w+)\\s\*=\\s\*/.+", pattern) - if m: + m = re.match(r"^(\w+)", pattern) + if m and m.group(1) in path_token_names: tokens.append(m.group(1)) if tokens: result[skill_name] = tokens @@ -393,6 +428,32 @@ def test_every_contracted_skill_has_emit_instruction() -> None: ) +def test_contracted_path_capture_skills_includes_backslash_s_patterns() -> None: + """_get_contracted_path_capture_skills must return all skills with \\S+-terminated patterns.""" + raw = yaml.safe_load(SKILL_CONTRACTS_PATH.read_text()) + skills_data = raw.get("skills", {}) if isinstance(raw, dict) else {} + + # Dynamically find skills whose contracts include \S+-terminated path-capture patterns. + backslash_s_pattern_re = re.compile(r"\\[Ss]\+\s*$") + expected: list[str] = [] + for skill_name, contract in skills_data.items(): + if not isinstance(contract, dict): + continue + for pattern in contract.get("expected_output_patterns", []): + if backslash_s_pattern_re.search(pattern): + expected.append(skill_name) + break + + assert expected, "No skills with \\S+-terminated patterns found in contracts — test is vacuous" + + contracted = _get_contracted_path_capture_skills() + missing = [s for s in expected if s not in contracted] + assert not missing, ( + f"Skills with \\S+-terminated patterns not returned by " + f"_get_contracted_path_capture_skills: {missing}" + ) + + @pytest.mark.parametrize("skill_name,token_names", list(PATH_CAPTURE_SKILLS.items())) def test_path_capture_token_instruction_mentions_absolute( skill_name: str, token_names: list[str] diff --git a/tests/skills/test_skill_placeholder_contracts.py b/tests/skills/test_skill_placeholder_contracts.py index f10603ccc4..36aa34e386 100644 --- a/tests/skills/test_skill_placeholder_contracts.py +++ b/tests/skills/test_skill_placeholder_contracts.py @@ -22,7 +22,7 @@ extract_declared_ingredients, shell_vars_assigned, ) -from autoskillit.recipe.rules_skill_content import ( +from autoskillit.recipe.rules.rules_skill_content import ( _PSEUDOCODE_ALLOWLIST as _PROD_PSEUDOCODE_ALLOWLIST, ) diff --git a/tests/test_skill_preambles.py b/tests/skills/test_skill_preambles.py similarity index 57% rename from tests/test_skill_preambles.py rename to tests/skills/test_skill_preambles.py index 2d96af860a..5f7233e5a9 100644 --- a/tests/test_skill_preambles.py +++ b/tests/skills/test_skill_preambles.py @@ -8,8 +8,6 @@ - FRICT-5-3: external repo path validation """ -import pytest - from autoskillit.workspace.skills import DefaultSkillResolver @@ -19,39 +17,6 @@ def _skill_md(skill_name: str) -> str: return result.path.read_text() -CODE_INDEX_SKILLS = [ - "investigate", - "audit-impl", - "make-plan", - "make-groups", - "rectify", - "triage-issues", - "resolve-failures", -] - - -@pytest.mark.parametrize("skill_name", CODE_INDEX_SKILLS) -def test_code_index_skills_have_set_project_path(skill_name): - """Each code-index skill must call set_project_path in its workflow preamble.""" - content = _skill_md(skill_name) - assert "set_project_path" in content, ( - f"{skill_name}/SKILL.md is missing a set_project_path call. " - "Agents using code-index tools without initialization fail with " - "'Project path not set' (FRICT-1B-1)." - ) - - -@pytest.mark.parametrize("skill_name", CODE_INDEX_SKILLS) -def test_code_index_skills_have_relative_path_example(skill_name): - """Each updated skill must include a project-relative path example.""" - content = _skill_md(skill_name) - assert "src/" in content, ( - f"{skill_name}/SKILL.md is missing a project-relative path example " - "(e.g., src//some_module.py). Agents copy absolute " - "paths from Read output and code-index rejects them (FRICT-1C-2)." - ) - - def test_implement_worktree_has_pre_implementation_checklist(): """implement-worktree SKILL.md must contain the pre-implementation checklist.""" content = _skill_md("implement-worktree") diff --git a/tests/skills/test_skill_tool_syntax_contracts.py b/tests/skills/test_skill_tool_syntax_contracts.py new file mode 100644 index 0000000000..538967e627 --- /dev/null +++ b/tests/skills/test_skill_tool_syntax_contracts.py @@ -0,0 +1,63 @@ +"""Validates that SKILL.md bash blocks do not contain grep BRE \\| alternation patterns +that would be silently broken if copied to the Grep tool's pattern parameter. +""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from autoskillit.recipe._skill_placeholder_parser import extract_bash_blocks +from autoskillit.recipe.rules.rules_skill_content import _GIT_GREP_BRE_RE, _GREP_BRE_ALTERNATION_RE + +_REPO_ROOT = Path(__file__).parent.parent.parent +_SKILL_DIRS = [ + _REPO_ROOT / "src" / "autoskillit" / "skills", + _REPO_ROOT / "src" / "autoskillit" / "skills_extended", +] + + +def _all_skill_dirs() -> list[Path]: + result = [] + for skill_dir in _SKILL_DIRS: + if not skill_dir.exists(): + continue + result.extend(p for p in sorted(skill_dir.iterdir()) if p.is_dir()) + return result + + +def _is_git_grep_bre(line: str) -> bool: + """Return True if the \\| is inside a --grep= argument (git BRE context — allowed).""" + return bool(_GIT_GREP_BRE_RE.search(line)) + + +@pytest.mark.parametrize("skill_dir", _all_skill_dirs()) +def test_no_grep_bre_alternation_in_bash_blocks(skill_dir: Path) -> None: + """No SKILL.md bash block should contain grep '...\\|...' patterns. + + grep uses POSIX BRE where \\| is alternation, but the native Grep tool wraps + ripgrep which uses ERE where | (bare) is alternation. Models that copy \\| from + bash blocks into Grep tool pattern arguments get 0 results silently. + + Use rg 'foo|bar' in bash blocks (ripgrep syntax, identical to Grep tool). + Exception: --grep= arguments in git log/show commands are legitimate BRE. + """ + skill_md = skill_dir / "SKILL.md" + if not skill_md.exists(): + return + bash_blocks = extract_bash_blocks(skill_md.read_text()) + violations: list[str] = [] + for block in bash_blocks: + for line in block.splitlines(): + if _is_git_grep_bre(line): + continue # git --grep= BRE context: allowed + if _GREP_BRE_ALTERNATION_RE.search(line): + violations.append(f" Line: {line.strip()!r}") + assert not violations, ( + f"{skill_md.relative_to(Path.cwd())} contains grep BRE \\| patterns in bash " + f"blocks that could confuse models using the Grep tool:\n" + + "\n".join(violations) + + "\n\nFix: replace `grep 'foo\\|bar'` with `rg 'foo|bar'` " + "(ripgrep ERE matches Grep tool syntax)." + ) diff --git a/tests/skills/test_sous_chef_deferred_escalation.py b/tests/skills/test_sous_chef_deferred_escalation.py new file mode 100644 index 0000000000..e843a02a81 --- /dev/null +++ b/tests/skills/test_sous_chef_deferred_escalation.py @@ -0,0 +1,31 @@ +"""Contract tests for sous-chef deferred issue escalation (T6/T7).""" + +from __future__ import annotations + +from pathlib import Path + + +def _sous_chef_text() -> str: + return ( + Path(__file__).resolve().parent.parent.parent + / "src" + / "autoskillit" + / "skills" + / "sous-chef" + / "SKILL.md" + ).read_text() + + +def test_sous_chef_skillmd_has_deferred_escalation() -> None: + content = _sous_chef_text() + assert "deferred_issues" in content + assert "AskUserQuestion" in content + assert "Wait" in content and "Proceed" in content and "Drop" in content + assert "release_issue" in content + assert "headless" in content.lower() + + +def test_sous_chef_has_headless_wait_rule() -> None: + content = _sous_chef_text() + assert "denied" in content and "Wait" in content + assert "success: false" in content diff --git a/tests/skills/test_validate_audit_contracts.py b/tests/skills/test_validate_audit_contracts.py index e13bda345e..a592bb97d4 100644 --- a/tests/skills/test_validate_audit_contracts.py +++ b/tests/skills/test_validate_audit_contracts.py @@ -55,11 +55,14 @@ def test_two_output_files(self) -> None: assert "contested_findings_" in text # T-VAL-012 - def test_handles_all_three_audit_formats(self) -> None: + def test_handles_all_six_audit_formats(self) -> None: text = _skill_text() assert "audit-arch" in text assert "audit-tests" in text assert "audit-cohesion" in text + assert "audit-feature-gates" in text or "feature_gates" in text + assert "audit-docs" in text or "docs" in text + assert "audit-review-decisions" in text or "review_decisions" in text # T-VAL-013 def test_interactive_headless_distinction(self) -> None: @@ -67,8 +70,107 @@ def test_interactive_headless_distinction(self) -> None: # T-VAL-014 def test_output_dir(self) -> None: - assert "{{AUTOSKILLIT_TEMP}}/validate-audit/" in _skill_text() + assert "{{AUTOSKILLIT_TEMP}}/validate-audit-" in _skill_text() # T-VAL-015 def test_history_research_agent(self) -> None: assert "history research agent" in _skill_text().lower() + + # T-VAL-026 + def test_validate_audit_extended_filenames_have_no_timestamp_suffix(self) -> None: + """Extended validate-audit filenames must not carry timestamp suffixes.""" + text = _skill_text() + assert "validated_report_{source}.md" in text, ( + "Extended validate-audit/SKILL.md should use 'validated_report_{source}.md' " + "(no timestamp suffix)" + ) + + +class TestValidateAuditNewSteps: + # T-VAL-016 + def test_cross_validation_subagent_is_read_only(self) -> None: + text = _skill_text().lower() + assert "cross-valid" in text # "cross-validator" or "cross-validation" + assert "read-only" in text + + # T-VAL-017 + def test_ticket_grouping_manifest_with_finding_ids(self) -> None: + text = _skill_text().lower() + assert "grouping manifest" in text + assert "finding id" in text + + # T-VAL-018 + def test_validation_summary_separate_file(self) -> None: + assert "validation_summary_" in _skill_text() + + # T-VAL-019 + def test_validated_findings_contains_only_valid(self) -> None: + text = _skill_text() + assert ( + "do NOT include VALID BUT EXCEPTION WARRANTED" in text + or "exception-warranted findings go exclusively" in text.lower() + or "exception-warranted findings must not appear" in text.lower() + ) + + +class TestValidateAuditFeatureGates: + # T-VAL-020 + def test_feature_gates_severity_mapping_documented(self) -> None: + """validate-audit must document BLOCK->HIGH, WARN->MEDIUM, INFO->LOW severity mapping.""" + text = _skill_text() + assert "BLOCK" in text and "HIGH" in text, ( + "validate-audit must document severity normalization for feature-gates: BLOCK->HIGH" + ) + block_idx = text.find("BLOCK") + assert block_idx != -1, "BLOCK not found in validate-audit SKILL.md" + high_idx = text.find("HIGH", block_idx) + assert high_idx != -1 and (high_idx - block_idx) < 300, ( + "BLOCK->HIGH mapping must appear as a co-located severity mapping table, " + "not just incidental mentions" + ) + + # T-VAL-021 + def test_feature_gates_d1_d5_table_handling_documented(self) -> None: + """validate-audit must document special handling for D1/D5 table-format findings.""" + text = _skill_text() + has_d1_d5_handling = ("D1" in text and "D5" in text) and ( + "table" in text.lower() + or "table-format" in text.lower() + or "cross-cutting" in text.lower() + ) + assert has_d1_d5_handling, ( + "validate-audit must document that D1 (Config Projection) and D5 (Boundary Coupling) " + "produce table-format findings without file:line and must be handled in the " + "cross-cutting batch with direct file verification" + ) + + +class TestValidateAuditReviewDecisions: + # T-VAL-022 + def test_handles_review_decisions_audit_format(self) -> None: + """validate-audit must recognize 'Review Decisions Audit' as a source format.""" + text = _skill_text() + assert "Review Decisions Audit" in text + + # T-VAL-023 + def test_review_decisions_source_name(self) -> None: + """validate-audit must use 'review_decisions' as the source name.""" + text = _skill_text() + assert "review_decisions" in text + + # T-VAL-024 + def test_review_decisions_pr_provenance_preserved(self) -> None: + """validate-audit must preserve PR provenance metadata for review_decisions findings.""" + text = _skill_text().lower() + assert "pr number" in text or "pr provenance" in text or "pr #" in text + + # T-VAL-025 + def test_error_message_lists_six_formats(self) -> None: + """Error message for unrecognized format must list all 6 recognized formats.""" + text = _skill_text() + assert "Review Decisions Audit" in text + assert "Architectural Audit" in text + assert "Test Suite Audit" in text + assert "Cohesion Audit" in text + assert "Feature Gate Audit" in text + assert "Documentation Audit" in text diff --git a/tests/skills/test_validate_review_decisions_contracts.py b/tests/skills/test_validate_review_decisions_contracts.py new file mode 100644 index 0000000000..2492ebf9a6 --- /dev/null +++ b/tests/skills/test_validate_review_decisions_contracts.py @@ -0,0 +1,203 @@ +"""Contract tests for the validate-review-decisions skill SKILL.md.""" + +from __future__ import annotations + +import functools +import re + +from autoskillit.core.types import SkillSource +from autoskillit.workspace.skills import DefaultSkillResolver + + +@functools.cache +def _skill_text() -> str: + info = DefaultSkillResolver().resolve("validate-review-decisions") + assert info is not None, "validate-review-decisions skill not found" + return info.path.read_text(encoding="utf-8") + + +class TestValidateReviewDecisionsSkillExists: + # T-VRD-001 + def test_skill_exists(self) -> None: + info = DefaultSkillResolver().resolve("validate-review-decisions") + assert info is not None + assert info.source == SkillSource.BUNDLED_EXTENDED + assert info.path.exists() + + # T-VRD-002 + def test_has_audit_category(self) -> None: + info = DefaultSkillResolver().resolve("validate-review-decisions") + assert info is not None + assert "audit" in info.categories + + +class TestValidateReviewDecisionsContent: + # T-VRD-003 + def test_validated_true_marker_required(self) -> None: + assert "validated: true" in _skill_text() + + # T-VRD-004 + def test_parallel_single_message_dispatch(self) -> None: + text = _skill_text().lower() + assert "single message" in text + assert "parallel" in text + + # T-VRD-005 + def test_three_verdict_types_defined(self) -> None: + text = _skill_text() + assert "VALID" in text + assert "VALID BUT EXCEPTION WARRANTED" in text + assert "CONTESTED" in text + + # T-VRD-006 + def test_output_files(self) -> None: + text = _skill_text() + assert "validated_report_" in text + assert "contested_findings_" in text + + # T-VRD-007 + def test_handles_review_decisions_format_only(self) -> None: + text = _skill_text() + assert "audit-review-decisions" in text + assert "Review Decisions Audit" in text + + # T-VRD-008 + def test_history_research_agent(self) -> None: + assert "history research agent" in _skill_text().lower() + + # T-VRD-009 + def test_cross_validation_subagent_is_read_only(self) -> None: + text = _skill_text().lower() + assert "cross-valid" in text + assert "read-only" in text + + # T-VRD-010 + def test_ticket_grouping_manifest_with_finding_ids(self) -> None: + text = _skill_text().lower() + assert "grouping manifest" in text + assert "finding id" in text + + # T-VRD-011 + def test_validation_summary_separate_file(self) -> None: + assert "validation_summary_" in _skill_text() + + # T-VRD-012 + def test_validated_findings_contains_only_valid(self) -> None: + text = _skill_text() + assert "do NOT include VALID BUT EXCEPTION WARRANTED" in text + + # T-VRD-013 + def test_interactive_headless_distinction(self) -> None: + assert "Interactive vs Headless" in _skill_text() + + # T-VRD-014 + def test_audit_run_dir_support(self) -> None: + assert "AUTOSKILLIT_AUDIT_RUN_DIR" in _skill_text() + + # T-VRD-015 + def test_pr_provenance_metadata_preserved(self) -> None: + text = _skill_text() + assert "PR provenance" in text or "PR Number" in text + assert "Reviewer Quote" in text + + # T-VRD-016 + def test_review_decisions_source_name(self) -> None: + text = _skill_text() + assert "review_decisions" in text + + # T-VRD-017 + def test_validated_report_path_emit(self) -> None: + assert "validated_report_path" in _skill_text() + + +class TestValidateReviewDecisionsIntentAnalysis: + # T-VRD-018 + def test_intent_analysis_step_exists(self) -> None: + text = _skill_text().lower() + assert "intent analysis" in text + + # T-VRD-019 + def test_intent_analysis_docstring_check(self) -> None: + text = _skill_text().lower() + assert "docstring" in text + + # T-VRD-020 + def test_intent_analysis_git_provenance(self) -> None: + text = _skill_text().lower() + assert "git log" in text or "git provenance" in text + + # T-VRD-021 + def test_intent_analysis_test_coverage(self) -> None: + text = _skill_text().lower() + assert "test coverage" in text + + # T-VRD-022 + def test_intent_analysis_contract_analysis(self) -> None: + text = _skill_text().lower() + assert "contract analysis" in text or "consumer" in text + + # T-VRD-023 + def test_intent_analysis_architectural_constraint(self) -> None: + text = _skill_text() + assert "architectural constraint" in text or "IL import" in text + + # T-VRD-024 + def test_intent_analysis_behavioral_simulation(self) -> None: + text = _skill_text().lower() + assert "simulation" in text or "simulate" in text + + +class TestValidateReviewDecisionsEvidenceGatheringRules: + # T-VRD-025 + def test_rule_docstring_as_contract(self) -> None: + text = _skill_text().lower() + assert "docstring" in text + assert "contract" in text + + # T-VRD-026 + def test_rule_deliberate_change_detection(self) -> None: + text = _skill_text().lower() + assert "deliberate" in text + assert "intentional" in text + + # T-VRD-027 + def test_rule_test_as_intent_signal(self) -> None: + text = _skill_text().lower() + assert "test-as-intent-signal" in text + + # T-VRD-028 + def test_rule_consumer_impact_verification(self) -> None: + text = _skill_text().lower() + assert "consumer" in text + assert "impact" in text + + # T-VRD-029 + def test_rule_architectural_feasibility_check(self) -> None: + text = _skill_text().lower() + assert "architectural" in text + assert "feasibility" in text + + # T-VRD-030 + def test_rule_behavioral_simulation(self) -> None: + text = _skill_text().lower() + assert "behavioral" in text + assert "simulation" in text + + # T-VRD-031 + def test_rule_symmetry_as_design(self) -> None: + text = _skill_text().lower() + assert "symmetry" in text + assert "design" in text + + +class TestValidateReviewDecisionsInputHandling: + # T-VRD-032 + def test_auto_discover_audit_review_decisions(self) -> None: + assert "{{AUTOSKILLIT_TEMP}}/audit-review-decisions/" in _skill_text() + + # T-VRD-033 + def test_evidence_rules_are_generalizable(self) -> None: + # Must not reference specific finding IDs like F01, F07, F13 + text = _skill_text() + finding_ids = re.findall(r"F\d{2}", text) + assert len(finding_ids) == 0, f"Found specific finding IDs: {finding_ids}" diff --git a/tests/skills/test_validate_test_audit_contracts.py b/tests/skills/test_validate_test_audit_contracts.py new file mode 100644 index 0000000000..3969932161 --- /dev/null +++ b/tests/skills/test_validate_test_audit_contracts.py @@ -0,0 +1,181 @@ +"""Contract tests for the validate-test-audit skill SKILL.md.""" + +from __future__ import annotations + +import functools +import re + +from autoskillit.core.types import SkillSource +from autoskillit.workspace.skills import DefaultSkillResolver + + +@functools.cache +def _skill_text() -> str: + info = DefaultSkillResolver().resolve("validate-test-audit") + assert info is not None, "validate-test-audit skill not found" + return info.path.read_text(encoding="utf-8") + + +class TestValidateTestAuditSkillExists: + # T-VTA-001 + def test_validate_test_audit_skill_exists(self) -> None: + info = DefaultSkillResolver().resolve("validate-test-audit") + assert info is not None + assert info.source == SkillSource.BUNDLED_EXTENDED + assert info.path.exists() + + # T-VTA-002 + def test_validate_test_audit_has_audit_category(self) -> None: + info = DefaultSkillResolver().resolve("validate-test-audit") + assert info is not None + assert "audit" in info.categories + + +class TestValidateTestAuditContent: + # T-VTA-003 + def test_validated_true_marker_required(self) -> None: + assert "validated: true" in _skill_text() + + # T-VTA-004 + def test_parallel_single_message_dispatch(self) -> None: + text = _skill_text().lower() + assert "single message" in text + assert "parallel" in text + + # T-VTA-005 + def test_three_verdict_types_defined(self) -> None: + text = _skill_text() + assert "VALID" in text + assert "VALID BUT EXCEPTION WARRANTED" in text + assert "CONTESTED" in text + + # T-VTA-006 + def test_output_files(self) -> None: + text = _skill_text() + assert "validated_report_" in text + assert "contested_findings_" in text + + # T-VTA-007 + def test_handles_audit_tests_format_only(self) -> None: + text = _skill_text() + assert "audit-tests" in text + assert "Test Suite Audit" in text + + # T-VTA-008 + def test_output_dir_matches_validate_audit(self) -> None: + assert "{{AUTOSKILLIT_TEMP}}/validate-audit-{YYYY-MM-DD_HHMMSS}/" in _skill_text() + + # T-VTA-009 + def test_history_research_agent(self) -> None: + assert "history research agent" in _skill_text().lower() + + # T-VTA-010 + def test_cross_validation_subagent_is_read_only(self) -> None: + text = _skill_text().lower() + assert "cross-valid" in text + assert "read-only" in text + + # T-VTA-011 + def test_ticket_grouping_manifest_with_finding_ids(self) -> None: + text = _skill_text().lower() + assert "grouping manifest" in text + assert "finding id" in text + + # T-VTA-012 + def test_validation_summary_separate_file(self) -> None: + assert "validation_summary_" in _skill_text() + + # T-VTA-013 + def test_validated_findings_contains_only_valid(self) -> None: + text = _skill_text() + assert ( + "do NOT include VALID BUT EXCEPTION WARRANTED" in text + or "exception-warranted findings go exclusively" in text.lower() + ) + + # T-VTA-014 + def test_interactive_headless_distinction(self) -> None: + assert "Interactive vs Headless" in _skill_text() + + # T-VTA-027 + def test_validate_test_audit_uses_per_run_subdirectory_pattern(self) -> None: + """validate-test-audit must use per-run subdirectory pattern.""" + text = _skill_text() + assert "validate-audit-{YYYY-MM-DD_HHMMSS}/" in text, ( + "validate-test-audit/SKILL.md missing per-run subdirectory pattern" + ) + + +class TestValidateTestAuditSemanticRules: + # T-VTA-015 + def test_semantic_rule_import_path_as_contract(self) -> None: + text = _skill_text().lower() + assert "import" in text, "expected 'import' in skill text" + assert "contract" in text, "expected 'contract' in skill text" + assert "existence guard" in text + + # T-VTA-016 + def test_semantic_rule_precondition_as_assertion(self) -> None: + text = _skill_text().lower() + assert "precondition" in text, "expected 'precondition' in skill text" + assert "assertion" in text, "expected 'assertion' in skill text" + + # T-VTA-017 + def test_semantic_rule_provenance_verification(self) -> None: + text = _skill_text().lower() + assert "__module__" in text or "provenance" in text + assert "provenance guard" in text or "provenance verification" in text + + # T-VTA-018 + def test_semantic_rule_split_era_lifecycle(self) -> None: + text = _skill_text().lower() + assert "split-era" in text, "expected 'split-era' in skill text" + assert "lifecycle" in text or "structural contract" in text + + # T-VTA-019 + def test_semantic_rule_deletion_vs_improvement(self) -> None: + text = _skill_text().lower() + assert "deletion" in text, "expected 'deletion' in skill text" + assert "improvement" in text, "expected 'improvement' in skill text" + assert "improving" in text or "improve" in text + + +class TestValidateTestAuditIntentAnalysis: + # T-VTA-020 + def test_intent_analysis_step_exists(self) -> None: + text = _skill_text().lower() + assert "intent analysis" in text or "intent determination" in text + + # T-VTA-021 + def test_intent_analysis_git_provenance(self) -> None: + text = _skill_text().lower() + assert "git log" in text or "git provenance" in text + assert "introducing commit" in text or "commit message" in text + + # T-VTA-022 + def test_intent_analysis_co_creation_context(self) -> None: + text = _skill_text().lower() + assert "co-creation" in text or "same commit" in text + + # T-VTA-023 + def test_intent_analysis_naming_signals(self) -> None: + text = _skill_text().lower() + assert "naming signal" in text or "test file name" in text or "function name" in text + + # T-VTA-024 + def test_intent_analysis_redundancy_check(self) -> None: + text = _skill_text().lower() + assert "redundancy check" in text or "covered by another test" in text + + +class TestValidateTestAuditInputHandling: + # T-VTA-025 + def test_auto_discover_audit_tests(self) -> None: + assert "{{AUTOSKILLIT_TEMP}}/audit-tests/" in _skill_text() + + # T-VTA-026 + def test_semantic_rules_are_generalizable(self) -> None: + # Must not reference specific finding IDs like C1-3, C1-4 + text = _skill_text() + finding_ids = re.findall(r"C\d+-\d+", text) + assert len(finding_ids) == 0, f"Found specific finding IDs: {finding_ids}" diff --git a/tests/skills/test_vis_lens_methodology_norms.py b/tests/skills/test_vis_lens_methodology_norms.py new file mode 100644 index 0000000000..7daac51d44 --- /dev/null +++ b/tests/skills/test_vis_lens_methodology_norms.py @@ -0,0 +1,505 @@ +"""Skill-level integration tests for vis-lens-methodology-norms tradition expansion.""" + +from __future__ import annotations + +import os +from pathlib import Path + +import pytest +import yaml + +from autoskillit.core import RETIRED_SKILL_NAMES +from autoskillit.recipe.methodology_disambiguation import ( + disambiguate, +) +from autoskillit.recipe.methodology_tradition_registry import ( + MethodologyTraditionSpec, + get_methodology_tradition_by_name, + is_out_of_scope_tradition, + load_all_methodology_traditions, +) +from autoskillit.recipe.methodology_tradition_router import ( + classify_methodology, +) +from autoskillit.recipe.methodology_venue_appendix import ( + resolve_venue_appendices, +) +from autoskillit.workspace.skills import DefaultSkillResolver + +EXPECTED_TRADITIONS: list[str] = [ + "controlled_intervention", + "systematic_synthesis", + "observational_correlational", + "diagnostic_accuracy", + "prediction_model_validation", + "simulation_modeling_tradition", + "measurement_instrument_validation_tradition", + "quality_improvement", + "economic_evaluation", + "animal_preclinical", + "qualitative_interpretive_tradition", + "method_comparison_benchmarking", +] + +TIER_C_PLAN_FIXTURES: list[tuple[str, str]] = [ + ( + "controlled_intervention", + "This randomized controlled trial evaluated the efficacy of a novel " + "intervention using CONSORT-compliant design. Participants were randomly " + "allocated to treatment or placebo arms. Primary endpoints were assessed " + "at 12-week follow-up using intention-to-treat analysis.", + ), + ( + "systematic_synthesis", + "We conducted a systematic review following PRISMA 2020 guidelines. " + "A comprehensive meta-analysis synthesized effect sizes across 42 eligible " + "studies. Heterogeneity was assessed using the I-squared statistic and " + "publication bias was evaluated via funnel plot asymmetry.", + ), + ( + "observational_correlational", + "This prospective cohort study followed 5,000 participants over ten years. " + "STROBE reporting guidelines were followed throughout. Hazard ratios were " + "estimated using Cox proportional hazards models adjusting for confounders.", + ), + ( + "diagnostic_accuracy", + "We evaluated the diagnostic accuracy of a novel biomarker panel using " + "STARD-compliant methodology. Sensitivity and specificity were calculated " + "against the histopathology reference standard. ROC analysis determined " + "optimal cutoff thresholds.", + ), + ( + "prediction_model_validation", + "This study developed and externally validated a clinical prediction model " + "following TRIPOD guidelines. Model calibration was assessed using the " + "Hosmer-Lemeshow test. Discrimination was quantified via the concordance " + "statistic on the held-out validation cohort.", + ), + ( + "simulation_modeling_tradition", + "We constructed an agent-based model to evaluate healthcare resource " + "allocation under pandemic scenarios. Parameter sweep experiments revealed " + "emergent behavior patterns at the population level. Individual-based model " + "dynamics were validated against observational data.", + ), + ( + "measurement_instrument_validation_tradition", + "This psychometric validation study followed COSMIN methodology to evaluate " + "a patient-reported outcome measure. Test-retest reliability and internal " + "consistency were assessed. Structural validity was established through " + "confirmatory factor analysis.", + ), + ( + "quality_improvement", + "This quality improvement initiative applied the SQUIRE 2.0 framework to " + "reduce medication errors in an inpatient setting. Three iterative PDSA " + "cycles refined the intervention. Statistical process control charts " + "monitored performance over the implementation period.", + ), + ( + "economic_evaluation", + "We conducted a cost-effectiveness analysis alongside the clinical trial " + "following CHEERS 2022 reporting standards. Incremental cost-effectiveness " + "ratios were computed from the healthcare payer perspective. Probabilistic " + "sensitivity analysis explored parameter uncertainty.", + ), + ( + "animal_preclinical", + "This preclinical study adhered to ARRIVE 2.0 reporting guidelines for " + "animal research. Male Sprague-Dawley rats were randomized to treatment " + "groups. Histological endpoints were scored by blinded observers using " + "validated grading systems.", + ), + ( + "qualitative_interpretive_tradition", + "We employed reflexive thematic analysis within a constructivist paradigm " + "following COREQ reporting standards. Semi-structured interviews were " + "conducted with 24 participants until thematic saturation. Credibility was " + "enhanced through member checking and peer debriefing.", + ), + ( + "method_comparison_benchmarking", + "This benchmark study compared five transformer architectures on standard " + "NLP tasks using an ablation study design. Statistical significance was " + "assessed via bootstrap confidence intervals and performance comparison " + "across five random seeds.", + ), +] + +DISAMBIGUATION_PLAN_FIXTURES: list[tuple[str, str, str, tuple[str, ...]]] = [ + ( + "prisma_dominance", + "This systematic review and meta-analysis followed PRISMA guidelines to " + "synthesize evidence from randomized controlled trial and clinical trial " + "designs. CONSORT reporting standards were applied. Forest plots summarized " + "pooled effect estimates across included studies.", + "systematic_synthesis", + (), + ), + ( + "rct_economic_union", + "This randomized controlled trial included a prospective economic evaluation. " + "CONSORT reporting was followed for the clinical endpoints. Cost-effectiveness " + "analysis adhered to CHEERS 2022 standards with an NHS perspective. " + "Incremental cost per QALY was the primary economic outcome.", + "controlled_intervention", + ("CHEERS_union",), + ), + ( + "arrive_supersedes_consort", + "This preclinical animal study used a randomized controlled trial design " + "following ARRIVE 2.0 guidelines. Treatment and vehicle groups of C57BL/6 " + "mice were compared. CONSORT reporting standards and clinical trial " + "registration were applied. Behavioral endpoints were assessed by blinded raters.", + "animal_preclinical", + (), + ), + ( + "benchmarking_prediction_nested", + "This benchmark study used an ablation study design to compare prediction " + "models following TRIPOD guidelines. TRIPOD-adherent calibration and " + "discrimination metrics were computed. External validation cohorts confirmed " + "the c-statistic across the benchmark evaluation protocol.", + "method_comparison_benchmarking", + ("TRIPOD_nested",), + ), +] + +VENUE_PLAN_FIXTURES: list[tuple[str, str, str, bool]] = [ + ( + "foundation_models", + "This study benchmarks foundation model performance across few-shot learning tasks. Standard NLP and vision benchmarks measure generalization. Ablation experiments isolate pretraining data effects.", + "method_comparison_benchmarking", + False, + ), + ( + "foundation_models", + "We evaluate foundation model calibration on held-out validation sets. Few-shot scaling laws are analyzed across model sizes. Zero-shot validation performance is compared against fine-tuned baselines.", + "prediction_model_validation", + True, + ), + ( + "foundation_models", + "This psychometric analysis evaluates latent trait measurement properties of foundation model outputs. Item response theory models assess construct validity. Cronbach alpha quantifies internal consistency of generated scales.", + "measurement_instrument_validation_tradition", + True, + ), + ( + "reinforcement_learning", + "This benchmark compares deep reinforcement learning algorithms on Atari and MuJoCo environments. Ablation studies isolate architectural contributions. Statistical significance is assessed across 10 random seeds.", + "method_comparison_benchmarking", + False, + ), + ( + "reinforcement_learning", + "We study multi-agent reinforcement learning with emergent behavior in population dynamics environments. Swarm intelligence metrics quantify collective coordination. Agent-based simulation validates scaling.", + "simulation_modeling_tradition", + True, + ), + ( + "reinforcement_learning", + "This study evaluates reinforcement learning policy on held-out task distributions. Sample efficiency curves measure data requirements. Policy evaluation protocols follow systematic benchmarking methodology.", + "prediction_model_validation", + True, + ), + ( + "supervised_classification", + "This study benchmarks supervised classification models on tabular and image datasets. Ablation experiments measure feature importance. Bootstrap confidence intervals assess statistical significance.", + "method_comparison_benchmarking", + False, + ), + ( + "supervised_classification", + "We evaluate supervised classification for medical imaging diagnosis of radiology findings. Sensitivity and specificity are computed against pathology gold standard. ROC analysis determines optimal thresholds.", + "diagnostic_accuracy", + True, + ), + ( + "supervised_classification", + "This supervised classification model predicts patient outcome trajectories. Prognosis endpoints include 30-day mortality and risk prediction scores. Clinical decision support integration is evaluated.", + "prediction_model_validation", + True, + ), + ( + "nlp", + "This NLP benchmark compares transformer architectures on text classification and generation tasks. Ablation experiments measure attention head contribution. Results are averaged over five random seeds.", + "method_comparison_benchmarking", + False, + ), + ( + "nlp", + "We develop a clinical NLP pipeline for automated risk scoring from patient notes. The clinical text extraction module predicts patient outcome categories. External validation uses held-out hospital data.", + "prediction_model_validation", + True, + ), + ( + "nlp", + "This NLP study applies discourse analysis to conversational AI outputs. Narrative structure is examined through thematic coding of dialogue turns. Conversation analysis reveals pragmatic patterns in generated text.", + "qualitative_interpretive_tradition", + True, + ), + ( + "computer_vision", + "This computer vision benchmark compares object detection architectures on COCO and ImageNet. Ablation experiments measure backbone contribution. mAP scores are reported across five training runs.", + "method_comparison_benchmarking", + False, + ), + ( + "computer_vision", + "We evaluate a computer vision model for medical imaging classification in radiology screening. Sensitivity and specificity of pathology detection are calculated against expert consensus. STARD methodology is followed.", + "diagnostic_accuracy", + True, + ), + ( + "computer_vision", + "This computer vision system predicts patient outcome from retinal imaging biomarkers. Clinical decision support thresholds are derived from ROC analysis. Prognosis models are validated on external cohorts.", + "prediction_model_validation", + True, + ), + ( + "generative_models", + "This study benchmarks generative model architectures on image synthesis quality. FID and IS scores are compared across diffusion and GAN variants. Ablation experiments isolate conditioning mechanism effects.", + "method_comparison_benchmarking", + False, + ), + ( + "generative_models", + "We develop a human evaluation instrument for generative model output quality. Inter-rater reliability is assessed via ICC. Construct validity is established through scale development methodology.", + "measurement_instrument_validation_tradition", + True, + ), + ( + "generative_models", + "This study applies qualitative assessment of generative model outputs through expert rating sessions. Human evaluation uses content analysis protocols with thematic coding. Evaluator agreement is reported.", + "qualitative_interpretive_tradition", + True, + ), + ( + "agentic_systems", + "This benchmark compares agentic system architectures on task completion metrics. Ablation experiments measure planning module contribution. Results are averaged across 20 evaluation episodes.", + "method_comparison_benchmarking", + False, + ), + ( + "agentic_systems", + "We study multi-agent coordination through agent-based model simulation. Emergent behavior patterns are analyzed in swarm intelligence scenarios. Population-level dynamics emerge from individual agent rules.", + "simulation_modeling_tradition", + True, + ), + ( + "agentic_systems", + "This agentic system targets process improvement in automated workflows. Workflow optimization metrics track task efficiency gains. Error reduction rates are measured across deployment iterations.", + "quality_improvement", + True, + ), + ( + "time_series", + "This study benchmarks time series forecasting architectures on standard datasets. Ablation experiments isolate temporal encoding contributions. MAE and RMSE are reported with bootstrap confidence intervals.", + "method_comparison_benchmarking", + False, + ), + ( + "time_series", + "We develop a clinical forecasting model for patient trajectory prediction. Risk prediction scores for mortality are validated on external cohorts. Calibration curves assess probability reliability.", + "prediction_model_validation", + True, + ), + ( + "time_series", + "This time series study employs a dynamical system model with differential equation formulation. Compartmental model dynamics are simulated across parameter ranges. System dynamics analysis reveals stability regions.", + "simulation_modeling_tradition", + True, + ), +] + + +@pytest.mark.parametrize("tradition_name", EXPECTED_TRADITIONS) +def test_tradition_schema_valid(tradition_name: str) -> None: + spec = get_methodology_tradition_by_name(tradition_name) + assert spec is not None, f"Tradition '{tradition_name}' not found" + assert isinstance(spec, MethodologyTraditionSpec) + assert spec.name == tradition_name + assert spec.display_name, "display_name must be non-empty" + assert spec.schema_version == "1.0" + assert isinstance(spec.priority, int) and spec.priority >= 1 + assert len(spec.detection_keywords) >= 2, "need >= 2 detection keywords for matching" + assert isinstance(spec.mandatory_figures, list) + assert isinstance(spec.strongly_expected_figures, list) + assert isinstance(spec.anti_patterns, list) + assert isinstance(spec.canonical_guideline, dict) + assert {"name", "governing_body", "stable_for_decade", "canonical"} == set( + spec.canonical_guideline.keys() + ) + assert isinstance(spec.venue_specific_appendices, tuple) + + +@pytest.mark.parametrize( + "tradition_slug,plan_text", + TIER_C_PLAN_FIXTURES, + ids=[t[0] for t in TIER_C_PLAN_FIXTURES], +) +def test_tier_c_selection(tradition_slug: str, plan_text: str) -> None: + result = classify_methodology(plan_text) + assert result.primary_tradition == tradition_slug + assert result.precedence_trace == "stage1_single_match" + assert tradition_slug in result.candidate_set + assert result.applied_union_rules == () + + +@pytest.mark.parametrize( + "rule_name,plan_text,expected_primary,expected_union_rules", + DISAMBIGUATION_PLAN_FIXTURES, + ids=[f[0] for f in DISAMBIGUATION_PLAN_FIXTURES], +) +def test_disambiguation_rule( + rule_name: str, + plan_text: str, + expected_primary: str, + expected_union_rules: tuple[str, ...], +) -> None: + router_result = classify_methodology(plan_text) + assert len(router_result.candidate_set) >= 2, ( + f"Expected multi-match for {rule_name}, got {router_result.candidate_set}" + ) + + disamb_result = disambiguate(set(router_result.candidate_set)) + assert disamb_result.primary_tradition == expected_primary + for rule in expected_union_rules: + assert rule in disamb_result.applied_union_rules + assert f"rule_{rule_name}" in disamb_result.precedence_trace + + +@pytest.mark.parametrize( + "sub_area,plan_text,expected_parent,expected_rerouted", + VENUE_PLAN_FIXTURES, + ids=[f"{f[0]}_{f[2]}{'_rerouted' if f[3] else '_primary'}" for f in VENUE_PLAN_FIXTURES], +) +def test_venue_appendix_branching( + sub_area: str, + plan_text: str, + expected_parent: str, + expected_rerouted: bool, +) -> None: + matches = resolve_venue_appendices(plan_text) + sub_area_matches = [m for m in matches if m.sub_area == sub_area] + assert len(sub_area_matches) >= 1, f"No match for sub_area={sub_area}" + match = sub_area_matches[0] + assert match.resolved_parent == expected_parent + assert match.re_routed == expected_rerouted + + +def test_no_mandatory_figures_path() -> None: + spec = get_methodology_tradition_by_name("qualitative_interpretive_tradition") + assert spec is not None + assert is_out_of_scope_tradition(spec) is True + assert spec.mandatory_figures == [] + assert len(spec.strongly_expected_figures) >= 1 + for fig in spec.strongly_expected_figures: + assert "figure" in fig and "source" in fig + assert spec.canonical_guideline["name"] == "COREQ/SRQR" + + +def test_renamed_skill_discoverable() -> None: + assert "vis-lens-domain-norms" in RETIRED_SKILL_NAMES + resolver = DefaultSkillResolver() + info = resolver.resolve("vis-lens-methodology-norms") + assert info is not None, "DefaultSkillResolver must find vis-lens-methodology-norms" + assert info.name == "vis-lens-methodology-norms" + assert info.path.is_file() + + +def test_classification_determinism() -> None: + plan_text = TIER_C_PLAN_FIXTURES[0][1] + results = [classify_methodology(plan_text) for _ in range(10)] + first = results[0] + for r in results[1:]: + assert r.primary_tradition == first.primary_tradition + assert r.candidate_set == first.candidate_set + assert r.precedence_trace == first.precedence_trace + assert r.applied_union_rules == first.applied_union_rules + + +def test_cache_invalidation(tmp_path: Path) -> None: + user_dir = tmp_path / ".autoskillit" / "methodology-traditions" + user_dir.mkdir(parents=True) + custom_yaml: dict = { + "name": "cache_test_tradition", + "display_name": "Cache Test", + "schema_version": "1.0", + "priority": 99, + "canonical_guideline": { + "name": "TEST", + "governing_body": "Test", + "stable_for_decade": True, + "canonical": True, + }, + "fields_spanned": ["test"], + "detection_keywords": ["cache_test_keyword_alpha", "cache_test_keyword_beta"], + "mandatory_figures": [{"figure": "test", "source": "test"}], + "strongly_expected_figures": [], + "anti_patterns": [], + } + yaml_path = user_dir / "cache_test_tradition.yaml" + yaml_path.write_text(yaml.dump(custom_yaml)) + + r1 = load_all_methodology_traditions(project_dir=tmp_path) + names_1 = {s.name for s in r1} + assert "cache_test_tradition" in names_1 + + old_mt = user_dir.stat().st_mtime + yaml_path.unlink() + os.utime(user_dir, (old_mt + 2, old_mt + 2)) + + r2 = load_all_methodology_traditions(project_dir=tmp_path) + names_2 = {s.name for s in r2} + assert "cache_test_tradition" not in names_2 + + +class TestIntegratedPipeline: + def test_single_tradition_full_pipeline(self) -> None: + plan_text = ( + "This benchmark study compares five supervised classification architectures " + "on standard image datasets using an ablation study design. Statistical " + "significance was assessed via bootstrap confidence intervals across seeds." + ) + router_result = classify_methodology(plan_text) + assert router_result.primary_tradition == "method_comparison_benchmarking" + + venue_matches = resolve_venue_appendices(plan_text) + sub_areas = {m.sub_area for m in venue_matches} + assert "supervised_classification" in sub_areas + + def test_multi_tradition_disambiguation_then_venue(self) -> None: + plan_text = ( + "This systematic review synthesized evidence from randomized controlled " + "trials following PRISMA 2020 guidelines. A clinical trial registry was " + "consulted. Meta-analysis pooled effect sizes across CONSORT-compliant RCTs." + ) + router_result = classify_methodology(plan_text) + assert len(router_result.candidate_set) >= 2 + + disamb_result = disambiguate(set(router_result.candidate_set)) + assert disamb_result.primary_tradition == "systematic_synthesis" + + def test_out_of_scope_skips_venue_appendix(self) -> None: + spec = get_methodology_tradition_by_name("qualitative_interpretive_tradition") + assert spec is not None + assert is_out_of_scope_tradition(spec) is True + assert spec.mandatory_figures == [] + assert len(spec.strongly_expected_figures) >= 1 + + def test_venue_reroute_produces_correct_appendix(self) -> None: + plan_text = ( + "We evaluate a supervised classification model for medical imaging " + "diagnosis in radiology screening. Sensitivity and specificity are " + "computed against the pathology reference standard." + ) + matches = resolve_venue_appendices(plan_text) + sc_matches = [m for m in matches if m.sub_area == "supervised_classification"] + assert len(sc_matches) >= 1 + match = sc_matches[0] + assert match.re_routed is True + assert match.resolved_parent == "diagnostic_accuracy" + assert match.appendix.sub_area == "supervised_classification" + assert len(match.appendix.expectations) >= 1 diff --git a/tests/skills/test_vis_lens_structural.py b/tests/skills/test_vis_lens_structural.py index 9c658f3dac..1c1f396650 100644 --- a/tests/skills/test_vis_lens_structural.py +++ b/tests/skills/test_vis_lens_structural.py @@ -13,7 +13,7 @@ "chart-select", "uncertainty", "antipattern", - "domain-norms", + "methodology-norms", "always-on", "multi-compare", "temporal", @@ -108,3 +108,17 @@ def test_frontmatter_activate_deps(slug: str) -> None: assert fm.get("activate_deps") == ["mermaid"], ( f"vis-lens-{slug} frontmatter must have activate_deps: [mermaid]" ) + + +def test_methodology_norms_documents_tradition_slug() -> None: + """vis-lens-methodology-norms must document tradition_slug in context file handling.""" + text = _read("methodology-norms") + assert "tradition_slug" in text + + +def test_methodology_norms_documents_two_stage_matching() -> None: + """vis-lens-methodology-norms must document Stage A and Stage B two-stage matching.""" + text = _read("methodology-norms") + assert "Stage A" in text or "stage A" in text + assert "Stage B" in text or "stage B" in text + assert "venue_specific_appendices" in text or "venue appendix" in text diff --git a/tests/skills_extended/CLAUDE.md b/tests/skills_extended/CLAUDE.md new file mode 100644 index 0000000000..7acc5b354a --- /dev/null +++ b/tests/skills_extended/CLAUDE.md @@ -0,0 +1,10 @@ +# skills_extended/ + +Extended (Tier 2/3) skill tests. + +## Files + +| File | Purpose | +|------|---------| +| `__init__.py` | empty | +| `test_bundle_local_report.py` | Tests for the bundle-local-report skill renderer | diff --git a/tests/test_conftest.py b/tests/test_conftest.py index b291e1ccc5..0a2f5c8cbe 100644 --- a/tests/test_conftest.py +++ b/tests/test_conftest.py @@ -6,11 +6,11 @@ def test_tool_ctx_provides_isolated_gate(tool_ctx): - """tool_ctx fixture provides a ToolContext with gate enabled.""" + """tool_ctx fixture provides a ToolContext with gate closed (matching production).""" from autoskillit.pipeline.gate import DefaultGateState assert isinstance(tool_ctx.gate, DefaultGateState) - assert tool_ctx.gate.enabled is True + assert tool_ctx.gate.enabled is False def test_tool_ctx_provides_isolated_audit(tool_ctx): @@ -25,7 +25,7 @@ def test_tool_ctx_provides_isolated_token_log(tool_ctx): async def test_mock_subprocess_runner_push_and_pop(tmp_path: Path): """MockSubprocessRunner.push() queues results, __call__ pops in order.""" - from tests.conftest import MockSubprocessRunner + from tests.fakes import MockSubprocessRunner runner = MockSubprocessRunner() r1 = SubprocessResult(0, "out1", "", TerminationReason.NATURAL_EXIT, 100) @@ -41,7 +41,7 @@ async def test_mock_subprocess_runner_push_and_pop(tmp_path: Path): async def test_mock_subprocess_runner_default_when_empty(tmp_path: Path): """MockSubprocessRunner returns a zero-exit default when queue is empty.""" - from tests.conftest import MockSubprocessRunner + from tests.fakes import MockSubprocessRunner runner = MockSubprocessRunner() result = await runner(["cmd"], cwd=tmp_path, timeout=30.0) @@ -99,7 +99,7 @@ def test_structlog_does_not_write_to_stdout_in_tests(capsys): writes to sys.stdout. The autouse _structlog_to_null fixture must intercept all log output before it reaches stdout. """ - from autoskillit.execution.quota import _log as quota_log + from autoskillit.execution.quota import logger as quota_log quota_log.warning("test_sentinel_should_not_reach_stdout", probe=True) captured = capsys.readouterr() @@ -133,3 +133,210 @@ def test_tool_ctx_log_dir_is_isolated_from_production(tool_ctx): xdg = os.environ.get("XDG_DATA_HOME") or os.path.expanduser("~/.local/share") production_path = os.path.join(xdg, "autoskillit", "logs") assert not os.path.abspath(log_dir).startswith(production_path) + + +def test_minimal_ctx_imports_only_core_pipeline_and_config(): + """minimal_ctx fixture must only import from autoskillit.core, .pipeline, and .config.""" + import ast + from pathlib import Path + + conftest_path = Path(__file__).parent / "conftest.py" + tree = ast.parse(conftest_path.read_text(), filename=str(conftest_path)) + + func = None + for node in ast.walk(tree): + if isinstance(node, ast.FunctionDef) and node.name == "minimal_ctx": + func = node + break + assert func is not None, "minimal_ctx fixture not found in conftest.py" + + ALLOWED_PREFIXES = ("autoskillit.core", "autoskillit.pipeline", "autoskillit.config") + + violations = [] + for node in ast.walk(func): + if ( + isinstance(node, ast.ImportFrom) + and node.module + and node.module.startswith("autoskillit.") + ): + if not any(node.module.startswith(p) for p in ALLOWED_PREFIXES): + violations.append(node.module) + + assert not violations, ( + f"minimal_ctx imports from forbidden modules: {violations}. " + f"Only autoskillit.core, autoskillit.pipeline, and autoskillit.config are allowed." + ) + + +def test_minimal_ctx_has_no_server_factory_dependency(): + """minimal_ctx must not import from autoskillit.server or reference make_context.""" + import ast + from pathlib import Path + + conftest_path = Path(__file__).parent / "conftest.py" + tree = ast.parse(conftest_path.read_text(), filename=str(conftest_path)) + + func = None + for node in ast.walk(tree): + if isinstance(node, ast.FunctionDef) and node.name == "minimal_ctx": + func = node + break + assert func is not None, "minimal_ctx fixture not found in conftest.py" + + for node in ast.walk(func): + if isinstance(node, ast.ImportFrom) and node.module: + assert not node.module.startswith("autoskillit.server"), ( + f"minimal_ctx imports from server module: {node.module}" + ) + names = [alias.name for alias in node.names] + assert "make_context" not in names, ( + "minimal_ctx imports make_context — use direct ToolContext construction" + ) + + +def test_clear_headless_env_no_server_import(): + """_clear_headless_env must not import from autoskillit.server.""" + import ast + from pathlib import Path + + conftest_path = Path(__file__).parent / "conftest.py" + tree = ast.parse(conftest_path.read_text(), filename=str(conftest_path)) + + func = None + for node in ast.walk(tree): + if isinstance(node, ast.FunctionDef) and node.name == "_clear_headless_env": + func = node + break + assert func is not None, "_clear_headless_env fixture not found in conftest.py" + + for node in ast.walk(func): + if isinstance(node, ast.ImportFrom) and node.module: + assert not node.module.startswith("autoskillit.server"), ( + f"_clear_headless_env imports from server module: {node.module}. " + f"MCP tag resets belong in tests/server/conftest.py." + ) + + +def test_minimal_ctx_provides_isolated_gate(minimal_ctx): + """minimal_ctx fixture provides a ToolContext with gate closed (matching production).""" + from autoskillit.pipeline.gate import DefaultGateState + + assert isinstance(minimal_ctx.gate, DefaultGateState) + assert minimal_ctx.gate.enabled is False + + +def test_is_test_feature_enabled_reads_project_config(monkeypatch): + """When AUTOSKILLIT_TEST_FEATURES is unset, fleet resolves True via experimental_enabled.""" + monkeypatch.delenv("AUTOSKILLIT_TEST_FEATURES", raising=False) + from tests.conftest import _is_test_feature_enabled, _resolve_test_config + + _resolve_test_config.cache_clear() + try: + result = _is_test_feature_enabled("fleet", env_val=None) + assert result is True + finally: + _resolve_test_config.cache_clear() + + +def test_is_test_feature_enabled_dynaconf_env_overrides(monkeypatch): + """AUTOSKILLIT_FEATURES__FLEET=false overrides experimental_enabled in test resolution.""" + monkeypatch.delenv("AUTOSKILLIT_TEST_FEATURES", raising=False) + monkeypatch.setenv("AUTOSKILLIT_FEATURES__FLEET", "false") + from tests.conftest import _is_test_feature_enabled, _resolve_test_config + + _resolve_test_config.cache_clear() + try: + result = _is_test_feature_enabled("fleet", env_val=None) + assert result is False + finally: + _resolve_test_config.cache_clear() + + +def test_is_test_feature_enabled_respects_experimental_enabled(monkeypatch): + """EXPERIMENTAL feature resolves True via experimental_enabled=True in config.""" + import autoskillit.core.types._type_constants as tc + from autoskillit.core.types._type_constants import FeatureDef + from autoskillit.core.types._type_enums import FeatureLifecycle + from tests.conftest import _is_test_feature_enabled, _resolve_test_config + + monkeypatch.delenv("AUTOSKILLIT_TEST_FEATURES", raising=False) + exp_feat = FeatureDef( + lifecycle=FeatureLifecycle.EXPERIMENTAL, + description="test", + tool_tags=frozenset(), + skill_categories=frozenset(), + import_package=None, + default_enabled=False, + ) + monkeypatch.setitem(tc.FEATURE_REGISTRY, "conftest_test_exp", exp_feat) + _resolve_test_config.cache_clear() + try: + # defaults.yaml has experimental_enabled=true, so EXPERIMENTAL features are enabled + result = _is_test_feature_enabled("conftest_test_exp", env_val=None) + assert result is True + finally: + _resolve_test_config.cache_clear() + + +def test_is_test_feature_enabled_disabled_lifecycle_always_false(monkeypatch): + """_is_test_feature_enabled returns False for DISABLED feature regardless of config.""" + import autoskillit.core.types._type_constants as tc + from autoskillit.core.types._type_constants import FeatureDef + from autoskillit.core.types._type_enums import FeatureLifecycle + from tests.conftest import _is_test_feature_enabled, _resolve_test_config + + monkeypatch.delenv("AUTOSKILLIT_TEST_FEATURES", raising=False) + disabled_feat = FeatureDef( + lifecycle=FeatureLifecycle.DISABLED, + description="disabled test", + tool_tags=frozenset(), + skill_categories=frozenset(), + import_package=None, + ) + monkeypatch.setitem(tc.FEATURE_REGISTRY, "conftest_test_disabled", disabled_feat) + _resolve_test_config.cache_clear() + try: + result = _is_test_feature_enabled("conftest_test_disabled", env_val=None) + assert result is False + finally: + _resolve_test_config.cache_clear() + + +def test_no_per_test_config_cache_clear_fixture(): + """The function-scoped autouse fixture that cleared _resolve_test_config cache + after every test has been removed. Verify it no longer exists in conftest.""" + import ast + from pathlib import Path + + import pytest + + conftest_path = Path(__file__).resolve().parent / "conftest.py" + assert conftest_path.exists(), f"conftest.py not found at {conftest_path}" + tree = ast.parse(conftest_path.read_text()) + for node in ast.walk(tree): + if isinstance(node, ast.FunctionDef) and node.name == "_clear_resolve_test_config_cache": + for decorator in node.decorator_list: + if isinstance(decorator, ast.Call): + for kw in decorator.keywords: + if kw.arg == "scope" and isinstance(kw.value, ast.Constant): + if kw.value.value == "session": + return # session-scoped is ok + pytest.fail( + "_clear_resolve_test_config_cache still exists as a non-session fixture. " + "It should be removed to stop per-test cache thrash." + ) + + +def test_resolve_test_config_cache_persists_across_calls(): + """_resolve_test_config LRU cache should persist — calling it twice returns the same object.""" + from tests.conftest import _resolve_test_config + + _resolve_test_config.cache_clear() + try: + result1 = _resolve_test_config() + result2 = _resolve_test_config() + assert result1 is result2, "LRU cache should return the same object on repeated calls" + info = _resolve_test_config.cache_info() + assert info.hits >= 1, "Expected at least 1 cache hit" + finally: + _resolve_test_config.cache_clear() diff --git a/tests/test_fakes_conformance.py b/tests/test_fakes_conformance.py new file mode 100644 index 0000000000..a1f44de626 --- /dev/null +++ b/tests/test_fakes_conformance.py @@ -0,0 +1,331 @@ +"""Protocol conformance and behavioral tests for tests/fakes.py.""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from autoskillit.core.types import ( + CIWatcher, + DatabaseReader, + HeadlessExecutor, + MergeQueueWatcher, + RecipeRepository, + SkillResult, + SubprocessRunner, + TestResult, + TestRunner, +) +from tests.fakes import ( + InMemoryCIWatcher, + InMemoryDatabaseReader, + InMemoryHeadlessExecutor, + InMemoryMergeQueueWatcher, + InMemoryRecipeRepository, + InMemoryTestRunner, + MockSubprocessRunner, +) + +# --------------------------------------------------------------------------- +# T1: isinstance protocol conformance +# --------------------------------------------------------------------------- + + +def test_in_memory_headless_executor_satisfies_protocol(): + assert isinstance(InMemoryHeadlessExecutor(), HeadlessExecutor) + + +def test_in_memory_test_runner_satisfies_protocol(): + assert isinstance(InMemoryTestRunner(), TestRunner) + + +def test_in_memory_recipe_repository_satisfies_protocol(): + assert isinstance(InMemoryRecipeRepository(), RecipeRepository) + + +def test_in_memory_ci_watcher_satisfies_protocol(): + assert isinstance(InMemoryCIWatcher(), CIWatcher) + + +def test_in_memory_merge_queue_watcher_satisfies_protocol(): + assert isinstance(InMemoryMergeQueueWatcher(), MergeQueueWatcher) + + +def test_in_memory_database_reader_satisfies_protocol(): + assert isinstance(InMemoryDatabaseReader(), DatabaseReader) + + +def test_mock_subprocess_runner_satisfies_protocol(): + assert isinstance(MockSubprocessRunner(), SubprocessRunner) + + +# --------------------------------------------------------------------------- +# T2: InMemoryHeadlessExecutor behavioral tests +# --------------------------------------------------------------------------- + + +@pytest.mark.anyio +async def test_executor_returns_configured_result(): + result = SkillResult( + success=True, + result="ok", + session_id="s1", + subtype="success", + is_error=False, + exit_code=0, + needs_retry=False, + retry_reason="none", + stderr="", + ) + executor = InMemoryHeadlessExecutor(default_result=result) + got = await executor.run("/skill", "/cwd") + assert got == result + + +@pytest.mark.anyio +async def test_executor_records_calls(): + executor = InMemoryHeadlessExecutor() + await executor.run("/skill", "/cwd", model="opus") + assert len(executor.calls) == 1 + assert executor.calls[0].skill_command == "/skill" + assert executor.calls[0].cwd == "/cwd" + assert executor.calls[0].model == "opus" + + +@pytest.mark.anyio +async def test_executor_pops_from_queue(): + r1 = SkillResult( + success=True, + result="first", + session_id="", + subtype="success", + is_error=False, + exit_code=0, + needs_retry=False, + retry_reason="none", + stderr="", + ) + r2 = SkillResult( + success=False, + result="second", + session_id="", + subtype="error", + is_error=True, + exit_code=1, + needs_retry=False, + retry_reason="none", + stderr="", + ) + executor = InMemoryHeadlessExecutor() + executor.push(r1) + executor.push(r2) + assert (await executor.run("/s", "/c")).result == "first" + assert (await executor.run("/s", "/c")).result == "second" + + +# --------------------------------------------------------------------------- +# T3: InMemoryTestRunner behavioral tests +# --------------------------------------------------------------------------- + + +@pytest.mark.anyio +async def test_runner_returns_configured_results_in_order(): + runner = InMemoryTestRunner( + results=[ + TestResult(passed=True, stdout="ok", stderr=""), + TestResult(passed=False, stdout="fail", stderr="err"), + ] + ) + assert (await runner.run(Path("/a"))).passed is True + assert (await runner.run(Path("/b"))).passed is False + + +@pytest.mark.anyio +async def test_runner_fallback_after_exhaustion(): + runner = InMemoryTestRunner(results=[]) + r = await runner.run(Path("/a")) + assert r.passed is True + + +@pytest.mark.anyio +async def test_runner_tracks_call_count(): + runner = InMemoryTestRunner(results=[]) + assert runner.call_count == 0 + await runner.run(Path("/a")) + assert runner.call_count == 1 + await runner.run(Path("/b")) + assert runner.call_count == 2 + + +# --------------------------------------------------------------------------- +# T4: InMemoryCIWatcher and InMemoryMergeQueueWatcher behavioral tests +# --------------------------------------------------------------------------- + + +@pytest.mark.anyio +async def test_ci_watcher_returns_configured_wait_result(): + watcher = InMemoryCIWatcher(wait_result={"run_id": 42, "conclusion": "failure"}) + result = await watcher.wait("main") + assert result["run_id"] == 42 + assert result["conclusion"] == "failure" + + +@pytest.mark.anyio +async def test_ci_watcher_records_wait_calls(): + watcher = InMemoryCIWatcher() + await watcher.wait("main", repo="owner/repo") + assert len(watcher.wait_calls) == 1 + assert watcher.wait_calls[0]["branch"] == "main" + assert watcher.wait_calls[0]["repo"] == "owner/repo" + + +@pytest.mark.anyio +async def test_ci_watcher_side_effect_raises(): + watcher = InMemoryCIWatcher() + watcher.wait_side_effect = RuntimeError("network timeout") + with pytest.raises(RuntimeError, match="network timeout"): + await watcher.wait("main") + + +@pytest.mark.anyio +async def test_merge_queue_watcher_records_calls(): + watcher = InMemoryMergeQueueWatcher() + await watcher.wait(123, "main") + assert len(watcher.wait_calls) == 1 + assert watcher.wait_calls[0]["pr_number"] == 123 + assert watcher.wait_calls[0]["target_branch"] == "main" + + +@pytest.mark.anyio +async def test_merge_queue_watcher_toggle(): + watcher = InMemoryMergeQueueWatcher() + result = await watcher.toggle(42, "main") + assert result["toggled"] is True + assert len(watcher.toggle_calls) == 1 + + +@pytest.mark.anyio +async def test_database_reader_returns_configured_result(): + reader = InMemoryDatabaseReader( + query_result={"columns": ["id"], "rows": [[1]], "row_count": 1} + ) + result = reader.query("test.db", "SELECT 1", [], 30, 100) + assert result["row_count"] == 1 + assert len(reader.calls) == 1 + + +# --------------------------------------------------------------------------- +# T5: InMemoryRecipeRepository call recording +# --------------------------------------------------------------------------- + + +def test_recipe_repository_find_records_call(): + repo = InMemoryRecipeRepository() + repo.find("my-recipe", Path("/proj")) + assert len(repo.calls) == 1 + call = repo.calls[0] + assert call["method"] == "find" + assert call["name"] == "my-recipe" + assert call["project_dir"] == Path("/proj") + + +def test_recipe_repository_list_records_call(): + repo = InMemoryRecipeRepository() + repo.list(Path("/proj")) + assert len(repo.calls) == 1 + call = repo.calls[0] + assert call["method"] == "list" + assert call["project_dir"] == Path("/proj") + + +def test_recipe_repository_load_and_validate_records_call(): + repo = InMemoryRecipeRepository() + repo.load_and_validate( + "my-recipe", + Path("/proj"), + suppressed=["rule-a"], + resolved_defaults={"k": "v"}, + ingredient_overrides={"x": "y"}, + temp_dir=Path("/tmp"), + temp_dir_relpath=".autoskillit/temp", + ) + assert len(repo.calls) == 1 + call = repo.calls[0] + assert call["method"] == "load_and_validate" + assert call["name"] == "my-recipe" + assert call["project_dir"] == Path("/proj") + assert call["suppressed"] == ["rule-a"] + assert call["resolved_defaults"] == {"k": "v"} + assert call["ingredient_overrides"] == {"x": "y"} + assert call["temp_dir"] == Path("/tmp") + assert call["temp_dir_relpath"] == ".autoskillit/temp" + + +def test_recipe_repository_list_all_records_call(): + repo = InMemoryRecipeRepository() + repo.list_all(project_dir=Path("/proj")) + assert len(repo.calls) == 1 + call = repo.calls[0] + assert call["method"] == "list_all" + assert call["project_dir"] == Path("/proj") + + +def test_recipe_repository_list_all_records_call_with_none_project_dir(): + repo = InMemoryRecipeRepository() + repo.list_all() + assert len(repo.calls) == 1 + assert repo.calls[0]["project_dir"] is None + + +def test_recipe_repository_calls_accumulate(): + repo = InMemoryRecipeRepository() + repo.find("r1", Path("/a")) + repo.list_all() + repo.load_and_validate("r1", Path("/a")) + assert len(repo.calls) == 3 + assert repo.calls[0]["method"] == "find" + assert repo.calls[1]["method"] == "list_all" + assert repo.calls[2]["method"] == "load_and_validate" + + +def test_recipe_repository_calls_starts_empty(): + repo = InMemoryRecipeRepository() + assert repo.calls == [] + + +def test_recipe_repository_validate_from_path_records_call(): + repo = InMemoryRecipeRepository() + script_path = Path("/proj/recipe.yaml") + repo.validate_from_path(script_path, ".autoskillit/temp") + assert len(repo.calls) == 1 + call = repo.calls[0] + assert call["method"] == "validate_from_path" + assert call["script_path"] == script_path + assert call["temp_dir_relpath"] == ".autoskillit/temp" + + +def test_recipe_repository_validate_from_path_records_call_default_relpath(): + repo = InMemoryRecipeRepository() + script_path = Path("/proj/recipe.yaml") + repo.validate_from_path(script_path) + assert len(repo.calls) == 1 + call = repo.calls[0] + assert call["method"] == "validate_from_path" + assert call["script_path"] == script_path + assert call["temp_dir_relpath"] == ".autoskillit/temp" + + +# --------------------------------------------------------------------------- +# T5: InMemoryHeadlessExecutor accepts resume_session_id (conformance test) +# --------------------------------------------------------------------------- + + +@pytest.mark.anyio +async def test_in_memory_executor_accepts_resume_session_id(): + """InMemoryHeadlessExecutor.run() must accept resume_session_id kwarg.""" + executor = InMemoryHeadlessExecutor() + # Must not raise TypeError + await executor.run("/implement foo", "/tmp", resume_session_id="sess-123") + assert len(executor.calls) == 1 + assert executor.calls[0].resume_session_id == "sess-123" diff --git a/tests/test_hook_registry.py b/tests/test_hook_registry.py deleted file mode 100644 index d973065c09..0000000000 --- a/tests/test_hook_registry.py +++ /dev/null @@ -1,187 +0,0 @@ -"""Tests for hook_registry.py — L0 hook identity model.""" - -from __future__ import annotations - -import json -from pathlib import Path - -from autoskillit.hook_registry import ( - HOOK_REGISTRY, - HOOKS_DIR, - _extract_script_basenames, - _is_own_hook, - canonical_script_basenames, - find_broken_hook_scripts, - generate_hooks_json, -) - - -# HR-FILTER-1: command with "autoskillit" substring -> True -def test_is_own_hook_autoskillit_substring() -> None: - assert _is_own_hook("python3 /path/to/autoskillit/hooks/quota_guard.py") is True - - -# HR-FILTER-2: command ending with known script basename -> True -def test_is_own_hook_known_basename() -> None: - assert _is_own_hook("python3 /some/other/path/quota_guard.py") is True - - -# HR-FILTER-3: unrelated command -> False -def test_is_own_hook_unrelated_command() -> None: - assert _is_own_hook('wsl-notify-send.exe "Done!"') is False - assert _is_own_hook("python3 /home/user/my_guard.py") is False - - -# HR-FILTER-4: known basename with different path prefix -> True -def test_is_own_hook_different_prefix() -> None: - assert ( - _is_own_hook("python3 /home/user/.local/share/uv/tools/lib/hooks/skill_cmd_guard.py") - is True - ) - - -# HR-BASENAME-1: canonical hooks dict -> returns set of bare filenames -def test_extract_script_basenames_canonical() -> None: - canonical = generate_hooks_json() - result = _extract_script_basenames(canonical.get("hooks", {})) - expected = canonical_script_basenames() - assert result == expected - - -# HR-BASENAME-2: different path prefixes -> returns same basenames -def test_extract_script_basenames_different_prefix() -> None: - foreign_dir = ( - "/home/user/.local/share/uv/tools/autoskillit/lib/python3.13" - "/site-packages/autoskillit/hooks" - ) - by_event: dict[str, list[dict]] = {} - for hdef in HOOK_REGISTRY: - hook_commands = [ - {"type": "command", "command": f"python3 {foreign_dir}/{script}"} - for script in hdef.scripts - ] - entry: dict = {"hooks": hook_commands} - if hdef.event_type != "SessionStart": - entry["matcher"] = hdef.matcher - by_event.setdefault(hdef.event_type, []).append(entry) - - result = _extract_script_basenames(by_event) - assert result == canonical_script_basenames() - - -# HR-BASENAME-3: mixed autoskillit + user hooks -> returns only autoskillit basenames -def test_extract_script_basenames_filters_user_hooks() -> None: - canonical = generate_hooks_json() - hooks = canonical["hooks"] - hooks.setdefault("PreToolUse", []).append( - { - "matcher": ".*", - "hooks": [ - {"type": "command", "command": "python3 /home/user/my_guard.py"}, - {"type": "command", "command": 'wsl-notify-send.exe "Done!"'}, - ], - } - ) - result = _extract_script_basenames(hooks) - assert result == canonical_script_basenames() - assert "my_guard.py" not in result - - -# --------------------------------------------------------------------------- -# T7 — Hook file renames: new names exist, old names removed -# --------------------------------------------------------------------------- - -_HOOKS_DIR = Path(__file__).parent.parent / "src" / "autoskillit" / "hooks" - - -def test_renamed_hook_files_exist() -> None: - """New hook filenames must exist on disk after rename.""" - for expected in [ - "quota_guard.py", - "quota_post_hook.py", - "skill_cmd_guard.py", - "pretty_output_hook.py", - "session_start_hook.py", - "token_summary_hook.py", - ]: - assert (_HOOKS_DIR / expected).exists(), f"Missing renamed hook: {expected}" - - -def test_old_hook_names_removed() -> None: - """Old hook filenames must NOT exist on disk after rename.""" - for old in [ - "quota_check.py", - "quota_post_check.py", - "skill_cmd_check.py", - "pretty_output.py", - "session_start_reminder.py", - "token_summary_appender.py", - ]: - assert not (_HOOKS_DIR / old).exists(), f"Old hook file still present: {old}" - - -def test_hook_registry_uses_new_filenames() -> None: - """HOOK_REGISTRY must reference only the new filenames, not the old ones.""" - all_scripts = {s for h in HOOK_REGISTRY for s in h.scripts} - old_names = { - "quota_check.py", - "quota_post_check.py", - "skill_cmd_check.py", - "pretty_output.py", - "session_start_reminder.py", - "token_summary_appender.py", - } - for old in old_names: - assert old not in all_scripts, f"HOOK_REGISTRY still references old name: {old}" - - -def test_find_broken_hook_scripts_ignores_user_inline_hooks(tmp_path: Path) -> None: - """User inline hooks like 'wsl-notify-send.exe Done' must not be flagged as broken.""" - settings = tmp_path / "settings.json" - settings.write_text( - json.dumps( - { - "hooks": { - "PostToolUse": [{"hooks": [{"command": 'wsl-notify-send.exe "Build done"'}]}] - } - } - ) - ) - broken = find_broken_hook_scripts(settings) - assert broken == [] - - -def test_find_broken_hook_scripts_flags_missing_autoskillit_script(tmp_path: Path) -> None: - """A python3 /abs/path/autoskillit/hooks/script.py that doesn't exist must be flagged.""" - missing = str(HOOKS_DIR / "nonexistent_guard.py") - settings = tmp_path / "settings.json" - settings.write_text( - json.dumps( - { - "hooks": { - "PreToolUse": [{"matcher": ".*", "hooks": [{"command": f"python3 {missing}"}]}] - } - } - ) - ) - broken = find_broken_hook_scripts(settings) - assert any("nonexistent_guard.py" in b for b in broken) - - -def test_find_broken_hook_scripts_does_not_flag_user_python_scripts(tmp_path: Path) -> None: - """A user's python3 script outside the autoskillit hooks dir must not be flagged.""" - user_script = "/home/user/my_custom_guard.py" - settings = tmp_path / "settings.json" - settings.write_text( - json.dumps( - { - "hooks": { - "PreToolUse": [ - {"matcher": ".*", "hooks": [{"command": f"python3 {user_script}"}]} - ] - } - } - ) - ) - broken = find_broken_hook_scripts(settings) - assert broken == [] diff --git a/tests/test_no_orchestration_tier_language.py b/tests/test_no_orchestration_tier_language.py new file mode 100644 index 0000000000..b8b581a78e --- /dev/null +++ b/tests/test_no_orchestration_tier_language.py @@ -0,0 +1,39 @@ +"""Guard: legacy orchestration-tier language must not appear in critical source files.""" + +from __future__ import annotations + +import pathlib + +import pytest + +pytestmark = [pytest.mark.layer("core"), pytest.mark.small] + +_REPO_ROOT = pathlib.Path(__file__).parent.parent + +_ORCHESTRATION_FILES = [ + "src/autoskillit/core/types/_type_enums.py", + "src/autoskillit/pipeline/gate.py", + "src/autoskillit/server/_guards.py", + "src/autoskillit/hooks/guards/skill_orchestration_guard.py", +] + +_FORBIDDEN = [ + "Tier 1 session", + "Tier 2 worker", + "mid-tier", + "bottom-tier", + "Tier discriminator", + "Tier 2) may", + "leaf-tier", + "Tier invariant", + "orchestrator-tier", + "fleet-tier", +] + + +def test_no_legacy_orchestration_tier_language_in_source(): + """Old ad-hoc tier language must be absent from orchestration-critical files.""" + for rel in _ORCHESTRATION_FILES: + text = (_REPO_ROOT / rel).read_text() + for phrase in _FORBIDDEN: + assert phrase not in text, f"Legacy tier phrase {phrase!r} found in {rel}" diff --git a/tests/test_smoke_utils.py b/tests/test_smoke_utils.py index b3b0077dc1..42ebc7806d 100644 --- a/tests/test_smoke_utils.py +++ b/tests/test_smoke_utils.py @@ -1,12 +1,21 @@ -"""Tests for smoke_utils — check_bug_report_non_empty callable.""" +"""Tests for smoke_utils callables.""" from __future__ import annotations -import ast import json +import subprocess from pathlib import Path +from unittest.mock import patch -from autoskillit.smoke_utils import check_bug_report_non_empty +from autoskillit.smoke_utils import ( + annotate_pr_diff, + check_bug_report_non_empty, + check_loop_iteration, + check_review_loop, + enrich_diff_context, + patch_pr_token_summary, +) +from tests.infra._token_summary_helpers import _resolve_session_label # T_SU1 @@ -40,8 +49,244 @@ def test_returns_false_when_bug_report_malformed(tmp_path: Path) -> None: assert result == {"non_empty": "false"} +# --------------------------------------------------------------------------- +# T_CRL6–T_CRL8: check_review_loop tests (pure iteration guard) +# --------------------------------------------------------------------------- + + +# T_CRL6 +def test_crl_next_iteration_increments() -> None: + """next_iteration increments from current_iteration: "" → "1", "1" → "2", "2" → "3".""" + r1 = check_review_loop("1", current_iteration="") + assert r1["next_iteration"] == "1" + + r2 = check_review_loop("1", current_iteration="1") + assert r2["next_iteration"] == "2" + + r3 = check_review_loop("1", current_iteration="2") + assert r3["next_iteration"] == "3" + + +# T_CRL7 +def test_crl_max_exceeded_when_next_iteration_ge_max() -> None: + """max_exceeded=true when next_iteration >= max_iterations.""" + result = check_review_loop("1", current_iteration="2", max_iterations="3") + assert result["max_exceeded"] == "true" + assert result["next_iteration"] == "3" + + +# T_CRL8 +def test_crl_max_not_exceeded_when_below_max() -> None: + """max_exceeded=false when next_iteration < max_iterations.""" + result = check_review_loop("1", current_iteration="1", max_iterations="3") + assert result["max_exceeded"] == "false" + + +def test_check_review_loop_always_continues_when_iterations_remain() -> None: + """After a resolve cycle, check_review_loop must indicate continuation + when max_iterations is not exceeded — regardless of GitHub thread state. + + The function is a pure iteration guard: if next_iteration < max_iterations, + it must return max_exceeded=false so the recipe routes back to review_pr. + """ + result = check_review_loop( + pr_number="42", + current_iteration="0", + max_iterations="3", + ) + assert result["max_exceeded"] == "false" + assert result["next_iteration"] == "1" + + +def test_check_review_loop_stops_at_max_iterations() -> None: + """When current_iteration reaches max_iterations, max_exceeded must be true.""" + result = check_review_loop( + pr_number="42", + current_iteration="2", + max_iterations="3", + ) + assert result["max_exceeded"] == "true" + assert result["next_iteration"] == "3" + + +def test_check_review_loop_returns_expected_fields() -> None: + """check_review_loop must return next_iteration, max_exceeded, and had_blocking.""" + result = check_review_loop(pr_number="42") + assert set(result.keys()) == {"next_iteration", "max_exceeded", "had_blocking"} + + +# T_CRL11 — verify check_review_loop has no subprocess calls +def test_check_review_loop_has_no_subprocess_calls() -> None: + """The simplified check_review_loop must not use subprocess at all.""" + import ast + + src = Path("src/autoskillit/smoke_utils.py").read_text() + tree = ast.parse(src) + + # Find the check_review_loop function node + for node in ast.walk(tree): + if isinstance(node, ast.FunctionDef) and node.name == "check_review_loop": + for child in ast.walk(node): + if isinstance(child, ast.Attribute) and child.attr == "run": + if isinstance(child.value, ast.Name) and child.value.id == "subprocess": + raise AssertionError( + "check_review_loop should not use subprocess.run() — " + "it is a pure iteration guard" + ) + break + + +# T_CRL12 +def test_crl_had_blocking_true_when_changes_requested() -> None: + """had_blocking=true when previous_verdict is changes_requested.""" + result = check_review_loop("42", previous_verdict="changes_requested") + assert result["had_blocking"] == "true" + + +# T_CRL13 +def test_crl_had_blocking_false_when_approved_with_comments() -> None: + """had_blocking=false when previous_verdict is approved_with_comments.""" + result = check_review_loop("42", previous_verdict="approved_with_comments") + assert result["had_blocking"] == "false" + + +# T_CRL14 +def test_crl_had_blocking_false_when_empty_verdict() -> None: + """had_blocking=false when previous_verdict is absent (first-pass guard).""" + result = check_review_loop("42") + assert result["had_blocking"] == "false" + + +# --------------------------------------------------------------------------- +# T_CRL15–T_CRL18: check_review_loop with local_review_rounds parameter +# --------------------------------------------------------------------------- + + +def test_crl_local_rounds_not_exhausted_approved_is_blocking() -> None: + """When local_review_rounds > 0 and iteration < local_rounds, approved is blocking. + + The approved verdict must trigger re-review until local_review_rounds are + exhausted, so review_loop_count advances on every local round. + """ + result = check_review_loop( + pr_number="42", + current_iteration="0", + max_iterations="6", + previous_verdict="approved", + local_review_rounds="3", + ) + assert result["had_blocking"] == "true" + assert result["next_iteration"] == "1" + assert result["max_exceeded"] == "false" + + +def test_crl_local_rounds_exhausted_approved_is_non_blocking() -> None: + """When iteration >= local_review_rounds, approved is non-blocking. + + Once local rounds are exhausted, approved exits to CI immediately. + """ + result = check_review_loop( + pr_number="42", + current_iteration="3", + max_iterations="6", + previous_verdict="approved", + local_review_rounds="3", + ) + assert result["had_blocking"] == "false" + assert result["next_iteration"] == "4" + + +def test_crl_changes_requested_always_blocking_regardless_of_local_rounds() -> None: + """changes_requested is always blocking, even after local rounds exhausted.""" + result = check_review_loop( + pr_number="42", + current_iteration="5", + max_iterations="6", + previous_verdict="changes_requested", + local_review_rounds="3", + ) + assert result["had_blocking"] == "true" + + +def test_crl_no_local_rounds_approved_is_non_blocking() -> None: + """When local_review_rounds is absent or zero, approved is non-blocking as before. + + This preserves backward compatibility: without local_review_rounds configured, + the only blocking verdict is changes_requested. + """ + result = check_review_loop( + pr_number="42", + current_iteration="0", + max_iterations="3", + previous_verdict="approved", + local_review_rounds="", + ) + assert result["had_blocking"] == "false" + + +def test_crl_local_rounds_zero_approved_is_non_blocking() -> None: + """local_review_rounds="0" means no local rounds, approved is non-blocking.""" + result = check_review_loop( + pr_number="42", + current_iteration="0", + max_iterations="3", + previous_verdict="approved", + local_review_rounds="0", + ) + assert result["had_blocking"] == "false" + + +def test_crl_needs_human_non_blocking_when_local_rounds_exhausted() -> None: + """needs_human is non-blocking when local rounds are exhausted.""" + result = check_review_loop( + pr_number="42", + current_iteration="3", + max_iterations="6", + previous_verdict="needs_human", + local_review_rounds="3", + ) + assert result["had_blocking"] == "false" + + +# --------------------------------------------------------------------------- +# T_SU_LI1–T_SU_LI5: check_loop_iteration tests (generic loop iteration guard) +# --------------------------------------------------------------------------- + + +def test_check_loop_iteration_first_call() -> None: + """First iteration (empty string) → next=1, max_exceeded=false for max=2.""" + result = check_loop_iteration(current_iteration="", max_iterations="2") + assert result == {"next_iteration": "1", "max_exceeded": "false"} + + +def test_check_loop_iteration_at_budget() -> None: + """iteration=1, max=2 → next=2, max_exceeded=true.""" + result = check_loop_iteration(current_iteration="1", max_iterations="2") + assert result == {"next_iteration": "2", "max_exceeded": "true"} + + +def test_check_loop_iteration_over_budget() -> None: + """iteration=5, max=2 → max_exceeded=true.""" + result = check_loop_iteration(current_iteration="5", max_iterations="2") + assert result == {"next_iteration": "6", "max_exceeded": "true"} + + +def test_check_loop_iteration_custom_max() -> None: + """iteration=3, max=5 → next=4, max_exceeded=false.""" + result = check_loop_iteration(current_iteration="3", max_iterations="5") + assert result == {"next_iteration": "4", "max_exceeded": "false"} + + +def test_check_loop_iteration_defaults() -> None: + """No arguments → iteration=0, max=2 → next=1, max_exceeded=false.""" + result = check_loop_iteration() + assert result == {"next_iteration": "1", "max_exceeded": "false"} + + def test_subprocess_calls_have_timeout() -> None: """All subprocess.run() calls in smoke_utils.py must have a timeout= argument.""" + import ast + src = Path("src/autoskillit/smoke_utils.py").read_text() tree = ast.parse(src) for node in ast.walk(tree): @@ -54,3 +299,613 @@ def test_subprocess_calls_have_timeout() -> None: assert "timeout" in kw_names, ( f"subprocess.run() at line {node.lineno} in smoke_utils.py missing timeout=" ) + + +# --------------------------------------------------------------------------- +# T_PTS1–T_PTS7: patch_pr_token_summary tests +# --------------------------------------------------------------------------- + +PR_URL = "https://github.com/TestOwner/TestRepo/pull/42" + + +def _write_test_sessions(log_root: Path, entries: list[dict]) -> None: + lines = [] + for entry in entries: + index_entry = { + "dir_name": entry["dir_name"], + "cwd": entry.get("cwd", ""), + "kitchen_id": entry.get("kitchen_id", ""), + "order_id": entry.get("order_id", ""), + "timestamp": entry.get("timestamp", "2026-01-01T00:00:00+00:00"), + } + lines.append(json.dumps(index_entry)) + session_dir = log_root / "sessions" / entry["dir_name"] + session_dir.mkdir(parents=True, exist_ok=True) + token_data = { + "session_label": _resolve_session_label(entry), + "input_tokens": entry.get("input_tokens", 1000), + "output_tokens": entry.get("output_tokens", 500), + "cache_creation_input_tokens": entry.get("cache_creation_input_tokens", 100), + "cache_read_input_tokens": entry.get("cache_read_input_tokens", 200), + "timing_seconds": entry.get("timing_seconds", 10.0), + "order_id": entry.get("order_id", ""), + "loc_insertions": entry.get("loc_insertions", 0), + "loc_deletions": entry.get("loc_deletions", 0), + } + (session_dir / "token_usage.json").write_text(json.dumps(token_data)) + (log_root / "sessions.jsonl").write_text("\n".join(lines) + "\n") + + +def _make_gh_mock(get_body: str = "", get_rc: int = 0, patch_rc: int = 0): + def _mock_run(cmd, **_kwargs): + if "--method" in cmd and "PATCH" in cmd: + return subprocess.CompletedProcess( + cmd, patch_rc, "", "" if patch_rc == 0 else "patch error" + ) + return subprocess.CompletedProcess( + cmd, get_rc, get_body if get_rc == 0 else "", "" if get_rc == 0 else "read error" + ) + + return _mock_run + + +# T_PTS1 +@patch("time.sleep") +@patch("subprocess.run") +def test_pts_happy_path_appends_table(mock_run, _mock_sleep, tmp_path: Path) -> None: + cwd = "/clone/test" + _write_test_sessions( + tmp_path, + [ + { + "dir_name": "s1", + "cwd": cwd, + "step_name": "plan", + "input_tokens": 1000, + "output_tokens": 500, + }, + { + "dir_name": "s2", + "cwd": cwd, + "step_name": "implement", + "input_tokens": 2000, + "output_tokens": 1000, + }, + { + "dir_name": "s3", + "cwd": cwd, + "step_name": "compose_pr", + "input_tokens": 500, + "output_tokens": 250, + }, + ], + ) + mock_run.side_effect = _make_gh_mock(get_body="## Summary\nSome PR body") + result = patch_pr_token_summary(PR_URL, cwd, log_dir=str(tmp_path)) + assert result["success"] == "true" + assert result["sessions_loaded"] == "3" + patch_call = mock_run.call_args_list[-1] + body_arg = [a for a in patch_call[0][0] if a.startswith("body=")][0] + assert "## Token Usage Summary" in body_arg + assert "plan" in body_arg + assert "implement" in body_arg + assert "compose_pr" in body_arg + + +# T_PTS2 +@patch("time.sleep") +@patch("subprocess.run") +def test_pts_replaces_existing_partial_table(mock_run, _mock_sleep, tmp_path: Path) -> None: + cwd = "/clone/test" + _write_test_sessions( + tmp_path, + [ + { + "dir_name": "s1", + "cwd": cwd, + "step_name": "plan", + "input_tokens": 1000, + "output_tokens": 500, + }, + { + "dir_name": "s2", + "cwd": cwd, + "step_name": "implement", + "input_tokens": 2000, + "output_tokens": 1000, + }, + { + "dir_name": "s3", + "cwd": cwd, + "step_name": "compose_pr", + "input_tokens": 500, + "output_tokens": 250, + }, + ], + ) + existing_body = ( + "## Summary\nSome text\n\n## Token Usage Summary\n\n" + "| Step | old partial table |\n| compose_pr | 500 |" + ) + mock_run.side_effect = _make_gh_mock(get_body=existing_body) + result = patch_pr_token_summary(PR_URL, cwd, log_dir=str(tmp_path)) + assert result["success"] == "true" + patch_call = mock_run.call_args_list[-1] + body_arg = [a for a in patch_call[0][0] if a.startswith("body=")][0] + assert body_arg.count("## Token Usage Summary") == 1 + assert "plan" in body_arg + + +# T_PTS3 +def test_pts_invalid_pr_url() -> None: + result = patch_pr_token_summary("not-a-url", "/clone/test") + assert result["success"] == "false" + assert "Invalid PR URL" in result["error"] + + +# T_PTS4 +def test_pts_zero_sessions(tmp_path: Path) -> None: + (tmp_path / "sessions.jsonl").write_text("") + result = patch_pr_token_summary(PR_URL, "/clone/test", log_dir=str(tmp_path)) + assert result["success"] == "false" + assert result["sessions_loaded"] == "0" + + +# T_PTS5 +@patch("time.sleep") +@patch("subprocess.run") +def test_pts_cross_kitchen_sessions(mock_run, _mock_sleep, tmp_path: Path) -> None: + cwd = "/clone/test" + entries = [ + { + "dir_name": f"s{i}", + "cwd": cwd, + "kitchen_id": "aaa", + "step_name": f"step_a{i}", + "input_tokens": 100, + "output_tokens": 50, + } + for i in range(3) + ] + [ + { + "dir_name": f"s{i + 3}", + "cwd": cwd, + "kitchen_id": "bbb", + "step_name": f"step_b{i}", + "input_tokens": 100, + "output_tokens": 50, + } + for i in range(3) + ] + _write_test_sessions(tmp_path, entries) + mock_run.side_effect = _make_gh_mock(get_body="## Summary\nBody") + result = patch_pr_token_summary(PR_URL, cwd, log_dir=str(tmp_path)) + assert result["success"] == "true" + assert result["sessions_loaded"] == "6" + + +# T_PTS6 +@patch("subprocess.run") +def test_pts_gh_api_read_failure(mock_run, tmp_path: Path) -> None: + cwd = "/clone/test" + _write_test_sessions( + tmp_path, + [ + {"dir_name": "s1", "cwd": cwd, "step_name": "plan"}, + ], + ) + mock_run.side_effect = _make_gh_mock(get_rc=1) + result = patch_pr_token_summary(PR_URL, cwd, log_dir=str(tmp_path)) + assert result["success"] == "false" + assert "Failed to read PR" in result["error"] + + +# T_PTS7 +@patch("time.sleep") +@patch("subprocess.run") +def test_pts_gh_api_patch_failure(mock_run, _mock_sleep, tmp_path: Path) -> None: + cwd = "/clone/test" + _write_test_sessions( + tmp_path, + [ + {"dir_name": "s1", "cwd": cwd, "step_name": "plan"}, + ], + ) + mock_run.side_effect = _make_gh_mock(get_body="## Summary\nBody", patch_rc=1) + result = patch_pr_token_summary(PR_URL, cwd, log_dir=str(tmp_path)) + assert result["success"] == "false" + assert "Failed to patch PR" in result["error"] + + +# --------------------------------------------------------------------------- +# Null-safety tests (run_python None-input coercion) +# --------------------------------------------------------------------------- + + +def test_check_loop_iteration_none_current() -> None: + result = check_loop_iteration(current_iteration=None) # type: ignore[arg-type] + assert result["next_iteration"] == "1" + assert result["max_exceeded"] == "false" + + +def test_check_loop_iteration_none_max() -> None: + result = check_loop_iteration(current_iteration="0", max_iterations=None) # type: ignore[arg-type] + assert result["next_iteration"] == "1" + assert result["max_exceeded"] == "false" + + +def test_check_review_loop_none_current() -> None: + result = check_review_loop(pr_number="1", current_iteration=None) # type: ignore[arg-type] + assert result["next_iteration"] == "1" + + +def test_check_review_loop_none_verdict() -> None: + result = check_review_loop(pr_number="1", previous_verdict=None) # type: ignore[arg-type] + assert result["had_blocking"] == "false" + + +# --------------------------------------------------------------------------- +# T_PTS8–T_PTS11: order_id, efficiency table, and env-based scoping tests +# --------------------------------------------------------------------------- + + +# T_PTS8 +@patch("time.sleep") +@patch("subprocess.run") +def test_pts_order_id_captures_cross_clone_sessions(mock_run, _mock_sleep, tmp_path: Path) -> None: + """patch_pr_token_summary with order_id loads sessions from multiple cwd paths.""" + _write_test_sessions( + tmp_path, + [ + { + "dir_name": "s-clone-a", + "cwd": "/clone-A", + "order_id": "issue-42", + "step_name": "rectify", + "input_tokens": 1000, + }, + { + "dir_name": "s-clone-b", + "cwd": "/clone-B", + "order_id": "issue-42", + "step_name": "implement", + "input_tokens": 2000, + }, + ], + ) + mock_run.side_effect = _make_gh_mock(get_body="## Summary\nBody") + result = patch_pr_token_summary(PR_URL, order_id="issue-42", log_dir=str(tmp_path)) + assert result["success"] == "true" + assert result["sessions_loaded"] == "2" + patch_call = mock_run.call_args_list[-1] + body_arg = [a for a in patch_call[0][0] if a.startswith("body=")][0] + assert "rectify" in body_arg + assert "implement" in body_arg + + +# T_PTS9 +@patch("time.sleep") +@patch("subprocess.run") +def test_pts_generates_efficiency_table_when_loc_data_present( + mock_run, _mock_sleep, tmp_path: Path +) -> None: + """patch_pr_token_summary emits Token Efficiency table when LoC data exists.""" + _write_test_sessions( + tmp_path, + [ + { + "dir_name": "s1", + "cwd": "/clone/test", + "step_name": "implement", + "input_tokens": 1000, + "output_tokens": 500, + "loc_insertions": 120, + }, + ], + ) + mock_run.side_effect = _make_gh_mock(get_body="## Summary\nBody") + result = patch_pr_token_summary(PR_URL, cwd="/clone/test", log_dir=str(tmp_path)) + assert result["success"] == "true" + patch_call = mock_run.call_args_list[-1] + body_arg = [a for a in patch_call[0][0] if a.startswith("body=")][0] + assert "## Token Efficiency" in body_arg + + +# T_PTS10 +@patch("time.sleep") +@patch("subprocess.run") +def test_pts_preserves_or_regenerates_efficiency_table( + mock_run, _mock_sleep, tmp_path: Path +) -> None: + """When overwriting existing summary, efficiency table is regenerated, not lost.""" + _write_test_sessions( + tmp_path, + [ + { + "dir_name": "s1", + "cwd": "/clone/test", + "step_name": "implement", + "input_tokens": 1000, + "output_tokens": 500, + "loc_insertions": 100, + }, + ], + ) + existing_body = ( + "## Summary\nSome text\n\n## Token Usage Summary\n\n" + "| Step | old |\n\n" + "## Token Efficiency\n\n| Step | old eff |\n" + ) + mock_run.side_effect = _make_gh_mock(get_body=existing_body) + result = patch_pr_token_summary(PR_URL, cwd="/clone/test", log_dir=str(tmp_path)) + assert result["success"] == "true" + patch_call = mock_run.call_args_list[-1] + body_arg = [a for a in patch_call[0][0] if a.startswith("body=")][0] + assert body_arg.count("## Token Usage Summary") == 1 + assert "## Token Efficiency" in body_arg + + +# T_PTS11 +@patch("time.sleep") +@patch("subprocess.run") +def test_pts_reads_order_id_from_dispatch_env( + mock_run, _mock_sleep, tmp_path: Path, monkeypatch +) -> None: + """patch_pr_token_summary auto-reads AUTOSKILLIT_DISPATCH_ID when order_id not passed.""" + monkeypatch.setenv("AUTOSKILLIT_DISPATCH_ID", "issue-42") + _write_test_sessions( + tmp_path, + [ + { + "dir_name": "s-clone-a", + "cwd": "/clone-A", + "order_id": "issue-42", + "step_name": "rectify", + "input_tokens": 1000, + }, + { + "dir_name": "s-clone-b", + "cwd": "/clone-B", + "order_id": "issue-42", + "step_name": "implement", + "input_tokens": 2000, + }, + ], + ) + mock_run.side_effect = _make_gh_mock(get_body="## Summary\nBody") + result = patch_pr_token_summary(PR_URL, cwd="", log_dir=str(tmp_path)) + assert result["success"] == "true" + assert result["sessions_loaded"] == "2" + + +# --------------------------------------------------------------------------- +# T_EDC1–T_EDC3: enrich_diff_context tests +# --------------------------------------------------------------------------- + +_ANNOTATED_DIFF_CONTENT = ( + "+++ b/src/app.py\n" + "@@ -38,10 +38,12 @@ def main():\n" + "[L38] existing_line_38\n" + "[L39] existing_line_39\n" + "[L40]+new_import\n" + "[L41]+another_import\n" + "[L42] existing_42\n" + "[L43] existing_43\n" + "[L44]+added_44\n" + "[L45] existing_45\n" +) + + +def _setup_handoff(tmp_path: Path, entries: list[dict]) -> None: + review_dir = tmp_path / ".autoskillit" / "temp" / "review-pr" + review_dir.mkdir(parents=True) + handoff = {"schema_version": 1, "context_entries": entries} + (review_dir / "diff_context_123.json").write_text(json.dumps(handoff)) + (review_dir / "annotated_diff_123.txt").write_text(_ANNOTATED_DIFF_CONTENT) + + +# T_EDC1 +def test_enrich_diff_context_fills_empty_code_regions(tmp_path: Path) -> None: + """enrich_diff_context populates empty code_region from annotated diff.""" + _setup_handoff( + tmp_path, + [ + {"path": "src/app.py", "line": 42, "severity": "critical", "code_region": ""}, + ], + ) + result = enrich_diff_context(pr_number="123", work_dir=str(tmp_path)) + assert result["enriched"] == "true" + assert result["enriched_count"] == "1" + + handoff_path = tmp_path / ".autoskillit" / "temp" / "review-pr" / "diff_context_123.json" + handoff = json.loads(handoff_path.read_text()) + assert "[L42]" in handoff["context_entries"][0]["code_region"] + + +# T_EDC2 +def test_enrich_diff_context_preserves_existing_code_regions(tmp_path: Path) -> None: + """enrich_diff_context does not overwrite non-empty code_region values.""" + _setup_handoff( + tmp_path, + [ + { + "path": "src/app.py", + "line": 42, + "severity": "critical", + "code_region": "pre-existing", + }, + {"path": "src/app.py", "line": 40, "severity": "warning", "code_region": ""}, + ], + ) + result = enrich_diff_context(pr_number="123", work_dir=str(tmp_path)) + assert result["enriched"] == "true" + assert result["enriched_count"] == "1" + + handoff_path = tmp_path / ".autoskillit" / "temp" / "review-pr" / "diff_context_123.json" + handoff = json.loads(handoff_path.read_text()) + assert handoff["context_entries"][0]["code_region"] == "pre-existing" + assert "[L40]" in handoff["context_entries"][1]["code_region"] + + +# T_EDC3 +def test_enrich_diff_context_missing_handoff_file(tmp_path: Path) -> None: + """enrich_diff_context returns gracefully when handoff file does not exist.""" + result = enrich_diff_context(pr_number="999", work_dir=str(tmp_path)) + assert result["enriched"] == "false" + assert result["reason"] == "handoff_not_found" + + +# --------------------------------------------------------------------------- +# T3.1–T3.7: annotate_pr_diff review_mode tests +# --------------------------------------------------------------------------- + +_DIFF_OUTPUT = "+++ b/src/app.py\n@@ -1,3 +1,4 @@\n line1\n+added\n" + + +@patch("subprocess.run") +def test_annotate_pr_diff_returns_review_mode_local(mock_run, tmp_path: Path) -> None: + """T3.1: iteration < local_rounds → review_mode=local.""" + mock_run.return_value = subprocess.CompletedProcess([], 0, _DIFF_OUTPUT, "") + result = annotate_pr_diff( + pr_number="123", + cwd=str(tmp_path), + output_dir=str(tmp_path), + local_review_rounds="3", + current_iteration="0", + base_branch="main", + ) + assert result["review_mode"] == "local" + + +@patch("subprocess.run") +def test_annotate_pr_diff_returns_review_mode_github(mock_run, tmp_path: Path) -> None: + """T3.2: iteration >= local_rounds → review_mode=github.""" + mock_run.return_value = subprocess.CompletedProcess([], 0, _DIFF_OUTPUT, "") + result = annotate_pr_diff( + pr_number="123", + cwd=str(tmp_path), + output_dir=str(tmp_path), + local_review_rounds="3", + current_iteration="3", + ) + assert result["review_mode"] == "github" + + +@patch("subprocess.run") +def test_annotate_pr_diff_local_mode_uses_git_diff(mock_run, tmp_path: Path) -> None: + """T3.3: local mode calls git diff with base...HEAD.""" + mock_run.return_value = subprocess.CompletedProcess([], 0, _DIFF_OUTPUT, "") + annotate_pr_diff( + pr_number="123", + cwd=str(tmp_path), + output_dir=str(tmp_path), + local_review_rounds="2", + current_iteration="0", + base_branch="main", + ) + args = mock_run.call_args[0][0] + assert args[:3] == ["git", "diff", "main...HEAD"] + mock_run.assert_called_once() + + +@patch("subprocess.run") +def test_annotate_pr_diff_github_mode_uses_gh_pr_diff(mock_run, tmp_path: Path) -> None: + """T3.4: github mode calls gh pr diff.""" + mock_run.return_value = subprocess.CompletedProcess([], 0, _DIFF_OUTPUT, "") + annotate_pr_diff( + pr_number="123", + cwd=str(tmp_path), + output_dir=str(tmp_path), + local_review_rounds="2", + current_iteration="2", + base_branch="", + ) + mock_run.assert_called_once() + args = mock_run.call_args[0][0] + assert args[:3] == ["gh", "pr", "diff"] + + +@patch("subprocess.run") +def test_annotate_pr_diff_zero_local_rounds_always_github(mock_run, tmp_path: Path) -> None: + """T3.5: local_review_rounds=0 → always github.""" + mock_run.return_value = subprocess.CompletedProcess([], 0, _DIFF_OUTPUT, "") + result = annotate_pr_diff( + pr_number="123", + cwd=str(tmp_path), + output_dir=str(tmp_path), + local_review_rounds="0", + current_iteration="0", + ) + assert result["review_mode"] == "github" + + +@patch("subprocess.run") +def test_annotate_pr_diff_missing_iteration_defaults_zero(mock_run, tmp_path: Path) -> None: + """T3.6: empty current_iteration defaults to 0 → local mode when local_rounds > 0.""" + mock_run.return_value = subprocess.CompletedProcess([], 0, _DIFF_OUTPUT, "") + result = annotate_pr_diff( + pr_number="123", + cwd=str(tmp_path), + output_dir=str(tmp_path), + local_review_rounds="3", + current_iteration="", + base_branch="main", + ) + assert result["review_mode"] == "local" + + +@patch("subprocess.run") +def test_annotate_pr_diff_local_mode_empty_base_branch_falls_back_to_github( + mock_run, tmp_path: Path +) -> None: + """T3.8: local mode with empty base_branch falls back to gh pr diff and returns github.""" + mock_run.return_value = subprocess.CompletedProcess([], 0, _DIFF_OUTPUT, "") + result = annotate_pr_diff( + pr_number="123", + cwd=str(tmp_path), + output_dir=str(tmp_path), + local_review_rounds="3", + current_iteration="0", + base_branch="", + ) + assert result["review_mode"] == "github" + args = mock_run.call_args[0][0] + assert args[:3] == ["gh", "pr", "diff"] + + +@patch("subprocess.run") +def test_annotate_pr_diff_backward_compat_no_new_params(mock_run, tmp_path: Path) -> None: + """T3.7: old 3-arg call works and defaults review_mode=github.""" + mock_run.return_value = subprocess.CompletedProcess([], 0, _DIFF_OUTPUT, "") + result = annotate_pr_diff( + pr_number="123", + cwd=str(tmp_path), + output_dir=str(tmp_path), + ) + assert "review_mode" in result + assert result["review_mode"] == "github" + + +# ─── Type coercion: annotate_pr_diff with int pr_number (Step 1b) ─────────── + + +@patch("subprocess.run") +def test_annotate_pr_diff_int_pr_number(mock_run, tmp_path: Path) -> None: + """annotate_pr_diff handles int pr_number from LLM JSON boundary. + + Without the type coercion fix, passing pr_number=42 (int) causes + TypeError: argument of type 'int' is not iterable when constructing + the gh subprocess command list. + """ + mock_run.return_value = subprocess.CompletedProcess( + args=[], returncode=0, stdout="+ diff content", stderr="" + ) + output_dir = tmp_path / "output" + output_dir.mkdir() + result = annotate_pr_diff(pr_number=42, cwd=str(tmp_path), output_dir=str(output_dir)) # type: ignore[arg-type] + assert result["annotated_diff_path"] + + # Verify the subprocess call received str "42", not int 42 + gh_diff_call = mock_run.call_args_list[0] + cmd_list = gh_diff_call[0][0] + assert "42" in cmd_list, f"Expected '42' in command, got {cmd_list}" diff --git a/tests/test_test_filter.py b/tests/test_test_filter.py new file mode 100644 index 0000000000..8badcef615 --- /dev/null +++ b/tests/test_test_filter.py @@ -0,0 +1,967 @@ +"""Tests for tests/_test_filter.py — standalone test-path filtering logic.""" + +from __future__ import annotations + +import ast +import subprocess +from pathlib import Path +from unittest.mock import Mock + +import pytest + +import tests._test_filter as tf_mod +from autoskillit._test_filter import apply_manifest as manifest_apply_manifest +from autoskillit._test_filter import load_manifest as manifest_load_manifest +from tests._test_filter import ( + ALWAYS_RUN_AGGRESSIVE, + ALWAYS_RUN_CONSERVATIVE, + LAYER_CASCADE_AGGRESSIVE, + LAYER_CASCADE_CONSERVATIVE, + ASTImportWalker, + FilterMode, + FullRunReason, + ImportContext, + _expand_reexport_closure, + apply_manifest, + build_test_scope, + check_bucket_a, + git_changed_files, + load_manifest, +) + +PROJECT_ROOT = Path(__file__).resolve().parents[1] +MANIFEST_PATH = PROJECT_ROOT / ".autoskillit" / "test-filter-manifest.yaml" + +# --------------------------------------------------------------------------- +# Walker Tests (W1–W8) +# --------------------------------------------------------------------------- + + +class TestASTImportWalker: + def test_walker_top_level_import(self) -> None: + tree = ast.parse("import os") + walker = ASTImportWalker() + walker.visit(tree) + assert ("os", ImportContext.TOP_LEVEL) in walker.imports + + def test_walker_top_level_from_import(self) -> None: + tree = ast.parse("from pathlib import Path") + walker = ASTImportWalker() + walker.visit(tree) + assert ("pathlib", ImportContext.TOP_LEVEL) in walker.imports + + def test_walker_relative_from_import(self) -> None: + tree = ast.parse("from .sub import X") + walker = ASTImportWalker() + walker.visit(tree) + assert (".sub", ImportContext.TOP_LEVEL) in walker.imports + + def test_walker_conditional_import(self) -> None: + source = "import sys\nif sys.platform == 'linux':\n import foo" + tree = ast.parse(source) + walker = ASTImportWalker() + walker.visit(tree) + assert ("foo", ImportContext.CONDITIONAL) in walker.imports + + def test_walker_type_checking_guard(self) -> None: + source = "from typing import TYPE_CHECKING\nif TYPE_CHECKING:\n from foo import Bar" + tree = ast.parse(source) + walker = ASTImportWalker() + walker.visit(tree) + assert ("foo", ImportContext.TYPE_CHECKING) in walker.imports + + def test_walker_type_checking_attribute(self) -> None: + source = "import typing\nif typing.TYPE_CHECKING:\n from bar import Baz" + tree = ast.parse(source) + walker = ASTImportWalker() + walker.visit(tree) + assert ("bar", ImportContext.TYPE_CHECKING) in walker.imports + + def test_walker_deferred_import(self) -> None: + source = "def f():\n import foo" + tree = ast.parse(source) + walker = ASTImportWalker() + walker.visit(tree) + assert ("foo", ImportContext.DEFERRED) in walker.imports + + def test_walker_importlib_literal(self) -> None: + source = 'import importlib\nimportlib.import_module("foo")' + tree = ast.parse(source) + walker = ASTImportWalker() + walker.visit(tree) + assert ("foo", ImportContext.IMPORTLIB) in walker.imports + + +# --------------------------------------------------------------------------- +# Bucket A Tests (B1–B9) +# --------------------------------------------------------------------------- + + +class TestCheckBucketA: + def test_bucket_a_conftest(self) -> None: + assert check_bucket_a({"tests/conftest.py"}) is True + + def test_bucket_a_helpers(self) -> None: + assert check_bucket_a({"tests/_helpers.py"}) is True + + def test_bucket_a_arch_helpers(self) -> None: + assert check_bucket_a({"tests/arch/_helpers.py"}) is True + assert check_bucket_a({"tests/arch/_rules.py"}) is True + + def test_bucket_a_pyproject(self) -> None: + assert check_bucket_a({"pyproject.toml"}) is True + + def test_bucket_a_uv_lock(self) -> None: + assert check_bucket_a({"uv.lock"}) is True + + def test_bucket_a_precommit(self) -> None: + assert check_bucket_a({".pre-commit-config.yaml"}) is True + + def test_bucket_a_factory(self) -> None: + assert check_bucket_a({"src/autoskillit/server/_factory.py"}) is True + + def test_bucket_a_subdir_conftest(self) -> None: + assert check_bucket_a({"tests/execution/conftest.py"}) is True + + def test_bucket_a_negative(self) -> None: + assert check_bucket_a({"src/autoskillit/core/io.py"}) is False + + +# --------------------------------------------------------------------------- +# build_test_scope Tests (S1–S10) +# --------------------------------------------------------------------------- + + +class TestBuildTestScope: + def test_scope_none_changed_returns_git_unavailable(self, tmp_path: Path) -> None: + result = build_test_scope( + changed_files=None, + mode=FilterMode.CONSERVATIVE, + tests_root=tmp_path / "tests", + ) + assert result is FullRunReason.GIT_UNAVAILABLE + + def test_scope_large_changeset_returns_full_run_reason(self, tmp_path: Path) -> None: + files = {f"src/autoskillit/core/f{i}.py" for i in range(31)} + result = build_test_scope( + changed_files=files, + mode=FilterMode.CONSERVATIVE, + tests_root=tmp_path / "tests", + ) + assert result is FullRunReason.LARGE_CHANGESET + + def test_scope_bucket_a_returns_full_run_reason(self, tmp_path: Path) -> None: + result = build_test_scope( + changed_files={"pyproject.toml"}, + mode=FilterMode.CONSERVATIVE, + tests_root=tmp_path / "tests", + ) + assert result is FullRunReason.BUCKET_A + + def test_scope_l0_core_conservative(self, tmp_path: Path) -> None: + tests_root = tmp_path / "tests" + for d in [ + "core", + "config", + "execution", + "pipeline", + "workspace", + "recipe", + "migration", + "server", + "cli", + "hooks", + "skills", + "arch", + "contracts", + "infra", + "docs", + ]: + (tests_root / d).mkdir(parents=True, exist_ok=True) + + result = build_test_scope( + changed_files={"src/autoskillit/core/io.py"}, + mode=FilterMode.CONSERVATIVE, + tests_root=tests_root, + ) + assert result is not None + dir_names = {p.name for p in result} + for expected in [ + "core", + "config", + "execution", + "pipeline", + "workspace", + "recipe", + "migration", + "server", + "cli", + "hooks", + "skills", + ]: + assert expected in dir_names, f"{expected} missing from cascade" + assert "arch" in dir_names, "arch always-run missing" + assert "contracts" in dir_names, "contracts always-run missing" + # infra and docs are not directories for a non-triggering change; + # 6 infra + 3 hooks unconditional files appear in result via direct_test_files + result_names = {p.name for p in result} + from tests._test_filter import _HOOKS_UNCONDITIONAL_FILES, _INFRA_UNCONDITIONAL_FILES + + for fname in _INFRA_UNCONDITIONAL_FILES: + assert fname in result_names, f"unconditional infra file {fname!r} missing" + for fname in _HOOKS_UNCONDITIONAL_FILES: + assert fname in result_names, f"unconditional hooks file {fname!r} missing" + assert "test_doc_counts.py" in result_names + + def test_scope_l1_execution_conservative(self, tmp_path: Path) -> None: + tests_root = tmp_path / "tests" + for d in [ + "execution", + "core", + "workspace", + "migration", + "server", + "cli", + "infra", + "skills", + "arch", + "contracts", + "docs", + ]: + (tests_root / d).mkdir(parents=True, exist_ok=True) + + result = build_test_scope( + changed_files={"src/autoskillit/execution/headless.py"}, + mode=FilterMode.CONSERVATIVE, + tests_root=tests_root, + ) + assert result is not None + dir_names = {p.name for p in result} + for expected in [ + "execution", + "core", + "workspace", + "migration", + "server", + "cli", + "infra", + "skills", + ]: + assert expected in dir_names, f"{expected} missing from cascade" + + def test_scope_l2_recipe_conservative(self, tmp_path: Path) -> None: + tests_root = tmp_path / "tests" + for d in ["recipe", "arch", "contracts", "docs"]: + (tests_root / d).mkdir(parents=True, exist_ok=True) + for f in [ + "server/test_factory.py", + "server/test_tools_kitchen_envelope.py", + "cli/test_cli_prompts.py", + "cli/test_cook_order_picker.py", + "execution/test_headless_path_validation.py", + "execution/test_zero_write_detection.py", + "migration/test_api.py", + "migration/test_engine.py", + "hooks/test_recipe_write_advisor.py", + "infra/test_pretty_output_recipe.py", + "skills/test_planner_skill_contracts.py", + "skills/test_skill_placeholder_contracts.py", + "skills/test_review_design_guards.py", + "skills/test_skill_tool_syntax_contracts.py", + "core/test_type_constants.py", + "core/test_kitchen_state.py", + "core/test_session_registry.py", + ]: + p = tests_root / f + p.parent.mkdir(parents=True, exist_ok=True) + p.touch() + (tests_root / "test_llm_triage.py").touch() + + result = build_test_scope( + changed_files={"src/autoskillit/recipe/schema.py"}, + mode=FilterMode.CONSERVATIVE, + tests_root=tests_root, + ) + assert result is not None + result_names = {p.name for p in result} + assert "recipe" in result_names, "recipe missing" + for expected in [ + "test_factory.py", + "test_tools_kitchen_envelope.py", + "test_cli_prompts.py", + "test_headless_path_validation.py", + "test_zero_write_detection.py", + "test_pretty_output_recipe.py", + "test_skill_placeholder_contracts.py", + # hooks/test_recipe_write_advisor.py is covered by the directory-level "hooks" entry + "hooks", + "core", + "migration", + ]: + assert expected in result_names, f"{expected} missing" + for absent in [ + "execution", + "infra", + "skills", + "server", + "cli", + ]: + assert absent not in result_names, f"{absent} should not be a full directory" + + def test_scope_l3_server_conservative(self, tmp_path: Path) -> None: + tests_root = tmp_path / "tests" + fleet_dir = tests_root / "fleet" + for d in ["server", "cli", "fleet", "infra", "arch", "contracts", "docs"]: + (tests_root / d).mkdir(parents=True, exist_ok=True) + (fleet_dir / "test_pack_enforcement.py").touch() + + result = build_test_scope( + changed_files={"src/autoskillit/server/helpers.py"}, + mode=FilterMode.CONSERVATIVE, + tests_root=tests_root, + ) + assert result is not None + result_names = {p.name for p in result} + assert "server" in result_names + assert "cli" in result_names + assert "test_pack_enforcement.py" in result_names + from tests._test_filter import _HOOKS_UNCONDITIONAL_FILES, _INFRA_UNCONDITIONAL_FILES + + for fname in _INFRA_UNCONDITIONAL_FILES: + assert fname in result_names, f"unconditional infra file {fname!r} missing" + for fname in _HOOKS_UNCONDITIONAL_FILES: + assert fname in result_names, f"unconditional hooks file {fname!r} missing" + assert "infra" not in {p.name for p in result if (tests_root / p.name).is_dir()}, ( + "full infra dir should not appear for pure server change" + ) + + def test_scope_test_file_included_directly(self, tmp_path: Path) -> None: + tests_root = tmp_path / "tests" + for d in ["arch", "contracts", "infra", "docs"]: + (tests_root / d).mkdir(parents=True, exist_ok=True) + + result = build_test_scope( + changed_files={"tests/core/test_io.py"}, + mode=FilterMode.CONSERVATIVE, + tests_root=tests_root, + ) + assert result is not None + assert Path("tests/core/test_io.py") in result + + def test_scope_nonpython_no_manifest_only_alwaysrun(self, tmp_path: Path) -> None: + """Non-Python file with manifest=None → fail-open → FullRunReason.UNMAPPED_FILE.""" + tests_root = tmp_path / "tests" + for d in ["arch", "contracts", "infra", "docs"]: + (tests_root / d).mkdir(parents=True, exist_ok=True) + + result = build_test_scope( + changed_files={"README.md"}, + mode=FilterMode.CONSERVATIVE, + tests_root=tests_root, + ) + assert result is FullRunReason.UNMAPPED_FILE + + def test_scope_nonpython_unmatched_manifest_full_run(self, tmp_path: Path) -> None: + tests_root = tmp_path / "tests" + for d in ["arch", "contracts", "infra", "docs"]: + (tests_root / d).mkdir(parents=True, exist_ok=True) + manifest = {"docs/**/*.md": ["docs"]} + result = build_test_scope( + changed_files={"some/unknown/file.txt"}, + mode=FilterMode.CONSERVATIVE, + manifest=manifest, + tests_root=tests_root, + ) + assert result is FullRunReason.UNMAPPED_FILE + + def test_scope_nonpython_matched_manifest_adds_dirs(self, tmp_path: Path) -> None: + """A non-Python file that DOES match a manifest pattern contributes its test dirs.""" + tests_root = tmp_path / "tests" + for d in ["arch", "contracts", "infra", "docs"]: + (tests_root / d).mkdir(parents=True, exist_ok=True) + manifest = {"docs/**/*.md": ["docs"]} + result = build_test_scope( + changed_files={"docs/README.md"}, + mode=FilterMode.CONSERVATIVE, + manifest=manifest, + tests_root=tests_root, + ) + assert result is not None + dir_names = {p.name for p in result} + assert "arch" in dir_names + assert "contracts" in dir_names + assert "docs" in dir_names + assert "infra" not in dir_names # docs change doesn't trigger full infra + result_names = {p.name for p in result} + from tests._test_filter import _HOOKS_UNCONDITIONAL_FILES, _INFRA_UNCONDITIONAL_FILES + + for fname in _INFRA_UNCONDITIONAL_FILES: + assert fname in result_names, f"unconditional infra file {fname!r} missing" + for fname in _HOOKS_UNCONDITIONAL_FILES: + assert fname in result_names, f"unconditional hooks file {fname!r} missing" + + def test_scope_none_mode_returns_disabled(self, tmp_path: Path) -> None: + result = build_test_scope( + changed_files={"src/autoskillit/core/io.py"}, + mode=FilterMode.NONE, + tests_root=tmp_path / "tests", + ) + assert result is FullRunReason.DISABLED + + def test_scope_empty_changeset(self, tmp_path: Path) -> None: + tests_root = tmp_path / "tests" + for d in ["arch", "contracts", "infra", "docs"]: + (tests_root / d).mkdir(parents=True, exist_ok=True) + + result = build_test_scope( + changed_files=set(), + mode=FilterMode.CONSERVATIVE, + tests_root=tests_root, + ) + assert result is not None + dir_names = {p.name for p in result} + assert dir_names == {"arch", "contracts", "infra", "docs"} + + def test_scope_tests_claude_md_not_unmapped(self, tmp_path: Path) -> None: + tests_root = tmp_path / "tests" + for d in ["core", "arch", "contracts"]: + (tests_root / d).mkdir(parents=True, exist_ok=True) + manifest = {"tests/**/CLAUDE.md": []} + result = build_test_scope( + changed_files={"tests/core/CLAUDE.md"}, + mode=FilterMode.CONSERVATIVE, + manifest=manifest, + tests_root=tests_root, + ) + assert result is not None + assert result is not FullRunReason.UNMAPPED_FILE + + +# --------------------------------------------------------------------------- +# Conservative vs Aggressive Tests (M1–M4) +# --------------------------------------------------------------------------- + + +class TestFilterModes: + def test_conservative_always_run_includes_infra(self) -> None: + assert "arch" in ALWAYS_RUN_CONSERVATIVE + assert "contracts" in ALWAYS_RUN_CONSERVATIVE + assert "infra" in ALWAYS_RUN_CONSERVATIVE + assert "docs" in ALWAYS_RUN_CONSERVATIVE + + def test_aggressive_always_run_excludes_infra(self) -> None: + assert "arch" in ALWAYS_RUN_AGGRESSIVE + assert "contracts" in ALWAYS_RUN_AGGRESSIVE + assert "infra" not in ALWAYS_RUN_AGGRESSIVE + assert "docs" not in ALWAYS_RUN_AGGRESSIVE + + def test_aggressive_ast_refinement(self, tmp_path: Path) -> None: + tests_root = tmp_path / "tests" + (tests_root / "core").mkdir(parents=True) + (tests_root / "arch").mkdir() + (tests_root / "contracts").mkdir() + + result_aggressive = build_test_scope( + changed_files={"src/autoskillit/core/io.py"}, + mode=FilterMode.AGGRESSIVE, + tests_root=tests_root, + ) + assert result_aggressive is not None + dir_names = {p.name for p in result_aggressive} + assert "core" in dir_names + assert "arch" in dir_names + assert "contracts" in dir_names + + def test_conservative_wider_cascade(self) -> None: + for pkg in LAYER_CASCADE_CONSERVATIVE: + if pkg in LAYER_CASCADE_AGGRESSIVE: + assert LAYER_CASCADE_AGGRESSIVE[pkg] <= LAYER_CASCADE_CONSERVATIVE[pkg], ( + f"Aggressive cascade for {pkg} is not a subset of conservative" + ) + + +# --------------------------------------------------------------------------- +# Git Diff Edge Cases (G1–G5) +# --------------------------------------------------------------------------- + + +class TestGitChangedFiles: + def test_git_changed_files_success(self, monkeypatch: pytest.MonkeyPatch) -> None: + mock_run = Mock( + side_effect=[ + subprocess.CompletedProcess(args=[], returncode=0, stdout="abc123\n"), + subprocess.CompletedProcess( + args=[], + returncode=0, + stdout="src/autoskillit/core/io.py\ntests/core/test_io.py\n", + ), + subprocess.CompletedProcess(args=[], returncode=0, stdout=""), + ] + ) + monkeypatch.setattr(subprocess, "run", mock_run) + result = git_changed_files("/fake", base_ref="main") + assert result == {"src/autoskillit/core/io.py", "tests/core/test_io.py"} + assert mock_run.call_count == 3 + + def test_git_changed_files_failure_returns_none( + self, + monkeypatch: pytest.MonkeyPatch, + ) -> None: + def _raise(*a: object, **kw: object) -> None: + raise subprocess.CalledProcessError(1, "git") + + monkeypatch.setattr(subprocess, "run", _raise) + result = git_changed_files("/fake", base_ref="main") + assert result is None + + def test_git_changed_files_timeout_returns_none( + self, + monkeypatch: pytest.MonkeyPatch, + ) -> None: + def _raise(*a: object, **kw: object) -> None: + raise subprocess.TimeoutExpired("git", 10) + + monkeypatch.setattr(subprocess, "run", _raise) + result = git_changed_files("/fake", base_ref="main") + assert result is None + + def test_git_changed_files_env_override( + self, + monkeypatch: pytest.MonkeyPatch, + ) -> None: + monkeypatch.setenv("AUTOSKILLIT_TEST_BASE_REF", "feature-branch") + monkeypatch.delenv("GITHUB_BASE_REF", raising=False) + + mock_run = Mock( + side_effect=[ + subprocess.CompletedProcess(args=[], returncode=0, stdout="abc123\n"), + subprocess.CompletedProcess(args=[], returncode=0, stdout=""), + subprocess.CompletedProcess(args=[], returncode=0, stdout=""), + ] + ) + monkeypatch.setattr(subprocess, "run", mock_run) + git_changed_files("/fake") + assert mock_run.call_count == 3 + first_call_args = list(mock_run.call_args_list[0][0][0]) + assert first_call_args[:3] == ["git", "merge-base", "HEAD"] + assert first_call_args[3] == "feature-branch" + + def test_git_changed_files_github_base_ref( + self, + monkeypatch: pytest.MonkeyPatch, + ) -> None: + monkeypatch.delenv("AUTOSKILLIT_TEST_BASE_REF", raising=False) + monkeypatch.setenv("GITHUB_BASE_REF", "main") + + mock_run = Mock( + side_effect=[ + subprocess.CompletedProcess(args=[], returncode=0, stdout="abc123\n"), + subprocess.CompletedProcess(args=[], returncode=0, stdout=""), + subprocess.CompletedProcess(args=[], returncode=0, stdout=""), + ] + ) + monkeypatch.setattr(subprocess, "run", mock_run) + git_changed_files("/fake") + assert mock_run.call_count == 3 + first_call_args = list(mock_run.call_args_list[0][0][0]) + assert first_call_args[:3] == ["git", "merge-base", "HEAD"] + assert first_call_args[3] == "main" + + def test_git_changed_files_includes_unstaged_tracked( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + mock_run = Mock( + side_effect=[ + subprocess.CompletedProcess(args=[], returncode=0, stdout="abc123\n"), + subprocess.CompletedProcess( + args=[], returncode=0, stdout="src/autoskillit/core/io.py\n" + ), + subprocess.CompletedProcess(args=[], returncode=0, stdout=""), + ] + ) + monkeypatch.setattr(subprocess, "run", mock_run) + result = git_changed_files("/fake", base_ref="main") + assert result == {"src/autoskillit/core/io.py"} + assert mock_run.call_count == 3 + + def test_git_changed_files_includes_untracked_files( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + mock_run = Mock( + side_effect=[ + subprocess.CompletedProcess(args=[], returncode=0, stdout="abc123\n"), + subprocess.CompletedProcess(args=[], returncode=0, stdout=""), + subprocess.CompletedProcess(args=[], returncode=0, stdout="new_script.py\n"), + ] + ) + monkeypatch.setattr(subprocess, "run", mock_run) + result = git_changed_files("/fake", base_ref="main") + assert result == {"new_script.py"} + assert mock_run.call_count == 3 + + def test_git_changed_files_ls_files_failure_is_nonfatal( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + mock_run = Mock( + side_effect=[ + subprocess.CompletedProcess(args=[], returncode=0, stdout="abc123\n"), + subprocess.CompletedProcess( + args=[], returncode=0, stdout="src/autoskillit/core/io.py\n" + ), + subprocess.CompletedProcess(args=[], returncode=1, stdout=""), + ] + ) + monkeypatch.setattr(subprocess, "run", mock_run) + result = git_changed_files("/fake", base_ref="main") + assert result == {"src/autoskillit/core/io.py"} + assert mock_run.call_count == 3 + + +# --------------------------------------------------------------------------- +# Re-export Closure Tests (R1–R2) +# --------------------------------------------------------------------------- + + +class TestReexportClosure: + def test_reexport_closure_direct_init(self, tmp_path: Path) -> None: + pkg = tmp_path / "pkg" + pkg.mkdir() + (pkg / "sub.py").write_text("x = 1\n") + (pkg / "__init__.py").write_text("from .sub import x\n") + + result = _expand_reexport_closure({"pkg/sub.py"}, tmp_path) + assert "pkg/__init__.py" in result + assert "pkg/sub.py" in result + + def test_reexport_closure_no_match(self, tmp_path: Path) -> None: + pkg = tmp_path / "pkg" + pkg.mkdir() + (pkg / "other.py").write_text("y = 2\n") + (pkg / "__init__.py").write_text("from .sub import x\n") + + result = _expand_reexport_closure({"pkg/other.py"}, tmp_path) + assert "pkg/__init__.py" not in result + assert "pkg/other.py" in result + + +# --------------------------------------------------------------------------- +# Re-export Closure Integration Tests (build_test_scope wiring) +# --------------------------------------------------------------------------- + + +class TestReexportClosureIntegration: + def test_build_test_scope_calls_expand_reexport_closure( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + captured: list[set[str]] = [] + original = tf_mod._expand_reexport_closure + + def spy(changed_src_files: set[str], src_root: object) -> set[str]: + captured.append(set(changed_src_files)) + return original(changed_src_files, src_root) + + monkeypatch.setattr(tf_mod, "_expand_reexport_closure", spy) + + tests_root = tmp_path / "tests" + for d in list(LAYER_CASCADE_AGGRESSIVE["core"]) + list(ALWAYS_RUN_AGGRESSIVE): + (tests_root / d).mkdir(parents=True, exist_ok=True) + + result = build_test_scope( + changed_files={"src/autoskillit/core/io.py"}, + mode=FilterMode.AGGRESSIVE, + tests_root=tests_root, + ) + + assert captured, "_expand_reexport_closure was not called from build_test_scope" + assert "src/autoskillit/core/io.py" in captured[0] + assert result is not None + assert tests_root / "core" in result + + def test_core_io_change_expands_to_core_init( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + """Integration: changing core/io.py expands to include core/__init__.py.""" + core_src = tmp_path / "src" / "autoskillit" / "core" + core_src.mkdir(parents=True) + (core_src / "io.py").write_text("# io\n") + (core_src / "__init__.py").write_text("from .io import atomic_write\n") + + captured_expanded: list[set[str]] = [] + original = tf_mod._expand_reexport_closure + + def spy(changed_src_files: set[str], src_root: object) -> set[str]: + result = original(changed_src_files, src_root) + captured_expanded.append(set(result)) + return result + + monkeypatch.setattr(tf_mod, "_expand_reexport_closure", spy) + + tests_root = tmp_path / "tests" + for d in list(LAYER_CASCADE_CONSERVATIVE["core"]) + list(ALWAYS_RUN_CONSERVATIVE): + (tests_root / d).mkdir(parents=True, exist_ok=True) + + result = build_test_scope( + changed_files={"src/autoskillit/core/io.py"}, + mode=FilterMode.CONSERVATIVE, + tests_root=tests_root, + ) + + assert result is not None + assert captured_expanded, "_expand_reexport_closure was not called" + assert "src/autoskillit/core/__init__.py" in captured_expanded[0], ( + "core/__init__.py was not found in expansion of core/io.py" + ) + + def test_expansion_error_is_fail_open( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + def explode(changed_src_files: set[str], src_root: object) -> set[str]: + raise RuntimeError("simulated expansion failure") + + monkeypatch.setattr(tf_mod, "_expand_reexport_closure", explode) + + tests_root = tmp_path / "tests" + for d in list(LAYER_CASCADE_AGGRESSIVE["core"]) + list(ALWAYS_RUN_AGGRESSIVE): + (tests_root / d).mkdir(parents=True, exist_ok=True) + + result = build_test_scope( + changed_files={"src/autoskillit/core/io.py"}, + mode=FilterMode.AGGRESSIVE, + tests_root=tests_root, + ) + # Expansion error must NOT propagate; scope still computed from original classification + assert result is not None + assert tests_root / "core" in result + + def test_unclassifiable_init_does_not_cause_full_run( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + """An __init__.py added by expansion that maps to no cascade entry is silently skipped.""" + + tests_root = tmp_path / "tests" + for d in list(LAYER_CASCADE_AGGRESSIVE["core"]) + list(ALWAYS_RUN_AGGRESSIVE): + (tests_root / d).mkdir(parents=True, exist_ok=True) + + baseline = build_test_scope( + changed_files={"src/autoskillit/core/io.py"}, + mode=FilterMode.AGGRESSIVE, + tests_root=tests_root, + ) + + # Expansion returns an __init__.py outside any known package + def always_expand(changed_src_files: set[str], src_root: object) -> set[str]: + return set(changed_src_files) | {"src/autoskillit/__init__.py"} + + monkeypatch.setattr(tf_mod, "_expand_reexport_closure", always_expand) + + result = build_test_scope( + changed_files={"src/autoskillit/core/io.py"}, + mode=FilterMode.AGGRESSIVE, + tests_root=tests_root, + ) + # Must NOT return None — unclassifiable init.py is skipped, not a fail-open trigger + assert result is not None + # The unclassifiable init.py must not expand the scope beyond the base classification + assert result == baseline + + +# --------------------------------------------------------------------------- +# Manifest Tests — tests/_test_filter (MA1–MA4) +# --------------------------------------------------------------------------- + + +class TestLoadManifest: + def test_load_manifest_absent(self, tmp_path: Path) -> None: + result = load_manifest(tmp_path) + assert result is None + + def test_load_manifest_valid(self, tmp_path: Path) -> None: + manifest_dir = tmp_path / ".autoskillit" + manifest_dir.mkdir() + (manifest_dir / "test-filter-manifest.yaml").write_text( + "patterns:\n 'docs/*.md':\n - docs\n" + ) + result = load_manifest(tmp_path) + assert result is not None + assert "patterns" in result + assert "docs/*.md" in result["patterns"] + + def test_load_manifest_malformed_yaml(self, tmp_path: Path) -> None: + manifest_dir = tmp_path / ".autoskillit" + manifest_dir.mkdir() + (manifest_dir / "test-filter-manifest.yaml").write_text(":\n - :\n bad: [") + import warnings + + with warnings.catch_warnings(record=True) as w: + warnings.simplefilter("always") + result = load_manifest(tmp_path) + assert result is None + assert any("Malformed YAML" in str(warning.message) for warning in w) + + +class TestApplyManifest: + def test_apply_manifest_none(self) -> None: + result = apply_manifest({"README.md"}, None) + assert result is None + + def test_apply_manifest_match(self) -> None: + manifest = {"docs/*.md": ["docs"]} + result = apply_manifest({"docs/README.md"}, manifest) + assert result == {"docs"} + + def test_apply_manifest_no_match(self) -> None: + manifest = {"tests/*.py": ["unit"]} + result = apply_manifest({"src/foo.py"}, manifest) + assert result is None + + def test_apply_manifest_list_dirs(self) -> None: + manifest = {"*.yaml": ["config", "infra"]} + result = apply_manifest({"defaults.yaml"}, manifest) + assert result == {"config", "infra"} + + def test_apply_manifest_doublestar_zero_segments(self) -> None: + """docs/**/*.md must match docs/README.md (zero intermediate path segments). + + This pattern is in the production manifest. fnmatch fails this; pathspec passes it. + """ + manifest = {"docs/**/*.md": ["docs"]} + result = apply_manifest({"docs/README.md"}, manifest) + assert result == {"docs"} + + def test_apply_manifest_doublestar_nested(self) -> None: + """docs/**/*.md must match docs/developer/SETUP.md (non-zero intermediate segments).""" + manifest = {"docs/**/*.md": ["docs"]} + result = apply_manifest({"docs/developer/SETUP.md"}, manifest) + assert result == {"docs"} + + def test_apply_manifest_tests_claude_md_empty_cascade(self) -> None: + manifest = {"tests/**/CLAUDE.md": []} + result = apply_manifest({"tests/core/CLAUDE.md"}, manifest) + assert result == set() + + def test_apply_manifest_tests_nested_claude_md(self) -> None: + manifest = {"tests/**/CLAUDE.md": []} + result = apply_manifest({"tests/execution/CLAUDE.md"}, manifest) + assert result == set() + + +# --------------------------------------------------------------------------- +# Behavioral equivalence cross-validation (EQ1–EQ2) +# --------------------------------------------------------------------------- + + +class TestApplyManifestEquivalence: + """Cross-validates conftest apply_manifest against production manifest_apply_manifest. + + Both modules must agree on matched outputs. When a file matches patterns in one, + it must match the same patterns in the other. The two implementations are allowed + to differ only in how they handle the unmatched case (both must return None there too). + """ + + MANIFEST = { + "src/autoskillit/recipes/*.yaml": ["recipe", "contracts"], + "docs/**/*.md": ["docs"], + "Taskfile.yml": ["infra"], + "tests/recipe/fixtures/*": ["recipe"], + ".pre-commit-config.yaml": ["infra"], + } + + @pytest.mark.parametrize( + "file_path,expected_dirs", + [ + ("src/autoskillit/recipes/my.yaml", {"recipe", "contracts"}), + ("docs/README.md", {"docs"}), + ("docs/sub/dir/deep.md", {"docs"}), + ("Taskfile.yml", {"infra"}), + ("tests/recipe/fixtures/test.yaml", {"recipe"}), + (".pre-commit-config.yaml", {"infra"}), + ], + ) + def test_matching_cases_agree(self, file_path: str, expected_dirs: set[str]) -> None: + """For files that match manifest patterns, both implementations return the same dirs.""" + conftest_result = apply_manifest({file_path}, self.MANIFEST) + production_result = manifest_apply_manifest([file_path], self.MANIFEST) + assert conftest_result == production_result == expected_dirs, ( + f"Implementations disagree for {file_path!r}: " + f"conftest={conftest_result!r}, production={production_result!r}" + ) + + @pytest.mark.parametrize( + "file_path", + [ + "some/unknown/file.txt", + "README.md", + "pyproject.toml", + ], + ) + def test_unmatched_cases_both_return_none(self, file_path: str) -> None: + """For files matching no manifest pattern, both implementations return None.""" + conftest_result = apply_manifest({file_path}, self.MANIFEST) + production_result = manifest_apply_manifest([file_path], self.MANIFEST) + assert conftest_result is None, f"conftest returned {conftest_result!r} for {file_path!r}" + assert production_result is None, ( + f"production returned {production_result!r} for {file_path!r}" + ) + + +# --------------------------------------------------------------------------- +# Manifest Tests — autoskillit._test_filter (pathspec-based) +# --------------------------------------------------------------------------- + + +class TestManifestLoadManifest: + def test_load_manifest_parses_yaml(self) -> None: + manifest = manifest_load_manifest(MANIFEST_PATH) + assert isinstance(manifest, dict) + assert len(manifest) >= 22 + non_empty_count = 0 + for pattern, dirs in manifest.items(): + assert isinstance(dirs, list) + assert all(isinstance(d, str) for d in dirs) + if len(dirs) > 0: + non_empty_count += 1 + assert non_empty_count >= 22 + + def test_load_manifest_missing_file_raises(self, tmp_path: Path) -> None: + with pytest.raises(FileNotFoundError): + manifest_load_manifest(tmp_path / "nonexistent.yaml") + + +class TestManifestApplyManifest: + def test_apply_manifest_single_star_glob(self) -> None: + manifest = {"src/autoskillit/recipes/*.yaml": ["recipe/"]} + result = manifest_apply_manifest(["src/autoskillit/recipes/implementation.yaml"], manifest) + assert result == {"recipe/"} + + def test_apply_manifest_doublestar_glob(self) -> None: + manifest = {"docs/**/*.md": ["docs/"]} + # Deep nested path + result = manifest_apply_manifest(["docs/developer/README.md"], manifest) + assert result == {"docs/"} + # Zero intermediate segments + result = manifest_apply_manifest(["docs/README.md"], manifest) + assert result == {"docs/"} + + def test_apply_manifest_no_match_returns_none(self) -> None: + manifest = {"src/autoskillit/recipes/*.yaml": ["recipe/"]} + result = manifest_apply_manifest(["some/unknown/file.txt"], manifest) + assert result is None + + def test_apply_manifest_multiple_files_union(self) -> None: + manifest = { + "src/autoskillit/recipes/*.yaml": ["recipe/"], + "docs/**/*.md": ["docs/"], + } + result = manifest_apply_manifest( + ["src/autoskillit/recipes/cook.yaml", "docs/guide.md"], manifest + ) + assert result == {"recipe/", "docs/"} + + def test_apply_manifest_sub_recipes_yaml(self) -> None: + manifest = manifest_load_manifest(MANIFEST_PATH) + assert manifest is not None + result = manifest_apply_manifest( + ["src/autoskillit/recipes/sub-recipes/research.yaml"], + manifest, + ) + assert result == {"recipe/"} diff --git a/tests/test_test_filter_cascade.py b/tests/test_test_filter_cascade.py new file mode 100644 index 0000000000..bf221ff5f4 --- /dev/null +++ b/tests/test_test_filter_cascade.py @@ -0,0 +1,272 @@ +"""REQ-FILT-003: cascade entries for planner, _llm_triage, smoke_utils, version.""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from tests._test_filter import FilterMode, build_test_scope + + +class TestCascadeNewEntries: + """REQ-FILT-003: four new packages must not force a full test run.""" + + @pytest.mark.parametrize( + "filepath,mode,items_to_create,expected_in_result", + [ + # Conservative: planner only touches its own tests + ( + "src/autoskillit/planner/__init__.py", + FilterMode.CONSERVATIVE, + ["planner"], + ["planner"], + ), + # Conservative: _llm_triage cascades into server + direct test file + ( + "src/autoskillit/_llm_triage.py", + FilterMode.CONSERVATIVE, + ["server", "test_llm_triage.py"], + ["server", "test_llm_triage.py"], + ), + # Conservative: smoke_utils cascades into recipe + direct test file + ( + "src/autoskillit/smoke_utils.py", + FilterMode.CONSERVATIVE, + ["recipe", "test_smoke_utils.py"], + ["recipe", "test_smoke_utils.py"], + ), + # Conservative: version cascades into server + direct test file + ( + "src/autoskillit/version.py", + FilterMode.CONSERVATIVE, + ["server", "test_version.py"], + ["server", "test_version.py"], + ), + # Aggressive: planner scoped to its own tests directory + ( + "src/autoskillit/planner/__init__.py", + FilterMode.AGGRESSIVE, + ["planner"], + ["planner"], + ), + # Aggressive: _llm_triage scoped to its direct test file + ( + "src/autoskillit/_llm_triage.py", + FilterMode.AGGRESSIVE, + ["test_llm_triage.py"], + ["test_llm_triage.py"], + ), + # Aggressive: smoke_utils scoped to its direct test file + ( + "src/autoskillit/smoke_utils.py", + FilterMode.AGGRESSIVE, + ["test_smoke_utils.py"], + ["test_smoke_utils.py"], + ), + # Aggressive: version scoped to its direct test file + ( + "src/autoskillit/version.py", + FilterMode.AGGRESSIVE, + ["test_version.py"], + ["test_version.py"], + ), + # Conservative: quota_guard.py cascades into hooks tests + + # execution/test_quota_sleep.py + ( + "src/autoskillit/hooks/guards/quota_guard.py", + FilterMode.CONSERVATIVE, + ["hooks", "execution", "execution/test_quota_sleep.py"], + ["hooks", "test_quota_sleep.py"], + ), + ], + ) + def test_cascade_new_entries_not_full_run( + self, + tmp_path: Path, + filepath: str, + mode: FilterMode, + items_to_create: list[str], + expected_in_result: list[str], + ) -> None: + tests_root = tmp_path / "tests" + tests_root.mkdir(parents=True, exist_ok=True) + for item in items_to_create: + if item.endswith(".py"): + (tests_root / item).touch() + else: + (tests_root / item).mkdir(parents=True, exist_ok=True) + + result = build_test_scope( + changed_files={filepath}, + mode=mode, + tests_root=tests_root, + ) + assert result is not None, ( + f"{filepath} still forces a full test run in {mode} mode — " + "cascade entry is missing from LAYER_CASCADE_" + + ("CONSERVATIVE" if mode == FilterMode.CONSERVATIVE else "AGGRESSIVE") + ) + result_names = {p.name for p in result} + for expected in expected_in_result: + assert expected in result_names, ( + f"{expected!r} not found in result scope {result_names!r} " + f"for {filepath} in {mode} mode" + ) + + +_SERVER_FILE_LEVEL_ENTRIES = [ + "test_factory.py", + "test_tools_load_recipe.py", + "test_server_tool_registration.py", + "test_mcp_overrides.py", + "test_smoke_pipeline.py", + "test_tools_dispatch.py", + "test_tools_kitchen_envelope.py", + "test_service_wrappers.py", + "test_tools_list_recipes.py", +] + +_CLI_FILE_LEVEL_ENTRIES = [ + "test_cli_prompts.py", + "test_l3_orchestrator_prompt.py", + "test_cook_order_picker.py", +] + +_FLEET_FILE_LEVEL_ENTRIES = [ + "test_fleet_e2e.py", + "test_campaign_capture.py", + "test_pack_enforcement.py", + "test_pack_enforcement_e2e.py", + "test_dispatch_failure_semantics.py", +] + + +class TestRecipeCascadeNarrowing: + """REQ-RECIPE-001/002/003: recipe cascade uses file-level entries for server/cli, + and uses file-level entries for migration/hooks (not full-directory entries).""" + + def test_recipe_cascade_server_file_level_only(self, tmp_path: Path) -> None: + tests_root = tmp_path / "tests" + server_dir = tests_root / "server" + server_dir.mkdir(parents=True, exist_ok=True) + for fname in _SERVER_FILE_LEVEL_ENTRIES: + (server_dir / fname).touch() + (server_dir / "test_serve_guard.py").touch() + + result = build_test_scope( + changed_files={"src/autoskillit/recipe/__init__.py"}, + mode=FilterMode.CONSERVATIVE, + tests_root=tests_root, + ) + assert result is not None + result_names = {p.name for p in result} + for fname in _SERVER_FILE_LEVEL_ENTRIES: + assert fname in result_names, f"{fname!r} missing from recipe cascade result" + assert "test_serve_guard.py" not in result_names + + def test_recipe_cascade_cli_file_level_only(self, tmp_path: Path) -> None: + tests_root = tmp_path / "tests" + cli_dir = tests_root / "cli" + cli_dir.mkdir(parents=True, exist_ok=True) + for fname in _CLI_FILE_LEVEL_ENTRIES: + (cli_dir / fname).touch() + (cli_dir / "test_app.py").touch() + + result = build_test_scope( + changed_files={"src/autoskillit/recipe/schema.py"}, + mode=FilterMode.CONSERVATIVE, + tests_root=tests_root, + ) + assert result is not None + result_names = {p.name for p in result} + for fname in _CLI_FILE_LEVEL_ENTRIES: + assert fname in result_names, f"{fname!r} missing from recipe cascade result" + assert "test_app.py" not in result_names + + def test_recipe_cascade_no_migration_full_directory(self, tmp_path: Path) -> None: + # migration/test_store.py is not in the file-level cascade; it should be excluded. + tests_root = tmp_path / "tests" + migration_dir = tests_root / "migration" + migration_dir.mkdir(parents=True, exist_ok=True) + (migration_dir / "test_store.py").touch() + + result = build_test_scope( + changed_files={"src/autoskillit/recipe/schema.py"}, + mode=FilterMode.CONSERVATIVE, + tests_root=tests_root, + ) + assert result is not None + assert not any("/migration/" in str(p) for p in result), ( + f"migration/test_store.py (not in cascade) should not appear; got {result}" + ) + + def test_recipe_cascade_no_hooks_full_directory(self, tmp_path: Path) -> None: + # hooks/test_fmt_status.py is not in the file-level cascade or hook unconditionals; + # it should be excluded. Unconditional hook files (test_hook_executability.py etc.) + # are allowed to appear since they run regardless of trigger. + tests_root = tmp_path / "tests" + hooks_dir = tests_root / "hooks" + hooks_dir.mkdir(parents=True, exist_ok=True) + (hooks_dir / "test_fmt_status.py").touch() + + result = build_test_scope( + changed_files={"src/autoskillit/recipe/schema.py"}, + mode=FilterMode.CONSERVATIVE, + tests_root=tests_root, + ) + assert result is not None + result_names = {p.name for p in result} + assert "test_fmt_status.py" not in result_names, ( + "test_fmt_status.py (not in cascade or hook unconditionals) should not appear" + ) + + def test_recipe_cascade_fleet_file_level_only(self, tmp_path: Path) -> None: + tests_root = tmp_path / "tests" + fleet_dir = tests_root / "fleet" + fleet_dir.mkdir(parents=True, exist_ok=True) + for fname in _FLEET_FILE_LEVEL_ENTRIES: + (fleet_dir / fname).touch() + (fleet_dir / "test_fleet_cli.py").touch() + + result = build_test_scope( + changed_files={"src/autoskillit/recipe/schema.py"}, + mode=FilterMode.CONSERVATIVE, + tests_root=tests_root, + ) + assert len(result) >= len(_FLEET_FILE_LEVEL_ENTRIES) + result_names = {p.name for p in result} + for fname in _FLEET_FILE_LEVEL_ENTRIES: + assert fname in result_names, f"{fname!r} missing from recipe cascade result" + assert "test_fleet_cli.py" not in result_names + + +class TestServerFleetCascadeNarrowing: + """REQ-FLEET-002: server cascade targets only fleet/test_pack_enforcement.py.""" + + def test_server_source_change_targets_pack_enforcement_only(self, tmp_path: Path) -> None: + """A server source change cascades to fleet/test_pack_enforcement.py only, + not to other fleet test files.""" + tests_root = tmp_path / "tests" + fleet_dir = tests_root / "fleet" + fleet_dir.mkdir(parents=True, exist_ok=True) + (fleet_dir / "test_pack_enforcement.py").touch() + (fleet_dir / "test_fleet.py").touch() + server_dir = tests_root / "server" + server_dir.mkdir(parents=True, exist_ok=True) + (server_dir / "test_factory.py").touch() + + result = build_test_scope( + changed_files={"src/autoskillit/server/tools/tools_kitchen.py"}, + mode=FilterMode.CONSERVATIVE, + tests_root=tests_root, + ) + assert result is not None, "server source change should not force a full run" + result_names = {p.name for p in result} + assert "test_pack_enforcement.py" in result_names, ( + "fleet/test_pack_enforcement.py must appear in server cascade" + ) + assert "test_fleet.py" not in result_names, ( + "fleet/test_fleet.py must NOT appear in server cascade — " + "only test_pack_enforcement.py has server imports" + ) diff --git a/tests/test_test_filter_content_aware.py b/tests/test_test_filter_content_aware.py new file mode 100644 index 0000000000..f706508dc1 --- /dev/null +++ b/tests/test_test_filter_content_aware.py @@ -0,0 +1,619 @@ +"""Tests for content-aware Bucket A check and build_test_scope integration (T1–T5).""" + +from __future__ import annotations + +import subprocess +from pathlib import Path +from unittest.mock import Mock + +import pytest + +from tests._test_filter import ( + _CORE_UNIVERSAL_EXCLUSIONS, + _CORE_UNIVERSAL_MODULES, + LAYER_CASCADE_CONSERVATIVE, + FilterMode, + FullRunReason, + _is_additive_only, + build_test_scope, +) + +# --------------------------------------------------------------------------- +# Content-Aware Bucket A Tests (T1) +# --------------------------------------------------------------------------- + + +class TestCheckBucketAContentAware: + def test_content_aware_version_only_pyproject_not_triggered( + self, + monkeypatch: pytest.MonkeyPatch, + ) -> None: + """pyproject.toml with only version= line change: content-aware check returns False.""" + from tests._test_filter import check_bucket_a_content_aware + + diff_output = ( + "--- a/pyproject.toml\n+++ b/pyproject.toml\n@@ -5 +5 @@\n" + '-version = "0.9.107"\n+version = "0.9.108"\n' + ) + mock_run = Mock( + side_effect=[ + subprocess.CompletedProcess( + args=[], returncode=0, stdout="deadbeefdeadbeefdeadbeefdeadbeefdeadbeef\n" + ), # merge-base + subprocess.CompletedProcess(args=[], returncode=0, stdout=diff_output), # diff + ] + ) + monkeypatch.setattr(subprocess, "run", mock_run) + result = check_bucket_a_content_aware({"pyproject.toml"}, "/fake", "main") + assert result is False + + def test_content_aware_uv_lock_version_only_not_triggered( + self, + monkeypatch: pytest.MonkeyPatch, + ) -> None: + """uv.lock with only version= line change: content-aware check returns False.""" + from tests._test_filter import check_bucket_a_content_aware + + diff_output = ( + "--- a/uv.lock\n+++ b/uv.lock\n@@ -10 +10 @@\n" + '-version = "0.9.107"\n+version = "0.9.108"\n' + ) + mock_run = Mock( + side_effect=[ + subprocess.CompletedProcess( + args=[], returncode=0, stdout="deadbeefdeadbeefdeadbeefdeadbeefdeadbeef\n" + ), + subprocess.CompletedProcess(args=[], returncode=0, stdout=diff_output), + ] + ) + monkeypatch.setattr(subprocess, "run", mock_run) + result = check_bucket_a_content_aware({"uv.lock"}, "/fake", "main") + assert result is False + + def test_content_aware_pyproject_structural_change_triggers( + self, + monkeypatch: pytest.MonkeyPatch, + ) -> None: + """pyproject.toml with non-version line change: content-aware check returns True.""" + from tests._test_filter import check_bucket_a_content_aware + + diff_output = ( + "--- a/pyproject.toml\n+++ b/pyproject.toml\n" + '@@ -5 +5 @@\n-version = "0.9.107"\n+version = "0.9.108"\n' + '@@ -20 +20 @@\n-requires-python = ">=3.11"\n+requires-python = ">=3.12"\n' + ) + mock_run = Mock( + side_effect=[ + subprocess.CompletedProcess( + args=[], returncode=0, stdout="deadbeefdeadbeefdeadbeefdeadbeefdeadbeef\n" + ), + subprocess.CompletedProcess(args=[], returncode=0, stdout=diff_output), + ] + ) + monkeypatch.setattr(subprocess, "run", mock_run) + result = check_bucket_a_content_aware({"pyproject.toml"}, "/fake", "main") + assert result is True + + def test_content_aware_git_failure_falls_back_to_full_run( + self, + monkeypatch: pytest.MonkeyPatch, + ) -> None: + """Git failure: content-aware check returns True (fail-open).""" + from tests._test_filter import check_bucket_a_content_aware + + def _raise(*a: object, **kw: object) -> None: + raise subprocess.CalledProcessError(1, "git") + + monkeypatch.setattr(subprocess, "run", _raise) + result = check_bucket_a_content_aware({"pyproject.toml"}, "/fake", "main") + assert result is True + + def test_content_aware_other_bucket_a_pattern_unaffected( + self, + monkeypatch: pytest.MonkeyPatch, + ) -> None: + """Other Bucket A patterns (not pyproject/uv.lock) trigger immediately, no git call.""" + from tests._test_filter import check_bucket_a_content_aware + + mock_run = Mock() + monkeypatch.setattr(subprocess, "run", mock_run) + result = check_bucket_a_content_aware({"tests/conftest.py"}, "/fake", "main") + assert result is True + mock_run.assert_not_called() # no git diff needed + + def test_content_aware_version_only_both_files_not_triggered( + self, + monkeypatch: pytest.MonkeyPatch, + ) -> None: + """Both pyproject.toml and uv.lock with version-only changes: returns False.""" + from tests._test_filter import check_bucket_a_content_aware + + diff_output = ( + "--- a/pyproject.toml\n+++ b/pyproject.toml\n@@ -5 +5 @@\n" + '-version = "0.9.107"\n+version = "0.9.108"\n' + "--- a/uv.lock\n+++ b/uv.lock\n@@ -10 +10 @@\n" + '-version = "0.9.107"\n+version = "0.9.108"\n' + ) + mock_run = Mock( + side_effect=[ + subprocess.CompletedProcess( + args=[], returncode=0, stdout="deadbeefdeadbeefdeadbeefdeadbeefdeadbeef\n" + ), + subprocess.CompletedProcess(args=[], returncode=0, stdout=diff_output), + ] + ) + monkeypatch.setattr(subprocess, "run", mock_run) + result = check_bucket_a_content_aware({"pyproject.toml", "uv.lock"}, "/fake", "main") + assert result is False + + +# --------------------------------------------------------------------------- +# build_test_scope content-aware integration tests (T2) +# --------------------------------------------------------------------------- + + +class TestBuildTestScopeContentAware: + def test_scope_pyproject_version_only_with_cwd_no_full_run( + self, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + ) -> None: + """build_test_scope: pyproject.toml version-only with cwd= does NOT force full run.""" + tests_root = tmp_path / "tests" + for d in ["arch", "contracts", "infra", "docs"]: + (tests_root / d).mkdir(parents=True, exist_ok=True) + diff_output = ( + "--- a/pyproject.toml\n+++ b/pyproject.toml\n@@ -5 +5 @@\n" + '-version = "0.9.107"\n+version = "0.9.108"\n' + ) + mock_run = Mock( + side_effect=[ + subprocess.CompletedProcess( + args=[], returncode=0, stdout="deadbeefdeadbeefdeadbeefdeadbeefdeadbeef\n" + ), + subprocess.CompletedProcess(args=[], returncode=0, stdout=diff_output), + ] + ) + monkeypatch.setattr(subprocess, "run", mock_run) + result = build_test_scope( + changed_files={"pyproject.toml"}, + mode=FilterMode.CONSERVATIVE, + tests_root=tests_root, + cwd=str(tmp_path), + base_ref="main", + ) + assert result is not None # filtered run, not full suite + assert result == {tests_root / d for d in ["arch", "contracts", "infra", "docs"]} + + def test_scope_pyproject_without_cwd_still_full_run(self, tmp_path: Path) -> None: + """build_test_scope: pyproject.toml without cwd= still forces full run.""" + tests_root = tmp_path / "tests" + for d in ["arch", "contracts", "infra", "docs"]: + (tests_root / d).mkdir(parents=True, exist_ok=True) + result = build_test_scope( + changed_files={"pyproject.toml"}, + mode=FilterMode.CONSERVATIVE, + tests_root=tests_root, + # no cwd or base_ref + ) + assert result is FullRunReason.BUCKET_A # still full run without cwd + + +# --------------------------------------------------------------------------- +# _is_additive_only unit tests (T3) +# --------------------------------------------------------------------------- + + +class TestIsAdditiveOnly: + def test_additive_only_new_class( + self, + monkeypatch: pytest.MonkeyPatch, + ) -> None: + """Pure addition of a new class: returns True.""" + diff_output = "+class NewType:\n+ field: str" + mock_run = Mock( + side_effect=[ + subprocess.CompletedProcess( + args=[], returncode=0, stdout="deadbeefdeadbeefdeadbeefdeadbeefdeadbeef\n" + ), + subprocess.CompletedProcess(args=[], returncode=0, stdout=diff_output), + ] + ) + monkeypatch.setattr(subprocess, "run", mock_run) + result = _is_additive_only("/fake", "main", "src/core/_type_enums.py") + assert result is True + + def test_additive_only_new_field( + self, + monkeypatch: pytest.MonkeyPatch, + ) -> None: + """Adding a field to existing TypedDict: returns True.""" + diff_output = "+ new_field: int" + mock_run = Mock( + side_effect=[ + subprocess.CompletedProcess( + args=[], returncode=0, stdout="deadbeefdeadbeefdeadbeefdeadbeefdeadbeef\n" + ), + subprocess.CompletedProcess(args=[], returncode=0, stdout=diff_output), + ] + ) + monkeypatch.setattr(subprocess, "run", mock_run) + result = _is_additive_only("/fake", "main", "src/core/_type_enums.py") + assert result is True + + def test_removal_detected_class( + self, + monkeypatch: pytest.MonkeyPatch, + ) -> None: + """Removed class definition: returns False, triggers full cascade.""" + diff_output = "-class OldType:\n- field: str" + mock_run = Mock( + side_effect=[ + subprocess.CompletedProcess( + args=[], returncode=0, stdout="deadbeefdeadbeefdeadbeefdeadbeefdeadbeef\n" + ), + subprocess.CompletedProcess(args=[], returncode=0, stdout=diff_output), + ] + ) + monkeypatch.setattr(subprocess, "run", mock_run) + result = _is_additive_only("/fake", "main", "src/core/_type_enums.py") + assert result is False + + def test_removal_detected_function( + self, + monkeypatch: pytest.MonkeyPatch, + ) -> None: + """Removed function definition: returns False.""" + diff_output = "-def old_func():" + mock_run = Mock( + side_effect=[ + subprocess.CompletedProcess( + args=[], returncode=0, stdout="deadbeefdeadbeefdeadbeefdeadbeefdeadbeef\n" + ), + subprocess.CompletedProcess(args=[], returncode=0, stdout=diff_output), + ] + ) + monkeypatch.setattr(subprocess, "run", mock_run) + result = _is_additive_only("/fake", "main", "src/core/_type_enums.py") + assert result is False + + def test_removal_detected_constant( + self, + monkeypatch: pytest.MonkeyPatch, + ) -> None: + """Removed UPPER_CASE constant: returns False.""" + diff_output = "-SOME_CONST = 42" + mock_run = Mock( + side_effect=[ + subprocess.CompletedProcess( + args=[], returncode=0, stdout="deadbeefdeadbeefdeadbeefdeadbeefdeadbeef\n" + ), + subprocess.CompletedProcess(args=[], returncode=0, stdout=diff_output), + ] + ) + monkeypatch.setattr(subprocess, "run", mock_run) + result = _is_additive_only("/fake", "main", "src/core/_type_enums.py") + assert result is False + + def test_rename_detected( + self, + monkeypatch: pytest.MonkeyPatch, + ) -> None: + """Class rename (removal + addition): returns False.""" + diff_output = "-class OldName:\n+class NewName:" + mock_run = Mock( + side_effect=[ + subprocess.CompletedProcess( + args=[], returncode=0, stdout="deadbeefdeadbeefdeadbeefdeadbeefdeadbeef\n" + ), + subprocess.CompletedProcess(args=[], returncode=0, stdout=diff_output), + ] + ) + monkeypatch.setattr(subprocess, "run", mock_run) + result = _is_additive_only("/fake", "main", "src/core/_type_enums.py") + assert result is False + + def test_mixed_diff_with_removal( + self, + monkeypatch: pytest.MonkeyPatch, + ) -> None: + """Mixed diff with a removal: returns False.""" + diff_output = "+class New:\n-class Old:" + mock_run = Mock( + side_effect=[ + subprocess.CompletedProcess( + args=[], returncode=0, stdout="deadbeefdeadbeefdeadbeefdeadbeefdeadbeef\n" + ), + subprocess.CompletedProcess(args=[], returncode=0, stdout=diff_output), + ] + ) + monkeypatch.setattr(subprocess, "run", mock_run) + result = _is_additive_only("/fake", "main", "src/core/_type_enums.py") + assert result is False + + def test_empty_diff_is_additive( + self, + monkeypatch: pytest.MonkeyPatch, + ) -> None: + """Empty diff: returns True (vacuously additive).""" + diff_output = "" + mock_run = Mock( + side_effect=[ + subprocess.CompletedProcess( + args=[], returncode=0, stdout="deadbeefdeadbeefdeadbeefdeadbeefdeadbeef\n" + ), + subprocess.CompletedProcess(args=[], returncode=0, stdout=diff_output), + ] + ) + monkeypatch.setattr(subprocess, "run", mock_run) + result = _is_additive_only("/fake", "main", "src/core/_type_enums.py") + assert result is True + + def test_git_error_returns_false( + self, + monkeypatch: pytest.MonkeyPatch, + ) -> None: + """Git failure: fail-open returns False (full cascade used).""" + + def _raise(*a: object, **kw: object) -> None: + raise subprocess.CalledProcessError(1, "git") + + monkeypatch.setattr(subprocess, "run", _raise) + result = _is_additive_only("/fake", "main", "src/core/_type_enums.py") + assert result is False + + def test_invalid_sha_returns_false( + self, + monkeypatch: pytest.MonkeyPatch, + ) -> None: + """merge-base returns invalid SHA: fail-open returns False.""" + mock_run = Mock( + side_effect=[ + subprocess.CompletedProcess(args=[], returncode=0, stdout="not-a-sha\n"), + ] + ) + monkeypatch.setattr(subprocess, "run", mock_run) + result = _is_additive_only("/fake", "main", "src/core/_type_enums.py") + assert result is False + + def test_lowercase_field_change_is_additive( + self, + monkeypatch: pytest.MonkeyPatch, + ) -> None: + """Lowercase field type change (no +/- prefix match): treated as additive.""" + diff_output = "- name: str\n+ name: str | None" + mock_run = Mock( + side_effect=[ + subprocess.CompletedProcess( + args=[], returncode=0, stdout="deadbeefdeadbeefdeadbeefdeadbeefdeadbeef\n" + ), + subprocess.CompletedProcess(args=[], returncode=0, stdout=diff_output), + ] + ) + monkeypatch.setattr(subprocess, "run", mock_run) + result = _is_additive_only("/fake", "main", "src/core/_type_enums.py") + assert result is True + + def test_indented_class_removal_detected( + self, + monkeypatch: pytest.MonkeyPatch, + ) -> None: + """Indented class removal still matches the regex: returns False.""" + diff_output = "- class InnerClass:" + mock_run = Mock( + side_effect=[ + subprocess.CompletedProcess( + args=[], returncode=0, stdout="deadbeefdeadbeefdeadbeefdeadbeefdeadbeef\n" + ), + subprocess.CompletedProcess(args=[], returncode=0, stdout=diff_output), + ] + ) + monkeypatch.setattr(subprocess, "run", mock_run) + result = _is_additive_only("/fake", "main", "src/core/_type_enums.py") + assert result is False + + +# --------------------------------------------------------------------------- +# build_test_scope integration tests for content-aware cascade (T4) +# --------------------------------------------------------------------------- + + +class TestBuildTestScopeUniversalExclusions: + ALL_DIRS = [ + "core", + "config", + "execution", + "pipeline", + "workspace", + "recipe", + "migration", + "fleet", + "server", + "cli", + "hooks", + "skills", + "_llm_triage", + "_test_filter", + "hook_registry", + "planner", + "smoke_utils", + "arch", + "contracts", + "infra", + "docs", + ] + + @staticmethod + def _make_tests_root(tmp_path: Path) -> Path: + tests_root = tmp_path / "tests" + for d in TestBuildTestScopeUniversalExclusions.ALL_DIRS: + (tests_root / d).mkdir(parents=True, exist_ok=True) + return tests_root + + def test_universal_additive_only_uses_narrowed_cascade( + self, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + ) -> None: + """_type_enums with additive-only diff: cascade excludes hooks and skills.""" + tests_root = self._make_tests_root(tmp_path) + diff_output = "+class NewType:\n+ field: str" + mock_run = Mock( + side_effect=[ + subprocess.CompletedProcess( + args=[], returncode=0, stdout="deadbeefdeadbeefdeadbeefdeadbeefdeadbeef\n" + ), + subprocess.CompletedProcess(args=[], returncode=0, stdout=diff_output), + ] + ) + monkeypatch.setattr(subprocess, "run", mock_run) + result = build_test_scope( + changed_files={"src/autoskillit/core/_type_enums.py"}, + mode=FilterMode.CONSERVATIVE, + tests_root=tests_root, + cwd=str(tmp_path), + base_ref="main", + ) + assert isinstance(result, set) + dir_names = {p.name for p in result if p.is_dir()} + for pkg in ["core", "config", "execution", "pipeline", "server", "cli"]: + assert pkg in dir_names, f"narrowed cascade should include {pkg}" + for excluded in ["hooks", "skills"]: + assert excluded not in dir_names, f"narrowed cascade should exclude {excluded}" + + def test_universal_breaking_change_uses_full_cascade( + self, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + ) -> None: + """_type_enums with removal in diff: uses full cascade.""" + tests_root = self._make_tests_root(tmp_path) + diff_output = "-class OldType:" + mock_run = Mock( + side_effect=[ + subprocess.CompletedProcess( + args=[], returncode=0, stdout="deadbeefdeadbeefdeadbeefdeadbeefdeadbeef\n" + ), + subprocess.CompletedProcess(args=[], returncode=0, stdout=diff_output), + ] + ) + monkeypatch.setattr(subprocess, "run", mock_run) + result = build_test_scope( + changed_files={"src/autoskillit/core/_type_enums.py"}, + mode=FilterMode.CONSERVATIVE, + tests_root=tests_root, + cwd=str(tmp_path), + base_ref="main", + ) + assert isinstance(result, set) + dir_names = {p.name for p in result if p.is_dir()} + for pkg in ["core", "config", "execution", "hooks", "skills"]: + assert pkg in dir_names, f"full cascade should include {pkg}" + + def test_universal_without_exclusion_entry_uses_full_cascade( + self, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + ) -> None: + """io.py changed (universal but no exclusion entry): full cascade.""" + tests_root = self._make_tests_root(tmp_path) + diff_output = "+class NewType:" + mock_run = Mock( + side_effect=[ + subprocess.CompletedProcess( + args=[], returncode=0, stdout="deadbeefdeadbeefdeadbeefdeadbeefdeadbeef\n" + ), + subprocess.CompletedProcess(args=[], returncode=0, stdout=diff_output), + ] + ) + monkeypatch.setattr(subprocess, "run", mock_run) + result = build_test_scope( + changed_files={"src/autoskillit/core/io.py"}, + mode=FilterMode.CONSERVATIVE, + tests_root=tests_root, + cwd=str(tmp_path), + base_ref="main", + ) + assert isinstance(result, set) + dir_names = {p.name for p in result if p.is_dir()} + for pkg in ["core", "config", "execution", "hooks", "skills"]: + assert pkg in dir_names, f"full cascade should include {pkg}" + mock_run.assert_not_called() + + def test_universal_without_cwd_uses_full_cascade( + self, + tmp_path: Path, + ) -> None: + """_type_enums with additive diff but no cwd/base_ref: full cascade.""" + tests_root = self._make_tests_root(tmp_path) + result = build_test_scope( + changed_files={"src/autoskillit/core/_type_enums.py"}, + mode=FilterMode.CONSERVATIVE, + tests_root=tests_root, + ) + assert isinstance(result, set) + dir_names = {p.name for p in result if p.is_dir()} + for pkg in ["core", "config", "execution", "hooks", "skills"]: + assert pkg in dir_names, f"full cascade should include {pkg}" + + def test_universal_git_error_uses_full_cascade( + self, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + ) -> None: + """_type_enums with git error: full cascade (fail-open).""" + tests_root = self._make_tests_root(tmp_path) + + def _raise(*a: object, **kw: object) -> None: + raise subprocess.CalledProcessError(1, "git") + + monkeypatch.setattr(subprocess, "run", _raise) + result = build_test_scope( + changed_files={"src/autoskillit/core/_type_enums.py"}, + mode=FilterMode.CONSERVATIVE, + tests_root=tests_root, + cwd=str(tmp_path), + base_ref="main", + ) + assert isinstance(result, set) + dir_names = {p.name for p in result if p.is_dir()} + for pkg in ["core", "config", "execution", "hooks", "skills"]: + assert pkg in dir_names, f"full cascade should include {pkg}" + + +# --------------------------------------------------------------------------- +# _CORE_UNIVERSAL_EXCLUSIONS contract tests (T5) +# --------------------------------------------------------------------------- + + +class TestCoreUniversalExclusions: + def test_exclusions_dict_exists(self) -> None: + """_CORE_UNIVERSAL_EXCLUSIONS is a dict.""" + assert isinstance(_CORE_UNIVERSAL_EXCLUSIONS, dict) + + def test_all_exclusion_keys_are_strings(self) -> None: + """All keys in _CORE_UNIVERSAL_EXCLUSIONS are strings.""" + assert all(isinstance(k, str) for k in _CORE_UNIVERSAL_EXCLUSIONS) + + def test_all_exclusion_values_are_frozensets(self) -> None: + """All values in _CORE_UNIVERSAL_EXCLUSIONS are frozenset[str].""" + assert all( + isinstance(v, frozenset) and all(isinstance(x, str) for x in v) + for v in _CORE_UNIVERSAL_EXCLUSIONS.values() + ) + + def test_type_enums_exclusion(self) -> None: + """_type_enums maps to hooks and skills.""" + assert _CORE_UNIVERSAL_EXCLUSIONS.get("_type_enums") == frozenset({"hooks", "skills"}) + + def test_exclusion_keys_are_universal_modules(self) -> None: + """Every key in _CORE_UNIVERSAL_EXCLUSIONS is in _CORE_UNIVERSAL_MODULES.""" + assert all(stem in _CORE_UNIVERSAL_MODULES for stem in _CORE_UNIVERSAL_EXCLUSIONS) + + def test_exclusion_dirs_are_subset_of_core_cascade(self) -> None: + """Every exclusion dir is in LAYER_CASCADE_CONSERVATIVE[core].""" + core_dirs = LAYER_CASCADE_CONSERVATIVE["core"] + for stem, exclusions in _CORE_UNIVERSAL_EXCLUSIONS.items(): + assert exclusions <= core_dirs, ( + f"{stem} exclusion {exclusions} is not a subset of core cascade {core_dirs}" + ) diff --git a/tests/test_test_filter_core_cascade.py b/tests/test_test_filter_core_cascade.py new file mode 100644 index 0000000000..e8952a0785 --- /dev/null +++ b/tests/test_test_filter_core_cascade.py @@ -0,0 +1,567 @@ +"""REQ-CORE-001..004: module-level cascade map for core/ submodules.""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +import tests._test_filter as tf_mod +from tests._test_filter import ( + _CORE_UNIVERSAL_MODULES, + MODULE_CASCADE_CORE, + FilterMode, + build_test_scope, +) + + +class TestCoreUniversalModules: + """REQ-CORE-001: _CORE_UNIVERSAL_MODULES must exist and contain the right stems.""" + + def test_required_stems_present(self) -> None: + required = { + "io", + "logging", + "types", + "_type_constants", + "_type_protocols_logging", + "_type_protocols_execution", + "_type_protocols_github", + "_type_protocols_recipe", + "_type_protocols_infra", + "_type_enums", + "_type_subprocess", + } + assert required <= _CORE_UNIVERSAL_MODULES + + def test_paths_and_init_not_in_universal(self) -> None: + # __init__ handled separately by stem == "__init__" check, not via this frozenset + assert "__init__" not in _CORE_UNIVERSAL_MODULES + + +class TestModuleCascadeCore: + """REQ-CORE-002: MODULE_CASCADE_CORE must exist with validated consumer sets.""" + + def test_all_values_are_frozensets(self) -> None: + for stem, consumers in MODULE_CASCADE_CORE.items(): + assert isinstance(consumers, frozenset), f"{stem} value must be frozenset" + + def test_all_consumers_include_core(self) -> None: + # Every narrow module's cascade must include 'core' (its own tests) + for stem, consumers in MODULE_CASCADE_CORE.items(): + assert "core" in consumers, f"{stem} cascade missing 'core'" + + def test_feature_flags_cascade(self) -> None: + assert MODULE_CASCADE_CORE["feature_flags"] == frozenset( + {"core", "cli", "config", "execution", "recipe", "server", "workspace"} + ) + + def test_branch_guard_cascade(self) -> None: + assert MODULE_CASCADE_CORE["branch_guard"] == frozenset( + {"core", "pipeline", "server", "workspace"} + ) + + def test_no_universal_stem_in_map(self) -> None: + # Universal modules must not appear in MODULE_CASCADE_CORE + overlap = _CORE_UNIVERSAL_MODULES & set(MODULE_CASCADE_CORE.keys()) + assert not overlap, f"Universal modules in MODULE_CASCADE_CORE: {overlap}" + + def test_all_entries_present(self) -> None: + expected_stems = { + "feature_flags", + "branch_guard", + "_plugin_ids", + "_terminal_table", + "_plugin_cache", + "github_url", + "paths", + "_claude_env", + "_version_snapshot", + "claude_conventions", + "_type_resume", + "_type_helpers", + "_type_protocols_workspace", + "_install_detect", + "_linux_proc", + "_type_plugin_source", + "kitchen_state", + "readiness", + "session_provenance", + "session_registry", + "tool_sequence_analysis", + "_type_checkpoint", + "_type_results", + "_type_results_execution", + } + assert set(MODULE_CASCADE_CORE.keys()) == expected_stems + + def test_type_resume_cascade(self) -> None: + assert MODULE_CASCADE_CORE["_type_resume"] == frozenset({"core", "cli", "execution"}) + + def test_type_helpers_cascade(self) -> None: + assert MODULE_CASCADE_CORE["_type_helpers"] == frozenset( + {"core", "execution", "fleet", "pipeline", "recipe", "server"} + ) + + def test_type_protocols_workspace_cascade(self) -> None: + assert MODULE_CASCADE_CORE["_type_protocols_workspace"] == frozenset( + {"core", "pipeline", "recipe", "workspace"} + ) + + def test_type_checkpoint_entry_exists(self) -> None: + assert "_type_checkpoint" in MODULE_CASCADE_CORE + assert MODULE_CASCADE_CORE["_type_checkpoint"] == frozenset( + {"core", "execution", "fleet", "server"} + ) + + def test_type_results_cascade(self) -> None: + assert MODULE_CASCADE_CORE["_type_results"] == frozenset( + { + "core", + "execution", + "pipeline", + "workspace", + "recipe", + "migration", + "fleet", + "server", + "cli", + "_llm_triage", + "_test_filter", + "hook_registry", + "smoke_utils", + } + ) + + +class TestBuildTestScopeCoreCascade: + """REQ-CORE-003/004: build_test_scope routes core modules correctly.""" + + def _make_tests_root(self, tmp_path: Path, dirs: list[str]) -> Path: + tests_root = tmp_path / "tests" + for d in dirs: + (tests_root / d).mkdir(parents=True, exist_ok=True) + return tests_root + + ALL_DIRS = [ + "core", + "config", + "execution", + "pipeline", + "workspace", + "recipe", + "migration", + "fleet", + "server", + "cli", + "hooks", + "skills", + "planner", + "arch", + "contracts", + "infra", + "docs", + ] + + def test_universal_module_triggers_full_cascade(self, tmp_path: Path) -> None: + """io.py is universal → full 12-package cascade.""" + tests_root = self._make_tests_root(tmp_path, self.ALL_DIRS) + result = build_test_scope( + changed_files={"src/autoskillit/core/io.py"}, + mode=FilterMode.CONSERVATIVE, + tests_root=tests_root, + ) + assert result is not None + dir_names = {p.name for p in result} + for pkg in [ + "core", + "config", + "execution", + "pipeline", + "workspace", + "recipe", + "migration", + "server", + "cli", + "hooks", + "skills", + ]: + assert pkg in dir_names, f"universal io.py should cascade to {pkg}" + + def test_init_triggers_full_cascade(self, tmp_path: Path) -> None: + """__init__.py always triggers full cascade.""" + tests_root = self._make_tests_root(tmp_path, self.ALL_DIRS) + result = build_test_scope( + changed_files={"src/autoskillit/core/__init__.py"}, + mode=FilterMode.CONSERVATIVE, + tests_root=tests_root, + ) + assert result is not None + dir_names = {p.name for p in result} + for pkg in ["core", "config", "execution", "server", "cli"]: + assert pkg in dir_names + + def test_kitchen_state_narrow_cascade(self, tmp_path: Path) -> None: + """kitchen_state.py → narrow cascade of {"core"} only.""" + tests_root = self._make_tests_root(tmp_path, self.ALL_DIRS) + result = build_test_scope( + changed_files={"src/autoskillit/core/runtime/kitchen_state.py"}, + mode=FilterMode.CONSERVATIVE, + tests_root=tests_root, + ) + assert result is not None + dir_names = {p.name for p in result} + assert "core" in dir_names + for excluded in ["config", "execution", "pipeline", "fleet", "migration", "workspace"]: + assert excluded not in dir_names, f"narrow cascade should not include {excluded}" + + def test_unknown_core_module_fails_open_to_full_cascade(self, tmp_path: Path) -> None: + """An unknown core module stem → full cascade (fail-open, not None).""" + tests_root = self._make_tests_root(tmp_path, self.ALL_DIRS) + result = build_test_scope( + changed_files={"src/autoskillit/core/_new_future_module.py"}, + mode=FilterMode.CONSERVATIVE, + tests_root=tests_root, + ) + assert result is not None + dir_names = {p.name for p in result} + # Fail-open: should include all packages + for pkg in ["core", "config", "execution", "server", "cli"]: + assert pkg in dir_names, f"fail-open should include {pkg}" + + def test_aggressive_mode_unaffected(self, tmp_path: Path) -> None: + """AGGRESSIVE mode still maps core → {core} regardless of stem.""" + tests_root = self._make_tests_root(tmp_path, self.ALL_DIRS) + result = build_test_scope( + changed_files={"src/autoskillit/core/runtime/kitchen_state.py"}, + mode=FilterMode.AGGRESSIVE, + tests_root=tests_root, + ) + assert result is not None + dir_names = {p.name for p in result} + assert "core" in dir_names + # AGGRESSIVE only maps core → core, no other dirs (except always-run) + for excluded in [ + "execution", + "pipeline", + "workspace", + "recipe", + "migration", + "server", + "cli", + "hooks", + ]: + assert excluded not in dir_names + + def test_paths_cascade_includes_most_packages(self, tmp_path: Path) -> None: + """paths.py is used by almost everything → large but not full cascade.""" + tests_root = self._make_tests_root(tmp_path, self.ALL_DIRS) + result = build_test_scope( + changed_files={"src/autoskillit/core/paths.py"}, + mode=FilterMode.CONSERVATIVE, + tests_root=tests_root, + ) + assert result is not None + dir_names = {p.name for p in result} + for pkg in [ + "core", + "cli", + "config", + "execution", + "fleet", + "migration", + "recipe", + "server", + "workspace", + ]: + assert pkg in dir_names + + def test_readiness_narrow_cascade(self, tmp_path: Path) -> None: + """readiness.py → narrow cascade of {"core", "server"} only.""" + tests_root = self._make_tests_root(tmp_path, self.ALL_DIRS) + result = build_test_scope( + changed_files={"src/autoskillit/core/runtime/readiness.py"}, + mode=FilterMode.CONSERVATIVE, + tests_root=tests_root, + ) + assert result is not None + dir_names = {p.name for p in result} + for pkg in ["core", "server"]: + assert pkg in dir_names, f"narrow cascade should include {pkg}" + for excluded in [ + "cli", + "config", + "execution", + "fleet", + "migration", + "recipe", + "workspace", + ]: + assert excluded not in dir_names, f"narrow cascade should not include {excluded}" + + def test_type_resume_narrow_routing(self, tmp_path: Path) -> None: + tests_root = self._make_tests_root(tmp_path, self.ALL_DIRS) + result = build_test_scope( + changed_files={"src/autoskillit/core/types/_type_resume.py"}, + mode=FilterMode.CONSERVATIVE, + tests_root=tests_root, + ) + assert result is not None + dir_names = {p.name for p in result} + for pkg in ["core", "cli", "execution"]: + assert pkg in dir_names, f"_type_resume cascade should include {pkg}" + for excluded in [ + "server", + "recipe", + "pipeline", + "fleet", + "workspace", + "migration", + "hooks", + ]: + assert excluded not in dir_names, f"_type_resume cascade should not include {excluded}" + + def test_type_helpers_narrow_routing(self, tmp_path: Path) -> None: + tests_root = self._make_tests_root(tmp_path, self.ALL_DIRS) + result = build_test_scope( + changed_files={"src/autoskillit/core/types/_type_helpers.py"}, + mode=FilterMode.CONSERVATIVE, + tests_root=tests_root, + ) + assert result is not None + dir_names = {p.name for p in result} + for pkg in ["core", "execution", "fleet", "pipeline", "recipe", "server"]: + assert pkg in dir_names, f"_type_helpers cascade should include {pkg}" + for excluded in ["cli", "hooks", "workspace", "migration"]: + assert excluded not in dir_names, ( + f"_type_helpers cascade should not include {excluded}" + ) + + def test_type_protocols_workspace_narrow_routing(self, tmp_path: Path) -> None: + tests_root = self._make_tests_root(tmp_path, self.ALL_DIRS) + result = build_test_scope( + changed_files={"src/autoskillit/core/types/_type_protocols_workspace.py"}, + mode=FilterMode.CONSERVATIVE, + tests_root=tests_root, + ) + assert result is not None + dir_names = {p.name for p in result} + for pkg in ["core", "pipeline", "recipe", "workspace"]: + assert pkg in dir_names, f"_type_protocols_workspace cascade should include {pkg}" + for excluded in ["cli", "server", "execution", "fleet", "migration", "hooks"]: + assert excluded not in dir_names, ( + f"_type_protocols_workspace cascade should not include {excluded}" + ) + + def test_type_checkpoint_narrow_routing(self, tmp_path: Path) -> None: + tests_root = self._make_tests_root(tmp_path, self.ALL_DIRS) + result = build_test_scope( + changed_files={"src/autoskillit/core/types/_type_checkpoint.py"}, + mode=FilterMode.CONSERVATIVE, + tests_root=tests_root, + ) + assert result is not None + dir_names = {p.name for p in result} + for pkg in ["core", "execution", "fleet", "server"]: + assert pkg in dir_names, f"_type_checkpoint cascade should include {pkg}" + for excluded in ["cli", "config", "pipeline", "workspace", "recipe", "migration", "hooks"]: + assert excluded not in dir_names, ( + f"_type_checkpoint cascade should not include {excluded}" + ) + + def test_type_results_narrow_routing(self, tmp_path: Path) -> None: + tests_root = self._make_tests_root(tmp_path, self.ALL_DIRS) + result = build_test_scope( + changed_files={"src/autoskillit/core/types/_type_results.py"}, + mode=FilterMode.CONSERVATIVE, + tests_root=tests_root, + ) + assert result is not None + dir_names = {p.name for p in result} + for pkg in ["core", "execution", "server", "fleet", "migration"]: + assert pkg in dir_names, f"_type_results cascade should include {pkg}" + for excluded in ["config", "hooks", "skills", "planner"]: + assert excluded not in dir_names, ( + f"_type_results cascade should not include {excluded}" + ) + + +class TestClosureCoreNarrowCascade: + """Closure-added core/__init__.py uses MODULE_CASCADE_CORE when all causes are narrow.""" + + ALL_DIRS = [ + "core", + "config", + "execution", + "pipeline", + "workspace", + "recipe", + "migration", + "fleet", + "server", + "cli", + "hooks", + "skills", + "arch", + "contracts", + "infra", + "docs", + ] + + def _make_core_layout(self, tmp_path: Path, modules: dict[str, str]) -> Path: + core_dir = tmp_path / "src" / "autoskillit" / "core" + core_dir.mkdir(parents=True) + for name, content in modules.items(): + (core_dir / name).write_text(content) + tests_root = tmp_path / "tests" + for d in self.ALL_DIRS: + (tests_root / d).mkdir(parents=True, exist_ok=True) + return tests_root + + def test_closure_init_uses_narrow_cascade_single_module(self, tmp_path: Path) -> None: + """Single narrow cause → closure __init__.py uses that cause's MODULE_CASCADE_CORE.""" + tests_root = self._make_core_layout( + tmp_path, + { + "_plugin_ids.py": "", + "__init__.py": "from ._plugin_ids import DIRECT_PREFIX\n", + }, + ) + result = build_test_scope( + changed_files={"src/autoskillit/core/_plugin_ids.py"}, + mode=FilterMode.CONSERVATIVE, + tests_root=tests_root, + ) + assert result is not None + dir_names = {p.name for p in result} + for pkg in ["core", "cli", "server"]: + assert pkg in dir_names, f"narrow cascade should include {pkg}" + for excluded in [ + "execution", + "pipeline", + "workspace", + "recipe", + "migration", + "hooks", + ]: + assert excluded not in dir_names, f"narrow cascade should not include {excluded}" + + def test_closure_init_uses_union_for_multiple_modules(self, tmp_path: Path) -> None: + """Multiple narrow causes → union of their MODULE_CASCADE_CORE entries.""" + tests_root = self._make_core_layout( + tmp_path, + { + "_plugin_ids.py": "", + "_plugin_cache.py": "", + "__init__.py": ( + "from ._plugin_ids import DIRECT_PREFIX\n" + "from ._plugin_cache import is_cached\n" + ), + }, + ) + result = build_test_scope( + changed_files={ + "src/autoskillit/core/_plugin_ids.py", + "src/autoskillit/core/_plugin_cache.py", + }, + mode=FilterMode.CONSERVATIVE, + tests_root=tests_root, + ) + assert result is not None + dir_names = {p.name for p in result} + # Union: _plugin_ids={core,cli,server} ∪ _plugin_cache={core,cli,server} + for pkg in ["core", "cli", "server"]: + assert pkg in dir_names, f"union cascade should include {pkg}" + for excluded in [ + "execution", + "pipeline", + "workspace", + "recipe", + "migration", + "hooks", + ]: + assert excluded not in dir_names, f"union cascade should not include {excluded}" + + def test_closure_init_falls_back_when_universal_cause_present(self, tmp_path: Path) -> None: + """A universal cause among the core changes → full cascade (fail-open).""" + tests_root = self._make_core_layout( + tmp_path, + { + "io.py": "", + "kitchen_state.py": "", + "__init__.py": ( + "from .io import atomic_write\nfrom .kitchen_state import KitchenState\n" + ), + }, + ) + result = build_test_scope( + changed_files={ + "src/autoskillit/core/io.py", + "src/autoskillit/core/kitchen_state.py", + }, + mode=FilterMode.CONSERVATIVE, + tests_root=tests_root, + ) + assert result is not None + dir_names = {p.name for p in result} + # io is universal → full cascade + for pkg in ["core", "execution", "pipeline", "server", "cli"]: + assert pkg in dir_names, f"universal fallback should include {pkg}" + + def test_closure_init_falls_back_when_unknown_cause(self, tmp_path: Path) -> None: + """An unmapped cause → full cascade (fail-open).""" + tests_root = self._make_core_layout( + tmp_path, + { + "kitchen_state.py": "", + "_brand_new_module.py": "", + "__init__.py": ( + "from .kitchen_state import KitchenState\n" + "from ._brand_new_module import something\n" + ), + }, + ) + result = build_test_scope( + changed_files={ + "src/autoskillit/core/kitchen_state.py", + "src/autoskillit/core/_brand_new_module.py", + }, + mode=FilterMode.CONSERVATIVE, + tests_root=tests_root, + ) + assert result is not None + dir_names = {p.name for p in result} + for pkg in ["core", "execution", "pipeline", "server", "cli"]: + assert pkg in dir_names, f"unknown-cause fallback should include {pkg}" + + def test_closure_init_falls_back_when_no_core_causes( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + """No core files in changed_src_py → full cascade (fail-open).""" + tests_root = self._make_core_layout( + tmp_path, + { + "__init__.py": "from .kitchen_state import KitchenState\n", + "kitchen_state.py": "", + }, + ) + + original_expand = tf_mod._expand_reexport_closure + + def _patched_expand(changed_src_files: set[str], src_root: str | Path) -> set[str]: + result = original_expand(changed_src_files, src_root) + result.add("src/autoskillit/core/__init__.py") + return result + + monkeypatch.setattr(tf_mod, "_expand_reexport_closure", _patched_expand) + + result = build_test_scope( + changed_files={"src/autoskillit/server/__init__.py"}, + mode=FilterMode.CONSERVATIVE, + tests_root=tests_root, + ) + assert result is not None + dir_names = {p.name for p in result} + # No core causes → full cascade for the closure-added __init__ + for pkg in ["core", "execution", "pipeline", "server", "cli"]: + assert pkg in dir_names, f"no-causes fallback should include {pkg}" diff --git a/tests/test_test_filter_coverage_map.py b/tests/test_test_filter_coverage_map.py new file mode 100644 index 0000000000..82503d3eb4 --- /dev/null +++ b/tests/test_test_filter_coverage_map.py @@ -0,0 +1,74 @@ +"""Tests for load_coverage_map in tests/_test_filter.py.""" + +from __future__ import annotations + +from pathlib import Path + +from tests._test_filter import load_coverage_map + + +class TestLoadCoverageMap: + def test_returns_dict_from_valid_json(self, tmp_path: Path) -> None: + """Parses a valid JSON map and returns dict[str, set[str]].""" + map_file = tmp_path / "test-source-map.json" + map_file.write_text( + '{"src/autoskillit/recipe/rules_dataflow.py": ' + '["tests/recipe/test_rules_dataflow.py", "tests/recipe/test_rules_structure.py"]}', + encoding="utf-8", + ) + result = load_coverage_map(map_file) + assert result is not None + assert "src/autoskillit/recipe/rules_dataflow.py" in result + assert isinstance(result["src/autoskillit/recipe/rules_dataflow.py"], set) + assert ( + "tests/recipe/test_rules_dataflow.py" + in result["src/autoskillit/recipe/rules_dataflow.py"] + ) + + def test_missing_file_returns_none(self, tmp_path: Path) -> None: + """Returns None when the map file does not exist.""" + result = load_coverage_map(tmp_path / "nonexistent.json") + assert result is None + + def test_stale_file_returns_none(self, tmp_path: Path) -> None: + """Returns None when the file mtime is older than max_age_days.""" + import os + import time + + map_file = tmp_path / "test-source-map.json" + map_file.write_text('{"src/foo.py": ["tests/test_foo.py"]}', encoding="utf-8") + # Backdate mtime by 31 days + old_mtime = time.time() - (31 * 24 * 3600) + os.utime(map_file, (old_mtime, old_mtime)) + result = load_coverage_map(map_file, max_age_days=30) + assert result is None + + def test_fresh_file_returns_data(self, tmp_path: Path) -> None: + """Returns data when the file mtime is within max_age_days.""" + map_file = tmp_path / "test-source-map.json" + map_file.write_text('{"src/foo.py": ["tests/test_foo.py"]}', encoding="utf-8") + result = load_coverage_map(map_file, max_age_days=30) + assert result is not None + assert result["src/foo.py"] == {"tests/test_foo.py"} + + def test_custom_max_age_days_respected(self, tmp_path: Path) -> None: + """Custom max_age_days threshold is respected.""" + import os + import time + + map_file = tmp_path / "test-source-map.json" + map_file.write_text('{"src/foo.py": ["tests/test_foo.py"]}', encoding="utf-8") + # Backdate by 2 days — stale with max_age_days=1, fresh with max_age_days=3 + old_mtime = time.time() - (2 * 24 * 3600) + os.utime(map_file, (old_mtime, old_mtime)) + assert load_coverage_map(map_file, max_age_days=1) is None + result_fresh = load_coverage_map(map_file, max_age_days=3) + assert result_fresh is not None + assert result_fresh["src/foo.py"] == {"tests/test_foo.py"} + + def test_malformed_json_returns_none(self, tmp_path: Path) -> None: + """Returns None on JSON parse failure.""" + map_file = tmp_path / "test-source-map.json" + map_file.write_text("{bad json}", encoding="utf-8") + result = load_coverage_map(map_file) + assert result is None diff --git a/tests/test_test_filter_execution_cascade.py b/tests/test_test_filter_execution_cascade.py new file mode 100644 index 0000000000..6355687ef4 --- /dev/null +++ b/tests/test_test_filter_execution_cascade.py @@ -0,0 +1,388 @@ +"""REQ-EXEC-001..004: module-level cascade map for execution/ submodules.""" + +from __future__ import annotations + +from pathlib import Path + +from tests._test_filter import ( + MODULE_CASCADE_EXECUTION, + SUBPKG_CASCADE_EXECUTION, + FilterMode, + build_test_scope, +) + +_ALL_DIRS = [ + "core", + "config", + "execution", + "pipeline", + "workspace", + "recipe", + "migration", + "fleet", + "server", + "cli", + "hooks", + "skills", + "arch", + "contracts", + "infra", + "docs", +] + + +def test_all_entries_present() -> None: + """All documented module stems are present in MODULE_CASCADE_EXECUTION.""" + expected = { + "anomaly_detection", + "clone_guard", + "ci", + "diff_annotator", + "pr_analysis", + "testing", + "db", + "recording", + "github", + "remote_resolver", + "session", + "quota", + "session_log", + "linux_tracing", + "commands", + } + assert expected <= set(MODULE_CASCADE_EXECUTION.keys()) + + +class TestBuildTestScopeExecutionCascade: + """Routing via MODULE_CASCADE_EXECUTION in build_test_scope (CONSERVATIVE mode).""" + + ALL_DIRS = _ALL_DIRS + + def _make_tests_root(self, tmp_path: Path, dirs: list[str]) -> Path: + tests_root = tmp_path / "tests" + for d in dirs: + (tests_root / d).mkdir(parents=True, exist_ok=True) + return tests_root + + def test_narrow_module_uses_narrow_scope(self, tmp_path: Path) -> None: + """anomaly_detection.py change → scope is {execution} only (+ always-run).""" + tests_root = self._make_tests_root(tmp_path, self.ALL_DIRS) + result = build_test_scope( + changed_files={"src/autoskillit/execution/anomaly_detection.py"}, + mode=FilterMode.CONSERVATIVE, + tests_root=tests_root, + ) + assert result is not None + dir_names = {p.name for p in result} + assert "execution" in dir_names + # Must NOT include any dirs not declared in the narrow entry + for excluded in [ + "core", + "cli", + "server", + "workspace", + "migration", + "pipeline", + "recipe", + ]: + assert excluded not in dir_names, ( + f"narrow cascade for anomaly_detection should not include {excluded}" + ) + # Always-run dirs must still be present + assert "arch" in dir_names + assert "contracts" in dir_names + + def test_narrow_modules_all_resolve_to_execution_only(self, tmp_path: Path) -> None: + """Each of the narrowest stems maps to frozenset({"execution"}).""" + narrow_stems = [ + "anomaly_detection", + "clone_guard", + ] + tests_root = self._make_tests_root(tmp_path, self.ALL_DIRS) + for stem in narrow_stems: + result = build_test_scope( + changed_files={f"src/autoskillit/execution/{stem}.py"}, + mode=FilterMode.CONSERVATIVE, + tests_root=tests_root, + ) + assert result is not None, f"{stem} should return non-None result" + dir_names = {p.name for p in result} + assert "execution" in dir_names, f"{stem} should cascade to 'execution'" + for excluded in ["core", "cli", "server", "workspace", "migration"]: + assert excluded not in dir_names, ( + f"{stem} narrow cascade should not include {excluded}" + ) + + def test_unknown_execution_stem_falls_through_to_cascade(self, tmp_path: Path) -> None: + """headless.py (not in MODULE_CASCADE_EXECUTION) → cascade_map["execution"].""" + tests_root = self._make_tests_root(tmp_path, self.ALL_DIRS) + result = build_test_scope( + changed_files={"src/autoskillit/execution/headless.py"}, + mode=FilterMode.CONSERVATIVE, + tests_root=tests_root, + ) + assert result is not None + dir_names = {p.name for p in result} + # LAYER_CASCADE_CONSERVATIVE["execution"] includes execution, core, workspace, + # migration, server, cli, infra, skills + for pkg in ["execution", "server", "cli", "workspace"]: + assert pkg in dir_names, f"fail-open cascade for headless.py should include {pkg}" + + def test_medium_scope_module_ci(self, tmp_path: Path) -> None: + """ci.py → frozenset({"execution"}) (its MODULE_CASCADE_EXECUTION entry).""" + tests_root = self._make_tests_root(tmp_path, self.ALL_DIRS) + result = build_test_scope( + changed_files={"src/autoskillit/execution/ci.py"}, + mode=FilterMode.CONSERVATIVE, + tests_root=tests_root, + ) + assert result is not None + dir_names = {p.name for p in result} + assert "execution" in dir_names + for excluded in ["core", "cli", "server", "workspace", "migration"]: + assert excluded not in dir_names, f"ci narrow cascade should not include {excluded}" + + def test_recording_medium_scope_includes_server_files(self, tmp_path: Path) -> None: + """recording.py entry includes specific server/ test files.""" + tests_root = self._make_tests_root(tmp_path, self.ALL_DIRS) + # Create the specific server test file so it resolves in the result + (tests_root / "server" / "test_factory_recording.py").write_text("") + result = build_test_scope( + changed_files={"src/autoskillit/execution/recording.py"}, + mode=FilterMode.CONSERVATIVE, + tests_root=tests_root, + ) + assert result is not None + result_paths = {str(p) for p in result} + result_names = {p.name for p in result} + assert "execution" in result_names + assert "test_factory_recording.py" in result_names, ( + "recording.py cascade should include server/test_factory_recording.py" + ) + assert str(tests_root / "server" / "test_factory_recording.py") in result_paths + + def test_aggressive_mode_skips_execution_cascade_branch(self, tmp_path: Path) -> None: + """In AGGRESSIVE mode the execution branch is not taken; maps to {"execution"}.""" + tests_root = self._make_tests_root(tmp_path, self.ALL_DIRS) + result = build_test_scope( + changed_files={"src/autoskillit/execution/anomaly_detection.py"}, + mode=FilterMode.AGGRESSIVE, + tests_root=tests_root, + ) + assert result is not None + dir_names = {p.name for p in result} + assert "execution" in dir_names + # AGGRESSIVE maps execution → {execution} only; no widening via MODULE_CASCADE_EXECUTION + for excluded in ["core", "cli", "server", "workspace", "migration"]: + assert excluded not in dir_names, ( + f"AGGRESSIVE mode should not widen execution cascade to {excluded}" + ) + + +class TestClosureExecutionNarrowCascade: + """__init__.py closure expansion for execution package.""" + + ALL_DIRS = _ALL_DIRS + + def _make_execution_layout(self, tmp_path: Path, modules: dict[str, str]) -> Path: + exec_dir = tmp_path / "src" / "autoskillit" / "execution" + exec_dir.mkdir(parents=True) + for name, content in modules.items(): + (exec_dir / name).write_text(content) + tests_root = tmp_path / "tests" + for d in self.ALL_DIRS: + (tests_root / d).mkdir(parents=True, exist_ok=True) + return tests_root + + def test_init_closure_narrow_single_cause(self, tmp_path: Path) -> None: + """ + Changing anomaly_detection.py triggers closure to add execution/__init__.py. + Because anomaly_detection is narrow, the __init__ back-propagation + should still resolve to frozenset({"execution"}). + """ + tests_root = self._make_execution_layout( + tmp_path, + { + "anomaly_detection.py": "", + "__init__.py": "from .anomaly_detection import AnomalyDetector\n", + }, + ) + result = build_test_scope( + changed_files={"src/autoskillit/execution/anomaly_detection.py"}, + mode=FilterMode.CONSERVATIVE, + tests_root=tests_root, + ) + assert result is not None + dir_names = {p.name for p in result} + assert "execution" in dir_names + for excluded in ["core", "cli", "server", "workspace", "migration"]: + assert excluded not in dir_names, ( + f"narrow closure cascade should not include {excluded}" + ) + + def test_init_closure_mixed_causes_falls_through(self, tmp_path: Path) -> None: + """ + Changing both anomaly_detection.py (narrow) and headless.py (wide/unknown) + → __init__ closure must fall through to cascade_map["execution"]. + """ + tests_root = self._make_execution_layout( + tmp_path, + { + "anomaly_detection.py": "", + "headless.py": "", + "__init__.py": ( + "from .anomaly_detection import AnomalyDetector\n" + "from .headless import HeadlessSession\n" + ), + }, + ) + result = build_test_scope( + changed_files={ + "src/autoskillit/execution/anomaly_detection.py", + "src/autoskillit/execution/headless.py", + }, + mode=FilterMode.CONSERVATIVE, + tests_root=tests_root, + ) + assert result is not None + dir_names = {p.name for p in result} + # headless is not in MODULE_CASCADE_EXECUTION → fail-open → full execution cascade + for pkg in ["execution", "server", "cli"]: + assert pkg in dir_names, f"mixed-cause closure fallback should include {pkg}" + + def test_init_closure_all_narrow_causes_union(self, tmp_path: Path) -> None: + """ + Changing ci.py + clone_guard.py (both narrow to {"execution"}) → + __init__ closure union is still {"execution"}. + """ + tests_root = self._make_execution_layout( + tmp_path, + { + "ci.py": "", + "clone_guard.py": "", + "__init__.py": ( + "from .ci import CIWatcher\nfrom .clone_guard import CloneGuard\n" + ), + }, + ) + result = build_test_scope( + changed_files={ + "src/autoskillit/execution/ci.py", + "src/autoskillit/execution/clone_guard.py", + }, + mode=FilterMode.CONSERVATIVE, + tests_root=tests_root, + ) + assert result is not None + dir_names = {p.name for p in result} + assert "execution" in dir_names + for excluded in ["core", "cli", "server", "workspace", "migration"]: + assert excluded not in dir_names, ( + f"ci+clone_guard union closure should not include {excluded}" + ) + + +class TestSubpkgCascadeExecution: + """REQ-MQ-001..004: subpackage-aware cascade routing for execution/ subpackages.""" + + ALL_DIRS = _ALL_DIRS + + def _make_tests_root(self, tmp_path: Path, dirs: list[str]) -> Path: + tests_root = tmp_path / "tests" + for d in dirs: + (tests_root / d).mkdir(parents=True, exist_ok=True) + return tests_root + + def _make_execution_subpkg_layout( + self, + tmp_path: Path, + subpkg: str, + modules: dict[str, str], + ) -> Path: + subpkg_dir = tmp_path / "src" / "autoskillit" / "execution" / subpkg + subpkg_dir.mkdir(parents=True) + for name, content in modules.items(): + (subpkg_dir / name).write_text(content) + tests_root = tmp_path / "tests" + for d in self.ALL_DIRS: + (tests_root / d).mkdir(parents=True, exist_ok=True) + return tests_root + + def test_merge_queue_init_uses_narrow_cascade(self, tmp_path: Path) -> None: + """execution/merge_queue/__init__.py → narrow {execution} cascade.""" + tests_root = self._make_tests_root(tmp_path, self.ALL_DIRS) + result = build_test_scope( + changed_files={"src/autoskillit/execution/merge_queue/__init__.py"}, + mode=FilterMode.CONSERVATIVE, + tests_root=tests_root, + ) + assert result is not None + dir_names = {p.name for p in result} + assert "execution" in dir_names + for excluded in ["core", "cli", "server", "workspace", "migration"]: + assert excluded not in dir_names, ( + f"merge_queue/__init__ cascade should not include {excluded}" + ) + + def test_merge_queue_private_module_uses_narrow_cascade(self, tmp_path: Path) -> None: + """execution/merge_queue/_merge_queue_classifier.py → narrow {execution} cascade.""" + tests_root = self._make_tests_root(tmp_path, self.ALL_DIRS) + result = build_test_scope( + changed_files={"src/autoskillit/execution/merge_queue/_merge_queue_classifier.py"}, + mode=FilterMode.CONSERVATIVE, + tests_root=tests_root, + ) + assert result is not None + dir_names = {p.name for p in result} + assert "execution" in dir_names + for excluded in ["core", "cli", "server", "workspace", "migration"]: + assert excluded not in dir_names, ( + f"merge_queue private module cascade should not include {excluded}" + ) + + def test_merge_queue_and_non_subpkg_file_fails_open(self, tmp_path: Path) -> None: + """merge_queue file + headless.py (not in any map) → full execution cascade.""" + tests_root = self._make_tests_root(tmp_path, self.ALL_DIRS) + result = build_test_scope( + changed_files={ + "src/autoskillit/execution/merge_queue/_merge_queue_classifier.py", + "src/autoskillit/execution/headless.py", + }, + mode=FilterMode.CONSERVATIVE, + tests_root=tests_root, + ) + assert result is not None + dir_names = {p.name for p in result} + for pkg in ["execution", "server", "cli", "workspace"]: + assert pkg in dir_names, ( + f"mixed merge_queue+headless should fail open to include {pkg}" + ) + + def test_merge_queue_closure_expansion_stays_narrow(self, tmp_path: Path) -> None: + """ + _merge_queue_classifier.py re-exported by merge_queue/__init__.py → + closure expansion of __init__ stays narrow to {execution}. + """ + tests_root = self._make_execution_subpkg_layout( + tmp_path, + "merge_queue", + { + "__init__.py": ("from ._merge_queue_classifier import MergeQueueClassifier\n"), + "_merge_queue_classifier.py": "", + }, + ) + result = build_test_scope( + changed_files={"src/autoskillit/execution/merge_queue/_merge_queue_classifier.py"}, + mode=FilterMode.CONSERVATIVE, + tests_root=tests_root, + ) + assert result is not None + dir_names = {p.name for p in result} + assert "execution" in dir_names + for excluded in ["core", "cli", "server", "workspace", "migration"]: + assert excluded not in dir_names, f"merge_queue closure should not include {excluded}" + + def test_subpkg_cascade_exported(self) -> None: + """SUBPKG_CASCADE_EXECUTION is importable and contains 'merge_queue' key.""" + assert "merge_queue" in SUBPKG_CASCADE_EXECUTION + assert SUBPKG_CASCADE_EXECUTION["merge_queue"] == frozenset({"execution"}) diff --git a/tests/test_test_filter_plugin.py b/tests/test_test_filter_plugin.py new file mode 100644 index 0000000000..e90761fc5e --- /dev/null +++ b/tests/test_test_filter_plugin.py @@ -0,0 +1,415 @@ +"""Tests for tests/_test_filter.py — conftest filter plugin and shadow-diff tests.""" + +from __future__ import annotations + +import json + +import pytest + +pytest_plugins = ["pytester"] + + +# --------------------------------------------------------------------------- +# Conftest filter plugin – pytester integration tests (P1–P8) +# --------------------------------------------------------------------------- + +_CONFTEST_HOOKS_SOURCE = """ +import json +import os +import warnings +import pytest +from pathlib import Path + +_scope_key = pytest.StashKey[set | None]() +_filter_mode_key = pytest.StashKey[str | None]() +_selected_count_key = pytest.StashKey[int | None]() +_deselected_count_key = pytest.StashKey[int | None]() + +# Module-level accumulator for xdist worker-to-controller IPC. +# Populated by pytest_testnodedown (controller); cleared by pytest_configure +# at session start so in-process pytester reruns don't leak stale data. +_worker_filter_counts: dict[str, int | None] = {} + +def pytest_addoption(parser): + parser.addoption("--filter-mode", default=None, + choices=("none", "conservative", "aggressive")) + parser.addoption("--filter-base-ref", default=None) + +def pytest_configure(config): + _worker_filter_counts.clear() + config.stash[_scope_key] = None + config.stash[_filter_mode_key] = None + cli_mode = config.getoption("--filter-mode", default=None) + env_val = os.environ.get("AUTOSKILLIT_TEST_FILTER", "") + if not cli_mode and not env_val: + return + if not cli_mode and env_val.lower() in ("0", "false", "no"): + return + try: + mode = cli_mode or ("conservative" if env_val.lower() in ("1", "true", "yes") else env_val) + if mode == "none": + return + config.stash[_filter_mode_key] = mode + # Stub scope: only include files under subdir_a/ + config.stash[_scope_key] = {config.rootpath / "subdir_a"} + except Exception as exc: + warnings.warn(f"Test filter setup failed: {exc}", stacklevel=1) + +def pytest_collection_modifyitems(items, config): + scope = config.stash.get(_scope_key, None) + if scope is None: + return + try: + selected, deselected = [], [] + for item in items: + matched = any( + item.path == sp if sp.is_file() else _is_under(item.path, sp) + for sp in scope + ) + (selected if matched else deselected).append(item) + if deselected: + config.hook.pytest_deselected(items=deselected) + items[:] = selected + warnings.warn( + f"Test filter: {len(selected)} selected, {len(deselected)} deselected", + stacklevel=1, + ) + + config.stash[_selected_count_key] = len(items) + config.stash[_deselected_count_key] = len(deselected) + except Exception as exc: + warnings.warn(f"Test filter deselection failed: {exc}", stacklevel=1) + +def pytest_sessionfinish(session, exitstatus): + if hasattr(session.config, "workerinput"): + session.config.workeroutput["filter_selected"] = session.config.stash.get( + _selected_count_key, None + ) + session.config.workeroutput["filter_deselected"] = session.config.stash.get( + _deselected_count_key, None + ) + return + out_path = os.environ.get("AUTOSKILLIT_FILTER_STATS_FILE") + if not out_path: + return + filter_mode = session.config.stash.get(_filter_mode_key, None) + selected = session.config.stash.get(_selected_count_key, None) + deselected = session.config.stash.get(_deselected_count_key, None) + if selected is None and _worker_filter_counts: + selected = _worker_filter_counts.get("selected") + if deselected is None and _worker_filter_counts: + deselected = _worker_filter_counts.get("deselected") + if filter_mode is None: + return + Path(out_path).write_text(json.dumps({ + "filter_mode": filter_mode, + "tests_selected": selected, + "tests_deselected": deselected, + })) + +@pytest.hookimpl(optionalhook=True) +def pytest_testnodedown(node, error): + '''Aggregate filter counts from the first xdist worker that reports. + + Mirrors production conftest.py: captures the first worker that reports both + selected and deselected counts as non-None. + ''' + if _worker_filter_counts: + return + wo = getattr(node, "workeroutput", {}) + selected = wo.get("filter_selected") + deselected = wo.get("filter_deselected") + if selected is not None and deselected is not None: + _worker_filter_counts["selected"] = selected + _worker_filter_counts["deselected"] = deselected + +def _is_under(path, parent): + try: + path.relative_to(parent) + return True + except ValueError: + return False +""" + +_CONFTEST_ERROR_CONFIGURE_SOURCE = """ +import os +import warnings +import pytest + +_scope_key = pytest.StashKey[set | None]() + +def pytest_addoption(parser): + parser.addoption("--filter-mode", default=None) + parser.addoption("--filter-base-ref", default=None) + +def pytest_configure(config): + config.stash[_scope_key] = None + env_val = os.environ.get("AUTOSKILLIT_TEST_FILTER", "") + if not env_val: + return + try: + raise RuntimeError("simulated configure failure") + except Exception as exc: + warnings.warn(f"Test filter setup failed: {exc}", stacklevel=1) +""" + +_CONFTEST_ERROR_MODIFYITEMS_SOURCE = """ +import os +import warnings +import pytest + +_scope_key = pytest.StashKey[set | None]() + +def pytest_addoption(parser): + parser.addoption("--filter-mode", default=None) + parser.addoption("--filter-base-ref", default=None) + +def pytest_configure(config): + config.stash[_scope_key] = None + env_val = os.environ.get("AUTOSKILLIT_TEST_FILTER", "") + if env_val: + config.stash[_scope_key] = {"will_cause_error"} + +def pytest_collection_modifyitems(items, config): + scope = config.stash.get(_scope_key, None) + if scope is None: + return + try: + raise RuntimeError("simulated modifyitems failure") + except Exception as exc: + warnings.warn(f"Test filter deselection failed: {exc}", stacklevel=1) +""" + + +class TestConftestFilterPlugin: + """pytester-based integration tests for conftest filter hook wiring.""" + + def test_filter_inactive_by_default( + self, pytester: pytest.Pytester, monkeypatch: pytest.MonkeyPatch + ) -> None: + monkeypatch.delenv("AUTOSKILLIT_TEST_FILTER", raising=False) + pytester.makeconftest(_CONFTEST_HOOKS_SOURCE) + pytester.makepyfile(test_a="def test_one(): pass", test_b="def test_two(): pass") + result = pytester.runpytest("-v") + result.assert_outcomes(passed=2) + + def test_filter_activates_with_env_var( + self, pytester: pytest.Pytester, monkeypatch: pytest.MonkeyPatch + ) -> None: + monkeypatch.setenv("AUTOSKILLIT_TEST_FILTER", "1") + pytester.makeconftest(_CONFTEST_HOOKS_SOURCE) + pytester.mkdir("subdir_a") + pytester.makepyfile(**{"subdir_a/test_a": "def test_one(): pass"}) + pytester.makepyfile(test_b="def test_two(): pass") + result = pytester.runpytest("-v") + result.assert_outcomes(passed=1, deselected=1) + + def test_deselection_reports_correctly( + self, pytester: pytest.Pytester, monkeypatch: pytest.MonkeyPatch + ) -> None: + monkeypatch.setenv("AUTOSKILLIT_TEST_FILTER", "1") + pytester.makeconftest(_CONFTEST_HOOKS_SOURCE) + pytester.makepyfile(test_keep="def test_keep(): pass") + pytester.makepyfile(test_drop="def test_drop(): pass") + result = pytester.runpytest("-v") + # Both are at root level, not under subdir_a — both deselected + result.assert_outcomes(deselected=2) + + def test_fail_open_on_configure_error( + self, pytester: pytest.Pytester, monkeypatch: pytest.MonkeyPatch + ) -> None: + monkeypatch.setenv("AUTOSKILLIT_TEST_FILTER", "1") + pytester.makeconftest(_CONFTEST_ERROR_CONFIGURE_SOURCE) + pytester.makepyfile(test_a="def test_one(): pass") + result = pytester.runpytest("-v") + result.assert_outcomes(passed=1) + + def test_fail_open_on_modifyitems_error( + self, pytester: pytest.Pytester, monkeypatch: pytest.MonkeyPatch + ) -> None: + monkeypatch.setenv("AUTOSKILLIT_TEST_FILTER", "1") + pytester.makeconftest(_CONFTEST_ERROR_MODIFYITEMS_SOURCE) + pytester.makepyfile(test_a="def test_one(): pass") + result = pytester.runpytest("-v") + result.assert_outcomes(passed=1) + + def test_filter_mode_cli_flag(self, pytester: pytest.Pytester) -> None: + pytester.makeconftest(_CONFTEST_HOOKS_SOURCE) + pytester.makepyfile(test_a="def test_one(): pass") + result = pytester.runpytest("--filter-mode=none", "-v") + result.assert_outcomes(passed=1) + + def test_filter_base_ref_cli_flag( + self, pytester: pytest.Pytester, monkeypatch: pytest.MonkeyPatch + ) -> None: + monkeypatch.delenv("AUTOSKILLIT_TEST_FILTER", raising=False) + pytester.makeconftest(_CONFTEST_HOOKS_SOURCE) + pytester.makepyfile(test_a="def test_one(): pass") + result = pytester.runpytest("--filter-base-ref=main", "-v") + result.assert_outcomes(passed=1) + + def test_summary_warning_emitted( + self, pytester: pytest.Pytester, monkeypatch: pytest.MonkeyPatch + ) -> None: + monkeypatch.setenv("AUTOSKILLIT_TEST_FILTER", "1") + pytester.makeconftest(_CONFTEST_HOOKS_SOURCE) + pytester.mkdir("subdir_a") + pytester.makepyfile(**{"subdir_a/test_keep": "def test_keep(): pass"}) + pytester.makepyfile(test_drop="def test_drop(): pass") + result = pytester.runpytest("-v", "-W", "always") + result.stdout.fnmatch_lines(["*Test filter:*selected*deselected*"]) + + def test_conftest_writes_filter_sidecar( + self, + pytester: pytest.Pytester, + tmp_path, + monkeypatch: pytest.MonkeyPatch, + ) -> None: + """pytest_sessionfinish writes filter stats JSON when sidecar env var is set.""" + sidecar = tmp_path / "filter-stats.json" + monkeypatch.setenv("AUTOSKILLIT_FILTER_STATS_FILE", str(sidecar)) + pytester.makeconftest(_CONFTEST_HOOKS_SOURCE) + pytester.mkdir("subdir_a") + (pytester.path / "subdir_a" / "test_in_scope.py").write_text("def test_ok(): pass\n") + pytester.makepyfile(test_out_scope="def test_skip(): pass") + pytester.runpytest("--filter-mode=conservative") + assert sidecar.is_file(), "Sidecar file must be written by pytest_sessionfinish" + data = json.loads(sidecar.read_text()) + assert data["filter_mode"] == "conservative" + assert isinstance(data["tests_selected"], int) + assert isinstance(data["tests_deselected"], int) + + def test_conftest_no_sidecar_when_env_unset(self, pytester: pytest.Pytester) -> None: + """No sidecar written when AUTOSKILLIT_FILTER_STATS_FILE is not in env.""" + pytester.makeconftest(_CONFTEST_HOOKS_SOURCE) + pytester.mkdir("subdir_a") + (pytester.path / "subdir_a" / "test_simple.py").write_text("def test_a(): pass\n") + result = pytester.runpytest("--filter-mode=conservative") + result.assert_outcomes(passed=1) + + def test_conftest_sidecar_zero_deselection_has_integer_counts( + self, pytester: pytest.Pytester, tmp_path, monkeypatch: pytest.MonkeyPatch + ) -> None: + """AC2: When filter is active but all tests are in scope, sidecar must + have integer counts (not null).""" + sidecar = tmp_path / "filter-stats.json" + monkeypatch.setenv("AUTOSKILLIT_FILTER_STATS_FILE", str(sidecar)) + pytester.makeconftest(_CONFTEST_HOOKS_SOURCE) + pytester.mkdir("subdir_a") + # All test files inside subdir_a — nothing to deselect + (pytester.path / "subdir_a" / "test_all_in.py").write_text( + "def test_a(): pass\ndef test_b(): pass\n" + ) + pytester.runpytest("--filter-mode=conservative") + assert sidecar.is_file() + data = json.loads(sidecar.read_text()) + assert data["filter_mode"] == "conservative" + assert data["tests_selected"] == 2 + assert data["tests_deselected"] == 0 + + def test_conftest_writes_filter_sidecar_under_xdist( + self, + pytester: pytest.Pytester, + tmp_path, + monkeypatch: pytest.MonkeyPatch, + ) -> None: + """pytest_sessionfinish writes integer counts under xdist (-n 2). + + Regression test for the controller/worker stash split: under xdist + pytest_collection_modifyitems runs on workers while pytest_sessionfinish + runs on the controller, so the workeroutput IPC channel is required. + """ + sidecar = tmp_path / "filter-stats.json" + monkeypatch.setenv("AUTOSKILLIT_FILTER_STATS_FILE", str(sidecar)) + pytester.makeconftest(_CONFTEST_HOOKS_SOURCE) + pytester.mkdir("subdir_a") + (pytester.path / "subdir_a" / "test_in_scope.py").write_text("def test_ok(): pass\n") + pytester.makepyfile(test_out_scope="def test_skip(): pass") + # Use out-of-process subprocess mode (not runpytest_inprocess): xdist spawns + # real worker subprocesses that communicate via workeroutput IPC, which cannot + # be exercised in-process. Resource contention risk is low because pytester + # runs its own isolated pytest session in a temp directory. + pytester.runpytest("-n", "2", "--filter-mode=conservative") + assert sidecar.is_file(), "Sidecar file must be written by pytest_sessionfinish" + data = json.loads(sidecar.read_text()) + assert data["filter_mode"] == "conservative" + assert isinstance(data["tests_selected"], int), ( + f"tests_selected must be int under xdist, got {data['tests_selected']!r}" + ) + assert isinstance(data["tests_deselected"], int), ( + f"tests_deselected must be int under xdist, got {data['tests_deselected']!r}" + ) + + +# --------------------------------------------------------------------------- +# Shadow-diff verification tests (SD1) +# --------------------------------------------------------------------------- + + +class TestShadowDiff: + """Shadow-diff verification tests (SD1).""" + + @staticmethod + def _missed(full_ids: list[str], filtered_ids: list[str]) -> list[str]: + """Return sorted IDs present in full but absent from filtered.""" + return sorted(set(full_ids) - set(filtered_ids)) + + def test_shadow_diff_detects_missed_tests(self) -> None: + """IDs in full but not in filtered are 'missed'.""" + full_ids = [ + "tests/core/test_core.py::test_a", + "tests/core/test_core.py::test_b", + "tests/execution/test_headless.py::test_c", + "tests/pipeline/test_gate.py::test_d", + "tests/server/test_init.py::test_e", + ] + filtered_ids = [ + "tests/core/test_core.py::test_a", + "tests/core/test_core.py::test_b", + "tests/server/test_init.py::test_e", + ] + assert self._missed(full_ids, filtered_ids) == [ + "tests/execution/test_headless.py::test_c", + "tests/pipeline/test_gate.py::test_d", + ] + + def test_shadow_diff_no_missed_tests(self) -> None: + """When filtered is a superset of full, no missed tests.""" + ids = [ + "tests/core/test_core.py::test_a", + "tests/core/test_core.py::test_b", + ] + assert self._missed(ids, ids) == [] + + def test_shadow_diff_empty_filtered(self) -> None: + """When filter selects nothing, all full IDs are missed.""" + full_ids = sorted(["tests/core/test_core.py::test_a", "tests/core/test_core.py::test_b"]) + assert self._missed(full_ids, []) == full_ids + + def test_shadow_conftest_has_workeroutput_propagation(self) -> None: + """Shadow conftest must define pytest_testnodedown and workeroutput propagation. + + Structural guard: ensures _CONFTEST_HOOKS_SOURCE stays in sync with the + production conftest's xdist IPC pathway. Without this guard the shadow could + silently lose the workeroutput mechanism, causing pytester-based xdist tests + to pass against stale shadow code that doesn't match production behavior. + """ + import ast + + tree = ast.parse(_CONFTEST_HOOKS_SOURCE) + func_names = {node.name for node in ast.walk(tree) if isinstance(node, ast.FunctionDef)} + assert "pytest_testnodedown" in func_names, ( + "Shadow conftest is missing pytest_testnodedown hook — " + "xdist worker-to-controller propagation not present" + ) + # Verify pytest_sessionfinish contains a workeroutput assignment + for node in ast.walk(tree): + if isinstance(node, ast.FunctionDef) and node.name == "pytest_sessionfinish": + func_src = ast.unparse(node) + assert "workeroutput" in func_src, ( + "pytest_sessionfinish in shadow conftest must assign workeroutput — " + "xdist IPC channel missing" + ) + break + else: + raise AssertionError("pytest_sessionfinish not found in shadow conftest") diff --git a/tests/test_test_filter_scope_extras.py b/tests/test_test_filter_scope_extras.py new file mode 100644 index 0000000000..8dcb920f0a --- /dev/null +++ b/tests/test_test_filter_scope_extras.py @@ -0,0 +1,51 @@ +"""Tests for file-level scope support in build_test_scope (AC3, AC4).""" + +from __future__ import annotations + +from pathlib import Path + +from tests._test_filter import FilterMode, build_test_scope + + +def test_build_test_scope_includes_file_entries_from_manifest(tmp_path: Path) -> None: + """AC3: Manifest entries pointing to files (not dirs) must be included in the scope.""" + tests_root = tmp_path / "tests" + tests_root.mkdir() + target_file = tests_root / "test_version.py" + target_file.write_text("def test_v(): pass\n") + sub_dir = tests_root / "contracts" + sub_dir.mkdir() + + manifest = {"some/config.json": ["contracts/", "test_version.py"]} + scope = build_test_scope( + {"some/config.json"}, + FilterMode.CONSERVATIVE, + manifest=manifest, + tests_root=str(tests_root), + ) + assert scope is not None + assert target_file in scope + + +def test_build_test_scope_skills_extended_manifest(tmp_path: Path) -> None: + """AC4: skills_extended/*/SKILL.md changes must include skills_extended/ in scope.""" + tests_root = tmp_path / "tests" + tests_root.mkdir() + (tests_root / "skills_extended").mkdir() + + manifest = { + "src/autoskillit/skills_extended/foo/SKILL.md": [ + "skills/", + "contracts/", + "recipe/", + "skills_extended/", + ] + } + scope = build_test_scope( + {"src/autoskillit/skills_extended/foo/SKILL.md"}, + FilterMode.CONSERVATIVE, + manifest=manifest, + tests_root=str(tests_root), + ) + assert scope is not None + assert tests_root / "skills_extended" in scope diff --git a/tests/test_test_filter_step7.py b/tests/test_test_filter_step7.py new file mode 100644 index 0000000000..3588752eb5 --- /dev/null +++ b/tests/test_test_filter_step7.py @@ -0,0 +1,209 @@ +"""Tests for build_test_scope step 7 — aggressive mode file-level filtering (S11–S17).""" + +from __future__ import annotations + +import os +import time +from pathlib import Path + +from tests._test_filter import ( + FilterMode, + build_test_scope, +) + + +class TestBuildTestScopeStep7: + def test_step7_file_level_substitution_single_file(self, tmp_path: Path) -> None: + """Coverage map entry replaces the cascade directory with specific test files.""" + tests_root = tmp_path / "tests" + (tests_root / "core").mkdir(parents=True) + (tests_root / "arch").mkdir() + (tests_root / "contracts").mkdir() + + specific_test = tests_root / "core" / "test_io.py" + specific_test.write_text("") + + map_file = tmp_path / "test-source-map.json" + map_file.write_text( + '{"src/autoskillit/core/io.py": ["tests/core/test_io.py"]}', + encoding="utf-8", + ) + + result = build_test_scope( + changed_files={"src/autoskillit/core/io.py"}, + mode=FilterMode.AGGRESSIVE, + tests_root=tests_root, + coverage_map_path=map_file, + ) + assert result is not None + assert not any(p.is_dir() and p.name == "core" for p in result) + paths = {str(p) for p in result} + assert any("test_io.py" in p for p in paths) + assert any("arch" in p for p in paths) + assert any("contracts" in p for p in paths) + + def test_step7_fallback_to_dir_when_file_not_in_map(self, tmp_path: Path) -> None: + """When src file has no coverage map entry, directory-level entry is preserved.""" + tests_root = tmp_path / "tests" + (tests_root / "core").mkdir(parents=True) + (tests_root / "arch").mkdir() + (tests_root / "contracts").mkdir() + + map_file = tmp_path / "test-source-map.json" + map_file.write_text( + '{"src/autoskillit/execution/headless.py": ["tests/execution/test_headless.py"]}', + encoding="utf-8", + ) + + result = build_test_scope( + changed_files={"src/autoskillit/core/io.py"}, + mode=FilterMode.AGGRESSIVE, + tests_root=tests_root, + coverage_map_path=map_file, + ) + assert result is not None + dir_names = {p.name for p in result if p.is_dir()} + assert "core" in dir_names + + def test_step7_no_path_uses_directory_level(self, tmp_path: Path) -> None: + """When coverage_map_path is None, step 7 is skipped entirely.""" + tests_root = tmp_path / "tests" + (tests_root / "core").mkdir(parents=True) + (tests_root / "arch").mkdir() + (tests_root / "contracts").mkdir() + + result = build_test_scope( + changed_files={"src/autoskillit/core/io.py"}, + mode=FilterMode.AGGRESSIVE, + tests_root=tests_root, + coverage_map_path=None, + ) + assert result is not None + dir_names = {p.name for p in result} + assert "core" in dir_names + + def test_step7_stale_oracle_falls_back_to_directory(self, tmp_path: Path) -> None: + """When load_coverage_map returns None (stale file), dir-level is preserved.""" + tests_root = tmp_path / "tests" + (tests_root / "core").mkdir(parents=True) + (tests_root / "arch").mkdir() + (tests_root / "contracts").mkdir() + + map_file = tmp_path / "test-source-map.json" + map_file.write_text( + '{"src/autoskillit/core/io.py": ["tests/core/test_io.py"]}', + encoding="utf-8", + ) + old_mtime = time.time() - (31 * 24 * 3600) + os.utime(map_file, (old_mtime, old_mtime)) + + result = build_test_scope( + changed_files={"src/autoskillit/core/io.py"}, + mode=FilterMode.AGGRESSIVE, + tests_root=tests_root, + coverage_map_path=map_file, + ) + assert result is not None + dir_names = {p.name for p in result if p.is_dir()} + assert "core" in dir_names + + def test_step7_conservative_mode_unaffected(self, tmp_path: Path) -> None: + """Conservative mode is completely unaffected by step 7 and coverage_map_path.""" + tests_root = tmp_path / "tests" + for d in [ + "core", + "config", + "execution", + "pipeline", + "workspace", + "recipe", + "migration", + "server", + "cli", + "hooks", + "skills", + "arch", + "contracts", + "infra", + "docs", + ]: + (tests_root / d).mkdir(parents=True, exist_ok=True) + + map_file = tmp_path / "test-source-map.json" + map_file.write_text( + '{"src/autoskillit/core/io.py": ["tests/core/test_io.py"]}', + encoding="utf-8", + ) + + result = build_test_scope( + changed_files={"src/autoskillit/core/io.py"}, + mode=FilterMode.CONSERVATIVE, + tests_root=tests_root, + coverage_map_path=map_file, + ) + assert result is not None + dir_names = {p.name for p in result} + for expected in [ + "core", + "config", + "execution", + "pipeline", + "workspace", + "recipe", + "migration", + "server", + "cli", + ]: + assert expected in dir_names, f"conservative cascade lost {expected}" + + def test_step7_mixed_coverage_keeps_directory(self, tmp_path: Path) -> None: + """If two src files map to same cascade dir and one lacks coverage data, dir is kept.""" + tests_root = tmp_path / "tests" + (tests_root / "core").mkdir(parents=True) + (tests_root / "arch").mkdir() + (tests_root / "contracts").mkdir() + + map_file = tmp_path / "test-source-map.json" + map_file.write_text( + '{"src/autoskillit/core/io.py": ["tests/core/test_io.py"]}', + encoding="utf-8", + ) + + result = build_test_scope( + changed_files={"src/autoskillit/core/io.py", "src/autoskillit/core/logging.py"}, + mode=FilterMode.AGGRESSIVE, + tests_root=tests_root, + coverage_map_path=map_file, + ) + assert result is not None + dir_names = {p.name for p in result if p.is_dir()} + assert "core" in dir_names + + def test_step7_all_files_covered_replaces_directory(self, tmp_path: Path) -> None: + """When all cascade-dir src files have oracle data, dir is replaced with specific files.""" + tests_root = tmp_path / "tests" + (tests_root / "core").mkdir(parents=True) + (tests_root / "arch").mkdir() + (tests_root / "contracts").mkdir() + (tests_root / "core" / "test_io.py").write_text("") + (tests_root / "core" / "test_logging.py").write_text("") + + map_file = tmp_path / "test-source-map.json" + map_file.write_text( + '{"src/autoskillit/core/io.py": ["tests/core/test_io.py"],' + ' "src/autoskillit/core/logging.py": ["tests/core/test_logging.py"]}', + encoding="utf-8", + ) + + result = build_test_scope( + changed_files={"src/autoskillit/core/io.py", "src/autoskillit/core/logging.py"}, + mode=FilterMode.AGGRESSIVE, + tests_root=tests_root, + coverage_map_path=map_file, + ) + assert result is not None + result_paths = list(result) + assert not any(p.is_dir() and p.name == "core" for p in result_paths) + names = {p.name for p in result_paths} + assert "test_io.py" in names + assert "test_logging.py" in names diff --git a/tests/test_test_filter_tiered_always_run.py b/tests/test_test_filter_tiered_always_run.py new file mode 100644 index 0000000000..f359ffd77a --- /dev/null +++ b/tests/test_test_filter_tiered_always_run.py @@ -0,0 +1,171 @@ +"""Tests for tiered conditional always-run logic (REQ-TIER-001 through REQ-TIER-005).""" + +from pathlib import Path + +from tests._test_filter import ( + _HOOKS_UNCONDITIONAL_FILES, + _INFRA_UNCONDITIONAL_FILES, + FilterMode, + build_test_scope, +) + + +def _make_tests_root(tmp_path: Path, dirs: list[str]) -> Path: + tests_root = tmp_path / "tests" + for d in dirs: + (tests_root / d).mkdir(parents=True, exist_ok=True) + return tests_root + + +ALL_DIRS = [ + "core", + "config", + "execution", + "pipeline", + "workspace", + "recipe", + "migration", + "server", + "cli", + "hooks", + "skills", + "arch", + "contracts", + "infra", + "docs", +] + + +class TestTieredAlwaysRun: + def test_pure_cli_change_skips_infra_dir_and_docs_dir(self, tmp_path: Path) -> None: + """REQ-TIER-001/002/003: cli change → arch+contracts present; infra/docs NOT as dirs; + 6 infra + 3 hooks unconditional files in result; test_doc_counts.py in result.""" + tests_root = _make_tests_root(tmp_path, ALL_DIRS) + result = build_test_scope( + changed_files={"src/autoskillit/cli/app.py"}, + mode=FilterMode.CONSERVATIVE, + tests_root=tests_root, + ) + assert result is not None + dir_names = {p.name for p in result if (tests_root / p.name).is_dir()} + assert "arch" in dir_names + assert "contracts" in dir_names + assert "infra" not in dir_names, "full infra dir should not appear for pure cli change" + assert "docs" not in dir_names, "docs dir should not appear for pure cli change" + result_names = {p.name for p in result} + for fname in _INFRA_UNCONDITIONAL_FILES: + assert fname in result_names, f"unconditional infra file {fname!r} missing" + for fname in _HOOKS_UNCONDITIONAL_FILES: + assert fname in result_names, f"unconditional hooks file {fname!r} missing" + # Verify parent directories + for p in result: + if p.name in _INFRA_UNCONDITIONAL_FILES: + assert p.parent.name == "infra", f"{p.name} expected under infra/" + if p.name in _HOOKS_UNCONDITIONAL_FILES: + assert p.parent.name == "hooks", f"{p.name} expected under hooks/" + assert "test_doc_counts.py" in result_names + + def test_docs_change_includes_docs_dir(self, tmp_path: Path) -> None: + """REQ-TIER-002: docs/ change → docs directory included.""" + tests_root = _make_tests_root(tmp_path, ALL_DIRS) + result = build_test_scope( + changed_files={"docs/developer/SETUP.md"}, + mode=FilterMode.CONSERVATIVE, + tests_root=tests_root, + manifest={"docs/**/*.md": ["docs"]}, + ) + assert result is not None + dir_names = {p.name for p in result} + assert "docs" in dir_names + + def test_hooks_change_includes_full_infra_dir(self, tmp_path: Path) -> None: + """REQ-TIER-003: hooks source change → full infra directory included.""" + tests_root = _make_tests_root(tmp_path, ALL_DIRS) + result = build_test_scope( + changed_files={"src/autoskillit/hooks/guards/quota_guard.py"}, + mode=FilterMode.CONSERVATIVE, + tests_root=tests_root, + ) + assert result is not None + dir_names = {p.name for p in result} + assert "infra" in dir_names + + def test_github_workflow_change_includes_full_infra_dir(self, tmp_path: Path) -> None: + """REQ-TIER-003: .github/ change → full infra directory included.""" + tests_root = _make_tests_root(tmp_path, ALL_DIRS) + result2 = build_test_scope( + changed_files={".github/workflows/ci.yml"}, + mode=FilterMode.CONSERVATIVE, + tests_root=tests_root, + manifest={".github/workflows/*.yml": ["infra"]}, + ) + assert result2 is not None + dir_names = {p.name for p in result2} + assert "infra" in dir_names + + def test_empty_changed_files_uses_full_always_run(self, tmp_path: Path) -> None: + """REQ-TIER-004: empty changed_files → fail-open → full always-run set as dirs.""" + tests_root = _make_tests_root(tmp_path, ALL_DIRS) + result = build_test_scope( + changed_files=set(), + mode=FilterMode.CONSERVATIVE, + tests_root=tests_root, + ) + assert result is not None + dir_names = {p.name for p in result} + for d in ["arch", "contracts", "infra", "docs"]: + assert d in dir_names, f"fail-open: {d} must be present for empty changeset" + + def test_unconditional_files_constants_have_correct_counts(self) -> None: + """_INFRA_UNCONDITIONAL_FILES has 6 entries; _HOOKS_UNCONDITIONAL_FILES has 3 entries.""" + assert len(_INFRA_UNCONDITIONAL_FILES) == 6 + assert len(_HOOKS_UNCONDITIONAL_FILES) == 3 + + def test_infra_unconditional_files_resolve_under_infra_dir(self, tmp_path: Path) -> None: + """Infra unconditional files must resolve to tests/infra/.""" + tests_root = _make_tests_root(tmp_path, ALL_DIRS) + # .touch() is load-bearing: build_test_scope resolves unconditional file paths + # by scanning the filesystem; files must exist for parent-path assertions to hold. + for fname in _INFRA_UNCONDITIONAL_FILES: + (tests_root / "infra" / fname).touch() + result = build_test_scope( + changed_files={"src/autoskillit/cli/app.py"}, + mode=FilterMode.CONSERVATIVE, + tests_root=tests_root, + ) + assert result is not None + infra_results = [p for p in result if p.name in _INFRA_UNCONDITIONAL_FILES] + assert len(infra_results) == len(_INFRA_UNCONDITIONAL_FILES) + for p in infra_results: + assert p.parent.name == "infra", f"{p.name} expected under infra/, got {p.parent.name}" + + def test_hook_unconditional_files_resolve_under_hooks_dir(self, tmp_path: Path) -> None: + """Hook unconditional files must resolve to tests/hooks/, not tests/infra/.""" + tests_root = _make_tests_root(tmp_path, ALL_DIRS) + # Create placeholder files so path assertions are meaningful + for fname in _HOOKS_UNCONDITIONAL_FILES: + (tests_root / "hooks" / fname).touch() + result = build_test_scope( + changed_files={"src/autoskillit/cli/app.py"}, + mode=FilterMode.CONSERVATIVE, + tests_root=tests_root, + ) + assert result is not None + hooks_results = [p for p in result if p.name in _HOOKS_UNCONDITIONAL_FILES] + assert len(hooks_results) == len(_HOOKS_UNCONDITIONAL_FILES) + for p in hooks_results: + assert p.parent.name == "hooks", f"{p.name} expected under hooks/, got {p.parent.name}" + + def test_aggressive_mode_unaffected_by_tiered_logic(self, tmp_path: Path) -> None: + """Tiered logic only applies to CONSERVATIVE mode; aggressive uses its own always-run.""" + tests_root = _make_tests_root(tmp_path, ["core", "arch", "contracts"]) + result = build_test_scope( + changed_files={"src/autoskillit/cli/app.py"}, + mode=FilterMode.AGGRESSIVE, + tests_root=tests_root, + ) + assert result is not None + dir_names = {p.name for p in result} + assert "arch" in dir_names + assert "contracts" in dir_names + assert "infra" not in dir_names # not in ALWAYS_RUN_AGGRESSIVE diff --git a/tests/test_version.py b/tests/test_version.py index 3ef5a05355..1ffbcdf366 100644 --- a/tests/test_version.py +++ b/tests/test_version.py @@ -18,7 +18,11 @@ def test_returns_expected_keys(self, tmp_path): from autoskillit.version import version_info info = version_info(plugin_dir=tmp_path) - assert set(info.keys()) == {"package_version", "plugin_json_version", "match"} + assert set(info.keys()) == { + "package_version", + "plugin_json_version", + "match", + } def test_missing_plugin_json(self, tmp_path): from autoskillit.version import version_info diff --git a/tests/workspace/CLAUDE.md b/tests/workspace/CLAUDE.md new file mode 100644 index 0000000000..a1e1fa7d50 --- /dev/null +++ b/tests/workspace/CLAUDE.md @@ -0,0 +1,33 @@ +# workspace/ + +Workspace cleanup, clone lifecycle, session skills, and worktree tests. + +## Files + +| File | Purpose | +|------|---------| +| `__init__.py` | empty | +| `conftest.py` | Shared fixtures for tests/workspace/ | +| `test_cleanup.py` | L1 unit tests for workspace/cleanup.py — CleanupResult and directory deletion | +| `test_clone_core.py` | Core clone_repo + remove_clone tests — setup, paths, error handling, origin contracts | +| `test_clone_detect.py` | detect_source_dir, detect_branch, detect_uncommitted_changes, classify_remote_url | +| `test_clone_push.py` | push_to_remote tests — E2E, mocked, protected branches, force-with-lease | +| `test_clone_remote.py` | Remote resolution — probe helpers, URL resolution, stale-clone regression | +| `test_clone_ci_contract.py` | Cross-boundary contract tests: clone isolation × CI/merge-queue resolution | +| `test_clone_registry.py` | Tests for autoskillit.workspace.clone_registry module | +| `test_clone_split.py` | Structural guard for clone test split | +| `test_clone_timeouts.py` | Static analysis: git network commands in clone.py must have timeouts | +| `test_constants.py` | Asserts that workspace directory name constants are exported from workspace/__init__ | +| `test_project_local_overrides.py` | Tests for project-local skill override detection and enforcement (T-OVR-001..011) | +| `test_session_skills_allow_only_and_closure.py` | Phase 2 tests: session_skills module — allow_only filter and compute_skill_closure | +| `test_session_skills_deps.py` | Phase 2 tests: session_skills module — activate_deps resolution | +| `test_session_skills_features.py` | Phase 2 tests: session_skills module — feature-gate skill filtering | +| `test_session_skills_filtering.py` | Phase 2 tests: session_skills module — subset/disabled-category and pack filtering | +| `test_session_skills_provider.py` | Phase 2 tests: session_skills module — provider and core manager | +| `test_skill_content_substitution.py` | Tests for SkillsDirectoryProvider.get_skill_content placeholder substitution | +| `test_skills.py` | Tests for skill resolution hierarchy | +| `test_worktree.py` | Worktree tests | + +## Architecture Notes + +`conftest.py` provides shared fixtures for workspace tests. The `test_clone_*.py` files are split by concern from the original test_clone.py: core clone_repo behavior, push_to_remote, remote resolution/probing, and detect helpers. The `test_session_skills_*.py` files are split by concern across five files testing different aspects of the `session_skills` module. diff --git a/tests/workspace/conftest.py b/tests/workspace/conftest.py index 3d7e11d5a9..25a31ea3e8 100644 --- a/tests/workspace/conftest.py +++ b/tests/workspace/conftest.py @@ -1,32 +1,95 @@ -"""Workspace test fixtures.""" +"""Shared fixtures for tests/workspace/.""" from __future__ import annotations import subprocess +from pathlib import Path import pytest -@pytest.fixture(scope="session") -def clone_isolation_repo(tmp_path_factory): - """ - Creates a clone-isolated git repo mirroring what clone_repo produces: - - origin → file://{clone_path} (isolation) - - upstream → https://github.com/testowner/testrepo.git (real remote) +@pytest.fixture +def bare_remote(tmp_path: Path) -> Path: + """Create a bare git remote (simulates GitHub/origin).""" + remote = tmp_path / "remote.git" + remote.mkdir() + subprocess.run(["git", "init", "--bare", str(remote)], check=True, capture_output=True) + return remote + - Session-scoped to avoid repeated git subprocess calls per test. - Use shutil.copytree in tests that mutate remotes. +@pytest.fixture +def local_with_remote(tmp_path: Path, bare_remote: Path) -> Path: + """Local repo with origin configured, main pushed, feature/local-only unpublished.""" + local = tmp_path / "local" + local.mkdir() + subprocess.run(["git", "init", str(local)], check=True, capture_output=True) + subprocess.run( + ["git", "-C", str(local), "config", "user.email", "t@t.com"], + check=True, + capture_output=True, + ) + subprocess.run( + ["git", "-C", str(local), "config", "user.name", "T"], + check=True, + capture_output=True, + ) + subprocess.run( + ["git", "-C", str(local), "remote", "add", "origin", str(bare_remote)], + check=True, + capture_output=True, + ) + subprocess.run( + ["git", "-C", str(local), "commit", "--allow-empty", "-m", "init"], + check=True, + capture_output=True, + ) + subprocess.run( + ["git", "-C", str(local), "branch", "-M", "main"], + check=True, + capture_output=True, + ) + subprocess.run( + ["git", "-C", str(local), "push", "-u", "origin", "main"], + check=True, + capture_output=True, + ) + # Create local-only branch (never pushed to origin) + subprocess.run( + ["git", "-C", str(local), "checkout", "-b", "feature/local-only"], + check=True, + capture_output=True, + ) + subprocess.run( + ["git", "-C", str(local), "commit", "--allow-empty", "-m", "local"], + check=True, + capture_output=True, + ) + return local + + +@pytest.fixture +def git_repo(tmp_path: Path) -> Path: + """Create a minimal git repo with one empty commit. + + Returns tmp_path / 'repo' (a subdirectory) so that clone_repo output lands at + tmp_path / 'autoskillit-runs' — inside the test's isolated tmp_path boundary. """ - base = tmp_path_factory.mktemp("clone_isolation") - repo = base / "clone" + repo = tmp_path / "repo" repo.mkdir() - subprocess.run(["git", "init", str(repo)], check=True) + subprocess.run(["git", "init", str(repo)], check=True, capture_output=True) subprocess.run( - ["git", "remote", "add", "origin", f"file://{base}/other"], cwd=str(repo), check=True + ["git", "-C", str(repo), "config", "user.email", "test@test.com"], + check=True, + capture_output=True, + ) + subprocess.run( + ["git", "-C", str(repo), "config", "user.name", "Test"], + check=True, + capture_output=True, ) subprocess.run( - ["git", "remote", "add", "upstream", "https://github.com/testowner/testrepo.git"], - cwd=str(repo), + ["git", "-C", str(repo), "commit", "--allow-empty", "-m", "init"], check=True, + capture_output=True, ) return repo diff --git a/tests/workspace/test_cleanup.py b/tests/workspace/test_cleanup.py index 99872bc992..93ef02b502 100644 --- a/tests/workspace/test_cleanup.py +++ b/tests/workspace/test_cleanup.py @@ -5,8 +5,12 @@ from pathlib import Path from unittest.mock import patch +import pytest + from autoskillit.workspace import CleanupResult, _delete_directory_contents +pytestmark = [pytest.mark.layer("workspace"), pytest.mark.small] + class TestCleanupResult: """CleanupResult dataclass contract.""" diff --git a/tests/workspace/test_clone.py b/tests/workspace/test_clone.py deleted file mode 100644 index f3c721abe5..0000000000 --- a/tests/workspace/test_clone.py +++ /dev/null @@ -1,1961 +0,0 @@ -"""Tests for autoskillit.workspace.clone module.""" - -from __future__ import annotations - -import inspect -import re -import shutil -import subprocess -from pathlib import Path -from unittest.mock import MagicMock, patch - -import pytest -import structlog.testing - -from autoskillit.workspace.clone import ( - DefaultCloneManager, - classify_remote_url, - clone_repo, - detect_branch, - detect_source_dir, - detect_uncommitted_changes, - push_to_remote, - remove_clone, -) - - -@pytest.fixture -def bare_remote(tmp_path: Path) -> Path: - """Create a bare git remote (simulates GitHub/origin).""" - remote = tmp_path / "remote.git" - remote.mkdir() - subprocess.run(["git", "init", "--bare", str(remote)], check=True, capture_output=True) - return remote - - -@pytest.fixture -def local_with_remote(tmp_path: Path, bare_remote: Path) -> Path: - """Local repo with origin configured, main pushed, feature/local-only unpublished.""" - local = tmp_path / "local" - local.mkdir() - subprocess.run(["git", "init", str(local)], check=True, capture_output=True) - subprocess.run( - ["git", "-C", str(local), "config", "user.email", "t@t.com"], - check=True, - capture_output=True, - ) - subprocess.run( - ["git", "-C", str(local), "config", "user.name", "T"], - check=True, - capture_output=True, - ) - subprocess.run( - ["git", "-C", str(local), "remote", "add", "origin", str(bare_remote)], - check=True, - capture_output=True, - ) - subprocess.run( - ["git", "-C", str(local), "commit", "--allow-empty", "-m", "init"], - check=True, - capture_output=True, - ) - subprocess.run( - ["git", "-C", str(local), "branch", "-M", "main"], - check=True, - capture_output=True, - ) - subprocess.run( - ["git", "-C", str(local), "push", "-u", "origin", "main"], - check=True, - capture_output=True, - ) - # Create local-only branch (never pushed to origin) - subprocess.run( - ["git", "-C", str(local), "checkout", "-b", "feature/local-only"], - check=True, - capture_output=True, - ) - subprocess.run( - ["git", "-C", str(local), "commit", "--allow-empty", "-m", "local"], - check=True, - capture_output=True, - ) - return local - - -@pytest.fixture -def git_repo(tmp_path: Path) -> Path: - """Create a minimal git repo with one empty commit. - - Returns tmp_path / 'repo' (a subdirectory) so that clone_repo output lands at - tmp_path / 'autoskillit-runs' — inside the test's isolated tmp_path boundary. - """ - repo = tmp_path / "repo" - repo.mkdir() - subprocess.run(["git", "init", str(repo)], check=True, capture_output=True) - subprocess.run( - ["git", "-C", str(repo), "config", "user.email", "test@test.com"], - check=True, - capture_output=True, - ) - subprocess.run( - ["git", "-C", str(repo), "config", "user.name", "Test"], - check=True, - capture_output=True, - ) - subprocess.run( - ["git", "-C", str(repo), "commit", "--allow-empty", "-m", "init"], - check=True, - capture_output=True, - ) - return repo - - -class TestCloneRepo: - def test_success_result_and_directory_layout(self, git_repo: Path) -> None: - """Clone creates a sibling directory with expected keys, parent, and git repo.""" - result = clone_repo(str(git_repo), "myrun", strategy="clone_local") - clone_path = Path(result["clone_path"]) - assert clone_path.is_dir() - assert (clone_path / ".git").is_dir() - assert "clone_path" in result - assert "source_dir" in result - assert "remote_url" in result - assert result["source_dir"] == str(git_repo.resolve()) - expected_parent = git_repo.parent / "autoskillit-runs" - assert clone_path.parent == expected_parent - - def test_clone_path_name_format(self, git_repo: Path) -> None: - """Clone directory name follows run_name-YYYYMMDD-HHMMSS-ffffff format.""" - result = clone_repo(str(git_repo), "myrun", strategy="clone_local") - clone_path = Path(result["clone_path"]) - assert re.match(r"myrun-\d{8}-\d{6}-\d{6}$", clone_path.name), clone_path.name - - def test_invalid_source_dir_raises_value_error(self, tmp_path: Path) -> None: - with pytest.raises(ValueError, match="source_dir does not exist"): - clone_repo("/nonexistent/path/that/does/not/exist", "name") - - def test_non_git_dir_raises_runtime_error(self, tmp_path: Path) -> None: - # tmp_path exists but is not a git repo — probe fails with reason="error" - with pytest.raises(RuntimeError, match="clone_origin_probe_failed"): - clone_repo(str(tmp_path), "name") - - def test_cb1_explicit_branch_is_checked_out_in_clone(self, git_repo: Path) -> None: - """T_CB1: explicit branch is checked out in the clone.""" - subprocess.run( - ["git", "-C", str(git_repo), "checkout", "-b", "dev"], - check=True, - capture_output=True, - ) - subprocess.run( - ["git", "-C", str(git_repo), "commit", "--allow-empty", "-m", "dev-commit"], - check=True, - capture_output=True, - ) - result = clone_repo(str(git_repo), "test", branch="dev", strategy="clone_local") - clone_path = Path(result["clone_path"]) - head = subprocess.run( - ["git", "-C", str(clone_path), "rev-parse", "--abbrev-ref", "HEAD"], - capture_output=True, - text=True, - check=True, - ).stdout.strip() - assert head == "dev" - - def test_cb2_auto_detects_current_branch_when_branch_empty(self, git_repo: Path) -> None: - """T_CB2: branch="" auto-detects current HEAD branch and clones it.""" - subprocess.run( - ["git", "-C", str(git_repo), "checkout", "-b", "feature"], - check=True, - capture_output=True, - ) - subprocess.run( - ["git", "-C", str(git_repo), "commit", "--allow-empty", "-m", "feature-commit"], - check=True, - capture_output=True, - ) - result = clone_repo(str(git_repo), "test", branch="", strategy="clone_local") - clone_path = Path(result["clone_path"]) - head = subprocess.run( - ["git", "-C", str(clone_path), "rev-parse", "--abbrev-ref", "HEAD"], - capture_output=True, - text=True, - check=True, - ).stdout.strip() - assert head == "feature" - - def test_cb3_returns_uncommitted_changes_warning_when_dirty(self, git_repo: Path) -> None: - """T_CB3: untracked files trigger warning dict; no clone is created.""" - (git_repo / "untracked.txt").write_text("dirty") - result = clone_repo(str(git_repo), "test") - assert result["uncommitted_changes"] == "true" - assert "changed_files" in result - assert "clone_path" not in result - - def test_cb4_strategy_proceed_skips_uncommitted_check(self, local_with_remote: Path) -> None: - """T_CB4: strategy='proceed' clones without uncommitted changes check.""" - (local_with_remote / "untracked.txt").write_text("dirty") - result = clone_repo(str(local_with_remote), "test", branch="main", strategy="proceed") - assert "clone_path" in result - clone_path = Path(result["clone_path"]) - assert not (clone_path / "untracked.txt").exists() - - def test_cb5_strategy_clone_local_includes_uncommitted_changes(self, git_repo: Path) -> None: - """T_CB5: strategy='clone_local' copytree includes untracked files.""" - (git_repo / "untracked.txt").write_text("dirty") - result = clone_repo(str(git_repo), "test", strategy="clone_local") - assert "clone_path" in result - clone_path = Path(result["clone_path"]) - assert (clone_path / "untracked.txt").exists() - - def test_cb6_detached_head_falls_back_to_no_branch_flag(self, git_repo: Path) -> None: - """T_CB6: detached HEAD yields branch=''; git clone succeeds without --branch.""" - sha = subprocess.run( - ["git", "-C", str(git_repo), "rev-parse", "HEAD"], - capture_output=True, - text=True, - check=True, - ).stdout.strip() - subprocess.run( - ["git", "-C", str(git_repo), "checkout", sha], - check=True, - capture_output=True, - ) - result = clone_repo(str(git_repo), "test", branch="", strategy="clone_local") - assert "clone_path" in result - clone_path = Path(result["clone_path"]) - assert clone_path.is_dir() - - def test_returns_unpublished_branch_warning_when_not_on_remote( - self, local_with_remote: Path - ) -> None: - """clone_repo returns unpublished_branch warning dict when branch has no origin ref.""" - result = clone_repo(str(local_with_remote), "test-run", branch="feature/local-only") - assert result.get("unpublished_branch") == "true" - assert result.get("branch") == "feature/local-only" - assert result.get("source_dir") == str(local_with_remote) - assert "clone_path" not in result # sentinel: no clone was created - - def test_published_branch_proceeds_to_clone(self, local_with_remote: Path) -> None: - """clone_repo clones normally when branch is on origin.""" - result = clone_repo(str(local_with_remote), "test-run", branch="main") - assert "clone_path" in result - assert "unpublished_branch" not in result - - def test_strategy_proceed_with_unpublished_branch_fails_from_remote( - self, local_with_remote: Path - ) -> None: - """strategy='proceed' with unpublished branch now fails — remote is always used. - - Previously 'proceed' bypassed the guard by silently falling back to the - local path. After the fix the clone always targets the remote, so a branch - that doesn't exist on the remote causes git clone to fail. - """ - with pytest.raises(RuntimeError, match="git clone failed"): - clone_repo( - str(local_with_remote), - "test-run", - branch="feature/local-only", - strategy="proceed", - ) - - -def test_clone_output_stays_within_test_isolation_boundary(tmp_path: Path, git_repo: Path) -> None: - """clone_repo output must be a descendant of tmp_path, never its parent. - - Fails when the git_repo fixture returns tmp_path itself, because clone_repo - places autoskillit-runs/ at source.parent = tmp_path.parent (worker-shared). - Passes once git_repo returns tmp_path / 'repo' (a subdirectory). - """ - result = clone_repo(str(git_repo), "isolation-check", strategy="clone_local") - clone_path = Path(result["clone_path"]) - assert clone_path.is_relative_to(tmp_path), ( - f"clone_repo placed output at {clone_path!r}, which is outside " - f"the test's tmp_path {tmp_path!r}.\n" - "The git_repo fixture must return a SUBDIRECTORY of tmp_path " - "(e.g. tmp_path / 'repo'), not tmp_path itself. " - "When git_repo is tmp_path, clone destination is tmp_path.parent — " - "a directory shared across all tests in the same xdist worker." - ) - shutil.rmtree(clone_path.parent, ignore_errors=True) - - -class TestCloneOriginContract: - """Contract: clone's origin is a unique file:// URL; real URL is in the upstream remote.""" - - def test_clone_origin_is_file_url_and_upstream_holds_real_url(self, tmp_path: Path) -> None: - """Clone's origin must be a file:// URL (not source_dir or bare remote). - - After the fix: clone's origin = file://, upstream = real network URL. - This prevents Claude Code from aliasing the clone session to the source project. - """ - bare_remote = tmp_path / "bare.git" - bare_remote.mkdir() - subprocess.run( - ["git", "init", "--bare", "--initial-branch=main", str(bare_remote)], check=True - ) - - source = tmp_path / "source" - subprocess.run(["git", "clone", str(bare_remote), str(source)], check=True) - subprocess.run(["git", "-C", str(source), "config", "user.email", "t@t.com"], check=True) - subprocess.run(["git", "-C", str(source), "config", "user.name", "T"], check=True) - (source / "README.md").write_text("hello") - subprocess.run(["git", "-C", str(source), "add", "."], check=True) - subprocess.run(["git", "-C", str(source), "commit", "-m", "init"], check=True) - src_branch = subprocess.run( - ["git", "-C", str(source), "rev-parse", "--abbrev-ref", "HEAD"], - capture_output=True, - text=True, - check=True, - ).stdout.strip() - subprocess.run(["git", "-C", str(source), "push", "origin", src_branch], check=True) - - result = clone_repo(str(source), "contract-test") - clone_path = Path(result["clone_path"]) - remote_url = result["remote_url"] - - try: - origin_in_clone = subprocess.run( - ["git", "remote", "get-url", "origin"], - cwd=str(clone_path), - capture_output=True, - text=True, - check=True, - ).stdout.strip() - upstream_in_clone = subprocess.run( - ["git", "remote", "get-url", "upstream"], - cwd=str(clone_path), - capture_output=True, - text=True, - check=True, - ).stdout.strip() - - # origin is now a local file URL unique to this clone - assert origin_in_clone.startswith("file://"), ( - f"Clone's origin ({origin_in_clone!r}) must be a file:// URL" - ) - assert origin_in_clone != str(source), ( - "Clone's origin must not be the local source_dir path" - ) - # upstream holds the real push target - assert upstream_in_clone == remote_url, ( - f"Clone's upstream ({upstream_in_clone!r}) should equal" - f" remote_url ({remote_url!r})" - ) - finally: - shutil.rmtree(clone_path.parent, ignore_errors=True) - - def test_clone_local_strategy_rewrites_origin_to_file_url(self, tmp_path: Path) -> None: - """clone_local strategy (copytree) rewrites origin to file:// unconditionally. - - Covers the #377 compounding regression: the isolation rewrite was previously - skipped when effective_url was empty (no remote origin). After the fix, - _ensure_origin_isolated fires unconditionally for every successful clone. - """ - source = tmp_path / "source" - subprocess.run(["git", "init", str(source)], check=True) - subprocess.run(["git", "-C", str(source), "config", "user.email", "t@t.com"], check=True) - subprocess.run(["git", "-C", str(source), "config", "user.name", "T"], check=True) - (source / "f.txt").write_text("x") - subprocess.run(["git", "-C", str(source), "add", "."], check=True) - subprocess.run(["git", "-C", str(source), "commit", "-m", "init"], check=True) - - result = clone_repo(str(source), "no-upstream-test", strategy="clone_local") - try: - clone_path = Path(result["clone_path"]) - origin_in_clone = subprocess.run( - ["git", "-C", str(clone_path), "remote", "get-url", "origin"], - check=True, - capture_output=True, - text=True, - ).stdout.strip() - assert origin_in_clone == f"file://{clone_path}", ( - f"Expected file://{clone_path!r}, got {origin_in_clone!r}" - ) - finally: - shutil.rmtree(Path(result["clone_path"]).parent, ignore_errors=True) - - def test_clone_local_strategy_also_sets_correct_remotes(self, tmp_path: Path) -> None: - """clone_local strategy (copytree) must also set origin=file:// and upstream=real URL. - - The rewrite applies to both the proceed (git clone) and clone_local (copytree) paths. - """ - bare_remote = tmp_path / "bare.git" - bare_remote.mkdir() - subprocess.run(["git", "init", "--bare", str(bare_remote)], check=True) - source = tmp_path / "source" - subprocess.run(["git", "clone", str(bare_remote), str(source)], check=True) - subprocess.run(["git", "-C", str(source), "config", "user.email", "t@t.com"], check=True) - subprocess.run(["git", "-C", str(source), "config", "user.name", "T"], check=True) - (source / "README.md").write_text("hello") - subprocess.run(["git", "-C", str(source), "add", "."], check=True) - subprocess.run(["git", "-C", str(source), "commit", "-m", "init"], check=True) - - result = clone_repo(str(source), "clone-local-contract", strategy="clone_local") - clone_path = Path(result["clone_path"]) - remote_url = result["remote_url"] - - try: - origin_in_clone = subprocess.run( - ["git", "remote", "get-url", "origin"], - cwd=str(clone_path), - capture_output=True, - text=True, - check=True, - ).stdout.strip() - upstream_in_clone = subprocess.run( - ["git", "remote", "get-url", "upstream"], - cwd=str(clone_path), - capture_output=True, - text=True, - check=True, - ).stdout.strip() - - assert origin_in_clone.startswith("file://"), ( - f"clone_local origin ({origin_in_clone!r}) must be a file:// URL" - ) - assert upstream_in_clone == remote_url, ( - f"clone_local upstream ({upstream_in_clone!r}) should equal" - f" remote_url ({remote_url!r})" - ) - finally: - shutil.rmtree(clone_path.parent, ignore_errors=True) - - -class TestCloneRepoRemoteUrlOverride: - """Tests for the remote_url parameter on clone_repo (T_RU1, T_RU2).""" - - def _make_source_with_bare_remote(self, tmp_path: Path) -> tuple[Path, Path]: - """Helper: create a bare remote and source repo pointing to it.""" - bare = tmp_path / "bare.git" - bare.mkdir() - subprocess.run(["git", "init", "--bare", str(bare)], check=True, capture_output=True) - - source = tmp_path / "source" - subprocess.run(["git", "clone", str(bare), str(source)], check=True, capture_output=True) - subprocess.run( - ["git", "-C", str(source), "config", "user.email", "t@t.com"], - check=True, - capture_output=True, - ) - subprocess.run( - ["git", "-C", str(source), "config", "user.name", "T"], - check=True, - capture_output=True, - ) - (source / "f.txt").write_text("x") - subprocess.run(["git", "-C", str(source), "add", "."], check=True, capture_output=True) - subprocess.run( - ["git", "-C", str(source), "commit", "-m", "init"], - check=True, - capture_output=True, - ) - src_branch = subprocess.run( - ["git", "-C", str(source), "rev-parse", "--abbrev-ref", "HEAD"], - capture_output=True, - text=True, - check=True, - ).stdout.strip() - subprocess.run( - ["git", "-C", str(source), "push", "origin", src_branch], - check=True, - capture_output=True, - ) - return source, bare - - def test_clone_repo_remote_url_override_applied(self, tmp_path: Path) -> None: - """When remote_url is provided, clone's upstream is set to that URL (T_RU1). - - After the fix: origin = file://, upstream = override_url. - """ - source, _bare = self._make_source_with_bare_remote(tmp_path) - override_url = "https://github.com/example/repo.git" - - result = clone_repo(str(source), "test-run", strategy="proceed", remote_url=override_url) - assert "clone_path" in result - clone_path = Path(result["clone_path"]) - - try: - actual_origin = subprocess.run( - ["git", "remote", "get-url", "origin"], - cwd=str(clone_path), - capture_output=True, - text=True, - ).stdout.strip() - actual_upstream = subprocess.run( - ["git", "remote", "get-url", "upstream"], - cwd=str(clone_path), - capture_output=True, - text=True, - ).stdout.strip() - # origin is now a local file URL unique to this clone - assert actual_origin.startswith("file://") - # upstream holds the real push target (the override URL) - assert actual_upstream == override_url - assert result["remote_url"] == override_url - finally: - shutil.rmtree(clone_path.parent, ignore_errors=True) - - def test_clone_repo_without_remote_url_uses_detected(self, tmp_path: Path) -> None: - """Without remote_url, clone's upstream is set to the source's detected origin (T_RU2). - - After the fix: origin = file://, upstream = detected bare remote path. - """ - source, bare = self._make_source_with_bare_remote(tmp_path) - - result = clone_repo(str(source), "test-run", strategy="proceed") - assert "clone_path" in result - clone_path = Path(result["clone_path"]) - - try: - actual_origin = subprocess.run( - ["git", "remote", "get-url", "origin"], - cwd=str(clone_path), - capture_output=True, - text=True, - ).stdout.strip() - actual_upstream = subprocess.run( - ["git", "remote", "get-url", "upstream"], - cwd=str(clone_path), - capture_output=True, - text=True, - ).stdout.strip() - # origin is now a local file URL unique to this clone - assert actual_origin.startswith("file://") - # upstream holds the real push target (the detected bare remote) - assert actual_upstream == str(bare) - finally: - shutil.rmtree(clone_path.parent, ignore_errors=True) - - -class TestRemoveClone: - def test_keep_false_removes_directory(self, git_repo: Path) -> None: - result = clone_repo(str(git_repo), "test", strategy="clone_local") - clone_path = result["clone_path"] - remove_result = remove_clone(clone_path, keep="false") - assert remove_result == {"removed": "true"} - assert not Path(clone_path).exists() - # Cleanup parent runs dir - import shutil - - shutil.rmtree(Path(clone_path), ignore_errors=True) - - def test_keep_true_preserves_directory(self, git_repo: Path) -> None: - result = clone_repo(str(git_repo), "test", strategy="clone_local") - clone_path = result["clone_path"] - remove_result = remove_clone(clone_path, keep="true") - assert remove_result == {"removed": "false", "reason": "keep=true"} - assert Path(clone_path).exists() - # Cleanup - import shutil - - shutil.rmtree(Path(clone_path), ignore_errors=True) - - def test_missing_path_returns_not_found(self) -> None: - result = remove_clone("/nonexistent/clone/path", keep="false") - assert result == {"removed": "false", "reason": "not_found"} - - -class TestPushToRemote: - def test_push_to_remote_propagates_to_upstream(self, tmp_path: Path) -> None: - # 1. Create bare remote (simulates GitHub) - remote = tmp_path / "remote.git" - subprocess.run(["git", "init", "--bare", str(remote)], check=True, capture_output=True) - - # 2. Clone remote into source (simulates user's local checkout) - source = tmp_path / "source" - subprocess.run(["git", "clone", str(remote), str(source)], check=True, capture_output=True) - subprocess.run( - ["git", "-C", str(source), "config", "user.email", "t@t.com"], - check=True, - capture_output=True, - ) - subprocess.run( - ["git", "-C", str(source), "config", "user.name", "T"], - check=True, - capture_output=True, - ) - subprocess.run( - ["git", "-C", str(source), "commit", "--allow-empty", "-m", "init"], - check=True, - capture_output=True, - ) - src_branch = subprocess.run( - ["git", "-C", str(source), "rev-parse", "--abbrev-ref", "HEAD"], - capture_output=True, - text=True, - check=True, - ).stdout.strip() - subprocess.run( - ["git", "-C", str(source), "push", "origin", src_branch], - check=True, - capture_output=True, - ) - - # 3. Pipeline clones source - clone_result = clone_repo(str(source), "pushtest") - clone_path = clone_result["clone_path"] - - # 4. Make a commit in pipeline-clone - subprocess.run( - ["git", "-C", clone_path, "config", "user.email", "t@t.com"], - check=True, - capture_output=True, - ) - subprocess.run( - ["git", "-C", clone_path, "config", "user.name", "T"], - check=True, - capture_output=True, - ) - subprocess.run( - ["git", "-C", clone_path, "commit", "--allow-empty", "-m", "pipeline-commit"], - check=True, - capture_output=True, - ) - branch = subprocess.run( - ["git", "-C", clone_path, "rev-parse", "--abbrev-ref", "HEAD"], - capture_output=True, - text=True, - check=True, - ).stdout.strip() - - # Record source HEAD before push (must not change) - source_head_before = subprocess.run( - ["git", "-C", str(source), "rev-parse", "HEAD"], - capture_output=True, - text=True, - check=True, - ).stdout.strip() - - result = push_to_remote(clone_path, str(source), branch, protected_branches=[]) - assert result["success"] is True - - # Commit landed in remote - remote_log = subprocess.run( - ["git", "-C", str(remote), "log", "--oneline", "-3"], - capture_output=True, - text=True, - check=True, - ).stdout - assert "pipeline-commit" in remote_log - - # source_dir HEAD unchanged - source_head_after = subprocess.run( - ["git", "-C", str(source), "rev-parse", "HEAD"], - capture_output=True, - text=True, - check=True, - ).stdout.strip() - assert source_head_before == source_head_after - - def test_push_to_remote_establishes_tracking_ref(self, tmp_path: Path) -> None: - """push_to_remote must establish a tracking ref so remove_clone_guard passes.""" - remote = tmp_path / "remote.git" - subprocess.run(["git", "init", "--bare", str(remote)], check=True, capture_output=True) - source = tmp_path / "source" - subprocess.run(["git", "clone", str(remote), str(source)], check=True, capture_output=True) - subprocess.run( - ["git", "-C", str(source), "config", "user.email", "t@t.com"], - check=True, - capture_output=True, - ) - subprocess.run( - ["git", "-C", str(source), "config", "user.name", "T"], - check=True, - capture_output=True, - ) - subprocess.run( - ["git", "-C", str(source), "commit", "--allow-empty", "-m", "init"], - check=True, - capture_output=True, - ) - src_branch = subprocess.run( - ["git", "-C", str(source), "rev-parse", "--abbrev-ref", "HEAD"], - capture_output=True, - text=True, - check=True, - ).stdout.strip() - subprocess.run( - ["git", "-C", str(source), "push", "origin", src_branch], - check=True, - capture_output=True, - ) - - clone_result = clone_repo(str(source), "tracktest") - clone_path = clone_result["clone_path"] - subprocess.run( - ["git", "-C", clone_path, "config", "user.email", "t@t.com"], - check=True, - capture_output=True, - ) - subprocess.run( - ["git", "-C", clone_path, "config", "user.name", "T"], - check=True, - capture_output=True, - ) - subprocess.run( - ["git", "-C", clone_path, "commit", "--allow-empty", "-m", "pipeline-commit"], - check=True, - capture_output=True, - ) - - result = push_to_remote(clone_path, str(source), src_branch, protected_branches=[]) - assert result["success"] is True - - upstream_rc = subprocess.run( - ["git", "-C", clone_path, "rev-parse", "@{upstream}"], - capture_output=True, - text=True, - ).returncode - assert upstream_rc == 0, "@{upstream} must be set after push_to_remote" - shutil.rmtree(Path(clone_path).parent, ignore_errors=True) - - def test_push_to_remote_fails_when_source_has_no_origin(self, tmp_path: Path) -> None: - source = tmp_path / "source" - source.mkdir() - subprocess.run(["git", "init", str(source)], check=True, capture_output=True) - - result = push_to_remote("/nonexistent/clone", str(source), "main", protected_branches=[]) - assert result["success"] is False - assert len(result["stderr"]) > 0 - - -class TestDetectSourceDir: - def test_ds1_returns_git_toplevel(self) -> None: - """T_DS1: returns git rev-parse --show-toplevel when returncode=0.""" - mock_result = MagicMock() - mock_result.returncode = 0 - mock_result.stdout = "/repo/root\n" - with patch("subprocess.run", return_value=mock_result): - assert detect_source_dir("/any/cwd") == "/repo/root" - - def test_ds2_falls_back_on_nonzero_returncode(self) -> None: - """T_DS2: returns cwd unchanged when git exits non-zero.""" - mock_result = MagicMock() - mock_result.returncode = 128 - mock_result.stdout = "" - with patch("subprocess.run", return_value=mock_result): - assert detect_source_dir("/any/cwd") == "/any/cwd" - - -class TestCloneRepoDetectAndExpand: - def test_ds4_expands_tilde(self) -> None: - """T_DS4: clone_repo raises ValueError with 'resolved to' when tilde path doesn't exist.""" - with pytest.raises(ValueError, match="resolved to"): - clone_repo("~/nonexistent-autoskillit-test-xyz", "test-run") - - def test_ds5_raises_value_error_with_clear_message(self, tmp_path) -> None: - """T_DS5: non-existent path raises ValueError with 'resolved to' in message.""" - nonexistent = str(tmp_path / "does-not-exist") - with pytest.raises(ValueError, match="resolved to"): - clone_repo(nonexistent, "test-run") - - def test_cb7_calls_detect_branch_when_branch_empty(self, tmp_path) -> None: - """T_CB7: detect_branch is called with source_dir when branch="".""" - with patch( - "autoskillit.workspace.clone.detect_branch", return_value="main" - ) as mock_detect: - with patch("autoskillit.workspace.clone.detect_uncommitted_changes", return_value=[]): - mock_clone = MagicMock() - mock_clone.returncode = 0 - with patch("subprocess.run", return_value=mock_clone): - clone_repo(str(tmp_path), "test-run", branch="") - mock_detect.assert_called_once_with(str(tmp_path)) - - def test_cb9_passes_branch_flag_to_git(self, tmp_path) -> None: - """T_CB9: --branch and branch name appear in the git clone subprocess call.""" - with patch("autoskillit.workspace.clone.detect_uncommitted_changes", return_value=[]): - mock_clone = MagicMock() - mock_clone.returncode = 0 - with patch("subprocess.run", return_value=mock_clone) as mock_run: - clone_repo(str(tmp_path), "test-run", branch="dev") - git_clone_calls = [ - call for call in mock_run.call_args_list if call.args and "clone" in call.args[0] - ] - assert git_clone_calls, "git clone was not called" - cmd = git_clone_calls[0].args[0] - assert "--branch" in cmd - assert "dev" in cmd - - def test_cb10_returns_warning_dict_on_uncommitted_changes(self, tmp_path) -> None: - """T_CB10: uncommitted changes produce warning dict; git clone not called.""" - with patch("autoskillit.workspace.clone.detect_branch", return_value="main"): - with patch( - "autoskillit.workspace.clone.detect_uncommitted_changes", - return_value=[" M file.py"], - ): - with patch("subprocess.run") as mock_run: - result = clone_repo(str(tmp_path), "test-run") - assert result["uncommitted_changes"] == "true" - mock_run.assert_not_called() - - -class TestDetectBranch: - def test_cb11_returns_branch_name_on_success(self) -> None: - """T_CB11: returns branch name when git rev-parse succeeds.""" - mock_result = MagicMock() - mock_result.returncode = 0 - mock_result.stdout = "main\n" - with patch("subprocess.run", return_value=mock_result): - assert detect_branch("/any") == "main" - - def test_cb12_returns_empty_string_on_nonzero_returncode(self) -> None: - """T_CB12: returns "" when git exits non-zero.""" - mock_result = MagicMock() - mock_result.returncode = 128 - mock_result.stdout = "" - with patch("subprocess.run", return_value=mock_result): - assert detect_branch("/any") == "" - - def test_cb13_returns_head_literal_for_detached_state(self) -> None: - """T_CB13: returns literal 'HEAD' in detached HEAD state; caller treats as no branch.""" - mock_result = MagicMock() - mock_result.returncode = 0 - mock_result.stdout = "HEAD\n" - with patch("subprocess.run", return_value=mock_result): - assert detect_branch("/any") == "HEAD" - - -class TestDetectUncommittedChanges: - def test_cb14_returns_empty_list_when_clean(self) -> None: - """T_CB14: returns [] when working tree is clean.""" - mock_result = MagicMock() - mock_result.returncode = 0 - mock_result.stdout = "" - with patch("subprocess.run", return_value=mock_result): - assert detect_uncommitted_changes("/any") == [] - - def test_cb15_returns_changed_file_lines_when_dirty(self) -> None: - """T_CB15: returns non-empty status lines when changes exist.""" - mock_result = MagicMock() - mock_result.returncode = 0 - mock_result.stdout = " M file.py\n?? new.txt\n" - with patch("subprocess.run", return_value=mock_result): - assert detect_uncommitted_changes("/any") == [" M file.py", "?? new.txt"] - - def test_cb16_returns_empty_list_on_git_failure(self) -> None: - """T_CB16: returns [] when git exits non-zero.""" - mock_result = MagicMock() - mock_result.returncode = 128 - with patch("subprocess.run", return_value=mock_result): - assert detect_uncommitted_changes("/any") == [] - - -class TestPushToRemoteProtectedBranch: - """push_to_remote rejects pushes to protected branches.""" - - @pytest.mark.parametrize("branch", ["main", "integration", "stable"]) - def test_push_to_remote_rejects_protected_branch(self, tmp_path: Path, branch: str) -> None: - """push_to_remote must reject when branch is a protected branch.""" - clone = tmp_path / "clone" - clone.mkdir() - - result = push_to_remote( - clone_path=str(clone), - branch=branch, - remote_url="https://github.com/example/repo.git", - protected_branches=["main", "integration", "stable"], - ) - - assert result["success"] is False - assert result.get("error_type") == "protected_branch_push" - - -class TestPushToRemoteMocked: - def test_ds6_push_to_remote_calls_get_url_then_push(self) -> None: - """T_DS6: push_to_remote calls git remote get-url origin then git push -u upstream.""" - mock_url = MagicMock() - mock_url.returncode = 0 - mock_url.stdout = "git@github.com:org/repo.git\n" - mock_url.stderr = "" - - mock_push = MagicMock() - mock_push.returncode = 0 - mock_push.stderr = "" - - with patch("subprocess.run", side_effect=[mock_url, mock_push]) as mock_run: - result = push_to_remote( - "/clone", "/source", "main", protected_branches=[], force=False - ) - - assert result == {"success": True, "stderr": ""} - # First call: git remote get-url origin from source_dir - first_call = mock_run.call_args_list[0] - assert first_call[0][0] == ["git", "remote", "get-url", "origin"] - assert first_call[1]["cwd"] == "/source" - # Second call: git push -u upstream from clone_path — no --force-with-lease - second_call = mock_run.call_args_list[1] - assert second_call[0][0] == ["git", "push", "-u", "upstream", "main"] - assert second_call[1]["cwd"] == "/clone" - - def test_ds7_push_to_remote_fails_when_no_origin(self) -> None: - """T_DS7: push_to_remote returns error when git remote get-url fails, no push attempted.""" - mock_fail = MagicMock() - mock_fail.returncode = 128 - mock_fail.stdout = "" - mock_fail.stderr = "error: No such remote 'origin'" - - with patch("subprocess.run", return_value=mock_fail) as mock_run: - result = push_to_remote("/clone", "/source", "main", protected_branches=[]) - - assert result["success"] is False - assert "origin" in result["stderr"] - assert mock_run.call_count == 1 # no push attempted - - def test_push_to_remote_with_force_injects_force_with_lease(self) -> None: - """T1: push_to_remote with force=True appends --force-with-lease to push command.""" - mock_url = MagicMock() - mock_url.returncode = 0 - mock_url.stdout = "git@github.com:org/repo.git\n" - mock_url.stderr = "" - - mock_push = MagicMock() - mock_push.returncode = 0 - mock_push.stderr = "" - - with patch("subprocess.run", side_effect=[mock_url, mock_push]) as mock_run: - result = push_to_remote("/clone", "/source", "main", protected_branches=[], force=True) - - assert result == {"success": True, "stderr": ""} - second_call = mock_run.call_args_list[1] - cmd = second_call[0][0] - assert "--force-with-lease" in cmd - assert "git" in cmd - assert "push" in cmd - assert "upstream" in cmd - assert "main" in cmd - assert second_call[1]["cwd"] == "/clone" - - def test_push_to_remote_default_force_false_does_not_inject_lease(self) -> None: - """T2: push_to_remote with force=False (default) does not inject --force-with-lease.""" - mock_url = MagicMock() - mock_url.returncode = 0 - mock_url.stdout = "git@github.com:org/repo.git\n" - mock_url.stderr = "" - - mock_push = MagicMock() - mock_push.returncode = 0 - mock_push.stderr = "" - - with patch("subprocess.run", side_effect=[mock_url, mock_push]) as mock_run: - result = push_to_remote( - "/clone", "/source", "main", protected_branches=[], force=False - ) - - assert result == {"success": True, "stderr": ""} - second_call = mock_run.call_args_list[1] - cmd = second_call[0][0] - assert cmd == ["git", "push", "-u", "upstream", "main"] - - def test_default_clone_manager_push_to_remote_accepts_force_param(self) -> None: - """T5: DefaultCloneManager.push_to_remote has force keyword param with default False.""" - sig = inspect.signature(DefaultCloneManager.push_to_remote) - assert "force" in sig.parameters, ( - "DefaultCloneManager.push_to_remote must have 'force' param" - ) - param = sig.parameters["force"] - assert param.default is False, "force param must default to False" - - def test_force_with_lease_stale_returns_error_type(self) -> None: - """push_to_remote returns error_type=force_with_lease_stale when git reports stale info.""" - mock_url = MagicMock() - mock_url.returncode = 0 - mock_url.stdout = "git@github.com:org/repo.git\n" - mock_url.stderr = "" - - mock_push = MagicMock() - mock_push.returncode = 1 - mock_push.stderr = "! [rejected] main -> main (stale info)" - - with patch("subprocess.run", side_effect=[mock_url, mock_push]): - result = push_to_remote("/clone", "/source", "main", protected_branches=[], force=True) - - assert result["success"] is False - assert result.get("error_type") == "force_with_lease_stale" - - def test_force_with_lease_no_upstream_returns_error_type(self) -> None: - """push_to_remote returns error_type=force_with_lease_no_upstream for missing upstream.""" - mock_url = MagicMock() - mock_url.returncode = 0 - mock_url.stdout = "git@github.com:org/repo.git\n" - mock_url.stderr = "" - - mock_push = MagicMock() - mock_push.returncode = 1 - mock_push.stderr = "error: The current branch main has no upstream branch." - - with patch("subprocess.run", side_effect=[mock_url, mock_push]): - result = push_to_remote("/clone", "/source", "main", protected_branches=[], force=True) - - assert result["success"] is False - assert result.get("error_type") == "force_with_lease_no_upstream" - - def test_force_push_generic_failure_has_no_error_type(self) -> None: - """push_to_remote returns no error_type for generic force-push failures.""" - mock_url = MagicMock() - mock_url.returncode = 0 - mock_url.stdout = "git@github.com:org/repo.git\n" - mock_url.stderr = "" - - mock_push = MagicMock() - mock_push.returncode = 1 - mock_push.stderr = "error: failed to push some refs" - - with patch("subprocess.run", side_effect=[mock_url, mock_push]): - result = push_to_remote("/clone", "/source", "main", protected_branches=[], force=True) - - assert result["success"] is False - assert "error_type" not in result - - def test_non_force_failure_has_no_error_type(self) -> None: - """push_to_remote returns no error_type for non-force push failures.""" - mock_url = MagicMock() - mock_url.returncode = 0 - mock_url.stdout = "git@github.com:org/repo.git\n" - mock_url.stderr = "" - - mock_push = MagicMock() - mock_push.returncode = 1 - mock_push.stderr = "! [rejected] main -> main (stale info)" - - with patch("subprocess.run", side_effect=[mock_url, mock_push]): - result = push_to_remote( - "/clone", "/source", "main", protected_branches=[], force=False - ) - - assert result["success"] is False - assert "error_type" not in result - - -class TestPushToRemoteNonBare: - """push_to_remote fails with error_type when remote is a local non-bare repo.""" - - def test_push_fails_with_local_nonbare_remote(self, tmp_path: Path) -> None: - """push_to_remote returns error_type=local_non_bare_remote for non-bare local origin.""" - # upstream is a non-bare local repo with main checked out - upstream = tmp_path / "upstream" - subprocess.run(["git", "init", str(upstream)], check=True, capture_output=True) - subprocess.run( - ["git", "-C", str(upstream), "config", "user.email", "t@t.com"], - check=True, - capture_output=True, - ) - subprocess.run( - ["git", "-C", str(upstream), "config", "user.name", "T"], - check=True, - capture_output=True, - ) - subprocess.run( - ["git", "-C", str(upstream), "commit", "--allow-empty", "-m", "init"], - check=True, - capture_output=True, - ) - - source = tmp_path / "source" - subprocess.run( - ["git", "clone", str(upstream), str(source)], check=True, capture_output=True - ) - subprocess.run( - ["git", "-C", str(source), "config", "user.email", "t@t.com"], - check=True, - capture_output=True, - ) - subprocess.run( - ["git", "-C", str(source), "config", "user.name", "T"], - check=True, - capture_output=True, - ) - subprocess.run( - ["git", "-C", str(source), "commit", "--allow-empty", "-m", "src"], - check=True, - capture_output=True, - ) - - # upstream has main checked out — push from source will be refused - clone_result = clone_repo(str(source), "test-nonbare", strategy="proceed") - clone_path = clone_result["clone_path"] - subprocess.run( - ["git", "-C", clone_path, "config", "user.email", "t@t.com"], - check=True, - capture_output=True, - ) - subprocess.run( - ["git", "-C", clone_path, "config", "user.name", "T"], - check=True, - capture_output=True, - ) - subprocess.run( - ["git", "-C", clone_path, "commit", "--allow-empty", "-m", "pipeline"], - check=True, - capture_output=True, - ) - - result = push_to_remote(clone_path, str(source), "main", protected_branches=[]) - - assert result["success"] is False - assert result.get("error_type") == "local_non_bare_remote" - - import shutil - - shutil.rmtree(Path(clone_path).parent, ignore_errors=True) - - def test_clone_repo_returns_remote_url(self, tmp_path: Path) -> None: - """clone_repo result dict includes remote_url field.""" - remote = tmp_path / "remote.git" - subprocess.run( - ["git", "init", "--bare", "--initial-branch=main", str(remote)], - check=True, - capture_output=True, - ) - source = tmp_path / "source" - subprocess.run(["git", "clone", str(remote), str(source)], check=True, capture_output=True) - subprocess.run( - ["git", "-C", str(source), "config", "user.email", "t@t.com"], - check=True, - capture_output=True, - ) - subprocess.run( - ["git", "-C", str(source), "config", "user.name", "T"], - check=True, - capture_output=True, - ) - subprocess.run( - ["git", "-C", str(source), "commit", "--allow-empty", "-m", "init"], - check=True, - capture_output=True, - ) - subprocess.run( - ["git", "-C", str(source), "push", "-u", "origin", "HEAD:main"], - check=True, - capture_output=True, - ) - - result = clone_repo(str(source), "test-remoteurl", strategy="proceed") - - assert "remote_url" in result - assert str(remote) in result["remote_url"] - - # After the fix: origin is a file:// URL, upstream holds the real remote_url - origin_in_clone = subprocess.run( - ["git", "remote", "get-url", "origin"], - cwd=result["clone_path"], - capture_output=True, - text=True, - check=True, - ).stdout.strip() - upstream_in_clone = subprocess.run( - ["git", "remote", "get-url", "upstream"], - cwd=result["clone_path"], - capture_output=True, - text=True, - check=True, - ).stdout.strip() - assert origin_in_clone.startswith("file://"), ( - "Clone's origin must be a file:// URL after the fix" - ) - assert upstream_in_clone == result["remote_url"], ( - "Clone's upstream must equal the returned remote_url" - ) - - import shutil - - shutil.rmtree(Path(result["clone_path"]).parent, ignore_errors=True) - - def test_clone_repo_local_strategy_returns_discriminator_when_no_origin( - self, tmp_path: Path - ) -> None: - """clone_local strategy returns discriminator with clone_source_type=local.""" - source = tmp_path / "source" - subprocess.run(["git", "init", str(source)], check=True, capture_output=True) - subprocess.run( - ["git", "-C", str(source), "config", "user.email", "t@t.com"], - check=True, - capture_output=True, - ) - subprocess.run( - ["git", "-C", str(source), "config", "user.name", "T"], - check=True, - capture_output=True, - ) - subprocess.run( - ["git", "-C", str(source), "commit", "--allow-empty", "-m", "init"], - check=True, - capture_output=True, - ) - - result = clone_repo(str(source), "test-noremote", strategy="clone_local") - - assert "clone_path" in result - assert result["remote_url"] == "" - assert result["clone_source_type"] == "local" - assert result["clone_source_reason"] == "strategy_clone_local" - assert Path(result["clone_path"]).exists() - - import shutil - - shutil.rmtree(Path(result["clone_path"]).parent, ignore_errors=True) - - def test_push_to_remote_uses_explicit_remote_url_without_reading_source_dir( - self, tmp_path: Path - ) -> None: - """When remote_url is explicit, source_dir is not accessed for URL lookup.""" - remote = tmp_path / "remote.git" - subprocess.run(["git", "init", "--bare", str(remote)], check=True, capture_output=True) - source = tmp_path / "source" - subprocess.run(["git", "clone", str(remote), str(source)], check=True, capture_output=True) - subprocess.run( - ["git", "-C", str(source), "config", "user.email", "t@t.com"], - check=True, - capture_output=True, - ) - subprocess.run( - ["git", "-C", str(source), "config", "user.name", "T"], - check=True, - capture_output=True, - ) - subprocess.run( - ["git", "-C", str(source), "commit", "--allow-empty", "-m", "init"], - check=True, - capture_output=True, - ) - subprocess.run( - ["git", "-C", str(source), "branch", "-M", "main"], - check=True, - capture_output=True, - ) - subprocess.run( - ["git", "-C", str(source), "push", "origin", "main"], - check=True, - capture_output=True, - ) - - clone_result = clone_repo(str(source), "test-explicit", strategy="proceed") - clone_path = clone_result["clone_path"] - subprocess.run( - ["git", "-C", clone_path, "config", "user.email", "t@t.com"], - check=True, - capture_output=True, - ) - subprocess.run( - ["git", "-C", clone_path, "config", "user.name", "T"], - check=True, - capture_output=True, - ) - subprocess.run( - ["git", "-C", clone_path, "commit", "--allow-empty", "-m", "impl"], - check=True, - capture_output=True, - ) - - # Pass explicit remote_url — source_dir is not needed - result = push_to_remote( - clone_path, remote_url=str(remote), branch="main", protected_branches=[] - ) - - assert result["success"] is True - - import shutil - - shutil.rmtree(Path(clone_path).parent, ignore_errors=True) - - -class TestClassifyRemoteUrl: - """classify_remote_url correctly identifies remote URL types.""" - - @pytest.mark.parametrize( - "url,expected_type", - [ - ("git@github.com:org/repo.git", "network"), - ("https://github.com/org/repo.git", "network"), - ("ssh://git@github.com/org/repo.git", "network"), - ("http://example.com/repo.git", "network"), - ("git://github.com/org/repo.git", "network"), - ("file:///tmp/repo.git", "network"), - ], - ) - def test_network_urls(self, url: str, expected_type: str) -> None: - result = classify_remote_url(url) - assert result == expected_type - - def test_bare_local_path(self, tmp_path: Path) -> None: - import subprocess - - bare = tmp_path / "repo.git" - subprocess.run(["git", "init", "--bare", str(bare)], check=True, capture_output=True) - result = classify_remote_url(str(bare)) - assert result == "bare_local" - - def test_nonbare_local_path(self, tmp_path: Path) -> None: - import subprocess - - repo = tmp_path / "repo" - subprocess.run(["git", "init", str(repo)], check=True, capture_output=True) - result = classify_remote_url(str(repo)) - assert result == "nonbare_local" - - def test_empty_string_returns_none_type(self) -> None: - result = classify_remote_url("") - assert result == "none" - - def test_nonexistent_path_returns_unknown(self, tmp_path: Path) -> None: - result = classify_remote_url(str(tmp_path / "nonexistent")) - assert result == "unknown" - - -class TestCloneDecontamination: - """clone_repo strips tracked and on-disk generated files from clones.""" - - def test_clone_repo_untracks_inherited_generated_files(self, tmp_path: Path) -> None: - """Tracked generated files in source are untracked in the clone.""" - repo = tmp_path / "repo" - repo.mkdir() - subprocess.run(["git", "init", str(repo)], check=True, capture_output=True) - subprocess.run( - ["git", "-C", str(repo), "config", "user.email", "t@t.com"], - check=True, - capture_output=True, - ) - subprocess.run( - ["git", "-C", str(repo), "config", "user.name", "T"], - check=True, - capture_output=True, - ) - # Create and track a generated file - hooks_dir = repo / "src" / "autoskillit" / "hooks" - hooks_dir.mkdir(parents=True) - (hooks_dir / "hooks.json").write_text('{"hooks": {}}') - subprocess.run( - ["git", "-C", str(repo), "add", "src/autoskillit/hooks/hooks.json"], - check=True, - capture_output=True, - ) - subprocess.run( - ["git", "-C", str(repo), "commit", "-m", "add generated file"], - check=True, - capture_output=True, - ) - - result = clone_repo(str(repo), "decontam-test", strategy="clone_local") - clone_path = Path(result["clone_path"]) - try: - ls_result = subprocess.run( - ["git", "ls-files", "--", "src/autoskillit/hooks/hooks.json"], - cwd=str(clone_path), - capture_output=True, - text=True, - ) - assert ls_result.stdout.strip() == "", "Generated file should be untracked in clone" - finally: - shutil.rmtree(clone_path.parent, ignore_errors=True) - - def test_clone_repo_deletes_untracked_generated_files_from_disk(self, tmp_path: Path) -> None: - """clone_local copies untracked generated files; decontamination deletes them.""" - repo = tmp_path / "repo" - repo.mkdir() - subprocess.run(["git", "init", str(repo)], check=True, capture_output=True) - subprocess.run( - ["git", "-C", str(repo), "config", "user.email", "t@t.com"], - check=True, - capture_output=True, - ) - subprocess.run( - ["git", "-C", str(repo), "config", "user.name", "T"], - check=True, - capture_output=True, - ) - subprocess.run( - ["git", "-C", str(repo), "commit", "--allow-empty", "-m", "init"], - check=True, - capture_output=True, - ) - # Create on-disk generated file WITHOUT tracking it - hooks_dir = repo / "src" / "autoskillit" / "hooks" - hooks_dir.mkdir(parents=True) - (hooks_dir / "hooks.json").write_text('{"hooks": {}}') - - result = clone_repo(str(repo), "disk-cleanup-test", strategy="clone_local") - clone_path = Path(result["clone_path"]) - try: - assert not (clone_path / "src" / "autoskillit" / "hooks" / "hooks.json").exists(), ( - "Untracked generated file should be deleted from clone disk" - ) - finally: - shutil.rmtree(clone_path.parent, ignore_errors=True) - - def test_clone_repo_noop_when_no_generated_files_tracked(self, git_repo: Path) -> None: - """Clean repo with no tracked generated files clones without errors.""" - result = clone_repo(str(git_repo), "clean-test", strategy="clone_local") - clone_path = Path(result["clone_path"]) - try: - assert clone_path.is_dir() - assert (clone_path / ".git").is_dir() - finally: - shutil.rmtree(clone_path.parent, ignore_errors=True) - - -class TestCloneRemoteUrlResolution: - """T1: clone_repo resolves remote URL before cloning when origin is configured.""" - - # T1-A - def test_clone_uses_remote_url_as_clone_source( - self, local_with_remote: Path, bare_remote: Path - ) -> None: - """clone_repo uses the remote URL (not the local path) as git clone source.""" - with patch( - "autoskillit.workspace.clone.subprocess.run", - wraps=subprocess.run, - ) as spy: - result = clone_repo(str(local_with_remote), "test", branch="main", strategy="proceed") - clone_path = Path(result["clone_path"]) - try: - clone_calls = [ - call - for call in spy.call_args_list - if call[0] and isinstance(call[0][0], list) and call[0][0][:2] == ["git", "clone"] - ] - assert len(clone_calls) == 1, ( - f"Expected exactly one git clone call, got {len(clone_calls)}" - ) - clone_args = clone_calls[0][0][0] - # source is second-to-last positional arg (before clone_path) - clone_source = clone_args[-2] - assert clone_source == str(bare_remote), ( - f"Expected clone source to be remote URL {bare_remote!r}, " - f"got {clone_source!r} (local path was used instead)" - ) - assert result["clone_source_type"] == "remote", ( - f"Expected clone_source_type='remote', got {result.get('clone_source_type')!r}" - ) - finally: - shutil.rmtree(clone_path.parent, ignore_errors=True) - - # T1-B - def test_clone_raises_on_no_origin_with_proceed_strategy(self, git_repo: Path) -> None: - """clone_repo raises RuntimeError when no remote origin and strategy=proceed. - - Previously the code silently fell back to cloning from the local path - (git clone /abs/path via local transport). After the fix, the no_origin - probe result causes an immediate RuntimeError, instructing the caller to - use strategy="clone_local" for an intentional local-only clone. - """ - with pytest.raises(RuntimeError, match="clone_origin_probe_failed.*no_origin"): - clone_repo(str(git_repo), "test", strategy="proceed") - - # T1-C - def test_clone_result_remote_url_correct_after_remote_clone( - self, local_with_remote: Path, bare_remote: Path - ) -> None: - """clone_repo result['remote_url'] equals the remote URL after cloning.""" - result = clone_repo(str(local_with_remote), "test", branch="main", strategy="proceed") - clone_path = Path(result["clone_path"]) - try: - assert result["remote_url"] == str(bare_remote) - assert result["clone_source_type"] == "remote" - assert result["clone_source_reason"] == "ok" - finally: - shutil.rmtree(clone_path.parent, ignore_errors=True) - - # T1-D - def test_clone_uses_remote_when_branch_not_on_remote( - self, local_with_remote: Path, bare_remote: Path - ) -> None: - """Regression: clone always uses remote URL even when branch not on remote. - - Previously the origin probe collapse allowed falling back to the local path - when ls-remote did not find the branch on the remote. After the fix the probe - always uses the remote URL when one is configured, so git clone fails (correctly) - instead of silently cloning local state. - """ - with patch( - "autoskillit.workspace.clone.subprocess.run", - wraps=subprocess.run, - ) as spy: - with pytest.raises(RuntimeError, match="git clone failed"): - clone_repo( - str(local_with_remote), - "test", - branch="feature/local-only", - strategy="proceed", - ) - clone_calls = [ - call - for call in spy.call_args_list - if call[0] and isinstance(call[0][0], list) and call[0][0][:2] == ["git", "clone"] - ] - assert len(clone_calls) == 1, "Expected exactly one git clone call" - cmd = clone_calls[0][0][0] - # Extract positional args from git clone, skipping flags and their values. - # Handles any optional flags (--branch, --no-hardlinks, --depth, etc.) - # without relying on a fixed index like [-2]. - positional: list[str] = [] - i = cmd.index("clone") + 1 - while i < len(cmd): - if cmd[i].startswith("-"): - i += 2 # skip flag and its value - else: - positional.append(cmd[i]) - i += 1 - assert len(positional) == 2, f"Expected [source, target] in clone cmd: {cmd}" - clone_source = positional[0] - assert clone_source == str(bare_remote), ( - f"Expected remote URL {bare_remote!r} as clone source, " - f"got {clone_source!r} — local path fallback detected" - ) - - -class TestProbeSingleRemote: - """Unit tests for _probe_single_remote helper.""" - - def test_probe_single_remote_returns_ok_reason_when_remote_configured( - self, local_with_remote: Path, bare_remote: Path - ) -> None: - from autoskillit.workspace.clone import _probe_single_remote - - resolution = _probe_single_remote(local_with_remote, "origin") - assert resolution.reason == "ok" - assert resolution.url == str(bare_remote) - assert resolution.stderr == "" - - def test_probe_single_remote_returns_no_origin_reason_when_no_remote( - self, git_repo: Path - ) -> None: - from autoskillit.workspace.clone import _probe_single_remote - - resolution = _probe_single_remote(git_repo, "origin") - assert resolution.reason == "no_origin" - assert resolution.url == "" - - def test_probe_single_remote_returns_timeout_reason_on_timeout(self, tmp_path: Path) -> None: - from autoskillit.workspace.clone import _probe_single_remote - - with patch( - "autoskillit.workspace.clone.subprocess.run", - side_effect=subprocess.TimeoutExpired(cmd=["git"], timeout=30), - ): - resolution = _probe_single_remote(tmp_path, "origin") - assert resolution.reason == "timeout" - assert resolution.url == "" - - def test_probe_single_remote_returns_error_reason_on_non_zero_rc(self, tmp_path: Path) -> None: - from autoskillit.workspace.clone import _probe_single_remote - - mock_result = MagicMock() - mock_result.returncode = 1 - mock_result.stdout = "" - mock_result.stderr = "fatal: not a git repository" - with patch("autoskillit.workspace.clone.subprocess.run", return_value=mock_result): - resolution = _probe_single_remote(tmp_path, "origin") - assert resolution.reason == "error" - assert resolution.url == "" - assert resolution.stderr == "fatal: not a git repository" - - -class TestProbeCloneSourceUrl: - """Unit tests for the updated _probe_clone_source_url URL resolution logic.""" - - def test_prefers_upstream_network_url_over_file_origin(self, tmp_path: Path) -> None: - """When origin=file:// and upstream=network URL, uses upstream (the key bug fix). - - This is the exact scenario that caused stale clones in multi-batch pipelines: - source_dir is a previous autoskillit clone with origin rewritten to file://. - """ - from autoskillit.workspace.clone import _probe_clone_source_url - - bare = tmp_path / "bare.git" - subprocess.run(["git", "init", "--bare", str(bare)], check=True, capture_output=True) - source = tmp_path / "source" - subprocess.run(["git", "init", str(source)], check=True, capture_output=True) - subprocess.run( - ["git", "-C", str(source), "config", "user.email", "t@t.com"], - check=True, - capture_output=True, - ) - subprocess.run( - ["git", "-C", str(source), "config", "user.name", "T"], check=True, capture_output=True - ) - subprocess.run( - ["git", "-C", str(source), "commit", "--allow-empty", "-m", "init"], - check=True, - capture_output=True, - ) - # Simulate a previous _ensure_origin_isolated call: origin=file://, upstream=real remote - subprocess.run( - ["git", "-C", str(source), "remote", "add", "origin", f"file://{source}"], - check=True, - capture_output=True, - ) - subprocess.run( - ["git", "-C", str(source), "remote", "add", "upstream", str(bare)], - check=True, - capture_output=True, - ) - - result = _probe_clone_source_url(source) - - assert result.reason == "ok" - assert result.url == str(bare), ( - f"Expected upstream URL {bare!r}, got {result.url!r}. " - "When origin=file://, upstream should be preferred." - ) - - def test_falls_back_to_origin_when_no_upstream(self, tmp_path: Path) -> None: - """Without upstream remote, falls back to origin URL (existing behavior preserved).""" - from autoskillit.workspace.clone import _probe_clone_source_url - - bare = tmp_path / "bare.git" - subprocess.run(["git", "init", "--bare", str(bare)], check=True, capture_output=True) - source = tmp_path / "source" - subprocess.run(["git", "clone", str(bare), str(source)], capture_output=True) - subprocess.run( - ["git", "-C", str(source), "config", "user.email", "t@t.com"], - check=True, - capture_output=True, - ) - subprocess.run( - ["git", "-C", str(source), "config", "user.name", "T"], check=True, capture_output=True - ) - subprocess.run( - ["git", "-C", str(source), "commit", "--allow-empty", "-m", "init"], - check=True, - capture_output=True, - ) - # Only origin is configured, no upstream - - result = _probe_clone_source_url(source) - - assert result.reason == "ok" - assert result.url == str(bare) - - def test_uses_origin_when_upstream_is_file_url_and_origin_is_network( - self, tmp_path: Path - ) -> None: - """When upstream=file:// and origin=non-file-local-path that is network-equivalent, - falls through to origin result (covers edge cases where upstream is also local).""" - from autoskillit.workspace.clone import _probe_clone_source_url - - bare = tmp_path / "bare.git" - subprocess.run(["git", "init", "--bare", str(bare)], check=True, capture_output=True) - source = tmp_path / "source" - subprocess.run(["git", "init", str(source)], check=True, capture_output=True) - subprocess.run( - ["git", "-C", str(source), "config", "user.email", "t@t.com"], - check=True, - capture_output=True, - ) - subprocess.run( - ["git", "-C", str(source), "config", "user.name", "T"], check=True, capture_output=True - ) - subprocess.run( - ["git", "-C", str(source), "commit", "--allow-empty", "-m", "init"], - check=True, - capture_output=True, - ) - # upstream is a file:// URL (not a real network URL), origin is the bare path - subprocess.run( - ["git", "-C", str(source), "remote", "add", "upstream", f"file://{source}"], - check=True, - capture_output=True, - ) - subprocess.run( - ["git", "-C", str(source), "remote", "add", "origin", str(bare)], - check=True, - capture_output=True, - ) - - result = _probe_clone_source_url(source) - - # upstream is file:// → excluded by _is_not_file_url → falls through to origin - assert result.reason == "ok" - assert result.url == str(bare) - - def test_returns_no_origin_for_repo_without_remotes(self, tmp_path: Path) -> None: - """Repo with no remotes returns reason='no_origin' (unchanged from current behavior).""" - from autoskillit.workspace.clone import _probe_clone_source_url - - source = tmp_path / "source" - subprocess.run(["git", "init", str(source)], check=True, capture_output=True) - subprocess.run( - ["git", "-C", str(source), "config", "user.email", "t@t.com"], - check=True, - capture_output=True, - ) - subprocess.run( - ["git", "-C", str(source), "config", "user.name", "T"], check=True, capture_output=True - ) - subprocess.run( - ["git", "-C", str(source), "commit", "--allow-empty", "-m", "init"], - check=True, - capture_output=True, - ) - - result = _probe_clone_source_url(source) - - assert result.reason == "no_origin" - assert result.url == "" - - -class TestCloneFromPreviousAutoskillitClone: - """Regression: cloning from a previous autoskillit clone must use the network remote.""" - - def test_clone_from_previous_clone_uses_upstream_not_stale_local(self, tmp_path: Path) -> None: - """Batch N+1 clones from batch N's clone must get fresh HEAD from the network remote, - not the stale local state of the previous clone. - - Setup: - bare_remote (has commit A + commit B) - source (has commit A only — stale) - source has: origin=file://source (isolation), upstream=bare_remote - - Expected: clone_from_source gets commit B (fetched from bare_remote via upstream). - Bug behavior (pre-fix): gets only commit A (cloned from file://source via origin). - """ - bare_remote = tmp_path / "bare.git" - subprocess.run( - ["git", "init", "--bare", "--initial-branch=main", str(bare_remote)], - check=True, - capture_output=True, - ) - - # source: has commit A, stale (does not have commit B) - source = tmp_path / "source" - subprocess.run(["git", "init", str(source)], check=True, capture_output=True) - subprocess.run( - ["git", "-C", str(source), "config", "user.email", "t@t.com"], - check=True, - capture_output=True, - ) - subprocess.run( - ["git", "-C", str(source), "config", "user.name", "T"], check=True, capture_output=True - ) - (source / "a.txt").write_text("commit A") - subprocess.run(["git", "-C", str(source), "add", "."], check=True, capture_output=True) - subprocess.run( - ["git", "-C", str(source), "commit", "-m", "commit A"], check=True, capture_output=True - ) - subprocess.run( - ["git", "-C", str(source), "branch", "-M", "main"], check=True, capture_output=True - ) - subprocess.run( - ["git", "-C", str(source), "push", str(bare_remote), "main"], - check=True, - capture_output=True, - ) - - # Simulate _ensure_origin_isolated: source.origin = file://, source.upstream = bare_remote - subprocess.run( - ["git", "-C", str(source), "remote", "add", "origin", f"file://{source}"], - check=True, - capture_output=True, - ) - subprocess.run( - ["git", "-C", str(source), "remote", "add", "upstream", str(bare_remote)], - check=True, - capture_output=True, - ) - - # Add commit B to bare_remote directly (simulates another batch merging) - tmp_push = tmp_path / "push_helper" - subprocess.run( - ["git", "clone", str(bare_remote), str(tmp_push)], check=True, capture_output=True - ) - subprocess.run( - ["git", "-C", str(tmp_push), "config", "user.email", "t@t.com"], - check=True, - capture_output=True, - ) - subprocess.run( - ["git", "-C", str(tmp_push), "config", "user.name", "T"], - check=True, - capture_output=True, - ) - (tmp_push / "b.txt").write_text("commit B") - subprocess.run(["git", "-C", str(tmp_push), "add", "."], check=True, capture_output=True) - subprocess.run( - ["git", "-C", str(tmp_push), "commit", "-m", "commit B"], - check=True, - capture_output=True, - ) - subprocess.run( - ["git", "-C", str(tmp_push), "push", "origin", "main"], check=True, capture_output=True - ) - - # Now clone from source (which is stale — missing commit B) - result = clone_repo(str(source), "batch2", branch="main", strategy="proceed") - clone_path = Path(result["clone_path"]) - - try: - # The clone MUST have b.txt (from bare_remote commit B), not just a.txt - assert (clone_path / "b.txt").exists(), ( - "Clone is missing b.txt — it cloned from the stale local source instead of " - "the network remote (bare_remote). This is the #817 regression." - ) - finally: - shutil.rmtree(clone_path.parent, ignore_errors=True) - shutil.rmtree(tmp_push, ignore_errors=True) - - -class TestCloneOriginProbeFailFast: - """Tests 2A-2C: clone_repo raises RuntimeError when origin probe fails.""" - - def test_clone_repo_raises_on_timeout_when_source_has_origin( - self, local_with_remote: Path - ) -> None: - """The reported bug reproduced: timeout on origin probe raises RuntimeError. - - Patches subprocess.run to raise TimeoutExpired only on get-url origin - while delegating all other calls to the real subprocess.run. - """ - _real_run = subprocess.run - - def _selective_side_effect(cmd, **kwargs): # type: ignore[no-untyped-def] - if isinstance(cmd, list) and cmd == ["git", "remote", "get-url", "origin"]: - raise subprocess.TimeoutExpired(cmd=cmd, timeout=30) - return _real_run(cmd, **kwargs) - - with patch( - "autoskillit.workspace.clone.subprocess.run", - side_effect=_selective_side_effect, - ): - with structlog.testing.capture_logs() as cap: - with pytest.raises(RuntimeError, match="clone_origin_probe_failed.*timeout"): - clone_repo(str(local_with_remote), "t", strategy="proceed") - - warning_events = [e for e in cap if e.get("event") == "clone_origin_probe_failed"] - assert warning_events, "Expected clone_origin_probe_failed warning to be emitted" - assert warning_events[0].get("reason") == "timeout" - - def test_clone_repo_raises_on_non_zero_get_url_when_source_has_origin( - self, local_with_remote: Path - ) -> None: - """Non-zero returncode on get-url origin raises RuntimeError.""" - _real_run = subprocess.run - - def _selective_side_effect(cmd, **kwargs): # type: ignore[no-untyped-def] - if isinstance(cmd, list) and cmd == ["git", "remote", "get-url", "origin"]: - mock = MagicMock() - mock.returncode = 1 - mock.stdout = "" - mock.stderr = "fatal: some error" - return mock - return _real_run(cmd, **kwargs) - - with patch( - "autoskillit.workspace.clone.subprocess.run", - side_effect=_selective_side_effect, - ): - with pytest.raises(RuntimeError, match="clone_origin_probe_failed"): - clone_repo(str(local_with_remote), "t", strategy="proceed") - - def test_clone_repo_raises_on_no_origin_without_explicit_local_strategy( - self, git_repo: Path - ) -> None: - """clone_repo raises RuntimeError when source has no origin and strategy=proceed. - - Error message must instruct the caller to pass strategy="clone_local". - """ - with pytest.raises( - RuntimeError, - match=r"clone_origin_probe_failed.*no_origin.*strategy=.?clone_local", - ): - clone_repo(str(git_repo), "t", strategy="proceed") - - -class TestCloneDiscriminator: - """Tests 3A-3B and 5: typed return contract with clone_source_type discriminator.""" - - def test_clone_local_strategy_succeeds_on_no_origin_with_discriminator( - self, git_repo: Path - ) -> None: - """strategy=clone_local returns clone_source_type=local with discriminator.""" - result = clone_repo(str(git_repo), "t", strategy="clone_local") - clone_path = Path(result["clone_path"]) - try: - assert result["clone_source_type"] == "local" - assert result["clone_source_reason"] == "strategy_clone_local" - assert result["remote_url"] == "" - assert clone_path.exists() - finally: - shutil.rmtree(clone_path.parent, ignore_errors=True) - - def test_clone_proceed_strategy_succeeds_with_remote_discriminator( - self, local_with_remote: Path, bare_remote: Path - ) -> None: - """strategy=proceed returns clone_source_type=remote with discriminator.""" - result = clone_repo(str(local_with_remote), "t", branch="main", strategy="proceed") - clone_path = Path(result["clone_path"]) - try: - assert result["clone_source_type"] == "remote" - assert result["clone_source_reason"] == "ok" - assert result["remote_url"] == str(bare_remote) - finally: - shutil.rmtree(clone_path.parent, ignore_errors=True) - - @pytest.mark.parametrize( - "fixture_name,strategy,expected_source_type", - [ - ("local_with_remote", "proceed", "remote"), - ("git_repo", "clone_local", "local"), - ], - ) - def test_clone_result_always_has_discriminator_key( - self, - fixture_name: str, - strategy: str, - expected_source_type: str, - request: pytest.FixtureRequest, - ) -> None: - """Structural invariant: every success result carries clone_source_type.""" - source = request.getfixturevalue(fixture_name) - kwargs = {"strategy": strategy} - if fixture_name == "local_with_remote": - kwargs["branch"] = "main" - result = clone_repo(str(source), "t", **kwargs) # type: ignore[arg-type] - clone_path = Path(result["clone_path"]) - try: - assert result["clone_source_type"] == expected_source_type - assert "clone_source_reason" in result - finally: - shutil.rmtree(clone_path.parent, ignore_errors=True) - - -class TestOriginIsolationUnconditional: - """Tests 4A-4B: _ensure_origin_isolated fires unconditionally.""" - - def test_clone_local_strategy_still_rewrites_origin_to_file_url(self, git_repo: Path) -> None: - """clone_local (no remote) rewrites origin to file:// — covers #377 regression. - - Inverts the deleted test_clone_origin_unchanged_when_no_upstream. - """ - result = clone_repo(str(git_repo), "t", strategy="clone_local") - clone_path = Path(result["clone_path"]) - try: - origin = subprocess.run( - ["git", "-C", str(clone_path), "remote", "get-url", "origin"], - check=True, - capture_output=True, - text=True, - ).stdout.strip() - assert origin == f"file://{clone_path}", ( - f"Expected file://{clone_path!r}, got {origin!r}" - ) - finally: - shutil.rmtree(clone_path.parent, ignore_errors=True) - - def test_clone_remote_strategy_rewrites_origin_to_file_url( - self, local_with_remote: Path - ) -> None: - """clone via remote URL also rewrites origin to file:// unconditionally.""" - result = clone_repo(str(local_with_remote), "t", branch="main", strategy="proceed") - clone_path = Path(result["clone_path"]) - try: - origin = subprocess.run( - ["git", "-C", str(clone_path), "remote", "get-url", "origin"], - check=True, - capture_output=True, - text=True, - ).stdout.strip() - assert origin == f"file://{clone_path}", ( - f"Expected file://{clone_path!r}, got {origin!r}" - ) - finally: - shutil.rmtree(clone_path.parent, ignore_errors=True) diff --git a/tests/workspace/test_clone_ci_contract.py b/tests/workspace/test_clone_ci_contract.py index 47884c3da4..985a313b3d 100644 --- a/tests/workspace/test_clone_ci_contract.py +++ b/tests/workspace/test_clone_ci_contract.py @@ -17,6 +17,8 @@ from autoskillit.execution import resolve_remote_repo from autoskillit.workspace import clone_repo +pytestmark = [pytest.mark.layer("workspace"), pytest.mark.medium] + # --------------------------------------------------------------------------- # Primary cross-boundary integration test # --------------------------------------------------------------------------- @@ -64,38 +66,3 @@ async def test_resolve_remote_repo_after_clone_uses_upstream(tmp_path: Path) -> assert resolved == "testowner/testrepo" finally: shutil.rmtree(Path(clone_path).parent, ignore_errors=True) - - -# --------------------------------------------------------------------------- -# Regression guard: infer_repo_from_remote with file:// origin -# --------------------------------------------------------------------------- - - -@pytest.mark.anyio -async def test_infer_repo_from_remote_returns_empty_for_file_url( - clone_isolation_repo: Path, - tmp_path: Path, -) -> None: - """Regression guard: file:// origin, no upstream → infer_repo_from_remote returns ''. - - Uses a temporary copy of clone_isolation_repo so the session-scoped - fixture is not mutated by removing the upstream remote. - """ - import shutil - - from autoskillit.server.helpers import infer_repo_from_remote - - work_copy = tmp_path / "work_copy" - shutil.copytree(clone_isolation_repo, work_copy) - try: - # Remove the upstream remote so only file:// origin remains - subprocess.run( - ["git", "remote", "remove", "upstream"], - cwd=str(work_copy), - check=True, - ) - - result = await infer_repo_from_remote(str(work_copy)) - assert result == "" - finally: - shutil.rmtree(work_copy, ignore_errors=True) diff --git a/tests/workspace/test_clone_core.py b/tests/workspace/test_clone_core.py new file mode 100644 index 0000000000..4f8452239b --- /dev/null +++ b/tests/workspace/test_clone_core.py @@ -0,0 +1,873 @@ +"""Core clone_repo tests — setup, paths, error handling, origin contracts.""" + +from __future__ import annotations + +import re +import shutil +import subprocess +from pathlib import Path +from unittest.mock import MagicMock, patch + +import pytest +import structlog.testing + +from autoskillit.workspace.clone import ( + clone_repo, + remove_clone, +) + +pytestmark = [pytest.mark.layer("workspace"), pytest.mark.medium] + + +class TestCloneRepo: + def test_success_result_and_directory_layout(self, git_repo: Path) -> None: + """Clone creates a sibling directory with expected keys, parent, and git repo.""" + result = clone_repo(str(git_repo), "myrun", strategy="clone_local") + clone_path = Path(result["clone_path"]) + assert clone_path.is_dir() + assert (clone_path / ".git").is_dir() + assert "clone_path" in result + assert "source_dir" in result + assert "remote_url" in result + assert result["source_dir"] == str(git_repo.resolve()) + expected_parent = git_repo.parent / "autoskillit-runs" + assert clone_path.parent == expected_parent + + def test_clone_path_name_format(self, git_repo: Path) -> None: + """Clone directory name follows run_name-YYYYMMDD-HHMMSS-ffffff format.""" + result = clone_repo(str(git_repo), "myrun", strategy="clone_local") + clone_path = Path(result["clone_path"]) + assert re.match(r"myrun-\d{8}-\d{6}-\d{6}$", clone_path.name), clone_path.name + + def test_invalid_source_dir_raises_value_error(self, tmp_path: Path) -> None: + with pytest.raises(ValueError, match="source_dir does not exist"): + clone_repo("/nonexistent/path/that/does/not/exist", "name") + + def test_non_git_dir_raises_runtime_error(self, tmp_path: Path) -> None: + # tmp_path exists but is not a git repo — probe fails with reason="error" + with pytest.raises(RuntimeError, match="clone_origin_probe_failed"): + clone_repo(str(tmp_path), "name") + + def test_cb1_explicit_branch_is_checked_out_in_clone(self, git_repo: Path) -> None: + """T_CB1: explicit branch is checked out in the clone.""" + subprocess.run( + ["git", "-C", str(git_repo), "checkout", "-b", "dev"], + check=True, + capture_output=True, + ) + subprocess.run( + ["git", "-C", str(git_repo), "commit", "--allow-empty", "-m", "dev-commit"], + check=True, + capture_output=True, + ) + result = clone_repo(str(git_repo), "test", branch="dev", strategy="clone_local") + clone_path = Path(result["clone_path"]) + head = subprocess.run( + ["git", "-C", str(clone_path), "rev-parse", "--abbrev-ref", "HEAD"], + capture_output=True, + text=True, + check=True, + ).stdout.strip() + assert head == "dev" + + def test_cb2_auto_detects_current_branch_when_branch_empty(self, git_repo: Path) -> None: + """T_CB2: branch="" auto-detects current HEAD branch and clones it.""" + subprocess.run( + ["git", "-C", str(git_repo), "checkout", "-b", "feature"], + check=True, + capture_output=True, + ) + subprocess.run( + ["git", "-C", str(git_repo), "commit", "--allow-empty", "-m", "feature-commit"], + check=True, + capture_output=True, + ) + result = clone_repo(str(git_repo), "test", branch="", strategy="clone_local") + clone_path = Path(result["clone_path"]) + head = subprocess.run( + ["git", "-C", str(clone_path), "rev-parse", "--abbrev-ref", "HEAD"], + capture_output=True, + text=True, + check=True, + ).stdout.strip() + assert head == "feature" + + def test_cb3_returns_uncommitted_changes_warning_when_dirty(self, git_repo: Path) -> None: + """T_CB3: untracked files trigger warning dict; no clone is created.""" + (git_repo / "untracked.txt").write_text("dirty") + result = clone_repo(str(git_repo), "test") + assert result["uncommitted_changes"] == "true" + assert "changed_files" in result + assert "clone_path" not in result + + def test_cb4_strategy_proceed_skips_uncommitted_check(self, local_with_remote: Path) -> None: + """T_CB4: strategy='proceed' clones without uncommitted changes check.""" + (local_with_remote / "untracked.txt").write_text("dirty") + result = clone_repo(str(local_with_remote), "test", branch="main", strategy="proceed") + assert "clone_path" in result + clone_path = Path(result["clone_path"]) + assert not (clone_path / "untracked.txt").exists() + + def test_cb5_strategy_clone_local_includes_uncommitted_changes(self, git_repo: Path) -> None: + """T_CB5: strategy='clone_local' copytree includes untracked files.""" + (git_repo / "untracked.txt").write_text("dirty") + result = clone_repo(str(git_repo), "test", strategy="clone_local") + assert "clone_path" in result + clone_path = Path(result["clone_path"]) + assert (clone_path / "untracked.txt").exists() + + def test_cb6_detached_head_falls_back_to_no_branch_flag(self, git_repo: Path) -> None: + """T_CB6: detached HEAD yields branch=''; git clone succeeds without --branch.""" + sha = subprocess.run( + ["git", "-C", str(git_repo), "rev-parse", "HEAD"], + capture_output=True, + text=True, + check=True, + ).stdout.strip() + subprocess.run( + ["git", "-C", str(git_repo), "checkout", sha], + check=True, + capture_output=True, + ) + result = clone_repo(str(git_repo), "test", branch="", strategy="clone_local") + assert "clone_path" in result + clone_path = Path(result["clone_path"]) + assert clone_path.is_dir() + + def test_returns_unpublished_branch_warning_when_not_on_remote( + self, local_with_remote: Path + ) -> None: + """clone_repo returns unpublished_branch warning dict when branch has no origin ref.""" + result = clone_repo(str(local_with_remote), "test-run", branch="feature/local-only") + assert result.get("unpublished_branch") == "true" + assert result.get("branch") == "feature/local-only" + assert result.get("source_dir") == str(local_with_remote) + assert "clone_path" not in result # sentinel: no clone was created + + def test_published_branch_proceeds_to_clone(self, local_with_remote: Path) -> None: + """clone_repo clones normally when branch is on origin.""" + result = clone_repo(str(local_with_remote), "test-run", branch="main") + assert "clone_path" in result + assert "unpublished_branch" not in result + + def test_strategy_proceed_with_unpublished_branch_fails_from_remote( + self, local_with_remote: Path + ) -> None: + """strategy='proceed' with unpublished branch now fails — remote is always used. + + Previously 'proceed' bypassed the guard by silently falling back to the + local path. After the fix the clone always targets the remote, so a branch + that doesn't exist on the remote causes git clone to fail. + """ + with pytest.raises(RuntimeError, match="git clone failed"): + clone_repo( + str(local_with_remote), + "test-run", + branch="feature/local-only", + strategy="proceed", + ) + + +def test_clone_output_stays_within_test_isolation_boundary(tmp_path: Path, git_repo: Path) -> None: + """clone_repo output must be a descendant of tmp_path, never its parent. + + Fails when the git_repo fixture returns tmp_path itself, because clone_repo + places autoskillit-runs/ at source.parent = tmp_path.parent (worker-shared). + Passes once git_repo returns tmp_path / 'repo' (a subdirectory). + """ + result = clone_repo(str(git_repo), "isolation-check", strategy="clone_local") + clone_path = Path(result["clone_path"]) + assert clone_path.is_relative_to(tmp_path), ( + f"clone_repo placed output at {clone_path!r}, which is outside " + f"the test's tmp_path {tmp_path!r}.\n" + "The git_repo fixture must return a SUBDIRECTORY of tmp_path " + "(e.g. tmp_path / 'repo'), not tmp_path itself. " + "When git_repo is tmp_path, clone destination is tmp_path.parent — " + "a directory shared across all tests in the same xdist worker." + ) + shutil.rmtree(clone_path.parent, ignore_errors=True) + + +class TestCloneOriginContract: + """Contract: clone's origin is a unique file:// URL; real URL is in the upstream remote.""" + + def test_clone_origin_is_file_url_and_upstream_holds_real_url(self, tmp_path: Path) -> None: + """Clone's origin must be a file:// URL (not source_dir or bare remote). + + After the fix: clone's origin = file://, upstream = real network URL. + This prevents Claude Code from aliasing the clone session to the source project. + """ + bare_remote = tmp_path / "bare.git" + bare_remote.mkdir() + subprocess.run( + ["git", "init", "--bare", "--initial-branch=main", str(bare_remote)], check=True + ) + + source = tmp_path / "source" + subprocess.run(["git", "clone", str(bare_remote), str(source)], check=True) + subprocess.run(["git", "-C", str(source), "config", "user.email", "t@t.com"], check=True) + subprocess.run(["git", "-C", str(source), "config", "user.name", "T"], check=True) + (source / "README.md").write_text("hello") + subprocess.run(["git", "-C", str(source), "add", "."], check=True) + subprocess.run(["git", "-C", str(source), "commit", "-m", "init"], check=True) + src_branch = subprocess.run( + ["git", "-C", str(source), "rev-parse", "--abbrev-ref", "HEAD"], + capture_output=True, + text=True, + check=True, + ).stdout.strip() + subprocess.run(["git", "-C", str(source), "push", "origin", src_branch], check=True) + + result = clone_repo(str(source), "contract-test") + clone_path = Path(result["clone_path"]) + remote_url = result["remote_url"] + + try: + origin_in_clone = subprocess.run( + ["git", "remote", "get-url", "origin"], + cwd=str(clone_path), + capture_output=True, + text=True, + check=True, + ).stdout.strip() + upstream_in_clone = subprocess.run( + ["git", "remote", "get-url", "upstream"], + cwd=str(clone_path), + capture_output=True, + text=True, + check=True, + ).stdout.strip() + + # origin is now a local file URL unique to this clone + assert origin_in_clone.startswith("file://"), ( + f"Clone's origin ({origin_in_clone!r}) must be a file:// URL" + ) + assert origin_in_clone != str(source), ( + "Clone's origin must not be the local source_dir path" + ) + # upstream holds the real push target + assert upstream_in_clone == remote_url, ( + f"Clone's upstream ({upstream_in_clone!r}) should equal" + f" remote_url ({remote_url!r})" + ) + finally: + shutil.rmtree(clone_path.parent, ignore_errors=True) + + def test_clone_local_strategy_rewrites_origin_to_file_url(self, tmp_path: Path) -> None: + """clone_local strategy (copytree) rewrites origin to file:// unconditionally. + + Covers the #377 compounding regression: the isolation rewrite was previously + skipped when effective_url was empty (no remote origin). After the fix, + _ensure_origin_isolated fires unconditionally for every successful clone. + """ + source = tmp_path / "source" + subprocess.run(["git", "init", str(source)], check=True) + subprocess.run(["git", "-C", str(source), "config", "user.email", "t@t.com"], check=True) + subprocess.run(["git", "-C", str(source), "config", "user.name", "T"], check=True) + (source / "f.txt").write_text("x") + subprocess.run(["git", "-C", str(source), "add", "."], check=True) + subprocess.run(["git", "-C", str(source), "commit", "-m", "init"], check=True) + + result = clone_repo(str(source), "no-upstream-test", strategy="clone_local") + try: + clone_path = Path(result["clone_path"]) + origin_in_clone = subprocess.run( + ["git", "-C", str(clone_path), "remote", "get-url", "origin"], + check=True, + capture_output=True, + text=True, + ).stdout.strip() + assert origin_in_clone == f"file://{clone_path}", ( + f"Expected file://{clone_path!r}, got {origin_in_clone!r}" + ) + finally: + shutil.rmtree(Path(result["clone_path"]).parent, ignore_errors=True) + + def test_clone_local_strategy_also_sets_correct_remotes(self, tmp_path: Path) -> None: + """clone_local strategy (copytree) must also set origin=file:// and upstream=real URL. + + The rewrite applies to both the proceed (git clone) and clone_local (copytree) paths. + """ + bare_remote = tmp_path / "bare.git" + bare_remote.mkdir() + subprocess.run(["git", "init", "--bare", str(bare_remote)], check=True) + source = tmp_path / "source" + subprocess.run(["git", "clone", str(bare_remote), str(source)], check=True) + subprocess.run(["git", "-C", str(source), "config", "user.email", "t@t.com"], check=True) + subprocess.run(["git", "-C", str(source), "config", "user.name", "T"], check=True) + (source / "README.md").write_text("hello") + subprocess.run(["git", "-C", str(source), "add", "."], check=True) + subprocess.run(["git", "-C", str(source), "commit", "-m", "init"], check=True) + + result = clone_repo(str(source), "clone-local-contract", strategy="clone_local") + clone_path = Path(result["clone_path"]) + remote_url = result["remote_url"] + + try: + origin_in_clone = subprocess.run( + ["git", "remote", "get-url", "origin"], + cwd=str(clone_path), + capture_output=True, + text=True, + check=True, + ).stdout.strip() + upstream_in_clone = subprocess.run( + ["git", "remote", "get-url", "upstream"], + cwd=str(clone_path), + capture_output=True, + text=True, + check=True, + ).stdout.strip() + + assert origin_in_clone.startswith("file://"), ( + f"clone_local origin ({origin_in_clone!r}) must be a file:// URL" + ) + assert upstream_in_clone == remote_url, ( + f"clone_local upstream ({upstream_in_clone!r}) should equal" + f" remote_url ({remote_url!r})" + ) + finally: + shutil.rmtree(clone_path.parent, ignore_errors=True) + + +class TestCloneRepoRemoteUrlOverride: + """Tests for the remote_url parameter on clone_repo (T_RU1, T_RU2).""" + + def _make_source_with_bare_remote(self, tmp_path: Path) -> tuple[Path, Path]: + """Helper: create a bare remote and source repo pointing to it.""" + bare = tmp_path / "bare.git" + bare.mkdir() + subprocess.run(["git", "init", "--bare", str(bare)], check=True, capture_output=True) + + source = tmp_path / "source" + subprocess.run(["git", "clone", str(bare), str(source)], check=True, capture_output=True) + subprocess.run( + ["git", "-C", str(source), "config", "user.email", "t@t.com"], + check=True, + capture_output=True, + ) + subprocess.run( + ["git", "-C", str(source), "config", "user.name", "T"], + check=True, + capture_output=True, + ) + (source / "f.txt").write_text("x") + subprocess.run(["git", "-C", str(source), "add", "."], check=True, capture_output=True) + subprocess.run( + ["git", "-C", str(source), "commit", "-m", "init"], + check=True, + capture_output=True, + ) + src_branch = subprocess.run( + ["git", "-C", str(source), "rev-parse", "--abbrev-ref", "HEAD"], + capture_output=True, + text=True, + check=True, + ).stdout.strip() + subprocess.run( + ["git", "-C", str(source), "push", "origin", src_branch], + check=True, + capture_output=True, + ) + return source, bare + + def test_clone_repo_remote_url_override_applied(self, tmp_path: Path) -> None: + """When remote_url is provided, clone's upstream is set to that URL (T_RU1). + + After the fix: origin = file://, upstream = override_url. + """ + source, _bare = self._make_source_with_bare_remote(tmp_path) + override_url = "https://github.com/example/repo.git" + + result = clone_repo(str(source), "test-run", strategy="proceed", remote_url=override_url) + assert "clone_path" in result + clone_path = Path(result["clone_path"]) + + try: + actual_origin = subprocess.run( + ["git", "remote", "get-url", "origin"], + cwd=str(clone_path), + capture_output=True, + text=True, + check=True, + ).stdout.strip() + actual_upstream = subprocess.run( + ["git", "remote", "get-url", "upstream"], + cwd=str(clone_path), + capture_output=True, + text=True, + check=True, + ).stdout.strip() + # origin is now a local file URL unique to this clone + assert actual_origin.startswith("file://") + # upstream holds the real push target (the override URL) + assert actual_upstream == override_url + assert result["remote_url"] == override_url + finally: + shutil.rmtree(clone_path.parent, ignore_errors=True) + + def test_clone_repo_without_remote_url_uses_detected(self, tmp_path: Path) -> None: + """Without remote_url, clone's upstream is set to the source's detected origin (T_RU2). + + After the fix: origin = file://, upstream = detected bare remote path. + """ + source, bare = self._make_source_with_bare_remote(tmp_path) + + result = clone_repo(str(source), "test-run", strategy="proceed") + assert "clone_path" in result + clone_path = Path(result["clone_path"]) + + try: + actual_origin = subprocess.run( + ["git", "remote", "get-url", "origin"], + cwd=str(clone_path), + capture_output=True, + text=True, + check=True, + ).stdout.strip() + actual_upstream = subprocess.run( + ["git", "remote", "get-url", "upstream"], + cwd=str(clone_path), + capture_output=True, + text=True, + check=True, + ).stdout.strip() + # origin is now a local file URL unique to this clone + assert actual_origin.startswith("file://") + # upstream holds the real push target (the detected bare remote) + assert actual_upstream == str(bare) + finally: + shutil.rmtree(clone_path.parent, ignore_errors=True) + + +class TestRemoveClone: + def test_keep_false_removes_directory(self, git_repo: Path) -> None: + result = clone_repo(str(git_repo), "test", strategy="clone_local") + clone_path = result["clone_path"] + remove_result = remove_clone(clone_path, keep="false") + assert remove_result == {"removed": "true"} + assert not Path(clone_path).exists() + # Cleanup parent runs dir + shutil.rmtree(Path(clone_path), ignore_errors=True) + + def test_keep_true_preserves_directory(self, git_repo: Path) -> None: + result = clone_repo(str(git_repo), "test", strategy="clone_local") + clone_path = result["clone_path"] + remove_result = remove_clone(clone_path, keep="true") + assert remove_result == {"removed": "false", "reason": "keep=true"} + assert Path(clone_path).exists() + # Cleanup + shutil.rmtree(Path(clone_path), ignore_errors=True) + + def test_missing_path_returns_not_found(self) -> None: + result = remove_clone("/nonexistent/clone/path", keep="false") + assert result == {"removed": "false", "reason": "not_found"} + + +class TestCloneRepoDetectAndExpand: + def test_ds4_expands_tilde(self) -> None: + """T_DS4: clone_repo raises ValueError with 'resolved to' when tilde path doesn't exist.""" + with pytest.raises(ValueError, match="resolved to"): + clone_repo("~/nonexistent-autoskillit-test-xyz", "test-run") + + def test_ds5_raises_value_error_with_clear_message(self, tmp_path: Path) -> None: + """T_DS5: non-existent path raises ValueError with 'resolved to' in message.""" + nonexistent = str(tmp_path / "does-not-exist") + with pytest.raises(ValueError, match="resolved to"): + clone_repo(nonexistent, "test-run") + + def test_cb7_calls_detect_branch_when_branch_empty(self, tmp_path: Path) -> None: + """T_CB7: detect_branch is called with source_dir when branch="".""" + with patch( + "autoskillit.workspace.clone.detect_branch", return_value="main" + ) as mock_detect: + with patch("autoskillit.workspace.clone.detect_uncommitted_changes", return_value=[]): + mock_clone = MagicMock() + mock_clone.returncode = 0 + with patch("autoskillit.workspace.clone.subprocess.run", return_value=mock_clone): + clone_repo(str(tmp_path), "test-run", branch="") + mock_detect.assert_called_once_with(str(tmp_path)) + + def test_cb9_passes_branch_flag_to_git(self, tmp_path: Path) -> None: + """T_CB9: --branch and branch name appear in the git clone subprocess call.""" + with patch("autoskillit.workspace.clone.detect_uncommitted_changes", return_value=[]): + mock_clone = MagicMock() + mock_clone.returncode = 0 + with patch( + "autoskillit.workspace.clone.subprocess.run", return_value=mock_clone + ) as mock_run: + clone_repo(str(tmp_path), "test-run", branch="dev") + git_clone_calls = [ + call for call in mock_run.call_args_list if call.args and "clone" in call.args[0] + ] + assert git_clone_calls, "git clone was not called" + cmd = git_clone_calls[0].args[0] + assert "--branch" in cmd + assert "dev" in cmd + + def test_cb10_returns_warning_dict_on_uncommitted_changes(self, tmp_path: Path) -> None: + """T_CB10: uncommitted changes produce warning dict; git clone not called.""" + with patch("autoskillit.workspace.clone.detect_branch", return_value="main"): + with patch( + "autoskillit.workspace.clone.detect_uncommitted_changes", + return_value=[" M file.py"], + ): + with patch("autoskillit.workspace.clone.subprocess.run") as mock_run: + result = clone_repo(str(tmp_path), "test-run") + assert result["uncommitted_changes"] == "true" + mock_run.assert_not_called() + + +class TestCloneDecontamination: + """clone_repo strips tracked and on-disk generated files from clones.""" + + def test_clone_repo_untracks_inherited_generated_files(self, tmp_path: Path) -> None: + """Tracked generated files in source are untracked in the clone.""" + repo = tmp_path / "repo" + repo.mkdir() + subprocess.run(["git", "init", str(repo)], check=True, capture_output=True) + subprocess.run( + ["git", "-C", str(repo), "config", "user.email", "t@t.com"], + check=True, + capture_output=True, + ) + subprocess.run( + ["git", "-C", str(repo), "config", "user.name", "T"], + check=True, + capture_output=True, + ) + # Create and track a generated file + hooks_dir = repo / "src" / "autoskillit" / "hooks" + hooks_dir.mkdir(parents=True) + (hooks_dir / "hooks.json").write_text('{"hooks": {}}') + subprocess.run( + ["git", "-C", str(repo), "add", "src/autoskillit/hooks/hooks.json"], + check=True, + capture_output=True, + ) + subprocess.run( + ["git", "-C", str(repo), "commit", "-m", "add generated file"], + check=True, + capture_output=True, + ) + + result = clone_repo(str(repo), "decontam-test", strategy="clone_local") + clone_path = Path(result["clone_path"]) + try: + ls_result = subprocess.run( + ["git", "ls-files", "--", "src/autoskillit/hooks/hooks.json"], + cwd=str(clone_path), + capture_output=True, + text=True, + check=True, + ) + assert ls_result.stdout.strip() == "", "Generated file should be untracked in clone" + finally: + shutil.rmtree(clone_path.parent, ignore_errors=True) + + def test_clone_repo_deletes_untracked_generated_files_from_disk(self, tmp_path: Path) -> None: + """clone_local copies untracked generated files; decontamination deletes them.""" + repo = tmp_path / "repo" + repo.mkdir() + subprocess.run(["git", "init", str(repo)], check=True, capture_output=True) + subprocess.run( + ["git", "-C", str(repo), "config", "user.email", "t@t.com"], + check=True, + capture_output=True, + ) + subprocess.run( + ["git", "-C", str(repo), "config", "user.name", "T"], + check=True, + capture_output=True, + ) + subprocess.run( + ["git", "-C", str(repo), "commit", "--allow-empty", "-m", "init"], + check=True, + capture_output=True, + ) + # Create on-disk generated file WITHOUT tracking it + hooks_dir = repo / "src" / "autoskillit" / "hooks" + hooks_dir.mkdir(parents=True) + (hooks_dir / "hooks.json").write_text('{"hooks": {}}') + + result = clone_repo(str(repo), "disk-cleanup-test", strategy="clone_local") + clone_path = Path(result["clone_path"]) + try: + assert not (clone_path / "src" / "autoskillit" / "hooks" / "hooks.json").exists(), ( + "Untracked generated file should be deleted from clone disk" + ) + finally: + shutil.rmtree(clone_path.parent, ignore_errors=True) + + def test_clone_repo_noop_when_no_generated_files_tracked(self, git_repo: Path) -> None: + """Clean repo with no tracked generated files clones without errors.""" + result = clone_repo(str(git_repo), "clean-test", strategy="clone_local") + clone_path = Path(result["clone_path"]) + try: + assert clone_path.is_dir() + assert (clone_path / ".git").is_dir() + finally: + shutil.rmtree(clone_path.parent, ignore_errors=True) + + +class TestCloneRepoResultShape: + """clone_repo result-value contracts — remote_url field and clone_source_type discriminator.""" + + def test_clone_repo_returns_remote_url(self, tmp_path: Path) -> None: + """clone_repo result dict includes remote_url field.""" + remote = tmp_path / "remote.git" + subprocess.run( + ["git", "init", "--bare", "--initial-branch=main", str(remote)], + check=True, + capture_output=True, + ) + source = tmp_path / "source" + subprocess.run(["git", "clone", str(remote), str(source)], check=True, capture_output=True) + subprocess.run( + ["git", "-C", str(source), "config", "user.email", "t@t.com"], + check=True, + capture_output=True, + ) + subprocess.run( + ["git", "-C", str(source), "config", "user.name", "T"], + check=True, + capture_output=True, + ) + subprocess.run( + ["git", "-C", str(source), "commit", "--allow-empty", "-m", "init"], + check=True, + capture_output=True, + ) + subprocess.run( + ["git", "-C", str(source), "push", "-u", "origin", "HEAD:main"], + check=True, + capture_output=True, + ) + + result = clone_repo(str(source), "test-remoteurl", strategy="proceed") + + assert "remote_url" in result + assert str(remote) in result["remote_url"] + + # After the fix: origin is a file:// URL, upstream holds the real remote_url + origin_in_clone = subprocess.run( + ["git", "remote", "get-url", "origin"], + cwd=result["clone_path"], + capture_output=True, + text=True, + check=True, + ).stdout.strip() + upstream_in_clone = subprocess.run( + ["git", "remote", "get-url", "upstream"], + cwd=result["clone_path"], + capture_output=True, + text=True, + check=True, + ).stdout.strip() + assert origin_in_clone.startswith("file://"), ( + "Clone's origin must be a file:// URL after the fix" + ) + assert upstream_in_clone == result["remote_url"], ( + "Clone's upstream must equal the returned remote_url" + ) + + shutil.rmtree(Path(result["clone_path"]).parent, ignore_errors=True) + + def test_clone_repo_local_strategy_returns_discriminator_when_no_origin( + self, tmp_path: Path + ) -> None: + """clone_local strategy returns discriminator with clone_source_type=local.""" + source = tmp_path / "source" + subprocess.run(["git", "init", str(source)], check=True, capture_output=True) + subprocess.run( + ["git", "-C", str(source), "config", "user.email", "t@t.com"], + check=True, + capture_output=True, + ) + subprocess.run( + ["git", "-C", str(source), "config", "user.name", "T"], + check=True, + capture_output=True, + ) + subprocess.run( + ["git", "-C", str(source), "commit", "--allow-empty", "-m", "init"], + check=True, + capture_output=True, + ) + + result = clone_repo(str(source), "test-noremote", strategy="clone_local") + + assert "clone_path" in result + assert result["remote_url"] == "" + assert result["clone_source_type"] == "local" + assert result["clone_source_reason"] == "strategy_clone_local" + assert Path(result["clone_path"]).exists() + + shutil.rmtree(Path(result["clone_path"]).parent, ignore_errors=True) + + +class TestCloneOriginProbeFailFast: + """Tests 2A-2C: clone_repo raises RuntimeError when origin probe fails.""" + + def test_clone_repo_raises_on_timeout_when_source_has_origin( + self, local_with_remote: Path + ) -> None: + """The reported bug reproduced: timeout on origin probe raises RuntimeError. + + Patches subprocess.run to raise TimeoutExpired only on get-url origin + while delegating all other calls to the real subprocess.run. + """ + _real_run = subprocess.run + + def _selective_side_effect(cmd, **kwargs): # type: ignore[no-untyped-def] + if isinstance(cmd, list) and cmd == ["git", "remote", "get-url", "origin"]: + raise subprocess.TimeoutExpired(cmd=cmd, timeout=30) + return _real_run(cmd, **kwargs) + + with patch( + "autoskillit.workspace.clone.subprocess.run", + side_effect=_selective_side_effect, + ): + with structlog.testing.capture_logs() as cap: + with pytest.raises(RuntimeError, match="clone_origin_probe_failed.*timeout"): + clone_repo(str(local_with_remote), "t", strategy="proceed") + + warning_events = [e for e in cap if e.get("event") == "clone_origin_probe_failed"] + assert warning_events, "Expected clone_origin_probe_failed warning to be emitted" + assert warning_events[0].get("reason") == "timeout" + + def test_clone_repo_raises_on_non_zero_get_url_when_source_has_origin( + self, local_with_remote: Path + ) -> None: + """Non-zero returncode on get-url origin raises RuntimeError.""" + _real_run = subprocess.run + + def _selective_side_effect(cmd, **kwargs): # type: ignore[no-untyped-def] + if isinstance(cmd, list) and cmd == ["git", "remote", "get-url", "origin"]: + mock = MagicMock() + mock.returncode = 1 + mock.stdout = "" + mock.stderr = "fatal: some error" + return mock + return _real_run(cmd, **kwargs) + + with patch( + "autoskillit.workspace.clone.subprocess.run", + side_effect=_selective_side_effect, + ): + with pytest.raises(RuntimeError, match="clone_origin_probe_failed"): + clone_repo(str(local_with_remote), "t", strategy="proceed") + + def test_clone_repo_raises_on_no_origin_without_explicit_local_strategy( + self, git_repo: Path + ) -> None: + """clone_repo raises RuntimeError when source has no origin and strategy=proceed. + + Error message must instruct the caller to pass strategy="clone_local". + """ + with pytest.raises( + RuntimeError, + match=r"clone_origin_probe_failed.*no_origin.*strategy=.?clone_local", + ): + clone_repo(str(git_repo), "t", strategy="proceed") + + +class TestCloneDiscriminator: + """Tests 3A-3B and 5: typed return contract with clone_source_type discriminator.""" + + def test_clone_local_strategy_succeeds_on_no_origin_with_discriminator( + self, git_repo: Path + ) -> None: + """strategy=clone_local returns clone_source_type=local with discriminator.""" + result = clone_repo(str(git_repo), "t", strategy="clone_local") + clone_path = Path(result["clone_path"]) + try: + assert result["clone_source_type"] == "local" + assert result["clone_source_reason"] == "strategy_clone_local" + assert result["remote_url"] == "" + assert clone_path.exists() + finally: + shutil.rmtree(clone_path.parent, ignore_errors=True) + + def test_clone_proceed_strategy_succeeds_with_remote_discriminator( + self, local_with_remote: Path, bare_remote: Path + ) -> None: + """strategy=proceed returns clone_source_type=remote with discriminator.""" + result = clone_repo(str(local_with_remote), "t", branch="main", strategy="proceed") + clone_path = Path(result["clone_path"]) + try: + assert result["clone_source_type"] == "remote" + assert result["clone_source_reason"] == "ok" + assert result["remote_url"] == str(bare_remote) + finally: + shutil.rmtree(clone_path.parent, ignore_errors=True) + + @pytest.mark.parametrize( + "fixture_name,strategy,expected_source_type", + [ + ("local_with_remote", "proceed", "remote"), + ("git_repo", "clone_local", "local"), + ], + ) + def test_clone_result_always_has_discriminator_key( + self, + fixture_name: str, + strategy: str, + expected_source_type: str, + request: pytest.FixtureRequest, + ) -> None: + """Structural invariant: every success result carries clone_source_type.""" + source = request.getfixturevalue(fixture_name) + kwargs = {"strategy": strategy} + if fixture_name == "local_with_remote": + kwargs["branch"] = "main" + result = clone_repo(str(source), "t", **kwargs) # type: ignore[arg-type] + clone_path = Path(result["clone_path"]) + try: + assert result["clone_source_type"] == expected_source_type + assert "clone_source_reason" in result + finally: + shutil.rmtree(clone_path.parent, ignore_errors=True) + + +class TestOriginIsolationUnconditional: + """Tests 4A-4B: _ensure_origin_isolated fires unconditionally.""" + + def test_clone_local_strategy_still_rewrites_origin_to_file_url(self, git_repo: Path) -> None: + """clone_local (no remote) rewrites origin to file:// — covers #377 regression. + + Inverts the deleted test_clone_origin_unchanged_when_no_upstream. + """ + result = clone_repo(str(git_repo), "t", strategy="clone_local") + clone_path = Path(result["clone_path"]) + try: + origin = subprocess.run( + ["git", "-C", str(clone_path), "remote", "get-url", "origin"], + check=True, + capture_output=True, + text=True, + ).stdout.strip() + assert origin == f"file://{clone_path}", ( + f"Expected file://{clone_path!r}, got {origin!r}" + ) + finally: + shutil.rmtree(clone_path.parent, ignore_errors=True) + + def test_clone_remote_strategy_rewrites_origin_to_file_url( + self, local_with_remote: Path + ) -> None: + """clone via remote URL also rewrites origin to file:// unconditionally.""" + result = clone_repo(str(local_with_remote), "t", branch="main", strategy="proceed") + clone_path = Path(result["clone_path"]) + try: + origin = subprocess.run( + ["git", "-C", str(clone_path), "remote", "get-url", "origin"], + check=True, + capture_output=True, + text=True, + ).stdout.strip() + assert origin == f"file://{clone_path}", ( + f"Expected file://{clone_path!r}, got {origin!r}" + ) + finally: + shutil.rmtree(clone_path.parent, ignore_errors=True) diff --git a/tests/workspace/test_clone_detect.py b/tests/workspace/test_clone_detect.py new file mode 100644 index 0000000000..ff8f98a449 --- /dev/null +++ b/tests/workspace/test_clone_detect.py @@ -0,0 +1,126 @@ +"""Tests for detect_* helpers and classify_remote_url from autoskillit.workspace._clone_detect.""" + +from __future__ import annotations + +import subprocess +from pathlib import Path +from unittest.mock import MagicMock, patch + +import pytest + +from autoskillit.workspace.clone import ( + classify_remote_url, + detect_branch, + detect_source_dir, + detect_uncommitted_changes, +) + +pytestmark = [pytest.mark.layer("workspace"), pytest.mark.medium] + + +class TestDetectSourceDir: + def test_ds1_returns_git_toplevel(self) -> None: + """T_DS1: returns git rev-parse --show-toplevel when returncode=0.""" + mock_result = MagicMock() + mock_result.returncode = 0 + mock_result.stdout = "/repo/root\n" + with patch("autoskillit.workspace._clone_detect.subprocess.run", return_value=mock_result): + assert detect_source_dir("/any/cwd") == "/repo/root" + + def test_ds2_falls_back_on_nonzero_returncode(self) -> None: + """T_DS2: returns cwd unchanged when git exits non-zero.""" + mock_result = MagicMock() + mock_result.returncode = 128 + mock_result.stdout = "" + with patch("autoskillit.workspace._clone_detect.subprocess.run", return_value=mock_result): + assert detect_source_dir("/any/cwd") == "/any/cwd" + + +class TestDetectBranch: + def test_cb11_returns_branch_name_on_success(self) -> None: + """T_CB11: returns branch name when git rev-parse succeeds.""" + mock_result = MagicMock() + mock_result.returncode = 0 + mock_result.stdout = "main\n" + with patch("autoskillit.workspace._clone_detect.subprocess.run", return_value=mock_result): + assert detect_branch("/any") == "main" + + def test_cb12_returns_empty_string_on_nonzero_returncode(self) -> None: + """T_CB12: returns "" when git exits non-zero.""" + mock_result = MagicMock() + mock_result.returncode = 128 + mock_result.stdout = "" + with patch("autoskillit.workspace._clone_detect.subprocess.run", return_value=mock_result): + assert detect_branch("/any") == "" + + def test_cb13_returns_head_literal_for_detached_state(self) -> None: + """T_CB13: returns literal 'HEAD' in detached HEAD state; caller treats as no branch.""" + mock_result = MagicMock() + mock_result.returncode = 0 + mock_result.stdout = "HEAD\n" + with patch("autoskillit.workspace._clone_detect.subprocess.run", return_value=mock_result): + assert detect_branch("/any") == "HEAD" + + +class TestDetectUncommittedChanges: + def test_cb14_returns_empty_list_when_clean(self) -> None: + """T_CB14: returns [] when working tree is clean.""" + mock_result = MagicMock() + mock_result.returncode = 0 + mock_result.stdout = "" + with patch("autoskillit.workspace._clone_detect.subprocess.run", return_value=mock_result): + assert detect_uncommitted_changes("/any") == [] + + def test_cb15_returns_changed_file_lines_when_dirty(self) -> None: + """T_CB15: returns non-empty status lines when changes exist.""" + mock_result = MagicMock() + mock_result.returncode = 0 + mock_result.stdout = " M file.py\n?? new.txt\n" + with patch("autoskillit.workspace._clone_detect.subprocess.run", return_value=mock_result): + assert detect_uncommitted_changes("/any") == [" M file.py", "?? new.txt"] + + def test_cb16_returns_empty_list_on_git_failure(self) -> None: + """T_CB16: returns [] when git exits non-zero.""" + mock_result = MagicMock() + mock_result.returncode = 128 + with patch("autoskillit.workspace._clone_detect.subprocess.run", return_value=mock_result): + assert detect_uncommitted_changes("/any") == [] + + +class TestClassifyRemoteUrl: + """classify_remote_url correctly identifies remote URL types.""" + + @pytest.mark.parametrize( + "url,expected_type", + [ + ("git@github.com:org/repo.git", "network"), + ("https://github.com/org/repo.git", "network"), + ("ssh://git@github.com/org/repo.git", "network"), + ("http://example.com/repo.git", "network"), + ("git://github.com/org/repo.git", "network"), + ("file:///tmp/repo.git", "network"), + ], + ) + def test_network_urls(self, url: str, expected_type: str) -> None: + result = classify_remote_url(url) + assert result == expected_type + + def test_bare_local_path(self, tmp_path: Path) -> None: + bare = tmp_path / "repo.git" + subprocess.run(["git", "init", "--bare", str(bare)], check=True, capture_output=True) + result = classify_remote_url(str(bare)) + assert result == "bare_local" + + def test_nonbare_local_path(self, tmp_path: Path) -> None: + repo = tmp_path / "repo" + subprocess.run(["git", "init", str(repo)], check=True, capture_output=True) + result = classify_remote_url(str(repo)) + assert result == "nonbare_local" + + def test_empty_string_returns_none_type(self) -> None: + result = classify_remote_url("") + assert result == "none" + + def test_nonexistent_path_returns_unknown(self, tmp_path: Path) -> None: + result = classify_remote_url(str(tmp_path / "nonexistent")) + assert result == "unknown" diff --git a/tests/workspace/test_clone_push.py b/tests/workspace/test_clone_push.py new file mode 100644 index 0000000000..8f3a911e29 --- /dev/null +++ b/tests/workspace/test_clone_push.py @@ -0,0 +1,579 @@ +"""push_to_remote tests — E2E, mocked, protected branches, force-with-lease, merge queue.""" + +from __future__ import annotations + +import inspect +import shutil +import subprocess +from pathlib import Path +from unittest.mock import MagicMock, patch + +import pytest + +from autoskillit.workspace.clone import ( + DefaultCloneManager, + clone_repo, + push_to_remote, +) + +pytestmark = [pytest.mark.layer("workspace"), pytest.mark.medium] + + +class TestPushToRemote: + def test_push_to_remote_propagates_to_upstream(self, tmp_path: Path) -> None: + # 1. Create bare remote (simulates GitHub) + remote = tmp_path / "remote.git" + subprocess.run(["git", "init", "--bare", str(remote)], check=True, capture_output=True) + + # 2. Clone remote into source (simulates user's local checkout) + source = tmp_path / "source" + subprocess.run(["git", "clone", str(remote), str(source)], check=True, capture_output=True) + subprocess.run( + ["git", "-C", str(source), "config", "user.email", "t@t.com"], + check=True, + capture_output=True, + ) + subprocess.run( + ["git", "-C", str(source), "config", "user.name", "T"], + check=True, + capture_output=True, + ) + subprocess.run( + ["git", "-C", str(source), "commit", "--allow-empty", "-m", "init"], + check=True, + capture_output=True, + ) + src_branch = subprocess.run( + ["git", "-C", str(source), "rev-parse", "--abbrev-ref", "HEAD"], + capture_output=True, + text=True, + check=True, + ).stdout.strip() + subprocess.run( + ["git", "-C", str(source), "push", "origin", src_branch], + check=True, + capture_output=True, + ) + + # 3. Pipeline clones source + clone_result = clone_repo(str(source), "pushtest") + clone_path = clone_result["clone_path"] + + # 4. Make a commit in pipeline-clone + subprocess.run( + ["git", "-C", clone_path, "config", "user.email", "t@t.com"], + check=True, + capture_output=True, + ) + subprocess.run( + ["git", "-C", clone_path, "config", "user.name", "T"], + check=True, + capture_output=True, + ) + subprocess.run( + ["git", "-C", clone_path, "commit", "--allow-empty", "-m", "pipeline-commit"], + check=True, + capture_output=True, + ) + branch = subprocess.run( + ["git", "-C", clone_path, "rev-parse", "--abbrev-ref", "HEAD"], + capture_output=True, + text=True, + check=True, + ).stdout.strip() + + # Record source HEAD before push (must not change) + source_head_before = subprocess.run( + ["git", "-C", str(source), "rev-parse", "HEAD"], + capture_output=True, + text=True, + check=True, + ).stdout.strip() + + result = push_to_remote(clone_path, str(source), branch, protected_branches=[]) + assert result["success"] is True + + # Commit landed in remote + remote_log = subprocess.run( + ["git", "-C", str(remote), "log", "--oneline", "-3"], + capture_output=True, + text=True, + check=True, + ).stdout + assert "pipeline-commit" in remote_log + + # source_dir HEAD unchanged + source_head_after = subprocess.run( + ["git", "-C", str(source), "rev-parse", "HEAD"], + capture_output=True, + text=True, + check=True, + ).stdout.strip() + assert source_head_before == source_head_after + + def test_push_to_remote_establishes_tracking_ref(self, tmp_path: Path) -> None: + """push_to_remote must establish a tracking ref so remove_clone_guard passes.""" + remote = tmp_path / "remote.git" + subprocess.run(["git", "init", "--bare", str(remote)], check=True, capture_output=True) + source = tmp_path / "source" + subprocess.run(["git", "clone", str(remote), str(source)], check=True, capture_output=True) + subprocess.run( + ["git", "-C", str(source), "config", "user.email", "t@t.com"], + check=True, + capture_output=True, + ) + subprocess.run( + ["git", "-C", str(source), "config", "user.name", "T"], + check=True, + capture_output=True, + ) + subprocess.run( + ["git", "-C", str(source), "commit", "--allow-empty", "-m", "init"], + check=True, + capture_output=True, + ) + src_branch = subprocess.run( + ["git", "-C", str(source), "rev-parse", "--abbrev-ref", "HEAD"], + capture_output=True, + text=True, + check=True, + ).stdout.strip() + subprocess.run( + ["git", "-C", str(source), "push", "origin", src_branch], + check=True, + capture_output=True, + ) + + clone_result = clone_repo(str(source), "tracktest") + clone_path = clone_result["clone_path"] + subprocess.run( + ["git", "-C", clone_path, "config", "user.email", "t@t.com"], + check=True, + capture_output=True, + ) + subprocess.run( + ["git", "-C", clone_path, "config", "user.name", "T"], + check=True, + capture_output=True, + ) + subprocess.run( + ["git", "-C", clone_path, "commit", "--allow-empty", "-m", "pipeline-commit"], + check=True, + capture_output=True, + ) + + result = push_to_remote(clone_path, str(source), src_branch, protected_branches=[]) + assert result["success"] is True + + upstream_rc = subprocess.run( + ["git", "-C", clone_path, "rev-parse", "@{upstream}"], + capture_output=True, + text=True, + ).returncode + assert upstream_rc == 0, "@{upstream} must be set after push_to_remote" + shutil.rmtree(Path(clone_path).parent, ignore_errors=True) + + def test_push_to_remote_fails_when_source_has_no_origin(self, tmp_path: Path) -> None: + source = tmp_path / "source" + source.mkdir() + subprocess.run(["git", "init", str(source)], check=True, capture_output=True) + + result = push_to_remote("/nonexistent/clone", str(source), "main", protected_branches=[]) + assert result["success"] is False + assert len(result["stderr"]) > 0 + + +class TestPushToRemoteProtectedBranch: + """push_to_remote rejects pushes to protected branches.""" + + @pytest.mark.parametrize("branch", ["main", "develop", "stable"]) + def test_push_to_remote_rejects_protected_branch(self, tmp_path: Path, branch: str) -> None: + """push_to_remote must reject when branch is a protected branch.""" + clone = tmp_path / "clone" + clone.mkdir() + + result = push_to_remote( + clone_path=str(clone), + branch=branch, + remote_url="https://github.com/example/repo.git", + protected_branches=["main", "develop", "stable"], + ) + + assert result["success"] is False + assert result.get("error_type") == "protected_branch_push" + + +class TestPushToRemoteMocked: + def test_ds6_push_to_remote_calls_get_url_then_push(self) -> None: + """T_DS6: push_to_remote calls git remote get-url origin then git push -u upstream.""" + mock_url = MagicMock() + mock_url.returncode = 0 + mock_url.stdout = "git@github.com:org/repo.git\n" + mock_url.stderr = "" + + mock_push = MagicMock() + mock_push.returncode = 0 + mock_push.stderr = "" + + with patch( + "autoskillit.workspace.clone.subprocess.run", side_effect=[mock_url, mock_push] + ) as mock_run: + result = push_to_remote( + "/clone", "/source", "main", protected_branches=[], force=False + ) + + assert result == {"success": True, "stderr": ""} + # First call: git remote get-url origin from source_dir + first_call = mock_run.call_args_list[0] + assert first_call[0][0] == ["git", "remote", "get-url", "origin"] + assert first_call[1]["cwd"] == "/source" + # Second call: git push -u upstream from clone_path — no --force-with-lease + second_call = mock_run.call_args_list[1] + assert second_call[0][0] == ["git", "push", "-u", "upstream", "main"] + assert second_call[1]["cwd"] == "/clone" + + def test_ds7_push_to_remote_fails_when_no_origin(self) -> None: + """T_DS7: push_to_remote returns error when git remote get-url fails, no push attempted.""" + mock_fail = MagicMock() + mock_fail.returncode = 128 + mock_fail.stdout = "" + mock_fail.stderr = "error: No such remote 'origin'" + + with patch( + "autoskillit.workspace.clone.subprocess.run", return_value=mock_fail + ) as mock_run: + result = push_to_remote("/clone", "/source", "main", protected_branches=[]) + + assert result["success"] is False + assert "origin" in result["stderr"] + assert mock_run.call_count == 1 # no push attempted + + def test_push_to_remote_with_force_injects_force_with_lease(self) -> None: + """T1: push_to_remote with force=True appends --force-with-lease to push command.""" + mock_url = MagicMock() + mock_url.returncode = 0 + mock_url.stdout = "git@github.com:org/repo.git\n" + mock_url.stderr = "" + + mock_push = MagicMock() + mock_push.returncode = 0 + mock_push.stderr = "" + + with patch( + "autoskillit.workspace.clone.subprocess.run", side_effect=[mock_url, mock_push] + ) as mock_run: + result = push_to_remote("/clone", "/source", "main", protected_branches=[], force=True) + + assert result == {"success": True, "stderr": ""} + second_call = mock_run.call_args_list[1] + cmd = second_call[0][0] + assert "--force-with-lease" in cmd + assert "git" in cmd + assert "push" in cmd + assert "upstream" in cmd + assert "main" in cmd + assert second_call[1]["cwd"] == "/clone" + + def test_push_to_remote_default_force_false_does_not_inject_lease(self) -> None: + """T2: push_to_remote with force=False (default) does not inject --force-with-lease.""" + mock_url = MagicMock() + mock_url.returncode = 0 + mock_url.stdout = "git@github.com:org/repo.git\n" + mock_url.stderr = "" + + mock_push = MagicMock() + mock_push.returncode = 0 + mock_push.stderr = "" + + with patch( + "autoskillit.workspace.clone.subprocess.run", side_effect=[mock_url, mock_push] + ) as mock_run: + result = push_to_remote( + "/clone", "/source", "main", protected_branches=[], force=False + ) + + assert result == {"success": True, "stderr": ""} + second_call = mock_run.call_args_list[1] + cmd = second_call[0][0] + assert cmd == ["git", "push", "-u", "upstream", "main"] + + def test_default_clone_manager_push_to_remote_accepts_force_param(self) -> None: + """T5: DefaultCloneManager.push_to_remote has force keyword param with default False.""" + sig = inspect.signature(DefaultCloneManager.push_to_remote) + assert "force" in sig.parameters, ( + "DefaultCloneManager.push_to_remote must have 'force' param" + ) + param = sig.parameters["force"] + assert param.default is False, "force param must default to False" + + def test_force_with_lease_stale_returns_error_type(self) -> None: + """push_to_remote returns error_type=force_with_lease_stale when git reports stale info.""" + mock_url = MagicMock() + mock_url.returncode = 0 + mock_url.stdout = "git@github.com:org/repo.git\n" + mock_url.stderr = "" + + mock_push = MagicMock() + mock_push.returncode = 1 + mock_push.stderr = "! [rejected] main -> main (stale info)" + + with patch( + "autoskillit.workspace.clone.subprocess.run", side_effect=[mock_url, mock_push] + ): + result = push_to_remote("/clone", "/source", "main", protected_branches=[], force=True) + + assert result["success"] is False + assert result.get("error_type") == "force_with_lease_stale" + + def test_force_with_lease_no_upstream_returns_error_type(self) -> None: + """push_to_remote returns error_type=force_with_lease_no_upstream for missing upstream.""" + mock_url = MagicMock() + mock_url.returncode = 0 + mock_url.stdout = "git@github.com:org/repo.git\n" + mock_url.stderr = "" + + mock_push = MagicMock() + mock_push.returncode = 1 + mock_push.stderr = "error: The current branch main has no upstream branch." + + with patch( + "autoskillit.workspace.clone.subprocess.run", side_effect=[mock_url, mock_push] + ): + result = push_to_remote("/clone", "/source", "main", protected_branches=[], force=True) + + assert result["success"] is False + assert result.get("error_type") == "force_with_lease_no_upstream" + + def test_force_push_generic_failure_has_no_error_type(self) -> None: + """push_to_remote returns no error_type for generic force-push failures.""" + mock_url = MagicMock() + mock_url.returncode = 0 + mock_url.stdout = "git@github.com:org/repo.git\n" + mock_url.stderr = "" + + mock_push = MagicMock() + mock_push.returncode = 1 + mock_push.stderr = "error: failed to push some refs" + + with patch( + "autoskillit.workspace.clone.subprocess.run", side_effect=[mock_url, mock_push] + ): + result = push_to_remote("/clone", "/source", "main", protected_branches=[], force=True) + + assert result["success"] is False + assert "error_type" not in result + + def test_non_force_failure_has_no_error_type(self) -> None: + """push_to_remote returns no error_type for non-force push failures.""" + mock_url = MagicMock() + mock_url.returncode = 0 + mock_url.stdout = "git@github.com:org/repo.git\n" + mock_url.stderr = "" + + mock_push = MagicMock() + mock_push.returncode = 1 + mock_push.stderr = "! [rejected] main -> main (stale info)" + + with patch( + "autoskillit.workspace.clone.subprocess.run", side_effect=[mock_url, mock_push] + ): + result = push_to_remote( + "/clone", "/source", "main", protected_branches=[], force=False + ) + + assert result["success"] is False + assert "error_type" not in result + + def test_gh006_returns_queued_branch_error_type(self) -> None: + """GH006 stderr produces error_type=queued_branch.""" + mock_url = MagicMock() + mock_url.returncode = 0 + mock_url.stdout = "git@github.com:org/repo.git\n" + mock_url.stderr = "" + + mock_push = MagicMock() + mock_push.returncode = 1 + mock_push.stderr = ( + "remote: error: GH006: Protected branch update failed for refs/heads/main.\n" + "remote: error: Changes must be made through a merge queue." + ) + + with patch( + "autoskillit.workspace.clone.subprocess.run", side_effect=[mock_url, mock_push] + ): + result = push_to_remote("/clone", "/source", "main", protected_branches=[], force=True) + + assert result["success"] is False + assert result.get("error_type") == "queued_branch" + + def test_merge_queue_stderr_returns_queued_branch_error_type(self) -> None: + """'protected by merge queue' stderr variant also produces queued_branch.""" + mock_url = MagicMock() + mock_url.returncode = 0 + mock_url.stdout = "git@github.com:org/repo.git\n" + mock_url.stderr = "" + + mock_push = MagicMock() + mock_push.returncode = 1 + mock_push.stderr = "remote: error: branch is protected by merge queue" + + with patch( + "autoskillit.workspace.clone.subprocess.run", side_effect=[mock_url, mock_push] + ): + result = push_to_remote("/clone", "/source", "main", protected_branches=[], force=True) + + assert result["success"] is False + assert result.get("error_type") == "queued_branch" + + def test_gh006_non_force_also_returns_queued_branch(self) -> None: + """GH006 fires for non-force pushes too (queue protects all push modes).""" + mock_url = MagicMock() + mock_url.returncode = 0 + mock_url.stdout = "git@github.com:org/repo.git\n" + mock_url.stderr = "" + + mock_push = MagicMock() + mock_push.returncode = 1 + mock_push.stderr = "remote: error: GH006: Protected branch update failed." + + with patch( + "autoskillit.workspace.clone.subprocess.run", side_effect=[mock_url, mock_push] + ): + result = push_to_remote( + "/clone", "/source", "main", protected_branches=[], force=False + ) + + assert result["success"] is False + assert result.get("error_type") == "queued_branch" + + +class TestPushToRemoteNonBare: + """push_to_remote fails with error_type when remote is a local non-bare repo.""" + + def test_push_fails_with_local_nonbare_remote(self, tmp_path: Path) -> None: + """push_to_remote returns error_type=local_non_bare_remote for non-bare local origin.""" + # upstream is a non-bare local repo with main checked out + upstream = tmp_path / "upstream" + subprocess.run(["git", "init", str(upstream)], check=True, capture_output=True) + subprocess.run( + ["git", "-C", str(upstream), "config", "user.email", "t@t.com"], + check=True, + capture_output=True, + ) + subprocess.run( + ["git", "-C", str(upstream), "config", "user.name", "T"], + check=True, + capture_output=True, + ) + subprocess.run( + ["git", "-C", str(upstream), "commit", "--allow-empty", "-m", "init"], + check=True, + capture_output=True, + ) + + source = tmp_path / "source" + subprocess.run( + ["git", "clone", str(upstream), str(source)], check=True, capture_output=True + ) + subprocess.run( + ["git", "-C", str(source), "config", "user.email", "t@t.com"], + check=True, + capture_output=True, + ) + subprocess.run( + ["git", "-C", str(source), "config", "user.name", "T"], + check=True, + capture_output=True, + ) + subprocess.run( + ["git", "-C", str(source), "commit", "--allow-empty", "-m", "src"], + check=True, + capture_output=True, + ) + + # upstream has main checked out — push from source will be refused + clone_result = clone_repo(str(source), "test-nonbare", strategy="proceed") + clone_path = clone_result["clone_path"] + subprocess.run( + ["git", "-C", clone_path, "config", "user.email", "t@t.com"], + check=True, + capture_output=True, + ) + subprocess.run( + ["git", "-C", clone_path, "config", "user.name", "T"], + check=True, + capture_output=True, + ) + subprocess.run( + ["git", "-C", clone_path, "commit", "--allow-empty", "-m", "pipeline"], + check=True, + capture_output=True, + ) + + result = push_to_remote(clone_path, str(source), "main", protected_branches=[]) + + assert result["success"] is False + assert result.get("error_type") == "local_non_bare_remote" + + shutil.rmtree(Path(clone_path).parent, ignore_errors=True) + + def test_push_to_remote_uses_explicit_remote_url_without_reading_source_dir( + self, tmp_path: Path + ) -> None: + """When remote_url is explicit, source_dir is not accessed for URL lookup.""" + remote = tmp_path / "remote.git" + subprocess.run(["git", "init", "--bare", str(remote)], check=True, capture_output=True) + source = tmp_path / "source" + subprocess.run(["git", "clone", str(remote), str(source)], check=True, capture_output=True) + subprocess.run( + ["git", "-C", str(source), "config", "user.email", "t@t.com"], + check=True, + capture_output=True, + ) + subprocess.run( + ["git", "-C", str(source), "config", "user.name", "T"], + check=True, + capture_output=True, + ) + subprocess.run( + ["git", "-C", str(source), "commit", "--allow-empty", "-m", "init"], + check=True, + capture_output=True, + ) + subprocess.run( + ["git", "-C", str(source), "branch", "-M", "main"], + check=True, + capture_output=True, + ) + subprocess.run( + ["git", "-C", str(source), "push", "origin", "main"], + check=True, + capture_output=True, + ) + + clone_result = clone_repo(str(source), "test-explicit", strategy="proceed") + clone_path = clone_result["clone_path"] + subprocess.run( + ["git", "-C", clone_path, "config", "user.email", "t@t.com"], + check=True, + capture_output=True, + ) + subprocess.run( + ["git", "-C", clone_path, "config", "user.name", "T"], + check=True, + capture_output=True, + ) + subprocess.run( + ["git", "-C", clone_path, "commit", "--allow-empty", "-m", "impl"], + check=True, + capture_output=True, + ) + + # Pass explicit remote_url — source_dir is not needed + result = push_to_remote( + clone_path, remote_url=str(remote), branch="main", protected_branches=[] + ) + + assert result["success"] is True + + shutil.rmtree(Path(clone_path).parent, ignore_errors=True) diff --git a/tests/workspace/test_clone_registry.py b/tests/workspace/test_clone_registry.py index 34008fdc79..bd6d68bf5f 100644 --- a/tests/workspace/test_clone_registry.py +++ b/tests/workspace/test_clone_registry.py @@ -15,6 +15,8 @@ register_clone, ) +pytestmark = [pytest.mark.layer("workspace"), pytest.mark.small] + class TestRegisterClone: """register_clone contract tests.""" @@ -130,8 +132,12 @@ def test_register_clone_persists_owner_field(self, tmp_path: Path) -> None: class TestRegisterCloneRejectsEmptyOwner: """T2: register_clone raises ValueError when owner is empty string.""" - def test_register_clone_rejects_empty_owner(self, tmp_path: Path) -> None: - """T2 — register_clone raises ValueError and does not write registry when owner=''.""" + def test_register_clone_rejects_empty_owner( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + """T2 — raises ValueError when owner='' and no env fallback is set.""" + monkeypatch.delenv("AUTOSKILLIT_CAMPAIGN_ID", raising=False) + monkeypatch.delenv("AUTOSKILLIT_KITCHEN_SESSION_ID", raising=False) reg = str(tmp_path / "registry.json") with pytest.raises(ValueError, match="owner is required"): register_clone("/tmp/a", "success", owner="", registry_path=reg) @@ -359,3 +365,123 @@ def test_batch_delete_other_owner_entries_untouched_on_disk(self, tmp_path: Path remaining = [e["path"] for e in data["clones"]] assert "/tmp/mine" not in remaining # kit-1 entry pruned assert "/tmp/theirs" in remaining # kit-2 entry untouched + + +class TestRegisterCloneEnvDefaulting: + """T20–T25: register_clone env-var defaulting for owner.""" + + def test_register_clone_defaults_owner_from_campaign_id_env( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + """T20 — AUTOSKILLIT_CAMPAIGN_ID env var is used as owner when no explicit owner given.""" + monkeypatch.setenv("AUTOSKILLIT_CAMPAIGN_ID", "camp-1") + monkeypatch.delenv("AUTOSKILLIT_KITCHEN_SESSION_ID", raising=False) + reg = str(tmp_path / "registry.json") + + register_clone("/c", "success", registry_path=reg) + + data = json.loads(Path(reg).read_text()) + assert data["clones"][0]["owner"] == "camp-1" + + def test_register_clone_defaults_owner_from_kitchen_session_id_env( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + """T21 — AUTOSKILLIT_KITCHEN_SESSION_ID is used as owner when CAMPAIGN_ID is absent.""" + monkeypatch.delenv("AUTOSKILLIT_CAMPAIGN_ID", raising=False) + monkeypatch.setenv("AUTOSKILLIT_KITCHEN_SESSION_ID", "kit-session-77") + reg = str(tmp_path / "registry.json") + + register_clone("/c", "success", registry_path=reg) + + data = json.loads(Path(reg).read_text()) + assert data["clones"][0]["owner"] == "kit-session-77" + + def test_register_clone_campaign_id_takes_precedence_over_kitchen_session_id( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + """T22 — When both env vars are set, CAMPAIGN_ID wins.""" + monkeypatch.setenv("AUTOSKILLIT_CAMPAIGN_ID", "camp-priority") + monkeypatch.setenv("AUTOSKILLIT_KITCHEN_SESSION_ID", "kit-session-99") + reg = str(tmp_path / "registry.json") + + register_clone("/c", "success", registry_path=reg) + + data = json.loads(Path(reg).read_text()) + assert data["clones"][0]["owner"] == "camp-priority" + + def test_register_clone_raises_when_no_owner_source( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + """T23 — ValueError raised when neither env var is set and no explicit owner given.""" + monkeypatch.delenv("AUTOSKILLIT_CAMPAIGN_ID", raising=False) + monkeypatch.delenv("AUTOSKILLIT_KITCHEN_SESSION_ID", raising=False) + reg = str(tmp_path / "registry.json") + + with pytest.raises(ValueError, match="owner is required"): + register_clone("/c", "success", registry_path=reg) + assert not Path(reg).exists() + + def test_register_clone_explicit_owner_overrides_env( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + """T24 — Explicit owner argument takes precedence over env vars.""" + monkeypatch.setenv("AUTOSKILLIT_CAMPAIGN_ID", "camp-env") + monkeypatch.setenv("AUTOSKILLIT_KITCHEN_SESSION_ID", "kit-env") + reg = str(tmp_path / "registry.json") + + register_clone("/c", "success", owner="explicit", registry_path=reg) + + data = json.loads(Path(reg).read_text()) + assert data["clones"][0]["owner"] == "explicit" + + def test_register_clone_env_default_works_with_batch_delete( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + """T25 — env-default owner is correctly scoped by batch_delete.""" + campaign_id = "camp-batch-test" + monkeypatch.setenv("AUTOSKILLIT_CAMPAIGN_ID", campaign_id) + monkeypatch.delenv("AUTOSKILLIT_KITCHEN_SESSION_ID", raising=False) + reg = str(tmp_path / "registry.json") + + register_clone("/c1", "success", registry_path=reg) + register_clone("/c2", "success", "other-owner", registry_path=reg) + + mock_remove = MagicMock(return_value={"removed": "true"}) + result = batch_delete(reg, mock_remove, owner=campaign_id) + + assert "/c1" in result["deleted"] + assert "/c2" not in result["deleted"] + mock_remove.assert_called_once_with("/c1", "false") + + +class TestTwoCampaignsIsolation: + """T31: two campaigns register clones via env-default; cleanup is isolated.""" + + def test_two_campaigns_register_and_cleanup_isolated( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + """T31 — Campaign-1 cleanup removes only campaign-1's clones; campaign-2's remain.""" + reg = str(tmp_path / "registry.json") + camp1, camp2 = "camp-alpha", "camp-beta" + + # Campaign 1 registers its clone via env-defaulting + monkeypatch.setenv("AUTOSKILLIT_CAMPAIGN_ID", camp1) + monkeypatch.delenv("AUTOSKILLIT_KITCHEN_SESSION_ID", raising=False) + register_clone("/clone-alpha", "success", registry_path=reg) + + # Campaign 2 registers its clone via env-defaulting + monkeypatch.setenv("AUTOSKILLIT_CAMPAIGN_ID", camp2) + register_clone("/clone-beta", "success", registry_path=reg) + + # Campaign 1 runs cleanup; only its clones are removed + mock_remove = MagicMock(return_value={"removed": "true"}) + result = batch_delete(reg, mock_remove, owner=camp1) + + assert "/clone-alpha" in result["deleted"] + assert "/clone-beta" not in result["deleted"] + + # Campaign 2's clone remains on disk + data = json.loads(Path(reg).read_text()) + remaining = [e["path"] for e in data["clones"]] + assert "/clone-beta" in remaining + assert "/clone-alpha" not in remaining diff --git a/tests/workspace/test_clone_remote.py b/tests/workspace/test_clone_remote.py new file mode 100644 index 0000000000..cd8d9d555a --- /dev/null +++ b/tests/workspace/test_clone_remote.py @@ -0,0 +1,412 @@ +"""Remote resolution tests — _probe_single_remote, _probe_clone_source_url, URL resolution.""" + +from __future__ import annotations + +import shutil +import subprocess +from pathlib import Path +from unittest.mock import MagicMock, patch + +import pytest + +from autoskillit.workspace._clone_remote import ( + _probe_clone_source_url, + _probe_single_remote, +) +from autoskillit.workspace.clone import clone_repo + +pytestmark = [pytest.mark.layer("workspace"), pytest.mark.medium] + + +class TestCloneRemoteUrlResolution: + """T1: clone_repo resolves remote URL before cloning when origin is configured.""" + + # T1-A + def test_clone_uses_remote_url_as_clone_source( + self, local_with_remote: Path, bare_remote: Path + ) -> None: + """clone_repo uses the remote URL (not the local path) as git clone source.""" + with patch( + "autoskillit.workspace.clone.subprocess.run", + wraps=subprocess.run, + ) as spy: + result = clone_repo(str(local_with_remote), "test", branch="main", strategy="proceed") + clone_path = Path(result["clone_path"]) + try: + clone_calls = [ + call + for call in spy.call_args_list + if call[0] and isinstance(call[0][0], list) and call[0][0][:2] == ["git", "clone"] + ] + assert len(clone_calls) == 1, ( + f"Expected exactly one git clone call, got {len(clone_calls)}" + ) + cmd = clone_calls[0][0][0] + # Extract positional args by skipping flags and their values. + positional: list[str] = [] + i = cmd.index("clone") + 1 + while i < len(cmd): + if cmd[i].startswith("-"): + i += 2 # skip flag and its value + else: + positional.append(cmd[i]) + i += 1 + assert len(positional) == 2, f"Expected [source, target] in clone cmd: {cmd}" + clone_source = positional[0] + assert clone_source == str(bare_remote), ( + f"Expected clone source to be remote URL {bare_remote!r}, " + f"got {clone_source!r} (local path was used instead)" + ) + assert result["clone_source_type"] == "remote", ( + f"Expected clone_source_type='remote', got {result.get('clone_source_type')!r}" + ) + finally: + shutil.rmtree(clone_path.parent, ignore_errors=True) + + # T1-B + def test_clone_raises_on_no_origin_with_proceed_strategy(self, git_repo: Path) -> None: + """clone_repo raises RuntimeError when no remote origin and strategy=proceed. + + Previously the code silently fell back to cloning from the local path + (git clone /abs/path via local transport). After the fix, the no_origin + probe result causes an immediate RuntimeError, instructing the caller to + use strategy="clone_local" for an intentional local-only clone. + """ + with pytest.raises(RuntimeError, match="clone_origin_probe_failed.*no_origin"): + clone_repo(str(git_repo), "test", strategy="proceed") + + # T1-C + def test_clone_result_remote_url_correct_after_remote_clone( + self, local_with_remote: Path, bare_remote: Path + ) -> None: + """clone_repo result['remote_url'] equals the remote URL after cloning.""" + result = clone_repo(str(local_with_remote), "test", branch="main", strategy="proceed") + clone_path = Path(result["clone_path"]) + try: + assert result["remote_url"] == str(bare_remote) + assert result["clone_source_type"] == "remote" + assert result["clone_source_reason"] == "ok" + finally: + shutil.rmtree(clone_path.parent, ignore_errors=True) + + # T1-D + def test_clone_uses_remote_when_branch_not_on_remote( + self, local_with_remote: Path, bare_remote: Path + ) -> None: + """Regression: clone always uses remote URL even when branch not on remote. + + Previously the origin probe collapse allowed falling back to the local path + when ls-remote did not find the branch on the remote. After the fix the probe + always uses the remote URL when one is configured, so git clone fails (correctly) + instead of silently cloning local state. + """ + with patch( + "autoskillit.workspace.clone.subprocess.run", + wraps=subprocess.run, + ) as spy: + with pytest.raises(RuntimeError, match="git clone failed"): + clone_repo( + str(local_with_remote), + "test", + branch="feature/local-only", + strategy="proceed", + ) + clone_calls = [ + call + for call in spy.call_args_list + if call[0] and isinstance(call[0][0], list) and call[0][0][:2] == ["git", "clone"] + ] + assert len(clone_calls) == 1, "Expected exactly one git clone call" + cmd = clone_calls[0][0][0] + # Extract positional args from git clone, skipping flags and their values. + # Handles any optional flags (--branch, --no-hardlinks, --depth, etc.) + # without relying on a fixed index like [-2]. + positional: list[str] = [] + i = cmd.index("clone") + 1 + while i < len(cmd): + if cmd[i].startswith("-"): + i += 2 # skip flag and its value + else: + positional.append(cmd[i]) + i += 1 + assert len(positional) == 2, f"Expected [source, target] in clone cmd: {cmd}" + clone_source = positional[0] + assert clone_source == str(bare_remote), ( + f"Expected remote URL {bare_remote!r} as clone source, " + f"got {clone_source!r} — local path fallback detected" + ) + + +class TestProbeSingleRemote: + """Unit tests for _probe_single_remote helper.""" + + def test_probe_single_remote_returns_ok_reason_when_remote_configured( + self, local_with_remote: Path, bare_remote: Path + ) -> None: + resolution = _probe_single_remote(local_with_remote, "origin") + assert resolution.reason == "ok" + assert resolution.url == str(bare_remote) + assert resolution.stderr == "" + + def test_probe_single_remote_returns_no_origin_reason_when_no_remote( + self, git_repo: Path + ) -> None: + resolution = _probe_single_remote(git_repo, "origin") + assert resolution.reason == "no_origin" + assert resolution.url == "" + + def test_probe_single_remote_returns_timeout_reason_on_timeout(self, tmp_path: Path) -> None: + with patch( + "autoskillit.workspace._clone_remote.subprocess.run", + side_effect=subprocess.TimeoutExpired(cmd=["git"], timeout=30), + ): + resolution = _probe_single_remote(tmp_path, "origin") + assert resolution.reason == "timeout" + assert resolution.url == "" + + def test_probe_single_remote_returns_error_reason_on_non_zero_rc(self, tmp_path: Path) -> None: + mock_result = MagicMock() + mock_result.returncode = 1 + mock_result.stdout = "" + mock_result.stderr = "fatal: not a git repository" + with patch("autoskillit.workspace._clone_remote.subprocess.run", return_value=mock_result): + resolution = _probe_single_remote(tmp_path, "origin") + assert resolution.reason == "error" + assert resolution.url == "" + assert resolution.stderr == "fatal: not a git repository" + + +class TestProbeCloneSourceUrl: + """Unit tests for the updated _probe_clone_source_url URL resolution logic.""" + + def test_prefers_upstream_network_url_over_file_origin(self, tmp_path: Path) -> None: + """When origin=file:// and upstream=network URL, uses upstream (the key bug fix). + + This is the exact scenario that caused stale clones in multi-batch pipelines: + source_dir is a previous autoskillit clone with origin rewritten to file://. + """ + bare = tmp_path / "bare.git" + subprocess.run(["git", "init", "--bare", str(bare)], check=True, capture_output=True) + source = tmp_path / "source" + subprocess.run(["git", "init", str(source)], check=True, capture_output=True) + subprocess.run( + ["git", "-C", str(source), "config", "user.email", "t@t.com"], + check=True, + capture_output=True, + ) + subprocess.run( + ["git", "-C", str(source), "config", "user.name", "T"], check=True, capture_output=True + ) + subprocess.run( + ["git", "-C", str(source), "commit", "--allow-empty", "-m", "init"], + check=True, + capture_output=True, + ) + # Simulate a previous _ensure_origin_isolated call: origin=file://, upstream=real remote + subprocess.run( + ["git", "-C", str(source), "remote", "add", "origin", f"file://{source}"], + check=True, + capture_output=True, + ) + subprocess.run( + ["git", "-C", str(source), "remote", "add", "upstream", str(bare)], + check=True, + capture_output=True, + ) + + result = _probe_clone_source_url(source) + + assert result.reason == "ok" + assert result.url == str(bare), ( + f"Expected upstream URL {bare!r}, got {result.url!r}. " + "When origin=file://, upstream should be preferred." + ) + + def test_falls_back_to_origin_when_no_upstream(self, tmp_path: Path) -> None: + """Without upstream remote, falls back to origin URL (existing behavior preserved).""" + bare = tmp_path / "bare.git" + subprocess.run(["git", "init", "--bare", str(bare)], check=True, capture_output=True) + source = tmp_path / "source" + subprocess.run(["git", "clone", str(bare), str(source)], check=True, capture_output=True) + subprocess.run( + ["git", "-C", str(source), "config", "user.email", "t@t.com"], + check=True, + capture_output=True, + ) + subprocess.run( + ["git", "-C", str(source), "config", "user.name", "T"], check=True, capture_output=True + ) + subprocess.run( + ["git", "-C", str(source), "commit", "--allow-empty", "-m", "init"], + check=True, + capture_output=True, + ) + # Only origin is configured, no upstream + + result = _probe_clone_source_url(source) + + assert result.reason == "ok" + assert result.url == str(bare) + + def test_uses_origin_when_upstream_is_file_url_and_origin_is_network( + self, tmp_path: Path + ) -> None: + """When upstream=file:// and origin=non-file-local-path that is network-equivalent, + falls through to origin result (covers edge cases where upstream is also local).""" + bare = tmp_path / "bare.git" + subprocess.run(["git", "init", "--bare", str(bare)], check=True, capture_output=True) + source = tmp_path / "source" + subprocess.run(["git", "init", str(source)], check=True, capture_output=True) + subprocess.run( + ["git", "-C", str(source), "config", "user.email", "t@t.com"], + check=True, + capture_output=True, + ) + subprocess.run( + ["git", "-C", str(source), "config", "user.name", "T"], check=True, capture_output=True + ) + subprocess.run( + ["git", "-C", str(source), "commit", "--allow-empty", "-m", "init"], + check=True, + capture_output=True, + ) + # upstream is a file:// URL (not a real network URL), origin is the bare path + subprocess.run( + ["git", "-C", str(source), "remote", "add", "upstream", f"file://{source}"], + check=True, + capture_output=True, + ) + subprocess.run( + ["git", "-C", str(source), "remote", "add", "origin", str(bare)], + check=True, + capture_output=True, + ) + + result = _probe_clone_source_url(source) + + # upstream is file:// → excluded by _is_not_file_url → falls through to origin + assert result.reason == "ok" + assert result.url == str(bare) + + def test_returns_no_origin_for_repo_without_remotes(self, tmp_path: Path) -> None: + """Repo with no remotes returns reason='no_origin' (unchanged from current behavior).""" + source = tmp_path / "source" + subprocess.run(["git", "init", str(source)], check=True, capture_output=True) + subprocess.run( + ["git", "-C", str(source), "config", "user.email", "t@t.com"], + check=True, + capture_output=True, + ) + subprocess.run( + ["git", "-C", str(source), "config", "user.name", "T"], check=True, capture_output=True + ) + subprocess.run( + ["git", "-C", str(source), "commit", "--allow-empty", "-m", "init"], + check=True, + capture_output=True, + ) + + result = _probe_clone_source_url(source) + + assert result.reason == "no_origin" + assert result.url == "" + + +class TestCloneFromPreviousAutoskillitClone: + """Regression: cloning from a previous autoskillit clone must use the network remote.""" + + def test_clone_from_previous_clone_uses_upstream_not_stale_local(self, tmp_path: Path) -> None: + """Batch N+1 clones from batch N's clone must get fresh HEAD from the network remote, + not the stale local state of the previous clone. + + Setup: + bare_remote (has commit A + commit B) + source (has commit A only — stale) + source has: origin=file://source (isolation), upstream=bare_remote + + Expected: clone_from_source gets commit B (fetched from bare_remote via upstream). + Bug behavior (pre-fix): gets only commit A (cloned from file://source via origin). + """ + bare_remote = tmp_path / "bare.git" + subprocess.run( + ["git", "init", "--bare", "--initial-branch=main", str(bare_remote)], + check=True, + capture_output=True, + ) + + # source: has commit A, stale (does not have commit B) + source = tmp_path / "source" + subprocess.run(["git", "init", str(source)], check=True, capture_output=True) + subprocess.run( + ["git", "-C", str(source), "config", "user.email", "t@t.com"], + check=True, + capture_output=True, + ) + subprocess.run( + ["git", "-C", str(source), "config", "user.name", "T"], check=True, capture_output=True + ) + (source / "a.txt").write_text("commit A") + subprocess.run(["git", "-C", str(source), "add", "."], check=True, capture_output=True) + subprocess.run( + ["git", "-C", str(source), "commit", "-m", "commit A"], check=True, capture_output=True + ) + subprocess.run( + ["git", "-C", str(source), "branch", "-M", "main"], check=True, capture_output=True + ) + subprocess.run( + ["git", "-C", str(source), "push", str(bare_remote), "main"], + check=True, + capture_output=True, + ) + + # Simulate _ensure_origin_isolated: source.origin = file://, source.upstream = bare_remote + subprocess.run( + ["git", "-C", str(source), "remote", "add", "origin", f"file://{source}"], + check=True, + capture_output=True, + ) + subprocess.run( + ["git", "-C", str(source), "remote", "add", "upstream", str(bare_remote)], + check=True, + capture_output=True, + ) + + # Add commit B to bare_remote directly (simulates another batch merging) + tmp_push = tmp_path / "push_helper" + subprocess.run( + ["git", "clone", str(bare_remote), str(tmp_push)], check=True, capture_output=True + ) + subprocess.run( + ["git", "-C", str(tmp_push), "config", "user.email", "t@t.com"], + check=True, + capture_output=True, + ) + subprocess.run( + ["git", "-C", str(tmp_push), "config", "user.name", "T"], + check=True, + capture_output=True, + ) + (tmp_push / "b.txt").write_text("commit B") + subprocess.run(["git", "-C", str(tmp_push), "add", "."], check=True, capture_output=True) + subprocess.run( + ["git", "-C", str(tmp_push), "commit", "-m", "commit B"], + check=True, + capture_output=True, + ) + subprocess.run( + ["git", "-C", str(tmp_push), "push", "origin", "main"], check=True, capture_output=True + ) + + # Now clone from source (which is stale — missing commit B) + result = clone_repo(str(source), "batch2", branch="main", strategy="proceed") + clone_path = Path(result["clone_path"]) + + try: + # The clone MUST have b.txt (from bare_remote commit B), not just a.txt + assert (clone_path / "b.txt").exists(), ( + "Clone is missing b.txt — it cloned from the stale local source instead of " + "the network remote (bare_remote). This is the #817 regression." + ) + finally: + shutil.rmtree(clone_path.parent, ignore_errors=True) + shutil.rmtree(tmp_push, ignore_errors=True) diff --git a/tests/workspace/test_clone_split.py b/tests/workspace/test_clone_split.py new file mode 100644 index 0000000000..519e1e977d --- /dev/null +++ b/tests/workspace/test_clone_split.py @@ -0,0 +1,27 @@ +import pytest + +pytestmark = [pytest.mark.layer("workspace"), pytest.mark.small] + + +def test_detect_helpers_importable_from_submodule(): + from autoskillit.workspace._clone_detect import ( + classify_remote_url, + ) + + assert callable(classify_remote_url) + + +def test_remote_helpers_importable_from_submodule(): + from autoskillit.workspace._clone_remote import ( + CloneSourceResolution, + ) + + assert CloneSourceResolution is not None + + +def test_workspace_init_surface_unchanged(): + from autoskillit.workspace import ( + clone_repo, + ) + + assert callable(clone_repo) diff --git a/tests/workspace/test_clone_timeouts.py b/tests/workspace/test_clone_timeouts.py index 4736dee187..038b5e75e8 100644 --- a/tests/workspace/test_clone_timeouts.py +++ b/tests/workspace/test_clone_timeouts.py @@ -5,12 +5,16 @@ import ast from pathlib import Path +import pytest + +pytestmark = [pytest.mark.layer("workspace"), pytest.mark.small] + _GIT_NETWORK_SUBCOMMANDS = {"push", "clone", "fetch", "pull", "ls-remote"} def test_git_network_commands_have_timeout() -> None: """All subprocess.run() calls with git network commands must have timeout=.""" - src = Path("src/autoskillit/workspace/clone.py").read_text() + src = (Path(__file__).parent.parent.parent / "src/autoskillit/workspace/clone.py").read_text() tree = ast.parse(src) for node in ast.walk(tree): if not ( diff --git a/tests/workspace/test_constants.py b/tests/workspace/test_constants.py index c47718efbc..5266cee506 100644 --- a/tests/workspace/test_constants.py +++ b/tests/workspace/test_constants.py @@ -1,6 +1,10 @@ # tests/workspace/test_constants.py """Asserts that workspace directory name constants are exported from workspace/__init__.""" +import pytest + +pytestmark = [pytest.mark.layer("workspace"), pytest.mark.small] + def test_runs_dir_constant_is_exported(): from autoskillit.workspace import RUNS_DIR diff --git a/tests/workspace/test_project_local_overrides.py b/tests/workspace/test_project_local_overrides.py index 1d7f6de01e..4c54f5c844 100644 --- a/tests/workspace/test_project_local_overrides.py +++ b/tests/workspace/test_project_local_overrides.py @@ -2,8 +2,12 @@ from __future__ import annotations +import pytest + from autoskillit.core.types import PACK_REGISTRY +pytestmark = [pytest.mark.layer("workspace"), pytest.mark.small] + # Tags for packs that are disabled by default (e.g. research, exp-lens). # Shared by T-OVR-014 and T-OVR-017 to avoid duplication. _DEFAULT_DISABLED_TAGS: frozenset[str] = frozenset( @@ -133,11 +137,11 @@ def test_init_session_includes_non_overridden_skills(tmp_path): def test_init_session_subset_and_override_compose(tmp_path): """T-OVR-010: Subset disable and override compose independently.""" - from autoskillit.config import AutomationConfig, SubsetsConfig from autoskillit.workspace.session_skills import ( DefaultSessionSkillManager, SkillsDirectoryProvider, ) + from tests._helpers import make_subsetsconfig, make_test_config project_dir = tmp_path / "project" project_dir.mkdir() @@ -146,7 +150,7 @@ def test_init_session_subset_and_override_compose(tmp_path): override.mkdir(parents=True) (override / "SKILL.md").write_text("# custom") # Config disables "github" subset (which covers open-pr) - config = AutomationConfig(subsets=SubsetsConfig(disabled=["github"])) + config = make_test_config(subsets=make_subsetsconfig(disabled=["github"])) mgr = DefaultSessionSkillManager(SkillsDirectoryProvider(), tmp_path / "ephemeral") skills_dir = mgr.init_session("sess-004", config=config, project_dir=project_dir) # "open-pr" absent due to subset; "review-pr" absent due to override @@ -202,13 +206,13 @@ def test_init_session_cook_session_excludes_project_local_overrides(tmp_path): def test_init_session_cook_session_ignores_disabled_subsets(tmp_path): """T-OVR-013: cook_session=True includes subset-disabled skills.""" - from autoskillit.config import AutomationConfig, SubsetsConfig from autoskillit.workspace.session_skills import ( DefaultSessionSkillManager, SkillsDirectoryProvider, ) + from tests._helpers import make_subsetsconfig, make_test_config - config = AutomationConfig(subsets=SubsetsConfig(disabled=["github"])) + config = make_test_config(subsets=make_subsetsconfig(disabled=["github"])) mgr = DefaultSessionSkillManager(SkillsDirectoryProvider(), tmp_path / "ephemeral") skills_dir = mgr.init_session("sess-cook2", cook_session=True, config=config) assert (skills_dir / ".claude" / "skills" / "compose-pr" / "SKILL.md").exists(), ( @@ -224,13 +228,13 @@ def test_init_session_cook_full_skill_set_invariant(tmp_path): The cook bypasses explicit subset-disable filtering but NOT channel deduplication and NOT default pack gating. """ - from autoskillit.config import AutomationConfig, SubsetsConfig from autoskillit.core.types import SkillSource from autoskillit.workspace.session_skills import ( DefaultSessionSkillManager, SkillsDirectoryProvider, ) from autoskillit.workspace.skills import DefaultSkillResolver + from tests._helpers import make_subsetsconfig, make_test_config # Override exactly one extended skill to test override exclusion project_dir = tmp_path / "project" @@ -240,8 +244,9 @@ def test_init_session_cook_full_skill_set_invariant(tmp_path): (override / "SKILL.md").write_text("# override") # Config disables all known categories — cook should bypass this - config = AutomationConfig( - subsets=SubsetsConfig(disabled=["github", "audit", "arch-lens", "ci"]) + config = make_test_config( + subsets=make_subsetsconfig(disabled=["github", "audit", "arch-lens", "ci"]), + features={"fleet": True}, ) mgr = DefaultSessionSkillManager(SkillsDirectoryProvider(), tmp_path / "ephemeral") skills_dir = mgr.init_session( @@ -305,6 +310,7 @@ def test_cook_session_retains_non_colliding_extended_skills(tmp_path): SkillsDirectoryProvider, ) from autoskillit.workspace.skills import DefaultSkillResolver + from tests._helpers import make_test_config # Override exactly one extended skill project_dir = tmp_path / "project" @@ -313,8 +319,11 @@ def test_cook_session_retains_non_colliding_extended_skills(tmp_path): override.mkdir(parents=True) (override / "SKILL.md").write_text("# custom") + config = make_test_config(features={"fleet": True}) mgr = DefaultSessionSkillManager(SkillsDirectoryProvider(), tmp_path / "ephemeral") - skills_dir = mgr.init_session("sess-retain", cook_session=True, project_dir=project_dir) + skills_dir = mgr.init_session( + "sess-retain", cook_session=True, config=config, project_dir=project_dir + ) resolver = DefaultSkillResolver() expected = { diff --git a/tests/workspace/test_session_skills.py b/tests/workspace/test_session_skills.py deleted file mode 100644 index d9fe82c6cf..0000000000 --- a/tests/workspace/test_session_skills.py +++ /dev/null @@ -1,1041 +0,0 @@ -"""Phase 2 tests: session_skills module (resolver, provider, manager).""" - -from __future__ import annotations - -import os -import re -import time -from pathlib import Path - -import pytest -import yaml - -from autoskillit.workspace.session_skills import ( - DefaultSessionSkillManager, - SkillsDirectoryProvider, - resolve_ephemeral_root, -) - - -def test_resolve_ephemeral_root_returns_writable_dir( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch -) -> None: - import autoskillit.workspace.session_skills as ss - - monkeypatch.setattr(ss, "_CANDIDATE_ROOTS", [tmp_path]) - root = resolve_ephemeral_root() - assert root.exists() - assert root.is_dir() - test_file = root / "write_test.tmp" - test_file.write_text("ok") - test_file.unlink() - - -def test_resolve_ephemeral_root_fallback(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: - import autoskillit.workspace.session_skills as ss - - monkeypatch.setattr(ss, "_CANDIDATE_ROOTS", [Path("/nonexistent"), tmp_path]) - root = ss.resolve_ephemeral_root() - assert root.exists() - - -def test_skills_directory_provider_lists_all_skills() -> None: - provider = SkillsDirectoryProvider() - skills = provider.list_skills() - names = {s.name for s in skills} - assert "open-kitchen" in names - assert "close-kitchen" in names - assert "implement-worktree" in names - assert "sous-chef" not in names # internal, excluded - - -def test_provider_injects_disable_model_invocation_for_tier2() -> None: - provider = SkillsDirectoryProvider() - content = provider.get_skill_content("open-kitchen", gated=True) - fm_match = re.match(r"^---\n(.*?)\n---", content, re.DOTALL) - assert fm_match, "Content must have YAML frontmatter" - fm = yaml.safe_load(fm_match.group(1)) - assert fm.get("disable-model-invocation") is True - - -def test_provider_does_not_inject_for_cook_session() -> None: - # Use mermaid (skills_extended/, no flag at rest) to verify that gated=False - # returns unmodified content without injecting disable-model-invocation. - # open-kitchen and close-kitchen carry disable-model-invocation: true in their source - # (human-only skills), so they cannot be used to assert "flag not present". - provider = SkillsDirectoryProvider() - content = provider.get_skill_content("mermaid", gated=False) - fm_match = re.match(r"^---\n(.*?)\n---", content, re.DOTALL) - assert fm_match, "Content must have YAML frontmatter" - fm = yaml.safe_load(fm_match.group(1)) - assert fm.get("disable-model-invocation") is not True - - -def test_session_skill_manager_creates_ephemeral_dir(tmp_path: Path) -> None: - provider = SkillsDirectoryProvider() - mgr = DefaultSessionSkillManager(provider, ephemeral_root=tmp_path) - session_path = mgr.init_session("test-session-abc", cook_session=False) - assert session_path.exists() - assert session_path.is_dir() - skill_files = list(session_path.glob(".claude/skills/*/SKILL.md")) - assert len(skill_files) > 0 - - -def test_session_manager_injects_disable_for_tier2(tmp_path: Path) -> None: - """Non-cook init_session injects disable-model-invocation for tier2 skills.""" - from autoskillit.config.settings import AutomationConfig, SkillsConfig - - provider = SkillsDirectoryProvider() - mgr = DefaultSessionSkillManager(provider, ephemeral_root=tmp_path) - config = AutomationConfig( - skills=SkillsConfig( - tier1=["open-kitchen", "close-kitchen"], - tier2=["mermaid"], - tier3=[], - ) - ) - session_path = mgr.init_session("test-session-xyz", cook_session=False, config=config) - mermaid_md = session_path / ".claude" / "skills" / "mermaid" / "SKILL.md" - assert mermaid_md.exists() - content = mermaid_md.read_text() - fm_match = re.match(r"^---\n(.*?)\n---", content, re.DOTALL) - assert fm_match - fm = yaml.safe_load(fm_match.group(1)) - assert fm.get("disable-model-invocation") is True - - -def test_session_manager_no_flag_for_cook_session(tmp_path: Path) -> None: - """Cook session does not inject disable-model-invocation even for tier2 skills.""" - from autoskillit.config.settings import AutomationConfig, SkillsConfig - - provider = SkillsDirectoryProvider() - mgr = DefaultSessionSkillManager(provider, ephemeral_root=tmp_path) - config = AutomationConfig( - skills=SkillsConfig( - tier1=["open-kitchen", "close-kitchen"], - tier2=["mermaid"], - tier3=[], - ) - ) - session_path = mgr.init_session("cook-session-123", cook_session=True, config=config) - mermaid_md = session_path / ".claude" / "skills" / "mermaid" / "SKILL.md" - content = mermaid_md.read_text() - fm_match = re.match(r"^---\n(.*?)\n---", content, re.DOTALL) - assert fm_match - fm = yaml.safe_load(fm_match.group(1)) - assert fm.get("disable-model-invocation") is not True - - -def test_activate_skill_deps_removes_flag(tmp_path: Path) -> None: - from autoskillit.config.settings import AutomationConfig, SkillsConfig - - provider = SkillsDirectoryProvider() - mgr = DefaultSessionSkillManager(provider, ephemeral_root=tmp_path) - config = AutomationConfig( - skills=SkillsConfig( - tier1=["open-kitchen", "close-kitchen"], - tier2=["mermaid"], - tier3=[], - ) - ) - mgr.init_session("session-toggle", cook_session=False, config=config) - result = mgr.activate_skill_deps("session-toggle", "mermaid") - assert result is True - mermaid_md = tmp_path / "session-toggle" / ".claude" / "skills" / "mermaid" / "SKILL.md" - content = mermaid_md.read_text() - fm_match = re.match(r"^---\n(.*?)\n---", content, re.DOTALL) - assert fm_match - fm = yaml.safe_load(fm_match.group(1)) - assert "disable-model-invocation" not in fm or fm.get("disable-model-invocation") is not True - - -def test_cleanup_stale_removes_old_dirs(tmp_path: Path) -> None: - provider = SkillsDirectoryProvider() - mgr = DefaultSessionSkillManager(provider, ephemeral_root=tmp_path) - stale_dir = tmp_path / "stale-session" - stale_dir.mkdir() - os.utime(stale_dir, (time.time() - 90000, time.time() - 90000)) # 25h old - fresh_dir = tmp_path / "fresh-session" - fresh_dir.mkdir() - count = mgr.cleanup_stale(max_age_seconds=86400) - assert count == 1 - assert not stale_dir.exists() - assert fresh_dir.exists() - - -def test_tier2_skills_constant_removed() -> None: - """TIER2_SKILLS no longer exported from workspace (superseded by config).""" - import autoskillit.workspace as ws - - assert not hasattr(ws, "TIER2_SKILLS") - - -def test_init_session_unknown_skill_logs_warning(tmp_path: Path) -> None: - """Unknown skill name in config.skills.tier2 logs a warning (REQ-TIER-010).""" - import structlog.testing - - from autoskillit.config.settings import AutomationConfig, SkillsConfig - - config = AutomationConfig( - skills=SkillsConfig( - tier1=["open-kitchen", "close-kitchen"], - tier2=["this-skill-does-not-exist-anywhere"], - tier3=[], - ) - ) - mgr = DefaultSessionSkillManager(SkillsDirectoryProvider(), ephemeral_root=tmp_path) - with structlog.testing.capture_logs() as cap_logs: - mgr.init_session("test_unknown_warn", cook_session=False, config=config) - assert any( - "this-skill-does-not-exist-anywhere" in str(entry.get("event", "")) - for entry in cap_logs - if entry.get("log_level") == "warning" - ) - - -# T-VIS-006 -def test_init_session_skips_disabled_builtin_category(tmp_path: Path) -> None: - """Skills whose SKILL.md categories overlap disabled built-in tags are excluded.""" - from unittest.mock import MagicMock - - from autoskillit.config.settings import AutomationConfig, SubsetsConfig - from autoskillit.workspace.session_skills import DefaultSessionSkillManager - from autoskillit.workspace.skills import SkillInfo, SkillSource - - skill_dir = tmp_path / "skills" / "fake-github-skill" - skill_dir.mkdir(parents=True) - (skill_dir / "SKILL.md").write_text("---\ncategories:\n - github\n---\n# Fake GitHub Skill\n") - - provider = MagicMock() - provider.list_skills.return_value = [ - SkillInfo( - name="fake-github-skill", - source=SkillSource.BUNDLED_EXTENDED, - path=skill_dir / "SKILL.md", - categories=frozenset({"github"}), - ) - ] - - config = AutomationConfig(subsets=SubsetsConfig(disabled=["github"])) - root = tmp_path / "sessions" - root.mkdir() - mgr = DefaultSessionSkillManager(provider, ephemeral_root=root) - session_path = mgr.init_session("test-subset-skip", config=config) - - assert not (session_path / ".claude" / "skills" / "fake-github-skill").exists(), ( - "Skill with disabled category 'github' must not be copied to ephemeral dir" - ) - - -# T-VIS-007 -def test_init_session_skips_disabled_custom_tag(tmp_path: Path) -> None: - """Skills listed under a custom_tag that is disabled are excluded.""" - from unittest.mock import MagicMock - - from autoskillit.config.settings import AutomationConfig, SubsetsConfig - from autoskillit.workspace.session_skills import DefaultSessionSkillManager - from autoskillit.workspace.skills import SkillInfo, SkillSource - - skill_dir = tmp_path / "skills" / "my-custom-skill" - skill_dir.mkdir(parents=True) - (skill_dir / "SKILL.md").write_text("---\ncategories: []\n---\n# My Custom Skill\n") - - provider = MagicMock() - provider.list_skills.return_value = [ - SkillInfo( - name="my-custom-skill", - source=SkillSource.BUNDLED_EXTENDED, - path=skill_dir / "SKILL.md", - categories=frozenset(), - ) - ] - provider.get_skill_content.return_value = "---\ncategories: []\n---\n# My Custom Skill\n" - - config = AutomationConfig( - subsets=SubsetsConfig( - disabled=["data-infra"], - custom_tags={"data-infra": ["my-custom-skill"]}, - ) - ) - root = tmp_path / "sessions" - root.mkdir() - mgr = DefaultSessionSkillManager(provider, ephemeral_root=root) - session_path = mgr.init_session("test-custom-skip", config=config) - - assert not (session_path / ".claude" / "skills" / "my-custom-skill").exists(), ( - "Skill listed under disabled custom_tag must not be copied" - ) - - -# T-VIS-008 -def test_init_session_includes_non_disabled_skills(tmp_path: Path) -> None: - """Skills not in any disabled category are still copied to ephemeral dir.""" - from unittest.mock import MagicMock - - from autoskillit.config.settings import AutomationConfig, SubsetsConfig - from autoskillit.workspace.session_skills import DefaultSessionSkillManager - from autoskillit.workspace.skills import SkillInfo, SkillSource - - skill_dir = tmp_path / "skills" / "safe-skill" - skill_dir.mkdir(parents=True) - (skill_dir / "SKILL.md").write_text("---\ncategories:\n - audit\n---\n# Safe Skill\n") - - provider = MagicMock() - provider.list_skills.return_value = [ - SkillInfo( - name="safe-skill", - source=SkillSource.BUNDLED_EXTENDED, - path=skill_dir / "SKILL.md", - categories=frozenset({"audit"}), - ) - ] - provider.get_skill_content.return_value = "---\ncategories:\n - audit\n---\n# Safe Skill\n" - - config = AutomationConfig(subsets=SubsetsConfig(disabled=["github"])) - root = tmp_path / "sessions" - root.mkdir() - mgr = DefaultSessionSkillManager(provider, ephemeral_root=root) - session_path = mgr.init_session("test-safe-include", config=config) - - assert (session_path / ".claude" / "skills" / "safe-skill").exists(), ( - "Skills in non-disabled categories must be included" - ) - - -# REQ-EPH-002 -def test_cleanup_stale_default_is_72_hours() -> None: - import inspect - - sig = inspect.signature(DefaultSessionSkillManager.cleanup_stale) - default = sig.parameters["max_age_seconds"].default - assert default == 259200, f"Expected 259200 (72h), got {default}" - - -# REQ-PACK-005 / REQ-PACK-006: _resolve_effective_disabled -def test_resolve_effective_disabled_default_disabled_excluded() -> None: - from autoskillit.core import PACK_REGISTRY - from autoskillit.workspace.session_skills import _resolve_effective_disabled - - result = _resolve_effective_disabled( - explicit_disabled=[], - pack_registry=PACK_REGISTRY, - packs_enabled=[], - recipe_packs=None, - ) - # Default-disabled packs (research, exp-lens) should be in result - assert "research" in result - assert "exp-lens" in result - # Default-enabled packs should NOT be disabled - assert "github" not in result - - -def test_resolve_effective_disabled_packs_enabled_overrides_default() -> None: - from autoskillit.core import PACK_REGISTRY - from autoskillit.workspace.session_skills import _resolve_effective_disabled - - result = _resolve_effective_disabled( - explicit_disabled=[], - pack_registry=PACK_REGISTRY, - packs_enabled=["research"], - recipe_packs=None, - ) - assert "research" not in result # enabled by packs.enabled - - -# REQ-PACK-004: subsets.disabled always overrides packs.enabled -def test_resolve_effective_disabled_explicit_wins_over_pack_enabled() -> None: - from autoskillit.core import PACK_REGISTRY - from autoskillit.workspace.session_skills import _resolve_effective_disabled - - result = _resolve_effective_disabled( - explicit_disabled=["github"], # explicitly disabled - pack_registry=PACK_REGISTRY, - packs_enabled=["github"], # also in packs.enabled — explicit wins - recipe_packs=None, - ) - assert "github" in result # explicit disable survives - - -def test_resolve_effective_disabled_recipe_packs_overrides_default() -> None: - from autoskillit.core import PACK_REGISTRY - from autoskillit.workspace.session_skills import _resolve_effective_disabled - - result = _resolve_effective_disabled( - explicit_disabled=[], - pack_registry=PACK_REGISTRY, - packs_enabled=[], - recipe_packs=frozenset(["research"]), - ) - assert "research" not in result # enabled by recipe - - -# REQ-PACK-006: Cook sessions skip default-disabled packs -def test_cook_session_skips_default_disabled_packs(tmp_path: Path) -> None: - """Cook session excludes default-disabled pack skills when packs.enabled=[].""" - from unittest.mock import MagicMock - - from autoskillit.config.settings import AutomationConfig - from autoskillit.workspace.skills import SkillInfo, SkillSource - - skill_dir = tmp_path / "skills" / "research-skill" - skill_dir.mkdir(parents=True) - (skill_dir / "SKILL.md").write_text("---\ncategories:\n - research\n---\n# Research Skill\n") - - provider = MagicMock() - provider.list_skills.return_value = [ - SkillInfo( - name="research-skill", - source=SkillSource.BUNDLED_EXTENDED, - path=skill_dir / "SKILL.md", - categories=frozenset({"research"}), - ) - ] - provider.get_skill_content.return_value = ( - "---\ncategories:\n - research\n---\n# Research Skill\n" - ) - - root = tmp_path / "sessions" - root.mkdir() - config = AutomationConfig() # packs.enabled=[] by default - mgr = DefaultSessionSkillManager(provider, ephemeral_root=root) - session_path = mgr.init_session("cook-research", cook_session=True, config=config) - - assert not (session_path / ".claude" / "skills" / "research-skill").exists(), ( - "Default-disabled pack skill must not be in cook session when packs.enabled=[]" - ) - - -# REQ-PACK-005: headless sessions exclude default-disabled packs -def test_headless_session_excludes_default_disabled_pack_skills(tmp_path: Path) -> None: - """Skills in 'exp-lens' pack are excluded from headless session when packs.enabled=[].""" - from unittest.mock import MagicMock - - from autoskillit.config.settings import AutomationConfig - from autoskillit.workspace.skills import SkillInfo, SkillSource - - skill_dir = tmp_path / "skills" / "exp-skill" - skill_dir.mkdir(parents=True) - (skill_dir / "SKILL.md").write_text( - "---\ncategories:\n - exp-lens\n---\n# Experimental Skill\n" - ) - - provider = MagicMock() - provider.list_skills.return_value = [ - SkillInfo( - name="exp-skill", - source=SkillSource.BUNDLED_EXTENDED, - path=skill_dir / "SKILL.md", - categories=frozenset({"exp-lens"}), - ) - ] - - root = tmp_path / "sessions" - root.mkdir() - config = AutomationConfig() # packs.enabled=[] by default - mgr = DefaultSessionSkillManager(provider, ephemeral_root=root) - session_path = mgr.init_session("headless-exp", cook_session=False, config=config) - - assert not (session_path / ".claude" / "skills" / "exp-skill").exists(), ( - "Default-disabled pack skill must not be in headless session when packs.enabled=[]" - ) - - -def test_init_session_recipe_packs_enables_default_disabled(tmp_path: Path) -> None: - """recipe_packs param enables default-disabled pack skills for this session.""" - from unittest.mock import MagicMock - - from autoskillit.config.settings import AutomationConfig - from autoskillit.workspace.skills import SkillInfo, SkillSource - - skill_dir = tmp_path / "skills" / "research-skill" - skill_dir.mkdir(parents=True) - (skill_dir / "SKILL.md").write_text("---\ncategories:\n - research\n---\n# Research Skill\n") - - provider = MagicMock() - provider.list_skills.return_value = [ - SkillInfo( - name="research-skill", - source=SkillSource.BUNDLED_EXTENDED, - path=skill_dir / "SKILL.md", - categories=frozenset({"research"}), - ) - ] - provider.get_skill_content.return_value = ( - "---\ncategories:\n - research\n---\n# Research Skill\n" - ) - - root = tmp_path / "sessions" - root.mkdir() - config = AutomationConfig() # packs.enabled=[] by default - mgr = DefaultSessionSkillManager(provider, ephemeral_root=root) - session_path = mgr.init_session( - "headless-recipe-research", - cook_session=False, - config=config, - recipe_packs=frozenset(["research"]), - ) - - assert (session_path / ".claude" / "skills" / "research-skill").exists(), ( - "Default-disabled pack skill should be present when enabled via recipe_packs" - ) - - -# ── Tests: _parse_activate_deps ───────────────────────────────────────────── - - -class TestParseActivateDeps: - def test_parses_single_dep(self) -> None: - from autoskillit.workspace.session_skills import _parse_activate_deps - - content = "---\nname: foo\nactivate_deps: [arch-lens]\n---\nBody" - assert _parse_activate_deps(content) == ["arch-lens"] - - def test_parses_multiple_deps(self) -> None: - from autoskillit.workspace.session_skills import _parse_activate_deps - - content = "---\nname: foo\nactivate_deps: [arch-lens, mermaid]\n---\nBody" - assert _parse_activate_deps(content) == ["arch-lens", "mermaid"] - - def test_empty_deps(self) -> None: - from autoskillit.workspace.session_skills import _parse_activate_deps - - content = "---\nname: foo\nactivate_deps: []\n---\nBody" - assert _parse_activate_deps(content) == [] - - def test_no_activate_deps_field(self) -> None: - from autoskillit.workspace.session_skills import _parse_activate_deps - - content = "---\nname: foo\n---\nBody" - assert _parse_activate_deps(content) == [] - - def test_no_frontmatter(self) -> None: - from autoskillit.workspace.session_skills import _parse_activate_deps - - content = "Just body text" - assert _parse_activate_deps(content) == [] - - -# ── Tests: activate_skill_deps transitive dependency resolution ────────────────── - - -def _write_skill_md(base: Path, session_id: str, skill_name: str, content: str) -> Path: - """Helper to write a SKILL.md in the ephemeral session layout.""" - skill_dir = base / session_id / ".claude" / "skills" / skill_name - skill_dir.mkdir(parents=True, exist_ok=True) - md = skill_dir / "SKILL.md" - md.write_text(content) - return md - - -def _is_gated(base: Path, session_id: str, skill_name: str) -> bool: - """Return True if the skill has disable-model-invocation: true.""" - md = base / session_id / ".claude" / "skills" / skill_name / "SKILL.md" - content = md.read_text() - return "disable-model-invocation: true" in content - - -class TestActivateDepsResolution: - def test_activate_skill_deps_resolves_pack_deps(self, tmp_path: Path) -> None: - """Activating a skill with activate_deps: [arch-lens] ungates all arch-lens skills.""" - from unittest.mock import MagicMock - - from autoskillit.core.types import SkillSource - from autoskillit.workspace.skills import SkillInfo - - session_id = "test-pack-deps" - gate = "disable-model-invocation: true" - # Parent skill with pack dep - _write_skill_md( - tmp_path, - session_id, - "make-plan", - f"---\nname: make-plan\nactivate_deps: [arch-lens]\n{gate}\n---\n# Plan", - ) - # Three arch-lens skills - for name in ["arch-lens-a", "arch-lens-b", "arch-lens-c"]: - _write_skill_md( - tmp_path, - session_id, - name, - f"---\nname: {name}\ncategories: [arch-lens]\n{gate}\n---\n# Lens", - ) - - provider = MagicMock() - resolver = MagicMock() - provider.resolver = resolver - - def resolve_fn(name: str) -> SkillInfo | None: - if name.startswith("arch-lens-"): - return SkillInfo( - name=name, - source=SkillSource.BUNDLED_EXTENDED, - path=tmp_path / session_id / ".claude" / "skills" / name / "SKILL.md", - categories=frozenset({"arch-lens"}), - ) - return SkillInfo( - name=name, - source=SkillSource.BUNDLED_EXTENDED, - path=tmp_path / session_id / ".claude" / "skills" / name / "SKILL.md", - categories=frozenset(), - ) - - resolver.resolve.side_effect = resolve_fn - - mgr = DefaultSessionSkillManager(provider, ephemeral_root=tmp_path) - result = mgr.activate_skill_deps(session_id, "make-plan") - assert result is True - assert not _is_gated(tmp_path, session_id, "make-plan") - for name in ["arch-lens-a", "arch-lens-b", "arch-lens-c"]: - assert not _is_gated(tmp_path, session_id, name), f"{name} should be ungated" - - def test_activate_skill_deps_resolves_individual_skill_dep(self, tmp_path: Path) -> None: - """Activating a skill with activate_deps: [mermaid] ungates mermaid specifically.""" - from unittest.mock import MagicMock - - session_id = "test-individual-dep" - gate = "disable-model-invocation: true" - _write_skill_md( - tmp_path, - session_id, - "parent-skill", - f"---\nname: parent-skill\nactivate_deps: [mermaid]\n{gate}\n---\n# Parent", - ) - _write_skill_md( - tmp_path, - session_id, - "mermaid", - "---\nname: mermaid\ndisable-model-invocation: true\n---\n# Mermaid", - ) - - provider = MagicMock() - provider.resolver.resolve.return_value = None - - mgr = DefaultSessionSkillManager(provider, ephemeral_root=tmp_path) - result = mgr.activate_skill_deps(session_id, "parent-skill") - assert result is True - assert not _is_gated(tmp_path, session_id, "mermaid") - - def test_activate_skill_deps_resolves_two_level_transitive(self, tmp_path: Path) -> None: - """make-plan -> arch-lens-* -> mermaid: all three levels get ungated.""" - from unittest.mock import MagicMock - - from autoskillit.core.types import SkillSource - from autoskillit.workspace.skills import SkillInfo - - session_id = "test-two-level" - gate = "disable-model-invocation: true" - _write_skill_md( - tmp_path, - session_id, - "make-plan", - f"---\nname: make-plan\nactivate_deps: [arch-lens]\n{gate}\n---\n# Plan", - ) - _write_skill_md( - tmp_path, - session_id, - "arch-lens-x", - ( - f"---\nname: arch-lens-x\ncategories: [arch-lens]\n" - f"activate_deps: [mermaid]\n{gate}\n---\n# Lens" - ), - ) - _write_skill_md( - tmp_path, - session_id, - "mermaid", - "---\nname: mermaid\ndisable-model-invocation: true\n---\n# Mermaid", - ) - - provider = MagicMock() - resolver = MagicMock() - provider.resolver = resolver - - def resolve_fn(name: str) -> SkillInfo | None: - cats = frozenset({"arch-lens"}) if name.startswith("arch-lens-") else frozenset() - return SkillInfo( - name=name, - source=SkillSource.BUNDLED_EXTENDED, - path=tmp_path / session_id / ".claude" / "skills" / name / "SKILL.md", - categories=cats, - ) - - resolver.resolve.side_effect = resolve_fn - - mgr = DefaultSessionSkillManager(provider, ephemeral_root=tmp_path) - mgr.activate_skill_deps(session_id, "make-plan") - assert not _is_gated(tmp_path, session_id, "make-plan") - assert not _is_gated(tmp_path, session_id, "arch-lens-x") - assert not _is_gated(tmp_path, session_id, "mermaid") - - def test_activate_skill_deps_handles_circular_deps(self, tmp_path: Path) -> None: - """Circular activate_deps do not cause infinite recursion.""" - from unittest.mock import MagicMock - - session_id = "test-circular" - gate = "disable-model-invocation: true" - _write_skill_md( - tmp_path, - session_id, - "skill-a", - f"---\nname: skill-a\nactivate_deps: [skill-b]\n{gate}\n---\n# A", - ) - _write_skill_md( - tmp_path, - session_id, - "skill-b", - f"---\nname: skill-b\nactivate_deps: [skill-a]\n{gate}\n---\n# B", - ) - - provider = MagicMock() - provider.resolver.resolve.return_value = None - - mgr = DefaultSessionSkillManager(provider, ephemeral_root=tmp_path) - result = mgr.activate_skill_deps(session_id, "skill-a") - assert result is True - assert not _is_gated(tmp_path, session_id, "skill-a") - assert not _is_gated(tmp_path, session_id, "skill-b") - - def test_tier3_target_activates_tier2_deps(self, tmp_path: Path) -> None: - """A tier3 (ungated) skill's activate_deps still triggers tier2 dependency ungating.""" - from unittest.mock import MagicMock - - session_id = "test-tier3-deps" - # tier3 parent (not gated) - _write_skill_md( - tmp_path, - session_id, - "open-pr", - "---\nname: open-pr\nactivate_deps: [mermaid]\n---\n# Open PR", - ) - # tier2 dep (gated) - _write_skill_md( - tmp_path, - session_id, - "mermaid", - "---\nname: mermaid\ndisable-model-invocation: true\n---\n# Mermaid", - ) - - provider = MagicMock() - provider.resolver.resolve.return_value = None - - mgr = DefaultSessionSkillManager(provider, ephemeral_root=tmp_path) - result = mgr.activate_skill_deps(session_id, "open-pr") - assert result is True - assert not _is_gated(tmp_path, session_id, "mermaid") - - def test_pack_dep_absent_skills_noop(self, tmp_path: Path) -> None: - """Pack dep referencing skills not in ephemeral dir does not error.""" - from unittest.mock import MagicMock - - session_id = "test-absent-pack" - gate = "disable-model-invocation: true" - _write_skill_md( - tmp_path, - session_id, - "parent-skill", - f"---\nname: parent-skill\nactivate_deps: [exp-lens]\n{gate}\n---\n# Parent", - ) - - provider = MagicMock() - provider.resolver.resolve.return_value = None - - mgr = DefaultSessionSkillManager(provider, ephemeral_root=tmp_path) - result = mgr.activate_skill_deps(session_id, "parent-skill") - assert result is True - assert not _is_gated(tmp_path, session_id, "parent-skill") - - def test_activate_deps_strips_marker_from_dependency_body(self, tmp_path: Path) -> None: - """Dependency SKILL.md bodies must have %%ORDER_UP%% stripped after activation.""" - from unittest.mock import MagicMock - - session_id = "test-strip-marker" - gate = "disable-model-invocation: true" - marker_body = ( - "# Sub-Skill\n\nDo the work.\n\n" - "ORCHESTRATION DIRECTIVE: When your task is complete, " - "your final text output MUST end with: %%ORDER_UP%%\n" - "CRITICAL: Append %%ORDER_UP%% at the very end of your substantive response, " - "in the SAME message. Do NOT output %%ORDER_UP%% as a separate standalone message." - ) - _write_skill_md( - tmp_path, - session_id, - "parent-skill", - f"---\nname: parent-skill\nactivate_deps: [dep-skill]\n{gate}\n---\n# Parent", - ) - _write_skill_md( - tmp_path, - session_id, - "dep-skill", - f"---\nname: dep-skill\n{gate}\n---\n{marker_body}", - ) - - provider = MagicMock() - provider.resolver.resolve.return_value = None - - mgr = DefaultSessionSkillManager(provider, ephemeral_root=tmp_path) - mgr.activate_skill_deps(session_id, "parent-skill") - - dep_md = tmp_path / session_id / ".claude" / "skills" / "dep-skill" / "SKILL.md" - dep_content = dep_md.read_text() - assert "%%ORDER_UP%%" not in dep_content - assert "# Sub-Skill" in dep_content - assert "Do the work." in dep_content - - def test_activate_deps_preserves_marker_in_root_skill(self, tmp_path: Path) -> None: - """The root (directly targeted) skill keeps its %%ORDER_UP%% body intact.""" - from unittest.mock import MagicMock - - session_id = "test-preserve-root-marker" - gate = "disable-model-invocation: true" - _write_skill_md( - tmp_path, - session_id, - "root-skill", - f"---\nname: root-skill\n{gate}\n---\n# Root\n\n%%ORDER_UP%%", - ) - - provider = MagicMock() - provider.resolver.resolve.return_value = None - - mgr = DefaultSessionSkillManager(provider, ephemeral_root=tmp_path) - mgr.activate_skill_deps(session_id, "root-skill") - - root_md = tmp_path / session_id / ".claude" / "skills" / "root-skill" / "SKILL.md" - root_content = root_md.read_text() - assert "%%ORDER_UP%%" in root_content - - -def _make_synthetic_provider( - tmp_path: Path, - skills: dict[str, dict], -): - """Build a mocked SkillsDirectoryProvider serving synthetic SKILL.md files. - - skills: mapping of name -> {"deps": [...], "categories": [...]} - """ - from unittest.mock import MagicMock - - from autoskillit.workspace.skills import SkillInfo, SkillSource - - tmp_path.mkdir(parents=True, exist_ok=True) - skill_infos: list[SkillInfo] = [] - for name, spec in skills.items(): - skill_dir = tmp_path / name - skill_dir.mkdir() - deps = spec.get("deps", []) - categories = spec.get("categories", []) - fm_lines = [f"name: {name}"] - if categories: - fm_lines.append(f"categories: [{', '.join(categories)}]") - if deps: - fm_lines.append(f"activate_deps: [{', '.join(deps)}]") - content = "---\n" + "\n".join(fm_lines) + "\n---\nbody\n" - (skill_dir / "SKILL.md").write_text(content) - skill_infos.append( - SkillInfo( - name=name, - source=SkillSource.BUNDLED_EXTENDED, - path=skill_dir / "SKILL.md", - categories=frozenset(categories), - ) - ) - - by_name = {info.name: info for info in skill_infos} - - provider = MagicMock() - provider.list_skills.return_value = skill_infos - provider.resolver = MagicMock() - provider.resolver.resolve.side_effect = lambda n: by_name.get(n) - - def _get_content(name: str, *, gated: bool = True) -> str: - if name not in by_name: - raise FileNotFoundError(name) - return by_name[name].path.read_text() - - provider.get_skill_content.side_effect = _get_content - return provider - - -class TestInitSessionAllowOnly: - """Tests for the ``allow_only`` filter on ``DefaultSessionSkillManager.init_session``.""" - - def test_allow_only_writes_only_named_skill(self, tmp_path: Path) -> None: - provider = _make_synthetic_provider( - tmp_path / "skills", - {"alpha": {}, "beta": {}, "gamma": {}}, - ) - root = tmp_path / "sessions" - root.mkdir() - mgr = DefaultSessionSkillManager(provider, ephemeral_root=root) - session_path = mgr.init_session("s1", allow_only=frozenset({"alpha"})) - - skills_base = session_path / ".claude" / "skills" - assert (skills_base / "alpha" / "SKILL.md").exists() - assert not (skills_base / "beta").exists() - assert not (skills_base / "gamma").exists() - - def test_allow_only_empty_writes_no_skills(self, tmp_path: Path) -> None: - provider = _make_synthetic_provider( - tmp_path / "skills", - {"alpha": {}, "beta": {}}, - ) - root = tmp_path / "sessions" - root.mkdir() - mgr = DefaultSessionSkillManager(provider, ephemeral_root=root) - session_path = mgr.init_session("s2", allow_only=frozenset()) - - skills_base = session_path / ".claude" / "skills" - assert skills_base.exists() - assert list(skills_base.glob("*/SKILL.md")) == [] - - def test_allow_only_none_preserves_full_injection(self, tmp_path: Path) -> None: - provider = _make_synthetic_provider( - tmp_path / "skills", - {"alpha": {}, "beta": {}, "gamma": {}}, - ) - root = tmp_path / "sessions" - root.mkdir() - mgr = DefaultSessionSkillManager(provider, ephemeral_root=root) - session_path = mgr.init_session("s3", allow_only=None) - - skills_base = session_path / ".claude" / "skills" - names = {p.parent.name for p in skills_base.glob("*/SKILL.md")} - assert names == {"alpha", "beta", "gamma"} - - def test_allow_only_intersects_effective_disabled_disabled_wins(self, tmp_path: Path) -> None: - from autoskillit.config.settings import AutomationConfig, SubsetsConfig - - provider = _make_synthetic_provider( - tmp_path / "skills", - {"alpha": {"categories": ["github"]}, "beta": {}}, - ) - root = tmp_path / "sessions" - root.mkdir() - mgr = DefaultSessionSkillManager(provider, ephemeral_root=root) - config = AutomationConfig(subsets=SubsetsConfig(disabled=["github"])) - session_path = mgr.init_session( - "s4", config=config, allow_only=frozenset({"alpha", "beta"}) - ) - - skills_base = session_path / ".claude" / "skills" - assert not (skills_base / "alpha").exists() # disabled wins - assert (skills_base / "beta" / "SKILL.md").exists() - - def test_allow_only_skips_project_local_overrides(self, tmp_path: Path) -> None: - provider = _make_synthetic_provider( - tmp_path / "skills", - {"alpha": {}, "beta": {}}, - ) - # Simulate a project-local override of "alpha" - project_dir = tmp_path / "project" - local_skill = project_dir / ".claude" / "skills" / "alpha" - local_skill.mkdir(parents=True) - (local_skill / "SKILL.md").write_text("---\nname: alpha\n---\nlocal\n") - - root = tmp_path / "sessions" - root.mkdir() - mgr = DefaultSessionSkillManager(provider, ephemeral_root=root) - session_path = mgr.init_session( - "s5", - project_dir=project_dir, - allow_only=frozenset({"alpha", "beta"}), - ) - - skills_base = session_path / ".claude" / "skills" - # alpha is suppressed by channel-dedup (project-local override exists) - assert not (skills_base / "alpha").exists() - assert (skills_base / "beta" / "SKILL.md").exists() - - def test_allow_only_logs_debug_skip(self, tmp_path: Path) -> None: - import structlog - - provider = _make_synthetic_provider( - tmp_path / "skills", - {"alpha": {}, "beta": {}, "gamma": {}}, - ) - root = tmp_path / "sessions" - root.mkdir() - mgr = DefaultSessionSkillManager(provider, ephemeral_root=root) - - with structlog.testing.capture_logs() as cap_logs: - mgr.init_session("s6", allow_only=frozenset({"alpha"})) - - skipped = { - entry["skill"] - for entry in cap_logs - if entry.get("event") == "init_session_allow_only_skip" - } - assert skipped == {"beta", "gamma"} - - -class TestComputeSkillClosure: - """Tests for ``compute_skill_closure`` and its top-level helper.""" - - def test_closure_standalone_returns_only_self(self, tmp_path: Path) -> None: - from autoskillit.workspace.session_skills import compute_skill_closure - - provider = _make_synthetic_provider(tmp_path, {"lone": {}}) - assert compute_skill_closure("lone", provider) == frozenset({"lone"}) - - def test_closure_pack_dep_expands_to_pack_members(self) -> None: - from autoskillit.workspace.session_skills import compute_skill_closure - - provider = SkillsDirectoryProvider() - closure = compute_skill_closure("make-plan", provider) - assert "make-plan" in closure - assert "mermaid" in closure # transitive via arch-lens-* deps - arch_members = {n for n in closure if n.startswith("arch-lens-")} - assert len(arch_members) >= 1 - - def test_closure_individual_skill_dep(self, tmp_path: Path) -> None: - from autoskillit.workspace.session_skills import compute_skill_closure - - provider = _make_synthetic_provider( - tmp_path, - {"target": {"deps": ["other"]}, "other": {}}, - ) - assert compute_skill_closure("target", provider) == frozenset({"target", "other"}) - - def test_closure_two_level_transitive(self, tmp_path: Path) -> None: - from autoskillit.workspace.session_skills import compute_skill_closure - - provider = _make_synthetic_provider( - tmp_path, - {"a": {"deps": ["b"]}, "b": {"deps": ["c"]}, "c": {}}, - ) - assert compute_skill_closure("a", provider) == frozenset({"a", "b", "c"}) - - def test_closure_cycle_safe(self, tmp_path: Path) -> None: - from autoskillit.workspace.session_skills import compute_skill_closure - - provider = _make_synthetic_provider( - tmp_path, - {"a": {"deps": ["b"]}, "b": {"deps": ["a"]}}, - ) - assert compute_skill_closure("a", provider) == frozenset({"a", "b"}) - - def test_closure_unknown_dep_silently_ignored(self, tmp_path: Path) -> None: - from autoskillit.workspace.session_skills import compute_skill_closure - - provider = _make_synthetic_provider( - tmp_path, - {"target": {"deps": ["ghost"]}}, - ) - assert compute_skill_closure("target", provider) == frozenset({"target"}) - - def test_closure_unknown_target_returns_empty_frozenset(self, tmp_path: Path) -> None: - from autoskillit.workspace.session_skills import compute_skill_closure - - provider = _make_synthetic_provider(tmp_path, {"alpha": {}}) - assert compute_skill_closure("nonexistent", provider) == frozenset() - - def test_closure_pack_dep_with_no_members_returns_only_target(self, tmp_path: Path) -> None: - from autoskillit.workspace.session_skills import compute_skill_closure - - # 'audit' is a real PACK_REGISTRY key, but no synthetic skills declare it. - provider = _make_synthetic_provider( - tmp_path, - {"target": {"deps": ["audit"]}}, - ) - assert compute_skill_closure("target", provider) == frozenset({"target"}) diff --git a/tests/workspace/test_session_skills_allow_only_and_closure.py b/tests/workspace/test_session_skills_allow_only_and_closure.py new file mode 100644 index 0000000000..0938d319fd --- /dev/null +++ b/tests/workspace/test_session_skills_allow_only_and_closure.py @@ -0,0 +1,462 @@ +"""Phase 2 tests: session_skills module — allow_only filter and compute_skill_closure.""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from autoskillit.core import FEATURE_REGISTRY +from autoskillit.workspace.session_skills import ( + DefaultSessionSkillManager, + SkillsDirectoryProvider, +) + +pytestmark = [pytest.mark.layer("workspace"), pytest.mark.small] + +_FEATURE_SKILL_CATEGORY_PARAMS = [ + pytest.param(feat_name, cat, id=f"{feat_name}-{cat}") + for feat_name, feat_def in FEATURE_REGISTRY.items() + for cat in feat_def.skill_categories +] +assert len(_FEATURE_SKILL_CATEGORY_PARAMS) > 0, ( + "FEATURE_REGISTRY has no entries with skill_categories — " + "test_allow_only_overrides_all_feature_registry_entries would be silently skipped" +) + + +def _make_synthetic_provider( + tmp_path: Path, + skills: dict[str, dict], +): + """Build a mocked SkillsDirectoryProvider serving synthetic SKILL.md files. + + skills: mapping of name -> {"deps": [...], "categories": [...]} + """ + from unittest.mock import MagicMock + + from autoskillit.workspace.skills import SkillInfo, SkillSource + + tmp_path.mkdir(parents=True, exist_ok=True) + skill_infos: list[SkillInfo] = [] + for name, spec in skills.items(): + skill_dir = tmp_path / name + skill_dir.mkdir() + deps = spec.get("deps", []) + categories = spec.get("categories", []) + fm_lines = [f"name: {name}"] + if categories: + fm_lines.append(f"categories: [{', '.join(categories)}]") + if deps: + fm_lines.append(f"activate_deps: [{', '.join(deps)}]") + content = "---\n" + "\n".join(fm_lines) + "\n---\nbody\n" + (skill_dir / "SKILL.md").write_text(content) + skill_infos.append( + SkillInfo( + name=name, + source=SkillSource.BUNDLED_EXTENDED, + path=skill_dir / "SKILL.md", + categories=frozenset(categories), + ) + ) + + by_name = {info.name: info for info in skill_infos} + + provider = MagicMock() + provider.list_skills.return_value = skill_infos + provider.resolver = MagicMock() + provider.resolver.resolve.side_effect = lambda n: by_name.get(n) + + def _get_content(name: str, *, gated: bool = True) -> str: + if name not in by_name: + raise FileNotFoundError(name) + return by_name[name].path.read_text() + + provider.get_skill_content.side_effect = _get_content + return provider + + +class TestInitSessionAllowOnly: + """Tests for the ``allow_only`` filter on ``DefaultSessionSkillManager.init_session``.""" + + def test_allow_only_writes_only_named_skill(self, tmp_path: Path) -> None: + provider = _make_synthetic_provider( + tmp_path / "skills", + {"alpha": {}, "beta": {}, "gamma": {}}, + ) + root = tmp_path / "sessions" + root.mkdir() + mgr = DefaultSessionSkillManager(provider, ephemeral_root=root) + session_path = mgr.init_session("s1", allow_only=frozenset({"alpha"})) + + skills_base = session_path / ".claude" / "skills" + assert (skills_base / "alpha" / "SKILL.md").exists() + assert not (skills_base / "beta").exists() + assert not (skills_base / "gamma").exists() + + def test_allow_only_empty_writes_no_skills(self, tmp_path: Path) -> None: + provider = _make_synthetic_provider( + tmp_path / "skills", + {"alpha": {}, "beta": {}}, + ) + root = tmp_path / "sessions" + root.mkdir() + mgr = DefaultSessionSkillManager(provider, ephemeral_root=root) + session_path = mgr.init_session("s2", allow_only=frozenset()) + + skills_base = session_path / ".claude" / "skills" + assert skills_base.exists() + assert list(skills_base.glob("*/SKILL.md")) == [] + + def test_allow_only_none_preserves_full_injection(self, tmp_path: Path) -> None: + provider = _make_synthetic_provider( + tmp_path / "skills", + {"alpha": {}, "beta": {}, "gamma": {}}, + ) + root = tmp_path / "sessions" + root.mkdir() + mgr = DefaultSessionSkillManager(provider, ephemeral_root=root) + session_path = mgr.init_session("s3", allow_only=None) + + skills_base = session_path / ".claude" / "skills" + names = {p.parent.name for p in skills_base.glob("*/SKILL.md")} + assert names == {"alpha", "beta", "gamma"} + + def test_allow_only_does_not_override_explicit_subsets_disabled(self, tmp_path: Path) -> None: + from tests._helpers import make_subsetsconfig, make_test_config + + provider = _make_synthetic_provider( + tmp_path / "skills", + {"alpha": {"categories": ["github"]}, "beta": {}}, + ) + root = tmp_path / "sessions" + root.mkdir() + mgr = DefaultSessionSkillManager(provider, ephemeral_root=root) + config = make_test_config(subsets=make_subsetsconfig(disabled=["github"])) + session_path = mgr.init_session( + "s4", config=config, allow_only=frozenset({"alpha", "beta"}) + ) + + skills_base = session_path / ".claude" / "skills" + assert not (skills_base / "alpha").exists() # disabled wins + assert (skills_base / "beta" / "SKILL.md").exists() + + def test_allow_only_skips_project_local_overrides(self, tmp_path: Path) -> None: + provider = _make_synthetic_provider( + tmp_path / "skills", + {"alpha": {}, "beta": {}}, + ) + # Simulate a project-local override of "alpha" + project_dir = tmp_path / "project" + local_skill = project_dir / ".claude" / "skills" / "alpha" + local_skill.mkdir(parents=True) + (local_skill / "SKILL.md").write_text("---\nname: alpha\n---\nlocal\n") + + root = tmp_path / "sessions" + root.mkdir() + mgr = DefaultSessionSkillManager(provider, ephemeral_root=root) + session_path = mgr.init_session( + "s5", + project_dir=project_dir, + allow_only=frozenset({"alpha", "beta"}), + ) + + skills_base = session_path / ".claude" / "skills" + # alpha is suppressed by channel-dedup (project-local override exists) + assert not (skills_base / "alpha").exists() + assert (skills_base / "beta" / "SKILL.md").exists() + + def test_allow_only_logs_debug_skip(self, tmp_path: Path) -> None: + import structlog + + provider = _make_synthetic_provider( + tmp_path / "skills", + {"alpha": {}, "beta": {}, "gamma": {}}, + ) + root = tmp_path / "sessions" + root.mkdir() + mgr = DefaultSessionSkillManager(provider, ephemeral_root=root) + + with structlog.testing.capture_logs() as cap_logs: + mgr.init_session("s6", allow_only=frozenset({"alpha"})) + + skipped = { + entry["skill"] + for entry in cap_logs + if entry.get("event") == "init_session_allow_only_skip" + } + assert skipped == {"beta", "gamma"} + + def test_allow_only_overrides_feature_gate_for_explicitly_requested_skills( + self, tmp_path: Path + ) -> None: + from tests._helpers import make_test_config + + provider = _make_synthetic_provider( + tmp_path / "skills", + {"planner-skill": {"categories": ["planner"]}}, + ) + root = tmp_path / "sessions" + root.mkdir() + mgr = DefaultSessionSkillManager(provider, ephemeral_root=root) + config = make_test_config(features={}, experimental_enabled=False) + session_path = mgr.init_session( + "s-feat-bypass", + allow_only=frozenset({"planner-skill"}), + config=config, + ) + + skills_base = session_path / ".claude" / "skills" + assert (skills_base / "planner-skill" / "SKILL.md").exists(), ( + "planner-skill must be written when allow_only explicitly requests it, " + "even though the planner feature gate is off (experimental_enabled=False)" + ) + + def test_allow_only_nonempty_but_zero_skills_raises(self, tmp_path: Path) -> None: + provider = _make_synthetic_provider( + tmp_path / "skills", + {"alpha": {}, "beta": {}}, + ) + root = tmp_path / "sessions" + root.mkdir() + mgr = DefaultSessionSkillManager(provider, ephemeral_root=root) + with pytest.raises(RuntimeError, match="allow_only") as exc_info: + mgr.init_session("s-zero", allow_only=frozenset({"ghost-skill"})) + assert "zero skills" in str(exc_info.value) + + @pytest.mark.parametrize("feature_name,category", _FEATURE_SKILL_CATEGORY_PARAMS) + def test_allow_only_overrides_all_feature_registry_entries( + self, tmp_path: Path, feature_name: str, category: str + ) -> None: + from tests._helpers import make_test_config + + provider = _make_synthetic_provider( + tmp_path / "skills", + {"test-skill": {"categories": [category]}}, + ) + root = tmp_path / "sessions" + root.mkdir() + mgr = DefaultSessionSkillManager(provider, ephemeral_root=root) + config = make_test_config(features={feature_name: False}, experimental_enabled=False) + session_path = mgr.init_session( + "s-feat-registry", + allow_only=frozenset({"test-skill"}), + config=config, + ) + + skills_base = session_path / ".claude" / "skills" + assert (skills_base / "test-skill" / "SKILL.md").exists(), ( + f"test-skill (category={category!r}) must be written when allow_only explicitly " + f"requests it, even though feature {feature_name!r} is disabled" + ) + + +_CROSS_AXIS_PARAMS = [ + # (allow_only, feature_enabled, subsets_disabled, cook_session, expected_target_written) + # allow_only=None cases + pytest.param(None, True, [], False, True, id="ao-none_feat-on_disabled-no_cook-no"), + pytest.param(None, True, ["planner"], False, False, id="ao-none_feat-on_disabled-yes_cook-no"), + pytest.param(None, False, [], False, False, id="ao-none_feat-off_disabled-no_cook-no"), + pytest.param( + None, False, ["planner"], False, False, id="ao-none_feat-off_disabled-yes_cook-no" + ), + pytest.param(None, True, [], True, True, id="ao-none_feat-on_disabled-no_cook-yes"), + pytest.param(None, True, ["planner"], True, True, id="ao-none_feat-on_disabled-yes_cook-yes"), + pytest.param(None, False, [], True, True, id="ao-none_feat-off_disabled-no_cook-yes"), + pytest.param( + None, False, ["planner"], True, True, id="ao-none_feat-off_disabled-yes_cook-yes" + ), + # allow_only=frozenset({"target","sibling"}) cases + pytest.param( + frozenset({"target", "sibling"}), + True, + [], + False, + True, + id="ao-set_feat-on_disabled-no_cook-no", + ), + pytest.param( + frozenset({"target", "sibling"}), + True, + ["planner"], + False, + False, + id="ao-set_feat-on_disabled-yes_cook-no", + ), + pytest.param( + frozenset({"target", "sibling"}), + False, + [], + False, + True, + id="ao-set_feat-off_disabled-no_cook-no", # THE BUG CASE + ), + pytest.param( + frozenset({"target", "sibling"}), + False, + ["planner"], + False, + False, + id="ao-set_feat-off_disabled-yes_cook-no", + ), + pytest.param( + frozenset({"target", "sibling"}), + True, + [], + True, + True, + id="ao-set_feat-on_disabled-no_cook-yes", + ), + pytest.param( + frozenset({"target", "sibling"}), + True, + ["planner"], + True, + True, + id="ao-set_feat-on_disabled-yes_cook-yes", + ), + pytest.param( + frozenset({"target", "sibling"}), + False, + [], + True, + True, + id="ao-set_feat-off_disabled-no_cook-yes", + ), + pytest.param( + frozenset({"target", "sibling"}), + False, + ["planner"], + True, + True, + id="ao-set_feat-off_disabled-yes_cook-yes", + ), +] + + +class TestCrossAxisGatingMatrix: + """Cross-axis parametrized test matrix: allow_only × feature_gate × subsets.disabled × cook.""" + + @pytest.mark.parametrize( + "allow_only,feature_enabled,subsets_disabled,cook_session,expected_target_written", + _CROSS_AXIS_PARAMS, + ) + def test_cross_axis_gating_matrix( + self, + tmp_path: Path, + allow_only: frozenset[str] | None, + feature_enabled: bool, + subsets_disabled: list[str], + cook_session: bool, + expected_target_written: bool, + ) -> None: + from tests._helpers import make_subsetsconfig, make_test_config + + provider = _make_synthetic_provider( + tmp_path / "skills", + {"target": {"categories": ["planner"]}, "sibling": {}}, + ) + root = tmp_path / "sessions" + root.mkdir() + mgr = DefaultSessionSkillManager(provider, ephemeral_root=root) + + features = {"planner": True} if feature_enabled else {} + config = make_test_config( + features=features, + experimental_enabled=False, + subsets=make_subsetsconfig(disabled=subsets_disabled), + ) + + session_path = mgr.init_session( + "s-matrix", + config=config, + allow_only=allow_only, + cook_session=cook_session, + ) + + skills_base = session_path / ".claude" / "skills" + target_written = (skills_base / "target").exists() + assert target_written == expected_target_written, ( + f"target (category=planner) written={target_written!r}, " + f"expected={expected_target_written!r} for " + f"allow_only={allow_only!r}, feature_enabled={feature_enabled!r}, " + f"subsets_disabled={subsets_disabled!r}, cook_session={cook_session!r}" + ) + sibling_written = (skills_base / "sibling").exists() + assert sibling_written, ( + f"sibling (no categories) must always be written — " + f"allow_only={allow_only!r}, feature_enabled={feature_enabled!r}, " + f"subsets_disabled={subsets_disabled!r}, cook_session={cook_session!r}" + ) + + +class TestComputeSkillClosure: + """Tests for ``compute_skill_closure`` and its top-level helper.""" + + def test_closure_standalone_returns_only_self(self, tmp_path: Path) -> None: + from autoskillit.workspace.session_skills import compute_skill_closure + + provider = _make_synthetic_provider(tmp_path, {"lone": {}}) + assert compute_skill_closure("lone", provider) == frozenset({"lone"}) + + def test_closure_pack_dep_expands_to_pack_members(self) -> None: + from autoskillit.workspace.session_skills import compute_skill_closure + + provider = SkillsDirectoryProvider() + closure = compute_skill_closure("make-plan", provider) + assert "make-plan" in closure + assert "mermaid" in closure # transitive via arch-lens-* deps + arch_members = {n for n in closure if n.startswith("arch-lens-")} + assert len(arch_members) >= 1 + + def test_closure_individual_skill_dep(self, tmp_path: Path) -> None: + from autoskillit.workspace.session_skills import compute_skill_closure + + provider = _make_synthetic_provider( + tmp_path, + {"target": {"deps": ["other"]}, "other": {}}, + ) + assert compute_skill_closure("target", provider) == frozenset({"target", "other"}) + + def test_closure_two_level_transitive(self, tmp_path: Path) -> None: + from autoskillit.workspace.session_skills import compute_skill_closure + + provider = _make_synthetic_provider( + tmp_path, + {"a": {"deps": ["b"]}, "b": {"deps": ["c"]}, "c": {}}, + ) + assert compute_skill_closure("a", provider) == frozenset({"a", "b", "c"}) + + def test_closure_cycle_safe(self, tmp_path: Path) -> None: + from autoskillit.workspace.session_skills import compute_skill_closure + + provider = _make_synthetic_provider( + tmp_path, + {"a": {"deps": ["b"]}, "b": {"deps": ["a"]}}, + ) + assert compute_skill_closure("a", provider) == frozenset({"a", "b"}) + + def test_closure_unknown_dep_silently_ignored(self, tmp_path: Path) -> None: + from autoskillit.workspace.session_skills import compute_skill_closure + + provider = _make_synthetic_provider( + tmp_path, + {"target": {"deps": ["ghost"]}}, + ) + assert compute_skill_closure("target", provider) == frozenset({"target"}) + + def test_closure_unknown_target_returns_empty_frozenset(self, tmp_path: Path) -> None: + from autoskillit.workspace.session_skills import compute_skill_closure + + provider = _make_synthetic_provider(tmp_path, {"alpha": {}}) + assert compute_skill_closure("nonexistent", provider) == frozenset() + + def test_closure_pack_dep_with_no_members_returns_only_target(self, tmp_path: Path) -> None: + from autoskillit.workspace.session_skills import compute_skill_closure + + # 'audit' is a real PACK_REGISTRY key, but no synthetic skills declare it. + provider = _make_synthetic_provider( + tmp_path, + {"target": {"deps": ["audit"]}}, + ) + assert compute_skill_closure("target", provider) == frozenset({"target"}) diff --git a/tests/workspace/test_session_skills_deps.py b/tests/workspace/test_session_skills_deps.py new file mode 100644 index 0000000000..a468713c4f --- /dev/null +++ b/tests/workspace/test_session_skills_deps.py @@ -0,0 +1,337 @@ +"""Phase 2 tests: session_skills module — activate_deps resolution.""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from autoskillit.workspace.session_skills import DefaultSessionSkillManager + +pytestmark = [pytest.mark.layer("workspace"), pytest.mark.small] + + +# ── Tests: _parse_activate_deps ───────────────────────────────────────────── + + +class TestParseActivateDeps: + def test_parses_single_dep(self) -> None: + from autoskillit.workspace.session_skills import _parse_activate_deps + + content = "---\nname: foo\nactivate_deps: [arch-lens]\n---\nBody" + assert _parse_activate_deps(content) == ["arch-lens"] + + def test_parses_multiple_deps(self) -> None: + from autoskillit.workspace.session_skills import _parse_activate_deps + + content = "---\nname: foo\nactivate_deps: [arch-lens, mermaid]\n---\nBody" + assert _parse_activate_deps(content) == ["arch-lens", "mermaid"] + + def test_empty_deps(self) -> None: + from autoskillit.workspace.session_skills import _parse_activate_deps + + content = "---\nname: foo\nactivate_deps: []\n---\nBody" + assert _parse_activate_deps(content) == [] + + def test_no_activate_deps_field(self) -> None: + from autoskillit.workspace.session_skills import _parse_activate_deps + + content = "---\nname: foo\n---\nBody" + assert _parse_activate_deps(content) == [] + + def test_no_frontmatter(self) -> None: + from autoskillit.workspace.session_skills import _parse_activate_deps + + content = "Just body text" + assert _parse_activate_deps(content) == [] + + +# ── Tests: activate_skill_deps transitive dependency resolution ────────────────── + + +def _write_skill_md(base: Path, session_id: str, skill_name: str, content: str) -> Path: + """Helper to write a SKILL.md in the ephemeral session layout.""" + skill_dir = base / session_id / ".claude" / "skills" / skill_name + skill_dir.mkdir(parents=True, exist_ok=True) + md = skill_dir / "SKILL.md" + md.write_text(content) + return md + + +def _is_gated(base: Path, session_id: str, skill_name: str) -> bool: + """Return True if the skill has disable-model-invocation: true.""" + md = base / session_id / ".claude" / "skills" / skill_name / "SKILL.md" + content = md.read_text() + return "disable-model-invocation: true" in content + + +class TestActivateDepsResolution: + def test_activate_skill_deps_resolves_pack_deps(self, tmp_path: Path) -> None: + """Activating a skill with activate_deps: [arch-lens] ungates all arch-lens skills.""" + from unittest.mock import MagicMock + + from autoskillit.core.types import SkillSource + from autoskillit.workspace.skills import SkillInfo + + session_id = "test-pack-deps" + gate = "disable-model-invocation: true" + # Parent skill with pack dep + _write_skill_md( + tmp_path, + session_id, + "make-plan", + f"---\nname: make-plan\nactivate_deps: [arch-lens]\n{gate}\n---\n# Plan", + ) + # Three arch-lens skills + for name in ["arch-lens-a", "arch-lens-b", "arch-lens-c"]: + _write_skill_md( + tmp_path, + session_id, + name, + f"---\nname: {name}\ncategories: [arch-lens]\n{gate}\n---\n# Lens", + ) + + provider = MagicMock() + resolver = MagicMock() + provider.resolver = resolver + + def resolve_fn(name: str) -> SkillInfo | None: + if name.startswith("arch-lens-"): + return SkillInfo( + name=name, + source=SkillSource.BUNDLED_EXTENDED, + path=tmp_path / session_id / ".claude" / "skills" / name / "SKILL.md", + categories=frozenset({"arch-lens"}), + ) + return SkillInfo( + name=name, + source=SkillSource.BUNDLED_EXTENDED, + path=tmp_path / session_id / ".claude" / "skills" / name / "SKILL.md", + categories=frozenset(), + ) + + resolver.resolve.side_effect = resolve_fn + + mgr = DefaultSessionSkillManager(provider, ephemeral_root=tmp_path) + result = mgr.activate_skill_deps(session_id, "make-plan") + assert result is True + assert not _is_gated(tmp_path, session_id, "make-plan") + for name in ["arch-lens-a", "arch-lens-b", "arch-lens-c"]: + assert not _is_gated(tmp_path, session_id, name), f"{name} should be ungated" + + def test_activate_skill_deps_resolves_individual_skill_dep(self, tmp_path: Path) -> None: + """Activating a skill with activate_deps: [mermaid] ungates mermaid specifically.""" + from unittest.mock import MagicMock + + session_id = "test-individual-dep" + gate = "disable-model-invocation: true" + _write_skill_md( + tmp_path, + session_id, + "parent-skill", + f"---\nname: parent-skill\nactivate_deps: [mermaid]\n{gate}\n---\n# Parent", + ) + _write_skill_md( + tmp_path, + session_id, + "mermaid", + "---\nname: mermaid\ndisable-model-invocation: true\n---\n# Mermaid", + ) + + provider = MagicMock() + provider.resolver.resolve.return_value = None + + mgr = DefaultSessionSkillManager(provider, ephemeral_root=tmp_path) + result = mgr.activate_skill_deps(session_id, "parent-skill") + assert result is True + assert not _is_gated(tmp_path, session_id, "mermaid") + + def test_activate_skill_deps_resolves_two_level_transitive(self, tmp_path: Path) -> None: + """make-plan -> arch-lens-* -> mermaid: all three levels get ungated.""" + from unittest.mock import MagicMock + + from autoskillit.core.types import SkillSource + from autoskillit.workspace.skills import SkillInfo + + session_id = "test-two-level" + gate = "disable-model-invocation: true" + _write_skill_md( + tmp_path, + session_id, + "make-plan", + f"---\nname: make-plan\nactivate_deps: [arch-lens]\n{gate}\n---\n# Plan", + ) + _write_skill_md( + tmp_path, + session_id, + "arch-lens-x", + ( + f"---\nname: arch-lens-x\ncategories: [arch-lens]\n" + f"activate_deps: [mermaid]\n{gate}\n---\n# Lens" + ), + ) + _write_skill_md( + tmp_path, + session_id, + "mermaid", + "---\nname: mermaid\ndisable-model-invocation: true\n---\n# Mermaid", + ) + + provider = MagicMock() + resolver = MagicMock() + provider.resolver = resolver + + def resolve_fn(name: str) -> SkillInfo | None: + cats = frozenset({"arch-lens"}) if name.startswith("arch-lens-") else frozenset() + return SkillInfo( + name=name, + source=SkillSource.BUNDLED_EXTENDED, + path=tmp_path / session_id / ".claude" / "skills" / name / "SKILL.md", + categories=cats, + ) + + resolver.resolve.side_effect = resolve_fn + + mgr = DefaultSessionSkillManager(provider, ephemeral_root=tmp_path) + mgr.activate_skill_deps(session_id, "make-plan") + assert not _is_gated(tmp_path, session_id, "make-plan") + assert not _is_gated(tmp_path, session_id, "arch-lens-x") + assert not _is_gated(tmp_path, session_id, "mermaid") + + def test_activate_skill_deps_handles_circular_deps(self, tmp_path: Path) -> None: + """Circular activate_deps do not cause infinite recursion.""" + from unittest.mock import MagicMock + + session_id = "test-circular" + gate = "disable-model-invocation: true" + _write_skill_md( + tmp_path, + session_id, + "skill-a", + f"---\nname: skill-a\nactivate_deps: [skill-b]\n{gate}\n---\n# A", + ) + _write_skill_md( + tmp_path, + session_id, + "skill-b", + f"---\nname: skill-b\nactivate_deps: [skill-a]\n{gate}\n---\n# B", + ) + + provider = MagicMock() + provider.resolver.resolve.return_value = None + + mgr = DefaultSessionSkillManager(provider, ephemeral_root=tmp_path) + result = mgr.activate_skill_deps(session_id, "skill-a") + assert result is True + assert not _is_gated(tmp_path, session_id, "skill-a") + assert not _is_gated(tmp_path, session_id, "skill-b") + + def test_tier3_target_activates_tier2_deps(self, tmp_path: Path) -> None: + """A tier3 (ungated) skill's activate_deps still triggers tier2 dependency ungating.""" + from unittest.mock import MagicMock + + session_id = "test-tier3-deps" + # tier3 parent (not gated) + _write_skill_md( + tmp_path, + session_id, + "open-pr", + "---\nname: open-pr\nactivate_deps: [mermaid]\n---\n# Open PR", + ) + # tier2 dep (gated) + _write_skill_md( + tmp_path, + session_id, + "mermaid", + "---\nname: mermaid\ndisable-model-invocation: true\n---\n# Mermaid", + ) + + provider = MagicMock() + provider.resolver.resolve.return_value = None + + mgr = DefaultSessionSkillManager(provider, ephemeral_root=tmp_path) + result = mgr.activate_skill_deps(session_id, "open-pr") + assert result is True + assert not _is_gated(tmp_path, session_id, "mermaid") + + def test_pack_dep_absent_skills_noop(self, tmp_path: Path) -> None: + """Pack dep referencing skills not in ephemeral dir does not error.""" + from unittest.mock import MagicMock + + session_id = "test-absent-pack" + gate = "disable-model-invocation: true" + _write_skill_md( + tmp_path, + session_id, + "parent-skill", + f"---\nname: parent-skill\nactivate_deps: [exp-lens]\n{gate}\n---\n# Parent", + ) + + provider = MagicMock() + provider.resolver.resolve.return_value = None + + mgr = DefaultSessionSkillManager(provider, ephemeral_root=tmp_path) + result = mgr.activate_skill_deps(session_id, "parent-skill") + assert result is True + assert not _is_gated(tmp_path, session_id, "parent-skill") + + def test_activate_deps_strips_marker_from_dependency_body(self, tmp_path: Path) -> None: + """Dependency SKILL.md bodies must have %%ORDER_UP%% stripped after activation.""" + from unittest.mock import MagicMock + + session_id = "test-strip-marker" + gate = "disable-model-invocation: true" + marker_body = ( + "# Sub-Skill\n\nDo the work.\n\n" + "ORCHESTRATION DIRECTIVE: When your task is complete, " + "your final text output MUST end with: %%ORDER_UP%%\n" + "CRITICAL: Append %%ORDER_UP%% at the very end of your substantive response, " + "in the SAME message. Do NOT output %%ORDER_UP%% as a separate standalone message." + ) + _write_skill_md( + tmp_path, + session_id, + "parent-skill", + f"---\nname: parent-skill\nactivate_deps: [dep-skill]\n{gate}\n---\n# Parent", + ) + _write_skill_md( + tmp_path, + session_id, + "dep-skill", + f"---\nname: dep-skill\n{gate}\n---\n{marker_body}", + ) + + provider = MagicMock() + provider.resolver.resolve.return_value = None + + mgr = DefaultSessionSkillManager(provider, ephemeral_root=tmp_path) + mgr.activate_skill_deps(session_id, "parent-skill") + + dep_md = tmp_path / session_id / ".claude" / "skills" / "dep-skill" / "SKILL.md" + dep_content = dep_md.read_text() + assert "%%ORDER_UP%%" not in dep_content + assert "# Sub-Skill" in dep_content + assert "Do the work." in dep_content + + def test_activate_deps_preserves_marker_in_root_skill(self, tmp_path: Path) -> None: + """The root (directly targeted) skill keeps its %%ORDER_UP%% body intact.""" + from unittest.mock import MagicMock + + session_id = "test-preserve-root-marker" + gate = "disable-model-invocation: true" + _write_skill_md( + tmp_path, + session_id, + "root-skill", + f"---\nname: root-skill\n{gate}\n---\n# Root\n\n%%ORDER_UP%%", + ) + + provider = MagicMock() + provider.resolver.resolve.return_value = None + + mgr = DefaultSessionSkillManager(provider, ephemeral_root=tmp_path) + mgr.activate_skill_deps(session_id, "root-skill") + + root_md = tmp_path / session_id / ".claude" / "skills" / "root-skill" / "SKILL.md" + root_content = root_md.read_text() + assert "%%ORDER_UP%%" in root_content diff --git a/tests/workspace/test_session_skills_features.py b/tests/workspace/test_session_skills_features.py new file mode 100644 index 0000000000..2a9c96b81c --- /dev/null +++ b/tests/workspace/test_session_skills_features.py @@ -0,0 +1,247 @@ +"""Phase 2 tests: session_skills module — feature-gate skill filtering.""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from autoskillit.workspace.session_skills import ( + DefaultSessionSkillManager, + SkillsDirectoryProvider, +) + +pytestmark = [pytest.mark.layer("workspace"), pytest.mark.small] + + +# ── Tests: feature-gate skill filtering ───────────────────────────────────── + + +def test_skill_disabled_when_feature_off(tmp_path: Path) -> None: + """make-campaign excluded from session skills when features.fleet=false.""" + from tests._helpers import make_test_config + + config = make_test_config(features={"fleet": False}) + provider = SkillsDirectoryProvider() + mgr = DefaultSessionSkillManager(provider, ephemeral_root=tmp_path) + session_path = mgr.init_session("test-fleet-off", cook_session=False, config=config) + skill_names = {p.parent.name for p in session_path.glob(".claude/skills/*/SKILL.md")} + assert "make-campaign" not in skill_names, ( + "make-campaign must be excluded when fleet feature is disabled" + ) + + +def test_skill_enabled_when_feature_on(tmp_path: Path) -> None: + """make-campaign present in session skills when features.fleet=true.""" + from tests._helpers import make_test_config + + config = make_test_config(features={"fleet": True}) + provider = SkillsDirectoryProvider() + mgr = DefaultSessionSkillManager(provider, ephemeral_root=tmp_path) + session_path = mgr.init_session("test-fleet-on", cook_session=False, config=config) + skill_names = {p.parent.name for p in session_path.glob(".claude/skills/*/SKILL.md")} + assert "make-campaign" in skill_names, ( + "make-campaign must be present when fleet feature is enabled" + ) + + +def test_skill_suppressed_when_no_features_config(tmp_path: Path) -> None: + """Feature-gated skills use default_enabled when config is None (no features dict).""" + provider = SkillsDirectoryProvider() + mgr = DefaultSessionSkillManager(provider, ephemeral_root=tmp_path) + session_path = mgr.init_session("test-no-features-config", cook_session=False, config=None) + skill_names = {p.parent.name for p in session_path.glob(".claude/skills/*/SKILL.md")} + assert "make-campaign" not in skill_names, ( + "make-campaign must be absent when no features config is provided " + "(fleet.default_enabled=False)" + ) + + +def test_other_skills_unaffected_by_fleet_feature(tmp_path: Path) -> None: + """Non-fleet skills unaffected when fleet feature is disabled.""" + from tests._helpers import make_test_config + + config_off = make_test_config(features={"fleet": False}) + config_on = make_test_config(features={"fleet": True}) + provider = SkillsDirectoryProvider() + mgr = DefaultSessionSkillManager(provider, ephemeral_root=tmp_path) + session_off = mgr.init_session( + "test-others-unaffected-off", cook_session=False, config=config_off + ) + session_on = mgr.init_session( + "test-others-unaffected-on", cook_session=False, config=config_on + ) + skill_names_off = {p.parent.name for p in session_off.glob(".claude/skills/*/SKILL.md")} + skill_names_on = {p.parent.name for p in session_on.glob(".claude/skills/*/SKILL.md")} + assert "make-plan" in skill_names_off + assert "implement-worktree" in skill_names_off + # Count invariant: fleet=False removes exactly the fleet-category skills + assert skill_names_off.issubset(skill_names_on), ( + "fleet=False must only remove skills, not introduce new suppressions" + ) + suppressed = skill_names_on - skill_names_off + assert suppressed, "fleet=False must suppress at least one skill" + assert "make-campaign" in suppressed, "make-campaign must be in the suppressed set" + assert "make-plan" not in suppressed, "make-plan must not be suppressed by fleet=False" + assert "implement-worktree" not in suppressed, ( + "implement-worktree must not be suppressed by fleet=False" + ) + + +# ── Tests: cook session + feature-gate interaction ───────────────────────── + + +def test_cook_session_bypasses_feature_gate(tmp_path: Path) -> None: + """Cook sessions see all skills regardless of feature flags.""" + from tests._helpers import make_test_config + + config = make_test_config(features={"fleet": False}) + provider = SkillsDirectoryProvider() + mgr = DefaultSessionSkillManager(provider, ephemeral_root=tmp_path) + session_path = mgr.init_session("cook-feat-gate", cook_session=True, config=config) + skill_names = {p.parent.name for p in session_path.glob(".claude/skills/*/SKILL.md")} + assert "make-campaign" in skill_names, ( + "cook_session=True should bypass feature gates — " + "make-campaign must be available even when fleet is disabled" + ) + + +def test_cook_session_disabled_feature_tags_empty(tmp_path: Path) -> None: + """disabled_feature_tags is empty frozenset for cook sessions.""" + from tests._helpers import make_test_config + + config = make_test_config(features={"fleet": False}) + provider = SkillsDirectoryProvider() + mgr = DefaultSessionSkillManager(provider, ephemeral_root=tmp_path) + session_path = mgr.init_session("cook-tags-empty", cook_session=True, config=config) + # Verify via the integration effect: fleet-tagged skills are present + skill_names = {p.parent.name for p in session_path.glob(".claude/skills/*/SKILL.md")} + # If disabled_feature_tags were non-empty, fleet skills would be suppressed + # via _resolve_effective_disabled even without the _is_skill_disabled feature loop + assert "make-campaign" in skill_names + + +def test_non_cook_session_still_suppresses_feature_gated_skills(tmp_path: Path) -> None: + """Non-cook sessions with fleet=False still suppress make-campaign.""" + from tests._helpers import make_test_config + + config = make_test_config(features={"fleet": False}) + provider = SkillsDirectoryProvider() + mgr = DefaultSessionSkillManager(provider, ephemeral_root=tmp_path) + session_path = mgr.init_session("non-cook-feat", cook_session=False, config=config) + skill_names = {p.parent.name for p in session_path.glob(".claude/skills/*/SKILL.md")} + assert "make-campaign" not in skill_names, ( + "Non-cook session with fleet=False must suppress make-campaign" + ) + + +def test_feature_gate_suppresses_when_allow_only_is_none(tmp_path: Path) -> None: + """Feature gate still suppresses planner skills when allow_only=None.""" + from tests._helpers import make_test_config + + config = make_test_config(features={}, experimental_enabled=False) + provider = SkillsDirectoryProvider() + mgr = DefaultSessionSkillManager(provider, ephemeral_root=tmp_path) + session_path = mgr.init_session( + "feat-gate-allow-none", cook_session=False, config=config, allow_only=None + ) + skill_names = {p.parent.name for p in session_path.glob(".claude/skills/*/SKILL.md")} + assert "planner-elaborate-phase" not in skill_names, ( + "planner-elaborate-phase must be suppressed by the planner feature gate " + "when allow_only=None (no orchestrator-requested override)" + ) + + +# ── Tests: recipe_features parameter ───────────────────────────────────────── + + +def test_recipe_features_enables_planner_skills(tmp_path: Path) -> None: + """Planner skills are available when recipe declares requires_features: [planner].""" + from tests._helpers import make_test_config + + config = make_test_config(features={}, experimental_enabled=False) + provider = SkillsDirectoryProvider() + mgr = DefaultSessionSkillManager(provider, ephemeral_root=tmp_path) + session_path = mgr.init_session( + "recipe-feat-enables", + cook_session=False, + config=config, + recipe_features=frozenset({"planner"}), + ) + skill_names = {p.parent.name for p in session_path.glob(".claude/skills/*/SKILL.md")} + assert "planner-elaborate-phase" in skill_names, ( + "planner-elaborate-phase must be present when recipe_features includes 'planner'" + ) + + +def test_recipe_features_do_not_override_explicit_user_disable(tmp_path: Path) -> None: + """Explicit features.planner=False in config wins over recipe_features.""" + from tests._helpers import make_test_config + + config = make_test_config(features={"planner": False}, experimental_enabled=False) + provider = SkillsDirectoryProvider() + mgr = DefaultSessionSkillManager(provider, ephemeral_root=tmp_path) + session_path = mgr.init_session( + "recipe-feat-no-override", + cook_session=False, + config=config, + recipe_features=frozenset({"planner"}), + ) + skill_names = {p.parent.name for p in session_path.glob(".claude/skills/*/SKILL.md")} + assert "planner-elaborate-phase" not in skill_names, ( + "recipe_features must not override explicit user config features.planner=False" + ) + + +def test_recipe_features_enables_multiple_features(tmp_path: Path) -> None: + """Recipe declaring requires_features: [planner, fleet] enables both.""" + from tests._helpers import make_test_config + + config = make_test_config(features={}, experimental_enabled=False) + provider = SkillsDirectoryProvider() + mgr = DefaultSessionSkillManager(provider, ephemeral_root=tmp_path) + session_path = mgr.init_session( + "recipe-feat-multi", + cook_session=False, + config=config, + recipe_features=frozenset({"planner", "fleet"}), + ) + skill_names = {p.parent.name for p in session_path.glob(".claude/skills/*/SKILL.md")} + assert "planner-elaborate-phase" in skill_names, ( + "planner-elaborate-phase must be present when recipe_features includes 'planner'" + ) + assert "make-campaign" in skill_names, ( + "make-campaign must be present when recipe_features includes 'fleet'" + ) + + +@pytest.mark.parametrize( + "recipe_features,expected", + [ + (None, False), + (frozenset(), False), + (frozenset({"planner"}), True), + ], + ids=["none", "empty", "planner"], +) +def test_recipe_features_cross_axis( + tmp_path: Path, recipe_features: frozenset | None, expected: bool +) -> None: + """recipe_features axis interacts correctly with feature gate.""" + from tests._helpers import make_test_config + + config = make_test_config(features={}, experimental_enabled=False) + provider = SkillsDirectoryProvider() + mgr = DefaultSessionSkillManager(provider, ephemeral_root=tmp_path) + session_path = mgr.init_session( + f"recipe-feat-cross-{id(recipe_features)}", + cook_session=False, + config=config, + recipe_features=recipe_features, + ) + skill_names = {p.parent.name for p in session_path.glob(".claude/skills/*/SKILL.md")} + present = "planner-elaborate-phase" in skill_names + assert present == expected, ( + f"recipe_features={recipe_features!r}: expected planner-elaborate-phase " + f"present={expected}, got {present}" + ) diff --git a/tests/workspace/test_session_skills_filtering.py b/tests/workspace/test_session_skills_filtering.py new file mode 100644 index 0000000000..0cd48311d0 --- /dev/null +++ b/tests/workspace/test_session_skills_filtering.py @@ -0,0 +1,338 @@ +"""Phase 2 tests: session_skills module — subset/disabled-category and pack filtering.""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from autoskillit.workspace.session_skills import ( + DefaultSessionSkillManager, + SkillsDirectoryProvider, +) + +pytestmark = [pytest.mark.layer("workspace"), pytest.mark.small] + + +def test_init_session_unknown_skill_logs_warning(tmp_path: Path) -> None: + """Unknown skill name in config.skills.tier2 logs a warning (REQ-TIER-010).""" + import structlog.testing + + from tests._helpers import make_skills_config, make_test_config + + config = make_test_config( + skills=make_skills_config( + tier1=["open-kitchen", "close-kitchen"], + tier2=["this-skill-does-not-exist-anywhere"], + tier3=[], + ) + ) + mgr = DefaultSessionSkillManager(SkillsDirectoryProvider(), ephemeral_root=tmp_path) + with structlog.testing.capture_logs() as cap_logs: + mgr.init_session("test_unknown_warn", cook_session=False, config=config) + assert any( + "this-skill-does-not-exist-anywhere" in str(entry.get("event", "")) + for entry in cap_logs + if entry.get("log_level") == "warning" + ) + + +# T-VIS-006 +def test_init_session_skips_disabled_builtin_category(tmp_path: Path) -> None: + """Skills whose SKILL.md categories overlap disabled built-in tags are excluded.""" + from unittest.mock import MagicMock + + from autoskillit.workspace.session_skills import DefaultSessionSkillManager + from autoskillit.workspace.skills import SkillInfo, SkillSource + from tests._helpers import make_subsetsconfig, make_test_config + + skill_dir = tmp_path / "skills" / "fake-github-skill" + skill_dir.mkdir(parents=True) + (skill_dir / "SKILL.md").write_text("---\ncategories:\n - github\n---\n# Fake GitHub Skill\n") + + provider = MagicMock() + provider.list_skills.return_value = [ + SkillInfo( + name="fake-github-skill", + source=SkillSource.BUNDLED_EXTENDED, + path=skill_dir / "SKILL.md", + categories=frozenset({"github"}), + ) + ] + + config = make_test_config(subsets=make_subsetsconfig(disabled=["github"])) + root = tmp_path / "sessions" + root.mkdir() + mgr = DefaultSessionSkillManager(provider, ephemeral_root=root) + session_path = mgr.init_session("test-subset-skip", config=config) + + assert not (session_path / ".claude" / "skills" / "fake-github-skill").exists(), ( + "Skill with disabled category 'github' must not be copied to ephemeral dir" + ) + + +# T-VIS-007 +def test_init_session_skips_disabled_custom_tag(tmp_path: Path) -> None: + """Skills listed under a custom_tag that is disabled are excluded.""" + from unittest.mock import MagicMock + + from autoskillit.workspace.session_skills import DefaultSessionSkillManager + from autoskillit.workspace.skills import SkillInfo, SkillSource + from tests._helpers import make_subsetsconfig, make_test_config + + skill_dir = tmp_path / "skills" / "my-custom-skill" + skill_dir.mkdir(parents=True) + (skill_dir / "SKILL.md").write_text("---\ncategories: []\n---\n# My Custom Skill\n") + + provider = MagicMock() + provider.list_skills.return_value = [ + SkillInfo( + name="my-custom-skill", + source=SkillSource.BUNDLED_EXTENDED, + path=skill_dir / "SKILL.md", + categories=frozenset(), + ) + ] + provider.get_skill_content.return_value = "---\ncategories: []\n---\n# My Custom Skill\n" + + config = make_test_config( + subsets=make_subsetsconfig( + disabled=["data-infra"], + custom_tags={"data-infra": ["my-custom-skill"]}, + ) + ) + root = tmp_path / "sessions" + root.mkdir() + mgr = DefaultSessionSkillManager(provider, ephemeral_root=root) + session_path = mgr.init_session("test-custom-skip", config=config) + + assert not (session_path / ".claude" / "skills" / "my-custom-skill").exists(), ( + "Skill listed under disabled custom_tag must not be copied" + ) + + +# T-VIS-008 +def test_init_session_includes_non_disabled_skills(tmp_path: Path) -> None: + """Skills not in any disabled category are still copied to ephemeral dir.""" + from unittest.mock import MagicMock + + from autoskillit.workspace.session_skills import DefaultSessionSkillManager + from autoskillit.workspace.skills import SkillInfo, SkillSource + from tests._helpers import make_subsetsconfig, make_test_config + + skill_dir = tmp_path / "skills" / "safe-skill" + skill_dir.mkdir(parents=True) + (skill_dir / "SKILL.md").write_text("---\ncategories:\n - audit\n---\n# Safe Skill\n") + + provider = MagicMock() + provider.list_skills.return_value = [ + SkillInfo( + name="safe-skill", + source=SkillSource.BUNDLED_EXTENDED, + path=skill_dir / "SKILL.md", + categories=frozenset({"audit"}), + ) + ] + provider.get_skill_content.return_value = "---\ncategories:\n - audit\n---\n# Safe Skill\n" + + config = make_test_config(subsets=make_subsetsconfig(disabled=["github"])) + root = tmp_path / "sessions" + root.mkdir() + mgr = DefaultSessionSkillManager(provider, ephemeral_root=root) + session_path = mgr.init_session("test-safe-include", config=config) + + assert (session_path / ".claude" / "skills" / "safe-skill").exists(), ( + "Skills in non-disabled categories must be included" + ) + + +# REQ-EPH-002 +def test_cleanup_stale_default_is_72_hours() -> None: + import inspect + + sig = inspect.signature(DefaultSessionSkillManager.cleanup_stale) + default = sig.parameters["max_age_seconds"].default + assert default == 259200, f"Expected 259200 (72h), got {default}" + + +# REQ-PACK-005 / REQ-PACK-006: _resolve_effective_disabled +def test_resolve_effective_disabled_default_disabled_excluded() -> None: + from autoskillit.core import PACK_REGISTRY + from autoskillit.workspace.session_skills import _resolve_effective_disabled + + result = _resolve_effective_disabled( + explicit_disabled=[], + pack_registry=PACK_REGISTRY, + packs_enabled=[], + recipe_packs=None, + ) + # Default-disabled packs (research, exp-lens) should be in result + assert "research" in result + assert "exp-lens" in result + # Default-enabled packs should NOT be disabled + assert "github" not in result + + +def test_resolve_effective_disabled_packs_enabled_overrides_default() -> None: + from autoskillit.core import PACK_REGISTRY + from autoskillit.workspace.session_skills import _resolve_effective_disabled + + result = _resolve_effective_disabled( + explicit_disabled=[], + pack_registry=PACK_REGISTRY, + packs_enabled=["research"], + recipe_packs=None, + ) + assert "research" not in result # enabled by packs.enabled + + +# REQ-PACK-004: subsets.disabled always overrides packs.enabled +def test_resolve_effective_disabled_explicit_wins_over_pack_enabled() -> None: + from autoskillit.core import PACK_REGISTRY + from autoskillit.workspace.session_skills import _resolve_effective_disabled + + result = _resolve_effective_disabled( + explicit_disabled=["github"], # explicitly disabled + pack_registry=PACK_REGISTRY, + packs_enabled=["github"], # also in packs.enabled — explicit wins + recipe_packs=None, + ) + assert "github" in result # explicit disable survives + + +def test_resolve_effective_disabled_recipe_packs_overrides_default() -> None: + from autoskillit.core import PACK_REGISTRY + from autoskillit.workspace.session_skills import _resolve_effective_disabled + + result = _resolve_effective_disabled( + explicit_disabled=[], + pack_registry=PACK_REGISTRY, + packs_enabled=[], + recipe_packs=frozenset(["research"]), + ) + assert "research" not in result # enabled by recipe + + +def test_resolve_effective_disabled_includes_feature_tags() -> None: + from autoskillit.core import PACK_REGISTRY + from autoskillit.workspace.session_skills import _resolve_effective_disabled + + result = _resolve_effective_disabled( + explicit_disabled=[], + pack_registry=PACK_REGISTRY, + packs_enabled=[], + recipe_packs=None, + disabled_feature_tags=frozenset({"fleet"}), + ) + assert "fleet" in result + + +# REQ-PACK-006: Cook sessions skip default-disabled packs +def test_cook_session_skips_default_disabled_packs(tmp_path: Path) -> None: + """Cook session excludes default-disabled pack skills when packs.enabled=[].""" + from unittest.mock import MagicMock + + from autoskillit.workspace.skills import SkillInfo, SkillSource + from tests._helpers import make_test_config + + skill_dir = tmp_path / "skills" / "research-skill" + skill_dir.mkdir(parents=True) + (skill_dir / "SKILL.md").write_text("---\ncategories:\n - research\n---\n# Research Skill\n") + + provider = MagicMock() + provider.list_skills.return_value = [ + SkillInfo( + name="research-skill", + source=SkillSource.BUNDLED_EXTENDED, + path=skill_dir / "SKILL.md", + categories=frozenset({"research"}), + ) + ] + provider.get_skill_content.return_value = ( + "---\ncategories:\n - research\n---\n# Research Skill\n" + ) + + root = tmp_path / "sessions" + root.mkdir() + config = make_test_config() # packs.enabled=[] by default + mgr = DefaultSessionSkillManager(provider, ephemeral_root=root) + session_path = mgr.init_session("cook-research", cook_session=True, config=config) + + assert not (session_path / ".claude" / "skills" / "research-skill").exists(), ( + "Default-disabled pack skill must not be in cook session when packs.enabled=[]" + ) + + +# REQ-PACK-005: headless sessions exclude default-disabled packs +def test_headless_session_excludes_default_disabled_pack_skills(tmp_path: Path) -> None: + """Skills in 'exp-lens' pack are excluded from headless session when packs.enabled=[].""" + from unittest.mock import MagicMock + + from autoskillit.workspace.skills import SkillInfo, SkillSource + from tests._helpers import make_test_config + + skill_dir = tmp_path / "skills" / "exp-skill" + skill_dir.mkdir(parents=True) + (skill_dir / "SKILL.md").write_text( + "---\ncategories:\n - exp-lens\n---\n# Experimental Skill\n" + ) + + provider = MagicMock() + provider.list_skills.return_value = [ + SkillInfo( + name="exp-skill", + source=SkillSource.BUNDLED_EXTENDED, + path=skill_dir / "SKILL.md", + categories=frozenset({"exp-lens"}), + ) + ] + + root = tmp_path / "sessions" + root.mkdir() + config = make_test_config() # packs.enabled=[] by default + mgr = DefaultSessionSkillManager(provider, ephemeral_root=root) + session_path = mgr.init_session("headless-exp", cook_session=False, config=config) + + assert not (session_path / ".claude" / "skills" / "exp-skill").exists(), ( + "Default-disabled pack skill must not be in headless session when packs.enabled=[]" + ) + + +def test_init_session_recipe_packs_enables_default_disabled(tmp_path: Path) -> None: + """recipe_packs param enables default-disabled pack skills for this session.""" + from unittest.mock import MagicMock + + from autoskillit.workspace.skills import SkillInfo, SkillSource + from tests._helpers import make_test_config + + skill_dir = tmp_path / "skills" / "research-skill" + skill_dir.mkdir(parents=True) + (skill_dir / "SKILL.md").write_text("---\ncategories:\n - research\n---\n# Research Skill\n") + + provider = MagicMock() + provider.list_skills.return_value = [ + SkillInfo( + name="research-skill", + source=SkillSource.BUNDLED_EXTENDED, + path=skill_dir / "SKILL.md", + categories=frozenset({"research"}), + ) + ] + provider.get_skill_content.return_value = ( + "---\ncategories:\n - research\n---\n# Research Skill\n" + ) + + root = tmp_path / "sessions" + root.mkdir() + config = make_test_config() # packs.enabled=[] by default + mgr = DefaultSessionSkillManager(provider, ephemeral_root=root) + session_path = mgr.init_session( + "headless-recipe-research", + cook_session=False, + config=config, + recipe_packs=frozenset(["research"]), + ) + + assert (session_path / ".claude" / "skills" / "research-skill").exists(), ( + "Default-disabled pack skill should be present when enabled via recipe_packs" + ) diff --git a/tests/workspace/test_session_skills_provider.py b/tests/workspace/test_session_skills_provider.py new file mode 100644 index 0000000000..57a2f5de34 --- /dev/null +++ b/tests/workspace/test_session_skills_provider.py @@ -0,0 +1,172 @@ +"""Phase 2 tests: session_skills module — provider and core manager.""" + +from __future__ import annotations + +import os +import re +import time +from pathlib import Path + +import pytest +import yaml + +from autoskillit.workspace.session_skills import ( + DefaultSessionSkillManager, + SkillsDirectoryProvider, + resolve_ephemeral_root, +) + +pytestmark = [pytest.mark.layer("workspace"), pytest.mark.small] + + +def test_resolve_ephemeral_root_returns_writable_dir( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + import autoskillit.workspace.session_skills as ss + + monkeypatch.setattr(ss, "_CANDIDATE_ROOTS", [tmp_path]) + root = resolve_ephemeral_root() + assert root.exists() + assert root.is_dir() + test_file = root / "write_test.tmp" + test_file.write_text("ok") + test_file.unlink() + + +def test_resolve_ephemeral_root_fallback(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: + import autoskillit.workspace.session_skills as ss + + monkeypatch.setattr(ss, "_CANDIDATE_ROOTS", [Path("/nonexistent"), tmp_path]) + root = ss.resolve_ephemeral_root() + assert root.exists() + + +def test_skills_directory_provider_lists_all_skills() -> None: + provider = SkillsDirectoryProvider() + skills = provider.list_skills() + names = {s.name for s in skills} + assert "open-kitchen" in names + assert "close-kitchen" in names + assert "implement-worktree" in names + assert "sous-chef" not in names # internal, excluded + + +def test_provider_injects_disable_model_invocation_for_tier2() -> None: + provider = SkillsDirectoryProvider() + content = provider.get_skill_content("open-kitchen", gated=True) + fm_match = re.match(r"^---\n(.*?)\n---", content, re.DOTALL) + assert fm_match, "Content must have YAML frontmatter" + fm = yaml.safe_load(fm_match.group(1)) + assert fm.get("disable-model-invocation") is True + + +def test_provider_does_not_inject_for_cook_session() -> None: + # Use mermaid (skills_extended/, no flag at rest) to verify that gated=False + # returns unmodified content without injecting disable-model-invocation. + # open-kitchen and close-kitchen carry disable-model-invocation: true in their source + # (human-only skills), so they cannot be used to assert "flag not present". + provider = SkillsDirectoryProvider() + content = provider.get_skill_content("mermaid", gated=False) + fm_match = re.match(r"^---\n(.*?)\n---", content, re.DOTALL) + assert fm_match, "Content must have YAML frontmatter" + fm = yaml.safe_load(fm_match.group(1)) + assert fm.get("disable-model-invocation") is not True + + +def test_session_skill_manager_creates_ephemeral_dir(tmp_path: Path) -> None: + provider = SkillsDirectoryProvider() + mgr = DefaultSessionSkillManager(provider, ephemeral_root=tmp_path) + session_path = mgr.init_session("test-session-abc", cook_session=False) + assert session_path.exists() + assert session_path.is_dir() + skill_files = list(session_path.glob(".claude/skills/*/SKILL.md")) + assert len(skill_files) > 0 + + +def test_session_manager_injects_disable_for_tier2(tmp_path: Path) -> None: + """Non-cook init_session injects disable-model-invocation for tier2 skills.""" + from tests._helpers import make_skills_config, make_test_config + + provider = SkillsDirectoryProvider() + mgr = DefaultSessionSkillManager(provider, ephemeral_root=tmp_path) + config = make_test_config( + skills=make_skills_config( + tier1=["open-kitchen", "close-kitchen"], + tier2=["mermaid"], + tier3=[], + ) + ) + session_path = mgr.init_session("test-session-xyz", cook_session=False, config=config) + mermaid_md = session_path / ".claude" / "skills" / "mermaid" / "SKILL.md" + assert mermaid_md.exists() + content = mermaid_md.read_text() + fm_match = re.match(r"^---\n(.*?)\n---", content, re.DOTALL) + assert fm_match + fm = yaml.safe_load(fm_match.group(1)) + assert fm.get("disable-model-invocation") is True + + +def test_session_manager_no_flag_for_cook_session(tmp_path: Path) -> None: + """Cook session does not inject disable-model-invocation even for tier2 skills.""" + from tests._helpers import make_skills_config, make_test_config + + provider = SkillsDirectoryProvider() + mgr = DefaultSessionSkillManager(provider, ephemeral_root=tmp_path) + config = make_test_config( + skills=make_skills_config( + tier1=["open-kitchen", "close-kitchen"], + tier2=["mermaid"], + tier3=[], + ) + ) + session_path = mgr.init_session("cook-session-123", cook_session=True, config=config) + mermaid_md = session_path / ".claude" / "skills" / "mermaid" / "SKILL.md" + content = mermaid_md.read_text() + fm_match = re.match(r"^---\n(.*?)\n---", content, re.DOTALL) + assert fm_match + fm = yaml.safe_load(fm_match.group(1)) + assert fm.get("disable-model-invocation") is not True + + +def test_activate_skill_deps_removes_flag(tmp_path: Path) -> None: + from tests._helpers import make_skills_config, make_test_config + + provider = SkillsDirectoryProvider() + mgr = DefaultSessionSkillManager(provider, ephemeral_root=tmp_path) + config = make_test_config( + skills=make_skills_config( + tier1=["open-kitchen", "close-kitchen"], + tier2=["mermaid"], + tier3=[], + ) + ) + mgr.init_session("session-toggle", cook_session=False, config=config) + result = mgr.activate_skill_deps("session-toggle", "mermaid") + assert result is True + mermaid_md = tmp_path / "session-toggle" / ".claude" / "skills" / "mermaid" / "SKILL.md" + content = mermaid_md.read_text() + fm_match = re.match(r"^---\n(.*?)\n---", content, re.DOTALL) + assert fm_match + fm = yaml.safe_load(fm_match.group(1)) + assert "disable-model-invocation" not in fm or fm.get("disable-model-invocation") is not True + + +def test_cleanup_stale_removes_old_dirs(tmp_path: Path) -> None: + provider = SkillsDirectoryProvider() + mgr = DefaultSessionSkillManager(provider, ephemeral_root=tmp_path) + stale_dir = tmp_path / "stale-session" + stale_dir.mkdir() + os.utime(stale_dir, (time.time() - 90000, time.time() - 90000)) # 25h old + fresh_dir = tmp_path / "fresh-session" + fresh_dir.mkdir() + count = mgr.cleanup_stale(max_age_seconds=86400) + assert count == 1 + assert not stale_dir.exists() + assert fresh_dir.exists() + + +def test_tier2_skills_constant_removed() -> None: + """TIER2_SKILLS no longer exported from workspace (superseded by config).""" + import autoskillit.workspace as ws + + assert not hasattr(ws, "TIER2_SKILLS") diff --git a/tests/workspace/test_skill_content_substitution.py b/tests/workspace/test_skill_content_substitution.py index 883318f049..285fbcbf18 100644 --- a/tests/workspace/test_skill_content_substitution.py +++ b/tests/workspace/test_skill_content_substitution.py @@ -11,6 +11,8 @@ SkillsDirectoryProvider, ) +pytestmark = [pytest.mark.layer("workspace"), pytest.mark.small] + class _StubInfo: def __init__(self, path: Path, name: str) -> None: @@ -36,8 +38,11 @@ def _provider_with_synth( body: str, *, temp_dir_relpath: str = ".autoskillit/temp", + default_base_branch: str = "main", ) -> tuple[SkillsDirectoryProvider, _StubInfo]: - provider = SkillsDirectoryProvider(temp_dir_relpath=temp_dir_relpath) + provider = SkillsDirectoryProvider( + temp_dir_relpath=temp_dir_relpath, default_base_branch=default_base_branch + ) info = _StubInfo(_make_synth_skill_md(tmp_path, name, body), name) monkeypatch.setattr(provider._resolver, "resolve", lambda n: info if n == name else None) return provider, info @@ -54,6 +59,57 @@ def test_get_skill_content_substitutes_default_temp_dir( assert ".autoskillit/temp/foo/output.md" in content +def test_get_skill_content_substitutes_default_base_branch( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + """{{DEFAULT_BASE_BRANCH}} resolves to the default when no explicit value set.""" + provider, _ = _provider_with_synth( + monkeypatch, + tmp_path, + "synth_base_branch", + "Merging into {{DEFAULT_BASE_BRANCH}}", + default_base_branch="main", + ) + content = provider.get_skill_content("synth_base_branch", gated=False) + assert "{{DEFAULT_BASE_BRANCH}}" not in content + assert "main" in content + + +def test_get_skill_content_substitutes_explicit_base_branch( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + """{{DEFAULT_BASE_BRANCH}} resolves to the explicit value passed to constructor.""" + provider, _ = _provider_with_synth( + monkeypatch, + tmp_path, + "synth_develop", + "Merging into {{DEFAULT_BASE_BRANCH}}", + default_base_branch="develop", + ) + content = provider.get_skill_content("synth_develop", gated=False) + assert "{{DEFAULT_BASE_BRANCH}}" not in content + assert "develop" in content + + +def test_get_skill_content_substitutes_both_placeholders( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + """Both {{AUTOSKILLIT_TEMP}} and {{DEFAULT_BASE_BRANCH}} are substituted.""" + provider, _ = _provider_with_synth( + monkeypatch, + tmp_path, + "synth_both", + "Using {{AUTOSKILLIT_TEMP}} and {{DEFAULT_BASE_BRANCH}}", + temp_dir_relpath=".autoskillit/temp", + default_base_branch="develop", + ) + content = provider.get_skill_content("synth_both", gated=False) + assert "{{AUTOSKILLIT_TEMP}}" not in content + assert "{{DEFAULT_BASE_BRANCH}}" not in content + assert ".autoskillit/temp" in content + assert "develop" in content + + def test_get_skill_content_substitutes_custom_temp_dir( monkeypatch: pytest.MonkeyPatch, tmp_path: Path ) -> None: diff --git a/tests/workspace/test_skills.py b/tests/workspace/test_skills.py index cce9533cd3..d097265f03 100644 --- a/tests/workspace/test_skills.py +++ b/tests/workspace/test_skills.py @@ -5,8 +5,10 @@ import re from pathlib import Path +import pytest import yaml +import autoskillit.workspace.skills as _skills_mod from autoskillit.core.types import SkillSource from autoskillit.workspace.skills import ( DefaultSkillResolver, @@ -14,6 +16,8 @@ bundled_skills_extended_dir, ) +pytestmark = [pytest.mark.layer("workspace"), pytest.mark.small] + BUNDLED_SKILLS = [ "analyze-prs", "arch-lens-c4-container", @@ -34,12 +38,18 @@ "audit-claims", "audit-cohesion", "audit-defense-standards", + "audit-docs", + "audit-feature-gates", "audit-friction", "audit-impl", + "audit-review-decisions", "audit-tests", + "build-execution-map", "bundle-local-report", "close-kitchen", "collapse-issues", + "compose-pr", + "compose-research-pr", "design-guards", "diagnose-ci", "dry-walkthrough", @@ -63,12 +73,14 @@ "exp-lens-unit-interference", "exp-lens-validity-threats", "exp-lens-variance-stability", + "generate-report", "implement-experiment", "implement-worktree", "implement-worktree-no-merge", "investigate", "issue-splitter", "make-arch-diag", + "make-campaign", "make-experiment-diag", "make-groups", "make-plan", @@ -78,14 +90,29 @@ "migrate-recipes", "open-integration-pr", "open-kitchen", - "open-pr", - "open-research-pr", "pipeline-summary", "plan-experiment", "plan-visualization", + "planner-analyze", + "planner-assess-review-approach", + "planner-consolidate-wps", + "planner-elaborate-assignments", + "planner-elaborate-phase", + "planner-elaborate-wps", + "planner-extract-domain", + "planner-generate-phases", + "planner-reconcile-deps", + "planner-refine", + "planner-refine-assignments", + "planner-refine-phases", + "planner-refine-wps", "prepare-issue", + "prepare-pr", + "prepare-research-pr", "process-issues", + "promote-to-main", "rectify", + "reload-session", "report-bug", "resolve-claims-review", "resolve-design-review", @@ -103,18 +130,18 @@ "setup-project", "smoke-task", "sous-chef", - "sprint-planner", "stage-data", "triage-issues", "troubleshoot-experiment", "validate-audit", + "validate-review-decisions", "verify-diag", "vis-lens-always-on", "vis-lens-antipattern", "vis-lens-caption-annot", "vis-lens-chart-select", "vis-lens-color-access", - "vis-lens-domain-norms", + "vis-lens-methodology-norms", "vis-lens-figure-table", "vis-lens-multi-compare", "vis-lens-reproducibility", @@ -122,7 +149,6 @@ "vis-lens-temporal", "vis-lens-uncertainty", "write-recipe", - "generate-report", ] # Internal-only skill documents: injected programmatically, never invocable as slash commands. @@ -172,6 +198,10 @@ "audit-cohesion", "audit-defense-standards", "validate-audit", + "validate-test-audit", + "validate-review-decisions", + "audit-docs", + "audit-review-decisions", ] BUNDLED_SKILL_NAMES = set(BUNDLED_SKILLS) @@ -255,39 +285,6 @@ def test_skill_md_cross_references_are_namespaced(self) -> None: skill_file = f"{skill_md.parent.name}/SKILL.md" assert False, f"{skill_file}: /{name} should be /autoskillit:{name}" - def test_skill_md_yaml_examples_are_valid_workflows(self) -> None: - """YAML workflow examples embedded in SKILL.md files must pass validation.""" - import yaml as _yaml - - from autoskillit.recipe.io import ( - _parse_recipe as _parse_workflow, - ) - from autoskillit.recipe.validator import ( - validate_recipe as validate_workflow, - ) - - yaml_block_re = re.compile(r"```yaml\n(.*?)```", re.DOTALL) - - for skills_dir in _all_skill_roots(): - for skill_md in skills_dir.rglob("SKILL.md"): - content = skill_md.read_text() - for match in yaml_block_re.finditer(content): - block = match.group(1) - # Only validate blocks that look like full workflow definitions - if "steps:" not in block or "name:" not in block: - continue - # Skip format templates that use {placeholder} syntax - if "{script-name}" in block or "{mcp_tool_name}" in block: - continue - data = _yaml.safe_load(block) - if not isinstance(data, dict) or "steps" not in data: - continue - wf = _parse_workflow(data) - errors = [e for e in validate_workflow(wf) if "kitchen_rules" not in e.lower()] - assert not errors, ( - f"{skill_md.parent.name}/SKILL.md has invalid YAML example:\n {errors}" - ) - def test_skill_md_has_critical_constraints(self) -> None: """Every user-invocable SKILL.md must have Critical Constraints (NEVER/ALWAYS blocks).""" failures: list[str] = [] @@ -311,6 +308,7 @@ def test_skill_md_has_critical_constraints(self) -> None: def test_file_producing_skills_have_output_guard(self) -> None: """File-producing skills must have a negative output constraint in NEVER block.""" FILE_PRODUCING_SKILLS = { + "build-execution-map": ".autoskillit/temp/build-execution-map/", "investigate": ".autoskillit/temp/investigate/", "make-groups": ".autoskillit/temp/make-groups/", "make-plan": ".autoskillit/temp/make-plan/", @@ -371,7 +369,7 @@ def test_make_groups_skill_documents_per_group_output(self) -> None: assert "manifest" in content.lower() def test_bundled_skills_list_matches_filesystem(self) -> None: - """make-script-skill SKILL.md bundled skills list must match filesystem.""" + """write-recipe SKILL.md bundled skills list must match filesystem.""" skill_md = DefaultSkillResolver().resolve("write-recipe").path content = skill_md.read_text() @@ -401,7 +399,7 @@ def test_bundled_skills_list_matches_filesystem(self) -> None: actual_skills = sorted(s.name for s in DefaultSkillResolver().list_all()) assert listed_skills == actual_skills, ( - f"make-script-skill bundled skills list doesn't match filesystem.\n" + f"write-recipe bundled skills list doesn't match filesystem.\n" f" Listed: {listed_skills}\n" f" On disk: {actual_skills}" ) @@ -450,18 +448,18 @@ def test_tier1_only_in_skills_dir(self) -> None: names = {d.name for d in bundled_skills_dir().iterdir() if d.is_dir()} assert names == {"open-kitchen", "close-kitchen", "sous-chef"} - def test_90_skills_in_skills_extended(self) -> None: - """skills_extended/ contains exactly 107 SKILL.md-carrying directories.""" + def test_skills_in_skills_extended(self) -> None: + """skills_extended/ contains at least 125 SKILL.md-carrying directories.""" skills = [ d for d in bundled_skills_extended_dir().iterdir() if d.is_dir() and (d / "SKILL.md").is_file() ] - assert len(skills) == 107 + assert len(skills) >= 125 - def test_skill_resolver_list_all_total_count(self) -> None: - """list_all() returns 109 public skills (2 Tier-1 + 107 extended).""" - assert len(DefaultSkillResolver().list_all()) == 109 + def test_skill_resolver_list_all_minimum_count(self) -> None: + """list_all() returns at least 128 public skills (2 Tier-1 + extended).""" + assert len(DefaultSkillResolver().list_all()) >= 128 def test_skill_resolver_resolve_extended_skill(self) -> None: """resolve() finds a skill living in skills_extended/ with BUNDLED_EXTENDED source.""" @@ -493,6 +491,14 @@ def test_list_all_no_cross_directory_name_collision(self) -> None: f"Skill name collision across skills/ and skills_extended/: {sorted(dupes)}" ) + def test_list_all_excludes_retired_skill_names(self) -> None: + """list_all() must not surface any skill whose name is in RETIRED_SKILL_NAMES.""" + from autoskillit.core import RETIRED_SKILL_NAMES + + names = {s.name for s in DefaultSkillResolver().list_all()} + surfaced = RETIRED_SKILL_NAMES & names + assert not surfaced, f"list_all() returned retired skill name(s): {sorted(surfaced)}" + class TestSkillCategories: # T6 — read_skill_categories() and SkillInfo.categories @@ -575,6 +581,8 @@ def test_all_audit_skills_have_audit_category(self) -> None: "audit-friction", "audit-impl", "validate-audit", + "audit-docs", + "audit-review-decisions", ]: info = resolver.resolve(name) assert info is not None @@ -605,6 +613,40 @@ def test_make_experiment_diag_has_exp_lens_category(self) -> None: assert info is not None assert "exp-lens" in info.categories + def test_make_campaign_has_fleet_category(self) -> None: + """make-campaign must declare both orchestration-family and fleet categories.""" + info = DefaultSkillResolver().resolve("make-campaign") + assert info is not None + assert "fleet" in info.categories, "make-campaign missing 'fleet' category" + assert "orchestration-family" in info.categories, ( + "make-campaign must retain 'orchestration-family' category" + ) + + def test_planner_analyze_has_planner_category(self) -> None: + info = DefaultSkillResolver().resolve("planner-analyze") + assert info is not None + assert "planner" in info.categories + + def test_planner_extract_domain_has_planner_category(self) -> None: + info = DefaultSkillResolver().resolve("planner-extract-domain") + assert info is not None + assert "planner" in info.categories + + def test_planner_generate_phases_has_planner_category(self) -> None: + info = DefaultSkillResolver().resolve("planner-generate-phases") + assert info is not None + assert "planner" in info.categories + + def test_planner_elaborate_phase_has_planner_category(self) -> None: + info = DefaultSkillResolver().resolve("planner-elaborate-phase") + assert info is not None + assert "planner" in info.categories + + def test_planner_elaborate_assignments_has_planner_category(self) -> None: + info = DefaultSkillResolver().resolve("planner-elaborate-assignments") + assert info is not None + assert "planner" in info.categories + RESEARCH_SKILL_NAMES = { "scope", @@ -687,3 +729,89 @@ def test_resolve_claims_review_skill_md_exists() -> None: info = resolver.resolve("resolve-claims-review") assert info is not None, "resolve-claims-review skill not found" assert info.path.exists(), f"SKILL.md missing at {info.path}" + + +def test_audit_docs_skill_md_exists() -> None: + resolver = DefaultSkillResolver() + info = resolver.resolve("audit-docs") + assert info is not None + assert info.path.exists() + + +# --------------------------------------------------------------------------- +# Cache tests +# --------------------------------------------------------------------------- + + +def test_list_all_cache_hit(monkeypatch: pytest.MonkeyPatch) -> None: + """Second list_all() call returns cached result without re-scanning.""" + monkeypatch.setattr(_skills_mod, "_LIST_ALL_CACHE", None) + monkeypatch.setattr(_skills_mod, "_LIST_ALL_CACHE_KEY", (0.0, 0.0)) + resolver = DefaultSkillResolver() + result1 = resolver.list_all() + call_count = 0 + original_scan = _skills_mod._scan_directory + + def counting_scan(*args: object, **kwargs: object) -> list[object]: + nonlocal call_count + call_count += 1 + return original_scan(*args, **kwargs) # type: ignore[no-any-return] + + monkeypatch.setattr(_skills_mod, "_scan_directory", counting_scan) + result2 = resolver.list_all() + assert result2 == result1 + assert call_count == 0 # Cache hit — no re-scan + + +def test_list_all_cache_invalidation_on_mtime_change(monkeypatch: pytest.MonkeyPatch) -> None: + """list_all() re-scans when directory mtime changes.""" + monkeypatch.setattr(_skills_mod, "_LIST_ALL_CACHE", None) + monkeypatch.setattr(_skills_mod, "_LIST_ALL_CACHE_KEY", (0.0, 0.0)) + resolver = DefaultSkillResolver() + result1 = resolver.list_all() + monkeypatch.setattr(_skills_mod, "_LIST_ALL_CACHE_KEY", (999.0, 999.0)) + call_count = 0 + original_scan = _skills_mod._scan_directory + + def counting_scan(*args: object, **kwargs: object) -> list[object]: + nonlocal call_count + call_count += 1 + return original_scan(*args, **kwargs) # type: ignore[no-any-return] + + monkeypatch.setattr(_skills_mod, "_scan_directory", counting_scan) + result2 = resolver.list_all() + assert result2 == result1 # Same content + assert call_count == 2 # Re-scanned both directories + + +def test_resolve_instance_cache_hit(monkeypatch: pytest.MonkeyPatch) -> None: + """Second resolve() for same name returns cached SkillInfo without disk I/O.""" + monkeypatch.setattr(_skills_mod, "_LIST_ALL_CACHE", None) + monkeypatch.setattr(_skills_mod, "_LIST_ALL_CACHE_KEY", (0.0, 0.0)) + resolver = DefaultSkillResolver() + result1 = resolver.resolve("make-plan") + assert result1 is not None + call_count = 0 + original_fn = _skills_mod._skill_info_from_frontmatter + + def counting_fn(*args: object, **kwargs: object) -> object: + nonlocal call_count + call_count += 1 + return original_fn(*args, **kwargs) + + monkeypatch.setattr(_skills_mod, "_skill_info_from_frontmatter", counting_fn) + result2 = resolver.resolve("make-plan") + assert result2 is not None + assert result2.name == result1.name + assert call_count == 0 # Cache hit + + +def test_resolve_instance_cache_caches_none(monkeypatch: pytest.MonkeyPatch) -> None: + """resolve() caches None for unknown skills to avoid repeated is_file() checks.""" + monkeypatch.setattr(_skills_mod, "_LIST_ALL_CACHE", None) + monkeypatch.setattr(_skills_mod, "_LIST_ALL_CACHE_KEY", (0.0, 0.0)) + resolver = DefaultSkillResolver() + result1 = resolver.resolve("nonexistent-skill-xyz") + assert result1 is None + assert "nonexistent-skill-xyz" in resolver._resolve_cache + assert resolver._resolve_cache["nonexistent-skill-xyz"] is None diff --git a/tests/workspace/test_worktree.py b/tests/workspace/test_worktree.py index 79b2a77a24..60cb954e01 100644 --- a/tests/workspace/test_worktree.py +++ b/tests/workspace/test_worktree.py @@ -5,12 +5,15 @@ import pytest from autoskillit.core import CleanupResult -from autoskillit.workspace.worktree import ( +from autoskillit.workspace import ( list_git_worktrees, remove_git_worktree, remove_worktree_sidecar, + write_worktree_sidecar, ) +pytestmark = [pytest.mark.layer("workspace"), pytest.mark.small] + class TestListGitWorktrees: """list_git_worktrees(project_root) returns paths of linked worktrees under root.""" @@ -115,3 +118,42 @@ def test_missing_sidecar_is_skipped_not_failed(self, tmp_path): assert any("impl-absent" in s for s in result.skipped), ( f"Expected impl-absent path in skipped list; got: {result.skipped}" ) + + +class TestWriteWorktreeSidecar: + """write_worktree_sidecar creates .autoskillit/temp/worktrees//base-branch.""" + + def test_writes_base_branch_file(self, tmp_path) -> None: + """File is created at the correct path with correct content.""" + path = write_worktree_sidecar(tmp_path, "impl-foo-20260101-120000", "develop") + assert path.exists() + assert path.read_text().strip() == "develop" + + def test_creates_intermediate_directories(self, tmp_path) -> None: + """Intermediate directories are created if they don't exist.""" + write_worktree_sidecar(tmp_path, "impl-bar-20260201-130000", "main") + expected = tmp_path / ".autoskillit" / "temp" / "worktrees" / "impl-bar-20260201-130000" + assert (expected / "base-branch").exists() + + def test_overwrites_existing_sidecar(self, tmp_path) -> None: + """Writing again replaces the existing content.""" + name = "impl-baz-20260301-140000" + write_worktree_sidecar(tmp_path, name, "main") + write_worktree_sidecar(tmp_path, name, "develop") + path = tmp_path / ".autoskillit" / "temp" / "worktrees" / name / "base-branch" + assert path.read_text().strip() == "develop" + + def test_returns_written_path(self, tmp_path) -> None: + """Return value is the path to the sidecar file.""" + path = write_worktree_sidecar(tmp_path, "impl-ret-20260401-150000", "main") + assert path.name == "base-branch" + assert "impl-ret-20260401-150000" in str(path) + + def test_roundtrip_file_content(self, tmp_path) -> None: + """Write via write_worktree_sidecar, verify file readable with stdlib.""" + name = "impl-rt-20260501-160000" + branch = "develop" + sidecar_path = write_worktree_sidecar(tmp_path, name, branch) + + assert sidecar_path.is_file() + assert sidecar_path.read_text().strip() == branch diff --git a/uv.lock b/uv.lock index 12779bcb76..97c785486b 100644 --- a/uv.lock +++ b/uv.lock @@ -73,7 +73,7 @@ wheels = [ [[package]] name = "autoskillit" -version = "0.9.0" +version = "0.9.569" source = { editable = "." } dependencies = [ { name = "anyio" }, @@ -82,23 +82,30 @@ dependencies = [ { name = "fastmcp" }, { name = "httpx" }, { name = "igraph" }, + { name = "lazy-loader" }, { name = "markdown-it-py" }, + { name = "orjson" }, { name = "packaging" }, + { name = "pathspec" }, { name = "psutil" }, { name = "pygments" }, { name = "pyjwt" }, { name = "pyyaml" }, + { name = "regex" }, { name = "structlog" }, + { name = "uvloop", marker = "sys_platform != 'win32'" }, ] [package.optional-dependencies] dev = [ - { name = "api-simulator" }, + { name = "api-simulator", marker = "sys_platform == 'linux'" }, { name = "import-linter" }, { name = "packaging" }, { name = "pytest" }, { name = "pytest-asyncio" }, + { name = "pytest-cov" }, { name = "pytest-httpx" }, + { name = "pytest-testmon" }, { name = "pytest-timeout" }, { name = "pytest-xdist" }, { name = "ruff" }, @@ -107,27 +114,34 @@ dev = [ [package.metadata] requires-dist = [ { name = "anyio", specifier = ">=4.6.0" }, - { name = "api-simulator", marker = "extra == 'dev'", git = "https://github.com/TalonT-Org/api-simulator?subdirectory=crates%2Fapi-simulator-py&rev=8f711f958d4d93a8e53a3c3b51279fb420870f3a" }, + { name = "api-simulator", marker = "sys_platform == 'linux' and extra == 'dev'", git = "https://github.com/TalonT-Org/api-simulator?subdirectory=crates%2Fapi-simulator-py&rev=8f711f958d4d93a8e53a3c3b51279fb420870f3a" }, { name = "cyclopts", specifier = ">=4.0" }, { name = "dynaconf", specifier = ">=3.2.13,<4.0" }, { name = "fastmcp", specifier = ">=3.2.0,<4.0" }, - { name = "httpx", specifier = ">=0.28,<0.29" }, + { name = "httpx", specifier = ">=0.28,<1.0" }, { name = "igraph", specifier = ">=1.0,<2.0" }, { name = "import-linter", marker = "extra == 'dev'", specifier = ">=2.1" }, + { name = "lazy-loader", specifier = ">=0.4" }, { name = "markdown-it-py", specifier = ">=3.0" }, + { name = "orjson", specifier = ">=3.11" }, { name = "packaging", specifier = ">=23.0" }, { name = "packaging", marker = "extra == 'dev'", specifier = ">=25.0" }, + { name = "pathspec", specifier = ">=1.0,<2.0" }, { name = "psutil", specifier = ">=7.2.0" }, { name = "pygments", specifier = ">=2.20.0" }, { name = "pyjwt", specifier = ">=2.12.0" }, { name = "pytest", marker = "extra == 'dev'", specifier = ">=9.0.0" }, { name = "pytest-asyncio", marker = "extra == 'dev'", specifier = ">=1.0.0" }, + { name = "pytest-cov", marker = "extra == 'dev'", specifier = ">=7.0" }, { name = "pytest-httpx", marker = "extra == 'dev'", specifier = ">=0.36" }, + { name = "pytest-testmon", marker = "extra == 'dev'", specifier = ">=2.2.0" }, { name = "pytest-timeout", marker = "extra == 'dev'", specifier = ">=2.3" }, { name = "pytest-xdist", marker = "extra == 'dev'", specifier = ">=3.5.0" }, { name = "pyyaml", specifier = ">=6.0" }, + { name = "regex", specifier = ">=2026.1" }, { name = "ruff", marker = "extra == 'dev'", specifier = ">=0.15.0" }, { name = "structlog", specifier = ">=25.0,<26.0" }, + { name = "uvloop", marker = "sys_platform != 'win32'", specifier = ">=0.21.0" }, ] provides-extras = ["dev"] @@ -283,6 +297,110 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/d1/d6/3965ed04c63042e047cb6a3e6ed1a63a35087b6a609aa3a15ed8ac56c221/colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6", size = 25335, upload-time = "2022-10-25T02:36:20.889Z" }, ] +[[package]] +name = "coverage" +version = "7.13.5" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/9d/e0/70553e3000e345daff267cec284ce4cbf3fc141b6da229ac52775b5428f1/coverage-7.13.5.tar.gz", hash = "sha256:c81f6515c4c40141f83f502b07bbfa5c240ba25bbe73da7b33f1e5b6120ff179", size = 915967, upload-time = "2026-03-17T10:33:18.341Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/4b/37/d24c8f8220ff07b839b2c043ea4903a33b0f455abe673ae3c03bbdb7f212/coverage-7.13.5-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:66a80c616f80181f4d643b0f9e709d97bcea413ecd9631e1dedc7401c8e6695d", size = 219381, upload-time = "2026-03-17T10:30:14.68Z" }, + { url = "https://files.pythonhosted.org/packages/35/8b/cd129b0ca4afe886a6ce9d183c44d8301acbd4ef248622e7c49a23145605/coverage-7.13.5-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:145ede53ccbafb297c1c9287f788d1bc3efd6c900da23bf6931b09eafc931587", size = 219880, upload-time = "2026-03-17T10:30:16.231Z" }, + { url = "https://files.pythonhosted.org/packages/55/2f/e0e5b237bffdb5d6c530ce87cc1d413a5b7d7dfd60fb067ad6d254c35c76/coverage-7.13.5-cp311-cp311-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:0672854dc733c342fa3e957e0605256d2bf5934feeac328da9e0b5449634a642", size = 250303, upload-time = "2026-03-17T10:30:17.748Z" }, + { url = "https://files.pythonhosted.org/packages/92/be/b1afb692be85b947f3401375851484496134c5554e67e822c35f28bf2fbc/coverage-7.13.5-cp311-cp311-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:ec10e2a42b41c923c2209b846126c6582db5e43a33157e9870ba9fb70dc7854b", size = 252218, upload-time = "2026-03-17T10:30:19.804Z" }, + { url = "https://files.pythonhosted.org/packages/da/69/2f47bb6fa1b8d1e3e5d0c4be8ccb4313c63d742476a619418f85740d597b/coverage-7.13.5-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:be3d4bbad9d4b037791794ddeedd7d64a56f5933a2c1373e18e9e568b9141686", size = 254326, upload-time = "2026-03-17T10:30:21.321Z" }, + { url = "https://files.pythonhosted.org/packages/d5/d0/79db81da58965bd29dabc8f4ad2a2af70611a57cba9d1ec006f072f30a54/coverage-7.13.5-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:4d2afbc5cc54d286bfb54541aa50b64cdb07a718227168c87b9e2fb8f25e1743", size = 256267, upload-time = "2026-03-17T10:30:23.094Z" }, + { url = "https://files.pythonhosted.org/packages/e5/32/d0d7cc8168f91ddab44c0ce4806b969df5f5fdfdbb568eaca2dbc2a04936/coverage-7.13.5-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:3ad050321264c49c2fa67bb599100456fc51d004b82534f379d16445da40fb75", size = 250430, upload-time = "2026-03-17T10:30:25.311Z" }, + { url = "https://files.pythonhosted.org/packages/4d/06/a055311d891ddbe231cd69fdd20ea4be6e3603ffebddf8704b8ca8e10a3c/coverage-7.13.5-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:7300c8a6d13335b29bb76d7651c66af6bd8658517c43499f110ddc6717bfc209", size = 252017, upload-time = "2026-03-17T10:30:27.284Z" }, + { url = "https://files.pythonhosted.org/packages/d6/f6/d0fd2d21e29a657b5f77a2fe7082e1568158340dceb941954f776dce1b7b/coverage-7.13.5-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:eb07647a5738b89baab047f14edd18ded523de60f3b30e75c2acc826f79c839a", size = 250080, upload-time = "2026-03-17T10:30:29.481Z" }, + { url = "https://files.pythonhosted.org/packages/4e/ab/0d7fb2efc2e9a5eb7ddcc6e722f834a69b454b7e6e5888c3a8567ecffb31/coverage-7.13.5-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:9adb6688e3b53adffefd4a52d72cbd8b02602bfb8f74dcd862337182fd4d1a4e", size = 253843, upload-time = "2026-03-17T10:30:31.301Z" }, + { url = "https://files.pythonhosted.org/packages/ba/6f/7467b917bbf5408610178f62a49c0ed4377bb16c1657f689cc61470da8ce/coverage-7.13.5-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:7c8d4bc913dd70b93488d6c496c77f3aff5ea99a07e36a18f865bca55adef8bd", size = 249802, upload-time = "2026-03-17T10:30:33.358Z" }, + { url = "https://files.pythonhosted.org/packages/75/2c/1172fb689df92135f5bfbbd69fc83017a76d24ea2e2f3a1154007e2fb9f8/coverage-7.13.5-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:0e3c426ffc4cd952f54ee9ffbdd10345709ecc78a3ecfd796a57236bfad0b9b8", size = 250707, upload-time = "2026-03-17T10:30:35.2Z" }, + { url = "https://files.pythonhosted.org/packages/67/21/9ac389377380a07884e3b48ba7a620fcd9dbfaf1d40565facdc6b36ec9ef/coverage-7.13.5-cp311-cp311-win32.whl", hash = "sha256:259b69bb83ad9894c4b25be2528139eecba9a82646ebdda2d9db1ba28424a6bf", size = 221880, upload-time = "2026-03-17T10:30:36.775Z" }, + { url = "https://files.pythonhosted.org/packages/af/7f/4cd8a92531253f9d7c1bbecd9fa1b472907fb54446ca768c59b531248dc5/coverage-7.13.5-cp311-cp311-win_amd64.whl", hash = "sha256:258354455f4e86e3e9d0d17571d522e13b4e1e19bf0f8596bcf9476d61e7d8a9", size = 222816, upload-time = "2026-03-17T10:30:38.891Z" }, + { url = "https://files.pythonhosted.org/packages/12/a6/1d3f6155fb0010ca68eba7fe48ca6c9da7385058b77a95848710ecf189b1/coverage-7.13.5-cp311-cp311-win_arm64.whl", hash = "sha256:bff95879c33ec8da99fc9b6fe345ddb5be6414b41d6d1ad1c8f188d26f36e028", size = 221483, upload-time = "2026-03-17T10:30:40.463Z" }, + { url = "https://files.pythonhosted.org/packages/a0/c3/a396306ba7db865bf96fc1fb3b7fd29bcbf3d829df642e77b13555163cd6/coverage-7.13.5-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:460cf0114c5016fa841214ff5564aa4864f11948da9440bc97e21ad1f4ba1e01", size = 219554, upload-time = "2026-03-17T10:30:42.208Z" }, + { url = "https://files.pythonhosted.org/packages/a6/16/a68a19e5384e93f811dccc51034b1fd0b865841c390e3c931dcc4699e035/coverage-7.13.5-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:0e223ce4b4ed47f065bfb123687686512e37629be25cc63728557ae7db261422", size = 219908, upload-time = "2026-03-17T10:30:43.906Z" }, + { url = "https://files.pythonhosted.org/packages/29/72/20b917c6793af3a5ceb7fb9c50033f3ec7865f2911a1416b34a7cfa0813b/coverage-7.13.5-cp312-cp312-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:6e3370441f4513c6252bf042b9c36d22491142385049243253c7e48398a15a9f", size = 251419, upload-time = "2026-03-17T10:30:45.545Z" }, + { url = "https://files.pythonhosted.org/packages/8c/49/cd14b789536ac6a4778c453c6a2338bc0a2fb60c5a5a41b4008328b9acc1/coverage-7.13.5-cp312-cp312-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:03ccc709a17a1de074fb1d11f217342fb0d2b1582ed544f554fc9fc3f07e95f5", size = 254159, upload-time = "2026-03-17T10:30:47.204Z" }, + { url = "https://files.pythonhosted.org/packages/9d/00/7b0edcfe64e2ed4c0340dac14a52ad0f4c9bd0b8b5e531af7d55b703db7c/coverage-7.13.5-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3f4818d065964db3c1c66dc0fbdac5ac692ecbc875555e13374fdbe7eedb4376", size = 255270, upload-time = "2026-03-17T10:30:48.812Z" }, + { url = "https://files.pythonhosted.org/packages/93/89/7ffc4ba0f5d0a55c1e84ea7cee39c9fc06af7b170513d83fbf3bbefce280/coverage-7.13.5-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:012d5319e66e9d5a218834642d6c35d265515a62f01157a45bcc036ecf947256", size = 257538, upload-time = "2026-03-17T10:30:50.77Z" }, + { url = "https://files.pythonhosted.org/packages/81/bd/73ddf85f93f7e6fa83e77ccecb6162d9415c79007b4bc124008a4995e4a7/coverage-7.13.5-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:8dd02af98971bdb956363e4827d34425cb3df19ee550ef92855b0acb9c7ce51c", size = 251821, upload-time = "2026-03-17T10:30:52.5Z" }, + { url = "https://files.pythonhosted.org/packages/a0/81/278aff4e8dec4926a0bcb9486320752811f543a3ce5b602cc7a29978d073/coverage-7.13.5-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:f08fd75c50a760c7eb068ae823777268daaf16a80b918fa58eea888f8e3919f5", size = 253191, upload-time = "2026-03-17T10:30:54.543Z" }, + { url = "https://files.pythonhosted.org/packages/70/ee/fe1621488e2e0a58d7e94c4800f0d96f79671553488d401a612bebae324b/coverage-7.13.5-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:843ea8643cf967d1ac7e8ecd4bb00c99135adf4816c0c0593fdcc47b597fcf09", size = 251337, upload-time = "2026-03-17T10:30:56.663Z" }, + { url = "https://files.pythonhosted.org/packages/37/a6/f79fb37aa104b562207cc23cb5711ab6793608e246cae1e93f26b2236ed9/coverage-7.13.5-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:9d44d7aa963820b1b971dbecd90bfe5fe8f81cff79787eb6cca15750bd2f79b9", size = 255404, upload-time = "2026-03-17T10:30:58.427Z" }, + { url = "https://files.pythonhosted.org/packages/75/f0/ed15262a58ec81ce457ceb717b7f78752a1713556b19081b76e90896e8d4/coverage-7.13.5-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:7132bed4bd7b836200c591410ae7d97bf7ae8be6fc87d160b2bd881df929e7bf", size = 250903, upload-time = "2026-03-17T10:31:00.093Z" }, + { url = "https://files.pythonhosted.org/packages/0f/e9/9129958f20e7e9d4d56d51d42ccf708d15cac355ff4ac6e736e97a9393d2/coverage-7.13.5-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:a698e363641b98843c517817db75373c83254781426e94ada3197cabbc2c919c", size = 252780, upload-time = "2026-03-17T10:31:01.916Z" }, + { url = "https://files.pythonhosted.org/packages/a4/d7/0ad9b15812d81272db94379fe4c6df8fd17781cc7671fdfa30c76ba5ff7b/coverage-7.13.5-cp312-cp312-win32.whl", hash = "sha256:bdba0a6b8812e8c7df002d908a9a2ea3c36e92611b5708633c50869e6d922fdf", size = 222093, upload-time = "2026-03-17T10:31:03.642Z" }, + { url = "https://files.pythonhosted.org/packages/29/3d/821a9a5799fac2556bcf0bd37a70d1d11fa9e49784b6d22e92e8b2f85f18/coverage-7.13.5-cp312-cp312-win_amd64.whl", hash = "sha256:d2c87e0c473a10bffe991502eac389220533024c8082ec1ce849f4218dded810", size = 222900, upload-time = "2026-03-17T10:31:05.651Z" }, + { url = "https://files.pythonhosted.org/packages/d4/fa/2238c2ad08e35cf4f020ea721f717e09ec3152aea75d191a7faf3ef009a8/coverage-7.13.5-cp312-cp312-win_arm64.whl", hash = "sha256:bf69236a9a81bdca3bff53796237aab096cdbf8d78a66ad61e992d9dac7eb2de", size = 221515, upload-time = "2026-03-17T10:31:07.293Z" }, + { url = "https://files.pythonhosted.org/packages/74/8c/74fedc9663dcf168b0a059d4ea756ecae4da77a489048f94b5f512a8d0b3/coverage-7.13.5-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:5ec4af212df513e399cf11610cc27063f1586419e814755ab362e50a85ea69c1", size = 219576, upload-time = "2026-03-17T10:31:09.045Z" }, + { url = "https://files.pythonhosted.org/packages/0c/c9/44fb661c55062f0818a6ffd2685c67aa30816200d5f2817543717d4b92eb/coverage-7.13.5-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:941617e518602e2d64942c88ec8499f7fbd49d3f6c4327d3a71d43a1973032f3", size = 219942, upload-time = "2026-03-17T10:31:10.708Z" }, + { url = "https://files.pythonhosted.org/packages/5f/13/93419671cee82b780bab7ea96b67c8ef448f5f295f36bf5031154ec9a790/coverage-7.13.5-cp313-cp313-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:da305e9937617ee95c2e39d8ff9f040e0487cbf1ac174f777ed5eddd7a7c1f26", size = 250935, upload-time = "2026-03-17T10:31:12.392Z" }, + { url = "https://files.pythonhosted.org/packages/ac/68/1666e3a4462f8202d836920114fa7a5ee9275d1fa45366d336c551a162dd/coverage-7.13.5-cp313-cp313-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:78e696e1cc714e57e8b25760b33a8b1026b7048d270140d25dafe1b0a1ee05a3", size = 253541, upload-time = "2026-03-17T10:31:14.247Z" }, + { url = "https://files.pythonhosted.org/packages/4e/5e/3ee3b835647be646dcf3c65a7c6c18f87c27326a858f72ab22c12730773d/coverage-7.13.5-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:02ca0eed225b2ff301c474aeeeae27d26e2537942aa0f87491d3e147e784a82b", size = 254780, upload-time = "2026-03-17T10:31:16.193Z" }, + { url = "https://files.pythonhosted.org/packages/44/b3/cb5bd1a04cfcc49ede6cd8409d80bee17661167686741e041abc7ee1b9a9/coverage-7.13.5-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:04690832cbea4e4663d9149e05dba142546ca05cb1848816760e7f58285c970a", size = 256912, upload-time = "2026-03-17T10:31:17.89Z" }, + { url = "https://files.pythonhosted.org/packages/1b/66/c1dceb7b9714473800b075f5c8a84f4588f887a90eb8645282031676e242/coverage-7.13.5-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:0590e44dd2745c696a778f7bab6aa95256de2cbc8b8cff4f7db8ff09813d6969", size = 251165, upload-time = "2026-03-17T10:31:19.605Z" }, + { url = "https://files.pythonhosted.org/packages/b7/62/5502b73b97aa2e53ea22a39cf8649ff44827bef76d90bf638777daa27a9d/coverage-7.13.5-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:d7cfad2d6d81dd298ab6b89fe72c3b7b05ec7544bdda3b707ddaecff8d25c161", size = 252908, upload-time = "2026-03-17T10:31:21.312Z" }, + { url = "https://files.pythonhosted.org/packages/7d/37/7792c2d69854397ca77a55c4646e5897c467928b0e27f2d235d83b5d08c6/coverage-7.13.5-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:e092b9499de38ae0fbfbc603a74660eb6ff3e869e507b50d85a13b6db9863e15", size = 250873, upload-time = "2026-03-17T10:31:23.565Z" }, + { url = "https://files.pythonhosted.org/packages/a3/23/bc866fb6163be52a8a9e5d708ba0d3b1283c12158cefca0a8bbb6e247a43/coverage-7.13.5-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:48c39bc4a04d983a54a705a6389512883d4a3b9862991b3617d547940e9f52b1", size = 255030, upload-time = "2026-03-17T10:31:25.58Z" }, + { url = "https://files.pythonhosted.org/packages/7d/8b/ef67e1c222ef49860701d346b8bbb70881bef283bd5f6cbba68a39a086c7/coverage-7.13.5-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:2d3807015f138ffea1ed9afeeb8624fd781703f2858b62a8dd8da5a0994c57b6", size = 250694, upload-time = "2026-03-17T10:31:27.316Z" }, + { url = "https://files.pythonhosted.org/packages/46/0d/866d1f74f0acddbb906db212e096dee77a8e2158ca5e6bb44729f9d93298/coverage-7.13.5-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:ee2aa19e03161671ec964004fb74b2257805d9710bf14a5c704558b9d8dbaf17", size = 252469, upload-time = "2026-03-17T10:31:29.472Z" }, + { url = "https://files.pythonhosted.org/packages/7a/f5/be742fec31118f02ce42b21c6af187ad6a344fed546b56ca60caacc6a9a0/coverage-7.13.5-cp313-cp313-win32.whl", hash = "sha256:ce1998c0483007608c8382f4ff50164bfc5bd07a2246dd272aa4043b75e61e85", size = 222112, upload-time = "2026-03-17T10:31:31.526Z" }, + { url = "https://files.pythonhosted.org/packages/66/40/7732d648ab9d069a46e686043241f01206348e2bbf128daea85be4d6414b/coverage-7.13.5-cp313-cp313-win_amd64.whl", hash = "sha256:631efb83f01569670a5e866ceb80fe483e7c159fac6f167e6571522636104a0b", size = 222923, upload-time = "2026-03-17T10:31:33.633Z" }, + { url = "https://files.pythonhosted.org/packages/48/af/fea819c12a095781f6ccd504890aaddaf88b8fab263c4940e82c7b770124/coverage-7.13.5-cp313-cp313-win_arm64.whl", hash = "sha256:f4cd16206ad171cbc2470dbea9103cf9a7607d5fe8c242fdf1edf36174020664", size = 221540, upload-time = "2026-03-17T10:31:35.445Z" }, + { url = "https://files.pythonhosted.org/packages/23/d2/17879af479df7fbbd44bd528a31692a48f6b25055d16482fdf5cdb633805/coverage-7.13.5-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:0428cbef5783ad91fe240f673cc1f76b25e74bbfe1a13115e4aa30d3f538162d", size = 220262, upload-time = "2026-03-17T10:31:37.184Z" }, + { url = "https://files.pythonhosted.org/packages/5b/4c/d20e554f988c8f91d6a02c5118f9abbbf73a8768a3048cb4962230d5743f/coverage-7.13.5-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:e0b216a19534b2427cc201a26c25da4a48633f29a487c61258643e89d28200c0", size = 220617, upload-time = "2026-03-17T10:31:39.245Z" }, + { url = "https://files.pythonhosted.org/packages/29/9c/f9f5277b95184f764b24e7231e166dfdb5780a46d408a2ac665969416d61/coverage-7.13.5-cp313-cp313t-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:972a9cd27894afe4bc2b1480107054e062df08e671df7c2f18c205e805ccd806", size = 261912, upload-time = "2026-03-17T10:31:41.324Z" }, + { url = "https://files.pythonhosted.org/packages/d5/f6/7f1ab39393eeb50cfe4747ae8ef0e4fc564b989225aa1152e13a180d74f8/coverage-7.13.5-cp313-cp313t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:4b59148601efcd2bac8c4dbf1f0ad6391693ccf7a74b8205781751637076aee3", size = 263987, upload-time = "2026-03-17T10:31:43.724Z" }, + { url = "https://files.pythonhosted.org/packages/a0/d7/62c084fb489ed9c6fbdf57e006752e7c516ea46fd690e5ed8b8617c7d52e/coverage-7.13.5-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:505d7083c8b0c87a8fa8c07370c285847c1f77739b22e299ad75a6af6c32c5c9", size = 266416, upload-time = "2026-03-17T10:31:45.769Z" }, + { url = "https://files.pythonhosted.org/packages/a9/f6/df63d8660e1a0bff6125947afda112a0502736f470d62ca68b288ea762d8/coverage-7.13.5-cp313-cp313t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:60365289c3741e4db327e7baff2a4aaacf22f788e80fa4683393891b70a89fbd", size = 267558, upload-time = "2026-03-17T10:31:48.293Z" }, + { url = "https://files.pythonhosted.org/packages/5b/02/353ca81d36779bd108f6d384425f7139ac3c58c750dcfaafe5d0bee6436b/coverage-7.13.5-cp313-cp313t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:1b88c69c8ef5d4b6fe7dea66d6636056a0f6a7527c440e890cf9259011f5e606", size = 261163, upload-time = "2026-03-17T10:31:50.125Z" }, + { url = "https://files.pythonhosted.org/packages/2c/16/2e79106d5749bcaf3aee6d309123548e3276517cd7851faa8da213bc61bf/coverage-7.13.5-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:5b13955d31d1633cf9376908089b7cebe7d15ddad7aeaabcbe969a595a97e95e", size = 263981, upload-time = "2026-03-17T10:31:51.961Z" }, + { url = "https://files.pythonhosted.org/packages/29/c7/c29e0c59ffa6942030ae6f50b88ae49988e7e8da06de7ecdbf49c6d4feae/coverage-7.13.5-cp313-cp313t-musllinux_1_2_i686.whl", hash = "sha256:f70c9ab2595c56f81a89620e22899eea8b212a4041bd728ac6f4a28bf5d3ddd0", size = 261604, upload-time = "2026-03-17T10:31:53.872Z" }, + { url = "https://files.pythonhosted.org/packages/40/48/097cdc3db342f34006a308ab41c3a7c11c3f0d84750d340f45d88a782e00/coverage-7.13.5-cp313-cp313t-musllinux_1_2_ppc64le.whl", hash = "sha256:084b84a8c63e8d6fc7e3931b316a9bcafca1458d753c539db82d31ed20091a87", size = 265321, upload-time = "2026-03-17T10:31:55.997Z" }, + { url = "https://files.pythonhosted.org/packages/bb/1f/4994af354689e14fd03a75f8ec85a9a68d94e0188bbdab3fc1516b55e512/coverage-7.13.5-cp313-cp313t-musllinux_1_2_riscv64.whl", hash = "sha256:ad14385487393e386e2ea988b09d62dd42c397662ac2dabc3832d71253eee479", size = 260502, upload-time = "2026-03-17T10:31:58.308Z" }, + { url = "https://files.pythonhosted.org/packages/22/c6/9bb9ef55903e628033560885f5c31aa227e46878118b63ab15dc7ba87797/coverage-7.13.5-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:7f2c47b36fe7709a6e83bfadf4eefb90bd25fbe4014d715224c4316f808e59a2", size = 262688, upload-time = "2026-03-17T10:32:00.141Z" }, + { url = "https://files.pythonhosted.org/packages/14/4f/f5df9007e50b15e53e01edea486814783a7f019893733d9e4d6caad75557/coverage-7.13.5-cp313-cp313t-win32.whl", hash = "sha256:67e9bc5449801fad0e5dff329499fb090ba4c5800b86805c80617b4e29809b2a", size = 222788, upload-time = "2026-03-17T10:32:02.246Z" }, + { url = "https://files.pythonhosted.org/packages/e1/98/aa7fccaa97d0f3192bec013c4e6fd6d294a6ed44b640e6bb61f479e00ed5/coverage-7.13.5-cp313-cp313t-win_amd64.whl", hash = "sha256:da86cdcf10d2519e10cabb8ac2de03da1bcb6e4853790b7fbd48523332e3a819", size = 223851, upload-time = "2026-03-17T10:32:04.416Z" }, + { url = "https://files.pythonhosted.org/packages/3d/8b/e5c469f7352651e5f013198e9e21f97510b23de957dd06a84071683b4b60/coverage-7.13.5-cp313-cp313t-win_arm64.whl", hash = "sha256:0ecf12ecb326fe2c339d93fc131816f3a7367d223db37817208905c89bded911", size = 222104, upload-time = "2026-03-17T10:32:06.65Z" }, + { url = "https://files.pythonhosted.org/packages/8e/77/39703f0d1d4b478bfd30191d3c14f53caf596fac00efb3f8f6ee23646439/coverage-7.13.5-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:fbabfaceaeb587e16f7008f7795cd80d20ec548dc7f94fbb0d4ec2e038ce563f", size = 219621, upload-time = "2026-03-17T10:32:08.589Z" }, + { url = "https://files.pythonhosted.org/packages/e2/3e/51dff36d99ae14639a133d9b164d63e628532e2974d8b1edb99dd1ebc733/coverage-7.13.5-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:9bb2a28101a443669a423b665939381084412b81c3f8c0fcfbac57f4e30b5b8e", size = 219953, upload-time = "2026-03-17T10:32:10.507Z" }, + { url = "https://files.pythonhosted.org/packages/6a/6c/1f1917b01eb647c2f2adc9962bd66c79eb978951cab61bdc1acab3290c07/coverage-7.13.5-cp314-cp314-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:bd3a2fbc1c6cccb3c5106140d87cc6a8715110373ef42b63cf5aea29df8c217a", size = 250992, upload-time = "2026-03-17T10:32:12.41Z" }, + { url = "https://files.pythonhosted.org/packages/22/e5/06b1f88f42a5a99df42ce61208bdec3bddb3d261412874280a19796fc09c/coverage-7.13.5-cp314-cp314-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:6c36ddb64ed9d7e496028d1d00dfec3e428e0aabf4006583bb1839958d280510", size = 253503, upload-time = "2026-03-17T10:32:14.449Z" }, + { url = "https://files.pythonhosted.org/packages/80/28/2a148a51e5907e504fa7b85490277734e6771d8844ebcc48764a15e28155/coverage-7.13.5-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:380e8e9084d8eb38db3a9176a1a4f3c0082c3806fa0dc882d1d87abc3c789247", size = 254852, upload-time = "2026-03-17T10:32:16.56Z" }, + { url = "https://files.pythonhosted.org/packages/61/77/50e8d3d85cc0b7ebe09f30f151d670e302c7ff4a1bf6243f71dd8b0981fa/coverage-7.13.5-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:e808af52a0513762df4d945ea164a24b37f2f518cbe97e03deaa0ee66139b4d6", size = 257161, upload-time = "2026-03-17T10:32:19.004Z" }, + { url = "https://files.pythonhosted.org/packages/3b/c4/b5fd1d4b7bf8d0e75d997afd3925c59ba629fc8616f1b3aae7605132e256/coverage-7.13.5-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:e301d30dd7e95ae068671d746ba8c34e945a82682e62918e41b2679acd2051a0", size = 251021, upload-time = "2026-03-17T10:32:21.344Z" }, + { url = "https://files.pythonhosted.org/packages/f8/66/6ea21f910e92d69ef0b1c3346ea5922a51bad4446c9126db2ae96ee24c4c/coverage-7.13.5-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:800bc829053c80d240a687ceeb927a94fd108bbdc68dfbe505d0d75ab578a882", size = 252858, upload-time = "2026-03-17T10:32:23.506Z" }, + { url = "https://files.pythonhosted.org/packages/9e/ea/879c83cb5d61aa2a35fb80e72715e92672daef8191b84911a643f533840c/coverage-7.13.5-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:0b67af5492adb31940ee418a5a655c28e48165da5afab8c7fa6fd72a142f8740", size = 250823, upload-time = "2026-03-17T10:32:25.516Z" }, + { url = "https://files.pythonhosted.org/packages/8a/fb/616d95d3adb88b9803b275580bdeee8bd1b69a886d057652521f83d7322f/coverage-7.13.5-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:c9136ff29c3a91e25b1d1552b5308e53a1e0653a23e53b6366d7c2dcbbaf8a16", size = 255099, upload-time = "2026-03-17T10:32:27.944Z" }, + { url = "https://files.pythonhosted.org/packages/1c/93/25e6917c90ec1c9a56b0b26f6cad6408e5f13bb6b35d484a0d75c9cf000d/coverage-7.13.5-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:cff784eef7f0b8f6cb28804fbddcfa99f89efe4cc35fb5627e3ac58f91ed3ac0", size = 250638, upload-time = "2026-03-17T10:32:29.914Z" }, + { url = "https://files.pythonhosted.org/packages/fc/7b/dc1776b0464145a929deed214aef9fb1493f159b59ff3c7eeeedf91eddd0/coverage-7.13.5-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:68a4953be99b17ac3c23b6efbc8a38330d99680c9458927491d18700ef23ded0", size = 252295, upload-time = "2026-03-17T10:32:31.981Z" }, + { url = "https://files.pythonhosted.org/packages/ea/fb/99cbbc56a26e07762a2740713f3c8f9f3f3106e3a3dd8cc4474954bccd34/coverage-7.13.5-cp314-cp314-win32.whl", hash = "sha256:35a31f2b1578185fbe6aa2e74cea1b1d0bbf4c552774247d9160d29b80ed56cc", size = 222360, upload-time = "2026-03-17T10:32:34.233Z" }, + { url = "https://files.pythonhosted.org/packages/8d/b7/4758d4f73fb536347cc5e4ad63662f9d60ba9118cb6785e9616b2ce5d7fa/coverage-7.13.5-cp314-cp314-win_amd64.whl", hash = "sha256:2aa055ae1857258f9e0045be26a6d62bdb47a72448b62d7b55f4820f361a2633", size = 223174, upload-time = "2026-03-17T10:32:36.369Z" }, + { url = "https://files.pythonhosted.org/packages/2c/f2/24d84e1dfe70f8ac9fdf30d338239860d0d1d5da0bda528959d0ebc9da28/coverage-7.13.5-cp314-cp314-win_arm64.whl", hash = "sha256:1b11eef33edeae9d142f9b4358edb76273b3bfd30bc3df9a4f95d0e49caf94e8", size = 221739, upload-time = "2026-03-17T10:32:38.736Z" }, + { url = "https://files.pythonhosted.org/packages/60/5b/4a168591057b3668c2428bff25dd3ebc21b629d666d90bcdfa0217940e84/coverage-7.13.5-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:10a0c37f0b646eaff7cce1874c31d1f1ccb297688d4c747291f4f4c70741cc8b", size = 220351, upload-time = "2026-03-17T10:32:41.196Z" }, + { url = "https://files.pythonhosted.org/packages/f5/21/1fd5c4dbfe4a58b6b99649125635df46decdfd4a784c3cd6d410d303e370/coverage-7.13.5-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:b5db73ba3c41c7008037fa731ad5459fc3944cb7452fc0aa9f822ad3533c583c", size = 220612, upload-time = "2026-03-17T10:32:43.204Z" }, + { url = "https://files.pythonhosted.org/packages/d6/fe/2a924b3055a5e7e4512655a9d4609781b0d62334fa0140c3e742926834e2/coverage-7.13.5-cp314-cp314t-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:750db93a81e3e5a9831b534be7b1229df848b2e125a604fe6651e48aa070e5f9", size = 261985, upload-time = "2026-03-17T10:32:45.514Z" }, + { url = "https://files.pythonhosted.org/packages/d7/0d/c8928f2bd518c45990fe1a2ab8db42e914ef9b726c975facc4282578c3eb/coverage-7.13.5-cp314-cp314t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:9ddb4f4a5479f2539644be484da179b653273bca1a323947d48ab107b3ed1f29", size = 264107, upload-time = "2026-03-17T10:32:47.971Z" }, + { url = "https://files.pythonhosted.org/packages/ef/ae/4ae35bbd9a0af9d820362751f0766582833c211224b38665c0f8de3d487f/coverage-7.13.5-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d8a7a2049c14f413163e2bdabd37e41179b1d1ccb10ffc6ccc4b7a718429c607", size = 266513, upload-time = "2026-03-17T10:32:50.1Z" }, + { url = "https://files.pythonhosted.org/packages/9c/20/d326174c55af36f74eac6ae781612d9492f060ce8244b570bb9d50d9d609/coverage-7.13.5-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:e1c85e0b6c05c592ea6d8768a66a254bfb3874b53774b12d4c89c481eb78cb90", size = 267650, upload-time = "2026-03-17T10:32:52.391Z" }, + { url = "https://files.pythonhosted.org/packages/7a/5e/31484d62cbd0eabd3412e30d74386ece4a0837d4f6c3040a653878bfc019/coverage-7.13.5-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:777c4d1eff1b67876139d24288aaf1817f6c03d6bae9c5cc8d27b83bcfe38fe3", size = 261089, upload-time = "2026-03-17T10:32:54.544Z" }, + { url = "https://files.pythonhosted.org/packages/e9/d8/49a72d6de146eebb0b7e48cc0f4bc2c0dd858e3d4790ab2b39a2872b62bd/coverage-7.13.5-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:6697e29b93707167687543480a40f0db8f356e86d9f67ddf2e37e2dfd91a9dab", size = 263982, upload-time = "2026-03-17T10:32:56.803Z" }, + { url = "https://files.pythonhosted.org/packages/06/3b/0351f1bd566e6e4dd39e978efe7958bde1d32f879e85589de147654f57bb/coverage-7.13.5-cp314-cp314t-musllinux_1_2_i686.whl", hash = "sha256:8fdf453a942c3e4d99bd80088141c4c6960bb232c409d9c3558e2dbaa3998562", size = 261579, upload-time = "2026-03-17T10:32:59.466Z" }, + { url = "https://files.pythonhosted.org/packages/5d/ce/796a2a2f4017f554d7810f5c573449b35b1e46788424a548d4d19201b222/coverage-7.13.5-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:32ca0c0114c9834a43f045a87dcebd69d108d8ffb666957ea65aa132f50332e2", size = 265316, upload-time = "2026-03-17T10:33:01.847Z" }, + { url = "https://files.pythonhosted.org/packages/3d/16/d5ae91455541d1a78bc90abf495be600588aff8f6db5c8b0dae739fa39c9/coverage-7.13.5-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:8769751c10f339021e2638cd354e13adeac54004d1941119b2c96fe5276d45ea", size = 260427, upload-time = "2026-03-17T10:33:03.945Z" }, + { url = "https://files.pythonhosted.org/packages/48/11/07f413dba62db21fb3fad5d0de013a50e073cc4e2dc4306e770360f6dfc8/coverage-7.13.5-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:cec2d83125531bd153175354055cdb7a09987af08a9430bd173c937c6d0fba2a", size = 262745, upload-time = "2026-03-17T10:33:06.285Z" }, + { url = "https://files.pythonhosted.org/packages/91/15/d792371332eb4663115becf4bad47e047d16234b1aff687b1b18c58d60ae/coverage-7.13.5-cp314-cp314t-win32.whl", hash = "sha256:0cd9ed7a8b181775459296e402ca4fb27db1279740a24e93b3b41942ebe4b215", size = 223146, upload-time = "2026-03-17T10:33:08.756Z" }, + { url = "https://files.pythonhosted.org/packages/db/51/37221f59a111dca5e85be7dbf09696323b5b9f13ff65e0641d535ed06ea8/coverage-7.13.5-cp314-cp314t-win_amd64.whl", hash = "sha256:301e3b7dfefecaca37c9f1aa6f0049b7d4ab8dd933742b607765d757aca77d43", size = 224254, upload-time = "2026-03-17T10:33:11.174Z" }, + { url = "https://files.pythonhosted.org/packages/54/83/6acacc889de8987441aa7d5adfbdbf33d288dad28704a67e574f1df9bcbb/coverage-7.13.5-cp314-cp314t-win_arm64.whl", hash = "sha256:9dacc2ad679b292709e0f5fc1ac74a6d4d5562e424058962c7bb0c658ad25e45", size = 222276, upload-time = "2026-03-17T10:33:13.466Z" }, + { url = "https://files.pythonhosted.org/packages/9e/ee/a4cf96b8ce1e566ed238f0659ac2d3f007ed1d14b181bcb684e19561a69a/coverage-7.13.5-py3-none-any.whl", hash = "sha256:34b02417cf070e173989b3db962f7ed56d2f644307b2cf9d5a0f258e13084a61", size = 211346, upload-time = "2026-03-17T10:33:15.691Z" }, +] + +[package.optional-dependencies] +toml = [ + { name = "tomli", marker = "python_full_version <= '3.11'" }, +] + [[package]] name = "cryptography" version = "46.0.7" @@ -799,6 +917,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/81/db/e655086b7f3a705df045bf0933bdd9c2f79bb3c97bfef1384598bb79a217/keyring-25.7.0-py3-none-any.whl", hash = "sha256:be4a0b195f149690c166e850609a477c532ddbfbaed96a404d4e43f8d5e2689f", size = 39160, upload-time = "2025-11-16T16:26:08.402Z" }, ] +[[package]] +name = "lazy-loader" +version = "0.5" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "packaging" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/49/ac/21a1f8aa3777f5658576777ea76bfb124b702c520bbe90edf4ae9915eafa/lazy_loader-0.5.tar.gz", hash = "sha256:717f9179a0dbed357012ddad50a5ad3d5e4d9a0b8712680d4e687f5e6e6ed9b3", size = 15294, upload-time = "2026-03-06T15:45:09.054Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/8a/a1/8d812e53a5da1687abb10445275d41a8b13adb781bbf7196ddbcf8d88505/lazy_loader-0.5-py3-none-any.whl", hash = "sha256:ab0ea149e9c554d4ffeeb21105ac60bed7f3b4fd69b1d2360a4add51b170b005", size = 8044, upload-time = "2026-03-06T15:45:07.668Z" }, +] + [[package]] name = "markdown-it-py" version = "4.0.0" @@ -879,6 +1009,74 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/cf/df/d3f1ddf4bb4cb50ed9b1139cc7b1c54c34a1e7ce8fd1b9a37c0d1551a6bd/opentelemetry_api-1.39.1-py3-none-any.whl", hash = "sha256:2edd8463432a7f8443edce90972169b195e7d6a05500cd29e6d13898187c9950", size = 66356, upload-time = "2025-12-11T13:32:17.304Z" }, ] +[[package]] +name = "orjson" +version = "3.11.9" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/7e/0c/964746fcafbd16f8ff53219ad9f6b412b34f345c75f384ad434ceaadb538/orjson-3.11.9.tar.gz", hash = "sha256:4fef17e1f8722c11587a6ef18e35902450221da0028e65dbaaa543619e68e48f", size = 5599163, upload-time = "2026-05-06T15:11:08.309Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/1e/51/3fb9e65ae76ee97bd611869a503fa3fc0a6e81dd8b737cf3003f682df7ff/orjson-3.11.9-cp311-cp311-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:f01c4818b3fc9b0da8e096722a84318071eaa118df35f6ed2344da0e73a5444f", size = 228522, upload-time = "2026-05-06T15:09:35.362Z" }, + { url = "https://files.pythonhosted.org/packages/16/fa/9d54b07cb3f3b0bfd57841478e42d7a0ece4a9f49f9907eecf5a45461687/orjson-3.11.9-cp311-cp311-macosx_15_0_arm64.whl", hash = "sha256:3ebca4179031ee716ed076ffadc29428e900512f6fccee8614c9983157fcf19c", size = 128463, upload-time = "2026-05-06T15:09:37.063Z" }, + { url = "https://files.pythonhosted.org/packages/88/b1/6ceafc2eefd0a553e3be77ce6c49d107e772485d9568629376171c50e634/orjson-3.11.9-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:48ee05097750de0ff69ed5b7bbcf0732182fd57a24043dcc2a1da780a5ead3a5", size = 132306, upload-time = "2026-05-06T15:09:38.299Z" }, + { url = "https://files.pythonhosted.org/packages/ea/76/f11311285324a40aab1e3031385c50b635a7cd0734fdaf60c7e89a696f60/orjson-3.11.9-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:a6082706765a95a6680d812e1daf1c0cfe8adec7831b3ff3b625693f3b461b1c", size = 127988, upload-time = "2026-05-06T15:09:39.597Z" }, + { url = "https://files.pythonhosted.org/packages/9e/85/0ef63bcf1337f44031ce9b91b1919563f62a37527b3ea4368bb15a22e5d7/orjson-3.11.9-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:277fefe9d76ee17eb14debf399e3533d4d63b5f677a4d3719eb763536af1f4bd", size = 135188, upload-time = "2026-05-06T15:09:40.957Z" }, + { url = "https://files.pythonhosted.org/packages/05/94/b0d27090ea8a2095db3c2bd1b1c96f96f19bbb494d7fef33130e846e613d/orjson-3.11.9-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:03db380e3780fa0015ed776a90f20e8e20bb11dde13b216ce19e5718e3dfba62", size = 145937, upload-time = "2026-05-06T15:09:42.249Z" }, + { url = "https://files.pythonhosted.org/packages/09/eb/75d50c29c05b8054013e221e598820a365c8e64065312e75e202ed880709/orjson-3.11.9-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:33d7d766701847dc6729846362dc27895d2f2d2251264f9d10e7cb9878194877", size = 132758, upload-time = "2026-05-06T15:09:43.945Z" }, + { url = "https://files.pythonhosted.org/packages/49/bd/360686f39348aa88827cb6fbf7dc606fd41c831a35235e1abf1db8e3a9e6/orjson-3.11.9-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:147302878da387104b66bb4a8b0227d1d487e976ce41a8501916161072ed87b1", size = 133971, upload-time = "2026-05-06T15:09:45.239Z" }, + { url = "https://files.pythonhosted.org/packages/0e/30/3178eb16f3221aeef068b6f1f1ebe05f656ea5c6dffe9f6c917329fe17a3/orjson-3.11.9-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:3513550321f8c8c811a7c3297b8a630e82dc08e4c10216d07703c997776236cd", size = 141685, upload-time = "2026-05-06T15:09:46.858Z" }, + { url = "https://files.pythonhosted.org/packages/5f/f1/ff2f19ed0225f9680fafa42febca3570dd59444ebf190980738d376214c2/orjson-3.11.9-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:c5d001196b89fa9cf0a4ab79766cd835b991a166e4b621ba95089edc50c429ff", size = 415167, upload-time = "2026-05-06T15:09:48.312Z" }, + { url = "https://files.pythonhosted.org/packages/9b/61/863bddf0da6e9e586765414debd54b4e58db05f560902b6d00658cb88636/orjson-3.11.9-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:16969c9d369c98eb084889c6e4d2d39b77c7eb38ceccf8da2a9fff62ae908980", size = 147913, upload-time = "2026-05-06T15:09:49.733Z" }, + { url = "https://files.pythonhosted.org/packages/b6/8a/4081492586d75b073d60c5271a8d0f05a0955cabf1e34c8473f6fcd84235/orjson-3.11.9-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:63e0efbc991250c0b3143488fa57d95affcabbfc63c99c48d625dd37779aafe2", size = 136959, upload-time = "2026-05-06T15:09:51.311Z" }, + { url = "https://files.pythonhosted.org/packages/0d/bd/70b6ab193594d7abb875320c0a7c8335e846f28968c432c31042409c3c8d/orjson-3.11.9-cp311-cp311-win32.whl", hash = "sha256:14ed654580c1ed2bc217352ec82f91b047aef82951aa71c7f64e0dcb03c0e180", size = 131533, upload-time = "2026-05-06T15:09:52.637Z" }, + { url = "https://files.pythonhosted.org/packages/3f/17/1a1a228183d62d1b77e2c30d210f47dd4768b310ebe1607c63e3c0e3a71e/orjson-3.11.9-cp311-cp311-win_amd64.whl", hash = "sha256:57ea77fb70a448ce87d18fca050193202a3da5e54598f6501ca5476fb66cfe02", size = 127106, upload-time = "2026-05-06T15:09:54.204Z" }, + { url = "https://files.pythonhosted.org/packages/b8/95/285de5fa296d09681ee9c546cd4a8aeb773b701cf343dc125994f4d52953/orjson-3.11.9-cp311-cp311-win_arm64.whl", hash = "sha256:19b72ed11572a2ee51a67a903afbe5af504f84ed6f529c0fe44b0ab3fb5cc697", size = 126848, upload-time = "2026-05-06T15:09:55.551Z" }, + { url = "https://files.pythonhosted.org/packages/16/6d/11867a3ffa3a3608d84a4de51ef4dd0896d6b5cc9132fbe1daf593e677bc/orjson-3.11.9-cp312-cp312-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:9ef6fe90aadef185c7b128859f40beb24720b4ecea95379fc9000931179c3a49", size = 228515, upload-time = "2026-05-06T15:09:57.265Z" }, + { url = "https://files.pythonhosted.org/packages/24/75/05912954c8b288f34fcf5cd4b9b071cb4f6e77b9961e175e56ebb258089f/orjson-3.11.9-cp312-cp312-macosx_15_0_arm64.whl", hash = "sha256:e5c9b8f28e726e97d97696c826bc7bea5d71cecd63576dba92924a32c1961291", size = 128409, upload-time = "2026-05-06T15:09:59.063Z" }, + { url = "https://files.pythonhosted.org/packages/ab/86/1c3a47df3bc8191ea9ac51603bbb872a95167a364320c269f2557911f406/orjson-3.11.9-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:26a473dbb4162108b27901492546f83c76fdcea3d0eadff00ae7a07e18dcce09", size = 132106, upload-time = "2026-05-06T15:10:00.798Z" }, + { url = "https://files.pythonhosted.org/packages/d7/cf/b33b5f3e695ae7d63feef9d915c37cc3b8f465493dcd4f8e0b4c697a2366/orjson-3.11.9-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:011382e2a60fda9d46f1cdee31068cfc52ffe952b587d683ec0463002802a0f4", size = 127864, upload-time = "2026-05-06T15:10:02.15Z" }, + { url = "https://files.pythonhosted.org/packages/31/6a/6cf69385a58208024fcb8c014e2141b8ce838aba6492b589f8acfff97fab/orjson-3.11.9-cp312-cp312-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c2d3dc759490128c5c1711a53eeaa8ee1d437fd0038ffd2b6008abf46db3f882", size = 135213, upload-time = "2026-05-06T15:10:03.515Z" }, + { url = "https://files.pythonhosted.org/packages/e8/f8/0b1bd3e8f2efcdd376af5c8cfd79eaf13f018080c0089c80ebd724e3c7fb/orjson-3.11.9-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d8ea516b3726d190e1b4297e6f4e7a8650347ae053868a18163b4dd3641d1fff", size = 145994, upload-time = "2026-05-06T15:10:05.083Z" }, + { url = "https://files.pythonhosted.org/packages/f3/59/dab79f61044c529d2c81aecdc589b1f833a1c8dec11ba3b1c2498a02ca7e/orjson-3.11.9-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:380cdce7ba24989af81d0a7013d0aaec5d0e2a21734c0e2681b1bc4f141957fe", size = 132744, upload-time = "2026-05-06T15:10:06.853Z" }, + { url = "https://files.pythonhosted.org/packages/0e/a4/82b7a2fe5d8a67a59ed831b24d59a3d46ea7d207b66e1602d376541d94a6/orjson-3.11.9-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:be4fa4f0af7fa18951f7ab3fc2148e223af211bf03f59e1c6034ec3f97f21d61", size = 134014, upload-time = "2026-05-06T15:10:08.213Z" }, + { url = "https://files.pythonhosted.org/packages/50/c7/375e83a76851b73b2e39f3bcf0e5a19e2b89bad13e5bca97d0b293d27f24/orjson-3.11.9-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:a8f5f8bc7ce7d59f08d9f99fa510c06496164a24cb5f3d34537dbd9ca30132e2", size = 141509, upload-time = "2026-05-06T15:10:09.595Z" }, + { url = "https://files.pythonhosted.org/packages/7f/7c/49d5d82a3d3097f641f094f552131f1e2723b0b8cb0fa2874ab65ecfffa6/orjson-3.11.9-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:4d7fde5501b944f83b3e665e1b31343ff6e154b15560a16b7130ea1e594a4206", size = 415127, upload-time = "2026-05-06T15:10:11.049Z" }, + { url = "https://files.pythonhosted.org/packages/3a/dc/7446c538590d55f455647e5f3c61fc33f7108714e7afcffa6a2a033f8350/orjson-3.11.9-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:cde1a448023ba7d5bb4c01c5afb48894380b5e4956e0627266526587ef4e535f", size = 148025, upload-time = "2026-05-06T15:10:12.842Z" }, + { url = "https://files.pythonhosted.org/packages/df/e5/4d2d8af06f788329b4f78f8cc3679bb395392fcaa1e4d8d3c33e85308fa4/orjson-3.11.9-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:71e63adb0e1f1ed5d9e168f50a91ceb93ae6420731d222dc7da5c69409aa47aa", size = 136943, upload-time = "2026-05-06T15:10:14.405Z" }, + { url = "https://files.pythonhosted.org/packages/06/69/850264ccf6d80f6b174620d30a87f65c9b1490aba33fe6b62798e618cad3/orjson-3.11.9-cp312-cp312-win32.whl", hash = "sha256:2d057a602cdd19a0ad680417527c45b6961a095081c0f46fe0e03e304aac6470", size = 131606, upload-time = "2026-05-06T15:10:15.791Z" }, + { url = "https://files.pythonhosted.org/packages/b9/d5/973a43fc9c55e20f2051e9830997649f669be0cb3ca52192087c0143f118/orjson-3.11.9-cp312-cp312-win_amd64.whl", hash = "sha256:59e403b1cc5a676da8eaf31f6254801b7341b3e29efa85f92b48d272637e77be", size = 127101, upload-time = "2026-05-06T15:10:17.129Z" }, + { url = "https://files.pythonhosted.org/packages/fe/ae/495470f0e4a18f73fa10b7f6b84b464ec4cc5291c4e0c7c2a6c400bef006/orjson-3.11.9-cp312-cp312-win_arm64.whl", hash = "sha256:9af678d6488357948f1f84c6cd1c1d397c014e1ae2f98ae082a44eb48f602624", size = 126736, upload-time = "2026-05-06T15:10:18.645Z" }, + { url = "https://files.pythonhosted.org/packages/32/33/93fcc25907235c344ae73122f8a4e01d2d393ef062b4af7d2e2487a32c37/orjson-3.11.9-cp313-cp313-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:4bab1b2d6141fe7b32ae71dac905666ece4f94936efbfb13d55bb7739a3a6021", size = 228458, upload-time = "2026-05-06T15:10:20.079Z" }, + { url = "https://files.pythonhosted.org/packages/8f/27/b1e6dadb3c080313c03fdd8067b85e6a0460c7d8d6a1c3984ef77b904e4d/orjson-3.11.9-cp313-cp313-macosx_15_0_arm64.whl", hash = "sha256:844417969855fc7a41be124aafe83dc424592a7f77cd4501900c67307122b92c", size = 128368, upload-time = "2026-05-06T15:10:21.549Z" }, + { url = "https://files.pythonhosted.org/packages/21/0f/c9ede0bf052f6b4051e64a7d4fa91b725cccf8321a6a786e86eb03519f00/orjson-3.11.9-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ffe02797b5e9f3a9d8292ddcd289b474ad13e81ad83cd1891a240811f1d2cb81", size = 132070, upload-time = "2026-05-06T15:10:23.371Z" }, + { url = "https://files.pythonhosted.org/packages/fd/26/d398e28048dc18205bbe812f2c88cb9b40313db2470778e25964796458fe/orjson-3.11.9-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:0e4eed3b200023042814d2fc8a5d2e880f13b52e1ed2485e83da4f3962f7dc1a", size = 127892, upload-time = "2026-05-06T15:10:24.714Z" }, + { url = "https://files.pythonhosted.org/packages/66/60/52b0054c4c700d5aa7fc5b7ca96917400d8f061307778578e67a10e25852/orjson-3.11.9-cp313-cp313-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:8aff7da9952a5ad1cef8e68017724d96c7b9a66e99e91d6252e1b133d67a7b10", size = 135217, upload-time = "2026-05-06T15:10:26.084Z" }, + { url = "https://files.pythonhosted.org/packages/d5/97/1e3dc2b2a28b7b2528f403d2fc1d79ec5f39af3bc143ab65d3ec26426385/orjson-3.11.9-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:4d4e98d6f3b8afed8bc8cd9718ec0cdf46661826beefb53fe8eafb37f2bf0362", size = 145980, upload-time = "2026-05-06T15:10:28.062Z" }, + { url = "https://files.pythonhosted.org/packages/fc/39/31fbfe7850f2de32dee7e7e5c09f26d403ab01e440ac96001c6b01ad3c99/orjson-3.11.9-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3a81d52442a7c99b3662333235b3adf96a1715864658b35bb797212be7bddb97", size = 132738, upload-time = "2026-05-06T15:10:29.727Z" }, + { url = "https://files.pythonhosted.org/packages/a1/08/dca0082dd2a194acb93e5457e73455388e2e2ca464a2672449a9ddbb679d/orjson-3.11.9-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4e39364e726a8fff737309aff059ff67d8a8c8d5b677be7bb49a8b3e84b7e218", size = 134033, upload-time = "2026-05-06T15:10:31.152Z" }, + { url = "https://files.pythonhosted.org/packages/11/d4/5bdb0626801230139987385554c5d4c42255218ac906525bf4347f22cd95/orjson-3.11.9-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:4fd66214623f1b17501df9f0543bef0b833979ab5b6ded1e1d123222866aa8c9", size = 141492, upload-time = "2026-05-06T15:10:32.641Z" }, + { url = "https://files.pythonhosted.org/packages/fa/88/a21fb53b3ede6703aede6dce4710ed4111e5b201cfa6bbff5e544f9d47d7/orjson-3.11.9-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:8ecc30f10465fa1e0ce13fd01d9e22c316e5053a719a8d915d4545a09a5ff677", size = 415087, upload-time = "2026-05-06T15:10:34.438Z" }, + { url = "https://files.pythonhosted.org/packages/3d/57/1b30daf70f0d8180e9a73cefbfbdd99e4bf19eb020466502b01fba7e0e50/orjson-3.11.9-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:97db4c94a7db398a5bd636273324f0b3fd58b350bbbac8bb380ceb825a9b40f4", size = 148031, upload-time = "2026-05-06T15:10:36.358Z" }, + { url = "https://files.pythonhosted.org/packages/04/83/45fbb6d962e260807f99441db9613cee868ceda4baceda59b3720a563f97/orjson-3.11.9-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:9f78cf8fec5bd627f4082b8dfeac7871b43d7f3274904492a43dab39f18a19a0", size = 136915, upload-time = "2026-05-06T15:10:38.013Z" }, + { url = "https://files.pythonhosted.org/packages/5f/cc/2d10025f9056d376e4127ec05a5808b218d46f035fdc08178a5411b34250/orjson-3.11.9-cp313-cp313-win32.whl", hash = "sha256:d4087e5c0209a0a8efe4de3303c234b9c44d1174161dcd851e8eea07c7560b32", size = 131613, upload-time = "2026-05-06T15:10:39.569Z" }, + { url = "https://files.pythonhosted.org/packages/67/bd/2775ff28bfe883b9aa1ff348300542eb2ef1ee18d8ae0e3a49846817a865/orjson-3.11.9-cp313-cp313-win_amd64.whl", hash = "sha256:051b102c93b4f634e89f3866b07b9a9a98915ada541f4ec30f177067b2694979", size = 127086, upload-time = "2026-05-06T15:10:41.262Z" }, + { url = "https://files.pythonhosted.org/packages/91/2b/d26799e580939e32a7da9a39531bc9e58e15ca32ffaa6a8cb3e9bb0d22cd/orjson-3.11.9-cp313-cp313-win_arm64.whl", hash = "sha256:cce9127885941bd28f080cecf1f1d288336b7e0d812c345b08be88b572796254", size = 126696, upload-time = "2026-05-06T15:10:42.651Z" }, + { url = "https://files.pythonhosted.org/packages/8e/eb/5da01e356015aee6ecfa1187ced87aef51364e306f5e695dd52719bf0e78/orjson-3.11.9-cp314-cp314-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:b6ef1979adc4bc243523f1a2ba91418030a8e29b0a99cbe7e0e2d6807d4dce6e", size = 228465, upload-time = "2026-05-06T15:10:44.097Z" }, + { url = "https://files.pythonhosted.org/packages/64/62/3e0e0c14c957133bcd855395c62b55ed4e3b0af23ffea11b032cb1dcbdb1/orjson-3.11.9-cp314-cp314-macosx_15_0_arm64.whl", hash = "sha256:f36b7f32c7c0db4a719f1fc5824db4a9c6f8bd1a354debb91faf26ebf3a4c71e", size = 128364, upload-time = "2026-05-06T15:10:45.839Z" }, + { url = "https://files.pythonhosted.org/packages/5a/5a/07d8aa117211a8ed7630bda80c8c0b14d04e0f8dcf99bcf49656e4a710eb/orjson-3.11.9-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:08f4d8ebb44925c794e535b2bebc507cebf32209df81de22ae285fb0d8d66de0", size = 132063, upload-time = "2026-05-06T15:10:47.267Z" }, + { url = "https://files.pythonhosted.org/packages/d6/ec/4acaf21483e18aa945be74a474c74b434f284b549f275a0a39b9f98956e9/orjson-3.11.9-cp314-cp314-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:6cc7923789694fd58f001cbcac7e47abc13af4d560ebbfcf3b41a8b1a0748124", size = 122356, upload-time = "2026-05-06T15:10:48.765Z" }, + { url = "https://files.pythonhosted.org/packages/13/d8/5f0555e7638801323b7a75850f92e7dfa891bc84fe27a1ba4449170d1200/orjson-3.11.9-cp314-cp314-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ea5c46eb2d3af39e806b986f4b09d5c2706a1f5afde3cbf7544ce6616127173c", size = 129592, upload-time = "2026-05-06T15:10:50.13Z" }, + { url = "https://files.pythonhosted.org/packages/b6/30/ed9860412a3603ceb3c5955bfd72d28b9d0e7ba6ed81add14f83d7114236/orjson-3.11.9-cp314-cp314-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:f5d89a2ed90731df3be64bab0aa44f78bff39fdc9d71c291f4a8023aa46425b7", size = 140491, upload-time = "2026-05-06T15:10:51.582Z" }, + { url = "https://files.pythonhosted.org/packages/d0/17/adc514dea7ac7c505527febf884934b815d34f0c7b8693c1a8b39c5c4a57/orjson-3.11.9-cp314-cp314-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:25e4aed0312d292c09f61af25bba34e0b2c88546041472b09088c39a4d828af1", size = 127309, upload-time = "2026-05-06T15:10:53.329Z" }, + { url = "https://files.pythonhosted.org/packages/76/3e/c0b690253f0b82d86e99949af13533363acfb5432ecb5d53dd5b3bce9c34/orjson-3.11.9-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:aaea64f3f467d22e70eeed68bdccb3bc4f83f650446c4a03c59f2cba28a108db", size = 134030, upload-time = "2026-05-06T15:10:54.988Z" }, + { url = "https://files.pythonhosted.org/packages/c1/7a/bc82a0bb25e9faaf92dc4d9ef002732efc09737706af83e346788641d4a7/orjson-3.11.9-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:a028425d1b440c5d92a6be1e1a020739dfe67ea87d96c6dbe828c1b30041728b", size = 141482, upload-time = "2026-05-06T15:10:56.663Z" }, + { url = "https://files.pythonhosted.org/packages/01/55/e69188b939f77d5d32a9833745ace31ea5ccae3ab613a1ec185d3cd2c4fb/orjson-3.11.9-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:5b192c6cf397e4455b11523c5cf2b18ed084c1bbd61b6c0926344d2129481972", size = 415178, upload-time = "2026-05-06T15:10:58.446Z" }, + { url = "https://files.pythonhosted.org/packages/2e/1a/b8a5a7ac527e80b9cb11d51e3f6689b709279183264b9ec5c7bc680bb8b5/orjson-3.11.9-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:ea407d4ccf5891d667d045fecae97a7a1e5e87b3b97f97ae1803c2e741130be0", size = 148089, upload-time = "2026-05-06T15:11:00.441Z" }, + { url = "https://files.pythonhosted.org/packages/97/4e/00503f64204bf859b37213a63927028f30fb6268cd8677fb0a5ad48155e1/orjson-3.11.9-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:5f63aaf97afd9f6dec5b1a68e1b8da12bfccb4cb9a9a65c3e0b6c847849e7586", size = 136921, upload-time = "2026-05-06T15:11:02.176Z" }, + { url = "https://files.pythonhosted.org/packages/0d/ba/a23b82a0a8d0ed7bed4e5f5035aae751cad4ff6a1e8d2ecd14d8860f5929/orjson-3.11.9-cp314-cp314-win32.whl", hash = "sha256:e30ab17845bb9fa54ccf67fa4f9f5282652d54faa6d17452f47d0f369d038673", size = 131638, upload-time = "2026-05-06T15:11:03.696Z" }, + { url = "https://files.pythonhosted.org/packages/f3/c3/0c6798456bade745c75c452342dabacce5798196483e77e643be1f53877d/orjson-3.11.9-cp314-cp314-win_amd64.whl", hash = "sha256:32ef5f4283a3be81913947d19608eacb7c6608026851123790cd9cc8982af34b", size = 127078, upload-time = "2026-05-06T15:11:05.123Z" }, + { url = "https://files.pythonhosted.org/packages/16/21/5a3f1e8913103b703a436a5664238e5b965ec392b555fe68943ea3691e6b/orjson-3.11.9-cp314-cp314-win_arm64.whl", hash = "sha256:eebdbdeef0094e4f5aefa20dcd4eb2368ab5e7a3b4edea27f1e7b2892e009cf9", size = 126687, upload-time = "2026-05-06T15:11:06.602Z" }, +] + [[package]] name = "packaging" version = "26.0" @@ -897,6 +1095,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/52/96/5a770e5c461462575474468e5af931cff9de036e7c2b4fea23c1c58d2cbe/pathable-0.5.0-py3-none-any.whl", hash = "sha256:646e3d09491a6351a0c82632a09c02cdf70a252e73196b36d8a15ba0a114f0a6", size = 16867, upload-time = "2026-02-20T08:46:59.536Z" }, ] +[[package]] +name = "pathspec" +version = "1.0.4" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/fa/36/e27608899f9b8d4dff0617b2d9ab17ca5608956ca44461ac14ac48b44015/pathspec-1.0.4.tar.gz", hash = "sha256:0210e2ae8a21a9137c0d470578cb0e595af87edaa6ebf12ff176f14a02e0e645", size = 131200, upload-time = "2026-01-27T03:59:46.938Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ef/3c/2c197d226f9ea224a9ab8d197933f9da0ae0aac5b6e0f884e2b8d9c8e9f7/pathspec-1.0.4-py3-none-any.whl", hash = "sha256:fb6ae2fd4e7c921a165808a552060e722767cfa526f99ca5156ed2ce45a5c723", size = 55206, upload-time = "2026-01-27T03:59:45.137Z" }, +] + [[package]] name = "platformdirs" version = "4.9.2" @@ -1169,6 +1376,20 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/e5/35/f8b19922b6a25bc0880171a2f1a003eaeb93657475193ab516fd87cac9da/pytest_asyncio-1.3.0-py3-none-any.whl", hash = "sha256:611e26147c7f77640e6d0a92a38ed17c3e9848063698d5c93d5aa7aa11cebff5", size = 15075, upload-time = "2025-11-10T16:07:45.537Z" }, ] +[[package]] +name = "pytest-cov" +version = "7.1.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "coverage", extra = ["toml"] }, + { name = "pluggy" }, + { name = "pytest" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/b1/51/a849f96e117386044471c8ec2bd6cfebacda285da9525c9106aeb28da671/pytest_cov-7.1.0.tar.gz", hash = "sha256:30674f2b5f6351aa09702a9c8c364f6a01c27aae0c1366ae8016160d1efc56b2", size = 55592, upload-time = "2026-03-21T20:11:16.284Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/9d/7a/d968e294073affff457b041c2be9868a40c1c71f4a35fcc1e45e5493067b/pytest_cov-7.1.0-py3-none-any.whl", hash = "sha256:a0461110b7865f9a271aa1b51e516c9a95de9d696734a2f71e3e78f46e1d4678", size = 22876, upload-time = "2026-03-21T20:11:14.438Z" }, +] + [[package]] name = "pytest-httpx" version = "0.36.0" @@ -1182,6 +1403,19 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/e2/d2/1eb1ea9c84f0d2033eb0b49675afdc71aa4ea801b74615f00f3c33b725e3/pytest_httpx-0.36.0-py3-none-any.whl", hash = "sha256:bd4c120bb80e142df856e825ec9f17981effb84d159f9fa29ed97e2357c3a9c8", size = 20229, upload-time = "2025-12-02T16:34:56.45Z" }, ] +[[package]] +name = "pytest-testmon" +version = "2.2.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "coverage" }, + { name = "pytest" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/4d/1d/3e4230cc67cd6205bbe03c3527500c0ccaf7f0c78b436537eac71590ee4a/pytest_testmon-2.2.0.tar.gz", hash = "sha256:01f488e955ed0e0049777bee598bf1f647dd524e06f544c31a24e68f8d775a51", size = 23108, upload-time = "2025-12-01T07:30:24.76Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/61/55/ebb3c2f59fb089f08d00f764830d35780fc4e4c41dffcadafa3264682b65/pytest_testmon-2.2.0-py3-none-any.whl", hash = "sha256:2604ca44a54d61a2e830d9ce828b41a837075e4ebc1f81b148add8e90d34815b", size = 25199, upload-time = "2025-12-01T07:30:23.623Z" }, +] + [[package]] name = "pytest-timeout" version = "2.3.1" @@ -1322,6 +1556,110 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/2c/58/ca301544e1fa93ed4f80d724bf5b194f6e4b945841c5bfd555878eea9fcb/referencing-0.37.0-py3-none-any.whl", hash = "sha256:381329a9f99628c9069361716891d34ad94af76e461dcb0335825aecc7692231", size = 26766, upload-time = "2025-10-13T15:30:47.625Z" }, ] +[[package]] +name = "regex" +version = "2026.4.4" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/cb/0e/3a246dbf05666918bd3664d9d787f84a9108f6f43cc953a077e4a7dfdb7e/regex-2026.4.4.tar.gz", hash = "sha256:e08270659717f6973523ce3afbafa53515c4dc5dcad637dc215b6fd50f689423", size = 416000, upload-time = "2026-04-03T20:56:28.155Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e0/7a/617356cbecdb452812a5d42f720d6d5096b360d4a4c1073af700ea140ad2/regex-2026.4.4-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:b4c36a85b00fadb85db9d9e90144af0a980e1a3d2ef9cd0f8a5bef88054657c6", size = 489415, upload-time = "2026-04-03T20:53:11.645Z" }, + { url = "https://files.pythonhosted.org/packages/20/e6/bf057227144d02e3ba758b66649e87531d744dda5f3254f48660f18ae9d8/regex-2026.4.4-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:dcb5453ecf9cd58b562967badd1edbf092b0588a3af9e32ee3d05c985077ce87", size = 291205, upload-time = "2026-04-03T20:53:13.289Z" }, + { url = "https://files.pythonhosted.org/packages/eb/3b/637181b787dd1a820ba1c712cee2b4144cd84a32dc776ca067b12b2d70c8/regex-2026.4.4-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:6aa809ed4dc3706cc38594d67e641601bd2f36d5555b2780ff074edfcb136cf8", size = 289225, upload-time = "2026-04-03T20:53:16.002Z" }, + { url = "https://files.pythonhosted.org/packages/05/21/bac05d806ed02cd4b39d9c8e5b5f9a2998c94c3a351b7792e80671fa5315/regex-2026.4.4-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:33424f5188a7db12958246a54f59a435b6cb62c5cf9c8d71f7cc49475a5fdada", size = 792434, upload-time = "2026-04-03T20:53:17.414Z" }, + { url = "https://files.pythonhosted.org/packages/d9/17/c65d1d8ae90b772d5758eb4014e1e011bb2db353fc4455432e6cc9100df7/regex-2026.4.4-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:7d346fccdde28abba117cc9edc696b9518c3307fbfcb689e549d9b5979018c6d", size = 861730, upload-time = "2026-04-03T20:53:18.903Z" }, + { url = "https://files.pythonhosted.org/packages/ad/64/933321aa082a2c6ee2785f22776143ba89840189c20d3b6b1d12b6aae16b/regex-2026.4.4-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:415a994b536440f5011aa77e50a4274d15da3245e876e5c7f19da349caaedd87", size = 906495, upload-time = "2026-04-03T20:53:20.561Z" }, + { url = "https://files.pythonhosted.org/packages/01/ea/4c8d306e9c36ac22417336b1e02e7b358152c34dc379673f2d331143725f/regex-2026.4.4-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:21e5eb86179b4c67b5759d452ea7c48eb135cd93308e7a260aa489ed2eb423a4", size = 799810, upload-time = "2026-04-03T20:53:22.961Z" }, + { url = "https://files.pythonhosted.org/packages/29/ce/7605048f00e1379eba89d610c7d644d8f695dc9b26d3b6ecfa3132b872ff/regex-2026.4.4-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:312ec9dd1ae7d96abd8c5a36a552b2139931914407d26fba723f9e53c8186f86", size = 774242, upload-time = "2026-04-03T20:53:25.015Z" }, + { url = "https://files.pythonhosted.org/packages/e9/77/283e0d5023fde22cd9e86190d6d9beb21590a452b195ffe00274de470691/regex-2026.4.4-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:a0d2b28aa1354c7cd7f71b7658c4326f7facac106edd7f40eda984424229fd59", size = 781257, upload-time = "2026-04-03T20:53:26.918Z" }, + { url = "https://files.pythonhosted.org/packages/8b/fb/7f3b772be101373c8626ed34c5d727dcbb8abd42a7b1219bc25fd9a3cc04/regex-2026.4.4-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:349d7310eddff40429a099c08d995c6d4a4bfaf3ff40bd3b5e5cb5a5a3c7d453", size = 854490, upload-time = "2026-04-03T20:53:29.065Z" }, + { url = "https://files.pythonhosted.org/packages/85/30/56547b80f34f4dd2986e1cdd63b1712932f63b6c4ce2f79c50a6cd79d1c2/regex-2026.4.4-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:e7ab63e9fe45a9ec3417509e18116b367e89c9ceb6219222a3396fa30b147f80", size = 763544, upload-time = "2026-04-03T20:53:30.917Z" }, + { url = "https://files.pythonhosted.org/packages/ac/2f/ce060fdfea8eff34a8997603532e44cdb7d1f35e3bc253612a8707a90538/regex-2026.4.4-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:fe896e07a5a2462308297e515c0054e9ec2dd18dfdc9427b19900b37dfe6f40b", size = 844442, upload-time = "2026-04-03T20:53:32.463Z" }, + { url = "https://files.pythonhosted.org/packages/e5/44/810cb113096a1dacbe82789fbfab2823f79d19b7f1271acecb7009ba9b88/regex-2026.4.4-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:eb59c65069498dbae3c0ef07bbe224e1eaa079825a437fb47a479f0af11f774f", size = 789162, upload-time = "2026-04-03T20:53:34.039Z" }, + { url = "https://files.pythonhosted.org/packages/20/96/9647dd7f2ecf6d9ce1fb04dfdb66910d094e10d8fe53e9c15096d8aa0bd2/regex-2026.4.4-cp311-cp311-win32.whl", hash = "sha256:2a5d273181b560ef8397c8825f2b9d57013de744da9e8257b8467e5da8599351", size = 266227, upload-time = "2026-04-03T20:53:35.601Z" }, + { url = "https://files.pythonhosted.org/packages/33/80/74e13262460530c3097ff343a17de9a34d040a5dc4de9cf3a8241faab51c/regex-2026.4.4-cp311-cp311-win_amd64.whl", hash = "sha256:9542ccc1e689e752594309444081582f7be2fdb2df75acafea8a075108566735", size = 278399, upload-time = "2026-04-03T20:53:37.021Z" }, + { url = "https://files.pythonhosted.org/packages/1c/3c/39f19f47f19dcefa3403f09d13562ca1c0fd07ab54db2bc03148f3f6b46a/regex-2026.4.4-cp311-cp311-win_arm64.whl", hash = "sha256:b5f9fb784824a042be3455b53d0b112655686fdb7a91f88f095f3fee1e2a2a54", size = 270473, upload-time = "2026-04-03T20:53:38.633Z" }, + { url = "https://files.pythonhosted.org/packages/e5/28/b972a4d3df61e1d7bcf1b59fdb3cddef22f88b6be43f161bb41ebc0e4081/regex-2026.4.4-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:c07ab8794fa929e58d97a0e1796b8b76f70943fa39df225ac9964615cf1f9d52", size = 490434, upload-time = "2026-04-03T20:53:40.219Z" }, + { url = "https://files.pythonhosted.org/packages/84/20/30041446cf6dc3e0eab344fc62770e84c23b6b68a3b657821f9f80cb69b4/regex-2026.4.4-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:2c785939dc023a1ce4ec09599c032cc9933d258a998d16ca6f2b596c010940eb", size = 292061, upload-time = "2026-04-03T20:53:41.862Z" }, + { url = "https://files.pythonhosted.org/packages/62/c8/3baa06d75c98c46d4cc4262b71fd2edb9062b5665e868bca57859dadf93a/regex-2026.4.4-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:1b1ce5c81c9114f1ce2f9288a51a8fd3aeea33a0cc440c415bf02da323aa0a76", size = 289628, upload-time = "2026-04-03T20:53:43.701Z" }, + { url = "https://files.pythonhosted.org/packages/31/87/3accf55634caad8c0acab23f5135ef7d4a21c39f28c55c816ae012931408/regex-2026.4.4-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:760ef21c17d8e6a4fe8cf406a97cf2806a4df93416ccc82fc98d25b1c20425be", size = 796651, upload-time = "2026-04-03T20:53:45.379Z" }, + { url = "https://files.pythonhosted.org/packages/f6/0c/aaa2c83f34efedbf06f61cb1942c25f6cf1ee3b200f832c4d05f28306c2e/regex-2026.4.4-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:7088fcdcb604a4417c208e2169715800d28838fefd7455fbe40416231d1d47c1", size = 865916, upload-time = "2026-04-03T20:53:47.064Z" }, + { url = "https://files.pythonhosted.org/packages/d9/f6/8c6924c865124643e8f37823eca845dc27ac509b2ee58123685e71cd0279/regex-2026.4.4-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:07edca1ba687998968f7db5bc355288d0c6505caa7374f013d27356d93976d13", size = 912287, upload-time = "2026-04-03T20:53:49.422Z" }, + { url = "https://files.pythonhosted.org/packages/11/0e/a9f6f81013e0deaf559b25711623864970fe6a098314e374ccb1540a4152/regex-2026.4.4-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:993f657a7c1c6ec51b5e0ba97c9817d06b84ea5fa8d82e43b9405de0defdc2b9", size = 801126, upload-time = "2026-04-03T20:53:51.096Z" }, + { url = "https://files.pythonhosted.org/packages/71/61/3a0cc8af2dc0c8deb48e644dd2521f173f7e6513c6e195aad9aa8dd77ac5/regex-2026.4.4-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:2b69102a743e7569ebee67e634a69c4cb7e59d6fa2e1aa7d3bdbf3f61435f62d", size = 776788, upload-time = "2026-04-03T20:53:52.889Z" }, + { url = "https://files.pythonhosted.org/packages/64/0b/8bb9cbf21ef7dee58e49b0fdb066a7aded146c823202e16494a36777594f/regex-2026.4.4-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:6dac006c8b6dda72d86ea3d1333d45147de79a3a3f26f10c1cf9287ca4ca0ac3", size = 785184, upload-time = "2026-04-03T20:53:55.627Z" }, + { url = "https://files.pythonhosted.org/packages/99/c2/d3e80e8137b25ee06c92627de4e4d98b94830e02b3e6f81f3d2e3f504cf5/regex-2026.4.4-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:50a766ee2010d504554bfb5f578ed2e066898aa26411d57e6296230627cdefa0", size = 859913, upload-time = "2026-04-03T20:53:57.249Z" }, + { url = "https://files.pythonhosted.org/packages/bc/e6/9d5d876157d969c804622456ef250017ac7a8f83e0e14f903b9e6df5ce95/regex-2026.4.4-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:9e2f5217648f68e3028c823df58663587c1507a5ba8419f4fdfc8a461be76043", size = 765732, upload-time = "2026-04-03T20:53:59.428Z" }, + { url = "https://files.pythonhosted.org/packages/82/80/b568935b4421388561c8ed42aff77247285d3ae3bb2a6ca22af63bae805e/regex-2026.4.4-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:39d8de85a08e32632974151ba59c6e9140646dcc36c80423962b1c5c0a92e244", size = 852152, upload-time = "2026-04-03T20:54:01.505Z" }, + { url = "https://files.pythonhosted.org/packages/39/29/f0f81217e21cd998245da047405366385d5c6072048038a3d33b37a79dc0/regex-2026.4.4-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:55d9304e0e7178dfb1e106c33edf834097ddf4a890e2f676f6c5118f84390f73", size = 789076, upload-time = "2026-04-03T20:54:03.323Z" }, + { url = "https://files.pythonhosted.org/packages/49/1d/1d957a61976ab9d4e767dd4f9d04b66cc0c41c5e36cf40e2d43688b5ae6f/regex-2026.4.4-cp312-cp312-win32.whl", hash = "sha256:04bb679bc0bde8a7bfb71e991493d47314e7b98380b083df2447cda4b6edb60f", size = 266700, upload-time = "2026-04-03T20:54:05.639Z" }, + { url = "https://files.pythonhosted.org/packages/c5/5c/bf575d396aeb58ea13b06ef2adf624f65b70fafef6950a80fc3da9cae3bc/regex-2026.4.4-cp312-cp312-win_amd64.whl", hash = "sha256:db0ac18435a40a2543dbb3d21e161a6c78e33e8159bd2e009343d224bb03bb1b", size = 277768, upload-time = "2026-04-03T20:54:07.312Z" }, + { url = "https://files.pythonhosted.org/packages/c9/27/049df16ec6a6828ccd72add3c7f54b4df029669bea8e9817df6fff58be90/regex-2026.4.4-cp312-cp312-win_arm64.whl", hash = "sha256:4ce255cc05c1947a12989c6db801c96461947adb7a59990f1360b5983fab4983", size = 270568, upload-time = "2026-04-03T20:54:09.484Z" }, + { url = "https://files.pythonhosted.org/packages/9d/83/c4373bc5f31f2cf4b66f9b7c31005bd87fe66f0dce17701f7db4ee79ee29/regex-2026.4.4-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:62f5519042c101762509b1d717b45a69c0139d60414b3c604b81328c01bd1943", size = 490273, upload-time = "2026-04-03T20:54:11.202Z" }, + { url = "https://files.pythonhosted.org/packages/46/f8/fe62afbcc3cf4ad4ac9adeaafd98aa747869ae12d3e8e2ac293d0593c435/regex-2026.4.4-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:3790ba9fb5dd76715a7afe34dbe603ba03f8820764b1dc929dd08106214ed031", size = 291954, upload-time = "2026-04-03T20:54:13.412Z" }, + { url = "https://files.pythonhosted.org/packages/5a/92/4712b9fe6a33d232eeb1c189484b80c6c4b8422b90e766e1195d6e758207/regex-2026.4.4-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:8fae3c6e795d7678963f2170152b0d892cf6aee9ee8afc8c45e6be38d5107fe7", size = 289487, upload-time = "2026-04-03T20:54:15.824Z" }, + { url = "https://files.pythonhosted.org/packages/88/2c/f83b93f85e01168f1070f045a42d4c937b69fdb8dd7ae82d307253f7e36e/regex-2026.4.4-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:298c3ec2d53225b3bf91142eb9691025bab610e0c0c51592dde149db679b3d17", size = 796646, upload-time = "2026-04-03T20:54:18.229Z" }, + { url = "https://files.pythonhosted.org/packages/df/55/61a2e17bf0c4dc57e11caf8dd11771280d8aaa361785f9e3bc40d653f4a7/regex-2026.4.4-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:e9638791082eaf5b3ac112c587518ee78e083a11c4b28012d8fe2a0f536dfb17", size = 865904, upload-time = "2026-04-03T20:54:20.019Z" }, + { url = "https://files.pythonhosted.org/packages/45/32/1ac8ed1b5a346b5993a3d256abe0a0f03b0b73c8cc88d928537368ac65b6/regex-2026.4.4-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:ae3e764bd4c5ff55035dc82a8d49acceb42a5298edf6eb2fc4d328ee5dd7afae", size = 912304, upload-time = "2026-04-03T20:54:22.403Z" }, + { url = "https://files.pythonhosted.org/packages/26/47/2ee5c613ab546f0eddebf9905d23e07beb933416b1246c2d8791d01979b4/regex-2026.4.4-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ffa81f81b80047ba89a3c69ae6a0f78d06f4a42ce5126b0eb2a0a10ad44e0b2e", size = 801126, upload-time = "2026-04-03T20:54:24.308Z" }, + { url = "https://files.pythonhosted.org/packages/75/cd/41dacd129ca9fd20bd7d02f83e0fad83e034ac8a084ec369c90f55ef37e2/regex-2026.4.4-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:f56ebf9d70305307a707911b88469213630aba821e77de7d603f9d2f0730687d", size = 776772, upload-time = "2026-04-03T20:54:26.319Z" }, + { url = "https://files.pythonhosted.org/packages/89/6d/5af0b588174cb5f46041fa7dd64d3fd5cd2fe51f18766703d1edc387f324/regex-2026.4.4-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:773d1dfd652bbffb09336abf890bfd64785c7463716bf766d0eb3bc19c8b7f27", size = 785228, upload-time = "2026-04-03T20:54:28.387Z" }, + { url = "https://files.pythonhosted.org/packages/b7/3b/f5a72b7045bd59575fc33bf1345f156fcfd5a8484aea6ad84b12c5a82114/regex-2026.4.4-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:d51d20befd5275d092cdffba57ded05f3c436317ee56466c8928ac32d960edaf", size = 860032, upload-time = "2026-04-03T20:54:30.641Z" }, + { url = "https://files.pythonhosted.org/packages/39/a4/72a317003d6fcd7a573584a85f59f525dfe8f67e355ca74eb6b53d66a5e2/regex-2026.4.4-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:0a51cdb3c1e9161154f976cb2bef9894bc063ac82f31b733087ffb8e880137d0", size = 765714, upload-time = "2026-04-03T20:54:32.789Z" }, + { url = "https://files.pythonhosted.org/packages/25/1e/5672e16f34dbbcb2560cc7e6a2fbb26dfa8b270711e730101da4423d3973/regex-2026.4.4-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:ae5266a82596114e41fb5302140e9630204c1b5f325c770bec654b95dd54b0aa", size = 852078, upload-time = "2026-04-03T20:54:34.546Z" }, + { url = "https://files.pythonhosted.org/packages/f7/0d/c813f0af7c6cc7ed7b9558bac2e5120b60ad0fa48f813e4d4bd55446f214/regex-2026.4.4-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:c882cd92ec68585e9c1cf36c447ec846c0d94edd706fe59e0c198e65822fd23b", size = 789181, upload-time = "2026-04-03T20:54:36.642Z" }, + { url = "https://files.pythonhosted.org/packages/ea/6d/a344608d1adbd2a95090ddd906cec09a11be0e6517e878d02a5123e0917f/regex-2026.4.4-cp313-cp313-win32.whl", hash = "sha256:05568c4fbf3cb4fa9e28e3af198c40d3237cf6041608a9022285fe567ec3ad62", size = 266690, upload-time = "2026-04-03T20:54:38.343Z" }, + { url = "https://files.pythonhosted.org/packages/31/07/54049f89b46235ca6f45cd6c88668a7050e77d4a15555e47dd40fde75263/regex-2026.4.4-cp313-cp313-win_amd64.whl", hash = "sha256:3384df51ed52db0bea967e21458ab0a414f67cdddfd94401688274e55147bb81", size = 277733, upload-time = "2026-04-03T20:54:40.11Z" }, + { url = "https://files.pythonhosted.org/packages/0e/21/61366a8e20f4d43fb597708cac7f0e2baadb491ecc9549b4980b2be27d16/regex-2026.4.4-cp313-cp313-win_arm64.whl", hash = "sha256:acd38177bd2c8e69a411d6521760806042e244d0ef94e2dd03ecdaa8a3c99427", size = 270565, upload-time = "2026-04-03T20:54:41.883Z" }, + { url = "https://files.pythonhosted.org/packages/f1/1e/3a2b9672433bef02f5d39aa1143ca2c08f311c1d041c464a42be9ae648dc/regex-2026.4.4-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:f94a11a9d05afcfcfa640e096319720a19cc0c9f7768e1a61fceee6a3afc6c7c", size = 494126, upload-time = "2026-04-03T20:54:43.602Z" }, + { url = "https://files.pythonhosted.org/packages/4e/4b/c132a4f4fe18ad3340d89fcb56235132b69559136036b845be3c073142ed/regex-2026.4.4-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:36bcb9d6d1307ab629edc553775baada2aefa5c50ccc0215fbfd2afcfff43141", size = 293882, upload-time = "2026-04-03T20:54:45.41Z" }, + { url = "https://files.pythonhosted.org/packages/f4/5f/eaa38092ce7a023656280f2341dbbd4ad5f05d780a70abba7bb4f4bea54c/regex-2026.4.4-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:261c015b3e2ed0919157046d768774ecde57f03d8fa4ba78d29793447f70e717", size = 292334, upload-time = "2026-04-03T20:54:47.051Z" }, + { url = "https://files.pythonhosted.org/packages/5f/f6/dd38146af1392dac33db7074ab331cec23cced3759167735c42c5460a243/regex-2026.4.4-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c228cf65b4a54583763645dcd73819b3b381ca8b4bb1b349dee1c135f4112c07", size = 811691, upload-time = "2026-04-03T20:54:49.074Z" }, + { url = "https://files.pythonhosted.org/packages/7a/f0/dc54c2e69f5eeec50601054998ec3690d5344277e782bd717e49867c1d29/regex-2026.4.4-cp313-cp313t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:dd2630faeb6876fb0c287f664d93ddce4d50cd46c6e88e60378c05c9047e08ca", size = 871227, upload-time = "2026-04-03T20:54:51.035Z" }, + { url = "https://files.pythonhosted.org/packages/a1/af/cb16bd5dc61621e27df919a4449bbb7e5a1034c34d307e0a706e9cc0f3e3/regex-2026.4.4-cp313-cp313t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:6a50ab11b7779b849472337191f3a043e27e17f71555f98d0092fa6d73364520", size = 917435, upload-time = "2026-04-03T20:54:52.994Z" }, + { url = "https://files.pythonhosted.org/packages/5c/71/8b260897f22996b666edd9402861668f45a2ca259f665ac029e6104a2d7d/regex-2026.4.4-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0734f63afe785138549fbe822a8cfeaccd1bae814c5057cc0ed5b9f2de4fc883", size = 816358, upload-time = "2026-04-03T20:54:54.884Z" }, + { url = "https://files.pythonhosted.org/packages/1c/60/775f7f72a510ef238254906c2f3d737fc80b16ca85f07d20e318d2eea894/regex-2026.4.4-cp313-cp313t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:c4ee50606cb1967db7e523224e05f32089101945f859928e65657a2cbb3d278b", size = 785549, upload-time = "2026-04-03T20:54:57.01Z" }, + { url = "https://files.pythonhosted.org/packages/58/42/34d289b3627c03cf381e44da534a0021664188fa49ba41513da0b4ec6776/regex-2026.4.4-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:6c1818f37be3ca02dcb76d63f2c7aaba4b0dc171b579796c6fbe00148dfec6b1", size = 801364, upload-time = "2026-04-03T20:54:58.981Z" }, + { url = "https://files.pythonhosted.org/packages/fc/20/f6ecf319b382a8f1ab529e898b222c3f30600fcede7834733c26279e7465/regex-2026.4.4-cp313-cp313t-musllinux_1_2_ppc64le.whl", hash = "sha256:f5bfc2741d150d0be3e4a0401a5c22b06e60acb9aa4daa46d9e79a6dcd0f135b", size = 866221, upload-time = "2026-04-03T20:55:00.88Z" }, + { url = "https://files.pythonhosted.org/packages/92/6a/9f16d3609d549bd96d7a0b2aee1625d7512ba6a03efc01652149ef88e74d/regex-2026.4.4-cp313-cp313t-musllinux_1_2_riscv64.whl", hash = "sha256:504ffa8a03609a087cad81277a629b6ce884b51a24bd388a7980ad61748618ff", size = 772530, upload-time = "2026-04-03T20:55:03.213Z" }, + { url = "https://files.pythonhosted.org/packages/fa/f6/aa9768bc96a4c361ac96419fbaf2dcdc33970bb813df3ba9b09d5d7b6d96/regex-2026.4.4-cp313-cp313t-musllinux_1_2_s390x.whl", hash = "sha256:70aadc6ff12e4b444586e57fc30771f86253f9f0045b29016b9605b4be5f7dfb", size = 856989, upload-time = "2026-04-03T20:55:05.087Z" }, + { url = "https://files.pythonhosted.org/packages/4d/b4/c671db3556be2473ae3e4bb7a297c518d281452871501221251ea4ecba57/regex-2026.4.4-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:f4f83781191007b6ef43b03debc35435f10cad9b96e16d147efe84a1d48bdde4", size = 803241, upload-time = "2026-04-03T20:55:07.162Z" }, + { url = "https://files.pythonhosted.org/packages/2a/5c/83e3b1d89fa4f6e5a1bc97b4abd4a9a97b3c1ac7854164f694f5f0ba98a0/regex-2026.4.4-cp313-cp313t-win32.whl", hash = "sha256:e014a797de43d1847df957c0a2a8e861d1c17547ee08467d1db2c370b7568baa", size = 269921, upload-time = "2026-04-03T20:55:09.62Z" }, + { url = "https://files.pythonhosted.org/packages/28/07/077c387121f42cdb4d92b1301133c0d93b5709d096d1669ab847dda9fe2e/regex-2026.4.4-cp313-cp313t-win_amd64.whl", hash = "sha256:b15b88b0d52b179712632832c1d6e58e5774f93717849a41096880442da41ab0", size = 281240, upload-time = "2026-04-03T20:55:11.521Z" }, + { url = "https://files.pythonhosted.org/packages/9d/22/ead4a4abc7c59a4d882662aa292ca02c8b617f30b6e163bc1728879e9353/regex-2026.4.4-cp313-cp313t-win_arm64.whl", hash = "sha256:586b89cdadf7d67bf86ae3342a4dcd2b8d70a832d90c18a0ae955105caf34dbe", size = 272440, upload-time = "2026-04-03T20:55:13.365Z" }, + { url = "https://files.pythonhosted.org/packages/f0/f5/ed97c2dc47b5fbd4b73c0d7d75f9ebc8eca139f2bbef476bba35f28c0a77/regex-2026.4.4-cp314-cp314-macosx_10_13_universal2.whl", hash = "sha256:2da82d643fa698e5e5210e54af90181603d5853cf469f5eedf9bfc8f59b4b8c7", size = 490343, upload-time = "2026-04-03T20:55:15.241Z" }, + { url = "https://files.pythonhosted.org/packages/80/e9/de4828a7385ec166d673a5790ad06ac48cdaa98bc0960108dd4b9cc1aef7/regex-2026.4.4-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:54a1189ad9d9357760557c91103d5e421f0a2dabe68a5cdf9103d0dcf4e00752", size = 291909, upload-time = "2026-04-03T20:55:17.558Z" }, + { url = "https://files.pythonhosted.org/packages/b4/d6/5cfbfc97f3201a4d24b596a77957e092030dcc4205894bc035cedcfce62f/regex-2026.4.4-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:76d67d5afb1fe402d10a6403bae668d000441e2ab115191a804287d53b772951", size = 289692, upload-time = "2026-04-03T20:55:20.561Z" }, + { url = "https://files.pythonhosted.org/packages/8e/ac/f2212d9fd56fe897e36d0110ba30ba2d247bd6410c5bd98499c7e5a1e1f2/regex-2026.4.4-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e7cd3e4ee8d80447a83bbc9ab0c8459781fa77087f856c3e740d7763be0df27f", size = 796979, upload-time = "2026-04-03T20:55:22.56Z" }, + { url = "https://files.pythonhosted.org/packages/c9/e3/a016c12675fbac988a60c7e1c16e67823ff0bc016beb27bd7a001dbdabc6/regex-2026.4.4-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:2e19e18c568d2866d8b6a6dfad823db86193503f90823a8f66689315ba28fbe8", size = 866744, upload-time = "2026-04-03T20:55:24.646Z" }, + { url = "https://files.pythonhosted.org/packages/af/a4/0b90ca4cf17adc3cb43de80ec71018c37c88ad64987e8d0d481a95ca60b5/regex-2026.4.4-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:7698a6f38730fd1385d390d1ed07bb13dce39aa616aca6a6d89bea178464b9a4", size = 911613, upload-time = "2026-04-03T20:55:27.033Z" }, + { url = "https://files.pythonhosted.org/packages/8e/3b/2b3dac0b82d41ab43aa87c6ecde63d71189d03fe8854b8ca455a315edac3/regex-2026.4.4-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:173a66f3651cdb761018078e2d9487f4cf971232c990035ec0eb1cdc6bf929a9", size = 800551, upload-time = "2026-04-03T20:55:29.532Z" }, + { url = "https://files.pythonhosted.org/packages/25/fe/5365eb7aa0e753c4b5957815c321519ecab033c279c60e1b1ae2367fa810/regex-2026.4.4-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:fa7922bbb2cc84fa062d37723f199d4c0cd200245ce269c05db82d904db66b83", size = 776911, upload-time = "2026-04-03T20:55:31.526Z" }, + { url = "https://files.pythonhosted.org/packages/aa/b3/7fb0072156bba065e3b778a7bc7b0a6328212be5dd6a86fd207e0c4f2dab/regex-2026.4.4-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:59f67cd0a0acaf0e564c20bbd7f767286f23e91e2572c5703bf3e56ea7557edb", size = 785751, upload-time = "2026-04-03T20:55:33.797Z" }, + { url = "https://files.pythonhosted.org/packages/02/1a/9f83677eb699273e56e858f7bd95acdbee376d42f59e8bfca2fd80d79df3/regex-2026.4.4-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:475e50f3f73f73614f7cba5524d6de49dee269df00272a1b85e3d19f6d498465", size = 860484, upload-time = "2026-04-03T20:55:35.745Z" }, + { url = "https://files.pythonhosted.org/packages/3b/7a/93937507b61cfcff8b4c5857f1b452852b09f741daa9acae15c971d8554e/regex-2026.4.4-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:a1c0c7d67b64d85ac2e1879923bad2f08a08f3004055f2f406ef73c850114bd4", size = 765939, upload-time = "2026-04-03T20:55:37.972Z" }, + { url = "https://files.pythonhosted.org/packages/86/ea/81a7f968a351c6552b1670ead861e2a385be730ee28402233020c67f9e0f/regex-2026.4.4-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:1371c2ccbb744d66ee63631cc9ca12aa233d5749972626b68fe1a649dd98e566", size = 851417, upload-time = "2026-04-03T20:55:39.92Z" }, + { url = "https://files.pythonhosted.org/packages/4c/7e/323c18ce4b5b8f44517a36342961a0306e931e499febbd876bb149d900f0/regex-2026.4.4-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:59968142787042db793348a3f5b918cf24ced1f23247328530e063f89c128a95", size = 789056, upload-time = "2026-04-03T20:55:42.303Z" }, + { url = "https://files.pythonhosted.org/packages/c0/af/e7510f9b11b1913b0cd44eddb784b2d650b2af6515bfce4cffcc5bfd1d38/regex-2026.4.4-cp314-cp314-win32.whl", hash = "sha256:59efe72d37fd5a91e373e5146f187f921f365f4abc1249a5ab446a60f30dd5f8", size = 272130, upload-time = "2026-04-03T20:55:44.995Z" }, + { url = "https://files.pythonhosted.org/packages/9a/51/57dae534c915e2d3a21490e88836fa2ae79dde3b66255ecc0c0a155d2c10/regex-2026.4.4-cp314-cp314-win_amd64.whl", hash = "sha256:e0aab3ff447845049d676827d2ff714aab4f73f340e155b7de7458cf53baa5a4", size = 280992, upload-time = "2026-04-03T20:55:47.316Z" }, + { url = "https://files.pythonhosted.org/packages/0a/5e/abaf9f4c3792e34edb1434f06717fae2b07888d85cb5cec29f9204931bf8/regex-2026.4.4-cp314-cp314-win_arm64.whl", hash = "sha256:a7a5bb6aa0cf62208bb4fa079b0c756734f8ad0e333b425732e8609bd51ee22f", size = 273563, upload-time = "2026-04-03T20:55:49.273Z" }, + { url = "https://files.pythonhosted.org/packages/ff/06/35da85f9f217b9538b99cbb170738993bcc3b23784322decb77619f11502/regex-2026.4.4-cp314-cp314t-macosx_10_13_universal2.whl", hash = "sha256:97850d0638391bdc7d35dc1c1039974dcb921eaafa8cc935ae4d7f272b1d60b3", size = 494191, upload-time = "2026-04-03T20:55:51.258Z" }, + { url = "https://files.pythonhosted.org/packages/54/5b/1bc35f479eef8285c4baf88d8c002023efdeebb7b44a8735b36195486ae7/regex-2026.4.4-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:ee7337f88f2a580679f7bbfe69dc86c043954f9f9c541012f49abc554a962f2e", size = 293877, upload-time = "2026-04-03T20:55:53.214Z" }, + { url = "https://files.pythonhosted.org/packages/39/5b/f53b9ad17480b3ddd14c90da04bfb55ac6894b129e5dea87bcaf7d00e336/regex-2026.4.4-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:7429f4e6192c11d659900c0648ba8776243bf396ab95558b8c51a345afeddde6", size = 292410, upload-time = "2026-04-03T20:55:55.736Z" }, + { url = "https://files.pythonhosted.org/packages/bb/56/52377f59f60a7c51aa4161eecf0b6032c20b461805aca051250da435ffc9/regex-2026.4.4-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:dc4f10fbd5dd13dcf4265b4cc07d69ca70280742870c97ae10093e3d66000359", size = 811831, upload-time = "2026-04-03T20:55:57.802Z" }, + { url = "https://files.pythonhosted.org/packages/dd/63/8026310bf066f702a9c361f83a8c9658f3fe4edb349f9c1e5d5273b7c40c/regex-2026.4.4-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:a152560af4f9742b96f3827090f866eeec5becd4765c8e0d3473d9d280e76a5a", size = 871199, upload-time = "2026-04-03T20:56:00.333Z" }, + { url = "https://files.pythonhosted.org/packages/20/9f/a514bbb00a466dbb506d43f187a04047f7be1505f10a9a15615ead5080ee/regex-2026.4.4-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:54170b3e95339f415d54651f97df3bff7434a663912f9358237941bbf9143f55", size = 917649, upload-time = "2026-04-03T20:56:02.445Z" }, + { url = "https://files.pythonhosted.org/packages/cb/6b/8399f68dd41a2030218839b9b18360d79b86d22b9fab5ef477c7f23ca67c/regex-2026.4.4-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:07f190d65f5a72dcb9cf7106bfc3d21e7a49dd2879eda2207b683f32165e4d99", size = 816388, upload-time = "2026-04-03T20:56:04.595Z" }, + { url = "https://files.pythonhosted.org/packages/1e/9c/103963f47c24339a483b05edd568594c2be486188f688c0170fd504b2948/regex-2026.4.4-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:9a2741ce5a29d3c84b0b94261ba630ab459a1b847a0d6beca7d62d188175c790", size = 785746, upload-time = "2026-04-03T20:56:07.13Z" }, + { url = "https://files.pythonhosted.org/packages/fa/ee/7f6054c0dec0cee3463c304405e4ff42e27cff05bf36fcb34be549ab17bd/regex-2026.4.4-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:b26c30df3a28fd9793113dac7385a4deb7294a06c0f760dd2b008bd49a9139bc", size = 801483, upload-time = "2026-04-03T20:56:09.365Z" }, + { url = "https://files.pythonhosted.org/packages/30/c2/51d3d941cf6070dc00c3338ecf138615fc3cce0421c3df6abe97a08af61a/regex-2026.4.4-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:421439d1bee44b19f4583ccf42670ca464ffb90e9fdc38d37f39d1ddd1e44f1f", size = 866331, upload-time = "2026-04-03T20:56:12.039Z" }, + { url = "https://files.pythonhosted.org/packages/16/e8/76d50dcc122ac33927d939f350eebcfe3dbcbda96913e03433fc36de5e63/regex-2026.4.4-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:b40379b53ecbc747fd9bdf4a0ea14eb8188ca1bd0f54f78893a39024b28f4863", size = 772673, upload-time = "2026-04-03T20:56:14.558Z" }, + { url = "https://files.pythonhosted.org/packages/a5/6e/5f6bf75e20ea6873d05ba4ec78378c375cbe08cdec571c83fbb01606e563/regex-2026.4.4-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:08c55c13d2eef54f73eeadc33146fb0baaa49e7335eb1aff6ae1324bf0ddbe4a", size = 857146, upload-time = "2026-04-03T20:56:16.663Z" }, + { url = "https://files.pythonhosted.org/packages/0b/33/3c76d9962949e487ebba353a18e89399f292287204ac8f2f4cfc3a51c233/regex-2026.4.4-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:9776b85f510062f5a75ef112afe5f494ef1635607bf1cc220c1391e9ac2f5e81", size = 803463, upload-time = "2026-04-03T20:56:18.923Z" }, + { url = "https://files.pythonhosted.org/packages/19/eb/ef32dcd2cb69b69bc0c3e55205bce94a7def48d495358946bc42186dcccc/regex-2026.4.4-cp314-cp314t-win32.whl", hash = "sha256:385edaebde5db5be103577afc8699fea73a0e36a734ba24870be7ffa61119d74", size = 275709, upload-time = "2026-04-03T20:56:20.996Z" }, + { url = "https://files.pythonhosted.org/packages/a0/86/c291bf740945acbf35ed7dbebf8e2eea2f3f78041f6bd7cdab80cb274dc0/regex-2026.4.4-cp314-cp314t-win_amd64.whl", hash = "sha256:5d354b18839328927832e2fa5f7c95b7a3ccc39e7a681529e1685898e6436d45", size = 285622, upload-time = "2026-04-03T20:56:23.641Z" }, + { url = "https://files.pythonhosted.org/packages/d5/e7/ec846d560ae6a597115153c02ca6138a7877a1748b2072d9521c10a93e58/regex-2026.4.4-cp314-cp314t-win_arm64.whl", hash = "sha256:af0384cb01a33600c49505c27c6c57ab0b27bf84a74e28524c92ca897ebdac9d", size = 275773, upload-time = "2026-04-03T20:56:26.07Z" }, +] + [[package]] name = "rich" version = "14.3.3" @@ -1538,6 +1876,60 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/24/99/4772b8e00a136f3e01236de33b0efda31ee7077203ba5967fcc76da94d65/texttable-1.7.0-py2.py3-none-any.whl", hash = "sha256:72227d592c82b3d7f672731ae73e4d1f88cd8e2ef5b075a7a7f01a23a3743917", size = 10768, upload-time = "2023-10-03T09:48:10.434Z" }, ] +[[package]] +name = "tomli" +version = "2.4.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/22/de/48c59722572767841493b26183a0d1cc411d54fd759c5607c4590b6563a6/tomli-2.4.1.tar.gz", hash = "sha256:7c7e1a961a0b2f2472c1ac5b69affa0ae1132c39adcb67aba98568702b9cc23f", size = 17543, upload-time = "2026-03-25T20:22:03.828Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f4/11/db3d5885d8528263d8adc260bb2d28ebf1270b96e98f0e0268d32b8d9900/tomli-2.4.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:f8f0fc26ec2cc2b965b7a3b87cd19c5c6b8c5e5f436b984e85f486d652285c30", size = 154704, upload-time = "2026-03-25T20:21:10.473Z" }, + { url = "https://files.pythonhosted.org/packages/6d/f7/675db52c7e46064a9aa928885a9b20f4124ecb9bc2e1ce74c9106648d202/tomli-2.4.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:4ab97e64ccda8756376892c53a72bd1f964e519c77236368527f758fbc36a53a", size = 149454, upload-time = "2026-03-25T20:21:12.036Z" }, + { url = "https://files.pythonhosted.org/packages/61/71/81c50943cf953efa35bce7646caab3cf457a7d8c030b27cfb40d7235f9ee/tomli-2.4.1-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:96481a5786729fd470164b47cdb3e0e58062a496f455ee41b4403be77cb5a076", size = 237561, upload-time = "2026-03-25T20:21:13.098Z" }, + { url = "https://files.pythonhosted.org/packages/48/c1/f41d9cb618acccca7df82aaf682f9b49013c9397212cb9f53219e3abac37/tomli-2.4.1-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5a881ab208c0baf688221f8cecc5401bd291d67e38a1ac884d6736cbcd8247e9", size = 243824, upload-time = "2026-03-25T20:21:14.569Z" }, + { url = "https://files.pythonhosted.org/packages/22/e4/5a816ecdd1f8ca51fb756ef684b90f2780afc52fc67f987e3c61d800a46d/tomli-2.4.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:47149d5bd38761ac8be13a84864bf0b7b70bc051806bc3669ab1cbc56216b23c", size = 242227, upload-time = "2026-03-25T20:21:15.712Z" }, + { url = "https://files.pythonhosted.org/packages/6b/49/2b2a0ef529aa6eec245d25f0c703e020a73955ad7edf73e7f54ddc608aa5/tomli-2.4.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:ec9bfaf3ad2df51ace80688143a6a4ebc09a248f6ff781a9945e51937008fcbc", size = 247859, upload-time = "2026-03-25T20:21:17.001Z" }, + { url = "https://files.pythonhosted.org/packages/83/bd/6c1a630eaca337e1e78c5903104f831bda934c426f9231429396ce3c3467/tomli-2.4.1-cp311-cp311-win32.whl", hash = "sha256:ff2983983d34813c1aeb0fa89091e76c3a22889ee83ab27c5eeb45100560c049", size = 97204, upload-time = "2026-03-25T20:21:18.079Z" }, + { url = "https://files.pythonhosted.org/packages/42/59/71461df1a885647e10b6bb7802d0b8e66480c61f3f43079e0dcd315b3954/tomli-2.4.1-cp311-cp311-win_amd64.whl", hash = "sha256:5ee18d9ebdb417e384b58fe414e8d6af9f4e7a0ae761519fb50f721de398dd4e", size = 108084, upload-time = "2026-03-25T20:21:18.978Z" }, + { url = "https://files.pythonhosted.org/packages/b8/83/dceca96142499c069475b790e7913b1044c1a4337e700751f48ed723f883/tomli-2.4.1-cp311-cp311-win_arm64.whl", hash = "sha256:c2541745709bad0264b7d4705ad453b76ccd191e64aa6f0fc66b69a293a45ece", size = 95285, upload-time = "2026-03-25T20:21:20.309Z" }, + { url = "https://files.pythonhosted.org/packages/c1/ba/42f134a3fe2b370f555f44b1d72feebb94debcab01676bf918d0cb70e9aa/tomli-2.4.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:c742f741d58a28940ce01d58f0ab2ea3ced8b12402f162f4d534dfe18ba1cd6a", size = 155924, upload-time = "2026-03-25T20:21:21.626Z" }, + { url = "https://files.pythonhosted.org/packages/dc/c7/62d7a17c26487ade21c5422b646110f2162f1fcc95980ef7f63e73c68f14/tomli-2.4.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:7f86fd587c4ed9dd76f318225e7d9b29cfc5a9d43de44e5754db8d1128487085", size = 150018, upload-time = "2026-03-25T20:21:23.002Z" }, + { url = "https://files.pythonhosted.org/packages/5c/05/79d13d7c15f13bdef410bdd49a6485b1c37d28968314eabee452c22a7fda/tomli-2.4.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ff18e6a727ee0ab0388507b89d1bc6a22b138d1e2fa56d1ad494586d61d2eae9", size = 244948, upload-time = "2026-03-25T20:21:24.04Z" }, + { url = "https://files.pythonhosted.org/packages/10/90/d62ce007a1c80d0b2c93e02cab211224756240884751b94ca72df8a875ca/tomli-2.4.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:136443dbd7e1dee43c68ac2694fde36b2849865fa258d39bf822c10e8068eac5", size = 253341, upload-time = "2026-03-25T20:21:25.177Z" }, + { url = "https://files.pythonhosted.org/packages/1a/7e/caf6496d60152ad4ed09282c1885cca4eea150bfd007da84aea07bcc0a3e/tomli-2.4.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:5e262d41726bc187e69af7825504c933b6794dc3fbd5945e41a79bb14c31f585", size = 248159, upload-time = "2026-03-25T20:21:26.364Z" }, + { url = "https://files.pythonhosted.org/packages/99/e7/c6f69c3120de34bbd882c6fba7975f3d7a746e9218e56ab46a1bc4b42552/tomli-2.4.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:5cb41aa38891e073ee49d55fbc7839cfdb2bc0e600add13874d048c94aadddd1", size = 253290, upload-time = "2026-03-25T20:21:27.46Z" }, + { url = "https://files.pythonhosted.org/packages/d6/2f/4a3c322f22c5c66c4b836ec58211641a4067364f5dcdd7b974b4c5da300c/tomli-2.4.1-cp312-cp312-win32.whl", hash = "sha256:da25dc3563bff5965356133435b757a795a17b17d01dbc0f42fb32447ddfd917", size = 98141, upload-time = "2026-03-25T20:21:28.492Z" }, + { url = "https://files.pythonhosted.org/packages/24/22/4daacd05391b92c55759d55eaee21e1dfaea86ce5c571f10083360adf534/tomli-2.4.1-cp312-cp312-win_amd64.whl", hash = "sha256:52c8ef851d9a240f11a88c003eacb03c31fc1c9c4ec64a99a0f922b93874fda9", size = 108847, upload-time = "2026-03-25T20:21:29.386Z" }, + { url = "https://files.pythonhosted.org/packages/68/fd/70e768887666ddd9e9f5d85129e84910f2db2796f9096aa02b721a53098d/tomli-2.4.1-cp312-cp312-win_arm64.whl", hash = "sha256:f758f1b9299d059cc3f6546ae2af89670cb1c4d48ea29c3cacc4fe7de3058257", size = 95088, upload-time = "2026-03-25T20:21:30.677Z" }, + { url = "https://files.pythonhosted.org/packages/07/06/b823a7e818c756d9a7123ba2cda7d07bc2dd32835648d1a7b7b7a05d848d/tomli-2.4.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:36d2bd2ad5fb9eaddba5226aa02c8ec3fa4f192631e347b3ed28186d43be6b54", size = 155866, upload-time = "2026-03-25T20:21:31.65Z" }, + { url = "https://files.pythonhosted.org/packages/14/6f/12645cf7f08e1a20c7eb8c297c6f11d31c1b50f316a7e7e1e1de6e2e7b7e/tomli-2.4.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:eb0dc4e38e6a1fd579e5d50369aa2e10acfc9cace504579b2faabb478e76941a", size = 149887, upload-time = "2026-03-25T20:21:33.028Z" }, + { url = "https://files.pythonhosted.org/packages/5c/e0/90637574e5e7212c09099c67ad349b04ec4d6020324539297b634a0192b0/tomli-2.4.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c7f2c7f2b9ca6bdeef8f0fa897f8e05085923eb091721675170254cbc5b02897", size = 243704, upload-time = "2026-03-25T20:21:34.51Z" }, + { url = "https://files.pythonhosted.org/packages/10/8f/d3ddb16c5a4befdf31a23307f72828686ab2096f068eaf56631e136c1fdd/tomli-2.4.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f3c6818a1a86dd6dca7ddcaaf76947d5ba31aecc28cb1b67009a5877c9a64f3f", size = 251628, upload-time = "2026-03-25T20:21:36.012Z" }, + { url = "https://files.pythonhosted.org/packages/e3/f1/dbeeb9116715abee2485bf0a12d07a8f31af94d71608c171c45f64c0469d/tomli-2.4.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:d312ef37c91508b0ab2cee7da26ec0b3ed2f03ce12bd87a588d771ae15dcf82d", size = 247180, upload-time = "2026-03-25T20:21:37.136Z" }, + { url = "https://files.pythonhosted.org/packages/d3/74/16336ffd19ed4da28a70959f92f506233bd7cfc2332b20bdb01591e8b1d1/tomli-2.4.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:51529d40e3ca50046d7606fa99ce3956a617f9b36380da3b7f0dd3dd28e68cb5", size = 251674, upload-time = "2026-03-25T20:21:38.298Z" }, + { url = "https://files.pythonhosted.org/packages/16/f9/229fa3434c590ddf6c0aa9af64d3af4b752540686cace29e6281e3458469/tomli-2.4.1-cp313-cp313-win32.whl", hash = "sha256:2190f2e9dd7508d2a90ded5ed369255980a1bcdd58e52f7fe24b8162bf9fedbd", size = 97976, upload-time = "2026-03-25T20:21:39.316Z" }, + { url = "https://files.pythonhosted.org/packages/6a/1e/71dfd96bcc1c775420cb8befe7a9d35f2e5b1309798f009dca17b7708c1e/tomli-2.4.1-cp313-cp313-win_amd64.whl", hash = "sha256:8d65a2fbf9d2f8352685bc1364177ee3923d6baf5e7f43ea4959d7d8bc326a36", size = 108755, upload-time = "2026-03-25T20:21:40.248Z" }, + { url = "https://files.pythonhosted.org/packages/83/7a/d34f422a021d62420b78f5c538e5b102f62bea616d1d75a13f0a88acb04a/tomli-2.4.1-cp313-cp313-win_arm64.whl", hash = "sha256:4b605484e43cdc43f0954ddae319fb75f04cc10dd80d830540060ee7cd0243cd", size = 95265, upload-time = "2026-03-25T20:21:41.219Z" }, + { url = "https://files.pythonhosted.org/packages/3c/fb/9a5c8d27dbab540869f7c1f8eb0abb3244189ce780ba9cd73f3770662072/tomli-2.4.1-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:fd0409a3653af6c147209d267a0e4243f0ae46b011aa978b1080359fddc9b6cf", size = 155726, upload-time = "2026-03-25T20:21:42.23Z" }, + { url = "https://files.pythonhosted.org/packages/62/05/d2f816630cc771ad836af54f5001f47a6f611d2d39535364f148b6a92d6b/tomli-2.4.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:a120733b01c45e9a0c34aeef92bf0cf1d56cfe81ed9d47d562f9ed591a9828ac", size = 149859, upload-time = "2026-03-25T20:21:43.386Z" }, + { url = "https://files.pythonhosted.org/packages/ce/48/66341bdb858ad9bd0ceab5a86f90eddab127cf8b046418009f2125630ecb/tomli-2.4.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:559db847dc486944896521f68d8190be1c9e719fced785720d2216fe7022b662", size = 244713, upload-time = "2026-03-25T20:21:44.474Z" }, + { url = "https://files.pythonhosted.org/packages/df/6d/c5fad00d82b3c7a3ab6189bd4b10e60466f22cfe8a08a9394185c8a8111c/tomli-2.4.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:01f520d4f53ef97964a240a035ec2a869fe1a37dde002b57ebc4417a27ccd853", size = 252084, upload-time = "2026-03-25T20:21:45.62Z" }, + { url = "https://files.pythonhosted.org/packages/00/71/3a69e86f3eafe8c7a59d008d245888051005bd657760e96d5fbfb0b740c2/tomli-2.4.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:7f94b27a62cfad8496c8d2513e1a222dd446f095fca8987fceef261225538a15", size = 247973, upload-time = "2026-03-25T20:21:46.937Z" }, + { url = "https://files.pythonhosted.org/packages/67/50/361e986652847fec4bd5e4a0208752fbe64689c603c7ae5ea7cb16b1c0ca/tomli-2.4.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:ede3e6487c5ef5d28634ba3f31f989030ad6af71edfb0055cbbd14189ff240ba", size = 256223, upload-time = "2026-03-25T20:21:48.467Z" }, + { url = "https://files.pythonhosted.org/packages/8c/9a/b4173689a9203472e5467217e0154b00e260621caa227b6fa01feab16998/tomli-2.4.1-cp314-cp314-win32.whl", hash = "sha256:3d48a93ee1c9b79c04bb38772ee1b64dcf18ff43085896ea460ca8dec96f35f6", size = 98973, upload-time = "2026-03-25T20:21:49.526Z" }, + { url = "https://files.pythonhosted.org/packages/14/58/640ac93bf230cd27d002462c9af0d837779f8773bc03dee06b5835208214/tomli-2.4.1-cp314-cp314-win_amd64.whl", hash = "sha256:88dceee75c2c63af144e456745e10101eb67361050196b0b6af5d717254dddf7", size = 109082, upload-time = "2026-03-25T20:21:50.506Z" }, + { url = "https://files.pythonhosted.org/packages/d5/2f/702d5e05b227401c1068f0d386d79a589bb12bf64c3d2c72ce0631e3bc49/tomli-2.4.1-cp314-cp314-win_arm64.whl", hash = "sha256:b8c198f8c1805dc42708689ed6864951fd2494f924149d3e4bce7710f8eb5232", size = 96490, upload-time = "2026-03-25T20:21:51.474Z" }, + { url = "https://files.pythonhosted.org/packages/45/4b/b877b05c8ba62927d9865dd980e34a755de541eb65fffba52b4cc495d4d2/tomli-2.4.1-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:d4d8fe59808a54658fcc0160ecfb1b30f9089906c50b23bcb4c69eddc19ec2b4", size = 164263, upload-time = "2026-03-25T20:21:52.543Z" }, + { url = "https://files.pythonhosted.org/packages/24/79/6ab420d37a270b89f7195dec5448f79400d9e9c1826df982f3f8e97b24fd/tomli-2.4.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:7008df2e7655c495dd12d2a4ad038ff878d4ca4b81fccaf82b714e07eae4402c", size = 160736, upload-time = "2026-03-25T20:21:53.674Z" }, + { url = "https://files.pythonhosted.org/packages/02/e0/3630057d8eb170310785723ed5adcdfb7d50cb7e6455f85ba8a3deed642b/tomli-2.4.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1d8591993e228b0c930c4bb0db464bdad97b3289fb981255d6c9a41aedc84b2d", size = 270717, upload-time = "2026-03-25T20:21:55.129Z" }, + { url = "https://files.pythonhosted.org/packages/7a/b4/1613716072e544d1a7891f548d8f9ec6ce2faf42ca65acae01d76ea06bb0/tomli-2.4.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:734e20b57ba95624ecf1841e72b53f6e186355e216e5412de414e3c51e5e3c41", size = 278461, upload-time = "2026-03-25T20:21:56.228Z" }, + { url = "https://files.pythonhosted.org/packages/05/38/30f541baf6a3f6df77b3df16b01ba319221389e2da59427e221ef417ac0c/tomli-2.4.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:8a650c2dbafa08d42e51ba0b62740dae4ecb9338eefa093aa5c78ceb546fcd5c", size = 274855, upload-time = "2026-03-25T20:21:57.653Z" }, + { url = "https://files.pythonhosted.org/packages/77/a3/ec9dd4fd2c38e98de34223b995a3b34813e6bdadf86c75314c928350ed14/tomli-2.4.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:504aa796fe0569bb43171066009ead363de03675276d2d121ac1a4572397870f", size = 283144, upload-time = "2026-03-25T20:21:59.089Z" }, + { url = "https://files.pythonhosted.org/packages/ef/be/605a6261cac79fba2ec0c9827e986e00323a1945700969b8ee0b30d85453/tomli-2.4.1-cp314-cp314t-win32.whl", hash = "sha256:b1d22e6e9387bf4739fbe23bfa80e93f6b0373a7f1b96c6227c32bef95a4d7a8", size = 108683, upload-time = "2026-03-25T20:22:00.214Z" }, + { url = "https://files.pythonhosted.org/packages/12/64/da524626d3b9cc40c168a13da8335fe1c51be12c0a63685cc6db7308daae/tomli-2.4.1-cp314-cp314t-win_amd64.whl", hash = "sha256:2c1c351919aca02858f740c6d33adea0c5deea37f9ecca1cc1ef9e884a619d26", size = 121196, upload-time = "2026-03-25T20:22:01.169Z" }, + { url = "https://files.pythonhosted.org/packages/5a/cd/e80b62269fc78fc36c9af5a6b89c835baa8af28ff5ad28c7028d60860320/tomli-2.4.1-cp314-cp314t-win_arm64.whl", hash = "sha256:eab21f45c7f66c13f2a9e0e1535309cee140182a9cdae1e041d02e47291e8396", size = 100393, upload-time = "2026-03-25T20:22:02.137Z" }, + { url = "https://files.pythonhosted.org/packages/7b/61/cceae43728b7de99d9b847560c262873a1f6c98202171fd5ed62640b494b/tomli-2.4.1-py3-none-any.whl", hash = "sha256:0d85819802132122da43cb86656f8d1f8c6587d54ae7dcaf30e90533028b49fe", size = 14583, upload-time = "2026-03-25T20:22:03.012Z" }, +] + [[package]] name = "typing-extensions" version = "4.15.0" @@ -1581,6 +1973,44 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/83/e4/d04a086285c20886c0daad0e026f250869201013d18f81d9ff5eada73a88/uvicorn-0.41.0-py3-none-any.whl", hash = "sha256:29e35b1d2c36a04b9e180d4007ede3bcb32a85fbdfd6c6aeb3f26839de088187", size = 68783, upload-time = "2026-02-16T23:07:22.357Z" }, ] +[[package]] +name = "uvloop" +version = "0.22.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/06/f0/18d39dbd1971d6d62c4629cc7fa67f74821b0dc1f5a77af43719de7936a7/uvloop-0.22.1.tar.gz", hash = "sha256:6c84bae345b9147082b17371e3dd5d42775bddce91f885499017f4607fdaf39f", size = 2443250, upload-time = "2025-10-16T22:17:19.342Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c7/d5/69900f7883235562f1f50d8184bb7dd84a2fb61e9ec63f3782546fdbd057/uvloop-0.22.1-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:c60ebcd36f7b240b30788554b6f0782454826a0ed765d8430652621b5de674b9", size = 1352420, upload-time = "2025-10-16T22:16:21.187Z" }, + { url = "https://files.pythonhosted.org/packages/a8/73/c4e271b3bce59724e291465cc936c37758886a4868787da0278b3b56b905/uvloop-0.22.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:3b7f102bf3cb1995cfeaee9321105e8f5da76fdb104cdad8986f85461a1b7b77", size = 748677, upload-time = "2025-10-16T22:16:22.558Z" }, + { url = "https://files.pythonhosted.org/packages/86/94/9fb7fad2f824d25f8ecac0d70b94d0d48107ad5ece03769a9c543444f78a/uvloop-0.22.1-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:53c85520781d84a4b8b230e24a5af5b0778efdb39142b424990ff1ef7c48ba21", size = 3753819, upload-time = "2025-10-16T22:16:23.903Z" }, + { url = "https://files.pythonhosted.org/packages/74/4f/256aca690709e9b008b7108bc85fba619a2bc37c6d80743d18abad16ee09/uvloop-0.22.1-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:56a2d1fae65fd82197cb8c53c367310b3eabe1bbb9fb5a04d28e3e3520e4f702", size = 3804529, upload-time = "2025-10-16T22:16:25.246Z" }, + { url = "https://files.pythonhosted.org/packages/7f/74/03c05ae4737e871923d21a76fe28b6aad57f5c03b6e6bfcfa5ad616013e4/uvloop-0.22.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:40631b049d5972c6755b06d0bfe8233b1bd9a8a6392d9d1c45c10b6f9e9b2733", size = 3621267, upload-time = "2025-10-16T22:16:26.819Z" }, + { url = "https://files.pythonhosted.org/packages/75/be/f8e590fe61d18b4a92070905497aec4c0e64ae1761498cad09023f3f4b3e/uvloop-0.22.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:535cc37b3a04f6cd2c1ef65fa1d370c9a35b6695df735fcff5427323f2cd5473", size = 3723105, upload-time = "2025-10-16T22:16:28.252Z" }, + { url = "https://files.pythonhosted.org/packages/3d/ff/7f72e8170be527b4977b033239a83a68d5c881cc4775fca255c677f7ac5d/uvloop-0.22.1-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:fe94b4564e865d968414598eea1a6de60adba0c040ba4ed05ac1300de402cd42", size = 1359936, upload-time = "2025-10-16T22:16:29.436Z" }, + { url = "https://files.pythonhosted.org/packages/c3/c6/e5d433f88fd54d81ef4be58b2b7b0cea13c442454a1db703a1eea0db1a59/uvloop-0.22.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:51eb9bd88391483410daad430813d982010f9c9c89512321f5b60e2cddbdddd6", size = 752769, upload-time = "2025-10-16T22:16:30.493Z" }, + { url = "https://files.pythonhosted.org/packages/24/68/a6ac446820273e71aa762fa21cdcc09861edd3536ff47c5cd3b7afb10eeb/uvloop-0.22.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:700e674a166ca5778255e0e1dc4e9d79ab2acc57b9171b79e65feba7184b3370", size = 4317413, upload-time = "2025-10-16T22:16:31.644Z" }, + { url = "https://files.pythonhosted.org/packages/5f/6f/e62b4dfc7ad6518e7eff2516f680d02a0f6eb62c0c212e152ca708a0085e/uvloop-0.22.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7b5b1ac819a3f946d3b2ee07f09149578ae76066d70b44df3fa990add49a82e4", size = 4426307, upload-time = "2025-10-16T22:16:32.917Z" }, + { url = "https://files.pythonhosted.org/packages/90/60/97362554ac21e20e81bcef1150cb2a7e4ffdaf8ea1e5b2e8bf7a053caa18/uvloop-0.22.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:e047cc068570bac9866237739607d1313b9253c3051ad84738cbb095be0537b2", size = 4131970, upload-time = "2025-10-16T22:16:34.015Z" }, + { url = "https://files.pythonhosted.org/packages/99/39/6b3f7d234ba3964c428a6e40006340f53ba37993f46ed6e111c6e9141d18/uvloop-0.22.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:512fec6815e2dd45161054592441ef76c830eddaad55c8aa30952e6fe1ed07c0", size = 4296343, upload-time = "2025-10-16T22:16:35.149Z" }, + { url = "https://files.pythonhosted.org/packages/89/8c/182a2a593195bfd39842ea68ebc084e20c850806117213f5a299dfc513d9/uvloop-0.22.1-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:561577354eb94200d75aca23fbde86ee11be36b00e52a4eaf8f50fb0c86b7705", size = 1358611, upload-time = "2025-10-16T22:16:36.833Z" }, + { url = "https://files.pythonhosted.org/packages/d2/14/e301ee96a6dc95224b6f1162cd3312f6d1217be3907b79173b06785f2fe7/uvloop-0.22.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:1cdf5192ab3e674ca26da2eada35b288d2fa49fdd0f357a19f0e7c4e7d5077c8", size = 751811, upload-time = "2025-10-16T22:16:38.275Z" }, + { url = "https://files.pythonhosted.org/packages/b7/02/654426ce265ac19e2980bfd9ea6590ca96a56f10c76e63801a2df01c0486/uvloop-0.22.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6e2ea3d6190a2968f4a14a23019d3b16870dd2190cd69c8180f7c632d21de68d", size = 4288562, upload-time = "2025-10-16T22:16:39.375Z" }, + { url = "https://files.pythonhosted.org/packages/15/c0/0be24758891ef825f2065cd5db8741aaddabe3e248ee6acc5e8a80f04005/uvloop-0.22.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0530a5fbad9c9e4ee3f2b33b148c6a64d47bbad8000ea63704fa8260f4cf728e", size = 4366890, upload-time = "2025-10-16T22:16:40.547Z" }, + { url = "https://files.pythonhosted.org/packages/d2/53/8369e5219a5855869bcee5f4d317f6da0e2c669aecf0ef7d371e3d084449/uvloop-0.22.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:bc5ef13bbc10b5335792360623cc378d52d7e62c2de64660616478c32cd0598e", size = 4119472, upload-time = "2025-10-16T22:16:41.694Z" }, + { url = "https://files.pythonhosted.org/packages/f8/ba/d69adbe699b768f6b29a5eec7b47dd610bd17a69de51b251126a801369ea/uvloop-0.22.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:1f38ec5e3f18c8a10ded09742f7fb8de0108796eb673f30ce7762ce1b8550cad", size = 4239051, upload-time = "2025-10-16T22:16:43.224Z" }, + { url = "https://files.pythonhosted.org/packages/90/cd/b62bdeaa429758aee8de8b00ac0dd26593a9de93d302bff3d21439e9791d/uvloop-0.22.1-cp314-cp314-macosx_10_13_universal2.whl", hash = "sha256:3879b88423ec7e97cd4eba2a443aa26ed4e59b45e6b76aabf13fe2f27023a142", size = 1362067, upload-time = "2025-10-16T22:16:44.503Z" }, + { url = "https://files.pythonhosted.org/packages/0d/f8/a132124dfda0777e489ca86732e85e69afcd1ff7686647000050ba670689/uvloop-0.22.1-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:4baa86acedf1d62115c1dc6ad1e17134476688f08c6efd8a2ab076e815665c74", size = 752423, upload-time = "2025-10-16T22:16:45.968Z" }, + { url = "https://files.pythonhosted.org/packages/a3/94/94af78c156f88da4b3a733773ad5ba0b164393e357cc4bd0ab2e2677a7d6/uvloop-0.22.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:297c27d8003520596236bdb2335e6b3f649480bd09e00d1e3a99144b691d2a35", size = 4272437, upload-time = "2025-10-16T22:16:47.451Z" }, + { url = "https://files.pythonhosted.org/packages/b5/35/60249e9fd07b32c665192cec7af29e06c7cd96fa1d08b84f012a56a0b38e/uvloop-0.22.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c1955d5a1dd43198244d47664a5858082a3239766a839b2102a269aaff7a4e25", size = 4292101, upload-time = "2025-10-16T22:16:49.318Z" }, + { url = "https://files.pythonhosted.org/packages/02/62/67d382dfcb25d0a98ce73c11ed1a6fba5037a1a1d533dcbb7cab033a2636/uvloop-0.22.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:b31dc2fccbd42adc73bc4e7cdbae4fc5086cf378979e53ca5d0301838c5682c6", size = 4114158, upload-time = "2025-10-16T22:16:50.517Z" }, + { url = "https://files.pythonhosted.org/packages/f0/7a/f1171b4a882a5d13c8b7576f348acfe6074d72eaf52cccef752f748d4a9f/uvloop-0.22.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:93f617675b2d03af4e72a5333ef89450dfaa5321303ede6e67ba9c9d26878079", size = 4177360, upload-time = "2025-10-16T22:16:52.646Z" }, + { url = "https://files.pythonhosted.org/packages/79/7b/b01414f31546caf0919da80ad57cbfe24c56b151d12af68cee1b04922ca8/uvloop-0.22.1-cp314-cp314t-macosx_10_13_universal2.whl", hash = "sha256:37554f70528f60cad66945b885eb01f1bb514f132d92b6eeed1c90fd54ed6289", size = 1454790, upload-time = "2025-10-16T22:16:54.355Z" }, + { url = "https://files.pythonhosted.org/packages/d4/31/0bb232318dd838cad3fa8fb0c68c8b40e1145b32025581975e18b11fab40/uvloop-0.22.1-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:b76324e2dc033a0b2f435f33eb88ff9913c156ef78e153fb210e03c13da746b3", size = 796783, upload-time = "2025-10-16T22:16:55.906Z" }, + { url = "https://files.pythonhosted.org/packages/42/38/c9b09f3271a7a723a5de69f8e237ab8e7803183131bc57c890db0b6bb872/uvloop-0.22.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:badb4d8e58ee08dad957002027830d5c3b06aea446a6a3744483c2b3b745345c", size = 4647548, upload-time = "2025-10-16T22:16:57.008Z" }, + { url = "https://files.pythonhosted.org/packages/c1/37/945b4ca0ac27e3dc4952642d4c900edd030b3da6c9634875af6e13ae80e5/uvloop-0.22.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b91328c72635f6f9e0282e4a57da7470c7350ab1c9f48546c0f2866205349d21", size = 4467065, upload-time = "2025-10-16T22:16:58.206Z" }, + { url = "https://files.pythonhosted.org/packages/97/cc/48d232f33d60e2e2e0b42f4e73455b146b76ebe216487e862700457fbf3c/uvloop-0.22.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:daf620c2995d193449393d6c62131b3fbd40a63bf7b307a1527856ace637fe88", size = 4328384, upload-time = "2025-10-16T22:16:59.36Z" }, + { url = "https://files.pythonhosted.org/packages/e4/16/c1fd27e9549f3c4baf1dc9c20c456cd2f822dbf8de9f463824b0c0357e06/uvloop-0.22.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:6cde23eeda1a25c75b2e07d39970f3374105d5eafbaab2a4482be82f272d5a5e", size = 4296730, upload-time = "2025-10-16T22:17:00.744Z" }, +] + [[package]] name = "watchfiles" version = "1.1.1"